From 53f42acaa6a6d99c6a7dd095f85735ef1c1f5b62 Mon Sep 17 00:00:00 2001 From: Linwei-Ying Date: Mon, 7 Jul 2025 20:02:39 +0800 Subject: [PATCH] compare state detect improve --- .../msprobe/core/common/const.py | 6 +- .../msprobe/core/compare/acc_compare.py | 41 +++-- .../msprobe/core/compare/highlight.py | 6 +- .../compare/layer_mapping/layer_mapping.py | 11 +- .../core/compare/multiprocessing_compute.py | 8 +- .../msprobe/core/compare/utils.py | 113 ++++++------- .../test/core_ut/compare/test_acc_compare.py | 63 ++++---- .../core_ut/compare/test_acc_compare_utils.py | 148 +++++++++--------- .../core_ut/compare/test_cmp_highlight.py | 8 +- .../test_cmp_multiprocessing_compute.py | 2 +- 10 files changed, 199 insertions(+), 207 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/common/const.py b/debug/accuracy_tools/msprobe/core/common/const.py index 7b90110f29..a86b87ce0d 100644 --- a/debug/accuracy_tools/msprobe/core/common/const.py +++ b/debug/accuracy_tools/msprobe/core/common/const.py @@ -237,6 +237,8 @@ class Const: MEAN = 'Mean' NORM = 'Norm' DATA_NAME = 'data_name' + STATE = 'state' + API_ORIGIN_NAME = 'api_origin_name' TENSOR_STAT_INDEX = 'tensor_stat_index' SUMMARY_METRICS_LIST = [MAX, MIN, MEAN, NORM] @@ -642,9 +644,9 @@ class CompareConst: OP_NAME_X = 'op_name_x' MATCH_RESULT_COLUMNS = [ - OP_NAME_X, 'dtype_x', 'shape_x', 'summary_x', 'stack_info_x', 'data_name_x', + OP_NAME_X, 'dtype_x', 'shape_x', 'summary_x', 'stack_info_x', 'state_x', 'api_origin_name_x', 'data_name_x', CMP_KEY, CMP_SHAPE, - 'op_name_y', 'dtype_y', 'shape_y', 'summary_y', 'stack_info_y', 'data_name_y', + 'op_name_y', 'dtype_y', 'shape_y', 'summary_y', 'stack_info_y', 'state_y', 'api_origin_name_y', 'data_name_y' ] INTERNAL_API_MAPPING_FILE = 'ms_to_pt_api.yaml' diff --git a/debug/accuracy_tools/msprobe/core/compare/acc_compare.py b/debug/accuracy_tools/msprobe/core/compare/acc_compare.py index 9fd13743ab..7b34b73d2e 100644 --- a/debug/accuracy_tools/msprobe/core/compare/acc_compare.py +++ b/debug/accuracy_tools/msprobe/core/compare/acc_compare.py @@ -118,6 +118,7 @@ class Comparator: highlight = HighLight(self.mode_config) if self.mode_config.compared_file_type == Const.DUMP_JSON_FILE: highlight.find_compare_result_error_rows(result_df, highlight_dict) + result_df.drop(columns=['state', 'api_origin_name'], inplace=True) # 删除中间数据，两列不落盘 highlight.highlight_rows_xlsx(result_df, highlight_dict, file_path) # output compare analysis suggestions @@ -179,10 +180,12 @@ class ParseData: Const.DTYPE: [], Const.SHAPE: [], Const.SUMMARY: [], - Const.STACK_INFO: [] + Const.STACK_INFO: [], + Const.STATE: [], + Const.API_ORIGIN_NAME: [] } if self.mode_config.dump_mode == Const.ALL: - result['data_name'] = [] + result[Const.DATA_NAME] = [] elif self.mode_config.dump_mode == Const.MD5: result[Const.MD5] = [] @@ -203,20 +206,22 @@ class ParseData: op_name_list = merge_list.get(CompareConst.OP_NAME) summary_list = merge_list.get(Const.SUMMARY) - data_name_list = merge_list.get('data_name') - op_name_reorder, summary_reorder, data_name_reorder = reorder_op_x_list(op_name_list, - summary_list, - data_name_list) + data_name_list = merge_list.get(Const.DATA_NAME) + state_list = merge_list.get(Const.STATE) + op_name_reorder, summary_reorder, data_name_reorder, state_reorder = reorder_op_x_list(op_name_list, + summary_list, + data_name_list, + state_list) # 遍历单个API的所有item - for index, op_name in enumerate(op_name_reorder): + for index, (op_name, state) in enumerate(zip(op_name_reorder, state_reorder)): result[CompareConst.OP_NAME].append(op_name) - if (CompareConst.INPUT_PATTERN in op_name) or (CompareConst.KWARGS_PATTERN in op_name): + if state == Const.INPUT or state == Const.KWARGS: info_list = merge_list[CompareConst.INPUT_STRUCT] - elif CompareConst.OUTPUT_PATTERN in op_name: + elif state == Const.OUTPUT: info_list = merge_list[CompareConst.OUTPUT_STRUCT] - elif CompareConst.PARAMS_PATTERN in op_name: + elif state == Const.PARAMS: info_list = merge_list[CompareConst.PARAMS_STRUCT] - elif CompareConst.PARAMS_GRAD_PATTERN in op_name: + elif state == Const.PARAMS_GRAD: info_list = merge_list[CompareConst.PARAMS_GRAD_STRUCT] else: info_list = merge_list[CompareConst.DEBUG_STRUCT] @@ -241,7 +246,10 @@ class ParseData: if self.mode_config.dump_mode == Const.ALL: check_api_info_len(op_name, data_name_reorder, 1) - result['data_name'].append(data_name_reorder.pop(0)) + result[Const.DATA_NAME].append(data_name_reorder.pop(0)) + + result[Const.STATE].append(state) + result[Const.API_ORIGIN_NAME].append(data_name) progress_bar.update(1) progress_bar.close() return pd.DataFrame(result) @@ -413,8 +421,8 @@ class Match: @staticmethod def put_unmatched_in_table(match_result, npu_op_item): npu_columns = npu_op_item.index.tolist()[:-2] - new_columns = [name[:-1] + 'y' for name in npu_columns] - na_series = pd.Series([CompareConst.N_A] * len(new_columns), index=new_columns) + bench_columns = [name + '_y' for name in npu_columns] + na_series = pd.Series([CompareConst.N_A] * len(bench_columns), index=bench_columns) new_result_item = pd.concat([npu_op_item, na_series]).to_frame().T new_result_item.columns = CompareConst.MATCH_RESULT_COLUMNS match_result = pd.concat([match_result, new_result_item]) @@ -610,7 +618,9 @@ class CreateTable: 'md5_x': CompareConst.NPU_MD5, 'md5_y': CompareConst.BENCH_MD5, 'data_name_x': CompareConst.DATA_NAME, - 'stack_info_x': CompareConst.STACK}, inplace=True) + 'stack_info_x': CompareConst.STACK, + 'state_x': Const.STATE, + 'api_origin_name_x': Const.API_ORIGIN_NAME}, inplace=True) # process summary data npu_summary = [CompareConst.NPU_MAX, CompareConst.NPU_MIN, CompareConst.NPU_MEAN, CompareConst.NPU_NORM] @@ -623,6 +633,7 @@ class CreateTable: result[npu_summary] = result['summary_x'].apply(self.set_summary).tolist() result[bench_summary] = result['summary_y'].apply(self.set_summary).tolist() + header.extend([Const.STATE, Const.API_ORIGIN_NAME]) result_df = pd.DataFrame(columns=header) for h in header: if h in result.columns: diff --git a/debug/accuracy_tools/msprobe/core/compare/highlight.py b/debug/accuracy_tools/msprobe/core/compare/highlight.py index 71959d77d1..37eec42169 100644 --- a/debug/accuracy_tools/msprobe/core/compare/highlight.py +++ b/debug/accuracy_tools/msprobe/core/compare/highlight.py @@ -29,7 +29,7 @@ from msprobe.core.common.const import CompareConst, Const from msprobe.core.common.file_utils import save_workbook from msprobe.core.common.log import logger from msprobe.core.common.utils import get_header_index, safe_get_value -from msprobe.core.compare.utils import table_value_is_valid, get_name_and_state, CompareException +from msprobe.core.compare.utils import table_value_is_valid, CompareException from msprobe.core.compare.config import ModeConfig @@ -275,8 +275,8 @@ class HighLight: result = result_df.values api_batches = [] for i, res_i in enumerate(result): - api_full_name = safe_get_value(res_i, 0, "res_i") - api_name, state = get_name_and_state(api_full_name) + api_name = safe_get_value(res_i, -1, "res_i") # 内部定义倒数第一个元素必是api_origin_name + state = safe_get_value(res_i, -2, "res_i") # 内部定义倒数第二个元素必是state self.api_batches_update(api_batches, api_name, state, i) with tqdm(total=len(api_batches), desc="API/Module Analyse Progress", unit="item", ncols=100) as progress_bar: for api_batch in api_batches: diff --git a/debug/accuracy_tools/msprobe/core/compare/layer_mapping/layer_mapping.py b/debug/accuracy_tools/msprobe/core/compare/layer_mapping/layer_mapping.py index 4845adb048..91927f963a 100644 --- a/debug/accuracy_tools/msprobe/core/compare/layer_mapping/layer_mapping.py +++ b/debug/accuracy_tools/msprobe/core/compare/layer_mapping/layer_mapping.py @@ -208,7 +208,8 @@ def generate_data_mapping(npu_json_path, bench_json_path, api_mapping, output_pa def read_full_op_names(data, op_name): op_parsed_list = read_op(data.get(op_name, {}), op_name) full_op_names = [op_parsed.get('full_op_name') for op_parsed in op_parsed_list] - return full_op_names + states = [op_parsed.get(Const.STATE) for op_parsed in op_parsed_list] + return full_op_names, states def generate_op_data_mapping(npu_op_name, npu_full_op_names, bench_op_name, bench_full_op_names): suffix_to_full_op_name = {} @@ -228,10 +229,10 @@ def generate_data_mapping(npu_json_path, bench_json_path, api_mapping, output_pa for npu_op_name, bench_op_name in api_mapping.items(): if not npu_op_name: continue - npu_full_op_names = read_full_op_names(npu_data, npu_op_name) - bench_full_op_names = read_full_op_names(bench_data, bench_op_name) - npu_full_op_names_reorder = reorder_op_name_list(npu_full_op_names) - bench_full_op_names_reorder = reorder_op_name_list(bench_full_op_names) + npu_full_op_names, npu_states = read_full_op_names(npu_data, npu_op_name) + bench_full_op_names, bench_states = read_full_op_names(bench_data, bench_op_name) + npu_full_op_names_reorder, _ = reorder_op_name_list(npu_full_op_names, npu_states) + bench_full_op_names_reorder, _ = reorder_op_name_list(bench_full_op_names, bench_states) mapping = generate_op_data_mapping(npu_op_name, npu_full_op_names_reorder, bench_op_name, bench_full_op_names_reorder) data_mapping.update(mapping) diff --git a/debug/accuracy_tools/msprobe/core/compare/multiprocessing_compute.py b/debug/accuracy_tools/msprobe/core/compare/multiprocessing_compute.py index 8e23f9f8b9..cb0e13e383 100644 --- a/debug/accuracy_tools/msprobe/core/compare/multiprocessing_compute.py +++ b/debug/accuracy_tools/msprobe/core/compare/multiprocessing_compute.py @@ -94,8 +94,8 @@ class CompareRealData: @staticmethod def read_dump_data(result_df): try: - npu_dump_name_list = result_df.iloc[0:, 0].tolist() - dump_tensor_pair_list = result_df.iloc[0:, -1].tolist() + npu_dump_name_list = result_df.loc[0:, CompareConst.NPU_NAME].tolist() + dump_tensor_pair_list = result_df.loc[0:, CompareConst.DATA_NAME].tolist() op_name_mapping_dict = {} for index, npu_dump_name in enumerate(npu_dump_name_list): dump_tensor_pair = dump_tensor_pair_list[index] @@ -104,9 +104,9 @@ class CompareRealData: except ValueError as e: logger.error('result dataframe is not found.') raise CompareException(CompareException.INVALID_DATA_ERROR) from e - except IndexError as e: + except KeyError as e: logger.error('result dataframe elements can not be access.') - raise CompareException(CompareException.INDEX_OUT_OF_BOUNDS_ERROR) from e + raise CompareException(CompareException.INVALID_KEY_ERROR) from e @staticmethod def _save_cmp_result(offset, result: ComparisonResult, result_df, lock): diff --git a/debug/accuracy_tools/msprobe/core/compare/utils.py b/debug/accuracy_tools/msprobe/core/compare/utils.py index 1e67a8020d..e72d6bef96 100644 --- a/debug/accuracy_tools/msprobe/core/compare/utils.py +++ b/debug/accuracy_tools/msprobe/core/compare/utils.py @@ -95,17 +95,21 @@ def check_and_return_dir_contents(dump_dir, prefix): def read_op(op_data, op_name): split_name = op_name.split(Const.SEP) - if Const.DEBUG in split_name or Const.PARAMS_GRAD in split_name: - op_parsed_list = op_item_parse(op_data, op_name) + if Const.DEBUG in split_name: + op_parsed_list = op_item_parse(op_data, op_name, Const.DEBUG) + elif Const.PARAMS_GRAD in split_name: + op_parsed_list = op_item_parse(op_data, op_name, Const.PARAMS_GRAD) else: op_parsed_list = [] for name in CompareConst.IO_NAME_MAPPING: if name in op_data: - op_parsed_list.extend(op_item_parse(op_data[name], op_name + CompareConst.IO_NAME_MAPPING[name])) + op_parsed_list.extend(op_item_parse(op_data[name], op_name + CompareConst.IO_NAME_MAPPING[name], name)) return op_parsed_list -def op_item_parse(op_data, op_name: str, depth: int = 0) -> list: +def op_item_parse(op_data, op_name: str, state: str, depth: int = 0) -> list: + if state == Const.INPUT_ARGS or state == Const.INPUT_KWARGS: + state = Const.INPUT default_item = { 'full_op_name': op_name, 'type': None, @@ -117,7 +121,8 @@ def op_item_parse(op_data, op_name: str, depth: int = 0) -> list: 'shape': None, 'md5': None, 'value': None, - 'data_name': '-1' + 'data_name': '-1', + 'state': state } if depth > Const.MAX_DEPTH: @@ -133,14 +138,14 @@ def op_item_parse(op_data, op_name: str, depth: int = 0) -> list: if isinstance(op_data, list): for i, data in enumerate(op_data): if Const.PARAMS_GRAD not in op_name.split(Const.SEP): - item_list.extend(op_item_parse(data, op_name + Const.SEP + str(i), depth + 1)) + item_list.extend(op_item_parse(data, op_name + Const.SEP + str(i), state, depth + 1)) else: - item_list.extend(op_item_parse(data, op_name, depth + 1)) + item_list.extend(op_item_parse(data, op_name, state, depth + 1)) elif isinstance(op_data, dict): if is_leaf_data(op_data): - return [gen_op_item(op_data, op_name)] + return [gen_op_item(op_data, op_name, state)] for sub_name, sub_data in op_data.items(): - item_list.extend(op_item_parse(sub_data, op_name + Const.SEP + str(sub_name), depth + 1)) + item_list.extend(op_item_parse(sub_data, op_name + Const.SEP + str(sub_name), state, depth + 1)) return item_list @@ -148,12 +153,13 @@ def is_leaf_data(op_data): return 'type' in op_data and isinstance(op_data['type'], str) -def gen_op_item(op_data, op_name): +def gen_op_item(op_data, op_name, state): op_item = {} op_item.update(op_data) data_name = op_data.get('data_name') if op_data.get('data_name') else '-1' # 如果是""也返回-1 op_item['data_name'] = data_name op_item['full_op_name'] = data_name.rsplit(Const.SEP, 1)[0] if data_name != '-1' else op_name + op_item[Const.STATE] = state params = ['Max', 'Min', 'Mean', 'Norm'] for i in params: @@ -205,7 +211,8 @@ def merge_tensor(tensor_list, dump_mode): CompareConst.PARAMS_GRAD_STRUCT, CompareConst.DEBUG_STRUCT, Const.SUMMARY, - Const.STACK_INFO + Const.STACK_INFO, + Const.STATE ] op_dict = {key: [] for key in keys} @@ -215,12 +222,13 @@ def merge_tensor(tensor_list, dump_mode): for tensor in tensor_list: # A dict(len=2) with 'full_op_name' and 'full_info' is added to the tensor only if self.stack_mode is True if len(tensor) == 2: - op_dict[Const.STACK_INFO].append(tensor['full_info']) + op_dict[Const.STACK_INFO].append(tensor.get('full_info')) break - op_dict[CompareConst.OP_NAME].append(tensor['full_op_name']) + op_dict[CompareConst.OP_NAME].append(tensor.get('full_op_name')) + state = tensor.get(Const.STATE) + op_dict[Const.STATE].append(state) - _, state = get_name_and_state(tensor['full_op_name']) struct_key = CompareConst.STATE_TO_STRUCT_MAPPING.get(state) if not struct_key: continue @@ -266,83 +274,54 @@ def table_value_is_valid(value: str) -> bool: return True -def get_name_and_state(name): - """ - Get api/module name and state - example: - name = 'conv2d.forward.1.input.0' - return: ('conv2d.forward.1.', 'input') - - name = 'Functional.pad.0.backward.output.0' - return: ('Functional.pad.0.backward.', 'output') - - name = 'x_tensor.0.debug.{index}' - return: ('x_tensor.0.', 'debug') - - state type: input, output, kwargs, parameters, parameters_grad, debug - """ - if not isinstance(name, str): - logger.error(f'Invalid name: {name}, type should be string, please check.') - raise CompareException(CompareException.INVALID_API_NAME_ERROR) - - if Const.DEBUG in name.split(Const.SEP): - return name.split(Const.DEBUG)[0], Const.DEBUG - if Const.PARAMS_GRAD in name.split(Const.SEP): - return name.split(Const.PARAMS_GRAD)[0], Const.PARAMS_GRAD - - split = re.split(Const.REGEX_FORWARD_BACKWARD, name) - if len(split) < 3: - logger.error(f'Invalid name string: {name}, can not be split by forward/backward, please check.') - raise CompareException(CompareException.INVALID_API_NAME_ERROR) - api = f'{split[0]}.{split[1]}.' - state_str = split[2] - match = re.match(r'^(\d+\.)?(input|output|kwargs|parameters)\..+$', state_str) - if not match: - raise CompareException(f'Invalid name string: {name}') - if match.group(1): - api = f'{api}{match.group(1)}' - state = match.group(2) - return api, state - - -def reorder_op_name_list(op_name_list): +def reorder_op_name_list(op_name_list, state_list): if not op_name_list: - return op_name_list + return op_name_list, state_list parameters = [] output = [] parameters_grad = [] others = [] - for x in op_name_list: - state = get_name_and_state(x)[1] + parameters_s = [] + output_s = [] + parameters_grad_s = [] + others_s = [] + for op_name, state in zip(op_name_list, state_list): if state == Const.PARAMS: - parameters.append(x) + parameters.append(op_name) + parameters_s.append(state) elif state == Const.OUTPUT: - output.append(x) + output.append(op_name) + output_s.append(state) elif state == Const.PARAMS_GRAD: - parameters_grad.append(x) + parameters_grad.append(op_name) + parameters_grad_s.append(state) else: - others.append(x) + others.append(op_name) + others_s.append(state) # 合并others, parameters, 和output，确保parameters排在output前面 op_name_reorder = others + parameters + output + parameters_grad - return op_name_reorder + state_reorder = others_s + parameters_s + output_s + parameters_grad_s + return op_name_reorder, state_reorder -def reorder_op_x_list(op_name_list, summary_list, data_name_list): - """对op_name, summary, data_name重新排序，把parameters放到input后output前，data_name由于统计量比对时，为None，单独处理""" +def reorder_op_x_list(op_name_list, summary_list, data_name_list, state_list): + """ + 对op_name, summary, data_name, state重新排序，把parameters放到input后output前，data_name由于统计量比对时，为None，单独处理 + """ if not op_name_list or not summary_list: - return op_name_list, summary_list, data_name_list + return op_name_list, summary_list, data_name_list, state_list index_map = {name: index for index, name in enumerate(op_name_list)} - op_name_reorder = reorder_op_name_list(op_name_list) + op_name_reorder, state_order = reorder_op_name_list(op_name_list, state_list) summary_reorder = [summary_list[index_map.get(name)] for name in op_name_reorder] if data_name_list: data_name_reorder = [data_name_list[index_map.get(name)] for name in op_name_reorder] else: data_name_reorder = data_name_list - return op_name_reorder, summary_reorder, data_name_reorder + return op_name_reorder, summary_reorder, data_name_reorder, state_order def process_summary_data(summary_data): diff --git a/debug/accuracy_tools/msprobe/test/core_ut/compare/test_acc_compare.py b/debug/accuracy_tools/msprobe/test/core_ut/compare/test_acc_compare.py index 107581b5fe..639be03606 100644 --- a/debug/accuracy_tools/msprobe/test/core_ut/compare/test_acc_compare.py +++ b/debug/accuracy_tools/msprobe/test/core_ut/compare/test_acc_compare.py @@ -350,7 +350,8 @@ class TestUtilsMethods(unittest.TestCase): 'params_struct': [], 'params_grad_struct': [], 'stack_info': [['File']], - 'summary': [[1, 1, 1, 1]] + 'summary': [[1, 1, 1, 1]], + 'state': ['input'] } stack_mode = True @@ -394,10 +395,10 @@ class TestUtilsMethods(unittest.TestCase): o_data = [ ['Functional.linear.0.forward.input.0', 'Functional.linear.0.forward.input.0', 'torch.float32', 'torch.float32', '[2, 2]', '[2, 2]', 0, 0, 0, 0, '0.0%', 'N/A', '0.0%', '0.0%', - 2, 0, 1, 1, 2, 0, 1, 1, '', '', ['File'] + 2, 0, 1, 1, 2, 0, 1, 1, '', '', ['File'], 'input', 'Functional.linear.0.forward' ] ] - columns = CompareConst.SUMMARY_COMPARE_RESULT_HEADER + ['NPU_Stack_Info'] + columns = CompareConst.SUMMARY_COMPARE_RESULT_HEADER + ['NPU_Stack_Info'] + ['state', 'api_origin_name'] o_result = pd.DataFrame(o_data, columns=columns, dtype=object) self.assertTrue(np.array_equal(result.to_numpy(), o_result.to_numpy())) @@ -426,8 +427,8 @@ class TestParseData(unittest.TestCase): npu_df, bench_df = parse_data.parse(file_list) target_df = pd.DataFrame( - [['Functional.linear.0.forward.input.0', 'torch.float32', [2, 2], [2, 0, 1, 1], ['File']]], - columns=['op_name', 'dtype', 'shape', 'summary', 'stack_info'] + [['Functional.linear.0.forward.input.0', 'torch.float32', [2, 2], [2, 0, 1, 1], ['File'], 'input', 'Functional.linear.0.forward']], + columns=['op_name', 'dtype', 'shape', 'summary', 'stack_info', 'state', 'api_origin_name'] ) self.assertTrue(npu_df.equals(target_df)) self.assertTrue(bench_df.equals(target_df)) @@ -444,8 +445,8 @@ class TestParseData(unittest.TestCase): npu_df = parse_data.gen_data_df(npu_json_data, stack_json_data) target_df = pd.DataFrame( - [['Functional.linear.0.forward.input.0', 'torch.float32', [2, 2], [2, 0, 1, 1], ['File']]], - columns=['op_name', 'dtype', 'shape', 'summary', 'stack_info'] + [['Functional.linear.0.forward.input.0', 'torch.float32', [2, 2], [2, 0, 1, 1], ['File'], 'input', 'Functional.linear.0.forward']], + columns=['op_name', 'dtype', 'shape', 'summary', 'stack_info', 'state', 'api_origin_name'] ) self.assertTrue(npu_df.equals(target_df)) @@ -461,8 +462,8 @@ class TestParseData(unittest.TestCase): npu_df = parse_data.gen_data_df(npu_json_data, stack_json_data) target_df = pd.DataFrame( - [['Functional.linear.0.forward.input.0', 'torch.float32', [2, 2], [2, 0, 1, 1], ['File'], 'Functional.linear.0.forward.input.0.pt']], - columns=['op_name', 'dtype', 'shape', 'summary', 'stack_info', 'data_name'] + [['Functional.linear.0.forward.input.0', 'torch.float32', [2, 2], [2, 0, 1, 1], ['File'], 'input', 'Functional.linear.0.forward', 'Functional.linear.0.forward.input.0.pt']], + columns=['op_name', 'dtype', 'shape', 'summary', 'stack_info', 'state', 'api_origin_name', 'data_name'] ) self.assertTrue(npu_df.equals(target_df)) @@ -478,8 +479,8 @@ class TestParseData(unittest.TestCase): npu_df = parse_data.gen_data_df(npu_json_data, stack_json_data) target_df = pd.DataFrame( - [['Functional.linear.0.forward.input.0', 'torch.float32', [2, 2], [2, 0, 1, 1], ['File'], 123456]], - columns=['op_name', 'dtype', 'shape', 'summary', 'stack_info', 'md5'] + [['Functional.linear.0.forward.input.0', 'torch.float32', [2, 2], [2, 0, 1, 1], ['File'], 'input', 'Functional.linear.0.forward', 123456]], + columns=['op_name', 'dtype', 'shape', 'summary', 'stack_info', 'state', 'api_origin_name', 'md5'] ) self.assertTrue(npu_df.equals(target_df)) @@ -502,7 +503,8 @@ class TestParseData(unittest.TestCase): 'params_grad_struct': [], 'params_struct': [], 'stack_info': [['File']], - 'summary': [[2, 0, 1, 1]] + 'summary': [[2, 0, 1, 1]], + 'state': ['input'] } self.assertEqual(merge_list, target_dict) @@ -672,13 +674,14 @@ class TestMatch(unittest.TestCase): match = Match(mode_config, mapping_config, cross_frame=False) match_result = pd.DataFrame(columns=CompareConst.MATCH_RESULT_COLUMNS) - npu_op_item = pd.Series(['op', 'float32', [1, 2], 'summary', 'stack_info', 'data_name', 'op', [1, 2]], - index=['op_name_x', 'dtype_x', 'shape_x', 'summary_x', 'stack_info_x', 'data_name_x', + npu_op_item = pd.Series(['op', 'float32', [1, 2], 'summary', 'stack_info', 'input', 'op_origin', 'data_name', 'op', [1, 2]], + index=['op_name_x', 'dtype_x', 'shape_x', 'summary_x', 'stack_info_x', + 'state_x', 'api_origin_name_x', 'data_name_x', 'compare_key', 'compare_shape'] ) match_result = match.put_unmatched_in_table(match_result, npu_op_item) - target_match_result = pd.DataFrame([['op', 'float32', [1, 2], 'summary', 'stack_info', 'data_name', 'op', [1, 2], - 'N/A', 'N/A', 'N/A', 'N/A', 'N/A', 'N/A']], + target_match_result = pd.DataFrame([['op', 'float32', [1, 2], 'summary', 'stack_info', 'input', 'op_origin', 'data_name', 'op', [1, 2], + 'N/A', 'N/A', 'N/A', 'N/A', 'N/A', 'N/A', 'N/A', 'N/A']], columns=CompareConst.MATCH_RESULT_COLUMNS) self.assertTrue(match_result.equals(target_match_result)) @@ -688,17 +691,17 @@ class TestMatch(unittest.TestCase): match = Match(mode_config, mapping_config, cross_frame=False) match_result = pd.DataFrame(columns=CompareConst.MATCH_RESULT_COLUMNS) - npu_op_item = pd.Series(['op', 'float32', [1, 2], 'summary', 'stack_info', 'data_name', 'op', [1, 2]], - index=['op_name_x', 'dtype_x', 'shape_x', 'summary_x', 'stack_info_x', 'data_name_x', + npu_op_item = pd.Series(['op', 'float32', [1, 2], 'summary', 'stack_info', 'input', 'op_origin', 'data_name', 'op', [1, 2]], + index=['op_name_x', 'dtype_x', 'shape_x', 'summary_x', 'stack_info_x', 'state_x', 'api_origin_name_x', 'data_name_x', 'compare_key', 'compare_shape'] ) - bench_op_item = pd.Series(['op', 'float32', [1, 2], 'summary', 'stack_info', 'data_name', 'op', [1, 2]], - index=['op_name_y', 'dtype_y', 'shape_y', 'summary_y', 'stack_info_y', 'data_name_y', + bench_op_item = pd.Series(['op', 'float32', [1, 2], 'summary', 'stack_info', 'input', 'op_origin', 'data_name', 'op', [1, 2]], + index=['op_name_y', 'dtype_y', 'shape_y', 'summary_y', 'stack_info_y', 'state_y', 'api_origin_name_y', 'data_name_y', 'compare_key', 'compare_shape'] ) match_result = match.put_matched_in_table(match_result, npu_op_item, bench_op_item) - target_match_result = pd.DataFrame([['op', 'float32', [1, 2], 'summary', 'stack_info', 'data_name', 'op', [1, 2], - 'op', 'float32', [1, 2], 'summary', 'stack_info', 'data_name']], + target_match_result = pd.DataFrame([['op', 'float32', [1, 2], 'summary', 'stack_info', 'input', 'op_origin', 'data_name', 'op', [1, 2], + 'op', 'float32', [1, 2], 'summary', 'stack_info', 'input', 'op_origin', 'data_name']], columns=CompareConst.MATCH_RESULT_COLUMNS) self.assertTrue(match_result.equals(target_match_result)) @@ -741,19 +744,19 @@ class TestMatch(unittest.TestCase): match = Match(mode_config, mapping_config, cross_frame=False) npu_df = pd.DataFrame([ - ['Functional.conv2d.3.forward.input.0', 'float32', [1, 2], 'summary', 'stack_info', 'Functional.conv2d.3.forward.input.0.pt', 'Functional.conv2d.3.forward.input.0', [1, 2]], - ['Functional.amax.1.forward.input.0', 'float32', [1, 2], 'summary', 'stack_info', 'Functional.amax.0.forward.input.0.pt', 'Functional.amax.1.forward.input.0', [1, 2]] - ], columns=['op_name', 'dtype', 'shape', 'summary', 'stack_info', 'data_name', 'compare_key', 'compare_shape']) + ['Functional.conv2d.3.forward.input.0', 'float32', [1, 2], 'summary', 'stack_info', 'input', 'Functional.conv2d.3.forward','Functional.conv2d.3.forward.input.0.pt', 'Functional.conv2d.3.forward.input.0', [1, 2]], + ['Functional.amax.1.forward.input.0', 'float32', [1, 2], 'summary', 'stack_info', 'input', 'Functional.amax.1.forward', 'Functional.amax.0.forward.input.0.pt', 'Functional.amax.1.forward.input.0', [1, 2]] + ], columns=['op_name', 'dtype', 'shape', 'summary', 'stack_info', 'state', 'api_origin_name', 'data_name', 'compare_key', 'compare_shape']) bench_df = pd.DataFrame([ - ['Functional.conv2d.0.forward.input.0', 'float32', [1, 2], 'summary', 'stack_info', 'Functional.conv2d.0.forward.input.0.pt', 'Functional.conv2d.0.forward.input.0', [1, 2]], - ['Functional.amax.0.forward.input.0', 'float32', [1, 2], 'summary', 'stack_info', 'Functional.amax.0.forward.input.0.pt', 'Functional.amax.0.forward.input.0', [1, 2]] - ], columns=['op_name', 'dtype', 'shape', 'summary', 'stack_info', 'data_name', 'compare_key', 'compare_shape']) + ['Functional.conv2d.0.forward.input.0', 'float32', [1, 2], 'summary', 'stack_info', 'input', 'Functional.conv2d.0.forward', 'Functional.conv2d.0.forward.input.0.pt', 'Functional.conv2d.0.forward.input.0', [1, 2]], + ['Functional.amax.0.forward.input.0', 'float32', [1, 2], 'summary', 'stack_info', 'input', 'Functional.amax.0.forward', 'Functional.amax.0.forward.input.0.pt', 'Functional.amax.0.forward.input.0', [1, 2]] + ], columns=['op_name', 'dtype', 'shape', 'summary', 'stack_info', 'state', 'api_origin_name', 'data_name', 'compare_key', 'compare_shape']) match_result = match.process_fuzzy_match(npu_df, bench_df) expected = pd.DataFrame( [ - ['Functional.conv2d.3.forward.input.0', 'float32', [1, 2], 'summary', 'stack_info', 'Functional.conv2d.3.forward.input.0.pt', 'Functional.conv2d.3.forward.input.0', [1, 2], 'Functional.conv2d.0.forward.input.0', 'float32', [1, 2], 'summary', 'stack_info', 'Functional.conv2d.0.forward.input.0.pt'], - ['Functional.amax.1.forward.input.0', 'float32', [1, 2], 'summary', 'stack_info', 'Functional.amax.0.forward.input.0.pt', 'Functional.amax.1.forward.input.0', [1, 2], 'Functional.amax.0.forward.input.0', 'float32', [1, 2], 'summary', 'stack_info', 'Functional.amax.0.forward.input.0.pt'] + ['Functional.conv2d.3.forward.input.0', 'float32', [1, 2], 'summary', 'stack_info', 'input', 'Functional.conv2d.3.forward', 'Functional.conv2d.3.forward.input.0.pt', 'Functional.conv2d.3.forward.input.0', [1, 2], 'Functional.conv2d.0.forward.input.0', 'float32', [1, 2], 'summary', 'stack_info', 'input', 'Functional.conv2d.0.forward', 'Functional.conv2d.0.forward.input.0.pt'], + ['Functional.amax.1.forward.input.0', 'float32', [1, 2], 'summary', 'stack_info', 'input', 'Functional.amax.1.forward', 'Functional.amax.0.forward.input.0.pt', 'Functional.amax.1.forward.input.0', [1, 2], 'Functional.amax.0.forward.input.0', 'float32', [1, 2], 'summary', 'stack_info', 'input', 'Functional.amax.0.forward', 'Functional.amax.0.forward.input.0.pt'] ] , columns=CompareConst.MATCH_RESULT_COLUMNS) diff --git a/debug/accuracy_tools/msprobe/test/core_ut/compare/test_acc_compare_utils.py b/debug/accuracy_tools/msprobe/test/core_ut/compare/test_acc_compare_utils.py index 4a7c421a1b..4d264dd74c 100644 --- a/debug/accuracy_tools/msprobe/test/core_ut/compare/test_acc_compare_utils.py +++ b/debug/accuracy_tools/msprobe/test/core_ut/compare/test_acc_compare_utils.py @@ -13,7 +13,7 @@ from msprobe.core.common.const import CompareConst, Const from msprobe.core.common.utils import CompareException from msprobe.core.compare.utils import ApiItemInfo, _compare_parser, check_and_return_dir_contents, extract_json, \ count_struct, get_accuracy, get_rela_diff_summary_mode, merge_tensor, op_item_parse, read_op, result_item_init, \ - stack_column_process, table_value_is_valid, get_name_and_state, reorder_op_name_list, reorder_op_x_list, gen_op_item + stack_column_process, table_value_is_valid, reorder_op_name_list, reorder_op_x_list, gen_op_item # test_read_op_1 op_data = { @@ -31,15 +31,15 @@ op_name = "Tensor.add_0.0.forward" op_result = [ {'type': 'torch.Tensor', 'dtype': 'torch.float32', 'shape': [16, 1, 3, 3], 'md5': '00000000', 'Max': 0.33033010363578796, 'Min': -0.331031858921051, 'Mean': -0.030964046716690063, 'data_name': '-1', - 'Norm': 2.2533628940582275, 'requires_grad': True, 'full_op_name': 'Tensor.add_0.0.forward.input.0'}, + 'Norm': 2.2533628940582275, 'requires_grad': True, 'full_op_name': 'Tensor.add_0.0.forward.input.0', 'state': 'input'}, {'type': 'torch.Tensor', 'dtype': 'torch.float32', 'shape': [16, 1, 3, 3], 'md5': '00000000', 'Max': 0.003992878366261721, 'Min': -0.008102823048830032, 'Mean': -0.0002002553956117481, 'data_name': '-1', - 'Norm': 0.02844562754034996, 'requires_grad': False, 'full_op_name': 'Tensor.add_0.0.forward.input.1'}, + 'Norm': 0.02844562754034996, 'requires_grad': False, 'full_op_name': 'Tensor.add_0.0.forward.input.1', 'state': 'input'}, {'full_op_name': 'Tensor.add_0.0.forward.input.alpha', 'dtype': "", 'shape': '[]', 'md5': '0dae4479', - 'Max': -0.1, 'Min': -0.1, 'Mean': -0.1, 'Norm': -0.1, 'data_name': '-1', 'type': 'float', 'value': -0.1}, + 'Max': -0.1, 'Min': -0.1, 'Mean': -0.1, 'Norm': -0.1, 'data_name': '-1', 'type': 'float', 'value': -0.1, 'state': 'input'}, {'type': 'torch.Tensor', 'dtype': 'torch.float32', 'shape': [16, 1, 3, 3], 'md5': '00000000', 'Max': 0.33033010363578796, 'Min': -0.331031858921051, 'Mean': -0.030964046716690063, 'data_name': '-1', - 'Norm': 2.2533628940582275, 'requires_grad': True, 'full_op_name': 'Tensor.add_0.0.forward.output.0'}] + 'Norm': 2.2533628940582275, 'requires_grad': True, 'full_op_name': 'Tensor.add_0.0.forward.output.0', 'state': 'output'}] # test_read_op_1 op_data_b = { @@ -56,13 +56,13 @@ op_name_b = "Tensor.add_0.0.backward" op_result_b = [ {'type': 'torch.Tensor', 'dtype': 'torch.float32', 'shape': [16, 1, 3, 3], 'data_name': '-1', 'md5': '00000000', 'Max': 0.33033010363578796, 'Min': -0.331031858921051, 'Mean': -0.030964046716690063, - 'Norm': 2.2533628940582275, 'requires_grad': True, 'full_op_name': 'Tensor.add_0.0.backward.input.0'}, + 'Norm': 2.2533628940582275, 'requires_grad': True, 'full_op_name': 'Tensor.add_0.0.backward.input.0', 'state': 'input'}, {'type': 'torch.Tensor', 'dtype': 'torch.float32', 'shape': [16, 1, 3, 3], 'data_name': '-1', 'md5': '00000000', 'Max': 0.003992878366261721, 'Min': -0.008102823048830032, 'Mean': -0.0002002553956117481, - 'Norm': 0.02844562754034996, 'requires_grad': False, 'full_op_name': 'Tensor.add_0.0.backward.input.1'}, + 'Norm': 0.02844562754034996, 'requires_grad': False, 'full_op_name': 'Tensor.add_0.0.backward.input.1', 'state': 'input'}, {'type': 'torch.Tensor', 'dtype': 'torch.float32', 'shape': [16, 1, 3, 3], 'data_name': '-1', 'md5': '00000000', 'Max': 0.33033010363578796, 'Min': -0.331031858921051, 'Mean': -0.030964046716690063, - 'Norm': 2.2533628940582275, 'requires_grad': True, 'full_op_name': 'Tensor.add_0.0.backward.output.0'}] + 'Norm': 2.2533628940582275, 'requires_grad': True, 'full_op_name': 'Tensor.add_0.0.backward.output.0', 'state': 'output'}] # test_op_item_parse parse_item = [ @@ -78,12 +78,12 @@ parse_top_bool = True o_result_parse = [ {'Max': 4097.0, 'Mean': 820.2, 'Min': 0.0, 'Norm': 4097.0, 'dtype': 'torch.int64', 'requires_grad': False, 'shape': [5], 'type': 'torch.Tensor', 'full_op_name': 'Distributed.broadcast.0.forward.input.0', - 'data_name': '-1', 'md5': '00000000'}, + 'data_name': '-1', 'md5': '00000000', 'state': 'input'}, {'full_op_name': 'Distributed.broadcast.0.forward.input.1', 'dtype': "", 'shape': '[]', - 'md5': 'f4dbdf21', 'Max': 0, 'Min': 0, 'Mean': 0, 'Norm': 0, 'data_name': '-1', 'type': 'int', 'value': 0}, + 'md5': 'f4dbdf21', 'Max': 0, 'Min': 0, 'Mean': 0, 'Norm': 0, 'data_name': '-1', 'type': 'int', 'value': 0, 'state': 'input'}, {'Max': None, 'Mean': None, 'Min': None, 'Norm': None, 'data_name': '-1', 'dtype': 'slice', 'type': 'slice', 'full_op_name': 'Distributed.broadcast.0.forward.input.2', 'md5': '5fbbe87f', 'shape': '(3,)', - 'value': [None, None, None]} + 'value': [None, None, None], 'state': 'input'} ] # test_resolve_api_special_parameters @@ -254,15 +254,15 @@ o_result_unmatch_3 = [ tensor_list = [ {'type': 'torch.Tensor', 'dtype': 'torch.float32', 'shape': [16, 1, 3, 3], 'Max': 0.33033010363578796, 'Min': -0.331031858921051, 'Mean': -0.030964046716690063, 'Norm': 2.2533628940582275, 'requires_grad': True, - 'full_op_name': 'Tensor.add_.0.forward.input.0'}, + 'full_op_name': 'Tensor.add_.0.forward.input.0', 'state': 'input'}, {'type': 'torch.Tensor', 'dtype': 'torch.float32', 'shape': [16, 1, 3, 3], 'Max': 0.003992878366261721, 'Min': -0.008102823048830032, 'Mean': -0.0002002553956117481, - 'Norm': 0.02844562754034996, 'requires_grad': False, 'full_op_name': 'Tensor.add_.0.forward.input.1'}, + 'Norm': 0.02844562754034996, 'requires_grad': False, 'full_op_name': 'Tensor.add_.0.forward.input.1', 'state': 'input'}, {'full_op_name': 'Tensor.add_.0.forward.input.alpha.0', 'dtype': "", "shape": '[]', 'md5': None, - 'Max': -0.1, 'Min': -0.1, 'Mean': -0.1, 'Norm': -0.1, 'data_name': '-1'}, + 'Max': -0.1, 'Min': -0.1, 'Mean': -0.1, 'Norm': -0.1, 'data_name': '-1', 'state': 'input'}, {'type': 'torch.Tensor', 'dtype': 'torch.float32', 'shape': [16, 1, 3, 3], 'Max': 0.33033010363578796, 'Min': -0.331031858921051, 'Mean': -0.030964046716690063, - 'Norm': 2.2533628940582275, 'requires_grad': True, 'full_op_name': 'Tensor.add_.0.forward.output.0'} + 'Norm': 2.2533628940582275, 'requires_grad': True, 'full_op_name': 'Tensor.add_.0.forward.output.0', 'state': 'output'} ] result_op_dict = {'op_name': ['Tensor.add_.0.forward.input.0', 'Tensor.add_.0.forward.input.1', 'Tensor.add_.0.forward.input.alpha.0', 'Tensor.add_.0.forward.output.0'], @@ -277,17 +277,18 @@ result_op_dict = {'op_name': ['Tensor.add_.0.forward.input.0', 'Tensor.add_.0.fo 0.02844562754034996], [-0.1, -0.1, -0.1, -0.1], [0.33033010363578796, -0.331031858921051, -0.030964046716690063, 2.2533628940582275]], - 'stack_info': []} + 'stack_info': [], + 'state': ['input', 'input', 'input', 'output']} tensor_list_md5 = [ {'type': 'torch.Tensor', 'dtype': 'torch.float32', 'shape': [16, 1, 3, 3], 'Max': 0.003992878366261721, 'Min': -0.008102823048830032, 'Mean': -0.0002002553956117481, - 'Norm': 0.02844562754034996, 'requires_grad': False, 'full_op_name': 'Tensor.add_.0.forward.input.0', 'md5': 1}, + 'Norm': 0.02844562754034996, 'requires_grad': False, 'full_op_name': 'Tensor.add_.0.forward.input.0', 'md5': 1, 'state': 'input'}, {'full_op_name': 'Tensor.add_.0.forward.kwargs.alpha.0', 'dtype': "", "shape": '[]', 'md5': None, - 'Max': -0.1, 'Min': -0.1, 'Mean': -0.1, 'Norm': -0.1, 'data_name': '-1'}, + 'Max': -0.1, 'Min': -0.1, 'Mean': -0.1, 'Norm': -0.1, 'data_name': '-1', 'state': 'input'}, {'type': 'torch.Tensor', 'dtype': 'torch.float32', 'shape': [16, 1, 3, 3], 'Max': 0.33033010363578796, 'Min': -0.331031858921051, 'Mean': -0.030964046716690063, - 'Norm': 2.2533628940582275, 'requires_grad': True, 'full_op_name': 'Tensor.add_.0.forward.output.0', 'md5': 2} + 'Norm': 2.2533628940582275, 'requires_grad': True, 'full_op_name': 'Tensor.add_.0.forward.output.0', 'md5': 2, 'state': 'output'} ] result_op_dict_md5 = {'op_name': ['Tensor.add_.0.forward.input.0', 'Tensor.add_.0.forward.kwargs.alpha.0', 'Tensor.add_.0.forward.output.0'], @@ -300,7 +301,8 @@ result_op_dict_md5 = {'op_name': ['Tensor.add_.0.forward.input.0', 'Tensor.add_. [0.003992878366261721, -0.008102823048830032, -0.0002002553956117481, 0.02844562754034996], [-0.1, -0.1, -0.1, -0.1], [0.33033010363578796, -0.331031858921051, -0.030964046716690063, 2.2533628940582275]], - 'stack_info': []} + 'stack_info': [], + 'state': ['input', 'input', 'output']} base_dir1 = os.path.join(os.path.dirname(os.path.abspath(__file__)), f'test_acc_compare_utils1') base_dir2 = os.path.join(os.path.dirname(os.path.abspath(__file__)), f'test_acc_compare_utils2') @@ -363,12 +365,12 @@ class TestUtilsMethods(unittest.TestCase): self.assertEqual(result, op_result_b) def test_op_item_parse(self): - result = op_item_parse(parse_item, parse_op_name) + result = op_item_parse(parse_item, parse_op_name, 'input') self.assertEqual(result, o_result_parse) def test_op_item_parse_max_depth(self): with self.assertRaises(CompareException) as context: - op_item_parse(parse_item, parse_op_name, depth=11) + op_item_parse(parse_item, parse_op_name, 'input', depth=11) self.assertEqual(context.exception.code, CompareException.RECURSION_LIMIT_ERROR) def test_get_rela_diff_summary_mode_float_or_int(self): @@ -554,57 +556,34 @@ class TestUtilsMethods(unittest.TestCase): self.assertFalse(result) -class TestGetNameAndState(unittest.TestCase): - def test_valid_forward_input(self): - name = 'conv2d.forward.1.input.0' - expected_api = 'conv2d.forward.1.' - expected_state = 'input' - self.assertEqual(get_name_and_state(name), (expected_api, expected_state)) - - def test_valid_backward_output(self): - name = 'Functional.pad.0.backward.output.0' - expected_api = 'Functional.pad.0.backward.' - expected_state = 'output' - self.assertEqual(get_name_and_state(name), (expected_api, expected_state)) - - def test_valid_with_kwargs(self): - name = 'layer.norm.2.forward.kwargs.attr' - expected_api = 'layer.norm.2.forward.' - expected_state = 'kwargs' - self.assertEqual(get_name_and_state(name), (expected_api, expected_state)) - - def test_no_numeric_index(self): - name = 'conv2d.forward.input.0' - expected_api = 'conv2d.forward.' - expected_state = 'input' - self.assertEqual(get_name_and_state(name), (expected_api, expected_state)) - - def test_invalid__state(self): - name = 'conv2d.forward.1.invalidstate.0' - with self.assertRaises(CompareException) as context: - get_name_and_state(name) - self.assertIn('Invalid name string', str(context.exception.code)) - - class TestReorderOpNameList(unittest.TestCase): def test_reorder_op_name_list(self): # 标准顺序 op_name_list = ["op.forward.input.0.0", "op.forward.output.0", "op.forward.output.1", "op.forward.parameters.1", "op.forward.parameters.2", "op.parameters_grad.0"] - result = reorder_op_name_list(op_name_list) - expected = ["op.forward.input.0.0", "op.forward.parameters.1", "op.forward.parameters.2", "op.forward.output.0", "op.forward.output.1", "op.parameters_grad.0"] - self.assertEqual(result, expected) + state_list = ["input", "output", "output", "parameters", "parameters", "parameters_grad"] + op_name_reorder, state_reorder = reorder_op_name_list(op_name_list, state_list) + expected_result = ["op.forward.input.0.0", "op.forward.parameters.1", "op.forward.parameters.2", "op.forward.output.0", "op.forward.output.1", "op.parameters_grad.0"] + expected_state = ["input", "parameters", "parameters", "output", "output", "parameters_grad"] + self.assertEqual(op_name_reorder, expected_result) + self.assertEqual(state_reorder, expected_state) # 只有输入元素 op_name_list = ["op.forward.input.0", "op.forward.input.1"] - result = reorder_op_name_list(op_name_list) - expected = ["op.forward.input.0", "op.forward.input.1"] - self.assertEqual(result, expected) + state_list = ["input", "input"] + op_name_reorder, state_reorder = reorder_op_name_list(op_name_list, state_list) + expected_result = ["op.forward.input.0", "op.forward.input.1"] + expected_state = ["input", "input"] + self.assertEqual(op_name_reorder, expected_result) + self.assertEqual(state_reorder, expected_state) # 输入为空 op_name_list = [] - result = reorder_op_name_list(op_name_list) - expected = [] - self.assertEqual(result, expected) + state_list = [] + op_name_reorder, state_reorder = reorder_op_name_list(op_name_list, state_list) + expected_result = [] + expected_state = [] + self.assertEqual(op_name_reorder, expected_result) + self.assertEqual(state_reorder, expected_state) class TestReorderOpXList(unittest.TestCase): @@ -613,37 +592,45 @@ class TestReorderOpXList(unittest.TestCase): op_name_list = ["op.forward.input.0", "op.forward.output.0", "op.forward.parameters.weight"] summary_list = ["summary1", "summary2", "summary3"] data_name_list = ["data1", "data2", "data3"] - result_op_name, result_summary, result_data_name = reorder_op_x_list(op_name_list, summary_list, data_name_list) + state_list = ["input", "output", "parameters"] + result_op_name, result_summary, result_data_name, result_state = reorder_op_x_list(op_name_list, summary_list, data_name_list, state_list) self.assertEqual(result_op_name, ["op.forward.input.0", "op.forward.parameters.weight", "op.forward.output.0"]) self.assertEqual(result_summary, ["summary1", "summary3", "summary2"]) self.assertEqual(result_data_name, ["data1", "data3", "data2"]) + self.assertEqual(result_state, ["input", "parameters", "output"]) # 空 op_name_list 或 summary_list op_name_list = [] summary_list = [] data_name_list = ["data1", "data2", "data3"] - result_op_name, result_summary, result_data_name = reorder_op_x_list(op_name_list, summary_list, data_name_list) + state_list = [] + result_op_name, result_summary, result_data_name, result_state = reorder_op_x_list(op_name_list, summary_list, data_name_list, state_list) self.assertEqual(result_op_name, []) self.assertEqual(result_summary, []) self.assertEqual(result_data_name, ["data1", "data2", "data3"]) + self.assertEqual(result_state, []) # 空 data_name_list op_name_list = ["op.forward.input.0", "op.forward.output.0", "op.forward.parameters.weight"] summary_list = ["summary1", "summary2", "summary3"] data_name_list = [] - result_op_name, result_summary, result_data_name = reorder_op_x_list(op_name_list, summary_list, data_name_list) + state_list = ["input", "output", "parameters"] + result_op_name, result_summary, result_data_name, result_state = reorder_op_x_list(op_name_list, summary_list, data_name_list, state_list) self.assertEqual(result_op_name, ["op.forward.input.0", "op.forward.parameters.weight", "op.forward.output.0"]) self.assertEqual(result_summary, ["summary1", "summary3", "summary2"]) self.assertEqual(result_data_name, []) + self.assertEqual(result_state, ["input", "parameters", "output"]) # data_name_list 为 None op_name_list = ["op.forward.input.0", "op.forward.output.0", "op.forward.parameters.weight"] summary_list = ["summary1", "summary2", "summary3"] data_name_list = None - result_op_name, result_summary, result_data_name = reorder_op_x_list(op_name_list, summary_list, data_name_list) + state_list = ["input", "output", "parameters"] + result_op_name, result_summary, result_data_name, result_state = reorder_op_x_list(op_name_list, summary_list, data_name_list, state_list) self.assertEqual(result_op_name, ["op.forward.input.0", "op.forward.parameters.weight", "op.forward.output.0"]) self.assertEqual(result_summary, ["summary1", "summary3", "summary2"]) self.assertEqual(result_data_name, None) + self.assertEqual(result_state, ["input", "parameters", "output"]) class TestGenOpItem(unittest.TestCase): @@ -661,7 +648,7 @@ class TestGenOpItem(unittest.TestCase): } op_name = 'op_test' - result = gen_op_item(op_data, op_name) + result = gen_op_item(op_data, op_name, 'input') self.assertEqual(result['data_name'], 'test_data') self.assertEqual(result['full_op_name'], 'test_data') @@ -672,6 +659,7 @@ class TestGenOpItem(unittest.TestCase): self.assertEqual(result['Mean'], 2) self.assertEqual(result['Norm'], 2) self.assertEqual(result['md5'], f"{zlib.crc32(str(op_data['value']).encode()):08x}") + self.assertEqual(result['state'], 'input') def test_gen_op_item_with_empty_data_name(self): op_data = { @@ -681,11 +669,12 @@ class TestGenOpItem(unittest.TestCase): } op_name = 'op_test' - result = gen_op_item(op_data, op_name) + result = gen_op_item(op_data, op_name, 'input') # data_name为空时，应该被设置为'-1' self.assertEqual(result['data_name'], '-1') self.assertEqual(result['full_op_name'], op_name) + self.assertEqual(result['state'], 'input') def test_gen_op_item_with_none_data_name(self): op_data = { @@ -695,11 +684,12 @@ class TestGenOpItem(unittest.TestCase): } op_name = 'op_test' - result = gen_op_item(op_data, op_name) + result = gen_op_item(op_data, op_name, 'input') # data_name为None时，应该被设置为'-1' self.assertEqual(result['data_name'], '-1') self.assertEqual(result['full_op_name'], op_name) + self.assertEqual(result['state'], 'input') def test_gen_op_item_with_type_torch_size(self): op_data = { @@ -709,7 +699,7 @@ class TestGenOpItem(unittest.TestCase): } op_name = 'op_test' - result = gen_op_item(op_data, op_name) + result = gen_op_item(op_data, op_name, 'input') self.assertEqual(result['dtype'], 'torch.Size') self.assertEqual(result['shape'], '[2, 3, 4]') @@ -717,6 +707,7 @@ class TestGenOpItem(unittest.TestCase): self.assertEqual(result['Min'], None) self.assertEqual(result['Mean'], None) self.assertEqual(result['Norm'], None) + self.assertEqual(result['state'], 'input') def test_gen_op_item_with_type_slice(self): op_data = { @@ -726,10 +717,11 @@ class TestGenOpItem(unittest.TestCase): } op_name = 'op_test' - result = gen_op_item(op_data, op_name) + result = gen_op_item(op_data, op_name, 'input') self.assertEqual(result['dtype'], 'slice') self.assertEqual(result['shape'], str(np.shape(np.array(op_data['value'])))) + self.assertEqual(result['state'], 'input') def test_gen_op_item_with_type_ellipsis(self): op_data = { @@ -739,7 +731,7 @@ class TestGenOpItem(unittest.TestCase): } op_name = 'op_test' - result = gen_op_item(op_data, op_name) + result = gen_op_item(op_data, op_name, 'input') self.assertEqual(result['dtype'], 'ellipsis') self.assertEqual(result['shape'], '[]') @@ -747,6 +739,7 @@ class TestGenOpItem(unittest.TestCase): self.assertEqual(result['Min'], '...') self.assertEqual(result['Mean'], '...') self.assertEqual(result['Norm'], '...') + self.assertEqual(result['state'], 'input') def test_gen_op_item_with_type_torch_process_group(self): op_data = { @@ -756,7 +749,7 @@ class TestGenOpItem(unittest.TestCase): } op_name = 'op_test' - result = gen_op_item(op_data, op_name) + result = gen_op_item(op_data, op_name, 'input') self.assertEqual(result['dtype'], 'torch.ProcessGroup') self.assertEqual(result['shape'], '[]') @@ -764,6 +757,7 @@ class TestGenOpItem(unittest.TestCase): self.assertEqual(result['Min'], '[0, 1]') self.assertEqual(result['Mean'], '[0, 1]') self.assertEqual(result['Norm'], '[0, 1]') + self.assertEqual(result['state'], 'input') def test_gen_op_item_with_default_dtype(self): op_data = { @@ -773,10 +767,11 @@ class TestGenOpItem(unittest.TestCase): } op_name = 'op_test' - result = gen_op_item(op_data, op_name) + result = gen_op_item(op_data, op_name, 'input') self.assertEqual(result['dtype'], str(type(op_data['value']))) self.assertEqual(result['shape'], '[]') + self.assertEqual(result['state'], 'input') def test_gen_op_item_with_md5(self): op_data = { @@ -786,7 +781,8 @@ class TestGenOpItem(unittest.TestCase): } op_name = 'op_test' - result = gen_op_item(op_data, op_name) + result = gen_op_item(op_data, op_name, 'input') expected_md5 = f"{zlib.crc32(str(op_data['value']).encode()):08x}" self.assertEqual(result['md5'], expected_md5) + self.assertEqual(result['state'], 'input') diff --git a/debug/accuracy_tools/msprobe/test/core_ut/compare/test_cmp_highlight.py b/debug/accuracy_tools/msprobe/test/core_ut/compare/test_cmp_highlight.py index 5ffc0013fa..933c846013 100644 --- a/debug/accuracy_tools/msprobe/test/core_ut/compare/test_cmp_highlight.py +++ b/debug/accuracy_tools/msprobe/test/core_ut/compare/test_cmp_highlight.py @@ -37,19 +37,19 @@ summary_line_3 = ['Functional_batch_norm_0_forward.output.2', 'Functional_batch_ line_input = ['Functional.batch.norm.0.forward.input.0', 'Functional.batch.norm.0.forward.input.0', 'torch.float16', 'torch.float32', [256, 256, 14, 14], [256, 256, 14, 14], 1, 0.5, 1, 1, 0.95, 1, 1, 1, 1, 1, 1.01, 1, 1, 1, - 'Yes', ''] + 'Yes', '', 'input', 'Functional.batch.norm.0.forward'] line_1 = ['Functional.batch.norm.0.forward.output.0', 'Functional.batch.norm.0.forward.output.0', 'torch.float16', 'torch.float32', [256, 256, 14, 14], [256, 256, 14, 14], 0.8, 0.5, 1, 1, 0.59, 1, 'nan', 0, 1, 1, 19, 1, 1, 1, - 'Yes', ''] + 'Yes', '', 'output', 'Functional.batch.norm.0.forward'] line_2 = ['Functional.batch.norm.0.forward.output.1', 'Functional.batch.norm.0.forward.output.1', 'torch.float16', 'torch.float32', [256, 256, 14, 14], [256, 256, 14, 14], 0.9, 0.5, 1, 1, 0.8, 1, 0, 0.12, 0, 1, 1, 0.1, 1, 1, - 'Yes', ''] + 'Yes', '', 'output', 'Functional.batch.norm.0.forward'] line_3 = ['Functional.batch.norm.0.forward.output.2', 'Functional.batch.norm.0.forward.output.2', 'torch.float16', 'torch.float32', [256, 256, 14, 14], [256, 256, 14, 14], 0.8, 0.5, 1.1e+10, 1, 0.85, 1, 9, 0.12, 0, 1, 1, 0.1, 1, 1, - 'Yes', ''] + 'Yes', '', 'output', 'Functional.batch.norm.0.forward'] base_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), f'test_highlight') diff --git a/debug/accuracy_tools/msprobe/test/core_ut/compare/test_cmp_multiprocessing_compute.py b/debug/accuracy_tools/msprobe/test/core_ut/compare/test_cmp_multiprocessing_compute.py index 1e2a8c90ab..d95abf6d91 100644 --- a/debug/accuracy_tools/msprobe/test/core_ut/compare/test_cmp_multiprocessing_compute.py +++ b/debug/accuracy_tools/msprobe/test/core_ut/compare/test_cmp_multiprocessing_compute.py @@ -103,7 +103,7 @@ class TestCompareRealData(unittest.TestCase): # index error with self.assertRaises(CompareException) as context: result = compare_real_data.read_dump_data(pd.DataFrame()) - self.assertEqual(context.exception.code, CompareException.INDEX_OUT_OF_BOUNDS_ERROR) + self.assertEqual(context.exception.code, CompareException.INVALID_KEY_ERROR) def test_save_cmp_result_success(self): file_reader = read_real_data -- Gitee