From c6c112eca2f0fee6e2bbb7888150151e05bff611 Mon Sep 17 00:00:00 2001 From: Linwei-Ying Date: Wed, 22 May 2024 16:30:52 +0800 Subject: [PATCH 1/7] =?UTF-8?q?compare=E9=80=82=E9=85=8D=E6=96=B0=E7=9A=84?= =?UTF-8?q?=E8=BE=93=E5=87=BA=E4=BB=B6=E6=A0=BC=E5=BC=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/python/ptdbg_ascend/common/utils.py | 84 ++++- .../ptdbg_ascend/compare/acc_compare.py | 312 +++++++++++++----- .../compare/distributed_compare.py | 48 ++- 3 files changed, 328 insertions(+), 116 deletions(-) diff --git a/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/common/utils.py b/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/common/utils.py index 6b2b62dba..2ea2a7767 100644 --- a/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/common/utils.py +++ b/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/common/utils.py @@ -126,8 +126,8 @@ class CompareConst: # compare result column name NPU_NAME = "NPU Name" BENCH_NAME = "Bench Name" - NPU_DTYPE = "NPU Tensor Dtype" - BENCH_DTYPE = "Bench Tensor Dtype" + NPU_DTYPE = "NPU Dtype" + BENCH_DTYPE = "Bench Dtype" NPU_SHAPE = "NPU Tensor Shape" BENCH_SHAPE = "Bench Tensor Shape" NPU_MAX = "NPU max" @@ -147,6 +147,7 @@ class CompareConst: MAX_RELATIVE_ERR = "MaxRelativeErr" ACCURACY = "Accuracy Reached or Not" STACK = "NPU_Stack_Info" + DATA_NAME = "Data_name" ERROR_MESSAGE = "Err_message" ONE_THOUSANDTH_ERR_RATIO = "One Thousandth Err Ratio" FIVE_THOUSANDTHS_ERR_RATIO = "Five Thousandths Err Ratio" @@ -171,6 +172,7 @@ class CompareConst: # compare result data NAN = 'Nan' + NONE = 'None' SHAPE_UNMATCH = 'shape unmatched' DTYPE_UNMATCH = 'dtype unmatched' PASS = 'Pass' @@ -235,6 +237,7 @@ class CompareException(Exception): INVALID_COMPARE_MODE = 17 OVER_SIZE_FILE_ERROR = 18 INVALID_SUMMARY_MODE = 19 + INVALID_TASK_ERROR = 20 def __init__(self, code, error_info: str = ""): super(CompareException, self).__init__() @@ -369,19 +372,21 @@ def check_summary_only_valid(summary_only): return summary_only -def check_compare_param(input_parma, output_path, stack_mode=False, summary_compare=False): # 添加默认值来让不传参时能通过参数检查 +def check_compare_param(input_parma, output_path, stack_mode=False, summary_compare=False, md5_compare=False): if not (isinstance(input_parma, dict) and isinstance(output_path, str)): print_error_log("Invalid input parameters") raise CompareException(CompareException.INVALID_PARAM_ERROR) - check_file_or_directory_path(input_parma.get("npu_pkl_path"), False) - check_file_or_directory_path(input_parma.get("bench_pkl_path"), False) - if not summary_compare: + check_file_or_directory_path(input_parma.get("npu_json_path"), False) + check_file_or_directory_path(input_parma.get("bench_json_path"), False) + check_file_or_directory_path(input_parma.get("stack_json_path"), False) + if not summary_compare and not md5_compare: check_file_or_directory_path(input_parma.get("npu_dump_data_dir"), True) check_file_or_directory_path(input_parma.get("bench_dump_data_dir"), True) check_file_or_directory_path(output_path, True) - with FileOpen(input_parma.get("npu_pkl_path"), "r") as npu_pkl, \ - FileOpen(input_parma.get("bench_pkl_path"), "r") as bench_pkl: - check_pkl_file(input_parma, npu_pkl, bench_pkl, stack_mode) + with FileOpen(input_parma.get("npu_json_path"), "r") as npu_json, \ + FileOpen(input_parma.get("bench_json_path"), "r") as bench_json, \ + FileOpen(input_parma.get("stack_json_path"), "r") as stack_json: + check_json_file(input_parma, npu_json, bench_json, stack_json, stack_mode) def is_summary_compare(input_param): @@ -414,6 +419,53 @@ def is_md5_compare(input_parma): return False +def md5_find(data): + for key_op in data: + for key_xxput in data[key_op]: + if isinstance(data[key_op][key_xxput], list): + for i in range(len(data[key_op][key_xxput])): + if data[key_op][key_xxput][i] == None: + continue + elif 'md5' in data[key_op][key_xxput][i]: + return True + elif 'md5' in data[key_op][key_xxput]: + return True + return False + + +def task_dumppath_get(input_param): + npu_json_path = input_param.get("npu_json_path", None) + bench_json_path = input_param.get("bench_json_path", None) + if not npu_json_path or not bench_json_path: + print_error_log(f"Please check the json path is valid.") + raise CompareException(CompareException.INVALID_PATH_ERROR) + with open(npu_json_path, 'r') as npu_f: + npu_json_data = json.load(npu_f) + with open(bench_json_path, 'r') as bench_f: + bench_json_data = json.load(bench_f) + if npu_json_data['task'] != bench_json_data['task']: + print_error_log(f"Please check the dump task is consistent.") + raise CompareException(CompareException.INVALID_TASK_ERROR) + else: + if npu_json_data['task'] == 'tensor': + summary_compare = False + md5_compare = False + elif npu_json_data['task'] == 'statistics': + md5_compare = md5_find(npu_json_data['data']) + if md5_compare: + summary_compare = False + else: + summary_compare = True + else: + print_error_log(f"Compare is not required for overflow_check.") + raise CompareException(CompareException.INVALID_TASK_ERROR) + input_param['npu_dump_data_dir'] = npu_json_data['dump_data_dir'] + input_param['bench_dump_data_dir'] = bench_json_data['dump_data_dir'] + return summary_compare, md5_compare + print_error_log(f"Please check the dump data dir is valid.") + raise CompareException(CompareException.INVALID_PATH_ERROR) + + def check_configuration_param(stack_mode=False, auto_analyze=True, fuzzy_match=False): if not (isinstance(stack_mode, bool) and isinstance(auto_analyze, bool) and isinstance(fuzzy_match, bool)): print_error_log("Invalid input parameters which should be only bool type.") @@ -445,6 +497,14 @@ def _check_pkl(pkl_file_handle, file_name): pkl_file_handle.seek(0, 0) +def _check_json(json_file_handle, file_name): + tensor_line = json_file_handle.readline() + if len(tensor_line) == 0: + print_error_log("dump file {} have empty line!".format(file_name)) + raise CompareException(CompareException.INVALID_DUMP_FILE) + json_file_handle.seek(0, 0) + + def is_starts_with(string, prefix_list): return any(string.startswith(prefix) for prefix in prefix_list) @@ -487,6 +547,12 @@ def check_pkl_file(input_param, npu_pkl, bench_pkl, stack_mode): raise CompareException(CompareException.INVALID_COMPARE_MODE) +def check_json_file(input_param, npu_json, bench_json, stack_json, stack_mode): + _check_json(npu_json, input_param.get("npu_json_path")) + _check_json(bench_json, input_param.get("bench_json_path")) + _check_json(stack_json, input_param.get("stack_json_path")) + + def check_file_size(input_file, max_size): try: file_size = os.path.getsize(input_file) diff --git a/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/compare/acc_compare.py b/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/compare/acc_compare.py index be1e0dae7..cedfaea0e 100644 --- a/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/compare/acc_compare.py +++ b/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/compare/acc_compare.py @@ -20,6 +20,7 @@ import multiprocessing import os.path import stat import sys +import torch import numpy as np import pandas as pd @@ -29,7 +30,7 @@ from ..advisor.advisor import Advisor from ..common.utils import check_compare_param, add_time_as_suffix, \ print_info_log, print_warn_log, print_error_log, CompareException, Const, \ CompareConst, format_value, check_file_not_exists, check_configuration_param, \ - is_summary_compare, is_md5_compare + is_summary_compare, is_md5_compare, task_dumppath_get from ..common.file_check_util import FileChecker, FileCheckConst, change_mode, FileOpen @@ -227,67 +228,47 @@ def rename_api(npu_name, process): return torch_func -def merge_tensor(tensor_list): +def merge_tensor(tensor_list, summary_compare, md5_compare): op_dict = {} op_dict["op_name"] = [] op_dict["input_struct"] = [] + op_dict["kwargs_struct"] = [] op_dict["output_struct"] = [] op_dict["summery"] = [] op_dict["stack_info"] = [] + all_mode_bool = summary_compare == False and md5_compare == False + if all_mode_bool: + op_dict["data_name"] = [] + for tensor in tensor_list: - if tensor[0].find("stack_info") != -1: - if len(tensor) != Const.STACK_COLUMN_NUM: - print_error_log(f"This stack_info data is not complete. {tensor}") - raise CompareException(CompareException.INVALID_DATA_ERROR) - op_dict["stack_info"].append(tensor[1]) + if len(tensor) == 2: + op_dict['stack_info'].append(tensor['full_info']) break - op_dict["op_name"].append(tensor[0]) - if len(tensor) != Const.SUMMARY_COLUMN_NUM: - print_error_log(f"This summary data is not complete. {tensor}") - raise CompareException(CompareException.INVALID_DATA_ERROR) - if tensor[0].find("input") != -1: - op_dict["input_struct"].append((tensor[3], tensor[4], tensor[2])) - elif tensor[0].find("output") != -1: - op_dict["output_struct"].append((tensor[3], tensor[4], tensor[2])) - - if tensor[1] <= Const.DUMP_RATIO_MAX: - op_dict["summery"].append(tensor[5]) - - return op_dict - + op_dict["op_name"].append(tensor['full_op_name']) + if not md5_compare: + if tensor['full_op_name'].find("input") != -1: + op_dict["input_struct"].append((tensor['dtype'], tensor['shape'])) + elif tensor['full_op_name'].find("kwarg") != -1: + op_dict["kwargs_struct"].append((tensor['dtype'], tensor['shape'])) + elif tensor['full_op_name'].find("output") != -1: + op_dict["output_struct"].append((tensor['dtype'], tensor['shape'])) + else: + if tensor['full_op_name'].find("input") != -1: + op_dict["input_struct"].append((tensor['dtype'], tensor['shape'], tensor['md5'])) + elif tensor['full_op_name'].find("kwarg") != -1: + op_dict["kwargs_struct"].append((tensor['dtype'], tensor['shape'], tensor['md5'])) + elif tensor['full_op_name'].find("output") != -1: + op_dict["output_struct"].append((tensor['dtype'], tensor['shape'], tensor['md5'])) -def read_op(ops_queue, pkl_file_handle, stack_mode): - tensor_list = [] - read_err = False - read_output_flag = {"last_line": False, "curr_line": False} - end_flag = "stack_info" if stack_mode is True else "output" + op_dict["summery"].append([tensor['Max'], tensor['Min'], tensor['Mean'], tensor['Norm']]) - while True: - curr_pos = pkl_file_handle.tell() - tensor_line = pkl_file_handle.readline() - if len(tensor_line) == 0 and not read_output_flag.get("curr_line"): - read_err = True - break - if tensor_line == '\n': - continue - if len(tensor_line) != 0: - tensor_data = json.loads(tensor_line) - if not isinstance(tensor_data, list): - print_error_log(f"This data is not a list, please check the dump data pkl file. {tensor_data}") - raise CompareException(CompareException.INVALID_DATA_ERROR) - read_output_flag["last_line"] = read_output_flag.get("curr_line") - read_output_flag["curr_line"] = True if tensor_data[0].find(end_flag) != -1 else False - - if (read_output_flag.get("last_line") and not read_output_flag.get("curr_line")) \ - or (len(tensor_line) == 0 and read_output_flag.get("curr_line")): # end of file scenario - ops_queue.append(merge_tensor(tensor_list)) - # the pos of the handle needs to restore to the start of the next api. - pkl_file_handle.seek(curr_pos, 0) - break - tensor_list.append(tensor_data) + if all_mode_bool: + op_dict["data_name"].append(tensor['data_name']) - return not read_err + if not op_dict["kwargs_struct"]: + del op_dict["kwargs_struct"] + return op_dict def match_op(npu_queue, bench_queue, fuzzy_match): @@ -308,7 +289,17 @@ def get_accuracy(result, n_dict, b_dict, summary_compare=False, md5_compare=Fals npu_stack_info = n_dict.get("stack_info", None) bench_stack_info = b_dict.get("stack_info", None) has_stack = npu_stack_info and bench_stack_info + + all_mode_bool = summary_compare == False and md5_compare == False + if all_mode_bool: + npu_data_name = n_dict.get("data_name", None) + bench_data_name = b_dict.get("data_name", None) + has_data_name = False + for index in range(min_len): + if all_mode_bool: + has_data_name = npu_data_name[n_start + index] and bench_data_name[b_start + index] + n_name = n_dict['op_name'][n_start + index] b_name = b_dict['op_name'][b_start + index] n_struct = n_dict[key][index] @@ -319,6 +310,10 @@ def get_accuracy(result, n_dict, b_dict, summary_compare=False, md5_compare=Fals n_struct[2], b_struct[2], CompareConst.PASS if n_struct[2] == b_struct[2] else CompareConst.DIFF] if has_stack and index == 0 and key == "input_struct": result_item.extend(npu_stack_info) + else: + result_item.append(CompareConst.NONE) + if all_mode_bool and has_data_name: + result_item.append(npu_data_name[n_start + index]) result.append(result_item) continue @@ -345,7 +340,8 @@ def get_accuracy(result, n_dict, b_dict, summary_compare=False, md5_compare=Fals if magnitude_diff > 0.5: warning_flag = True else: - result_item[start_idx + i] = CompareConst.NAN + # result_item[start_idx + i] = CompareConst.NAN + result_item[start_idx + i] = CompareConst.NONE accuracy_check = CompareConst.WARNING if warning_flag else "" err_msg += "Need double check api accuracy." if warning_flag else "" result_item[start_idx:] = [f'{str(x)}\t' if str(x) in ('inf', '-inf', 'nan') else x for x in result_item[start_idx:]] @@ -354,11 +350,18 @@ def get_accuracy(result, n_dict, b_dict, summary_compare=False, md5_compare=Fals result_item.append(err_msg) if has_stack and index == 0 and key == "input_struct": result_item.extend(npu_stack_info) + else: + result_item.append(CompareConst.NONE) + if all_mode_bool and has_data_name: + result_item.append(npu_data_name[n_start + index]) result.append(result_item) if n_len > b_len: for index in range(b_len, n_len): + if all_mode_bool: + has_data_name = npu_data_name[n_start + index] and bench_data_name[b_start + index] + n_name = n_dict['op_name'][n_start + index] n_struct = n_dict[key][index] if md5_compare: @@ -379,6 +382,10 @@ def get_accuracy(result, n_dict, b_dict, summary_compare=False, md5_compare=Fals if has_stack and index == 0 and key == "input_struct": result_item.extend(npu_stack_info) + else: + result_item.append(CompareConst.NONE) + if all_mode_bool and has_data_name: + result_item.append(npu_data_name[n_start + index]) result.append(result_item) @@ -386,10 +393,13 @@ def get_accuracy(result, n_dict, b_dict, summary_compare=False, md5_compare=Fals b_num = len(b_dict['op_name']) n_num_input = len([name for name in n_dict['op_name'] if 'input' in name]) b_num_input = len([name for name in b_dict['op_name'] if 'input' in name]) - n_num_output = n_num - n_num_input - b_num_output = b_num - b_num_input + n_num_kwarg = len([name for name in n_dict['op_name'] if 'kwarg' in name]) + b_num_kwarg = len([name for name in b_dict['op_name'] if 'kwarg' in name]) + n_num_output = n_num - n_num_input - n_num_kwarg + b_num_output = b_num - b_num_input - b_num_kwarg get_accuracy_core(0, n_num_input, 0, b_num_input, 'input_struct') - get_accuracy_core(n_num_input, n_num_output, b_num_input, b_num_output, 'output_struct') + get_accuracy_core(n_num_input, n_num_kwarg, b_num_input, b_num_kwarg, "kwargs_struct") + get_accuracy_core(n_num_input + n_num_kwarg, n_num_output, b_num_input + b_num_kwarg, b_num_output, 'output_struct') def _do_multi_process(input_parma, result_path): @@ -407,12 +417,14 @@ def read_dump_path(result_path): try: csv_pd = pd.read_csv(result_path) npu_dump_name_list = csv_pd.iloc[0:, 0].tolist() - bench_dump_name_list = csv_pd.iloc[0:, 1].tolist() + npu_dump_tensor_list = csv_pd.iloc[0:, -1].tolist() + # bench_dump_name_list = csv_pd.iloc[0:, 1].tolist() op_name_mapping_dict = {} for index, _ in enumerate(npu_dump_name_list): npu_dump_name = npu_dump_name_list[index] - bench_dump_name = bench_dump_name_list[index] - op_name_mapping_dict[npu_dump_name] = [npu_dump_name, bench_dump_name] + npu_dump_tensor = npu_dump_tensor_list[index] + # bench_dump_name = bench_dump_name_list[index] + op_name_mapping_dict[npu_dump_name] = [npu_dump_tensor, npu_dump_tensor] return op_name_mapping_dict except FileNotFoundError as e: print_error_log('{} file is not found.'.format(result_path)) @@ -464,7 +476,12 @@ def compare_ops(idx, fusion_op_names, dump_path_dict, result_path, lock, input_p for i, op_name in enumerate(fusion_op_names): if is_print_compare_log: print("start compare: {}".format(op_name)) - cos_sim, max_abs_err, max_relative_err, err_msg, one_thousand_err_ratio, five_thousand_err_ratio = compare_by_op(op_name, dump_path_dict, input_parma) + + if op_name == '-1': + cos_sim = max_abs_err = max_relative_err = err_msg = one_thousand_err_ratio = five_thousand_err_ratio = CompareConst.NONE + else: + cos_sim, max_abs_err, max_relative_err, err_msg, one_thousand_err_ratio, five_thousand_err_ratio = compare_by_op(op_name, dump_path_dict, input_parma) + if is_print_compare_log: print("[{}] Compare result: cosine {}, max_abs_err {}, max_relative_err {}, {}, one_thousand_err_ratio {}, five_thousand_err_ratio {}".format(op_name, cos_sim, max_abs_err, max_relative_err, err_msg, one_thousand_err_ratio, five_thousand_err_ratio)) cos_result.append(cos_sim) @@ -506,15 +523,15 @@ def _save_cmp_result(idx, cos_result, max_err_result, max_relative_err_result, e def check_accuracy(cos, max_abs_err): if cos == CompareConst.SHAPE_UNMATCH: return CompareConst.ACCURACY_CHECK_UNMATCH - if cos == CompareConst.NAN or max_abs_err == CompareConst.NAN: - return CompareConst.NAN + if cos == CompareConst.NONE or max_abs_err == CompareConst.NONE: + return CompareConst.NONE if cos == "N/A" or max_abs_err == "N/A": return CompareConst.ACCURACY_CHECK_NO try: cos, max_abs_err = float(cos), float(max_abs_err) except ValueError: print_warn_log("Cosine or MaxAbsErr can not get float value.") - return CompareConst.NAN + return CompareConst.NONE if cos < CompareConst.COS_THRESHOLD and max_abs_err > CompareConst.MAX_ABS_ERR_THRESHOLD: return CompareConst.ACCURACY_CHECK_NO if cos < CompareConst.COS_MAX_THRESHOLD or max_abs_err > CompareConst.MAX_ABS_ERR_MAX_THRESHOLD: @@ -524,19 +541,19 @@ def check_accuracy(cos, max_abs_err): def compare_by_op(op_name, op_name_mapping_dict, input_parma): npu_bench_name_list = op_name_mapping_dict[op_name] - if npu_bench_name_list[1] == CompareConst.NAN: - return CompareConst.NAN, CompareConst.NAN, CompareConst.NAN, CompareConst.NO_BENCH, CompareConst.NAN, CompareConst.NAN + if int(npu_bench_name_list[1]) == -1: + return CompareConst.NONE, CompareConst.NONE, CompareConst.NONE, CompareConst.NO_BENCH, CompareConst.NONE, CompareConst.NONE try: - n_path = os.path.join(input_parma.get("npu_dump_data_dir"), npu_bench_name_list[0] + ".npy") - b_path = os.path.join(input_parma.get("bench_dump_data_dir"), npu_bench_name_list[1] + ".npy") + n_path = os.path.join(input_parma.get("npu_dump_data_dir"), npu_bench_name_list[0]) + b_path = os.path.join(input_parma.get("bench_dump_data_dir"), npu_bench_name_list[1]) n_path_checker = FileChecker(n_path, FileCheckConst.FILE, FileCheckConst.READ_ABLE, - FileCheckConst.NUMPY_SUFFIX, False) + FileCheckConst.PT_SUFFIX, False) b_path_checker = FileChecker(b_path, FileCheckConst.FILE, FileCheckConst.READ_ABLE, - FileCheckConst.NUMPY_SUFFIX, False) + FileCheckConst.PT_SUFFIX, False) n_path = n_path_checker.common_check() b_path = b_path_checker.common_check() - n_value = np.load(n_path) - b_value = np.load(b_path) + n_value = torch.load(n_path).detach().numpy() + b_value = torch.load(b_path).detach().numpy() except IOError as error: return CompareConst.NAN, CompareConst.NAN, CompareConst.NAN, "Dump file: {} not found.".format(error.filename), CompareConst.NAN, CompareConst.NAN relative_err = get_relative_err(n_value, b_value) @@ -601,10 +618,9 @@ def handle_inf_nan(n_value, b_value): def compare(input_parma, output_path, stack_mode=False, auto_analyze=True, fuzzy_match=False): try: - summary_compare = is_summary_compare(input_parma) - md5_compare = is_md5_compare(input_parma) + summary_compare, md5_compare = task_dumppath_get(input_parma) check_configuration_param(stack_mode, auto_analyze, fuzzy_match) - check_compare_param(input_parma, output_path, stack_mode, summary_compare) + check_compare_param(input_parma, output_path, stack_mode, summary_compare, md5_compare) except CompareException as error: print_error_log('Compare failed. Please check the arguments and do it again!') sys.exit(error.code) @@ -620,11 +636,12 @@ def compare_core(input_parma, output_path, stack_mode=False, auto_analyze=True, file_path = os.path.join(os.path.realpath(output_path), file_name) check_file_not_exists(file_path) - with FileOpen(input_parma.get("npu_pkl_path"), "r") as npu_pkl, \ - FileOpen(input_parma.get("bench_pkl_path"), "r") as bench_pkl, \ + with FileOpen(input_parma.get("npu_json_path"), "r") as npu_json, \ + FileOpen(input_parma.get("bench_json_path"), "r") as bench_json, \ + FileOpen(input_parma.get("stack_json_path"), "r") as stack_json, \ os.fdopen(os.open(file_path, os.O_RDWR | os.O_CREAT, stat.S_IWUSR | stat.S_IRUSR | stat.S_IRGRP), 'w+') \ as fout: - compare_process([npu_pkl, bench_pkl, fout], stack_mode, fuzzy_match, summary_compare, md5_compare) + compare_process([npu_json, bench_json, stack_json, fout], stack_mode, fuzzy_match, summary_compare, md5_compare) if summary_compare: print_info_log(f"Summary compare result is {file_path}") @@ -671,19 +688,137 @@ def parse(pkl_file, module_name_prefix): print(summery_info) +def op_item_parse(item, op_name, index, item_list=[], top_bool=True): + if item == None or (isinstance(item, dict) and len(item) == 0): + if not top_bool: + tmp = {'full_op_name': op_name + '.' + str(index), 'Max': None, 'Min': None, 'Mean': None, 'Norm': None, 'dtype': None, 'shape': None, 'md5': None, 'data_name': '-1'} + else: + tmp = {'full_op_name': op_name + '.0', 'Max': None, 'Min': None, 'Mean': None, 'Norm': None, 'dtype': None, 'shape': None, 'md5': None, 'data_name': '-1'} + item_list.append(tmp) + return item_list + if index == None: + if isinstance(item, dict): + full_op_name = op_name + '.0' + else: + full_op_name = op_name + else: + full_op_name = op_name + '.' + str(index) + if isinstance(item, dict): + if 'dtype' in item: + parsed_item = item + parsed_item['full_op_name'] = full_op_name + item_list.append(parsed_item) + else: + parsed_item = {} + if item['type'] == 'slice': + parsed_item['full_op_name'] = full_op_name + parsed_item['dtype'] = 'slice' + parsed_item['shape'] = str(np.shape(np.array(item['value']))) + parsed_item['md5'] = None + parsed_item['Max'] = None + parsed_item['Min'] = None + parsed_item['Mean'] = None + parsed_item['Norm'] = None + parsed_item['data_name'] = '-1' + item_list.append(parsed_item) + else: + parsed_item['full_op_name'] = full_op_name + parsed_item['dtype'] = str(type(item['value'])) + parsed_item['shape'] = '[]' + parsed_item['md5'] = None + parsed_item['Max'] = item['value'] + parsed_item['Min'] = item['value'] + parsed_item['Mean'] = item['value'] + parsed_item['Norm'] = item['value'] + parsed_item['data_name'] = '-1' + item_list.append(parsed_item) + else: + for j in range(len(item)): + op_item_parse(item[j], full_op_name, j, top_bool=False) + return item_list + + +def read_op(op_data, opname): + if 'input_args' in op_data: + input_item = op_data['input_args'] + input_parsed_list = op_item_parse(input_item, opname + '_input', None) + op_parsed_list = input_parsed_list.copy() + input_parsed_list.clear() + if 'input_kwargs' in op_data: + kwargs_item = op_data['input_kwargs'] + if isinstance(kwargs_item, dict) and "type" in kwargs_item or isinstance(kwargs_item, list): + kwarg_parsed_list = op_item_parse(kwargs_item, opname + '_input', None) + op_parsed_list += kwarg_parsed_list + kwarg_parsed_list.clear() + elif kwargs_item: + for kwarg in kwargs_item: + kwarg_parsed_list = op_item_parse(kwargs_item[kwarg], opname + '_input.' + kwarg, None) + op_parsed_list += kwarg_parsed_list + kwarg_parsed_list.clear() + if 'output' in op_data: + output_item = op_data['output'] + output_parsed_list = op_item_parse(output_item, opname + '_output', None) + op_parsed_list += output_parsed_list + output_parsed_list.clear() + return op_parsed_list + + def compare_process(file_handles, stack_mode, fuzzy_match, summary_compare=False, md5_compare=False): - npu_pkl_handle, bench_pkl_handle, output_csv_handle = file_handles + npu_json_handle, bench_json_handle, stack_json_handle, output_csv_handle = file_handles + npu_json_data = json.load(npu_json_handle) + bench_json_data = json.load(bench_json_handle) + stack_json_data = json.load(stack_json_handle) + if fuzzy_match: print_warn_log("This task uses fuzzy matching, which may affect the accuracy of the comparison.") + npu_ops_queue = [] bench_ops_queue = [] result = [] + + ops_npu_iter = iter(npu_json_data['data']) + ops_bench_iter = iter(bench_json_data['data']) + read_err_npu = True + read_err_bench = True + while True: - npu_file_flag = read_op(npu_ops_queue, npu_pkl_handle, stack_mode) - bench_file_flag = read_op(bench_ops_queue, bench_pkl_handle, stack_mode) - if (not npu_file_flag and not bench_file_flag) \ - or (len(npu_ops_queue) == 0 or len(bench_ops_queue) == 0): + if not read_err_npu or not read_err_bench: break + try: + opname_npu = next(ops_npu_iter) + read_err_npu = True + + npuop_data = npu_json_data['data'][opname_npu] + npuop_parsed_list = read_op(npuop_data, opname_npu) + if opname_npu in stack_json_data: + npuop_parsed_list.append({'full_op_name': opname_npu, 'full_info': stack_json_data[opname_npu]}) + else: + npuop_parsed_list.append({'full_op_name': opname_npu, 'full_info': None}) + + npu_ops_queue.append(merge_tensor(npuop_parsed_list, summary_compare, md5_compare)) + except StopIteration: + read_err_npu = False + continue + try: + opname_bench = next(ops_bench_iter) + read_err_bench = True + + benchop_data = bench_json_data['data'][opname_bench] + benchop_parsed_list = read_op(benchop_data, opname_bench) + if opname_bench in stack_json_data: + benchop_parsed_list.append( + {'full_op_name': opname_bench, 'full_info': stack_json_data[opname_bench]}) + else: + benchop_parsed_list.append({'full_op_name': opname_bench, 'full_info': None}) + + bench_ops_queue.append(merge_tensor(benchop_parsed_list, summary_compare, md5_compare)) + except StopIteration: + read_err_bench = False + continue + + if len(npu_ops_queue) == 0 or len(bench_ops_queue) == 0: + break + n_match_point, b_match_point = match_op(npu_ops_queue, bench_ops_queue, fuzzy_match) if n_match_point == -1 and b_match_point == -1: continue @@ -706,8 +841,23 @@ def compare_process(file_handles, stack_mode, fuzzy_match, summary_compare=False header = CompareConst.SUMMARY_COMPARE_RESULT_HEADER[:] else: header = CompareConst.COMPARE_RESULT_HEADER[:] + + all_mode_bool = summary_compare == False and md5_compare == False if stack_mode: - header.append(CompareConst.STACK) + if all_mode_bool: + header.append(CompareConst.STACK) + header.append(CompareConst.DATA_NAME) + else: + header.append(CompareConst.STACK) + else: + if all_mode_bool: + for row in result: + del row[-2] + header.append(CompareConst.DATA_NAME) + else: + for row in result: + del row[-1] + result_df = pd.DataFrame(result, columns=header) result_df.to_csv(output_csv_handle, index=False) diff --git a/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/compare/distributed_compare.py b/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/compare/distributed_compare.py index 85f9bb95f..077bcd132 100644 --- a/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/compare/distributed_compare.py +++ b/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/compare/distributed_compare.py @@ -18,7 +18,7 @@ import os import sys import re from ..common.utils import print_error_log, CompareException, check_compare_param, check_file_or_directory_path, \ - check_configuration_param, is_summary_compare, is_md5_compare + check_configuration_param, is_summary_compare, is_md5_compare, task_dumppath_get from .acc_compare import compare_core @@ -36,28 +36,23 @@ def compare_distributed(npu_dump_dir, bench_dump_dir, output_path, **kwargs): raise CompareException(CompareException.INVALID_PATH_ERROR) return contents - def extract_pkl_and_data_dir(dirname): - pkl_path, dump_data_dir, pkl_name, dump_data_dirname = '', '', '', '' + + def extract_json(dirname, stack_json=False): + json_path = '' for fname in os.listdir(dirname): full_path = os.path.join(dirname, fname) - if os.path.isdir(full_path): - dump_data_dir = full_path - dump_data_dirname = fname - elif full_path.endswith('.pkl'): - pkl_path = full_path - pkl_name = fname + if full_path.endswith('.json'): + json_path = full_path + if not stack_json and 'stack' not in json_path: + break + if stack_json and 'stack' in json_path: + break + # Provide robustness on invalid directory inputs - if not pkl_path: + if not json_path: print_error_log(f'No file is found in dump dir {dirname}. ') raise CompareException(CompareException.NO_DUMP_FILE_ERROR) - name_body, ext = os.path.splitext(pkl_name) - pattern = re.compile(f'{name_body}$') - match = pattern.match(dump_data_dirname) - if dump_data_dir and match is None: - print_error_log('The names of pkl and directory do not match! ' - f'Please check the names and remove irrelevant files in {dirname}. ') - raise CompareException(CompareException.INVALID_FILE_ERROR) - return pkl_path, dump_data_dir + return json_path if kwargs.get('suffix'): @@ -77,18 +72,19 @@ def compare_distributed(npu_dump_dir, bench_dump_dir, output_path, **kwargs): for nr, br in zip(npu_ranks, bench_ranks): n_dir = os.path.join(npu_dump_dir, nr) b_dir = os.path.join(bench_dump_dir, br) - npu_pkl_path, npu_dump_data_dir = extract_pkl_and_data_dir(n_dir) - bench_pkl_path, bench_dump_data_dir = extract_pkl_and_data_dir(b_dir) + s_dir = b_dir + npu_json_path = extract_json(n_dir, stack_json=False) + bench_json_path = extract_json(b_dir, stack_json=False) + stack_json_path = extract_json(s_dir, stack_json=True) + dump_result_param = { - 'npu_pkl_path': npu_pkl_path, - 'bench_pkl_path': bench_pkl_path, - 'npu_dump_data_dir': npu_dump_data_dir, - 'bench_dump_data_dir': bench_dump_data_dir, + 'npu_json_path': npu_json_path, + 'bench_json_path': bench_json_path, + 'stack_json_path': stack_json_path, 'is_print_compare_log': True } try: - summary_compare = is_summary_compare(dump_result_param) - md5_compare = is_md5_compare(dump_result_param) + summary_compare, md5_compare = task_dumppath_get(dump_result_param) check_configuration_param(stack_mode, auto_analyze, fuzzy_match) check_compare_param(dump_result_param, output_path, stack_mode=stack_mode, summary_compare=summary_compare) except CompareException as error: -- Gitee From 916495eaa3bc841923d6b5beac3660d55dcb3d36 Mon Sep 17 00:00:00 2001 From: Linwei-Ying Date: Wed, 22 May 2024 16:35:44 +0800 Subject: [PATCH 2/7] =?UTF-8?q?=E5=88=A0=E9=99=A4=E8=80=81=E7=9A=84summary?= =?UTF-8?q?=E3=80=81md5=E5=88=A4=E8=AF=BB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/python/ptdbg_ascend/common/utils.py | 30 ------------------- .../ptdbg_ascend/compare/acc_compare.py | 2 +- 2 files changed, 1 insertion(+), 31 deletions(-) diff --git a/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/common/utils.py b/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/common/utils.py index 2ea2a7767..6e68f44c0 100644 --- a/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/common/utils.py +++ b/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/common/utils.py @@ -389,36 +389,6 @@ def check_compare_param(input_parma, output_path, stack_mode=False, summary_comp check_json_file(input_parma, npu_json, bench_json, stack_json, stack_mode) -def is_summary_compare(input_param): - npu_pkl_path = input_param.get("npu_pkl_path", None) - bench_pkl_path = input_param.get("bench_pkl_path", None) - npu_dump_data_dir = input_param.get("npu_dump_data_dir", None) - bench_dump_data_dir = input_param.get("bench_dump_data_dir", None) - if not npu_pkl_path or not bench_pkl_path: - print_error_log(f"Please check the pkl path is valid.") - raise CompareException(CompareException.INVALID_PATH_ERROR) - if not (npu_dump_data_dir and bench_dump_data_dir): - return True - if npu_dump_data_dir and bench_dump_data_dir: - return False - print_error_log(f"Please check the dump data dir is valid.") - raise CompareException(CompareException.INVALID_PATH_ERROR) - - -def is_md5_compare(input_parma): - with FileOpen(input_parma.get("npu_pkl_path"), "r") as npu_pkl: - pkl_lines = npu_pkl.readline() - try: - line = json.loads(pkl_lines) - except JSONDecodeError as err: - raise CompareException(CompareException.INVALID_FILE_ERROR) from err - if len(line) < 3: - return False - if line[2]: - return True - return False - - def md5_find(data): for key_op in data: for key_xxput in data[key_op]: diff --git a/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/compare/acc_compare.py b/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/compare/acc_compare.py index cedfaea0e..faa538653 100644 --- a/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/compare/acc_compare.py +++ b/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/compare/acc_compare.py @@ -30,7 +30,7 @@ from ..advisor.advisor import Advisor from ..common.utils import check_compare_param, add_time_as_suffix, \ print_info_log, print_warn_log, print_error_log, CompareException, Const, \ CompareConst, format_value, check_file_not_exists, check_configuration_param, \ - is_summary_compare, is_md5_compare, task_dumppath_get + task_dumppath_get from ..common.file_check_util import FileChecker, FileCheckConst, change_mode, FileOpen -- Gitee From c890c5f05f53019f8205d774548f4d0460dfc58f Mon Sep 17 00:00:00 2001 From: Linwei-Ying Date: Wed, 22 May 2024 17:00:04 +0800 Subject: [PATCH 3/7] =?UTF-8?q?=E5=88=A0=E9=99=A4=E8=80=81=E7=9A=84summary?= =?UTF-8?q?=E3=80=81md5=E5=88=A4=E8=AF=BB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ptdbg_ascend/src/python/ptdbg_ascend/common/utils.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/common/utils.py b/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/common/utils.py index 6e68f44c0..7e01619ba 100644 --- a/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/common/utils.py +++ b/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/common/utils.py @@ -432,8 +432,6 @@ def task_dumppath_get(input_param): input_param['npu_dump_data_dir'] = npu_json_data['dump_data_dir'] input_param['bench_dump_data_dir'] = bench_json_data['dump_data_dir'] return summary_compare, md5_compare - print_error_log(f"Please check the dump data dir is valid.") - raise CompareException(CompareException.INVALID_PATH_ERROR) def check_configuration_param(stack_mode=False, auto_analyze=True, fuzzy_match=False): -- Gitee From c618615b8bbb7a3e219ade54bf137b97fd4d72d0 Mon Sep 17 00:00:00 2001 From: Linwei-Ying Date: Wed, 22 May 2024 20:15:10 +0800 Subject: [PATCH 4/7] DT --- .../test/ut/compare/test_acc_compare.py | 90 ++++++++++--------- 1 file changed, 50 insertions(+), 40 deletions(-) diff --git a/debug/accuracy_tools/ptdbg_ascend/test/ut/compare/test_acc_compare.py b/debug/accuracy_tools/ptdbg_ascend/test/ut/compare/test_acc_compare.py index d51dbfb93..4271fe850 100644 --- a/debug/accuracy_tools/ptdbg_ascend/test/ut/compare/test_acc_compare.py +++ b/debug/accuracy_tools/ptdbg_ascend/test/ut/compare/test_acc_compare.py @@ -6,24 +6,33 @@ from ptdbg_ascend.compare import acc_compare as compare from ptdbg_ascend.common.utils import CompareConst -npu_dict = {'op_name': ['Functional_conv2d_0_forward_input.0', 'Functional_conv2d_0_forward_input.1', 'Functional_conv2d_0_forward_input.2', 'Functional_conv2d_0_forward_output'],\ - 'input_struct': [('torch.float32', [1, 1, 28, 28]), ('torch.float32', [16, 1, 5, 5]), ('torch.float32', [16])],\ - 'output_struct': [('torch.float32', [1, 16, 28, 28])], 'summery': [[3.029174327850342, -2.926689624786377, -0.06619918346405029], \ - [0.19919930398464203, -0.19974489510059357, 0.006269412115216255], [0.19734230637550354, -0.18177609145641327, 0.007903944700956345], [2.1166646480560303, -2.190781354904175, -0.003579073818400502]], 'stack_info': []} -bench_dict = {'op_name': ['Functional_conv2d_0_forward_input.0', 'Functional_conv2d_0_forward_input.1', 'Functional_conv2d_0_forward_input.2', 'Functional_conv2d_0_forward_output'],\ - 'input_struct': [('torch.float32', [1, 1, 28, 28]), ('torch.float32', [16, 1, 5, 5]), ('torch.float32', [16])],\ - 'output_struct': [('torch.float32', [1, 16, 28, 28])], 'summery': [[3.029174327850342, -2.926689624786377, -0.06619918346405029], \ - [0.19919930398464203, -0.19974489510059357, 0.006269412115216255], [0.19734230637550354, -0.18177609145641327, 0.007903944700956345], [2.1166646480560303, -2.190781354904175, -0.003579073818400502]], 'stack_info': []} -tensor_list = [['Functional_conv2d_0_forward_input.0', 1, [], 'torch.float32', [1, 1, 28, 28], [3.029174327850342, -2.926689624786377, -0.06619918346405029]],\ - ['Functional_conv2d_0_forward_input.1', 1, [], 'torch.float32', [16, 1, 5, 5], [0.19919930398464203, -0.19974489510059357, 0.006269412115216255]], \ - ['Functional_conv2d_0_forward_input.2', 1, [], 'torch.float32', [16], [0.19734230637550354, -0.18177609145641327, 0.007903944700956345]],\ - ['Functional_conv2d_0_forward_output', 1, [], 'torch.float32', [1, 16, 28, 28], [2.1166646480560303, -2.190781354904175, -0.003579073818400502]]] -result_op_dict = {'op_name': ['Functional_conv2d_0_forward_input.0', 'Functional_conv2d_0_forward_input.1', 'Functional_conv2d_0_forward_input.2', 'Functional_conv2d_0_forward_output'], \ -'input_struct': [('torch.float32', [1, 1, 28, 28], []), ('torch.float32', [16, 1, 5, 5], []), ('torch.float32', [16], [])], \ -'output_struct': [('torch.float32', [1, 16, 28, 28], [])], 'summery': [[3.029174327850342, -2.926689624786377, -0.06619918346405029], [0.19919930398464203, -0.19974489510059357, 0.006269412115216255], \ -[0.19734230637550354, -0.18177609145641327, 0.007903944700956345], [2.1166646480560303, -2.190781354904175, -0.003579073818400502]], 'stack_info': []} - -o_result = [['Functional_conv2d_0_forward_input.0', 'Functional_conv2d_0_forward_input.0', 'torch.float32', 'torch.float32', [1, 1, 28, 28], [1, 1, 28, 28], ' ', ' ', ' ', ' ', ' ', 3.029174327850342, -2.926689624786377, -0.06619918346405029, 3.029174327850342, -2.926689624786377, -0.06619918346405029, 'Yes', ''], ['Functional_conv2d_0_forward_input.1', 'Functional_conv2d_0_forward_input.1', 'torch.float32', 'torch.float32', [16, 1, 5, 5], [16, 1, 5, 5], ' ', ' ', ' ', ' ', ' ', 0.19919930398464203, -0.19974489510059357, 0.006269412115216255, 0.19919930398464203, -0.19974489510059357, 0.006269412115216255, 'Yes', ''], ['Functional_conv2d_0_forward_input.2', 'Functional_conv2d_0_forward_input.2', 'torch.float32', 'torch.float32', [16], [16], ' ', ' ', ' ', ' ', ' ', 0.19734230637550354, -0.18177609145641327, 0.007903944700956345, 0.19734230637550354, -0.18177609145641327, 0.007903944700956345, 'Yes', ''], ['Functional_conv2d_0_forward_output', 'Functional_conv2d_0_forward_output', 'torch.float32', 'torch.float32', [1, 16, 28, 28], [1, 16, 28, 28], ' ', ' ', ' ', ' ', ' ', 2.1166646480560303, -2.190781354904175, -0.003579073818400502, 2.1166646480560303, -2.190781354904175, -0.003579073818400502, 'Yes', '']] +# npu_dict = {'op_name': ['Functional_conv2d_0_forward_input.0', 'Functional_conv2d_0_forward_input.1', 'Functional_conv2d_0_forward_input.2', 'Functional_conv2d_0_forward_output'],\ +# 'input_struct': [('torch.float32', [1, 1, 28, 28]), ('torch.float32', [16, 1, 5, 5]), ('torch.float32', [16])],\ +# 'output_struct': [('torch.float32', [1, 16, 28, 28])], 'summery': [[3.029174327850342, -2.926689624786377, -0.06619918346405029], \ +# [0.19919930398464203, -0.19974489510059357, 0.006269412115216255], [0.19734230637550354, -0.18177609145641327, 0.007903944700956345], [2.1166646480560303, -2.190781354904175, -0.003579073818400502]], 'stack_info': []} +# bench_dict = {'op_name': ['Functional_conv2d_0_forward_input.0', 'Functional_conv2d_0_forward_input.1', 'Functional_conv2d_0_forward_input.2', 'Functional_conv2d_0_forward_output'],\ +# 'input_struct': [('torch.float32', [1, 1, 28, 28]), ('torch.float32', [16, 1, 5, 5]), ('torch.float32', [16])],\ +# 'output_struct': [('torch.float32', [1, 16, 28, 28])], 'summery': [[3.029174327850342, -2.926689624786377, -0.06619918346405029], \ +# [0.19919930398464203, -0.19974489510059357, 0.006269412115216255], [0.19734230637550354, -0.18177609145641327, 0.007903944700956345], [2.1166646480560303, -2.190781354904175, -0.003579073818400502]], 'stack_info': []} +# tensor_list = [['Functional_conv2d_0_forward_input.0', 1, [], 'torch.float32', [1, 1, 28, 28], [3.029174327850342, -2.926689624786377, -0.06619918346405029]],\ +# ['Functional_conv2d_0_forward_input.1', 1, [], 'torch.float32', [16, 1, 5, 5], [0.19919930398464203, -0.19974489510059357, 0.006269412115216255]], \ +# ['Functional_conv2d_0_forward_input.2', 1, [], 'torch.float32', [16], [0.19734230637550354, -0.18177609145641327, 0.007903944700956345]],\ +# ['Functional_conv2d_0_forward_output', 1, [], 'torch.float32', [1, 16, 28, 28], [2.1166646480560303, -2.190781354904175, -0.003579073818400502]]] +# result_op_dict = {'op_name': ['Functional_conv2d_0_forward_input.0', 'Functional_conv2d_0_forward_input.1', 'Functional_conv2d_0_forward_input.2', 'Functional_conv2d_0_forward_output'], \ +# 'input_struct': [('torch.float32', [1, 1, 28, 28], []), ('torch.float32', [16, 1, 5, 5], []), ('torch.float32', [16], [])], \ +# 'output_struct': [('torch.float32', [1, 16, 28, 28], [])], 'summery': [[3.029174327850342, -2.926689624786377, -0.06619918346405029], [0.19919930398464203, -0.19974489510059357, 0.006269412115216255], \ +# [0.19734230637550354, -0.18177609145641327, 0.007903944700956345], [2.1166646480560303, -2.190781354904175, -0.003579073818400502]], 'stack_info': []} +# +# o_result = [['Functional_conv2d_0_forward_input.0', 'Functional_conv2d_0_forward_input.0', 'torch.float32', 'torch.float32', [1, 1, 28, 28], [1, 1, 28, 28], ' ', ' ', ' ', ' ', ' ', 3.029174327850342, -2.926689624786377, -0.06619918346405029, 3.029174327850342, -2.926689624786377, -0.06619918346405029, 'Yes', ''], ['Functional_conv2d_0_forward_input.1', 'Functional_conv2d_0_forward_input.1', 'torch.float32', 'torch.float32', [16, 1, 5, 5], [16, 1, 5, 5], ' ', ' ', ' ', ' ', ' ', 0.19919930398464203, -0.19974489510059357, 0.006269412115216255, 0.19919930398464203, -0.19974489510059357, 0.006269412115216255, 'Yes', ''], ['Functional_conv2d_0_forward_input.2', 'Functional_conv2d_0_forward_input.2', 'torch.float32', 'torch.float32', [16], [16], ' ', ' ', ' ', ' ', ' ', 0.19734230637550354, -0.18177609145641327, 0.007903944700956345, 0.19734230637550354, -0.18177609145641327, 0.007903944700956345, 'Yes', ''], ['Functional_conv2d_0_forward_output', 'Functional_conv2d_0_forward_output', 'torch.float32', 'torch.float32', [1, 16, 28, 28], [1, 16, 28, 28], ' ', ' ', ' ', ' ', ' ', 2.1166646480560303, -2.190781354904175, -0.003579073818400502, 2.1166646480560303, -2.190781354904175, -0.003579073818400502, 'Yes', '']] + +npu_dict = {'op_name': ['modulemodel.linear.Linear.0.forward_input.0', 'modulemodel.linear.Linear.0.forward_input.1', 'modulemodel.linear.Linear.0.forward_input.2.0', 'modulemodel.linear.Linear.0.forward_input.2.1', 'modulemodel.linear.Linear.0.forward_output.0'], 'input_struct': [('torch.float32', [10, 10], '7f84caad'), (None, None, None), ("", '[]', None), ("", '[]', None)], 'output_struct': [('torch.float32', [10, 10], '3e8354f5')], 'summery': [[2.8386683464050293, -2.158618688583374, 0.11464785784482956, 10.07983684539795], [None, None, None, None], [2, 2, 2, 2], [2, 2, 2, 2], [1.1663073301315308, -1.6045000553131104, -0.1430426388978958, 6.108779430389404]], 'stack_info': [['File run_sample.py, line 11, in forward, \n return self.relu(self.linear(x))', 'File /home/louyujing/miniconda3/envs/pytorch21/lib/python3.8/site-packages/torch/nn/modules/module.py, line 1568, in _call_impl, \n result = forward_call(*args, **kwargs)', 'File /home/louyujing/miniconda3/envs/pytorch21/lib/python3.8/site-packages/torch/nn/modules/module.py, line 1518, in _wrapped_call_impl, \n return self._call_impl(*args, **kwargs)', 'File run_sample.py, line 21, in forward, \n return self.linear(self.model(x))', 'File /home/louyujing/miniconda3/envs/pytorch21/lib/python3.8/site-packages/torch/nn/modules/module.py, line 1528, in _call_impl, \n return forward_call(*args, **kwargs)', 'File /home/louyujing/miniconda3/envs/pytorch21/lib/python3.8/site-packages/torch/nn/modules/module.py, line 1518, in _wrapped_call_impl, \n return self._call_impl(*args, **kwargs)', 'File run_sample.py, line 30, in , \n y = model(x)']]} +bench_dict = {'op_name': ['modulemodel.linear.Linear.0.forward_input.0', 'modulemodel.linear.Linear.0.forward_input.1', 'modulemodel.linear.Linear.0.forward_input.2.0', 'modulemodel.linear.Linear.0.forward_input.2.1', 'modulemodel.linear.Linear.0.forward_output.0'], 'input_struct': [('torch.float32', [10, 10], '7f84caad'), (None, None, None), ("", '[]', None), ("", '[]', None)], 'output_struct': [('torch.float32', [10, 10], '3e8354f5')], 'summery': [[2.8386683464050293, -2.158618688583374, 0.11464785784482956, 10.07983684539795], [None, None, None, None], [2, 2, 2, 2], [2, 2, 2, 2], [1.1663073301315308, -1.6045000553131104, -0.1430426388978958, 6.108779430389404]], 'stack_info': [['File run_sample.py, line 11, in forward, \n return self.relu(self.linear(x))', 'File /home/louyujing/miniconda3/envs/pytorch21/lib/python3.8/site-packages/torch/nn/modules/module.py, line 1568, in _call_impl, \n result = forward_call(*args, **kwargs)', 'File /home/louyujing/miniconda3/envs/pytorch21/lib/python3.8/site-packages/torch/nn/modules/module.py, line 1518, in _wrapped_call_impl, \n return self._call_impl(*args, **kwargs)', 'File run_sample.py, line 21, in forward, \n return self.linear(self.model(x))', 'File /home/louyujing/miniconda3/envs/pytorch21/lib/python3.8/site-packages/torch/nn/modules/module.py, line 1528, in _call_impl, \n return forward_call(*args, **kwargs)', 'File /home/louyujing/miniconda3/envs/pytorch21/lib/python3.8/site-packages/torch/nn/modules/module.py, line 1518, in _wrapped_call_impl, \n return self._call_impl(*args, **kwargs)', 'File run_sample.py, line 30, in , \n y = model(x)']]} +tensor_list = [{'type': 'torch.Tensor', 'dtype': 'torch.float32', 'shape': [10, 10], 'Max': 2.8386683464050293, 'Min': -2.158618688583374, 'Mean': 0.11464785784482956, 'Norm': 10.07983684539795, 'requires_grad': False, 'md5': '7f84caad', 'full_op_name': 'modulemodel.linear.Linear.0.forward_input.0'}, {'full_op_name': 'modulemodel.linear.Linear.0.forward_input.1', 'Max': None, 'Min': None, 'Mean': None, 'Norm': None, 'dtype': None, 'shape': None, 'md5': None, 'data_name': '-1'}, {'full_op_name': 'modulemodel.linear.Linear.0.forward_input.2.0', 'dtype': "", 'shape': '[]', 'md5': None, 'Max': 2, 'Min': 2, 'Mean': 2, 'Norm': 2, 'data_name': '-1'}, {'full_op_name': 'modulemodel.linear.Linear.0.forward_input.2.1', 'dtype': "", 'shape': '[]', 'md5': None, 'Max': 2, 'Min': 2, 'Mean': 2, 'Norm': 2, 'data_name': '-1'}, {'type': 'torch.Tensor', 'dtype': 'torch.float32', 'shape': [10, 10], 'Max': 1.1663073301315308, 'Min': -1.6045000553131104, 'Mean': -0.1430426388978958, 'Norm': 6.108779430389404, 'requires_grad': True, 'md5': '3e8354f5', 'full_op_name': 'modulemodel.linear.Linear.0.forward_output.0'}, {'full_op_name': 'modulemodel.linear.Linear.0.forward', 'full_info': ['File run_sample.py, line 11, in forward, \n return self.relu(self.linear(x))', 'File /home/louyujing/miniconda3/envs/pytorch21/lib/python3.8/site-packages/torch/nn/modules/module.py, line 1568, in _call_impl, \n result = forward_call(*args, **kwargs)', 'File /home/louyujing/miniconda3/envs/pytorch21/lib/python3.8/site-packages/torch/nn/modules/module.py, line 1518, in _wrapped_call_impl, \n return self._call_impl(*args, **kwargs)', 'File run_sample.py, line 21, in forward, \n return self.linear(self.model(x))', 'File /home/louyujing/miniconda3/envs/pytorch21/lib/python3.8/site-packages/torch/nn/modules/module.py, line 1528, in _call_impl, \n return forward_call(*args, **kwargs)', 'File /home/louyujing/miniconda3/envs/pytorch21/lib/python3.8/site-packages/torch/nn/modules/module.py, line 1518, in _wrapped_call_impl, \n return self._call_impl(*args, **kwargs)', 'File run_sample.py, line 30, in , \n y = model(x)']}] +result_op_dict = {'op_name': ['modulemodel.linear.Linear.0.forward_input.0', 'modulemodel.linear.Linear.0.forward_input.1', 'modulemodel.linear.Linear.0.forward_input.2.0', 'modulemodel.linear.Linear.0.forward_input.2.1', 'modulemodel.linear.Linear.0.forward_output.0'], 'input_struct': [('torch.float32', [10, 10], '7f84caad'), (None, None, None), ("", '[]', None), ("", '[]', None)], 'output_struct': [('torch.float32', [10, 10], '3e8354f5')], 'summery': [[2.8386683464050293, -2.158618688583374, 0.11464785784482956, 10.07983684539795], [None, None, None, None], [2, 2, 2, 2], [2, 2, 2, 2], [1.1663073301315308, -1.6045000553131104, -0.1430426388978958, 6.108779430389404]], 'stack_info': [['File run_sample.py, line 11, in forward, \n return self.relu(self.linear(x))', 'File /home/louyujing/miniconda3/envs/pytorch21/lib/python3.8/site-packages/torch/nn/modules/module.py, line 1568, in _call_impl, \n result = forward_call(*args, **kwargs)', 'File /home/louyujing/miniconda3/envs/pytorch21/lib/python3.8/site-packages/torch/nn/modules/module.py, line 1518, in _wrapped_call_impl, \n return self._call_impl(*args, **kwargs)', 'File run_sample.py, line 21, in forward, \n return self.linear(self.model(x))', 'File /home/louyujing/miniconda3/envs/pytorch21/lib/python3.8/site-packages/torch/nn/modules/module.py, line 1528, in _call_impl, \n return forward_call(*args, **kwargs)', 'File /home/louyujing/miniconda3/envs/pytorch21/lib/python3.8/site-packages/torch/nn/modules/module.py, line 1518, in _wrapped_call_impl, \n return self._call_impl(*args, **kwargs)', 'File run_sample.py, line 30, in , \n y = model(x)']]} +o_result = [['modulemodel.linear.Linear.0.forward_input.0', 'modulemodel.linear.Linear.0.forward_input.0', 'torch.float32', 'torch.float32', [10, 10], [10, 10], '7f84caad', '7f84caad', 'Pass', ['File run_sample.py, line 11, in forward, \n return self.relu(self.linear(x))', 'File /home/louyujing/miniconda3/envs/pytorch21/lib/python3.8/site-packages/torch/nn/modules/module.py, line 1568, in _call_impl, \n result = forward_call(*args, **kwargs)', 'File /home/louyujing/miniconda3/envs/pytorch21/lib/python3.8/site-packages/torch/nn/modules/module.py, line 1518, in _wrapped_call_impl, \n return self._call_impl(*args, **kwargs)', 'File run_sample.py, line 21, in forward, \n return self.linear(self.model(x))', 'File /home/louyujing/miniconda3/envs/pytorch21/lib/python3.8/site-packages/torch/nn/modules/module.py, line 1528, in _call_impl, \n return forward_call(*args, **kwargs)', 'File /home/louyujing/miniconda3/envs/pytorch21/lib/python3.8/site-packages/torch/nn/modules/module.py, line 1518, in _wrapped_call_impl, \n return self._call_impl(*args, **kwargs)', 'File run_sample.py, line 30, in , \n y = model(x)']], ['modulemodel.linear.Linear.0.forward_input.1', 'modulemodel.linear.Linear.0.forward_input.1', None, None, None, None, None, None, 'Pass', 'None'], ['modulemodel.linear.Linear.0.forward_input.2.0', 'modulemodel.linear.Linear.0.forward_input.2.0', "", "", '[]', '[]', None, None, 'Pass', 'None'], ['modulemodel.linear.Linear.0.forward_input.2.1', 'modulemodel.linear.Linear.0.forward_input.2.1', "", "", '[]', '[]', None, None, 'Pass', 'None'], ['modulemodel.linear.Linear.0.forward_output.0', 'modulemodel.linear.Linear.0.forward_output.0', 'torch.float32', 'torch.float32', [10, 10], [10, 10], '3e8354f5', '3e8354f5', 'Pass', 'None']] +npuop_data = {'input_args': [{'type': 'torch.Tensor', 'dtype': 'torch.float32', 'shape': [10, 10], 'Max': 2.8386683464050293, 'Min': -2.158618688583374, 'Mean': 0.11464785784482956, 'Norm': 10.07983684539795, 'requires_grad': False, 'md5': '7f84caad'}, None, [{'type': 'int', 'value': 2}, {'type': 'int', 'value': 2}]], 'input_kwargs': {}, 'output': [{'type': 'torch.Tensor', 'dtype': 'torch.float32', 'shape': [10, 10], 'Max': 1.1663073301315308, 'Min': -1.6045000553131104, 'Mean': -0.1430426388978958, 'Norm': 6.108779430389404, 'requires_grad': True, 'md5': '3e8354f5'}]} +result_1 = [{'type': 'torch.Tensor', 'dtype': 'torch.float32', 'shape': [10, 10], 'Max': 2.8386683464050293, 'Min': -2.158618688583374, 'Mean': 0.11464785784482956, 'Norm': 10.07983684539795, 'requires_grad': False, 'md5': '7f84caad', 'full_op_name': 'modulemodel.linear.Linear.0.forward_input.0'}, {'full_op_name': 'modulemodel.linear.Linear.0.forward_input.1', 'Max': None, 'Min': None, 'Mean': None, 'Norm': None, 'dtype': None, 'shape': None, 'md5': None, 'data_name': '-1'}, {'full_op_name': 'modulemodel.linear.Linear.0.forward_input.2.0', 'dtype': "", 'shape': '[]', 'md5': None, 'Max': 2, 'Min': 2, 'Mean': 2, 'Norm': 2, 'data_name': '-1'}, {'full_op_name': 'modulemodel.linear.Linear.0.forward_input.2.1', 'dtype': "", 'shape': '[]', 'md5': None, 'Max': 2, 'Min': 2, 'Mean': 2, 'Norm': 2, 'data_name': '-1'}, {'type': 'torch.Tensor', 'dtype': 'torch.float32', 'shape': [10, 10], 'Max': 1.1663073301315308, 'Min': -1.6045000553131104, 'Mean': -0.1430426388978958, 'Norm': 6.108779430389404, 'requires_grad': True, 'md5': '3e8354f5', 'full_op_name': 'modulemodel.linear.Linear.0.forward_output.0'}] +aten_result = [['Aten__native_batch_norm_legit_functional.default_0_forward_input.0', 'Functional_batch_norm_0_forward_input.0', 'torch.float16', 'torch.float32', [256, 256, 14, 14], [256, 256, 14, 14], 136.56337118148804, -124.33742618560791, -0.010397066915174946, ' ', 139.625, -127.5625, -0.0103607177734375, 3.061628818511963, -3.22507381439209, 3.634914173744619e-05, 'Warning', 'Need double check api accuracy.', 'None'], ['Aten__native_batch_norm_legit_functional.default_0_forward_input.1', 'Functional_batch_norm_0_forward_input.1', 'torch.float32', 'torch.float32', [256], [256], 2.527024927258026, -2.1782388387364335, -0.0008296193100250093, ' ', 2.5276029109954834, -2.1788690090179443, -0.0008259844034910202, 0.0005779837374575436, -0.0006301702815108001, 3.634906533989124e-06, 'Warning', 'Need double check api accuracy.', 'None'], ['Aten__native_batch_norm_legit_functional.default_0_forward_input.2', 'Functional_batch_norm_0_forward_input.2', 'torch.float32', 'torch.float32', [256], [256], 1.5384095311164856, -3.7736878395080566, -0.9390918612480164, ' ', 2.472219944000244, -2.845968723297119, -0.008756577968597412, 0.9338104128837585, 0.9277191162109375, 0.930335283279419, 'Warning', 'Need double check api accuracy.', 'None'], ['Aten__native_batch_norm_legit_functional.default_0_forward_input.3', 'Functional_batch_norm_0_forward_input.3', 'torch.float32', 'torch.float32', [256], [256], 1.763145923614502, -4.398397922515869, -1.0521326325833797, ' ', 2.763145923614502, -3.398397922515869, -0.052132632583379745, 1.0, 1.0, 1.0, 'Warning', 'Need double check api accuracy.', 'None'], ['Aten__native_batch_norm_legit_functional.default_0_forward_input.4', 'Functional_batch_norm_0_forward_input.4', 'torch.float32', 'torch.float32', [256], [256], 2.673110008239746, -3.149275064468384, 0.01613386906683445, ' ', 2.673110008239746, -3.149275064468384, 0.01613386906683445, 0.0, 0.0, 0.0, 'Warning', 'Need double check api accuracy.', 'None'], ['Aten__native_batch_norm_legit_functional.default_0_forward_output.0', 'Functional_batch_norm_0_forward_output', 'torch.float16', 'torch.float32', [256, 256, 14, 14], [256, 256, 14, 14], 8.156781196594238, -4.843813419342041, -0.008758545174714527, ' ', 13.5546875, -10.640625, -0.008758544921875, 5.397906303405762, -5.796811580657959, 2.5283952709287405e-10, 'Warning', 'Need double check api accuracy.', 'None'], ['Aten__native_batch_norm_legit_functional.default_0_forward_output.1', 'Nan', 'torch.float32', 'Nan', [256], 'Nan', ' ', ' ', ' ', ' ', ' ', 0.30550330877304077, -0.24485322833061218, -0.010361209511756897, 'Nan', 'Nan', 'Nan', 'Yes', '', 'None'], ['Aten__native_batch_norm_legit_functional.default_0_forward_output.2', 'Nan', 'torch.float32', 'Nan', [256], 'Nan', ' ', ' ', ' ', ' ', ' ', 623.9192504882812, 432.96826171875, 520.2276611328125, 'Nan', 'Nan', 'Nan', 'Yes', '', 'None'], ['Aten__native_batch_norm_legit_functional.default_0_forward_output.3', 'Nan', 'torch.float32', 'Nan', [256], 'Nan', ' ', ' ', ' ', ' ', ' ', 2.4797861576080322, -3.055997371673584, -0.04795549064874649, 'Nan', 'Nan', 'Nan', 'Yes', '', 'None'], ['Aten__native_batch_norm_legit_functional.default_0_forward_output.4', 'Nan', 'torch.float32', 'Nan', [256], 'Nan', ' ', ' ', ' ', ' ', ' ', 61.7945556640625, 42.59713363647461, 52.03831481933594, 'Nan', 'Nan', 'Nan', 'Yes', '', 'None']] npu_dict_aten = {'op_name': ['Aten__native_batch_norm_legit_functional.default_0_forward_input.0', 'Aten__native_batch_norm_legit_functional.default_0_forward_input.1', @@ -61,17 +70,17 @@ bench_dict_functional = {'op_name': ['Functional_batch_norm_0_forward_input.0', [5.397906303405762, -5.796811580657959, 2.5283952709287405e-10]] } -aten_result = [['Aten__native_batch_norm_legit_functional.default_0_forward_input.0', 'Functional_batch_norm_0_forward_input.0', 'torch.float16', 'torch.float32', [256, 256, 14, 14], [256, 256, 14, 14], ' ', ' ', ' ', ' ', ' ', 139.625, -127.5625, -0.0103607177734375, 3.061628818511963, -3.22507381439209, 3.634914173744619e-05, 'Yes', ''], - ['Aten__native_batch_norm_legit_functional.default_0_forward_input.1', 'Functional_batch_norm_0_forward_input.1', 'torch.float32', 'torch.float32', [256], [256], ' ', ' ', ' ', ' ', ' ', 2.5276029109954834, -2.1788690090179443, -0.0008259844034910202, 0.0005779837374575436, -0.0006301702815108001, 3.634906533989124e-06, 'Yes', ''], - ['Aten__native_batch_norm_legit_functional.default_0_forward_input.2', 'Functional_batch_norm_0_forward_input.2', 'torch.float32', 'torch.float32', [256], [256], ' ', ' ', ' ', ' ', ' ', 2.472219944000244, -2.845968723297119, -0.008756577968597412, 0.9338104128837585, 0.9277191162109375, 0.930335283279419, 'Yes', ''], - ['Aten__native_batch_norm_legit_functional.default_0_forward_input.3', 'Functional_batch_norm_0_forward_input.3', 'torch.float32', 'torch.float32', [256], [256], ' ', ' ', ' ', ' ', ' ', 2.763145923614502, -3.398397922515869, -0.052132632583379745, 1.0, 1.0, 1.0, 'Yes', ''], - ['Aten__native_batch_norm_legit_functional.default_0_forward_input.4', 'Functional_batch_norm_0_forward_input.4', 'torch.float32', 'torch.float32', [256], [256], ' ', ' ', ' ', ' ', ' ', 2.673110008239746, -3.149275064468384, 0.01613386906683445, 0.0, 0.0, 0.0, 'Yes', ''], - ['Aten__native_batch_norm_legit_functional.default_0_forward_output.0', 'Functional_batch_norm_0_forward_output', 'torch.float16', 'torch.float32', [256, 256, 14, 14], [256, 256, 14, 14], ' ', ' ', ' ', ' ', ' ', 13.5546875, -10.640625, -0.008758544921875, 5.397906303405762, -5.796811580657959, 2.5283952709287405e-10, 'Yes', ''], - ['Aten__native_batch_norm_legit_functional.default_0_forward_output.1', 'Nan', 'torch.float32', 'Nan', [256], 'Nan', ' ', ' ', ' ', ' ', ' ', 0.30550330877304077, -0.24485322833061218, -0.010361209511756897, 'Nan', 'Nan', 'Nan', 'Yes', ''], - ['Aten__native_batch_norm_legit_functional.default_0_forward_output.2', 'Nan', 'torch.float32', 'Nan', [256], 'Nan', ' ', ' ', ' ', ' ', ' ', 623.9192504882812, 432.96826171875, 520.2276611328125, 'Nan', 'Nan', 'Nan', 'Yes', ''], - ['Aten__native_batch_norm_legit_functional.default_0_forward_output.3', 'Nan', 'torch.float32', 'Nan', [256], 'Nan', ' ', ' ', ' ', ' ', ' ', 2.4797861576080322, -3.055997371673584, -0.04795549064874649, 'Nan', 'Nan', 'Nan', 'Yes', ''], - ['Aten__native_batch_norm_legit_functional.default_0_forward_output.4', 'Nan', 'torch.float32', 'Nan', [256], 'Nan', ' ', ' ', ' ', ' ', ' ', 61.7945556640625, 42.59713363647461, 52.03831481933594, 'Nan', 'Nan', 'Nan', 'Yes', ''] - ] +# aten_result = [['Aten__native_batch_norm_legit_functional.default_0_forward_input.0', 'Functional_batch_norm_0_forward_input.0', 'torch.float16', 'torch.float32', [256, 256, 14, 14], [256, 256, 14, 14], ' ', ' ', ' ', ' ', ' ', 139.625, -127.5625, -0.0103607177734375, 3.061628818511963, -3.22507381439209, 3.634914173744619e-05, 'Yes', ''], +# ['Aten__native_batch_norm_legit_functional.default_0_forward_input.1', 'Functional_batch_norm_0_forward_input.1', 'torch.float32', 'torch.float32', [256], [256], ' ', ' ', ' ', ' ', ' ', 2.5276029109954834, -2.1788690090179443, -0.0008259844034910202, 0.0005779837374575436, -0.0006301702815108001, 3.634906533989124e-06, 'Yes', ''], +# ['Aten__native_batch_norm_legit_functional.default_0_forward_input.2', 'Functional_batch_norm_0_forward_input.2', 'torch.float32', 'torch.float32', [256], [256], ' ', ' ', ' ', ' ', ' ', 2.472219944000244, -2.845968723297119, -0.008756577968597412, 0.9338104128837585, 0.9277191162109375, 0.930335283279419, 'Yes', ''], +# ['Aten__native_batch_norm_legit_functional.default_0_forward_input.3', 'Functional_batch_norm_0_forward_input.3', 'torch.float32', 'torch.float32', [256], [256], ' ', ' ', ' ', ' ', ' ', 2.763145923614502, -3.398397922515869, -0.052132632583379745, 1.0, 1.0, 1.0, 'Yes', ''], +# ['Aten__native_batch_norm_legit_functional.default_0_forward_input.4', 'Functional_batch_norm_0_forward_input.4', 'torch.float32', 'torch.float32', [256], [256], ' ', ' ', ' ', ' ', ' ', 2.673110008239746, -3.149275064468384, 0.01613386906683445, 0.0, 0.0, 0.0, 'Yes', ''], +# ['Aten__native_batch_norm_legit_functional.default_0_forward_output.0', 'Functional_batch_norm_0_forward_output', 'torch.float16', 'torch.float32', [256, 256, 14, 14], [256, 256, 14, 14], ' ', ' ', ' ', ' ', ' ', 13.5546875, -10.640625, -0.008758544921875, 5.397906303405762, -5.796811580657959, 2.5283952709287405e-10, 'Yes', ''], +# ['Aten__native_batch_norm_legit_functional.default_0_forward_output.1', 'Nan', 'torch.float32', 'Nan', [256], 'Nan', ' ', ' ', ' ', ' ', ' ', 0.30550330877304077, -0.24485322833061218, -0.010361209511756897, 'Nan', 'Nan', 'Nan', 'Yes', ''], +# ['Aten__native_batch_norm_legit_functional.default_0_forward_output.2', 'Nan', 'torch.float32', 'Nan', [256], 'Nan', ' ', ' ', ' ', ' ', ' ', 623.9192504882812, 432.96826171875, 520.2276611328125, 'Nan', 'Nan', 'Nan', 'Yes', ''], +# ['Aten__native_batch_norm_legit_functional.default_0_forward_output.3', 'Nan', 'torch.float32', 'Nan', [256], 'Nan', ' ', ' ', ' ', ' ', ' ', 2.4797861576080322, -3.055997371673584, -0.04795549064874649, 'Nan', 'Nan', 'Nan', 'Yes', ''], +# ['Aten__native_batch_norm_legit_functional.default_0_forward_output.4', 'Nan', 'torch.float32', 'Nan', [256], 'Nan', ' ', ' ', ' ', ' ', ' ', 61.7945556640625, 42.59713363647461, 52.03831481933594, 'Nan', 'Nan', 'Nan', 'Yes', ''] +# ] class TestUtilsMethods(unittest.TestCase): @@ -164,19 +173,20 @@ class TestUtilsMethods(unittest.TestCase): self.assertEqual(result, True) def test_merge_tensor(self): - op_dict = compare.merge_tensor(tensor_list) + op_dict = compare.merge_tensor(tensor_list, False, True) self.assertEqual(op_dict, result_op_dict) def test_read_op(self): - base_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) - - pkl_dir = os.path.join(base_dir, "resources/compare/npu_test.pkl") - - npu_ops_queue = [] - npu_pkl_handle = open(pkl_dir, "r") - stack_mode = False - result = compare.read_op(npu_ops_queue, npu_pkl_handle, stack_mode) - self.assertEqual(result, True) + # base_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + # + # pkl_dir = os.path.join(base_dir, "resources/compare/npu_test.pkl") + # + # npu_ops_queue = [] + # npu_pkl_handle = open(pkl_dir, "r") + # stack_mode = False + opname_npu = 'modulemodle.linear.Linear.0.forward' + result = compare.read_op(npuop_data, opname_npu) + self.assertEqual(result, result_1) def test_match_op(self): @@ -187,7 +197,7 @@ class TestUtilsMethods(unittest.TestCase): def test_get_accuracy(self): result = [] - compare.get_accuracy(result, npu_dict, bench_dict) + compare.get_accuracy(result, npu_dict, bench_dict, False, True) self.assertEqual(result, o_result) -- Gitee From 2a0560bb5f372bac019dc876274c46a9e197126d Mon Sep 17 00:00:00 2001 From: Linwei-Ying Date: Wed, 22 May 2024 20:59:35 +0800 Subject: [PATCH 5/7] =?UTF-8?q?=E5=88=A0=E9=99=A4=E8=80=81=E7=9A=84summary?= =?UTF-8?q?=E3=80=81md5=E5=88=A4=E8=AF=BB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/python/ptdbg_ascend/common/utils.py | 19 ++++++------ .../ptdbg_ascend/compare/acc_compare.py | 31 +++++++++---------- .../compare/distributed_compare.py | 2 +- .../test/ut/compare/test_acc_compare.py | 9 +----- 4 files changed, 27 insertions(+), 34 deletions(-) diff --git a/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/common/utils.py b/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/common/utils.py index 7e01619ba..6424021b5 100644 --- a/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/common/utils.py +++ b/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/common/utils.py @@ -391,14 +391,15 @@ def check_compare_param(input_parma, output_path, stack_mode=False, summary_comp def md5_find(data): for key_op in data: - for key_xxput in data[key_op]: - if isinstance(data[key_op][key_xxput], list): - for i in range(len(data[key_op][key_xxput])): - if data[key_op][key_xxput][i] == None: + # data_key_op = data.get(key_op, None) + for api_info in data[key_op]: + if isinstance(data[key_op][api_info], list): + for i in range(len(data[key_op][api_info])): + if data[key_op][api_info][i] == None: continue - elif 'md5' in data[key_op][key_xxput][i]: + elif 'md5' in data[key_op][api_info][i]: return True - elif 'md5' in data[key_op][key_xxput]: + elif 'md5' in data[key_op][api_info]: return True return False @@ -409,9 +410,9 @@ def task_dumppath_get(input_param): if not npu_json_path or not bench_json_path: print_error_log(f"Please check the json path is valid.") raise CompareException(CompareException.INVALID_PATH_ERROR) - with open(npu_json_path, 'r') as npu_f: + with FileOpen(npu_json_path, 'r') as npu_f: npu_json_data = json.load(npu_f) - with open(bench_json_path, 'r') as bench_f: + with FileOpen(bench_json_path, 'r') as bench_f: bench_json_data = json.load(bench_f) if npu_json_data['task'] != bench_json_data['task']: print_error_log(f"Please check the dump task is consistent.") @@ -515,7 +516,7 @@ def check_pkl_file(input_param, npu_pkl, bench_pkl, stack_mode): raise CompareException(CompareException.INVALID_COMPARE_MODE) -def check_json_file(input_param, npu_json, bench_json, stack_json, stack_mode): +def check_json_file(input_param, npu_json, bench_json, stack_json): _check_json(npu_json, input_param.get("npu_json_path")) _check_json(bench_json, input_param.get("bench_json_path")) _check_json(stack_json, input_param.get("stack_json_path")) diff --git a/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/compare/acc_compare.py b/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/compare/acc_compare.py index faa538653..da9e5089e 100644 --- a/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/compare/acc_compare.py +++ b/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/compare/acc_compare.py @@ -340,7 +340,6 @@ def get_accuracy(result, n_dict, b_dict, summary_compare=False, md5_compare=Fals if magnitude_diff > 0.5: warning_flag = True else: - # result_item[start_idx + i] = CompareConst.NAN result_item[start_idx + i] = CompareConst.NONE accuracy_check = CompareConst.WARNING if warning_flag else "" err_msg += "Need double check api accuracy." if warning_flag else "" @@ -785,33 +784,33 @@ def compare_process(file_handles, stack_mode, fuzzy_match, summary_compare=False if not read_err_npu or not read_err_bench: break try: - opname_npu = next(ops_npu_iter) + op_name_npu = next(ops_npu_iter) read_err_npu = True - npuop_data = npu_json_data['data'][opname_npu] - npuop_parsed_list = read_op(npuop_data, opname_npu) - if opname_npu in stack_json_data: - npuop_parsed_list.append({'full_op_name': opname_npu, 'full_info': stack_json_data[opname_npu]}) + npu_op_data = npu_json_data['data'][op_name_npu] + npu_op_parsed_list = read_op(npu_op_data, op_name_npu) + if op_name_npu in stack_json_data: + npu_op_parsed_list.append({'full_op_name': op_name_npu, 'full_info': stack_json_data[op_name_npu]}) else: - npuop_parsed_list.append({'full_op_name': opname_npu, 'full_info': None}) + npu_op_parsed_list.append({'full_op_name': op_name_npu, 'full_info': None}) - npu_ops_queue.append(merge_tensor(npuop_parsed_list, summary_compare, md5_compare)) + npu_ops_queue.append(merge_tensor(npu_op_parsed_list, summary_compare, md5_compare)) except StopIteration: read_err_npu = False continue try: - opname_bench = next(ops_bench_iter) + op_name_bench = next(ops_bench_iter) read_err_bench = True - benchop_data = bench_json_data['data'][opname_bench] - benchop_parsed_list = read_op(benchop_data, opname_bench) - if opname_bench in stack_json_data: - benchop_parsed_list.append( - {'full_op_name': opname_bench, 'full_info': stack_json_data[opname_bench]}) + bench_op_data = bench_json_data['data'][op_name_bench] + bench_op_parsed_list = read_op(bench_op_data, op_name_bench) + if op_name_bench in stack_json_data: + bench_op_parsed_list.append( + {'full_op_name': op_name_bench, 'full_info': stack_json_data[op_name_bench]}) else: - benchop_parsed_list.append({'full_op_name': opname_bench, 'full_info': None}) + bench_op_parsed_list.append({'full_op_name': op_name_bench, 'full_info': None}) - bench_ops_queue.append(merge_tensor(benchop_parsed_list, summary_compare, md5_compare)) + bench_ops_queue.append(merge_tensor(bench_op_parsed_list, summary_compare, md5_compare)) except StopIteration: read_err_bench = False continue diff --git a/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/compare/distributed_compare.py b/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/compare/distributed_compare.py index 077bcd132..e6bce19a6 100644 --- a/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/compare/distributed_compare.py +++ b/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/compare/distributed_compare.py @@ -18,7 +18,7 @@ import os import sys import re from ..common.utils import print_error_log, CompareException, check_compare_param, check_file_or_directory_path, \ - check_configuration_param, is_summary_compare, is_md5_compare, task_dumppath_get + check_configuration_param, task_dumppath_get from .acc_compare import compare_core diff --git a/debug/accuracy_tools/ptdbg_ascend/test/ut/compare/test_acc_compare.py b/debug/accuracy_tools/ptdbg_ascend/test/ut/compare/test_acc_compare.py index 4271fe850..7ef2e1219 100644 --- a/debug/accuracy_tools/ptdbg_ascend/test/ut/compare/test_acc_compare.py +++ b/debug/accuracy_tools/ptdbg_ascend/test/ut/compare/test_acc_compare.py @@ -177,13 +177,6 @@ class TestUtilsMethods(unittest.TestCase): self.assertEqual(op_dict, result_op_dict) def test_read_op(self): - # base_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) - # - # pkl_dir = os.path.join(base_dir, "resources/compare/npu_test.pkl") - # - # npu_ops_queue = [] - # npu_pkl_handle = open(pkl_dir, "r") - # stack_mode = False opname_npu = 'modulemodle.linear.Linear.0.forward' result = compare.read_op(npuop_data, opname_npu) self.assertEqual(result, result_1) @@ -203,5 +196,5 @@ class TestUtilsMethods(unittest.TestCase): def test_get_accuracy_graph_mode(self): result = [] - compare.get_accuracy(result, npu_dict_aten, bench_dict_functional) + compare.get_accuracy(result, npu_dict_aten, bench_dict_functional, True, False) self.assertEqual(result, aten_result) -- Gitee From 32284dc16314468c7da392f82ccc10e3c2e216a2 Mon Sep 17 00:00:00 2001 From: Linwei-Ying Date: Wed, 22 May 2024 21:13:57 +0800 Subject: [PATCH 6/7] =?UTF-8?q?=E5=88=A0=E9=99=A4=E8=80=81=E7=9A=84summary?= =?UTF-8?q?=E3=80=81md5=E5=88=A4=E8=AF=BB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ptdbg_ascend/src/python/ptdbg_ascend/common/utils.py | 1 - debug/accuracy_tools/ptdbg_ascend/test/ut/test_utils.py | 6 +----- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/common/utils.py b/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/common/utils.py index 6424021b5..acf25a38e 100644 --- a/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/common/utils.py +++ b/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/common/utils.py @@ -391,7 +391,6 @@ def check_compare_param(input_parma, output_path, stack_mode=False, summary_comp def md5_find(data): for key_op in data: - # data_key_op = data.get(key_op, None) for api_info in data[key_op]: if isinstance(data[key_op][api_info], list): for i in range(len(data[key_op][api_info])): diff --git a/debug/accuracy_tools/ptdbg_ascend/test/ut/test_utils.py b/debug/accuracy_tools/ptdbg_ascend/test/ut/test_utils.py index 9ae980102..b55095423 100644 --- a/debug/accuracy_tools/ptdbg_ascend/test/ut/test_utils.py +++ b/debug/accuracy_tools/ptdbg_ascend/test/ut/test_utils.py @@ -3,7 +3,7 @@ import torch import pytest import ptdbg_ascend.common.utils as utils -from ptdbg_ascend.common.utils import CompareException, is_md5_compare, get_md5_for_tensor +from ptdbg_ascend.common.utils import CompareException, get_md5_for_tensor from ptdbg_ascend.common.file_check_util import FileCheckException @@ -32,10 +32,6 @@ class TestUtilsMethods(unittest.TestCase): utils.check_file_size(file, 0) self.assertEqual(error.value.code, CompareException.INVALID_FILE_ERROR) - def test_is_md5_compare(self): - input_param = {"npu_pkl_path": "resources/compare/npu_test.pkl"} - result = is_md5_compare(input_param) - self.assertFalse(result) def test_get_md5_for_tensor(self): data = [[1, 2], [3, 4]] -- Gitee From da100a711e7600c15dcba4366c10d54ce6deddf3 Mon Sep 17 00:00:00 2001 From: Linwei-Ying Date: Wed, 22 May 2024 21:20:18 +0800 Subject: [PATCH 7/7] =?UTF-8?q?=E5=88=A0=E9=99=A4=E8=80=81=E7=9A=84summary?= =?UTF-8?q?=E3=80=81md5=E5=88=A4=E8=AF=BB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../test/ut/compare/test_acc_compare.py | 37 ++----------------- 1 file changed, 3 insertions(+), 34 deletions(-) diff --git a/debug/accuracy_tools/ptdbg_ascend/test/ut/compare/test_acc_compare.py b/debug/accuracy_tools/ptdbg_ascend/test/ut/compare/test_acc_compare.py index 7ef2e1219..5580de579 100644 --- a/debug/accuracy_tools/ptdbg_ascend/test/ut/compare/test_acc_compare.py +++ b/debug/accuracy_tools/ptdbg_ascend/test/ut/compare/test_acc_compare.py @@ -6,31 +6,12 @@ from ptdbg_ascend.compare import acc_compare as compare from ptdbg_ascend.common.utils import CompareConst -# npu_dict = {'op_name': ['Functional_conv2d_0_forward_input.0', 'Functional_conv2d_0_forward_input.1', 'Functional_conv2d_0_forward_input.2', 'Functional_conv2d_0_forward_output'],\ -# 'input_struct': [('torch.float32', [1, 1, 28, 28]), ('torch.float32', [16, 1, 5, 5]), ('torch.float32', [16])],\ -# 'output_struct': [('torch.float32', [1, 16, 28, 28])], 'summery': [[3.029174327850342, -2.926689624786377, -0.06619918346405029], \ -# [0.19919930398464203, -0.19974489510059357, 0.006269412115216255], [0.19734230637550354, -0.18177609145641327, 0.007903944700956345], [2.1166646480560303, -2.190781354904175, -0.003579073818400502]], 'stack_info': []} -# bench_dict = {'op_name': ['Functional_conv2d_0_forward_input.0', 'Functional_conv2d_0_forward_input.1', 'Functional_conv2d_0_forward_input.2', 'Functional_conv2d_0_forward_output'],\ -# 'input_struct': [('torch.float32', [1, 1, 28, 28]), ('torch.float32', [16, 1, 5, 5]), ('torch.float32', [16])],\ -# 'output_struct': [('torch.float32', [1, 16, 28, 28])], 'summery': [[3.029174327850342, -2.926689624786377, -0.06619918346405029], \ -# [0.19919930398464203, -0.19974489510059357, 0.006269412115216255], [0.19734230637550354, -0.18177609145641327, 0.007903944700956345], [2.1166646480560303, -2.190781354904175, -0.003579073818400502]], 'stack_info': []} -# tensor_list = [['Functional_conv2d_0_forward_input.0', 1, [], 'torch.float32', [1, 1, 28, 28], [3.029174327850342, -2.926689624786377, -0.06619918346405029]],\ -# ['Functional_conv2d_0_forward_input.1', 1, [], 'torch.float32', [16, 1, 5, 5], [0.19919930398464203, -0.19974489510059357, 0.006269412115216255]], \ -# ['Functional_conv2d_0_forward_input.2', 1, [], 'torch.float32', [16], [0.19734230637550354, -0.18177609145641327, 0.007903944700956345]],\ -# ['Functional_conv2d_0_forward_output', 1, [], 'torch.float32', [1, 16, 28, 28], [2.1166646480560303, -2.190781354904175, -0.003579073818400502]]] -# result_op_dict = {'op_name': ['Functional_conv2d_0_forward_input.0', 'Functional_conv2d_0_forward_input.1', 'Functional_conv2d_0_forward_input.2', 'Functional_conv2d_0_forward_output'], \ -# 'input_struct': [('torch.float32', [1, 1, 28, 28], []), ('torch.float32', [16, 1, 5, 5], []), ('torch.float32', [16], [])], \ -# 'output_struct': [('torch.float32', [1, 16, 28, 28], [])], 'summery': [[3.029174327850342, -2.926689624786377, -0.06619918346405029], [0.19919930398464203, -0.19974489510059357, 0.006269412115216255], \ -# [0.19734230637550354, -0.18177609145641327, 0.007903944700956345], [2.1166646480560303, -2.190781354904175, -0.003579073818400502]], 'stack_info': []} -# -# o_result = [['Functional_conv2d_0_forward_input.0', 'Functional_conv2d_0_forward_input.0', 'torch.float32', 'torch.float32', [1, 1, 28, 28], [1, 1, 28, 28], ' ', ' ', ' ', ' ', ' ', 3.029174327850342, -2.926689624786377, -0.06619918346405029, 3.029174327850342, -2.926689624786377, -0.06619918346405029, 'Yes', ''], ['Functional_conv2d_0_forward_input.1', 'Functional_conv2d_0_forward_input.1', 'torch.float32', 'torch.float32', [16, 1, 5, 5], [16, 1, 5, 5], ' ', ' ', ' ', ' ', ' ', 0.19919930398464203, -0.19974489510059357, 0.006269412115216255, 0.19919930398464203, -0.19974489510059357, 0.006269412115216255, 'Yes', ''], ['Functional_conv2d_0_forward_input.2', 'Functional_conv2d_0_forward_input.2', 'torch.float32', 'torch.float32', [16], [16], ' ', ' ', ' ', ' ', ' ', 0.19734230637550354, -0.18177609145641327, 0.007903944700956345, 0.19734230637550354, -0.18177609145641327, 0.007903944700956345, 'Yes', ''], ['Functional_conv2d_0_forward_output', 'Functional_conv2d_0_forward_output', 'torch.float32', 'torch.float32', [1, 16, 28, 28], [1, 16, 28, 28], ' ', ' ', ' ', ' ', ' ', 2.1166646480560303, -2.190781354904175, -0.003579073818400502, 2.1166646480560303, -2.190781354904175, -0.003579073818400502, 'Yes', '']] - npu_dict = {'op_name': ['modulemodel.linear.Linear.0.forward_input.0', 'modulemodel.linear.Linear.0.forward_input.1', 'modulemodel.linear.Linear.0.forward_input.2.0', 'modulemodel.linear.Linear.0.forward_input.2.1', 'modulemodel.linear.Linear.0.forward_output.0'], 'input_struct': [('torch.float32', [10, 10], '7f84caad'), (None, None, None), ("", '[]', None), ("", '[]', None)], 'output_struct': [('torch.float32', [10, 10], '3e8354f5')], 'summery': [[2.8386683464050293, -2.158618688583374, 0.11464785784482956, 10.07983684539795], [None, None, None, None], [2, 2, 2, 2], [2, 2, 2, 2], [1.1663073301315308, -1.6045000553131104, -0.1430426388978958, 6.108779430389404]], 'stack_info': [['File run_sample.py, line 11, in forward, \n return self.relu(self.linear(x))', 'File /home/louyujing/miniconda3/envs/pytorch21/lib/python3.8/site-packages/torch/nn/modules/module.py, line 1568, in _call_impl, \n result = forward_call(*args, **kwargs)', 'File /home/louyujing/miniconda3/envs/pytorch21/lib/python3.8/site-packages/torch/nn/modules/module.py, line 1518, in _wrapped_call_impl, \n return self._call_impl(*args, **kwargs)', 'File run_sample.py, line 21, in forward, \n return self.linear(self.model(x))', 'File /home/louyujing/miniconda3/envs/pytorch21/lib/python3.8/site-packages/torch/nn/modules/module.py, line 1528, in _call_impl, \n return forward_call(*args, **kwargs)', 'File /home/louyujing/miniconda3/envs/pytorch21/lib/python3.8/site-packages/torch/nn/modules/module.py, line 1518, in _wrapped_call_impl, \n return self._call_impl(*args, **kwargs)', 'File run_sample.py, line 30, in , \n y = model(x)']]} bench_dict = {'op_name': ['modulemodel.linear.Linear.0.forward_input.0', 'modulemodel.linear.Linear.0.forward_input.1', 'modulemodel.linear.Linear.0.forward_input.2.0', 'modulemodel.linear.Linear.0.forward_input.2.1', 'modulemodel.linear.Linear.0.forward_output.0'], 'input_struct': [('torch.float32', [10, 10], '7f84caad'), (None, None, None), ("", '[]', None), ("", '[]', None)], 'output_struct': [('torch.float32', [10, 10], '3e8354f5')], 'summery': [[2.8386683464050293, -2.158618688583374, 0.11464785784482956, 10.07983684539795], [None, None, None, None], [2, 2, 2, 2], [2, 2, 2, 2], [1.1663073301315308, -1.6045000553131104, -0.1430426388978958, 6.108779430389404]], 'stack_info': [['File run_sample.py, line 11, in forward, \n return self.relu(self.linear(x))', 'File /home/louyujing/miniconda3/envs/pytorch21/lib/python3.8/site-packages/torch/nn/modules/module.py, line 1568, in _call_impl, \n result = forward_call(*args, **kwargs)', 'File /home/louyujing/miniconda3/envs/pytorch21/lib/python3.8/site-packages/torch/nn/modules/module.py, line 1518, in _wrapped_call_impl, \n return self._call_impl(*args, **kwargs)', 'File run_sample.py, line 21, in forward, \n return self.linear(self.model(x))', 'File /home/louyujing/miniconda3/envs/pytorch21/lib/python3.8/site-packages/torch/nn/modules/module.py, line 1528, in _call_impl, \n return forward_call(*args, **kwargs)', 'File /home/louyujing/miniconda3/envs/pytorch21/lib/python3.8/site-packages/torch/nn/modules/module.py, line 1518, in _wrapped_call_impl, \n return self._call_impl(*args, **kwargs)', 'File run_sample.py, line 30, in , \n y = model(x)']]} tensor_list = [{'type': 'torch.Tensor', 'dtype': 'torch.float32', 'shape': [10, 10], 'Max': 2.8386683464050293, 'Min': -2.158618688583374, 'Mean': 0.11464785784482956, 'Norm': 10.07983684539795, 'requires_grad': False, 'md5': '7f84caad', 'full_op_name': 'modulemodel.linear.Linear.0.forward_input.0'}, {'full_op_name': 'modulemodel.linear.Linear.0.forward_input.1', 'Max': None, 'Min': None, 'Mean': None, 'Norm': None, 'dtype': None, 'shape': None, 'md5': None, 'data_name': '-1'}, {'full_op_name': 'modulemodel.linear.Linear.0.forward_input.2.0', 'dtype': "", 'shape': '[]', 'md5': None, 'Max': 2, 'Min': 2, 'Mean': 2, 'Norm': 2, 'data_name': '-1'}, {'full_op_name': 'modulemodel.linear.Linear.0.forward_input.2.1', 'dtype': "", 'shape': '[]', 'md5': None, 'Max': 2, 'Min': 2, 'Mean': 2, 'Norm': 2, 'data_name': '-1'}, {'type': 'torch.Tensor', 'dtype': 'torch.float32', 'shape': [10, 10], 'Max': 1.1663073301315308, 'Min': -1.6045000553131104, 'Mean': -0.1430426388978958, 'Norm': 6.108779430389404, 'requires_grad': True, 'md5': '3e8354f5', 'full_op_name': 'modulemodel.linear.Linear.0.forward_output.0'}, {'full_op_name': 'modulemodel.linear.Linear.0.forward', 'full_info': ['File run_sample.py, line 11, in forward, \n return self.relu(self.linear(x))', 'File /home/louyujing/miniconda3/envs/pytorch21/lib/python3.8/site-packages/torch/nn/modules/module.py, line 1568, in _call_impl, \n result = forward_call(*args, **kwargs)', 'File /home/louyujing/miniconda3/envs/pytorch21/lib/python3.8/site-packages/torch/nn/modules/module.py, line 1518, in _wrapped_call_impl, \n return self._call_impl(*args, **kwargs)', 'File run_sample.py, line 21, in forward, \n return self.linear(self.model(x))', 'File /home/louyujing/miniconda3/envs/pytorch21/lib/python3.8/site-packages/torch/nn/modules/module.py, line 1528, in _call_impl, \n return forward_call(*args, **kwargs)', 'File /home/louyujing/miniconda3/envs/pytorch21/lib/python3.8/site-packages/torch/nn/modules/module.py, line 1518, in _wrapped_call_impl, \n return self._call_impl(*args, **kwargs)', 'File run_sample.py, line 30, in , \n y = model(x)']}] result_op_dict = {'op_name': ['modulemodel.linear.Linear.0.forward_input.0', 'modulemodel.linear.Linear.0.forward_input.1', 'modulemodel.linear.Linear.0.forward_input.2.0', 'modulemodel.linear.Linear.0.forward_input.2.1', 'modulemodel.linear.Linear.0.forward_output.0'], 'input_struct': [('torch.float32', [10, 10], '7f84caad'), (None, None, None), ("", '[]', None), ("", '[]', None)], 'output_struct': [('torch.float32', [10, 10], '3e8354f5')], 'summery': [[2.8386683464050293, -2.158618688583374, 0.11464785784482956, 10.07983684539795], [None, None, None, None], [2, 2, 2, 2], [2, 2, 2, 2], [1.1663073301315308, -1.6045000553131104, -0.1430426388978958, 6.108779430389404]], 'stack_info': [['File run_sample.py, line 11, in forward, \n return self.relu(self.linear(x))', 'File /home/louyujing/miniconda3/envs/pytorch21/lib/python3.8/site-packages/torch/nn/modules/module.py, line 1568, in _call_impl, \n result = forward_call(*args, **kwargs)', 'File /home/louyujing/miniconda3/envs/pytorch21/lib/python3.8/site-packages/torch/nn/modules/module.py, line 1518, in _wrapped_call_impl, \n return self._call_impl(*args, **kwargs)', 'File run_sample.py, line 21, in forward, \n return self.linear(self.model(x))', 'File /home/louyujing/miniconda3/envs/pytorch21/lib/python3.8/site-packages/torch/nn/modules/module.py, line 1528, in _call_impl, \n return forward_call(*args, **kwargs)', 'File /home/louyujing/miniconda3/envs/pytorch21/lib/python3.8/site-packages/torch/nn/modules/module.py, line 1518, in _wrapped_call_impl, \n return self._call_impl(*args, **kwargs)', 'File run_sample.py, line 30, in , \n y = model(x)']]} o_result = [['modulemodel.linear.Linear.0.forward_input.0', 'modulemodel.linear.Linear.0.forward_input.0', 'torch.float32', 'torch.float32', [10, 10], [10, 10], '7f84caad', '7f84caad', 'Pass', ['File run_sample.py, line 11, in forward, \n return self.relu(self.linear(x))', 'File /home/louyujing/miniconda3/envs/pytorch21/lib/python3.8/site-packages/torch/nn/modules/module.py, line 1568, in _call_impl, \n result = forward_call(*args, **kwargs)', 'File /home/louyujing/miniconda3/envs/pytorch21/lib/python3.8/site-packages/torch/nn/modules/module.py, line 1518, in _wrapped_call_impl, \n return self._call_impl(*args, **kwargs)', 'File run_sample.py, line 21, in forward, \n return self.linear(self.model(x))', 'File /home/louyujing/miniconda3/envs/pytorch21/lib/python3.8/site-packages/torch/nn/modules/module.py, line 1528, in _call_impl, \n return forward_call(*args, **kwargs)', 'File /home/louyujing/miniconda3/envs/pytorch21/lib/python3.8/site-packages/torch/nn/modules/module.py, line 1518, in _wrapped_call_impl, \n return self._call_impl(*args, **kwargs)', 'File run_sample.py, line 30, in , \n y = model(x)']], ['modulemodel.linear.Linear.0.forward_input.1', 'modulemodel.linear.Linear.0.forward_input.1', None, None, None, None, None, None, 'Pass', 'None'], ['modulemodel.linear.Linear.0.forward_input.2.0', 'modulemodel.linear.Linear.0.forward_input.2.0', "", "", '[]', '[]', None, None, 'Pass', 'None'], ['modulemodel.linear.Linear.0.forward_input.2.1', 'modulemodel.linear.Linear.0.forward_input.2.1', "", "", '[]', '[]', None, None, 'Pass', 'None'], ['modulemodel.linear.Linear.0.forward_output.0', 'modulemodel.linear.Linear.0.forward_output.0', 'torch.float32', 'torch.float32', [10, 10], [10, 10], '3e8354f5', '3e8354f5', 'Pass', 'None']] -npuop_data = {'input_args': [{'type': 'torch.Tensor', 'dtype': 'torch.float32', 'shape': [10, 10], 'Max': 2.8386683464050293, 'Min': -2.158618688583374, 'Mean': 0.11464785784482956, 'Norm': 10.07983684539795, 'requires_grad': False, 'md5': '7f84caad'}, None, [{'type': 'int', 'value': 2}, {'type': 'int', 'value': 2}]], 'input_kwargs': {}, 'output': [{'type': 'torch.Tensor', 'dtype': 'torch.float32', 'shape': [10, 10], 'Max': 1.1663073301315308, 'Min': -1.6045000553131104, 'Mean': -0.1430426388978958, 'Norm': 6.108779430389404, 'requires_grad': True, 'md5': '3e8354f5'}]} +npu_op_data = {'input_args': [{'type': 'torch.Tensor', 'dtype': 'torch.float32', 'shape': [10, 10], 'Max': 2.8386683464050293, 'Min': -2.158618688583374, 'Mean': 0.11464785784482956, 'Norm': 10.07983684539795, 'requires_grad': False, 'md5': '7f84caad'}, None, [{'type': 'int', 'value': 2}, {'type': 'int', 'value': 2}]], 'input_kwargs': {}, 'output': [{'type': 'torch.Tensor', 'dtype': 'torch.float32', 'shape': [10, 10], 'Max': 1.1663073301315308, 'Min': -1.6045000553131104, 'Mean': -0.1430426388978958, 'Norm': 6.108779430389404, 'requires_grad': True, 'md5': '3e8354f5'}]} result_1 = [{'type': 'torch.Tensor', 'dtype': 'torch.float32', 'shape': [10, 10], 'Max': 2.8386683464050293, 'Min': -2.158618688583374, 'Mean': 0.11464785784482956, 'Norm': 10.07983684539795, 'requires_grad': False, 'md5': '7f84caad', 'full_op_name': 'modulemodel.linear.Linear.0.forward_input.0'}, {'full_op_name': 'modulemodel.linear.Linear.0.forward_input.1', 'Max': None, 'Min': None, 'Mean': None, 'Norm': None, 'dtype': None, 'shape': None, 'md5': None, 'data_name': '-1'}, {'full_op_name': 'modulemodel.linear.Linear.0.forward_input.2.0', 'dtype': "", 'shape': '[]', 'md5': None, 'Max': 2, 'Min': 2, 'Mean': 2, 'Norm': 2, 'data_name': '-1'}, {'full_op_name': 'modulemodel.linear.Linear.0.forward_input.2.1', 'dtype': "", 'shape': '[]', 'md5': None, 'Max': 2, 'Min': 2, 'Mean': 2, 'Norm': 2, 'data_name': '-1'}, {'type': 'torch.Tensor', 'dtype': 'torch.float32', 'shape': [10, 10], 'Max': 1.1663073301315308, 'Min': -1.6045000553131104, 'Mean': -0.1430426388978958, 'Norm': 6.108779430389404, 'requires_grad': True, 'md5': '3e8354f5', 'full_op_name': 'modulemodel.linear.Linear.0.forward_output.0'}] aten_result = [['Aten__native_batch_norm_legit_functional.default_0_forward_input.0', 'Functional_batch_norm_0_forward_input.0', 'torch.float16', 'torch.float32', [256, 256, 14, 14], [256, 256, 14, 14], 136.56337118148804, -124.33742618560791, -0.010397066915174946, ' ', 139.625, -127.5625, -0.0103607177734375, 3.061628818511963, -3.22507381439209, 3.634914173744619e-05, 'Warning', 'Need double check api accuracy.', 'None'], ['Aten__native_batch_norm_legit_functional.default_0_forward_input.1', 'Functional_batch_norm_0_forward_input.1', 'torch.float32', 'torch.float32', [256], [256], 2.527024927258026, -2.1782388387364335, -0.0008296193100250093, ' ', 2.5276029109954834, -2.1788690090179443, -0.0008259844034910202, 0.0005779837374575436, -0.0006301702815108001, 3.634906533989124e-06, 'Warning', 'Need double check api accuracy.', 'None'], ['Aten__native_batch_norm_legit_functional.default_0_forward_input.2', 'Functional_batch_norm_0_forward_input.2', 'torch.float32', 'torch.float32', [256], [256], 1.5384095311164856, -3.7736878395080566, -0.9390918612480164, ' ', 2.472219944000244, -2.845968723297119, -0.008756577968597412, 0.9338104128837585, 0.9277191162109375, 0.930335283279419, 'Warning', 'Need double check api accuracy.', 'None'], ['Aten__native_batch_norm_legit_functional.default_0_forward_input.3', 'Functional_batch_norm_0_forward_input.3', 'torch.float32', 'torch.float32', [256], [256], 1.763145923614502, -4.398397922515869, -1.0521326325833797, ' ', 2.763145923614502, -3.398397922515869, -0.052132632583379745, 1.0, 1.0, 1.0, 'Warning', 'Need double check api accuracy.', 'None'], ['Aten__native_batch_norm_legit_functional.default_0_forward_input.4', 'Functional_batch_norm_0_forward_input.4', 'torch.float32', 'torch.float32', [256], [256], 2.673110008239746, -3.149275064468384, 0.01613386906683445, ' ', 2.673110008239746, -3.149275064468384, 0.01613386906683445, 0.0, 0.0, 0.0, 'Warning', 'Need double check api accuracy.', 'None'], ['Aten__native_batch_norm_legit_functional.default_0_forward_output.0', 'Functional_batch_norm_0_forward_output', 'torch.float16', 'torch.float32', [256, 256, 14, 14], [256, 256, 14, 14], 8.156781196594238, -4.843813419342041, -0.008758545174714527, ' ', 13.5546875, -10.640625, -0.008758544921875, 5.397906303405762, -5.796811580657959, 2.5283952709287405e-10, 'Warning', 'Need double check api accuracy.', 'None'], ['Aten__native_batch_norm_legit_functional.default_0_forward_output.1', 'Nan', 'torch.float32', 'Nan', [256], 'Nan', ' ', ' ', ' ', ' ', ' ', 0.30550330877304077, -0.24485322833061218, -0.010361209511756897, 'Nan', 'Nan', 'Nan', 'Yes', '', 'None'], ['Aten__native_batch_norm_legit_functional.default_0_forward_output.2', 'Nan', 'torch.float32', 'Nan', [256], 'Nan', ' ', ' ', ' ', ' ', ' ', 623.9192504882812, 432.96826171875, 520.2276611328125, 'Nan', 'Nan', 'Nan', 'Yes', '', 'None'], ['Aten__native_batch_norm_legit_functional.default_0_forward_output.3', 'Nan', 'torch.float32', 'Nan', [256], 'Nan', ' ', ' ', ' ', ' ', ' ', 2.4797861576080322, -3.055997371673584, -0.04795549064874649, 'Nan', 'Nan', 'Nan', 'Yes', '', 'None'], ['Aten__native_batch_norm_legit_functional.default_0_forward_output.4', 'Nan', 'torch.float32', 'Nan', [256], 'Nan', ' ', ' ', ' ', ' ', ' ', 61.7945556640625, 42.59713363647461, 52.03831481933594, 'Nan', 'Nan', 'Nan', 'Yes', '', 'None']] @@ -70,18 +51,6 @@ bench_dict_functional = {'op_name': ['Functional_batch_norm_0_forward_input.0', [5.397906303405762, -5.796811580657959, 2.5283952709287405e-10]] } -# aten_result = [['Aten__native_batch_norm_legit_functional.default_0_forward_input.0', 'Functional_batch_norm_0_forward_input.0', 'torch.float16', 'torch.float32', [256, 256, 14, 14], [256, 256, 14, 14], ' ', ' ', ' ', ' ', ' ', 139.625, -127.5625, -0.0103607177734375, 3.061628818511963, -3.22507381439209, 3.634914173744619e-05, 'Yes', ''], -# ['Aten__native_batch_norm_legit_functional.default_0_forward_input.1', 'Functional_batch_norm_0_forward_input.1', 'torch.float32', 'torch.float32', [256], [256], ' ', ' ', ' ', ' ', ' ', 2.5276029109954834, -2.1788690090179443, -0.0008259844034910202, 0.0005779837374575436, -0.0006301702815108001, 3.634906533989124e-06, 'Yes', ''], -# ['Aten__native_batch_norm_legit_functional.default_0_forward_input.2', 'Functional_batch_norm_0_forward_input.2', 'torch.float32', 'torch.float32', [256], [256], ' ', ' ', ' ', ' ', ' ', 2.472219944000244, -2.845968723297119, -0.008756577968597412, 0.9338104128837585, 0.9277191162109375, 0.930335283279419, 'Yes', ''], -# ['Aten__native_batch_norm_legit_functional.default_0_forward_input.3', 'Functional_batch_norm_0_forward_input.3', 'torch.float32', 'torch.float32', [256], [256], ' ', ' ', ' ', ' ', ' ', 2.763145923614502, -3.398397922515869, -0.052132632583379745, 1.0, 1.0, 1.0, 'Yes', ''], -# ['Aten__native_batch_norm_legit_functional.default_0_forward_input.4', 'Functional_batch_norm_0_forward_input.4', 'torch.float32', 'torch.float32', [256], [256], ' ', ' ', ' ', ' ', ' ', 2.673110008239746, -3.149275064468384, 0.01613386906683445, 0.0, 0.0, 0.0, 'Yes', ''], -# ['Aten__native_batch_norm_legit_functional.default_0_forward_output.0', 'Functional_batch_norm_0_forward_output', 'torch.float16', 'torch.float32', [256, 256, 14, 14], [256, 256, 14, 14], ' ', ' ', ' ', ' ', ' ', 13.5546875, -10.640625, -0.008758544921875, 5.397906303405762, -5.796811580657959, 2.5283952709287405e-10, 'Yes', ''], -# ['Aten__native_batch_norm_legit_functional.default_0_forward_output.1', 'Nan', 'torch.float32', 'Nan', [256], 'Nan', ' ', ' ', ' ', ' ', ' ', 0.30550330877304077, -0.24485322833061218, -0.010361209511756897, 'Nan', 'Nan', 'Nan', 'Yes', ''], -# ['Aten__native_batch_norm_legit_functional.default_0_forward_output.2', 'Nan', 'torch.float32', 'Nan', [256], 'Nan', ' ', ' ', ' ', ' ', ' ', 623.9192504882812, 432.96826171875, 520.2276611328125, 'Nan', 'Nan', 'Nan', 'Yes', ''], -# ['Aten__native_batch_norm_legit_functional.default_0_forward_output.3', 'Nan', 'torch.float32', 'Nan', [256], 'Nan', ' ', ' ', ' ', ' ', ' ', 2.4797861576080322, -3.055997371673584, -0.04795549064874649, 'Nan', 'Nan', 'Nan', 'Yes', ''], -# ['Aten__native_batch_norm_legit_functional.default_0_forward_output.4', 'Nan', 'torch.float32', 'Nan', [256], 'Nan', ' ', ' ', ' ', ' ', ' ', 61.7945556640625, 42.59713363647461, 52.03831481933594, 'Nan', 'Nan', 'Nan', 'Yes', ''] -# ] - class TestUtilsMethods(unittest.TestCase): def test_correct_data(self): @@ -177,8 +146,8 @@ class TestUtilsMethods(unittest.TestCase): self.assertEqual(op_dict, result_op_dict) def test_read_op(self): - opname_npu = 'modulemodle.linear.Linear.0.forward' - result = compare.read_op(npuop_data, opname_npu) + op_name_npu = 'modulemodel.linear.Linear.0.forward' + result = compare.read_op(npu_op_data, op_name_npu) self.assertEqual(result, result_1) -- Gitee