diff --git a/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/common/utils.py b/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/common/utils.py index 942391d4b101f7012348cf7491d86f777f106eef..dde00e1009b1fef0e744b5221c5eae90e3a44dbf 100644 --- a/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/common/utils.py +++ b/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/common/utils.py @@ -23,6 +23,7 @@ import stat import subprocess import sys import time +import hashlib from datetime import datetime, timezone from functools import wraps from pathlib import Path @@ -743,3 +744,8 @@ def check_inplace_op(prefix): match_op = re.findall(r"Distributed_(.+?)_\d", prefix) op_name = match_op[0] if match_op else None return op_name in Const.INPLACE_LIST + +def get_md5_for_tensor(x): + tensor_bytes = x.cpu().detach().float().numpy().tobytes() + md5_hash = hashlib.md5(tensor_bytes) + return md5_hash.hexdigest() diff --git a/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/dump/dump.py b/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/dump/dump.py index f0e041a4a4515eef021afcd22f496ab07be40dbb..e2e6d49cb649977f4611207f42d90c3b8e34f06f 100644 --- a/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/dump/dump.py +++ b/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/dump/dump.py @@ -33,7 +33,7 @@ else: is_gpu = False from .utils import DumpUtil, check_if_in_api_list, make_dump_data_dir, get_tensor_rank, create_dirs_if_not_exist -from ..common.utils import print_warn_log, Const, print_info_log, modify_dump_path, check_inplace_op, CompareConst +from ..common.utils import print_warn_log, Const, print_info_log, modify_dump_path, check_inplace_op, CompareConst, get_md5_for_tensor from ..dump.utils import check_writable from ..common.file_check_util import FileOpen, change_mode, FileCheckConst, check_path_pattern_vaild, check_path_length @@ -69,7 +69,8 @@ def get_not_float_tensor_info(data): tensor_max = torch._C._VariableFunctionsClass.max(data).cpu().detach().float().numpy().tolist() tensor_min = torch._C._VariableFunctionsClass.min(data).cpu().detach().float().numpy().tolist() tensor_mean = torch._C._VariableFunctionsClass.mean(data.float()).cpu().detach().float().numpy().tolist() - return get_tensor_data_info(data, tensor_max, tensor_min, tensor_mean, CompareConst.NAN) + data_md5 = get_md5_for_tensor(data) + return get_tensor_data_info(data, tensor_max, tensor_min, tensor_mean, CompareConst.NAN, data_md5) def get_scalar_data_info(data): @@ -82,7 +83,8 @@ def get_float_tensor_info(data): tensor_min = torch._C._VariableFunctionsClass.min(data).cpu().detach().float().numpy().tolist() tensor_mean = torch._C._VariableFunctionsClass.mean(data).cpu().detach().float().numpy().tolist() tensor_norm = torch._C._VariableFunctionsClass.norm(data).cpu().detach().float().numpy().tolist() - return get_tensor_data_info(data, tensor_max, tensor_min, tensor_mean, tensor_norm) + data_md5 = get_md5_for_tensor(data) + return get_tensor_data_info(data, tensor_max, tensor_min, tensor_mean, tensor_norm, data_md5) def get_tensor_data_info(data, *tensor_args): diff --git a/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/parse_tool/lib/compare.py b/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/parse_tool/lib/compare.py index ba8692578620ca0e6a006bbee65f03efc77fb3f7..a4a76ae5d44a3cc99a08abef898b42306f73e5f1 100644 --- a/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/parse_tool/lib/compare.py +++ b/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/parse_tool/lib/compare.py @@ -16,6 +16,8 @@ """ import os +import time + import numpy as np from .utils import Util from .config import Const @@ -148,3 +150,101 @@ class Compare: err_percent = float(err_cnt / total_cnt) self.util.print(self.util.create_columns([err_table, top_table])) return total_cnt, all_close, cos_sim, err_percent + + + def compare_npy(self, file, bench_file, output_path): + data = np.load(file) + bench_data = np.load(bench_file) + shape, dtype = data.shape, data.dtype + bench_shape, bench_dtype = bench_data.shape, bench_data.dtype + filename = os.path.basename(file) + bench_filename = os.path. basename(bench_file) + if shape != bench_shape or dtype != bench_dtype: + return False + md5_consistency = False + if self.util.get_md5_for_numpy(data) and self.util.get_md5_for_numpy(bench_data): + md5_consistency = True + data_mean = np.mean(data) + abs_error = np.abs(data - bench_data) + rel_error = 0 + if not np.any(bench_data == 0): + rel_error = abs_error / bench_data + abs_diff_max = abs_error.max() + rel_diff_max = np.max(rel_error) + compare_result = [[filename, bench_filename, data_mean, md5_consistency, abs_diff_max, rel_diff_max]] + self.util.write_csv(compare_result,output_path) + return True + + def compare_all_file_in_directory(self, my_dump_dir, golden_dump_dir, output_path): + if self.util.is_subdir_count_equal(my_dump_dir, golden_dump_dir) and self.util.check_npy_files_valid_in_dir(my_dump_dir) and self.util.check_npy_files_valid_in_dir(golden_dump_dir): + my_npy_files = self.util.get_sorted_files_names(my_dump_dir) + golden_npy_files = self.util.get_sorted_files_names(golden_dump_dir) + for my_npy_file_name, golden_npy_file_name in zip(my_npy_files, golden_npy_files): + my_npy_path = os.path.join(my_dump_dir, my_npy_file_name) + golden_npy_path = os.path.join(golden_dump_dir, golden_npy_file_name) + if not self.compare_npy(my_npy_path, golden_npy_path, output_path): + if os.path.exists(output_path): + os.remove(output_path) + raise ParseException("Inconsistent shape or dtype") + else: + if os.path.exists(output_path): + os.remove(output_path) + raise ParseException("Inconsistent directory structure") + + def compare_timestamp_directory(self, my_dump_dir, golden_dump_dir, output_path): + if self.util.is_subdir_count_equal(my_dump_dir, golden_dump_dir): + my_ordered_subdirs = self.util.get_sorted_subdirectories_names(my_dump_dir) + golden_ordered_subdirs = self.util.get_sorted_subdirectories_names(golden_dump_dir) + for my_subdir_name, golden_subdir_name in zip(my_ordered_subdirs, golden_ordered_subdirs): + my_subdir_path = os.path.join(my_dump_dir, my_subdir_name) + golden_subdir_path = os.path.join(golden_dump_dir, golden_subdir_name) + self.compare_all_file_in_directory(my_subdir_path, golden_subdir_path, output_path) + else: + if os.path.exists(output_path): + os.remove(output_path) + raise ParseException("Inconsistent directory structure") + + def compare_converted_dir(self, my_dump_dir, golden_dump_dir, output_dir): + my_dump_dir = self.util.path_strip(my_dump_dir) + golden_dump_dir = self.util.path_strip(golden_dump_dir) + output_dir = self.util.path_strip(output_dir) + timestamp = int(time.time()) + output_file_name = f"batch_compare_{timestamp}.csv" + output_path = os.path.join(output_dir, output_file_name) + + title_rows = [[ + "NPU File Name", + "Bench File Name", + "Mean", + "Md5 Consistency", + "Max Abs Error", + "Max Relative Error" + ]] + self.util.write_csv(title_rows, output_path) + if self.util.is_subdir_count_equal(my_dump_dir, golden_dump_dir): + my_ordered_subdirs = self.util.get_sorted_subdirectories_names(my_dump_dir) + golden_ordered_subdirs = self.util.get_sorted_subdirectories_names(golden_dump_dir) + for my_subdir_name, golden_subdir_name in zip(my_ordered_subdirs, golden_ordered_subdirs): + if my_subdir_name == golden_subdir_name: + my_subdir_path = os.path.join(my_dump_dir, my_subdir_name) + golden_subdir_path = os.path.join(golden_dump_dir, golden_subdir_name) + self.compare_timestamp_directory(my_subdir_path, golden_subdir_path, output_path) + else: + if os.path.exists(output_path): + os.remove(output_path) + raise ParseException("Inconsistent acl dump files") + else: + if os.path.exists(output_path): + os.remove(output_path) + raise ParseException("Inconsistent directory structure") + + def convert_api_dir_to_npy(self, dump_dir, param, output_dir, msaccucmp_path): + dump_dir = self.util.path_strip(dump_dir) + for root, dirs, files in os.walk(dump_dir): + for file in files: + file_path = os.path.join(root, file) + file_name = os.path.basename(file_path) + op_name = file_name.split('.')[1] + timestamp = file_name.split('.')[-1] + output_path = os.path.join(output_dir, op_name, timestamp) + self.convert_dump_to_npy(file_path, param, output_path, msaccucmp_path) diff --git a/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/parse_tool/lib/config.py b/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/parse_tool/lib/config.py index 9cf479d64488497be8d0816f20cbc250b5ced831..ac1c6cb830d25db88e1295ee5bacc4390e0e87b8 100644 --- a/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/parse_tool/lib/config.py +++ b/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/parse_tool/lib/config.py @@ -27,6 +27,8 @@ class Const: DATA_ROOT_DIR = os.path.join(ROOT_DIR, 'parse_data') DUMP_CONVERT_DIR = os.path.join(DATA_ROOT_DIR, 'dump_convert') COMPARE_DIR = os.path.join(DATA_ROOT_DIR, 'compare_result') + BATCH_DUMP_CONVERT_DIR = os.path.join(DATA_ROOT_DIR, 'batch_dump_convert') + BATCH_COMPARE_DIR = os.path.join(DATA_ROOT_DIR, 'batch_compare_result') OFFLINE_DUMP_CONVERT_PATTERN = \ r"^([A-Za-z0-9_-]+)\.([A-Za-z0-9_-]+)\.([0-9]+)(\.[0-9]+)?\.([0-9]{1,255})" \ r"\.([a-z]+)\.([0-9]{1,255})(\.[x0-9]+)?\.npy$" diff --git a/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/parse_tool/lib/interactive_cli.py b/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/parse_tool/lib/interactive_cli.py index df1de148e21219361b39a9994c51df0e8fac77b5..4dbdf329c03398ff685ea405d225093f9afdbb99 100644 --- a/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/parse_tool/lib/interactive_cli.py +++ b/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/parse_tool/lib/interactive_cli.py @@ -65,3 +65,11 @@ class InteractiveCli(cmd.Cmd): def do_cn(self, line=''): self.parse_tool.do_compare_data(self._parse_argv(line)) + + def do_cad(self,line=''): + self.parse_tool.do_convert_api_dir(self._parse_argv(line)) + + def do_ccd(self, line=''): + self.parse_tool.do_compare_converted_dir(self._parse_argv(line)) + + diff --git a/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/parse_tool/lib/parse_tool.py b/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/parse_tool/lib/parse_tool.py index 8b587eb3999e9c6174a07cd1424ad97d2f5d437e..b22de5cd5e5d2eb4e4461130cc9caa14ffcb8300 100644 --- a/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/parse_tool/lib/parse_tool.py +++ b/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/parse_tool/lib/parse_tool.py @@ -139,3 +139,45 @@ class ParseTool: self.util.check_path_format(args.my_dump_path, Const.NPY_SUFFIX) self.util.check_path_format(args.golden_dump_path, Const.NPY_SUFFIX) self.compare.compare_data(args.my_dump_path, args.golden_dump_path, args.save, args.rtol, args.atol, args.count) + + @catch_exception + def do_compare_converted_dir(self, argv=None): + """compare two dir""" + parser = argparse.ArgumentParser() + parser.add_argument( + "-m", "--my_dump_path", dest="my_dump_path", default=None, + help=" my dump path, the data compared with golden data", + required=True + ) + parser.add_argument( + "-g", "--golden_dump_path", dest="golden_dump_path", default=None, + help=" the golden dump data path", + required=True + ) + parser.add_argument( + '-out', '--output_path', dest='output_path', required=False, default=None, help='output path') + args = parser.parse_args(argv) + self.util.check_path_valid(args.my_dump_path) + self.util.check_path_valid(args.golden_dump_path) + output_path = self.util.path_strip(args.output_path) if args.output_path else Const.BATCH_COMPARE_DIR + self.compare.compare_converted_dir(args.my_dump_path, args.golden_dump_path, output_path) + + @catch_exception + def do_convert_api_dir(self, argv=None): + parser = argparse.ArgumentParser() + parser.add_argument( + "-m", "--my_dump_path", dest="my_dump_path", default=None, + help=" my dump path, the data compared with golden data", + required=True + ) + parser.add_argument( + '-out', '--output_path', dest='output_path', required=False, default=None, help='output path') + parser.add_argument( + "-asc", "--msaccucmp_path", dest="msaccucmp_path", default=None, + help=" the msaccucmp.py file path", required=False) + args = parser.parse_args(argv) + self.util.check_path_valid(args.my_dump_path) + self.util.check_files_in_path(args.my_dump_path) + output_path = self.util.path_strip(args.output_path) if args.output_path else Const.BATCH_DUMP_CONVERT_DIR + msaccucmp_path = self.util.path_strip(args.msaccucmp_path) if args.msaccucmp_path else Const.MS_ACCU_CMP_PATH + self.compare.convert_api_dir_to_npy(args.my_dump_path, None, output_path, msaccucmp_path) diff --git a/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/parse_tool/lib/utils.py b/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/parse_tool/lib/utils.py index 2d512224f7e5d70173e0f716d37864be6687708f..213d5b6de973b6625e3764d94746b25c3d3dc242 100644 --- a/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/parse_tool/lib/utils.py +++ b/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/parse_tool/lib/utils.py @@ -16,9 +16,12 @@ """ import logging import os +import io import re import sys import subprocess +import hashlib +import csv import numpy as np from .config import Const from .file_desc import DumpDecodeFileDesc, FileDesc @@ -26,6 +29,7 @@ from .parse_exception import ParseException from ...common.file_check_util import change_mode, check_other_user_writable,\ check_path_executable, check_path_owner_consistent from ...common.file_check_util import FileCheckConst +from ...common.file_check_util import FileOpen try: from rich.traceback import install @@ -189,6 +193,7 @@ class Util: if path.endswith(Const.NPY_SUFFIX) and file_size > Const.TEN_GB: self.log.error('The file {} size is greater than 10GB.'.format(path)) raise ParseException(ParseException.PARSE_INVALID_PATH_ERROR) + return True def check_files_in_path(self, path): if os.path.isdir(path) and len(os.listdir(path)) == 0: @@ -251,3 +256,73 @@ class Util: if not re.match(Const.FILE_PATTERN, param): self.log.error('The parameter {} contains special characters.'.format(param)) raise ParseException(ParseException.PARSE_INVALID_PARAM_ERROR) + + def get_timestamp_from_str(self, str): + ts_pattern = r'\d+' + match = re.search(ts_pattern, str) + if match: + return int(match.group) + else: + return 0 + + def get_sorted_files_in_dir(self, directory): + files = os.listdir(directory) + sorted_files = sorted(files, key=lambda x: self.get_timestamp_from_str(x)) + return sorted_files + + def get_subdir_count(self, directory): + subdir_count = 0 + for root, dirs, files in os.walk(directory): + subdir_count += len(dirs) + break + return subdir_count + + def get_subfiles_count(self, directory): + file_count = 0 + for root, dirs, files in os.walk(directory): + file_count += len(files) + return file_count + + def is_subdir_count_equal(self, dir1, dir2): + dir1_count = self.get_subdir_count(dir1) + dir2_count = self.get_subdir_count(dir2) + if dir1_count == dir2_count: + return True + else: + return False + + def get_sorted_subdirectories_names(self, directory): + subdirectories = [] + for item in os.listdir(directory): + item_path = os.path.join(directory, item) + if os.path.isdir(item_path): + subdirectories.append(item) + return sorted(subdirectories) + + def get_sorted_files_names(self, directory): + files = [] + for item in os.listdir(directory): + item_path = os.path.join(directory,item) + if os.path.isfile(item_path): + files.append(item) + return sorted(files) + + def check_npy_files_valid_in_dir(self, dir_path): + for file_name in os.listdir(dir_path): + file_path = os.path.join(dir_path, file_name) + if not self.check_path_valid(file_path): + return False + _, file_extension = os.path.splitext(file_path) + if not file_extension == '.npy': + return False + return True + + def get_md5_for_numpy(self, obj): + np_bytes = obj.tobytes() + md5_hash = hashlib.md5(np_bytes) + return md5_hash.hexdigest() + + def write_csv(self, data, filepath): + with FileOpen(filepath, 'a') as f: + writer = csv.writer(f) + writer.writerows(data) \ No newline at end of file