diff --git a/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/dump/dump.py b/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/dump/dump.py index 4d330e26fa760f36e5b40e35145b1d8a67ec4d7d..f0e041a4a4515eef021afcd22f496ab07be40dbb 100644 --- a/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/dump/dump.py +++ b/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/dump/dump.py @@ -195,12 +195,8 @@ def rename_(): global rank global pkl_name if rank is not None and pkl_name is not None: - if DumpUtil.target_iter: - dir_name = os.path.join(DumpUtil.dump_root, "step{}".format(DumpUtil.iter_num), "rank{}".format(os.getpid())) - new_name = os.path.join(DumpUtil.dump_root, "step{}".format(DumpUtil.iter_num), "rank{}".format(rank)) - else: - dir_name = os.path.join(DumpUtil.dump_root, "rank{}".format(os.getpid())) - new_name = os.path.join(DumpUtil.dump_root, "rank{}".format(rank)) + dir_name = os.path.join(DumpUtil.dump_root, "step{}".format(DumpUtil.iter_num), "rank{}".format(os.getpid())) + new_name = os.path.join(DumpUtil.dump_root, "step{}".format(DumpUtil.iter_num), "rank{}".format(rank)) if not os.path.exists(new_name) and os.path.exists(dir_name): _, file_name = os.path.split(pkl_name) os.rename(dir_name, new_name) @@ -218,11 +214,10 @@ def dump_acc_cmp(name, in_feat, out_feat, dump_step, module): dump_file = modify_dump_path(dump_file, DumpUtil.dump_switch_mode) global rank dump_dir, dump_filename = os.path.split(dump_file) - if DumpUtil.target_iter: - dump_dir = os.path.join(dump_dir, "step{}".format(DumpUtil.iter_num)) - if not os.path.exists(dump_dir): - Path(dump_dir).mkdir(mode=FileCheckConst.DATA_DIR_AUTHORITY, exist_ok=True) - dump_file = os.path.join(dump_dir, dump_filename) + dump_dir = os.path.join(dump_dir, "step{}".format(DumpUtil.iter_num)) + if not os.path.exists(dump_dir): + Path(dump_dir).mkdir(mode=FileCheckConst.DATA_DIR_AUTHORITY, exist_ok=True) + dump_file = os.path.join(dump_dir, dump_filename) rank_this = get_tensor_rank(in_feat, out_feat) DumpUtil.dump_root = os.path.dirname(DumpUtil.dump_path) if rank_this is not None and rank != rank_this: @@ -233,10 +228,7 @@ def dump_acc_cmp(name, in_feat, out_feat, dump_step, module): npy_dir = dump_filename[:-4] else: npy_dir = dump_filename - if DumpUtil.target_iter: - DumpUtil.dump_data_dir = os.path.join(DumpUtil.dump_root, "step{}".format(DumpUtil.iter_num), "rank{}".format(rank), npy_dir) - else: - DumpUtil.dump_data_dir = os.path.join(DumpUtil.dump_root, "rank{}".format(rank), npy_dir) + DumpUtil.dump_data_dir = os.path.join(DumpUtil.dump_root, "step{}".format(DumpUtil.iter_num), "rank{}".format(rank), npy_dir) if DumpUtil.target_rank is not None: if rank != DumpUtil.target_rank: return diff --git a/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/overflow_check/overflow_check.py b/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/overflow_check/overflow_check.py index 10875743c76161f093c08d959d85b941c48b5115..afa0fea202659619c7df8f4f67e63fd819c48228 100644 --- a/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/overflow_check/overflow_check.py +++ b/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/overflow_check/overflow_check.py @@ -16,6 +16,7 @@ from .utils import OverFlowUtil, dump_overflow, check_overflow_npu, clear_overfl from ..dump.utils import DumpUtil, Const, get_tensor_rank, create_dirs_if_not_exist from .info_dump import write_api_info_json, ForwardAPIInfo, BackwardAPIInfo from ..dump import dump +from ..common.file_check_util import FileCheckConst backward_init_status = False api_overflow = [] @@ -91,12 +92,11 @@ def overflow_check(name, **kwargs): return dump_file = DumpUtil.get_dump_path() global rank - if DumpUtil.target_iter: - dump_dir, dump_filename = os.path.split(dump_file) - dump_dir = os.path.join(dump_dir, "step{}".format(DumpUtil.iter_num)) - if not os.path.exists(dump_dir): - Path(dump_dir).mkdir(mode=0o750, exist_ok=True) - dump_file = os.path.join(dump_dir, dump_filename) + dump_dir, dump_filename = os.path.split(dump_file) + dump_dir = os.path.join(dump_dir, "step{}".format(DumpUtil.iter_num)) + if not os.path.exists(dump_dir): + Path(dump_dir).mkdir(mode=FileCheckConst.DATA_DIR_AUTHORITY, exist_ok=True) + dump_file = os.path.join(dump_dir, dump_filename) rank_this = get_tensor_rank(in_feat, out_feat) DumpUtil.dump_root = os.path.dirname(DumpUtil.dump_path) if rank_this is not None and rank != rank_this: