diff --git a/debug/accuracy_tools/msprobe/pytorch/online_dispatch/compare.py b/debug/accuracy_tools/msprobe/pytorch/online_dispatch/compare.py index 985e0f743cf30e69164fb084aaea2f88aa3bddd5..5755e1eccefc809a813e4ff0d257622797303cfa 100644 --- a/debug/accuracy_tools/msprobe/pytorch/online_dispatch/compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/online_dispatch/compare.py @@ -8,7 +8,7 @@ from rich.table import Table from rich.console import Console from msprobe.core.common.const import CompareConst, FileCheckConst from msprobe.core.common.file_check import FileOpen, change_mode -from .single_compare import single_benchmark_compare_wrap +from msprobe.pytorch.online_dispatch.single_compare import single_benchmark_compare_wrap from msprobe.pytorch.common.log import logger from msprobe.core.common.utils import CompareException diff --git a/debug/accuracy_tools/msprobe/pytorch/online_dispatch/dispatch.py b/debug/accuracy_tools/msprobe/pytorch/online_dispatch/dispatch.py index f634609504630f249a11289ce4817c723effd0f1..6faa152aae0855f1e3a0da81eefc5cb70f6fb1f2 100644 --- a/debug/accuracy_tools/msprobe/pytorch/online_dispatch/dispatch.py +++ b/debug/accuracy_tools/msprobe/pytorch/online_dispatch/dispatch.py @@ -1,8 +1,7 @@ import os import time import json -from pathlib import Path -from multiprocessing import Manager, Pool +from multiprocessing import Pool import torch @@ -15,13 +14,14 @@ except ImportError: else: is_npu = True -from msprobe.core.common.utils import check_file_or_directory_path, check_path_before_create, load_yaml +from msprobe.core.common.utils import check_file_or_directory_path, check_path_before_create, load_yaml, create_directory from msprobe.core.common.const import Const, CompareConst +from msprobe.core.common.file_check import FileOpen from msprobe.pytorch.common.log import logger -from .dump_compare import dispatch_workflow, dispatch_multiprocess, error_call, TimeStatistics, \ +from msprobe.pytorch.online_dispatch.dump_compare import dispatch_workflow, dispatch_multiprocess, error_call, TimeStatistics, \ DispatchRunParam, DisPatchDataInfo -from .utils import get_callstack, data_to_cpu, get_sys_info, DispatchException, COMPARE_LOGO -from .compare import Comparator +from msprobe.pytorch.online_dispatch.utils import get_callstack, data_to_cpu, get_sys_info, DispatchException, COMPARE_LOGO +from msprobe.pytorch.online_dispatch.compare import Comparator current_time = time.strftime("%Y%m%d%H%M%S") @@ -59,8 +59,8 @@ class PtdbgDispatch(TorchDispatchMode): self.root_npu_path = os.path.join(self.root_path, f'npu') check_path_before_create(self.root_cpu_path) check_path_before_create(self.root_npu_path) - Path(self.root_cpu_path).mkdir(mode=0o750, parents=True, exist_ok=True) - Path(self.root_npu_path).mkdir(mode=0o750, parents=True, exist_ok=True) + create_directory(self.root_cpu_path) + create_directory(self.root_npu_path) self.result_csv_path = os.path.join(self.root_path, RESULT_FILE_NAME) self.detail_csv_path = os.path.join(self.root_path, DETAILS_FILE_NAME) @@ -94,7 +94,7 @@ class PtdbgDispatch(TorchDispatchMode): logger.error("Please check train log, An exception may have occurred!") return check_file_or_directory_path(summary_path, False) - fp_handle = open(summary_path, "r") + fp_handle = FileOpen(summary_path, "r") while True: json_line_data = fp_handle.readline() if json_line_data == '\n': diff --git a/debug/accuracy_tools/msprobe/pytorch/online_dispatch/dump_compare.py b/debug/accuracy_tools/msprobe/pytorch/online_dispatch/dump_compare.py index 5b2e94e3ceb28f77e4afb98bc18d1b710d56e28b..b0f59ffa34cfde5d35eb762afabebe3e6c52cf77 100644 --- a/debug/accuracy_tools/msprobe/pytorch/online_dispatch/dump_compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/online_dispatch/dump_compare.py @@ -7,7 +7,7 @@ import pandas as pd import torch from msprobe.pytorch.common.log import logger from msprobe.core.common.file_check import FileOpen -from .utils import np_save_data +from msprobe.pytorch.online_dispatch.utils import np_save_data class DispatchRunParam: diff --git a/debug/accuracy_tools/msprobe/pytorch/online_dispatch/utils.py b/debug/accuracy_tools/msprobe/pytorch/online_dispatch/utils.py index 85786bf60a256af0c2fef6f1af1b5f9073cc5417..0f59f9827c1d604e07df67aaf45020589ba6a7da 100644 --- a/debug/accuracy_tools/msprobe/pytorch/online_dispatch/utils.py +++ b/debug/accuracy_tools/msprobe/pytorch/online_dispatch/utils.py @@ -11,9 +11,9 @@ except ImportError: else: pta_cpu_device = torch.device("cpu") -from msprobe.core.common.const import CompareConst, FileCheckConst -from msprobe.core.common.file_check import change_mode -from msprobe.pytorch.common.log import logger +from msprobe.core.common.const import CompareConst +from msprobe.core.common.utils import save_npy +from msprobe.core.common.log import logger cpu_device = torch._C.device("cpu") COLOR_RED = '\033[31m' @@ -74,8 +74,7 @@ def np_save_data(data, file_name, data_path): if hasattr(data, "numpy"): data = data.numpy() dump_path = os.path.join(data_path, f'{file_name}.npy') - np.save(dump_path, data) - change_mode(dump_path, FileCheckConst.DATA_FILE_AUTHORITY) + save_npy(data, dump_path) except Exception as e: logger.error("save numpy failed, error: {}".format(e)) finally: