diff --git a/debug/accuracy_tools/api_accuracy_checker/common/base_api.py b/debug/accuracy_tools/api_accuracy_checker/common/base_api.py index 5bcc0e78af7ad24ee7e0f73f7e3e5ff86648aa26..c08eed78e6755c2ef5b6611a818c3e1ee25a31a9 100644 --- a/debug/accuracy_tools/api_accuracy_checker/common/base_api.py +++ b/debug/accuracy_tools/api_accuracy_checker/common/base_api.py @@ -1,7 +1,7 @@ import os import torch from api_accuracy_checker.common.utils import print_error_log, write_pt, create_directory -from ptdbg_ascend.src.python.ptdbg_ascend.common.utils import check_path_before_create +from common.file_check_util import check_path_before_create class BaseAPIInfo: diff --git a/debug/accuracy_tools/api_accuracy_checker/common/config.py b/debug/accuracy_tools/api_accuracy_checker/common/config.py index f9b882f47b107a474b9f888e76fe94defd2b26fe..f90e3cdd57387072920cafb0a36a2e81bf8ebffc 100644 --- a/debug/accuracy_tools/api_accuracy_checker/common/config.py +++ b/debug/accuracy_tools/api_accuracy_checker/common/config.py @@ -2,7 +2,7 @@ import os import yaml from api_accuracy_checker.common.utils import check_file_or_directory_path from api_accuracy_checker.hook_module.utils import WrapFunctionalOps, WrapTensorOps, WrapTorchOps -from ptdbg_ascend.src.python.ptdbg_ascend.common.file_check_util import FileOpen +from common.file_check_util import FileOpen WrapApi = set(WrapFunctionalOps) | set(WrapTensorOps) | set(WrapTorchOps) diff --git a/debug/accuracy_tools/api_accuracy_checker/common/utils.py b/debug/accuracy_tools/api_accuracy_checker/common/utils.py index f8648eb20ccd7572c39a752db287e8b63a0e096c..027f83dbfb8709851071dfedc9b9dec0cee740b8 100644 --- a/debug/accuracy_tools/api_accuracy_checker/common/utils.py +++ b/debug/accuracy_tools/api_accuracy_checker/common/utils.py @@ -36,8 +36,8 @@ except ImportError: else: IS_GPU = False -from ptdbg_ascend.src.python.ptdbg_ascend.common.file_check_util import FileCheckConst, FileChecker, FileOpen -from ptdbg_ascend.src.python.ptdbg_ascend.common import file_check_util +from common.file_check_util import FileCheckConst, FileChecker, FileOpen +from common import file_check_util torch_without_guard_version_list = ['2.1'] for version in torch_without_guard_version_list: diff --git a/debug/accuracy_tools/api_accuracy_checker/dump/info_dump.py b/debug/accuracy_tools/api_accuracy_checker/dump/info_dump.py index 69de65912fbc2a0d0ca3ceabe4d15a4f7f65e0c5..dade146917f62ea6ea27e02cf763f8a1ea2dbbe8 100644 --- a/debug/accuracy_tools/api_accuracy_checker/dump/info_dump.py +++ b/debug/accuracy_tools/api_accuracy_checker/dump/info_dump.py @@ -5,11 +5,10 @@ import threading from api_accuracy_checker.dump.api_info import ForwardAPIInfo, BackwardAPIInfo from api_accuracy_checker.common.utils import check_file_or_directory_path, initialize_save_path, create_directory -from ptdbg_ascend.src.python.ptdbg_ascend.common.utils import check_path_before_create from api_accuracy_checker.common.config import msCheckerConfig -from ptdbg_ascend.src.python.ptdbg_ascend.common.file_check_util import FileOpen, FileCheckConst, FileChecker +from common.file_check_util import FileOpen, FileCheckConst, FileChecker, check_path_before_create lock = threading.Lock() diff --git a/debug/accuracy_tools/api_accuracy_checker/hook_module/utils.py b/debug/accuracy_tools/api_accuracy_checker/hook_module/utils.py index 7d16ac993ed45faa0f9b48bb64050592e15ef4d2..c81d747a1cb534d858a50648eeed28e55580f662 100644 --- a/debug/accuracy_tools/api_accuracy_checker/hook_module/utils.py +++ b/debug/accuracy_tools/api_accuracy_checker/hook_module/utils.py @@ -18,7 +18,7 @@ import os import yaml -from ptdbg_ascend.src.python.ptdbg_ascend.common.file_check_util import FileOpen +from common.file_check_util import FileOpen cur_path = os.path.dirname(os.path.realpath(__file__)) yaml_path = os.path.join(cur_path, "support_wrap_ops.yaml") diff --git a/debug/accuracy_tools/api_accuracy_checker/hook_module/wrap_functional.py b/debug/accuracy_tools/api_accuracy_checker/hook_module/wrap_functional.py index 967e9efc84123533422240c1f1fe530e73800bf0..79f685347a14b9d3bdca2da59918563e3a29b970 100644 --- a/debug/accuracy_tools/api_accuracy_checker/hook_module/wrap_functional.py +++ b/debug/accuracy_tools/api_accuracy_checker/hook_module/wrap_functional.py @@ -24,7 +24,6 @@ from api_accuracy_checker.hook_module.hook_module import HOOKModule from api_accuracy_checker.common.utils import torch_device_guard from api_accuracy_checker.common.config import msCheckerConfig from api_accuracy_checker.hook_module.utils import WrapFunctionalOps -from ptdbg_ascend.src.python.ptdbg_ascend.common.file_check_util import FileOpen for f in dir(torch.nn.functional): locals().update({f: getattr(torch.nn.functional, f)}) diff --git a/debug/accuracy_tools/api_accuracy_checker/hook_module/wrap_tensor.py b/debug/accuracy_tools/api_accuracy_checker/hook_module/wrap_tensor.py index b105ae8353fbe31ac3f1ffe2916beb2c8be2724e..06c618e525ea22d3313c93c5c086fbde433eca46 100644 --- a/debug/accuracy_tools/api_accuracy_checker/hook_module/wrap_tensor.py +++ b/debug/accuracy_tools/api_accuracy_checker/hook_module/wrap_tensor.py @@ -24,7 +24,6 @@ from api_accuracy_checker.hook_module.hook_module import HOOKModule from api_accuracy_checker.common.utils import torch_device_guard from api_accuracy_checker.common.config import msCheckerConfig from api_accuracy_checker.hook_module.utils import WrapTensorOps -from ptdbg_ascend.src.python.ptdbg_ascend.common.file_check_util import FileOpen def get_tensor_ops(): diff --git a/debug/accuracy_tools/api_accuracy_checker/hook_module/wrap_torch.py b/debug/accuracy_tools/api_accuracy_checker/hook_module/wrap_torch.py index 7eca0fbaba0a6cf883b05696ff032ff97eb14ccf..bf8bbe25fe7112177f01b4c42a0777399a32a07b 100644 --- a/debug/accuracy_tools/api_accuracy_checker/hook_module/wrap_torch.py +++ b/debug/accuracy_tools/api_accuracy_checker/hook_module/wrap_torch.py @@ -24,7 +24,6 @@ from api_accuracy_checker.hook_module.hook_module import HOOKModule from api_accuracy_checker.common.utils import torch_device_guard from api_accuracy_checker.common.config import msCheckerConfig from api_accuracy_checker.hook_module.utils import WrapTorchOps -from ptdbg_ascend.src.python.ptdbg_ascend.common.file_check_util import FileOpen def get_torch_ops(): diff --git a/debug/accuracy_tools/api_accuracy_checker/run_ut/run_overflow_check.py b/debug/accuracy_tools/api_accuracy_checker/run_ut/run_overflow_check.py index fb455434cf5464d416b16cb6a903aa9d29c2236b..a0adb2573090a529a12acdb19ed29b6328b825ff 100644 --- a/debug/accuracy_tools/api_accuracy_checker/run_ut/run_overflow_check.py +++ b/debug/accuracy_tools/api_accuracy_checker/run_ut/run_overflow_check.py @@ -8,7 +8,7 @@ from api_accuracy_checker.run_ut.run_ut import exec_api, generate_device_params, from api_accuracy_checker.common.utils import print_info_log, print_warn_log, get_json_contents, api_info_preprocess, \ print_error_log -from ptdbg_ascend.src.python.ptdbg_ascend.common.file_check_util import FileCheckConst, check_file_suffix, check_link +from common.file_check_util import FileCheckConst, check_file_suffix, check_link def check_tensor_overflow(x): diff --git a/debug/accuracy_tools/api_accuracy_checker/run_ut/run_ut.py b/debug/accuracy_tools/api_accuracy_checker/run_ut/run_ut.py index 0c0f3305c7104e87f64d6002996ed63c342c2eb9..fab598636cea4aa3e423721add1e3a11ec970463 100644 --- a/debug/accuracy_tools/api_accuracy_checker/run_ut/run_ut.py +++ b/debug/accuracy_tools/api_accuracy_checker/run_ut/run_ut.py @@ -26,7 +26,7 @@ from api_accuracy_checker.hook_module.wrap_torch import TorchOPTemplate from api_accuracy_checker.run_ut.ut_api_info import UtAPIInfo from api_accuracy_checker.common.config import msCheckerConfig -from ptdbg_ascend.src.python.ptdbg_ascend.common.file_check_util import FileOpen, FileCheckConst, FileChecker, \ +from common.file_check_util import FileOpen, FileCheckConst, FileChecker, \ change_mode, check_file_suffix, check_link ut_error_data_dir = 'ut_error_data' diff --git a/debug/accuracy_tools/api_accuracy_checker/run_ut/ut_api_info.py b/debug/accuracy_tools/api_accuracy_checker/run_ut/ut_api_info.py index 7d345ac0ab8c7d40f6663739c9363f75e06886be..c419923bf788ea519bf97ce1596f01d7b0c5909d 100644 --- a/debug/accuracy_tools/api_accuracy_checker/run_ut/ut_api_info.py +++ b/debug/accuracy_tools/api_accuracy_checker/run_ut/ut_api_info.py @@ -2,7 +2,7 @@ import os from api_accuracy_checker.common.config import msCheckerConfig from api_accuracy_checker.common.base_api import BaseAPIInfo from api_accuracy_checker.common.utils import write_pt, create_directory -from ptdbg_ascend.src.python.ptdbg_ascend.common.utils import check_path_before_create +from common.file_check_util import check_path_before_create class UtAPIInfo(BaseAPIInfo): diff --git a/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/common/file_check_util.py b/debug/accuracy_tools/common/file_check_util.py similarity index 94% rename from debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/common/file_check_util.py rename to debug/accuracy_tools/common/file_check_util.py index 2323124f53be2953229e10cf6af9d4a5951b5517..731851f60e264f5d71163020e28518fcdb0eb2b2 100644 --- a/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/common/file_check_util.py +++ b/debug/accuracy_tools/common/file_check_util.py @@ -36,6 +36,7 @@ class FileCheckConst: PT_SUFFIX = ".pt" CSV_SUFFIX = ".csv" YAML_SUFFIX = ".yaml" + FILE_PATTERN = r'^[a-zA-Z0-9_./-]+$' MAX_PKL_SIZE = 1 * 1024 * 1024 * 1024 MAX_NUMPY_SIZE = 10 * 1024 * 1024 * 1024 MAX_JSON_SIZE = 1 * 1024 * 1024 * 1024 @@ -296,3 +297,13 @@ def change_mode(path, mode): print_error_log('Failed to change {} authority. {}'.format(path, str(ex))) raise FileCheckException(FileCheckException.INVALID_PERMISSION_ERROR) + +def check_path_before_create(path): + if len(os.path.realpath(path)) > FileCheckConst.DIRECTORY_LENGTH or len(os.path.basename(path)) > \ + FileCheckConst.FILE_NAME_LENGTH: + print_error_log('The file path length exceeds limit.') + raise CompareException(CompareException.INVALID_PATH_ERROR) + + if not re.match(FileCheckConst.FILE_PATTERN, os.path.realpath(path)): + print_error_log('The file path {} contains special characters.'.format(path)) + raise CompareException(CompareException.INVALID_PATH_ERROR) \ No newline at end of file diff --git a/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/common/log.py b/debug/accuracy_tools/common/log.py similarity index 100% rename from debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/common/log.py rename to debug/accuracy_tools/common/log.py diff --git a/debug/accuracy_tools/ptdbg_ascend/CMakeLists.txt b/debug/accuracy_tools/ptdbg_ascend/CMakeLists.txt index 13cf18f7fd7116687241c79d53e1b6dc2d719d4d..79dd56d016520b8c755699deb5a6ac3802d59ea8 100644 --- a/debug/accuracy_tools/ptdbg_ascend/CMakeLists.txt +++ b/debug/accuracy_tools/ptdbg_ascend/CMakeLists.txt @@ -12,6 +12,7 @@ file(STRINGS "${CMAKE_CURRENT_LIST_DIR}/tools/PYTHON_BIN_PATH" PYTHON_BIN_PATH) add_custom_target(ptdbg_ascend ALL COMMAND ${CMAKE_COMMAND} -E copy_directory ${CMAKE_CURRENT_LIST_DIR}/src/python ${CMAKE_BINARY_DIR}/ptdbg_ascend + COMMAND ${CMAKE_COMMAND} -E copy_directory ${CMAKE_CURRENT_LIST_DIR}/../common ${CMAKE_BINARY_DIR}/ptdbg_ascend/ptdbg_ascend/common COMMAND cd ${CMAKE_BINARY_DIR}/ptdbg_ascend && ${PYTHON_BIN_PATH} setup.py bdist_wheel VERBATIM ) diff --git a/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/common/utils.py b/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/common/utils.py index a78eb595f6eeed6ebbf8e2194e31bb6a0be3f375..5572f99364239f27fc78fb6f2e5bf0094d3edf02 100644 --- a/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/common/utils.py +++ b/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/common/utils.py @@ -698,17 +698,6 @@ def check_file_valid(file_path): raise CompareException(CompareException.INVALID_PATH_ERROR) -def check_path_before_create(path): - if len(os.path.realpath(path)) > Const.DIRECTORY_LENGTH or len(os.path.basename(path)) > \ - Const.FILE_NAME_LENGTH: - print_error_log('The file path length exceeds limit.') - raise CompareException(CompareException.INVALID_PATH_ERROR) - - if not re.match(Const.FILE_PATTERN, os.path.realpath(path)): - print_error_log('The file path {} contains special characters.'.format(path)) - raise CompareException(CompareException.INVALID_PATH_ERROR) - - def check_inplace_op(prefix): if len(prefix) > Const.DISTRIBUTED_PREFIX_LENGTH: return False diff --git a/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/dump/utils.py b/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/dump/utils.py index b619d0cf493ff63529f4cb1d70c53e8abe7a45a8..e774eb37253bb1afc0a21fe52f190dfd63ac3a1f 100644 --- a/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/dump/utils.py +++ b/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/dump/utils.py @@ -7,9 +7,10 @@ import torch from ..dump import dump from ..common.utils import print_error_log, CompareException, DumpException, Const, get_time, print_info_log, \ - check_mode_valid, get_api_name_from_matcher, check_switch_valid, check_dump_mode_valid, check_summary_only_valid, generate_compare_script, \ - check_is_npu, check_file_valid, make_dump_path_if_not_exists, check_path_before_create -from ..common.file_check_util import FileChecker, FileCheckConst, check_path_length, check_path_pattern_vaild + check_mode_valid, check_switch_valid, check_dump_mode_valid, check_summary_only_valid, generate_compare_script, \ + check_is_npu, make_dump_path_if_not_exists +from ..common.file_check_util import FileChecker, FileCheckConst, check_path_length, check_path_pattern_vaild, \ + check_path_before_create, check_file_valid from ..common.version import __version__ diff --git a/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/online_dispatch/dispatch.py b/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/online_dispatch/dispatch.py index 05b4444e92f81f050fb6175af29295ef19442418..428e76c2baed8a1317d983d216df3a869e564b7d 100644 --- a/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/online_dispatch/dispatch.py +++ b/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/online_dispatch/dispatch.py @@ -16,13 +16,12 @@ except ImportError: else: is_npu = True -from ..common.utils import Const, CompareConst, add_time_as_suffix, check_file_or_directory_path, \ - check_path_before_create +from ..common.utils import Const, CompareConst, add_time_as_suffix, check_file_or_directory_path from ..common.version import __version__ from .dump_compare import dispatch_workflow, dispatch_multiprocess, error_call, TimeStatistics, \ DispatchRunParam, save_csv from .utils import get_callstack, data_to_cpu, logger_debug, logger_error, logger_warn, logger_logo, get_sys_info -from ..common.file_check_util import FileOpen +from ..common.file_check_util import FileOpen, check_path_before_create class PtdbgDispatch(TorchDispatchMode):