From f831b72baa1b5266e5ab07a3c40e06b70d3a8184 Mon Sep 17 00:00:00 2001 From: gitee Date: Mon, 11 Dec 2023 20:21:40 +0800 Subject: [PATCH 1/3] =?UTF-8?q?=E6=8F=90=E5=8F=96common?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../accuracy_tools/api_accuracy_checker/common/base_api.py | 2 +- debug/accuracy_tools/api_accuracy_checker/common/config.py | 2 +- debug/accuracy_tools/api_accuracy_checker/common/utils.py | 4 ++-- .../accuracy_tools/api_accuracy_checker/dump/info_dump.py | 3 +-- .../api_accuracy_checker/hook_module/utils.py | 2 +- .../api_accuracy_checker/hook_module/wrap_functional.py | 1 - .../api_accuracy_checker/hook_module/wrap_tensor.py | 1 - .../api_accuracy_checker/hook_module/wrap_torch.py | 1 - .../api_accuracy_checker/run_ut/run_overflow_check.py | 2 +- debug/accuracy_tools/api_accuracy_checker/run_ut/run_ut.py | 2 +- .../api_accuracy_checker/run_ut/ut_api_info.py | 2 +- .../src/python/ptdbg_ascend => }/common/file_check_util.py | 0 .../src/python/ptdbg_ascend => }/common/log.py | 0 debug/accuracy_tools/ptdbg_ascend/CMakeLists.txt | 1 + .../ptdbg_ascend/src/python/ptdbg_ascend/dump/utils.py | 7 ++++--- .../src/python/ptdbg_ascend/online_dispatch/dispatch.py | 5 ++--- 16 files changed, 16 insertions(+), 19 deletions(-) rename debug/accuracy_tools/{ptdbg_ascend/src/python/ptdbg_ascend => }/common/file_check_util.py (100%) rename debug/accuracy_tools/{ptdbg_ascend/src/python/ptdbg_ascend => }/common/log.py (100%) diff --git a/debug/accuracy_tools/api_accuracy_checker/common/base_api.py b/debug/accuracy_tools/api_accuracy_checker/common/base_api.py index 5bcc0e78af7..c08eed78e67 100644 --- a/debug/accuracy_tools/api_accuracy_checker/common/base_api.py +++ b/debug/accuracy_tools/api_accuracy_checker/common/base_api.py @@ -1,7 +1,7 @@ import os import torch from api_accuracy_checker.common.utils import print_error_log, write_pt, create_directory -from ptdbg_ascend.src.python.ptdbg_ascend.common.utils import check_path_before_create +from common.file_check_util import check_path_before_create class BaseAPIInfo: diff --git a/debug/accuracy_tools/api_accuracy_checker/common/config.py b/debug/accuracy_tools/api_accuracy_checker/common/config.py index f9b882f47b1..f90e3cdd573 100644 --- a/debug/accuracy_tools/api_accuracy_checker/common/config.py +++ b/debug/accuracy_tools/api_accuracy_checker/common/config.py @@ -2,7 +2,7 @@ import os import yaml from api_accuracy_checker.common.utils import check_file_or_directory_path from api_accuracy_checker.hook_module.utils import WrapFunctionalOps, WrapTensorOps, WrapTorchOps -from ptdbg_ascend.src.python.ptdbg_ascend.common.file_check_util import FileOpen +from common.file_check_util import FileOpen WrapApi = set(WrapFunctionalOps) | set(WrapTensorOps) | set(WrapTorchOps) diff --git a/debug/accuracy_tools/api_accuracy_checker/common/utils.py b/debug/accuracy_tools/api_accuracy_checker/common/utils.py index f8648eb20cc..027f83dbfb8 100644 --- a/debug/accuracy_tools/api_accuracy_checker/common/utils.py +++ b/debug/accuracy_tools/api_accuracy_checker/common/utils.py @@ -36,8 +36,8 @@ except ImportError: else: IS_GPU = False -from ptdbg_ascend.src.python.ptdbg_ascend.common.file_check_util import FileCheckConst, FileChecker, FileOpen -from ptdbg_ascend.src.python.ptdbg_ascend.common import file_check_util +from common.file_check_util import FileCheckConst, FileChecker, FileOpen +from common import file_check_util torch_without_guard_version_list = ['2.1'] for version in torch_without_guard_version_list: diff --git a/debug/accuracy_tools/api_accuracy_checker/dump/info_dump.py b/debug/accuracy_tools/api_accuracy_checker/dump/info_dump.py index 69de65912fb..dade146917f 100644 --- a/debug/accuracy_tools/api_accuracy_checker/dump/info_dump.py +++ b/debug/accuracy_tools/api_accuracy_checker/dump/info_dump.py @@ -5,11 +5,10 @@ import threading from api_accuracy_checker.dump.api_info import ForwardAPIInfo, BackwardAPIInfo from api_accuracy_checker.common.utils import check_file_or_directory_path, initialize_save_path, create_directory -from ptdbg_ascend.src.python.ptdbg_ascend.common.utils import check_path_before_create from api_accuracy_checker.common.config import msCheckerConfig -from ptdbg_ascend.src.python.ptdbg_ascend.common.file_check_util import FileOpen, FileCheckConst, FileChecker +from common.file_check_util import FileOpen, FileCheckConst, FileChecker, check_path_before_create lock = threading.Lock() diff --git a/debug/accuracy_tools/api_accuracy_checker/hook_module/utils.py b/debug/accuracy_tools/api_accuracy_checker/hook_module/utils.py index 7d16ac993ed..c81d747a1cb 100644 --- a/debug/accuracy_tools/api_accuracy_checker/hook_module/utils.py +++ b/debug/accuracy_tools/api_accuracy_checker/hook_module/utils.py @@ -18,7 +18,7 @@ import os import yaml -from ptdbg_ascend.src.python.ptdbg_ascend.common.file_check_util import FileOpen +from common.file_check_util import FileOpen cur_path = os.path.dirname(os.path.realpath(__file__)) yaml_path = os.path.join(cur_path, "support_wrap_ops.yaml") diff --git a/debug/accuracy_tools/api_accuracy_checker/hook_module/wrap_functional.py b/debug/accuracy_tools/api_accuracy_checker/hook_module/wrap_functional.py index 967e9efc841..79f685347a1 100644 --- a/debug/accuracy_tools/api_accuracy_checker/hook_module/wrap_functional.py +++ b/debug/accuracy_tools/api_accuracy_checker/hook_module/wrap_functional.py @@ -24,7 +24,6 @@ from api_accuracy_checker.hook_module.hook_module import HOOKModule from api_accuracy_checker.common.utils import torch_device_guard from api_accuracy_checker.common.config import msCheckerConfig from api_accuracy_checker.hook_module.utils import WrapFunctionalOps -from ptdbg_ascend.src.python.ptdbg_ascend.common.file_check_util import FileOpen for f in dir(torch.nn.functional): locals().update({f: getattr(torch.nn.functional, f)}) diff --git a/debug/accuracy_tools/api_accuracy_checker/hook_module/wrap_tensor.py b/debug/accuracy_tools/api_accuracy_checker/hook_module/wrap_tensor.py index b105ae8353f..06c618e525e 100644 --- a/debug/accuracy_tools/api_accuracy_checker/hook_module/wrap_tensor.py +++ b/debug/accuracy_tools/api_accuracy_checker/hook_module/wrap_tensor.py @@ -24,7 +24,6 @@ from api_accuracy_checker.hook_module.hook_module import HOOKModule from api_accuracy_checker.common.utils import torch_device_guard from api_accuracy_checker.common.config import msCheckerConfig from api_accuracy_checker.hook_module.utils import WrapTensorOps -from ptdbg_ascend.src.python.ptdbg_ascend.common.file_check_util import FileOpen def get_tensor_ops(): diff --git a/debug/accuracy_tools/api_accuracy_checker/hook_module/wrap_torch.py b/debug/accuracy_tools/api_accuracy_checker/hook_module/wrap_torch.py index 7eca0fbaba0..bf8bbe25fe7 100644 --- a/debug/accuracy_tools/api_accuracy_checker/hook_module/wrap_torch.py +++ b/debug/accuracy_tools/api_accuracy_checker/hook_module/wrap_torch.py @@ -24,7 +24,6 @@ from api_accuracy_checker.hook_module.hook_module import HOOKModule from api_accuracy_checker.common.utils import torch_device_guard from api_accuracy_checker.common.config import msCheckerConfig from api_accuracy_checker.hook_module.utils import WrapTorchOps -from ptdbg_ascend.src.python.ptdbg_ascend.common.file_check_util import FileOpen def get_torch_ops(): diff --git a/debug/accuracy_tools/api_accuracy_checker/run_ut/run_overflow_check.py b/debug/accuracy_tools/api_accuracy_checker/run_ut/run_overflow_check.py index fb455434cf5..a0adb257309 100644 --- a/debug/accuracy_tools/api_accuracy_checker/run_ut/run_overflow_check.py +++ b/debug/accuracy_tools/api_accuracy_checker/run_ut/run_overflow_check.py @@ -8,7 +8,7 @@ from api_accuracy_checker.run_ut.run_ut import exec_api, generate_device_params, from api_accuracy_checker.common.utils import print_info_log, print_warn_log, get_json_contents, api_info_preprocess, \ print_error_log -from ptdbg_ascend.src.python.ptdbg_ascend.common.file_check_util import FileCheckConst, check_file_suffix, check_link +from common.file_check_util import FileCheckConst, check_file_suffix, check_link def check_tensor_overflow(x): diff --git a/debug/accuracy_tools/api_accuracy_checker/run_ut/run_ut.py b/debug/accuracy_tools/api_accuracy_checker/run_ut/run_ut.py index 0c0f3305c71..376e3c4ef9e 100644 --- a/debug/accuracy_tools/api_accuracy_checker/run_ut/run_ut.py +++ b/debug/accuracy_tools/api_accuracy_checker/run_ut/run_ut.py @@ -26,7 +26,7 @@ from api_accuracy_checker.hook_module.wrap_torch import TorchOPTemplate from api_accuracy_checker.run_ut.ut_api_info import UtAPIInfo from api_accuracy_checker.common.config import msCheckerConfig -from ptdbg_ascend.src.python.ptdbg_ascend.common.file_check_util import FileOpen, FileCheckConst, FileChecker, \ +from common.file_check_util import FileCheckConst, FileChecker, \ change_mode, check_file_suffix, check_link ut_error_data_dir = 'ut_error_data' diff --git a/debug/accuracy_tools/api_accuracy_checker/run_ut/ut_api_info.py b/debug/accuracy_tools/api_accuracy_checker/run_ut/ut_api_info.py index 7d345ac0ab8..c419923bf78 100644 --- a/debug/accuracy_tools/api_accuracy_checker/run_ut/ut_api_info.py +++ b/debug/accuracy_tools/api_accuracy_checker/run_ut/ut_api_info.py @@ -2,7 +2,7 @@ import os from api_accuracy_checker.common.config import msCheckerConfig from api_accuracy_checker.common.base_api import BaseAPIInfo from api_accuracy_checker.common.utils import write_pt, create_directory -from ptdbg_ascend.src.python.ptdbg_ascend.common.utils import check_path_before_create +from common.file_check_util import check_path_before_create class UtAPIInfo(BaseAPIInfo): diff --git a/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/common/file_check_util.py b/debug/accuracy_tools/common/file_check_util.py similarity index 100% rename from debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/common/file_check_util.py rename to debug/accuracy_tools/common/file_check_util.py diff --git a/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/common/log.py b/debug/accuracy_tools/common/log.py similarity index 100% rename from debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/common/log.py rename to debug/accuracy_tools/common/log.py diff --git a/debug/accuracy_tools/ptdbg_ascend/CMakeLists.txt b/debug/accuracy_tools/ptdbg_ascend/CMakeLists.txt index 13cf18f7fd7..79dd56d0165 100644 --- a/debug/accuracy_tools/ptdbg_ascend/CMakeLists.txt +++ b/debug/accuracy_tools/ptdbg_ascend/CMakeLists.txt @@ -12,6 +12,7 @@ file(STRINGS "${CMAKE_CURRENT_LIST_DIR}/tools/PYTHON_BIN_PATH" PYTHON_BIN_PATH) add_custom_target(ptdbg_ascend ALL COMMAND ${CMAKE_COMMAND} -E copy_directory ${CMAKE_CURRENT_LIST_DIR}/src/python ${CMAKE_BINARY_DIR}/ptdbg_ascend + COMMAND ${CMAKE_COMMAND} -E copy_directory ${CMAKE_CURRENT_LIST_DIR}/../common ${CMAKE_BINARY_DIR}/ptdbg_ascend/ptdbg_ascend/common COMMAND cd ${CMAKE_BINARY_DIR}/ptdbg_ascend && ${PYTHON_BIN_PATH} setup.py bdist_wheel VERBATIM ) diff --git a/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/dump/utils.py b/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/dump/utils.py index b619d0cf493..e774eb37253 100644 --- a/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/dump/utils.py +++ b/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/dump/utils.py @@ -7,9 +7,10 @@ import torch from ..dump import dump from ..common.utils import print_error_log, CompareException, DumpException, Const, get_time, print_info_log, \ - check_mode_valid, get_api_name_from_matcher, check_switch_valid, check_dump_mode_valid, check_summary_only_valid, generate_compare_script, \ - check_is_npu, check_file_valid, make_dump_path_if_not_exists, check_path_before_create -from ..common.file_check_util import FileChecker, FileCheckConst, check_path_length, check_path_pattern_vaild + check_mode_valid, check_switch_valid, check_dump_mode_valid, check_summary_only_valid, generate_compare_script, \ + check_is_npu, make_dump_path_if_not_exists +from ..common.file_check_util import FileChecker, FileCheckConst, check_path_length, check_path_pattern_vaild, \ + check_path_before_create, check_file_valid from ..common.version import __version__ diff --git a/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/online_dispatch/dispatch.py b/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/online_dispatch/dispatch.py index 05b4444e92f..428e76c2bae 100644 --- a/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/online_dispatch/dispatch.py +++ b/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/online_dispatch/dispatch.py @@ -16,13 +16,12 @@ except ImportError: else: is_npu = True -from ..common.utils import Const, CompareConst, add_time_as_suffix, check_file_or_directory_path, \ - check_path_before_create +from ..common.utils import Const, CompareConst, add_time_as_suffix, check_file_or_directory_path from ..common.version import __version__ from .dump_compare import dispatch_workflow, dispatch_multiprocess, error_call, TimeStatistics, \ DispatchRunParam, save_csv from .utils import get_callstack, data_to_cpu, logger_debug, logger_error, logger_warn, logger_logo, get_sys_info -from ..common.file_check_util import FileOpen +from ..common.file_check_util import FileOpen, check_path_before_create class PtdbgDispatch(TorchDispatchMode): -- Gitee From 7b277a923f4f10ecd08038d90e35293b8060e086 Mon Sep 17 00:00:00 2001 From: gitee Date: Mon, 11 Dec 2023 21:08:22 +0800 Subject: [PATCH 2/3] =?UTF-8?q?=E6=8F=90=E5=8F=96common?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../accuracy_tools/common/file_check_util.py | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/debug/accuracy_tools/common/file_check_util.py b/debug/accuracy_tools/common/file_check_util.py index 2323124f53b..9904551695a 100644 --- a/debug/accuracy_tools/common/file_check_util.py +++ b/debug/accuracy_tools/common/file_check_util.py @@ -296,3 +296,37 @@ def change_mode(path, mode): print_error_log('Failed to change {} authority. {}'.format(path, str(ex))) raise FileCheckException(FileCheckException.INVALID_PERMISSION_ERROR) + +def check_file_valid(file_path): + if os.path.islink(file_path): + print_error_log('The file path {} is a soft link.'.format(file_path)) + raise CompareException(CompareException.INVALID_PATH_ERROR) + + if len(os.path.realpath(file_path)) > Const.DIRECTORY_LENGTH or len(os.path.basename(file_path)) > \ + Const.FILE_NAME_LENGTH: + print_error_log('The file path length exceeds limit.') + raise CompareException(CompareException.INVALID_PATH_ERROR) + + if not re.match(Const.FILE_PATTERN, os.path.realpath(file_path)): + print_error_log('The file path {} contains special characters.'.format(file_path)) + raise CompareException(CompareException.INVALID_PATH_ERROR) + + if os.path.isfile(file_path): + file_size = os.path.getsize(file_path) + if file_path.endswith(Const.PKL_SUFFIX) and file_size > Const.ONE_GB: + print_error_log('The file {} size is greater than 1GB.'.format(file_path)) + raise CompareException(CompareException.INVALID_PATH_ERROR) + if file_path.endswith(Const.NUMPY_SUFFIX) and file_size > Const.TEN_GB: + print_error_log('The file {} size is greater than 10GB.'.format(file_path)) + raise CompareException(CompareException.INVALID_PATH_ERROR) + + +def check_path_before_create(path): + if len(os.path.realpath(path)) > Const.DIRECTORY_LENGTH or len(os.path.basename(path)) > \ + Const.FILE_NAME_LENGTH: + print_error_log('The file path length exceeds limit.') + raise CompareException(CompareException.INVALID_PATH_ERROR) + + if not re.match(Const.FILE_PATTERN, os.path.realpath(path)): + print_error_log('The file path {} contains special characters.'.format(path)) + raise CompareException(CompareException.INVALID_PATH_ERROR) \ No newline at end of file -- Gitee From 19b22390a92c57af859829dac5e798857b48c66c Mon Sep 17 00:00:00 2001 From: gitee Date: Mon, 11 Dec 2023 21:55:54 +0800 Subject: [PATCH 3/3] =?UTF-8?q?=E6=8F=90=E5=8F=96common?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../api_accuracy_checker/run_ut/run_ut.py | 2 +- .../accuracy_tools/common/file_check_util.py | 31 +++---------------- .../src/python/ptdbg_ascend/common/utils.py | 11 ------- 3 files changed, 5 insertions(+), 39 deletions(-) diff --git a/debug/accuracy_tools/api_accuracy_checker/run_ut/run_ut.py b/debug/accuracy_tools/api_accuracy_checker/run_ut/run_ut.py index 376e3c4ef9e..fab598636ce 100644 --- a/debug/accuracy_tools/api_accuracy_checker/run_ut/run_ut.py +++ b/debug/accuracy_tools/api_accuracy_checker/run_ut/run_ut.py @@ -26,7 +26,7 @@ from api_accuracy_checker.hook_module.wrap_torch import TorchOPTemplate from api_accuracy_checker.run_ut.ut_api_info import UtAPIInfo from api_accuracy_checker.common.config import msCheckerConfig -from common.file_check_util import FileCheckConst, FileChecker, \ +from common.file_check_util import FileOpen, FileCheckConst, FileChecker, \ change_mode, check_file_suffix, check_link ut_error_data_dir = 'ut_error_data' diff --git a/debug/accuracy_tools/common/file_check_util.py b/debug/accuracy_tools/common/file_check_util.py index 9904551695a..731851f60e2 100644 --- a/debug/accuracy_tools/common/file_check_util.py +++ b/debug/accuracy_tools/common/file_check_util.py @@ -36,6 +36,7 @@ class FileCheckConst: PT_SUFFIX = ".pt" CSV_SUFFIX = ".csv" YAML_SUFFIX = ".yaml" + FILE_PATTERN = r'^[a-zA-Z0-9_./-]+$' MAX_PKL_SIZE = 1 * 1024 * 1024 * 1024 MAX_NUMPY_SIZE = 10 * 1024 * 1024 * 1024 MAX_JSON_SIZE = 1 * 1024 * 1024 * 1024 @@ -297,36 +298,12 @@ def change_mode(path, mode): raise FileCheckException(FileCheckException.INVALID_PERMISSION_ERROR) -def check_file_valid(file_path): - if os.path.islink(file_path): - print_error_log('The file path {} is a soft link.'.format(file_path)) - raise CompareException(CompareException.INVALID_PATH_ERROR) - - if len(os.path.realpath(file_path)) > Const.DIRECTORY_LENGTH or len(os.path.basename(file_path)) > \ - Const.FILE_NAME_LENGTH: - print_error_log('The file path length exceeds limit.') - raise CompareException(CompareException.INVALID_PATH_ERROR) - - if not re.match(Const.FILE_PATTERN, os.path.realpath(file_path)): - print_error_log('The file path {} contains special characters.'.format(file_path)) - raise CompareException(CompareException.INVALID_PATH_ERROR) - - if os.path.isfile(file_path): - file_size = os.path.getsize(file_path) - if file_path.endswith(Const.PKL_SUFFIX) and file_size > Const.ONE_GB: - print_error_log('The file {} size is greater than 1GB.'.format(file_path)) - raise CompareException(CompareException.INVALID_PATH_ERROR) - if file_path.endswith(Const.NUMPY_SUFFIX) and file_size > Const.TEN_GB: - print_error_log('The file {} size is greater than 10GB.'.format(file_path)) - raise CompareException(CompareException.INVALID_PATH_ERROR) - - def check_path_before_create(path): - if len(os.path.realpath(path)) > Const.DIRECTORY_LENGTH or len(os.path.basename(path)) > \ - Const.FILE_NAME_LENGTH: + if len(os.path.realpath(path)) > FileCheckConst.DIRECTORY_LENGTH or len(os.path.basename(path)) > \ + FileCheckConst.FILE_NAME_LENGTH: print_error_log('The file path length exceeds limit.') raise CompareException(CompareException.INVALID_PATH_ERROR) - if not re.match(Const.FILE_PATTERN, os.path.realpath(path)): + if not re.match(FileCheckConst.FILE_PATTERN, os.path.realpath(path)): print_error_log('The file path {} contains special characters.'.format(path)) raise CompareException(CompareException.INVALID_PATH_ERROR) \ No newline at end of file diff --git a/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/common/utils.py b/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/common/utils.py index a78eb595f6e..5572f993642 100644 --- a/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/common/utils.py +++ b/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/common/utils.py @@ -698,17 +698,6 @@ def check_file_valid(file_path): raise CompareException(CompareException.INVALID_PATH_ERROR) -def check_path_before_create(path): - if len(os.path.realpath(path)) > Const.DIRECTORY_LENGTH or len(os.path.basename(path)) > \ - Const.FILE_NAME_LENGTH: - print_error_log('The file path length exceeds limit.') - raise CompareException(CompareException.INVALID_PATH_ERROR) - - if not re.match(Const.FILE_PATTERN, os.path.realpath(path)): - print_error_log('The file path {} contains special characters.'.format(path)) - raise CompareException(CompareException.INVALID_PATH_ERROR) - - def check_inplace_op(prefix): if len(prefix) > Const.DISTRIBUTED_PREFIX_LENGTH: return False -- Gitee