From 381cd17d7990a038bc7ba40279dd7bb74d16bb11 Mon Sep 17 00:00:00 2001 From: yang-minghai22 Date: Thu, 26 Oct 2023 14:11:51 +0800 Subject: [PATCH 1/2] add check before create dir catch indexError --- .../python/ptdbg_ascend/online_dispatch/dispatch.py | 9 +++++++-- .../src/python/ptdbg_ascend/parse_tool/lib/utils.py | 10 ++++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/online_dispatch/dispatch.py b/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/online_dispatch/dispatch.py index 6b3c98c33..85495ce18 100644 --- a/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/online_dispatch/dispatch.py +++ b/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/online_dispatch/dispatch.py @@ -157,8 +157,13 @@ class PtdbgDispatch(TorchDispatchMode): logger_error("Please confirm you run environment installed torch_npu!") return func(*args, **kwargs) - aten_api = func.__name__.split(".")[0] - aten_api_overload_name = func.__name__.split(".")[1] + func_name_split_list = func.__name__.split(".") + aten_api = func_name_split_list[0] + try: + aten_api_overload_name = func_name_split_list[1] + except IndexError: + logger_error(f"Please check the func name {func.__name__}!") + return func(*args, **kwargs) if aten_api in self.aten_ops_blacklist: npu_out = func(*args, **kwargs) diff --git a/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/parse_tool/lib/utils.py b/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/parse_tool/lib/utils.py index 02cf0a215..04b4fa3af 100644 --- a/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/parse_tool/lib/utils.py +++ b/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/parse_tool/lib/utils.py @@ -104,6 +104,7 @@ class Util: path = self.path_strip(path) if os.path.exists(path): return + self.check_path_name(path) try: os.makedirs(path, mode=0o750) except OSError as e: @@ -221,3 +222,12 @@ class Util: else: self.log.error("The file path %s is invalid" % path) raise ParseException(ParseException.PARSE_INVALID_PATH_ERROR) + + def check_path_name(self, path): + if len(os.path.realpath(path)) > Const.DIRECTORY_LENGTH or len(os.path.basename(path)) > \ + Const.FILE_NAME_LENGTH: + self.log.error('The file path length exceeds limit.') + raise ParseException(ParseException.PARSE_INVALID_PATH_ERROR) + if not re.match(Const.FILE_PATTERN, os.path.realpath(path)): + self.log.error('The file path {} contains special characters.'.format(path)) + raise ParseException(ParseException.PARSE_INVALID_PATH_ERROR) -- Gitee From 07ac31f10834abcfd7227fc53ba4114268dc9f93 Mon Sep 17 00:00:00 2001 From: yang-minghai22 Date: Fri, 27 Oct 2023 11:37:28 +0800 Subject: [PATCH 2/2] check path before create --- .../src/python/ptdbg_ascend/common/utils.py | 11 +++++++++++ .../src/python/ptdbg_ascend/dump/utils.py | 4 +++- .../python/ptdbg_ascend/online_dispatch/dispatch.py | 5 ++++- 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/common/utils.py b/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/common/utils.py index ecfc420fd..d9eb7dec4 100644 --- a/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/common/utils.py +++ b/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/common/utils.py @@ -668,3 +668,14 @@ def check_file_valid(file_path): if file_path.endswith(Const.NUMPY_SUFFIX) and file_size > Const.TEN_GB: print_error_log('The file {} size is greater than 10GB.'.format(file_path)) raise CompareException(CompareException.INVALID_PATH_ERROR) + + +def check_path_before_create(path): + if len(os.path.realpath(path)) > Const.DIRECTORY_LENGTH or len(os.path.basename(path)) > \ + Const.FILE_NAME_LENGTH: + print_error_log('The file path length exceeds limit.') + raise CompareException(CompareException.INVALID_PATH_ERROR) + + if not re.match(Const.FILE_PATTERN, os.path.realpath(path)): + print_error_log('The file path {} contains special characters.'.format(path)) + raise CompareException(CompareException.INVALID_PATH_ERROR) diff --git a/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/dump/utils.py b/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/dump/utils.py index 0f65156e0..2fcc0f4db 100644 --- a/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/dump/utils.py +++ b/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/dump/utils.py @@ -8,7 +8,7 @@ import torch from ..dump import dump from ..common.utils import print_error_log, CompareException, DumpException, Const, get_time, print_info_log, \ check_mode_valid, get_api_name_from_matcher, check_switch_valid, check_dump_mode_valid, check_summary_only_valid, generate_compare_script, \ - check_is_npu, check_file_valid, make_dump_path_if_not_exists + check_is_npu, check_file_valid, make_dump_path_if_not_exists, check_path_before_create from ..common.version import __version__ @@ -246,6 +246,7 @@ def make_dump_data_dir(dump_file_name): dump_path, file_name = os.path.split(os.path.realpath(dump_file_name)) name_body, name_extension = os.path.splitext(file_name) output_dir = os.path.join(dump_path, f"{name_body}") + check_path_before_create(output_dir) if not os.path.exists(output_dir): Path(output_dir).mkdir(mode=0o750, exist_ok=True) else: @@ -258,6 +259,7 @@ def make_dump_dirs(): dump_file_name, dump_file_name_body = "dump.pkl", "dump" dump_root_dir = load_env_dump_path(DumpUtil.dump_path) tag_dir = os.path.join(dump_root_dir, DumpUtil.dump_dir_tag + f'_v{__version__}') + check_path_before_create(tag_dir) Path(tag_dir).mkdir(mode=0o750, parents=True, exist_ok=True) DumpUtil.dump_dir = tag_dir dump_file_path = os.path.join(tag_dir, dump_file_name) diff --git a/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/online_dispatch/dispatch.py b/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/online_dispatch/dispatch.py index 85495ce18..126e5f3b2 100644 --- a/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/online_dispatch/dispatch.py +++ b/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/online_dispatch/dispatch.py @@ -14,7 +14,8 @@ except ImportError: else: is_npu = True -from ..common.utils import Const, CompareConst, add_time_as_suffix, check_file_or_directory_path +from ..common.utils import Const, CompareConst, add_time_as_suffix, check_file_or_directory_path, \ + check_path_before_create from ..common.version import __version__ from .dump_compare import dispatch_workflow, dispatch_multiprocess, error_call, TimeStatistics, \ DispatchRunParam, save_csv @@ -55,6 +56,8 @@ class PtdbgDispatch(TorchDispatchMode): self.root_npu_path = os.path.join(self.root_path, f'npu') file_name = add_time_as_suffix(f'compare_result_rank{self.device_id}') self.csv_path = os.path.join(self.root_path, file_name) + check_path_before_create(self.root_cpu_path) + check_path_before_create(self.root_npu_path) Path(self.root_cpu_path).mkdir(mode=0o750, parents=True, exist_ok=True) Path(self.root_npu_path).mkdir(mode=0o750, parents=True, exist_ok=True) -- Gitee