diff --git a/debug/accuracy_tools/api_accuracy_checker/common/base_api.py b/debug/accuracy_tools/api_accuracy_checker/common/base_api.py index 627bd769766863683d2191509a54f5e28660a623..a4f9a4ea8373f5d3b0d9d624066645b1e34d944d 100644 --- a/debug/accuracy_tools/api_accuracy_checker/common/base_api.py +++ b/debug/accuracy_tools/api_accuracy_checker/common/base_api.py @@ -41,7 +41,6 @@ class BaseAPIInfo: raise NotImplementedError(msg) return out - def analyze_tensor(self, arg): single_arg = {} if not self.is_save_data: diff --git a/debug/accuracy_tools/api_accuracy_checker/common/config.py b/debug/accuracy_tools/api_accuracy_checker/common/config.py index 02d40973c10b96160e5f22aaa4f48eaa93691de9..821842ddd9be54b824d9ed05710fa1977a85df0a 100644 --- a/debug/accuracy_tools/api_accuracy_checker/common/config.py +++ b/debug/accuracy_tools/api_accuracy_checker/common/config.py @@ -1,11 +1,12 @@ import yaml import os from api_accuracy_checker.common.utils import check_file_or_directory_path +from ptdbg_ascend.src.python.ptdbg_ascend.common.file_check_util import FileOpen class Config: def __init__(self, yaml_file): check_file_or_directory_path(yaml_file, False) - with open(yaml_file, 'r') as file: + with FileOpen(yaml_file, 'r') as file: config = yaml.safe_load(file) self.config = {key: self.validate(key, value) for key, value in config.items()} @@ -20,7 +21,7 @@ class Config: 'target_iter': int, 'precision': int } - if not isinstance(value, validators[key]): + if not isinstance(value, validators.get(key)): raise ValueError(f"{key} must be {validators[key].__name__} type") if key == 'target_iter' and value < 0: raise ValueError("target_iter must be greater than 0") diff --git a/debug/accuracy_tools/api_accuracy_checker/common/utils.py b/debug/accuracy_tools/api_accuracy_checker/common/utils.py index 065c0fe46de551d045f8014ed85931f58b0371a7..b031b92a18afbe429d0a043760e427589c2865e1 100644 --- a/debug/accuracy_tools/api_accuracy_checker/common/utils.py +++ b/debug/accuracy_tools/api_accuracy_checker/common/utils.py @@ -29,6 +29,9 @@ import numpy as np import torch import csv +from ptdbg_ascend.src.python.ptdbg_ascend.common.file_check_util import FileCheckConst, FileChecker, FileOpen +from ptdbg_ascend.src.python.ptdbg_ascend.common import file_check_util + try: import torch_npu except ImportError: @@ -86,7 +89,7 @@ class Const: API_PATTERN = r"^[A-Za-z0-9]+[_]+([A-Za-z0-9]+[_]*[A-Za-z0-9]+)[_]+[0-9]+[_]+[A-Za-z0-9]+" WRITE_FLAGS = os.O_WRONLY | os.O_CREAT WRITE_MODES = stat.S_IWUSR | stat.S_IRUSR - + RAISE_PRECISION = { "torch.float16" : "torch.float32", "torch.bfloat16" : "torch.float32", @@ -192,12 +195,12 @@ class DumpException(CompareException): pass def read_json(file): - with open(file, 'r') as f: + with FileOpen(file, 'r') as f: obj = json.load(f) return obj def write_csv(data, filepath): - with open(filepath, 'a') as f: + with FileOpen(filepath, 'a') as f: writer = csv.writer(f) writer.writerows(data) @@ -377,7 +380,7 @@ def create_directory(dir_path): """ if not os.path.exists(dir_path): try: - os.makedirs(dir_path, mode=0o700) + os.makedirs(dir_path, mode=FileCheckConst.DATA_DIR_AUTHORITY) except OSError as ex: print_error_log( 'Failed to create {}.Please check the path permission or disk space .{}'.format(dir_path, str(ex))) @@ -513,8 +516,7 @@ def get_json_contents(file_path): def get_file_content_bytes(file): - check_input_file_valid(file) - with open(file, 'rb') as file_handle: + with FileOpen(file, 'rb') as file_handle: return file_handle.read() @@ -606,12 +608,15 @@ def initialize_save_path(save_path, dir_name): if os.path.exists(data_path): raise ValueError(f"file {data_path} already exists, please remove it first") else: - os.mkdir(data_path, mode = 0o750) - check_file_or_directory_path(data_path, True) + os.mkdir(data_path, mode=FileCheckConst.DATA_DIR_AUTHORITY) + data_path_checker = FileChecker(data_path, FileCheckConst.DIR) + data_path_checker.common_check() + def write_pt(file_path, tensor): if os.path.exists(file_path): raise ValueError(f"File {file_path} already exists") torch.save(tensor, file_path) - full_path = os.path.abspath(file_path) - return full_path \ No newline at end of file + full_path = os.path.realpath(file_path) + file_check_util.change_mode(full_path, FileCheckConst.DATA_FILE_AUTHORITY) + return full_path diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/save_result.py b/debug/accuracy_tools/api_accuracy_checker/compare/save_result.py deleted file mode 100644 index 06191a4a54f20b3f82bb55f98f446415a76089a0..0000000000000000000000000000000000000000 --- a/debug/accuracy_tools/api_accuracy_checker/compare/save_result.py +++ /dev/null @@ -1 +0,0 @@ -# 比对表格输出,并将失败api数据进行保存 \ No newline at end of file diff --git a/debug/accuracy_tools/api_accuracy_checker/dump/dump.py b/debug/accuracy_tools/api_accuracy_checker/dump/dump.py index 08385882d301d97b4c31f32854fdab0efd1f2975..830ff5e4fcf1548e11119dc0f36bf7641a94143e 100644 --- a/debug/accuracy_tools/api_accuracy_checker/dump/dump.py +++ b/debug/accuracy_tools/api_accuracy_checker/dump/dump.py @@ -17,12 +17,15 @@ from api_accuracy_checker.dump.api_info import ForwardAPIInfo, BackwardAPIInfo from api_accuracy_checker.dump.info_dump import write_api_info_json, initialize_output_json -from api_accuracy_checker.common.utils import print_error_log +from api_accuracy_checker.common.utils import print_error_log, CompareException from api_accuracy_checker.hook_module.register_hook import initialize_hook from api_accuracy_checker.common.config import msCheckerConfig def set_dump_switch(switch): + if switch not in ["ON", "OFF"]: + print_error_log("Please set switch with 'ON' or 'OFF'.") + raise CompareException(CompareException.INVALID_PARAM_ERROR) if switch == "ON": initialize_hook(pretest_hook) initialize_output_json() @@ -39,8 +42,8 @@ class DumpUtil(object): @staticmethod def get_dump_switch(): return DumpUtil.dump_switch == "ON" - - @staticmethod + + @staticmethod def incr_iter_num_maybe_exit(): if DumpUtil.call_num == msCheckerConfig.target_iter or not msCheckerConfig.enable_dataloader: set_dump_switch("ON") @@ -48,7 +51,7 @@ class DumpUtil(object): raise Exception("Model pretest: exit after iteration {}".format(msCheckerConfig.target_iter)) else: set_dump_switch("OFF") - DumpUtil.call_num += 1 + DumpUtil.call_num += 1 class DumpConst: @@ -59,7 +62,7 @@ class DumpConst: def pretest_info_dump(name, out_feat, module, phase): if not DumpUtil.get_dump_switch(): - return + return if phase == DumpConst.forward: api_info = ForwardAPIInfo(name, module.input_args, module.input_kwargs) elif phase == DumpConst.backward: @@ -68,14 +71,14 @@ def pretest_info_dump(name, out_feat, module, phase): msg = "Unexpected training phase {}.".format(phase) print_error_log(msg) raise NotImplementedError(msg) - + write_api_info_json(api_info) def pretest_hook(name, phase): def pretest_info_dump_hook(module, in_feat, out_feat): pretest_info_dump(name, out_feat, module, phase) if hasattr(module, "input_args"): - del module.input_args + del module.input_args if hasattr(module, "input_kwargs"): - del module.input_kwargs - return pretest_info_dump_hook + del module.input_kwargs + return pretest_info_dump_hook diff --git a/debug/accuracy_tools/api_accuracy_checker/dump/info_dump.py b/debug/accuracy_tools/api_accuracy_checker/dump/info_dump.py index f2a96bd0fac56b0da20c980bbf70caed2a6f6161..f915d779a84609f8cf39ff098792a19374d64048 100644 --- a/debug/accuracy_tools/api_accuracy_checker/dump/info_dump.py +++ b/debug/accuracy_tools/api_accuracy_checker/dump/info_dump.py @@ -7,6 +7,8 @@ from .api_info import ForwardAPIInfo, BackwardAPIInfo from ..common.utils import check_file_or_directory_path, initialize_save_path from ..common.config import msCheckerConfig +from ptdbg_ascend.src.python.ptdbg_ascend.common.file_check_util import FileOpen, FileCheckConst, FileChecker + lock = threading.Lock() def write_api_info_json(api_info): @@ -27,10 +29,10 @@ def write_api_info_json(api_info): def write_json(file_path, data, indent=None): check_file_or_directory_path(os.path.dirname(file_path),True) if not os.path.exists(file_path): - with open(file_path, 'w') as f: + with FileOpen(file_path, 'w') as f: f.write("{\n}") lock.acquire() - with open(file_path, 'a+') as f: + with FileOpen(file_path, 'a+') as f: fcntl.flock(f, fcntl.LOCK_EX) try: f.seek(0, os.SEEK_END) @@ -49,8 +51,8 @@ def write_json(file_path, data, indent=None): def initialize_output_json(): - dump_path = os.path.realpath(msCheckerConfig.dump_path) - check_file_or_directory_path(dump_path, True) + dump_path_checker = FileChecker(msCheckerConfig.dump_path, FileCheckConst.DIR) + dump_path = dump_path_checker.common_check() files = ['forward_info.json', 'backward_info.json', 'stack_info.json'] if msCheckerConfig.real_data: initialize_save_path(dump_path, 'forward_real_data') diff --git a/debug/accuracy_tools/api_accuracy_checker/dump/utils.py b/debug/accuracy_tools/api_accuracy_checker/dump/utils.py deleted file mode 100644 index 93af6f0981aa0bd3d3c42606a50f04b79bc1c37b..0000000000000000000000000000000000000000 --- a/debug/accuracy_tools/api_accuracy_checker/dump/utils.py +++ /dev/null @@ -1,15 +0,0 @@ -import os -import numpy as np - - -def create_folder(path): - if not os.path.exists(path): - os.makedirs(path, mode=0o750) - return path - -def write_npy(file_path, tensor): - if os.path.exists(file_path): - raise ValueError(f"File {file_path} already exists") - np.save(file_path, tensor) - full_path = os.path.abspath(file_path) - return full_path diff --git a/debug/accuracy_tools/api_accuracy_checker/hook_module/wrap_functional.py b/debug/accuracy_tools/api_accuracy_checker/hook_module/wrap_functional.py index dbe27a134c0ad5351099613f4fa0fe1fbe87ae0b..a1c9af127f25afc713bd16a1153cf3bfcab292f2 100644 --- a/debug/accuracy_tools/api_accuracy_checker/hook_module/wrap_functional.py +++ b/debug/accuracy_tools/api_accuracy_checker/hook_module/wrap_functional.py @@ -22,10 +22,11 @@ import yaml from api_accuracy_checker.hook_module.hook_module import HOOKModule from api_accuracy_checker.common.utils import torch_device_guard +from ptdbg_ascend.src.python.ptdbg_ascend.common.file_check_util import FileOpen cur_path = os.path.dirname(os.path.realpath(__file__)) yaml_path = os.path.join(cur_path, "support_wrap_ops.yaml") -with open(yaml_path, 'r') as f: +with FileOpen(yaml_path, 'r') as f: WrapFunctionalOps = yaml.safe_load(f).get('functional') for f in dir(torch.nn.functional): diff --git a/debug/accuracy_tools/api_accuracy_checker/hook_module/wrap_tensor.py b/debug/accuracy_tools/api_accuracy_checker/hook_module/wrap_tensor.py index 93d92923c666fec64fa25683be35bdecb98e8b81..547955ec1876e1f4a464ff0268ec845c46b2f1a9 100644 --- a/debug/accuracy_tools/api_accuracy_checker/hook_module/wrap_tensor.py +++ b/debug/accuracy_tools/api_accuracy_checker/hook_module/wrap_tensor.py @@ -22,10 +22,11 @@ import yaml from api_accuracy_checker.hook_module.hook_module import HOOKModule from api_accuracy_checker.common.utils import torch_device_guard +from ptdbg_ascend.src.python.ptdbg_ascend.common.file_check_util import FileOpen cur_path = os.path.dirname(os.path.realpath(__file__)) yaml_path = os.path.join(cur_path, "support_wrap_ops.yaml") -with open(yaml_path, 'r') as f: +with FileOpen(yaml_path, 'r') as f: WrapTensorOps = yaml.safe_load(f).get('tensor') diff --git a/debug/accuracy_tools/api_accuracy_checker/hook_module/wrap_torch.py b/debug/accuracy_tools/api_accuracy_checker/hook_module/wrap_torch.py index 07a037b779b4ce43773e06398eebc238f5c4279d..c42ada0ed4e0ecdab8d86bc09afbbd933ce74d93 100644 --- a/debug/accuracy_tools/api_accuracy_checker/hook_module/wrap_torch.py +++ b/debug/accuracy_tools/api_accuracy_checker/hook_module/wrap_torch.py @@ -23,9 +23,11 @@ import yaml from api_accuracy_checker.hook_module.hook_module import HOOKModule from api_accuracy_checker.common.utils import torch_device_guard +from ptdbg_ascend.src.python.ptdbg_ascend.common.file_check_util import FileOpen + cur_path = os.path.dirname(os.path.realpath(__file__)) yaml_path = os.path.join(cur_path, "support_wrap_ops.yaml") -with open(yaml_path, 'r') as f: +with FileOpen(yaml_path, 'r') as f: WrapTorchOps = yaml.safe_load(f).get('torch') diff --git a/debug/accuracy_tools/api_accuracy_checker/run_ut/run_overflow_check.py b/debug/accuracy_tools/api_accuracy_checker/run_ut/run_overflow_check.py index 7c0fa0f6a6b73f1cd86b256ec791262d53638745..03e4e545aafb020091a68dd80ed36e3f8ede4004 100644 --- a/debug/accuracy_tools/api_accuracy_checker/run_ut/run_overflow_check.py +++ b/debug/accuracy_tools/api_accuracy_checker/run_ut/run_overflow_check.py @@ -9,6 +9,7 @@ from api_accuracy_checker.run_ut.run_ut import exec_api, generate_npu_params, ru from api_accuracy_checker.common.utils import print_info_log, print_warn_log, get_json_contents, api_info_preprocess, \ print_error_log +from ptdbg_ascend.src.python.ptdbg_ascend.common.file_check_util import FileCheckConst, check_file_suffix NO_GRAD_APIS = ["hardtanh"] @@ -137,10 +138,8 @@ def _run_overflow_check(): backward_file = "" if args.backward_input_file: backward_file = os.path.realpath(args.backward_input_file) - if not backward_file.endswith(".json"): - raise ValueError("The backward_input_file should be a json file!") - if not forward_file.endswith(".json"): - raise ValueError("The forward_input_file should be a json file!") + check_file_suffix(backward_file, FileCheckConst.JSON_SUFFIX) + check_file_suffix(forward_file, FileCheckConst.JSON_SUFFIX) try: torch.npu.set_device(npu_device) except Exception: diff --git a/debug/accuracy_tools/api_accuracy_checker/run_ut/run_ut.py b/debug/accuracy_tools/api_accuracy_checker/run_ut/run_ut.py index 4cc27880c791ba069c4ef7ca06a062dc618d49f3..e7f240185f44a492d072ec7a0d4d5996f441027e 100644 --- a/debug/accuracy_tools/api_accuracy_checker/run_ut/run_ut.py +++ b/debug/accuracy_tools/api_accuracy_checker/run_ut/run_ut.py @@ -16,13 +16,16 @@ from api_accuracy_checker.hook_module.wrap_torch import TorchOPTemplate from ut_api_info import UtAPIInfo from api_accuracy_checker.common.config import msCheckerConfig +from ptdbg_ascend.src.python.ptdbg_ascend.common.file_check_util import FileOpen, FileCheckConst, FileChecker, \ + change_mode, check_file_suffix + NO_GRAD_APIS = ["hardtanh"] def init_environment(): cur_path = os.path.dirname(os.path.realpath(__file__)) yaml_path = os.path.join(cur_path, "../hook_module/support_wrap_ops.yaml") - with open(yaml_path, 'r') as f: + with FileOpen(yaml_path, 'r') as f: WrapFunctionalOps = yaml.safe_load(f).get('functional') for f in dir(torch.nn.functional): if f != "__name__": @@ -121,6 +124,8 @@ def run_ut(forward_file, backward_file, out_path, save_error_data): else: print_error_log(f"Run {api_full_name} UT Error: %s" % str(err)) compare.write_summary_csv((api_full_name, "SKIP", "SKIP", str(err))) + change_mode(compare.save_path, FileCheckConst.DATA_FILE_AUTHORITY) + change_mode(compare.detail_save_path, FileCheckConst.DATA_FILE_AUTHORITY) compare.print_pretest_result() @@ -208,8 +213,9 @@ def run_backward(api_full_name, args, backward_content, grad_index, npu_args, np def initialize_save_error_data(): - error_data_path = os.path.realpath(msCheckerConfig.error_data_path) - check_file_or_directory_path(error_data_path, True) + error_data_path_checker = FileChecker(msCheckerConfig.error_data_path, FileCheckConst.DIR, + ability=FileCheckConst.WRITE_ABLE) + error_data_path = error_data_path_checker.common_check() initialize_save_path(error_data_path, 'ut_error_data') @@ -246,9 +252,11 @@ def _run_ut(): raise NotImplementedError forward_file = os.path.realpath(args.forward_input_file) backward_file = os.path.realpath(args.backward_input_file) - if not forward_file.endswith(".json") or not backward_file.endswith(".json"): - raise ValueError("The forward_input_file and backward_input_file should be a json file!") + check_file_suffix(forward_file, FileCheckConst.JSON_SUFFIX) + check_file_suffix(backward_file, FileCheckConst.JSON_SUFFIX) out_path = os.path.realpath(args.out_path) if args.out_path else "./" + out_path_checker = FileChecker(out_path, FileCheckConst.DIR, ability=FileCheckConst.WRITE_ABLE) + out_path = out_path_checker.common_check() save_error_data = args.save_error_data if save_error_data: initialize_save_error_data() @@ -264,6 +272,7 @@ class UtDataInfo: self.grad_in = grad_in self.in_fwd_data_list = in_fwd_data_list + if __name__ == '__main__': _run_ut() print_info_log("UT task completed.") diff --git a/debug/accuracy_tools/ptdbg_ascend/__init__.py b/debug/accuracy_tools/ptdbg_ascend/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..8400fd5ecd1246eaee795cebfccfacc80a94f08c --- /dev/null +++ b/debug/accuracy_tools/ptdbg_ascend/__init__.py @@ -0,0 +1,14 @@ +# Copyright (c) 2023, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/common/file_check_util.py b/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/common/file_check_util.py new file mode 100644 index 0000000000000000000000000000000000000000..b8f7f4ed2dfaabf5c20ddce46ee606f96bfaa6f1 --- /dev/null +++ b/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/common/file_check_util.py @@ -0,0 +1,294 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +# Copyright (C) 2022-2023. Huawei Technologies Co., Ltd. All rights reserved. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +import os +import re + +from .utils import print_warn_log, print_error_log + + +class FileCheckConst: + """ + Class for file check const + """ + READ_ABLE = "read" + WRITE_ABLE = "write" + READ_WRITE_ABLE = "read and write" + DIRECTORY_LENGTH = 4096 + FILE_NAME_LENGTH = 255 + FILE_VALID_PATTERN = r"^[a-zA-Z0-9_.:/-]+$" + PKL_SUFFIX = ".pkl" + NUMPY_SUFFIX = ".npy" + JSON_SUFFIX = ".json" + PT_SUFFIX = ".pt" + MAX_PKL_SIZE = 1 * 1024 * 1024 * 1024 + MAX_NUMPY_SIZE = 10 * 1024 * 1024 * 1024 + MAX_JSON_SIZE = 1 * 1024 * 1024 * 1024 + MAX_PT_SIZE = 10 * 1024 * 1024 * 1024 + DIR = "dir" + FILE = "file" + DATA_DIR_AUTHORITY = 0o750 + DATA_FILE_AUTHORITY = 0o640 + + +class FileCheckException(Exception): + """ + Class for File Check Exception + """ + NONE_ERROR = 0 + INVALID_PATH_ERROR = 1 + INVALID_FILE_TYPE_ERROR = 2 + INVALID_PARAM_ERROR = 3 + INVALID_PERMISSION_ERROR = 3 + + def __init__(self, code, error_info: str = ""): + super(FileCheckException, self).__init__() + self.code = code + self.error_info = error_info + + def __str__(self): + return self.error_info + + +class FileChecker: + """ + The class for check file. + + Attributes: + file_path: The file or dictionary path to be verified. + path_type: file or dictionary + ability(str): FileCheckConst.WRITE_ABLE or FileCheckConst.READ_ABLE to set file has writability or readability + file_type(str): The correct file type for file + """ + def __init__(self, file_path, path_type, ability=None, file_type=None): + self.file_path = file_path + self.path_type = self._check_path_type(path_type) + self.ability = ability + self.file_type = file_type + + @staticmethod + def _check_path_type(path_type): + if path_type not in [FileCheckConst.DIR, FileCheckConst.FILE]: + print_error_log(f'The path_type must be {FileCheckConst.DIR} or {FileCheckConst.FILE}.') + raise FileCheckException(FileCheckException.INVALID_PARAM_ERROR) + return path_type + + def common_check(self): + """ + 功能:用户校验基本文件权限:软连接、文件长度、是否存在、读写权限、文件属组、文件特殊字符 + 注意:文件后缀的合法性,非通用操作,可使用其他独立接口实现 + """ + check_link(self.file_path) + check_path_length(self.file_path) + check_path_exists(self.file_path) + self.check_path_ability() + check_path_owner_consistent(self.file_path) + check_path_pattern_vaild(self.file_path) + check_common_file_size(self.file_path) + check_file_suffix(self.file_path, self.file_type) + return os.path.realpath(self.file_path) + + def check_path_ability(self): + if self.ability == FileCheckConst.WRITE_ABLE: + check_path_writability(self.file_path) + if self.ability == FileCheckConst.READ_ABLE: + check_path_readability(self.file_path) + if self.ability == FileCheckConst.READ_WRITE_ABLE: + check_path_readability(self.file_path) + check_path_writability(self.file_path) + + +class FileOpen: + """ + The class for open file by a safe way. + + Attributes: + file_path: The file or dictionary path to be opened. + mode(str): The file open mode + """ + SUPPORT_READ_MODE = ["r", "rb"] + SUPPORT_WRITE_MODE = ["w", "wb", "a", "ab"] + SUPPORT_READ_WRITE_MODE = ["r+", "rb+", "w+", "wb+", "a+", "ab+"] + + def __init__(self, file_path, mode): + self.file_path = file_path + self.mode = mode + self._handle = None + + def __enter__(self): + self.check_file_path() + self._handle = open(self.file_path, self.mode) + return self._handle + + def __exit__(self, exc_type, exc_val, exc_tb): + if self._handle: + self._handle.close() + + def check_file_path(self): + support_mode = self.SUPPORT_READ_MODE + self.SUPPORT_WRITE_MODE + self.SUPPORT_READ_WRITE_MODE + if self.mode not in support_mode: + print_error_log("File open not support %s mode" % self.mode) + raise FileCheckException(FileCheckException.INVALID_PARAM_ERROR) + check_link(self.file_path) + check_path_length(self.file_path) + self.check_ability_and_owner() + check_path_pattern_vaild(self.file_path) + if os.path.exists(self.file_path): + check_common_file_size(self.file_path) + + def check_ability_and_owner(self): + if self.mode in self.SUPPORT_READ_MODE: + check_path_exists(self.file_path) + check_path_readability(self.file_path) + check_path_owner_consistent(self.file_path) + if self.mode in self.SUPPORT_WRITE_MODE and os.path.exists(self.file_path): + check_path_writability(self.file_path) + check_path_owner_consistent(self.file_path) + if self.mode in self.SUPPORT_READ_WRITE_MODE and os.path.exists(self.file_path): + check_path_readability(self.file_path) + check_path_writability(self.file_path) + check_path_owner_consistent(self.file_path) + + +def check_link(path): + abs_path = os.path.abspath(path) + if os.path.islink(abs_path): + print_error_log('The file path {} is a soft link.'.format(path)) + raise FileCheckException(FileCheckException.INVALID_PATH_ERROR) + + +def check_path_length(path): + if len(os.path.realpath(path)) > FileCheckConst.DIRECTORY_LENGTH or \ + len(os.path.basename(path)) > FileCheckConst.FILE_NAME_LENGTH: + print_error_log('The file path length exceeds limit.') + raise FileCheckException(FileCheckException.INVALID_PATH_ERROR) + + +def check_path_exists(path): + real_path = os.path.realpath(path) + if not os.path.exists(real_path): + print_error_log('The file path %s does not exist.' % path) + raise FileCheckException(FileCheckException.INVALID_PATH_ERROR) + + +def check_path_readability(path): + real_path = os.path.realpath(path) + if not os.access(real_path, os.R_OK): + print_error_log('The file path %s is not readable.' % path) + raise FileCheckException(FileCheckException.INVALID_PERMISSION_ERROR) + + +def check_path_writability(path): + real_path = os.path.realpath(path) + if not os.access(real_path, os.W_OK): + print_error_log('The file path %s is not writable.' % path) + raise FileCheckException(FileCheckException.INVALID_PERMISSION_ERROR) + + +def _user_interactive_confirm(message): + while True: + check_message = input(message + " Enter 'c' to continue or enter 'e' to exit: ") + if check_message == "c": + break + elif check_message == "e": + print_warn_log("User canceled.") + raise FileCheckException(FileCheckException.INVALID_PATH_ERROR) + else: + print("Input is error, please enter 'c' or 'e'.") + + +def check_path_owner_consistent(path): + real_path = os.path.realpath(path) + file_owner = os.stat(real_path).st_uid + if file_owner != os.getuid(): + _user_interactive_confirm('The file path %s may be insecure because is does not belong to you.' + 'Do you want to continue?' % path) + + +def check_path_pattern_vaild(path): + if not re.match(FileCheckConst.FILE_VALID_PATTERN, os.path.realpath(path)): + print_error_log('The file path {} contains special characters.'.format(path)) + raise FileCheckException(FileCheckException.INVALID_PATH_ERROR) + + +def check_file_size(file_path, max_size): + real_path = os.path.realpath(file_path) + file_size = os.path.getsize(real_path) + if file_size >= max_size: + _user_interactive_confirm(f'The size of file path {file_path} exceeds {max_size} bytes.' + f'Do you want to continue?') + + +def check_common_file_size(file_path): + if os.path.isfile(file_path): + if file_path.endswith(FileCheckConst.PKL_SUFFIX): + check_file_size(file_path, FileCheckConst.MAX_PKL_SIZE) + if file_path.endswith(FileCheckConst.NUMPY_SUFFIX): + check_file_size(file_path, FileCheckConst.MAX_NUMPY_SIZE) + if file_path.endswith(FileCheckConst.JSON_SUFFIX): + check_file_size(file_path, FileCheckConst.MAX_JSON_SIZE) + if file_path.endswith(FileCheckConst.PT_SUFFIX): + check_file_size(file_path, FileCheckConst.MAX_PT_SIZE) + + +def check_file_suffix(file_path, file_suffix): + if file_suffix: + real_path = os.path.realpath(file_path) + if not real_path.endswith(file_suffix): + print_error_log(f"The {file_path} should be a {file_suffix} file!") + raise FileCheckException(FileCheckException.INVALID_FILE_TYPE_ERROR) + + +def check_file_type(file_path, file_type): + real_path = os.path.realpath(file_path) + if file_type == FileCheckConst.FILE: + if not os.path.isfile(real_path): + print_error_log(f"The {file_path} should be a file!") + raise FileCheckException(FileCheckException.INVALID_FILE_TYPE_ERROR) + if file_type == FileCheckConst.DIR: + if not os.path.isdir(real_path): + print_error_log(f"The {file_path} should be a dictionary!") + raise FileCheckException(FileCheckException.INVALID_FILE_TYPE_ERROR) + + +def create_directory(dir_path): + """ + Function Description: + creating a directory with specified permissions + Parameter: + dir_path: directory path + Exception Description: + when invalid data throw exception + """ + dir_path = os.path.realpath(dir_path) + if not os.path.exists(dir_path): + try: + os.makedirs(dir_path, mode=FileCheckConst.DATA_DIR_AUTHORITY) + except OSError as ex: + print_error_log( + 'Failed to create {}.Please check the path permission or disk space .{}'.format(dir_path, str(ex))) + raise FileCheckException(FileCheckException.INVALID_PATH_ERROR) + + +def change_mode(path, mode): + if not os.path.exists(path) or os.path.islink(path): + return + try: + os.chmod(path, mode) + except PermissionError as ex: + print_error_log('Failed to change {} authority. {}'.format(path, str(ex))) + raise FileCheckException(FileCheckException.INVALID_PERMISSION_ERROR) + diff --git a/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/common/version.py b/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/common/version.py new file mode 100644 index 0000000000000000000000000000000000000000..8aa4e6974e74ce2b6037a2e81569a71561791c2b --- /dev/null +++ b/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/common/version.py @@ -0,0 +1 @@ +__version__ = '4.0' \ No newline at end of file