From 15013d5d8bc684263841c977dae2af7db4addb44 Mon Sep 17 00:00:00 2001 From: brightlyking Date: Mon, 23 Oct 2023 17:39:47 +0800 Subject: [PATCH 01/11] add file check util --- .../ptdbg_ascend/common/file_check_util.py | 288 ++++++++++++++++++ 1 file changed, 288 insertions(+) create mode 100644 debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/common/file_check_util.py diff --git a/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/common/file_check_util.py b/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/common/file_check_util.py new file mode 100644 index 0000000000..aeac1b0149 --- /dev/null +++ b/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/common/file_check_util.py @@ -0,0 +1,288 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +# Copyright (C) 2022-2023. Huawei Technologies Co., Ltd. All rights reserved. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +import os +import re + +from .utils import print_warn_log, print_error_log + + +class FileCheckConst: + """ + Class for file check const + """ + READ_ABLE = "read" + WRITE_ABLE = "write" + READ_WRITE_ABLE = "read and write" + DIRECTORY_LENGTH = 4096 + FILE_NAME_LENGTH = 255 + FILE_VALID_PATTERN = r"^[a-zA-Z0-9_.:/-]+$" + PKL_SUFFIX = ".pkl" + NUMPY_SUFFIX = ".npy" + JSON_SUFFIX = ".json" + MAX_PKL_SIZE = 1 * 1024 * 1024 * 1024 + MAX_NUMPY_SIZE = 10 * 1024 * 1024 * 1024 + MAX_JSON_SIZE = 1 * 1024 * 1024 * 1024 + DIR = "dir" + FILE = "file" + DATA_DIR_AUTHORITY = 0o750 + DATA_FILE_AUTHORITY = 0o640 + + +class FileCheckException(Exception): + """ + Class for File Check Exception + """ + NONE_ERROR = 0 + INVALID_PATH_ERROR = 1 + INVALID_FILE_TYPE_ERROR = 2 + INVALID_PARAM_ERROR = 3 + INVALID_PERMISSION_ERROR = 3 + + def __init__(self, code, error_info: str = ""): + super(FileCheckException, self).__init__() + self.code = code + self.error_info = error_info + + def __str__(self): + return self.error_info + + +class FileChecker: + """ + The class for check file. + + Attributes: + file_path: The file or dictionary path to be verified. + ability(str): FileCheckConst.WRITE_ABLE or FileCheckConst.READ_ABLE to set file has writability or readability + file_type(str): The correct file type for file + """ + def __init__(self, file_path, path_type, ability=None, file_type=None): + self.file_path = file_path + self.path_type = self._check_path_type(path_type) + self.ability = ability + self.file_type = file_type + + @staticmethod + def _check_path_type(path_type): + if path_type not in [FileCheckConst.DIR, FileCheckConst.FILE]: + print_error_log(f'The path_type must be {FileCheckConst.DIR} or {FileCheckConst.FILE}.') + raise FileCheckException(FileCheckException.INVALID_PARAM_ERROR) + return path_type + + def common_check(self): + """ + 功能：用户校验基本文件权限：软连接、文件长度、是否存在、读写权限、文件属组、文件特殊字符 + 注意：文件后缀的合法性，非通用操作，可使用其他独立接口实现 + """ + check_link(self.file_path) + check_path_length(self.file_path) + check_path_exists(self.file_path) + self.check_path_ability() + check_path_owner_consistent(self.file_path) + check_path_pattern_vaild(self.file_path) + check_common_file_size(self.file_path) + check_file_suffix(self.file_path, self.file_type) + return os.path.realpath(self.file_path) + + def check_path_ability(self): + if self.ability == FileCheckConst.WRITE_ABLE: + check_path_writability(self.file_path) + if self.ability == FileCheckConst.READ_ABLE: + check_path_readability(self.file_path) + if self.ability == FileCheckConst.READ_WRITE_ABLE: + check_path_readability(self.file_path) + check_path_writability(self.file_path) + + +class FileOpen: + """ + The class for open file by a safe way. + + Attributes: + file_path: The file or dictionary path to be opened. + mode(str): The file open mode + """ + SUPPORT_READ_MODE = ["r", "rb"] + SUPPORT_WRITE_MODE = ["w", "wb", "a", "ab"] + SUPPORT_READ_WRITE_MODE = ["r+", "rb+", "w+", "wb+", "a+", "ab+"] + + def __init__(self, file_path, mode): + self.file_path = file_path + self.mode = mode + self._handle = None + + def __enter__(self): + self.check_file_path() + self._handle = open(self.file_path, self.mode) + return self._handle + + def __exit__(self, exc_type, exc_val, exc_tb): + if self._handle: + self._handle.close() + + def check_file_path(self): + support_mode = self.SUPPORT_READ_MODE + self.SUPPORT_WRITE_MODE + self.SUPPORT_READ_WRITE_MODE + if self.mode not in support_mode: + print_error_log("File open not support %s mode" % self.mode) + check_link(self.file_path) + check_path_length(self.file_path) + self.check_ability_and_owner() + check_path_pattern_vaild(self.file_path) + if os.path.exists(self.file_path): + check_common_file_size(self.file_path) + + def check_ability_and_owner(self): + if self.mode in self.SUPPORT_READ_MODE: + check_path_exists(self.file_path) + check_path_readability(self.file_path) + check_path_owner_consistent(self.file_path) + if self.mode in self.SUPPORT_WRITE_MODE and os.path.exists(self.file_path): + check_path_writability(self.file_path) + check_path_owner_consistent(self.file_path) + if self.mode in self.SUPPORT_READ_WRITE_MODE and os.path.exists(self.file_path): + check_path_readability(self.file_path) + check_path_writability(self.file_path) + check_path_owner_consistent(self.file_path) + + +def check_link(path): + abs_path = os.path.abspath(path) + if os.path.islink(abs_path): + print_error_log('The file path {} is a soft link.'.format(path)) + raise FileCheckException(FileCheckException.INVALID_PATH_ERROR) + + +def check_path_length(path): + if len(os.path.realpath(path)) > FileCheckConst.DIRECTORY_LENGTH or \ + len(os.path.basename(path)) > FileCheckConst.FILE_NAME_LENGTH: + print_error_log('The file path length exceeds limit.') + raise FileCheckException(FileCheckException.INVALID_PATH_ERROR) + + +def check_path_exists(path): + real_path = os.path.realpath(path) + if not os.path.exists(real_path): + print_error_log('The file path %s does not exist.' % path) + raise FileCheckException(FileCheckException.INVALID_PATH_ERROR) + + +def check_path_readability(path): + real_path = os.path.realpath(path) + if not os.access(real_path, os.R_OK): + print_error_log('The file path %s is not readable.' % path) + raise FileCheckException(FileCheckException.INVALID_PERMISSION_ERROR) + + +def check_path_writability(path): + real_path = os.path.realpath(path) + if not os.access(real_path, os.W_OK): + print_error_log('The file path %s is not writable.' % path) + raise FileCheckException(FileCheckException.INVALID_PERMISSION_ERROR) + + +def _user_interactive_confirm(message): + while True: + check_message = input(message + " Enter 'c' to continue or enter 'e' to exit: ") + if check_message == "c": + break + elif check_message == "e": + print_warn_log("User canceled.") + raise FileCheckException(FileCheckException.INVALID_PATH_ERROR) + else: + print("Input is error, please enter 'c' or 'e'.") + + +def check_path_owner_consistent(path): + real_path = os.path.realpath(path) + file_owner = os.stat(real_path).st_uid + if file_owner != os.getuid(): + _user_interactive_confirm('The file path %s may be insecure because is does not belong to you.' + 'Do you want to continue?' % path) + + +def check_path_pattern_vaild(path): + if not re.match(FileCheckConst.FILE_VALID_PATTERN, os.path.realpath(path)): + print_error_log('The file path {} contains special characters.'.format(path)) + raise FileCheckException(FileCheckException.INVALID_PATH_ERROR) + + +def check_file_size(file_path, max_size): + real_path = os.path.realpath(file_path) + file_size = os.path.getsize(real_path) + if file_size >= max_size: + _user_interactive_confirm(f'The size of file path {file_path} exceeds {max_size} bytes.' + f'Do you want to continue?') + + +def check_common_file_size(file_path): + if os.path.isfile(file_path): + if file_path.endswith(FileCheckConst.PKL_SUFFIX): + check_file_size(file_path, FileCheckConst.MAX_PKL_SIZE) + if file_path.endswith(FileCheckConst.NUMPY_SUFFIX): + check_file_size(file_path, FileCheckConst.MAX_NUMPY_SIZE) + if file_path.endswith(FileCheckConst.JSON_SUFFIX): + check_file_size(file_path, FileCheckConst.MAX_JSON_SIZE) + + +def check_file_suffix(file_path, file_suffix): + if file_suffix: + real_path = os.path.realpath(file_path) + if not real_path.endswith(file_suffix): + print_error_log(f"The {file_path} should be a {file_suffix} file!") + raise FileCheckException(FileCheckException.INVALID_FILE_TYPE_ERROR) + + +def check_file_type(file_path, file_type): + real_path = os.path.realpath(file_path) + if file_type == FileCheckConst.FILE: + if not os.path.isfile(real_path): + print_error_log(f"The {file_path} should be a file!") + raise FileCheckException(FileCheckException.INVALID_FILE_TYPE_ERROR) + if file_type == FileCheckConst.DIR: + if not os.path.isdir(real_path): + print_error_log(f"The {file_path} should be a dictionary!") + raise FileCheckException(FileCheckException.INVALID_FILE_TYPE_ERROR) + + +def create_directory(dir_path): + """ + Function Description: + creating a directory with specified permissions + Parameter: + dir_path: directory path + Exception Description: + when invalid data throw exception + """ + dir_path = os.path.realpath(dir_path) + if not os.path.exists(dir_path): + try: + os.makedirs(dir_path, mode=FileCheckConst.DATA_DIR_AUTHORITY) + except OSError as ex: + print_error_log( + 'Failed to create {}.Please check the path permission or disk space .{}'.format(dir_path, str(ex))) + raise FileCheckException(FileCheckException.INVALID_PATH_ERROR) + + +def change_mode(path, mode): + if not os.path.exists(path) or os.path.islink(path): + return + try: + os.chmod(path, mode) + except PermissionError as ex: + print_error_log('Failed to change {} authority. {}'.format(path, str(ex))) + raise FileCheckException(FileCheckException.INVALID_PERMISSION_ERROR) + -- Gitee From ea1f818a9ee907a98d2793e954d69f8d73aae37c Mon Sep 17 00:00:00 2001 From: brightlyking Date: Mon, 23 Oct 2023 19:12:02 +0800 Subject: [PATCH 02/11] add Exception --- .../src/python/ptdbg_ascend/common/file_check_util.py | 1 + 1 file changed, 1 insertion(+) diff --git a/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/common/file_check_util.py b/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/common/file_check_util.py index aeac1b0149..dcca5ef37b 100644 --- a/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/common/file_check_util.py +++ b/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/common/file_check_util.py @@ -138,6 +138,7 @@ class FileOpen: support_mode = self.SUPPORT_READ_MODE + self.SUPPORT_WRITE_MODE + self.SUPPORT_READ_WRITE_MODE if self.mode not in support_mode: print_error_log("File open not support %s mode" % self.mode) + raise FileCheckException(FileCheckException.INVALID_PARAM_ERROR) check_link(self.file_path) check_path_length(self.file_path) self.check_ability_and_owner() -- Gitee From 852a683a49f9ab69cc9b9227cbb15b804cf78d13 Mon Sep 17 00:00:00 2001 From: brightlyking Date: Mon, 23 Oct 2023 20:31:03 +0800 Subject: [PATCH 03/11] add accuracy_tool file check --- .../api_accuracy_checker/common/config.py | 2 +- .../api_accuracy_checker/common/utils.py | 18 ++++++++----- .../api_accuracy_checker/dump/info_dump.py | 7 ++++-- .../api_accuracy_checker/dump/utils.py | 15 ----------- .../run_ut/run_overflow_check.py | 13 +++++----- .../api_accuracy_checker/run_ut/run_ut.py | 25 +++++++++++++------ debug/accuracy_tools/ptdbg_ascend/__init__.py | 14 +++++++++++ .../ptdbg_ascend/common/file_check_util.py | 1 + 8 files changed, 58 insertions(+), 37 deletions(-) delete mode 100644 debug/accuracy_tools/api_accuracy_checker/dump/utils.py create mode 100644 debug/accuracy_tools/ptdbg_ascend/__init__.py diff --git a/debug/accuracy_tools/api_accuracy_checker/common/config.py b/debug/accuracy_tools/api_accuracy_checker/common/config.py index 02d40973c1..d07ddb7d80 100644 --- a/debug/accuracy_tools/api_accuracy_checker/common/config.py +++ b/debug/accuracy_tools/api_accuracy_checker/common/config.py @@ -20,7 +20,7 @@ class Config: 'target_iter': int, 'precision': int } - if not isinstance(value, validators[key]): + if not isinstance(value, validators.get(key)): raise ValueError(f"{key} must be {validators[key].__name__} type") if key == 'target_iter' and value < 0: raise ValueError("target_iter must be greater than 0") diff --git a/debug/accuracy_tools/api_accuracy_checker/common/utils.py b/debug/accuracy_tools/api_accuracy_checker/common/utils.py index 065c0fe46d..b14ff6b8b6 100644 --- a/debug/accuracy_tools/api_accuracy_checker/common/utils.py +++ b/debug/accuracy_tools/api_accuracy_checker/common/utils.py @@ -29,6 +29,9 @@ import numpy as np import torch import csv +from ptdbg_ascend.src.python.ptdbg_ascend.common.file_check_util import FileCheckConst, FileChecker +from ptdbg_ascend.src.python.ptdbg_ascend.common import file_check_util + try: import torch_npu except ImportError: @@ -86,7 +89,7 @@ class Const: API_PATTERN = r"^[A-Za-z0-9]+[_]+([A-Za-z0-9]+[_]*[A-Za-z0-9]+)[_]+[0-9]+[_]+[A-Za-z0-9]+" WRITE_FLAGS = os.O_WRONLY | os.O_CREAT WRITE_MODES = stat.S_IWUSR | stat.S_IRUSR - + RAISE_PRECISION = { "torch.float16" : "torch.float32", "torch.bfloat16" : "torch.float32", @@ -377,7 +380,7 @@ def create_directory(dir_path): """ if not os.path.exists(dir_path): try: - os.makedirs(dir_path, mode=0o700) + os.makedirs(dir_path, mode=FileCheckConst.DATA_DIR_AUTHORITY) except OSError as ex: print_error_log( 'Failed to create {}.Please check the path permission or disk space .{}'.format(dir_path, str(ex))) @@ -606,12 +609,15 @@ def initialize_save_path(save_path, dir_name): if os.path.exists(data_path): raise ValueError(f"file {data_path} already exists, please remove it first") else: - os.mkdir(data_path, mode = 0o750) - check_file_or_directory_path(data_path, True) + os.mkdir(data_path, mode=FileCheckConst.DATA_DIR_AUTHORITY) + data_path_checker = FileChecker(data_path, FileCheckConst.DIR) + data_path_checker.common_check() + def write_pt(file_path, tensor): if os.path.exists(file_path): raise ValueError(f"File {file_path} already exists") torch.save(tensor, file_path) - full_path = os.path.abspath(file_path) - return full_path \ No newline at end of file + full_path = os.path.realpath(file_path) + file_check_util.change_mode(full_path, FileCheckConst.DATA_FILE_AUTHORITY) + return full_path diff --git a/debug/accuracy_tools/api_accuracy_checker/dump/info_dump.py b/debug/accuracy_tools/api_accuracy_checker/dump/info_dump.py index f2a96bd0fa..bd17df8128 100644 --- a/debug/accuracy_tools/api_accuracy_checker/dump/info_dump.py +++ b/debug/accuracy_tools/api_accuracy_checker/dump/info_dump.py @@ -7,6 +7,9 @@ from .api_info import ForwardAPIInfo, BackwardAPIInfo from ..common.utils import check_file_or_directory_path, initialize_save_path from ..common.config import msCheckerConfig +from ptdbg_ascend.src.python.ptdbg_ascend.common.file_check_util import FileOpen, FileCheckConst, FileChecker, \ + change_mode + lock = threading.Lock() def write_api_info_json(api_info): @@ -49,8 +52,8 @@ def write_json(file_path, data, indent=None): def initialize_output_json(): - dump_path = os.path.realpath(msCheckerConfig.dump_path) - check_file_or_directory_path(dump_path, True) + dump_path_checker = FileChecker(msCheckerConfig.dump_path, FileCheckConst.DIR) + dump_path = dump_path_checker.common_check() files = ['forward_info.json', 'backward_info.json', 'stack_info.json'] if msCheckerConfig.real_data: initialize_save_path(dump_path, 'forward_real_data') diff --git a/debug/accuracy_tools/api_accuracy_checker/dump/utils.py b/debug/accuracy_tools/api_accuracy_checker/dump/utils.py deleted file mode 100644 index 93af6f0981..0000000000 --- a/debug/accuracy_tools/api_accuracy_checker/dump/utils.py +++ /dev/null @@ -1,15 +0,0 @@ -import os -import numpy as np - - -def create_folder(path): - if not os.path.exists(path): - os.makedirs(path, mode=0o750) - return path - -def write_npy(file_path, tensor): - if os.path.exists(file_path): - raise ValueError(f"File {file_path} already exists") - np.save(file_path, tensor) - full_path = os.path.abspath(file_path) - return full_path diff --git a/debug/accuracy_tools/api_accuracy_checker/run_ut/run_overflow_check.py b/debug/accuracy_tools/api_accuracy_checker/run_ut/run_overflow_check.py index 7c0fa0f6a6..725b5fd87b 100644 --- a/debug/accuracy_tools/api_accuracy_checker/run_ut/run_overflow_check.py +++ b/debug/accuracy_tools/api_accuracy_checker/run_ut/run_overflow_check.py @@ -9,6 +9,7 @@ from api_accuracy_checker.run_ut.run_ut import exec_api, generate_npu_params, ru from api_accuracy_checker.common.utils import print_info_log, print_warn_log, get_json_contents, api_info_preprocess, \ print_error_log +from ptdbg_ascend.src.python.ptdbg_ascend.common.file_check_util import FileCheckConst, FileChecker NO_GRAD_APIS = ["hardtanh"] @@ -133,14 +134,14 @@ def _run_overflow_check(): args = parser.parse_args(sys.argv[1:]) torch.npu.set_compile_mode(jit_compile=args.jit_compile) npu_device = "npu:" + str(args.device_id) - forward_file = os.path.realpath(args.forward_input_file) + forward_file_checker = FileChecker(args.forward_input_file, FileCheckConst.FILE, ability=FileCheckConst.READ_ABLE, + file_type=FileCheckConst.JSON_SUFFIX) + forward_file = forward_file_checker.common_check() backward_file = "" if args.backward_input_file: - backward_file = os.path.realpath(args.backward_input_file) - if not backward_file.endswith(".json"): - raise ValueError("The backward_input_file should be a json file!") - if not forward_file.endswith(".json"): - raise ValueError("The forward_input_file should be a json file!") + backward_file_checker = FileChecker(args.backward_input_file, FileCheckConst.FILE, + ability=FileCheckConst.READ_ABLE, file_type=FileCheckConst.JSON_SUFFIX) + backward_file = backward_file_checker.common_check() try: torch.npu.set_device(npu_device) except Exception: diff --git a/debug/accuracy_tools/api_accuracy_checker/run_ut/run_ut.py b/debug/accuracy_tools/api_accuracy_checker/run_ut/run_ut.py index 4cc27880c7..71daf0e906 100644 --- a/debug/accuracy_tools/api_accuracy_checker/run_ut/run_ut.py +++ b/debug/accuracy_tools/api_accuracy_checker/run_ut/run_ut.py @@ -16,13 +16,16 @@ from api_accuracy_checker.hook_module.wrap_torch import TorchOPTemplate from ut_api_info import UtAPIInfo from api_accuracy_checker.common.config import msCheckerConfig +from ptdbg_ascend.src.python.ptdbg_ascend.common.file_check_util import FileOpen, FileCheckConst, FileChecker, \ + change_mode + NO_GRAD_APIS = ["hardtanh"] def init_environment(): cur_path = os.path.dirname(os.path.realpath(__file__)) yaml_path = os.path.join(cur_path, "../hook_module/support_wrap_ops.yaml") - with open(yaml_path, 'r') as f: + with FileOpen(yaml_path, 'r') as f: WrapFunctionalOps = yaml.safe_load(f).get('functional') for f in dir(torch.nn.functional): if f != "__name__": @@ -121,6 +124,8 @@ def run_ut(forward_file, backward_file, out_path, save_error_data): else: print_error_log(f"Run {api_full_name} UT Error: %s" % str(err)) compare.write_summary_csv((api_full_name, "SKIP", "SKIP", str(err))) + change_mode(compare.save_path, FileCheckConst.DATA_FILE_AUTHORITY) + change_mode(compare.detail_save_path, FileCheckConst.DATA_FILE_AUTHORITY) compare.print_pretest_result() @@ -208,8 +213,9 @@ def run_backward(api_full_name, args, backward_content, grad_index, npu_args, np def initialize_save_error_data(): - error_data_path = os.path.realpath(msCheckerConfig.error_data_path) - check_file_or_directory_path(error_data_path, True) + error_data_path_checker = FileChecker(msCheckerConfig.error_data_path, FileCheckConst.DIR, + ability=FileCheckConst.WRITE_ABLE) + error_data_path = error_data_path_checker.common_check() initialize_save_path(error_data_path, 'ut_error_data') @@ -244,11 +250,15 @@ def _run_ut(): except Exception: print_error_log(f"Set NPU device id failed. device id is: {args.device_id}") raise NotImplementedError - forward_file = os.path.realpath(args.forward_input_file) - backward_file = os.path.realpath(args.backward_input_file) - if not forward_file.endswith(".json") or not backward_file.endswith(".json"): - raise ValueError("The forward_input_file and backward_input_file should be a json file!") + forward_file_checker = FileChecker(args.forward_input_file, FileCheckConst.FILE, ability=FileCheckConst.READ_ABLE, + file_type=FileCheckConst.JSON_SUFFIX) + forward_file = forward_file_checker.common_check() + backward_file_checker = FileChecker(args.backward_input_file, FileCheckConst.FILE, + ability=FileCheckConst.READ_ABLE, file_type=FileCheckConst.JSON_SUFFIX) + backward_file = backward_file_checker.common_check() out_path = os.path.realpath(args.out_path) if args.out_path else "./" + out_path_checker = FileChecker(out_path, FileCheckConst.DIR, ability=FileCheckConst.WRITE_ABLE) + out_path = out_path_checker.common_check() save_error_data = args.save_error_data if save_error_data: initialize_save_error_data() @@ -264,6 +274,7 @@ class UtDataInfo: self.grad_in = grad_in self.in_fwd_data_list = in_fwd_data_list + if __name__ == '__main__': _run_ut() print_info_log("UT task completed.") diff --git a/debug/accuracy_tools/ptdbg_ascend/__init__.py b/debug/accuracy_tools/ptdbg_ascend/__init__.py new file mode 100644 index 0000000000..8400fd5ecd --- /dev/null +++ b/debug/accuracy_tools/ptdbg_ascend/__init__.py @@ -0,0 +1,14 @@ +# Copyright (c) 2023, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/common/file_check_util.py b/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/common/file_check_util.py index dcca5ef37b..83d090142e 100644 --- a/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/common/file_check_util.py +++ b/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/common/file_check_util.py @@ -67,6 +67,7 @@ class FileChecker: Attributes: file_path: The file or dictionary path to be verified. + path_type: file or dictionary ability(str): FileCheckConst.WRITE_ABLE or FileCheckConst.READ_ABLE to set file has writability or readability file_type(str): The correct file type for file """ -- Gitee From e0167df3c898a71b7cdd1d51f60a9ce1d4167669 Mon Sep 17 00:00:00 2001 From: brightlyking Date: Tue, 24 Oct 2023 09:57:24 +0800 Subject: [PATCH 04/11] add accuracy_tool file check --- .../api_accuracy_checker/common/base_api.py | 1 - .../accuracy_tools/api_accuracy_checker/common/config.py | 3 ++- debug/accuracy_tools/api_accuracy_checker/common/utils.py | 8 ++++---- .../api_accuracy_checker/compare/save_result.py | 1 - .../accuracy_tools/api_accuracy_checker/dump/info_dump.py | 7 +++---- .../api_accuracy_checker/hook_module/wrap_functional.py | 3 ++- .../api_accuracy_checker/hook_module/wrap_tensor.py | 3 ++- .../api_accuracy_checker/hook_module/wrap_torch.py | 4 +++- .../api_accuracy_checker/run_ut/data_generate.py | 5 +++-- .../src/python/ptdbg_ascend/common/file_check_util.py | 4 ++++ 10 files changed, 23 insertions(+), 16 deletions(-) delete mode 100644 debug/accuracy_tools/api_accuracy_checker/compare/save_result.py diff --git a/debug/accuracy_tools/api_accuracy_checker/common/base_api.py b/debug/accuracy_tools/api_accuracy_checker/common/base_api.py index 627bd76976..a4f9a4ea83 100644 --- a/debug/accuracy_tools/api_accuracy_checker/common/base_api.py +++ b/debug/accuracy_tools/api_accuracy_checker/common/base_api.py @@ -41,7 +41,6 @@ class BaseAPIInfo: raise NotImplementedError(msg) return out - def analyze_tensor(self, arg): single_arg = {} if not self.is_save_data: diff --git a/debug/accuracy_tools/api_accuracy_checker/common/config.py b/debug/accuracy_tools/api_accuracy_checker/common/config.py index d07ddb7d80..821842ddd9 100644 --- a/debug/accuracy_tools/api_accuracy_checker/common/config.py +++ b/debug/accuracy_tools/api_accuracy_checker/common/config.py @@ -1,11 +1,12 @@ import yaml import os from api_accuracy_checker.common.utils import check_file_or_directory_path +from ptdbg_ascend.src.python.ptdbg_ascend.common.file_check_util import FileOpen class Config: def __init__(self, yaml_file): check_file_or_directory_path(yaml_file, False) - with open(yaml_file, 'r') as file: + with FileOpen(yaml_file, 'r') as file: config = yaml.safe_load(file) self.config = {key: self.validate(key, value) for key, value in config.items()} diff --git a/debug/accuracy_tools/api_accuracy_checker/common/utils.py b/debug/accuracy_tools/api_accuracy_checker/common/utils.py index b14ff6b8b6..a59fc4efa0 100644 --- a/debug/accuracy_tools/api_accuracy_checker/common/utils.py +++ b/debug/accuracy_tools/api_accuracy_checker/common/utils.py @@ -29,7 +29,7 @@ import numpy as np import torch import csv -from ptdbg_ascend.src.python.ptdbg_ascend.common.file_check_util import FileCheckConst, FileChecker +from ptdbg_ascend.src.python.ptdbg_ascend.common.file_check_util import FileCheckConst, FileChecker, FileOpen from ptdbg_ascend.src.python.ptdbg_ascend.common import file_check_util try: @@ -195,12 +195,12 @@ class DumpException(CompareException): pass def read_json(file): - with open(file, 'r') as f: + with FileOpen(file, 'r') as f: obj = json.load(f) return obj def write_csv(data, filepath): - with open(filepath, 'a') as f: + with FileOpen(filepath, 'a') as f: writer = csv.writer(f) writer.writerows(data) @@ -517,7 +517,7 @@ def get_json_contents(file_path): def get_file_content_bytes(file): check_input_file_valid(file) - with open(file, 'rb') as file_handle: + with FileOpen(file, 'rb') as file_handle: return file_handle.read() diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/save_result.py b/debug/accuracy_tools/api_accuracy_checker/compare/save_result.py deleted file mode 100644 index 06191a4a54..0000000000 --- a/debug/accuracy_tools/api_accuracy_checker/compare/save_result.py +++ /dev/null @@ -1 +0,0 @@ -# 比对表格输出，并将失败api数据进行保存 \ No newline at end of file diff --git a/debug/accuracy_tools/api_accuracy_checker/dump/info_dump.py b/debug/accuracy_tools/api_accuracy_checker/dump/info_dump.py index bd17df8128..f915d779a8 100644 --- a/debug/accuracy_tools/api_accuracy_checker/dump/info_dump.py +++ b/debug/accuracy_tools/api_accuracy_checker/dump/info_dump.py @@ -7,8 +7,7 @@ from .api_info import ForwardAPIInfo, BackwardAPIInfo from ..common.utils import check_file_or_directory_path, initialize_save_path from ..common.config import msCheckerConfig -from ptdbg_ascend.src.python.ptdbg_ascend.common.file_check_util import FileOpen, FileCheckConst, FileChecker, \ - change_mode +from ptdbg_ascend.src.python.ptdbg_ascend.common.file_check_util import FileOpen, FileCheckConst, FileChecker lock = threading.Lock() @@ -30,10 +29,10 @@ def write_api_info_json(api_info): def write_json(file_path, data, indent=None): check_file_or_directory_path(os.path.dirname(file_path),True) if not os.path.exists(file_path): - with open(file_path, 'w') as f: + with FileOpen(file_path, 'w') as f: f.write("{\n}") lock.acquire() - with open(file_path, 'a+') as f: + with FileOpen(file_path, 'a+') as f: fcntl.flock(f, fcntl.LOCK_EX) try: f.seek(0, os.SEEK_END) diff --git a/debug/accuracy_tools/api_accuracy_checker/hook_module/wrap_functional.py b/debug/accuracy_tools/api_accuracy_checker/hook_module/wrap_functional.py index dbe27a134c..a1c9af127f 100644 --- a/debug/accuracy_tools/api_accuracy_checker/hook_module/wrap_functional.py +++ b/debug/accuracy_tools/api_accuracy_checker/hook_module/wrap_functional.py @@ -22,10 +22,11 @@ import yaml from api_accuracy_checker.hook_module.hook_module import HOOKModule from api_accuracy_checker.common.utils import torch_device_guard +from ptdbg_ascend.src.python.ptdbg_ascend.common.file_check_util import FileOpen cur_path = os.path.dirname(os.path.realpath(__file__)) yaml_path = os.path.join(cur_path, "support_wrap_ops.yaml") -with open(yaml_path, 'r') as f: +with FileOpen(yaml_path, 'r') as f: WrapFunctionalOps = yaml.safe_load(f).get('functional') for f in dir(torch.nn.functional): diff --git a/debug/accuracy_tools/api_accuracy_checker/hook_module/wrap_tensor.py b/debug/accuracy_tools/api_accuracy_checker/hook_module/wrap_tensor.py index 93d92923c6..547955ec18 100644 --- a/debug/accuracy_tools/api_accuracy_checker/hook_module/wrap_tensor.py +++ b/debug/accuracy_tools/api_accuracy_checker/hook_module/wrap_tensor.py @@ -22,10 +22,11 @@ import yaml from api_accuracy_checker.hook_module.hook_module import HOOKModule from api_accuracy_checker.common.utils import torch_device_guard +from ptdbg_ascend.src.python.ptdbg_ascend.common.file_check_util import FileOpen cur_path = os.path.dirname(os.path.realpath(__file__)) yaml_path = os.path.join(cur_path, "support_wrap_ops.yaml") -with open(yaml_path, 'r') as f: +with FileOpen(yaml_path, 'r') as f: WrapTensorOps = yaml.safe_load(f).get('tensor') diff --git a/debug/accuracy_tools/api_accuracy_checker/hook_module/wrap_torch.py b/debug/accuracy_tools/api_accuracy_checker/hook_module/wrap_torch.py index 07a037b779..c42ada0ed4 100644 --- a/debug/accuracy_tools/api_accuracy_checker/hook_module/wrap_torch.py +++ b/debug/accuracy_tools/api_accuracy_checker/hook_module/wrap_torch.py @@ -23,9 +23,11 @@ import yaml from api_accuracy_checker.hook_module.hook_module import HOOKModule from api_accuracy_checker.common.utils import torch_device_guard +from ptdbg_ascend.src.python.ptdbg_ascend.common.file_check_util import FileOpen + cur_path = os.path.dirname(os.path.realpath(__file__)) yaml_path = os.path.join(cur_path, "support_wrap_ops.yaml") -with open(yaml_path, 'r') as f: +with FileOpen(yaml_path, 'r') as f: WrapTorchOps = yaml.safe_load(f).get('torch') diff --git a/debug/accuracy_tools/api_accuracy_checker/run_ut/data_generate.py b/debug/accuracy_tools/api_accuracy_checker/run_ut/data_generate.py index 3946c34014..9f7f15ec7f 100644 --- a/debug/accuracy_tools/api_accuracy_checker/run_ut/data_generate.py +++ b/debug/accuracy_tools/api_accuracy_checker/run_ut/data_generate.py @@ -21,6 +21,7 @@ import numpy as np from api_accuracy_checker.common.utils import Const, check_file_or_directory_path, check_object_type, print_warn_log, print_error_log, \ CompareException +from ptdbg_ascend.src.python.ptdbg_ascend.common.file_check_util import FileCheckConst, FileChecker, FileOpen TORCH_TYPE = ["torch.device", "torch.dtype"] TENSOR_DATA_LIST = ["torch.Tensor", "torch.nn.parameter.Parameter"] @@ -65,8 +66,8 @@ def gen_real_tensor(data_path, convert_type): data_path: API data path convert_type: convert ori_type to dist_type flag. """ - data_path = os.path.realpath(data_path) - check_file_or_directory_path(data_path) + data_path_checker = FileChecker(data_path, FileCheckConst.FILE) + data_path = data_path_checker.common_check() if not data_path.endswith('.pt') and not data_path.endswith('.npy'): print_error_log(f"The file: {data_path} is not a pt or numpy file.") raise CompareException.INVALID_FILE_ERROR diff --git a/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/common/file_check_util.py b/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/common/file_check_util.py index 83d090142e..b8f7f4ed2d 100644 --- a/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/common/file_check_util.py +++ b/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/common/file_check_util.py @@ -33,9 +33,11 @@ class FileCheckConst: PKL_SUFFIX = ".pkl" NUMPY_SUFFIX = ".npy" JSON_SUFFIX = ".json" + PT_SUFFIX = ".pt" MAX_PKL_SIZE = 1 * 1024 * 1024 * 1024 MAX_NUMPY_SIZE = 10 * 1024 * 1024 * 1024 MAX_JSON_SIZE = 1 * 1024 * 1024 * 1024 + MAX_PT_SIZE = 10 * 1024 * 1024 * 1024 DIR = "dir" FILE = "file" DATA_DIR_AUTHORITY = 0o750 @@ -238,6 +240,8 @@ def check_common_file_size(file_path): check_file_size(file_path, FileCheckConst.MAX_NUMPY_SIZE) if file_path.endswith(FileCheckConst.JSON_SUFFIX): check_file_size(file_path, FileCheckConst.MAX_JSON_SIZE) + if file_path.endswith(FileCheckConst.PT_SUFFIX): + check_file_size(file_path, FileCheckConst.MAX_PT_SIZE) def check_file_suffix(file_path, file_suffix): -- Gitee From 63ca9afafdd67b4afff5ee6d4e9124da80c03a45 Mon Sep 17 00:00:00 2001 From: brightlyking Date: Tue, 24 Oct 2023 10:25:58 +0800 Subject: [PATCH 05/11] add accuracy_tool file check --- debug/accuracy_tools/api_accuracy_checker/common/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/api_accuracy_checker/common/config.py b/debug/accuracy_tools/api_accuracy_checker/common/config.py index 821842ddd9..a43212dac4 100644 --- a/debug/accuracy_tools/api_accuracy_checker/common/config.py +++ b/debug/accuracy_tools/api_accuracy_checker/common/config.py @@ -35,7 +35,7 @@ class Config: def __str__(self): return '\n'.join(f"{key}={value}" for key, value in self.config.items()) - def update_config(self, dump_path, real_data=False, enable_dataloader=False, target_iter=1): + def update_config(self, dump_path, real_data=False, enable_dataloader=True, target_iter=1): args = { "dump_path": dump_path, "real_data": real_data, -- Gitee From 3b5cdc1bd807029b11d02594b0da436368dd65e2 Mon Sep 17 00:00:00 2001 From: brightlyking Date: Tue, 24 Oct 2023 10:33:54 +0800 Subject: [PATCH 06/11] add accuracy_tool file check --- .../api_accuracy_checker/dump/dump.py | 21 +++++++++++-------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/debug/accuracy_tools/api_accuracy_checker/dump/dump.py b/debug/accuracy_tools/api_accuracy_checker/dump/dump.py index 08385882d3..830ff5e4fc 100644 --- a/debug/accuracy_tools/api_accuracy_checker/dump/dump.py +++ b/debug/accuracy_tools/api_accuracy_checker/dump/dump.py @@ -17,12 +17,15 @@ from api_accuracy_checker.dump.api_info import ForwardAPIInfo, BackwardAPIInfo from api_accuracy_checker.dump.info_dump import write_api_info_json, initialize_output_json -from api_accuracy_checker.common.utils import print_error_log +from api_accuracy_checker.common.utils import print_error_log, CompareException from api_accuracy_checker.hook_module.register_hook import initialize_hook from api_accuracy_checker.common.config import msCheckerConfig def set_dump_switch(switch): + if switch not in ["ON", "OFF"]: + print_error_log("Please set switch with 'ON' or 'OFF'.") + raise CompareException(CompareException.INVALID_PARAM_ERROR) if switch == "ON": initialize_hook(pretest_hook) initialize_output_json() @@ -39,8 +42,8 @@ class DumpUtil(object): @staticmethod def get_dump_switch(): return DumpUtil.dump_switch == "ON" - - @staticmethod + + @staticmethod def incr_iter_num_maybe_exit(): if DumpUtil.call_num == msCheckerConfig.target_iter or not msCheckerConfig.enable_dataloader: set_dump_switch("ON") @@ -48,7 +51,7 @@ class DumpUtil(object): raise Exception("Model pretest: exit after iteration {}".format(msCheckerConfig.target_iter)) else: set_dump_switch("OFF") - DumpUtil.call_num += 1 + DumpUtil.call_num += 1 class DumpConst: @@ -59,7 +62,7 @@ class DumpConst: def pretest_info_dump(name, out_feat, module, phase): if not DumpUtil.get_dump_switch(): - return + return if phase == DumpConst.forward: api_info = ForwardAPIInfo(name, module.input_args, module.input_kwargs) elif phase == DumpConst.backward: @@ -68,14 +71,14 @@ def pretest_info_dump(name, out_feat, module, phase): msg = "Unexpected training phase {}.".format(phase) print_error_log(msg) raise NotImplementedError(msg) - + write_api_info_json(api_info) def pretest_hook(name, phase): def pretest_info_dump_hook(module, in_feat, out_feat): pretest_info_dump(name, out_feat, module, phase) if hasattr(module, "input_args"): - del module.input_args + del module.input_args if hasattr(module, "input_kwargs"): - del module.input_kwargs - return pretest_info_dump_hook + del module.input_kwargs + return pretest_info_dump_hook -- Gitee From b940cc219f2956f93957c90f9190ce09c03e7496 Mon Sep 17 00:00:00 2001 From: brightlyking Date: Tue, 24 Oct 2023 11:08:24 +0800 Subject: [PATCH 07/11] add version --- .../ptdbg_ascend/src/python/ptdbg_ascend/common/version.py | 1 + 1 file changed, 1 insertion(+) create mode 100644 debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/common/version.py diff --git a/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/common/version.py b/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/common/version.py new file mode 100644 index 0000000000..8aa4e6974e --- /dev/null +++ b/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/common/version.py @@ -0,0 +1 @@ +__version__ = '4.0' \ No newline at end of file -- Gitee From df51db7b0dc25d731109e96b5630c27dd337e3b9 Mon Sep 17 00:00:00 2001 From: brightlyking Date: Tue, 24 Oct 2023 11:56:54 +0800 Subject: [PATCH 08/11] fix smoke --- .../api_accuracy_checker/run_ut/data_generate.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/debug/accuracy_tools/api_accuracy_checker/run_ut/data_generate.py b/debug/accuracy_tools/api_accuracy_checker/run_ut/data_generate.py index 9f7f15ec7f..3946c34014 100644 --- a/debug/accuracy_tools/api_accuracy_checker/run_ut/data_generate.py +++ b/debug/accuracy_tools/api_accuracy_checker/run_ut/data_generate.py @@ -21,7 +21,6 @@ import numpy as np from api_accuracy_checker.common.utils import Const, check_file_or_directory_path, check_object_type, print_warn_log, print_error_log, \ CompareException -from ptdbg_ascend.src.python.ptdbg_ascend.common.file_check_util import FileCheckConst, FileChecker, FileOpen TORCH_TYPE = ["torch.device", "torch.dtype"] TENSOR_DATA_LIST = ["torch.Tensor", "torch.nn.parameter.Parameter"] @@ -66,8 +65,8 @@ def gen_real_tensor(data_path, convert_type): data_path: API data path convert_type: convert ori_type to dist_type flag. """ - data_path_checker = FileChecker(data_path, FileCheckConst.FILE) - data_path = data_path_checker.common_check() + data_path = os.path.realpath(data_path) + check_file_or_directory_path(data_path) if not data_path.endswith('.pt') and not data_path.endswith('.npy'): print_error_log(f"The file: {data_path} is not a pt or numpy file.") raise CompareException.INVALID_FILE_ERROR -- Gitee From 195b4f71652f60f407ea002c9d87d91d2f231728 Mon Sep 17 00:00:00 2001 From: brightlyking Date: Tue, 24 Oct 2023 14:07:13 +0800 Subject: [PATCH 09/11] fix smoke --- debug/accuracy_tools/api_accuracy_checker/common/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/api_accuracy_checker/common/config.py b/debug/accuracy_tools/api_accuracy_checker/common/config.py index a43212dac4..821842ddd9 100644 --- a/debug/accuracy_tools/api_accuracy_checker/common/config.py +++ b/debug/accuracy_tools/api_accuracy_checker/common/config.py @@ -35,7 +35,7 @@ class Config: def __str__(self): return '\n'.join(f"{key}={value}" for key, value in self.config.items()) - def update_config(self, dump_path, real_data=False, enable_dataloader=True, target_iter=1): + def update_config(self, dump_path, real_data=False, enable_dataloader=False, target_iter=1): args = { "dump_path": dump_path, "real_data": real_data, -- Gitee From bf1eab97f11cd81345d7cda61e54ee36d41f7daf Mon Sep 17 00:00:00 2001 From: brightlyking Date: Tue, 24 Oct 2023 15:28:49 +0800 Subject: [PATCH 10/11] fix review --- .../run_ut/run_overflow_check.py | 12 +++++------- .../api_accuracy_checker/run_ut/run_ut.py | 12 +++++------- 2 files changed, 10 insertions(+), 14 deletions(-) diff --git a/debug/accuracy_tools/api_accuracy_checker/run_ut/run_overflow_check.py b/debug/accuracy_tools/api_accuracy_checker/run_ut/run_overflow_check.py index 725b5fd87b..03e4e545aa 100644 --- a/debug/accuracy_tools/api_accuracy_checker/run_ut/run_overflow_check.py +++ b/debug/accuracy_tools/api_accuracy_checker/run_ut/run_overflow_check.py @@ -9,7 +9,7 @@ from api_accuracy_checker.run_ut.run_ut import exec_api, generate_npu_params, ru from api_accuracy_checker.common.utils import print_info_log, print_warn_log, get_json_contents, api_info_preprocess, \ print_error_log -from ptdbg_ascend.src.python.ptdbg_ascend.common.file_check_util import FileCheckConst, FileChecker +from ptdbg_ascend.src.python.ptdbg_ascend.common.file_check_util import FileCheckConst, check_file_suffix NO_GRAD_APIS = ["hardtanh"] @@ -134,14 +134,12 @@ def _run_overflow_check(): args = parser.parse_args(sys.argv[1:]) torch.npu.set_compile_mode(jit_compile=args.jit_compile) npu_device = "npu:" + str(args.device_id) - forward_file_checker = FileChecker(args.forward_input_file, FileCheckConst.FILE, ability=FileCheckConst.READ_ABLE, - file_type=FileCheckConst.JSON_SUFFIX) - forward_file = forward_file_checker.common_check() + forward_file = os.path.realpath(args.forward_input_file) backward_file = "" if args.backward_input_file: - backward_file_checker = FileChecker(args.backward_input_file, FileCheckConst.FILE, - ability=FileCheckConst.READ_ABLE, file_type=FileCheckConst.JSON_SUFFIX) - backward_file = backward_file_checker.common_check() + backward_file = os.path.realpath(args.backward_input_file) + check_file_suffix(backward_file, FileCheckConst.JSON_SUFFIX) + check_file_suffix(forward_file, FileCheckConst.JSON_SUFFIX) try: torch.npu.set_device(npu_device) except Exception: diff --git a/debug/accuracy_tools/api_accuracy_checker/run_ut/run_ut.py b/debug/accuracy_tools/api_accuracy_checker/run_ut/run_ut.py index 71daf0e906..e7f240185f 100644 --- a/debug/accuracy_tools/api_accuracy_checker/run_ut/run_ut.py +++ b/debug/accuracy_tools/api_accuracy_checker/run_ut/run_ut.py @@ -17,7 +17,7 @@ from ut_api_info import UtAPIInfo from api_accuracy_checker.common.config import msCheckerConfig from ptdbg_ascend.src.python.ptdbg_ascend.common.file_check_util import FileOpen, FileCheckConst, FileChecker, \ - change_mode + change_mode, check_file_suffix NO_GRAD_APIS = ["hardtanh"] @@ -250,12 +250,10 @@ def _run_ut(): except Exception: print_error_log(f"Set NPU device id failed. device id is: {args.device_id}") raise NotImplementedError - forward_file_checker = FileChecker(args.forward_input_file, FileCheckConst.FILE, ability=FileCheckConst.READ_ABLE, - file_type=FileCheckConst.JSON_SUFFIX) - forward_file = forward_file_checker.common_check() - backward_file_checker = FileChecker(args.backward_input_file, FileCheckConst.FILE, - ability=FileCheckConst.READ_ABLE, file_type=FileCheckConst.JSON_SUFFIX) - backward_file = backward_file_checker.common_check() + forward_file = os.path.realpath(args.forward_input_file) + backward_file = os.path.realpath(args.backward_input_file) + check_file_suffix(forward_file, FileCheckConst.JSON_SUFFIX) + check_file_suffix(backward_file, FileCheckConst.JSON_SUFFIX) out_path = os.path.realpath(args.out_path) if args.out_path else "./" out_path_checker = FileChecker(out_path, FileCheckConst.DIR, ability=FileCheckConst.WRITE_ABLE) out_path = out_path_checker.common_check() -- Gitee From 50ade036a8ef6a894df8a9b34a88b93e2368cad6 Mon Sep 17 00:00:00 2001 From: brightlyking Date: Tue, 24 Oct 2023 15:55:40 +0800 Subject: [PATCH 11/11] fix review --- debug/accuracy_tools/api_accuracy_checker/common/utils.py | 1 - 1 file changed, 1 deletion(-) diff --git a/debug/accuracy_tools/api_accuracy_checker/common/utils.py b/debug/accuracy_tools/api_accuracy_checker/common/utils.py index a59fc4efa0..b031b92a18 100644 --- a/debug/accuracy_tools/api_accuracy_checker/common/utils.py +++ b/debug/accuracy_tools/api_accuracy_checker/common/utils.py @@ -516,7 +516,6 @@ def get_json_contents(file_path): def get_file_content_bytes(file): - check_input_file_valid(file) with FileOpen(file, 'rb') as file_handle: return file_handle.read() -- Gitee