From 541a91aaef94f8ed664ad60dbeb927cd6c1e43ee Mon Sep 17 00:00:00 2001 From: sunyiming Date: Tue, 1 Aug 2023 07:40:47 +0000 Subject: [PATCH 1/8] update debug/accuracy_tools/api_accuracy_checker/dump/info_dump.py. Signed-off-by: sunyiming --- .../api_accuracy_checker/dump/info_dump.py | 69 ++++++++++++++++++- 1 file changed, 68 insertions(+), 1 deletion(-) diff --git a/debug/accuracy_tools/api_accuracy_checker/dump/info_dump.py b/debug/accuracy_tools/api_accuracy_checker/dump/info_dump.py index 2cbd1750f1..adf08e402d 100644 --- a/debug/accuracy_tools/api_accuracy_checker/dump/info_dump.py +++ b/debug/accuracy_tools/api_accuracy_checker/dump/info_dump.py @@ -1 +1,68 @@ -# 基于api——info信息，将其落盘为json文件 \ No newline at end of file +import os +import json +import numpy as np +from .utils import DumpUtil +from .api_info import ForwardAPIInfo, BackwardAPIInfo +import threading +import fcntl +from ..common.utils import check_file_or_directory_path + +lock = threading.Lock() + +def write_api_info_json(api_info): + dump_path = DumpUtil.dump_path + initialize_output_json() + if isinstance(api_info, ForwardAPIInfo): + file_path = os.path.join(dump_path, 'forward_info.json') + stack_file_path = os.path.join(dump_path, 'stack_info.json') + write_json(file_path, api_info.api_info_struct) + write_json(stack_file_path, api_info.stack_info_struct, indent=4) + + elif isinstance(api_info, BackwardAPIInfo): + file_path = os.path.join(dump_path, 'backward_info.json') + write_json(file_path, api_info.grad_info_struct) + else: + raise ValueError(f"Invalid api_info type {type(api_info)}") + +def write_npy(name, tensor): + dump_path = DumpUtil.dump_path + if not os.path.exists(dump_path): + raise ValueError(f"Path {dump_path} does not exist") + npy_folder = os.path.join(dump_path, 'npy') + if not os.path.exists(npy_folder): + os.makedirs(npy_folder, mode=0o750) + npy_path = os.path.join(npy_folder, name) + if os.path.exists(npy_path): + raise ValueError(f"File {npy_path} already exists") + np.save(npy_path, tensor) + +def write_json(file_path, data, indent=None): + if not os.path.exists(file_path): + with open(file_path, 'w') as f: + f.write("{\n}") + try: + lock.acquire() + with open(file_path, 'a+') as f: + fcntl.flock(f, fcntl.LOCK_EX) + f.seek(0, os.SEEK_END) + f.seek(f.tell() - 1, os.SEEK_SET) + f.truncate() + if f.tell() > 3: + f.seek(f.tell() - 1, os.SEEK_SET) + f.truncate() + f.write(',\n') + f.write(json.dumps(data, indent=indent)[1:-1] + '\n}') + except Exception as e: + raise ValueError(f"Json save failed:{e}") + finally: + fcntl.flock(f, fcntl.LOCK_UN) + lock.release() + +def initialize_output_json(): + dump_path = DumpUtil.dump_path + check_file_or_directory_path(dump_path) + files = ['forward_info.json', 'backward_info.json', 'stack_info.json'] + for file in files: + file_path = os.path.join(dump_path, file) + if os.path.exists(file_path): + raise ValueError(f"file {file_path} already exists, please remove it first or use a new dump path") \ No newline at end of file -- Gitee From e20a4df339a184dd29145dc93ee7ed58cdb257fd Mon Sep 17 00:00:00 2001 From: sunyiming Date: Tue, 1 Aug 2023 07:41:06 +0000 Subject: [PATCH 2/8] update debug/accuracy_tools/api_accuracy_checker/dump/utils.py. Signed-off-by: sunyiming --- .../api_accuracy_checker/dump/utils.py | 220 +++--------------- 1 file changed, 29 insertions(+), 191 deletions(-) diff --git a/debug/accuracy_tools/api_accuracy_checker/dump/utils.py b/debug/accuracy_tools/api_accuracy_checker/dump/utils.py index e5773e097c..30666dd977 100644 --- a/debug/accuracy_tools/api_accuracy_checker/dump/utils.py +++ b/debug/accuracy_tools/api_accuracy_checker/dump/utils.py @@ -2,7 +2,7 @@ import os import shutil import sys from pathlib import Path - +import numpy as np from ..common.utils import print_error_log, CompareException, DumpException, Const, get_time, print_info_log, \ check_mode_valid, get_api_name_from_matcher @@ -11,20 +11,35 @@ from ..common.version import __version__ dump_count = 0 range_begin_flag, range_end_flag = False, False +class DumpConst: + delimiter = '*' + forward = 'forward' + backward = 'backward' + +def create_folder(path): + if not os.path.exists(path): + os.makedirs(path, mode=0o750) + return path + +def write_npy(file_path, tensor): + if os.path.exists(file_path): + raise ValueError(f"File {file_path} already exists") + np.save(file_path, tensor) + full_path = os.path.abspath(file_path+'.npy') + return full_path + +def check_path(dump_path): + if not os.path.exists(dump_path): + raise ValueError(f"Path {dump_path} does not exist") + else: + current_permission = oct(os.stat(dump_path).st_mode)[-3:] + if current_permission != '750': + raise ValueError(f"Path {dump_path} does not have the required 0o750 permissions") class DumpUtil(object): - dump_data_dir = None - dump_path = None + save_real_data = False + dump_path = './random_data_jsons' dump_switch = None - dump_switch_mode = Const.ALL - dump_switch_scope = [] - dump_init_enable = False - dump_api_list = [] - dump_filter_switch = None - dump_mode = Const.ALL - backward_input = {} - dump_dir_tag = 'ptdbg_dump' - dump_config = None @staticmethod def set_dump_path(save_path): @@ -32,191 +47,14 @@ class DumpUtil(object): DumpUtil.dump_init_enable = True @staticmethod - def set_dump_config(dump_config): - DumpUtil.dump_config = dump_config - - @staticmethod - def set_dump_switch(switch, mode, scope, api_list, filter_switch, dump_mode): + def set_dump_switch(switch): DumpUtil.dump_switch = switch - DumpUtil.dump_switch_mode = mode - DumpUtil.dump_init_enable = True - DumpUtil.dump_switch_scope = scope - DumpUtil.dump_api_list = [api.lower() for api in api_list] - DumpUtil.dump_filter_switch = filter_switch - DumpUtil.dump_mode = dump_mode - if mode == Const.ACL: - DumpUtil.dump_switch_scope = [api_name.replace("backward", "forward") for api_name in scope] - - def check_list_or_acl_mode(name_prefix): - global dump_count - for item in DumpUtil.dump_switch_scope: - if name_prefix.startswith(item): - dump_count = dump_count + 1 - return True - - def check_range_mode(name_prefix): - global range_begin_flag - global range_end_flag - if name_prefix.startswith(DumpUtil.dump_switch_scope[0]): - range_begin_flag = True - return True - if name_prefix.startswith(DumpUtil.dump_switch_scope[1]): - range_end_flag = True - return True - if range_begin_flag and not range_end_flag: - return True - return False - - def check_stack_mode(name_prefix): - if len(DumpUtil.dump_switch_scope) == 0: - return True - elif len(DumpUtil.dump_switch_scope) == 1: - return name_prefix.startswith(DumpUtil.dump_switch_scope[0]) - elif len(DumpUtil.dump_switch_scope) == 2: - return DumpUtil.check_range_mode(name_prefix) - else: - print_error_log("dump scope is invalid, Please set the scope mode in" - " set_dump_switch with 'all', 'list', 'range', 'stack', 'acl', 'api_list'!") - return False - - check_mapper = { - Const.LIST: check_list_or_acl_mode, - Const.ACL: check_list_or_acl_mode, - Const.RANGE: check_range_mode, - Const.STACK: check_stack_mode - } - - @staticmethod - def check_switch_scope(name_prefix): - if DumpUtil.dump_switch_mode in DumpUtil.check_mapper: - check_func = DumpUtil.check_mapper[DumpUtil.dump_switch_mode] - return check_func(name_prefix) - return False @staticmethod def get_dump_path(): if DumpUtil.dump_path: return DumpUtil.dump_path - if DumpUtil.dump_switch_mode == Const.ALL: - raise RuntimeError("get_dump_path: the file path is empty," - " you must use set_dump_path to set a valid dump path!!!") - else: - dir_path = os.path.realpath("./") - dump_file_name = "scope_dump_{}_{}_{}.pkl".format( - DumpUtil.dump_switch_mode, DumpUtil.dump_switch_scope[0], get_time()) - DumpUtil.dump_path = os.path.join(dir_path, dump_file_name) - return DumpUtil.dump_path - @staticmethod def get_dump_switch(): - return DumpUtil.dump_switch == "ON" - - -def set_dump_path(fpath=None, dump_tag='ptdbg_dump'): - if fpath is None: - raise RuntimeError("set_dump_path '{}' error, please set a valid filename".format(fpath)) - return - real_path = os.path.realpath(fpath) - if not os.path.isdir(real_path): - print_error_log( - "set_dump_path '{}' error, the path is not a directory please set a valid directory.".format(real_path)) - raise DumpException(DumpException.INVALID_PATH_ERROR) - DumpUtil.set_dump_path(real_path) - DumpUtil.dump_dir_tag = dump_tag - - -def generate_dump_path_str(): - if DumpUtil.dump_switch_mode == 'acl': - if DumpUtil.dump_config == '': - print_error_log("Please provide dump config for register hook before turning on dump switch!") - raise DumpException(DumpException.NONE_ERROR) - dump_path = f"according to dump config {DumpUtil.dump_config}" - else: - dump_path = f"to {DumpUtil.dump_path}" - return dump_path - - -def set_dump_switch(switch, mode=Const.ALL, scope=[], api_list=[], filter_switch=Const.ON, dump_mode=Const.ALL): - try: - check_mode_valid(mode) - assert switch in ["ON", "OFF"], "Please set dump switch with 'ON' or 'OFF'." - assert filter_switch in ["ON", "OFF"], "Please set filter_switch with 'ON' or 'OFF'." - assert dump_mode in ["all", "forward", "backward"], \ - "Please set dump_mode with 'all' or 'forward' or 'backward'." - if mode == Const.RANGE: - assert len(scope) == 2, "set_dump_switch, scope param set invalid, it's must be [start, end]." - if mode == Const.LIST: - assert len(scope) != 0, "set_dump_switch, scope param set invalid, it's should not be an empty list." - if mode == Const.STACK: - assert len(scope) <= 2, "set_dump_switch, scope param set invalid, it's must be [start, end] or []." - if mode == Const.ACL: - assert len(scope) == 1, \ - "set_dump_switch, scope param set invalid, only one api name is supported in acl mode." - if mode == Const.API_LIST: - assert isinstance(api_list, list) and len(api_list) >= 1, \ - "Current dump mode is 'api_list', but the content of api_list parameter is empty or valid." - except (CompareException, AssertionError) as err: - print_error_log(str(err)) - sys.exit() - - if switch == "OFF": - dump_path_str = generate_dump_path_str() - DumpUtil.set_dump_switch(switch, mode=mode, scope=scope, api_list=api_list, - filter_switch=filter_switch, dump_mode=dump_mode) - if switch == "ON": - dump_path_str = generate_dump_path_str() - - global dump_count - if switch == "ON": - print_info_log(f"Dump switch is turned on. Dump data will be saved {dump_path_str}. ") - if mode == Const.LIST: - dump_count = 0 - else: - print_info_log(f"Dump switch is turned off. Dump data has been saved {dump_path_str}. ") - if mode == Const.LIST: - print_info_log("The number of matched dump is {}".format(dump_count)) - -def _set_dump_switch4api_list(name): - if DumpUtil.dump_api_list: - api_name = get_api_name_from_matcher(name) - DumpUtil.dump_switch = "ON" if api_name in DumpUtil.dump_api_list else "OFF" - - -def set_backward_input(backward_input): - for index, api_name in enumerate(DumpUtil.dump_switch_scope): - DumpUtil.backward_input[api_name] = backward_input[index] - - -def make_dump_data_dir(dump_file_name): - dump_path, file_name = os.path.split(os.path.realpath(dump_file_name)) - name_body, name_extension = os.path.splitext(file_name) - output_dir = os.path.join(dump_path, f"{name_body}") - if not os.path.exists(output_dir): - os.mkdir(output_dir, mode=0o750) - else: - shutil.rmtree(output_dir, ignore_errors=True) - os.mkdir(output_dir, mode=0o750) - return output_dir - - -def make_dump_dirs(rank): - dump_file_name, dump_file_name_body = "dump.pkl", "dump" - dump_root_dir = DumpUtil.dump_path if DumpUtil.dump_path else "./" - tag_dir = os.path.join(dump_root_dir, DumpUtil.dump_dir_tag + f'_v{__version__}') - Path(tag_dir).mkdir(mode=0o750, parents=True, exist_ok=True) - rank_dir = os.path.join(tag_dir, 'rank' + str(rank)) - if not os.path.exists(rank_dir): - os.mkdir(rank_dir, mode=0o750) - DumpUtil.dump_dir = rank_dir - dump_file_path = os.path.join(rank_dir, dump_file_name) - DumpUtil.set_dump_path(dump_file_path) - - -def check_writable(dump_file): - if not os.access(dump_file, os.W_OK): - print_error_log( - 'The path {} does not have permission to write. Please check the path permission'.format( - dump_file)) - raise DumpException(DumpException.INVALID_PATH_ERROR) - + return DumpUtil.dump_switch == "ON" \ No newline at end of file -- Gitee From ac7a2299833a791eaa58b209e6fb710bf23a997e Mon Sep 17 00:00:00 2001 From: sunyiming Date: Tue, 1 Aug 2023 07:42:35 +0000 Subject: [PATCH 3/8] update debug/accuracy_tools/api_accuracy_checker/dump/utils.py. Signed-off-by: sunyiming --- debug/accuracy_tools/api_accuracy_checker/dump/utils.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/debug/accuracy_tools/api_accuracy_checker/dump/utils.py b/debug/accuracy_tools/api_accuracy_checker/dump/utils.py index 30666dd977..aec1bfa8f2 100644 --- a/debug/accuracy_tools/api_accuracy_checker/dump/utils.py +++ b/debug/accuracy_tools/api_accuracy_checker/dump/utils.py @@ -28,14 +28,6 @@ def write_npy(file_path, tensor): full_path = os.path.abspath(file_path+'.npy') return full_path -def check_path(dump_path): - if not os.path.exists(dump_path): - raise ValueError(f"Path {dump_path} does not exist") - else: - current_permission = oct(os.stat(dump_path).st_mode)[-3:] - if current_permission != '750': - raise ValueError(f"Path {dump_path} does not have the required 0o750 permissions") - class DumpUtil(object): save_real_data = False dump_path = './random_data_jsons' -- Gitee From 8977590f8a1c8e5c0df92d65f4e195d07ba07d24 Mon Sep 17 00:00:00 2001 From: sunyiming Date: Tue, 1 Aug 2023 08:04:39 +0000 Subject: [PATCH 4/8] update debug/accuracy_tools/api_accuracy_checker/dump/utils.py. Signed-off-by: sunyiming --- .../api_accuracy_checker/dump/utils.py | 103 +++++++++--------- 1 file changed, 54 insertions(+), 49 deletions(-) diff --git a/debug/accuracy_tools/api_accuracy_checker/dump/utils.py b/debug/accuracy_tools/api_accuracy_checker/dump/utils.py index aec1bfa8f2..b1200fe104 100644 --- a/debug/accuracy_tools/api_accuracy_checker/dump/utils.py +++ b/debug/accuracy_tools/api_accuracy_checker/dump/utils.py @@ -1,52 +1,57 @@ +import fcntl +import json import os -import shutil -import sys -from pathlib import Path +import threading import numpy as np -from ..common.utils import print_error_log, CompareException, DumpException, Const, get_time, print_info_log, \ - check_mode_valid, get_api_name_from_matcher -from ..common.version import __version__ - -dump_count = 0 -range_begin_flag, range_end_flag = False, False - -class DumpConst: - delimiter = '*' - forward = 'forward' - backward = 'backward' - -def create_folder(path): - if not os.path.exists(path): - os.makedirs(path, mode=0o750) - return path - -def write_npy(file_path, tensor): - if os.path.exists(file_path): - raise ValueError(f"File {file_path} already exists") - np.save(file_path, tensor) - full_path = os.path.abspath(file_path+'.npy') - return full_path - -class DumpUtil(object): - save_real_data = False - dump_path = './random_data_jsons' - dump_switch = None - - @staticmethod - def set_dump_path(save_path): - DumpUtil.dump_path = save_path - DumpUtil.dump_init_enable = True - - @staticmethod - def set_dump_switch(switch): - DumpUtil.dump_switch = switch - - @staticmethod - def get_dump_path(): - if DumpUtil.dump_path: - return DumpUtil.dump_path - - @staticmethod - def get_dump_switch(): - return DumpUtil.dump_switch == "ON" \ No newline at end of file +from .api_info import ForwardAPIInfo, BackwardAPIInfo +from .utils import DumpUtil +from ..common.utils import check_file_or_directory_path + +lock = threading.Lock() + +def write_api_info_json(api_info): + dump_path = DumpUtil.dump_path + initialize_output_json() + if isinstance(api_info, ForwardAPIInfo): + file_path = os.path.join(dump_path, 'forward_info.json') + stack_file_path = os.path.join(dump_path, 'stack_info.json') + write_json(file_path, api_info.api_info_struct) + write_json(stack_file_path, api_info.stack_info_struct, indent=4) + + elif isinstance(api_info, BackwardAPIInfo): + file_path = os.path.join(dump_path, 'backward_info.json') + write_json(file_path, api_info.grad_info_struct) + else: + raise ValueError(f"Invalid api_info type {type(api_info)}") + +def write_json(file_path, data, indent=None): + check_file_or_directory_path(file_path,False) + with open(file_path, 'w') as f: + f.write("{\n}") + try: + lock.acquire() + with open(file_path, 'a+') as f: + fcntl.flock(f, fcntl.LOCK_EX) + f.seek(0, os.SEEK_END) + f.seek(f.tell() - 1, os.SEEK_SET) + f.truncate() + if f.tell() > 3: + f.seek(f.tell() - 1, os.SEEK_SET) + f.truncate() + f.write(',\n') + f.write(json.dumps(data, indent=indent)[1:-1] + '\n}') + except Exception as e: + raise ValueError(f"Json save failed:{e}") + finally: + fcntl.flock(f, fcntl.LOCK_UN) + lock.release() + +def initialize_output_json(): + dump_path = DumpUtil.dump_path + check_file_or_directory_path(dump_path,True) + files = ['forward_info.json', 'backward_info.json', 'stack_info.json'] + for file in files: + file_path = os.path.join(dump_path, file) + if os.path.exists(file_path): + raise ValueError(f"file {file_path} already exists, please remove it first or use a new dump path") \ No newline at end of file -- Gitee From 46fb48b8aed623f4c3f29486139b43ddc2dff825 Mon Sep 17 00:00:00 2001 From: sunyiming Date: Tue, 1 Aug 2023 08:05:47 +0000 Subject: [PATCH 5/8] Revert "update debug/accuracy_tools/api_accuracy_checker/dump/utils.py." This reverts commit 8977590f8a1c8e5c0df92d65f4e195d07ba07d24. --- .../api_accuracy_checker/dump/utils.py | 103 +++++++++--------- 1 file changed, 49 insertions(+), 54 deletions(-) diff --git a/debug/accuracy_tools/api_accuracy_checker/dump/utils.py b/debug/accuracy_tools/api_accuracy_checker/dump/utils.py index b1200fe104..aec1bfa8f2 100644 --- a/debug/accuracy_tools/api_accuracy_checker/dump/utils.py +++ b/debug/accuracy_tools/api_accuracy_checker/dump/utils.py @@ -1,57 +1,52 @@ -import fcntl -import json import os -import threading +import shutil +import sys +from pathlib import Path import numpy as np +from ..common.utils import print_error_log, CompareException, DumpException, Const, get_time, print_info_log, \ + check_mode_valid, get_api_name_from_matcher -from .api_info import ForwardAPIInfo, BackwardAPIInfo -from .utils import DumpUtil -from ..common.utils import check_file_or_directory_path - -lock = threading.Lock() - -def write_api_info_json(api_info): - dump_path = DumpUtil.dump_path - initialize_output_json() - if isinstance(api_info, ForwardAPIInfo): - file_path = os.path.join(dump_path, 'forward_info.json') - stack_file_path = os.path.join(dump_path, 'stack_info.json') - write_json(file_path, api_info.api_info_struct) - write_json(stack_file_path, api_info.stack_info_struct, indent=4) - - elif isinstance(api_info, BackwardAPIInfo): - file_path = os.path.join(dump_path, 'backward_info.json') - write_json(file_path, api_info.grad_info_struct) - else: - raise ValueError(f"Invalid api_info type {type(api_info)}") - -def write_json(file_path, data, indent=None): - check_file_or_directory_path(file_path,False) - with open(file_path, 'w') as f: - f.write("{\n}") - try: - lock.acquire() - with open(file_path, 'a+') as f: - fcntl.flock(f, fcntl.LOCK_EX) - f.seek(0, os.SEEK_END) - f.seek(f.tell() - 1, os.SEEK_SET) - f.truncate() - if f.tell() > 3: - f.seek(f.tell() - 1, os.SEEK_SET) - f.truncate() - f.write(',\n') - f.write(json.dumps(data, indent=indent)[1:-1] + '\n}') - except Exception as e: - raise ValueError(f"Json save failed:{e}") - finally: - fcntl.flock(f, fcntl.LOCK_UN) - lock.release() - -def initialize_output_json(): - dump_path = DumpUtil.dump_path - check_file_or_directory_path(dump_path,True) - files = ['forward_info.json', 'backward_info.json', 'stack_info.json'] - for file in files: - file_path = os.path.join(dump_path, file) - if os.path.exists(file_path): - raise ValueError(f"file {file_path} already exists, please remove it first or use a new dump path") \ No newline at end of file +from ..common.version import __version__ + +dump_count = 0 +range_begin_flag, range_end_flag = False, False + +class DumpConst: + delimiter = '*' + forward = 'forward' + backward = 'backward' + +def create_folder(path): + if not os.path.exists(path): + os.makedirs(path, mode=0o750) + return path + +def write_npy(file_path, tensor): + if os.path.exists(file_path): + raise ValueError(f"File {file_path} already exists") + np.save(file_path, tensor) + full_path = os.path.abspath(file_path+'.npy') + return full_path + +class DumpUtil(object): + save_real_data = False + dump_path = './random_data_jsons' + dump_switch = None + + @staticmethod + def set_dump_path(save_path): + DumpUtil.dump_path = save_path + DumpUtil.dump_init_enable = True + + @staticmethod + def set_dump_switch(switch): + DumpUtil.dump_switch = switch + + @staticmethod + def get_dump_path(): + if DumpUtil.dump_path: + return DumpUtil.dump_path + + @staticmethod + def get_dump_switch(): + return DumpUtil.dump_switch == "ON" \ No newline at end of file -- Gitee From 65dcbb59f61895e2f7d4879dcd8cce58fdae373f Mon Sep 17 00:00:00 2001 From: sunyiming Date: Tue, 1 Aug 2023 08:06:16 +0000 Subject: [PATCH 6/8] update debug/accuracy_tools/api_accuracy_checker/dump/info_dump.py. Signed-off-by: sunyiming --- .../api_accuracy_checker/dump/info_dump.py | 39 +++++++------------ 1 file changed, 14 insertions(+), 25 deletions(-) diff --git a/debug/accuracy_tools/api_accuracy_checker/dump/info_dump.py b/debug/accuracy_tools/api_accuracy_checker/dump/info_dump.py index adf08e402d..0f76069a7f 100644 --- a/debug/accuracy_tools/api_accuracy_checker/dump/info_dump.py +++ b/debug/accuracy_tools/api_accuracy_checker/dump/info_dump.py @@ -1,10 +1,11 @@ -import os +import fcntl import json +import os +import threading import numpy as np -from .utils import DumpUtil + from .api_info import ForwardAPIInfo, BackwardAPIInfo -import threading -import fcntl +from .utils import DumpUtil from ..common.utils import check_file_or_directory_path lock = threading.Lock() @@ -24,22 +25,10 @@ def write_api_info_json(api_info): else: raise ValueError(f"Invalid api_info type {type(api_info)}") -def write_npy(name, tensor): - dump_path = DumpUtil.dump_path - if not os.path.exists(dump_path): - raise ValueError(f"Path {dump_path} does not exist") - npy_folder = os.path.join(dump_path, 'npy') - if not os.path.exists(npy_folder): - os.makedirs(npy_folder, mode=0o750) - npy_path = os.path.join(npy_folder, name) - if os.path.exists(npy_path): - raise ValueError(f"File {npy_path} already exists") - np.save(npy_path, tensor) - def write_json(file_path, data, indent=None): - if not os.path.exists(file_path): - with open(file_path, 'w') as f: - f.write("{\n}") + check_file_or_directory_path(file_path,False) + with open(file_path, 'w') as f: + f.write("{\n}") try: lock.acquire() with open(file_path, 'a+') as f: @@ -52,15 +41,15 @@ def write_json(file_path, data, indent=None): f.truncate() f.write(',\n') f.write(json.dumps(data, indent=indent)[1:-1] + '\n}') - except Exception as e: - raise ValueError(f"Json save failed:{e}") - finally: - fcntl.flock(f, fcntl.LOCK_UN) - lock.release() + except Exception as e: + raise ValueError(f"Json save failed:{e}") + finally: + fcntl.flock(f, fcntl.LOCK_UN) + lock.release() def initialize_output_json(): dump_path = DumpUtil.dump_path - check_file_or_directory_path(dump_path) + check_file_or_directory_path(dump_path,True) files = ['forward_info.json', 'backward_info.json', 'stack_info.json'] for file in files: file_path = os.path.join(dump_path, file) -- Gitee From ec5725709407c1182c10e9bc0d8f8bcd1484602f Mon Sep 17 00:00:00 2001 From: sunyiming Date: Tue, 1 Aug 2023 08:28:43 +0000 Subject: [PATCH 7/8] update debug/accuracy_tools/api_accuracy_checker/dump/utils.py. Signed-off-by: sunyiming --- debug/accuracy_tools/api_accuracy_checker/dump/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/api_accuracy_checker/dump/utils.py b/debug/accuracy_tools/api_accuracy_checker/dump/utils.py index aec1bfa8f2..9011a74ffa 100644 --- a/debug/accuracy_tools/api_accuracy_checker/dump/utils.py +++ b/debug/accuracy_tools/api_accuracy_checker/dump/utils.py @@ -25,7 +25,7 @@ def write_npy(file_path, tensor): if os.path.exists(file_path): raise ValueError(f"File {file_path} already exists") np.save(file_path, tensor) - full_path = os.path.abspath(file_path+'.npy') + full_path = os.path.abspath(file_path) return full_path class DumpUtil(object): -- Gitee From 350ed7ae4119c92ffd2d197d25d40dedfd4102a4 Mon Sep 17 00:00:00 2001 From: sunyiming Date: Wed, 2 Aug 2023 02:15:14 +0000 Subject: [PATCH 8/8] update debug/accuracy_tools/api_accuracy_checker/dump/utils.py. Signed-off-by: sunyiming --- debug/accuracy_tools/api_accuracy_checker/dump/utils.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/debug/accuracy_tools/api_accuracy_checker/dump/utils.py b/debug/accuracy_tools/api_accuracy_checker/dump/utils.py index 9011a74ffa..1aaf7a3d53 100644 --- a/debug/accuracy_tools/api_accuracy_checker/dump/utils.py +++ b/debug/accuracy_tools/api_accuracy_checker/dump/utils.py @@ -28,6 +28,9 @@ def write_npy(file_path, tensor): full_path = os.path.abspath(file_path) return full_path +def set_dump_switch(switch): + DumpUtil.set_dump_switch(switch) + class DumpUtil(object): save_real_data = False dump_path = './random_data_jsons' -- Gitee