diff --git a/debug/accuracy_tools/api_accuracy_checker/dump/info_dump.py b/debug/accuracy_tools/api_accuracy_checker/dump/info_dump.py index 2cbd1750f1fb0adcbe22b930bb1f4b4278f6056c..0f76069a7f9e81cd6ddf08a467c5cdf2f7a55f5c 100644 --- a/debug/accuracy_tools/api_accuracy_checker/dump/info_dump.py +++ b/debug/accuracy_tools/api_accuracy_checker/dump/info_dump.py @@ -1 +1,57 @@ -# 基于api——info信息，将其落盘为json文件 \ No newline at end of file +import fcntl +import json +import os +import threading +import numpy as np + +from .api_info import ForwardAPIInfo, BackwardAPIInfo +from .utils import DumpUtil +from ..common.utils import check_file_or_directory_path + +lock = threading.Lock() + +def write_api_info_json(api_info): + dump_path = DumpUtil.dump_path + initialize_output_json() + if isinstance(api_info, ForwardAPIInfo): + file_path = os.path.join(dump_path, 'forward_info.json') + stack_file_path = os.path.join(dump_path, 'stack_info.json') + write_json(file_path, api_info.api_info_struct) + write_json(stack_file_path, api_info.stack_info_struct, indent=4) + + elif isinstance(api_info, BackwardAPIInfo): + file_path = os.path.join(dump_path, 'backward_info.json') + write_json(file_path, api_info.grad_info_struct) + else: + raise ValueError(f"Invalid api_info type {type(api_info)}") + +def write_json(file_path, data, indent=None): + check_file_or_directory_path(file_path,False) + with open(file_path, 'w') as f: + f.write("{\n}") + try: + lock.acquire() + with open(file_path, 'a+') as f: + fcntl.flock(f, fcntl.LOCK_EX) + f.seek(0, os.SEEK_END) + f.seek(f.tell() - 1, os.SEEK_SET) + f.truncate() + if f.tell() > 3: + f.seek(f.tell() - 1, os.SEEK_SET) + f.truncate() + f.write(',\n') + f.write(json.dumps(data, indent=indent)[1:-1] + '\n}') + except Exception as e: + raise ValueError(f"Json save failed:{e}") + finally: + fcntl.flock(f, fcntl.LOCK_UN) + lock.release() + +def initialize_output_json(): + dump_path = DumpUtil.dump_path + check_file_or_directory_path(dump_path,True) + files = ['forward_info.json', 'backward_info.json', 'stack_info.json'] + for file in files: + file_path = os.path.join(dump_path, file) + if os.path.exists(file_path): + raise ValueError(f"file {file_path} already exists, please remove it first or use a new dump path") \ No newline at end of file diff --git a/debug/accuracy_tools/api_accuracy_checker/dump/utils.py b/debug/accuracy_tools/api_accuracy_checker/dump/utils.py index e5773e097cfbecfc7f9ba2fdb1d6c4886642d5fa..1aaf7a3d53a0c4da4ef6cf7b24cb84e843898355 100644 --- a/debug/accuracy_tools/api_accuracy_checker/dump/utils.py +++ b/debug/accuracy_tools/api_accuracy_checker/dump/utils.py @@ -2,7 +2,7 @@ import os import shutil import sys from pathlib import Path - +import numpy as np from ..common.utils import print_error_log, CompareException, DumpException, Const, get_time, print_info_log, \ check_mode_valid, get_api_name_from_matcher @@ -11,20 +11,30 @@ from ..common.version import __version__ dump_count = 0 range_begin_flag, range_end_flag = False, False +class DumpConst: + delimiter = '*' + forward = 'forward' + backward = 'backward' + +def create_folder(path): + if not os.path.exists(path): + os.makedirs(path, mode=0o750) + return path + +def write_npy(file_path, tensor): + if os.path.exists(file_path): + raise ValueError(f"File {file_path} already exists") + np.save(file_path, tensor) + full_path = os.path.abspath(file_path) + return full_path + +def set_dump_switch(switch): + DumpUtil.set_dump_switch(switch) class DumpUtil(object): - dump_data_dir = None - dump_path = None + save_real_data = False + dump_path = './random_data_jsons' dump_switch = None - dump_switch_mode = Const.ALL - dump_switch_scope = [] - dump_init_enable = False - dump_api_list = [] - dump_filter_switch = None - dump_mode = Const.ALL - backward_input = {} - dump_dir_tag = 'ptdbg_dump' - dump_config = None @staticmethod def set_dump_path(save_path): @@ -32,191 +42,14 @@ class DumpUtil(object): DumpUtil.dump_init_enable = True @staticmethod - def set_dump_config(dump_config): - DumpUtil.dump_config = dump_config - - @staticmethod - def set_dump_switch(switch, mode, scope, api_list, filter_switch, dump_mode): + def set_dump_switch(switch): DumpUtil.dump_switch = switch - DumpUtil.dump_switch_mode = mode - DumpUtil.dump_init_enable = True - DumpUtil.dump_switch_scope = scope - DumpUtil.dump_api_list = [api.lower() for api in api_list] - DumpUtil.dump_filter_switch = filter_switch - DumpUtil.dump_mode = dump_mode - if mode == Const.ACL: - DumpUtil.dump_switch_scope = [api_name.replace("backward", "forward") for api_name in scope] - - def check_list_or_acl_mode(name_prefix): - global dump_count - for item in DumpUtil.dump_switch_scope: - if name_prefix.startswith(item): - dump_count = dump_count + 1 - return True - - def check_range_mode(name_prefix): - global range_begin_flag - global range_end_flag - if name_prefix.startswith(DumpUtil.dump_switch_scope[0]): - range_begin_flag = True - return True - if name_prefix.startswith(DumpUtil.dump_switch_scope[1]): - range_end_flag = True - return True - if range_begin_flag and not range_end_flag: - return True - return False - - def check_stack_mode(name_prefix): - if len(DumpUtil.dump_switch_scope) == 0: - return True - elif len(DumpUtil.dump_switch_scope) == 1: - return name_prefix.startswith(DumpUtil.dump_switch_scope[0]) - elif len(DumpUtil.dump_switch_scope) == 2: - return DumpUtil.check_range_mode(name_prefix) - else: - print_error_log("dump scope is invalid, Please set the scope mode in" - " set_dump_switch with 'all', 'list', 'range', 'stack', 'acl', 'api_list'!") - return False - - check_mapper = { - Const.LIST: check_list_or_acl_mode, - Const.ACL: check_list_or_acl_mode, - Const.RANGE: check_range_mode, - Const.STACK: check_stack_mode - } - - @staticmethod - def check_switch_scope(name_prefix): - if DumpUtil.dump_switch_mode in DumpUtil.check_mapper: - check_func = DumpUtil.check_mapper[DumpUtil.dump_switch_mode] - return check_func(name_prefix) - return False @staticmethod def get_dump_path(): if DumpUtil.dump_path: return DumpUtil.dump_path - if DumpUtil.dump_switch_mode == Const.ALL: - raise RuntimeError("get_dump_path: the file path is empty," - " you must use set_dump_path to set a valid dump path!!!") - else: - dir_path = os.path.realpath("./") - dump_file_name = "scope_dump_{}_{}_{}.pkl".format( - DumpUtil.dump_switch_mode, DumpUtil.dump_switch_scope[0], get_time()) - DumpUtil.dump_path = os.path.join(dir_path, dump_file_name) - return DumpUtil.dump_path - @staticmethod def get_dump_switch(): - return DumpUtil.dump_switch == "ON" - - -def set_dump_path(fpath=None, dump_tag='ptdbg_dump'): - if fpath is None: - raise RuntimeError("set_dump_path '{}' error, please set a valid filename".format(fpath)) - return - real_path = os.path.realpath(fpath) - if not os.path.isdir(real_path): - print_error_log( - "set_dump_path '{}' error, the path is not a directory please set a valid directory.".format(real_path)) - raise DumpException(DumpException.INVALID_PATH_ERROR) - DumpUtil.set_dump_path(real_path) - DumpUtil.dump_dir_tag = dump_tag - - -def generate_dump_path_str(): - if DumpUtil.dump_switch_mode == 'acl': - if DumpUtil.dump_config == '': - print_error_log("Please provide dump config for register hook before turning on dump switch!") - raise DumpException(DumpException.NONE_ERROR) - dump_path = f"according to dump config {DumpUtil.dump_config}" - else: - dump_path = f"to {DumpUtil.dump_path}" - return dump_path - - -def set_dump_switch(switch, mode=Const.ALL, scope=[], api_list=[], filter_switch=Const.ON, dump_mode=Const.ALL): - try: - check_mode_valid(mode) - assert switch in ["ON", "OFF"], "Please set dump switch with 'ON' or 'OFF'." - assert filter_switch in ["ON", "OFF"], "Please set filter_switch with 'ON' or 'OFF'." - assert dump_mode in ["all", "forward", "backward"], \ - "Please set dump_mode with 'all' or 'forward' or 'backward'." - if mode == Const.RANGE: - assert len(scope) == 2, "set_dump_switch, scope param set invalid, it's must be [start, end]." - if mode == Const.LIST: - assert len(scope) != 0, "set_dump_switch, scope param set invalid, it's should not be an empty list." - if mode == Const.STACK: - assert len(scope) <= 2, "set_dump_switch, scope param set invalid, it's must be [start, end] or []." - if mode == Const.ACL: - assert len(scope) == 1, \ - "set_dump_switch, scope param set invalid, only one api name is supported in acl mode." - if mode == Const.API_LIST: - assert isinstance(api_list, list) and len(api_list) >= 1, \ - "Current dump mode is 'api_list', but the content of api_list parameter is empty or valid." - except (CompareException, AssertionError) as err: - print_error_log(str(err)) - sys.exit() - - if switch == "OFF": - dump_path_str = generate_dump_path_str() - DumpUtil.set_dump_switch(switch, mode=mode, scope=scope, api_list=api_list, - filter_switch=filter_switch, dump_mode=dump_mode) - if switch == "ON": - dump_path_str = generate_dump_path_str() - - global dump_count - if switch == "ON": - print_info_log(f"Dump switch is turned on. Dump data will be saved {dump_path_str}. ") - if mode == Const.LIST: - dump_count = 0 - else: - print_info_log(f"Dump switch is turned off. Dump data has been saved {dump_path_str}. ") - if mode == Const.LIST: - print_info_log("The number of matched dump is {}".format(dump_count)) - -def _set_dump_switch4api_list(name): - if DumpUtil.dump_api_list: - api_name = get_api_name_from_matcher(name) - DumpUtil.dump_switch = "ON" if api_name in DumpUtil.dump_api_list else "OFF" - - -def set_backward_input(backward_input): - for index, api_name in enumerate(DumpUtil.dump_switch_scope): - DumpUtil.backward_input[api_name] = backward_input[index] - - -def make_dump_data_dir(dump_file_name): - dump_path, file_name = os.path.split(os.path.realpath(dump_file_name)) - name_body, name_extension = os.path.splitext(file_name) - output_dir = os.path.join(dump_path, f"{name_body}") - if not os.path.exists(output_dir): - os.mkdir(output_dir, mode=0o750) - else: - shutil.rmtree(output_dir, ignore_errors=True) - os.mkdir(output_dir, mode=0o750) - return output_dir - - -def make_dump_dirs(rank): - dump_file_name, dump_file_name_body = "dump.pkl", "dump" - dump_root_dir = DumpUtil.dump_path if DumpUtil.dump_path else "./" - tag_dir = os.path.join(dump_root_dir, DumpUtil.dump_dir_tag + f'_v{__version__}') - Path(tag_dir).mkdir(mode=0o750, parents=True, exist_ok=True) - rank_dir = os.path.join(tag_dir, 'rank' + str(rank)) - if not os.path.exists(rank_dir): - os.mkdir(rank_dir, mode=0o750) - DumpUtil.dump_dir = rank_dir - dump_file_path = os.path.join(rank_dir, dump_file_name) - DumpUtil.set_dump_path(dump_file_path) - - -def check_writable(dump_file): - if not os.access(dump_file, os.W_OK): - print_error_log( - 'The path {} does not have permission to write. Please check the path permission'.format( - dump_file)) - raise DumpException(DumpException.INVALID_PATH_ERROR) - + return DumpUtil.dump_switch == "ON" \ No newline at end of file