From 216a7b9289c7ba5bd91929a6017b5558db4342d7 Mon Sep 17 00:00:00 2001 From: wangchao Date: Thu, 16 Nov 2023 15:11:08 +0800 Subject: [PATCH 1/2] add soft_link check --- .../api_accuracy_checker/run_ut/run_overflow_check.py | 4 +++- debug/accuracy_tools/api_accuracy_checker/run_ut/run_ut.py | 4 +++- .../src/python/ptdbg_ascend/compare/acc_compare.py | 7 +++++-- .../src/python/ptdbg_ascend/compare/distributed_compare.py | 3 ++- 4 files changed, 13 insertions(+), 5 deletions(-) diff --git a/debug/accuracy_tools/api_accuracy_checker/run_ut/run_overflow_check.py b/debug/accuracy_tools/api_accuracy_checker/run_ut/run_overflow_check.py index 81460820d0..55a344b729 100644 --- a/debug/accuracy_tools/api_accuracy_checker/run_ut/run_overflow_check.py +++ b/debug/accuracy_tools/api_accuracy_checker/run_ut/run_overflow_check.py @@ -9,7 +9,7 @@ from api_accuracy_checker.run_ut.run_ut import exec_api, generate_npu_params, ru from api_accuracy_checker.common.utils import print_info_log, print_warn_log, get_json_contents, api_info_preprocess, \ print_error_log -from ptdbg_ascend.src.python.ptdbg_ascend.common.file_check_util import FileCheckConst, check_file_suffix +from ptdbg_ascend.src.python.ptdbg_ascend.common.file_check_util import FileCheckConst, check_file_suffix, check_link init_environment() @@ -133,9 +133,11 @@ def _run_overflow_check(): args = parser.parse_args(sys.argv[1:]) torch.npu.set_compile_mode(jit_compile=args.jit_compile) npu_device = "npu:" + str(args.device_id) + check_link(args.forward_input_file) forward_file = os.path.realpath(args.forward_input_file) backward_file = "" if args.backward_input_file: + check_link(args.backward_input_file) backward_file = os.path.realpath(args.backward_input_file) check_file_suffix(backward_file, FileCheckConst.JSON_SUFFIX) check_file_suffix(forward_file, FileCheckConst.JSON_SUFFIX) diff --git a/debug/accuracy_tools/api_accuracy_checker/run_ut/run_ut.py b/debug/accuracy_tools/api_accuracy_checker/run_ut/run_ut.py index 86d6c4d79d..b1860bc183 100644 --- a/debug/accuracy_tools/api_accuracy_checker/run_ut/run_ut.py +++ b/debug/accuracy_tools/api_accuracy_checker/run_ut/run_ut.py @@ -18,7 +18,7 @@ from api_accuracy_checker.run_ut.ut_api_info import UtAPIInfo from api_accuracy_checker.common.config import msCheckerConfig from ptdbg_ascend.src.python.ptdbg_ascend.common.file_check_util import FileOpen, FileCheckConst, FileChecker, \ - change_mode, check_file_suffix + change_mode, check_file_suffix, check_link @@ -250,6 +250,8 @@ def _run_ut(): except Exception as error: print_error_log(f"Set NPU device id failed. device id is: {args.device_id}") raise NotImplementedError from error + check_link(args.forward_input_file) + check_link(args.backward_input_file) forward_file = os.path.realpath(args.forward_input_file) backward_file = os.path.realpath(args.backward_input_file) check_file_suffix(forward_file, FileCheckConst.JSON_SUFFIX) diff --git a/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/compare/acc_compare.py b/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/compare/acc_compare.py index 7db518793c..67d252814e 100644 --- a/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/compare/acc_compare.py +++ b/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/compare/acc_compare.py @@ -28,7 +28,7 @@ from ..advisor.advisor import Advisor from ..common.utils import check_compare_param, add_time_as_suffix, \ print_warn_log, print_error_log, CompareException, Const,\ CompareConst, format_value, check_file_not_exists -from ..common.file_check_util import FileChecker, FileCheckConst, change_mode +from ..common.file_check_util import FileChecker, FileCheckConst, change_mode, FileOpen def correct_data(result): @@ -537,7 +537,10 @@ def compare_core(input_parma, output_path, npu_pkl, bench_pkl, stack_mode=False, def parse(pkl_file, module_name_prefix): - pkl_handle = open(pkl_file, "r") + pkl_handle = FileOpen(pkl_file, "r") + if not isinstance(module_name_prefix, str): + print_error_log("The parameter:module_name_prefix is not a string.") + raise CompareException(CompareException.INVALID_PARAM_ERROR) done = False title_printed = False while not done: diff --git a/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/compare/distributed_compare.py b/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/compare/distributed_compare.py index c58db903dc..615a35e7b7 100644 --- a/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/compare/distributed_compare.py +++ b/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/compare/distributed_compare.py @@ -17,12 +17,13 @@ import os import sys import re -from ..common.utils import print_error_log, CompareException, check_compare_param +from ..common.utils import print_error_log, CompareException, check_compare_param, check_file_or_directory_path from .acc_compare import compare_core def compare_distributed(npu_dump_dir, bench_dump_dir, output_path, **kwargs): def check_and_return_dir_contents(dump_dir, prefix): + check_file_or_directory_path(dump_dir, True) contents = os.listdir(dump_dir) pattern = re.compile(f'^{prefix}[0-9]+$') for name in contents: -- Gitee From f5dff9dc36187ddd0f251c2527f329a9e268aaa2 Mon Sep 17 00:00:00 2001 From: wangchao Date: Thu, 16 Nov 2023 16:11:57 +0800 Subject: [PATCH 2/2] add soft_link check --- .../ptdbg_ascend/compare/acc_compare.py | 59 +++++++++---------- 1 file changed, 29 insertions(+), 30 deletions(-) diff --git a/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/compare/acc_compare.py b/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/compare/acc_compare.py index 67d252814e..343c2c9ece 100644 --- a/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/compare/acc_compare.py +++ b/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/compare/acc_compare.py @@ -537,39 +537,38 @@ def compare_core(input_parma, output_path, npu_pkl, bench_pkl, stack_mode=False, def parse(pkl_file, module_name_prefix): - pkl_handle = FileOpen(pkl_file, "r") if not isinstance(module_name_prefix, str): print_error_log("The parameter:module_name_prefix is not a string.") raise CompareException(CompareException.INVALID_PARAM_ERROR) - done = False - title_printed = False - while not done: - pkl_line = pkl_handle.readline() - if pkl_line == '\n': - continue - if len(pkl_line) == 0: - done = True - break - - msg = json.loads(pkl_line) - info_prefix = msg[0] - if not info_prefix.startswith(module_name_prefix): - continue - - if info_prefix.find("stack_info") != -1: - print("\nTrace back({}):".format(msg[0])) - for item in reversed(msg[1]): - print(" File \"{}\", line {}, in {}".format(item[0], item[1], item[2])) - print(" {}".format(item[3])) - continue - if len(msg) > 5: - summery_info = " [{}][dtype: {}][shape: {}][max: {}][min: {}][mean: {}]" \ - .format(msg[0], msg[3], msg[4], msg[5][0], msg[5][1], msg[5][2]) - if not title_printed: - print("\nStatistic Info:") - title_printed = True - print(summery_info) - pkl_handle.close() + with FileOpen(pkl_file, "r") as f: + done = False + title_printed = False + while not done: + pkl_line = f.readline() + if pkl_line == '\n': + continue + if len(pkl_line) == 0: + done = True + break + + msg = json.loads(pkl_line) + info_prefix = msg[0] + if not info_prefix.startswith(module_name_prefix): + continue + + if info_prefix.find("stack_info") != -1: + print("\nTrace back({}):".format(msg[0])) + for item in reversed(msg[1]): + print(" File \"{}\", line {}, in {}".format(item[0], item[1], item[2])) + print(" {}".format(item[3])) + continue + if len(msg) > 5: + summery_info = " [{}][dtype: {}][shape: {}][max: {}][min: {}][mean: {}]" \ + .format(msg[0], msg[3], msg[4], msg[5][0], msg[5][1], msg[5][2]) + if not title_printed: + print("\nStatistic Info:") + title_printed = True + print(summery_info) def compare_process(npu_pkl_handle, bench_pkl_handle, stack_mode, fuzzy_match): -- Gitee