From 62bb9d2bad8faa08cbc8c55b3da254a3cadcd59d Mon Sep 17 00:00:00 2001 From: s30048155 Date: Fri, 8 Sep 2023 09:01:28 +0800 Subject: [PATCH 1/6] update --- .../api_accuracy_checker/compare/compare.py | 5 ++++- .../accuracy_tools/api_accuracy_checker/run_ut/run_ut.py | 8 ++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/compare.py b/debug/accuracy_tools/api_accuracy_checker/compare/compare.py index 2192359a02..244eb053bc 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/compare.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/compare.py @@ -86,6 +86,8 @@ class Comparator: subject_prefix = test_result[0] fwd_result = test_result[3] bwd_result = test_result[4] + shape_info = test_result[5] + test_rows.append(["Shape Info"] + list(shape_info)) if isinstance(fwd_result, list): for i, test_subject in enumerate(fwd_result): subject = subject_prefix + ".forward.output." + str(i) @@ -106,6 +108,7 @@ class Comparator: self.compare_alg.update({name: (compare_func, standard)}) def compare_output(self, api_name, bench_out, npu_out, bench_grad=None, npu_grad=None): + shape_info = npu_out.shape if "dropout" in api_name: is_fwd_success, fwd_compare_alg_results = self._compare_dropout(bench_out, npu_out) else: @@ -117,7 +120,7 @@ class Comparator: is_bwd_success, bwd_compare_alg_results = self._compare_core_wrapper(bench_grad, npu_grad) else: is_bwd_success, bwd_compare_alg_results = CompareConst.NA, None - self.record_results(api_name, is_fwd_success, is_bwd_success, fwd_compare_alg_results, bwd_compare_alg_results) + self.record_results(api_name, is_fwd_success, is_bwd_success, fwd_compare_alg_results, bwd_compare_alg_results, shape_info) if is_fwd_success and is_bwd_success: self.test_result_cnt['success_num'] += 1 elif not is_fwd_success and not is_bwd_success: diff --git a/debug/accuracy_tools/api_accuracy_checker/run_ut/run_ut.py b/debug/accuracy_tools/api_accuracy_checker/run_ut/run_ut.py index 707d6cbed9..e899a006c1 100644 --- a/debug/accuracy_tools/api_accuracy_checker/run_ut/run_ut.py +++ b/debug/accuracy_tools/api_accuracy_checker/run_ut/run_ut.py @@ -205,8 +205,8 @@ def run_backward(api_full_name, args, backward_content, grad_index, npu_args, np return grad_out, npu_grad_out, grad, npu_grad -def initialize_save_error_data(): - error_data_path = os.path.realpath(msCheckerConfig.error_data_path) +def initialize_save_error_data(out_path): + error_data_path = os.path.realpath(out_path) check_file_or_directory_path(error_data_path, True) initialize_save_path(error_data_path, 'ut_error_data') @@ -225,7 +225,7 @@ def _run_ut_parser(parser): required=False) parser.add_argument('-save_error_data', dest="save_error_data", action="store_true", help=" Save compare failed api output.", required=False) - parser.add_argument("-c", "--jit_compile", dest="jit_compile", help=" whether to turn on jit compile", + parser.add_argument("-j", "--jit_compile", dest="jit_compile", help=" whether to turn on jit compile", default=False, required=False) parser.add_argument("-d", "--device", dest="device_id", type=int, help=" set NPU device id to run ut", default=0, required=False) @@ -249,7 +249,7 @@ def _run_ut(): out_path = os.path.realpath(args.out_path) if args.out_path else "./" save_error_data = args.save_error_data if save_error_data: - initialize_save_error_data() + initialize_save_error_data(out_path) run_ut(forward_file, backward_file, out_path, save_error_data) -- Gitee From 815f6ea84cde0bef932cb1013aa952755919f38e Mon Sep 17 00:00:00 2001 From: s30048155 Date: Sat, 9 Sep 2023 11:09:31 +0800 Subject: [PATCH 2/6] update --- .../api_accuracy_checker/compare/compare.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/compare.py b/debug/accuracy_tools/api_accuracy_checker/compare/compare.py index 244eb053bc..1ac2cb719e 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/compare.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/compare.py @@ -57,7 +57,7 @@ class Comparator: write_csv(summary_test_rows, self.save_path) detail_test_rows = [[ - "Npu Name", "Bench Dtype", "NPU Dtype", + "Npu Name", "Bench Dtype", "NPU Dtype", "Shape", "Cosine Similarity", "Cosine Similarity Message", "Max Rel Error", "Max Rel Err Message", "Relative Error (dual thousandth)", "Relative Error (dual thousandth) Message", @@ -86,8 +86,6 @@ class Comparator: subject_prefix = test_result[0] fwd_result = test_result[3] bwd_result = test_result[4] - shape_info = test_result[5] - test_rows.append(["Shape Info"] + list(shape_info)) if isinstance(fwd_result, list): for i, test_subject in enumerate(fwd_result): subject = subject_prefix + ".forward.output." + str(i) @@ -108,7 +106,6 @@ class Comparator: self.compare_alg.update({name: (compare_func, standard)}) def compare_output(self, api_name, bench_out, npu_out, bench_grad=None, npu_grad=None): - shape_info = npu_out.shape if "dropout" in api_name: is_fwd_success, fwd_compare_alg_results = self._compare_dropout(bench_out, npu_out) else: @@ -120,7 +117,7 @@ class Comparator: is_bwd_success, bwd_compare_alg_results = self._compare_core_wrapper(bench_grad, npu_grad) else: is_bwd_success, bwd_compare_alg_results = CompareConst.NA, None - self.record_results(api_name, is_fwd_success, is_bwd_success, fwd_compare_alg_results, bwd_compare_alg_results, shape_info) + self.record_results(api_name, is_fwd_success, is_bwd_success, fwd_compare_alg_results, bwd_compare_alg_results) if is_fwd_success and is_bwd_success: self.test_result_cnt['success_num'] += 1 elif not is_fwd_success and not is_bwd_success: @@ -136,12 +133,14 @@ class Comparator: detailed_result_total = [] bench_dtype_total = [] npu_dtype_total = [] + shape_total = [] test_success_total = True for name in self.compare_alg.keys(): alg = self.compare_alg[name][0] detailed_result, test_success, bench_dtype, npu_dtype = compare_core(bench_out, npu_out, alg) bench_dtype_total = bench_dtype npu_dtype_total = npu_dtype + shape_total = npu_out.shape if name != "Max Relative Error": test_success_total = test_success_total and test_success if detailed_result_total: @@ -154,6 +153,7 @@ class Comparator: detailed_result = list(detailed_result_total[i]) detailed_result.insert(0, bench_dtype_total[i]) detailed_result.insert(1, npu_dtype_total[i]) + detailed_result.insert(2, shape_total[i]) detailed_result.append(str(test_success_total)) detailed_result_total[i] = tuple(detailed_result) return test_success_total, detailed_result_total -- Gitee From de21e702638bcc835bdc411700e9bc5565e383c1 Mon Sep 17 00:00:00 2001 From: s30048155 Date: Sat, 9 Sep 2023 11:46:25 +0800 Subject: [PATCH 3/6] update --- .../api_accuracy_checker/compare/algorithm.py | 17 ++++++++++++----- .../api_accuracy_checker/compare/compare.py | 4 ++-- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py b/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py index b0b1aaf605..041aa0591f 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py @@ -152,24 +152,26 @@ def compare_core(bench_out, npu_out, alg): if not isinstance(bench_out, type(npu_out)): return [(CompareConst.NAN, "bench and npu output type is different.")], False, CompareConst.NA, CompareConst.NA if isinstance(bench_out, (list, tuple)): - compare_result, test_success, bench_dtype, npu_dtype = [], True, [], [] + compare_result, test_success, bench_dtype, npu_dtype, shape = [], True, [], [], [] if len(bench_out) != len(npu_out): return [(CompareConst.NAN, "bench and npu output structure is different")], False, CompareConst.NA, CompareConst.NA for b_out_i, n_out_i in zip(bench_out, npu_out): - compare_result_i, test_success_i, bench_dtype_i, npu_dtype_i = compare_core(b_out_i, n_out_i, alg) + compare_result_i, test_success_i, bench_dtype_i, npu_dtype_i, shape_i = compare_core(b_out_i, n_out_i, alg) compare_result.append(compare_result_i) test_success = test_success and test_success_i bench_dtype.append(bench_dtype_i) npu_dtype.append(npu_dtype_i) + shape.append(shape_i) elif isinstance(bench_out, dict): b_keys, n_keys = set(bench_out.keys()), set(npu_out.keys()) if b_keys != n_keys: - compare_result, test_success, bench_dtype, npu_dtype = [(CompareConst.NAN, "bench and npu output dict keys are different")], False, \ + compare_result, test_success, bench_dtype, npu_dtype, shape = [(CompareConst.NAN, "bench and npu output dict keys are different")], False, \ CompareConst.NA, CompareConst.NA - compare_result, test_success, bench_dtype, npu_dtype = compare_core(list(bench_out.values()), list(npu_out.values()), alg) + compare_result, test_success, bench_dtype, npu_dtype, shape = compare_core(list(bench_out.values()), list(npu_out.values()), alg) elif isinstance(bench_out, torch.Tensor): bench_dtype = str(bench_out.dtype) npu_dtype = str(npu_out.dtype) + shape = str(npu_out.shape) if bench_out.dtype in [torch.float32, torch.float64] and bench_out.dtype != npu_out.dtype: npu_out = npu_out.type(bench_out.dtype) compare_result, test_success, msg = compare_torch_tensor(bench_out.detach().numpy(), npu_out.detach().cpu().numpy(), alg) @@ -177,15 +179,18 @@ def compare_core(bench_out, npu_out, alg): compare_result, test_success, msg = compare_builtin_type(bench_out, npu_out) bench_dtype = str(type(bench_out)) npu_dtype = str(type(npu_out)) + shape = str(type(npu_out)) elif bench_out is None: compare_result, test_success, msg = CompareConst.NA, True, "output is None" bench_dtype = CompareConst.NAN npu_dtype = CompareConst.NAN + shape = CompareConst.NAN else: compare_result, test_success, msg = CompareConst.NA, True, "Unexpected output type \ in compare_core: {}".format(type(bench_out)) bench_dtype = CompareConst.NAN npu_dtype = CompareConst.NAN + shape = CompareConst.NAN if isinstance(compare_result, list): compare_result = flatten_compare_result(compare_result) else: @@ -193,9 +198,11 @@ def compare_core(bench_out, npu_out, alg): if isinstance(bench_dtype, list): bench_dtype = flatten_compare_result(bench_dtype) npu_dtype = flatten_compare_result(npu_dtype) + shape = flatten_compare_result(shape) else: bench_dtype = [bench_dtype] npu_dtype = [npu_dtype] - return compare_result, test_success, bench_dtype, npu_dtype + shape = [shape] + return compare_result, test_success, bench_dtype, npu_dtype, shape diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/compare.py b/debug/accuracy_tools/api_accuracy_checker/compare/compare.py index 1ac2cb719e..f46d0f3ce8 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/compare.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/compare.py @@ -137,10 +137,10 @@ class Comparator: test_success_total = True for name in self.compare_alg.keys(): alg = self.compare_alg[name][0] - detailed_result, test_success, bench_dtype, npu_dtype = compare_core(bench_out, npu_out, alg) + detailed_result, test_success, bench_dtype, npu_dtype, shape = compare_core(bench_out, npu_out, alg) bench_dtype_total = bench_dtype npu_dtype_total = npu_dtype - shape_total = npu_out.shape + shape_total = shape if name != "Max Relative Error": test_success_total = test_success_total and test_success if detailed_result_total: -- Gitee From ee9cc7eaaf49623ddc58e951bf3f40cd1a602048 Mon Sep 17 00:00:00 2001 From: s30048155 Date: Sat, 9 Sep 2023 12:44:28 +0800 Subject: [PATCH 4/6] fix --- debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py b/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py index 041aa0591f..58b8ebb49c 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py @@ -171,7 +171,7 @@ def compare_core(bench_out, npu_out, alg): elif isinstance(bench_out, torch.Tensor): bench_dtype = str(bench_out.dtype) npu_dtype = str(npu_out.dtype) - shape = str(npu_out.shape) + shape = list(npu_out.shape) if bench_out.dtype in [torch.float32, torch.float64] and bench_out.dtype != npu_out.dtype: npu_out = npu_out.type(bench_out.dtype) compare_result, test_success, msg = compare_torch_tensor(bench_out.detach().numpy(), npu_out.detach().cpu().numpy(), alg) -- Gitee From d574b004595bf99e11d9376ace42893a72c0cfca Mon Sep 17 00:00:00 2001 From: s30048155 Date: Mon, 11 Sep 2023 10:37:10 +0800 Subject: [PATCH 5/6] update --- debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py b/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py index ae1549f192..b6ce2fa044 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py @@ -171,7 +171,7 @@ def compare_core(bench_out, npu_out, alg): b_keys, n_keys = set(bench_out.keys()), set(npu_out.keys()) if b_keys != n_keys: compare_result, test_success, bench_dtype, npu_dtype, shape = [(CompareConst.NAN, "bench and npu output dict keys are different")], False, \ - CompareConst.NA, CompareConst.NA + CompareConst.NA, CompareConst.NA, CompareConst.NA compare_result, test_success, bench_dtype, npu_dtype, shape = compare_core(list(bench_out.values()), list(npu_out.values()), alg) elif isinstance(bench_out, torch.Tensor): bench_dtype = str(bench_out.dtype) -- Gitee From f21764643d06b94511f9c5e9daedfb7830b0e9e2 Mon Sep 17 00:00:00 2001 From: s30048155 Date: Mon, 11 Sep 2023 10:46:46 +0800 Subject: [PATCH 6/6] revert --- debug/accuracy_tools/api_accuracy_checker/run_ut/run_ut.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/debug/accuracy_tools/api_accuracy_checker/run_ut/run_ut.py b/debug/accuracy_tools/api_accuracy_checker/run_ut/run_ut.py index e899a006c1..18522776c4 100644 --- a/debug/accuracy_tools/api_accuracy_checker/run_ut/run_ut.py +++ b/debug/accuracy_tools/api_accuracy_checker/run_ut/run_ut.py @@ -205,8 +205,8 @@ def run_backward(api_full_name, args, backward_content, grad_index, npu_args, np return grad_out, npu_grad_out, grad, npu_grad -def initialize_save_error_data(out_path): - error_data_path = os.path.realpath(out_path) +def initialize_save_error_data(): + error_data_path = os.path.realpath(msCheckerConfig.error_data_path) check_file_or_directory_path(error_data_path, True) initialize_save_path(error_data_path, 'ut_error_data') @@ -249,7 +249,7 @@ def _run_ut(): out_path = os.path.realpath(args.out_path) if args.out_path else "./" save_error_data = args.save_error_data if save_error_data: - initialize_save_error_data(out_path) + initialize_save_error_data() run_ut(forward_file, backward_file, out_path, save_error_data) -- Gitee