From cbfe1dd18b0da1f3ac5f71833732c1df0c11e56c Mon Sep 17 00:00:00 2001 From: gitee Date: Fri, 19 Apr 2024 10:54:28 +0800 Subject: [PATCH 1/5] fix --- .../compare/api_precision_compare.py | 31 ++++++----- .../api_accuracy_checker/compare/compare.py | 24 ++++++--- .../api_accuracy_checker/run_ut/run_ut.py | 51 ++++++++++++------- .../run_ut/run_ut_utils.py | 4 ++ 4 files changed, 72 insertions(+), 38 deletions(-) create mode 100644 debug/accuracy_tools/api_accuracy_checker/run_ut/run_ut_utils.py diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/api_precision_compare.py b/debug/accuracy_tools/api_accuracy_checker/compare/api_precision_compare.py index ad38cb956..34e7e7a85 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/api_precision_compare.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/api_precision_compare.py @@ -191,20 +191,25 @@ def analyse_csv(npu_data, gpu_data, config): msg = f'This API : {full_api_name_with_direction_status} has multiple records in the GPU data.' raise CompareException(CompareException.INVALID_DATA_ERROR, msg) row_gpu = row_gpu.iloc[0] + new_status = CompareConst.SPACE #当前API的输出为空（例如反向过程中requires_grad=False）,跳过比对 if row_npu[ApiPrecisionCompareColumn.DEVICE_DTYPE].isspace(): - continue - _, api_name, _ = full_api_name.split("*") - new_status = CompareConst.SPACE - compare_column.api_name = full_api_name_with_direction_status - if row_npu[ApiPrecisionCompareColumn.DEVICE_DTYPE] not in BINARY_COMPARE_UNSUPPORT_LIST or api_name in BinaryStandardApi: - new_status = record_binary_consistency_result(api_name, compare_column, row_npu) - elif api_name in AbsoluteStandardApi: - new_status = record_absolute_threshold_result(compare_column, row_npu) - elif row_npu[ApiPrecisionCompareColumn.DEVICE_DTYPE] in BENCHMARK_COMPARE_SUPPORT_LIST: - bs = BenchmarkStandard(full_api_name_with_direction_status, row_npu, row_gpu) - new_status = record_benchmark_compare_result(compare_column, bs) - write_detail_csv(compare_column.to_column_value(), config.details_csv_path) + compare_column.api_name = full_api_name_with_direction_status + compare_column.compare_result = CompareConst.SKIP + compare_column.message = row_npu[ApiPrecisionCompareColumn.MESSAGE] + new_status = CompareConst.SKIP + write_detail_csv(compare_column.to_column_value(), config.details_csv_path) + else: + _, api_name, _ = full_api_name.split("*") + compare_column.api_name = full_api_name_with_direction_status + if row_npu[ApiPrecisionCompareColumn.DEVICE_DTYPE] not in BINARY_COMPARE_UNSUPPORT_LIST or api_name in BinaryStandardApi: + new_status = record_binary_consistency_result(api_name, compare_column, row_npu) + elif api_name in AbsoluteStandardApi: + new_status = record_absolute_threshold_result(compare_column, row_npu) + elif row_npu[ApiPrecisionCompareColumn.DEVICE_DTYPE] in BENCHMARK_COMPARE_SUPPORT_LIST: + bs = BenchmarkStandard(full_api_name_with_direction_status, row_npu, row_gpu) + new_status = record_benchmark_compare_result(compare_column, bs) + write_detail_csv(compare_column.to_column_value(), config.details_csv_path) if full_last_api_name is not None and full_api_name != full_last_api_name: if last_api_dtype in API_PRECISION_COMPARE_UNSUPPORT_LIST: @@ -279,6 +284,8 @@ def get_absolute_threshold_result(row_npu): def get_api_checker_result(status): if not status: return CompareConst.SPACE + if all(item == CompareConst.SKIP for item in status): + return CompareConst.SKIP for const in (CompareConst.ERROR, CompareConst.WARNING): if const in status: return const diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/compare.py b/debug/accuracy_tools/api_accuracy_checker/compare/compare.py index bd10f7797..9ef8ae1a0 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/compare.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/compare.py @@ -97,7 +97,7 @@ class Comparator: self.test_result_cnt["total_skip_num"] += 1 continue self.test_result_cnt["total_num"] += 1 - if column1 == CompareConst.PASS and column2 in [CompareConst.PASS, CompareConst.SPACE]: + if column1 == CompareConst.PASS and column2 in [CompareConst.PASS, CompareConst.SPACE, CompareConst.SKIP]: self.test_result_cnt['success_num'] += 1 elif column1 == CompareConst.ERROR and column2 == CompareConst.ERROR: self.test_result_cnt['forward_and_backward_fail_num'] += 1 @@ -125,7 +125,7 @@ class Comparator: name = test_result[0] df_row = list(test_result[:3]) - if test_result[1] == "SKIP" or test_result[2] == "SKIP": + if test_result[1] == "SKIP": df_row.append(test_result[3]) if self.stack_info: stack_info = "\n".join(self.stack_info[name]) @@ -158,18 +158,28 @@ class Comparator: self.write_summary_csv(args) self.write_detail_csv(args) - def compare_output(self, full_api_name, bench_output, device_output, bench_grad=None, npu_grad=None): + def compare_output(self, full_api_name, data_info): _, api_name, _ = full_api_name.split("*") + bench_output = data_info.bench_output + device_output = data_info.device_output + bench_grad = data_info.bench_grad + device_grad = data_info.device_grad + backward_message = data_info.backward_message compare_func = self._compare_dropout if "dropout" in full_api_name else self._compare_core_wrapper fwd_success_status, fwd_compare_alg_results = compare_func(api_name, bench_output, device_output) - if not (bench_grad and npu_grad): + if not (bench_grad and device_grad): bwd_success_status, bwd_compare_alg_results = (CompareConst.SPACE, []) else: if "dropout" in full_api_name: - bwd_success_status, bwd_compare_alg_results = compare_func(api_name, bench_grad[0], npu_grad[0]) + bwd_success_status, bwd_compare_alg_results = compare_func(api_name, bench_grad[0], device_grad[0]) else: - bwd_success_status, bwd_compare_alg_results = compare_func(api_name, bench_grad, npu_grad) - self.record_results(full_api_name, fwd_success_status, bwd_success_status if bwd_compare_alg_results is not None else CompareConst.SPACE, fwd_compare_alg_results, bwd_compare_alg_results) + bwd_success_status, bwd_compare_alg_results = compare_func(api_name, bench_grad, device_grad) + if backward_message: + backward_column = CompareColumn() + bwd_compare_alg_results = backward_column.to_column_value(CompareConst.SKIP, backward_message) + self.record_results(full_api_name, fwd_success_status, CompareConst.SKIP, fwd_compare_alg_results, [bwd_compare_alg_results]) + else: + self.record_results(full_api_name, fwd_success_status, bwd_success_status if bwd_compare_alg_results is not None else CompareConst.SPACE, fwd_compare_alg_results, bwd_compare_alg_results) return fwd_success_status == CompareConst.PASS, bwd_success_status == CompareConst.PASS \ or bwd_success_status == CompareConst.SPACE diff --git a/debug/accuracy_tools/api_accuracy_checker/run_ut/run_ut.py b/debug/accuracy_tools/api_accuracy_checker/run_ut/run_ut.py index e9e8b161e..251e488c0 100644 --- a/debug/accuracy_tools/api_accuracy_checker/run_ut/run_ut.py +++ b/debug/accuracy_tools/api_accuracy_checker/run_ut/run_ut.py @@ -17,9 +17,12 @@ else: import torch from tqdm import tqdm from api_accuracy_checker.run_ut.data_generate import gen_api_params, gen_args +from api_accuracy_checker.run_ut.run_ut_utils import Backward_Message from api_accuracy_checker.common.utils import print_info_log, print_warn_log, get_json_contents, api_info_preprocess, \ print_error_log, initialize_save_path, Const, create_directory from api_accuracy_checker.compare.compare import Comparator +from api_accuracy_checker.compare.compare_column import CompareColumn +from api_accuracy_checker.compare.compare_utils import CompareConst from api_accuracy_checker.hook_module.wrap_tensor import TensorOPTemplate from api_accuracy_checker.hook_module.wrap_functional import FunctionalOPTemplate from api_accuracy_checker.hook_module.wrap_torch import TorchOPTemplate @@ -172,10 +175,7 @@ def run_ut(config): continue data_info = run_torch_api(api_full_name, config.real_data_path, config.backward_content, api_info_dict) is_fwd_success, is_bwd_success = compare.compare_output(api_full_name, - data_info.bench_out, - data_info.device_out, - data_info.bench_grad_out, - data_info.device_grad_out) + data_info) if config.save_error_data: do_save_error_data(api_full_name, data_info, is_fwd_success, is_bwd_success) except Exception as err: @@ -185,7 +185,9 @@ def run_ut(config): f"'int32_to_int64' list in accuracy_tools/api_accuracy_check/common/utils.py file.") else: print_error_log(f"Run {api_full_name} UT Error: %s" % str(err)) - compare.write_summary_csv((api_full_name, "SKIP", "SKIP", str(err))) + err_column = CompareColumn() + fwd_compare_alg_results = err_column.to_column_value(CompareConst.SKIP, str(err)) + compare.record_results(api_full_name, CompareConst.SKIP, CompareConst.SKIP, [fwd_compare_alg_results], None) finally: if is_gpu: torch.cuda.empty_cache() @@ -211,6 +213,7 @@ def do_save_error_data(api_full_name, data_info, is_fwd_success, is_bwd_success) def run_torch_api(api_full_name, real_data_path, backward_content, api_info_dict): in_fwd_data_list = [] + backward_message = '' [api_type, api_name, _] = api_full_name.split("*") args, kwargs, need_grad = get_api_info(api_info_dict, api_name, real_data_path) in_fwd_data_list.append(args) @@ -219,10 +222,12 @@ def run_torch_api(api_full_name, real_data_path, backward_content, api_info_dict if not need_grad: print_warn_log("%s function with out=... arguments don't support automatic differentiation, skip backward." % api_full_name) + backward_message += Backward_Message.UNSUPPORT_BACKWARD_MESSAGE if api_name in not_backward_list: need_grad = False print_warn_log( "%s function backward result is None, skip backward." % api_full_name) + backward_message += Backward_Message.NO_BACKWARD_RESULT_MESSAGE need_backward = need_backward and need_grad if kwargs.get("device"): del kwargs["device"] @@ -241,14 +246,17 @@ def run_torch_api(api_full_name, real_data_path, backward_content, api_info_dict grad_index = grad_input_index.get('grad_index') if need_backward: - backward_args = backward_content[api_full_name] - grad = gen_args(backward_args, real_data_path=real_data_path)[0] - bench_grad, _ = generate_cpu_params(grad, {}, False, api_name) - bench_grad_out = run_backward(cpu_args, bench_grad, grad_index, out) - device_grad = grad.clone().detach().to(current_device) - device_grad_out = run_backward(device_args, device_grad, grad_index, device_out) + if need_to_backward(grad_index, out): + backward_args = backward_content[api_full_name] + grad = gen_args(backward_args, real_data_path=real_data_path)[0] + bench_grad, _ = generate_cpu_params(grad, {}, False, api_name) + bench_grad_out = run_backward(cpu_args, bench_grad, grad_index, out) + device_grad = grad.clone().detach().to(current_device) + device_grad_out = run_backward(device_args, device_grad, grad_index, device_out) + else: + backward_message += Backward_Message.MULTIPLE_BACKWARD_MESSAGE - return UtDataInfo(bench_grad_out, device_grad_out, device_out, out, bench_grad, in_fwd_data_list) + return UtDataInfo(bench_grad_out, device_grad_out, device_out, out, bench_grad, in_fwd_data_list, backward_message) def get_api_info(api_info_dict, api_name, real_data_path): @@ -260,12 +268,16 @@ def get_api_info(api_info_dict, api_name, real_data_path): return args, kwargs, need_grad +def need_to_backward(grad_index, out): + if grad_index is None and isinstance(out, (list, tuple)): + return False + return True + + def run_backward(args, grad, grad_index, out): if grad_index is not None: out[grad_index].backward(grad) - elif isinstance(out, (list, tuple)): - raise NotImplementedError("Multiple backward is not supported.") else: out.backward(grad) args_grad = [] @@ -437,13 +449,14 @@ def run_ut_command(args): class UtDataInfo: - def __init__(self, bench_grad_out, device_grad_out, device_out, bench_out, grad_in, in_fwd_data_list): - self.bench_grad_out = bench_grad_out - self.device_grad_out = device_grad_out - self.device_out = device_out - self.bench_out = bench_out + def __init__(self, bench_grad, device_grad, device_output, bench_output, grad_in, in_fwd_data_list, backward_message): + self.bench_grad = bench_grad + self.device_grad = device_grad + self.device_output = device_output + self.bench_output = bench_output self.grad_in = grad_in self.in_fwd_data_list = in_fwd_data_list + self.backward_message = backward_message class UtAPIInfo(APIInfo): diff --git a/debug/accuracy_tools/api_accuracy_checker/run_ut/run_ut_utils.py b/debug/accuracy_tools/api_accuracy_checker/run_ut/run_ut_utils.py new file mode 100644 index 000000000..490d8a5db --- /dev/null +++ b/debug/accuracy_tools/api_accuracy_checker/run_ut/run_ut_utils.py @@ -0,0 +1,4 @@ +class Backward_Message: + MULTIPLE_BACKWARD_MESSAGE = "Multiple backward is not supported." + UNSUPPORT_BACKWARD_MESSAGE = "function with out=... arguments don't support automatic differentiation, skip backward." + NO_BACKWARD_RESULT_MESSAGE = "function backward result is None, skip backward." \ No newline at end of file -- Gitee From 06e8148a1e90034a1e7abdcf8e3d7b6a81c4b06c Mon Sep 17 00:00:00 2001 From: gitee Date: Fri, 19 Apr 2024 16:43:31 +0800 Subject: [PATCH 2/5] fix --- debug/accuracy_tools/api_accuracy_checker/compare/compare.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/compare.py b/debug/accuracy_tools/api_accuracy_checker/compare/compare.py index 9ef8ae1a0..94785413a 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/compare.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/compare.py @@ -126,7 +126,7 @@ class Comparator: name = test_result[0] df_row = list(test_result[:3]) if test_result[1] == "SKIP": - df_row.append(test_result[3]) + df_row.append(test_result[3][0][-1]) if self.stack_info: stack_info = "\n".join(self.stack_info[name]) df_row.append(stack_info) -- Gitee From 7297bc70d49e466ec9c282d003f90cd78920ce46 Mon Sep 17 00:00:00 2001 From: gitee Date: Fri, 19 Apr 2024 17:05:40 +0800 Subject: [PATCH 3/5] fix --- .../api_accuracy_checker/compare/api_precision_compare.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/api_precision_compare.py b/debug/accuracy_tools/api_accuracy_checker/compare/api_precision_compare.py index 34e7e7a85..322b8c0b0 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/api_precision_compare.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/api_precision_compare.py @@ -196,7 +196,7 @@ def analyse_csv(npu_data, gpu_data, config): if row_npu[ApiPrecisionCompareColumn.DEVICE_DTYPE].isspace(): compare_column.api_name = full_api_name_with_direction_status compare_column.compare_result = CompareConst.SKIP - compare_column.message = row_npu[ApiPrecisionCompareColumn.MESSAGE] + compare_column.compare_message = row_npu[ApiPrecisionCompareColumn.MESSAGE] new_status = CompareConst.SKIP write_detail_csv(compare_column.to_column_value(), config.details_csv_path) else: -- Gitee From 6667ccd9a4c605d1d626301f7fe2c6f106076208 Mon Sep 17 00:00:00 2001 From: gitee Date: Fri, 19 Apr 2024 17:51:19 +0800 Subject: [PATCH 4/5] fix --- debug/accuracy_tools/api_accuracy_checker/run_ut/run_ut.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/debug/accuracy_tools/api_accuracy_checker/run_ut/run_ut.py b/debug/accuracy_tools/api_accuracy_checker/run_ut/run_ut.py index 251e488c0..05bd4305a 100644 --- a/debug/accuracy_tools/api_accuracy_checker/run_ut/run_ut.py +++ b/debug/accuracy_tools/api_accuracy_checker/run_ut/run_ut.py @@ -220,13 +220,11 @@ def run_torch_api(api_full_name, real_data_path, backward_content, api_info_dict in_fwd_data_list.append(kwargs) need_backward = api_full_name in backward_content if not need_grad: - print_warn_log("%s function with out=... arguments don't support automatic differentiation, skip backward." - % api_full_name) + print_warn_log(f"{api_full_name} {Backward_Message.UNSUPPORT_BACKWARD_MESSAGE.format(api_full_name)}") backward_message += Backward_Message.UNSUPPORT_BACKWARD_MESSAGE if api_name in not_backward_list: need_grad = False - print_warn_log( - "%s function backward result is None, skip backward." % api_full_name) + print_warn_log(f"{api_full_name} {Backward_Message.NO_BACKWARD_RESULT_MESSAGE.format(api_full_name)}") backward_message += Backward_Message.NO_BACKWARD_RESULT_MESSAGE need_backward = need_backward and need_grad if kwargs.get("device"): -- Gitee From 7868b817ba58f87c99ddcefd56ded5a2b3881820 Mon Sep 17 00:00:00 2001 From: gitee Date: Sat, 20 Apr 2024 10:23:01 +0800 Subject: [PATCH 5/5] fix --- .../test/ut/compare/test_compare.py | 7 +++++-- .../test/ut/run_ut/test_run_ut.py | 11 ++++++----- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/debug/accuracy_tools/api_accuracy_checker/test/ut/compare/test_compare.py b/debug/accuracy_tools/api_accuracy_checker/test/ut/compare/test_compare.py index 4ce73ce55..c59a0a6c7 100644 --- a/debug/accuracy_tools/api_accuracy_checker/test/ut/compare/test_compare.py +++ b/debug/accuracy_tools/api_accuracy_checker/test/ut/compare/test_compare.py @@ -9,6 +9,7 @@ import torch.nn.functional from api_accuracy_checker.compare.compare import Comparator from api_accuracy_checker.compare.compare_column import CompareColumn +from api_accuracy_checker.run_ut.run_ut import UtDataInfo current_time = time.strftime("%Y%m%d%H%M%S") RESULT_FILE_NAME = "accuracy_checking_result_" + current_time + ".csv" @@ -70,13 +71,15 @@ class TestCompare(unittest.TestCase): bench_out, npu_out = torch.randn(100, 100), torch.randn(100, 100) bench_grad, npu_grad = [torch.randn(100, 100)], [torch.randn(100, 100)] api_name = 'Functional*conv2d*0' - is_fwd_success, is_bwd_success = self.compare.compare_output(api_name, bench_out, npu_out, bench_grad, npu_grad) + data_info = UtDataInfo(bench_grad, npu_grad, bench_out, npu_out, None, None, None) + is_fwd_success, is_bwd_success = self.compare.compare_output(api_name, data_info) self.assertFalse(is_fwd_success) self.assertFalse(is_bwd_success) dummy_input = torch.randn(100, 100) bench_out, npu_out = dummy_input, dummy_input - is_fwd_success, is_bwd_success = self.compare.compare_output(api_name, bench_out, npu_out) + data_info = UtDataInfo(None, None, bench_out, npu_out, None, None, None) + is_fwd_success, is_bwd_success = self.compare.compare_output(api_name, data_info) self.assertTrue(is_fwd_success) self.assertTrue(is_bwd_success) diff --git a/debug/accuracy_tools/api_accuracy_checker/test/ut/run_ut/test_run_ut.py b/debug/accuracy_tools/api_accuracy_checker/test/ut/run_ut/test_run_ut.py index fdcc1cfdd..3c180fa23 100644 --- a/debug/accuracy_tools/api_accuracy_checker/test/ut/run_ut/test_run_ut.py +++ b/debug/accuracy_tools/api_accuracy_checker/test/ut/run_ut/test_run_ut.py @@ -61,10 +61,11 @@ class TestRunUtMethods(unittest.TestCase): self.assertEqual(cpu_kwargs, {'inplace': False}) def test_UtDataInfo(self): - data_info = UtDataInfo(None, None, None, None, None, None) - self.assertIsNone(data_info.bench_grad_out) - self.assertIsNone(data_info.device_grad_out) - self.assertIsNone(data_info.device_out) - self.assertIsNone(data_info.bench_out) + data_info = UtDataInfo(None, None, None, None, None, None, None) + self.assertIsNone(data_info.bench_grad) + self.assertIsNone(data_info.device_grad) + self.assertIsNone(data_info.device_output) + self.assertIsNone(data_info.bench_output) self.assertIsNone(data_info.grad_in) self.assertIsNone(data_info.in_fwd_data_list) + self.assertIsNone(data_info.backward_message) -- Gitee