From e5265cd227ad256228b7e24449bc847e608647d1 Mon Sep 17 00:00:00 2001 From: gitee Date: Tue, 20 Feb 2024 15:55:44 +0800 Subject: [PATCH 1/3] ut --- .../test/ut/compare/test_algorithm.py | 70 ++++++++++++++++--- .../test/ut/compare/test_compare.py | 8 +++ 2 files changed, 70 insertions(+), 8 deletions(-) diff --git a/debug/accuracy_tools/api_accuracy_checker/test/ut/compare/test_algorithm.py b/debug/accuracy_tools/api_accuracy_checker/test/ut/compare/test_algorithm.py index 90e18d166f5..0cd6c8b4d1d 100644 --- a/debug/accuracy_tools/api_accuracy_checker/test/ut/compare/test_algorithm.py +++ b/debug/accuracy_tools/api_accuracy_checker/test/ut/compare/test_algorithm.py @@ -4,13 +4,60 @@ import torch from api_accuracy_checker.compare import compare as cmp from api_accuracy_checker.compare import algorithm as alg + class TestAlgorithmMethods(unittest.TestCase): + def setUp(self): + self.bench_data = np.array([1.0, 1.0, 9.0], dtype=np.float16) + self.device_data = np.array([5.0, 2.0, 1.0], dtype=np.float16) + self.abs_err = np.abs(self.device_data - self.bench_data) + self.rel_err_orrign = np.abs(self.abs_err / self.bench_data) + eps = np.finfo(self.bench_data.dtype).eps + abs_bench = np.abs(self.bench_data) + abs_bench_with_eps = abs_bench + eps + self.rel_err = self.abs_err / abs_bench_with_eps + + def test_cosine_sim(self): + cpu_output = np.array([1.0, 2.0, 3.0]) + npu_output = np.array([1.0, 2.0, 3.0]) + self.assertEqual(alg.cosine_sim(cpu_output, npu_output), (1.0, True, '')) + + def test_get_rmse(self): + inf_nan_mask = [False, False, False] + self.assertAlmostEqual(alg.get_rmse(self.abs_err, inf_nan_mask), 5.196, 3) + + def test_get_error_balance(self): + self.assertEqual(alg.get_error_balance(self.bench_data, self.device_data), 1/3) + + def test_get_small_value_err_ratio(self): + small_value_mask = [True, True, True, False, True] + abs_err_greater_mask = [False, True, True, True, False] + self.assertEqual(alg.get_small_value_err_ratio(small_value_mask, abs_err_greater_mask), 0.5) + + def get_rel_err(self): + eps = np.finfo(self.bench_data.dtype).eps + abs_bench = np.abs(self.bench_data) + abs_bench_with_eps = abs_bench + eps + small_value_mask = [False, False, False] + inf_nan_mask = [False, False, False] + rel_err = self.abs_err / abs_bench_with_eps + self.assertListEqual(list(alg.get_rel_err(self.abs_err, abs_bench_with_eps, small_value_mask, inf_nan_mask)), + list(rel_err)) + + def test_get_abs_err(self): + self.assertListEqual(list(alg.get_abs_err(self.bench_data, self.device_data)), [4.0, 1.0, 8.0]) + + def test_get_rel_err_origin(self): + self.assertListEqual(list(alg.get_rel_err_origin(self.abs_err, self.bench_data)), list(self.rel_err_orrign)) + def test_get_max_abs_err(self): - b_value = np.array([1.0, 2.0, 3.0]) - n_value = np.array([1.0, 2.0, 3.0]) - abs_err = np.abs(b_value - n_value) - self.assertEqual(alg.get_max_abs_err(abs_err), (0.0, True)) + self.assertEqual(alg.get_max_abs_err(self.abs_err), (8.0, False)) + + def test_get_max_rel_err(self): + self.assertAlmostEqual(alg.get_max_rel_err(self.rel_err), 3.996, 3) + + def test_get_mean_rel_err(self): + self.assertAlmostEqual(alg.get_mean_rel_err(self.rel_err), 1.961, 3) def test_get_rel_err_ratio_thousandth(self): b_value = np.array([1.0, 2.0, 3.0]) @@ -26,7 +73,14 @@ class TestAlgorithmMethods(unittest.TestCase): rel_err = alg.get_rel_err_origin(abs_err, b_value) self.assertEqual(alg.get_rel_err_ratio(rel_err, 0.0001), (1.0, True)) - def test_cosine_sim(self): - cpu_output = np.array([1.0, 2.0, 3.0]) - npu_output = np.array([1.0, 2.0, 3.0]) - self.assertEqual(alg.cosine_sim(cpu_output, npu_output), (1.0, True, '')) + def test_get_finite_and_infinite_mask(self): + both_finite_mask, inf_nan_mask = alg.get_finite_and_infinite_mask(sekf.bench_data, self.device_data) + self.assertListEqual(list(both_finite_mask), [True, True, True]) + self.assertListEqual(list(inf_nan_mask), [False, False, False]) + + def test_get_small_value_mask(self): + b_value = np.array([1e-7, 1.0, 2e-6], dtype=np.float16) + abs_bench = np.abs(b_value) + both_finite_mask = [True, True, True] + small_value_mask = alg.get_small_value_mask(abs_bench, both_finite_mask, 1e-3) + self.assertListEqual(list(small_value_mask), [True, False, True]) diff --git a/debug/accuracy_tools/api_accuracy_checker/test/ut/compare/test_compare.py b/debug/accuracy_tools/api_accuracy_checker/test/ut/compare/test_compare.py index 242db8584af..c49d93f7d6b 100644 --- a/debug/accuracy_tools/api_accuracy_checker/test/ut/compare/test_compare.py +++ b/debug/accuracy_tools/api_accuracy_checker/test/ut/compare/test_compare.py @@ -103,3 +103,11 @@ class TestCompare(unittest.TestCase): npu_out = 1 status, compare_result, message = self.compare._compare_builtin_type(bench_out, npu_out, compare_column) self.assertEqual((status, compare_result.error_rate, message), ('pass', 0, '')) + + def test_compare_float_tensor(self): + cpu_output = torch.Tensor([1.0, 2.0, 3.0]) + npu_output = torch.Tensor([1.0, 2.0, 3.0]) + compare_column = CompareColumn() + status, compare_column, message = self.compare._compare_float_tensor(cpu_output.numpy(), npu_output.numpy(), + compare_column, npu_output.dtype) + self.assertEqual(status, "pass") -- Gitee From 976b1b86996ecae34b732ef1cef437232021d94f Mon Sep 17 00:00:00 2001 From: gitee Date: Tue, 20 Feb 2024 16:03:48 +0800 Subject: [PATCH 2/3] fix --- .../api_accuracy_checker/test/ut/compare/test_algorithm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/api_accuracy_checker/test/ut/compare/test_algorithm.py b/debug/accuracy_tools/api_accuracy_checker/test/ut/compare/test_algorithm.py index 0cd6c8b4d1d..44570d73644 100644 --- a/debug/accuracy_tools/api_accuracy_checker/test/ut/compare/test_algorithm.py +++ b/debug/accuracy_tools/api_accuracy_checker/test/ut/compare/test_algorithm.py @@ -74,7 +74,7 @@ class TestAlgorithmMethods(unittest.TestCase): self.assertEqual(alg.get_rel_err_ratio(rel_err, 0.0001), (1.0, True)) def test_get_finite_and_infinite_mask(self): - both_finite_mask, inf_nan_mask = alg.get_finite_and_infinite_mask(sekf.bench_data, self.device_data) + both_finite_mask, inf_nan_mask = alg.get_finite_and_infinite_mask(self.bench_data, self.device_data) self.assertListEqual(list(both_finite_mask), [True, True, True]) self.assertListEqual(list(inf_nan_mask), [False, False, False]) -- Gitee From 1593cef699285692514fc7a7bde1adc5f7a55635 Mon Sep 17 00:00:00 2001 From: gitee Date: Tue, 20 Feb 2024 16:37:12 +0800 Subject: [PATCH 3/3] fix --- .../api_accuracy_checker/test/ut/compare/test_algorithm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/api_accuracy_checker/test/ut/compare/test_algorithm.py b/debug/accuracy_tools/api_accuracy_checker/test/ut/compare/test_algorithm.py index 44570d73644..8146d72e3e3 100644 --- a/debug/accuracy_tools/api_accuracy_checker/test/ut/compare/test_algorithm.py +++ b/debug/accuracy_tools/api_accuracy_checker/test/ut/compare/test_algorithm.py @@ -27,7 +27,7 @@ class TestAlgorithmMethods(unittest.TestCase): self.assertAlmostEqual(alg.get_rmse(self.abs_err, inf_nan_mask), 5.196, 3) def test_get_error_balance(self): - self.assertEqual(alg.get_error_balance(self.bench_data, self.device_data), 1/3) + self.assertEqual(alg.get_error_balance(self.bench_data, self.device_data), 1 / 3) def test_get_small_value_err_ratio(self): small_value_mask = [True, True, True, False, True] -- Gitee