diff --git a/debug/accuracy_tools/api_accuracy_checker/common/base_api.py b/debug/accuracy_tools/api_accuracy_checker/common/base_api.py index 627bd769766863683d2191509a54f5e28660a623..78ce01957ffdcd0532ae366522e70a9a6ea2b65c 100644 --- a/debug/accuracy_tools/api_accuracy_checker/common/base_api.py +++ b/debug/accuracy_tools/api_accuracy_checker/common/base_api.py @@ -119,9 +119,9 @@ class BaseAPIInfo: elif operator == 'min': return False not in data if operator == 'max': - return torch._C._VariableFunctionsClass.max(data).item() + return torch._C._VariableFunctionsClass.max(data.float()).item() else: - return torch._C._VariableFunctionsClass.min(data).item() + return torch._C._VariableFunctionsClass.min(data.float()).item() def get_type_name(self, name): diff --git a/debug/accuracy_tools/api_accuracy_checker/dump/info_dump.py b/debug/accuracy_tools/api_accuracy_checker/dump/info_dump.py index f2a96bd0fac56b0da20c980bbf70caed2a6f6161..7ccfd84382a2b3666e699ac1e5ac7f8e355c8fb4 100644 --- a/debug/accuracy_tools/api_accuracy_checker/dump/info_dump.py +++ b/debug/accuracy_tools/api_accuracy_checker/dump/info_dump.py @@ -3,9 +3,9 @@ import json import os import threading -from .api_info import ForwardAPIInfo, BackwardAPIInfo -from ..common.utils import check_file_or_directory_path, initialize_save_path -from ..common.config import msCheckerConfig +from api_accuracy_checker.dump.api_info import ForwardAPIInfo, BackwardAPIInfo +from api_accuracy_checker.common.utils import check_file_or_directory_path, initialize_save_path +from api_accuracy_checker.common.config import msCheckerConfig lock = threading.Lock() diff --git a/debug/accuracy_tools/api_accuracy_checker/run_ut/run_ut.py b/debug/accuracy_tools/api_accuracy_checker/run_ut/run_ut.py index 034586cc1d29064eebc9c3cd09534caf46287b01..5b1142c3e590af6ba301bcc1104bac39fb6f649d 100644 --- a/debug/accuracy_tools/api_accuracy_checker/run_ut/run_ut.py +++ b/debug/accuracy_tools/api_accuracy_checker/run_ut/run_ut.py @@ -13,7 +13,7 @@ from api_accuracy_checker.compare.compare import Comparator from api_accuracy_checker.hook_module.wrap_tensor import TensorOPTemplate from api_accuracy_checker.hook_module.wrap_functional import FunctionalOPTemplate from api_accuracy_checker.hook_module.wrap_torch import TorchOPTemplate -from ut_api_info import UtAPIInfo +from api_accuracy_checker.run_ut.ut_api_info import UtAPIInfo from api_accuracy_checker.common.config import msCheckerConfig NO_GRAD_APIS = ["hardtanh"] diff --git a/debug/accuracy_tools/api_accuracy_checker/test/ut/common/test_base_api.py b/debug/accuracy_tools/api_accuracy_checker/test/ut/common/test_base_api.py new file mode 100644 index 0000000000000000000000000000000000000000..ba260c052a4333e1765fe8db905fb4a36dfb03ab --- /dev/null +++ b/debug/accuracy_tools/api_accuracy_checker/test/ut/common/test_base_api.py @@ -0,0 +1,66 @@ +import unittest +import torch +import os +import shutil +from api_accuracy_checker.common.base_api import BaseAPIInfo + +class TestBaseAPI(unittest.TestCase): + def setUp(self): + if os.path.exists('./forward'): + shutil.rmtree('./forward') + os.makedirs('./forward', mode=0o755) + self.api = BaseAPIInfo("test_api", True, True, "./", "forward", "backward") + + def test_analyze_element(self): + element = [1, 2, 3] + result = self.api.analyze_element(element) + self.assertEqual(result, [{'type': 'int', 'value': 1}, {'type': 'int', 'value': 2}, {'type': 'int', 'value': 3}]) + + def test_analyze_tensor(self): + tensor = torch.tensor([1, 2, 3], dtype=torch.float32, requires_grad=True) + result = self.api.analyze_tensor(tensor) + self.assertEqual(result['type'], 'torch.Tensor') + self.assertEqual(result['requires_grad'], True) + self.assertTrue(os.path.exists(result['datapath'])) + + def test_analyze_builtin(self): + arg = slice(1, 10, 2) + result = self.api.analyze_builtin(arg) + self.assertEqual(result, {'type': 'slice', 'value': [1, 10, 2]}) + + def test_transfer_types(self): + data = 10 + dtype = 'int' + result = self.api.transfer_types(data, dtype) + self.assertEqual(result, 10) + + def test_is_builtin_class(self): + element = 10 + result = self.api.is_builtin_class(element) + self.assertEqual(result, True) + + def test_analyze_device_in_kwargs(self): + element = torch.device('cuda:0') + result = self.api.analyze_device_in_kwargs(element) + self.assertEqual(result, {'type': 'torch.device', 'value': 'cuda:0'}) + + def test_analyze_dtype_in_kwargs(self): + element = torch.float32 + result = self.api.analyze_dtype_in_kwargs(element) + self.assertEqual(result, {'type': 'torch.dtype', 'value': 'torch.float32'}) + + def test_get_tensor_extremum(self): + data = torch.tensor([1, 2, 3]) + result_max = self.api.get_tensor_extremum(data, 'max') + result_min = self.api.get_tensor_extremum(data, 'min') + self.assertEqual(result_max, 3) + self.assertEqual(result_min, 1) + + def test_get_type_name(self): + name = "" + result = self.api.get_type_name(name) + self.assertEqual(result, 'int') + + def tearDown(self): + if os.path.exists('./forward'): + shutil.rmtree('./forward') \ No newline at end of file diff --git a/debug/accuracy_tools/api_accuracy_checker/test/ut/common/test_common_utils.py b/debug/accuracy_tools/api_accuracy_checker/test/ut/common/test_common_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..f8bae77bcc632d6f06519e52b29f4d6ab02c1448 --- /dev/null +++ b/debug/accuracy_tools/api_accuracy_checker/test/ut/common/test_common_utils.py @@ -0,0 +1,128 @@ +import unittest +import os +import numpy as np +import torch +from api_accuracy_checker.common.utils import * + +class TestUtils(unittest.TestCase): + + def test_read_json(self): + test_dict = {"key": "value"} + with open('test.json', 'w') as f: + json.dump(test_dict, f) + self.assertEqual(read_json('test.json'), test_dict) + os.remove('test.json') + + def test_write_csv(self): + test_data = [["name", "age"], ["Alice", "20"], ["Bob", "30"]] + write_csv(test_data, 'test.csv') + with open('test.csv', 'r') as f: + reader = csv.reader(f) + for i, row in enumerate(reader): + self.assertEqual(row, test_data[i]) + os.remove('test.csv') + + def test_print_info_log(self): + try: + print_info_log("Test message") + except Exception as e: + self.fail(f"print_info_log raised exception {e}") + + def test_check_mode_valid(self): + try: + check_mode_valid(Const.ALL) + except Exception as e: + self.fail(f"check_mode_valid raised exception {e}") + + def test_check_object_type(self): + try: + check_object_type(123, int) + except Exception as e: + self.fail(f"check_object_type raised exception {e}") + + def test_check_file_or_directory_path(self): + try: + check_file_or_directory_path(__file__) + except Exception as e: + self.fail(f"check_file_or_directory_path raised exception {e}") + + def test_get_dump_data_path(self): + path, exist = get_dump_data_path(os.path.dirname(__file__)) + self.assertTrue(exist) + + def test_get_api_name_from_matcher(self): + api_name = get_api_name_from_matcher("api_stack_1_add") + self.assertEqual(api_name, "stack") + + def test_create_directory(self): + create_directory('test_dir') + self.assertTrue(os.path.exists('test_dir')) + os.rmdir('test_dir') + + def test_execute_command(self): + execute_command(['echo', 'Hello, World!']) + + def test_parse_arg_value(self): + values = "1,2,3;4,5,6" + expected_result = [[1, 2, 3], [4, 5, 6]] + self.assertEqual(parse_arg_value(values), expected_result) + + def test_parse_value_by_comma(self): + value = "1,2,3" + expected_result = [1, 2, 3] + self.assertEqual(parse_value_by_comma(value), expected_result) + + def test_get_data_len_by_shape(self): + shape = [2, 3, 4] + expected_result = 24 + self.assertEqual(get_data_len_by_shape(shape), expected_result) + + def test_add_time_as_suffix(self): + name = "test" + result = add_time_as_suffix(name) + self.assertTrue(result.startswith(name)) + + def test_get_time(self): + result = get_time() + self.assertTrue(isinstance(result, str)) + + def test_format_value(self): + value = 123.456789 + expected_result = '123.456789' + self.assertEqual(format_value(value), expected_result) + + def test_seed_all(self): + seed_all(1234) + + def test_get_process_rank(self): + model = torch.nn.Linear(10, 10) + rank, _ = get_process_rank(model) + self.assertEqual(rank, 0) + + def test_get_json_contents(self): + test_dict = {"key": "value"} + with open('test.json', 'w') as f: + json.dump(test_dict, f) + self.assertEqual(get_json_contents('test.json'), test_dict) + os.remove('test.json') + + def test_get_file_content_bytes(self): + with open('test.txt', 'w') as f: + f.write("Hello, World!") + self.assertEqual(get_file_content_bytes('test.txt'), b"Hello, World!") + os.remove('test.txt') + + def test_islink(self): + self.assertFalse(islink(__file__)) + + def test_check_path_length_valid(self): + self.assertTrue(check_path_length_valid(__file__)) + + def test_check_path_pattern_valid(self): + self.assertIsNone(check_path_pattern_valid(__file__)) + + def test_check_input_file_valid(self): + self.assertIsNone(check_input_file_valid(__file__)) + + def test_check_need_convert(self): + self.assertIsNone(check_need_convert("unknown_api")) diff --git a/debug/accuracy_tools/api_accuracy_checker/test/ut/common/test_config.py b/debug/accuracy_tools/api_accuracy_checker/test/ut/common/test_config.py new file mode 100644 index 0000000000000000000000000000000000000000..ed764987d5a9287293f183c0bde1d86afd90ccae --- /dev/null +++ b/debug/accuracy_tools/api_accuracy_checker/test/ut/common/test_config.py @@ -0,0 +1,21 @@ +import unittest +import os +from api_accuracy_checker.common.config import Config + +class TestConfig(unittest.TestCase): + def setUp(self): + cur_path = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__))))) + yaml_path = os.path.join(cur_path, "config.yaml") + self.yaml_file = yaml_path + self.config = Config(self.yaml_file) + + def test_validate(self): + self.assertEqual(self.config.validate('dump_path', '/path/to/dump'), '/path/to/dump') + + with self.assertRaises(ValueError): + self.config.validate('dump_path', 123) + + + def test_update_config(self): + self.config.update_config(dump_path='/new/path/to/dump', enable_dataloader=False) + self.assertEqual(self.config.dump_path, '/new/path/to/dump') diff --git a/debug/accuracy_tools/api_accuracy_checker/test/ut/compare/test_algorithm.py b/debug/accuracy_tools/api_accuracy_checker/test/ut/compare/test_algorithm.py new file mode 100644 index 0000000000000000000000000000000000000000..701ea6f7ae132c58801cbcac81be88238217e6b2 --- /dev/null +++ b/debug/accuracy_tools/api_accuracy_checker/test/ut/compare/test_algorithm.py @@ -0,0 +1,72 @@ +import unittest +import numpy as np +from api_accuracy_checker.compare import algorithm as alg + +class TestAlgorithmMethods(unittest.TestCase): + + def test_compare_torch_tensor(self): + cpu_output = np.array([1, 2, 3]) + npu_output = np.array([1, 2, 3]) + compare_alg = alg.get_max_rel_err + self.assertEqual(alg.compare_torch_tensor(cpu_output, npu_output, compare_alg), (0.0, True, '')) + + def test_compare_bool_tensor(self): + cpu_output = np.array([True, False, True]) + npu_output = np.array([True, False, True]) + self.assertEqual(alg.compare_bool_tensor(cpu_output, npu_output), (0.0, True, '')) + + def test_get_msg_and_handle_value(self): + b_value = np.array([1.0, 2.0, 3.0]) + n_value = np.array([1.0, 2.0, 3.0]) + self.assertEqual(alg.get_msg_and_handle_value(b_value, n_value), (b_value, n_value, '')) + + def test_get_max_rel_err(self): + b_value = np.array([1.0, 2.0, 3.0]) + n_value = np.array([1.0, 2.0, 3.0]) + self.assertEqual(alg.get_max_rel_err(b_value, n_value), (0.0, True, '')) + + def test_get_max_abs_err(self): + b_value = np.array([1.0, 2.0, 3.0]) + n_value = np.array([1.0, 2.0, 3.0]) + self.assertEqual(alg.get_max_abs_err(b_value, n_value), (0.0, True, '')) + + def test_get_rel_err_ratio_thousandth(self): + b_value = np.array([1.0, 2.0, 3.0]) + n_value = np.array([1.0, 2.0, 3.0]) + self.assertEqual(alg.get_rel_err_ratio_thousandth(b_value, n_value), (1.0, True, '')) + + def test_get_rel_err_ratio_ten_thousandth(self): + b_value = np.array([1.0, 2.0, 3.0]) + n_value = np.array([1.0, 2.0, 3.0]) + self.assertEqual(alg.get_rel_err_ratio_ten_thousandth(b_value, n_value), (1.0, True, '')) + + def test_max_rel_err_standard(self): + max_rel_errs = [0.0001, 0.0002, 0.0003] + result, arr = alg.max_rel_err_standard(max_rel_errs) + self.assertEqual(result, True) + self.assertTrue((arr == np.array([True, True, True])).all()) + + def test_cosine_standard(self): + compare_result = [0.9999, 0.9999, 0.9999] + result, arr = alg.cosine_standard(compare_result) + self.assertEqual(result, True) + self.assertTrue((arr == np.array([True, True, True])).all()) + + def test_cosine_sim(self): + cpu_output = np.array([1.0, 2.0, 3.0]) + npu_output = np.array([1.0, 2.0, 3.0]) + self.assertEqual(alg.cosine_sim(cpu_output, npu_output), (1.0, True, '')) + + def test_compare_uint8_data(self): + b_value = np.array([1, 2, 3], dtype=np.uint8) + n_value = np.array([1, 2, 3], dtype=np.uint8) + self.assertEqual(alg.compare_uint8_data(b_value, n_value), (1, True)) + + def test_compare_builtin_type(self): + bench_out = 1 + npu_out = 1 + self.assertEqual(alg.compare_builtin_type(bench_out, npu_out), (True, True, '')) + + def test_flatten_compare_result(self): + result = [[1, 2], [3, 4]] + self.assertEqual(alg.flatten_compare_result(result), [1, 2, 3, 4]) diff --git a/debug/accuracy_tools/api_accuracy_checker/test/ut/compare/test_compare_utils.py b/debug/accuracy_tools/api_accuracy_checker/test/ut/compare/test_compare_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..4e83c0643ef452c28d11c02bbbc2fee359a1ea2e --- /dev/null +++ b/debug/accuracy_tools/api_accuracy_checker/test/ut/compare/test_compare_utils.py @@ -0,0 +1,25 @@ +import unittest +import numpy as np +from api_accuracy_checker.compare.compare_utils import CompareConst, check_dtype_comparable + +class TestCompareUtils(unittest.TestCase): + def test_check_dtype_comparable(self): + x = np.array([1, 2, 3], dtype=np.int32) + y = np.array([4, 5, 6], dtype=np.int32) + self.assertTrue(check_dtype_comparable(x, y)) + + x = np.array([1.0, 2.0, 3.0], dtype=np.float32) + y = np.array([4.0, 5.0, 6.0], dtype=np.float32) + self.assertTrue(check_dtype_comparable(x, y)) + + x = np.array([True, False, True], dtype=np.bool_) + y = np.array([False, True, False], dtype=np.bool_) + self.assertTrue(check_dtype_comparable(x, y)) + + x = np.array([1, 2, 3], dtype=np.int32) + y = np.array([4.0, 5.0, 6.0], dtype=np.float32) + self.assertFalse(check_dtype_comparable(x, y)) + + x = np.array([1, 2, 3], dtype=np.int32) + y = np.array([True, False, True], dtype=np.bool_) + self.assertFalse(check_dtype_comparable(x, y)) diff --git a/debug/accuracy_tools/api_accuracy_checker/test/ut/dump/test_api_info.py b/debug/accuracy_tools/api_accuracy_checker/test/ut/dump/test_api_info.py new file mode 100644 index 0000000000000000000000000000000000000000..5ed5cd64452a11e7cf1cfda9b9f0b775eb011f8e --- /dev/null +++ b/debug/accuracy_tools/api_accuracy_checker/test/ut/dump/test_api_info.py @@ -0,0 +1,29 @@ +import unittest +from api_accuracy_checker.dump.api_info import APIInfo, ForwardAPIInfo, BackwardAPIInfo +from api_accuracy_checker.common.config import msCheckerConfig + +class TestAPIInfo(unittest.TestCase): + def test_APIInfo(self): + api_info = APIInfo("test_api", True, True, "save_path") + self.assertEqual(api_info.api_name, "test_api") + self.assertEqual(api_info.is_forward, True) + self.assertEqual(api_info.is_save_data, True) + self.assertEqual(api_info.save_path, "save_path") + self.assertEqual(api_info.forward_path, "forward_real_data") + self.assertEqual(api_info.backward_path, "backward_real_data") + + def test_ForwardAPIInfo(self): + forward_api_info = ForwardAPIInfo("test_forward_api", [1, 2, 3], {"a": 1, "b": 2}) + self.assertEqual(forward_api_info.api_name, "test_forward_api") + self.assertEqual(forward_api_info.is_forward, True) + self.assertEqual(forward_api_info.is_save_data, msCheckerConfig.real_data) + self.assertEqual(forward_api_info.save_path, msCheckerConfig.dump_path) + self.assertEqual(forward_api_info.api_info_struct, {"test_forward_api": {"args": [{'type': 'int', 'value': 1},{'type': 'int', 'value': 2},{'type': 'int', 'value': 3},], "kwargs": {'a': {'type': 'int', 'value': 1}, 'b': {'type': 'int', 'value': 2}}}}) + + def test_BackwardAPIInfo(self): + backward_api_info = BackwardAPIInfo("test_backward_api", [1, 2, 3]) + self.assertEqual(backward_api_info.api_name, "test_backward_api") + self.assertEqual(backward_api_info.is_forward, False) + self.assertEqual(backward_api_info.is_save_data, msCheckerConfig.real_data) + self.assertEqual(backward_api_info.save_path, msCheckerConfig.dump_path) + self.assertEqual(backward_api_info.grad_info_struct, {"test_backward_api": [{'type': 'int', 'value': 1},{'type': 'int', 'value': 2},{'type': 'int', 'value': 3}]}) diff --git a/debug/accuracy_tools/api_accuracy_checker/test/ut/dump/test_dump.py b/debug/accuracy_tools/api_accuracy_checker/test/ut/dump/test_dump.py new file mode 100644 index 0000000000000000000000000000000000000000..4b543c5276456e5c628ea383b0ef5f2f9d86ffe7 --- /dev/null +++ b/debug/accuracy_tools/api_accuracy_checker/test/ut/dump/test_dump.py @@ -0,0 +1,32 @@ +import unittest +from api_accuracy_checker.dump.dump import * + +class TestDumpUtil(unittest.TestCase): + def test_set_dump_switch(self): + set_dump_switch("ON") + self.assertEqual(DumpUtil.dump_switch, "ON") + set_dump_switch("OFF") + self.assertEqual(DumpUtil.dump_switch, "OFF") + + def test_get_dump_switch(self): + DumpUtil.dump_switch = "ON" + self.assertTrue(DumpUtil.get_dump_switch()) + DumpUtil.dump_switch = "OFF" + self.assertFalse(DumpUtil.get_dump_switch()) + + def test_incr_iter_num_maybe_exit(self): + msCheckerConfig.target_iter = 5 + msCheckerConfig.enable_dataloader = True + + DumpUtil.call_num = 6 + with self.assertRaises(Exception): + DumpUtil.incr_iter_num_maybe_exit() + + DumpUtil.call_num = 4 + DumpUtil.incr_iter_num_maybe_exit() + self.assertEqual(DumpUtil.dump_switch, "OFF") + + msCheckerConfig.enable_dataloader = False + DumpUtil.call_num = 5 + DumpUtil.incr_iter_num_maybe_exit() + self.assertEqual(DumpUtil.dump_switch, "ON") diff --git a/debug/accuracy_tools/api_accuracy_checker/test/ut/dump/test_dump_scopr.py b/debug/accuracy_tools/api_accuracy_checker/test/ut/dump/test_dump_scopr.py new file mode 100644 index 0000000000000000000000000000000000000000..addba38e38446b177942a104b4194efe910b1f7c --- /dev/null +++ b/debug/accuracy_tools/api_accuracy_checker/test/ut/dump/test_dump_scopr.py @@ -0,0 +1,21 @@ +import unittest +from api_accuracy_checker.dump.dump_scope import * +from api_accuracy_checker.dump.dump import DumpUtil + +class TestDumpScope(unittest.TestCase): + def test_iter_tracer(self): + DumpUtil.call_num = 0 + def dummy_func(): + return "Hello, World!" + + wrapped_func = iter_tracer(dummy_func) + result = wrapped_func() + self.assertEqual(DumpUtil.dump_switch, "ON") + self.assertEqual(result, "Hello, World!") + + def another_dummy_func(): + return 123 + wrapped_func = iter_tracer(another_dummy_func) + result = wrapped_func() + self.assertEqual(DumpUtil.dump_switch, "ON") + self.assertEqual(result, 123) diff --git a/debug/accuracy_tools/api_accuracy_checker/test/ut/dump/test_info_dump.py b/debug/accuracy_tools/api_accuracy_checker/test/ut/dump/test_info_dump.py new file mode 100644 index 0000000000000000000000000000000000000000..27efdf02ac3041b0866d06b9e147bfe7a386bd80 --- /dev/null +++ b/debug/accuracy_tools/api_accuracy_checker/test/ut/dump/test_info_dump.py @@ -0,0 +1,28 @@ +import unittest +import os +import fcntl +from unittest.mock import patch +from api_accuracy_checker.dump.api_info import APIInfo, ForwardAPIInfo, BackwardAPIInfo +from api_accuracy_checker.dump.info_dump import write_api_info_json, write_json, initialize_output_json +from api_accuracy_checker.common.utils import check_file_or_directory_path, initialize_save_path + +class TestInfoDump(unittest.TestCase): + + def test_write_api_info_json_backward(self): + api_info = BackwardAPIInfo("test_backward_api", [1, 2, 3]) + with patch('api_accuracy_checker.dump.info_dump.write_json') as mock_write_json: + write_api_info_json(api_info) + rank = os.getpid() + mock_write_json.assert_called_with(f'./backward_info_{rank}.json', api_info.grad_info_struct) + + def test_write_api_info_json_invalid_type(self): + api_info = APIInfo("test_api", True, True, "save_path") + with self.assertRaises(ValueError): + write_api_info_json(api_info) + + def tearDown(self): + rank = os.getpid() + files = [f'./backward_info_{rank}.json'] + for file in files: + if os.path.exists(file): + os.remove(file) diff --git a/debug/accuracy_tools/api_accuracy_checker/test/ut/run_ut/test_run_ut.py b/debug/accuracy_tools/api_accuracy_checker/test/ut/run_ut/test_run_ut.py new file mode 100644 index 0000000000000000000000000000000000000000..2d26356e5c177b6cae22790e15d2f9fe1c080f86 --- /dev/null +++ b/debug/accuracy_tools/api_accuracy_checker/test/ut/run_ut/test_run_ut.py @@ -0,0 +1,13 @@ +import unittest +from api_accuracy_checker.run_ut.run_ut import UtDataInfo + +class TestRunUt(unittest.TestCase): + + def test_UtDataInfo(self): + data_info = UtDataInfo(None, None, None, None, None, None) + self.assertIsNone(data_info.bench_grad_out) + self.assertIsNone(data_info.npu_grad_out) + self.assertIsNone(data_info.npu_out) + self.assertIsNone(data_info.bench_out) + self.assertIsNone(data_info.grad_in) + self.assertIsNone(data_info.in_fwd_data_list)