From 38e96c19547e8d3e92fa2289e02c3732f0660217 Mon Sep 17 00:00:00 2001 From: Linwei-Ying Date: Fri, 1 Nov 2024 10:14:51 +0800 Subject: [PATCH 1/5] compare ut add --- .../msprobe/test/core_ut/common/test_utils.py | 53 ++++++++++++++++++- 1 file changed, 52 insertions(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/test/core_ut/common/test_utils.py b/debug/accuracy_tools/msprobe/test/core_ut/common/test_utils.py index c0235840d..20527ecfc 100644 --- a/debug/accuracy_tools/msprobe/test/core_ut/common/test_utils.py +++ b/debug/accuracy_tools/msprobe/test/core_ut/common/test_utils.py @@ -19,6 +19,7 @@ import os import tempfile from unittest import TestCase from unittest.mock import patch, MagicMock, mock_open +import numpy as np from msprobe.core.common.const import Const from msprobe.core.common.file_utils import (FileCheckConst, @@ -41,7 +42,10 @@ from msprobe.core.common.utils import (CompareException, get_dump_mode, get_real_step_or_rank, get_step_or_rank_from_string, - get_stack_construct_by_dump_json_path) + get_stack_construct_by_dump_json_path, + safe_get_value, + MsprobeBaseException) + class TestUtils(TestCase): @@ -312,3 +316,50 @@ class TestUtils(TestCase): self.assertEqual(stack, {'stack_key': 'stack_value'}) self.assertEqual(construct, {'construct_key': 'construct_value'}) + + def test_safe_get_value_dict_valid_key_index(self): + # Test valid key and index in a dictionary + dict_container = {'a': [1, 2, 3], 'b': [4, 5, 6]} + self.assertEqual(safe_get_value(dict_container, 1, 'dict_container', key='a'), 2) + + def test_safe_get_value_invalid_key(self): + # Test invalid key in dictionary + dict_container = {'a': [1, 2, 3], 'b': [4, 5, 6]} + with self.assertRaises(MsprobeBaseException) as context: + safe_get_value(dict_container, 1, 'dict_container', key='invalid_key') + self.assertEqual(context.exception.error_code, MsprobeBaseException.INVALID_KEY_ERROR) + + def test_safe_get_value_valid_key_invalid_index(self): + # Test invalid index in dictionary[key] + dict_container = {'a': [1, 2, 3], 'b': [4, 5, 6]} + with self.assertRaises(MsprobeBaseException) as context: + safe_get_value(dict_container, 5, 'dict_container', key='a') + self.assertEqual(context.exception.error_code, MsprobeBaseException.INDEX_OUT_OF_BOUNDS_ERROR) + + def test_safe_get_value_list_valid_index(self): + # Test valid index in a list + list_container = [10, 20, 30] + self.assertEqual(safe_get_value(list_container, 1, 'list_container'), 20) + + def test_safe_get_value_list_index_out_of_bounds(self): + # Test index out of bounds in a list + list_container = [10, 20, 30] + with self.assertRaises(MsprobeBaseException) as context: + safe_get_value(list_container, 10, 'list_container') + self.assertEqual(context.exception.error_code, MsprobeBaseException.INDEX_OUT_OF_BOUNDS_ERROR) + + def test_safe_get_value_tuple_valid_index(self): + # Test valid index in a tuple + tuple_container = (100, 200, 300) + self.assertEqual(safe_get_value(tuple_container, 2, 'tuple_container'), 300) + + def test_safe_get_value_array_valid_index(self): + # Test valid index in a numpy array + array_container = np.array([1000, 2000, 3000]) + self.assertEqual(safe_get_value(array_container, 0, 'array_container'), 1000) + + def test_safe_get_value_unsupported_container_type(self): + # Test unsupported container type (e.g., a string) + with self.assertRaises(MsprobeBaseException) as context: + safe_get_value("unsupported_type", 0, 'string_container') + self.assertEqual(context.exception.error_code, MsprobeBaseException.INVALID_OBJECT_TYPE_ERROR) -- Gitee From 4ea6f706327c2eb50b17554b1bfaeea5959d3a0b Mon Sep 17 00:00:00 2001 From: Linwei-Ying Date: Mon, 4 Nov 2024 10:24:17 +0800 Subject: [PATCH 2/5] compare ut add --- .../msprobe/core/compare/utils.py | 40 +++++++++++ .../core_ut/compare/test_acc_compare_utils.py | 72 ++++++++++++++++++- 2 files changed, 110 insertions(+), 2 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/compare/utils.py b/debug/accuracy_tools/msprobe/core/compare/utils.py index b23840c3c..4a2b221ce 100644 --- a/debug/accuracy_tools/msprobe/core/compare/utils.py +++ b/debug/accuracy_tools/msprobe/core/compare/utils.py @@ -16,6 +16,7 @@ import os import re import math +from dataclasses import dataclass import numpy as np from msprobe.core.common.const import Const, CompareConst from msprobe.core.common.utils import CompareException, check_regex_prefix_format_valid, logger @@ -477,3 +478,42 @@ def _compare_parser(parser): help=" The data mapping file path.", required=False) parser.add_argument("-lm", "--layer_mapping", dest="layer_mapping", type=str, nargs='?', const=True, help=" The layer mapping file path.", required=False) + + +@dataclass +class ApiItemInfo: + name: str + struct: tuple + stack_info: list + + +def stack_column_process(result_item, has_stack, index, key, npu_stack_info): + if has_stack and index == 0 and key == CompareConst.INPUT_STRUCT: + result_item.extend(npu_stack_info) + else: + result_item.append(CompareConst.NONE) + return result_item + + +def result_item_init(n_info, b_info, dump_mode): + n_len = len(n_info.struct) + b_len = len(b_info.struct) + struct_long_enough = (n_len > 2 and b_len > 2) if dump_mode == Const.MD5 else (n_len > 1 and b_len > 1) + if struct_long_enough: + result_item = [ + n_info.name, b_info.name, n_info.struct[0], b_info.struct[0], n_info.struct[1], b_info.struct[1] + ] + if dump_mode == Const.MD5: + md5_compare_result = CompareConst.PASS if n_info.struct[2] == b_info.struct[2] else CompareConst.DIFF + result_item.extend([n_info.struct[2], b_info.struct[2], md5_compare_result]) + elif dump_mode == Const.SUMMARY: + result_item.extend([" "] * 8) + else: + result_item.extend([" "] * 5) + else: + err_msg = "index out of bounds error will occur in result_item_init, please check!\n" \ + f"npu_info_struct is {n_info.struct}\n" \ + f"bench_info_struct is {b_info.struct}" + logger.error(err_msg) + raise CompareException(CompareException.INDEX_OUT_OF_BOUNDS_ERROR) + return result_item diff --git a/debug/accuracy_tools/msprobe/test/core_ut/compare/test_acc_compare_utils.py b/debug/accuracy_tools/msprobe/test/core_ut/compare/test_acc_compare_utils.py index 3150ee14f..d10acbda7 100644 --- a/debug/accuracy_tools/msprobe/test/core_ut/compare/test_acc_compare_utils.py +++ b/debug/accuracy_tools/msprobe/test/core_ut/compare/test_acc_compare_utils.py @@ -6,9 +6,10 @@ import unittest import argparse from msprobe.core.compare.utils import extract_json, rename_api, read_op, op_item_parse, \ check_and_return_dir_contents, resolve_api_special_parameters, get_rela_diff_summary_mode, \ - get_accuracy, get_un_match_accuracy, merge_tensor, _compare_parser + get_accuracy, get_un_match_accuracy, merge_tensor, _compare_parser, stack_column_process, result_item_init, \ + ApiItemInfo from msprobe.core.common.utils import CompareException -from msprobe.core.common.const import Const +from msprobe.core.common.const import Const, CompareConst # test_read_op_1 @@ -367,3 +368,70 @@ class TestUtilsMethods(unittest.TestCase): self.assertIsNone(args.api_mapping) # 默认值应为 None self.assertEqual(args.data_mapping, "data_mapping.txt") self.assertEqual(args.layer_mapping, "layer_mapping.txt") + + def test_stack_column_process_stack_info(self): + result_item = [] + has_stack = True + index = 0 + key = CompareConst.INPUT_STRUCT + npu_stack_info = ['abc'] + result_item = stack_column_process(result_item, has_stack, index, key, npu_stack_info) + self.assertEqual(result_item, ['abc']) + + def test_stack_column_process_None(self): + result_item = [] + has_stack = True + index = 1 + key = CompareConst.INPUT_STRUCT + npu_stack_info = ['abc'] + result_item = stack_column_process(result_item, has_stack, index, key, npu_stack_info) + self.assertEqual(result_item, [None]) + + def test_result_item_init_all_and_summary(self): + n_name = 'Tensor.add.0.forward.input.0' + n_struct = ('torch.float32', [96]) + npu_stack_info = ['abc'] + b_name = 'Tensor.add.0.forward.input.0' + b_struct = ('torch.float32', [96]) + bench_stack_info = ['abc'] + n_info = ApiItemInfo(n_name, n_struct, npu_stack_info) + b_info = ApiItemInfo(b_name, b_struct, bench_stack_info) + + dump_mode = Const.ALL + result_item = result_item_init(n_info, b_info, dump_mode) + self.assertEqual(result_item, []) + + dump_mode = Const.SUMMARY + result_item = result_item_init(n_info, b_info, dump_mode) + self.assertEqual(result_item, []) + + def test_result_item_init_md5(self): + n_name = 'Tensor.add.0.forward.input.0' + n_struct = ('torch.float32', [96], 'e87000dc') + npu_stack_info = ['abc'] + b_name = 'Tensor.add.0.forward.input.0' + b_struct = ('torch.float32', [96], 'e87000dc') + bench_stack_info = ['abc'] + n_info = ApiItemInfo(n_name, n_struct, npu_stack_info) + b_info = ApiItemInfo(b_name, b_struct, bench_stack_info) + + dump_mode = Const.MD5 + result_item = result_item_init(n_info, b_info, dump_mode) + self.assertEqual(result_item, []) + + def test_result_item_init_md5_index_error(self): + n_name = 'Tensor.add.0.forward.input.0' + n_struct = ('torch.float32', [96]) + npu_stack_info = ['abc'] + b_name = 'Tensor.add.0.forward.input.0' + b_struct = ('torch.float32', [96]) + bench_stack_info = ['abc'] + n_info = ApiItemInfo(n_name, n_struct, npu_stack_info) + b_info = ApiItemInfo(b_name, b_struct, bench_stack_info) + + dump_mode = Const.MD5 + result_item_init(n_info, b_info, dump_mode) + + with self.assertRaises(CompareException) as context: + result_item = result_item_init(n_info, b_info, dump_mode) + self.assertEqual(context.exception.code, CompareException.INDEX_OUT_OF_BOUNDS_ERROR) -- Gitee From c12725806b407e7a332344fa8deb17d3df132195 Mon Sep 17 00:00:00 2001 From: Linwei-Ying Date: Mon, 4 Nov 2024 10:46:10 +0800 Subject: [PATCH 3/5] compare ut add --- debug/accuracy_tools/msprobe/core/common/utils.py | 7 +++++++ .../msprobe/test/core_ut/common/test_utils.py | 8 ++++---- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/common/utils.py b/debug/accuracy_tools/msprobe/core/common/utils.py index 204377a56..4aca49b92 100644 --- a/debug/accuracy_tools/msprobe/core/common/utils.py +++ b/debug/accuracy_tools/msprobe/core/common/utils.py @@ -407,3 +407,10 @@ def safe_get_value(container, index, container_name, key=None): f"{container_name} is {container}" logger.error(err_msg) raise MsprobeBaseException(MsprobeBaseException.INVALID_KEY_ERROR) from e + except TypeError as e: + err_msg = "wrong type, please check!\n" \ + f"{container_name} is {container}\n" \ + f"index is {index}\n" \ + f"key is {key}" + logger.error(err_msg) + raise MsprobeBaseException(MsprobeBaseException.INVALID_OBJECT_TYPE_ERROR) from e diff --git a/debug/accuracy_tools/msprobe/test/core_ut/common/test_utils.py b/debug/accuracy_tools/msprobe/test/core_ut/common/test_utils.py index 20527ecfc..e16f71387 100644 --- a/debug/accuracy_tools/msprobe/test/core_ut/common/test_utils.py +++ b/debug/accuracy_tools/msprobe/test/core_ut/common/test_utils.py @@ -327,14 +327,14 @@ class TestUtils(TestCase): dict_container = {'a': [1, 2, 3], 'b': [4, 5, 6]} with self.assertRaises(MsprobeBaseException) as context: safe_get_value(dict_container, 1, 'dict_container', key='invalid_key') - self.assertEqual(context.exception.error_code, MsprobeBaseException.INVALID_KEY_ERROR) + self.assertEqual(context.exception.code, MsprobeBaseException.INVALID_OBJECT_TYPE_ERROR) def test_safe_get_value_valid_key_invalid_index(self): # Test invalid index in dictionary[key] dict_container = {'a': [1, 2, 3], 'b': [4, 5, 6]} with self.assertRaises(MsprobeBaseException) as context: safe_get_value(dict_container, 5, 'dict_container', key='a') - self.assertEqual(context.exception.error_code, MsprobeBaseException.INDEX_OUT_OF_BOUNDS_ERROR) + self.assertEqual(context.exception.code, MsprobeBaseException.INDEX_OUT_OF_BOUNDS_ERROR) def test_safe_get_value_list_valid_index(self): # Test valid index in a list @@ -346,7 +346,7 @@ class TestUtils(TestCase): list_container = [10, 20, 30] with self.assertRaises(MsprobeBaseException) as context: safe_get_value(list_container, 10, 'list_container') - self.assertEqual(context.exception.error_code, MsprobeBaseException.INDEX_OUT_OF_BOUNDS_ERROR) + self.assertEqual(context.exception.code, MsprobeBaseException.INDEX_OUT_OF_BOUNDS_ERROR) def test_safe_get_value_tuple_valid_index(self): # Test valid index in a tuple @@ -362,4 +362,4 @@ class TestUtils(TestCase): # Test unsupported container type (e.g., a string) with self.assertRaises(MsprobeBaseException) as context: safe_get_value("unsupported_type", 0, 'string_container') - self.assertEqual(context.exception.error_code, MsprobeBaseException.INVALID_OBJECT_TYPE_ERROR) + self.assertEqual(context.exception.code, MsprobeBaseException.INVALID_OBJECT_TYPE_ERROR) -- Gitee From 2024084ea75f8aa45298a85b20da469e5f89c3af Mon Sep 17 00:00:00 2001 From: Linwei-Ying Date: Mon, 4 Nov 2024 10:55:06 +0800 Subject: [PATCH 4/5] compare ut add --- .../test/core_ut/compare/test_acc_compare_utils.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/debug/accuracy_tools/msprobe/test/core_ut/compare/test_acc_compare_utils.py b/debug/accuracy_tools/msprobe/test/core_ut/compare/test_acc_compare_utils.py index d10acbda7..f755eeec7 100644 --- a/debug/accuracy_tools/msprobe/test/core_ut/compare/test_acc_compare_utils.py +++ b/debug/accuracy_tools/msprobe/test/core_ut/compare/test_acc_compare_utils.py @@ -385,7 +385,7 @@ class TestUtilsMethods(unittest.TestCase): key = CompareConst.INPUT_STRUCT npu_stack_info = ['abc'] result_item = stack_column_process(result_item, has_stack, index, key, npu_stack_info) - self.assertEqual(result_item, [None]) + self.assertEqual(result_item, ['None']) def test_result_item_init_all_and_summary(self): n_name = 'Tensor.add.0.forward.input.0' @@ -399,11 +399,13 @@ class TestUtilsMethods(unittest.TestCase): dump_mode = Const.ALL result_item = result_item_init(n_info, b_info, dump_mode) - self.assertEqual(result_item, []) + self.assertEqual(result_item, ['Tensor.add.0.forward.input.0', 'Tensor.add.0.forward.input.0', + 'torch.float32', 'torch.float32', [96], [96], '', '', '', '', '']) dump_mode = Const.SUMMARY result_item = result_item_init(n_info, b_info, dump_mode) - self.assertEqual(result_item, []) + self.assertEqual(result_item, ['Tensor.add.0.forward.input.0', 'Tensor.add.0.forward.input.0', + 'torch.float32', 'torch.float32', [96], [96], '', '', '', '', '', '', '', '']) def test_result_item_init_md5(self): n_name = 'Tensor.add.0.forward.input.0' @@ -417,7 +419,8 @@ class TestUtilsMethods(unittest.TestCase): dump_mode = Const.MD5 result_item = result_item_init(n_info, b_info, dump_mode) - self.assertEqual(result_item, []) + self.assertEqual(result_item, ['Tensor.add.0.forward.input.0', 'Tensor.add.0.forward.input.0', + 'torch.float32', 'torch.float32', [96], [96], 'e87000dc', 'e87000dc', 'pass']) def test_result_item_init_md5_index_error(self): n_name = 'Tensor.add.0.forward.input.0' @@ -430,8 +433,6 @@ class TestUtilsMethods(unittest.TestCase): b_info = ApiItemInfo(b_name, b_struct, bench_stack_info) dump_mode = Const.MD5 - result_item_init(n_info, b_info, dump_mode) - with self.assertRaises(CompareException) as context: result_item = result_item_init(n_info, b_info, dump_mode) self.assertEqual(context.exception.code, CompareException.INDEX_OUT_OF_BOUNDS_ERROR) -- Gitee From 846f93cda6239e9829fcf8d6dce5e0fab75b909f Mon Sep 17 00:00:00 2001 From: Linwei-Ying Date: Mon, 4 Nov 2024 10:58:51 +0800 Subject: [PATCH 5/5] compare ut add --- .../msprobe/test/core_ut/compare/test_acc_compare_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/debug/accuracy_tools/msprobe/test/core_ut/compare/test_acc_compare_utils.py b/debug/accuracy_tools/msprobe/test/core_ut/compare/test_acc_compare_utils.py index f755eeec7..145fec43f 100644 --- a/debug/accuracy_tools/msprobe/test/core_ut/compare/test_acc_compare_utils.py +++ b/debug/accuracy_tools/msprobe/test/core_ut/compare/test_acc_compare_utils.py @@ -400,12 +400,12 @@ class TestUtilsMethods(unittest.TestCase): dump_mode = Const.ALL result_item = result_item_init(n_info, b_info, dump_mode) self.assertEqual(result_item, ['Tensor.add.0.forward.input.0', 'Tensor.add.0.forward.input.0', - 'torch.float32', 'torch.float32', [96], [96], '', '', '', '', '']) + 'torch.float32', 'torch.float32', [96], [96], ' ', ' ', ' ', ' ', ' ']) dump_mode = Const.SUMMARY result_item = result_item_init(n_info, b_info, dump_mode) self.assertEqual(result_item, ['Tensor.add.0.forward.input.0', 'Tensor.add.0.forward.input.0', - 'torch.float32', 'torch.float32', [96], [96], '', '', '', '', '', '', '', '']) + 'torch.float32', 'torch.float32', [96], [96], ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ']) def test_result_item_init_md5(self): n_name = 'Tensor.add.0.forward.input.0' -- Gitee