diff --git a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/base.py b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/base.py index eb09df95a6827a72aec148506913c3a6a8efc3f3..bb734743b99e4338c3247076ba80c0b7180af9e4 100644 --- a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/base.py +++ b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/base.py @@ -89,7 +89,7 @@ class TensorStatInfo: class BaseDataProcessor: _recursive_key_stack = [] special_type = ( - np.integer, np.floating, np.bool_, np.complexfloating, np.str_, np.byte, np.unicode_, + np.integer, np.floating, np.bool_, np.complexfloating, np.str_, np.byte, np.unicode_, np.ndarray, bool, int, float, str, slice, type(Ellipsis) ) @@ -216,8 +216,22 @@ class BaseDataProcessor: return single_arg @staticmethod - def _analyze_numpy(value, numpy_type): - return {"type": numpy_type, "value": value} + def _analyze_numpy(ndarray, numpy_type): + ndarray_json = {} + ndarray_json.update({'type': 'numpy.ndarray'}) + ndarray_json.update({'dtype': str(ndarray.dtype)}) + ndarray_json.update({'shape': ndarray.shape}) + if ndarray.size > 0: + ndarray_json.update({"Max": np.max(ndarray).item()}) + ndarray_json.update({"Min": np.min(ndarray).item()}) + ndarray_json.update({"Mean": np.mean(ndarray).item()}) + ndarray_json.update({"Norm": np.linalg.norm(ndarray).item()}) + else: + ndarray_json.update({"Max": None}) + ndarray_json.update({"Min": None}) + ndarray_json.update({"Mean": None}) + ndarray_json.update({"Norm": None}) + return ndarray_json @staticmethod def _get_allowed_data_mode(data_mode): diff --git a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/mindspore_processor.py b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/mindspore_processor.py index 6765e38ac21426874581eeedcf01c4e48eaa3b23..c16eccefdbaca81c87df847a9fa62486bc8364e5 100644 --- a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/mindspore_processor.py +++ b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/mindspore_processor.py @@ -140,11 +140,13 @@ class MindsporeDataProcessor(BaseDataProcessor): converted_numpy, numpy_type = self._convert_numpy_to_builtin(element) if converted_numpy is not element: - return self._analyze_numpy(converted_numpy, numpy_type) + return {"type": numpy_type, "value": converted_numpy} if isinstance(element, Number): return self.analyze_dtype_in_kwargs(element) if isinstance(element, ms.Tensor): return self._analyze_tensor(element, Const.SEP.join([str(suffix) for suffix in suffix_stack])) + if isinstance(element, np.ndarray): + return self._analyze_numpy(element, Const.SEP.join([str(suffix) for suffix in suffix_stack])) if isinstance(element, (bool, int, float, str, slice, type(Ellipsis))): return self._analyze_builtin(element) return {} @@ -189,6 +191,13 @@ class TensorDataProcessor(MindsporeDataProcessor): else: save_tensor_as_npy(tensor, file_path) return single_arg + + def _analyze_numpy(self, ndarray, suffix): + dump_data_name, file_path = self.get_save_file_path(suffix) + np.save(file_path, ndarray) + ndarray_json = super()._analyze_numpy(ndarray, suffix) + ndarray_json.update({"data_name": dump_data_name}) + return ndarray_json class OverflowCheckDataProcessor(MindsporeDataProcessor): diff --git a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py index d1c03e167b0d36e156ed54b66da9d6246aafd580..a5d783de9c73713b14738b5be54b40cf05a1e37e 100644 --- a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py +++ b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py @@ -211,9 +211,11 @@ class PytorchDataProcessor(BaseDataProcessor): return self._analyze_process_group(element) converted_numpy, numpy_type = self._convert_numpy_to_builtin(element) if converted_numpy is not element: - return self._analyze_numpy(converted_numpy, numpy_type) + return {"type": numpy_type, "value": converted_numpy} if isinstance(element, torch.Tensor): return self._analyze_tensor(element, Const.SEP.join([str(suffix) for suffix in suffix_stack])) + if isinstance(element, np.ndarray): + return self._analyze_numpy(element, Const.SEP.join([str(suffix) for suffix in suffix_stack])) if isinstance(element, (bool, int, float, str, slice, type(Ellipsis))): return self._analyze_builtin(element) return {} @@ -272,6 +274,13 @@ class TensorDataProcessor(PytorchDataProcessor): saved_tensor = tensor.clone().contiguous().detach() save_pt(saved_tensor, file_path) return single_arg + + def _analyze_numpy(self, ndarray, suffix): + dump_data_name, file_path = self.get_save_file_path(suffix) + torch.save(ndarray, file_path) + ndarray_json = super()._analyze_numpy(ndarray, suffix) + ndarray_json.update({"data_name": dump_data_name}) + return ndarray_json class OverflowCheckDataProcessor(PytorchDataProcessor): diff --git a/debug/accuracy_tools/msprobe/test/core_ut/data_dump/data_processor/test_base.py b/debug/accuracy_tools/msprobe/test/core_ut/data_dump/data_processor/test_base.py index 49fa52c73a3f2adff985e9c674022e1c7bfdf0ee..9b4bbc720cc57e79e4e9a1ea7421a23de158635f 100644 --- a/debug/accuracy_tools/msprobe/test/core_ut/data_dump/data_processor/test_base.py +++ b/debug/accuracy_tools/msprobe/test/core_ut/data_dump/data_processor/test_base.py @@ -112,13 +112,23 @@ class TestBaseDataProcessor(unittest.TestCase): expected = {'type': 'int', 'value': 1} self.assertEqual(result, expected) - def test_analyze_numpy(self): - result = BaseDataProcessor._analyze_numpy(5, 'int32') - self.assertEqual(result, {'type': 'int32', 'value': 5}) - def test_get_special_types(self): self.assertIn(int, BaseDataProcessor.get_special_types()) + def test_analyze_numpy(self): + ndarray = np.array([[1, 2, 3], [4, 5, 6]], dtype=np.int32) + result = BaseDataProcessor._analyze_numpy(ndarray, 'numpy.ndarray') + expected_result = { + 'type': 'numpy.ndarray', + 'dtype': 'int32', + 'shape': (2, 3), + 'Max': 6, + 'Min': 1, + 'Mean': 3.5, + 'Norm':9.539392014169456 + } + self.assertEqual(result, expected_result) + def test_recursive_apply_transform(self): transform = lambda x, _: x * 2 Test = namedtuple("Test", ['a']) diff --git a/debug/accuracy_tools/msprobe/test/core_ut/data_dump/data_processor/test_pytorch_processor.py b/debug/accuracy_tools/msprobe/test/core_ut/data_dump/data_processor/test_pytorch_processor.py index 269231df40eca19d3a62e041b1c75abeaad9c796..36ab6b987d721b7c1e84d548206e9b92e82df4b2 100644 --- a/debug/accuracy_tools/msprobe/test/core_ut/data_dump/data_processor/test_pytorch_processor.py +++ b/debug/accuracy_tools/msprobe/test/core_ut/data_dump/data_processor/test_pytorch_processor.py @@ -254,7 +254,7 @@ class TestPytorchDataProcessor(unittest.TestCase): numpy_element = np.int64(1) converted_numpy, numpy_type = self.processor._convert_numpy_to_builtin(numpy_element) result = self.processor.analyze_single_element(numpy_element, []) - expected_result = self.processor._analyze_numpy(converted_numpy, numpy_type) + expected_result = {"type": numpy_type, "value": converted_numpy} self.assertEqual(result, expected_result) def test_analyze_single_element_tensor(self):