From 5ec25defd8946b8a8c99c90c6b28d154d55ca25d Mon Sep 17 00:00:00 2001 From: TAJh Date: Thu, 26 Sep 2024 17:21:16 +0800 Subject: [PATCH 01/13] add modify mapping UT --- .../msprobe/test/core_ut/common/test_utils.py | 60 +++++++ .../core_ut/compare/test_modify_mapping.py | 157 ++++++++++++++++++ 2 files changed, 217 insertions(+) create mode 100644 debug/accuracy_tools/msprobe/test/core_ut/compare/test_modify_mapping.py diff --git a/debug/accuracy_tools/msprobe/test/core_ut/common/test_utils.py b/debug/accuracy_tools/msprobe/test/core_ut/common/test_utils.py index 96f746000..65c2762fc 100644 --- a/debug/accuracy_tools/msprobe/test/core_ut/common/test_utils.py +++ b/debug/accuracy_tools/msprobe/test/core_ut/common/test_utils.py @@ -276,3 +276,63 @@ class TestUtils(TestCase): self.assertEqual(result, [1, 2, 3, 4, 5, 10]) result = get_real_step_or_rank([10, "1-3", 3], "step") self.assertEqual(result, [1, 2, 3, 10]) + + def test_struct_json_get_pt_invalid_path_then_fail(self): + input_param = { + "bench_json_path": None + } + framework = Const.PT_FRAMEWORK + with self.assertRaises(CompareException) as context: + stack, construct = struct_json_get(input_param, framework) + self.assertEqual(context.exception.code, CompareException.INVALID_PATH_ERROR) + + def test_struct_json_get_ms_invalid_path_then_fail(self): + input_param = { + "npu_json_path": None + } + framework = Const.MS_FRAMEWORK + with self.assertRaises(CompareException) as context: + stack, construct = struct_json_get(input_param, framework) + self.assertEqual(context.exception.code, CompareException.INVALID_PATH_ERROR) + + @patch('msprobe.core.common.utils.load_json') + def test_struct_json_get_pt_framework_valid_paths_then_pass(self, mock_load_json): + def load_json_side_effect(path): + if path.endswith("stack.json"): + return {'stack_key': 'stack_value'} + elif path.endswith("construct.json"): + return {'construct_key': 'construct_value'} + else: + raise FileNotFoundError + # 设置框架为PT_FRAMEWORK + input_param = {} + framework = Const.PT_FRAMEWORK + prefix = 'bench' + frame_json_path = './dump.json' + input_param[f'{prefix}_json_path'] = frame_json_path + mock_load_json.side_effect = load_json_side_effect + + stack, construct = struct_json_get(input_param, framework) + self.assertEqual(stack, {'stack_key': 'stack_value'}) + self.assertEqual(construct, {'construct_key': 'construct_value'}) + + @patch('msprobe.core.common.utils.load_json') + def test_struct_json_get_ms_framework_valid_paths_then_pass(self, mock_load_json): + def load_json_side_effect(path): + if path.endswith("stack.json"): + return {'stack_key': 'stack_value'} + elif path.endswith("construct.json"): + return {'construct_key': 'construct_value'} + else: + raise FileNotFoundError + # 设置框架为PT_FRAMEWORK + input_param = {} + framework = Const.MS_FRAMEWORK + prefix = 'npu' + frame_json_path = './dump.json' + input_param[f'{prefix}_json_path'] = frame_json_path + mock_load_json.side_effect = load_json_side_effect + + stack, construct = struct_json_get(input_param, framework) + self.assertEqual(stack, {'stack_key': 'stack_value'}) + self.assertEqual(construct, {'construct_key': 'construct_value'}) diff --git a/debug/accuracy_tools/msprobe/test/core_ut/compare/test_modify_mapping.py b/debug/accuracy_tools/msprobe/test/core_ut/compare/test_modify_mapping.py new file mode 100644 index 000000000..9c4348600 --- /dev/null +++ b/debug/accuracy_tools/msprobe/test/core_ut/compare/test_modify_mapping.py @@ -0,0 +1,157 @@ +import unittest +from msprobe.mindspore.compare.modify_mapping import * +from unittest.mock import patch, MagicMock + +class TestModifyMapping(unittest.TestCase): + + def setUp(self): + # 这里可以设置常量和初始化 + self.lines = [ + "file1.py, func_a, other_info", + "file2.py, func_b, other_info", + "file3.py, func_c, other_info" + ] + self.pt_construct = { + "Functional.max_pool2d.0.forward": "Module.pool1.MaxPool2d.forward.0", + "Funtional.conv2d.1.forward": "Module.con2.Conv2d.forward.0", + "Functional.linear.5.backward": "Module.fc3.Linear.backward.1", + "Module.conv1.Conv2d.backward.1": null + } + self.ms_construct = { + "Functional.add.0.forward": "Cell.transformer_layers.0.attention.core_attention.scale_mask_softmax.ScaleMaskSoftmax.forward.0", + "Tensor.reshape.2.forward": "Cell.transformer_layers.0.attention.ParallelAttention.forward.0" + } + self.ms_stack = { + "Functional.add.0.forward": [ + "File /home/user/envs/python3.9/site-packages/mindspore/nn/cell.py, line 507, \ + in _run_construct, \n output = self._run_forward_hook(inputs, output)", + "File /home/user/envs/python3.9/site-packages/mindspore/nn/cell.py, line 745, \ + in __call__, \n return self._run_construct(*args, **kwargs)", + "File /home/user/envs/python3.9/site-packages/msprobe/mindspore/dump/hook_cell/hook_cell.py, line 48, \ + in __call__, \n out = super(HOOKCell, self).__call__(*args, **kwargs)", + "File /home/user/envs/python3.9/site-packages/msprobe/mindspore/dump/hook_cell/wrap_api.py, line 92, \ + in api_function, \n return ApiTemplate(api_name, api_dict, prefix, hook)(*args, **kwargs)" + "File /home/user/envs/python3.9/site-packages/mindformers/transformer/utils.py, line 38, in attn_mask_add, \n attention_scores = ops.add(", + "File /home/user/envs/python3.9/site-packages/mindformers/transformer/scale_mask_softmax.py, line 65, in construct, \n masked_input = self.mask_func(x, mask) if mask is not None else x", + "File /home/user/envs/python3.9/site-packages/mindspore/nn/cell.py, line 2460, \ + in _backward_hook_construct, \n outputs = self.construct(*outputs, **kwargs)", + "File /home/user/envs/python3.9/site-packages/mindspore/nn/cell.py, line 498, \ + in _run_construct, \n output = self._backward_hook_construct(*outputs, **kwargs)", + "File /home/user/envs/python3.9/site-packages/mindspore/nn/cell.py, line 745, \ + in __call__, \n return self._run_construct(*args, **kwargs)", + "File /home/user/envs/python3.9/site-packages/mindformers/transformer/transformer.py, line 533, \ + in construct, \n attention_output = self.attention(norm_output, attention_mask, rotary_pos_emb)", + "File /home/user/envs/python3.9/site-packages/mindspore/nn/cell.py, line 2462, \ + in _backward_hook_construct, \n outputs = self.construct(output, **kwargs)", + "File /home/user/envs/python3.9/site-packages/mindspore/nn/cell.py, line 2460, \ + in _backward_hook_construct, \n outputs = self.construct(*outputs, **kwargs)", + "File /home/user/envs/python3.9/site-packages/mindspore/nn/cell.py, line 498, \ + in _run_construct, \n output = self._backward_hook_construct(*outputs, **kwargs)", + "File /home/user/envs/python3.9/site-packages/mindspore/nn/cell.py, line 745, \ + in __call__, \n return self._run_construct(*args, **kwargs)", + "File /home/user/envs/python3.9/site-packages/mindformers/transformer/transformer.py, line 533, \ + in construct, \n attention_output = self.attention(norm_output, attention_mask, rotary_pos_emb)", + "File /home/user/envs/python3.9/site-packages/mindspore/nn/cell.py, line 2462, \ + in _backward_hook_construct, \n outputs = self.construct(output, **kwargs)" + ], + "Tensor.reshape.2.forward": [ + "File /home/user/envs/python3.9/site-packages/mindspore/nn/cell.py, line 507, \ + in _run_construct, \n output = self._run_forward_hook(inputs, output)", + "File /home/user/envs/python3.9/site-packages/mindspore/nn/cell.py, line 745, \ + in __call__, \n return self._run_construct(*args, **kwargs)", + "File /home/user/envs/python3.9/site-packages/msprobe/mindspore/dump/hook_cell/hook_cell.py, line 48, \ + in __call__, \n out = super(HOOKCell, self).__call__(*args, **kwargs)", + "File /home/user/envs/python3.9/site-packages/msprobe/mindspore/dump/hook_cell/wrap_api.py, line 92, \ + in api_function, \n return ApiTemplate(api_name, api_dict, prefix, hook)(*args, **kwargs)", + "File /home/user/envs/python3.9/site-packages/mindformers/transformer/transformer.py, line 367, \ + in construct, \n query = query.reshape(bs, seq_len, -1, self.head_dim).transpose((0, 2, 1, 3))", + "File /home/user/envs/python3.9/site-packages/mindspore/nn/cell.py, line 2460, \ + in _backward_hook_construct, \n outputs = self.construct(*outputs, **kwargs)", + "File /home/user/envs/python3.9/site-packages/mindspore/nn/cell.py, line 498, \ + in _run_construct, \n output = self._backward_hook_construct(*outputs, **kwargs)", + "File /home/user/envs/python3.9/site-packages/mindspore/nn/cell.py, line 745, \ + in __call__, \n return self._run_construct(*args, **kwargs)", + "File /home/user/envs/python3.9/site-packages/mindformers/transformer/transformer.py, line 533, \ + in construct, \n attention_output = self.attention(norm_output, attention_mask, rotary_pos_emb)", + "File /home/user/envs/python3.9/site-packages/mindspore/nn/cell.py, line 2462, \ + in _backward_hook_construct, \n outputs = self.construct(output, **kwargs)" + ], + } + self.pt_stack = { + "Functional.max_pool2d.0.forward": [ + "File /home/user/envs/lib/python3.9/site-packages/msprobe/pytorch/hook_module/wrap_funtional.py, line 97, \ + in functional_op_template, \n return FunctionalOPTemplate(op_name, hook)(*args, **kwargs)", + "File /home/user/envs/lib/python3.9/site-packages/torch/nn/modules/pooling.py, line 166, \ + in forward, \n return F.max_pool2d(input, self.kernel_size, self.stride,", + "File /home/user/envs/lib/python3.9/site-packages/torch/nn/modules/module.py, line 1568, \ + in _call_impl, \n result = foward_call(*args, **kwargs)", + "File /home/user/envs/lib/python3.9/site-packages/torch/nn/modules/module.py, line 1518, \ + in _wrapped_call_impl, \n return self._call_impl(*args, **kwargs)", + "File lenet.py, line 28, in forward, \n x = self.pool1(x)", + "File /home/user/envs/lib/python3.9/site-packages/torch/nn/modules/module.py, line 1527, \ + in _call_impl, \n return forward_call(*args, **kwargs)", + "File /home/user/envs/lib/python3.9/site-packages/torch/nn/modules/module.py, line 1518, \ + in _wrapped_call_impl, \n return self._call_impl(*args, **kwargs)", + "File lenet.py, line 47, in train_one_epoch, \n output = model(image)", + "File lenet.py, line 83, in TestLeNetE2E, \n train_one_epoch(step, steps, train_loader, \ + model, optimizer, criterion)", + "File lenet.py, line 88, in , \n TestLeNetE2E()" + ], + "Funtional.conv2d.1.forward": [ + "File /home/user/envs/lib/python3.9/site-packages/msprobe/pytorch/hook_module/wrap_funtional.py, line 97, \ + in functional_op_template, \n return FunctionalOPTemplate(op_name, hook)(*args, **kwargs)", + "File /home/user/envs/lib/python3.9/site-packages/torch/nn/modules/conv.py, line 456, \ + in _conv_forward, \n return F.conv2d(input, weight, bias, self.stride,", + "File /home/user/envs/lib/python3.9/site-packages/torch/nn/modules/conv.py, line 460, \ + in forward, \n return self._conv_forward(input, self.weight, self.bias)", + "File /home/user/envs/lib/python3.9/site-packages/torch/nn/modules/module.py, line 1568, \ + in _call_impl, \n result = foward_call(*args, **kwargs)", + "File /home/user/envs/lib/python3.9/site-packages/torch/nn/modules/module.py, line 1518, \ + in _wrapped_call_impl, \n return self._call_impl(*args, **kwargs)", + "File lenet.py, line 29, in forward, \n x = F.relu(self.conv2(x))", + "File /home/user/envs/lib/python3.9/site-packages/torch/nn/modules/module.py, line 1527, \ + in _call_impl, \n return forward_call(*args, **kwargs)", + "File /home/user/envs/lib/python3.9/site-packages/torch/nn/modules/module.py, line 1518, \ + in _wrapped_call_impl, \n return self._call_impl(*args, **kwargs)", + "File lenet.py, line 47, in train_one_epoch, \n output = model(image)", + "File lenet.py, line 83, in TestLeNetE2E, \n train_one_epoch(step, steps, train_loader, \ + model, optimizer, criterion)", + "File lenet.py, line 88, in , \n TestLeNetE2E()" + ] + } + + def test_find_regard_scope(self): + start_sign = "func_a" + end_sign = "func_c" + start_pos, end_pos = find_regard_scope(self.lines, start_sign, end_sign) + self.assertEqual(start_pos, 0) + self.assertEqual(end_pos, -1) + + def test_find_stack_func_list(self): + result = find_stack_func_list(self.lines) + self.assertEqual(result, ['func_a', 'func_b', 'func_c']) + + def test_get_duplicated_name(self): + components = ["Module", "Func", "Name"] + duplicated_name = get_duplicated_name(components) + self.assertEqual(duplicated_name, ["Module", "Func", "Name", "Func", "Name"]) + + def test_modify_mapping_with_stack_when_pt_valid_then_pass(self): + result = modify_mapping_with_stack(self.pt_stack, self.pt_construct) + print(result) + expected_result = { + "Cell.SomeFunc": { + "origin_data": "Cell.SomeFunc", + "scope": "Cell.SomeFunc.Parent", + "stack": None + }, + "Module.FuncName": { + "origin_data": "Module.FuncName", + "scope": "Module.FuncName.Parent", + "stack": "line1,line2,line3" + } + } + self.assertIn("Cell.SomeFunc", result) + self.assertIn("Module.FuncName", result) + self.assertEqual(result["Module.FuncName"]["origin_data"], "Module.FuncName") + self.assertEqual(result["Cell.SomeFunc"]["origin_data"], "Cell.SomeFunc") -- Gitee From 7d3801f0c65f79a04febcc292bd1c4e23e645397 Mon Sep 17 00:00:00 2001 From: TAJh Date: Thu, 26 Sep 2024 18:33:19 +0800 Subject: [PATCH 02/13] bugfix --- .../msprobe/mindspore/compare/modify_mapping.py | 6 +++--- .../msprobe/test/core_ut/compare/test_modify_mapping.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/debug/accuracy_tools/msprobe/mindspore/compare/modify_mapping.py b/debug/accuracy_tools/msprobe/mindspore/compare/modify_mapping.py index 82bddefcd..c2ba67fe9 100644 --- a/debug/accuracy_tools/msprobe/mindspore/compare/modify_mapping.py +++ b/debug/accuracy_tools/msprobe/mindspore/compare/modify_mapping.py @@ -96,12 +96,12 @@ def modify_mapping_with_stack(stack, construct): func_stack_list = find_stack_func_list(regard_scope) # 组合逻辑：parent的节点名（到节点名字为止）加上调用栈名[reversed_list]加上原来key重复key的节点名[key_components[1:-2] + key_components[-3:]] - final_res_key = Const.SEP.join(parent[:parent_idx + 1] + reversed_list + + final_res_key = Const.SEP.join(parent[:parent_idx + 1] + func_stack_list + key_components[1:Const.CONSTRUCT_NAME_INDEX + 1] + key_components[Const.CONSTRUCT_NAME_INDEX:]) final_res_key = final_res_key.strip(".forward").strip(".backward") else: final_res_key = Const.SEP.join(key_components[:-2] + [key_components[-1]]) - reversed_list = [] + func_stack_list = [] final_pres[final_res_key] = {Const.ORIGIN_DATA: key, Const.SCOPE: construct[key], - Const.STACK: Const.SEP.join(reversed_list) if reversed_list else None} + Const.STACK: Const.SEP.join(func_stack_list) if func_stack_list else None} return final_pres diff --git a/debug/accuracy_tools/msprobe/test/core_ut/compare/test_modify_mapping.py b/debug/accuracy_tools/msprobe/test/core_ut/compare/test_modify_mapping.py index 9c4348600..2e269cf2e 100644 --- a/debug/accuracy_tools/msprobe/test/core_ut/compare/test_modify_mapping.py +++ b/debug/accuracy_tools/msprobe/test/core_ut/compare/test_modify_mapping.py @@ -13,9 +13,9 @@ class TestModifyMapping(unittest.TestCase): ] self.pt_construct = { "Functional.max_pool2d.0.forward": "Module.pool1.MaxPool2d.forward.0", - "Funtional.conv2d.1.forward": "Module.con2.Conv2d.forward.0", + "Funtional.conv2d.1.forward": "Module.conv2.Conv2d.forward.0", "Functional.linear.5.backward": "Module.fc3.Linear.backward.1", - "Module.conv1.Conv2d.backward.1": null + "Module.conv1.Conv2d.backward.1": None } self.ms_construct = { "Functional.add.0.forward": "Cell.transformer_layers.0.attention.core_attention.scale_mask_softmax.ScaleMaskSoftmax.forward.0", -- Gitee From f2a63282bd3243ca3b30af53d57b527ba0602d3e Mon Sep 17 00:00:00 2001 From: TAJh Date: Thu, 26 Sep 2024 19:54:42 +0800 Subject: [PATCH 03/13] fix some senario --- .../mindspore/compare/modify_mapping.py | 65 +++++++++---------- .../core_ut/compare/test_modify_mapping.py | 43 +++++++++++- 2 files changed, 72 insertions(+), 36 deletions(-) diff --git a/debug/accuracy_tools/msprobe/mindspore/compare/modify_mapping.py b/debug/accuracy_tools/msprobe/mindspore/compare/modify_mapping.py index c2ba67fe9..dd4caf36f 100644 --- a/debug/accuracy_tools/msprobe/mindspore/compare/modify_mapping.py +++ b/debug/accuracy_tools/msprobe/mindspore/compare/modify_mapping.py @@ -54,47 +54,46 @@ def modify_mapping_with_stack(stack, construct): # 查看归属关系 for key in construct: key_components = key.split(Const.SEP) + code_list = stack.get(key, None) + parent_node = construct.get(key, None) # 名称如果非标准开头，转为标准开头 if not key.startswith(("Module", "Cell")): - code_list = stack.get(key, None) - if not code_list: - logger.info(f"Key not found in code stack") - continue - # 如果没有拿到父属scope name，默认顶级域名为Module或Cell - if not construct.get(key, None): - if not key.startswith(("Module", "Cell")): - # 将节点名字转为标准的Module或Cell - key_components[0] = "Cell" if is_ms else "Module" - # 重复该节点的名字作为类型 如add.add add在-3位置 - duplicated_components = get_duplicated_name(key_components) - modified_key = Const.SEP.join(duplicated_components) - else: - modified_key = key + if not parent_node: + # 将节点名字转为标准的Module或Cell + key_components[0] = "Cell" if is_ms else "Module" + # 重复该节点的名字作为类型 如add.add add在-3位置 + duplicated_components = get_duplicated_name(key_components) + modified_key = Const.SEP.join(duplicated_components) + modified_key = modified_key.replace(".forward", "").replace(".backward", "") final_pres[modified_key] = {Const.ORIGIN_DATA: key, Const.SCOPE: None, Const.STACK: None} continue - - parent = construct[key].split(Const.SEP) + parent = parent_node.split(Const.SEP) if len(parent) < 4: - logger.warning(f"Parent name in construct.json is not valid") + logger.info(f"Parent name in construct.json is not valid") continue - # {name}.Class.count_number.X ward Or {name}.Class.count_number.X ward.ele_number - parent_idx = Const.NAME_FIRST_POSSIBLE_INDEX if not parent[Const.NAME_FIRST_POSSIBLE_INDEX].isdigit() else Const.NAME_SECOND_POSSIBLE_INDEX - + parent_idx = Const.NAME_FIRST_POSSIBLE_INDEX if not \ + parent[Const.NAME_FIRST_POSSIBLE_INDEX].isdigit() else Const.NAME_SECOND_POSSIBLE_INDEX parent_name = parent[parent_idx] - if parent_name.endswith('s'): - parent_name = parent_name[:-1] - # {name}.count_number.X ward - func_name = key_components[-3] - - start_pos, end_pos = find_regard_scope(code_list, func_name, parent_name) - - # 获取指定范围的代码 - regard_scope = code_list[start_pos:end_pos] - - func_stack_list = find_stack_func_list(regard_scope) - + if code_list: + # {name}.Class.count_number.X ward Or {name}.Class.count_number.X ward.ele_number + + if parent_name.endswith('s'): + parent_name = parent_name[:-1] + if len(key_components) < 3: + logger.info("The length of key in construct is less than 3, please check") + continue + # {name}.count_number.X ward + func_name = key_components[-3] + start_pos, end_pos = find_regard_scope(code_list, func_name, parent_name) + + # 获取指定范围的代码 + regard_scope = code_list[start_pos:end_pos] + + func_stack_list = find_stack_func_list(regard_scope) + else: + func_stack_list = [] # 组合逻辑：parent的节点名（到节点名字为止）加上调用栈名[reversed_list]加上原来key重复key的节点名[key_components[1:-2] + key_components[-3:]] final_res_key = Const.SEP.join(parent[:parent_idx + 1] + func_stack_list + key_components[1:Const.CONSTRUCT_NAME_INDEX + 1] + key_components[Const.CONSTRUCT_NAME_INDEX:]) @@ -102,6 +101,6 @@ def modify_mapping_with_stack(stack, construct): else: final_res_key = Const.SEP.join(key_components[:-2] + [key_components[-1]]) func_stack_list = [] - final_pres[final_res_key] = {Const.ORIGIN_DATA: key, Const.SCOPE: construct[key], + final_pres[final_res_key] = {Const.ORIGIN_DATA: key, Const.SCOPE: parent_node, Const.STACK: Const.SEP.join(func_stack_list) if func_stack_list else None} return final_pres diff --git a/debug/accuracy_tools/msprobe/test/core_ut/compare/test_modify_mapping.py b/debug/accuracy_tools/msprobe/test/core_ut/compare/test_modify_mapping.py index 2e269cf2e..51c51544f 100644 --- a/debug/accuracy_tools/msprobe/test/core_ut/compare/test_modify_mapping.py +++ b/debug/accuracy_tools/msprobe/test/core_ut/compare/test_modify_mapping.py @@ -19,7 +19,8 @@ class TestModifyMapping(unittest.TestCase): } self.ms_construct = { "Functional.add.0.forward": "Cell.transformer_layers.0.attention.core_attention.scale_mask_softmax.ScaleMaskSoftmax.forward.0", - "Tensor.reshape.2.forward": "Cell.transformer_layers.0.attention.ParallelAttention.forward.0" + "Tensor.reshape.2.forward": "Cell.transformer_layers.0.attention.ParallelAttention.forward.0", + "Functional.add.4.forward": "Cell.transformer_layers.0.attention.core_attention.scale_mask_softmax.ScaleMaskSoftmax.forward.0" } self.ms_stack = { "Functional.add.0.forward": [ @@ -31,8 +32,10 @@ class TestModifyMapping(unittest.TestCase): in __call__, \n out = super(HOOKCell, self).__call__(*args, **kwargs)", "File /home/user/envs/python3.9/site-packages/msprobe/mindspore/dump/hook_cell/wrap_api.py, line 92, \ in api_function, \n return ApiTemplate(api_name, api_dict, prefix, hook)(*args, **kwargs)" - "File /home/user/envs/python3.9/site-packages/mindformers/transformer/utils.py, line 38, in attn_mask_add, \n attention_scores = ops.add(", - "File /home/user/envs/python3.9/site-packages/mindformers/transformer/scale_mask_softmax.py, line 65, in construct, \n masked_input = self.mask_func(x, mask) if mask is not None else x", + "File /home/user/envs/python3.9/site-packages/mindformers/transformer/utils.py, line 38, \ + in attn_mask_add, \n attention_scores = ops.add(", + "File /home/user/envs/python3.9/site-packages/mindformers/transformer/scale_mask_softmax.py, line 65, \ + in construct, \n masked_input = self.mask_func(x, mask) if mask is not None else x", "File /home/user/envs/python3.9/site-packages/mindspore/nn/cell.py, line 2460, \ in _backward_hook_construct, \n outputs = self.construct(*outputs, **kwargs)", "File /home/user/envs/python3.9/site-packages/mindspore/nn/cell.py, line 498, \ @@ -76,6 +79,40 @@ class TestModifyMapping(unittest.TestCase): "File /home/user/envs/python3.9/site-packages/mindspore/nn/cell.py, line 2462, \ in _backward_hook_construct, \n outputs = self.construct(output, **kwargs)" ], + "Functional.add.4.forward": [ + "File /home/user/envs/python3.9/site-packages/mindspore/nn/cell.py, line 507, \ + in _run_construct, \n output = self._run_forward_hook(inputs, output)", + "File /home/user/envs/python3.9/site-packages/mindspore/nn/cell.py, line 745, \ + in __call__, \n return self._run_construct(*args, **kwargs)", + "File /home/user/envs/python3.9/site-packages/msprobe/mindspore/dump/hook_cell/hook_cell.py, line 48, \ + in __call__, \n out = super(HOOKCell, self).__call__(*args, **kwargs)", + "File /home/user/envs/python3.9/site-packages/msprobe/mindspore/dump/hook_cell/wrap_api.py, line 92, \ + in api_function, \n return ApiTemplate(api_name, api_dict, prefix, hook)(*args, **kwargs)", + "File /home/user/envs/python3.9/site-packages/mindformers/transformer/utils.py, line 38, \ + in attn_mask_add, \n attention_scores = ops.add(", + "File /home/user/envs/python3.9/site-packages/mindformers/transformer/scale_mask_softmax.py, line 65, \ + in construct, \n masked_input = self.mask_func(x, mask) if mask is not None else x", + "File /home/user/envs/python3.9/site-packages/mindspore/nn/cell.py, line 2460, \ + in _backward_hook_construct, \n outputs = self.construct(*outputs, **kwargs)", + "File /home/user/envs/python3.9/site-packages/mindspore/nn/cell.py, line 498, \ + in _run_construct, \n output = self._backward_hook_construct(*outputs, **kwargs)", + "File /home/user/envs/python3.9/site-packages/mindspore/nn/cell.py, line 745, \ + in __call__, \n return self._run_construct(*args, **kwargs)", + "File /home/user/envs/python3.9/site-packages/mindformers/transformer/transformer.py, line 533, \ + in construct, \n attention_output = self.attention(norm_output, attention_mask, rotary_pos_emb)", + "File /home/user/envs/python3.9/site-packages/mindspore/nn/cell.py, line 2462, \ + in _backward_hook_construct, \n outputs = self.construct(output, **kwargs)", + "File /home/user/envs/python3.9/site-packages/mindspore/nn/cell.py, line 2460, \ + in _backward_hook_construct, \n outputs = self.construct(*outputs, **kwargs)", + "File /home/user/envs/python3.9/site-packages/mindspore/nn/cell.py, line 498, \ + in _run_construct, \n output = self._backward_hook_construct(*outputs, **kwargs)", + "File /home/user/envs/python3.9/site-packages/mindspore/nn/cell.py, line 745, \ + in __call__, \n return self._run_construct(*args, **kwargs)", + "File /home/user/envs/python3.9/site-packages/mindformers/transformer/transformer.py, line 533, \ + in construct, \n attention_output = self.attention(norm_output, attention_mask, rotary_pos_emb)", + "File /home/user/envs/python3.9/site-packages/mindspore/nn/cell.py, line 2462, \ + in _backward_hook_construct, \n outputs = self.construct(output, **kwargs)" + ] } self.pt_stack = { "Functional.max_pool2d.0.forward": [ -- Gitee From 982632421db4e7eab2504716f3f5f12c9cf57ec2 Mon Sep 17 00:00:00 2001 From: TAJh Date: Thu, 26 Sep 2024 20:50:41 +0800 Subject: [PATCH 04/13] add UT --- .../msprobe/core/common/const.py | 4 +- .../mindspore/compare/modify_mapping.py | 5 +- .../core_ut/compare/test_modify_mapping.py | 84 ++++++++++++++----- 3 files changed, 68 insertions(+), 25 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/common/const.py b/debug/accuracy_tools/msprobe/core/common/const.py index c8aa91ffb..2217be97b 100644 --- a/debug/accuracy_tools/msprobe/core/common/const.py +++ b/debug/accuracy_tools/msprobe/core/common/const.py @@ -116,7 +116,9 @@ class Const: STACK_FILE_INDEX = 0 - STACK_FUNC_INDEX = 1 + STACK_FUNC_INDEX = 2 + + STACK_FUNC_ELE_INDEX = 1 CONSTRUCT_NAME_INDEX = -3 diff --git a/debug/accuracy_tools/msprobe/mindspore/compare/modify_mapping.py b/debug/accuracy_tools/msprobe/mindspore/compare/modify_mapping.py index dd4caf36f..c37eae4ff 100644 --- a/debug/accuracy_tools/msprobe/mindspore/compare/modify_mapping.py +++ b/debug/accuracy_tools/msprobe/mindspore/compare/modify_mapping.py @@ -9,6 +9,7 @@ def find_regard_scope(lines, start_sign, end_sign): start_pos = idx elif end_sign in ii: end_pos = idx + break return start_pos, end_pos @@ -25,7 +26,7 @@ def find_stack_func_list(lines): if any(ii in func_ele for ii in Const.FUNC_SKIP_LIST): continue - in_func_name = func_ele.split()[Const.STACK_FUNC_INDEX] + in_func_name = func_ele.split()[Const.STACK_FUNC_ELE_INDEX] res_list.append(in_func_name) # 反转res_list并生成final_res @@ -76,9 +77,9 @@ def modify_mapping_with_stack(stack, construct): parent_idx = Const.NAME_FIRST_POSSIBLE_INDEX if not \ parent[Const.NAME_FIRST_POSSIBLE_INDEX].isdigit() else Const.NAME_SECOND_POSSIBLE_INDEX parent_name = parent[parent_idx] + if code_list: # {name}.Class.count_number.X ward Or {name}.Class.count_number.X ward.ele_number - if parent_name.endswith('s'): parent_name = parent_name[:-1] if len(key_components) < 3: diff --git a/debug/accuracy_tools/msprobe/test/core_ut/compare/test_modify_mapping.py b/debug/accuracy_tools/msprobe/test/core_ut/compare/test_modify_mapping.py index 51c51544f..0c22cc2e6 100644 --- a/debug/accuracy_tools/msprobe/test/core_ut/compare/test_modify_mapping.py +++ b/debug/accuracy_tools/msprobe/test/core_ut/compare/test_modify_mapping.py @@ -7,9 +7,38 @@ class TestModifyMapping(unittest.TestCase): def setUp(self): # 这里可以设置常量和初始化 self.lines = [ - "file1.py, func_a, other_info", - "file2.py, func_b, other_info", - "file3.py, func_c, other_info" + "File /home/user/envs/python3.9/site-packages/mindspore/nn/cell.py, line 507, \ + in _run_construct, \n output = self._run_forward_hook(inputs, output)", + "File /home/user/envs/python3.9/site-packages/mindspore/nn/cell.py, line 745, \ + in __call__, \n return self._run_construct(*args, **kwargs)", + "File /home/user/envs/python3.9/site-packages/msprobe/mindspore/dump/hook_cell/hook_cell.py, line 48, \ + in __call__, \n out = super(HOOKCell, self).__call__(*args, **kwargs)", + "File /home/user/envs/python3.9/site-packages/msprobe/mindspore/dump/hook_cell/wrap_api.py, line 92, \ + in api_function, \n return ApiTemplate(api_name, api_dict, prefix, hook)(*args, **kwargs)", + "File /home/user/envs/python3.9/site-packages/mindformers/transformer/utils.py, line 38, \ + in attn_mask_add, \n attention_scores = ops.add(", + "File /home/user/envs/python3.9/site-packages/mindformers/transformer/scale_mask_softmax.py, line 65, \ + in construct, \n masked_input = self.mask_func(x, mask) if mask is not None else x", + "File /home/user/envs/python3.9/site-packages/mindspore/nn/cell.py, line 2460, \ + in _backward_hook_construct, \n outputs = self.construct(*outputs, **kwargs)", + "File /home/user/envs/python3.9/site-packages/mindspore/nn/cell.py, line 498, \ + in _run_construct, \n output = self._backward_hook_construct(*outputs, **kwargs)", + "File /home/user/envs/python3.9/site-packages/mindspore/nn/cell.py, line 745, \ + in __call__, \n return self._run_construct(*args, **kwargs)", + "File /home/user/envs/python3.9/site-packages/mindformers/transformer/transformer.py, line 533, \ + in construct, \n attention_output = self.attention(norm_output, attention_mask, rotary_pos_emb)", + "File /home/user/envs/python3.9/site-packages/mindspore/nn/cell.py, line 2462, \ + in _backward_hook_construct, \n outputs = self.construct(output, **kwargs)", + "File /home/user/envs/python3.9/site-packages/mindspore/nn/cell.py, line 2460, \ + in _backward_hook_construct, \n outputs = self.construct(*outputs, **kwargs)", + "File /home/user/envs/python3.9/site-packages/mindspore/nn/cell.py, line 498, \ + in _run_construct, \n output = self._backward_hook_construct(*outputs, **kwargs)", + "File /home/user/envs/python3.9/site-packages/mindspore/nn/cell.py, line 745, \ + in __call__, \n return self._run_construct(*args, **kwargs)", + "File /home/user/envs/python3.9/site-packages/mindformers/transformer/transformer.py, line 533, \ + in construct, \n attention_output = self.attention(norm_output, attention_mask, rotary_pos_emb)", + "File /home/user/envs/python3.9/site-packages/mindspore/nn/cell.py, line 2462, \ + in _backward_hook_construct, \n outputs = self.construct(output, **kwargs)" ] self.pt_construct = { "Functional.max_pool2d.0.forward": "Module.pool1.MaxPool2d.forward.0", @@ -158,37 +187,48 @@ class TestModifyMapping(unittest.TestCase): } def test_find_regard_scope(self): - start_sign = "func_a" - end_sign = "func_c" + start_sign = "add" + end_sign = "attention" start_pos, end_pos = find_regard_scope(self.lines, start_sign, end_sign) - self.assertEqual(start_pos, 0) - self.assertEqual(end_pos, -1) + self.assertEqual(start_pos, 4) + self.assertEqual(end_pos, 9) def test_find_stack_func_list(self): result = find_stack_func_list(self.lines) - self.assertEqual(result, ['func_a', 'func_b', 'func_c']) + self.assertEqual(result, ['attn_mask_add']) def test_get_duplicated_name(self): - components = ["Module", "Func", "Name"] + components = ["Module", "max_pool2d", "0", "forward"] duplicated_name = get_duplicated_name(components) - self.assertEqual(duplicated_name, ["Module", "Func", "Name", "Func", "Name"]) + self.assertEqual(duplicated_name, ["Module", "max_pool2d", "max_pool2d", "0", "forward"]) def test_modify_mapping_with_stack_when_pt_valid_then_pass(self): result = modify_mapping_with_stack(self.pt_stack, self.pt_construct) - print(result) expected_result = { - "Cell.SomeFunc": { - "origin_data": "Cell.SomeFunc", - "scope": "Cell.SomeFunc.Parent", + "Module.pool1.max_pool2d.max_pool2d.0": { + "origin_data": "Functional.max_pool2d.0.forward", + "scope": "Module.pool1.MaxPool2d.forward.0", "stack": None }, - "Module.FuncName": { - "origin_data": "Module.FuncName", - "scope": "Module.FuncName.Parent", - "stack": "line1,line2,line3" + "Module.conv2.conv2d.conv2d.1": { + "origin_data": "Module.conv1.Conv2d.backward.1", + "scope": None, + "stack": None + }, + "Module.fc3.linear.linear.5": { + "origin_data": "Functional.linear.5.backward", + "scope": "Module.fc3.Linear.backward.1", + "stack": None + }, + "Module.conv1.Conv2d.1": { + "origin_data": "Module.conv1.Conv2d.1", + "scope": "Module.fc3.Linear.backward.1", + "stack": None } } - self.assertIn("Cell.SomeFunc", result) - self.assertIn("Module.FuncName", result) - self.assertEqual(result["Module.FuncName"]["origin_data"], "Module.FuncName") - self.assertEqual(result["Cell.SomeFunc"]["origin_data"], "Cell.SomeFunc") + self.assertIn("Module.pool1.max_pool2d.max_pool2d.0", result) + self.assertIn("Module.conv2.conv2d.conv2d.1", result) + self.assertIn("Module.fc3.linear.linear.5", result) + self.assertIn("Module.conv1.Conv2d.1", result) + self.assertEqual(result["Module.pool1.max_pool2d.max_pool2d.0"]["origin_data"], "Functional.max_pool2d.0.forward") + self.assertEqual(result["Module.conv1.Conv2d.1"]["origin_data"], "Module.conv1.Conv2d.backward.1") -- Gitee From 3c778990f63013fc3b4156cb46ca7d4bdc52ddb0 Mon Sep 17 00:00:00 2001 From: TAJh Date: Fri, 27 Sep 2024 09:37:16 +0800 Subject: [PATCH 05/13] add UT --- .../core_ut/compare/test_modify_mapping.py | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/debug/accuracy_tools/msprobe/test/core_ut/compare/test_modify_mapping.py b/debug/accuracy_tools/msprobe/test/core_ut/compare/test_modify_mapping.py index 0c22cc2e6..a91d1c095 100644 --- a/debug/accuracy_tools/msprobe/test/core_ut/compare/test_modify_mapping.py +++ b/debug/accuracy_tools/msprobe/test/core_ut/compare/test_modify_mapping.py @@ -232,3 +232,34 @@ class TestModifyMapping(unittest.TestCase): self.assertIn("Module.conv1.Conv2d.1", result) self.assertEqual(result["Module.pool1.max_pool2d.max_pool2d.0"]["origin_data"], "Functional.max_pool2d.0.forward") self.assertEqual(result["Module.conv1.Conv2d.1"]["origin_data"], "Module.conv1.Conv2d.backward.1") + + def test_modify_mapping_with_stack_when_ms_valid_then_pass(self): + result = modify_mapping_with_stack(self.ms_stack, self.ms_construct) + expected_result = { + "Module.pool1.max_pool2d.max_pool2d.0": { + "origin_data": "Functional.max_pool2d.0.forward", + "scope": "Module.pool1.MaxPool2d.forward.0", + "stack": None + }, + "Module.conv2.conv2d.conv2d.1": { + "origin_data": "Module.conv1.Conv2d.backward.1", + "scope": None, + "stack": None + }, + "Module.fc3.linear.linear.5": { + "origin_data": "Functional.linear.5.backward", + "scope": "Module.fc3.Linear.backward.1", + "stack": None + }, + "Module.conv1.Conv2d.1": { + "origin_data": "Module.conv1.Conv2d.1", + "scope": "Module.fc3.Linear.backward.1", + "stack": None + } + } + self.assertIn("Module.pool1.max_pool2d.max_pool2d.0", result) + self.assertIn("Module.conv2.conv2d.conv2d.1", result) + self.assertIn("Module.fc3.linear.linear.5", result) + self.assertIn("Module.conv1.Conv2d.1", result) + self.assertEqual(result["Module.pool1.max_pool2d.max_pool2d.0"]["origin_data"], "Functional.max_pool2d.0.forward") + self.assertEqual(result["Module.conv1.Conv2d.1"]["origin_data"], "Module.conv1.Conv2d.backward.1") -- Gitee From f44490c5f69238642943c853f4f77806d0506ac9 Mon Sep 17 00:00:00 2001 From: TAJh Date: Fri, 27 Sep 2024 09:45:43 +0800 Subject: [PATCH 06/13] add UT --- .../core_ut/compare/test_modify_mapping.py | 45 +++++++++---------- 1 file changed, 22 insertions(+), 23 deletions(-) diff --git a/debug/accuracy_tools/msprobe/test/core_ut/compare/test_modify_mapping.py b/debug/accuracy_tools/msprobe/test/core_ut/compare/test_modify_mapping.py index a91d1c095..76f3860e6 100644 --- a/debug/accuracy_tools/msprobe/test/core_ut/compare/test_modify_mapping.py +++ b/debug/accuracy_tools/msprobe/test/core_ut/compare/test_modify_mapping.py @@ -60,7 +60,7 @@ class TestModifyMapping(unittest.TestCase): "File /home/user/envs/python3.9/site-packages/msprobe/mindspore/dump/hook_cell/hook_cell.py, line 48, \ in __call__, \n out = super(HOOKCell, self).__call__(*args, **kwargs)", "File /home/user/envs/python3.9/site-packages/msprobe/mindspore/dump/hook_cell/wrap_api.py, line 92, \ - in api_function, \n return ApiTemplate(api_name, api_dict, prefix, hook)(*args, **kwargs)" + in api_function, \n return ApiTemplate(api_name, api_dict, prefix, hook)(*args, **kwargs)", "File /home/user/envs/python3.9/site-packages/mindformers/transformer/utils.py, line 38, \ in attn_mask_add, \n attention_scores = ops.add(", "File /home/user/envs/python3.9/site-packages/mindformers/transformer/scale_mask_softmax.py, line 65, \ @@ -236,30 +236,29 @@ class TestModifyMapping(unittest.TestCase): def test_modify_mapping_with_stack_when_ms_valid_then_pass(self): result = modify_mapping_with_stack(self.ms_stack, self.ms_construct) expected_result = { - "Module.pool1.max_pool2d.max_pool2d.0": { - "origin_data": "Functional.max_pool2d.0.forward", - "scope": "Module.pool1.MaxPool2d.forward.0", - "stack": None + "Cell.transformer_layers.0.attention.core_attention.scale_mask_softmax.attn_mask_add.add.add.0": { + "origin_data": "Functional.add.0.forward", + "scope": "Cell.transformer_layers.0.attention.core_attention.\ + scale_mask_softmax.ScaleMaskSoftmax.forward.0", + "stack": "attn_mask_add" }, - "Module.conv2.conv2d.conv2d.1": { - "origin_data": "Module.conv1.Conv2d.backward.1", - "scope": None, + "Cell.transformer_layers.0.attention.reshape.reshape.2": { + "origin_data": "Tensor.reshape.2.forward", + "scope": "Cell.transformer_layers.0.attention.ParallelAttention.forward.0", "stack": None }, - "Module.fc3.linear.linear.5": { - "origin_data": "Functional.linear.5.backward", - "scope": "Module.fc3.Linear.backward.1", - "stack": None - }, - "Module.conv1.Conv2d.1": { - "origin_data": "Module.conv1.Conv2d.1", - "scope": "Module.fc3.Linear.backward.1", - "stack": None + "Cell.transformer_layers.0.attention.core_attention.scale_mask_softmax.attn_mask_add.add.add.4": { + "origin_data": "Functional.add.4.forward", + "scope": "Cell.transformer_layers.0.attention.core_attention.\ + scale_mask_softmax.ScaleMaskSoftmax.forward.0", + "stack": "attn_mask_add" } } - self.assertIn("Module.pool1.max_pool2d.max_pool2d.0", result) - self.assertIn("Module.conv2.conv2d.conv2d.1", result) - self.assertIn("Module.fc3.linear.linear.5", result) - self.assertIn("Module.conv1.Conv2d.1", result) - self.assertEqual(result["Module.pool1.max_pool2d.max_pool2d.0"]["origin_data"], "Functional.max_pool2d.0.forward") - self.assertEqual(result["Module.conv1.Conv2d.1"]["origin_data"], "Module.conv1.Conv2d.backward.1") + self.assertIn("Cell.transformer_layers.0.attention.core_attention.scale_mask_softmax.attn_mask_add.add.add.0", result) + self.assertIn("Cell.transformer_layers.0.attention.reshape.reshape.2", result) + self.assertIn("Cell.transformer_layers.0.attention.core_attention.scale_mask_softmax.attn_mask_add.add.add.4", result) + self.assertEqual(result["Cell.transformer_layers.0.attention.core_attention.scale_mask_softmax.attn_mask_add.add.add.0"]["origin_data"], "Functional.add.0.forward") + self.assertEqual(result["Cell.transformer_layers.0.attention.core_attention.scale_mask_softmax.attn_mask_add.add.add.0"]["scope"], "Cell.transformer_layers.0.attention.core_attention.scale_mask_softmax.ScaleMaskSoftmax.forward.0") + self.assertEqual(result["Cell.transformer_layers.0.attention.core_attention.scale_mask_softmax.attn_mask_add.add.add.0"]["stack"], "attn_mask_add") + self.assertEqual(result["Cell.transformer_layers.0.attention.reshape.reshape.2"]["origin_data"], "Tensor.reshape.2.forward") + self.assertEqual(result["Cell.transformer_layers.0.attention.reshape.reshape.2"]["scope"], "Cell.transformer_layers.0.attention.ParallelAttention.forward.0") -- Gitee From 00263038c8f2036fbcdd64da3cca1164da4d978f Mon Sep 17 00:00:00 2001 From: TAJh Date: Fri, 27 Sep 2024 10:14:44 +0800 Subject: [PATCH 07/13] add UT --- .../compare/test_modify_mapping.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) rename debug/accuracy_tools/msprobe/test/{core_ut => mindspore_ut}/compare/test_modify_mapping.py (98%) diff --git a/debug/accuracy_tools/msprobe/test/core_ut/compare/test_modify_mapping.py b/debug/accuracy_tools/msprobe/test/mindspore_ut/compare/test_modify_mapping.py similarity index 98% rename from debug/accuracy_tools/msprobe/test/core_ut/compare/test_modify_mapping.py rename to debug/accuracy_tools/msprobe/test/mindspore_ut/compare/test_modify_mapping.py index 76f3860e6..985262610 100644 --- a/debug/accuracy_tools/msprobe/test/core_ut/compare/test_modify_mapping.py +++ b/debug/accuracy_tools/msprobe/test/mindspore_ut/compare/test_modify_mapping.py @@ -212,7 +212,7 @@ class TestModifyMapping(unittest.TestCase): }, "Module.conv2.conv2d.conv2d.1": { "origin_data": "Module.conv1.Conv2d.backward.1", - "scope": None, + "scope": "Module.conv2.Conv2d.forward.0", "stack": None }, "Module.fc3.linear.linear.5": { @@ -221,8 +221,8 @@ class TestModifyMapping(unittest.TestCase): "stack": None }, "Module.conv1.Conv2d.1": { - "origin_data": "Module.conv1.Conv2d.1", - "scope": "Module.fc3.Linear.backward.1", + "origin_data": "Module.conv1.Conv2d.backward.1", + "scope": None, "stack": None } } @@ -232,6 +232,7 @@ class TestModifyMapping(unittest.TestCase): self.assertIn("Module.conv1.Conv2d.1", result) self.assertEqual(result["Module.pool1.max_pool2d.max_pool2d.0"]["origin_data"], "Functional.max_pool2d.0.forward") self.assertEqual(result["Module.conv1.Conv2d.1"]["origin_data"], "Module.conv1.Conv2d.backward.1") + self.assertEqual(result["Module.conv2.conv2d.conv2d.1"]["origin_data"], "Funtional.conv2d.1.forward") def test_modify_mapping_with_stack_when_ms_valid_then_pass(self): result = modify_mapping_with_stack(self.ms_stack, self.ms_construct) -- Gitee From ce941972a896c35a9ffe799e6bc67345e1171603 Mon Sep 17 00:00:00 2001 From: CSNIU Date: Thu, 26 Sep 2024 19:44:40 +0800 Subject: [PATCH 08/13] =?UTF-8?q?=E3=80=90bugfix=E3=80=91cell=E7=BA=A7dump?= =?UTF-8?q?=E6=95=B0=E6=8D=AE=E6=AF=94=E5=AF=B9=E6=95=B0=E7=BB=84=E8=B6=8A?= =?UTF-8?q?=E7=95=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../msprobe/core/compare/utils.py | 31 +++++++++---------- 1 file changed, 14 insertions(+), 17 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/compare/utils.py b/debug/accuracy_tools/msprobe/core/compare/utils.py index 9f0b19a02..285a49b58 100644 --- a/debug/accuracy_tools/msprobe/core/compare/utils.py +++ b/debug/accuracy_tools/msprobe/core/compare/utils.py @@ -316,15 +316,12 @@ def get_accuracy(result, n_dict, b_dict, summary_compare=False, md5_compare=Fals n_num = len(n_dict['op_name']) b_num = len(b_dict['op_name']) - n_num_input = len([name for name in n_dict['op_name'] if Const.INPUT in name]) - b_num_input = len([name for name in b_dict['op_name'] if Const.INPUT in name]) - n_num_kwarg = len([name for name in n_dict['op_name'] if 'kwarg' in name]) - b_num_kwarg = len([name for name in b_dict['op_name'] if 'kwarg' in name]) - n_num_output = n_num - n_num_input - n_num_kwarg - b_num_output = b_num - b_num_input - b_num_kwarg + n_num_input = len([name for name in n_dict['op_name'] if Const.INPUT in name.split(Const.SEP) or Const.KWARGS in name.split(Const.SEP)]) + b_num_input = len([name for name in b_dict['op_name'] if Const.INPUT in name.split(Const.SEP) or Const.KWARGS in name.split(Const.SEP)]) + n_num_output = n_num - n_num_input + b_num_output = b_num - b_num_input get_accuracy_core(0, n_num_input, 0, b_num_input, 'input_struct') - get_accuracy_core(n_num_input, n_num_kwarg, b_num_input, b_num_kwarg, "kwargs_struct") - get_accuracy_core(n_num_input + n_num_kwarg, n_num_output, b_num_input + b_num_kwarg, b_num_output, 'output_struct') + get_accuracy_core(n_num_input, n_num_output, b_num_input, b_num_output, 'output_struct') def get_un_match_accuracy(result, n_dict, md5_compare, summary_compare): @@ -334,7 +331,8 @@ def get_un_match_accuracy(result, n_dict, md5_compare, summary_compare): err_msg = CompareConst.NO_BENCH accuracy_check_res = CompareConst.N_A for index, n_name in enumerate(n_dict["op_name"]): - if n_name.find("input") != -1: + name_ele_list = n_name.split(Const.SEP) + if "input" in name_ele_list: n_struct = n_dict["input_struct"][index] else: n_struct = n_dict["output_struct"][index_out] @@ -386,21 +384,21 @@ def merge_tensor(tensor_list, summary_compare, md5_compare): op_dict['stack_info'].append(tensor['full_info']) break op_dict["op_name"].append(tensor['full_op_name']) + name_ele_list = tensor['full_op_name'].split(Const.SEP) if not md5_compare: - if tensor['full_op_name'].find("input") != -1: + if "input" in name_ele_list: op_dict["input_struct"].append((tensor['dtype'], tensor['shape'])) - elif tensor['full_op_name'].find("kwarg") != -1: + elif "kwarg" in name_ele_list: op_dict["kwargs_struct"].append((tensor['dtype'], tensor['shape'])) - elif tensor['full_op_name'].find("output") != -1: + elif "output" in name_ele_list: op_dict["output_struct"].append((tensor['dtype'], tensor['shape'])) else: - if tensor['full_op_name'].find("input") != -1: + if "input" in name_ele_list: op_dict["input_struct"].append((tensor['dtype'], tensor['shape'], tensor['md5'])) - elif tensor['full_op_name'].find("kwarg") != -1: + if "kwarg" in name_ele_list: op_dict["kwargs_struct"].append((tensor['dtype'], tensor['shape'], tensor['md5'])) - elif tensor['full_op_name'].find("output") != -1: + elif "output" in name_ele_list: op_dict["output_struct"].append((tensor['dtype'], tensor['shape'], tensor['md5'])) - op_dict["summary"].append([tensor['Max'], tensor['Min'], tensor['Mean'], tensor['Norm']]) if all_mode_bool: @@ -409,7 +407,6 @@ def merge_tensor(tensor_list, summary_compare, md5_compare): if data_name != "-1": op_dict["op_name"][-1] = data_name - if not op_dict["kwargs_struct"]: del op_dict["kwargs_struct"] return op_dict if op_dict["op_name"] else {} -- Gitee From b29488290e3263b261434dc1c7e5c6aeabf6c350 Mon Sep 17 00:00:00 2001 From: jiandaobao Date: Wed, 25 Sep 2024 17:27:07 +0800 Subject: [PATCH 09/13] free benchmark cleancode. --- .../pytorch/free_benchmark/__init__.py | 25 +++++-- .../pytorch/free_benchmark/common/params.py | 15 ++++ .../pytorch/free_benchmark/common/utils.py | 18 ++++- .../free_benchmark/compare/grad_saver.py | 35 +++++++-- .../compare/single_benchmark.py | 15 ++++ .../msprobe/pytorch/free_benchmark/main.py | 23 ++++-- .../perturbed_layers/base_layer.py | 15 ++++ .../perturbed_layers/layer_factory.py | 23 ++++-- .../perturbed_layers/npu/add_noise.py | 15 ++++ .../perturbed_layers/npu/bit_noise.py | 15 ++++ .../perturbed_layers/npu/change_value.py | 15 ++++ .../perturbed_layers/npu/improve_precision.py | 15 ++++ .../perturbed_layers/npu/no_change.py | 15 ++++ .../perturbed_layers/npu/npu_base_layser.py | 15 ++++ .../perturbed_layers/run_cpu.py | 15 ++++ .../result_handlers/base_handler.py | 71 ++++++++++++++----- .../result_handlers/check_handler.py | 15 ++++ .../result_handlers/fix_handler.py | 15 ++++ .../result_handlers/handler_factory.py | 15 ++++ .../result_handlers/preheat_handler.py | 19 ++++- 20 files changed, 369 insertions(+), 40 deletions(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/free_benchmark/__init__.py b/debug/accuracy_tools/msprobe/pytorch/free_benchmark/__init__.py index f0b40ccb2..effc2de41 100644 --- a/debug/accuracy_tools/msprobe/pytorch/free_benchmark/__init__.py +++ b/debug/accuracy_tools/msprobe/pytorch/free_benchmark/__init__.py @@ -1,8 +1,23 @@ -from msprobe.pytorch.common.log import logger -from msprobe.core.common.exceptions import FreeBenchmarkException +# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +__all__ = ["FreeBenchmarkCheck", "UnequalRow"] + from msprobe.core.common.const import Const +from msprobe.core.common.exceptions import FreeBenchmarkException +from msprobe.pytorch.common.log import logger -from .main import FreeBenchmarkCheck from .common.params import UnequalRow - -__all__ = [FreeBenchmarkCheck, UnequalRow] +from .main import FreeBenchmarkCheck diff --git a/debug/accuracy_tools/msprobe/pytorch/free_benchmark/common/params.py b/debug/accuracy_tools/msprobe/pytorch/free_benchmark/common/params.py index bbfc245a6..a6a8bfd0a 100644 --- a/debug/accuracy_tools/msprobe/pytorch/free_benchmark/common/params.py +++ b/debug/accuracy_tools/msprobe/pytorch/free_benchmark/common/params.py @@ -1,3 +1,18 @@ +# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from dataclasses import dataclass from typing import Any, Callable, Dict, List, Optional, Tuple diff --git a/debug/accuracy_tools/msprobe/pytorch/free_benchmark/common/utils.py b/debug/accuracy_tools/msprobe/pytorch/free_benchmark/common/utils.py index 631beeb85..5926e0953 100644 --- a/debug/accuracy_tools/msprobe/pytorch/free_benchmark/common/utils.py +++ b/debug/accuracy_tools/msprobe/pytorch/free_benchmark/common/utils.py @@ -1,3 +1,18 @@ +# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import torch from msprobe.pytorch.free_benchmark.common.enums import DeviceType @@ -75,7 +90,8 @@ class Tools: ) return type(origin)(result) return origin - + + class TorchC: sum = torch._C._VariableFunctionsClass.sum isinf = torch._C._VariableFunctionsClass.isinf diff --git a/debug/accuracy_tools/msprobe/pytorch/free_benchmark/compare/grad_saver.py b/debug/accuracy_tools/msprobe/pytorch/free_benchmark/compare/grad_saver.py index e58223e59..bc168fd0d 100644 --- a/debug/accuracy_tools/msprobe/pytorch/free_benchmark/compare/grad_saver.py +++ b/debug/accuracy_tools/msprobe/pytorch/free_benchmark/compare/grad_saver.py @@ -1,8 +1,27 @@ +# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import torch from msprobe.core.common.exceptions import FreeBenchmarkException from msprobe.pytorch.free_benchmark import logger from msprobe.pytorch.free_benchmark.common.constant import CommonField -from msprobe.pytorch.free_benchmark.common.params import DataParams, HandlerParams, data_pre_deal +from msprobe.pytorch.free_benchmark.common.params import ( + DataParams, + HandlerParams, + data_pre_deal, +) from msprobe.pytorch.free_benchmark.perturbed_layers.layer_factory import LayerFactory from msprobe.pytorch.free_benchmark.result_handlers.handler_factory import ( FuzzHandlerFactory, @@ -84,7 +103,7 @@ class GradSaver: if self.perturbed_grad_input is None: raise FreeBenchmarkException( FreeBenchmarkException.InvalidGrad, - f"grad not exists : {self.api_name}." + f"grad not exists : {self.api_name}.", ) with torch.no_grad(): perturbed_grad = self.perturbed_grad_input[new_grad_index].to( @@ -94,7 +113,7 @@ class GradSaver: raise FreeBenchmarkException( FreeBenchmarkException.InvalidGrad, f"grad shapes are inconsistent. api:{self.handler_params.api_name}." - f"origin:{origin_grad.shape}, perturbation: {perturbed_grad.shape}" + f"origin:{origin_grad.shape}, perturbation: {perturbed_grad.shape}", ) return perturbed_grad @@ -150,8 +169,8 @@ class GradSaver: else: _real_input.append(object_) kwargs = self.kwargs.copy() - if 'inplace' in kwargs: - kwargs['inplace'] = False + if "inplace" in kwargs: + kwargs["inplace"] = False return self.origin_func(*_real_input, **kwargs) _, grad_input = torch.autograd.functional.vjp( @@ -159,12 +178,14 @@ class GradSaver: ) return grad_input - def calculate_perturbed_grad_input(self, grad_output, need_grad_tensors, inner_args): + def calculate_perturbed_grad_input( + self, grad_output, need_grad_tensors, inner_args + ): data_params = data_pre_deal( self.handler_params.api_name, self.get_grad_input_from_vjp, [need_grad_tensors, grad_output, inner_args], - {} + {}, ) layer = LayerFactory.create( self.handler_params.api_name, diff --git a/debug/accuracy_tools/msprobe/pytorch/free_benchmark/compare/single_benchmark.py b/debug/accuracy_tools/msprobe/pytorch/free_benchmark/compare/single_benchmark.py index 59239fcd0..cf4bac803 100644 --- a/debug/accuracy_tools/msprobe/pytorch/free_benchmark/compare/single_benchmark.py +++ b/debug/accuracy_tools/msprobe/pytorch/free_benchmark/compare/single_benchmark.py @@ -1,3 +1,18 @@ +# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import math import torch diff --git a/debug/accuracy_tools/msprobe/pytorch/free_benchmark/main.py b/debug/accuracy_tools/msprobe/pytorch/free_benchmark/main.py index 69ece0a0c..66d7b7e10 100644 --- a/debug/accuracy_tools/msprobe/pytorch/free_benchmark/main.py +++ b/debug/accuracy_tools/msprobe/pytorch/free_benchmark/main.py @@ -1,3 +1,18 @@ +# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from abc import ABC import torch @@ -36,9 +51,9 @@ class FreeBenchmarkCheck(ABC): def update_iter(self, update_iter): self.current_iter = update_iter - + def if_fix(self): - if self.config.handler_type==HandlerType.FIX: + if self.config.handler_type == HandlerType.FIX: return True return False @@ -73,9 +88,9 @@ class FreeBenchmarkCheck(ABC): layer.handle(data_params) handler_params = make_handler_params(name, self.config, self.current_iter) handler = FuzzHandlerFactory.create(handler_params) - perturbed_output = handler.handle(data_params) + perturbed_output = handler.handle(data_params) return perturbed_output, handler.get_unequal_rows() - + def backward(self, name, module, grad_output): if not self.config.fuzz_stage == Const.BACKWARD: diff --git a/debug/accuracy_tools/msprobe/pytorch/free_benchmark/perturbed_layers/base_layer.py b/debug/accuracy_tools/msprobe/pytorch/free_benchmark/perturbed_layers/base_layer.py index f64a201d5..0f99d08b6 100644 --- a/debug/accuracy_tools/msprobe/pytorch/free_benchmark/perturbed_layers/base_layer.py +++ b/debug/accuracy_tools/msprobe/pytorch/free_benchmark/perturbed_layers/base_layer.py @@ -1,3 +1,18 @@ +# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from abc import ABC, abstractmethod from typing import Any diff --git a/debug/accuracy_tools/msprobe/pytorch/free_benchmark/perturbed_layers/layer_factory.py b/debug/accuracy_tools/msprobe/pytorch/free_benchmark/perturbed_layers/layer_factory.py index 0ea9107aa..79256cd40 100644 --- a/debug/accuracy_tools/msprobe/pytorch/free_benchmark/perturbed_layers/layer_factory.py +++ b/debug/accuracy_tools/msprobe/pytorch/free_benchmark/perturbed_layers/layer_factory.py @@ -1,14 +1,29 @@ +# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from msprobe.pytorch.free_benchmark import FreeBenchmarkException from msprobe.pytorch.free_benchmark.common.enums import DeviceType, PerturbationMode -from msprobe.pytorch.free_benchmark.perturbed_layers.npu.improve_precision import ( - ImprovePrecisionLayer, -) from msprobe.pytorch.free_benchmark.perturbed_layers.npu.add_noise import AddNoiseLayer from msprobe.pytorch.free_benchmark.perturbed_layers.npu.bit_noise import BitNoiseLayer -from msprobe.pytorch.free_benchmark.perturbed_layers.npu.no_change import NoChangeLayer from msprobe.pytorch.free_benchmark.perturbed_layers.npu.change_value import ( ChangeValueLayer, ) +from msprobe.pytorch.free_benchmark.perturbed_layers.npu.improve_precision import ( + ImprovePrecisionLayer, +) +from msprobe.pytorch.free_benchmark.perturbed_layers.npu.no_change import NoChangeLayer from msprobe.pytorch.free_benchmark.perturbed_layers.run_cpu import CpuLayer diff --git a/debug/accuracy_tools/msprobe/pytorch/free_benchmark/perturbed_layers/npu/add_noise.py b/debug/accuracy_tools/msprobe/pytorch/free_benchmark/perturbed_layers/npu/add_noise.py index 2ccc2bfcf..1e022b4b1 100644 --- a/debug/accuracy_tools/msprobe/pytorch/free_benchmark/perturbed_layers/npu/add_noise.py +++ b/debug/accuracy_tools/msprobe/pytorch/free_benchmark/perturbed_layers/npu/add_noise.py @@ -1,3 +1,18 @@ +# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import torch from msprobe.pytorch.free_benchmark import logger from msprobe.pytorch.free_benchmark.common.constant import ThresholdConfig diff --git a/debug/accuracy_tools/msprobe/pytorch/free_benchmark/perturbed_layers/npu/bit_noise.py b/debug/accuracy_tools/msprobe/pytorch/free_benchmark/perturbed_layers/npu/bit_noise.py index a0ac21691..0f3547a78 100644 --- a/debug/accuracy_tools/msprobe/pytorch/free_benchmark/perturbed_layers/npu/bit_noise.py +++ b/debug/accuracy_tools/msprobe/pytorch/free_benchmark/perturbed_layers/npu/bit_noise.py @@ -1,3 +1,18 @@ +# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import torch from msprobe.pytorch.free_benchmark import logger from msprobe.pytorch.free_benchmark.common.constant import ThresholdConfig diff --git a/debug/accuracy_tools/msprobe/pytorch/free_benchmark/perturbed_layers/npu/change_value.py b/debug/accuracy_tools/msprobe/pytorch/free_benchmark/perturbed_layers/npu/change_value.py index ae5bf9f03..96429213d 100644 --- a/debug/accuracy_tools/msprobe/pytorch/free_benchmark/perturbed_layers/npu/change_value.py +++ b/debug/accuracy_tools/msprobe/pytorch/free_benchmark/perturbed_layers/npu/change_value.py @@ -1,3 +1,18 @@ +# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import torch from msprobe.pytorch.free_benchmark import logger from msprobe.pytorch.free_benchmark.common.enums import PerturbationMode diff --git a/debug/accuracy_tools/msprobe/pytorch/free_benchmark/perturbed_layers/npu/improve_precision.py b/debug/accuracy_tools/msprobe/pytorch/free_benchmark/perturbed_layers/npu/improve_precision.py index b5a106dac..393d82a3a 100644 --- a/debug/accuracy_tools/msprobe/pytorch/free_benchmark/perturbed_layers/npu/improve_precision.py +++ b/debug/accuracy_tools/msprobe/pytorch/free_benchmark/perturbed_layers/npu/improve_precision.py @@ -1,3 +1,18 @@ +# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import torch from msprobe.core.common.const import Const from msprobe.pytorch.free_benchmark import logger diff --git a/debug/accuracy_tools/msprobe/pytorch/free_benchmark/perturbed_layers/npu/no_change.py b/debug/accuracy_tools/msprobe/pytorch/free_benchmark/perturbed_layers/npu/no_change.py index fa775e00e..58c9dd7eb 100644 --- a/debug/accuracy_tools/msprobe/pytorch/free_benchmark/perturbed_layers/npu/no_change.py +++ b/debug/accuracy_tools/msprobe/pytorch/free_benchmark/perturbed_layers/npu/no_change.py @@ -1,3 +1,18 @@ +# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import torch from msprobe.pytorch.free_benchmark import logger from msprobe.pytorch.free_benchmark.common.enums import PerturbationMode diff --git a/debug/accuracy_tools/msprobe/pytorch/free_benchmark/perturbed_layers/npu/npu_base_layser.py b/debug/accuracy_tools/msprobe/pytorch/free_benchmark/perturbed_layers/npu/npu_base_layser.py index 1a8594814..b7d3b7d17 100644 --- a/debug/accuracy_tools/msprobe/pytorch/free_benchmark/perturbed_layers/npu/npu_base_layser.py +++ b/debug/accuracy_tools/msprobe/pytorch/free_benchmark/perturbed_layers/npu/npu_base_layser.py @@ -1,3 +1,18 @@ +# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from abc import abstractmethod from typing import Any diff --git a/debug/accuracy_tools/msprobe/pytorch/free_benchmark/perturbed_layers/run_cpu.py b/debug/accuracy_tools/msprobe/pytorch/free_benchmark/perturbed_layers/run_cpu.py index 376f4ee3e..9422a0610 100644 --- a/debug/accuracy_tools/msprobe/pytorch/free_benchmark/perturbed_layers/run_cpu.py +++ b/debug/accuracy_tools/msprobe/pytorch/free_benchmark/perturbed_layers/run_cpu.py @@ -1,3 +1,18 @@ +# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import torch from msprobe.pytorch.free_benchmark import logger from msprobe.pytorch.free_benchmark.common.params import DataParams diff --git a/debug/accuracy_tools/msprobe/pytorch/free_benchmark/result_handlers/base_handler.py b/debug/accuracy_tools/msprobe/pytorch/free_benchmark/result_handlers/base_handler.py index e36f58673..4b343d6d4 100644 --- a/debug/accuracy_tools/msprobe/pytorch/free_benchmark/result_handlers/base_handler.py +++ b/debug/accuracy_tools/msprobe/pytorch/free_benchmark/result_handlers/base_handler.py @@ -1,8 +1,23 @@ +# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import math from abc import ABC, abstractmethod from typing import Any, Optional, Tuple -import numpy as np +import numpy as np import torch from msprobe.core.common.const import Const from msprobe.pytorch.free_benchmark import logger @@ -35,7 +50,9 @@ class FuzzHandler(ABC): origin_ouput = origin_ouput.values perturbed_output = perturbed_output.values if hasattr(perturbed_output, "dtype"): - abs_tol = ThresholdConfig.ABS_TOL_VALUE_DICT.get(perturbed_output.dtype, FuzzThreshold.F32_THD) + abs_tol = ThresholdConfig.ABS_TOL_VALUE_DICT.get( + perturbed_output.dtype, FuzzThreshold.F32_THD + ) else: abs_tol = FuzzThreshold.F32_THD return ( @@ -53,16 +70,22 @@ class FuzzHandler(ABC): :return origin_output_chunks: 切块后原始输出列表 :return perturbed_output_chunks: 切块后扰动后输出列表 """ - single_output_mem = origin_output.element_size() * origin_output.nelement() / Const.ONE_MB + single_output_mem = ( + origin_output.element_size() * origin_output.nelement() / Const.ONE_MB + ) if single_output_mem == 0 or origin_output.ndim == 0: return [origin_output], [perturbed_output] # 张量大小和批数之间的关系：chunks_exp=math.log(M,2)-4, chunks=2**chunks_exp (M为对比张量数据大小[Mb]) chunks_exp = int(math.log(single_output_mem, 2)) - 4 - chunks = 2 ** chunks_exp + chunks = 2**chunks_exp chunks = max(chunks, 1) chunks = min(chunks, ThresholdConfig.TENSOR_SPLIT_MAX_CHUNK) - origin_output_chunks = TorchC.tensor_split(TorchC.reshape(origin_output, (-1,)), chunks) - perturbed_output_chunks = TorchC.tensor_split(TorchC.reshape(perturbed_output, (-1,)), chunks) + origin_output_chunks = TorchC.tensor_split( + TorchC.reshape(origin_output, (-1,)), chunks + ) + perturbed_output_chunks = TorchC.tensor_split( + TorchC.reshape(perturbed_output, (-1,)), chunks + ) return origin_output_chunks, perturbed_output_chunks @staticmethod @@ -80,14 +103,16 @@ class FuzzHandler(ABC): pass def get_ratio_from_specific_norm( - self, origin_output, perturbed_output, norm_type, abs_tol + self, origin_output, perturbed_output, norm_type, abs_tol ): if norm_type == NormType.ENDLESS_NORM: return self.calculate_error(origin_output, perturbed_output, abs_tol) return ThresholdConfig.COMP_CONSISTENT def calculate_error(self, origin_output, perturbed_output, abs_tol): - origin_output_chunks, perturbed_output_chunks = self.tensor_split_for_error_calculate(origin_output, perturbed_output) + origin_output_chunks, perturbed_output_chunks = ( + self.tensor_split_for_error_calculate(origin_output, perturbed_output) + ) norm1 = -np.inf norm2 = -np.inf norm3 = np.inf @@ -95,11 +120,25 @@ class FuzzHandler(ABC): if chunk_origin.nelement() == 0: break chunk_perturbed = perturbed_output_chunks[i] - ratio_tensor1 = TorchC.where(TorchC.abs(chunk_perturbed) > abs_tol, - TorchC.div(TorchC.clamp(chunk_origin, min=abs_tol), TorchC.clamp(chunk_perturbed, min=abs_tol)), 1) - ratio_tensor2 = TorchC.where(TorchC.abs(chunk_origin) > abs_tol, - TorchC.div(TorchC.clamp(chunk_perturbed, min=abs_tol), TorchC.clamp(chunk_origin, min=abs_tol)), 1) - norm_values = TorchC.stack([TorchC.max(ratio_tensor1), TorchC.max(ratio_tensor2)]) + ratio_tensor1 = TorchC.where( + TorchC.abs(chunk_perturbed) > abs_tol, + TorchC.div( + TorchC.clamp(chunk_origin, min=abs_tol), + TorchC.clamp(chunk_perturbed, min=abs_tol), + ), + 1, + ) + ratio_tensor2 = TorchC.where( + TorchC.abs(chunk_origin) > abs_tol, + TorchC.div( + TorchC.clamp(chunk_perturbed, min=abs_tol), + TorchC.clamp(chunk_origin, min=abs_tol), + ), + 1, + ) + norm_values = TorchC.stack( + [TorchC.max(ratio_tensor1), TorchC.max(ratio_tensor2)] + ) max_ratio1, max_ratio2 = norm_values.tolist() norm1 = max(norm1, self.convert_overflow_ratio_to_consistent(max_ratio1)) norm2 = max(norm2, self.convert_overflow_ratio_to_consistent(max_ratio2)) @@ -126,13 +165,13 @@ class FuzzHandler(ABC): if self.params.fuzz_stage == Const.BACKWARD: abs_tol = ThresholdConfig.BACKWARD_OUTPUT_LOWER_BOUND else: - abs_tol = abs_tol ** 0.5 + abs_tol = abs_tol**0.5 return self.get_ratio_from_specific_norm( origin_output, perturbed_output, norm_type, abs_tol ) def npu_compare( - self, origin_output, perturbed_output + self, origin_output, perturbed_output ) -> Tuple[bool, Optional[float]]: if isinstance(perturbed_output, int): @@ -189,7 +228,7 @@ class FuzzHandler(ABC): max_fuzz_ratio if ratio is None else max(max_fuzz_ratio, ratio) ) data_params.is_consistent = ( - is_consistent and data_params.is_consistent + is_consistent and data_params.is_consistent ) if not is_consistent and data_params.grad_unequal_flag: self.unequal_rows.append( diff --git a/debug/accuracy_tools/msprobe/pytorch/free_benchmark/result_handlers/check_handler.py b/debug/accuracy_tools/msprobe/pytorch/free_benchmark/result_handlers/check_handler.py index c16284eb0..9feec1531 100644 --- a/debug/accuracy_tools/msprobe/pytorch/free_benchmark/result_handlers/check_handler.py +++ b/debug/accuracy_tools/msprobe/pytorch/free_benchmark/result_handlers/check_handler.py @@ -1,3 +1,18 @@ +# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from typing import Any from msprobe.pytorch.free_benchmark import logger diff --git a/debug/accuracy_tools/msprobe/pytorch/free_benchmark/result_handlers/fix_handler.py b/debug/accuracy_tools/msprobe/pytorch/free_benchmark/result_handlers/fix_handler.py index a1d90035e..bc7d0a5f2 100644 --- a/debug/accuracy_tools/msprobe/pytorch/free_benchmark/result_handlers/fix_handler.py +++ b/debug/accuracy_tools/msprobe/pytorch/free_benchmark/result_handlers/fix_handler.py @@ -1,3 +1,18 @@ +# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from typing import Any from msprobe.pytorch.free_benchmark.common.params import DataParams diff --git a/debug/accuracy_tools/msprobe/pytorch/free_benchmark/result_handlers/handler_factory.py b/debug/accuracy_tools/msprobe/pytorch/free_benchmark/result_handlers/handler_factory.py index 46efd8283..7a8e0aa7f 100644 --- a/debug/accuracy_tools/msprobe/pytorch/free_benchmark/result_handlers/handler_factory.py +++ b/debug/accuracy_tools/msprobe/pytorch/free_benchmark/result_handlers/handler_factory.py @@ -1,3 +1,18 @@ +# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from msprobe.pytorch.free_benchmark import FreeBenchmarkException from msprobe.pytorch.free_benchmark.common.constant import PreheatConfig from msprobe.pytorch.free_benchmark.common.enums import HandlerType diff --git a/debug/accuracy_tools/msprobe/pytorch/free_benchmark/result_handlers/preheat_handler.py b/debug/accuracy_tools/msprobe/pytorch/free_benchmark/result_handlers/preheat_handler.py index d78e43036..dd71e679c 100644 --- a/debug/accuracy_tools/msprobe/pytorch/free_benchmark/result_handlers/preheat_handler.py +++ b/debug/accuracy_tools/msprobe/pytorch/free_benchmark/result_handlers/preheat_handler.py @@ -1,3 +1,18 @@ +# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import math from typing import Any @@ -136,9 +151,7 @@ class PreheatHandler(FuzzHandler): def _adjust_threshold_for_dtype(self, dtype_str, compare_result): con_ratio = [ratio for ratio, is_consistent in compare_result if is_consistent] - incon_ratio = [ - ratio for ratio, is_consistent in compare_result if not is_consistent - ] + incon_ratio = [ratio for ratio, is_consistent in compare_result if not is_consistent] old_thd = preheat_counter.get_api_thd(self.pure_name, dtype_str) new_thd = old_thd # 正例负例都存在 -- Gitee From ed2e66a7655c6994149e911f30d5163929a77498 Mon Sep 17 00:00:00 2001 From: Linwei-Ying Date: Thu, 26 Sep 2024 17:21:20 +0800 Subject: [PATCH 10/13] compare ut test_acc_compare_check_modify --- .../test/core_ut/compare/test_acc_compare_check.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/debug/accuracy_tools/msprobe/test/core_ut/compare/test_acc_compare_check.py b/debug/accuracy_tools/msprobe/test/core_ut/compare/test_acc_compare_check.py index 108a21abd..32333aad3 100644 --- a/debug/accuracy_tools/msprobe/test/core_ut/compare/test_acc_compare_check.py +++ b/debug/accuracy_tools/msprobe/test/core_ut/compare/test_acc_compare_check.py @@ -161,18 +161,6 @@ class TestUtilsMethods(unittest.TestCase): self.assertEqual(context.exception.code, CompareException.INVALID_CHAR_ERROR) def test_check_stack_json_str_5(self): - stack_info = ['&File', 'File'] - with self.assertRaises(CompareException) as context: - check_stack_json_str(stack_info, op_name) - self.assertEqual(context.exception.code, CompareException.INVALID_CHAR_ERROR) - - def test_check_stack_json_str_6(self): - stack_info = ['\File', 'File'] - with self.assertRaises(CompareException) as context: - check_stack_json_str(stack_info, op_name) - self.assertEqual(context.exception.code, CompareException.INVALID_CHAR_ERROR) - - def test_check_stack_json_str_7(self): stack_info = ['@File', 'File'] with self.assertRaises(CompareException) as context: check_stack_json_str(stack_info, op_name) -- Gitee From 2bf56d1b855d2277d706b28bedbeee19595b139d Mon Sep 17 00:00:00 2001 From: cai-weiwei1989 <734267852@qq.com> Date: Tue, 24 Sep 2024 20:35:19 +0800 Subject: [PATCH 11/13] =?UTF-8?q?[msprobe]FAQ=E8=B5=84=E6=96=99=E8=A1=A5?= =?UTF-8?q?=E5=85=85=E4=B8=80=E4=B8=AAFAQ?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../msprobe/docs/{FAQ_PyTorch.md => FAQ.md} | 24 ++++++++++++++----- 1 file changed, 18 insertions(+), 6 deletions(-) rename debug/accuracy_tools/msprobe/docs/{FAQ_PyTorch.md => FAQ.md} (94%) diff --git a/debug/accuracy_tools/msprobe/docs/FAQ_PyTorch.md b/debug/accuracy_tools/msprobe/docs/FAQ.md similarity index 94% rename from debug/accuracy_tools/msprobe/docs/FAQ_PyTorch.md rename to debug/accuracy_tools/msprobe/docs/FAQ.md index 6d724975b..fd9652583 100644 --- a/debug/accuracy_tools/msprobe/docs/FAQ_PyTorch.md +++ b/debug/accuracy_tools/msprobe/docs/FAQ.md @@ -1,4 +1,16 @@ -# 1 精度预检工具 + + +# 1 数据采集 + +1. dump.json中API或Module统计信息里出现null或None值的原因是什么？ + + dump.json里出现null或None值的可能性较多，常见的场景有： + + - 输入或者输出参数本身是一个None值。 + - 输入参数或输出参数类型当前工具不支持，会有日志打印提醒。 + - 输入或者输出tensor的dtype为bool时，Mean和Norm等字段为null。 + +# 2 精度预检(PyTorch) 1. 预检工具在 dump 和 run_ut 的过程中，是否需要同时开启或关闭 jit 编译（jit_compile）？ @@ -61,11 +73,11 @@ | `__truediv__` | 同 `__div__` | | `__xor__` | ^ | -# 2 精度比对工具 +# 3 精度比对(PyTorch) -## 2.1 工具使用 +## 3.1 工具使用 -### 2.1.1 dump 指定融合算子 +### 3.1.1 dump 指定融合算子 数据采集当前支持融合算子的输入输出，需要在 `mstt/debug/accuracy_tools/msprobe/pytorch/hook_module/support_wrap_ops.yaml` 中添加，比如以下代码段调用的 softmax 融合算子。 @@ -83,7 +95,7 @@ def npu_forward_fused_softmax(self, input_, mask): （npu_scaled_masked_softmax 融合算子工具已支持 dump，本例仅供参考）。 -## 2.2 常见问题 +## 3.2 常见问题 1. 在同一个目录多次执行 dump 会冲突吗？ @@ -97,7 +109,7 @@ def npu_forward_fused_softmax(self, input_, mask): 答：torch 版本和硬件差异属于正常情况。 -## 2.3 异常情况 +## 3.3 异常情况 1. HCCL 报错： error code: EI0006。 -- Gitee From a7f679a76274adfbad94896962db7478a95668b2 Mon Sep 17 00:00:00 2001 From: l30036321 Date: Thu, 26 Sep 2024 20:47:41 +0800 Subject: [PATCH 12/13] fix norm compute --- .../core/data_dump/data_processor/mindspore_processor.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/mindspore_processor.py b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/mindspore_processor.py index 910c508f7..a8c4cfee4 100644 --- a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/mindspore_processor.py +++ b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/mindspore_processor.py @@ -75,7 +75,10 @@ class MindsporeDataProcessor(BaseDataProcessor): get_max_value = api_register.mint_ops_ori_attr.get("max", mint.max) get_min_value = api_register.mint_ops_ori_attr.get("min", mint.min) get_mean_value = api_register.mint_ops_ori_attr.get("mean", mint.mean) - get_norm_value = api_register.functional_ori_attr.get("norm", ops.norm) + if hasattr(mint, "norm"): + get_norm_value = api_register.mint_ops_ori_attr.get("norm", mint.norm) + else: + get_norm_value = api_register.functional_ori_attr.get("norm", ops.norm) tensor_stat.max = get_max_value(data).item() tensor_stat.min = get_min_value(data).item() tensor_stat.mean = get_mean_value(data).item() -- Gitee From 52a7ca9258c43fb7a74cf442582ace3d194fc5bf Mon Sep 17 00:00:00 2001 From: wangchao Date: Thu, 26 Sep 2024 17:26:22 +0800 Subject: [PATCH 13/13] add parse_json ut --- .../msprobe/pytorch/common/parse_json.py | 20 +++--- .../compare/test_modify_mapping.py | 8 ++- .../test/pytorch_ut/common/test_parse_json.py | 63 +++++++++++++++++++ .../test/resources/common/test_backward.json | 7 +++ .../resources/common/test_no_dump_data.json | 3 + .../resources/common/test_no_fb_data.json | 6 ++ 6 files changed, 96 insertions(+), 11 deletions(-) create mode 100644 debug/accuracy_tools/msprobe/test/pytorch_ut/common/test_parse_json.py create mode 100644 debug/accuracy_tools/msprobe/test/resources/common/test_backward.json create mode 100644 debug/accuracy_tools/msprobe/test/resources/common/test_no_dump_data.json create mode 100644 debug/accuracy_tools/msprobe/test/resources/common/test_no_fb_data.json diff --git a/debug/accuracy_tools/msprobe/pytorch/common/parse_json.py b/debug/accuracy_tools/msprobe/pytorch/common/parse_json.py index 89edd834c..ad2413f2f 100644 --- a/debug/accuracy_tools/msprobe/pytorch/common/parse_json.py +++ b/debug/accuracy_tools/msprobe/pytorch/common/parse_json.py @@ -5,14 +5,6 @@ from msprobe.core.common.file_utils import FileOpen def parse_json_info_forward_backward(json_path): - def parse_data_name_with_pattern(data_name, pattern): - name_struct = data_name.split('.') - if not name_struct[-1] == pattern: - raise ParseJsonException(ParseJsonException.UnexpectedNameStruct, - f"{data_name} in file {json_path}") - api_name = '.'.join(name_struct[:-1]) - return api_name - with FileOpen(json_path, 'r') as f: dump_json = json.load(f) @@ -27,13 +19,21 @@ def parse_json_info_forward_backward(json_path): if "Module" in data_name: continue if "forward" in data_name: - api_name = parse_data_name_with_pattern(data_name, "forward") + api_name = parse_data_name_with_pattern(data_name, "forward", json_path) forward_data.update({api_name: data_item}) elif "backward" in data_name: - api_name = parse_data_name_with_pattern(data_name, "backward") + api_name = parse_data_name_with_pattern(data_name, "backward", json_path) backward_data.update({api_name: data_item}) else: raise ParseJsonException(ParseJsonException.UnexpectedNameStruct, f"{data_name} in file {json_path}.") return forward_data, backward_data, real_data_path + + +def parse_data_name_with_pattern(data_name, pattern, json_path): + name_struct = data_name.split('.') + if not name_struct[-1] == pattern: + raise ParseJsonException(ParseJsonException.UnexpectedNameStruct, f"{data_name} in file {json_path}") + api_name = '.'.join(name_struct[:-1]) + return api_name diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/compare/test_modify_mapping.py b/debug/accuracy_tools/msprobe/test/mindspore_ut/compare/test_modify_mapping.py index 985262610..6cc9e8bf0 100644 --- a/debug/accuracy_tools/msprobe/test/mindspore_ut/compare/test_modify_mapping.py +++ b/debug/accuracy_tools/msprobe/test/mindspore_ut/compare/test_modify_mapping.py @@ -44,7 +44,8 @@ class TestModifyMapping(unittest.TestCase): "Functional.max_pool2d.0.forward": "Module.pool1.MaxPool2d.forward.0", "Funtional.conv2d.1.forward": "Module.conv2.Conv2d.forward.0", "Functional.linear.5.backward": "Module.fc3.Linear.backward.1", - "Module.conv1.Conv2d.backward.1": None + "Module.conv1.Conv2d.backward.1": None, + "Functional.conv2d.2.forward": None } self.ms_construct = { "Functional.add.0.forward": "Cell.transformer_layers.0.attention.core_attention.scale_mask_softmax.ScaleMaskSoftmax.forward.0", @@ -224,6 +225,11 @@ class TestModifyMapping(unittest.TestCase): "origin_data": "Module.conv1.Conv2d.backward.1", "scope": None, "stack": None + }, + "Module.conv2d.conv2d.2": { + "origin_data": "Functional.conv2d.2.forward", + "scope": None, + "stack": None } } self.assertIn("Module.pool1.max_pool2d.max_pool2d.0", result) diff --git a/debug/accuracy_tools/msprobe/test/pytorch_ut/common/test_parse_json.py b/debug/accuracy_tools/msprobe/test/pytorch_ut/common/test_parse_json.py new file mode 100644 index 000000000..e802088c0 --- /dev/null +++ b/debug/accuracy_tools/msprobe/test/pytorch_ut/common/test_parse_json.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +# Copyright (C) 2024-2024. Huawei Technologies Co., Ltd. All rights reserved. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +import json +import os +import shutil +from unittest import TestCase +from unittest.mock import patch, MagicMock, mock_open + +from msprobe.core.common.exceptions import ParseJsonException +from msprobe.pytorch.common.parse_json import parse_data_name_with_pattern +from msprobe.pytorch.common.parse_json import parse_json_info_forward_backward + + +class TestParseJson(TestCase): + + def setUp(self): + self.base_test_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__)))) + self.input_dir = os.path.join(self.base_test_dir, 'resources') + self.output_path = os.path.abspath(os.path.join(self.base_test_dir, 'test_output')) + + os.makedirs(self.output_path, mode=0o700, exist_ok=True) + self.has_error = False + + def tearDown(self) -> None: + shutil.rmtree(self.output_path, ignore_errors=True) + + def test_parse_data_name_with_pattern_exception(self): + invalid_data_name = "matmul.unforward" + pattern = "forward" + json_path = "test/test.json" + + with self.assertRaises(ParseJsonException): + parse_data_name_with_pattern(invalid_data_name, pattern, json_path) + + def test_parse_json_info_forward_backward_no_dump_data_exception(self): + no_dump_data_json = os.path.join(self.input_dir, 'common', 'test_no_dump_data.json') + with self.assertRaises(ParseJsonException): + parse_json_info_forward_backward(no_dump_data_json) + + def test_parse_json_info_forward_backward(self): + backward_json = os.path.join(self.input_dir, 'common', 'test_backward.json') + golden_result = ({}, {"matmul": 1}, "./") + result = parse_json_info_forward_backward(backward_json) + self.assertEqual(result, golden_result) + + def test_parse_json_info_forward_backward_no_fb_data_exception(self): + no_fb_data_json = os.path.join(self.input_dir, 'common', 'test_no_fb_data.json') + with self.assertRaises(ParseJsonException): + parse_json_info_forward_backward(no_fb_data_json) diff --git a/debug/accuracy_tools/msprobe/test/resources/common/test_backward.json b/debug/accuracy_tools/msprobe/test/resources/common/test_backward.json new file mode 100644 index 000000000..bae40c199 --- /dev/null +++ b/debug/accuracy_tools/msprobe/test/resources/common/test_backward.json @@ -0,0 +1,7 @@ +{ + "dump_data_dir": "./", + "data": { + "Module": 1, + "matmul.backward": 1 + } +} \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/test/resources/common/test_no_dump_data.json b/debug/accuracy_tools/msprobe/test/resources/common/test_no_dump_data.json new file mode 100644 index 000000000..aa84bf177 --- /dev/null +++ b/debug/accuracy_tools/msprobe/test/resources/common/test_no_dump_data.json @@ -0,0 +1,3 @@ +{ + "dump_data_dir": "./" +} \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/test/resources/common/test_no_fb_data.json b/debug/accuracy_tools/msprobe/test/resources/common/test_no_fb_data.json new file mode 100644 index 000000000..0ebc95562 --- /dev/null +++ b/debug/accuracy_tools/msprobe/test/resources/common/test_no_fb_data.json @@ -0,0 +1,6 @@ +{ + "dump_data_dir": "./", + "data": { + "invalid_value": 1 + } +} \ No newline at end of file -- Gitee