From aeb3c568d121f1506cadc5a8ecacbf1aec8c72ab Mon Sep 17 00:00:00 2001 From: sunyiming Date: Tue, 31 Oct 2023 02:30:21 +0000 Subject: [PATCH 1/6] support save real data in rank fold Signed-off-by: sunyiming --- .../api_accuracy_checker/common/base_api.py | 30 +++++++++++++++++-- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/debug/accuracy_tools/api_accuracy_checker/common/base_api.py b/debug/accuracy_tools/api_accuracy_checker/common/base_api.py index 2c3086184c3..83096974a23 100644 --- a/debug/accuracy_tools/api_accuracy_checker/common/base_api.py +++ b/debug/accuracy_tools/api_accuracy_checker/common/base_api.py @@ -54,12 +54,18 @@ class BaseAPIInfo: else: api_args = self.api_name + '.' + str(self.args_num) + rank = self.get_tensor_rank(arg) + if rank is not None: + rank = "rank"+str(rank) if self.is_forward: - forward_real_data_path = os.path.join(self.save_path, self.forward_path) - + forward_real_data_path = os.path.join(self.save_path, self.forward_path, rank) if rank else os.path.join(self.save_path, self.forward_path) + if not os.path.exists(forward_real_data_path): + os.makedirs(forward_real_data_path) file_path = os.path.join(forward_real_data_path, f'{api_args}.pt') else: - backward_real_data_path = os.path.join(self.save_path, self.backward_path) + backward_real_data_path = os.path.join(self.save_path, self.backward_path, rank) if rank else os.path.join(self.save_path, self.backward_path) + if not os.path.exists(backward_real_data_path): + os.makedirs(backward_real_data_path) file_path = os.path.join(backward_real_data_path, f'{api_args}.pt') self.args_num += 1 pt_path = write_pt(file_path, arg.contiguous().cpu().detach()) @@ -68,6 +74,24 @@ class BaseAPIInfo: single_arg.update({'requires_grad': arg.requires_grad}) return single_arg + def get_tensor_rank(self, arg): + def get_tensor_rank_single(x): + if isinstance(x, (list, tuple)): + if len(x) > 0: + return get_tensor_rank_single(x[0]) + return None + elif isinstance(x, torch.Tensor): + device = x.device + if device.type == 'cpu': + return None + else: + return device.index + return None + rank = get_tensor_rank_single(arg) + if rank is None: + return None + return rank + def analyze_builtin(self, arg): single_arg = {} if self.is_save_data: -- Gitee From 1195a0fd2db12a423f67a7b49b9bc08262e19377 Mon Sep 17 00:00:00 2001 From: sunyiming Date: Tue, 31 Oct 2023 02:36:31 +0000 Subject: [PATCH 2/6] Revert "change ut assert" This reverts commit 58c6b789f2d52bf8553b4bf46c93c9ba58440bc8. --- debug/accuracy_tools/ptdbg_ascend/test/ut/test_hooks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/ptdbg_ascend/test/ut/test_hooks.py b/debug/accuracy_tools/ptdbg_ascend/test/ut/test_hooks.py index 82f3d8dfedd..7874d3c2fa9 100644 --- a/debug/accuracy_tools/ptdbg_ascend/test/ut/test_hooks.py +++ b/debug/accuracy_tools/ptdbg_ascend/test/ut/test_hooks.py @@ -17,7 +17,7 @@ class TestUtilsMethods(unittest.TestCase): self.assertTrue(dump_util.dump_init_enable) self.assertEqual(dump_util.dump_switch_scope, []) self.assertEqual(dump_util.dump_api_list, []) - self.assertEqual(dump_util.dump_filter_switch, "OFF") + self.assertEqual(dump_util.dump_filter_switch, switch_on) self.assertEqual(dump_count, 0) def test_set_dump_switch_mode_is_list(self): -- Gitee From b2d5f43e96ae5e20bf4c3414205a0916096e0120 Mon Sep 17 00:00:00 2001 From: sunyiming Date: Tue, 31 Oct 2023 02:36:42 +0000 Subject: [PATCH 3/6] Revert "change filter switch default to off" This reverts commit f735168eafe7970ddcaf2c04eeaf21a6d1e30c91. --- .../src/python/ptdbg_ascend/debugger/precision_debugger.py | 2 +- .../ptdbg_ascend/src/python/ptdbg_ascend/dump/utils.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/debugger/precision_debugger.py b/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/debugger/precision_debugger.py index b305769cf2f..de1ff96b263 100644 --- a/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/debugger/precision_debugger.py +++ b/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/debugger/precision_debugger.py @@ -40,7 +40,7 @@ class PrecisionDebugger: hook_dict = {"dump": self.configure_full_dump, "overflow_check": self.configure_overflow_dump} return hook_dict.get(hook_name, lambda: ValueError("hook name {} is not in ['dump', 'overflow_check']".format(hook_name))) - def configure_full_dump(self, mode='api_stack', scope=[], api_list=[], filter_switch=Const.OFF, + def configure_full_dump(self, mode='api_stack', scope=[], api_list=[], filter_switch=Const.ON, input_output_mode=[Const.ALL], acl_config=None, backward_input=[], summary_only=False): set_dump_switch_config(mode=mode, scope=scope, api_list=api_list, filter_switch=filter_switch, dump_mode=input_output_mode, summary_only=summary_only) diff --git a/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/dump/utils.py b/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/dump/utils.py index 7fd8b4e9b04..e6cfa1f444b 100644 --- a/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/dump/utils.py +++ b/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/dump/utils.py @@ -195,7 +195,7 @@ def generate_dump_path_str(): return dump_path -def set_dump_switch(switch, mode=Const.ALL, scope=[], api_list=[], filter_switch=Const.OFF, dump_mode=[Const.ALL], summary_only=False): +def set_dump_switch(switch, mode=Const.ALL, scope=[], api_list=[], filter_switch=Const.ON, dump_mode=[Const.ALL], summary_only=False): check_switch_valid(switch) if not DumpUtil.dump_path: set_dump_path() @@ -209,7 +209,7 @@ def set_dump_switch(switch, mode=Const.ALL, scope=[], api_list=[], filter_switch set_dump_switch_config(mode=mode, scope=scope, api_list=api_list, filter_switch=filter_switch, dump_mode=dump_mode,summary_only=summary_only) -def set_dump_switch_config(mode=Const.ALL, scope=[], api_list=[], filter_switch=Const.OFF, dump_mode=[Const.ALL], summary_only=False): +def set_dump_switch_config(mode=Const.ALL, scope=[], api_list=[], filter_switch=Const.ON, dump_mode=[Const.ALL], summary_only=False): try: check_mode_valid(mode, scope, api_list) check_switch_valid(filter_switch) -- Gitee From f96844fbec58d14e29886df422967d5373b6e960 Mon Sep 17 00:00:00 2001 From: sunyiming Date: Tue, 31 Oct 2023 03:04:14 +0000 Subject: [PATCH 4/6] update debug/accuracy_tools/api_accuracy_checker/test/ut/common/test_config.py. Signed-off-by: sunyiming --- .../api_accuracy_checker/test/ut/common/test_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/api_accuracy_checker/test/ut/common/test_config.py b/debug/accuracy_tools/api_accuracy_checker/test/ut/common/test_config.py index ed764987d5a..a68057dfb41 100644 --- a/debug/accuracy_tools/api_accuracy_checker/test/ut/common/test_config.py +++ b/debug/accuracy_tools/api_accuracy_checker/test/ut/common/test_config.py @@ -17,5 +17,5 @@ class TestConfig(unittest.TestCase): def test_update_config(self): - self.config.update_config(dump_path='/new/path/to/dump', enable_dataloader=False) + self.config.update_config(dump_path='/new/path/to/dump') self.assertEqual(self.config.dump_path, '/new/path/to/dump') -- Gitee From d51a80cafaa328bb0234d5ae994f367cd05890e9 Mon Sep 17 00:00:00 2001 From: sunyiming Date: Tue, 31 Oct 2023 03:04:35 +0000 Subject: [PATCH 5/6] update debug/accuracy_tools/api_accuracy_checker/test/ut/dump/test_dump_scopr.py. Signed-off-by: sunyiming --- .../api_accuracy_checker/test/ut/dump/test_dump_scopr.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/debug/accuracy_tools/api_accuracy_checker/test/ut/dump/test_dump_scopr.py b/debug/accuracy_tools/api_accuracy_checker/test/ut/dump/test_dump_scopr.py index addba38e384..b892a6077a3 100644 --- a/debug/accuracy_tools/api_accuracy_checker/test/ut/dump/test_dump_scopr.py +++ b/debug/accuracy_tools/api_accuracy_checker/test/ut/dump/test_dump_scopr.py @@ -10,12 +10,12 @@ class TestDumpScope(unittest.TestCase): wrapped_func = iter_tracer(dummy_func) result = wrapped_func() - self.assertEqual(DumpUtil.dump_switch, "ON") + self.assertEqual(DumpUtil.dump_switch, "OFF") self.assertEqual(result, "Hello, World!") def another_dummy_func(): return 123 wrapped_func = iter_tracer(another_dummy_func) result = wrapped_func() - self.assertEqual(DumpUtil.dump_switch, "ON") + self.assertEqual(DumpUtil.dump_switch, "OFF") self.assertEqual(result, 123) -- Gitee From 40a699790fbec9ac9e1c9c6fd245fe786f0ddd4e Mon Sep 17 00:00:00 2001 From: sunyiming Date: Wed, 1 Nov 2023 02:26:13 +0000 Subject: [PATCH 6/6] clearcode Signed-off-by: sunyiming --- debug/accuracy_tools/api_accuracy_checker/common/base_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/api_accuracy_checker/common/base_api.py b/debug/accuracy_tools/api_accuracy_checker/common/base_api.py index 83096974a23..49fd004c060 100644 --- a/debug/accuracy_tools/api_accuracy_checker/common/base_api.py +++ b/debug/accuracy_tools/api_accuracy_checker/common/base_api.py @@ -56,7 +56,7 @@ class BaseAPIInfo: api_args = self.api_name + '.' + str(self.args_num) rank = self.get_tensor_rank(arg) if rank is not None: - rank = "rank"+str(rank) + rank = "rank" + str(rank) if self.is_forward: forward_real_data_path = os.path.join(self.save_path, self.forward_path, rank) if rank else os.path.join(self.save_path, self.forward_path) if not os.path.exists(forward_real_data_path): -- Gitee