From 89738df1426d2d9503c8ab635a0d84b722964adb Mon Sep 17 00:00:00 2001 From: qianzhengxin Date: Fri, 23 Aug 2024 16:23:26 +0800 Subject: [PATCH 1/4] grad_monitor ut --- .../grad_tool/test/ut_ms/config.yaml | 6 +++ .../grad_tool/test/ut_ms/only_optimizer.py | 39 +++++++++++++++++++ .../grad_tool/test/ut_ms/test_grad_monitor.py | 36 +++++++++++++++++ 3 files changed, 81 insertions(+) create mode 100644 debug/accuracy_tools/grad_tool/test/ut_ms/config.yaml create mode 100644 debug/accuracy_tools/grad_tool/test/ut_ms/only_optimizer.py diff --git a/debug/accuracy_tools/grad_tool/test/ut_ms/config.yaml b/debug/accuracy_tools/grad_tool/test/ut_ms/config.yaml new file mode 100644 index 0000000000..83c815bc14 --- /dev/null +++ b/debug/accuracy_tools/grad_tool/test/ut_ms/config.yaml @@ -0,0 +1,6 @@ +level: L0 +param_list: +rank: +step: [1, 3] +bounds: +output_path: "" \ No newline at end of file diff --git a/debug/accuracy_tools/grad_tool/test/ut_ms/only_optimizer.py b/debug/accuracy_tools/grad_tool/test/ut_ms/only_optimizer.py new file mode 100644 index 0000000000..56ead5a255 --- /dev/null +++ b/debug/accuracy_tools/grad_tool/test/ut_ms/only_optimizer.py @@ -0,0 +1,39 @@ +import mindspore +from mindspore import nn, Tensor +from mindspore.nn import SGD +import numpy as np +import os + +file_path = os.path.abspath(__file__) +directory = os.path.dirname(file_path) + +from grad_tool.grad_monitor import GradientMonitor + + +def main(): + gm = GradientMonitor(os.path.join(directory, "config.yaml"), framework="MindSpore") + + class SimpleNet(nn.Cell): + def __init__(self): + super().__init__() + self.my_dense = nn.Dense(16, 5) + + def construct(self, x): + x = self.flatten(x) + logits = self.my_dense(x) + return logits + model = SimpleNet() + optimizer = SGD(model.trainable_parameter(), learning_rate=0.001) + + gm.monitor(optimizer) + + fix_gradient = tuple([Tensor(np.arange(5*16).reshape((5, 16)), dtype=mindspore.float32), + Tensor(np.arange(5).reshape(5), dtype=mindspore.float32)]) + + steps = 10 + + for _ in range(steps): + optimizer(fix_gradient) + +if __name__ == "__main__": + main() diff --git a/debug/accuracy_tools/grad_tool/test/ut_ms/test_grad_monitor.py b/debug/accuracy_tools/grad_tool/test/ut_ms/test_grad_monitor.py index e69de29bb2..fad0d0d976 100644 --- a/debug/accuracy_tools/grad_tool/test/ut_ms/test_grad_monitor.py +++ b/debug/accuracy_tools/grad_tool/test/ut_ms/test_grad_monitor.py @@ -0,0 +1,36 @@ +import os + +from unittest import TestCase +from unittest.mock import patch + +from grad_tool.test.ut_ms.only_optimizer import main + +import yaml +file_path = os.path.abspath(__file__) +directory = os.path.dirname(file_path) + + +def update_yaml_key_value(yaml_file, key, new_value): + try: + with open(yaml_file, 'r') as file: + data = yaml.safe_load(file) + + if key in data: + data[key] = new_value + with open(yaml_file, 'w') as file: + yaml.safe_dump(data, file, indent=4) + print(f"键 '{key}' 的值已更新为 '{new_value}'。") + else: + print(f"键 '{key}' 在YAML文件中不存在。") + except yaml.YAMLError as e: + print(f"YAML解析错误: {e}") + + + +class TestMsGradientMonitor(TestCase): + def test_gradient_monitor(self): + config_path = os.path.join(directory, "config.yaml") + gradient_output_path = os.path.join(directory, "gradient_output") + update_yaml_key_value(config_path, "output_path", gradient_output_path) + main() + update_yaml_key_value(config_path, "output_path", "") \ No newline at end of file -- Gitee From 18ee7f2e427bf5860b9fa808f2006242da4294d9 Mon Sep 17 00:00:00 2001 From: qianzhengxin Date: Fri, 23 Aug 2024 16:34:42 +0800 Subject: [PATCH 2/4] fix ut --- .../grad_tool/test/ut_ms/config.yaml | 6 ---- .../grad_tool/test/ut_ms/only_optimizer.py | 2 +- .../grad_tool/test/ut_ms/test_grad_monitor.py | 33 ++++++++++--------- 3 files changed, 18 insertions(+), 23 deletions(-) delete mode 100644 debug/accuracy_tools/grad_tool/test/ut_ms/config.yaml diff --git a/debug/accuracy_tools/grad_tool/test/ut_ms/config.yaml b/debug/accuracy_tools/grad_tool/test/ut_ms/config.yaml deleted file mode 100644 index 83c815bc14..0000000000 --- a/debug/accuracy_tools/grad_tool/test/ut_ms/config.yaml +++ /dev/null @@ -1,6 +0,0 @@ -level: L0 -param_list: -rank: -step: [1, 3] -bounds: -output_path: "" \ No newline at end of file diff --git a/debug/accuracy_tools/grad_tool/test/ut_ms/only_optimizer.py b/debug/accuracy_tools/grad_tool/test/ut_ms/only_optimizer.py index 56ead5a255..775bb2e09d 100644 --- a/debug/accuracy_tools/grad_tool/test/ut_ms/only_optimizer.py +++ b/debug/accuracy_tools/grad_tool/test/ut_ms/only_optimizer.py @@ -23,7 +23,7 @@ def main(): logits = self.my_dense(x) return logits model = SimpleNet() - optimizer = SGD(model.trainable_parameter(), learning_rate=0.001) + optimizer = SGD(model.trainable_params(), learning_rate=0.001) gm.monitor(optimizer) diff --git a/debug/accuracy_tools/grad_tool/test/ut_ms/test_grad_monitor.py b/debug/accuracy_tools/grad_tool/test/ut_ms/test_grad_monitor.py index fad0d0d976..fa2d62d99d 100644 --- a/debug/accuracy_tools/grad_tool/test/ut_ms/test_grad_monitor.py +++ b/debug/accuracy_tools/grad_tool/test/ut_ms/test_grad_monitor.py @@ -10,20 +10,13 @@ file_path = os.path.abspath(__file__) directory = os.path.dirname(file_path) -def update_yaml_key_value(yaml_file, key, new_value): - try: - with open(yaml_file, 'r') as file: - data = yaml.safe_load(file) +def save_dict_as_yaml(data, yaml_file_path): + # 将字典保存为YAML文件 + with open(yaml_file_path, 'w') as yaml_file: + yaml.dump(data, yaml_file, default_flow_style=False) + print(f"字典已保存为YAML文件: {yaml_file_path}") + - if key in data: - data[key] = new_value - with open(yaml_file, 'w') as file: - yaml.safe_dump(data, file, indent=4) - print(f"键 '{key}' 的值已更新为 '{new_value}'。") - else: - print(f"键 '{key}' 在YAML文件中不存在。") - except yaml.YAMLError as e: - print(f"YAML解析错误: {e}") @@ -31,6 +24,14 @@ class TestMsGradientMonitor(TestCase): def test_gradient_monitor(self): config_path = os.path.join(directory, "config.yaml") gradient_output_path = os.path.join(directory, "gradient_output") - update_yaml_key_value(config_path, "output_path", gradient_output_path) - main() - update_yaml_key_value(config_path, "output_path", "") \ No newline at end of file + config_dict = { + "level": "L2", + "param_list": None, + "rank": None, + "step": [1, 3], + "bounds": None, + "output_path": gradient_output_path, + } + save_dict_as_yaml(config_dict, config_path) + + main() \ No newline at end of file -- Gitee From 53618b71d70e43b187465fcb1216b12e5129cb05 Mon Sep 17 00:00:00 2001 From: qianzhengxin Date: Fri, 23 Aug 2024 18:25:04 +0800 Subject: [PATCH 3/4] ut fix --- .../grad_tool/test/ut_ms/test_grad_monitor.py | 42 ++++++++++++++++--- 1 file changed, 36 insertions(+), 6 deletions(-) diff --git a/debug/accuracy_tools/grad_tool/test/ut_ms/test_grad_monitor.py b/debug/accuracy_tools/grad_tool/test/ut_ms/test_grad_monitor.py index fa2d62d99d..288536df3a 100644 --- a/debug/accuracy_tools/grad_tool/test/ut_ms/test_grad_monitor.py +++ b/debug/accuracy_tools/grad_tool/test/ut_ms/test_grad_monitor.py @@ -1,11 +1,12 @@ import os - +import numpy as np +import shutil +import yaml from unittest import TestCase -from unittest.mock import patch - +import hashlib from grad_tool.test.ut_ms.only_optimizer import main -import yaml + file_path = os.path.abspath(__file__) directory = os.path.dirname(file_path) @@ -17,7 +18,32 @@ def save_dict_as_yaml(data, yaml_file_path): print(f"字典已保存为YAML文件: {yaml_file_path}") +def get_hash(file_path): + with open(file_path, 'rb') as file: + hash_object = hashlib.md5() + for chunk in iter(lambda: file.read(4096), b""): + hash_object.update(chunk) + return hash_object.hexdigest() + + +def check_npy(gradient_output_path): + my_dense_bias_path = os.path.join(gradient_output_path, "rank0", "step_1", "my_dense,bias.npy") + assert os.path.isfile(my_dense_bias_path) + my_dense_bias_real = np.load(my_dense_bias_path) + my_dense_bias_target = np.arange(5).reshape(5) > 0 + + assert (my_dense_bias_real == my_dense_bias_target).all() + + my_dense_weight_path = os.path.join(gradient_output_path, "rank0", "step_1", "my_dense,weight.npy") + assert os.path.isfile(my_dense_weight_path) + my_dense_weight_real = np.load(my_dense_weight_path) + my_dense_weight_target = np.arange(5*16).reshape((5, 16)) > 0 + + assert (my_dense_weight_real == my_dense_weight_target).all() +def check_stat_csv(csv_path, md5_value): + real_md5_value = get_hash(csv_path) + assert real_md5_value == md5_value class TestMsGradientMonitor(TestCase): @@ -28,10 +54,14 @@ class TestMsGradientMonitor(TestCase): "level": "L2", "param_list": None, "rank": None, - "step": [1, 3], + "step": [1], "bounds": None, "output_path": gradient_output_path, } save_dict_as_yaml(config_dict, config_path) - main() \ No newline at end of file + main() + check_npy(gradient_output_path) + check_stat_csv(os.path.join(gradient_output_path, "rank_0", "grad_summary_1.csv"), "b00499bee5c3e3ca96aaee773a092dd7") + os.remove(config_path) + shutil.rmtree(gradient_output_path) \ No newline at end of file -- Gitee From aedb9c36ea9ab3e1d68e1e1c1fb144b216515229 Mon Sep 17 00:00:00 2001 From: qianzhengxin Date: Fri, 23 Aug 2024 18:31:14 +0800 Subject: [PATCH 4/4] bug fix --- .../accuracy_tools/grad_tool/test/ut_ms/test_grad_monitor.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/debug/accuracy_tools/grad_tool/test/ut_ms/test_grad_monitor.py b/debug/accuracy_tools/grad_tool/test/ut_ms/test_grad_monitor.py index 288536df3a..d8bbf75903 100644 --- a/debug/accuracy_tools/grad_tool/test/ut_ms/test_grad_monitor.py +++ b/debug/accuracy_tools/grad_tool/test/ut_ms/test_grad_monitor.py @@ -27,14 +27,14 @@ def get_hash(file_path): def check_npy(gradient_output_path): - my_dense_bias_path = os.path.join(gradient_output_path, "rank0", "step_1", "my_dense,bias.npy") + my_dense_bias_path = os.path.join(gradient_output_path, "rank_0", "step_1", "my_dense.bias.npy") assert os.path.isfile(my_dense_bias_path) my_dense_bias_real = np.load(my_dense_bias_path) my_dense_bias_target = np.arange(5).reshape(5) > 0 assert (my_dense_bias_real == my_dense_bias_target).all() - my_dense_weight_path = os.path.join(gradient_output_path, "rank0", "step_1", "my_dense,weight.npy") + my_dense_weight_path = os.path.join(gradient_output_path, "rank_0", "step_1", "my_dense.weight.npy") assert os.path.isfile(my_dense_weight_path) my_dense_weight_real = np.load(my_dense_weight_path) my_dense_weight_target = np.arange(5*16).reshape((5, 16)) > 0 -- Gitee