From 68fa587956fcc872c4be9966397a030570439bbe Mon Sep 17 00:00:00 2001 From: xiaobulv <3026463191@qq.com> Date: Fri, 19 Feb 2021 10:27:50 +0800 Subject: [PATCH] Code extraction from debugging in pynative mode --- .../debug_in_pynative_mode/README_CN.md | 33 +++++ .../debug_in_pynative_mode.py | 112 +++++++++++++++++ .../debug_in_pynative_mode_with_ms_func.py | 116 ++++++++++++++++++ 3 files changed, 261 insertions(+) create mode 100644 tutorials/tutorial_code/debug_in_pynative_mode/README_CN.md create mode 100644 tutorials/tutorial_code/debug_in_pynative_mode/debug_in_pynative_mode.py create mode 100644 tutorials/tutorial_code/debug_in_pynative_mode/debug_in_pynative_mode_with_ms_func.py diff --git a/tutorials/tutorial_code/debug_in_pynative_mode/README_CN.md b/tutorials/tutorial_code/debug_in_pynative_mode/README_CN.md new file mode 100644 index 0000000000..2fb06744eb --- /dev/null +++ b/tutorials/tutorial_code/debug_in_pynative_mode/README_CN.md @@ -0,0 +1,33 @@ +# 代码运行说明 + +## 安装版本 + +MindSpore 1.1.1 GPU版本 + +## 数据集下载 + +训练数据集：http://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz + +训练数据集的标签：http://yann.lecun.com/exdb/mnist/train-labels-idx1-ubyte.gz + +## 数据集放置 + +```text +└─MNIST_Data + │ + └─train + train-images.idx3-ubyte + train-labels.idx1-ubyte +``` + +## 运行方式 + +```bash +python debug_in_pynative_mode.py +``` + +或者 + +```bash +python debug_in_pynative_mode_with_ms_func.py +``` \ No newline at end of file diff --git a/tutorials/tutorial_code/debug_in_pynative_mode/debug_in_pynative_mode.py b/tutorials/tutorial_code/debug_in_pynative_mode/debug_in_pynative_mode.py new file mode 100644 index 0000000000..5601be9ef7 --- /dev/null +++ b/tutorials/tutorial_code/debug_in_pynative_mode/debug_in_pynative_mode.py @@ -0,0 +1,112 @@ +"""debug_in_pynative_mode +run in GPU +""" +import numpy as np +import mindspore.nn as nn +import mindspore.ops as ops +from mindspore import dtype as mstype +from mindspore import context, Tensor, ParameterTuple +from mindspore.common.initializer import Normal +from mindspore.nn import Dense, WithLossCell, SoftmaxCrossEntropyWithLogits, Momentum +import mindspore.dataset as ds +import mindspore.dataset.transforms.c_transforms as C +import mindspore.dataset.vision.c_transforms as CV +from mindspore.dataset.vision import Inter +from mindspore import dtype as mstype + +context.set_context(mode=context.PYNATIVE_MODE, device_target="GPU") + +def create_dataset(data_path, batch_size=32, repeat_size=1, + num_parallel_workers=8): + """ create dataset for train or test + Args: + data_path: Data path + batch_size: The number of data records in each group + repeat_size: The number of replicated data records + num_parallel_workers: The number of parallel workers + """ + # define dataset + mnist_ds = ds.MnistDataset(data_path) + + # define map operations + trans_c = [ + CV.Resize((32, 32), interpolation=Inter.LINEAR), # 双线性插值法将图片上采样为32*32 + CV.Rescale(1 / 0.3081, -1 * 0.1307 / 0.3081), # 数据标准化 + CV.Rescale(1.0 / 255.0, 0.0), # 数据归一化 + CV.HWC2CHW() # 通道切换 + + ] + type_cast_op = C.TypeCast(mstype.int32) + + # apply map operations on images + mnist_ds = mnist_ds.map(operations=type_cast_op, input_columns="label", num_parallel_workers=num_parallel_workers) + mnist_ds = mnist_ds.map(operations=trans_c, input_columns="image", num_parallel_workers=num_parallel_workers) + + # apply DatasetOps + buffer_size = 10000 # 10000张图片的混洗空间 + mnist_ds = mnist_ds.shuffle(buffer_size=buffer_size) # 图片混洗 + mnist_ds = mnist_ds.batch(batch_size, drop_remainder=True) # 图片成组，同时设置最后一组batch数量不足时丢弃。 + mnist_ds = mnist_ds.repeat(repeat_size) # batch的复制增强 + + return mnist_ds + +class LeNet5(nn.Cell): + """ + Lenet network structure + """ + #define the operator required + def __init__(self, num_class=10, num_channel=1): + super(LeNet5, self).__init__() + self.conv1 = nn.Conv2d(num_channel, 6, 5, pad_mode='valid') + self.conv2 = nn.Conv2d(6, 16, 5, pad_mode='valid') + self.fc1 = nn.Dense(16 * 5 * 5, 120, weight_init=Normal(0.02)) + self.fc2 = nn.Dense(120, 84, weight_init=Normal(0.02)) + self.fc3 = nn.Dense(84, num_class, weight_init=Normal(0.02)) + self.relu = nn.ReLU() + self.max_pool2d = nn.MaxPool2d(kernel_size=2, stride=2) + self.flatten = nn.Flatten() + + #use the preceding operators to construct networks + def construct(self, x): + x = self.max_pool2d(self.relu(self.conv1(x))) + x = self.max_pool2d(self.relu(self.conv2(x))) + x = self.flatten(x) + x = self.relu(self.fc1(x)) + x = self.relu(self.fc2(x)) + x = self.fc3(x) + return x + +class GradWrap(nn.Cell): + """ GradWrap definition """ + def __init__(self, network): + super(GradWrap, self).__init__(auto_prefix=False) + self.network = network + self.weights = ParameterTuple(filter(lambda x: x.requires_grad, network.get_parameters())) # 从网络参数中过滤出可训练权重 + + def construct(self, x, label): + weights = self.weights + return ops.GradOperation(get_by_list=True)(self.network, weights)(x, label) # 对network中的可训练权重求梯度 + +net = LeNet5() +optimizer = Momentum(filter(lambda x: x.requires_grad, net.get_parameters()), 0.01, 0.9) +criterion = nn.SoftmaxCrossEntropyWithLogits(sparse=True, reduction='mean') +net_with_criterion = WithLossCell(net, criterion) +train_network = GradWrap(net_with_criterion) +train_network.set_train() + + +ds_train = create_dataset("./MNIST_Data/train/") +dict_data = ds_train.create_dict_iterator() +for step, batch_data in enumerate(dict_data): + input_data = batch_data["image"] + label = batch_data["label"] + # 特征网络输出 + output = net(Tensor(input_data)) + # 计算模型损失值 + loss_output = criterion(output, label) + # 计算当前模型中各权重梯度值 + grads = train_network(input_data, label) + # 将梯度值更新到模型中的权重 + success = optimizer(grads) + loss = loss_output.asnumpy() + print("step:", step+1, "loss value:", loss) \ No newline at end of file diff --git a/tutorials/tutorial_code/debug_in_pynative_mode/debug_in_pynative_mode_with_ms_func.py b/tutorials/tutorial_code/debug_in_pynative_mode/debug_in_pynative_mode_with_ms_func.py new file mode 100644 index 0000000000..90c6a4a336 --- /dev/null +++ b/tutorials/tutorial_code/debug_in_pynative_mode/debug_in_pynative_mode_with_ms_func.py @@ -0,0 +1,116 @@ + +"""debug_in_pynative_mode_with_ms_func +run in GPU +""" +import numpy as np +import mindspore.nn as nn +import mindspore.ops as ops +from mindspore import dtype as mstype +from mindspore import context, Tensor, ParameterTuple +from mindspore.common.initializer import Normal +from mindspore.nn import Dense, WithLossCell, SoftmaxCrossEntropyWithLogits, Momentum +import mindspore.dataset as ds +import mindspore.dataset.transforms.c_transforms as C +import mindspore.dataset.vision.c_transforms as CV +from mindspore.dataset.vision import Inter +from mindspore import dtype as mstype +from mindspore import ms_function + +context.set_context(mode=context.PYNATIVE_MODE, device_target="GPU") + +def create_dataset(data_path, batch_size=32, repeat_size=1, + num_parallel_workers=8): + """ create dataset for train or test + Args: + data_path: Data path + batch_size: The number of data records in each group + repeat_size: The number of replicated data records + num_parallel_workers: The number of parallel workers + """ + # define dataset + mnist_ds = ds.MnistDataset(data_path) + + # define map operations + trans_c = [ + CV.Resize((32, 32), interpolation=Inter.LINEAR), # 双线性插值法将图片上采样为32*32 + CV.Rescale(1 / 0.3081, -1 * 0.1307 / 0.3081), # 数据标准化 + CV.Rescale(1.0 / 255.0, 0.0), # 数据归一化 + CV.HWC2CHW() # 通道切换 + + ] + type_cast_op = C.TypeCast(mstype.int32) + + # apply map operations on images + mnist_ds = mnist_ds.map(operations=type_cast_op, input_columns="label", num_parallel_workers=num_parallel_workers) + mnist_ds = mnist_ds.map(operations=trans_c, input_columns="image", num_parallel_workers=num_parallel_workers) + + # apply DatasetOps + buffer_size = 10000 # 10000张图片的混洗空间 + mnist_ds = mnist_ds.shuffle(buffer_size=buffer_size) # 图片混洗 + mnist_ds = mnist_ds.batch(batch_size, drop_remainder=True) # 图片成组，同时设置最后一组batch数量不足时丢弃。 + mnist_ds = mnist_ds.repeat(repeat_size) # batch的复制增强 + + return mnist_ds + +class LeNet5(nn.Cell): + """ + Lenet network structure + """ + #define the operator required + def __init__(self, num_class=10, num_channel=1): + super(LeNet5, self).__init__() + self.conv1 = nn.Conv2d(num_channel, 6, 5, pad_mode='valid') + self.conv2 = nn.Conv2d(6, 16, 5, pad_mode='valid') + self.fc1 = nn.Dense(16 * 5 * 5, 120, weight_init=Normal(0.02)) + self.fc2 = nn.Dense(120, 84, weight_init=Normal(0.02)) + self.fc3 = nn.Dense(84, num_class, weight_init=Normal(0.02)) + self.relu = nn.ReLU() + self.max_pool2d = nn.MaxPool2d(kernel_size=2, stride=2) + self.flatten = nn.Flatten() + + #use the preceding operators to construct networks + @ms_function + def construct(self, x): + x = self.max_pool2d(self.relu(self.conv1(x))) + x = self.max_pool2d(self.relu(self.conv2(x))) + x = self.flatten(x) + x = self.relu(self.fc1(x)) + x = self.relu(self.fc2(x)) + x = self.fc3(x) + return x + +class GradWrap(nn.Cell): + """ GradWrap definition """ + def __init__(self, network): + super(GradWrap, self).__init__(auto_prefix=False) + self.network = network + self.weights = ParameterTuple(filter(lambda x: x.requires_grad, network.get_parameters())) # 从网络参数中过滤出可训练权重 + + @ms_function + def construct(self, x, label): + weights = self.weights + return ops.GradOperation(get_by_list=True)(self.network, weights)(x, label) # 对network中的可训练权重求梯度 + +net = LeNet5() +optimizer = Momentum(filter(lambda x: x.requires_grad, net.get_parameters()), 0.01, 0.9) +criterion = nn.SoftmaxCrossEntropyWithLogits(sparse=True, reduction='mean') +net_with_criterion = WithLossCell(net, criterion) +train_network = GradWrap(net_with_criterion) +train_network.set_train() + + +ds_train = create_dataset("./MNIST_Data/train/") +dict_data = ds_train.create_dict_iterator() +for step, batch_data in enumerate(dict_data): + input_data = batch_data["image"] + label = batch_data["label"] + # 特征网络输出 + output = net(Tensor(input_data)) + # 计算模型损失值 + loss_output = criterion(output, label) + # 计算当前模型中各权重梯度值 + grads = train_network(input_data, label) + # 将梯度值更新到模型中的权重 + success = optimizer(grads) + loss = loss_output.asnumpy() + print("step:", step+1, "loss value:", loss) \ No newline at end of file -- Gitee