From 7149633456c984a08e017c2f83efb5be94dbb532 Mon Sep 17 00:00:00 2001 From: Xiaoda Date: Mon, 27 Apr 2020 16:03:44 +0800 Subject: [PATCH 1/9] update tutorials/source_zh_cn/advanced_use/mixed_precision.md. --- .../advanced_use/mixed_precision.md | 32 +++++++++++++++++-- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/tutorials/source_zh_cn/advanced_use/mixed_precision.md b/tutorials/source_zh_cn/advanced_use/mixed_precision.md index 7fdc240fbc..89e9fb0930 100644 --- a/tutorials/source_zh_cn/advanced_use/mixed_precision.md +++ b/tutorials/source_zh_cn/advanced_use/mixed_precision.md @@ -45,9 +45,20 @@ MindSpore混合精度典型的计算流程如下图所示： 代码样例如下： ```python +import numpy as np +import mindspore.nn as nn +import mindspore.common.dtype as mstype +from mindspore import Tensor, context +from mindspore.ops import operations as P +from mindspore.nn import WithLossCell +from mindspore.nn import Momentum +from mindspore.nn.loss import MSELoss # The interface of Auto_mixed precision from mindspore.train import amp +context.set_context(mode=context.GRAPH_MODE) +context.set_context(device_target="Ascend") + # Define network class LeNet5(nn.Cell): def __init__(self): @@ -58,7 +69,7 @@ class LeNet5(nn.Cell): self.fc2 = nn.Dense(120, 84) self.fc3 = nn.Dense(84, 10) self.relu = nn.ReLU() - self.max_pool2d = nn.MaxPool2d(kernel_size=2) + self.max_pool2d = nn.MaxPool2d(kernel_size=2, stride=2) self.flatten = P.Flatten() def construct(self, x): @@ -103,6 +114,19 @@ MindSpore还支持手动混合精度。假定在网络中只有一个Dense Layer 代码样例如下： ```python +import numpy as np +import mindspore.nn as nn +import mindspore.common.dtype as mstype +from mindspore import Tensor, context +from mindspore.ops import operations as P +from mindspore.nn import WithLossCell, TrainOneStepWithLossScaleCell +from mindspore.nn import Momentum +from mindspore.nn.loss import MSELoss + + +context.set_context(mode=context.GRAPH_MODE) +context.set_context(device_target="Ascend") + # Define network class LeNet5(nn.Cell): def __init__(self): @@ -111,9 +135,9 @@ class LeNet5(nn.Cell): self.conv2 = nn.Conv2d(6, 16, 5, pad_mode='valid') self.fc1 = nn.Dense(16 * 5 * 5, 120) self.fc2 = nn.Dense(120, 84) - self.fc3 = nn.Dense(84, 10).to_float(mstype.float32) + self.fc3 = nn.Dense(84, 10) self.relu = nn.ReLU() - self.max_pool2d = nn.MaxPool2d(kernel_size=2) + self.max_pool2d = nn.MaxPool2d(kernel_size=2, stride=2) self.flatten = P.Flatten() def construct(self, x): @@ -128,6 +152,7 @@ class LeNet5(nn.Cell): # Initialize network and set mixing precision net = LeNet5() net.to_float(mstype.float16) +net.fc3.to_float(mstype.float32) # Define training data, label and sens predict = Tensor(np.ones([1, 1, 32, 32]).astype(np.float32) * 0.01) @@ -140,6 +165,7 @@ loss = MSELoss() optimizer = Momentum(params=net.trainable_params(), learning_rate=0.1, momentum=0.9) net_with_loss = WithLossCell(net, loss) train_network = TrainOneStepWithLossScaleCell(net_with_loss, optimizer) +train_network.set_train() # Run training output = train_network(predict, label, scaling_sens) -- Gitee From e3de99d12cf54d00e97a8ecc2ca881ad2fbc6f22 Mon Sep 17 00:00:00 2001 From: Xiaoda Date: Mon, 27 Apr 2020 17:09:02 +0800 Subject: [PATCH 2/9] update tutorials/source_zh_cn/advanced_use/mixed_precision.md. --- tutorials/source_zh_cn/advanced_use/mixed_precision.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tutorials/source_zh_cn/advanced_use/mixed_precision.md b/tutorials/source_zh_cn/advanced_use/mixed_precision.md index 89e9fb0930..e6988314a3 100644 --- a/tutorials/source_zh_cn/advanced_use/mixed_precision.md +++ b/tutorials/source_zh_cn/advanced_use/mixed_precision.md @@ -105,9 +105,9 @@ output = train_network(predict, label, scaling_sens) MindSpore还支持手动混合精度。假定在网络中只有一个Dense Layer要用FP32计算，其他Layer都用FP16计算。混合精度配置以Cell为粒度，Cell默认是FP32类型。 以下是一个手动混合精度的实现步骤： -1. 定义网络: 该步骤与自动混合精度中的步骤2类似；注意：在LeNet中的fc3算子，需要手动配置成FP32； +1. 定义网络: 该步骤与自动混合精度中的步骤2类似； -2. 配置混合精度: LeNet通过net.to_float(mstype.float16)，把该Cell及其子Cell中所有的算子都配置成FP16； +2. 配置混合精度: LeNet通过net.to_float(mstype.float16)，把该Cell及其子Cell中所有的算子都配置成FP16；然后，将LeNet中的fc3算子手动配置成FP32； 3. 使用TrainOneStepWithLossScaleCell封装网络模型和优化器。 -- Gitee From fada0b07b36f08ab66e545afb39d1aec1cd5490b Mon Sep 17 00:00:00 2001 From: Xiaoda Date: Mon, 27 Apr 2020 17:16:55 +0800 Subject: [PATCH 3/9] update tutorials/source_en/advanced_use/mixed_precision.md. --- .../source_en/advanced_use/mixed_precision.md | 36 ++++++++++++++++--- 1 file changed, 31 insertions(+), 5 deletions(-) diff --git a/tutorials/source_en/advanced_use/mixed_precision.md b/tutorials/source_en/advanced_use/mixed_precision.md index d7b4fb4532..10c87f9ee0 100644 --- a/tutorials/source_en/advanced_use/mixed_precision.md +++ b/tutorials/source_en/advanced_use/mixed_precision.md @@ -46,9 +46,20 @@ The procedure is as follows: A code example is as follows: ```python +import numpy as np +import mindspore.nn as nn +import mindspore.common.dtype as mstype +from mindspore import Tensor, context +from mindspore.ops import operations as P +from mindspore.nn import WithLossCell +from mindspore.nn import Momentum +from mindspore.nn.loss import MSELoss # The interface of Auto_mixed precision from mindspore.train import amp +context.set_context(mode=context.GRAPH_MODE) +context.set_context(device_target="Ascend") + # Define network class LeNet5(nn.Cell): def __init__(self): @@ -59,7 +70,7 @@ class LeNet5(nn.Cell): self.fc2 = nn.Dense(120, 84) self.fc3 = nn.Dense(84, 10) self.relu = nn.ReLU() - self.max_pool2d = nn.MaxPool2d(kernel_size=2) + self.max_pool2d = nn.MaxPool2d(kernel_size=2, stride=2) self.flatten = P.Flatten() def construct(self, x): @@ -95,15 +106,28 @@ output = train_network(predict, label, scaling_sens) MindSpore also supports manual mixed precision. It is assumed that only one dense layer in the network needs to be calculated by using FP32, and other layers are calculated by using FP16. The mixed precision is configured in the granularity of cell. The default format of a cell is FP32. The following is the procedure for implementing manual mixed precision: -1. Define the network. This step is similar to step 2 in the automatic mixed precision. NoteThe fc3 operator in LeNet needs to be manually set to FP32. +1. Define the network. This step is similar to step 2 in the automatic mixed precision. -2. Configure the mixed precision. Use net.to_float(mstype.float16) to set all operators of the cell and its sub-cells to FP16. +2. Configure the mixed precision. Use net.to_float(mstype.float16) to set all operators of the cell and its sub-cells to FP16. Then, configure the fc3 to FP32. 3. Use TrainOneStepWithLossScaleCell to encapsulate the network model and optimizer. A code example is as follows: ```python +import numpy as np +import mindspore.nn as nn +import mindspore.common.dtype as mstype +from mindspore import Tensor, context +from mindspore.ops import operations as P +from mindspore.nn import WithLossCell, TrainOneStepWithLossScaleCell +from mindspore.nn import Momentum +from mindspore.nn.loss import MSELoss + + +context.set_context(mode=context.GRAPH_MODE) +context.set_context(device_target="Ascend") + # Define network class LeNet5(nn.Cell): def __init__(self): @@ -112,9 +136,9 @@ class LeNet5(nn.Cell): self.conv2 = nn.Conv2d(6, 16, 5, pad_mode='valid') self.fc1 = nn.Dense(16 * 5 * 5, 120) self.fc2 = nn.Dense(120, 84) - self.fc3 = nn.Dense(84, 10).to_float(mstype.float32) + self.fc3 = nn.Dense(84, 10) self.relu = nn.ReLU() - self.max_pool2d = nn.MaxPool2d(kernel_size=2) + self.max_pool2d = nn.MaxPool2d(kernel_size=2, stride=2) self.flatten = P.Flatten() def construct(self, x): @@ -129,6 +153,7 @@ class LeNet5(nn.Cell): # Initialize network and set mixing precision net = LeNet5() net.to_float(mstype.float16) +net.fc3.to_float(mstype.float32) # Define training data, label and sens predict = Tensor(np.ones([1, 1, 32, 32]).astype(np.float32) * 0.01) @@ -141,6 +166,7 @@ loss = MSELoss() optimizer = Momentum(params=net.trainable_params(), learning_rate=0.1, momentum=0.9) net_with_loss = WithLossCell(net, loss) train_network = TrainOneStepWithLossScaleCell(net_with_loss, optimizer) +train_network.set_train() # Run training output = train_network(predict, label, scaling_sens) -- Gitee From eb2d9b47c3d1d04e198389252b0deeaa502e0252 Mon Sep 17 00:00:00 2001 From: Xiaoda Date: Mon, 27 Apr 2020 17:19:53 +0800 Subject: [PATCH 4/9] update tutorials/source_zh_cn/advanced_use/mixed_precision.md. --- tutorials/source_zh_cn/advanced_use/mixed_precision.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tutorials/source_zh_cn/advanced_use/mixed_precision.md b/tutorials/source_zh_cn/advanced_use/mixed_precision.md index e6988314a3..992b3910f1 100644 --- a/tutorials/source_zh_cn/advanced_use/mixed_precision.md +++ b/tutorials/source_zh_cn/advanced_use/mixed_precision.md @@ -12,7 +12,7 @@ ## 概述 -混合精度训练方法是通过混合使用单精度和半精度数据格式来加速深度神经网络训练的过程，同时保持了单精度训练所能达到的网络精度。混合精度训练能够加速计算过程，同时减少内存使用和存取，并使得在特定的硬件上可以训练更大的模型或batch size。 +混合精度训练方法是通过混合使用单精度和半精度数据格式来加速深度神经网络训练的过程，同时保持了单精度训练所能达到的网络精度。混合精度训练能够加速计算过程，同时减少内存使用和存取，并使得在特定的硬件上可以训练更大的模型或batch size。对于FP16的算子，若给定的数据类型是FP32的，MindSpore框架的后端则会进行降精度处理。 ## 计算流程 -- Gitee From ea6eb02d2d4525461e2a956f8229fd0c24bf7991 Mon Sep 17 00:00:00 2001 From: Xiaoda Date: Mon, 27 Apr 2020 20:18:47 +0800 Subject: [PATCH 5/9] update tutorials/source_zh_cn/advanced_use/mixed_precision.md. --- tutorials/source_zh_cn/advanced_use/mixed_precision.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tutorials/source_zh_cn/advanced_use/mixed_precision.md b/tutorials/source_zh_cn/advanced_use/mixed_precision.md index 992b3910f1..cb86208082 100644 --- a/tutorials/source_zh_cn/advanced_use/mixed_precision.md +++ b/tutorials/source_zh_cn/advanced_use/mixed_precision.md @@ -12,7 +12,9 @@ ## 概述 -混合精度训练方法是通过混合使用单精度和半精度数据格式来加速深度神经网络训练的过程，同时保持了单精度训练所能达到的网络精度。混合精度训练能够加速计算过程，同时减少内存使用和存取，并使得在特定的硬件上可以训练更大的模型或batch size。对于FP16的算子，若给定的数据类型是FP32的，MindSpore框架的后端则会进行降精度处理。 +混合精度训练方法是通过混合使用单精度和半精度数据格式来加速深度神经网络训练的过程，同时保持了单精度训练所能达到的网络精度。混合精度训练能够加速计算过程，同时减少内存使用和存取，并使得在特定的硬件上可以训练更大的模型或batch size。 + +对于FP16的算子，若给定的数据类型是FP32，MindSpore框架的后端则会进行降精度处理。 ## 计算流程 -- Gitee From 45755f3d34edfaffd7f7dc5fcf4d5f038b2af002 Mon Sep 17 00:00:00 2001 From: Xiaoda Date: Mon, 27 Apr 2020 20:19:25 +0800 Subject: [PATCH 6/9] update tutorials/source_zh_cn/advanced_use/mixed_precision.md. --- tutorials/source_zh_cn/advanced_use/mixed_precision.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tutorials/source_zh_cn/advanced_use/mixed_precision.md b/tutorials/source_zh_cn/advanced_use/mixed_precision.md index cb86208082..0fedaef609 100644 --- a/tutorials/source_zh_cn/advanced_use/mixed_precision.md +++ b/tutorials/source_zh_cn/advanced_use/mixed_precision.md @@ -14,7 +14,7 @@ 混合精度训练方法是通过混合使用单精度和半精度数据格式来加速深度神经网络训练的过程，同时保持了单精度训练所能达到的网络精度。混合精度训练能够加速计算过程，同时减少内存使用和存取，并使得在特定的硬件上可以训练更大的模型或batch size。 -对于FP16的算子，若给定的数据类型是FP32，MindSpore框架的后端则会进行降精度处理。 +对于FP16的算子，若给定的数据类型是FP32，MindSpore框架的后端则会进行降精度处理。用户可以开启INFO日志，并通过搜索关键字“reduce precision”查看降精度处理的算子。 ## 计算流程 -- Gitee From 7f7c2b5539174dd35ebc1b56b9dcea6470136ba9 Mon Sep 17 00:00:00 2001 From: Xiaoda Date: Mon, 27 Apr 2020 20:25:37 +0800 Subject: [PATCH 7/9] update tutorials/source_en/advanced_use/mixed_precision.md. --- tutorials/source_en/advanced_use/mixed_precision.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tutorials/source_en/advanced_use/mixed_precision.md b/tutorials/source_en/advanced_use/mixed_precision.md index 10c87f9ee0..b4f7531bd1 100644 --- a/tutorials/source_en/advanced_use/mixed_precision.md +++ b/tutorials/source_en/advanced_use/mixed_precision.md @@ -15,6 +15,8 @@ The mixed precision training method accelerates the deep learning neural network training process by using both the single-precision and half-precision data formats, and maintains the network precision achieved by the single-precision training at the same time. Mixed precision training can accelerate the computation process, reduce memory usage, and enable a larger model or batch size to be trained on specific hardware. +For FP16 operators, if the input data type is FP32, the backend of MindSpore will automatically handle it with reduced precision. Users could check the reduced-precision operators by enabling INFO log and then searching `reduce precision'. + ## Computation Process The following figure shows the typical computation process of mixed precision in MindSpore. -- Gitee From ae194a5fbbd2893b36f1a86d73e1cd51dd7c21a4 Mon Sep 17 00:00:00 2001 From: Xiaoda Date: Mon, 27 Apr 2020 20:49:49 +0800 Subject: [PATCH 8/9] update tutorials/source_en/advanced_use/mixed_precision.md. --- tutorials/source_en/advanced_use/mixed_precision.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tutorials/source_en/advanced_use/mixed_precision.md b/tutorials/source_en/advanced_use/mixed_precision.md index b4f7531bd1..51963dbd9a 100644 --- a/tutorials/source_en/advanced_use/mixed_precision.md +++ b/tutorials/source_en/advanced_use/mixed_precision.md @@ -15,7 +15,7 @@ The mixed precision training method accelerates the deep learning neural network training process by using both the single-precision and half-precision data formats, and maintains the network precision achieved by the single-precision training at the same time. Mixed precision training can accelerate the computation process, reduce memory usage, and enable a larger model or batch size to be trained on specific hardware. -For FP16 operators, if the input data type is FP32, the backend of MindSpore will automatically handle it with reduced precision. Users could check the reduced-precision operators by enabling INFO log and then searching `reduce precision'. +For FP16 operators, if the input data type is FP32, the backend of MindSpore will automatically handle it with reduced precision. Users could check the reduced-precision operators by enabling INFO log and then searching 'reduce precision'. ## Computation Process -- Gitee From 6bf02fb78feadc722e4a1c5c32436607035aa2ef Mon Sep 17 00:00:00 2001 From: Xiaoda Date: Mon, 27 Apr 2020 20:50:43 +0800 Subject: [PATCH 9/9] update tutorials/source_zh_cn/advanced_use/mixed_precision.md. --- tutorials/source_zh_cn/advanced_use/mixed_precision.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tutorials/source_zh_cn/advanced_use/mixed_precision.md b/tutorials/source_zh_cn/advanced_use/mixed_precision.md index 0fedaef609..afe38b53ee 100644 --- a/tutorials/source_zh_cn/advanced_use/mixed_precision.md +++ b/tutorials/source_zh_cn/advanced_use/mixed_precision.md @@ -14,7 +14,7 @@ 混合精度训练方法是通过混合使用单精度和半精度数据格式来加速深度神经网络训练的过程，同时保持了单精度训练所能达到的网络精度。混合精度训练能够加速计算过程，同时减少内存使用和存取，并使得在特定的硬件上可以训练更大的模型或batch size。 -对于FP16的算子，若给定的数据类型是FP32，MindSpore框架的后端则会进行降精度处理。用户可以开启INFO日志，并通过搜索关键字“reduce precision”查看降精度处理的算子。 +对于FP16的算子，若给定的数据类型是FP32，MindSpore框架的后端会进行降精度处理。用户可以开启INFO日志，并通过搜索关键字“reduce precision”查看降精度处理的算子。 ## 计算流程 -- Gitee