diff --git a/docs/lite/api/source_en/conf.py b/docs/lite/api/source_en/conf.py index b9ccde4eb64ac1cc90b36b51651e35cce8b60c49..9c11fa8f2990cdc93d41337d49680b5e6ea0731c 100644 --- a/docs/lite/api/source_en/conf.py +++ b/docs/lite/api/source_en/conf.py @@ -310,7 +310,7 @@ try: ("mindspore_lite/converter.py","del decorator", "@set_env","# generate api by del decorator set_env."), ("mindspore_lite/llm_engine.py","del decorator", - "@set_env","# generate api by del decorator set_env.")] + "@set_env","# generate api by del decorator set_env.")] for i in decorator_list: with open(os.path.join(base_path, os.path.normpath(i[0])), "r+", encoding="utf8") as f: diff --git a/docs/mindspore/api/api_cn/API_sample_and_requirements.md b/docs/mindspore/api/api_cn/API_sample_and_requirements.md index d1ccc6f405f930f3de46448f3b6ce3530b57ab31..dd0995371d6ab7a61b1fd3bcdd006ae25ef17a10 100644 --- a/docs/mindspore/api/api_cn/API_sample_and_requirements.md +++ b/docs/mindspore/api/api_cn/API_sample_and_requirements.md @@ -269,8 +269,8 @@ - **Error2** – 异常描述2。 教程样例: - - `自动混合精度 - 损失缩放 - `_ + - `Graph Mode加速 + `_ ``` @@ -283,8 +283,8 @@ xxxx 教程样例: - - `自动混合精度 - 损失缩放 - `_ + - `Graph Mode加速 + `_ ``` diff --git a/tutorials/source_en/beginner/accelerate_with_static_graph.md b/tutorials/source_en/beginner/accelerate_with_static_graph.md index 3315bb899283e060a97dc04b69d8e73047cef457..cecbf71c56e9f5df5684819515f5cf308120d695 100644 --- a/tutorials/source_en/beginner/accelerate_with_static_graph.md +++ b/tutorials/source_en/beginner/accelerate_with_static_graph.md @@ -1,6 +1,6 @@ [![View Source On Gitee](https://mindspore-website.obs.cn-north-4.myhuaweicloud.com/website-images/master/resource/_static/logo_source_en.svg)](https://gitee.com/mindspore/docs/blob/master/tutorials/source_en/beginner/accelerate_with_static_graph.md) -[Introduction](https://www.mindspore.cn/tutorials/en/master/beginner/introduction.html) || [Quick Start](https://www.mindspore.cn/tutorials/en/master/beginner/quick_start.html) || [Tensor](https://www.mindspore.cn/tutorials/en/master/beginner/tensor.html) || [Data Loading and Processing](https://www.mindspore.cn/tutorials/en/master/beginner/dataset.html) || [Model](https://www.mindspore.cn/tutorials/en/master/beginner/model.html) || [Autograd](https://gitee.com/mindspore/docs/blob/master/tutorials/source_en/beginner/autograd.md) || [Train](https://www.mindspore.cn/tutorials/en/master/beginner/train.html) || [Save and Load](https://www.mindspore.cn/tutorials/en/master/beginner/save_load.html) || **Accelerating with Static Graphs**|| [Mixed Precision](https://www.mindspore.cn/tutorials/en/master/beginner/mixed_precision.html) +[Introduction](https://www.mindspore.cn/tutorials/en/master/beginner/introduction.html) || [Quick Start](https://www.mindspore.cn/tutorials/en/master/beginner/quick_start.html) || [Tensor](https://www.mindspore.cn/tutorials/en/master/beginner/tensor.html) || [Data Loading and Processing](https://www.mindspore.cn/tutorials/en/master/beginner/dataset.html) || [Model](https://www.mindspore.cn/tutorials/en/master/beginner/model.html) || [Autograd](https://gitee.com/mindspore/docs/blob/master/tutorials/source_en/beginner/autograd.md) || [Train](https://www.mindspore.cn/tutorials/en/master/beginner/train.html) || [Save and Load](https://www.mindspore.cn/tutorials/en/master/beginner/save_load.html) || **Accelerating with Static Graphs** # Accelerating with Static Graphs diff --git a/tutorials/source_en/beginner/autograd.md b/tutorials/source_en/beginner/autograd.md index 44758b62996b6231e96652b1dc84f297dd7a0510..1f5589fc070916d9080c0facdeacf90c77572611 100644 --- a/tutorials/source_en/beginner/autograd.md +++ b/tutorials/source_en/beginner/autograd.md @@ -1,6 +1,6 @@ [![View Source On Gitee](https://mindspore-website.obs.cn-north-4.myhuaweicloud.com/website-images/master/resource/_static/logo_source_en.svg)](https://gitee.com/mindspore/docs/blob/master/tutorials/source_en/beginner/autograd.md) -[Introduction](https://www.mindspore.cn/tutorials/en/master/beginner/introduction.html) || [Quick Start](https://www.mindspore.cn/tutorials/en/master/beginner/quick_start.html) || [Tensor](https://www.mindspore.cn/tutorials/en/master/beginner/tensor.html) || [Data Loading and Processing](https://www.mindspore.cn/tutorials/en/master/beginner/dataset.html) || [Model](https://www.mindspore.cn/tutorials/en/master/beginner/model.html) || **Autograd** || [Train](https://www.mindspore.cn/tutorials/en/master/beginner/train.html) || [Save and Load](https://www.mindspore.cn/tutorials/en/master/beginner/save_load.html) || [Accelerating with Static Graphs](https://www.mindspore.cn/tutorials/en/master/beginner/accelerate_with_static_graph.html) || [Mixed Precision](https://www.mindspore.cn/tutorials/en/master/beginner/mixed_precision.html) +[Introduction](https://www.mindspore.cn/tutorials/en/master/beginner/introduction.html) || [Quick Start](https://www.mindspore.cn/tutorials/en/master/beginner/quick_start.html) || [Tensor](https://www.mindspore.cn/tutorials/en/master/beginner/tensor.html) || [Data Loading and Processing](https://www.mindspore.cn/tutorials/en/master/beginner/dataset.html) || [Model](https://www.mindspore.cn/tutorials/en/master/beginner/model.html) || **Autograd** || [Train](https://www.mindspore.cn/tutorials/en/master/beginner/train.html) || [Save and Load](https://www.mindspore.cn/tutorials/en/master/beginner/save_load.html) || [Accelerating with Static Graphs](https://www.mindspore.cn/tutorials/en/master/beginner/accelerate_with_static_graph.html) # Automatic Differentiation diff --git a/tutorials/source_en/beginner/dataset.md b/tutorials/source_en/beginner/dataset.md index 2388cb71deda88136e54dcf744b9ab5e735b2052..bb36f2c8da1f96023152d226e5ba9a894ec1f561 100644 --- a/tutorials/source_en/beginner/dataset.md +++ b/tutorials/source_en/beginner/dataset.md @@ -1,6 +1,6 @@ [![View Source On Gitee](https://mindspore-website.obs.cn-north-4.myhuaweicloud.com/website-images/master/resource/_static/logo_source_en.svg)](https://gitee.com/mindspore/docs/blob/master/tutorials/source_en/beginner/dataset.md) -[Introduction](https://www.mindspore.cn/tutorials/en/master/beginner/introduction.html) || [Quick Start](https://www.mindspore.cn/tutorials/en/master/beginner/quick_start.html) || [Tensor](https://www.mindspore.cn/tutorials/en/master/beginner/tensor.html) || **Data Loading and Processing** || [Model](https://www.mindspore.cn/tutorials/en/master/beginner/model.html) || [Autograd](https://www.mindspore.cn/tutorials/en/master/beginner/autograd.html) || [Train](https://www.mindspore.cn/tutorials/en/master/beginner/train.html) || [Save and Load](https://www.mindspore.cn/tutorials/en/master/beginner/save_load.html) || [Accelerating with Static Graphs](https://www.mindspore.cn/tutorials/en/master/beginner/accelerate_with_static_graph.html) || [Mixed Precision](https://www.mindspore.cn/tutorials/en/master/beginner/mixed_precision.html) +[Introduction](https://www.mindspore.cn/tutorials/en/master/beginner/introduction.html) || [Quick Start](https://www.mindspore.cn/tutorials/en/master/beginner/quick_start.html) || [Tensor](https://www.mindspore.cn/tutorials/en/master/beginner/tensor.html) || **Data Loading and Processing** || [Model](https://www.mindspore.cn/tutorials/en/master/beginner/model.html) || [Autograd](https://www.mindspore.cn/tutorials/en/master/beginner/autograd.html) || [Train](https://www.mindspore.cn/tutorials/en/master/beginner/train.html) || [Save and Load](https://www.mindspore.cn/tutorials/en/master/beginner/save_load.html) || [Accelerating with Static Graphs](https://www.mindspore.cn/tutorials/en/master/beginner/accelerate_with_static_graph.html) # Data Loading and Processing diff --git a/tutorials/source_en/beginner/images/fp16_vs_fp32.png b/tutorials/source_en/beginner/images/fp16_vs_fp32.png deleted file mode 100644 index 83f965ddfa7b2bfed024e774d554f8fe15e6ab3f..0000000000000000000000000000000000000000 Binary files a/tutorials/source_en/beginner/images/fp16_vs_fp32.png and /dev/null differ diff --git a/tutorials/source_en/beginner/introduction.md b/tutorials/source_en/beginner/introduction.md index 50885a09f28e2a1a9dcc6ad1e7efbe7573dc504c..745fedf41897cac53826d74fe64d956074403e05 100644 --- a/tutorials/source_en/beginner/introduction.md +++ b/tutorials/source_en/beginner/introduction.md @@ -1,6 +1,6 @@ [![View Source On Gitee](https://mindspore-website.obs.cn-north-4.myhuaweicloud.com/website-images/master/resource/_static/logo_source_en.svg)](https://gitee.com/mindspore/docs/blob/master/tutorials/source_en/beginner/introduction.md) -**Introduction** || [Quick Start](https://www.mindspore.cn/tutorials/en/master/beginner/quick_start.html#) || [Tensor](https://www.mindspore.cn/tutorials/en/master/beginner/tensor.html) || [Data Loading and Processing](https://www.mindspore.cn/tutorials/en/master/beginner/dataset.html) || [Model](https://www.mindspore.cn/tutorials/en/master/beginner/model.html) || [Autograd](https://www.mindspore.cn/tutorials/en/master/beginner/autograd.html) || [Train](https://www.mindspore.cn/tutorials/en/master/beginner/train.html) || [Save and Load](https://www.mindspore.cn/tutorials/en/master/beginner/save_load.html) || [Accelerating with Static Graphs](https://www.mindspore.cn/tutorials/en/master/beginner/accelerate_with_static_graph.html) || [Mixed Precision](https://www.mindspore.cn/tutorials/en/master/beginner/mixed_precision.html) +**Introduction** || [Quick Start](https://www.mindspore.cn/tutorials/en/master/beginner/quick_start.html#) || [Tensor](https://www.mindspore.cn/tutorials/en/master/beginner/tensor.html) || [Data Loading and Processing](https://www.mindspore.cn/tutorials/en/master/beginner/dataset.html) || [Model](https://www.mindspore.cn/tutorials/en/master/beginner/model.html) || [Autograd](https://www.mindspore.cn/tutorials/en/master/beginner/autograd.html) || [Train](https://www.mindspore.cn/tutorials/en/master/beginner/train.html) || [Save and Load](https://www.mindspore.cn/tutorials/en/master/beginner/save_load.html) || [Accelerating with Static Graphs](https://www.mindspore.cn/tutorials/en/master/beginner/accelerate_with_static_graph.html) # Overview diff --git a/tutorials/source_en/beginner/mixed_precision.md b/tutorials/source_en/beginner/mixed_precision.md deleted file mode 100644 index fd9b3f85abda9d3c3e6dcc3d90d6d65a1c5f843e..0000000000000000000000000000000000000000 --- a/tutorials/source_en/beginner/mixed_precision.md +++ /dev/null @@ -1,321 +0,0 @@ -[![View Source On Gitee](https://mindspore-website.obs.cn-north-4.myhuaweicloud.com/website-images/master/resource/_static/logo_source_en.svg)](https://gitee.com/mindspore/docs/blob/master/tutorials/source_en/beginner/mixed_precision.md) - -[Introduction](https://www.mindspore.cn/tutorials/en/master/beginner/introduction.html) || [Quick Start](https://www.mindspore.cn/tutorials/en/master/beginner/quick_start.html#) || [Tensor](https://www.mindspore.cn/tutorials/en/master/beginner/tensor.html) || [Data Loading and Processing](https://www.mindspore.cn/tutorials/en/master/beginner/dataset.html) || [Model](https://www.mindspore.cn/tutorials/en/master/beginner/model.html) || [Autograd](https://www.mindspore.cn/tutorials/en/master/beginner/autograd.html) || [Train](https://www.mindspore.cn/tutorials/en/master/beginner/train.html) || [Save and Load](https://www.mindspore.cn/tutorials/en/master/beginner/save_load.html) || [Accelerating with Static Graphs](https://www.mindspore.cn/tutorials/en/master/beginner/accelerate_with_static_graph.html)|| **Mixed Precision** - -# Automatic Mixed Precision - -Mixed precision training is a computing strategy that uses different numerical precision for different operations of the neural network during training. In neural network operations, some operations are not sensitive to numerical precision, and using lower precision can achieve significant acceleration (such as conv, matmul), while some of the operations usually need to retain high precision to ensure the correctness of the results due to the large difference between the input and output values (such as log, softmax). - -The hardware acceleration modules are usually designed on current AI accelerator cards for targeting computationally intensive, precision-insensitive operations, such as TensorCore for NVIDIA GPUs and Cube for Ascend NPU. For neural networks with a larger share of operations, such as conv, matmul, their training speed usually has a larger acceleration ratio. - -The `mindspore.amp` module provides a convenient interface for automatic mixed precision, allowing users to obtain training acceleration at different hardware backends with simple interface calls. In the following, we introduce the calculation principle of mixed precision, and then introduce the automatic mixed precision usage of MindSpore by example. - -## Principle of Mixed Precision Calculation - -Floating-point data types include double-precision (FP64), single-precision (FP32), and half-precision (FP16). In a training process of a neural network model, an FP32 data type is generally used by default to indicate a network model weight and other parameters. The following is a brief introduction to floating-point data types. - -According to [IEEE 754](https://en.wikipedia.org/wiki/IEEE_754), floating-point data types are classified into double-precision (FP64), single-precision (FP32), and half-precision (FP16). Each type is represented by three different bits. FP64 indicates a data type that uses 8 bytes (64 bits in total) for encoding and storage. FP32 indicates a data type that uses 4 bytes (32 bits in total) and FP16 indicates a data type that uses 2 bytes (16 bits in total). As shown in the following figure: - -![fp16_vs_FP32](https://mindspore-website.obs.cn-north-4.myhuaweicloud.com/website-images/master/tutorials/source_en/beginner/images/fp16_vs_fp32.png) - -As shown in the figure, the storage space of FP16 is half that of FP32. Similarly, the storage space of FP32 is half that of FP64. Therefore, using FP16 for computing has the following advantages: - -- Reduce memory usage: The bit width of FP16 is half that of FP32, so the memory used for parameters such as weights is also half of the original, saving memory for larger network models or training with more data. -- Higher computational efficiency: On special AI-accelerated chips such as Huawei Atlas training series and Atlas 200/300/500 inference product series, or GPUs on NVIDIA VOLTA architecture, execution performance is faster using FP16 than FP32. -- Accelerate communication efficiency: For distributed training, especially in the process of training large models, the communication overhead constrains the overall performance of network model training. Less bit-width of communication means that communication performance can be improved, waiting time can be reduced, and the flow of data can be accelerated. - -But the use of FP16 also poses a number of problems: - -- Data overflow: The valid data representation range for FP16 is $[5.9\times10^{-8}, 65504]$ and for FP32 is $[1.4\times10^{-45}, 1.7\times10^{38}]$. It can be seen that the effective range of FP16 is much narrower than that of FP32, and using FP16 to replace FP32 will result in overflow and underflow. In deep learning, the gradient (first-order derivative) of the weights in the network model needs to be calculated, so the gradient will be even smaller than the weight value and often prone to underflow. -- Rounding error: Rounding Error is when the backward gradient of the network model is small, which is generally represented by FP32. But the conversion to FP16 will be smaller than the minimum interval in the current interval and will lead to data overflow. If `0.00006666666` can be expressed normally in FP32, it will be expressed as `0.000067` after conversion to FP16, and the numbers that do not meet the minimum interval of FP16 will be forced to be rounded. - -Therefore, the solution of the FP16 introduction problem needs to be considered while using mixed precision to obtain training speedup and memory savings. Loss Scale, a solution to the FP16 type data overflow problem, expands the loss by a certain number of times when calculating the loss value loss. According to the chain rule, the gradient is expanded accordingly and then scaled down by a corresponding multiple when the optimizer updates the weights, thus avoiding data underflow. - -Based on the principles described above, a typical mixed precision computation process is shown in the following figure: - -![mix precision](https://mindspore-website.obs.cn-north-4.myhuaweicloud.com/website-images/master/tutorials/source_zh_cn/beginner/images/mix_precision_fp16.png) - -1. Parameters stored in FP32. -2. During forward computation, when it comes to FP16 operators, the operator inputs and parameters need to be cast from FP32 to FP16 for computation. -3. Set the Loss layer to FP32 for computation. -4. During the inverse computation, the Loss Scale value is first multiplied to avoid underflow due to a too small inverse gradient. -5. FP16 parameters are involved in the gradient computation and their results will be cast back to FP32. -6. Dividing by the Loss scale value to restore the amplified gradient. -7. Determine if there is an overflow in the gradient, and skip the update if there is an overflow, otherwise the optimizer updates the original parameters with FP32. - -In the following, we demonstrate the automatic mixed precision implementation of MindSpore by importing the handwritten digit recognition model and dataset from [Quick Start](https://www.mindspore.cn/tutorials/en/master/beginner/quick_start.html). - -```python -import mindspore as ms -from mindspore import nn -from mindspore import value_and_grad -from mindspore.dataset import vision, transforms -from mindspore.dataset import MnistDataset - -# Download data from open datasets -from download import download - -url = "https://mindspore-website.obs.cn-north-4.myhuaweicloud.com/" \ - "notebook/datasets/MNIST_Data.zip" -path = download(url, "./", kind="zip", replace=True) - - -def datapipe(path, batch_size): - image_transforms = [ - vision.Rescale(1.0 / 255.0, 0), - vision.Normalize(mean=(0.1307,), std=(0.3081,)), - vision.HWC2CHW() - ] - label_transform = transforms.TypeCast(ms.int32) - - dataset = MnistDataset(path) - dataset = dataset.map(image_transforms, 'image') - dataset = dataset.map(label_transform, 'label') - dataset = dataset.batch(batch_size) - return dataset - -train_dataset = datapipe('MNIST_Data/train', 64) - -# Define model -class Network(nn.Cell): - def __init__(self): - super().__init__() - self.flatten = nn.Flatten() - self.dense_relu_sequential = nn.SequentialCell( - nn.Dense(28*28, 512), - nn.ReLU(), - nn.Dense(512, 512), - nn.ReLU(), - nn.Dense(512, 10) - ) - - def construct(self, x): - x = self.flatten(x) - logits = self.dense_relu_sequential(x) - return logits -``` - -```text -Downloading data from https://mindspore-website.obs.cn-north-4.myhuaweicloud.com/notebook/datasets/MNIST_Data.zip (10.3 MB) - -file_sizes: 100%|██████████████████████████| 10.8M/10.8M [00:07<00:00, 1.53MB/s] -Extracting zip file... -Successfully downloaded / unzipped to ./ -``` - -## Type Conversions - -Mixed precision calculations require type conversion of operations that require low precision, converting their input to FP16 types, and then converting them back to FP32 types after the output is obtained. MindSpore provides both automatic and manual type conversion methods to meet the different needs for ease of use and flexibility, which are described below. - -### Automatic Type Conversion - -The `mindspore.amp.auto_mixed_precision` interface provides the function to do automatic type conversion for networks. Automatic type conversion follows a blacklist and white list mechanism with five levels configured according to common operator precision conventions, as follows: - -- 'O0': Neural network keeps FP32. -- 'O1': Operation cast to FP16 by whitelist. -- 'O2': Retain FP32 by blacklist and the rest of operations cast to FP16. -- 'O3': The neural network is fully cast to FP16. -- 'auto': Cast operators in whitelist to FP16, cast operators in blacklist to FP32, and the rest of operators choose the highest floating-point precision of operator input for conversion. - -The following is an example of using automatic type conversion: - -```python -from mindspore.amp import auto_mixed_precision - -model = Network() -model = auto_mixed_precision(model, 'O2') -``` - -### Manual Type Conversion - -Usually automatic type conversion can be used to satisfy most of the mixed precision training needs. But when users need to finely control the precision of operations in different parts of the neural network, they can be controlled by means of manual type conversion. - -> Manual type conversions need to take into account the precision of each module in the model and are generally used only when extreme performance is required. - -Below we adapt `Network` in the previous article to demonstrate different ways of manual type conversion. - -#### Cell Granularity Type Conversion - -The `nn.Cell` class provides the `to_float` method to configure the module's operator precision with a single click, automatically casting the module input to the specified precision. - -```python -class NetworkFP16(nn.Cell): - def __init__(self): - super().__init__() - self.flatten = nn.Flatten() - self.dense_relu_sequential = nn.SequentialCell( - nn.Dense(28*28, 512).to_float(ms.float16), - nn.ReLU(), - nn.Dense(512, 512).to_float(ms.float16), - nn.ReLU(), - nn.Dense(512, 10).to_float(ms.float16) - ) - - def construct(self, x): - x = self.flatten(x) - logits = self.dense_relu_sequential(x) - return logits -``` - -#### Custom Granularity Type Conversion - -When the user needs to configure the precision of operations in a single operation, or a combination of multiple modules, Cell granularity often can not meet the purpose of custom granularity control by directly casting the type of input data. - -```python -class NetworkFP16Manual(nn.Cell): - def __init__(self): - super().__init__() - self.flatten = nn.Flatten() - self.dense_relu_sequential = nn.SequentialCell( - nn.Dense(28*28, 512), - nn.ReLU(), - nn.Dense(512, 512), - nn.ReLU(), - nn.Dense(512, 10) - ) - - def construct(self, x): - x = self.flatten(x) - x = x.astype(ms.float16) - logits = self.dense_relu_sequential(x) - logits = logits.astype(ms.float32) - return logits -``` - -## Loss Scaling - -Two implementations of Loss Scale are provided in MindSpore, [mindspore.amp.StaticLossScaler](https://www.mindspore.cn/docs/en/master/api_python/amp/mindspore.amp.StaticLossScaler.html) and [mindspore.amp.DynamicLossScaler](https://www.mindspore.cn/docs/en/master/api_python/amp/mindspore.amp.DynamicLossScaler.html), whose difference is whether the loss scale value is dynamically adjusted. The following is an example of `DynamicLossScaler`, which implements the neural network training logic according to the mixed precision calculation process. - -First, instantiate the LossScaler and manually scale up the loss value when defining the forward network. - -```python -from mindspore.amp import DynamicLossScaler - -# Instantiate loss function and optimizer -loss_fn = nn.CrossEntropyLoss() -optimizer = nn.SGD(model.trainable_params(), 1e-2) - -# Define LossScaler -loss_scaler = DynamicLossScaler(scale_value=2**16, scale_factor=2, scale_window=50) - -def forward_fn(data, label): - logits = model(data) - loss = loss_fn(logits, label) - # scale up the loss value - loss = loss_scaler.scale(loss) - return loss, logits -``` - -Next, a function transformation is performed to obtain the gradient function. - -```python -grad_fn = value_and_grad(forward_fn, None, model.trainable_params(), has_aux=True) -``` - -Define the training step: Calculates the current gradient value and recovers the loss. Use `all_finite` to determine if there is a gradient underflow problem. If there is no overflow, restore the gradient and update the network weight, while if there is overflow, skip this step. - -```python -from mindspore.amp import all_finite - -@ms.jit -def train_step(data, label): - (loss, _), grads = grad_fn(data, label) - loss = loss_scaler.unscale(loss) - - is_finite = all_finite(grads) - if is_finite: - grads = loss_scaler.unscale(grads) - optimizer(grads) - loss_scaler.adjust(is_finite) - - return loss -``` - -Finally, we train 1 epoch and observe the convergence of the loss trained using automatic mixed precision. - -```python -size = train_dataset.get_dataset_size() -model.set_train() -for batch, (data, label) in enumerate(train_dataset.create_tuple_iterator()): - loss = train_step(data, label) - - if batch % 100 == 0: - loss, current = loss.asnumpy(), batch - print(f"loss: {loss:>7f} [{current:>3d}/{size:>3d}]") -``` - -```text -loss: 2.305425 [ 0/938] -loss: 2.289585 [100/938] -loss: 2.259094 [200/938] -loss: 2.176874 [300/938] -loss: 1.856715 [400/938] -loss: 1.398342 [500/938] -loss: 0.889620 [600/938] -loss: 0.709884 [700/938] -loss: 0.750509 [800/938] -loss: 0.482525 [900/938] -``` - -It can be seen that the loss convergence is normal and there is no overflow problem. - -## Automatic Mixed Precision for `Cell` Configuration - -MindSpore supports a programming paradigm that uses Cell to encapsulate the full computational graph. When the [mindspore.amp.build_train_network](https://www.mindspore.cn/docs/en/master/api_python/amp/mindspore.amp.build_train_network.html) interface can be used to automatically perform the type conversion and pass in the Loss Scale as part of the full graph computation. At this point, you only need to configure the mixed precision level and `LossScaleManager` to get the computational graph with the configured automatic mixed precision. - -[mindspore.amp.FixedLossScaleManager](https://www.mindspore.cn/docs/en/master/api_python/amp/mindspore.amp.FixedLossScaleManager.html) and [mindspore.amp.DynamicLossScaleManager](https://www.mindspore.cn/docs/en/master/api_python/amp/mindspore.amp.DynamicLossScaleManager.html) are the Loss scale management interfaces for configuring the automatic mixed precision with `Cell`, corresponding to `StaticLossScalar` and `DynamicLossScalar`, respectively. For detailed information, refer to [mindspore.amp](https://www.mindspore.cn/docs/en/master/api_python/mindspore.amp.html). - -> Automated mixed precision training with `Cell` configuration supports only `GPU` and `Ascend`. - -```python -from mindspore.amp import build_train_network, FixedLossScaleManager - -model = Network() -loss_scale_manager = FixedLossScaleManager() - -model = build_train_network(model, optimizer, loss_fn, level="O2", loss_scale_manager=loss_scale_manager) -``` - -## `Model` Configures Automatic Mixed Precision - -[mindspore.train.Model](https://www.mindspore.cn/docs/en/master/api_python/train/mindspore.train.Model.html) is a high level encapsulation for fast training of neural networks, which encapsulates `mindspore.amp.build_train_network`, so again, only the mixed precision level and `LossScaleManager` need to be configured for automatic mixed precision training. - -> Automated mixed precision training with `Model` configuration supports only `GPU` and `Ascend`. - -```python -from mindspore.train import Model, LossMonitor -# Initialize network -model = Network() -loss_fn = nn.CrossEntropyLoss() -optimizer = nn.SGD(model.trainable_params(), 1e-2) - -loss_scale_manager = FixedLossScaleManager() -trainer = Model(model, loss_fn=loss_fn, optimizer=optimizer, metrics={'accuracy'}, amp_level="O2", loss_scale_manager=loss_scale_manager) - -loss_callback = LossMonitor(100) -trainer.train(10, train_dataset, callbacks=[loss_callback]) -``` - -```text -epoch: 1 step: 100, loss is 2.2883859 -epoch: 1 step: 200, loss is 2.2612116 -epoch: 1 step: 300, loss is 2.1563218 -epoch: 1 step: 400, loss is 1.9420109 -epoch: 1 step: 500, loss is 1.396821 -epoch: 1 step: 600, loss is 1.0450488 -epoch: 1 step: 700, loss is 0.69754004 -epoch: 1 step: 800, loss is 0.6924556 -epoch: 1 step: 900, loss is 0.57444984 -... -epoch: 10 step: 58, loss is 0.13086069 -epoch: 10 step: 158, loss is 0.07224723 -epoch: 10 step: 258, loss is 0.08281057 -epoch: 10 step: 358, loss is 0.09759849 -epoch: 10 step: 458, loss is 0.17265382 -epoch: 10 step: 558, loss is 0.10023793 -epoch: 10 step: 658, loss is 0.08235697 -epoch: 10 step: 758, loss is 0.10531154 -epoch: 10 step: 858, loss is 0.19084263 -``` - -> The image is quoted from [automatic-mixed-precision](https://developer.nvidia.com/automatic-mixed-precision). diff --git a/tutorials/source_en/beginner/model.md b/tutorials/source_en/beginner/model.md index 2cdee882556ed37d053a2a56dbf866b62b149a03..0f6a11d6ec79fcc75cda5dde31818ebc1e6e56a1 100644 --- a/tutorials/source_en/beginner/model.md +++ b/tutorials/source_en/beginner/model.md @@ -1,6 +1,6 @@ [![View Source On Gitee](https://mindspore-website.obs.cn-north-4.myhuaweicloud.com/website-images/master/resource/_static/logo_source_en.svg)](https://gitee.com/mindspore/docs/blob/master/tutorials/source_en/beginner/model.md) -[Introduction](https://www.mindspore.cn/tutorials/en/master/beginner/introduction.html) || [Quick Start](https://www.mindspore.cn/tutorials/en/master/beginner/quick_start.html) || [Tensor](https://www.mindspore.cn/tutorials/en/master/beginner/tensor.html) || [Data Loading and Processing](https://www.mindspore.cn/tutorials/en/master/beginner/dataset.html) || **Model** || [Autograd](https://www.mindspore.cn/tutorials/en/master/beginner/autograd.html) || [Train](https://www.mindspore.cn/tutorials/en/master/beginner/train.html) || [Save and Load](https://www.mindspore.cn/tutorials/en/master/beginner/save_load.html) || [Accelerating with Static Graphs](https://www.mindspore.cn/tutorials/en/master/beginner/accelerate_with_static_graph.html) || [Mixed Precision](https://www.mindspore.cn/tutorials/en/master/beginner/mixed_precision.html) +[Introduction](https://www.mindspore.cn/tutorials/en/master/beginner/introduction.html) || [Quick Start](https://www.mindspore.cn/tutorials/en/master/beginner/quick_start.html) || [Tensor](https://www.mindspore.cn/tutorials/en/master/beginner/tensor.html) || [Data Loading and Processing](https://www.mindspore.cn/tutorials/en/master/beginner/dataset.html) || **Model** || [Autograd](https://www.mindspore.cn/tutorials/en/master/beginner/autograd.html) || [Train](https://www.mindspore.cn/tutorials/en/master/beginner/train.html) || [Save and Load](https://www.mindspore.cn/tutorials/en/master/beginner/save_load.html) || [Accelerating with Static Graphs](https://www.mindspore.cn/tutorials/en/master/beginner/accelerate_with_static_graph.html) # Building a Network diff --git a/tutorials/source_en/beginner/quick_start.md b/tutorials/source_en/beginner/quick_start.md index ed25584c88f3b76c025f9edeedae11ca5161ddc5..2abc001ea0a3277f8e4336f928d03602e7edce73 100644 --- a/tutorials/source_en/beginner/quick_start.md +++ b/tutorials/source_en/beginner/quick_start.md @@ -1,6 +1,6 @@ [![View Source On Gitee](https://mindspore-website.obs.cn-north-4.myhuaweicloud.com/website-images/master/resource/_static/logo_source_en.svg)](https://gitee.com/mindspore/docs/blob/master/tutorials/source_en/beginner/quick_start.md) -[Introduction](https://www.mindspore.cn/tutorials/en/master/beginner/introduction.html) || **Quick Start** || [Tensor](https://www.mindspore.cn/tutorials/en/master/beginner/tensor.html) || [Data Loading and Processing](https://www.mindspore.cn/tutorials/en/master/beginner/dataset.html) || [Model](https://www.mindspore.cn/tutorials/en/master/beginner/model.html) || [Autograd](https://www.mindspore.cn/tutorials/en/master/beginner/autograd.html) || [Train](https://www.mindspore.cn/tutorials/en/master/beginner/train.html) || [Save and load](https://www.mindspore.cn/tutorials/en/master/beginner/save_load.html) || [Accelerating with Static Graphs](https://www.mindspore.cn/tutorials/en/master/beginner/accelerate_with_static_graph.html) || [Mixed Precision](https://www.mindspore.cn/tutorials/en/master/beginner/mixed_precision.html) +[Introduction](https://www.mindspore.cn/tutorials/en/master/beginner/introduction.html) || **Quick Start** || [Tensor](https://www.mindspore.cn/tutorials/en/master/beginner/tensor.html) || [Data Loading and Processing](https://www.mindspore.cn/tutorials/en/master/beginner/dataset.html) || [Model](https://www.mindspore.cn/tutorials/en/master/beginner/model.html) || [Autograd](https://www.mindspore.cn/tutorials/en/master/beginner/autograd.html) || [Train](https://www.mindspore.cn/tutorials/en/master/beginner/train.html) || [Save and load](https://www.mindspore.cn/tutorials/en/master/beginner/save_load.html) || [Accelerating with Static Graphs](https://www.mindspore.cn/tutorials/en/master/beginner/accelerate_with_static_graph.html) # Quick Start diff --git a/tutorials/source_en/beginner/save_load.md b/tutorials/source_en/beginner/save_load.md index c3e1c75b16b711ee719630c19ec0b52f69efccbc..c92ba00681dc3b6a6adac9ac58eeb495d4bd8a0d 100644 --- a/tutorials/source_en/beginner/save_load.md +++ b/tutorials/source_en/beginner/save_load.md @@ -1,6 +1,6 @@ [![View Source On Gitee](https://mindspore-website.obs.cn-north-4.myhuaweicloud.com/website-images/master/resource/_static/logo_source_en.svg)](https://gitee.com/mindspore/docs/blob/master/tutorials/source_en/beginner/save_load.md) -[Introduction](https://www.mindspore.cn/tutorials/en/master/beginner/introduction.html) || [Quick Start](https://www.mindspore.cn/tutorials/en/master/beginner/quick_start.html) || [Tensor](https://www.mindspore.cn/tutorials/en/master/beginner/tensor.html) || [Data Loading and Processing](https://www.mindspore.cn/tutorials/en/master/beginner/dataset.html) || [Model](https://www.mindspore.cn/tutorials/en/master/beginner/model.html) || [Autograd](https://www.mindspore.cn/tutorials/en/master/beginner/autograd.html) || [Train](https://www.mindspore.cn/tutorials/en/master/beginner/train.html) || **Save and Load** || [Accelerating with Static Graphs](https://www.mindspore.cn/tutorials/en/master/beginner/accelerate_with_static_graph.html) || [Mixed Precision](https://www.mindspore.cn/tutorials/en/master/beginner/mixed_precision.html) +[Introduction](https://www.mindspore.cn/tutorials/en/master/beginner/introduction.html) || [Quick Start](https://www.mindspore.cn/tutorials/en/master/beginner/quick_start.html) || [Tensor](https://www.mindspore.cn/tutorials/en/master/beginner/tensor.html) || [Data Loading and Processing](https://www.mindspore.cn/tutorials/en/master/beginner/dataset.html) || [Model](https://www.mindspore.cn/tutorials/en/master/beginner/model.html) || [Autograd](https://www.mindspore.cn/tutorials/en/master/beginner/autograd.html) || [Train](https://www.mindspore.cn/tutorials/en/master/beginner/train.html) || **Save and Load** || [Accelerating with Static Graphs](https://www.mindspore.cn/tutorials/en/master/beginner/accelerate_with_static_graph.html) # Saving and Loading the Model diff --git a/tutorials/source_en/beginner/tensor.md b/tutorials/source_en/beginner/tensor.md index 3b552faf5318e58b9ab9607976eccd453ffe66a0..eebcc25910c5391d04faa8a0b9568b98e011be1d 100644 --- a/tutorials/source_en/beginner/tensor.md +++ b/tutorials/source_en/beginner/tensor.md @@ -1,6 +1,6 @@ [![View Source On Gitee](https://mindspore-website.obs.cn-north-4.myhuaweicloud.com/website-images/master/resource/_static/logo_source_en.svg)](https://gitee.com/mindspore/docs/blob/master/tutorials/source_en/beginner/tensor.md) -[Introduction](https://www.mindspore.cn/tutorials/en/master/beginner/introduction.html) || [Quick Start](https://www.mindspore.cn/tutorials/en/master/beginner/quick_start.html) || **Tensor** || [Data Loading and Processing](https://www.mindspore.cn/tutorials/en/master/beginner/dataset.html) || [Model](https://www.mindspore.cn/tutorials/en/master/beginner/model.html) || [Autograd](https://www.mindspore.cn/tutorials/en/master/beginner/autograd.html) || [Train](https://www.mindspore.cn/tutorials/en/master/beginner/train.html) || [Save and Load](https://www.mindspore.cn/tutorials/en/master/beginner/save_load.html) || [Accelerating with Static Graphs](https://www.mindspore.cn/tutorials/en/master/beginner/accelerate_with_static_graph.html) || [Mixed Precision](https://www.mindspore.cn/tutorials/en/master/beginner/mixed_precision.html) +[Introduction](https://www.mindspore.cn/tutorials/en/master/beginner/introduction.html) || [Quick Start](https://www.mindspore.cn/tutorials/en/master/beginner/quick_start.html) || **Tensor** || [Data Loading and Processing](https://www.mindspore.cn/tutorials/en/master/beginner/dataset.html) || [Model](https://www.mindspore.cn/tutorials/en/master/beginner/model.html) || [Autograd](https://www.mindspore.cn/tutorials/en/master/beginner/autograd.html) || [Train](https://www.mindspore.cn/tutorials/en/master/beginner/train.html) || [Save and Load](https://www.mindspore.cn/tutorials/en/master/beginner/save_load.html) || [Accelerating with Static Graphs](https://www.mindspore.cn/tutorials/en/master/beginner/accelerate_with_static_graph.html) # Tensor diff --git a/tutorials/source_en/beginner/train.md b/tutorials/source_en/beginner/train.md index 39c1b0987c39e4aac786e7c1e05d162325edeb3d..125510f7d8d296ffc94acf08c37fb07343aa6208 100644 --- a/tutorials/source_en/beginner/train.md +++ b/tutorials/source_en/beginner/train.md @@ -1,6 +1,6 @@ [![View Source On Gitee](https://mindspore-website.obs.cn-north-4.myhuaweicloud.com/website-images/master/resource/_static/logo_source_en.svg)](https://gitee.com/mindspore/docs/blob/master/tutorials/source_en/beginner/train.md) -[Introduction](https://www.mindspore.cn/tutorials/en/master/beginner/introduction.html) || [Quick Start](https://www.mindspore.cn/tutorials/en/master/beginner/quick_start.html) || [Tensor](https://www.mindspore.cn/tutorials/en/master/beginner/tensor.html) || [Data Loading and Processing](https://www.mindspore.cn/tutorials/en/master/beginner/dataset.html) || [Model](https://www.mindspore.cn/tutorials/en/master/beginner/model.html) || [Autograd](https://www.mindspore.cn/tutorials/en/master/beginner/autograd.html) || **Train** || [Save and Load](https://www.mindspore.cn/tutorials/en/master/beginner/save_load.html) || [Accelerating with Static Graphs](https://www.mindspore.cn/tutorials/en/master/beginner/accelerate_with_static_graph.html) || [Mixed Precision](https://www.mindspore.cn/tutorials/en/master/beginner/mixed_precision.html) +[Introduction](https://www.mindspore.cn/tutorials/en/master/beginner/introduction.html) || [Quick Start](https://www.mindspore.cn/tutorials/en/master/beginner/quick_start.html) || [Tensor](https://www.mindspore.cn/tutorials/en/master/beginner/tensor.html) || [Data Loading and Processing](https://www.mindspore.cn/tutorials/en/master/beginner/dataset.html) || [Model](https://www.mindspore.cn/tutorials/en/master/beginner/model.html) || [Autograd](https://www.mindspore.cn/tutorials/en/master/beginner/autograd.html) || **Train** || [Save and Load](https://www.mindspore.cn/tutorials/en/master/beginner/save_load.html) || [Accelerating with Static Graphs](https://www.mindspore.cn/tutorials/en/master/beginner/accelerate_with_static_graph.html) # Model Training diff --git a/tutorials/source_en/index.rst b/tutorials/source_en/index.rst index 05e3a933cd2934f2f3ec63c10cd56bf5995c936f..9aea3c9585ce65e831a95e9d8b08194fb585a8d8 100644 --- a/tutorials/source_en/index.rst +++ b/tutorials/source_en/index.rst @@ -21,7 +21,6 @@ MindSpore Tutorial beginner/train beginner/save_load beginner/accelerate_with_static_graph - beginner/mixed_precision .. toctree:: :glob: diff --git a/tutorials/source_zh_cn/beginner/accelerate_with_static_graph.ipynb b/tutorials/source_zh_cn/beginner/accelerate_with_static_graph.ipynb index 0fd1f96ef37ba4182cfb99023aed73f0a6fb2d21..4f80abe32f06a91e830e7653ce5c9de0c907b58c 100644 --- a/tutorials/source_zh_cn/beginner/accelerate_with_static_graph.ipynb +++ b/tutorials/source_zh_cn/beginner/accelerate_with_static_graph.ipynb @@ -7,7 +7,7 @@ "source": [ "[![下载Notebook](https://mindspore-website.obs.cn-north-4.myhuaweicloud.com/website-images/master/resource/_static/logo_notebook.svg)](https://mindspore-website.obs.cn-north-4.myhuaweicloud.com/notebook/master/tutorials/zh_cn/beginner/mindspore_accelerate_with_static_graph.ipynb) [![下载样例代码](https://mindspore-website.obs.cn-north-4.myhuaweicloud.com/website-images/master/resource/_static/logo_download_code.svg)](https://mindspore-website.obs.cn-north-4.myhuaweicloud.com/notebook/master/tutorials/zh_cn/beginner/mindspore_accelerate_with_static_graph.py) [![查看源文件](https://mindspore-website.obs.cn-north-4.myhuaweicloud.com/website-images/master/resource/_static/logo_source.svg)](https://gitee.com/mindspore/docs/blob/master/tutorials/source_zh_cn/beginner/accelerate_with_static_graph.ipynb)\n", "\n", - "[基本介绍](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/introduction.html) || [快速入门](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/quick_start.html) || [张量 Tensor](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/tensor.html) || [数据加载与处理](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/dataset.html) || [网络构建](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/model.html) || [函数式自动微分](https://gitee.com/mindspore/docs/blob/master/tutorials/source_zh_cn/beginner/autograd.ipynb) || [模型训练](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/train.html) || [保存与加载](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/save_load.html) || **Graph Mode加速** || [自动混合精度](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/mixed_precision.html) ||\n", + "[基本介绍](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/introduction.html) || [快速入门](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/quick_start.html) || [张量 Tensor](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/tensor.html) || [数据加载与处理](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/dataset.html) || [网络构建](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/model.html) || [函数式自动微分](https://gitee.com/mindspore/docs/blob/master/tutorials/source_zh_cn/beginner/autograd.ipynb) || [模型训练](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/train.html) || [保存与加载](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/save_load.html) || **Graph Mode加速** ||\n", "\n", "# Graph Mode加速\n", "\n", diff --git a/tutorials/source_zh_cn/beginner/autograd.ipynb b/tutorials/source_zh_cn/beginner/autograd.ipynb index 94e8e5691015c9aab853f14d8016fe64db5ced70..d64ef996d09fcbca8142b5453cd839db3e8c0a40 100644 --- a/tutorials/source_zh_cn/beginner/autograd.ipynb +++ b/tutorials/source_zh_cn/beginner/autograd.ipynb @@ -6,7 +6,7 @@ "source": [ "[![下载Notebook](https://mindspore-website.obs.cn-north-4.myhuaweicloud.com/website-images/master/resource/_static/logo_notebook.svg)](https://mindspore-website.obs.cn-north-4.myhuaweicloud.com/notebook/master/tutorials/zh_cn/beginner/mindspore_autograd.ipynb) [![下载样例代码](https://mindspore-website.obs.cn-north-4.myhuaweicloud.com/website-images/master/resource/_static/logo_download_code.svg)](https://mindspore-website.obs.cn-north-4.myhuaweicloud.com/notebook/master/tutorials/zh_cn/beginner/mindspore_autograd.py) [![查看源文件](https://mindspore-website.obs.cn-north-4.myhuaweicloud.com/website-images/master/resource/_static/logo_source.svg)](https://gitee.com/mindspore/docs/blob/master/tutorials/source_zh_cn/beginner/autograd.ipynb)\n", "\n", - "[基本介绍](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/introduction.html) || [快速入门](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/quick_start.html) || [张量 Tensor](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/tensor.html) || [数据加载与处理](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/dataset.html) || [网络构建](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/model.html) || **函数式自动微分** || [模型训练](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/train.html) || [保存与加载](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/save_load.html) || [Graph Mode加速](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/accelerate_with_static_graph.html) || [自动混合精度](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/mixed_precision.html) ||" + "[基本介绍](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/introduction.html) || [快速入门](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/quick_start.html) || [张量 Tensor](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/tensor.html) || [数据加载与处理](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/dataset.html) || [网络构建](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/model.html) || **函数式自动微分** || [模型训练](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/train.html) || [保存与加载](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/save_load.html) || [Graph Mode加速](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/accelerate_with_static_graph.html) ||" ] }, { diff --git a/tutorials/source_zh_cn/beginner/dataset.ipynb b/tutorials/source_zh_cn/beginner/dataset.ipynb index 79b5225838e6c42afdd2dd056f0864ee55e3710f..3526f5aa076c2d260b91486e003813eb2c4b8495 100644 --- a/tutorials/source_zh_cn/beginner/dataset.ipynb +++ b/tutorials/source_zh_cn/beginner/dataset.ipynb @@ -7,7 +7,7 @@ "source": [ "[![下载Notebook](https://mindspore-website.obs.cn-north-4.myhuaweicloud.com/website-images/master/resource/_static/logo_notebook.svg)](https://mindspore-website.obs.cn-north-4.myhuaweicloud.com/notebook/master/tutorials/zh_cn/beginner/mindspore_dataset.ipynb) [![下载样例代码](https://mindspore-website.obs.cn-north-4.myhuaweicloud.com/website-images/master/resource/_static/logo_download_code.svg)](https://mindspore-website.obs.cn-north-4.myhuaweicloud.com/notebook/master/tutorials/zh_cn/beginner/mindspore_dataset.py) [![查看源文件](https://mindspore-website.obs.cn-north-4.myhuaweicloud.com/website-images/master/resource/_static/logo_source.svg)](https://gitee.com/mindspore/docs/blob/master/tutorials/source_zh_cn/beginner/dataset.ipynb)\n", "\n", - "[基本介绍](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/introduction.html) || [快速入门](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/quick_start.html) || [张量 Tensor](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/tensor.html) || **数据加载与处理** || [网络构建](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/model.html) || [函数式自动微分](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/autograd.html) || [模型训练](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/train.html) || [保存与加载](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/save_load.html) || [Graph Mode加速](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/accelerate_with_static_graph.html) || [自动混合精度](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/mixed_precision.html) ||" + "[基本介绍](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/introduction.html) || [快速入门](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/quick_start.html) || [张量 Tensor](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/tensor.html) || **数据加载与处理** || [网络构建](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/model.html) || [函数式自动微分](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/autograd.html) || [模型训练](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/train.html) || [保存与加载](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/save_load.html) || [Graph Mode加速](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/accelerate_with_static_graph.html) ||" ] }, { diff --git a/tutorials/source_zh_cn/beginner/images/fp16_vs_fp32.png b/tutorials/source_zh_cn/beginner/images/fp16_vs_fp32.png deleted file mode 100644 index 8f8e07801600a15f3fe20010f59f5669afad34d5..0000000000000000000000000000000000000000 Binary files a/tutorials/source_zh_cn/beginner/images/fp16_vs_fp32.png and /dev/null differ diff --git a/tutorials/source_zh_cn/beginner/images/mix_precision_fp16.png b/tutorials/source_zh_cn/beginner/images/mix_precision_fp16.png deleted file mode 100644 index 2c8771445dfaaf320ace866cb14f5ae581164560..0000000000000000000000000000000000000000 Binary files a/tutorials/source_zh_cn/beginner/images/mix_precision_fp16.png and /dev/null differ diff --git a/tutorials/source_zh_cn/beginner/introduction.ipynb b/tutorials/source_zh_cn/beginner/introduction.ipynb index c801892b0d12333e3af540ddca9009a294b3b0b7..70504f4a1a2a72b6f15fc7b8bb934f702cbc2c39 100644 --- a/tutorials/source_zh_cn/beginner/introduction.ipynb +++ b/tutorials/source_zh_cn/beginner/introduction.ipynb @@ -7,7 +7,7 @@ "source": [ "[![查看源文件](https://mindspore-website.obs.cn-north-4.myhuaweicloud.com/website-images/master/resource/_static/logo_source.svg)](https://gitee.com/mindspore/docs/blob/master/tutorials/source_zh_cn/beginner/introduction.ipynb)\n", "\n", - "**基本介绍** || [快速入门](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/quick_start.html#) || [张量 Tensor](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/tensor.html) || [数据加载与处理](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/dataset.html) || [网络构建](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/model.html) || [函数式自动微分](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/autograd.html) || [模型训练](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/train.html) || [保存与加载](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/save_load.html) || [Graph Mode加速](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/accelerate_with_static_graph.html) || [自动混合精度](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/mixed_precision.html) ||" + "**基本介绍** || [快速入门](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/quick_start.html#) || [张量 Tensor](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/tensor.html) || [数据加载与处理](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/dataset.html) || [网络构建](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/model.html) || [函数式自动微分](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/autograd.html) || [模型训练](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/train.html) || [保存与加载](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/save_load.html) || [Graph Mode加速](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/accelerate_with_static_graph.html) ||" ] }, { diff --git a/tutorials/source_zh_cn/beginner/mixed_precision.ipynb b/tutorials/source_zh_cn/beginner/mixed_precision.ipynb deleted file mode 100644 index ecbf71dd0035ca1c2a953b5f03ab58669339f27b..0000000000000000000000000000000000000000 --- a/tutorials/source_zh_cn/beginner/mixed_precision.ipynb +++ /dev/null @@ -1,581 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "3fca2d6c-be41-47be-b020-228c6e2acc98", - "metadata": {}, - "source": [ - "[![下载Notebook](https://mindspore-website.obs.cn-north-4.myhuaweicloud.com/website-images/master/resource/_static/logo_notebook.svg)](https://mindspore-website.obs.cn-north-4.myhuaweicloud.com/notebook/master/tutorials/zh_cn/beginner/mindspore_mixed_precision.ipynb) [![下载样例代码](https://mindspore-website.obs.cn-north-4.myhuaweicloud.com/website-images/master/resource/_static/logo_download_code.svg)](https://mindspore-website.obs.cn-north-4.myhuaweicloud.com/notebook/master/tutorials/zh_cn/beginner/mindspore_mixed_precision.py) [![查看源文件](https://mindspore-website.obs.cn-north-4.myhuaweicloud.com/website-images/master/resource/_static/logo_source.svg)](https://gitee.com/mindspore/docs/blob/master/tutorials/source_zh_cn/beginner/mixed_precision.ipynb)\n", - "\n", - "[基本介绍](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/introduction.html) || [快速入门](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/quick_start.html#) || [张量 Tensor](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/tensor.html) || [数据加载与处理](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/dataset.html) || [网络构建](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/model.html) || [函数式自动微分](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/autograd.html) || [模型训练](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/train.html) || [保存与加载](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/save_load.html) || [Graph Mode加速](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/accelerate_with_static_graph.html) || **自动混合精度** ||\n", - "\n", - "# 自动混合精度" - ] - }, - { - "cell_type": "markdown", - "id": "49ed0572-abea-4574-9710-be45d84191b3", - "metadata": {}, - "source": [ - "混合精度(Mix Precision)训练是指在训练时,对神经网络不同的运算采用不同的数值精度的运算策略。在神经网络运算中,部分运算对数值精度不敏感,此时使用较低精度可以达到明显的加速效果(如conv、matmul等);而部分运算由于输入和输出的数值差异大,通常需要保留较高精度以保证结果的正确性(如log、softmax等)。\n", - "\n", - "当前的AI加速卡通常针对计算密集、精度不敏感的运算设计了硬件加速模块,如NVIDIA GPU的TensorCore、Ascend NPU的Cube等。对于conv、matmul等运算占比较大的神经网络,其训练速度通常会有较大的加速比。\n", - "\n", - "`mindspore.amp`模块提供了便捷的自动混合精度接口,用户可以在不同的硬件后端通过简单的接口调用获得训练加速。下面我们简要介绍混合精度计算原理,并通过实例介绍MindSpore的自动混合精度用法。" - ] - }, - { - "cell_type": "markdown", - "id": "518888c6-a7ad-4a13-8b7e-8cef642457e3", - "metadata": {}, - "source": [ - "## 混合精度计算原理" - ] - }, - { - "cell_type": "markdown", - "id": "76ebb0cf-7c2d-49d4-8905-04d928567e76", - "metadata": {}, - "source": [ - "浮点数据类型主要分为双精度(FP64)、单精度(FP32)、半精度(FP16)。在神经网络模型的训练过程中,一般默认采用单精度(FP32)浮点数据类型,来表示网络模型权重和其他参数。在了解混合精度训练之前,先简单了解浮点数据类型。\n", - "\n", - "根据IEEE二进制浮点数算术标准([IEEE 754](https://en.wikipedia.org/wiki/IEEE_754))的定义,浮点数据类型分为双精度(FP64)、单精度(FP32)、半精度(FP16)三种。每一种都有三个不同的部分来表示。FP64采用8个字节共64位进行编码存储;FP32采用4个字节共32位;FP16则是采用2字节共16位。如图所示:\n", - "\n", - "![fp16-vs-fp32](https://mindspore-website.obs.cn-north-4.myhuaweicloud.com/website-images/master/tutorials/source_zh_cn/beginner/images/fp16_vs_fp32.png)\n", - "\n", - "从图中可以看出,与FP32相比,FP16的存储空间是FP32的一半。类似地,FP32则是FP64的一半。因此使用FP16进行运算具备以下优势:\n", - "\n", - "- 减少内存占用:FP16的位宽是FP32的一半,因此权重等参数所占用的内存也是原来的一半,节省下来的内存可以用于更大的网络模型或者更多的数据训练。\n", - "- 计算效率更高:在特殊的AI加速芯片(如华为Atlas训练系列、Atlas 200/300/500推理系列,或NVIDIA VOLTA架构的GPU)上,FP16运算性能比FP32更高。\n", - "- 加快通讯效率:针对分布式训练,特别是在大模型训练的过程中,通讯的开销制约了网络模型训练的整体性能。通讯的位宽减少可提升通讯性能,减少等待时间,加快数据的流通。\n", - "\n", - "但是使用FP16同样会带来一些问题:\n", - "\n", - "- 数据溢出:FP16的有效数据表示范围为 $[5.9\\times10^{-8}, 65504]$,FP32的有效数据表示范围为 $[1.4\\times10^{-45}, 1.7\\times10^{38}]$。可见FP16相比FP32的有效范围要窄很多,使用FP16替换FP32会出现上溢(Overflow)和下溢(Underflow)的情况。而在深度学习中,需要计算网络模型中权重的梯度(一阶导数),因此梯度会比权重值更加小,往往容易出现下溢情况。\n", - "- 舍入误差:Rounding Error是指当网络模型的反向梯度很小,一般FP32能够表示,但是转换到FP16会小于当前区间内的最小间隔,会导致数据溢出。如`0.00006666666`在FP32中能正常表示,转换到FP16后会表示成为`0.000067`,不满足FP16最小间隔的数会强制舍入。\n", - "\n", - "因此,在使用混合精度获得训练加速和内存节省的同时,需要考虑FP16带来的问题。Loss Scale(损失缩放)是FP16类型数据下溢问题的解决方案,其主要思想是在计算损失值loss的时候,将loss扩大一定的倍数。根据链式法则,梯度也会相应扩大,然后在优化器更新权重时再缩小相应的倍数,从而避免数据下溢。" - ] - }, - { - "cell_type": "markdown", - "id": "981dd3d3-bec1-4b43-a48e-6563547f8411", - "metadata": {}, - "source": [ - "根据上述原理介绍,典型的混合精度计算流程如下图所示:\n", - "\n", - "![mix precision](https://mindspore-website.obs.cn-north-4.myhuaweicloud.com/website-images/master/tutorials/source_zh_cn/beginner/images/mix_precision_fp16.png)\n", - "\n", - "1. 参数以FP32存储;\n", - "2. 正向计算过程中,遇到FP16算子,需要把算子输入和参数从FP32 cast成FP16进行计算;\n", - "3. 将Loss层设置为FP32进行计算;\n", - "4. 反向计算过程中,首先乘以Loss Scale值,避免反向梯度过小而产生下溢;\n", - "5. FP16参数参与梯度计算,其结果将被cast回FP32;\n", - "6. 除以Loss scale值,还原被放大的梯度;\n", - "7. 判断梯度是否存在溢出,如果溢出则跳过更新,否则优化器以FP32对原始参数进行更新。" - ] - }, - { - "cell_type": "markdown", - "id": "81c60159-5242-4ea2-a73e-6015de4a675c", - "metadata": {}, - "source": [ - "下面我们通过导入[快速入门](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/quick_start.html)中的手写数字识别模型及数据集,演示MindSpore的自动混合精度实现。" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "id": "0dcf1b26-3050-42e6-ade8-889f3b3dc79d", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Downloading data from https://mindspore-website.obs.cn-north-4.myhuaweicloud.com/notebook/datasets/MNIST_Data.zip (10.3 MB)\n", - "\n", - "file_sizes: 100%|██████████████████████████| 10.8M/10.8M [00:07<00:00, 1.53MB/s]\n", - "Extracting zip file...\n", - "Successfully downloaded / unzipped to ./\n" - ] - } - ], - "source": [ - "import mindspore as ms\n", - "from mindspore import nn\n", - "from mindspore import value_and_grad\n", - "from mindspore.dataset import vision, transforms\n", - "from mindspore.dataset import MnistDataset\n", - "\n", - "# Download data from open datasets\n", - "from download import download\n", - "\n", - "url = \"https://mindspore-website.obs.cn-north-4.myhuaweicloud.com/\" \\\n", - " \"notebook/datasets/MNIST_Data.zip\"\n", - "path = download(url, \"./\", kind=\"zip\", replace=True)\n", - "\n", - "\n", - "def datapipe(path, batch_size):\n", - " image_transforms = [\n", - " vision.Rescale(1.0 / 255.0, 0),\n", - " vision.Normalize(mean=(0.1307,), std=(0.3081,)),\n", - " vision.HWC2CHW()\n", - " ]\n", - " label_transform = transforms.TypeCast(ms.int32)\n", - "\n", - " dataset = MnistDataset(path)\n", - " dataset = dataset.map(image_transforms, 'image')\n", - " dataset = dataset.map(label_transform, 'label')\n", - " dataset = dataset.batch(batch_size)\n", - " return dataset\n", - "\n", - "train_dataset = datapipe('MNIST_Data/train', 64)\n", - "\n", - "# Define model\n", - "class Network(nn.Cell):\n", - " def __init__(self):\n", - " super().__init__()\n", - " self.flatten = nn.Flatten()\n", - " self.dense_relu_sequential = nn.SequentialCell(\n", - " nn.Dense(28*28, 512),\n", - " nn.ReLU(),\n", - " nn.Dense(512, 512),\n", - " nn.ReLU(),\n", - " nn.Dense(512, 10)\n", - " )\n", - "\n", - " def construct(self, x):\n", - " x = self.flatten(x)\n", - " logits = self.dense_relu_sequential(x)\n", - " return logits" - ] - }, - { - "cell_type": "markdown", - "id": "b22c0430-ad94-47b5-bd04-278e2c71775c", - "metadata": {}, - "source": [ - "## 类型转换\n", - "\n", - "混合精度计算需要将需要使用低精度的运算进行类型转换,将其输入转为FP16类型,得到输出后进将其重新转回FP32类型。MindSpore同时提供了自动和手动类型转换的方法,满足对易用性和灵活性的不同需求,下面我们分别对其进行介绍。\n", - "\n", - "### 自动类型转换" - ] - }, - { - "cell_type": "markdown", - "id": "8bdf54b3", - "metadata": { - "pycharm": { - "name": "#%% md\n" - } - }, - "source": [ - "`mindspore.amp.auto_mixed_precision` 接口提供对网络做自动类型转换的功能。自动类型转换遵循黑白名单机制,根据常用的运算精度习惯配置了5个等级,分别为:\n", - "\n", - "- 'O0':神经网络保持FP32;\n", - "- 'O1':按白名单将运算cast为FP16;\n", - "- 'O2':按黑名单保留FP32,其余运算cast为FP16;\n", - "- 'O3':神经网络完全cast为FP16。\n", - "- 'auto':白名单内的算子运算cast为FP16,黑名单内的算子运算cast为FP32,其余算子选择算子输入的最高浮点精度来转换。\n", - "\n", - "下面是使用自动类型转换的示例:" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "id": "ead4d2ea-95a0-44e6-a722-0701a3e11bfb", - "metadata": {}, - "outputs": [], - "source": [ - "from mindspore.amp import auto_mixed_precision\n", - "\n", - "model = Network()\n", - "model = auto_mixed_precision(model, 'O2')" - ] - }, - { - "cell_type": "markdown", - "id": "8f89dc3c-e419-484d-8d7c-4e991d85cfd1", - "metadata": {}, - "source": [ - "### 手动类型转换\n", - "\n", - "通常情况下,自动类型转换可以满足大部分混合精度训练的需求。但当用户需要精细化控制神经网络不同部分的运算精度时,可以通过手动类型转换的方式来实现。\n", - "\n", - "> 手动类型转换需考虑模型各个模块的运算精度,一般仅在需要获得极致性能的情况下使用。\n", - "\n", - "下面我们对前文的`Network`进行改造,演示手动类型转换的不同方式。" - ] - }, - { - "cell_type": "markdown", - "id": "064acdb8-8cba-4e60-979d-06e667fc70a3", - "metadata": {}, - "source": [ - "#### Cell粒度类型转换\n", - "\n", - "`nn.Cell`类提供了`to_float`方法,可以一键配置该模块的运算精度,自动将模块输入cast为指定的精度:" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "id": "112b57a8-c279-49cb-8069-cc261d92e857", - "metadata": {}, - "outputs": [], - "source": [ - "class NetworkFP16(nn.Cell):\n", - " def __init__(self):\n", - " super().__init__()\n", - " self.flatten = nn.Flatten()\n", - " self.dense_relu_sequential = nn.SequentialCell(\n", - " nn.Dense(28*28, 512).to_float(ms.float16),\n", - " nn.ReLU(),\n", - " nn.Dense(512, 512).to_float(ms.float16),\n", - " nn.ReLU(),\n", - " nn.Dense(512, 10).to_float(ms.float16)\n", - " )\n", - "\n", - " def construct(self, x):\n", - " x = self.flatten(x)\n", - " logits = self.dense_relu_sequential(x)\n", - " return logits" - ] - }, - { - "cell_type": "markdown", - "id": "a169af9b-df4c-4644-8377-833d4cf7100a", - "metadata": {}, - "source": [ - "#### 自定义粒度类型转换\n", - "\n", - "当用户需要在单个运算,或多个模块组合配置运算精度时,Cell粒度往往无法满足,此时可以直接通过对输入数据的类型进行cast来达到自定义粒度控制的目的。" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "id": "d99f7986-bde4-4254-bc97-3baa1c7353d9", - "metadata": {}, - "outputs": [], - "source": [ - "class NetworkFP16Manual(nn.Cell):\n", - " def __init__(self):\n", - " super().__init__()\n", - " self.flatten = nn.Flatten()\n", - " self.dense_relu_sequential = nn.SequentialCell(\n", - " nn.Dense(28*28, 512),\n", - " nn.ReLU(),\n", - " nn.Dense(512, 512),\n", - " nn.ReLU(),\n", - " nn.Dense(512, 10)\n", - " )\n", - "\n", - " def construct(self, x):\n", - " x = self.flatten(x)\n", - " x = x.astype(ms.float16)\n", - " logits = self.dense_relu_sequential(x)\n", - " logits = logits.astype(ms.float32)\n", - " return logits" - ] - }, - { - "cell_type": "markdown", - "id": "42d9a6e5-765e-4c28-9ad4-2c1f59996a0b", - "metadata": {}, - "source": [ - "## 损失缩放\n", - "\n", - "MindSpore中提供了两种Loss Scale的实现,分别为[mindspore.amp.StaticLossScaler](https://www.mindspore.cn/docs/zh-CN/master/api_python/amp/mindspore.amp.StaticLossScaler.html)和[mindspore.amp.DynamicLossScaler](https://www.mindspore.cn/docs/zh-CN/master/api_python/amp/mindspore.amp.DynamicLossScaler.html),其差异为损失缩放值scale value是否进行动态调整。下面以`DynamicLossScaler`为例,根据混合精度计算流程实现神经网络训练逻辑。\n", - "\n", - "首先,实例化LossScaler,并在定义前向网络时,手动放大loss值。" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "id": "9f3abade", - "metadata": { - "collapsed": false, - "jupyter": { - "outputs_hidden": false - }, - "pycharm": { - "name": "#%%\n" - } - }, - "outputs": [], - "source": [ - "from mindspore.amp import DynamicLossScaler\n", - "\n", - "# Instantiate loss function and optimizer\n", - "loss_fn = nn.CrossEntropyLoss()\n", - "optimizer = nn.SGD(model.trainable_params(), 1e-2)\n", - "\n", - "# Define LossScaler\n", - "loss_scaler = DynamicLossScaler(scale_value=2**16, scale_factor=2, scale_window=50)\n", - "\n", - "def forward_fn(data, label):\n", - " logits = model(data)\n", - " loss = loss_fn(logits, label)\n", - " # scale up the loss value\n", - " loss = loss_scaler.scale(loss)\n", - " return loss, logits" - ] - }, - { - "cell_type": "markdown", - "id": "09ca284d", - "metadata": {}, - "source": [ - "接下来进行函数变换,获得梯度函数。" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "id": "95715438-38f9-4476-afc3-5ff655b66966", - "metadata": { - "collapsed": false, - "jupyter": { - "outputs_hidden": false - }, - "pycharm": { - "name": "#%%\n" - } - }, - "outputs": [], - "source": [ - "grad_fn = value_and_grad(forward_fn, None, model.trainable_params(), has_aux=True)" - ] - }, - { - "cell_type": "markdown", - "id": "2ccec7bf", - "metadata": {}, - "source": [ - "定义训练step:计算当前梯度值并恢复损失。使用 `all_finite` 判断是否出现梯度下溢问题,如果无溢出,恢复梯度并更新网络权重;如果溢出,跳过此step。" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "id": "5a3c1cc3", - "metadata": { - "collapsed": false, - "jupyter": { - "outputs_hidden": false - }, - "pycharm": { - "name": "#%%\n" - } - }, - "outputs": [], - "source": [ - "from mindspore.amp import all_finite\n", - "\n", - "@ms.jit\n", - "def train_step(data, label):\n", - " (loss, _), grads = grad_fn(data, label)\n", - " loss = loss_scaler.unscale(loss)\n", - "\n", - " is_finite = all_finite(grads)\n", - " if is_finite:\n", - " grads = loss_scaler.unscale(grads)\n", - " optimizer(grads)\n", - " loss_scaler.adjust(is_finite)\n", - "\n", - " return loss" - ] - }, - { - "cell_type": "markdown", - "id": "70519642-58d2-485a-9614-773e270889dd", - "metadata": {}, - "source": [ - "最后,我们训练1个epoch,观察使用自动混合精度训练的loss收敛情况。" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "id": "0bb34af6-b3e2-494b-afbe-4ff68896552c", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "loss: 2.305425 [ 0/938]\n", - "loss: 2.289585 [100/938]\n", - "loss: 2.259094 [200/938]\n", - "loss: 2.176874 [300/938]\n", - "loss: 1.856715 [400/938]\n", - "loss: 1.398342 [500/938]\n", - "loss: 0.889620 [600/938]\n", - "loss: 0.709884 [700/938]\n", - "loss: 0.750509 [800/938]\n", - "loss: 0.482525 [900/938]\n" - ] - } - ], - "source": [ - "size = train_dataset.get_dataset_size()\n", - "model.set_train()\n", - "for batch, (data, label) in enumerate(train_dataset.create_tuple_iterator()):\n", - " loss = train_step(data, label)\n", - "\n", - " if batch % 100 == 0:\n", - " loss, current = loss.asnumpy(), batch\n", - " print(f\"loss: {loss:>7f} [{current:>3d}/{size:>3d}]\")" - ] - }, - { - "cell_type": "markdown", - "id": "17ff0777", - "metadata": {}, - "source": [ - "可以看到loss收敛趋势正常,没有出现溢出问题。" - ] - }, - { - "cell_type": "markdown", - "id": "e307279c-1694-44d6-96be-93a7fbb169eb", - "metadata": {}, - "source": [ - "## `Cell`配置自动混合精度\n", - "\n", - "MindSpore支持使用Cell封装完整计算图的编程范式,此时可以使用[mindspore.amp.build_train_network](https://www.mindspore.cn/docs/zh-CN/master/api_python/amp/mindspore.amp.build_train_network.html)接口,自动进行类型转换,并将Loss Scale传入,作为整图计算的一部分。\n", - "此时仅需要配置混合精度等级和`LossScaleManager`即可获得配置好自动混合精度的计算图。\n", - "\n", - "[mindspore.amp.FixedLossScaleManager](https://www.mindspore.cn/docs/zh-CN/master/api_python/amp/mindspore.amp.FixedLossScaleManager.html)和[mindspore.amp.DynamicLossScaleManager](https://www.mindspore.cn/docs/zh-CN/master/api_python/amp/mindspore.amp.DynamicLossScaleManager.html)是`Cell`配置自动混合精度的Loss scale管理接口,分别与`StaticLossScalar`和`DynamicLossScalar`对应,具体详见[mindspore.amp](https://www.mindspore.cn/docs/zh-CN/master/api_python/mindspore.amp.html)。\n", - "\n", - "> 使用`Cell`配置自动混合精度训练仅支持`GPU`和`Ascend`。" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "ff433a1f-30b5-486b-bfc2-218bca91d85d", - "metadata": {}, - "outputs": [], - "source": [ - "from mindspore.amp import build_train_network, FixedLossScaleManager\n", - "\n", - "model = Network()\n", - "loss_scale_manager = FixedLossScaleManager()\n", - "\n", - "model = build_train_network(model, optimizer, loss_fn, level=\"O2\", loss_scale_manager=loss_scale_manager)" - ] - }, - { - "cell_type": "markdown", - "id": "e09b86b5", - "metadata": {}, - "source": [ - "## `Model` 配置自动混合精度" - ] - }, - { - "cell_type": "markdown", - "id": "d5c8adf4-71bb-4061-8801-9a4f97168dc5", - "metadata": {}, - "source": [ - "[mindspore.train.Model](https://www.mindspore.cn/docs/zh-CN/master/api_python/train/mindspore.train.Model.html)是神经网络快速训练的高阶封装,其将`mindspore.amp.build_train_network`封装在内,因此同样只需要配置混合精度等级和`LossScaleManager`,即可进行自动混合精度训练。\n", - "\n", - "> 使用`Model`配置自动混合精度训练仅支持`GPU`和`Ascend`。" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "4e855c91", - "metadata": { - "collapsed": false, - "jupyter": { - "outputs_hidden": false - }, - "pycharm": { - "name": "#%%\n" - } - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "epoch: 1 step: 100, loss is 2.2883859\n", - "epoch: 1 step: 200, loss is 2.2612116\n", - "epoch: 1 step: 300, loss is 2.1563218\n", - "epoch: 1 step: 400, loss is 1.9420109\n", - "epoch: 1 step: 500, loss is 1.396821\n", - "epoch: 1 step: 600, loss is 1.0450488\n", - "epoch: 1 step: 700, loss is 0.69754004\n", - "epoch: 1 step: 800, loss is 0.6924556\n", - "epoch: 1 step: 900, loss is 0.57444984\n", - "...\n", - "epoch: 10 step: 58, loss is 0.13086069\n", - "epoch: 10 step: 158, loss is 0.07224723\n", - "epoch: 10 step: 258, loss is 0.08281057\n", - "epoch: 10 step: 358, loss is 0.09759849\n", - "epoch: 10 step: 458, loss is 0.17265382\n", - "epoch: 10 step: 558, loss is 0.10023793\n", - "epoch: 10 step: 658, loss is 0.08235697\n", - "epoch: 10 step: 758, loss is 0.10531154\n", - "epoch: 10 step: 858, loss is 0.19084263\n" - ] - } - ], - "source": [ - "from mindspore.train import Model, LossMonitor\n", - "# Initialize network\n", - "model = Network()\n", - "loss_fn = nn.CrossEntropyLoss()\n", - "optimizer = nn.SGD(model.trainable_params(), 1e-2)\n", - "\n", - "loss_scale_manager = FixedLossScaleManager()\n", - "trainer = Model(model, loss_fn=loss_fn, optimizer=optimizer, metrics={'accuracy'}, amp_level=\"O2\", loss_scale_manager=loss_scale_manager)\n", - "\n", - "loss_callback = LossMonitor(100)\n", - "trainer.train(10, train_dataset, callbacks=[loss_callback])" - ] - }, - { - "cell_type": "markdown", - "id": "7e24bdca", - "metadata": {}, - "source": [ - "> 图片引用自[automatic-mixed-precision](https://developer.nvidia.com/automatic-mixed-precision)。" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "MindSpore", - "language": "python", - "name": "mindspore" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.5 (default, Oct 25 2019, 15:51:11) \n[GCC 7.3.0]" - }, - "vscode": { - "interpreter": { - "hash": "8c9da313289c39257cb28b126d2dadd33153d4da4d524f730c81a4aaccbd2ca7" - } - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/tutorials/source_zh_cn/beginner/model.ipynb b/tutorials/source_zh_cn/beginner/model.ipynb index 0bc8993acdfd9c29d90d4935ee6959e613e88d50..bc559fbeb17b4f8e74e7169d1c442612ab343e82 100644 --- a/tutorials/source_zh_cn/beginner/model.ipynb +++ b/tutorials/source_zh_cn/beginner/model.ipynb @@ -6,7 +6,7 @@ "source": [ "[![下载Notebook](https://mindspore-website.obs.cn-north-4.myhuaweicloud.com/website-images/master/resource/_static/logo_notebook.svg)](https://mindspore-website.obs.cn-north-4.myhuaweicloud.com/notebook/master/tutorials/zh_cn/beginner/mindspore_model.ipynb) [![下载样例代码](https://mindspore-website.obs.cn-north-4.myhuaweicloud.com/website-images/master/resource/_static/logo_download_code.svg)](https://mindspore-website.obs.cn-north-4.myhuaweicloud.com/notebook/master/tutorials/zh_cn/beginner/mindspore_model.py) [![查看源文件](https://mindspore-website.obs.cn-north-4.myhuaweicloud.com/website-images/master/resource/_static/logo_source.svg)](https://gitee.com/mindspore/docs/blob/master/tutorials/source_zh_cn/beginner/model.ipynb)\n", "\n", - "[基本介绍](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/introduction.html) || [快速入门](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/quick_start.html) || [张量 Tensor](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/tensor.html) || [数据加载与处理](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/dataset.html) || **网络构建** || [函数式自动微分](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/autograd.html) || [模型训练](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/train.html) || [保存与加载](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/save_load.html) || [Graph Mode加速](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/accelerate_with_static_graph.html) || [自动混合精度](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/mixed_precision.html) ||" + "[基本介绍](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/introduction.html) || [快速入门](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/quick_start.html) || [张量 Tensor](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/tensor.html) || [数据加载与处理](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/dataset.html) || **网络构建** || [函数式自动微分](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/autograd.html) || [模型训练](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/train.html) || [保存与加载](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/save_load.html) || [Graph Mode加速](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/accelerate_with_static_graph.html) ||" ] }, { diff --git a/tutorials/source_zh_cn/beginner/quick_start.ipynb b/tutorials/source_zh_cn/beginner/quick_start.ipynb index 82f58972ca2be322574828329ff4e2519dda825e..cb0660d243ac6c8c8f1c50cc725fdd67e2286e44 100644 --- a/tutorials/source_zh_cn/beginner/quick_start.ipynb +++ b/tutorials/source_zh_cn/beginner/quick_start.ipynb @@ -6,7 +6,7 @@ "source": [ "[![下载Notebook](https://mindspore-website.obs.cn-north-4.myhuaweicloud.com/website-images/master/resource/_static/logo_notebook.svg)](https://mindspore-website.obs.cn-north-4.myhuaweicloud.com/notebook/master/tutorials/zh_cn/beginner/mindspore_quick_start.ipynb) [![下载样例代码](https://mindspore-website.obs.cn-north-4.myhuaweicloud.com/website-images/master/resource/_static/logo_download_code.svg)](https://mindspore-website.obs.cn-north-4.myhuaweicloud.com/notebook/master/tutorials/zh_cn/beginner/mindspore_quick_start.py) [![查看源文件](https://mindspore-website.obs.cn-north-4.myhuaweicloud.com/website-images/master/resource/_static/logo_source.svg)](https://gitee.com/mindspore/docs/blob/master/tutorials/source_zh_cn/beginner/quick_start.ipynb)\n", "\n", - "[基本介绍](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/introduction.html) || **快速入门** || [张量 Tensor](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/tensor.html) || [数据加载与处理](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/dataset.html) || [网络构建](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/model.html) || [函数式自动微分](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/autograd.html) || [模型训练](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/train.html) || [保存与加载](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/save_load.html) || [Graph Mode加速](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/accelerate_with_static_graph.html) || [自动混合精度](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/mixed_precision.html) ||" + "[基本介绍](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/introduction.html) || **快速入门** || [张量 Tensor](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/tensor.html) || [数据加载与处理](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/dataset.html) || [网络构建](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/model.html) || [函数式自动微分](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/autograd.html) || [模型训练](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/train.html) || [保存与加载](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/save_load.html) || [Graph Mode加速](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/accelerate_with_static_graph.html) ||" ] }, { diff --git a/tutorials/source_zh_cn/beginner/save_load.ipynb b/tutorials/source_zh_cn/beginner/save_load.ipynb index c6bd427e6aaf4c00de85ea1a92bb8231a2d75bf2..9badb402548f98efc68f41044af3f414594d85d3 100644 --- a/tutorials/source_zh_cn/beginner/save_load.ipynb +++ b/tutorials/source_zh_cn/beginner/save_load.ipynb @@ -8,7 +8,7 @@ "[![下载样例代码](https://mindspore-website.obs.cn-north-4.myhuaweicloud.com/website-images/master/resource/_static/logo_download_code.svg)](https://mindspore-website.obs.cn-north-4.myhuaweicloud.com/notebook/master/tutorials/zh_cn/beginner/mindspore_save_load.py) \n", "[![查看源文件](https://mindspore-website.obs.cn-north-4.myhuaweicloud.com/website-images/master/resource/_static/logo_source.svg)](https://gitee.com/mindspore/docs/blob/master/tutorials/source_zh_cn/beginner/save_load.ipynb)\n", "\n", - "[基本介绍](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/introduction.html) || [快速入门](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/quick_start.html) || [张量 Tensor](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/tensor.html) || [数据加载与处理](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/dataset.html) || [网络构建](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/model.html) || [函数式自动微分](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/autograd.html) || [模型训练](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/train.html) || **保存与加载** || [Graph Mode加速](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/accelerate_with_static_graph.html) || [自动混合精度](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/mixed_precision.html) ||" + "[基本介绍](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/introduction.html) || [快速入门](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/quick_start.html) || [张量 Tensor](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/tensor.html) || [数据加载与处理](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/dataset.html) || [网络构建](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/model.html) || [函数式自动微分](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/autograd.html) || [模型训练](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/train.html) || **保存与加载** || [Graph Mode加速](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/accelerate_with_static_graph.html) ||" ] }, { diff --git a/tutorials/source_zh_cn/beginner/tensor.ipynb b/tutorials/source_zh_cn/beginner/tensor.ipynb index 9a6696bad5195551ef037fa531ed550000dbb1eb..af801d8418ba6e1cbc75e039a69f8abc72b0b49d 100644 --- a/tutorials/source_zh_cn/beginner/tensor.ipynb +++ b/tutorials/source_zh_cn/beginner/tensor.ipynb @@ -7,7 +7,7 @@ "[![下载Notebook](https://mindspore-website.obs.cn-north-4.myhuaweicloud.com/website-images/master/resource/_static/logo_notebook.svg)](https://mindspore-website.obs.cn-north-4.myhuaweicloud.com/notebook/master/tutorials/zh_cn/beginner/mindspore_tensor.ipynb) [![下载样例代码](https://mindspore-website.obs.cn-north-4.myhuaweicloud.com/website-images/master/resource/_static/logo_download_code.svg)](https://mindspore-website.obs.cn-north-4.myhuaweicloud.com/notebook/master/tutorials/zh_cn/beginner/mindspore_tensor.py)\n", " [![查看源文件](https://mindspore-website.obs.cn-north-4.myhuaweicloud.com/website-images/master/resource/_static/logo_source.svg)](https://gitee.com/mindspore/docs/blob/master/tutorials/source_zh_cn/beginner/tensor.ipynb)\n", "\n", - "[基本介绍](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/introduction.html) || [快速入门](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/quick_start.html) || **张量 Tensor** || [数据加载与处理](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/dataset.html) || [网络构建](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/model.html) || [函数式自动微分](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/autograd.html) || [模型训练](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/train.html) || [保存与加载](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/save_load.html) || [Graph Mode加速](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/accelerate_with_static_graph.html) || [自动混合精度](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/mixed_precision.html) ||" + "[基本介绍](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/introduction.html) || [快速入门](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/quick_start.html) || **张量 Tensor** || [数据加载与处理](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/dataset.html) || [网络构建](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/model.html) || [函数式自动微分](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/autograd.html) || [模型训练](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/train.html) || [保存与加载](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/save_load.html) || [Graph Mode加速](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/accelerate_with_static_graph.html) ||" ] }, { diff --git a/tutorials/source_zh_cn/beginner/train.ipynb b/tutorials/source_zh_cn/beginner/train.ipynb index 3c5df84a60879bf739810d5fc75243ecfa1eb7c4..414a7284ed187bed92701e7bdaaded819f3c4e76 100644 --- a/tutorials/source_zh_cn/beginner/train.ipynb +++ b/tutorials/source_zh_cn/beginner/train.ipynb @@ -10,7 +10,7 @@ "source": [ "[![下载Notebook](https://mindspore-website.obs.cn-north-4.myhuaweicloud.com/website-images/master/resource/_static/logo_notebook.svg)](https://mindspore-website.obs.cn-north-4.myhuaweicloud.com/notebook/master/tutorials/zh_cn/beginner/mindspore_train.ipynb) [![下载样例代码](https://mindspore-website.obs.cn-north-4.myhuaweicloud.com/website-images/master/resource/_static/logo_download_code.svg)](https://mindspore-website.obs.cn-north-4.myhuaweicloud.com/notebook/master/tutorials/zh_cn/beginner/mindspore_train.py) [![查看源文件](https://mindspore-website.obs.cn-north-4.myhuaweicloud.com/website-images/master/resource/_static/logo_source.svg)](https://gitee.com/mindspore/docs/blob/master/tutorials/source_zh_cn/beginner/train.ipynb)\n", "\n", - "[基本介绍](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/introduction.html) || [快速入门](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/quick_start.html) || [张量 Tensor](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/tensor.html) || [数据加载与处理](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/dataset.html) || [网络构建](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/model.html) || [函数式自动微分](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/autograd.html) || **模型训练** || [保存与加载](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/save_load.html) || [Graph Mode加速](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/accelerate_with_static_graph.html) || [自动混合精度](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/mixed_precision.html) ||" + "[基本介绍](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/introduction.html) || [快速入门](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/quick_start.html) || [张量 Tensor](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/tensor.html) || [数据加载与处理](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/dataset.html) || [网络构建](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/model.html) || [函数式自动微分](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/autograd.html) || **模型训练** || [保存与加载](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/save_load.html) || [Graph Mode加速](https://www.mindspore.cn/tutorials/zh-CN/master/beginner/accelerate_with_static_graph.html) ||" ] }, { diff --git a/tutorials/source_zh_cn/index.rst b/tutorials/source_zh_cn/index.rst index 0889bc1948576c8e2e74b0a8a9b3f08ec3852988..4493ad476236bbef21514902e7b22458a123c839 100644 --- a/tutorials/source_zh_cn/index.rst +++ b/tutorials/source_zh_cn/index.rst @@ -21,7 +21,6 @@ MindSpore教程 beginner/train beginner/save_load beginner/accelerate_with_static_graph - beginner/mixed_precision .. toctree:: :glob: