From 231eae393c92cc94a21ef4f9884c0a85a6f74ca0 Mon Sep 17 00:00:00 2001 From: M-Zif Date: Tue, 1 Jun 2021 00:31:25 +0800 Subject: [PATCH 1/8] Translate Initializer.md --- .../source_en/initializer.md | 325 +++++++++++++++++- 1 file changed, 323 insertions(+), 2 deletions(-) diff --git a/docs/programming_guide/source_en/initializer.md b/docs/programming_guide/source_en/initializer.md index cd934f345f..1834c40fe4 100644 --- a/docs/programming_guide/source_en/initializer.md +++ b/docs/programming_guide/source_en/initializer.md @@ -1,5 +1,326 @@ # Initialization of Network Parameters -No English version right now, welcome to contribute. + - \ No newline at end of file +## Overview + +MindSpore provides a weight initialization module that allows users to initialize network parameters by wrapping operators and initializer methods to call strings, Initializer subclasses or custom Tensor. Class Initializer is the basic data structure in MindSpore to initialize network parameters,its subclasses contain several different types of data distributions (Zero, One, XavierUniform, HeUniform, HeNormal, Constant, Uniform, Normal, TruncatedNormal). The following is a detailed description of the two parameter initialization modes, the wrapper operator and the initializer method. + +## Parameter initialization using wrapper operator + +MindSpore provides various ways to initialize parameters and encapsulates parameter initialization in some operators. In this section, we will introduce the methods of initializing parameters in operators with parameter initialization function, taking the Conv2d operator as an example, and introduce the initialization of parameters in the network by strings, Initializer subclasses and custom Tensor, etc. The following code examples take Normal, a subclass of Initializer, as an example. Normal can be replaced by any of the Initializer subclasses. + +## String + +Using a string to initialize the network parameters, the content of the string needs to be consistent with the name of the subclass of Initializer, using the string approach to initialization will use the default parameters in the Initializer subclass, for example, using the string Normal is equivalent to using the Initializer subclass Normal (), code examples are as follows: + +```python +import numpy as np +import mindspore.nn as nn +from mindspore import Tensor +from mindspore.common import set_seed + +set_seed(1) + +input_data = Tensor(np.ones([1, 3, 16, 50], dtype=np.float32)) +net = nn.Conv2d(3, 64, 3, weight_init='Normal') +output = net(input_data) +print(output) +``` + +```python +[[[[ 3.10382620e-02 4.38603461e-02 4.38603461e-02 ... 4.38603461e-02 + 4.38603461e-02 1.38719045e-02] + [ 3.26051228e-02 3.54298912e-02 3.54298912e-02 ... 3.54298912e-02 + 3.54298912e-02 -5.54019120e-03] + [ 3.26051228e-02 3.54298912e-02 3.54298912e-02 ... 3.54298912e-02 + 3.54298912e-02 -5.54019120e-03] + ... + [ 3.26051228e-02 3.54298912e-02 3.54298912e-02 ... 3.54298912e-02 + 3.54298912e-02 -5.54019120e-03] + [ 3.26051228e-02 3.54298912e-02 3.54298912e-02 ... 3.54298912e-02 + 3.54298912e-02 -5.54019120e-03] + [ 9.66199022e-03 1.24104535e-02 1.24104535e-02 ... 1.24104535e-02 + 1.24104535e-02 -1.38977719e-02]] + + ... + + [[ 3.98553275e-02 -1.35465711e-03 -1.35465711e-03 ... -1.35465711e-03 + -1.35465711e-03 -1.00310734e-02] + [ 4.38403059e-03 -3.60766202e-02 -3.60766202e-02 ... -3.60766202e-02 + -3.60766202e-02 -2.95619294e-02] + [ 4.38403059e-03 -3.60766202e-02 -3.60766202e-02 ... -3.60766202e-02 + -3.60766202e-02 -2.95619294e-02] + ... + [ 4.38403059e-03 -3.60766202e-02 -3.60766202e-02 ... -3.60766202e-02 + -3.60766202e-02 -2.95619294e-02] + [ 4.38403059e-03 -3.60766202e-02 -3.60766202e-02 ... -3.60766202e-02 + -3.60766202e-02 -2.95619294e-02] + [ 1.33139016e-02 6.74417242e-05 6.74417242e-05 ... 6.74417242e-05 + 6.74417242e-05 -2.27325838e-02]]]] +``` + +## Initializer subclass + +Using the Initializer subclass to initialize the network parameters is similar to using a string to initialize the parameters, the difference is that using a string to initialize the parameters is using the default parameters of the Initializer subclass, such as to use the parameters in the Initializer subclass, you must use the Initializer subclass way to initialize the parameters To initialize, take Normal(0.2) as an example, the code sample is as follows: + +```python +import numpy as np +import mindspore.nn as nn +from mindspore import Tensor +from mindspore.common import set_seed +from mindspore.common.initializer import Normal + +set_seed(1) + +input_data = Tensor(np.ones([1, 3, 16, 50], dtype=np.float32)) +net = nn.Conv2d(3, 64, 3, weight_init=Normal(0.2)) +output = net(input_data) +print(output) +``` + +```python +[[[[ 6.2076533e-01 8.7720710e-01 8.7720710e-01 ... 8.7720710e-01 + 8.7720710e-01 2.7743810e-01] + [ 6.5210247e-01 7.0859784e-01 7.0859784e-01 ... 7.0859784e-01 + 7.0859784e-01 -1.1080378e-01] + [ 6.5210247e-01 7.0859784e-01 7.0859784e-01 ... 7.0859784e-01 + 7.0859784e-01 -1.1080378e-01] + ... + [ 6.5210247e-01 7.0859784e-01 7.0859784e-01 ... 7.0859784e-01 + 7.0859784e-01 -1.1080378e-01] + [ 6.5210247e-01 7.0859784e-01 7.0859784e-01 ... 7.0859784e-01 + 7.0859784e-01 -1.1080378e-01] + [ 1.9323981e-01 2.4820906e-01 2.4820906e-01 ... 2.4820906e-01 + 2.4820906e-01 -2.7795550e-01]] + + ... + + [[ 7.9710668e-01 -2.7093157e-02 -2.7093157e-02 ... -2.7093157e-02 + -2.7093157e-02 -2.0062150e-01] + [ 8.7680638e-02 -7.2153252e-01 -7.2153252e-01 ... -7.2153252e-01 + -7.2153252e-01 -5.9123868e-01] + [ 8.7680638e-02 -7.2153252e-01 -7.2153252e-01 ... -7.2153252e-01 + -7.2153252e-01 -5.9123868e-01] + ... + [ 8.7680638e-02 -7.2153252e-01 -7.2153252e-01 ... -7.2153252e-01 + -7.2153252e-01 -5.9123868e-01] + [ 8.7680638e-02 -7.2153252e-01 -7.2153252e-01 ... -7.2153252e-01 + -7.2153252e-01 -5.9123868e-01] + [ 2.6627803e-01 1.3488382e-03 1.3488382e-03 ... 1.3488382e-03 + 1.3488382e-03 -4.5465171e-01]]]] +``` + +## Customized Tensor + +In addition to the above two initialization methods, when the network wants to initialize the parameters with data types not available in MindSpore, the user can initialize the parameters by means of a custom Tensor, as shown in the following code example. + + + +```python +import numpy as np +import mindspore.nn as nn +from mindspore import Tensor +from mindspore import dtype as mstype + +weight = Tensor(np.ones([64, 3, 3, 3]), dtype=mstype.float32) +input_data = Tensor(np.ones([1, 3, 16, 50], dtype=np.float32)) +net = nn.Conv2d(3, 64, 3, weight_init=weight) +output = net(input_data) +print(output) +``` + +```python +[[[[12. 18. 18. ... 18. 18. 12.] + [18. 27. 27. ... 27. 27. 18.] + [18. 27. 27. ... 27. 27. 18.] + ... + [18. 27. 27. ... 27. 27. 18.] + [18. 27. 27. ... 27. 27. 18.] + [12. 18. 18. ... 18. 18. 12.]] + + ... + + [[12. 18. 18. ... 18. 18. 12.] + [18. 27. 27. ... 27. 27. 18.] + [18. 27. 27. ... 27. 27. 18.] + ... + [18. 27. 27. ... 27. 27. 18.] + [18. 27. 27. ... 27. 27. 18.] + [12. 18. 18. ... 18. 18. 12.]]]] +``` + +## Initializing parameters using the initializer method + +In the above code sample, it is given how to initialize the parameters in the network, such as using the nn layer to encapsulate the Conv2d operator in the network, the parameter weight_init is passed to the Conv2d operator as the data type to be initialized, and the operator will initialize the parameters by calling the Parameter class, which in turn calls the initializer method encapsulated in the Parameter class to complete the initialization of the parameter. However, some operators do not encapsulate the function of parameter initialization internally as Conv2d does, such as the weights of the Conv3d operator are passed into the Conv3d operator as parameters, and the initialization of the weights needs to be defined manually. + +When initializing parameters, you can use the initializer method to call different data types in the Initializer subclass to initialize the parameters and thus generate different types of data. + +When using initializer for parameter initialization, the parameters passed in are init, shape, and dtype: + + + +init: support passing in Tensor, str, and subclass of Initializer. + + + +shape: support passing list, tuple, int. + + + +dtype: support passing mindspore.dtype. + + + +### init parameter is Tensor + +Sample code is as follows. + +```python +import numpy as np +from mindspore import Tensor +from mindspore import dtype as mstype +from mindspore.common import set_seed +from mindspore.common.initializer import initializer +from mindspore.ops.operations import nn_ops as nps + +set_seed(1) + +input_data = Tensor(np.ones([16, 3, 10, 32, 32]), dtype=mstype.float32) +weight_init = Tensor(np.ones([32, 3, 4, 3, 3]), dtype=mstype.float32) +weight = initializer(weight_init, shape=[32, 3, 4, 3, 3]) +conv3d = nps.Conv3D(out_channel=32, kernel_size=(4, 3, 3)) +output = conv3d(input_data, weight) +print(output) +``` + +The output is as follows. + +```python +[[[[[108 108 108 ... 108 108 108] + [108 108 108 ... 108 108 108] + [108 108 108 ... 108 108 108] + ... + [108 108 108 ... 108 108 108] + [108 108 108 ... 108 108 108] + [108 108 108 ... 108 108 108]] + ... + [[108 108 108 ... 108 108 108] + [108 108 108 ... 108 108 108] + [108 108 108 ... 108 108 108] + ... + [108 108 108 ... 108 108 108] + [108 108 108 ... 108 108 108] + [108 108 108 ... 108 108 108]]]]] +``` + +### init parameter is str + +Sample code is as follows. + +```python +import numpy as np +from mindspore import Tensor +from mindspore import dtype as mstype +from mindspore.common import set_seed +from mindspore.common.initializer import initializer +from mindspore.ops.operations import nn_ops as nps + +set_seed(1) + +input_data = Tensor(np.ones([16, 3, 10, 32, 32]), dtype=mstype.float32) +weight = initializer('Normal', shape=[32, 3, 4, 3, 3], dtype=mstype.float32) +conv3d = nps.Conv3D(out_channel=32, kernel_size=(4, 3, 3)) +output = conv3d(input_data, weight) +print(output) +``` + +The output is as follows. + +```python +[[[[[0 0 0 ... 0 0 0] + [0 0 0 ... 0 0 0] + [0 0 0 ... 0 0 0]] + ... + [0 0 0 ... 0 0 0] + [0 0 0 ... 0 0 0] + [0 0 0 ... 0 0 0]] + ... + [[0 0 0 ... 0 0 0] + [0 0 0 ... 0 0 0] + [0 0 0 ... 0 0 0]] + ... + [0 0 0 ... 0 0 0] + [0 0 0 ... 0 0 0] + [0 0 0 ... 0 0 0]]]]] +``` + +### The init parameter is a subclass of Initializer + +Sample code is as follows. + +```python +import numpy as np +from mindspore import Tensor +from mindspore import dtype as mstype +from mindspore.common import set_seed +from mindspore.ops.operations import nn_ops as nps +from mindspore.common.initializer import Normal, initializer + +set_seed(1) + +input_data = Tensor(np.ones([16, 3, 10, 32, 32]), dtype=mstype.float32) +weight = initializer(Normal(0.2), shape=[32, 3, 4, 3, 3], dtype=mstype.float32) +conv3d = nps.Conv3D(out_channel=32, kernel_size=(4, 3, 3)) +output = conv3d(input_data, weight) +print(output) +``` + +``` +[[[[[0 0 0 ... 0 0 0] + [0 0 0 ... 0 0 0] + [0 0 0 ... 0 0 0]] + ... + [0 0 0 ... 0 0 0] + [0 0 0 ... 0 0 0] + [0 0 0 ... 0 0 0]] + ... + [[0 0 0 ... 0 0 0] + [0 0 0 ... 0 0 0] + [0 0 0 ... 0 0 0]] + ... + [0 0 0 ... 0 0 0] + [0 0 0 ... 0 0 0] + [0 0 0 ... 0 0 0]]]]] +``` + +## Application in Parameter + +The code example is as follows. + +```python +import numpy as np +from mindspore import dtype as mstype +from mindspore.common import set_seed +from mindspore.ops import operations as ops +from mindspore import Tensor, Parameter, context +from mindspore.common.initializer import Normal, initializer + +set_seed(1) + +weight1 = Parameter(initializer('Normal', [5, 4], mstype.float32), name="w1") +weight2 = Parameter(initializer(Normal(0.2), [5, 4], mstype.float32), name="w2") +input_data = Tensor(np.arange(20).reshape(5, 4), dtype=mstype.float32) +net = ops.Add() +output = net(input_data, weight1) +output = net(output, weight2) +print(output) +``` + +```python +[[-0.3305102 1.0412874 2.0412874 3.0412874] + [ 4.0412874 4.9479127 5.9479127 6.9479127] + [ 7.947912 9.063009 10.063009 11.063009 ] + [12.063009 13.536987 14.536987 14.857441 ] + [15.751231 17.073082 17.808317 19.364822 ]] +``` \ No newline at end of file -- Gitee From a0e854194bf8a01a198f8e024ce7e41fb3274c2a Mon Sep 17 00:00:00 2001 From: M-Zif Date: Tue, 1 Jun 2021 10:10:55 +0800 Subject: [PATCH 2/8] Translate Initializer --- .../source_en/initializer.ipynb | 326 ++++++++++++++++++ 1 file changed, 326 insertions(+) create mode 100644 docs/programming_guide/source_en/initializer.ipynb diff --git a/docs/programming_guide/source_en/initializer.ipynb b/docs/programming_guide/source_en/initializer.ipynb new file mode 100644 index 0000000000..1834c40fe4 --- /dev/null +++ b/docs/programming_guide/source_en/initializer.ipynb @@ -0,0 +1,326 @@ +# Initialization of Network Parameters + + + +## Overview + +MindSpore provides a weight initialization module that allows users to initialize network parameters by wrapping operators and initializer methods to call strings, Initializer subclasses or custom Tensor. Class Initializer is the basic data structure in MindSpore to initialize network parameters,its subclasses contain several different types of data distributions (Zero, One, XavierUniform, HeUniform, HeNormal, Constant, Uniform, Normal, TruncatedNormal). The following is a detailed description of the two parameter initialization modes, the wrapper operator and the initializer method. + +## Parameter initialization using wrapper operator + +MindSpore provides various ways to initialize parameters and encapsulates parameter initialization in some operators. In this section, we will introduce the methods of initializing parameters in operators with parameter initialization function, taking the Conv2d operator as an example, and introduce the initialization of parameters in the network by strings, Initializer subclasses and custom Tensor, etc. The following code examples take Normal, a subclass of Initializer, as an example. Normal can be replaced by any of the Initializer subclasses. + +## String + +Using a string to initialize the network parameters, the content of the string needs to be consistent with the name of the subclass of Initializer, using the string approach to initialization will use the default parameters in the Initializer subclass, for example, using the string Normal is equivalent to using the Initializer subclass Normal (), code examples are as follows: + +```python +import numpy as np +import mindspore.nn as nn +from mindspore import Tensor +from mindspore.common import set_seed + +set_seed(1) + +input_data = Tensor(np.ones([1, 3, 16, 50], dtype=np.float32)) +net = nn.Conv2d(3, 64, 3, weight_init='Normal') +output = net(input_data) +print(output) +``` + +```python +[[[[ 3.10382620e-02 4.38603461e-02 4.38603461e-02 ... 4.38603461e-02 + 4.38603461e-02 1.38719045e-02] + [ 3.26051228e-02 3.54298912e-02 3.54298912e-02 ... 3.54298912e-02 + 3.54298912e-02 -5.54019120e-03] + [ 3.26051228e-02 3.54298912e-02 3.54298912e-02 ... 3.54298912e-02 + 3.54298912e-02 -5.54019120e-03] + ... + [ 3.26051228e-02 3.54298912e-02 3.54298912e-02 ... 3.54298912e-02 + 3.54298912e-02 -5.54019120e-03] + [ 3.26051228e-02 3.54298912e-02 3.54298912e-02 ... 3.54298912e-02 + 3.54298912e-02 -5.54019120e-03] + [ 9.66199022e-03 1.24104535e-02 1.24104535e-02 ... 1.24104535e-02 + 1.24104535e-02 -1.38977719e-02]] + + ... + + [[ 3.98553275e-02 -1.35465711e-03 -1.35465711e-03 ... -1.35465711e-03 + -1.35465711e-03 -1.00310734e-02] + [ 4.38403059e-03 -3.60766202e-02 -3.60766202e-02 ... -3.60766202e-02 + -3.60766202e-02 -2.95619294e-02] + [ 4.38403059e-03 -3.60766202e-02 -3.60766202e-02 ... -3.60766202e-02 + -3.60766202e-02 -2.95619294e-02] + ... + [ 4.38403059e-03 -3.60766202e-02 -3.60766202e-02 ... -3.60766202e-02 + -3.60766202e-02 -2.95619294e-02] + [ 4.38403059e-03 -3.60766202e-02 -3.60766202e-02 ... -3.60766202e-02 + -3.60766202e-02 -2.95619294e-02] + [ 1.33139016e-02 6.74417242e-05 6.74417242e-05 ... 6.74417242e-05 + 6.74417242e-05 -2.27325838e-02]]]] +``` + +## Initializer subclass + +Using the Initializer subclass to initialize the network parameters is similar to using a string to initialize the parameters, the difference is that using a string to initialize the parameters is using the default parameters of the Initializer subclass, such as to use the parameters in the Initializer subclass, you must use the Initializer subclass way to initialize the parameters To initialize, take Normal(0.2) as an example, the code sample is as follows: + +```python +import numpy as np +import mindspore.nn as nn +from mindspore import Tensor +from mindspore.common import set_seed +from mindspore.common.initializer import Normal + +set_seed(1) + +input_data = Tensor(np.ones([1, 3, 16, 50], dtype=np.float32)) +net = nn.Conv2d(3, 64, 3, weight_init=Normal(0.2)) +output = net(input_data) +print(output) +``` + +```python +[[[[ 6.2076533e-01 8.7720710e-01 8.7720710e-01 ... 8.7720710e-01 + 8.7720710e-01 2.7743810e-01] + [ 6.5210247e-01 7.0859784e-01 7.0859784e-01 ... 7.0859784e-01 + 7.0859784e-01 -1.1080378e-01] + [ 6.5210247e-01 7.0859784e-01 7.0859784e-01 ... 7.0859784e-01 + 7.0859784e-01 -1.1080378e-01] + ... + [ 6.5210247e-01 7.0859784e-01 7.0859784e-01 ... 7.0859784e-01 + 7.0859784e-01 -1.1080378e-01] + [ 6.5210247e-01 7.0859784e-01 7.0859784e-01 ... 7.0859784e-01 + 7.0859784e-01 -1.1080378e-01] + [ 1.9323981e-01 2.4820906e-01 2.4820906e-01 ... 2.4820906e-01 + 2.4820906e-01 -2.7795550e-01]] + + ... + + [[ 7.9710668e-01 -2.7093157e-02 -2.7093157e-02 ... -2.7093157e-02 + -2.7093157e-02 -2.0062150e-01] + [ 8.7680638e-02 -7.2153252e-01 -7.2153252e-01 ... -7.2153252e-01 + -7.2153252e-01 -5.9123868e-01] + [ 8.7680638e-02 -7.2153252e-01 -7.2153252e-01 ... -7.2153252e-01 + -7.2153252e-01 -5.9123868e-01] + ... + [ 8.7680638e-02 -7.2153252e-01 -7.2153252e-01 ... -7.2153252e-01 + -7.2153252e-01 -5.9123868e-01] + [ 8.7680638e-02 -7.2153252e-01 -7.2153252e-01 ... -7.2153252e-01 + -7.2153252e-01 -5.9123868e-01] + [ 2.6627803e-01 1.3488382e-03 1.3488382e-03 ... 1.3488382e-03 + 1.3488382e-03 -4.5465171e-01]]]] +``` + +## Customized Tensor + +In addition to the above two initialization methods, when the network wants to initialize the parameters with data types not available in MindSpore, the user can initialize the parameters by means of a custom Tensor, as shown in the following code example. + + + +```python +import numpy as np +import mindspore.nn as nn +from mindspore import Tensor +from mindspore import dtype as mstype + +weight = Tensor(np.ones([64, 3, 3, 3]), dtype=mstype.float32) +input_data = Tensor(np.ones([1, 3, 16, 50], dtype=np.float32)) +net = nn.Conv2d(3, 64, 3, weight_init=weight) +output = net(input_data) +print(output) +``` + +```python +[[[[12. 18. 18. ... 18. 18. 12.] + [18. 27. 27. ... 27. 27. 18.] + [18. 27. 27. ... 27. 27. 18.] + ... + [18. 27. 27. ... 27. 27. 18.] + [18. 27. 27. ... 27. 27. 18.] + [12. 18. 18. ... 18. 18. 12.]] + + ... + + [[12. 18. 18. ... 18. 18. 12.] + [18. 27. 27. ... 27. 27. 18.] + [18. 27. 27. ... 27. 27. 18.] + ... + [18. 27. 27. ... 27. 27. 18.] + [18. 27. 27. ... 27. 27. 18.] + [12. 18. 18. ... 18. 18. 12.]]]] +``` + +## Initializing parameters using the initializer method + +In the above code sample, it is given how to initialize the parameters in the network, such as using the nn layer to encapsulate the Conv2d operator in the network, the parameter weight_init is passed to the Conv2d operator as the data type to be initialized, and the operator will initialize the parameters by calling the Parameter class, which in turn calls the initializer method encapsulated in the Parameter class to complete the initialization of the parameter. However, some operators do not encapsulate the function of parameter initialization internally as Conv2d does, such as the weights of the Conv3d operator are passed into the Conv3d operator as parameters, and the initialization of the weights needs to be defined manually. + +When initializing parameters, you can use the initializer method to call different data types in the Initializer subclass to initialize the parameters and thus generate different types of data. + +When using initializer for parameter initialization, the parameters passed in are init, shape, and dtype: + + + +init: support passing in Tensor, str, and subclass of Initializer. + + + +shape: support passing list, tuple, int. + + + +dtype: support passing mindspore.dtype. + + + +### init parameter is Tensor + +Sample code is as follows. + +```python +import numpy as np +from mindspore import Tensor +from mindspore import dtype as mstype +from mindspore.common import set_seed +from mindspore.common.initializer import initializer +from mindspore.ops.operations import nn_ops as nps + +set_seed(1) + +input_data = Tensor(np.ones([16, 3, 10, 32, 32]), dtype=mstype.float32) +weight_init = Tensor(np.ones([32, 3, 4, 3, 3]), dtype=mstype.float32) +weight = initializer(weight_init, shape=[32, 3, 4, 3, 3]) +conv3d = nps.Conv3D(out_channel=32, kernel_size=(4, 3, 3)) +output = conv3d(input_data, weight) +print(output) +``` + +The output is as follows. + +```python +[[[[[108 108 108 ... 108 108 108] + [108 108 108 ... 108 108 108] + [108 108 108 ... 108 108 108] + ... + [108 108 108 ... 108 108 108] + [108 108 108 ... 108 108 108] + [108 108 108 ... 108 108 108]] + ... + [[108 108 108 ... 108 108 108] + [108 108 108 ... 108 108 108] + [108 108 108 ... 108 108 108] + ... + [108 108 108 ... 108 108 108] + [108 108 108 ... 108 108 108] + [108 108 108 ... 108 108 108]]]]] +``` + +### init parameter is str + +Sample code is as follows. + +```python +import numpy as np +from mindspore import Tensor +from mindspore import dtype as mstype +from mindspore.common import set_seed +from mindspore.common.initializer import initializer +from mindspore.ops.operations import nn_ops as nps + +set_seed(1) + +input_data = Tensor(np.ones([16, 3, 10, 32, 32]), dtype=mstype.float32) +weight = initializer('Normal', shape=[32, 3, 4, 3, 3], dtype=mstype.float32) +conv3d = nps.Conv3D(out_channel=32, kernel_size=(4, 3, 3)) +output = conv3d(input_data, weight) +print(output) +``` + +The output is as follows. + +```python +[[[[[0 0 0 ... 0 0 0] + [0 0 0 ... 0 0 0] + [0 0 0 ... 0 0 0]] + ... + [0 0 0 ... 0 0 0] + [0 0 0 ... 0 0 0] + [0 0 0 ... 0 0 0]] + ... + [[0 0 0 ... 0 0 0] + [0 0 0 ... 0 0 0] + [0 0 0 ... 0 0 0]] + ... + [0 0 0 ... 0 0 0] + [0 0 0 ... 0 0 0] + [0 0 0 ... 0 0 0]]]]] +``` + +### The init parameter is a subclass of Initializer + +Sample code is as follows. + +```python +import numpy as np +from mindspore import Tensor +from mindspore import dtype as mstype +from mindspore.common import set_seed +from mindspore.ops.operations import nn_ops as nps +from mindspore.common.initializer import Normal, initializer + +set_seed(1) + +input_data = Tensor(np.ones([16, 3, 10, 32, 32]), dtype=mstype.float32) +weight = initializer(Normal(0.2), shape=[32, 3, 4, 3, 3], dtype=mstype.float32) +conv3d = nps.Conv3D(out_channel=32, kernel_size=(4, 3, 3)) +output = conv3d(input_data, weight) +print(output) +``` + +``` +[[[[[0 0 0 ... 0 0 0] + [0 0 0 ... 0 0 0] + [0 0 0 ... 0 0 0]] + ... + [0 0 0 ... 0 0 0] + [0 0 0 ... 0 0 0] + [0 0 0 ... 0 0 0]] + ... + [[0 0 0 ... 0 0 0] + [0 0 0 ... 0 0 0] + [0 0 0 ... 0 0 0]] + ... + [0 0 0 ... 0 0 0] + [0 0 0 ... 0 0 0] + [0 0 0 ... 0 0 0]]]]] +``` + +## Application in Parameter + +The code example is as follows. + +```python +import numpy as np +from mindspore import dtype as mstype +from mindspore.common import set_seed +from mindspore.ops import operations as ops +from mindspore import Tensor, Parameter, context +from mindspore.common.initializer import Normal, initializer + +set_seed(1) + +weight1 = Parameter(initializer('Normal', [5, 4], mstype.float32), name="w1") +weight2 = Parameter(initializer(Normal(0.2), [5, 4], mstype.float32), name="w2") +input_data = Tensor(np.arange(20).reshape(5, 4), dtype=mstype.float32) +net = ops.Add() +output = net(input_data, weight1) +output = net(output, weight2) +print(output) +``` + +```python +[[-0.3305102 1.0412874 2.0412874 3.0412874] + [ 4.0412874 4.9479127 5.9479127 6.9479127] + [ 7.947912 9.063009 10.063009 11.063009 ] + [12.063009 13.536987 14.536987 14.857441 ] + [15.751231 17.073082 17.808317 19.364822 ]] +``` \ No newline at end of file -- Gitee From 187384e802566c246cac70f9361b6d21d71005f0 Mon Sep 17 00:00:00 2001 From: M-Zif Date: Tue, 1 Jun 2021 10:19:27 +0800 Subject: [PATCH 3/8] Translation of Initializer --- .../programming_guide/source_en/initialize.md | 326 ++++++++++++++++++ 1 file changed, 326 insertions(+) create mode 100644 docs/programming_guide/source_en/initialize.md diff --git a/docs/programming_guide/source_en/initialize.md b/docs/programming_guide/source_en/initialize.md new file mode 100644 index 0000000000..1834c40fe4 --- /dev/null +++ b/docs/programming_guide/source_en/initialize.md @@ -0,0 +1,326 @@ +# Initialization of Network Parameters + + + +## Overview + +MindSpore provides a weight initialization module that allows users to initialize network parameters by wrapping operators and initializer methods to call strings, Initializer subclasses or custom Tensor. Class Initializer is the basic data structure in MindSpore to initialize network parameters,its subclasses contain several different types of data distributions (Zero, One, XavierUniform, HeUniform, HeNormal, Constant, Uniform, Normal, TruncatedNormal). The following is a detailed description of the two parameter initialization modes, the wrapper operator and the initializer method. + +## Parameter initialization using wrapper operator + +MindSpore provides various ways to initialize parameters and encapsulates parameter initialization in some operators. In this section, we will introduce the methods of initializing parameters in operators with parameter initialization function, taking the Conv2d operator as an example, and introduce the initialization of parameters in the network by strings, Initializer subclasses and custom Tensor, etc. The following code examples take Normal, a subclass of Initializer, as an example. Normal can be replaced by any of the Initializer subclasses. + +## String + +Using a string to initialize the network parameters, the content of the string needs to be consistent with the name of the subclass of Initializer, using the string approach to initialization will use the default parameters in the Initializer subclass, for example, using the string Normal is equivalent to using the Initializer subclass Normal (), code examples are as follows: + +```python +import numpy as np +import mindspore.nn as nn +from mindspore import Tensor +from mindspore.common import set_seed + +set_seed(1) + +input_data = Tensor(np.ones([1, 3, 16, 50], dtype=np.float32)) +net = nn.Conv2d(3, 64, 3, weight_init='Normal') +output = net(input_data) +print(output) +``` + +```python +[[[[ 3.10382620e-02 4.38603461e-02 4.38603461e-02 ... 4.38603461e-02 + 4.38603461e-02 1.38719045e-02] + [ 3.26051228e-02 3.54298912e-02 3.54298912e-02 ... 3.54298912e-02 + 3.54298912e-02 -5.54019120e-03] + [ 3.26051228e-02 3.54298912e-02 3.54298912e-02 ... 3.54298912e-02 + 3.54298912e-02 -5.54019120e-03] + ... + [ 3.26051228e-02 3.54298912e-02 3.54298912e-02 ... 3.54298912e-02 + 3.54298912e-02 -5.54019120e-03] + [ 3.26051228e-02 3.54298912e-02 3.54298912e-02 ... 3.54298912e-02 + 3.54298912e-02 -5.54019120e-03] + [ 9.66199022e-03 1.24104535e-02 1.24104535e-02 ... 1.24104535e-02 + 1.24104535e-02 -1.38977719e-02]] + + ... + + [[ 3.98553275e-02 -1.35465711e-03 -1.35465711e-03 ... -1.35465711e-03 + -1.35465711e-03 -1.00310734e-02] + [ 4.38403059e-03 -3.60766202e-02 -3.60766202e-02 ... -3.60766202e-02 + -3.60766202e-02 -2.95619294e-02] + [ 4.38403059e-03 -3.60766202e-02 -3.60766202e-02 ... -3.60766202e-02 + -3.60766202e-02 -2.95619294e-02] + ... + [ 4.38403059e-03 -3.60766202e-02 -3.60766202e-02 ... -3.60766202e-02 + -3.60766202e-02 -2.95619294e-02] + [ 4.38403059e-03 -3.60766202e-02 -3.60766202e-02 ... -3.60766202e-02 + -3.60766202e-02 -2.95619294e-02] + [ 1.33139016e-02 6.74417242e-05 6.74417242e-05 ... 6.74417242e-05 + 6.74417242e-05 -2.27325838e-02]]]] +``` + +## Initializer subclass + +Using the Initializer subclass to initialize the network parameters is similar to using a string to initialize the parameters, the difference is that using a string to initialize the parameters is using the default parameters of the Initializer subclass, such as to use the parameters in the Initializer subclass, you must use the Initializer subclass way to initialize the parameters To initialize, take Normal(0.2) as an example, the code sample is as follows: + +```python +import numpy as np +import mindspore.nn as nn +from mindspore import Tensor +from mindspore.common import set_seed +from mindspore.common.initializer import Normal + +set_seed(1) + +input_data = Tensor(np.ones([1, 3, 16, 50], dtype=np.float32)) +net = nn.Conv2d(3, 64, 3, weight_init=Normal(0.2)) +output = net(input_data) +print(output) +``` + +```python +[[[[ 6.2076533e-01 8.7720710e-01 8.7720710e-01 ... 8.7720710e-01 + 8.7720710e-01 2.7743810e-01] + [ 6.5210247e-01 7.0859784e-01 7.0859784e-01 ... 7.0859784e-01 + 7.0859784e-01 -1.1080378e-01] + [ 6.5210247e-01 7.0859784e-01 7.0859784e-01 ... 7.0859784e-01 + 7.0859784e-01 -1.1080378e-01] + ... + [ 6.5210247e-01 7.0859784e-01 7.0859784e-01 ... 7.0859784e-01 + 7.0859784e-01 -1.1080378e-01] + [ 6.5210247e-01 7.0859784e-01 7.0859784e-01 ... 7.0859784e-01 + 7.0859784e-01 -1.1080378e-01] + [ 1.9323981e-01 2.4820906e-01 2.4820906e-01 ... 2.4820906e-01 + 2.4820906e-01 -2.7795550e-01]] + + ... + + [[ 7.9710668e-01 -2.7093157e-02 -2.7093157e-02 ... -2.7093157e-02 + -2.7093157e-02 -2.0062150e-01] + [ 8.7680638e-02 -7.2153252e-01 -7.2153252e-01 ... -7.2153252e-01 + -7.2153252e-01 -5.9123868e-01] + [ 8.7680638e-02 -7.2153252e-01 -7.2153252e-01 ... -7.2153252e-01 + -7.2153252e-01 -5.9123868e-01] + ... + [ 8.7680638e-02 -7.2153252e-01 -7.2153252e-01 ... -7.2153252e-01 + -7.2153252e-01 -5.9123868e-01] + [ 8.7680638e-02 -7.2153252e-01 -7.2153252e-01 ... -7.2153252e-01 + -7.2153252e-01 -5.9123868e-01] + [ 2.6627803e-01 1.3488382e-03 1.3488382e-03 ... 1.3488382e-03 + 1.3488382e-03 -4.5465171e-01]]]] +``` + +## Customized Tensor + +In addition to the above two initialization methods, when the network wants to initialize the parameters with data types not available in MindSpore, the user can initialize the parameters by means of a custom Tensor, as shown in the following code example. + + + +```python +import numpy as np +import mindspore.nn as nn +from mindspore import Tensor +from mindspore import dtype as mstype + +weight = Tensor(np.ones([64, 3, 3, 3]), dtype=mstype.float32) +input_data = Tensor(np.ones([1, 3, 16, 50], dtype=np.float32)) +net = nn.Conv2d(3, 64, 3, weight_init=weight) +output = net(input_data) +print(output) +``` + +```python +[[[[12. 18. 18. ... 18. 18. 12.] + [18. 27. 27. ... 27. 27. 18.] + [18. 27. 27. ... 27. 27. 18.] + ... + [18. 27. 27. ... 27. 27. 18.] + [18. 27. 27. ... 27. 27. 18.] + [12. 18. 18. ... 18. 18. 12.]] + + ... + + [[12. 18. 18. ... 18. 18. 12.] + [18. 27. 27. ... 27. 27. 18.] + [18. 27. 27. ... 27. 27. 18.] + ... + [18. 27. 27. ... 27. 27. 18.] + [18. 27. 27. ... 27. 27. 18.] + [12. 18. 18. ... 18. 18. 12.]]]] +``` + +## Initializing parameters using the initializer method + +In the above code sample, it is given how to initialize the parameters in the network, such as using the nn layer to encapsulate the Conv2d operator in the network, the parameter weight_init is passed to the Conv2d operator as the data type to be initialized, and the operator will initialize the parameters by calling the Parameter class, which in turn calls the initializer method encapsulated in the Parameter class to complete the initialization of the parameter. However, some operators do not encapsulate the function of parameter initialization internally as Conv2d does, such as the weights of the Conv3d operator are passed into the Conv3d operator as parameters, and the initialization of the weights needs to be defined manually. + +When initializing parameters, you can use the initializer method to call different data types in the Initializer subclass to initialize the parameters and thus generate different types of data. + +When using initializer for parameter initialization, the parameters passed in are init, shape, and dtype: + + + +init: support passing in Tensor, str, and subclass of Initializer. + + + +shape: support passing list, tuple, int. + + + +dtype: support passing mindspore.dtype. + + + +### init parameter is Tensor + +Sample code is as follows. + +```python +import numpy as np +from mindspore import Tensor +from mindspore import dtype as mstype +from mindspore.common import set_seed +from mindspore.common.initializer import initializer +from mindspore.ops.operations import nn_ops as nps + +set_seed(1) + +input_data = Tensor(np.ones([16, 3, 10, 32, 32]), dtype=mstype.float32) +weight_init = Tensor(np.ones([32, 3, 4, 3, 3]), dtype=mstype.float32) +weight = initializer(weight_init, shape=[32, 3, 4, 3, 3]) +conv3d = nps.Conv3D(out_channel=32, kernel_size=(4, 3, 3)) +output = conv3d(input_data, weight) +print(output) +``` + +The output is as follows. + +```python +[[[[[108 108 108 ... 108 108 108] + [108 108 108 ... 108 108 108] + [108 108 108 ... 108 108 108] + ... + [108 108 108 ... 108 108 108] + [108 108 108 ... 108 108 108] + [108 108 108 ... 108 108 108]] + ... + [[108 108 108 ... 108 108 108] + [108 108 108 ... 108 108 108] + [108 108 108 ... 108 108 108] + ... + [108 108 108 ... 108 108 108] + [108 108 108 ... 108 108 108] + [108 108 108 ... 108 108 108]]]]] +``` + +### init parameter is str + +Sample code is as follows. + +```python +import numpy as np +from mindspore import Tensor +from mindspore import dtype as mstype +from mindspore.common import set_seed +from mindspore.common.initializer import initializer +from mindspore.ops.operations import nn_ops as nps + +set_seed(1) + +input_data = Tensor(np.ones([16, 3, 10, 32, 32]), dtype=mstype.float32) +weight = initializer('Normal', shape=[32, 3, 4, 3, 3], dtype=mstype.float32) +conv3d = nps.Conv3D(out_channel=32, kernel_size=(4, 3, 3)) +output = conv3d(input_data, weight) +print(output) +``` + +The output is as follows. + +```python +[[[[[0 0 0 ... 0 0 0] + [0 0 0 ... 0 0 0] + [0 0 0 ... 0 0 0]] + ... + [0 0 0 ... 0 0 0] + [0 0 0 ... 0 0 0] + [0 0 0 ... 0 0 0]] + ... + [[0 0 0 ... 0 0 0] + [0 0 0 ... 0 0 0] + [0 0 0 ... 0 0 0]] + ... + [0 0 0 ... 0 0 0] + [0 0 0 ... 0 0 0] + [0 0 0 ... 0 0 0]]]]] +``` + +### The init parameter is a subclass of Initializer + +Sample code is as follows. + +```python +import numpy as np +from mindspore import Tensor +from mindspore import dtype as mstype +from mindspore.common import set_seed +from mindspore.ops.operations import nn_ops as nps +from mindspore.common.initializer import Normal, initializer + +set_seed(1) + +input_data = Tensor(np.ones([16, 3, 10, 32, 32]), dtype=mstype.float32) +weight = initializer(Normal(0.2), shape=[32, 3, 4, 3, 3], dtype=mstype.float32) +conv3d = nps.Conv3D(out_channel=32, kernel_size=(4, 3, 3)) +output = conv3d(input_data, weight) +print(output) +``` + +``` +[[[[[0 0 0 ... 0 0 0] + [0 0 0 ... 0 0 0] + [0 0 0 ... 0 0 0]] + ... + [0 0 0 ... 0 0 0] + [0 0 0 ... 0 0 0] + [0 0 0 ... 0 0 0]] + ... + [[0 0 0 ... 0 0 0] + [0 0 0 ... 0 0 0] + [0 0 0 ... 0 0 0]] + ... + [0 0 0 ... 0 0 0] + [0 0 0 ... 0 0 0] + [0 0 0 ... 0 0 0]]]]] +``` + +## Application in Parameter + +The code example is as follows. + +```python +import numpy as np +from mindspore import dtype as mstype +from mindspore.common import set_seed +from mindspore.ops import operations as ops +from mindspore import Tensor, Parameter, context +from mindspore.common.initializer import Normal, initializer + +set_seed(1) + +weight1 = Parameter(initializer('Normal', [5, 4], mstype.float32), name="w1") +weight2 = Parameter(initializer(Normal(0.2), [5, 4], mstype.float32), name="w2") +input_data = Tensor(np.arange(20).reshape(5, 4), dtype=mstype.float32) +net = ops.Add() +output = net(input_data, weight1) +output = net(output, weight2) +print(output) +``` + +```python +[[-0.3305102 1.0412874 2.0412874 3.0412874] + [ 4.0412874 4.9479127 5.9479127 6.9479127] + [ 7.947912 9.063009 10.063009 11.063009 ] + [12.063009 13.536987 14.536987 14.857441 ] + [15.751231 17.073082 17.808317 19.364822 ]] +``` \ No newline at end of file -- Gitee From 9b490d633f65360156fa72ecf5180d16f30b2624 Mon Sep 17 00:00:00 2001 From: M-Zif Date: Tue, 1 Jun 2021 10:29:11 +0800 Subject: [PATCH 4/8] Translate --- docs/programming_guide/source_en/initialize.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/programming_guide/source_en/initialize.md b/docs/programming_guide/source_en/initialize.md index 1834c40fe4..522cb3b654 100644 --- a/docs/programming_guide/source_en/initialize.md +++ b/docs/programming_guide/source_en/initialize.md @@ -172,7 +172,7 @@ When using initializer for parameter initialization, the parameters passed in ar -### init parameter is Tensor +## init parameter is Tensor Sample code is as follows. @@ -214,7 +214,7 @@ The output is as follows. [108 108 108 ... 108 108 108]]]]] ``` -### init parameter is str +## init parameter is str Sample code is as follows. @@ -255,7 +255,7 @@ The output is as follows. [0 0 0 ... 0 0 0]]]]] ``` -### The init parameter is a subclass of Initializer +## The init parameter is a subclass of Initializer Sample code is as follows. -- Gitee From ce9ddbba7d248160cda8fc264e2dcf187e9b8482 Mon Sep 17 00:00:00 2001 From: M-Zif Date: Tue, 1 Jun 2021 10:52:17 +0800 Subject: [PATCH 5/8] Translate --- .../source_en/initialize.ipynb | 484 ++++++++++++++++++ 1 file changed, 484 insertions(+) create mode 100644 docs/programming_guide/source_en/initialize.ipynb diff --git a/docs/programming_guide/source_en/initialize.ipynb b/docs/programming_guide/source_en/initialize.ipynb new file mode 100644 index 0000000000..b2271fb094 --- /dev/null +++ b/docs/programming_guide/source_en/initialize.ipynb @@ -0,0 +1,484 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Initialization of Network Parameters\n", + "\n", + "[![](https://gitee.com/mindspore/docs/raw/master/resource/_static/logo_source.png)](https://gitee.com/mindspore/docs/blob/master/docs/programming_guide/source_zh_cn/initializer.ipynb) [![](https://gitee.com/mindspore/docs/raw/master/resource/_static/logo_notebook.png)](https://obs.dualstack.cn-north-4.myhuaweicloud.com/mindspore-website/notebook/master/programming_guide/mindspore_initializer.ipynb) [![](https://gitee.com/mindspore/docs/raw/master/resource/_static/logo_modelarts.png)](https://authoring-modelarts-cnnorth4.huaweicloud.com/console/lab?share-url-b64=aHR0cHM6Ly9vYnMuZHVhbHN0YWNrLmNuLW5vcnRoLTQubXlodWF3ZWljbG91ZC5jb20vbWluZHNwb3JlLXdlYnNpdGUvbm90ZWJvb2svbW9kZWxhcnRzL3Byb2dyYW1taW5nX2d1aWRlL21pbmRzcG9yZV9pbml0aWFsaXplci5pcHluYg==&imagename=MindSpore1.1.1)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Overview\n", + "\n", + "MindSpore provides a weight initialization module that allows users to initialize network parameters by wrapping operators and initializer methods to call strings, Initializer subclasses or custom Tensor. Class Initializer is the basic data structure in MindSpore to initialize network parameters,its subclasses contain several different types of data distributions (Zero, One, XavierUniform, HeUniform, HeNormal, Constant, Uniform, Normal, TruncatedNormal). The following is a detailed description of the two parameter initialization modes, the wrapper operator and the initializer method." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Parameter initialization using wrapper operator \n", + "MindSpore provides various ways to initialize parameters and encapsulates parameter initialization in some operators. In this section, we will introduce the methods of initializing parameters in operators with parameter initialization function, taking the Conv2d operator as an example, and introduce the initialization of parameters in the network by strings, Initializer subclasses and custom Tensor, etc. The following code examples take Normal, a subclass of Initializer, as an example. Normal can be replaced by any of the Initializer subclasses." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### String \n", + "Using a string to initialize the network parameters, the content of the string needs to be consistent with the name of the subclass of Initializer, using the string approach to initialization will use the default parameters in the Initializer subclass, for example, using the string Normal is equivalent to using the Initializer subclass Normal (), code examples are as follows:" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[[[[ 3.10382620e-02 4.38603461e-02 4.38603461e-02 ... 4.38603461e-02\n", + " 4.38603461e-02 1.38719045e-02]\n", + " [ 3.26051228e-02 3.54298912e-02 3.54298912e-02 ... 3.54298912e-02\n", + " 3.54298912e-02 -5.54019120e-03]\n", + " [ 3.26051228e-02 3.54298912e-02 3.54298912e-02 ... 3.54298912e-02\n", + " 3.54298912e-02 -5.54019120e-03]\n", + " ...\n", + " [ 3.26051228e-02 3.54298912e-02 3.54298912e-02 ... 3.54298912e-02\n", + " 3.54298912e-02 -5.54019120e-03]\n", + " [ 3.26051228e-02 3.54298912e-02 3.54298912e-02 ... 3.54298912e-02\n", + " 3.54298912e-02 -5.54019120e-03]\n", + " [ 9.66199022e-03 1.24104535e-02 1.24104535e-02 ... 1.24104535e-02\n", + " 1.24104535e-02 -1.38977719e-02]]\n", + "\n", + " ...\n", + "\n", + " [[ 3.98553275e-02 -1.35465711e-03 -1.35465711e-03 ... -1.35465711e-03\n", + " -1.35465711e-03 -1.00310734e-02]\n", + " [ 4.38403059e-03 -3.60766202e-02 -3.60766202e-02 ... -3.60766202e-02\n", + " -3.60766202e-02 -2.95619294e-02]\n", + " [ 4.38403059e-03 -3.60766202e-02 -3.60766202e-02 ... -3.60766202e-02\n", + " -3.60766202e-02 -2.95619294e-02]\n", + " ...\n", + " [ 4.38403059e-03 -3.60766202e-02 -3.60766202e-02 ... -3.60766202e-02\n", + " -3.60766202e-02 -2.95619294e-02]\n", + " [ 4.38403059e-03 -3.60766202e-02 -3.60766202e-02 ... -3.60766202e-02\n", + " -3.60766202e-02 -2.95619294e-02]\n", + " [ 1.33139016e-02 6.74417242e-05 6.74417242e-05 ... 6.74417242e-05\n", + " 6.74417242e-05 -2.27325838e-02]]]]\n" + ] + } + ], + "source": [ + "import numpy as np\n", + "import mindspore.nn as nn\n", + "from mindspore import Tensor\n", + "from mindspore.common import set_seed\n", + "\n", + "set_seed(1)\n", + "\n", + "input_data = Tensor(np.ones([1, 3, 16, 50], dtype=np.float32))\n", + "net = nn.Conv2d(3, 64, 3, weight_init='Normal')\n", + "output = net(input_data)\n", + "print(output)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Initializer subclass \n", + "Using the Initializer subclass to initialize the network parameters is similar to using a string to initialize the parameters, the difference is that using a string to initialize the parameters is using the default parameters of the Initializer subclass, such as to use the parameters in the Initializer subclass, you must use the Initializer subclass way to initialize the parameters To initialize, take Normal(0.2) as an example, the code sample is as follows:" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[[[[ 6.2076533e-01 8.7720710e-01 8.7720710e-01 ... 8.7720710e-01\n", + " 8.7720710e-01 2.7743810e-01]\n", + " [ 6.5210247e-01 7.0859784e-01 7.0859784e-01 ... 7.0859784e-01\n", + " 7.0859784e-01 -1.1080378e-01]\n", + " [ 6.5210247e-01 7.0859784e-01 7.0859784e-01 ... 7.0859784e-01\n", + " 7.0859784e-01 -1.1080378e-01]\n", + " ...\n", + " [ 6.5210247e-01 7.0859784e-01 7.0859784e-01 ... 7.0859784e-01\n", + " 7.0859784e-01 -1.1080378e-01]\n", + " [ 6.5210247e-01 7.0859784e-01 7.0859784e-01 ... 7.0859784e-01\n", + " 7.0859784e-01 -1.1080378e-01]\n", + " [ 1.9323981e-01 2.4820906e-01 2.4820906e-01 ... 2.4820906e-01\n", + " 2.4820906e-01 -2.7795550e-01]]\n", + "\n", + " ...\n", + "\n", + " [[ 7.9710668e-01 -2.7093157e-02 -2.7093157e-02 ... -2.7093157e-02\n", + " -2.7093157e-02 -2.0062150e-01]\n", + " [ 8.7680638e-02 -7.2153252e-01 -7.2153252e-01 ... -7.2153252e-01\n", + " -7.2153252e-01 -5.9123868e-01]\n", + " [ 8.7680638e-02 -7.2153252e-01 -7.2153252e-01 ... -7.2153252e-01\n", + " -7.2153252e-01 -5.9123868e-01]\n", + " ...\n", + " [ 8.7680638e-02 -7.2153252e-01 -7.2153252e-01 ... -7.2153252e-01\n", + " -7.2153252e-01 -5.9123868e-01]\n", + " [ 8.7680638e-02 -7.2153252e-01 -7.2153252e-01 ... -7.2153252e-01\n", + " -7.2153252e-01 -5.9123868e-01]\n", + " [ 2.6627803e-01 1.3488382e-03 1.3488382e-03 ... 1.3488382e-03\n", + " 1.3488382e-03 -4.5465171e-01]]]]\n" + ] + } + ], + "source": [ + "import numpy as np\n", + "import mindspore.nn as nn\n", + "from mindspore import Tensor\n", + "from mindspore.common import set_seed\n", + "from mindspore.common.initializer import Normal\n", + "\n", + "set_seed(1)\n", + "\n", + "input_data = Tensor(np.ones([1, 3, 16, 50], dtype=np.float32))\n", + "net = nn.Conv2d(3, 64, 3, weight_init=Normal(0.2))\n", + "output = net(input_data)\n", + "print(output)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Customized Tensor \n", + "In addition to the above two initialization methods, when the network wants to initialize the parameters with data types not available in MindSpore, the user can initialize the parameters by means of a custom Tensor, as shown in the following code example." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[[[[12. 18. 18. ... 18. 18. 12.]\n", + " [18. 27. 27. ... 27. 27. 18.]\n", + " [18. 27. 27. ... 27. 27. 18.]\n", + " ...\n", + " [18. 27. 27. ... 27. 27. 18.]\n", + " [18. 27. 27. ... 27. 27. 18.]\n", + " [12. 18. 18. ... 18. 18. 12.]]\n", + "\n", + " ...\n", + "\n", + " [[12. 18. 18. ... 18. 18. 12.]\n", + " [18. 27. 27. ... 27. 27. 18.]\n", + " [18. 27. 27. ... 27. 27. 18.]\n", + " ...\n", + " [18. 27. 27. ... 27. 27. 18.]\n", + " [18. 27. 27. ... 27. 27. 18.]\n", + " [12. 18. 18. ... 18. 18. 12.]]]]\n" + ] + } + ], + "source": [ + "import numpy as np\n", + "import mindspore.nn as nn\n", + "from mindspore import Tensor\n", + "from mindspore import dtype as mstype\n", + "\n", + "weight = Tensor(np.ones([64, 3, 3, 3]), dtype=mstype.float32)\n", + "input_data = Tensor(np.ones([1, 3, 16, 50], dtype=np.float32))\n", + "net = nn.Conv2d(3, 64, 3, weight_init=weight)\n", + "output = net(input_data)\n", + "print(output)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Initializing parameters using the initializer method\n", + "\n", + "In the above code sample, it is given how to initialize the parameters in the network, such as using the nn layer to encapsulate the Conv2d operator in the network, the parameter weight_init is passed to the Conv2d operator as the data type to be initialized, and the operator will initialize the parameters by calling the Parameter class, which in turn calls the initializer method encapsulated in the Parameter class to complete the initialization of the parameter. However, some operators do not encapsulate the function of parameter initialization internally as Conv2d does, such as the weights of the Conv3d operator are passed into the Conv3d operator as parameters, and the initialization of the weights needs to be defined manually.\n", + "\n", + "When initializing parameters, you can use the initializer method to call different data types in the Initializer subclass to initialize the parameters and thus generate different types of data.\n", + "\n", + "When using initializer for parameter initialization, the parameters passed in are`init`、`shape`、`dtype`：\n", + "\n", + "- `init`：support passing in`Tensor`, `str`, `Subclass of Initializer`。\n", + "\n", + "- `shape`：support passing`list`, `tuple`, `int`.\n", + "\n", + "- `dtype`：support passing`mindspore.dtype`." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### init parameter is Tensor" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Sample code is as follows:" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "ExecuteTime": { + "end_time": "2021-02-03T02:59:50.340750Z", + "start_time": "2021-02-03T02:59:49.571048Z" + } + }, + "source": [ + "```python\n", + "import numpy as np\n", + "from mindspore import Tensor\n", + "from mindspore import dtype as mstype\n", + "from mindspore.common import set_seed\n", + "from mindspore.common.initializer import initializer\n", + "from mindspore.ops.operations import nn_ops as nps\n", + "\n", + "set_seed(1)\n", + "\n", + "input_data = Tensor(np.ones([16, 3, 10, 32, 32]), dtype=mstype.float32)\n", + "weight_init = Tensor(np.ones([32, 3, 4, 3, 3]), dtype=mstype.float32)\n", + "weight = initializer(weight_init, shape=[32, 3, 4, 3, 3])\n", + "conv3d = nps.Conv3D(out_channel=32, kernel_size=(4, 3, 3))\n", + "output = conv3d(input_data, weight)\n", + "print(output)\n", + "```" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The output is as follows：\n", + "\n", + "```text\n", + "[[[[[108 108 108 ... 108 108 108]\n", + " [108 108 108 ... 108 108 108]\n", + " [108 108 108 ... 108 108 108]\n", + " ...\n", + " [108 108 108 ... 108 108 108]\n", + " [108 108 108 ... 108 108 108]\n", + " [108 108 108 ... 108 108 108]]\n", + " ...\n", + " [[108 108 108 ... 108 108 108]\n", + " [108 108 108 ... 108 108 108]\n", + " [108 108 108 ... 108 108 108]\n", + " ...\n", + " [108 108 108 ... 108 108 108]\n", + " [108 108 108 ... 108 108 108]\n", + " [108 108 108 ... 108 108 108]]]]]\n", + "```" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### init parameter is str" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Sample code is as follows：" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "```python\n", + "import numpy as np\n", + "from mindspore import Tensor\n", + "from mindspore import dtype as mstype\n", + "from mindspore.common import set_seed\n", + "from mindspore.common.initializer import initializer\n", + "from mindspore.ops.operations import nn_ops as nps\n", + "\n", + "set_seed(1)\n", + "\n", + "input_data = Tensor(np.ones([16, 3, 10, 32, 32]), dtype=mstype.float32)\n", + "weight = initializer('Normal', shape=[32, 3, 4, 3, 3], dtype=mstype.float32)\n", + "conv3d = nps.Conv3D(out_channel=32, kernel_size=(4, 3, 3))\n", + "output = conv3d(input_data, weight)\n", + "print(output)\n", + "```" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The output is as follows：\n", + "\n", + "```text\n", + "[[[[[0 0 0 ... 0 0 0]\n", + " [0 0 0 ... 0 0 0]\n", + " [0 0 0 ... 0 0 0]]\n", + " ...\n", + " [0 0 0 ... 0 0 0]\n", + " [0 0 0 ... 0 0 0]\n", + " [0 0 0 ... 0 0 0]]\n", + " ...\n", + " [[0 0 0 ... 0 0 0]\n", + " [0 0 0 ... 0 0 0]\n", + " [0 0 0 ... 0 0 0]]\n", + " ...\n", + " [0 0 0 ... 0 0 0]\n", + " [0 0 0 ... 0 0 0]\n", + " [0 0 0 ... 0 0 0]]]]]\n", + "```" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### The init parameter is a subclass of Initializer" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Sample code is as follows：" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "```python\n", + "import numpy as np\n", + "from mindspore import Tensor\n", + "from mindspore import dtype as mstype\n", + "from mindspore.common import set_seed\n", + "from mindspore.ops.operations import nn_ops as nps\n", + "from mindspore.common.initializer import Normal, initializer\n", + "\n", + "set_seed(1)\n", + "\n", + "input_data = Tensor(np.ones([16, 3, 10, 32, 32]), dtype=mstype.float32)\n", + "weight = initializer(Normal(0.2), shape=[32, 3, 4, 3, 3], dtype=mstype.float32)\n", + "conv3d = nps.Conv3D(out_channel=32, kernel_size=(4, 3, 3))\n", + "output = conv3d(input_data, weight)\n", + "print(output)\n", + "```" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "```text\n", + "[[[[[0 0 0 ... 0 0 0]\n", + " [0 0 0 ... 0 0 0]\n", + " [0 0 0 ... 0 0 0]]\n", + " ...\n", + " [0 0 0 ... 0 0 0]\n", + " [0 0 0 ... 0 0 0]\n", + " [0 0 0 ... 0 0 0]]\n", + " ...\n", + " [[0 0 0 ... 0 0 0]\n", + " [0 0 0 ... 0 0 0]\n", + " [0 0 0 ... 0 0 0]]\n", + " ...\n", + " [0 0 0 ... 0 0 0]\n", + " [0 0 0 ... 0 0 0]\n", + " [0 0 0 ... 0 0 0]]]]]\n", + "```" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Application in Parameter" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The code example is as follows:" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[[-0.3305102 1.0412874 2.0412874 3.0412874]\n", + " [ 4.0412874 4.9479127 5.9479127 6.9479127]\n", + " [ 7.947912 9.063009 10.063009 11.063009 ]\n", + " [12.063009 13.536987 14.536987 14.857441 ]\n", + " [15.751231 17.073082 17.808317 19.364822 ]]\n" + ] + } + ], + "source": [ + "import numpy as np\n", + "from mindspore import dtype as mstype\n", + "from mindspore.common import set_seed\n", + "from mindspore.ops import operations as ops\n", + "from mindspore import Tensor, Parameter, context\n", + "from mindspore.common.initializer import Normal, initializer\n", + "\n", + "set_seed(1)\n", + "\n", + "weight1 = Parameter(initializer('Normal', [5, 4], mstype.float32), name=\"w1\")\n", + "weight2 = Parameter(initializer(Normal(0.2), [5, 4], mstype.float32), name=\"w2\")\n", + "input_data = Tensor(np.arange(20).reshape(5, 4), dtype=mstype.float32)\n", + "net = ops.Add()\n", + "output = net(input_data, weight1)\n", + "output = net(output, weight2)\n", + "print(output)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.6" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} -- Gitee From 765bfa7ee514f7ed63da342120e934fd2dcd2624 Mon Sep 17 00:00:00 2001 From: M-Zif Date: Tue, 1 Jun 2021 11:10:27 +0800 Subject: [PATCH 6/8] Translate --- .../source_zh_cn/initializer.ipynb | 56 +++++++++---------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/docs/programming_guide/source_zh_cn/initializer.ipynb b/docs/programming_guide/source_zh_cn/initializer.ipynb index b70044f5ea..b2271fb094 100644 --- a/docs/programming_guide/source_zh_cn/initializer.ipynb +++ b/docs/programming_guide/source_zh_cn/initializer.ipynb @@ -4,7 +4,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# 网络参数的初始化\n", + "# Initialization of Network Parameters\n", "\n", "[![](https://gitee.com/mindspore/docs/raw/master/resource/_static/logo_source.png)](https://gitee.com/mindspore/docs/blob/master/docs/programming_guide/source_zh_cn/initializer.ipynb) [![](https://gitee.com/mindspore/docs/raw/master/resource/_static/logo_notebook.png)](https://obs.dualstack.cn-north-4.myhuaweicloud.com/mindspore-website/notebook/master/programming_guide/mindspore_initializer.ipynb) [![](https://gitee.com/mindspore/docs/raw/master/resource/_static/logo_modelarts.png)](https://authoring-modelarts-cnnorth4.huaweicloud.com/console/lab?share-url-b64=aHR0cHM6Ly9vYnMuZHVhbHN0YWNrLmNuLW5vcnRoLTQubXlodWF3ZWljbG91ZC5jb20vbWluZHNwb3JlLXdlYnNpdGUvbm90ZWJvb2svbW9kZWxhcnRzL3Byb2dyYW1taW5nX2d1aWRlL21pbmRzcG9yZV9pbml0aWFsaXplci5pcHluYg==&imagename=MindSpore1.1.1)" ] @@ -13,25 +13,25 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## 概述\n", + "## Overview\n", "\n", - "MindSpore提供了权重初始化模块，用户可以通过封装算子和initializer方法来调用字符串、Initializer子类或自定义Tensor等方式完成对网络参数进行初始化。Initializer类是MindSpore中用于进行初始化的基本数据结构，其子类包含了几种不同类型的数据分布（Zero，One，XavierUniform，HeUniform，HeNormal，Constant，Uniform，Normal，TruncatedNormal）。下面针对封装算子和initializer方法两种参数初始化模式进行详细介绍。" + "MindSpore provides a weight initialization module that allows users to initialize network parameters by wrapping operators and initializer methods to call strings, Initializer subclasses or custom Tensor. Class Initializer is the basic data structure in MindSpore to initialize network parameters,its subclasses contain several different types of data distributions (Zero, One, XavierUniform, HeUniform, HeNormal, Constant, Uniform, Normal, TruncatedNormal). The following is a detailed description of the two parameter initialization modes, the wrapper operator and the initializer method." ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## 使用封装算子对参数初始化 \n", - "MindSpore提供了多种参数初始化的方式，并在部分算子中封装了参数初始化的功能。本节将介绍带有参数初始化功能的算子对参数进行初始化的方法，以`Conv2d`算子为例，分别介绍以字符串，`Initializer`子类和自定义`Tensor`等方式对网络中的参数进行初始化，以下代码示例中均以`Initializer`的子类`Normal`为例，代码示例中`Normal`均可替换成`Initializer`子类中任何一个。" + "## Parameter initialization using wrapper operator \n", + "MindSpore provides various ways to initialize parameters and encapsulates parameter initialization in some operators. In this section, we will introduce the methods of initializing parameters in operators with parameter initialization function, taking the Conv2d operator as an example, and introduce the initialization of parameters in the network by strings, Initializer subclasses and custom Tensor, etc. The following code examples take Normal, a subclass of Initializer, as an example. Normal can be replaced by any of the Initializer subclasses." ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "### 字符串 \n", - "使用字符串对网络参数进行初始化，字符串的内容需要与`Initializer`子类的名称保持一致，使用字符串方式进行初始化将使用`Initializer`子类中的默认参数，例如使用字符串`Normal`等同于使用`Initializer`的子类`Normal()`，代码样例如下：" + "### String \n", + "Using a string to initialize the network parameters, the content of the string needs to be consistent with the name of the subclass of Initializer, using the string approach to initialization will use the default parameters in the Initializer subclass, for example, using the string Normal is equivalent to using the Initializer subclass Normal (), code examples are as follows:" ] }, { @@ -93,8 +93,8 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "### Initializer子类 \n", - "使用`Initializer`子类对网络参数进行初始化，与使用字符串对参数进行初始化的效果类似，不同的是使用字符串进行参数初始化是使用`Initializer`子类的默认参数，如要使用`Initializer`子类中的参数，就必须使用`Initializer`子类的方式对参数进行初始化，以`Normal(0.2)`为例，代码样例如下：" + "### Initializer subclass \n", + "Using the Initializer subclass to initialize the network parameters is similar to using a string to initialize the parameters, the difference is that using a string to initialize the parameters is using the default parameters of the Initializer subclass, such as to use the parameters in the Initializer subclass, you must use the Initializer subclass way to initialize the parameters To initialize, take Normal(0.2) as an example, the code sample is as follows:" ] }, { @@ -157,8 +157,8 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "### 自定义的Tensor \n", - "除上述两种初始化方法外，当网络要使用MindSpore中没有的数据类型对参数进行初始化，用户可以通过自定义`Tensor`的方式来对参数进行初始化，代码样例如下：" + "### Customized Tensor \n", + "In addition to the above two initialization methods, when the network wants to initialize the parameters with data types not available in MindSpore, the user can initialize the parameters by means of a custom Tensor, as shown in the following code example." ] }, { @@ -207,33 +207,33 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## 使用initializer方法对参数初始化\n", + "## Initializing parameters using the initializer method\n", "\n", - "在上述代码样例中，给出了如何在网络中进行参数初始化的方法，如在网络中使用nn层封装`Conv2d`算子，参数`weight_init`作为要初始化的数据类型传入`Conv2d`算子，算子会在初始化时通过调用`Parameter`类，进而调用封装在`Parameter`类中的`initializer`方法来完成对参数的初始化。然而有一些算子并没有像`Conv2d`那样在内部对参数初始化的功能进行封装，如`Conv3d`算子的权重就是作为参数传入`Conv3d`算子，此时就需要手动的定义权重的初始化。\n", + "In the above code sample, it is given how to initialize the parameters in the network, such as using the nn layer to encapsulate the Conv2d operator in the network, the parameter weight_init is passed to the Conv2d operator as the data type to be initialized, and the operator will initialize the parameters by calling the Parameter class, which in turn calls the initializer method encapsulated in the Parameter class to complete the initialization of the parameter. However, some operators do not encapsulate the function of parameter initialization internally as Conv2d does, such as the weights of the Conv3d operator are passed into the Conv3d operator as parameters, and the initialization of the weights needs to be defined manually.\n", "\n", - "当对参数进行初始化时，可以使用`initializer`方法调用`Initializer`子类中不同的数据类型来对参数进行初始化，进而产生不同类型的数据。\n", + "When initializing parameters, you can use the initializer method to call different data types in the Initializer subclass to initialize the parameters and thus generate different types of data.\n", "\n", - "使用initializer进行参数初始化时，支持传入的参数有`init`、`shape`、`dtype`：\n", + "When using initializer for parameter initialization, the parameters passed in are`init`、`shape`、`dtype`：\n", "\n", - "- `init`：支持传入`Tensor`、 `str`、 `Initializer的子类`。\n", + "- `init`：support passing in`Tensor`, `str`, `Subclass of Initializer`。\n", "\n", - "- `shape`：支持传入`list`、 `tuple`、 `int`。\n", + "- `shape`：support passing`list`, `tuple`, `int`.\n", "\n", - "- `dtype`：支持传入`mindspore.dtype`。" + "- `dtype`：support passing`mindspore.dtype`." ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "### init参数为Tensor" + "### init parameter is Tensor" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "代码样例如下：" + "Sample code is as follows:" ] }, { @@ -268,7 +268,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "输出如下：\n", + "The output is as follows：\n", "\n", "```text\n", "[[[[[108 108 108 ... 108 108 108]\n", @@ -293,14 +293,14 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "### init参数为str" + "### init parameter is str" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "代码样例如下：" + "Sample code is as follows：" ] }, { @@ -329,7 +329,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "输出如下：\n", + "The output is as follows：\n", "\n", "```text\n", "[[[[[0 0 0 ... 0 0 0]\n", @@ -354,14 +354,14 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "### init参数为Initializer子类" + "### The init parameter is a subclass of Initializer" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "代码样例如下：" + "Sample code is as follows：" ] }, { @@ -413,14 +413,14 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "### 在Parameter中的应用" + "### Application in Parameter" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "代码样例如下：" + "The code example is as follows:" ] }, { -- Gitee From ed6a8db81cf330cb0e7ea4a20ac5734917ecb018 Mon Sep 17 00:00:00 2001 From: M-Zif Date: Tue, 1 Jun 2021 11:36:04 +0800 Subject: [PATCH 7/8] translate --- .../source_en/initializer.ipynb | 810 +++++++++++------- 1 file changed, 484 insertions(+), 326 deletions(-) diff --git a/docs/programming_guide/source_en/initializer.ipynb b/docs/programming_guide/source_en/initializer.ipynb index 1834c40fe4..b2271fb094 100644 --- a/docs/programming_guide/source_en/initializer.ipynb +++ b/docs/programming_guide/source_en/initializer.ipynb @@ -1,326 +1,484 @@ -# Initialization of Network Parameters - - - -## Overview - -MindSpore provides a weight initialization module that allows users to initialize network parameters by wrapping operators and initializer methods to call strings, Initializer subclasses or custom Tensor. Class Initializer is the basic data structure in MindSpore to initialize network parameters,its subclasses contain several different types of data distributions (Zero, One, XavierUniform, HeUniform, HeNormal, Constant, Uniform, Normal, TruncatedNormal). The following is a detailed description of the two parameter initialization modes, the wrapper operator and the initializer method. - -## Parameter initialization using wrapper operator - -MindSpore provides various ways to initialize parameters and encapsulates parameter initialization in some operators. In this section, we will introduce the methods of initializing parameters in operators with parameter initialization function, taking the Conv2d operator as an example, and introduce the initialization of parameters in the network by strings, Initializer subclasses and custom Tensor, etc. The following code examples take Normal, a subclass of Initializer, as an example. Normal can be replaced by any of the Initializer subclasses. - -## String - -Using a string to initialize the network parameters, the content of the string needs to be consistent with the name of the subclass of Initializer, using the string approach to initialization will use the default parameters in the Initializer subclass, for example, using the string Normal is equivalent to using the Initializer subclass Normal (), code examples are as follows: - -```python -import numpy as np -import mindspore.nn as nn -from mindspore import Tensor -from mindspore.common import set_seed - -set_seed(1) - -input_data = Tensor(np.ones([1, 3, 16, 50], dtype=np.float32)) -net = nn.Conv2d(3, 64, 3, weight_init='Normal') -output = net(input_data) -print(output) -``` - -```python -[[[[ 3.10382620e-02 4.38603461e-02 4.38603461e-02 ... 4.38603461e-02 - 4.38603461e-02 1.38719045e-02] - [ 3.26051228e-02 3.54298912e-02 3.54298912e-02 ... 3.54298912e-02 - 3.54298912e-02 -5.54019120e-03] - [ 3.26051228e-02 3.54298912e-02 3.54298912e-02 ... 3.54298912e-02 - 3.54298912e-02 -5.54019120e-03] - ... - [ 3.26051228e-02 3.54298912e-02 3.54298912e-02 ... 3.54298912e-02 - 3.54298912e-02 -5.54019120e-03] - [ 3.26051228e-02 3.54298912e-02 3.54298912e-02 ... 3.54298912e-02 - 3.54298912e-02 -5.54019120e-03] - [ 9.66199022e-03 1.24104535e-02 1.24104535e-02 ... 1.24104535e-02 - 1.24104535e-02 -1.38977719e-02]] - - ... - - [[ 3.98553275e-02 -1.35465711e-03 -1.35465711e-03 ... -1.35465711e-03 - -1.35465711e-03 -1.00310734e-02] - [ 4.38403059e-03 -3.60766202e-02 -3.60766202e-02 ... -3.60766202e-02 - -3.60766202e-02 -2.95619294e-02] - [ 4.38403059e-03 -3.60766202e-02 -3.60766202e-02 ... -3.60766202e-02 - -3.60766202e-02 -2.95619294e-02] - ... - [ 4.38403059e-03 -3.60766202e-02 -3.60766202e-02 ... -3.60766202e-02 - -3.60766202e-02 -2.95619294e-02] - [ 4.38403059e-03 -3.60766202e-02 -3.60766202e-02 ... -3.60766202e-02 - -3.60766202e-02 -2.95619294e-02] - [ 1.33139016e-02 6.74417242e-05 6.74417242e-05 ... 6.74417242e-05 - 6.74417242e-05 -2.27325838e-02]]]] -``` - -## Initializer subclass - -Using the Initializer subclass to initialize the network parameters is similar to using a string to initialize the parameters, the difference is that using a string to initialize the parameters is using the default parameters of the Initializer subclass, such as to use the parameters in the Initializer subclass, you must use the Initializer subclass way to initialize the parameters To initialize, take Normal(0.2) as an example, the code sample is as follows: - -```python -import numpy as np -import mindspore.nn as nn -from mindspore import Tensor -from mindspore.common import set_seed -from mindspore.common.initializer import Normal - -set_seed(1) - -input_data = Tensor(np.ones([1, 3, 16, 50], dtype=np.float32)) -net = nn.Conv2d(3, 64, 3, weight_init=Normal(0.2)) -output = net(input_data) -print(output) -``` - -```python -[[[[ 6.2076533e-01 8.7720710e-01 8.7720710e-01 ... 8.7720710e-01 - 8.7720710e-01 2.7743810e-01] - [ 6.5210247e-01 7.0859784e-01 7.0859784e-01 ... 7.0859784e-01 - 7.0859784e-01 -1.1080378e-01] - [ 6.5210247e-01 7.0859784e-01 7.0859784e-01 ... 7.0859784e-01 - 7.0859784e-01 -1.1080378e-01] - ... - [ 6.5210247e-01 7.0859784e-01 7.0859784e-01 ... 7.0859784e-01 - 7.0859784e-01 -1.1080378e-01] - [ 6.5210247e-01 7.0859784e-01 7.0859784e-01 ... 7.0859784e-01 - 7.0859784e-01 -1.1080378e-01] - [ 1.9323981e-01 2.4820906e-01 2.4820906e-01 ... 2.4820906e-01 - 2.4820906e-01 -2.7795550e-01]] - - ... - - [[ 7.9710668e-01 -2.7093157e-02 -2.7093157e-02 ... -2.7093157e-02 - -2.7093157e-02 -2.0062150e-01] - [ 8.7680638e-02 -7.2153252e-01 -7.2153252e-01 ... -7.2153252e-01 - -7.2153252e-01 -5.9123868e-01] - [ 8.7680638e-02 -7.2153252e-01 -7.2153252e-01 ... -7.2153252e-01 - -7.2153252e-01 -5.9123868e-01] - ... - [ 8.7680638e-02 -7.2153252e-01 -7.2153252e-01 ... -7.2153252e-01 - -7.2153252e-01 -5.9123868e-01] - [ 8.7680638e-02 -7.2153252e-01 -7.2153252e-01 ... -7.2153252e-01 - -7.2153252e-01 -5.9123868e-01] - [ 2.6627803e-01 1.3488382e-03 1.3488382e-03 ... 1.3488382e-03 - 1.3488382e-03 -4.5465171e-01]]]] -``` - -## Customized Tensor - -In addition to the above two initialization methods, when the network wants to initialize the parameters with data types not available in MindSpore, the user can initialize the parameters by means of a custom Tensor, as shown in the following code example. - - - -```python -import numpy as np -import mindspore.nn as nn -from mindspore import Tensor -from mindspore import dtype as mstype - -weight = Tensor(np.ones([64, 3, 3, 3]), dtype=mstype.float32) -input_data = Tensor(np.ones([1, 3, 16, 50], dtype=np.float32)) -net = nn.Conv2d(3, 64, 3, weight_init=weight) -output = net(input_data) -print(output) -``` - -```python -[[[[12. 18. 18. ... 18. 18. 12.] - [18. 27. 27. ... 27. 27. 18.] - [18. 27. 27. ... 27. 27. 18.] - ... - [18. 27. 27. ... 27. 27. 18.] - [18. 27. 27. ... 27. 27. 18.] - [12. 18. 18. ... 18. 18. 12.]] - - ... - - [[12. 18. 18. ... 18. 18. 12.] - [18. 27. 27. ... 27. 27. 18.] - [18. 27. 27. ... 27. 27. 18.] - ... - [18. 27. 27. ... 27. 27. 18.] - [18. 27. 27. ... 27. 27. 18.] - [12. 18. 18. ... 18. 18. 12.]]]] -``` - -## Initializing parameters using the initializer method - -In the above code sample, it is given how to initialize the parameters in the network, such as using the nn layer to encapsulate the Conv2d operator in the network, the parameter weight_init is passed to the Conv2d operator as the data type to be initialized, and the operator will initialize the parameters by calling the Parameter class, which in turn calls the initializer method encapsulated in the Parameter class to complete the initialization of the parameter. However, some operators do not encapsulate the function of parameter initialization internally as Conv2d does, such as the weights of the Conv3d operator are passed into the Conv3d operator as parameters, and the initialization of the weights needs to be defined manually. - -When initializing parameters, you can use the initializer method to call different data types in the Initializer subclass to initialize the parameters and thus generate different types of data. - -When using initializer for parameter initialization, the parameters passed in are init, shape, and dtype: - - - -init: support passing in Tensor, str, and subclass of Initializer. - - - -shape: support passing list, tuple, int. - - - -dtype: support passing mindspore.dtype. - - - -### init parameter is Tensor - -Sample code is as follows. - -```python -import numpy as np -from mindspore import Tensor -from mindspore import dtype as mstype -from mindspore.common import set_seed -from mindspore.common.initializer import initializer -from mindspore.ops.operations import nn_ops as nps - -set_seed(1) - -input_data = Tensor(np.ones([16, 3, 10, 32, 32]), dtype=mstype.float32) -weight_init = Tensor(np.ones([32, 3, 4, 3, 3]), dtype=mstype.float32) -weight = initializer(weight_init, shape=[32, 3, 4, 3, 3]) -conv3d = nps.Conv3D(out_channel=32, kernel_size=(4, 3, 3)) -output = conv3d(input_data, weight) -print(output) -``` - -The output is as follows. - -```python -[[[[[108 108 108 ... 108 108 108] - [108 108 108 ... 108 108 108] - [108 108 108 ... 108 108 108] - ... - [108 108 108 ... 108 108 108] - [108 108 108 ... 108 108 108] - [108 108 108 ... 108 108 108]] - ... - [[108 108 108 ... 108 108 108] - [108 108 108 ... 108 108 108] - [108 108 108 ... 108 108 108] - ... - [108 108 108 ... 108 108 108] - [108 108 108 ... 108 108 108] - [108 108 108 ... 108 108 108]]]]] -``` - -### init parameter is str - -Sample code is as follows. - -```python -import numpy as np -from mindspore import Tensor -from mindspore import dtype as mstype -from mindspore.common import set_seed -from mindspore.common.initializer import initializer -from mindspore.ops.operations import nn_ops as nps - -set_seed(1) - -input_data = Tensor(np.ones([16, 3, 10, 32, 32]), dtype=mstype.float32) -weight = initializer('Normal', shape=[32, 3, 4, 3, 3], dtype=mstype.float32) -conv3d = nps.Conv3D(out_channel=32, kernel_size=(4, 3, 3)) -output = conv3d(input_data, weight) -print(output) -``` - -The output is as follows. - -```python -[[[[[0 0 0 ... 0 0 0] - [0 0 0 ... 0 0 0] - [0 0 0 ... 0 0 0]] - ... - [0 0 0 ... 0 0 0] - [0 0 0 ... 0 0 0] - [0 0 0 ... 0 0 0]] - ... - [[0 0 0 ... 0 0 0] - [0 0 0 ... 0 0 0] - [0 0 0 ... 0 0 0]] - ... - [0 0 0 ... 0 0 0] - [0 0 0 ... 0 0 0] - [0 0 0 ... 0 0 0]]]]] -``` - -### The init parameter is a subclass of Initializer - -Sample code is as follows. - -```python -import numpy as np -from mindspore import Tensor -from mindspore import dtype as mstype -from mindspore.common import set_seed -from mindspore.ops.operations import nn_ops as nps -from mindspore.common.initializer import Normal, initializer - -set_seed(1) - -input_data = Tensor(np.ones([16, 3, 10, 32, 32]), dtype=mstype.float32) -weight = initializer(Normal(0.2), shape=[32, 3, 4, 3, 3], dtype=mstype.float32) -conv3d = nps.Conv3D(out_channel=32, kernel_size=(4, 3, 3)) -output = conv3d(input_data, weight) -print(output) -``` - -``` -[[[[[0 0 0 ... 0 0 0] - [0 0 0 ... 0 0 0] - [0 0 0 ... 0 0 0]] - ... - [0 0 0 ... 0 0 0] - [0 0 0 ... 0 0 0] - [0 0 0 ... 0 0 0]] - ... - [[0 0 0 ... 0 0 0] - [0 0 0 ... 0 0 0] - [0 0 0 ... 0 0 0]] - ... - [0 0 0 ... 0 0 0] - [0 0 0 ... 0 0 0] - [0 0 0 ... 0 0 0]]]]] -``` - -## Application in Parameter - -The code example is as follows. - -```python -import numpy as np -from mindspore import dtype as mstype -from mindspore.common import set_seed -from mindspore.ops import operations as ops -from mindspore import Tensor, Parameter, context -from mindspore.common.initializer import Normal, initializer - -set_seed(1) - -weight1 = Parameter(initializer('Normal', [5, 4], mstype.float32), name="w1") -weight2 = Parameter(initializer(Normal(0.2), [5, 4], mstype.float32), name="w2") -input_data = Tensor(np.arange(20).reshape(5, 4), dtype=mstype.float32) -net = ops.Add() -output = net(input_data, weight1) -output = net(output, weight2) -print(output) -``` - -```python -[[-0.3305102 1.0412874 2.0412874 3.0412874] - [ 4.0412874 4.9479127 5.9479127 6.9479127] - [ 7.947912 9.063009 10.063009 11.063009 ] - [12.063009 13.536987 14.536987 14.857441 ] - [15.751231 17.073082 17.808317 19.364822 ]] -``` \ No newline at end of file +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Initialization of Network Parameters\n", + "\n", + "[![](https://gitee.com/mindspore/docs/raw/master/resource/_static/logo_source.png)](https://gitee.com/mindspore/docs/blob/master/docs/programming_guide/source_zh_cn/initializer.ipynb) [![](https://gitee.com/mindspore/docs/raw/master/resource/_static/logo_notebook.png)](https://obs.dualstack.cn-north-4.myhuaweicloud.com/mindspore-website/notebook/master/programming_guide/mindspore_initializer.ipynb) [![](https://gitee.com/mindspore/docs/raw/master/resource/_static/logo_modelarts.png)](https://authoring-modelarts-cnnorth4.huaweicloud.com/console/lab?share-url-b64=aHR0cHM6Ly9vYnMuZHVhbHN0YWNrLmNuLW5vcnRoLTQubXlodWF3ZWljbG91ZC5jb20vbWluZHNwb3JlLXdlYnNpdGUvbm90ZWJvb2svbW9kZWxhcnRzL3Byb2dyYW1taW5nX2d1aWRlL21pbmRzcG9yZV9pbml0aWFsaXplci5pcHluYg==&imagename=MindSpore1.1.1)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Overview\n", + "\n", + "MindSpore provides a weight initialization module that allows users to initialize network parameters by wrapping operators and initializer methods to call strings, Initializer subclasses or custom Tensor. Class Initializer is the basic data structure in MindSpore to initialize network parameters,its subclasses contain several different types of data distributions (Zero, One, XavierUniform, HeUniform, HeNormal, Constant, Uniform, Normal, TruncatedNormal). The following is a detailed description of the two parameter initialization modes, the wrapper operator and the initializer method." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Parameter initialization using wrapper operator \n", + "MindSpore provides various ways to initialize parameters and encapsulates parameter initialization in some operators. In this section, we will introduce the methods of initializing parameters in operators with parameter initialization function, taking the Conv2d operator as an example, and introduce the initialization of parameters in the network by strings, Initializer subclasses and custom Tensor, etc. The following code examples take Normal, a subclass of Initializer, as an example. Normal can be replaced by any of the Initializer subclasses." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### String \n", + "Using a string to initialize the network parameters, the content of the string needs to be consistent with the name of the subclass of Initializer, using the string approach to initialization will use the default parameters in the Initializer subclass, for example, using the string Normal is equivalent to using the Initializer subclass Normal (), code examples are as follows:" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[[[[ 3.10382620e-02 4.38603461e-02 4.38603461e-02 ... 4.38603461e-02\n", + " 4.38603461e-02 1.38719045e-02]\n", + " [ 3.26051228e-02 3.54298912e-02 3.54298912e-02 ... 3.54298912e-02\n", + " 3.54298912e-02 -5.54019120e-03]\n", + " [ 3.26051228e-02 3.54298912e-02 3.54298912e-02 ... 3.54298912e-02\n", + " 3.54298912e-02 -5.54019120e-03]\n", + " ...\n", + " [ 3.26051228e-02 3.54298912e-02 3.54298912e-02 ... 3.54298912e-02\n", + " 3.54298912e-02 -5.54019120e-03]\n", + " [ 3.26051228e-02 3.54298912e-02 3.54298912e-02 ... 3.54298912e-02\n", + " 3.54298912e-02 -5.54019120e-03]\n", + " [ 9.66199022e-03 1.24104535e-02 1.24104535e-02 ... 1.24104535e-02\n", + " 1.24104535e-02 -1.38977719e-02]]\n", + "\n", + " ...\n", + "\n", + " [[ 3.98553275e-02 -1.35465711e-03 -1.35465711e-03 ... -1.35465711e-03\n", + " -1.35465711e-03 -1.00310734e-02]\n", + " [ 4.38403059e-03 -3.60766202e-02 -3.60766202e-02 ... -3.60766202e-02\n", + " -3.60766202e-02 -2.95619294e-02]\n", + " [ 4.38403059e-03 -3.60766202e-02 -3.60766202e-02 ... -3.60766202e-02\n", + " -3.60766202e-02 -2.95619294e-02]\n", + " ...\n", + " [ 4.38403059e-03 -3.60766202e-02 -3.60766202e-02 ... -3.60766202e-02\n", + " -3.60766202e-02 -2.95619294e-02]\n", + " [ 4.38403059e-03 -3.60766202e-02 -3.60766202e-02 ... -3.60766202e-02\n", + " -3.60766202e-02 -2.95619294e-02]\n", + " [ 1.33139016e-02 6.74417242e-05 6.74417242e-05 ... 6.74417242e-05\n", + " 6.74417242e-05 -2.27325838e-02]]]]\n" + ] + } + ], + "source": [ + "import numpy as np\n", + "import mindspore.nn as nn\n", + "from mindspore import Tensor\n", + "from mindspore.common import set_seed\n", + "\n", + "set_seed(1)\n", + "\n", + "input_data = Tensor(np.ones([1, 3, 16, 50], dtype=np.float32))\n", + "net = nn.Conv2d(3, 64, 3, weight_init='Normal')\n", + "output = net(input_data)\n", + "print(output)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Initializer subclass \n", + "Using the Initializer subclass to initialize the network parameters is similar to using a string to initialize the parameters, the difference is that using a string to initialize the parameters is using the default parameters of the Initializer subclass, such as to use the parameters in the Initializer subclass, you must use the Initializer subclass way to initialize the parameters To initialize, take Normal(0.2) as an example, the code sample is as follows:" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[[[[ 6.2076533e-01 8.7720710e-01 8.7720710e-01 ... 8.7720710e-01\n", + " 8.7720710e-01 2.7743810e-01]\n", + " [ 6.5210247e-01 7.0859784e-01 7.0859784e-01 ... 7.0859784e-01\n", + " 7.0859784e-01 -1.1080378e-01]\n", + " [ 6.5210247e-01 7.0859784e-01 7.0859784e-01 ... 7.0859784e-01\n", + " 7.0859784e-01 -1.1080378e-01]\n", + " ...\n", + " [ 6.5210247e-01 7.0859784e-01 7.0859784e-01 ... 7.0859784e-01\n", + " 7.0859784e-01 -1.1080378e-01]\n", + " [ 6.5210247e-01 7.0859784e-01 7.0859784e-01 ... 7.0859784e-01\n", + " 7.0859784e-01 -1.1080378e-01]\n", + " [ 1.9323981e-01 2.4820906e-01 2.4820906e-01 ... 2.4820906e-01\n", + " 2.4820906e-01 -2.7795550e-01]]\n", + "\n", + " ...\n", + "\n", + " [[ 7.9710668e-01 -2.7093157e-02 -2.7093157e-02 ... -2.7093157e-02\n", + " -2.7093157e-02 -2.0062150e-01]\n", + " [ 8.7680638e-02 -7.2153252e-01 -7.2153252e-01 ... -7.2153252e-01\n", + " -7.2153252e-01 -5.9123868e-01]\n", + " [ 8.7680638e-02 -7.2153252e-01 -7.2153252e-01 ... -7.2153252e-01\n", + " -7.2153252e-01 -5.9123868e-01]\n", + " ...\n", + " [ 8.7680638e-02 -7.2153252e-01 -7.2153252e-01 ... -7.2153252e-01\n", + " -7.2153252e-01 -5.9123868e-01]\n", + " [ 8.7680638e-02 -7.2153252e-01 -7.2153252e-01 ... -7.2153252e-01\n", + " -7.2153252e-01 -5.9123868e-01]\n", + " [ 2.6627803e-01 1.3488382e-03 1.3488382e-03 ... 1.3488382e-03\n", + " 1.3488382e-03 -4.5465171e-01]]]]\n" + ] + } + ], + "source": [ + "import numpy as np\n", + "import mindspore.nn as nn\n", + "from mindspore import Tensor\n", + "from mindspore.common import set_seed\n", + "from mindspore.common.initializer import Normal\n", + "\n", + "set_seed(1)\n", + "\n", + "input_data = Tensor(np.ones([1, 3, 16, 50], dtype=np.float32))\n", + "net = nn.Conv2d(3, 64, 3, weight_init=Normal(0.2))\n", + "output = net(input_data)\n", + "print(output)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Customized Tensor \n", + "In addition to the above two initialization methods, when the network wants to initialize the parameters with data types not available in MindSpore, the user can initialize the parameters by means of a custom Tensor, as shown in the following code example." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[[[[12. 18. 18. ... 18. 18. 12.]\n", + " [18. 27. 27. ... 27. 27. 18.]\n", + " [18. 27. 27. ... 27. 27. 18.]\n", + " ...\n", + " [18. 27. 27. ... 27. 27. 18.]\n", + " [18. 27. 27. ... 27. 27. 18.]\n", + " [12. 18. 18. ... 18. 18. 12.]]\n", + "\n", + " ...\n", + "\n", + " [[12. 18. 18. ... 18. 18. 12.]\n", + " [18. 27. 27. ... 27. 27. 18.]\n", + " [18. 27. 27. ... 27. 27. 18.]\n", + " ...\n", + " [18. 27. 27. ... 27. 27. 18.]\n", + " [18. 27. 27. ... 27. 27. 18.]\n", + " [12. 18. 18. ... 18. 18. 12.]]]]\n" + ] + } + ], + "source": [ + "import numpy as np\n", + "import mindspore.nn as nn\n", + "from mindspore import Tensor\n", + "from mindspore import dtype as mstype\n", + "\n", + "weight = Tensor(np.ones([64, 3, 3, 3]), dtype=mstype.float32)\n", + "input_data = Tensor(np.ones([1, 3, 16, 50], dtype=np.float32))\n", + "net = nn.Conv2d(3, 64, 3, weight_init=weight)\n", + "output = net(input_data)\n", + "print(output)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Initializing parameters using the initializer method\n", + "\n", + "In the above code sample, it is given how to initialize the parameters in the network, such as using the nn layer to encapsulate the Conv2d operator in the network, the parameter weight_init is passed to the Conv2d operator as the data type to be initialized, and the operator will initialize the parameters by calling the Parameter class, which in turn calls the initializer method encapsulated in the Parameter class to complete the initialization of the parameter. However, some operators do not encapsulate the function of parameter initialization internally as Conv2d does, such as the weights of the Conv3d operator are passed into the Conv3d operator as parameters, and the initialization of the weights needs to be defined manually.\n", + "\n", + "When initializing parameters, you can use the initializer method to call different data types in the Initializer subclass to initialize the parameters and thus generate different types of data.\n", + "\n", + "When using initializer for parameter initialization, the parameters passed in are`init`、`shape`、`dtype`：\n", + "\n", + "- `init`：support passing in`Tensor`, `str`, `Subclass of Initializer`。\n", + "\n", + "- `shape`：support passing`list`, `tuple`, `int`.\n", + "\n", + "- `dtype`：support passing`mindspore.dtype`." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### init parameter is Tensor" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Sample code is as follows:" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "ExecuteTime": { + "end_time": "2021-02-03T02:59:50.340750Z", + "start_time": "2021-02-03T02:59:49.571048Z" + } + }, + "source": [ + "```python\n", + "import numpy as np\n", + "from mindspore import Tensor\n", + "from mindspore import dtype as mstype\n", + "from mindspore.common import set_seed\n", + "from mindspore.common.initializer import initializer\n", + "from mindspore.ops.operations import nn_ops as nps\n", + "\n", + "set_seed(1)\n", + "\n", + "input_data = Tensor(np.ones([16, 3, 10, 32, 32]), dtype=mstype.float32)\n", + "weight_init = Tensor(np.ones([32, 3, 4, 3, 3]), dtype=mstype.float32)\n", + "weight = initializer(weight_init, shape=[32, 3, 4, 3, 3])\n", + "conv3d = nps.Conv3D(out_channel=32, kernel_size=(4, 3, 3))\n", + "output = conv3d(input_data, weight)\n", + "print(output)\n", + "```" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The output is as follows：\n", + "\n", + "```text\n", + "[[[[[108 108 108 ... 108 108 108]\n", + " [108 108 108 ... 108 108 108]\n", + " [108 108 108 ... 108 108 108]\n", + " ...\n", + " [108 108 108 ... 108 108 108]\n", + " [108 108 108 ... 108 108 108]\n", + " [108 108 108 ... 108 108 108]]\n", + " ...\n", + " [[108 108 108 ... 108 108 108]\n", + " [108 108 108 ... 108 108 108]\n", + " [108 108 108 ... 108 108 108]\n", + " ...\n", + " [108 108 108 ... 108 108 108]\n", + " [108 108 108 ... 108 108 108]\n", + " [108 108 108 ... 108 108 108]]]]]\n", + "```" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### init parameter is str" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Sample code is as follows：" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "```python\n", + "import numpy as np\n", + "from mindspore import Tensor\n", + "from mindspore import dtype as mstype\n", + "from mindspore.common import set_seed\n", + "from mindspore.common.initializer import initializer\n", + "from mindspore.ops.operations import nn_ops as nps\n", + "\n", + "set_seed(1)\n", + "\n", + "input_data = Tensor(np.ones([16, 3, 10, 32, 32]), dtype=mstype.float32)\n", + "weight = initializer('Normal', shape=[32, 3, 4, 3, 3], dtype=mstype.float32)\n", + "conv3d = nps.Conv3D(out_channel=32, kernel_size=(4, 3, 3))\n", + "output = conv3d(input_data, weight)\n", + "print(output)\n", + "```" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The output is as follows：\n", + "\n", + "```text\n", + "[[[[[0 0 0 ... 0 0 0]\n", + " [0 0 0 ... 0 0 0]\n", + " [0 0 0 ... 0 0 0]]\n", + " ...\n", + " [0 0 0 ... 0 0 0]\n", + " [0 0 0 ... 0 0 0]\n", + " [0 0 0 ... 0 0 0]]\n", + " ...\n", + " [[0 0 0 ... 0 0 0]\n", + " [0 0 0 ... 0 0 0]\n", + " [0 0 0 ... 0 0 0]]\n", + " ...\n", + " [0 0 0 ... 0 0 0]\n", + " [0 0 0 ... 0 0 0]\n", + " [0 0 0 ... 0 0 0]]]]]\n", + "```" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### The init parameter is a subclass of Initializer" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Sample code is as follows：" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "```python\n", + "import numpy as np\n", + "from mindspore import Tensor\n", + "from mindspore import dtype as mstype\n", + "from mindspore.common import set_seed\n", + "from mindspore.ops.operations import nn_ops as nps\n", + "from mindspore.common.initializer import Normal, initializer\n", + "\n", + "set_seed(1)\n", + "\n", + "input_data = Tensor(np.ones([16, 3, 10, 32, 32]), dtype=mstype.float32)\n", + "weight = initializer(Normal(0.2), shape=[32, 3, 4, 3, 3], dtype=mstype.float32)\n", + "conv3d = nps.Conv3D(out_channel=32, kernel_size=(4, 3, 3))\n", + "output = conv3d(input_data, weight)\n", + "print(output)\n", + "```" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "```text\n", + "[[[[[0 0 0 ... 0 0 0]\n", + " [0 0 0 ... 0 0 0]\n", + " [0 0 0 ... 0 0 0]]\n", + " ...\n", + " [0 0 0 ... 0 0 0]\n", + " [0 0 0 ... 0 0 0]\n", + " [0 0 0 ... 0 0 0]]\n", + " ...\n", + " [[0 0 0 ... 0 0 0]\n", + " [0 0 0 ... 0 0 0]\n", + " [0 0 0 ... 0 0 0]]\n", + " ...\n", + " [0 0 0 ... 0 0 0]\n", + " [0 0 0 ... 0 0 0]\n", + " [0 0 0 ... 0 0 0]]]]]\n", + "```" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Application in Parameter" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The code example is as follows:" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[[-0.3305102 1.0412874 2.0412874 3.0412874]\n", + " [ 4.0412874 4.9479127 5.9479127 6.9479127]\n", + " [ 7.947912 9.063009 10.063009 11.063009 ]\n", + " [12.063009 13.536987 14.536987 14.857441 ]\n", + " [15.751231 17.073082 17.808317 19.364822 ]]\n" + ] + } + ], + "source": [ + "import numpy as np\n", + "from mindspore import dtype as mstype\n", + "from mindspore.common import set_seed\n", + "from mindspore.ops import operations as ops\n", + "from mindspore import Tensor, Parameter, context\n", + "from mindspore.common.initializer import Normal, initializer\n", + "\n", + "set_seed(1)\n", + "\n", + "weight1 = Parameter(initializer('Normal', [5, 4], mstype.float32), name=\"w1\")\n", + "weight2 = Parameter(initializer(Normal(0.2), [5, 4], mstype.float32), name=\"w2\")\n", + "input_data = Tensor(np.arange(20).reshape(5, 4), dtype=mstype.float32)\n", + "net = ops.Add()\n", + "output = net(input_data, weight1)\n", + "output = net(output, weight2)\n", + "print(output)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.6" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} -- Gitee From fbeee8438cc3ef345e588c61aeb223d5aee6722d Mon Sep 17 00:00:00 2001 From: M-Zif Date: Tue, 1 Jun 2021 13:05:55 +0800 Subject: [PATCH 8/8] Translate --- .../source_en/initializer.md | 52 ++++--------------- .../source_zh_cn/initializer.ipynb | 15 +++--- 2 files changed, 17 insertions(+), 50 deletions(-) diff --git a/docs/programming_guide/source_en/initializer.md b/docs/programming_guide/source_en/initializer.md index 1834c40fe4..ec627a5428 100644 --- a/docs/programming_guide/source_en/initializer.md +++ b/docs/programming_guide/source_en/initializer.md @@ -1,19 +1,15 @@ -# Initialization of Network Parameters +# Initialization of Network Parameters ## Overview - MindSpore provides a weight initialization module that allows users to initialize network parameters by wrapping operators and initializer methods to call strings, Initializer subclasses or custom Tensor. Class Initializer is the basic data structure in MindSpore to initialize network parameters,its subclasses contain several different types of data distributions (Zero, One, XavierUniform, HeUniform, HeNormal, Constant, Uniform, Normal, TruncatedNormal). The following is a detailed description of the two parameter initialization modes, the wrapper operator and the initializer method. ## Parameter initialization using wrapper operator - MindSpore provides various ways to initialize parameters and encapsulates parameter initialization in some operators. In this section, we will introduce the methods of initializing parameters in operators with parameter initialization function, taking the Conv2d operator as an example, and introduce the initialization of parameters in the network by strings, Initializer subclasses and custom Tensor, etc. The following code examples take Normal, a subclass of Initializer, as an example. Normal can be replaced by any of the Initializer subclasses. -## String - +### String Using a string to initialize the network parameters, the content of the string needs to be consistent with the name of the subclass of Initializer, using the string approach to initialization will use the default parameters in the Initializer subclass, for example, using the string Normal is equivalent to using the Initializer subclass Normal (), code examples are as follows: - ```python import numpy as np import mindspore.nn as nn @@ -60,10 +56,8 @@ print(output) 6.74417242e-05 -2.27325838e-02]]]] ``` -## Initializer subclass - +### Initializer subclass Using the Initializer subclass to initialize the network parameters is similar to using a string to initialize the parameters, the difference is that using a string to initialize the parameters is using the default parameters of the Initializer subclass, such as to use the parameters in the Initializer subclass, you must use the Initializer subclass way to initialize the parameters To initialize, take Normal(0.2) as an example, the code sample is as follows: - ```python import numpy as np import mindspore.nn as nn @@ -78,7 +72,6 @@ net = nn.Conv2d(3, 64, 3, weight_init=Normal(0.2)) output = net(input_data) print(output) ``` - ```python [[[[ 6.2076533e-01 8.7720710e-01 8.7720710e-01 ... 8.7720710e-01 8.7720710e-01 2.7743810e-01] @@ -111,12 +104,8 @@ print(output) 1.3488382e-03 -4.5465171e-01]]]] ``` -## Customized Tensor - +### Customized Tensor In addition to the above two initialization methods, when the network wants to initialize the parameters with data types not available in MindSpore, the user can initialize the parameters by means of a custom Tensor, as shown in the following code example. - - - ```python import numpy as np import mindspore.nn as nn @@ -129,7 +118,6 @@ net = nn.Conv2d(3, 64, 3, weight_init=weight) output = net(input_data) print(output) ``` - ```python [[[[12. 18. 18. ... 18. 18. 12.] [18. 27. 27. ... 27. 27. 18.] @@ -151,31 +139,19 @@ print(output) ``` ## Initializing parameters using the initializer method - In the above code sample, it is given how to initialize the parameters in the network, such as using the nn layer to encapsulate the Conv2d operator in the network, the parameter weight_init is passed to the Conv2d operator as the data type to be initialized, and the operator will initialize the parameters by calling the Parameter class, which in turn calls the initializer method encapsulated in the Parameter class to complete the initialization of the parameter. However, some operators do not encapsulate the function of parameter initialization internally as Conv2d does, such as the weights of the Conv3d operator are passed into the Conv3d operator as parameters, and the initialization of the weights needs to be defined manually. - When initializing parameters, you can use the initializer method to call different data types in the Initializer subclass to initialize the parameters and thus generate different types of data. - When using initializer for parameter initialization, the parameters passed in are init, shape, and dtype: - - init: support passing in Tensor, str, and subclass of Initializer. - - shape: support passing list, tuple, int. - - dtype: support passing mindspore.dtype. - ### init parameter is Tensor - Sample code is as follows. - ```python import numpy as np from mindspore import Tensor @@ -193,9 +169,7 @@ conv3d = nps.Conv3D(out_channel=32, kernel_size=(4, 3, 3)) output = conv3d(input_data, weight) print(output) ``` - -The output is as follows. - +The output is as follows: ```python [[[[[108 108 108 ... 108 108 108] [108 108 108 ... 108 108 108] @@ -215,9 +189,7 @@ The output is as follows. ``` ### init parameter is str - Sample code is as follows. - ```python import numpy as np from mindspore import Tensor @@ -234,9 +206,7 @@ conv3d = nps.Conv3D(out_channel=32, kernel_size=(4, 3, 3)) output = conv3d(input_data, weight) print(output) ``` - -The output is as follows. - +The output is as follows: ```python [[[[[0 0 0 ... 0 0 0] [0 0 0 ... 0 0 0] @@ -256,9 +226,7 @@ The output is as follows. ``` ### The init parameter is a subclass of Initializer - Sample code is as follows. - ```python import numpy as np from mindspore import Tensor @@ -294,10 +262,8 @@ print(output) [0 0 0 ... 0 0 0]]]]] ``` -## Application in Parameter - -The code example is as follows. - +### Application in Parameter +The code example is as follows: ```python import numpy as np from mindspore import dtype as mstype @@ -323,4 +289,4 @@ print(output) [ 7.947912 9.063009 10.063009 11.063009 ] [12.063009 13.536987 14.536987 14.857441 ] [15.751231 17.073082 17.808317 19.364822 ]] -``` \ No newline at end of file +``` diff --git a/docs/programming_guide/source_zh_cn/initializer.ipynb b/docs/programming_guide/source_zh_cn/initializer.ipynb index b2271fb094..a91dd474d5 100644 --- a/docs/programming_guide/source_zh_cn/initializer.ipynb +++ b/docs/programming_guide/source_zh_cn/initializer.ipynb @@ -158,7 +158,7 @@ "metadata": {}, "source": [ "### Customized Tensor \n", - "In addition to the above two initialization methods, when the network wants to initialize the parameters with data types not available in MindSpore, the user can initialize the parameters by means of a custom Tensor, as shown in the following code example." + "In addition to the above two initialization methods, when the network wants to initialize the parameters with data types not available in MindSpore, the user can initialize the parameters by means of a custom Tensor, as shown in the following code example：" ] }, { @@ -213,13 +213,13 @@ "\n", "When initializing parameters, you can use the initializer method to call different data types in the Initializer subclass to initialize the parameters and thus generate different types of data.\n", "\n", - "When using initializer for parameter initialization, the parameters passed in are`init`、`shape`、`dtype`：\n", + "When using initializer for parameter initialization, the parameters passed in are `init`、`shape`、`dtype`：\n", "\n", - "- `init`：support passing in`Tensor`, `str`, `Subclass of Initializer`。\n", + "- `init`：support passing in`Tensor`, `str`, `Subclass of Initializer`.\n", "\n", - "- `shape`：support passing`list`, `tuple`, `int`.\n", + "- `shape`：support passing in`list`, `tuple`, `int`。\n", "\n", - "- `dtype`：support passing`mindspore.dtype`." + "- `dtype`：support passing in`mindspore.dtype`。" ] }, { @@ -233,7 +233,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Sample code is as follows:" + "Sample code is as follows：" ] }, { @@ -420,7 +420,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "The code example is as follows:" + "The code example is as follows：" ] }, { @@ -482,3 +482,4 @@ "nbformat": 4, "nbformat_minor": 4 } + -- Gitee