diff --git a/tutorials/notebook/mindspore_save_model.ipynb b/tutorials/notebook/mindspore_save_model.ipynb index d7467d4e86dde7ad5743255777a83c2e3a670fab..ce40c37156d5f59b11b8ba1918c04ad3045f7199 100644 --- a/tutorials/notebook/mindspore_save_model.ipynb +++ b/tutorials/notebook/mindspore_save_model.ipynb @@ -81,16 +81,16 @@ "name": "stdout", "output_type": "stream", "text": [ - "--2020-11-30 16:14:55-- https://obs.dualstack.cn-north-4.myhuaweicloud.com/mindspore-website/notebook/datasets/MNIST_Data.zip\n", + "--2020-11-26 15:50:38-- https://obs.dualstack.cn-north-4.myhuaweicloud.com/mindspore-website/notebook/datasets/MNIST_Data.zip\n", "Resolving proxy-notebook.modelarts-dev-proxy.com (proxy-notebook.modelarts-dev-proxy.com)... 192.168.0.172\n", "Connecting to proxy-notebook.modelarts-dev-proxy.com (proxy-notebook.modelarts-dev-proxy.com)|192.168.0.172|:8083... connected.\n", "Proxy request sent, awaiting response... 200 OK\n", "Length: 10754903 (10M) [application/zip]\n", "Saving to: ‘MNIST_Data.zip’\n", "\n", - "MNIST_Data.zip 100%[===================>] 10.26M 64.9MB/s in 0.2s \n", + "MNIST_Data.zip 100%[===================>] 10.26M 65.1MB/s in 0.2s \n", "\n", - "2020-11-30 16:14:56 (64.9 MB/s) - ‘MNIST_Data.zip’ saved [10754903/10754903]\n", + "2020-11-26 15:50:39 (65.1 MB/s) - ‘MNIST_Data.zip’ saved [10754903/10754903]\n", "\n", "Archive: MNIST_Data.zip\n", " creating: MNIST_Data/test/\n", @@ -112,9 +112,11 @@ } ], "source": [ - "!wget -N https://obs.dualstack.cn-north-4.myhuaweicloud.com/mindspore-website/notebook/datasets/MNIST_Data.zip\n", - "!unzip -o MNIST_Data.zip -d ./datasets\n", - "!tree ./datasets/MNIST_Data/" + "! wget https://obs.dualstack.cn-north-4.myhuaweicloud.com/mindspore-website/notebook/datasets/MNIST_Data.zip\n", + "! unzip MNIST_Data.zip\n", + "! mkdir -p ./datasets/\n", + "! mv -f ./MNIST_Data/ ./datasets/\n", + "! tree ./datasets/MNIST_Data" ] }, { @@ -300,6 +302,13 @@ "\n", "context.set_context(mode=context.GRAPH_MODE, device_target=\"CPU\")\n", "\n", + "if os.name == \"nt\":\n", + " # clean up old run files before in Windows\n", + " os.system('del/f/s/q *.ckpt *.meta')\n", + "else:\n", + " # clean up old run files before in Linux\n", + " os.system('rm -f *.ckpt *.meta *.pb')\n", + "\n", "lr = 0.01\n", "momentum = 0.9 \n", "\n", @@ -373,14 +382,10 @@ "source": [ "from mindspore.train.callback import ModelCheckpoint, CheckpointConfig\n", "\n", - "model_path = './models/ckpt/mindspore_save_model/'\n", - "# clean up old run files before in Linux\n", - "os.system('rm -f {}*.ckpt {}*.meta {}*.pb'.format(model_path, model_path, model_path))\n", - "\n", "# define config_ck for specifying the steps to save the checkpoint and the maximum file numbers\n", "config_ck = CheckpointConfig(save_checkpoint_steps=375, keep_checkpoint_max=10)\n", "# define ckpoint_cb for specifying the prefix of the file and the saving directory\n", - "ckpoint_cb = ModelCheckpoint(prefix='lenet', directory=model_path, config=config_ck)\n", + "ckpoint_cb = ModelCheckpoint(prefix='lenet', directory='./models/ckpt/save_model', config=config_ck)\n", "#load the training dataset\n", "ds_train = create_dataset(os.path.join(mnist_path, \"train\"), 32, repeat_size)\n", "print(\"========== The Training is Starting. ==========\")\n", @@ -425,7 +430,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "./models/ckpt/mindspore_save_model\n", + "./models/ckpt/save_model\n", "├── lenet-1_1125.ckpt\n", "├── lenet-1_1500.ckpt\n", "├── lenet-1_1875.ckpt\n", @@ -438,7 +443,7 @@ } ], "source": [ - "! tree ./models/ckpt/mindspore_save_model" + "! tree ./models/ckpt/save_model" ] }, { @@ -481,14 +486,10 @@ ], "source": [ "from mindspore.train.callback import ModelCheckpoint, CheckpointConfig\n", - "import os\n", - "\n", - "# clean up old run files before in Linux\n", - "os.system('rm -f {}lenet_2*.ckpt'.format(model_path))\n", "\n", "config_ck = CheckpointConfig(save_checkpoint_steps=375, keep_checkpoint_max=10)\n", "# Specify that here the script is executed for the second time\n", - "ckpoint_cb = ModelCheckpoint(prefix='lenet_2', directory='./models/ckpt/mindspore_save_model', config=config_ck)\n", + "ckpoint_cb = ModelCheckpoint(prefix='lenet_2', directory='./models/ckpt/save_model', config=config_ck)\n", "ds_train = create_dataset(os.path.join(mnist_path, \"train\"), 32, repeat_size)\n", "print(\"========== The Training is Starting. ==========\")\n", "model.train(epoch_size, ds_train, callbacks=ckpoint_cb,dataset_sink_mode=False)\n", @@ -499,7 +500,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "生成的CheckPoint文件(以`lenet-2`为前缀的`.ckpt`文件)如下:" + "生成的CheckPoint文件如下:" ] }, { @@ -511,7 +512,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "./models/ckpt/mindspore_save_model\n", + "./models/ckpt/save_model\n", "├── lenet-1_1125.ckpt\n", "├── lenet-1_1500.ckpt\n", "├── lenet-1_1875.ckpt\n", @@ -530,7 +531,7 @@ } ], "source": [ - "! tree ./models/ckpt/mindspore_save_model" + "! tree ./models/ckpt/save_model" ] }, { @@ -587,15 +588,13 @@ ], "source": [ "from mindspore.train.callback import ModelCheckpoint, CheckpointConfig\n", - "import os\n", "\n", - "os.system('rm -f {}lenet_3*.ckpt'.format(model_path))\n", "# define config_ck for specifying the seconds to save the checkpoint and the maximum file numbers\n", - "config_ck = CheckpointConfig(save_checkpoint_steps=None, save_checkpoint_seconds=10, keep_checkpoint_max=None, keep_checkpoint_per_n_minutes=1)\n", + "config_ck = CheckpointConfig(save_checkpoint_steps=None, save_checkpoint_seconds=30, keep_checkpoint_max=None, keep_checkpoint_per_n_minutes=1)\n", "# define ckpoint_cb for specifying the prefix of the file and the saving directory\n", - "ckpoint_cb = ModelCheckpoint(prefix='lenet_3', directory='./models/ckpt/mindspore_save_model', config=config_ck)\n", + "ckpoint_cb = ModelCheckpoint(prefix='lenet_3', directory='./models/ckpt/save_model', config=config_ck)\n", "#load the training dataset\n", - "epoch_size = 2\n", + "epoch_size = 10\n", "ds_train = create_dataset(os.path.join(mnist_path, \"train\"), 32, repeat_size)\n", "print(\"========== The Training is Starting. ==========\")\n", "model.train(epoch_size, ds_train, callbacks=ckpoint_cb,dataset_sink_mode=False)\n", @@ -606,8 +605,8 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "这里`save_checkpoint_seconds`和`keep_checkpoint_per_n_minutes`这两个分别设置为10和1。
\n", - "意思是每10秒会保存一个CheckPoint文件,每隔1分钟会保留一个CheckPoint文件。假设训练持续了1分钟,那总共会生成7个CheckPoint文件,但是当训练结束后,实际上会看到4个CheckPoint文件(以`lenet-3`为前缀的`.ckpt`文件),即保存下来的3个文件和默认保存最后一个step的CheckPoint文件。" + "这里`save_checkpoint_seconds`和`keep_checkpoint_per_n_minutes`这两个分别设置为30和1。
\n", + "意思是每30秒会保存一个CheckPoint文件,每隔1分钟会保留一个CheckPoint文件。假设训练持续了3分钟,那总共会生成6个CheckPoint文件,但是当训练结束后,实际上会看到4个CheckPoint文件,即保存下来的3个文件和默认保存最后一个step的CheckPoint文件。" ] }, { @@ -626,7 +625,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "./models/ckpt/mindspore_save_model\n", + "./models/ckpt/save_model\n", "├── lenet-1_1125.ckpt\n", "├── lenet-1_1500.ckpt\n", "├── lenet-1_1875.ckpt\n", @@ -638,19 +637,23 @@ "├── lenet_2-1_375.ckpt\n", "├── lenet_2-1_750.ckpt\n", "├── lenet_2-graph.meta\n", - "├── lenet_3-1_1023.ckpt\n", - "├── lenet_3-2_1254.ckpt\n", - "├── lenet_3-2_1875.ckpt\n", - "├── lenet_3-2_194.ckpt\n", + "├── lenet_3-10_1875.ckpt\n", + "├── lenet_3-10_1.ckpt\n", + "├── lenet_3-2_452.ckpt\n", + "├── lenet_3-3_1029.ckpt\n", + "├── lenet_3-4_1581.ckpt\n", + "├── lenet_3-6_188.ckpt\n", + "├── lenet_3-7_758.ckpt\n", + "├── lenet_3-8_1331.ckpt\n", "├── lenet_3-graph.meta\n", "└── lenet-graph.meta\n", "\n", - "0 directories, 17 files\n" + "0 directories, 21 files\n" ] } ], "source": [ - "! tree ./models/ckpt/mindspore_save_model" + "! tree ./models/ckpt/save_model" ] }, { @@ -680,13 +683,13 @@ "metadata": {}, "outputs": [], "source": [ - "from mindspore import export, load_checkpoint, load_param_into_net\n", + "from mindspore.train.serialization import export, load_checkpoint, load_param_into_net\n", "from mindspore import Tensor\n", "import numpy as np\n", "\n", "lenet = LeNet5()\n", "# return a parameter dict for model\n", - "param_dict = load_checkpoint(\"./models/ckpt/mindspore_save_model/lenet-1_1875.ckpt\")\n", + "param_dict = load_checkpoint(\"./models/ckpt/save_model/lenet-1_1875.ckpt\")\n", "# load the parameter into net\n", "load_param_into_net(lenet, param_dict)\n", "input = np.random.uniform(0.0, 1.0, size=[32, 1, 32, 32]).astype(np.float32)\n", @@ -726,7 +729,7 @@ "import numpy as np\n", "lenet = LeNet5()\n", "# return a parameter dict for model\n", - "param_dict = load_checkpoint(\"./models/ckpt/mindspore_save_model/lenet-1_1875.ckpt\")\n", + "param_dict = load_checkpoint(\"./models/ckpt/save_model/lenet-1_1875.ckpt\")\n", "# load the parameter into net\n", "load_param_into_net(lenet, param_dict)\n", "input = np.random.uniform(0.0, 1.0, size=[32, 1, 32, 32]).astype(np.float32)\n", diff --git a/tutorials/training/source_en/advanced_use/custom_debugging_info.md b/tutorials/training/source_en/advanced_use/custom_debugging_info.md index 30fa019a5fed8eed8cfb6bc87be39f5f8dc62980..8f01a9fc36db3f514bc2d0cb2f7f68e9136bbcc2 100644 --- a/tutorials/training/source_en/advanced_use/custom_debugging_info.md +++ b/tutorials/training/source_en/advanced_use/custom_debugging_info.md @@ -429,7 +429,7 @@ Sub modules of MindSpore grouped by source directory: | mindspore/ccsrc/pybind_api | COMMON | | mindspore/ccsrc/runtime/device | DEVICE | | mindspore/ccsrc/transform/graph_ir | GE_ADPT | -| mindspore/ccsrc/transform/onnx | ONNX | +| mindspore/ccsrc/transform/express_ir | EXPRESS | | mindspore/ccsrc/utils | UTILS | | mindspore/ccsrc/vm | VM | | mindspore/ccsrc | ME | diff --git a/tutorials/training/source_zh_cn/advanced_use/custom_debugging_info.md b/tutorials/training/source_zh_cn/advanced_use/custom_debugging_info.md index ff75859da4a09eaaf12017dfd318b20a4b6cd135..aac4e00000f236dbcb5881818b56cf3299b6bc8f 100644 --- a/tutorials/training/source_zh_cn/advanced_use/custom_debugging_info.md +++ b/tutorials/training/source_zh_cn/advanced_use/custom_debugging_info.md @@ -434,7 +434,7 @@ MindSpore子模块按照目录划分如下: | mindspore/ccsrc/pybind_api | COMMON | | mindspore/ccsrc/runtime/device | DEVICE | | mindspore/ccsrc/transform/graph_ir | GE_ADPT | -| mindspore/ccsrc/transform/onnx | ONNX | +| mindspore/ccsrc/transform/express_ir | EXPRESS | | mindspore/ccsrc/utils | UTILS | | mindspore/ccsrc/vm | VM | | mindspore/ccsrc | ME |