From 6fc8c46e54d4cc6368b78ef81d3de3721144bb86 Mon Sep 17 00:00:00 2001 From: snowdong Date: Fri, 27 May 2022 14:05:49 +0800 Subject: [PATCH] =?UTF-8?q?=E6=8F=90=E4=BA=A43D=5FEDSR=5FID3005=5Ffor=5FPy?= =?UTF-8?q?Torch=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../others/3D_EDSR_ID3005_for_PyTorch/LICENSE | 201 ++++++++++++++++++ .../3D_EDSR_ID3005_for_PyTorch/README.md | 44 ++++ .../dataset_generator.py | 85 ++++++++ .../3D_EDSR_ID3005_for_PyTorch/edsr_x3_3d.py | 91 ++++++++ .../3D_EDSR_ID3005_for_PyTorch/load_data.py | 73 +++++++ .../3D_EDSR_ID3005_for_PyTorch/main_edsr.py | 145 +++++++++++++ .../modelarts_entry_acc.py | 63 ++++++ .../modelarts_entry_perf.py | 63 ++++++ .../modelzoo_level.txt | 3 + .../3D_EDSR_ID3005_for_PyTorch/option.py | 82 +++++++ .../requirements.txt | 2 + .../test/train_full_1p.sh | 186 ++++++++++++++++ .../test/train_performance_1p.sh | 186 ++++++++++++++++ 13 files changed, 1224 insertions(+) create mode 100644 PyTorch/contrib/cv/others/3D_EDSR_ID3005_for_PyTorch/LICENSE create mode 100644 PyTorch/contrib/cv/others/3D_EDSR_ID3005_for_PyTorch/README.md create mode 100644 PyTorch/contrib/cv/others/3D_EDSR_ID3005_for_PyTorch/dataset_generator.py create mode 100644 PyTorch/contrib/cv/others/3D_EDSR_ID3005_for_PyTorch/edsr_x3_3d.py create mode 100644 PyTorch/contrib/cv/others/3D_EDSR_ID3005_for_PyTorch/load_data.py create mode 100644 PyTorch/contrib/cv/others/3D_EDSR_ID3005_for_PyTorch/main_edsr.py create mode 100644 PyTorch/contrib/cv/others/3D_EDSR_ID3005_for_PyTorch/modelarts_entry_acc.py create mode 100644 PyTorch/contrib/cv/others/3D_EDSR_ID3005_for_PyTorch/modelarts_entry_perf.py create mode 100644 PyTorch/contrib/cv/others/3D_EDSR_ID3005_for_PyTorch/modelzoo_level.txt create mode 100644 PyTorch/contrib/cv/others/3D_EDSR_ID3005_for_PyTorch/option.py create mode 100644 PyTorch/contrib/cv/others/3D_EDSR_ID3005_for_PyTorch/requirements.txt create mode 100644 PyTorch/contrib/cv/others/3D_EDSR_ID3005_for_PyTorch/test/train_full_1p.sh create mode 100644 PyTorch/contrib/cv/others/3D_EDSR_ID3005_for_PyTorch/test/train_performance_1p.sh diff --git a/PyTorch/contrib/cv/others/3D_EDSR_ID3005_for_PyTorch/LICENSE b/PyTorch/contrib/cv/others/3D_EDSR_ID3005_for_PyTorch/LICENSE new file mode 100644 index 0000000000..eeac88fb9d --- /dev/null +++ b/PyTorch/contrib/cv/others/3D_EDSR_ID3005_for_PyTorch/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. \ No newline at end of file diff --git a/PyTorch/contrib/cv/others/3D_EDSR_ID3005_for_PyTorch/README.md b/PyTorch/contrib/cv/others/3D_EDSR_ID3005_for_PyTorch/README.md new file mode 100644 index 0000000000..755e945bce --- /dev/null +++ b/PyTorch/contrib/cv/others/3D_EDSR_ID3005_for_PyTorch/README.md @@ -0,0 +1,44 @@ +# 3D_EDSR + +The 3D EDSR (Enhanced Deep Super Resolution) convolution neural network used in this repository is based on the implementation of the CVPR2017 workshop Paper: "Enhanced Deep Residual Networks for Single Image Super-Resolution" (https://arxiv.org/pdf/1707.02921.pdf) using PyTorch on the X-Ray Micro-CT images, mainly modified from [GitHub - sci-sjj/EDSRmodelling](https://github.com/sci-sjj/EDSRmodelling). + +## 3D_EDSR Detail + +As of the current date, Ascend-Pytorch is still inefficient for contiguous operations.Therefore, 3D_EDSR is re-implemented using semantics such as custom OP. + +## Requirements + +- Install PyTorch ([pytorch.org](https://gitee.com/link?target=http%3A%2F%2Fpytorch.org)) +- `pip install -r requirements.txt` +- Download the Multi-resolution X-Ray micro-CT images of Bentheimer Sandstones dataset from [Zenodo at DOI: 10.5281/zenodo.5542624](https://doi.org/10.5281/zenodo.5542623). + +## training/testing dataset + +To generate suitable training/testing images (sub-slices of the full data above), the following code can be run: + +- dataset_generator.py. This generates LR and registered x3 HR sub-images for 3D EDSR training/testing. The LR/HR sub-images are separated into two different folders LR and HR + +## Training + +The 3D EDSR model can then be trained on the LR and HR sub-sampled data via: + +- main_edsr.py. This trains the 3D EDSR network on the LR/HR data. It requires the code load_data.py, which is the sub-image loader for 3D EDSR training. It also requires the 3D EDSR model structure code edsr_x3_3d.py. The code then saves the trained network as 3D_EDSR.pt. + +```shell +# training 1p accuracy +bash ./test/train_full_1p.sh --data_path=real_data_path + +# training 1p performance +bash ./test/train_performance_1p.sh --data_path=real_data_path +``` + +Log path: test/output/devie_id/train_${device_id}.log or obs://cann-idxxx/npu/workspace/MA-new-npu-xxx/log/modelarts-job-xxx-worker-0.log + +## 3D_EDSR result + +| PSNR (dB) | Npu_nums | Epochs | AMP_Type | +| --------- | -------- | ------ | -------- | +| 23.1217 | 1 | 50 | O2 | + + + diff --git a/PyTorch/contrib/cv/others/3D_EDSR_ID3005_for_PyTorch/dataset_generator.py b/PyTorch/contrib/cv/others/3D_EDSR_ID3005_for_PyTorch/dataset_generator.py new file mode 100644 index 0000000000..285e831f25 --- /dev/null +++ b/PyTorch/contrib/cv/others/3D_EDSR_ID3005_for_PyTorch/dataset_generator.py @@ -0,0 +1,85 @@ +# BSD 3-Clause License +# +# Copyright (c) 2017 xxxx +# All rights reserved. +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# * Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# ============================================================================ +# cropping Train/Test image patches +import os +from skimage import io +from option import args + +# Train and Test image save paths +Train_LR_folder = os.path.join(args.dir_train_data, 'LR') +Train_HR_folder = os.path.join(args.dir_train_data, 'HR') +Test_LR_folder = os.path.join(args.dir_test_data, 'LR') +Test_HR_folder = os.path.join(args.dir_test_data, 'HR') +if not os.path.exists(Train_LR_folder): + os.makedirs(Train_LR_folder) +if not os.path.exists(Train_HR_folder): + os.makedirs(Train_HR_folder) +if not os.path.exists(Test_LR_folder): + os.makedirs(Test_LR_folder) +if not os.path.exists(Test_HR_folder): + os.makedirs(Test_HR_folder) +# generate LR data +Core1_Subvol1_LR = io.imread(os.path.join(args.dir_data, 'Core1_Subvol1_LR.tif')) +Image_Size = 210 +LR_Image_Size = 15 +HR_Image_Size = LR_Image_Size * 3 +Train_Dataset_Size = 801 +Test_Dataset_Size = 10 +count1 = 1 +for z in range(LR_Image_Size, Image_Size, LR_Image_Size): + for y in range(LR_Image_Size, Image_Size, LR_Image_Size): + for x in range(LR_Image_Size, Image_Size, LR_Image_Size): + img = Core1_Subvol1_LR[z - LR_Image_Size:z + LR_Image_Size, y - LR_Image_Size:y + LR_Image_Size, x - LR_Image_Size:x + LR_Image_Size] + if count1 < Train_Dataset_Size: + io.imsave(os.path.join(Train_LR_folder, str(count1) + '.tif'), img) + count1 += 1 + elif count1 < Train_Dataset_Size + Test_Dataset_Size: + io.imsave(os.path.join(Test_LR_folder, str(count1) + '.tif'), img) + count1 += 1 + else: + break + +# generate HR data +Core1_Subvol1_HR = io.imread(os.path.join(args.dir_data, 'Core1_Subvol1_HR.tif')) +count2 = 1 +for z in range(HR_Image_Size, Image_Size * 3, HR_Image_Size): + for y in range(HR_Image_Size, Image_Size * 3, HR_Image_Size): + for x in range(HR_Image_Size, Image_Size * 3, HR_Image_Size): + img = Core1_Subvol1_HR[z - HR_Image_Size:z + HR_Image_Size, y - HR_Image_Size:y + HR_Image_Size, x - HR_Image_Size:x + HR_Image_Size] + if count2 < Train_Dataset_Size: + io.imsave(os.path.join(Train_HR_folder, str(count2) + '.tif'), img) + count2 += 1 + elif count2 < Train_Dataset_Size + Test_Dataset_Size: + io.imsave(os.path.join(Test_HR_folder, str(count2) + '.tif'), img) + count2 += 1 + else: + break diff --git a/PyTorch/contrib/cv/others/3D_EDSR_ID3005_for_PyTorch/edsr_x3_3d.py b/PyTorch/contrib/cv/others/3D_EDSR_ID3005_for_PyTorch/edsr_x3_3d.py new file mode 100644 index 0000000000..98badd9d5b --- /dev/null +++ b/PyTorch/contrib/cv/others/3D_EDSR_ID3005_for_PyTorch/edsr_x3_3d.py @@ -0,0 +1,91 @@ +# BSD 3-Clause License +# +# Copyright (c) 2017 xxxx +# All rights reserved. +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# * Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# ============================================================================ +import torch +import torch.nn as nn +from option import args +# 3D EDSR model structure +def default_conv(in_channelss, out_channels, kernel_size, bias=True): + return nn.Conv3d( + in_channelss, out_channels, kernel_size, + padding=(kernel_size // 2), bias=bias) + +class ResBlock(nn.Module): + def __init__( + self, conv, n_feat, kernel_size, + bias=True, bn=False, act=nn.ReLU(True), res_scale=1): + + super(ResBlock, self).__init__() + modules_body = [] + for i in range(2): + modules_body.append(conv(n_feat, n_feat, kernel_size, bias=bias)) + if bn: modules_body.append(nn.BatchNorm2d(n_feat)) + if i == 0: modules_body.append(act) + + self.body = nn.Sequential(*modules_body) + self.res_scale = res_scale + + def forward(self, x): + res = self.body(x).mul(self.res_scale) + res += x + return res + + +class EDSR(nn.Module): + def __init__(self, conv=default_conv): + super(EDSR, self).__init__() + n_feats = args.n_feats # 64 + kernel_size = args.kernel_size + n_resblock = args.n_resblocks # 16 + act = nn.ReLU(True) + res_scale = args.res_scale + scale = args.scale + + self.head = nn.Sequential(conv(1, n_feats, kernel_size)) + + modules_body = [ + ResBlock( + conv, n_feats, kernel_size, act=act, res_scale=res_scale) for _ in range(n_resblock)] + self.body = nn.Sequential(*modules_body) + + modules_tail = [ + nn.Upsample(scale_factor=scale, mode='trilinear', align_corners=False), + conv(n_feats, 1, kernel_size)] + self.tail = nn.Sequential(*modules_tail) + + def forward(self, x): + x = x.contiguous() + x = self.head(x) + res = self.body(x) + res += x + x = self.tail(res) + #x = torch.squeeze(x, dim=1) + return x diff --git a/PyTorch/contrib/cv/others/3D_EDSR_ID3005_for_PyTorch/load_data.py b/PyTorch/contrib/cv/others/3D_EDSR_ID3005_for_PyTorch/load_data.py new file mode 100644 index 0000000000..e612208386 --- /dev/null +++ b/PyTorch/contrib/cv/others/3D_EDSR_ID3005_for_PyTorch/load_data.py @@ -0,0 +1,73 @@ +# BSD 3-Clause License +# +# Copyright (c) 2017 xxxx +# All rights reserved. +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# * Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# ============================================================================ +from torch.utils.data import Dataset +from os import listdir +from skimage import io +from os.path import join +import numpy as np +from torchvision.transforms import Compose, ToTensor + + +# training/testing data generation +class DatasetFromFolder(Dataset): + def __init__(self, root): + super(DatasetFromFolder, self).__init__() + self.image_filenames1 = [join(root, x) for x in listdir(root)][0] # LR + self.image_filenames2 = [join(root, x) for x in listdir(root)][1] # HR + if "HR" in self.image_filenames1: + tmp = self.image_filenames1 + self.image_filenames1 = self.image_filenames2 + self.image_filenames2 = tmp + LIST1 = listdir(self.image_filenames1) + LIST2 = listdir(self.image_filenames2) + LIST1.sort(key=lambda x: int(x[:-4])) + LIST2.sort(key=lambda x: int(x[:-4])) + self.img1 = [join(self.image_filenames1, x) for x in LIST1] + self.img2 = [join(self.image_filenames2, x) for x in LIST2] + self.train_hr_transform = tensor_transform() + self.train_lr_transform = tensor_transform() + + def __getitem__(self, index): + lr_image = self.train_lr_transform(io.imread(self.img1[index])) # LR + hr_image = self.train_lr_transform(io.imread(self.img2[index])) # HR + hr_image = np.expand_dims(hr_image, axis=0) + lr_image = np.expand_dims(lr_image, axis=0) + return lr_image, hr_image + + def __len__(self): + return len(self.img1) + + +def tensor_transform(): + return Compose([ + ToTensor(), + ]) diff --git a/PyTorch/contrib/cv/others/3D_EDSR_ID3005_for_PyTorch/main_edsr.py b/PyTorch/contrib/cv/others/3D_EDSR_ID3005_for_PyTorch/main_edsr.py new file mode 100644 index 0000000000..3a25091b4a --- /dev/null +++ b/PyTorch/contrib/cv/others/3D_EDSR_ID3005_for_PyTorch/main_edsr.py @@ -0,0 +1,145 @@ +# BSD 3-Clause License +# +# Copyright (c) 2017 xxxx +# All rights reserved. +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# * Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# ============================================================================ +# main code for 3D EDSR training +import torch +import torch.backends.cudnn as cudnn +import torch.nn as nn +import torch.optim as optim +from torch.autograd import Variable +from torch.utils.data import DataLoader + +import torch.npu +from edsr_x3_3d import EDSR +from load_data import DatasetFromFolder +import math +import time +from option import args + + +def main(): + cudnn.benchmark = True + print("Random Seed: ", args.seed) + torch.manual_seed(args.seed) + if args.use_npu: + print("===> Use NPU") + device = torch.device("npu:{}".format(args.npu)) + print(device) + torch.npu.set_device(device) + else: + print("===> Use GPU") + device = torch.device("cuda:{}".format(args.cuda)) + + print("===> Loading datasets") + # load train/test data + train_set = DatasetFromFolder(args.dir_train_data) + test_set = DatasetFromFolder(args.dir_test_data) + training_data_loader = DataLoader(dataset=train_set, num_workers=args.threads, batch_size=args.batch_size, + shuffle=True) + test_data_loader = DataLoader(dataset=test_set, batch_size=1, shuffle=False) + print("===> Building model") + model = EDSR() + criterion = nn.L1Loss(reduction='mean') + print("===> Setting GPU/NPU") + model = model.to(device) + criterion = criterion.to(device) + + print("===> Setting Optimizer") + print("===> Training") + for epoch in range(1, args.epochs + 1): + optimizer = optim.Adam(model.parameters(), lr=args.lr, betas=args.betas) + lr = adjust_learning_rate(epoch - 1) + for param_group in optimizer.param_groups: + param_group["lr"] = lr + print("Epoch={} Train, lr={}".format(epoch, optimizer.param_groups[0]["lr"])) + train(training_data_loader, optimizer, model, criterion, epoch, device) + test(test_data_loader, model, criterion, epoch, device) + torch.save(model.state_dict(), './3D_EDSR.pt') + + +def adjust_learning_rate(epoch): + """Sets the learning rate to the initial LR decayed by 10""" + lr = args.lr * (args.gamma ** (epoch // args.step)) + return lr + + +def train(training_data_loader, optimizer, model, criterion, epoch, device): + model.train() + for iteration, batch in enumerate(training_data_loader, 1): + input, target = Variable(batch[0]), Variable(batch[1], requires_grad=False) # requires_grad=False + input = input.to(device) + target = target.to(device) + t0 = time.time() + sr = model(input) + loss = criterion(sr, target) + loss = loss.to(device) + optimizer.zero_grad() + loss.backward() + optimizer.step() + t1 = time.time() + mse = torch.mean((sr - target) ** 2) + psnr = 10 * math.log10(1.0 / torch.mean(mse)) + if iteration % 10 == 0: + print("===> Epoch[{}]({}/{}): Train Loss:{:.10f} psnr:{:.10f}".format( \ + epoch, iteration, len(training_data_loader), loss.item(), psnr)) + print('===> Timer:%.4f' % (t1 - t0)) + + +def test(test_data_loader, model, criterion, epoch, device): + model.eval() + loss = 0 + psnr = 0 + t1 = time.time() + count = 0 + for iteration, batch in enumerate(test_data_loader, 1): + with torch.no_grad(): + input, target = Variable(batch[0]), Variable(batch[1], requires_grad=False) + input = input.to(device) + target = target.to(device) + sr = model(input) + loss_tem = criterion(sr, target) + loss_tem = loss_tem.to(device) + loss += loss_tem.item() + mse = torch.mean((sr - target) ** 2) + psnr_tem = 10 * math.log10(1.0 / torch.mean(mse)) + psnr += psnr_tem + count += 1 + + tloss = loss / count + tpsnr = psnr / count + t2 = time.time() + print("Epoch[{}]: Test Loss:{:.10f} psnr:{:.10f} Timer:{:.4f}".format( \ + epoch, tloss, tpsnr, (t2 - t1))) + if epoch == args.epochs: + print("Final Accuracy accuracy psnr: {:.10f} ".format(tpsnr)) + +if __name__ == "__main__": + main() diff --git a/PyTorch/contrib/cv/others/3D_EDSR_ID3005_for_PyTorch/modelarts_entry_acc.py b/PyTorch/contrib/cv/others/3D_EDSR_ID3005_for_PyTorch/modelarts_entry_acc.py new file mode 100644 index 0000000000..13077b10e6 --- /dev/null +++ b/PyTorch/contrib/cv/others/3D_EDSR_ID3005_for_PyTorch/modelarts_entry_acc.py @@ -0,0 +1,63 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import argparse +import sys + +# 解析输入参数data_url +parser = argparse.ArgumentParser() +parser.add_argument("--data_url", type=str, default="/home/ma-user/modelarts/inputs/data_url_0") +parser.add_argument("--train_url", type=str, default="/home/ma-user/modelarts/outputs/train_url_0/") +config = parser.parse_args() + +print("[CANN-Modelzoo] code_dir path is [%s]" % (sys.path[0])) +code_dir = sys.path[0] +os.chdir(code_dir) +print("[CANN-Modelzoo] work_dir path is [%s]" % (os.getcwd())) + +print("[CANN-Modelzoo] before train - list my run files:") +os.system("ls -al /usr/local/Ascend/ascend-toolkit/") + +print("[CANN-Modelzoo] before train - list my dataset files:") +os.system("ls -al %s" % config.data_url) + +print("[CANN-Modelzoo] start run train shell") +# 设置sh文件格式为linux可执行 +os.system("dos2unix ./test/*") + +# 执行train_full_1p.sh或者train_performance_1p.sh，需要用户自己指定 +# full和performance的差异，performance只需要执行很少的step，控制在15分钟以内，主要关注性能FPS +os.system("bash ./test/train_full_1p.sh --data_path=%s --output_path=%s " % (config.data_url, config.train_url)) + +print("[CANN-Modelzoo] finish run train shell") + +# 将当前执行目录所有文件拷贝到obs的output进行备份 +print("[CANN-Modelzoo] after train - list my output files:") +os.system("cp -r %s %s " % (code_dir, config.train_url)) +os.system("ls -al %s" % config.train_url) diff --git a/PyTorch/contrib/cv/others/3D_EDSR_ID3005_for_PyTorch/modelarts_entry_perf.py b/PyTorch/contrib/cv/others/3D_EDSR_ID3005_for_PyTorch/modelarts_entry_perf.py new file mode 100644 index 0000000000..14384e227a --- /dev/null +++ b/PyTorch/contrib/cv/others/3D_EDSR_ID3005_for_PyTorch/modelarts_entry_perf.py @@ -0,0 +1,63 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import argparse +import sys + +# 解析输入参数data_url +parser = argparse.ArgumentParser() +parser.add_argument("--data_url", type=str, default="/home/ma-user/modelarts/inputs/data_url_0") +parser.add_argument("--train_url", type=str, default="/home/ma-user/modelarts/outputs/train_url_0/") +config = parser.parse_args() + +print("[CANN-Modelzoo] code_dir path is [%s]" % (sys.path[0])) +code_dir = sys.path[0] +os.chdir(code_dir) +print("[CANN-Modelzoo] work_dir path is [%s]" % (os.getcwd())) + +print("[CANN-Modelzoo] before train - list my run files:") +os.system("ls -al /usr/local/Ascend/ascend-toolkit/") + +print("[CANN-Modelzoo] before train - list my dataset files:") +os.system("ls -al %s" % config.data_url) + +print("[CANN-Modelzoo] start run train shell") +# 设置sh文件格式为linux可执行 +os.system("dos2unix ./test/*") + +# 执行train_full_1p.sh或者train_performance_1p.sh，需要用户自己指定 +# full和performance的差异，performance只需要执行很少的step，控制在15分钟以内，主要关注性能FPS +os.system("bash ./test/train_performance_1p.sh --data_path=%s --output_path=%s " % (config.data_url, config.train_url)) + +print("[CANN-Modelzoo] finish run train shell") + +# 将当前执行目录所有文件拷贝到obs的output进行备份 +print("[CANN-Modelzoo] after train - list my output files:") +os.system("cp -r %s %s " % (code_dir, config.train_url)) +os.system("ls -al %s" % config.train_url) diff --git a/PyTorch/contrib/cv/others/3D_EDSR_ID3005_for_PyTorch/modelzoo_level.txt b/PyTorch/contrib/cv/others/3D_EDSR_ID3005_for_PyTorch/modelzoo_level.txt new file mode 100644 index 0000000000..0b49b4fb26 --- /dev/null +++ b/PyTorch/contrib/cv/others/3D_EDSR_ID3005_for_PyTorch/modelzoo_level.txt @@ -0,0 +1,3 @@ +FuncStatus:OK +PerfStatus:OK +PrecisionStatus:OK \ No newline at end of file diff --git a/PyTorch/contrib/cv/others/3D_EDSR_ID3005_for_PyTorch/option.py b/PyTorch/contrib/cv/others/3D_EDSR_ID3005_for_PyTorch/option.py new file mode 100644 index 0000000000..463dfb8e56 --- /dev/null +++ b/PyTorch/contrib/cv/others/3D_EDSR_ID3005_for_PyTorch/option.py @@ -0,0 +1,82 @@ +# BSD 3-Clause License +# +# Copyright (c) 2017 xxxx +# All rights reserved. +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# * Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# ============================================================================ +import argparse + +parser = argparse.ArgumentParser(description='3D EDSR Training in Micro-CT Rock Images With Pytorch') + +# Hardware specifications +parser.add_argument('--threads', type=int, default=1, + help='number of threads for data loading') +parser.add_argument('--seed', type=int, default=1, + help='random seed') +parser.add_argument('--cuda', type=int, default=0, + help='explicitly specify to use gpu number') +# npu setting +parser.add_argument("--use_npu", action="store_true", + help="Use NPU to train the model") +parser.add_argument("--npu", default=0, type=int, help="NPU id to use") + +# Data specifications +parser.add_argument('--dir_data', type=str, default='./dataset', + help='dataset directory') +parser.add_argument('--dir_train_data', type=str, default='./dataset/train/', + help='train dataset name') +parser.add_argument('--dir_test_data', type=str, default='./dataset/test/', + help='test dataset name') +parser.add_argument('--scale', type=str, default='3', + help='super resolution scale') + +# Model specifications +parser.add_argument('--n_resblocks', type=int, default=32, + help='number of residual blocks') +parser.add_argument('--n_feats', type=int, default=64, + help='number of feature maps') +parser.add_argument('--kernel_size', type=int, default=3, + help='kernel size') +parser.add_argument('--res_scale', type=float, default=1, + help='residual scaling') + +# Training specifications +parser.add_argument("--epochs", type=int, default=50, + help="number of epochs to train") +parser.add_argument("--batch_size", type=int, default=1, + help="input batch size for training") +parser.add_argument('--lr', type=float, default=1e-4, + help='learning rate') +parser.add_argument('--gamma', type=float, default=0.1, + help='learning rate decay factor for step decay') +parser.add_argument('--betas', type=tuple, default=(0.9, 0.999), + help='ADAM beta') +parser.add_argument('--step', type=int, default=10, + help='learning rate to the initial LR decayed by step') + +args = parser.parse_args() diff --git a/PyTorch/contrib/cv/others/3D_EDSR_ID3005_for_PyTorch/requirements.txt b/PyTorch/contrib/cv/others/3D_EDSR_ID3005_for_PyTorch/requirements.txt new file mode 100644 index 0000000000..beb452e9e9 --- /dev/null +++ b/PyTorch/contrib/cv/others/3D_EDSR_ID3005_for_PyTorch/requirements.txt @@ -0,0 +1,2 @@ +Python 3.7.4 +Pytorch 1.8.1 \ No newline at end of file diff --git a/PyTorch/contrib/cv/others/3D_EDSR_ID3005_for_PyTorch/test/train_full_1p.sh b/PyTorch/contrib/cv/others/3D_EDSR_ID3005_for_PyTorch/test/train_full_1p.sh new file mode 100644 index 0000000000..2b40c37a1b --- /dev/null +++ b/PyTorch/contrib/cv/others/3D_EDSR_ID3005_for_PyTorch/test/train_full_1p.sh @@ -0,0 +1,186 @@ +#!/bin/bash + +########################################################## +#########第3行 至 100行，请一定不要、不要、不要修改########## +#########第3行 至 100行，请一定不要、不要、不要修改########## +#########第3行 至 100行，请一定不要、不要、不要修改########## +########################################################## +# shell脚本所在路径 +cur_path=`echo $(cd $(dirname $0);pwd)` + +# 判断当前shell是否是performance +perf_flag=`echo $0 | grep performance | wc -l` + +# 当前执行网络的名称 +Network=`echo $(cd $(dirname $0);pwd) | awk -F"/" '{print $(NF-1)}'` + +export RANK_SIZE=1 +export RANK_ID=0 +export JOB_ID=10087 + +# 路径参数初始化 +data_path="" +output_path="" + +# 帮助信息，不需要修改 +if [[ $1 == --help || $1 == -h ]];then + echo"usage:./train_performance_1P.sh " + echo " " + echo "parameter explain: + --data_path # dataset of training + --output_path # output of training + --train_steps # max_step for training + --train_epochs # max_epoch for training + --batch_size # batch size + -h/--help show help message + " + exit 1 +fi + +# 参数校验，不需要修改 +for para in $* +do + if [[ $para == --data_path* ]];then + data_path=`echo ${para#*=}` + elif [[ $para == --output_path* ]];then + output_path=`echo ${para#*=}` + elif [[ $para == --train_steps* ]];then + train_steps=`echo ${para#*=}` + elif [[ $para == --train_epochs* ]];then + train_epochs=`echo ${para#*=}` + elif [[ $para == --batch_size* ]];then + batch_size=`echo ${para#*=}` + fi +done + +# 校验是否传入data_path,不需要修改 +if [[ $data_path == "" ]];then + echo "[Error] para \"data_path\" must be config" + exit 1 +fi + +# 校验是否传入output_path,不需要修改 +if [[ $output_path == "" ]];then + output_path="./test/output/${ASCEND_DEVICE_ID}" +fi + +# 设置打屏日志文件名，请保留，文件名为${print_log} +print_log="./test/output/${ASCEND_DEVICE_ID}/train_${ASCEND_DEVICE_ID}.log" +modelarts_flag=`cat /etc/passwd |grep ma-user` +if [ x"${modelarts_flag}" != x ]; +then + echo "running with modelarts_flag..." + print_log_name=`ls /home/ma-user/modelarts/log/ | grep proc-rank` + print_log="/home/ma-user/modelarts/log/${print_log_name}" +fi +echo "### get your log here : ${print_log}" + +CaseName="" +function get_casename() +{ + if [ x"${perf_flag}" = x1 ]; + then + CaseName=${Network}_bs${batch_size}_${RANK_SIZE}'p'_'perf' + else + CaseName=${Network}_bs${batch_size}_${RANK_SIZE}'p'_'acc' + fi +} + +# 跳转到code目录 +cd ${cur_path}/../ +rm -rf ./test/output/${ASCEND_DEVICE_ID} +mkdir -p ./test/output/${ASCEND_DEVICE_ID} + +# 训练开始时间记录，不需要修改 +start_time=$(date +%s) +########################################################## +#########第3行 至 100行，请一定不要、不要、不要修改########## +#########第3行 至 100行，请一定不要、不要、不要修改########## +#########第3行 至 100行，请一定不要、不要、不要修改########## +########################################################## + +#========================================================= +#========================================================= +#========训练执行命令，需要根据您的网络进行修改============== +#========================================================= +#========================================================= +# 基础参数，需要模型审视修改 +# 您的训练数据集在${data_path}路径下，请直接使用这个变量获取 +# 您的训练输出目录在${output_path}路径下，请直接使用这个变量获取 +# 您的其他基础参数，可以自定义增加，但是batch_size请保留，并且设置正确的值 +batch_size=1 + +python3.7 ./dataset_generator.py --dir_data=${data_path} + +if [ x"${modelarts_flag}" != x ]; +then + python3.7 ./main_edsr.py --batch_size=${batch_size} --use_npu +else + python3.7 ./main_edsr.py --batch_size=${batch_size} --use_npu 1>${print_log} 2>&1 +fi + +# 性能相关数据计算 +StepTime=`grep "===> Timer:" ${print_log} | tail -n 10 | awk -F ":" '{print $NF}' | awk '{sum+=$1} END {print sum/NR}'` +FPS=`awk 'BEGIN{printf "%.2f\n", '${batch_size}'/'${StepTime}'}'` + +# 精度相关数据计算 +train_accuracy=`grep "Final Accuracy accuracy" ${print_log} | awk '{print $NF}'` +# 提取所有loss打印信息 +grep "Test Loss:" ${print_log} | awk -F ":" '{print $3}' | awk '{print $1}' > ./test/output/${ASCEND_DEVICE_ID}/my_output_loss.txt + +########################################################### +#########后面的所有内容请不要修改########################### +#########后面的所有内容请不要修改########################### +#########后面的所有内容请不要修改########################### +########################################################### + +# 判断本次执行是否正确使用Ascend NPU +tf_flag=`echo ${Network} | grep TensorFlow | wc -l` +use_npu_flag=`grep "The model has been compiled on the Ascend AI processor" ${print_log} | wc -l` +if [ x"${use_npu_flag}" == x0 -a x"${tf_flag}" == x1 ]; +then + echo "------------------ ERROR NOTICE START ------------------" + echo "ERROR, your task haven't used Ascend NPU, please check your npu Migration." + echo "------------------ ERROR NOTICE END------------------" +else + echo "------------------ INFO NOTICE START------------------" + echo "INFO, your task have used Ascend NPU, please check your result." + echo "------------------ INFO NOTICE END------------------" +fi + +# 获取最终的casename，请保留，case文件名为${CaseName} +get_casename + +# 重命名loss文件 +if [ -f ./test/output/${ASCEND_DEVICE_ID}/my_output_loss.txt ]; +then + mv ./test/output/${ASCEND_DEVICE_ID}/my_output_loss.txt ./test/output/${ASCEND_DEVICE_ID}/${CaseName}_loss.txt +fi + +# 训练端到端耗时 +end_time=$(date +%s) +e2e_time=$(( $end_time - $start_time )) + +echo "------------------ Final result ------------------" +# 输出性能FPS/单step耗时/端到端耗时 +echo "Final Performance images/sec : $FPS" +echo "Final Performance sec/step : $StepTime" +echo "E2E Training Duration sec : $e2e_time" + +# 输出训练精度 +echo "Final Train Accuracy : ${train_accuracy}" + +# 最后一个迭代loss值，不需要修改 +ActualLoss=(`awk 'END {print $NF}' $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}_loss.txt`) + +#关键信息打印到${CaseName}.log中，不需要修改 +echo "Network = ${Network}" > $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "RankSize = ${RANK_SIZE}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "BatchSize = ${batch_size}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "DeviceType = `uname -m`" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "CaseName = ${CaseName}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "ActualFPS = ${FPS}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "TrainingTime = ${StepTime}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "ActualLoss = ${ActualLoss}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "E2ETrainingTime = ${e2e_time}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "TrainAccuracy = ${train_accuracy}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log \ No newline at end of file diff --git a/PyTorch/contrib/cv/others/3D_EDSR_ID3005_for_PyTorch/test/train_performance_1p.sh b/PyTorch/contrib/cv/others/3D_EDSR_ID3005_for_PyTorch/test/train_performance_1p.sh new file mode 100644 index 0000000000..0a45d0a75e --- /dev/null +++ b/PyTorch/contrib/cv/others/3D_EDSR_ID3005_for_PyTorch/test/train_performance_1p.sh @@ -0,0 +1,186 @@ +#!/bin/bash + +########################################################## +#########第3行 至 100行，请一定不要、不要、不要修改########## +#########第3行 至 100行，请一定不要、不要、不要修改########## +#########第3行 至 100行，请一定不要、不要、不要修改########## +########################################################## +# shell脚本所在路径 +cur_path=`echo $(cd $(dirname $0);pwd)` + +# 判断当前shell是否是performance +perf_flag=`echo $0 | grep performance | wc -l` + +# 当前执行网络的名称 +Network=`echo $(cd $(dirname $0);pwd) | awk -F"/" '{print $(NF-1)}'` + +export RANK_SIZE=1 +export RANK_ID=0 +export JOB_ID=10087 + +# 路径参数初始化 +data_path="" +output_path="" + +# 帮助信息，不需要修改 +if [[ $1 == --help || $1 == -h ]];then + echo"usage:./train_performance_1P.sh " + echo " " + echo "parameter explain: + --data_path # dataset of training + --output_path # output of training + --train_steps # max_step for training + --train_epochs # max_epoch for training + --batch_size # batch size + -h/--help show help message + " + exit 1 +fi + +# 参数校验，不需要修改 +for para in $* +do + if [[ $para == --data_path* ]];then + data_path=`echo ${para#*=}` + elif [[ $para == --output_path* ]];then + output_path=`echo ${para#*=}` + elif [[ $para == --train_steps* ]];then + train_steps=`echo ${para#*=}` + elif [[ $para == --train_epochs* ]];then + train_epochs=`echo ${para#*=}` + elif [[ $para == --batch_size* ]];then + batch_size=`echo ${para#*=}` + fi +done + +# 校验是否传入data_path,不需要修改 +if [[ $data_path == "" ]];then + echo "[Error] para \"data_path\" must be config" + exit 1 +fi + +# 校验是否传入output_path,不需要修改 +if [[ $output_path == "" ]];then + output_path="./test/output/${ASCEND_DEVICE_ID}" +fi + +# 设置打屏日志文件名，请保留，文件名为${print_log} +print_log="./test/output/${ASCEND_DEVICE_ID}/train_${ASCEND_DEVICE_ID}.log" +modelarts_flag=`cat /etc/passwd |grep ma-user` +if [ x"${modelarts_flag}" != x ]; +then + echo "running with modelarts..." + print_log_name=`ls /home/ma-user/modelarts/log/ | grep proc-rank` + print_log="/home/ma-user/modelarts/log/${print_log_name}" +fi +echo "### get your log here : ${print_log}" + +CaseName="" +function get_casename() +{ + if [ x"${perf_flag}" = x1 ]; + then + CaseName=${Network}_bs${batch_size}_${RANK_SIZE}'p'_'perf' + else + CaseName=${Network}_bs${batch_size}_${RANK_SIZE}'p'_'acc' + fi +} + +# 跳转到code目录 +cd ${cur_path}/../ +rm -rf ./test/output/${ASCEND_DEVICE_ID} +mkdir -p ./test/output/${ASCEND_DEVICE_ID} + +# 训练开始时间记录，不需要修改 +start_time=$(date +%s) +########################################################## +#########第3行 至 100行，请一定不要、不要、不要修改########## +#########第3行 至 100行，请一定不要、不要、不要修改########## +#########第3行 至 100行，请一定不要、不要、不要修改########## +########################################################## + +#========================================================= +#========================================================= +#========训练执行命令，需要根据您的网络进行修改============== +#========================================================= +#========================================================= +# 基础参数，需要模型审视修改 +# 您的训练数据集在${data_path}路径下，请直接使用这个变量获取 +# 您的训练输出目录在${output_path}路径下，请直接使用这个变量获取 +# 您的其他基础参数，可以自定义增加，但是batch_size请保留，并且设置正确的值 +batch_size=1 +train_epochs=2 + +python3.7 ./dataset_generator.py --dir_data=${data_path} + +if [ x"${modelarts_flag}" != x ]; +then + python3.7 ./main_edsr.py --batch_size=${batch_size} --epochs=${train_epochs} --use_npu +else + python3.7 ./main_edsr.py --batch_size=${batch_size} --epochs=${train_epochs} --use_npu 1>${print_log} 2>&1 +fi + +# 性能相关数据计算 +StepTime=`grep "===> Timer:" ${print_log} | tail -n 10 | awk -F ":" '{print $NF}' | awk '{sum+=$1} END {print sum/NR}'` +FPS=`awk 'BEGIN{printf "%.2f\n", '${batch_size}'/'${StepTime}'}'` + +# 精度相关数据计算 +train_accuracy=`grep "Final Accuracy accuracy" ${print_log} | awk '{print $NF}'` +# 提取所有loss打印信息 +grep "Test Loss:" ${print_log} | awk -F ":" '{print $3}' | awk '{print $1}' > ./test/output/${ASCEND_DEVICE_ID}/my_output_loss.txt + +########################################################### +#########后面的所有内容请不要修改########################### +#########后面的所有内容请不要修改########################### +#########后面的所有内容请不要修改########################### +########################################################### + +# 判断本次执行是否正确使用Ascend NPU +tf_flag=`echo ${Network} | grep TensorFlow | wc -l` +use_npu_flag=`grep "The model has been compiled on the Ascend AI processor" ${print_log} | wc -l` +if [ x"${use_npu_flag}" == x0 -a x"${tf_flag}" == x1 ]; +then + echo "------------------ ERROR NOTICE START ------------------" + echo "ERROR, your task haven't used Ascend NPU, please check your npu Migration." + echo "------------------ ERROR NOTICE END------------------" +else + echo "------------------ INFO NOTICE START------------------" + echo "INFO, your task have used Ascend NPU, please check your result." + echo "------------------ INFO NOTICE END------------------" +fi + +# 获取最终的casename，请保留，case文件名为${CaseName} +get_casename + +# 重命名loss文件 +if [ -f ./test/output/${ASCEND_DEVICE_ID}/my_output_loss.txt ]; +then + mv ./test/output/${ASCEND_DEVICE_ID}/my_output_loss.txt ./test/output/${ASCEND_DEVICE_ID}/${CaseName}_loss.txt +fi + +# 训练端到端耗时 +end_time=$(date +%s) +e2e_time=$(( $end_time - $start_time )) + +echo "------------------ Final result ------------------" +# 输出性能FPS/单step耗时/端到端耗时 +echo "Final Performance images/sec : $FPS" +echo "Final Performance sec/step : $StepTime" +echo "E2E Training Duration sec : $e2e_time" + +# 输出训练精度 +echo "Final Train Accuracy : ${train_accuracy}" + +# 最后一个迭代loss值，不需要修改 +ActualLoss=(`awk 'END {print $NF}' $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}_loss.txt`) + +#关键信息打印到${CaseName}.log中，不需要修改 +echo "Network = ${Network}" > $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "RankSize = ${RANK_SIZE}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "BatchSize = ${batch_size}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "DeviceType = `uname -m`" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "CaseName = ${CaseName}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "ActualFPS = ${FPS}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "TrainingTime = ${StepTime}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "ActualLoss = ${ActualLoss}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "E2ETrainingTime = ${e2e_time}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log \ No newline at end of file -- Gitee