diff --git a/ACL_TensorFlow/contrib/cv/edsr/.keep b/ACL_TensorFlow/contrib/cv/edsr/.keep new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/ACL_TensorFlow/contrib/cv/edsr/README.md b/ACL_TensorFlow/contrib/cv/edsr/README.md new file mode 100644 index 0000000000000000000000000000000000000000..2d70a935f9826d62a1113de5961387f46b118588 --- /dev/null +++ b/ACL_TensorFlow/contrib/cv/edsr/README.md @@ -0,0 +1,89 @@ +# ATC EDSR + + +- references： + + ["Enhanced Deep Residual Networks for Single Image Super-Resolution"](https://arxiv.org/abs/1707.02921) + + +- training model： + + [EDSR_ID1263_for_TensorFlow](https://gitee.com/ascend/ModelZoo-TensorFlow/tree/master/TensorFlow/contrib/cv/EDSR_ID1263_for_TensorFlow) + + +# 1. ckpt to pb + +download **ckpt** and use **ckpt2pb.py** process ckpt to pb. + + **Note** ：Before the model transformation, the **run.py** and **edsr.py** files in the training code need to be replaced with those in the current directory. + +[ckpt](https://pan.baidu.com/s/17vlOrwXbygdce8l8OHErCA) +Password:x0j4 + +# 2. pb to om +Command: +``` +atc --model=./edsr.pb --framework=3 --input_shape="LR:1,48,48,3" --output=./edsr --soc_version=Ascend310 +``` +[Pb](https://pan.baidu.com/s/1vxU_Q3qorOlPvDJYQ9tTsw) +Password:1wbo + +[OM](https://pan.baidu.com/s/1QeQSRdckigMAbBRnXooRJA) +Password:djqe + +# 3. compile msame +Reference to https://gitee.com/ascend/tools/tree/master/msame, compile **msame** + +Compile msame command: +```bash +. /home/HwHiAiUser/Ascend/ascend-toolkit/set_env.sh +export DDK_PATH=/home/HwHiAiUser/Ascend/ascend-toolkit/latest +export NPU_HOST_LIB=/home/HwHiAiUser/Ascend/ascend-toolkit/latest/runtime/lib64/stub +cd $HOME/AscendProjects/tools/msame/ +./build.sh g++ $HOME/AscendProjects/tools/msame/out + +``` + +# 4. inference +Inference command: +```bash +cd $HOME/AscendProjects/tools/msame/out +model_path=/home/HwHiAiUser/AscendProjects/EDSR/edsr.om +input_path=/home/HwHiAiUser/AscendProjects/EDSR/testpic.bin +output_path=/home/HwHiAiUser/AscendProjects/EDSR/output +./msame --model ${model_path} --input ${input_path} --output ${output_path} --outfmt TXT +``` + + + +Part of **Inference sys output**: +```bash +[INFO] acl init success +[INFO] open device 0 success +[INFO] create context success +[INFO] create stream success +[INFO] get run mode success +[INFO] load model ./edsr.om success +[INFO] create model description success +[INFO] get input dynamic gear count success +[INFO] create model output success +./output/2022113_15_10_1_472972 +[INFO] start to process file:./testpic.bin +[INFO] model execute success +Inference time: 20.962ms +[INFO] get max dynamic batch size success +[INFO] output data success +Inference average time: 20.962000 ms +[INFO] destroy model input success +[INFO] unload model success, model Id is 1 +[INFO] Execute sample success +[INFO] end to destroy stream +[INFO] end to destroy context +[INFO] end to reset device is 0 +[INFO] end to finalize acl +``` +[Inference Result](https://pan.baidu.com/s/1PrIrKap_V0C_qe_bLNC7bA) +Password:y2ic + + + diff --git a/ACL_TensorFlow/contrib/cv/edsr/ckpt2pb.py b/ACL_TensorFlow/contrib/cv/edsr/ckpt2pb.py new file mode 100644 index 0000000000000000000000000000000000000000..08397e6a8ff29e3f195ce0f9284a433055efbb95 --- /dev/null +++ b/ACL_TensorFlow/contrib/cv/edsr/ckpt2pb.py @@ -0,0 +1,118 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import tensorflow as tf +import run +import os +import argparse +from cfg import make_config +#from tensorflow.core.protobuf.rewriter_config_pb2 import RewriterConfig +from tensorflow.python.tools import freeze_graph + +os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2' # gets rid of avx/fma warning + +flags = tf.flags +FLAGS = flags.FLAGS + +## Required parameters +flags.DEFINE_string("result", "result", "The result directory where the model checkpoints will be written.") +flags.DEFINE_string("dataset", "dataset", "dataset path") +flags.DEFINE_string("obs_dir", "obs://edsr/log", "obs result path, not need on gpu and apulis platform") + +## Other parametersresult +flags.DEFINE_float("lr", 0.0001, "The initial learning rate for Adam.") +flags.DEFINE_integer("num_blocks", 32, "number of resBlocks") +flags.DEFINE_integer("num_filters", 256, "number of filters") +flags.DEFINE_boolean("from_scrach", True, "train from scrach") +flags.DEFINE_integer("scale", 2, "Upscale an image with desired scale") +flags.DEFINE_integer("batch_size", 16, "batch size for one NPU") +flags.DEFINE_integer("train_steps", 100, "total epochs for training") +flags.DEFINE_integer("save_step", 5, "epochs for saving checkpoint") +flags.DEFINE_integer("decay_step", 500, "update the learning_rate value every decay_steps times") +flags.DEFINE_float("decay_rate", 0.9, "momentum used in optimizer") +flags.DEFINE_string("resume_path", None, "checkpoint path") +flags.DEFINE_string("chip", "npu", "Run on which chip, (npu or gpu or cpu)") +flags.DEFINE_string("platform", "apulis", + "Run on apulis/modelarts platform. Modelarts Platform has some extra data copy operations") + +## The following params only useful on NPU chip mode +flags.DEFINE_boolean("npu_dump_data", False, "dump data for precision or not") +flags.DEFINE_boolean("npu_dump_graph", False, "dump graph or not") +flags.DEFINE_boolean("npu_profiling", False, "profiling for performance or not") +flags.DEFINE_boolean("npu_auto_tune", False, "auto tune or not. And you must set tune_bank_path param.") + +checkpoint = "./model/edsr_ckpt" + +if __name__ == "__main__": + + # INIT + scale = FLAGS.scale + meanbgr = [103.1545782, 111.561547, 114.35629928] + + # Set checkpoint paths for different scales and models + ckpt_path = "" + if scale == 2: + ckpt_path = os.path.join(FLAGS.result, "x2") + elif scale == 3: + ckpt_path = os.path.join(FLAGS.result, "x3") + elif scale == 4: + ckpt_path = os.path.join(FLAGS.result, "x3") + else: + print("No checkpoint directory. Choose scale 2, 3 or 4. Or add checkpoint directory for this scale.") + exit() + + config = make_config(FLAGS) + # Create run instance + run = run.run(config, ckpt_path, FLAGS.scale, FLAGS.batch_size, FLAGS.train_steps, FLAGS.num_blocks, + FLAGS.num_filters, FLAGS.lr, FLAGS.from_scrach, meanbgr) + + + tf.reset_default_graph() + LR = tf.placeholder(tf.float32, shape=[None, 48, 48, 3], name="LR") + out = run.test_model(LR) + #out_nchw = tf.identity(out, name='out_nchw') + with tf.Session(config=config) as sess: + + graph_def = tf.get_default_graph().as_graph_def(add_shapes=True) + node_list = [n.name for n in graph_def.node] + for node in node_list: + print("node_name", node) + tf.train.write_graph(sess.graph_def, './pb_model', 'model.pb') + freeze_graph.freeze_graph( + input_graph='./pb_model/model.pb', + input_saver='', + input_binary=False, + input_checkpoint=checkpoint, + output_node_names='NCHW_output', + restore_op_name='save/restore_all', + filename_tensor_name='save/Const:0', + output_graph='./pb_model/edsr.pb', + clear_devices=False, + initializer_nodes='') + print("done") \ No newline at end of file diff --git a/ACL_TensorFlow/contrib/cv/edsr/edsr.py b/ACL_TensorFlow/contrib/cv/edsr/edsr.py new file mode 100644 index 0000000000000000000000000000000000000000..510b6a849684812336f84142fc8827134c46fc20 --- /dev/null +++ b/ACL_TensorFlow/contrib/cv/edsr/edsr.py @@ -0,0 +1,157 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from __future__ import print_function + +import cv2 +import tensorflow as tf +import numpy as np +import os + +class Edsr: + + def __init__(self, B, F, scale): + self.B = B + self.F = F + self.scale = scale + self.global_step = tf.placeholder(tf.int32, shape=[], name="global_step") + self.scaling_factor = 0.1 + self.bias_initializer = tf.constant_initializer(value=0.0) + self.PS = 3 * (scale*scale) #channels x scale^2 + self.xavier = tf.contrib.layers.xavier_initializer() + + # -- Filters & Biases -- + self.resFilters = list() + self.resBiases = list() + + for i in range(0, B*2): + self.resFilters.append( tf.get_variable("resFilter%d" % (i), shape=[3,3,F,F], initializer=self.xavier)) + self.resBiases.append(tf.get_variable(name="resBias%d" % (i), shape=[F], initializer=self.bias_initializer)) + + self.filter_one = tf.get_variable("resFilter_one", shape=[3,3,3,F], initializer=self.xavier) + self.filter_two = tf.get_variable("resFilter_two", shape=[3,3,F,F], initializer=self.xavier) + self.filter_three = tf.get_variable("resFilter_three", shape=[3,3,F,self.PS], initializer=self.xavier) + + self.bias_one = tf.get_variable(shape=[F], initializer=self.bias_initializer, name="BiasOne") + self.bias_two = tf.get_variable(shape=[F], initializer=self.bias_initializer, name="BiasTwo") + self.bias_three = tf.get_variable(shape=[self.PS], initializer=self.bias_initializer, name="BiasThree") + + + def model(self, x, y, lr): + """ + Implementation of EDSR: https://arxiv.org/abs/1707.02921. + """ + + # -- Model architecture -- + + # first conv + x = tf.nn.conv2d(x, filter=self.filter_one, strides=[1, 1, 1, 1], padding='SAME') + x = x + self.bias_one + out1 = tf.identity(x) + + # all residual blocks + for i in range(self.B): + x = self.resBlock(x, (i*2)) + + # last conv + x = tf.nn.conv2d(x, filter=self.filter_two, strides=[1, 1, 1, 1], padding='SAME') + x = x + self.bias_two + x = x + out1 + + # upsample via sub-pixel, equivalent to depth to space + x = tf.nn.conv2d(x, filter=self.filter_three, strides=[1, 1, 1, 1], padding='SAME') + x = x + self.bias_three + out = tf.nn.depth_to_space(x, self.scale, data_format='NHWC', name="NHWC_output") + + # -- -- + + # some outputs + out_nchw = tf.transpose(out, [0, 3, 1, 2], name="NCHW_output") + psnr = tf.image.psnr(out, y, max_val=255.0) + loss = tf.losses.absolute_difference(out, y) #L1 + ssim = tf.image.ssim(out, y, max_val=255.0) + + # (decaying) learning rate + lr = tf.train.exponential_decay(lr, + self.global_step, + decay_steps=15000, + decay_rate=0.95, + staircase=True) + # gradient clipping + optimizer = tf.train.AdamOptimizer(learning_rate=lr, beta1=0.9, beta2=0.999, epsilon=1e-08, use_locking=False, name='Adam') + gradients, variables = zip(*optimizer.compute_gradients(loss)) + gradients, _ = tf.clip_by_global_norm(gradients, 5.0) + train_op = optimizer.apply_gradients(zip(gradients, variables)) + + return out, loss, train_op, psnr, ssim, lr + + def model_pb(self, x): + """ + Implementation of EDSR: https://arxiv.org/abs/1707.02921. + """ + + # -- Model architecture -- + + # first conv + x = tf.nn.conv2d(x, filter=self.filter_one, strides=[1, 1, 1, 1], padding='SAME') + x = x + self.bias_one + out1 = tf.identity(x) + + # all residual blocks + for i in range(self.B): + x = self.resBlock(x, (i*2)) + + # last conv + x = tf.nn.conv2d(x, filter=self.filter_two, strides=[1, 1, 1, 1], padding='SAME') + x = x + self.bias_two + x = x + out1 + + # upsample via sub-pixel, equivalent to depth to space + x = tf.nn.conv2d(x, filter=self.filter_three, strides=[1, 1, 1, 1], padding='SAME') + x = x + self.bias_three + out = tf.nn.depth_to_space(x, self.scale, data_format='NHWC', name="NHWC_output") + + # -- -- + + # some outputs + out_nchw = tf.transpose(out, [0, 3, 1, 2], name="NCHW_output") + + return out_nchw + + + def resBlock(self, inpt, f_nr): + x = tf.nn.conv2d(inpt, filter=self.resFilters[f_nr], strides=[1, 1, 1, 1], padding='SAME') + x = x + self.resBiases[f_nr] + x = tf.nn.relu(x) + + x = tf.nn.conv2d(x, filter=self.resFilters[f_nr+1], strides=[1, 1, 1, 1], padding='SAME') + x = x + self.resBiases[f_nr+1] + x = x * self.scaling_factor + + return inpt + x \ No newline at end of file diff --git a/ACL_TensorFlow/contrib/cv/edsr/run.py b/ACL_TensorFlow/contrib/cv/edsr/run.py new file mode 100644 index 0000000000000000000000000000000000000000..2c7434f31551bd0967b71cb8cc1b98c101ff6828 --- /dev/null +++ b/ACL_TensorFlow/contrib/cv/edsr/run.py @@ -0,0 +1,411 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import tensorflow as tf +import os +import cv2 +import numpy as np +import math +import data_utils +import edsr +from tqdm import tqdm +from tensorflow.python.tools import freeze_graph +from tensorflow.python.tools import optimize_for_inference_lib +from tensorflow.tools.graph_transforms import TransformGraph + +class run: + def __init__(self, config, ckpt_path, scale, batch, epochs, B, F, lr, load_flag, meanBGR): + self.config = config + self.ckpt_path = ckpt_path + self.scale = scale + self.batch = batch + self.epochs = epochs + self.B = B + self.F = F + self.lr = lr + self.load_flag = load_flag + self.mean = meanBGR + + + def test_model(self, x): + + # Edsr model + print("\nRunning EDSR.") + edsrObj = edsr.Edsr(self.B, self.F, self.scale) + out = edsrObj.model_pb(x) + + + def train(self, imagefolder, validfolder): + + # Create training dataset + train_image_paths = data_utils.getpaths(imagefolder) + train_dataset = tf.data.Dataset.from_generator(generator=data_utils.make_dataset, + output_types=(tf.float32, tf.float32), + output_shapes=(tf.TensorShape([None, None, 3]), tf.TensorShape([None, None, 3])), + args=[train_image_paths, self.scale, self.mean]) + train_dataset = train_dataset.padded_batch(self.batch, padded_shapes=([None, None, 3],[None, None, 3])) + + # Create validation dataset + val_image_paths = data_utils.getpaths(validfolder) + val_dataset = tf.data.Dataset.from_generator(generator=data_utils.make_val_dataset, + output_types=(tf.float32, tf.float32), + output_shapes=(tf.TensorShape([None, None, 3]), tf.TensorShape([None, None, 3])), + args=[val_image_paths, self.scale, self.mean]) + val_dataset = val_dataset.padded_batch(1, padded_shapes=([None, None, 3],[None, None, 3])) + + # Make the iterator and its initializers + train_val_iterator = tf.data.Iterator.from_structure(train_dataset.output_types, train_dataset.output_shapes) + train_initializer = train_val_iterator.make_initializer(train_dataset) + val_initializer = train_val_iterator.make_initializer(val_dataset) + + handle = tf.placeholder(tf.string, shape=[]) + iterator = tf.data.Iterator.from_string_handle(handle, train_dataset.output_types, train_dataset.output_shapes) + LR, HR = iterator.get_next() + + # Edsr model + print("\nRunning EDSR.") + edsrObj = edsr.Edsr(self.B, self.F, self.scale) + out, loss, train_op, psnr, ssim, lr = edsrObj.model(x=LR, y=HR, lr=self.lr) + + # -- Training session + with tf.Session(config=self.config) as sess: + + train_writer = tf.summary.FileWriter('./logs/train', sess.graph) + sess.run(tf.global_variables_initializer()) + + saver = tf.train.Saver() + + # Create check points directory if not existed, and load previous model if specified. + # if not os.path.exists(self.ckpt_path): + # os.makedirs(self.ckpt_path) + # else: + # if os.path.isfile(self.ckpt_path + "edsr_ckpt" + ".meta"): + # if self.load_flag: + # saver.restore(sess, tf.train.latest_checkpoint(self.ckpt_path)) + # print("\nLoaded checkpoint.") + # if not self.load_flag: + # print("No checkpoint loaded. Training from scratch.") + # else: + # if os.path.isfile("./CKPT_dir/x2/" + "edsr_ckpt" + ".meta"): + # saver.restore(sess, tf.train.latest_checkpoint("./CKPT_dir/x2/")) + # print("Previous checkpoint does not exists. Will load model from x2") + # else: + # print("No checkpoint loaded. Training from scratch.") + + global_step = 0 + tf.convert_to_tensor(global_step) + + train_val_handle = sess.run(train_val_iterator.string_handle()) + + print("Training...") + for e in range(1, self.epochs+1): + + sess.run(train_initializer) + step, train_loss = 0, 0 + + try: + for _ in tqdm(range(len(train_image_paths))): + o, l, t, l_rate = sess.run([out, loss, train_op, lr], feed_dict={handle:train_val_handle, + edsrObj.global_step: global_step}) + train_loss += l + step += 1 + global_step += 1 + if step % 1000 == 0: + save_path = saver.save(sess, self.ckpt_path + "edsr_ckpt") + print("Step nr: [{}/{}] - Loss: {:.5f} - Lr: {:.7f}".format(step, "?", float(train_loss/step), l_rate)) + save_path = saver.save(sess, self.ckpt_path + "edsr_ckpt") + print("Step nr: [{}/{}] - Loss: {:.5f} - Lr: {:.7f}".format(step, "?", float(train_loss / step), + l_rate)) + except tf.errors.OutOfRangeError: + pass + if e % 5 == 0: + # Perform end-of-epoch calculations here. + sess.run(val_initializer) + tot_val_psnr, tot_val_ssim, val_im_cntr = 0, 0, 0 + try: + for _ in tqdm(range(len(val_image_paths))): + val_psnr, val_ssim = sess.run([psnr, ssim], feed_dict={handle:train_val_handle}) + + tot_val_psnr += val_psnr[0] + tot_val_ssim += val_ssim[0] + val_im_cntr += 1 + except tf.errors.OutOfRangeError: + pass + + print("Epoch nr: [{}/{}] - Loss: {:.5f} - val PSNR: {:.3f} - val SSIM: {:.3f}\n".format(e, + self.epochs, + float(train_loss/step), + (tot_val_psnr / val_im_cntr), + (tot_val_ssim / val_im_cntr))) + saver.save(sess, os.path.join(self.ckpt_path, "model")) + + print("Training finished.") + train_writer.close() + + def upscale(self, path): + """ + Upscales an image via model. This loads a checkpoint, not a .pb file. + """ + fullimg = cv2.imread(path, 3) + + floatimg = fullimg.astype(np.float32) - self.mean + + LR_input_ = floatimg.reshape(1, floatimg.shape[0], floatimg.shape[1], 3) + + with tf.Session(config=self.config) as sess: + print("\nUpscale image by a factor of {}:\n".format(self.scale)) + # load the model + ckpt_name = self.ckpt_path + "edsr_ckpt" + ".meta" + saver = tf.train.import_meta_graph(ckpt_name) + saver.restore(sess, tf.train.latest_checkpoint(self.ckpt_path)) + graph_def = sess.graph + LR_tensor = graph_def.get_tensor_by_name("IteratorGetNext:0") + HR_tensor = graph_def.get_tensor_by_name("NHWC_output:0") + + output = sess.run(HR_tensor, feed_dict={LR_tensor: LR_input_}) + + Y = output[0] + HR_image = (Y + self.mean).clip(min=0, max=255) + HR_image = (HR_image).astype(np.uint8) + + bicubic_image = cv2.resize(fullimg, None, fx=self.scale, fy=self.scale, interpolation=cv2.INTER_CUBIC) + + cv2.imshow('Original image', fullimg) + cv2.imshow('EDSR upscaled image', HR_image) + cv2.imshow('Bicubic upscaled image', bicubic_image) + cv2.waitKey(0) + + sess.close() + + def test(self, path): + """ + Test single image and calculate psnr. This loads a checkpoint, not a .pb file. + """ + fullimg = cv2.imread(path, 3) + width = fullimg.shape[0] + height = fullimg.shape[1] + + cropped = fullimg[0:(width - (width % self.scale)), 0:(height - (height % self.scale)), :] + img = cv2.resize(cropped, None, fx=1. / self.scale, fy=1. / self.scale, interpolation=cv2.INTER_CUBIC) + floatimg = img.astype(np.float32) - self.mean + + LR_input_ = floatimg.reshape(1, floatimg.shape[0], floatimg.shape[1], 3) + + with tf.Session(config=self.config) as sess: + print("\nTest model with psnr:\n") + # load the model + ckpt_name = self.ckpt_path + "edsr_ckpt" + ".meta" + saver = tf.train.import_meta_graph(ckpt_name) + saver.restore(sess, tf.train.latest_checkpoint(self.ckpt_path)) + graph_def = sess.graph + LR_tensor = graph_def.get_tensor_by_name("IteratorGetNext:0") + HR_tensor = graph_def.get_tensor_by_name("NHWC_output:0") + + output = sess.run(HR_tensor, feed_dict={LR_tensor: LR_input_}) + + Y = output[0] + HR_image = (Y + self.mean).clip(min=0, max=255) + HR_image = (HR_image).astype(np.uint8) + + bicubic_image = cv2.resize(img, None, fx=self.scale, fy=self.scale, interpolation=cv2.INTER_CUBIC) + + print(np.amax(Y), np.amax(LR_input_)) + print("PSNR of EDSR upscaled image: {}".format(self.psnr(cropped, HR_image))) + print("PSNR of bicubic upscaled image: {}".format(self.psnr(cropped, bicubic_image))) + + cv2.imshow('Original image', fullimg) + cv2.imshow('EDSR upscaled image', HR_image) + cv2.imshow('Bicubic upscaled image', bicubic_image) + + cv2.imwrite("./images/EdsrOutput.png", HR_image) + cv2.imwrite("./images/BicubicOutput.png", bicubic_image) + cv2.imwrite("./images/original.png", fullimg) + cv2.imwrite("./images/input.png", img) + + cv2.waitKey(0) + cv2.destroyAllWindows() + + sess.close() + + def load_pb(self, path_to_pb): + with tf.gfile.GFile(path_to_pb, "rb") as f: + graph_def = tf.GraphDef() + graph_def.ParseFromString(f.read()) + with tf.Graph().as_default() as graph: + tf.import_graph_def(graph_def, name='') + return graph + + def testFromPb(self, path): + """ + Test single image and calculate psnr. This loads a .pb file. + """ + # Read model + pbPath = "./models/EDSR_x{}.pb".format(self.scale) + + # Get graph + graph = self.load_pb(pbPath) + + fullimg = cv2.imread(path, 3) + width = fullimg.shape[0] + height = fullimg.shape[1] + + cropped = fullimg[0:(width - (width % self.scale)), 0:(height - (height % self.scale)), :] + img = cv2.resize(cropped, None, fx=1. / self.scale, fy=1. / self.scale, interpolation=cv2.INTER_CUBIC) + floatimg = img.astype(np.float32) - self.mean + + LR_input_ = floatimg.reshape(1, floatimg.shape[0], floatimg.shape[1], 3) + + LR_tensor = graph.get_tensor_by_name("IteratorGetNext:0") + HR_tensor = graph.get_tensor_by_name("NHWC_output:0") + + with tf.Session(graph=graph) as sess: + print("Loading pb...") + output = sess.run(HR_tensor, feed_dict={LR_tensor: LR_input_}) + Y = output[0] + HR_image = (Y + self.mean).clip(min=0, max=255) + HR_image = (HR_image).astype(np.uint8) + + bicubic_image = cv2.resize(img, None, fx=self.scale, fy=self.scale, interpolation=cv2.INTER_CUBIC) + + print(np.amax(Y), np.amax(LR_input_)) + print("PSNR of EDSR upscaled image: {}".format(self.psnr(cropped, HR_image))) + print("PSNR of bicubic upscaled image: {}".format(self.psnr(cropped, bicubic_image))) + + cv2.imshow('Original image', fullimg) + cv2.imshow('EDSR upscaled image', HR_image) + cv2.imshow('Bicubic upscaled image', bicubic_image) + + cv2.imwrite("./images/EdsrOutput.png", HR_image) + cv2.imwrite("./images/BicubicOutput.png", bicubic_image) + cv2.imwrite("./images/original.png", fullimg) + cv2.imwrite("./images/input.png", img) + + cv2.waitKey(0) + cv2.destroyAllWindows() + print("Done.") + + sess.close() + + def upscaleFromPb(self, path): + """ + Upscale single image by desired model. This loads a .pb file. + """ + # Read model + pbPath = "./models/EDSR_x{}.pb".format(self.scale) + + # Get graph + graph = self.load_pb(pbPath) + + fullimg = cv2.imread(path, 3) + floatimg = fullimg.astype(np.float32) - self.mean + LR_input_ = floatimg.reshape(1, floatimg.shape[0], floatimg.shape[1], 3) + + LR_tensor = graph.get_tensor_by_name("IteratorGetNext:0") + HR_tensor = graph.get_tensor_by_name("NHWC_output:0") + + with tf.Session(graph=graph) as sess: + print("Loading pb...") + output = sess.run(HR_tensor, feed_dict={LR_tensor: LR_input_}) + Y = output[0] + HR_image = (Y + self.mean).clip(min=0, max=255) + HR_image = (HR_image).astype(np.uint8) + + bicubic_image = cv2.resize(fullimg, None, fx=self.scale, fy=self.scale, interpolation=cv2.INTER_CUBIC) + + cv2.imshow('Original image', fullimg) + cv2.imshow('EDSR upscaled image', HR_image) + cv2.imshow('Bicubic upscaled image', bicubic_image) + + cv2.waitKey(0) + cv2.destroyAllWindows() + + sess.close() + + def export(self, quant): + print("Exporting model...") + + export_dir = "./models/" + if not os.path.exists(export_dir): + os.makedirs(export_dir) + + export_file = "EDSRorig_x{}.pb".format(self.scale) + + graph = tf.get_default_graph() + with graph.as_default(): + with tf.Session(config=self.config) as sess: + + ### Restore checkpoint + ckpt_name = self.ckpt_path + "edsr_ckpt" + ".meta" + saver = tf.train.import_meta_graph(ckpt_name) + saver.restore(sess, tf.train.latest_checkpoint(self.ckpt_path)) + + # Return a serialized GraphDef representation of this graph + graph_def = sess.graph.as_graph_def() + + # All variables to constants + graph_def = tf.graph_util.convert_variables_to_constants(sess, graph_def, ['NCHW_output']) + + # Optimize for inference + graph_def = optimize_for_inference_lib.optimize_for_inference(graph_def, ["IteratorGetNext"], + ["NCHW_output"], # ["NHWC_output"], + tf.float32.as_datatype_enum) + + # Implement certain file shrinking transforms. 2 is recommended. + transforms = ["sort_by_execution_order"] + if quant == 1: + print("Rounding weights for export.") + transforms = ["sort_by_execution_order", "round_weights"] + export_file = "EDSR_x{}_q1.pb".format(self.scale) + if quant == 2: + print("Quantizing for export.") + transforms = ["sort_by_execution_order", "quantize_weights"] + export_file = "EDSR_x{}.pb".format(self.scale) + if quant == 3: + print("Round weights and quantizing for export.") + transforms = ["sort_by_execution_order", "round_weights", "quantize_weights"] + export_file = "EDSR_x{}_q3.pb".format(self.scale) + + graph_def = TransformGraph(graph_def, ["IteratorGetNext"], + ["NCHW_output"], + transforms) + + print("Exported file = {}".format(export_dir+export_file)) + with tf.gfile.GFile(export_dir + export_file, 'wb') as f: + f.write(graph_def.SerializeToString()) + + tf.train.write_graph(graph_def, ".", 'train.pbtxt') + + sess.close() + + def psnr(self, img1, img2): + mse = np.mean( (img1 - img2) ** 2 ) + if mse == 0: + return 100 + PIXEL_MAX = 255.0 + return (20 * math.log10(PIXEL_MAX / math.sqrt(mse))) \ No newline at end of file diff --git a/ACL_TensorFlow/contrib/cv/edsr/testpic.bin b/ACL_TensorFlow/contrib/cv/edsr/testpic.bin new file mode 100644 index 0000000000000000000000000000000000000000..b7277b3b9501c47dbd8a30451a87a86d1ed18efa Binary files /dev/null and b/ACL_TensorFlow/contrib/cv/edsr/testpic.bin differ diff --git a/ACL_TensorFlow/contrib/cv/mnasnet/.keep b/ACL_TensorFlow/contrib/cv/mnasnet/.keep new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/ACL_TensorFlow/contrib/cv/mnasnet/README.md b/ACL_TensorFlow/contrib/cv/mnasnet/README.md new file mode 100644 index 0000000000000000000000000000000000000000..d1fc92a503ab53d407e1f3da25ba7f84dba8e21f --- /dev/null +++ b/ACL_TensorFlow/contrib/cv/mnasnet/README.md @@ -0,0 +1,89 @@ +# ATC MnasNet + + +- references： + + ["Platform-Aware Neural Architecture Search for Mobile"](https://arxiv.org/abs/1807.11626) + + +- training model： + + [MnasNet_ID0728_for_TensorFlow](https://gitee.com/ascend/ModelZoo-TensorFlow/tree/master/TensorFlow/contrib/cv/MnasNet_ID0728_for_TensorFlow) + + +# 1. ckpt to pb + +download **ckpt** and use **ckpt2pb.py** process ckpt to pb. + +[ckpt](https://pan.baidu.com/s/1-E3SQAxShCYcIVdkxbg19w) +Password:e3el + +# 2. pb to om +Command: +``` +atc --model=./mnasnet.pb --framework=3 --input_shape="input1:1,224,224,3" --output=./mnasnet --soc_version=Ascend910" +``` +[Pb](https://pan.baidu.com/s/1YhB_1zjYb2dz_h8P_kIGUQ) +Password:m6mx + +[OM](https://pan.baidu.com/s/1mKV8wkUBz3KiF8hpxUh9mA) +Password:zdo1 + +# 3. compile msame +Reference to https://gitee.com/ascend/tools/tree/master/msame, compile **msame** + +Compile msame command: +```bash +. /home/HwHiAiUser/Ascend/ascend-toolkit/set_env.sh +export DDK_PATH=/home/HwHiAiUser/Ascend/ascend-toolkit/latest +export NPU_HOST_LIB=/home/HwHiAiUser/Ascend/ascend-toolkit/latest/runtime/lib64/stub +cd $HOME/AscendProjects/tools/msame/ +./build.sh g++ $HOME/AscendProjects/tools/msame/out + +``` + +# 4. inference +Inference command: +```bash +cd $HOME/AscendProjects/tools/msame/out +model_path=/home/HwHiAiUser/AscendProjects/MnasNet/mnasnet.om +input_path=/home/HwHiAiUser/AscendProjects/MnasNet/pic.bin +output_path=/home/HwHiAiUser/AscendProjects/MnasNet/output +./msame --model ${model_path} --input ${input_path} --output ${output_path} --outfmt TXT +``` + + + +Part of **Inference sys output**: +```bash +[INFO] acl init success +[INFO] open device 0 success +[INFO] create context success +[INFO] create stream success +[INFO] get run mode success +[INFO] load model ./mnasnet.om success +[INFO] create model description success +[INFO] get input dynamic gear count success +[INFO] create model output success +./output/2022113_9_31_13_612340 +[INFO] start to process file:./pic.bin +[INFO] model execute success +Inference time: 1.359ms +[INFO] get max dynamic batch size success +[INFO] output data success +Inference average time: 1.359000 ms +[INFO] destroy model input success +[INFO] unload model success, model Id is 1 +[INFO] Execute sample success +[INFO] end to destroy stream +[INFO] end to destroy context +[INFO] end to reset device is 0 +[INFO] end to finalize acl + +``` + + +Part of **Inference sys output**: +```bash +0.000629902 0.000514507 0.000611782 0.000699043 0.000445127 0.00120354 0.00102234 0.00104713 0.0011034 0.000992775 0.000550747 0.00101948 0.00100136 0.000835419 0.000398874 0.000741005 0.000406742 0.00107861 +``` diff --git a/ACL_TensorFlow/contrib/cv/mnasnet/ckpt2pb.py b/ACL_TensorFlow/contrib/cv/mnasnet/ckpt2pb.py new file mode 100644 index 0000000000000000000000000000000000000000..de67137c8fed41aa5d1a2e75f1f356929709b0af --- /dev/null +++ b/ACL_TensorFlow/contrib/cv/mnasnet/ckpt2pb.py @@ -0,0 +1,40 @@ +import tensorflow as tf +from tensorflow.python.framework import graph_util +import mnasnet_models + +ckpt_path = "./models/model.ckpt-10009" + +def main(): + + tf.reset_default_graph() + + inputs1 = tf.placeholder(tf.float32, shape=[1, 224, 224, 3], name="input1") + + with tf.Session() as sess: + params = {'dropout_rate': 0.2, 'data_format': 'channels_last', 'num_classes': 1000, 'use_keras': True} + #params = {} + logits, _ = mnasnet_models.build_mnasnet_model(inputs1, "mnasnet-a1", training=True, override_params=params) + probs = tf.nn.softmax(logits) + probs = tf.squeeze(probs, name='pred_prob') + + graph = tf.get_default_graph() + input_graph_def = graph.as_graph_def() + + sess.run(tf.global_variables_initializer()) + + saver = tf.train.Saver() + saver.restore(sess, ckpt_path) #恢复图并得到数据 + + output_graph_def = graph_util.convert_variables_to_constants( # 模型持久化，将变量值固定 + sess=sess, + input_graph_def=input_graph_def, + output_node_names=['pred_prob']) # 如果有多个输出节点，以逗号隔开 + # 保存模型 + with tf.gfile.GFile("./pb_model/mnasnet.pb", "wb") as f: + f.write(output_graph_def.SerializeToString()) # 序列化输出 + print("%d ops in the final graph." % len(output_graph_def.node)) # 得到当前图有几个操作节点 + + print("done") + +if __name__ == '__main__': + main() diff --git a/ACL_TensorFlow/contrib/cv/mnasnet/pic.bin b/ACL_TensorFlow/contrib/cv/mnasnet/pic.bin new file mode 100644 index 0000000000000000000000000000000000000000..320a9c30e9cbc06f6adc99eb749df696d9a9f411 Binary files /dev/null and b/ACL_TensorFlow/contrib/cv/mnasnet/pic.bin differ