diff --git a/TensorFlow/contrib/cv/Mixnet_ID2072_for_Tensorflow/README.md b/TensorFlow/contrib/cv/Mixnet_ID2072_for_Tensorflow/README.md new file mode 100644 index 0000000000000000000000000000000000000000..f5f22e17783cc9e2d3bad103bdfb2dfbda5ad0e8 --- /dev/null +++ b/TensorFlow/contrib/cv/Mixnet_ID2072_for_Tensorflow/README.md @@ -0,0 +1,49 @@ +# MIXNET +mixnet即混合深度卷积，在一次卷积中自然地混合多个卷积核大小，大内核提取高级语义信息，小内盒提取位置边缘信息，以此获得更好的精度和效率。参考文章为 MixConv: Mixed Depthwise Convolutional Kernels 参考项目： https://github.com/tensorflow/tpu/tree/master/models/official/mnasnet/mixnet. + +# setup +* python 3.7.5+ +* tensorflow-gpu 1.15.0+ +* numpy 1.14.1+ + + +# Train + +数据集： 请用户自行准备好数据集，包含训练集和验证集两部分，可选用的数据集包括ImageNet2012，CIFAR10、Flower等，包含train和val两部分。格式为TFRecord文件。 + +Boot File：boot_modelarts.py + +Pretrained: obs://cann-lhz/data_url/result_mix + +# 精度对比 +测试集：Imagenet2012 + +论文精度： + +| Model | Top1 Accuracy | Top5 Accuracy | +| :--------: |:----------: | :-----------: | +| MixNet-S | 73.8% | 92.8% | + + +GPU目标精度： + +| Model | Top1 Accuracy | Top5 Accuracy | +| :--------: |:----------: | :-----------: | +| MixNet-S | 74.1% | 92.6% | + + +Ascend精度： + +| Model | Top1 Accuracy | Top5 Accuracy | +| :--------: |:----------: | :-----------: | +| MixNet-S | 74.3% | 92.8% | + + +# 性能对比： + +| GPU V100 | Ascend 910 | +| :--------: | --------| +| 5.2global_step/s | 11.0global_step/s | + + + diff --git a/TensorFlow/contrib/cv/Mixnet_ID2072_for_Tensorflow/boot_modelarts.py b/TensorFlow/contrib/cv/Mixnet_ID2072_for_Tensorflow/boot_modelarts.py new file mode 100644 index 0000000000000000000000000000000000000000..a1e412644d440ff2f2020c0e4d83dda103f4b37b --- /dev/null +++ b/TensorFlow/contrib/cv/Mixnet_ID2072_for_Tensorflow/boot_modelarts.py @@ -0,0 +1,53 @@ +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +This is the boot file for ModelArts platform. +Firstly, the train datasets are copyed from obs to ModelArts. +Then, the string of train shell command is concated and using 'os.system()' to execute +""" +import os +import numpy as np +import argparse +from help_modelarts import obs_data2modelarts + +print(os.system('env')) + +if __name__ == '__main__': + # Note: the code dir is not the same as work dir on ModelArts Platform!!! + code_dir = os.path.dirname(__file__) + work_dir = os.getcwd() + print("===>>>code_dir:{}, work_dir:{}".format(code_dir, work_dir)) + + parser = argparse.ArgumentParser() + parser.add_argument("--data_url", type=str, default=None) + parser.add_argument("--train_url", type=str, default=None) + parser.add_argument("--modelarts_data_dir", type=str, + default="/cache/data_url") + config = parser.parse_args() + + print("--------config----------") + for k in list(vars(config).keys()): + print("key:{}: value:{}".format(k, vars(config)[k])) + print("--------config----------") + + # copy dataset from obs to modelarts + obs_data2modelarts(config) + + # start to train on Modelarts platform + bash_header = os.path.join(code_dir, 'run_1p.sh') + arg_url = '%s %s %s %s' % (code_dir, config.modelarts_data_dir+'/ILSVRC2012', + config.modelarts_data_dir+'/result_mix', config.train_url) + bash_command = 'bash %s %s' % (bash_header, arg_url) + print("bash command:", bash_command) + os.system(bash_command) diff --git a/TensorFlow/contrib/cv/Mixnet_ID2072_for_Tensorflow/configs/__init__.py b/TensorFlow/contrib/cv/Mixnet_ID2072_for_Tensorflow/configs/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..20c4af897dbc69deafc901debc8e3cb0193edfe9 --- /dev/null +++ b/TensorFlow/contrib/cv/Mixnet_ID2072_for_Tensorflow/configs/__init__.py @@ -0,0 +1,28 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""The initial file of config.""" \ No newline at end of file diff --git a/TensorFlow/contrib/cv/Mixnet_ID2072_for_Tensorflow/configs/cloud/gpu.yaml b/TensorFlow/contrib/cv/Mixnet_ID2072_for_Tensorflow/configs/cloud/gpu.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3dd114758dc5464c376fd434909ea6275a3ddba3 --- /dev/null +++ b/TensorFlow/contrib/cv/Mixnet_ID2072_for_Tensorflow/configs/cloud/gpu.yaml @@ -0,0 +1,7 @@ +use_tpu: False +train_steps: 3503192 # 1281167 * 350 / train_batch_size +train_batch_size: 128 # 1024 / 8 +eval_batch_size: 128 # 1024 / 8 +model_name: 'mnasnet-a1' +dropout_rate: null +depth_multiplier: null diff --git a/TensorFlow/contrib/cv/Mixnet_ID2072_for_Tensorflow/configs/cloud/v2-32.yaml b/TensorFlow/contrib/cv/Mixnet_ID2072_for_Tensorflow/configs/cloud/v2-32.yaml new file mode 100644 index 0000000000000000000000000000000000000000..66a606796921f71e1fa8dc5b0632fb52a1b35141 --- /dev/null +++ b/TensorFlow/contrib/cv/Mixnet_ID2072_for_Tensorflow/configs/cloud/v2-32.yaml @@ -0,0 +1,10 @@ +train_steps: 109474 +train_batch_size: 4096 +eval_batch_size: 256 +iterations_per_loop: 100 +skip_host_call: false +model_name: 'mnasnet-a1' +dropout_rate: null +depth_multiplier: null +use_keras: true +precision: 'float32' diff --git a/TensorFlow/contrib/cv/Mixnet_ID2072_for_Tensorflow/configs/cloud/v2-8.yaml b/TensorFlow/contrib/cv/Mixnet_ID2072_for_Tensorflow/configs/cloud/v2-8.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f6a8eab2eb112255a511ddb515a91b9f1d8c678f --- /dev/null +++ b/TensorFlow/contrib/cv/Mixnet_ID2072_for_Tensorflow/configs/cloud/v2-8.yaml @@ -0,0 +1,8 @@ +train_steps: 437899 +train_batch_size: 1024 +eval_batch_size: 1024 +iterations_per_loop: 1251 +skip_host_call: True +model_name: 'mnasnet-a1' +dropout_rate: null +depth_multiplier: null diff --git a/TensorFlow/contrib/cv/Mixnet_ID2072_for_Tensorflow/configs/cloud/v3-32.yaml b/TensorFlow/contrib/cv/Mixnet_ID2072_for_Tensorflow/configs/cloud/v3-32.yaml new file mode 100644 index 0000000000000000000000000000000000000000..66a606796921f71e1fa8dc5b0632fb52a1b35141 --- /dev/null +++ b/TensorFlow/contrib/cv/Mixnet_ID2072_for_Tensorflow/configs/cloud/v3-32.yaml @@ -0,0 +1,10 @@ +train_steps: 109474 +train_batch_size: 4096 +eval_batch_size: 256 +iterations_per_loop: 100 +skip_host_call: false +model_name: 'mnasnet-a1' +dropout_rate: null +depth_multiplier: null +use_keras: true +precision: 'float32' diff --git a/TensorFlow/contrib/cv/Mixnet_ID2072_for_Tensorflow/configs/cloud/v3-8.yaml b/TensorFlow/contrib/cv/Mixnet_ID2072_for_Tensorflow/configs/cloud/v3-8.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f6a8eab2eb112255a511ddb515a91b9f1d8c678f --- /dev/null +++ b/TensorFlow/contrib/cv/Mixnet_ID2072_for_Tensorflow/configs/cloud/v3-8.yaml @@ -0,0 +1,8 @@ +train_steps: 437899 +train_batch_size: 1024 +eval_batch_size: 1024 +iterations_per_loop: 1251 +skip_host_call: True +model_name: 'mnasnet-a1' +dropout_rate: null +depth_multiplier: null diff --git a/TensorFlow/contrib/cv/Mixnet_ID2072_for_Tensorflow/configs/mnasnet_config.py b/TensorFlow/contrib/cv/Mixnet_ID2072_for_Tensorflow/configs/mnasnet_config.py new file mode 100644 index 0000000000000000000000000000000000000000..2773e5e6a5794a2345a3a24f2368a4039b2619cb --- /dev/null +++ b/TensorFlow/contrib/cv/Mixnet_ID2072_for_Tensorflow/configs/mnasnet_config.py @@ -0,0 +1,63 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Config to train MNasNet.""" + +MNASNET_CFG = { + 'use_tpu': False, + 'train_batch_size': 64, + 'eval_batch_size': 64, + 'num_train_images': 1281167, + 'num_eval_images': 50000, + 'iterations_per_loop': 1251, + 'num_parallel_calls': 64, + 'num_label_classes': 1000, + 'transpose_input': True, + 'base_learning_rate': 0.016, + 'momentum': 0.9, + 'moving_average_decay': 0.9999, + 'weight_decay': 0.00001, + 'label_smoothing': 0.1, + 'dropout_rate': 0.2, + 'use_cache': False, + 'use_async_checkpointing': False, + 'precision': 'float32', + 'use_keras': True, + 'skip_host_call': False, + 'input_image_size': 224, + 'train_steps': 150000, + 'model_name': 'mixnet-s', + 'data_format': 'channels_last', + 'batch_norm_momentum': None, + 'batch_norm_epsilon': None, + 'depth_multiplier': None, + 'depth_divisor': None, + 'min_depth': 0, +} + +MNASNET_RESTRICTIONS = [] diff --git a/TensorFlow/contrib/cv/Mixnet_ID2072_for_Tensorflow/eval_ckpt_main.py b/TensorFlow/contrib/cv/Mixnet_ID2072_for_Tensorflow/eval_ckpt_main.py new file mode 100644 index 0000000000000000000000000000000000000000..98eab5f9edabb9c5a19968616c943f1bb59edcd0 --- /dev/null +++ b/TensorFlow/contrib/cv/Mixnet_ID2072_for_Tensorflow/eval_ckpt_main.py @@ -0,0 +1,130 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Eval checkpoint driver. + +This is an example evaluation script for users to understand the model +checkpoints on CPU. To serve the model, please consider to export a `SavedModel` +from checkpoints and use tf-serving to serve. +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from absl import app +from absl import flags +import tensorflow.compat.v1 as tf + +import imagenet_input +import mnas_utils +import mnasnet_models +import preprocessing +from mixnet import mixnet_builder + +flags.DEFINE_string('model_name', 'mixnet-s', 'Model name to eval.') +flags.DEFINE_string('runmode', 'examples', + 'Running mode: examples or imagenet') +flags.DEFINE_string( + 'imagenet_eval_glob', None, 'Imagenet eval image glob, ' + 'such as /imagenet/ILSVRC2012*.JPEG') +flags.DEFINE_string( + 'imagenet_eval_label', None, 'Imagenet eval label file path, ' + 'such as /imagenet/ILSVRC2012_validation_ground_truth.txt') +flags.DEFINE_string('ckpt_dir', '/tmp/ckpt/', 'Checkpoint folders') +flags.DEFINE_boolean('enable_ema', True, 'Enable exponential moving average.') +flags.DEFINE_string('export_ckpt', None, 'Exported ckpt for eval graph.') +flags.DEFINE_string('example_img', '/tmp/panda.jpg', + 'Filepath for a single example image.') +flags.DEFINE_string('labels_map_file', '/tmp/labels_map.txt', + 'Labels map from label id to its meaning.') +flags.DEFINE_bool('include_background_label', False, + 'Whether to include background as label #0') +flags.DEFINE_integer('num_images', 5000, + 'Number of images to eval. Use -1 to eval all images.') + + +class EvalCkptDriver(mnas_utils.EvalCkptDriver): + """A driver for running eval inference.""" + + def build_model(self, features, is_training): + """Build model with input features.""" + if self.model_name.startswith('mixnet'): + model_builder = mixnet_builder + elif self.model_name.startswith('mnasnet'): + model_builder = mnasnet_models + else: + raise ValueError('Unknown model name {}'.format(self.model_name)) + + features -= tf.constant( + imagenet_input.MEAN_RGB, shape=[1, 1, 3], dtype=features.dtype) + features /= tf.constant( + imagenet_input.STDDEV_RGB, shape=[1, 1, 3], dtype=features.dtype) + logits, _ = model_builder.build_model(features, self.model_name, + is_training) + probs = tf.nn.softmax(logits) + probs = tf.squeeze(probs) + return probs + + def get_preprocess_fn(self): + """Build input dataset.""" + return preprocessing.preprocess_image + + +def get_eval_driver(model_name, include_background_label=False): + """Get a eval driver.""" + return EvalCkptDriver( + model_name=model_name, + batch_size=1, + image_size=224, + include_background_label=include_background_label) + + +# FLAGS should not be used before main. +FLAGS = flags.FLAGS + + +def main(unused_argv): + """The main function of evaluation.""" + tf.logging.set_verbosity(tf.logging.ERROR) + driver = get_eval_driver(FLAGS.model_name, FLAGS.include_background_label) + if FLAGS.runmode == 'examples': + # Run inference for an example image. + driver.eval_example_images(FLAGS.ckpt_dir, [FLAGS.example_img], + FLAGS.labels_map_file, FLAGS.enable_ema, + FLAGS.export_ckpt) + elif FLAGS.runmode == 'imagenet': + # Run inference for imagenet. + driver.eval_imagenet(FLAGS.ckpt_dir, FLAGS.imagenet_eval_glob, + FLAGS.imagenet_eval_label, FLAGS.num_images, + FLAGS.enable_ema, FLAGS.export_ckpt) + else: + print('must specify runmode: examples or imagenet') + + +if __name__ == '__main__': + app.run(main) diff --git a/TensorFlow/contrib/cv/Mixnet_ID2072_for_Tensorflow/g3doc/mnasnet_vs_mobilenetv2.png b/TensorFlow/contrib/cv/Mixnet_ID2072_for_Tensorflow/g3doc/mnasnet_vs_mobilenetv2.png new file mode 100644 index 0000000000000000000000000000000000000000..baecdf256932bfb467514e019d43617a820cc04e Binary files /dev/null and b/TensorFlow/contrib/cv/Mixnet_ID2072_for_Tensorflow/g3doc/mnasnet_vs_mobilenetv2.png differ diff --git a/TensorFlow/contrib/cv/Mixnet_ID2072_for_Tensorflow/g3doc/mnasnet_vs_mobilenetv2_2.png b/TensorFlow/contrib/cv/Mixnet_ID2072_for_Tensorflow/g3doc/mnasnet_vs_mobilenetv2_2.png new file mode 100644 index 0000000000000000000000000000000000000000..eb59c4d207ae47dbaaa580ef00666fe677d1f278 Binary files /dev/null and b/TensorFlow/contrib/cv/Mixnet_ID2072_for_Tensorflow/g3doc/mnasnet_vs_mobilenetv2_2.png differ diff --git a/TensorFlow/contrib/cv/Mixnet_ID2072_for_Tensorflow/help_modelarts.py b/TensorFlow/contrib/cv/Mixnet_ID2072_for_Tensorflow/help_modelarts.py new file mode 100644 index 0000000000000000000000000000000000000000..d2c3803440f9a7a7a9ea9c92bab7a04f30b681ce --- /dev/null +++ b/TensorFlow/contrib/cv/Mixnet_ID2072_for_Tensorflow/help_modelarts.py @@ -0,0 +1,90 @@ +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Help mox files between obs and modelarts.""" +import os +import datetime +import moxing as mox + + +def obs_data2modelarts(config): + """ + Copy train data from obs to modelarts by using moxing api. + """ + start = datetime.datetime.now() + print("===>>>Copy files from obs:{} to modelarts dir:{}".format( + config.data_url, config.modelarts_data_dir)) + mox.file.copy_parallel(src_url=config.data_url, + dst_url=config.modelarts_data_dir) + end = datetime.datetime.now() + print("===>>>Copy from obs to modelarts, time use:{}(s)".format( + (end - start).seconds)) + files = os.listdir(config.modelarts_data_dir) + print("===>>>Files:", files) + print("===>>>File_nums:", len(files)) + + +def modelarts_result2obs(FLAGS): + """ + Copy debug data from modelarts to obs. + According to the swich flags, the debug data may contains auto tune repository, + dump data for precision comparision, even the computation graph and profiling data. + """ + work_dir = os.getcwd() + + # copy result from modelarts to obs + obs_result_dir = os.path.join(FLAGS.obs_dir, 'result') + if not mox.file.exists(obs_result_dir): + mox.file.make_dirs(obs_result_dir) + mox.file.copy_parallel(src_url=FLAGS.model_dir, dst_url=obs_result_dir) + print("===>>>Copy Event or Checkpoint from modelarts dir:{} to obs:{}".format( + FLAGS.model_dir, obs_result_dir)) + + # Copy dump data. Comment this snippets if npu_dump_data is off. + if FLAGS.npu_dump_data: + obs_dump_data_dir = os.path.join(FLAGS.obs_dir, 'npu_dump_data') + if not mox.file.exists(obs_dump_data_dir): + mox.file.make_dirs(obs_dump_data_dir) + mox.file.copy_parallel(FLAGS.dump_dir, obs_dump_data_dir) + print("===>>>Dumped graph:{} on OBS dir:{}".format( + mox.file.list_directory(obs_dump_data_dir), obs_dump_data_dir)) + + if FLAGS.npu_profiling: + obs_profiling_dir = os.path.join(FLAGS.obs_dir, 'npu_profiling') + if not mox.file.exists(obs_profiling_dir): + mox.file.make_dirs(obs_profiling_dir) + mox.file.copy_parallel(FLAGS.profiling_dir, obs_profiling_dir) + print("===>>>Profiling data:{} on OBS dir:{}".format( + mox.file.list_directory(obs_profiling_dir), obs_profiling_dir)) + + # Copy compute graph. Comment this snippets if npu_dump_graph is off. + if FLAGS.npu_dump_graph: + modelarts_dump_graph_dir = os.path.join(work_dir, "npu_dump_graph") + obs_dump_graph_dir = os.path.join(FLAGS.obs_dir, 'npu_dump_graph') + if not mox.file.exists(obs_dump_graph_dir): + mox.file.make_dirs(obs_dump_graph_dir) + mox.file.copy_parallel(modelarts_dump_graph_dir, obs_dump_graph_dir) + print("===>>>Dumped data:{} on OBS dir:{}".format( + mox.file.list_directory(obs_dump_graph_dir), obs_dump_graph_dir)) + + # Copy profiling data. Comment this snippets if npu_profiling is off. + + # Copy auto tune repository. Comment this snippets if npu_auto_tune is off. + if FLAGS.npu_auto_tune: + modelarts_auto_tune_dir = os.path.join(work_dir, "npu_auto_tune") + obs_auto_tune_dir = os.path.join(FLAGS.obs_dir, 'npu_auto_tune') + if not mox.file.exists(obs_auto_tune_dir): + mox.file.make_dirs(obs_auto_tune_dir) + mox.file.copy_parallel(modelarts_auto_tune_dir, obs_auto_tune_dir) + print("===>>>Auto tune:{} on OBS dir:{}".format( + mox.file.list_directory(obs_auto_tune_dir), obs_auto_tune_dir)) diff --git a/TensorFlow/contrib/cv/Mixnet_ID2072_for_Tensorflow/hyperparameters/__init__.py b/TensorFlow/contrib/cv/Mixnet_ID2072_for_Tensorflow/hyperparameters/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..2a59a24e7b9170ab204aefdd341e1605a06a31a2 --- /dev/null +++ b/TensorFlow/contrib/cv/Mixnet_ID2072_for_Tensorflow/hyperparameters/__init__.py @@ -0,0 +1,28 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""The initial file of hyperparameters.""" diff --git a/TensorFlow/contrib/cv/Mixnet_ID2072_for_Tensorflow/hyperparameters/common_hparams_flags.py b/TensorFlow/contrib/cv/Mixnet_ID2072_for_Tensorflow/hyperparameters/common_hparams_flags.py new file mode 100644 index 0000000000000000000000000000000000000000..e2c32276d2d8956bac09a13381eaa1eccb2f97e1 --- /dev/null +++ b/TensorFlow/contrib/cv/Mixnet_ID2072_for_Tensorflow/hyperparameters/common_hparams_flags.py @@ -0,0 +1,124 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Defining common model params used across all the models.""" + +from absl import flags + + +def define_common_hparams_flags(): + """Define the common flags across models.""" + + flags.DEFINE_string( + 'data_dir', default=None, + help='The directory where the ImageNet input data is stored. Please see the README.md for the expected data format.') + + flags.DEFINE_string( + 'model_dir', default=None, + help='The directory where the model and training/evaluation summaries are stored.') + + flags.DEFINE_string( + 'dump_dir', default="/cache/dump_dir", + help='The directory where the dump data are stored.') + flags.DEFINE_string( + 'profiling_dir', default="/cache/profiling_dir", + help='The directory where the profiling data are stored.You need create it first and have the permissions of read and write.') + + flags.DEFINE_string("obs_dir", None, "obs dir") + + # The following params only useful on NPU chip mode + flags.DEFINE_boolean("npu_dump_data", True, + "dump data for precision or not") + flags.DEFINE_boolean("npu_profiling", False, + "profiling for performance or not") + flags.DEFINE_boolean("npu_dump_graph", False, "dump graph or not") + flags.DEFINE_boolean("npu_auto_tune", False, + "auto tune or not. And you must set tune_bank_path param.") + + flags.DEFINE_bool( + 'use_tpu', default=None, + help=('Use TPU to execute the model for training and evaluation. If' + ' --use_tpu=false, will use whatever devices are available to' + ' TensorFlow by default (e.g. CPU and GPU)')) + + flags.DEFINE_integer( + 'train_batch_size', default=None, help='Batch size for training.') + + flags.DEFINE_integer( + 'train_steps', default=None, + help=('The number of steps to use for training. This flag' + ' should be adjusted according to the --train_batch_size flag.')) + + flags.DEFINE_integer( + 'eval_batch_size', default=None, help='Batch size for evaluation.') + + flags.DEFINE_bool( + 'skip_host_call', default=None, + help=('Skip the host_call which is executed every training step. This is' + ' generally used for generating training summaries (train loss,' + ' learning rate, etc...). When --skip_host_call=false, there could' + ' be a performance drop if host_call function is slow and cannot' + ' keep up with the TPU-side computation.')) + + flags.DEFINE_integer( + 'iterations_per_loop', default=None, + help=('Number of steps to run on TPU before outfeeding metrics to the ' + 'CPU. If the number of iterations in the loop would exceed the ' + 'number of train steps, the loop will exit before reaching' + ' --iterations_per_loop. The larger this value is, the higher the' + ' utilization on the TPU.')) + + flags.DEFINE_string( + 'precision', default=None, + help=('Precision to use; one of: {bfloat16, float32}')) + + flags.DEFINE_string( + 'config_file', default=None, + help=('A YAML file which specifies overrides. Note that this file can be ' + 'used as an override template to override the default parameters ' + 'specified in Python. If the same parameter is specified in both ' + '`--config_file` and `--params_override`, the one in ' + '`--params_override` will be used finally.')) + + flags.DEFINE_string( + 'params_override', default=None, + help=('a YAML/JSON string or a YAML file which specifies additional ' + 'overrides over the default parameters and those specified in ' + '`--config_file`. Note that this is supposed to be used only to ' + 'override the model parameters, but not the parameters like TPU ' + 'specific flags. One canonical use case of `--config_file` and ' + '`--params_override` is users first define a template config file ' + 'using `--config_file`, then use `--params_override` to adjust the ' + 'minimal set of tuning parameters, for example setting up different' + ' `train_batch_size`. ' + 'The final override order of parameters: default_model_params --> ' + 'params from config_file --> params in params_override.' + 'See also the help message of `--config_file`.')) + + flags.DEFINE_bool( + 'eval_after_training', default=False, + help='Run one eval after the training finishes.') diff --git a/TensorFlow/contrib/cv/Mixnet_ID2072_for_Tensorflow/hyperparameters/common_tpu_flags.py b/TensorFlow/contrib/cv/Mixnet_ID2072_for_Tensorflow/hyperparameters/common_tpu_flags.py new file mode 100644 index 0000000000000000000000000000000000000000..95ec67647afa54758ed8628a184a7ef62f6c8d56 --- /dev/null +++ b/TensorFlow/contrib/cv/Mixnet_ID2072_for_Tensorflow/hyperparameters/common_tpu_flags.py @@ -0,0 +1,54 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Defining common TPU flags used across all the models.""" + +from absl import flags + + +def define_common_tpu_flags(): + """Define the flags related to TPU's.""" + flags.DEFINE_string( + 'tpu', default=None, + help='The Cloud TPU to use for training. This should be either the name ' + 'used when creating the Cloud TPU, or a grpc://ip.address.of.tpu:8470 ' + 'url.') + + flags.DEFINE_string( + 'gcp_project', default=None, + help='Project name for the Cloud TPU-enabled project. If not specified, we ' + 'will attempt to automatically detect the GCE project from metadata.') + + flags.DEFINE_string( + 'tpu_zone', default=None, + help='GCE zone where the Cloud TPU is located in. If not specified, we ' + 'will attempt to automatically detect the GCE project from metadata.') + + flags.DEFINE_string( + 'eval_master', default=None, + help='GRPC URL of the eval master. Set to an appropiate value when running ' + 'on CPU/GPU.') diff --git a/TensorFlow/contrib/cv/Mixnet_ID2072_for_Tensorflow/hyperparameters/flags_to_params.py b/TensorFlow/contrib/cv/Mixnet_ID2072_for_Tensorflow/hyperparameters/flags_to_params.py new file mode 100644 index 0000000000000000000000000000000000000000..348d90c54a32a155975154f6373c3e69d9821a57 --- /dev/null +++ b/TensorFlow/contrib/cv/Mixnet_ID2072_for_Tensorflow/hyperparameters/flags_to_params.py @@ -0,0 +1,101 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Functions to override model parameters from command-line flags.""" + +from absl import logging +from hyperparameters import params_dict + +ESSENTIAL_FLAGS = ['tpu', 'data_dir', 'model_dir'] + + +def override_params_from_input_flags(params, input_flags): + """Update params dictionary with input flags. + + Args: + params: ParamsDict object containing dictionary of model parameters. + input_flags: All the flags with non-null value of overridden model + parameters. + + Returns: + ParamsDict object containing dictionary of model parameters. + """ + if not isinstance(params, params_dict.ParamsDict): + raise ValueError( + 'The base parameter set must be a ParamsDict, was: {}'.format( + type(params))) + + essential_flag_dict = {} + for key in ESSENTIAL_FLAGS: + flag_value = input_flags.get_flag_value(key, None) + + if flag_value is None: + logging.warning('Flag %s is None.', key) + else: + essential_flag_dict[key] = flag_value + + params_dict.override_params_dict(params, + essential_flag_dict, + is_strict=False) + + normal_flag_dict = get_dictionary_from_flags(params.as_dict(), input_flags) + + params_dict.override_params_dict(params, + normal_flag_dict, + is_strict=False) + + return params + + +def get_dictionary_from_flags(params, input_flags): + """Generate dictionary from non-null flags. + + Args: + params: Python dictionary of model parameters. + input_flags: All the flags with non-null value of overridden model + parameters. + + Returns: + Python dict of overriding model parameters. + """ + if not isinstance(params, dict): + raise ValueError('The base parameter set must be a dict. ' + 'Was: {}'.format(type(params))) + flag_dict = {} + for k, v in params.items(): + if isinstance(v, dict): + d = get_dictionary_from_flags(v, input_flags) + flag_dict[k] = d + else: + try: + flag_value = input_flags.get_flag_value(k, None) + if flag_value is not None: + flag_dict[k] = flag_value + except AttributeError: + flag_dict[k] = v + + return flag_dict diff --git a/TensorFlow/contrib/cv/Mixnet_ID2072_for_Tensorflow/hyperparameters/params_dict.py b/TensorFlow/contrib/cv/Mixnet_ID2072_for_Tensorflow/hyperparameters/params_dict.py new file mode 100644 index 0000000000000000000000000000000000000000..32e1658283a2dde7afa4551c07d581921f19c58b --- /dev/null +++ b/TensorFlow/contrib/cv/Mixnet_ID2072_for_Tensorflow/hyperparameters/params_dict.py @@ -0,0 +1,441 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""A parameter dictionary class which supports the nest structure.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import collections +import copy +import re +import six +import tensorflow.compat.v1 as tf +import yaml + +# regex pattern that matches on key-value pairs in a comma-separated +# key-value pair string. It splits each k-v pair on the = sign, and +# matches on values that are within single quotes, double quotes, single +# values (e.g. floats, ints, etc.), and a lists within brackets. +_PARAM_RE = re.compile(r""" + (?P[a-zA-Z][\w\.]*) # variable name: "var" or "x" + \s*=\s* + ((?P\'(.*?)\' # single quote + | + \"(.*?)\" # double quote + | + [^,\[]* # single value + | + \[[^\]]*\])) # list of values + ($|,\s*)""", re.VERBOSE) + + +class ParamsDict(object): + """A hyperparameter container class.""" + + RESERVED_ATTR = ['_locked', '_restrictions'] + + def __init__(self, default_params=None, restrictions=None): + """Instantiate a ParamsDict. + + Instantiate a ParamsDict given a set of default parameters and a list of + restrictions. Upon initialization, it validates itself by checking all the + defined restrictions, and raise error if it finds inconsistency. + + Args: + default_params: a Python dict or another ParamsDict object including the + default parameters to initialize. + restrictions: a list of strings, which define a list of restrictions to + ensure the consistency of different parameters internally. Each + restriction string is defined as a binary relation with a set of + operators, including {'==', '!=', '<', '<=', '>', '>='}. + """ + self._locked = False + self._restrictions = [] + if restrictions: + self._restrictions = restrictions + if default_params is None: + default_params = {} + self.override(default_params, is_strict=False) + self.validate() + + def _set(self, k, v): + if isinstance(v, dict): + self.__dict__[k] = ParamsDict(v) + else: + self.__dict__[k] = copy.deepcopy(v) + + def __setattr__(self, k, v): + """Sets the value of the existing key. + + Note that this does not allow directly defining a new key. Use the + `override` method with `is_strict=False` instead. + + Args: + k: the key string. + v: the value to be used to set the key `k`. + + Raises: + KeyError: if k is not defined in the ParamsDict. + """ + if k not in ParamsDict.RESERVED_ATTR: + if k not in self.__dict__.keys(): + raise KeyError('The key `%{}` does not exist. ' + 'To extend the existing keys, use ' + '`override` with `is_strict` = True.'.format(k)) + if self._locked: + raise ValueError('The ParamsDict has been locked. ' + 'No change is allowed.') + self._set(k, v) + + def __getattr__(self, k): + """Gets the value of the existing key. + + Args: + k: the key string. + + Returns: + the value of the key. + + Raises: + KeyError: if k is not defined in the ParamsDict. + """ + if k == '__getstate__': + return self.as_dict + elif k == '__setstate__': + return self.__init__ + elif k not in self.__dict__.keys(): + raise KeyError('The key `{}` does not exist. '.format(k)) + return self.__dict__[k] + + def __delattr__(self, k): + """Deletes the key and removes its values. + + Args: + k: the key string. + + Raises: + KeyError: if k is not defined in the ParamsDict. + """ + if k in ParamsDict.RESERVED_ATTR: + raise KeyError('The key `{}` is reserved. ' + 'No change is allowes. '.format(k)) + if k not in self.__dict__.keys(): + raise KeyError('The key `{}` does not exist. '.format(k)) + if self._locked: + raise ValueError( + 'The ParamsDict has been locked. No change is allowed.') + del self.__dict__[k] + + def override(self, override_params, is_strict=True): + """Override the ParamsDict with a set of given params. + + Args: + override_params: a dict or a ParamsDict specifying the parameters to + be overridden. + is_strict: a boolean specifying whether override is strict or not. If + True, keys in `override_params` must be present in the ParamsDict. + If False, keys in `override_params` can be different from what is + currently defined in the ParamsDict. In this case, the ParamsDict will + be extended to include the new keys. + """ + if self._locked: + raise ValueError( + 'The ParamsDict has been locked. No change is allowed.') + if isinstance(override_params, ParamsDict): + override_params = override_params.as_dict() + self._override(override_params, + is_strict) # pylint: disable=protected-access + + def _override(self, override_dict, is_strict=True): + """The implementation of `override`.""" + for k, v in six.iteritems(override_dict): + if k in ParamsDict.RESERVED_ATTR: + raise KeyError('The key `%{}` is internally reserved. ' + 'Can not be overridden.') + if k not in self.__dict__.keys(): + if is_strict: + raise KeyError('The key `{}` does not exist. ' + 'To extend the existing keys, use ' + '`override` with `is_strict` = False.'.format(k)) + else: + self._set(k, v) + else: + if isinstance(v, dict): + self.__dict__[k]._override( + v, is_strict) # pylint: disable=protected-access + elif isinstance(v, ParamsDict): + self.__dict__[k]._override( + v.as_dict(), is_strict) # pylint: disable=protected-access + else: + self.__dict__[k] = copy.deepcopy(v) + + def lock(self): + """Makes the ParamsDict immutable.""" + self._locked = True + + def as_dict(self): + """Returns a dict representation of ParamsDict. + + For the nested ParamsDict, a nested dict will be returned. + """ + params_dict = {} + for k, v in six.iteritems(self.__dict__): + if k not in ParamsDict.RESERVED_ATTR: + if isinstance(v, ParamsDict): + params_dict[k] = v.as_dict() + else: + params_dict[k] = copy.deepcopy(v) + return params_dict + + def validate(self): + """Validate the parameters consistency based on the restrictions. + + This method validates the internal consistency using the pre-defined list of + restrictions. A restriction is defined as a string which specfiies a binary + operation. The supported binary operations are {'==', '!=', '<', '<=', '>', + '>='}. Note that the meaning of these operators are consistent with the + underlying Python immplementation. Users should make sure the define + restrictions on their type make sense. + + For example, for a ParamsDict like the following + ``` + a: + a1: 1 + a2: 2 + b: + bb: + bb1: 10 + bb2: 20 + ccc: + a1: 1 + a3: 3 + ``` + one can define two restrictions like this + ['a.a1 == b.ccc.a1', 'a.a2 <= b.bb.bb2'] + + What it enforces are: + - a.a1 = 1 == b.ccc.a1 = 2 + - a.a2 = 2 <= b.bb.bb2 = 20 + + Raises: + KeyError: if any of the following happens + (1) any of parameters in any of restrictions is not defined in + ParamsDict, + (2) any inconsistency violating the restriction is found. + ValueError: if the restriction defined in the string is not supported. + """ + def _get_kv(dotted_string, params_dict): + tokenized_params = dotted_string.split('.') + v = params_dict + for t in tokenized_params: + v = v[t] + return tokenized_params[-1], v + + def _get_kvs(tokens, params_dict): + if len(tokens) != 2: + raise ValueError( + 'Only support binary relation in restriction.') + stripped_tokens = [t.strip() for t in tokens] + left_k, left_v = _get_kv(stripped_tokens[0], params_dict) + right_k, right_v = _get_kv(stripped_tokens[1], params_dict) + return [left_k, left_v, right_k, right_v] + + params_dict = self.as_dict() + for restriction in self._restrictions: + if '==' in restriction: + tokens = restriction.split('==') + _, left_v, _, right_v = _get_kvs(tokens, params_dict) + if left_v != right_v: + raise KeyError('Found inconsistncy between key `{}` and key `{}`.' + .format(tokens[0], tokens[1])) + elif '!=' in restriction: + tokens = restriction.split('!=') + _, left_v, _, right_v = _get_kvs(tokens, params_dict) + if left_v == right_v: + raise KeyError('Found inconsistncy between key `{}` and key `{}`.' + .format(tokens[0], tokens[1])) + elif '<' in restriction: + tokens = restriction.split('<') + _, left_v, _, right_v = _get_kvs(tokens, params_dict) + if left_v >= right_v: + raise KeyError('Found inconsistncy between key `{}` and key `{}`.' + .format(tokens[0], tokens[1])) + elif '<=' in restriction: + tokens = restriction.split('<=') + _, left_v, _, right_v = _get_kvs(tokens, params_dict) + if left_v > right_v: + raise KeyError('Found inconsistncy between key `{}` and key `{}`.' + .format(tokens[0], tokens[1])) + elif '>' in restriction: + tokens = restriction.split('>') + _, left_v, _, right_v = _get_kvs(tokens, params_dict) + if left_v <= right_v: + raise KeyError('Found inconsistncy between key `{}` and key `{}`.' + .format(tokens[0], tokens[1])) + elif '>=' in restriction: + tokens = restriction.split('>=') + _, left_v, _, right_v = _get_kvs(tokens, params_dict) + if left_v < right_v: + raise KeyError('Found inconsistncy between key `{}` and key `{}`.' + .format(tokens[0], tokens[1])) + else: + raise ValueError('Unsupported relation in restriction.') + + +def read_yaml_to_params_dict(file_path): + """Reads a YAML file to a ParamsDict.""" + with tf.gfile.Open(file_path, 'r') as f: + params_dict = yaml.load(f) + return ParamsDict(params_dict) + + +def save_params_dict_to_yaml(params, file_path): + """Saves the input ParamsDict to a YAML file.""" + with tf.gfile.Open(file_path, 'w') as f: + def _my_list_rep(dumper, data): + # u'tag:yaml.org,2002:seq' is the YAML internal tag for sequence. + return dumper.represent_sequence( + u'tag:yaml.org,2002:seq', data, flow_style=True) + yaml.add_representer(list, _my_list_rep) + yaml.dump(params.as_dict(), f, default_flow_style=False) + + +def nested_csv_str_to_json_str(csv_str): + """Converts a nested (using '.') comma-separated k=v string to a JSON string. + + Converts a comma-separated string of key/value pairs that supports + nesting of keys to a JSON string. Nesting is implemented using + '.' between levels for a given key. + + Spacing between commas and = is supported (e.g. there is no difference between + "a=1,b=2", "a = 1, b = 2", or "a=1, b=2") but there should be no spaces before + keys or after values (e.g. " a=1,b=2" and "a=1,b=2 " are not supported). + + Note that this will only support values supported by CSV, meaning + values such as nested lists (e.g. "a=[[1,2,3],[4,5,6]]") are not + supported. Strings are supported as well, e.g. "a='hello'". + + An example conversion would be: + + "a=1, b=2, c.a=2, c.b=3, d.a.a=5" + + to + + "{ a: 1, b : 2, c: {a : 2, b : 3}, d: {a: {a : 5}}}" + + Args: + csv_str: the comma separated string. + + Returns: + the converted JSON string. + + Raises: + ValueError: If csv_str is not in a comma separated string or + if the string is formatted incorrectly. + """ + if not csv_str: + return '' + + formatted_entries = [] + nested_map = collections.defaultdict(list) + pos = 0 + while pos < len(csv_str): + m = _PARAM_RE.match(csv_str, pos) + if not m: + raise ValueError('Malformed hyperparameter value while parsing ' + 'CSV string: %s' % csv_str[pos:]) + pos = m.end() + # Parse the values. + m_dict = m.groupdict() + name = m_dict['name'] + v = m_dict['val'] + + # If a GCS path (e.g. gs://...) is provided, wrap this in quotes + # as yaml.load would otherwise throw an exception + if re.match(r'(?=[^\"\'])(?=[gs://])', v): + v = '\'{}\''.format(v) + + name_nested = name.split('.') + if len(name_nested) > 1: + grouping = name_nested[0] + value = '.'.join(name_nested[1:]) + '=' + v + nested_map[grouping].append(value) + else: + formatted_entries.append('%s : %s' % (name, v)) + + for grouping, value in nested_map.items(): + value = ','.join(value) + value = nested_csv_str_to_json_str(value) + formatted_entries.append('%s : %s' % (grouping, value)) + return '{' + ', '.join(formatted_entries) + '}' + + +def override_params_dict(params, dict_or_string_or_yaml_file, is_strict): + """Override a given ParamsDict using a dict, JSON/YAML/CSV string or YAML file. + + The logic of the function is outlined below: + 1. Test that the input is a dict. If not, proceed to 2. + 2. Tests that the input is a string. If not, raise unknown ValueError + 2.1. Test if the string is in a CSV format. If so, parse. + If not, proceed to 2.2. + 2.2. Try loading the string as a YAML/JSON. If successful, parse to + dict and use it to override. If not, proceed to 2.3. + 2.3. Try using the string as a file path and load the YAML file. + + Args: + params: a ParamsDict object to be overridden. + dict_or_string_or_yaml_file: a Python dict, JSON/YAML/CSV string or + path to a YAML file specifying the parameters to be overridden. + is_strict: a boolean specifying whether override is strict or not. + + Returns: + params: the overridden ParamsDict object. + + Raises: + ValueError: if failed to override the parameters. + """ + if not dict_or_string_or_yaml_file: + return params + if isinstance(dict_or_string_or_yaml_file, dict): + params.override(dict_or_string_or_yaml_file, is_strict) + elif isinstance(dict_or_string_or_yaml_file, six.string_types): + try: + dict_or_string_or_yaml_file = ( + nested_csv_str_to_json_str(dict_or_string_or_yaml_file)) + except ValueError: + pass + params_dict = yaml.load(dict_or_string_or_yaml_file) + if isinstance(params_dict, dict): + params.override(params_dict, is_strict) + else: + with tf.gfile.Open(dict_or_string_or_yaml_file) as f: + params.override(yaml.load(f), is_strict) + else: + raise ValueError('Unknown input type to parse.') + return params diff --git a/TensorFlow/contrib/cv/Mixnet_ID2072_for_Tensorflow/imagenet_input.py b/TensorFlow/contrib/cv/Mixnet_ID2072_for_Tensorflow/imagenet_input.py new file mode 100644 index 0000000000000000000000000000000000000000..b9e0c2d66d4aaaf972e104b0e1a545d60903feff --- /dev/null +++ b/TensorFlow/contrib/cv/Mixnet_ID2072_for_Tensorflow/imagenet_input.py @@ -0,0 +1,370 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Efficient ImageNet input pipeline using tf.data.Dataset.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import abc +import collections +import functools +import os +from absl import logging +import tensorflow.compat.v1 as tf + +import preprocessing + + +# The input tensor is in the range of [0, 255], we need to scale them. +MEAN_RGB = [0.485 * 255, 0.456 * 255, 0.406 * 255] +STDDEV_RGB = [0.229 * 255, 0.224 * 255, 0.225 * 255] + + +def build_image_serving_input_fn(image_size): + """Builds a serving input fn for raw images.""" + def _image_serving_input_fn(): + """Serving input fn for raw images.""" + def _preprocess_image(image_bytes): + """Preprocess a single raw image.""" + image = preprocessing.preprocess_image( + image_bytes=image_bytes, is_training=False, image_size=image_size) + return image + + image_bytes_list = tf.placeholder( + shape=[None], + dtype=tf.string, + ) + images = tf.map_fn( + _preprocess_image, image_bytes_list, back_prop=False, dtype=tf.float32) + return tf.estimator.export.ServingInputReceiver( + images, {'image_bytes': image_bytes_list}) + return _image_serving_input_fn + + +class ImageNetTFExampleInput(object): + """Base class for ImageNet input_fn generator. + + Attributes: + image_preprocessing_fn: function to preprocess images + is_training: `bool` for whether the input is for training + use_bfloat16: If True, use bfloat16 precision; else use float32. + num_cores: `int` for the number of TPU cores + image_size: `int` for image size (both width and height). + transpose_input: 'bool' for whether to use the double transpose trick + """ + __metaclass__ = abc.ABCMeta + + def __init__(self, + is_training, + use_bfloat16, + num_cores=8, + image_size=224, + transpose_input=False): + self.image_preprocessing_fn = preprocessing.preprocess_image + self.is_training = is_training + self.use_bfloat16 = use_bfloat16 + self.num_cores = num_cores + self.transpose_input = transpose_input + self.image_size = image_size + + def set_shapes(self, batch_size, images, labels): + """Statically set the batch_size dimension.""" + if self.transpose_input: + images.set_shape(images.get_shape().merge_with( + tf.TensorShape([None, None, None, batch_size]))) + labels.set_shape(labels.get_shape().merge_with( + tf.TensorShape([batch_size]))) + else: + images.set_shape(images.get_shape().merge_with( + tf.TensorShape([batch_size, None, None, None]))) + labels.set_shape(labels.get_shape().merge_with( + tf.TensorShape([batch_size]))) + + return images, labels + + def dataset_parser(self, value): + """Parses an image and its label from a serialized ResNet-50 TFExample. + + Args: + value: serialized string containing an ImageNet TFExample. + + Returns: + Returns a tuple of (image, label) from the TFExample. + """ + keys_to_features = { + 'image/encoded': tf.FixedLenFeature((), tf.string, ''), + 'image/class/label': tf.FixedLenFeature([], tf.int64, -1), + } + + parsed = tf.parse_single_example(value, keys_to_features) + image_bytes = tf.reshape(parsed['image/encoded'], shape=[]) + + image = self.image_preprocessing_fn( + image_bytes=image_bytes, + is_training=self.is_training, + image_size=self.image_size, + use_bfloat16=self.use_bfloat16) + + # Subtract one so that labels are in [0, 1000). + label = tf.cast( + tf.reshape(parsed['image/class/label'], shape=[]), dtype=tf.int32) - 1 + + return image, label + + @abc.abstractmethod + def make_source_dataset(self, index, num_hosts): + """Makes dataset of serialized TFExamples. + + The returned dataset will contain `tf.string` tensors, but these strings are + serialized `TFExample` records that will be parsed by `dataset_parser`. + + If self.is_training, the dataset should be infinite. + + Args: + index: current host index. + num_hosts: total number of hosts. + + Returns: + A `tf.data.Dataset` object. + """ + return + + def input_fn(self, params): + """Input function which provides a single batch for train or eval. + + Args: + params: `dict` of parameters passed from the `TPUEstimator`. + `params['batch_size']` is always provided and should be used as the + effective batch size. + + Returns: + A `tf.data.Dataset` object. + """ + # Retrieves the batch size for the current shard. The # of shards is + # computed according to the input pipeline deployment. See + # tf.estimator.tpu.RunConfig for details. + batch_size = params['train_batch_size'] + + if 'context' in params: + current_host = params['context'].current_input_fn_deployment()[1] + num_hosts = params['context'].num_hosts + else: + current_host = 0 + num_hosts = 1 + + dataset = self.make_source_dataset(current_host, num_hosts) + + # Use the fused map-and-batch operation. + # + # For XLA, we must used fixed shapes. Because we repeat the source training + # dataset indefinitely, we can use `drop_remainder=True` to get fixed-size + # batches without dropping any training examples. + # + # When evaluating, `drop_remainder=True` prevents accidentally evaluating + # the same image twice by dropping the final batch if it is less than a full + # batch size. As long as this validation is done with consistent batch size, + # exactly the same images will be used. + dataset = dataset.apply( + tf.data.experimental.map_and_batch( + self.dataset_parser, + batch_size=batch_size, + num_parallel_batches=self.num_cores, + drop_remainder=True)) + + # Transpose for performance on TPU + if self.transpose_input: + dataset = dataset.map( + lambda images, labels: (tf.transpose( + images, [1, 2, 3, 0]), labels), + num_parallel_calls=self.num_cores) + + # Assign static batch size dimension + dataset = dataset.map(functools.partial(self.set_shapes, batch_size)) + + # Prefetch overlaps in-feed with training + dataset = dataset.prefetch(tf.data.experimental.AUTOTUNE) + return dataset + + +class ImageNetInput(ImageNetTFExampleInput): + """Generates ImageNet input_fn from a series of TFRecord files. + + The training data is assumed to be in TFRecord format with keys as specified + in the dataset_parser below, sharded across 1024 files, named sequentially: + + train-00000-of-01024 + train-00001-of-01024 + ... + train-01023-of-01024 + + The validation data is in the same format but sharded in 128 files. + + The format of the data required is created by the script at: + https://github.com/tensorflow/tpu/blob/master/tools/datasets/imagenet_to_gcs.py + """ + + def __init__(self, + is_training, + use_bfloat16, + transpose_input, + data_dir, + image_size=224, + num_parallel_calls=64, + cache=False): + """Create an input from TFRecord files. + + Args: + is_training: `bool` for whether the input is for training + use_bfloat16: If True, use bfloat16 precision; else use float32. + transpose_input: 'bool' for whether to use the double transpose trick + data_dir: `str` for the directory of the training and validation data; + if 'null' (the literal string 'null') or implicitly False + then construct a null pipeline, consisting of empty images + and blank labels. + image_size: `int` for image size (both width and height). + num_parallel_calls: concurrency level to use when reading data from disk. + cache: if true, fill the dataset by repeating from its cache + """ + super(ImageNetInput, self).__init__( + is_training=is_training, + image_size=image_size, + use_bfloat16=use_bfloat16, + transpose_input=transpose_input) + self.data_dir = data_dir + if self.data_dir == 'null' or not self.data_dir: + self.data_dir = None + self.num_parallel_calls = num_parallel_calls + self.cache = cache + + def _get_null_input(self, data): + """Returns a null image (all black pixels). + + Args: + data: element of a dataset, ignored in this method, since it produces + the same null image regardless of the element. + + Returns: + a tensor representing a null image. + """ + del data # Unused since output is constant regardless of input + return tf.zeros([self.image_size, self.image_size, 3], tf.bfloat16 + if self.use_bfloat16 else tf.float32) + + def dataset_parser(self, value): + """See base class.""" + if not self.data_dir: + return value, tf.constant(0, tf.int32) + return super(ImageNetInput, self).dataset_parser(value) + + def make_source_dataset(self, index, num_hosts): + """See base class.""" + if not self.data_dir: + tf.logging.info('Undefined data_dir implies null input') + return tf.data.Dataset.range(1).repeat().map(self._get_null_input) + + # Shuffle the filenames to ensure better randomization. + file_pattern = os.path.join( + self.data_dir, 'train-*' if self.is_training else 'validation-*') + + # For multi-host training, we want each hosts to always process the same + # subset of files. Each host only sees a subset of the entire dataset, + # allowing us to cache larger datasets in memory. + dataset = tf.data.Dataset.list_files(file_pattern, shuffle=False) + dataset = dataset.shard(num_hosts, index) + + if self.is_training and not self.cache: + dataset = dataset.repeat() + + def fetch_dataset(filename): + """Fetch the dataset""" + buffer_size = 8 * 1024 * 1024 # 8 MiB per file + dataset = tf.data.TFRecordDataset( + filename, buffer_size=buffer_size) + return dataset + + # Read the data from disk in parallel + dataset = dataset.apply( + tf.data.experimental.parallel_interleave( + fetch_dataset, cycle_length=self.num_parallel_calls, sloppy=True)) + + if self.cache: + dataset = dataset.cache().apply( + tf.data.experimental.shuffle_and_repeat(1024 * 16)) + else: + dataset = dataset.shuffle(1024) + return dataset + + +# Defines a selection of data from a Cloud Bigtable. +BigtableSelection = collections.namedtuple('BigtableSelection', [ + 'project', 'instance', 'table', 'prefix', 'column_family', + 'column_qualifier' +]) + + +class ImageNetBigtableInput(ImageNetTFExampleInput): + """Generates ImageNet input_fn from a Bigtable for training or evaluation. + """ + + def __init__(self, is_training, use_bfloat16, transpose_input, selection): + """Constructs an ImageNet input from a BigtableSelection. + + Args: + is_training: `bool` for whether the input is for training + use_bfloat16: If True, use bfloat16 precision; else use float32. + transpose_input: 'bool' for whether to use the double transpose trick + selection: a BigtableSelection specifying a part of a Bigtable. + """ + super(ImageNetBigtableInput, self).__init__( + is_training=is_training, + use_bfloat16=use_bfloat16, + transpose_input=transpose_input) + self.selection = selection + + def make_source_dataset(self, index, num_hosts): + """See base class.""" + data = self.selection + try: + from tensorflow.contrib.cloud import BigtableClient # pylint: disable=g-import-not-at-top + except ImportError as e: + logging.exception('Bigtable is not supported in TensorFlow 2.x.') + raise e + + client = BigtableClient(data.project, data.instance) + table = client.table(data.table) + ds = table.parallel_scan_prefix(data.prefix, + columns=[(data.column_family, + data.column_qualifier)]) + # The Bigtable datasets will have the shape (row_key, data) + ds_data = ds.map(lambda index, data: data) + + if self.is_training: + ds_data = ds_data.repeat() + + return ds_data diff --git a/TensorFlow/contrib/cv/Mixnet_ID2072_for_Tensorflow/lars_optimizer.py b/TensorFlow/contrib/cv/Mixnet_ID2072_for_Tensorflow/lars_optimizer.py new file mode 100644 index 0000000000000000000000000000000000000000..6cc2f8d8d8ffe86fde3490222f8abedae3e6f782 --- /dev/null +++ b/TensorFlow/contrib/cv/Mixnet_ID2072_for_Tensorflow/lars_optimizer.py @@ -0,0 +1,202 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Layer-wise Adaptive Rate Scaling optimizer for large-batch training.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import tensorflow.compat.v1 as tf + + +class LARSOptimizer(tf.train.Optimizer): + """Layer-wise Adaptive Rate Scaling for large batch training. + + Introduced by "Large Batch Training of Convolutional Networks" by Y. You, + I. Gitman, and B. Ginsburg. (https://arxiv.org/abs/1708.03888) + + Implements the LARS learning rate scheme presented in the paper above. This + optimizer is useful when scaling the batch size to up to 32K without + significant performance degradation. It is recommended to use the optimizer + in conjunction with: + - Gradual learning rate warm-up + - Linear learning rate scaling + - Poly rule learning rate decay + + Note, LARS scaling is currently only enabled for dense tensors. Sparse tensors + use the default momentum optimizer. + """ + + def __init__( + self, + learning_rate, + momentum=0.9, + weight_decay=0.0001, + # The LARS coefficient is a hyperparameter + eeta=0.001, + epsilon=0.0, + name="LARSOptimizer", + # Enable skipping variables from LARS scaling. + + # todo(sameerkm): Enable a direct mechanism to pass a + # subset of variables to the optimizer. + + skip_list=None, + use_nesterov=False): + """Construct a new LARS Optimizer. + + Args: + learning_rate: A `Tensor` or floating point value. The base learning rate. + momentum: A floating point value. Momentum hyperparameter. + weight_decay: A floating point value. Weight decay hyperparameter. + eeta: LARS coefficient as used in the paper. Dfault set to LARS + coefficient from the paper. (eeta / weight_decay) determines the highest + scaling factor in LARS. + epsilon: Optional epsilon parameter to be set in models that have very + small gradients. Default set to 0.0. + name: Optional name prefix for variables and ops created by LARSOptimizer. + skip_list: List of strings to enable skipping variables from LARS scaling. + If any of the strings in skip_list is a subset of var.name, variable + 'var' is skipped from LARS scaling. For a typical classification model + with batch normalization, the skip_list is ['batch_normalization', + 'bias'] + use_nesterov: when set to True, nesterov momentum will be enabled + + Raises: + ValueError: If a hyperparameter is set to a non-sensical value. + """ + if momentum < 0.0: + raise ValueError("momentum should be positive: %s" % momentum) + if weight_decay < 0.0: + raise ValueError( + "weight_decay should be positive: %s" % weight_decay) + super(LARSOptimizer, self).__init__(use_locking=False, name=name) + + self._learning_rate = learning_rate + self._momentum = momentum + self._weight_decay = weight_decay + self._eeta = eeta + self._epsilon = epsilon + self._name = name + self._skip_list = skip_list + self._use_nesterov = use_nesterov + + if callable(learning_rate): + learning_rate = learning_rate() + self._learning_rate_tensor = tf.convert_to_tensor( + learning_rate, name="learning_rate") + if callable(momentum): + momentum = momentum() + self._momentum_tensor = tf.convert_to_tensor(momentum, name="momentum") + + def _create_slots(self, var_list): + for v in var_list: + self._zeros_slot(v, "momentum", self._name) + + def compute_lr(self, grad, var): + """Comepute the learning rate according to the weight regularization gradient and var""" + scaled_lr = self._learning_rate + if self._skip_list is None or not any(v in var.name + for v in self._skip_list): + w_norm = tf.norm(var, ord=2) + g_norm = tf.norm(grad, ord=2) + trust_ratio = tf.where( + tf.math.greater(w_norm, 0), + tf.where( + tf.math.greater(g_norm, 0), + (self._eeta * w_norm / + (g_norm + self._weight_decay * w_norm + self._epsilon)), 1.0), + 1.0) + scaled_lr = self._learning_rate * trust_ratio + # Add the weight regularization gradient + grad = grad + self._weight_decay * var + return scaled_lr, grad + + def _apply_dense(self, grad, var): + scaled_lr, grad = self.compute_lr(grad, var) + mom = self.get_slot(var, "momentum") + return tf.raw_ops.ApplyMomentum( + var, + mom, + tf.cast(1.0, var.dtype.base_dtype), + grad * scaled_lr, + self._momentum, + use_locking=False, + use_nesterov=self._use_nesterov) + + def _resource_apply_dense(self, grad, var): + scaled_lr, grad = self.compute_lr(grad, var) + mom = self.get_slot(var, "momentum") + return tf.raw_ops.ResourceApplyMomentum( + var=var.handle, + accum=mom.handle, + lr=tf.cast(1.0, var.dtype.base_dtype), + grad=grad * scaled_lr, + momentum=self._momentum, + use_locking=False, + use_nesterov=self._use_nesterov) + + # Fallback to momentum optimizer for sparse tensors + def _apply_sparse(self, grad, var): + mom = self.get_slot(var, "momentum") + return tf.raw_ops.SparseApplyMomentum( + var, + mom, + tf.cast(self._learning_rate_tensor, var.dtype.base_dtype), + grad.values, + grad.indices, + tf.cast(self._momentum_tensor, var.dtype.base_dtype), + use_locking=self._use_locking, + use_nesterov=self._use_nesterov).op + + def _resource_apply_sparse(self, grad, var, indices): + mom = self.get_slot(var, "momentum") + return tf.raw_ops.ResourceSparseApplyMomentum( + var.handle, + mom.handle, + tf.cast(self._learning_rate_tensor, grad.dtype), + grad, + indices, + tf.cast(self._momentum_tensor, grad.dtype), + use_locking=self._use_locking, + use_nesterov=self._use_nesterov) + + def _prepare(self): + """ + # error: Attribute '_xxxx' defined outside _init__ + learning_rate = self._learning_rate + if callable(learning_rate): + learning_rate = learning_rate() + self._learning_rate_tensor = tf.convert_to_tensor( + learning_rate, name="learning_rate") + momentum = self._momentum + if callable(momentum): + momentum = momentum() + self._momentum_tensor = tf.convert_to_tensor(momentum, name="momentum") + """ + pass diff --git a/TensorFlow/contrib/cv/Mixnet_ID2072_for_Tensorflow/mixnet/__init__.py b/TensorFlow/contrib/cv/Mixnet_ID2072_for_Tensorflow/mixnet/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..d02ecafe1369324dccd22f1d574e0d0a1b7a9f11 --- /dev/null +++ b/TensorFlow/contrib/cv/Mixnet_ID2072_for_Tensorflow/mixnet/__init__.py @@ -0,0 +1,28 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""The initial file of mixnet.""" diff --git a/TensorFlow/contrib/cv/Mixnet_ID2072_for_Tensorflow/mixnet/custom_layers.py b/TensorFlow/contrib/cv/Mixnet_ID2072_for_Tensorflow/mixnet/custom_layers.py new file mode 100644 index 0000000000000000000000000000000000000000..a74f4931a94ed7b1c63653b9d71ba51003609a38 --- /dev/null +++ b/TensorFlow/contrib/cv/Mixnet_ID2072_for_Tensorflow/mixnet/custom_layers.py @@ -0,0 +1,153 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Custom layers.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import tensorflow.compat.v1 as tf + + +def _get_conv2d(filters, kernel_size, use_keras, **kwargs): + """A helper function to create Conv2D layer.""" + if use_keras: + return tf.keras.layers.Conv2D( + filters=filters, kernel_size=kernel_size, **kwargs) + else: + return tf.layers.Conv2D(filters=filters, kernel_size=kernel_size, **kwargs) + + +def _split_channels(total_filters, num_groups): + split = [total_filters // num_groups for _ in range(num_groups)] + split[0] += total_filters - sum(split) + return split + + +def _get_shape_value(maybe_v2_shape): + if maybe_v2_shape is None: + return None + elif isinstance(maybe_v2_shape, int): + return maybe_v2_shape + else: + return maybe_v2_shape.value + + +class GroupedConv2D(object): + """Groupped convolution. + + Currently tf.keras and tf.layers don't support group convolution, so here we + use split/concat to implement this op. It reuses kernel_size for group + definition, where len(kernel_size) is number of groups. Notably, it allows + different group has different kernel size. + """ + + def __init__(self, filters, kernel_size, use_keras, **kwargs): + """Initialize the layer. + + Args: + filters: Integer, the dimensionality of the output space. + kernel_size: An integer or a list. If it is a single integer, then it is + same as the original Conv2D. If it is a list, then we split the channels + and perform different kernel for each group. + use_keras: An boolean value, whether to use keras layer. + **kwargs: other parameters passed to the original conv2d layer. + """ + self._groups = len(kernel_size) + self._channel_axis = -1 + + self._convs = [] + splits = _split_channels(filters, self._groups) + for i in range(self._groups): + self._convs.append( + _get_conv2d(splits[i], kernel_size[i], use_keras, **kwargs)) + + def __call__(self, inputs): + if len(self._convs) == 1: + return self._convs[0](inputs) + + filters = _get_shape_value(inputs.shape[self._channel_axis]) + splits = _split_channels(filters, len(self._convs)) + x_splits = tf.split(inputs, splits, self._channel_axis) + x_outputs = [c(x) for x, c in zip(x_splits, self._convs)] + x = tf.concat(x_outputs, self._channel_axis) + return x + + +class MixConv(object): + """MixConv with mixed depthwise convolutional kernels. + + MDConv is an improved depthwise convolution that mixes multiple kernels (e.g. + 3x3, 5x5, etc). Right now, we use an naive implementation that split channels + into multiple groups and perform different kernels for each group. + + See Mixnet paper for more details. + """ + + def __init__(self, kernel_size, strides, dilated=False, **kwargs): + """Initialize the layer. + + Most of args are the same as tf.keras.layers.DepthwiseConv2D except it has + an extra parameter "dilated" to indicate whether to use dilated conv to + simulate large kernel size. If dilated=True, then dilation_rate is ignored. + + Args: + kernel_size: An integer or a list. If it is a single integer, then it is + same as the original tf.keras.layers.DepthwiseConv2D. If it is a list, + then we split the channels and perform different kernel for each group. + strides: An integer or tuple/list of 2 integers, specifying the strides of + the convolution along the height and width. + dilated: Bool. indicate whether to use dilated conv to simulate large + kernel size. + **kwargs: other parameters passed to the original depthwise_conv layer. + """ + self._channel_axis = -1 + self._dilated = dilated + + self._convs = [] + for s in kernel_size: + d = 1 + if strides[0] == 1 and self._dilated: + # Only apply dilated conv for stride 1 if needed. + d, s = (s - 1) // 2, 3 + tf.logging.info( + 'Use dilated conv with dilation rate = {}'.format(d)) + self._convs.append( + tf.keras.layers.DepthwiseConv2D( + s, strides=strides, dilation_rate=d, **kwargs)) + + def __call__(self, inputs): + if len(self._convs) == 1: + return self._convs[0](inputs) + + filters = _get_shape_value(inputs.shape[self._channel_axis]) + splits = _split_channels(filters, len(self._convs)) + x_splits = tf.split(inputs, splits, self._channel_axis) + x_outputs = [c(x) for x, c in zip(x_splits, self._convs)] + x = tf.concat(x_outputs, self._channel_axis) + return x diff --git a/TensorFlow/contrib/cv/Mixnet_ID2072_for_Tensorflow/mixnet/g3doc/mixnet-flops.png b/TensorFlow/contrib/cv/Mixnet_ID2072_for_Tensorflow/mixnet/g3doc/mixnet-flops.png new file mode 100644 index 0000000000000000000000000000000000000000..885fc69c8d3ebf883329ec7c65d7f02411bb9570 Binary files /dev/null and b/TensorFlow/contrib/cv/Mixnet_ID2072_for_Tensorflow/mixnet/g3doc/mixnet-flops.png differ diff --git a/TensorFlow/contrib/cv/Mixnet_ID2072_for_Tensorflow/mixnet/mixnet_builder.py b/TensorFlow/contrib/cv/Mixnet_ID2072_for_Tensorflow/mixnet/mixnet_builder.py new file mode 100644 index 0000000000000000000000000000000000000000..478f458f2efcd6fc02e87ff742428274d007fdb9 --- /dev/null +++ b/TensorFlow/contrib/cv/Mixnet_ID2072_for_Tensorflow/mixnet/mixnet_builder.py @@ -0,0 +1,307 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Mixnet model builder (branched from MnasNet).""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import re +import tensorflow.compat.v1 as tf + +from mixnet import mixnet_model + + +class MixnetDecoder(object): + """A class of Mixnet decoder to get model configuration.""" + + def _decode_block_string(self, block_string): + """Gets a mixnet block through a string notation of arguments. + + E.g. r2_k3_a1_p1_s2_e1_i32_o16_se0.25_noskip: r - number of repeat blocks, + k - kernel size, s - strides (1-9), e - expansion ratio, i - input filters, + o - output filters, se - squeeze/excitation ratio + + Args: + block_string: a string, a string representation of block arguments. + + Returns: + A BlockArgs instance. + Raises: + ValueError: if the strides option is not correctly specified. + """ + assert isinstance(block_string, str) + ops = block_string.split('_') + options = {} + for op in ops: + splits = re.split(r'(\d.*)', op) + if len(splits) >= 2: + key, value = splits[:2] + options[key] = value + + if 's' not in options or len(options['s']) != 2: + raise ValueError('Strides options should be a pair of integers.') + + def _parse_ksize(ss): + return [int(k) for k in ss.split('.')] + + return mixnet_model.BlockArgs( + expand_ksize=_parse_ksize(options['a']), + dw_ksize=_parse_ksize(options['k']), + project_ksize=_parse_ksize(options['p']), + num_repeat=int(options['r']), + input_filters=int(options['i']), + output_filters=int(options['o']), + expand_ratio=int(options['e']), + id_skip=('noskip' not in block_string), + se_ratio=float(options['se']) if 'se' in options else None, + strides=[int(options['s'][0]), int(options['s'][1])], + swish=('sw' in block_string), + dilated=('dilated' in block_string)) + + def _encode_block_string(self, block): + """Encodes a Mixnet block to a string.""" + def _encode_ksize(arr): + return '.'.join([str(k) for k in arr]) + + args = [ + 'r%d' % block.num_repeat, + 'k%s' % _encode_ksize(block.dw_ksize), + 'a%s' % _encode_ksize(block.expand_ksize), + 'p%s' % _encode_ksize(block.project_ksize), + 's%d%d' % (block.strides[0], block.strides[1]), + 'e%s' % block.expand_ratio, + 'i%d' % block.input_filters, + 'o%d' % block.output_filters + ] + if (block.se_ratio is not None and block.se_ratio > 0 and + block.se_ratio <= 1): + args.append('se%s' % block.se_ratio) + if block.id_skip is False: # pylint: disable=g-bool-id-comparison + args.append('noskip') + if block.swish: + args.append('sw') + if block.dilated: + args.append('dilated') + return '_'.join(args) + + def decode(self, string_list): + """Decodes a list of string notations to specify blocks inside the network. + + Args: + string_list: a list of strings, each string is a notation of Mixnet + block.build_model_base + + Returns: + A list of namedtuples to represent Mixnet blocks arguments. + """ + assert isinstance(string_list, list) + blocks_args = [] + for block_string in string_list: + blocks_args.append(self._decode_block_string(block_string)) + return blocks_args + + def encode(self, blocks_args): + """Encodes a list of Mixnet Blocks to a list of strings. + + Args: + blocks_args: A list of namedtuples to represent Mixnet blocks arguments. + Returns: + a list of strings, each string is a notation of Mixnet block. + """ + block_strings = [] + for block in blocks_args: + block_strings.append(self._encode_block_string(block)) + return block_strings + + +def mixnet_s(depth_multiplier=None): + """Creates mixnet-s model. + + Args: + depth_multiplier: multiplier to number of filters per layer. + + Returns: + blocks_args: a list of BlocksArgs for internal Mixnet blocks. + global_params: GlobalParams, global parameters for the model. + """ + blocks_args = [ + 'r1_k3_a1_p1_s11_e1_i16_o16', + 'r1_k3_a1.1_p1.1_s22_e6_i16_o24', + 'r1_k3_a1.1_p1.1_s11_e3_i24_o24', + + 'r1_k3.5.7_a1_p1_s22_e6_i24_o40_se0.5_sw', + 'r3_k3.5_a1.1_p1.1_s11_e6_i40_o40_se0.5_sw', + + 'r1_k3.5.7_a1_p1.1_s22_e6_i40_o80_se0.25_sw', + 'r2_k3.5_a1_p1.1_s11_e6_i80_o80_se0.25_sw', + + 'r1_k3.5.7_a1.1_p1.1_s11_e6_i80_o120_se0.5_sw', + 'r2_k3.5.7.9_a1.1_p1.1_s11_e3_i120_o120_se0.5_sw', + + 'r1_k3.5.7.9.11_a1_p1_s22_e6_i120_o200_se0.5_sw', + 'r2_k3.5.7.9_a1_p1.1_s11_e6_i200_o200_se0.5_sw', + ] + global_params = mixnet_model.GlobalParams( + batch_norm_momentum=0.99, + batch_norm_epsilon=1e-3, + dropout_rate=0.2, + data_format='channels_last', + num_classes=1000, + depth_multiplier=depth_multiplier, + depth_divisor=8, + min_depth=None, + stem_size=16, + use_keras=True, + feature_size=1536) + decoder = MixnetDecoder() + return decoder.decode(blocks_args), global_params + + +def mixnet_m(depth_multiplier=None): + """Creates a mixnet-m model. + + Args: + depth_multiplier: multiplier to number of filters per layer. + + Returns: + blocks_args: a list of BlocksArgs for internal Mixnet blocks. + global_params: GlobalParams, global parameters for the model. + """ + blocks_args = [ + 'r1_k3_a1_p1_s11_e1_i24_o24', + 'r1_k3.5.7_a1.1_p1.1_s22_e6_i24_o32', + 'r1_k3_a1.1_p1.1_s11_e3_i32_o32', + + 'r1_k3.5.7.9_a1_p1_s22_e6_i32_o40_se0.5_sw', + 'r3_k3.5_a1.1_p1.1_s11_e6_i40_o40_se0.5_sw', + + 'r1_k3.5.7_a1_p1_s22_e6_i40_o80_se0.25_sw', + 'r3_k3.5.7.9_a1.1_p1.1_s11_e6_i80_o80_se0.25_sw', + + 'r1_k3_a1_p1_s11_e6_i80_o120_se0.5_sw', + 'r3_k3.5.7.9_a1.1_p1.1_s11_e3_i120_o120_se0.5_sw', + + 'r1_k3.5.7.9_a1_p1_s22_e6_i120_o200_se0.5_sw', + 'r3_k3.5.7.9_a1_p1.1_s11_e6_i200_o200_se0.5_sw', + ] + global_params = mixnet_model.GlobalParams( + batch_norm_momentum=0.99, + batch_norm_epsilon=1e-3, + dropout_rate=0.25, + data_format='channels_last', + num_classes=1000, + depth_multiplier=depth_multiplier, + depth_divisor=8, + min_depth=None, + stem_size=24, + use_keras=True, + feature_size=1536) + decoder = MixnetDecoder() + return decoder.decode(blocks_args), global_params + + +def mixnet_l(depth_multiplier=None): + d = 1.3 * depth_multiplier if depth_multiplier else 1.3 + return mixnet_m(d) + + +def get_model_params(model_name, override_params): + """Get the block args and global params for a given model.""" + if model_name == 'mixnet-s': + blocks_args, global_params = mixnet_s() + elif model_name == 'mixnet-m': + blocks_args, global_params = mixnet_m() + elif model_name == 'mixnet-l': + blocks_args, global_params = mixnet_l() + else: + raise NotImplementedError( + 'model name is not pre-defined: %s' % model_name) + + if override_params: + # ValueError will be raised here if override_params has fields not included + # in global_params. + global_params = global_params._replace(**override_params) + return blocks_args, global_params + + +def build_model(images, model_name, training, override_params=None): + """A helper functiion to create a Mixnet model and return predicted logits. + + Args: + images: input images tensor. + model_name: string, the model name of a pre-defined Mixnet. + training: boolean, whether the model is constructed for training. + override_params: A dictionary of params for overriding. Fields must exist in + mixnet_model.GlobalParams. + + Returns: + logits: the logits tensor of classes. + endpoints: the endpoints for each layer. + Raises: + When model_name specified an undefined model, raises NotImplementedError. + When override_params has invalid fields, raises ValueError. + """ + assert isinstance(images, tf.Tensor) + blocks_args, global_params = get_model_params(model_name, override_params) + tf.logging.info('blocks_args= {}'.format(blocks_args)) + tf.logging.info('global_params= {}'.format(global_params)) + with tf.variable_scope(model_name): + model = mixnet_model.MixnetModel(blocks_args, global_params) + logits = model(images, training=training) + + logits = tf.identity(logits, 'logits') + return logits, model.endpoints + + +def build_model_base(images, model_name, training, override_params=None): + """A helper functiion to create a Mixnet base model and return global_pool. + + Args: + images: input images tensor. + model_name: string, the model name of a pre-defined Mixnet. + training: boolean, whether the model is constructed for training. + override_params: A dictionary of params for overriding. Fields must exist in + mixnet_model.GlobalParams. + + Returns: + features: global pool features. + endpoints: the endpoints for each layer. + Raises: + When model_name specified an undefined model, raises NotImplementedError. + When override_params has invalid fields, raises ValueError. + """ + assert isinstance(images, tf.Tensor) + blocks_args, global_params = get_model_params(model_name, override_params) + + with tf.variable_scope(model_name): + model = mixnet_model.MixnetModel(blocks_args, global_params) + features = model(images, training=training, features_only=True) + + features = tf.identity(features, 'global_pool') + return features, model.endpoints diff --git a/TensorFlow/contrib/cv/Mixnet_ID2072_for_Tensorflow/mixnet/mixnet_model.py b/TensorFlow/contrib/cv/Mixnet_ID2072_for_Tensorflow/mixnet/mixnet_model.py new file mode 100644 index 0000000000000000000000000000000000000000..830039bc3dad02c64d5997b08be0fb686811a359 --- /dev/null +++ b/TensorFlow/contrib/cv/Mixnet_ID2072_for_Tensorflow/mixnet/mixnet_model.py @@ -0,0 +1,452 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Contains definitions for mixnet model (branched from MnasNet). + +[1] Mingxing Tan, Quoc V. Le + MixNet: Mixed Depthwise Convolutional Kernels. + BMVC 2019. https://arxiv.org/abs/1907.09595 +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import collections +import numpy as np +import six +from six.moves import xrange # pylint: disable=redefined-builtin +import tensorflow.compat.v1 as tf + +from mixnet import custom_layers + + +GlobalParams = collections.namedtuple('GlobalParams', [ + 'batch_norm_momentum', 'batch_norm_epsilon', 'dropout_rate', 'data_format', + 'num_classes', 'depth_multiplier', 'depth_divisor', 'min_depth', + 'use_keras', 'stem_size', 'feature_size', +]) +GlobalParams.__new__.__defaults__ = (None,) * len(GlobalParams._fields) + +BlockArgs = collections.namedtuple('BlockArgs', [ + 'dw_ksize', 'expand_ksize', 'project_ksize', 'num_repeat', 'input_filters', + 'output_filters', 'expand_ratio', 'id_skip', 'strides', 'se_ratio', + 'swish', 'dilated', +]) +# defaults will be a public argument for namedtuple in Python 3.7 +# https://docs.python.org/3/library/collections.html#collections.namedtuple +BlockArgs.__new__.__defaults__ = (None,) * len(BlockArgs._fields) + + +def conv_kernel_initializer(shape, dtype=None, partition_info=None): + """Initialization for convolutional kernels. + + The main difference with tf.variance_scaling_initializer is that + tf.variance_scaling_initializer uses a truncated normal with an uncorrected + standard deviation, whereas here we use a normal distribution. Similarly, + tf.contrib.layers.variance_scaling_initializer uses a truncated normal with + a corrected standard deviation. + + Args: + shape: shape of variable + dtype: dtype of variable + partition_info: unused + + Returns: + an initialization for the variable + """ + del partition_info + kernel_height, kernel_width, _, out_filters = shape + fan_out = int(kernel_height * kernel_width * out_filters) + return tf.random_normal( + shape, mean=0.0, stddev=np.sqrt(2.0 / fan_out), dtype=dtype) + + +def dense_kernel_initializer(shape, dtype=None, partition_info=None): + """Initialization for dense kernels. + + This initialization is equal to + tf.variance_scaling_initializer(scale=1.0/3.0, mode='fan_out', + distribution='uniform'). + It is written out explicitly here for clarity. + + Args: + shape: shape of variable + dtype: dtype of variable + partition_info: unused + + Returns: + an initialization for the variable + """ + del partition_info + init_range = 1.0 / np.sqrt(shape[1]) + return tf.random_uniform(shape, -init_range, init_range, dtype=dtype) + + +def round_filters(filters, global_params): + """Round number of filters based on depth multiplier.""" + multiplier = global_params.depth_multiplier + divisor = global_params.depth_divisor + min_depth = global_params.min_depth + if not multiplier: + return filters + + filters *= multiplier + min_depth = min_depth or divisor + new_filters = max(min_depth, int( + filters + divisor / 2) // divisor * divisor) + # Make sure that round down does not go down by more than 10%. + if new_filters < 0.9 * filters: + new_filters += divisor + return new_filters + + +class MixnetBlock(object): + """A class of Mixnet block. + + Attributes: + endpoints: dict. A list of internal tensors. + """ + + def __init__(self, block_args, global_params): + """Initializes the block. + + Args: + block_args: BlockArgs, arguments to create a MixnetBlock. + global_params: GlobalParams, a set of global parameters. + """ + self._block_args = block_args + self._batch_norm_momentum = global_params.batch_norm_momentum + self._batch_norm_epsilon = global_params.batch_norm_epsilon + self._use_keras = global_params.use_keras + self._data_format = global_params.data_format + if self._data_format == 'channels_first': + self._channel_axis = 1 + self._spatial_dims = [2, 3] + else: + self._channel_axis = -1 + self._spatial_dims = [1, 2] + self._has_se = (self._block_args.se_ratio is not None) and ( + self._block_args.se_ratio > 0) and (self._block_args.se_ratio <= 1) + self._relu_fn = tf.nn.swish if self._block_args.swish else tf.nn.relu + + self.endpoints = None + + # Builds the block accordings to arguments. + self._build() + + def block_args(self): + return self._block_args + + def _build(self): + """Builds block according to the arguments.""" + filters = self._block_args.input_filters * self._block_args.expand_ratio + if self._block_args.expand_ratio != 1: + # Expansion phase: + self._expand_conv = custom_layers.GroupedConv2D( + filters=filters, + kernel_size=self._block_args.expand_ksize, + strides=[1, 1], + kernel_initializer=conv_kernel_initializer, + padding='same', + use_bias=False, + use_keras=self._use_keras) + self._bn0 = tf.layers.BatchNormalization( + axis=self._channel_axis, + momentum=self._batch_norm_momentum, + epsilon=self._batch_norm_epsilon, + fused=True) + + kernel_size = self._block_args.dw_ksize + # Depth-wise convolution phase: + self._depthwise_conv = custom_layers.MixConv( + kernel_size, + strides=self._block_args.strides, + depthwise_initializer=conv_kernel_initializer, + padding='same', + data_format=self._data_format, + use_bias=False, + dilated=self._block_args.dilated) + self._bn1 = tf.layers.BatchNormalization( + axis=self._channel_axis, + momentum=self._batch_norm_momentum, + epsilon=self._batch_norm_epsilon, + fused=True) + + if self._has_se: + num_reduced_filters = max( + 1, int(self._block_args.input_filters * self._block_args.se_ratio)) + # Squeeze and Excitation layer. + self._se_reduce = custom_layers.GroupedConv2D( + num_reduced_filters, + kernel_size=[1], + strides=[1, 1], + kernel_initializer=conv_kernel_initializer, + padding='same', + use_bias=True, + data_format=self._data_format, + use_keras=self._use_keras) + self._se_expand = custom_layers.GroupedConv2D( + filters, + kernel_size=[1], + strides=[1, 1], + kernel_initializer=conv_kernel_initializer, + padding='same', + use_bias=True, + data_format=self._data_format, + use_keras=self._use_keras) + + # Output phase: + filters = self._block_args.output_filters + self._project_conv = custom_layers.GroupedConv2D( + filters, + kernel_size=self._block_args.project_ksize, + strides=[1, 1], + kernel_initializer=conv_kernel_initializer, + padding='same', + use_bias=False, + data_format=self._data_format, + use_keras=self._use_keras) + self._bn2 = tf.layers.BatchNormalization( + axis=self._channel_axis, + momentum=self._batch_norm_momentum, + epsilon=self._batch_norm_epsilon, + fused=True) + + def _call_se(self, input_tensor): + """Call Squeeze and Excitation layer. + + Args: + input_tensor: Tensor, a single input tensor for Squeeze/Excitation layer. + + Returns: + A output tensor, which should have the same shape as input. + """ + se_tensor = tf.reduce_mean( + input_tensor, self._spatial_dims, keepdims=True) + se_tensor = self._se_expand(self._relu_fn(self._se_reduce(se_tensor))) + tf.logging.info('Built Squeeze and Excitation with tensor shape: %s' % + (se_tensor.shape)) + return tf.sigmoid(se_tensor) * input_tensor + + def call(self, inputs, training=True): + """Implementation of MixnetBlock call(). + + Args: + inputs: the inputs tensor. + training: boolean, whether the model is constructed for training. + + Returns: + A output tensor. + """ + tf.logging.info('Block input: %s shape: %s' % + (inputs.name, inputs.shape)) + if self._block_args.expand_ratio != 1: + x = self._relu_fn( + self._bn0(self._expand_conv(inputs), training=training)) + else: + x = inputs + tf.logging.info('Expand: %s shape: %s' % (x.name, x.shape)) + + x = self._relu_fn( + self._bn1(self._depthwise_conv(x), training=training)) + tf.logging.info('DWConv: %s shape: %s' % (x.name, x.shape)) + + if self._has_se: + with tf.variable_scope('se'): + x = self._call_se(x) + + self.endpoints = {'expansion_output': x} + + x = self._bn2(self._project_conv(x), training=training) + if self._block_args.id_skip: + if all( + s == 1 for s in self._block_args.strides + ) and self._block_args.input_filters == self._block_args.output_filters: + x = tf.add(x, inputs) + tf.logging.info('Project: %s shape: %s' % (x.name, x.shape)) + return tf.identity(x) + + +class MixnetModel(tf.keras.Model): + """A class implements tf.keras.Model for mixnet model. + + Reference: https://arxiv.org/abs/1807.11626 + """ + + def __init__(self, blocks_args=None, global_params=None): + """Initializes an `MixnetModel` instance. + + Args: + blocks_args: A list of BlockArgs to construct Mixnet block modules. + global_params: GlobalParams, a set of global parameters. + + Raises: + ValueError: when blocks_args is not specified as a list. + """ + super(MixnetModel, self).__init__() + if not isinstance(blocks_args, list): + raise ValueError('blocks_args should be a list.') + self._global_params = global_params + self._blocks_args = blocks_args + # Use relu in default for head and stem. + self._relu_fn = tf.nn.relu + self.endpoints = None + self._build() + + def _build(self): + """Builds a Mixnet model.""" + self._blocks = [] + # Builds blocks. + for block_args in self._blocks_args: + assert block_args.num_repeat > 0 + # Update block input and output filters based on depth multiplier. + block_args = block_args._replace( + input_filters=round_filters(block_args.input_filters, + self._global_params), + output_filters=round_filters(block_args.output_filters, + self._global_params)) + + # The first block needs to take care of stride and filter size increase. + self._blocks.append(MixnetBlock(block_args, self._global_params)) + if block_args.num_repeat > 1: + # pylint: disable=protected-access + block_args = block_args._replace( + input_filters=block_args.output_filters, strides=[1, 1]) + # pylint: enable=protected-access + for _ in xrange(block_args.num_repeat - 1): + self._blocks.append(MixnetBlock( + block_args, self._global_params)) + + batch_norm_momentum = self._global_params.batch_norm_momentum + batch_norm_epsilon = self._global_params.batch_norm_epsilon + if self._global_params.data_format == 'channels_first': + channel_axis = 1 + else: + channel_axis = -1 + + # Stem part. + stem_size = self._global_params.stem_size + self._conv_stem = custom_layers.GroupedConv2D( + filters=round_filters(stem_size, self._global_params), + kernel_size=[3], + strides=[2, 2], + kernel_initializer=conv_kernel_initializer, + padding='same', + use_bias=False, + use_keras=self._global_params.use_keras) + self._bn0 = tf.layers.BatchNormalization( + axis=channel_axis, + momentum=batch_norm_momentum, + epsilon=batch_norm_epsilon, + fused=True) + + # Head part. + self._conv_head = custom_layers.GroupedConv2D( + filters=self._global_params.feature_size, + kernel_size=[1], + strides=[1, 1], + kernel_initializer=conv_kernel_initializer, + padding='same', + use_bias=False, + use_keras=self._global_params.use_keras) + self._bn1 = tf.layers.BatchNormalization( + axis=channel_axis, + momentum=batch_norm_momentum, + epsilon=batch_norm_epsilon, + fused=True) + + self._avg_pooling = tf.keras.layers.GlobalAveragePooling2D( + data_format=self._global_params.data_format) + if self._global_params.use_keras: + self._fc = tf.keras.layers.Dense( + self._global_params.num_classes, + kernel_initializer=dense_kernel_initializer) + else: + self._fc = tf.layers.Dense( + self._global_params.num_classes, + kernel_initializer=dense_kernel_initializer) + if self._global_params.dropout_rate > 0: + self._dropout = tf.keras.layers.Dropout( + self._global_params.dropout_rate) + else: + self._dropout = None + + def call(self, inputs, training=True, features_only=None): + """Implementation of MixnetModel call(). + + Args: + inputs: input tensors. + training: boolean, whether the model is constructed for training. + features_only: build the base feature network only. + + Returns: + output tensors. + """ + outputs = None + self.endpoints = {} + # Calls Stem layers + with tf.variable_scope('stem'): + outputs = self._relu_fn( + self._bn0(self._conv_stem(inputs), training=training)) + tf.logging.info('Built stem layers with output shape: %s' % + outputs.shape) + self.endpoints['stem'] = outputs + + # Calls blocks. + reduction_idx = 0 + for idx, block in enumerate(self._blocks): + is_reduction = False + if ((idx == len(self._blocks) - 1) or + self._blocks[idx + 1].block_args().strides[0] > 1): + is_reduction = True + reduction_idx += 1 + + with tf.variable_scope('blocks_%s' % idx): + outputs = block.call(outputs, training=training) + self.endpoints['block_%s' % idx] = outputs + if is_reduction: + self.endpoints['reduction_%s' % reduction_idx] = outputs + if block.endpoints: + for k, v in six.iteritems(block.endpoints): + self.endpoints['block_%s/%s' % (idx, k)] = v + if is_reduction: + self.endpoints['reduction_%s/%s' % + (reduction_idx, k)] = v + self.endpoints['global_pool'] = outputs + + if not features_only: + # Calls final layers and returns logits. + with tf.variable_scope('head'): + outputs = self._relu_fn( + self._bn1(self._conv_head(outputs), training=training)) + outputs = self._avg_pooling(outputs) + if self._dropout: + outputs = self._dropout(outputs, training=training) + outputs = self._fc(outputs) + self.endpoints['head'] = outputs + return outputs diff --git a/TensorFlow/contrib/cv/Mixnet_ID2072_for_Tensorflow/mnas_utils.py b/TensorFlow/contrib/cv/Mixnet_ID2072_for_Tensorflow/mnas_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..667b8f76880bee1f80f86536b2a36e7f91cc360d --- /dev/null +++ b/TensorFlow/contrib/cv/Mixnet_ID2072_for_Tensorflow/mnas_utils.py @@ -0,0 +1,44 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Utils for MnasNet.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +import utils as efficientnet_utils + + + +# Import common utils from efficientnet. +archive_ckpt = efficientnet_utils.archive_ckpt +build_learning_rate = efficientnet_utils.build_learning_rate +build_optimizer = efficientnet_utils.build_optimizer +drop_connect = efficientnet_utils.drop_connect +get_ema_vars = efficientnet_utils.get_ema_vars +DepthwiseConv2D = efficientnet_utils.DepthwiseConv2D +EvalCkptDriver = efficientnet_utils.EvalCkptDriver diff --git a/TensorFlow/contrib/cv/Mixnet_ID2072_for_Tensorflow/mnasnet_main.py b/TensorFlow/contrib/cv/Mixnet_ID2072_for_Tensorflow/mnasnet_main.py new file mode 100644 index 0000000000000000000000000000000000000000..e3e1bc050107d63a87b932a8733de22449c1f0a5 --- /dev/null +++ b/TensorFlow/contrib/cv/Mixnet_ID2072_for_Tensorflow/mnasnet_main.py @@ -0,0 +1,895 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Train a MnasNet on ImageNet on TPU.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import os +import time +from absl import app +from absl import flags +from absl import logging +import numpy as np +import tensorflow.compat.v1 as tf +import tensorflow.compat.v2 as tf2 + +from hyperparameters import common_hparams_flags +from hyperparameters import common_tpu_flags +from hyperparameters import flags_to_params +from hyperparameters import params_dict +import imagenet_input +import mnas_utils +import mnasnet_models +from configs import mnasnet_config +from mixnet import mixnet_builder + +from npu_bridge.npu_init import * +from npu_bridge.estimator.npu.npu_config import NPURunConfig +from npu_bridge.estimator import npu_ops +from npu_bridge.estimator.npu.npu_estimator import NPUEstimator, NPUEstimatorSpec + +common_tpu_flags.define_common_tpu_flags() +common_hparams_flags.define_common_hparams_flags() + +FLAGS = flags.FLAGS + +FAKE_DATA_DIR = 'gs://cloud-tpu-test-datasets/fake_imagenet' + +# Model specific flags +flags.DEFINE_string( + 'model_name', + default=None, + help=( + 'The model name to select models among existing MnasNet configurations.' + )) + +flags.DEFINE_enum('mode', 'train_and_eval', + ['train_and_eval', 'train', 'eval', 'export_only'], + 'One of {"train_and_eval", "train", "eval", "export_only"}.') + +flags.DEFINE_integer('input_image_size', default=None, + help='Input image size.') + +flags.DEFINE_integer( + 'num_train_images', default=None, help='Size of training data set.') + +flags.DEFINE_integer( + 'num_eval_images', default=None, help='Size of evaluation data set.') + +flags.DEFINE_integer( + 'steps_per_eval', + default=6255, + help=('Controls how often evaluation is performed. Since evaluation is' + ' fairly expensive, it is advised to evaluate as infrequently as' + ' possible (i.e. up to --train_steps, which evaluates the model only' + ' after finishing the entire training regime).')) + +flags.DEFINE_integer( + 'eval_timeout', + default=None, + help='Maximum seconds between checkpoints before evaluation terminates.') + +flags.DEFINE_integer( + 'num_parallel_calls', + default=None, + help=('Number of parallel threads in CPU for the input pipeline')) + +flags.DEFINE_string( + 'bigtable_project', None, + 'The Cloud Bigtable project. If None, --gcp_project will be used.') +flags.DEFINE_string('bigtable_instance', None, + 'The Cloud Bigtable instance to load data from.') +flags.DEFINE_string('bigtable_table', 'imagenet', + 'The Cloud Bigtable table to load data from.') +flags.DEFINE_string('bigtable_train_prefix', 'train_', + 'The prefix identifying training rows.') +flags.DEFINE_string('bigtable_eval_prefix', 'validation_', + 'The prefix identifying evaluation rows.') +flags.DEFINE_string('bigtable_column_family', 'tfexample', + 'The column family storing TFExamples.') +flags.DEFINE_string('bigtable_column_qualifier', 'example', + 'The column name storing TFExamples.') + +flags.DEFINE_string( + 'data_format', + default=None, + help=('A flag to override the data format used in the model. The value' + ' is either channels_first or channels_last. To run the network on' + ' CPU or TPU, channels_last should be used. For GPU, channels_first' + ' will improve performance.')) +flags.DEFINE_integer( + 'num_label_classes', default=None, help='Number of classes, at least 2') +flags.DEFINE_float( + 'batch_norm_momentum', + default=None, + help=('Batch normalization layer momentum of moving average to override.')) +flags.DEFINE_float( + 'batch_norm_epsilon', + default=None, + help=('Batch normalization layer epsilon to override..')) + +flags.DEFINE_bool( + 'transpose_input', + default=None, + help='Use TPU double transpose optimization') + +flags.DEFINE_string( + 'export_dir', + default=None, + help=('The directory where the exported SavedModel will be stored.')) +flags.DEFINE_bool( + 'export_to_tpu', + default=False, + help=('Whether to export additional metagraph with "serve, tpu" tags' + ' in addition to "serve" only metagraph.')) +flags.DEFINE_bool( + 'post_quantize', default=True, help=('Enable post quantization.')) + +flags.DEFINE_bool( + 'quantized_training', + default=False, + help=('Enable quantized training as it is required for Edge TPU.' + 'This should be used for fine-tuning rather than pre-training.')) + +flags.DEFINE_integer( + 'quantization_delay_epochs', + default=0, + help=('The number of epochs after which weights and activations are' + ' quantized during training.')) + +flags.DEFINE_bool( + 'export_moving_average', + default=True, + help=('Replace variables with corresponding moving average variables in ' + 'saved model export.')) + +flags.DEFINE_string( + 'init_checkpoint', + default=None, + help=('Initial checkpoint from a pre-trained MnasNet model.')) + +flags.DEFINE_float( + 'base_learning_rate', + default=None, + help=('Base learning rate when train batch size is 256.')) + +flags.DEFINE_float( + 'momentum', + default=None, + help=('Momentum parameter used in the MomentumOptimizer.')) + +flags.DEFINE_float( + 'moving_average_decay', default=None, help=('Moving average decay rate.')) + +flags.DEFINE_float( + 'weight_decay', + default=None, + help=('Weight decay coefficiant for l2 regularization.')) + +flags.DEFINE_float( + 'label_smoothing', + default=None, + help=('Label smoothing parameter used in the softmax_cross_entropy')) + +flags.DEFINE_float( + 'dropout_rate', + default=None, + help=('Dropout rate for the final output layer.')) + +flags.DEFINE_integer( + 'log_step_count_steps', 64, 'The number of steps at ' + 'which the global step information is logged.') + +flags.DEFINE_bool( + 'add_summaries', + default=None, + help=('Whether to write training/eval summaries for visualization.')) + +flags.DEFINE_bool( + 'use_cache', default=None, help=('Enable cache for training input.')) + +flags.DEFINE_float( + 'depth_multiplier', default=None, help=('Depth multiplier per layer.')) + +flags.DEFINE_float( + 'depth_divisor', default=None, help=('Depth divisor (default to 8).')) + +flags.DEFINE_float( + 'min_depth', default=None, help=('Minimal depth (default to None).')) + +flags.DEFINE_bool( + 'use_async_checkpointing', default=None, help=('Enable async checkpoint')) + +flags.DEFINE_bool( + 'use_keras', + default=None, + help=('Whether to use tf.keras.layers to construct networks.')) + +# Learning rate schedule +LR_SCHEDULE = [ # (multiplier, epoch to start) tuples + (1.0, 5), (0.1, 30), (0.01, 60), (0.001, 80) +] + + +def get_pretrained_variables_to_restore(checkpoint_path, + load_moving_average=False): + """Gets veriables_to_restore mapping from pretrained checkpoint. + + Args: + checkpoint_path: String. Path of checkpoint. + load_moving_average: Boolean, whether load moving average variables to + replace variables. + + Returns: + Mapping of variables to restore. + """ + checkpoint_reader = tf.train.load_checkpoint(checkpoint_path) + variable_shape_map = checkpoint_reader.get_variable_to_shape_map() + + variables_to_restore = {} + ema_vars = mnas_utils.get_ema_vars() + for v in tf.global_variables(): + # Skip variables if they are in excluded scopes. + is_excluded = False + for scope in ['global_step', 'ExponentialMovingAverage']: + if scope in v.op.name: + is_excluded = True + break + if is_excluded: + tf.logging.info( + 'Exclude [%s] from loading from checkpoint.', v.op.name) + continue + variable_name_ckpt = v.op.name + if load_moving_average and v in ema_vars: + # To load moving average variables into non-moving version for + # fine-tuning, maps variables here manually. + variable_name_ckpt = v.op.name + '/ExponentialMovingAverage' + + if variable_name_ckpt not in variable_shape_map: + tf.logging.info( + 'Skip init [%s] from [%s] as it is not in the checkpoint', + v.op.name, variable_name_ckpt) + continue + + variables_to_restore[variable_name_ckpt] = v + tf.logging.info('Init variable [%s] from [%s] in ckpt', v.op.name, + variable_name_ckpt) + return variables_to_restore + + +def build_model_fn(features, labels, mode, params): + """The model_fn for MnasNet to be used with TPUEstimator. + + Args: + features: `Tensor` of batched images. + labels: `Tensor` of labels for the data samples + mode: one of `tf.estimator.ModeKeys.{TRAIN,EVAL,PREDICT}` + params: `dict` of parameters passed to the model from the TPUEstimator, + `params['batch_size']` is always provided and should be used as the + effective batch size. + + Returns: + A `TPUEstimatorSpec` for the model + """ + is_training = (mode == tf.estimator.ModeKeys.TRAIN) + # This is essential, if using a keras-derived model. + tf.keras.backend.set_learning_phase(is_training) + + if isinstance(features, dict): + features = features['feature'] + + if mode == tf.estimator.ModeKeys.PREDICT: + # Adds an identify node to help TFLite export. + features = tf.identity(features, 'float_image_input') + + # In most cases, the default data format NCHW instead of NHWC should be + # used for a significant performance boost on GPU. NHWC should be used + # only if the network needs to be run on CPU since the pooling operations + # are only supported on NHWC. TPU uses XLA compiler to figure out best layout. + if params['data_format'] == 'channels_first': + assert not params['transpose_input'] # channels_first only for GPU + features = tf.transpose(features, [0, 3, 1, 2]) + stats_shape = [3, 1, 1] + else: + stats_shape = [1, 1, 3] + + if params['transpose_input'] and mode != tf.estimator.ModeKeys.PREDICT: + features = tf.transpose(features, [3, 0, 1, 2]) # HWCN to NHWC + + # Normalize the image to zero mean and unit variance. + features -= tf.constant( + imagenet_input.MEAN_RGB, shape=stats_shape, dtype=features.dtype) + features /= tf.constant( + imagenet_input.STDDEV_RGB, shape=stats_shape, dtype=features.dtype) + + has_moving_average_decay = (params['moving_average_decay'] > 0) + + tf.logging.info('Using open-source implementation for MnasNet definition.') + override_params = {} + if params['batch_norm_momentum']: + override_params['batch_norm_momentum'] = params['batch_norm_momentum'] + if params['batch_norm_epsilon']: + override_params['batch_norm_epsilon'] = params['batch_norm_epsilon'] + if params['dropout_rate']: + override_params['dropout_rate'] = params['dropout_rate'] + if params['data_format']: + override_params['data_format'] = params['data_format'] + if params['num_label_classes']: + override_params['num_classes'] = params['num_label_classes'] + if params['depth_multiplier']: + override_params['depth_multiplier'] = params['depth_multiplier'] + if params['depth_divisor']: + override_params['depth_divisor'] = params['depth_divisor'] + if params['min_depth']: + override_params['min_depth'] = params['min_depth'] + override_params['use_keras'] = params['use_keras'] + + def _build_model(model_name): + """Build the model for a given model name.""" + if model_name.startswith('mnasnet'): + return mnasnet_models.build_mnasnet_model( + features, + model_name=model_name, + training=is_training, + override_params=override_params) + elif model_name.startswith('mixnet'): + return mixnet_builder.build_model( + features, + model_name=model_name, + training=is_training, + override_params=override_params) + else: + raise ValueError('Unknown model name {}'.format(model_name)) + + if params['precision'] == 'bfloat16': + with tf.tpu.bfloat16_scope(): + logits, _ = _build_model(params['model_name']) + logits = tf.cast(logits, tf.float32) + else: # params['precision'] == 'float32' + logits, _ = _build_model(params['model_name']) + + if params['quantized_training']: + try: + from tensorflow.contrib import quantize # pylint: disable=g-import-not-at-top + except ImportError as e: + logging.exception( + 'Quantized training is not supported in TensorFlow 2.x') + raise e + + if is_training: + tf.logging.info('Adding fake quantization ops for training.') + quantize.create_training_graph( + quant_delay=int(params['steps_per_epoch'] * + FLAGS.quantization_delay_epochs)) + else: + tf.logging.info('Adding fake quantization ops for evaluation.') + quantize.create_eval_graph() + + if mode == tf.estimator.ModeKeys.PREDICT: + scaffold_fn = None + if FLAGS.export_moving_average: + # If the model is trained with moving average decay, to match evaluation + # metrics, we need to export the model using moving average variables. + restore_checkpoint = tf.train.latest_checkpoint(FLAGS.model_dir) + variables_to_restore = get_pretrained_variables_to_restore( + restore_checkpoint, load_moving_average=True) + tf.logging.info('Restoring from the latest checkpoint: %s', + restore_checkpoint) + tf.logging.info(str(variables_to_restore)) + + saver = tf.train.Saver(variables_to_restore) + + predictions = { + 'classes': tf.argmax(logits, axis=1), + 'probabilities': tf.nn.softmax(logits, name='softmax_tensor') + } + return NPUEstimatorSpec( + mode=mode, + predictions=predictions, + export_outputs={ + 'classify': tf.estimator.export.PredictOutput(predictions) + }, + scaffold=tf.train.Scaffold(saver=saver)) + + # If necessary, in the model_fn, use params['batch_size'] instead the batch + # size flags (--train_batch_size or --eval_batch_size). + batch_size = params['train_batch_size'] # pylint: disable=unused-variable + + # Calculate loss, which includes softmax cross entropy and L2 regularization. + one_hot_labels = tf.one_hot(labels, params['num_label_classes']) + cross_entropy = tf.losses.softmax_cross_entropy( + logits=logits, + onehot_labels=one_hot_labels, + label_smoothing=params['label_smoothing']) + + # Add weight decay to the loss for non-batch-normalization variables. + loss = cross_entropy + params['weight_decay'] * tf.add_n([ + tf.nn.l2_loss(v) + for v in tf.trainable_variables() + if 'batch_normalization' not in v.name + ]) + + global_step = tf.train.get_global_step() + if has_moving_average_decay: + ema = tf.train.ExponentialMovingAverage( + decay=params['moving_average_decay'], num_updates=global_step) + ema_vars = mnas_utils.get_ema_vars() + + host_call = None + if is_training: + # Compute the current epoch and associated learning rate from global_step. + current_epoch = ( + tf.cast(global_step, tf.float32) / params['steps_per_epoch']) + + scaled_lr = params['base_learning_rate'] * \ + (params['train_batch_size'] / + 256.0) # pylint: disable=line-too-long + learning_rate = mnas_utils.build_learning_rate(scaled_lr, global_step, + params['steps_per_epoch']) + optimizer = mnas_utils.build_optimizer(learning_rate) + if params['use_tpu']: + # When using TPU, wrap the optimizer with CrossShardOptimizer which + # handles synchronization details between different TPU cores. To the + # user, this should look like regular synchronous training. + optimizer = tf.tpu.CrossShardOptimizer(optimizer) + + if params['add_summaries']: + summary_writer = tf2.summary.create_file_writer( + FLAGS.model_dir, max_queue=params['iterations_per_loop']) + with summary_writer.as_default(): + should_record = tf.equal(global_step % params['iterations_per_loop'], + 0) + with tf2.summary.record_if(should_record): + tf2.summary.scalar('loss', loss, step=global_step) + tf2.summary.scalar( + 'learning_rate', learning_rate, step=global_step) + tf2.summary.scalar( + 'current_epoch', current_epoch, step=global_step) + + # Batch normalization requires UPDATE_OPS to be added as a dependency to + # the train operation. + update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) + + with tf.control_dependencies(update_ops + tf.summary.all_v2_summary_ops()): + train_op = optimizer.minimize(loss, global_step) + + if has_moving_average_decay: + with tf.control_dependencies([train_op]): + train_op = ema.apply(ema_vars) + + else: + train_op = None + + eval_metrics = None + if mode == tf.estimator.ModeKeys.EVAL: + + def metric_fn(labels, logits): + """Evaluation metric function. + + Evaluates accuracy. + + This function is executed on the CPU and should not directly reference + any Tensors in the rest of the `model_fn`. To pass Tensors from the model + to the `metric_fn`, provide as part of the `eval_metrics`. See + https://www.tensorflow.org/api_docs/python/tf/estimator/tpu/TPUEstimatorSpec + for more information. + + Arguments should match the list of `Tensor` objects passed as the second + element in the tuple passed to `eval_metrics`. + + Args: + labels: `Tensor` with shape `[batch]`. + logits: `Tensor` with shape `[batch, num_classes]`. + + Returns: + A dict of the metrics to return from evaluation. + """ + predictions = tf.argmax(logits, axis=1) + top_1_accuracy = tf.metrics.accuracy(labels, predictions) + in_top_5 = tf.cast(tf.nn.in_top_k(logits, labels, 5), tf.float32) + top_5_accuracy = tf.metrics.mean(in_top_5) + + return { + 'top_1_accuracy': top_1_accuracy, + 'top_5_accuracy': top_5_accuracy, + } + + #eval_metrics = (metric_fn, [labels, logits]) + eval_metrics = metric_fn(labels, logits) + + num_params = np.sum([np.prod(v.shape) for v in tf.trainable_variables()]) + tf.logging.info('number of trainable parameters: {}'.format(num_params)) + + # Prepares scaffold_fn if needed. + scaffold_fn = None + if is_training and FLAGS.init_checkpoint: + variables_to_restore = get_pretrained_variables_to_restore( + FLAGS.init_checkpoint, has_moving_average_decay) + tf.logging.info('Initializing from pretrained checkpoint: %s', + FLAGS.init_checkpoint) + if FLAGS.use_tpu: + + def init_scaffold(): + tf.train.init_from_checkpoint(FLAGS.init_checkpoint, + variables_to_restore) + return tf.train.Scaffold() + + scaffold_fn = init_scaffold + else: + tf.train.init_from_checkpoint( + FLAGS.init_checkpoint, variables_to_restore) + + restore_vars_dict = None + if not is_training and has_moving_average_decay: + # Load moving average variables for eval. + restore_vars_dict = ema.variables_to_restore(ema_vars) + + saver = tf.train.Saver(restore_vars_dict) + + return NPUEstimatorSpec( + mode=mode, + loss=loss, + train_op=train_op, + eval_metric_ops=eval_metrics, + scaffold=tf.train.Scaffold(saver=saver)) + + +def _verify_non_empty_string(value, field_name): + """Ensures that a given proposed field value is a non-empty string. + + Args: + value: proposed value for the field. + field_name: string name of the field, e.g. `project`. + + Returns: + The given value, provided that it passed the checks. + + Raises: + ValueError: the value is not a string, or is a blank string. + """ + if not isinstance(value, str): + raise ValueError( + 'Bigtable parameter "%s" must be a string.' % field_name) + if not value: + raise ValueError( + 'Bigtable parameter "%s" must be non-empty.' % field_name) + return value + + +def _select_tables_from_flags(): + """Construct training and evaluation Bigtable selections from flags. + + Returns: + [training_selection, evaluation_selection] + """ + project = _verify_non_empty_string( + FLAGS.bigtable_project or FLAGS.gcp_project, 'project') + instance = _verify_non_empty_string(FLAGS.bigtable_instance, 'instance') + table = _verify_non_empty_string(FLAGS.bigtable_table, 'table') + train_prefix = _verify_non_empty_string(FLAGS.bigtable_train_prefix, + 'train_prefix') + eval_prefix = _verify_non_empty_string(FLAGS.bigtable_eval_prefix, + 'eval_prefix') + column_family = _verify_non_empty_string(FLAGS.bigtable_column_family, + 'column_family') + column_qualifier = _verify_non_empty_string(FLAGS.bigtable_column_qualifier, + 'column_qualifier') + return [ + imagenet_input.BigtableSelection( + project=project, + instance=instance, + table=table, + prefix=p, + column_family=column_family, + column_qualifier=column_qualifier) + for p in (train_prefix, eval_prefix) + ] + + +def export(est, export_dir, params, post_quantize=True): + """Export graph to SavedModel and TensorFlow Lite. + + Args: + est: estimator instance. + export_dir: string, exporting directory. + params: `ParamsDict` passed to the model from the TPUEstimator. + post_quantize: boolean, whether to quantize model checkpoint after training. + + Raises: + ValueError: the export directory path is not specified. + """ + if not export_dir: + raise ValueError('The export directory path is not specified.') + # The guide to serve a exported TensorFlow model is at: + # https://www.tensorflow.org/serving/serving_basic + image_serving_input_fn = imagenet_input.build_image_serving_input_fn( + params.input_image_size) + tf.logging.info('Starting to export model.') + subfolder = est.export_saved_model( + export_dir_base=export_dir, + serving_input_receiver_fn=image_serving_input_fn) + + tf.logging.info('Starting to export TFLite.') + converter = tf.lite.TFLiteConverter.from_saved_model( + subfolder, input_arrays=['truediv'], output_arrays=['logits']) + if params.quantized_training: + # Export quantized tflite if it is trained with quantized ops. + converter.inference_type = tf.uint8 + converter.quantized_input_stats = {'truediv': (0., 2.)} + tflite_model = converter.convert() + tflite_file = os.path.join(export_dir, params.model_name + '.tflite') + tf.gfile.GFile(tflite_file, 'wb').write(tflite_model) + + if post_quantize: + tf.logging.info('Starting to export quantized TFLite.') + converter = tf.lite.TFLiteConverter.from_saved_model( + subfolder, input_arrays=['truediv'], output_arrays=['logits']) + converter.post_training_quantize = True + quant_tflite_model = converter.convert() + quant_tflite_file = os.path.join(export_dir, + params.model_name + '_postquant.tflite') + tf.gfile.GFile(quant_tflite_file, 'wb').write(quant_tflite_model) + + +def main(unused_argv): + params = params_dict.ParamsDict( + mnasnet_config.MNASNET_CFG, mnasnet_config.MNASNET_RESTRICTIONS) + params = params_dict.override_params_dict( + params, FLAGS.config_file, is_strict=True) + params = params_dict.override_params_dict( + params, FLAGS.params_override, is_strict=True) + + params = flags_to_params.override_params_from_input_flags(params, FLAGS) + + additional_params = { + 'steps_per_epoch': params.num_train_images / params.train_batch_size, + 'quantized_training': FLAGS.quantized_training, + 'add_summaries': FLAGS.add_summaries, + } + + params = params_dict.override_params_dict( + params, additional_params, is_strict=False) + + params.validate() + params.lock() + + if FLAGS.tpu or params.use_tpu: + tpu_cluster_resolver = tf.distribute.cluster_resolver.TPUClusterResolver( + FLAGS.tpu, zone=FLAGS.tpu_zone, project=FLAGS.gcp_project) + else: + tpu_cluster_resolver = None + + if params.use_async_checkpointing: + save_checkpoints_steps = None + else: + save_checkpoints_steps = max(100, params.iterations_per_loop) + + # Enables automatic outside compilation. Required in order to + # automatically detect summary ops to run on CPU instead of TPU. + tf.config.set_soft_device_placement(True) + + from tensorflow.core.protobuf.rewriter_config_pb2 import RewriterConfig + config = tf.ConfigProto() + custom_op = config.graph_options.rewrite_options.custom_optimizers.add() + custom_op.name = "NpuOptimizer" + + # Set the precision_mode:allow_mix_precision + custom_op.parameter_map['precision_mode'].s = tf.compat.as_bytes( + 'allow_mix_precision') + custom_op.parameter_map["use_off_line"].b = True + + # Set the dump path. + os.mkdir(FLAGS.dump_dir) + custom_op.parameter_map['dump_path'].s = tf.compat.as_bytes(FLAGS.dump_dir) + # Set the dump debug. + custom_op.parameter_map['enable_dump_debug'].b = True + custom_op.parameter_map['dump_debug_mode'].s = tf.compat.as_bytes('all') + + config.graph_options.rewrite_options.remapping = RewriterConfig.OFF # Must be set OFF. + config.graph_options.rewrite_options.memory_optimization = RewriterConfig.OFF # Must be set OFF. + + """ + # Set the profiling_config. + os.mkdir(FLAGS.profiling_dir) + profiling_options = '{"output":"%s","task_trace":"on"}' % FLAGS.profiling_dir + profiling_config = ProfilingConfig( + enable_profiling=True, profiling_options=profiling_options) + """ + runconfig = NPURunConfig( + # profiling_config=profiling_config, + model_dir=FLAGS.model_dir, + save_checkpoints_steps=save_checkpoints_steps, + log_step_count_steps=FLAGS.log_step_count_steps, + session_config=config) # pylint: disable=line-too-long + + # Validates Flags. + if params.precision == 'bfloat16' and params.use_keras: + raise ValueError( + 'Keras layers do not have full support to bfloat16 activation training.' + ' You have set precision as %s and use_keras as %s' % + (params.precision, params.use_keras)) + + # Initializes model parameters. + mnasnet_est = NPUEstimator( + model_fn=build_model_fn, + config=runconfig, + model_dir=FLAGS.model_dir, + params=params.as_dict()) + + if FLAGS.mode == 'export_only': + export(mnasnet_est, FLAGS.export_dir, params, FLAGS.post_quantize) + return + + # Input pipelines are slightly different (with regards to shuffling and + # preprocessing) between training and evaluation. + if FLAGS.bigtable_instance: + tf.logging.info('Using Bigtable dataset, table %s', + FLAGS.bigtable_table) + select_train, select_eval = _select_tables_from_flags() + imagenet_train, imagenet_eval = [imagenet_input.ImageNetBigtableInput( + is_training=is_training, + use_bfloat16=False, + transpose_input=params.transpose_input, + selection=selection) for (is_training, selection) in + [(True, select_train), + (False, select_eval)]] + else: + if FLAGS.data_dir == FAKE_DATA_DIR: + tf.logging.info('Using fake dataset.') + else: + tf.logging.info('Using dataset: %s', FLAGS.data_dir) + imagenet_train, imagenet_eval = [ + imagenet_input.ImageNetInput( + is_training=is_training, + data_dir=FLAGS.data_dir, + transpose_input=params.transpose_input, + cache=params.use_cache and is_training, + image_size=params.input_image_size, + num_parallel_calls=params.num_parallel_calls, + use_bfloat16=(params.precision == 'bfloat16')) for is_training in [True, False] + ] + + if FLAGS.mode == 'eval': + eval_steps = params.num_eval_images // params.eval_batch_size + # Run evaluation when there's a new checkpoint + for ckpt in tf.train.checkpoints_iterator( + FLAGS.model_dir, timeout=FLAGS.eval_timeout): + tf.logging.info('Starting to evaluate.') + try: + start_timestamp = time.time() # This time will include compilation time + eval_results = mnasnet_est.evaluate( + input_fn=imagenet_eval.input_fn, + steps=eval_steps, + checkpoint_path=ckpt) + elapsed_time = int(time.time() - start_timestamp) + tf.logging.info('Eval results: %s. Elapsed seconds: %d', eval_results, + elapsed_time) + mnas_utils.archive_ckpt( + eval_results, eval_results['top_1_accuracy'], ckpt) + + # Terminate eval job when final checkpoint is reached + current_step = int(os.path.basename(ckpt).split('-')[1]) + if current_step >= params.train_steps: + tf.logging.info('Evaluation finished after training step %d', + current_step) + break + + except tf.errors.NotFoundError: + # Since the coordinator is on a different job than the TPU worker, + # sometimes the TPU worker does not finish initializing until long after + # the CPU job tells it to start evaluating. In this case, the checkpoint + # file could have been deleted already. + tf.logging.info('Checkpoint %s no longer exists, skipping checkpoint', + ckpt) + + if FLAGS.export_dir: + export(mnasnet_est, FLAGS.export_dir, params, FLAGS.post_quantize) + else: # FLAGS.mode == 'train' or FLAGS.mode == 'train_and_eval' + try: + current_step = tf.train.load_variable(FLAGS.model_dir, + tf.GraphKeys.GLOBAL_STEP) + except (TypeError, ValueError, tf.errors.NotFoundError): + current_step = 0 + + tf.logging.info( + 'Training for %d steps (%.2f epochs in total). Current' + ' step %d.', params.train_steps, + params.train_steps / params.steps_per_epoch, current_step) + + start_timestamp = time.time() # This time will include compilation time + + if FLAGS.mode == 'train': + hooks = [] + if params.use_async_checkpointing: + try: + from tensorflow.contrib.tpu.python.tpu import async_checkpoint # pylint: disable=g-import-not-at-top + except ImportError as e: + logging.exception( + 'Async checkpointing is not supported in TensorFlow 2.x') + raise e + + hooks.append( + async_checkpoint.AsyncCheckpointSaverHook( + checkpoint_dir=FLAGS.model_dir, + save_steps=max(100, params.iterations_per_loop))) + mnasnet_est.train( + input_fn=imagenet_train.input_fn, + max_steps=params.train_steps, + hooks=hooks) + + else: + assert FLAGS.mode == 'train_and_eval' + while current_step < params.train_steps: + # Train for up to steps_per_eval number of steps. + # At the end of training, a checkpoint will be written to --model_dir. + next_checkpoint = min(current_step + FLAGS.steps_per_eval, + params.train_steps) + mnasnet_est.train( + input_fn=imagenet_train.input_fn, max_steps=next_checkpoint) + current_step = next_checkpoint + + tf.logging.info('Finished training up to step %d. Elapsed seconds %d.', + next_checkpoint, int(time.time() - start_timestamp)) + + # Evaluate the model on the most recent model in --model_dir. + # Since evaluation happens in batches of --eval_batch_size, some images + # may be excluded modulo the batch size. As long as the batch size is + # consistent, the evaluated images are also consistent. + tf.logging.info('Starting to evaluate.') + eval_results = mnasnet_est.evaluate( + input_fn=imagenet_eval.input_fn, + steps=params.num_eval_images // params.eval_batch_size) + tf.logging.info('Eval results at step %d: %s', next_checkpoint, + eval_results) + ckpt = tf.train.latest_checkpoint(FLAGS.model_dir) + mnas_utils.archive_ckpt( + eval_results, eval_results['top_1_accuracy'], ckpt) + + elapsed_time = int(time.time() - start_timestamp) + tf.logging.info('Finished training up to step %d. Elapsed seconds %d.', + params.train_steps, elapsed_time) + if FLAGS.export_dir: + export(mnasnet_est, FLAGS.export_dir, + params, FLAGS.post_quantize) + + from help_modelarts import modelarts_result2obs + modelarts_result2obs(FLAGS) + + +if __name__ == '__main__': + tf.logging.set_verbosity(tf.logging.INFO) + tf.disable_v2_behavior() + + flags.mark_flag_as_required("data_dir") + flags.mark_flag_as_required("model_dir") + flags.mark_flag_as_required("obs_dir") + flags.mark_flag_as_required("model_name") + + app.run(main) diff --git a/TensorFlow/contrib/cv/Mixnet_ID2072_for_Tensorflow/mnasnet_model.py b/TensorFlow/contrib/cv/Mixnet_ID2072_for_Tensorflow/mnasnet_model.py new file mode 100644 index 0000000000000000000000000000000000000000..6d51d0aa502f9088f67f06c1061d681695a6c186 --- /dev/null +++ b/TensorFlow/contrib/cv/Mixnet_ID2072_for_Tensorflow/mnasnet_model.py @@ -0,0 +1,490 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Contains definitions for MnesNet model. + +[1] Mingxing Tan, Bo Chen, Ruoming Pang, Vijay Vasudevan, Quoc V. Le + MnasNet: Platform-Aware Neural Architecture Search for Mobile. + arXiv:1807.11626 +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import collections +import numpy as np +import six +from six.moves import xrange # pylint: disable=redefined-builtin +import tensorflow.compat.v1 as tf + +import mnas_utils + +GlobalParams = collections.namedtuple('GlobalParams', [ + 'batch_norm_momentum', 'batch_norm_epsilon', 'dropout_rate', 'data_format', + 'num_classes', 'depth_multiplier', 'depth_divisor', 'min_depth', + 'stem_size', 'use_keras' +]) +GlobalParams.__new__.__defaults__ = (None,) * len(GlobalParams._fields) + +# todo(hongkuny): Consider rewrite an argument class with encoding/decoding. +BlockArgs = collections.namedtuple('BlockArgs', [ + 'kernel_size', 'num_repeat', 'input_filters', 'output_filters', + 'expand_ratio', 'id_skip', 'strides', 'se_ratio' +]) +# defaults will be a public argument for namedtuple in Python 3.7 +# https://docs.python.org/3/library/collections.html#collections.namedtuple +BlockArgs.__new__.__defaults__ = (None,) * len(BlockArgs._fields) + + +def conv_kernel_initializer(shape, dtype=None, partition_info=None): + """Initialization for convolutional kernels. + + The main difference with tf.variance_scaling_initializer is that + tf.variance_scaling_initializer uses a truncated normal with an uncorrected + standard deviation, whereas here we use a normal distribution. Similarly, + tf.contrib.layers.variance_scaling_initializer uses a truncated normal with + a corrected standard deviation. + + Args: + shape: shape of variable + dtype: dtype of variable + partition_info: unused + + Returns: + an initialization for the variable + """ + del partition_info + kernel_height, kernel_width, _, out_filters = shape + fan_out = int(kernel_height * kernel_width * out_filters) + return tf.random_normal( + shape, mean=0.0, stddev=np.sqrt(2.0 / fan_out), dtype=dtype) + + +def dense_kernel_initializer(shape, dtype=None, partition_info=None): + """Initialization for dense kernels. + + This initialization is equal to + tf.variance_scaling_initializer(scale=1.0/3.0, mode='fan_out', + distribution='uniform'). + It is written out explicitly here for clarity. + + Args: + shape: shape of variable + dtype: dtype of variable + partition_info: unused + + Returns: + an initialization for the variable + """ + del partition_info + init_range = 1.0 / np.sqrt(shape[1]) + return tf.random_uniform(shape, -init_range, init_range, dtype=dtype) + + +def round_filters(filters, global_params): + """Round number of filters based on depth multiplier.""" + multiplier = global_params.depth_multiplier + divisor = global_params.depth_divisor + min_depth = global_params.min_depth + if not multiplier: + return filters + + filters *= multiplier + min_depth = min_depth or divisor + new_filters = max(min_depth, int( + filters + divisor / 2) // divisor * divisor) + # Make sure that round down does not go down by more than 10%. + if new_filters < 0.9 * filters: + new_filters += divisor + return new_filters + + +def _get_conv2d(filters, + kernel_size, + strides, + kernel_initializer, + padding, + use_bias, + data_format='channels_last', + use_keras=True): + """A helper function to create Conv2D layer.""" + if use_keras: + return tf.keras.layers.Conv2D( + filters=filters, + kernel_size=kernel_size, + strides=strides, + kernel_initializer=kernel_initializer, + padding=padding, + data_format=data_format, + use_bias=use_bias) + else: + return tf.layers.Conv2D( + filters=filters, + kernel_size=kernel_size, + strides=strides, + kernel_initializer=kernel_initializer, + padding=padding, + data_format=data_format, + use_bias=use_bias) + + +class MnasBlock(object): + """A class of MnasNet Inveretd Residual Bottleneck. + + Attributes: + has_se: boolean. Whether the block contains a Squeeze and Excitation layer + inside. + endpoints: dict. A list of internal tensors. + """ + + def __init__(self, block_args, global_params): + """Initializes a MnasNet block. + + Args: + block_args: BlockArgs, arguments to create a MnasBlock. + global_params: GlobalParams, a set of global parameters. + """ + self._block_args = block_args + self._batch_norm_momentum = global_params.batch_norm_momentum + self._batch_norm_epsilon = global_params.batch_norm_epsilon + self._use_keras = global_params.use_keras + self._data_format = global_params.data_format + if self._data_format == 'channels_first': + self._channel_axis = 1 + self._spatial_dims = [2, 3] + else: + self._channel_axis = -1 + self._spatial_dims = [1, 2] + self.has_se = (self._block_args.se_ratio is not None) and ( + self._block_args.se_ratio > 0) and (self._block_args.se_ratio <= 1) + + self.endpoints = None + + # Builds the block accordings to arguments. + self._build() + + def block_args(self): + """Call the function of block_args.""" + return self._block_args + + def _build(self): + """Builds MnasNet block according to the arguments.""" + filters = self._block_args.input_filters * self._block_args.expand_ratio + if self._block_args.expand_ratio != 1: + # Expansion phase: + self._expand_conv = _get_conv2d( + filters=filters, + kernel_size=[1, 1], + strides=[1, 1], + kernel_initializer=conv_kernel_initializer, + padding='same', + use_bias=False, + data_format=self._data_format, + use_keras=self._use_keras) + # todo(hongkuny): b/120622234 need to manage update ops directly. + self._bn0 = tf.layers.BatchNormalization( + axis=self._channel_axis, + momentum=self._batch_norm_momentum, + epsilon=self._batch_norm_epsilon, + fused=True) + + kernel_size = self._block_args.kernel_size + # Depth-wise convolution phase: + if self._use_keras: + self._depthwise_conv = tf.keras.layers.DepthwiseConv2D( + [kernel_size, kernel_size], + strides=self._block_args.strides, + depthwise_initializer=conv_kernel_initializer, + padding='same', + data_format=self._data_format, + use_bias=False) + else: + self._depthwise_conv = mnas_utils.DepthwiseConv2D( + [kernel_size, kernel_size], + strides=self._block_args.strides, + depthwise_initializer=conv_kernel_initializer, + padding='same', + data_format=self._data_format, + use_bias=False) + self._bn1 = tf.layers.BatchNormalization( + axis=self._channel_axis, + momentum=self._batch_norm_momentum, + epsilon=self._batch_norm_epsilon, + fused=True) + + if self.has_se: + num_reduced_filters = max( + 1, int(self._block_args.input_filters * self._block_args.se_ratio)) + # Squeeze and Excitation layer. + self._se_reduce = _get_conv2d( + num_reduced_filters, + kernel_size=[1, 1], + strides=[1, 1], + kernel_initializer=conv_kernel_initializer, + padding='same', + use_bias=True, + data_format=self._data_format, + use_keras=self._use_keras) + self._se_expand = _get_conv2d( + filters, + kernel_size=[1, 1], + strides=[1, 1], + kernel_initializer=conv_kernel_initializer, + padding='same', + use_bias=True, + data_format=self._data_format, + use_keras=self._use_keras) + + # Output phase: + filters = self._block_args.output_filters + self._project_conv = _get_conv2d( + filters, + kernel_size=[1, 1], + strides=[1, 1], + kernel_initializer=conv_kernel_initializer, + padding='same', + use_bias=False, + data_format=self._data_format, + use_keras=self._use_keras) + self._bn2 = tf.layers.BatchNormalization( + axis=self._channel_axis, + momentum=self._batch_norm_momentum, + epsilon=self._batch_norm_epsilon, + fused=True) + + def _call_se(self, input_tensor): + """Call Squeeze and Excitation layer. + + Args: + input_tensor: Tensor, a single input tensor for Squeeze/Excitation layer. + + Returns: + A output tensor, which should have the same shape as input. + """ + se_tensor = tf.reduce_mean( + input_tensor, self._spatial_dims, keepdims=True) + se_tensor = self._se_expand(tf.nn.relu(self._se_reduce(se_tensor))) + tf.logging.info('Built Squeeze and Excitation with tensor shape: %s' % + (se_tensor.shape)) + return tf.sigmoid(se_tensor) * input_tensor + + def call(self, inputs, training=True): + """Implementation of MnasBlock call(). + + Args: + inputs: the inputs tensor. + training: boolean, whether the model is constructed for training. + + Returns: + A output tensor. + """ + tf.logging.info('Block input: %s shape: %s' % + (inputs.name, inputs.shape)) + if self._block_args.expand_ratio != 1: + x = tf.nn.relu( + self._bn0(self._expand_conv(inputs), training=training)) + else: + x = inputs + tf.logging.info('Expand: %s shape: %s' % (x.name, x.shape)) + + x = tf.nn.relu(self._bn1(self._depthwise_conv(x), training=training)) + tf.logging.info('DWConv: %s shape: %s' % (x.name, x.shape)) + + if self.has_se: + with tf.variable_scope('se'): + x = self._call_se(x) + + self.endpoints = {'expansion_output': x} + + x = self._bn2(self._project_conv(x), training=training) + if self._block_args.id_skip: + if all( + s == 1 for s in self._block_args.strides + ) and self._block_args.input_filters == self._block_args.output_filters: + x = tf.add(x, inputs) + tf.logging.info('Project: %s shape: %s' % (x.name, x.shape)) + return tf.identity(x) + + +class MnasNetModel(tf.keras.Model): + """A class implements tf.keras.Model for MnesNet model. + + Reference: https://arxiv.org/abs/1807.11626 + """ + + def __init__(self, blocks_args=None, global_params=None): + """Initializes an `MnasNetModel` instance. + + Args: + blocks_args: A list of BlockArgs to construct MnasNet block modules. + global_params: GlobalParams, a set of global parameters. + + Raises: + ValueError: when blocks_args is not specified as a list. + """ + super(MnasNetModel, self).__init__() + if not isinstance(blocks_args, list): + raise ValueError('blocks_args should be a list.') + self._global_params = global_params + self._blocks_args = blocks_args + self.endpoints = None + self._build() + + def _build(self): + """Builds a MnasNet model.""" + self._blocks = [] + # Builds blocks. + for block_args in self._blocks_args: + assert block_args.num_repeat > 0 + # Update block input and output filters based on depth multiplier. + block_args = block_args._replace( + input_filters=round_filters(block_args.input_filters, + self._global_params), + output_filters=round_filters(block_args.output_filters, + self._global_params)) + + # The first block needs to take care of stride and filter size increase. + self._blocks.append(MnasBlock(block_args, self._global_params)) + if block_args.num_repeat > 1: + # pylint: disable=protected-access + block_args = block_args._replace( + input_filters=block_args.output_filters, strides=[1, 1]) + # pylint: enable=protected-access + for _ in xrange(block_args.num_repeat - 1): + self._blocks.append(MnasBlock(block_args, self._global_params)) + + batch_norm_momentum = self._global_params.batch_norm_momentum + batch_norm_epsilon = self._global_params.batch_norm_epsilon + if self._global_params.data_format == 'channels_first': + channel_axis = 1 + else: + channel_axis = -1 + + # Stem part. + stem_size = self._global_params.stem_size + self._conv_stem = _get_conv2d( + filters=round_filters(stem_size, self._global_params), + kernel_size=[3, 3], + strides=[2, 2], + kernel_initializer=conv_kernel_initializer, + padding='same', + use_bias=False, + data_format=self._global_params.data_format, + use_keras=self._global_params.use_keras) + self._bn0 = tf.layers.BatchNormalization( + axis=channel_axis, + momentum=batch_norm_momentum, + epsilon=batch_norm_epsilon, + fused=True) + + # Head part. + self._conv_head = _get_conv2d( + filters=1280, + kernel_size=[1, 1], + strides=[1, 1], + kernel_initializer=conv_kernel_initializer, + padding='same', + use_bias=False, + data_format=self._global_params.data_format, + use_keras=self._global_params.use_keras) + self._bn1 = tf.layers.BatchNormalization( + axis=channel_axis, + momentum=batch_norm_momentum, + epsilon=batch_norm_epsilon, + fused=True) + + self._avg_pooling = tf.keras.layers.GlobalAveragePooling2D( + data_format=self._global_params.data_format) + if self._global_params.use_keras: + self._fc = tf.keras.layers.Dense( + self._global_params.num_classes, + kernel_initializer=dense_kernel_initializer) + else: + self._fc = tf.layers.Dense( + self._global_params.num_classes, + kernel_initializer=dense_kernel_initializer) + if self._global_params.dropout_rate > 0: + self._dropout = tf.keras.layers.Dropout( + self._global_params.dropout_rate) + else: + self._dropout = None + + def call(self, inputs, training=True, features_only=None): + """Implementation of MnasNetModel call(). + + Args: + inputs: input tensors. + training: boolean, whether the model is constructed for training. + features_only: build the base feature network only. + + Returns: + output tensors. + """ + outputs = None + self.endpoints = {} + # Calls Stem layers + with tf.variable_scope('mnas_stem'): + outputs = tf.nn.relu( + self._bn0(self._conv_stem(inputs), training=training)) + tf.logging.info('Built stem layers with output shape: %s' % + outputs.shape) + self.endpoints['stem'] = outputs + + # Calls blocks. + reduction_idx = 0 + for idx, block in enumerate(self._blocks): + is_reduction = False + if ((idx == len(self._blocks) - 1) or + self._blocks[idx + 1].block_args().strides[0] > 1): + is_reduction = True + reduction_idx += 1 + + with tf.variable_scope('mnas_blocks_%s' % idx): + outputs = block.call(outputs, training=training) + self.endpoints['block_%s' % idx] = outputs + if is_reduction: + self.endpoints['reduction_%s' % reduction_idx] = outputs + if block.endpoints: + for k, v in six.iteritems(block.endpoints): + self.endpoints['block_%s/%s' % (idx, k)] = v + if is_reduction: + self.endpoints['reduction_%s/%s' % + (reduction_idx, k)] = v + self.endpoints['global_pool'] = outputs + + if not features_only: + # Calls final layers and returns logits. + with tf.variable_scope('mnas_head'): + outputs = tf.nn.relu( + self._bn1(self._conv_head(outputs), training=training)) + outputs = self._avg_pooling(outputs) + if self._dropout: + outputs = self._dropout(outputs, training=training) + outputs = self._fc(outputs) + self.endpoints['head'] = outputs + return outputs diff --git a/TensorFlow/contrib/cv/Mixnet_ID2072_for_Tensorflow/mnasnet_models.py b/TensorFlow/contrib/cv/Mixnet_ID2072_for_Tensorflow/mnasnet_models.py new file mode 100644 index 0000000000000000000000000000000000000000..ea4150820fe506d1c8033f71e50a686672c98dfc --- /dev/null +++ b/TensorFlow/contrib/cv/Mixnet_ID2072_for_Tensorflow/mnasnet_models.py @@ -0,0 +1,358 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Predefined MnasNet models.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import re +import tensorflow.compat.v1 as tf + +import mnasnet_model + + +class MnasNetDecoder(object): + """A class of MnasNet decoder to get model configuration.""" + + def _decode_block_string(self, block_string): + """Gets a MNasNet block through a string notation of arguments. + + E.g. r2_k3_s2_e1_i32_o16_se0.25_noskip: r - number of repeat blocks, + k - kernel size, s - strides (1-9), e - expansion ratio, i - input filters, + o - output filters, se - squeeze/excitation ratio + + Args: + block_string: a string, a string representation of block arguments. + + Returns: + A BlockArgs instance. + Raises: + ValueError: if the strides option is not correctly specified. + """ + assert isinstance(block_string, str) + ops = block_string.split('_') + options = {} + for op in ops: + splits = re.split(r'(\d.*)', op) + if len(splits) >= 2: + key, value = splits[:2] + options[key] = value + + if 's' not in options or len(options['s']) != 2: + raise ValueError('Strides options should be a pair of integers.') + + return mnasnet_model.BlockArgs( + kernel_size=int(options['k']), + num_repeat=int(options['r']), + input_filters=int(options['i']), + output_filters=int(options['o']), + expand_ratio=int(options['e']), + id_skip=('noskip' not in block_string), + se_ratio=float(options['se']) if 'se' in options else None, + strides=[int(options['s'][0]), int(options['s'][1])]) + + def _encode_block_string(self, block): + """Encodes a MnasNet block to a string.""" + args = [ + 'r%d' % block.num_repeat, + 'k%d' % block.kernel_size, + 's%d%d' % (block.strides[0], block.strides[1]), + 'e%s' % block.expand_ratio, + 'i%d' % block.input_filters, + 'o%d' % block.output_filters + ] + if (block.se_ratio is not None and block.se_ratio > 0 and + block.se_ratio <= 1): + args.append('se%s' % block.se_ratio) + if block.id_skip is False: + args.append('noskip') + return '_'.join(args) + + def decode(self, string_list): + """Decodes a list of string notations to specify blocks inside the network. + + Args: + string_list: a list of strings, each string is a notation of MnasNet + block. + + Returns: + A list of namedtuples to represent MnasNet blocks arguments. + """ + assert isinstance(string_list, list) + blocks_args = [] + for block_string in string_list: + blocks_args.append(self._decode_block_string(block_string)) + return blocks_args + + def encode(self, blocks_args): + """Encodes a list of MnasNet Blocks to a list of strings. + + Args: + blocks_args: A list of namedtuples to represent MnasNet blocks arguments. + Returns: + a list of strings, each string is a notation of MnasNet block. + """ + block_strings = [] + for block in blocks_args: + block_strings.append(self._encode_block_string(block)) + return block_strings + + +def mnasnet_b1(depth_multiplier=None): + """Creates a mnasnet-b1 model. + + Args: + depth_multiplier: multiplier to number of filters per layer. + + Returns: + blocks_args: a list of BlocksArgs for internal MnasNet blocks. + global_params: GlobalParams, global parameters for the model. + """ + blocks_args = [ + 'r1_k3_s11_e1_i32_o16_noskip', 'r3_k3_s22_e3_i16_o24', + 'r3_k5_s22_e3_i24_o40', 'r3_k5_s22_e6_i40_o80', 'r2_k3_s11_e6_i80_o96', + 'r4_k5_s22_e6_i96_o192', 'r1_k3_s11_e6_i192_o320_noskip' + ] + decoder = MnasNetDecoder() + global_params = mnasnet_model.GlobalParams( + batch_norm_momentum=0.99, + batch_norm_epsilon=1e-3, + dropout_rate=0.2, + data_format='channels_last', + num_classes=1000, + depth_multiplier=depth_multiplier, + depth_divisor=8, + min_depth=None, + stem_size=32, + use_keras=True) + return decoder.decode(blocks_args), global_params + + +def mnasnet_a1(depth_multiplier=None): + """Creates a mnasnet-a1 model. + + Args: + depth_multiplier: multiplier to number of filters per layer. + + Returns: + blocks_args: a list of BlocksArgs for internal MnasNet blocks. + global_params: GlobalParams, global parameters for the model. + """ + blocks_args = [ + 'r1_k3_s11_e1_i32_o16_noskip', 'r2_k3_s22_e6_i16_o24', + 'r3_k5_s22_e3_i24_o40_se0.25', 'r4_k3_s22_e6_i40_o80', + 'r2_k3_s11_e6_i80_o112_se0.25', 'r3_k5_s22_e6_i112_o160_se0.25', + 'r1_k3_s11_e6_i160_o320' + ] + global_params = mnasnet_model.GlobalParams( + batch_norm_momentum=0.99, + batch_norm_epsilon=1e-3, + dropout_rate=0.2, + data_format='channels_last', + num_classes=1000, + depth_multiplier=depth_multiplier, + depth_divisor=8, + min_depth=None, + stem_size=32, + use_keras=True) + decoder = MnasNetDecoder() + return decoder.decode(blocks_args), global_params + + +def mnasnet_small(depth_multiplier=None): + """Creates a mnasnet-a1 model. + + Args: + depth_multiplier: multiplier to number of filters per layer. + + Returns: + blocks_args: a list of BlocksArgs for internal MnasNet blocks. + global_params: GlobalParams, global parameters for the model. + """ + blocks_args = [ + 'r1_k3_s11_e1_i16_o8', 'r1_k3_s22_e3_i8_o16', + 'r2_k3_s22_e6_i16_o16', 'r4_k5_s22_e6_i16_o32_se0.25', + 'r3_k3_s11_e6_i32_o32_se0.25', 'r3_k5_s22_e6_i32_o88_se0.25', + 'r1_k3_s11_e6_i88_o144' + ] + global_params = mnasnet_model.GlobalParams( + batch_norm_momentum=0.99, + batch_norm_epsilon=1e-3, + dropout_rate=0, + data_format='channels_last', + num_classes=1000, + depth_multiplier=depth_multiplier, + depth_divisor=8, + min_depth=None, + stem_size=8, + use_keras=True) + decoder = MnasNetDecoder() + return decoder.decode(blocks_args), global_params + + +def mnasnet_d1(depth_multiplier=None): + """Creates a jointly searched mnasnet backbone for mnas-fpn. + + Args: + depth_multiplier: multiplier to number of filters per layer. + + Returns: + blocks_args: a list of BlocksArgs for internal MnasNet blocks. + global_params: GlobalParams, global parameters for the model. + """ + blocks_args = [ + 'r1_k3_s11_e9_i32_o24', 'r3_k3_s22_e9_i24_o36', + 'r5_k3_s22_e9_i36_o48', 'r4_k5_s22_e9_i48_o96', + 'r5_k7_s11_e3_i96_o96', 'r3_k3_s22_e9_i96_o80', + 'r1_k7_s11_e6_i80_o320_noskip' + ] + global_params = mnasnet_model.GlobalParams( + batch_norm_momentum=0.99, + batch_norm_epsilon=1e-3, + dropout_rate=0.2, + data_format='channels_last', + num_classes=1000, + depth_multiplier=depth_multiplier, + depth_divisor=8, + min_depth=None, + stem_size=32, + use_keras=False) + decoder = MnasNetDecoder() + return decoder.decode(blocks_args), global_params + + +def mnasnet_d1_320(depth_multiplier=None): + """Creates a jointly searched mnasnet backbone for 320x320 input size. + + Args: + depth_multiplier: multiplier to number of filters per layer. + + Returns: + blocks_args: a list of BlocksArgs for internal MnasNet blocks. + global_params: GlobalParams, global parameters for the model. + """ + blocks_args = [ + 'r3_k5_s11_e6_i32_o24', 'r4_k7_s22_e9_i24_o36', + 'r5_k5_s22_e9_i36_o48', 'r5_k7_s22_e6_i48_o96', + 'r5_k3_s11_e9_i96_o144', 'r5_k5_s22_e6_i144_o160', + 'r1_k7_s11_e9_i160_o320' + ] + + global_params = mnasnet_model.GlobalParams( + batch_norm_momentum=0.99, + batch_norm_epsilon=1e-3, + dropout_rate=0.2, + data_format='channels_last', + num_classes=1000, + depth_multiplier=depth_multiplier, + depth_divisor=8, + min_depth=None, + stem_size=32, + use_keras=False) + decoder = MnasNetDecoder() + return decoder.decode(blocks_args), global_params + + +def get_model_params(model_name, override_params): + """Get the block args and global params for a given model.""" + if model_name == 'mnasnet-a1': + blocks_args, global_params = mnasnet_a1() + elif model_name == 'mnasnet-b1': + blocks_args, global_params = mnasnet_b1() + elif model_name == 'mnasnet-small': + blocks_args, global_params = mnasnet_small() + elif model_name == 'mnasnet-d1': + blocks_args, global_params = mnasnet_d1() + elif model_name == 'mnasnet-d1-320': + blocks_args, global_params = mnasnet_d1_320() + else: + raise NotImplementedError( + 'model name is not pre-defined: %s' % model_name) + + if override_params: + # ValueError will be raised here if override_params has fields not included + # in global_params. + global_params = global_params._replace(**override_params) + return blocks_args, global_params + + +def build_mnasnet_model(images, model_name, training, override_params=None): + """A helper functiion to create a MnasNet model and return predicted logits. + + Args: + images: input images tensor. + model_name: string, the model name of a pre-defined MnasNet. + training: boolean, whether the model is constructed for training. + override_params: A dictionary of params for overriding. Fields must exist in + mnasnet_model.GlobalParams. + + Returns: + logits: the logits tensor of classes. + endpoints: the endpoints for each layer. + Raises: + When model_name specified an undefined model, raises NotImplementedError. + When override_params has invalid fields, raises ValueError. + """ + assert isinstance(images, tf.Tensor) + blocks_args, global_params = get_model_params(model_name, override_params) + with tf.variable_scope(model_name): + model = mnasnet_model.MnasNetModel(blocks_args, global_params) + logits = model(images, training=training) + + logits = tf.squeeze(tf.expand_dims(logits, 0), 0) + logits = tf.identity(logits, 'logits') + return logits, model.endpoints + + +def build_mnasnet_base(images, model_name, training, override_params=None): + """A helper functiion to create a MnasNet base model and return global_pool. + + Args: + images: input images tensor. + model_name: string, the model name of a pre-defined MnasNet. + training: boolean, whether the model is constructed for training. + override_params: A dictionary of params for overriding. Fields must exist in + mnasnet_model.GlobalParams. + + Returns: + features: global pool features. + endpoints: the endpoints for each layer. + Raises: + When model_name specified an undefined model, raises NotImplementedError. + When override_params has invalid fields, raises ValueError. + """ + assert isinstance(images, tf.Tensor) + blocks_args, global_params = get_model_params(model_name, override_params) + + with tf.variable_scope(model_name): + model = mnasnet_model.MnasNetModel(blocks_args, global_params) + features = model(images, training=training, features_only=True) + + features = tf.identity(features, 'global_pool') + return features, model.endpoints diff --git a/TensorFlow/contrib/cv/Mixnet_ID2072_for_Tensorflow/modelarts_entry_acc.py b/TensorFlow/contrib/cv/Mixnet_ID2072_for_Tensorflow/modelarts_entry_acc.py new file mode 100644 index 0000000000000000000000000000000000000000..13077b10e660de32d6f7861257a50e1a01ede9ba --- /dev/null +++ b/TensorFlow/contrib/cv/Mixnet_ID2072_for_Tensorflow/modelarts_entry_acc.py @@ -0,0 +1,63 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import argparse +import sys + +# 解析输入参数data_url +parser = argparse.ArgumentParser() +parser.add_argument("--data_url", type=str, default="/home/ma-user/modelarts/inputs/data_url_0") +parser.add_argument("--train_url", type=str, default="/home/ma-user/modelarts/outputs/train_url_0/") +config = parser.parse_args() + +print("[CANN-Modelzoo] code_dir path is [%s]" % (sys.path[0])) +code_dir = sys.path[0] +os.chdir(code_dir) +print("[CANN-Modelzoo] work_dir path is [%s]" % (os.getcwd())) + +print("[CANN-Modelzoo] before train - list my run files:") +os.system("ls -al /usr/local/Ascend/ascend-toolkit/") + +print("[CANN-Modelzoo] before train - list my dataset files:") +os.system("ls -al %s" % config.data_url) + +print("[CANN-Modelzoo] start run train shell") +# 设置sh文件格式为linux可执行 +os.system("dos2unix ./test/*") + +# 执行train_full_1p.sh或者train_performance_1p.sh，需要用户自己指定 +# full和performance的差异，performance只需要执行很少的step，控制在15分钟以内，主要关注性能FPS +os.system("bash ./test/train_full_1p.sh --data_path=%s --output_path=%s " % (config.data_url, config.train_url)) + +print("[CANN-Modelzoo] finish run train shell") + +# 将当前执行目录所有文件拷贝到obs的output进行备份 +print("[CANN-Modelzoo] after train - list my output files:") +os.system("cp -r %s %s " % (code_dir, config.train_url)) +os.system("ls -al %s" % config.train_url) diff --git a/TensorFlow/contrib/cv/Mixnet_ID2072_for_Tensorflow/modelarts_entry_perf.py b/TensorFlow/contrib/cv/Mixnet_ID2072_for_Tensorflow/modelarts_entry_perf.py new file mode 100644 index 0000000000000000000000000000000000000000..14384e227a0fa90a514254590aef5078c62ff700 --- /dev/null +++ b/TensorFlow/contrib/cv/Mixnet_ID2072_for_Tensorflow/modelarts_entry_perf.py @@ -0,0 +1,63 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import argparse +import sys + +# 解析输入参数data_url +parser = argparse.ArgumentParser() +parser.add_argument("--data_url", type=str, default="/home/ma-user/modelarts/inputs/data_url_0") +parser.add_argument("--train_url", type=str, default="/home/ma-user/modelarts/outputs/train_url_0/") +config = parser.parse_args() + +print("[CANN-Modelzoo] code_dir path is [%s]" % (sys.path[0])) +code_dir = sys.path[0] +os.chdir(code_dir) +print("[CANN-Modelzoo] work_dir path is [%s]" % (os.getcwd())) + +print("[CANN-Modelzoo] before train - list my run files:") +os.system("ls -al /usr/local/Ascend/ascend-toolkit/") + +print("[CANN-Modelzoo] before train - list my dataset files:") +os.system("ls -al %s" % config.data_url) + +print("[CANN-Modelzoo] start run train shell") +# 设置sh文件格式为linux可执行 +os.system("dos2unix ./test/*") + +# 执行train_full_1p.sh或者train_performance_1p.sh，需要用户自己指定 +# full和performance的差异，performance只需要执行很少的step，控制在15分钟以内，主要关注性能FPS +os.system("bash ./test/train_performance_1p.sh --data_path=%s --output_path=%s " % (config.data_url, config.train_url)) + +print("[CANN-Modelzoo] finish run train shell") + +# 将当前执行目录所有文件拷贝到obs的output进行备份 +print("[CANN-Modelzoo] after train - list my output files:") +os.system("cp -r %s %s " % (code_dir, config.train_url)) +os.system("ls -al %s" % config.train_url) diff --git a/TensorFlow/contrib/cv/Mixnet_ID2072_for_Tensorflow/modelzoo_level.txt b/TensorFlow/contrib/cv/Mixnet_ID2072_for_Tensorflow/modelzoo_level.txt new file mode 100644 index 0000000000000000000000000000000000000000..83f8226cf34b739ad69eb22437e37c3279262d04 --- /dev/null +++ b/TensorFlow/contrib/cv/Mixnet_ID2072_for_Tensorflow/modelzoo_level.txt @@ -0,0 +1,7 @@ +GPUStatus:OK +NPUMigrationStatus:OK + +FuncStatus:OK +PrecisionStatus:OK +AutoTune:OK +PerfStatus:OK \ No newline at end of file diff --git a/TensorFlow/contrib/cv/Mixnet_ID2072_for_Tensorflow/post_quantization.py b/TensorFlow/contrib/cv/Mixnet_ID2072_for_Tensorflow/post_quantization.py new file mode 100644 index 0000000000000000000000000000000000000000..fe654ed8a579767ee51514c0e01b09b7a856bd09 --- /dev/null +++ b/TensorFlow/contrib/cv/Mixnet_ID2072_for_Tensorflow/post_quantization.py @@ -0,0 +1,124 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Post-training full quantization script from TF to TFLite.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from absl import app +from absl import flags +import tensorflow.compat.v1 as tf + +import imagenet_input + +flags.DEFINE_string("saved_model_dir", None, + "Path to input savedmodel bundle.") +flags.DEFINE_enum( + "input_name", "float_image_input", ["float_image_input", "truediv"], + "Name of the input node. `float_image_input` is for image " + "array input and `truediv` is for normalized input. Please " + "use `truediv` if require_int8=True and be aware that " + "users need to handle normalization in the client side.") +flags.DEFINE_string("output_name", "logits", "Name of the output node.") +flags.DEFINE_integer( + "num_steps", 1000, + "Number of post-training quantization calibration steps to run.") +flags.DEFINE_integer("image_size", 224, "Size of the input image.") +flags.DEFINE_integer("batch_size", 1, "Batch size of input tensor.") +flags.DEFINE_string("output_tflite", None, "Path to output tflite file.") +flags.DEFINE_string("data_dir", None, "Image dataset directory.") +flags.DEFINE_bool( + "require_int8", False, "Whether all ops should be built-in" + " int8, which is necessary for EdgeTPU.") + +FLAGS = flags.FLAGS + + +def representative_dataset_gen(): + """Gets a python generator of image numpy arrays for ImageNet.""" + params = dict(batch_size=FLAGS.batch_size) + imagenet_eval = imagenet_input.ImageNetInput( + is_training=False, + data_dir=FLAGS.data_dir, + transpose_input=False, + cache=False, + image_size=FLAGS.image_size, + num_parallel_calls=1, + use_bfloat16=False) + + data = imagenet_eval.input_fn(params) + + def preprocess_map_fn(images, labels): + """The function of preprocess the map.""" + del labels + if FLAGS.input_name == "truediv": + images -= tf.constant( + imagenet_input.MEAN_RGB, shape=[1, 1, 3], dtype=images.dtype) + images /= tf.constant( + imagenet_input.STDDEV_RGB, shape=[1, 1, 3], dtype=images.dtype) + return images + + data = data.map(preprocess_map_fn) + iterator = data.make_one_shot_iterator() + for _ in range(FLAGS.num_steps): + # In eager context, we can get a python generator from a dataset iterator. + images = iterator.get_next() + yield [images] + + +def main(_): + """ Enables eager context for TF 1.x. TF 2.x will use eager by default. + This is used to conveniently get a representative dataset generator using + TensorFlow training input helper. + """ + tf.enable_eager_execution() + + converter = tf.lite.TFLiteConverter.from_saved_model( + FLAGS.saved_model_dir, + input_arrays=[FLAGS.input_name], + output_arrays=[FLAGS.output_name]) + # Chooses a tf.lite.Optimize mode: + # https://www.tensorflow.org/api_docs/python/tf/lite/Optimize + converter.optimizations = [tf.lite.Optimize.DEFAULT] + converter.representative_dataset = tf.lite.RepresentativeDataset( + representative_dataset_gen) + if FLAGS.require_int8: + converter.target_spec.supported_ops = [ + tf.lite.OpsSet.TFLITE_BUILTINS_INT8] + + tflite_buffer = converter.convert() + tf.gfile.GFile(FLAGS.output_tflite, "wb").write(tflite_buffer) + print("tflite model written to %s" % FLAGS.output_tflite) + + +if __name__ == "__main__": + flags.mark_flag_as_required("saved_model_dir") + flags.mark_flag_as_required("output_tflite") + flags.mark_flag_as_required("data_dir") + app.run(main) diff --git a/TensorFlow/contrib/cv/Mixnet_ID2072_for_Tensorflow/preprocessing.py b/TensorFlow/contrib/cv/Mixnet_ID2072_for_Tensorflow/preprocessing.py new file mode 100644 index 0000000000000000000000000000000000000000..75a33bd8921ff733c56c06b25918df56dc725b0b --- /dev/null +++ b/TensorFlow/contrib/cv/Mixnet_ID2072_for_Tensorflow/preprocessing.py @@ -0,0 +1,211 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""ImageNet preprocessing for MnasNet.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import tensorflow.compat.v1 as tf + +IMAGE_SIZE = 224 +CROP_PADDING = 32 + + +def distorted_bounding_box_crop(image_bytes, + bbox, + min_object_covered=0.1, + aspect_ratio_range=None, + area_range=None, + max_attempts=100, + scope=None): + """Generates cropped_image using one of the bboxes randomly distorted. + + See `tf.image.sample_distorted_bounding_box` for more documentation. + + Args: + image_bytes: `Tensor` of binary image data. + bbox: `Tensor` of bounding boxes arranged `[1, num_boxes, coords]` + where each coordinate is [0, 1) and the coordinates are arranged + as `[ymin, xmin, ymax, xmax]`. If num_boxes is 0 then use the whole + image. + min_object_covered: An optional `float`. Defaults to `0.1`. The cropped + area of the image must contain at least this fraction of any bounding + box supplied. + aspect_ratio_range: An optional list of `float`s. The cropped area of the + image must have an aspect ratio = width / height within this range. + area_range: An optional list of `float`s. The cropped area of the image + must contain a fraction of the supplied image within in this range. + max_attempts: An optional `int`. Number of attempts at generating a cropped + region of the image of the specified constraints. After `max_attempts` + failures, return the entire image. + scope: Optional `str` for name scope. + Returns: + cropped image `Tensor` + """ + if aspect_ratio_range is None: + spect_ratio_range=(0.75, 1.33) + if area_range is None: + area_range=(0.05, 1.0) + with tf.name_scope(scope, 'distorted_bounding_box_crop', [image_bytes, bbox]): + shape = tf.image.extract_jpeg_shape(image_bytes) + sample_distorted_bounding_box = tf.image.sample_distorted_bounding_box( + shape, + bounding_boxes=bbox, + min_object_covered=min_object_covered, + aspect_ratio_range=aspect_ratio_range, + area_range=area_range, + max_attempts=max_attempts, + use_image_if_no_bounding_boxes=True) + bbox_begin, bbox_size, _ = sample_distorted_bounding_box + + # Crop the image to the specified bounding box. + offset_y, offset_x, _ = tf.unstack(bbox_begin) + target_height, target_width, _ = tf.unstack(bbox_size) + crop_window = tf.stack( + [offset_y, offset_x, target_height, target_width]) + image = tf.image.decode_and_crop_jpeg( + image_bytes, crop_window, channels=3) + + return image + + +def _at_least_x_are_equal(a, b, x): + """At least `x` of `a` and `b` `Tensors` are equal.""" + match = tf.equal(a, b) + match = tf.cast(match, tf.int32) + return tf.greater_equal(tf.reduce_sum(match), x) + + +def _decode_and_random_crop(image_bytes, image_size): + """Make a random crop of image_size.""" + bbox = tf.constant([0.0, 0.0, 1.0, 1.0], dtype=tf.float32, shape=[1, 1, 4]) + image = distorted_bounding_box_crop( + image_bytes, + bbox, + min_object_covered=0.1, + aspect_ratio_range=(3. / 4, 4. / 3.), + area_range=(0.08, 1.0), + max_attempts=10, + scope=None) + original_shape = tf.image.extract_jpeg_shape(image_bytes) + bad = _at_least_x_are_equal(original_shape, tf.shape(image), 3) + + image = tf.cond( + bad, + lambda: _decode_and_center_crop(image_bytes, image_size), + # pylint: disable=g-long-lambda + lambda: tf.image.resize([image], [image_size, image_size])[0]) + # pylint: enable=g-long-lambda + + return image + + +def _decode_and_center_crop(image_bytes, image_size): + """Crops to center of image with padding then scales image_size.""" + shape = tf.image.extract_jpeg_shape(image_bytes) + image_height = shape[0] + image_width = shape[1] + + padded_center_crop_size = tf.cast( + ((image_size / (image_size + CROP_PADDING)) * + tf.cast(tf.minimum(image_height, image_width), tf.float32)), tf.int32) + + offset_height = ((image_height - padded_center_crop_size) + 1) // 2 + offset_width = ((image_width - padded_center_crop_size) + 1) // 2 + crop_window = tf.stack([ + offset_height, offset_width, padded_center_crop_size, + padded_center_crop_size + ]) + image = tf.image.decode_and_crop_jpeg(image_bytes, crop_window, channels=3) + image = tf.image.resize([image], [image_size, image_size])[0] + + return image + + +def _flip(image): + """Random horizontal image flip.""" + image = tf.image.random_flip_left_right(image) + return image + + +def preprocess_for_train(image_bytes, use_bfloat16, image_size=IMAGE_SIZE): + """Preprocesses the given image for evaluation. + + Args: + image_bytes: `Tensor` representing an image binary of arbitrary size. + use_bfloat16: `bool` for whether to use bfloat16. + image_size: image size. + + Returns: + A preprocessed image `Tensor`. + """ + image = _decode_and_random_crop(image_bytes, image_size) + image = _flip(image) + image = tf.reshape(image, [image_size, image_size, 3]) + image = tf.image.convert_image_dtype( + image, dtype=tf.bfloat16 if use_bfloat16 else tf.float32) + return image + + +def preprocess_for_eval(image_bytes, use_bfloat16, image_size=IMAGE_SIZE): + """Preprocesses the given image for evaluation. + + Args: + image_bytes: `Tensor` representing an image binary of arbitrary size. + use_bfloat16: `bool` for whether to use bfloat16. + image_size: image size. + + Returns: + A preprocessed image `Tensor`. + """ + image = _decode_and_center_crop(image_bytes, image_size) + image = tf.reshape(image, [image_size, image_size, 3]) + image = tf.image.convert_image_dtype( + image, dtype=tf.bfloat16 if use_bfloat16 else tf.float32) + return image + + +def preprocess_image(image_bytes, + is_training=False, + use_bfloat16=False, + image_size=IMAGE_SIZE): + """Preprocesses the given image. + + Args: + image_bytes: `Tensor` representing an image binary of arbitrary size. + is_training: `bool` for whether the preprocessing is for training. + use_bfloat16: `bool` for whether to use bfloat16. + image_size: image size. + + Returns: + A preprocessed image `Tensor` with value range of [0, 255]. + """ + if is_training: + return preprocess_for_train(image_bytes, use_bfloat16, image_size) + else: + return preprocess_for_eval(image_bytes, use_bfloat16, image_size) diff --git a/TensorFlow/contrib/cv/Mixnet_ID2072_for_Tensorflow/requirements.txt b/TensorFlow/contrib/cv/Mixnet_ID2072_for_Tensorflow/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/TensorFlow/contrib/cv/Mixnet_ID2072_for_Tensorflow/run_1p.sh b/TensorFlow/contrib/cv/Mixnet_ID2072_for_Tensorflow/run_1p.sh new file mode 100644 index 0000000000000000000000000000000000000000..deb54415c1874d2c8070da65459787e1eb57e56c --- /dev/null +++ b/TensorFlow/contrib/cv/Mixnet_ID2072_for_Tensorflow/run_1p.sh @@ -0,0 +1,22 @@ +#!/bin/bash +### Do not need to Configure CANN Environment on Modelarts Platform, because it has been set already. +### Modelarts Platform command for train +export TF_CPP_MIN_LOG_LEVEL=2 ## Tensorflow api print Log Config +export ASCEND_SLOG_PRINT_TO_STDOUT=0 ## Print log on terminal on(1), off(0) + +code_dir=${1} +data_dir=${2} +result_dir=${3} +obs_url=${4} + +current_time=`date "+%Y-%m-%d-%H-%M-%S"` + +python3.7 ${code_dir}/mnasnet_main.py \ + --data_dir=${data_dir} \ + --model_dir=${result_dir} \ + --obs_dir=${obs_url} \ + --model_name='mixnet-s' \ + --npu_dump_data=True \ + --npu_profiling=True \ + --npu_dump_graph=False \ + --npu_auto_tune=False 2>&1 | tee ${result_dir}/${current_time}_train_npu.log \ No newline at end of file diff --git a/TensorFlow/contrib/cv/Mixnet_ID2072_for_Tensorflow/test/train_full_1p.sh b/TensorFlow/contrib/cv/Mixnet_ID2072_for_Tensorflow/test/train_full_1p.sh new file mode 100644 index 0000000000000000000000000000000000000000..418c62acc475729eefb82d10ea9fe114614b63f3 --- /dev/null +++ b/TensorFlow/contrib/cv/Mixnet_ID2072_for_Tensorflow/test/train_full_1p.sh @@ -0,0 +1,185 @@ +#!/bin/bash + +########################################################## +#########第3行 至 100行，请一定不要、不要、不要修改########## +#########第3行 至 100行，请一定不要、不要、不要修改########## +#########第3行 至 100行，请一定不要、不要、不要修改########## +########################################################## +# shell脚本所在路径 +cur_path=`echo $(cd $(dirname $0);pwd)` + +# 判断当前shell是否是performance +perf_flag=`echo $0 | grep performance | wc -l` + +# 当前执行网络的名称 +Network=`echo $(cd $(dirname $0);pwd) | awk -F"/" '{print $(NF-1)}'` + +export RANK_SIZE=1 +export RANK_ID=0 +export JOB_ID=10087 + +# 路径参数初始化 +data_path="" +output_path="" + +# 帮助信息，不需要修改 +if [[ $1 == --help || $1 == -h ]];then + echo"usage:./train_performance_1P.sh " + echo " " + echo "parameter explain: + --data_path # dataset of training + --output_path # output of training + --train_steps # max_step for training + --train_epochs # max_epoch for training + --batch_size # batch size + -h/--help show help message + " + exit 1 +fi + +# 参数校验，不需要修改 +for para in $* +do + if [[ $para == --data_path* ]];then + data_path=`echo ${para#*=}` + elif [[ $para == --output_path* ]];then + output_path=`echo ${para#*=}` + elif [[ $para == --train_steps* ]];then + train_steps=`echo ${para#*=}` + elif [[ $para == --train_epochs* ]];then + train_epochs=`echo ${para#*=}` + elif [[ $para == --batch_size* ]];then + batch_size=`echo ${para#*=}` + fi +done + +# 校验是否传入data_path,不需要修改 +if [[ $data_path == "" ]];then + echo "[Error] para \"data_path\" must be config" + exit 1 +fi + +# 校验是否传入output_path,不需要修改 +if [[ $output_path == "" ]];then + output_path="./test/output/${ASCEND_DEVICE_ID}" +fi + +# 设置打屏日志文件名，请保留，文件名为${print_log} +print_log="./test/output/${ASCEND_DEVICE_ID}/train_${ASCEND_DEVICE_ID}.log" +modelarts_flag=`cat /etc/passwd |grep ma-user` +if [ x"${modelarts_flag}" != x ]; +then + echo "running with modelarts_flag..." + print_log_name=`ls /home/ma-user/modelarts/log/ | grep proc-rank` + print_log="/home/ma-user/modelarts/log/${print_log_name}" +fi +echo "### get your log here : ${print_log}" + +CaseName="" +function get_casename() +{ + if [ x"${perf_flag}" = x1 ]; + then + CaseName=${Network}_bs${batch_size}_${RANK_SIZE}'p'_'perf' + else + CaseName=${Network}_bs${batch_size}_${RANK_SIZE}'p'_'acc' + fi +} + +# 跳转到code目录 +cd ${cur_path}/../ +rm -rf ./test/output/${ASCEND_DEVICE_ID} +mkdir -p ./test/output/${ASCEND_DEVICE_ID} + +# 训练开始时间记录，不需要修改 +start_time=$(date +%s) +########################################################## +#########第3行 至 100行，请一定不要、不要、不要修改########## +#########第3行 至 100行，请一定不要、不要、不要修改########## +#########第3行 至 100行，请一定不要、不要、不要修改########## +########################################################## + +#========================================================= +#========================================================= +#========训练执行命令，需要根据您的网络进行修改============== +#========================================================= +#========================================================= +# 基础参数，需要模型审视修改 +# 您的训练数据集在${data_path}路径下，请直接使用这个变量获取 +# 您的训练输出目录在${output_path}路径下，请直接使用这个变量获取 +# 您的其他基础参数，可以自定义增加，但是batch_size请保留，并且设置正确的值 +batch_size=64 + +if [ x"${modelarts_flag}" != x ]; +then + python3.7 ./mnasnet_main.py --data_dir=${data_dir}/ILSVRC2012 --model_dir=${data_dir}/result_mix --obs_dir=${output_path} --model_name='mixnet-s' --train_steps=${train_steps} +else + python3.7 ./mnasnet_main.py --data_dir=${data_dir}/ILSVRC2012 --model_dir=${data_dir}/result_mix --obs_dir=${output_path} --model_name='mixnet-s' --train_steps=${train_steps} 1>${print_log} 2>&1 +fi + +# 性能相关数据计算 +StepTime=`grep "sec/step :" ${print_log} | tail -n 10 | awk '{print $NF}' | awk '{sum+=$1} END {print sum/NR}'` +FPS=`awk 'BEGIN{printf "%.2f\n", '${batch_size}'/'${StepTime}'}'` + +# 精度相关数据计算 +train_accuracy=`grep "Final Accuracy accuracy" ${print_log} | awk '{print $NF}'` +# 提取所有loss打印信息 +grep "loss :" ${print_log} | awk -F ":" '{print $4}' | awk -F "-" '{print $1}' > ./test/output/${ASCEND_DEVICE_ID}/my_output_loss.txt + + +########################################################### +#########后面的所有内容请不要修改########################### +#########后面的所有内容请不要修改########################### +#########后面的所有内容请不要修改########################### +########################################################### + +# 判断本次执行是否正确使用Ascend NPU +tf_flag=`echo ${Network} | grep TensorFlow | wc -l` +use_npu_flag=`grep "The model has been compiled on the Ascend AI processor" ${print_log} | wc -l` +if [ x"${use_npu_flag}" == x0 -a x"${tf_flag}" == x1 ]; +then + echo "------------------ ERROR NOTICE START ------------------" + echo "ERROR, your task haven't used Ascend NPU, please check your npu Migration." + echo "------------------ ERROR NOTICE END------------------" +else + echo "------------------ INFO NOTICE START------------------" + echo "INFO, your task have used Ascend NPU, please check your result." + echo "------------------ INFO NOTICE END------------------" +fi + +# 获取最终的casename，请保留，case文件名为${CaseName} +get_casename + +# 重命名loss文件 +if [ -f ./test/output/${ASCEND_DEVICE_ID}/my_output_loss.txt ]; +then + mv ./test/output/${ASCEND_DEVICE_ID}/my_output_loss.txt ./test/output/${ASCEND_DEVICE_ID}/${CaseName}_loss.txt +fi + +# 训练端到端耗时 +end_time=$(date +%s) +e2e_time=$(( $end_time - $start_time )) + +echo "------------------ Final result ------------------" +# 输出性能FPS/单step耗时/端到端耗时 +echo "Final Performance images/sec : $FPS" +echo "Final Performance sec/step : $StepTime" +echo "E2E Training Duration sec : $e2e_time" + +# 输出训练精度 +echo "Final Train Accuracy : ${train_accuracy}" + +# 最后一个迭代loss值，不需要修改 +ActualLoss=(`awk 'END {print $NF}' $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}_loss.txt`) + +#关键信息打印到${CaseName}.log中，不需要修改 +echo "Network = ${Network}" > $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "RankSize = ${RANK_SIZE}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "BatchSize = ${batch_size}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "DeviceType = `uname -m`" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "CaseName = ${CaseName}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "ActualFPS = ${FPS}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "TrainingTime = ${StepTime}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "ActualLoss = ${ActualLoss}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "E2ETrainingTime = ${e2e_time}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "TrainAccuracy = ${train_accuracy}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log \ No newline at end of file diff --git a/TensorFlow/contrib/cv/Mixnet_ID2072_for_Tensorflow/test/train_performance_1p.sh b/TensorFlow/contrib/cv/Mixnet_ID2072_for_Tensorflow/test/train_performance_1p.sh new file mode 100644 index 0000000000000000000000000000000000000000..843d4abed86a1f5fc4a232b344425320ed58f515 --- /dev/null +++ b/TensorFlow/contrib/cv/Mixnet_ID2072_for_Tensorflow/test/train_performance_1p.sh @@ -0,0 +1,186 @@ +#!/bin/bash + +########################################################## +#########第3行 至 100行，请一定不要、不要、不要修改########## +#########第3行 至 100行，请一定不要、不要、不要修改########## +#########第3行 至 100行，请一定不要、不要、不要修改########## +########################################################## +# shell脚本所在路径 +cur_path=`echo $(cd $(dirname $0);pwd)` + +# 判断当前shell是否是performance +perf_flag=`echo $0 | grep performance | wc -l` + +# 当前执行网络的名称 +Network=`echo $(cd $(dirname $0);pwd) | awk -F"/" '{print $(NF-1)}'` + +export RANK_SIZE=1 +export RANK_ID=0 +export JOB_ID=10087 + +# 路径参数初始化 +data_path="" +output_path="" + +# 帮助信息，不需要修改 +if [[ $1 == --help || $1 == -h ]];then + echo"usage:./train_performance_1P.sh " + echo " " + echo "parameter explain: + --data_path # dataset of training + --output_path # output of training + --train_steps # max_step for training + --train_epochs # max_epoch for training + --batch_size # batch size + -h/--help show help message + " + exit 1 +fi + +# 参数校验，不需要修改 +for para in $* +do + if [[ $para == --data_path* ]];then + data_path=`echo ${para#*=}` + elif [[ $para == --output_path* ]];then + output_path=`echo ${para#*=}` + elif [[ $para == --train_steps* ]];then + train_steps=`echo ${para#*=}` + elif [[ $para == --train_epochs* ]];then + train_epochs=`echo ${para#*=}` + elif [[ $para == --batch_size* ]];then + batch_size=`echo ${para#*=}` + fi +done + +# 校验是否传入data_path,不需要修改 +if [[ $data_path == "" ]];then + echo "[Error] para \"data_path\" must be config" + exit 1 +fi + +# 校验是否传入output_path,不需要修改 +if [[ $output_path == "" ]];then + output_path="./test/output/${ASCEND_DEVICE_ID}" +fi + +# 设置打屏日志文件名，请保留，文件名为${print_log} +print_log="./test/output/${ASCEND_DEVICE_ID}/train_${ASCEND_DEVICE_ID}.log" +modelarts_flag=`cat /etc/passwd |grep ma-user` +if [ x"${modelarts_flag}" != x ]; +then + echo "running with modelarts..." + print_log_name=`ls /home/ma-user/modelarts/log/ | grep proc-rank` + print_log="/home/ma-user/modelarts/log/${print_log_name}" +fi +echo "### get your log here : ${print_log}" + +CaseName="" +function get_casename() +{ + if [ x"${perf_flag}" = x1 ]; + then + CaseName=${Network}_bs${batch_size}_${RANK_SIZE}'p'_'perf' + else + CaseName=${Network}_bs${batch_size}_${RANK_SIZE}'p'_'acc' + fi +} + +# 跳转到code目录 +cd ${cur_path}/../ +rm -rf ./test/output/${ASCEND_DEVICE_ID} +mkdir -p ./test/output/${ASCEND_DEVICE_ID} + +# 训练开始时间记录，不需要修改 +start_time=$(date +%s) +########################################################## +#########第3行 至 100行，请一定不要、不要、不要修改########## +#########第3行 至 100行，请一定不要、不要、不要修改########## +#########第3行 至 100行，请一定不要、不要、不要修改########## +########################################################## + +#========================================================= +#========================================================= +#========训练执行命令，需要根据您的网络进行修改============== +#========================================================= +#========================================================= +# 基础参数，需要模型审视修改 +# 您的训练数据集在${data_path}路径下，请直接使用这个变量获取 +# 您的训练输出目录在${output_path}路径下，请直接使用这个变量获取 +# 您的其他基础参数，可以自定义增加，但是batch_size请保留，并且设置正确的值 +train_epochs=2 +train_steps=100 +batch_size=64 + +if [ x"${modelarts_flag}" != x ]; +then + python3.7 ./mnasnet_main.py --data_dir=${data_dir}/ILSVRC2012 --model_dir=${data_dir}/result_mix --obs_dir=${output_path} --model_name='mixnet-s' --train_steps=${train_steps} +else + python3.7 ./mnasnet_main.py --data_dir=${data_dir}/ILSVRC2012 --model_dir=${data_dir}/result_mix --obs_dir=${output_path} --model_name='mixnet-s' --train_steps=${train_steps} 1>${print_log} 2>&1 +fi + +# 性能相关数据计算 +StepTime=`grep "sec/step :" ${print_log} | tail -n 10 | awk '{print $NF}' | awk '{sum+=$1} END {print sum/NR}'` +FPS=`awk 'BEGIN{printf "%.2f\n", '${batch_size}'/'${StepTime}'}'` + +# 精度相关数据计算 +train_accuracy=`grep "Final Accuracy accuracy" ${print_log} | awk '{print $NF}'` +# 提取所有loss打印信息 +grep "loss :" ${print_log} | awk -F ":" '{print $4}' | awk -F "-" '{print $1}' > ./test/output/${ASCEND_DEVICE_ID}/my_output_loss.txt + + +########################################################### +#########后面的所有内容请不要修改########################### +#########后面的所有内容请不要修改########################### +#########后面的所有内容请不要修改########################### +########################################################### + +# 判断本次执行是否正确使用Ascend NPU +tf_flag=`echo ${Network} | grep TensorFlow | wc -l` +use_npu_flag=`grep "The model has been compiled on the Ascend AI processor" ${print_log} | wc -l` +if [ x"${use_npu_flag}" == x0 -a x"${tf_flag}" == x1 ]; +then + echo "------------------ ERROR NOTICE START ------------------" + echo "ERROR, your task haven't used Ascend NPU, please check your npu Migration." + echo "------------------ ERROR NOTICE END------------------" +else + echo "------------------ INFO NOTICE START------------------" + echo "INFO, your task have used Ascend NPU, please check your result." + echo "------------------ INFO NOTICE END------------------" +fi + +# 获取最终的casename，请保留，case文件名为${CaseName} +get_casename + +# 重命名loss文件 +if [ -f ./test/output/${ASCEND_DEVICE_ID}/my_output_loss.txt ]; +then + mv ./test/output/${ASCEND_DEVICE_ID}/my_output_loss.txt ./test/output/${ASCEND_DEVICE_ID}/${CaseName}_loss.txt +fi + +# 训练端到端耗时 +end_time=$(date +%s) +e2e_time=$(( $end_time - $start_time )) + +echo "------------------ Final result ------------------" +# 输出性能FPS/单step耗时/端到端耗时 +echo "Final Performance images/sec : $FPS" +echo "Final Performance sec/step : $StepTime" +echo "E2E Training Duration sec : $e2e_time" + +# 输出训练精度 +echo "Final Train Accuracy : ${train_accuracy}" + +# 最后一个迭代loss值，不需要修改 +ActualLoss=(`awk 'END {print $NF}' $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}_loss.txt`) + +#关键信息打印到${CaseName}.log中，不需要修改 +echo "Network = ${Network}" > $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "RankSize = ${RANK_SIZE}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "BatchSize = ${batch_size}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "DeviceType = `uname -m`" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "CaseName = ${CaseName}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "ActualFPS = ${FPS}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "TrainingTime = ${StepTime}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "ActualLoss = ${ActualLoss}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "E2ETrainingTime = ${e2e_time}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log \ No newline at end of file diff --git a/TensorFlow/contrib/cv/Mixnet_ID2072_for_Tensorflow/utils.py b/TensorFlow/contrib/cv/Mixnet_ID2072_for_Tensorflow/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..0b3c80bf83e786ebff0fa7cee55ce888e599e08b --- /dev/null +++ b/TensorFlow/contrib/cv/Mixnet_ID2072_for_Tensorflow/utils.py @@ -0,0 +1,599 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Model utilities.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import json +import os +import sys + +from absl import flags +from absl import logging +import numpy as np +import tensorflow.compat.v1 as tf + +import lars_optimizer +from tensorflow.python.tpu import tpu_function # pylint:disable=g-direct-tensorflow-import + +FLAGS = flags.FLAGS + + +def build_learning_rate(initial_lr, + global_step, + steps_per_epoch=None, + lr_decay_type='exponential', + decay_factor=0.97, + decay_epochs=2.4, + total_steps=None, + warmup_epochs=5): + """Build learning rate.""" + if lr_decay_type == 'exponential': + assert steps_per_epoch is not None + decay_steps = steps_per_epoch * decay_epochs + lr = tf.train.exponential_decay( + initial_lr, global_step, decay_steps, decay_factor, staircase=True) + elif lr_decay_type == 'cosine': + assert total_steps is not None + lr = 0.5 * initial_lr * ( + 1 + tf.cos(np.pi * tf.cast(global_step, tf.float32) / total_steps)) + elif lr_decay_type == 'constant': + lr = initial_lr + elif lr_decay_type == 'poly': + tf.logging.info('Using poly LR schedule') + assert steps_per_epoch is not None + assert total_steps is not None + warmup_steps = int(steps_per_epoch * warmup_epochs) + min_step = tf.constant(1, dtype=tf.int64) + decay_steps = tf.maximum( + min_step, tf.subtract(global_step, warmup_steps)) + lr = tf.train.polynomial_decay( + initial_lr, + decay_steps, + total_steps - warmup_steps + 1, + end_learning_rate=0.1, + power=2.0) + else: + assert False, 'Unknown lr_decay_type : %s' % lr_decay_type + + if warmup_epochs: + logging.info('Learning rate warmup_epochs: %d', warmup_epochs) + warmup_steps = int(warmup_epochs * steps_per_epoch) + warmup_lr = ( + initial_lr * tf.cast(global_step, tf.float32) / tf.cast( + warmup_steps, tf.float32)) + lr = tf.cond(global_step < warmup_steps, lambda: warmup_lr, lambda: lr) + + return lr + + +def build_optimizer(learning_rate, + optimizer_name='rmsprop', + decay=0.9, + epsilon=0.001, + momentum=0.9, + lars_weight_decay=None, + lars_epsilon=None): + """Build optimizer.""" + if optimizer_name == 'sgd': + logging.info('Using SGD optimizer') + optimizer = tf.train.GradientDescentOptimizer( + learning_rate=learning_rate) + elif optimizer_name == 'momentum': + logging.info('Using Momentum optimizer') + optimizer = tf.train.MomentumOptimizer( + learning_rate=learning_rate, momentum=momentum) + elif optimizer_name == 'rmsprop': + logging.info('Using RMSProp optimizer') + optimizer = tf.train.RMSPropOptimizer(learning_rate, decay, momentum, + epsilon) + elif optimizer_name == 'lars': + logging.info('Using LARS optimizer') + assert lars_weight_decay is not None, 'LARS weight decay is None.' + assert lars_epsilon is not None, 'LARS epsilon is None.' + optimizer = lars_optimizer.LARSOptimizer( + learning_rate, + momentum=momentum, + weight_decay=lars_weight_decay, + skip_list=['batch_normalization', 'bias', 'beta', 'gamma'], + epsilon=lars_epsilon) + else: + logging.fatal('Unknown optimizer: %s', optimizer_name) + + return optimizer + + +class TpuBatchNormalization(tf.layers.BatchNormalization): + # class TpuBatchNormalization(tf.layers.BatchNormalization): + """Cross replica batch normalization.""" + + def __init__(self, fused=False, **kwargs): + if fused in (True, None): + raise ValueError( + 'TpuBatchNormalization does not support fused=True.') + super(TpuBatchNormalization, self).__init__(fused=fused, **kwargs) + + def _cross_replica_average(self, t, num_shards_per_group): + """Calculates the average value of input tensor across TPU replicas.""" + num_shards = tpu_function.get_tpu_context().number_of_shards + group_assignment = None + if num_shards_per_group > 1: + if num_shards % num_shards_per_group != 0: + raise ValueError('num_shards: %d mod shards_per_group: %d, should be 0' + % (num_shards, num_shards_per_group)) + num_groups = num_shards // num_shards_per_group + group_assignment = [[ + x for x in range(num_shards) if x // num_shards_per_group == y + ] for y in range(num_groups)] + return tf.tpu.cross_replica_sum(t, group_assignment) / tf.cast( + num_shards_per_group, t.dtype) + + def _moments(self, inputs, reduction_axes, keep_dims): + """Compute the mean and variance: it overrides the original _moments.""" + shard_mean, shard_variance = super(TpuBatchNormalization, self)._moments( + inputs, reduction_axes, keep_dims=keep_dims) + + num_shards = tpu_function.get_tpu_context().number_of_shards or 1 + if num_shards <= 8: # Skip cross_replica for 2x2 or smaller slices. + num_shards_per_group = 1 + else: + num_shards_per_group = max(8, num_shards // 8) + logging.info('TpuBatchNormalization with num_shards_per_group %s', + num_shards_per_group) + if num_shards_per_group > 1: + # Compute variance using: Var[X]= E[X^2] - E[X]^2. + shard_square_of_mean = tf.math.square(shard_mean) + shard_mean_of_square = shard_variance + shard_square_of_mean + group_mean = self._cross_replica_average( + shard_mean, num_shards_per_group) + group_mean_of_square = self._cross_replica_average( + shard_mean_of_square, num_shards_per_group) + group_variance = group_mean_of_square - tf.math.square(group_mean) + return (group_mean, group_variance) + else: + return (shard_mean, shard_variance) + + +class BatchNormalization(tf.layers.BatchNormalization): + """Fixed default name of BatchNormalization to match TpuBatchNormalization.""" + + def __init__(self, name='tpu_batch_normalization', **kwargs): + super(BatchNormalization, self).__init__(name=name, **kwargs) + + +def train_batch_norm(**kwargs): + if 'optimizer' in FLAGS and FLAGS.optimizer == 'lars': + return DistributedBatchNormalization(**kwargs) + return TpuBatchNormalization(**kwargs) + + +def eval_batch_norm(**kwargs): + if 'optimizer' in FLAGS and FLAGS.optimizer == 'lars': + return DistributedBatchNormalization(**kwargs) + return BatchNormalization(**kwargs) + + +class DistributedBatchNormalization(object): + """Distributed batch normalization used in https://arxiv.org/abs/2011.00071.""" + + def __init__(self, axis, momentum, epsilon): + self.axis = axis + self.momentum = momentum + self.epsilon = epsilon + + def __call__(self, x, training, distname='batch_normalization'): + shape = [x.shape[-1]] + with tf.variable_scope('batch_normalization'): + ones = tf.initializers.ones() + zeros = tf.initializers.zeros() + gamma = tf.get_variable( + 'gamma', shape, initializer=ones, trainable=True, use_resource=True) + beta = tf.get_variable( + 'beta', shape, initializer=zeros, trainable=True, use_resource=True) + moving_mean = tf.get_variable( + 'moving_mean', + shape, + initializer=zeros, + trainable=False, + use_resource=True) + moving_variance = tf.get_variable( + 'moving_variance', + shape, + initializer=ones, + trainable=False, + use_resource=True) + num_replicas = FLAGS.num_replicas + + x = tf.cast(x, tf.float32) + if training: + if num_replicas <= 8: + group_assign = None + group_shards = tf.cast(num_replicas, tf.float32) + else: + + group_shards = max( + 1, + int(FLAGS.batch_norm_batch_size / + (FLAGS.train_batch_size / num_replicas))) + group_assign = np.arange(num_replicas, dtype=np.int32) + group_assign = group_assign.reshape([-1, group_shards]) + group_assign = group_assign.tolist() + group_shards = tf.cast(group_shards, tf.float32) + + mean = tf.reduce_mean(x, [0, 1, 2]) + mean = tf.tpu.cross_replica_sum(mean, group_assign) / group_shards + + # Var[x] = E[x^2] - E[x]^2 + mean_sq = tf.reduce_mean(tf.math.square(x), [0, 1, 2]) + mean_sq = tf.tpu.cross_replica_sum( + mean_sq, group_assign) / group_shards + variance = mean_sq - tf.math.square(mean) + + decay = tf.cast(1. - self.momentum, tf.float32) + + def u(moving, normal, name): + """The util of subtract.""" + num_replicas_fp = tf.cast(num_replicas, tf.float32) + normal = tf.tpu.cross_replica_sum(normal) / num_replicas_fp + diff = decay * (moving - normal) + return tf.assign_sub(moving, diff, use_locking=True, name=name) + + tf.add_to_collection(tf.GraphKeys.UPDATE_OPS, + u(moving_mean, mean, name='moving_mean')) + tf.add_to_collection(tf.GraphKeys.UPDATE_OPS, + u(moving_variance, variance, name='moving_variance')) + + x = tf.nn.batch_normalization( + x, + mean=mean, + variance=variance, + offset=beta, + scale=gamma, + variance_epsilon=self.epsilon) + else: + + x, _, _ = tf.nn.fused_batch_norm( + x, + scale=gamma, + offset=beta, + mean=moving_mean, + variance=moving_variance, + epsilon=self.epsilon, + is_training=False) + + return x + + +def drop_connect(inputs, is_training, survival_prob): + """Drop the entire conv with given survival probability.""" + # "Deep Networks with Stochastic Depth", https://arxiv.org/pdf/1603.09382.pdf + if not is_training: + return inputs + + # Compute tensor. + batch_size = tf.shape(inputs)[0] + random_tensor = survival_prob + random_tensor += tf.random_uniform([batch_size, + 1, 1, 1], dtype=inputs.dtype) + binary_tensor = tf.floor(random_tensor) + # Unlike conventional way that multiply survival_prob at test time, here we + # divide survival_prob at training time, such that no addition compute is + # needed at test time. + output = tf.div(inputs, survival_prob) * binary_tensor + return output + + +def archive_ckpt(ckpt_eval, ckpt_objective, ckpt_path): + """Archive a checkpoint if the metric is better.""" + ckpt_dir, ckpt_name = os.path.split(ckpt_path) + + saved_objective_path = os.path.join(ckpt_dir, 'best_objective.txt') + saved_objective = float('-inf') + if tf.gfile.Exists(saved_objective_path): + with tf.gfile.GFile(saved_objective_path, 'r') as f: + saved_objective = float(f.read()) + if saved_objective > ckpt_objective: + logging.info('Ckpt %s is worse than %s', + ckpt_objective, saved_objective) + return False + + filenames = tf.gfile.Glob(ckpt_path + '.*') + if filenames is None: + logging.info('No files to copy for checkpoint %s', ckpt_path) + return False + + # Clear the old folder. + dst_dir = os.path.join(ckpt_dir, 'archive') + if tf.gfile.Exists(dst_dir): + tf.gfile.DeleteRecursively(dst_dir) + tf.gfile.MakeDirs(dst_dir) + + # Write checkpoints. + for f in filenames: + dest = os.path.join(dst_dir, os.path.basename(f)) + tf.gfile.Copy(f, dest, overwrite=True) + ckpt_state = tf.train.generate_checkpoint_state_proto( + dst_dir, + model_checkpoint_path=ckpt_name, + all_model_checkpoint_paths=[ckpt_name]) + with tf.gfile.GFile(os.path.join(dst_dir, 'checkpoint'), 'w') as f: + f.write(str(ckpt_state)) + with tf.gfile.GFile(os.path.join(dst_dir, 'best_eval.txt'), 'w') as f: + f.write('%s' % ckpt_eval) + + # Update the best objective. + with tf.gfile.GFile(saved_objective_path, 'w') as f: + f.write('%f' % ckpt_objective) + + logging.info('Copying checkpoint %s to %s', ckpt_path, dst_dir) + return True + + +def get_ema_vars(): + """Get all exponential moving average (ema) variables.""" + ema_vars = tf.trainable_variables() + tf.get_collection('moving_vars') + for v in tf.global_variables(): + # We maintain mva for batch norm moving mean and variance as well. + if 'moving_mean' in v.name or 'moving_variance' in v.name: + ema_vars.append(v) + return list(set(ema_vars)) + + +class DepthwiseConv2D(tf.keras.layers.DepthwiseConv2D, tf.layers.Layer): + """Wrap keras DepthwiseConv2D to tf.layers.""" + + pass + + +class Conv2D(tf.layers.Conv2D): + """Wrapper for Conv2D with specialization for fast inference.""" + + def _bias_activation(self, outputs): + if self.use_bias: + outputs = tf.nn.bias_add(outputs, self.bias, data_format='NCHW') + if self.activation is not None: + return self.activation(outputs) + return outputs + + def _can_run_fast_1x1(self, inputs): + batch_size = inputs.shape.as_list()[0] + return (self.data_format == 'channels_first' and + batch_size == 1 and + self.kernel_size == (1, 1)) + + def _call_fast_1x1(self, inputs): + # Compute the 1x1 convolution as a matmul. + inputs_shape = tf.shape(inputs) + flat_inputs = tf.reshape(inputs, [inputs_shape[1], -1]) + flat_outputs = tf.matmul( + tf.squeeze(self.kernel), + flat_inputs, + transpose_a=True) + outputs_shape = tf.concat( + [[1, self.filters], inputs_shape[2:]], axis=0) + outputs = tf.reshape(flat_outputs, outputs_shape) + + # Handle the bias and activation function. + return self._bias_activation(outputs) + + def call(self, inputs): + """Rewrite the function of call. """ + if self._can_run_fast_1x1(inputs): + return self._call_fast_1x1(inputs) + return super(Conv2D, self).call(inputs) + + +class EvalCkptDriver(object): + """A driver for running eval inference. + + Attributes: + model_name: str. Model name to eval. + batch_size: int. Eval batch size. + image_size: int. Input image size, determined by model name. + num_classes: int. Number of classes, default to 1000 for ImageNet. + include_background_label: whether to include extra background label. + advprop_preprocessing: whether to use advprop preprocessing. + """ + + def __init__(self, + model_name, + batch_size=1, + image_size=224, + num_classes=1000, + include_background_label=False, + advprop_preprocessing=False): + """Initialize internal variables.""" + self.model_name = model_name + self.batch_size = batch_size + self.num_classes = num_classes + self.include_background_label = include_background_label + self.image_size = image_size + self.advprop_preprocessing = advprop_preprocessing + + def restore_model(self, sess, ckpt_dir, enable_ema=True, export_ckpt=None): + """Restore variables from checkpoint dir.""" + sess.run(tf.global_variables_initializer()) + checkpoint = tf.train.latest_checkpoint(ckpt_dir) + if enable_ema: + ema = tf.train.ExponentialMovingAverage(decay=0.0) + ema_vars = get_ema_vars() + var_dict = ema.variables_to_restore(ema_vars) + ema_assign_op = ema.apply(ema_vars) + else: + var_dict = get_ema_vars() + ema_assign_op = None + + tf.train.get_or_create_global_step() + sess.run(tf.global_variables_initializer()) + saver = tf.train.Saver(var_dict, max_to_keep=1) + saver.restore(sess, checkpoint) + + if export_ckpt: + if ema_assign_op is not None: + sess.run(ema_assign_op) + saver = tf.train.Saver(max_to_keep=1, save_relative_paths=True) + saver.save(sess, export_ckpt) + + def build_model(self, features, is_training): + """Build model with input features.""" + del features, is_training + raise ValueError('Must be implemented by subclasses.') + + def get_preprocess_fn(self): + """Raise the error.""" + raise ValueError('Must be implemented by subclsses.') + + def build_dataset(self, filenames, labels, is_training): + """Build input dataset.""" + batch_drop_remainder = False + if 'condconv' in self.model_name and not is_training: + # CondConv layers can only be called with known batch dimension. Thus, we + # must drop all remaining examples that do not make up one full batch. + # To ensure all examples are evaluated, use a batch size that evenly + # divides the number of files. + batch_drop_remainder = True + num_files = len(filenames) + if num_files % self.batch_size != 0: + tf.logging.warn('Remaining examples in last batch are not being ' + 'evaluated.') + filenames = tf.constant(filenames) + labels = tf.constant(labels) + dataset = tf.data.Dataset.from_tensor_slices((filenames, labels)) + + def _parse_function(filename, label): + image_string = tf.read_file(filename) + preprocess_fn = self.get_preprocess_fn() + image_decoded = preprocess_fn( + image_string, is_training, image_size=self.image_size) + image = tf.cast(image_decoded, tf.float32) + return image, label + + dataset = dataset.map(_parse_function) + dataset = dataset.batch(self.batch_size, + drop_remainder=batch_drop_remainder) + + iterator = dataset.make_one_shot_iterator() + images, labels = iterator.get_next() + return images, labels + + def run_inference(self, + ckpt_dir, + image_files, + labels, + enable_ema=True, + export_ckpt=None): + """Build and run inference on the target images and labels.""" + label_offset = 1 if self.include_background_label else 0 + with tf.Graph().as_default(), tf.Session() as sess: + images, labels = self.build_dataset(image_files, labels, False) + probs = self.build_model(images, is_training=False) + if isinstance(probs, tuple): + probs = probs[0] + + self.restore_model(sess, ckpt_dir, enable_ema, export_ckpt) + + prediction_idx = [] + prediction_prob = [] + for _ in range(len(image_files) // self.batch_size): + out_probs = sess.run(probs) + idx = np.argsort(out_probs)[::-1] + prediction_idx.append(idx[:5] - label_offset) + prediction_prob.append([out_probs[pid] for pid in idx[:5]]) + + # Return the top 5 predictions (idx and prob) for each image. + return prediction_idx, prediction_prob + + def eval_example_images(self, + ckpt_dir, + image_files, + labels_map_file, + enable_ema=True, + export_ckpt=None): + """Eval a list of example images. + + Args: + ckpt_dir: str. Checkpoint directory path. + image_files: List[str]. A list of image file paths. + labels_map_file: str. The labels map file path. + enable_ema: enable expotential moving average. + export_ckpt: export ckpt folder. + + Returns: + A tuple (pred_idx, and pred_prob), where pred_idx is the top 5 prediction + index and pred_prob is the top 5 prediction probability. + """ + classes = json.loads(tf.gfile.Open(labels_map_file).read()) + pred_idx, pred_prob = self.run_inference( + ckpt_dir, image_files, [0] * len(image_files), enable_ema, export_ckpt) + for i in range(len(image_files)): + print('predicted class for image {}: '.format(image_files[i])) + for j, idx in enumerate(pred_idx[i]): + print(' -> top_{} ({:4.2f}%): {} '.format(j, pred_prob[i][j] * 100, + classes[str(idx)])) + return pred_idx, pred_prob + + def eval_imagenet(self, ckpt_dir, imagenet_eval_glob, + imagenet_eval_label, num_images, enable_ema, export_ckpt): + """Eval ImageNet images and report top1/top5 accuracy. + + Args: + ckpt_dir: str. Checkpoint directory path. + imagenet_eval_glob: str. File path glob for all eval images. + imagenet_eval_label: str. File path for eval label. + num_images: int. Number of images to eval: -1 means eval the whole + dataset. + enable_ema: enable expotential moving average. + export_ckpt: export checkpoint folder. + + Returns: + A tuple (top1, top5) for top1 and top5 accuracy. + """ + imagenet_val_labels = [int(i) + for i in tf.gfile.GFile(imagenet_eval_label)] + imagenet_filenames = sorted(tf.gfile.Glob(imagenet_eval_glob)) + if num_images < 0: + num_images = len(imagenet_filenames) + image_files = imagenet_filenames[:num_images] + labels = imagenet_val_labels[:num_images] + + pred_idx, _ = self.run_inference( + ckpt_dir, image_files, labels, enable_ema, export_ckpt) + top1_cnt, top5_cnt = 0.0, 0.0 + for i, label in enumerate(labels): + top1_cnt += label in pred_idx[i][:1] + top5_cnt += label in pred_idx[i][:5] + if i % 100 == 0: + print('Step {}: top1_acc = {:4.2f}% top5_acc = {:4.2f}%'.format( + i, 100 * top1_cnt / (i + 1), 100 * top5_cnt / (i + 1))) + sys.stdout.flush() + top1, top5 = 100 * top1_cnt / num_images, 100 * top5_cnt / num_images + print( + 'Final: top1_acc = {:4.2f}% top5_acc = {:4.2f}%'.format(top1, top5)) + return top1, top5