diff --git a/TensorFlow/contrib/cv/Discriminate_Loss_Function_ID1093_for_TensorFlow/.keep b/TensorFlow/contrib/cv/Discriminate_Loss_Function_ID1093_for_TensorFlow/.keep new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/TensorFlow/contrib/cv/Discriminate_Loss_Function_ID1093_for_TensorFlow/LICENSE b/TensorFlow/contrib/cv/Discriminate_Loss_Function_ID1093_for_TensorFlow/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..29f81d812f3e768fa89638d1f72920dbfd1413a8 --- /dev/null +++ b/TensorFlow/contrib/cv/Discriminate_Loss_Function_ID1093_for_TensorFlow/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/TensorFlow/contrib/cv/Discriminate_Loss_Function_ID1093_for_TensorFlow/README.md b/TensorFlow/contrib/cv/Discriminate_Loss_Function_ID1093_for_TensorFlow/README.md new file mode 100644 index 0000000000000000000000000000000000000000..dc100e1510824821dc7e08584bdc68cfb736a230 --- /dev/null +++ b/TensorFlow/contrib/cv/Discriminate_Loss_Function_ID1093_for_TensorFlow/README.md @@ -0,0 +1,48 @@ +GPU代码位置：https://github.com/hq-jiang/instance-segmentation-with-discriminative-loss-tensorflow + + +NPU代码： + +Files + +├── __data__ here the data should be stored +│        └── __tusimple_dataset_processing.py__ processes the TuSimple dataset +├── __doc__ documentation +├── __inference_test__ inference related data +│        └── __images__ for testing the inference +├── __trained_model__ pretrained model for finetuning +├── __clustering.py__ mean-shift clustering +├── __datagenerator.py__ feeds data for training and evaluation +├── __enet.py__ [Enet architecture](https://github.com/kwotsin/TensorFlow-ENet) +├── __inference.py__ tests inference on images +├── __loss.py__ defines discriminative loss function +├── __README.md__ +├── __training.py__ contains training pipeline +├── __utils.py__ contains utilities files for building and initializing the graph +└── __visualization.py__ contains visualization of the clustering and pixel embeddings + + +参数：默认参数 + +数据集：http://benchmark.tusimple.ai/#/t/1 + +训练命令：python training.py + +GPU 训练时间：168小时 +NPU 训练时间：24小时 + +GPU 精度：https://cann001.obs.cn-north-4.myhuaweicloud.com/log.1217215.out + +step_valid 18150 valid loss 0.09118794 valid l_var 0.07415567 valid l_dist 0.0039632297 valid l_reg 0.013069032 + +NPU 精度：https://cann001.obs.cn-north-4.myhuaweicloud.com/modelarts-job-4c06ae91-0a9b-4c9c-a9f6-4e32fe5c0bd7-proc-rank-0-device-0.txt + +step_valid 18150 valid loss 0.074521884 valid l_var 0.057265442 valid l_dist 0.0004725394 valid l_reg 0.016783904 + +ckpt转pb：https://gitee.com/ascend/modelzoo/blob/master/built-in/TensorFlow/Research/cv/image_segmentation/UNet3D_ID0057_for_TensorFlow/pbinference/unet3d_pb_frozen.py + +pb转om：atc --model=/home/HwHiAiUser/AscendProjects/NGNN/pb_model/model.ckpt-163150.pb --framework=3 --output=/home/HwHiAiUser/AscendProjects/NGNN/ngnn_acc --soc_version=Ascend310 --input_shape="Placeholder:1,512,512,3" + +离线推理命令：./msame --model /home/HwHiAiUser/AscendProjects/tools/msame/ngnn_acc.om --input /home/HwHiAiUser/AscendProjects/tools/msame/982.bin --output /home/HwHiAiUser/ljj/AMEXEC/out/output1 --outfmt TXT --loop 2 + +离线推理结果：https://cann001.obs.cn-north-4.myhuaweicloud.com/ngnn_acc_output_0.txt diff --git a/TensorFlow/contrib/cv/Discriminate_Loss_Function_ID1093_for_TensorFlow/__init__.py b/TensorFlow/contrib/cv/Discriminate_Loss_Function_ID1093_for_TensorFlow/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/TensorFlow/contrib/cv/Discriminate_Loss_Function_ID1093_for_TensorFlow/clustering.py b/TensorFlow/contrib/cv/Discriminate_Loss_Function_ID1093_for_TensorFlow/clustering.py new file mode 100644 index 0000000000000000000000000000000000000000..b23cda2a8fd0e4d3e96ad2ab426557e6378ebfdc --- /dev/null +++ b/TensorFlow/contrib/cv/Discriminate_Loss_Function_ID1093_for_TensorFlow/clustering.py @@ -0,0 +1,73 @@ +from npu_bridge.npu_init import * +import os +import numpy as np +from sklearn.cluster import MeanShift, estimate_bandwidth +import time +import cv2 + +COLOR=[np.array([255,0,0]), + np.array([0,255,0]), + np.array([0,0,255]), + np.array([125,125,0]), + np.array([0,125,125]), + np.array([125,0,125]), + np.array([50,100,50]), + np.array([100,50,100])] + +def cluster(prediction, bandwidth): + ms = MeanShift(bandwidth, bin_seeding=True) + print ('Mean shift clustering, might take some time ...') + tic = time.time() + ms.fit(prediction) + print ('time for clustering', time.time() - tic) + labels = ms.labels_ + cluster_centers = ms.cluster_centers_ + + num_clusters = cluster_centers.shape[0] + + return num_clusters, labels, cluster_centers + +def get_instance_masks(prediction, bandwidth): + batch_size, h, w, feature_dim = prediction.shape + + instance_masks = [] + for i in range(batch_size): + num_clusters, labels, cluster_centers = cluster(prediction[i].reshape([h*w, feature_dim]), bandwidth) + print ('Number of predicted clusters', num_clusters) + labels = np.array(labels, dtype=np.uint8).reshape([h,w]) + mask = np.zeros([h,w,3], dtype=np.uint8) + + num_clusters = min([num_clusters,8]) + for mask_id in range(num_clusters): + ind = np.where(labels==mask_id) + mask[ind] = COLOR[mask_id] + + + instance_masks.append(mask) + + return instance_masks + + +def save_instance_masks(prediction,output_dir, bandwidth, count): + batch_size, h, w, feature_dim = prediction.shape + + instance_masks = [] + for i in range(batch_size): + num_clusters, labels, cluster_centers = cluster(prediction[i].reshape([h*w, feature_dim]), bandwidth) + print ('Number of predicted clusters', num_clusters) + labels = np.array(labels, dtype=np.uint8).reshape([h,w]) + mask = np.zeros([h,w,3], dtype=np.uint8) + + num_clusters = min([num_clusters,8]) + for mask_id in range(num_clusters): + mask = np.zeros([h,w,3], dtype=np.uint8) + ind = np.where(labels==mask_id) + mask[ind] = np.array([255,255,255]) + output_file_name = os.path.join(output_dir, 'cluster_{}_{}.png'.format(str(count).zfill(4), str(mask_id))) + cv2.imwrite(output_file_name, mask) + + + instance_masks.append(mask) + + return instance_masks + diff --git a/TensorFlow/contrib/cv/Discriminate_Loss_Function_ID1093_for_TensorFlow/data/.keep b/TensorFlow/contrib/cv/Discriminate_Loss_Function_ID1093_for_TensorFlow/data/.keep new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/TensorFlow/contrib/cv/Discriminate_Loss_Function_ID1093_for_TensorFlow/data/tusimple_dataset_processing.py b/TensorFlow/contrib/cv/Discriminate_Loss_Function_ID1093_for_TensorFlow/data/tusimple_dataset_processing.py new file mode 100644 index 0000000000000000000000000000000000000000..d16d45eee487105170a50849741a9c5a43f948be --- /dev/null +++ b/TensorFlow/contrib/cv/Discriminate_Loss_Function_ID1093_for_TensorFlow/data/tusimple_dataset_processing.py @@ -0,0 +1,140 @@ +from npu_bridge.npu_init import * +import os +import json +import csv +import glob +import argparse +import math +import matplotlib.pyplot as plt +from tqdm import tqdm +import cv2 +import numpy as np + + +def read_json(data_dir, json_string): + print('xxx', data_dir) + json_paths = glob.glob(os.path.join(data_dir, json_string)) + print('sss', json_paths) + data = [] + for path in json_paths: + with open(path) as f: + d = (line.strip() for line in f) + d_str = "[{0}]".format(','.join(d)) + data.append(json.loads(d_str)) + + num_samples = 0 + for d in data: + num_samples += len(d) + print('Number of labeled images:', num_samples) + #print('data keys:', data[0][0].keys()) + + return data + + +def read_image_strings(data, input_dir): + img_paths = [] + for datum in data: + for d in datum: + path = os.path.join(input_dir, d['raw_file']) + img_paths.append(path) + + num_samples = 0 + for d in data: + num_samples += len(d) + assert len(img_paths) == num_samples, 'Number of samples do not match' + print(img_paths[0:2]) + + return img_paths + + +def save_input_images(output_dir, img_paths): + output_path2 = os.path.join(output_dir, 'images') + os.makedirs(output_path2) + for i, path in tqdm(enumerate(img_paths), total=len(img_paths)): + img = cv2.imread(path) + output_path = os.path.join(output_path2, '{}.png'.format(str(i).zfill(4))) + cv2.imwrite(output_path, img) + #mox.file.copy(output_path, 's3://cann001/dataset/dataset/') + + +def draw_lines(img, lanes, height, instancewise=False): + for i, lane in enumerate(lanes): + pts = [[x, y] for x, y in zip(lane, height) if (x != -2 and y != -2)] + pts = np.array([pts]) + if not instancewise: + cv2.polylines(img, pts, False, 255, thickness=7) + else: + cv2.polylines(img, pts, False, 50 * i + 20, thickness=7) + + +def draw_single_line(img, lane, height): + pts = [[x, y] for x, y in zip(lane, height) if (x != -2 and y != -2)] + pts = np.array([pts]) + cv2.polylines(img, pts, False, 255, thickness=15) + + +def save_label_images(output_dir, data, instancewise=True): + counter = 0 + output_path1 = os.path.join(output_dir, 'labels') + os.makedirs(output_path1) + for i in range(len(data)): + for j in tqdm(range(len(data[i]))): + img = np.zeros([720, 1280], dtype=np.uint8) + lanes = data[i][j]['lanes'] + height = data[i][j]['h_samples'] + draw_lines(img, lanes, height, instancewise) + output_path = os.path.join(output_path1, '{}.png'.format(str(counter).zfill(4))) + cv2.imwrite(output_path, img) + counter += 1 + #mox.file.copy(output_path, 's3://cann001/dataset/dataset/') + + +if __name__ == '__main__': + + print('begin to run') + + import moxing as mox + parser = argparse.ArgumentParser() + #parser.add_argument('srcdir', help="Source directory of TuSimple dataset") + parser.add_argument('--srcdir', type=str, default='s3://cann001/dataset/train_set', help="Source directory of TuSimple dataset") + parser.add_argument('--output', type=str, default='../dataset') + + parser.add_argument("--train_url0", type=str, default="s3://cann001/dataset/dataset/") + local_dir = '/cache/11' + local_dir2 = '/cache/22' + os.makedirs(local_dir2) + + #args = parser.parse_args() + args, unkown = parser.parse_known_args() + mox.file.copy_parallel(args.srcdir, local_dir) + + print('node2') + ''' + if not os.path.isdir(args.srcdir): + raise IOError('Directory does not exist') + + if not os.path.isdir('images'): + os.mkdir('images') + if not os.path.isdir('labels'): + os.mkdir('labels') + ''' + json_string = 'label_data_*.json' + data = read_json(local_dir, json_string) + img_paths = read_image_strings(data, local_dir) + print('dddd', data) + print('ppppp', img_paths) + save_input_images(local_dir2, img_paths) + save_label_images(local_dir2, data) + p = os.listdir(local_dir2) + print(p) + print('Yes') + ''' + opdir = os.path.join(args.train_url, 'result') + if not mox.file.exists(opdir): + mox.file.make_dirs(opdir) + #opdir = 's3://cann001/dataset/dataset/' + ''' + #local_dir = os.path.join(local_dir, 'images') + mox.file.copy_parallel(local_dir2, args.train_url0) + print('123456') + #mox.file.copy_parallel(local_dir2, args.train_url0) diff --git a/TensorFlow/contrib/cv/Discriminate_Loss_Function_ID1093_for_TensorFlow/datagenerator.py b/TensorFlow/contrib/cv/Discriminate_Loss_Function_ID1093_for_TensorFlow/datagenerator.py new file mode 100644 index 0000000000000000000000000000000000000000..82912931266f8da3d736d792b39397bdb4aea5e4 --- /dev/null +++ b/TensorFlow/contrib/cv/Discriminate_Loss_Function_ID1093_for_TensorFlow/datagenerator.py @@ -0,0 +1,86 @@ +from npu_bridge.npu_init import * +import os +from glob import glob +import numpy as np +import scipy.misc +import random +from sklearn.utils import shuffle +import shutil +import time +import tensorflow as tf +import cv2 +import matplotlib.pyplot as plt + + +### Mean and std deviation for whole training data set (RGB format) +mean = np.array([92.14031982, 103.20146942, 103.47182465]) +std = np.array([49.157, 54.9057, 59.4065]) + +INSTANCE_COLORS = [np.array([0,0,0]), + np.array([20.,20.,20.]), + np.array([70.,70.,70.]), + np.array([120.,120.,120.]), + np.array([170.,170.,170.]), + np.array([220.,220.,220.]) + ] + +def get_batches_fn(batch_size, image_shape, image_paths, label_paths): + """ + Create batches of training data + :param batch_size: Batch Size + :return: Batches of training data + """ + + #print ('Number of total labels:', len(label_paths)) + assert len(image_paths)==len(label_paths), 'Number of images and labels do not match' + + image_paths.sort() + label_paths.sort() + + #image_paths = image_paths[:10] + #label_paths = label_paths[:10] + + image_paths, label_paths = shuffle(image_paths, label_paths) + for batch_i in range(0, len(image_paths), batch_size): + images = [] + gt_images = [] + for image_file, gt_image_file in zip(image_paths[batch_i:batch_i+batch_size], label_paths[batch_i:batch_i+batch_size]): + + image = cv2.resize(cv2.imread(image_file), image_shape, interpolation=cv2.INTER_LINEAR) + image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) + #image = (image.astype(np.float32)-mean)/std + + gt_image = cv2.imread(gt_image_file, cv2.IMREAD_COLOR) + gt_image = cv2.resize(gt_image[:,:,0], image_shape, interpolation=cv2.INTER_NEAREST) + + images.append(image) + gt_images.append(gt_image) + + yield np.array(images), np.array(gt_images) + + +def get_validation_batch(data_dir, image_shape): + valid_image_paths = [os.path.join(data_dir,'images','0000.png')] + + valid_label_paths = [os.path.join(data_dir,'labels','0000.png')] + + images = [] + gt_images = [] + for image_file, gt_image_file in zip(valid_image_paths, valid_label_paths): + + image = cv2.resize(cv2.imread(image_file), image_shape, interpolation=cv2.INTER_LINEAR) + image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) + #image = (image.astype(np.float32)-mean)/std + + gt_image = cv2.imread(gt_image_file, cv2.IMREAD_COLOR) + gt_image = cv2.resize(gt_image[:,:,0], image_shape, interpolation=cv2.INTER_NEAREST) + + images.append(image) + gt_images.append(gt_image) + + return np.array(images), np.array(gt_images) + + + +if __name__=="__main__": + pass diff --git a/TensorFlow/contrib/cv/Discriminate_Loss_Function_ID1093_for_TensorFlow/demo.py b/TensorFlow/contrib/cv/Discriminate_Loss_Function_ID1093_for_TensorFlow/demo.py new file mode 100644 index 0000000000000000000000000000000000000000..58d15eea03976f983688484f1f97484e722a597b --- /dev/null +++ b/TensorFlow/contrib/cv/Discriminate_Loss_Function_ID1093_for_TensorFlow/demo.py @@ -0,0 +1,49 @@ +# coding=utf-8 +from npu_bridge.npu_init import * +import os +import argparse +import datetime +import moxing as mox + +## Code dir: /home/work/user-job-dir/code # ModelArtsϵĴ洢Ŀ¼Ŀ¼ᱻΪcode +## Work dir: /home/work/workspace/device2 # device idjob +print("===>>>{}".format(os.getcwd())) +print(os.system('env')) + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument("--train_url", type=str, default="../output") # PyCharm OBS Path· + parser.add_argument("--data_url", type=str, default="../dataset") # PyCharm Data Path in OBS· + config = parser.parse_args() + + # Ϊ˷㣬ǻὫѵʱõݼ /cache Ŀ¼£ImageNet TFRecordʽ148GҪԼ280sΪСļݼΪtarļ󿽱ȥٽѹ + # copy dataset from obs to local + # dataset will be saved under /cache/ilsvrc2012_tfrecord while the results will be saved under /cache/results + local_dir = '/cache/ilsvrc2012_tfrecord' + start = datetime.datetime.now() + print("===>>>Copy files from obs:{} to local dir:{}".format(config.data_url, local_dir)) + mox.file.copy_parallel(src_url=config.data_url, dst_url=local_dir) + end = datetime.datetime.now() + print("===>>>Copy from obs to local, time use:{}(s)".format((end - start).seconds)) + files = os.listdir(local_dir) + print("===>>>Files number:", len(files)) + + # ʼѵűֻҪѵűеdataset pathĬָΪ/cacheеӦݼ·ɣͬʱѵlog, snapshotļҲд/cacheµijһضļУع̬дÿηobsҪ죬ҪڴmoxȱֶkillͲᱣмԶʱcopyһ¡ + # run training + print("===>>>Begin training:") + os.system('bash /home/work/user-job-dir/code/run_1p.sh') # ʾľѵűΪrun_1p.sh + print("===>>>Training finished:") + + # ѵҪмobsĿobs·Ϊ֮ǰ--train_url + # copy results from local to obs + local_dir = '/cache/result' + remote_dir = os.path.join(config.train_url, 'result') + if not mox.file.exists(remote_dir): + mox.file.make_dirs(remote_dir) + start = datetime.datetime.now() + print("===>>>Copy files from local dir:{} to obs:{}".format(local_dir, remote_dir)) + mox.file.copy_parallel(src_url=local_dir, dst_url=remote_dir) + end = datetime.datetime.now() + print("===>>>Copy from local to obs, time use:{}(s)".format((end - start).seconds)) + files = os.listdir(local_dir) + print("===>>>Files number:", len(files)) diff --git a/TensorFlow/contrib/cv/Discriminate_Loss_Function_ID1093_for_TensorFlow/doc/.keep b/TensorFlow/contrib/cv/Discriminate_Loss_Function_ID1093_for_TensorFlow/doc/.keep new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/TensorFlow/contrib/cv/Discriminate_Loss_Function_ID1093_for_TensorFlow/doc/cluster_000000.png b/TensorFlow/contrib/cv/Discriminate_Loss_Function_ID1093_for_TensorFlow/doc/cluster_000000.png new file mode 100644 index 0000000000000000000000000000000000000000..92a2f65e05ce1b7c2e59364d46b26e41860e593d Binary files /dev/null and b/TensorFlow/contrib/cv/Discriminate_Loss_Function_ID1093_for_TensorFlow/doc/cluster_000000.png differ diff --git a/TensorFlow/contrib/cv/Discriminate_Loss_Function_ID1093_for_TensorFlow/doc/cluster_001000.png b/TensorFlow/contrib/cv/Discriminate_Loss_Function_ID1093_for_TensorFlow/doc/cluster_001000.png new file mode 100644 index 0000000000000000000000000000000000000000..a39f6add4b76e4b84da0555faff94bb3acf781a7 Binary files /dev/null and b/TensorFlow/contrib/cv/Discriminate_Loss_Function_ID1093_for_TensorFlow/doc/cluster_001000.png differ diff --git a/TensorFlow/contrib/cv/Discriminate_Loss_Function_ID1093_for_TensorFlow/doc/cluster_002000.png b/TensorFlow/contrib/cv/Discriminate_Loss_Function_ID1093_for_TensorFlow/doc/cluster_002000.png new file mode 100644 index 0000000000000000000000000000000000000000..7988ab36c560fb367f698d6e2b8cd2054794b81a Binary files /dev/null and b/TensorFlow/contrib/cv/Discriminate_Loss_Function_ID1093_for_TensorFlow/doc/cluster_002000.png differ diff --git a/TensorFlow/contrib/cv/Discriminate_Loss_Function_ID1093_for_TensorFlow/doc/cluster_012000.png b/TensorFlow/contrib/cv/Discriminate_Loss_Function_ID1093_for_TensorFlow/doc/cluster_012000.png new file mode 100644 index 0000000000000000000000000000000000000000..569f1dddea3a8744e2468530809caea5303a4143 Binary files /dev/null and b/TensorFlow/contrib/cv/Discriminate_Loss_Function_ID1093_for_TensorFlow/doc/cluster_012000.png differ diff --git a/TensorFlow/contrib/cv/Discriminate_Loss_Function_ID1093_for_TensorFlow/doc/cluster_015000.png b/TensorFlow/contrib/cv/Discriminate_Loss_Function_ID1093_for_TensorFlow/doc/cluster_015000.png new file mode 100644 index 0000000000000000000000000000000000000000..0449ff14949e879b8bc1521368a2b299959b5c1a Binary files /dev/null and b/TensorFlow/contrib/cv/Discriminate_Loss_Function_ID1093_for_TensorFlow/doc/cluster_015000.png differ diff --git a/TensorFlow/contrib/cv/Discriminate_Loss_Function_ID1093_for_TensorFlow/doc/cluster_016000.png b/TensorFlow/contrib/cv/Discriminate_Loss_Function_ID1093_for_TensorFlow/doc/cluster_016000.png new file mode 100644 index 0000000000000000000000000000000000000000..b97ec8e2f06b8f899ac672df8dce72a02fc5ee76 Binary files /dev/null and b/TensorFlow/contrib/cv/Discriminate_Loss_Function_ID1093_for_TensorFlow/doc/cluster_016000.png differ diff --git a/TensorFlow/contrib/cv/Discriminate_Loss_Function_ID1093_for_TensorFlow/doc/cluster_017000.png b/TensorFlow/contrib/cv/Discriminate_Loss_Function_ID1093_for_TensorFlow/doc/cluster_017000.png new file mode 100644 index 0000000000000000000000000000000000000000..815272fa8ba7bfbb5a56e21600202627cad800c2 Binary files /dev/null and b/TensorFlow/contrib/cv/Discriminate_Loss_Function_ID1093_for_TensorFlow/doc/cluster_017000.png differ diff --git a/TensorFlow/contrib/cv/Discriminate_Loss_Function_ID1093_for_TensorFlow/doc/cluster_018000.png b/TensorFlow/contrib/cv/Discriminate_Loss_Function_ID1093_for_TensorFlow/doc/cluster_018000.png new file mode 100644 index 0000000000000000000000000000000000000000..cf1171b66c6f9c3806055a02c90f0b660e6a5508 Binary files /dev/null and b/TensorFlow/contrib/cv/Discriminate_Loss_Function_ID1093_for_TensorFlow/doc/cluster_018000.png differ diff --git a/TensorFlow/contrib/cv/Discriminate_Loss_Function_ID1093_for_TensorFlow/doc/training_pipeline.png b/TensorFlow/contrib/cv/Discriminate_Loss_Function_ID1093_for_TensorFlow/doc/training_pipeline.png new file mode 100644 index 0000000000000000000000000000000000000000..0a052013f6514c2d057363eaf1c18a21c6bb249c Binary files /dev/null and b/TensorFlow/contrib/cv/Discriminate_Loss_Function_ID1093_for_TensorFlow/doc/training_pipeline.png differ diff --git a/TensorFlow/contrib/cv/Discriminate_Loss_Function_ID1093_for_TensorFlow/enet.py b/TensorFlow/contrib/cv/Discriminate_Loss_Function_ID1093_for_TensorFlow/enet.py new file mode 100644 index 0000000000000000000000000000000000000000..e09c96cea4fd6e852aa296bd406ae316521ffd39 --- /dev/null +++ b/TensorFlow/contrib/cv/Discriminate_Loss_Function_ID1093_for_TensorFlow/enet.py @@ -0,0 +1,524 @@ +# -*- coding: utf-8 -*- +from npu_bridge.npu_init import * +import tensorflow +import tensorflow.compat.v1 as tf + +from npu_bridge.estimator import npu_ops +tf.disable_v2_behavior() +from tensorflow.contrib.layers.python.layers import initializers +slim = tensorflow.contrib.slim + +''' +============================================================================ +ENet: A Deep Neural Network Architecture for Real-Time Semantic Segmentation +============================================================================ +Based on the paper: https://arxiv.org/pdf/1606.02147.pdf +''' +@slim.add_arg_scope +def prelu(x, scope, decoder=False): + ''' + Performs the parametric relu operation. This implementation is based on: + https://stackoverflow.com/questions/39975676/how-to-implement-prelu-activation-in-tensorflow + + For the decoder portion, prelu becomes just a normal prelu + + INPUTS: + - x(Tensor): a 4D Tensor that undergoes prelu + - scope(str): the string to name your prelu operation's alpha variable. + - decoder(bool): if True, prelu becomes a normal relu. + + OUTPUTS: + - pos + neg / x (Tensor): gives prelu output only during training; otherwise, just return x. + + ''' + #If decoder, then perform relu and just return the output + if decoder: + return tf.nn.relu(x, name=scope) + + alpha= tf.get_variable(scope + 'alpha', x.get_shape()[-1], + initializer=tf.constant_initializer(0.0), + dtype=tf.float32) + pos = tf.nn.relu(x) + neg = alpha * (x - abs(x)) * 0.5 + return pos + neg + +def spatial_dropout(x, p, seed, scope, is_training=True): + ''' + Performs a 2D spatial dropout that drops layers instead of individual elements in an input feature map. + Note that p stands for the probability of dropping, but tf.nn.relu uses probability of keeping. + + ------------------ + Technical Details + ------------------ + The noise shape must be of shape [batch_size, 1, 1, num_channels], with the height and width set to 1, because + it will represent either a 1 or 0 for each layer, and these 1 or 0 integers will be broadcasted to the entire + dimensions of each layer they interact with such that they can decide whether each layer should be entirely + 'dropped'/set to zero or have its activations entirely kept. + -------------------------- + + INPUTS: + - x(Tensor): a 4D Tensor of the input feature map. + - p(float): a float representing the probability of dropping a layer + - seed(int): an integer for random seeding the random_uniform distribution that runs under tf.nn.relu + - scope(str): the string name for naming the spatial_dropout + - is_training(bool): to turn on dropout only when training. Optional. + + OUTPUTS: + - output(Tensor): a 4D Tensor that is in exactly the same size as the input x, + with certain layers having their elements all set to 0 (i.e. dropped). + ''' + if is_training: + keep_prob = 1.0 - p + input_shape = x.get_shape().as_list() + noise_shape = tf.constant(value=[input_shape[0], 1, 1, input_shape[3]]) + output = tf.nn.dropout(x, keep_prob, noise_shape, seed=seed, name=scope) + #output = npu_ops.dropout(x, keep_prob, noise_shape, seed=seed, name=scope) + + return output + + return x + +def unpool(updates, mask, k_size=[1, 2, 2, 1], output_shape=None, scope=''): + ''' + Unpooling function based on the implementation by Panaetius at https://github.com/tensorflow/tensorflow/issues/2169 + + INPUTS: + - inputs(Tensor): a 4D tensor of shape [batch_size, height, width, num_channels] that represents the input block to be upsampled + - mask(Tensor): a 4D tensor that represents the argmax values/pooling indices of the previously max-pooled layer + - k_size(list): a list of values representing the dimensions of the unpooling filter. + - output_shape(list): a list of values to indicate what the final output shape should be after unpooling + - scope(str): the string name to name your scope + + OUTPUTS: + - ret(Tensor): the returned 4D tensor that has the shape of output_shape. + + ''' + with tf.variable_scope(scope): + mask = tf.cast(mask, tf.int32) + input_shape = tf.shape(updates, out_type=tf.int32) + # calculation new shape + if output_shape is None: + output_shape = (input_shape[0], input_shape[1] * ksize[1], input_shape[2] * ksize[2], input_shape[3]) + + # calculation indices for batch, height, width and feature maps + one_like_mask = tf.ones_like(mask, dtype=tf.int32) + batch_shape = tf.concat([[input_shape[0]], [1], [1], [1]], 0) + batch_range = tf.reshape(tf.range(output_shape[0], dtype=tf.int32), shape=batch_shape) + b = one_like_mask * batch_range + y = mask // (output_shape[2] * output_shape[3]) + x = (mask // output_shape[3]) % output_shape[2] #mask % (output_shape[2] * output_shape[3]) // output_shape[3] + feature_range = tf.range(output_shape[3], dtype=tf.int32) + f = one_like_mask * feature_range + + # transpose indices & reshape update values to one dimension + updates_size = tf.size(updates) + indices = tf.transpose(tf.reshape(tf.stack([b, y, x, f]), [4, updates_size])) + values = tf.reshape(updates, [updates_size]) + ret = tf.scatter_nd(indices, values, output_shape) + return ret + +@slim.add_arg_scope +def initial_block(inputs, is_training=True, scope='initial_block'): + ''' + The initial block for Enet has 2 branches: The convolution branch and Maxpool branch. + + The conv branch has 13 layers, while the maxpool branch gives 3 layers corresponding to the RGB channels. + Both output layers are then concatenated to give an output of 16 layers. + + NOTE: Does not need to store pooling indices since it won't be used later for the final upsampling. + + INPUTS: + - inputs(Tensor): A 4D tensor of shape [batch_size, height, width, channels] + + OUTPUTS: + - net_concatenated(Tensor): a 4D Tensor that contains the + ''' + #Convolutional branch + net_conv = slim.conv2d(inputs, 13, [3,3], stride=2, activation_fn=None, scope=scope+'_conv') + net_conv = slim.batch_norm(net_conv, is_training=is_training, fused=True, scope=scope+'_batchnorm') + net_conv = prelu(net_conv, scope=scope+'_prelu') + + #Max pool branch + net_pool = slim.max_pool2d(inputs, [2,2], stride=2, scope=scope+'_max_pool') + + #Concatenated output - does it matter max pool comes first or conv comes first? probably not. + net_concatenated = tf.concat([net_conv, net_pool], axis=3, name=scope+'_concat') + return net_concatenated + +@slim.add_arg_scope +def bottleneck(inputs, + output_depth, + filter_size, + regularizer_prob, + projection_ratio=4, + seed=0, + is_training=True, + downsampling=False, + upsampling=False, + pooling_indices=None, + output_shape=None, + dilated=False, + dilation_rate=None, + asymmetric=False, + decoder=False, + scope='bottleneck'): + ''' + The bottleneck module has three different kinds of variants: + + 1. A regular convolution which you can decide whether or not to downsample. + 2. A dilated convolution, which requires you to have a dilation factor. + 3. An asymmetric convolution that has a decomposed filter size of 5x1 and 1x5 separately. + + INPUTS: + - inputs(Tensor): a 4D Tensor of the previous convolutional block of shape [batch_size, height, width, num_channels]. + - output_depth(int): an integer indicating the output depth of the output convolutional block. + - filter_size(int): an integer that gives the height and width of the filter size to use for a regular/dilated convolution. + - regularizer_prob(float): the float p that represents the prob of dropping a layer for spatial dropout regularization. + - projection_ratio(int): the amount of depth to reduce for initial 1x1 projection. Depth is divided by projection ratio. Default is 4. + - seed(int): an integer for the random seed used in the random normal distribution within dropout. + - is_training(bool): a boolean value to indicate whether or not is training. Decides batch_norm and prelu activity. + + - downsampling(bool): if True, a max-pool2D layer is added to downsample the spatial sizes. + - upsampling(bool): if True, the upsampling bottleneck is activated but requires pooling indices to upsample. + - pooling_indices(Tensor): the argmax values that are obtained after performing tf.nn.max_pool_with_argmax. + - output_shape(list): A list of integers indicating the output shape of the unpooling layer. + - dilated(bool): if True, then dilated convolution is done, but requires a dilation rate to be given. + - dilation_rate(int): the dilation factor for performing atrous convolution/dilated convolution. + - asymmetric(bool): if True, then asymmetric convolution is done, and the only filter size used here is 5. + - decoder(bool): if True, then all the prelus become relus according to ENet author. + - scope(str): a string name that names your bottleneck. + + OUTPUTS: + - net(Tensor): The convolution block output after a bottleneck + - pooling_indices(Tensor): If downsample, then this tensor is produced for use in upooling later. + - inputs_shape(list): The shape of the input to the downsampling conv block. For use in unpooling later. + + ''' + #Calculate the depth reduction based on the projection ratio used in 1x1 convolution. + reduced_depth = int(inputs.get_shape().as_list()[3] / projection_ratio) + + with slim.arg_scope([prelu], decoder=decoder): + + #=============DOWNSAMPLING BOTTLENECK==================== + if downsampling: + #=============MAIN BRANCH============= + #Just perform a max pooling + net_main, pooling_indices = tf.nn.max_pool_with_argmax(inputs, + ksize=[1,2,2,1], + strides=[1,2,2,1], + padding='SAME', + name=scope+'_main_max_pool') + + #First get the difference in depth to pad, then pad with zeros only on the last dimension. + inputs_shape = inputs.get_shape().as_list() + depth_to_pad = abs(inputs_shape[3] - output_depth) + paddings = tf.convert_to_tensor([[0,0], [0,0], [0,0], [0, depth_to_pad]]) + net_main = tf.pad(net_main, paddings=paddings, name=scope+'_main_padding') + + #=============SUB BRANCH============== + #First projection that has a 2x2 kernel and stride 2 + net = slim.conv2d(inputs, reduced_depth, [2,2], stride=2, scope=scope+'_conv1') + net = slim.batch_norm(net, is_training=is_training, scope=scope+'_batch_norm1') + net = prelu(net, scope=scope+'_prelu1') + + #Second conv block + net = slim.conv2d(net, reduced_depth, [filter_size, filter_size], scope=scope+'_conv2') + net = slim.batch_norm(net, is_training=is_training, scope=scope+'_batch_norm2') + net = prelu(net, scope=scope+'_prelu2') + + #Final projection with 1x1 kernel + net = slim.conv2d(net, output_depth, [1,1], scope=scope+'_conv3') + net = slim.batch_norm(net, is_training=is_training, scope=scope+'_batch_norm3') + net = prelu(net, scope=scope+'_prelu3') + + #Regularizer + net = spatial_dropout(net, p=regularizer_prob, seed=seed, scope=scope+'_spatial_dropout') + + #Finally, combine the two branches together via an element-wise addition + net = tf.add(net, net_main, name=scope+'_add') + net = prelu(net, scope=scope+'_last_prelu') + + #also return inputs shape for convenience later + return net, pooling_indices, inputs_shape + + #============DILATION CONVOLUTION BOTTLENECK==================== + #Everything is the same as a regular bottleneck except for the dilation rate argument + elif dilated: + #Check if dilation rate is given + if not dilation_rate: + raise ValueError('Dilation rate is not given.') + + #Save the main branch for addition later + net_main = inputs + + #First projection with 1x1 kernel (dimensionality reduction) + net = slim.conv2d(inputs, reduced_depth, [1,1], scope=scope+'_conv1') + net = slim.batch_norm(net, is_training=is_training, scope=scope+'_batch_norm1') + net = prelu(net, scope=scope+'_prelu1') + + #Second conv block --- apply dilated convolution here + net = slim.conv2d(net, reduced_depth, [filter_size, filter_size], rate=dilation_rate, scope=scope+'_dilated_conv2') + net = slim.batch_norm(net, is_training=is_training, scope=scope+'_batch_norm2') + net = prelu(net, scope=scope+'_prelu2') + + #Final projection with 1x1 kernel (Expansion) + net = slim.conv2d(net, output_depth, [1,1], scope=scope+'_conv3') + net = slim.batch_norm(net, is_training=is_training, scope=scope+'_batch_norm3') + net = prelu(net, scope=scope+'_prelu3') + + #Regularizer + net = spatial_dropout(net, p=regularizer_prob, seed=seed, scope=scope+'_spatial_dropout') + net = prelu(net, scope=scope+'_prelu4') + + #Add the main branch + net = tf.add(net_main, net, name=scope+'_add_dilated') + net = prelu(net, scope=scope+'_last_prelu') + + return net + + #===========ASYMMETRIC CONVOLUTION BOTTLENECK============== + #Everything is the same as a regular bottleneck except for a [5,5] kernel decomposed into two [5,1] then [1,5] + elif asymmetric: + #Save the main branch for addition later + net_main = inputs + + #First projection with 1x1 kernel (dimensionality reduction) + net = slim.conv2d(inputs, reduced_depth, [1,1], scope=scope+'_conv1') + net = slim.batch_norm(net, is_training=is_training, scope=scope+'_batch_norm1') + net = prelu(net, scope=scope+'_prelu1') + + #Second conv block --- apply asymmetric conv here + net = slim.conv2d(net, reduced_depth, [filter_size, 1], scope=scope+'_asymmetric_conv2a') + net = slim.conv2d(net, reduced_depth, [1, filter_size], scope=scope+'_asymmetric_conv2b') + net = slim.batch_norm(net, is_training=is_training, scope=scope+'_batch_norm2') + net = prelu(net, scope=scope+'_prelu2') + + #Final projection with 1x1 kernel + net = slim.conv2d(net, output_depth, [1,1], scope=scope+'_conv3') + net = slim.batch_norm(net, is_training=is_training, scope=scope+'_batch_norm3') + net = prelu(net, scope=scope+'_prelu3') + + #Regularizer + net = spatial_dropout(net, p=regularizer_prob, seed=seed, scope=scope+'_spatial_dropout') + net = prelu(net, scope=scope+'_prelu4') + + #Add the main branch + net = tf.add(net_main, net, name=scope+'_add_asymmetric') + net = prelu(net, scope=scope+'_last_prelu') + + return net + + #============UPSAMPLING BOTTLENECK================ + #Everything is the same as a regular one, except convolution becomes transposed. + elif upsampling: + #Check if pooling indices is given + if pooling_indices == None: + raise ValueError('Pooling indices are not given.') + + #Check output_shape given or not + if output_shape == None: + raise ValueError('Output depth is not given') + + #=======MAIN BRANCH======= + #Main branch to upsample. output shape must match with the shape of the layer that was pooled initially, in order + #for the pooling indices to work correctly. However, the initial pooled layer was padded, so need to reduce dimension + #before unpooling. In the paper, padding is replaced with convolution for this purpose of reducing the depth! + net_unpool = slim.conv2d(inputs, output_depth, [1,1], scope=scope+'_main_conv1') + net_unpool = slim.batch_norm(net_unpool, is_training=is_training, scope=scope+'batch_norm1') + net_unpool = unpool(net_unpool, pooling_indices, output_shape=output_shape, scope='unpool') + + #======SUB BRANCH======= + #First 1x1 projection to reduce depth + net = slim.conv2d(inputs, reduced_depth, [1,1], scope=scope+'_conv1') + net = slim.batch_norm(net, is_training=is_training, scope=scope+'_batch_norm2') + net = prelu(net, scope=scope+'_prelu1') + + #Second conv block -----------------------------> NOTE: using tf.nn.conv2d_transpose for variable input shape. + net_unpool_shape = net_unpool.get_shape().as_list() + output_shape = [net_unpool_shape[0], net_unpool_shape[1], net_unpool_shape[2], reduced_depth] + output_shape = tf.convert_to_tensor(output_shape) + filter_size = [filter_size, filter_size, reduced_depth, reduced_depth] + filters = tf.get_variable(shape=filter_size, initializer=initializers.xavier_initializer(), dtype=tf.float32, name=scope+'_transposed_conv2_filters') + + # net = slim.conv2d_transpose(net, reduced_depth, [filter_size, filter_size], stride=2, scope=scope+'_transposed_conv2') + net = tf.nn.conv2d_transpose(net, filter=filters, strides=[1,2,2,1], output_shape=output_shape, name=scope+'_transposed_conv2') + net = slim.batch_norm(net, is_training=is_training, scope=scope+'_batch_norm3') + net = prelu(net, scope=scope+'_prelu2') + + #Final projection with 1x1 kernel + net = slim.conv2d(net, output_depth, [1,1], scope=scope+'_conv3') + net = slim.batch_norm(net, is_training=is_training, scope=scope+'_batch_norm4') + net = prelu(net, scope=scope+'_prelu3') + + #Regularizer + net = spatial_dropout(net, p=regularizer_prob, seed=seed, scope=scope+'_spatial_dropout') + net = prelu(net, scope=scope+'_prelu4') + + #Finally, add the unpooling layer and the sub branch together + net = tf.add(net, net_unpool, name=scope+'_add_upsample') + net = prelu(net, scope=scope+'_last_prelu') + + return net + + #OTHERWISE, just perform a regular bottleneck! + #==============REGULAR BOTTLENECK================== + #Save the main branch for addition later + net_main = inputs + + #First projection with 1x1 kernel + net = slim.conv2d(inputs, reduced_depth, [1,1], scope=scope+'_conv1') + net = slim.batch_norm(net, is_training=is_training, scope=scope+'_batch_norm1') + net = prelu(net, scope=scope+'_prelu1') + + #Second conv block + net = slim.conv2d(net, reduced_depth, [filter_size, filter_size], scope=scope+'_conv2') + net = slim.batch_norm(net, is_training=is_training, scope=scope+'_batch_norm2') + net = prelu(net, scope=scope+'_prelu2') + + #Final projection with 1x1 kernel + net = slim.conv2d(net, output_depth, [1,1], scope=scope+'_conv3') + net = slim.batch_norm(net, is_training=is_training, scope=scope+'_batch_norm3') + net = prelu(net, scope=scope+'_prelu3') + + #Regularizer + net = spatial_dropout(net, p=regularizer_prob, seed=seed, scope=scope+'_spatial_dropout') + net = prelu(net, scope=scope+'_prelu4') + + #Add the main branch + net = tf.add(net_main, net, name=scope+'_add_regular') + net = prelu(net, scope=scope+'_last_prelu') + + return net + +#Now actually start building the network +def ENet(inputs, + num_classes, + batch_size, + num_initial_blocks=1, + stage_two_repeat=2, + skip_connections=True, + reuse=None, + is_training=True, + scope='ENet'): + ''' + The ENet model for real-time semantic segmentation! + + INPUTS: + - inputs(Tensor): a 4D Tensor of shape [batch_size, image_height, image_width, num_channels] that represents one batch of preprocessed images. + - num_classes(int): an integer for the number of classes to predict. This will determine the final output channels as the answer. + - batch_size(int): the batch size to explictly set the shape of the inputs in order for operations to work properly. + - num_initial_blocks(int): the number of times to repeat the initial block. + - stage_two_repeat(int): the number of times to repeat stage two in order to make the network deeper. + - skip_connections(bool): if True, add the corresponding encoder feature maps to the decoder. They are of exact same shapes. + - reuse(bool): Whether or not to reuse the variables for evaluation. + - is_training(bool): if True, switch on batch_norm and prelu only during training, otherwise they are turned off. + - scope(str): a string that represents the scope name for the variables. + + OUTPUTS: + - net(Tensor): a 4D Tensor output of shape [batch_size, image_height, image_width, num_classes], where each pixel has a one-hot encoded vector + determining the label of the pixel. + ''' + #Set the shape of the inputs first to get the batch_size information + inputs_shape = inputs.get_shape().as_list() + inputs.set_shape(shape=(batch_size, inputs_shape[1], inputs_shape[2], inputs_shape[3])) + + with tf.variable_scope(scope, reuse=reuse): + #Set the primary arg scopes. Fused batch_norm is faster than normal batch norm. + with slim.arg_scope([initial_block, bottleneck], is_training=is_training),\ + slim.arg_scope([slim.batch_norm], fused=True), \ + slim.arg_scope([slim.conv2d, slim.conv2d_transpose], activation_fn=None): + #=================INITIAL BLOCK================= + net = initial_block(inputs, scope='initial_block_1') + for i in range(2, max(num_initial_blocks, 1) + 1): + net = initial_block(net, scope='initial_block_' + str(i)) + + #Save for skip connection later + if skip_connections: + net_one = net + + #===================STAGE ONE======================= + net, pooling_indices_1, inputs_shape_1 = bottleneck(net, output_depth=64, filter_size=3, regularizer_prob=0.01, downsampling=True, scope='bottleneck1_0') + net = bottleneck(net, output_depth=64, filter_size=3, regularizer_prob=0.01, scope='bottleneck1_1') + net = bottleneck(net, output_depth=64, filter_size=3, regularizer_prob=0.01, scope='bottleneck1_2') + net = bottleneck(net, output_depth=64, filter_size=3, regularizer_prob=0.01, scope='bottleneck1_3') + net = bottleneck(net, output_depth=64, filter_size=3, regularizer_prob=0.01, scope='bottleneck1_4') + + #Save for skip connection later + if skip_connections: + net_two = net + + #regularization prob is 0.1 from bottleneck 2.0 onwards + with slim.arg_scope([bottleneck], regularizer_prob=0.1): + net, pooling_indices_2, inputs_shape_2 = bottleneck(net, output_depth=128, filter_size=3, downsampling=True, scope='bottleneck2_0') + + #Repeat the stage two at least twice to get stage 2 and 3: + for i in range(2, max(stage_two_repeat, 2) + 2): + net = bottleneck(net, output_depth=128, filter_size=3, scope='bottleneck'+str(i)+'_1') + net = bottleneck(net, output_depth=128, filter_size=3, dilated=True, dilation_rate=2, scope='bottleneck'+str(i)+'_2') + net = bottleneck(net, output_depth=128, filter_size=5, asymmetric=True, scope='bottleneck'+str(i)+'_3') + net = bottleneck(net, output_depth=128, filter_size=3, dilated=True, dilation_rate=4, scope='bottleneck'+str(i)+'_4') + net = bottleneck(net, output_depth=128, filter_size=3, scope='bottleneck'+str(i)+'_5') + net = bottleneck(net, output_depth=128, filter_size=3, dilated=True, dilation_rate=8, scope='bottleneck'+str(i)+'_6') + net = bottleneck(net, output_depth=128, filter_size=5, asymmetric=True, scope='bottleneck'+str(i)+'_7') + net = bottleneck(net, output_depth=128, filter_size=3, dilated=True, dilation_rate=16, scope='bottleneck'+str(i)+'_8') + + with slim.arg_scope([bottleneck], regularizer_prob=0.1, decoder=True): + #===================STAGE FOUR======================== + bottleneck_scope_name = "bottleneck" + str(i + 1) + + #The decoder section, so start to upsample. + net = bottleneck(net, output_depth=64, filter_size=3, upsampling=True, + pooling_indices=pooling_indices_2, output_shape=inputs_shape_2, scope=bottleneck_scope_name+'_0') + + #Perform skip connections here + if skip_connections: + net = tf.add(net, net_two, name=bottleneck_scope_name+'_skip_connection') + + net = bottleneck(net, output_depth=64, filter_size=3, scope=bottleneck_scope_name+'_1') + net = bottleneck(net, output_depth=64, filter_size=3, scope=bottleneck_scope_name+'_2') + + #===================STAGE FIVE======================== + bottleneck_scope_name = "bottleneck" + str(i + 2) + + net = bottleneck(net, output_depth=16, filter_size=3, upsampling=True, + pooling_indices=pooling_indices_1, output_shape=inputs_shape_1, scope=bottleneck_scope_name+'_0') + + #perform skip connections here + if skip_connections: + net = tf.add(net, net_one, name=bottleneck_scope_name+'_skip_connection') + + net = bottleneck(net, output_depth=16, filter_size=3, scope=bottleneck_scope_name+'_1') + + #=============FINAL CONVOLUTION============= + logits = slim.conv2d_transpose(net, num_classes, [2,2], stride=2, scope='fullconv') + probabilities = tf.nn.softmax(logits, name='logits_to_softmax') + + return logits, probabilities + + +def ENet_arg_scope(weight_decay=2e-4, + batch_norm_decay=0.1, + batch_norm_epsilon=0.001): + ''' + The arg scope for enet model. The weight decay is 2e-4 as seen in the paper. + Batch_norm decay is 0.1 (momentum 0.1) according to official implementation. + + INPUTS: + - weight_decay(float): the weight decay for weights variables in conv2d and separable conv2d + - batch_norm_decay(float): decay for the moving average of batch_norm momentums. + - batch_norm_epsilon(float): small float added to variance to avoid dividing by zero. + + OUTPUTS: + - scope(arg_scope): a tf-slim arg_scope with the parameters needed for xception. + ''' + # Set weight_decay for weights in conv2d and separable_conv2d layers. + with slim.arg_scope([slim.conv2d], + weights_regularizer=slim.l2_regularizer(weight_decay), + biases_regularizer=slim.l2_regularizer(weight_decay)): + + # Set parameters for batch_norm. + with slim.arg_scope([slim.batch_norm], + decay=batch_norm_decay, + epsilon=batch_norm_epsilon) as scope: + return scope + diff --git a/TensorFlow/contrib/cv/Discriminate_Loss_Function_ID1093_for_TensorFlow/inference.py b/TensorFlow/contrib/cv/Discriminate_Loss_Function_ID1093_for_TensorFlow/inference.py new file mode 100644 index 0000000000000000000000000000000000000000..bec135de49b820d84beabb0a2c20453380c38956 --- /dev/null +++ b/TensorFlow/contrib/cv/Discriminate_Loss_Function_ID1093_for_TensorFlow/inference.py @@ -0,0 +1,131 @@ +from npu_bridge.npu_init import * +import os +import os.path + +import argparse +from glob import glob +import numpy as np +import cv2 +import tensorflow as tf +import tf_slim as slim +#slim = tf.contrib.slim +from enet import ENet, ENet_arg_scope +from clustering import cluster, get_instance_masks, save_instance_masks +import time + + +def rebuild_graph(sess, checkpoint_dir, input_image, batch_size, feature_dim): + checkpoint = tf.train.latest_checkpoint(checkpoint_dir) + + num_initial_blocks = 1 + skip_connections = False + stage_two_repeat = 2 + + with slim.arg_scope(ENet_arg_scope()): + _, _ = ENet(input_image, + num_classes=12, + batch_size=batch_size, + is_training=True, + reuse=None, + num_initial_blocks=num_initial_blocks, + stage_two_repeat=stage_two_repeat, + skip_connections=skip_connections) + + graph = tf.get_default_graph() + last_prelu = graph.get_tensor_by_name('ENet/bottleneck5_1_last_prelu:0') + logits = slim.conv2d_transpose(last_prelu, feature_dim, [2,2], stride=2, + scope='Instance/transfer_layer/conv2d_transpose') + + variables_to_restore = slim.get_variables_to_restore() + saver = tf.train.Saver(variables_to_restore) + saver.restore(sess, checkpoint) + + return logits + +def save_image_with_features_as_color(pred): + p_min = np.min(pred) + p_max = np.max(pred) + pred = (pred - p_min)*255/(p_max-p_min) + pred = pred.astype(np.uint8) + output_file_name = os.path.join(output_dir, 'color_{}.png'.format(str(i).zfill(4))) + cv2.imwrite(output_file_name, np.squeeze(pred)) + + +if __name__=='__main__': + + parser = argparse.ArgumentParser() + parser.add_argument('-m','--modeldir', default='trained_model', help="Directory of trained model") + parser.add_argument('-i', '--indir', default=os.path.join('inference_test', 'images'), help='Input image directory (jpg format)') + parser.add_argument('-o', '--outdir', default=os.path.join('inference_test', 'results'), help='Output directory for inference images') + args = parser.parse_args() + + data_dir = args.indir + output_dir = args.outdir + checkpoint_dir = args.modeldir + + if not os.path.isdir(output_dir): + os.mkdir(output_dir) + + image_paths = glob(os.path.join(data_dir, '*.jpg')) + image_paths.sort() + + num_images = len(image_paths) + + image_shape = (512, 512) + batch_size = 1 + feature_dim = 3 + + ### Limit GPU memory usage due to occasional crashes + config = tf.ConfigProto() + config.gpu_options.allow_growth = True + config.gpu_options.per_process_gpu_memory_fraction = 0.5 + + with tf.Session(config=npu_config_proto(config_proto=config)) as sess: + + input_image = tf.placeholder(tf.float32, shape=(None, image_shape[1], image_shape[0], 3)) + logits = rebuild_graph(sess, checkpoint_dir, input_image, batch_size, feature_dim) + + inference_time = 0 + cluster_time = 0 + for i, path in enumerate(image_paths): + + image = cv2.resize(cv2.imread(path), image_shape, interpolation=cv2.INTER_LINEAR) + image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) + image = np.expand_dims(image, axis=0) + + tic = time.time() + prediction = sess.run(logits, feed_dict={input_image: image}) + pred_time = time.time()-tic + print ('Inference time', pred_time) + inference_time += pred_time + + + pred_color = np.squeeze(prediction.copy()) + print ('Save prediction', i ) + #save_image_with_features_as_color(pred_color) + + pred_cluster = prediction.copy() + tic = time.time() + instance_mask = get_instance_masks(pred_cluster, bandwidth=1.)[0] + #save_instance_masks(prediction, output_dir, bandwidth=1., count=i) + print (instance_mask.shape) + output_file_name = os.path.join(output_dir, 'cluster_{}.png'.format(str(i).zfill(4))) + colors, counts = np.unique(instance_mask.reshape(image_shape[0]*image_shape[1],3), + return_counts=True, axis=0) + max_count = 0 + for color, count in zip(colors, counts): + if count > max_count: + max_count = count + bg_color = color + ind = np.where(instance_mask==bg_color) + instance_mask[ind] = 0. + instance_mask = cv2.addWeighted(np.squeeze(image), 1, instance_mask, 0.3, 0) + instance_mask = cv2.resize(instance_mask, (1280,720)) + clust_time = time.time()-tic + cluster_time += clust_time + cv2.imwrite(output_file_name, cv2.cvtColor(instance_mask, cv2.COLOR_RGB2BGR)) + + print ('Mean inference time:', inference_time/num_images, 'fps:', num_images/inference_time) + print ('Mean cluster time:', cluster_time/num_images, 'fps:', num_images/cluster_time) + print ('Mean total time:', cluster_time/num_images + inference_time/num_images, 'fps:', 1./(cluster_time/num_images + inference_time/num_images)) + diff --git a/TensorFlow/contrib/cv/Discriminate_Loss_Function_ID1093_for_TensorFlow/loss.py b/TensorFlow/contrib/cv/Discriminate_Loss_Function_ID1093_for_TensorFlow/loss.py new file mode 100644 index 0000000000000000000000000000000000000000..2c620908ab3910cd82906a7deb5187b905044152 --- /dev/null +++ b/TensorFlow/contrib/cv/Discriminate_Loss_Function_ID1093_for_TensorFlow/loss.py @@ -0,0 +1,139 @@ +from npu_bridge.npu_init import * +import tensorflow as tf + +def discriminative_loss_single(prediction, correct_label, feature_dim, label_shape, + delta_v, delta_d, param_var, param_dist, param_reg): + + ''' Discriminative loss for a single prediction/label pair. + :param prediction: inference of network + :param correct_label: instance label + :feature_dim: feature dimension of prediction + :param label_shape: shape of label + :param delta_v: cutoff variance distance + :param delta_d: curoff cluster distance + :param param_var: weight for intra cluster variance + :param param_dist: weight for inter cluster distances + :param param_reg: weight regularization + ''' + + ### Reshape so pixels are aligned along a vector + correct_label = tf.reshape(correct_label, [label_shape[1]*label_shape[0]]) + reshaped_pred = tf.reshape(prediction, [label_shape[1]*label_shape[0], feature_dim]) + + ### Count instances + unique_labels, unique_id, counts = tf.unique_with_counts(correct_label) + counts = tf.cast(counts, tf.float32) + num_instances = tf.size(unique_labels) + + segmented_sum = tf.unsorted_segment_sum(reshaped_pred, unique_id, num_instances) + + mu = tf.div(segmented_sum, tf.reshape(counts, (-1, 1))) + mu_expand = tf.gather(mu, unique_id) + + ### Calculate l_var + distance = tf.norm(tf.subtract(mu_expand, reshaped_pred), axis=1) + distance = tf.subtract(distance, delta_v) + distance = tf.clip_by_value(distance, 0., distance) + distance = tf.square(distance) + + l_var = tf.unsorted_segment_sum(distance, unique_id, num_instances) + l_var = tf.div(l_var, counts) + l_var = tf.reduce_sum(l_var) + l_var = tf.divide(l_var, tf.cast(num_instances, tf.float32)) + + ### Calculate l_dist + + # Get distance for each pair of clusters like this: + # mu_1 - mu_1 + # mu_2 - mu_1 + # mu_3 - mu_1 + # mu_1 - mu_2 + # mu_2 - mu_2 + # mu_3 - mu_2 + # mu_1 - mu_3 + # mu_2 - mu_3 + # mu_3 - mu_3 + + mu_interleaved_rep = tf.tile(mu, [num_instances, 1]) + mu_band_rep = tf.tile(mu, [1, num_instances]) + mu_band_rep = tf.reshape(mu_band_rep, (num_instances*num_instances, feature_dim)) + + mu_diff = tf.subtract(mu_band_rep, mu_interleaved_rep) + + # Filter out zeros from same cluster subtraction + intermediate_tensor = tf.reduce_sum(tf.abs(mu_diff),axis=1) + zero_vector = tf.zeros(1, dtype=tf.float32) + bool_mask = tf.not_equal(intermediate_tensor, zero_vector) + mu_diff_bool = tf.boolean_mask(mu_diff, bool_mask) + + mu_norm = tf.norm(mu_diff_bool, axis=1) + mu_norm = tf.subtract(2.*delta_d, mu_norm) + mu_norm = tf.clip_by_value(mu_norm, 0., mu_norm) + mu_norm = tf.square(mu_norm) + + l_dist = tf.reduce_mean(mu_norm) + + ### Calculate l_reg + l_reg = tf.reduce_mean(tf.norm(mu, axis=1)) + + param_scale = 1. + l_var = param_var * l_var + l_dist = param_dist * l_dist + l_reg = param_reg * l_reg + + loss = param_scale*(l_var + l_dist + l_reg) + + return loss, l_var, l_dist, l_reg + + +def discriminative_loss(prediction, correct_label, feature_dim, image_shape, + delta_v, delta_d, param_var, param_dist, param_reg): + ''' Iterate over a batch of prediction/label and cumulate loss + :return: discriminative loss and its three components + ''' + def cond(label, batch, out_loss, out_var, out_dist, out_reg, i): + return tf.less(i, tf.shape(batch)[0]) + + def body(label, batch, out_loss, out_var, out_dist, out_reg, i): + disc_loss, l_var, l_dist, l_reg = discriminative_loss_single(prediction[i], correct_label[i], feature_dim, image_shape, + delta_v, delta_d, param_var, param_dist, param_reg) + + out_loss = out_loss.write(i, disc_loss) + out_var = out_var.write(i, l_var) + out_dist = out_dist.write(i, l_dist) + out_reg = out_reg.write(i, l_reg) + + return label, batch, out_loss, out_var, out_dist, out_reg, i + 1 + + # TensorArray is a data structure that support dynamic writing + output_ta_loss = tf.TensorArray(dtype=tf.float32, + size=0, + dynamic_size=True) + output_ta_var = tf.TensorArray(dtype=tf.float32, + size=0, + dynamic_size=True) + output_ta_dist = tf.TensorArray(dtype=tf.float32, + size=0, + dynamic_size=True) + output_ta_reg = tf.TensorArray(dtype=tf.float32, + size=0, + dynamic_size=True) + + _, _, out_loss_op, out_var_op, out_dist_op, out_reg_op, _ = tf.while_loop(cond, body, [correct_label, + prediction, + output_ta_loss, + output_ta_var, + output_ta_dist, + output_ta_reg, + 0]) + out_loss_op = out_loss_op.stack() + out_var_op = out_var_op.stack() + out_dist_op = out_dist_op.stack() + out_reg_op = out_reg_op.stack() + + disc_loss = tf.reduce_mean(out_loss_op) + l_var = tf.reduce_mean(out_var_op) + l_dist = tf.reduce_mean(out_dist_op) + l_reg = tf.reduce_mean(out_reg_op) + + return disc_loss, l_var, l_dist, l_reg diff --git a/TensorFlow/contrib/cv/Discriminate_Loss_Function_ID1093_for_TensorFlow/modelzoo_level.txt b/TensorFlow/contrib/cv/Discriminate_Loss_Function_ID1093_for_TensorFlow/modelzoo_level.txt new file mode 100644 index 0000000000000000000000000000000000000000..0b49b4fb26c2694a86567bea1b462e7dcb03cc31 --- /dev/null +++ b/TensorFlow/contrib/cv/Discriminate_Loss_Function_ID1093_for_TensorFlow/modelzoo_level.txt @@ -0,0 +1,3 @@ +FuncStatus:OK +PerfStatus:OK +PrecisionStatus:OK \ No newline at end of file diff --git a/TensorFlow/contrib/cv/Discriminate_Loss_Function_ID1093_for_TensorFlow/requirement.txt b/TensorFlow/contrib/cv/Discriminate_Loss_Function_ID1093_for_TensorFlow/requirement.txt new file mode 100644 index 0000000000000000000000000000000000000000..61b19bdb2ffbf15152fae8e3749cc24a30058acf --- /dev/null +++ b/TensorFlow/contrib/cv/Discriminate_Loss_Function_ID1093_for_TensorFlow/requirement.txt @@ -0,0 +1 @@ +tensorflow 1.15 \ No newline at end of file diff --git a/TensorFlow/contrib/cv/Discriminate_Loss_Function_ID1093_for_TensorFlow/run.sh b/TensorFlow/contrib/cv/Discriminate_Loss_Function_ID1093_for_TensorFlow/run.sh new file mode 100644 index 0000000000000000000000000000000000000000..31f3a7ed9b10f08e9d1949c728ca4a66f6a38268 --- /dev/null +++ b/TensorFlow/contrib/cv/Discriminate_Loss_Function_ID1093_for_TensorFlow/run.sh @@ -0,0 +1 @@ +python data/tusimple_dataset_processing.py -srcdir s3://obs-lpf/data/ -o \ No newline at end of file diff --git a/TensorFlow/contrib/cv/Discriminate_Loss_Function_ID1093_for_TensorFlow/todo_semantic_segmentation/.keep b/TensorFlow/contrib/cv/Discriminate_Loss_Function_ID1093_for_TensorFlow/todo_semantic_segmentation/.keep new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/TensorFlow/contrib/cv/Discriminate_Loss_Function_ID1093_for_TensorFlow/todo_semantic_segmentation/helper.py b/TensorFlow/contrib/cv/Discriminate_Loss_Function_ID1093_for_TensorFlow/todo_semantic_segmentation/helper.py new file mode 100644 index 0000000000000000000000000000000000000000..a15d38df6175b9dd148bc346a7f16093cd191bea --- /dev/null +++ b/TensorFlow/contrib/cv/Discriminate_Loss_Function_ID1093_for_TensorFlow/todo_semantic_segmentation/helper.py @@ -0,0 +1,120 @@ +from npu_bridge.npu_init import * +import os +from glob import glob +import numpy as np +import random +from sklearn.utils import shuffle +import shutil +import time +import tensorflow as tf +import cv2 + + +### Mean and std deviation for whole training data set (RGB format) +mean = 0.#np.array([92.14031982, 103.20146942, 103.47182465]) +std = 1.#np.array([49.157, 54.9057, 59.4065]) + + +def get_batches_fn(batch_size, image_shape, image_paths, label_paths): + """ + Create batches of training data + :param batch_size: Batch Size + :param image_shape: input image shape + :param image_paths: list of paths for training or validation + :param label_paths: list of paths for training or validation + :return: Batches of training data + """ + + image_paths.sort() + label_paths.sort() + + #image_paths = image_paths[:20] + #label_paths = label_paths[:20] + + background_color = np.array([0, 0, 0]) + + image_paths, label_paths = shuffle(image_paths, label_paths) + + for batch_i in range(0, len(image_paths), batch_size): + images = [] + gt_images = [] + for image_file, gt_image_file in zip(image_paths[batch_i:batch_i+batch_size], label_paths[batch_i:batch_i+batch_size]): + + ### Image preprocessing + image = cv2.resize(cv2.imread(image_file), (image_shape[1], image_shape[0]), cv2.INTER_LINEAR) + image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) + #image = (image.astype(np.float32)-mean)/std + + ### Label preprocessing + gt_image = cv2.resize(cv2.imread(gt_image_file, cv2.IMREAD_COLOR), (image_shape[1], image_shape[0]), cv2.INTER_NEAREST) + gt_bg = np.all(gt_image == background_color, axis=2) + gt_bg = gt_bg.reshape(gt_bg.shape[0], gt_bg.shape[1], 1) + gt_image = np.concatenate((gt_bg, np.invert(gt_bg)), axis=2) + + images.append(image) + gt_images.append(gt_image) + + yield np.array(images), np.array(gt_images) + + +# Source http://www.pyimagesearch.com/2015/10/05/opencv-gamma-correction/ +def adjust_gamma(image, gamma=1.0): + # build a lookup table mapping the pixel values [0, 255] to + # their adjusted gamma values + invGamma = 1.0 / gamma + table = np.array([((i / 255.0) ** invGamma) * 255 + for i in np.arange(0, 256)]).astype("float32") + + # apply gamma correction using the lookup table + return cv2.LUT(image, table) + + +def gen_test_output(sess, logits, keep_prob, image_pl, data_folder, image_shape): + """ + Generate test output using the test images + :param sess: TF session + :param logits: TF Tensor for the logits + :param keep_prob: TF Placeholder for the dropout keep robability + :param image_pl: TF Placeholder for the image placeholder + :param data_folder: Path to the folder that contains the datasets + :param image_shape: Tuple - Shape of image + :return: Output for for each test image + """ + for image_file in glob(os.path.join(data_folder, 'test_images', '*.png'))[:40]: + image = cv2.resize(cv2.imread(image_file), (image_shape[1], image_shape[0]), cv2.INTER_LINEAR) + + ### Run inference + image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) + img_origin = image.copy() + #image = (image.astype(np.float32)-mean)/std + + im_softmax = sess.run( + tf.nn.softmax(logits), + {keep_prob: 1.0, image_pl: [image]}) + + ### Threshholding + im_softmax = im_softmax[:, 1].reshape(image_shape[0], image_shape[1]) + mask_ind = np.where(im_softmax > 0.3) + + ### Overlay class mask over original image + blend = np.zeros_like(img_origin) + blend[mask_ind] = np.array([0,255,0]) + blended = cv2.addWeighted(img_origin, 1, blend, 0.7, 0) + blended = cv2.cvtColor(blended, cv2.COLOR_BGR2RGB) + + yield os.path.basename(image_file), np.array(blended) + + +def save_inference_samples(runs_dir, data_dir, sess, image_shape, logits, keep_prob, input_image): + # Make folder for current run + output_dir = os.path.join(runs_dir, str(time.time())) + if os.path.exists(output_dir): + shutil.rmtree(output_dir) + os.makedirs(output_dir) + + # Run NN on test images and save them to HD + print('Training Finished. Saving test images to: {}'.format(output_dir)) + image_outputs = gen_test_output( + sess, logits, keep_prob, input_image, data_dir, image_shape) + for name, image in image_outputs: + cv2.imwrite(os.path.join(output_dir, name), image) diff --git a/TensorFlow/contrib/cv/Discriminate_Loss_Function_ID1093_for_TensorFlow/todo_semantic_segmentation/transfer_semantic.py b/TensorFlow/contrib/cv/Discriminate_Loss_Function_ID1093_for_TensorFlow/todo_semantic_segmentation/transfer_semantic.py new file mode 100644 index 0000000000000000000000000000000000000000..cf4b57a8bca1d95ff1894b7fb4bee9ef87ab0489 --- /dev/null +++ b/TensorFlow/contrib/cv/Discriminate_Loss_Function_ID1093_for_TensorFlow/todo_semantic_segmentation/transfer_semantic.py @@ -0,0 +1,206 @@ +import os.path +import tensorflow as tf +import helper +import warnings +from sklearn.utils import shuffle +from sklearn.model_selection import train_test_split +from glob import glob +from enet import ENet, ENet_arg_scope +from tensorflow.contrib.layers.python.layers import initializers +import numpy as np +slim = tf.contrib.slim + +# Check for a GPU +if not tf.test.gpu_device_name(): + warnings.warn('No GPU found. Please use a GPU to train your neural network.') +else: + print('Default GPU Device: {}'.format(tf.test.gpu_device_name())) + + +def load_enet(sess, checkpoint_dir, input_image, batch_size, num_classes): + checkpoint = tf.train.latest_checkpoint(checkpoint_dir) + + num_initial_blocks = 1 + skip_connections = False + stage_two_repeat = 2 + + with slim.arg_scope(ENet_arg_scope()): + logits, _ = ENet(input_image, + num_classes=12, + batch_size=batch_size, + is_training=True, + reuse=None, + num_initial_blocks=num_initial_blocks, + stage_two_repeat=stage_two_repeat, + skip_connections=skip_connections) + + + variables_to_restore = slim.get_variables_to_restore() + saver = tf.train.Saver(variables_to_restore) + saver.restore(sess, checkpoint) + graph = tf.get_default_graph() + + last_prelu = graph.get_tensor_by_name('ENet/bottleneck5_1_last_prelu:0') + output = slim.conv2d_transpose(last_prelu, num_classes, [2,2], stride=2, + weights_initializer=initializers.xavier_initializer(), + scope='Semantic/transfer_layer/conv2d_transpose') + + probabilities = tf.nn.softmax(output, name='Semantic/transfer_layer/logits_to_softmax') + + with tf.variable_scope('', reuse=True): + weight = tf.get_variable('Semantic/transfer_layer/conv2d_transpose/weights') + bias = tf.get_variable('Semantic/transfer_layer/conv2d_transpose/biases') + sess.run([weight.initializer, bias.initializer]) + + return output, probabilities + + + +def optimize(sess, logits, correct_label, learning_rate, num_classes, trainables, global_step): + """ + Build the TensorFLow loss and optimizer operations. + :param nn_last_layer: TF Tensor of the last layer in the neural network + :param correct_label: TF Placeholder for the correct label image + :param learning_rate: TF Placeholder for the learning rate + :param num_classes: Number of classes to classify + :return: Tuple of (logits, train_op, cross_entropy_loss) + """ + # TODO: Implement function + + #correct_label = tf.reshape(correct_label, (-1, num_classes)) + #logits = tf.reshape(nn_last_layer, (-1, num_classes)) + + weights = correct_label * np.array([1., 40.]) + weights = tf.reduce_sum(weights, axis=3) + loss = tf.reduce_mean(tf.losses.softmax_cross_entropy(onehot_labels=correct_label, logits=logits, weights=weights)) + + + #loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=correct_label, logits=logits)) + with tf.name_scope('Semantic/Adam'): + train_op = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(loss, var_list=trainables, global_step=global_step) + adam_initializers = [var.initializer for var in tf.global_variables() if 'Adam' in var.name] + sess.run(adam_initializers) + return logits, train_op, loss + + +def run(): + + ### Initialization + image_shape = (512, 512) # (width, height) + model_dir = '../checkpoint' + data_dir = '../../tusimple_api/clean_data' + log_dir = './log' + output_dir = './saved_model' + + num_classes = 2 + epochs = 20 + batch_size = 1 + starter_learning_rate = 1e-4 + learning_rate_decay_interval = 500 + learning_rate_decay_rate = 0.96 + ### Load images and labels + image_paths = glob(os.path.join(data_dir, 'images', '*.png')) + label_paths = glob(os.path.join(data_dir, 'labels', '*.png')) + + #image_paths = image_paths[:20] + #label_paths = label_paths[:20] + + X_train, X_valid, y_train, y_valid = train_test_split(image_paths, label_paths, test_size=0.20, random_state=42) + + ### Limit GPU memory usage due to ocassional crashes + config = tf.ConfigProto() + config.gpu_options.allow_growth = True + config.gpu_options.per_process_gpu_memory_fraction = 0.7 + + + + with tf.Session(config=config) as sess: + + ### Load ENet and replace layers + input_image = tf.placeholder(tf.float32, shape=[batch_size, image_shape[1], image_shape[0], 3]) + correct_label = tf.placeholder(dtype=tf.float32, shape=(None, image_shape[1], image_shape[0], 2), name='Semantic/input_image') + + logits, probabilities = load_enet(sess, model_dir, input_image, batch_size, num_classes) + predictions_val = tf.argmax(probabilities, axis=-1) + predictions_val = tf.cast(predictions_val, dtype=tf.float32) + predictions_val = tf.reshape(predictions_val, shape=[batch_size, image_shape[1], image_shape[0], 1]) + + + ### Set up learning rate decay + global_step = tf.Variable(0, trainable=False) + sess.run(global_step.initializer) + learning_rate = tf.train.exponential_decay(starter_learning_rate, global_step, + learning_rate_decay_interval, learning_rate_decay_rate, staircase=True) + + for i, var in enumerate(tf.trainable_variables()): + print i, var + tf.summary.histogram(var.name, var) + + trainables = [var for var in tf.trainable_variables() if 'bias' not in var.name and 'ENet/fullconv' not in var.name] + + ### Print variables which are actually trained + for var in trainables: + print var + + logits, train_op, cross_entropy_loss = optimize(sess, logits, correct_label, learning_rate, num_classes, trainables, global_step) + tf.summary.scalar('training_loss', cross_entropy_loss) + tf.summary.image('Images/Validation_original_image', input_image, max_outputs=1) + tf.summary.image('Images/Validation_segmentation_output', predictions_val, max_outputs=1) + summary_train = tf.summary.merge_all() + summary_valid = tf.summary.scalar('validation_loss', cross_entropy_loss) + + train_writer = tf.summary.FileWriter(log_dir) + + saver = tf.train.Saver() + + ### Training pipeline + step_train = 0 + step_valid = 0 + summary_cycle = 10 + for epoch in range(epochs): + print 'epoch', epoch + print 'training ...' + train_loss = 0 + for image, label in helper.get_batches_fn(batch_size, image_shape, X_train, y_train): + # Training + lr = sess.run(learning_rate) + if step_train%summary_cycle==0: + _, summary, loss = sess.run([train_op, summary_train, cross_entropy_loss], + feed_dict={input_image: image, correct_label: label}) + train_writer.add_summary(summary, step_train) + print 'epoch', epoch, '\t step_train', step_train, '\t batch loss', loss, '\t current learning rate', lr + else: + _, loss = sess.run([train_op, cross_entropy_loss], + feed_dict={input_image: image, correct_label: label}) + step_train+=1 + train_loss += loss + + if (step_train%5000==4999): + saver.save(sess, os.path.join(output_dir, 'model.ckpt'), global_step=global_step) + + + print 'train epoch loss', train_loss + + print 'validating ...' + valid_loss = 0 + for image, label in helper.get_batches_fn(batch_size, image_shape, X_valid, y_valid): + # Validation + if step_valid%summary_cycle==0: + summary, loss = sess.run([summary_valid, cross_entropy_loss], + feed_dict={input_image: image, correct_label: label}) + train_writer.add_summary(summary, step_valid) + print 'batch loss', loss + else: + loss = sess.run(cross_entropy_loss, + feed_dict={input_image: image, correct_label: label}) + + valid_loss += loss + step_valid+=1 + + print 'valid epoch loss', valid_loss + + + + +if __name__ == '__main__': + run() diff --git a/TensorFlow/contrib/cv/Discriminate_Loss_Function_ID1093_for_TensorFlow/training.py b/TensorFlow/contrib/cv/Discriminate_Loss_Function_ID1093_for_TensorFlow/training.py new file mode 100644 index 0000000000000000000000000000000000000000..45e9562c2aea18d5acb70f93b078823ea8cdea95 --- /dev/null +++ b/TensorFlow/contrib/cv/Discriminate_Loss_Function_ID1093_for_TensorFlow/training.py @@ -0,0 +1,271 @@ +# -*- coding: utf-8 -*- +from npu_bridge.npu_init import * +import os +import os.path + +import sys +import warnings +import copy +from glob import glob +import argparse + +import numpy as np +import cv2 +#import tensorflow as tf +import tensorflow.compat.v1 as tf +tf.disable_v2_behavior() + +from sklearn.model_selection import train_test_split +from tensorflow.contrib.layers.python.layers import initializers + +import matplotlib as mpl +mpl.use('Agg') +import matplotlib.pyplot as plt +from mpl_toolkits.mplot3d import Axes3D +import matplotlib.cm as cm + +import utils +from loss import discriminative_loss +import datagenerator +import visualization +import clustering + +from npu_bridge.npu_init import * + +from tensorflow.core.protobuf.rewriter_config_pb2 import RewriterConfig +import moxing as mox + +def run(): + parser = argparse.ArgumentParser() + # Directories + parser.add_argument('--srcdir', default='s3://cann001/dataset/dataset', help="Source directory of TuSimple dataset") + parser.add_argument('--modeldir', default='s3://cann001/dataset/pretrained_semantic_model', help="Output directory of extracted data") + parser.add_argument('--outdir', default='s3://cann001/output/', help="Directory for trained model") + parser.add_argument('--logdir', default='log', help="Log directory for tensorboard and evaluation files") + # Hyperparameters + parser.add_argument('--epochs', type=int, default=50, help="Number of epochs") + parser.add_argument('--var', type=float, default=1., help="Weight of variance loss") + parser.add_argument('--dist', type=float, default=1., help="Weight of distance loss") + parser.add_argument('--reg', type=float, default=0.001, help="Weight of regularization loss") + parser.add_argument('--dvar', type=float, default=0.5, help="Cutoff variance") + parser.add_argument('--ddist', type=float, default=1.5, help="Cutoff distance") + + #args = parser.parse_args() + args, unkown = parser.parse_known_args() + local_dir1 = '/cache/11' + mox.file.copy_parallel(args.srcdir, local_dir1) + local_dir2 = '/cache/22' + mox.file.copy_parallel(args.modeldir, local_dir2) + + if not os.path.isdir(local_dir1): + raise IOError('Directory does not exist') + if not os.path.isdir(local_dir2): + raise IOError('Directory does not exist') + if not os.path.isdir(args.logdir): + os.mkdir(args.logdir) + args.srcdir = local_dir1 + args.modeldir = local_dir2 + print('ccc') + + image_shape = (512, 512) + data_dir = args.srcdir #os.path.join('.', 'data') + model_dir = args.modeldir + output_dir = args.outdir + log_dir = args.logdir + + image_paths = glob(os.path.join(data_dir, 'images', '*.png')) + label_paths = glob(os.path.join(data_dir, 'labels', '*.png')) + + image_paths.sort() + label_paths.sort() + + #image_paths = image_paths[0:10] + #label_paths = label_paths[0:10] + + X_train, X_valid, y_train, y_valid = train_test_split(image_paths, label_paths, test_size=0.10, random_state=42) + + print ('Number of train samples', len(y_train)) + print ('Number of valid samples', len(y_valid)) + + + ### Debugging + debug_clustering = True + bandwidth = 0.7 + cluster_cycle = 5000 + eval_cycle=1000 + save_cycle=15000 + + ### Hyperparameters + epochs = args.epochs + batch_size = 1 + starter_learning_rate = 1e-4 + learning_rate_decay_rate = 0.96 + learning_rate_decay_interval = 5000 + + feature_dim = 3 + param_var = args.var + param_dist = args.dist + param_reg = args.reg + delta_v = args.dvar + delta_d = args.ddist + + param_string = 'fdim'+str(feature_dim)+'_var'+str(param_var)+'_dist'+str(param_dist)+'_reg'+str(param_reg) \ + +'_dv'+str(delta_v)+'_dd'+str(delta_d) \ + +'_lr'+str(starter_learning_rate)+'_btch'+str(batch_size) + + if not os.path.exists(os.path.join(log_dir, param_string)): + os.makedirs(os.path.join(log_dir, param_string)) + + if not os.path.exists(os.path.join(output_dir)): + os.makedirs(os.path.join(output_dir)) + + + ### Limit GPU memory usage due to ocassional crashes +# config = tf.compat.v1.ConfigProto + config = tf.ConfigProto(allow_soft_placement=True, log_device_placement=False) + custom_op = config.graph_options.rewrite_options.custom_optimizers.add() + custom_op.name = "NpuOptimizer" + config.graph_options.rewrite_options.remapping = RewriterConfig.OFF # 必须显式关闭 + config.graph_options.rewrite_options.memory_optimization = RewriterConfig.OFF # 必须显式关闭 + #sess = tf.Session(config=config) + #config.gpu_options.allow_growth = True + #config.gpu_options.per_process_gpu_memory_fraction = 0.5 + + + #with tf.Session(config=npu_config_proto(config_proto=config)) as sess: + with tf.Session(config=config) as sess: +# with tf.compat.v1.Session(config=config) as sess: + + ### Build network + input_image = tf.placeholder(tf.float32, shape=(None, image_shape[1], image_shape[0], 3)) + correct_label = tf.placeholder(dtype=tf.float32, shape=(None, image_shape[1], image_shape[0])) + + last_prelu = utils.load_enet(sess, model_dir, input_image, batch_size) + prediction = utils.add_transfer_layers_and_initialize(sess, last_prelu, feature_dim) + + print ('Number of parameters in the model', utils.count_parameters()) + ### Set up learning rate decay + global_step = tf.Variable(0, trainable=False) + sess.run(global_step.initializer) + learning_rate = tf.train.exponential_decay(starter_learning_rate, global_step, + learning_rate_decay_interval, learning_rate_decay_rate, staircase=True) + + ### Set variables to train + trainables = utils.get_trainable_variables_and_initialize(sess, debug=False) + + ### Optimization operations + disc_loss, l_var, l_dist, l_reg = discriminative_loss(prediction, correct_label, feature_dim, image_shape, + delta_v, delta_d, param_var, param_dist, param_reg) + with tf.name_scope('Instance/Adam'): + train_op = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(disc_loss, var_list=trainables, global_step=global_step) + adam_initializers = [var.initializer for var in tf.global_variables() if 'Adam' in var.name] + sess.run(adam_initializers) + + + ### Collect summaries + summary_op_train, summary_op_valid = utils.collect_summaries(disc_loss, l_var, l_dist, l_reg, input_image, prediction, correct_label) + + train_writer = tf.summary.FileWriter(log_dir) + + + ### Check if image and labels match + valid_image_chosen, valid_label_chosen = datagenerator.get_validation_batch(data_dir, image_shape) + print (valid_image_chosen.shape) + #visualization.save_image_overlay(valid_image_chosen.copy(), valid_label_chosen.copy()) + + + ### Training pipeline + saver = tf.train.Saver() + step_train=0 + step_valid=0 + for epoch in range(epochs): + print ('epoch', epoch) + + train_loss = 0 + for image, label in datagenerator.get_batches_fn(batch_size, image_shape, X_train, y_train): + + lr = sess.run(learning_rate) + + if (step_train%eval_cycle!=0): + ### Training + _, step_prediction, step_loss, step_l_var, step_l_dist, step_l_reg = sess.run([ + train_op, + prediction, + disc_loss, + l_var, + l_dist, + l_reg], + feed_dict={input_image: image, correct_label: label}) + else: + # First run normal training step and record summaries + print ('Evaluating on chosen images ...') + _, summary, step_prediction, step_loss, step_l_var, step_l_dist, step_l_reg = sess.run([ + train_op, + summary_op_train, + prediction, + disc_loss, + l_var, + l_dist, + l_reg], + feed_dict={input_image: image, correct_label: label}) + train_writer.add_summary(summary, step_train) + + # Then run model on some chosen images and save feature space visualization + valid_pred = sess.run(prediction, feed_dict={input_image: np.expand_dims(valid_image_chosen[0], axis=0), + correct_label: np.expand_dims(valid_label_chosen[0], axis=0)}) + visualization.evaluate_scatter_plot(log_dir, valid_pred, valid_label_chosen, feature_dim, param_string, step_train) + + # Perform mean-shift clustering on prediction + if (step_train%cluster_cycle==0): + if debug_clustering: + instance_masks = clustering.get_instance_masks(valid_pred, bandwidth) + for img_id, mask in enumerate(instance_masks): + cv2.imwrite(os.path.join(log_dir, param_string, 'cluster_{}_{}.png'.format(str(step_train).zfill(6), str(img_id)) ), mask) + + step_train += 1 + + ### Save intermediate model + if (step_train%save_cycle==(save_cycle-1)): + try: + print ('Saving model ...') + saver.save(sess, os.path.join(output_dir, 'model.ckpt'), global_step=step_train) + except: + print ('FAILED saving model') + #print 'gradient', step_gradient + print ('step', step_train, '\tloss', step_loss, '\tl_var', step_l_var, '\tl_dist', step_l_dist, '\tl_reg', step_l_reg, '\tcurrent lr', lr) + + + ### Regular validation + print ('Evaluating current model ...') + for image, label in datagenerator.get_batches_fn(batch_size, image_shape, X_valid, y_valid): + if step_valid%100==0: + summary, step_prediction, step_loss, step_l_var, step_l_dist, step_l_reg = sess.run([ + summary_op_valid, + prediction, + disc_loss, + l_var, + l_dist, + l_reg], + feed_dict={input_image: image, correct_label: label}) + train_writer.add_summary(summary, step_valid) + else: + step_prediction, step_loss, step_l_var, step_l_dist, step_l_reg = sess.run([ + prediction, + disc_loss, + l_var, + l_dist, + l_reg], + feed_dict={input_image: image, correct_label: label}) + step_valid += 1 + + + print ('step_valid', step_valid, 'valid loss', step_loss, '\tvalid l_var', step_l_var, '\tvalid l_dist', step_l_dist, '\tvalid l_reg', step_l_reg) + + + saver = tf.train.Saver() + saver.save(sess, os.path.join(output_dir, 'model.ckpt'), global_step=step_train) + + +if __name__ == '__main__': + run() + diff --git a/TensorFlow/contrib/cv/Discriminate_Loss_Function_ID1093_for_TensorFlow/utils.py b/TensorFlow/contrib/cv/Discriminate_Loss_Function_ID1093_for_TensorFlow/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..9e1b8b5c4e6bb1df676866a0485530678923989d --- /dev/null +++ b/TensorFlow/contrib/cv/Discriminate_Loss_Function_ID1093_for_TensorFlow/utils.py @@ -0,0 +1,124 @@ + +from npu_bridge.npu_init import * +import sys +sys.path.append('../base') +import tensorflow +import tensorflow.compat.v1 as tf +tf.disable_v2_behavior() +from enet import ENet, ENet_arg_scope +slim = tensorflow.contrib.slim + +def load_enet(sess, checkpoint_dir, input_image, batch_size): + checkpoint = tf.train.latest_checkpoint(checkpoint_dir) + + num_initial_blocks = 1 + skip_connections = False + stage_two_repeat = 2 + + with slim.arg_scope(ENet_arg_scope()): + _, _ = ENet(input_image, + num_classes=12, + batch_size=batch_size, + is_training=True, + reuse=None, + num_initial_blocks=num_initial_blocks, + stage_two_repeat=stage_two_repeat, + skip_connections=skip_connections) + + variables_to_restore = slim.get_variables_to_restore() + saver = tf.train.Saver(variables_to_restore) + saver.restore(sess, checkpoint) + + graph = tf.get_default_graph() + last_prelu = graph.get_tensor_by_name('ENet/bottleneck5_1_last_prelu:0') + return last_prelu + +def add_transfer_layers_and_initialize(sess, last_prelu, feature_dim): + + logits = slim.conv2d_transpose(last_prelu, feature_dim, [2,2], stride=2, + biases_initializer=tf.constant_initializer(10.0), + weights_initializer=tensorflow.contrib.layers.xavier_initializer(), + scope='Instance/transfer_layer/conv2d_transpose') + + with tf.variable_scope('', reuse=True): + weight = tf.get_variable('Instance/transfer_layer/conv2d_transpose/weights') + bias = tf.get_variable('Instance/transfer_layer/conv2d_transpose/biases') + sess.run([weight.initializer, bias.initializer]) + + return logits + +def get_trainable_variables_and_initialize(sess, debug=False): + ''' Determine which variables to train and reset + We accumulate all variables we want to train in a list to pass it to the optimizer. + As mentioned in the 'Fast Scene Understanding' paper we want to freeze stage 1 and 2 + from the ENet and train stage 3-5. The variables from the later stages are reseted. + Additionally all biases are not trained. + + :return: trainables: List of variables we want to train + + ''' + ### Freeze shared encode + trainables = [var for var in tf.trainable_variables() if 'bias' not in var.name]# and \ + #'ENet/fullconv' not in var.name and \ + #'ENet/initial_block_1' not in var.name and \ + #'ENet/bottleneck1' not in var.name and \ + #'ENet/bottleneck2' not in var.name + #] + if debug: + print ('All trainable variables') + for i, var in enumerate(tf.trainable_variables()): + print (i, var) + print ('variables which are actually trained') + for var in trainables: + print (var) + + ### Design choice: reset decoder network to default initialize weights + # Reset all trainable variables + #sess.run(tf.variables_initializer(trainables)) + # Additionally reset all biases in the decoder network + # Encoder retains pretrained biases + sess.run(tf.variables_initializer([var for var in tf.trainable_variables() if 'bias' in var.name and \ + 'ENet/initial_block_1' not in var.name and \ + 'ENet/bottleneck1' not in var.name and \ + 'ENet/bottleneck2' not in var.name]) + ) + return trainables + +def collect_summaries(disc_loss, l_var, l_dist, l_reg, input_image, prediction, correct_label): + + summaries = [] + # Collect all variables + for var in tf.trainable_variables(): + summaries.append(tf.summary.histogram(var.name, var)) + # Collect losses + summaries.append(tf.summary.scalar('Train/disc_loss', disc_loss)) + summaries.append(tf.summary.scalar('Train/l_var', l_var)) + summaries.append(tf.summary.scalar('Train/l_dist', l_dist)) + summaries.append(tf.summary.scalar('Train/l_reg', l_reg)) + # Collect images + summaries.append(tf.summary.image('Train/Images/Input', input_image, max_outputs=1)) + summaries.append(tf.summary.image('Train/Images/Prediction', tf.expand_dims(prediction[:,:,:,0], axis=3), max_outputs=1)) + summaries.append(tf.summary.image('Train/Images/Label', tf.expand_dims(correct_label, axis=3), max_outputs=1)) + + for summ in summaries: + tf.add_to_collection('CUSTOM_SUMMARIES', summ) + + summary_op_train = tf.summary.merge_all('CUSTOM_SUMMARIES') + + summaries_valid = [] + summaries_valid.append(tf.summary.image('Valid/Images/Input', input_image, max_outputs=1)) + summaries_valid.append(tf.summary.image('Valid/Images/Prediction', tf.expand_dims(prediction[:,:,:,0], axis=3), max_outputs=1)) + summaries_valid.append(tf.summary.image('Valid/Images/Label', tf.expand_dims(correct_label, axis=3), max_outputs=1)) + summaries_valid.append(tf.summary.scalar('Valid/disc_loss', disc_loss)) + summary_op_valid = tf.summary.merge(summaries_valid) + return summary_op_train, summary_op_valid + +def count_parameters(): + total_parameters = 0 + for var in tf.trainable_variables(): + shape = var.get_shape() + variable_parameters = 1 + for dim in shape: + variable_parameters *= dim.value + total_parameters += variable_parameters + return total_parameters diff --git a/TensorFlow/contrib/cv/Discriminate_Loss_Function_ID1093_for_TensorFlow/visualization.py b/TensorFlow/contrib/cv/Discriminate_Loss_Function_ID1093_for_TensorFlow/visualization.py new file mode 100644 index 0000000000000000000000000000000000000000..d7adb5ed5809da75729042be73caf2fe9511930d --- /dev/null +++ b/TensorFlow/contrib/cv/Discriminate_Loss_Function_ID1093_for_TensorFlow/visualization.py @@ -0,0 +1,75 @@ +from npu_bridge.npu_init import * +import os +import numpy as np +import cv2 +import matplotlib.pyplot as plt +from clustering import cluster + +def save_image_overlay(valid_image, valid_label): + + assert len(valid_image.shape)==3 and len(valid_label.shape)==2, \ + 'input dimensions should be [h,w,c]' + + num_unique = np.unique(valid_label) + blended = valid_image + for color_id, unique in enumerate(list(num_unique[1:])): + instance_ind = np.where(valid_label==unique) + alpha = np.zeros_like(valid_image) + alpha[instance_ind] = np.array([color_id*70, color_id*70, 255-color_id*50]) + + blended = cv2.addWeighted(blended, 1, alpha, 1, 0) + blended = cv2.cvtColor(blended, cv2.COLOR_RGB2BGR) + cv2.imwrite('overlayed_image.png', blended) + + +def evaluate_scatter_plot(log_dir, valid_pred, valid_label, feature_dim, param_string, step): + + assert len(valid_pred.shape)==4 and len(valid_label.shape)==3, \ + 'input dimensions should be [b,h,w,c] and [b,h,w]' + + assert valid_pred.shape[3]==feature_dim, 'feature dimension and prediction do not match' + + + fig = plt.figure() #plt.figure(figsize=(10,8)) + if feature_dim==2: + + #for i in range(valid_pred.shape[0]): + # plt.subplot(2,2,i+1) + # #valid_label = valid_label[0] + # #print 'valid_pred', valid_pred.shape + # #print 'valid_label', valid_label.shape + # num_unique = np.unique(valid_label[i]) + num_unique = np.unique(valid_label[0]) + for unique in list(num_unique): + instance_ind = np.where(valid_label[0]==unique) + #print 'instance id', instance_ind + #print valid_pr[instance_ind].shape + x = valid_pred[0,:,:,0][instance_ind] + y = valid_pred[0,:,:,1][instance_ind] + plt.plot(x, y, 'o') + #plt.imshow(valid_label[i]) + + elif feature_dim==3: + #for i in range(valid_pred.shape[0]): + # ax = fig.add_subplot(2,2,i+1, projection='3d') + # #valid_pred = valid_pred[0] + # #valid_label = valid_label[0] + ax = fig.add_subplot(1,1,1, projection='3d') + num_unique = np.unique(valid_label[0]) + colors = [(0., 0., 1., 0.05), 'g', 'r', 'c', 'm', 'y'] + for color_id, unique in enumerate(list(num_unique)): + instance_ind = np.where(valid_label[0]==unique) + #print 'instance id', instance_ind + #print valid_pr[instance_ind].shape + x = valid_pred[0,:,:,0][instance_ind] + y = valid_pred[0,:,:,1][instance_ind] + z = valid_pred[0,:,:,2][instance_ind] + + ax.scatter(x, y, z, c=colors[color_id]) + elif feature_dim > 3: + plt.close(fig) + return None + + plt.savefig(os.path.join(log_dir, param_string, 'cluster_{}.png'.format(str(step).zfill(6))), bbox_inches='tight') + plt.close(fig) +