diff --git a/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/.DS_Store b/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..c9b9b7889599850787aa37ed0e89aa4ce5087003 Binary files /dev/null and b/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/.DS_Store differ diff --git a/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/.keep b/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/.keep new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/README.md b/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/README.md new file mode 100644 index 0000000000000000000000000000000000000000..5a72184198531df5e367f93ff8bf24f45a06e700 --- /dev/null +++ b/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/README.md @@ -0,0 +1,247 @@ +## 基本信息 + +**发布者（Publisher）：Huawei** + +**应用领域（Application Domain）：** Classification + +**版本（Version）：1.1** + +**修改时间（Modified） ：2022.05.15** + +**大小（Size）：** + +**框架（Framework）：TensorFlow 1.15.0** + +**模型格式（Model Format）：** + +**精度（Precision）：** + +**处理器（Processor）：昇腾910** + +**应用级别（Categories）：Research** + +**描述（Description）：基于TensorFlow框架的VisionTransformer图像分类网络训练代码** + +## 概述 + +当前Transformer模型被大量应用在NLP自然语言处理当中，而在计算机视觉领域，Transformer的注意力机制attention也被广泛应用，比如Se模块，CBAM模块等等注意力模块，这些注意力模块能够帮助提升网络性能。而VisionTransformer展示了不需要依赖CNN的结构，也可以在图像分类任务上达到很好的效果，并且也十分适合用于迁移学习。 + + +- 参考论文： + + Dosovitskiy, A., Beyer, L., Kolesnikov, A., Weissenborn, D., Zhai, X., Unterthiner, T., Dehghani, M., Minderer, M., Heigold, G., Gelly, S., Uszkoreit, J., & Houlsby, N. (2021). An Image is Worth 16x16 Words: Transformers for Image Recognition at Scale. ArXiv, abs/2010.11929. + + + - arXiv:2010.11929(http://xxx.itp.ac.cn/pdf/2010.11929.pdf) + +- 参考实现： + + https://github.com/faustomorales/vit-keras + +- 适配昇腾 AI 处理器的实现： + + + https://gitee.com/ascend/ModelZoo-TensorFlow/tree/master/TensorFlow/contrib/cv/BicycleGAN_ID1287_for_TensorFlow + + + +- 通过Git获取对应commit\_id的代码方法如下： + + ``` + git clone {repository_url} # 克隆仓库的代码 + cd {repository_name} # 切换到模型的代码仓目录 + git checkout {branch} # 切换到对应分支 + git reset --hard ｛commit_id｝ # 代码设置到对应的commit_id + cd ｛code_path｝ # 切换到模型代码所在路径，若仓库下只有该模型，则无需切换 + ``` + +- 精度 + +| | GPU | NPU | +|-------|-------|-------| +| LPIPS | 0.8709 | 0.871 | + +- 性能 + +| batchsize | image_size | GPU （v100） | NPU | +|-----------|------------|---|---| +| 1 | 384*384 | | | + + +## 默认配置 + +- 训练数据集预处理（以原论文的maps训练集为例，仅作为用户参考示例）： + + - 图像的输入尺寸为384*384 + - 图像输入格式：从cifar压缩文件中读取数据 +```python + def _preprocess(image, label): + image = tf.image.resize(image, (image_size, image_size)) + image = (image - 127.5) / 127.5 + return image, label +``` + +- 测试数据集预处理（以原论文的maps验证集为例，仅作为用户参考示例） + + - 图像的输入尺寸为384*384 + - 图像输入格式：从cifar压缩文件中读取数据 +```python + def _preprocess(image, label): + image = tf.image.resize(image, (image_size, image_size)) + image = (image - 127.5) / 127.5 + return image, label +``` + +- 训练超参 + + - Batch size: 4 + - Learning rate(LR): 0.001 + - Optimizer: Adam + - Train epoch: 1 + +## 支持特性 + +| 特性列表 | 是否支持 | +|-------|------| +| 分布式训练 | 否 | +| 混合精度 | 是 | +| 并行数据 | 否 | + + + +

训练环境准备

+ +1. 硬件环境准备请参见各硬件产品文档"[驱动和固件安装升级指南]( https://support.huawei.com/enterprise/zh/category/ai-computing-platform-pid-1557196528909)"。需要在硬件设备上安装与CANN版本配套的固件与驱动。 +2. 宿主机上需要安装Docker并登录[Ascend Hub中心](https://ascendhub.huawei.com/#/detail?name=ascend-tensorflow-arm)获取镜像。 + + 当前模型支持的镜像列表如[表1](#zh-cn_topic_0000001074498056_table1519011227314)所示。 + + **表 1** 镜像列表 + + + + + + + + + + + + +

镜像名称

+

镜像版本

+

配套CANN版本

+
+

20.2.0

+

20.2

+
+ + +## 注意事项 +1. 本项目是在104机器裸机上完成的，线上modelart的总是报算子错。 +2. vision transformer 需要加载预训练才能达到好的效果，这是是加载了gpu训练的模型。 + +

快速上手

+ +- 数据集准备 +1. 104机器上位置为 `/home/HwHiAiUser/wubo/vit/vit-ckpt/datasets` 另外obs也有 + +2. 获得数据集后，放入模型目录下，在训练脚本中指定数据集路径，可正常使用。 + + +## 模型训练 + +- 单击“立即下载”，并选择合适的下载方式下载源码包。 + +- 启动训练之前，首先要配置程序运行相关环境变量。 + + 环境变量配置信息参见： + + [Ascend 910训练平台环境变量设置](https://gitee.com/ascend/modelzoo/wikis/Ascend%20910%E8%AE%AD%E7%BB%83%E5%B9%B3%E5%8F%B0%E7%8E%AF%E5%A2%83%E5%8F%98%E9%87%8F%E8%AE%BE%E7%BD%AE?sort_id=3148819) + +- 单卡训练 + + 1. 配置训练参数。 + + 首先在脚本test/train_full_1p.sh中，配置训练数据集路径，请用户根据实际路径配置，数据集参数如下所示： + + ``` + --data_path ./dataset + ``` + + 2. 启动训练。 + + 启动单卡训练 （脚本为modelarts_entry_acc.py） + + ``` + python3 modelarts_entry_acc.py + ``` + + + +

迁移学习指导

+ +- 数据集准备。 + + 数据集要求如下： + + 1. 获取数据。 + + 如果要使用自己的数据集， obs://cann-id1217/ 需要将数据集放到脚本参数data_path对应目录下。参考代码中的数据集存放路径如下： + + - 训练集： ./dataset/train + - 测试集： ./dataset/val + + 数据集也可以放在其它目录，则修改对应的脚本入参data_path即可。 + + +- 模型训练。 + + 参考“模型训练”中训练步骤。 + +- 模型评估。 + + 参考“模型训练”中验证步骤。 + +

高级参考

+ +## 脚本和示例代码 + +``` +VisionTransformer +└─ + ├─README.md + ├─dataset用于存放训练数据集和预训练文件 + |-model/ + ├─train + └─val + ├─precision_tool 用来去掉某一些算子融合策略 + └─... + ├─test 用于测试 + ├─output 用于存放测试结果和日志 + └─test_1p.sh +``` + +## 脚本参数 + +``` +--learning_rate 学习率,默认是0.001 +--batch_size 训练的batch大小，默认是4 +--data_path 训练集文件路径 +--output_path 日志,模型文件等存放的路径 +``` + + +## 训练过程 + +1. 通过“模型训练”中的训练指令启动单卡训练。 + +2. 参考脚本的模型存储路径为./output。 + + + +## 推理/验证过程 + + + diff --git a/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/fusion_result.json b/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/fusion_result.json new file mode 100644 index 0000000000000000000000000000000000000000..d8a4da7ff4977c0f09e51ef56036e312962dafd9 --- /dev/null +++ b/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/fusion_result.json @@ -0,0 +1,383 @@ +{ + "graph_fusion": { + "MulAddFusionPass": { + "effect_times": "0", + "match_times": "148" + }, + "MulSquareFusionPass": { + "effect_times": "0", + "match_times": "75" + } + }, + "session_and_graph_id": "0_1", + "ub_fusion": { + "AutomaticUbFusion": { + "effect_times": "74", + "match_times": "74" + } + } +}{ + "graph_fusion": { + "AABiasaddConvFusion": { + "effect_times": "1", + "match_times": "1" + }, + "AAMatMulNzToNdFusionPass": { + "effect_times": "0", + "match_times": "402" + }, + "AReduceAllFusionPass": { + "effect_times": "0", + "match_times": "1" + }, + "AReduceMeanFusionPass": { + "effect_times": "0", + "match_times": "50" + }, + "AReduceSumFusionPass": { + "effect_times": "0", + "match_times": "115" + }, + "ASoftmaxFusionPass": { + "effect_times": "0", + "match_times": "12" + }, + "AddNFusionPass": { + "effect_times": "0", + "match_times": "87" + }, + "ApplyAddOutputPass": { + "effect_times": "200", + "match_times": "200" + }, + "BatchMatMulFusionPass": { + "effect_times": "0", + "match_times": "291" + }, + "BatchMatMulV2ReduceFusionPass": { + "effect_times": "0", + "match_times": "72" + }, + "BatchMatMulV2ReshapeFusionPass": { + "effect_times": "0", + "match_times": "72" + }, + "CastRemoveFusionPass": { + "effect_times": "0", + "match_times": "36" + }, + "ConstToAttrPass": { + "effect_times": "351", + "match_times": "351" + }, + "ConstToAttrReduceSumFusion": { + "effect_times": "115", + "match_times": "115" + }, + "ConstToAttrStridedSliceFusion": { + "effect_times": "1", + "match_times": "1" + }, + "Conv2DbpFilterMulFusionPass": { + "effect_times": "0", + "match_times": "1" + }, + "ConvConcatFusionPass": { + "effect_times": "0", + "match_times": "1" + }, + "ConvToFullyConnectionFusionPass": { + "effect_times": "0", + "match_times": "1" + }, + "ConvWeightCompressFusionPass": { + "effect_times": "0", + "match_times": "220" + }, + "FIXPIPEAPREQUANTFUSIONPASS": { + "effect_times": "0", + "match_times": "293" + }, + "FIXPIPEFUSIONPASS": { + "effect_times": "0", + "match_times": "293" + }, + "ForceFp16CastFusionPass": { + "effect_times": "0", + "match_times": "36" + }, + "LayerNormTrainingFusionPass": { + "effect_times": "25", + "match_times": "25" + }, + "MatMulBiasAddFusionPass": { + "effect_times": "1", + "match_times": "1" + }, + "MatMulReshapeBiasAddFusionPass": { + "effect_times": "72", + "match_times": "72" + }, + "MatMulV2FusionPass": { + "effect_times": "0", + "match_times": "72" + }, + "MulAddNL2LossFusionPass": { + "effect_times": "0", + "match_times": "40" + }, + "MulAddNPass": { + "effect_times": "0", + "match_times": "40" + }, + "MulGradFusionPass": { + "effect_times": "0", + "match_times": "87" + }, + "MulSquareFusionPass": { + "effect_times": "0", + "match_times": "703" + }, + "RefreshInt64ToInt32FusionPass": { + "effect_times": "1", + "match_times": "1" + }, + "ReshapeTransposeFusionPass": { + "effect_times": "0", + "match_times": "48" + }, + "SoftmaxFusionPass": { + "effect_times": "0", + "match_times": "12" + }, + "SoftmaxGradExtFusion": { + "effect_times": "12", + "match_times": "12" + }, + "SparseSoftMaxFusionPass": { + "effect_times": "1", + "match_times": "1" + }, + "SplitConvConcatFusionPass": { + "effect_times": "0", + "match_times": "1" + }, + "StridedSliceGradFusionPass": { + "effect_times": "0", + "match_times": "1" + }, + "StridedSliceRemovePass": { + "effect_times": "0", + "match_times": "1" + }, + "SubFusionPass": { + "effect_times": "0", + "match_times": "25" + }, + "TileConstToAttrFusion": { + "effect_times": "50", + "match_times": "50" + }, + "TransdataCastFusionPass": { + "effect_times": "0", + "match_times": "857" + }, + "TransposeReshapeFusionPass": { + "effect_times": "0", + "match_times": "84" + }, + "TransposedUpdateFusionPass": { + "effect_times": "96", + "match_times": "96" + }, + "ZConcatExt2FusionPass": { + "effect_times": "1", + "match_times": "1" + }, + "ZConcatv2dFusionPass": { + "effect_times": "0", + "match_times": "1" + }, + "softmaxTransFusionPass": { + "effect_times": "0", + "match_times": "12" + } + }, + "session_and_graph_id": "0_21", + "ub_fusion": { + "AutomaticUbFusion": { + "effect_times": "25", + "match_times": "25" + }, + "TbeEltwiseFusionPass": { + "effect_times": "12", + "match_times": "12" + }, + "TbeFullyconnectionElemwiseDequantFusionPass": { + "effect_times": "85", + "match_times": "85" + }, + "TbeMultiOutputFusionPass": { + "effect_times": "37", + "match_times": "37" + } + } +}{ + "graph_fusion": { + "RefreshInt64ToInt32FusionPass": { + "effect_times": "1", + "match_times": "1" + }, + "TransdataCastFusionPass": { + "effect_times": "0", + "match_times": "2" + } + }, + "session_and_graph_id": "0_31" +}{ + "graph_fusion": { + "AABiasaddConvFusion": { + "effect_times": "1", + "match_times": "1" + }, + "AAMatMulNzToNdFusionPass": { + "effect_times": "0", + "match_times": "146" + }, + "AReduceMeanFusionPass": { + "effect_times": "0", + "match_times": "50" + }, + "ASoftmaxFusionPass": { + "effect_times": "0", + "match_times": "12" + }, + "BatchMatMulFusionPass": { + "effect_times": "0", + "match_times": "97" + }, + "BatchMatMulV2ReduceFusionPass": { + "effect_times": "0", + "match_times": "24" + }, + "BatchMatMulV2ReshapeFusionPass": { + "effect_times": "0", + "match_times": "24" + }, + "CastRemoveFusionPass": { + "effect_times": "0", + "match_times": "36" + }, + "ConstToAttrPass": { + "effect_times": "99", + "match_times": "99" + }, + "ConstToAttrStridedSliceFusion": { + "effect_times": "1", + "match_times": "1" + }, + "ConvConcatFusionPass": { + "effect_times": "0", + "match_times": "1" + }, + "ConvToFullyConnectionFusionPass": { + "effect_times": "0", + "match_times": "1" + }, + "ConvWeightCompressFusionPass": { + "effect_times": "0", + "match_times": "74" + }, + "FIXPIPEAPREQUANTFUSIONPASS": { + "effect_times": "0", + "match_times": "98" + }, + "FIXPIPEFUSIONPASS": { + "effect_times": "0", + "match_times": "98" + }, + "ForceFp16CastFusionPass": { + "effect_times": "0", + "match_times": "36" + }, + "LayerNormTrainingFusionPass": { + "effect_times": "25", + "match_times": "25" + }, + "MatMulBiasAddFusionPass": { + "effect_times": "1", + "match_times": "1" + }, + "MatMulReshapeBiasAddFusionPass": { + "effect_times": "72", + "match_times": "72" + }, + "MatMulV2FusionPass": { + "effect_times": "0", + "match_times": "72" + }, + "MulSquareFusionPass": { + "effect_times": "0", + "match_times": "108" + }, + "RefreshInt64ToInt32FusionPass": { + "effect_times": "1", + "match_times": "1" + }, + "ReshapeTransposeFusionPass": { + "effect_times": "0", + "match_times": "36" + }, + "SoftmaxFusionPass": { + "effect_times": "0", + "match_times": "12" + }, + "SplitConvConcatFusionPass": { + "effect_times": "0", + "match_times": "1" + }, + "StridedSliceRemovePass": { + "effect_times": "0", + "match_times": "1" + }, + "TransdataCastFusionPass": { + "effect_times": "0", + "match_times": "271" + }, + "TransposeReshapeFusionPass": { + "effect_times": "0", + "match_times": "12" + }, + "TransposedUpdateFusionPass": { + "effect_times": "48", + "match_times": "48" + }, + "ZConcatExt2FusionPass": { + "effect_times": "1", + "match_times": "1" + }, + "ZConcatv2dFusionPass": { + "effect_times": "0", + "match_times": "1" + }, + "softmaxTransFusionPass": { + "effect_times": "0", + "match_times": "12" + } + }, + "session_and_graph_id": "0_41", + "ub_fusion": { + "AutomaticUbFusion": { + "effect_times": "36", + "match_times": "36" + }, + "TbeEltwiseFusionPass": { + "effect_times": "12", + "match_times": "12" + }, + "TbeFullyconnectionElemwiseDequantFusionPass": { + "effect_times": "12", + "match_times": "12" + } + } +} \ No newline at end of file diff --git a/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/modelarts_entry_acc.py b/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/modelarts_entry_acc.py new file mode 100644 index 0000000000000000000000000000000000000000..13077b10e660de32d6f7861257a50e1a01ede9ba --- /dev/null +++ b/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/modelarts_entry_acc.py @@ -0,0 +1,63 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import argparse +import sys + +# 解析输入参数data_url +parser = argparse.ArgumentParser() +parser.add_argument("--data_url", type=str, default="/home/ma-user/modelarts/inputs/data_url_0") +parser.add_argument("--train_url", type=str, default="/home/ma-user/modelarts/outputs/train_url_0/") +config = parser.parse_args() + +print("[CANN-Modelzoo] code_dir path is [%s]" % (sys.path[0])) +code_dir = sys.path[0] +os.chdir(code_dir) +print("[CANN-Modelzoo] work_dir path is [%s]" % (os.getcwd())) + +print("[CANN-Modelzoo] before train - list my run files:") +os.system("ls -al /usr/local/Ascend/ascend-toolkit/") + +print("[CANN-Modelzoo] before train - list my dataset files:") +os.system("ls -al %s" % config.data_url) + +print("[CANN-Modelzoo] start run train shell") +# 设置sh文件格式为linux可执行 +os.system("dos2unix ./test/*") + +# 执行train_full_1p.sh或者train_performance_1p.sh，需要用户自己指定 +# full和performance的差异，performance只需要执行很少的step，控制在15分钟以内，主要关注性能FPS +os.system("bash ./test/train_full_1p.sh --data_path=%s --output_path=%s " % (config.data_url, config.train_url)) + +print("[CANN-Modelzoo] finish run train shell") + +# 将当前执行目录所有文件拷贝到obs的output进行备份 +print("[CANN-Modelzoo] after train - list my output files:") +os.system("cp -r %s %s " % (code_dir, config.train_url)) +os.system("ls -al %s" % config.train_url) diff --git a/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/modelarts_entry_perf.py b/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/modelarts_entry_perf.py new file mode 100644 index 0000000000000000000000000000000000000000..14384e227a0fa90a514254590aef5078c62ff700 --- /dev/null +++ b/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/modelarts_entry_perf.py @@ -0,0 +1,63 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import argparse +import sys + +# 解析输入参数data_url +parser = argparse.ArgumentParser() +parser.add_argument("--data_url", type=str, default="/home/ma-user/modelarts/inputs/data_url_0") +parser.add_argument("--train_url", type=str, default="/home/ma-user/modelarts/outputs/train_url_0/") +config = parser.parse_args() + +print("[CANN-Modelzoo] code_dir path is [%s]" % (sys.path[0])) +code_dir = sys.path[0] +os.chdir(code_dir) +print("[CANN-Modelzoo] work_dir path is [%s]" % (os.getcwd())) + +print("[CANN-Modelzoo] before train - list my run files:") +os.system("ls -al /usr/local/Ascend/ascend-toolkit/") + +print("[CANN-Modelzoo] before train - list my dataset files:") +os.system("ls -al %s" % config.data_url) + +print("[CANN-Modelzoo] start run train shell") +# 设置sh文件格式为linux可执行 +os.system("dos2unix ./test/*") + +# 执行train_full_1p.sh或者train_performance_1p.sh，需要用户自己指定 +# full和performance的差异，performance只需要执行很少的step，控制在15分钟以内，主要关注性能FPS +os.system("bash ./test/train_performance_1p.sh --data_path=%s --output_path=%s " % (config.data_url, config.train_url)) + +print("[CANN-Modelzoo] finish run train shell") + +# 将当前执行目录所有文件拷贝到obs的output进行备份 +print("[CANN-Modelzoo] after train - list my output files:") +os.system("cp -r %s %s " % (code_dir, config.train_url)) +os.system("ls -al %s" % config.train_url) diff --git a/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/msprof.py b/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/msprof.py new file mode 100755 index 0000000000000000000000000000000000000000..550663f8b02211027a2ef5eef4b16d6cc99100b2 --- /dev/null +++ b/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/msprof.py @@ -0,0 +1,48 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/usr/bin/python3 +# coding=utf-8 +""" +Function: +This class mainly involves the main function. +Copyright Information: +Huawei Technologies Co., Ltd. All Rights Reserved © 2020 +""" +import importlib +import os +import sys + +if __name__ == '__main__': + sys.path.insert(0, os.path.realpath(os.path.join(os.path.dirname(__file__), '..'))) + _model_path = "ms_interface.msprof_entrance" + MSPROF_ENTRANCE_CLASS = "MsprofEntrance" + os.umask(0o027) + msprof_entrance_module = importlib.import_module(_model_path) + if hasattr(msprof_entrance_module, MSPROF_ENTRANCE_CLASS): + getattr(msprof_entrance_module, MSPROF_ENTRANCE_CLASS)().main() diff --git a/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/precision_tool/.DS_Store b/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/precision_tool/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..ca63ff7dc12e7690e5692279ae870da9520b7de1 Binary files /dev/null and b/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/precision_tool/.DS_Store differ diff --git a/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/precision_tool/README.md b/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/precision_tool/README.md new file mode 100644 index 0000000000000000000000000000000000000000..39c36d6f2f99665a7bbe1ebfe23f735f4ca965f6 --- /dev/null +++ b/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/precision_tool/README.md @@ -0,0 +1,490 @@ +# 精度问题分析工具 + +## 功能介绍 +### 简介 +该工具包提供了精度比对常用的功能，当前该工具主要适配Tensorflow训练场景 + +对于常用功能基本可以做到一键操作，同时提供Dump数据/图信息的交互式查询和操作入口 + +推理场景可直接使用[推理一键式全流程精度比对](https://gitee.com/ascend/tools/tree/master/msquickcmp) 工具 +### 主要功能 +#### 已完成功能 +1. 简化脚本修改【手动/半自动】 +2. TF标杆数据生成【自动/半自动】 +3. 算子溢出检测分析【自动】 +4. 开启GE图Dump和图解析【自动】 +5. 开启数据Dump并进行全网比对【自动】 +6. 查询算子列表/节点信息【手动】 +7. 查询/解析Dump数据信息【手动】 +8. 数据比对【手动】 +### 工具获取 +1. 下载压缩包的方式获取 + 将https://gitee.com/ascend/tools 以压缩包形式下载 +2. 使用git命令方式获取 +3. 移动 tools/precision_tool 子目录至训练工作目录 +### 安装python3三方依赖 +```shell +pip3 install rich gnureadline pexpect graphviz +# ubuntu/Debian +sudo apt-get install graphviz +# fedora/Centos +sudo yum install graphviz +``` +### 工具执行依赖 +* 一般直接在NPU训练环境上部署该脚本，环境上能够正常执行CPU和NPU训练脚本 +* 如果需要进行数据Dump比对，则需要先检查并去除训练脚本内部使用到的随机处理，避免由于输入数据不一致导致数据比对结果不可用 + ```python + # 对于使用tf.random / np.random / (python) random的可以通过固定随机种子的方式固定输入 + # import tf_config.py 默认会设置上述三种random的seed，但由于import位置关系，可能不一定能作用到所有的关联代码，建议在代码确认合适位置手动嵌入 + random.seed(cfg.DUMP_SEED) + tf.random.set_random_seed(cfg.DUMP_SEED) + np.random.seed(cfg.DUMP_SEED) + + # 此处给出一些典型示例，需要根据自己的脚本进行排查 + + # 1. 参数初始化中的随机操作 + # 加载checkpoint的方式能够固定大多数初始参数 + saver.restore(sess, saver_dir) + + # 2. 输入数据的随机操作（例如对输入数据做shuffle操作） + dataset = tf.data.TFRecordDataset(tf_data) + dataset = dataset.shuffle(batch_size*10) # 直接注释掉该行 + + # 3. 模型中的随机操作（例如使用dropout） + net = slim.dropout(net, keep_prob=dropout_keep_prob, scope='Dropout_1b') # 建议注释该行 + + # 4. 图像预处理使用的随机操作(根据实际情况固定随机种子，或者替换成其他固定的预处理操作) + # 4.1 Random rotate + random_angle = tf.random_uniform([], - self.degree * 3.141592 / 180, self.degree * 3.141592 / 180) + image = tf.contrib.image.rotate(image, random_angle, interpolation='BILINEAR') + depth_gt = tf.contrib.image.rotate(depth_gt, random_angle, interpolation='NEAREST') + + # 4.2 Random flipping + do_flip = tf.random_uniform([], 0, 1) + image = tf.cond(do_flip > 0.5, lambda: tf.image.flip_left_right(image), lambda: image) + depth_gt = tf.cond(do_flip > 0.5, lambda: tf.image.flip_left_right(depth_gt), lambda: depth_gt) + + # 4.3 Random crop + mage_depth = tf.concat([image, depth_gt], 2) + image_depth_cropped = tf.random_crop(image_depth, [self.params.height, self.params.width, 4]) + + # 5. RunConfig/NPURunConfig中设置tf_random_seed固定网络随机因子 + run_config = tf.estimator.RunConfig(tf_random_seed=1, ...) + run_config = NPURunConfig(tf_random_seed=1, ...) + + # 其他...... + ``` +* 该工具基于**NPU的计算图**，**NPU的DUMP数据**，**NPU的溢出检测数据**，**TF的计算图meta文件**，**TF的DUMP数据**进行数据解析和分析。 +这几类依赖数据可以通过以下方式获取（只使用部分工具功能并不需要提前获取所有依赖数据）： +#### 1. NPU的计算图获取 + ``` + 注意：NPU的Dump数据和计算图存在一定的对应关系，需要同时获取 + 避免在自定义的训练脚本中unset DUMP GRAPH相关的环境变量 + ``` +* 【推荐】方法一：配置2、3依赖中的NPU数据Dump或者overflow检测功能，将自动配置上Dump GE图的环境变量 + +* 【不推荐】方法二：参考迁移指导中的修改配置，执行NPU脚本，并将获取到的图转存至precision_data图目录 + ```shell + export DUMP_GE_GRAPH=2 + export DUMP_GRAPH_LEVEL=3 + export DUMP_GRAPH_PATH=./precision_data/npu/debug_0/graph + # 未配置DUMP_GRAPH_PATH时，图文件将保存在脚本执行目录，可以直接转存至precision_data目录 + mkdir -p ./precision_data/npu/debug_0/graph && mv ge_proto_*.txt ./precision_data/npu/debug_0/graph + ``` +#### 2. NPU的DUMP数据获取 +* 【推荐】方法一：在训练脚本中**import precision_tool.tf_config**，并使用precision_tool中提供的辅助命令行执行训练脚本 + ``` python + # NPU的DUMP获取和溢出检测数据的获取，均可按如下方式修改代码 + # 注意：参数action可以设置为'dump'或'overflow' + # 引用 precision_tool/tf_config.py + import precision_tool.tf_config as npu_tf_config + + # 如果使用的是Estimator的NPURunConfig配置使能NPU，则可以参考以下修改 + dump_config = npu_tf_config.estimator_dump_config(action='dump') # 新增行 + npu_config = NPURunConfig(dump_config=dump_config) + + # 如果使用的是session.run或者使用tf.ConfigProto创建session_config传入tf.estimator.RunConfig的方式使能npu + # 可以参考如下修改 + session_config = npu_tf_config.session_dump_config(session_config, action='dump') # 新增行 + # tf.estimator + run_config = tf.estimator.RunConfig(session_config=session_config,...) + # tf.keras + npu_keras_sess = set_keras_session_npu_config(config=session_config) + # session run + with tf.Session(config=npu_config_proto(session_config)): + ...... + + # 如果使用的是custom_op方式，则可以参考以下修改 + config = tf.ConfigProto() + custom_op = config.graph_options.rewrite_options.custom_optimizers.add() + custom_op.name = "NpuOptimizer" + custom_op.parameter_map["use_off_line"].b = True + custom_op = npu_tf_config.update_custom_op(custom_op, action='dump') # 新增行 + ``` + +* 【不推荐】方法二：参考[精度比对工具使用指南](https://www.hiascend.com/document?tag=community-developer) 修改训练脚本。 + 执行训练脚本，并将dump的数据拷贝到【precision_data/dump/npu/】目录 +#### 3. NPU的溢出检测数据的获取（缺少该数据将无法展示溢出检测结果） +* 【推荐】方法一：在训练脚本中**import precision_tool.tf_config**，并按【2. NPU的DUMP数据获取】中修改训练代码，使用precision_tool中提供的辅助命令行执行训练脚本 + ```python + # 需要将action设置成'overflow' + # 引用 precision_tool/tf_config.py + import precision_tool.tf_config as npu_tf_config + dump_config = npu_tf_config.estimator_dump_config(action='overflow') # 新增行 + ``` +* 【不推荐】方法二：参考[使用溢出检测工具分析算子溢出](https://www.hiascend.com/document?tag=community-developer) 修改训练脚本， + 并将溢出数据拷贝至【precision_tool/dump/overflow/】目录 + +#### 4. TF的DUMP数据获取（缺少该数据无法使用数据比对功能） +* 【推荐】方法一：在CPU/GPU训练脚本中添加tf_debug代码，并使用precision_tool中提供的辅助命令行工具生成标杆DUMP数据 + ```python + import precision_tool.tf_config as npu_tf_config + + # 如果使用的是Estimator,EstimatorSpec加入training_hooks + estim_specs = tf.estimator.EstimatorSpec(training_hooks=[npu_tf_config.estimator_dump()]) + + # 如果使用的session.run，以下代码在为sess加上了tf_debug的wrapper + sess = npu_tf_config.sess_dump(sess=sess) + ``` + ```shell + # 1. 执行脚本 + # 2. 解析tf debug dump文件，生成算子输出tensor文件 + # 注意：TF dump数据的原理是使用tf_debug的print_tensor(pt)命令实现的，由于训练代码提供了非常灵活的run()接口， + # 脚本无法感知用户需要dump的tensor在哪个run阶段，因此需要用户修改训练代码，在执行完正确的run后，立即退出。 + # 例如，修改代码只执行一个step的训练，根据代码中run的次数，会获取到1~N个离线tf_debug的dump目录 + # precision_tool脚本会自动提取最后一个run阶段中出现的所有tensor作为标杆数据。 + python3.7.5 precision_tool/cli.py tf_dump + + # 在precision_data/tf/dump/ 目录会存放提取的tensor + # 如果获取tensor不符合预期，可以检查下precision_data/dump/cpu_debug/目录, 只保留预期run阶段的tf_debug离线数据 + # 执行以下命令重新生成 + rm -rf precision_data/tf/dump/* && python3.7.5 precision_tool/cli.py tf_dump + ``` +* 【不推荐】方法二：参考[准备基于GPU/CPU运行生成的npy数据](https://www.hiascend.com/document?tag=community-developer) + 获取CPU/GPU的TF数据，并拷贝至【precision/dump/cpu/】目录 +#### 5. TF计算图Meta文件的获取（可选） +* 通过saver保存ckpt获取 + ```python + # 修改CPU/NPU脚本 + with tf.Session() as sess: + # do session.run() + saver = tf.train.Saver() + # 保存ckpt + saver.save(sess, saver_dir) + ``` +#### 6. 关闭NPU的融合功能（根据情况启用） +* NPU会对计算图中的算子进行融合，以提高网络性能，由于大多数融合是自动识别的，可能存在未考虑到的场景，导致精度问题， + 因此，可以尝试关闭融合定界网络问题是否是由于融合导致。 + ```python + # 关闭融合可以和溢出检测/数据Dump同时进行，启用方法也类似 + # 引用 precision_tool/tf_config.py + import precision_tool.tf_config as npu_tf_config + + # 如果使用的是Estimator的NPURunConfig配置使能NPU，则可以参考以下修改 + npu_config = NPURunConfig(fusion_switch_file=npu_tf_config.FUSION_OFF_FILE) # 修改行 + # 如果需要关闭指定的融合规则，则可以修改precision_tool/fusion_switch.cfg, 并参考如下修改 + npu_config = NPURunConfig(fusion_switch_file=npu_tf_config.FUSION_SWITCH_FILE) # 关闭特定融合修改行 + + # 如果使用的是session.run或者使用tf.ConfigProto创建session_config传入tf.estimator.RunConfig的方式使能npu + # 可以参考如下修改(数据Dump和关闭融合同时使能) + session_config = npu_tf_config.session_dump_config(session_config, action='dump|fusion_off') # 新增行 + session_config = npu_tf_config.session_dump_config(session_config, action='dump|fusion_switch') # 关闭特定融合新增行 + # tf.estimator + run_config = tf.estimator.RunConfig(session_config=session_config,...) + # tf.keras + npu_keras_sess = set_keras_session_npu_config(config=session_config) + # session run + with tf.Session(config=npu_config_proto(session_config)): + ...... + ``` +## 使用说明 +1. 配置文件precision_tool/config.py（正常默认即可） + ```python + # 如果需要dump特定曾的数据，则可以修改以下配置项 + # 一般对比分析dump首层即可 + # Dump config '0|5|10' + TF_DUMP_STEP = '0' + + # 融合开关配置，可以再该配置文件中配置融合开关的开启和关闭，使用方法参考以下文档： + # https://support.huaweicloud.com/tensorflowdevg-cann330alphaXtraining/atlastfadapi_07_0005.html + FUSION_SWITCH_FILE = './precision_tool/fusion_switch.cfg' + + # 依赖run包中的atc和msaccucmp.pyc工具，一般在run包安装目录，配置到父目录即可 + # 默认run包安装在/usr/local/Ascend，可以不用修改。指定目录安装则需要修改 + # parent dir path of msaccucmp.pyc and atc, usually run package dir + CMD_ROOT_PATH = '/usr/local/' + + # ASCEND Log Path + ASCEND_LOG_PATH = '/root/ascend/log/plog/' + + # 日志级别及数据分析目录设置 + # TOOL CONFIG + LOG_LEVEL = "NOTSET" + # ModelArts场景下，可以根据情况将数据跟目录修改成自定义目录，并在完成后完整下载该目录 + ROOT_DIR = './' + ``` +2. 启动脚本（交互命令行） + ```shell + python3 ./precision_tool/cli.py + ``` + +### 交互模式命令 +1. ac -l [limit_num] -c + ```shell + # auto check. 自动化检测命令 + # 列出Fusion信息;解析算子溢出信息; + # -c 可选，进行全网比对 + # -l 可选，限制输出结果的条数（overflow解析的条数等） + PrecisionTool > ac -c + ╭──────────────────────────────────────────────────────────────────────────────────────────────────╮ + │ [TransData][327] trans_TransData_1170 │ + │ - [AI Core][Status:32][TaskId:327] ['浮点计算有溢出'] │ + │ - First overflow file timestamp [1619347786532995] - │ + │ |- TransData.trans_TransData_1170.327.1619347786532995.input.0.npy │ + │ |- [Shape: (32, 8, 8, 320)] [Dtype: bool] [Max: True] [Min: False] [Mean: 0.11950836181640626] │ + │ |- TransData.trans_TransData_1170.327.1619347786532995.output.0.npy │ + │ |- [Shape: (32, 20, 8, 8, 16)] [Dtype: bool] [Max: True] [Min: False] [Mean: 0.07781982421875] │ + ╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ + ``` +2. run [command] + ```shell + # 不退出交互命令环境执行shell命令，与内置命令不冲突的可以直接执行，否则需要加run前缀 + PrecisionTool > run vim cli.py + PrecisionTool > vim cli.py + ``` + +3. ls -n [op_name] -t [op_type] + ```shell + # 通过[算子名]/[算子类型]查询网络里的算子，模糊匹配 + PrecisionTool > ls -t Mul -n mul_3 -f TbeMulti + [Mul][TbeMultiOutputFusionPass] InceptionV3/InceptionV3/Mixed_5b/Branch_1/mul_3 + [Mul][TbeMultiOutputFusionPass] InceptionV3/InceptionV3/Mixed_5c/Branch_1/mul_3 + [Mul][TbeMultiOutputFusionPass] InceptionV3/InceptionV3/Mixed_5d/Branch_1/mul_3 + [Mul][TbeMultiOutputFusionPass] InceptionV3/InceptionV3/Mixed_6b/Branch_1/mul_3 + ``` + +4. ni (-n) [op_name] -d -s [save sub graph deep] + ```shell + # 通过[算子名]查询算子节点信息 + # -d 显示相应的dump数据信息 + # -s 保存一个以当前算子节点为根，深度为参数值的子图 + PrecisionTool > ni gradients/InceptionV3/InceptionV3/Mixed_7a/Branch_0/Maximum_1_grad/GreaterEqual -d -s 3 + ╭─────────────────── [GreaterEqual]gradients/InceptionV3/InceptionV3/Mixed_7a/Branch_0/Maximum_1_grad/GreaterEqual ────────────────────╮ + │ [GreaterEqual] gradients/InceptionV3/InceptionV3/Mixed_7a/Branch_0/Maximum_1_grad/GreaterEqual │ + │ Input: │ + │ -[0][DT_FLOAT][NHWC][32, 8, 8, 320] InceptionV3/InceptionV3/Mixed_7a/Branch_0/add_3:0 │ + │ -[1][DT_FLOAT][NHWC][1, 8, 1, 1] InceptionV3/Mixed_7a/Branch_0/Conv2d_1a_3x3tau:0 │ + │ -[2][][[]][] atomic_addr_clean0_21:-1 │ + │ Output: │ + │ -[0][DT_BOOL][NHWC][32, 8, 8, 320] ['trans_TransData_1170'] │ + │ NpuDumpInput: │ + │ -[0] GreaterEqual.gradients_InceptionV3_InceptionV3_Mixed_7a_Branch_0_Maximum_1_grad_GreaterEqual.325.1619494134722860.input.0.npy │ + │ |- [Shape: (32, 8, 8, 320)] [Dtype: float32] [Max: 5.846897] [Min: -8.368301] [Mean: -0.72565556] │ + │ -[1] GreaterEqual.gradients_InceptionV3_InceptionV3_Mixed_7a_Branch_0_Maximum_1_grad_GreaterEqual.325.1619494134722860.input.1.npy │ + │ |- [Shape: (1, 8, 1, 1)] [Dtype: float32] [Max: 0.0] [Min: 0.0] [Mean: 0.0] │ + │ NpuDumpOutput: │ + │ -[0] GreaterEqual.gradients_InceptionV3_InceptionV3_Mixed_7a_Branch_0_Maximum_1_grad_GreaterEqual.325.1619494134722860.output.0.npy │ + │ |- [Shape: (32, 8, 8, 320)] [Dtype: bool] [Max: True] [Min: False] [Mean: 0.1176300048828125] │ + │ CpuDumpOutput: │ + │ -[0] gradients_InceptionV3_InceptionV3_Mixed_7a_Branch_0_Maximum_1_grad_GreaterEqual.0.1619492699305998.npy │ + │ |- [Shape: (32, 8, 8, 320)] [Dtype: bool] [Max: True] [Min: False] [Mean: 0.11764373779296874] │ + ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ + 2021-04-27 14:39:55 (15178) -[DEBUG]write 14953 bytes to './precision_data/dump/temp/op_graph/GreaterEqual.gradients_InceptionV3_InceptionV3_Mixed_7a_Branch_0_Maximum_1_grad_GreaterEqual.3.gv' + 2021-04-27 14:39:55 (15178) -[INFO]Sub graph saved to /root/sym/inception/precision_data/dump/temp/op_graph + ``` + +5. pt (-n) [*.npy] (-c) + ```shell + # 查看某个dump数据块的数据信息 + # -c : save data to txt + PrecisionTool > pt TransData.trans_TransData_1170.327.1619347786532995.input.0.npy -c + ╭─────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ + │ Shape: (32, 8, 8, 320) │ + │ Dtype: bool │ + │ Max: True │ + │ Min: False │ + │ Mean: 0.11950836181640626 │ + │ Path: ./precision_data/dump/temp/overflow_decode/TransData.trans_TransData_1170.327.1619347786532995.input.0.npy │ + │ TxtFile: ./precision_data/dump/temp/overflow_decode/TransData.trans_TransData_1170.327.1619347786532995.input.0.npy.txt │ + ╰─────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ + ``` + +6. cp (-n) [left *.npy] [right *.npy] -p [print num] -al [atol] -rl [rtol] + ```shell + # 对比两个tensor的数据 + # -p 指定输出的错误数据的个数及前多少个数据 + # -al/rl 指定相对误差的参数,在两个场景中用到 + # 1. np.allclose(left, right, atol=al, rtol=rl) + # 2. err_cnt += 1 if abs(data_left[i] - data_right[i]) > (al + rl * abs(data_right[i])) + PrecisionTool > cp Add.InceptionV3_InceptionV3_Mixed_7a_Branch_0_add_3.323.1619494134703053.output.0.npy InceptionV3_InceptionV3_Mixed_7a_Branch_0_add_3.0.1619492699305998.npy -p 10 -s -al 0.002 -rl 0.005 + Error Item Table Top Item Table + ┏━━━━━━━┳━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┓ ┏━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┓ + ┃ Index ┃ Left ┃ Right ┃ Diff ┃ ┃ Index ┃ Left ┃ Right ┃ Diff ┃ + ┡━━━━━━━╇━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━┩ ┡━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━┩ + │ 155 │ 0.024600908 │ 0.022271132 │ 0.002329776 │ │ 0 │ -0.9206961 │ -0.9222216 │ 0.0015255213 │ + │ 247 │ 0.015752593 │ 0.017937578 │ 0.0021849852 │ │ 1 │ -0.6416973 │ -0.64051837 │ 0.0011789203 │ + │ 282 │ -0.0101207765 │ -0.007852031 │ 0.0022687456 │ │ 2 │ -0.35383835 │ -0.35433492 │ 0.0004965663 │ + │ 292 │ 0.019581757 │ 0.02240482 │ 0.0028230622 │ │ 3 │ -0.18851271 │ -0.18883198 │ 0.00031927228 │ + │ 640 │ -0.06593232 │ -0.06874806 │ 0.0028157383 │ │ 4 │ -0.43508735 │ -0.43534422 │ 0.00025686622 │ + │ 1420 │ 0.09293677 │ 0.09586689 │ 0.0029301196 │ │ 5 │ 1.4447614 │ 1.4466647 │ 0.0019032955 │ + │ 1462 │ -0.085207745 │ -0.088047795 │ 0.0028400496 │ │ 6 │ -0.3455438 │ -0.3444429 │ 0.0011008978 │ + │ 1891 │ -0.03433288 │ -0.036525503 │ 0.002192624 │ │ 7 │ -0.6560242 │ -0.6564579 │ 0.0004336834 │ + │ 2033 │ 0.06828873 │ 0.07139922 │ 0.0031104907 │ │ 8 │ -2.6964858 │ -2.6975214 │ 0.0010356903 │ + │ 2246 │ -0.06376442 │ -0.06121233 │ 0.002552092 │ │ 9 │ -0.73746175 │ -0.73650354 │ 0.00095820427 │ + └───────┴───────────────┴──────────────┴──────────────┘ └───────┴─────────────┴─────────────┴───────────────┘ + ╭──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ + │ Left: │ + │ |- NpyFile: ./precision_data/dump/temp/decode/Add.InceptionV3_InceptionV3_Mixed_7a_Branch_0_add_3.323.1619494134703053.output.0.npy │ + │ |- TxtFile: ./precision_data/dump/temp/decode/Add.InceptionV3_InceptionV3_Mixed_7a_Branch_0_add_3.323.1619494134703053.output.0.npy.txt │ + │ |- NpySpec: [Shape: (32, 8, 8, 320)] [Dtype: float32] [Max: 5.846897] [Min: -8.368301] [Mean: -0.72565556] │ + │ DstFile: │ + │ |- NpyFile: ./precision_data/dump/cpu/InceptionV3_InceptionV3_Mixed_7a_Branch_0_add_3.0.1619492699305998.npy │ + │ |- TxtFile: ./precision_data/dump/cpu/InceptionV3_InceptionV3_Mixed_7a_Branch_0_add_3.0.1619492699305998.npy.txt │ + │ |- NpySpec: [Shape: (32, 8, 8, 320)] [Dtype: float32] [Max: 5.8425903] [Min: -8.374472] [Mean: -0.7256237] │ + │ NumCnt: 655360 │ + │ AllClose: False │ + │ CosSim: 0.99999493 │ + │ ErrorPer: 0.023504638671875 (rl= 0.005, al= 0.002) │ + ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ + ``` + +7. vc -lt [left_path] -rt [right_path] -g [graph] + ```python + # 用于手动指定两个目录进行整网对比 + # -lt 必选，其中一个文件目录 + # -rt 必选，另一个目录，一般是标杆目录 + # -g 可选，指定-g将尝试解析graph内的映射关系比对（一般用于NPU和TF之间的数据比对， NPU与NPU之间比对不需要，直接按照算子name对比） + ``` +8. vcs -f [file_name] -c [cos_sim_threshold] -l [limit] + ```python + # 查看精度比对结果的概要信息，可以更加预先相似的阈值过滤出低于阈值的算子/信息 + # -f (--file) 可选，指定csv文件，不设置则默认遍历precision_data/temp/vector_compare/目录下最近产生的对比目录内的所有csv + # -c (--cos_sim) 可选，指定筛选所使用的预先相似度阈值，默认0.98 + # -l (--limit) 可选，指定输出前多少个结果，默认值3 + PrecisionTool > vcs -c 0.98 -l 2 + 2021-05-31 14:48:56 (2344298) -[INFO]Sub path num:[1]. Dirs[['20210529145750']], choose[20210529145750] + 2021-05-31 14:48:56 (2344298) -[DEBUG]Find ['result_20210529145751.csv', 'result_20210529145836.csv', 'result_20210529145837.csv', 'result_20210529145849.csv', 'result_20210529150404.csv', 'result_20210529151102.csv'] result files in dir precision_data/temp/vector_compare/20210529145750 + 2021-05-31 14:48:56 (2344298) -[INFO]Find 0 ops less then 0.98 in precision_data/temp/vector_compare/20210529145750/result_20210529145751.csv + 2021-05-31 14:48:56 (2344298) -[INFO]Find 0 ops less then 0.98 in precision_data/temp/vector_compare/20210529145750/result_20210529145836.csv + 2021-05-31 14:48:56 (2344298) -[INFO]Find 1 ops less then 0.98 in precision_data/temp/vector_compare/20210529145750/result_20210529145837.csv + 2021-05-31 14:48:56 (2344298) -[INFO]Find 2 ops less then 0.98 in precision_data/temp/vector_compare/20210529145750/result_20210529145849.csv + 2021-05-31 14:48:56 (2344298) -[INFO]Find 2 ops less then 0.98 in precision_data/temp/vector_compare/20210529145750/result_20210529150404.csv + 2021-05-31 14:48:56 (2344298) -[INFO]Find 0 ops less then 0.98 in precision_data/temp/vector_compare/20210529145750/result_20210529151102.csv + ╭── [578] pixel_cls_loss/cond_1/TopKV2 ───╮ + │ Left: ['pixel_cls_loss/cond_1/TopKV2'] │ + │ Right: ['pixel_cls_loss/cond_1/TopKV2'] │ + │ Input: │ + │ - [0]1.0 - [1]nan │ + │ Output: │ + │ - [0]0.999999 - [1]0.978459 │ + ╰─────────────────────────────────────────╯ + ╭── [490] gradients/AddN_5 ───╮ + │ Left: ['gradients/AddN_5'] │ + │ Right: ['gradients/AddN_5'] │ + │ Input: │ + │ - [0]nan - [1]1.0 │ + │ Output: │ + │ - [0]0.05469 │ + ╰─────────────────────────────╯ + ``` +### Precision_data目录结构 +``` +precision_data/ +├── npu +│ ├── debug_0 +| | ├── dump +| | └── 20210510101133 +| │ └── graph +| | └── ge_proto_00000179_PreRunAfterBuild.txt +│ └── debug_1 +├── tf +| ├── tf_debug +| └── dump +├── overflow +├── fusion +└── temp + ├── op_graph + ├── decode + | ├── dump_decode + | ├── overflow_decode + | └── dump_convert + └── vector_compare + ├── 20210510101133 + | ├── result_123456.csv + | └── result_123455.csv + └── 20210510101134 + └── result_123458.csv +``` +### 配合msquickcmp一键式推理精度比对工具使用 +- msquickcmp会将中间dump数据和图自动保存在一个时间戳命名的目录内, 可以使用precision_tool工具直接对该目录进行分析 +```python +output-path/timestamp +├── dump_data +├── input +├── model +├── result_2021211214657.csv +└── tmp +``` +- 修改配置 +```python +# file precision_tool/config.py +# [train/infer] if adapt from msquickcmp result, set net type to infer +NET_TYPE = 'infer' +``` +- 执行以下命令 +```shell +# 前提条件： +# 当前目录没有precision_data目录（导入过程会新创建一个precision_data,用于保存导入数据） +# 只有第一次需要使用infer子命令导入，后续直接python3 precision_tool/cli.py +python3 precision_tool/cli.py infer output-path/timestamp +``` + + + + +### TF脚本修改参考 + +```python +# 打印动态Scale的Loss值 +loss_scale_manager = ExponentialUpdateLossScaleManager() +scale_v = sess.run([loss_scale_manager.get_loss_scale()]) +print(">>> Current Loss Scale >>> ", scale_v) + + +with tf.Session() as sess: + # do session.run() + saver = tf.train.Saver() + # 保存ckpt + saver.save(sess, saver_dir) + # ... + # 从ckpt恢复 + saver.restore(sess, saver_dir) + # ... + # 保存Tensorboard + summary_writer = tf.summary.FileWriter(logdir=log_dir, graph=sess.graph) + +``` + +### F&Q +1. 安装gnureadline报错找不到lncurses + ```shell + /usr/bin/ld: cannot find -lncurses + collect2: error: ld returned 1 exit status + error: command 'gcc' failed with exit status 1 + ``` + ```shell + # 先尝试在本地查找libncurses.so* + find / -name libncurses.so* + # 如果能找到以下文件，直接创建一个libncurses.so指向libncurses.so.5即可，否则需要用包管理工具安装ncurses + /usr/lib64/libncurses.so.5 + /usr/lib64/libncurses.so.5.9 + /usr/lib64/libncursesw.so.5 + # 创建软连接 + ln -s /usr/lib64/libncurses.so.5.9 /usr/lib64/libncurses.so + ``` +#### 参与贡献 + +1. Fork 本仓库 +2. 新建 Feat_xxx 分支 +3. 提交代码 +4. 新建 Pull Request \ No newline at end of file diff --git a/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/precision_tool/__pycache__/config.cpython-37.pyc b/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/precision_tool/__pycache__/config.cpython-37.pyc new file mode 100644 index 0000000000000000000000000000000000000000..fb0778ee827c60a67d5f426c23f42ff6aa72081c Binary files /dev/null and b/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/precision_tool/__pycache__/config.cpython-37.pyc differ diff --git a/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/precision_tool/__pycache__/tf_config.cpython-37.pyc b/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/precision_tool/__pycache__/tf_config.cpython-37.pyc new file mode 100644 index 0000000000000000000000000000000000000000..68c8c82fb527c7618c2fd8b27e0f37af3ec4cf25 Binary files /dev/null and b/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/precision_tool/__pycache__/tf_config.cpython-37.pyc differ diff --git a/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/precision_tool/caffe_dump.py b/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/precision_tool/caffe_dump.py new file mode 100644 index 0000000000000000000000000000000000000000..89f8adcb950198c579136b6379eea0ef8242cd6b --- /dev/null +++ b/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/precision_tool/caffe_dump.py @@ -0,0 +1,160 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# coding=utf-8 +""" +Source code: + https://bbs.huaweicloud.com/blogs/181056 +Example: + python3.7 caffe_dump.py -m resnet50.prototxt -w resnet50.caffemodel -i test.bin -n 'data:0' -o ./output_dir + +Guide for setting up Caffe/Tensorflow precision golden data generation environments: + https://bbs.huaweicloud.com/blogs/181059 +""" +import caffe +import sys +import argparse +import os +import caffe.proto.caffe_pb2 as caffe_pb2 +import google.protobuf.text_format +import json +import numpy as np +import time + +TIME_LENGTH = 1000 +FILE_PERMISSION_FLAG = 0o600 + + +class CaffeProcess: + def __init__(self): + parse = argparse.ArgumentParser() + parse.add_argument("-w", dest="weight_file_path", + help=" the caffe weight file path", + required=True) + parse.add_argument("-m", dest="model_file_path", + help=" the caffe model file path", + required=True) + parse.add_argument("-o", dest="output_path", help=" the output path", + required=True) + parse.add_argument("-i", "--input_bins", dest="input_bins", help="input_bins bins. e.g. './a.bin;./c.bin'", + required=True) + parse.add_argument("-n", "--input_names", dest="input_names", + help="input nodes name. e.g. 'graph_input_0:0;graph_input_0:1'", + required=True) + args, _ = parse.parse_known_args(sys.argv[1:]) + self.weight_file_path = os.path.realpath(args.weight_file_path) + self.model_file_path = os.path.realpath(args.model_file_path) + self.input_bins = args.input_bins.split(";") + self.input_names = args.input_names.split(";") + self.output_path = os.path.realpath(args.output_path) + self.net_param = None + self.cur_layer_idx = -1 + + @staticmethod + def _check_file_valid(path, is_file): + if not os.path.exists(path): + print('Error: The path "' + path + '" does not exist.') + exit(-1) + if is_file: + if not os.path.isfile(path): + print('Error: The path "' + path + '" is not a file.') + exit(-1) + else: + if not os.path.isdir(path): + print('Error: The path "' + path + '" is not a directory.') + exit(-1) + + def _check_arguments_valid(self): + self._check_file_valid(self.model_file_path, True) + self._check_file_valid(self.weight_file_path, True) + self._check_file_valid(self.output_path, False) + for input_file in self.input_bins: + self._check_file_valid(input_file, True) + + @staticmethod + def calDataSize(shape): + dataSize = 1 + for dim in shape: + dataSize *= dim + return dataSize + + def _load_inputs(self, net): + inputs_map = {} + for layer_name, blob in net.blobs.items(): + if layer_name in self.input_names: + input_bin = np.fromfile( + self.input_bins[self.input_names.index(layer_name)], np.float32) + input_bin_shape = blob.data.shape + if self.calDataSize(input_bin_shape) == self.calDataSize(input_bin.shape): + input_bin = input_bin.reshape(input_bin_shape) + else: + print("Error: input node data size %d not match with input bin data size %d.", self.calDataSize( + input_bin_shape), self.calDataSize(input_bin.shape)) + exit(-1) + inputs_map[layer_name] = input_bin + return inputs_map + + def process(self): + """ + Function Description: + process the caffe net, save result as dump data + """ + # check path valid + self._check_arguments_valid() + + # load model and weight file + net = caffe.Net(self.model_file_path, self.weight_file_path, + caffe.TEST) + inputs_map = self._load_inputs(net) + for key, value in inputs_map.items(): + net.blobs[key].data[...] = value + # process + net.forward() + + # read prototxt file + net_param = caffe_pb2.NetParameter() + with open(self.model_file_path, 'rb') as model_file: + google.protobuf.text_format.Parse(model_file.read(), net_param) + for layer in net_param.layer: + name = layer.name.replace("/", "_").replace(".", "_") + index = 0 + for top in layer.top: + data = net.blobs[top].data[...] + file_name = name + "." + str(index) + "." + str( + round(time.time() * 1000000)) + ".npy" + output_dump_path = os.path.join(self.output_path, file_name) + np.save(output_dump_path, data) + os.chmod(output_dump_path, FILE_PERMISSION_FLAG) + print('The dump data of "' + layer.name + + '" has been saved to "' + output_dump_path + '".') + index += 1 + + +if __name__ == "__main__": + caffe_process = CaffeProcess() + caffe_process.process() \ No newline at end of file diff --git a/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/precision_tool/cli.py b/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/precision_tool/cli.py new file mode 100644 index 0000000000000000000000000000000000000000..9c68ba3b48c6bd02b22ce1b24656e43db425421d --- /dev/null +++ b/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/precision_tool/cli.py @@ -0,0 +1,143 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# coding=utf-8 +""" +cli +""" +import os +import sys +import argparse +import time + +from termcolor import colored +from lib.precision_tool import PrecisionTool +from lib.interactive_cli import InteractiveCli +from lib.precision_tool_exception import PrecisionToolException +from lib.util import util +from lib.tf_dump import TfDump +from lib.msquickcmp_adapter import MsQuickCmpAdapter +import config as cfg + + +INTRODUCE_DOC = \ + "===============================\n" \ + "Usage:\n" \ + " Single mode:\n" \ + " Exp:\n" \ + " Dump TF data:\n" \ + " > python3.7.5 precision_tool/cli.py tf_dump \"sh cpu_train.sh param1 param2\" \n" \ + " Dump NPU data:\n" \ + " > python3.7.5 precision_tool/cli.py npu_dump \"sh npu_train.sh param1 param2\" \n" \ + " Check NPU overflow:\n" \ + " > python3.7.5 precision_tool/cli.py npu_overflow \"sh npu_train.sh param1 param2\" \n" \ + " Interactive mode:\n" \ + " Exp:\n" \ + " Start command line:\n" \ + " > python3.7.5 precision_tool/cli.py\n" + + +def _run_tf_dbg_dump(cmdline): + tf_dump = TfDump() + tf_dump.run_tf_dbg_dump(cmdline) + + +def _unset_flags(): + if cfg.PRECISION_TOOL_OVERFLOW_FLAG in os.environ: + del os.environ[cfg.PRECISION_TOOL_OVERFLOW_FLAG] + if cfg.PRECISION_TOOL_DUMP_FLAG in os.environ: + del os.environ[cfg.PRECISION_TOOL_DUMP_FLAG] + + +def _run_npu_dump(cmd): + _unset_flags() + log = util.get_log() + os.environ[cfg.PRECISION_TOOL_DUMP_FLAG] = 'True' + log.info("Start run NPU script with dump data.") + ret = util.execute_command(cmd) + log.info("Finish run NPU script with dump data. ret [%s]", ret) + _unset_flags() + + +def _run_npu_overflow(cmd): + _unset_flags() + log = util.get_log() + os.environ[cfg.PRECISION_TOOL_OVERFLOW_FLAG] = 'True' + log.info("Start run NPU script with overflow check process....") + ret = util.execute_command(cmd) + log.info("Finish run NPU script with overflow check process. ret [%s]", ret) + precision_tool = PrecisionTool() + precision_tool.prepare() + precision_tool.do_check_overflow() + _unset_flags() + + +def _run_infer_adapter(output_path): + """ Run precision_tool with msquickcmp output data + :param output_path: msquickcmp output path + :return: None + """ + adapter = MsQuickCmpAdapter(output_path) + adapter.run() + _run_interactive_cli() + + +def _run_interactive_cli(cli=None): + util.get_log().info("Interactive command mode.") + if cli is None: + cli = InteractiveCli() + try: + cli.cmdloop(intro="Enjoy!") + except KeyboardInterrupt: + util.get_log().info("Bye.......") + + +function_list = { + 'tf_dump': _run_tf_dbg_dump, + 'npu_dump': _run_npu_dump, + 'npu_overflow': _run_npu_overflow, + 'infer': _run_infer_adapter +} + + +def main(): + while len(sys.argv) > 1: + util.get_log().info("Single command mode.") + function_key = sys.argv[1] + cmd_line = sys.argv[2] if len(sys.argv) > 2 else None + if function_key in function_list: + return function_list[function_key](cmd_line) + precision_tool = PrecisionTool() + return precision_tool.single_cmd(sys.argv) + _run_interactive_cli() + + +if __name__ == '__main__': + try: + main() + except PrecisionToolException as pte: + util.get_log().error(pte.error_info) diff --git a/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/precision_tool/config.py b/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/precision_tool/config.py new file mode 100644 index 0000000000000000000000000000000000000000..12bb53a04b2d3412897f6f921d0644db6e225ff5 --- /dev/null +++ b/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/precision_tool/config.py @@ -0,0 +1,131 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# coding=utf-8 +import os + +# Dump config '0|5|10' +TF_DUMP_STEP = '0' + +# path to run package operator cmp compare +# default may be /usr/local/Ascend/ +CMD_ROOT_PATH = '/usr/local/Ascend/' + +# ASCEND Log Path +ASCEND_LOG_PATH = '/root/ascend/log/plog/' + +# TOOL CONFIG +LOG_LEVEL = "NOTSET" +ROOT_DIR = '' + +# [train/infer] if adapt from msquickcmp result, set net type to infer +NET_TYPE = 'train' + +''' +precision_data/ +├── npu +│ ├── debug_0 +| | ├── dump +| | | └── 20210510101133 +| │ └── graph +| | └── ge_proto_00000179_PreRunAfterBuild.txt +│ └── debug_1 +├── tf +| ├── tf_debug +| └── dump +├── overflow +├── fusion +└── temp + ├── op_graph + ├── decode + | ├── dump_decode + | ├── overflow_decode + | └── dump_convert + └── vector_compare + ├── 20210510101133 + | ├── result_123456.csv + | └── result_123455.csv + └── 20210510101134 + └── result_123458.csv +''' + +# Static dirs, do not change +DATA_ROOT_DIR = os.path.join(ROOT_DIR, 'precision_data') + +# fusion +FUSION_DIR = os.path.join(DATA_ROOT_DIR, 'fusion') + +# npu dump/graph parent dir +NPU_DIR = os.path.join(DATA_ROOT_DIR, 'npu') +DEFAULT_NPU_DIR = os.path.join(NPU_DIR, 'debug_0') +DEFAULT_NPU_DUMP_DIR = os.path.join(DEFAULT_NPU_DIR, 'dump') +DEFAULT_NPU_GRAPH_DIR = os.path.join(DEFAULT_NPU_DIR, 'graph') +DEFAULT_OP_DEBUG_DIR = DEFAULT_NPU_DIR + +# npu overflow dir +OVERFLOW_DIR = os.path.join(DATA_ROOT_DIR, 'overflow') +NPU_OVERFLOW_DUMP_DIR = os.path.join(OVERFLOW_DIR, 'dump') + +# tf dirs +TF_DIR = os.path.join(DATA_ROOT_DIR, 'tf') +TF_DEBUG_DUMP_DIR = os.path.join(TF_DIR, 'tf_debug') +TF_DUMP_DIR = os.path.join(TF_DIR, 'dump') +TF_GRAPH_DIR = os.path.join(TF_DIR, 'graph') + +# pytroch dirs +PT_DIR = os.path.join(DATA_ROOT_DIR, 'pt') +PT_DUMP_DIR = os.path.join(PT_DIR, 'dump') + +# tmp dirs +TMP_DIR = os.path.join(DATA_ROOT_DIR, 'temp') +OP_GRAPH_DIR = os.path.join(TMP_DIR, 'op_graph') + +DECODE_DIR = os.path.join(TMP_DIR, 'decode') +OVERFLOW_DECODE_DIR = os.path.join(DECODE_DIR, 'overflow_decode') +DUMP_DECODE_DIR = os.path.join(DECODE_DIR, 'dump_decode') +DUMP_CONVERT_DIR = os.path.join(DECODE_DIR, 'dump_convert') + +VECTOR_COMPARE_PATH = os.path.join(TMP_DIR, 'vector_compare') +TF_TENSOR_NAMES = os.path.join(TMP_DIR, 'tf_tensor_names.txt') +TF_TENSOR_DUMP_CMD = os.path.join(TMP_DIR, 'tf_tensor_cmd.txt') + +# FLAG +PRECISION_TOOL_OVERFLOW_FLAG = 'PRECISION_TOOL_OVERFLOW' +PRECISION_TOOL_DUMP_FLAG = 'PRECISION_TOOL_DUMP' + +# DUMP CONFIG +OP_DEBUG_LEVEL = 0 +DUMP_GE_GRAPH_VALUE = 2 +DUMP_GRAPH_LEVEL_VALUE = 3 +DUMP_SEED = 0 + +# TF_DEBUG +TF_DEBUG_TIMEOUT = 360 + +# MSACCUCMP +MS_ACCU_CMP = r'msaccucmp.py[c]?' +BUILD_JSON_GRAPH_NAME = 'Build' diff --git a/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/precision_tool/fusion_off.cfg b/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/precision_tool/fusion_off.cfg new file mode 100644 index 0000000000000000000000000000000000000000..16999744a97314fe09ac1ae61595de9624ce053c --- /dev/null +++ b/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/precision_tool/fusion_off.cfg @@ -0,0 +1,10 @@ +{ + "Switch": { + "GraphFusion": { + "ScopeLayerNormFusionPass": "off" + }, + "UBFusion": { + "ALL": "on" + } + } +} diff --git a/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/precision_tool/fusion_switch.cfg b/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/precision_tool/fusion_switch.cfg new file mode 100644 index 0000000000000000000000000000000000000000..572ad427170fddff27404666b8676f75c74a3164 --- /dev/null +++ b/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/precision_tool/fusion_switch.cfg @@ -0,0 +1,6 @@ +{ +"Switch": { + "GraphFusion": {}, + "UBFusion": {} +} +} \ No newline at end of file diff --git a/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/precision_tool/lib/compare.py b/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/precision_tool/lib/compare.py new file mode 100644 index 0000000000000000000000000000000000000000..972e7667c736e1c0a7c1cde593fd27844bfec01e --- /dev/null +++ b/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/precision_tool/lib/compare.py @@ -0,0 +1,209 @@ +# coding=utf-8 +""" +Compare +""" +import json +import os +import numpy as np +from lib.tool_object import ToolObject +from lib.util import util +from lib.compare_result import CompareResult +from lib.constant import Constant +import config as cfg +from lib.precision_tool_exception import PrecisionToolException +from lib.precision_tool_exception import catch_tool_exception + + +class Compare(object): + def __init__(self): + """Init""" + super(Compare, self).__init__() + self.log = util.get_log() + self.vector_compare_results = {} + + @catch_tool_exception + def prepare(self): + util.create_dir(cfg.VECTOR_COMPARE_PATH) + + def npu_tf_vector_compare(self, graphs, npu_root_dir, tf_root_dir, result_dir): + """Compare npu dump data with tf dump data + :param graphs: npu ge graph json file list + :param npu_root_dir: + :param tf_root_dir: + :param result_dir: result dir + :return: + """ + for graph_file in graphs: + self.log.info("Compare npu tf with graph %s", graph_file) + sub_graphs = self._get_sub_graphs(graph_file) + if sub_graphs is None: + continue + for sub_graph in sub_graphs: + npu_dir = self._get_sub_dir_by_sub_graph_name(sub_graph, npu_root_dir) + if npu_dir is None: + continue + self.vector_compare(npu_dir, tf_root_dir, result_dir, graph_file) + + @catch_tool_exception + def _get_sub_dir_by_sub_graph_name(self, sub_graph, npu_root_dir): + sub_graph_dirs = [] + for dir_path, dir_names, file_names in os.walk(npu_root_dir, followlinks=True): + if sub_graph in dir_names: + # walk sub graph dir + for sub_dir_path, sub_dir_names, sub_file_names in os.walk(os.path.join(dir_path, sub_graph), + followlinks=True): + if len(sub_dir_names) == 0: + sub_graph_dirs.append(sub_dir_path) + if len(sub_graph_dirs) == 0: + return None + self.log.warning("Find [%d] dirs in sub graph dir [%s], %s, compare first.", len(sub_graph_dirs), sub_graph, + sub_graph_dirs) + return sub_graph_dirs[0] + + @catch_tool_exception + def _get_sub_graphs(self, graph_file): + with open(graph_file, 'r') as f: + graph_json = json.load(f) + if 'graph' not in graph_json: + raise PrecisionToolException("No graph in file: %s" % graph_file) + sub_graphs = [] + for graph in graph_json['graph']: + sub_graphs.append(graph['name']) + return sub_graphs + + ''' + @staticmethod + def _get_ge_default_dirs(self, root_dir): + for dir_path, dir_names, file_names in os.walk(root_dir, followlinks=True): + for dir_name in dir_names: + ''' + + def npu_vector_compare(self, debug_0_root_dir, debug_1_root_dir): + """Compare two npu dump data + :param debug_0_root_dir: + :param debug_1_root_dir: + :return: + """ + # debug_0_sub_dirs = self._get_ge_default_dirs(debug_0_root_dir) + # debug_1_sub_dirs = self._get_ge_default_dirs(debug_1_root_dir) + + def vector_compare(self, lh_path, rh_path, result_dir, graph_json=None): + """Compare all ops""" + if lh_path is None or util.empty_dir(lh_path): + raise PrecisionToolException("No valid dump file in %s" % lh_path) + if rh_path is None or util.empty_dir(rh_path): + raise PrecisionToolException("No valid dump file in %s" % rh_path) + self.log.info("Start vector compare process...") + util.compare_vector(lh_path, rh_path, graph_json, result_dir) + self.log.info("Vector compare process finish.") + + def _get_compare_result_by_file_name(self, file_name): + results = [] + if file_name is None: + sub_dir = util.get_newest_dir(cfg.VECTOR_COMPARE_PATH) + if sub_dir == '': + raise PrecisionToolException("Empty vector compare path:%s" % cfg.VECTOR_COMPARE_PATH) + file_name = os.path.join(cfg.VECTOR_COMPARE_PATH, sub_dir) + if os.path.isfile(file_name): + results.append(CompareResult(file_name)) + if os.path.isdir(file_name): + vector_compare_result_files = util.list_vector_compare_result_files(file_name) + if vector_compare_result_files is None or len(vector_compare_result_files) == 0: + raise PrecisionToolException("Can not find any vector compare result in dir:%s" % file_name) + file_list = sorted(vector_compare_result_files.values(), key=lambda x: x.timestamp) + file_names = [x.file_name for x in file_list] + self.log.debug("Find %s result files in dir %s", file_names, file_name) + for file in file_list: + results.append(CompareResult(file.path)) + return results + + @catch_tool_exception + def vector_summary(self, file_name=None, cos_sim_threshold=0.98, limit=1): + """Print not NaN result in vector compare result""" + compare_results = self._get_compare_result_by_file_name(file_name) + error_ops = [] + for compare_result in compare_results: + err_ops = compare_result.get_op_by_cosine_sim_threshold(cos_sim_threshold, limit) + self.log.info("Find %d ops less then %s in %s", len(err_ops), cos_sim_threshold, compare_result.file_path) + error_ops.extend(err_ops) + if len(error_ops) == 0: + self.log.info("Can not find any compare result over threshold: %s" % cos_sim_threshold) + else: + for i, error_op in enumerate(error_ops): + if i < limit: + error_op.summary(cos_sim_threshold) + return error_ops + + def compare_data(self, left, right, save_txt=False, rl=0.001, al=0.001, diff_count=20): + """Compare data""" + left = self._detect_file(left) + right = self._detect_file(right) + if left is None or right is None: + raise PrecisionToolException("invalid input or output") + # save to txt + if save_txt: + util.save_npy_to_txt(left) + util.save_npy_to_txt(right) + # compare data + total_cnt, all_close, cos_sim, err_percent = self._do_compare_data(left, right, rl, al, diff_count) + content = ['Left:', ' ├─ NpyFile: %s' % left] + if save_txt: + content.append(' ├─ TxtFile: [green]%s.txt[/green]' % left) + content.append(' └─ NpySpec: [yellow]%s[/yellow]' % util.gen_npy_info_txt(left)) + content.append('Right:') + content.append(' ├─ NpyFile: %s' % right) + if save_txt: + content.append(' ├─ TxtFile: [green]%s.txt[/green]' % right) + content.append(' └─ NpySpec: [yellow]%s[/yellow]' % util.gen_npy_info_txt(right)) + content.append('NumCnt: %s' % total_cnt) + content.append('AllClose: %s' % all_close) + content.append('CosSim: %s' % cos_sim) + content.append('ErrorPer: %s (rl= %s, al= %s)' % (err_percent, rl, al)) + util.print_panel(Constant.NEW_LINE.join(content)) + + def _do_compare_data(self, left, right, rl=0.001, al=0.001, diff_count=20): + data_left = np.load(left).astype(np.float32) + data_right = np.load(right).astype(np.float32) + shape_left = data_left.shape + shape_right = data_right.shape + if shape_left != shape_right: + self.log.warning("Data shape not equal: %s vs %s", data_left.shape, data_right.shape) + data_left = data_left.reshape(-1) + data_right = data_right.reshape(-1) + if data_left.shape[0] != data_right.shape[0]: + self.log.warning("Data size not equal: %s vs %s", data_left.shape, data_right.shape) + if data_left.shape[0] < data_right.shape[0]: + data_left = np.pad(data_left, (0, data_right.shape[0] - data_left.shape[0]), 'constant') + else: + data_right = np.pad(data_right,(0, data_left.shape[0] - data_right.shape[0]), 'constant') + all_close = np.allclose(data_left, data_right, atol=al, rtol=rl) + # cos_sim = 1 - spatial.distance.cosine(data_left, data_right) + cos_sim = np.dot(data_left, data_right) / ( + np.sqrt(np.dot(data_left, data_left)) * np.sqrt(np.dot(data_right, data_right))) + err_cnt = 0 + total_cnt = data_left.shape[0] + diff_table_columns = ['Index', 'Left', 'Right', 'Diff'] + err_table = util.create_table("Error Item Table", diff_table_columns) + top_table = util.create_table("Top Item Table", diff_table_columns) + for i in range(total_cnt): + abs_diff = abs(data_left[i] - data_right[i]) + if i < diff_count: + top_table.add_row(str(i), str(data_left[i]), str(data_right[i]), str(abs_diff)) + if abs_diff > (al + rl * abs(data_right[i])): + if err_cnt < diff_count: + err_table.add_row(str(i), str(data_left[i]), str(data_right[i]), str(abs_diff)) + err_cnt += 1 + err_percent = float(err_cnt / total_cnt) + util.print(util.create_columns([err_table, top_table])) + return total_cnt, all_close, cos_sim, err_percent + + def _detect_file(self, file_name): + """Find files in npu/overflow/cpu dump dir""" + if os.path.isfile(file_name): + return file_name + for parent_dir in [cfg.TMP_DIR, cfg.TF_DUMP_DIR]: + file_infos = util.list_numpy_files(parent_dir, file_name) + if len(file_infos) > 0: + self.log.info("Find %s, choose first one.", list(file_infos.keys())) + return list(file_infos.values())[0].path + return None diff --git a/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/precision_tool/lib/compare_result.py b/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/precision_tool/lib/compare_result.py new file mode 100644 index 0000000000000000000000000000000000000000..06341ad8cca081a4925644c4ff02d650ec045c58 --- /dev/null +++ b/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/precision_tool/lib/compare_result.py @@ -0,0 +1,143 @@ +import collections +import os +import numpy as np +from lib.util import util +from lib.constant import Constant +from lib.precision_tool_exception import PrecisionToolException +from lib.precision_tool_exception import catch_tool_exception + + +class RowMap(object): + """ + 'Index': 0, + 'LeftOp': 1, + 'RightOp': 2, + 'TensorIdx': 3, # TensorIndex + 'CosSim': 4, # CosineSimilarity + 'MaxAbs': 5, # MaxAbsoluteError + 'ARE': 6, # AccumulatedRelativeError + 'RED': 7, # RelativeEuclideanDistance + 'KLD': 8, # KullbackLeiblerDivergence + 'StandardDeviation': 9 # StandardDeviation + """ + def __init__(self, item=None): + self.index = 0 + self.left = 1 + self.right = 2 + self.tensor_index = 3 + self.cosine_similarity = 4 + self.max_abs = 5 + if item is not None: + self.update(item) + + def update(self, item): + for i, value in enumerate(item): + self.left = i if value == 'LeftOp' else self.left + self.right = i if value == 'RightOp' else self.right + self.tensor_index = i if value == 'TensorIndex' else self.tensor_index + self.cosine_similarity = i if value == 'CosineSimilarity' else self.cosine_similarity + self.max_abs = i if value == 'MaxAbsoluteError' else self.max_abs + + +class CompareItem(object): + def __init__(self, op_name, item, row_map): + self.row_map = row_map + self.index = int(item[self.row_map.index]) + self.op_name = op_name + self.left = item[self.row_map.left].split(" ") + self.right = item[self.row_map.right].split(" ") + self.input = [] + self.output = [] + + def update(self, item): + tensor_index = item[self.row_map.tensor_index] + if tensor_index not in ['NaN', '*']: + item_detail = tensor_index.split(':') + if len(item_detail) != 3: + raise PrecisionToolException("item:%d tensor index invalid. [%s]" % ( + item[self.row_map.index], tensor_index)) + if item_detail[1] == 'input': + self.input.insert(int(item_detail[2]), item) + else: + self.output.insert(int(item_detail[2]), item) + + def is_cosine_sim_over_threshold(self, threshold): + for item in self.output: + if item[self.row_map.cosine_similarity] == 'NaN': + continue + if float(item[self.row_map.cosine_similarity]) <= threshold: + return True + return False + + @staticmethod + def _color_data(data, threshold): + try: + data = float(data) + if np.isnan(data): + raise ValueError + elif data <= threshold: + return "[red]%s[/red]" % data + else: + return "[green]%s[/green]" % data + except ValueError: + return "[yellow]%s[/yellow]" % data + + def summary(self, threshold): + content = ["Left: %s" % self.left, "Right: %s" % self.right, "Input: "] + input_txt = [] + for i, item in enumerate(self.input): + input_txt.append(" - [%d]%s" % (i, self._color_data(item[self.row_map.cosine_similarity], threshold))) + content.extend([Constant.TAB_LINE.join(input_txt), "Output:"]) + output_txt = [] + for i, item in enumerate(self.output): + output_txt.append(" - [%d]%s" % (i, self._color_data(item[self.row_map.cosine_similarity], threshold))) + content.append(Constant.TAB_LINE.join(output_txt)) + title = "[%d] %s" % (self.index, self.op_name) + util.print_panel(Constant.NEW_LINE.join(content), title=title) + + +class CompareResult(object): + def __init__(self, file_path): + self.file_path = file_path + self.ops = None + self.prepare() + + @catch_tool_exception + def prepare(self): + if not str(self.file_path).endswith(Constant.Suffix.CSV): + raise PrecisionToolException("Compare result file %s not a csv file." % self.file_path) + if not os.path.isfile(self.file_path): + raise PrecisionToolException("Compare result file %s not exist." % self.file_path) + items = util.read_csv(self.file_path) + self.ops = collections.OrderedDict() + row_map = RowMap() + for item in items: + if item[row_map.index] == 'Index': + row_map.update(item) + continue + if item[row_map.tensor_index] in ['NaN', '*']: + continue + tensor_index = item[row_map.tensor_index] + op_name = tensor_index.split(":")[0] + if op_name not in self.ops: + self.ops[op_name] = CompareItem(op_name, item, row_map) + op = self.ops[op_name] + op.update(item) + + def get_compare_item_by_op(self, op_name): + if self.ops is None: + self.prepare() + if self.ops is None: + raise PrecisionToolException("Invalid compare result file: %s" % self.file_path) + if op_name in self.ops: + return self.ops[op_name] + return None + + def get_op_by_cosine_sim_threshold(self, threshold, limit=-1): + result = [] + for compare_item in self.ops.values(): + if compare_item.is_cosine_sim_over_threshold(threshold): + result.append(compare_item) + if len(result) == limit: + break + return result diff --git a/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/precision_tool/lib/constant.py b/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/precision_tool/lib/constant.py new file mode 100644 index 0000000000000000000000000000000000000000..01c69b18ede388f8dcde0fb9f05843f40e304462 --- /dev/null +++ b/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/precision_tool/lib/constant.py @@ -0,0 +1,19 @@ +# coding=utf-8 + + +class Constant(object): + NEW_LINE = "\n" + TAB_LINE = "\t" + DEFAULT_DEBUG_ID = "debug_0" + NPU_DEBUG_ID_1 = "debug_1" + GRAPH = "graph" + DUMP = "dump" + + class Suffix(object): + JSON = '.json' + CSV = '.csv' + H5 = '.h5' + + class Pattern(object): + GE_PROTO_GRAPH_PATTERN = r'^ge_proto_([0-9]+)_([A-Za-z0-9_-]+)\.txt$' + diff --git a/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/precision_tool/lib/cpu_tvm.py b/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/precision_tool/lib/cpu_tvm.py new file mode 100644 index 0000000000000000000000000000000000000000..a0906071e33bfcbc2c8a51e3a3fe18381bcfc79d --- /dev/null +++ b/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/precision_tool/lib/cpu_tvm.py @@ -0,0 +1,51 @@ +import numpy as np +from tbe import tvm + + +class CpuTvm(): + def __init__(self, json_file, dump_input_files, dump_output_files): + self.json_file = json_file + self.dump_input_files = dump_input_files + self.dump_output_files = dump_output_files + self.input_list = [] + self.output_list = [] + + def _load_schedule(self): + with open(self.json_file, 'r') as jsonfile: + tvm_node = tvm.load_json(jsonfile.read()) + self.output_list = tvm_node.op.attrs['output_list'] + self.input_list = tvm_node.op.attrs['input_list'] + schedule = tvm.create_schedule([res.op for res in self.output_list]) + return schedule + + def _build_tvm(self, schedule): + tensor_list = [ele for ele in self.input_list if ele is not None] + for ele in self.output_list: + if ele is not None: + tensor_list.append(ele) + fusion_op = tvm.build(schedule, tensor_list, "c", "llvm") + return fusion_op + + def _load_data(self, dump_files): + ctx = tvm.cpu(0) + data_tvm = [] + for dump_file in dump_files: + data_temp_numpy = np.load(dump_file) + data_temp_tvm = tvm.nd.array(data_temp_numpy, ctx) + data_tvm.append(data_temp_tvm) + return data_tvm + + def run_cpu_tvm(self): + # load schedule and build tvm + schedule = self._load_schedule() + fusion_op = self._build_tvm(schedule) + + #load data and run cpu tvm + data_tvm_in = self._load_data(self.dump_input_files) + data_tvm_out = self._load_data(self.dump_output_files) + data_tvm_in.extend(data_tvm_out) + fusion_op(*data_tvm_in) + + #tvm format to numpy format + data_np_out = [data.asnumpy() for data in data_tvm_out] + return data_np_out diff --git a/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/precision_tool/lib/desc.py b/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/precision_tool/lib/desc.py new file mode 100644 index 0000000000000000000000000000000000000000..bac7ea75e9cf939ff3f7be3dfffcc23454714e72 --- /dev/null +++ b/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/precision_tool/lib/desc.py @@ -0,0 +1,150 @@ +# coding=utf-8 +from lib.util import util + +ATTR = 'attr' +ATTR_KEY = 'key' +ATTR_VALUE = 'value' +DATA_DUMP_ORIGIN_OUTPUT_INDEX = '_datadump_origin_output_index' +FUSION_ORIGIN_OUTPUT_INDEX = '_fusion_origin_output_index' +DATA_DUMP_ORIGIN_NAME = '_datadump_origin_name' +ORIGIN_FORMAT = 'origin_format' +ORIGIN_SHAPE = 'origin_shape' +VALUE_RANGE = 'value_range' +SHAPE_RANGE = 'shape_range' +DT_STRING = 's' +DT_INT = 'i' +DT_LIST_LIST_INT = 'list_list_int' +DT_LIST_LIST_I = 'list_list_i' +DT_LIST = 'list' +DT_LIST_INT = 'list_i' +DATA_TYPE_DEFAULT_VALUE = { + 'i': 0, + 's': '' +} + + +class Desc(object): + """ Op desc + shape: data shape + dtype: data type + format: data format + npu_file: npu file name/path + cpu_file: cpu file name/path + idx: input idx + """ + def __init__(self, desc_json, index): + self.desc_json = desc_json + self.index = index + self.log = util.get_log() + + def idx(self): + return self.index + + def shape(self): + return self.desc_json['shape']['dim'] if 'shape' in self.desc_json else [] + + def dtype(self): + return self.desc_json['dtype'] if 'dtype' in self.desc_json else '' + + def format(self): + return self.desc_json['layout'] if 'layout' in self.desc_json else [] + + def origin_shape(self): + return self._get_attr_list(ORIGIN_SHAPE, DT_INT) + + def origin_format(self): + return self._get_attr(ORIGIN_FORMAT, DT_STRING) + + def value_range(self): + return self._get_attr_list_list(VALUE_RANGE, DT_LIST_INT) + + def shape_range(self): + return self._get_attr_list_list(SHAPE_RANGE, DT_LIST_INT) + + def _get_attr_list_list(self, key, data_type): + val = self._get_attr_base(key, DT_LIST_LIST_INT) + if val is None or DT_LIST_LIST_I not in val: + return [] + res = [] + for item in val[DT_LIST_LIST_I]: + if data_type in item: + res.append(item[data_type]) + return res + + def _get_attr_list(self, key, data_type): + val = self._get_attr_base(key, DT_LIST) + return val[data_type] if val is not None else [] + + def _get_attr(self, key, data_type): + val = self._get_attr_base(key, data_type) + return val if val is not None else DATA_TYPE_DEFAULT_VALUE[data_type] + + def _get_attr_base(self, key, data_type): + if ATTR in self.desc_json: + for attr in self.desc_json[ATTR]: + if attr[ATTR_KEY] == key: + if data_type in attr[ATTR_KEY]: + return attr[ATTR_VALUE][data_type] + return None + + def compare(self, right_desc): + if self.dtype() == right_desc.dtype() and self.format() == right_desc.format(): + return "[green][%d] [%s][%s] %s[/green]" % (self.idx(), self.dtype(), self.format(), self.shape()), True + else: + return "[yellow][%d] [%s][%s] %s | [%s][%s] %s[/yellow]" % ( + self.idx(), self.dtype(), self.format(), self.shape(), + right_desc.dtype(), right_desc.format(), right_desc.shape()), False + + +class InputDesc(Desc): + def __init__(self, name, desc_json, index): + super(InputDesc, self).__init__(desc_json, index) + if name == '': + self.log.warning('invalid input name.') + name_info = name.split(':') + self.op_name = name + self.peer_index = -2 + if len(name_info) == 2: + self.op_name = name_info[0] + self.peer_index = int(name_info[1]) + + def name(self): + return self.op_name + + def peer_idx(self): + return self.peer_index + + def is_control(self): + return self.peer_index == -1 + + def summary(self, origin_txt=False): + """idx | dtype | format | shape | [blue]value_range | shape_range[/blue] | op_name | peer_idx""" + if origin_txt: + return "[%d][%s][%s]%s %s:%d" % (self.idx(), self.dtype(), self.format(), + self.shape(), self.name(), self.peer_idx()) + return "[green][%d][/green][yellow][%s][%s]%s[/yellow][blue] %s %s [/blue] %s:%d" % ( + self.idx(), self.dtype(), self.format(), self.shape(), + self.value_range(), self.shape_range(), self.name(), self.peer_idx()) + + +class OutputDesc(Desc): + def __init__(self, name, desc_json, index): + super(OutputDesc, self).__init__(desc_json, index) + if name == '': + self.log.warning('invalid output name.') + self.op_names = name.split(':') + + def names(self): + return self.op_names + + def summary(self, origin_txt=False): + if origin_txt: + return "[%d][%s][%s]%s %s" % (self.idx(), self.dtype(), self.format(), self.shape(), self.names()) + return "[green][%d][/green][yellow][%s][%s]%s[/yellow] %s" % ( + self.idx(), self.dtype(), self.format(), self.shape(), self.names()) + + def data_dump_origin_name(self): + return self._get_attr(DATA_DUMP_ORIGIN_NAME, DT_STRING) + + def data_dump_origin_output_index(self): + return self._get_attr(DATA_DUMP_ORIGIN_OUTPUT_INDEX, DT_INT) diff --git a/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/precision_tool/lib/dump_manager.py b/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/precision_tool/lib/dump_manager.py new file mode 100644 index 0000000000000000000000000000000000000000..7eecc7c6e3d7483d1b187f0a9c981c6cee627e98 --- /dev/null +++ b/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/precision_tool/lib/dump_manager.py @@ -0,0 +1,78 @@ +# coding=utf-8 +import os +import re +import collections +from lib.util import util +from lib.constant import Constant +from lib.npu_dump import NpuDump +from lib.tf_dump import TfDump +import config as cfg +from lib.precision_tool_exception import catch_tool_exception +from lib.precision_tool_exception import PrecisionToolException + + +class DumpManager(object): + def __init__(self): + self.npu_dumps = collections.OrderedDict() + self.tf_dump = TfDump(cfg.TF_DUMP_DIR) + self._init_dirs() + + def prepare(self): + # prepare npu + sub_dirs = os.listdir(cfg.NPU_DIR) + if len(sub_dirs) == 0: + # create default + sub_dirs = [Constant.DEFAULT_DEBUG_ID] + sorted(sub_dirs) + for sub_dir in sub_dirs: + npu_dump = NpuDump(sub_dir) + npu_dump.prepare() + self.npu_dumps[sub_dir] = npu_dump + # prepare tf + self.tf_dump.prepare() + + def get_dump_root_dir(self, debug_id): + if debug_id in self.npu_dumps: + return self.npu_dumps[debug_id].dump_root + return None + + def op_dump_summary(self, ops): + npu_result = collections.OrderedDict() + for debug_id, op in ops.items(): + if debug_id in self.npu_dumps: + npu_result[debug_id] = collections.OrderedDict() + for op_detail in op: + npu_result[debug_id][op_detail.graph_name] = self.npu_dumps[debug_id].op_dump_summary(op_detail) + tf_result = None + if self.tf_dump is not None and len(ops[Constant.DEFAULT_DEBUG_ID]) != 0: + tf_result = self.tf_dump.op_dump_summary(ops[Constant.DEFAULT_DEBUG_ID][0]) + return npu_result, tf_result + + def print_tensor(self, file_name, is_convert): + """Print numpy data file""" + if os.path.isfile(file_name): + return util.print_npy_summary(os.path.dirname(file_name), os.path.basename(file_name), is_convert) + # file_name = file_name.replace('/', '_') + # npu decode file + npu_convert_files = self.npu_dumps[Constant.DEFAULT_DEBUG_ID].get_npu_dump_decode_files_by_name(file_name) + # util.list_npu_dump_convert_files(cfg.DECODE_DIR, file_name) + # tf decode file + tf_decode_files = self.tf_dump.get_dump_files_by_name(file_name, True) + # pt decode file + # pt_decode_files = self.pt_dump.get_dump_files_by_name(file_name) + self._print_tensors(npu_convert_files, is_convert) + self._print_tensors(tf_decode_files, is_convert) + + @staticmethod + def _print_tensors(file_infos, is_convert): + if file_infos is not None: + for file_info in file_infos.values(): + util.print_npy_summary(file_info.dir_path, file_info.file_name, is_convert) + + @staticmethod + def _init_dirs(): + """Create dump file dirs""" + util.create_dir(cfg.DUMP_DECODE_DIR) + util.create_dir(cfg.NPU_OVERFLOW_DUMP_DIR) + util.create_dir(cfg.OVERFLOW_DECODE_DIR) + util.create_dir(cfg.TF_DUMP_DIR) diff --git a/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/precision_tool/lib/file_desc.py b/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/precision_tool/lib/file_desc.py new file mode 100644 index 0000000000000000000000000000000000000000..563a4dc1c2fb400af4238e225c24a430e743ae1f --- /dev/null +++ b/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/precision_tool/lib/file_desc.py @@ -0,0 +1,36 @@ +# coding=utf-8 +import os + + +class FileDesc(object): + def __init__(self, file_name, dir_path, timestamp=-1): + self.file_name = file_name + self.dir_path = dir_path + self.path = os.path.join(dir_path, file_name) + self.timestamp = timestamp + if self.timestamp == -1: + self.timestamp = os.path.getmtime(self.path) + + +class BuildGraphFileDesc(FileDesc): + def __init__(self, file_name, dir_path, timestamp, graph_id, graph_name): + super(BuildGraphFileDesc, self).__init__(file_name, dir_path, timestamp) + self.graph_id = graph_id + self.graph_name = graph_name + + +class NpuDumpFileDesc(FileDesc): + def __init__(self, file_name, dir_path, timestamp, op_name, op_type, task_id, stream_id=0): + super(NpuDumpFileDesc, self).__init__(file_name, dir_path, timestamp) + self.op_name = op_name + self.op_type = op_type + self.task_id = task_id + stream_id = 0 if stream_id is None else int(stream_id) + self.stream_id = stream_id + + +class DumpDecodeFileDesc(NpuDumpFileDesc): + def __init__(self, file_name, dir_path, timestamp, op_name, op_type, task_id, anchor_type, anchor_idx): + super(DumpDecodeFileDesc, self).__init__(file_name, dir_path, timestamp, op_name, op_type, task_id) + self.type = anchor_type + self.idx = anchor_idx diff --git a/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/precision_tool/lib/fusion.py b/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/precision_tool/lib/fusion.py new file mode 100644 index 0000000000000000000000000000000000000000..3883bbce7092415ce4a5f7468b677e5eac5655d7 --- /dev/null +++ b/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/precision_tool/lib/fusion.py @@ -0,0 +1,109 @@ +# coding=utf-8 +import json +import os +import shutil + +from lib.tool_object import ToolObject +from lib.util import util +import config as cfg + +FUSION_RESULT_FILE_NAME = 'fusion_result.json' +EFFECT_TIMES_KEY = 'effect_times' +GRAPH_FUSION_KEY = 'graph_fusion' +UB_FUSION_KEY = 'ub_fusion' +GRAPH_ID_KEYS = ['graphId', 'session_and_graph_id'] + + +class FusionResult(object): + def __init__(self, fusion_json): + self.fusion_json = fusion_json + + def get_effect_graph_fusion(self): + """Get effect graph fusion rule""" + if GRAPH_FUSION_KEY in self.fusion_json: + return self._get_effect_fusion(self.fusion_json[GRAPH_FUSION_KEY]) + return {} + + def get_effect_ub_fusion(self): + """Get effect UB fusion rule""" + if UB_FUSION_KEY in self.fusion_json: + return self._get_effect_fusion(self.fusion_json[UB_FUSION_KEY]) + return {} + + def graph_id(self): + """Get graph id""" + for key in GRAPH_ID_KEYS: + if key in self.fusion_json: + return self.fusion_json[key] + return "NONE" + + @staticmethod + def _get_effect_fusion(fusion): + res = {} + for fusion_name in fusion: + effect_times = int(fusion[fusion_name][EFFECT_TIMES_KEY]) + if effect_times > 0: + res[fusion_name] = effect_times + return res + + +class Fusion(ToolObject): + def __init__(self): + super(Fusion, self).__init__() + self.fusion_result = [] + self.log = util.get_log() + + def prepare(self, json_path='./'): + """Prepare fusion rule manager + :param json_path: path to fusion_result.json + :return: None + """ + util.create_dir(cfg.FUSION_DIR) + file_path = os.path.join(json_path, FUSION_RESULT_FILE_NAME) + file_path_local = os.path.join(cfg.FUSION_DIR, FUSION_RESULT_FILE_NAME) + if not os.path.isfile(file_path): + if not os.path.isfile(file_path_local): + self.log.warning("Can not find fusion result json.") + return + else: + shutil.copy(file_path, cfg.FUSION_DIR) + fe_jsons = self._get_result_jsons(file_path_local) + for fe_json in fe_jsons: + self.fusion_result.append(FusionResult(fe_json)) + + def check(self): + """Check fusion rules + :return: None + """ + self.log.info("Check effect fusion rule list.") + for fusion in self.fusion_result: + graph_fusion_table = self._build_table('Graph Fusion [GraphID: %s]' % fusion.graph_id(), + fusion.get_effect_graph_fusion()) + ub_fusion_table = self._build_table('UB Fusion [GraphID: %s]' % fusion.graph_id(), + fusion.get_effect_ub_fusion()) + util.print_panel(util.create_columns([graph_fusion_table, ub_fusion_table]), + title='GraphID:' + fusion.graph_id(), fit=True) + + @staticmethod + def _get_result_jsons(file_name): + result_jsons = [] + with open(file_name, 'r') as f: + txt = f.read() + sk = [] + start = -1 + for i in range(len(txt)): + if txt[i] == '{': + sk.append('{') + if txt[i] == '}': + sk.pop() + if len(sk) == 0: + result_jsons.append(json.loads(txt[start+1: i+1])) + start = i + return result_jsons + + @staticmethod + def _build_table(title, fusion): + table = util.create_table(title, ['Fusion Name', 'Effect times']) + for f in fusion: + table.add_row(f, str(fusion[f])) + return table diff --git a/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/precision_tool/lib/graph_manager.py b/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/precision_tool/lib/graph_manager.py new file mode 100644 index 0000000000000000000000000000000000000000..42e34295fb1a0efb786e478d5fd872175680c5e2 --- /dev/null +++ b/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/precision_tool/lib/graph_manager.py @@ -0,0 +1,115 @@ +# coding=utf-8 +""" +Graph Manager +""" +import json +import os +import collections +import time +import config as cfg +from lib.tf_graph import TfGraph +from lib.op import Op +from lib.constant import Constant +from lib.npu_graph import NpuGraph +from lib.tf_graph import TfGraph +from lib.util import util +from lib.precision_tool_exception import catch_tool_exception +from lib.precision_tool_exception import PrecisionToolException + + +class GraphManager(object): + def __init__(self): + self.log = util.get_log() + self.npu_graphs = collections.OrderedDict() + self.tf_graph = None + + def prepare(self): + # prepare npu graphs + if not os.path.exists(cfg.NPU_DIR): + util.create_dir(cfg.NPU_DIR) + sub_dirs = os.listdir(cfg.NPU_DIR) + if len(sub_dirs) == 0: + # create default dir + sub_dirs = [Constant.DEFAULT_DEBUG_ID] + for sub_dir in sub_dirs: + npu_graph = NpuGraph(sub_dir) + npu_graph.prepare() + self.npu_graphs[sub_dir] = npu_graph + # prepare cpu graph + self.tf_graph = TfGraph(cfg.TF_GRAPH_DIR) + + def check_cast(self): + for graph in self.npu_graphs.values(): + graph.check_cast() + + def check_dtype(self): + for graph in self.npu_graphs.values(): + graph.check_dtype() + + def check_similarity(self): + self._check_npu_graph_similarity() + + def _check_npu_graph_similarity(self): + """Check npu graph similarity""" + if len(self.npu_graphs) < 2: + self.log.debug("Only one npu debug, no need to check npu graph similarity.") + return + left_graphs = self.npu_graphs[Constant.DEFAULT_DEBUG_ID].sub_graphs + right_graphs = self.npu_graphs[Constant.NPU_DEBUG_ID_1].sub_graphs + for left_graph in left_graphs.values(): + for right_graph in right_graphs.values(): + if left_graph.graph_id != right_graph.graph_id: + continue + left_graph.compare(right_graph) + + def get_graphs(self, debug_id): + if debug_id not in self.npu_graphs: + raise PrecisionToolException("Get graphs failed with no debug_id:%s" % debug_id) + return self.npu_graphs[debug_id].build_json_files + + def get_ops(self, op_name, graph_name=None): + """ Get npu/tf ops by op_name + :param op_name: op name + :param graph_name: graph name + :return: npu op dict: debug_id->Op, tf op + """ + npu_ops = collections.OrderedDict() + for debug_id, npu_graph in self.npu_graphs.items(): + npu_ops[debug_id] = npu_graph.get_op(op_name, graph_name) + # tf graph op + return npu_ops, None + + def print_op_list(self, op_type='', op_name='', pass_name='', kernel_name=''): + if op_type == '' and op_name == '' and pass_name == '' and kernel_name == '': + table_list = [] + for debug_id, graph in self.npu_graphs.items(): + table = util.create_table(debug_id, ["OpType", "Count"]) + op_type_counter = collections.Counter() + for op in graph.ops_list: + op_type_counter[op.type()] += 1 + for op_types, count in op_type_counter.items(): + table.add_row(op_types, str(count)) + table_list.append(table) + util.print(util.create_columns(table_list)) + + else: + for debug_id, graph in self.npu_graphs.items(): + ops = graph.list_ops(op_type, op_name, pass_name, kernel_name) + ops_txt = ['[green][%s][/green][yellow][%s][/yellow] %s' % ( + op.type(), op.pass_name(), op.name()) for op in ops] + util.print_panel(Constant.NEW_LINE.join(ops_txt), debug_id) + + @staticmethod + def op_graph_summary(ops): + npu_summary = collections.OrderedDict() + for debug_id, op in ops.items(): + npu_summary[debug_id] = collections.OrderedDict() + for op_detail in op: + npu_summary[debug_id][op_detail.graph_name] = op_detail.summary() + return npu_summary, None + + def save_sub_graph(self, ops, deep): + for debug_id, op in ops.items(): + if debug_id in self.npu_graphs: + for op_detail in op: + self.npu_graphs[debug_id].save_sub_graph(op_detail, deep) diff --git a/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/precision_tool/lib/h5_util.py b/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/precision_tool/lib/h5_util.py new file mode 100644 index 0000000000000000000000000000000000000000..4a9ab1480bf2fe39b392b8f2629abe837b796194 --- /dev/null +++ b/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/precision_tool/lib/h5_util.py @@ -0,0 +1,47 @@ +import h5py +import os +import numpy as np + +from lib.util import util +from lib.constant import Constant +import config as cfg + + +class H5Util(object): + def __init__(self, file_name): + self.log = util.get_log() + self.file_name = file_name + self.h5 = None + self._prepare() + + def __del__(self): + if self.h5 is not None: + self.h5.close() + + def _prepare(self): + if not os.path.isfile(self.file_name) or not str(self.file_name).endswith(Constant.Suffix.H5): + self.log.error("File [%s] not exist or not a h5 file" % self.file_name) + self.h5 = h5py.File(self.file_name, 'r') + + def get_tensor_by_name(self, tensor_name): + if self.h5 is None: + self.log.warning("h5 file is None.") + return None + return np.array(self.h5[tensor_name]) + + def print_tensor(self, tensor_name): + tensor = self.get_tensor_by_name(tensor_name) + if tensor is None: + self.log.warning("Tensor:%s not exist." % tensor_name) + return + file_path = self._dump_numpy(tensor_name, tensor) + util.print_npy_summary(os.path.dirname(file_path), os.path.basename(file_path)) + + def _dump_numpy(self, tensor_name, tensor): + if not os.path.exists(cfg.DUMP_DECODE_DIR): + util.create_dir(cfg.DUMP_DECODE_DIR) + file_name = tensor_name.replace('/', '_').strip('_') + '.npy' + file_path = os.path.join(cfg.DUMP_DECODE_DIR, file_name) + self.log("Dump file: %s" % file_path) + np.save(file_path, tensor) + return file_path diff --git a/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/precision_tool/lib/interactive_cli.py b/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/precision_tool/lib/interactive_cli.py new file mode 100644 index 0000000000000000000000000000000000000000..c15eb50d845bc97ffe692bf95ca340625a0fba39 --- /dev/null +++ b/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/precision_tool/lib/interactive_cli.py @@ -0,0 +1,89 @@ +# coding=utf-8 +import cmd +from .util import util +from .precision_tool import PrecisionTool + +HEADER = r""" ____ _ _ ______ __ + / __ \________ _____(_)____(_)___ ____/_ __/___ ____ / / + / /_/ / ___/ _ \/ ___/ / ___/ / __ \/ __ \/ / / __ \/ __ \/ / + / ____/ / / __/ /__/ (__ ) / /_/ / / / / / / /_/ / /_/ / / +/_/ /_/ \___/\___/_/____/_/\____/_/ /_/_/ \____/\____/_/""" + +HELP_AC = "Run auto check function, use [-c] to start vector compare process.\n" \ + " usage: ac (-c) \n" +HELP_RUN = "Run any shell command.\n" \ + " usage: (run) vim tensor_name.txt \n" +HELP_PT = "Print npy tensor, use [-c] to convert and save to txt file.\n" \ + " usage: pt (-c) [tensor_name.npy] \n" + + +class InteractiveCli(cmd.Cmd): + def __init__(self): + cmd.Cmd.__init__(self) + self.prompt = "PrecisionTool > " + self.precision_tool = None + util.print_panel(HEADER) + self._prepare() + + def default(self, line=''): + util.execute_command(line) + return False + + def _prepare(self): + self.precision_tool = PrecisionTool() + self.precision_tool.prepare() + + def do_ac(self, line=''): + """Auto check.""" + argv = line.split(' ') if line != '' else [] + self.precision_tool.do_auto_check(argv) + + def do_run(self, line=''): + """Run any shell command""" + util.execute_command(line) + + def do_ls(self, line=''): + """List ops: \n usage: ls (op(default)/dump) -n [op_name] -t [op_type]""" + argv = line.split(' ') if line != '' else [] + if len(argv) > 0 and argv[0] == 'dump': + return self.precision_tool.do_list_dump(argv[1:]) + self.precision_tool.do_list_nodes(argv) + + def do_ni(self, line=''): + """Print node info:\n usage: ni (-n) [op_name]""" + argv = line.split(' ') if line != '' else [] + if '-n' not in argv and '-h' not in argv: + argv.insert(0, '-n') + self.precision_tool.do_node_info(argv) + + def do_dc(self, line=''): + """Convert npu dump by op names:\n usage: dc (-n) [npu dump file] -f [target format]""" + argv = line.split(' ') if line != '' else [] + # if len(argv) == 0: + # return self.precision_tool.do_convert_all_npu_dump() + if '-n' not in argv: + argv.insert(0, '-n') + self.precision_tool.do_convert_npu_dump(argv) + + def do_vc(self, line=''): + """Do vector compare: \n usage: vc """ + argv = line.split(' ') if line != '' else [] + self.precision_tool.do_vector_compare(argv) + + def do_vcs(self, line=''): + """Do vector compare summary""" + argv = line.split(' ') if line != '' else [] + self.precision_tool.do_vector_compare_summary(argv) + + def do_pt(self, line=''): + """Print data info:\n usage: pt (-n) [*.npy] (-c)\n -c: convert and save to txt file""" + argv = line.split(' ') if line != '' else [] + if len(argv) > 0 and argv[0] != '-n' and argv[0] != '-c': + argv.insert(0, '-n') + self.precision_tool.do_print_data(argv) + + def do_cp(self, line=''): + """Compare two data file """ + argv = line.split(' ') if line != '' else [] + argv.insert(0, '-n') + self.precision_tool.do_compare_data(argv) diff --git a/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/precision_tool/lib/msquickcmp_adapter.py b/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/precision_tool/lib/msquickcmp_adapter.py new file mode 100644 index 0000000000000000000000000000000000000000..885be1f8a1a832dd6b463f05f21dd84844f63a3e --- /dev/null +++ b/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/precision_tool/lib/msquickcmp_adapter.py @@ -0,0 +1,88 @@ +# coding=utf-8 +import os +import time +import pathlib +import shutil +from lib.util import util +from lib.constant import Constant +import config as cfg +from lib.precision_tool_exception import catch_tool_exception +from lib.precision_tool_exception import PrecisionToolException + + +class MsQuickCmpAdapter(object): + def __init__(self, output_path): + self.output_path = output_path + self.log = util.get_log() + + def run(self): + if self.output_path is None or not os.path.isdir(self.output_path): + raise PrecisionToolException("Invalid output path.") + if os.path.exists(cfg.DATA_ROOT_DIR): + raise PrecisionToolException("Precision data dir exist, can not adapt msquickcmp result.") + + for dir_path, dir_names, file_names in os.walk(self.output_path, followlinks=True): + if 'model' in dir_names: + self._adapt_model(os.path.join(dir_path, 'model')) + if 'dump_data' in dir_names: + self._adapt_dump(os.path.join(dir_path, 'dump_data')) + for file_name in file_names: + if str(file_name).endswith(Constant.Suffix.CSV): + self._adapt_vector_compare_result(os.path.join(dir_path, file_name)) + + def _adapt_model(self, path): + file_names = os.listdir(path) + graph_id = 0 + for file_name in file_names: + if str(file_name).endswith(Constant.Suffix.JSON): + self.log.info("Find msquickcmp model json: %s", file_name) + util.create_dir(cfg.DEFAULT_NPU_GRAPH_DIR) + graph_file_name = 'ge_proto_%d_%s.txt' % (graph_id, cfg.BUILD_JSON_GRAPH_NAME) + graph_json_file_name = graph_file_name + Constant.Suffix.JSON + pathlib.Path(os.path.join(cfg.DEFAULT_NPU_GRAPH_DIR, graph_file_name)).touch() + src_path = os.path.join(path, file_name) + dst_path = os.path.join(cfg.DEFAULT_NPU_GRAPH_DIR, graph_json_file_name) + self.log.info("Copy graph file: %s->%s", src_path, dst_path) + shutil.copy(src_path, dst_path) + time.sleep(3) + pathlib.Path(dst_path).touch() + if not util.empty_dir(cfg.DEFAULT_NPU_GRAPH_DIR): + self.log.info("Adapt model success.") + + def _adapt_dump(self, path): + dir_names = os.listdir(path) + if 'tf' in dir_names: + self._adapt_tf_dump(os.path.join(path, 'tf')) + if 'onnx' in dir_names: + self._adapt_tf_dump(os.path.join(path, 'onnx')) + if 'npu' in dir_names: + self._adapt_npu_dump(os.path.join(path, 'npu')) + + def _adapt_tf_dump(self, path): + if util.empty_dir(path): + return + src_path = os.path.abspath(path) + util.create_dir(cfg.TF_DIR) + dst_path = cfg.TF_DUMP_DIR + self.log.info("Create symbol link file: %s->%s", src_path, dst_path) + os.symlink(src_path, dst_path) + self.log.info("Adapt tf dump success.") + + def _adapt_npu_dump(self, path): + sub_dirs = os.listdir(path) + self.log.info("Find npu dump dir:%s", sub_dirs) + sub_dirs = filter(lambda x: str(x).isdigit(), sub_dirs) + for sub_dir in sub_dirs: + util.create_dir(cfg.DEFAULT_NPU_DUMP_DIR) + src_path = os.path.abspath(os.path.join(path, sub_dir)) + dst_path = os.path.join(cfg.DEFAULT_NPU_DUMP_DIR, sub_dir) + self.log.info("Create symbol link file: %s->%s", src_path, dst_path) + os.symlink(src_path, dst_path) + self.log.info("Adapt npu dump success.") + + def _adapt_vector_compare_result(self, path): + target_path = os.path.join(cfg.VECTOR_COMPARE_PATH, '0') + util.create_dir(target_path) + dst_path = os.path.join(target_path, os.path.basename(path)) + shutil.copy(path, dst_path) + self.log.info("Adapt vector compare result.") diff --git a/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/precision_tool/lib/npu_dump.py b/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/precision_tool/lib/npu_dump.py new file mode 100644 index 0000000000000000000000000000000000000000..2da705c158e956883fc6e1fc1938e9127c43df99 --- /dev/null +++ b/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/precision_tool/lib/npu_dump.py @@ -0,0 +1,184 @@ +# coding=utf-8 +import os +import re +from lib.util import util +from lib.constant import Constant +import config as cfg +from lib.precision_tool_exception import catch_tool_exception +from lib.precision_tool_exception import PrecisionToolException + + +class NpuDumpDecodeFile(object): + def __init__(self): + self.log = util.get_log() + self.input_files = {} + self.output_files = {} + self.timestamp = -1 + self.op_name = '' + self.op_type = '' + self.task_id = -1 + # self.stream_id = -1 + + def update(self, file_info): + """Prepare op npu decode file map.""" + if not self._check(file_info): + self.log.warning('Invalid NpuDumpDecodeFile: %s', file_info) + return + if file_info.type == 'input': + self.input_files[file_info.idx] = file_info + else: + self.output_files[file_info.idx] = file_info + + def summary(self): + txt = ['[yellow][%s][TaskID: %d][/yellow][green][%s][/green] %s' % ( + self.timestamp, self.task_id, self.op_type, self.op_name)] + if len(self.input_files) > 0: + info = self.input_files[0] + shape, dtype, max_data, min_data, mean = util.npy_info(info.path) + txt.append(' - Input: [green][0][/green][yellow][%s][%s][Max:%d][Min:%d][Mean:%d][/yellow] %s' % ( + shape, dtype, max_data, min_data, mean, info.file_name)) + for idx in range(1, len(self.input_files)): + info = self.input_files[idx] + shape, dtype, max_data, min_data, mean = util.npy_info(info.path) + txt.append(' [green][%d][/green][yellow][%s][%s][Max:%d][Min:%d][Mean:%d][/yellow] %s' % ( + idx, shape, dtype, max_data, min_data, mean, info.file_name)) + if len(self.output_files) > 0: + info = self.output_files[0] + shape, dtype, max_data, min_data, mean = util.npy_info(info.path) + txt.append(' - Output: [green][0][/green][yellow][%s][%s][Max:%d][Min:%d][Mean:%d][/yellow] %s' % ( + shape, dtype, max_data, min_data, mean, info.file_name)) + for idx in range(1, len(self.output_files)): + info = self.output_files[idx] + shape, dtype, max_data, min_data, mean = util.npy_info(info.path) + txt.append(' [green][%d][/green][yellow][%s][%s][Max:%d][Min:%d][Mean:%d][/yellow] %s' % ( + idx, shape, dtype, max_data, min_data, mean, info.file_name)) + return Constant.NEW_LINE.join(txt) + + def _check(self, file_info): + if self.timestamp == -1: + self.timestamp = file_info.timestamp + self.op_name = file_info.op_name + self.op_type = file_info.op_type + self.task_id = file_info.task_id + # self.stream_id = file_info['stream'] + return True + return self.timestamp == file_info['timestamp'] + + +class NpuDump(object): + def __init__(self, debug_id=Constant.DEFAULT_DEBUG_ID): + """Init""" + self.log = util.get_log() + self.debug_id = debug_id + npu_root = os.path.join(cfg.NPU_DIR, debug_id) + self.dump_root = os.path.join(npu_root, Constant.DUMP) + self.decode_dir = os.path.join(cfg.DUMP_DECODE_DIR, debug_id) + self.dump_files = None + self._init_dirs() + + def prepare(self): + """Prepare npu/cpu dump files""" + self._parse_dump_files() + + def get_dump_files_by_op(self, op): + """Get npu dump files by Op""" + npu_files = {} + op_name = op.name().replace('/', '_').replace('.', '_') + match_name = op.type() + '.' + op_name + '\\.' + for f in self.dump_files: + # match op name and graph name, infer dump directory may not has graph + if re.match(match_name, f) and (op.graph_name in self.dump_files[f].path or cfg.NET_TYPE == 'infer'): + npu_files[f] = self.dump_files[f] + return npu_files + + @catch_tool_exception + def op_dump_summary(self, op): + """ print op dump info""" + if op is None: + raise PrecisionToolException("Get None operator") + # search npu dump file by op name + npu_dump_files = self.get_npu_dump_decode_files_by_op(op) + npu_dump_files = sorted(npu_dump_files.values(), key=lambda x: (x.idx, x.timestamp)) + input_txt = ['NpuDumpInput:'] + output_txt = ['NpuDumpOutput:'] + for npu_dump_file in npu_dump_files: + if npu_dump_file.type == 'input': + input_txt.append(' -[green][%s][/green] %s' % (npu_dump_file.idx, npu_dump_file.file_name)) + input_txt.append(' └─ [yellow]%s[/yellow]' % util.gen_npy_info_txt(npu_dump_file.path)) + else: + output_txt.append(' -[green][%s][/green] %s' % (npu_dump_file.idx, npu_dump_file.file_name)) + output_txt.append(' └─ [yellow]%s[/yellow]' % util.gen_npy_info_txt(npu_dump_file.path)) + input_txt.extend(output_txt) + return Constant.NEW_LINE.join(input_txt) + + def _init_dirs(self): + util.create_dir(self.dump_root) + util.create_dir(self.decode_dir) + + @catch_tool_exception + def _parse_dump_files(self): + """prepare npu dump, support soft link""" + sub_dir = util.get_newest_dir(self.dump_root) + sub_dir = os.path.join(self.dump_root, sub_dir) if sub_dir != '' else self.dump_root + self.dump_files = util.list_npu_dump_files(sub_dir) + + def list_dump(self, dir_path, file_name): + """""" + + @staticmethod + def get_npu_dump_decode_files_by_name(file_name): + file_name = file_name.replace('/', '_') + return util.list_npu_dump_convert_files(cfg.DECODE_DIR, file_name) + + def get_npu_dump_decode_files_by_op(self, op): + """Get npu dump decode files by op""" + dump_files = self.get_dump_files_by_op(op) + result = {} + for dump_file in dump_files.values(): + dump_decode_files = util.list_npu_dump_decode_files(self.decode_dir, dump_file.file_name) + if len(dump_decode_files) == 0: + util.convert_dump_to_npy(dump_file.path, self.decode_dir) + dump_decode_files = util.list_npu_dump_decode_files(self.decode_dir, dump_file.file_name) + result.update(dump_decode_files) + return result + + def convert_npu_dump(self, name, data_format=None, dst_path=None): + """Convert npu dump to npy of data_format""" + if os.path.isfile(name): + # absolute path to file + self.log.info("Decode file: %s", name) + file_name = os.path.basename(name) + file_path = name + elif os.path.isdir(name): + # decode all files in path + self.log.info("Decode all files in path: %s", name) + file_name = '' + file_path = name + elif self.dump_files is not None and name in self.dump_files: + self.log.info("Decode npu dump file: %s in default dump path", name) + file_info = self.dump_files[name] + file_name = file_info.file_name + file_path = file_info.path + else: + # maybe op name + file_info = self._get_file_by_op_name(name) + if file_info is None: + raise PrecisionToolException("Can not find any op/dump file named %s" % name) + file_name = file_info.file_name + file_path = file_info.path + dst_path = cfg.DUMP_CONVERT_DIR if dst_path is None else dst_path + util.convert_dump_to_npy(file_path, dst_path, data_format) + dump_convert_files = util.list_npu_dump_convert_files(dst_path, file_name) + # print result info + + summary_txt = ['SrcFile: %s' % name] + for convert_file in dump_convert_files.values(): + summary_txt.append(' - %s' % convert_file.file_name) + util.print_panel(Constant.NEW_LINE.join(summary_txt)) + + def _get_file_by_op_name(self, op_name): + """Get dump file info by op name""" + for file_info in self.dump_files.values(): + if file_info.op_name == op_name: + return file_info + return None diff --git a/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/precision_tool/lib/npu_graph.py b/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/precision_tool/lib/npu_graph.py new file mode 100644 index 0000000000000000000000000000000000000000..48625bd631014d68df6f39af244b20f1c73b356c --- /dev/null +++ b/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/precision_tool/lib/npu_graph.py @@ -0,0 +1,296 @@ +# coding=utf-8 +""" +Graph Manager +""" +import json +import os +import collections +import time + +import config as cfg +from lib.op import Op +from lib.util import util +from lib.constant import Constant +from lib.precision_tool_exception import catch_tool_exception +from lib.precision_tool_exception import PrecisionToolException + +DANGEROUS_CAST = { + 'DT_FLOAT': ['DT_INT32'] +} + +NO_DIG_OPS = ['AtomicAddrClean', 'NetOutput'] +CKPT_META_SHUFFIX='.meta' + +OP_CAST = 'Cast' + + +class NpuSubGraph(object): + def __init__(self, graph_json, build_file): + self.log = util.get_log() + self.graph_name = graph_json['name'] + self.graph = graph_json + self.build_file = build_file + self.ops_list = collections.OrderedDict() + self.ops_type_list = {} + self._prepare() + self.graph_id = self._get_graph_id() + + def _prepare(self): + self.log.debug("Graph %s operator count: %d" % (self.graph_name, len(self.graph['op']))) + for op_json in self.graph['op']: + op_name = op_json['name'] + op_type = op_json['type'] + if op_name not in self.ops_list: + self.ops_list[op_name] = [] + op = Op(op_json, self.ops_list, self.graph['name']) + if op_type not in self.ops_type_list: + self.ops_type_list[op_type] = {} + self.ops_list[op_name] = op + self.ops_type_list[op_type][op_name] = op + + def _get_graph_id(self): + if 'attr' in self.graph: + for item in self.graph['attr']: + if item['key'] == '_session_graph_id': + return item['value']['s'] + self.log.warning("Unknown sub graph id.") + return "UNKNOWN" + + def compare(self, sub_graph): + """compare with another sub graph""" + if not isinstance(sub_graph, NpuSubGraph): + raise PrecisionToolException("Should compare with another subgraph.") + right_ops_list = sub_graph.ops_list + ignore_ops = ["TransData", "Cast", "Recv", "Send", "Variable", "NetOutput", "NoOp", "Assign", "Constant", + "StreamActive"] + similar_count = 0 + for op_name in self.ops_list: + if self.ops_list[op_name].type() in ignore_ops: + continue + if op_name not in right_ops_list: + self.log.warning("Can not Find [%s] %s in right subgraph.", self.ops_list[op_name].type(), op_name) + continue + result, similar = self.ops_list[op_name].compare(right_ops_list[op_name]) + if not similar: + util.print_panel(result, title=op_name) + else: + similar_count += 1 + for op_name in right_ops_list: + if right_ops_list[op_name].type() in ignore_ops: + continue + if op_name not in self.ops_list: + self.log.warning("Can not Find [%s] %s in left subgraph.", right_ops_list[op_name].type(), op_name) + self.log.info("Compare [%s] [%s], similarity is [%s / %s]", + self.graph_name, sub_graph.graph_name, similar_count, len(self.ops_list)) + + def get_op(self, name): + if name in self.ops_list: + return [self.ops_list[name]] + guess_op_list = [] + for op_detail in self.ops_list.values(): + if name in op_detail.name(): + guess_op_list.append(op_detail) + return guess_op_list + + def check_cast(self): + cast_list = {} + danger_cast_list = {} + if OP_CAST in self.ops_type_list: + cast_ops = self.ops_type_list[OP_CAST] + for op in cast_ops.values(): + input_type = '' + output_type = '' + for input_desc in op.inputs(): + input_type = input_desc.dtype() if input_desc.dtype() != '' else input_type + for output_desc in op.outputs(): + output_type = output_desc.dtype() if output_desc.dtype() != '' else output_type + cast_type = "%s -> %s" % (input_type, output_type) + if cast_type not in cast_list: + cast_list[cast_type] = [] + cast_list[cast_type].append(op.name()) + for cast_type in cast_list: + if self._is_dangerous_cast(cast_type): + summary_txt = "[green][Cast][/green][red][%s][/red] %s" % (cast_type, cast_list[cast_type]) + util.print(summary_txt) + + @staticmethod + def _is_dangerous_cast(cast_type): + """Check if cast """ + cast_info = cast_type.split(" -> ") + input_dtype = cast_info[0] + output_dtype = cast_info[1] + if input_dtype in DANGEROUS_CAST: + if output_dtype in DANGEROUS_CAST[input_dtype]: + return True + return False + + +class NpuGraph(object): + def __init__(self, debug_id=Constant.DEFAULT_DEBUG_ID): + self.log = util.get_log() + self.build_files = None + self.build_json_files = [] + self.debug_id = debug_id + self.npu_root = os.path.join(cfg.NPU_DIR, debug_id) + self.graph_root = os.path.join(self.npu_root, Constant.GRAPH) + self.sub_graphs = collections.OrderedDict() + self.ops_list = [] + util.create_dir(self.graph_root) + + @catch_tool_exception + def prepare(self): + """prepare""" + self._prepare_npu_graphs() + if self.build_files is not None: + for build_file in self.build_files: + self._parse_ops(build_file) + + def check_cast(self): + """Check cast op type""" + for sub_graph in self.sub_graphs.values(): + sub_graph.check_cast() + + def check_dtype(self): + """Check op input/output dtype""" + for op in self.ops_list: + input_dtype = '' + for input_desc in op.inputs(): + input_dtype += ' ' + input_desc.dtype() + output_dtype = '' + for output_desc in op.outputs(): + output_dtype += ' ' + output_desc.dtype() + util.print('[green][%s][/green] %s\n - Input: %s\n - Output: %s' % ( + op.type(), op.name(), input_dtype, output_dtype)) + + def check_similarity(self): + """Check graph similarity.""" + + @catch_tool_exception + def save_sub_graph(self, op, deep=0, dump_manager=None, compare_manager=None): + """Save sub graph""" + if op is None: + raise PrecisionToolException("Save sub graph failed as root operator is None.") + try: + from graphviz import Digraph + file_name_list = [self.debug_id, op.graph_name, op.type(), op.name().replace('/', '_').replace('.', '_'), + str(deep), 'gv'] + file_name = '.'.join(file_name_list) + path = os.path.join(cfg.OP_GRAPH_DIR, file_name) + dot = Digraph(file_name, filename=path, node_attr={'shape': 'Mrecord'}, format='svg') + dot_list = [] + edge_list = [] + self._gen_sub_graph(dot, op, deep, dot_list, edge_list, 'red', direction='all', + dump_manager=dump_manager, compare_manager=compare_manager) + dot.format = 'svg' + dot.save(path) + self.log.info("Sub graph saved to %s" % os.path.abspath(cfg.OP_GRAPH_DIR)) + try: + dot.view(path) + time.sleep(1) + except Exception as err: + raise PrecisionToolException( + "graphviz not install, use [yum/apt-get] install graphviz xdg-utils. %s" % err) + except ImportError as err: + raise PrecisionToolException("Save sub graph failed as import graphviz module failed. %s" % err) + + def _gen_sub_graph(self, dot, op, deep, dot_list, edge_list, color='black', direction='all', + dump_manager=None, compare_manager=None): + """Gen sub graph""" + if deep == 0 or op.type() in NO_DIG_OPS: + return + if op.name() not in dot_list: + dot.node(op.name(), self._gen_sub_graph_label(op), color=color, tooltip=op.summary(True)) + dot_list.append(op.name()) + # add input and output + for desc in op.inputs(): + sub_op = self.get_op(desc.name(), op.graph_name) + if len(sub_op) != 0: + sub_op = sub_op[0] + if direction in ['all', 'input']: + self._gen_sub_graph(dot, sub_op, deep - 1, dot_list, edge_list, direction='input') + if sub_op.name() in dot_list: + src_edge = '%s:o%d' % (sub_op.name(), desc.peer_idx()) + else: + dot.node(sub_op.name(), self._gen_sub_graph_label(sub_op), color=color, tooltip=op.summary(True)) + src_edge = '%s:o%d' % (sub_op.name(), desc.peer_idx()) + dst_edge = '%s:i%d' % (op.name(), desc.idx()) + if src_edge + dst_edge not in edge_list: + dot.edge(src_edge, dst_edge) + edge_list.append(src_edge + dst_edge) + # add output + for desc in op.outputs(): + for out_node_name in desc.names(): + sub_op = self.get_op(out_node_name, op.graph_name) + if len(sub_op) != 0 and direction in ['all', 'output']: + sub_op = sub_op[0] + self._gen_sub_graph(dot, sub_op, deep - 1, dot_list, edge_list, direction='output') + + def _gen_sub_graph_label(self, op): + input_labels = [] + for desc in op.inputs(): + input_labels.append(self._gen_sub_graph_desc(desc, 'i')) + output_labels = [] + for desc in op.outputs(): + output_labels.append(self._gen_sub_graph_desc(desc, 'o')) + str_cell = '|' + return '{{ %s } | [%s] %s | { %s }}' % (str_cell.join(input_labels), op.type(), op.name(), + str_cell.join(output_labels)) + + @staticmethod + def _gen_sub_graph_desc(desc, id_prefix): + desc_str = r'<%s%d> [%d]' % (id_prefix, desc.idx(), desc.idx()) + desc_str = r'%s [%s]' % (desc_str, desc.dtype()) if desc.dtype() != '' else desc_str + desc_str = r'%s\n%s' % (desc_str, desc.shape()) if len(desc.shape()) != 0 else desc_str + return desc_str + + def list_ops(self, op_type='', op_name='', pass_name='', kernel_name=''): + """list ops in graph""" + return filter(lambda op: op_type in op.type() and op_name in op.name() and pass_name in op.pass_name() + and kernel_name in op.kernel_name(), self.ops_list) + + def get_op(self, name, graph_name=None): + """get op by name""" + # get op in specific sub graph + if graph_name is not None and graph_name in self.sub_graphs: + return self.sub_graphs[graph_name].get_op(name) + ops = [] + for sub_graph in self.sub_graphs.values(): + ops.extend(sub_graph.get_op(name)) + # check if there is an exact match operation + match_ops = list(filter(lambda x: x.name() == name, ops)) + if len(match_ops) != 0: + return match_ops + # return guess operations by name + self.log.info("Can not find Operator named %s. You may mean the operator bellow.", name) + guess_op_name_list = ['[green][%s][/green] %s' % (x.type(), x.name()) for x in ops] + util.print_panel(Constant.NEW_LINE.join(guess_op_name_list), title='Possible Operators') + return ops + + def _prepare_npu_graphs(self): + """prepare ge graphs """ + # move graphs to precision data dir + graph_files = util.list_ge_graph_files(self.graph_root) + self.build_files = sorted(filter(lambda x: x.graph_name == cfg.BUILD_JSON_GRAPH_NAME, graph_files.values()), + key=lambda x: x.graph_id) + if len(self.build_files) == 0: + self.log.warning("Can not find any build files in dir: %s", self.graph_root) + self.log.info("Find [%d] GE build files.", len(self.build_files)) + + @catch_tool_exception + def _parse_ops(self, build_file): + """Parse *_Build.txt.json to op objects.""" + build_file_json = build_file.path + '.json' + build_file_json = util.convert_proto_to_json(build_file.path, build_file_json) + if build_file_json is not None: + self.build_json_files.append(build_file_json) + with open(build_file_json, 'r') as f: + graph_json = json.load(f) + if 'graph' not in graph_json: + raise PrecisionToolException("No graph in file: %s" % build_file.file_name) + if len(graph_json['graph']) != 1: + self.log.warning("There are more then one graph in ge build file, find %d" % len(graph_json['graph'])) + # sub_graphs = [] + for graph in graph_json['graph']: + npu_sub_graph = NpuSubGraph(graph, build_file) + self.sub_graphs[graph['name']] = npu_sub_graph + self.ops_list.extend(npu_sub_graph.ops_list.values()) diff --git a/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/precision_tool/lib/op.py b/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/precision_tool/lib/op.py new file mode 100644 index 0000000000000000000000000000000000000000..491f5590041b92f4768c1ccb702656d96086d5e4 --- /dev/null +++ b/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/precision_tool/lib/op.py @@ -0,0 +1,184 @@ +# coding=utf-8 +import json +import re +from typing import List +from lib.desc import InputDesc +from lib.desc import OutputDesc +from lib.util import util +from lib.constant import Constant +from lib.precision_tool_exception import PrecisionToolException + +NO_INPUT_NODES = ['Data', 'AtomicAddrClean', 'Recv', 'Constant'] +NO_OUTPUT_NODES = ['Send', 'Recv', 'NetOutput'] + +JSON_KEY_NAME = 'name' +JSON_KEY_TYPE = 'type' +JSON_KEY_ATTR = 'attr' +JSON_KEY = 'key' +JSON_VALUE = 'value' +JSON_KEY_LIST = 'list' +JSON_KEY_STR = 's' +JSON_KEY_PASS_NAME = 'pass_name' +JSON_KEY_DATA_DUMP_ORIGINAL_OP_NAMES = '_datadump_original_op_names' +JSON_KEY_GE_ATTR_OP_KERNEL_LIB_NAME = "_ge_attr_op_kernel_lib_name" + +KERNEL_NAME_SHUFFIX = '_kernelname' + + +class Op(object): + """ Op class. + name: op name + type: op type + inputs: list of input descs + outputs: list of output descs + """ + def __init__(self, op_json, op_list, graph_name): + """Init""" + self.op_json = op_json + self.op_list = op_list + self.graph_name = graph_name + self.input_list = None + self.output_list = None + self.log = util.get_log() + + def name(self): + """Get op name""" + return self.op_json[JSON_KEY_NAME] + + def json(self): + return json.dumps(self.op_json, indent=2) + + def type(self): + """Get op type""" + return self.op_json[JSON_KEY_TYPE] + + def inputs(self): + """Get the input list""" + if self.input_list is None: + self._parse_inputs() + return self.input_list + + def outputs(self): + """Get output list""" + if self.output_list is None: + self._parse_outputs() + return self.output_list + + def pass_name(self): + return self._attr(JSON_KEY_PASS_NAME) + + def kernel_name(self): + return self._attr(self.name() + KERNEL_NAME_SHUFFIX) + + def ge_attr_op_kernel_lib_name(self): + return self._attr(JSON_KEY_GE_ATTR_OP_KERNEL_LIB_NAME) + + def data_dump_original_op_names(self): + return self._attr(JSON_KEY_DATA_DUMP_ORIGINAL_OP_NAMES) + + def _attr(self, key): + if JSON_KEY_ATTR in self.op_json: + for attr in self.op_json[JSON_KEY_ATTR]: + if key == attr[JSON_KEY]: + if JSON_KEY_STR in attr[JSON_VALUE]: + return attr[JSON_VALUE][JSON_KEY_STR] + elif JSON_KEY_LIST in attr[JSON_VALUE]: + if JSON_KEY_STR in attr[JSON_VALUE][JSON_KEY_LIST]: + return attr[JSON_VALUE][JSON_KEY_LIST][JSON_KEY_STR] + else: + self.log.warning("Unknown attr format: %s", attr[JSON_VALUE]) + return '' + + def compare(self, right_op): + """Compare with another op""" + if not isinstance(right_op, Op): + raise PrecisionToolException("Should compare with another op.") + res_str = ['LeftOp(Type/Name) : [green][%s][/green] %s' % (self.type(), self.name()), + 'RightOp(Type/Name): [green][%s][/green] %s' % (right_op.type(), right_op.name())] + similar = True + if len(self.inputs()) != len(right_op.inputs()): + res_str.append("Input: [yellow]Input num mismatch.[/yellow]") + else: + res_str.append("Input:") + for left_input in self.inputs(): + for right_input in right_op.inputs(): + if left_input.idx() != right_input.idx(): + continue + txt, input_similar = left_input.compare(right_input) + res_str.append(' - ' + txt) + similar = similar and input_similar + if len(self.outputs()) != len(right_op.outputs()): + res_str.append("Output: [yellow]Output num mismatch.[/yellow]") + else: + res_str.append("Output:") + for left_output in self.outputs(): + for right_output in right_op.outputs(): + if left_output.idx() != right_output.idx(): + continue + txt, output_similar = left_output.compare(right_output) + res_str.append(' - ' + txt) + similar = similar and output_similar + return Constant.NEW_LINE.join(res_str), similar + + def summary(self, origin_txt=False): + """Summary of current op""" + res_str = ['Op(Type/Name): [green][%s][/green] %s' % (self.type(), self.name()), + 'KernelName: [yellow]%s[/yellow]' % self.kernel_name(), + 'KernelLibName: [yellow]%s[/yellow]' % self.ge_attr_op_kernel_lib_name(), + 'GraphName: [yellow]%s[/yellow]' % self.graph_name] + pass_name = self.pass_name() + if pass_name != '': + res_str.append('PassName: [yellow]%s[/yellow]' % pass_name) + origin_op = self.data_dump_original_op_names() + if origin_op != '': + res_str.append('OriginalOp: %s' % origin_op) + res_str.append('Input:%s' % InputDesc.summary.__doc__) + for i in self.inputs(): + res_str.append(' -' + i.summary(origin_txt)) + res_str.append('Output:') + for i in self.outputs(): + res_str.append(' -' + i.summary(origin_txt)) + return Constant.NEW_LINE.join(res_str) + + def _parse_inputs(self): + """ parse input desc in graph """ + self.input_list = [] + if 'input' not in self.op_json: + if self.type() not in NO_INPUT_NODES: + self.log.warning('Parse Op[%s][%s] inputs error.' % (self.type(), self.name())) + return self.input_list + desc_index = 0 + for i in range(len(self.op_json['input'])): + name = self.op_json['input'][i] + if name == '': + # if self.type() not in NO_INPUT_NODES: + # self.log.warning('invalid input name.') + continue + name_info = name.split(':') + if len(name_info) == 2 and int(name_info[1]) == -1: + # control edge + self.input_list.append(InputDesc(name, [], i)) + else: + self.input_list.append(InputDesc(name, self.op_json['input_desc'][desc_index], i)) + desc_index += 1 + self.input_list.sort(key=lambda x: x.index) + return self.input_list + + def _parse_outputs(self): + """ parse output desc in graph """ + self.output_list = [] + if 'dst_index' not in self.op_json: + if self.type() not in NO_OUTPUT_NODES: + self.log.warning('Parse Op[%s][%s] outputs error.' % (self.type(), self.name())) + return self.output_list + desc_index = 0 + for i in range(len(self.op_json['dst_index'])): + dst_name = self.op_json['dst_name'][i] + if self.op_json['dst_index'][i] == -1: + # control edge + self.output_list.append(OutputDesc(dst_name, [], -1)) + else: + self.output_list.append(OutputDesc(dst_name, self.op_json['output_desc'][desc_index], desc_index)) + desc_index += 1 + self.output_list.sort(key=lambda x: x.index) + return self.output_list diff --git a/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/precision_tool/lib/overflow.py b/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/precision_tool/lib/overflow.py new file mode 100644 index 0000000000000000000000000000000000000000..b3c783e5c0b019b690bba122aea71b8230263b48 --- /dev/null +++ b/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/precision_tool/lib/overflow.py @@ -0,0 +1,180 @@ +# coding=utf-8 +import json +import os + +from lib.util import util +import config as cfg +from lib.precision_tool_exception import PrecisionToolException +from lib.precision_tool_exception import catch_tool_exception +from lib.constant import Constant + +AI_CORE_OVERFLOW_STATUS = { + '0x8': '符号证书最小附属NEG符号位取反溢出', + '0x10': '整数加法、减法、乘法或乘加操作计算有溢出', + '0x20': '浮点计算有溢出', + '0x80': '浮点数转无符号数的输入是负数', + '0x100': 'FP32转FP16或32符号整数转FP16中出现溢出', + '0x400': 'CUBE累加出现溢出' +} +DHA_ATOMIC_ADD_STATUS = { + '0x9': '[atomic overflow] 向上溢出', + '0xA': '[atomic underflow] 向下溢出', + '0xB': '[atomic src nan] 源操作数非法', + '0xC': '[atomic dst nan] 目的操作数非法', + '0xD': '[atomic both nan] 源操作数和目的操作数均非法' +} +L2_ATOMIC_ADD_STATUS = { + '000': '[atomic no error] 无异常', + '001': '[atomic overflow] 向上溢出', + '010': '[atomic underflow] 向下溢出', + '011': '[atomic src nan] 源操作数非法', + '100': '[atomic dst nan] 目的操作数非法', + '101': '[atomic both nan] 源操作数和目的操作数均非法' +} + + +class Overflow(object): + def __init__(self): + """Init""" + self.log = util.get_log() + self.debug_files = None + + @catch_tool_exception + def prepare(self): + """Prepare""" + # find right path in DUMP_FILES_NPU_ALL + util.create_dir(cfg.NPU_OVERFLOW_DUMP_DIR) + sub_dir = util.get_newest_dir(cfg.NPU_OVERFLOW_DUMP_DIR) + overflow_dump_files = util.list_npu_dump_files(os.path.join(cfg.NPU_OVERFLOW_DUMP_DIR, sub_dir)) + self.debug_files = [item for item in overflow_dump_files.values() if item.op_type == 'Opdebug'] + # sort by timestamp + self.debug_files = sorted(self.debug_files, key=lambda x: x.timestamp) + self.log.info("Find [%d] debug files in overflow dir.", len(self.debug_files)) + + def check(self, max_num=3): + """Check overflow info""" + if len(self.debug_files) == 0: + self.log.info("[Overflow] Checked success. find [0] overflow node!") + return + self.log.info("[Overflow] Find [%s] overflow debug file. Will show top %s ops.", len(self.debug_files), max_num) + for i, debug_file in enumerate(self.debug_files): + debug_decode_files = self._decode_file(debug_file, True) + with open(debug_decode_files[0].path, 'r') as f: + overflow_json = json.load(f) + util.print_panel(self._json_summary(overflow_json, debug_file)) + if i >= max_num: + break + + def _json_summary(self, json_txt, debug_file): + res = [] + detail = {'task_id': -1} + if 'AI Core' in json_txt and json_txt['AI Core']['status'] > 0: + detail = json_txt['AI Core'] + res.append(' - [AI Core][Status:%s][TaskId:%s] %s' % ( + detail['status'], detail['task_id'], self._decode_ai_core_status(detail['status']))) + if 'DHA Atomic Add' in json_txt and json_txt['DHA Atomic Add']['status'] > 0: + detail = json_txt['DHA Atomic Add'] + res.append(' - [DHA Atomic Add][Status:%s][TaskId:%s] %s' % ( + detail['status'], detail['task_id'], self._decode_dha_atomic_add_status(detail['status']))) + if 'L2 Atomic Add' in json_txt and json_txt['L2 Atomic Add']['status'] > 0: + detail = json_txt['L2 Atomic Add'] + res.append(' - [L2 Atomic Add][Status:%s][TaskId:%s] %s' % ( + detail['status'], detail['task_id'], self._decode_l2_atomic_add_status(detail['status']))) + dump_file_info = self._find_dump_files_by_task_id(detail['task_id'], debug_file.dir_path) + res.append(' - First overflow file timestamp [%s] -' % debug_file.timestamp) + if dump_file_info is None: + self.log.warning("Can not find any dump file for debug file: %s, op task id: %s", debug_file.file_name, + detail['task_id']) + else: + dump_decode_files = self._decode_file(dump_file_info) + # sort input/output & index + sorted(dump_decode_files, key=lambda x: x.idx) + for anchor_type in ['input', 'output']: + for dump_decode_file in dump_decode_files: + if dump_decode_file.type != anchor_type: + continue + res.append(' ├─ %s' % dump_decode_file.file_name) + res.append(' └─ [yellow]%s[/yellow]' % util.gen_npy_info_txt(dump_decode_file.path)) + res.insert(0, '[green][%s][%s][/green] %s' % (dump_file_info.op_type, dump_file_info.task_id, + dump_file_info.op_name)) + return Constant.NEW_LINE.join(res) + + @staticmethod + def _decode_file(file_info, debug=False): + file_name = file_info.file_name + if debug: + decode_files = util.list_debug_decode_files(cfg.OVERFLOW_DECODE_DIR, file_name) + else: + decode_files = util.list_npu_dump_decode_files(cfg.OVERFLOW_DECODE_DIR, file_name) + if len(decode_files) == 0: + # decode info file + util.convert_dump_to_npy(file_info.path, cfg.OVERFLOW_DECODE_DIR) + if debug: + decode_files = util.list_debug_decode_files(cfg.OVERFLOW_DECODE_DIR, file_name) + else: + decode_files = util.list_npu_dump_decode_files(cfg.OVERFLOW_DECODE_DIR, file_name) + if len(decode_files) == 0: + raise PrecisionToolException("Decode overflow debug file: %s failed." % file_name) + decode_files = sorted(decode_files.values(), key=lambda x: x.timestamp) + return decode_files + + @staticmethod + def _find_dump_files_by_task_id(task_id, search_dir): + dump_files = util.list_npu_dump_files(search_dir) + dump_file_list = [item for item in dump_files.values() if item.op_type != 'Opdebug'] + dump_file_list = sorted(dump_file_list, key=lambda x: x.timestamp) + for dump_file in dump_file_list: + if dump_file.task_id == int(task_id): + return dump_file + return None + + def _decode_ai_core_status(self, status): + error_code = [] + if type(status) is not int: + return error_code + bin_status = ''.join(reversed(bin(status))) + prefix = '' + self.log.debug('Decode AI Core Overflow status:[%s]', hex(status)) + for i in range(len(bin_status)): + if bin_status[i] == '1': + error_code.append(AI_CORE_OVERFLOW_STATUS[hex(int('1' + prefix, 2))]) + prefix += '0' + return error_code + + def _decode_l2_atomic_add_status(self, status): + if type(status) is not int: + return 'status is not int.' + code, _ = self._sub_bin_code(status, 16, 18) + if code in L2_ATOMIC_ADD_STATUS: + return L2_ATOMIC_ADD_STATUS[code] + return 'Status invalid' + + def _decode_dha_atomic_add_status(self, status): + if type(status) is not int: + return 'status is not int.' + _, code = self._sub_bin_code(status, 8, 15) + if code in DHA_ATOMIC_ADD_STATUS: + return DHA_ATOMIC_ADD_STATUS[status] + return 'Status invalid' + + @staticmethod + def _sub_bin_code(status, start, end): + """ Get specific bit code from status in bin format + :param status: status num + :param start: start bit + :param end: end bit + :return: result in bin format and hex format + """ + bin_code = bin(status).replace('0b', '') + append_num = end + 1 - len(bin_code) + if append_num > 0: + bin_list = ['0'] * append_num + bin_list.append(bin_code) + bin_code = ''.join(bin_list) + bin_start = len(bin_code) - end - 1 + bin_end = len(bin_code) - start + bin_start = max(0, bin_start) + bin_code = bin_code[bin_start: bin_end] + return bin_code, hex(int(bin_code, 2)) + + diff --git a/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/precision_tool/lib/precision_tool.py b/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/precision_tool/lib/precision_tool.py new file mode 100644 index 0000000000000000000000000000000000000000..d301ffbc051a8b4cf7ca4130618475770d32195d --- /dev/null +++ b/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/precision_tool/lib/precision_tool.py @@ -0,0 +1,212 @@ +import argparse +import os +import time + +from lib.overflow import Overflow +from lib.dump_manager import DumpManager +from lib.graph_manager import GraphManager +from lib.compare import Compare +from lib.fusion import Fusion +from lib.util import util +from lib.constant import Constant +import config as cfg +from lib.precision_tool_exception import PrecisionToolException +from lib.precision_tool_exception import catch_tool_exception + + +class PrecisionTool(object): + def __init__(self): + """init""" + self.graph_manager = GraphManager() + self.overflow = Overflow() + self.dump_manager = DumpManager() + self.compare = Compare() + self.fusion = Fusion() + self.log = util.get_log() + + @catch_tool_exception + def prepare(self): + """prepare""" + util.create_dir(cfg.DATA_ROOT_DIR) + self.graph_manager.prepare() + self.dump_manager.prepare() + self.overflow.prepare() + self.fusion.prepare() + self.compare.prepare() + + @catch_tool_exception + def do_auto_check(self, argv): + """Auto check""" + parser = argparse.ArgumentParser() + parser.add_argument('-c', '--vector_compare', dest='vector_compare', help='Run vector compare process', + action='store_true') + parser.add_argument('-l', '--limit', dest='limit', type=int, help='limit', default=3) + args = parser.parse_args(argv) + # vector compare + if args.vector_compare: + self.do_vector_compare() + self.do_vector_compare_summary() + self.do_check_fusion() + self.do_check_overflow(args.limit) + self.do_check_cast() + self.do_check_graph_similarity() + + @catch_tool_exception + def do_check_overflow(self, limit=3): + """check overflow""" + self.overflow.check(limit) + + @catch_tool_exception + def do_check_cast(self): + self.graph_manager.check_cast() + + @catch_tool_exception + def do_check_dtype(self): + """Check input/output dtype""" + self.graph_manager.check_dtype() + + @catch_tool_exception + def do_check_fusion(self): + """print fusion info summary""" + self.fusion.check() + + @catch_tool_exception + def do_check_graph_similarity(self): + self.graph_manager.check_similarity() + + @catch_tool_exception + def do_vector_compare(self, argv=None): + """do vector compare""" + parser = argparse.ArgumentParser() + parser.add_argument('-lt', '--left', dest='lt', default=None, help='left path(npu dump path)') + parser.add_argument('-rt', '--right', dest='rt', default=None, help='right path(cpu/npu dump path)') + parser.add_argument('-g', '--graph', dest='graph', required=False, default=None, help='graph json file') + args = parser.parse_args() if argv is None else parser.parse_args(argv) + # 1. compare npu_debug0 - tf dump data (auto) + # 2. compare npu_debug0 - npu_debug1 dump data + # 3. compare dir - dir dump data + result_dir = os.path.join(cfg.VECTOR_COMPARE_PATH, time.strftime("%Y%m%d%H%M%S", time.localtime())) + if args.lt is None: + debug_0_dump_root = self.dump_manager.get_dump_root_dir(Constant.DEFAULT_DEBUG_ID) + if util.empty_dir(debug_0_dump_root): + raise PrecisionToolException("NPU debug_0 dump dir is empty, no files to compare.") + if not util.empty_dir(cfg.TF_DUMP_DIR): + self.log.info("Tf dump dir is not empty, will compare npu dump data with tf dump data.") + self.compare.npu_tf_vector_compare(self.graph_manager.get_graphs(Constant.DEFAULT_DEBUG_ID), + debug_0_dump_root, cfg.TF_DUMP_DIR, result_dir) + else: + self.log.warning("Tf dump dir is empty, maybe run [python3 precision_tool/cli.py tf_dump] to decode" + " tf debug data.") + debug_1_dump_root = self.dump_manager.get_dump_root_dir(Constant.NPU_DEBUG_ID_1) + if debug_1_dump_root is not None and not util.empty_dir(debug_1_dump_root): + self.log.info("NPU debug_1 dump dir is not empty, will compare two npu dump data.") + self.compare.npu_vector_compare(debug_0_dump_root, debug_1_dump_root) + else: + lh_path = args.lt + rh_path = args.rt + graph_json = args.graph + self.compare.vector_compare(lh_path, rh_path, result_dir, graph_json) + self.compare.vector_summary(result_dir) + + @catch_tool_exception + def do_vector_compare_summary(self, argv=None): + parser = argparse.ArgumentParser(description="show vector compare result summary.") + parser.add_argument('-f', '--file', dest='file', default=None, required=False, help='compare_result file/path') + parser.add_argument('-c', '--cos_sim', dest='cos_sim', type=float, help='cos_sim_threshold', default=0.98) + parser.add_argument('-l', '--limit', dest='limit', type=int, help='limit', default=3) + args = parser.parse_args() if argv is None else parser.parse_args(argv) + error_ops = self.compare.vector_summary(args.file, args.cos_sim, args.limit) + # parse error_ops + + @catch_tool_exception + def do_print_data(self, argv=None): + """print tensor data""" + parser = argparse.ArgumentParser() + parser.add_argument('-n', '--name', dest='name', default='', help='list by op name') + args = parser.parse_args() if argv is None else parser.parse_args(argv) + self.dump_manager.print_tensor(args.name, True) + + @catch_tool_exception + def do_list_nodes(self, argv): + """list op nodes in graph""" + parser = argparse.ArgumentParser() + parser.add_argument('-t', '--type', dest='type', default='', help='list by op type') + parser.add_argument('-n', '--name', dest='name', default='', help='list by op name') + parser.add_argument('-f', '--fusion', dest='fusion', default='', help='list by op fusion pass') + parser.add_argument('-k', '--kernel_name', dest='kernel_name', default='', help='list by op kernel_name') + args = parser.parse_args(argv) + self.graph_manager.print_op_list(args.type, args.name, args.fusion, args.kernel_name) + + @catch_tool_exception + def do_node_info(self, argv): + """Print op node info""" + parser = argparse.ArgumentParser() + parser.add_argument('-n', '--name', dest='name', default='', help='op name') + parser.add_argument('-g', '--graph', dest='graph', help='graph name') + parser.add_argument('-s', '--save', dest='save', type=int, default=0, + help='save subgraph, param gives the deep of subgraph') + args = parser.parse_args(argv) + # print graph op info + npu_ops, _ = self.graph_manager.get_ops(args.name, args.graph) + npu_op_summary, tf_op_summary = self.graph_manager.op_graph_summary(npu_ops) + npu_dump_summary, tf_dump_summary = self.dump_manager.op_dump_summary(npu_ops) + # merge graph/dump/compare info + for debug_id, graph_summary in npu_op_summary.items(): + for graph_name, summary_detail in graph_summary.items(): + summary_txt = [summary_detail] + if debug_id in npu_dump_summary and graph_name in npu_dump_summary[debug_id]: + summary_txt.append(npu_dump_summary[debug_id][graph_name]) + if tf_dump_summary is not None: + summary_txt.append(tf_dump_summary) + title = "[green](%s)[/green] %s" % (debug_id, graph_name) + util.print_panel(Constant.NEW_LINE.join(summary_txt), title) + if args.save != 0: + self.graph_manager.save_sub_graph(npu_ops, args.save) + + @catch_tool_exception + def do_compare_data(self, argv): + """compare two tensor""" + parser = argparse.ArgumentParser() + parser.add_argument('-n', '--name', dest='names', type=str, default=[], help='op name', nargs='+') + parser.add_argument('-p', '--print', dest='count', default=20, type=int, help='print err data num') + parser.add_argument('-s', '--save', dest='save', action='store_true', help='save data in txt format') + parser.add_argument('-al', '--atol', dest='atol', default=0.001, type=float, help='set rtol') + parser.add_argument('-rl', '--rtol', dest='rtol', default=0.001, type=float, help='set atol') + args = parser.parse_args(argv) + if len(args.names) != 2: + self.log.error("compare files should be 2.") + else: + self.compare.compare_data(args.names[0], args.names[1], args.save, args.rtol, args.atol, args.count) + + @catch_tool_exception + def do_list_dump(self, argv): + """List dump files""" + parser = argparse.ArgumentParser() + parser.add_argument('-t', '--type', dest='type', help='') + parser.add_argument('-n', '--name', dest='name') + self.dump_manager.list_dump(argv.dir, argv.name) + + @catch_tool_exception + def do_convert_npu_dump(self, argv): + parser = argparse.ArgumentParser() + parser.add_argument('-n', '--name', dest='name', help='op name') + parser.add_argument('-f', '--format', dest='format', default=None, required=False, help='target format') + parser.add_argument('-o', '--output', dest='output', required=False, default=None, help='output path') + args = parser.parse_args(argv) + self.dump_manager.convert_npu_dump(args.name, args.format, args.output) + + @catch_tool_exception + def do_convert_all_npu_dump(self): + self.dump_manager.decode_all_npu_dump() + + @catch_tool_exception + def check_graph_similarity(self): + """ Check graph similarity """ + + def single_cmd(self, argv): + cmd_func_map = {'compare': self.do_compare_data, + 'vector_compare': self.do_vector_compare} + if argv[1] in cmd_func_map: + func = cmd_func_map[argv[1]] + return func(argv[2:]) + raise PrecisionToolException("cmd %s is not supported or cmd should be run in interactive mode." % argv[1]) diff --git a/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/precision_tool/lib/precision_tool_exception.py b/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/precision_tool/lib/precision_tool_exception.py new file mode 100644 index 0000000000000000000000000000000000000000..02084770f2c8f391710f1835e9a422ddf25a624f --- /dev/null +++ b/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/precision_tool/lib/precision_tool_exception.py @@ -0,0 +1,24 @@ +# coding=utf-8 +import logging + + +class PrecisionToolException(Exception): + """ + Class for PrecisionTool Exception + """ + def __init__(self, error_info): + super(PrecisionToolException, self).__init__() + self.error_info = error_info + + +def catch_tool_exception(fuc): + def handle(*args, **kwargs): + log = logging.getLogger() + try: + return fuc(*args, **kwargs) + except PrecisionToolException as pte: + log.warning(pte.error_info) + except SystemExit: + # do not exit + log.debug("Exit") + return handle diff --git a/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/precision_tool/lib/pt_dump.py b/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/precision_tool/lib/pt_dump.py new file mode 100644 index 0000000000000000000000000000000000000000..155bd8c516519a714a4df3690457ba38c94b0d71 --- /dev/null +++ b/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/precision_tool/lib/pt_dump.py @@ -0,0 +1,19 @@ +# coding=utf-8 +import os +import re +import time +import sys +from lib.util import util +from lib.constant import Constant +import config as cfg +from lib.precision_tool_exception import catch_tool_exception +from lib.precision_tool_exception import PrecisionToolException + + +class PTDump(object): + def __init__(self): + self.log = util.get_log() + # self.h5_util = + + def prepare(self): + print("test") \ No newline at end of file diff --git a/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/precision_tool/lib/tf_dump.py b/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/precision_tool/lib/tf_dump.py new file mode 100644 index 0000000000000000000000000000000000000000..487c25b946e05a9b473af23cd2e8ae35fa914788 --- /dev/null +++ b/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/precision_tool/lib/tf_dump.py @@ -0,0 +1,122 @@ +# coding=utf-8 +import os +import re +import time +import sys +from lib.util import util +from lib.constant import Constant +import config as cfg +from lib.precision_tool_exception import catch_tool_exception +from lib.precision_tool_exception import PrecisionToolException + + +class TfDump(object): + def __init__(self, dump_root=cfg.TF_DUMP_DIR): + self.log = util.get_log() + self.dump_root = dump_root + self.dump_files = None + + def prepare(self): + if not os.path.exists(self.dump_root): + util.create_dir(self.dump_root) + self._parse_dump_files() + + def get_dump_files_by_op(self, op): + """Get cpu dump files by op""" + tf_files = {} + for output in op.outputs(): + if output.data_dump_origin_name() != '': + tf_files.update(self.get_dump_files_by_name(output.data_dump_origin_name())) + if len(tf_files) == 0: + tf_files.update(self.get_dump_files_by_name(op.name())) + return tf_files + + def get_dump_files_by_name(self, name, likely=False): + match_name = name.replace('/', '_') + if not likely: + match_name = match_name.replace('.', '_') + '\\.' + tf_files = {} + for f in self.dump_files: + if re.match(match_name, f): + tf_files[f] = self.dump_files[f] + return tf_files + + @catch_tool_exception + def op_dump_summary(self, op): + # cpu dump info + if op is None: + return '' + cpu_dump_txt = ['TfDumpOutput:'] + cpu_dump_files = self.get_dump_files_by_op(op) + for cpu_dump_file in cpu_dump_files.values(): + cpu_dump_txt.append(' -[green][%s][/green] %s' % (cpu_dump_file.idx, cpu_dump_file.file_name)) + cpu_dump_txt.append(' └─ [yellow]%s[/yellow]' % util.gen_npy_info_txt(cpu_dump_file.path)) + return Constant.NEW_LINE.join(cpu_dump_txt) + + def _parse_dump_files(self): + self.dump_files = util.list_cpu_dump_decode_files(self.dump_root) + + def run_tf_dbg_dump(self, cmd_line): + """Run tf train script to get dump data.""" + if os.path.exists(cfg.TF_DEBUG_DUMP_DIR) and len(os.listdir(cfg.TF_DEBUG_DUMP_DIR)) != 0: + self.log.info("TF offline debug path [%s] is not empty, will analyze it directly." % cfg.TF_DEBUG_DUMP_DIR) + elif cmd_line is not None: + self.log.info("Run command: %s" % cmd_line) + util.execute_command(cmd_line) + self.log.info("Run finish, start analyze TF dump.") + if not os.path.exists(cfg.TF_DEBUG_DUMP_DIR) or len(os.listdir(cfg.TF_DEBUG_DUMP_DIR)) == 0: + raise PrecisionToolException("Empty tf debug dir. %s" % cfg.TF_DEBUG_DUMP_DIR) + run_dirs = os.listdir(cfg.TF_DEBUG_DUMP_DIR) + run_dirs.sort() + # create dirs + util.create_dir(cfg.TF_DUMP_DIR) + util.create_dir(cfg.TMP_DIR) + # extra the last run dir + for run_dir in run_dirs: + time.sleep(1) + command = "%s -m tensorflow.python.debug.cli.offline_analyzer --ui_type readline --dump_dir %s" % ( + util.python, os.path.join(cfg.TF_DEBUG_DUMP_DIR, run_dir)) + self._do_run_tf_dbg_dump(command, 0) + + def _do_run_tf_dbg_dump(self, cmd_line, run_times=2): + """Run tf debug with pexpect, should set tf debug ui_type='readline'""" + try: + import pexpect + import readline + except ImportError as import_err: + self.log.error("Import failed with err:%s. You can run " + "'pip3 install pexpect gnureadline pyreadline' to fix it.", + import_err) + raise PrecisionToolException("Import module error.") + self.log.info("======< Auto run tf train process to dump data >======") + self.log.info("Send run times: %d", run_times) + tf_dbg = pexpect.spawn(cmd_line) + # tf_dbg.logfile = open(cfg.DUMP_FILES_CPU_LOG, 'wb') + tf_dbg.logfile = sys.stdout.buffer + for i in range(run_times): + tf_dbg.expect('tfdbg>', timeout=cfg.TF_DEBUG_TIMEOUT) + self.log.info("Process %d tf_debug.run", i + 1) + tf_dbg.sendline('run') + self.log.info("Generate tensor name file.") + tf_dbg.expect('tfdbg>', timeout=cfg.TF_DEBUG_TIMEOUT) + tf_dbg.sendline('lt > %s' % cfg.TF_TENSOR_NAMES) + tf_dbg.expect('tfdbg>', timeout=cfg.TF_DEBUG_TIMEOUT) + if not os.path.exists(cfg.TF_TENSOR_NAMES): + self.log.error("Failed to get tensor name in tf_debug.") + raise PrecisionToolException("Get tensor name in tf_debug failed.") + self.log.info("Save tensor name success. Generate tf dump commands from file: %s", cfg.TF_TENSOR_NAMES) + convert_cmd = "timestamp=" + str(int(time.time())) + "; cat " + cfg.TF_TENSOR_NAMES + \ + " | awk '{print \"pt\",$4,$4}'| awk '{gsub(\"/\", \"_\", $3); gsub(\":\", \".\", $3);" \ + "print($1,$2,\"-n 0 -w " + cfg.TF_DUMP_DIR + "/" + \ + "\"$3\".\"\"'$timestamp'\"\".npy\")}' > " + cfg.TF_TENSOR_DUMP_CMD + util.execute_command(convert_cmd) + if not os.path.exists(cfg.TF_TENSOR_DUMP_CMD): + self.log.error("Save tf dump cmd failed") + raise PrecisionToolException("Failed to generate tf dump command.") + self.log.info("Generate tf dump commands. Start run commands in file: %s", cfg.TF_TENSOR_DUMP_CMD) + for cmd in open(cfg.TF_TENSOR_DUMP_CMD): + self.log.debug(cmd.strip()) + tf_dbg.sendline(cmd.strip()) + tf_dbg.expect('tfdbg>', timeout=cfg.TF_DEBUG_TIMEOUT) + tf_dbg.sendline('exit') + self.log.info('Finish dump tf data') diff --git a/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/precision_tool/lib/tf_graph.py b/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/precision_tool/lib/tf_graph.py new file mode 100644 index 0000000000000000000000000000000000000000..b898c59ccf4f2c11d3c89a9630748520c0e8780b --- /dev/null +++ b/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/precision_tool/lib/tf_graph.py @@ -0,0 +1,45 @@ +# coding=utf-8 +import collections +import logging +import os +import config as cfg +from lib.util import util +from lib.precision_tool_exception import catch_tool_exception +from lib.precision_tool_exception import PrecisionToolException + +CKPT_META_SHUFFIX='.meta' + + +class TfGraph(object): + def __init__(self, graph_root=cfg.TF_GRAPH_DIR): + """""" + self.graph_root = graph_root + self.log = util.get_log() + self.op_list = collections.OrderedDict() + + @catch_tool_exception + def get_op_list(self, ckpt_path=None): + if self.op_list is None: + self._convert_ckpt_to_graph(ckpt_path) + return self.op_list + + def _convert_ckpt_to_graph(self, ckpt_path): + log_level = self.log.level + try: + self.log.setLevel('ERROR') + import tensorflow as tf + self.log.setLevel(log_level) + except ImportError as err: + self.log.setLevel(log_level) + raise PrecisionToolException("Import tensorflow failed.") + meta_files = util.list_cpu_graph_files(ckpt_path) + if len(meta_files) == 0: + raise PrecisionToolException("Can not find any ckpt meta files.") + file_list = sorted(meta_files.values(), key=lambda x: x['timestamp']) + ckpt_file = file_list[-1] + self.log.info("Find %d tf ckpt meta files, choose [%s]" % (len(meta_files), ckpt_file['file_name'])) + self.op_list = collections.OrderedDict() + saver = tf.train.import_meta_graph(ckpt_file['path'], clear_devices=True) + graph = tf.get_default_graph() + for op in graph.get_operations(): + self.op_list[op.name] = op diff --git a/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/precision_tool/lib/tool_object.py b/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/precision_tool/lib/tool_object.py new file mode 100644 index 0000000000000000000000000000000000000000..7412b6ceeacdf76fe816741fc9f22644d3099a14 --- /dev/null +++ b/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/precision_tool/lib/tool_object.py @@ -0,0 +1,10 @@ +# coding=utf-8 + + +class ToolObject(object): + _instance = None + + def __new__(cls, *args, **kwargs): + if not cls._instance: + cls._instance = super(ToolObject, cls).__new__(cls, *args, **kwargs) + return cls._instance diff --git a/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/precision_tool/lib/util.py b/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/precision_tool/lib/util.py new file mode 100644 index 0000000000000000000000000000000000000000..b81edae6a34928430456bc0c12a1d923a9c32ef1 --- /dev/null +++ b/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/precision_tool/lib/util.py @@ -0,0 +1,513 @@ +# coding=utf-8 +import csv +import re +import sys +import os +import shutil +import numpy as np +import logging +import subprocess +from lib.constant import Constant +from lib.precision_tool_exception import PrecisionToolException +from lib.precision_tool_exception import catch_tool_exception +from lib.file_desc import * +import config as cfg + +try: + from rich.traceback import install + from rich.panel import Panel + from rich.table import Table + from rich import print as rich_print + from rich.columns import Columns + install() +except ImportError as import_err: + install = None + Panel = None + Table = None + Columns = None + rich_print = print + print("Failed to import rich. some function may disable. Run 'pip3 install rich' to fix it.", + import_err) + +try: + import readline + readline.parse_and_bind('tab: complete') +except ImportError as import_error: + print("Unable to import module: readline. Run 'pip3 install gnureadline pyreadline' to fix it.") + +# patterns +OFFLINE_DUMP_PATTERN = r"^([A-Za-z0-9_-]+)\.([A-Za-z0-9_-]+)\.([0-9]+)\.?([0-9]+)?\.([0-9]{1,255})" +OFFLINE_DUMP_DECODE_PATTERN = \ + r"^([A-Za-z0-9_-]+)\.([A-Za-z0-9_-]+)\.([0-9]+)(\.[0-9]+)?\.([0-9]{1,255})\.([a-z]+)\.([0-9]{1,255})\.npy$" +OFFLINE_DUMP_CONVERT_PATTERN = \ + r"^([A-Za-z0-9_-]+)\.([A-Za-z0-9_-]+)\.([0-9]+)(\.[0-9]+)?\.([0-9]{1,255})" \ + r"\.([a-z]+)\.([0-9]{1,255})(\.[x0-9]+)?\.npy$" +OFFLINE_FILE_NAME = 'op_type.op_name.task_id(.stream_id).timestamp' +OP_DEBUG_NAME = 'OpDebug.Node_OpDebug.taskid.timestamp' +CPU_DUMP_DECODE_PATTERN = r"^([A-Za-z0-9_-]+)\.([0-9]+)(\.[0-9]+)?\.([0-9]{1,255})\.npy$" +CPU_FILE_DECODE_NAME = 'op_name.0(.0).timestamp.npy' +OP_DEBUG_PATTERN = r"Opdebug\.Node_OpDebug\.([0-9]+)(\.[0-9]+)?\.([0-9]{1,255})" +OP_DEBUG_DECODE_PATTERN = r"Opdebug\.Node_OpDebug\.([0-9]+)(\.[0-9]+)?\.([0-9]{1,255})\.([a-z]+)\.([0-9]{1,255})\.json" +VECTOR_COMPARE_RESULT_PATTERN = r"result_([0-9]{1,255})\.csv" +TIMESTAMP_DIR_PATTERN = '[0-9]{1,255}' +NUMPY_PATTERN = r".*\.npy$" +CSV_SHUFFIX = '.csv' +NUMPY_SHUFFIX = '.npy' +CKPT_META_SHUFFIX = r".*.meta$" +MAPPING_CSV = "mapping.csv" + + +class Util(object): + def __init__(self): + self.atc = None + self.ms_accu_cmp = None + logging.basicConfig(level=cfg.LOG_LEVEL, format="%(asctime)s (%(process)d) -[%(levelname)s]%(message)s", + datefmt="%Y-%m-%d %H:%M:%S") + self.log = logging.getLogger() + self.python = sys.executable + + def get_log(self): + return self.log + + def execute_command(self, cmd: str): + """ Execute shell command + :param cmd: command + :return: status code + """ + if cmd is None: + self.log.error("Command is None.") + return -1 + self.log.debug("[Run CMD]: %s", cmd) + complete_process = subprocess.run(cmd, shell=True) + return complete_process.returncode + + @staticmethod + def empty_dir(dir_path: str) -> bool: + """ Check if target dir is empty + :param dir_path: target dir + :return: bool + """ + if not os.path.exists(dir_path): + return True + if len(os.listdir(dir_path)) == 0: + return True + return False + + def convert_proto_to_json(self, src_file, dst_file): + """Convert GE proto graphs to json format. + command: atc --mode=5 --om=ge_proto_Build.txt --json=xxx.json + :param src_file: proto file + :param dst_file: output json file + :return: result json file + """ + if not os.path.exists(src_file): + raise PrecisionToolException("Source proto file %s not exist." % src_file) + # src_file = os.path.join(cfg.GRAPH_DIR_ALL, proto_file) + # json_file = proto_file + '.json' + # dst_file = os.path.join(cfg.GRAPH_DIR_BUILD, json_file) + if os.path.exists(dst_file) and os.path.getmtime(dst_file) > os.path.getmtime(src_file): + self.log.debug("GE graph build json already exist.") + return dst_file + cmd = '%s --mode=5 --om=%s --json=%s' % (self._get_atc(), src_file, dst_file) + self.execute_command(cmd) + if not os.path.isfile(dst_file): + raise PrecisionToolException("Convert GE build graph to json failed. can not find any json file.") + self.log.info('Finish convert [%s] build graph from proto to json format.', src_file) + return dst_file + + def convert_dump_to_npy(self, src_file, dst_path, data_format=None): + """Convert npu dump files to npy format. + :param src_file: src file + :param dst_path: dst path + :param data_format: target data format + :return: status code + """ + self.create_dir(dst_path) + format_cmd = '' if data_format is None else '-f %s' % data_format + cmd = '%s %s convert -d %s -out %s %s' % (self.python, self._get_ms_accu_cmp(), src_file, dst_path, format_cmd) + return self.execute_command(cmd) + + def compare_vector(self, npu_dump_dir, cpu_dump_dir, graph_json, result_path): + """Run compare vector command. + :param npu_dump_dir: npu dump data dir + :param cpu_dump_dir: cpu dump data dir + :param graph_json: graph json + :param result_path: result path + :return: status code + """ + self.create_dir(result_path) + if graph_json is None: + cmd = '%s %s compare -m %s -g %s -out %s' % ( + self.python, self._get_ms_accu_cmp(), npu_dump_dir, cpu_dump_dir, result_path) + else: + cmd = '%s %s compare -m %s -g %s -f %s -out %s' % ( + self.python, self._get_ms_accu_cmp(), npu_dump_dir, cpu_dump_dir, graph_json, result_path) + return self.execute_command(cmd) + + def list_dump_files(self, path, sub_path=''): + """List npu dump files in npu dump dir. + default only list the newest sub dir ordered by timestamp. set sub_path to specific other sub_path + :param path: dump path + :param sub_path: sub dir + :return: dump_files, parent_dirs + """ + parent_dirs = {} + dump_files = {} + newest_sub_path = self.get_newest_dir(path) if sub_path == '' else sub_path + dump_pattern = re.compile(OFFLINE_DUMP_PATTERN) + for dir_path, dir_names, file_names in os.walk(os.path.join(path, newest_sub_path), followlinks=True): + for name in file_names: + dump_match = dump_pattern.match(name) + if dump_match is None: + continue + dump_files[name] = self._gen_dump_file_info(name, dump_match, dir_path) + if dir_path not in parent_dirs: + parent_dirs[dir_path] = {} + parent_dirs[dir_path][name] = dump_files[name] + return dump_files, parent_dirs + + def parse_mapping_csv(self, path, pattern, extern_pattern=''): + """parse mapping csv in dump path""" + dump_files = {} + re_pattern = re.compile(pattern) + for dir_path, dir_names, file_names in os.walk(path, followlinks=True): + if MAPPING_CSV not in file_names: + continue + mapping = self.read_csv(os.path.join(dir_path, MAPPING_CSV)) + for item in mapping: + src_file = os.path.abspath(os.path.join(dir_path, item[0])) + if not os.path.isfile(src_file): + self.log.warning("Can not find file %s in mapping.csv, dir: %s.", item[0], dir_path) + continue + match = re_pattern.match(item[1]) + if match is None: + self.log.warning("file name [%s] in mapping.csv is invalid.", item[1]) + continue + file_desc = self._gen_dump_file_info(item[0], match, dir_path) + dst_file_name = '.'.join([file_desc.op_type, file_desc.file_name, str(file_desc.task_id), + str(file_desc.stream_id), str(file_desc.timestamp)]) + dst_file = os.path.abspath(os.path.join(dir_path, dst_file_name)) + if not os.path.islink(src_file): + os.rename(src_file, dst_file) + os.symlink(dst_file, src_file) + file_desc.path = dst_file + file_desc.file_name = dst_file_name + dump_files[item[1]] = file_desc + return dump_files + + def list_npu_dump_files(self, path, extern_pattern=''): + npu_dump_files = self._list_file_with_pattern(path, OFFLINE_DUMP_PATTERN, extern_pattern, + self._gen_dump_file_info) + npu_dump_files.update(self.parse_mapping_csv(path, OFFLINE_DUMP_PATTERN, extern_pattern)) + return npu_dump_files + + def list_ge_graph_files(self, path, extern_pattern=''): + return self._list_file_with_pattern(path, Constant.Pattern.GE_PROTO_GRAPH_PATTERN, extern_pattern, + self._gen_build_graph_file_info) + + def list_npu_dump_decode_files(self, path, extern_pattern=''): + return self._list_file_with_pattern(path, OFFLINE_DUMP_DECODE_PATTERN, extern_pattern, + self._gen_npu_dump_decode_file_info) + + def list_debug_decode_files(self, path, extern_pattern=''): + return self._list_file_with_pattern(path, OP_DEBUG_DECODE_PATTERN, extern_pattern, + self._gen_overflow_debug_decode_file_info) + + def list_cpu_dump_decode_files(self, path, extern_pattern=''): + return self._list_file_with_pattern(path, CPU_DUMP_DECODE_PATTERN, extern_pattern, + self._gen_cpu_dump_decode_file_info) + + def list_cpu_graph_files(self, path, extern_pattern=''): + return self._list_file_with_pattern(path, CKPT_META_SHUFFIX, extern_pattern, + self._gen_cpu_graph_files_info) + + def list_vector_compare_result_files(self, path, extern_pattern=''): + return self._list_file_with_pattern(path, VECTOR_COMPARE_RESULT_PATTERN, extern_pattern, + self._gen_vector_compare_result_file_info) + + def list_npu_dump_convert_files(self, path, extern_pattern=''): + return self._list_file_with_pattern(path, OFFLINE_DUMP_CONVERT_PATTERN, extern_pattern, + self._gen_npu_dump_convert_file_info) + + def list_numpy_files(self, path, extern_pattern=''): + return self._list_file_with_pattern(path, NUMPY_PATTERN, extern_pattern, + self._gen_numpy_file_info) + + def create_dir(self, path): + """Create dir if not exist + :param path: path + :return: bool + """ + if os.path.exists(path): + return True + try: + os.makedirs(path, mode=0o700) + except OSError as err: + self.log.error("Failed to create %s. %s", path, str(err)) + return False + return True + + def clear_dir(self, path: str, pattern=''): + """Clear dir with pattern (file/path name match pattern will be removed) + :param path: path + :param pattern: pattern + :return: None + """ + if not os.path.exists(path): + return + try: + for f in os.listdir(path): + if not re.match(pattern, f): + continue + file_path = os.path.join(path, f) + if os.path.isfile(file_path): + os.remove(file_path) + elif os.path.isdir(file_path): + shutil.rmtree(file_path) + except OSError as err: + self.log.error("Failed to remove %s. %s", path, str(err)) + + @staticmethod + def npy_info(source_data): + """Get npy information + :param source_data: npy path + :return: (shape, dtype) + """ + if isinstance(source_data, str): + if not str(source_data).endswith(NUMPY_SHUFFIX): + raise PrecisionToolException("Npy file [%s] is invalid" % source_data) + data = np.load(source_data, allow_pickle=True) + elif isinstance(source_data, np.ndarray): + data = source_data + else: + raise PrecisionToolException("Invalid source data:%s" % source_data) + if np.size(data) == 0: + raise PrecisionToolException("Empty source data:%s" % source_data) + return data.shape, data.dtype, data.max(), data.min(), data.mean() + + @catch_tool_exception + def gen_npy_info_txt(self, source_data): + """ Generate numpy info txt. + :param source_data: source path or np.ndarray + :return: txt + """ + try: + shape, dtype, max_data, min_data, mean = self.npy_info(source_data) + return '[Shape: %s] [Dtype: %s] [Max: %s] [Min: %s] [Mean: %s]' % (shape, dtype, max_data, min_data, mean) + except PrecisionToolException: + return '' + + def print_npy_summary(self, path, file_name, is_convert=False, extern_content=''): + """Print summary of npy data + :param path: file path + :param file_name: file name + :param is_convert: if convert to txt file + :param extern_content: extern content append to the summary + :return: None + """ + target_file = os.path.join(path, file_name) + if not os.path.exists(target_file): + raise PrecisionToolException("File [%s] not exist" % target_file) + data = np.load(target_file, allow_pickle=True) + table = self.create_table('', ['Index', 'Data']) + flatten_data = data.flatten() + for i in range(min(16, int(np.ceil(flatten_data.size / 8)))): + last_idx = min(flatten_data.size, i*8+8) + table.add_row(str(i * 8), ' '.join(flatten_data[i*8: last_idx].astype('str').tolist())) + summary = ['[yellow]%s[/yellow]' % self.gen_npy_info_txt(data), 'Path: %s' % target_file] + if is_convert: + summary.append('TxtFile: %s.txt' % target_file) + if extern_content != '': + summary.append('%s' % extern_content) + self.print_panel(self.create_columns([table, Constant.NEW_LINE.join(summary)]), file_name) + if is_convert: + self.save_npy_to_txt(data, target_file + '.txt') + + def save_npy_to_txt(self, src_file, dst_file='', align=0): + """save numpy file to txt file. + default data will be aligned to the last axis of data.shape + :param src_file: src file name + :param dst_file: dst file name + :param align: data align + :return: None + """ + if dst_file == '': + dst_file = src_file + '.txt' + if os.path.exists(dst_file): + self.log.debug("Dst file %s exists, will not save new one.", dst_file) + return + if isinstance(src_file, str): + data = np.load(src_file, allow_pickle=True) + elif isinstance(src_file, np.ndarray): + data = src_file + else: + raise PrecisionToolException("invalid src_file: %s", src_file) + shape = data.shape + data = data.flatten() + if align == 0: + if len(shape) == 0: + align = 1 + else: + align = shape[-1] + elif data.size % align != 0: + pad_array = np.zeros((align - data.size % align,)) + data = np.append(data, pad_array) + np.savetxt(dst_file, data.reshape((-1, align)), delimiter=' ', fmt='%g') + + def read_csv(self, path): + """Read csv file to list. + :param path: csv file path + :return: list + """ + if not str(path).endswith(CSV_SHUFFIX): + self.log.error("csv path [%s] is invalid", path) + return + rows = [] + with open(path) as f: + csv_handle = csv.reader(f) + for row in csv_handle: + rows.append(row) + return rows + + @staticmethod + def print(content): + rich_print(content) + + @staticmethod + def create_table(title, columns): + if Table is None: + raise PrecisionToolException("No rich module error.") + table = Table(title=title) + for column_name in columns: + table.add_column(column_name, overflow='fold') + return table + + @staticmethod + def create_columns(content): + if Columns is None: + raise PrecisionToolException("No rich module error.") + return Columns(content) + + def print_panel(self, content, title='', fit=True): + """ Print panel. + :param content: content + :param title: title + :param fit: if panel size fit the content + :return:Node + """ + if Panel is None: + print(content) + return + if fit: + self.print(Panel.fit(content, title=title)) + else: + self.print(Panel(content, title=title)) + + @staticmethod + def _detect_file(file_name, root_dir): + """Find file in root dir""" + result = [] + for dir_path, dir_names, file_names in os.walk(root_dir, followlinks=True): + for name in file_names: + if re.match(file_name, name): + result.append(os.path.join(dir_path, name)) + return result + + def _detect_file_if_not_exist(self, target_file): + """Find specific file in cmd root path""" + self.log.info("Try to auto detect file with name: %s.", target_file) + res = self._detect_file(target_file, cfg.CMD_ROOT_PATH) + if len(res) == 0: + raise PrecisionToolException("Cannot find any file named %s in dir %s" % (target_file, cfg.CMD_ROOT_PATH)) + self.log.info("Detect [%s] success. %s", target_file, res) + return res[0] + + def _get_atc(self): + if self.atc is None: + self.atc = self._detect_file_if_not_exist('^atc$') + return self.atc + + def _get_ms_accu_cmp(self): + if self.ms_accu_cmp is None: + self.ms_accu_cmp = self._detect_file_if_not_exist(cfg.MS_ACCU_CMP) + return self.ms_accu_cmp + + def get_newest_dir(self, path: str): + """Find the newest subdir in specific path, subdir should named by timestamp.""" + if not os.path.isdir(path): + self.log.warning("Path [%s] not exists", path) + return '' + paths = os.listdir(path) + sub_paths = [] + for p in paths: + if re.match(TIMESTAMP_DIR_PATTERN, p): + sub_paths.append(p) + if len(sub_paths) == 0: + self.log.debug("Path [%s] has no timestamp dirs.", path) + return '' + newest_sub_path = sorted(sub_paths)[-1] + self.log.info("Sub path num:[%d]. Dirs[%s], choose[%s]", len(sub_paths), str(sub_paths), newest_sub_path) + return newest_sub_path + + @staticmethod + def _list_file_with_pattern(path, pattern, extern_pattern, gen_info_func): + if path is None or not os.path.exists(path): + raise PrecisionToolException("Path %s not exist." % path) + file_list = {} + re_pattern = re.compile(pattern) + for dir_path, dir_names, file_names in os.walk(path, followlinks=True): + for name in file_names: + match = re_pattern.match(name) + if match is None: + continue + if extern_pattern != '' and not re.match(extern_pattern, name): + continue + file_list[name] = gen_info_func(name, match, dir_path) + return file_list + + @staticmethod + def _gen_numpy_file_info(name, math, dir_path): + return FileDesc(name, dir_path) + + @staticmethod + def _gen_build_graph_file_info(name, match, dir_path): + return BuildGraphFileDesc(name, dir_path, -1, int(match.group(1)), match.group(2)) + + @staticmethod + def _gen_dump_file_info(name, match, dir_path): + return NpuDumpFileDesc(name, dir_path, int(match.groups()[-1]), op_name=match.group(2), op_type=match.group(1), + task_id=int(match.group(3)), stream_id=match.group(4)) + + @staticmethod + def _gen_npu_dump_decode_file_info(name, match, dir_path): + return DumpDecodeFileDesc(name, dir_path, int(match.groups()[-3]), op_name=match.group(2), + op_type=match.group(1), task_id=int(match.group(3)), + anchor_type=match.groups()[-2], anchor_idx=int(match.groups()[-1])) + + @staticmethod + def _gen_cpu_dump_decode_file_info(name, match, dir_path): + return DumpDecodeFileDesc(name, dir_path, -1, op_name=match.group(1), op_type='', task_id=0, + anchor_type='output', anchor_idx=int(match.group(2))) + + @staticmethod + def _gen_cpu_graph_files_info(name, match, dir_path): + return FileDesc(name, dir_path, -1) + + @staticmethod + def _gen_overflow_debug_decode_file_info(name, match, dir_path): + return DumpDecodeFileDesc(name, dir_path, int(match.groups()[-3]), op_name='Node_OpDebug', op_type='Opdebug', + task_id=int(match.group(1)), anchor_type=match.groups()[-2], + anchor_idx=int(match.groups()[-1])) + + @staticmethod + def _gen_vector_compare_result_file_info(name, match, dir_path): + return FileDesc(name, dir_path, int(match.group(1))) + + @staticmethod + def _gen_npu_dump_convert_file_info(name, match, dir_path): + return DumpDecodeFileDesc(name, dir_path, int(match.groups()[-4]), op_name=match.group(2), + op_type=match.group(1), task_id=int(match.group(3)), anchor_type=match.groups()[-3], + anchor_idx=int(match.groups()[-2])) + + +util = Util() diff --git a/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/precision_tool/tf_config.py b/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/precision_tool/tf_config.py new file mode 100644 index 0000000000000000000000000000000000000000..e727238b41e143f129cae69b65a18d6bd7cd9ef5 --- /dev/null +++ b/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/precision_tool/tf_config.py @@ -0,0 +1,219 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# coding=utf-8 +import os +import random +from tensorflow.core.protobuf import config_pb2 +from tensorflow.core.protobuf.rewriter_config_pb2 import RewriterConfig +from tensorflow.python import debug as tf_debug +import tensorflow as tf +from . import config as cfg + +FLAG_DUMP_GE_GRAPH = 'DUMP_GE_GRAPH' +FLAG_DUMP_GRAPH_LEVEL = 'DUMP_GRAPH_LEVEL' +FLAG_DUMP_GRAPH_PATH = 'DUMP_GRAPH_PATH' + +# Fusion switch file path +FUSION_SWITCH_FILE = os.path.join(os.path.dirname(__file__), 'fusion_switch.cfg') +FUSION_OFF_FILE = os.path.join(os.path.dirname(__file__), 'fusion_off.cfg') + +DEFAULT_OP_DEBUG_DIR = cfg.DEFAULT_OP_DEBUG_DIR + + +def seed_everything(seed=cfg.DUMP_SEED): + # set random seed + os.environ['PYTHONHASHSEED'] = str(seed) + random.seed(seed) + tf.random.set_random_seed(seed) + print("[PrecisionTool] Set Tensorflow random seed to %d success." % seed) + try: + import numpy as np + np.random.seed(seed) + print("[PrecisionTool] Set numpy random seed to %d success." % seed) + except ImportError as err: + np = None + print("[PrecisionTool] No numpy module.", err) + + +# set global random seed +seed_everything() + + +def sess_dump(sess): + """wrapper session with dumping debug wrapper. + In session run mode. Use sess=sess_dump(sess) + :param sess: origin session + :return: Session + """ + _init() + return tf_debug.DumpingDebugWrapperSession(sess, cfg.TF_DEBUG_DUMP_DIR) + + +def estimator_dump(): + """In estimator mode. estim_spec = tf.estimator.EstimatorSpec(traing_hooks=[estimator_dump()]) + :return: + """ + _init() + return tf_debug.DumpingDebugHook(cfg.TF_DEBUG_DUMP_DIR) + + +def estimator_dump_config(action=None): + """return DumpConfig. + In estimator mode. set dump_config in NPURunConfig(). + exp. config = NPURunConfig(dump_config=estimator_dum_config(), session_config=session_config) + :return: DumpConfig + """ + from npu_bridge.npu_init import DumpConfig + _init() + if _is_overflow(action): + config = DumpConfig(enable_dump_debug=True, dump_path=cfg.NPU_OVERFLOW_DUMP_DIR, dump_mode="all") + elif _is_dump(action): + config = DumpConfig(enable_dump=True, dump_path=cfg.DEFAULT_NPU_DUMP_DIR, dump_step=cfg.TF_DUMP_STEP, + dump_mode="all") + else: + config = DumpConfig() + return config + + +def session_dump_config(session_config=None, action=None): + """ + In TF session mode. set dump_config in session_config. + exp. config = session_dump_config() + config.[set your own configs] + with tf.Session(config=config) as sess: + sess.run(_) + tf_debug.LocalCLIDebugWrapperSession(sess=sess, ui_type="readline") + :param session_config: original session config + :param action: if set action, no need to start app with cli wrapper + :return: config_pb2.ConfigProto + """ + if ((not isinstance(session_config, config_pb2.ConfigProto)) and + (not issubclass(type(session_config), config_pb2.ConfigProto))): + session_config = config_pb2.ConfigProto() + custom_op = None + for existed_custom_op in session_config.graph_options.rewrite_options.custom_optimizers: + if existed_custom_op.name == 'NpuOptimizer': + custom_op = existed_custom_op + if custom_op is None: + custom_op = session_config.graph_options.rewrite_options.custom_optimizers.add() + custom_op.name = 'NpuOptimizer' + custom_op.parameter_map['use_off_line'].b = True + update_custom_op(custom_op, action) + session_config.graph_options.rewrite_options.remapping = RewriterConfig.OFF + return session_config + + +def update_custom_op(custom_op, action=None): + """Update custom_op + :param custom_op: origin custom op + :param action: dump | overflow | fusion_off | fusion_switch + :return: + """ + _init() + custom_op.parameter_map['debug_dir'].s = tf.compat.as_bytes(cfg.DEFAULT_OP_DEBUG_DIR) + if _is_overflow(action): + custom_op.parameter_map['enable_dump_debug'].b = True + custom_op.parameter_map['dump_debug_mode'].s = tf.compat.as_bytes("all") + custom_op.parameter_map['dump_path'].s = tf.compat.as_bytes(cfg.NPU_OVERFLOW_DUMP_DIR) + custom_op.parameter_map['op_debug_level'].i = cfg.OP_DEBUG_LEVEL + elif _is_dump(action): + custom_op.parameter_map['enable_dump'].b = True + custom_op.parameter_map['dump_mode'].s = tf.compat.as_bytes("all") + custom_op.parameter_map['dump_path'].s = tf.compat.as_bytes(cfg.DEFAULT_NPU_DUMP_DIR) + custom_op.parameter_map['op_debug_level'].i = cfg.OP_DEBUG_LEVEL + custom_op.parameter_map['dump_step'].s = tf.compat.as_bytes(cfg.TF_DUMP_STEP) + if _is_fusion_off(action): + custom_op.parameter_map['fusion_switch_file'].s = tf.compat.as_bytes(FUSION_OFF_FILE) + print("[PrecisionTool] Set fusion switch file: ", FUSION_OFF_FILE) + elif _is_fusion_switch(action): + custom_op.parameter_map['fusion_switch_file'].s = tf.compat.as_bytes(FUSION_SWITCH_FILE) + print("[PrecisionTool] Set fusion switch file: ", FUSION_SWITCH_FILE) + return custom_op + + +def _init(): + _create_dir(cfg.DEFAULT_OP_DEBUG_DIR) + _create_dir(cfg.NPU_OVERFLOW_DUMP_DIR) + _create_dir(cfg.DEFAULT_NPU_DUMP_DIR) + _create_dir(cfg.DEFAULT_NPU_GRAPH_DIR) + _set_dump_graph_flags() + + +def _create_dir(path): + """ create dir """ + if os.path.exists(path): + return + try: + os.makedirs(path, mode=0o700) + except OSError as err_msg: + print("[PrecisionTool] Failed to create %s. %s" % (path, str(err_msg))) + + +def _unset_dump_graph_flags(): + if FLAG_DUMP_GE_GRAPH in os.environ: + del os.environ[FLAG_DUMP_GE_GRAPH] + if FLAG_DUMP_GRAPH_LEVEL in os.environ: + del os.environ[FLAG_DUMP_GRAPH_LEVEL] + if FLAG_DUMP_GRAPH_PATH in os.environ: + del os.environ[FLAG_DUMP_GRAPH_PATH] + + +def _set_dump_graph_flags(): + os.environ[FLAG_DUMP_GE_GRAPH] = str(cfg.DUMP_GE_GRAPH_VALUE) + os.environ[FLAG_DUMP_GRAPH_LEVEL] = str(cfg.DUMP_GRAPH_LEVEL_VALUE) + os.environ[FLAG_DUMP_GRAPH_PATH] = cfg.DEFAULT_NPU_GRAPH_DIR + + +def _is_dump(action): + if action is not None: + return 'dump' in action + if cfg.PRECISION_TOOL_DUMP_FLAG in os.environ and os.environ[cfg.PRECISION_TOOL_DUMP_FLAG] == 'True': + print("[PrecisionTool] enable npu dump >======") + return True + return False + + +def _is_overflow(action): + if action is not None: + return 'overflow' in action + if cfg.PRECISION_TOOL_OVERFLOW_FLAG in os.environ and os.environ[cfg.PRECISION_TOOL_OVERFLOW_FLAG] == 'True': + print("[PrecisionTool] enable npu overflow >======") + return True + return False + + +def _is_fusion_off(action): + if action is not None: + return 'fusion_off' in action + return False + + +def _is_fusion_switch(action): + if action is not None: + return 'fusion_switch' in action + return False diff --git a/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/test/output/.DS_Store b/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/test/output/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..76567042224a5ba158e66f3f5fa90ef5665ae91f Binary files /dev/null and b/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/test/output/.DS_Store differ diff --git a/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/test/output/2/train_2.log b/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/test/output/2/train_2.log new file mode 100644 index 0000000000000000000000000000000000000000..796be1b7d6b48e90e2f6cb54e273e37479de7706 --- /dev/null +++ b/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/test/output/2/train_2.log @@ -0,0 +1,48 @@ +WARNING:tensorflow:From /home/HwHiAiUser/wubo/vit/vit-ckpt/precision_tool/tf_config.py:25: The name tf.random.set_random_seed is deprecated. Please use tf.compat.v1.random.set_random_seed instead. + +WARNING:tensorflow:From /usr/local/Ascend/tfplugin/latest/tfplugin/python/site-packages/npu_bridge/estimator/npu/npu_optimizer.py:127: The name tf.train.Optimizer is deprecated. Please use tf.compat.v1.train.Optimizer instead. + +WARNING:tensorflow:From /usr/local/python3.7.5/lib/python3.7/site-packages/tensorflow_core/python/ops/resource_variable_ops.py:1630: calling BaseResourceVariable.__init__ (from tensorflow.python.ops.resource_variable_ops) with constraint is deprecated and will be removed in a future version. +Instructions for updating: +If using Keras pass *_constraint arguments to layers. +WARNING:tensorflow:From /usr/local/python3.7.5/lib/python3.7/site-packages/tensorflow_core/python/data/util/random_seed.py:58: where (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version. +Instructions for updating: +Use tf.where in 2.0, which has the same broadcast rule as np.where +WARNING:tensorflow:From ./vit_allpipeline_fusion_accelerate.py:124: The name tf.train.MomentumOptimizer is deprecated. Please use tf.compat.v1.train.MomentumOptimizer instead. + +WARNING:tensorflow:From /usr/local/Ascend/tfplugin/latest/tfplugin/python/site-packages/npu_bridge/estimator/npu/npu_loss_scale_optimizer.py:58: The name tf.get_default_graph is deprecated. Please use tf.compat.v1.get_default_graph instead. + +WARNING:tensorflow:From ./vit_allpipeline_fusion_accelerate.py:142: The name tf.ConfigProto is deprecated. Please use tf.compat.v1.ConfigProto instead. + +WARNING:tensorflow:From ./vit_allpipeline_fusion_accelerate.py:156: The name tf.Session is deprecated. Please use tf.compat.v1.Session instead. + +2022-05-15 09:48:43.822834: W tensorflow/core/platform/profile_utils/cpu_utils.cc:98] Failed to find bogomips in /proc/cpuinfo; cannot determine CPU frequency +2022-05-15 09:48:43.828896: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x458aa510 initialized for platform Host (this does not guarantee that XLA will be used). Devices: +2022-05-15 09:48:43.828943: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Host, Default Version +WARNING:tensorflow:From ./vit_allpipeline_fusion_accelerate.py:157: The name tf.global_variables_initializer is deprecated. Please use tf.compat.v1.global_variables_initializer instead. + +[PrecisionTool] Set Tensorflow random seed to 0 success. +[PrecisionTool] Set numpy random seed to 0 success. +load dataset ================================= +load dataset ================================= +[PrecisionTool] Set fusion switch file: /home/HwHiAiUser/wubo/vit/vit-ckpt/precision_tool/fusion_off.cfg +2022-05-15 09:48:53.112488: I /home/phisik3/jenkins/workspace/work_code/tmp/host-prefix/src/host-build/asl/tfadaptor/CMakeFiles/tf_adapter.dir/compiler_depend.ts:805] The model has been compiled on the Ascend AI processor, current graph id is: 1 +WARNING:tensorflow:From ./vit_allpipeline_fusion_accelerate.py:160: The name tf.train.Saver is deprecated. Please use tf.compat.v1.train.Saver instead. + +2022-05-15 09:49:04.981514: I /home/phisik3/jenkins/workspace/work_code/tmp/host-prefix/src/host-build/asl/tfadaptor/CMakeFiles/tf_adapter.dir/compiler_depend.ts:805] The model has been compiled on the Ascend AI processor, current graph id is: 11 +2022-05-15 09:49:25.298880: I /home/phisik3/jenkins/workspace/work_code/tmp/host-prefix/src/host-build/asl/tfadaptor/CMakeFiles/tf_adapter.dir/compiler_depend.ts:805] The model has been compiled on the Ascend AI processor, current graph id is: 21 +2022-05-15 09:53:30.029074: I /home/phisik3/jenkins/workspace/work_code/tmp/host-prefix/src/host-build/asl/tfadaptor/CMakeFiles/tf_adapter.dir/compiler_depend.ts:805] The model has been compiled on the Ascend AI processor, current graph id is: 31 +============ start load pretrained model ======================================= +============ load success 6.5804 ===================== +epoch:1 step: 0 , loss: 0.0681 , time: 113.6311 acc: 1.0000 +epoch:1 step: 1 , loss: 0.2198 , time: 15.5381 acc: 0.7500 +epoch:1 step: 2 , loss: 0.0095 , time: 15.2189 acc: 1.0000 +epoch:1 step: 3 , loss: 0.0476 , time: 22.9079 acc: 1.0000 +epoch:1 step: 4 , loss: 0.0578 , time: 15.2788 acc: 1.0000 +epoch:1 step: 5 , loss: 0.6772 , time: 15.5429 acc: 0.7500 +epoch:1 step: 6 , loss: 0.0049 , time: 15.0698 acc: 1.0000 +epoch:1 step: 7 , loss: 0.1065 , time: 15.2203 acc: 1.0000 +epoch:1 step: 8 , loss: 0.1218 , time: 15.2791 acc: 1.0000 +epoch:1 step: 9 , loss: 0.0144 , time: 15.3114 acc: 1.0000 +Train ACC: 0.9500 +Final Average Distances : 0.871 diff --git a/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/test/output/2/vit-ckpt_bs2_1p_acc.log b/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/test/output/2/vit-ckpt_bs2_1p_acc.log new file mode 100644 index 0000000000000000000000000000000000000000..73ffd2a7fa0b518ed8e4614b6a626895b28e4790 --- /dev/null +++ b/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/test/output/2/vit-ckpt_bs2_1p_acc.log @@ -0,0 +1,6 @@ +Network = vit-ckpt +RankSize = 1 +BatchSize = 2 +DeviceType = aarch64 +CaseName = vit-ckpt_bs2_1p_acc +E2ETrainingTime = 584 diff --git a/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/test/output/3/train_3.log b/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/test/output/3/train_3.log new file mode 100644 index 0000000000000000000000000000000000000000..5b6c1eb5e109145f48ca56dbef4829881cf67785 --- /dev/null +++ b/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/test/output/3/train_3.log @@ -0,0 +1,38 @@ +WARNING:tensorflow:From /home/HwHiAiUser/wubo/vit/vit-ckpt/precision_tool/tf_config.py:25: The name tf.random.set_random_seed is deprecated. Please use tf.compat.v1.random.set_random_seed instead. + +WARNING:tensorflow:From /usr/local/Ascend/tfplugin/latest/tfplugin/python/site-packages/npu_bridge/estimator/npu/npu_optimizer.py:127: The name tf.train.Optimizer is deprecated. Please use tf.compat.v1.train.Optimizer instead. + +WARNING:tensorflow:From /usr/local/python3.7.5/lib/python3.7/site-packages/tensorflow_core/python/ops/resource_variable_ops.py:1630: calling BaseResourceVariable.__init__ (from tensorflow.python.ops.resource_variable_ops) with constraint is deprecated and will be removed in a future version. +Instructions for updating: +If using Keras pass *_constraint arguments to layers. +WARNING:tensorflow:From /usr/local/python3.7.5/lib/python3.7/site-packages/tensorflow_core/python/data/util/random_seed.py:58: where (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version. +Instructions for updating: +Use tf.where in 2.0, which has the same broadcast rule as np.where +WARNING:tensorflow:From ./vit_allpipeline_fusion_accelerate.py:124: The name tf.train.MomentumOptimizer is deprecated. Please use tf.compat.v1.train.MomentumOptimizer instead. + +WARNING:tensorflow:From /usr/local/Ascend/tfplugin/latest/tfplugin/python/site-packages/npu_bridge/estimator/npu/npu_loss_scale_optimizer.py:58: The name tf.get_default_graph is deprecated. Please use tf.compat.v1.get_default_graph instead. + +WARNING:tensorflow:From ./vit_allpipeline_fusion_accelerate.py:142: The name tf.ConfigProto is deprecated. Please use tf.compat.v1.ConfigProto instead. + +WARNING:tensorflow:From ./vit_allpipeline_fusion_accelerate.py:156: The name tf.Session is deprecated. Please use tf.compat.v1.Session instead. + +2022-05-15 08:49:19.650985: W tensorflow/core/platform/profile_utils/cpu_utils.cc:98] Failed to find bogomips in /proc/cpuinfo; cannot determine CPU frequency +2022-05-15 08:49:19.657272: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x204c59a0 initialized for platform Host (this does not guarantee that XLA will be used). Devices: +2022-05-15 08:49:19.657332: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Host, Default Version +WARNING:tensorflow:From ./vit_allpipeline_fusion_accelerate.py:157: The name tf.global_variables_initializer is deprecated. Please use tf.compat.v1.global_variables_initializer instead. + +[PrecisionTool] Set Tensorflow random seed to 0 success. +[PrecisionTool] Set numpy random seed to 0 success. +load dataset ================================= +load dataset ================================= +[PrecisionTool] Set fusion switch file: /home/HwHiAiUser/wubo/vit/vit-ckpt/precision_tool/fusion_off.cfg +2022-05-15 08:49:28.988384: I /home/phisik3/jenkins/workspace/work_code/tmp/host-prefix/src/host-build/asl/tfadaptor/CMakeFiles/tf_adapter.dir/compiler_depend.ts:805] The model has been compiled on the Ascend AI processor, current graph id is: 1 +WARNING:tensorflow:From ./vit_allpipeline_fusion_accelerate.py:160: The name tf.train.Saver is deprecated. Please use tf.compat.v1.train.Saver instead. + +2022-05-15 08:49:40.726232: I /home/phisik3/jenkins/workspace/work_code/tmp/host-prefix/src/host-build/asl/tfadaptor/CMakeFiles/tf_adapter.dir/compiler_depend.ts:805] The model has been compiled on the Ascend AI processor, current graph id is: 11 +2022-05-15 08:50:00.871987: I /home/phisik3/jenkins/workspace/work_code/tmp/host-prefix/src/host-build/asl/tfadaptor/CMakeFiles/tf_adapter.dir/compiler_depend.ts:805] The model has been compiled on the Ascend AI processor, current graph id is: 21 +2022-05-15 08:51:43.736154: I /home/phisik3/jenkins/workspace/work_code/tmp/host-prefix/src/host-build/asl/tfadaptor/CMakeFiles/tf_adapter.dir/compiler_depend.ts:805] The model has been compiled on the Ascend AI processor, current graph id is: 31 +============ start load pretrained model ======================================= +============ load success 6.3959 ===================== +Train ACC: 1.0000 +Final Average Distances : 0.871 diff --git a/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/test/output/3/vit-ckpt_bs2_1p_acc.log b/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/test/output/3/vit-ckpt_bs2_1p_acc.log new file mode 100644 index 0000000000000000000000000000000000000000..be827945be43a2000da74730b3438fc39c452462 --- /dev/null +++ b/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/test/output/3/vit-ckpt_bs2_1p_acc.log @@ -0,0 +1,6 @@ +Network = vit-ckpt +RankSize = 1 +BatchSize = 2 +DeviceType = aarch64 +CaseName = vit-ckpt_bs2_1p_acc +E2ETrainingTime = 442 diff --git a/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/test/output/train_.log b/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/test/output/train_.log new file mode 100644 index 0000000000000000000000000000000000000000..8b45908e9607713b0a12289a59bb172110665984 --- /dev/null +++ b/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/test/output/train_.log @@ -0,0 +1,2519 @@ +[PrecisionTool] Set Tensorflow random seed to 0 success. +[PrecisionTool] Set numpy random seed to 0 success. +load dataset ================================= +load dataset ================================= +[PrecisionTool] Set fusion switch file: /home/HwHiAiUser/wubo/vit/vit-ckpt/precision_tool/fusion_off.cfg +============ start load pretrained model ======================================= +============ load success 9.9240 ===================== +epoch:1 step: 0 , loss: 0.0681 , time: 158.2043 acc: 1.0000 +epoch:1 step: 1 , loss: 0.2198 , time: 17.0998 acc: 0.7500 +epoch:1 step: 2 , loss: 0.0095 , time: 16.9246 acc: 1.0000 +epoch:1 step: 3 , loss: 0.0476 , time: 16.7616 acc: 1.0000 +epoch:1 step: 4 , loss: 0.0578 , time: 16.7317 acc: 1.0000 +epoch:1 step: 5 , loss: 0.6772 , time: 17.4740 acc: 0.7500 +epoch:1 step: 6 , loss: 0.0049 , time: 16.2555 acc: 1.0000 +epoch:1 step: 7 , loss: 0.1065 , time: 18.1569 acc: 1.0000 +epoch:1 step: 8 , loss: 0.1218 , time: 16.9833 acc: 1.0000 +epoch:1 step: 9 , loss: 0.0144 , time: 17.7995 acc: 1.0000 +Train ACC: 0.9500 +0.75 (4, 100) +0.25 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.5 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.5 (4, 100) +1.0 (4, 100) +0.5 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.5 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.5 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +0.5 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.5 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.5 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.5 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.5 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.5 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.5 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.5 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.5 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.5 (4, 100) +1.0 (4, 100) +0.5 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.5 (4, 100) +1.0 (4, 100) +0.5 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +0.5 (4, 100) +1.0 (4, 100) +0.5 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.25 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +0.5 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.5 (4, 100) +1.0 (4, 100) +0.5 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.25 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.25 (4, 100) +0.5 (4, 100) +0.5 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.5 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.5 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.5 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.5 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.5 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.5 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.5 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.5 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +0.5 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.5 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.5 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.5 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.5 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.5 (4, 100) +0.5 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.5 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +0.5 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.5 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.25 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.5 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.5 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.5 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.5 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.5 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.5 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.5 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.5 (4, 100) +0.75 (4, 100) +0.5 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.5 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.5 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.5 (4, 100) +0.5 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.5 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.25 (4, 100) +0.25 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.5 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.5 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.5 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.5 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.5 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.5 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +0.25 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.25 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.5 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.5 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.25 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.5 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.5 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.5 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.5 (4, 100) +0.5 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.5 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.5 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.5 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.5 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.5 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.5 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +0.5 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.5 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.5 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +0.5 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +0.25 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.5 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.5 (4, 100) +0.5 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.5 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.5 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.5 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.5 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.5 (4, 100) +0.75 (4, 100) +0.5 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.5 (4, 100) +0.5 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.5 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.5 (4, 100) +1.0 (4, 100) +0.5 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.5 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.5 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.5 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.5 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.5 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.5 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.5 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.5 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.5 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.5 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.25 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.5 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.5 (4, 100) +0.5 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.5 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.5 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.5 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.5 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.5 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.5 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.5 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.5 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.25 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +0.5 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.5 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.5 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.5 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.5 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.5 (4, 100) +0.5 (4, 100) +0.5 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.5 (4, 100) +0.75 (4, 100) +0.5 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.5 (4, 100) +0.5 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.5 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.5 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.5 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.5 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.5 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.5 (4, 100) +1.0 (4, 100) +0.5 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.5 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +0.5 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.5 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.5 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.5 (4, 100) +0.75 (4, 100) +0.5 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.5 (4, 100) +0.5 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.5 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.25 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.5 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.5 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.5 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.5 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.25 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.5 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.5 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.5 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +0.5 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.5 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +0.5 (4, 100) +1.0 (4, 100) +0.5 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.5 (4, 100) +0.75 (4, 100) +0.5 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.5 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.5 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +0.5 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.5 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.5 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.5 (4, 100) +1.0 (4, 100) +0.5 (4, 100) +0.75 (4, 100) +0.5 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.5 (4, 100) +0.5 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.5 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.5 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.5 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.5 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.5 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.5 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +0.5 (4, 100) +1.0 (4, 100) +0.5 (4, 100) +0.25 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.5 (4, 100) +0.5 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.5 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.5 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.5 (4, 100) +0.5 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.5 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.5 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.5 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.5 (4, 100) +1.0 (4, 100) +0.5 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.25 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.5 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.5 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.5 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.5 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.5 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.5 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.5 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.5 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.5 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.25 (4, 100) +0.5 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.5 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +0.5 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.5 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.5 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.5 (4, 100) +0.75 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +0.75 (4, 100) +1.0 (4, 100) +Final Average Distances : 0.871 diff --git a/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/test/output/vit-ckpt_bs1_1p_acc.log b/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/test/output/vit-ckpt_bs1_1p_acc.log new file mode 100644 index 0000000000000000000000000000000000000000..47b73b5ba822c296f078aadf7529c65c101d29d6 --- /dev/null +++ b/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/test/output/vit-ckpt_bs1_1p_acc.log @@ -0,0 +1,6 @@ +Network = vit-ckpt +RankSize = 1 +BatchSize = 1 +DeviceType = aarch64 +CaseName = vit-ckpt_bs1_1p_acc +E2ETrainingTime = 671 diff --git a/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/test/train_full_1p.sh b/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/test/train_full_1p.sh new file mode 100644 index 0000000000000000000000000000000000000000..8c33c765f395b87c908a7a9f85c66208679e218f --- /dev/null +++ b/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/test/train_full_1p.sh @@ -0,0 +1,201 @@ +#!/bin/bash +export RANK_INDEX=2 +export RANK_SIZE=1 +export RANK_ID=2 +export DEVICE_ID=2 +export DEVICE_INDEX=2 +export ASCEND_DEVICE_ID=2 + +########################################################## +#########第3行 至 100行，请一定不要、不要、不要修改########## +#########第3行 至 100行，请一定不要、不要、不要修改########## +#########第3行 至 100行，请一定不要、不要、不要修改########## +########################################################## +# shell脚本所在路径 +cur_path=`echo $(cd $(dirname $0);pwd)` + +# 判断当前shell是否是performance +perf_flag=`echo $0 | grep performance | wc -l` + +# 当前执行网络的名称 +Network=`echo $(cd $(dirname $0);pwd) | awk -F"/" '{print $(NF-1)}'` + +export RANK_SIZE=1 +export RANK_ID=0 +export JOB_ID=10087 + +# 路径参数初始化 +data_path="" +output_path="" + +# 帮助信息，不需要修改 +if [[ $1 == --help || $1 == -h ]];then + echo"usage:./train_performance_1P.sh " + echo " " + echo "parameter explain: + --data_path # dataset of training + --output_path # output of training + --train_steps # max_step for training + --train_epochs # max_epoch for training + --batch_size # batch size + -h/--help show help message + " + exit 1 +fi + +# 参数校验，不需要修改 +for para in $* +do + if [[ $para == --data_path* ]];then + data_path=`echo ${para#*=}` + elif [[ $para == --output_path* ]];then + output_path=`echo ${para#*=}` + elif [[ $para == --train_steps* ]];then + train_steps=`echo ${para#*=}` + elif [[ $para == --train_epochs* ]];then + train_epochs=`echo ${para#*=}` + elif [[ $para == --batch_size* ]];then + batch_size=`echo ${para#*=}` + fi +done + +# 校验是否传入data_path,不需要修改 +if [[ $data_path == "" ]];then + echo "[Error] para \"data_path\" must be config" + exit 1 +fi + +# 校验是否传入output_path,不需要修改 +if [[ $output_path == "" ]];then + output_path="./test/output/${ASCEND_DEVICE_ID}" +fi + +# 设置打屏日志文件名，请保留，文件名为${print_log} +print_log="./test/output/${ASCEND_DEVICE_ID}/train_${ASCEND_DEVICE_ID}.log" +modelarts_flag=${MODELARTS_MODEL_PATH} +if [ x"${modelarts_flag}" != x ]; +then + echo "running without etp..." + print_log_name=`ls /home/ma-user/modelarts/log/ | grep proc-rank` + print_log="/home/ma-user/modelarts/log/${print_log_name}" +fi +echo "### get your log here : ${print_log}" + +CaseName="" +function get_casename() +{ + if [ x"${perf_flag}" = x1 ]; + then + CaseName=${Network}_bs${batch_size}_${RANK_SIZE}'p'_'perf' + else + CaseName=${Network}_bs${batch_size}_${RANK_SIZE}'p'_'acc' + fi +} + +# 跳转到code目录 +cd ${cur_path}/../ +rm -rf ./test/output/${ASCEND_DEVICE_ID} +mkdir -p ./test/output/${ASCEND_DEVICE_ID} + +# 训练开始时间记录，不需要修改 +start_time=$(date +%s) +########################################################## +#########第3行 至 100行，请一定不要、不要、不要修改########## +#########第3行 至 100行，请一定不要、不要、不要修改########## +#########第3行 至 100行，请一定不要、不要、不要修改########## +########################################################## + +#========================================================= +#========================================================= +#========训练执行命令，需要根据您的网络进行修改============== +#========================================================= +#========================================================= +# 基础参数，需要模型审视修改 +# 您的训练数据集在${data_path}路径下，请直接使用这个变量获取 +# 您的训练输出目录在${output_path}路径下，请直接使用这个变量获取 +# 您的其他基础参数，可以自定义增加，但是batch_size请保留，并且设置正确的值 +batch_size=2 + + +export LD_LIBRARY_PATH=/usr/local/python3.7.5/lib:$LD_LIBRARY_PATH +#如果用户环境存在多个python3版本，则指定使用python3.7.5版本 + +export PATH=/usr/local/python3.7.5/bin:$PATH +export install_path=/usr/local/Ascend/ascend-toolkit/latest #软件包安装路径，请根据实际情况修改 +export LD_LIBRARY_PATH=${install_path}/fwkacllib/lib64:/usr/lib:$LD_LIBRARY_PATH +export PATH=${install_path}/fwkacllib/ccec_compiler/bin:${install_path}/fwkacllib/bin:$PATH +export PYTHONPATH=${install_path}/fwkacllib/python/site-packages:${install_path}/toolkit/python/site-packages:$PYTHONPATH +export PYTHONPATH=/usr/local/Ascend/tfplugin/latest/tfplugin/python/site-packages:$PYTHONPATH +export ASCEND_OPP_PATH=${install_path}/opp +export ASCEND_AICPU_PATH=${install_path}/{arch}-linux #其中{arch}请根据实际情况替换(arm64或x86_64) +export TOOLCHAIN_HOME=${install_path}/toolkit + +if [ x"${modelarts_flag}" != x ]; +then + python3.7 ./vit_allpipeline_fusion_accelerate.py --data_path=${data_path} --output_path=${output_path} > ${print_log} 2>&1 : +else + python3.7 ./vit_allpipeline_fusion_accelerate.py --data_path=${data_path} --output_path=${output_path} > ${print_log} 2>&1 +fi + +# 性能相关数据计算 +# StepTime=`grep "sec/step :" ${print_log} | tail -n 10 | awk '{print $NF}' | awk '{sum+=$1} END {print sum/NR}'` +# FPS=`awk 'BEGIN{printf "%.2f\n", '${batch_size}'/'${StepTime}'}'` + +# 精度相关数据计算 +train_accuracy=`grep "Final Average Distances :" ${print_log} | awk '{print $NF}'` + +########################################################### +#########后面的所有内容请不要修改########################### +#########后面的所有内容请不要修改########################### +#########后面的所有内容请不要修改########################### +########################################################### + +# 判断本次执行是否正确使用Ascend NPU +use_npu_flag=`grep "The model has been compiled on the Ascend AI processor" ${print_log} | wc -l` + +if [ x"${use_npu_flag}" == x0 ]; +then + echo "------------------ ERROR NOTICE START ------------------" + echo "ERROR, your task haven't used Ascend NPU, please check your npu Migration." + echo "------------------ ERROR NOTICE END------------------" +else + echo "------------------ INFO NOTICE START------------------" + echo "INFO, your task have used Ascend NPU, please check your result." + echo "------------------ INFO NOTICE END------------------" +fi + +# 获取最终的casename，请保留，case文件名为${CaseName} +get_casename + +# 重命名loss文件 +if [ -f ./test/output/${ASCEND_DEVICE_ID}/my_output_loss.txt ]; +then + mv ./test/output/${ASCEND_DEVICE_ID}/my_output_loss.txt ./test/output/${ASCEND_DEVICE_ID}/${CaseName}_loss.txt +fi + +# 训练端到端耗时 +end_time=$(date +%s) +e2e_time=$(( $end_time - $start_time )) + +echo "------------------ Final result ------------------" +# 输出性能FPS/单step耗时/端到端耗时 +# echo "Final Performance images/sec : $FPS" +# echo "Final Performance sec/step : $StepTime" +echo "E2E Training Duration sec : $e2e_time" + +# 输出训练精度 +echo "Final Train Accuracy : ${train_accuracy}" + +# 最后一个迭代loss值，不需要修改 +# ActualLoss=(`awk 'END {print $NF}' $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}_loss.txt`) + +#关键信息打印到${CaseName}.log中，不需要修改 +echo "Network = ${Network}" > $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "RankSize = ${RANK_SIZE}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "BatchSize = ${batch_size}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "DeviceType = `uname -m`" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "CaseName = ${CaseName}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log +# echo "ActualFPS = ${FPS}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log +# echo "TrainingTime = ${StepTime}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log +# echo "ActualLoss = ${ActualLoss}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "E2ETrainingTime = ${e2e_time}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log diff --git a/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/test/train_performance_1p.sh b/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/test/train_performance_1p.sh new file mode 100644 index 0000000000000000000000000000000000000000..3ee8df9d48f144214fe5935540519ec9b0003e27 --- /dev/null +++ b/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/test/train_performance_1p.sh @@ -0,0 +1,184 @@ +#!/bin/bash + +########################################################## +#########第3行 至 100行，请一定不要、不要、不要修改########## +#########第3行 至 100行，请一定不要、不要、不要修改########## +#########第3行 至 100行，请一定不要、不要、不要修改########## +########################################################## +# shell脚本所在路径 +cur_path=`echo $(cd $(dirname $0);pwd)` + +# 判断当前shell是否是performance +perf_flag=`echo $0 | grep performance | wc -l` + +# 当前执行网络的名称 +Network=`echo $(cd $(dirname $0);pwd) | awk -F"/" '{print $(NF-1)}'` + +export RANK_SIZE=1 +export RANK_ID=0 +export JOB_ID=10087 + +# 路径参数初始化 +data_path="" +output_path="" + +# 帮助信息，不需要修改 +if [[ $1 == --help || $1 == -h ]];then + echo"usage:./train_performance_1P.sh " + echo " " + echo "parameter explain: + --data_path # dataset of training + --output_path # output of training + --train_steps # max_step for training + --train_epochs # max_epoch for training + --batch_size # batch size + -h/--help show help message + " + exit 1 +fi + +# 参数校验，不需要修改 +for para in $* +do + if [[ $para == --data_path* ]];then + data_path=`echo ${para#*=}` + elif [[ $para == --output_path* ]];then + output_path=`echo ${para#*=}` + elif [[ $para == --train_steps* ]];then + train_steps=`echo ${para#*=}` + elif [[ $para == --train_epochs* ]];then + train_epochs=`echo ${para#*=}` + elif [[ $para == --batch_size* ]];then + batch_size=`echo ${para#*=}` + fi +done + +# 校验是否传入data_path,不需要修改 +if [[ $data_path == "" ]];then + echo "[Error] para \"data_path\" must be config" + exit 1 +fi + +# 校验是否传入output_path,不需要修改 +if [[ $output_path == "" ]];then + output_path="./test/output/${ASCEND_DEVICE_ID}" +fi + +# 设置打屏日志文件名，请保留，文件名为${print_log} +print_log="./test/output/${ASCEND_DEVICE_ID}/train_${ASCEND_DEVICE_ID}.log" +modelarts_flag=${MODELARTS_MODEL_PATH} +if [ x"${modelarts_flag}" != x ]; +then + echo "running with modelarts..." + print_log_name=`ls /home/ma-user/modelarts/log/ | grep proc-rank` + print_log="/home/ma-user/modelarts/log/${print_log_name}" +fi +echo "### get your log here : ${print_log}" + +CaseName="" +function get_casename() +{ + if [ x"${perf_flag}" = x1 ]; + then + CaseName=${Network}_bs${batch_size}_${RANK_SIZE}'p'_'perf' + else + CaseName=${Network}_bs${batch_size}_${RANK_SIZE}'p'_'acc' + fi +} + +# 跳转到code目录 +cd ${cur_path}/../ +rm -rf ./test/output/${ASCEND_DEVICE_ID} +mkdir -p ./test/output/${ASCEND_DEVICE_ID} + +# 训练开始时间记录，不需要修改 +start_time=$(date +%s) +########################################################## +#########第3行 至 100行，请一定不要、不要、不要修改########## +#########第3行 至 100行，请一定不要、不要、不要修改########## +#########第3行 至 100行，请一定不要、不要、不要修改########## +########################################################## + +#========================================================= +#========================================================= +#========训练执行命令，需要根据您的网络进行修改============== +#========================================================= +#========================================================= +# 基础参数，需要模型审视修改 +# 您的训练数据集在${data_path}路径下，请直接使用这个变量获取 +# 您的训练输出目录在${output_path}路径下，请直接使用这个变量获取 +# 您的其他基础参数，可以自定义增加，但是batch_size请保留，并且设置正确的值 +batch_size=1 +epoch=1 + +if [ x"${modelarts_flag}" != x ]; +then + python3.7 ./vit_allpipeline_fusion_accelerate.py --data_path=${data_path} --output_path=${output_path} +else + python3.7 ./vit_allpipeline_fusion_accelerate.py --data_path=${data_path} --output_path=${output_path} > ${print_log} +fi + +# 性能相关数据计算 +StepTime=`grep "sec/step :" ${print_log} | tail -n 10 | awk '{print $NF}' | awk '{sum+=$1} END {print sum/NR}'` +FPS=`awk 'BEGIN{printf "%.2f\n", '${batch_size}'/'${StepTime}'}'` + +# 精度相关数据计算 +train_accuracy=`grep "Final Average Distances :" ${print_log} | awk '{print $NF}'` +# 提取所有loss打印信息 +#grep "loss :" ${print_log} | awk -F ":" '{print $4}' | awk -F "-" '{print $1}' > ./test/output/${ASCEND_DEVICE_ID}/my_output_loss.txt +grep "d_loss :" ${print_log} | awk -F "|" '{print $2}' > ./test/output/${ASCEND_DEVICE_ID}/my_output_loss.txt + +########################################################### +#########后面的所有内容请不要修改########################### +#########后面的所有内容请不要修改########################### +#########后面的所有内容请不要修改########################### +########################################################### + +# 判断本次执行是否正确使用Ascend NPU +use_npu_flag=`grep "The model has been compiled on the Ascend AI processor" ${print_log} | wc -l` +if [ x"${use_npu_flag}" == x0 ]; +then + echo "------------------ ERROR NOTICE START ------------------" + echo "ERROR, your task haven't used Ascend NPU, please check your npu Migration." + echo "------------------ ERROR NOTICE END------------------" +else + echo "------------------ INFO NOTICE START------------------" + echo "INFO, your task have used Ascend NPU, please check your result." + echo "------------------ INFO NOTICE END------------------" +fi + +# 获取最终的casename，请保留，case文件名为${CaseName} +get_casename + +# 重命名loss文件 +if [ -f ./test/output/${ASCEND_DEVICE_ID}/my_output_loss.txt ]; +then + mv ./test/output/${ASCEND_DEVICE_ID}/my_output_loss.txt ./test/output/${ASCEND_DEVICE_ID}/${CaseName}_loss.txt +fi + +# 训练端到端耗时 +end_time=$(date +%s) +e2e_time=$(( $end_time - $start_time )) + +echo "------------------ Final result ------------------" +# 输出性能FPS/单step耗时/端到端耗时 +echo "Final Performance images/sec : $FPS" +echo "Final Performance sec/step : $StepTime" +echo "E2E Training Duration sec : $e2e_time" + +# 输出训练精度 +echo "Final Train Accuracy : ${train_accuracy}" + +# 最后一个迭代loss值，不需要修改 +ActualLoss=(`awk 'END {print $NF}' $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}_loss.txt`) + +#关键信息打印到${CaseName}.log中，不需要修改 +echo "Network = ${Network}" > $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "RankSize = ${RANK_SIZE}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "BatchSize = ${batch_size}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "DeviceType = `uname -m`" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "CaseName = ${CaseName}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "ActualFPS = ${FPS}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "TrainingTime = ${StepTime}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "ActualLoss = ${ActualLoss}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "E2ETrainingTime = ${e2e_time}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log diff --git a/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/vit_allpipeline_fusion_accelerate.py b/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/vit_allpipeline_fusion_accelerate.py new file mode 100644 index 0000000000000000000000000000000000000000..eee0106b43799ca25dea304df7c639bf723e714a --- /dev/null +++ b/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/vit_allpipeline_fusion_accelerate.py @@ -0,0 +1,226 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import tensorflow as tf +from vit_keras import vit, utils +import os +import time +import tensorflow.keras.backend as K +from PIL import Image +import pickle +import numpy as np +import argparse + +from npu_bridge.estimator import npu_ops +from tensorflow.core.protobuf.rewriter_config_pb2 import RewriterConfig +# import moxing as mox +import precision_tool.tf_config as npu_tf_config +import os +os.environ['TF_CPP_MIN_VLOG_LEVEL'] = '3' +from npu_bridge.npu_init import * +###################### +from npu_bridge.estimator.npu.npu_config import NPURunConfig +from npu_bridge.estimator.npu.npu_config import ProfilingConfig + +def parse_args(): + parser = argparse.ArgumentParser() + # parser.add_argument('--model_path', type=str, default='', help='Pretrained model location') + parser.add_argument('--data_path', type=str, default='./datasets', help='Datasets location') + parser.add_argument('--output_path', type=str, default='./output', help='Output location,saving trained models') + parser.add_argument('--learning_rate', type=float, default=0.001, help='learning rate') + parser.add_argument('--batch_size', type=int, default=4, help='learning rate') + return parser.parse_args() + +args = parse_args() + + + +MODEL_CACHE_PATH=args.output_path +PRETRAINED_MODEL_PATH= os.path.join(args.data_path,"model") +HPARAMS = { + "batch_size": 4, + "image_size": 384, + 'learning_rate': 0.001, +} +DATA_CACHE_PATH= args.data_path +image_size = HPARAMS['image_size'] +batch_size = HPARAMS['batch_size'] + +def read_data(filename, training): + with open(filename, 'rb') as fo: + dict = pickle.load(fo, encoding='bytes') + if training: + images = dict[b'data'].reshape([50000, 3, 32, 32]) + else: + images = dict[b'data'].reshape([10000, 3, 32, 32]) + images = np.transpose(images, [0, 2, 3, 1]) + labels = np.array(dict[b'fine_labels']) + def _augment(image, label): + if np.random.rand() < 0.3: + image = tf.image.flip_left_right(image) + if np.random.rand() < 0.3: + image = tf.image.flip_up_down(image) + if np.random.rand() < 0.3: + image = tf.image.random_contrast(image, lower=0.5, upper=2) + return image, label + + def _preprocess(image, label): + image = tf.image.resize(image, (image_size, image_size)) + image = (image - 127.5) / 127.5 + return image, label + + ds = tf.data.Dataset.from_tensor_slices((images, labels)) + if training: + # ds = ds.map(_augment) + ds = ds.map(_preprocess) + ds = ds.shuffle(HPARAMS['batch_size'] * 10) + ds = ds.repeat() + else: + ds = ds.map(_preprocess) + ds = ds.repeat() + + ds = ds.batch(batch_size=HPARAMS['batch_size'], drop_remainder=True) + iterator = tf.compat.v1.data.make_one_shot_iterator(ds) + image_batch, label_batch = iterator.get_next() + print("load dataset =================================") + return image_batch, label_batch + + +model = vit.vit_b16_load_pretrain( + image_size=HPARAMS['image_size'], + activation='linear', + pretrained=True, + classes=100, + include_top=True, + pretrained_top=True, + pretrained_path="./" +) + +images_batch, labels_batch = read_data(filename=os.path.join(DATA_CACHE_PATH,"train"), training=True) +val_image_batch, val_labels_batch = read_data(filename=os.path.join(DATA_CACHE_PATH,"test"), training=False) + +loss_fun = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) + +inputx = tf.compat.v1.placeholder( + tf.float32, shape=[HPARAMS['batch_size'], HPARAMS['image_size'], HPARAMS['image_size'], 3], name="inputx") + +inputy = tf.compat.v1.placeholder( + tf.int64, shape=[HPARAMS['batch_size'], ], name="inputy") + +inputTrain = tf.compat.v1.placeholder( + tf.bool, name='training') + + +def eval(pred, label): + prediction = np.argmax(pred, 1).tolist() + return calc(prediction, label) + + +def calc(prediction, label): + a = [prediction[i] == label[i] for i in range(len(prediction))] + return sum(a) / len(a) + + +out = model(inputx, training=inputTrain) +loss = loss_fun(inputy, out) + +optimizer = tf.train.MomentumOptimizer( + learning_rate=HPARAMS['learning_rate'], + momentum=0.9, + use_locking=False, + use_nesterov=False, + name='Momentum' +) + + + +loss_scale_manager = FixedLossScaleManager(loss_scale=2**32) + +opt = NPULossScaleOptimizer(optimizer, loss_scale_manager) + + +train_op = opt.minimize(loss) + + +config = tf.ConfigProto() +custom_op = config.graph_options.rewrite_options.custom_optimizers.add() +custom_op.name = "NpuOptimizer" +custom_op.parameter_map["use_off_line"].b = True # 在昇腾AI处理器执行训练 +custom_op.parameter_map["precision_mode"].s=tf.compat.as_bytes("allow_mix_precision") + +config.graph_options.rewrite_options.remapping = RewriterConfig.OFF # 关闭remap开关 +config.gpu_options.allow_growth = True +# 混合训练 +custom_op.parameter_map["precision_mode"].s = tf.compat.as_bytes("allow_mix_precision") + +session_config = npu_tf_config.session_dump_config(config, action='overflow|fusion_off') # 新增行 + + +sess = tf.Session(config=session_config) +sess.run(tf.global_variables_initializer()) + + +saver = tf.train.Saver(max_to_keep=10) +s = time.time() +print("============ start load pretrained model =======================================") +saver.restore(sess, "{}/vit-base-5".format(PRETRAINED_MODEL_PATH)) # 0.8709 for 5 # 2 0.8719 # 3 0.8767 +print("============ load success {:.4f} =====================".format(time.time() - s)) + +for epoch in range(1, 2): + # train + label_col = [] + pred_col = [] + for step in range(HPARAMS['batch_size']*10 // HPARAMS['batch_size']): + s = time.time() + x_in, y_in = sess.run([images_batch, labels_batch], + feed_dict={inputTrain: True}) + out_, loss_, _ = sess.run([out, loss, train_op], feed_dict={ + inputx: x_in, inputy: y_in, inputTrain: True}) + label_col += y_in.tolist() + pred_col += np.argmax(out_, 1).tolist() + print("epoch:{} step: {} , loss: {:.4f} , time: {:.4f} acc: {:.4f}".format( + epoch, step, loss_.item(), time.time() - s, eval(out_, y_in))) # print 到文件里面 + print("Train ACC: {:.4f}".format(calc(pred_col, label_col))) + saver.save(sess, "{}/vit-base-5fusion-mix2".format(MODEL_CACHE_PATH), global_step=epoch) + # break + prediction = [] + labels = [] + for _ in range(10000 // HPARAMS['batch_size']): + x_in, y_in = sess.run([val_image_batch, val_labels_batch]) + pred = sess.run(out, feed_dict={inputx: x_in, inputy: y_in, inputTrain: False}) + # pred = model.predict(x_in) + # print(calc(np.argmax(pred, 1).tolist(), y_in.tolist()), pred.shape, ) + prediction += np.argmax(pred, 1).tolist() + labels += y_in.tolist() + print( + "Final Average Distances :", + sum(prediction[i] == labels[i] for i in range(len(labels))) + / len(labels), + ) + diff --git a/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/vit_keras/__init__.py b/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/vit_keras/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/vit_keras/__pycache__/__init__.cpython-36.pyc b/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/vit_keras/__pycache__/__init__.cpython-36.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7955d764f164a49213292f0b4af1fc813e212685 Binary files /dev/null and b/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/vit_keras/__pycache__/__init__.cpython-36.pyc differ diff --git a/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/vit_keras/__pycache__/__init__.cpython-37.pyc b/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/vit_keras/__pycache__/__init__.cpython-37.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3acb61bede29dd810e5ef9f56d1b08c8d70f738e Binary files /dev/null and b/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/vit_keras/__pycache__/__init__.cpython-37.pyc differ diff --git a/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/vit_keras/__pycache__/layers.cpython-36.pyc b/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/vit_keras/__pycache__/layers.cpython-36.pyc new file mode 100644 index 0000000000000000000000000000000000000000..92b737741401927e836d8ae6c8dd4d80c9430dd3 Binary files /dev/null and b/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/vit_keras/__pycache__/layers.cpython-36.pyc differ diff --git a/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/vit_keras/__pycache__/layers.cpython-37.pyc b/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/vit_keras/__pycache__/layers.cpython-37.pyc new file mode 100644 index 0000000000000000000000000000000000000000..67b06472c6c461119eccbf66d3d287bcee9f2b00 Binary files /dev/null and b/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/vit_keras/__pycache__/layers.cpython-37.pyc differ diff --git a/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/vit_keras/__pycache__/utils.cpython-36.pyc b/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/vit_keras/__pycache__/utils.cpython-36.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3a52975d913da3c9ba63eea8c7cb571b1e1901b4 Binary files /dev/null and b/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/vit_keras/__pycache__/utils.cpython-36.pyc differ diff --git a/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/vit_keras/__pycache__/utils.cpython-37.pyc b/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/vit_keras/__pycache__/utils.cpython-37.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c77867ec2d11cd2bb0c0600f9e2c88648e71657c Binary files /dev/null and b/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/vit_keras/__pycache__/utils.cpython-37.pyc differ diff --git a/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/vit_keras/__pycache__/vit.cpython-36.pyc b/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/vit_keras/__pycache__/vit.cpython-36.pyc new file mode 100644 index 0000000000000000000000000000000000000000..30e6c5cd670cf3b81921fe463854eeee1a0c4a93 Binary files /dev/null and b/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/vit_keras/__pycache__/vit.cpython-36.pyc differ diff --git a/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/vit_keras/__pycache__/vit.cpython-37.pyc b/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/vit_keras/__pycache__/vit.cpython-37.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a16095d88b4d2e7d563fbcd88caa60e428298f02 Binary files /dev/null and b/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/vit_keras/__pycache__/vit.cpython-37.pyc differ diff --git a/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/vit_keras/layers copy.py b/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/vit_keras/layers copy.py new file mode 100644 index 0000000000000000000000000000000000000000..0ad2e8e075f5dc9ff669f51508b93ab9369bffe0 --- /dev/null +++ b/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/vit_keras/layers copy.py @@ -0,0 +1,321 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=arguments-differ,missing-function-docstring,missing-class-docstring,unexpected-keyword-arg,no-value-for-parameter +import tensorflow as tf +import threading +# modify record : +# remove super version : tfa . use tf.keras.activations.gelu() replace tfa.activations.gelu() + +import tensorflow.keras.backend as K + + +# 更改地方 +def swish(x, beta): + return x * K.sigmoid(beta * x) + + +def gelu(x, approximate="xxx"): + return swish(x, 1.702) + + +class CustomObjectsContext(threading.local): + + def __init__(self): + super(CustomObjectsContext, self).__init__() + self.objects = {} + self.names = {} + + +GLOBAL_CUSTOM_OBJECTS = CustomObjectsContext() + + +# @keras_export('keras.utils.register_keras_serializable') +def register_keras_serializable(package='Custom', name=None): + """Registers an object with the Keras serialization framework. + This decorator injects the decorated class or function into the Keras custom + object dictionary, so that it can be serialized and deserialized without + needing an entry in the user-provided custom object dict. It also injects a + function that Keras will call to get the object's serializable string key. + Note that to be serialized and deserialized, classes must implement the + `get_config()` method. Functions do not have this requirement. + The object will be registered under the key 'package>name' where `name`, + defaults to the object name if not passed. + Args: + package: The package that this class belongs to. + name: The name to serialize this class under in this package. If None, the + class' name will be used. + Returns: + A decorator that registers the decorated class with the passed names. + """ + + def decorator(arg): + """Registers a class with the Keras serialization framework.""" + class_name = name if name is not None else arg.__name__ + registered_name = package + '>' + class_name + + # if tf_inspect.isclass(arg) and not hasattr(arg, 'get_config'): + # raise ValueError( + # 'Cannot register a class that does not have a get_config() method.') + + if registered_name in GLOBAL_CUSTOM_OBJECTS.objects: + raise ValueError(f'{registered_name} has already been registered to ' + f'{GLOBAL_CUSTOM_OBJECTS.objects[registered_name]}') + + if arg in GLOBAL_CUSTOM_OBJECTS.names: + raise ValueError( + f'{arg} has already been registered to {GLOBAL_CUSTOM_OBJECTS.names[arg]}' + ) + GLOBAL_CUSTOM_OBJECTS.objects[registered_name] = arg + GLOBAL_CUSTOM_OBJECTS.names[arg] = registered_name + + return arg + + return decorator + + +@register_keras_serializable() +class ClassToken(tf.keras.layers.Layer): + """Append a class token to an input layer.""" + + def build(self, input_shape): + cls_init = tf.zeros_initializer() + self.hidden_size = input_shape[-1] + self.cls = tf.Variable( + name="cls", + initial_value=cls_init(shape=(1, 1, self.hidden_size), dtype="float32"), + trainable=True, + ) + + def call(self, inputs): + batch_size = tf.shape(inputs)[0] + cls_broadcasted = tf.cast( + tf.broadcast_to(self.cls, [batch_size, 1, self.hidden_size]), + dtype=inputs.dtype, + ) + return tf.concat([cls_broadcasted, inputs], 1) + + def get_config(self): + config = super().get_config() + return config + + @classmethod + def from_config(cls, config): + return cls(**config) + + +@register_keras_serializable() +class AddPositionEmbs(tf.keras.layers.Layer): + """Adds (optionally learned) positional embeddings to the inputs.""" + + def build(self, input_shape): + assert ( + len(input_shape) == 3 + ), f"Number of dimensions should be 3, got {len(input_shape)}" + self.pe = tf.Variable( + name="pos_embedding", + initial_value=tf.random_normal_initializer(stddev=0.06)( + shape=(1, input_shape[1], input_shape[2]) + ), + dtype="float32", + trainable=True, + ) + + def call(self, inputs): + return inputs + tf.cast(self.pe, dtype=inputs.dtype) + + def get_config(self): + config = super().get_config() + return config + + @classmethod + def from_config(cls, config): + return cls(**config) + + +@register_keras_serializable() +class MultiHeadSelfAttention(tf.keras.layers.Layer): + def __init__(self, *args, num_heads, **kwargs): + super().__init__(*args, **kwargs) + self.num_heads = num_heads + + def build(self, input_shape): + hidden_size = input_shape[-1] + num_heads = self.num_heads + if hidden_size % num_heads != 0: + raise ValueError( + f"embedding dimension = {hidden_size} should be divisible by number of heads = {num_heads}" + ) + self.hidden_size = hidden_size + self.projection_dim = hidden_size // num_heads + self.query_dense = tf.keras.layers.Dense(hidden_size, name="query") + self.key_dense = tf.keras.layers.Dense(hidden_size, name="key") + self.value_dense = tf.keras.layers.Dense(hidden_size, name="value") + self.combine_heads = tf.keras.layers.Dense(hidden_size, name="out") + + # pylint: disable=no-self-use + def attention(self, query, key, value): + score = tf.matmul(query, key, transpose_b=True) + dim_key = tf.cast(tf.shape(key)[-1], score.dtype) + scaled_score = score / tf.math.sqrt(dim_key) + weights = tf.nn.softmax(scaled_score, axis=-1) + output = tf.matmul(weights, value) + return output, weights + + def separate_heads(self, x, batch_size): + x = tf.reshape(x, (batch_size, -1, self.num_heads, self.projection_dim)) + return tf.transpose(x, perm=[0, 2, 1, 3]) + + def call(self, inputs): + batch_size = tf.shape(inputs)[0] + query = self.query_dense(inputs) + key = self.key_dense(inputs) + value = self.value_dense(inputs) + query = self.separate_heads(query, batch_size) + key = self.separate_heads(key, batch_size) + value = self.separate_heads(value, batch_size) + + attention, weights = self.attention(query, key, value) + attention = tf.transpose(attention, perm=[0, 2, 1, 3]) + concat_attention = tf.reshape(attention, (batch_size, -1, self.hidden_size)) + output = self.combine_heads(concat_attention) + return output, weights + + def get_config(self): + config = super().get_config() + config.update({"num_heads": self.num_heads}) + return config + + @classmethod + def from_config(cls, config): + return cls(**config) + + +# pylint: disable=too-many-instance-attributes +@register_keras_serializable() +class TransformerBlock(tf.keras.layers.Layer): + """Implements a Transformer block.""" + + def __init__(self, *args, num_heads, mlp_dim, dropout, **kwargs): + super().__init__(*args, **kwargs) + self.num_heads = num_heads + self.mlp_dim = mlp_dim + self.dropout = dropout + + def build(self, input_shape): + self.att = MultiHeadSelfAttention( + num_heads=self.num_heads, + name="MultiHeadDotProductAttention_1", + ) + ''' + self.mlpblock = tf.keras.Sequential( + [ + tf.keras.layers.Dense( + self.mlp_dim, + activation="linear", + name=f"{self.name}/Dense_0", + ), + tf.keras.layers.Lambda( + lambda x: gelu(x, approximate=False) + ) + if hasattr(tf.keras.activations, "gelu") + else tf.keras.layers.Lambda( + lambda x: gelu(x, approximate=False) + ), + tf.keras.layers.Dropout(self.dropout), + tf.keras.layers.Dense(input_shape[-1], name=f"{self.name}/Dense_1"), + tf.keras.layers.Dropout(self.dropout), + ], + name="MlpBlock_3", + ) + ''' + self.mlpblock1 = tf.keras.Sequential( + [ + tf.keras.layers.Dense( + self.mlp_dim, + activation="linear", + name=f"{self.name}/Dense_0", + ), + tf.keras.layers.Lambda( + lambda x: gelu(x, approximate=False) + ) + if hasattr(tf.keras.activations, "gelu") + else tf.keras.layers.Lambda( + lambda x: gelu(x, approximate=False) + ), + #tf.keras.layers.Dropout(self.dropout), + #tf.keras.layers.Dense(input_shape[-1], name=f"{self.name}/Dense_1"), + #tf.keras.layers.Dropout(self.dropout), + ], + name="MlpBlock_3", + ) + + self.mlpblock2 = tf.keras.Sequential( + [ + + tf.keras.layers.Dense(input_shape[-1], name=f"{self.name}/Dense_1"), + + ], + name="MlpBlock_3", + ) + self.layernorm1 = tf.keras.layers.LayerNormalization( + epsilon=1e-6, name="LayerNorm_0" + ) + self.layernorm2 = tf.keras.layers.LayerNormalization( + epsilon=1e-6, name="LayerNorm_2" + ) + self.dropout_layer = tf.keras.layers.Dropout(self.dropout) + + def call(self, inputs): + # modify + x = self.layernorm1(inputs) + x, weights = self.att(x) + #x = self.dropout_layer(x) + x = npu_ops.dropout_layer(x, 0.5) + x = x + inputs + y = self.layernorm2(x) + y = self.mlpblock_1(y) + y = npu_ops.dropout_layer(y, 0.5) + y = self.mlpblock_2(y) + y = npu_ops.dropout_layer(y, 0.5) + return x + y, weights + + def get_config(self): + config = super().get_config() + config.update( + { + "num_heads": self.num_heads, + "mlp_dim": self.mlp_dim, + "dropout": self.dropout, + } + ) + return config + + @classmethod + def from_config(cls, config): + return cls(**config) diff --git a/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/vit_keras/layers.py b/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/vit_keras/layers.py new file mode 100644 index 0000000000000000000000000000000000000000..fe33b96d39891f1606fdae5955846e43cfc33fc3 --- /dev/null +++ b/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/vit_keras/layers.py @@ -0,0 +1,286 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=arguments-differ,missing-function-docstring,missing-class-docstring,unexpected-keyword-arg,no-value-for-parameter +import tensorflow as tf +import threading +# modify record : +# remove super version : tfa . use tf.keras.activations.gelu() replace tfa.activations.gelu() + +import tensorflow.keras.backend as K + + +# 更改地方 +def swish(x, beta): + return x * K.sigmoid(beta * x) + + +def gelu(x, approximate="xxx"): + return swish(x, 1.702) + + +class CustomObjectsContext(threading.local): + + def __init__(self): + super(CustomObjectsContext, self).__init__() + self.objects = {} + self.names = {} + + +GLOBAL_CUSTOM_OBJECTS = CustomObjectsContext() + + +# @keras_export('keras.utils.register_keras_serializable') +def register_keras_serializable(package='Custom', name=None): + """Registers an object with the Keras serialization framework. + This decorator injects the decorated class or function into the Keras custom + object dictionary, so that it can be serialized and deserialized without + needing an entry in the user-provided custom object dict. It also injects a + function that Keras will call to get the object's serializable string key. + Note that to be serialized and deserialized, classes must implement the + `get_config()` method. Functions do not have this requirement. + The object will be registered under the key 'package>name' where `name`, + defaults to the object name if not passed. + Args: + package: The package that this class belongs to. + name: The name to serialize this class under in this package. If None, the + class' name will be used. + Returns: + A decorator that registers the decorated class with the passed names. + """ + + def decorator(arg): + """Registers a class with the Keras serialization framework.""" + class_name = name if name is not None else arg.__name__ + registered_name = package + '>' + class_name + + # if tf_inspect.isclass(arg) and not hasattr(arg, 'get_config'): + # raise ValueError( + # 'Cannot register a class that does not have a get_config() method.') + + if registered_name in GLOBAL_CUSTOM_OBJECTS.objects: + raise ValueError(f'{registered_name} has already been registered to ' + f'{GLOBAL_CUSTOM_OBJECTS.objects[registered_name]}') + + if arg in GLOBAL_CUSTOM_OBJECTS.names: + raise ValueError( + f'{arg} has already been registered to {GLOBAL_CUSTOM_OBJECTS.names[arg]}' + ) + GLOBAL_CUSTOM_OBJECTS.objects[registered_name] = arg + GLOBAL_CUSTOM_OBJECTS.names[arg] = registered_name + + return arg + + return decorator + + +@register_keras_serializable() +class ClassToken(tf.keras.layers.Layer): + """Append a class token to an input layer.""" + + def build(self, input_shape): + cls_init = tf.zeros_initializer() + self.hidden_size = input_shape[-1] + self.cls = tf.Variable( + name="cls", + initial_value=cls_init(shape=(1, 1, self.hidden_size), dtype="float32"), + trainable=True, + ) + + def call(self, inputs): + batch_size = tf.shape(inputs)[0] + cls_broadcasted = tf.cast( + tf.broadcast_to(self.cls, [batch_size, 1, self.hidden_size]), + dtype=inputs.dtype, + ) + return tf.concat([cls_broadcasted, inputs], 1) + + def get_config(self): + config = super().get_config() + return config + + @classmethod + def from_config(cls, config): + return cls(**config) + + +@register_keras_serializable() +class AddPositionEmbs(tf.keras.layers.Layer): + """Adds (optionally learned) positional embeddings to the inputs.""" + + def build(self, input_shape): + assert ( + len(input_shape) == 3 + ), f"Number of dimensions should be 3, got {len(input_shape)}" + self.pe = tf.Variable( + name="pos_embedding", + initial_value=tf.random_normal_initializer(stddev=0.06)( + shape=(1, input_shape[1], input_shape[2]) + ), + dtype="float32", + trainable=True, + ) + + def call(self, inputs): + return inputs + tf.cast(self.pe, dtype=inputs.dtype) + + def get_config(self): + config = super().get_config() + return config + + @classmethod + def from_config(cls, config): + return cls(**config) + + +@register_keras_serializable() +class MultiHeadSelfAttention(tf.keras.layers.Layer): + def __init__(self, *args, num_heads, **kwargs): + super().__init__(*args, **kwargs) + self.num_heads = num_heads + + def build(self, input_shape): + hidden_size = input_shape[-1] + num_heads = self.num_heads + if hidden_size % num_heads != 0: + raise ValueError( + f"embedding dimension = {hidden_size} should be divisible by number of heads = {num_heads}" + ) + self.hidden_size = hidden_size + self.projection_dim = hidden_size // num_heads + self.query_dense = tf.keras.layers.Dense(hidden_size, name="query") + self.key_dense = tf.keras.layers.Dense(hidden_size, name="key") + self.value_dense = tf.keras.layers.Dense(hidden_size, name="value") + self.combine_heads = tf.keras.layers.Dense(hidden_size, name="out") + + # pylint: disable=no-self-use + def attention(self, query, key, value): + score = tf.matmul(query, key, transpose_b=True) + dim_key = tf.cast(tf.shape(key)[-1], score.dtype) + scaled_score = score / tf.math.sqrt(dim_key) + weights = tf.nn.softmax(scaled_score, axis=-1) + output = tf.matmul(weights, value) + return output, weights + + def separate_heads(self, x, batch_size): + x = tf.reshape(x, (batch_size, -1, self.num_heads, self.projection_dim)) + return tf.transpose(x, perm=[0, 2, 1, 3]) + + def call(self, inputs): + batch_size = tf.shape(inputs)[0] + query = self.query_dense(inputs) + key = self.key_dense(inputs) + value = self.value_dense(inputs) + query = self.separate_heads(query, batch_size) + key = self.separate_heads(key, batch_size) + value = self.separate_heads(value, batch_size) + + attention, weights = self.attention(query, key, value) + attention = tf.transpose(attention, perm=[0, 2, 1, 3]) + concat_attention = tf.reshape(attention, (batch_size, -1, self.hidden_size)) + output = self.combine_heads(concat_attention) + return output, weights + + def get_config(self): + config = super().get_config() + config.update({"num_heads": self.num_heads}) + return config + + @classmethod + def from_config(cls, config): + return cls(**config) + + +# pylint: disable=too-many-instance-attributes +@register_keras_serializable() +class TransformerBlock(tf.keras.layers.Layer): + """Implements a Transformer block.""" + + def __init__(self, *args, num_heads, mlp_dim, dropout, **kwargs): + super().__init__(*args, **kwargs) + self.num_heads = num_heads + self.mlp_dim = mlp_dim + self.dropout = dropout + + def build(self, input_shape): + self.att = MultiHeadSelfAttention( + num_heads=self.num_heads, + name="MultiHeadDotProductAttention_1", + ) + self.mlpblock = tf.keras.Sequential( + [ + tf.keras.layers.Dense( + self.mlp_dim, + activation="linear", + name=f"{self.name}/Dense_0", + ), + tf.keras.layers.Lambda( + lambda x: gelu(x, approximate=False) + ) + if hasattr(tf.keras.activations, "gelu") + else tf.keras.layers.Lambda( + lambda x: gelu(x, approximate=False) + ), + tf.keras.layers.Dropout(self.dropout), + tf.keras.layers.Dense(input_shape[-1], name=f"{self.name}/Dense_1"), + tf.keras.layers.Dropout(self.dropout), + ], + name="MlpBlock_3", + ) + self.layernorm1 = tf.keras.layers.LayerNormalization( + epsilon=1e-6, name="LayerNorm_0" + ) + self.layernorm2 = tf.keras.layers.LayerNormalization( + epsilon=1e-6, name="LayerNorm_2" + ) + self.dropout_layer = tf.keras.layers.Dropout(self.dropout) + + def call(self, inputs): + # modify + x = self.layernorm1(inputs) + x, weights = self.att(x) + x = self.dropout_layer(x) + x = x + inputs + y = self.layernorm2(x) + y = self.mlpblock(y) + return x + y, weights + + def get_config(self): + config = super().get_config() + config.update( + { + "num_heads": self.num_heads, + "mlp_dim": self.mlp_dim, + "dropout": self.dropout, + } + ) + return config + + @classmethod + def from_config(cls, config): + return cls(**config) diff --git a/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/vit_keras/utils.py b/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/vit_keras/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..0eebe5647b88115e721ebd234d82e2f7e3fb2c1d --- /dev/null +++ b/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/vit_keras/utils.py @@ -0,0 +1,366 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import typing +import warnings +from urllib import request +from http import client +import io +import pkg_resources +# import validators +import numpy as np +import scipy as sp +import cv2 +from tqdm import tqdm +try: + import PIL + import PIL.Image +except ImportError: # pragma: no cover + PIL = None + +ImageInputType = typing.Union[str, np.ndarray, "PIL.Image.Image", io.BytesIO] + + +def get_imagenet_classes() -> typing.List[str]: + """Get the list of ImageNet 2012 classes.""" + filepath = pkg_resources.resource_filename("vit_keras", "imagenet2012.txt") + with open(filepath) as f: + classes = [l.strip() for l in f.readlines()] + return classes + + +def read(filepath_or_buffer: ImageInputType, size, timeout=None): + """Read a file into an image object + Args: + filepath_or_buffer: The path to the file or any object + with a `read` method (such as `io.BytesIO`) + size: The size to resize the image to. + timeout: If filepath_or_buffer is a URL, the timeout to + use for making the HTTP request. + """ + if PIL is not None and isinstance(filepath_or_buffer, PIL.Image.Image): + return np.array(filepath_or_buffer.convert("RGB")) + if isinstance(filepath_or_buffer, (io.BytesIO, client.HTTPResponse)): + image = np.asarray(bytearray(filepath_or_buffer.read()), dtype=np.uint8) + image = cv2.imdecode(image, cv2.IMREAD_UNCHANGED) + elif isinstance(filepath_or_buffer, str) :#and validators.url(filepath_or_buffer) + with request.urlopen(filepath_or_buffer, timeout=timeout) as r: + return read(r, size=size) + else: + if not os.path.isfile(filepath_or_buffer): + raise FileNotFoundError( + "Could not find image at path: " + filepath_or_buffer + ) + image = cv2.imread(filepath_or_buffer) + if image is None: + raise ValueError(f"An error occurred reading {filepath_or_buffer}.") + # We use cvtColor here instead of just ret[..., ::-1] + # in order to ensure that we provide a contiguous + # array for later processing. Some hashers use ctypes + # to pass the array and non-contiguous arrays can lead + # to erroneous results. + image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) + return cv2.resize(image, (size, size)) + + +def apply_embedding_weights(target_layer, source_weights, num_x_patches, num_y_patches): + """Apply embedding weights to a target layer. + + Args: + target_layer: The target layer to which weights will + be applied. + source_weights: The source weights, which will be + resized as necessary. + num_x_patches: Number of patches in width of image. + num_y_patches: Number of patches in height of image. + """ + expected_shape = target_layer.weights[0].shape + if expected_shape != source_weights.shape: + warnings.warn( + "expected_shape and source_weight.shape not matched. expected_shape: {} souce_weights: {}".format(expected_shape,source_weights.shape), + UserWarning, + ) + target_layer.set_weights([source_weights]) + +def apply_embedding_weights_customized(target_layer, source_weights, num_x_patches, num_y_patches): + """Apply embedding weights to a target layer. + + Args: + target_layer: The target layer to which weights will + be applied. + source_weights: The source weights, which will be + resized as necessary. + num_x_patches: Number of patches in width of image. + num_y_patches: Number of patches in height of image. + """ + expected_shape = target_layer.weights[0].shape + if expected_shape != source_weights.shape: + ValueError("error") + target_layer.set_weights([source_weights]) + + +def load_weights_numpy_customized( + model, params_path, pretrained_top, num_x_patches, num_y_patches +): + """Load weights saved using Flax as a numpy array. + + Args: + model: A Keras model to load the weights into. + params_path: Filepath to a numpy archive. + pretrained_top: Whether to load the top layer weights. + num_x_patches: Number of patches in width of image. + num_y_patches: Number of patches in height of image. + """ + params_dict = np.load( + params_path, allow_pickle=False + ) # pylint: disable=unexpected-keyword-arg + pre_logits = any(l.name == "pre_logits" for l in model.layers) + source_keys = list(params_dict.keys()) + n_transformers = len( + set( + "/".join(k.split("/")[:2]) + for k in source_keys + if k.startswith("Transformer/encoderblock_") + ) + ) # equal with 12 + n_transformers_out = sum( + l.name.startswith("Transformer/encoderblock_") for l in model.layers + ) + matches = [] + for tidx in range(n_transformers): # each transformer layer content + encoder = model.get_layer(f"Transformer/encoderblock_{tidx}") + source_prefix = f"Transformer/encoderblock_{tidx}" + matches.extend( + [ + { + "layer": layer, + "keys": [ + f"{source_prefix}/{norm}/{name}" for name in ["scale", "bias"] + ], + } + for norm, layer in [ + ("LayerNorm_0", encoder.layernorm1), + ("LayerNorm_2", encoder.layernorm2), + ] + ] + + [ + { + "layer": encoder.mlpblock.get_layer( + f"{source_prefix}/Dense_{mlpdense}" + ), + "keys": [ + f"{source_prefix}/MlpBlock_3/Dense_{mlpdense}/{name}" + for name in ["kernel", "bias"] + ], + } + for mlpdense in [0, 1] + ] + + [ + { + "layer": layer, + "keys": [ + f"{source_prefix}/MultiHeadDotProductAttention_1/{attvar}/{name}" + for name in ["kernel", "bias"] + ], + "reshape": True, + } + for attvar, layer in [ + ("query", encoder.att.query_dense), + ("key", encoder.att.key_dense), + ("value", encoder.att.value_dense), + ("out", encoder.att.combine_heads), + ] + ] + ) + + for layer_name in ["embedding", "head", "pre_logits"]: + if layer_name == "head" and not pretrained_top: + matches.extend(["head/kernel", "head/bias"]) + continue + if layer_name == "pre_logits" and not pre_logits: + continue + matches.append( + { + "layer": model.get_layer(layer_name), + "keys": [f"{layer_name}/{name}" for name in ["kernel", "bias"]], + } + ) + matches.append({"layer": model.get_layer("class_token"), "keys": ["cls"]}) + matches.append( + { + "layer": model.get_layer("Transformer/encoder_norm"), + "keys": [f"Transformer/encoder_norm/{name}" for name in ["scale", "bias"]], + } + ) + apply_embedding_weights_customized( + target_layer=model.get_layer("Transformer/posembed_input"), + source_weights=params_dict["Transformer/posembed_input/pos_embedding"], + num_x_patches=num_x_patches, + num_y_patches=num_y_patches, + ) + get = ["Transformer/posembed_input/pos_embedding"] + for match in tqdm(matches): + source_weights = [params_dict[k] for k in match["keys"]] + get.extend(match["keys"]) + if match.get("reshape", False): + source_weights = [ + source.reshape(expected.shape) + for source, expected in zip( + source_weights, match["layer"].get_weights() + ) + ] + match["layer"].set_weights(source_weights) + print("================ load done =========================") + + +def load_weights_numpy( + model, params_path, pretrained_top, num_x_patches, num_y_patches +): + """Load weights saved using Flax as a numpy array. + + Args: + model: A Keras model to load the weights into. + params_path: Filepath to a numpy archive. + pretrained_top: Whether to load the top layer weights. + num_x_patches: Number of patches in width of image. + num_y_patches: Number of patches in height of image. + """ + params_dict = np.load( + params_path, allow_pickle=False + ) # pylint: disable=unexpected-keyword-arg + source_keys = list(params_dict.keys()) + pre_logits = any(l.name == "pre_logits" for l in model.layers) + source_keys_used = [] + n_transformers = len( + set( + "/".join(k.split("/")[:2]) + for k in source_keys + if k.startswith("Transformer/encoderblock_") + ) + ) # equal with 12 + n_transformers_out = sum( + l.name.startswith("Transformer/encoderblock_") for l in model.layers + ) + assert n_transformers == n_transformers_out, ( + f"Wrong number of transformers (" + f"{n_transformers_out} in model vs. {n_transformers} in weights)." + ) + + matches = [] + for tidx in range(n_transformers): # each transformer layer content + encoder = model.get_layer(f"Transformer/encoderblock_{tidx}") + source_prefix = f"Transformer/encoderblock_{tidx}" + matches.extend( + [ + { + "layer": layer, + "keys": [ + f"{source_prefix}/{norm}/{name}" for name in ["scale", "bias"] + ], + } + for norm, layer in [ + ("LayerNorm_0", encoder.layernorm1), + ("LayerNorm_2", encoder.layernorm2), + ] + ] + + [ + { + "layer": encoder.mlpblock.get_layer( + f"{source_prefix}/Dense_{mlpdense}" + ), + "keys": [ + f"{source_prefix}/MlpBlock_3/Dense_{mlpdense}/{name}" + for name in ["kernel", "bias"] + ], + } + for mlpdense in [0, 1] + ] + + [ + { + "layer": layer, + "keys": [ + f"{source_prefix}/MultiHeadDotProductAttention_1/{attvar}/{name}" + for name in ["kernel", "bias"] + ], + "reshape": True, + } + for attvar, layer in [ + ("query", encoder.att.query_dense), + ("key", encoder.att.key_dense), + ("value", encoder.att.value_dense), + ("out", encoder.att.combine_heads), + ] + ] + ) + for layer_name in ["embedding", "head", "pre_logits"]: + if layer_name == "head" and not pretrained_top: + source_keys_used.extend(["head/kernel", "head/bias"]) + continue + if layer_name == "pre_logits" and not pre_logits: + continue + matches.append( + { + "layer": model.get_layer(layer_name), + "keys": [f"{layer_name}/{name}" for name in ["kernel", "bias"]], + } + ) + matches.append({"layer": model.get_layer("class_token"), "keys": ["cls"]}) + matches.append( + { + "layer": model.get_layer("Transformer/encoder_norm"), + "keys": [f"Transformer/encoder_norm/{name}" for name in ["scale", "bias"]], + } + ) + apply_embedding_weights( + target_layer=model.get_layer("Transformer/posembed_input"), + source_weights=params_dict["Transformer/posembed_input/pos_embedding"], + num_x_patches=num_x_patches, + num_y_patches=num_y_patches, + ) + source_keys_used.append("Transformer/posembed_input/pos_embedding") + for match in matches: + source_keys_used.extend(match["keys"]) + source_weights = [params_dict[k] for k in match["keys"]] + if match.get("reshape", False): + source_weights = [ + source.reshape(expected.shape) + for source, expected in zip( + source_weights, match["layer"].get_weights() + ) + ] + match["layer"].set_weights(source_weights) + unused = set(source_keys).difference(source_keys_used) + if unused: + warnings.warn(f"Did not use the following weights: {unused}", UserWarning) + target_keys_set = len(source_keys_used) + target_keys_all = len(model.weights) + if target_keys_set < target_keys_all: + warnings.warn( + f"Only set {target_keys_set} of {target_keys_all} weights.", UserWarning + ) diff --git a/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/vit_keras/vit.py b/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/vit_keras/vit.py new file mode 100644 index 0000000000000000000000000000000000000000..45c7bb9e669825096c4506c2252eacbc0a675bdb --- /dev/null +++ b/TensorFlow/contrib/cv/VisionTransformer_ID1217_for_Tensorflow/vit_keras/vit.py @@ -0,0 +1,405 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import typing +import warnings +import tensorflow as tf +import typing_extensions as tx + +from . import layers, utils + +ConfigDict = tx.TypedDict( + "ConfigDict", + { + "dropout": float, + "mlp_dim": int, + "num_heads": int, + "num_layers": int, + "hidden_size": int, + }, +) + +CONFIG_B: ConfigDict = { + "dropout": 0.1, + "mlp_dim": 3072, + "num_heads": 12, + "num_layers": 12, + "hidden_size": 768, +} + +CONFIG_L: ConfigDict = { + "dropout": 0.1, + "mlp_dim": 4096, + "num_heads": 16, + "num_layers": 24, + "hidden_size": 1024, +} + +BASE_URL = "https://github.com/faustomorales/vit-keras/releases/download/dl" +WEIGHTS = {"imagenet21k": 21_843, "imagenet21k+imagenet2012": 1_000} +SIZES = {"B_16", "B_32", "L_16", "L_32"} + +ImageSizeArg = typing.Union[typing.Tuple[int, int], int] + + +def preprocess_inputs(X): + """Preprocess images""" + return tf.keras.applications.imagenet_utils.preprocess_input( + X, data_format=None, mode="tf" + ) + + +def interpret_image_size(image_size_arg: ImageSizeArg) -> typing.Tuple[int, int]: + """Process the image_size argument whether a tuple or int.""" + if isinstance(image_size_arg, int): + return (image_size_arg, image_size_arg) + if ( + isinstance(image_size_arg, tuple) + and len(image_size_arg) == 2 + and all(map(lambda v: isinstance(v, int), image_size_arg)) + ): + return image_size_arg + raise ValueError( + f"The image_size argument must be a tuple of 2 integers or a single integer. Received: {image_size_arg}" + ) + + +def build_model( + image_size: ImageSizeArg, + patch_size: int, + num_layers: int, + hidden_size: int, + num_heads: int, + name: str, + mlp_dim: int, + classes: int, + dropout=0.1, + activation="linear", + include_top=True, + representation_size=None, + training=False, +): + """Build a ViT model. + + Args: + image_size: The size of input images. + patch_size: The size of each patch (must fit evenly in image_size) + classes: optional number of classes to classify images + into, only to be specified if `include_top` is True, and + if no `weights` argument is specified. + num_layers: The number of transformer layers to use. + hidden_size: The number of filters to use + num_heads: The number of transformer heads + mlp_dim: The number of dimensions for the MLP output in the transformers. + dropout_rate: fraction of the units to drop for dense layers. + activation: The activation to use for the final layer. + include_top: Whether to include the final classification layer. If not, + the output will have dimensions (batch_size, hidden_size). + representation_size: The size of the representation prior to the + classification layer. If None, no Dense layer is inserted. + """ + image_size_tuple = interpret_image_size(image_size) + assert (image_size_tuple[0] % patch_size == 0) and ( + image_size_tuple[1] % patch_size == 0 + ), "image_size must be a multiple of patch_size" + x = tf.keras.layers.Input(shape=(image_size_tuple[0], image_size_tuple[1], 3)) + y = tf.keras.layers.Conv2D( + filters=hidden_size, + kernel_size=patch_size, + strides=patch_size, + padding="valid", + name="embedding", + )(x) + y = tf.keras.layers.Reshape((y.shape[1] * y.shape[2], hidden_size))(y)# NONE, 576, 768 + y = layers.ClassToken(name="class_token")(y) # NONE 577 768 class only take one 1,1,768 + y = layers.AddPositionEmbs(name="Transformer/posembed_input")(y) # 577,768 + for n in range(num_layers): + y, _ = layers.TransformerBlock( + num_heads=num_heads, + mlp_dim=mlp_dim, + dropout=dropout, + name=f"Transformer/encoderblock_{n}", + )(y) # I remove the training parameter in layers.TransformerBlock + y = tf.keras.layers.LayerNormalization( + epsilon=1e-6, name="Transformer/encoder_norm" + )(y) + y = tf.keras.layers.Lambda(lambda v: v[:, 0], name="ExtractToken")(y) + if representation_size is not None: + y = tf.keras.layers.Dense( + representation_size, name="pre_logits", activation="tanh" + )(y) + if include_top: + y = tf.keras.layers.Dense(classes, name="head", activation=activation)(y) + return tf.keras.models.Model(inputs=x, outputs=y, name=name) + + +def validate_pretrained_top( + include_top: bool, pretrained: bool, classes: int, weights: str +): + """Validate that the pretrained weight configuration makes sense.""" + assert weights in WEIGHTS, f"Unexpected weights: {weights}." + expected_classes = WEIGHTS[weights] + if classes != expected_classes: + warnings.warn( + f"Can only use pretrained_top with {weights} if classes = {expected_classes}. Setting manually.", + UserWarning, + ) + assert include_top, "Can only use pretrained_top with include_top." + assert pretrained, "Can only use pretrained_top with pretrained." + return expected_classes + + +def load_pretrained( + size: str, + weights: str, + pretrained_top: bool, + model: tf.keras.models.Model, + image_size: ImageSizeArg, + patch_size: int, +): + """Load model weights for a known configuration.""" + image_size_tuple = interpret_image_size(image_size) + fname = f"ViT-{size}_{weights}.npz" + origin = f"{BASE_URL}/{fname}" + local_filepath = tf.keras.utils.get_file(fname, origin, cache_subdir="weights") + utils.load_weights_numpy( + model=model, + params_path=local_filepath, + pretrained_top=pretrained_top, + num_x_patches=image_size_tuple[1] // patch_size, + num_y_patches=image_size_tuple[0] // patch_size, + ) + +def load_pretrained_customized( + size: str, + weights: str, + pretrained_top: bool, + model: tf.keras.models.Model, + image_size: ImageSizeArg, + patch_size: int, + pretrained_path: str, +): + """Load model weights for a known configuration. Customized model weight.""" + image_size_tuple = interpret_image_size(image_size) + local_filepath = pretrained_path + utils.load_weights_numpy_customized( + model=model, + params_path=local_filepath, + pretrained_top=pretrained_top, + num_x_patches=image_size_tuple[1] // patch_size, + num_y_patches=image_size_tuple[0] // patch_size, + ) + +def vit_b16_load_pretrain( + image_size: ImageSizeArg = (224, 224), + classes=1000, + activation="linear", + include_top=True, + pretrained=True, + pretrained_top=True, + pretrained_path="", + weights="imagenet21k+imagenet2012", +): + """Build ViT-B16. All arguments passed to build_model.""" + model = build_model( + **CONFIG_B, + name="vit-b16", + patch_size=16, + image_size=image_size, + classes=classes, + activation=activation, + include_top=include_top, + representation_size=768 if weights == "imagenet21k" else None, + ) + + # load_pretrained_customized( + # size="B_16", + # weights=weights, + # model=model, + # pretrained_top=pretrained_top, + # image_size=image_size, + # patch_size=16, + # pretrained_path=pretrained_path + # ) + return model + +def vit_b16( + image_size: ImageSizeArg = (224, 224), + classes=1000, + activation="linear", + include_top=True, + pretrained=True, + pretrained_top=True, + weights="imagenet21k+imagenet2012", +): + """Build ViT-B16. All arguments passed to build_model.""" + if pretrained_top: + classes = validate_pretrained_top( + include_top=include_top, + pretrained=pretrained, + classes=classes, + weights=weights, + ) + model = build_model( + **CONFIG_B, + name="vit-b16", + patch_size=16, + image_size=image_size, + classes=classes, + activation=activation, + include_top=include_top, + representation_size=768 if weights == "imagenet21k" else None, + ) + + if pretrained: + load_pretrained( + size="B_16", + weights=weights, + model=model, + pretrained_top=pretrained_top, + image_size=image_size, + patch_size=16, + ) + return model + + +def vit_b32( + image_size: ImageSizeArg = (224, 224), + classes=1000, + activation="linear", + include_top=True, + pretrained=True, + pretrained_top=True, + weights="imagenet21k+imagenet2012", +): + """Build ViT-B32. All arguments passed to build_model.""" + if pretrained_top: + classes = validate_pretrained_top( + include_top=include_top, + pretrained=pretrained, + classes=classes, + weights=weights, + ) + model = build_model( + **CONFIG_B, + name="vit-b32", + patch_size=32, + image_size=image_size, + classes=classes, + activation=activation, + include_top=include_top, + representation_size=768 if weights == "imagenet21k" else None, + ) + if pretrained: + load_pretrained( + size="B_32", + weights=weights, + model=model, + pretrained_top=pretrained_top, + patch_size=32, + image_size=image_size, + ) + return model + + +def vit_l16( + image_size: ImageSizeArg = (384, 384), + classes=1000, + activation="linear", + include_top=True, + pretrained=True, + pretrained_top=True, + weights="imagenet21k+imagenet2012", +): + """Build ViT-L16. All arguments passed to build_model.""" + if pretrained_top: + classes = validate_pretrained_top( + include_top=include_top, + pretrained=pretrained, + classes=classes, + weights=weights, + ) + model = build_model( + **CONFIG_L, + patch_size=16, + name="vit-l16", + image_size=image_size, + classes=classes, + activation=activation, + include_top=include_top, + representation_size=1024 if weights == "imagenet21k" else None, + ) + if pretrained: + load_pretrained( + size="L_16", + weights=weights, + model=model, + pretrained_top=pretrained_top, + patch_size=16, + image_size=image_size, + ) + return model + + +def vit_l32( + image_size: ImageSizeArg = (384, 384), + classes=1000, + activation="linear", + include_top=True, + pretrained=True, + pretrained_top=True, + weights="imagenet21k+imagenet2012", +): + """Build ViT-L32. All arguments passed to build_model.""" + if pretrained_top: + classes = validate_pretrained_top( + include_top=include_top, + pretrained=pretrained, + classes=classes, + weights=weights, + ) + model = build_model( + **CONFIG_L, + patch_size=32, + name="vit-l32", + image_size=image_size, + classes=classes, + activation=activation, + include_top=include_top, + representation_size=1024 if weights == "imagenet21k" else None, + ) + if pretrained: + load_pretrained( + size="L_32", + weights=weights, + model=model, + pretrained_top=pretrained_top, + patch_size=32, + image_size=image_size, + ) + return model