diff --git a/contrib/Retinaface/README.md b/contrib/Retinaface/README.md new file mode 100644 index 0000000000000000000000000000000000000000..e46390a4e09ebaae078ed53108a98842664b29d6 --- /dev/null +++ b/contrib/Retinaface/README.md @@ -0,0 +1,238 @@ +## Retinaface目标检测 + +## 1介绍 + +Retinaface基于MindX_SDK开发，在昇腾芯片上进行目标检测，并实现可视化呈现。输入单张图片，对其进行推理，输出推理结果。 + +### 1.1 支持的产品 + +本产品以昇腾310（推理）卡为硬件平台。 + +### 1.2 支持的版本 + +该项目支持的SDK版本为2.0.4，CANN版本为5.0.4。 + +### 1.3 软件方案介绍 + +表1.1 系统方案各子系统功能描述： + +| 序号 | 子系统 | 功能描述 | +| ---- | -------------- | ------------------------------------------------------------ | +| 1 | 图片输入 | 接收外部调用接口的输入视频路径，对视频进行拉流，并将拉去的裸流存储到缓冲区（buffer）中，并发送到下游插件。 | +| 2 | 模型推理插件 | 目标检测。 | +| 3 | 模型后处理插件 | 对模型输出的张量进行后处理，得到物体类型数据。 | + + + +### 1.4 代码目录结构与说明 + +本项目名为Retinaface目标检测，项目目录如下所示： + +```` +. +├── build.sh +├── run.sh +├── config +│   └── aipp.cfg +│ ├── face_Retina.cfg +├── main.py +├── test.py +├── utils.py +├── model +│   └── run.sh +├── plugin +│   └── build.sh +│   ├── CMakeLists.txt +│   ├── TotalYunetPostProcess.cpp +│   └── TotalYunetPostProcess.h +├── README.md +```` + + + +### 1.5 技术实现流程图 + +![流程图](images/process.png) + + + +### 1.6 特性及适用场景 + +本项目根据widerface数据集训练得到，适用于人脸检测，并且将人脸位置与五官位置标出。 + +本项目在绝大多数情况下准确，但是存在以下检测异常的情况： +1.正常检测中，由于模型本身的限制，会出现部分物体检测不到/检测错误； +2.图片中无检测目标时，会出现可视化异常，输出结果无图片信息 + + + + +## 2 环境依赖 + +推荐系统为ubuntu 18.04,环境软件和版本如下： + +| 软件名称 | 版本 | 说明 | 获取方式 | +| ------------------- | ----- | ----------------------------- | :-------------------------------------------------------- | +| MindX SDK | 2.0.4 | mxVision软件包 | [链接](https://www.hiascend.com/software/Mindx-sdk) | +| ubuntu | 18.04 | 操作系统 | 请上ubuntu官网获取 | +| Ascend-CANN-toolkit | 5.0.4 | Ascend-cann-toolkit开发套件包 | [链接](https://www.hiascend.com/software/cann/commercial) | + + + +在编译运行项目前，需要设置环境变量： + +MindSDK 环境变量： + +``` +. ${SDK-path}/set_env.sh +``` + +CANN 环境变量： + +``` +. ${ascend-toolkit-path}/set_env.sh +``` + +- 环境变量介绍 + +``` +SDK-path: mxVision SDK 安装路径 +ascend-toolkit-path: CANN 安装路径。 +``` + +## 3 软件依赖说明 + +本项目无特定软件依赖。 + +## 4.推理文件准备 +进入项目根目录，执行以下命令，创建所需文件夹： +``` +mkdir include +mkdir -p evaluate/widerface_txt/ +``` +下载本模型论文官方源码，可通过git clone下载： +``` +git clone https://github.com/biubug6/Pytorch_Retinaface.git +``` +或者直接下载zip代码包解压。 + +1、准备include目录中的文件 + +* 将Pytorch_Retinaface项目中的 + * Pytorch_Retinaface/layers/functions/prior_box.py + * Pytorch_Retinaface/utils/box_utils.py + * Pytorch_Retinaface/utils/nms/py_cpu_nms.py + + 放入本项目include文件夹下。 + +2、准备evaluate目录中的文件 +* 将Pytorch_Retinaface项目中的 + * Pytorch_Retinaface/widerface_evaluate + + 文件夹放入本项目evaluate文件夹下。 + +3、编译测试依赖代码 +进入evaluate/widerface_evaluate路径下： +``` + python3 setup.py build_ext --inplace +``` + +4、准备模型及标签文件 +在ModelZoo社区下载“ATC Retinaface(FP16) from Pytorch.zip”模型代码包并上传至服务器解压 +* 将模型代码包中的"retinaface.onnx"模型拷贝至项目根目录的"model"目录下 +* 将模型代码包中的"Retinaface/data/widerface/val/wider_val.txt"标签文件拷贝至"evaluate"目录下； + +## 4 模型转化 + +本项目中使用的模型是Retinaface模型，onnx模型可以直接[下载](https://www.hiascend.com/zh/software/modelzoo/models/detail/1/7270b02a457d4c4ab262277a646517f9)。下载后解包，得到`Retinaface.onnx`，使用模型转换工具ATC将onnx模型转换为om模型，模型转换工具相关介绍参考[链接](https://support.huaweicloud.com/tg-cannApplicationDev330/atlasatc_16_0005.html) + +模型转换步骤如下： + +1、按照2环境依赖设置环境变量 + +2、`cd`到`model`文件夹，运行 + +```` +bash run.sh +```` + +3、执行该命令后会在指定输出.om模型路径生成项目指定模型文件newRetinaface.om。若模型转换成功则输出： + +``` +ATC start working now, please wait for a moment. +ATC run success, welcome to the next use. +``` + +aipp文件配置如下： + +``` +aipp_op { +aipp_mode: static + +input_format :RGB888_U8 +src_image_size_w :1000 +src_image_size_h :1000 + +mean_chn_0 :104 +mean_chn_1 :117 +mean_chn_2 :123 + +var_reci_chn_0 :1 +var_reci_chn_1 :1 +var_reci_chn_2 :1 +} + +``` + +## 5 编译运行 + +`main.py`：用来生成单张图片推理的可视化结果，以提供推理模型的应用实例。 + +1、在项目根目录下，cd到plugin目录，并执行以下命令进行编译： + +``` +mkdir build +cd build +cmake .. +make -j +make install +``` + +将build文件夹下`libtotalyunetpostprocess.so`修改权限为`640` +并且复制到MindSDK安装路径的`lib/modelpostprocessors`目录。 + + +2、查看项目根目录下的config/aipp.cfg权限是否为640，若不是请修改。 + + + +3、准备好测试图片`test.jpg`，放置在项目根目录。 + +4、运行`main.py`程序 + +在根目录，运行 + +```` +bash run.sh +```` + +最后会得到`result.jpg`即为输出结果 + + + +## 6 精度验证 + +本模型使用widerface数据集进行精度评估。 + +1.[下载](https://share.weiyun.com/5ot9Qv1)数据集放到Retinaface目录下 + +2. 打开test.py文件,在开头修改路径参数： +* RNDB修改为widerface验证集的位置。 +* RNDY修改为保存结果txt文件的文件夹位置。 + +3.在Retinaface目录运行 +``` +python3 test.py +``` +4、在`evaluate/widerface_evaluate`目录运行`python3 evaluation.py -p -g `，等待一段时间后即可得到结果。其中与2中RNDB相同，是widerface_evaluate中的groun_truth文件夹。 + diff --git a/contrib/Retinaface/build.sh b/contrib/Retinaface/build.sh new file mode 100644 index 0000000000000000000000000000000000000000..0e678a3a6389446cc803205615eab4340d31efca --- /dev/null +++ b/contrib/Retinaface/build.sh @@ -0,0 +1,37 @@ +#!/bin/bash +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e +current_folder="$( cd "$(dirname "$0")" ;pwd -P )" + + +SAMPLE_FOLDER=( +/plugin/ +) + + +err_flag=0 +for sample in "${SAMPLE_FOLDER[@]}";do + cd "${current_folder}/${sample}" + bash build.sh || { + echo -e "Failed to build ${sample}" + err_flag=1 + } +done + +if [ ${err_flag} -eq 1 ]; then + exit 1 +fi +exit 0 \ No newline at end of file diff --git a/contrib/Retinaface/config/aipp.cfg b/contrib/Retinaface/config/aipp.cfg new file mode 100644 index 0000000000000000000000000000000000000000..47339dd2827dc08f914e6d5ded8cb74eaa1b9630 --- /dev/null +++ b/contrib/Retinaface/config/aipp.cfg @@ -0,0 +1,15 @@ +aipp_op { +aipp_mode: static + +input_format :RGB888_U8 +src_image_size_w :1000 +src_image_size_h :1000 + +mean_chn_0 :104 +mean_chn_1 :117 +mean_chn_2 :123 + +var_reci_chn_0 :1 +var_reci_chn_1 :1 +var_reci_chn_2 :1 +} diff --git a/contrib/Retinaface/config/face_Retina.cfg b/contrib/Retinaface/config/face_Retina.cfg new file mode 100644 index 0000000000000000000000000000000000000000..30e9ff3c25931b11dbabf613f210b923bfdbd1f9 --- /dev/null +++ b/contrib/Retinaface/config/face_Retina.cfg @@ -0,0 +1,4 @@ +CLASS_NUM=1 +SCORE_THRESH=0.8 +IOU_THRESH=0.9 + diff --git a/contrib/Retinaface/images/process.png b/contrib/Retinaface/images/process.png new file mode 100644 index 0000000000000000000000000000000000000000..d6cabc352174c741be4f649a7bea8ab850a7df4d Binary files /dev/null and b/contrib/Retinaface/images/process.png differ diff --git a/contrib/Retinaface/main.py b/contrib/Retinaface/main.py new file mode 100644 index 0000000000000000000000000000000000000000..eac04925cf6bef0748dfbe0412ce2a3a30f82523 --- /dev/null +++ b/contrib/Retinaface/main.py @@ -0,0 +1,142 @@ +#!/usr/bin/env python +# coding=utf-8 + +""" +Copyright(C) 2021. Huawei Technologies Co.,Ltd. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import shutil +import json +import argparse +import os +import tqdm +import numpy as np +import cv2 +import MxpiDataType_pb2 as MxpiDataType +from PIL import Image +from utils import preprocess +from StreamManagerApi import StreamManagerApi, MxDataInput, StringVector +from StreamManagerApi import StreamManagerApi, MxProtobufIn, InProtobufVector, StringVector + +if __name__ == '__main__': + streamManagerApi = StreamManagerApi() + ret = streamManagerApi.InitManager() + if ret != 0: + print("Failed to init Stream manager, ret=%s" % str(ret)) + exit() + + pipeline = { + "Retinaface": { + "stream_config": { + "deviceId": "3" + }, + "appsrc0": { + "props": { + "blocksize": "409600" + }, + "factory": "appsrc", + "next": "mxpi_tensorinfer0" + }, + "mxpi_tensorinfer0": { + "props": { + "singleBatchInfer": "1", + "dataSource": "appsrc0", + "modelPath": "./model/newRetinaface.om" + }, + "factory": "mxpi_tensorinfer", + "next": "mxpi_objectpostprocessor0" + }, + "mxpi_objectpostprocessor0": { + "props": { + "dataSource": "mxpi_tensorinfer0", + "postProcessConfigPath": "./config/face_Retina.cfg", + "postProcessLibPath": "libtotalyunetpostprocess.so" + }, + "factory": "mxpi_objectpostprocessor", + "next": "appsink0" + }, + "appsink0": { + "props": { + "blocksize": "409600" + }, + "factory": "appsink" + } + } + } + pipelineStr = json.dumps(pipeline).encode() + ret = streamManagerApi.CreateMultipleStreams(pipelineStr) + if ret != 0: + print("Failed to create Stream, ret=%s" % str(ret)) + exit() + + tensor_data , return_img = preprocess("./test.jpg") + tensor = tensor_data[None, :] + + STREAMNAME = b"Retinaface" + INPLUGINID = 0 + visionList = MxpiDataType.MxpiVisionList() + visionVec = visionList.visionVec.add() + visionInfo = visionVec.visionInfo + + visionInfo.width = 1000 + visionInfo.height = 1000 + visionInfo.widthAligned = 1000 + visionInfo.heightAligned = 1000 + visionData = visionVec.visionData + visionData.dataStr = tensor.tobytes() + visionData.deviceId = 0 + visionData.memType = 0 + visionData.dataSize = len(tensor) + + KEY_VALUE = b"appsrc0" + protobufVec = InProtobufVector() + + protobuf = MxProtobufIn() + protobuf.key = KEY_VALUE + protobuf.type = b"MxTools.MxpiVisionList" + protobuf.protobuf = visionList.SerializeToString() + protobufVec.push_back(protobuf) + + uniqueId = streamManagerApi.SendProtobuf(STREAMNAME, INPLUGINID, protobufVec) + + keys = [b'mxpi_objectpostprocessor0'] + keyVec = StringVector() + for key in keys: + keyVec.push_back(key) + infer_result = streamManagerApi.GetProtobuf(STREAMNAME, 0, keyVec) + if infer_result.size() == 0: + print("infer_result is null") + exit() + + if infer_result[0].errorCode != 0: + print("GetProtobuf error. errorCode=%d" % (infer_result[0].errorCode)) + exit() + objectList = MxpiDataType.MxpiObjectList() + objectList.ParseFromString(infer_result[0].messageBuf) + + img = cv2.imread('test.jpg') + result = objectList.objectVec + resize , left, top, right, bottom = return_img + for x in result: + new_x0 = max(int((x.x0-left)/resize), 0) + new_x1 = max(int((x.x1-left)/resize), 0) + new_y0 = max(int((x.y0-top)/resize), 0) + new_y1 = max(int((x.y1-top)/resize), 0) + + confidence = x.classVec[0].confidence + cv2.rectangle(img, (new_x0, new_y0), (new_x1, new_y1), (255, 0, 0), 2) + + cv2.imwrite("./result.jpg", img) + streamManagerApi.DestroyAllStreams() diff --git a/contrib/Retinaface/model/run.sh b/contrib/Retinaface/model/run.sh new file mode 100644 index 0000000000000000000000000000000000000000..1772eb922a0f481530d664059f1157fd84115986 --- /dev/null +++ b/contrib/Retinaface/model/run.sh @@ -0,0 +1,18 @@ +#!/bin/bash + + +# 设置环境变量（请确认install_path路径是否正确） +# Set environment PATH (Please confirm that the install_path is correct). + +export install_path=/usr/local/Ascend/ascend-toolkit/latest +export PATH=/usr/local/python3/bin:${install_path}/atc/ccec_compiler/bin:${install_path}/atc/bin:$PATH +export PYTHONPATH=${install_path}/atc/python/site-packages:${install_path}/atc/python/site-packages/auto_tune.egg/auto_tune:${install_path}/atc/python/site-packages/schedule_search.egg +export LD_LIBRARY_PATH=${install_path}/atc/lib64:$LD_LIBRARY_PATH +export ASCEND_OPP_PATH=${install_path}/opp + + +# 执行，转换Retinaface模型 +# Execute, transform Retinaface model. + +atc --framework=5 --model=retinaface.onnx --output=newRetinaface --input_format=NCHW --input_shape="image:1,3,1000,1000" --log=debug --soc_version=Ascend310 --insert_op_conf=../config/aipp.cfg +# 说明：out_nodes制定了输出节点的顺序，需要与模型后处理适配。 diff --git a/contrib/Retinaface/plugin/CMakeLists.txt b/contrib/Retinaface/plugin/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..46341a0c18adfca7a98531f1f3ddc806c3e9c099 --- /dev/null +++ b/contrib/Retinaface/plugin/CMakeLists.txt @@ -0,0 +1,31 @@ +cmake_minimum_required(VERSION 3.5.2) +project(totalyunetpostprocess) + +add_definitions(-D_GLIBCXX_USE_CXX11_ABI=0) +add_definitions(-Dgoogle=mindxsdk_private) + +set(PLUGIN_NAME "totalyunetpostprocess") +set(TARGET_LIBRARY ${PLUGIN_NAME}) +set(MX_SDK_HOME "$ENV{MX_SDK_HOME}") + +include_directories(${MX_SDK_HOME}/include) +include_directories(${MX_SDK_HOME}/opensource/include) +include_directories(${MX_SDK_HOME}/opensource/include/gstreamer-1.0) +include_directories(${MX_SDK_HOME}/opensource/include/glib-2.0) +include_directories(${MX_SDK_HOME}/opensource/lib/glib-2.0/include) +include_directories(${MX_SDK_HOME}/opensource/include/opencv4) + +link_directories(${MX_SDK_HOME}/opensource/lib/) +link_directories(${MX_SDK_HOME}/lib) + +add_compile_options(-std=c++11 -fPIC -fstack-protector-all -pie -Wno-deprecated-declarations) +add_compile_options("-DPLUGIN_NAME=${PLUGIN_NAME}") + +add_definitions(-DENABLE_DVPP_INTERFACE) +add_library(${TARGET_LIBRARY} SHARED TotalYunetPostProcess.cpp) + +target_link_libraries(${TARGET_LIBRARY} glib-2.0 gstreamer-1.0 gobject-2.0 gstbase-1.0 gmodule-2.0) +target_link_libraries(${TARGET_LIBRARY} plugintoolkit mxpidatatype mxbase opencv_world) +target_link_libraries(${TARGET_LIBRARY} -Wl,-z,relro,-z,now,-z,noexecstack -s) + +install(TARGETS ${TARGET_LIBRARY} LIBRARY DESTINATION ${MX_SDK_HOME}/lib/modelpostprocessors) diff --git a/contrib/Retinaface/plugin/TotalYunetPostProcess.cpp b/contrib/Retinaface/plugin/TotalYunetPostProcess.cpp new file mode 100644 index 0000000000000000000000000000000000000000..29972a1e550432b0b58590ca7ea6886358cd4483 --- /dev/null +++ b/contrib/Retinaface/plugin/TotalYunetPostProcess.cpp @@ -0,0 +1,283 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "TotalYunetPostProcess.h" +#include "MxBase/Log/Log.h" +#include +namespace { + const uint32_t LEFTTOPX = 0; + const uint32_t LEFTTOPY = 1; + const uint32_t RIGHTTOPX = 2; + const uint32_t RIGHTTOPY = 3; + const int PRIOR_PARAMETERS[3][2] = {{16, 32}, {64, 128}, {256, 512}}; + const int PRIOR_PARAMETERS_COUNT = 2; + const float IMAGE_WIDTH = 1000.0; + const float IMAGE_HEIGHT = 1000.0; + const float STEPS[3] = {8.0, 16.0, 32.0}; + const float VARIANCE[2] = {0.1, 0.2}; + const uint32_t RECTANGLEPOINT = 4; + const uint32_t KEYPOINTNUM = 5; + const uint32_t POINT_SIZE = 1; + const uint32_t DIM = 2; + const uint32_t RECTANGLE_COLOR = 1; + const uint32_t KEYPOINT_COLOR = 2; + const uint32_t DIV_TWO = 2; +} +namespace MxBase { + TotalYunetPostProcess& TotalYunetPostProcess::operator=(const TotalYunetPostProcess& other) + { + if (this == &other) { + return *this; + } + ObjectPostProcessBase::operator=(other); + return *this; + } + + APP_ERROR TotalYunetPostProcess::Init(const std::map >& postConfig) + { + LogInfo << "Start to Init TotalYunetPostProcess."; + APP_ERROR ret = ObjectPostProcessBase::Init(postConfig); + if (ret != APP_ERR_OK) { + LogError << GetError(ret) << "Fail to superInit in ObjectPostProcessBase."; + return ret; + } + LogInfo << "End to Init TotalYunetPostProcess."; + return APP_ERR_OK; + } + + APP_ERROR TotalYunetPostProcess::DeInit() + { + return APP_ERR_OK; + } + + void TotalYunetPostProcess::ObjectDetectionOutput(const std::vector & tensors, + std::vector >& objectInfos, + const std::vector & resizedImageInfos) + { + LogInfo << "TotalYunetPostProcess start to write results."; + + for (auto num : { objectInfoTensor_, objectConfTensor_ }) { + if ((num >= tensors.size()) || (num < 0)) { + LogError << GetError(APP_ERR_INVALID_PARAM) << "TENSOR(" << num + << ") must ben less than tensors'size(" << tensors.size() << ") and larger than 0."; + } + } + auto loc = tensors[0].GetBuffer(); + auto conf = tensors[1].GetBuffer(); + auto shape = tensors[0].GetShape(); + auto keyshape = tensors[2].GetShape(); + auto key = tensors[2].GetBuffer(); + + cv::Mat PriorBox; + cv::Mat location = cv::Mat(shape[1], shape[2], CV_32FC1, tensors[0].GetBuffer()); + cv::Mat keylocation = cv::Mat(keyshape[1], keyshape[2], CV_32FC1, tensors[2].GetBuffer()); + GeneratePriorBox(PriorBox); + + float width_resize = resizedImageInfos[0].widthResize; + float height_resize = resizedImageInfos[0].heightResize; + float width_original = resizedImageInfos[0].widthOriginal; + float height_original = resizedImageInfos[0].heightOriginal; + float width_resize_scale = width_resize / width_original; + float height_resize_scale = height_resize / height_original; + float resize_scale_factor = 1.0; + if (width_resize_scale >= height_resize_scale) { + resize_scale_factor = height_resize_scale; + } else { + resize_scale_factor = width_resize_scale; + } + + cv::Mat res = decode_for_loc(location, PriorBox, keylocation, resize_scale_factor); + + uint32_t batchSize = shape[0]; + uint32_t VectorNum = shape[1]; + + std::map match; + for (uint32_t i = 0; i < batchSize; i++) { + std::vector objectInfo; + std::vector objectInfoSorted; + std::vector keypointInfo; + std::vector keypointInfoSorted; + auto dataPtr_Conf = (float *) tensors[1].GetBuffer() + i * tensors[1].GetByteSize() / batchSize; + + for (uint32_t j = 0; j < VectorNum; j++) { + float* begin_Conf = dataPtr_Conf + j * 2; + float conf = *(begin_Conf + 1); + + if (conf > confThresh_) { + ObjectInfo objInfo; + objInfo.confidence = j; + objInfo.x0 = res.at(j, LEFTTOPX) * IMAGE_WIDTH / width_resize_scale; + objInfo.y0 = res.at(j, LEFTTOPY) * IMAGE_HEIGHT / height_resize_scale; + objInfo.x1 = res.at(j, RIGHTTOPX) * IMAGE_WIDTH / width_resize_scale; + objInfo.y1 = res.at(j, RIGHTTOPY) * IMAGE_HEIGHT / height_resize_scale; + objInfo.classId = RECTANGLE_COLOR; + + objectInfo.push_back(objInfo); + } + } + MxBase::NmsSort(objectInfo, iouThresh_); + + for (uint32_t j = 0; j < objectInfo.size(); j++) { + ObjectInfo obj = objectInfo[j]; + KeyPointDetectionInfo kpInfo; + int keypoint_Pos = objectInfo[j].confidence; + float* begin_Conf = dataPtr_Conf + keypoint_Pos * 2; + float conf = *(begin_Conf + 1); + objectInfo[j].confidence = conf; + objectInfoSorted.push_back(objectInfo[j]); + + for (int k = 0; k < KEYPOINTNUM; k++) + { + float x = res.at(keypoint_Pos, RECTANGLEPOINT + k * DIM) * IMAGE_WIDTH / width_resize_scale; + float y = res.at(keypoint_Pos, RECTANGLEPOINT + k * DIM + 1) * IMAGE_HEIGHT / height_resize_scale; + ObjectInfo objInfo; + + objInfo.x0= x - POINT_SIZE; + objInfo.x1= x + POINT_SIZE; + objInfo.y0= y - POINT_SIZE; + objInfo.y1= y + POINT_SIZE; + objInfo.confidence = 0; + objInfo.classId = KEYPOINT_COLOR; + objectInfoSorted.push_back(objInfo); + } + } + + objectInfos.push_back(objectInfoSorted); + } + LogInfo << "TotalYunetPostProcess write results successed."; + } + APP_ERROR TotalYunetPostProcess::Process(const std::vector &tensors, + std::vector> &objectInfos, + const std::vector &resizedImageInfos, + const std::map> &configParamMap) + { + LogInfo << "Start to Process TotalYunetPostProcess."; + APP_ERROR ret = APP_ERR_OK; + auto inputs = tensors; + ret = CheckAndMoveTensors(inputs); + if (ret != APP_ERR_OK) { + LogError << "CheckAndMoveTensors failed. ret=" << ret; + return ret; + } + ObjectDetectionOutput(inputs, objectInfos, resizedImageInfos); + LogInfo << "End to Process TotalYunetPostProcess."; + return APP_ERR_OK; + } + + /* + * @description: Generate prior boxes for detection boxes decoding + * @param anchors A Matrix used to save prior boxes that contains box coordinates(x0,y0,x1,y1), shape[21824,4] + */ + void TotalYunetPostProcess::GeneratePriorBox(cv::Mat &anchors) + { + std::vector>feature_maps(RIGHTTOPY, std::vector(DIM)); + for (int i = 0; i < feature_maps.size(); i++) { + feature_maps[i][0] = ceil(IMAGE_HEIGHT / STEPS[i]); + feature_maps[i][1] = ceil(IMAGE_WIDTH / STEPS[i]); + } + for (int k = 0; k < feature_maps.size(); k++) { + auto f = feature_maps[k]; + + float step = (float)STEPS[k]; + for (int i = 0; i < f[0]; i++) { + for (int j = 0; j < f[1]; j++) { + for (int l = 0; l < PRIOR_PARAMETERS_COUNT && PRIOR_PARAMETERS[k][l] != -1; l++) { + float min_size = PRIOR_PARAMETERS[k][l]; + cv::Mat anchor(1, RECTANGLEPOINT * DIM, CV_32F); + float center_x = (j + 0.5f) * step; + float center_y = (i + 0.5f) * step; + + float xmin = (center_x - min_size / 2.f) / IMAGE_WIDTH; + float ymin = (center_y - min_size / 2.f) / IMAGE_HEIGHT; + float xmax = (center_x + min_size / 2.f) / IMAGE_WIDTH; + float ymax = (center_y + min_size / 2.f) / IMAGE_HEIGHT; + + float prior_width = xmax - xmin; + float prior_height = ymax - ymin; + float prior_center_x = (xmin + xmax) / 2; + float prior_center_y = (ymin + ymax) / 2; + + anchor.at(0, LEFTTOPX) = center_x / IMAGE_WIDTH; + anchor.at(0, LEFTTOPY) = center_y / IMAGE_HEIGHT; + anchor.at(0, RIGHTTOPX) = min_size / IMAGE_WIDTH; + anchor.at(0, RIGHTTOPY) = min_size / IMAGE_HEIGHT; + + anchor.at(0, LEFTTOPX + RECTANGLEPOINT) = prior_width; + anchor.at(0, LEFTTOPY + RECTANGLEPOINT) = prior_height; + anchor.at(0, RIGHTTOPX + RECTANGLEPOINT) = prior_center_x; + anchor.at(0, RIGHTTOPY + RECTANGLEPOINT) = prior_center_y; + + anchors.push_back(anchor); + } + } + } + } + } + /* + * @description: Generate prior boxes for detection boxes decoding + * @param loc: The matrix which contains box biases, shape[21824, 4] + * @param prior: The matrix which contains prior box coordinates, shape[21824,4] + * @param resize_scale_factor: The factor of min(WidthOriginal/WidthResize, HeightOriginal/HeightResize) + * @param boxes: The matrix which contains detection box coordinates(x0,y0,x1,y1), shape[21824,4] + */ + cv::Mat TotalYunetPostProcess::decode_for_loc(cv::Mat &loc, cv::Mat &prior, cv::Mat &key, float resize_scale_factor) { + LogInfo << loc.rows; + LogInfo << loc.cols; + LogInfo << prior.rows; + LogInfo << prior.cols; + LogInfo << key.rows; + LogInfo << key.cols; + cv::Mat loc_first = loc.colRange(0, 2); + cv::Mat loc_last = loc.colRange(2, 4); + cv::Mat prior_first = prior.colRange(0, 2); + cv::Mat prior_last = prior.colRange(2, 4); + cv::Mat prior_first2 = prior.colRange(4, 6); + cv::Mat prior_last2 = prior.colRange(6, 8); + cv::Mat facepoint = key.colRange(0, 10); + cv::Mat boxes1 = prior_first + (loc_first * VARIANCE[0]).mul(prior_last); + cv::Mat boxes2; + cv::exp(loc_last * VARIANCE[1], boxes2); + boxes2 = boxes2.mul(prior_last); + boxes1 = boxes1 - boxes2 / DIV_TWO; + boxes2 = boxes2 + boxes1; + cv::Mat boxes3; + for (int i = 0; i < KEYPOINTNUM; i++) + { + cv::Mat singlepoint = facepoint.colRange(i * 2, (i + 1) * 2); + singlepoint = prior_last2 + (singlepoint * VARIANCE[0]).mul(prior_first2); + if (i == 0) boxes3 = singlepoint; + else cv::hconcat(boxes3, singlepoint, boxes3); + } + + cv::Mat boxes; + cv::hconcat(boxes1, boxes2, boxes); + cv::hconcat(boxes, boxes3, boxes); + if (resize_scale_factor == 0) { + LogError << "resize_scale_factor is 0."; + } + return boxes; + } + + extern "C" { + std::shared_ptr GetObjectInstance() + { + LogInfo << "Begin to get TotalYunetPostProcess instance."; + auto instance = std::make_shared(); + LogInfo << "End to get TotalYunetPostProcess instance."; + return instance; + } + } +} diff --git a/contrib/Retinaface/plugin/TotalYunetPostProcess.h b/contrib/Retinaface/plugin/TotalYunetPostProcess.h new file mode 100644 index 0000000000000000000000000000000000000000..b65a9cb79c7ace644276c3568f03c41009af623e --- /dev/null +++ b/contrib/Retinaface/plugin/TotalYunetPostProcess.h @@ -0,0 +1,67 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef Yunet_POST_PROCESS_H +#define Yunet_POST_PROCESS_H +#include "MxBase/PostProcessBases/ObjectPostProcessBase.h" +#include "MxBase/CV/ObjectDetection/Nms/Nms.h" +#include "opencv2/opencv.hpp" +#define DEFAULT_OBJECT_CONF_TENSOR 1 +#define DEFAULT_OBJECT_INFO_TENSOR 0 +#define DEFAULT_IOU_THRESH 0.4 +#define DEFAULT_CONFIDENCE_THRESH 0.2 + +namespace MxBase { + bool operator<(const ObjectInfo &a, const ObjectInfo &b) { + return a.confidence < b.confidence; + } + + class TotalYunetPostProcess : public ObjectPostProcessBase { + public: + TotalYunetPostProcess() = default; + + ~TotalYunetPostProcess() = default; + + TotalYunetPostProcess(const TotalYunetPostProcess& other); + + TotalYunetPostProcess& operator=(const TotalYunetPostProcess& other); + + APP_ERROR Init(const std::map >& postConfig) override; + + APP_ERROR DeInit() override; + + APP_ERROR Process(const std::vector &tensors, std::vector> &objectInfos, + const std::vector &resizedImageInfos = {}, + const std::map> &configParamMap = {}) override; + + protected: + void GeneratePriorBox(cv::Mat &anchors); + cv::Mat decode_for_loc(cv::Mat &loc, cv::Mat &prior, cv::Mat &key, float resize_scale_factor); + void ObjectDetectionOutput(const std::vector & tensors, + std::vector >& objectInfos, + const std::vector & resizedImageInfos = {}); + private: + uint32_t objectConfTensor_ = DEFAULT_OBJECT_CONF_TENSOR; + uint32_t objectInfoTensor_ = DEFAULT_OBJECT_INFO_TENSOR; + float iouThresh_ = DEFAULT_IOU_THRESH; + float confThresh_ = DEFAULT_CONFIDENCE_THRESH; + }; + + extern "C" { + std::shared_ptr GetObjectInstance(); + } +} +#endif diff --git a/contrib/Retinaface/plugin/build.sh b/contrib/Retinaface/plugin/build.sh new file mode 100644 index 0000000000000000000000000000000000000000..006f86ba8c189fb585e3523b066f5e0a4ec66c07 --- /dev/null +++ b/contrib/Retinaface/plugin/build.sh @@ -0,0 +1,36 @@ +#!/bin/bash +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License.mitations under the License. + +set -e + +current_folder="$( cd "$(dirname "$0")" ;pwd -P )" + +function build_plugin() { + build_path=$current_folder/build + if [ -d "$build_path" ]; then + rm -rf "$build_path" + else + echo "file $build_path is not exist." + fi + mkdir -p "$build_path" + cd "$build_path" + cmake .. + make -j + cd .. + exit 0 +} + +build_plugin +exit 0 \ No newline at end of file diff --git a/contrib/Retinaface/run.sh b/contrib/Retinaface/run.sh new file mode 100644 index 0000000000000000000000000000000000000000..65177cb838ab57905d02ed00d760b13ada5632c7 --- /dev/null +++ b/contrib/Retinaface/run.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +CUR_PATH=$(cd "$(dirname "$0")" || { warn "Failed to check path/to/run.sh" ; exit ; } ; pwd) + +# Simple log helper functions +info() { echo -e "\033[1;34m[INFO ][MxStream] $1\033[1;37m" ; } +warn() { echo >&2 -e "\033[1;31m[WARN ][MxStream] $1\033[1;37m" ; } + +export LD_LIBRARY_PATH=${MX_SDK_HOME}/lib:${MX_SDK_HOME}/opensource/lib:${MX_SDK_HOME}/opensource/lib64:/usr/local/Ascend/ascend-toolkit/latest/acllib/lib64:${LD_LIBRARY_PATH} +export GST_PLUGIN_SCANNER=${MX_SDK_HOME}/opensource/libexec/gstreamer-1.0/gst-plugin-scanner +export GST_PLUGIN_PATH=${MX_SDK_HOME}/opensource/lib/gstreamer-1.0:${MX_SDK_HOME}/lib/plugins + +#to set PYTHONPATH, import the StreamManagerApi.py +export PYTHONPATH=$PYTHONPATH:${MX_SDK_HOME}/python + +python3 main.py +exit 0 diff --git a/contrib/Retinaface/test.py b/contrib/Retinaface/test.py new file mode 100644 index 0000000000000000000000000000000000000000..3ba71bfdd724a4567c834fd9e191f41574931226 --- /dev/null +++ b/contrib/Retinaface/test.py @@ -0,0 +1,139 @@ +#!/usr/bin/env python +# coding=utf-8 + +""" +Copyright(C) 2021. Huawei Technologies Co.,Ltd. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import os +import stat +import argparse +import json +import shutil +import tqdm +import cv2 +import numpy as np +import MxpiDataType_pb2 as MxpiDataType +from PIL import Image +from utils import preprocess, postprocess +from StreamManagerApi import StreamManagerApi, MxDataInput, StringVector +from StreamManagerApi import StreamManagerApi, MxProtobufIn, InProtobufVector, StringVector + +RNDB = "./widerface/val/images/" +RNDY = "./evaluate/widerface_txt" +img_addresses = [] +streamManagerApi = StreamManagerApi() +ret = streamManagerApi.InitManager() +if ret != 0: + print("Failed to init Stream manager, ret=%s" % str(ret)) + exit() + +pipeline = { + "Retinaface": { + "stream_config": { + "deviceId": "3" + }, + "appsrc0": { + "props": { + "blocksize": "409600" + }, + "factory": "appsrc", + "next": "mxpi_tensorinfer0" + }, + "mxpi_tensorinfer0": { + "props": { + "singleBatchInfer": "1", + "dataSource": "appsrc0", + "modelPath": "./model/newRetinaface.om" + }, + "factory": "mxpi_tensorinfer", + "next": "appsink0" + }, + "appsink0": { + "props": { + "blocksize": "409600" + }, + "factory": "appsink" + } + } +} + +pipelineStr = json.dumps(pipeline).encode() +ret = streamManagerApi.CreateMultipleStreams(pipelineStr) +if ret != 0: + print("Failed to create Stream, ret=%s" % str(ret)) + exit() + +FLAGS = os.O_WRONLY | os.O_CREAT +MODES = stat.S_IWUSR | stat.S_IRUSR +with os.fdopen(os.open('./evaluate/wider_val.txt', FLAGS, MODES), 'r') as fr: + for img_address in fr: + print(img_address) + tensor_data , info = preprocess(RNDB + img_address.strip('\r\n')) + tensor = tensor_data[None, :] + STREAMNAME = b"Retinaface" + INPLUGINID = 0 + visionList = MxpiDataType.MxpiVisionList() + visionVec = visionList.visionVec.add() + visionInfo = visionVec.visionInfo + + visionInfo.width = 1000 + visionInfo.height = 1000 + visionInfo.widthAligned = 1000 + visionInfo.heightAligned = 1000 + visionData = visionVec.visionData + visionData.dataStr = tensor.tobytes() + visionData.deviceId = 0 + visionData.memType = 0 + visionData.dataSize = len(tensor) + + KEY_VALUE = b"appsrc0" + protobufVec = InProtobufVector() + + protobuf = MxProtobufIn() + protobuf.key = KEY_VALUE + protobuf.type = b"MxTools.MxpiVisionList" + protobuf.protobuf = visionList.SerializeToString() + protobufVec.push_back(protobuf) + + uniqueId = streamManagerApi.SendProtobuf(STREAMNAME, INPLUGINID, protobufVec) + KEY_VALUE = b"mxpi_tensorinfer0" + keyVec = StringVector() + keyVec.push_back(KEY_VALUE) + inferResult = streamManagerApi.GetProtobuf(STREAMNAME, 0, keyVec) + + infer_list = MxpiDataType.MxpiTensorPackageList() + infer_list.ParseFromString(inferResult[0].messageBuf) + infer_data0 = infer_list.tensorPackageVec[0].tensorVec[0].dataStr + loc = np.frombuffer(infer_data0, dtype=np.float32) + infer_data1 = infer_list.tensorPackageVec[0].tensorVec[1].dataStr + conf = np.frombuffer(infer_data1, dtype=np.float32) + infer_data2 = infer_list.tensorPackageVec[0].tensorVec[2].dataStr + landms = np.frombuffer(infer_data2, dtype=np.float32) + result , count = postprocess(loc , conf , landms , info) + dir_name = RNDY + "/" + os.path.split(img_address.strip('.jpg\r\n'))[0] + if not os.path.isdir(dir_name): + os.makedirs(dir_name) + print(dir_name) + txt_name = RNDY + "/" + img_address.strip('.jpg\r\n') + '.txt' + res_name = os.path.split(img_address.strip('.jpg\r\n'))[1] + "\n" + with os.fdopen(os.open(txt_name, FLAGS, MODES), 'w') as f: + f.write(res_name) + f.write('{:d}\n'.format(count)) + f.write(result) + f.close() + +streamManagerApi.DestroyAllStreams() + diff --git a/contrib/Retinaface/utils.py b/contrib/Retinaface/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..4d91b278d4f6f276f8044f58a46aaa2e8a5f24d5 --- /dev/null +++ b/contrib/Retinaface/utils.py @@ -0,0 +1,129 @@ +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import print_function +import os +import argparse +from glob import glob +import sys +import cv2 +import torch +import numpy as np +from PIL import Image +from PIL import ImageDraw + +from include.box_utils import decode, decode_landm +from include.prior_box import PriorBox +from include.py_cpu_nms import py_cpu_nms +cfg_mnet = { + 'name': 'mobilenet0.25', + 'min_sizes': [[16, 32], [64, 128], [256, 512]], + 'steps': [8, 16, 32], + 'variance': [0.1, 0.2], + 'clip': False, + 'loc_weight': 2.0, + 'gpu_train': True, + 'batch_size': 32, + 'ngpu': 1, + 'epoch': 250, + 'decay1': 190, + 'decay2': 220, + 'image_size': 640, + 'pretrain': True, + 'return_layers': {'stage1': 1, 'stage2': 2, 'stage3': 3}, + 'in_channel': 32, + 'out_channel': 64 +} + + +def preprocess(image_path): + img_raw = cv2.imread(image_path, cv2.IMREAD_COLOR) + img = np.float32(img_raw) + # testing scale + target_size = 1000 + im_shape = img.shape + im_size_min = np.min(im_shape[0:2]) + im_size_max = np.max(im_shape[0:2]) + resize = target_size / im_size_max + img = cv2.resize(img, None, None, fx=resize, fy=resize, interpolation=cv2.INTER_NEAREST) + width_pad = target_size - img.shape[1] + left = 0 + right = width_pad + height_pad = target_size - img.shape[0] + top = 0 + bottom = height_pad + img = cv2.copyMakeBorder(img, top, bottom, left, right, cv2.BORDER_CONSTANT, value=(0, 0, 0)) + im_height, im_width, _ = img.shape + scale = torch.Tensor([img.shape[1], img.shape[0], img.shape[1], img.shape[0]]) + img = torch.from_numpy(img).unsqueeze(0).byte() + info = np.array(resize, dtype=np.float32) + return img.numpy() , info + + +def postprocess(loc , conf , landms , resize): + loc = np.reshape(loc, [1, 41236, 4]) + conf = np.reshape(conf , [1, 41236, 2]) + landms = np.reshape(landms , [1, 41236, 10]) + scale = torch.ones(4,).fill_(1000) + loc = torch.Tensor(loc) + conf = torch.Tensor(conf) + landms = torch.Tensor(landms) + priorbox = PriorBox(cfg_mnet, image_size=(1000, 1000)) + priors = priorbox.forward() + prior_data = priors.data + boxes = decode(loc.data.squeeze(0), prior_data, [0.1, 0.2]) + boxes = boxes * scale / resize + boxes = boxes.numpy() + scores = conf.squeeze(0).data.numpy()[:, 1] + landms = decode_landm(landms.data.squeeze(0), prior_data, [0.1, 0.2]) + scale1 = torch.ones(10,).fill_(1000) + + landms = landms * scale1 / resize + landms = landms.numpy() + + inds = np.where(scores > 0.02)[0] + boxes = boxes[inds] + landms = landms[inds] + scores = scores[inds] + # keep top-K before NMS + order = scores.argsort()[::-1] + boxes = boxes[order] + landms = landms[order] + scores = scores[order] + + # do NMS + dets = np.hstack((boxes, scores[:, np.newaxis])).astype(np.float32, copy=False) + keep = py_cpu_nms(dets, 0.4) + dets = dets[keep, :] + landms = landms[keep] + + dets = np.concatenate((dets, landms), axis=1) + bboxs = dets + bboxs_num = str(len(bboxs)) + "\n" + a = Image.open("test.jpg") + result = '' + count = 0 + for box in bboxs: + x = int(box[0]) + y = int(box[1]) + w = int(box[2]) - int(box[0]) + h = int(box[3]) - int(box[1]) + confidence = str(box[4]) + if confidence != 0: + count += 1 + line = str(x) + " " + str(y) + " " + str(w) + " " + str(h) + " " + confidence + " \n" + result += line + aa = ImageDraw.ImageDraw(a) + aa.rectangle((x, y, x+w, y+h), outline='red', width=3) + return result , count