diff --git a/mxVision/PPYOLOEPlusDetection/CMakeLists.txt b/mxVision/PPYOLOEPlusDetection/CMakeLists.txt index 7fc2520f6d7c6e56d1b94244e7475d908764ef13..ac78fe546a18ac7db0c29f1b7890d5c884c3008c 100644 --- a/mxVision/PPYOLOEPlusDetection/CMakeLists.txt +++ b/mxVision/PPYOLOEPlusDetection/CMakeLists.txt @@ -25,5 +25,6 @@ target_link_libraries(sample glog mxbase cpprest + opencv_world ppyoloepostprocess ) \ No newline at end of file diff --git a/mxVision/PPYOLOEPlusDetection/README.md b/mxVision/PPYOLOEPlusDetection/README.md index 918d4df7808d3d3c884712aecea13717072c15e4..a2c690f30821f997c5a177c2864ae8ce278726b0 100644 --- a/mxVision/PPYOLOEPlusDetection/README.md +++ b/mxVision/PPYOLOEPlusDetection/README.md @@ -28,20 +28,23 @@ paddlepaddle框架的ppyoloe模型推理时，前处理方案包括解码为BGR- . ├── run.sh # 编译运行main.cpp脚本 ├── main.cpp # mxBasev2接口推理样例流程 -├── PPYoloePostProcess.h # ppyoloe后处理插件编译头文件(需要被main.cpp引入) -├── PPYoloePostProcess.cpp # ppyoloe后处理插件实现 +├── plugin +│ ├── PPYoloePostProcess.h # ppyoloe后处理插件编译头文件(需要被main.cpp引入) +│ ├── PPYoloePostProcess.cpp # ppyoloe后处理插件实现 +│ └── CMakeLists.txt # 用于编译后处理插件 ├── model │ ├── coco.names # 需要下载，下载链接在下方 │ └── ppyoloe.cfg # 模型后处理配置文件，配置说明参考《mxVision用户指南》中已有模型支持->模型后处理配置参数->YOLOv5模型后处理配置参数 ├── pipeline -│ └── Sample.pipeline # 参考pipeline文件，用户需要根据自己需求和模型输入类型进行修改 +│ ├── Sample.pipeline # 参考pipeline文件，用户需要根据自己需求和模型输入类型进行修改 +│ └── SampleYuv.pipeline # 参考pipeline文件，用于配置yuv模型，用户需要根据自己需求和模型输入类型进行修改 ├── test.jpg # 需要用户自行添加测试数据 ├── CMakeLists.txt # 编译main.cpp所需的CMakeLists.txt, 编译插件所需的CMakeLists.txt请查阅用户指南 └── README.md ``` -注：coco.names文件源于[链接](../Collision/model/coco.names)的coco2014.names文件，下载之后，放到models目录下。 +注：coco.names文件源于[链接](https://gitee.com/ascend/mindxsdk-referenceapps/blob/master/contrib/Collision/model/coco.names)的coco2014.names文件，下载之后，放到models目录下。 @@ -70,7 +73,11 @@ SDK-path: mxVision SDK 安装路径 ascend-toolkit-path: CANN 安装路径。 ``` -## 3. 模型转换 +## 3. 模型转换 + +关键依赖版本说明 +paddle >=1.0.2 +paddlepaddle >= 2.3.2 **步骤1** 建议通过[链接](https://github.com/PaddlePaddle/PaddleYOLO/blob/develop/docs/MODEL_ZOO_cn.md#PP-YOLOE)中 部署模型->PP-YOLOE+_l ->导出后的权重->(w/nms)下载paddle模型。 @@ -91,8 +98,8 @@ python prune_paddle_model.py --model_dir ${input_model_dir} --model_filename ${p 参考工具[链接](https://github.com/PaddlePaddle/Paddle2ONNX/blob/develop/README.md)转换为onnx模型 **步骤4** -将onnx模型转换为om模型 -配置aipp.cfg参考: +将onnx模型转换为om模型 +RGB输入模型配置aipp.cfg参考: ``` aipp_op { aipp_mode : static @@ -101,9 +108,32 @@ aipp_op { src_image_size_h : 640 csc_switch : false - rbuv_swap_switch : true + rbuv_swap_switch : false +} +``` + +转换为YUVSP420输入模型参考 +``` +aipp_op { + aipp_mode : static + input_format : YUV420SP_U8 + csc_switch : true + rbuv_swap_switch : false + matrix_r0c0 : 256 + matrix_r0c1 : 0 + matrix_r0c2 : 359 + matrix_r1c0 : 256 + matrix_r1c1 : -88 + matrix_r1c2 : -183 + matrix_r2c0 : 256 + matrix_r2c1 : 454 + matrix_r2c2 : 0 + input_bias_0 : 0 + input_bias_1 : 128 + input_bias_2 : 128 } ``` + atc转换模型命令 ``` atc --framework=5 --model=${onnx_model} --output={output_name} --input_format=NCHW --input_shape="image:1, 3, 640, 640" --log=error --soc_version={soc_name} --insert_op_conf=${aipp_cfg_file} --output_type=FP32 @@ -135,16 +165,13 @@ atc --framework=5 --model=${onnx_model} --output={output_name} --input_format=NC 放入待测图片。将一张图片放项目根路径下，命名为 test.jpg。 **步骤3** -对main.cpp样例中加载的模型路径、模型配置文件路径进行检查，确保对应位置存在相关文件，包括： -string modelPath = "models/ppyoloe.om"; -string ppyoloeConfigPath = "models/ppyoloe.cfg"; -string ppyoloeLabelPath = "models/coco.names"; +对main.cpp样例中加载的模型路径、模型配置文件路径进行检查，确保对应位置存在相关文件，请参考run.sh中的说明。 **步骤4** 图片检测。在项目路径根目录下运行命令： ``` -bash run.sh +bash run.sh -m model_path -c model_config_path -l model_label_path -i image_path [-y] ``` ### 4.2 pipeline推理业务流程 diff --git a/mxVision/PPYOLOEPlusDetection/main.cpp b/mxVision/PPYOLOEPlusDetection/main.cpp index fa917b7635a1cd49eec42668bb638226bbf0ecc3..9e86d096ff030f9e74c2856e822f9219f0b3feb9 100644 --- a/mxVision/PPYOLOEPlusDetection/main.cpp +++ b/mxVision/PPYOLOEPlusDetection/main.cpp @@ -24,6 +24,7 @@ #include #include +#include "opencv2/imgproc.hpp" #include "MxBase/MxBase.h" #include "MxBase/MemoryHelper/MemoryHelper.h" #include "MxBase/DeviceManager/DeviceManager.h" @@ -37,6 +38,9 @@ const int MODEL_INPUT_WIDTH = 640; const int MODEL_INPUT_HEIGHT = 640; const int AVG_PARAM = 2; const long MAX_FILE_SIZE = 1024 * 1024 * 1024; // 1g +const int RGB_EXTEND = 3; +const int ARG_NUM = 10; +const int YUV_DIVISION = 2; APP_ERROR CheckFileVaild(const std::string &filePath) { @@ -131,45 +135,84 @@ APP_ERROR DvppPreprocessorYuv(ImageProcessor &imageProcessor, std::string &image return APP_ERR_OK; } -APP_ERROR DvppPreprocessor(std::string &imagePath, vector &ppyoloeInputs, +APP_ERROR OpenCVPreProcessor(std::string &imagePath, vector &ppyoloeInputs, std::vector &imagePreProcessInfos, int deviceId) +{ + auto image = cv::imread(imagePath); + cv::cvtColor(image, image, cv::COLOR_BGR2RGB); + size_t originalWidth = image.cols; + size_t originalHeight = image.rows; + float scaleWidth = MODEL_INPUT_WIDTH * 1.0 / originalWidth; + float scaleHeight = MODEL_INPUT_HEIGHT * 1.0 / originalHeight; + cv::Mat resizedImg; + cv::resize(image, resizedImg, cv::Size(MODEL_INPUT_WIDTH, MODEL_INPUT_HEIGHT), 1, 1, cv::INTER_CUBIC); + uint32_t dataSize = MODEL_INPUT_HEIGHT * MODEL_INPUT_WIDTH * RGB_EXTEND; + MxBase::Size imageSize(MODEL_INPUT_WIDTH, MODEL_INPUT_HEIGHT); + uint8_t *pasteHostData = (uint8_t *)malloc(dataSize); + if (pasteHostData == nullptr) { + return APP_ERR_ACL_BAD_ALLOC; + } + for (size_t i = 0; i < dataSize; i++) { + pasteHostData[i] = resizedImg.data[i]; + } + std::shared_ptr dataPaste((uint8_t *)pasteHostData, free); + Image pastedImage(dataPaste, dataSize, -1, imageSize, ImageFormat::RGB_888); + pastedImage.ToDevice(deviceId); + ppyoloeInputs.push_back(pastedImage.ConvertToTensor()); + ResizedImageInfo imagePreProcessInfo(MODEL_INPUT_WIDTH, MODEL_INPUT_HEIGHT, originalWidth, originalHeight, + RESIZER_STRETCHING, 0); + imagePreProcessInfos.push_back(imagePreProcessInfo); + return APP_ERR_OK; +} + +APP_ERROR DvppPreprocessor(std::string &imagePath, vector &ppyoloeInputs, + std::vector &imagePreProcessInfos, int deviceId, bool isYuvInput) { ImageProcessor imageProcessor(deviceId); - if (DeviceManager::IsAscend310P()) { - Image decodeImage; - APP_ERROR ret = imageProcessor.Decode(imagePath, decodeImage, ImageFormat::BGR_888); - if (ret != APP_ERR_OK) { - LogError << "ImageProcessor decode failed."; - return ret; - } - Image resizeImage; - uint32_t originalWidth = decodeImage.GetOriginalSize().width; - uint32_t originalHeight = decodeImage.GetOriginalSize().height; - ret = imageProcessor.Resize(decodeImage, MxBase::Size(MODEL_INPUT_WIDTH, MODEL_INPUT_HEIGHT), resizeImage, - Interpolation::BILINEAR_SIMILAR_OPENCV); - if (ret != APP_ERR_OK) { - LogError << "ImageProcessor resize failed."; - return ret; - } - ppyoloeInputs.push_back(resizeImage.ConvertToTensor()); - ResizedImageInfo imagePreProcessInfo(MODEL_INPUT_WIDTH, MODEL_INPUT_HEIGHT, originalWidth, originalHeight, - RESIZER_STRETCHING, 0); - imagePreProcessInfos.push_back(imagePreProcessInfo); - } else { + if (isYuvInput) { return DvppPreprocessorYuv(imageProcessor, imagePath, ppyoloeInputs, imagePreProcessInfos, deviceId); + } else { + if (DeviceManager::IsAscend310P()) { + Image decodeImage; + APP_ERROR ret = imageProcessor.Decode(imagePath, decodeImage, ImageFormat::RGB_888); + if (ret != APP_ERR_OK) { + LogError << "ImageProcessor decode failed."; + return OpenCVPreProcessor(imagePath, ppyoloeInputs, imagePreProcessInfos, deviceId); + } + Image resizeImage; + uint32_t originalWidth = decodeImage.GetOriginalSize().width; + uint32_t originalHeight = decodeImage.GetOriginalSize().height; + ret = imageProcessor.Resize(decodeImage, MxBase::Size(MODEL_INPUT_WIDTH, MODEL_INPUT_HEIGHT), resizeImage, + Interpolation::BILINEAR_SIMILAR_OPENCV); + if (ret != APP_ERR_OK) { + LogError << "ImageProcessor resize failed."; + return ret; + } + ppyoloeInputs.push_back(resizeImage.ConvertToTensor()); + ResizedImageInfo imagePreProcessInfo(MODEL_INPUT_WIDTH, MODEL_INPUT_HEIGHT, originalWidth, originalHeight, + RESIZER_STRETCHING, 0); + imagePreProcessInfos.push_back(imagePreProcessInfo); + } else { + return OpenCVPreProcessor(imagePath, ppyoloeInputs, imagePreProcessInfos, deviceId); + } } return APP_ERR_OK; } -APP_ERROR E2eInfer(std::string imagePath, int32_t deviceId) +APP_ERROR E2eInfer(std::map pathMap, int32_t deviceId, bool isYuvInput) { vector ppyoloeInputs; std::vector resizedImageInfos; - APP_ERROR ret = DvppPreprocessor(imagePath, ppyoloeInputs, resizedImageInfos, deviceId); + std::string imagePath = pathMap["imgPath"]; + APP_ERROR ret = CheckFileVaild(imagePath); if (ret != APP_ERR_OK) { return ret; } - string modelPath = "model/ppyoloe.om"; + ret = DvppPreprocessor(imagePath, ppyoloeInputs, resizedImageInfos, deviceId, isYuvInput); + if (ret != APP_ERR_OK) { + return ret; + } + string modelPath = pathMap["modelPath"]; ret = CheckFileVaild(modelPath); if (ret != APP_ERR_OK) { return ret; @@ -177,7 +220,7 @@ APP_ERROR E2eInfer(std::string imagePath, int32_t deviceId) Model ppyoloe(modelPath, deviceId); vector ppyoloeOutputs = ppyoloe.Infer(ppyoloeInputs); - if (ppyoloeInputs.size() == 0) { + if (ppyoloeOutputs.size() == 0) { LogError << "PPYOLOE infer failed."; return APP_ERR_COMM_FAILURE; } @@ -185,8 +228,8 @@ APP_ERROR E2eInfer(std::string imagePath, int32_t deviceId) ppyoloeOutputs[i].ToHost(); } std::vector cropConfigVec; - string ppyoloeConfigPath = "model/ppyoloe.cfg"; - string ppyoloeLabelPath = "model/ppyoloe.names"; + string ppyoloeConfigPath = pathMap["modelConfigPath"]; + string ppyoloeLabelPath = pathMap["modelLabelPath"]; ret = CheckFileVaild(ppyoloeConfigPath); if (ret != APP_ERR_OK) { @@ -201,16 +244,56 @@ APP_ERROR E2eInfer(std::string imagePath, int32_t deviceId) return APP_ERR_OK; } +void usage() +{ + std::cout << "Usage: " << std::endl; + std::cout << "./sample -m model_path -c model_config_path -l model_label_path -i image_path [-y] " << std::endl; +} + int main(int argc, char *argv[]) { MxInit(); + if (argc > ARG_NUM || argc < ARG_NUM - 1) { + usage(); + return 0; + } int32_t deviceId = 0; - std::string imgPath = "test.jpg"; - APP_ERROR ret = CheckFileVaild(imgPath); - if (ret != APP_ERR_OK) { - return ret; + bool isYuvInput = false; + std::map pathMap; + int input; + const char* optString = "i:m:c:l:yh"; + while ((input = getopt(argc, argv, optString)) != -1) { + switch (input) { + case 'm': + pathMap.insert({ "modelPath", optarg }); + break; + case 'i': + pathMap.insert({ "imgPath", optarg }); + break; + case 'c': + pathMap.insert({ "modelConfigPath", optarg }); + break; + case 'l': + pathMap.insert({ "modelLabelPath", optarg }); + break; + case 'y': + isYuvInput = true; + break; + case 'h': + usage(); + return 0; + case '?': + usage(); + return 0; + } + } + if (pathMap.count("modelPath") <= 0 || pathMap.count("imgPath") <= 0 || pathMap.count("modelConfigPath") <= 0 || + pathMap.count("modelLabelPath") <= 0) { + LogError << "Invalid input params"; + usage(); + return 0; } - ret = E2eInfer(imgPath, deviceId); + APP_ERROR ret = E2eInfer(pathMap, deviceId, isYuvInput); if (ret != APP_ERR_OK) { LogError << "Failed to run E2eInfer"; } diff --git a/mxVision/PPYOLOEPlusDetection/model/ppyoloe.cfg b/mxVision/PPYOLOEPlusDetection/model/ppyoloe.cfg index 0fa152808c762d771100c8b23a7446becfa4c646..8cbb1f8c3f10b61527fe8e8dc0130b285e3cfb9e 100644 --- a/mxVision/PPYOLOEPlusDetection/model/ppyoloe.cfg +++ b/mxVision/PPYOLOEPlusDetection/model/ppyoloe.cfg @@ -1,4 +1,4 @@ CLASS_NUM=80 SCORE_THRESH=0.5 OBJECTNESS_THRESH=0.5 -IOUT_THRESH=0.7 \ No newline at end of file +IOU_THRESH=0.7 \ No newline at end of file diff --git a/mxVision/PPYOLOEPlusDetection/pipeline/Sample.pipeline b/mxVision/PPYOLOEPlusDetection/pipeline/Sample.pipeline index 407bbc2e4850e50d56da05a91dcbc12b709b72a7..409985462f3f4241d272844f93af815668e2d533 100644 --- a/mxVision/PPYOLOEPlusDetection/pipeline/Sample.pipeline +++ b/mxVision/PPYOLOEPlusDetection/pipeline/Sample.pipeline @@ -12,15 +12,17 @@ }, "mxpi_imagedecoder0": { "props": { - "formatAdaptation" : "on" + "cvProcessor" : "opencv", + "outputDataFormat" : "RGB" }, "factory": "mxpi_imagedecoder", "next": "mxpi_imageresize0" }, "mxpi_imageresize0": { "props": { - "paddingHeight": "640", - "paddingWidth": "640" + "cvProcessor" : "opencv", + "resizeHeight": "640", + "resizeWidth": "640" }, "factory": "mxpi_imageresize", "next": "mxpi_tensorinfer0" diff --git a/mxVision/PPYOLOEPlusDetection/pipeline/SampleYuv.pipeline b/mxVision/PPYOLOEPlusDetection/pipeline/SampleYuv.pipeline new file mode 100644 index 0000000000000000000000000000000000000000..558bd75d9c0d0b7bdc0f89c60b7e161fcea5187f --- /dev/null +++ b/mxVision/PPYOLOEPlusDetection/pipeline/SampleYuv.pipeline @@ -0,0 +1,58 @@ +{ + "detection": { + "stream_config": { + "deviceId": "0" + }, + "appsrc0": { + "props": { + "blocksize": "409600" + }, + "factory": "appsrc", + "next": "mxpi_imagedecoder0" + }, + "mxpi_imagedecoder0": { + "factory": "mxpi_imagedecoder", + "next": "mxpi_imageresize0" + }, + "mxpi_imageresize0": { + "props": { + "resizeHeight": "640", + "resizeWidth": "640" + }, + "factory": "mxpi_imageresize", + "next": "mxpi_tensorinfer0" + }, + "mxpi_tensorinfer0": { + "props": { + "dataSource": "mxpi_imageresize0", + "modelPath": "../model/ppyoloe.om" + }, + "factory": "mxpi_tensorinfer", + "next": "mxpi_objectpostprocessor0" + }, + "mxpi_objectpostprocessor0": { + "props": { + "dataSource": "mxpi_tensorinfer0", + "postProcessConfigPath":"../model/ppyoloe.cfg", + "labelPath": "../model/coco.names", + "postProcessLibPath": "libppyoloepostprocess.so" + }, + "factory": "mxpi_objectpostprocessor", + "next": "mxpi_dataserialize0" + }, + "mxpi_dataserialize0": { + "props": { + "outputDataKeys": "mxpi_objectpostprocessor0" + }, + "factory": "mxpi_dataserialize", + "next": "appsink0" + }, + + "appsink0": { + "props": { + "blocksize": "4096000" + }, + "factory": "appsink" + } + } +} diff --git a/mxVision/PPYOLOEPlusDetection/plugin/CMakeLists.txt b/mxVision/PPYOLOEPlusDetection/plugin/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..b9706d52cb631d85f34b16451b5db990015f1f25 --- /dev/null +++ b/mxVision/PPYOLOEPlusDetection/plugin/CMakeLists.txt @@ -0,0 +1,29 @@ +cmake_minimum_required(VERSION 3.5.2) +project(ppyoloepostprocess) + +add_definitions(-D_GLIBCXX_USE_CXX11_ABI=0) +add_definitions(-Dgoogle=mindxsdk_private) + +set(PLUGIN_NAME "ppyoloepostprocess") +set(TARGET_LIBRARY ${PLUGIN_NAME}) + +include_directories(${PROJECT_SOURCE_DIR}/../../include) +include_directories(${PROJECT_SOURCE_DIR}/../../opensource/include) +include_directories(${PROJECT_SOURCE_DIR}/../../opensource/include/gstreamer-1.0) +include_directories(${PROJECT_SOURCE_DIR}/../../opensource/include/glib-2.0) +include_directories(${PROJECT_SOURCE_DIR}/../../opensource/lib/glib-2.0/include) + +link_directories(${PROJECT_SOURCE_DIR}/../../opensource/lib/) +link_directories(${PROJECT_SOURCE_DIR}/../../lib) + +add_compile_options(-fPIC -std=c++11 -fstack-protector-all -Wl, -z,relro,-z,now,-z,noexecstack -s -pie) +add_compile_options("-DPLUGIN_NAME=${PLUGIN_NAME}") + +add_definitions(-DENABLE_DVPP_INTERFACE) +add_library(${TARGET_LIBRARY} SHARED PPYoloePostProcess.cpp) + +target_link_libraries(${TARGET_LIBRARY} glib-2.0 gstreamer-1.0 gobject-2.0 gstbase-1.0 gmodule-2.0) +target_link_libraries(${TARGET_LIBRARY} plugintoolkit mxpidatatype mxbase) +target_link_libraries(${TARGET_LIBRARY} -Wl,-z,relro,-z,now,-z,noexecstack -s) + +install(TARGETS ${TARGET_LIBRARY} PERMISSIONS OWNER_READ GROUP_READ LIBRARY DESTINATION ${PROJECT_SOURCE_DIR}/../../lib/modelpostprocessors) \ No newline at end of file diff --git a/mxVision/PPYOLOEPlusDetection/PPYoloePostProcess.cpp b/mxVision/PPYOLOEPlusDetection/plugin/PPYoloePostProcess.cpp similarity index 89% rename from mxVision/PPYOLOEPlusDetection/PPYoloePostProcess.cpp rename to mxVision/PPYOLOEPlusDetection/plugin/PPYoloePostProcess.cpp index daf63241a6a460a06ca6fc07f015e59fd0c4ca43..59394ec4c64a88238a9caaa2f69e555c6c0aa3d1 100644 --- a/mxVision/PPYOLOEPlusDetection/PPYoloePostProcess.cpp +++ b/mxVision/PPYOLOEPlusDetection/plugin/PPYoloePostProcess.cpp @@ -41,41 +41,41 @@ PPYoloePostProcess &PPYoloePostProcess::operator = (const PPYoloePostProcess &ot APP_ERROR PPYoloePostProcess::Init(const std::map> &postConfig) { - LogDebug << "Start to Init PPYoloePostProcess. "; + LogDebug << "Start to Init PPYoloePostProcess."; APP_ERROR ret = ObjectPostProcessBase::Init(postConfig); if (ret != APP_ERR_OK) { - LogError << GetError(ret) << "Fail to superInit in ObjectPostProcessorBase"; + LogError << GetError(ret) << "Fail to superInit in ObjectPostProcessBase."; return ret; } ret = configData_.GetFileValue("OBJECTNESS_THRESH", objectnessThresh_, 0.0f, 1.0f); if (ret != APP_ERR_OK) { - LogWarn << GetError(ret) << "Fail to read OBJECTNESS_THRESH from config, default is :" << objectnessThresh_; + LogWarn << GetError(ret) << "Fail to read OBJECTNESS_THRESH from config, default is : " << objectnessThresh_; } ret = configData_.GetFileValue("IOU_THRESH", iouThresh_, 0.0f, 1.0f); if (ret != APP_ERR_OK) { - LogWarn << GetError(ret) << "Fail to read IOU_THRESH from config, default is :" << iouThresh_; + LogWarn << GetError(ret) << "Fail to read IOU_THRESH from config, default is : " << iouThresh_; } - LogDebug << "End to Init PPYoloePostProcess. "; + LogDebug << "End to Init PPYoloePostProcess."; return APP_ERR_OK; } APP_ERROR PPYoloePostProcess::Init(const std::map &postConfig) { - LogDebug << "Start to Init PPYoloePostProcess. "; + LogDebug << "Start to Init PPYoloePostProcess."; APP_ERROR ret = ObjectPostProcessBase::Init(postConfig); if (ret != APP_ERR_OK) { - LogError << GetError(ret) << "Fail to superInit in ObjectPostProcessorBase"; + LogError << GetError(ret) << "Fail to superInit in ObjectPostProcessBase."; return ret; } ret = configData_.GetFileValue("OBJECTNESS_THRESH", objectnessThresh_, 0.0f, 1.0f); if (ret != APP_ERR_OK) { - LogWarn << GetError(ret) << "Fail to read OBJECTNESS_THRESH from config, default is :" << objectnessThresh_; + LogWarn << GetError(ret) << "Fail to read OBJECTNESS_THRESH from config, default is : " << objectnessThresh_; } ret = configData_.GetFileValue("IOU_THRESH", iouThresh_, 0.0f, 1.0f); if (ret != APP_ERR_OK) { - LogWarn << GetError(ret) << "Fail to read IOU_THRESH from config, default is :" << iouThresh_; + LogWarn << GetError(ret) << "Fail to read IOU_THRESH from config, default is : " << iouThresh_; } - LogDebug << "End to Init PPYoloePostProcess. "; + LogDebug << "End to Init PPYoloePostProcess."; return APP_ERR_OK; } @@ -95,8 +95,9 @@ void PPYoloePostProcess::ConstructBoxFromOutput(float *output, float *boxOutput, classId = labelIdx; } } - if (classId < 0 || resizedImageInfo.widthOriginal == 0 || resizedImageInfo.heightOriginal == 0) + if (classId < 0 || resizedImageInfo.widthOriginal == 0 || resizedImageInfo.heightOriginal == 0) { return; + } float xGain = resizedImageInfo.widthResize * 1.0 / resizedImageInfo.widthOriginal; float yGain = resizedImageInfo.heightResize * 1.0 / resizedImageInfo.heightOriginal; auto leftX = boxOutput[offset * BOX_DIM] / xGain; @@ -104,10 +105,10 @@ void PPYoloePostProcess::ConstructBoxFromOutput(float *output, float *boxOutput, auto rightX = (boxOutput[offset * BOX_DIM + RIGHTX_IDX]) / xGain; auto rightY = (boxOutput[offset * BOX_DIM + RIGHTY_IDX]) / yGain; ObjectInfo obj; - obj.x0 = std::round(leftX); - obj.y0 = std::round(leftY); - obj.x1 = std::round(rightX); - obj.y1 = std::round(rightY); + obj.x0 = leftX; + obj.y0 = leftY; + obj.x1 = rightX; + obj.y1 = rightY; obj.confidence = maxProb; obj.classId = classId; obj.className = configData_.GetClassName(obj.classId); @@ -140,7 +141,7 @@ APP_ERROR PPYoloePostProcess::Process(const std::vector &tensors, return APP_ERR_INPUT_NOT_MATCH; } if (tensors[1].GetShape()[1] != classNum_) { - LogError << "The model output tensor[1][1] != classNum_"; + LogError << "The model output tensor[1][1] != classNum_."; return APP_ERR_INPUT_NOT_MATCH; } uint32_t batchSize = tensors[0].GetShape()[0]; @@ -167,9 +168,9 @@ APP_ERROR PPYoloePostProcess::Process(const std::vector &tensors, extern "C" { std::shared_ptr GetObjectInstance() { - LogInfo << "Begin to get PPYoloePostProcess instance"; + LogInfo << "Begin to get PPYoloePostProcess instance."; auto instance = std::make_shared(); - LogInfo << "End to get PPYoloePostProcess instance"; + LogInfo << "End to get PPYoloePostProcess instance."; return instance; } } diff --git a/mxVision/PPYOLOEPlusDetection/PPYoloePostProcess.h b/mxVision/PPYOLOEPlusDetection/plugin/PPYoloePostProcess.h similarity index 100% rename from mxVision/PPYOLOEPlusDetection/PPYoloePostProcess.h rename to mxVision/PPYOLOEPlusDetection/plugin/PPYoloePostProcess.h diff --git a/mxVision/PPYOLOEPlusDetection/run.sh b/mxVision/PPYOLOEPlusDetection/run.sh index e91a1fc9f4287ec5025d2f86ffa54806b0b571c2..79b314a20e02c9adc2f72df7a4ed8f99f4cf9b95 100644 --- a/mxVision/PPYOLOEPlusDetection/run.sh +++ b/mxVision/PPYOLOEPlusDetection/run.sh @@ -10,6 +10,81 @@ make -j || { exit ${ret} } +func() { + echo "Usage:" + echo "bash run.sh -m model_path -c model_config_path -l model_label_path -i image_path [-y]" + echo "Description:" + echo "-m model path" + echo "-c model config file path" + echo "-l label path for model" + echo "-i image path to infer" + echo "-y [Optional] use yuv model, default is not yuv" + exit -1 +} + +is_yuv=0 +argc=5 +args=0 +while getopts "i:m:c:l:yh" arg +do + if [ "$args" -gt "$argc" ]; then + echo "Error: Wrong usage, too many arguments." + func + exit 1 + fi + case "$arg" in + i) + img_path="$OPTARG" + ;; + m) + model_path="$OPTARG" + ;; + c) + model_config_path="$OPTARG" + ;; + l) + model_label_path="$OPTARG" + ;; + y) + is_yuv=1 + ;; + h) + func + exit 1 + ;; + ?) + echo "Error: Wrong usage, unknown argument." + func + exit 1 + ;; + esac + args=$(($args+1)) +done + +if [ ! -n "$model_path" ]; then + echo "Error: Required argument \"-m model_path\" is missing." + func + exit 1 +fi +if [ ! -n "$model_config_path" ]; then + echo "Error: Required argument \"-c model_config_path\" is missing." + func + exit 1 +fi +if [ ! -n "$model_label_path" ]; then + echo "Error: Required argument \"-l model_label_path\" is missing." + func + exit 1 +fi +if [ ! -n "$img_path" ]; then + echo "Error: Required argument \"-i img_path\" is missing." + func + exit 1 +fi cd .. -./sample +if [ "$is_yuv" -gt 0 ]; then + ./sample -m "$model_path" -c "$model_config_path" -l "$model_label_path" -i "$img_path" -y +else + ./sample -m "$model_path" -c "$model_config_path" -l "$model_label_path" -i "$img_path" +fi exit 0 \ No newline at end of file diff --git a/mxVision/YOLOv7Detection/README.md b/mxVision/YOLOv7Detection/README.md index 8e7db48540cc97e5622d44b999ccb5a3ee7087bc..f84d7146fa6e8ac81733cb5e06ea859007971c8a 100644 --- a/mxVision/YOLOv7Detection/README.md +++ b/mxVision/YOLOv7Detection/README.md @@ -28,21 +28,25 @@ Pytorch框架对yolov7模型推理时，前处理方案包括解码为BGR->等 . ├── run.sh # 编译运行main.cpp脚本 ├── main.cpp # mxBasev2接口推理样例流程 -├── Yolov7PostProcess.h # yolov7后处理插件编译头文件(需要被main.cpp引入) -├── Yolov7PostProcess.cpp # yolov7后处理插件实现 +├── plugin +│ ├── Yolov7PostProcess.h # yolov7后处理插件编译头文件(需要被main.cpp引入) +│ ├── Yolov7PostProcess.cpp # yolov7后处理插件实现 +│ └── CMakeLists.txt # 用于编译后处理插件 ├── model │ ├── coco.names # 需要下载，下载链接在下方 │ └── yolov7.cfg # 模型后处理配置文件，配置说明参考《mxVision用户指南》中已有模型支持->模型后处理配置参数->YOLOv5模型后处理配置参数 ├── pipeline -│ └── Sample.pipeline # 参考pipeline文件，用户需要根据自己需求和模型输入类型进行修改 +│ ├── Sample.pipeline # 参考pipeline文件，用户需要根据自己需求和模型输入类型进行修改 +│ └── SampleYuv.pipeline # 参考pipeline文件，用于配置yuv模型，用户需要根据自己需求和模型输入类型进行修改， ├── CMakeLists.txt # 编译main.cpp所需的CMakeLists.txt, 编译插件所需的CMakeLists.txt请查阅用户指南 ├── test.jpg # 需要用户自行添加测试数据 └── README.md ``` -注：coco.names文件源于[链接](../Collision/model/coco.names)的coco2014.names文件，下载之后，放到models目录下。 - +注：coco.names文件源于[链接](https://gitee.com/ascend/mindxsdk-referenceapps/blob/master/contrib/Collision/model/coco.names)的coco2014.names文件，下载之后，放到models目录下。 +另外，yolov7.cfg中新增了一个配置项PADDING_TYPE用于区分补边的情况，若采用dvpp补边则填写0，采用opencv补边则填写1，默认为1。 +SampleYuv.pipeline中resize插件需要选用双线性插值的方式，需要根据310和310P环境填写interpolation的参数。在310上面需要设置为1。 ## 2 环境依赖 @@ -68,13 +72,65 @@ CANN 环境变量： ``` SDK-path: mxVision SDK 安装路径 ascend-toolkit-path: CANN 安装路径。 -``` +``` + +## 3. 模型转换 -## 3. 模型转换 +关键依赖版本说明 +PyTorch >=1.8.0 请参考[链接](https://gitee.com/ascend/modelzoo-GPL/tree/master/built-in/ACL_Pytorch/Yolov7_for_Pytorch)对模型进行下载和转换为om。 注意：由于main.cpp样例在310P环境下解码后的图片为BGR格式，因此使用aipp转换至om时，请将上述链接中的教程中 4. 使用aipp预处理 aipp_op中的rbuv_swap_switch项设置为true。 -转换完成后，将该模型放到model路径下。 +转换完成后，将该模型放到model路径下。 + +转换为BGR输入参考 +``` +aipp_op{ + aipp_mode : static + input_format : RGB888_U8 + src_image_size_w : 640 + src_image_size_h : 640 + + csc_switch : false + rbuv_swap_switch : true + + min_chn_0 : 0 + min_chn_1 : 0 + min_chn_2 : 0 + var_reci_chn_0: 0.0039215686274509803921568627451 + var_reci_chn_1: 0.0039215686274509803921568627451 + var_reci_chn_2: 0.0039215686274509803921568627451 + +} +``` +转换为YUVSP420输入模型参考 +``` +aipp_op { + aipp_mode : static + input_format : YUV420SP_U8 + csc_switch : true + rbuv_swap_switch : false + matrix_r0c0 : 256 + matrix_r0c1 : 0 + matrix_r0c2 : 359 + matrix_r1c0 : 256 + matrix_r1c1 : -88 + matrix_r1c2 : -183 + matrix_r2c0 : 256 + matrix_r2c1 : 454 + matrix_r2c2 : 0 + input_bias_0 : 0 + input_bias_1 : 128 + input_bias_2 : 128 + + min_chn_0 : 0 + min_chn_1 : 0 + min_chn_2 : 0 + var_reci_chn_0: 0.0039215686274509803921568627451 + var_reci_chn_1: 0.0039215686274509803921568627451 + var_reci_chn_2: 0.0039215686274509803921568627451 +} +``` ## 4. 编译与运行 @@ -93,16 +149,13 @@ ascend-toolkit-path: CANN 安装路径。 放入待测图片。将一张图片放项目根路径下，命名为 test.jpg。 **步骤3** -对样例main.cpp中加载的模型路径、模型配置文件路径进行检查，确保对应位置存在相关文件，包括： -string modelPath = "models/yolov7.om"; -string yolov7ConfigPath = "models/yolov7.cfg"; -string yolov7LabelPath = "models/coco.names"; +对样例main.cpp中加载的模型路径、模型配置文件路径进行检查，确保对应位置存在相关文件，请参考run.sh中的说明。 **步骤4** 图片检测。在项目路径根目录下运行命令： ``` -bash run.sh +bash run.sh -m model_path -c model_config_path -l model_label_path -i image_path [-y] ``` ### 4.2 pipeline推理业务流程 diff --git a/mxVision/YOLOv7Detection/main.cpp b/mxVision/YOLOv7Detection/main.cpp index 1035101718671782cf129572832e5a15a84dbcb9..4809544929e86d90c87c08adc57676bcb200ca56 100644 --- a/mxVision/YOLOv7Detection/main.cpp +++ b/mxVision/YOLOv7Detection/main.cpp @@ -24,6 +24,7 @@ #include #include +#include "opencv2/imgproc.hpp" #include "MxBase/MxBase.h" #include "MxBase/MemoryHelper/MemoryHelper.h" #include "MxBase/DeviceManager/DeviceManager.h" @@ -41,9 +42,23 @@ const int OPENCV_8UC3 = 16; const int YUV_DIVISION = 2; const int R_CHANNEL = 2; const int AVG_PARAM = 2; - +const int ARG_NUM = 10; const long MAX_FILE_SIZE = 1024 * 1024 * 1024; // 1g +const float YUV_Y_R = 0.299; +const float YUV_Y_G = 0.587; +const float YUV_Y_B = 0.114; +const float YUV_U_R = -0.169; +const float YUV_U_G = 0.331; +const float YUV_U_B = 0.500; +const float YUV_V_R = 0.500; +const float YUV_V_G = 0.419; +const float YUV_V_B = 0.081; +const int YUV_DATA_SIZE = 3; +const int YUV_OFFSET = 2; +const int YUV_OFFSET_S = 1; +const int YUV_OFFSET_UV = 128; +const int ALIGN_LEFT = 16; APP_ERROR CheckFileVaild(const std::string &filePath) { struct stat buf; @@ -129,11 +144,13 @@ APP_ERROR PaddingProcess(ImageProcessor &imageProcessor, std::pair res LogError << "Failed to mallloc and copy dvpp memory."; return APP_ERR_ACL_BAD_COPY; } - cv::Mat resizedHost(resizeImage.GetSize().height, resizeImage.GetSize().width, OPENCV_8UC3, resHostData.ptrData); + cv::Mat resizedHost(resizeImage.GetSize().height, resizeImage.GetSize().width, OPENCV_8UC3, + resHostData.ptrData); cv::Rect roi = cv::Rect(0, 0, resizedWidth, resizedHeight); cv::Mat extendedImage; - cv::copyMakeBorder(resizedHost(roi), extendedImage, 0, 0, leftOffset, MODEL_INPUT_WIDTH - leftOffset - resizedWidth, - cv::BORDER_CONSTANT, cv::Scalar(PAD_COLOR, PAD_COLOR, PAD_COLOR)); + cv::copyMakeBorder(resizedHost(roi), extendedImage, 0, 0, leftOffset, + MODEL_INPUT_WIDTH - leftOffset - resizedWidth, cv::BORDER_CONSTANT, + cv::Scalar(PAD_COLOR, PAD_COLOR, PAD_COLOR)); int maxFillRow = std::min(MODEL_INPUT_WIDTH, (int)resizeImage.GetSize().width + leftOffset); for (int col = 0; col < MODEL_INPUT_WIDTH; col++) { for (int row = resizedWidth + leftOffset; row < maxFillRow; row++) { @@ -176,6 +193,40 @@ APP_ERROR PaddingProcess(ImageProcessor &imageProcessor, std::pair res return APP_ERR_OK; } +APP_ERROR SetImageBackground(MxBase::MemoryData& data) +{ + auto dataPtr = data.ptrData; + float yuvY = YUV_Y_R * PAD_COLOR + YUV_Y_G * PAD_COLOR + YUV_Y_B * PAD_COLOR; + float yuvU = YUV_U_R * PAD_COLOR - YUV_U_G * PAD_COLOR + YUV_U_B * PAD_COLOR + YUV_OFFSET_UV; + float yuvV = YUV_V_R * PAD_COLOR - YUV_V_G * PAD_COLOR - YUV_V_B * PAD_COLOR + YUV_OFFSET_UV; + + APP_ERROR ret = MxBase::MemoryHelper::MxbsMemset(data, (int)yuvY, data.size); + if (ret != APP_ERR_OK) { + LogError << "Failed to memset dvpp memory"; + return ret; + } + int offsetSize = MODEL_INPUT_HEIGHT * MODEL_INPUT_WIDTH / YUV_OFFSET; + data.ptrData = (uint8_t *)data.ptrData + MODEL_INPUT_HEIGHT * MODEL_INPUT_WIDTH; + ret = MxBase::MemoryHelper::MxbsMemset(data, (int)yuvU, offsetSize); + if (ret != APP_ERR_OK) { + LogError << "Failed to memset dvpp memory"; + data.ptrData = dataPtr; + return ret; + } + data.ptrData = (uint8_t *)data.ptrData + YUV_OFFSET_S; + for (int i = 0; i < offsetSize / YUV_OFFSET; i++) { + ret = MxBase::MemoryHelper::MxbsMemset(data, (int)yuvV, YUV_OFFSET_S); + if (ret != APP_ERR_OK) { + LogError << "Failed to memset dvpp memory"; + data.ptrData = dataPtr; + return ret; + } + data.ptrData = (uint8_t *)data.ptrData + YUV_OFFSET; + } + data.ptrData = dataPtr; + return APP_ERR_OK; +} + APP_ERROR DvppPreprocessorYuv(ImageProcessor &imageProcessor, std::string &imagePath, vector &yolov7Inputs, std::vector &imagePreProcessInfos, int deviceId) { @@ -207,12 +258,14 @@ APP_ERROR DvppPreprocessorYuv(ImageProcessor &imageProcessor, std::string &image return APP_ERR_ACL_BAD_ALLOC; } std::shared_ptr pastedData((uint8_t*)imgData.ptrData, imgData.free); - if (MemoryHelper::Memset(imgData, PAD_COLOR, dataSize) != APP_ERR_OK) { + if (SetImageBackground(imgData) != APP_ERR_OK) { LogError << "Failed to memset dvpp memory."; return APP_ERR_ACL_BAD_ALLOC; } int leftOffset = (MODEL_INPUT_WIDTH - resizedWidth) / AVG_PARAM; int topOffset = (MODEL_INPUT_HEIGHT - resizedHeight) / AVG_PARAM; + topOffset = topOffset % AVG_PARAM == 0 ? topOffset : topOffset - 1; + leftOffset = leftOffset < ALIGN_LEFT ? 0 : leftOffset / ALIGN_LEFT * ALIGN_LEFT; Rect RectSrc(0, 0, resizedWidth, resizedHeight); Rect RectDst(leftOffset, topOffset, leftOffset + resizedWidth, topOffset + resizedHeight); std::pair cropPasteRect = {RectSrc, RectDst}; @@ -223,62 +276,109 @@ APP_ERROR DvppPreprocessorYuv(ImageProcessor &imageProcessor, std::string &image } yolov7Inputs.push_back(pastedImgTmp.ConvertToTensor()); ResizedImageInfo imagePreProcessInfo(resizedWidth, resizedHeight, originalWidth, originalHeight, - RESIZER_MS_KEEP_ASPECT_RATIO, minScale); + RESIZER_TF_KEEP_ASPECT_RATIO, minScale); imagePreProcessInfos.push_back(imagePreProcessInfo); return APP_ERR_OK; } -APP_ERROR DvppPreprocessor(std::string &imagePath, vector &yolov7Inputs, +APP_ERROR OpenCVPreProcessor(std::string &imagePath, vector &yolov7Inputs, std::vector &imagePreProcessInfos, int deviceId) +{ + auto image = cv::imread(imagePath); + size_t originalWidth = image.cols; + size_t originalHeight = image.rows; + float scaleWidth = MODEL_INPUT_WIDTH * 1.0 / originalWidth; + float scaleHeight = MODEL_INPUT_HEIGHT * 1.0 / originalHeight; + float minScale = scaleWidth < scaleHeight ? scaleWidth : scaleHeight; + int resizedWidth = std::round(originalWidth * minScale); + int resizedHeight = std::round(originalHeight * minScale); + cv::Mat resizedImg; + cv::resize(image, resizedImg, cv::Size(resizedWidth, resizedHeight)); + int leftOffset = (MODEL_INPUT_WIDTH - resizedWidth) / AVG_PARAM; + int topOffset = (MODEL_INPUT_HEIGHT - resizedHeight) / AVG_PARAM; + uint32_t dataSize = MODEL_INPUT_HEIGHT * MODEL_INPUT_WIDTH * RGB_EXTEND; + MxBase::Size imageSize(MODEL_INPUT_WIDTH, MODEL_INPUT_HEIGHT); + cv::Mat extendedImage; + cv::copyMakeBorder(resizedImg, extendedImage, topOffset, MODEL_INPUT_HEIGHT - topOffset - resizedHeight, leftOffset, + MODEL_INPUT_WIDTH - leftOffset - resizedWidth, cv::BORDER_CONSTANT, + cv::Scalar(PAD_COLOR, PAD_COLOR, PAD_COLOR)); + uint8_t *pasteHostData = (uint8_t *)malloc(dataSize); + if (pasteHostData == nullptr) { + return APP_ERR_ACL_BAD_ALLOC; + } + for (size_t i = 0; i < dataSize; i++) { + pasteHostData[i] = extendedImage.data[i]; + } + std::shared_ptr dataPaste((uint8_t *)pasteHostData, free); + Image pastedImage(dataPaste, dataSize, -1, imageSize, ImageFormat::BGR_888); + pastedImage.ToDevice(deviceId); + yolov7Inputs.push_back(pastedImage.ConvertToTensor()); + ResizedImageInfo imagePreProcessInfo(resizedWidth, resizedHeight, originalWidth, originalHeight, + RESIZER_TF_KEEP_ASPECT_RATIO, minScale); + imagePreProcessInfos.push_back(imagePreProcessInfo); + return APP_ERR_OK; +} + +APP_ERROR DvppPreprocessor(std::string &imagePath, vector &yolov7Inputs, + std::vector &imagePreProcessInfos, int deviceId, bool isYuvInput) { ImageProcessor imageProcessor(deviceId); - if (DeviceManager::IsAscend310P()) { - Image decodeImage; - APP_ERROR ret = imageProcessor.Decode(imagePath, decodeImage, ImageFormat::BGR_888); - if (ret != APP_ERR_OK) { - LogError << "ImageProcessor decode failed."; - return ret; - } - Image resizeImage; - uint32_t originalWidth = decodeImage.GetOriginalSize().width; - uint32_t originalHeight = decodeImage.GetOriginalSize().height; - float scaleWidth = MODEL_INPUT_WIDTH * 1.0 / originalWidth; - float scaleHeight = MODEL_INPUT_HEIGHT * 1.0 / originalHeight; - float minScale = scaleWidth < scaleHeight ? scaleWidth : scaleHeight; - int resizedWidth = std::round(originalWidth * minScale); - int resizedHeight = std::round(originalHeight * minScale); - ret = imageProcessor.Resize(decodeImage, MxBase::Size(resizedWidth, resizedHeight), resizeImage, - Interpolation::BILINEAR_SIMILAR_OPENCV); - if (ret != APP_ERR_OK) { - LogError << "ImageProcessor resize failed."; - return ret; - } - Image pastedImage; - std::pair resizedInfo(resizedWidth, resizedHeight); - ret = PaddingProcess(imageProcessor, resizedInfo, deviceId, resizeImage, pastedImage); - if (ret != APP_ERR_OK) { - LogError << "ImageProcessor padding failed."; - return ret; - } - yolov7Inputs.push_back(pastedImage.ConvertToTensor()); - ResizedImageInfo imagePreProcessInfo(resizedWidth, resizedHeight, originalWidth, originalHeight, - RESIZER_STRETCHING, 0); - imagePreProcessInfos.push_back(imagePreProcessInfo); - } else { + if (isYuvInput) { return DvppPreprocessorYuv(imageProcessor, imagePath, yolov7Inputs, imagePreProcessInfos, deviceId); + } else { + if (DeviceManager::IsAscend310P()) { + Image decodeImage; + APP_ERROR ret = imageProcessor.Decode(imagePath, decodeImage, ImageFormat::BGR_888); + if (ret != APP_ERR_OK) { + LogError << "ImageProcessor decode failed."; + return OpenCVPreProcessor(imagePath, yolov7Inputs, imagePreProcessInfos, deviceId); + } + Image resizeImage; + uint32_t originalWidth = decodeImage.GetOriginalSize().width; + uint32_t originalHeight = decodeImage.GetOriginalSize().height; + float scaleWidth = MODEL_INPUT_WIDTH * 1.0 / originalWidth; + float scaleHeight = MODEL_INPUT_HEIGHT * 1.0 / originalHeight; + float minScale = scaleWidth < scaleHeight ? scaleWidth : scaleHeight; + int resizedWidth = std::round(originalWidth * minScale); + int resizedHeight = std::round(originalHeight * minScale); + ret = imageProcessor.Resize(decodeImage, MxBase::Size(resizedWidth, resizedHeight), resizeImage, + Interpolation::BILINEAR_SIMILAR_OPENCV); + if (ret != APP_ERR_OK) { + LogError << "ImageProcessor resize failed."; + return ret; + } + Image pastedImage; + std::pair resizedInfo(resizedWidth, resizedHeight); + ret = PaddingProcess(imageProcessor, resizedInfo, deviceId, resizeImage, pastedImage); + if (ret != APP_ERR_OK) { + LogError << "ImageProcessor padding failed."; + return ret; + } + yolov7Inputs.push_back(pastedImage.ConvertToTensor()); + ResizedImageInfo imagePreProcessInfo(resizedWidth, resizedHeight, originalWidth, originalHeight, + RESIZER_TF_KEEP_ASPECT_RATIO, minScale); + imagePreProcessInfos.push_back(imagePreProcessInfo); + } else { + return OpenCVPreProcessor(imagePath, yolov7Inputs, imagePreProcessInfos, deviceId); + } } return APP_ERR_OK; } -APP_ERROR E2eInfer(std::string imagePath, int32_t deviceId) +APP_ERROR E2eInfer(std::map pathMap, int32_t deviceId, bool isYuvInput) { + std::string imagePath = pathMap["imgPath"]; + APP_ERROR ret = CheckFileVaild(imagePath); + if (ret != APP_ERR_OK) { + return ret; + } vector yolov7Inputs; std::vector resizedImageInfos; - APP_ERROR ret = DvppPreprocessor(imagePath, yolov7Inputs, resizedImageInfos, deviceId); + ret = DvppPreprocessor(imagePath, yolov7Inputs, resizedImageInfos, deviceId, isYuvInput); if (ret != APP_ERR_OK) { return ret; } - string modelPath = "model/yolov7.om"; + string modelPath = pathMap["modelPath"]; ret = CheckFileVaild(modelPath); if (ret != APP_ERR_OK) { return ret; @@ -286,7 +386,7 @@ APP_ERROR E2eInfer(std::string imagePath, int32_t deviceId) Model yolov7(modelPath, deviceId); vector yolov7Outputs = yolov7.Infer(yolov7Inputs); - if (yolov7Inputs.size() == 0) { + if (yolov7Outputs.size() == 0) { LogError << "YOLOv7 infer failed."; return APP_ERR_COMM_FAILURE; } @@ -294,8 +394,8 @@ APP_ERROR E2eInfer(std::string imagePath, int32_t deviceId) yolov7Outputs[i].ToHost(); } std::vector cropConfigVec; - string yolov7ConfigPath = "model/yolov7.cfg"; - string yolov7LabelPath = "model/coco.names"; + string yolov7ConfigPath = pathMap["modelConfigPath"]; + string yolov7LabelPath = pathMap["modelLabelPath"]; ret = CheckFileVaild(yolov7ConfigPath); if (ret != APP_ERR_OK) { @@ -310,17 +410,56 @@ APP_ERROR E2eInfer(std::string imagePath, int32_t deviceId) return APP_ERR_OK; } +void usage() +{ + std::cout << "Usage: " << std::endl; + std::cout << "./sample -m model_path -c model_config_path -l model_label_path -i image_path [-y] " << std::endl; +} + int main(int argc, char *argv[]) { MxInit(); + if (argc > ARG_NUM || argc < ARG_NUM - 1) { + usage(); + return 0; + } int32_t deviceId = 0; - std::string imgPath = "test.jpg"; - APP_ERROR ret = CheckFileVaild(imgPath); - if (ret != APP_ERR_OK) { - return ret; + bool isYuvInput = 0; + std::map pathMap; + int input; + const char* optString = "i:m:c:l:yh"; + while ((input = getopt(argc, argv, optString)) != -1) { + switch (input) { + case 'm': + pathMap.insert({ "modelPath", optarg }); + break; + case 'i': + pathMap.insert({ "imgPath", optarg }); + break; + case 'c': + pathMap.insert({ "modelConfigPath", optarg }); + break; + case 'l': + pathMap.insert({ "modelLabelPath", optarg }); + break; + case 'y': + isYuvInput = true; + break; + case 'h': + usage(); + return 0; + case '?': + usage(); + return 0; + } } - - ret = E2eInfer(imgPath, deviceId); + if (pathMap.count("modelPath") <= 0 || pathMap.count("imgPath") <= 0 || pathMap.count("modelConfigPath") <= 0 || + pathMap.count("modelLabelPath") <= 0) { + LogError << "Invalid input params"; + usage(); + return 0; + } + APP_ERROR ret = E2eInfer(pathMap, deviceId, isYuvInput); if (ret != APP_ERR_OK) { LogError << "Failed to run E2eInfer"; } diff --git a/mxVision/YOLOv7Detection/model/yolov.cfg b/mxVision/YOLOv7Detection/model/yolov.cfg index 895471d65fc3c2ee01180b6cd7028bff608a2fea..3c5674b325afd545aec2e3c1896a71cc53d58cba 100644 --- a/mxVision/YOLOv7Detection/model/yolov.cfg +++ b/mxVision/YOLOv7Detection/model/yolov.cfg @@ -1,4 +1,5 @@ CLASS_NUM=80 SCORE_THRESH=0.25 OBJECTNESS_THRESH=0.25 -IOUT_THRESH=0.45 \ No newline at end of file +IOU_THRESH=0.45 +PADDING_TYPE=1 \ No newline at end of file diff --git a/mxVision/YOLOv7Detection/pipeline/Sample.pipeline b/mxVision/YOLOv7Detection/pipeline/Sample.pipeline index ba756a50fdb3a3fbc7247a79405e23ac1283efac..f6542cabf35e11dcc739de6ade210a39892a6166 100644 --- a/mxVision/YOLOv7Detection/pipeline/Sample.pipeline +++ b/mxVision/YOLOv7Detection/pipeline/Sample.pipeline @@ -11,19 +11,23 @@ "next": "mxpi_imagedecoder0" }, "mxpi_imagedecoder0": { + "props" : { + "cvProcessor" : "opencv" + }, "factory": "mxpi_imagedecoder", "next": "mxpi_imageresize0" }, "mxpi_imageresize0": { "props": { - "dataSource": "mxpi_imagedecoder0", + "cvProcessor" : "opencv", "resizeType": "Resizer_KeepAspectRatio_Fit", "paddingType": "Padding_Around", "paddingHeight": "640", "paddingWidth": "640", "paddingColorB": "114", "paddingColorG": "114", - "paddingColorR": "114" + "paddingColorR": "114", + "interpolation" : "0" }, "factory": "mxpi_imageresize", "next": "mxpi_tensorinfer0" @@ -39,7 +43,7 @@ "mxpi_objectpostprocessor0": { "props": { "dataSource": "mxpi_tensorinfer0", - "postProcessConfigPath":"../model/yolo7.cfg", + "postProcessConfigPath":"../model/yolov7.cfg", "labelPath": "../model/coco.names", "postProcessLibPath": "libyolov7postprocess.so" }, diff --git a/mxVision/YOLOv7Detection/pipeline/SampleYuv.pipeline b/mxVision/YOLOv7Detection/pipeline/SampleYuv.pipeline new file mode 100644 index 0000000000000000000000000000000000000000..74fa8fda811ffdc31d5adb3ae6b20efd928da237 --- /dev/null +++ b/mxVision/YOLOv7Detection/pipeline/SampleYuv.pipeline @@ -0,0 +1,62 @@ +{ + "detection": { + "stream_config": { + "deviceId": "0" + }, + "appsrc0": { + "props": { + "blocksize": "409600" + }, + "factory": "appsrc", + "next": "mxpi_imagedecoder0" + }, + "mxpi_imagedecoder0": { + "factory": "mxpi_imagedecoder", + "next": "mxpi_imageresize0" + }, + "mxpi_imageresize0": { + "props": { + "dataSource": "mxpi_imagedecoder0", + "resizeType": "Resizer_KeepAspectRatio_Fit", + "paddingType": "Padding_Around", + "paddingHeight": "640", + "paddingWidth": "640", + "RGBValue" : "114, 114, 114", + "interpolation" : "0" + }, + "factory": "mxpi_imageresize", + "next": "mxpi_tensorinfer0" + }, + "mxpi_tensorinfer0": { + "props": { + "dataSource": "mxpi_imageresize0", + "modelPath": "../model/yolov7.om" + }, + "factory": "mxpi_tensorinfer", + "next": "mxpi_objectpostprocessor0" + }, + "mxpi_objectpostprocessor0": { + "props": { + "dataSource": "mxpi_tensorinfer0", + "postProcessConfigPath":"../model/yolov7.cfg", + "labelPath": "../model/coco.names", + "postProcessLibPath": "libyolov7postprocess.so" + }, + "factory": "mxpi_objectpostprocessor", + "next": "mxpi_dataserialize0" + }, + "mxpi_dataserialize0": { + "props": { + "outputDataKeys": "mxpi_objectpostprocessor0" + }, + "factory": "mxpi_dataserialize", + "next": "appsink0" + }, + "appsink0": { + "props": { + "blocksize": "4096000" + }, + "factory": "appsink" + } + } +} diff --git a/mxVision/YOLOv7Detection/plugin/CMakeLists.txt b/mxVision/YOLOv7Detection/plugin/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..1af840a68d78a3624ddb02de5332fcff82cba346 --- /dev/null +++ b/mxVision/YOLOv7Detection/plugin/CMakeLists.txt @@ -0,0 +1,29 @@ +cmake_minimum_required(VERSION 3.5.2) +project(yolov7postprocess) + +add_definitions(-D_GLIBCXX_USE_CXX11_ABI=0) +add_definitions(-Dgoogle=mindxsdk_private) + +set(PLUGIN_NAME "yolov7postprocess") +set(TARGET_LIBRARY ${PLUGIN_NAME}) + +include_directories(${PROJECT_SOURCE_DIR}/../../include) +include_directories(${PROJECT_SOURCE_DIR}/../../opensource/include) +include_directories(${PROJECT_SOURCE_DIR}/../../opensource/include/gstreamer-1.0) +include_directories(${PROJECT_SOURCE_DIR}/../../opensource/include/glib-2.0) +include_directories(${PROJECT_SOURCE_DIR}/../../opensource/lib/glib-2.0/include) + +link_directories(${PROJECT_SOURCE_DIR}/../../opensource/lib/) +link_directories(${PROJECT_SOURCE_DIR}/../../lib) + +add_compile_options(-fPIC -std=c++11 -fstack-protector-all -Wl, -z,relro,-z,now,-z,noexecstack -s -pie) +add_compile_options("-DPLUGIN_NAME=${PLUGIN_NAME}") + +add_definitions(-DENABLE_DVPP_INTERFACE) +add_library(${TARGET_LIBRARY} SHARED Yolov7PostProcess.cpp) + +target_link_libraries(${TARGET_LIBRARY} glib-2.0 gstreamer-1.0 gobject-2.0 gstbase-1.0 gmodule-2.0) +target_link_libraries(${TARGET_LIBRARY} plugintoolkit mxpidatatype mxbase) +target_link_libraries(${TARGET_LIBRARY} -Wl,-z,relro,-z,now,-z,noexecstack -s) + +install(TARGETS ${TARGET_LIBRARY} PERMISSIONS OWNER_READ GROUP_READ LIBRARY DESTINATION ${PROJECT_SOURCE_DIR}/../../lib/modelpostprocessors) \ No newline at end of file diff --git a/mxVision/YOLOv7Detection/Yolov7PostProcess.cpp b/mxVision/YOLOv7Detection/plugin/Yolov7PostProcess.cpp similarity index 82% rename from mxVision/YOLOv7Detection/Yolov7PostProcess.cpp rename to mxVision/YOLOv7Detection/plugin/Yolov7PostProcess.cpp index e42964c68a6eeca9d10fdd7242dae81f89c40df1..ecc4fbc8411792f0bade5f7ec947e793a5a0e647 100644 --- a/mxVision/YOLOv7Detection/Yolov7PostProcess.cpp +++ b/mxVision/YOLOv7Detection/plugin/Yolov7PostProcess.cpp @@ -19,9 +19,11 @@ #include "MxBase/Log/Log.h" #include "MxBase/Maths/FastMath.h" #include "MxBase/CV/ObjectDetection/Nms/Nms.h" +#include "MxBase/DeviceManager/DeviceManager.h" namespace MxBase { const int MODEL_INPUT_SIZE = 640; +const int ALIGN_LEFT = 16; const int CONFIDENCE_IDX = 4; const int LABEL_START_OFFSET = 5; const int OUTPUT_DIMS = 3; @@ -39,6 +41,7 @@ Yolov7PostProcess &Yolov7PostProcess::operator = (const Yolov7PostProcess &other ObjectPostProcessBase::operator = (other); objectnessThresh_ = other.objectnessThresh_; iouThresh_ = other.iouThresh_; + paddingType_ = other.paddingType_; return *this; } @@ -58,6 +61,10 @@ APP_ERROR Yolov7PostProcess::Init(const std::map("PADDING_TYPE", paddingType_, 0, 1); + if (ret != APP_ERR_OK) { + LogWarn << GetError(ret) << "Fail to read PADDING_TYPE from config, default is :" << paddingType_; + } LogDebug << "End to Init Yolov7PostProcess. "; return APP_ERR_OK; } @@ -78,6 +85,10 @@ APP_ERROR Yolov7PostProcess::Init(const std::map &post if (ret != APP_ERR_OK) { LogWarn << GetError(ret) << "Fail to read IOU_THRESH from config, default is :" << iouThresh_; } + ret = configData_.GetFileValue("PADDING_TYPE", paddingType_, 0, 1); + if (ret != APP_ERR_OK) { + LogWarn << GetError(ret) << "Fail to read PADDING_TYPE from config, default is :" << paddingType_; + } LogDebug << "End to Init Yolov7PostProcess. "; return APP_ERR_OK; } @@ -105,22 +116,32 @@ void Yolov7PostProcess::ConstructBoxFromOutput(float *output, size_t offset, std if (classId < 0) { return; } + int tmpResizedWidth = resizedImageInfo.widthResize; + int tmpResizedHeight = resizedImageInfo.heightResize; double division = 1; if (std::fabs(resizedImageInfo.keepAspectRatioScaling) > EPSILON) { division = resizedImageInfo.keepAspectRatioScaling; } - int offsetLeft = (MODEL_INPUT_SIZE - resizedImageInfo.widthResize) / AVG_PARAM; - int offsetTop = (MODEL_INPUT_SIZE - resizedImageInfo.heightResize) / AVG_PARAM; + if (tmpResizedWidth == tmpResizedHeight && tmpResizedHeight == MODEL_INPUT_SIZE) { + tmpResizedWidth = std::round(resizedImageInfo.widthOriginal * division); + tmpResizedHeight = std::round(resizedImageInfo.heightOriginal * division); + } + int offsetLeft = (MODEL_INPUT_SIZE - tmpResizedWidth) / AVG_PARAM; + int offsetTop = (MODEL_INPUT_SIZE - tmpResizedHeight) / AVG_PARAM; + if (paddingType_ == 0) { + offsetTop = offsetTop % AVG_PARAM ? offsetTop : offsetTop - 1; + offsetLeft = offsetLeft < ALIGN_LEFT ? 0 : offsetLeft / ALIGN_LEFT * ALIGN_LEFT; + } auto leftX = (output[index] - output[index + XOFFSET] / AVG_PARAM - offsetLeft) / division; auto leftY = (output[index + 1] - output[index + YOFFSET] / AVG_PARAM - offsetTop) / division; auto rightX = (output[index] + output[index + XOFFSET] / AVG_PARAM - offsetLeft) / division; auto rightY = (output[index + 1] + output[index + YOFFSET] / AVG_PARAM - offsetTop) / division; ObjectInfo obj; - obj.x0 = std::round(leftX); - obj.y0 = std::round(leftY); - obj.x1 = std::round(rightX); - obj.y1 = std::round(rightY); + obj.x0 = leftX < 0.0 ? 0.0 : leftX; + obj.y0 = leftY < 0.0 ? 0.0 : leftY; + obj.x1 = rightX > resizedImageInfo.widthOriginal ? resizedImageInfo.widthOriginal : rightX; + obj.y1 = rightY > resizedImageInfo.heightOriginal ? resizedImageInfo.heightOriginal : rightY; obj.confidence = maxProb; obj.classId = classId; obj.className = configData_.GetClassName(obj.classId); @@ -158,6 +179,7 @@ APP_ERROR Yolov7PostProcess::Process(const std::vector &tensors, } if (resizedImageInfos.size() != tensors.size()) { LogError << "The size of resizedImageInfos does not match the size of tensors."; + return APP_ERR_INPUT_NOT_MATCH; } uint32_t batchSize = tensors[0].GetShape()[0]; size_t rows = tensors[0].GetSize() / ((classNum_ + LABEL_START_OFFSET) * batchSize); diff --git a/mxVision/YOLOv7Detection/Yolov7PostProcess.h b/mxVision/YOLOv7Detection/plugin/Yolov7PostProcess.h similarity index 98% rename from mxVision/YOLOv7Detection/Yolov7PostProcess.h rename to mxVision/YOLOv7Detection/plugin/Yolov7PostProcess.h index 02b00b607b1cb7e82655760977fe8624b08de716..7cb4f96c5c2f4dec9db18d2c67759044354bc197 100644 --- a/mxVision/YOLOv7Detection/Yolov7PostProcess.h +++ b/mxVision/YOLOv7Detection/plugin/Yolov7PostProcess.h @@ -48,6 +48,7 @@ private: std::vector &objectInfo, const ResizedImageInfo &resizedImageInfo); float objectnessThresh_ = DEFAULT_OBJECTNESS_THRESH; float iouThresh_ = DEFAULT_IOU_THRESH; + int paddingType_ = 1; }; extern "C" { diff --git a/mxVision/YOLOv7Detection/run.sh b/mxVision/YOLOv7Detection/run.sh index e91a1fc9f4287ec5025d2f86ffa54806b0b571c2..79b314a20e02c9adc2f72df7a4ed8f99f4cf9b95 100644 --- a/mxVision/YOLOv7Detection/run.sh +++ b/mxVision/YOLOv7Detection/run.sh @@ -10,6 +10,81 @@ make -j || { exit ${ret} } +func() { + echo "Usage:" + echo "bash run.sh -m model_path -c model_config_path -l model_label_path -i image_path [-y]" + echo "Description:" + echo "-m model path" + echo "-c model config file path" + echo "-l label path for model" + echo "-i image path to infer" + echo "-y [Optional] use yuv model, default is not yuv" + exit -1 +} + +is_yuv=0 +argc=5 +args=0 +while getopts "i:m:c:l:yh" arg +do + if [ "$args" -gt "$argc" ]; then + echo "Error: Wrong usage, too many arguments." + func + exit 1 + fi + case "$arg" in + i) + img_path="$OPTARG" + ;; + m) + model_path="$OPTARG" + ;; + c) + model_config_path="$OPTARG" + ;; + l) + model_label_path="$OPTARG" + ;; + y) + is_yuv=1 + ;; + h) + func + exit 1 + ;; + ?) + echo "Error: Wrong usage, unknown argument." + func + exit 1 + ;; + esac + args=$(($args+1)) +done + +if [ ! -n "$model_path" ]; then + echo "Error: Required argument \"-m model_path\" is missing." + func + exit 1 +fi +if [ ! -n "$model_config_path" ]; then + echo "Error: Required argument \"-c model_config_path\" is missing." + func + exit 1 +fi +if [ ! -n "$model_label_path" ]; then + echo "Error: Required argument \"-l model_label_path\" is missing." + func + exit 1 +fi +if [ ! -n "$img_path" ]; then + echo "Error: Required argument \"-i img_path\" is missing." + func + exit 1 +fi cd .. -./sample +if [ "$is_yuv" -gt 0 ]; then + ./sample -m "$model_path" -c "$model_config_path" -l "$model_label_path" -i "$img_path" -y +else + ./sample -m "$model_path" -c "$model_config_path" -l "$model_label_path" -i "$img_path" +fi exit 0 \ No newline at end of file