From ce0a780fe1a63f728f4cc99b6590101025519218 Mon Sep 17 00:00:00 2001 From: chenjianbinC <2312144889@qq.com> Date: Tue, 15 Nov 2022 11:45:37 +0800 Subject: [PATCH] add ReIDv2.py --- contrib/ReID/README.md | 31 ++- contrib/ReID/ReIDv2.py | 430 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 459 insertions(+), 2 deletions(-) create mode 100644 contrib/ReID/ReIDv2.py diff --git a/contrib/ReID/README.md b/contrib/ReID/README.md index 56ded5ba3..860e81b1b 100644 --- a/contrib/ReID/README.md +++ b/contrib/ReID/README.md @@ -278,10 +278,37 @@ python3 main.py --queryFilePath='data/querySet' --galleryFilePath='data/galleryS 执行`main.py`文件后，可在“项目所在目录/result”路径下查看结果。 -## 7 参考链接 +## 7 V2接口测试 + +7.1 获取om模型 +``` +步骤详见4： 模型转换 +``` +7.2 准备数据集 +``` +步骤详见5： 数据集 +``` +7.3 配置环境变量 +``` +步骤详见6.4： 配置环境变量 +``` +7.4 执行 +``` +python3 ReIDv2.py --queryFilePath='data/querySet' --galleryFilePath='data/gallerySet' --matchThreshold=0.3 +``` +7.5 查看结果 +``` +执行`ReIDv2.py`文件后，可在“项目所在目录/result”路径下查看结果。 +``` +7.6 精度性能说明 +``` +没有误测少测，功能通过，性能与V1对齐，V2检测结果与V1一样 +``` + +## 8 参考链接 > 特定行人检索：[Person Search Demo](https://github.com/songwsx/person_search_demo) -## 8 Q&A +## 9 Q&A · 在运行main.py时出现"Vpc cropping failed"，或者"The image height zoom ratio is out of range [1/32, 16]" > 这里的错误是因为yolov3模型检测到的目标过小，抠图后放大的比例超过系统给定的阈值[1/32, 16]，更新“项目所在目录/models/yolov3.cfg”文件，将OBJECTNESS_THRESH适度调大可解决该问题 \ No newline at end of file diff --git a/contrib/ReID/ReIDv2.py b/contrib/ReID/ReIDv2.py new file mode 100644 index 000000000..483f128c0 --- /dev/null +++ b/contrib/ReID/ReIDv2.py @@ -0,0 +1,430 @@ +#!/usr/bin/env python +# coding=utf-8 + +""" +Copyright(C) Huawei Technologies Co.,Ltd. 2012-2021 All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import argparse +import os +import cv2 +import numpy as np +import io +from mindx.sdk import base +from mindx.sdk.base import ImageProcessor +import MxpiDataType_pb2 as MxpiDataType +from PIL import Image +from base64 import decode +from mindx.sdk.base import Tensor, Model, Size, Rect, log, ImageProcessor, post, BTensor +import time + +GALLERY_STREAM_NAME = b'galleryImageProcess' +QUERY_STREAM_NAME = b'queryImageProcess' + +IN_PLUGIN_ID = 0 + +INITIAL_MIN_DISTANCE = 99 + +LINE_THICKNESS = 2 +FONT_SCALE = 1.0 +FIND_COLOR = (0, 255, 0) +NONE_FIND_COLOR = (255, 0, 0) + +DEFAULT_MATCH_THRESHOLD = 0.3 +FEATURE_RESHAPE_ROW = -1 +FEATURE_RESHAPE_COLUMN = 2048 +ADMM_BETA = 1 +ADMM_ALPHA = -2 + +MIN_IMAGE_SIZE = 32 +MAX_IMAGE_SIZE = 8192 +MIN_IMAGE_WIDTH = 6 +device_ID = 2 +matchThreshold = DEFAULT_MATCH_THRESHOLD +NUM_THREADS_OF_INFER = 32 +yolov3_model_path = "./models/yolov3.om" +yolov3 = base.model(yolov3_model_path, deviceId=device_ID) +reid_model_path = "./models/ReID.om" +reid = base.model(reid_model_path, deviceId=device_ID) + + +def extract_query_feature(queryPath): + """ + Extract the features of query images, return the feature vector and the corresponding Pid vector + + + :arg: + queryPath: the directory of query images + streamApi: stream api + + :return: + queryFeatures: the vectors of queryFeatures + queryPid: the vectors of queryPid + """ + queryFeatures = [] + queryPid = [] + + + # check the query file + if os.path.exists(queryPath) != 1: + errorMessage = 'The query file does not exist.' + print(errorMessage) + exit() + if len(os.listdir(queryPath)) == 0: + errorMessage = 'The query file is empty.' + print(errorMessage) + exit() + + # extract the features for all images in query file + for root, dirs, files in os.walk(queryPath): + for file in files: + if file.endswith('.jpg'): + # store the corresponding pid + # we use the market1501 as dataset, which is named by + # 0001(person id)_c1(camera id)s1(sequence id)_000151(frame id)_00(box id).jpg + # if you use other dataset, modify it to identify the person label + queryPid.append(file[:4]) + imageProcessor = ImageProcessor(device_ID) + filePath = os.path.join(root, file) + decodedImg = imageProcessor.decode(filePath, base.nv12) + size_cof = Size(128, 256) + resizedImg = imageProcessor.resize(decodedImg, size_cof, base.huaweiu_high_order_filter) + imgTensor = [resizedImg.to_tensor()] + reid_model_path = "./models/ReID.om" + reid = base.model(reid_model_path, deviceId=device_ID) + reid_output = reid.infer(imgTensor) + reid_output0 = reid_output[0] + reid_output0.to_host() + queryFeature = np.array(reid_output0) + cv2.normalize(src=queryFeature, dst=queryFeature, norm_type=cv2.NORM_L2) + queryFeatures.append(queryFeature) + + else: + print('Input image only support jpg') + exit() + # print ("行人底库特征向量:{}行人ID：{}".format(queryFeatures, queryPid)) + return queryFeatures, queryPid + + +def get_pipeline_results(filePath): + """ + Get the results of current gallery image in pipeline + + :arg: + filePath: directory of current gallery image + streamApi: stream api + + :return: + objectList: results from mxpi_objectpostprocessor0 + featureList: results from mxpi_tensorinfer1 + """ + + try: + image = Image.open(filePath) + if image.format != 'JPEG': + print('Input image only support jpg') + exit() + elif image.width < MIN_IMAGE_SIZE or image.width > MAX_IMAGE_SIZE: + print('Input image width must in range [32, 8192], curr is {}'.format(image.width)) + exit() + elif image.height < MIN_IMAGE_SIZE or image.height > MAX_IMAGE_SIZE: + print('Input image height must in range [32, 8192], curr is {}'.format(image.height)) + exit() + else: + # read input image bytes + image_bytes = io.BytesIO() + image.save(image_bytes, format='JPEG') + # galleryDataInput.data = image_bytes.getvalue() + except IOError: + print('An IOError occurred while opening {}, maybe your input is not a picture'.format(filePath)) + exit() + + resizeInfo = base.ResizedImageInfo() + resizeInfo.heightResize = 416 + resizeInfo.widthResize = 416 + resizeInfo.heightOriginal = 1080 + resizeInfo.widthOriginal = 1920 + + imageProcessor1 = ImageProcessor(device_ID) + decodedImg = imageProcessor1.decode(filePath, base.nv12) + size_cof = Size(416, 416) + resizedImg = imageProcessor1.resize(decodedImg, size_cof, base.huaweiu_high_order_filter) + imgTensor1 = [resizedImg.to_tensor()] + yolov3_outputs = yolov3.infer(imgTensor1) + label_path = "/home/yuanlei2/cjb/mindxsdk-referenceapps/contrib/ReID/models/coco.names" # 分类标签文件的路径 + config_path = "/home/yuanlei2/cjb/mindxsdk-referenceapps/contrib/ReID/models/yolov3.cfg" + yolov3_post = post.Yolov3PostProcess(config_path=config_path, label_path=label_path) + + # 构造后处理的输入 + inputs = [] + for i in range(len(yolov3_outputs)): + yolov3_outputs[i].to_host() + n = np.array(yolov3_outputs[i]) + tensor = BTensor(n) + inputs.append(base.batch([tensor] * 2, keep_dims=True)) + yolov3_post_results = yolov3_post.process(inputs, [resizeInfo] * 2) + cropResizeVec = [] + objectList = [] + for i in range(len(yolov3_post_results)): + for j in range(len(yolov3_post_results[i])): + x0 = int(yolov3_post_results[i][j].x0) + y0 = int(yolov3_post_results[i][j].y0) + x1 = int(yolov3_post_results[i][j].x1) + y1 = int(yolov3_post_results[i][j].y1) + className = yolov3_post_results[i][j].className + objectList.append([x0, y0, x1, y1, className]) + cropResizeVec.append((Rect(x0, y0, x1, y1), Size(128, 256))) + # print ("yolov3检测结果：", objectList) + yolov3_crop = imageProcessor1.crop_resize(decodedImg, cropResizeVec) + imgTensor2 = [x.to_tensor() for x in yolov3_crop] + + featureList = [] + for i in range(len(imgTensor2)): + reid_output = reid.infer([imgTensor2[i]]) + reid_output[0].to_host() + featureList.append(np.array(reid_output[0])) + # print ("ReID检测结果", featureList) + + return objectList, featureList + + +def compute_feature_distance(objectList, featureList, queryFeatures): + """ + Record the location and features of the person in gallery image + Compute the feature distance between persons in gallery image and query image + + :arg: + objectList: the results from mxpi_objectpostprocessor0 + featureList: the results from mxpi_tensorinfer1 + queryFeatures: the vectors of queryFeatures + + :return: + detectedPersonInformation: location information of the detected person in gallery image + detectedPersonFeature: feature of the detected person in gallery image + galleryFeatureLength: the length of gallery feature set + queryFeatureLength: the length of query feature set + minDistanceIndexMatrix: the index of minimal distance in distance matrix + minDistanceMatrix: the index of minimal distance value in distance matrix + """ + # store the information and features for detected person + detectedPersonInformation = [] + detectedPersonFeature = [] + filterImageCount = 0 + personDetectedFlag = False + + # select the detected person, and store its location and features + for detectedItemIndex in range(0, len(objectList)): + detectedItem = objectList[detectedItemIndex] + xLength = int(detectedItem[2]) - int(detectedItem[0]) + yLength = int(detectedItem[3]) - int(detectedItem[1]) + if xLength < MIN_IMAGE_SIZE or yLength < MIN_IMAGE_WIDTH: + filterImageCount += 1 + continue + + if detectedItem[4] == "person": + personDetectedFlag = True + detectedPersonInformation.append({'x0': int(detectedItem[0]), 'x1': int(detectedItem[2]), + 'y0': int(detectedItem[1]), 'y1': int(detectedItem[3])}) + # detectedFeature = featureList[detectedItemIndex - filterImageCount] + detectedFeature = featureList[detectedItemIndex] + # print ("detectedFeature :", detectedFeature) + # print ("queryFeatureVector", queryFeatureVector) + cv2.normalize(src=detectedFeature, dst=detectedFeature, norm_type=cv2.NORM_L2) + detectedPersonFeature.append(detectedFeature) + + if not personDetectedFlag: + return None + + # get the number of the query images + queryFeatureLength = len(queryFeatures) + queryFeatureVector1 = np.array(queryFeatureVector).reshape(queryFeatureLength, FEATURE_RESHAPE_COLUMN) + + # get the number of the detected persons in this gallery image + galleryFeatureLength = len(detectedPersonFeature) + detectedPersonFeature = np.array(detectedPersonFeature).reshape(galleryFeatureLength, FEATURE_RESHAPE_COLUMN) + + # # compute the distance between query feature and gallery feature + distanceMatrix = np.tile(np.sum(np.power(queryFeatureVector1, 2), axis=1, keepdims=True), + reps=galleryFeatureLength) + \ + np.tile(np.sum(np.power(detectedPersonFeature, 2), axis=1, keepdims=True), + reps=queryFeatureLength).T + distanceMatrix = ADMM_BETA * distanceMatrix + \ + ADMM_ALPHA * np.dot(queryFeatureVector1, detectedPersonFeature.T) + + # find minimal distance for each query image + minDistanceIndexMatrix = distanceMatrix.argmin(axis=1) + minDistanceMatrix = distanceMatrix.min(axis=1) + # print ("minDistanceMatrix.shape:", minDistanceMatrix.shape) + return {'detectedPersonInformation': detectedPersonInformation, + 'galleryFeatureLength': galleryFeatureLength, 'queryFeatureLength': queryFeatureLength, + 'minDistanceIndexMatrix': minDistanceIndexMatrix, 'minDistanceMatrix': minDistanceMatrix} + + +def label_for_gallery_image(galleryFeatureLength, queryFeatureLength, queryPid, minDistanceIndexMatrix, + minDistanceMatrix, matchThreshold): + """ + Label each detected person in gallery image, find the most possible Pid + + :arg: + galleryFeatureLength: the length of gallery feature set + queryFeatureLength: the length of query feature set + queryPid: the vectors of queryPid + minDistanceIndexMatrix: the index of minimal distance in distance matrix + minDistanceMatrix: the index of minimal distance value in distance matrix + matchThreshold: match threshold + + :return: + galleryLabelSet: labels for current gallery image + """ + # one person only exists once in each gallery image, thus the Pid in this galleryLabelSet must be unique + galleryLabelSet = np.full(shape=galleryFeatureLength, fill_value='None') + galleryLabelDistance = np.full(shape=galleryFeatureLength, fill_value=INITIAL_MIN_DISTANCE, dtype=float) + + for queryIndex in range(0, queryFeatureLength): + currentPid = queryPid[queryIndex] + preferGalleryIndex = minDistanceIndexMatrix[queryIndex] + preferDistance = minDistanceMatrix[queryIndex] + if preferDistance < matchThreshold: + pidExistSet = np.where(galleryLabelSet == currentPid) + pidExistIndex = pidExistSet[0] + if len(pidExistIndex) == 0: + if galleryLabelSet[preferGalleryIndex] == 'None': + galleryLabelSet[preferGalleryIndex] = currentPid + galleryLabelDistance[preferGalleryIndex] = preferDistance + else: + if preferDistance < galleryLabelDistance[preferGalleryIndex]: + galleryLabelSet[preferGalleryIndex] = currentPid + galleryLabelDistance[preferGalleryIndex] = preferDistance + else: + if preferDistance < galleryLabelDistance[pidExistIndex]: + galleryLabelSet[pidExistIndex] = 'None' + galleryLabelDistance[pidExistIndex] = INITIAL_MIN_DISTANCE + galleryLabelSet[preferGalleryIndex] = currentPid + galleryLabelDistance[preferGalleryIndex] = preferDistance + return galleryLabelSet + + +def draw_results(filePath, galleryFeatureLength, detectedPersonInformation, galleryLabelSet, file): + """ + Draw and label the detection and re-identification results + + :arg: + filePath: directory of current gallery image + galleryFeatureLength: the length of gallery feature set + detectedPersonInformation: location information of the detected person in gallery image + galleryLabelSet: labels for current gallery image + file: name of current gallery image + + :return: + None + """ + # read the original image and label the detection results + image = cv2.imread(filePath) + + for galleryIndex in range(0, galleryFeatureLength): + # get the locations of the detected person in gallery image + locations = detectedPersonInformation[galleryIndex] + # if some pid meets the constraints, change the legendText and color + if galleryLabelSet[galleryIndex] == 'None': + color = NONE_FIND_COLOR + else: + color = FIND_COLOR + # label the detected person in the original image + cv2.rectangle(image, (locations.get('x0'), locations.get('y0')), + (locations.get('x1'), locations.get('y1')), color, LINE_THICKNESS) + cv2.putText(image, galleryLabelSet[galleryIndex], (locations.get('x0'), locations.get('y0')), + cv2.FONT_HERSHEY_SIMPLEX, FONT_SCALE, color, LINE_THICKNESS) + cv2.imwrite("./result/result_{}".format(str(file)), image) + print("Detect ", file, " successfully.") + + +def process_reid(galleryPath, queryFeatures, queryPid, matchThreshold): + """ + Detect and re-identify person in gallery image + + :arg: + galleryPath: the directory of gallery images + queryFeatures: the vectors of queryFeatures + queryPid: the vectors of queryPid + streamApi: stream api + matchThreshold: match threshold + + :return: + None + """ + # check the gallery file + if os.path.exists(galleryPath) != 1: + errorMessage = 'The gallery file does not exist.' + print(errorMessage) + exit() + if len(os.listdir(galleryPath)) == 0: + errorMessage = 'The gallery file is empty.' + print(errorMessage) + exit() + outputPath = 'result' + if os.path.exists(outputPath) != 1: + errorMessage = 'The result file does not exist.' + print(errorMessage) + exit() + total_time = [] + # detect and crop all person for all images in query file, and then extract the features + for root, dirs, files in os.walk(galleryPath): + for file in files: + if file.endswith('.jpg'): + filePath = os.path.join(root, file) + start = time.time() + objectList, featureList = get_pipeline_results(filePath) + end = time.time() + total_time.append(end - start) + print('Running time: %s Seconds' % sum(total_time)) + metricDirectory = compute_feature_distance(objectList, featureList, queryFeatures) + + if not metricDirectory: + print("Cannot detect person for image:", file) + continue + + detectedPersonInformation = metricDirectory.get('detectedPersonInformation') + galleryFeatureLength = metricDirectory.get('galleryFeatureLength') + queryFeatureLength = metricDirectory.get('queryFeatureLength') + minDistanceIndexMatrix = metricDirectory.get('minDistanceIndexMatrix') + minDistanceMatrix = metricDirectory.get('minDistanceMatrix') + + galleryLabelSet = label_for_gallery_image(galleryFeatureLength, queryFeatureLength, queryPid, + minDistanceIndexMatrix, minDistanceMatrix, matchThreshold) + + draw_results(filePath, galleryFeatureLength, detectedPersonInformation, galleryLabelSet, file) + else: + print('Input image only support jpg') + exit() + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--queryFilePath', type=str, default='data/querySet/', help="Query File Path") + parser.add_argument('--galleryFilePath', type=str, default='data/gallerySet/', help="Gallery File Path") + parser.add_argument('--matchThreshold', type=float, default=DEFAULT_MATCH_THRESHOLD, + help="Match Threshold for ReID Processing") + opt = parser.parse_args() + + queryFeatureVector, queryPidVector = extract_query_feature(opt.queryFilePath) + process_reid(opt.galleryFilePath, queryFeatureVector, queryPidVector, opt.matchThreshold) + + + + + + + + -- Gitee