diff --git a/Cpp_example/C06_test_qr_code_detector/CMakeLists.txt b/Cpp_example/C06_test_qr_code_detector/CMakeLists.txt index 5bc112182bad8bea270137c6426b101469f7d190..17721c6c85aecdedbc586a2bddd4c79b286f59d3 100644 --- a/Cpp_example/C06_test_qr_code_detector/CMakeLists.txt +++ b/Cpp_example/C06_test_qr_code_detector/CMakeLists.txt @@ -26,7 +26,7 @@ find_package(LockzhinerVisionModule REQUIRED) # 定义 ZXing SDK 路径 set(ZXing_ROOT_PATH "${PROJECT_ROOT_PATH}/third_party/zxing-cpp-v2.2.1-lockzhiner-vision-module") set(ZXing_INCLUDE_DIRS "${ZXing_ROOT_PATH}/include") -set(ZXing_LIBRARIES "${ZXing_ROOT_PATH}/lib/libzxing.a") +set(ZXing_LIBRARIES "${ZXing_ROOT_PATH}/lib/libZXing.a") # 基本图像处理示例 add_executable(Test-qr_code-detector test_qr_code_detector.cc) diff --git a/Cpp_example/D10_yolov5/README.md b/Cpp_example/D10_yolov5/README.md index 98ce23290acc789ba0a1b3639bf35300f3e6468c..575ff2d0d15544ff22c4e753dc6b759348b5d818 100644 --- a/Cpp_example/D10_yolov5/README.md +++ b/Cpp_example/D10_yolov5/README.md @@ -1,120 +1,165 @@ # YOLOV5目标检测 + 本章节基于 Lockzhiner Vision Module 和 YOLOv5 目标检测模型，实现实时目标检测功能。 + ## 1. 基本知识简介 + ### 1.1 目标检测简介 + 目标检测是计算机视觉领域中的一个关键任务，它不仅需要识别图像中存在哪些对象，还需要定位这些对象的位置。具体来说，目标检测算法会输出每个检测到的对象的边界框（Bounding Box）以及其所属类别的概率或置信度得分。 + - 应用场景：目标检测技术广泛应用于多个领域，包括但不限于安全监控、自动驾驶汽车、智能零售和医疗影像分析。 + ### 1.2 YOLOv5简介 + YOLOv5 是 Ultralytics 在 2020 年基于 PyTorch 开发的一种高效目标检测模型，属于 YOLO 系列。它通过一次前向传播即可预测目标的类别和位置，适用于实时检测任务。YOLOv5 提供了多种模型大小（如 YOLOv5s、m、l、x），适应不同硬件条件。其结构包括骨干网络 CSPDarknet、特征融合层和检测头，支持多尺度预测和自定义训练，广泛用于各种检测场景。 ## 2. API 文档 + ### 2.1 YOLOv5模型类 + #### 2.1.1 头文件 + ```cpp #include "yolov5.h" ``` + #### 2.1.2 模型初始化函数 + ```cpp int init_yolov5_model(const char* model_path, rknn_app_context_t* ctx); ``` + - 作用：加载YOLOv5 RKNN模型并初始化推理上下文 - 参数 - - model_path：RKNN模型文件路径 - - ctx：模型上下文指针 + - model_path：RKNN模型文件路径 + - ctx：模型上下文指针 - 返回值： - - 0：初始化成功 - - -1：初始化失败 + - 0：初始化成功 + - -1：初始化失败 + #### 2.1.3 模型推理函数 + ```cpp int inference_yolov5_model(rknn_app_context_t* ctx, object_detect_result_list* od_results); ``` + - 作用：执行模型推理并获取检测结果 - 参数： - - ctx：已初始化的模型上下文 - - od_results：检测结果存储结构体指针 + - ctx：已初始化的模型上下文 + - od_results：检测结果存储结构体指针 - 返回值： - - 0：推理成功 - - -1 ：推理失败 + - 0：推理成功 + - -1 ：推理失败 + #### 2.1.4 模型释放函数 + ```cpp void release_yolov5_model(rknn_app_context_t* ctx); ``` + - 作用：释放模型相关资源 - 参数： - - ctx：待释放的模型上下文 + - ctx：待释放的模型上下文 - 返回值：无 + ### 2.2 图像处理函数 + #### 2.2.1 Letterbox处理 + ```cpp cv::Mat letterbox(cv::Mat& image); ``` + - 作用：保持图像比例进行缩放，添加灰边填充 - 参数： -image:输入图像(RGB格式) - 返回值： - - 返回预处理图像 + - 返回预处理图像 + #### 2.2.2 坐标映射函数 + ```cpp void mapCoordinates(float& x, float& y); ``` + - 作用：将模型输出坐标映射回原始图像坐标系 - 参数： - - x/y：模型输出坐标（输入输出参数） + - x/y：模型输出坐标（输入输出参数） - 返回值:无 + ### 2.3 结果处理函数 + #### 2.3.1 后处理初始化 + ```cpp void init_post_process(); ``` + - 作用：加载类别标签文件 - 参数：无 - 返回值：无 + #### 2.3.2 结果绘制函数 + ```cpp void draw_detections(int count, object_detect_result* results, cv::Mat& frame, void (*mapFunc)(float&, float&)); ``` + - 作用：在图像上绘制检测框和标签 - 参数： - - count:检测结果数量 - - results：检测结果数组 - - frame：目标图像帧 - - mapFunc：坐标映射函数指针 + - count:检测结果数量 + - results：检测结果数组 + - frame：目标图像帧 + - mapFunc：坐标映射函数指针 - 返回值：无 + ## 3. 代码解析 + ### 3.1 流程图 ### 3.2核心代码解析 + - 模型初始化 + ```cpp rknn_app_context_t rknn_app_ctx; init_yolov5_model("yolov5s.rknn", &rknn_app_ctx); ``` + - 图像预处理 + ```cpp cv::Mat letterboxImage = letterbox(frame); // 保持比例缩放 memcpy(rknn_app_ctx.input_mems[0]->virt_addr, letterboxImage.data, model_width * model_height * 3); ``` + - 模型推理 + ```cpp object_detect_result_list od_results; inference_yolov5_model(&rknn_app_ctx, &od_results); ``` + - 结果可视化 + ```cpp draw_detections(od_results.count, od_results.results, frame, mapCoordinates); ``` + ### 3.3 完整代码实现 + ```cpp #include #include @@ -146,117 +191,121 @@ char *lable; int main(int argc, char *argv[]) { - if (argc != 4) - { - LOGGER_INFO("Usage: %s ./yolov5_main model_path ./label size\n ./label_txt"); - } - obj_class_num = atoi(argv[2]); - lable = argv[3]; - // Rknn model - char text[16]; - // rknn上下文结构体 - rknn_app_context_t rknn_app_ctx; - object_detect_result_list od_results; - int ret; - const char *model_path = argv[1]; - memset(&rknn_app_ctx, 0, sizeof(rknn_app_context_t)); - - // Step 1: Load RKNN model - if (init_yolov5_model(model_path, &rknn_app_ctx) != 0) - { - printf("❌ Failed to load RKNN model!\n"); - return -1; - } - printf("✅ RKNN model loaded successfully.\n"); - - // 加载标签文件 - init_post_process(); - - // 打开摄像头 - lockzhiner_vision_module::edit::Edit edit; - if (!edit.StartAndAcceptConnection()) - { - std::cerr << "Error: Failed to start and accept connection." << std::endl; - return EXIT_FAILURE; - } - - cv::VideoCapture cap; - cap.set(cv::CAP_PROP_FRAME_WIDTH, 640); - cap.set(cv::CAP_PROP_FRAME_HEIGHT, 480); - cap.open(0); - - if (!cap.isOpened()) - { - std::cerr << "Error: Could not open camera." << std::endl; - return 1; - } - cv::Mat frame; - // 在 while 循环外声明 start 和 end 变量 - std::chrono::steady_clock::time_point start, end; - - while (true) - { - // 记录开始时间 - start = std::chrono::steady_clock::now(); - // Step 2: Load image from command line - cap >> frame; - if (frame.empty()) - { - LOGGER_INFO("❌ Failed to read frame from camera.\n"); - continue; - } - cv::resize(frame, frame, cv::Size(width, height), 0, 0, cv::INTER_LINEAR); - cv::Mat letterboxImage = letterbox(frame); - - if (letterboxImage.empty() || letterboxImage.total() * letterboxImage.elemSize() != model_width * model_height * 3) - { - - LOGGER_ERROR("❌ Input image format or size mismatch!\n"); - release_yolov5_model(&rknn_app_ctx); - return -1; - } - - if (rknn_app_ctx.input_mems == nullptr || rknn_app_ctx.input_mems[0] == nullptr) - { - - LOGGER_ERROR("❌ RKNN input memory not allocated!\n"); - release_yolov5_model(&rknn_app_ctx); - return -1; - } - - memcpy(rknn_app_ctx.input_mems[0]->virt_addr, letterboxImage.data, model_width * model_height * 3); - - if (inference_yolov5_model(&rknn_app_ctx, &od_results) != 0) - { - LOGGER_ERROR("inference_yolov5_model failed"); - release_yolov5_model(&rknn_app_ctx); - return -1; - } - - draw_detections(od_results.count, // 传入结果数量 - od_results.results, // 传入结果数组 - frame, // 图像帧 - mapCoordinates); // 直接使用现有坐标映射函数 - - edit.Print(frame); - // 记录结束时间 - end = std::chrono::steady_clock::now(); - // 计算耗时（秒） - double elapsed_time = std::chrono::duration(end - start).count(); - printf("Frame processed in %.4f seconds\n", elapsed_time); - } - release_yolov5_model(&rknn_app_ctx); - deinit_post_process(); - cap.release(); - return 0; + if (argc != 4) + { + LOGGER_INFO("Usage: %s ./yolov5_main model_path ./label size\n ./label_txt"); + } + obj_class_num = atoi(argv[2]); + lable = argv[3]; + // Rknn model + char text[16]; + // rknn上下文结构体 + rknn_app_context_t rknn_app_ctx; + object_detect_result_list od_results; + int ret; + const char *model_path = argv[1]; + memset(&rknn_app_ctx, 0, sizeof(rknn_app_context_t)); + + // Step 1: Load RKNN model + if (init_yolov5_model(model_path, &rknn_app_ctx) != 0) + { + printf("❌ Failed to load RKNN model!\n"); + return -1; + } + printf("✅ RKNN model loaded successfully.\n"); + + // 加载标签文件 + init_post_process(); + + // 打开摄像头 + lockzhiner_vision_module::edit::Edit edit; + if (!edit.StartAndAcceptConnection()) + { + std::cerr << "Error: Failed to start and accept connection." << std::endl; + return EXIT_FAILURE; + } + + cv::VideoCapture cap; + cap.set(cv::CAP_PROP_FRAME_WIDTH, 640); + cap.set(cv::CAP_PROP_FRAME_HEIGHT, 480); + cap.open(0); + + if (!cap.isOpened()) + { + std::cerr << "Error: Could not open camera." << std::endl; + return 1; + } + cv::Mat frame; + // 在 while 循环外声明 start 和 end 变量 + std::chrono::steady_clock::time_point start, end; + + while (true) + { + // 记录开始时间 + start = std::chrono::steady_clock::now(); + // Step 2: Load image from command line + cap >> frame; + if (frame.empty()) + { + LOGGER_INFO("❌ Failed to read frame from camera.\n"); + continue; + } + cv::resize(frame, frame, cv::Size(width, height), 0, 0, cv::INTER_LINEAR); + cv::Mat letterboxImage = letterbox(frame); + + if (letterboxImage.empty() || letterboxImage.total() * letterboxImage.elemSize() != model_width * model_height * 3) + { + + LOGGER_ERROR("❌ Input image format or size mismatch!\n"); + release_yolov5_model(&rknn_app_ctx); + return -1; + } + + if (rknn_app_ctx.input_mems == nullptr || rknn_app_ctx.input_mems[0] == nullptr) + { + + LOGGER_ERROR("❌ RKNN input memory not allocated!\n"); + release_yolov5_model(&rknn_app_ctx); + return -1; + } + + memcpy(rknn_app_ctx.input_mems[0]->virt_addr, letterboxImage.data, model_width * model_height * 3); + + if (inference_yolov5_model(&rknn_app_ctx, &od_results) != 0) + { + LOGGER_ERROR("inference_yolov5_model failed"); + release_yolov5_model(&rknn_app_ctx); + return -1; + } + + draw_detections(od_results.count, // 传入结果数量 + od_results.results, // 传入结果数组 + frame, // 图像帧 + mapCoordinates); // 直接使用现有坐标映射函数 + + edit.Print(frame); + // 记录结束时间 + end = std::chrono::steady_clock::now(); + // 计算耗时（秒） + double elapsed_time = std::chrono::duration(end - start).count(); + printf("Frame processed in %.4f seconds\n", elapsed_time); + } + release_yolov5_model(&rknn_app_ctx); + deinit_post_process(); + cap.release(); + return 0; } ``` ## 4. 编译过程 + ### 4.1 编译环境搭建 + - 请确保你已经按照 [开发环境搭建指南](../../../../docs/introductory_tutorial/cpp_development_environment.md) 正确配置了开发环境。 - 同时以正确连接开发板。 + ### 4.2 Cmake介绍 + ```cmake # CMake最低版本要求 cmake_minimum_required(VERSION 3.10) @@ -302,8 +351,11 @@ install( RUNTIME DESTINATION . ) ``` + ### 4.3 编译项目 + 使用 Docker Destop 打开 LockzhinerVisionModule 容器并执行以下命令来编译项目 + ```bash # 进入Demo所在目录 cd /LockzhinerVisionModuleWorkSpace/LockzhinerVisionModule/Cpp_example/D10_yolov5 @@ -320,14 +372,17 @@ make -j8 && make install 在执行完上述命令后，会在build目录下生成可执行文件。 ## 5. 例程运行示例 + ### 5.1 运行 + ```shell chmod 777 yolov5_main # 在实际应用的过程中LZ-Picodet需要替换为下载的或者你的rknn模型 ./yolov5_main ./voc320.rknn 20 label ``` + ### 5.2 结果展示 - 可以看到我们可以正确识别多种类别的 -![](./images/D10_yolov5.png) \ No newline at end of file +![](./images/D10_yolov5.png) diff --git "a/Cpp_example/D10_yolov5/\346\250\241\345\236\213\346\226\207\344\273\266\350\275\254\346\215\242.md" "b/Cpp_example/D10_yolov5/\346\250\241\345\236\213\346\226\207\344\273\266\350\275\254\346\215\242.md" new file mode 100644 index 0000000000000000000000000000000000000000..1be139b6c89a12d30b4dbf76579c5cc9f8207a0b --- /dev/null +++ "b/Cpp_example/D10_yolov5/\346\250\241\345\236\213\346\226\207\344\273\266\350\275\254\346\215\242.md" @@ -0,0 +1,223 @@ +# 模型训练以及转换 + +## 1. 模型训练 + +### 1.1 下载模型训练代码 + +```shell +git clone https://github.com/airockchip/yolov5.git +``` + +### 1.2 准备数据集 + +将自定义数据集转换为yolo格式，下面以Labelme工具标注的数据集为例，运行以下代码进行数据集的转换： + +```python +import os +import json +import shutil +import random +from pathlib import Path + +def convert_labelme_to_yolo(src_dir, dest_dir, yaml_output_path, split_ratio=0.8): + src_dir = Path(src_dir) + dest_dir = Path(dest_dir) + yaml_output_path = Path(yaml_output_path) + + # 输入路径设置 + annotations_dir = src_dir / 'annotations' + images_dir = src_dir / 'images' + + if not annotations_dir.exists(): + print(f"找不到标注文件夹: {annotations_dir}") + return + + # 1. 创建 YOLOv5 所需目录结构 + for split in ['train', 'val']: + (dest_dir / 'images' / split).mkdir(parents=True, exist_ok=True) + (dest_dir / 'labels' / split).mkdir(parents=True, exist_ok=True) + + # 获取所有 json 文件 + json_files = list(annotations_dir.glob('*.json')) + if not json_files: + print("未找到任何 JSON 标注文件！") + return + + # 2. 随机打乱用于划分训练集 (80%) 和验证集 (20%) + random.seed(42) + random.shuffle(json_files) + train_count = int(len(json_files) * split_ratio) + + # 类别映射（目前只有一个类别 fire，这里需要根据你的数据集类别进行修改） + classes = ['fire'] + class_to_id = {cls: i for i, cls in enumerate(classes)} + + print(f"总计找到 {len(json_files)} 个标注文件，开始转换...") + + success_count = 0 + for i, json_path in enumerate(json_files): + split = 'train' if i < train_count else 'val' + + with open(json_path, 'r', encoding='utf-8') as f: + data = json.load(f) + + # 获取图像宽高 + img_w = data.get('imageWidth') + img_h = data.get('imageHeight') + + if not img_w or not img_h: + print(f"警告: {json_path.name} 中没有图像宽高数据，跳过。") + continue + + # 解析图像文件名 (兼容 Windows '\\' 和 Linux '/') + img_name = data.get('imagePath', '').replace('\\', '/').split('/')[-1] + if not img_name: + img_name = json_path.stem + '.png' + + src_img_path = images_dir / img_name + + # 3. 如果图片不存在，尝试其它常见的图片后缀 + if not src_img_path.exists(): + for ext in ['.jpg', '.jpeg', '.JPG', '.PNG']: + alt_img_path = images_dir / (json_path.stem + ext) + if alt_img_path.exists(): + src_img_path = alt_img_path + img_name = alt_img_path.name + break + else: + print(f"警告: 找不到对应的图像文件 {src_img_path}，跳过此样本。") + continue + + # 4. 准备输出 label 内容 + yolo_labels = [] + for shape in data.get('shapes', []): + label = shape.get('label') + if label not in class_to_id: + continue # 如果存在其它意外的类别则忽略 + + class_id = class_to_id[label] + points = shape.get('points') + + # 确保是矩形框并且有两个点 + if shape.get('shape_type') == 'rectangle' and len(points) == 2: + x1, y1 = points[0] + x2, y2 = points[1] + + # 确保坐标极值正确 + x_min, x_max = min(x1, x2), max(x1, x2) + y_min, y_max = min(y1, y2), max(y1, y2) + + # 转换为 YOLO 的中心点及宽高，并且归一化到 0~1 + x_center = ((x_min + x_max) / 2.0) / img_w + y_center = ((y_min + y_max) / 2.0) / img_h + w = (x_max - x_min) / img_w + h = (y_max - y_min) / img_h + + yolo_labels.append(f"{class_id} {x_center:.6f} {y_center:.6f} {w:.6f} {h:.6f}") + + # 5. 写入 YOLO 格式的 txt 标签文件 + dest_txt_path = dest_dir / 'labels' / split / (json_path.stem + '.txt') + with open(dest_txt_path, 'w', encoding='utf-8') as f: + f.write('\n'.join(yolo_labels)) + + # 6. 复制图像文件到 YOLO 目录 + dest_img_path = dest_dir / 'images' / split / img_name + shutil.copy(src_img_path, dest_img_path) + success_count += 1 + + # 7. 生成 YOLOv8 训练所需的 YAML 文件 + yaml_content = f"""path: {dest_dir.absolute()} # 数据集根目录 +train: images/train # 训练集图片目录 (相对于 path) +val: images/val # 验证集图片目录 (相对于 path) + +# 类别定义 +names: + 0: fire +""" + yaml_output_path.parent.mkdir(parents=True, exist_ok=True) + with open(yaml_output_path, 'w', encoding='utf-8') as f: + f.write(yaml_content) + + print(f"\n转换完成！成功处理 {success_count} 个样本。") + print(f"YOLO 格式数据集保存在: {dest_dir}") + print(f"YOLO 配置文件已生成: {yaml_output_path}") + +if __name__ == '__main__': + # 原始数据集路径 + SOURCE_DIR = 'path/to/yolov5/Dataset' + # 转换后YOLO数据集的存放路径 + DEST_DIR = 'path/to/yolov5/Dataset_YOLO' + # 要求的 yaml 配置文件生成路径 + YAML_PATH = 'path/to/yolov5/data/data.yaml' + + convert_labelme_to_yolo(SOURCE_DIR, DEST_DIR, YAML_PATH) +``` + +### 1.3 开始训练 + +运行以下代码开始训练yolov5模型 + +```python +python train.py --data ./data/data.yaml --epochs 300 --weights yolov5s.pt --cfg yolov5s.yaml --batch-size 32 +``` + +训练好的模型会保存在./runs/train/weights目录下。 + +## 2. 模型转换 + +### 2.1 导出onnx模型 + +由于是在rv1106上运行，对于模型转换请参考如下文档： + +[yolov5_export_onnx](https://github.com/airockchip/yolov5/blob/master/README_rkopt.md) + +在满足 ./requirements.txt 的环境要求后，执行以下语句导出模型 + +```shell +python export.py --rknpu --weight path/to/your/xxx.pt +``` + +执行完毕后，会生成 ONNX 模型. 假如原始模型为 yolov8s.pt，则生成 yolov8s.onnx 模型。 转换后的onnx模型会保存在runs/detect/train/weights目录下。 + +### 2.2 准备校准数据集 + +在量化时，提供 20~100 张有代表性的图片给 RKNN 即可。请在你的终端中运行以下命令，从你的 Dataset_YOLO/images/train 中提取部分图片路径生成校准文件： + +```shell +cd path/to/yolov5 +# 随机选取50张训练集的图片路径，写入 dataset_subset.txt +ls /path/to/yolov5/Dataset_YOLO/images/train/*.png | head -n 50 > dataset_subset.txt +``` + +- 注意：如果你的图片后缀是 .jpg ，请将上面的 *.png 替换为*.jpg + +### 2.3 转RKNN模型 + +#### 2.3.1 克隆rknn_model_zoo + +```shell +git clone https://github.com/airockchip/rknn_model_zoo.git +cd /path/to/rknn_model_zoo/examples/yolov5/python +``` + +#### 2.3.2 修改convert.py代码 + +主要是将 DATASET_PATH 指向刚才我们生成的 dataset_subset.txt ： + +```python +DATASET_PATH = 'path/to/dataset_subset.txt' +``` + +#### 2.3.3 运行convert.py + +在运行转换脚本前请确保你已经配置好了所需要的环境。并且将之前转换好的onnx模型放到以下目录： + +- /path/to/rknn_model_zoo/examples/yolov5/model/ + +执行以下命令开始转换 + +```shell +python convert.py ../model/xxxx.onnx rv1106 +``` + +执行完毕后，如果没有报错，你就会在 rknn_model_zoo/examples/yolov5/model/ 目录下得到 yolov5.rknn 文件，这个文件就可以直接放到你的 RV1106 板子上进行部署板上运行了！ diff --git a/Cpp_example/D15_yolov8/CMakeLists.txt b/Cpp_example/D15_yolov8/CMakeLists.txt new file mode 100755 index 0000000000000000000000000000000000000000..289ff4cb39ea4e8fe98a36d651510d55fb0c4ba8 --- /dev/null +++ b/Cpp_example/D15_yolov8/CMakeLists.txt @@ -0,0 +1,43 @@ +# CMake最低版本要求 +cmake_minimum_required(VERSION 3.10) + +project(yolov8) + +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED ON) + +# 定义项目根目录路径 +set(PROJECT_ROOT_PATH "${CMAKE_CURRENT_SOURCE_DIR}/../..") +message("PROJECT_ROOT_PATH = " ${PROJECT_ROOT_PATH}) + +include("${PROJECT_ROOT_PATH}/toolchains/arm-rockchip830-linux-uclibcgnueabihf.toolchain.cmake") + +# 定义 OpenCV SDK 路径 +set(OpenCV_ROOT_PATH "${PROJECT_ROOT_PATH}/third_party/opencv-mobile-4.10.0-lockzhiner-vision-module") +set(OpenCV_DIR "${OpenCV_ROOT_PATH}/lib/cmake/opencv4") +find_package(OpenCV REQUIRED) +set(OPENCV_LIBRARIES "${OpenCV_LIBS}") +# 定义 LockzhinerVisionModule SDK 路径 +set(LockzhinerVisionModule_ROOT_PATH "${PROJECT_ROOT_PATH}/third_party/lockzhiner_vision_module_sdk") +set(LockzhinerVisionModule_DIR "${LockzhinerVisionModule_ROOT_PATH}/lib/cmake/lockzhiner_vision_module") +find_package(LockzhinerVisionModule REQUIRED) + +set(RKNPU2_BACKEND_BASE_DIR "${LockzhinerVisionModule_ROOT_PATH}/include/lockzhiner_vision_module/vision/deep_learning/runtime") +if(NOT EXISTS ${RKNPU2_BACKEND_BASE_DIR}) + message(FATAL_ERROR "RKNPU2 backend base dir missing: ${RKNPU2_BACKEND_BASE_DIR}") +endif() + +add_executable(yolov8_main + main.cc + postprocess.cc + yolov8.cc + yolov8.h + postprocess.h +) +target_include_directories(yolov8_main PRIVATE ${LOCKZHINER_VISION_MODULE_INCLUDE_DIRS} ${rknpu2_INCLUDE_DIRS} ${RKNPU2_BACKEND_BASE_DIR}) +target_link_libraries(yolov8_main PRIVATE ${OPENCV_LIBRARIES} ${LOCKZHINER_VISION_MODULE_LIBRARIES}) + +install( + TARGETS yolov8_main + RUNTIME DESTINATION . +) \ No newline at end of file diff --git a/Cpp_example/D15_yolov8/README.md b/Cpp_example/D15_yolov8/README.md new file mode 100644 index 0000000000000000000000000000000000000000..33a170e547d9f6252aa359d4eea364b005322604 --- /dev/null +++ b/Cpp_example/D15_yolov8/README.md @@ -0,0 +1,521 @@ +# YOLOv8 目标检测 + +本章节基于 Lockzhiner Vision Module 和 YOLOv8 目标检测模型，实现实时目标检测功能。 + +## 1. 基本知识简介 + +### 1.1 目标检测简介 + +目标检测是计算机视觉领域中的一个关键任务，它不仅需要识别图像中存在哪些对象，还需要定位这些对象的位置。具体来说，目标检测算法会输出每个检测到的对象的边界框（Bounding Box）以及其所属类别的概率或置信度得分。 + +- 应用场景：目标检测技术广泛应用于多个领域，包括但不限于安全监控、自动驾驶汽车、智能零售和医疗影像分析。 + +### 1.2 YOLOv8 简介 + +YOLOv8 是 Ultralytics 在 2023 年推出的最新一代 YOLO 系列目标检测模型。相比 YOLOv5，YOLOv8 在架构和性能上都有显著提升： + +- **Anchor-Free 设计**：摒弃了传统的 Anchor 框机制，采用更简洁的 Anchor-Free 方式，提高了检测精度和泛化能力 +- **解耦头结构**：将分类和回归任务分离，使用两个独立的分支处理，提升了模型性能 +- **新的损失函数**：采用 CIoU Loss 和 DFL (Distribution Focal Loss) 组合，边界框定位更精准 +- **多尺度特征融合**：改进的 PANet 结构，更好地融合不同尺度的特征 +- **高效的主干网络**：优化的 CSPDarknet 结构，在速度和精度之间取得更好的平衡 + +YOLOv8 同样提供了多种模型规格（YOLOv8n/s/m/l/x），适用于从边缘设备到高性能服务器的各种应用场景。 + +## 2. API 文档 + +### 2.1 YOLOv8 模型类 + +#### 2.1.1 头文件 + +```cpp +#include "yolov8.h" +``` + +#### 2.1.2 模型初始化函数 + +```cpp +int init_yolov8_model(const char* model_path, rknn_app_context_t* ctx); +``` + +- 作用：加载 YOLOv8 RKNN 模型并初始化推理上下文 +- 参数： + - model_path：RKNN 模型文件路径 + - ctx：模型上下文指针 +- 返回值： + - 0：初始化成功 + - -1：初始化失败 + +#### 2.1.3 模型推理函数 + +```cpp +int inference_yolov8_model(rknn_app_context_t* ctx, + object_detect_result_list* od_results); +``` + +- 作用：执行模型推理并获取检测结果 +- 参数： + - ctx：已初始化的模型上下文 + - od_results：检测结果存储结构体指针 +- 返回值： + - 0：推理成功 + - -1：推理失败 + +#### 2.1.4 模型释放函数 + +```cpp +int release_yolov8_model(rknn_app_context_t* ctx); +``` + +- 作用：释放模型相关资源 +- 参数： + - ctx：待释放的模型上下文 +- 返回值： + - 0：释放成功 + - -1：释放失败 + +### 2.2 图像处理函数 + +#### 2.2.1 Letterbox 处理 + +```cpp +cv::Mat letterbox(cv::Mat input); +``` + +- 作用：保持图像比例进行缩放，添加灰边填充 +- 参数： + - input：输入图像 (RGB 格式) +- 返回值： + - 返回预处理后的 640x640 图像 + +#### 2.2.2 坐标映射函数 + +```cpp +void mapCoordinates(int *x, int *y); +``` + +- 作用：将模型输出的坐标映射回原始图像坐标系 +- 参数： + - x/y：模型输出坐标指针（输入输出参数） +- 返回值：无 + +### 2.3 结果处理函数 + +#### 2.3.1 后处理初始化 + +```cpp +int init_post_process(); +``` + +- 作用：加载类别标签文件 +- 参数：无 +- 返回值： + - 0：初始化成功 + - -1：初始化失败 + +#### 2.3.2 后处理释放 + +```cpp +void deinit_post_process(); +``` + +- 作用：释放后处理相关资源 +- 参数：无 +- 返回值：无 + +#### 2.3.3 类别名称获取 + +```cpp +char *coco_cls_to_name(int cls_id); +``` + +- 作用：将类别 ID 转换为可读的类别名称 +- 参数： + - cls_id：类别 ID +- 返回值： + - 类别名称字符串 + +#### 2.3.4 结果绘制函数 + +```cpp +void draw_detections(int count, + object_detect_result* results, + cv::Mat& frame, + void (*mapCoord)(int *, int *)); +``` + +- 作用：在图像上绘制检测框和标签 +- 参数： + - count：检测结果数量 + - results：检测结果数组 + - frame：目标图像帧 + - mapCoord：坐标映射函数指针 +- 返回值：无 + +#### 2.3.5 后处理核心函数 + +```cpp +int post_process(rknn_app_context_t *app_ctx, + void *outputs, + float conf_threshold, + float nms_threshold, + object_detect_result_list *od_results); +``` + +- 作用：解码模型输出，执行 DFL 和 NMS 处理 +- 参数： + - app_ctx：模型上下文 + - outputs：模型输出内存数组 + - conf_threshold：置信度阈值（默认 0.25） + - nms_threshold：NMS 阈值（默认 0.45） + - od_results：检测结果输出 +- 返回值： + - 0：处理成功 + - -1：处理失败 + +## 3. 代码解析 + +### 3.1 流程图 + +![](./images/yolov8.png) + +### 3.2 核心代码解析 + +- **模型初始化** + +```cpp +rknn_app_context_t rknn_app_ctx; +const char *model_path = argv[1]; +if (init_yolov8_model(model_path, &rknn_app_ctx) != 0) +{ + printf("❌ Failed to load RKNN model!\n"); + return -1; +} +printf("✅ RKNN model loaded successfully.\n"); +``` + +- **图像预处理** + +```cpp +cv::resize(frame, frame, cv::Size(width, height), 0, 0, cv::INTER_LINEAR); +cv::Mat letterboxImage = letterbox(frame); // 保持比例缩放，填充灰边 + +// 复制数据到 RKNN 输入内存 +memcpy(rknn_app_ctx.input_mems[0]->virt_addr, + letterboxImage.data, + model_width * model_height * 3); +``` + +- **模型推理** + +```cpp +object_detect_result_list od_results; +if (inference_yolov8_model(&rknn_app_ctx, &od_results) != 0) +{ + LOGGER_ERROR("inference_yolov8_model failed"); + return -1; +} +``` + +- **后处理（在 inference_yolov8_model 内部调用）** + +```cpp +// 解码三个检测头输出 +// 1. Box 分支：边界框回归 +// 2. Score_Sum 分支：快速过滤 +// 3. Score 分支：类别置信度 + +// DFL (Distribution Focal Loss) 解码边界框 +compute_dfl(before_dfl, dfl_len, box); + +// NMS (非极大值抑制) 过滤重叠框 +nms(validCount, filterBoxes, classId, indexArray, c, nms_threshold); +``` + +- **结果可视化** + +```cpp +draw_detections(od_results.count, + od_results.results, + frame, + mapCoordinates); +``` + +### 3.3 完整代码实现 + +```cpp +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "yolov8.h" + +// output img size +#define DISP_WIDTH 640 +#define DISP_HEIGHT 480 + +// disp size +int width = DISP_WIDTH; +int height = DISP_HEIGHT; + +// model size +int model_width = 640; +int model_height = 640; + +int leftPadding; +int topPadding; + +// label size +extern int obj_class_num; +char *lable; + +int main(int argc, char *argv[]) +{ + if (argc != 4) + { + LOGGER_INFO("Usage: %s ./yolov8_main model_path ./label size ./label_txt", argv[0]); + return -1; + } + obj_class_num = atoi(argv[2]); + lable = argv[3]; + if (obj_class_num <= 0) + { + LOGGER_ERROR("Invalid class count: %d", obj_class_num); + return -1; + } + + // rknn 上下文结构体 + rknn_app_context_t rknn_app_ctx; + object_detect_result_list od_results; + int ret; + const char *model_path = argv[1]; + memset(&rknn_app_ctx, 0, sizeof(rknn_app_context_t)); + + // Step 1: Load RKNN model + if (init_yolov8_model(model_path, &rknn_app_ctx) != 0) + { + printf("❌ Failed to load RKNN model!\n"); + return -1; + } + printf("✅ RKNN model loaded successfully.\n"); + + // 检查模型输出分支，动态调整类别数量 + int output_per_branch = rknn_app_ctx.io_num.n_output / 3; + int score_idx = output_per_branch > 0 ? (output_per_branch - 1) : -1; + if (score_idx >= 0) + { + int model_class_num = rknn_app_ctx.output_attrs[score_idx].dims[3]; + if (model_class_num > 0 && obj_class_num != model_class_num) + { + LOGGER_INFO("Class count mismatch, use model class count %d instead of arg %d", + model_class_num, obj_class_num); + obj_class_num = model_class_num; + } + } + + // 加载标签文件 + if (init_post_process() != 0) + { + LOGGER_ERROR("init_post_process failed"); + release_yolov8_model(&rknn_app_ctx); + return -1; + } + + // 打开摄像头 + lockzhiner_vision_module::edit::Edit edit; + if (!edit.StartAndAcceptConnection()) + { + std::cerr << "Error: Failed to start and accept connection." << std::endl; + return EXIT_FAILURE; + } + + cv::VideoCapture cap; + cap.set(cv::CAP_PROP_FRAME_WIDTH, 640); + cap.set(cv::CAP_PROP_FRAME_HEIGHT, 480); + cap.open(0); + + if (!cap.isOpened()) + { + std::cerr << "Error: Could not open camera." << std::endl; + return 1; + } + cv::Mat frame; + std::chrono::steady_clock::time_point start, end; + + while (true) + { + start = std::chrono::steady_clock::now(); + cap >> frame; + if (frame.empty()) + { + LOGGER_INFO("❌ Failed to read frame from camera.\n"); + continue; + } + cv::resize(frame, frame, cv::Size(width, height), 0, 0, cv::INTER_LINEAR); + cv::Mat letterboxImage = letterbox(frame); + + if (letterboxImage.empty() || + letterboxImage.total() * letterboxImage.elemSize() != model_width * model_height * 3) + { + LOGGER_ERROR("❌ Input image format or size mismatch!\n"); + release_yolov8_model(&rknn_app_ctx); + return -1; + } + + if (rknn_app_ctx.input_mems == nullptr || + rknn_app_ctx.input_mems[0] == nullptr) + { + LOGGER_ERROR("❌ RKNN input memory not allocated!\n"); + release_yolov8_model(&rknn_app_ctx); + return -1; + } + + memcpy(rknn_app_ctx.input_mems[0]->virt_addr, + letterboxImage.data, + model_width * model_height * 3); + + if (inference_yolov8_model(&rknn_app_ctx, &od_results) != 0) + { + LOGGER_ERROR("inference_yolov8_model failed"); + release_yolov8_model(&rknn_app_ctx); + return -1; + } + + draw_detections(od_results.count, + od_results.results, + frame, + mapCoordinates); + + edit.Print(frame); + + end = std::chrono::steady_clock::now(); + double elapsed_time = std::chrono::duration(end - start).count(); + printf("Frame processed in %.4f seconds\n", elapsed_time); + } + + release_yolov8_model(&rknn_app_ctx); + deinit_post_process(); + cap.release(); + return 0; +} +``` + +## 4. 编译过程 + +### 4.1 编译环境搭建 + +- 请确保你已经按照 [开发环境搭建指南](../../../../docs/introductory_tutorial/cpp_development_environment.md) 正确配置了开发环境。 +- 同时已正确连接开发板。 + +### 4.2 CMake 介绍 + +```cmake +# CMake 最低版本要求 +cmake_minimum_required(VERSION 3.10) + +project(yolov8) + +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED ON) + +# 定义项目根目录路径 +set(PROJECT_ROOT_PATH "${CMAKE_CURRENT_SOURCE_DIR}/../..") +message("PROJECT_ROOT_PATH = " ${PROJECT_ROOT_PATH}) + +include("${PROJECT_ROOT_PATH}/toolchains/arm-rockchip830-linux-uclibcgnueabihf.toolchain.cmake") + +# 定义 OpenCV SDK 路径 +set(OpenCV_ROOT_PATH "${PROJECT_ROOT_PATH}/third_party/opencv-mobile-4.10.0-lockzhiner-vision-module") +set(OpenCV_DIR "${OpenCV_ROOT_PATH}/lib/cmake/opencv4") +find_package(OpenCV REQUIRED) +set(OPENCV_LIBRARIES "${OpenCV_LIBS}") + +# 定义 LockzhinerVisionModule SDK 路径 +set(LockzhinerVisionModule_ROOT_PATH "${PROJECT_ROOT_PATH}/third_party/lockzhiner_vision_module_sdk") +set(LockzhinerVisionModule_DIR "${LockzhinerVisionModule_ROOT_PATH}/lib/cmake/lockzhiner_vision_module") +find_package(LockzhinerVisionModule REQUIRED) + +set(RKNPU2_BACKEND_BASE_DIR "${LockzhinerVisionModule_ROOT_PATH}/include/lockzhiner_vision_module/vision/deep_learning/runtime") +if(NOT EXISTS ${RKNPU2_BACKEND_BASE_DIR}) + message(FATAL_ERROR "RKNPU2 backend base dir missing: ${RKNPU2_BACKEND_BASE_DIR}") +endif() + +add_executable(yolov8_main + main.cc + postprocess.cc + yolov8.cc + yolov8.h + postprocess.h +) +target_include_directories(yolov8_main PRIVATE + ${LOCKZHINER_VISION_MODULE_INCLUDE_DIRS} + ${rknpu2_INCLUDE_DIRS} + ${RKNPU2_BACKEND_BASE_DIR} +) +target_link_libraries(yolov8_main PRIVATE + ${OPENCV_LIBRARIES} + ${LOCKZHINER_VISION_MODULE_LIBRARIES} +) + +install( + TARGETS yolov8_main + RUNTIME DESTINATION . +) +``` + +### 4.3 编译项目 + +使用 Docker Desktop 打开 LockzhinerVisionModule 容器并执行以下命令来编译项目： + +```bash +# 进入 Demo 所在目录 +cd /LockzhinerVisionModuleWorkSpace/LockzhinerVisionModule/Cpp_example/D15_yolov8 + +# 创建编译目录 +rm -rf build && mkdir build && cd build + +# 配置交叉编译工具链 +export TOOLCHAIN_ROOT_PATH="/LockzhinerVisionModuleWorkSpace/arm-rockchip830-linux-uclibcgnueabihf" + +# 使用 cmake 配置项目 +cmake .. + +# 执行编译项目 +make -j8 && make install +``` + +在执行完上述命令后，会在 build 目录下生成可执行文件 `yolov8_main`。 + +## 5. 例程运行示例 + +### 5.1 运行 + +```shell +chmod 777 yolov8_main + +# 语法格式 +./yolov8_main + +# 实际示例（使用自定义火焰数据集训练的 1 类别模型） +./yolov8_main ./yolov8_fire_rv1106.rknn 1 ./labels.txt + +``` + +**参数说明：** + +- `model_path`：YOLOv8 RKNN 模型文件路径 +- `class_count`：类别数量 +- `label_file_path`：标签文件路径，每行一个类别名称 + +### 5.2 结果展示 + +- 可以准确识别火焰 +- 实时显示检测框、类别名称和置信度 +- 支持摄像头实时视频流检测 + +![](./images/yolov8_demo.png) diff --git a/Cpp_example/D15_yolov8/images/yolov8.png b/Cpp_example/D15_yolov8/images/yolov8.png new file mode 100755 index 0000000000000000000000000000000000000000..75a71d3a0f064a4796a2e77606f085a9419b1060 Binary files /dev/null and b/Cpp_example/D15_yolov8/images/yolov8.png differ diff --git a/Cpp_example/D15_yolov8/images/yolov8_demo.png b/Cpp_example/D15_yolov8/images/yolov8_demo.png new file mode 100755 index 0000000000000000000000000000000000000000..644656ccd232e88de1109049128febbdc8cf2fff Binary files /dev/null and b/Cpp_example/D15_yolov8/images/yolov8_demo.png differ diff --git a/Cpp_example/D15_yolov8/main.cc b/Cpp_example/D15_yolov8/main.cc new file mode 100755 index 0000000000000000000000000000000000000000..5221bf1a979fb92d0dc6990206ba08c4bc3041f9 --- /dev/null +++ b/Cpp_example/D15_yolov8/main.cc @@ -0,0 +1,161 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "yolov8.h" + +// output img size +#define DISP_WIDTH 640 +#define DISP_HEIGHT 480 + +// disp size +int width = DISP_WIDTH; +int height = DISP_HEIGHT; + +// model size +int model_width = 640; +int model_height = 640; + +int leftPadding; +int topPadding; + +// label size +extern int obj_class_num; +char *lable; + +int main(int argc, char *argv[]) +{ + if (argc != 4) + { + LOGGER_INFO("Usage: %s ./yolov8_main model_path ./label size ./label_txt", argv[0]); + return -1; + } + obj_class_num = atoi(argv[2]); + lable = argv[3]; + if (obj_class_num <= 0) + { + LOGGER_ERROR("Invalid class count: %d", obj_class_num); + return -1; + } + + // rknn上下文结构体 + rknn_app_context_t rknn_app_ctx; + object_detect_result_list od_results; + int ret; + const char *model_path = argv[1]; + memset(&rknn_app_ctx, 0, sizeof(rknn_app_context_t)); + + // Step 1: Load RKNN model + if (init_yolov8_model(model_path, &rknn_app_ctx) != 0) + { + printf("❌ Failed to load RKNN model!\n"); + return -1; + } + printf("✅ RKNN model loaded successfully.\n"); + + int output_per_branch = rknn_app_ctx.io_num.n_output / 3; + int score_idx = output_per_branch > 0 ? (output_per_branch - 1) : -1; + if (score_idx >= 0) + { + int model_class_num = rknn_app_ctx.output_attrs[score_idx].dims[3]; + if (model_class_num > 0 && obj_class_num != model_class_num) + { + LOGGER_INFO("Class count mismatch, use model class count %d instead of arg %d", model_class_num, obj_class_num); + obj_class_num = model_class_num; + } + } + + // 加载标签文件 + if (init_post_process() != 0) + { + LOGGER_ERROR("init_post_process failed"); + release_yolov8_model(&rknn_app_ctx); + return -1; + } + + // 打开摄像头 + lockzhiner_vision_module::edit::Edit edit; + if (!edit.StartAndAcceptConnection()) + { + std::cerr << "Error: Failed to start and accept connection." << std::endl; + return EXIT_FAILURE; + } + + cv::VideoCapture cap; + cap.set(cv::CAP_PROP_FRAME_WIDTH, 640); + cap.set(cv::CAP_PROP_FRAME_HEIGHT, 480); + cap.open(0); + + if (!cap.isOpened()) + { + std::cerr << "Error: Could not open camera." << std::endl; + return 1; + } + cv::Mat frame; + std::chrono::steady_clock::time_point start, end; + + while (true) + { + start = std::chrono::steady_clock::now(); + cap >> frame; + if (frame.empty()) + { + LOGGER_INFO("❌ Failed to read frame from camera.\n"); + continue; + } + cv::resize(frame, frame, cv::Size(width, height), 0, 0, cv::INTER_LINEAR); + cv::Mat letterboxImage = letterbox(frame); + + if (letterboxImage.empty() || letterboxImage.total() * letterboxImage.elemSize() != model_width * model_height * 3) + { + LOGGER_ERROR("❌ Input image format or size mismatch!\n"); + release_yolov8_model(&rknn_app_ctx); + return -1; + } + + if (rknn_app_ctx.input_mems == nullptr || rknn_app_ctx.input_mems[0] == nullptr) + { + LOGGER_ERROR("❌ RKNN input memory not allocated!\n"); + release_yolov8_model(&rknn_app_ctx); + return -1; + } + + memcpy(rknn_app_ctx.input_mems[0]->virt_addr, letterboxImage.data, model_width * model_height * 3); + + if (inference_yolov8_model(&rknn_app_ctx, &od_results) != 0) + { + LOGGER_ERROR("inference_yolov8_model failed"); + release_yolov8_model(&rknn_app_ctx); + return -1; + } + + // printf("draw_detections start, count=%d\n", od_results.count); + // fflush(stdout); + draw_detections(od_results.count, + od_results.results, + frame, + mapCoordinates); + // printf("draw_detections done\n"); + // fflush(stdout); + + // printf("edit.Print start\n"); + // fflush(stdout); + + edit.Print(frame); + + // printf("edit.Print done\n"); + // fflush(stdout); + end = std::chrono::steady_clock::now(); + double elapsed_time = std::chrono::duration(end - start).count(); + printf("Frame processed in %.4f seconds\n", elapsed_time); + } + release_yolov8_model(&rknn_app_ctx); + deinit_post_process(); + cap.release(); + return 0; +} \ No newline at end of file diff --git a/Cpp_example/D15_yolov8/model/fire_label.txt b/Cpp_example/D15_yolov8/model/fire_label.txt new file mode 100755 index 0000000000000000000000000000000000000000..69b21656c0ecea94332352c454ba30bf1e0586bb --- /dev/null +++ b/Cpp_example/D15_yolov8/model/fire_label.txt @@ -0,0 +1 @@ +fire \ No newline at end of file diff --git a/Cpp_example/D15_yolov8/postprocess.cc b/Cpp_example/D15_yolov8/postprocess.cc new file mode 100755 index 0000000000000000000000000000000000000000..ec6c51ce746d0f56d5972ea22b406ae75ddd863a --- /dev/null +++ b/Cpp_example/D15_yolov8/postprocess.cc @@ -0,0 +1,564 @@ +#include "yolov8.h" + +#include +#include +#include +#include +#include +#include + +#include +#include +#define LABEL_NALE_TXT_PATH "./model/coco_80_labels_list.txt" + +int OBJ_CLASS_NUM = 1; // 默认值（通过 init_post_process 修改） +static char** labels = nullptr; // 动态分配的标签数组 +int obj_class_num = 0; + + +inline static int clamp(float val, int min, int max) { return val > min ? (val < max ? val : max) : min; } +extern char *lable; +static char *readLine(FILE *fp, char *buffer, int *len) +{ + int ch; + int i = 0; + size_t buff_len = 0; + + buffer = (char *)malloc(buff_len + 1); + if (!buffer) + return NULL; // Out of memory + + while ((ch = fgetc(fp)) != '\n' && ch != EOF) + { + if (ch == '\r') continue; // 过滤 Windows 的 \r 换行符，防止 cv::putText 卡死 + buff_len++; + void *tmp = realloc(buffer, buff_len + 1); + if (tmp == NULL) + { + free(buffer); + return NULL; // Out of memory + } + buffer = (char *)tmp; + + buffer[i] = (char)ch; + i++; + } + buffer[i] = '\0'; + + *len = buff_len; + + // Detect end + if (ch == EOF && (i == 0 || ferror(fp))) + { + free(buffer); + return NULL; + } + return buffer; +} + +static int readLines(const char *fileName, char *lines[], int max_line) +{ + FILE *file = fopen(fileName, "r"); + char *s = NULL; + int i = 0; + int n = 0; + + if (file == NULL) + { + printf("Open %s fail!\n", fileName); + return -1; + } + + while ((s = readLine(file, s, &n)) != NULL) + { + lines[i++] = s; + if (i >= max_line) + break; + } + fclose(file); + return i; +} + +static int loadLabelName(const char *locationFilename, char *label[]) +{ + printf("load lable %s\n", locationFilename); + readLines(locationFilename, label, OBJ_CLASS_NUM); + return 0; +} + +static float CalculateOverlap(float xmin0, float ymin0, float xmax0, float ymax0, float xmin1, float ymin1, float xmax1, + float ymax1) +{ + float w = fmax(0.f, fmin(xmax0, xmax1) - fmax(xmin0, xmin1) + 1.0); + float h = fmax(0.f, fmin(ymax0, ymax1) - fmax(ymin0, ymin1) + 1.0); + float i = w * h; + float u = (xmax0 - xmin0 + 1.0) * (ymax0 - ymin0 + 1.0) + (xmax1 - xmin1 + 1.0) * (ymax1 - ymin1 + 1.0) - i; + return u <= 0.f ? 0.f : (i / u); +} + +static int nms(int validCount, std::vector &outputLocations, std::vector classIds, std::vector &order, + int filterId, float threshold) +{ + for (int i = 0; i < validCount; ++i) + { + int n = order[i]; + if (n == -1 || classIds[n] != filterId) + { + continue; + } + for (int j = i + 1; j < validCount; ++j) + { + int m = order[j]; + if (m == -1 || classIds[m] != filterId) + { + continue; + } + float xmin0 = outputLocations[n * 4 + 0]; + float ymin0 = outputLocations[n * 4 + 1]; + float xmax0 = outputLocations[n * 4 + 0] + outputLocations[n * 4 + 2]; + float ymax0 = outputLocations[n * 4 + 1] + outputLocations[n * 4 + 3]; + + float xmin1 = outputLocations[m * 4 + 0]; + float ymin1 = outputLocations[m * 4 + 1]; + float xmax1 = outputLocations[m * 4 + 0] + outputLocations[m * 4 + 2]; + float ymax1 = outputLocations[m * 4 + 1] + outputLocations[m * 4 + 3]; + + float iou = CalculateOverlap(xmin0, ymin0, xmax0, ymax0, xmin1, ymin1, xmax1, ymax1); + + if (iou > threshold) + { + order[j] = -1; + } + } + } + return 0; +} + +static int quick_sort_indice_inverse(std::vector &input, int left, int right, std::vector &indices) +{ + float key; + int key_index; + int low = left; + int high = right; + if (left < right) + { + key_index = indices[left]; + key = input[left]; + while (low < high) + { + while (low < high && input[high] <= key) + { + high--; + } + input[low] = input[high]; + indices[low] = indices[high]; + while (low < high && input[low] >= key) + { + low++; + } + input[high] = input[low]; + indices[high] = indices[low]; + } + input[low] = key; + indices[low] = key_index; + quick_sort_indice_inverse(input, left, low - 1, indices); + quick_sort_indice_inverse(input, low + 1, right, indices); + } + return low; +} + +static float sigmoid(float x) { return 1.0 / (1.0 + expf(-x)); } + +static float unsigmoid(float y) { return -1.0 * logf((1.0 / y) - 1.0); } + +inline static int32_t __clip(float val, float min, float max) +{ + float f = val <= min ? min : (val >= max ? max : val); + return f; +} + +static int8_t qnt_f32_to_affine(float f32, int32_t zp, float scale) +{ + float dst_val = (f32 / scale) + zp; + int8_t res = (int8_t)__clip(dst_val, -128, 127); + return res; +} + +static float deqnt_affine_to_f32(int8_t qnt, int32_t zp, float scale) { return ((float)qnt - (float)zp) * scale; } + +static void compute_dfl(float* tensor, int dfl_len, float* box){ + for (int b=0; b<4; b++){ + float exp_t[dfl_len]; + float exp_sum=0; + float acc_sum=0; + for (int i=0; i< dfl_len; i++){ + exp_t[i] = exp(tensor[i+b*dfl_len]); + exp_sum += exp_t[i]; + } + + for (int i=0; i< dfl_len; i++){ + acc_sum += exp_t[i]/exp_sum *i; + } + box[b] = acc_sum; + } +} + +static int process_i8(int8_t *box_tensor, int32_t box_zp, float box_scale, + int8_t *score_tensor, int32_t score_zp, float score_scale, + int8_t *score_sum_tensor, int32_t score_sum_zp, float score_sum_scale, + int grid_h, int grid_w, int stride, int dfl_len, + std::vector &boxes, + std::vector &objProbs, + std::vector &classId, + float threshold) +{ + int validCount = 0; + int grid_len = grid_h * grid_w; + int8_t score_thres_i8 = qnt_f32_to_affine(threshold, score_zp, score_scale); + int8_t score_sum_thres_i8 = qnt_f32_to_affine(threshold, score_sum_zp, score_sum_scale); + + for (int i = 0; i < grid_h; i++) + { + for (int j = 0; j < grid_w; j++) + { + int offset = i* grid_w + j; + int max_class_id = -1; + + // 通过 score sum 起到快速过滤的作用 + if (score_sum_tensor != nullptr){ + if (score_sum_tensor[offset] < score_sum_thres_i8){ + continue; + } + } + + int8_t max_score = -score_zp; + for (int c= 0; c< OBJ_CLASS_NUM; c++){ + if ((score_tensor[offset] > score_thres_i8) && (score_tensor[offset] > max_score)) + { + max_score = score_tensor[offset]; + max_class_id = c; + } + offset += grid_len; + } + + // compute box + if (max_score> score_thres_i8){ + offset = i* grid_w + j; + float box[4]; + float before_dfl[dfl_len*4]; + for (int k=0; k< dfl_len*4; k++){ + before_dfl[k] = deqnt_affine_to_f32(box_tensor[offset], box_zp, box_scale); + offset += grid_len; + } + compute_dfl(before_dfl, dfl_len, box); + + float x1,y1,x2,y2,w,h; + x1 = (-box[0] + j + 0.5)*stride; + y1 = (-box[1] + i + 0.5)*stride; + x2 = (box[2] + j + 0.5)*stride; + y2 = (box[3] + i + 0.5)*stride; + w = x2 - x1; + h = y2 - y1; + boxes.push_back(x1); + boxes.push_back(y1); + boxes.push_back(w); + boxes.push_back(h); + + objProbs.push_back(deqnt_affine_to_f32(max_score, score_zp, score_scale)); + classId.push_back(max_class_id); + validCount ++; + } + } + } + return validCount; +} + +static int process_fp32(float *box_tensor, float *score_tensor, float *score_sum_tensor, + int grid_h, int grid_w, int stride, int dfl_len, + std::vector &boxes, + std::vector &objProbs, + std::vector &classId, + float threshold) +{ + int validCount = 0; + int grid_len = grid_h * grid_w; + for (int i = 0; i < grid_h; i++) + { + for (int j = 0; j < grid_w; j++) + { + int offset = i* grid_w + j; + int max_class_id = -1; + + // 通过 score sum 起到快速过滤的作用 + if (score_sum_tensor != nullptr){ + if (score_sum_tensor[offset] < threshold){ + continue; + } + } + + float max_score = 0; + for (int c= 0; c< OBJ_CLASS_NUM; c++){ + if ((score_tensor[offset] > threshold) && (score_tensor[offset] > max_score)) + { + max_score = score_tensor[offset]; + max_class_id = c; + } + offset += grid_len; + } + + // compute box + if (max_score> threshold){ + offset = i* grid_w + j; + float box[4]; + float before_dfl[dfl_len*4]; + for (int k=0; k< dfl_len*4; k++){ + before_dfl[k] = box_tensor[offset]; + offset += grid_len; + } + compute_dfl(before_dfl, dfl_len, box); + + float x1,y1,x2,y2,w,h; + x1 = (-box[0] + j + 0.5)*stride; + y1 = (-box[1] + i + 0.5)*stride; + x2 = (box[2] + j + 0.5)*stride; + y2 = (box[3] + i + 0.5)*stride; + w = x2 - x1; + h = y2 - y1; + boxes.push_back(x1); + boxes.push_back(y1); + boxes.push_back(w); + boxes.push_back(h); + + objProbs.push_back(max_score); + classId.push_back(max_class_id); + validCount ++; + } + } + } + return validCount; +} + + +static int process_i8_rv1106(int8_t *box_tensor, int32_t box_zp, float box_scale, + int8_t *score_tensor, int32_t score_zp, float score_scale, + int8_t *score_sum_tensor, int32_t score_sum_zp, float score_sum_scale, + int grid_h, int grid_w, int stride, int dfl_len, int score_channels, + std::vector &boxes, + std::vector &objProbs, + std::vector &classId, + float threshold) { + int validCount = 0; + int grid_len = grid_h * grid_w; + int8_t score_thres_i8 = qnt_f32_to_affine(threshold, score_zp, score_scale); + int8_t score_sum_thres_i8 = qnt_f32_to_affine(threshold, score_sum_zp, score_sum_scale); + + for (int i = 0; i < grid_h; i++) { + for (int j = 0; j < grid_w; j++) { + int offset = i * grid_w + j; + int max_class_id = -1; + + // 通过 score sum 起到快速过滤的作用 + if (score_sum_tensor != nullptr) { + //score_sum_tensor [1, 1, 80, 80] + if (score_sum_tensor[offset] < score_sum_thres_i8) { + continue; + } + } + + int8_t max_score = -128; + int class_count = OBJ_CLASS_NUM < score_channels ? OBJ_CLASS_NUM : score_channels; + offset = offset * score_channels; + for (int c = 0; c < class_count; c++) { + int8_t score = score_tensor[offset + c]; + if ((score > score_thres_i8) && (score > max_score)) { + max_score = score; + max_class_id = c; + } + } + + // compute box + if (max_class_id >= 0 && max_score > score_thres_i8) { + offset = (i * grid_w + j) * 4 * dfl_len; + float box[4]; + float before_dfl[dfl_len*4]; + for (int k=0; k< dfl_len*4; k++){ + before_dfl[k] = deqnt_affine_to_f32(box_tensor[offset + k], box_zp, box_scale); + } + compute_dfl(before_dfl, dfl_len, box); + + float x1, y1, x2, y2, w, h; + x1 = (-box[0] + j + 0.5) * stride; + y1 = (-box[1] + i + 0.5) * stride; + x2 = (box[2] + j + 0.5) * stride; + y2 = (box[3] + i + 0.5) * stride; + w = x2 - x1; + h = y2 - y1; + boxes.push_back(x1); + boxes.push_back(y1); + boxes.push_back(w); + boxes.push_back(h); + + objProbs.push_back(deqnt_affine_to_f32(max_score, score_zp, score_scale)); + classId.push_back(max_class_id); + validCount ++; + } + } + } + return validCount; +} + +int post_process(rknn_app_context_t *app_ctx, void *outputs, float conf_threshold, float nms_threshold, object_detect_result_list *od_results) +{ + rknn_tensor_mem **_outputs = (rknn_tensor_mem **)outputs; + std::vector filterBoxes; + std::vector objProbs; + std::vector classId; + int validCount = 0; + int stride = 0; + int grid_h = 0; + int grid_w = 0; + int model_in_w = app_ctx->model_width; + int model_in_h = app_ctx->model_height; + + memset(od_results, 0, sizeof(object_detect_result_list)); + + // default 3 branch + int output_per_branch = app_ctx->io_num.n_output / 3; + for (int i = 0; i < 3; i++) + { + int box_idx = i * output_per_branch; + int dfl_len = app_ctx->output_attrs[box_idx].dims[3] / 4; + void *score_sum = nullptr; + int32_t score_sum_zp = 0; + float score_sum_scale = 1.0; + if (output_per_branch == 3) { + score_sum = _outputs[i * output_per_branch + 1]->virt_addr; // 修复：YOLOv8 中 score_sum 在索引 1 + score_sum_zp = app_ctx->output_attrs[i * output_per_branch + 1].zp; + score_sum_scale = app_ctx->output_attrs[i * output_per_branch + 1].scale; + } + int score_idx = i * output_per_branch + 2; // 修复：YOLOv8 中 score 在索引 2 + grid_h = app_ctx->output_attrs[box_idx].dims[1]; + grid_w = app_ctx->output_attrs[box_idx].dims[2]; + stride = model_in_h / grid_h; + + if (app_ctx->is_quant) { + validCount += process_i8_rv1106((int8_t *)_outputs[box_idx]->virt_addr, app_ctx->output_attrs[box_idx].zp, app_ctx->output_attrs[box_idx].scale, + (int8_t *)_outputs[score_idx]->virt_addr, app_ctx->output_attrs[score_idx].zp, + app_ctx->output_attrs[score_idx].scale, (int8_t *)score_sum, score_sum_zp, score_sum_scale, + grid_h, grid_w, stride, dfl_len, app_ctx->output_attrs[score_idx].dims[3], filterBoxes, objProbs, classId, conf_threshold); + } + else + { + printf("RV1106 only support quantization mode\n"); + return -1; + } + } + + // no object detect + if (validCount <= 0) + { + return 0; + } + std::vector indexArray; + for (int i = 0; i < validCount; ++i) + { + indexArray.push_back(i); + } + quick_sort_indice_inverse(objProbs, 0, validCount - 1, indexArray); + + std::set class_set(std::begin(classId), std::end(classId)); + + for (auto c : class_set) + { + nms(validCount, filterBoxes, classId, indexArray, c, nms_threshold); + } + + int last_count = 0; + od_results->count = 0; + + /* box valid detect target */ + for (int i = 0; i < validCount; ++i) + { + if (indexArray[i] == -1 || last_count >= OBJ_NUMB_MAX_SIZE) + { + continue; + } + int n = indexArray[i]; + + float x1 = filterBoxes[n * 4 + 0]; + float y1 = filterBoxes[n * 4 + 1]; + float x2 = x1 + filterBoxes[n * 4 + 2]; + float y2 = y1 + filterBoxes[n * 4 + 3]; + int id = classId[n]; + float obj_conf = objProbs[n]; + + od_results->results[last_count].box.left = (int)(clamp(x1, 0, model_in_w)); + od_results->results[last_count].box.top = (int)(clamp(y1, 0, model_in_h)); + od_results->results[last_count].box.right = (int)(clamp(x2, 0, model_in_w)); + od_results->results[last_count].box.bottom = (int)(clamp(y2, 0, model_in_h)); + od_results->results[last_count].prop = obj_conf; + od_results->results[last_count].cls_id = id; + last_count++; + } + od_results->count = last_count; + return 0; +} + +int init_post_process() +{ + OBJ_CLASS_NUM = obj_class_num; + + // 释放旧内存（防止内存泄漏） + if (labels != nullptr) { + deinit_post_process(); + } + + // 分配新内存并清零（防止野指针） + labels = new char*[OBJ_CLASS_NUM]; + memset(labels, 0, sizeof(char*) * OBJ_CLASS_NUM); + + int ret = 0; + ret = loadLabelName(lable, labels); + if (ret < 0) + { + printf("Load %s failed!\n", lable); + return -1; + } + return 0; +} + +char *coco_cls_to_name(int cls_id) +{ + + static char null_str[] = "null"; + + if (cls_id >= OBJ_CLASS_NUM) + { + return null_str; + } + + if (labels[cls_id]) + { + return labels[cls_id]; + } + + return null_str; +} + +void deinit_post_process() +{ + if (labels == nullptr) + { + return; + } + for (int i = 0; i < OBJ_CLASS_NUM; i++) + { + if (labels[i] != nullptr) + { + free(labels[i]); + labels[i] = nullptr; + } + } + delete[] labels; + labels = nullptr; +} \ No newline at end of file diff --git a/Cpp_example/D15_yolov8/postprocess.h b/Cpp_example/D15_yolov8/postprocess.h new file mode 100755 index 0000000000000000000000000000000000000000..d467c307692010d69e898264b4581fd21a592652 --- /dev/null +++ b/Cpp_example/D15_yolov8/postprocess.h @@ -0,0 +1,38 @@ +#ifndef _RKNN_YOLOV8_DEMO_POSTPROCESS_H_ +#define _RKNN_YOLOV8_DEMO_POSTPROCESS_H_ + +#include +#include +#include "rknpu2_backend/rknpu2_backend.h" + +#define OBJ_NAME_MAX_SIZE 64 +#define OBJ_NUMB_MAX_SIZE 128 +#define NMS_THRESH 0.45 +#define BOX_THRESH 0.25 + +typedef struct { + int left; + int top; + int right; + int bottom; +} image_rect_t; + +typedef struct { + image_rect_t box; + float prop; + int cls_id; +} object_detect_result; + +typedef struct { + int id; + int count; + object_detect_result results[OBJ_NUMB_MAX_SIZE]; +} object_detect_result_list; + +int init_post_process(); +void deinit_post_process(); +char *coco_cls_to_name(int cls_id); +int post_process(rknn_app_context_t *app_ctx, void *outputs, float conf_threshold, float nms_threshold, object_detect_result_list *od_results); + +void deinitPostProcess(); +#endif //_RKNN_YOLOV8_DEMO_POSTPROCESS_H_ \ No newline at end of file diff --git a/Cpp_example/D15_yolov8/yolov8.cc b/Cpp_example/D15_yolov8/yolov8.cc new file mode 100755 index 0000000000000000000000000000000000000000..fdb6258723d01f54640143c45eb776ea5bae799c --- /dev/null +++ b/Cpp_example/D15_yolov8/yolov8.cc @@ -0,0 +1,273 @@ +#include +#include +#include +#include +#include +#include +#include "yolov8.h" + +static void dump_tensor_attr(rknn_tensor_attr *attr) +{ + printf(" index=%d, name=%s, n_dims=%d, dims=[%d, %d, %d, %d], n_elems=%d, size=%d, fmt=%s, type=%s, qnt_type=%s, " + "zp=%d, scale=%f\n", + attr->index, attr->name, attr->n_dims, attr->dims[0], attr->dims[1], attr->dims[2], attr->dims[3], + attr->n_elems, attr->size, get_format_string(attr->fmt), get_type_string(attr->type), + get_qnt_type_string(attr->qnt_type), attr->zp, attr->scale); +} + +int init_yolov8_model(const char *model_path, rknn_app_context_t *app_ctx) +{ + int ret; + int model_len = 0; + char *model; + rknn_context ctx = 0; + + ret = rknn_init(&ctx, (char *)model_path, 0, 0, NULL); + if (ret < 0) + { + printf("rknn_init fail! ret=%d\n", ret); + return -1; + } + + // Get Model Input Output Number + rknn_input_output_num io_num; + ret = rknn_query(ctx, RKNN_QUERY_IN_OUT_NUM, &io_num, sizeof(io_num)); + if (ret != RKNN_SUCC) + { + printf("rknn_query fail! ret=%d\n", ret); + return -1; + } + printf("model input num: %d, output num: %d\n", io_num.n_input, io_num.n_output); + + // Get Model Input Info + printf("input tensors:\n"); + rknn_tensor_attr input_attrs[io_num.n_input]; + memset(input_attrs, 0, sizeof(input_attrs)); + for (int i = 0; i < io_num.n_input; i++) + { + input_attrs[i].index = i; + ret = rknn_query(ctx, RKNN_QUERY_NATIVE_INPUT_ATTR, &(input_attrs[i]), sizeof(rknn_tensor_attr)); + if (ret != RKNN_SUCC) + { + printf("rknn_query fail! ret=%d\n", ret); + return -1; + } + dump_tensor_attr(&(input_attrs[i])); + } + + // Get Model Output Info + printf("output tensors:\n"); + rknn_tensor_attr output_attrs[io_num.n_output]; + memset(output_attrs, 0, sizeof(output_attrs)); + for (int i = 0; i < io_num.n_output; i++) + { + output_attrs[i].index = i; + //When using the zero-copy API interface, query the native output tensor attribute + ret = rknn_query(ctx, RKNN_QUERY_NATIVE_NHWC_OUTPUT_ATTR, &(output_attrs[i]), sizeof(rknn_tensor_attr)); + if (ret != RKNN_SUCC) + { + printf("rknn_query fail! ret=%d\n", ret); + return -1; + } + dump_tensor_attr(&(output_attrs[i])); + } + + // default input type is int8 (normalize and quantize need compute in outside) + // if set uint8, will fuse normalize and quantize to npu + input_attrs[0].type = RKNN_TENSOR_UINT8; + // default fmt is NHWC,1106 npu only support NHWC in zero copy mode + input_attrs[0].fmt = RKNN_TENSOR_NHWC; + printf("input_attrs[0].size_with_stride=%d\n", input_attrs[0].size_with_stride); + app_ctx->input_mems[0] = rknn_create_mem(ctx, input_attrs[0].size_with_stride); + + // Set input tensor memory + ret = rknn_set_io_mem(ctx, app_ctx->input_mems[0], &input_attrs[0]); + if (ret < 0) { + printf("input_mems rknn_set_io_mem fail! ret=%d\n", ret); + return -1; + } + + // Set output tensor memory + for (uint32_t i = 0; i < io_num.n_output; ++i) { + app_ctx->output_mems[i] = rknn_create_mem(ctx, output_attrs[i].size_with_stride); + ret = rknn_set_io_mem(ctx, app_ctx->output_mems[i], &output_attrs[i]); + if (ret < 0) { + printf("output_mems rknn_set_io_mem fail! ret=%d\n", ret); + return -1; + } + } + + // Set to context + app_ctx->rknn_ctx = ctx; + + // TODO + if (output_attrs[0].qnt_type == RKNN_TENSOR_QNT_AFFINE_ASYMMETRIC) + { + app_ctx->is_quant = true; + } + else + { + app_ctx->is_quant = false; + } + + app_ctx->io_num = io_num; + app_ctx->input_attrs = (rknn_tensor_attr *)malloc(io_num.n_input * sizeof(rknn_tensor_attr)); + memcpy(app_ctx->input_attrs, input_attrs, io_num.n_input * sizeof(rknn_tensor_attr)); + app_ctx->output_attrs = (rknn_tensor_attr *)malloc(io_num.n_output * sizeof(rknn_tensor_attr)); + memcpy(app_ctx->output_attrs, output_attrs, io_num.n_output * sizeof(rknn_tensor_attr)); + + if (input_attrs[0].fmt == RKNN_TENSOR_NCHW) + { + printf("model is NCHW input fmt\n"); + app_ctx->model_channel = input_attrs[0].dims[1]; + app_ctx->model_height = input_attrs[0].dims[2]; + app_ctx->model_width = input_attrs[0].dims[3]; + } else + { + printf("model is NHWC input fmt\n"); + app_ctx->model_height = input_attrs[0].dims[1]; + app_ctx->model_width = input_attrs[0].dims[2]; + app_ctx->model_channel = input_attrs[0].dims[3]; + } + + printf("model input height=%d, width=%d, channel=%d\n", + app_ctx->model_height, app_ctx->model_width, app_ctx->model_channel); + + return 0; +} + +int release_yolov8_model(rknn_app_context_t *app_ctx) +{ + if (app_ctx->input_attrs != NULL) + { + free(app_ctx->input_attrs); + app_ctx->input_attrs = NULL; + } + if (app_ctx->output_attrs != NULL) + { + free(app_ctx->output_attrs); + app_ctx->output_attrs = NULL; + } + for (int i = 0; i < app_ctx->io_num.n_input; i++) { + if (app_ctx->input_mems[i] != NULL) { + rknn_destroy_mem(app_ctx->rknn_ctx, app_ctx->input_mems[i]); + } + } + for (int i = 0; i < app_ctx->io_num.n_output; i++) { + if (app_ctx->output_mems[i] != NULL) { + rknn_destroy_mem(app_ctx->rknn_ctx, app_ctx->output_mems[i]); + } + } + if (app_ctx->rknn_ctx != 0) + { + rknn_destroy(app_ctx->rknn_ctx); + app_ctx->rknn_ctx = 0; + } + + printf("Release success\n"); + return 0; +} + +int inference_yolov8_model(rknn_app_context_t *app_ctx, object_detect_result_list *od_results) +{ + int ret; + const float nms_threshold = NMS_THRESH; + const float box_conf_threshold = BOX_THRESH; + + + memset(od_results, 0x00, sizeof(*od_results)); + + printf("rknn_run...\n"); + auto rknn_start = std::chrono::steady_clock::now(); + ret = rknn_run(app_ctx->rknn_ctx, nullptr); + auto rknn_end = std::chrono::steady_clock::now(); + double rknn_time = std::chrono::duration(rknn_end - rknn_start).count(); + printf("rknn_run done, ret=%d, cost time: %.4f seconds\n", ret, rknn_time); + if (ret < 0) { + printf("rknn_run fail! ret=%d\n", ret); + return -1; + } + + post_process(app_ctx, app_ctx->output_mems, box_conf_threshold, nms_threshold, od_results); +out: + return ret; +} + +/* +预处理 +图片填充缩放 +*/ +float scale; +extern int model_width; +extern int model_height; + +extern int width; +extern int height; + +extern int leftPadding; +extern int topPadding; +cv::Mat letterbox(cv::Mat input) +{ + float scaleX = (float)model_width / (float)width; + float scaleY = (float)model_height / (float)height; + scale = scaleX < scaleY ? scaleX : scaleY; + + int inputWidth = (int)((float)width * scale); + int inputHeight = (int)((float)height * scale); + + leftPadding = (model_width - inputWidth) / 2; + topPadding = (model_height - inputHeight) / 2; + + cv::Mat inputScale; + cv::resize(input, inputScale, cv::Size(inputWidth, inputHeight), 0, 0, cv::INTER_LINEAR); + cv::Mat letterboxImage(model_width, model_width, CV_8UC3, cv::Scalar(0, 0, 0)); + cv::Rect roi(leftPadding, topPadding, inputWidth, inputHeight); + inputScale.copyTo(letterboxImage(roi)); + + return letterboxImage; +} +/* +将经过 letterbox 处理后的坐标映射回原始图像坐标系 +*/ +void mapCoordinates(int *x, int *y) +{ + int mx = *x - leftPadding; + int my = *y - topPadding; + + *x = (int)((float)mx / scale); + *y = (int)((float)my / scale); +} + +void draw_detections(int count, + object_detect_result *results, + cv::Mat &frame, + void (*mapCoord)(int *, int *)) +{ + char text[128]; + for (int i = 0; i < count; i++) + { + object_detect_result *det = &results[i]; + + int sX = (int)det->box.left; + int sY = (int)det->box.top; + int eX = (int)det->box.right; + int eY = (int)det->box.bottom; + + mapCoord(&sX, &sY); + mapCoord(&eX, &eY); + + cv::rectangle(frame, + cv::Point(sX, sY), + cv::Point(eX, eY), + cv::Scalar(0, 255, 0), 1); + + sprintf(text, "%s %.1f%%", + coco_cls_to_name(det->cls_id), + det->prop * 100); + + cv::putText(frame, text, + cv::Point(sX, sY - 8), + cv::FONT_HERSHEY_SIMPLEX, 1, + cv::Scalar(0, 255, 0), 1); + } +} \ No newline at end of file diff --git a/Cpp_example/D15_yolov8/yolov8.h b/Cpp_example/D15_yolov8/yolov8.h new file mode 100755 index 0000000000000000000000000000000000000000..e516765004787559ff264ebe3987bc254236cc87 --- /dev/null +++ b/Cpp_example/D15_yolov8/yolov8.h @@ -0,0 +1,45 @@ +#ifndef _RKNN_DEMO_YOLOV8_H_ +#define _RKNN_DEMO_YOLOV8_H_ + +#include "rknn_api.h" + +typedef struct { + char *dma_buf_virt_addr; + int dma_buf_fd; + int size; +} rknn_dma_buf; + +typedef struct { + rknn_context rknn_ctx; + rknn_input_output_num io_num; + rknn_tensor_attr* input_attrs; + rknn_tensor_attr* output_attrs; + + rknn_tensor_mem* input_mems[1]; + rknn_tensor_mem* output_mems[9]; + rknn_dma_buf img_dma_buf; + + int model_channel; + int model_width; + int model_height; + bool is_quant; +} rknn_app_context_t; + +#include "postprocess.h" + + +int init_yolov8_model(const char* model_path, rknn_app_context_t* app_ctx); + +int release_yolov8_model(rknn_app_context_t* app_ctx); + +int inference_yolov8_model(rknn_app_context_t* app_ctx, object_detect_result_list* od_results); + +#endif //_RKNN_DEMO_YOLOV8_H_ + +cv::Mat letterbox(cv::Mat input); +void mapCoordinates(int *x, int *y); + +void draw_detections(int count, + object_detect_result *results, + cv::Mat &frame, + void (*mapCoord)(int *, int *)); \ No newline at end of file diff --git "a/Cpp_example/D15_yolov8/\346\250\241\345\236\213\346\226\207\344\273\266\350\275\254\346\215\242.md" "b/Cpp_example/D15_yolov8/\346\250\241\345\236\213\346\226\207\344\273\266\350\275\254\346\215\242.md" new file mode 100755 index 0000000000000000000000000000000000000000..5a799cebe480068ecf7329bc8292b619e3930948 --- /dev/null +++ "b/Cpp_example/D15_yolov8/\346\250\241\345\236\213\346\226\207\344\273\266\350\275\254\346\215\242.md" @@ -0,0 +1,230 @@ +# 模型训练以及转换 + +## 1. 模型训练 + +### 1.1 下载模型训练代码 + +```shell +git clone https://github.com/airockchip/ultralytics_yolov8.git +``` + +### 1.2 准备数据集 + +将自定义数据集转换为yolo格式，下面以Labelme工具标注的数据集为例，运行以下代码进行数据集的转换： + +```python +import os +import json +import shutil +import random +from pathlib import Path + +def convert_labelme_to_yolo(src_dir, dest_dir, yaml_output_path, split_ratio=0.8): + src_dir = Path(src_dir) + dest_dir = Path(dest_dir) + yaml_output_path = Path(yaml_output_path) + + # 输入路径设置 + annotations_dir = src_dir / 'annotations' + images_dir = src_dir / 'images' + + if not annotations_dir.exists(): + print(f"找不到标注文件夹: {annotations_dir}") + return + + # 1. 创建 YOLOv8 所需目录结构 + for split in ['train', 'val']: + (dest_dir / 'images' / split).mkdir(parents=True, exist_ok=True) + (dest_dir / 'labels' / split).mkdir(parents=True, exist_ok=True) + + # 获取所有 json 文件 + json_files = list(annotations_dir.glob('*.json')) + if not json_files: + print("未找到任何 JSON 标注文件！") + return + + # 2. 随机打乱用于划分训练集 (80%) 和验证集 (20%) + random.seed(42) + random.shuffle(json_files) + train_count = int(len(json_files) * split_ratio) + + # 类别映射（目前只有一个类别 fire，这里需要根据你的数据集类别进行修改） + classes = ['fire'] + class_to_id = {cls: i for i, cls in enumerate(classes)} + + print(f"总计找到 {len(json_files)} 个标注文件，开始转换...") + + success_count = 0 + for i, json_path in enumerate(json_files): + split = 'train' if i < train_count else 'val' + + with open(json_path, 'r', encoding='utf-8') as f: + data = json.load(f) + + # 获取图像宽高 + img_w = data.get('imageWidth') + img_h = data.get('imageHeight') + + if not img_w or not img_h: + print(f"警告: {json_path.name} 中没有图像宽高数据，跳过。") + continue + + # 解析图像文件名 (兼容 Windows '\\' 和 Linux '/') + img_name = data.get('imagePath', '').replace('\\', '/').split('/')[-1] + if not img_name: + img_name = json_path.stem + '.png' + + src_img_path = images_dir / img_name + + # 3. 如果图片不存在，尝试其它常见的图片后缀 + if not src_img_path.exists(): + for ext in ['.jpg', '.jpeg', '.JPG', '.PNG']: + alt_img_path = images_dir / (json_path.stem + ext) + if alt_img_path.exists(): + src_img_path = alt_img_path + img_name = alt_img_path.name + break + else: + print(f"警告: 找不到对应的图像文件 {src_img_path}，跳过此样本。") + continue + + # 4. 准备输出 label 内容 + yolo_labels = [] + for shape in data.get('shapes', []): + label = shape.get('label') + if label not in class_to_id: + continue # 如果存在其它意外的类别则忽略 + + class_id = class_to_id[label] + points = shape.get('points') + + # 确保是矩形框并且有两个点 + if shape.get('shape_type') == 'rectangle' and len(points) == 2: + x1, y1 = points[0] + x2, y2 = points[1] + + # 确保坐标极值正确 + x_min, x_max = min(x1, x2), max(x1, x2) + y_min, y_max = min(y1, y2), max(y1, y2) + + # 转换为 YOLO 的中心点及宽高，并且归一化到 0~1 + x_center = ((x_min + x_max) / 2.0) / img_w + y_center = ((y_min + y_max) / 2.0) / img_h + w = (x_max - x_min) / img_w + h = (y_max - y_min) / img_h + + yolo_labels.append(f"{class_id} {x_center:.6f} {y_center:.6f} {w:.6f} {h:.6f}") + + # 5. 写入 YOLO 格式的 txt 标签文件 + dest_txt_path = dest_dir / 'labels' / split / (json_path.stem + '.txt') + with open(dest_txt_path, 'w', encoding='utf-8') as f: + f.write('\n'.join(yolo_labels)) + + # 6. 复制图像文件到 YOLO 目录 + dest_img_path = dest_dir / 'images' / split / img_name + shutil.copy(src_img_path, dest_img_path) + success_count += 1 + + # 7. 生成 YOLOv8 训练所需的 YAML 文件 + yaml_content = f"""path: {dest_dir.absolute()} # 数据集根目录 +train: images/train # 训练集图片目录 (相对于 path) +val: images/val # 验证集图片目录 (相对于 path) + +# 类别定义 +names: + 0: fire +""" + yaml_output_path.parent.mkdir(parents=True, exist_ok=True) + with open(yaml_output_path, 'w', encoding='utf-8') as f: + f.write(yaml_content) + + print(f"\n转换完成！成功处理 {success_count} 个样本。") + print(f"YOLO 格式数据集保存在: {dest_dir}") + print(f"YOLO 配置文件已生成: {yaml_output_path}") + +if __name__ == '__main__': + # 原始数据集路径 + SOURCE_DIR = 'path/to/ultralytics_yolov8/Dataset' + # 转换后YOLO数据集的存放路径 + DEST_DIR = 'path/to/ultralytics_yolov8/Dataset_YOLO' + # 要求的 yaml 配置文件生成路径 + YAML_PATH = 'path/to/ultralytics_yolov8/datasets/data.yaml' + + convert_labelme_to_yolo(SOURCE_DIR, DEST_DIR, YAML_PATH) +``` + +### 1.3 开始训练 + +运行以下代码开始训练yolov8模型 + +```python +from ultralytics import YOLO +from ultralytics.utils import LOGGER +model = YOLO("yolov8n.pt") +# model = YOLO("./ultralytics/cfg/models/v8/yolov8s.yaml") +results = model.train(data="datasets/data.yaml", epochs=300, workers=8,imgsz=640, batch=32, device=0) +``` + +训练好的模型会保存在./runs/detect/train/weights目录下。 + +## 2. 模型转换 + +### 2.1 导出onnx模型 + +由于是在rv1106上运行，对于模型转换请参考如下文档： + +[RKOPT_README.zh-CN.md](https://github.com/airockchip/ultralytics_yolov8/blob/main/RKOPT_README.zh-CN.md) + +- 调整 ./ultralytics/cfg/default.yaml 中 model 文件路径，默认为 yolov8n.pt，若自己训练模型，请调接至对应的路径。支持检测、分割、姿态、旋转框检测模型。 + +在满足 ./requirements.txt 的环境要求后，执行以下语句导出模型 + +```shell +export PYTHONPATH=./ +python ./ultralytics/engine/exporter.py +``` + +执行完毕后，会生成 ONNX 模型. 假如原始模型为 yolov8n.pt，则生成 yolov8n.onnx 模型。 转换后的onnx模型会保存在runs/detect/train/weights目录下。 + +### 2.2 准备校准数据集 + +在量化时，提供 20~100 张有代表性的图片给 RKNN 即可。请在你的终端中运行以下命令，从你的 Dataset_YOLO/images/train 中提取部分图片路径生成校准文件： + +```shell +cd path/to/ultralytics_yolov8 +# 随机选取50张训练集的图片路径，写入 dataset_subset.txt +ls /path/to/ultralytics_yolov8/Dataset_YOLO/images/train/*.png | head -n 50 > dataset_subset.txt +``` + +- 注意：如果你的图片后缀是 .jpg ，请将上面的 *.png 替换为*.jpg + +### 2.3 转RKNN模型 + +#### 2.3.1 克隆rknn_model_zoo + +```shell +git clone https://github.com/airockchip/rknn_model_zoo.git +cd /path/to/rknn_model_zoo/examples/yolov8/python +``` + +#### 2.3.2 修改convert.py代码 + +主要是将 DATASET_PATH 指向刚才我们生成的 dataset_subset.txt ： + +```python +DATASET_PATH = 'path/to/dataset_subset.txt' +``` + +#### 2.3.3 运行convert.py + +在运行转换脚本前请确保你已经配置好了所需要的环境。并且将之前转换好的onnx模型放到以下目录： + +- /path/to/rknn_model_zoo/examples/yolov8/model/ + +执行以下命令开始转换 + +```shell +python convert.py /path/to/xxxx.onnx rv1106 i8 ../model/yolov8.rknn +``` + +执行完毕后，如果没有报错，你就会在 rknn_model_zoo/examples/yolov8/model/ 目录下得到 yolov8.rknn 文件，这个文件就可以直接放到你的 RV1106 板子上进行部署板上运行了！ diff --git a/README.md b/README.md index 06ae7ed6c6f149b9e2a523f3f36de79b4e2e68db..2a8168663c36086cd0e1ac66b11287919f3ffa4d 100644 --- a/README.md +++ b/README.md @@ -135,6 +135,7 @@ OpenCV 是一个开源的计算机视觉库，它提供了一组功能强大的 目标检测（Object Detection）是深度学习中计算机视觉领域的重要任务之一，旨在识别图像或视频中所有感兴趣的物体，并准确地定位这些物体的边界框（Bounding Box）。与目标分类不同，目标检测不仅需要预测物体的类别，还需要标注它们在图像中的位置。一般来说，目标检测任务的标注过程比较复杂，适合既需要对目标进行分类，有需要对目标进行定位的场景。 * [凌智视觉模块通用检测模型部署指南](./example/vision/detetcion) +* [YOLOv5 模型部署指南](./example/vision/yolov5) ### 👍 条码检测识别案例 @@ -185,6 +186,12 @@ OCR（Optical Character Recognition，光学字符识别）是一种将图像中 * [YOLOv5目标检测](./Cpp_example/D10_yolov5//README.md) +### 👍 YOLOv8目标检测 + +目标检测（Object Detection）是深度学习中计算机视觉领域的重要任务之一，旨在识别图像或视频中所有感兴趣的物体，并准确地定位这些物体的边界框（Bounding Box）。与目标分类不同，目标检测不仅需要预测物体的类别，还需要标注它们在图像中的位置。一般来说，目标检测任务的标注过程比较复杂，适合既需要对目标进行分类，有需要对目标进行定位的场景。该案例使用YOLOv8进行目标检测。 + +* [YOLOv8目标检测](./Cpp_example/D15_yolov8//README.md) + ## 🏀 C++ 开发案例 C++ 开发案例以A、B、C、D进行不同类别进行分类，方便初学者进行使用和二次开发。 @@ -229,6 +236,7 @@ C++ 开发案例以A、B、C、D进行不同类别进行分类，方便初学者 | D12 | 神经网络类 | PPOCRv4-Det | [文字检测](./Cpp_example/D12_ppocrv4_det/README.md) | | D13 | 神经网络类 | target_tracking | [多目标跟踪](./Cpp_example/D13_target_tracking/README.md)| | D14 | 神经网络类 | fatugue_detection | [疲劳检测](./Cpp_example/D14_fatigue_detection/README.md)| +| D15 | 神经网络类 | YOLOv8 | [YOLOv8目标检测](./Cpp_example/D15_yolov8/README.md)| | E01 | 使用示例类 | Test_find_Laser | [激光跟踪](./Cpp_example/E01_find_Laser/README.md)| | E02 | 使用示例类 | Test_find_number | [数字识别](./Cpp_example/E02_find_number/README.md)| diff --git "a/docs/Linux/linux\347\256\200\346\230\216\346\225\231\347\250\213.md" "b/docs/Linux/linux\347\256\200\346\230\216\346\225\231\347\250\213.md" index 9898840b14bba37e0273e0a3c5fc6ab34f0c45f8..9aa2eba364153d9b988df7c4ad6db6e3ca050f2d 100644 --- "a/docs/Linux/linux\347\256\200\346\230\216\346\225\231\347\250\213.md" +++ "b/docs/Linux/linux\347\256\200\346\230\216\346\225\231\347\250\213.md" @@ -1,4 +1,4 @@ -

凌智视觉模块**linux**简明教程

+

凌智视觉模块 Linux 简明教程

发布版本：V0.0.0 @@ -26,7 +26,7 @@ | **日期** | **版本** | **作者** | **修改说明** | |:-----------|--------|--------|-----------| -| 2026/02/04 | 0.0.0 |钟海滨| 初始化linux教程 | +| 2026/02/07 | 0.0.0 |钟海滨| 初始化linux教程 | **注**： 此篇教程参考[菜鸟教程](https://www.runoob.com/linux/linux-tutorial.html) diff --git "a/docs/opencv-python\346\216\245\345\217\243\346\226\207\346\241\243.md" "b/docs/opencv-python\346\216\245\345\217\243\346\226\207\346\241\243.md" index 6c3b2027c2cb18bb910657c16afc8e207057fc53..25479e202cebdc8bec4db39b41356d886ae9e236 100644 --- "a/docs/opencv-python\346\216\245\345\217\243\346\226\207\346\241\243.md" +++ "b/docs/opencv-python\346\216\245\345\217\243\346\226\207\346\241\243.md" @@ -2,7 +2,7 @@ 发布版本：V0.0.6 -日期：2026/02/04 +日期：2026/02/14 文件密级：□绝密 □秘密 □内部资料 ■公开 @@ -28,6 +28,7 @@ |:-----------|--------|--------|-----------| | 2026/02/04 | 0.0.6 |钟海滨| 完善接口文档内容 | | 2026/02/06 | 0.0.6 |钟海滨| 增加大量函数封装 | +| 2026/02/14 | 0.0.6 |钟海滨| 增加部分接口 | --- ## 目录 @@ -323,6 +324,55 @@ cv2.putText(img, text, org, fontFace, fontScale, color, thickness=1, cv2.putText(img, "Hello World", (10, 50), cv2.FONT_HERSHEY_SIMPLEX, 1, (255, 255, 255)) ``` +#### polylines - 绘制多段折线 +```python +# 语法 +cv2.polylines(img, pts, isClosed, color, thickness=1, lineType=cv2.LINE_8, shift=0) + +# 示例（pts 为点列表的列表） +pts = [[(10, 10), (100, 10), (100, 100)]] +cv2.polylines(img, pts, True, (0, 255, 0), 2) +``` + +#### rectangle_rect - 使用 Rect 绘制矩形 +```python +# 语法 +cv2.rectangle_rect(img, rect, color, thickness=1, lineType=cv2.LINE_8, shift=0) + +# 示例 +rect = cv2.Rect(50, 50, 100, 80) +cv2.rectangle_rect(img, rect, (255, 0, 0), 2) +``` + +#### ellipse - 绘制椭圆 +```python +# 语法 +cv2.ellipse(img, center, axes, angle, startAngle, endAngle, color, thickness=1, lineType=cv2.LINE_8, shift=0) + +# 示例 +cv2.ellipse(img, (100, 100), (60, 30), 0, 0, 360, (255, 255, 0), 2) +``` + +#### fillPoly - 填充多边形 +```python +# 语法 +cv2.fillPoly(img, pts, color, lineType=cv2.LINE_8, shift=0) + +# 示例 +polys = [[(10, 10), (80, 10), (80, 60), (10, 60)]] +cv2.fillPoly(img, polys, (0, 0, 255)) +``` + +#### fillConvexPoly - 填充凸多边形 +```python +# 语法 +cv2.fillConvexPoly(img, pts, color, lineType=cv2.LINE_8, shift=0) + +# 示例 +pts = [(30, 30), (120, 50), (80, 120)] +cv2.fillConvexPoly(img, pts, (0, 255, 255)) +``` + ### 4.2.4 轮廓处理 #### findContours - 查找轮廓 @@ -362,6 +412,69 @@ rect = cv2.boundingRect(points) bbox = cv2.boundingRect(contour) ``` +#### arcLength - 计算轮廓周长 +```python +# 语法 +length = cv2.arcLength(curve, closed) + +# 示例 +length = cv2.arcLength(contour, True) +``` + +#### approxPolyDP - 轮廓近似 +```python +# 语法 +approx = cv2.approxPolyDP(curve, epsilon, closed) + +# 示例 +approx = cv2.approxPolyDP(contour, 2.0, True) +``` + +#### moments - 计算几何矩 +```python +# 语法 +m = cv2.moments(contour, binaryImage=False) + +# 示例 +m = cv2.moments(contour) +``` + +#### HuMoments - Hu矩 +```python +# 语法 +hu = cv2.HuMoments(contour, binaryImage=False) + +# 示例 +hu = cv2.HuMoments(contour) +``` + +#### contourCentroid - 轮廓质心 +```python +# 语法 +center = cv2.contourCentroid(contour) + +# 示例 +center = cv2.contourCentroid(contour) +``` + +#### convexHull - 轮廓凸包 +```python +# 语法 +hull = cv2.convexHull(contour, clockwise=False) + +# 示例 +hull = cv2.convexHull(contour) +``` + +#### convexityDefects - 凸包缺陷 +```python +# 语法 +defects = cv2.convexityDefects(contour) + +# 示例 +defects = cv2.convexityDefects(contour) +``` + ### 4.2.5 霍夫变换 #### HoughLines - 标准霍夫线变换 @@ -488,12 +601,48 @@ equalized = cv2.equalizeHist(gray) #### applyCLAHE - 自适应直方图均衡化 ```python # 语法 -dst = cv2.applyCLAHE(inputImage, clipLimit=40.0, tileSize=(8, 8)) +dst = cv2.applyCLAHE(inputImage, clipLimit, tileGridSize) # 示例 clahe_result = cv2.applyCLAHE(gray, 2.0, (8, 8)) ``` +#### copyMakeBorder - 扩展边界 +```python +# 语法 +dst = cv2.copyMakeBorder(img, top, bottom, left, right, borderType, value=(0, 0, 0, 0)) + +# 示例 +bordered = cv2.copyMakeBorder(img, 10, 10, 20, 20, cv2.BORDER_CONSTANT, (0, 0, 0, 0)) +``` + +#### matchTemplate - 模板匹配 +```python +# 语法 +result = cv2.matchTemplate(image, templ, method) + +# 示例 +result = cv2.matchTemplate(img, templ, cv2.TM_CCOEFF_NORMED) +``` + +#### cornerHarris - Harris 角点检测 +```python +# 语法 +dst = cv2.cornerHarris(src, blockSize, ksize, k, borderType=cv2.BORDER_DEFAULT) + +# 示例 +harris = cv2.cornerHarris(gray, 2, 3, 0.04) +``` + +#### distanceTransform - 距离变换 +```python +# 语法 +dist = cv2.distanceTransform(src, distanceType, maskSize) + +# 示例 +dist = cv2.distanceTransform(binary, cv2.DIST_L2, 3) +``` + ### 4.2.11 特征点与形状分析 #### goodFeaturesToTrack - 角点检测 @@ -524,8 +673,9 @@ merged = cv2.merge(bgr_planes) small = cv2.pyrDown(img) large = cv2.pyrUp(small) -# 直方图计算 -hist = cv2.calcHist([gray], [0], None, [256], [0, 256]) +# 直方图计算（histSize 与 ranges 必填；mask 为空时使用 Mat） +mask = cv2.Mat() +hist = cv2.calcHist([gray], [0], mask, [256], [0, 256]) # 查找表变换 result = cv2.LUT(gray, lut) @@ -557,8 +707,12 @@ affine = cv2.warpAffine(img, M_affine, (img.cols, img.rows)) M_persp = cv2.getPerspectiveTransform(src_points4, dst_points4) persp = cv2.warpPerspective(img, M_persp, (img.cols, img.rows)) -# 翻转和转置 +# 重映射 +remapped = cv2.remap(img, map1, map2, cv2.INTER_LINEAR) + +# 翻转、旋转和转置 flipped = cv2.flip(img, 1) +rotated90 = cv2.rotate(img, cv2.ROTATE_90_CLOCKWISE) transposed = cv2.transpose(img) ``` @@ -638,11 +792,12 @@ cv2.imwrite('output.jpg', processed_img) #### imencode - 图像编码 ```python -# 语法（当前实现返回编码成功与字节数据） -success, buffer = cv2.imencode(extension, img, params=None) +# 语法（param 可为 None / int / list / tuple） +success, buffer = cv2.imencode(extension, img, param=None) # 示例 ret, jpg_bytes = cv2.imencode('.jpg', img) +ret, jpg_bytes = cv2.imencode('.jpg', img, 90) # jpg_bytes 类型为 bytes，可直接写入文件或通过网络发送 ``` @@ -704,6 +859,12 @@ sub_img = cv2.subtract(img1, img2) mul_img = cv2.multiply(img1, img2) div_img = cv2.divide(img1, img2) +# 标量运算 +add_s = cv2.addScalar(img, (10, 10, 10, 0)) +sub_s = cv2.subtractScalar(img, (10, 10, 10, 0)) +mul_s = cv2.multiplyScalar(img, (1.2, 1.2, 1.2, 1.0)) +div_s = cv2.divideScalar(img, (2, 2, 2, 1)) + # 线性组合与差分 scaled = cv2.scaleAdd(img1, 0.5, img2) abs_diff = cv2.absdiff(img1, img2) diff --git "a/docs/\346\216\245\345\217\243\346\226\207\346\241\243/lockzhiner_vision_module/cv2/core_functionality.py" "b/docs/\346\216\245\345\217\243\346\226\207\346\241\243/lockzhiner_vision_module/cv2/core_functionality.py" index c5a28823996c978d170523808f4d826d787be450..4455bcd5d5ef01c96e82ccb8c4e86335707a9a42 100644 --- "a/docs/\346\216\245\345\217\243\346\226\207\346\241\243/lockzhiner_vision_module/cv2/core_functionality.py" +++ "b/docs/\346\216\245\345\217\243\346\226\207\346\241\243/lockzhiner_vision_module/cv2/core_functionality.py" @@ -41,20 +41,40 @@ def add(src1, src2, mask=None): return cv2.add(src1, src2, mask) +def addScalar(src, scalar, mask=None): + if mask is None: + mask = cv2.Mat() + return cv2.addScalar(src, scalar, mask) + + def subtract(src1, src2, mask=None): if mask is None: mask = cv2.Mat() return cv2.subtract(src1, src2, mask) +def subtractScalar(src, scalar, mask=None): + if mask is None: + mask = cv2.Mat() + return cv2.subtractScalar(src, scalar, mask) + + def multiply(src1, src2, scale=1.0): return cv2.multiply(src1, src2, scale) +def multiplyScalar(src, scalar, scale=1.0): + return cv2.multiplyScalar(src, scalar, scale) + + def divide(src1, src2, scale=1.0): return cv2.divide(src1, src2, scale) +def divideScalar(src, scalar, scale=1.0): + return cv2.divideScalar(src, scalar, scale) + + def scaleAdd(src1, alpha, src2): return cv2.scaleAdd(src1, alpha, src2) @@ -92,8 +112,15 @@ def compareHist(H1, H2, method): def imencode(ext: str, img_data, param=None): if param is None: - param = [] - ret, buf = cv2.imencode(ext, img_data, param) + params = [] + elif isinstance(param, int): + quality_flag = getattr(cv2, "IMWRITE_JPEG_QUALITY", 1) + params = [quality_flag, param] + elif isinstance(param, (list, tuple)): + params = list(param) + else: + raise TypeError("param must be int, list, tuple, or None") + ret, buf = cv2.imencode(ext, img_data, params) return ret, buf diff --git "a/docs/\346\216\245\345\217\243\346\226\207\346\241\243/lockzhiner_vision_module/cv2/image_processing.py" "b/docs/\346\216\245\345\217\243\346\226\207\346\241\243/lockzhiner_vision_module/cv2/image_processing.py" index e44b6bda736c6646b88fe7bfd8dee31643710c67..20a8386344d56a5e567837bb0c51b9aa020bb85a 100644 --- "a/docs/\346\216\245\345\217\243\346\226\207\346\241\243/lockzhiner_vision_module/cv2/image_processing.py" +++ "b/docs/\346\216\245\345\217\243\346\226\207\346\241\243/lockzhiner_vision_module/cv2/image_processing.py" @@ -177,6 +177,31 @@ def contourArea(contour, oriented=False): return area +def moments(contour, binaryImage=False): + temp_contour = [convert2point(p) for p in contour] + return cv2.moments(temp_contour, binaryImage) + + +def HuMoments(contour, binaryImage=False): + temp_contour = [convert2point(p) for p in contour] + return cv2.HuMoments(temp_contour, binaryImage) + + +def contourCentroid(contour): + temp_contour = [convert2point(p) for p in contour] + return cv2.contourCentroid(temp_contour) + + +def convexHull(contour, clockwise=False): + temp_contour = [convert2point(p) for p in contour] + return cv2.convexHull(temp_contour, clockwise) + + +def convexityDefects(contour): + temp_contour = [convert2point(p) for p in contour] + return cv2.convexityDefects(temp_contour) + + def goodFeaturesToTrack(image, maxCorners, qualityLevel, minDistance, mask=None, blockSize=3, useHarrisDetector=False, k=0.04): if mask is None: mask_mat = cv2.Mat() diff --git "a/docs/\346\216\245\345\217\243\346\226\207\346\241\243/opencv\346\226\260\345\242\236\346\216\245\345\217\243\346\265\213\350\257\225\344\276\213\347\250\213/2026-0214\346\226\260\345\242\236.py" "b/docs/\346\216\245\345\217\243\346\226\207\346\241\243/opencv\346\226\260\345\242\236\346\216\245\345\217\243\346\265\213\350\257\225\344\276\213\347\250\213/2026-0214\346\226\260\345\242\236.py" new file mode 100644 index 0000000000000000000000000000000000000000..7ea5805884edc5ddc8d37b92a60c4ab6080b5beb --- /dev/null +++ "b/docs/\346\216\245\345\217\243\346\226\207\346\241\243/opencv\346\226\260\345\242\236\346\216\245\345\217\243\346\265\213\350\257\225\344\276\213\347\250\213/2026-0214\346\226\260\345\242\236.py" @@ -0,0 +1,92 @@ +import os +import sys + +import lockzhiner_vision_module.cv2 as cv2 + + + +def require_image(path): + if not os.path.isfile(path): + raise RuntimeError("Image file not found: " + path) + img = cv2.imread(path, cv2.IMREAD_COLOR) + if img.empty(): + raise RuntimeError("Failed to read image: " + path) + return img + + +def main(): + if len(sys.argv) < 2: + print("Usage: python test_new_apis_no_numpy.py ") + return 1 + + img = require_image(sys.argv[1]) + print("img:", img.cols, img.rows, img.channels(), img.type()) + + ok, encoded = cv2.imencode(".jpg", img, 80) + print("imencode ok:", ok, "bytes:", len(encoded)) + + print("at(0,0):", img.at(0, 0)) + print("ptr(0) len:", len(img.ptr(0))) + print("ptr(0,0) len:", len(img.ptr(0, 0))) + + reshaped = img.reshape(img.channels(), img.rows) + print("reshape:", reshaped.cols, reshaped.rows) + + converted = img.convertTo(img.type(), 1.0, 0.0) + print("convertTo:", converted.cols, converted.rows) + + scalar = cv2.Scalar(10, 20, 30) + add_s = cv2.addScalar(img, scalar, cv2.Mat()) + sub_s = cv2.subtractScalar(img, scalar, cv2.Mat()) + mul_s = cv2.multiplyScalar(img, scalar, 1.0) + div_s = cv2.divideScalar(img, scalar, 1.0) + print("addScalar/subtractScalar/multiplyScalar/divideScalar:", + add_s.cols, sub_s.cols, mul_s.cols, div_s.cols) + + gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY, 0) + _, binary = cv2.threshold(gray, 0, 255, cv2.THRESH_BINARY_INV + cv2.THRESH_OTSU) + contours, _ = cv2.findContours(binary, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) + print("contours:", len(contours)) + + result = img.copy() + + if contours: + c = contours[0] + area = cv2.contourArea(c, False) + perimeter = cv2.arcLength(c, True) + approx = cv2.approxPolyDP(c, 2.0, True) + + m = cv2.moments(c, False) + hu = cv2.HuMoments(c, False) + cx, cy = cv2.contourCentroid(c) + hull = cv2.convexHull(c, False) + defects = cv2.convexityDefects(c) + + print("area:", area) + print("arcLength:", perimeter) + print("approx size:", len(approx)) + print("moments m00:", m.get("m00")) + print("HuMoments:", hu) + print("centroid:", cx, cy) + print("convexHull size:", len(hull)) + print("convexityDefects:", len(defects)) + + cv2.drawContours(result, [c], -1, (0, 255, 0), 2, cv2.LINE_8, 0) + cv2.drawContours(result, [hull], -1, (0, 0, 255), 2, cv2.LINE_8, 0) + cv2.circle(result, (int(cx), int(cy)), 4, (255, 0, 0), 2, cv2.LINE_8, 0) + + rect = cv2.boundingRect(c) + cv2.rectangle_rect(result, rect, (255, 255, 0), 2, cv2.LINE_8, 0) + + text = "A:{:.2f} P:{:.2f}".format(area, perimeter) + cv2.putText(result, text, (10, 30), cv2.FONT_HERSHEY_SIMPLEX, 0.7, (255, 255, 255), 2, cv2.LINE_8, False) + + out_path = os.path.join(os.path.dirname(os.path.abspath(sys.argv[1])), "result_new_apis.png") + ok = cv2.imwrite(out_path, result) + print("save:", out_path, "ok:", ok) + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) \ No newline at end of file diff --git "a/docs/\346\216\245\345\217\243\346\226\207\346\241\243/opencv\346\226\260\345\242\236\346\216\245\345\217\243\346\265\213\350\257\225\344\276\213\347\250\213/result_new_apis.png" "b/docs/\346\216\245\345\217\243\346\226\207\346\241\243/opencv\346\226\260\345\242\236\346\216\245\345\217\243\346\265\213\350\257\225\344\276\213\347\250\213/result_new_apis.png" new file mode 100644 index 0000000000000000000000000000000000000000..f4d1cfe7d2101d4e86eaad5a501ca21d389ba05a Binary files /dev/null and "b/docs/\346\216\245\345\217\243\346\226\207\346\241\243/opencv\346\226\260\345\242\236\346\216\245\345\217\243\346\265\213\350\257\225\344\276\213\347\250\213/result_new_apis.png" differ diff --git a/example/vision/yolov5/convert.py b/example/vision/yolov5/convert.py new file mode 100644 index 0000000000000000000000000000000000000000..cd8ce7b5150437320b3f637632e7e64b2eedea40 --- /dev/null +++ b/example/vision/yolov5/convert.py @@ -0,0 +1,75 @@ +import sys + +from rknn.api import RKNN + +DATASET_PATH = './dataset.txt' +DEFAULT_RKNN_PATH = './yolov5.rknn' +DEFAULT_QUANT = True + +def parse_arg(): + if len(sys.argv) < 3: + print("Usage: python3 {} onnx_model_path [platform] [dtype(optional)] [output_rknn_path(optional)]".format(sys.argv[0])) + print(" platform choose from [rk3562, rk3566, rk3568, rk3576, rk3588, rv1103, rv1106, rv1126b, rv1109, rv1126, rk1808]") + print(" dtype choose from [i8, fp] for [rk3562, rk3566, rk3568, rk3576, rk3588, rv1103, rv1106, rv1126b]") + print(" dtype choose from [u8, fp] for [rv1109, rv1126, rk1808]") + exit(1) + + model_path = sys.argv[1] + platform = sys.argv[2] + + do_quant = DEFAULT_QUANT + if len(sys.argv) > 3: + model_type = sys.argv[3] + if model_type not in ['i8', 'u8', 'fp']: + print("ERROR: Invalid model type: {}".format(model_type)) + exit(1) + elif model_type in ['i8', 'u8']: + do_quant = True + else: + do_quant = False + + if len(sys.argv) > 4: + output_path = sys.argv[4] + else: + output_path = DEFAULT_RKNN_PATH + + return model_path, platform, do_quant, output_path + +if __name__ == '__main__': + model_path, platform, do_quant, output_path = parse_arg() + + # Create RKNN object + rknn = RKNN(verbose=False) + + # Pre-process config + print('--> Config model') + rknn.config(mean_values=[[0, 0, 0]], std_values=[ + [255, 255, 255]], target_platform=platform) + print('done') + + # Load model + print('--> Loading model') + ret = rknn.load_onnx(model=model_path) + if ret != 0: + print('Load model failed!') + exit(ret) + print('done') + + # Build model + print('--> Building model') + ret = rknn.build(do_quantization=do_quant, dataset=DATASET_PATH) + if ret != 0: + print('Build model failed!') + exit(ret) + print('done') + + # Export rknn model + print('--> Export rknn model') + ret = rknn.export_rknn(output_path) + if ret != 0: + print('Export rknn model failed!') + exit(ret) + print('done') + + # Release + rknn.release() \ No newline at end of file diff --git a/example/vision/yolov5/data.py b/example/vision/yolov5/data.py new file mode 100644 index 0000000000000000000000000000000000000000..c93cb90cfa310a9b7cf708bf8005ffc270e0e2a9 --- /dev/null +++ b/example/vision/yolov5/data.py @@ -0,0 +1,9 @@ +import os + +# 获取目录下所有文件名 +file_list = os.listdir("./images/train2017") +path = "./images/train2017/" +# 写入文件，每行一个 +with open('dataset.txt', 'w', encoding='utf-8') as f: + for filename in file_list: + f.write(path+filename + '\n') \ No newline at end of file diff --git a/example/vision/yolov5/test_yolov5.py b/example/vision/yolov5/test_yolov5.py new file mode 100755 index 0000000000000000000000000000000000000000..d4d99a366673618d395b68571c7041d2345006bb --- /dev/null +++ b/example/vision/yolov5/test_yolov5.py @@ -0,0 +1,157 @@ +import lockzhiner_vision_module.cv2 as cv2 +import sys +import os +from lockzhiner_vision_module.vision.detection import YoloV5 +from lockzhiner_vision_module.vision.utils import visualize +from lockzhiner_vision_module.edit import Edit + +def test_yolov5_image(model_path, image_path, label_path=None, output_path='yolov5_result.jpg'): + """ + 测试 YOLOv5 模型在单张图像上的目标检测功能 + + Args: + model_path (str): YOLOv5 模型文件路径 + image_path (str): 测试图像路径 + label_path (str, optional): 标签文件路径 + output_path (str, optional): 输出图像路径，默认为 'yolov5_result.jpg' + """ + print(f"正在初始化 YOLOv5 模型...") + print(f"模型路径: {model_path}") + + model = YoloV5() + + if not model.initialize(model_path): + print("模型初始化失败！") + return False + + print("模型初始化成功！") + + if model.load_labels(label_path): + labels = model.get_labels() + print(f"Loaded labels: {labels}") + else: + labels = [] + print("No labels loaded, will use label_id") + model.set_threshold(conf_threshold=0.25, nms_threshold=0.45) + print("检测阈值已设置: conf_threshold=0.25, nms_threshold=0.45") + + print(f"\n正在读取图像: {image_path}") + image = cv2.imread(image_path) + + if image is None: + print(f"无法读取图像: {image_path}") + return False + + print(f"图像尺寸: {image.cols}x{image.rows}") + print(f"图像尺寸: {image.shape[1]}x{image.shape[0]}") + print("\n正在进行目标检测...") + results = model.predict(image) + + print(f"检测完成！共检测到 {len(results)} 个目标") + + for i, result in enumerate(results): + print(f"\n目标 {i + 1}:") + print(f" 矩形框: x={result.box.x}, y={result.box.y}, " + f"width={result.box.width}, height={result.box.height}") + print(f" 标签 ID: {result.label_id}") + print(f" 置信度: {result.score:.4f}") + + print(f"\n正在可视化检测结果...") + output_image = visualize(image, results,labels) + + print(f"正在保存结果图像到: {output_path}") + cv2.imwrite(output_path, output_image) + + print(f"\n测试完成！结果已保存到 {output_path}") + return True + + +def test_yolov5_camera(model_path, label_path=None): + """ + 测试 YOLOv5 模型在摄像头实时视频上的目标检测功能 + + Args: + model_path (str): YOLOv5 模型文件路径 + label_path (str, optional): 标签文件路径 + """ + print(f"正在初始化 YOLOv5 模型...") + print(f"模型路径: {model_path}") + width = 640 + height = 480 + edit = Edit() + edit.start_and_accept_connection() + + model = YoloV5() + + if not model.initialize(model_path): + print("模型初始化失败！") + return False + + print("模型初始化成功！") + + if label_path: + print(f"正在加载标签文件: {label_path}") + if model.load_labels(label_path): + labels = model.get_labels() + print("标签文件加载成功！") + else: + labels = [] + print("标签文件加载失败，将使用默认标签 ID") + + + model.set_threshold(conf_threshold=0.25, nms_threshold=0.45) + print("检测阈值已设置: conf_threshold=0.25, nms_threshold=0.45") + + print("\n正在打开摄像头...") + cap = cv2.VideoCapture() + + if not cap.open(0): + print("无法打开摄像头！") + return False + + print("摄像头已打开，按 'q' 键退出") + + while True: + ret, frame = cap.read() + + if not ret: + print("无法读取摄像头画面") + break + + results = model.predict(frame) + + + output_frame = visualize(frame, results,labels) + edit.print(output_frame) + + + cap.release() + cv2.destroyAllWindows() + print("\n摄像头测试结束") + return True + + +if __name__ == "__main__": + if len(sys.argv) < 3: + print("使用方法:") + print(" 测试图像: python test_yolov5.py [label_path] [output_path]") + print(" 测试摄像头: python test_yolov5.py camera [label_path]") + print("\n参数说明:") + print(" model_path - YOLOv5 模型文件路径") + print(" image_path - 测试图像路径") + print(" camera - 使用摄像头进行实时检测") + print(" label_path - (可选) 标签文件路径") + print(" output_path - (可选) 输出图像路径，默认为 'yolov5_result.jpg'") + sys.exit(1) + + model_path = sys.argv[1] + mode = sys.argv[2] + + if mode == "camera": + label_path = sys.argv[3] if len(sys.argv) > 3 else None + test_yolov5_camera(model_path, label_path) + else: + image_path = mode + label_path = sys.argv[3] if len(sys.argv) > 3 else None + output_path = sys.argv[4] if len(sys.argv) > 4 else 'yolov5_result.jpg' + test_yolov5_image(model_path, image_path, label_path, output_path) diff --git "a/example/vision/yolov5/yolov5 \350\256\255\347\273\203\344\270\216\351\203\250\347\275\262\346\226\207\346\241\243.md" "b/example/vision/yolov5/yolov5 \350\256\255\347\273\203\344\270\216\351\203\250\347\275\262\346\226\207\346\241\243.md" new file mode 100644 index 0000000000000000000000000000000000000000..38b01dde961958c8bdc493c4d6a59c6d8369cc21 --- /dev/null +++ "b/example/vision/yolov5/yolov5 \350\256\255\347\273\203\344\270\216\351\203\250\347\275\262\346\226\207\346\241\243.md" @@ -0,0 +1,418 @@ +

凌智视觉模块 YOLOv5 目标检测模型部署指南

+ +发布版本：V0.0.0 + +日期：2026-03-04 + +文件密级：□绝密 □秘密 □内部资料 ■公开 + +--- + +**免责声明** + +本文档按**现状**提供，福州凌睿智捷电子有限公司（以下简称**本公司**）不对本文档中的任何陈述、信息和内容的准确性、可靠性、完整性、适销性、适用性及非侵权性提供任何明示或暗示的声明或保证。本文档仅作为使用指导的参考。 + +由于产品版本升级或其他原因，本文档可能在未经任何通知的情况下不定期更新或修改。 + +**读者对象** + +本教程适用于以下工程师： + +- 技术支持工程师 +- 软件开发工程师 + +**修订记录** + +| **日期** | **版本** | **作者** | **修改说明** | +|:-----------| -------- |--------| ------------ | +| 2026/03/04 | 0.0.0 | 钟海滨 | 初始版本 | + +## 1 简介 + +在深度学习中，目标检测是一项重要的计算机视觉任务，旨在在图像或视频中精确定位和识别多个目标的位置和类别。这项技术广泛应用于自动驾驶、安防监控、工业检测、智能零售等多种场景。 + +为了实现高效且准确的目标检测，我们基于 Lockzhiner Vision Module 的 YOLOv5 模型，训练了一个高性能的目标检测模型。YOLOv5（You Only Look Once version 5）是一种单阶段目标检测算法，以其出色的速度和精度平衡而闻名。 + +## 2 运行前的准备 + +* 请确保你已经下载了 [凌智视觉模块图片传输助手](https://gitee.com/LockzhinerAI/LockzhinerVisionModule/releases/download/v0.0.0/LockzhinerVisionModuleImageFetcher.exe) + +## 3 模型训练与转换 + +### 3.1 拉取 YOLOv5 仓库 + +```bash +git clone https://github.com/airockchip/yolov5.git +cd yolov5 +pip install -r requirements.txt +``` + +### 3.2 训练模型 + +```bash +python train.py --data coco.yaml --img 640 --batch 16 --epochs 300 --weights yolov5s.pt +``` + +### 3.3 导出 ONNX 模型 + +```bash +python export.py --rknpu --weights /media/lzdz/data/yolo-rk/runs/train/exp6/weights/best.pt +``` + +### 3.4 转换为 RKNN 模型 + +**构建量化数据集** + +```bash +python data.py +``` + +**安装转换工具** + +```bash +pip install rknn-toolkit2 +``` + +**转换模型** + +```bash +python convert.py /media/lzdz/data/yolo-rk/runs/train/exp8/weights/best.onnx rv1106 +``` + +## 4 在凌智视觉模块上部署模型 + +训练完模型后，请参考以下教程在凌智视觉模块上部署检测模型例程： + +* [凌智视觉模块 YOLOv5 目标检测模型 Python 部署指南](#5-python-api-文档) + +## 5 Python API 文档 + +### 5.1 Rect 类 + +Rect 类来自 cv2 模块，用于表示矩形框信息。 + +```python +from lockzhiner_vision_module.cv2 import Rect + +# Rect 类属性 +rect.x # 矩形左上角坐标点的 x 坐标 +rect.y # 矩形左上角坐标点的 y 坐标 +rect.width # 矩形的宽 +rect.height # 矩形的高 +``` + +### 5.2 DetectionResult 类 + +```python +class DetectionResult: + """ + 检测结果类，用于封装和处理目标检测结果数据。 + + 该类主要提供了一个包装层，用于访问和管理由视觉模块产生的检测结果。 + """ + + @property + def box(self): + """ + 获取目标检测模型检测结果的矩形框信息 + + Returns: + Rect: 矩形框信息 + """ + return self.detection_result.box + + @property + def score(self): + """ + 获取目标检测模型检测结果的得分信息 + + Returns: + float: 得分信息 + """ + return self.detection_result.score + + @property + def label_id(self): + """ + 获取目标检测模型检测结果的分类标签信息 + + Returns: + int: 分类标签信息 + """ + return self.detection_result.label_id +``` + +### 5.3 YoloV5 类 + +```python +from lockzhiner_vision_module.vision.detection import YoloV5 + +class YoloV5: + """ + YoloV5 类 - 用于目标检测的 YOLOv5 模型封装。 + + 该类封装了 YOLOv5 框架下的目标检测模型，提供了初始化和预测的方法。 + """ + + def __init__(self): + """ + 构造函数 - 初始化 YoloV5 对象。 + """ + self.model = vision.YoloV5() + + def initialize(self, model_path): + """ + 初始化模型 - 加载预训练的 YOLOv5 模型。 + + Args: + model_path (str): 模型文件的路径。 + + Returns: + bool: 初始化是否成功。 + """ + return self.model.initialize(model_path) + + def load_labels(self, label_path): + """ + 加载标签文件 - 加载类别标签。 + + Args: + label_path (str): 标签文件的路径。 + + Returns: + bool: 加载是否成功。 + """ + return self.model.load_labels(label_path) + + def set_threshold(self, conf_threshold=0.25, nms_threshold=0.45): + """ + 设置目标检测阈值 + + Args: + conf_threshold (float): 目标检测置信度阈值，默认为 0.25 + nms_threshold (float): 目标检测 NMS 阈值，默认为 0.45 + + """ + self.model.set_threshold(conf_threshold, nms_threshold) + + def predict(self, input_mat): + """ + 进行预测 - 使用加载的模型对输入数据进行目标检测预测。 + + Args: + input_mat (cv2.Mat): 输入的图像数据，通常是一个 cv2.Mat 变量。 + + Returns: + list(DetectionResult): 预测结果对象列表，每一个预测结果包含了矩形框、标签信息和置信度等信息。 + """ + return self.model.predict(input_mat) +``` + +## 6 项目介绍 + +为了方便大家入手，我们做了一个简易的目标检测例程。该程序可以使用摄像头进行端到端推理，也可以对单张图像进行检测。 +**注**： YOLOV5 类使用时，除了加载模型外文件外，还需要提供标签文件，该文件存放了模型所对应的标签。 +```python +import lockzhiner_vision_module.cv2 as cv2 +import sys +import os +from lockzhiner_vision_module.vision.detection import YoloV5 +from lockzhiner_vision_module.vision.utils import visualize +from lockzhiner_vision_module.edit import Edit + +def test_yolov5_image(model_path, image_path, label_path=None, output_path='yolov5_result.jpg'): + """ + 测试 YOLOv5 模型在单张图像上的目标检测功能 + + Args: + model_path (str): YOLOv5 模型文件路径 + image_path (str): 测试图像路径 + label_path (str, optional): 标签文件路径 + output_path (str, optional): 输出图像路径，默认为 'yolov5_result.jpg' + """ + print(f"正在初始化 YOLOv5 模型...") + print(f"模型路径: {model_path}") + + model = YoloV5() + + if not model.initialize(model_path): + print("模型初始化失败！") + return False + + print("模型初始化成功！") + + if label_path: + print(f"正在加载标签文件: {label_path}") + if model.load_labels(label_path): + print("标签文件加载成功！") + else: + print("标签文件加载失败，将使用默认标签 ID") + + model.set_threshold(conf_threshold=0.25, nms_threshold=0.45) + print("检测阈值已设置: conf_threshold=0.25, nms_threshold=0.45") + + print(f"\n正在读取图像: {image_path}") + image = cv2.imread(image_path) + + if image is None: + print(f"无法读取图像: {image_path}") + return False + + print(f"图像尺寸: {image.cols}x{image.rows}") + print(f"图像尺寸: {image.shape[1]}x{image.shape[0]}") + print("\n正在进行目标检测...") + results = model.predict(image) + + print(f"检测完成！共检测到 {len(results)} 个目标") + + for i, result in enumerate(results): + print(f"\n目标 {i + 1}:") + print(f" 矩形框: x={result.box.x}, y={result.box.y}, " + f"width={result.box.width}, height={result.box.height}") + print(f" 标签 ID: {result.label_id}") + print(f" 置信度: {result.score:.4f}") + + print(f"\n正在可视化检测结果...") + output_image = visualize(image, results) + + print(f"正在保存结果图像到: {output_path}") + cv2.imwrite(output_path, output_image) + + print(f"\n测试完成！结果已保存到 {output_path}") + return True + + +def test_yolov5_camera(model_path, label_path=None): + """ + 测试 YOLOv5 模型在摄像头实时视频上的目标检测功能 + + Args: + model_path (str): YOLOv5 模型文件路径 + label_path (str, optional): 标签文件路径 + """ + print(f"正在初始化 YOLOv5 模型...") + print(f"模型路径: {model_path}") + width = 640 + height = 480 + edit = Edit() + edit.start_and_accept_connection() + + model = YoloV5() + + if not model.initialize(model_path): + print("模型初始化失败！") + return False + + print("模型初始化成功！") + + if label_path: + print(f"正在加载标签文件: {label_path}") + if model.load_labels(label_path): + print("标签文件加载成功！") + else: + print("标签文件加载失败，将使用默认标签 ID") + + model.set_threshold(conf_threshold=0.25, nms_threshold=0.45) + print("检测阈值已设置: conf_threshold=0.25, nms_threshold=0.45") + + print("\n正在打开摄像头...") + cap = cv2.VideoCapture() + + if not cap.open(0): + print("无法打开摄像头！") + return False + + print("摄像头已打开，按 'q' 键退出") + + while True: + ret, frame = cap.read() + + if not ret: + print("无法读取摄像头画面") + break + + results = model.predict(frame) + + + output_frame = visualize(frame, results) + edit.print(output_frame) + + + cap.release() + cv2.destroyAllWindows() + print("\n摄像头测试结束") + return True + + +if __name__ == "__main__": + if len(sys.argv) < 3: + print("使用方法:") + print(" 测试图像: python test_yolov5.py [label_path] [output_path]") + print(" 测试摄像头: python test_yolov5.py camera [label_path]") + print("\n参数说明:") + print(" model_path - YOLOv5 模型文件路径") + print(" image_path - 测试图像路径") + print(" camera - 使用摄像头进行实时检测") + print(" label_path - (可选) 标签文件路径") + print(" output_path - (可选) 输出图像路径，默认为 'yolov5_result.jpg'") + sys.exit(1) + + model_path = sys.argv[1] + mode = sys.argv[2] + + if mode == "camera": + label_path = sys.argv[3] if len(sys.argv) > 3 else None + test_yolov5_camera(model_path, label_path) + else: + image_path = mode + label_path = sys.argv[3] if len(sys.argv) > 3 else None + output_path = sys.argv[4] if len(sys.argv) > 4 else 'yolov5_result.jpg' + test_yolov5_image(model_path, image_path, label_path, output_path) +``` + +## 7 上传并测试 Python 程序 + +参考 [连接设备指南](../../../../docs/introductory_tutorial/connect_device_using_ssh.md) 正确连接 Lockzhiner Vision Module 设备。 + +请使用 Electerm Sftp 依次上传以下文件: + +- 进入存放 **test_yolov5.py** 脚本文件的目录，将 **test_yolov5.py** 上传到 Lockzhiner Vision Module +- 进入存放 **.rknn** 模型存放的目录（模型存放在训练模型后下载的 output 文件夹内），将 **.rknn** 模型上传到 Lockzhiner Vision Module +- （可选）如果有标签文件，将标签文件也上传到 Lockzhiner Vision Module + +请使用 Electerm Ssh 并在命令行中执行以下命令: + +**测试单张图像：** + +```bash +python test_yolov5.py [label_path] [output_path] +``` + +**测试摄像头：** + +```bash +python test_yolov5.py camera [label_path] +``` + +连接凌智视觉模块图片传输助手后，选择连接设备，运行程序后，屏幕上开始打印矩形框信息、标签信息和置信度，并在一段时间后输出 FPS 值。 + +## 8 各模型性能指标 + +以下测试数据为模型执行 Predict 函数运行 1000 次耗时的平均时间。 + +支持的模型输入尺寸为： +- 640x640：推理时间 100ms +- 416x416：推理时间 70ms +- 320x320：推理时间 50ms + +请根据需求自行调整。 + +| 目标检测模型 | 输入尺寸 | 推理时间(ms) | FPS(帧/s) | +|:------:|:--------:|:--------:|:--------:| +| YOLOv5 | 640x640 | 100 | 10 | +| YOLOv5 | 416x416 | 70 | 14 | +| YOLOv5 | 320x320 | 50 | 20 | + + + +