diff --git a/Cpp_example/D03_face_recognition_system/README.md b/Cpp_example/D03_face_recognition_system/README.md index 5ab0c2b82ccd7ff551344a716b6a611f41b919fd..72b1d9b594a1df78177f0b37ab71ffa6e7c7b049 100755 --- a/Cpp_example/D03_face_recognition_system/README.md +++ b/Cpp_example/D03_face_recognition_system/README.md @@ -258,7 +258,7 @@ make -j8 && make install 在凌智视觉模块输入以下命令： ```shell chmod 777 Test-face-recognition-system -./Test-face-recognition-system LZ-Face LZ-ArcFace BaseDataset CropDataset +./Test-face-recognition-system LZ-Face.rknn LZ-ArcFace.rknn BaseDataset CropDataset ``` ### 5.3 运行效果 ![title](./images/result.png) diff --git a/Cpp_example/D10_yolov5/README.md b/Cpp_example/D10_yolov5/README.md index 98ce23290acc789ba0a1b3639bf35300f3e6468c..575ff2d0d15544ff22c4e753dc6b759348b5d818 100644 --- a/Cpp_example/D10_yolov5/README.md +++ b/Cpp_example/D10_yolov5/README.md @@ -1,120 +1,165 @@ # YOLOV5目标检测 + 本章节基于 Lockzhiner Vision Module 和 YOLOv5 目标检测模型，实现实时目标检测功能。 + ## 1. 基本知识简介 + ### 1.1 目标检测简介 + 目标检测是计算机视觉领域中的一个关键任务，它不仅需要识别图像中存在哪些对象，还需要定位这些对象的位置。具体来说，目标检测算法会输出每个检测到的对象的边界框（Bounding Box）以及其所属类别的概率或置信度得分。 + - 应用场景：目标检测技术广泛应用于多个领域，包括但不限于安全监控、自动驾驶汽车、智能零售和医疗影像分析。 + ### 1.2 YOLOv5简介 + YOLOv5 是 Ultralytics 在 2020 年基于 PyTorch 开发的一种高效目标检测模型，属于 YOLO 系列。它通过一次前向传播即可预测目标的类别和位置，适用于实时检测任务。YOLOv5 提供了多种模型大小（如 YOLOv5s、m、l、x），适应不同硬件条件。其结构包括骨干网络 CSPDarknet、特征融合层和检测头，支持多尺度预测和自定义训练，广泛用于各种检测场景。 ## 2. API 文档 + ### 2.1 YOLOv5模型类 + #### 2.1.1 头文件 + ```cpp #include "yolov5.h" ``` + #### 2.1.2 模型初始化函数 + ```cpp int init_yolov5_model(const char* model_path, rknn_app_context_t* ctx); ``` + - 作用：加载YOLOv5 RKNN模型并初始化推理上下文 - 参数 - - model_path：RKNN模型文件路径 - - ctx：模型上下文指针 + - model_path：RKNN模型文件路径 + - ctx：模型上下文指针 - 返回值： - - 0：初始化成功 - - -1：初始化失败 + - 0：初始化成功 + - -1：初始化失败 + #### 2.1.3 模型推理函数 + ```cpp int inference_yolov5_model(rknn_app_context_t* ctx, object_detect_result_list* od_results); ``` + - 作用：执行模型推理并获取检测结果 - 参数： - - ctx：已初始化的模型上下文 - - od_results：检测结果存储结构体指针 + - ctx：已初始化的模型上下文 + - od_results：检测结果存储结构体指针 - 返回值： - - 0：推理成功 - - -1 ：推理失败 + - 0：推理成功 + - -1 ：推理失败 + #### 2.1.4 模型释放函数 + ```cpp void release_yolov5_model(rknn_app_context_t* ctx); ``` + - 作用：释放模型相关资源 - 参数： - - ctx：待释放的模型上下文 + - ctx：待释放的模型上下文 - 返回值：无 + ### 2.2 图像处理函数 + #### 2.2.1 Letterbox处理 + ```cpp cv::Mat letterbox(cv::Mat& image); ``` + - 作用：保持图像比例进行缩放，添加灰边填充 - 参数： -image:输入图像(RGB格式) - 返回值： - - 返回预处理图像 + - 返回预处理图像 + #### 2.2.2 坐标映射函数 + ```cpp void mapCoordinates(float& x, float& y); ``` + - 作用：将模型输出坐标映射回原始图像坐标系 - 参数： - - x/y：模型输出坐标（输入输出参数） + - x/y：模型输出坐标（输入输出参数） - 返回值:无 + ### 2.3 结果处理函数 + #### 2.3.1 后处理初始化 + ```cpp void init_post_process(); ``` + - 作用：加载类别标签文件 - 参数：无 - 返回值：无 + #### 2.3.2 结果绘制函数 + ```cpp void draw_detections(int count, object_detect_result* results, cv::Mat& frame, void (*mapFunc)(float&, float&)); ``` + - 作用：在图像上绘制检测框和标签 - 参数： - - count:检测结果数量 - - results：检测结果数组 - - frame：目标图像帧 - - mapFunc：坐标映射函数指针 + - count:检测结果数量 + - results：检测结果数组 + - frame：目标图像帧 + - mapFunc：坐标映射函数指针 - 返回值：无 + ## 3. 代码解析 + ### 3.1 流程图 ### 3.2核心代码解析 + - 模型初始化 + ```cpp rknn_app_context_t rknn_app_ctx; init_yolov5_model("yolov5s.rknn", &rknn_app_ctx); ``` + - 图像预处理 + ```cpp cv::Mat letterboxImage = letterbox(frame); // 保持比例缩放 memcpy(rknn_app_ctx.input_mems[0]->virt_addr, letterboxImage.data, model_width * model_height * 3); ``` + - 模型推理 + ```cpp object_detect_result_list od_results; inference_yolov5_model(&rknn_app_ctx, &od_results); ``` + - 结果可视化 + ```cpp draw_detections(od_results.count, od_results.results, frame, mapCoordinates); ``` + ### 3.3 完整代码实现 + ```cpp #include #include @@ -146,117 +191,121 @@ char *lable; int main(int argc, char *argv[]) { - if (argc != 4) - { - LOGGER_INFO("Usage: %s ./yolov5_main model_path ./label size\n ./label_txt"); - } - obj_class_num = atoi(argv[2]); - lable = argv[3]; - // Rknn model - char text[16]; - // rknn上下文结构体 - rknn_app_context_t rknn_app_ctx; - object_detect_result_list od_results; - int ret; - const char *model_path = argv[1]; - memset(&rknn_app_ctx, 0, sizeof(rknn_app_context_t)); - - // Step 1: Load RKNN model - if (init_yolov5_model(model_path, &rknn_app_ctx) != 0) - { - printf("❌ Failed to load RKNN model!\n"); - return -1; - } - printf("✅ RKNN model loaded successfully.\n"); - - // 加载标签文件 - init_post_process(); - - // 打开摄像头 - lockzhiner_vision_module::edit::Edit edit; - if (!edit.StartAndAcceptConnection()) - { - std::cerr << "Error: Failed to start and accept connection." << std::endl; - return EXIT_FAILURE; - } - - cv::VideoCapture cap; - cap.set(cv::CAP_PROP_FRAME_WIDTH, 640); - cap.set(cv::CAP_PROP_FRAME_HEIGHT, 480); - cap.open(0); - - if (!cap.isOpened()) - { - std::cerr << "Error: Could not open camera." << std::endl; - return 1; - } - cv::Mat frame; - // 在 while 循环外声明 start 和 end 变量 - std::chrono::steady_clock::time_point start, end; - - while (true) - { - // 记录开始时间 - start = std::chrono::steady_clock::now(); - // Step 2: Load image from command line - cap >> frame; - if (frame.empty()) - { - LOGGER_INFO("❌ Failed to read frame from camera.\n"); - continue; - } - cv::resize(frame, frame, cv::Size(width, height), 0, 0, cv::INTER_LINEAR); - cv::Mat letterboxImage = letterbox(frame); - - if (letterboxImage.empty() || letterboxImage.total() * letterboxImage.elemSize() != model_width * model_height * 3) - { - - LOGGER_ERROR("❌ Input image format or size mismatch!\n"); - release_yolov5_model(&rknn_app_ctx); - return -1; - } - - if (rknn_app_ctx.input_mems == nullptr || rknn_app_ctx.input_mems[0] == nullptr) - { - - LOGGER_ERROR("❌ RKNN input memory not allocated!\n"); - release_yolov5_model(&rknn_app_ctx); - return -1; - } - - memcpy(rknn_app_ctx.input_mems[0]->virt_addr, letterboxImage.data, model_width * model_height * 3); - - if (inference_yolov5_model(&rknn_app_ctx, &od_results) != 0) - { - LOGGER_ERROR("inference_yolov5_model failed"); - release_yolov5_model(&rknn_app_ctx); - return -1; - } - - draw_detections(od_results.count, // 传入结果数量 - od_results.results, // 传入结果数组 - frame, // 图像帧 - mapCoordinates); // 直接使用现有坐标映射函数 - - edit.Print(frame); - // 记录结束时间 - end = std::chrono::steady_clock::now(); - // 计算耗时（秒） - double elapsed_time = std::chrono::duration(end - start).count(); - printf("Frame processed in %.4f seconds\n", elapsed_time); - } - release_yolov5_model(&rknn_app_ctx); - deinit_post_process(); - cap.release(); - return 0; + if (argc != 4) + { + LOGGER_INFO("Usage: %s ./yolov5_main model_path ./label size\n ./label_txt"); + } + obj_class_num = atoi(argv[2]); + lable = argv[3]; + // Rknn model + char text[16]; + // rknn上下文结构体 + rknn_app_context_t rknn_app_ctx; + object_detect_result_list od_results; + int ret; + const char *model_path = argv[1]; + memset(&rknn_app_ctx, 0, sizeof(rknn_app_context_t)); + + // Step 1: Load RKNN model + if (init_yolov5_model(model_path, &rknn_app_ctx) != 0) + { + printf("❌ Failed to load RKNN model!\n"); + return -1; + } + printf("✅ RKNN model loaded successfully.\n"); + + // 加载标签文件 + init_post_process(); + + // 打开摄像头 + lockzhiner_vision_module::edit::Edit edit; + if (!edit.StartAndAcceptConnection()) + { + std::cerr << "Error: Failed to start and accept connection." << std::endl; + return EXIT_FAILURE; + } + + cv::VideoCapture cap; + cap.set(cv::CAP_PROP_FRAME_WIDTH, 640); + cap.set(cv::CAP_PROP_FRAME_HEIGHT, 480); + cap.open(0); + + if (!cap.isOpened()) + { + std::cerr << "Error: Could not open camera." << std::endl; + return 1; + } + cv::Mat frame; + // 在 while 循环外声明 start 和 end 变量 + std::chrono::steady_clock::time_point start, end; + + while (true) + { + // 记录开始时间 + start = std::chrono::steady_clock::now(); + // Step 2: Load image from command line + cap >> frame; + if (frame.empty()) + { + LOGGER_INFO("❌ Failed to read frame from camera.\n"); + continue; + } + cv::resize(frame, frame, cv::Size(width, height), 0, 0, cv::INTER_LINEAR); + cv::Mat letterboxImage = letterbox(frame); + + if (letterboxImage.empty() || letterboxImage.total() * letterboxImage.elemSize() != model_width * model_height * 3) + { + + LOGGER_ERROR("❌ Input image format or size mismatch!\n"); + release_yolov5_model(&rknn_app_ctx); + return -1; + } + + if (rknn_app_ctx.input_mems == nullptr || rknn_app_ctx.input_mems[0] == nullptr) + { + + LOGGER_ERROR("❌ RKNN input memory not allocated!\n"); + release_yolov5_model(&rknn_app_ctx); + return -1; + } + + memcpy(rknn_app_ctx.input_mems[0]->virt_addr, letterboxImage.data, model_width * model_height * 3); + + if (inference_yolov5_model(&rknn_app_ctx, &od_results) != 0) + { + LOGGER_ERROR("inference_yolov5_model failed"); + release_yolov5_model(&rknn_app_ctx); + return -1; + } + + draw_detections(od_results.count, // 传入结果数量 + od_results.results, // 传入结果数组 + frame, // 图像帧 + mapCoordinates); // 直接使用现有坐标映射函数 + + edit.Print(frame); + // 记录结束时间 + end = std::chrono::steady_clock::now(); + // 计算耗时（秒） + double elapsed_time = std::chrono::duration(end - start).count(); + printf("Frame processed in %.4f seconds\n", elapsed_time); + } + release_yolov5_model(&rknn_app_ctx); + deinit_post_process(); + cap.release(); + return 0; } ``` ## 4. 编译过程 + ### 4.1 编译环境搭建 + - 请确保你已经按照 [开发环境搭建指南](../../../../docs/introductory_tutorial/cpp_development_environment.md) 正确配置了开发环境。 - 同时以正确连接开发板。 + ### 4.2 Cmake介绍 + ```cmake # CMake最低版本要求 cmake_minimum_required(VERSION 3.10) @@ -302,8 +351,11 @@ install( RUNTIME DESTINATION . ) ``` + ### 4.3 编译项目 + 使用 Docker Destop 打开 LockzhinerVisionModule 容器并执行以下命令来编译项目 + ```bash # 进入Demo所在目录 cd /LockzhinerVisionModuleWorkSpace/LockzhinerVisionModule/Cpp_example/D10_yolov5 @@ -320,14 +372,17 @@ make -j8 && make install 在执行完上述命令后，会在build目录下生成可执行文件。 ## 5. 例程运行示例 + ### 5.1 运行 + ```shell chmod 777 yolov5_main # 在实际应用的过程中LZ-Picodet需要替换为下载的或者你的rknn模型 ./yolov5_main ./voc320.rknn 20 label ``` + ### 5.2 结果展示 - 可以看到我们可以正确识别多种类别的 -![](./images/D10_yolov5.png) \ No newline at end of file +![](./images/D10_yolov5.png) diff --git "a/Cpp_example/D10_yolov5/\346\250\241\345\236\213\346\226\207\344\273\266\350\275\254\346\215\242.md" "b/Cpp_example/D10_yolov5/\346\250\241\345\236\213\346\226\207\344\273\266\350\275\254\346\215\242.md" new file mode 100644 index 0000000000000000000000000000000000000000..1be139b6c89a12d30b4dbf76579c5cc9f8207a0b --- /dev/null +++ "b/Cpp_example/D10_yolov5/\346\250\241\345\236\213\346\226\207\344\273\266\350\275\254\346\215\242.md" @@ -0,0 +1,223 @@ +# 模型训练以及转换 + +## 1. 模型训练 + +### 1.1 下载模型训练代码 + +```shell +git clone https://github.com/airockchip/yolov5.git +``` + +### 1.2 准备数据集 + +将自定义数据集转换为yolo格式，下面以Labelme工具标注的数据集为例，运行以下代码进行数据集的转换： + +```python +import os +import json +import shutil +import random +from pathlib import Path + +def convert_labelme_to_yolo(src_dir, dest_dir, yaml_output_path, split_ratio=0.8): + src_dir = Path(src_dir) + dest_dir = Path(dest_dir) + yaml_output_path = Path(yaml_output_path) + + # 输入路径设置 + annotations_dir = src_dir / 'annotations' + images_dir = src_dir / 'images' + + if not annotations_dir.exists(): + print(f"找不到标注文件夹: {annotations_dir}") + return + + # 1. 创建 YOLOv5 所需目录结构 + for split in ['train', 'val']: + (dest_dir / 'images' / split).mkdir(parents=True, exist_ok=True) + (dest_dir / 'labels' / split).mkdir(parents=True, exist_ok=True) + + # 获取所有 json 文件 + json_files = list(annotations_dir.glob('*.json')) + if not json_files: + print("未找到任何 JSON 标注文件！") + return + + # 2. 随机打乱用于划分训练集 (80%) 和验证集 (20%) + random.seed(42) + random.shuffle(json_files) + train_count = int(len(json_files) * split_ratio) + + # 类别映射（目前只有一个类别 fire，这里需要根据你的数据集类别进行修改） + classes = ['fire'] + class_to_id = {cls: i for i, cls in enumerate(classes)} + + print(f"总计找到 {len(json_files)} 个标注文件，开始转换...") + + success_count = 0 + for i, json_path in enumerate(json_files): + split = 'train' if i < train_count else 'val' + + with open(json_path, 'r', encoding='utf-8') as f: + data = json.load(f) + + # 获取图像宽高 + img_w = data.get('imageWidth') + img_h = data.get('imageHeight') + + if not img_w or not img_h: + print(f"警告: {json_path.name} 中没有图像宽高数据，跳过。") + continue + + # 解析图像文件名 (兼容 Windows '\\' 和 Linux '/') + img_name = data.get('imagePath', '').replace('\\', '/').split('/')[-1] + if not img_name: + img_name = json_path.stem + '.png' + + src_img_path = images_dir / img_name + + # 3. 如果图片不存在，尝试其它常见的图片后缀 + if not src_img_path.exists(): + for ext in ['.jpg', '.jpeg', '.JPG', '.PNG']: + alt_img_path = images_dir / (json_path.stem + ext) + if alt_img_path.exists(): + src_img_path = alt_img_path + img_name = alt_img_path.name + break + else: + print(f"警告: 找不到对应的图像文件 {src_img_path}，跳过此样本。") + continue + + # 4. 准备输出 label 内容 + yolo_labels = [] + for shape in data.get('shapes', []): + label = shape.get('label') + if label not in class_to_id: + continue # 如果存在其它意外的类别则忽略 + + class_id = class_to_id[label] + points = shape.get('points') + + # 确保是矩形框并且有两个点 + if shape.get('shape_type') == 'rectangle' and len(points) == 2: + x1, y1 = points[0] + x2, y2 = points[1] + + # 确保坐标极值正确 + x_min, x_max = min(x1, x2), max(x1, x2) + y_min, y_max = min(y1, y2), max(y1, y2) + + # 转换为 YOLO 的中心点及宽高，并且归一化到 0~1 + x_center = ((x_min + x_max) / 2.0) / img_w + y_center = ((y_min + y_max) / 2.0) / img_h + w = (x_max - x_min) / img_w + h = (y_max - y_min) / img_h + + yolo_labels.append(f"{class_id} {x_center:.6f} {y_center:.6f} {w:.6f} {h:.6f}") + + # 5. 写入 YOLO 格式的 txt 标签文件 + dest_txt_path = dest_dir / 'labels' / split / (json_path.stem + '.txt') + with open(dest_txt_path, 'w', encoding='utf-8') as f: + f.write('\n'.join(yolo_labels)) + + # 6. 复制图像文件到 YOLO 目录 + dest_img_path = dest_dir / 'images' / split / img_name + shutil.copy(src_img_path, dest_img_path) + success_count += 1 + + # 7. 生成 YOLOv8 训练所需的 YAML 文件 + yaml_content = f"""path: {dest_dir.absolute()} # 数据集根目录 +train: images/train # 训练集图片目录 (相对于 path) +val: images/val # 验证集图片目录 (相对于 path) + +# 类别定义 +names: + 0: fire +""" + yaml_output_path.parent.mkdir(parents=True, exist_ok=True) + with open(yaml_output_path, 'w', encoding='utf-8') as f: + f.write(yaml_content) + + print(f"\n转换完成！成功处理 {success_count} 个样本。") + print(f"YOLO 格式数据集保存在: {dest_dir}") + print(f"YOLO 配置文件已生成: {yaml_output_path}") + +if __name__ == '__main__': + # 原始数据集路径 + SOURCE_DIR = 'path/to/yolov5/Dataset' + # 转换后YOLO数据集的存放路径 + DEST_DIR = 'path/to/yolov5/Dataset_YOLO' + # 要求的 yaml 配置文件生成路径 + YAML_PATH = 'path/to/yolov5/data/data.yaml' + + convert_labelme_to_yolo(SOURCE_DIR, DEST_DIR, YAML_PATH) +``` + +### 1.3 开始训练 + +运行以下代码开始训练yolov5模型 + +```python +python train.py --data ./data/data.yaml --epochs 300 --weights yolov5s.pt --cfg yolov5s.yaml --batch-size 32 +``` + +训练好的模型会保存在./runs/train/weights目录下。 + +## 2. 模型转换 + +### 2.1 导出onnx模型 + +由于是在rv1106上运行，对于模型转换请参考如下文档： + +[yolov5_export_onnx](https://github.com/airockchip/yolov5/blob/master/README_rkopt.md) + +在满足 ./requirements.txt 的环境要求后，执行以下语句导出模型 + +```shell +python export.py --rknpu --weight path/to/your/xxx.pt +``` + +执行完毕后，会生成 ONNX 模型. 假如原始模型为 yolov8s.pt，则生成 yolov8s.onnx 模型。 转换后的onnx模型会保存在runs/detect/train/weights目录下。 + +### 2.2 准备校准数据集 + +在量化时，提供 20~100 张有代表性的图片给 RKNN 即可。请在你的终端中运行以下命令，从你的 Dataset_YOLO/images/train 中提取部分图片路径生成校准文件： + +```shell +cd path/to/yolov5 +# 随机选取50张训练集的图片路径，写入 dataset_subset.txt +ls /path/to/yolov5/Dataset_YOLO/images/train/*.png | head -n 50 > dataset_subset.txt +``` + +- 注意：如果你的图片后缀是 .jpg ，请将上面的 *.png 替换为*.jpg + +### 2.3 转RKNN模型 + +#### 2.3.1 克隆rknn_model_zoo + +```shell +git clone https://github.com/airockchip/rknn_model_zoo.git +cd /path/to/rknn_model_zoo/examples/yolov5/python +``` + +#### 2.3.2 修改convert.py代码 + +主要是将 DATASET_PATH 指向刚才我们生成的 dataset_subset.txt ： + +```python +DATASET_PATH = 'path/to/dataset_subset.txt' +``` + +#### 2.3.3 运行convert.py + +在运行转换脚本前请确保你已经配置好了所需要的环境。并且将之前转换好的onnx模型放到以下目录： + +- /path/to/rknn_model_zoo/examples/yolov5/model/ + +执行以下命令开始转换 + +```shell +python convert.py ../model/xxxx.onnx rv1106 +``` + +执行完毕后，如果没有报错，你就会在 rknn_model_zoo/examples/yolov5/model/ 目录下得到 yolov5.rknn 文件，这个文件就可以直接放到你的 RV1106 板子上进行部署板上运行了！ diff --git a/Cpp_example/D15_yolov8/CMakeLists.txt b/Cpp_example/D15_yolov8/CMakeLists.txt new file mode 100755 index 0000000000000000000000000000000000000000..289ff4cb39ea4e8fe98a36d651510d55fb0c4ba8 --- /dev/null +++ b/Cpp_example/D15_yolov8/CMakeLists.txt @@ -0,0 +1,43 @@ +# CMake最低版本要求 +cmake_minimum_required(VERSION 3.10) + +project(yolov8) + +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED ON) + +# 定义项目根目录路径 +set(PROJECT_ROOT_PATH "${CMAKE_CURRENT_SOURCE_DIR}/../..") +message("PROJECT_ROOT_PATH = " ${PROJECT_ROOT_PATH}) + +include("${PROJECT_ROOT_PATH}/toolchains/arm-rockchip830-linux-uclibcgnueabihf.toolchain.cmake") + +# 定义 OpenCV SDK 路径 +set(OpenCV_ROOT_PATH "${PROJECT_ROOT_PATH}/third_party/opencv-mobile-4.10.0-lockzhiner-vision-module") +set(OpenCV_DIR "${OpenCV_ROOT_PATH}/lib/cmake/opencv4") +find_package(OpenCV REQUIRED) +set(OPENCV_LIBRARIES "${OpenCV_LIBS}") +# 定义 LockzhinerVisionModule SDK 路径 +set(LockzhinerVisionModule_ROOT_PATH "${PROJECT_ROOT_PATH}/third_party/lockzhiner_vision_module_sdk") +set(LockzhinerVisionModule_DIR "${LockzhinerVisionModule_ROOT_PATH}/lib/cmake/lockzhiner_vision_module") +find_package(LockzhinerVisionModule REQUIRED) + +set(RKNPU2_BACKEND_BASE_DIR "${LockzhinerVisionModule_ROOT_PATH}/include/lockzhiner_vision_module/vision/deep_learning/runtime") +if(NOT EXISTS ${RKNPU2_BACKEND_BASE_DIR}) + message(FATAL_ERROR "RKNPU2 backend base dir missing: ${RKNPU2_BACKEND_BASE_DIR}") +endif() + +add_executable(yolov8_main + main.cc + postprocess.cc + yolov8.cc + yolov8.h + postprocess.h +) +target_include_directories(yolov8_main PRIVATE ${LOCKZHINER_VISION_MODULE_INCLUDE_DIRS} ${rknpu2_INCLUDE_DIRS} ${RKNPU2_BACKEND_BASE_DIR}) +target_link_libraries(yolov8_main PRIVATE ${OPENCV_LIBRARIES} ${LOCKZHINER_VISION_MODULE_LIBRARIES}) + +install( + TARGETS yolov8_main + RUNTIME DESTINATION . +) \ No newline at end of file diff --git a/Cpp_example/D15_yolov8/README.md b/Cpp_example/D15_yolov8/README.md new file mode 100644 index 0000000000000000000000000000000000000000..33a170e547d9f6252aa359d4eea364b005322604 --- /dev/null +++ b/Cpp_example/D15_yolov8/README.md @@ -0,0 +1,521 @@ +# YOLOv8 目标检测 + +本章节基于 Lockzhiner Vision Module 和 YOLOv8 目标检测模型，实现实时目标检测功能。 + +## 1. 基本知识简介 + +### 1.1 目标检测简介 + +目标检测是计算机视觉领域中的一个关键任务，它不仅需要识别图像中存在哪些对象，还需要定位这些对象的位置。具体来说，目标检测算法会输出每个检测到的对象的边界框（Bounding Box）以及其所属类别的概率或置信度得分。 + +- 应用场景：目标检测技术广泛应用于多个领域，包括但不限于安全监控、自动驾驶汽车、智能零售和医疗影像分析。 + +### 1.2 YOLOv8 简介 + +YOLOv8 是 Ultralytics 在 2023 年推出的最新一代 YOLO 系列目标检测模型。相比 YOLOv5，YOLOv8 在架构和性能上都有显著提升： + +- **Anchor-Free 设计**：摒弃了传统的 Anchor 框机制，采用更简洁的 Anchor-Free 方式，提高了检测精度和泛化能力 +- **解耦头结构**：将分类和回归任务分离，使用两个独立的分支处理，提升了模型性能 +- **新的损失函数**：采用 CIoU Loss 和 DFL (Distribution Focal Loss) 组合，边界框定位更精准 +- **多尺度特征融合**：改进的 PANet 结构，更好地融合不同尺度的特征 +- **高效的主干网络**：优化的 CSPDarknet 结构，在速度和精度之间取得更好的平衡 + +YOLOv8 同样提供了多种模型规格（YOLOv8n/s/m/l/x），适用于从边缘设备到高性能服务器的各种应用场景。 + +## 2. API 文档 + +### 2.1 YOLOv8 模型类 + +#### 2.1.1 头文件 + +```cpp +#include "yolov8.h" +``` + +#### 2.1.2 模型初始化函数 + +```cpp +int init_yolov8_model(const char* model_path, rknn_app_context_t* ctx); +``` + +- 作用：加载 YOLOv8 RKNN 模型并初始化推理上下文 +- 参数： + - model_path：RKNN 模型文件路径 + - ctx：模型上下文指针 +- 返回值： + - 0：初始化成功 + - -1：初始化失败 + +#### 2.1.3 模型推理函数 + +```cpp +int inference_yolov8_model(rknn_app_context_t* ctx, + object_detect_result_list* od_results); +``` + +- 作用：执行模型推理并获取检测结果 +- 参数： + - ctx：已初始化的模型上下文 + - od_results：检测结果存储结构体指针 +- 返回值： + - 0：推理成功 + - -1：推理失败 + +#### 2.1.4 模型释放函数 + +```cpp +int release_yolov8_model(rknn_app_context_t* ctx); +``` + +- 作用：释放模型相关资源 +- 参数： + - ctx：待释放的模型上下文 +- 返回值： + - 0：释放成功 + - -1：释放失败 + +### 2.2 图像处理函数 + +#### 2.2.1 Letterbox 处理 + +```cpp +cv::Mat letterbox(cv::Mat input); +``` + +- 作用：保持图像比例进行缩放，添加灰边填充 +- 参数： + - input：输入图像 (RGB 格式) +- 返回值： + - 返回预处理后的 640x640 图像 + +#### 2.2.2 坐标映射函数 + +```cpp +void mapCoordinates(int *x, int *y); +``` + +- 作用：将模型输出的坐标映射回原始图像坐标系 +- 参数： + - x/y：模型输出坐标指针（输入输出参数） +- 返回值：无 + +### 2.3 结果处理函数 + +#### 2.3.1 后处理初始化 + +```cpp +int init_post_process(); +``` + +- 作用：加载类别标签文件 +- 参数：无 +- 返回值： + - 0：初始化成功 + - -1：初始化失败 + +#### 2.3.2 后处理释放 + +```cpp +void deinit_post_process(); +``` + +- 作用：释放后处理相关资源 +- 参数：无 +- 返回值：无 + +#### 2.3.3 类别名称获取 + +```cpp +char *coco_cls_to_name(int cls_id); +``` + +- 作用：将类别 ID 转换为可读的类别名称 +- 参数： + - cls_id：类别 ID +- 返回值： + - 类别名称字符串 + +#### 2.3.4 结果绘制函数 + +```cpp +void draw_detections(int count, + object_detect_result* results, + cv::Mat& frame, + void (*mapCoord)(int *, int *)); +``` + +- 作用：在图像上绘制检测框和标签 +- 参数： + - count：检测结果数量 + - results：检测结果数组 + - frame：目标图像帧 + - mapCoord：坐标映射函数指针 +- 返回值：无 + +#### 2.3.5 后处理核心函数 + +```cpp +int post_process(rknn_app_context_t *app_ctx, + void *outputs, + float conf_threshold, + float nms_threshold, + object_detect_result_list *od_results); +``` + +- 作用：解码模型输出，执行 DFL 和 NMS 处理 +- 参数： + - app_ctx：模型上下文 + - outputs：模型输出内存数组 + - conf_threshold：置信度阈值（默认 0.25） + - nms_threshold：NMS 阈值（默认 0.45） + - od_results：检测结果输出 +- 返回值： + - 0：处理成功 + - -1：处理失败 + +## 3. 代码解析 + +### 3.1 流程图 + +![](./images/yolov8.png) + +### 3.2 核心代码解析 + +- **模型初始化** + +```cpp +rknn_app_context_t rknn_app_ctx; +const char *model_path = argv[1]; +if (init_yolov8_model(model_path, &rknn_app_ctx) != 0) +{ + printf("❌ Failed to load RKNN model!\n"); + return -1; +} +printf("✅ RKNN model loaded successfully.\n"); +``` + +- **图像预处理** + +```cpp +cv::resize(frame, frame, cv::Size(width, height), 0, 0, cv::INTER_LINEAR); +cv::Mat letterboxImage = letterbox(frame); // 保持比例缩放，填充灰边 + +// 复制数据到 RKNN 输入内存 +memcpy(rknn_app_ctx.input_mems[0]->virt_addr, + letterboxImage.data, + model_width * model_height * 3); +``` + +- **模型推理** + +```cpp +object_detect_result_list od_results; +if (inference_yolov8_model(&rknn_app_ctx, &od_results) != 0) +{ + LOGGER_ERROR("inference_yolov8_model failed"); + return -1; +} +``` + +- **后处理（在 inference_yolov8_model 内部调用）** + +```cpp +// 解码三个检测头输出 +// 1. Box 分支：边界框回归 +// 2. Score_Sum 分支：快速过滤 +// 3. Score 分支：类别置信度 + +// DFL (Distribution Focal Loss) 解码边界框 +compute_dfl(before_dfl, dfl_len, box); + +// NMS (非极大值抑制) 过滤重叠框 +nms(validCount, filterBoxes, classId, indexArray, c, nms_threshold); +``` + +- **结果可视化** + +```cpp +draw_detections(od_results.count, + od_results.results, + frame, + mapCoordinates); +``` + +### 3.3 完整代码实现 + +```cpp +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "yolov8.h" + +// output img size +#define DISP_WIDTH 640 +#define DISP_HEIGHT 480 + +// disp size +int width = DISP_WIDTH; +int height = DISP_HEIGHT; + +// model size +int model_width = 640; +int model_height = 640; + +int leftPadding; +int topPadding; + +// label size +extern int obj_class_num; +char *lable; + +int main(int argc, char *argv[]) +{ + if (argc != 4) + { + LOGGER_INFO("Usage: %s ./yolov8_main model_path ./label size ./label_txt", argv[0]); + return -1; + } + obj_class_num = atoi(argv[2]); + lable = argv[3]; + if (obj_class_num <= 0) + { + LOGGER_ERROR("Invalid class count: %d", obj_class_num); + return -1; + } + + // rknn 上下文结构体 + rknn_app_context_t rknn_app_ctx; + object_detect_result_list od_results; + int ret; + const char *model_path = argv[1]; + memset(&rknn_app_ctx, 0, sizeof(rknn_app_context_t)); + + // Step 1: Load RKNN model + if (init_yolov8_model(model_path, &rknn_app_ctx) != 0) + { + printf("❌ Failed to load RKNN model!\n"); + return -1; + } + printf("✅ RKNN model loaded successfully.\n"); + + // 检查模型输出分支，动态调整类别数量 + int output_per_branch = rknn_app_ctx.io_num.n_output / 3; + int score_idx = output_per_branch > 0 ? (output_per_branch - 1) : -1; + if (score_idx >= 0) + { + int model_class_num = rknn_app_ctx.output_attrs[score_idx].dims[3]; + if (model_class_num > 0 && obj_class_num != model_class_num) + { + LOGGER_INFO("Class count mismatch, use model class count %d instead of arg %d", + model_class_num, obj_class_num); + obj_class_num = model_class_num; + } + } + + // 加载标签文件 + if (init_post_process() != 0) + { + LOGGER_ERROR("init_post_process failed"); + release_yolov8_model(&rknn_app_ctx); + return -1; + } + + // 打开摄像头 + lockzhiner_vision_module::edit::Edit edit; + if (!edit.StartAndAcceptConnection()) + { + std::cerr << "Error: Failed to start and accept connection." << std::endl; + return EXIT_FAILURE; + } + + cv::VideoCapture cap; + cap.set(cv::CAP_PROP_FRAME_WIDTH, 640); + cap.set(cv::CAP_PROP_FRAME_HEIGHT, 480); + cap.open(0); + + if (!cap.isOpened()) + { + std::cerr << "Error: Could not open camera." << std::endl; + return 1; + } + cv::Mat frame; + std::chrono::steady_clock::time_point start, end; + + while (true) + { + start = std::chrono::steady_clock::now(); + cap >> frame; + if (frame.empty()) + { + LOGGER_INFO("❌ Failed to read frame from camera.\n"); + continue; + } + cv::resize(frame, frame, cv::Size(width, height), 0, 0, cv::INTER_LINEAR); + cv::Mat letterboxImage = letterbox(frame); + + if (letterboxImage.empty() || + letterboxImage.total() * letterboxImage.elemSize() != model_width * model_height * 3) + { + LOGGER_ERROR("❌ Input image format or size mismatch!\n"); + release_yolov8_model(&rknn_app_ctx); + return -1; + } + + if (rknn_app_ctx.input_mems == nullptr || + rknn_app_ctx.input_mems[0] == nullptr) + { + LOGGER_ERROR("❌ RKNN input memory not allocated!\n"); + release_yolov8_model(&rknn_app_ctx); + return -1; + } + + memcpy(rknn_app_ctx.input_mems[0]->virt_addr, + letterboxImage.data, + model_width * model_height * 3); + + if (inference_yolov8_model(&rknn_app_ctx, &od_results) != 0) + { + LOGGER_ERROR("inference_yolov8_model failed"); + release_yolov8_model(&rknn_app_ctx); + return -1; + } + + draw_detections(od_results.count, + od_results.results, + frame, + mapCoordinates); + + edit.Print(frame); + + end = std::chrono::steady_clock::now(); + double elapsed_time = std::chrono::duration(end - start).count(); + printf("Frame processed in %.4f seconds\n", elapsed_time); + } + + release_yolov8_model(&rknn_app_ctx); + deinit_post_process(); + cap.release(); + return 0; +} +``` + +## 4. 编译过程 + +### 4.1 编译环境搭建 + +- 请确保你已经按照 [开发环境搭建指南](../../../../docs/introductory_tutorial/cpp_development_environment.md) 正确配置了开发环境。 +- 同时已正确连接开发板。 + +### 4.2 CMake 介绍 + +```cmake +# CMake 最低版本要求 +cmake_minimum_required(VERSION 3.10) + +project(yolov8) + +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED ON) + +# 定义项目根目录路径 +set(PROJECT_ROOT_PATH "${CMAKE_CURRENT_SOURCE_DIR}/../..") +message("PROJECT_ROOT_PATH = " ${PROJECT_ROOT_PATH}) + +include("${PROJECT_ROOT_PATH}/toolchains/arm-rockchip830-linux-uclibcgnueabihf.toolchain.cmake") + +# 定义 OpenCV SDK 路径 +set(OpenCV_ROOT_PATH "${PROJECT_ROOT_PATH}/third_party/opencv-mobile-4.10.0-lockzhiner-vision-module") +set(OpenCV_DIR "${OpenCV_ROOT_PATH}/lib/cmake/opencv4") +find_package(OpenCV REQUIRED) +set(OPENCV_LIBRARIES "${OpenCV_LIBS}") + +# 定义 LockzhinerVisionModule SDK 路径 +set(LockzhinerVisionModule_ROOT_PATH "${PROJECT_ROOT_PATH}/third_party/lockzhiner_vision_module_sdk") +set(LockzhinerVisionModule_DIR "${LockzhinerVisionModule_ROOT_PATH}/lib/cmake/lockzhiner_vision_module") +find_package(LockzhinerVisionModule REQUIRED) + +set(RKNPU2_BACKEND_BASE_DIR "${LockzhinerVisionModule_ROOT_PATH}/include/lockzhiner_vision_module/vision/deep_learning/runtime") +if(NOT EXISTS ${RKNPU2_BACKEND_BASE_DIR}) + message(FATAL_ERROR "RKNPU2 backend base dir missing: ${RKNPU2_BACKEND_BASE_DIR}") +endif() + +add_executable(yolov8_main + main.cc + postprocess.cc + yolov8.cc + yolov8.h + postprocess.h +) +target_include_directories(yolov8_main PRIVATE + ${LOCKZHINER_VISION_MODULE_INCLUDE_DIRS} + ${rknpu2_INCLUDE_DIRS} + ${RKNPU2_BACKEND_BASE_DIR} +) +target_link_libraries(yolov8_main PRIVATE + ${OPENCV_LIBRARIES} + ${LOCKZHINER_VISION_MODULE_LIBRARIES} +) + +install( + TARGETS yolov8_main + RUNTIME DESTINATION . +) +``` + +### 4.3 编译项目 + +使用 Docker Desktop 打开 LockzhinerVisionModule 容器并执行以下命令来编译项目： + +```bash +# 进入 Demo 所在目录 +cd /LockzhinerVisionModuleWorkSpace/LockzhinerVisionModule/Cpp_example/D15_yolov8 + +# 创建编译目录 +rm -rf build && mkdir build && cd build + +# 配置交叉编译工具链 +export TOOLCHAIN_ROOT_PATH="/LockzhinerVisionModuleWorkSpace/arm-rockchip830-linux-uclibcgnueabihf" + +# 使用 cmake 配置项目 +cmake .. + +# 执行编译项目 +make -j8 && make install +``` + +在执行完上述命令后，会在 build 目录下生成可执行文件 `yolov8_main`。 + +## 5. 例程运行示例 + +### 5.1 运行 + +```shell +chmod 777 yolov8_main + +# 语法格式 +./yolov8_main + +# 实际示例（使用自定义火焰数据集训练的 1 类别模型） +./yolov8_main ./yolov8_fire_rv1106.rknn 1 ./labels.txt + +``` + +**参数说明：** + +- `model_path`：YOLOv8 RKNN 模型文件路径 +- `class_count`：类别数量 +- `label_file_path`：标签文件路径，每行一个类别名称 + +### 5.2 结果展示 + +- 可以准确识别火焰 +- 实时显示检测框、类别名称和置信度 +- 支持摄像头实时视频流检测 + +![](./images/yolov8_demo.png) diff --git a/Cpp_example/D15_yolov8/images/yolov8.png b/Cpp_example/D15_yolov8/images/yolov8.png new file mode 100755 index 0000000000000000000000000000000000000000..75a71d3a0f064a4796a2e77606f085a9419b1060 Binary files /dev/null and b/Cpp_example/D15_yolov8/images/yolov8.png differ diff --git a/Cpp_example/D15_yolov8/images/yolov8_demo.png b/Cpp_example/D15_yolov8/images/yolov8_demo.png new file mode 100755 index 0000000000000000000000000000000000000000..644656ccd232e88de1109049128febbdc8cf2fff Binary files /dev/null and b/Cpp_example/D15_yolov8/images/yolov8_demo.png differ diff --git a/Cpp_example/D15_yolov8/main.cc b/Cpp_example/D15_yolov8/main.cc new file mode 100755 index 0000000000000000000000000000000000000000..5221bf1a979fb92d0dc6990206ba08c4bc3041f9 --- /dev/null +++ b/Cpp_example/D15_yolov8/main.cc @@ -0,0 +1,161 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "yolov8.h" + +// output img size +#define DISP_WIDTH 640 +#define DISP_HEIGHT 480 + +// disp size +int width = DISP_WIDTH; +int height = DISP_HEIGHT; + +// model size +int model_width = 640; +int model_height = 640; + +int leftPadding; +int topPadding; + +// label size +extern int obj_class_num; +char *lable; + +int main(int argc, char *argv[]) +{ + if (argc != 4) + { + LOGGER_INFO("Usage: %s ./yolov8_main model_path ./label size ./label_txt", argv[0]); + return -1; + } + obj_class_num = atoi(argv[2]); + lable = argv[3]; + if (obj_class_num <= 0) + { + LOGGER_ERROR("Invalid class count: %d", obj_class_num); + return -1; + } + + // rknn上下文结构体 + rknn_app_context_t rknn_app_ctx; + object_detect_result_list od_results; + int ret; + const char *model_path = argv[1]; + memset(&rknn_app_ctx, 0, sizeof(rknn_app_context_t)); + + // Step 1: Load RKNN model + if (init_yolov8_model(model_path, &rknn_app_ctx) != 0) + { + printf("❌ Failed to load RKNN model!\n"); + return -1; + } + printf("✅ RKNN model loaded successfully.\n"); + + int output_per_branch = rknn_app_ctx.io_num.n_output / 3; + int score_idx = output_per_branch > 0 ? (output_per_branch - 1) : -1; + if (score_idx >= 0) + { + int model_class_num = rknn_app_ctx.output_attrs[score_idx].dims[3]; + if (model_class_num > 0 && obj_class_num != model_class_num) + { + LOGGER_INFO("Class count mismatch, use model class count %d instead of arg %d", model_class_num, obj_class_num); + obj_class_num = model_class_num; + } + } + + // 加载标签文件 + if (init_post_process() != 0) + { + LOGGER_ERROR("init_post_process failed"); + release_yolov8_model(&rknn_app_ctx); + return -1; + } + + // 打开摄像头 + lockzhiner_vision_module::edit::Edit edit; + if (!edit.StartAndAcceptConnection()) + { + std::cerr << "Error: Failed to start and accept connection." << std::endl; + return EXIT_FAILURE; + } + + cv::VideoCapture cap; + cap.set(cv::CAP_PROP_FRAME_WIDTH, 640); + cap.set(cv::CAP_PROP_FRAME_HEIGHT, 480); + cap.open(0); + + if (!cap.isOpened()) + { + std::cerr << "Error: Could not open camera." << std::endl; + return 1; + } + cv::Mat frame; + std::chrono::steady_clock::time_point start, end; + + while (true) + { + start = std::chrono::steady_clock::now(); + cap >> frame; + if (frame.empty()) + { + LOGGER_INFO("❌ Failed to read frame from camera.\n"); + continue; + } + cv::resize(frame, frame, cv::Size(width, height), 0, 0, cv::INTER_LINEAR); + cv::Mat letterboxImage = letterbox(frame); + + if (letterboxImage.empty() || letterboxImage.total() * letterboxImage.elemSize() != model_width * model_height * 3) + { + LOGGER_ERROR("❌ Input image format or size mismatch!\n"); + release_yolov8_model(&rknn_app_ctx); + return -1; + } + + if (rknn_app_ctx.input_mems == nullptr || rknn_app_ctx.input_mems[0] == nullptr) + { + LOGGER_ERROR("❌ RKNN input memory not allocated!\n"); + release_yolov8_model(&rknn_app_ctx); + return -1; + } + + memcpy(rknn_app_ctx.input_mems[0]->virt_addr, letterboxImage.data, model_width * model_height * 3); + + if (inference_yolov8_model(&rknn_app_ctx, &od_results) != 0) + { + LOGGER_ERROR("inference_yolov8_model failed"); + release_yolov8_model(&rknn_app_ctx); + return -1; + } + + // printf("draw_detections start, count=%d\n", od_results.count); + // fflush(stdout); + draw_detections(od_results.count, + od_results.results, + frame, + mapCoordinates); + // printf("draw_detections done\n"); + // fflush(stdout); + + // printf("edit.Print start\n"); + // fflush(stdout); + + edit.Print(frame); + + // printf("edit.Print done\n"); + // fflush(stdout); + end = std::chrono::steady_clock::now(); + double elapsed_time = std::chrono::duration(end - start).count(); + printf("Frame processed in %.4f seconds\n", elapsed_time); + } + release_yolov8_model(&rknn_app_ctx); + deinit_post_process(); + cap.release(); + return 0; +} \ No newline at end of file diff --git a/Cpp_example/D15_yolov8/model/fire_label.txt b/Cpp_example/D15_yolov8/model/fire_label.txt new file mode 100755 index 0000000000000000000000000000000000000000..69b21656c0ecea94332352c454ba30bf1e0586bb --- /dev/null +++ b/Cpp_example/D15_yolov8/model/fire_label.txt @@ -0,0 +1 @@ +fire \ No newline at end of file diff --git a/Cpp_example/D15_yolov8/postprocess.cc b/Cpp_example/D15_yolov8/postprocess.cc new file mode 100755 index 0000000000000000000000000000000000000000..ec6c51ce746d0f56d5972ea22b406ae75ddd863a --- /dev/null +++ b/Cpp_example/D15_yolov8/postprocess.cc @@ -0,0 +1,564 @@ +#include "yolov8.h" + +#include +#include +#include +#include +#include +#include + +#include +#include +#define LABEL_NALE_TXT_PATH "./model/coco_80_labels_list.txt" + +int OBJ_CLASS_NUM = 1; // 默认值（通过 init_post_process 修改） +static char** labels = nullptr; // 动态分配的标签数组 +int obj_class_num = 0; + + +inline static int clamp(float val, int min, int max) { return val > min ? (val < max ? val : max) : min; } +extern char *lable; +static char *readLine(FILE *fp, char *buffer, int *len) +{ + int ch; + int i = 0; + size_t buff_len = 0; + + buffer = (char *)malloc(buff_len + 1); + if (!buffer) + return NULL; // Out of memory + + while ((ch = fgetc(fp)) != '\n' && ch != EOF) + { + if (ch == '\r') continue; // 过滤 Windows 的 \r 换行符，防止 cv::putText 卡死 + buff_len++; + void *tmp = realloc(buffer, buff_len + 1); + if (tmp == NULL) + { + free(buffer); + return NULL; // Out of memory + } + buffer = (char *)tmp; + + buffer[i] = (char)ch; + i++; + } + buffer[i] = '\0'; + + *len = buff_len; + + // Detect end + if (ch == EOF && (i == 0 || ferror(fp))) + { + free(buffer); + return NULL; + } + return buffer; +} + +static int readLines(const char *fileName, char *lines[], int max_line) +{ + FILE *file = fopen(fileName, "r"); + char *s = NULL; + int i = 0; + int n = 0; + + if (file == NULL) + { + printf("Open %s fail!\n", fileName); + return -1; + } + + while ((s = readLine(file, s, &n)) != NULL) + { + lines[i++] = s; + if (i >= max_line) + break; + } + fclose(file); + return i; +} + +static int loadLabelName(const char *locationFilename, char *label[]) +{ + printf("load lable %s\n", locationFilename); + readLines(locationFilename, label, OBJ_CLASS_NUM); + return 0; +} + +static float CalculateOverlap(float xmin0, float ymin0, float xmax0, float ymax0, float xmin1, float ymin1, float xmax1, + float ymax1) +{ + float w = fmax(0.f, fmin(xmax0, xmax1) - fmax(xmin0, xmin1) + 1.0); + float h = fmax(0.f, fmin(ymax0, ymax1) - fmax(ymin0, ymin1) + 1.0); + float i = w * h; + float u = (xmax0 - xmin0 + 1.0) * (ymax0 - ymin0 + 1.0) + (xmax1 - xmin1 + 1.0) * (ymax1 - ymin1 + 1.0) - i; + return u <= 0.f ? 0.f : (i / u); +} + +static int nms(int validCount, std::vector &outputLocations, std::vector classIds, std::vector &order, + int filterId, float threshold) +{ + for (int i = 0; i < validCount; ++i) + { + int n = order[i]; + if (n == -1 || classIds[n] != filterId) + { + continue; + } + for (int j = i + 1; j < validCount; ++j) + { + int m = order[j]; + if (m == -1 || classIds[m] != filterId) + { + continue; + } + float xmin0 = outputLocations[n * 4 + 0]; + float ymin0 = outputLocations[n * 4 + 1]; + float xmax0 = outputLocations[n * 4 + 0] + outputLocations[n * 4 + 2]; + float ymax0 = outputLocations[n * 4 + 1] + outputLocations[n * 4 + 3]; + + float xmin1 = outputLocations[m * 4 + 0]; + float ymin1 = outputLocations[m * 4 + 1]; + float xmax1 = outputLocations[m * 4 + 0] + outputLocations[m * 4 + 2]; + float ymax1 = outputLocations[m * 4 + 1] + outputLocations[m * 4 + 3]; + + float iou = CalculateOverlap(xmin0, ymin0, xmax0, ymax0, xmin1, ymin1, xmax1, ymax1); + + if (iou > threshold) + { + order[j] = -1; + } + } + } + return 0; +} + +static int quick_sort_indice_inverse(std::vector &input, int left, int right, std::vector &indices) +{ + float key; + int key_index; + int low = left; + int high = right; + if (left < right) + { + key_index = indices[left]; + key = input[left]; + while (low < high) + { + while (low < high && input[high] <= key) + { + high--; + } + input[low] = input[high]; + indices[low] = indices[high]; + while (low < high && input[low] >= key) + { + low++; + } + input[high] = input[low]; + indices[high] = indices[low]; + } + input[low] = key; + indices[low] = key_index; + quick_sort_indice_inverse(input, left, low - 1, indices); + quick_sort_indice_inverse(input, low + 1, right, indices); + } + return low; +} + +static float sigmoid(float x) { return 1.0 / (1.0 + expf(-x)); } + +static float unsigmoid(float y) { return -1.0 * logf((1.0 / y) - 1.0); } + +inline static int32_t __clip(float val, float min, float max) +{ + float f = val <= min ? min : (val >= max ? max : val); + return f; +} + +static int8_t qnt_f32_to_affine(float f32, int32_t zp, float scale) +{ + float dst_val = (f32 / scale) + zp; + int8_t res = (int8_t)__clip(dst_val, -128, 127); + return res; +} + +static float deqnt_affine_to_f32(int8_t qnt, int32_t zp, float scale) { return ((float)qnt - (float)zp) * scale; } + +static void compute_dfl(float* tensor, int dfl_len, float* box){ + for (int b=0; b<4; b++){ + float exp_t[dfl_len]; + float exp_sum=0; + float acc_sum=0; + for (int i=0; i< dfl_len; i++){ + exp_t[i] = exp(tensor[i+b*dfl_len]); + exp_sum += exp_t[i]; + } + + for (int i=0; i< dfl_len; i++){ + acc_sum += exp_t[i]/exp_sum *i; + } + box[b] = acc_sum; + } +} + +static int process_i8(int8_t *box_tensor, int32_t box_zp, float box_scale, + int8_t *score_tensor, int32_t score_zp, float score_scale, + int8_t *score_sum_tensor, int32_t score_sum_zp, float score_sum_scale, + int grid_h, int grid_w, int stride, int dfl_len, + std::vector &boxes, + std::vector &objProbs, + std::vector &classId, + float threshold) +{ + int validCount = 0; + int grid_len = grid_h * grid_w; + int8_t score_thres_i8 = qnt_f32_to_affine(threshold, score_zp, score_scale); + int8_t score_sum_thres_i8 = qnt_f32_to_affine(threshold, score_sum_zp, score_sum_scale); + + for (int i = 0; i < grid_h; i++) + { + for (int j = 0; j < grid_w; j++) + { + int offset = i* grid_w + j; + int max_class_id = -1; + + // 通过 score sum 起到快速过滤的作用 + if (score_sum_tensor != nullptr){ + if (score_sum_tensor[offset] < score_sum_thres_i8){ + continue; + } + } + + int8_t max_score = -score_zp; + for (int c= 0; c< OBJ_CLASS_NUM; c++){ + if ((score_tensor[offset] > score_thres_i8) && (score_tensor[offset] > max_score)) + { + max_score = score_tensor[offset]; + max_class_id = c; + } + offset += grid_len; + } + + // compute box + if (max_score> score_thres_i8){ + offset = i* grid_w + j; + float box[4]; + float before_dfl[dfl_len*4]; + for (int k=0; k< dfl_len*4; k++){ + before_dfl[k] = deqnt_affine_to_f32(box_tensor[offset], box_zp, box_scale); + offset += grid_len; + } + compute_dfl(before_dfl, dfl_len, box); + + float x1,y1,x2,y2,w,h; + x1 = (-box[0] + j + 0.5)*stride; + y1 = (-box[1] + i + 0.5)*stride; + x2 = (box[2] + j + 0.5)*stride; + y2 = (box[3] + i + 0.5)*stride; + w = x2 - x1; + h = y2 - y1; + boxes.push_back(x1); + boxes.push_back(y1); + boxes.push_back(w); + boxes.push_back(h); + + objProbs.push_back(deqnt_affine_to_f32(max_score, score_zp, score_scale)); + classId.push_back(max_class_id); + validCount ++; + } + } + } + return validCount; +} + +static int process_fp32(float *box_tensor, float *score_tensor, float *score_sum_tensor, + int grid_h, int grid_w, int stride, int dfl_len, + std::vector &boxes, + std::vector &objProbs, + std::vector &classId, + float threshold) +{ + int validCount = 0; + int grid_len = grid_h * grid_w; + for (int i = 0; i < grid_h; i++) + { + for (int j = 0; j < grid_w; j++) + { + int offset = i* grid_w + j; + int max_class_id = -1; + + // 通过 score sum 起到快速过滤的作用 + if (score_sum_tensor != nullptr){ + if (score_sum_tensor[offset] < threshold){ + continue; + } + } + + float max_score = 0; + for (int c= 0; c< OBJ_CLASS_NUM; c++){ + if ((score_tensor[offset] > threshold) && (score_tensor[offset] > max_score)) + { + max_score = score_tensor[offset]; + max_class_id = c; + } + offset += grid_len; + } + + // compute box + if (max_score> threshold){ + offset = i* grid_w + j; + float box[4]; + float before_dfl[dfl_len*4]; + for (int k=0; k< dfl_len*4; k++){ + before_dfl[k] = box_tensor[offset]; + offset += grid_len; + } + compute_dfl(before_dfl, dfl_len, box); + + float x1,y1,x2,y2,w,h; + x1 = (-box[0] + j + 0.5)*stride; + y1 = (-box[1] + i + 0.5)*stride; + x2 = (box[2] + j + 0.5)*stride; + y2 = (box[3] + i + 0.5)*stride; + w = x2 - x1; + h = y2 - y1; + boxes.push_back(x1); + boxes.push_back(y1); + boxes.push_back(w); + boxes.push_back(h); + + objProbs.push_back(max_score); + classId.push_back(max_class_id); + validCount ++; + } + } + } + return validCount; +} + + +static int process_i8_rv1106(int8_t *box_tensor, int32_t box_zp, float box_scale, + int8_t *score_tensor, int32_t score_zp, float score_scale, + int8_t *score_sum_tensor, int32_t score_sum_zp, float score_sum_scale, + int grid_h, int grid_w, int stride, int dfl_len, int score_channels, + std::vector &boxes, + std::vector &objProbs, + std::vector &classId, + float threshold) { + int validCount = 0; + int grid_len = grid_h * grid_w; + int8_t score_thres_i8 = qnt_f32_to_affine(threshold, score_zp, score_scale); + int8_t score_sum_thres_i8 = qnt_f32_to_affine(threshold, score_sum_zp, score_sum_scale); + + for (int i = 0; i < grid_h; i++) { + for (int j = 0; j < grid_w; j++) { + int offset = i * grid_w + j; + int max_class_id = -1; + + // 通过 score sum 起到快速过滤的作用 + if (score_sum_tensor != nullptr) { + //score_sum_tensor [1, 1, 80, 80] + if (score_sum_tensor[offset] < score_sum_thres_i8) { + continue; + } + } + + int8_t max_score = -128; + int class_count = OBJ_CLASS_NUM < score_channels ? OBJ_CLASS_NUM : score_channels; + offset = offset * score_channels; + for (int c = 0; c < class_count; c++) { + int8_t score = score_tensor[offset + c]; + if ((score > score_thres_i8) && (score > max_score)) { + max_score = score; + max_class_id = c; + } + } + + // compute box + if (max_class_id >= 0 && max_score > score_thres_i8) { + offset = (i * grid_w + j) * 4 * dfl_len; + float box[4]; + float before_dfl[dfl_len*4]; + for (int k=0; k< dfl_len*4; k++){ + before_dfl[k] = deqnt_affine_to_f32(box_tensor[offset + k], box_zp, box_scale); + } + compute_dfl(before_dfl, dfl_len, box); + + float x1, y1, x2, y2, w, h; + x1 = (-box[0] + j + 0.5) * stride; + y1 = (-box[1] + i + 0.5) * stride; + x2 = (box[2] + j + 0.5) * stride; + y2 = (box[3] + i + 0.5) * stride; + w = x2 - x1; + h = y2 - y1; + boxes.push_back(x1); + boxes.push_back(y1); + boxes.push_back(w); + boxes.push_back(h); + + objProbs.push_back(deqnt_affine_to_f32(max_score, score_zp, score_scale)); + classId.push_back(max_class_id); + validCount ++; + } + } + } + return validCount; +} + +int post_process(rknn_app_context_t *app_ctx, void *outputs, float conf_threshold, float nms_threshold, object_detect_result_list *od_results) +{ + rknn_tensor_mem **_outputs = (rknn_tensor_mem **)outputs; + std::vector filterBoxes; + std::vector objProbs; + std::vector classId; + int validCount = 0; + int stride = 0; + int grid_h = 0; + int grid_w = 0; + int model_in_w = app_ctx->model_width; + int model_in_h = app_ctx->model_height; + + memset(od_results, 0, sizeof(object_detect_result_list)); + + // default 3 branch + int output_per_branch = app_ctx->io_num.n_output / 3; + for (int i = 0; i < 3; i++) + { + int box_idx = i * output_per_branch; + int dfl_len = app_ctx->output_attrs[box_idx].dims[3] / 4; + void *score_sum = nullptr; + int32_t score_sum_zp = 0; + float score_sum_scale = 1.0; + if (output_per_branch == 3) { + score_sum = _outputs[i * output_per_branch + 1]->virt_addr; // 修复：YOLOv8 中 score_sum 在索引 1 + score_sum_zp = app_ctx->output_attrs[i * output_per_branch + 1].zp; + score_sum_scale = app_ctx->output_attrs[i * output_per_branch + 1].scale; + } + int score_idx = i * output_per_branch + 2; // 修复：YOLOv8 中 score 在索引 2 + grid_h = app_ctx->output_attrs[box_idx].dims[1]; + grid_w = app_ctx->output_attrs[box_idx].dims[2]; + stride = model_in_h / grid_h; + + if (app_ctx->is_quant) { + validCount += process_i8_rv1106((int8_t *)_outputs[box_idx]->virt_addr, app_ctx->output_attrs[box_idx].zp, app_ctx->output_attrs[box_idx].scale, + (int8_t *)_outputs[score_idx]->virt_addr, app_ctx->output_attrs[score_idx].zp, + app_ctx->output_attrs[score_idx].scale, (int8_t *)score_sum, score_sum_zp, score_sum_scale, + grid_h, grid_w, stride, dfl_len, app_ctx->output_attrs[score_idx].dims[3], filterBoxes, objProbs, classId, conf_threshold); + } + else + { + printf("RV1106 only support quantization mode\n"); + return -1; + } + } + + // no object detect + if (validCount <= 0) + { + return 0; + } + std::vector indexArray; + for (int i = 0; i < validCount; ++i) + { + indexArray.push_back(i); + } + quick_sort_indice_inverse(objProbs, 0, validCount - 1, indexArray); + + std::set class_set(std::begin(classId), std::end(classId)); + + for (auto c : class_set) + { + nms(validCount, filterBoxes, classId, indexArray, c, nms_threshold); + } + + int last_count = 0; + od_results->count = 0; + + /* box valid detect target */ + for (int i = 0; i < validCount; ++i) + { + if (indexArray[i] == -1 || last_count >= OBJ_NUMB_MAX_SIZE) + { + continue; + } + int n = indexArray[i]; + + float x1 = filterBoxes[n * 4 + 0]; + float y1 = filterBoxes[n * 4 + 1]; + float x2 = x1 + filterBoxes[n * 4 + 2]; + float y2 = y1 + filterBoxes[n * 4 + 3]; + int id = classId[n]; + float obj_conf = objProbs[n]; + + od_results->results[last_count].box.left = (int)(clamp(x1, 0, model_in_w)); + od_results->results[last_count].box.top = (int)(clamp(y1, 0, model_in_h)); + od_results->results[last_count].box.right = (int)(clamp(x2, 0, model_in_w)); + od_results->results[last_count].box.bottom = (int)(clamp(y2, 0, model_in_h)); + od_results->results[last_count].prop = obj_conf; + od_results->results[last_count].cls_id = id; + last_count++; + } + od_results->count = last_count; + return 0; +} + +int init_post_process() +{ + OBJ_CLASS_NUM = obj_class_num; + + // 释放旧内存（防止内存泄漏） + if (labels != nullptr) { + deinit_post_process(); + } + + // 分配新内存并清零（防止野指针） + labels = new char*[OBJ_CLASS_NUM]; + memset(labels, 0, sizeof(char*) * OBJ_CLASS_NUM); + + int ret = 0; + ret = loadLabelName(lable, labels); + if (ret < 0) + { + printf("Load %s failed!\n", lable); + return -1; + } + return 0; +} + +char *coco_cls_to_name(int cls_id) +{ + + static char null_str[] = "null"; + + if (cls_id >= OBJ_CLASS_NUM) + { + return null_str; + } + + if (labels[cls_id]) + { + return labels[cls_id]; + } + + return null_str; +} + +void deinit_post_process() +{ + if (labels == nullptr) + { + return; + } + for (int i = 0; i < OBJ_CLASS_NUM; i++) + { + if (labels[i] != nullptr) + { + free(labels[i]); + labels[i] = nullptr; + } + } + delete[] labels; + labels = nullptr; +} \ No newline at end of file diff --git a/Cpp_example/D15_yolov8/postprocess.h b/Cpp_example/D15_yolov8/postprocess.h new file mode 100755 index 0000000000000000000000000000000000000000..d467c307692010d69e898264b4581fd21a592652 --- /dev/null +++ b/Cpp_example/D15_yolov8/postprocess.h @@ -0,0 +1,38 @@ +#ifndef _RKNN_YOLOV8_DEMO_POSTPROCESS_H_ +#define _RKNN_YOLOV8_DEMO_POSTPROCESS_H_ + +#include +#include +#include "rknpu2_backend/rknpu2_backend.h" + +#define OBJ_NAME_MAX_SIZE 64 +#define OBJ_NUMB_MAX_SIZE 128 +#define NMS_THRESH 0.45 +#define BOX_THRESH 0.25 + +typedef struct { + int left; + int top; + int right; + int bottom; +} image_rect_t; + +typedef struct { + image_rect_t box; + float prop; + int cls_id; +} object_detect_result; + +typedef struct { + int id; + int count; + object_detect_result results[OBJ_NUMB_MAX_SIZE]; +} object_detect_result_list; + +int init_post_process(); +void deinit_post_process(); +char *coco_cls_to_name(int cls_id); +int post_process(rknn_app_context_t *app_ctx, void *outputs, float conf_threshold, float nms_threshold, object_detect_result_list *od_results); + +void deinitPostProcess(); +#endif //_RKNN_YOLOV8_DEMO_POSTPROCESS_H_ \ No newline at end of file diff --git a/Cpp_example/D15_yolov8/yolov8.cc b/Cpp_example/D15_yolov8/yolov8.cc new file mode 100755 index 0000000000000000000000000000000000000000..fdb6258723d01f54640143c45eb776ea5bae799c --- /dev/null +++ b/Cpp_example/D15_yolov8/yolov8.cc @@ -0,0 +1,273 @@ +#include +#include +#include +#include +#include +#include +#include "yolov8.h" + +static void dump_tensor_attr(rknn_tensor_attr *attr) +{ + printf(" index=%d, name=%s, n_dims=%d, dims=[%d, %d, %d, %d], n_elems=%d, size=%d, fmt=%s, type=%s, qnt_type=%s, " + "zp=%d, scale=%f\n", + attr->index, attr->name, attr->n_dims, attr->dims[0], attr->dims[1], attr->dims[2], attr->dims[3], + attr->n_elems, attr->size, get_format_string(attr->fmt), get_type_string(attr->type), + get_qnt_type_string(attr->qnt_type), attr->zp, attr->scale); +} + +int init_yolov8_model(const char *model_path, rknn_app_context_t *app_ctx) +{ + int ret; + int model_len = 0; + char *model; + rknn_context ctx = 0; + + ret = rknn_init(&ctx, (char *)model_path, 0, 0, NULL); + if (ret < 0) + { + printf("rknn_init fail! ret=%d\n", ret); + return -1; + } + + // Get Model Input Output Number + rknn_input_output_num io_num; + ret = rknn_query(ctx, RKNN_QUERY_IN_OUT_NUM, &io_num, sizeof(io_num)); + if (ret != RKNN_SUCC) + { + printf("rknn_query fail! ret=%d\n", ret); + return -1; + } + printf("model input num: %d, output num: %d\n", io_num.n_input, io_num.n_output); + + // Get Model Input Info + printf("input tensors:\n"); + rknn_tensor_attr input_attrs[io_num.n_input]; + memset(input_attrs, 0, sizeof(input_attrs)); + for (int i = 0; i < io_num.n_input; i++) + { + input_attrs[i].index = i; + ret = rknn_query(ctx, RKNN_QUERY_NATIVE_INPUT_ATTR, &(input_attrs[i]), sizeof(rknn_tensor_attr)); + if (ret != RKNN_SUCC) + { + printf("rknn_query fail! ret=%d\n", ret); + return -1; + } + dump_tensor_attr(&(input_attrs[i])); + } + + // Get Model Output Info + printf("output tensors:\n"); + rknn_tensor_attr output_attrs[io_num.n_output]; + memset(output_attrs, 0, sizeof(output_attrs)); + for (int i = 0; i < io_num.n_output; i++) + { + output_attrs[i].index = i; + //When using the zero-copy API interface, query the native output tensor attribute + ret = rknn_query(ctx, RKNN_QUERY_NATIVE_NHWC_OUTPUT_ATTR, &(output_attrs[i]), sizeof(rknn_tensor_attr)); + if (ret != RKNN_SUCC) + { + printf("rknn_query fail! ret=%d\n", ret); + return -1; + } + dump_tensor_attr(&(output_attrs[i])); + } + + // default input type is int8 (normalize and quantize need compute in outside) + // if set uint8, will fuse normalize and quantize to npu + input_attrs[0].type = RKNN_TENSOR_UINT8; + // default fmt is NHWC,1106 npu only support NHWC in zero copy mode + input_attrs[0].fmt = RKNN_TENSOR_NHWC; + printf("input_attrs[0].size_with_stride=%d\n", input_attrs[0].size_with_stride); + app_ctx->input_mems[0] = rknn_create_mem(ctx, input_attrs[0].size_with_stride); + + // Set input tensor memory + ret = rknn_set_io_mem(ctx, app_ctx->input_mems[0], &input_attrs[0]); + if (ret < 0) { + printf("input_mems rknn_set_io_mem fail! ret=%d\n", ret); + return -1; + } + + // Set output tensor memory + for (uint32_t i = 0; i < io_num.n_output; ++i) { + app_ctx->output_mems[i] = rknn_create_mem(ctx, output_attrs[i].size_with_stride); + ret = rknn_set_io_mem(ctx, app_ctx->output_mems[i], &output_attrs[i]); + if (ret < 0) { + printf("output_mems rknn_set_io_mem fail! ret=%d\n", ret); + return -1; + } + } + + // Set to context + app_ctx->rknn_ctx = ctx; + + // TODO + if (output_attrs[0].qnt_type == RKNN_TENSOR_QNT_AFFINE_ASYMMETRIC) + { + app_ctx->is_quant = true; + } + else + { + app_ctx->is_quant = false; + } + + app_ctx->io_num = io_num; + app_ctx->input_attrs = (rknn_tensor_attr *)malloc(io_num.n_input * sizeof(rknn_tensor_attr)); + memcpy(app_ctx->input_attrs, input_attrs, io_num.n_input * sizeof(rknn_tensor_attr)); + app_ctx->output_attrs = (rknn_tensor_attr *)malloc(io_num.n_output * sizeof(rknn_tensor_attr)); + memcpy(app_ctx->output_attrs, output_attrs, io_num.n_output * sizeof(rknn_tensor_attr)); + + if (input_attrs[0].fmt == RKNN_TENSOR_NCHW) + { + printf("model is NCHW input fmt\n"); + app_ctx->model_channel = input_attrs[0].dims[1]; + app_ctx->model_height = input_attrs[0].dims[2]; + app_ctx->model_width = input_attrs[0].dims[3]; + } else + { + printf("model is NHWC input fmt\n"); + app_ctx->model_height = input_attrs[0].dims[1]; + app_ctx->model_width = input_attrs[0].dims[2]; + app_ctx->model_channel = input_attrs[0].dims[3]; + } + + printf("model input height=%d, width=%d, channel=%d\n", + app_ctx->model_height, app_ctx->model_width, app_ctx->model_channel); + + return 0; +} + +int release_yolov8_model(rknn_app_context_t *app_ctx) +{ + if (app_ctx->input_attrs != NULL) + { + free(app_ctx->input_attrs); + app_ctx->input_attrs = NULL; + } + if (app_ctx->output_attrs != NULL) + { + free(app_ctx->output_attrs); + app_ctx->output_attrs = NULL; + } + for (int i = 0; i < app_ctx->io_num.n_input; i++) { + if (app_ctx->input_mems[i] != NULL) { + rknn_destroy_mem(app_ctx->rknn_ctx, app_ctx->input_mems[i]); + } + } + for (int i = 0; i < app_ctx->io_num.n_output; i++) { + if (app_ctx->output_mems[i] != NULL) { + rknn_destroy_mem(app_ctx->rknn_ctx, app_ctx->output_mems[i]); + } + } + if (app_ctx->rknn_ctx != 0) + { + rknn_destroy(app_ctx->rknn_ctx); + app_ctx->rknn_ctx = 0; + } + + printf("Release success\n"); + return 0; +} + +int inference_yolov8_model(rknn_app_context_t *app_ctx, object_detect_result_list *od_results) +{ + int ret; + const float nms_threshold = NMS_THRESH; + const float box_conf_threshold = BOX_THRESH; + + + memset(od_results, 0x00, sizeof(*od_results)); + + printf("rknn_run...\n"); + auto rknn_start = std::chrono::steady_clock::now(); + ret = rknn_run(app_ctx->rknn_ctx, nullptr); + auto rknn_end = std::chrono::steady_clock::now(); + double rknn_time = std::chrono::duration(rknn_end - rknn_start).count(); + printf("rknn_run done, ret=%d, cost time: %.4f seconds\n", ret, rknn_time); + if (ret < 0) { + printf("rknn_run fail! ret=%d\n", ret); + return -1; + } + + post_process(app_ctx, app_ctx->output_mems, box_conf_threshold, nms_threshold, od_results); +out: + return ret; +} + +/* +预处理 +图片填充缩放 +*/ +float scale; +extern int model_width; +extern int model_height; + +extern int width; +extern int height; + +extern int leftPadding; +extern int topPadding; +cv::Mat letterbox(cv::Mat input) +{ + float scaleX = (float)model_width / (float)width; + float scaleY = (float)model_height / (float)height; + scale = scaleX < scaleY ? scaleX : scaleY; + + int inputWidth = (int)((float)width * scale); + int inputHeight = (int)((float)height * scale); + + leftPadding = (model_width - inputWidth) / 2; + topPadding = (model_height - inputHeight) / 2; + + cv::Mat inputScale; + cv::resize(input, inputScale, cv::Size(inputWidth, inputHeight), 0, 0, cv::INTER_LINEAR); + cv::Mat letterboxImage(model_width, model_width, CV_8UC3, cv::Scalar(0, 0, 0)); + cv::Rect roi(leftPadding, topPadding, inputWidth, inputHeight); + inputScale.copyTo(letterboxImage(roi)); + + return letterboxImage; +} +/* +将经过 letterbox 处理后的坐标映射回原始图像坐标系 +*/ +void mapCoordinates(int *x, int *y) +{ + int mx = *x - leftPadding; + int my = *y - topPadding; + + *x = (int)((float)mx / scale); + *y = (int)((float)my / scale); +} + +void draw_detections(int count, + object_detect_result *results, + cv::Mat &frame, + void (*mapCoord)(int *, int *)) +{ + char text[128]; + for (int i = 0; i < count; i++) + { + object_detect_result *det = &results[i]; + + int sX = (int)det->box.left; + int sY = (int)det->box.top; + int eX = (int)det->box.right; + int eY = (int)det->box.bottom; + + mapCoord(&sX, &sY); + mapCoord(&eX, &eY); + + cv::rectangle(frame, + cv::Point(sX, sY), + cv::Point(eX, eY), + cv::Scalar(0, 255, 0), 1); + + sprintf(text, "%s %.1f%%", + coco_cls_to_name(det->cls_id), + det->prop * 100); + + cv::putText(frame, text, + cv::Point(sX, sY - 8), + cv::FONT_HERSHEY_SIMPLEX, 1, + cv::Scalar(0, 255, 0), 1); + } +} \ No newline at end of file diff --git a/Cpp_example/D15_yolov8/yolov8.h b/Cpp_example/D15_yolov8/yolov8.h new file mode 100755 index 0000000000000000000000000000000000000000..e516765004787559ff264ebe3987bc254236cc87 --- /dev/null +++ b/Cpp_example/D15_yolov8/yolov8.h @@ -0,0 +1,45 @@ +#ifndef _RKNN_DEMO_YOLOV8_H_ +#define _RKNN_DEMO_YOLOV8_H_ + +#include "rknn_api.h" + +typedef struct { + char *dma_buf_virt_addr; + int dma_buf_fd; + int size; +} rknn_dma_buf; + +typedef struct { + rknn_context rknn_ctx; + rknn_input_output_num io_num; + rknn_tensor_attr* input_attrs; + rknn_tensor_attr* output_attrs; + + rknn_tensor_mem* input_mems[1]; + rknn_tensor_mem* output_mems[9]; + rknn_dma_buf img_dma_buf; + + int model_channel; + int model_width; + int model_height; + bool is_quant; +} rknn_app_context_t; + +#include "postprocess.h" + + +int init_yolov8_model(const char* model_path, rknn_app_context_t* app_ctx); + +int release_yolov8_model(rknn_app_context_t* app_ctx); + +int inference_yolov8_model(rknn_app_context_t* app_ctx, object_detect_result_list* od_results); + +#endif //_RKNN_DEMO_YOLOV8_H_ + +cv::Mat letterbox(cv::Mat input); +void mapCoordinates(int *x, int *y); + +void draw_detections(int count, + object_detect_result *results, + cv::Mat &frame, + void (*mapCoord)(int *, int *)); \ No newline at end of file diff --git "a/Cpp_example/D15_yolov8/\346\250\241\345\236\213\346\226\207\344\273\266\350\275\254\346\215\242.md" "b/Cpp_example/D15_yolov8/\346\250\241\345\236\213\346\226\207\344\273\266\350\275\254\346\215\242.md" new file mode 100755 index 0000000000000000000000000000000000000000..5a799cebe480068ecf7329bc8292b619e3930948 --- /dev/null +++ "b/Cpp_example/D15_yolov8/\346\250\241\345\236\213\346\226\207\344\273\266\350\275\254\346\215\242.md" @@ -0,0 +1,230 @@ +# 模型训练以及转换 + +## 1. 模型训练 + +### 1.1 下载模型训练代码 + +```shell +git clone https://github.com/airockchip/ultralytics_yolov8.git +``` + +### 1.2 准备数据集 + +将自定义数据集转换为yolo格式，下面以Labelme工具标注的数据集为例，运行以下代码进行数据集的转换： + +```python +import os +import json +import shutil +import random +from pathlib import Path + +def convert_labelme_to_yolo(src_dir, dest_dir, yaml_output_path, split_ratio=0.8): + src_dir = Path(src_dir) + dest_dir = Path(dest_dir) + yaml_output_path = Path(yaml_output_path) + + # 输入路径设置 + annotations_dir = src_dir / 'annotations' + images_dir = src_dir / 'images' + + if not annotations_dir.exists(): + print(f"找不到标注文件夹: {annotations_dir}") + return + + # 1. 创建 YOLOv8 所需目录结构 + for split in ['train', 'val']: + (dest_dir / 'images' / split).mkdir(parents=True, exist_ok=True) + (dest_dir / 'labels' / split).mkdir(parents=True, exist_ok=True) + + # 获取所有 json 文件 + json_files = list(annotations_dir.glob('*.json')) + if not json_files: + print("未找到任何 JSON 标注文件！") + return + + # 2. 随机打乱用于划分训练集 (80%) 和验证集 (20%) + random.seed(42) + random.shuffle(json_files) + train_count = int(len(json_files) * split_ratio) + + # 类别映射（目前只有一个类别 fire，这里需要根据你的数据集类别进行修改） + classes = ['fire'] + class_to_id = {cls: i for i, cls in enumerate(classes)} + + print(f"总计找到 {len(json_files)} 个标注文件，开始转换...") + + success_count = 0 + for i, json_path in enumerate(json_files): + split = 'train' if i < train_count else 'val' + + with open(json_path, 'r', encoding='utf-8') as f: + data = json.load(f) + + # 获取图像宽高 + img_w = data.get('imageWidth') + img_h = data.get('imageHeight') + + if not img_w or not img_h: + print(f"警告: {json_path.name} 中没有图像宽高数据，跳过。") + continue + + # 解析图像文件名 (兼容 Windows '\\' 和 Linux '/') + img_name = data.get('imagePath', '').replace('\\', '/').split('/')[-1] + if not img_name: + img_name = json_path.stem + '.png' + + src_img_path = images_dir / img_name + + # 3. 如果图片不存在，尝试其它常见的图片后缀 + if not src_img_path.exists(): + for ext in ['.jpg', '.jpeg', '.JPG', '.PNG']: + alt_img_path = images_dir / (json_path.stem + ext) + if alt_img_path.exists(): + src_img_path = alt_img_path + img_name = alt_img_path.name + break + else: + print(f"警告: 找不到对应的图像文件 {src_img_path}，跳过此样本。") + continue + + # 4. 准备输出 label 内容 + yolo_labels = [] + for shape in data.get('shapes', []): + label = shape.get('label') + if label not in class_to_id: + continue # 如果存在其它意外的类别则忽略 + + class_id = class_to_id[label] + points = shape.get('points') + + # 确保是矩形框并且有两个点 + if shape.get('shape_type') == 'rectangle' and len(points) == 2: + x1, y1 = points[0] + x2, y2 = points[1] + + # 确保坐标极值正确 + x_min, x_max = min(x1, x2), max(x1, x2) + y_min, y_max = min(y1, y2), max(y1, y2) + + # 转换为 YOLO 的中心点及宽高，并且归一化到 0~1 + x_center = ((x_min + x_max) / 2.0) / img_w + y_center = ((y_min + y_max) / 2.0) / img_h + w = (x_max - x_min) / img_w + h = (y_max - y_min) / img_h + + yolo_labels.append(f"{class_id} {x_center:.6f} {y_center:.6f} {w:.6f} {h:.6f}") + + # 5. 写入 YOLO 格式的 txt 标签文件 + dest_txt_path = dest_dir / 'labels' / split / (json_path.stem + '.txt') + with open(dest_txt_path, 'w', encoding='utf-8') as f: + f.write('\n'.join(yolo_labels)) + + # 6. 复制图像文件到 YOLO 目录 + dest_img_path = dest_dir / 'images' / split / img_name + shutil.copy(src_img_path, dest_img_path) + success_count += 1 + + # 7. 生成 YOLOv8 训练所需的 YAML 文件 + yaml_content = f"""path: {dest_dir.absolute()} # 数据集根目录 +train: images/train # 训练集图片目录 (相对于 path) +val: images/val # 验证集图片目录 (相对于 path) + +# 类别定义 +names: + 0: fire +""" + yaml_output_path.parent.mkdir(parents=True, exist_ok=True) + with open(yaml_output_path, 'w', encoding='utf-8') as f: + f.write(yaml_content) + + print(f"\n转换完成！成功处理 {success_count} 个样本。") + print(f"YOLO 格式数据集保存在: {dest_dir}") + print(f"YOLO 配置文件已生成: {yaml_output_path}") + +if __name__ == '__main__': + # 原始数据集路径 + SOURCE_DIR = 'path/to/ultralytics_yolov8/Dataset' + # 转换后YOLO数据集的存放路径 + DEST_DIR = 'path/to/ultralytics_yolov8/Dataset_YOLO' + # 要求的 yaml 配置文件生成路径 + YAML_PATH = 'path/to/ultralytics_yolov8/datasets/data.yaml' + + convert_labelme_to_yolo(SOURCE_DIR, DEST_DIR, YAML_PATH) +``` + +### 1.3 开始训练 + +运行以下代码开始训练yolov8模型 + +```python +from ultralytics import YOLO +from ultralytics.utils import LOGGER +model = YOLO("yolov8n.pt") +# model = YOLO("./ultralytics/cfg/models/v8/yolov8s.yaml") +results = model.train(data="datasets/data.yaml", epochs=300, workers=8,imgsz=640, batch=32, device=0) +``` + +训练好的模型会保存在./runs/detect/train/weights目录下。 + +## 2. 模型转换 + +### 2.1 导出onnx模型 + +由于是在rv1106上运行，对于模型转换请参考如下文档： + +[RKOPT_README.zh-CN.md](https://github.com/airockchip/ultralytics_yolov8/blob/main/RKOPT_README.zh-CN.md) + +- 调整 ./ultralytics/cfg/default.yaml 中 model 文件路径，默认为 yolov8n.pt，若自己训练模型，请调接至对应的路径。支持检测、分割、姿态、旋转框检测模型。 + +在满足 ./requirements.txt 的环境要求后，执行以下语句导出模型 + +```shell +export PYTHONPATH=./ +python ./ultralytics/engine/exporter.py +``` + +执行完毕后，会生成 ONNX 模型. 假如原始模型为 yolov8n.pt，则生成 yolov8n.onnx 模型。 转换后的onnx模型会保存在runs/detect/train/weights目录下。 + +### 2.2 准备校准数据集 + +在量化时，提供 20~100 张有代表性的图片给 RKNN 即可。请在你的终端中运行以下命令，从你的 Dataset_YOLO/images/train 中提取部分图片路径生成校准文件： + +```shell +cd path/to/ultralytics_yolov8 +# 随机选取50张训练集的图片路径，写入 dataset_subset.txt +ls /path/to/ultralytics_yolov8/Dataset_YOLO/images/train/*.png | head -n 50 > dataset_subset.txt +``` + +- 注意：如果你的图片后缀是 .jpg ，请将上面的 *.png 替换为*.jpg + +### 2.3 转RKNN模型 + +#### 2.3.1 克隆rknn_model_zoo + +```shell +git clone https://github.com/airockchip/rknn_model_zoo.git +cd /path/to/rknn_model_zoo/examples/yolov8/python +``` + +#### 2.3.2 修改convert.py代码 + +主要是将 DATASET_PATH 指向刚才我们生成的 dataset_subset.txt ： + +```python +DATASET_PATH = 'path/to/dataset_subset.txt' +``` + +#### 2.3.3 运行convert.py + +在运行转换脚本前请确保你已经配置好了所需要的环境。并且将之前转换好的onnx模型放到以下目录： + +- /path/to/rknn_model_zoo/examples/yolov8/model/ + +执行以下命令开始转换 + +```shell +python convert.py /path/to/xxxx.onnx rv1106 i8 ../model/yolov8.rknn +``` + +执行完毕后，如果没有报错，你就会在 rknn_model_zoo/examples/yolov8/model/ 目录下得到 yolov8.rknn 文件，这个文件就可以直接放到你的 RV1106 板子上进行部署板上运行了！ diff --git a/README.md b/README.md index d2fdd63629d385c643fc94b13af846941db87d30..606e7939861b715e1ac06965db40f9a76fb4979a 100644 --- a/README.md +++ b/README.md @@ -48,7 +48,7 @@ 这一部分教程旨在为你提供一个系统化的学习路径，帮助你快速上手 Lockzhiner Vision Module。通过一系列详细的教程, 你将学会如何烧录镜像、连接设备、搭建开发环境和编写简单的程序。 * [凌智视觉模块烧录镜像指南](./docs/introductory_tutorial/burn_image.md) -* [凌智视觉模块连接设备指南](./docs/introductory_tutorial/connect_device_using_ssh.md) +* [凌智视觉模块连接设备指南](./docs/introductory_tutorial/connect_device_using_ssh_new.md) * [凌智视觉模块WiFi配置指南](./docs/introductory_tutorial/wifi_config.md) * [凌智视觉模块 Python 开发环境搭建指南](./docs/introductory_tutorial/python_development_environment.md) * [凌智视觉模块 C++ 开发环境搭建指南](./docs/introductory_tutorial/cpp_development_environment.md) @@ -180,6 +180,12 @@ OCR（Optical Character Recognition，光学字符识别）是一种将图像中 * [YOLOv5目标检测](./Cpp_example/D10_yolov5//README.md) +### 👍 YOLOv8目标检测 + +目标检测（Object Detection）是深度学习中计算机视觉领域的重要任务之一，旨在识别图像或视频中所有感兴趣的物体，并准确地定位这些物体的边界框（Bounding Box）。与目标分类不同，目标检测不仅需要预测物体的类别，还需要标注它们在图像中的位置。一般来说，目标检测任务的标注过程比较复杂，适合既需要对目标进行分类，有需要对目标进行定位的场景。该案例使用YOLOv8进行目标检测。 + +* [YOLOv8目标检测](./Cpp_example/D15_yolov8//README.md) + ## 🏀 C++ 开发案例 C++ 开发案例以A、B、C、D进行不同类别进行分类，方便初学者进行使用和二次开发。 @@ -224,6 +230,7 @@ C++ 开发案例以A、B、C、D进行不同类别进行分类，方便初学者 | D12 | 神经网络类 | PPOCRv4-Det | [文字检测](./Cpp_example/D12_ppocrv4_det/README.md) | | D13 | 神经网络类 | target_tracking | [多目标跟踪](./Cpp_example/D13_target_tracking/README.md)| | D14 | 神经网络类 | fatugue_detection | [疲劳检测](./Cpp_example/D14_fatigue_detection/README.md)| +| D15 | 神经网络类 | YOLOv8 | [YOLOv8目标检测](./Cpp_example/D15_yolov8/README.md)| | E01 | 使用示例类 | Test_find_Laser | [激光跟踪](./Cpp_example/E01_find_Laser/README.md)| | E02 | 使用示例类 | Test_find_number | [数字识别](./Cpp_example/E02_find_number/README.md)| diff --git a/docs/introductory_tutorial/connect_device_using_ssh_new.md b/docs/introductory_tutorial/connect_device_using_ssh_new.md new file mode 100644 index 0000000000000000000000000000000000000000..eeaee16096a577420b143500b52c0b5f6814d971 --- /dev/null +++ b/docs/introductory_tutorial/connect_device_using_ssh_new.md @@ -0,0 +1,74 @@ +

凌智视觉模块连接设备指南

+ +发布版本：V0.0.6 + +日期：2026-1-22 + +文件密级：□绝密 □秘密 □内部资料 ■公开 + +--- + +**免责声明** + +本文档按**现状**提供，福州凌睿智捷电子有限公司（以下简称**本公司**）不对本文档中的任何陈述、信息和内容的准确性、可靠性、完整性、适销性、适用性及非侵权性提供任何明示或暗示的声明或保证。本文档仅作为使用指导的参考。 + +由于产品版本升级或其他原因，本文档可能在未经任何通知的情况下不定期更新或修改。 + +**读者对象** + +本教程适用于以下工程师： + +- 技术支持工程师 +- 软件开发工程师 + +**修订记录** + +| **日期** | **版本** | **作者** | **修改说明** | +| :--------- | -------- | -------- | ------------ | +| 2026/1/22 | 0.0.6 | 李铸成 | 最新版本 | + +## 1 简介 + +SSH 是一种用于在不安全网络上安全地访问和传输数据的协议，Lockzhiner Vision Module 使用 SSH 来让用户和设备进行通信。在本章节中，你将学会如何使用 SSH 连接设备并在屏幕上输出 Hello World。 + +## 2 下载并安装 electerm (可选) + +[electerm](https://github.com/electerm/electerm) 是一款跨平台的 (linux, mac, win) 开源终端客户端，支持 ssh/sftp 等多种通信方式。我们默认使用 electerm 进行 SSH 和 FTP 通信，当然你也可以自由的选择其他的客户端来完成通信。 + +在开始安装前，请往 [electerm 百度网盘(提取码 772f)](https://sourceforge.net/projects/electerm.mirror/files/) 来下载 electerm。 + +解压你下载的压缩包，打开你 electerm 安装包，一直点击下一步即可完成安装。 + +![](images/connect_device_using_ssh/electerm_install.png) + +## 3 使用 SSH 连接设备 + +为了方便大家使用，Lockzhiner Vision Module 在开机时默认启动 SSH 服务器并虚拟化一个网口。Lockzhiner Vision Module 的 SSH 详细信息如下： + +``` +登录账号：root +登录密码：lzdz +静态IP地址：10.1.1.144 +``` + +打开 electerm 或者你本地的 SSH 编辑器，打开网络配置，将 IP、用户名、密码分别进行以下配置 + +![](images/connect_device_using_ssh/ssh_config.png) + +点击保存并连接来连接到设备 + +![](images/connect_device_using_ssh/connect_to_device.png) + +屏幕上将出现以下界面 + +![](images/connect_device_using_ssh/ssh_success.png) + +## 4 在屏幕上打印 Hello World + +我们使用 Linux 命令行，输入以下命令在屏幕上打印 Hello World + +```bash +echo "Hello World" +``` + +![](images/connect_device_using_ssh/ssh_hello_world.png) \ No newline at end of file diff --git a/docs/introductory_tutorial/python_development_environment.md b/docs/introductory_tutorial/python_development_environment.md index 7465b1d0958f1360b1fba61fae3599a1a829be8c..4c379e0dccfbb9bcf5fd7ad279092c151e03c7e2 100644 --- a/docs/introductory_tutorial/python_development_environment.md +++ b/docs/introductory_tutorial/python_development_environment.md @@ -33,7 +33,7 @@ Lockzhiner Vision Module 的 Python 开发不需要像 C++ 一样的交叉编译 ## 2 下载/更新 LockzhinerVisionModule SDK -点击 [Lockzhiner Vision Module SDK 下载链接](https://gitee.com/LockzhinerAI/LockzhinerVisionModule/releases/download/v0.0.5/lockzhiner_vision_module_sdk.zip) 下载 Lockzhiner Vision Module SDK。解压到本地后，请使用解压软件解压 SDK，一般我们推荐使用 Bandzip。 +点击 [Lockzhiner Vision Module SDK 下载链接](https://gitee.com/LockzhinerAI/LockzhinerVisionModule/releases/download/v0.0.6/lockzhiner_vision_module_sdk.zip) 下载 Lockzhiner Vision Module SDK。解压到本地后，请使用解压软件解压 SDK，一般我们推荐使用 Bandzip。 ![](images/development_environment/python_update_lockzhiner_vision_module_0.png) diff --git a/example/vision/detetcion/python/README.md b/example/vision/detetcion/python/README.md index dd0551cddec086ddca323ac76044814b6a518c3b..c0cd9d131812df967476f1f013748058f07a582f 100644 --- a/example/vision/detetcion/python/README.md +++ b/example/vision/detetcion/python/README.md @@ -158,7 +158,7 @@ class PaddleDet: nms_threshold (float): 目标检测 NMS 阈值，默认为 0.3 """ - self.model.initialize(score_threshold, nms_threshold) + self.model.set_threshold(score_threshold, nms_threshold) def predict(self, input_mat): """ diff --git a/example/vision/detetcion/python/test_detection.py b/example/vision/detetcion/python/test_detection.py index 633e6eb2b3f55923a2be74f9ba61f40b316a1115..6ee2b9215273747d8f4741cb272737d02b14e386 100644 --- a/example/vision/detetcion/python/test_detection.py +++ b/example/vision/detetcion/python/test_detection.py @@ -14,6 +14,9 @@ if __name__ == "__main__": edit.start_and_accept_connection() model = PaddleDet() + + # 设置阈值 + model.set_threshold(0.8, 0.45) if model.initialize(args[1]) is False: print("Failed to initialize PaddleDet") exit(1)