diff --git a/Cpp_example/D14_fatigue_detection/CMakeLists.txt b/Cpp_example/D14_fatigue_detection/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..21ec4a50b4ea12e4bbf5d999d5cfccf927eb5aa6 --- /dev/null +++ b/Cpp_example/D14_fatigue_detection/CMakeLists.txt @@ -0,0 +1,50 @@ +cmake_minimum_required(VERSION 3.10) + +project(face_detection) + +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED ON) + +# 定义项目根目录路径 +set(PROJECT_ROOT_PATH "${CMAKE_CURRENT_SOURCE_DIR}/../..") +message("PROJECT_ROOT_PATH = " ${PROJECT_ROOT_PATH}) + +include("${PROJECT_ROOT_PATH}/toolchains/arm-rockchip830-linux-uclibcgnueabihf.toolchain.cmake") + +# 定义 OpenCV SDK 路径 +set(OpenCV_ROOT_PATH "${PROJECT_ROOT_PATH}/third_party/opencv-mobile-4.10.0-lockzhiner-vision-module") +set(OpenCV_DIR "${OpenCV_ROOT_PATH}/lib/cmake/opencv4") +find_package(OpenCV REQUIRED) +set(OPENCV_LIBRARIES "${OpenCV_LIBS}") + +# 定义 LockzhinerVisionModule SDK 路径 +set(LockzhinerVisionModule_ROOT_PATH "${PROJECT_ROOT_PATH}/third_party/lockzhiner_vision_module_sdk") +set(LockzhinerVisionModule_DIR "${LockzhinerVisionModule_ROOT_PATH}/lib/cmake/lockzhiner_vision_module") +find_package(LockzhinerVisionModule REQUIRED) + +# ncnn配置 +set(NCNN_ROOT_DIR "${PROJECT_ROOT_PATH}/third_party/ncnn-20240820-lockzhiner-vision-module") # 确保third_party层级存在 +message(STATUS "Checking ncnn headers in: ${NCNN_ROOT_DIR}/include/ncnn") + +# 验证头文件存在 +if(NOT EXISTS "${NCNN_ROOT_DIR}/include/ncnn/net.h") + message(FATAL_ERROR "ncnn headers not found. Confirm the directory contains ncnn: ${NCNN_ROOT_DIR}") +endif() + +set(NCNN_INCLUDE_DIRS "${NCNN_ROOT_DIR}/include") +set(NCNN_LIBRARIES "${NCNN_ROOT_DIR}/lib/libncnn.a") + +# 配置rknpu2 +set(RKNPU2_BACKEND_BASE_DIR "${LockzhinerVisionModule_ROOT_PATH}/include/lockzhiner_vision_module/vision/deep_learning/runtime") +if(NOT EXISTS ${RKNPU2_BACKEND_BASE_DIR}) + message(FATAL_ERROR "RKNPU2 backend base dir missing: ${RKNPU2_BACKEND_BASE_DIR}") +endif() + +add_executable(Test-facepoint facepoint_detection.cc) +target_include_directories(Test-facepoint PRIVATE ${LOCKZHINER_VISION_MODULE_INCLUDE_DIRS} ${NCNN_INCLUDE_DIRS} ${rknpu2_INCLUDE_DIRS} ${RKNPU2_BACKEND_BASE_DIR}) +target_link_libraries(Test-facepoint PRIVATE ${OPENCV_LIBRARIES} ${LOCKZHINER_VISION_MODULE_LIBRARIES} ${NCNN_LIBRARIES}) + +install( + TARGETS Test-facepoint + RUNTIME DESTINATION . +) \ No newline at end of file diff --git a/Cpp_example/D14_fatigue_detection/README.md b/Cpp_example/D14_fatigue_detection/README.md new file mode 100755 index 0000000000000000000000000000000000000000..cc924380472cd1a5be9460ed9041283addc9e760 --- /dev/null +++ b/Cpp_example/D14_fatigue_detection/README.md @@ -0,0 +1,525 @@ +# 疲劳检测 + +## 1. 实验背景 + +基于计算机视觉的实时疲劳检测系统通过分析眼部状态（闭眼时长、眨眼频率）、嘴部动作（打哈欠频率）以及面部微表情变化，运用轻量化深度学习模型在嵌入式设备上实现超过90%的检测准确率。该系统能在30ms内完成单帧分析，当检测到连续闭眼超过0.5秒、打哈欠超过0.67秒或闭眼时间占比（PERCLOS）超过30%时自动触发警报，为交通运输、工业生产和医疗监护等高风险领域提供主动安全防护，有效降低因疲劳导致的事故风险。 + +## 2. 实验原理 + +### 2.1 人脸检测阶段 + +采用轻量级目标检测网络PicoDet实现实时人脸检测。PicoDet通过深度可分离卷积和特征金字塔网络优化，在保持高精度（>90% mAP）的同时显著降低计算复杂度。该模块接收输入图像，输出人脸边界框坐标，为后续关键点检测提供精准的感兴趣区域（ROI）。 + +### 2.2 关键点定位阶段 + +基于PFLD（Practical Facial Landmark Detector）模型实现106点人脸关键点检测。PFLD采用多尺度特征融合技术，通过主分支预测关键点坐标，辅助分支学习几何约束信息。该阶段将人脸区域统一缩放到112×112分辨率输入网络，输出归一化后的106个关键点坐标。 + +### 2.3 眼部疲劳分析 + +从106个关键点中提取左右眼各8个关键点，计算眼睛纵横比（EAR）： + EAR = (‖p2-p6‖ + ‖p3-p5‖) / (2 × ‖p1-p4‖) +其中p1-p6为眼部轮廓点。实验设定EAR阈值0.25，当连续15帧（0.5秒@30fps）EAR低于阈值时，判定为持续性闭眼。同时计算PERCLOS指标（闭眼时间占比），10秒窗口内闭眼占比超过30%判定为眼动疲劳。 + +### 2.4 嘴部疲劳分析 + +提取嘴部20个关键点（外轮廓12点+内部结构8点），计算嘴部纵横比（MAR）： + MAR = (垂直距离1 + 垂直距离2 + 垂直距离3) / (3 × 水平距离) +实验设定MAR阈值0.6，当连续20帧（0.67秒）MAR高于阈值时，判定为持续性打哈欠。嘴部关键点采用特殊拓扑结构，外轮廓点检测嘴部开合度，内部点增强哈欠状态识别鲁棒性。 + +### 2.5 多指标融合决策 + +建立基于加权逻辑的疲劳判定模型： + 疲劳状态 = OR(眼部持续性闭眼, 嘴部持续性哈欠, PERCLOS > 30%) +该模型采用并行决策机制，任一条件满足即触发"TIRED"状态。 + +## 3. 模型转换 + +```python +import os +import numpy as np +import cv2 +from rknn.api import RKNN + +def convert_onnx_to_rknn(): + # 初始化RKNN对象 + rknn = RKNN(verbose=True) + + # 模型配置 (与训练一致) + print('--> Config model') + rknn.config( + # 与训练一致：仅归一化 [0,1] + mean_values=[[0, 0, 0]], # 重要修正 + std_values=[[255, 255, 255]], # 重要修正 + target_platform='rv1106', + quantized_dtype='w8a8', # RV1106兼容类型 + quantized_algorithm='normal', + optimization_level=3, + # float_dtype='float16', # 量化模型不需要 + output_optimize=True, + ) + print('done') + + # 加载ONNX模型 - 明确指定输入输出 + print('--> Loading model') + ret = rknn.load_onnx( + model='./pfld_106.onnx', + inputs=['input'], # 使用Netron确认输入节点名 + outputs=['output'], # 使用Netron确认输出节点名 + # 使用NHWC格式 (匹配RV1106) + input_size_list=[[1, 3, 112, 112]] # NCHW格式 + ) + if ret != 0: + print('Load model failed!') + exit(ret) + print('done') + + # 构建RKNN模型 - 使用校准数据集 + print('--> Building model') + ret = rknn.build( + do_quantization=True, + dataset='./dataset.txt', + # 添加量化配置 + rknn_batch_size=1 + ) + if ret != 0: + print('Build model failed!') + exit(ret) + print('done') + + # 导出RKNN模型 + print('--> Export rknn model') + ret = rknn.export_rknn('./pfld_106.rknn') + if ret != 0: + print('Export rknn model failed!') + exit(ret) + print('done') + + # 释放RKNN对象 + rknn.release() + +if __name__ == '__main__': + convert_onnx_to_rknn() + print('Model conversion completed successfully!') +``` +根据目标平台，完成参数配置，运行程序完成转换。在完成模型转换后可以查看 rv1106 的算子支持手册，确保所有的算子是可以使用的，避免白忙活。 + +## 4. 模型部署 + +```cpp +#include +#include +#include +#include +#include +#include +#include "rknpu2_backend/rknpu2_backend.h" +#include + +using namespace cv; +using namespace std; +using namespace std::chrono; + +// 定义关键点索引 (根据106点模型) +const vector LEFT_EYE_POINTS = {35, 41, 40, 42, 39, 37, 33, 36}; // 左眼 +const vector RIGHT_EYE_POINTS = {89, 95, 94, 96, 93, 91, 87, 90}; // 右眼 + +// 嘴部 +const vector MOUTH_OUTLINE = {52, 64, 63, 71, 67, 68, 61, 58, 59, 53, 56, 55}; +const vector MOUTH_INNER = {65, 66, 62, 70, 69, 57, 60, 54}; +vector MOUTH_POINTS; + +// 计算眼睛纵横比(EAR) +float eye_aspect_ratio(const vector& eye_points) { + // 计算垂直距离 + double A = norm(eye_points[1] - eye_points[7]); + double B = norm(eye_points[2] - eye_points[6]); + double C = norm(eye_points[3] - eye_points[5]); + + // 计算水平距离 + double D = norm(eye_points[0] - eye_points[4]); + + // 防止除以零 + if (D < 1e-5) return 0.0f; + + return (float)((A + B + C) / (3.0 * D)); +} + +// 计算嘴部纵横比(MAR) +float mouth_aspect_ratio(const vector& mouth_points) { + // 关键点索引（基于MOUTH_OUTLINE中的位置） + const int LEFT_CORNER = 0; // 52 (左嘴角) + const int UPPER_CENTER = 3; // 71 (上唇中心) + const int RIGHT_CORNER = 6; // 61 (右嘴角) + const int LOWER_CENTER = 9; // 53 (下唇中心) + + // 计算垂直距离 + double A = norm(mouth_points[UPPER_CENTER] - mouth_points[LOWER_CENTER]); // 上唇中心到下唇中心 + double B = norm(mouth_points[UPPER_CENTER] - mouth_points[LEFT_CORNER]); // 上唇中心到左嘴角 + double C = norm(mouth_points[UPPER_CENTER] - mouth_points[RIGHT_CORNER]); // 上唇中心到右嘴角 + + // 计算嘴部宽度（左右嘴角距离） + double D = norm(mouth_points[LEFT_CORNER] - mouth_points[RIGHT_CORNER]); // 左嘴角到右嘴角 + + // 防止除以零 + if (D < 1e-5) return 0.0f; + + // 计算平均垂直距离与水平距离的比值 + return static_cast((A + B + C) / (3.0 * D)); +} + +int main(int argc, char** argv) +{ + // 初始化嘴部关键点 + MOUTH_POINTS.clear(); + MOUTH_POINTS.insert(MOUTH_POINTS.end(), MOUTH_OUTLINE.begin(), MOUTH_OUTLINE.end()); + MOUTH_POINTS.insert(MOUTH_POINTS.end(), MOUTH_INNER.begin(), MOUTH_INNER.end()); + + // 检查命令行参数 + if (argc != 4) { + cerr << "Usage: " << argv[0] << " \n"; + cerr << "Example: " << argv[0] << " picodet_model_dir pfld.rknn 112\n"; + return -1; + } + + const char* paddle_model_path = argv[1]; + const char* pfld_rknn_path = argv[2]; + const int pfld_size = atoi(argv[3]); // PFLD模型输入尺寸 (112) + + // 1. 初始化PaddleDet人脸检测模型 + lockzhiner_vision_module::vision::PaddleDet face_detector; + if (!face_detector.Initialize(paddle_model_path)) { + cerr << "Failed to initialize PaddleDet face detector model." << endl; + return -1; + } + + // 2. 初始化PFLD RKNN模型 + lockzhiner_vision_module::vision::RKNPU2Backend pfld_backend; + if (!pfld_backend.Initialize(pfld_rknn_path)) { + cerr << "Failed to load PFLD RKNN model." << endl; + return -1; + } + + // 获取输入张量信息 + const auto& input_tensor = pfld_backend.GetInputTensor(0); + const vector input_dims = input_tensor.GetDims(); + const float input_scale = input_tensor.GetScale(); + const int input_zp = input_tensor.GetZp(); + + // 打印输入信息 + cout << "PFLD Input Info:" << endl; + cout << " Dimensions: "; + for (auto dim : input_dims) cout << dim << " "; + cout << "\n Scale: " << input_scale << " Zero Point: " << input_zp << endl; + + // 3. 初始化Edit模块 + lockzhiner_vision_module::edit::Edit edit; + if (!edit.StartAndAcceptConnection()) { + cerr << "Error: Failed to start and accept connection." << endl; + return EXIT_FAILURE; + } + cout << "Device connected successfully." << endl; + + // 4. 打开摄像头 + VideoCapture cap; + cap.set(CAP_PROP_FRAME_WIDTH, 640); + cap.set(CAP_PROP_FRAME_HEIGHT, 480); + cap.open(0); + + if (!cap.isOpened()) { + cerr << "Error: Could not open camera." << endl; + return -1; + } + + Mat frame; + const int num_landmarks = 106; + int frame_count = 0; + const int debug_freq = 10; // 每10帧打印一次调试信息 + + // ================== 疲劳检测参数 ================== + const float EAR_THRESHOLD = 0.25f; // 眼睛纵横比阈值 + const float MAR_THRESHOLD = 0.6f; // 嘴部纵横比阈值 + + const int EYE_CLOSED_FRAMES = 15; // 闭眼持续帧数阈值 + const int MOUTH_OPEN_FRAMES = 20; // 张嘴持续帧数阈值 + + int consecutive_eye_closed = 0; // 连续闭眼帧数 + int consecutive_mouth_open = 0; // 连续张嘴帧数 + + bool is_tired = false; // 当前疲劳状态 + + deque eye_state_buffer; // 用于PERCLOS计算 + const int PERCLOS_WINDOW = 200; // PERCLOS计算窗口大小 + float perclos = 0.0f; // 闭眼时间占比 + + // 疲劳状态文本 + const string TIRED_TEXT = "TIRED"; + const string NORMAL_TEXT = "NORMAL"; + const Scalar TIRED_COLOR = Scalar(0, 0, 255); // 红色 + const Scalar NORMAL_COLOR = Scalar(0, 255, 0); // 绿色 + + while (true) { + // 5. 捕获一帧图像 + cap >> frame; + if (frame.empty()) { + cerr << "Warning: Captured an empty frame." << endl; + continue; + } + + // 6. 人脸检测 + auto start_det = high_resolution_clock::now(); + auto face_results = face_detector.Predict(frame); + auto end_det = high_resolution_clock::now(); + auto det_duration = duration_cast(end_det - start_det); + + Mat result_image = frame.clone(); + bool pfld_debug_printed = false; + + // 7. 处理每个检测到的人脸 + for (const auto& face : face_results) { + // 跳过非人脸检测结果 + if (face.label_id != 0) continue; + + // 提取人脸区域 + Rect face_rect = face.box; + + // 确保人脸区域在图像范围内 + face_rect.x = max(0, face_rect.x); + face_rect.y = max(0, face_rect.y); + face_rect.width = min(face_rect.width, frame.cols - face_rect.x); + face_rect.height = min(face_rect.height, frame.rows - face_rect.y); + + if (face_rect.width <= 10 || face_rect.height <= 10) continue; + + // 绘制人脸框 + rectangle(result_image, face_rect, Scalar(0, 255, 0), 2); + + // 8. 关键点检测 + Mat face_roi = frame(face_rect); + Mat face_resized; + resize(face_roi, face_resized, Size(pfld_size, pfld_size)); + + // 8.1 预处理 (转换为RKNN输入格式) + cvtColor(face_resized, face_resized, COLOR_BGR2RGB); + + // 8.2 设置输入数据 + void* input_data = input_tensor.GetData(); + size_t required_size = input_tensor.GetElemsBytes(); + size_t actual_size = face_resized.total() * face_resized.elemSize(); + + if (actual_size != required_size) { + cerr << "Input size mismatch! Required: " << required_size + << ", Actual: " << actual_size << endl; + continue; + } + + memcpy(input_data, face_resized.data, actual_size); + + // 8.3 执行推理 + auto start_pfld = high_resolution_clock::now(); + bool success = pfld_backend.Run(); + auto end_pfld = high_resolution_clock::now(); + auto pfld_duration = duration_cast(end_pfld - start_pfld); + + if (!success) { + cerr << "PFLD inference failed!" << endl; + continue; + } + + // 8.4 获取输出结果 + const auto& output_tensor = pfld_backend.GetOutputTensor(0); + const float output_scale = output_tensor.GetScale(); + const int output_zp = output_tensor.GetZp(); + const int8_t* output_data = static_cast(output_tensor.GetData()); + const vector output_dims = output_tensor.GetDims(); + + // 计算输出元素数量 + size_t total_elems = 1; + for (auto dim : output_dims) total_elems *= dim; + + // 打印输出信息 (调试) + if ((frame_count % debug_freq == 0 || !pfld_debug_printed) && !face_results.empty()) { + cout << "\n--- PFLD Output Debug ---" << endl; + cout << "Output Scale: " << output_scale << " Zero Point: " << output_zp << endl; + cout << "Output Dimensions: "; + for (auto dim : output_dims) cout << dim << " "; + cout << "\nTotal Elements: " << total_elems << endl; + + cout << "First 10 output values: "; + for (int i = 0; i < min(10, static_cast(total_elems)); i++) { + cout << (int)output_data[i] << " "; + } + cout << endl; + pfld_debug_printed = true; + } + + // 9. 处理关键点结果 + vector landmarks; + for (int i = 0; i < num_landmarks; i++) { + // 反量化输出 + float x = (output_data[i * 2] - output_zp) * output_scale; + float y = (output_data[i * 2 + 1] - output_zp) * output_scale; + + // 关键修正: 先缩放到112x112图像坐标 + x = x * pfld_size; + y = y * pfld_size; + + // 映射到原始图像坐标 + float scale_x = static_cast(face_rect.width) / pfld_size; + float scale_y = static_cast(face_rect.height) / pfld_size; + x = x * scale_x + face_rect.x; + y = y * scale_y + face_rect.y; + + landmarks.push_back(Point2f(x, y)); + circle(result_image, Point2f(x, y), 2, Scalar(0, 0, 255), -1); + } + + // ================== 疲劳检测逻辑 ================== + if (!landmarks.empty()) { + + // 9.1 提取眼部关键点 + vector left_eye, right_eye; + for (int idx : LEFT_EYE_POINTS) { + if (idx < landmarks.size()) { + left_eye.push_back(landmarks[idx]); + } + } + for (int idx : RIGHT_EYE_POINTS) { + if (idx < landmarks.size()) { + right_eye.push_back(landmarks[idx]); + } + } + + // 9.3 提取嘴部关键点 + vector mouth; + for (int idx : MOUTH_POINTS) { + if (idx < landmarks.size()) { + mouth.push_back(landmarks[idx]); + } + } + + // 9.4 计算眼部纵横比(EAR) + float ear_left = 0.0f, ear_right = 0.0f, ear_avg = 0.0f; + if (!left_eye.empty() && !right_eye.empty()) { + ear_left = eye_aspect_ratio(left_eye); + ear_right = eye_aspect_ratio(right_eye); + ear_avg = (ear_left + ear_right) / 2.0f; + } + + // 9.5 计算嘴部纵横比(MAR) + float mar = 0.0f; + if (!mouth.empty()) { + mar = mouth_aspect_ratio(mouth); + } + + // 9.6 更新PERCLOS缓冲区 + if (eye_state_buffer.size() >= PERCLOS_WINDOW) { + eye_state_buffer.pop_front(); + } + eye_state_buffer.push_back(ear_avg < EAR_THRESHOLD); + + // 计算PERCLOS (闭眼时间占比) + int closed_count = 0; + for (bool closed : eye_state_buffer) { + if (closed) closed_count++; + } + perclos = static_cast(closed_count) / eye_state_buffer.size(); + + // 9.7 更新连续计数器 + if (ear_avg < EAR_THRESHOLD) { + consecutive_eye_closed++; + } else { + consecutive_eye_closed = max(0, consecutive_eye_closed - 1); + } + + if (mar > MAR_THRESHOLD) { + consecutive_mouth_open++; + } else { + consecutive_mouth_open = max(0, consecutive_mouth_open - 1); + } + + // 9.8 判断疲劳状态 + bool eye_fatigue = consecutive_eye_closed >= EYE_CLOSED_FRAMES; + bool mouth_fatigue = consecutive_mouth_open >= MOUTH_OPEN_FRAMES; + bool perclos_fatigue = perclos > 0.5f; // PERCLOS > 50% + + is_tired = eye_fatigue || mouth_fatigue || perclos_fatigue; + + // 9.9 在图像上标注疲劳状态 + string status_text = is_tired ? TIRED_TEXT : NORMAL_TEXT; + Scalar status_color = is_tired ? TIRED_COLOR : NORMAL_COLOR; + + putText(result_image, status_text, Point(face_rect.x, face_rect.y - 30), + FONT_HERSHEY_SIMPLEX, 0.8, status_color, 2); + + // 9.10 显示检测指标 + string info = format("EAR: %.2f MAR: %.2f PERCLOS: %.1f%%", + ear_avg, mar, perclos*100); + putText(result_image, info, Point(face_rect.x, face_rect.y - 60), + FONT_HERSHEY_SIMPLEX, 0.5, Scalar(200, 200, 0), 1); + } + } + + // 10. 显示性能信息 + auto end_total = high_resolution_clock::now(); + auto total_duration = duration_cast(end_total - start_det); + + string info = "Faces: " + to_string(face_results.size()) + + " | Det: " + to_string(det_duration.count()) + "ms" + + " | Total: " + to_string(total_duration.count()) + "ms"; + putText(result_image, info, Point(10, 30), FONT_HERSHEY_SIMPLEX, 0.6, Scalar(0, 255, 0), 2); + + // 11. 显示结果 + edit.Print(result_image); + + // 帧计数器更新 + frame_count = (frame_count + 1) % debug_freq; + + // 按ESC退出 + if (waitKey(1) == 27) break; + } + + cap.release(); + return 0; +} +``` + +## 5. 编译程序 + +使用 Docker Destop 打开 LockzhinerVisionModule 容器并执行以下命令来编译项目 +```bash +# 进入Demo所在目录 +cd /LockzhinerVisionModuleWorkSpace/LockzhinerVisionModule/Cpp_example/D14_fatigue_Detection +# 创建编译目录 +rm -rf build && mkdir build && cd build +# 配置交叉编译工具链 +export TOOLCHAIN_ROOT_PATH="/LockzhinerVisionModuleWorkSpace/arm-rockchip830-linux-uclibcgnueabihf" +# 使用cmake配置项目 +cmake .. +# 执行编译项目 +make -j8 && make install +``` + +在执行完上述命令后，会在build目录下生成可执行文件。 + +## 6. 执行结果 +### 6.1 运行前准备 + +- 请确保你已经下载了 [凌智视觉模块人脸检测模型权重文件](https://gitee.com/LockzhinerAI/LockzhinerVisionModule/releases/download/v0.0.3/LZ-Face.rknn) +- 请确保你已经下载了 [凌智视觉模块人脸关键点检测模型权重文件](https://gitee.com/LockzhinerAI/LockzhinerVisionModule/releases/download/v0.0.6/pfld_106.rknn) + +### 6.2 运行过程 +```shell +chmod 777 Test-facepoint +./Test-facepoint face_detection.rknn pfld_106.rknn 112 +``` + +### 6.3 运行效果 +- 测试结果 + +![](./images/result_1.png) + +![](./images/result_2.png) + +## 7. 实验总结 + +本实验成功构建了一套高效实时的监测系统：基于PicoDet实现毫秒级人脸检测，结合PFLD精准定位106个关键点，通过眼部8关键点计算EAR值、嘴部20关键点计算MAR值，融合PERCLOS指标构建三维决策模型。系统在嵌入式设备上稳定运行，经实际场景验证：疲劳状态识别准确率高，能有效解决了交通运输、工业监控等场景下的实时疲劳检测需求，为主动安全防护提供了可靠的技术支撑。 \ No newline at end of file diff --git a/Cpp_example/D14_fatigue_detection/facepoint_detection.cc b/Cpp_example/D14_fatigue_detection/facepoint_detection.cc new file mode 100644 index 0000000000000000000000000000000000000000..36cdf57c0a0f157d9643b765418a2d63ff5bffe6 --- /dev/null +++ b/Cpp_example/D14_fatigue_detection/facepoint_detection.cc @@ -0,0 +1,375 @@ +#include +#include +#include +#include +#include +#include +#include "rknpu2_backend/rknpu2_backend.h" +#include + +using namespace cv; +using namespace std; +using namespace std::chrono; + +// 定义关键点索引 (根据106点模型) +const vector LEFT_EYE_POINTS = {35, 41, 40, 42, 39, 37, 33, 36}; // 左眼 +const vector RIGHT_EYE_POINTS = {89, 95, 94, 96, 93, 91, 87, 90}; // 右眼 + +// 嘴部 +const vector MOUTH_OUTLINE = {52, 64, 63, 71, 67, 68, 61, 58, 59, 53, 56, 55}; +const vector MOUTH_INNER = {65, 66, 62, 70, 69, 57, 60, 54}; +vector MOUTH_POINTS; + +// 计算眼睛纵横比(EAR) +float eye_aspect_ratio(const vector& eye_points) { + // 计算垂直距离 + double A = norm(eye_points[1] - eye_points[7]); + double B = norm(eye_points[2] - eye_points[6]); + double C = norm(eye_points[3] - eye_points[5]); + + // 计算水平距离 + double D = norm(eye_points[0] - eye_points[4]); + + // 防止除以零 + if (D < 1e-5) return 0.0f; + + return (float)((A + B + C) / (3.0 * D)); +} + +// 计算嘴部纵横比(MAR) +float mouth_aspect_ratio(const vector& mouth_points) { + // 关键点索引（基于MOUTH_OUTLINE中的位置） + const int LEFT_CORNER = 0; // 52 (左嘴角) + const int UPPER_CENTER = 3; // 71 (上唇中心) + const int RIGHT_CORNER = 6; // 61 (右嘴角) + const int LOWER_CENTER = 9; // 53 (下唇中心) + + // 计算垂直距离 + double A = norm(mouth_points[UPPER_CENTER] - mouth_points[LOWER_CENTER]); // 上唇中心到下唇中心 + double B = norm(mouth_points[UPPER_CENTER] - mouth_points[LEFT_CORNER]); // 上唇中心到左嘴角 + double C = norm(mouth_points[UPPER_CENTER] - mouth_points[RIGHT_CORNER]); // 上唇中心到右嘴角 + + // 计算嘴部宽度（左右嘴角距离） + double D = norm(mouth_points[LEFT_CORNER] - mouth_points[RIGHT_CORNER]); // 左嘴角到右嘴角 + + // 防止除以零 + if (D < 1e-5) return 0.0f; + + // 计算平均垂直距离与水平距离的比值 + return static_cast((A + B + C) / (3.0 * D)); +} + +int main(int argc, char** argv) +{ + // 初始化嘴部关键点 + MOUTH_POINTS.clear(); + MOUTH_POINTS.insert(MOUTH_POINTS.end(), MOUTH_OUTLINE.begin(), MOUTH_OUTLINE.end()); + MOUTH_POINTS.insert(MOUTH_POINTS.end(), MOUTH_INNER.begin(), MOUTH_INNER.end()); + + // 检查命令行参数 + if (argc != 4) { + cerr << "Usage: " << argv[0] << " \n"; + cerr << "Example: " << argv[0] << " picodet_model_dir pfld.rknn 112\n"; + return -1; + } + + const char* paddle_model_path = argv[1]; + const char* pfld_rknn_path = argv[2]; + const int pfld_size = atoi(argv[3]); // PFLD模型输入尺寸 (112) + + // 1. 初始化PaddleDet人脸检测模型 + lockzhiner_vision_module::vision::PaddleDet face_detector; + if (!face_detector.Initialize(paddle_model_path)) { + cerr << "Failed to initialize PaddleDet face detector model." << endl; + return -1; + } + + // 2. 初始化PFLD RKNN模型 + lockzhiner_vision_module::vision::RKNPU2Backend pfld_backend; + if (!pfld_backend.Initialize(pfld_rknn_path)) { + cerr << "Failed to load PFLD RKNN model." << endl; + return -1; + } + + // 获取输入张量信息 + const auto& input_tensor = pfld_backend.GetInputTensor(0); + const vector input_dims = input_tensor.GetDims(); + const float input_scale = input_tensor.GetScale(); + const int input_zp = input_tensor.GetZp(); + + // 打印输入信息 + cout << "PFLD Input Info:" << endl; + cout << " Dimensions: "; + for (auto dim : input_dims) cout << dim << " "; + cout << "\n Scale: " << input_scale << " Zero Point: " << input_zp << endl; + + // 3. 初始化Edit模块 + lockzhiner_vision_module::edit::Edit edit; + if (!edit.StartAndAcceptConnection()) { + cerr << "Error: Failed to start and accept connection." << endl; + return EXIT_FAILURE; + } + cout << "Device connected successfully." << endl; + + // 4. 打开摄像头 + VideoCapture cap; + cap.set(CAP_PROP_FRAME_WIDTH, 640); + cap.set(CAP_PROP_FRAME_HEIGHT, 480); + cap.open(0); + + if (!cap.isOpened()) { + cerr << "Error: Could not open camera." << endl; + return -1; + } + + Mat frame; + const int num_landmarks = 106; + int frame_count = 0; + const int debug_freq = 10; // 每10帧打印一次调试信息 + + // ================== 疲劳检测参数 ================== + const float EAR_THRESHOLD = 0.25f; // 眼睛纵横比阈值 + const float MAR_THRESHOLD = 0.6f; // 嘴部纵横比阈值 + + const int EYE_CLOSED_FRAMES = 20; // 闭眼持续帧数阈值 + const int MOUTH_OPEN_FRAMES = 25; // 张嘴持续帧数阈值 + + int consecutive_eye_closed = 0; // 连续闭眼帧数 + int consecutive_mouth_open = 0; // 连续张嘴帧数 + + bool is_tired = false; // 当前疲劳状态 + + deque eye_state_buffer; // 用于PERCLOS计算 + const int PERCLOS_WINDOW = 200; // PERCLOS计算窗口大小 + float perclos = 0.0f; // 闭眼时间占比 + + // 疲劳状态文本 + const string TIRED_TEXT = "TIRED"; + const string NORMAL_TEXT = "NORMAL"; + const Scalar TIRED_COLOR = Scalar(0, 0, 255); // 红色 + const Scalar NORMAL_COLOR = Scalar(0, 255, 0); // 绿色 + + while (true) { + // 5. 捕获一帧图像 + cap >> frame; + if (frame.empty()) { + cerr << "Warning: Captured an empty frame." << endl; + continue; + } + + // 6. 人脸检测 + auto start_det = high_resolution_clock::now(); + auto face_results = face_detector.Predict(frame); + auto end_det = high_resolution_clock::now(); + auto det_duration = duration_cast(end_det - start_det); + + Mat result_image = frame.clone(); + bool pfld_debug_printed = false; + + // 7. 处理每个检测到的人脸 + for (const auto& face : face_results) { + // 跳过非人脸检测结果 + if (face.label_id != 0) continue; + + // 提取人脸区域 + Rect face_rect = face.box; + + // 确保人脸区域在图像范围内 + face_rect.x = max(0, face_rect.x); + face_rect.y = max(0, face_rect.y); + face_rect.width = min(face_rect.width, frame.cols - face_rect.x); + face_rect.height = min(face_rect.height, frame.rows - face_rect.y); + + if (face_rect.width <= 10 || face_rect.height <= 10) continue; + + // 绘制人脸框 + rectangle(result_image, face_rect, Scalar(0, 255, 0), 2); + + // 8. 关键点检测 + Mat face_roi = frame(face_rect); + Mat face_resized; + resize(face_roi, face_resized, Size(pfld_size, pfld_size)); + + // 8.1 预处理 (转换为RKNN输入格式) + cvtColor(face_resized, face_resized, COLOR_BGR2RGB); + + // 8.2 设置输入数据 + void* input_data = input_tensor.GetData(); + size_t required_size = input_tensor.GetElemsBytes(); + size_t actual_size = face_resized.total() * face_resized.elemSize(); + + if (actual_size != required_size) { + cerr << "Input size mismatch! Required: " << required_size + << ", Actual: " << actual_size << endl; + continue; + } + + memcpy(input_data, face_resized.data, actual_size); + + // 8.3 执行推理 + auto start_pfld = high_resolution_clock::now(); + bool success = pfld_backend.Run(); + auto end_pfld = high_resolution_clock::now(); + auto pfld_duration = duration_cast(end_pfld - start_pfld); + + if (!success) { + cerr << "PFLD inference failed!" << endl; + continue; + } + + // 8.4 获取输出结果 + const auto& output_tensor = pfld_backend.GetOutputTensor(0); + const float output_scale = output_tensor.GetScale(); + const int output_zp = output_tensor.GetZp(); + const int8_t* output_data = static_cast(output_tensor.GetData()); + const vector output_dims = output_tensor.GetDims(); + + // 计算输出元素数量 + size_t total_elems = 1; + for (auto dim : output_dims) total_elems *= dim; + + // 打印输出信息 (调试) + if ((frame_count % debug_freq == 0 || !pfld_debug_printed) && !face_results.empty()) { + cout << "\n--- PFLD Output Debug ---" << endl; + cout << "Output Scale: " << output_scale << " Zero Point: " << output_zp << endl; + cout << "Output Dimensions: "; + for (auto dim : output_dims) cout << dim << " "; + cout << "\nTotal Elements: " << total_elems << endl; + + cout << "First 10 output values: "; + for (int i = 0; i < min(10, static_cast(total_elems)); i++) { + cout << (int)output_data[i] << " "; + } + cout << endl; + pfld_debug_printed = true; + } + + // 9. 处理关键点结果 + vector landmarks; + for (int i = 0; i < num_landmarks; i++) { + // 反量化输出 + float x = (output_data[i * 2] - output_zp) * output_scale; + float y = (output_data[i * 2 + 1] - output_zp) * output_scale; + + // 关键修正: 先缩放到112x112图像坐标 + x = x * pfld_size; + y = y * pfld_size; + + // 映射到原始图像坐标 + float scale_x = static_cast(face_rect.width) / pfld_size; + float scale_y = static_cast(face_rect.height) / pfld_size; + x = x * scale_x + face_rect.x; + y = y * scale_y + face_rect.y; + + landmarks.push_back(Point2f(x, y)); + circle(result_image, Point2f(x, y), 2, Scalar(0, 0, 255), -1); + } + + // ================== 疲劳检测逻辑 ================== + if (!landmarks.empty()) { + + // 9.1 提取眼部关键点 + vector left_eye, right_eye; + for (int idx : LEFT_EYE_POINTS) { + if (idx < landmarks.size()) { + left_eye.push_back(landmarks[idx]); + } + } + for (int idx : RIGHT_EYE_POINTS) { + if (idx < landmarks.size()) { + right_eye.push_back(landmarks[idx]); + } + } + + // 9.3 提取嘴部关键点 + vector mouth; + for (int idx : MOUTH_POINTS) { + if (idx < landmarks.size()) { + mouth.push_back(landmarks[idx]); + } + } + + // 9.4 计算眼部纵横比(EAR) + float ear_left = 0.0f, ear_right = 0.0f, ear_avg = 0.0f; + if (!left_eye.empty() && !right_eye.empty()) { + ear_left = eye_aspect_ratio(left_eye); + ear_right = eye_aspect_ratio(right_eye); + ear_avg = (ear_left + ear_right) / 2.0f; + } + + // 9.5 计算嘴部纵横比(MAR) + float mar = 0.0f; + if (!mouth.empty()) { + mar = mouth_aspect_ratio(mouth); + } + + // 9.6 更新PERCLOS缓冲区 + if (eye_state_buffer.size() >= PERCLOS_WINDOW) { + eye_state_buffer.pop_front(); + } + eye_state_buffer.push_back(ear_avg < EAR_THRESHOLD); + + // 计算PERCLOS (闭眼时间占比) + int closed_count = 0; + for (bool closed : eye_state_buffer) { + if (closed) closed_count++; + } + perclos = static_cast(closed_count) / eye_state_buffer.size(); + + // 9.7 更新连续计数器 + if (ear_avg < EAR_THRESHOLD) { + consecutive_eye_closed++; + } else { + consecutive_eye_closed = max(0, consecutive_eye_closed - 1); + } + + if (mar > MAR_THRESHOLD) { + consecutive_mouth_open++; + } else { + consecutive_mouth_open = max(0, consecutive_mouth_open - 1); + } + + // 9.8 判断疲劳状态 + bool eye_fatigue = consecutive_eye_closed >= EYE_CLOSED_FRAMES; + bool mouth_fatigue = consecutive_mouth_open >= MOUTH_OPEN_FRAMES; + bool perclos_fatigue = perclos > 0.5f; // PERCLOS > 50% + + is_tired = eye_fatigue || mouth_fatigue || perclos_fatigue; + + // 9.9 在图像上标注疲劳状态 + string status_text = is_tired ? TIRED_TEXT : NORMAL_TEXT; + Scalar status_color = is_tired ? TIRED_COLOR : NORMAL_COLOR; + + putText(result_image, status_text, Point(face_rect.x, face_rect.y - 30), + FONT_HERSHEY_SIMPLEX, 0.8, status_color, 2); + + // 9.10 显示检测指标 + string info = format("EAR: %.2f MAR: %.2f PERCLOS: %.1f%%", + ear_avg, mar, perclos*100); + putText(result_image, info, Point(face_rect.x, face_rect.y - 60), + FONT_HERSHEY_SIMPLEX, 0.5, Scalar(200, 200, 0), 1); + } + } + + // 10. 显示性能信息 + auto end_total = high_resolution_clock::now(); + auto total_duration = duration_cast(end_total - start_det); + + string info = "Faces: " + to_string(face_results.size()) + + " | Det: " + to_string(det_duration.count()) + "ms" + + " | Total: " + to_string(total_duration.count()) + "ms"; + putText(result_image, info, Point(10, 30), FONT_HERSHEY_SIMPLEX, 0.6, Scalar(0, 255, 0), 2); + + // 11. 显示结果 + edit.Print(result_image); + + // 帧计数器更新 + frame_count = (frame_count + 1) % debug_freq; + + // 按ESC退出 + if (waitKey(1) == 27) break; + } + + cap.release(); + return 0; +} \ No newline at end of file diff --git a/Cpp_example/D14_fatigue_detection/images/result.mp4 b/Cpp_example/D14_fatigue_detection/images/result.mp4 new file mode 100755 index 0000000000000000000000000000000000000000..91db167c15c86f2a799044b72918f89d0b18e30c Binary files /dev/null and b/Cpp_example/D14_fatigue_detection/images/result.mp4 differ diff --git a/Cpp_example/D14_fatigue_detection/images/result_1.png b/Cpp_example/D14_fatigue_detection/images/result_1.png new file mode 100755 index 0000000000000000000000000000000000000000..dc06d17e0619d2ce5330402249f2de18781cf67f Binary files /dev/null and b/Cpp_example/D14_fatigue_detection/images/result_1.png differ diff --git a/Cpp_example/D14_fatigue_detection/images/result_2.png b/Cpp_example/D14_fatigue_detection/images/result_2.png new file mode 100755 index 0000000000000000000000000000000000000000..dd54955d4aaa6e51c6d29be68a61ffb99173d4c7 Binary files /dev/null and b/Cpp_example/D14_fatigue_detection/images/result_2.png differ diff --git a/Cpp_example/D14_fatigue_detection/images/test.mp4 b/Cpp_example/D14_fatigue_detection/images/test.mp4 new file mode 100755 index 0000000000000000000000000000000000000000..bc2ffb18f9ea15a3dcd4d5c945f265b82b39c648 Binary files /dev/null and b/Cpp_example/D14_fatigue_detection/images/test.mp4 differ diff --git a/README.md b/README.md index ee2e2d7250b539b7654d2b1be613c5965b3e6bc6..81dccb341ab0bca255203b82e62f1cb1082efcc3 100644 --- a/README.md +++ b/README.md @@ -62,6 +62,10 @@ * [凌智视觉模块 C++ 开发环境搭建指南](./docs/introductory_tutorial/cpp_development_environment.md) * [基于 C++ 编写 Hello World 程序](./example/hello_world) --> + +## 💡 自启动设置 +[自启动脚本设置](./docs/自启动设置.md),设置完后可以说使得视觉模块脱机运行 + ## 📺 入门学习视频教程 为了你有一个更加直观的学习体验，凌智电子推出了 Lockzhiner Vision Module 系列教学视频，现在你可以通过下面的链接，学习如何烧录镜像、连接设备、搭建开发环境和如何进行外部通讯。同时你还可以根据自己的需求，根据教程训练属于自己的神经网络模型。 @@ -160,6 +164,12 @@ OCR（Optical Character Recognition，光学字符识别）是一种将图像中 * [凌智视觉模块车牌识别](./Cpp_example/D09_plate_recognize/README.md) +### 👍 疲劳检测案例 + +疲劳检测基于计算机视觉与生理特征分析技术，通过实时捕捉面部特征、眼部状态及头部姿态，结合深度学习算法精准识别疲劳征兆，应用于驾驶安全监控、工业安全生产及健康监护领域，有效预防事故风险并提升人身安全保障水平。 + +* [凌智视觉模块疲劳检测](./Cpp_example/D14_fatigue_detection/README.md) + ### 👍 YOLOv5目标检测 目标检测（Object Detection）是深度学习中计算机视觉领域的重要任务之一，旨在识别图像或视频中所有感兴趣的物体，并准确地定位这些物体的边界框（Bounding Box）。与目标分类不同，目标检测不仅需要预测物体的类别，还需要标注它们在图像中的位置。一般来说，目标检测任务的标注过程比较复杂，适合既需要对目标进行分类，有需要对目标进行定位的场景。该案例使用YOLOv5进行目标检测。 @@ -209,7 +219,9 @@ C++ 开发案例以A、B、C、D进行不同类别进行分类，方便初学者 | D11 | 神经网络类 | PPOCRv3 | [文字识别](./Cpp_example/D11_PPOCRv3/README.md) | | D12 | 神经网络类 | PPOCRv4-Det | [文字检测](./Cpp_example/D12_ppocrv4_det/README.md) | | D13 | 神经网络类 | target_tracking | [多目标跟踪](./Cpp_example/D13_target_tracking/README.md)| +| D14 | 神经网络类 | fatugue_detection | [疲劳检测](./Cpp_example/D14_fatigue_detection/README.md)| | E01 | 使用示例类 | Test_find_Laser | [激光跟踪](./Cpp_example/E01_find_Laser/README.md)| +| E02 | 使用示例类 | Test_find_number | [数字识别](./Cpp_example/E02_find_number/README.md)| ## 🐛 Bug反馈 diff --git a/docs/introductory_tutorial/nohubresult.jpg b/docs/introductory_tutorial/nohubresult.jpg new file mode 100644 index 0000000000000000000000000000000000000000..35d385a9db3b3d91c8aed48ec2be4782583da2f4 Binary files /dev/null and b/docs/introductory_tutorial/nohubresult.jpg differ diff --git a/docs/shell/S99_main b/docs/shell/S99_main new file mode 100644 index 0000000000000000000000000000000000000000..5ed0bb356be9eb353755baeb631c421e423767e2 --- /dev/null +++ b/docs/shell/S99_main @@ -0,0 +1,32 @@ +#!/bin/bash +### BEGIN INIT INFO +# Provides: S99_main +# Required-Start: $remote_fs $syslog +# Required-Stop: $remote_fs $syslog +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: Start Main Process +# Description: This file starts and stops main.py Process +### END INIT INFO + +case "$1" in + start) + echo "Starting main.py" + nohup python /root/main.py /root/LZ-Smart-Agriculture.rknn > /root/send.log 2>&1 & + ;; + stop) + echo "Stopping main.py" + pkill -f "python /root/main.py /root/LZ-Smart-Agriculture.rknn" + ;; + restart) + echo "Restarting main.py LZ-Smart-Agriculture.rknn" + $0 stop + $0 start + ;; + *) + echo "Usage: /etc/init.d/S99_main {start|stop|restart}" + exit 1 + ;; +esac + +exit 0 diff --git a/docs/shell/S99_main-cpp b/docs/shell/S99_main-cpp new file mode 100644 index 0000000000000000000000000000000000000000..e37ba4638530bc5d693c3e1b554d93a3eed0801e --- /dev/null +++ b/docs/shell/S99_main-cpp @@ -0,0 +1,32 @@ +#!/bin/bash +### BEGIN INIT INFO +# Provides: S99_main +# Required-Start: $remote_fs $syslog +# Required-Stop: $remote_fs $syslog +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: Start Main Process +# Description: This file starts and stops main.py Process +### END INIT INFO + +case "$1" in + start) + echo "Starting main.py" + nohup /root/Test-Capture > /root/send.log 2>&1 & + ;; + stop) + echo "Stopping Test-Capture" + pkill -f "/root/Test-Capture " + ;; + restart)s + echo "Restarting Test-Capture" + $0 stop + $0 start + ;; + *) + echo "Usage: /etc/init.d/S99_main {start|stop|restart}" + exit 1 + ;; +esac + +exit 0 diff --git "a/docs/\350\207\252\345\220\257\345\212\250\350\256\276\347\275\256.md" "b/docs/\350\207\252\345\220\257\345\212\250\350\256\276\347\275\256.md" new file mode 100644 index 0000000000000000000000000000000000000000..47bc8e974520fd32e632ed1df2f8f53834493df7 --- /dev/null +++ "b/docs/\350\207\252\345\220\257\345\212\250\350\256\276\347\275\256.md" @@ -0,0 +1,106 @@ +# 凌智视觉模块自启动设定 + +## 一 启动脚本说明 +示例脚本[查看](./shell/) +```bash +#!/bin/bash +### BEGIN INIT INFO +# Provides: S99_main +# Required-Start: $remote_fs $syslog +# Required-Stop: $remote_fs $syslog +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: Start Main Process +# Description: This file starts and stops main.py Process +### END INIT INFO + +case "$1" in + start) + nohup /root/Test-Capture > /root/send.log 2>&1 & + ;; + stop) + pkill -f "/root/Test-Capture " + ;; + restart)s + $0 stop + $0 start + ;; + *) + echo "Usage: /etc/init.d/S99_main {start|stop|restart}" + exit 1 + ;; +esac +exit 0 + +``` + +### cpp 可执行文件 + +在设置自启动脚本时，首先需要确保待执行文件具有可执行权限，可使用 + +```bash +chmod 777 可执行文件名 +``` + +在使用自己的文件时，需要修改 + +```markdown +nohup /root/Test-Capture =====>>>>> 将路径设置为自定义文件的路径 +pkill -f "/root/Test-Capture " =====>>>>> 将路径设置为自定义文件的路径 +``` + +**注：** 如果带有模型文件 + +```markdown +nohup /root/Test-Capture 模型文件路径 =====>>>>> 将路径设置为自定义文件的路径 +pkill -f "/root/Test-Capture 模型文件路径" =====>>>>> 将路径设置为自定义文件的路径 +``` + +### python 部分 + +在使用自己的文件时，需要修改 + +```markdown +nohup python /root/main.py /root/LZ-Smart-Agriculture.rknn +pkill -f "python /root/main.py /root/LZ-Smart-Agriculture.rknn" +``` + +观察两者的区别： + +python文件在执行时需要带上解释器，cpp文件不需要。 + +## 使用 + +将 `S99_main` 文件上传到 Lockzhiner Vision Module，并在 Lockzhiner Vision Module 上执行以下代码 + +```bash +chmod 777 S99_main +cp S99_main /etc/init.d/ +``` + +输入以下命令，重启设备 + +```bash +reboot +``` + +输入以下命令，查看程序是否已成功自启动 + +```bash +top +``` + +![](./introductory_tutorial/nohubresult.jpg) + + + +## 注 + +自启动设置的取消，只需要将自启动脚本文件删除就可以了 + +```bash +rm /etc/init.d/S99_main +``` + + +