diff --git a/vendor/.gitkeep b/vendor/.gitkeep deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/vendor/opensource/xvp_osd/include/easy_log.h b/vendor/opensource/xvp_osd/include/easy_log.h new file mode 100755 index 0000000000000000000000000000000000000000..451699b9b804d9e10145f3a13e733dc42773b685 --- /dev/null +++ b/vendor/opensource/xvp_osd/include/easy_log.h @@ -0,0 +1,100 @@ +#ifndef _RO_LOG_H_ +#define _RO_LOG_H_ + +#include +#include +#include + +#ifdef __cplusplus +#if __cplusplus +extern "C"{ +#endif +#endif /* End of #ifdef __cplusplus */ + +#define HL_RED //"\033[1;31m" //亮红 +#define HL_GREEN //"\033[1;32m" //亮绿 +#define HL_YEL //"\033[1;33m" //亮黄 +#define BROWN //"\033[0;33m" //灰 +#define WHITE //"\033[1;37m" //白 +#define NORMAL //"\033[0m" //恢复正常 + +enum LOG_LEVEL +{ + LOG_LEVEL_OFF = 0, + LOG_LEVEL_ERR, + LOG_LEVEL_WARN, + LOG_LEVEL_INFO, + LOG_LEVEL_DBG, + LOG_LEVEL_ALL, +}; + +static const int log_level = LOG_LEVEL_ALL; + +#define LOGE(fmt, ...) \ +do \ +{ \ + time_t timeSec; \ + struct tm timeTm; \ + timeSec = time(NULL); \ + localtime_r(&timeSec, &timeTm); \ + struct timeval tv = {0}; \ + gettimeofday(&tv, NULL); \ + if(log_level >= LOG_LEVEL_ERR) \ + printf(HL_RED "%04d/%02d/%02d %02d:%02d:%02d.%04ld[%s line:%d]:" fmt "\n", \ + 1900 + timeTm.tm_year, 1 + timeTm.tm_mon, timeTm.tm_mday, timeTm.tm_hour, timeTm.tm_min, timeTm.tm_sec, (long int)tv.tv_usec/1000, __FUNCTION__, __LINE__, ##__VA_ARGS__); \ +} \ +while(0) + +#define LOGW(fmt, ...) \ +do \ +{ \ + time_t timeSec; \ + struct tm timeTm; \ + timeSec = time(NULL); \ + localtime_r(&timeSec, &timeTm); \ + struct timeval tv = {0}; \ + gettimeofday(&tv, NULL); \ + if(log_level >= LOG_LEVEL_ERR) \ + printf(HL_YEL "%04d/%02d/%02d %02d:%02d:%02d.%04ld[%s line:%d]:" fmt "\n", \ + 1900 + timeTm.tm_year, 1 + timeTm.tm_mon, timeTm.tm_mday, timeTm.tm_hour, timeTm.tm_min, timeTm.tm_sec, (long int)(tv.tv_usec/1000), __FUNCTION__, __LINE__, ##__VA_ARGS__); \ +} \ +while(0) + +#define LOGI(fmt, ...) \ +do \ +{ \ + time_t timeSec; \ + struct tm timeTm; \ + timeSec = time(NULL); \ + localtime_r(&timeSec, &timeTm); \ + struct timeval tv = {0}; \ + gettimeofday(&tv, NULL); \ + if(log_level >= LOG_LEVEL_ERR) \ + printf(WHITE "%04d/%02d/%02d %02d:%02d:%02d.%04ld[%s line:%d]:" fmt "\n", \ + 1900 + timeTm.tm_year, 1 + timeTm.tm_mon, timeTm.tm_mday, timeTm.tm_hour, timeTm.tm_min, timeTm.tm_sec, (long int)(tv.tv_usec/1000), __FUNCTION__, __LINE__, ##__VA_ARGS__); \ +} \ +while(0) + +#define LOGD(fmt, ...) \ +do \ +{ \ + time_t timeSec; \ + struct tm timeTm; \ + timeSec = time(NULL); \ + localtime_r(&timeSec, &timeTm); \ + struct timeval tv = {0}; \ + gettimeofday(&tv, NULL); \ + if(log_level >= LOG_LEVEL_ERR) \ + printf(HL_GREEN "%04d/%02d/%02d %02d:%02d:%02d.%04ld[%s line:%d]:" fmt "\n", \ + 1900 + timeTm.tm_year, 1 + timeTm.tm_mon, timeTm.tm_mday, timeTm.tm_hour, timeTm.tm_min, timeTm.tm_sec, (long int)(tv.tv_usec/1000), __FUNCTION__, __LINE__, ##__VA_ARGS__); \ +} \ +while(0) + +#ifdef __cplusplus +#if __cplusplus +} +#endif +#endif /* End of #ifdef __cplusplus */ + +#endif /* End of #ifndef */ + diff --git a/vendor/opensource/xvp_osd/include/xvp_common_osd.h b/vendor/opensource/xvp_osd/include/xvp_common_osd.h new file mode 100755 index 0000000000000000000000000000000000000000..29b878f49910eb8235a155073df436e98c4697c2 --- /dev/null +++ b/vendor/opensource/xvp_osd/include/xvp_common_osd.h @@ -0,0 +1,42 @@ +#ifndef _XVP_COMMON_OSD_ +#define _XVP_COMMON_OSD_ + +#include "ot_common_region.h" + +#ifdef __cplusplus +extern "C" { +#endif /* __cplusplus */ + +typedef struct { + int factor; + unsigned int font_color; + unsigned int border_color; + int stride; + int width; + int height; +}xvp_osd_info; + +typedef struct { + ot_rgn_chn_attr chn_attr; + xvp_osd_info osd_info; + td_s32 handle; + ot_mpp_chn chn; +} sample_mpp_osd_info; + +const char *xvp_common_osd_get_version(void); +int xvp_common_osd_init(char *hzk_file); +void xvp_common_osd_exit(void); +void xvp_common_osd_draw_text(unsigned char *bitmap_data, unsigned char *text_string, int offset_x, int offset_y, xvp_osd_info *osd_info); +int xvp_common_osd_draw_poit(unsigned char *bitmap_data, int start_x, int start_y, xvp_osd_info *osd_info); +int xvp_common_osd_draw_line(unsigned char *bitmap_data, int start_x, int start_y, int end_x, int end_y, unsigned char border_width, xvp_osd_info *osd_info); +int xvp_common_osd_draw_rect(unsigned char *bitmap_data, int start_x, int start_y, int end_x, int end_y, unsigned char border_width, xvp_osd_info *osd_info); +int xvp_common_osd_parse_string(char *str, int *max_len, int *line_cnt); +int xvp_common_osd_calculate_canvas_size(char *str, int factor, int *width, int *height); +int xvp_common_osd_string_filter(char *src, char *dst, int dst_len, int max_line_num, int max_line_len); +int xvp_common_osd_utf8_to_gb2312(const unsigned char *in_buf, int in_len, unsigned char *out_buf); + +#ifdef __cplusplus +} +#endif /* __cplusplus */ + +#endif diff --git a/vendor/opensource/xvp_osd/lib/libxvp_osd.a b/vendor/opensource/xvp_osd/lib/libxvp_osd.a new file mode 100755 index 0000000000000000000000000000000000000000..aa2ca940a8574f89dc400312b059c8708ccd6ac1 Binary files /dev/null and b/vendor/opensource/xvp_osd/lib/libxvp_osd.a differ diff --git a/vendor/zsks/README.md b/vendor/zsks/README.md new file mode 100755 index 0000000000000000000000000000000000000000..2298efacde2c78b2d3ad6a17c9121d83be375701 --- /dev/null +++ b/vendor/zsks/README.md @@ -0,0 +1,82 @@ +## 1.项目介绍 + +* 本目录是由中山旷视微电子科技有限责任公司与海思深度合作的生态开源项目，旨在为广大开发者提供便利、高效、好用的第三方软件及工具。助力海思生态更上一个台阶。 +* demo目录是我们基于SS928平台开发的实用案例，包括：基于OpenCV的人脸检测案例、基于HNR的夜间超微光案例、基于YoloV8的水果识别及语音播报案例、基于KCF的目标跟踪案例、基于YoloV8的强逆光下的人脸检测案例，同时我们还基于IVE实现了OpenCV的硬件加速，帮助熟悉OpenCV的开发者，以最小的学习成本，就能够很好的使用海思SS928平台的硬件加速模块。 +* 我们基于SS928平台移植的常见第三方软件，包括：python、numpy、OpenCV、libv4l2、alsa-lib、ffmpeg、libacamera第三方软件，doc目录就是我们提供的相关移植文档和开发文档，旨在帮助开发者快速移植场见的第三方软件。同时我们还对海思SS928的MPP模块进行了python化的封装，让开发者只需要使用几行python代码，就能够很好的完成一个海思案例的开发。 +* patch目录，是我们基于SS928的SDK包，提供的一些补丁代码，方便开发者更好的使用我们的参考案例。 + +## 2.参考案例简介 + +### [2.1.opencv_dnn](./demo/opencv_dnn/README.md) + +* 本案例主要是基于OpenCV，将USB camera采集到的图片送到人脸检测模型进行推理，将得到的结果通过HDMI，显示在外接显示屏上面。 + +### [2.2.hnr_auto](./demo/hnr_auto/README.md) + +* 本案例主要是基于海思的hnr案例，实现夜间超微光的功能，当外接ISO到达一定阈值是，自动切换到hnr模型，使得在黑暗条件下也能清楚看到画面。 + +### [2.3.fruit_identify](./demo/fruit_identify/README.md) + +* 本案例是基于YoloV8实现的一个水果分类的案例，通过USB Camera，将采集到的图片送到水果检测模型中进行推理，当检测到特定水果时，会通过外接显示屏实时显示水果的种类以及执行度，并框住水果的具体位置，并通过耳机播放出此时识别到的水果类别。 + +### [2.4.sample_kcf_track](./demo/sample_kcf_track/README.md) + +* 基于KCF+Track模型，通过USB Camera，将采集到的图片送到KFC+trak模型，实现对目标的实时跟踪效果。 + +### [2.5.face_detection](./demo/face_detection/README.md) + +* 本案例主要使用pytorch框架，基于YoloV8，使用大量的特殊场景下的人脸数据，训练出来的强逆光下的人脸检测模型，通过USB Camera，将采集到的图片送到人脸检测模型中进行推理，当检测到人脸时，会在人脸区域画出矩形，并实时的通过外接HDMI线显示在外接显示屏上。 + +## 3.第三方软件 + +### [3.1.python](./docs/python/README.md) + +* 文档主要包括python源码包下载、依赖软件安装、交叉编译及安装、板端测试等内容。 + +### [3.2.numpy](./docs/numpy/README.md) + +* 文档主要包括numpy源码包下载、交叉编译及安装、板端测试等内容。 +* 注意：在板端使用numpy之前，必须按照3.1中的内容，把python移植好。 + +### [3.3.opencv](./docs/opencv/README.md) + +* 文档主要包括python源码包下载、交叉编译及安装、板端测试(包括python代码和CPP的测试)等内容。 +* 注意：如果想使用python代码调用OpenCV接口，必须按照3.1中的内容，把python移植好。 + +### [3.4.libv4l2](./docs/libv4l2/README.md) + +* 文档主要包括libv4l2源码包下载、依赖软件安装、交叉编译及安装、板端测试等内容。 + +### [3.5.alsa-lib](./docs/alsa-lib/README.md) + +* 文档主要包括alsa-lib源码包下载、依赖软件安装、交叉编译及安装、板端测试等内容。 + +### [3.6.ffmpeg](./docs/ffmpeg/README.md) + +* 文档主要包括ffmpeg源码包下载、依赖软件安装、交叉编译及安装、板端测试，以及ffmpeg硬件加速（调用SS928的硬件编解码模块）等内容。 + +### [3.7.libcamera](./docs/libcamera/README.md) + +* 文档主要包括libcamera源码包下载、依赖软件安装、交叉编译及安装、板端测试等内容。 + + + +## 4.patch介绍 + +* 本补丁主要是基于海思的MPP sample添加v4l2等适配代码，确保USB camera能够正常使用。 +* 如果在a55_linux/mpp/sample/common/sample_comm.h中没有相关代码，说明你还没有合并我们的patch。 + +image-20250918185148122 + +* 打补丁的具体步骤如下，首先把cust_mpp_sample_patch.patch复制到SS928的sample目录下，然后再进入到ss928的sample目录下，执行下面的命令。 + +``` +cd pegasus + +cp vendor/cust/patch/cust_mpp_sample_patch.patch platform/SS928V100/smp/a55_linux/mpp/sample/ + +cd platform/SS928V100/smp/a55_linux/mpp/sample + +patch -p1 < cust_mpp_sample_patch.patch +``` + diff --git a/vendor/zsks/demo/face_detection/Makefile b/vendor/zsks/demo/face_detection/Makefile new file mode 100755 index 0000000000000000000000000000000000000000..5c0d0b328341f5ab30d765ceaea25a4d8476f267 --- /dev/null +++ b/vendor/zsks/demo/face_detection/Makefile @@ -0,0 +1,44 @@ +# 编译所需的公共文件 +include ../mk.param + +# 编译后生成的可执行文件名 +TARGET := main + +# 编译此工程需要包含的头文件 +CFLAGS += -I$(SDK_LINUX_INCLUDE_DIR)/ +CFLAGS += -I$(SDK_SAMPLE_COMMON_DIR)/ +CFLAGS += -I$(SDK_LINUX_INCLUDE_DIR)/svp_npu + +# 编译此工程需要依赖的.c或.cpp文件 +SMP_SRCS := $(SDK_SAMPLE_COMMON_DIR)/sample_comm_sys.c +SMP_SRCS += $(SDK_SAMPLE_COMMON_DIR)/sample_ipc.c +SMP_SRCS += $(SDK_SAMPLE_COMMON_DIR)/sample_comm_isp.c +SMP_SRCS += $(SDK_SAMPLE_COMMON_DIR)/sample_comm_vi.c +SMP_SRCS += $(SDK_SAMPLE_COMMON_DIR)/sample_comm_vo.c +SMP_SRCS += $(SDK_SAMPLE_COMMON_DIR)/sample_comm_mipi_tx.c +SMP_SRCS += $(SDK_SAMPLE_COMMON_DIR)/sample_comm_vpss.c +SMP_SRCS += $(SDK_SAMPLE_COMMON_DIR)/loadbmp.c +SMP_SRCS += $(SDK_SAMPLE_COMMON_DIR)/sample_comm_vdec.c +SMP_SRCS += $(SDK_SAMPLE_COMMON_DIR)/sample_comm_venc.c +SMP_SRCS += $(SDK_SAMPLE_COMMON_DIR)/sdk_init.c +SMP_SRCS += $(SDK_SAMPLE_COMMON_DIR)/sdk_exit.c +SMP_SRCS += $(wildcard $(SVP_COMMON_DIR)/*.c) +SMP_SRCS += $(FACE_DETECTION_SAMPLE_DIR)/face_detection.c + +# 编译此工程需要包含的库和宏 +CFLAGS += -O3 -g -Wall -DUSE_USB_CAMERA +CFLAGS += -fstack-protector-strong + +# 根据实际的类型设置，可以用set_sensor脚本设置 +SENSOR0_TYPE = -DSENSOR0_TYPE=HY_S0603_MIPI_8M_30FPS_12BIT +SENSOR1_TYPE = -DSENSOR1_TYPE=HY_S0603_MIPI_8M_30FPS_12BIT +SENSOR2_TYPE = -DSENSOR1_TYPE=HY_S0603_MIPI_8M_30FPS_12BIT +SENSOR3_TYPE = -DSENSOR1_TYPE=HY_S0603_MIPI_8M_30FPS_12BIT + +CFLAGS += $(SENSOR0_TYPE) +CFLAGS += $(SENSOR1_TYPE) +CFLAGS += $(SENSOR2_TYPE) +CFLAGS += $(SENSOR3_TYPE) +# 编译工程所需的mk文件 + +include ../sdk_linux.mk diff --git a/vendor/zsks/demo/face_detection/README.md b/vendor/zsks/demo/face_detection/README.md new file mode 100755 index 0000000000000000000000000000000000000000..b11d554f2f32b0aa9f4e8ac5e3ae30d20fc6c2cc --- /dev/null +++ b/vendor/zsks/demo/face_detection/README.md @@ -0,0 +1,129 @@ +## 2.5、face_detection 操作指导 + +### 2.5.1、face_detection 程序简介 + +* face_detection sample基于SS928V100平台开发，以EulerPi套件为例，face_detection sample是通过USB Camera，将采集到的图片送到人脸检测模型中进行推理，当检测到人脸时，会在人脸区域画出矩形框，并通过外接HDMI线实时的显示在外接显示屏上。 +* face_detection 案例主要是使用pytorch框架，基于YoloV8网络，使用widerface、FDDB、UFDD等开源数据集，共1.3万多张训练出来在强光、逆光、暗光下的人脸检测模型。 + +### 2.5.2、目录 + +```shell +pegasus/vendor/cust/demo/face_detection +|── data # 模型文件 +|── Makefile # 编译脚本 +└── face_detection.c # face_detection sample业务代码 +``` + +![image-20250919144926220](pic/image-20250919144926220.png) + +* 根据外接显示屏的输出参数进行配置，比如说我的外接显示屏的输出是1080P60，如果smp/a55_linux/mpp/sample/svp/common/sample_common_svp.c ample_common_svp_get_def_vo_cfg函数的intf_sync为OT_VO_OUT_1080P30，则需要把他修改为OT_VO_OUT_1080P60。 + +![image-20251023102053075](pic/image-20251023102053075.png) + +### 2.5.3、编译 + +* **注意：在编译cust的demo之前，请确保你已经按照[patch介绍中的步骤](../../README.md#4patch介绍)把补丁打入海思SDK的sample目录下了**。 + +* 步骤1：先根据自己选择的操作系统，进入到对应的Pegasus所在目录下。 + +* 步骤2：使用Makefile的方式进行单编 + +* 在Ubuntu的命令行终端，分步执行下面的命令，单编 face_detection sample + + ``` + cd pegasus/vendor/cust/demo/face_detection + + make clean && make + ``` + + * 在face_detection/out目录下，生成一个名为 main 的 可执行文件，如下图所示： + + ![image-20251023095337519](pic/image-20251023095337519.png) + +### 2.5.4、拷贝可执行程序和依赖文件至开发板的mnt目录下 + +**方式一：使用SD卡进行资料文件的拷贝** + +* 首先需要自己准备一张Micro sd卡(16G 左右即可)，还得有一个Micro sd卡的读卡器。 + +image-20221114150205685 + +* 步骤1：将编译后生成的可执行文件main和资源文件data，拷贝到SD卡中。 + +![image-20251023100347550](pic/image-20251023100347550.png) + +* 步骤2：可执行文件拷贝成功后，将内存卡插入开发板的SD卡槽中，可通过挂载的方式挂载到板端，可选择SD卡 mount指令进行挂载。 + +image-20250210161601541 + +* 在开发板的终端，执行下面的命令进行SD卡的挂载 + * 如果挂载失败，可以参考[这个issue尝试解决](https://gitee.com/HiSpark/HiSpark_NICU2022/issues/I54932?from=project-issue) + + +```shell +mount -t vfat /dev/mmcblk1p1 /mnt +# 其中/dev/mmcblk1p1需要根据实际块设备号修改 +``` + +* 挂载成功后，如下图所示： + +![image-20251023102549783](pic/image-20251023102549783.png) + +**方式二：使用NFS挂载的方式进行资料文件的拷贝** + +* 首先需要自己准备一根网线 +* 步骤1：参考[博客链接](https://blog.csdn.net/Wu_GuiMing/article/details/115872995?spm=1001.2014.3001.5501)中的内容，进行nfs的环境搭建 +* 步骤2：将编译后生成的可执行文件main和资源文件data，拷贝到Windows的nfs共享路径下 + +![image-20251023100136196](pic/image-20251023100136196.png) + +* 步骤3：在开发板的终端执行下面的命令，将Windows的nfs共享路径挂载至开发板的mnt目录下 + * 注意：这里IP地址请根据你开发板和电脑主机的实际IP地址进行填写 + + +``` +ifconfig eth0 192.168.100.100 + +mount -o nolock,addr=192.168.100.10 -t nfs 192.168.100.10:/d/nfs /mnt +``` + +![image-20251023101210815](pic/image-20251023101210815.png) + +### 2.5.5、硬件连接 + +* 准备一个外接显示器和HDMI数据线，将HDMI的一头接在开发板的HDMI输出口，将HDMI的另外一头接在外接显示器的HDMI输入口。 + +![image-20250213112932380](pic/image-20250213112932380.png) + +* 将USB 摄像头接在EulerPi开发板的USB接口上。 + +image-20250919150630870 + +### 2.5.6、功能验证 + +* 在开发板的终端执行下面的命令，运行可执行文件 + +``` +cd /mnt + +chmod +x main + +./main +``` + +![image-20251023101503216](pic/image-20251023101503216.png) + +* 此时，在HDMI的外接显示屏上即可出现实时码流，如下图所示： + +image-20250210170027454 + +* 如果您看到的现象和下图现象不一致，可以确认一下USB涉嫌头是否插入到开发板的USB口，并且在开发板的/dev目录下能够看到video0 和 video1两个设备节点。如果没有这两个设备节点，请确保镜像烧录正常。 + +![image-20250919151018659](pic/image-20250919151018659.png) + +* 正常情况下，我们会在外接显示屏上看到人脸的区域被框出来，且在比较黑暗或者强逆光条件下，也能比较好的检测到人脸。![image-20250919151605632](pic/image-20250919151605632.png) + +* 敲两下回车即可关闭程序 + +![image-20250919151519310](pic/image-20250919151519310.png) + diff --git a/vendor/zsks/demo/face_detection/data/model/face_detection.om b/vendor/zsks/demo/face_detection/data/model/face_detection.om new file mode 100755 index 0000000000000000000000000000000000000000..4765828d501e8d424ae023e6cedff0bb4cf24a5e Binary files /dev/null and b/vendor/zsks/demo/face_detection/data/model/face_detection.om differ diff --git a/vendor/zsks/demo/face_detection/face_detection.c b/vendor/zsks/demo/face_detection/face_detection.c new file mode 100755 index 0000000000000000000000000000000000000000..3dc48c83984a0fb299d61373ea8f4d66b23ba398 --- /dev/null +++ b/vendor/zsks/demo/face_detection/face_detection.c @@ -0,0 +1,515 @@ +/* + * Copyright (c) 2025 HiSilicon (Shanghai) Technologies Co., Ltd. + * Copyright (c) 2025 Zhongshan Kuangshi Microelectronics Technology Co., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Modifications Author: yaohongtao (yht@cust.edu.cn) +*/ +#include +#include +#include +#include +#include +#include +#include +#include "securec.h" +#include "svp_acl.h" +#include "svp_acl_rt.h" +#include "svp_acl_ext.h" +#include "sdk_module_init.h" +#include "sample_common_svp.h" +#include "sample_common_svp_npu.h" +#include "sample_common_svp_npu_model.h" + +/* 新增v4l2相关头文件 */ +#include // 文件控制 +#include // 错误处理 +#include // POSIX API +#include // IO控制 +#include // 内存映射 +#include // select复用 +#include // v4l2核心 + +#define SAMPLE_SVP_NPU_RESNET50_INPUT_FILE_NUM 1 +#define SAMPLE_SVP_NPU_LSTM_INPUT_FILE_NUM 4 +#define SAMPLE_SVP_NPU_SHAERD_WORK_BUF_NUM 1 +#define SAMPLE_SVP_NPU_YOLO_TYPE_NUM 12 +#define SAMPLE_SVP_NPU_YOLO_THRESHOLD_NUM 1 + +#define SAMPLE_SVP_NPU_PATH_LEN 0x100 +static td_bool g_svp_npu_terminate_signal = TD_FALSE; +static td_s32 g_svp_npu_dev_id = 0; +static sample_svp_npu_task_info g_svp_npu_task[SAMPLE_SVP_NPU_MAX_TASK_NUM] = {0}; +static sample_svp_npu_shared_work_buf g_svp_npu_shared_work_buf[SAMPLE_SVP_NPU_SHAERD_WORK_BUF_NUM] = {0}; + +static sample_svp_npu_threshold g_svp_npu_yolo_threshold[SAMPLE_SVP_NPU_YOLO_TYPE_NUM] = { + {0.9, 0.15, 1.0, 1.0, "rpn_data"}, // yolov8 +}; + +static sample_svp_npu_roi_info g_svp_npu_yolo_roi_info[SAMPLE_SVP_NPU_YOLO_TYPE_NUM] = { + {"output0", "output0_"}, // yolov8 +}; + +static ot_sample_svp_rect_info g_svp_npu_rect_info = {0}; +static td_bool g_svp_npu_thread_stop = TD_FALSE; +static pthread_t g_svp_npu_thread = 0; +static sample_vo_cfg g_svp_npu_vo_cfg = { 0 }; +static pthread_t g_svp_npu_vdec_thread = 0; +static ot_vb_pool_info g_svp_npu_vb_pool_info; +static td_void *g_svp_npu_vb_virt_addr = TD_NULL; + +static ot_sample_svp_media_cfg g_svp_npu_media_cfg = { + .svp_switch = {TD_FALSE, TD_TRUE}, + .pic_type = {PIC_1080P, PIC_CIF}, + .chn_num = OT_SVP_MAX_VPSS_CHN_NUM, +}; + +static sample_vdec_attr g_svp_npu_vdec_cfg = { + .type = OT_PT_MJPEG, + .mode = OT_VDEC_SEND_MODE_FRAME, + .width = FHD_WIDTH, + .height = FHD_HEIGHT, + .sample_vdec_picture.pixel_format = OT_PIXEL_FORMAT_YVU_SEMIPLANAR_420, /* 420:pixel_format */ + .sample_vdec_picture.alpha = 0, /* 0:alpha */ + .display_frame_num = 2, /* 2:display_frame_num */ + .frame_buf_cnt = 5, /* 5:2+2+1 */ +}; + +static vdec_thread_param g_svp_npu_vdec_param = { + .chn_id = 0, + + .v4l2_device = "/dev/video0", // 摄像头设备 + .width = FHD_WIDTH, // 采集分辨率宽 + .height = FHD_HEIGHT, // 采集分辨率高 + .pixel_format = V4L2_PIX_FMT_MJPEG, // MJPEG格式 + + .type = OT_PT_MJPEG, + .stream_mode = OT_VDEC_SEND_MODE_FRAME, + .interval_time = 1000, /* 1000:interval_time */ + .pts_init = 0, + .pts_increase = 0, + .e_thread_ctrl = THREAD_CTRL_START, + .circle_send = TD_TRUE, + .milli_sec = 0, + .min_buf_size = (FHD_WIDTH * FHD_HEIGHT * 3) >> 1, /* 3:chn_size */ + .fps = 30, +}; + +static td_void sample_svp_npu_acl_terminate(td_void) +{ + if (g_svp_npu_terminate_signal == TD_TRUE) { + printf("\033[0;31mprogram termination abnormally!\033[0;39m\n"); + } +} + +/* function : svp npu signal handle */ +td_void sample_svp_npu_acl_handle_sig(td_void) +{ + g_svp_npu_terminate_signal = TD_TRUE; +} + +static td_void sample_svp_npu_acl_deinit(td_void) +{ + svp_acl_error ret; + + ret = svp_acl_rt_reset_device(g_svp_npu_dev_id); + if (ret != SVP_ACL_SUCCESS) { + sample_svp_trace_err("reset device fail\n"); + } + sample_svp_trace_info("end to reset device is %d\n", g_svp_npu_dev_id); + + ret = svp_acl_finalize(); + if (ret != SVP_ACL_SUCCESS) { + sample_svp_trace_err("finalize acl fail\n"); + } + sample_svp_trace_info("end to finalize acl\n"); + (td_void)sample_common_svp_check_sys_exit(); +} + +static td_s32 sample_svp_npu_acl_init(const td_char *acl_config_path, td_bool vi_en) +{ + /* svp acl init */ + svp_acl_rt_run_mode run_mode; + svp_acl_error ret; + td_bool is_mpi_init; + + is_mpi_init = sample_common_svp_check_sys_init(vi_en); + sample_svp_check_exps_return(is_mpi_init != TD_TRUE, TD_FAILURE, SAMPLE_SVP_ERR_LEVEL_ERROR, "mpi init failed!\n"); + + ret = svp_acl_init(acl_config_path); + sample_svp_check_exps_return(ret != SVP_ACL_SUCCESS, TD_FAILURE, SAMPLE_SVP_ERR_LEVEL_ERROR, "acl init failed!\n"); + + sample_svp_trace_info("svp acl init success!\n"); + + /* open device */ + ret = svp_acl_rt_set_device(g_svp_npu_dev_id); + if (ret != SVP_ACL_SUCCESS) { + (td_void)svp_acl_finalize(); + sample_svp_trace_err("svp acl open device %d failed!\n", g_svp_npu_dev_id); + return TD_FAILURE; + } + sample_svp_trace_info("open device %d success!\n", g_svp_npu_dev_id); + + /* get run mode */ + ret = svp_acl_rt_get_run_mode(&run_mode); + if ((ret != SVP_ACL_SUCCESS) || (run_mode != SVP_ACL_DEVICE)) { + (td_void)svp_acl_rt_reset_device(g_svp_npu_dev_id); + (td_void)svp_acl_finalize(); + sample_svp_trace_err("acl get run mode failed!\n"); + return TD_FAILURE; + } + sample_svp_trace_info("get run mode success!\n"); + + return TD_SUCCESS; +} + +static td_s32 sample_svp_npu_acl_dataset_init(td_u32 task_idx) +{ + td_s32 ret = sample_common_svp_npu_create_input(&g_svp_npu_task[task_idx]); + sample_svp_check_exps_return(ret != TD_SUCCESS, TD_FAILURE, SAMPLE_SVP_ERR_LEVEL_ERROR, "create input failed!\n"); + + ret = sample_common_svp_npu_create_output(&g_svp_npu_task[task_idx]); + if (ret != TD_SUCCESS) { + sample_common_svp_npu_destroy_input(&g_svp_npu_task[task_idx]); + sample_svp_trace_err("execute create output fail.\n"); + return TD_FAILURE; + } + return TD_SUCCESS; +} + +static td_void sample_svp_npu_acl_dataset_deinit(td_u32 task_idx) +{ + (td_void)sample_common_svp_npu_destroy_input(&g_svp_npu_task[task_idx]); + (td_void)sample_common_svp_npu_destroy_output(&g_svp_npu_task[task_idx]); +} + +static td_void sample_svp_npu_acl_deinit_task(td_u32 task_num, td_u32 shared_work_buf_idx) +{ + td_u32 task_idx; + + for (task_idx = 0; task_idx < task_num; task_idx++) { + (td_void)sample_common_svp_npu_destroy_work_buf(&g_svp_npu_task[task_idx]); + (td_void)sample_common_svp_npu_destroy_task_buf(&g_svp_npu_task[task_idx]); + (td_void)sample_svp_npu_acl_dataset_deinit(task_idx); + (td_void)memset_s(&g_svp_npu_task[task_idx], sizeof(sample_svp_npu_task_cfg), 0, + sizeof(sample_svp_npu_task_cfg)); + } + if (g_svp_npu_shared_work_buf[shared_work_buf_idx].work_buf_ptr != TD_NULL) { + (td_void)svp_acl_rt_free(g_svp_npu_shared_work_buf[shared_work_buf_idx].work_buf_ptr); + g_svp_npu_shared_work_buf[shared_work_buf_idx].work_buf_ptr = TD_NULL; + g_svp_npu_shared_work_buf[shared_work_buf_idx].work_buf_size = 0; + g_svp_npu_shared_work_buf[shared_work_buf_idx].work_buf_stride = 0; + } +} + +static td_s32 sample_svp_npu_acl_create_shared_work_buf(td_u32 task_num, td_u32 shared_work_buf_idx) +{ + td_u32 task_idx; + td_u32 work_buf_size; + td_u32 work_buf_stride; + td_s32 ret; + + for (task_idx = 0; task_idx < task_num; task_idx++) { + ret = sample_common_svp_npu_get_work_buf_info(&g_svp_npu_task[task_idx], &work_buf_size, &work_buf_stride); + sample_svp_check_exps_return(ret != TD_SUCCESS, ret, SAMPLE_SVP_ERR_LEVEL_ERROR, + "get %u-th task work buf info failed!\n", task_idx); + + if (g_svp_npu_shared_work_buf[shared_work_buf_idx].work_buf_size < work_buf_size) { + g_svp_npu_shared_work_buf[shared_work_buf_idx].work_buf_size = work_buf_size; + g_svp_npu_shared_work_buf[shared_work_buf_idx].work_buf_stride = work_buf_stride; + } + } + ret = svp_acl_rt_malloc_cached(&g_svp_npu_shared_work_buf[shared_work_buf_idx].work_buf_ptr, + g_svp_npu_shared_work_buf[shared_work_buf_idx].work_buf_size, SVP_ACL_MEM_MALLOC_NORMAL_ONLY); + sample_svp_check_exps_return(ret != TD_SUCCESS, TD_FAILURE, SAMPLE_SVP_ERR_LEVEL_ERROR, + "malloc %u-th shared work buf failed!\n", shared_work_buf_idx); + + (td_void)svp_acl_rt_mem_flush(g_svp_npu_shared_work_buf[shared_work_buf_idx].work_buf_ptr, + g_svp_npu_shared_work_buf[shared_work_buf_idx].work_buf_size); + return TD_SUCCESS; +} + +static td_s32 sample_svp_npu_acl_init_task(td_u32 task_num, td_bool is_share_work_buf, + td_u32 shared_work_buf_idx) +{ + td_u32 task_idx; + td_s32 ret; + + if (is_share_work_buf == TD_TRUE) { + ret = sample_svp_npu_acl_create_shared_work_buf(task_num, shared_work_buf_idx); + sample_svp_check_exps_return(ret != TD_SUCCESS, TD_FAILURE, SAMPLE_SVP_ERR_LEVEL_ERROR, + "create shared work buf failed!\n"); + } + + for (task_idx = 0; task_idx < task_num; task_idx++) { + ret = sample_svp_npu_acl_dataset_init(task_idx); + if (ret != TD_SUCCESS) { + goto task_init_end_0; + } + ret = sample_common_svp_npu_create_task_buf(&g_svp_npu_task[task_idx]); + if (ret != TD_SUCCESS) { + sample_svp_trace_err("create task buf failed.\n"); + goto task_init_end_0; + } + if (is_share_work_buf == TD_FALSE) { + ret = sample_common_svp_npu_create_work_buf(&g_svp_npu_task[task_idx]); + } else { + /* if all tasks are on the same stream, work buf can be shared */ + ret = sample_common_svp_npu_share_work_buf(&g_svp_npu_shared_work_buf[shared_work_buf_idx], + &g_svp_npu_task[task_idx]); + } + if (ret != TD_SUCCESS) { + sample_svp_trace_err("create work buf failed.\n"); + goto task_init_end_0; + } + } + return TD_SUCCESS; + +task_init_end_0: + (td_void)sample_svp_npu_acl_deinit_task(task_num, shared_work_buf_idx); + return ret; +} + +static td_s32 sample_svp_npu_acl_frame_proc(const ot_video_frame_info *ext_frame, + const ot_video_frame_info *base_frame, td_void *args) +{ + td_s32 ret; + td_void *virt_addr = TD_NULL; + sample_svp_npu_detection_info *detection_info = TD_NULL; + sample_svp_npu_thread_args *thread_args = (sample_svp_npu_thread_args *)args; + td_u32 size = (td_u32)(ext_frame->video_frame.height * ext_frame->video_frame.stride[0] * + SAMPLE_SVP_NPU_IMG_THREE_CHN / SAMPLE_SVP_NPU_DOUBLE); + + virt_addr = g_svp_npu_vb_virt_addr + + (ext_frame->video_frame.phys_addr[0] - g_svp_npu_vb_pool_info.pool_phy_addr); + ret = sample_common_svp_npu_update_input_data_buffer_info(virt_addr, size, + ext_frame->video_frame.stride[0], 0, &g_svp_npu_task[0]); + sample_svp_check_exps_return(ret != TD_SUCCESS, ret, SAMPLE_SVP_ERR_LEVEL_ERROR, "update data buffer failed!\n"); + + ret = sample_common_svp_npu_model_execute(&g_svp_npu_task[0]); + sample_svp_check_exps_return(ret != TD_SUCCESS, ret, SAMPLE_SVP_ERR_LEVEL_ERROR, "model execute failed!\n"); + + if (thread_args->model_name == SAMPLE_SVP_NPU_RFCN || thread_args->model_name == SAMPLE_SVP_NPU_YOLO) { + detection_info = thread_args->detection_info; + if (detection_info->is_cpu_rpn == TD_FALSE) { + ret = sample_common_svp_npu_roi_to_rect(&g_svp_npu_task[0], detection_info, ext_frame, base_frame, + &g_svp_npu_rect_info); + sample_svp_check_exps_return(ret != TD_SUCCESS, ret, SAMPLE_SVP_ERR_LEVEL_ERROR, "roi to rect failed!\n"); + } else { + ret = sample_common_svp_npu_roi_to_rect_cpu(&g_svp_npu_task[0], detection_info, ext_frame, base_frame, + &g_svp_npu_rect_info); + sample_svp_check_exps_return(ret != TD_SUCCESS, ret, SAMPLE_SVP_ERR_LEVEL_ERROR, "roi to rect failed!\n"); + } + ret = sample_common_svp_vgs_fill_rect(base_frame, &g_svp_npu_rect_info, SAMPLE_SVP_NPU_RECT_COLOR); + sample_svp_check_exps_return(ret != TD_SUCCESS, ret, SAMPLE_SVP_ERR_LEVEL_ERROR, "vgs fill rect failed!\n"); + for (int i = 0; i < g_svp_npu_rect_info.num; i++) { + printf("score:%lf, class_id:%d\r\n", g_svp_npu_rect_info.rect[i].score, + g_svp_npu_rect_info.rect[i].class_id); + } + } + return ret; +} + +static td_s32 sample_svp_npu_acl_vb_map(td_u32 vb_pool_idx) +{ + td_s32 ret; + + if (g_svp_npu_vb_virt_addr == TD_NULL) { + ret = ss_mpi_vb_get_pool_info(g_svp_npu_media_cfg.vb_pool[vb_pool_idx], &g_svp_npu_vb_pool_info); + sample_svp_check_exps_return(ret != TD_SUCCESS, TD_FAILURE, SAMPLE_SVP_ERR_LEVEL_ERROR, + "get pool info failed!\n"); + g_svp_npu_vb_virt_addr = ss_mpi_sys_mmap(g_svp_npu_vb_pool_info.pool_phy_addr, + g_svp_npu_vb_pool_info.pool_size); + sample_svp_check_exps_return(g_svp_npu_vb_virt_addr == TD_NULL, TD_FAILURE, SAMPLE_SVP_ERR_LEVEL_ERROR, + "map vb pool failed!\n"); + } + return TD_SUCCESS; +} + +static td_void *sample_svp_npu_acl_vdec_to_vo(td_void *args) +{ + td_s32 ret; + ot_video_frame_info base_frame; + ot_video_frame_info ext_frame; + const td_s32 milli_sec = SAMPLE_SVP_NPU_MILLIC_SEC; + const ot_vo_layer vo_layer = 0; + const ot_vo_chn vo_chn = 0; + const td_s32 vpss_grp = 0; + td_s32 vpss_chn[] = { OT_VPSS_CHN0, OT_VPSS_CHN1 }; + td_u32 size; + td_u32 stride; + td_u8 *data = TD_NULL; + + (td_void)prctl(PR_SET_NAME, "svp_npu_vdec_to_vo", 0, 0, 0); + + ret = svp_acl_rt_set_device(g_svp_npu_dev_id); + sample_svp_check_exps_return(ret != TD_SUCCESS, TD_NULL, SAMPLE_SVP_ERR_LEVEL_ERROR, "open device failed!\n"); + + ret = sample_svp_npu_acl_vb_map(OT_VPSS_CHN1); + sample_svp_check_exps_goto(ret != TD_SUCCESS, fail_0, SAMPLE_SVP_ERR_LEVEL_ERROR, "map vb pool failed!\n"); + + ret = sample_common_svp_npu_get_input_data_buffer_info(&g_svp_npu_task[0], 0, &data, &size, &stride); + sample_svp_check_exps_goto(ret != TD_SUCCESS, fail_1, SAMPLE_SVP_ERR_LEVEL_ERROR, + "Error(%#x),get_input_data_buffer_info failed!\n", ret); + + while (g_svp_npu_thread_stop == TD_FALSE) { + ret = ss_mpi_vpss_get_chn_frame(vpss_grp, vpss_chn[1], &ext_frame, milli_sec); + sample_svp_check_exps_continue(ret != TD_SUCCESS, SAMPLE_SVP_ERR_LEVEL_ERROR, + "Error(%#x),ss_mpi_vpss_get_chn_frame failed, vpss_grp(%d), vpss_chn(%d)!\n", ret, vpss_grp, vpss_chn[1]); + + ret = ss_mpi_vpss_get_chn_frame(vpss_grp, vpss_chn[0], &base_frame, milli_sec); + sample_svp_check_exps_goto(ret != TD_SUCCESS, ext_release, SAMPLE_SVP_ERR_LEVEL_ERROR, + "Error(%#x),ss_mpi_vpss_get_chn_frame failed, VPSS_GRP(%d), VPSS_CHN(%d)!\n", ret, vpss_grp, vpss_chn[0]); + + ret = sample_svp_npu_acl_frame_proc(&ext_frame, &base_frame, args); + sample_svp_check_exps_goto(ret != TD_SUCCESS, base_release, SAMPLE_SVP_ERR_LEVEL_ERROR, + "Error(%#x),sample_svp_npu_acl_frame_proc failed!\n", ret); + + ret = sample_common_svp_venc_vo_send_stream(&g_svp_npu_media_cfg.svp_switch, 0, vo_layer, vo_chn, &base_frame); + sample_svp_check_exps_goto(ret != TD_SUCCESS, base_release, SAMPLE_SVP_ERR_LEVEL_ERROR, + "Error(%#x),sample_common_svp_venc_vo_send_stream failed!\n", ret); +base_release: + ret = ss_mpi_vpss_release_chn_frame(vpss_grp, vpss_chn[0], &base_frame); + sample_svp_check_exps_trace(ret != TD_SUCCESS, SAMPLE_SVP_ERR_LEVEL_ERROR, + "Error(%#x),release_frame failed,grp(%d) chn(%d)!\n", ret, vpss_grp, vpss_chn[0]); +ext_release: + ret = ss_mpi_vpss_release_chn_frame(vpss_grp, vpss_chn[1], &ext_frame); + sample_svp_check_exps_trace(ret != TD_SUCCESS, SAMPLE_SVP_ERR_LEVEL_ERROR, + "Error(%#x),release_frame failed,grp(%d) chn(%d)!\n", ret, vpss_grp, vpss_chn[1]); + } + ret = sample_common_svp_npu_update_input_data_buffer_info(data, size, stride, 0, &g_svp_npu_task[0]); + sample_svp_check_exps_trace(ret != TD_SUCCESS, SAMPLE_SVP_ERR_LEVEL_ERROR, "update buffer failed!\n"); +fail_1: + (td_void)ss_mpi_sys_munmap(g_svp_npu_vb_virt_addr, g_svp_npu_vb_pool_info.pool_size); +fail_0: + (td_void)svp_acl_rt_reset_device(g_svp_npu_dev_id); + return TD_NULL; +} + +static td_void sample_svp_npu_acl_pause(td_void) +{ + printf("---------------press Enter key to exit!---------------\n"); + if (g_svp_npu_terminate_signal == TD_TRUE) { + return; + } + (td_void)getchar(); + if (g_svp_npu_terminate_signal == TD_TRUE) { + return; + } +} + +static td_void sample_svp_npu_acl_set_task_info(td_u32 model_idx) +{ + g_svp_npu_task[0].cfg.max_batch_num = 1; + g_svp_npu_task[0].cfg.dynamic_batch_num = 1; + g_svp_npu_task[0].cfg.total_t = 0; + g_svp_npu_task[0].cfg.is_cached = TD_TRUE; + g_svp_npu_task[0].cfg.model_idx = model_idx; +} + +static td_void sample_svp_npu_acl_set_detection_info(sample_svp_npu_detection_info *detection_info, td_u32 index, + td_bool is_cpu_rpn) +{ + detection_info->num_name = g_svp_npu_yolo_roi_info[index].roi_num_name; + detection_info->roi_name = g_svp_npu_yolo_roi_info[index].roi_class_name; + detection_info->has_background = TD_FALSE; + detection_info->is_cpu_rpn = is_cpu_rpn; + detection_info->idx = index; + /* use PIC_BUTT to be a flag, get the input resolution form om */ + g_svp_npu_media_cfg.pic_type[1] = PIC_BUTT; + g_svp_npu_terminate_signal = TD_FALSE; +} + +/* function : show the sample of yolo. RPN processed by npu */ +td_void sample_face_detection(void) +{ + td_s32 ret; + const td_u32 model_idx = 0; + const td_char *om_model_path = "./data/model/face_detection.om"; + sample_svp_npu_detection_info detection_info = {0}; + sample_svp_npu_thread_args args = {SAMPLE_SVP_NPU_YOLO, &detection_info}; + + sample_svp_npu_acl_set_detection_info(&detection_info, 0, TD_FALSE); + + if (g_svp_npu_terminate_signal == TD_FALSE) { + /* init acl */ + ret = sample_svp_npu_acl_init(TD_NULL, TD_FALSE); + sample_svp_check_exps_return_void(ret != TD_SUCCESS, SAMPLE_SVP_ERR_LEVEL_ERROR, "init failed!\n"); + + /* load model */ + ret = sample_common_svp_npu_load_model(om_model_path, model_idx, TD_FALSE); + sample_svp_check_exps_goto(ret != TD_SUCCESS, process_end0, SAMPLE_SVP_ERR_LEVEL_ERROR, + "load model failed!\n"); + + /* get input resolution */ + ret = sample_common_svp_npu_get_input_resolution(model_idx, 0, &g_svp_npu_media_cfg.pic_size[1]); + sample_svp_check_exps_goto(ret != TD_SUCCESS, process_end1, SAMPLE_SVP_ERR_LEVEL_ERROR, + "load model failed!\n"); + + /* start vdec vpss venc vo */ + ret = sample_common_svp_create_vb_start_vdec_vpss_vo(&g_svp_npu_vdec_cfg, &g_svp_npu_vdec_param, + &g_svp_npu_vdec_thread, &g_svp_npu_media_cfg, &g_svp_npu_vo_cfg); + sample_svp_check_exps_goto(ret != TD_SUCCESS, process_end1, SAMPLE_SVP_ERR_LEVEL_DEBUG, + "init media failed!\n"); + + /* set cfg */ + sample_svp_npu_acl_set_task_info(model_idx); + + ret = sample_svp_npu_acl_init_task(1, TD_FALSE, 0); + sample_svp_check_exps_goto(ret != TD_SUCCESS, process_end2, SAMPLE_SVP_ERR_LEVEL_ERROR, "init task failed!\n"); + } + /* process */ + if (g_svp_npu_terminate_signal == TD_FALSE) { + ret = sample_common_svp_npu_set_threshold(&g_svp_npu_yolo_threshold[0], SAMPLE_SVP_NPU_YOLO_THRESHOLD_NUM, + &g_svp_npu_task[0]); + sample_svp_check_exps_goto(ret != TD_SUCCESS, process_end3, SAMPLE_SVP_ERR_LEVEL_ERROR, + "set threshold failed!\n"); + + g_svp_npu_thread_stop = TD_FALSE; + ret = pthread_create(&g_svp_npu_thread, 0, sample_svp_npu_acl_vdec_to_vo, (td_void*)&args); + + sample_svp_check_exps_goto(ret != 0, process_end3, SAMPLE_SVP_ERR_LEVEL_ERROR, "create thread failed!\n"); + + (td_void)sample_svp_npu_acl_pause(); + + g_svp_npu_thread_stop = TD_TRUE; + pthread_join(g_svp_npu_thread, TD_NULL); + } + +process_end3: + (td_void)sample_svp_npu_acl_deinit_task(1, 0); +process_end2: + (td_void)sample_common_svp_destroy_vb_stop_vdec_vpss_vo(&g_svp_npu_vdec_param, &g_svp_npu_vdec_thread, + &g_svp_npu_media_cfg, &g_svp_npu_vo_cfg); +process_end1: + (td_void)sample_common_svp_npu_unload_model(model_idx); +process_end0: + (td_void)sample_svp_npu_acl_deinit(); + (td_void)sample_svp_npu_acl_terminate(); +} + +int main(void) +{ +#ifdef CONFIG_USER_SPACE + SDK_init(); +#endif + + sample_face_detection(); + +#ifdef CONFIG_USER_SPACE + SDK_exit(); +#endif + + return 0; +} diff --git a/vendor/zsks/demo/face_detection/pic/image-20221114150205685.png b/vendor/zsks/demo/face_detection/pic/image-20221114150205685.png new file mode 100755 index 0000000000000000000000000000000000000000..4dc4215c962c57ba0e31548fb98c29c789f149d9 Binary files /dev/null and b/vendor/zsks/demo/face_detection/pic/image-20221114150205685.png differ diff --git a/vendor/zsks/demo/face_detection/pic/image-20250210161601541.png b/vendor/zsks/demo/face_detection/pic/image-20250210161601541.png new file mode 100755 index 0000000000000000000000000000000000000000..80845ece8131933a4cd56ae3bd6119c38e0abd26 Binary files /dev/null and b/vendor/zsks/demo/face_detection/pic/image-20250210161601541.png differ diff --git a/vendor/zsks/demo/face_detection/pic/image-20250210170027454.png b/vendor/zsks/demo/face_detection/pic/image-20250210170027454.png new file mode 100755 index 0000000000000000000000000000000000000000..c06081fc2c963af4d1fa8af506804beedefe1b0b Binary files /dev/null and b/vendor/zsks/demo/face_detection/pic/image-20250210170027454.png differ diff --git a/vendor/zsks/demo/face_detection/pic/image-20250213112932380.png b/vendor/zsks/demo/face_detection/pic/image-20250213112932380.png new file mode 100755 index 0000000000000000000000000000000000000000..8b591b3437544859f4ec76b9ca03867cc401078c Binary files /dev/null and b/vendor/zsks/demo/face_detection/pic/image-20250213112932380.png differ diff --git a/vendor/zsks/demo/face_detection/pic/image-20250919144926220.png b/vendor/zsks/demo/face_detection/pic/image-20250919144926220.png new file mode 100755 index 0000000000000000000000000000000000000000..ce52c2e8b435b703f1dcec3eccaa917d4dcd189a Binary files /dev/null and b/vendor/zsks/demo/face_detection/pic/image-20250919144926220.png differ diff --git a/vendor/zsks/demo/face_detection/pic/image-20250919150630870.png b/vendor/zsks/demo/face_detection/pic/image-20250919150630870.png new file mode 100755 index 0000000000000000000000000000000000000000..2d661646429af7ee314241ca3f86556060d7911e Binary files /dev/null and b/vendor/zsks/demo/face_detection/pic/image-20250919150630870.png differ diff --git a/vendor/zsks/demo/face_detection/pic/image-20250919151018659.png b/vendor/zsks/demo/face_detection/pic/image-20250919151018659.png new file mode 100755 index 0000000000000000000000000000000000000000..9b09507893b65d2142be49e6461daa31a81855c4 Binary files /dev/null and b/vendor/zsks/demo/face_detection/pic/image-20250919151018659.png differ diff --git a/vendor/zsks/demo/face_detection/pic/image-20250919151519310.png b/vendor/zsks/demo/face_detection/pic/image-20250919151519310.png new file mode 100755 index 0000000000000000000000000000000000000000..7f6d881a5d0e6d3f72b7a43d089b5c5a75d7f048 Binary files /dev/null and b/vendor/zsks/demo/face_detection/pic/image-20250919151519310.png differ diff --git a/vendor/zsks/demo/face_detection/pic/image-20250919151605632.png b/vendor/zsks/demo/face_detection/pic/image-20250919151605632.png new file mode 100755 index 0000000000000000000000000000000000000000..c6aa9aad5685495d6bad5ba5d4e0c09e31ed3c7a Binary files /dev/null and b/vendor/zsks/demo/face_detection/pic/image-20250919151605632.png differ diff --git a/vendor/zsks/demo/face_detection/pic/image-20251023095337519.png b/vendor/zsks/demo/face_detection/pic/image-20251023095337519.png new file mode 100755 index 0000000000000000000000000000000000000000..b0478a2959f250a93b12e43570a388d69b2f60d1 Binary files /dev/null and b/vendor/zsks/demo/face_detection/pic/image-20251023095337519.png differ diff --git a/vendor/zsks/demo/face_detection/pic/image-20251023100136196.png b/vendor/zsks/demo/face_detection/pic/image-20251023100136196.png new file mode 100755 index 0000000000000000000000000000000000000000..c1b6906ce6ab14d4703a09cf465fc3caf00ec3d0 Binary files /dev/null and b/vendor/zsks/demo/face_detection/pic/image-20251023100136196.png differ diff --git a/vendor/zsks/demo/face_detection/pic/image-20251023100347550.png b/vendor/zsks/demo/face_detection/pic/image-20251023100347550.png new file mode 100755 index 0000000000000000000000000000000000000000..5b02ccd845367df696cd076e73a53b45f811ae09 Binary files /dev/null and b/vendor/zsks/demo/face_detection/pic/image-20251023100347550.png differ diff --git a/vendor/zsks/demo/face_detection/pic/image-20251023101210815.png b/vendor/zsks/demo/face_detection/pic/image-20251023101210815.png new file mode 100755 index 0000000000000000000000000000000000000000..de9d64bdfa6a7b200ab4878ac8caebff709a349d Binary files /dev/null and b/vendor/zsks/demo/face_detection/pic/image-20251023101210815.png differ diff --git a/vendor/zsks/demo/face_detection/pic/image-20251023101503216.png b/vendor/zsks/demo/face_detection/pic/image-20251023101503216.png new file mode 100755 index 0000000000000000000000000000000000000000..684eb0e29b050e472c5e4ce1ea65861f2d6f5ff3 Binary files /dev/null and b/vendor/zsks/demo/face_detection/pic/image-20251023101503216.png differ diff --git a/vendor/zsks/demo/face_detection/pic/image-20251023102053075.png b/vendor/zsks/demo/face_detection/pic/image-20251023102053075.png new file mode 100755 index 0000000000000000000000000000000000000000..6fc62dd35e3dcfeb36a6e3ba1e7cea3e008660ed Binary files /dev/null and b/vendor/zsks/demo/face_detection/pic/image-20251023102053075.png differ diff --git a/vendor/zsks/demo/face_detection/pic/image-20251023102549783.png b/vendor/zsks/demo/face_detection/pic/image-20251023102549783.png new file mode 100755 index 0000000000000000000000000000000000000000..5b213c9e78709eaf94c0764a24761bd124d8dc75 Binary files /dev/null and b/vendor/zsks/demo/face_detection/pic/image-20251023102549783.png differ diff --git a/vendor/zsks/demo/fruit_identify/Makefile b/vendor/zsks/demo/fruit_identify/Makefile new file mode 100755 index 0000000000000000000000000000000000000000..655332a1e534facf1e535f30523f129824e25f72 --- /dev/null +++ b/vendor/zsks/demo/fruit_identify/Makefile @@ -0,0 +1,59 @@ +# 编译所需的公共文件 +include ../mk.param + +# 编译后生成的可执行文件名 +TARGET := main + +# 编译此工程需要包含的头文件 +CFLAGS += -I$(SDK_LINUX_INCLUDE_DIR)/ +CFLAGS += -I$(SDK_SAMPLE_COMMON_DIR)/ +CFLAGS += -I$(AUDIO_SAMPLE)/adp/ +CFLAGS += -I$(SDK_LINUX_INCLUDE_DIR)/svp_npu +CFLAGS += -I$(FRUIT_IDENTIFY_SAMPLE_DIR)/ +CFLAGS += -I$(FRUIT_IDENTIFY_SAMPLE_DIR)/common/ +CFLAGS += -I$(THIRD_PARTY_DIR)/xvp_osd/include/ + +# 编译此工程需要依赖的.c或.cpp文件 +SMP_SRCS := $(AUDIO_SAMPLE)/adp/audio_aac_adp.c +SMP_SRCS += $(AUDIO_SAMPLE)/adp/audio_dl_adp.c +SMP_SRCS += $(SDK_SAMPLE_COMMON_DIR)/sample_comm_sys.c +SMP_SRCS += $(SDK_SAMPLE_COMMON_DIR)/sample_comm_audio.c +SMP_SRCS += $(SDK_SAMPLE_COMMON_DIR)/sample_ipc.c +SMP_SRCS += $(SDK_SAMPLE_COMMON_DIR)/sample_comm_isp.c +SMP_SRCS += $(SDK_SAMPLE_COMMON_DIR)/sample_comm_vi.c +SMP_SRCS += $(SDK_SAMPLE_COMMON_DIR)/sample_comm_vo.c +SMP_SRCS += $(SDK_SAMPLE_COMMON_DIR)/sample_comm_mipi_tx.c +SMP_SRCS += $(SDK_SAMPLE_COMMON_DIR)/sample_comm_vpss.c +SMP_SRCS += $(SDK_SAMPLE_COMMON_DIR)/loadbmp.c +SMP_SRCS += $(SDK_SAMPLE_COMMON_DIR)/sample_comm_vdec.c +SMP_SRCS += $(SDK_SAMPLE_COMMON_DIR)/sample_comm_venc.c +SMP_SRCS += $(SDK_SAMPLE_COMMON_DIR)/sdk_init.c +SMP_SRCS += $(SDK_SAMPLE_COMMON_DIR)/sdk_exit.c +SMP_SRCS += $(FRUIT_IDENTIFY_SAMPLE_DIR)/fruit_identify.c +SMP_SRCS += $(FRUIT_IDENTIFY_SAMPLE_DIR)/sample_audio.c +SMP_SRCS += $(FRUIT_IDENTIFY_SAMPLE_DIR)/common/sample_common_svp_npu_model.c +SMP_SRCS += $(FRUIT_IDENTIFY_SAMPLE_DIR)/common/sample_common_svp_npu.c +SMP_SRCS += $(FRUIT_IDENTIFY_SAMPLE_DIR)/common/sample_common_svp_osd.c +SMP_SRCS += $(FRUIT_IDENTIFY_SAMPLE_DIR)/common/sample_common_svp.c + +# 编译此工程需要包含的库和宏 +CFLAGS += -O3 -g -Wall -DUSE_USB_CAMERA +CFLAGS += -fstack-protector-strong +MPI_LIBS += $(THIRD_PARTY_DIR)/xvp_osd/lib/libxvp_osd.a + +# 根据实际的类型设置，可以用set_sensor脚本设置 +SENSOR0_TYPE = -DSENSOR0_TYPE=HY_S0603_MIPI_8M_30FPS_12BIT +SENSOR1_TYPE = -DSENSOR1_TYPE=HY_S0603_MIPI_8M_30FPS_12BIT +SENSOR2_TYPE = -DSENSOR1_TYPE=HY_S0603_MIPI_8M_30FPS_12BIT +SENSOR3_TYPE = -DSENSOR1_TYPE=HY_S0603_MIPI_8M_30FPS_12BIT + +CFLAGS += $(SENSOR0_TYPE) +CFLAGS += $(SENSOR1_TYPE) +CFLAGS += $(SENSOR2_TYPE) +CFLAGS += $(SENSOR3_TYPE) + +# 编译工程所需的mk文件 + +include ../sdk_linux.mk + + diff --git a/vendor/zsks/demo/fruit_identify/README.md b/vendor/zsks/demo/fruit_identify/README.md new file mode 100755 index 0000000000000000000000000000000000000000..44984e18206a689662937189114c8649b8d16b39 --- /dev/null +++ b/vendor/zsks/demo/fruit_identify/README.md @@ -0,0 +1,130 @@ +## 2.3、fruit_identify操作指导 + +### 2.3.1、fruit_identify程序简介 + +* fruit_identify sample基于SS928V100平台开发，以EulerPi套件为例，fruit_identify sample是通过USB Camera，将采集到的图片送到水果检测模型中进行推理，当检测到特定水果时，会通过外接显示屏实时显示水果的种类以及执行度，并框住水果的具体位置，并通过耳机播放出此时识别到的水果类别。 +* fruit_identify案例主要是使用pytorch框架，基于YoloV8网络，使用自行标注的水果数据集训练出来的水果分类模型。 + +### 2.3.2、目录 + +```shell +pegasus/vendor/cust/demo/fruit_identify +|── common # 基于海思mpp/sample中的svp/common/目录下修改的代码，新增了OSD部分的内容 +|── data # 模型文件和汉字库 +|── fruit_audio # 不同水果的音频文件 +|── Makefile # 编译脚本 +|── fruit_identify.c # fruit_identif sample业务代码 +|── sample_audio.c # 音频播放的业务代码 +└── sample_audio.h # 音频播放的头文件 +``` + +![image-20250919173612332](pic/image-20250919173612332.png) + +### 2.3.3、编译 + +* **注意：在编译cust的demo之前，请确保你已经按照[patch介绍中的步骤](../../README.md#4patch介绍)把补丁打入海思SDK的sample目录下了**。 + +* 步骤1：先根据自己选择的操作系统，进入到对应的Pegasus所在目录下。 + +* 步骤2：使用Makefile的方式进行单编 + +* 在Ubuntu的命令行终端，分步执行下面的命令，单编 fruit_identify sample + + ``` + cd pegasus/vendor/cust/demo/fruit_identify + + make clean && make + ``` + + * 在fruit_identify/out目录下，生成一个名为main的 可执行文件，如下图所示： + + ![image-20251023103457181](pic/image-20251023103457181.png) + +### 2.3.4、拷贝可执行程序和依赖文件至开发板的mnt目录下 + +**方式一：使用SD卡进行资料文件的拷贝** + +* 首先需要自己准备一张Micro sd卡(16G 左右即可)，还得有一个Micro sd卡的读卡器。 + +image-20221114150205685 + +* 步骤1：将编译后生成的可执行文件、data目录（里面有模型和字库），音频文件都拷贝到SD卡中。 + +![image-20251023103623167](pic/image-20251023103623167.png) + +* 步骤2：可执行文件拷贝成功后，将内存卡插入开发板的SD卡槽中，可通过挂载的方式挂载到板端，可选择SD卡 mount指令进行挂载。 + +image-20250210161601541 + +* 在开发板的终端，执行下面的命令进行SD卡的挂载 + * 如果挂载失败，可以参考[这个issue尝试解决](https://gitee.com/HiSpark/HiSpark_NICU2022/issues/I54932?from=project-issue) + + +```shell +mount -t vfat /dev/mmcblk1p1 /mnt +# 其中/dev/mmcblk1p1需要根据实际块设备号修改 +``` + +* 挂载成功后，如下图所示： + +![image-20251023103850695](pic/image-20251023103850695.png) + +**方式二：使用NFS挂载的方式进行资料文件的拷贝** + +* 首先需要自己准备一根网线 +* 步骤1：参考[博客链接](https://blog.csdn.net/Wu_GuiMing/article/details/115872995?spm=1001.2014.3001.5501)中的内容，进行nfs的环境搭建 +* 步骤2：将编译后生成的可执行文件、data目录（里面有模型和字库），音频文件都拷贝到Windows的nfs共享路径下 + +![image-20251023103700404](pic/image-20251023103700404.png) + +* 步骤3：在开发板的终端执行下面的命令，将Windows的nfs共享路径挂载至开发板的mnt目录下 + * 注意：这里IP地址请根据你开发板和电脑主机的实际IP地址进行填写 + + +``` +ifconfig eth0 192.168.100.100 + +mount -o nolock,addr=192.168.100.10 -t nfs 192.168.100.10:/d/nfs /mnt +``` + +![image-20251023103944837](pic/image-20251023103944837.png) + +### 2.3.5、硬件连接 + +* 准备一个外接显示器和HDMI数据线，将HDMI的一头接在开发板的HDMI输出口，将HDMI的另外一头接在外接显示器的HDMI输入口。 + +![image-20250213112932380](pic/image-20250213112932380.png) + +* 将USB 摄像头接在EulerPi开发板的USB接口上。 + +image-20250919150630870 + +### 2.3.6、功能验证 + +* 在开发板的终端执行下面的命令，运行可执行文件 + +``` +cd /mnt + +chmod +x main + +./main +``` + +![image-20251023110809454](pic/image-20251023110809454.png) + +* 此时，在HDMI的外接显示屏上即可出现实时码流，如下图所示： + +image-20250210170027454 + +* 如果您看到的现象和下图现象不一致，可以确认一下USB涉嫌头是否插入到开发板的USB口，并且在开发板的/dev目录下能够看到video0 和 video1两个设备节点。如果没有这两个设备节点，请确保镜像烧录正常。 + +![image-20250919151018659](pic/image-20250919151018659.png) + +* 正常情况下，我们会在外接显示屏上看到有水果的区域被框出来。且在框框的左上角显示水果的种类和置信度，并且你可以在开发板上面接上耳机，可以听到检测到的水果的具体语音播报。 + +![image-20250919180706355](pic/image-20250919180706355.png) + +* 敲两下回车即可关闭程序 + +![image-20250919151519310](pic/image-20250919151519310.png) \ No newline at end of file diff --git a/vendor/zsks/demo/fruit_identify/common/sample_common_svp.c b/vendor/zsks/demo/fruit_identify/common/sample_common_svp.c new file mode 100755 index 0000000000000000000000000000000000000000..79778ec97254e1250f4ea6820348267955f8df4d --- /dev/null +++ b/vendor/zsks/demo/fruit_identify/common/sample_common_svp.c @@ -0,0 +1,1036 @@ +/* + * Copyright (c) 2025 HiSilicon (Shanghai) Technologies Co., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "sample_common_svp.h" + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "ot_common.h" +#include "ot_common_video.h" +#include "ot_common_sys.h" +#include "ot_common_svp.h" +#include "ss_mpi_vb.h" +#include "sample_comm.h" + +#define OT_VPSS_CHN_NUM 2 +#define OT_SAMPLE_IVE_MAX_POOL_CNT 128 +#define OT_SAMPLE_IVE_DRAW_THICK 2 + +#define SAMPLE_SVP_BLK_CNT 16 +#define SMAPLE_SVP_DISPLAY_BUF_LEN 3 +#define SAMPLE_SVP_VI_CHN_INTERVAL 4 +#define SAMPLE_SVP_VDEC_CHN_0 0 +#define SAMPLE_SVP_VDEC_CHN_NUM 1 +#define SAMPLE_SVP_VPSS_BORDER_WIDTH 2 +#define SAMPLE_SVP_VO_DIS_BUF_LEN 3 +#define SAMPLE_SVP_MAX_WIDTH 32768 +#define SAMPLE_SVP_NUM_TWO 2 +#define SAMPLE_SVP_DSP_BIN_NUM_PER 4 +#define SAMPLE_SVP_DSP_MEM_TYPE_SYS_DDR 0 +#define SAMPLE_SVP_DSP_MEM_TYPE_IRAM 1 +#define SAMPLE_SVP_DSP_MEM_TYPE_DRAM_0 2 +#define SAMPLE_SVP_DSP_MEM_TYPE_DRAM_1 3 + +static td_bool g_sample_svp_init_flag = TD_FALSE; + +static td_s32 sample_common_svp_get_pic_type_by_sns_type(sample_sns_type sns_type, ot_pic_size size[], td_u32 num) +{ + sample_svp_check_exps_return(num > OT_VPSS_CHN_NUM, TD_FAILURE, SAMPLE_SVP_ERR_LEVEL_ERROR, + "num(%u) can't be larger than (%u)\n", num, OT_VPSS_CHN_NUM); + switch (sns_type) { + case HY_S0603_MIPI_8M_30FPS_12BIT: + case HY_S0603_MIPI_8M_30FPS_10BIT_WDR2TO1: + size[0] = PIC_3840X2160; + break; + default: + size[0] = PIC_3840X2160; + break; + } + return TD_SUCCESS; +} + +static td_s32 sample_comm_svp_get_pic_size(ot_pic_size pic_type, td_u64 *size) +{ + td_s32 ret; + ot_pic_buf_attr pic_buf_attr = {0}; + ot_size pic_size = {0}; + + ret = sample_comm_sys_get_pic_size(pic_type, &pic_size); + sample_svp_check_exps_return(ret != TD_SUCCESS, ret, SAMPLE_SVP_ERR_LEVEL_ERROR, + "sample_comm_sys_get_pic_size failed,Error(%#x)!\n", ret); + pic_buf_attr.width = pic_size.width; + pic_buf_attr.height = pic_size.height; + pic_buf_attr.compress_mode = OT_COMPRESS_MODE_NONE; + pic_buf_attr.align = OT_DEFAULT_ALIGN; + pic_buf_attr.bit_width = OT_DATA_BIT_WIDTH_8; + pic_buf_attr.pixel_format = OT_PIXEL_FORMAT_YVU_SEMIPLANAR_422; + *size = ot_common_get_pic_buf_size(&pic_buf_attr); + return TD_SUCCESS; +} + +/* System init */ +static td_s32 sample_comm_svp_sys_init(td_bool vi_en) +{ + td_s32 ret; + ot_vb_cfg vb_cfg; + ot_pic_size pic_type; + td_u64 size = 0; + sample_sns_type sns_type = SENSOR0_TYPE; + + ret = ss_mpi_sys_exit(); + sample_svp_check_exps_return(ret != TD_SUCCESS, ret, SAMPLE_SVP_ERR_LEVEL_ERROR, + "Error(%#x):ss_mpi_sys_exit failed!\n", ret); + ret = ss_mpi_vb_exit(); + sample_svp_check_exps_return(ret != TD_SUCCESS, ret, SAMPLE_SVP_ERR_LEVEL_ERROR, + "Error(%#x):ss_mpi_vb_exit failed!\n", ret); + + (td_void)memset_s(&vb_cfg, sizeof(ot_vb_cfg), 0, sizeof(ot_vb_cfg)); + + if (vi_en == TD_FALSE) { + vb_cfg.max_pool_cnt = SAMPLE_SVP_VB_POOL_NUM; + vb_cfg.common_pool[1].blk_size = SAMPLE_SVP_D1_PAL_WIDTH * SAMPLE_SVP_D1_PAL_HEIGHT * SAMPLE_SVP_VB_POOL_NUM; + vb_cfg.common_pool[1].blk_cnt = 1; + } else { + ret = sample_common_svp_get_pic_type_by_sns_type(sns_type, &pic_type, OT_SVP_MAX_VPSS_CHN_NUM); + sample_svp_check_exps_return(ret != TD_SUCCESS, ret, SAMPLE_SVP_ERR_LEVEL_ERROR, + "sample_common_svp_get_pic_type_by_sns_type failed!\n"); + ret = sample_comm_svp_get_pic_size(pic_type, &size); + sample_svp_check_exps_return(ret != TD_SUCCESS, ret, SAMPLE_SVP_ERR_LEVEL_ERROR, + "Error(%#x):sample_common_svp_npu_get_pic_size failed!\n", ret); + vb_cfg.max_pool_cnt = SAMPLE_SVP_VB_POOL_NUM; + vb_cfg.common_pool[1].blk_size = size; + vb_cfg.common_pool[1].blk_cnt = SAMPLE_SVP_BLK_CNT; + } + + ret = ss_mpi_vb_set_cfg((const ot_vb_cfg *)&vb_cfg); + sample_svp_check_exps_return(ret != TD_SUCCESS, ret, SAMPLE_SVP_ERR_LEVEL_ERROR, + "Error(%#x):ss_mpi_vb_set_config failed!\n", ret); + + ret = ss_mpi_vb_init(); + sample_svp_check_exps_return(ret != TD_SUCCESS, ret, SAMPLE_SVP_ERR_LEVEL_ERROR, + "Error(%#x):ss_mpi_vb_init failed!\n", ret); + + ret = ss_mpi_sys_init(); + sample_svp_check_exps_return(ret != TD_SUCCESS, ret, SAMPLE_SVP_ERR_LEVEL_ERROR, + "Error(%#x):ss_mpi_sys_init failed!\n", ret); + + return ret; +} + +/* System exit */ +static td_s32 sample_comm_svp_sys_exit(td_void) +{ + td_s32 ret; + + ret = ss_mpi_sys_exit(); + sample_svp_check_exps_return(ret != TD_SUCCESS, ret, SAMPLE_SVP_ERR_LEVEL_ERROR, + "Error(%#x):ss_mpi_sys_exit failed!\n", ret); + + ret = ss_mpi_vb_exit(); + sample_svp_check_exps_return(ret != TD_SUCCESS, ret, SAMPLE_SVP_ERR_LEVEL_ERROR, + "Error(%#x):ss_mpi_vb_exit failed!\n", ret); + + return TD_SUCCESS; +} + +/* System init */ +td_s32 sample_common_svp_check_sys_init(td_bool vi_en) +{ + if (g_sample_svp_init_flag == TD_FALSE) { + if (sample_comm_svp_sys_init(vi_en) != TD_SUCCESS) { + sample_svp_trace_err("Svp mpi init failed!\n"); + return TD_FALSE; + } + g_sample_svp_init_flag = TD_TRUE; + } + + sample_svp_trace_info("Svp mpi init ok!\n"); + return TD_TRUE; +} + +/* System exit */ +td_void sample_common_svp_check_sys_exit(td_void) +{ + td_s32 ret; + + if (g_sample_svp_init_flag == TD_TRUE) { + ret = sample_comm_svp_sys_exit(); + if (ret != TD_SUCCESS) { + sample_svp_trace_err("svp mpi exit failed!\n"); + } + } + g_sample_svp_init_flag = TD_FALSE; + sample_svp_trace_info("Svp mpi exit ok!\n"); +} + +/* Align */ +td_u32 sample_common_svp_align(td_u32 size, td_u16 align) +{ + td_u32 stride; + + sample_svp_check_exps_return(align == 0, 0, SAMPLE_SVP_ERR_LEVEL_ERROR, "align can't be zero!\n"); + sample_svp_check_exps_return((size < 1) || (size > SAMPLE_SVP_MAX_WIDTH), 0, SAMPLE_SVP_ERR_LEVEL_ERROR, + "size(%u) must be [1, %u]\n", size, SAMPLE_SVP_MAX_WIDTH); + stride = size + (align - size % align) % align; + return stride; +} + +/* Create mem info */ +td_s32 sample_common_svp_create_mem_info(ot_svp_mem_info *mem_info, td_u32 size, td_u32 addr_offset) +{ + td_s32 ret = TD_FAILURE; + td_u32 size_tmp; + + sample_svp_check_exps_return(mem_info == TD_NULL, ret, SAMPLE_SVP_ERR_LEVEL_ERROR, "mem_info can't be zero\n"); + + size_tmp = size + addr_offset; + mem_info->size = size; + ret = ss_mpi_sys_mmz_alloc((td_phys_addr_t *)(&mem_info->phys_addr), + (void **)&mem_info->virt_addr, TD_NULL, TD_NULL, size_tmp); + sample_svp_check_exps_return(ret != TD_SUCCESS, ret, SAMPLE_SVP_ERR_LEVEL_ERROR, + "Error(%#x):ss_mpi_sys_alloc failed!\n", ret); + + mem_info->phys_addr += addr_offset; + mem_info->virt_addr += addr_offset; + + return ret; +} + +/* Destory mem info */ +td_void sample_common_svp_destroy_mem_info(ot_svp_mem_info *mem_info, td_u32 addr_offset) +{ + sample_svp_check_exps_return_void(mem_info == TD_NULL, SAMPLE_SVP_ERR_LEVEL_ERROR, "mem_info can't be zero\n"); + + if ((mem_info->virt_addr != 0) && (mem_info->phys_addr != 0)) { + (td_void)ss_mpi_sys_mmz_free(mem_info->phys_addr - addr_offset, + sample_svp_convert_addr_to_ptr(void, (mem_info->virt_addr - addr_offset))); + } + (td_void)memset_s(mem_info, sizeof(*mem_info), 0, sizeof(*mem_info)); +} +/* Malloc memory */ +td_s32 sample_common_svp_malloc_mem(td_char *mmb, td_char *zone, td_phys_addr_t *phys_addr, + td_void **virt_addr, td_u32 size) +{ + td_s32 ret = TD_FAILURE; + + sample_svp_check_exps_return(phys_addr == TD_NULL, ret, SAMPLE_SVP_ERR_LEVEL_ERROR, "phys_addr can't be null\n"); + sample_svp_check_exps_return(virt_addr == TD_NULL, ret, SAMPLE_SVP_ERR_LEVEL_ERROR, "virt_addr can't be null\n"); + ret = ss_mpi_sys_mmz_alloc((td_phys_addr_t *)phys_addr, virt_addr, mmb, zone, size); + + return ret; +} + +/* Malloc memory with cached */ +td_s32 sample_common_svp_malloc_cached(td_char *mmb, td_char *zone, td_phys_addr_t *phys_addr, + td_void **virt_addr, td_u32 size) +{ + td_s32 ret = TD_FAILURE; + + sample_svp_check_exps_return(phys_addr == TD_NULL, ret, SAMPLE_SVP_ERR_LEVEL_ERROR, "phys_addr can't be null\n"); + sample_svp_check_exps_return(virt_addr == TD_NULL, ret, SAMPLE_SVP_ERR_LEVEL_ERROR, "virt_addr can't be null\n"); + ret = ss_mpi_sys_mmz_alloc_cached((td_phys_addr_t *)phys_addr, virt_addr, mmb, zone, size); + + return ret; +} + +/* Fulsh cached */ +td_s32 sample_common_svp_flush_cache(td_phys_addr_t phys_addr, td_void *virt_addr, td_u32 size) +{ + td_s32 ret = TD_FAILURE; + + sample_svp_check_exps_return(virt_addr == TD_NULL, ret, SAMPLE_SVP_ERR_LEVEL_ERROR, "virt_addr can't be null\n"); + ret = ss_mpi_sys_flush_cache((td_phys_addr_t)phys_addr, virt_addr, size); + return ret; +} + +/* + * function : Init Vb + */ +static td_s32 sample_common_svp_vb_init(ot_pic_size *pic_type, ot_size *pic_size, + td_u32 vpss_chn_num) +{ + td_s32 ret; + td_u32 i; + ot_vb_cfg vb_cfg = {0}; + ot_pic_buf_attr pic_buf_attr; + ot_vb_calc_cfg calc_cfg; + ot_vi_vpss_mode_type mode_type = OT_VI_ONLINE_VPSS_OFFLINE; + ot_vi_video_mode video_mode = OT_VI_VIDEO_MODE_NORM; + + vb_cfg.max_pool_cnt = OT_SAMPLE_IVE_MAX_POOL_CNT; + + ret = sample_comm_sys_get_pic_size(pic_type[0], &pic_size[0]); + sample_svp_check_exps_goto(ret != TD_SUCCESS, vb_fail_0, SAMPLE_SVP_ERR_LEVEL_ERROR, + "sample_comm_sys_get_pic_size failed,Error(%#x)!\n", ret); + pic_buf_attr.width = pic_size[0].width; + pic_buf_attr.height = pic_size[0].height; + pic_buf_attr.align = OT_DEFAULT_ALIGN; + pic_buf_attr.bit_width = OT_DATA_BIT_WIDTH_8; + pic_buf_attr.pixel_format = OT_PIXEL_FORMAT_YVU_SEMIPLANAR_422; + pic_buf_attr.compress_mode = OT_COMPRESS_MODE_NONE; + + ot_common_get_pic_buf_cfg(&pic_buf_attr, &calc_cfg); + + vb_cfg.common_pool[0].blk_size = calc_cfg.vb_size; + vb_cfg.common_pool[0].blk_cnt = SAMPLE_SVP_BLK_CNT; + + for (i = 1; (i < vpss_chn_num) && (i < OT_VB_MAX_COMMON_POOLS); i++) { + ret = sample_comm_sys_get_pic_size(pic_type[i], &pic_size[i]); + sample_svp_check_exps_goto(ret != TD_SUCCESS, vb_fail_0, SAMPLE_SVP_ERR_LEVEL_ERROR, + "sample_comm_sys_get_pic_size failed,Error(%#x)!\n", ret); + pic_buf_attr.width = pic_size[i].width; + pic_buf_attr.height = pic_size[i].height; + pic_buf_attr.compress_mode = OT_COMPRESS_MODE_NONE; + pic_buf_attr.align = OT_DEFAULT_ALIGN; + + ot_common_get_pic_buf_cfg(&pic_buf_attr, &calc_cfg); + + /* comm video buffer */ + vb_cfg.common_pool[i].blk_size = calc_cfg.vb_size; + vb_cfg.common_pool[i].blk_cnt = SAMPLE_SVP_BLK_CNT; + } + + ret = sample_comm_sys_init_with_vb_supplement(&vb_cfg, OT_VB_SUPPLEMENT_BNR_MOT_MASK); + sample_svp_check_exps_goto(ret != TD_SUCCESS, vb_fail_1, SAMPLE_SVP_ERR_LEVEL_ERROR, + "sample_comm_sys_init failed,Error(%#x)!\n", ret); + + ret = sample_comm_vi_set_vi_vpss_mode(mode_type, video_mode); + sample_svp_check_exps_goto(ret != TD_SUCCESS, vb_fail_1, SAMPLE_SVP_ERR_LEVEL_ERROR, + "sample_comm_vi_set_vi_vpss_mode failed!\n"); + return ret; +vb_fail_1: + sample_comm_sys_exit(); +vb_fail_0: + return ret; +} + +td_s32 sample_common_svp_vgs_fill_rect(const ot_video_frame_info *frame_info, + ot_sample_svp_rect_info *rect, td_u32 color) +{ + ot_vgs_handle vgs_handle = -1; + td_s32 ret = TD_FAILURE; + td_u16 i; + ot_vgs_task_attr vgs_task; + ot_cover vgs_add_cover; + + sample_svp_check_exps_return(frame_info == TD_NULL, ret, SAMPLE_SVP_ERR_LEVEL_ERROR, "frame_info can't be null\n"); + sample_svp_check_exps_return(rect == TD_NULL, ret, SAMPLE_SVP_ERR_LEVEL_ERROR, "rect can't be null\n"); + sample_svp_check_exps_return(rect->num > OT_SVP_RECT_NUM, ret, + SAMPLE_SVP_ERR_LEVEL_ERROR, "rect->num can't lager than %u\n", OT_SVP_RECT_NUM); + if (rect->num == 0) { + return TD_SUCCESS; + } + + ret = ss_mpi_vgs_begin_job(&vgs_handle); + sample_svp_check_exps_return(ret != TD_SUCCESS, ret, SAMPLE_SVP_ERR_LEVEL_ERROR, + "Vgs begin job fail,Error(%#x)\n", ret); + + ret = memcpy_s(&vgs_task.img_in, sizeof(ot_video_frame_info), frame_info, sizeof(ot_video_frame_info)); + sample_svp_check_exps_goto(ret != EOK, fail, SAMPLE_SVP_ERR_LEVEL_ERROR, "get img_in failed\n"); + ret = memcpy_s(&vgs_task.img_out, sizeof(ot_video_frame_info), frame_info, sizeof(ot_video_frame_info)); + sample_svp_check_exps_goto(ret != EOK, fail, SAMPLE_SVP_ERR_LEVEL_ERROR, "get img_out failed\n"); + + vgs_add_cover.type = OT_COVER_QUAD; + vgs_add_cover.color = color; + for (i = 0; i < rect->num; i++) { + vgs_add_cover.quad.is_solid = TD_FALSE; + vgs_add_cover.quad.thick = OT_SAMPLE_IVE_DRAW_THICK; + ret = memcpy_s(vgs_add_cover.quad.point, sizeof(rect->rect[i].point), + rect->rect[i].point, sizeof(rect->rect[i].point)); + sample_svp_check_exps_goto(ret != EOK, fail, SAMPLE_SVP_ERR_LEVEL_ERROR, "get point failed\n"); + ret = ss_mpi_vgs_add_cover_task(vgs_handle, &vgs_task, &vgs_add_cover, 1); + sample_svp_check_exps_goto(ret != TD_SUCCESS, fail, SAMPLE_SVP_ERR_LEVEL_ERROR, + "ss_mpi_vgs_add_cover_task fail,Error(%#x)\n", ret); + } + ret = ss_mpi_vgs_end_job(vgs_handle); + sample_svp_check_exps_goto(ret != TD_SUCCESS, fail, SAMPLE_SVP_ERR_LEVEL_ERROR, + "ss_mpi_vgs_end_job fail,Error(%#x)\n", ret); + return ret; +fail: + ss_mpi_vgs_cancel_job(vgs_handle); + return ret; +} + +/* function : Start Vpss */ +static td_s32 sample_common_svp_start_vpss(td_s32 vpss_grp_cnt, ot_size *pic_size, td_u32 vpss_chn_num) +{ + td_u32 i; + ot_vpss_chn_attr vpss_chn_attr[OT_VPSS_MAX_CHN_NUM]; + ot_vpss_grp_attr vpss_grp_attr; + td_bool chn_enable[OT_VPSS_MAX_CHN_NUM] = { TD_TRUE, TD_TRUE, TD_FALSE, TD_FALSE }; + ot_vpss_grp vpss_grp; + td_s32 ret; + + (td_void)memset_s(&vpss_grp_attr, sizeof(ot_vpss_grp_attr), 0, sizeof(ot_vpss_grp_attr)); + sample_comm_vpss_get_default_grp_attr(&vpss_grp_attr); + vpss_grp_attr.max_width = pic_size[0].width; + vpss_grp_attr.max_height = pic_size[0].height; + vpss_grp_attr.nr_attr.compress_mode = OT_COMPRESS_MODE_NONE; + /* VPSS only onle channel0 support compress seg mode */ + sample_comm_vpss_get_default_chn_attr(&vpss_chn_attr[0]); + vpss_chn_attr[0].width = pic_size[0].width; + vpss_chn_attr[0].height = pic_size[0].height; + vpss_chn_attr[0].compress_mode = OT_COMPRESS_MODE_NONE; + vpss_chn_attr[0].depth = 1; + + for (i = 1; i < vpss_chn_num; i++) { + (td_void)memset_s(&vpss_chn_attr[i], sizeof(ot_vpss_chn_attr), 0, sizeof(ot_vpss_chn_attr)); + sample_comm_vpss_get_default_chn_attr(&vpss_chn_attr[i]); + vpss_chn_attr[i].width = pic_size[i].width; + vpss_chn_attr[i].height = pic_size[i].height; + vpss_chn_attr[i].compress_mode = OT_COMPRESS_MODE_NONE; + vpss_chn_attr[i].depth = 1; + } + + for (vpss_grp = 0; vpss_grp < vpss_grp_cnt; vpss_grp++) { + ret = sample_common_vpss_start(vpss_grp, chn_enable, &vpss_grp_attr, vpss_chn_attr, OT_VPSS_MAX_CHN_NUM); + if (ret != TD_SUCCESS) { + sample_svp_trace_err("failed with %#x!\n", ret); + return TD_FAILURE; + } + } + return TD_SUCCESS; +} + +/* function : Stop Vpss */ +static td_void sample_common_svp_stop_vpss(td_s32 vpss_grp_cnt, td_u32 vpss_chn_num) +{ + ot_vpss_grp vpss_grp = 0; + td_bool chn_enable[OT_VPSS_MAX_CHN_NUM] = { TD_FALSE, TD_FALSE, TD_FALSE, TD_FALSE }; + td_s32 i; + for (i = 0; (i < vpss_chn_num) && (i < OT_VPSS_MAX_CHN_NUM); i++) { + chn_enable[i] = TD_TRUE; + } + for (i = 0; (i < vpss_grp_cnt) && (i < OT_VPSS_MAX_CHN_NUM); i++) { + sample_common_vpss_stop(vpss_grp, chn_enable, OT_VPSS_MAX_CHN_NUM); + vpss_grp++; + } +} + +static td_s32 sample_common_svp_get_def_vo_cfg(sample_vo_cfg *vo_cfg) +{ + ot_rect def_disp_rect = {0, 0, 1920, 1080}; + ot_size def_img_size = {1920, 1080}; + + if (vo_cfg == TD_NULL) { + sample_svp_trace_err("error:argument can not be TD_NULL\n"); + return TD_FAILURE; + } + + vo_cfg->vo_dev = SAMPLE_VO_DEV_UHD; + + vo_cfg->vo_intf_type = OT_VO_INTF_HDMI; + vo_cfg->intf_sync = OT_VO_OUT_1080P60; + vo_cfg->bg_color = COLOR_RGB_BLACK; + vo_cfg->pix_format = OT_PIXEL_FORMAT_YVU_SEMIPLANAR_420; + vo_cfg->disp_rect = def_disp_rect; + vo_cfg->image_size = def_img_size; + vo_cfg->vo_part_mode = OT_VO_PARTITION_MODE_SINGLE; + vo_cfg->dis_buf_len = 3; /* 3: def buf len for single */ + vo_cfg->dst_dynamic_range = OT_DYNAMIC_RANGE_SDR8; + vo_cfg->vo_mode = VO_MODE_1MUX; + vo_cfg->compress_mode = OT_COMPRESS_MODE_NONE; + + return TD_SUCCESS; +} + +/* + * function : Start Vo + */ +static td_s32 sample_common_svp_start_vo(const ot_sample_svp_switch *vo_venc_switch, sample_vo_cfg *vo_cfg) +{ + td_s32 ret; + sample_svp_check_exps_return(vo_venc_switch == TD_NULL, TD_FAILURE, SAMPLE_SVP_ERR_LEVEL_ERROR, + "vo_venc_switch can't be null\n"); + sample_svp_check_exps_return(vo_cfg == TD_NULL, TD_FAILURE, SAMPLE_SVP_ERR_LEVEL_ERROR, "vo_cfg can't be null\n"); + + if (vo_venc_switch->is_vo_open == TD_FALSE) { + return TD_SUCCESS; + } + ret = sample_common_svp_get_def_vo_cfg(vo_cfg); + sample_svp_check_exps_return(ret != TD_SUCCESS, ret, SAMPLE_SVP_ERR_LEVEL_ERROR, + "Error(%#x),sample_common_svp_get_def_vo_cfg failed!\n", ret); + ret = sample_comm_vo_start_vo(vo_cfg); + sample_svp_check_exps_return(ret != TD_SUCCESS, ret, SAMPLE_SVP_ERR_LEVEL_ERROR, + "Error(%#x),sample_comm_vo_start_vo failed!\n", ret); + return ret; +} + +/* + * function : Stop Vo + */ +static td_void sample_common_svp_stop_vo(const ot_sample_svp_switch *vo_venc_switch, sample_vo_cfg *vo_cfg) +{ + sample_svp_check_exps_return_void(vo_venc_switch == TD_NULL, SAMPLE_SVP_ERR_LEVEL_ERROR, + "vo_venc_switch can't be null\n"); + sample_svp_check_exps_return_void(vo_cfg == TD_NULL, SAMPLE_SVP_ERR_LEVEL_ERROR, "vo_cfg can't be null\n"); + + if (vo_venc_switch->is_vo_open == TD_FALSE) { + return; + } + (td_void)sample_comm_vo_stop_vo(vo_cfg); +} + +static td_s32 sample_common_svp_vi_bind_multi_vpss(td_s32 vpss_grp_cnt, td_s32 vi_chn_cnt, + td_s32 vi_chn_interval) +{ + td_s32 ret; + td_s32 loop; + ot_vi_chn vi_chn; + ot_vpss_grp vpss_grp = 0; + + for (loop = 0; loop < vi_chn_cnt && vpss_grp < vpss_grp_cnt; loop++) { + vi_chn = loop * vi_chn_interval; + ret = sample_comm_vi_bind_vpss(0, vi_chn, vpss_grp, 0); + if (ret != TD_SUCCESS) { + sample_svp_trace_err("vi bind vpss failed!\n"); + return ret; + } + vpss_grp++; + } + + return TD_SUCCESS; +} + +static td_s32 sample_common_svp_vi_unbind_multi_vpss(td_s32 vpss_grp_cnt, td_s32 vi_chn_cnt, + td_s32 vi_chn_interval) +{ + td_s32 ret; + td_s32 loop; + ot_vi_chn vi_chn; + ot_vpss_grp vpss_grp = 0; + + for (loop = 0; loop < vi_chn_cnt && vpss_grp < vpss_grp_cnt; loop++) { + vi_chn = loop * vi_chn_interval; + ret = sample_comm_vi_un_bind_vpss(0, vi_chn, vpss_grp, 0); + if (ret != TD_SUCCESS) { + sample_svp_trace_err("vi bind vpss failed!\n"); + return ret; + } + + vpss_grp++; + } + + return TD_SUCCESS; +} + +static td_s32 sample_common_svp_set_vi_cfg(sample_vi_cfg *vi_cfg, ot_pic_size *pic_type, + td_u32 pic_type_len, ot_pic_size *ext_pic_size_type, sample_sns_type sns_type) +{ + sample_comm_vi_get_default_vi_cfg(sns_type, vi_cfg); + sample_svp_check_exps_return(pic_type_len < OT_VPSS_CHN_NUM, + TD_FAILURE, SAMPLE_SVP_ERR_LEVEL_ERROR, "pic_type_len is illegal!\n"); + pic_type[1] = *ext_pic_size_type; + + return TD_SUCCESS; +} + +static td_s32 sample_common_svp_start_venc(const ot_sample_svp_switch *switch_ptr, + sample_vo_cfg *vo_cfg, ot_pic_size *pic_type, td_u32 chn_num) +{ + td_s32 ret = TD_SUCCESS; + ot_venc_chn h264_chn = 0; + sample_comm_venc_chn_param chn_param; + ot_size ven_size = {1920, 1080}; + + sample_svp_check_exps_goto(chn_num == 0, end_0, SAMPLE_SVP_ERR_LEVEL_ERROR, "Error(%#x),chn_num(%u) can't be 0!\n", + ret, chn_num); + if (switch_ptr->is_venc_open == TD_TRUE) { + ret = sample_comm_venc_get_gop_attr(OT_VENC_GOP_MODE_NORMAL_P, &chn_param.gop_attr); + sample_svp_check_exps_goto(ret != TD_SUCCESS, end_0, SAMPLE_SVP_ERR_LEVEL_ERROR, + "Error(%#x),sample_comm_venc_get_gop_attr failed!\n", ret); + chn_param.frame_rate = 30; /* 30 is a number */ + chn_param.stats_time = 1; /* 1 is a number */ + chn_param.gop = 30; /* 30 is a number */ + chn_param.venc_size = ven_size; + chn_param.size = pic_type[0]; + chn_param.profile = 0; + chn_param.is_rcn_ref_share_buf = TD_TRUE; + + chn_param.type = OT_PT_H264; + chn_param.rc_mode = SAMPLE_RC_CBR; + + ret = sample_comm_venc_start(h264_chn, &chn_param); + sample_svp_check_exps_goto(ret != TD_SUCCESS, end_0, SAMPLE_SVP_ERR_LEVEL_ERROR, + "Error(%#x),sample_comm_venc_start failed!\n", ret); + + ret = sample_comm_venc_start_get_stream(&h264_chn, 1); + sample_svp_check_exps_goto(ret != TD_SUCCESS, end_1, SAMPLE_SVP_ERR_LEVEL_ERROR, + "Error(%#x),sample_comm_venc_start_get_stream failed!\n", ret); + } + return ret; + +end_1: + if (switch_ptr->is_venc_open == TD_TRUE) { + sample_comm_venc_stop(h264_chn); + } +end_0: + sample_common_svp_stop_vo(switch_ptr, vo_cfg); + return ret; +} + +/* + * function : Start Vi/Vpss/Venc/Vo + */ +td_s32 sample_common_svp_start_vi_vpss_venc_vo(sample_vi_cfg *vi_cfg, + ot_sample_svp_switch *switch_ptr, ot_pic_size *ext_pic_size_type) +{ + ot_size pic_size[OT_VPSS_CHN_NUM]; + ot_pic_size pic_type[OT_VPSS_CHN_NUM]; + sample_vo_cfg vo_cfg; + ot_pic_size venc_pic_type[SAMPLE_SVP_VDEC_CHN_NUM] = {PIC_1080P}; + + const td_s32 vpss_grp_cnt = 1; + td_s32 ret = TD_FAILURE; + sample_sns_type sns_type = SENSOR0_TYPE; + + sample_svp_check_exps_return(vi_cfg == TD_NULL, ret, SAMPLE_SVP_ERR_LEVEL_ERROR, "vi_cfg can't be null\n"); + sample_svp_check_exps_return(switch_ptr == TD_NULL, ret, SAMPLE_SVP_ERR_LEVEL_ERROR, "switch_ptr can't be null\n"); + sample_svp_check_exps_return(ext_pic_size_type == TD_NULL, ret, + SAMPLE_SVP_ERR_LEVEL_ERROR, "ext_pic_size_type can't be null\n"); + + ret = sample_common_svp_get_pic_type_by_sns_type(sns_type, pic_type, OT_VPSS_CHN_NUM); + sample_svp_check_exps_return(ret != TD_SUCCESS, ret, SAMPLE_SVP_ERR_LEVEL_ERROR, + "sample_common_svp_get_pic_type_by_sns_type failed!\n"); + ret = sample_common_svp_set_vi_cfg(vi_cfg, pic_type, OT_VPSS_CHN_NUM, ext_pic_size_type, sns_type); + sample_svp_check_exps_return(ret != TD_SUCCESS, ret, SAMPLE_SVP_ERR_LEVEL_ERROR, + "sample_common_svp_set_vi_cfg failed,Error:%#x\n", ret); + + /* step 1: Init vb */ + ret = sample_common_svp_vb_init(pic_type, pic_size, OT_VPSS_CHN_NUM); + sample_svp_check_exps_return(ret != TD_SUCCESS, ret, SAMPLE_SVP_ERR_LEVEL_ERROR, + "Error(%#x),sample_common_svp_vb_init failed!\n", ret); + + /* step 2: Start vi */ + ret = sample_comm_vi_start_vi(vi_cfg); + sample_svp_check_exps_goto(ret != TD_SUCCESS, end_init_1, SAMPLE_SVP_ERR_LEVEL_ERROR, + "Error(%#x),sample_comm_vi_start_vi failed!\n", ret); + + /* step 3: Bind vpss to vi */ + ret = sample_common_svp_vi_bind_multi_vpss(vpss_grp_cnt, 1, 1); + sample_svp_check_exps_goto(ret != TD_SUCCESS, end_init_2, SAMPLE_SVP_ERR_LEVEL_ERROR, + "Error(%#x),sample_common_vi_bind_multi_vpss failed!\n", ret); + + /* step 4: Start vpss */ + ret = sample_common_svp_start_vpss(vpss_grp_cnt, pic_size, OT_VPSS_CHN_NUM); + sample_svp_check_exps_goto(ret != TD_SUCCESS, end_init_3, SAMPLE_SVP_ERR_LEVEL_ERROR, + "Error(%#x),sample_common_svp_start_vpss failed!\n", ret); + + /* step 5: Set vi frame, Start Vo */ + ret = sample_common_svp_start_vo(switch_ptr, &vo_cfg); + sample_svp_check_exps_goto(ret != TD_SUCCESS, end_init_4, SAMPLE_SVP_ERR_LEVEL_ERROR, + "Error(%#x),sample_common_svp_set_vi_frame failed!\n", ret); + + /* step 6: Start Venc */ + ret = sample_common_svp_start_venc(switch_ptr, &vo_cfg, venc_pic_type, SAMPLE_SVP_VDEC_CHN_NUM); + sample_svp_check_exps_goto(ret != TD_SUCCESS, end_init_4, SAMPLE_SVP_ERR_LEVEL_ERROR, + "Error(%#x),sample_common_svp_start_vencb failed!\n", ret); + + return TD_SUCCESS; +end_init_4: + sample_common_svp_stop_vpss(vpss_grp_cnt, OT_VPSS_CHN_NUM); +end_init_3: + ret = sample_common_svp_vi_unbind_multi_vpss(vpss_grp_cnt, 1, 1); + sample_svp_check_exps_trace(ret != TD_SUCCESS, SAMPLE_SVP_ERR_LEVEL_ERROR, "svp_vi_unbind_multi_vpss failed!\n"); +end_init_2: + sample_comm_vi_stop_vi(vi_cfg); +end_init_1: /* system exit */ + sample_comm_sys_exit(); + (td_void)memset_s(vi_cfg, sizeof(sample_vi_cfg), 0, sizeof(sample_vi_cfg)); + return ret; +} +static td_void sample_common_svp_stop_venc(const ot_sample_svp_switch *vo_venc_switch) +{ + if (vo_venc_switch->is_venc_open == TD_TRUE) { + sample_comm_venc_stop_get_stream(1); + sample_comm_venc_stop(0); + } +} +/* + * function : Stop Vi/Vpss/Venc/Vo + */ +td_void sample_common_svp_stop_vi_vpss_venc_vo(sample_vi_cfg *vi_cfg, + ot_sample_svp_switch *switch_ptr) +{ + sample_vo_cfg vo_cfg = {0}; + td_s32 ret; + const td_s32 vpss_grp_cnt = 1; + + sample_svp_check_exps_return_void(vi_cfg == TD_NULL, SAMPLE_SVP_ERR_LEVEL_ERROR, "vi_cfg can't be null\n"); + sample_svp_check_exps_return_void(switch_ptr == TD_NULL, SAMPLE_SVP_ERR_LEVEL_ERROR, "switch_ptr can't be null\n"); + + sample_common_svp_stop_venc(switch_ptr); + + if (switch_ptr->is_vo_open == TD_TRUE) { + (td_void)sample_common_svp_get_def_vo_cfg(&vo_cfg); + sample_common_svp_stop_vo(switch_ptr, &vo_cfg); + } + + ret = sample_common_svp_vi_unbind_multi_vpss(vpss_grp_cnt, 1, 1); + sample_svp_check_exps_trace(ret != TD_SUCCESS, SAMPLE_SVP_ERR_LEVEL_ERROR, + "sample_common_svp_vi_unbind_multi_vpss failed\n"); + sample_common_svp_stop_vpss(vpss_grp_cnt, OT_VPSS_CHN_NUM); + sample_comm_vi_stop_vi(vi_cfg); + sample_comm_sys_exit(); + + (td_void)memset_s(vi_cfg, sizeof(sample_vi_cfg), 0, sizeof(sample_vi_cfg)); +} + +static td_s32 sample_common_svp_start_vdec(td_u32 chn_num, sample_vdec_attr *vdec_attr, td_u32 arr_len) +{ + td_s32 ret; + ret = sample_comm_vdec_init_vb_pool(chn_num, vdec_attr, arr_len); + sample_svp_check_exps_return(ret != TD_SUCCESS, ret, SAMPLE_SVP_ERR_LEVEL_ERROR, + "Error(%#x),sample_comm_vdec_init_vb_pool failed!\n", ret); + + ret = sample_comm_vdec_start(chn_num, vdec_attr, arr_len); + sample_svp_check_exps_goto(ret != TD_SUCCESS, end_1, SAMPLE_SVP_ERR_LEVEL_ERROR, + "Error(%#x), sample_comm_vdec_start failed!\n", ret); + + ret = ss_mpi_vdec_set_display_mode(SAMPLE_SVP_VDEC_CHN_0, OT_VIDEO_DISPLAY_MODE_PREVIEW); + sample_svp_check_exps_goto(ret != TD_SUCCESS, end_1, SAMPLE_SVP_ERR_LEVEL_ERROR, + "Error(%#x), sample_comm_vdec_start failed!\n", ret); + + return ret; +end_1: + sample_comm_vdec_exit_vb_pool(); + return ret; +} + +static td_void sample_common_svp_stop_vdec(td_u32 chn_num) +{ + td_s32 ret; + ret = sample_comm_vdec_stop(chn_num); + sample_svp_check_exps_trace(ret != TD_SUCCESS, SAMPLE_SVP_ERR_LEVEL_ERROR, "sample_comm_vdec_stop failed\n"); + sample_comm_vdec_exit_vb_pool(); +} + +static td_s32 sample_common_svp_vpss_bind_vdec(td_s32 vpss_grp_cnt, td_s32 vdec_chn) +{ + td_s32 i; + td_s32 j; + td_s32 ret; + for (i = 0; i < vpss_grp_cnt; i++) { + ret = sample_comm_vdec_bind_vpss(vdec_chn, i); + sample_svp_check_exps_goto(ret != TD_SUCCESS, end_1, SAMPLE_SVP_ERR_LEVEL_ERROR, + "Error(%#x), sample_comm_vdec_bind_vpss failed!\n", ret); + } + return ret; +end_1: + for (j = 0; j < i; j++) { + sample_comm_vdec_un_bind_vpss(vdec_chn, j); + } + return ret; +} + +static td_void sample_common_svp_vpss_unbind_vdec(td_s32 vpss_grp_cnt, td_s32 vdec_chn) +{ + td_s32 i; + for (i = 0; i < vpss_grp_cnt; i++) { + sample_comm_vdec_un_bind_vpss(vdec_chn, i); + } +} + +static td_s32 sample_common_svp_create_vb(ot_sample_svp_media_cfg *media_cfg) +{ + td_s32 ret = OT_INVALID_VALUE; + td_u32 i; + td_u32 j; + td_u64 blk_size; + ot_vb_pool_cfg vb_pool_cfg = { 0 }; + ot_pic_buf_attr pic_buf_attr = {0}; + + sample_svp_check_exps_return(media_cfg == TD_NULL, ret, SAMPLE_SVP_ERR_LEVEL_ERROR, "media_cfg is NULL!\n"); + + sample_svp_check_exps_return((media_cfg->chn_num == 0) || (media_cfg->chn_num > OT_SVP_MAX_VPSS_CHN_NUM), ret, + SAMPLE_SVP_ERR_LEVEL_ERROR, "media_cfg->chn_num must be [1, %d],Error(%#x)!\n", OT_SVP_MAX_VPSS_CHN_NUM, ret); + + for (i = 0; i < media_cfg->chn_num; i++) { + if (media_cfg->pic_type[i] != PIC_BUTT) { + ret = sample_comm_sys_get_pic_size(media_cfg->pic_type[i], &media_cfg->pic_size[i]); + sample_svp_check_exps_goto(ret != TD_SUCCESS, vb_fail_0, SAMPLE_SVP_ERR_LEVEL_ERROR, + "sample_comm_sys_get_pic_size failed,Error(%#x)!\n", ret); + } + pic_buf_attr.width = media_cfg->pic_size[i].width; + pic_buf_attr.height = media_cfg->pic_size[i].height; + pic_buf_attr.compress_mode = OT_COMPRESS_MODE_NONE; + pic_buf_attr.align = OT_DEFAULT_ALIGN; + pic_buf_attr.bit_width = OT_DATA_BIT_WIDTH_8; + pic_buf_attr.pixel_format = OT_PIXEL_FORMAT_YVU_SEMIPLANAR_422; + blk_size = ot_common_get_pic_buf_size(&pic_buf_attr); + vb_pool_cfg.blk_size = blk_size; + vb_pool_cfg.blk_cnt = SAMPLE_SVP_BLK_CNT; + media_cfg->vb_pool[i] = ss_mpi_vb_create_pool(&vb_pool_cfg); + } + return TD_SUCCESS; +vb_fail_0: + for (j = 0; j < i; j++) { + (td_void)ss_mpi_vb_destroy_pool(media_cfg->vb_pool[j]); + } + return TD_FAILURE; +} + +static td_void sample_common_svp_destroy_vb(ot_sample_svp_media_cfg *media_cfg) +{ + td_u32 i; + for (i = 0; i < media_cfg->chn_num; i++) { + (td_void)ss_mpi_vb_destroy_pool(media_cfg->vb_pool[i]); + } +} +static td_s32 sample_common_svp_vpss_attach_vb(ot_sample_svp_media_cfg *media_cfg, td_s32 vpss_grp_cnt) +{ + ot_vpss_grp grp; + ot_vpss_grp grp_tmp; + ot_vpss_chn chn; + ot_vpss_grp chn_tmp; + td_s32 ret; + + for (grp = 0; grp < vpss_grp_cnt; grp++) { + for (chn = 0; chn < media_cfg->chn_num; chn++) { + ret = ss_mpi_vpss_attach_vb_pool(grp, chn, media_cfg->vb_pool[chn]); + if (ret == TD_SUCCESS) { + continue; + } + for (chn_tmp = 0; chn_tmp < chn; chn_tmp++) { + (td_void)ss_mpi_vpss_detach_vb_pool(grp, chn_tmp); + } + sample_svp_check_exps_goto(ret != TD_SUCCESS, end_0, SAMPLE_SVP_ERR_LEVEL_ERROR, + "Error(%#x), ss_mpi_vpss_attach_vb_pool failed!\n", ret); + } + } + return ret; +end_0: + for (grp_tmp = 0; grp_tmp < grp; grp_tmp++) { + for (chn_tmp = 0; chn_tmp < media_cfg->chn_num; chn_tmp++) { + (td_void)ss_mpi_vpss_detach_vb_pool(grp_tmp, chn_tmp); + } + } + return ret; +} + +static td_void sample_common_svp_vpss_detach_vb(ot_sample_svp_media_cfg *media_cfg, td_s32 vpss_grp_cnt) +{ + ot_vpss_grp grp; + ot_vpss_chn chn; + + for (grp = 0; grp < vpss_grp_cnt; grp++) { + for (chn = 0; chn < media_cfg->chn_num; chn++) { + (td_void)ss_mpi_vpss_detach_vb_pool(grp, chn); + } + } +} + +td_s32 sample_common_svp_create_vb_start_vdec_vpss_vo(sample_vdec_attr *vdec_attr, vdec_thread_param *vdec_send, + pthread_t *vdec_thread, ot_sample_svp_media_cfg *media_cfg, sample_vo_cfg *vo_cfg) +{ + const td_s32 vpss_grp_cnt = 1; + td_s32 ret; + /* step 1: Init vb */ + ret = sample_common_svp_create_vb(media_cfg); + sample_svp_check_exps_return(ret != TD_SUCCESS, ret, SAMPLE_SVP_ERR_LEVEL_ERROR, + "Error(%#x),sample_common_svp_create_vb failed!\n", ret); + /* step 2: start vdec */ + ret = sample_common_svp_start_vdec(SAMPLE_SVP_VDEC_CHN_NUM, vdec_attr, SAMPLE_SVP_VDEC_CHN_NUM); + sample_svp_check_exps_goto(ret != TD_SUCCESS, end_init_1, SAMPLE_SVP_ERR_LEVEL_ERROR, + "Error(%#x), sample_common_svp_start_vdec failed!\n", ret); + /* step 3: Start vpss */ + ret = sample_common_svp_start_vpss(vpss_grp_cnt, media_cfg->pic_size, media_cfg->chn_num); + sample_svp_check_exps_goto(ret != TD_SUCCESS, end_init_2, SAMPLE_SVP_ERR_LEVEL_ERROR, + "Error(%#x),sample_common_svp_start_vpss failed!\n", ret); + + ret = sample_common_svp_vpss_attach_vb(media_cfg, vpss_grp_cnt); + sample_svp_check_exps_goto(ret != TD_SUCCESS, end_init_3, SAMPLE_SVP_ERR_LEVEL_ERROR, + "Error(%#x),sample_common_svp_start_vpss failed!\n", ret); + + /* step 4: Bind vpss to vdec */ + ret = sample_common_svp_vpss_bind_vdec(vpss_grp_cnt, SAMPLE_SVP_VDEC_CHN_0); + sample_svp_check_exps_goto(ret != TD_SUCCESS, end_init_4, SAMPLE_SVP_ERR_LEVEL_ERROR, + "Error(%#x),sample_comm_vdec_bind_vpss failed!\n", ret); + + /* step 5: Start vo */ + ret = sample_common_svp_start_vo(&(media_cfg->svp_switch), vo_cfg); + sample_svp_check_exps_goto(ret != TD_SUCCESS, end_init_5, SAMPLE_SVP_ERR_LEVEL_ERROR, + "Error(%#x),sample_common_svp_start_vo failed!\n", ret); + + /* step 5: Start Send thread */ + sample_comm_vdec_start_send_stream(SAMPLE_SVP_VDEC_CHN_NUM, vdec_send, vdec_thread, SAMPLE_SVP_VDEC_CHN_NUM, + SAMPLE_SVP_VDEC_CHN_NUM); + + /* step 6: Start Venc */ + ret = sample_common_svp_start_venc(&(media_cfg->svp_switch), vo_cfg, media_cfg->pic_type, + SAMPLE_SVP_VDEC_CHN_NUM); + sample_svp_check_exps_goto(ret != TD_SUCCESS, end_init_6, SAMPLE_SVP_ERR_LEVEL_ERROR, + "Error(%#x),sample_common_svp_start_venc failed!\n", ret); + + return ret; +end_init_6: + sample_comm_vdec_stop_send_stream(SAMPLE_SVP_VDEC_CHN_NUM, vdec_send, vdec_thread, SAMPLE_SVP_VDEC_CHN_NUM, + SAMPLE_SVP_VDEC_CHN_NUM); + sample_common_svp_stop_vo(&(media_cfg->svp_switch), vo_cfg); +end_init_5: + sample_common_svp_vpss_unbind_vdec(vpss_grp_cnt, SAMPLE_SVP_VDEC_CHN_0); +end_init_4: + sample_common_svp_vpss_detach_vb(media_cfg, vpss_grp_cnt); +end_init_3: + sample_common_svp_stop_vpss(vpss_grp_cnt, media_cfg->chn_num); +end_init_2: + sample_common_svp_stop_vdec(SAMPLE_SVP_VDEC_CHN_NUM); +end_init_1: + sample_common_svp_destroy_vb(media_cfg); + return ret; +} + +td_void sample_common_svp_destroy_vb_stop_vdec_vpss_vo(vdec_thread_param *vdec_send, pthread_t *vdec_thread, + ot_sample_svp_media_cfg *media_cfg, sample_vo_cfg *vo_cfg) +{ + const td_s32 vpss_grp_cnt = 1; + + sample_svp_check_exps_return_void(media_cfg == TD_NULL, SAMPLE_SVP_ERR_LEVEL_ERROR, "media_cfg is NULL!\n"); + + sample_svp_check_exps_return_void((media_cfg->chn_num == 0) || (media_cfg->chn_num > OT_SVP_MAX_VPSS_CHN_NUM), + SAMPLE_SVP_ERR_LEVEL_ERROR, "media_cfg->chn_num must be [1, %u]!\n", OT_SVP_MAX_VPSS_CHN_NUM); + + sample_common_svp_stop_venc(&(media_cfg->svp_switch)); + + sample_comm_vdec_stop_send_stream(SAMPLE_SVP_VDEC_CHN_NUM, vdec_send, vdec_thread, SAMPLE_SVP_VDEC_CHN_NUM, + SAMPLE_SVP_VDEC_CHN_NUM); + sample_common_svp_stop_vo(&(media_cfg->svp_switch), vo_cfg); + sample_common_svp_vpss_unbind_vdec(vpss_grp_cnt, SAMPLE_SVP_VDEC_CHN_0); + sample_common_svp_vpss_detach_vb(media_cfg, vpss_grp_cnt); + sample_common_svp_stop_vpss(vpss_grp_cnt, media_cfg->chn_num); + sample_common_svp_stop_vdec(SAMPLE_SVP_VDEC_CHN_NUM); + sample_common_svp_destroy_vb(media_cfg); +} + +td_s32 sample_common_svp_venc_vo_send_stream(const ot_sample_svp_switch *vo_venc_switch, ot_venc_chn venc_chn, + ot_vo_layer vo_layer, ot_vo_chn vo_chn, const ot_video_frame_info *frame) +{ + td_s32 ret = OT_INVALID_VALUE; + + sample_svp_check_exps_return((vo_venc_switch == TD_NULL || frame == TD_NULL), ret, SAMPLE_SVP_ERR_LEVEL_ERROR, + "vo_venc_switch is NULL!\n"); + if (vo_venc_switch->is_venc_open) { + ret = ss_mpi_venc_send_frame(venc_chn, frame, OT_SVP_TIMEOUT); + sample_svp_check_exps_return(ret != TD_SUCCESS, ret, SAMPLE_SVP_ERR_LEVEL_ERROR, + "Error(%#x),ss_mpi_venc_send_frame failed!\n", ret); + } + if (vo_venc_switch->is_vo_open) { + ret = ss_mpi_vo_send_frame(vo_layer, vo_chn, frame, OT_SVP_TIMEOUT); + sample_svp_check_exps_return(ret != TD_SUCCESS, ret, SAMPLE_SVP_ERR_LEVEL_ERROR, + "Error(%#x),ss_mpi_vo_send_frame failed!\n", ret); + } + return TD_SUCCESS; +} + +static td_s32 sample_common_svp_npu_get_pic_size(ot_sample_svp_media_cfg *media_cfg) +{ + td_s32 ret; + ret = sample_comm_sys_get_pic_size(media_cfg->pic_type[0], &media_cfg->pic_size[0]); + sample_svp_check_exps_return(ret != TD_SUCCESS, ret, SAMPLE_SVP_ERR_LEVEL_ERROR, + "sample_comm_sys_get_pic_size failed,Error(%#x)!\n", ret); + return TD_SUCCESS; +} + +/* + * function : Start Vi/Vpss/Vo + */ +td_s32 sample_common_svp_npu_start_vi_vpss_vo(sample_vi_cfg *vi_cfg, ot_sample_svp_media_cfg *media_cfg, + sample_vo_cfg *vo_cfg) +{ + td_s32 ret; + const td_s32 vpss_grp_cnt = 1; + sample_sns_type sns_type = SENSOR0_TYPE; + + ret = sample_common_svp_get_pic_type_by_sns_type(sns_type, media_cfg->pic_type, OT_SVP_MAX_VPSS_CHN_NUM); + sample_svp_check_exps_return(ret != TD_SUCCESS, ret, SAMPLE_SVP_ERR_LEVEL_ERROR, + "sample_common_svp_get_pic_type_by_sns_type failed!\n"); + sample_comm_vi_get_default_vi_cfg(sns_type, vi_cfg); + + ret = sample_common_svp_npu_get_pic_size(media_cfg); + sample_svp_check_exps_return(ret != TD_SUCCESS, ret, SAMPLE_SVP_ERR_LEVEL_ERROR, + "Error(%#x):sample_common_svp_npu_get_pic_size failed!\n", ret); + + /* step 1: Init vb */ + ret = sample_common_svp_create_vb(media_cfg); + sample_svp_check_exps_return(ret != TD_SUCCESS, ret, SAMPLE_SVP_ERR_LEVEL_ERROR, + "Error(%#x),sample_common_svp_create_vb failed!\n", ret); + + /* step 2: Start vi */ + ret = sample_comm_vi_start_vi(vi_cfg); + sample_svp_check_exps_goto(ret != TD_SUCCESS, end_init_1, SAMPLE_SVP_ERR_LEVEL_ERROR, + "Error(%#x),sample_comm_vi_start_vi failed!\n", ret); + + /* step 3: Bind vpss to vi */ + ret = sample_common_svp_vi_bind_multi_vpss(vpss_grp_cnt, 1, 1); + sample_svp_check_exps_goto(ret != TD_SUCCESS, end_init_2, SAMPLE_SVP_ERR_LEVEL_ERROR, + "Error(%#x),sample_common_vi_bind_multi_vpss failed!\n", ret); + + /* step 4: Start vpss */ + ret = sample_common_svp_start_vpss(vpss_grp_cnt, media_cfg->pic_size, media_cfg->chn_num); + sample_svp_check_exps_goto(ret != TD_SUCCESS, end_init_3, SAMPLE_SVP_ERR_LEVEL_ERROR, + "Error(%#x),sample_common_svp_start_vpss failed!\n", ret); + + /* step 5: attach vb */ + ret = sample_common_svp_vpss_attach_vb(media_cfg, vpss_grp_cnt); + sample_svp_check_exps_goto(ret != TD_SUCCESS, end_init_4, SAMPLE_SVP_ERR_LEVEL_ERROR, + "Error(%#x),sample_common_svp_start_vpss failed!\n", ret); + + /* step 6: Start vo */ + ret = sample_common_svp_start_vo(&(media_cfg->svp_switch), vo_cfg); + sample_svp_check_exps_goto(ret != TD_SUCCESS, end_init_5, SAMPLE_SVP_ERR_LEVEL_ERROR, + "Error(%#x),sample_common_svp_start_vo failed!\n", ret); + + return ret; + +end_init_5: + sample_common_svp_vpss_detach_vb(media_cfg, vpss_grp_cnt); +end_init_4: + sample_common_svp_stop_vpss(vpss_grp_cnt, media_cfg->chn_num); +end_init_3: + sample_common_svp_vi_unbind_multi_vpss(vpss_grp_cnt, 1, 1); +end_init_2: + sample_comm_vi_stop_vi(vi_cfg); +end_init_1: + sample_common_svp_destroy_vb(media_cfg); + return ret; +} + +td_void sample_common_svp_destroy_vb_stop_vi_vpss_vo(sample_vi_cfg *vi_cfg, ot_sample_svp_media_cfg *media_cfg, + sample_vo_cfg *vo_cfg) +{ + const td_s32 vpss_grp_cnt = 1; + + sample_svp_check_exps_return_void(media_cfg == TD_NULL, SAMPLE_SVP_ERR_LEVEL_ERROR, "media_cfg is NULL!\n"); + + sample_svp_check_exps_return_void((media_cfg->chn_num == 0) || (media_cfg->chn_num > OT_SVP_MAX_VPSS_CHN_NUM), + SAMPLE_SVP_ERR_LEVEL_ERROR, "media_cfg->chn_num must be [1, %u]!\n", OT_SVP_MAX_VPSS_CHN_NUM); + + sample_common_svp_stop_vo(&(media_cfg->svp_switch), vo_cfg); + sample_common_svp_vpss_detach_vb(media_cfg, vpss_grp_cnt); + sample_common_svp_stop_vpss(vpss_grp_cnt, media_cfg->chn_num); + sample_common_svp_vi_unbind_multi_vpss(vpss_grp_cnt, 1, 1); + sample_comm_vi_stop_vi(vi_cfg); + sample_common_svp_destroy_vb(media_cfg); +} diff --git a/vendor/zsks/demo/fruit_identify/common/sample_common_svp.h b/vendor/zsks/demo/fruit_identify/common/sample_common_svp.h new file mode 100755 index 0000000000000000000000000000000000000000..4ba556b17669a3b834ffea68f50343a60fc6d3ea --- /dev/null +++ b/vendor/zsks/demo/fruit_identify/common/sample_common_svp.h @@ -0,0 +1,298 @@ +/* + * Copyright (c) 2025 HiSilicon (Shanghai) Technologies Co., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef SAMPLE_COMMON_SVP_H +#define SAMPLE_COMMON_SVP_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "ss_mpi_sys.h" +#include "ot_common.h" +#include "ot_common_svp.h" +#include "sample_comm.h" + +#ifdef __cplusplus +#if __cplusplus +extern "C"{ +#endif +#endif /* __cplusplus */ + +#define OT_SVP_RECT_NUM 64 +#define OT_POINT_NUM 4 +#define OT_SVP_MAX_VPSS_CHN_NUM 2 +#define OT_SVP_TIMEOUT 2000 + +typedef enum { + SAMPLE_SVP_ERR_LEVEL_DEBUG = 0x0, /* debug-level */ + SAMPLE_SVP_ERR_LEVEL_INFO = 0x1, /* informational */ + SAMPLE_SVP_ERR_LEVEL_NOTICE = 0x2, /* normal but significant condition */ + SAMPLE_SVP_ERR_LEVEL_WARNING = 0x3, /* warning conditions */ + SAMPLE_SVP_ERR_LEVEL_ERROR = 0x4, /* error conditions */ + SAMPLE_SVP_ERR_LEVEL_CRIT = 0x5, /* critical conditions */ + SAMPLE_SVP_ERR_LEVEL_ALERT = 0x6, /* action must be taken immediately */ + SAMPLE_SVP_ERR_LEVEL_FATAL = 0x7, /* just for compatibility with previous version */ + + SAMPLE_SVP_ERR_LEVEL_BUTT +} sample_svp_err_level; + +typedef struct { + ot_point point[OT_POINT_NUM]; + td_float score; + td_u16 class_id; +} ot_sample_svp_rect; + +typedef struct { + td_u16 num; + ot_sample_svp_rect rect[OT_SVP_RECT_NUM]; +} ot_sample_svp_rect_info; + +typedef struct { + td_bool is_venc_open; + td_bool is_vo_open; +} ot_sample_svp_switch; + +typedef struct { + ot_sample_svp_switch svp_switch; + ot_size pic_size[OT_SVP_MAX_VPSS_CHN_NUM]; + ot_pic_size pic_type[OT_SVP_MAX_VPSS_CHN_NUM]; + ot_vb_pool vb_pool[OT_SVP_MAX_VPSS_CHN_NUM]; + td_u32 chn_num; +} ot_sample_svp_media_cfg; + +#define sample_svp_printf(level_str, msg, ...) \ +do { \ + fprintf(stderr, "[level]:%s,[func]:%s [line]:%d [info]:"msg, level_str, __FUNCTION__, __LINE__, ##__VA_ARGS__); \ +} while (0) + +#define sample_svp_printf_red(level_str, msg, ...) \ +do { \ + fprintf(stderr, "\033[0;31m [level]:%s,[func]:%s [line]:%d [info]:"msg"\033[0;39m\n", level_str, __FUNCTION__, \ + __LINE__, ##__VA_ARGS__); \ +} while (0) +/* system is unusable */ +#define sample_svp_trace_fatal(msg, ...) sample_svp_printf_red("Fatal", msg, ##__VA_ARGS__) +/* action must be taken immediately */ +#define sample_svp_trace_alert(msg, ...) sample_svp_printf_red("Alert", msg, ##__VA_ARGS__) +/* critical conditions */ +#define sample_svp_trace_critical(msg, ...) sample_svp_printf_red("Critical", msg, ##__VA_ARGS__) +/* error conditions */ +#define sample_svp_trace_err(msg, ...) sample_svp_printf_red("Error", msg, ##__VA_ARGS__) +/* warning conditions */ +#define sample_svp_trace_warning(msg, ...) sample_svp_printf("Warning", msg, ##__VA_ARGS__) +/* normal but significant condition */ +#define sample_svp_trace_notic(msg, ...) sample_svp_printf("Notice", msg, ##__VA_ARGS__) +/* informational */ +#define sample_svp_trace_info(msg, ...) sample_svp_printf("Info", msg, ##__VA_ARGS__) +/* debug-level messages */ +#define sample_svp_trace_debug(msg, ...) sample_svp_printf("Debug", msg, ##__VA_ARGS__) + +/* exps is true, goto */ +#define sample_svp_check_exps_goto(exps, label, level, msg, ...) \ +do { \ + if ((exps)) { \ + sample_svp_trace_err(msg, ## __VA_ARGS__); \ + goto label; \ + } \ +} while (0) +/* exps is true, return td_void */ +#define sample_svp_check_exps_return_void(exps, level, msg, ...) \ +do { \ + if ((exps)) { \ + sample_svp_trace_err(msg, ##__VA_ARGS__); \ + return; \ + } \ +} while (0) +/* exps is true, return ret */ +#define sample_svp_check_exps_return(exps, ret, level, msg, ...) \ +do { \ + if ((exps)) { \ + sample_svp_trace_err(msg, ##__VA_ARGS__); \ + return (ret); \ + } \ +} while (0) \ +/* exps is true, trace */ +#define sample_svp_check_exps_trace(exps, level, msg, ...) \ +do { \ + if ((exps)) { \ + sample_svp_trace_err(msg, ##__VA_ARGS__); \ + } \ +} while (0) + +#define sample_svp_check_exps_continue(exps, level, msg, ...) \ +do { \ + if ((exps)) { \ + sample_svp_trace_err(msg, ##__VA_ARGS__); \ + continue; \ + } \ +} while (0) \ + +/* exps is not success, trace */ +#define sample_svp_check_failed_trace(exps, level, msg, ...) \ +do { \ + if ((exps) != TD_SUCCESS) { \ + sample_svp_trace_err(msg, ##__VA_ARGS__); \ + } \ +} while (0) + +/* exps is not success, goto */ +#define sample_svp_check_failed_goto(exps, label, level, msg, ...) \ +do { \ + if ((exps) != TD_SUCCESS) { \ + sample_svp_trace_err(msg, ## __VA_ARGS__); \ + goto label; \ + } \ +} while (0) \ + +/* exps is not sucecss, return */ +#define sample_svp_check_failed_return(exps, ret, level, msg, ...) \ +do { \ + if ((exps) != TD_SUCCESS) { \ + sample_svp_trace_err(msg, ##__VA_ARGS__); \ + return (ret); \ + } \ +} while (0) + +/* exps is not equal to success, goto with level SAMPLE_SVP_ERR_LEVEL_ERROR */ +#define sample_svp_check_failed_err_level_goto(exps, label, msg, ...) \ +do { \ + if ((exps) != TD_SUCCESS) { \ + sample_svp_trace_err(msg, ## __VA_ARGS__); \ + goto label; \ + } \ +} while (0) + +/* exps is not equal to success, return with level SAMPLE_SVP_ERR_LEVEL_ERROR */ +#define sample_svp_check_failed_err_level_return(exps, ret, msg, ...) \ +do { \ + if ((exps) != TD_SUCCESS) { \ + sample_svp_trace_err(msg, ## __VA_ARGS__); \ + return (ret); \ + } \ +} while (0) + +/* exps is not equal success, trace with level SAMPLE_SVP_ERR_LEVEL_ERROR */ +#define sample_svp_check_failed_err_level_trace(exps, msg, ...) \ +do { \ + if ((exps) != TD_SUCCESS) { \ + sample_svp_trace_err(msg, ##__VA_ARGS__); \ + } \ +} while (0) + +#define sample_svp_pause() \ +do { \ + printf("---------------press Enter key to exit!---------------\n"); \ + (void)getchar(); \ +} while (0) + +#define SAMPLE_SVP_VB_POOL_NUM 2 +#define SAMPLE_SVP_ALIGN_16 16 +#define SAMPLE_SVP_ALIGN_32 32 +#define SAMPLE_SVP_D1_PAL_HEIGHT 576 +#define SAMPLE_SVP_D1_PAL_WIDTH 704 +#define sample_svp_convert_addr_to_ptr(type, addr) ((type *)(td_uintptr_t)(addr)) +#define sample_svp_convert_ptr_to_addr(type, addr) ((type)(td_uintptr_t)(addr)) + +/* free mmz */ +#define sample_svp_mmz_free(phys, virt) \ +do { \ + if (((phys) != 0) && ((virt) != 0)) { \ + ss_mpi_sys_mmz_free((td_phys_addr_t)(phys), (td_void*)(td_uintptr_t)(virt)); \ + (phys) = 0; \ + (virt) = 0; \ + } \ +} while (0) + +#define sample_svp_close_file(fp) \ +do { \ + if ((fp) != TD_NULL) { \ + fclose((fp)); \ + (fp) = TD_NULL; \ + } \ +} while (0) + +/* System init */ +td_s32 sample_common_svp_check_sys_init(td_bool vi_en); + +/* System exit */ +td_void sample_common_svp_check_sys_exit(td_void); + +/* Align */ +td_u32 sample_common_svp_align(td_u32 size, td_u16 align); + +/* Create mem info */ +td_s32 sample_common_svp_create_mem_info(ot_svp_mem_info *mem_info, td_u32 size, td_u32 addr_offsset); + +/* Destory mem info */ +td_void sample_common_svp_destroy_mem_info(ot_svp_mem_info *mem_info, td_u32 addr_offsset); + +/* Malloc memory */ +td_s32 sample_common_svp_malloc_mem(td_char *mmb, td_char *zone, td_phys_addr_t *phys_addr, + td_void **virt_addr, td_u32 size); + +/* Malloc memory with cached */ +td_s32 sample_common_svp_malloc_cached(td_char *mmb, td_char *zone, td_phys_addr_t *phys_addr, + td_void **virt_addr, td_u32 size); + +/* Fulsh cached */ +td_s32 sample_common_svp_flush_cache(td_phys_addr_t phys_addr, td_void *virt_addr, td_u32 size); + +/* function : Call vgs to fill rect */ +td_s32 sample_common_svp_vgs_fill_rect(const ot_video_frame_info *frame_info, + ot_sample_svp_rect_info *rect, td_u32 color); + +/* function : Start Vi/Vpss/Venc/Vo */ +td_s32 sample_common_svp_start_vi_vpss_venc_vo(sample_vi_cfg *vi_config, + ot_sample_svp_switch *switch_ptr, ot_pic_size *ext_pic_size_type); + +/* function : Stop Vi/Vpss/Venc/Vo */ +td_void sample_common_svp_stop_vi_vpss_venc_vo(sample_vi_cfg *vi_config, + ot_sample_svp_switch *switch_ptr); + +/* + * Start Vdec/Vpss/Vo + */ +td_s32 sample_common_svp_create_vb_start_vdec_vpss_vo(sample_vdec_attr *vdec_attr, + vdec_thread_param *vdec_send, pthread_t *vdec_thread, ot_sample_svp_media_cfg *media_cfg, + sample_vo_cfg *vo_cfg); +/* + * Stop Vdec/Vpss/Vo + */ +td_void sample_common_svp_destroy_vb_stop_vdec_vpss_vo(vdec_thread_param *vdec_send, + pthread_t *vdec_thread, ot_sample_svp_media_cfg *media_cfg, sample_vo_cfg *vo_cfg); +/* + * Send stream to venc/vo + */ +td_s32 sample_common_svp_venc_vo_send_stream(const ot_sample_svp_switch *vo_venc_switch, + ot_venc_chn venc_chn, ot_vo_layer vo_layer, ot_vo_chn vo_chn, const ot_video_frame_info *frame); + +td_s32 sample_common_svp_npu_start_vi_vpss_vo(sample_vi_cfg *vi_cfg, ot_sample_svp_media_cfg *media_cfg, + sample_vo_cfg *vo_cfg); + +td_void sample_common_svp_destroy_vb_stop_vi_vpss_vo(sample_vi_cfg *vi_cfg, ot_sample_svp_media_cfg *media_cfg, + sample_vo_cfg *vo_cfg); +#ifdef __cplusplus +#if __cplusplus +} +#endif +#endif /* __cplusplus */ +#endif /* SAMPLE_COMMON_SVP_H */ diff --git a/vendor/zsks/demo/fruit_identify/common/sample_common_svp_npu.c b/vendor/zsks/demo/fruit_identify/common/sample_common_svp_npu.c new file mode 100755 index 0000000000000000000000000000000000000000..64ae73c394907fe461c04bc7611bcce526affccf --- /dev/null +++ b/vendor/zsks/demo/fruit_identify/common/sample_common_svp_npu.c @@ -0,0 +1,69 @@ +/* + * Copyright (c) 2025 HiSilicon (Shanghai) Technologies Co., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "sample_common_svp_npu.h" +#include "ot_common.h" +#include "ot_common_svp.h" +#include "sample_comm.h" +#include "sample_common_svp.h" + +/* acl init */ + td_s32 sample_common_svp_npu_acl_init(const td_char *acl_config_path, td_s32 dev_id) +{ + svp_acl_rt_run_mode run_mode; + svp_acl_error ret; + /* svp acl init */ + ret = svp_acl_init(acl_config_path); + sample_svp_check_exps_return(ret != SVP_ACL_SUCCESS, TD_FAILURE, SAMPLE_SVP_ERR_LEVEL_ERROR, "acl init failed!\n"); + sample_svp_trace_info("svp acl init success!\n"); + /* open device */ + ret = svp_acl_rt_set_device(dev_id); + sample_svp_check_exps_goto(ret != SVP_ACL_SUCCESS, end_0, SAMPLE_SVP_ERR_LEVEL_ERROR, + "svp acl open device %d failed!\n", dev_id); + + sample_svp_trace_info("open device %d success!\n", dev_id); + /* get run mode */ + ret = svp_acl_rt_get_run_mode(&run_mode); + sample_svp_check_exps_goto(ret != SVP_ACL_SUCCESS, end_1, SAMPLE_SVP_ERR_LEVEL_ERROR, + "acl get run mode failed!\n"); + sample_svp_check_exps_goto(run_mode != SVP_ACL_DEVICE, end_1, SAMPLE_SVP_ERR_LEVEL_ERROR, + "run mode(%d) is not equal to %d!\n", run_mode, SVP_ACL_DEVICE); + + sample_svp_trace_info("get run mode success!\n"); + return TD_SUCCESS; +end_1: + (td_void)svp_acl_rt_reset_device(dev_id); +end_0: + (td_void)svp_acl_finalize(); + return TD_FAILURE; +} + +/* acl deinit */ +td_void sample_common_svp_npu_acl_deinit(td_s32 dev_id) +{ + svp_acl_error ret; + + ret = svp_acl_rt_reset_device(dev_id); + if (ret != SVP_ACL_SUCCESS) { + sample_svp_trace_err("reset device fail\n"); + } + sample_svp_trace_info("end to reset device is %d\n", dev_id); + + ret = svp_acl_finalize(); + if (ret != SVP_ACL_SUCCESS) { + sample_svp_trace_err("finalize acl fail\n"); + } + sample_svp_trace_info("end to finalize acl\n"); +} diff --git a/vendor/zsks/demo/fruit_identify/common/sample_common_svp_npu.h b/vendor/zsks/demo/fruit_identify/common/sample_common_svp_npu.h new file mode 100755 index 0000000000000000000000000000000000000000..79c1e54cf7bdc1f99e3b78d8719b1a8d79b97138 --- /dev/null +++ b/vendor/zsks/demo/fruit_identify/common/sample_common_svp_npu.h @@ -0,0 +1,163 @@ +/* + * Copyright (c) 2025 HiSilicon (Shanghai) Technologies Co., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef SAMPLE_COMMON_SVP_NPU_H +#define SAMPLE_COMMON_SVP_NPU_H +#include "ot_type.h" +#include "svp_acl.h" +#include "svp_acl_mdl.h" +#include "xvp_common_osd.h" + +#define SAMPLE_SVP_NPU_MAX_THREAD_NUM 16 +#define SAMPLE_SVP_NPU_MAX_TASK_NUM 16 +#define SAMPLE_SVP_NPU_MAX_MODEL_NUM 1 +#define SAMPLE_SVP_NPU_EXTRA_INPUT_NUM 2 +#define SAMPLE_SVP_NPU_BYTE_BIT_NUM 8 +#define SAMPLE_SVP_NPU_SHOW_TOP_NUM 5 +#define SAMPLE_SVP_NPU_MAX_NAME_LEN 32 +#define SAMPLE_SVP_NPU_MAX_MEM_SIZE 0xFFFFFFFF +#define SAMPLE_SVP_NPU_RECT_LEFT_TOP 0 +#define SAMPLE_SVP_NPU_RECT_RIGHT_TOP 1 +#define SAMPLE_SVP_NPU_RECT_RIGHT_BOTTOM 2 +#define SAMPLE_SVP_NPU_RECT_LEFT_BOTTOM 3 +#define SAMPLE_SVP_NPU_THRESHOLD_NUM 4 + +#define SAMPLE_SVP_NPU_RFCN_THRESHOLD_NUM 2 +#define SAMPLE_SVP_NPU_AICPU_WAIT_TIME 1000 +#define SAMPLE_SVP_NPU_RECT_COLOR 0x0000FF00 +#define SAMPLE_SVP_NPU_MILLIC_SEC 20000 +#define SAMPLE_SVP_NPU_IMG_THREE_CHN 3 +#define SAMPLE_SVP_NPU_DOUBLE 2 +#define SAMPLE_SVP_NPU_YOLO_CLASS_NUM 80 +#define SAMPLE_SVP_NPU_YOLO_CPU_OUT_NUM 3 +#define SAMPLE_SVP_NPU_YOLO_SCALE_SIZE 3 +#define SAMPLE_SVP_NPU_YOLO_OUT_PARM_NUM 85 +#define SAMPLE_SVP_NPU_YOLO_CLASS_MEMBER_NUM 6 +#define SAMPLE_SVP_NPU_YOLO_SCORE_THRESHOLE 0.5 + +typedef struct { + td_u32 model_id; + td_bool is_load_flag; + td_ulong model_mem_size; + td_void *model_mem_ptr; + svp_acl_mdl_desc *model_desc; + size_t input_num; + size_t output_num; + size_t dynamic_batch_idx; +} sample_svp_npu_model_info; + +typedef struct { + td_u32 max_batch_num; + td_u32 dynamic_batch_num; + td_u32 total_t; + td_bool is_cached; + td_u32 model_idx; +} sample_svp_npu_task_cfg; + +typedef struct { + sample_svp_npu_task_cfg cfg; + svp_acl_mdl_dataset *input_dataset; + svp_acl_mdl_dataset *output_dataset; + td_void *task_buf_ptr; + size_t task_buf_size; + size_t task_buf_stride; + td_void *work_buf_ptr; + size_t work_buf_size; + size_t work_buf_stride; +} sample_svp_npu_task_info; + +typedef struct { + td_void *work_buf_ptr; + size_t work_buf_size; + size_t work_buf_stride; +} sample_svp_npu_shared_work_buf; + +typedef struct { + td_float score; + td_u32 class_id; +} sample_svp_npu_top_n_result; + +typedef struct { + td_char *num_name; + td_char *roi_name; + td_bool has_background; + td_u32 roi_offset; + td_bool is_cpu_rpn; + td_u32 idx; +} sample_svp_npu_detection_info; + +typedef enum { + SAMPLE_SVP_NPU_RFCN = 0, + SAMPLE_SVP_NPU_YOLO = 1, + SAMPLE_SVP_NPU_HRNET = 2, + SAMPLE_SVP_NPU_MODEL_NAME_BUTT, +} svp_npu_model_name; + +typedef struct { + svp_npu_model_name model_name; + sample_svp_npu_detection_info *detection_info; + sample_mpp_osd_info mpp_osd_info; +} sample_svp_npu_thread_args; + +typedef struct { + td_char roi_num_name[SAMPLE_SVP_NPU_MAX_NAME_LEN]; + td_char roi_class_name[SAMPLE_SVP_NPU_MAX_NAME_LEN]; +} sample_svp_npu_roi_info; + +typedef struct { + td_float nms_threshold; + td_float score_threshold; + td_float min_height; + td_float min_width; + td_char *name; +} sample_svp_npu_threshold; + +typedef struct { + td_float score; + td_float xcenter; + td_float ycenter; + td_float w; + td_float h; + td_float class_id; +} sample_svp_npu_valid_box; + +typedef struct { + td_float lx; + td_float ly; + td_float rx; + td_float ry; + td_float score; + td_float class_id; + td_float area; +} sample_svp_npu_roi_box; + +typedef struct { + td_float *out_data; + size_t idx; + size_t w_stride_offset; + td_float score_thr; + td_u32 out_width; + td_u32 chn_step; + td_u32 out_height_idx; + td_u32 obj_score_offset; +} sample_svp_npu_detection_inner_param; + +/* acl init */ +td_s32 sample_common_svp_npu_acl_init(const td_char *acl_config_path, td_s32 dev_id); +/* acl deinit */ +td_void sample_common_svp_npu_acl_deinit(td_s32 dev_id); + +#endif diff --git a/vendor/zsks/demo/fruit_identify/common/sample_common_svp_npu_model.c b/vendor/zsks/demo/fruit_identify/common/sample_common_svp_npu_model.c new file mode 100755 index 0000000000000000000000000000000000000000..69810ed42427c37772e05602b63092caaee089b8 --- /dev/null +++ b/vendor/zsks/demo/fruit_identify/common/sample_common_svp_npu_model.c @@ -0,0 +1,1503 @@ +/* + * Copyright (c) 2025 HiSilicon (Shanghai) Technologies Co., Ltd. + * Copyright (c) 2025 Zhongshan Kuangshi Microelectronics Technology Co., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Modifications Author: yaohongtao (yht@cust.edu.cn) + */ +#include +#include +#include "svp_acl_rt.h" +#include "svp_acl_ext.h" +#include "sample_common_svp.h" +#include "sample_common_svp_npu_model.h" + +#define SAMPLE_SVP_NPU_H_DIM_IDX 2 +#define SAMPLE_SVP_NPU_W_DIM_IDX 3 +#define SAMPLE_SVP_NPU_DIM_NUM 4 +static sample_svp_npu_model_info g_svp_npu_model[SAMPLE_SVP_NPU_MAX_MODEL_NUM] = {0}; + +static td_s32 sample_svp_npu_get_model_base_info(td_u32 model_index) +{ + svp_acl_error ret; + + g_svp_npu_model[model_index].input_num = svp_acl_mdl_get_num_inputs(g_svp_npu_model[model_index].model_desc); + sample_svp_check_exps_return(g_svp_npu_model[model_index].input_num < SAMPLE_SVP_NPU_EXTRA_INPUT_NUM + 1, + TD_FAILURE, SAMPLE_SVP_ERR_LEVEL_ERROR, "get input num failed!\n"); + + g_svp_npu_model[model_index].output_num = svp_acl_mdl_get_num_outputs(g_svp_npu_model[model_index].model_desc); + sample_svp_check_exps_return(g_svp_npu_model[model_index].output_num < 1, + TD_FAILURE, SAMPLE_SVP_ERR_LEVEL_ERROR, "get output num failed!\n"); + + ret = svp_acl_mdl_get_input_index_by_name(g_svp_npu_model[model_index].model_desc, + SVP_ACL_DYNAMIC_TENSOR_NAME, &g_svp_npu_model[model_index].dynamic_batch_idx); + sample_svp_check_exps_return(ret != SVP_ACL_SUCCESS, TD_FAILURE, SAMPLE_SVP_ERR_LEVEL_ERROR, + "get dynamic batch idx failed, model id is %u, error code is %d!\n", model_index, ret); + + return TD_SUCCESS; +} + +static td_s32 sample_svp_npu_get_line_num_and_line_byte_num(const sample_svp_npu_task_info *task, td_u32 idx, + td_bool is_input, td_u32 *total_line_num, td_u32 *line_byte_num) +{ + td_s32 ret; + td_u32 i; + svp_acl_mdl_io_dims dims; + svp_acl_data_type data_type; + size_t data_size; + + if (is_input == TD_TRUE) { + ret = svp_acl_mdl_get_input_dims(g_svp_npu_model[task->cfg.model_idx].model_desc, idx, &dims); + } else { + ret = svp_acl_mdl_get_output_dims(g_svp_npu_model[task->cfg.model_idx].model_desc, idx, &dims); + } + sample_svp_check_exps_return(ret != TD_SUCCESS, TD_FAILURE, SAMPLE_SVP_ERR_LEVEL_ERROR, + "get %u-th input/output dims failed!\n", idx); + + if (is_input == TD_TRUE) { + data_type = svp_acl_mdl_get_input_data_type(g_svp_npu_model[task->cfg.model_idx].model_desc, idx); + } else { + data_type = svp_acl_mdl_get_output_data_type(g_svp_npu_model[task->cfg.model_idx].model_desc, idx); + } + sample_svp_check_exps_return(ret != TD_SUCCESS, TD_FAILURE, SAMPLE_SVP_ERR_LEVEL_ERROR, + "get %u-th input/output data type failed!\n", idx); + + data_size = svp_acl_data_type_size(data_type); + sample_svp_check_exps_return(data_size == 0, TD_FAILURE, SAMPLE_SVP_ERR_LEVEL_ERROR, "get data size failed!\n"); + + *line_byte_num = dims.dims[dims.dim_count - 1] * + ((data_size + SAMPLE_SVP_NPU_BYTE_BIT_NUM - 1) / SAMPLE_SVP_NPU_BYTE_BIT_NUM); + + *total_line_num = 1; + for (i = 0; i < dims.dim_count - 1; i++) { + *total_line_num *= dims.dims[i]; + } + /* lstm xt line num */ + if ((task->cfg.total_t != 0) && (idx == 0)) { + sample_svp_check_exps_return(task->cfg.total_t > dims.dims[0], TD_FAILURE, SAMPLE_SVP_ERR_LEVEL_ERROR, + "total t(%u) can't be greater than max total t(%ld)!\n", task->cfg.total_t, dims.dims[0]); + *total_line_num /= dims.dims[0]; + *total_line_num *= task->cfg.total_t; + } + return TD_SUCCESS; +} + +static td_s32 sample_svp_check_task_cfg(const sample_svp_npu_task_info *task) +{ + sample_svp_check_exps_return(task == TD_NULL, TD_FAILURE, SAMPLE_SVP_ERR_LEVEL_ERROR, "task is NULL!\n"); + + sample_svp_check_exps_return(task->cfg.max_batch_num == 0, TD_FAILURE, SAMPLE_SVP_ERR_LEVEL_ERROR, + "max_batch_num(%u) is 0!\n", task->cfg.max_batch_num); + + sample_svp_check_exps_return(task->cfg.dynamic_batch_num == 0, TD_FAILURE, SAMPLE_SVP_ERR_LEVEL_ERROR, + "dynamic_batch_num(%u) is 0!\n", task->cfg.dynamic_batch_num); + + sample_svp_check_exps_return(task->cfg.total_t != 0 && task->cfg.dynamic_batch_num != 1, TD_FAILURE, + SAMPLE_SVP_ERR_LEVEL_ERROR, "dynamic_batch_num(%u) should be 1 when total_t(%u) is not 0!\n", + task->cfg.dynamic_batch_num, task->cfg.total_t); + + sample_svp_check_exps_return((task->cfg.is_cached != TD_TRUE && task->cfg.is_cached != TD_FALSE), TD_FAILURE, + SAMPLE_SVP_ERR_LEVEL_ERROR, "is_cached(%u) should be [%u, %u]!\n", task->cfg.is_cached, TD_FALSE, TD_TRUE); + + sample_svp_check_exps_return(task->cfg.model_idx >= SAMPLE_SVP_NPU_MAX_MODEL_NUM, TD_FAILURE, + SAMPLE_SVP_ERR_LEVEL_ERROR, "model_idx(%u) should be less than %u!\n", + task->cfg.model_idx, SAMPLE_SVP_NPU_MAX_MODEL_NUM); + + sample_svp_check_exps_return(g_svp_npu_model[task->cfg.model_idx].model_desc == TD_NULL, TD_FAILURE, + SAMPLE_SVP_ERR_LEVEL_ERROR, "%u-th model_desc is NULL!\n", task->cfg.model_idx); + return TD_SUCCESS; +} + +td_s32 sample_common_svp_npu_get_input_data(const td_char *src[], td_u32 file_num, + const sample_svp_npu_task_info *task) +{ + td_s32 ret; + td_u32 i; + td_u32 line; + td_u32 total_line_num; + td_u32 line_byte_num; + td_char path[PATH_MAX] = { 0 }; + size_t stride; + size_t size; + size_t input_num; + FILE *fp = TD_NULL; + td_void *data = TD_NULL; + svp_acl_data_buffer *data_buffer = TD_NULL; + + if (sample_svp_check_task_cfg(task) != TD_SUCCESS) { + sample_svp_trace_err("check task cfg failed!\n"); + return TD_FAILURE; + } + sample_svp_check_exps_return(src == TD_NULL, TD_FAILURE, SAMPLE_SVP_ERR_LEVEL_ERROR, "src is NULL!\n"); + + input_num = g_svp_npu_model[task->cfg.model_idx].input_num - SAMPLE_SVP_NPU_EXTRA_INPUT_NUM; + sample_svp_check_exps_return(input_num != file_num, TD_FAILURE, SAMPLE_SVP_ERR_LEVEL_ERROR, + "input file num(%u) should be equal to %lu!\n", file_num, input_num); + + for (i = 0; i < file_num; i++) { + sample_svp_check_exps_return(realpath(src[i], path) == TD_NULL, TD_FAILURE, SAMPLE_SVP_ERR_LEVEL_ERROR, + "Invalid file!\n"); + + fp = fopen(path, "rb"); + sample_svp_check_exps_return(fp == TD_NULL, TD_FAILURE, SAMPLE_SVP_ERR_LEVEL_ERROR, "open file failed!\n"); + + ret = sample_svp_npu_get_line_num_and_line_byte_num(task, i, TD_TRUE, &total_line_num, &line_byte_num); + sample_svp_check_exps_goto(ret != TD_SUCCESS, end, SAMPLE_SVP_ERR_LEVEL_ERROR, "get line num failed!\n"); + + data_buffer = svp_acl_mdl_get_dataset_buffer(task->input_dataset, i); + sample_svp_check_exps_goto(data_buffer == TD_NULL, end, SAMPLE_SVP_ERR_LEVEL_ERROR, "get data buffer NULL!\n"); + + data = svp_acl_get_data_buffer_addr(data_buffer); + sample_svp_check_exps_goto(data == TD_NULL, end, SAMPLE_SVP_ERR_LEVEL_ERROR, "get data addr NULL!\n"); + + stride = svp_acl_get_data_buffer_stride(data_buffer); + sample_svp_check_exps_goto(stride == 0, end, SAMPLE_SVP_ERR_LEVEL_ERROR, "get data stride failed!\n"); + + size = svp_acl_get_data_buffer_size(data_buffer); + sample_svp_check_exps_goto(size < (td_u64)task->cfg.dynamic_batch_num * total_line_num * stride, end, + SAMPLE_SVP_ERR_LEVEL_ERROR, "%u-th data buffer size(%lu) is less than needed(%llu)!\n", + i, size, (td_u64)task->cfg.dynamic_batch_num * total_line_num * stride); + + for (line = 0; line < task->cfg.dynamic_batch_num * total_line_num; line++) { + ret = fread(data + line * stride, line_byte_num, 1, fp); + sample_svp_check_exps_goto(ret != 1, end, SAMPLE_SVP_ERR_LEVEL_ERROR, "Read file failed!\n"); + } + if (task->cfg.is_cached == TD_TRUE) { + (td_void)svp_acl_rt_mem_flush(data, task->cfg.dynamic_batch_num * total_line_num * stride); + } + (td_void)fclose(fp); + } + return TD_SUCCESS; + +end: + if (fp != TD_NULL) { + (td_void)fclose(fp); + } + return TD_FAILURE; +} + +static td_s32 sample_svp_npu_read_model(const td_char *model_path, td_u32 model_index, td_bool is_cached) +{ + FILE *fp = TD_NULL; + td_s32 ret; + + /* Get model file size */ + fp = fopen(model_path, "rb"); + sample_svp_check_exps_return(fp == TD_NULL, TD_FAILURE, SAMPLE_SVP_ERR_LEVEL_ERROR, + "open model file failed, model file is %s!\n", model_path); + + ret = fseek(fp, 0L, SEEK_END); + sample_svp_check_exps_goto(ret == -1, end_0, SAMPLE_SVP_ERR_LEVEL_ERROR, "fseek failed!\n"); + + g_svp_npu_model[model_index].model_mem_size = ftell(fp); + sample_svp_check_exps_goto(g_svp_npu_model[model_index].model_mem_size <= 0, end_0, + SAMPLE_SVP_ERR_LEVEL_ERROR, "ftell failed!\n"); + + ret = fseek(fp, 0L, SEEK_SET); + sample_svp_check_exps_goto(ret == -1, end_0, SAMPLE_SVP_ERR_LEVEL_ERROR, "fseek failed!\n"); + + /* malloc model file mem */ + if (is_cached == TD_TRUE) { + ret = svp_acl_rt_malloc_cached(&g_svp_npu_model[model_index].model_mem_ptr, + g_svp_npu_model[model_index].model_mem_size, SVP_ACL_MEM_MALLOC_NORMAL_ONLY); + } else { + ret = svp_acl_rt_malloc(&g_svp_npu_model[model_index].model_mem_ptr, + g_svp_npu_model[model_index].model_mem_size, SVP_ACL_MEM_MALLOC_NORMAL_ONLY); + } + sample_svp_check_exps_goto(ret != SVP_ACL_SUCCESS, end_0, SAMPLE_SVP_ERR_LEVEL_ERROR, + "malloc mem failed, erroe code %d!\n", ret); + + ret = fread(g_svp_npu_model[model_index].model_mem_ptr, g_svp_npu_model[model_index].model_mem_size, 1, fp); + sample_svp_check_exps_goto(ret != 1, end_1, SAMPLE_SVP_ERR_LEVEL_ERROR, "read model file failed!\n"); + + if (is_cached == TD_TRUE) { + ret = svp_acl_rt_mem_flush(g_svp_npu_model[model_index].model_mem_ptr, + g_svp_npu_model[model_index].model_mem_size); + sample_svp_check_exps_goto(ret != SVP_ACL_SUCCESS, end_1, SAMPLE_SVP_ERR_LEVEL_ERROR, + "flush mem failed!, error code is %d\n", ret); + } + (td_void)fclose(fp); + return TD_SUCCESS; +end_1: + (td_void)svp_acl_rt_free(g_svp_npu_model[model_index].model_mem_ptr); +end_0: + (td_void)fclose(fp); + return TD_FAILURE; +} + +static td_s32 sample_svp_npu_create_desc(td_u32 model_index) +{ + svp_acl_error ret; + + g_svp_npu_model[model_index].model_desc = svp_acl_mdl_create_desc(); + sample_svp_check_exps_return(g_svp_npu_model[model_index].model_desc == TD_NULL, TD_FAILURE, + SAMPLE_SVP_ERR_LEVEL_ERROR, "create model description failed!\n"); + + ret = svp_acl_mdl_get_desc(g_svp_npu_model[model_index].model_desc, g_svp_npu_model[model_index].model_id); + sample_svp_check_exps_return(ret != SVP_ACL_SUCCESS, TD_FAILURE, + SAMPLE_SVP_ERR_LEVEL_ERROR, "get model description failed, error code is %d!\n", ret); + + sample_svp_trace_info("create model description success!\n"); + + return TD_SUCCESS; +} + +td_s32 sample_common_svp_npu_load_model(const td_char *model_path, td_u32 model_index, td_bool is_cached) +{ + td_s32 ret; + + sample_svp_check_exps_return(g_svp_npu_model[model_index].is_load_flag == TD_TRUE, TD_FAILURE, + SAMPLE_SVP_ERR_LEVEL_ERROR, "%u-th node has already loaded a model!\n", model_index); + + ret = sample_svp_npu_read_model(model_path, model_index, is_cached); + sample_svp_check_exps_return(ret != SVP_ACL_SUCCESS, TD_FAILURE, + SAMPLE_SVP_ERR_LEVEL_ERROR, "query model failed, model file is %s!\n", model_path); + + ret = svp_acl_mdl_load_from_mem(g_svp_npu_model[model_index].model_mem_ptr, + g_svp_npu_model[model_index].model_mem_size, &g_svp_npu_model[model_index].model_id); + sample_svp_check_exps_goto(ret != SVP_ACL_SUCCESS, end_0, + SAMPLE_SVP_ERR_LEVEL_ERROR, "load model from mem failed, error code is %d!\n", ret); + + ret = sample_svp_npu_create_desc(model_index); + sample_svp_check_exps_goto(ret != SVP_ACL_SUCCESS, end_0, + SAMPLE_SVP_ERR_LEVEL_ERROR, "create desc failed, model file is %s!\n", model_path); + + ret = sample_svp_npu_get_model_base_info(model_index); + sample_svp_check_exps_goto(ret != SVP_ACL_SUCCESS, end_1, + SAMPLE_SVP_ERR_LEVEL_ERROR, "get model base info failed, model file is %s!\n", model_path); + + sample_svp_trace_info("load mem_size:%lu, id:%d!\n", g_svp_npu_model[model_index].model_mem_size, + g_svp_npu_model[model_index].model_id); + + g_svp_npu_model[model_index].is_load_flag = TD_TRUE; + sample_svp_trace_info("load model %s success!\n", model_path); + + return TD_SUCCESS; +end_1: + (td_void)svp_acl_mdl_destroy_desc(g_svp_npu_model[model_index].model_desc); + g_svp_npu_model[model_index].model_desc = TD_NULL; +end_0: + (td_void)svp_acl_rt_free(g_svp_npu_model[model_index].model_mem_ptr); + g_svp_npu_model[model_index].model_mem_ptr = TD_NULL; + g_svp_npu_model[model_index].model_mem_size = 0; + return TD_FAILURE; +} + +td_void sample_common_svp_npu_unload_model(td_u32 model_index) +{ + svp_acl_error ret; + + sample_svp_check_exps_return_void(g_svp_npu_model[model_index].is_load_flag != TD_TRUE, + SAMPLE_SVP_ERR_LEVEL_ERROR, "%u-th node has not loaded a model!\n", model_index); + + ret = svp_acl_mdl_unload(g_svp_npu_model[model_index].model_id); + if (ret != SVP_ACL_SUCCESS) { + sample_svp_trace_err("unload model failed, model_id is %u, error code is %d!\n", + g_svp_npu_model[model_index].model_id, ret); + } + + if (g_svp_npu_model[model_index].model_desc != TD_NULL) { + (td_void)svp_acl_mdl_destroy_desc(g_svp_npu_model[model_index].model_desc); + g_svp_npu_model[model_index].model_desc = TD_NULL; + } + + if (g_svp_npu_model[model_index].model_mem_ptr != TD_NULL) { + (td_void)svp_acl_rt_free(g_svp_npu_model[model_index].model_mem_ptr); + g_svp_npu_model[model_index].model_mem_ptr = TD_NULL; + g_svp_npu_model[model_index].model_mem_size = 0; + } + + g_svp_npu_model[model_index].is_load_flag = TD_FALSE; + sample_svp_trace_info("unload model SUCCESS, model id is %u!\n", g_svp_npu_model[model_index].model_id); +} + +static td_s32 sample_svp_npu_malloc_mem(td_void **buffer, td_u32 buffer_size, td_bool is_cached) +{ + svp_acl_error ret; + + if (is_cached == TD_TRUE) { + ret = svp_acl_rt_malloc_cached(buffer, buffer_size, SVP_ACL_MEM_MALLOC_NORMAL_ONLY); + } else { + ret = svp_acl_rt_malloc(buffer, buffer_size, SVP_ACL_MEM_MALLOC_NORMAL_ONLY); + } + sample_svp_check_exps_return(ret != SVP_ACL_SUCCESS, TD_FAILURE, SAMPLE_SVP_ERR_LEVEL_ERROR, + "can't malloc buffer, size is %u, error code is %d!\n", buffer_size, ret); + + (td_void)memset_s(*buffer, buffer_size, 0, buffer_size); + if (is_cached == TD_TRUE) { + (td_void)svp_acl_rt_mem_flush(*buffer, buffer_size); + } + return ret; +} + +static td_void sample_svp_npu_destroy_data_buffer(svp_acl_data_buffer *input_data) +{ + td_void *data = svp_acl_get_data_buffer_addr(input_data); + (td_void)svp_acl_rt_free(data); + (td_void)svp_acl_destroy_data_buffer(input_data); +} + +static svp_acl_data_buffer *sample_common_svp_npu_create_input_data_buffer(sample_svp_npu_task_info *task, td_u32 idx) +{ + size_t buffer_size; + size_t stride; + td_void *input_buffer = TD_NULL; + svp_acl_data_buffer *input_data = TD_NULL; + + stride = svp_acl_mdl_get_input_default_stride(g_svp_npu_model[task->cfg.model_idx].model_desc, idx); + sample_svp_check_exps_return(stride == 0, input_data, SAMPLE_SVP_ERR_LEVEL_ERROR, + "get %u-th input stride failed!\n", idx); + + buffer_size = svp_acl_mdl_get_input_size_by_index(g_svp_npu_model[task->cfg.model_idx].model_desc, idx) * + (td_u64)task->cfg.max_batch_num; + sample_svp_check_exps_return((buffer_size == 0 || buffer_size > SAMPLE_SVP_NPU_MAX_MEM_SIZE), input_data, + SAMPLE_SVP_ERR_LEVEL_ERROR, "buffer_size(%lu) can't be 0 and should be less than %u!\n", + buffer_size, SAMPLE_SVP_NPU_MAX_MEM_SIZE); + + if (sample_svp_npu_malloc_mem(&input_buffer, (td_u32)buffer_size, task->cfg.is_cached) != TD_SUCCESS) { + sample_svp_trace_err("%u-th input malloc mem failed!\n", idx); + return input_data; + } + + input_data = svp_acl_create_data_buffer(input_buffer, buffer_size, stride); + if (input_data == TD_NULL) { + sample_svp_trace_err("can't create %u-th input data buffer!\n", idx); + (td_void)svp_acl_rt_free(input_buffer); + return input_data; + } + if (idx == g_svp_npu_model[task->cfg.model_idx].input_num - SAMPLE_SVP_NPU_EXTRA_INPUT_NUM) { + task->task_buf_ptr = input_buffer; + task->task_buf_size = buffer_size; + task->task_buf_stride = stride; + } else if (idx == g_svp_npu_model[task->cfg.model_idx].input_num - 1) { + task->work_buf_ptr = input_buffer; + task->work_buf_size = buffer_size; + task->work_buf_stride = stride; + } + return input_data; +} + +static svp_acl_data_buffer *sample_common_svp_npu_create_output_data_buffer(const sample_svp_npu_task_info *task, + td_u32 idx) +{ + size_t buffer_size; + size_t stride; + td_void *output_buffer = TD_NULL; + svp_acl_data_buffer *output_data = TD_NULL; + + stride = svp_acl_mdl_get_output_default_stride(g_svp_npu_model[task->cfg.model_idx].model_desc, idx); + sample_svp_check_exps_return(stride == 0, output_data, SAMPLE_SVP_ERR_LEVEL_ERROR, + "get %u-th output stride failed!\n", idx); + + buffer_size = svp_acl_mdl_get_output_size_by_index(g_svp_npu_model[task->cfg.model_idx].model_desc, idx) * + (td_u64)task->cfg.max_batch_num; + sample_svp_check_exps_return((buffer_size == 0 || buffer_size > SAMPLE_SVP_NPU_MAX_MEM_SIZE), output_data, + SAMPLE_SVP_ERR_LEVEL_ERROR, "buffer_size(%lu) can't be 0 and should be less than %u!\n", + buffer_size, SAMPLE_SVP_NPU_MAX_MEM_SIZE); + + if (sample_svp_npu_malloc_mem(&output_buffer, buffer_size, task->cfg.is_cached) != TD_SUCCESS) { + sample_svp_trace_err("%u-th output malloc mem failed!\n", idx); + return output_data; + } + + output_data = svp_acl_create_data_buffer(output_buffer, buffer_size, stride); + if (output_data == TD_NULL) { + sample_svp_trace_err("can't create %u-th output data buffer!\n", idx); + (td_void)svp_acl_rt_free(output_buffer); + return output_data; + } + return output_data; +} + +td_s32 sample_common_svp_npu_create_input(sample_svp_npu_task_info *task) +{ + svp_acl_error ret; + td_u32 i; + svp_acl_data_buffer *input_data = TD_NULL; + + if (sample_svp_check_task_cfg(task) != TD_SUCCESS) { + sample_svp_trace_err("check task cfg failed!\n"); + return TD_FAILURE; + } + + task->input_dataset = svp_acl_mdl_create_dataset(); + sample_svp_check_exps_return(task->input_dataset == TD_NULL, TD_FAILURE, SAMPLE_SVP_ERR_LEVEL_ERROR, + "create input dataset failed!\n"); + + for (i = 0; i < g_svp_npu_model[task->cfg.model_idx].input_num - SAMPLE_SVP_NPU_EXTRA_INPUT_NUM; i++) { + input_data = sample_common_svp_npu_create_input_data_buffer(task, i); + if (input_data == TD_NULL) { + sample_svp_trace_err("create %u-th input data buffer failed!\n", i); + (td_void)sample_common_svp_npu_destroy_input(task); + return TD_FAILURE; + } + + ret = svp_acl_mdl_add_dataset_buffer(task->input_dataset, input_data); + if (ret != SVP_ACL_SUCCESS) { + sample_svp_trace_err("add %u-th input data buffer failed!\n", i); + (td_void)sample_svp_npu_destroy_data_buffer(input_data); + (td_void)sample_common_svp_npu_destroy_input(task); + return TD_FAILURE; + } + } + return TD_SUCCESS; +} + +td_s32 sample_common_svp_npu_create_output(sample_svp_npu_task_info *task) +{ + svp_acl_error ret; + td_u32 i; + svp_acl_data_buffer *output_data = TD_NULL; + + if (sample_svp_check_task_cfg(task) != TD_SUCCESS) { + sample_svp_trace_err("check task cfg failed!\n"); + return TD_FAILURE; + } + + task->output_dataset = svp_acl_mdl_create_dataset(); + sample_svp_check_exps_return(task->input_dataset == TD_NULL, TD_FAILURE, SAMPLE_SVP_ERR_LEVEL_ERROR, + "create output dataset failed!\n"); + + for (i = 0; i < g_svp_npu_model[task->cfg.model_idx].output_num; i++) { + output_data = sample_common_svp_npu_create_output_data_buffer(task, i); + if (output_data == TD_NULL) { + sample_svp_trace_err("create %u-th output data buffer failed!\n", i); + (td_void)sample_common_svp_npu_destroy_input(task); + return TD_FAILURE; + } + ret = svp_acl_mdl_add_dataset_buffer(task->output_dataset, output_data); + if (ret != SVP_ACL_SUCCESS) { + sample_svp_trace_err("add %u-th output data buffer failed!\n", i); + (td_void)sample_svp_npu_destroy_data_buffer(output_data); + (td_void)sample_common_svp_npu_destroy_output(task); + return TD_FAILURE; + } + } + + return TD_SUCCESS; +} + +td_void sample_common_svp_npu_destroy_input(sample_svp_npu_task_info *task) +{ + td_u32 i; + size_t input_num; + svp_acl_data_buffer *data_buffer = TD_NULL; + td_void *data = TD_NULL; + + if (sample_svp_check_task_cfg(task) != TD_SUCCESS) { + sample_svp_trace_err("check task cfg failed!\n"); + return; + } + + if (task->input_dataset == TD_NULL) { + return; + } + + input_num = svp_acl_mdl_get_dataset_num_buffers(task->input_dataset); + for (i = 0; i < input_num; i++) { + data_buffer = svp_acl_mdl_get_dataset_buffer(task->input_dataset, i); + if (i < g_svp_npu_model[task->cfg.model_idx].input_num - SAMPLE_SVP_NPU_EXTRA_INPUT_NUM) { + data = svp_acl_get_data_buffer_addr(data_buffer); + (td_void)svp_acl_rt_free(data); + } + (td_void)svp_acl_destroy_data_buffer(data_buffer); + } + (td_void)svp_acl_mdl_destroy_dataset(task->input_dataset); + task->input_dataset = TD_NULL; +} + +td_void sample_common_svp_npu_destroy_output(sample_svp_npu_task_info *task) +{ + td_u32 i; + size_t output_num; + svp_acl_data_buffer *data_buffer = TD_NULL; + td_void *data = TD_NULL; + + if (sample_svp_check_task_cfg(task) != TD_SUCCESS) { + sample_svp_trace_err("check task cfg failed!\n"); + return; + } + + if (task->output_dataset == TD_NULL) { + return; + } + + output_num = svp_acl_mdl_get_dataset_num_buffers(task->output_dataset); + + for (i = 0; i < output_num; i++) { + data_buffer = svp_acl_mdl_get_dataset_buffer(task->output_dataset, i); + data = svp_acl_get_data_buffer_addr(data_buffer); + (td_void)svp_acl_rt_free(data); + (td_void)svp_acl_destroy_data_buffer(data_buffer); + } + + (td_void)svp_acl_mdl_destroy_dataset(task->output_dataset); + task->output_dataset = TD_NULL; +} + +td_void sample_common_svp_npu_destroy_task_buf(sample_svp_npu_task_info *task) +{ + if (sample_svp_check_task_cfg(task) != TD_SUCCESS) { + sample_svp_trace_err("check task cfg failed!\n"); + return; + } + + if (task->task_buf_ptr == TD_NULL) { + return; + } + (td_void)svp_acl_rt_free(task->task_buf_ptr); + task->task_buf_ptr = TD_NULL; + task->task_buf_stride = 0; + task->task_buf_size = 0; +} + +td_void sample_common_svp_npu_destroy_work_buf(sample_svp_npu_task_info *task) +{ + if (sample_svp_check_task_cfg(task) != TD_SUCCESS) { + sample_svp_trace_err("check task cfg failed!\n"); + return; + } + + if (task->work_buf_ptr == TD_NULL) { + return; + } + (td_void)svp_acl_rt_free(task->work_buf_ptr); + task->work_buf_ptr = TD_NULL; + task->work_buf_stride = 0; + task->work_buf_size = 0; +} + +td_s32 sample_common_svp_npu_create_task_buf(sample_svp_npu_task_info *task) +{ + size_t num; + svp_acl_data_buffer *task_buf = TD_NULL; + svp_acl_error ret; + + if (sample_svp_check_task_cfg(task) != TD_SUCCESS) { + sample_svp_trace_err("check task cfg failed!\n"); + return TD_FAILURE; + } + sample_svp_check_exps_return(task->input_dataset == TD_NULL, TD_FAILURE, SAMPLE_SVP_ERR_LEVEL_ERROR, + "input_dataset is NULL!\n"); + + num = svp_acl_mdl_get_dataset_num_buffers(task->input_dataset); + sample_svp_check_exps_return(num != g_svp_npu_model[task->cfg.model_idx].input_num - SAMPLE_SVP_NPU_EXTRA_INPUT_NUM, + TD_FAILURE, SAMPLE_SVP_ERR_LEVEL_ERROR, "num of data buffer(%lu) should be %lu when create task buf!\n", + num, g_svp_npu_model[task->cfg.model_idx].input_num - SAMPLE_SVP_NPU_EXTRA_INPUT_NUM); + + task_buf = sample_common_svp_npu_create_input_data_buffer(task, + g_svp_npu_model[task->cfg.model_idx].input_num - SAMPLE_SVP_NPU_EXTRA_INPUT_NUM); + sample_svp_check_exps_return(task_buf == TD_NULL, TD_FAILURE, SAMPLE_SVP_ERR_LEVEL_ERROR, + "create task buf failed!\n"); + + ret = svp_acl_mdl_add_dataset_buffer(task->input_dataset, task_buf); + if (ret != SVP_ACL_SUCCESS) { + sample_svp_trace_err("add task buf failed!\n"); + (td_void)sample_svp_npu_destroy_data_buffer(task_buf); + return TD_FAILURE; + } + return TD_SUCCESS; +} + +td_s32 sample_common_svp_npu_create_work_buf(sample_svp_npu_task_info *task) +{ + size_t num; + svp_acl_data_buffer *work_buf = TD_NULL; + svp_acl_error ret; + + if (sample_svp_check_task_cfg(task) != TD_SUCCESS) { + sample_svp_trace_err("check task cfg failed!\n"); + return TD_FAILURE; + } + + sample_svp_check_exps_return(task->input_dataset == TD_NULL, TD_FAILURE, SAMPLE_SVP_ERR_LEVEL_ERROR, + "input_dataset is NULL!\n"); + + num = svp_acl_mdl_get_dataset_num_buffers(task->input_dataset); + sample_svp_check_exps_return(num != g_svp_npu_model[task->cfg.model_idx].input_num - 1, + TD_FAILURE, SAMPLE_SVP_ERR_LEVEL_ERROR, "num of data buffer(%lu) should be %lu when create work buf!\n", + num, g_svp_npu_model[task->cfg.model_idx].input_num - 1); + + work_buf = sample_common_svp_npu_create_input_data_buffer(task, g_svp_npu_model[task->cfg.model_idx].input_num - 1); + sample_svp_check_exps_return(work_buf == TD_NULL, TD_FAILURE, SAMPLE_SVP_ERR_LEVEL_ERROR, + "create work buf failed!\n"); + + ret = svp_acl_mdl_add_dataset_buffer(task->input_dataset, work_buf); + if (ret != SVP_ACL_SUCCESS) { + sample_svp_trace_err("add work buf failed!\n"); + (td_void)sample_svp_npu_destroy_data_buffer(work_buf); + return TD_FAILURE; + } + return TD_SUCCESS; +} + +td_s32 sample_common_svp_npu_get_work_buf_info(const sample_svp_npu_task_info *task, + td_u32 *work_buf_size, td_u32 *work_buf_stride) +{ + if (sample_svp_check_task_cfg(task) != TD_SUCCESS) { + sample_svp_trace_err("check task cfg failed!\n"); + return TD_FAILURE; + } + + *work_buf_stride = (td_u32)svp_acl_mdl_get_input_default_stride(g_svp_npu_model[task->cfg.model_idx].model_desc, + g_svp_npu_model[task->cfg.model_idx].input_num - 1); + if (*work_buf_stride == 0) { + sample_svp_trace_err("get work buf stride failed!\n"); + return TD_FAILURE; + } + + *work_buf_size = (td_u32)svp_acl_mdl_get_input_size_by_index(g_svp_npu_model[task->cfg.model_idx].model_desc, + g_svp_npu_model[task->cfg.model_idx].input_num - 1); + if (*work_buf_size == 0) { + sample_svp_trace_err("get work buf size failed!\n"); + return TD_FAILURE; + } + return TD_SUCCESS; +} + +td_s32 sample_common_svp_npu_share_work_buf(const sample_svp_npu_shared_work_buf *shared_work_buf, + const sample_svp_npu_task_info *task) +{ + svp_acl_error ret; + svp_acl_data_buffer *work_buf = TD_NULL; + + if (sample_svp_check_task_cfg(task) != TD_SUCCESS) { + sample_svp_trace_err("check task cfg failed!\n"); + return TD_FAILURE; + } + + sample_svp_check_exps_return(shared_work_buf == TD_NULL, TD_FAILURE, SAMPLE_SVP_ERR_LEVEL_ERROR, + "shared_work_buf is NULL!\n"); + + sample_svp_check_exps_return(shared_work_buf->work_buf_ptr == TD_NULL, TD_FAILURE, SAMPLE_SVP_ERR_LEVEL_ERROR, + "work_buf_ptr is NULL!\n"); + + sample_svp_check_exps_return(task->work_buf_ptr != TD_NULL, TD_FAILURE, SAMPLE_SVP_ERR_LEVEL_ERROR, + "task has created work buf!\n"); + + work_buf = svp_acl_create_data_buffer(shared_work_buf->work_buf_ptr, shared_work_buf->work_buf_size, + shared_work_buf->work_buf_stride); + sample_svp_check_exps_return(work_buf == TD_NULL, TD_FAILURE, SAMPLE_SVP_ERR_LEVEL_ERROR, + "create work buf failed!\n"); + + ret = svp_acl_mdl_add_dataset_buffer(task->input_dataset, work_buf); + if (ret != SVP_ACL_SUCCESS) { + sample_svp_trace_err("add work buf failed!\n"); + (td_void)svp_acl_destroy_data_buffer(work_buf); + return TD_FAILURE; + } + + return TD_SUCCESS; +} + +/* print the top n confidence values with indexes */ +static td_void sample_svp_npu_sort_output_result(const td_float *src, td_u32 src_len, + sample_svp_npu_top_n_result *dst, td_u32 dst_len) +{ + td_u32 i; + td_u32 j; + td_u32 index; + td_bool charge; + + for (i = 0; i < src_len; i++) { + charge = TD_FALSE; + + for (j = 0; j < dst_len; j++) { + if (src[i] > dst[j].score) { + index = j; + charge = TD_TRUE; + break; + } + } + + if (charge == TD_TRUE) { + for (j = dst_len - 1; j > index; j--) { + dst[j].score = dst[j - 1].score; + dst[j].class_id = dst[j - 1].class_id; + } + dst[index].score = src[i]; + dst[index].class_id = i; + } + } +} + +td_void sample_common_svp_npu_output_classification_result(const sample_svp_npu_task_info *task) +{ + svp_acl_data_buffer *data_buffer = TD_NULL; + td_void *data = TD_NULL; + td_u32 i; + td_u32 j; + td_u32 n; + svp_acl_error ret; + size_t stride; + svp_acl_mdl_io_dims dims; + sample_svp_npu_top_n_result top[SAMPLE_SVP_NPU_SHOW_TOP_NUM] = { 0 }; + + if (sample_svp_check_task_cfg(task) != TD_SUCCESS) { + sample_svp_trace_err("check task cfg failed!\n"); + return; + } + + sample_svp_check_exps_return_void(task->output_dataset == TD_NULL, SAMPLE_SVP_ERR_LEVEL_ERROR, + "task->output_dataset is NULL!\n"); + + for (i = 0; i < svp_acl_mdl_get_dataset_num_buffers(task->output_dataset); i++) { + data_buffer = svp_acl_mdl_get_dataset_buffer(task->output_dataset, i); + sample_svp_check_exps_return_void(data_buffer == TD_NULL, SAMPLE_SVP_ERR_LEVEL_ERROR, + "get %u-th data buffer is NULL!\n", i); + + data = svp_acl_get_data_buffer_addr(data_buffer); + sample_svp_check_exps_return_void(data == TD_NULL, SAMPLE_SVP_ERR_LEVEL_ERROR, + "get %u-th data addr is NULL!\n", i); + + stride = svp_acl_get_data_buffer_stride(data_buffer); + sample_svp_check_exps_return_void(data == TD_NULL, SAMPLE_SVP_ERR_LEVEL_ERROR, + "get %u-th data stride is 0!\n", i); + + ret = svp_acl_mdl_get_output_dims(g_svp_npu_model[task->cfg.model_idx].model_desc, i, &dims); + sample_svp_check_exps_return_void(data == TD_NULL, SAMPLE_SVP_ERR_LEVEL_ERROR, + "get %u-th output dims failed, error code is %d!\n", i, ret); + + for (n = 0; n < task->cfg.dynamic_batch_num; n++) { + (td_void)sample_svp_npu_sort_output_result(data, (td_u32)dims.dims[dims.dim_count - 1], + top, SAMPLE_SVP_NPU_SHOW_TOP_NUM); + sample_svp_trace_info("%u-th batch result:\n", n); + for (j = 0; j < SAMPLE_SVP_NPU_SHOW_TOP_NUM; j++) { + sample_svp_trace_info("top %d: value[%lf], class_id[%u]!\n", j, top[j].score, top[j].class_id); + } + data += stride; + (td_void)memset_s(top, sizeof(top), 0, sizeof(top)); + } + } + + sample_svp_trace_info("output data success!\n"); +} + +td_s32 sample_common_svp_npu_set_dynamic_batch(const sample_svp_npu_task_info *task) +{ + svp_acl_error ret; + + if (sample_svp_check_task_cfg(task) != TD_SUCCESS) { + sample_svp_trace_err("check task cfg failed!\n"); + return TD_FAILURE; + } + ret = svp_acl_mdl_set_dynamic_batch_size(g_svp_npu_model[task->cfg.model_idx].model_id, + task->input_dataset, g_svp_npu_model[task->cfg.model_idx].dynamic_batch_idx, task->cfg.dynamic_batch_num); + sample_svp_check_exps_return(ret != SVP_ACL_SUCCESS, TD_FAILURE, SAMPLE_SVP_ERR_LEVEL_ERROR, + "set dynamic batch size failed, model id is %u, error code is %d!\n", + g_svp_npu_model[task->cfg.model_idx].model_id, ret); + + if (task->cfg.total_t != 0) { + ret = svp_acl_mdl_set_total_t(g_svp_npu_model[task->cfg.model_idx].model_id, + task->input_dataset, task->cfg.total_t); + sample_svp_check_exps_return(ret != SVP_ACL_SUCCESS, TD_FAILURE, SAMPLE_SVP_ERR_LEVEL_ERROR, + "set total_t failed, model id is %u, error code is %d!\n", + g_svp_npu_model[task->cfg.model_idx].model_id, ret); + } + + return TD_SUCCESS; +} + +td_s32 sample_common_svp_npu_model_execute(const sample_svp_npu_task_info *task) +{ + svp_acl_data_buffer *data_buffer = TD_NULL; + td_void *data = TD_NULL; + size_t size; + td_u32 i; + svp_acl_error ret; + + if (sample_svp_check_task_cfg(task) != TD_SUCCESS) { + sample_svp_trace_err("check task cfg failed!\n"); + return TD_FAILURE; + } + + ret = svp_acl_mdl_execute(g_svp_npu_model[task->cfg.model_idx].model_id, task->input_dataset, task->output_dataset); + sample_svp_check_exps_return(ret != SVP_ACL_SUCCESS, TD_FAILURE, SAMPLE_SVP_ERR_LEVEL_ERROR, + "svp_acl_mdl_execute failed, model_id is %u, error code is %d!\n", + g_svp_npu_model[task->cfg.model_idx].model_id, ret); + + if (task->cfg.is_cached == TD_TRUE) { + for (i = 0; i < g_svp_npu_model[task->cfg.model_idx].output_num; i++) { + data_buffer = svp_acl_mdl_get_dataset_buffer(task->output_dataset, i); + sample_svp_check_exps_return(data_buffer == TD_NULL, TD_FAILURE, SAMPLE_SVP_ERR_LEVEL_ERROR, + "get %u-th output data_buffer is NULL!\n", i); + + data = svp_acl_get_data_buffer_addr(data_buffer); + sample_svp_check_exps_return(data == TD_NULL, TD_FAILURE, SAMPLE_SVP_ERR_LEVEL_ERROR, + "get %u-th output data is NULL!\n", i); + + size = svp_acl_get_data_buffer_size(data_buffer) / task->cfg.max_batch_num * task->cfg.dynamic_batch_num; + sample_svp_check_exps_return(size == 0, TD_FAILURE, SAMPLE_SVP_ERR_LEVEL_ERROR, + "get %u-th output data size is 0!\n", i); + + ret = svp_acl_rt_mem_flush(data, size); + sample_svp_check_exps_return(data_buffer == TD_NULL, TD_FAILURE, SAMPLE_SVP_ERR_LEVEL_ERROR, + "flush %u-th output data failed, error code is %d!\n", i, ret); + } + } + return ret; +} + +static td_void sample_svp_npu_dump_dataset(const sample_svp_npu_task_info *task, const td_char *file_name, + td_bool is_input) +{ + td_u32 i; + td_u32 line; + td_u32 total_line_num; + td_u32 line_byte_num; + td_u32 data_num; + svp_acl_error ret; + size_t size; + size_t stride; + svp_acl_data_buffer *data_buffer = TD_NULL; + svp_acl_mdl_dataset *dataset = TD_NULL; + td_void *data = TD_NULL; + FILE *fp = TD_NULL; + td_char name[SAMPLE_SVP_NPU_MAX_NAME_LEN] = {'\0'}; + + data_num = is_input == TD_TRUE ? g_svp_npu_model[task->cfg.model_idx].input_num - SAMPLE_SVP_NPU_EXTRA_INPUT_NUM : + g_svp_npu_model[task->cfg.model_idx].output_num; + dataset = is_input == TD_TRUE ? task->input_dataset : task->output_dataset; + + for (i = 0; i < data_num; i++) { + ret = sample_svp_npu_get_line_num_and_line_byte_num(task, i, is_input, &total_line_num, &line_byte_num); + sample_svp_check_exps_return_void(ret != TD_SUCCESS, SAMPLE_SVP_ERR_LEVEL_ERROR, + "get line num and line byte num failed!\n"); + + data_buffer = svp_acl_mdl_get_dataset_buffer(dataset, i); + sample_svp_check_exps_return_void(data_buffer == TD_NULL, SAMPLE_SVP_ERR_LEVEL_ERROR, + "get %u-th data_buffer failed!\n", i); + + data = svp_acl_get_data_buffer_addr(data_buffer); + sample_svp_check_exps_return_void(data == TD_NULL, SAMPLE_SVP_ERR_LEVEL_ERROR, + "get %u-th data addr failed!\n", i); + + stride = svp_acl_get_data_buffer_stride(data_buffer); + sample_svp_check_exps_return_void(stride == 0, SAMPLE_SVP_ERR_LEVEL_ERROR, + "get %u-th data stride failed!\n", i); + + size = svp_acl_get_data_buffer_size(data_buffer); + sample_svp_check_exps_return_void(size == 0, SAMPLE_SVP_ERR_LEVEL_ERROR, "get %u-th data size failed!\n", i); + + sample_svp_check_exps_return_void(size < (td_u64)task->cfg.dynamic_batch_num * total_line_num * stride, + SAMPLE_SVP_ERR_LEVEL_ERROR, "%u-th data buffer size(%lu) is less than needed(%llu)!\n", + i, size, (td_u64)task->cfg.dynamic_batch_num * total_line_num * stride); + + ret = snprintf_s(name, SAMPLE_SVP_NPU_MAX_NAME_LEN - 1, SAMPLE_SVP_NPU_MAX_NAME_LEN - 1, file_name, i); + sample_svp_check_exps_return_void(ret < 0, SAMPLE_SVP_ERR_LEVEL_ERROR, "create file name failed!\n"); + + fp = fopen(name, "wb"); + sample_svp_check_exps_return_void(fp == TD_NULL, SAMPLE_SVP_ERR_LEVEL_ERROR, "open file failed!\n"); + + for (line = 0; line < task->cfg.dynamic_batch_num * total_line_num; line++) { + ret = fwrite(data, line_byte_num, 1, fp); + if (ret != 1) { + fclose(fp); + sample_svp_check_exps_return_void(ret != 1, SAMPLE_SVP_ERR_LEVEL_ERROR, "write file failed!\n"); + } + data += stride; + } + (td_void)fclose(fp); + } +} + +td_void sample_common_svp_npu_dump_task_data(const sample_svp_npu_task_info *task) +{ + if (sample_svp_check_task_cfg(task) != TD_SUCCESS) { + sample_svp_trace_err("check task cfg failed!\n"); + return; + } + + sample_svp_check_exps_return_void(task->input_dataset == TD_NULL, SAMPLE_SVP_ERR_LEVEL_ERROR, + "task->input_dataset is NULL!\n"); + sample_svp_check_exps_return_void(task->output_dataset == TD_NULL, SAMPLE_SVP_ERR_LEVEL_ERROR, + "task->input_dataset is NULL!\n"); + + (td_void)sample_svp_npu_dump_dataset(task, "input_%u.bin", TD_TRUE); + (td_void)sample_svp_npu_dump_dataset(task, "output_%u.bin", TD_FALSE); +} + +td_s32 sample_common_svp_npu_update_input_data_buffer_info(td_u8 *virt_addr, td_u32 size, td_u32 stride, td_u32 idx, + const sample_svp_npu_task_info *task) +{ + svp_acl_data_buffer *data_buffer = TD_NULL; + svp_acl_error ret; + + data_buffer = svp_acl_mdl_get_dataset_buffer(task->input_dataset, idx); + sample_svp_check_exps_return(data_buffer == TD_NULL, TD_FAILURE, SAMPLE_SVP_ERR_LEVEL_ERROR, + "get %u-th data_buffer failed!\n", idx); + ret = svp_acl_update_data_buffer(data_buffer, (td_void *)virt_addr, size, stride); + sample_svp_check_exps_return(ret != SVP_ACL_SUCCESS, TD_FAILURE, SAMPLE_SVP_ERR_LEVEL_ERROR, + "update data buffer failed!\n"); + return TD_SUCCESS; +} + +td_s32 sample_common_svp_npu_get_input_data_buffer_info(const sample_svp_npu_task_info *task, td_u32 idx, + td_u8 **virt_addr, td_u32 *size, td_u32 *stride) +{ + svp_acl_data_buffer *data_buffer = TD_NULL; + + if (sample_svp_check_task_cfg(task) != TD_SUCCESS) { + sample_svp_trace_err("check task cfg failed!\n"); + return TD_FAILURE; + } + sample_svp_check_exps_return(virt_addr == TD_NULL, TD_FAILURE, SAMPLE_SVP_ERR_LEVEL_ERROR, "virt_addr is NULL!\n"); + sample_svp_check_exps_return(size == TD_NULL, TD_FAILURE, SAMPLE_SVP_ERR_LEVEL_ERROR, "size is NULL!\n"); + sample_svp_check_exps_return(stride == TD_NULL, TD_FAILURE, SAMPLE_SVP_ERR_LEVEL_ERROR, "size is NULL!\n"); + + data_buffer = svp_acl_mdl_get_dataset_buffer(task->input_dataset, idx); + sample_svp_check_exps_return(data_buffer == TD_NULL, TD_FAILURE, SAMPLE_SVP_ERR_LEVEL_ERROR, + "get 0-th data_buffer failed!\n"); + *size = (td_u32)svp_acl_get_data_buffer_size(data_buffer); + *stride = (td_u32)svp_acl_get_data_buffer_stride(data_buffer); + *virt_addr = (td_u8 *)svp_acl_get_data_buffer_addr(data_buffer); + return TD_SUCCESS; +} + +static td_s32 sample_svp_npu_get_roi_num(const sample_svp_npu_task_info *task, sample_svp_npu_detection_info *info, + ot_sample_svp_rect_info *rect_info) +{ + svp_acl_error ret; + size_t num_idx; + svp_acl_mdl_io_dims dims; + svp_acl_data_buffer *data_buffer = TD_NULL; + td_float *data = TD_NULL; + td_float total_num = 0.0; + td_u32 start = 0; + td_u32 i; + + ret = svp_acl_mdl_get_output_index_by_name(g_svp_npu_model[task->cfg.model_idx].model_desc, + info->num_name, &num_idx); + sample_svp_check_exps_return(ret != SVP_ACL_SUCCESS, TD_FAILURE, SAMPLE_SVP_ERR_LEVEL_ERROR, + "get num idx failed!\n"); + + ret = svp_acl_mdl_get_output_dims(g_svp_npu_model[task->cfg.model_idx].model_desc, num_idx, &dims); + sample_svp_check_exps_return(ret != SVP_ACL_SUCCESS, TD_FAILURE, SAMPLE_SVP_ERR_LEVEL_ERROR, + "get num dims failed!\n"); + + data_buffer = svp_acl_mdl_get_dataset_buffer(task->output_dataset, num_idx); + sample_svp_check_exps_return(data_buffer == TD_NULL, TD_FAILURE, SAMPLE_SVP_ERR_LEVEL_ERROR, + "get num data_buffer is NULL!\n"); + + data = (td_float *)svp_acl_get_data_buffer_addr(data_buffer); + sample_svp_check_exps_return(data == TD_NULL, TD_FAILURE, SAMPLE_SVP_ERR_LEVEL_ERROR, "get num data is NULL!\n"); + + if (info->has_background == TD_TRUE) { + info->roi_offset = (td_u32)(*data); + start++; + } + + for (i = start; i < dims.dims[dims.dim_count - 1]; i++) { + total_num += *(data + i); + } + rect_info->num = (td_u32)total_num; + return TD_SUCCESS; +} + +static td_s32 sample_svp_npu_get_roi(const sample_svp_npu_task_info *task, const sample_svp_npu_detection_info *info, + const ot_video_frame_info *proc_frame, const ot_video_frame_info *show_frame, ot_sample_svp_rect_info *rect_info) +{ + svp_acl_error ret; + size_t roi_idx; + size_t stride; + svp_acl_data_buffer *data_buffer = TD_NULL; + td_float *x_min = TD_NULL; + td_float *y_min = TD_NULL; + td_float *x_max = TD_NULL; + td_float *y_max = TD_NULL; + td_float *score = TD_NULL; + td_float *class_id = TD_NULL; + td_u32 roi_offset = info->roi_offset; + td_u32 i; + + ret = svp_acl_mdl_get_output_index_by_name(g_svp_npu_model[task->cfg.model_idx].model_desc, + info->roi_name, &roi_idx); + sample_svp_check_exps_return(ret != SVP_ACL_SUCCESS, TD_FAILURE, SAMPLE_SVP_ERR_LEVEL_ERROR, + "get roi idx failed!\n"); + + data_buffer = svp_acl_mdl_get_dataset_buffer(task->output_dataset, roi_idx); + sample_svp_check_exps_return(data_buffer == TD_NULL, TD_FAILURE, SAMPLE_SVP_ERR_LEVEL_ERROR, + "get roi data_buffer is NULL!\n"); + + stride = svp_acl_get_data_buffer_stride(data_buffer); + sample_svp_check_exps_return(stride == 0, TD_FAILURE, SAMPLE_SVP_ERR_LEVEL_ERROR, "get roi stride is 0!\n"); + + x_min = (td_float *)svp_acl_get_data_buffer_addr(data_buffer); + sample_svp_check_exps_return(x_min == TD_NULL, TD_FAILURE, SAMPLE_SVP_ERR_LEVEL_ERROR, "get roi data is NULL!\n"); + y_min = x_min + stride / sizeof(td_float); + x_max = y_min + stride / sizeof(td_float); + y_max = x_max + stride / sizeof(td_float); + score = y_max + stride / sizeof(td_float); + class_id = score + stride / sizeof(td_float); + + for (i = 0; i < rect_info->num; i++) { + rect_info->rect[i].point[SAMPLE_SVP_NPU_RECT_LEFT_TOP].x = (td_u32)((td_float)x_min[roi_offset] / + proc_frame->video_frame.width * show_frame->video_frame.width) & (~1); + rect_info->rect[i].point[SAMPLE_SVP_NPU_RECT_LEFT_TOP].y = (td_u32)((td_float)y_min[roi_offset] / + proc_frame->video_frame.height * show_frame->video_frame.height) & (~1); + + rect_info->rect[i].point[SAMPLE_SVP_NPU_RECT_RIGHT_TOP].x = (td_u32)((td_float)x_max[roi_offset] / + proc_frame->video_frame.width * show_frame->video_frame.width) & (~1); + rect_info->rect[i].point[SAMPLE_SVP_NPU_RECT_RIGHT_TOP].y = + rect_info->rect[i].point[SAMPLE_SVP_NPU_RECT_LEFT_TOP].y; + + rect_info->rect[i].point[SAMPLE_SVP_NPU_RECT_RIGHT_BOTTOM].x = + rect_info->rect[i].point[SAMPLE_SVP_NPU_RECT_RIGHT_TOP].x; + rect_info->rect[i].point[SAMPLE_SVP_NPU_RECT_RIGHT_BOTTOM].y = (td_u32)((td_float)y_max[roi_offset] / + proc_frame->video_frame.height * show_frame->video_frame.height) & (~1); + + rect_info->rect[i].point[SAMPLE_SVP_NPU_RECT_LEFT_BOTTOM].x = + rect_info->rect[i].point[SAMPLE_SVP_NPU_RECT_LEFT_TOP].x; + rect_info->rect[i].point[SAMPLE_SVP_NPU_RECT_LEFT_BOTTOM].y = + rect_info->rect[i].point[SAMPLE_SVP_NPU_RECT_RIGHT_BOTTOM].y; + + rect_info->rect[i].score = (td_float)((td_float)score[roi_offset]); + rect_info->rect[i].class_id = (td_u16)((td_float)class_id[roi_offset]); + + roi_offset++; + } + return TD_SUCCESS; +} + +static td_float sample_svp_npu_calc_iou(const sample_svp_npu_roi_box* box1, const sample_svp_npu_roi_box* box2) +{ + td_float area1 = box1->area; + td_float area2 = box2->area; + td_float xx1 = (box1->lx > box2->lx) ? box1->lx : box2->lx; + td_float yy1 = (box1->ly > box2->ly) ? box1->ly : box2->ly; + td_float xx2 = (box1->rx < box2->rx) ? box1->rx : box2->rx; + td_float yy2 = (box1->ry < box2->ry) ? box1->ry : box2->ry; + td_float w = (0.0f > (xx2 - xx1 + 1)) ? 0.0f : (xx2 - xx1 + 1); + td_float h = (0.0f > (yy2 - yy1 + 1)) ? 0.0f : (yy2 - yy1 + 1); + td_float inter = w * h; + td_float ovr = inter /(area1 + area2 - inter); + return ovr; +} + +static td_void sample_svp_npu_multiclass_nms(sample_svp_npu_roi_box *roi_box, td_u32 *roi_box_num, + sample_svp_npu_valid_box* valid_box, td_u32 valid_box_num) +{ + float nms_thr = 0.45; /* 0.45: threshold of ROI */ + for (td_u32 i = 0; i < valid_box_num; i++) { + sample_svp_npu_valid_box *item = &valid_box[i]; + td_float box_x_center = item->xcenter; + td_float box_y_center = item->ycenter; + td_float box_width = item->w; + td_float box_height = item->h; + + td_float x1 = (box_x_center - box_width / 2); + td_float y1 = (box_y_center - box_height / 2); + td_float x2 = (box_x_center + box_width / 2); + td_float y2 = (box_y_center + box_height / 2); + td_float area = (x2 - x1 + 1) * (y2 - y1 + 1); + td_bool keep = TD_TRUE; + /* lx, ly, rx, ry, score, class id, area */ + sample_svp_npu_roi_box box = {x1, y1, x2, y2, item->score, item->class_id, area}; + for (td_u32 j = 0; j < *roi_box_num; j++) { + if (sample_svp_npu_calc_iou(&box, &roi_box[j]) > nms_thr) { + keep = TD_FALSE; + break; + } + } + if (keep) { + if (*roi_box_num >= OT_SVP_RECT_NUM) { + sample_svp_trace_err("roi_box_num [%u] is large than %u!\n", *roi_box_num, OT_SVP_RECT_NUM); + return; + } + roi_box[*roi_box_num].lx = box.lx; + roi_box[*roi_box_num].ly = box.ly; + roi_box[*roi_box_num].rx = box.rx; + roi_box[*roi_box_num].ry = box.ry; + roi_box[*roi_box_num].score = box.score; + roi_box[*roi_box_num].class_id = box.class_id; + roi_box[*roi_box_num].area = box.area; + *roi_box_num = *roi_box_num + 1; + } + } +} + +inline static td_float sample_svp_npu_sig_mod(td_float a) +{ + return 1.0f / (1.0f + exp(-a)); +} + +static td_void sample_svp_npu_get_max_score_and_idx(td_u32 obj_score_idx, td_u32 chn_step, const td_float* out_data, + td_float *max_cls_score, td_u32 *max_cls_id) +{ + td_u32 cls_score_idx = obj_score_idx + chn_step; + for (td_u32 c = 0; c < SAMPLE_SVP_NPU_YOLO_CLASS_NUM; c++) { + td_float cls_score_val = out_data[cls_score_idx]; + if (cls_score_val > *max_cls_score) { + *max_cls_score = cls_score_val; + *max_cls_id = c; + } + cls_score_idx += chn_step; + } +} +static td_u32 g_sample_svp_npu_anchor_grids[SAMPLE_SVP_NPU_YOLO_CPU_OUT_NUM][SAMPLE_SVP_NPU_YOLO_CLASS_MEMBER_NUM] = { + {116, 90, 156, 198, 373, 326}, + {30, 61, 62, 45, 59, 119}, + {10, 13, 16, 30, 33, 23} +}; + +static td_void sample_svp_npu_process_per_dectection_inner(const sample_svp_npu_detection_inner_param *inner_param, + td_float *grids_x, td_float *grids_y, sample_svp_npu_valid_box* valid_box, td_u32 *valid_box_num) +{ + td_float box_hieght; + td_u32 expanded_strides[SAMPLE_SVP_NPU_YOLO_CPU_OUT_NUM] = { 32, 16, 8 }; + td_u32 out_height_idx = inner_param->out_height_idx; + td_u32 chn_step = inner_param->chn_step; + td_float score_thr = inner_param->score_thr; + td_float *out_data = inner_param->out_data; + size_t w_stride_offset = inner_param->w_stride_offset; + td_u32 obj_score_offset = inner_param->obj_score_offset; + td_u32 offset = out_height_idx * inner_param->w_stride_offset; + for (td_u32 j = 0; j < inner_param->out_width; j++) { + for (td_u32 k = 0; k < SAMPLE_SVP_NPU_YOLO_SCALE_SIZE; k++) { + offset = j + out_height_idx * w_stride_offset + k * chn_step * SAMPLE_SVP_NPU_YOLO_OUT_PARM_NUM; + td_u32 obj_score_idx = offset + obj_score_offset; + td_float obj_score_val = sample_svp_npu_sig_mod(out_data[obj_score_idx]); + if (obj_score_val <= score_thr) { + continue; + } + if (*valid_box_num >= OT_SVP_RECT_NUM) { + sample_svp_trace_err("valid_box_num [%u] is large than %u!\n", *valid_box_num, OT_SVP_RECT_NUM); + return; + } + /* max score */ + td_float max_cls_score = 0.0f; + td_u32 max_cls_id = 0; + sample_svp_npu_get_max_score_and_idx(obj_score_idx, chn_step, out_data, &max_cls_score, &max_cls_id); + + td_float confidence_score = sample_svp_npu_sig_mod(max_cls_score) * obj_score_val; + if (confidence_score > score_thr) { + /* gen box info */ + td_u32 x_center_idx = offset; + td_u32 y_center_idx = x_center_idx + chn_step; + td_u32 box_width_idx = y_center_idx + chn_step; + td_u32 box_height_idx = box_width_idx + chn_step; + td_float x_center = (sample_svp_npu_sig_mod(out_data[x_center_idx]) * SAMPLE_SVP_NPU_DOUBLE + + grids_x[j]) * expanded_strides[inner_param->idx]; + td_float y_center = (sample_svp_npu_sig_mod(out_data[y_center_idx]) * SAMPLE_SVP_NPU_DOUBLE + + grids_y[out_height_idx]) * expanded_strides[inner_param->idx]; + td_float tmp_value = sample_svp_npu_sig_mod(out_data[box_width_idx]) * SAMPLE_SVP_NPU_DOUBLE; + td_float box_width = tmp_value * tmp_value * g_sample_svp_npu_anchor_grids[inner_param->idx][(k << 1)]; + tmp_value = sample_svp_npu_sig_mod(out_data[box_height_idx]) * SAMPLE_SVP_NPU_DOUBLE; + box_hieght = tmp_value * tmp_value * g_sample_svp_npu_anchor_grids[inner_param->idx][(k << 1) +1]; + valid_box[*valid_box_num].score = confidence_score; + valid_box[*valid_box_num].xcenter = x_center; + valid_box[*valid_box_num].ycenter = y_center; + valid_box[*valid_box_num].w = box_width; + valid_box[*valid_box_num].h = box_hieght; + valid_box[*valid_box_num].class_id = (td_float)max_cls_id; + *valid_box_num = *valid_box_num + 1; + } + } + } +} + +static td_void sample_svp_npu_process_per_dectection(const sample_svp_npu_task_info *task, td_u32 idx, + sample_svp_npu_valid_box* valid_box, td_u32 *valid_box_num) +{ + td_u32 out_height; + svp_acl_mdl_io_dims out_dims; + sample_svp_npu_detection_inner_param inner_param; + svp_acl_data_buffer* data_buffer = TD_NULL; + td_u32 h_sizes[SAMPLE_SVP_NPU_YOLO_CPU_OUT_NUM] = { 20, 40, 80 }; + td_u32 w_sizes[SAMPLE_SVP_NPU_YOLO_CPU_OUT_NUM] = { 20, 40, 80 }; + /* gen grids */ + td_float grids_x[SAMPLE_SVP_NPU_YOLO_CLASS_NUM] = {0}; + td_float grids_y[SAMPLE_SVP_NPU_YOLO_CLASS_NUM] = {0}; + + svp_acl_mdl_get_output_dims(g_svp_npu_model[task->cfg.model_idx].model_desc, idx, &out_dims); + data_buffer = svp_acl_mdl_get_dataset_buffer(task->output_dataset, idx); + out_height = out_dims.dims[out_dims.dim_count - SAMPLE_SVP_NPU_DOUBLE]; + inner_param.out_width = out_dims.dims[out_dims.dim_count - 1]; + if (out_height > SAMPLE_SVP_NPU_YOLO_CLASS_NUM || inner_param.out_width > SAMPLE_SVP_NPU_YOLO_CLASS_NUM) { + sample_svp_trace_err("out_width[%u] or out_width[%u] is large than %u!\n", + out_height, inner_param.out_width, SAMPLE_SVP_NPU_YOLO_CLASS_NUM); + return; + } + + inner_param.score_thr = SAMPLE_SVP_NPU_YOLO_SCORE_THRESHOLE; + inner_param.idx = idx; + inner_param.out_data = (td_float *)svp_acl_get_data_buffer_addr(data_buffer); + inner_param.w_stride_offset = + svp_acl_mdl_get_output_default_stride(g_svp_npu_model[task->cfg.model_idx].model_desc, idx) / sizeof(td_float); + inner_param.chn_step = out_height * inner_param.w_stride_offset; + + for (td_u32 i = 0; i < h_sizes[idx]; i++) { + grids_y[i] = i - SAMPLE_SVP_NPU_YOLO_SCORE_THRESHOLE; + } + for (td_u32 i = 0; i < w_sizes[idx]; i++) { + grids_x[i] = i - SAMPLE_SVP_NPU_YOLO_SCORE_THRESHOLE; + } + inner_param.obj_score_offset = sizeof(td_float) * inner_param.chn_step; + for (td_u32 i = 0; i < out_height; i++) { + inner_param.out_height_idx = i; + sample_svp_npu_process_per_dectection_inner(&inner_param, grids_x, grids_y, valid_box, valid_box_num); + } +} + +static td_void sample_svp_npu_filter_yolo_box(const sample_svp_npu_task_info *task, + sample_svp_npu_valid_box* valid_box, td_u32 *valid_box_num) +{ + td_u32 out_num = svp_acl_mdl_get_num_outputs(g_svp_npu_model[task->cfg.model_idx].model_desc); + if (out_num > SAMPLE_SVP_NPU_YOLO_CPU_OUT_NUM) { + sample_svp_trace_err("output num [%u] is large than 3!\n", out_num); + return; + } + + /* gen box */ + for (td_u32 n = 0; n < out_num; n++) { + sample_svp_npu_process_per_dectection(task, n, valid_box, valid_box_num); + } +} + +static td_void sample_svp_npu_fill_rect_info(const ot_video_frame_info *proc_frame, + const ot_video_frame_info *show_frame, const sample_svp_npu_roi_box *roi_box, + td_u32 roi_box_num, ot_sample_svp_rect_info *rect_info) +{ + td_u32 rect_num = 0; + td_float w_multiples = (td_float)show_frame->video_frame.width / (td_float)proc_frame->video_frame.width; + td_float h_multiples = (td_float)show_frame->video_frame.height / (td_float)proc_frame->video_frame.height; + for (td_u32 i = 0; i < roi_box_num; i++) { + if (roi_box[i].score > 1.0f || roi_box[i].score < 0.0f) { + continue; + } + rect_info->rect[rect_num].point[SAMPLE_SVP_NPU_RECT_LEFT_TOP].x = (td_u32)(roi_box[i].lx * w_multiples) & (~1); + rect_info->rect[rect_num].point[SAMPLE_SVP_NPU_RECT_LEFT_TOP].y = (td_u32)(roi_box[i].ly * h_multiples) & (~1); + rect_info->rect[rect_num].point[SAMPLE_SVP_NPU_RECT_RIGHT_TOP].x = (td_u32)(roi_box[i].rx * w_multiples) & (~1); + rect_info->rect[rect_num].point[SAMPLE_SVP_NPU_RECT_RIGHT_TOP].y = + rect_info->rect[rect_num].point[SAMPLE_SVP_NPU_RECT_LEFT_TOP].y; + rect_info->rect[rect_num].point[SAMPLE_SVP_NPU_RECT_RIGHT_BOTTOM].x = + rect_info->rect[rect_num].point[SAMPLE_SVP_NPU_RECT_RIGHT_TOP].x; + rect_info->rect[rect_num].point[SAMPLE_SVP_NPU_RECT_RIGHT_BOTTOM].y = + (td_u32)(roi_box[i].ry * h_multiples) & (~1); + rect_info->rect[rect_num].point[SAMPLE_SVP_NPU_RECT_LEFT_BOTTOM].x = + rect_info->rect[rect_num].point[SAMPLE_SVP_NPU_RECT_LEFT_TOP].x; + rect_info->rect[rect_num].point[SAMPLE_SVP_NPU_RECT_LEFT_BOTTOM].y = + rect_info->rect[rect_num].point[SAMPLE_SVP_NPU_RECT_RIGHT_BOTTOM].y; + rect_num++; + } + rect_info->num = rect_num; +} + +static td_s32 cmp_valid_box(const void *item1, const void *item2) +{ + sample_svp_npu_valid_box *box1 = (sample_svp_npu_valid_box *)item1; + sample_svp_npu_valid_box *box2 = (sample_svp_npu_valid_box *)item2; + if (box1->score > box2->score) { + return 1; + } + return 0; +} + +static td_s32 cmp_roi_box(const void *item1, const void *item2) +{ + sample_svp_npu_roi_box *box1 = (sample_svp_npu_roi_box *)item1; + sample_svp_npu_roi_box *box2 = (sample_svp_npu_roi_box *)item2; + if (box1->class_id < box2->class_id) { + return 1; + } else if (box1->class_id == box2->class_id) { + return box1->score > box2->score; + } + return 0; +} + +static td_s32 sample_svp_npu_get_roi_by_cpu(const sample_svp_npu_task_info *task, + const sample_svp_npu_detection_info *info, const ot_video_frame_info *proc_frame, + const ot_video_frame_info *show_frame, ot_sample_svp_rect_info *rect_info) +{ + td_u32 valid_box_num = 0; + td_u32 roi_box_num = 0; + sample_svp_npu_valid_box valid_box[OT_SVP_RECT_NUM] = {0}; + sample_svp_npu_roi_box roi_box[OT_SVP_RECT_NUM] = {0}; + + sample_svp_npu_filter_yolo_box(task, valid_box, &valid_box_num); + qsort((td_void *)valid_box, valid_box_num, sizeof(sample_svp_npu_valid_box), cmp_valid_box); + + sample_svp_npu_multiclass_nms(roi_box, &roi_box_num, valid_box, valid_box_num); + if (roi_box_num == 0) { + sample_svp_trace_info("No target object is detected !!!\n"); + return TD_SUCCESS; + } + qsort((td_void *)roi_box, roi_box_num, sizeof(sample_svp_npu_roi_box), cmp_roi_box); + sample_svp_npu_fill_rect_info(proc_frame, show_frame, roi_box, roi_box_num, rect_info); + + return TD_SUCCESS; +} + +td_s32 sample_common_svp_npu_roi_to_rect(const sample_svp_npu_task_info *task, + sample_svp_npu_detection_info *detection_info, const ot_video_frame_info *proc_frame, + const ot_video_frame_info *show_frame, ot_sample_svp_rect_info *rect_info) +{ + td_s32 ret; + td_u32 i; + td_float max_confidence = 0.0f; + td_u32 max_index = 0; + + if (sample_svp_check_task_cfg(task) != TD_SUCCESS) { + sample_svp_trace_err("check task cfg failed!\n"); + return TD_FAILURE; + } + sample_svp_check_exps_return(detection_info == TD_NULL, TD_FAILURE, SAMPLE_SVP_ERR_LEVEL_ERROR, + "detection_info is NULL!\n"); + sample_svp_check_exps_return(proc_frame == TD_NULL, TD_FAILURE, SAMPLE_SVP_ERR_LEVEL_ERROR, + "proc_frame is NULL!\n"); + sample_svp_check_exps_return(show_frame == TD_NULL, TD_FAILURE, SAMPLE_SVP_ERR_LEVEL_ERROR, + "show_frame is NULL!\n"); + sample_svp_check_exps_return(rect_info == TD_NULL, TD_FAILURE, SAMPLE_SVP_ERR_LEVEL_ERROR, + "rect_info is NULL!\n"); + + detection_info->roi_offset = 0; + ret = sample_svp_npu_get_roi_num(task, detection_info, rect_info); + sample_svp_check_exps_return(ret != TD_SUCCESS, TD_FAILURE, SAMPLE_SVP_ERR_LEVEL_ERROR, + "get roi num failed!\n"); + + ret = sample_svp_npu_get_roi(task, detection_info, proc_frame, show_frame, rect_info); + sample_svp_check_exps_return(ret != TD_SUCCESS, TD_FAILURE, SAMPLE_SVP_ERR_LEVEL_ERROR, + "get roi failed!\n"); + + // 新增部分：筛选置信度最高的矩形框 + if (rect_info->num > 0) { + // 1. 查找置信度最高的矩形框 + for (i = 0; i < rect_info->num; i++) { + if (rect_info->rect[i].score > max_confidence) { + max_confidence = rect_info->rect[i].score; + max_index = i; + } + } + + // 2. 只保留置信度最高的矩形框 + if (rect_info->num > 1) { + rect_info->rect[0] = rect_info->rect[max_index]; + rect_info->num = 1; + } + } + + return TD_SUCCESS; +} + +td_s32 sample_common_svp_npu_roi_to_rect_cpu(const sample_svp_npu_task_info *task, + sample_svp_npu_detection_info *detection_info, const ot_video_frame_info *proc_frame, + const ot_video_frame_info *show_frame, ot_sample_svp_rect_info *rect_info) +{ + td_s32 ret; + + if (sample_svp_check_task_cfg(task) != TD_SUCCESS) { + sample_svp_trace_err("check task cfg failed!\n"); + return TD_FAILURE; + } + sample_svp_check_exps_return(detection_info == TD_NULL, TD_FAILURE, SAMPLE_SVP_ERR_LEVEL_ERROR, + "detection_info is NULL!\n"); + sample_svp_check_exps_return(proc_frame == TD_NULL, TD_FAILURE, SAMPLE_SVP_ERR_LEVEL_ERROR, + "proc_frame is NULL!\n"); + sample_svp_check_exps_return(show_frame == TD_NULL, TD_FAILURE, SAMPLE_SVP_ERR_LEVEL_ERROR, + "show_frame is NULL!\n"); + sample_svp_check_exps_return(rect_info == TD_NULL, TD_FAILURE, SAMPLE_SVP_ERR_LEVEL_ERROR, + "rect_info is NULL!\n"); + + ret = sample_svp_npu_get_roi_by_cpu(task, detection_info, proc_frame, show_frame, rect_info); + sample_svp_check_exps_return(ret != TD_SUCCESS, TD_FAILURE, SAMPLE_SVP_ERR_LEVEL_ERROR, + "get roi failed!\n"); + + return TD_SUCCESS; +} + +td_s32 sample_common_svp_npu_set_threshold(sample_svp_npu_threshold threshold[], td_u32 threshold_num, + const sample_svp_npu_task_info *task) +{ + td_u32 i; + td_u32 n; + svp_acl_error ret; + svp_acl_data_buffer *data_buffer = TD_NULL; + td_float *data = TD_NULL; + size_t idx; + size_t size; + + if (sample_svp_check_task_cfg(task) != TD_SUCCESS) { + sample_svp_trace_err("check task cfg failed!\n"); + return TD_FAILURE; + } + sample_svp_check_exps_return(threshold == TD_NULL, TD_FAILURE, SAMPLE_SVP_ERR_LEVEL_ERROR, "threshold is NULL!\n"); + sample_svp_check_exps_return(threshold_num == 0, TD_FAILURE, SAMPLE_SVP_ERR_LEVEL_ERROR, "threshold_num is 0!\n"); + + for (i = 0; i < threshold_num; i++) { + ret = svp_acl_mdl_get_input_index_by_name(g_svp_npu_model[task->cfg.model_idx].model_desc, + threshold[i].name, &idx); + sample_svp_check_exps_return(ret != SVP_ACL_SUCCESS, TD_FAILURE, SAMPLE_SVP_ERR_LEVEL_ERROR, + "get roi idx failed!\n"); + + data_buffer = svp_acl_mdl_get_dataset_buffer(task->input_dataset, idx); + sample_svp_check_exps_return(data_buffer == TD_NULL, TD_FAILURE, SAMPLE_SVP_ERR_LEVEL_ERROR, + "get roi data_buffer is NULL!\n"); + + size = svp_acl_get_data_buffer_size(data_buffer); + sample_svp_check_exps_return(size < SAMPLE_SVP_NPU_THRESHOLD_NUM * sizeof(td_float), TD_FAILURE, + SAMPLE_SVP_ERR_LEVEL_ERROR, "get size(%lu) is less than %lu!\n", + size, SAMPLE_SVP_NPU_THRESHOLD_NUM * sizeof(td_float)); + + data = (td_float *)svp_acl_get_data_buffer_addr(data_buffer); + sample_svp_check_exps_return(data == TD_NULL, TD_FAILURE, SAMPLE_SVP_ERR_LEVEL_ERROR, + "get roi data is NULL!\n"); + n = 0; + data[n++] = threshold[i].nms_threshold; + data[n++] = threshold[i].score_threshold; + data[n++] = threshold[i].min_height; + data[n++] = threshold[i].min_width; + } + return TD_SUCCESS; +} + +td_s32 sample_common_svp_npu_check_has_aicpu_task(const sample_svp_npu_task_info *task, td_bool *has_aicpu_task) +{ + td_u32 aicpu_task_num; + svp_acl_error ret; + + if (sample_svp_check_task_cfg(task) != TD_SUCCESS) { + sample_svp_trace_err("check task cfg failed!\n"); + return TD_FAILURE; + } + sample_svp_check_exps_return(has_aicpu_task == TD_NULL, TD_FAILURE, SAMPLE_SVP_ERR_LEVEL_ERROR, + "has_aicpu_task is NULL!\n"); + + ret = svp_acl_ext_get_mdl_aicpu_task_num(g_svp_npu_model[task->cfg.model_idx].model_id, &aicpu_task_num); + sample_svp_check_exps_return(ret != SVP_ACL_SUCCESS, TD_FAILURE, SAMPLE_SVP_ERR_LEVEL_ERROR, + "get aicpu task num failed, error code is %d!\n", ret); + + *has_aicpu_task = (aicpu_task_num == 0) ? TD_FALSE : TD_TRUE; + return TD_SUCCESS; +} + +sample_svp_npu_model_info* sample_common_svp_npu_get_model_info(td_u32 model_idx) +{ + sample_svp_check_exps_return(model_idx >= SAMPLE_SVP_NPU_MAX_MODEL_NUM, NULL, SAMPLE_SVP_ERR_LEVEL_ERROR, + "model_idx(%u) must be [0, %u)!\n", model_idx, SAMPLE_SVP_NPU_MAX_MODEL_NUM); + return &g_svp_npu_model[model_idx]; +} + +td_s32 sample_common_svp_npu_get_input_resolution(td_u32 model_idx, td_u32 input_idx, + ot_size *pic_size) +{ + svp_acl_error ret; + svp_acl_mdl_io_dims dims = {0}; + + ret = svp_acl_mdl_get_input_dims(g_svp_npu_model[model_idx].model_desc, input_idx, &dims); + sample_svp_check_exps_return(ret != SVP_ACL_SUCCESS, TD_FAILURE, SAMPLE_SVP_ERR_LEVEL_ERROR, + "get num dims failed!\n"); + pic_size->height = dims.dims[dims.dim_count - SAMPLE_SVP_NPU_H_DIM_IDX]; // NCHW + pic_size->width = dims.dims[dims.dim_count - 1]; // NCHW + return TD_SUCCESS; +} diff --git a/vendor/zsks/demo/fruit_identify/common/sample_common_svp_npu_model.h b/vendor/zsks/demo/fruit_identify/common/sample_common_svp_npu_model.h new file mode 100755 index 0000000000000000000000000000000000000000..18ff9e359a4c3cf2345f9c98d1821a1dcc2b5e93 --- /dev/null +++ b/vendor/zsks/demo/fruit_identify/common/sample_common_svp_npu_model.h @@ -0,0 +1,74 @@ +/* + * Copyright (c) 2025 HiSilicon (Shanghai) Technologies Co., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef SAMPLE_COMMON_SVP_NPU_MODEL_H +#define SAMPLE_COMMON_SVP_NPU_MODEL_H +#include "ot_type.h" +#include "svp_acl_mdl.h" +#include "sample_common_svp.h" +#include "sample_common_svp_npu.h" + +td_s32 sample_common_svp_npu_get_input_data(const td_char *src[], td_u32 file_num, + const sample_svp_npu_task_info *task); + +td_s32 sample_common_svp_npu_load_model(const td_char *model_path, td_u32 model_index, td_bool is_cached); +td_void sample_common_svp_npu_unload_model(td_u32 model_index); + +td_s32 sample_common_svp_npu_create_input(sample_svp_npu_task_info *task); +td_s32 sample_common_svp_npu_create_output(sample_svp_npu_task_info *task); +td_void sample_common_svp_npu_destroy_input(sample_svp_npu_task_info *task); +td_void sample_common_svp_npu_destroy_output(sample_svp_npu_task_info *task); + +td_s32 sample_common_svp_npu_create_task_buf(sample_svp_npu_task_info *task); +td_s32 sample_common_svp_npu_create_work_buf(sample_svp_npu_task_info *task); +td_void sample_common_svp_npu_destroy_task_buf(sample_svp_npu_task_info *task); +td_void sample_common_svp_npu_destroy_work_buf(sample_svp_npu_task_info *task); + +td_s32 sample_common_svp_npu_get_work_buf_info(const sample_svp_npu_task_info *task, + td_u32 *work_buf_size, td_u32 *work_buf_stride); +td_s32 sample_common_svp_npu_share_work_buf(const sample_svp_npu_shared_work_buf *shared_work_buf, + const sample_svp_npu_task_info *task); + +td_s32 sample_common_svp_npu_set_dynamic_batch(const sample_svp_npu_task_info *task); +td_s32 sample_common_svp_npu_model_execute(const sample_svp_npu_task_info *task); +td_void sample_common_svp_npu_output_classification_result(const sample_svp_npu_task_info *task); + +td_void sample_common_svp_npu_dump_task_data(const sample_svp_npu_task_info *task); + +td_s32 sample_common_svp_npu_set_threshold(sample_svp_npu_threshold threshold[], td_u32 threshold_num, + const sample_svp_npu_task_info *task); + +td_s32 sample_common_svp_npu_roi_to_rect(const sample_svp_npu_task_info *task, + sample_svp_npu_detection_info *detection_info, const ot_video_frame_info *proc_frame, + const ot_video_frame_info *show_frame, ot_sample_svp_rect_info *rect_info); + +td_s32 sample_common_svp_npu_roi_to_rect_cpu(const sample_svp_npu_task_info *task, + sample_svp_npu_detection_info *detection_info, const ot_video_frame_info *proc_frame, + const ot_video_frame_info *show_frame, ot_sample_svp_rect_info *rect_info); + +td_s32 sample_common_svp_npu_update_input_data_buffer_info(td_u8 *virt_addr, td_u32 size, td_u32 stride, td_u32 idx, + const sample_svp_npu_task_info *task); + +td_s32 sample_common_svp_npu_get_input_data_buffer_info(const sample_svp_npu_task_info *task, td_u32 idx, + td_u8 **virt_addr, td_u32 *size, td_u32 *stride); + +td_s32 sample_common_svp_npu_check_has_aicpu_task(const sample_svp_npu_task_info *task, td_bool *has_aicpu_task); + +sample_svp_npu_model_info* sample_common_svp_npu_get_model_info(td_u32 model_idx); + +td_s32 sample_common_svp_npu_get_input_resolution(td_u32 model_idx, td_u32 input_idx, + ot_size *pic_size); +#endif diff --git a/vendor/zsks/demo/fruit_identify/common/sample_common_svp_osd.c b/vendor/zsks/demo/fruit_identify/common/sample_common_svp_osd.c new file mode 100755 index 0000000000000000000000000000000000000000..2d0dcd4ef7ab8db1dcfdfa63d51f427e680806ff --- /dev/null +++ b/vendor/zsks/demo/fruit_identify/common/sample_common_svp_osd.c @@ -0,0 +1,210 @@ +/* + * Copyright (c) 2025 Zhongshan Kuangshi Microelectronics Technology Co., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Contact Information + * Author: yaohongtao + * Phone: +86-18604465633 + * Email: yht@cust.edu.cn + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "sample_comm.h" +#include "easy_log.h" +#include "xvp_common_osd.h" +#include "sample_common_svp_npu.h" + +#define OVERLAY_MIN_HANDLE 20 +#define BUFFER_SIZE 16 +#define CLASS_BUFFER_SIZE 10 +#define DOUBLE_OF_SIZE 2 +#define BACKGROUND_ALPHA 255 +#define OSD_BUFFER_SIZE 32 +#define WIDTH_OF_MIN 1920 +#define HEIGHT_OF_MIN 1080 +#define WIDTH_OF_MAX 2560 +#define HEIGHT_OF_MAX 1440 + +void sample_osd_region_init(ot_rgn_attr *region, ot_size img_size) +{ + td_s32 i; + region->type = OT_RGN_OVERLAYEX; + region->attr.overlay.canvas_num = 2; /* 2: canvas num */ + + for (i = 0; i < OT_RGN_CLUT_NUM; i++) { + region->attr.overlay.clut[i] = (td_u32)(0x000fff0f * (i + 1) * (i + 1)); + region->attr.overlay.clut[i] |= 0xff000000; + } + region->attr.overlay.pixel_format = OT_PIXEL_FORMAT_ARGB_1555; + region->attr.overlay.size.width = img_size.width * DOUBLE_OF_SIZE; + region->attr.overlay.size.height = img_size.height * DOUBLE_OF_SIZE; + region->attr.overlay.bg_color = 0xffff; +} + +void sample_osd_chn_attr_init(ot_rgn_chn_attr *chn_attr) +{ + chn_attr->type = OT_RGN_OVERLAYEX; + chn_attr->is_show = TD_TRUE; + chn_attr->attr.overlay_chn.bg_alpha = BACKGROUND_ALPHA; + chn_attr->attr.overlay_chn.fg_alpha = BACKGROUND_ALPHA; + chn_attr->attr.overlay_chn.qp_info.enable = TD_FALSE; + chn_attr->attr.overlay_chn.qp_info.is_abs_qp = TD_FALSE; + chn_attr->attr.overlay_chn.qp_info.qp_val = 0; + chn_attr->attr.overlay_chn.dst = OT_RGN_ATTACH_JPEG_MAIN; + chn_attr->attr.overlay_chn.layer = 0; +} + +void sample_osd_info_init(xvp_osd_info *osd_info, ot_rgn_attr region) +{ + osd_info->border_color = 0x0; + osd_info->font_color = SAMPLE_SVP_NPU_RECT_COLOR; + osd_info->factor = 2; /* 2: factor of osd info */ + osd_info->width = region.attr.overlay.size.width; + osd_info->height = region.attr.overlay.size.height; + osd_info->stride = region.attr.overlay.size.width; +} + +int sample_osd_init(sample_mpp_osd_info *mpp_osd_info) +{ + printf("sample_osd_init ...\r\n"); + td_s32 ret; + ot_rgn_attr region = {0}; + ot_mpp_chn chn = {0}; + ot_rgn_chn_attr chn_attr = {0}; + xvp_osd_info osd_info = {0}; + char osd_str[BUFFER_SIZE] = "scissors 0.00"; + unsigned char gb_str[BUFFER_SIZE] = {0}; + ot_size img_size = {BUFFER_SIZE}; + + chn.mod_id = OT_ID_VPSS; + chn.dev_id = 0; + chn.chn_id = 0; + + xvp_common_osd_utf8_to_gb2312((unsigned char *)osd_str, strlen(osd_str), gb_str); + xvp_common_osd_calculate_canvas_size(gb_str, 1, &img_size.width, &img_size.height); + printf("%s-%d l:%d w:%u h:%u\n", __func__, __LINE__, strlen(gb_str), img_size.width, img_size.height); + + sample_osd_region_init(®ion, img_size); + + ret = ss_mpi_rgn_create(mpp_osd_info->handle, ®ion); + if (ret != TD_SUCCESS) { + LOGE("ss_mpi_rgn_create failed with %#x!", ret); + } + + sample_osd_chn_attr_init(&chn_attr); + ret = ss_mpi_rgn_attach_to_chn(mpp_osd_info->handle, &chn, &chn_attr); + if (ret != TD_SUCCESS) { + LOGE("ss_mpi_rgn_attach_to_chn failed with %#x!", ret); + } + + sample_osd_info_init(&osd_info, region); + mpp_osd_info->chn = chn; + mpp_osd_info->chn_attr = chn_attr; + mpp_osd_info->osd_info = osd_info; + + xvp_common_osd_init(NULL); + + return 0; +} + +int sample_osd_exit(sample_mpp_osd_info *mpp_osd_info) +{ + printf("sample_osd_exit ...\r\n"); + td_s32 ret; + xvp_common_osd_exit(); + + ret = ss_mpi_rgn_detach_from_chn(mpp_osd_info->handle, &mpp_osd_info->chn); + if (ret != TD_SUCCESS) { + LOGE("ss_mpi_rgn_detach_from_chn failed with %#x!", ret); + } + + ret = ss_mpi_rgn_destroy(mpp_osd_info->handle); + if (ret != TD_SUCCESS) { + LOGE("ss_mpi_rgn_destroy failed with %#x!", ret); + } + + return 0; +} + +int sample_osd_draw_text(td_float score, td_u16 class_id, td_u16 x, td_u16 y, + sample_mpp_osd_info mpp_osd_info) +{ + printf("sample_osd_draw_text ...\r\n"); + td_s32 ret; + const char classes[][CLASS_BUFFER_SIZE] = {"梨子", "西瓜", "橙子", "香蕉", "苹果"}; + char osd_str[OSD_BUFFER_SIZE] = "西瓜 0.00"; + unsigned char gb_str[OSD_BUFFER_SIZE] = {0}; + ot_size img_size = {0}; + ot_bmp bitmap = {0}; + + bitmap.width = mpp_osd_info.osd_info.width; + bitmap.height = mpp_osd_info.osd_info.height; + bitmap.pixel_format = OT_PIXEL_FORMAT_ARGB_1555; + bitmap.data = (td_void*)malloc(bitmap.width * bitmap.height * DOUBLE_OF_SIZE); + + ret = snprintf(osd_str, sizeof(osd_str), "%s %0.2f", classes[class_id], score); + if (ret < 0) { + printf("%s:snprintf failed, ret:0x%x.\n", __FUNCTION__, ret); + } + memset_sp(bitmap.data, bitmap.width * bitmap.height * DOUBLE_OF_SIZE, 0x0, + bitmap.width * bitmap.height * DOUBLE_OF_SIZE); + memset_sp(gb_str, sizeof(gb_str), 0, sizeof(gb_str)); + xvp_common_osd_utf8_to_gb2312((unsigned char *)osd_str, strlen(osd_str), gb_str); + + xvp_common_osd_draw_text((unsigned char *)bitmap.data, gb_str, 0, 0, &mpp_osd_info.osd_info); + ret = ss_mpi_rgn_set_bmp(mpp_osd_info.handle, &bitmap); + if (ret != TD_SUCCESS) { + LOGE("ss_mpi_rgn_set_bmp failed with %#x!", ret); + } + int min_x = (int)(x * WIDTH_OF_MIN / WIDTH_OF_MAX); + int min_y = (int)(y * HEIGHT_OF_MIN / HEIGHT_OF_MAX); + + if ((min_x % DOUBLE_OF_SIZE) != 0) { + min_x = min_x + 1; + } + if ((min_y % DOUBLE_OF_SIZE) != 0) { + min_y = min_y + 1; + } + printf("min_x:%d, min_y:%d\r\n", min_x, min_y); + mpp_osd_info.chn_attr.attr.overlay_chn.point.x = min_x; + mpp_osd_info.chn_attr.attr.overlay_chn.point.y = min_y; + mpp_osd_info.chn_attr.is_show = TD_TRUE; + ss_mpi_rgn_set_display_attr(mpp_osd_info.handle, &mpp_osd_info.chn, &mpp_osd_info.chn_attr); + + if (bitmap.data != NULL) { + free(bitmap.data); + bitmap.data = NULL; + } + + return 0; +} + +int sample_osd_clean_text(sample_mpp_osd_info mpp_osd_info) +{ + ss_mpi_rgn_set_display_attr(mpp_osd_info.handle, &mpp_osd_info.chn, &mpp_osd_info.chn_attr); +} diff --git a/vendor/zsks/demo/fruit_identify/common/sample_common_svp_osd.h b/vendor/zsks/demo/fruit_identify/common/sample_common_svp_osd.h new file mode 100755 index 0000000000000000000000000000000000000000..96841f847f43bbdf44b8e098772a83f31cdf82b9 --- /dev/null +++ b/vendor/zsks/demo/fruit_identify/common/sample_common_svp_osd.h @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2025 Zhongshan Kuangshi Microelectronics Technology Co., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Contact Information + * Author: yaohongtao + * Phone: +86-18604465633 + * Email: yht@cust.edu.cn + */ + +#ifndef SAMPLE_COMMON_SVP_OSD_H +#define SAMPLE_COMMON_SVP_OSD_H +#include "xvp_common_osd.h" + +#ifdef __cplusplus +extern "C" { +#endif + +int sample_osd_init(sample_mpp_osd_info *mpp_osd_info); +int sample_osd_exit(sample_mpp_osd_info *mpp_osd_info); +int sample_osd_draw_text(td_float score, td_u16 class_id, td_u16 x, td_u16 y, sample_mpp_osd_info mpp_osd_info); +int sample_osd_clean_text(sample_mpp_osd_info mpp_osd_info); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/vendor/zsks/demo/fruit_identify/data/hzk/hzk16 b/vendor/zsks/demo/fruit_identify/data/hzk/hzk16 new file mode 100755 index 0000000000000000000000000000000000000000..460e0550372c3317fe0b901b14e84ce65ddc9e50 Binary files /dev/null and b/vendor/zsks/demo/fruit_identify/data/hzk/hzk16 differ diff --git a/vendor/zsks/demo/fruit_identify/data/model/fruit_identify.om b/vendor/zsks/demo/fruit_identify/data/model/fruit_identify.om new file mode 100755 index 0000000000000000000000000000000000000000..c8bda4529527724097744a302b5c311f530ccdb2 Binary files /dev/null and b/vendor/zsks/demo/fruit_identify/data/model/fruit_identify.om differ diff --git a/vendor/zsks/demo/fruit_identify/fruit_audio/apple/audio_chn0.aac b/vendor/zsks/demo/fruit_identify/fruit_audio/apple/audio_chn0.aac new file mode 100755 index 0000000000000000000000000000000000000000..be17a8c4b7dc63b33b3ec46196db5cedb1ab7a25 Binary files /dev/null and b/vendor/zsks/demo/fruit_identify/fruit_audio/apple/audio_chn0.aac differ diff --git a/vendor/zsks/demo/fruit_identify/fruit_audio/banana/audio_chn0.aac b/vendor/zsks/demo/fruit_identify/fruit_audio/banana/audio_chn0.aac new file mode 100755 index 0000000000000000000000000000000000000000..f7f529304bd73f4de175ba167ae501075c852b00 Binary files /dev/null and b/vendor/zsks/demo/fruit_identify/fruit_audio/banana/audio_chn0.aac differ diff --git a/vendor/zsks/demo/fruit_identify/fruit_audio/orange/audio_chn0.aac b/vendor/zsks/demo/fruit_identify/fruit_audio/orange/audio_chn0.aac new file mode 100755 index 0000000000000000000000000000000000000000..41d79b4688bcaba26971ff36e7bbd3bf42f47fbc Binary files /dev/null and b/vendor/zsks/demo/fruit_identify/fruit_audio/orange/audio_chn0.aac differ diff --git a/vendor/zsks/demo/fruit_identify/fruit_audio/pear/audio_chn0.aac b/vendor/zsks/demo/fruit_identify/fruit_audio/pear/audio_chn0.aac new file mode 100755 index 0000000000000000000000000000000000000000..cf40ce599eda4aaad8d6bc75465fd1117cd0d8ac Binary files /dev/null and b/vendor/zsks/demo/fruit_identify/fruit_audio/pear/audio_chn0.aac differ diff --git a/vendor/zsks/demo/fruit_identify/fruit_audio/watermelon/audio_chn0.aac b/vendor/zsks/demo/fruit_identify/fruit_audio/watermelon/audio_chn0.aac new file mode 100755 index 0000000000000000000000000000000000000000..7a13293d952ff025530b18bb49670a522013da11 Binary files /dev/null and b/vendor/zsks/demo/fruit_identify/fruit_audio/watermelon/audio_chn0.aac differ diff --git a/vendor/zsks/demo/fruit_identify/fruit_identify.c b/vendor/zsks/demo/fruit_identify/fruit_identify.c new file mode 100755 index 0000000000000000000000000000000000000000..d79b29e479c4829dc294934ca9b7b586c381b57b --- /dev/null +++ b/vendor/zsks/demo/fruit_identify/fruit_identify.c @@ -0,0 +1,709 @@ +/* + * Copyright (c) 2025 HiSilicon (Shanghai) Technologies Co., Ltd. + * Copyright (c) 2025 Zhongshan Kuangshi Microelectronics Technology Co., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Modifications Author: yaohongtao (yht@cust.edu.cn) + */ + +#include +#include +#include +#include +#include +/* 新增v4l2相关头文件 */ +#include // 文件控制 +#include // 错误处理 +#include // POSIX API +#include // 字符串操作 +#include // 内存映射 +#include // IO控制 +#include // select复用 +#include // v4l2核心 + +#include "svp_acl_rt.h" +#include "svp_acl.h" +#include "svp_acl_ext.h" +#include "sample_audio.h" // 包含音频文件 +#include "audio_aac_adp.h" +#include "sdk_module_init.h" +#include "common/sample_common_svp_osd.h" // osd头文件 +#include "common/sample_common_svp.h" +#include "common/sample_common_svp_npu.h" +#include "common/sample_common_svp_npu_model.h" + +#define SAMPLE_SVP_NPU_SHAERD_WORK_BUF_NUM 1 +#define SAMPLE_SVP_NPU_YOLO_TYPE_NUM 12 +#define SAMPLE_SVP_NPU_YOLO_THRESHOLD_NUM 1 +#define SAMPLE_SVP_NPU_PATH_LEN 0x100 + +static int g_last_played_class_id = -1; +static td_bool g_svp_npu_terminate_signal = TD_FALSE; +static td_s32 g_svp_npu_dev_id = 0; +static sample_svp_npu_task_info g_svp_npu_task[SAMPLE_SVP_NPU_MAX_TASK_NUM] = {0}; +static sample_svp_npu_shared_work_buf g_svp_npu_shared_work_buf[SAMPLE_SVP_NPU_SHAERD_WORK_BUF_NUM] = {0}; + +static sample_svp_npu_threshold g_svp_npu_rfcn_threshold[SAMPLE_SVP_NPU_RFCN_THRESHOLD_NUM] = { + {0.7, 0.0, 16.0, 16.0, "rpn_data"}, {0.3, 0.9, 16.0, 16.0, "rpn_data1"} +}; + +static sample_svp_npu_threshold g_svp_npu_yolo_threshold[SAMPLE_SVP_NPU_YOLO_TYPE_NUM] = { + {0.9, 0.15, 1.0, 1.0, "rpn_data"}, // yolov8 +}; + +static sample_svp_npu_roi_info g_svp_npu_yolo_roi_info[SAMPLE_SVP_NPU_YOLO_TYPE_NUM] = { + {"output0", "output0_"}, // yolov8 +}; + +static sample_vi_cfg g_vi_config; +static ot_sample_svp_rect_info g_svp_npu_rect_info = {0}; +static td_bool g_svp_npu_thread_stop = TD_FALSE; +static pthread_t g_svp_npu_thread = 0; +static sample_vo_cfg g_svp_npu_vo_cfg = { 0 }; +static pthread_t g_svp_npu_vdec_thread = 0; +static ot_vb_pool_info g_svp_npu_vb_pool_info; +static td_void *g_svp_npu_vb_virt_addr = TD_NULL; + +static ot_sample_svp_media_cfg g_svp_npu_media_cfg = { + .svp_switch = {TD_FALSE, TD_TRUE}, + .pic_type = {PIC_1080P, PIC_CIF}, + .chn_num = OT_SVP_MAX_VPSS_CHN_NUM, +}; + +static sample_vdec_attr g_svp_npu_vdec_cfg = { + .type = OT_PT_MJPEG, + .mode = OT_VDEC_SEND_MODE_FRAME, + .width = FHD_WIDTH, + .height = FHD_HEIGHT, + .sample_vdec_picture.pixel_format = OT_PIXEL_FORMAT_YVU_SEMIPLANAR_420, /* 420:pixel_format */ + .sample_vdec_picture.alpha = 0, /* 0:alpha */ + .display_frame_num = 2, /* 2:display_frame_num */ + .frame_buf_cnt = 5, /* 5:2+2+1 */ +}; + +static vdec_thread_param g_svp_npu_vdec_param = { + .chn_id = 0, + + .v4l2_device = "/dev/video0", // 摄像头设备 + .width = FHD_WIDTH, // 采集分辨率宽 + .height = FHD_HEIGHT, // 采集分辨率高 + .pixel_format = V4L2_PIX_FMT_MJPEG, // MJPEG格式 + + .type = OT_PT_MJPEG, + .stream_mode = OT_VDEC_SEND_MODE_FRAME, + .interval_time = 1000, /* 1000:interval_time */ + .pts_init = 0, + .pts_increase = 0, + .e_thread_ctrl = THREAD_CTRL_START, + .circle_send = TD_TRUE, + .milli_sec = 0, + .min_buf_size = (FHD_WIDTH * FHD_HEIGHT * 3) >> 1, /* 3:chn_size */ + .fps = 30, +}; + +// 音频线程 +typedef struct { + int class_id; +} audio_play_param; + +void* audio_play_thread(void* arg) +{ + audio_play_param* param = (audio_play_param*)arg; + td_s32 audio_ret = TD_SUCCESS; + + switch (param->class_id) { + case 0: + audio_ret = sample_audio_adec_ao("fruit_audio", "pear"); + break; + case 1: + audio_ret = sample_audio_adec_ao("fruit_audio", "watermelon"); + break; + case 2: /* 2: case of orange */ + audio_ret = sample_audio_adec_ao("fruit_audio", "orange"); + break; + case 3: /* 3: case of banana */ + audio_ret = sample_audio_adec_ao("fruit_audio", "banana"); + break; + case 4: /* 4: case of apple */ + audio_ret = sample_audio_adec_ao("fruit_audio", "apple"); + break; + default: + audio_ret = TD_SUCCESS; + break; + } + + if (audio_ret != TD_SUCCESS) { + sample_svp_trace_err("play audio failed!\n"); + } else { + sample_svp_trace_info("play audio success!\n"); + } + + free(param); // 释放参数 + return NULL; +} + +static td_void sample_svp_npu_acl_terminate(td_void) +{ + if (g_svp_npu_terminate_signal == TD_TRUE) { + printf("\033[0;31mprogram termination abnormally!\033[0;39m\n"); + } +} + +/* function : svp npu signal handle */ +td_void sample_svp_npu_acl_handle_sig(td_void) +{ + g_svp_npu_terminate_signal = TD_TRUE; +} + +static td_void sample_svp_npu_acl_deinit(td_void) +{ + svp_acl_error ret; + + ret = svp_acl_rt_reset_device(g_svp_npu_dev_id); + if (ret != SVP_ACL_SUCCESS) { + sample_svp_trace_err("reset device fail\n"); + } + sample_svp_trace_info("end to reset device is %d\n", g_svp_npu_dev_id); + + ret = svp_acl_finalize(); + if (ret != SVP_ACL_SUCCESS) { + sample_svp_trace_err("finalize acl fail\n"); + } + sample_svp_trace_info("end to finalize acl\n"); + (td_void)sample_common_svp_check_sys_exit(); +} + +static td_s32 sample_svp_npu_acl_init(const td_char *acl_config_path, td_bool vi_en) +{ + /* svp acl init */ + svp_acl_rt_run_mode run_mode; + svp_acl_error ret; + td_bool is_mpi_init; + + is_mpi_init = sample_common_svp_check_sys_init(vi_en); + sample_svp_check_exps_return(is_mpi_init != TD_TRUE, TD_FAILURE, SAMPLE_SVP_ERR_LEVEL_ERROR, "mpi init failed!\n"); + + ret = svp_acl_init(acl_config_path); + sample_svp_check_exps_return(ret != SVP_ACL_SUCCESS, TD_FAILURE, SAMPLE_SVP_ERR_LEVEL_ERROR, "acl init failed!\n"); + + sample_svp_trace_info("svp acl init success!\n"); + + /* open device */ + ret = svp_acl_rt_set_device(g_svp_npu_dev_id); + if (ret != SVP_ACL_SUCCESS) { + (td_void)svp_acl_finalize(); + sample_svp_trace_err("svp acl open device %d failed!\n", g_svp_npu_dev_id); + return TD_FAILURE; + } + sample_svp_trace_info("open device %d success!\n", g_svp_npu_dev_id); + + /* get run mode */ + ret = svp_acl_rt_get_run_mode(&run_mode); + if ((ret != SVP_ACL_SUCCESS) || (run_mode != SVP_ACL_DEVICE)) { + (td_void)svp_acl_rt_reset_device(g_svp_npu_dev_id); + (td_void)svp_acl_finalize(); + sample_svp_trace_err("acl get run mode failed!\n"); + return TD_FAILURE; + } + sample_svp_trace_info("get run mode success!\n"); + + return TD_SUCCESS; +} + +static td_s32 sample_svp_npu_acl_dataset_init(td_u32 task_idx) +{ + td_s32 ret = sample_common_svp_npu_create_input(&g_svp_npu_task[task_idx]); + sample_svp_check_exps_return(ret != TD_SUCCESS, TD_FAILURE, SAMPLE_SVP_ERR_LEVEL_ERROR, "create input failed!\n"); + + ret = sample_common_svp_npu_create_output(&g_svp_npu_task[task_idx]); + if (ret != TD_SUCCESS) { + sample_common_svp_npu_destroy_input(&g_svp_npu_task[task_idx]); + sample_svp_trace_err("execute create output fail.\n"); + return TD_FAILURE; + } + return TD_SUCCESS; +} + +static td_void sample_svp_npu_acl_dataset_deinit(td_u32 task_idx) +{ + (td_void)sample_common_svp_npu_destroy_input(&g_svp_npu_task[task_idx]); + (td_void)sample_common_svp_npu_destroy_output(&g_svp_npu_task[task_idx]); +} + +static td_void *sample_svp_npu_acl_thread_execute(td_void *args) +{ + td_s32 ret; + td_u32 task_idx = *(td_u32 *)args; + + ret = svp_acl_rt_set_device(g_svp_npu_dev_id); + sample_svp_check_exps_return(ret != TD_SUCCESS, TD_NULL, SAMPLE_SVP_ERR_LEVEL_ERROR, + "open device %d failed!\n", g_svp_npu_dev_id); + + ret = sample_common_svp_npu_model_execute(&g_svp_npu_task[task_idx]); + if (ret != SVP_ACL_SUCCESS) { + sample_svp_trace_err("execute inference failed of task[%u]!\n", task_idx); + } + + ret = svp_acl_rt_reset_device(g_svp_npu_dev_id); + if (ret != SVP_ACL_SUCCESS) { + sample_svp_trace_err("task[%u] reset device failed!\n", task_idx); + } + return TD_NULL; +} + +static td_void sample_svp_npu_acl_model_execute_multithread() +{ + pthread_t execute_threads[SAMPLE_SVP_NPU_MAX_THREAD_NUM] = {0}; + td_u32 idx[SAMPLE_SVP_NPU_MAX_THREAD_NUM] = {0}; + td_u32 task_idx; + + for (task_idx = 0; task_idx < SAMPLE_SVP_NPU_MAX_THREAD_NUM; task_idx++) { + idx[task_idx] = task_idx; + pthread_create(&execute_threads[task_idx], NULL, sample_svp_npu_acl_thread_execute, &idx[task_idx]); + } + + td_void *waitret[SAMPLE_SVP_NPU_MAX_THREAD_NUM]; + for (task_idx = 0; task_idx < SAMPLE_SVP_NPU_MAX_THREAD_NUM; task_idx++) { + pthread_join(execute_threads[task_idx], &waitret[task_idx]); + } + + for (task_idx = 0; task_idx < SAMPLE_SVP_NPU_MAX_THREAD_NUM; task_idx++) { + sample_svp_trace_info("output %u-th task data\n", task_idx); + sample_common_svp_npu_output_classification_result(&g_svp_npu_task[task_idx]); + } +} + +static td_void sample_svp_npu_acl_deinit_task(td_u32 task_num, td_u32 shared_work_buf_idx) +{ + td_u32 task_idx; + + for (task_idx = 0; task_idx < task_num; task_idx++) { + (td_void)sample_common_svp_npu_destroy_work_buf(&g_svp_npu_task[task_idx]); + (td_void)sample_common_svp_npu_destroy_task_buf(&g_svp_npu_task[task_idx]); + (td_void)sample_svp_npu_acl_dataset_deinit(task_idx); + (td_void)memset_s(&g_svp_npu_task[task_idx], sizeof(sample_svp_npu_task_cfg), 0, + sizeof(sample_svp_npu_task_cfg)); + } + if (g_svp_npu_shared_work_buf[shared_work_buf_idx].work_buf_ptr != TD_NULL) { + (td_void)svp_acl_rt_free(g_svp_npu_shared_work_buf[shared_work_buf_idx].work_buf_ptr); + g_svp_npu_shared_work_buf[shared_work_buf_idx].work_buf_ptr = TD_NULL; + g_svp_npu_shared_work_buf[shared_work_buf_idx].work_buf_size = 0; + g_svp_npu_shared_work_buf[shared_work_buf_idx].work_buf_stride = 0; + } +} + +static td_s32 sample_svp_npu_acl_create_shared_work_buf(td_u32 task_num, td_u32 shared_work_buf_idx) +{ + td_u32 task_idx; + td_u32 work_buf_size; + td_u32 work_buf_stride; + td_s32 ret; + + for (task_idx = 0; task_idx < task_num; task_idx++) { + ret = sample_common_svp_npu_get_work_buf_info(&g_svp_npu_task[task_idx], &work_buf_size, &work_buf_stride); + sample_svp_check_exps_return(ret != TD_SUCCESS, ret, SAMPLE_SVP_ERR_LEVEL_ERROR, + "get %u-th task work buf info failed!\n", task_idx); + + if (g_svp_npu_shared_work_buf[shared_work_buf_idx].work_buf_size < work_buf_size) { + g_svp_npu_shared_work_buf[shared_work_buf_idx].work_buf_size = work_buf_size; + g_svp_npu_shared_work_buf[shared_work_buf_idx].work_buf_stride = work_buf_stride; + } + } + ret = svp_acl_rt_malloc_cached(&g_svp_npu_shared_work_buf[shared_work_buf_idx].work_buf_ptr, + g_svp_npu_shared_work_buf[shared_work_buf_idx].work_buf_size, SVP_ACL_MEM_MALLOC_NORMAL_ONLY); + sample_svp_check_exps_return(ret != TD_SUCCESS, TD_FAILURE, SAMPLE_SVP_ERR_LEVEL_ERROR, + "malloc %u-th shared work buf failed!\n", shared_work_buf_idx); + + (td_void)svp_acl_rt_mem_flush(g_svp_npu_shared_work_buf[shared_work_buf_idx].work_buf_ptr, + g_svp_npu_shared_work_buf[shared_work_buf_idx].work_buf_size); + return TD_SUCCESS; +} + +static td_s32 sample_svp_npu_acl_init_task(td_u32 task_num, td_bool is_share_work_buf, + td_u32 shared_work_buf_idx) +{ + td_u32 task_idx; + td_s32 ret; + + if (is_share_work_buf == TD_TRUE) { + ret = sample_svp_npu_acl_create_shared_work_buf(task_num, shared_work_buf_idx); + sample_svp_check_exps_return(ret != TD_SUCCESS, TD_FAILURE, SAMPLE_SVP_ERR_LEVEL_ERROR, + "create shared work buf failed!\n"); + } + + for (task_idx = 0; task_idx < task_num; task_idx++) { + ret = sample_svp_npu_acl_dataset_init(task_idx); + if (ret != TD_SUCCESS) { + goto task_init_end_0; + } + ret = sample_common_svp_npu_create_task_buf(&g_svp_npu_task[task_idx]); + if (ret != TD_SUCCESS) { + sample_svp_trace_err("create task buf failed.\n"); + goto task_init_end_0; + } + if (is_share_work_buf == TD_FALSE) { + ret = sample_common_svp_npu_create_work_buf(&g_svp_npu_task[task_idx]); + } else { + /* if all tasks are on the same stream, work buf can be shared */ + ret = sample_common_svp_npu_share_work_buf(&g_svp_npu_shared_work_buf[shared_work_buf_idx], + &g_svp_npu_task[task_idx]); + } + if (ret != TD_SUCCESS) { + sample_svp_trace_err("create work buf failed.\n"); + goto task_init_end_0; + } + } + return TD_SUCCESS; + +task_init_end_0: + (td_void)sample_svp_npu_acl_deinit_task(task_num, shared_work_buf_idx); + return ret; +} + +// 辅助函数：根据class_id播放音频（如果类别发生变化） +static void play_audio_if_class_changed(void) +{ + int max_idx = -1; + double max_score = 0.5; + + // 检测到目标时，启动音频播放线程 + for (int i = 0; i < g_svp_npu_rect_info.num; i++) { + if (g_svp_npu_rect_info.rect[i].score >= max_score) { + if (max_idx == -1 || g_svp_npu_rect_info.rect[i].score > g_svp_npu_rect_info.rect[max_idx].score) { + max_idx = i; + } + } + } + // ...existing code... + if (max_idx != -1) { + int current_class_id = g_svp_npu_rect_info.rect[max_idx].class_id; + if (current_class_id != g_last_played_class_id) { + pthread_t audio_thread; + audio_play_param* param = malloc(sizeof(audio_play_param)); + if (param != NULL) { + param->class_id = current_class_id; + pthread_create(&audio_thread, NULL, audio_play_thread, param); + pthread_detach(audio_thread); // 不阻塞主流程 + g_last_played_class_id = current_class_id; // 只在播放时更新 + } + } + } +} + +static void get_detection_results(void) +{ + // 循环打印置信度和类别ID + for (int i = 0; i < g_svp_npu_rect_info.num; i++) { + const char* class_name = NULL; + switch (g_svp_npu_rect_info.rect[i].class_id) { + case 0: + class_name = "梨子"; + break; + case 1: + class_name = "西瓜"; + break; + case 2: /* 2: case of orange */ + class_name = "橘子"; + break; + case 3: /* 3: case of banana */ + class_name = "香蕉"; + break; + case 4: /* 4: case of apple */ + class_name = "苹果"; + break; + default: + class_name = "未知"; + break; + } + printf("score:%lf, class_id:%d, name:%s\r\n", + g_svp_npu_rect_info.rect[i].score, + g_svp_npu_rect_info.rect[i].class_id, + class_name); + } +} + +static td_s32 sample_svp_npu_acl_frame_proc(const ot_video_frame_info *ext_frame, + const ot_video_frame_info *base_frame, td_void *args) +{ + td_s32 ret; + td_void *virt_addr = TD_NULL; + sample_svp_npu_detection_info *detection_info = TD_NULL; + sample_svp_npu_thread_args *thread_args = (sample_svp_npu_thread_args *)args; + td_u32 size = (td_u32)(ext_frame->video_frame.height * ext_frame->video_frame.stride[0] * + SAMPLE_SVP_NPU_IMG_THREE_CHN / SAMPLE_SVP_NPU_DOUBLE); + + virt_addr = g_svp_npu_vb_virt_addr + + (ext_frame->video_frame.phys_addr[0] - g_svp_npu_vb_pool_info.pool_phy_addr); + ret = sample_common_svp_npu_update_input_data_buffer_info(virt_addr, size, + ext_frame->video_frame.stride[0], 0, &g_svp_npu_task[0]); + sample_svp_check_exps_return(ret != TD_SUCCESS, ret, SAMPLE_SVP_ERR_LEVEL_ERROR, "update data buffer failed!\n"); + + ret = sample_common_svp_npu_model_execute(&g_svp_npu_task[0]); + sample_svp_check_exps_return(ret != TD_SUCCESS, ret, SAMPLE_SVP_ERR_LEVEL_ERROR, "model execute failed!\n"); + + play_audio_if_class_changed(); + + // ...existing code... + if (thread_args->model_name == SAMPLE_SVP_NPU_RFCN || thread_args->model_name == SAMPLE_SVP_NPU_YOLO) { + detection_info = thread_args->detection_info; + if (detection_info->is_cpu_rpn == TD_FALSE) { + ret = sample_common_svp_npu_roi_to_rect(&g_svp_npu_task[0], detection_info, ext_frame, base_frame, + &g_svp_npu_rect_info); + sample_svp_check_exps_return(ret != TD_SUCCESS, ret, SAMPLE_SVP_ERR_LEVEL_ERROR, "roi to rect failed!\n"); + } else { + ret = sample_common_svp_npu_roi_to_rect_cpu(&g_svp_npu_task[0], detection_info, ext_frame, base_frame, + &g_svp_npu_rect_info); + sample_svp_check_exps_return(ret != TD_SUCCESS, ret, SAMPLE_SVP_ERR_LEVEL_ERROR, "roi to rect failed!\n"); + } + ret = sample_common_svp_vgs_fill_rect(base_frame, &g_svp_npu_rect_info, SAMPLE_SVP_NPU_RECT_COLOR); + sample_svp_check_exps_return(ret != TD_SUCCESS, ret, SAMPLE_SVP_ERR_LEVEL_ERROR, "vgs fill rect failed!\n"); + + // 在屏幕上打印信息检测到的水果信息 + int i; + for (i = 0; i < (g_svp_npu_rect_info.num); i++) { + sample_osd_draw_text(g_svp_npu_rect_info.rect[i].score, g_svp_npu_rect_info.rect[i].class_id, + g_svp_npu_rect_info.rect[i].point[0].x, g_svp_npu_rect_info.rect[i].point[0].y, + thread_args->mpp_osd_info); + } + if (g_svp_npu_rect_info.num == 0) { + thread_args->mpp_osd_info.chn_attr.is_show = TD_FALSE; + sample_osd_clean_text(thread_args->mpp_osd_info); + } + } + get_detection_results(); + + return ret; +} + +static td_s32 sample_svp_npu_acl_vb_map(td_u32 vb_pool_idx) +{ + td_s32 ret; + + if (g_svp_npu_vb_virt_addr == TD_NULL) { + ret = ss_mpi_vb_get_pool_info(g_svp_npu_media_cfg.vb_pool[vb_pool_idx], &g_svp_npu_vb_pool_info); + sample_svp_check_exps_return(ret != TD_SUCCESS, TD_FAILURE, SAMPLE_SVP_ERR_LEVEL_ERROR, + "get pool info failed!\n"); + g_svp_npu_vb_virt_addr = ss_mpi_sys_mmap(g_svp_npu_vb_pool_info.pool_phy_addr, + g_svp_npu_vb_pool_info.pool_size); + sample_svp_check_exps_return(g_svp_npu_vb_virt_addr == TD_NULL, TD_FAILURE, SAMPLE_SVP_ERR_LEVEL_ERROR, + "map vb pool failed!\n"); + } + return TD_SUCCESS; +} + +static td_void *sample_svp_npu_acl_vdec_to_vo(td_void *args) +{ + td_s32 ret; + ot_video_frame_info base_frame; + ot_video_frame_info ext_frame; + const td_s32 milli_sec = SAMPLE_SVP_NPU_MILLIC_SEC; + const ot_vo_layer vo_layer = 0; + const ot_vo_chn vo_chn = 0; + const td_s32 vpss_grp = 0; + td_s32 vpss_chn[] = { OT_VPSS_CHN0, OT_VPSS_CHN1 }; + td_u32 size; + td_u32 stride; + td_u8 *data = TD_NULL; + + (td_void)prctl(PR_SET_NAME, "svp_npu_vdec_to_vo", 0, 0, 0); + + ret = svp_acl_rt_set_device(g_svp_npu_dev_id); + sample_svp_check_exps_return(ret != TD_SUCCESS, TD_NULL, SAMPLE_SVP_ERR_LEVEL_ERROR, "open device failed!\n"); + + ret = sample_svp_npu_acl_vb_map(OT_VPSS_CHN1); + sample_svp_check_exps_goto(ret != TD_SUCCESS, fail_0, SAMPLE_SVP_ERR_LEVEL_ERROR, "map vb pool failed!\n"); + + ret = sample_common_svp_npu_get_input_data_buffer_info(&g_svp_npu_task[0], 0, &data, &size, &stride); + sample_svp_check_exps_goto(ret != TD_SUCCESS, fail_1, SAMPLE_SVP_ERR_LEVEL_ERROR, + "Error(%#x),get_input_data_buffer_info failed!\n", ret); + + while (g_svp_npu_thread_stop == TD_FALSE) { + ret = ss_mpi_vpss_get_chn_frame(vpss_grp, vpss_chn[1], &ext_frame, milli_sec); + sample_svp_check_exps_continue(ret != TD_SUCCESS, SAMPLE_SVP_ERR_LEVEL_ERROR, + "Error(%#x),ss_mpi_vpss_get_chn_frame failed, vpss_grp(%d), vpss_chn(%d)!\n", ret, vpss_grp, vpss_chn[1]); + + ret = ss_mpi_vpss_get_chn_frame(vpss_grp, vpss_chn[0], &base_frame, milli_sec); + sample_svp_check_exps_goto(ret != TD_SUCCESS, ext_release, SAMPLE_SVP_ERR_LEVEL_ERROR, + "Error(%#x),ss_mpi_vpss_get_chn_frame failed, VPSS_GRP(%d), VPSS_CHN(%d)!\n", ret, vpss_grp, vpss_chn[0]); + + ret = sample_svp_npu_acl_frame_proc(&ext_frame, &base_frame, args); + sample_svp_check_exps_goto(ret != TD_SUCCESS, base_release, SAMPLE_SVP_ERR_LEVEL_ERROR, + "Error(%#x),sample_svp_npu_acl_frame_proc failed!\n", ret); + + ret = sample_common_svp_venc_vo_send_stream(&g_svp_npu_media_cfg.svp_switch, 0, vo_layer, vo_chn, &base_frame); + sample_svp_check_exps_goto(ret != TD_SUCCESS, base_release, SAMPLE_SVP_ERR_LEVEL_ERROR, + "Error(%#x),sample_common_svp_venc_vo_send_stream failed!\n", ret); +base_release: + ret = ss_mpi_vpss_release_chn_frame(vpss_grp, vpss_chn[0], &base_frame); + sample_svp_check_exps_trace(ret != TD_SUCCESS, SAMPLE_SVP_ERR_LEVEL_ERROR, + "Error(%#x),release_frame failed,grp(%d) chn(%d)!\n", ret, vpss_grp, vpss_chn[0]); +ext_release: + ret = ss_mpi_vpss_release_chn_frame(vpss_grp, vpss_chn[1], &ext_frame); + sample_svp_check_exps_trace(ret != TD_SUCCESS, SAMPLE_SVP_ERR_LEVEL_ERROR, + "Error(%#x),release_frame failed,grp(%d) chn(%d)!\n", ret, vpss_grp, vpss_chn[1]); + } + ret = sample_common_svp_npu_update_input_data_buffer_info(data, size, stride, 0, &g_svp_npu_task[0]); + sample_svp_check_exps_trace(ret != TD_SUCCESS, SAMPLE_SVP_ERR_LEVEL_ERROR, "update buffer failed!\n"); +fail_1: + (td_void)ss_mpi_sys_munmap(g_svp_npu_vb_virt_addr, g_svp_npu_vb_pool_info.pool_size); +fail_0: + (td_void)svp_acl_rt_reset_device(g_svp_npu_dev_id); + return TD_NULL; +} + +static td_void sample_svp_npu_acl_pause(td_void) +{ + printf("---------------press Enter key to exit!---------------\n"); + if (g_svp_npu_terminate_signal == TD_TRUE) { + return; + } + (td_void)getchar(); + if (g_svp_npu_terminate_signal == TD_TRUE) { + return; + } +} + +static td_void sample_svp_npu_acl_set_task_info(td_u32 model_idx) +{ + g_svp_npu_task[0].cfg.max_batch_num = 1; + g_svp_npu_task[0].cfg.dynamic_batch_num = 1; + g_svp_npu_task[0].cfg.total_t = 0; + g_svp_npu_task[0].cfg.is_cached = TD_TRUE; + g_svp_npu_task[0].cfg.model_idx = model_idx; +} + +static td_void sample_svp_npu_acl_set_detection_info(sample_svp_npu_detection_info *detection_info, td_u32 index, + td_bool is_cpu_rpn) +{ + detection_info->num_name = g_svp_npu_yolo_roi_info[index].roi_num_name; + detection_info->roi_name = g_svp_npu_yolo_roi_info[index].roi_class_name; + detection_info->has_background = TD_FALSE; + detection_info->is_cpu_rpn = is_cpu_rpn; + detection_info->idx = index; + /* use PIC_BUTT to be a flag, get the input resolution form om */ + g_svp_npu_media_cfg.pic_type[1] = PIC_BUTT; + g_svp_npu_terminate_signal = TD_FALSE; +} + +static void sample_svp_audio_init() +{ + // 初始化音频编解码器 + ss_mpi_aenc_aac_init(); + ss_mpi_adec_aac_init(); +} + +static void sample_svp_audio_exit() +{ + // 反初始化音频 + ss_mpi_aenc_aac_deinit(); + ss_mpi_adec_aac_deinit(); +} + +/* 处理线程管理模块 */ +static td_s32 start_processing_thread(sample_svp_npu_thread_args *args) +{ + td_s32 ret; + + ret = sample_common_svp_npu_set_threshold(&g_svp_npu_yolo_threshold[0], + SAMPLE_SVP_NPU_YOLO_THRESHOLD_NUM, &g_svp_npu_task[0]); + + sample_svp_audio_init(); + g_svp_npu_thread_stop = TD_FALSE; + + ret = pthread_create(&g_svp_npu_thread, 0, sample_svp_npu_acl_vdec_to_vo, (td_void*)args); + + return TD_SUCCESS; +} + +/* function : show the sample of yolo. RPN processed by npu */ +td_void sample_svp_npu_acl_yolo_npu(void) +{ + td_s32 ret; + ot_mpp_chn chn; + td_s32 handle; + const td_u32 model_idx = 0; + const td_char *om_model_path = "./data/model/fruit_identify.om"; + sample_svp_npu_detection_info detection_info = {0}; + sample_svp_npu_thread_args args = {SAMPLE_SVP_NPU_YOLO, &detection_info}; + + sample_svp_npu_acl_set_detection_info(&detection_info, 0, TD_FALSE); + + if (g_svp_npu_terminate_signal == TD_FALSE) { + /* init acl */ + ret = sample_svp_npu_acl_init(TD_NULL, TD_FALSE); + sample_svp_check_exps_return_void(ret != TD_SUCCESS, SAMPLE_SVP_ERR_LEVEL_ERROR, "init failed!\n"); + + /* load model */ + ret = sample_common_svp_npu_load_model(om_model_path, model_idx, TD_FALSE); + sample_svp_check_exps_goto(ret != TD_SUCCESS, process_end0, SAMPLE_SVP_ERR_LEVEL_ERROR, + "load model failed!\n"); + + /* get input resolution */ + ret = sample_common_svp_npu_get_input_resolution(model_idx, 0, &g_svp_npu_media_cfg.pic_size[1]); + sample_svp_check_exps_goto(ret != TD_SUCCESS, process_end1, SAMPLE_SVP_ERR_LEVEL_ERROR, + "load model failed!\n"); + + /* start vdec vpss venc vo */ + ret = sample_common_svp_create_vb_start_vdec_vpss_vo(&g_svp_npu_vdec_cfg, &g_svp_npu_vdec_param, + &g_svp_npu_vdec_thread, &g_svp_npu_media_cfg, &g_svp_npu_vo_cfg); + sample_svp_check_exps_goto(ret != TD_SUCCESS, process_end1, SAMPLE_SVP_ERR_LEVEL_DEBUG, + "init media failed!\n"); + + // osd init + sample_osd_init(&args.mpp_osd_info); + + /* set cfg */ + sample_svp_npu_acl_set_task_info(model_idx); + + ret = sample_svp_npu_acl_init_task(1, TD_FALSE, 0); + sample_svp_check_exps_goto(ret != TD_SUCCESS, process_end2, SAMPLE_SVP_ERR_LEVEL_ERROR, "init task failed!\n"); + } + /* process */ + if (g_svp_npu_terminate_signal == TD_FALSE) { + // 处理线程启动 + ret = start_processing_thread(&args); + sample_svp_check_exps_goto(ret != 0, process_end3, SAMPLE_SVP_ERR_LEVEL_ERROR, "create thread failed!\n"); + + (td_void)sample_svp_npu_acl_pause(); + + g_svp_npu_thread_stop = TD_TRUE; + pthread_join(g_svp_npu_thread, TD_NULL); + } + +process_end3: + (td_void)sample_svp_npu_acl_deinit_task(1, 0); +process_end2: + sample_osd_exit(&args.mpp_osd_info); + (td_void)sample_common_svp_destroy_vb_stop_vdec_vpss_vo(&g_svp_npu_vdec_param, &g_svp_npu_vdec_thread, + &g_svp_npu_media_cfg, &g_svp_npu_vo_cfg); +process_end1: + (td_void)sample_common_svp_npu_unload_model(model_idx); +process_end0: + (td_void)sample_svp_npu_acl_deinit(); + (td_void)sample_svp_npu_acl_terminate(); + sample_svp_audio_exit(); +} + +int main(void) +{ +#ifdef CONFIG_USER_SPACE + SDK_init(); +#endif + sample_svp_npu_acl_yolo_npu(); +#ifdef CONFIG_USER_SPACE + SDK_exit(); +#endif + + return 0; +} diff --git a/vendor/zsks/demo/fruit_identify/pic/image-20221114150205685.png b/vendor/zsks/demo/fruit_identify/pic/image-20221114150205685.png new file mode 100755 index 0000000000000000000000000000000000000000..4dc4215c962c57ba0e31548fb98c29c789f149d9 Binary files /dev/null and b/vendor/zsks/demo/fruit_identify/pic/image-20221114150205685.png differ diff --git a/vendor/zsks/demo/fruit_identify/pic/image-20250210161601541.png b/vendor/zsks/demo/fruit_identify/pic/image-20250210161601541.png new file mode 100755 index 0000000000000000000000000000000000000000..80845ece8131933a4cd56ae3bd6119c38e0abd26 Binary files /dev/null and b/vendor/zsks/demo/fruit_identify/pic/image-20250210161601541.png differ diff --git a/vendor/zsks/demo/fruit_identify/pic/image-20250210170027454.png b/vendor/zsks/demo/fruit_identify/pic/image-20250210170027454.png new file mode 100755 index 0000000000000000000000000000000000000000..c06081fc2c963af4d1fa8af506804beedefe1b0b Binary files /dev/null and b/vendor/zsks/demo/fruit_identify/pic/image-20250210170027454.png differ diff --git a/vendor/zsks/demo/fruit_identify/pic/image-20250213112932380.png b/vendor/zsks/demo/fruit_identify/pic/image-20250213112932380.png new file mode 100755 index 0000000000000000000000000000000000000000..8b591b3437544859f4ec76b9ca03867cc401078c Binary files /dev/null and b/vendor/zsks/demo/fruit_identify/pic/image-20250213112932380.png differ diff --git a/vendor/zsks/demo/fruit_identify/pic/image-20250919150630870.png b/vendor/zsks/demo/fruit_identify/pic/image-20250919150630870.png new file mode 100755 index 0000000000000000000000000000000000000000..91f881e85b3292118336ce2c3d81968e4aee8c26 Binary files /dev/null and b/vendor/zsks/demo/fruit_identify/pic/image-20250919150630870.png differ diff --git a/vendor/zsks/demo/fruit_identify/pic/image-20250919151018659.png b/vendor/zsks/demo/fruit_identify/pic/image-20250919151018659.png new file mode 100755 index 0000000000000000000000000000000000000000..9b09507893b65d2142be49e6461daa31a81855c4 Binary files /dev/null and b/vendor/zsks/demo/fruit_identify/pic/image-20250919151018659.png differ diff --git a/vendor/zsks/demo/fruit_identify/pic/image-20250919151519310.png b/vendor/zsks/demo/fruit_identify/pic/image-20250919151519310.png new file mode 100755 index 0000000000000000000000000000000000000000..7f6d881a5d0e6d3f72b7a43d089b5c5a75d7f048 Binary files /dev/null and b/vendor/zsks/demo/fruit_identify/pic/image-20250919151519310.png differ diff --git a/vendor/zsks/demo/fruit_identify/pic/image-20250919173612332.png b/vendor/zsks/demo/fruit_identify/pic/image-20250919173612332.png new file mode 100755 index 0000000000000000000000000000000000000000..d6b77aee440df447f0a4abcade34334be11c9580 Binary files /dev/null and b/vendor/zsks/demo/fruit_identify/pic/image-20250919173612332.png differ diff --git a/vendor/zsks/demo/fruit_identify/pic/image-20250919180706355.png b/vendor/zsks/demo/fruit_identify/pic/image-20250919180706355.png new file mode 100755 index 0000000000000000000000000000000000000000..ba4971e35c5cdaaf64386cebfa6a896383e61c7d Binary files /dev/null and b/vendor/zsks/demo/fruit_identify/pic/image-20250919180706355.png differ diff --git a/vendor/zsks/demo/fruit_identify/pic/image-20251023103457181.png b/vendor/zsks/demo/fruit_identify/pic/image-20251023103457181.png new file mode 100755 index 0000000000000000000000000000000000000000..88b9ef12b22044790ffea9b20b9829e7b88d1792 Binary files /dev/null and b/vendor/zsks/demo/fruit_identify/pic/image-20251023103457181.png differ diff --git a/vendor/zsks/demo/fruit_identify/pic/image-20251023103623167.png b/vendor/zsks/demo/fruit_identify/pic/image-20251023103623167.png new file mode 100755 index 0000000000000000000000000000000000000000..f19b0fc1e3a19b42eebcd8202cdf8fd1d408c6f1 Binary files /dev/null and b/vendor/zsks/demo/fruit_identify/pic/image-20251023103623167.png differ diff --git a/vendor/zsks/demo/fruit_identify/pic/image-20251023103700404.png b/vendor/zsks/demo/fruit_identify/pic/image-20251023103700404.png new file mode 100755 index 0000000000000000000000000000000000000000..1d46490b430a11ceab5efbcae4e90690b6850fd5 Binary files /dev/null and b/vendor/zsks/demo/fruit_identify/pic/image-20251023103700404.png differ diff --git a/vendor/zsks/demo/fruit_identify/pic/image-20251023103850695.png b/vendor/zsks/demo/fruit_identify/pic/image-20251023103850695.png new file mode 100755 index 0000000000000000000000000000000000000000..f7266d4498a5b248efe056d3da9238bc0616dd27 Binary files /dev/null and b/vendor/zsks/demo/fruit_identify/pic/image-20251023103850695.png differ diff --git a/vendor/zsks/demo/fruit_identify/pic/image-20251023103944837.png b/vendor/zsks/demo/fruit_identify/pic/image-20251023103944837.png new file mode 100755 index 0000000000000000000000000000000000000000..f7d3b45140ff5f6eef00ea2e80bf5ce366a20793 Binary files /dev/null and b/vendor/zsks/demo/fruit_identify/pic/image-20251023103944837.png differ diff --git a/vendor/zsks/demo/fruit_identify/pic/image-20251023110809454.png b/vendor/zsks/demo/fruit_identify/pic/image-20251023110809454.png new file mode 100755 index 0000000000000000000000000000000000000000..4a0801ed60ba0dd2747ea0c163d402d046a3bcc2 Binary files /dev/null and b/vendor/zsks/demo/fruit_identify/pic/image-20251023110809454.png differ diff --git a/vendor/zsks/demo/fruit_identify/sample_audio.c b/vendor/zsks/demo/fruit_identify/sample_audio.c new file mode 100755 index 0000000000000000000000000000000000000000..22d334818f0b025fb71e80c4ec8196f06ea0bc8e --- /dev/null +++ b/vendor/zsks/demo/fruit_identify/sample_audio.c @@ -0,0 +1,242 @@ +/* + * Copyright (c) 2025 HiSilicon (Shanghai) Technologies Co., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "sample_comm.h" +#include "audio_aac_adp.h" +#include "audio_dl_adp.h" +#include "ot_resample.h" + +#include "sample_audio.h" + +static ot_payload_type g_payload_type = OT_PT_AAC; +static td_bool g_aio_resample = TD_FALSE; +static ot_audio_sample_rate g_in_sample_rate = OT_AUDIO_SAMPLE_RATE_BUTT; +static ot_audio_sample_rate g_out_sample_rate = OT_AUDIO_SAMPLE_RATE_BUTT; + +#define sample_dbg(ret) \ + do { \ + printf("ret = %#x, fuc:%s, line:%d\n", ret, __FUNCTION__, __LINE__); \ + } while (0) + +#define sample_res_check_null_ptr(ptr) \ + do { \ + if ((td_u8*)(ptr) == TD_NULL) { \ + printf("ptr is TD_NULL,fuc:%s,line:%d\n", __FUNCTION__, __LINE__); \ + return TD_FAILURE; \ + } \ + } while (0) + +/* function : PT number to string */ +static char *sample_audio_pt2_str(ot_payload_type type) +{ + if (type == OT_PT_G711A) { + return "g711a"; + } else if (type == OT_PT_G711U) { + return "g711u"; + } else if (type == OT_PT_ADPCMA) { + return "adpcm"; + } else if (type == OT_PT_G726) { + return "g726"; + } else if (type == OT_PT_LPCM) { + return "pcm"; + } else if (type == OT_PT_AAC) { + return "aac"; + } else { + return "data"; + } +} + +/* function : open adec file */ +static FILE *sample_audio_open_adec_file(ot_adec_chn ad_chn, ot_payload_type type, + const td_char *file_name1, const td_char *file_name2) +{ + FILE *fd = TD_NULL; + td_char asz_file_name[FILE_NAME_LEN] = {0}; + td_s32 ret; + td_char path[PATH_MAX] = {0}; + + /* create file for save stream */ +#ifdef __LITEOS__ + ret = snprintf_s(asz_file_name, FILE_NAME_LEN, FILE_NAME_LEN - 1, + "/sharefs/audio_chn%d.%s", ad_chn, sample_audio_pt2_str(type)); +#else + ret = snprintf_s(asz_file_name, FILE_NAME_LEN, FILE_NAME_LEN - 1, + "%s/%s/audio_chn%d.%s", file_name1, file_name2, ad_chn, sample_audio_pt2_str(type)); +#endif + if (ret < 0) { + printf("[func]:%s [line]:%d [info]:%s\n", __FUNCTION__, __LINE__, "get adec file name failed"); + return TD_NULL; + } + + if (asz_file_name[0] == '\0') { + printf("[func]:%s [line]:%d [info]:%s\n", __FUNCTION__, __LINE__, "adec file name is NULL"); + return TD_NULL; + } + + if (strlen(asz_file_name) > (FILE_NAME_LEN - 1)) { + printf("[func]:%s [line]:%d [info]:%s\n", __FUNCTION__, __LINE__, "adec file name extra long"); + return TD_NULL; + } + + if (realpath(asz_file_name, path) == TD_NULL) { + printf("[func]:%s [line]:%d [info]:%s\n", __FUNCTION__, __LINE__, "adec file name realpath fail"); + return TD_NULL; + } + + fd = fopen(path, "rb"); + if (fd == NULL) { + printf("%s: open file %s failed\n", __FUNCTION__, asz_file_name); + return NULL; + } + printf("open stream file:\"%s\" for adec ok\n", asz_file_name); + return fd; +} + +static td_void sample_audio_adec_ao_init_param(ot_aio_attr *aio_attr, ot_audio_dev *ao_dev) +{ + aio_attr->sample_rate = OT_AUDIO_SAMPLE_RATE_48000; + aio_attr->bit_width = OT_AUDIO_BIT_WIDTH_16; + aio_attr->work_mode = OT_AIO_MODE_I2S_MASTER; + aio_attr->snd_mode = OT_AUDIO_SOUND_MODE_STEREO; + aio_attr->expand_flag = 0; + aio_attr->frame_num = 30; /* 30:frame num */ + if (g_payload_type == OT_PT_AAC) { + aio_attr->point_num_per_frame = AACLC_SAMPLES_PER_FRAME; + } else { + aio_attr->point_num_per_frame = SAMPLE_AUDIO_POINT_NUM_PER_FRAME; + } + aio_attr->chn_cnt = 2; /* 2:chn num */ +#ifdef OT_ACODEC_TYPE_INNER + *ao_dev = SAMPLE_AUDIO_INNER_AO_DEV; + aio_attr->clk_share = 1; + aio_attr->i2s_type = OT_AIO_I2STYPE_INNERCODEC; +#else + *ao_dev = SAMPLE_AUDIO_EXTERN_AO_DEV; + aio_attr->clk_share = 1; + aio_attr->i2s_type = OT_AIO_I2STYPE_EXTERN; +#endif + g_aio_resample = TD_FALSE; + g_in_sample_rate = OT_AUDIO_SAMPLE_RATE_BUTT; + g_out_sample_rate = OT_AUDIO_SAMPLE_RATE_BUTT; +} + +td_void sample_audio_adec_ao_inner(ot_audio_dev ao_dev, ot_ao_chn ao_chn, ot_adec_chn ad_chn, + const td_char *file_name1, const td_char *file_name2) +{ + td_s32 ret; + FILE *fd = NULL; + + ret = sample_comm_audio_ao_bind_adec(ao_dev, ao_chn, ad_chn); + if (ret != TD_SUCCESS) { + sample_dbg(ret); + return; + } + + fd = sample_audio_open_adec_file(ad_chn, g_payload_type, file_name1, file_name2); + if (fd == TD_NULL) { + sample_dbg(TD_FAILURE); + goto adec_ao_err0; + } + + ret = sample_comm_audio_creat_trd_file_adec(ad_chn, fd); + if (ret != TD_SUCCESS) { + sample_dbg(ret); + fclose(fd); + fd = TD_NULL; + goto adec_ao_err0; + } + + printf("bind adec:%d to ao(%d,%d) ok \n", ad_chn, ao_dev, ao_chn); + + sleep(2); /* 2: sleep 2 seconds */ + + ret = sample_comm_audio_destory_trd_file_adec(ad_chn); + if (ret != TD_SUCCESS) { + sample_dbg(ret); + } + +adec_ao_err0: + ret = sample_comm_audio_ao_unbind_adec(ao_dev, ao_chn, ad_chn); + if (ret != TD_SUCCESS) { + sample_dbg(ret); + } + + return; +} + +/* function : file -> adec -> ao */ +td_s32 sample_audio_adec_ao(const td_char *file_name1, const td_char *file_name2) +{ + td_s32 ret; + td_u32 ao_chn_cnt; + td_u32 adec_chn_cnt; + ot_audio_dev ao_dev; + ot_aio_attr aio_attr; + const ot_ao_chn ao_chn = 0; + const ot_adec_chn ad_chn = 0; + + sample_audio_adec_ao_init_param(&aio_attr, &ao_dev); + + adec_chn_cnt = aio_attr.chn_cnt >> ((td_u32)aio_attr.snd_mode); + ret = sample_comm_audio_start_adec(adec_chn_cnt, g_payload_type); + if (ret != TD_SUCCESS) { + sample_dbg(ret); + goto adec_ao_err3; + } + + ao_chn_cnt = aio_attr.chn_cnt; + ret = sample_comm_audio_start_ao(ao_dev, ao_chn_cnt, &aio_attr, g_in_sample_rate, g_aio_resample); + if (ret != TD_SUCCESS) { + sample_dbg(ret); + goto adec_ao_err2; + } + + ret = sample_comm_audio_cfg_acodec(&aio_attr); + if (ret != TD_SUCCESS) { + sample_dbg(ret); + goto adec_ao_err1; + } + + sample_audio_adec_ao_inner(ao_dev, ao_chn, ad_chn, file_name1, file_name2); + +adec_ao_err1: + ret = sample_comm_audio_stop_ao(ao_dev, ao_chn_cnt, g_aio_resample); + if (ret != TD_SUCCESS) { + sample_dbg(ret); + } + +adec_ao_err2: + ret = sample_comm_audio_stop_adec(ad_chn); + if (ret != TD_SUCCESS) { + sample_dbg(ret); + } + +adec_ao_err3: + return ret; +} \ No newline at end of file diff --git a/vendor/zsks/demo/fruit_identify/sample_audio.h b/vendor/zsks/demo/fruit_identify/sample_audio.h new file mode 100755 index 0000000000000000000000000000000000000000..ebb5b5babcfbd3795ee222442abad02c792fec35 --- /dev/null +++ b/vendor/zsks/demo/fruit_identify/sample_audio.h @@ -0,0 +1,25 @@ +/* + * Copyright (c) 2025 HiSilicon (Shanghai) Technologies Co., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef __SAMPLE_AUDIO_ADEC_AO_H__ +#define __SAMPLE_AUDIO_ADEC_AO_H__ + +#include "sample_comm.h" + +td_s32 sample_audio_adec_ao(const td_char *file_name1, const td_char *file_name2); + + +#endif /* __SAMPLE_AUDIO_ADEC_AO_H__ */ \ No newline at end of file diff --git a/vendor/zsks/demo/mk.param b/vendor/zsks/demo/mk.param new file mode 100755 index 0000000000000000000000000000000000000000..591e8d1c38f8c037706dfc3cd5b598dce7bd49ea --- /dev/null +++ b/vendor/zsks/demo/mk.param @@ -0,0 +1,60 @@ +# 各模块makefile均应在前面include此文件 +export ROOT_DIR := /home/openharmony/pegasus +export SDK_ROOT := $(ROOT_DIR)/platform/ss928v100_clang +export CUST_ROOT := $(ROOT_DIR)/vendor/zsks/ +export THIRD_PARTY_DIR := $(ROOT_DIR)/vendor/opensource +export MPP_DIR := $(SDK_ROOT)/smp/a55_linux/mpp +export OUTPUT_DIR := $(CUST_ROOT)/demo/output/ +export OBJS_DIR := $(CUST_ROOT)/demo/obj/ + +# 交叉编译器 +CLANG_PATH := $(ROOT_DIR)/os/OpenHarmony/ohos/prebuilts/clang/ohos/linux-x86_64/llvm +export SYSROOT_PATH := $(ROOT_DIR)/os/OpenHarmony/ohos/out/hispark_ss928v100/ipcamera_hispark_ss928v100_linux/sysroot +COMPILER_PATH := $(CLANG_PATH)/bin/ +export CC:=$(COMPILER_PATH)/clang --target=aarch64-linux-ohos -Wno-unused-command-line-argument --sysroot $(SYSROOT_PATH) -isystem $(SYSROOT_PATH)/usr/include +export CXXCPP := $(COMPILER_PATH)/clang++ --target=aarch64-linux-ohos --sysroot=$(CLANG_PATH)/sysroot -nostdinc -isystem $(CLANG_PATH)/include/c++/v1 -isystem $(CLANG_PATH)/lib/clang/15.0.4/include -isystem $(SYSROOT_PATH)/usr/include/aarch64-linux-ohos -D_LIBCPP_TYPEINFO_COMPARISON_IMPLEMENTATION=2 -D__ARM_NEON=1 -D__ARM_NEON__=1 -mfpu=neon -resource-dir=$(CLANG_PATH)/lib/clang/15.0.4 +CC += -fuse-ld=lld -rtlib=compiler-rt + +# 编译所依赖的库目录 +export SDK_LINUX_OUT_LIB := $(MPP_DIR)/out/lib +export SDK_LINUX_OUT_SVP_NPU_LIB := $(SDK_LINUX_OUT_LIB)/svp_npu + +# 编译所依赖的头文件和源文件目录 +export SDK_LINUX_INCLUDE_DIR := $(MPP_DIR)/out/include +export SDK_SAMPLE_COMMON_DIR := $(MPP_DIR)/sample/common +export SVP_COMMON_DIR := $(MPP_DIR)/sample/svp/common + +# 第三方软件的库和头文件 +export OPENCV_LIB := $(THIRD_PARTY_DIR)/opencv/lib +export OPENCV_INCLUDE := $(THIRD_PARTY_DIR)/opencv/include/opencv4 + +# target输出的存储目录 +export TARGET_ROOT := ./ +export TARGET_DIR := $(TARGET_ROOT)/out/ + +export AUDIO_SAMPLE := $(MPP_DIR)/sample/audio/ +# cust 项目子模块目录 +export FACE_DETECTION_SAMPLE_DIR := $(CUST_ROOT)/demo/face_detection +export FRUIT_IDENTIFY_SAMPLE_DIR := $(CUST_ROOT)/demo/fruit_identify +export HNR_AUTO_SAMPLE_DIR := $(CUST_ROOT)/demo/hnr_auto +export KCF_TRACK_SAMPLE_DIR := $(CUST_ROOT)/demo/sample_kcf_track +export OPENCV_DNN_SAMPLE_DIR := $(CUST_ROOT)/demo/opencv_dnn + +################ select audio codec type for your sample ################ +#external acodec +#ACODEC_TYPE ?= ACODEC_TYPE_ES8388 +ACODEC_TYPE ?= ACODEC_TYPE_INNER +INNER_HDMI ?= y + +ifeq ($(ACODEC_TYPE), ACODEC_TYPE_ES8388) + CFLAGS += -DOT_ACODEC_TYPE_ES8388 +endif + +ifeq ($(INNER_HDMI), y) + CFLAGS += -DOT_ACODEC_TYPE_HDMI +endif + +ifeq ($(ACODEC_TYPE), ACODEC_TYPE_INNER) + CFLAGS += -DOT_ACODEC_TYPE_INNER +endif + diff --git a/vendor/zsks/demo/opencv_dnn/Makefile b/vendor/zsks/demo/opencv_dnn/Makefile new file mode 100755 index 0000000000000000000000000000000000000000..14ead07993fed2e673e88a02d595ed919dc0d5d9 --- /dev/null +++ b/vendor/zsks/demo/opencv_dnn/Makefile @@ -0,0 +1,54 @@ +# 编译所需的公共文件 +include ../mk.param + +# 编译后生成的可执行文件名 +TARGET := main + +# 编译此工程需要包含的头文件 +CFLAGS += -I$(SDK_LINUX_INCLUDE_DIR)/ +CFLAGS += -I$(SDK_SAMPLE_COMMON_DIR)/ +CFLAGS += -Wno-psabi -I$(OPENCV_INCLUDE)/ +CFLAGS += -I$(OPENCV_DNN_SAMPLE_DIR)/ + + +# 编译此工程需要依赖的.c文件 +SMP_SRCS := $(SDK_SAMPLE_COMMON_DIR)/sample_comm_sys.c +SMP_SRCS += $(SDK_SAMPLE_COMMON_DIR)/sample_ipc.c +SMP_SRCS += $(SDK_SAMPLE_COMMON_DIR)/sample_comm_isp.c +SMP_SRCS += $(SDK_SAMPLE_COMMON_DIR)/sample_comm_vi.c +SMP_SRCS += $(SDK_SAMPLE_COMMON_DIR)/sample_comm_vo.c +SMP_SRCS += $(SDK_SAMPLE_COMMON_DIR)/sample_comm_mipi_tx.c +SMP_SRCS += $(SDK_SAMPLE_COMMON_DIR)/sample_comm_vpss.c +SMP_SRCS += $(SDK_SAMPLE_COMMON_DIR)/loadbmp.c +SMP_SRCS += $(SDK_SAMPLE_COMMON_DIR)/sample_comm_vdec.c +SMP_SRCS += $(SDK_SAMPLE_COMMON_DIR)/sample_comm_venc.c +SMP_SRCS += $(SDK_SAMPLE_COMMON_DIR)/sdk_exit.c +SMP_SRCS += $(SDK_SAMPLE_COMMON_DIR)/sdk_init.c +SMP_SRCS += $(OPENCV_DNN_SAMPLE_DIR)/host_uvc.c +SMP_SRCS += $(OPENCV_DNN_SAMPLE_DIR)/media_vdec.c + +# 编译此工程需要依赖的.cpp文件 +CPP_SRCS += $(OPENCV_DNN_SAMPLE_DIR)/main.cpp + +# 编译此工程需要包含的库和宏 +CFLAGS += -O3 -g -Wall -DSAMPLE_HNR_SUPPORT +CFLAGS += -fstack-protector-strong +MPI_LIBS += -L$(OPENCV_LIB)/ +MPI_LIBS += -lopencv_videoio -lopencv_calib3d -lopencv_dnn -lopencv_core -lopencv_video +MPI_LIBS += -lopencv_flann -lopencv_features2d -lopencv_highgui -lopencv_imgcodecs +MPI_LIBS += -lopencv_imgproc -lopencv_ml -lopencv_objdetect -lopencv_photo -lopencv_stitching + +# 根据实际的类型设置，可以用set_sensor脚本设置 +SENSOR0_TYPE = -DSENSOR0_TYPE=HY_S0603_MIPI_8M_30FPS_12BIT +SENSOR1_TYPE = -DSENSOR1_TYPE=HY_S0603_MIPI_8M_30FPS_12BIT +SENSOR2_TYPE = -DSENSOR1_TYPE=HY_S0603_MIPI_8M_30FPS_12BIT +SENSOR3_TYPE = -DSENSOR1_TYPE=HY_S0603_MIPI_8M_30FPS_12BIT + +CFLAGS += $(SENSOR0_TYPE) +CFLAGS += $(SENSOR1_TYPE) +CFLAGS += $(SENSOR2_TYPE) +CFLAGS += $(SENSOR3_TYPE) +# 编译工程所需的mk文件 + +include ../sdk_linux.mk + diff --git a/vendor/zsks/demo/opencv_dnn/README.md b/vendor/zsks/demo/opencv_dnn/README.md new file mode 100755 index 0000000000000000000000000000000000000000..d3a89cdd625d6fa0f9aaf9d975ff25d70d9574f3 --- /dev/null +++ b/vendor/zsks/demo/opencv_dnn/README.md @@ -0,0 +1,141 @@ +## 2.1、opencv_dnn操作指导 + +### 2.1.1、opencv_dnn程序简介 + +* opencv_dnn sample基于SS928V100平台开发，以EulerPi套件为例，opencv_dnn sample是通过USB camera采集到的图片送到人脸检测模型进行推理，将得到的结果通过HDMI，显示在外接显示屏上面。 +* opencv_dnn案例，模型推理是在CPU上实现了，没有调用svp_npu的接口，都是调用opencv的接口来实现。 + +### 2.1.2、目录 + +```shell +pegasus/vendor/cust/demo/opencv_dnn +|── media_vdec.c # 主要是将推理的结果在HDMI外接显示器上显示 +|── media_vdec.h +|── Makefile # 编译脚本 +|── main.cpp # opencv_dnn sample业务代码 +|── host_uvc.c #参考海思SDK的host_uvc代码 +└── host_uvc.h +``` + +![image-20250919181207847](pic/image-20250919181207847.png) + +* 请在opencv_dnn目录下创建一个onnx的文件夹，然后访问链接下载[onnx模型](https://github.com/opencv/opencv_zoo/blob/main/models/face_detection_yunet/face_detection_yunet_2023mar.onnx)，并放入onnx目录下 + +![image-20251015164912907](pic/image-20251015164912907.png) + +![image-20251015165024778](pic/image-20251015165024778.png) + +* 在编译之前请确保你已经按照[opencv的移植文档](../../docs/opencv/README.md)，对opencv进行交叉编译并将生成的库文件复制到opensource目录下了。具体目录结构如下 + +![image-20251023111425708](pic/image-20251023111425708.png) + +### 2.1.3、编译 + +* **注意：在编译cust的demo之前，请确保你已经按照[patch介绍中的步骤](../../README.md#4patch介绍)把补丁打入海思SDK的sample目录下了**。 + +* 步骤1：先根据自己选择的操作系统，进入到对应的Pegasus所在目录下。 + +* 步骤2：使用Makefile的方式进行单编 + +* 在Ubuntu的命令行终端，分步执行下面的命令，单编 opencv_dnn sample + + ``` + cd pegasus/vendor/cust/demo/opencv_dnn + + make clean && make + ``` + + * 在opencv_dnn/out目录下，生成一个名为main的可执行文件，如下图所示： + + ![image-20251023111135567](pic/image-20251023111135567.png) + +### 2.1.4、拷贝可执行程序和依赖文件至开发板的mnt目录下 + +**方式一：使用SD卡进行资料文件的拷贝** + +* 首先需要自己准备一张Micro sd卡(16G 左右即可)，还得有一个Micro sd卡的读卡器。 + +image-20221114150205685 + +* 步骤1：将编译后生成的可执行文件、onnx模型文件、opencv的lib文件都拷贝到SD卡中。 + +![image-20251023112130092](pic/image-20251023112130092.png) + +* 步骤2：可执行文件拷贝成功后，将内存卡插入开发板的SD卡槽中，可通过挂载的方式挂载到板端，可选择SD卡 mount指令进行挂载。 + +image-20250210161601541 + +* 在开发板的终端，执行下面的命令进行SD卡的挂载 + * 如果挂载失败，可以参考[这个issue尝试解决](https://gitee.com/HiSpark/HiSpark_NICU2022/issues/I54932?from=project-issue) + + +```shell +mount -t vfat /dev/mmcblk1p1 /mnt +# 其中/dev/mmcblk1p1需要根据实际块设备号修改 +``` + +* 挂载成功后，如下图所示： + +![image-20251023112228717](pic/image-20251023112228717.png) + +**方式二：使用NFS挂载的方式进行资料文件的拷贝** + +* 首先需要自己准备一根网线 +* 步骤1：参考[博客链接](https://blog.csdn.net/Wu_GuiMing/article/details/115872995?spm=1001.2014.3001.5501)中的内容，进行nfs的环境搭建 +* 步骤2：将编译后生成的可执行文件、onnx模型文件、opencv库（在pegasus/vendor/opensource/opencv/lib目录下）都拷贝到Windows的nfs共享路径下 + +![image-20251023112328619](pic/image-20251023112328619.png) + +* 步骤3：在开发板的终端执行下面的命令，将Windows的nfs共享路径挂载至开发板的mnt目录下 + * 注意：这里IP地址请根据你开发板和电脑主机的实际IP地址进行填写 + + +``` +ifconfig eth0 192.168.100.100 +mount -o nolock,addr=192.168.100.10 -t nfs 192.168.100.10:/d/nfs /mnt +``` + +![image-20251023112352692](pic/image-20251023112352692.png) + +### 2.1.5、硬件连接 + +* 准备一个外接显示器和HDMI数据线，将HDMI的一头接在开发板的HDMI输出口，将HDMI的另外一头接在外接显示器的HDMI输入口。 + +![image-20250213112932380](pic/image-20250213112932380.png) + +* 将USB 摄像头接在EulerPi开发板的USB接口上。 + +image-20250919150630870 + +### 2.1.6、功能验证 + +* 在开发板的终端执行下面的命令，运行可执行文件 + +```c +# 把opencv的库添加到环境变量中 +export LD_LIBRARY_PATH=/mnt/lib:$LD_LIBRARY_PATH + +cd /mnt + +chmod +x main + +./main /dev/media0 -fMJPEG -s1920x1080 -Ftest.mjpg +``` + +![image-20251023112451423](pic/image-20251023112451423.png) + +* 此时，在HDMI的外接显示屏上即可出现实时码流，如下图所示： + +image-20250210170027454 + +* 如果您看到的现象和下图现象不一致，可以确认一下USB涉嫌头是否插入到开发板的USB口，并且在开发板的/dev目录下能够看到video0 和 video1两个设备节点。如果没有这两个设备节点，请确保镜像烧录正常。 + +![image-20250919151018659](pic/image-20250919151018659.png) + +* 正常情况下，我们会在外接显示屏上看到有人脸的区域被框出来，且在框框的左上角显示置信度。 + +![image-20250919183224310](pic/image-20250919183224310.png) + +* Ctrl + C，然后回车即可关闭程序 + +![image-20250919151519310](pic/image-20250919151519310.png) \ No newline at end of file diff --git a/vendor/zsks/demo/opencv_dnn/host_uvc.c b/vendor/zsks/demo/opencv_dnn/host_uvc.c new file mode 100755 index 0000000000000000000000000000000000000000..ee4620bdfdfcf0dd790ddd675b4f008e15a46ef2 --- /dev/null +++ b/vendor/zsks/demo/opencv_dnn/host_uvc.c @@ -0,0 +1,202 @@ +/* + * Copyright (c) 2025 HiSilicon (Shanghai) Technologies Co., Ltd. + * Copyright (c) 2025 Zhongshan Kuangshi Microelectronics Technology Co., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Modifications Author: yaohongtao (yht@cust.edu.cn) + */ +#include +#include +#include +#include +#include +#include +#include +#include "host_uvc.h" + +#ifdef __cplusplus +#if __cplusplus +extern "C" { +#endif +#endif + +#define array_size(a) (sizeof(a) / sizeof((a)[0])) + +format_info g_pixel_formats[] = { + { "YUYV", V4L2_PIX_FMT_YUYV, 1 }, + { "MJPEG", V4L2_PIX_FMT_MJPEG, 1 }, + { "H264", V4L2_PIX_FMT_H264, 1 }, + { "H265", V4L2_PIX_FMT_H265, 1 }, + { "NV12", V4L2_PIX_FMT_NV12, 1 }, + { "NV21", V4L2_PIX_FMT_NV21, 1 }, +}; + +static struct option g_opts[] = { + {"file", 2, 0, 'F'}, /* 2 is a number */ + {"format", 1, 0, 'f'}, + {"help", 0, 0, 'h'}, + {"size", 1, 0, 's'}, + {0, 0, 0, 0} +}; + +static const format_info *sample_uvc_v4l2_format_by_fourcc(td_u32 fourcc) +{ + td_u32 i; + + for (i = 0; i < array_size(g_pixel_formats); ++i) { + if (g_pixel_formats[i].fourcc == fourcc) { + return &g_pixel_formats[i]; + } + } + return TD_NULL; +} + +static td_void sample_uvc_usage(const td_char *argv0) +{ + printf("sample_uvc_usage: %s device [options]\n", argv0); + printf("supported options:\n"); + printf("-f, --format format set the video format\n"); + printf("-F, --file[=name] write file\n"); + printf("-h, --help show help info\n"); + printf("-s, --size WxH set the frame size (eg. 1920x1080)\n\n"); + printf("inquire USB device format: ./sample_uvc --enum-formats\n\n"); + printf("example of setting USB device format:\n"); + printf(" ./opencv_dnn /dev/media0 -fH264 -s1920x1080 -Ftest.h264\n"); + printf(" ./opencv_dnn /dev/media0 -fH265 -s1920x1080 -Ftest.h265\n"); + printf(" ./opencv_dnn /dev/media0 -fMJPEG -s1920x1080 -Ftest.mjpg\n"); + printf(" ./opencv_dnn /dev/media0 -fYUYV -s1920x1080 -Ftest.yuv\n"); + printf(" ./opencv_dnn /dev/media0 -fNV21 -s640x360 -Ftest.yuv\n\n"); + printf("note: set macro MEDIA_WORK to 0 to write file on disk.\n\n"); +} + +static const format_info *sample_uvc_v4l2_format_by_name(const td_char *name) +{ + td_u32 i; + for (i = 0; i < array_size(g_pixel_formats); ++i) { + if (!strcmp(g_pixel_formats[i].name, name)) { + return &g_pixel_formats[i]; + } + } + return TD_NULL; +} + +static td_void sample_uvc_list_formats(td_void) +{ + td_u32 i; + for (i = 0; i < array_size(g_pixel_formats); i++) { + sample_print("%s (\"%c%c%c%c\", %u planes)\n", g_pixel_formats[i].name, g_pixel_formats[i].fourcc & 0xff, + (g_pixel_formats[i].fourcc >> 8) & 0xff, (g_pixel_formats[i].fourcc >> 16) & 0xff, /* 8,16:shift */ + (g_pixel_formats[i].fourcc >> 24) & 0xff, g_pixel_formats[i].n_planes); /* 24:shift */ + } +} + +static td_s32 sample_uvc_get_opt(td_s32 argc, td_char *argv[], uvc_ctrl_info *ctrl_info) +{ + td_s32 c; + const format_info *info; + td_char *endptr; + + while ((c = getopt_long(argc, argv, "B:c::Cd:f:F::hi:Iln:p::q:r:R::s:t:uw:x:", g_opts, TD_NULL)) != -1) { + switch (c) { + case 'f': + if (!strcmp("help", optarg)) { + sample_uvc_list_formats(); + return TD_FAILURE; + } + ctrl_info->do_set_format = 1; + info = sample_uvc_v4l2_format_by_name(optarg); + if (info == TD_NULL) { + sample_print("Unsupported video format '%s'\n", optarg); + return TD_FAILURE; + } + ctrl_info->pixelformat = info->fourcc; + break; + case 'F': + ctrl_info->do_file = 1; + if (optarg) { + ctrl_info->pattern = optarg; + } + break; + case 'h': + sample_uvc_usage(argv[0]); + return TD_FAILURE; + case 's': + ctrl_info->do_capture = 1; + ctrl_info->do_set_format = 1; + ctrl_info->width = strtol(optarg, &endptr, 10); /* 10:base */ + if (*endptr != 'x' || endptr == optarg) { + sample_print("Invalid size '%s'\n", optarg); + return TD_FAILURE; + } + ctrl_info->height = strtol(endptr + 1, &endptr, 10); /* 10:base */ + if (*endptr != 0) { + sample_print("Invalid size '%s'\n", optarg); + return TD_FAILURE; + } + break; + default: + sample_print("Run %s -h for help.\n", argv[0]); + return TD_FAILURE; + } + } + return TD_SUCCESS; +} + +static td_void sample_uvc_ctrl_info_init(uvc_ctrl_info *ctrl_info) +{ + ctrl_info->pattern = "frame-#.bin"; + ctrl_info->width = 640; /* 640:width */ + ctrl_info->height = 480; /* 480:height */ + ctrl_info->pixelformat = V4L2_PIX_FMT_YUYV; +} + +const td_char *sample_uvc_v4l2_format_name(td_u32 fourcc) +{ + const format_info *format; + static td_char format_name[5]; /* 5: array len */ + td_u32 i; + format = sample_uvc_v4l2_format_by_fourcc(fourcc); + if (format) { + return format->name; + } + + for (i = 0; i < 4; ++i) { /* 4: format */ + format_name[i] = fourcc & 0xff; + fourcc >>= 8; /* 8: shift */ + } + + format_name[4] = '\0'; /* 4: end */ + return format_name; +} + +td_s32 sample_host_uvc(td_s32 argc, td_char *argv[], uvc_ctrl_info *ctrl_info) +{ + int ret; + sample_uvc_ctrl_info_init(ctrl_info); + ret = sample_uvc_get_opt(argc, argv, ctrl_info); + if (ret != TD_SUCCESS) { + return TD_FAILURE; + } + if (optind >= argc) { + sample_uvc_usage(argv[0]); + return TD_FAILURE; + } + return TD_SUCCESS ; +} + +#ifdef __cplusplus +#if __cplusplus +} +#endif +#endif /* End of #ifdef __cplusplus */ diff --git a/vendor/zsks/demo/opencv_dnn/host_uvc.h b/vendor/zsks/demo/opencv_dnn/host_uvc.h new file mode 100755 index 0000000000000000000000000000000000000000..2260d03101ecb6a09ad12f938aca663b297f8059 --- /dev/null +++ b/vendor/zsks/demo/opencv_dnn/host_uvc.h @@ -0,0 +1,60 @@ +/* + * Copyright (c) 2025 HiSilicon (Shanghai) Technologies Co., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef __HOST_UVC_H__ +#define __HOST_UVC_H__ + +#include +#include +#include +#include "sample_comm.h" +#ifdef __cplusplus +#if __cplusplus +extern "C" { +#endif +#endif + +#ifndef V4L2_PIX_FMT_H265 +#define V4L2_PIX_FMT_H265 v4l2_fourcc('H', '2', '6', '5') /* H.265 aka HEVC */ +#endif + +typedef struct { + const td_char *name; + td_u32 fourcc; + td_char n_planes; +} format_info; + +typedef struct { + const td_char *pattern; + const td_char *type_name; + td_s32 do_capture; + td_s32 do_set_format; + td_u32 pixelformat; + td_s32 do_file; + td_u32 input; + td_u32 width; + td_u32 height; +} uvc_ctrl_info; + +td_s32 sample_host_uvc(td_s32 argc, td_char *argv[], uvc_ctrl_info *ctrl_info); +const td_char *sample_uvc_v4l2_format_name(td_u32 fourcc); + +#ifdef __cplusplus +#if __cplusplus +} +#endif +#endif /* End of #ifdef __cplusplus */ + +#endif /* end of #ifndef __HOST_UVC_H__ */ diff --git a/vendor/zsks/demo/opencv_dnn/main.cpp b/vendor/zsks/demo/opencv_dnn/main.cpp new file mode 100755 index 0000000000000000000000000000000000000000..1b31d540155755992e62b75f60ee6d7e2ccc434b --- /dev/null +++ b/vendor/zsks/demo/opencv_dnn/main.cpp @@ -0,0 +1,306 @@ +/* + * Copyright (c) 2025 Zhongshan Kuangshi Microelectronics Technology Co., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Contact Information + * Author: yaohongtao + * Phone: +86-18604465633 + * Email: yht@cust.edu.cn + */ + +#include +#include +#include +#include +#include +#include +#include "host_uvc.h" +#include "media_vdec.h" +#include "sdk_module_init.h" + +using namespace cv; +using namespace std; +using namespace std::chrono; + +td_bool g_uvc_exit = TD_FALSE; + +// 优化内存管理类 +class MemoryManager { +public: + static void optimizeMemory() + { + cv::setUseOptimized(true); + cv::setNumThreads(4); /* 4: Number of CPU cores */ + } +}; + +static void sample_uvc_exit_signal_handler(int signal) +{ + printf("收到退出信号，准备退出...\n"); + g_uvc_exit = TD_TRUE; +} + +int main(int argc, char** argv) +{ + // 优化内存使用 + MemoryManager::optimizeMemory(); + + int ret; + uvc_ctrl_info ctrl_info = {0}; + struct sigaction sig_exit; + sig_exit.sa_handler = sample_uvc_exit_signal_handler; + sigaction(SIGINT, &sig_exit, nullptr); + sigaction(SIGTERM, &sig_exit, nullptr); + ret = sample_host_uvc(argc, argv, &ctrl_info); + if (ret != TD_SUCCESS) { + return TD_FAILURE; + } +#ifdef CONFIG_USER_SPACE + SDK_init(); +#endif + // OpenCV V4L2 video capture + VideoCapture cap(0, cv::CAP_V4L2); + if (!cap.isOpened()) { + printf("Error: Failed to open camera device!\n"); + return -1; + } + + int width = ctrl_info.width; + int height = ctrl_info.height; + int fps = 30; + + // 设置摄像头参数 + cap.set(CAP_PROP_FRAME_WIDTH, width); + cap.set(CAP_PROP_FRAME_HEIGHT, height); + cap.set(CAP_PROP_FPS, fps); + cap.set(CAP_PROP_BUFFERSIZE, 4); /* 4: Increase buffer size */ + + // 根据格式设置摄像头参数 + const char* type_name = sample_uvc_v4l2_format_name(ctrl_info.pixelformat); + if (strcmp(type_name, "MJPEG") == 0) { + cap.set(CAP_PROP_FOURCC, VideoWriter::fourcc('M', 'J', 'P', 'G')); + } else if (strcmp(type_name, "YUYV") == 0) { + cap.set(CAP_PROP_FOURCC, VideoWriter::fourcc('Y', 'U', 'Y', 'V')); + } else { + std::cerr << "只支持 YUYV 或 MJPEG 格式，请用 -fYUYV 或 -fMJPEG 参数启动！" << std::endl; + return -1; + } + + // 验证设置是否成功 + int actual_width = cap.get(CAP_PROP_FRAME_WIDTH); + int actual_height = cap.get(CAP_PROP_FRAME_HEIGHT); + double actual_fps = cap.get(CAP_PROP_FPS); + + printf("Requested resolution: %dx%d @ %d fps\n", width, height, fps); + printf("Actual resolution: %dx%d @ %.2f fps\n", actual_width, actual_height, actual_fps); + printf("Using format: %s\n", type_name); + + if (actual_width != width || actual_height != height) { + printf("Warning: Camera does not support requested resolution, using actual resolution\n"); + width = actual_width; + height = actual_height; + } + + // 初始化人脸检测器，使用更快的参数 + const string modelPath = "./onnx/face_detection_yunet_2023mar.onnx"; + Ptr detector = FaceDetectorYN::create( + /* 256 144 input width height 0.6 score_threshold 0.7 nms_threshold */ + modelPath, "", Size(256, 144), 0.6, 0.7, 100); /* 100 top_k */ + if (!detector) { + printf("Error: Failed to create face detector\n"); + return -1; + } + + // 初始化媒体链路 + if (sample_uvc_media_init(type_name, width, height) != TD_SUCCESS) { + std::cerr << "media start failed!" << std::endl; + return -1; + } + + const int MAX_RETRIES = 3; /* 3: max retries */ + int retry_count = 0; + bool device_reconnected = false; + int empty_frame_count = 0; + const int MAX_EMPTY_FRAMES = 10; /* 10: max empty frames */ + + // 预分配Mat对象 + Mat frame; + Mat faces; + Mat yuvFrame; + vector mjpeg_buffer; + mjpeg_buffer.reserve(width * height * 3 / 2); /* 3 / 2 YUV420 memory size */ + + // 创建MJPEG编码器 + vector mjpeg_params = { + IMWRITE_JPEG_QUALITY, 90, /* 90: JPEG quality */ + IMWRITE_JPEG_OPTIMIZE, 1, + IMWRITE_JPEG_PROGRESSIVE, 0 + }; + + // 预分配VPSS缓冲区 + ot_size pic_size; + pic_size.width = static_cast(width); + pic_size.height = static_cast(height); + + if (strcmp(type_name, "YUYV") == 0) { + // 预分配YUYV缓冲区，每个像素2字节 + yuvFrame.create(height, width, CV_8UC2); + } + + // 新增：人脸检测相关变量 + int frame_count = 0; + vector last_detected_faces; + vector last_confidence_scores; + const int DETECT_INTERVAL = 5; /* 5: Detect every 5 frames */ + + while (!g_uvc_exit) { + cap >> frame; + + if (frame.empty()) { + empty_frame_count++; + printf("Empty frame %d/%d, attempting to reconnect...\n", empty_frame_count, MAX_EMPTY_FRAMES); + + if (empty_frame_count >= MAX_EMPTY_FRAMES) { + if (retry_count < MAX_RETRIES) { + printf("Attempting to reconnect camera (attempt %d/%d)...\n", retry_count + 1, MAX_RETRIES); + cap.release(); + sleep(1); + + cap.open(0, cv::CAP_V4L2); + if (cap.isOpened()) { + cap.set(CAP_PROP_FRAME_WIDTH, width); + cap.set(CAP_PROP_FRAME_HEIGHT, height); + cap.set(CAP_PROP_FPS, fps); + cap.set(CAP_PROP_BUFFERSIZE, 4); /* 4 cap prop buffer size */ + if (strcmp(type_name, "MJPEG") == 0) { + cap.set(CAP_PROP_FOURCC, VideoWriter::fourcc('M', 'J', 'P', 'G')); + } else { + cap.set(CAP_PROP_FOURCC, VideoWriter::fourcc('Y', 'U', 'Y', 'V')); + } + + actual_width = cap.get(CAP_PROP_FRAME_WIDTH); + actual_height = cap.get(CAP_PROP_FRAME_HEIGHT); + actual_fps = cap.get(CAP_PROP_FPS); + + printf("Camera reconnected successfully\n"); + printf("New resolution: %dx%d @ %.2f fps\n", actual_width, actual_height, actual_fps); + + retry_count = 0; + empty_frame_count = 0; + device_reconnected = true; + continue; + } + retry_count++; + printf("Retry %d/%d failed\n", retry_count, MAX_RETRIES); + } else { + printf("Failed to reconnect after %d attempts\n", MAX_RETRIES); + break; + } + } + continue; + } + + empty_frame_count = 0; + retry_count = 0; + + if (device_reconnected) { + printf("Reinitializing media pipeline...\n"); + if (sample_uvc_media_init(type_name, width, height) != TD_SUCCESS) { + std::cerr << "media reinit failed!" << std::endl; + break; + } + device_reconnected = false; + } + + // 人脸检测逻辑 + frame_count++; + bool is_detect_frame = (frame_count % DETECT_INTERVAL == 0); + + if (is_detect_frame) { + // 预处理阶段 + Mat small_frame; + resize(frame, small_frame, Size(256, 144), 0, 0, INTER_NEAREST); /* 256 144 resize width height */ + + // 检测阶段 + detector->detect(small_frame, faces); + + // 处理检测结果 + const int minFaceSize = 50; /* 50 min face size */ + last_detected_faces.clear(); + last_confidence_scores.clear(); + + if (faces.rows > 0) { + float scale_x = static_cast(frame.cols) / small_frame.cols; + float scale_y = static_cast(frame.rows) / small_frame.rows; + + for (int i = 0; i < faces.rows; ++i) { + int x = static_cast(faces.at(i, 0) * scale_x); + int y = static_cast(faces.at(i, 1) * scale_y); + int w = static_cast(faces.at(i, 2) * scale_x); /* 2 the number of width */ + int h = static_cast(faces.at(i, 3) * scale_y); /* 3 the number of heigth */ + float score = faces.at(i, 4); /* 4 the number of socre */ + + if (w >= minFaceSize && h >= minFaceSize) { + last_detected_faces.emplace_back(x, y, w, h); + last_confidence_scores.push_back(score); + } + } + } + } + + // 绘制人脸框 + for (size_t i = 0; i < last_detected_faces.size(); ++i) { + rectangle(frame, last_detected_faces[i], Scalar(0, 255, 0), 2); /* 0 255 0 R G B, 2 Line width */ + char scoreText[20]; /* 20 text buffer size */ + ret = snprintf(scoreText, sizeof(scoreText), "Conf: %.2f", + last_confidence_scores[i] / 100); /* 100 get the percentage */ + if (ret < 0) { + printf("[func]:%s [line]:%d [info]:snprintf failed\n", __FUNCTION__, __LINE__); + break; + } + putText(frame, scoreText, Point(last_detected_faces[i].x, + last_detected_faces[i].y - 5), /* 5 pic height minus 5 */ + /* 0.5 scaling ratio 0 255 0 R G B, 1 Line width */ + FONT_HERSHEY_SIMPLEX, 0.5, Scalar(0, 255, 0), 1); + } + + // 后续处理保持不变 + if (strcmp(type_name, "MJPEG") == 0) { + imencode(".jpg", frame, mjpeg_buffer, mjpeg_params); + ret = sample_uvc_media_send_data(mjpeg_buffer.data(), mjpeg_buffer.size(), + width, &pic_size, "MJPEG"); + } else { + // YUYV格式转换 + cvtColor(frame, yuvFrame, COLOR_BGR2YUV_YUY2); + ret = sample_uvc_media_send_data(yuvFrame.data, yuvFrame.total() * yuvFrame.elemSize(), + yuvFrame.step, &pic_size, "YUYV"); + } + + if (ret != 0) { + ret = -errno; + std::cerr << "write error: " << strerror(-ret) << std::endl; + } + } + + printf("--media exit...\n"); + sample_uvc_media_stop_receive_data(); + sample_uvc_media_exit(); + cap.release(); + +#ifdef CONFIG_USER_SPACE + SDK_exit(); +#endif + + return 0; +} diff --git a/vendor/zsks/demo/opencv_dnn/media_vdec.c b/vendor/zsks/demo/opencv_dnn/media_vdec.c new file mode 100755 index 0000000000000000000000000000000000000000..82957d723c4931c394fc55b2444b60911ef79baf --- /dev/null +++ b/vendor/zsks/demo/opencv_dnn/media_vdec.c @@ -0,0 +1,803 @@ +/* + * Copyright (c) 2025 HiSilicon (Shanghai) Technologies Co., Ltd. + * Copyright (c) 2025 Zhongshan Kuangshi Microelectronics Technology Co., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Modifications Author: yaohongtao (yht@cust.edu.cn) + */ + +#include "media_vdec.h" +#include + +ot_vb_src g_vdec_vb_source = OT_VB_SRC_MOD; + +#define REF_NUM 2 +#define DISPLAY_NUM 2 + +static vdec_display_cfg g_vdec_display_cfg = { + .pic_size = PIC_1080P, + .intf_sync = OT_VO_OUT_1080P60, + .intf_type = OT_VO_INTF_HDMI, +}; + +static td_u32 g_input_width; +static td_u32 g_input_height; +static ot_size g_disp_size; +sample_vo_cfg g_vo_config; +static td_bool g_is_need_vdec = TD_TRUE; + +static td_s32 sample_uvc_init_module_vb(sample_vdec_attr *sample_vdec, td_u32 vdec_chn_num, ot_payload_type type, + td_u32 len) +{ + td_u32 i; + td_s32 ret; + for (i = 0; (i < vdec_chn_num) && (i < len); i++) { + sample_vdec[i].type = type; + sample_vdec[i].width = g_input_width; + sample_vdec[i].height = g_input_height; + sample_vdec[i].mode = sample_comm_vdec_get_lowdelay_en() ? OT_VDEC_SEND_MODE_COMPAT : + OT_VDEC_SEND_MODE_FRAME; + sample_vdec[i].sample_vdec_video.dec_mode = OT_VIDEO_DEC_MODE_IP; + sample_vdec[i].sample_vdec_video.bit_width = OT_DATA_BIT_WIDTH_8; + if (type == OT_PT_JPEG) { + sample_vdec[i].sample_vdec_video.ref_frame_num = 0; + } else { + sample_vdec[i].sample_vdec_video.ref_frame_num = REF_NUM; + } + sample_vdec[i].display_frame_num = DISPLAY_NUM; + sample_vdec[i].frame_buf_cnt = (type == OT_PT_JPEG) ? (sample_vdec[i].display_frame_num + 1) : + (sample_vdec[i].sample_vdec_video.ref_frame_num + sample_vdec[i].display_frame_num + 1); + if (type == OT_PT_JPEG) { + sample_vdec[i].sample_vdec_picture.pixel_format = OT_PIXEL_FORMAT_YVU_SEMIPLANAR_420; + sample_vdec[i].sample_vdec_picture.alpha = 255; /* 255:pic alpha value */ + } + } + ret = sample_comm_vdec_init_vb_pool(vdec_chn_num, &sample_vdec[0], len); + if (ret != TD_SUCCESS) { + sample_print("init mod common vb fail for %#x!\n", ret); + return ret; + } + return ret; +} + +static td_s32 sample_uvc_init_sys_and_vb(sample_vdec_attr *sample_vdec, td_u32 vdec_chn_num, ot_payload_type type, + td_u32 len) +{ + ot_vb_cfg vb_cfg; + ot_pic_buf_attr buf_attr = {0}; + td_s32 ret; + ret = sample_comm_sys_get_pic_size(g_vdec_display_cfg.pic_size, &g_disp_size); + if (ret != TD_SUCCESS) { + sample_print("sys get pic size fail for %#x!\n", ret); + return ret; + } + buf_attr.align = 0; + buf_attr.bit_width = OT_DATA_BIT_WIDTH_8; + buf_attr.compress_mode = OT_COMPRESS_MODE_SEG; + buf_attr.height = 2160; /* 2160:height */ + buf_attr.width = 3840; /* 3840:width */ + buf_attr.pixel_format = OT_PIXEL_FORMAT_YVU_SEMIPLANAR_420; + + (td_void)memset_s(&vb_cfg, sizeof(ot_vb_cfg), 0, sizeof(ot_vb_cfg)); + vb_cfg.max_pool_cnt = 1; + vb_cfg.common_pool[0].blk_cnt = 10 * vdec_chn_num; /* 10:common vb cnt */ + vb_cfg.common_pool[0].blk_size = ot_common_get_pic_buf_size(&buf_attr); + ret = sample_comm_sys_init(&vb_cfg); + if (ret != TD_SUCCESS) { + sample_print("init sys fail for %#x!\n", ret); + sample_comm_sys_exit(); + return ret; + } + + if (g_is_need_vdec == TD_TRUE) { + ret = sample_uvc_init_module_vb(&sample_vdec[0], vdec_chn_num, type, len); + if (ret != TD_SUCCESS) { + sample_print("init mod vb fail for %#x!\n", ret); + sample_comm_vdec_exit_vb_pool(); + sample_comm_sys_exit(); + return ret; + } + } + + return ret; +} + +static td_s32 sample_uvc_vdec_bind_vpss(td_u32 vpss_grp_num) +{ + td_u32 i; + td_s32 ret = TD_SUCCESS; + for (i = 0; i < vpss_grp_num; i++) { + ret = sample_comm_vdec_bind_vpss(i, i); + if (ret != TD_SUCCESS) { + sample_print("vdec bind vpss fail for %#x!\n", ret); + return ret; + } + } + return ret; +} + +static td_void sample_uvc_stop_vpss(ot_vpss_grp vpss_grp, td_bool *vpss_chn_enable, td_u32 chn_array_size) +{ + td_s32 i; + for (i = vpss_grp; i >= 0; i--) { + vpss_grp = i; + sample_common_vpss_stop(vpss_grp, &vpss_chn_enable[0], chn_array_size); + } +} + +static td_s32 sample_uvc_vdec_unbind_vpss(td_u32 vpss_grp_num) +{ + td_u32 i; + td_s32 ret = TD_SUCCESS; + for (i = 0; i < vpss_grp_num; i++) { + ret = sample_comm_vdec_un_bind_vpss(i, i); + if (ret != TD_SUCCESS) { + sample_print("vdec unbind vpss fail for %#x!\n", ret); + } + } + return ret; +} + +static td_void sample_uvc_config_vpss_grp_attr(ot_vpss_grp_attr *vpss_grp_attr) +{ + vpss_grp_attr->max_width = g_input_width; + vpss_grp_attr->max_height = g_input_height; + vpss_grp_attr->frame_rate.src_frame_rate = -1; + vpss_grp_attr->frame_rate.dst_frame_rate = -1; + vpss_grp_attr->pixel_format = OT_PIXEL_FORMAT_YVU_SEMIPLANAR_420; + vpss_grp_attr->nr_en = TD_FALSE; + vpss_grp_attr->ie_en = TD_FALSE; + vpss_grp_attr->dci_en = TD_FALSE; + vpss_grp_attr->dei_mode = OT_VPSS_DEI_MODE_OFF; + vpss_grp_attr->buf_share_en = TD_FALSE; +} + +static td_void sample_uvc_stop_vdec(td_u32 vdec_chn_num) +{ + if (g_is_need_vdec == TD_FALSE) { + return; + } + sample_comm_vdec_stop(vdec_chn_num); + sample_comm_vdec_exit_vb_pool(); + + return; +} + +static td_s32 sample_uvc_start_vdec(sample_vdec_attr *sample_vdec, td_u32 vdec_chn_num, td_u32 len) +{ + td_s32 ret; + + if (g_is_need_vdec == TD_FALSE) { + return TD_SUCCESS; + } + ret = sample_comm_vdec_start(vdec_chn_num, &sample_vdec[0], len); + if (ret != TD_SUCCESS) { + sample_print("start VDEC fail for %#x!\n", ret); + sample_uvc_stop_vdec(vdec_chn_num); + } + + return ret; +} + +static td_s32 sample_uvc_config_vpss_ldy_attr(td_u32 vpss_grp_num) +{ + td_u32 i; + td_s32 ret; + ot_low_delay_info vpss_ldy_info; + + for (i = 0; i < vpss_grp_num; i++) { + ret = ss_mpi_vpss_get_low_delay_attr(i, 0, &vpss_ldy_info); + if (ret != TD_SUCCESS) { + sample_print("vpss get low delay attr fail for %#x!\n", ret); + return ret; + } + vpss_ldy_info.enable = TD_TRUE; + vpss_ldy_info.line_cnt = g_disp_size.height / 4 * 1; /* 1/4:lowdelay line num */ + ret = ss_mpi_vpss_set_low_delay_attr(i, 0, &vpss_ldy_info); + if (ret != TD_SUCCESS) { + sample_print("vpss set low delay attr fail for %#x!\n", ret); + return ret; + } + } + return TD_SUCCESS; +} + +static td_s32 sample_uvc_start_vpss(ot_vpss_grp *vpss_grp, td_u32 vpss_grp_num, + td_bool *vpss_chn_enable, td_u32 arr_len) +{ + td_u32 i; + td_s32 ret; + ot_vpss_chn_attr vpss_chn_attr[OT_VPSS_MAX_CHN_NUM]; + ot_vpss_grp_attr vpss_grp_attr = {0}; + sample_uvc_config_vpss_grp_attr(&vpss_grp_attr); + (td_void)memset_s(vpss_chn_enable, arr_len * sizeof(td_bool), 0, arr_len * sizeof(td_bool)); + if (arr_len > 1) { + vpss_chn_enable[0] = TD_TRUE; + vpss_chn_attr[0].width = g_disp_size.width; + vpss_chn_attr[0].height = g_disp_size.height; + vpss_chn_attr[0].chn_mode = OT_VPSS_CHN_MODE_USER; + vpss_chn_attr[0].compress_mode = OT_COMPRESS_MODE_NONE; + vpss_chn_attr[0].pixel_format = OT_PIXEL_FORMAT_YVU_SEMIPLANAR_420; + vpss_chn_attr[0].video_format = OT_VIDEO_FORMAT_LINEAR; + vpss_chn_attr[0].frame_rate.src_frame_rate = -1; + vpss_chn_attr[0].frame_rate.dst_frame_rate = -1; + vpss_chn_attr[0].depth = 0; + vpss_chn_attr[0].mirror_en = TD_FALSE; + vpss_chn_attr[0].flip_en = TD_FALSE; + vpss_chn_attr[0].border_en = TD_FALSE; + vpss_chn_attr[0].aspect_ratio.mode = OT_ASPECT_RATIO_NONE; + } + + for (i = 0; i < vpss_grp_num; i++) { + *vpss_grp = i; + ret = sample_common_vpss_start(*vpss_grp, &vpss_chn_enable[0], + &vpss_grp_attr, vpss_chn_attr, OT_VPSS_MAX_CHN_NUM); + if (ret != TD_SUCCESS) { + sample_print("start VPSS fail for %#x!\n", ret); + sample_uvc_stop_vpss(*vpss_grp, &vpss_chn_enable[0], OT_VPSS_MAX_CHN_NUM); + return ret; + } + } + ret = sample_uvc_config_vpss_ldy_attr(vpss_grp_num); + if (ret != TD_SUCCESS) { + sample_uvc_stop_vpss(*vpss_grp, &vpss_chn_enable[0], OT_VPSS_MAX_CHN_NUM); + return ret; + } + ret = sample_uvc_vdec_bind_vpss(vpss_grp_num); + if (ret != TD_SUCCESS) { + sample_uvc_vdec_unbind_vpss(vpss_grp_num); + sample_uvc_stop_vpss(*vpss_grp, &vpss_chn_enable[0], OT_VPSS_MAX_CHN_NUM); + } + return ret; +} + +static td_s32 sample_uvc_vpss_bind_vo(sample_vo_cfg vo_config, td_u32 vpss_grp_num) +{ + td_u32 i; + ot_vo_layer vo_layer; + td_s32 ret = TD_SUCCESS; + vo_layer = vo_config.vo_dev; + for (i = 0; i < vpss_grp_num; i++) { + ret = sample_comm_vpss_bind_vo(i, 0, vo_layer, i); + if (ret != TD_SUCCESS) { + sample_print("vpss bind vo fail for %#x!\n", ret); + return ret; + } + } + return ret; +} + +static td_s32 sample_uvc_vpss_unbind_vo(td_u32 vpss_grp_num, sample_vo_cfg vo_config) +{ + td_u32 i; + ot_vo_layer vo_layer = vo_config.vo_dev; + td_s32 ret = TD_SUCCESS; + for (i = 0; i < vpss_grp_num; i++) { + ret = sample_comm_vpss_un_bind_vo(i, 0, vo_layer, i); + if (ret != TD_SUCCESS) { + sample_print("vpss unbind vo fail for %#x!\n", ret); + } + } + return ret; +} + +static td_void sample_uvc_get_default_vo_cfg(sample_vo_cfg *vo_config) +{ + vo_config->vo_dev = SAMPLE_VO_DEV_UHD; + vo_config->vo_intf_type = g_vdec_display_cfg.intf_type; + vo_config->intf_sync = g_vdec_display_cfg.intf_sync; + vo_config->pic_size = g_vdec_display_cfg.pic_size; + vo_config->bg_color = COLOR_RGB_BLUE; + vo_config->dis_buf_len = 3; /* 3:buf length */ + vo_config->dst_dynamic_range = OT_DYNAMIC_RANGE_SDR8; + vo_config->vo_mode = VO_MODE_1MUX; + vo_config->pix_format = OT_PIXEL_FORMAT_YVU_SEMIPLANAR_420; + vo_config->disp_rect.x = 0; + vo_config->disp_rect.y = 0; + vo_config->disp_rect.width = g_disp_size.width; + vo_config->disp_rect.height = g_disp_size.height; + vo_config->image_size.width = g_disp_size.width; + vo_config->image_size.height = g_disp_size.height; + vo_config->vo_part_mode = OT_VO_PARTITION_MODE_SINGLE; + vo_config->compress_mode = OT_COMPRESS_MODE_NONE; +} + +static td_s32 sample_uvc_start_vo(sample_vo_cfg *vo_config, td_u32 vpss_grp_num) +{ + td_s32 ret; + sample_uvc_get_default_vo_cfg(vo_config); + ret = sample_comm_vo_start_vo(vo_config); + if (ret != TD_SUCCESS) { + sample_print("start VO fail for %#x!\n", ret); + sample_comm_vo_stop_vo(vo_config); + return ret; + } + ret = sample_uvc_vpss_bind_vo(*vo_config, vpss_grp_num); + if (ret != TD_SUCCESS) { + sample_uvc_vpss_unbind_vo(vpss_grp_num, *vo_config); + sample_comm_vo_stop_vo(vo_config); + } + + return ret; +} + +static ot_payload_type sample_uvc_get_payload_type(const td_char *type_name) +{ + if (strcmp(type_name, "H264") == 0) { + return OT_PT_H264; + } else if (strcmp(type_name, "H265") == 0) { + return OT_PT_H265; + } else if (strcmp(type_name, "MJPEG") == 0) { + return OT_PT_JPEG; + } else { + sample_print("type name error!\n"); + return OT_PT_BUTT; + } +} + +static td_void sample_uvc_update_vdec_flag(const td_char *type_name) +{ + if ((strcmp(type_name, "H264") == 0) || + (strcmp(type_name, "H265") == 0) || + (strcmp(type_name, "MJPEG") == 0)) { + g_is_need_vdec = TD_TRUE; + } else { + g_is_need_vdec = TD_FALSE; + } +} + +td_s32 sample_uvc_media_init(const td_char *type_name, td_u32 width, td_u32 height) +{ + td_s32 ret; + td_u32 vdec_chn_num; + td_u32 vpss_grp_num; + sample_vdec_attr sample_vdec[OT_VDEC_MAX_CHN_NUM]; + td_bool vpss_chn_enable[OT_VPSS_MAX_CHN_NUM]; + ot_vpss_grp vpss_grp; + + ot_payload_type payload_type = OT_PT_H264; + vdec_chn_num = 1; + vpss_grp_num = vdec_chn_num; + g_input_width = width; + g_input_height = height; + sample_uvc_update_vdec_flag(type_name); + + if (g_is_need_vdec == TD_TRUE) { + payload_type = sample_uvc_get_payload_type(type_name); + if (payload_type == OT_PT_BUTT) { + return TD_FAILURE; + } + } + ret = sample_uvc_init_sys_and_vb(&sample_vdec[0], vdec_chn_num, payload_type, OT_VDEC_MAX_CHN_NUM); + if (ret != TD_SUCCESS) { + return ret; + } + ret = sample_uvc_start_vdec(&sample_vdec[0], vdec_chn_num, OT_VDEC_MAX_CHN_NUM); + if (ret != TD_SUCCESS) { + goto stop_sys; + } + ret = sample_uvc_start_vpss(&vpss_grp, vpss_grp_num, &vpss_chn_enable[0], OT_VPSS_MAX_CHN_NUM); + if (ret != TD_SUCCESS) { + goto stop_vdec; + } + ret = sample_uvc_start_vo(&g_vo_config, vpss_grp_num); + if (ret != TD_SUCCESS) { + goto stop_vpss; + } + + return TD_SUCCESS; + +stop_vpss: + ret = sample_uvc_vdec_unbind_vpss(vpss_grp_num); + sample_uvc_stop_vpss(vpss_grp, &vpss_chn_enable[0], OT_VPSS_MAX_CHN_NUM); +stop_vdec: + sample_uvc_stop_vdec(vdec_chn_num); +stop_sys: + sample_comm_sys_exit(); + + return TD_FAILURE; +} + +td_s32 sample_uvc_media_exit(td_void) +{ + td_u32 vdec_chn_num = 1; + td_u32 vpss_grp_num = 1; + ot_vpss_grp vpss_grp = 0; + td_bool vpss_chn_enable[OT_VPSS_MAX_CHN_NUM] = {0}; + vpss_chn_enable[0] = TD_TRUE; + + (td_void)sample_uvc_vpss_unbind_vo(vpss_grp_num, g_vo_config); + sample_comm_vo_stop_vo(&g_vo_config); + (td_void)sample_uvc_vdec_unbind_vpss(vpss_grp_num); + sample_uvc_stop_vpss(vpss_grp, &vpss_chn_enable[0], OT_VPSS_MAX_CHN_NUM); + sample_uvc_stop_vdec(vdec_chn_num); + sample_comm_sys_exit(); + + return TD_SUCCESS; +} + +static td_void sample_uvc_cut_stream_for_mjpeg(td_void *data, td_u32 size, td_s32 chn_id, + td_s32 *read_num, td_u32 *start) +{ + td_s32 i; + td_u32 len; + td_s32 read_len = size; + td_u8 *buf = data; + td_bool find_start = TD_FALSE; + for (i = 0; i < read_len - 1; i++) { + if (buf[i] == 0xFF && buf[i + 1] == 0xD8) { /* 0xFFD8: JPEG起始标记 */ + *start = i; + find_start = TD_TRUE; + i = i + 2; /* 2: 跳过标记长度 */ + break; + } + } + + /* 3: 检查下一个标记头所需长度 */ + for (; i < read_len - 3; i++) { + if ((buf[i] == 0xFF) && (buf[i + 1] & 0xF0) == 0xE0) { + len = (buf[i + 2] << 8) + buf[i + 3]; /* 2 3 index 8: 长度字段占2字节 */ + i += 1 + len; + } else { + break; + } + } + + for (; i < read_len - 1; i++) { + if (buf[i] == 0xFF && buf[i + 1] == 0xD9) { /* 0xFFD9: JPEG结束标记 */ + break; + } + } + read_len = i + 2; /* 2: 包含结束标记 */ + + if (find_start == TD_FALSE) { + sample_print("chn %d can not find JPEG start code! read_len %d!\n", chn_id, read_len); + } + + *read_num = read_len; +} + +static td_void sample_uvc_cut_stream_for_h264(td_void *data, td_u32 size, td_s32 chn_id, td_s32 *read_num) +{ + td_bool find_start = TD_FALSE; + td_bool find_end = TD_FALSE; + td_s32 i; + td_u8 *buf = data; + td_s32 read_len = size; + for (i = 0; i < read_len - 8; i++) { /* 8: 起始码最小长度 */ + int tmp = buf[i + 3] & 0x1F; /* 0x1F: NALU类型掩码 */ + /* 0x5: IDR帧, 0x1: 非IDR帧 0x80: 关键帧标志 2 index */ + if (buf[i] == 0 && buf[i + 1] == 0 && buf[i + 2] == 1 && + (((tmp == 0x5 || tmp == 0x1) && ((buf[i + 4] & 0x80) == 0x80)) || /* 4 index */ + (tmp == 20 && (buf[i + 7] & 0x80) == 0x80))) { /* 20 帧起始类型 7 index */ + find_start = TD_TRUE; + i += 8; /* 8: 跳过起始码 */ + break; + } + } + + for (; i < read_len - 8; i++) { /* 8: 查找结束所需最小长度 */ + int tmp = buf[i + 3] & 0x1F; /* 0x1F: NALU类型掩码 */ + if (buf[i] == 0 && buf[i + 1] == 0 && buf[i + 2] == 1 && /* 2 index */ + (tmp == 15 || tmp == 7 || tmp == 8 || tmp == 6 || /* 15 SEI 7 SPS 8 PPS 6 SEI */ + ((tmp == 5 || tmp == 1) && ((buf[i + 4] & 0x80) == 0x80)) || /* 5 1 帧起始类型 4 index */ + (tmp == 20 && (buf[i + 7] & 0x80) == 0x80))) { /* 20 帧起始类型 7 index */ + find_end = TD_TRUE; + break; + } + } + + if (i > 0) { + read_len = i; + } + + if (find_start == TD_FALSE) { + sample_print("chn %d can not find H264 start code! read_len %d!\n", chn_id, read_len); + } + + if (find_end == TD_FALSE) { + read_len = i + 8; /* 8: 包含下一帧起始码 */ + } + + *read_num = read_len; +} + +static td_void sample_uvc_cut_stream_for_h265(td_void *data, td_u32 size, td_s32 chn_id, td_s32 *read_num) +{ + td_bool find_start = TD_FALSE; + td_bool find_end = TD_FALSE; + td_bool new_pic = TD_FALSE; + td_s32 i; + td_u8 *buf = data; + td_s32 read_len = size; + for (i = 0; i < read_len - 6; i++) { /* 6:h265 frame start code length */ + td_u32 tmp = (buf[i + 3] & 0x7E) >> 1; /* 0x7E:frame start marker 3:index */ + new_pic = (buf[i + 0] == 0 && buf[i + 1] == 0 && buf[i + 2] == 1 && /* 1 2:index */ + (tmp <= 21) && ((buf[i + 5] & 0x80) == 0x80)); /* 5:index 21 0x80:frame start marker */ + + if (new_pic) { + find_start = TD_TRUE; + i += 6; /* 6:h265 frame start code length */ + break; + } + } + + for (; i < read_len - 6; i++) { /* 6:h265 frame start code length */ + td_u32 tmp = (buf[i + 3] & 0x7E) >> 1; /* 0x7E:frame start marker 3:index */ + new_pic = (buf[i + 0] == 0 && buf[i + 1] == 0 && buf[i + 2] == 1 && /* 1 2:index */ + (tmp == 32 || tmp == 33 || tmp == 34 || tmp == 39 || tmp == 40 || /* 32 33 34 39 40:frame start marker */ + ((tmp <= 21) && (buf[i + 5] & 0x80) == 0x80))); /* 5:index 21 0x80:frame start marker */ + + if (new_pic) { + find_end = TD_TRUE; + break; + } + } + + if (i > 0) { + read_len = i; + } + + if (find_start == TD_FALSE) { + sample_print("chn %d can not find H265 start code! read_len %d!\n", chn_id, read_len); + } + + if (find_end == TD_FALSE) { + read_len = i + 6; /* 6:h265 frame start code length */ + } + + *read_num = read_len; +} + +static td_s32 sample_uvc_prepare_frame_info(ot_vb_blk vb_blk, const ot_pic_buf_attr *buf_attr, + const ot_vb_calc_cfg *calc_cfg, ot_video_frame_info *video_frame) +{ + video_frame->video_frame.header_phys_addr[0] = ss_mpi_vb_handle_to_phys_addr(vb_blk); + if (video_frame->video_frame.header_phys_addr[0] == TD_NULL) { + sample_print("ss_mpi_vb_handle_to_phys_addr fail\n"); + return TD_FAILURE; + } + + video_frame->video_frame.header_virt_addr[0] = + (td_u8*)ss_mpi_sys_mmap(video_frame->video_frame.header_phys_addr[0], calc_cfg->vb_size); + if (video_frame->video_frame.header_virt_addr[0] == TD_NULL) { + sample_print("ss_mpi_sys_mmap fail\n"); + return TD_FAILURE; + } + + video_frame->mod_id = OT_ID_VGS; + video_frame->pool_id = ss_mpi_vb_handle_to_pool_id(vb_blk); + + video_frame->video_frame.header_phys_addr[1] = + video_frame->video_frame.header_phys_addr[0] + calc_cfg->head_y_size; + video_frame->video_frame.header_virt_addr[1] = + video_frame->video_frame.header_virt_addr[0] + calc_cfg->head_y_size; + video_frame->video_frame.phys_addr[0] = + video_frame->video_frame.header_phys_addr[0] + calc_cfg->head_size; + video_frame->video_frame.phys_addr[1] = + video_frame->video_frame.phys_addr[0] + calc_cfg->main_y_size; + video_frame->video_frame.virt_addr[0] = + video_frame->video_frame.header_virt_addr[0] + calc_cfg->head_size; + video_frame->video_frame.virt_addr[1] = + video_frame->video_frame.virt_addr[0] + calc_cfg->main_y_size; + video_frame->video_frame.header_stride[0] = calc_cfg->head_stride; + video_frame->video_frame.header_stride[1] = calc_cfg->head_stride; + video_frame->video_frame.stride[0] = calc_cfg->main_stride; + video_frame->video_frame.stride[1] = calc_cfg->main_stride; + + video_frame->video_frame.width = buf_attr->width; + video_frame->video_frame.height = buf_attr->height; + video_frame->video_frame.dynamic_range = OT_DYNAMIC_RANGE_SDR8; + video_frame->video_frame.compress_mode = OT_COMPRESS_MODE_NONE; + video_frame->video_frame.video_format = OT_VIDEO_FORMAT_LINEAR; + video_frame->video_frame.field = OT_VIDEO_FIELD_FRAME; + video_frame->video_frame.pixel_format = OT_PIXEL_FORMAT_YUV_SEMIPLANAR_420; + + return TD_SUCCESS; +} + +static td_void sample_uvc_buf_attr_init(const ot_size *pic_size, ot_pic_buf_attr *buf_attr) +{ + buf_attr->width = pic_size->width; + buf_attr->height = pic_size->height; + buf_attr->pixel_format = OT_PIXEL_FORMAT_YUV_SEMIPLANAR_420; + buf_attr->compress_mode = OT_COMPRESS_MODE_NONE; + buf_attr->align = 0; + buf_attr->bit_width = OT_DATA_BIT_WIDTH_8; +} + +static td_void sample_uvc_yuyv_to_nv12(td_char *image_in, td_u32 width, td_u32 height, + td_u32 size, td_char *image_out) +{ + td_s32 pixel_num = width * height; + td_u32 cycle_num = size / pixel_num / 2; /* 2: YUYV格式中每个像素占2字节 */ + + td_char *y = image_out; + td_char *uv = image_out + pixel_num; + + td_char *start = image_in; + td_u32 i = 0; + td_u32 j = 0; + td_u32 k = 0; + + /* Y */ + for (i = 0; i < cycle_num; i++) { + int index = 0; + for (j = 0; j < pixel_num * 2; j = j + 2) { /* 2: 跳过UV分量，每次取Y */ + *(y + index) = *(start + j); + index++; + } + start = image_in + pixel_num * 2 * i; /* 2: 每帧YUYV数据跨度 */ + y = y + pixel_num * 3 / 2; /* 3/2: NV12中Y+UV的总大小比例 */ + } + + /* UV */ + start = image_in; + for (i = 0; i < cycle_num; i++) { + int uv_index = 0; + for (j = 0; j < height; j = j + 2) { /* 2: 隔行采样UV */ + /* 2: YUYV宽度步进 4: 每两个像素的YUYV数据块 1: 取U分量的位置偏移 2: 取下一个V分量的偏移 */ + for (k = j * width * 2 + 1; k < width * 2 * (j + 1); k = k + 4) { + *(uv + uv_index) = *(start + k); /* U分量 */ + *(uv + uv_index + 1) = *(start + k + 2); /* 2 取下一个V分量的偏移 */ + uv_index += 2; /* 2: UV平面存储步进 */ + } + } + start = image_in + pixel_num * 2 * i; /* 2: 每帧YUYV数据跨度 */ + uv = uv + pixel_num * 3 / 2; /* 3/2: NV12中Y+UV的总大小比例 */ + } +} + +static td_void sample_uvc_update_vb_cfg(ot_pixel_format pixel_format, td_u32 stride, const ot_size *pic_size, + ot_vb_calc_cfg *calc_cfg) +{ + if (pixel_format != OT_PIXEL_FORMAT_YUYV_PACKAGE_422) { + return; + } + + calc_cfg->main_stride = stride >> 1; + calc_cfg->main_y_size = calc_cfg->main_stride * pic_size->height; +} + +static td_s32 sample_uvc_send_frame_to_vpss(td_void *data, td_u32 size, td_u32 stride, + const ot_size *pic_size, ot_pixel_format pixel_format) +{ + td_s32 ret; + ot_vpss_grp grp = 0; + ot_video_frame_info frame_info = {0}; + td_s32 milli_sec = -1; + ot_vb_blk vb_blk; + ot_pic_buf_attr buf_attr; + ot_vb_calc_cfg calc_cfg; + sample_uvc_buf_attr_init(pic_size, &buf_attr); + ot_common_get_pic_buf_cfg(&buf_attr, &calc_cfg); + sample_uvc_update_vb_cfg(pixel_format, stride, pic_size, &calc_cfg); + vb_blk = ss_mpi_vb_get_blk(OT_VB_INVALID_POOL_ID, calc_cfg.vb_size, TD_NULL); + if (vb_blk == OT_VB_INVALID_HANDLE) { + sample_print("get vb blk(size:%d) failed!\n", calc_cfg.vb_size); + return TD_FAILURE; + } + ret = sample_uvc_prepare_frame_info(vb_blk, &buf_attr, &calc_cfg, &frame_info); + if (ret != TD_SUCCESS) { + return ret; + } + + if (pixel_format == OT_PIXEL_FORMAT_YUYV_PACKAGE_422) { + sample_uvc_yuyv_to_nv12(data, pic_size->width, pic_size->height, size, frame_info.video_frame.virt_addr[0]); + } else { + frame_info.video_frame.pixel_format = pixel_format; + (td_void)memcpy_s(frame_info.video_frame.virt_addr[0], size, data, size); + } + ret = ss_mpi_vpss_send_frame(grp, &frame_info, milli_sec); + if (ret != TD_SUCCESS) { + sample_print("send frame to vpss failed!\n"); + } + ss_mpi_sys_munmap(frame_info.video_frame.virt_addr[0], calc_cfg.vb_size); + ret = ss_mpi_vb_release_blk(vb_blk); + if (ret != TD_SUCCESS) { + sample_print("release vb failed!\n"); + return TD_FAILURE; + } + + return ret; +} + +static td_s32 sample_uvc_get_pixel_format(const td_char *type_name, ot_pixel_format *pixel_format) +{ + if (strcmp(type_name, "YUYV") == 0) { + *pixel_format = OT_PIXEL_FORMAT_YUYV_PACKAGE_422; + } else if (strcmp(type_name, "NV12") == 0) { + *pixel_format = OT_PIXEL_FORMAT_YUV_SEMIPLANAR_420; + } else if (strcmp(type_name, "NV21") == 0) { + *pixel_format = OT_PIXEL_FORMAT_YVU_SEMIPLANAR_420; + } else { + sample_print("pixel format error!\n"); + return TD_FAILURE; + } + + return TD_SUCCESS; +} + +td_s32 sample_uvc_media_send_data(td_void *data, td_u32 size, td_u32 stride, + const ot_size *pic_size, const td_char *type_name) +{ + td_bool end_of_stream = TD_FALSE; + td_s32 read_len = size; + td_u8 *buf = data; + ot_vdec_stream vdec_stream; + td_u32 start = 0; + td_s32 ret; + td_s32 chn_id = 0; + ot_pixel_format pixel_format; + td_bool send_again = TD_TRUE; + + if (strcmp(type_name, "MJPEG") == 0) { + sample_uvc_cut_stream_for_mjpeg(data, size, chn_id, &read_len, &start); + } else if (strcmp(type_name, "H264") == 0) { + sample_uvc_cut_stream_for_h264(data, size, chn_id, &read_len); + } else if (strcmp(type_name, "H265") == 0) { + sample_uvc_cut_stream_for_h265(data, size, chn_id, &read_len); + } else { + ret = sample_uvc_get_pixel_format(type_name, &pixel_format); + if (ret != TD_SUCCESS) { + return ret; + } + return sample_uvc_send_frame_to_vpss(data, size, stride, pic_size, pixel_format); + } + + vdec_stream.addr = buf + start; + vdec_stream.len = read_len; + vdec_stream.end_of_frame = TD_TRUE; + vdec_stream.end_of_stream = end_of_stream; + vdec_stream.need_display = 1; + + while (send_again == TD_TRUE) { + ss_mpi_sys_get_cur_pts(&vdec_stream.pts); + ret = ss_mpi_vdec_send_stream(chn_id, &vdec_stream, 0); + if (ret != TD_SUCCESS) { + usleep(1000); /* 1000 sleep 1000us */ + sample_print("send stream to vdec failed!\n"); + continue; + } else { + end_of_stream = TD_FALSE; + } + send_again = TD_FALSE; + } + usleep(1000); /* 1000 sleep 1000us */ + + return TD_SUCCESS; +} + +td_s32 sample_uvc_media_stop_receive_data(td_void) +{ + ot_vdec_stream vdec_stream = {0}; + td_u32 vdec_chn_num = 1; + td_s32 chn_id = 0; + + if (g_is_need_vdec == TD_FALSE) { + return TD_SUCCESS; + } + + vdec_stream.end_of_stream = TD_TRUE; + ss_mpi_vdec_send_stream(chn_id, &vdec_stream, -1); + ss_mpi_vdec_stop_recv_stream(vdec_chn_num); + + return TD_SUCCESS; +} + +#ifdef __cplusplus +#if __cplusplus +} +#endif +#endif /* End of #ifdef __cplusplus */ diff --git a/vendor/zsks/demo/opencv_dnn/media_vdec.h b/vendor/zsks/demo/opencv_dnn/media_vdec.h new file mode 100755 index 0000000000000000000000000000000000000000..dcd6942734b98e09e5befc75a6cac348d7fa38c8 --- /dev/null +++ b/vendor/zsks/demo/opencv_dnn/media_vdec.h @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2025 HiSilicon (Shanghai) Technologies Co., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef __MEDIA_VDEC_H__ +#define __MEDIA_VDEC_H__ + +#include "sample_comm.h" + +#ifdef __cplusplus +#if __cplusplus +extern "C" { +#endif +#endif + +td_s32 sample_uvc_media_init(const td_char *type_name, td_u32 width, td_u32 height); +td_s32 sample_uvc_media_exit(td_void); +td_s32 sample_uvc_media_send_data(td_void *data, td_u32 data_size, td_u32 stride, + const ot_size *pic_size, const td_char *type_name); +td_s32 sample_uvc_media_stop_receive_data(td_void); + +#ifdef __cplusplus +#if __cplusplus +} +#endif +#endif /* End of #ifdef __cplusplus */ + + +#endif /* end of #ifndef __MEDIA_VDEC_H__ */ diff --git a/vendor/zsks/demo/opencv_dnn/pic/image-20221114150205685.png b/vendor/zsks/demo/opencv_dnn/pic/image-20221114150205685.png new file mode 100755 index 0000000000000000000000000000000000000000..4dc4215c962c57ba0e31548fb98c29c789f149d9 Binary files /dev/null and b/vendor/zsks/demo/opencv_dnn/pic/image-20221114150205685.png differ diff --git a/vendor/zsks/demo/opencv_dnn/pic/image-20250210161601541.png b/vendor/zsks/demo/opencv_dnn/pic/image-20250210161601541.png new file mode 100755 index 0000000000000000000000000000000000000000..80845ece8131933a4cd56ae3bd6119c38e0abd26 Binary files /dev/null and b/vendor/zsks/demo/opencv_dnn/pic/image-20250210161601541.png differ diff --git a/vendor/zsks/demo/opencv_dnn/pic/image-20250210170027454.png b/vendor/zsks/demo/opencv_dnn/pic/image-20250210170027454.png new file mode 100755 index 0000000000000000000000000000000000000000..c06081fc2c963af4d1fa8af506804beedefe1b0b Binary files /dev/null and b/vendor/zsks/demo/opencv_dnn/pic/image-20250210170027454.png differ diff --git a/vendor/zsks/demo/opencv_dnn/pic/image-20250213112932380.png b/vendor/zsks/demo/opencv_dnn/pic/image-20250213112932380.png new file mode 100755 index 0000000000000000000000000000000000000000..8b591b3437544859f4ec76b9ca03867cc401078c Binary files /dev/null and b/vendor/zsks/demo/opencv_dnn/pic/image-20250213112932380.png differ diff --git a/vendor/zsks/demo/opencv_dnn/pic/image-20250919150630870.png b/vendor/zsks/demo/opencv_dnn/pic/image-20250919150630870.png new file mode 100755 index 0000000000000000000000000000000000000000..91f881e85b3292118336ce2c3d81968e4aee8c26 Binary files /dev/null and b/vendor/zsks/demo/opencv_dnn/pic/image-20250919150630870.png differ diff --git a/vendor/zsks/demo/opencv_dnn/pic/image-20250919151018659.png b/vendor/zsks/demo/opencv_dnn/pic/image-20250919151018659.png new file mode 100755 index 0000000000000000000000000000000000000000..9b09507893b65d2142be49e6461daa31a81855c4 Binary files /dev/null and b/vendor/zsks/demo/opencv_dnn/pic/image-20250919151018659.png differ diff --git a/vendor/zsks/demo/opencv_dnn/pic/image-20250919151519310.png b/vendor/zsks/demo/opencv_dnn/pic/image-20250919151519310.png new file mode 100755 index 0000000000000000000000000000000000000000..7f6d881a5d0e6d3f72b7a43d089b5c5a75d7f048 Binary files /dev/null and b/vendor/zsks/demo/opencv_dnn/pic/image-20250919151519310.png differ diff --git a/vendor/zsks/demo/opencv_dnn/pic/image-20250919181207847.png b/vendor/zsks/demo/opencv_dnn/pic/image-20250919181207847.png new file mode 100755 index 0000000000000000000000000000000000000000..08df0f6a6eb2e5c602d2b818c18347651384099b Binary files /dev/null and b/vendor/zsks/demo/opencv_dnn/pic/image-20250919181207847.png differ diff --git a/vendor/zsks/demo/opencv_dnn/pic/image-20250919183224310.png b/vendor/zsks/demo/opencv_dnn/pic/image-20250919183224310.png new file mode 100755 index 0000000000000000000000000000000000000000..b539dd4175d5937062827ba614b2993165d510e4 Binary files /dev/null and b/vendor/zsks/demo/opencv_dnn/pic/image-20250919183224310.png differ diff --git a/vendor/zsks/demo/opencv_dnn/pic/image-20251015164912907.png b/vendor/zsks/demo/opencv_dnn/pic/image-20251015164912907.png new file mode 100755 index 0000000000000000000000000000000000000000..aa6e135f4c791b23615ebf00f1b70a85e7375162 Binary files /dev/null and b/vendor/zsks/demo/opencv_dnn/pic/image-20251015164912907.png differ diff --git a/vendor/zsks/demo/opencv_dnn/pic/image-20251015165024778.png b/vendor/zsks/demo/opencv_dnn/pic/image-20251015165024778.png new file mode 100755 index 0000000000000000000000000000000000000000..0648761c44f09bf97c9f57f2bc50e6002f8acd17 Binary files /dev/null and b/vendor/zsks/demo/opencv_dnn/pic/image-20251015165024778.png differ diff --git a/vendor/zsks/demo/opencv_dnn/pic/image-20251023111135567.png b/vendor/zsks/demo/opencv_dnn/pic/image-20251023111135567.png new file mode 100755 index 0000000000000000000000000000000000000000..62dfa69dc6c65a8bdbb15179c67c80488603cb4b Binary files /dev/null and b/vendor/zsks/demo/opencv_dnn/pic/image-20251023111135567.png differ diff --git a/vendor/zsks/demo/opencv_dnn/pic/image-20251023111425708.png b/vendor/zsks/demo/opencv_dnn/pic/image-20251023111425708.png new file mode 100755 index 0000000000000000000000000000000000000000..e332a7048949ea2d4ec6c2008065868f1e9f02f7 Binary files /dev/null and b/vendor/zsks/demo/opencv_dnn/pic/image-20251023111425708.png differ diff --git a/vendor/zsks/demo/opencv_dnn/pic/image-20251023112130092.png b/vendor/zsks/demo/opencv_dnn/pic/image-20251023112130092.png new file mode 100755 index 0000000000000000000000000000000000000000..e9116e45957d13d4768c7042ee3d4b538664e7d0 Binary files /dev/null and b/vendor/zsks/demo/opencv_dnn/pic/image-20251023112130092.png differ diff --git a/vendor/zsks/demo/opencv_dnn/pic/image-20251023112228717.png b/vendor/zsks/demo/opencv_dnn/pic/image-20251023112228717.png new file mode 100755 index 0000000000000000000000000000000000000000..574bbf96ec5810820bc59643b895b0dded7a6208 Binary files /dev/null and b/vendor/zsks/demo/opencv_dnn/pic/image-20251023112228717.png differ diff --git a/vendor/zsks/demo/opencv_dnn/pic/image-20251023112328619.png b/vendor/zsks/demo/opencv_dnn/pic/image-20251023112328619.png new file mode 100755 index 0000000000000000000000000000000000000000..32547b0c69a9fb73e9143b6db15d34b81f45b2ca Binary files /dev/null and b/vendor/zsks/demo/opencv_dnn/pic/image-20251023112328619.png differ diff --git a/vendor/zsks/demo/opencv_dnn/pic/image-20251023112352692.png b/vendor/zsks/demo/opencv_dnn/pic/image-20251023112352692.png new file mode 100755 index 0000000000000000000000000000000000000000..536a306e1a28d57fcc24ef1b92b3304c832760de Binary files /dev/null and b/vendor/zsks/demo/opencv_dnn/pic/image-20251023112352692.png differ diff --git a/vendor/zsks/demo/opencv_dnn/pic/image-20251023112451423.png b/vendor/zsks/demo/opencv_dnn/pic/image-20251023112451423.png new file mode 100755 index 0000000000000000000000000000000000000000..28b7023a83d7eff1212f38bbd2fb6e606f6be596 Binary files /dev/null and b/vendor/zsks/demo/opencv_dnn/pic/image-20251023112451423.png differ diff --git a/vendor/zsks/demo/sample_kcf_track/Makefile b/vendor/zsks/demo/sample_kcf_track/Makefile new file mode 100755 index 0000000000000000000000000000000000000000..3f3058a2e4d124d2bd7ecd595e454f3bf817acd4 --- /dev/null +++ b/vendor/zsks/demo/sample_kcf_track/Makefile @@ -0,0 +1,48 @@ +# 编译所需的公共文件 +include ../mk.param + +# 编译后生成的可执行文件名 +TARGET := main + +# 编译此工程需要包含的头文件 +CFLAGS += -I$(SDK_LINUX_INCLUDE_DIR)/ +CFLAGS += -I$(SDK_SAMPLE_COMMON_DIR)/ +CFLAGS += -I$(SDK_LINUX_INCLUDE_DIR)/svp_npu +CFLAGS += -I$(KCF_TRACK_SAMPLE_DIR)/ + +# 编译此工程需要依赖的.c或.cpp文件 +SMP_SRCS := $(SDK_SAMPLE_COMMON_DIR)/sample_comm_sys.c +SMP_SRCS += $(SDK_SAMPLE_COMMON_DIR)/sample_ipc.c +SMP_SRCS += $(SDK_SAMPLE_COMMON_DIR)/sample_comm_isp.c +SMP_SRCS += $(SDK_SAMPLE_COMMON_DIR)/sample_comm_vi.c +SMP_SRCS += $(SDK_SAMPLE_COMMON_DIR)/sample_comm_vo.c +SMP_SRCS += $(SDK_SAMPLE_COMMON_DIR)/sample_comm_mipi_tx.c +SMP_SRCS += $(SDK_SAMPLE_COMMON_DIR)/sample_comm_vpss.c +SMP_SRCS += $(SDK_SAMPLE_COMMON_DIR)/loadbmp.c +SMP_SRCS += $(SDK_SAMPLE_COMMON_DIR)/sample_comm_vdec.c +SMP_SRCS += $(SDK_SAMPLE_COMMON_DIR)/sample_comm_venc.c +SMP_SRCS += $(SDK_SAMPLE_COMMON_DIR)/sdk_init.c +SMP_SRCS += $(SDK_SAMPLE_COMMON_DIR)/sdk_exit.c +SMP_SRCS += $(wildcard $(SVP_COMMON_DIR)/*.c) +SMP_SRCS += $(KCF_TRACK_SAMPLE_DIR)/main.c +SMP_SRCS += $(KCF_TRACK_SAMPLE_DIR)/sample_kcf_track.c +SMP_SRCS += $(KCF_TRACK_SAMPLE_DIR)/sample_svp_npu_process.c + +# 编译此工程需要包含的库和宏 +CFLAGS += -O3 -g -Wall -DUSE_USB_CAMERA +CFLAGS += -fstack-protector-strong + +# 根据实际的类型设置，可以用set_sensor脚本设置 +SENSOR0_TYPE = -DSENSOR0_TYPE=HY_S0603_MIPI_8M_30FPS_12BIT +SENSOR1_TYPE = -DSENSOR1_TYPE=HY_S0603_MIPI_8M_30FPS_12BIT +SENSOR2_TYPE = -DSENSOR1_TYPE=HY_S0603_MIPI_8M_30FPS_12BIT +SENSOR3_TYPE = -DSENSOR1_TYPE=HY_S0603_MIPI_8M_30FPS_12BIT + +CFLAGS += $(SENSOR0_TYPE) +CFLAGS += $(SENSOR1_TYPE) +CFLAGS += $(SENSOR2_TYPE) +CFLAGS += $(SENSOR3_TYPE) +# 编译工程所需的mk文件 + +include ../sdk_linux.mk + diff --git a/vendor/zsks/demo/sample_kcf_track/README.md b/vendor/zsks/demo/sample_kcf_track/README.md new file mode 100755 index 0000000000000000000000000000000000000000..2ba36c5f5462680e15a73c22cfada33aeb65bf80 --- /dev/null +++ b/vendor/zsks/demo/sample_kcf_track/README.md @@ -0,0 +1,142 @@ +## 2.4、sample_kcf_track操作指导 + +### 2.4.1、sample_kcf_track程序简介 + +* sample_kcf_track 基于SS928V100平台开发，以EulerPi套件为例，sample_kcf_track 基于KCF+Track模型，通过USB Camera，将采集到的图片送到KFC+trak模型，实现对目标的实时跟踪效果。 + +### 2.4.2、目录 + +```shell +pegasus/vendor/cust/demo/sample_kcf_track +|── main.c # sample_kcf_track的主入口 +|── Makefile # 编译脚本 +|── sample_kcf_track.c # sample_kcf_track业务代码 +|── sample_kcf_track.h # sample_kcf_track业务代码头文件 +|── sample_svp_npu_process.c # 调用SVP_NPU的业务代码 +└── sample_svp_npu_process.c # 调用SVP_NPU的头文件 +``` + +![image-20250919184816530](pic/image-20250919184816530.png) + +* 根据外接显示屏的输出参数进行配置，比如说我的外接显示屏的输出是1080P60，如果smp/a55_linux/mpp/sample/svp/common/sample_common_svp.c ample_common_svp_get_def_vo_cfg函数的intf_sync为OT_VO_OUT_1080P30，则需要把他修改为OT_VO_OUT_1080P60。 + +![image-20251023102053075](pic/image-20251023102053075.png) + +### 2.4.3、编译 + +* **注意：在编译cust的demo之前，请确保你已经按照[patch介绍中的步骤](../../README.md#4patch介绍)把补丁打入海思SDK的sample目录下了**。 + +* 步骤1：先根据自己选择的操作系统，进入到对应的Pegasus所在目录下。 + +* 步骤2：使用Makefile的方式进行单编 + +* 在Ubuntu的命令行终端，分步执行下面的命令，单编 sample_kcf_track + + ``` + cd pegasus/vendor/cust/demo/sample_kcf_track + + make clean && make + ``` + + * 在sample_kcf_track/out目录下，生成一个名为main的可执行文件，如下图所示： + + ![image-20251023114014999](pic/image-20251023114014999.png) + +### 2.4.4、拷贝可执行程序和依赖文件至开发板的mnt目录下 + +**方式一：使用SD卡进行资料文件的拷贝** + +* 首先需要自己准备一张Micro sd卡(16G 左右即可)，还得有一个Micro sd卡的读卡器。 + +image-20221114150205685 + +* 步骤1：将编译后生成的可执行文件main，拷贝到SD卡中。 + +![image-20251023100505047](pic/image-20251023100505047.png) + +* 步骤2：可执行文件拷贝成功后，将内存卡插入开发板的SD卡槽中，可通过挂载的方式挂载到板端，可选择SD卡 mount指令进行挂载。 + +image-20250210161601541 + +* 在开发板的终端，执行下面的命令进行SD卡的挂载 + * 如果挂载失败，可以参考[这个issue尝试解决](https://gitee.com/HiSpark/HiSpark_NICU2022/issues/I54932?from=project-issue) + + +```shell +mount -t vfat /dev/mmcblk1p1 /mnt +# 其中/dev/mmcblk1p1需要根据实际块设备号修改 +``` + +* 挂载成功后，如下图所示： + +![image-20251023114317633](pic/image-20251023114317633.png) + +**方式二：使用NFS挂载的方式进行资料文件的拷贝** + +* 首先需要自己准备一根网线 +* 步骤1：参考[博客链接](https://blog.csdn.net/Wu_GuiMing/article/details/115872995?spm=1001.2014.3001.5501)中的内容，进行nfs的环境搭建 +* 步骤2：将编译后生成的可执行文件main，拷贝到Windows的nfs共享路径下 + +![image-20251023100536744](pic/image-20251023100536744.png) + +* 步骤3：在开发板的终端执行下面的命令，将Windows的nfs共享路径挂载至开发板的mnt目录下 + * 注意：这里IP地址请根据你开发板和电脑主机的实际IP地址进行填写 + + +``` +ifconfig eth0 192.168.100.100 + +mount -o nolock,addr=192.168.100.10 -t nfs 192.168.100.10:/d/nfs /mnt +``` + +![image-20251023114345977](pic/image-20251023114345977.png) + +### 2.4.5、硬件连接 + +* 准备一个外接显示器和HDMI数据线，将HDMI的一头接在开发板的HDMI输出口，将HDMI的另外一头接在外接显示器的HDMI输入口。 + +![image-20250213112932380](pic/image-20250213112932380.png) + +* 将USB 摄像头接在EulerPi开发板的USB接口上。 + +image-20250919150630870 + +### 2.4.6、功能验证 + +* 在开发板的终端执行下面的命令，运行可执行文件 + +``` +cd /mnt + +chmod +x main + +./main +``` + +![image-20251023114430657](pic/image-20251023114430657.png) + +* 此时，在HDMI的外接显示屏上即可出现实时码流，如下图所示： + +image-20250210170027454 + +* 如果您看到的现象和下图现象不一致，可以确认一下USB涉嫌头是否插入到开发板的USB口，并且在开发板的/dev目录下能够看到video0 和 video1两个设备节点。如果没有这两个设备节点，请确保镜像烧录正常。 + +![image-20250919151018659](pic/image-20250919151018659.png) + +* 正常情况下，我们会在外接显示屏上看到USB Camera采集到的画面。 +* 步骤1：按下键盘的空格键，会在屏幕的正中间显示一个红色的框框。 + +![image-20250919192349341](pic/image-20250919192349341.png) + +* 步骤2：把需要跟踪的目标放在红色框框内，然后再按一下空格键，此时红色框会变绿，如果没有变，可以重复一次。 +* 步骤3：移动需要跟踪的目标，绿色框会跟着目标一起移动。 + +![image-20250919192115350](pic/image-20250919192137187.png) + +* 当目标不在sensor视野范围内，绿框会变成黄色。 + +![image-20250919192243361](pic/image-20250919192243361.png) + +* 输入Q键，即可关闭程序 + +![image-20250919151519310](pic/image-20250919151519310.png) \ No newline at end of file diff --git a/vendor/zsks/demo/sample_kcf_track/main.c b/vendor/zsks/demo/sample_kcf_track/main.c new file mode 100755 index 0000000000000000000000000000000000000000..e25ac821e9157105a1a40b36e6b158cbc3614f8c --- /dev/null +++ b/vendor/zsks/demo/sample_kcf_track/main.c @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2025 HiSilicon (Shanghai) Technologies Co., Ltd. + * Copyright (c) 2025 Zhongshan Kuangshi Microelectronics Technology Co., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Modifications Author: yaohongtao (yht@cust.edu.cn) + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "sdk_module_init.h" +#include "securec.h" +#include "sample_svp_npu_process.h" + +int main(int argc, char *argv[]) +{ + td_s32 ret; + +#ifdef CONFIG_USER_SPACE + SDK_init(); +#endif + + sample_svp_npu_acl_v4l2_kcf(); + +#ifdef CONFIG_USER_SPACE + SDK_exit(); +#endif + return 0; +} diff --git a/vendor/zsks/demo/sample_kcf_track/pic/image-20221114150205685.png b/vendor/zsks/demo/sample_kcf_track/pic/image-20221114150205685.png new file mode 100755 index 0000000000000000000000000000000000000000..4dc4215c962c57ba0e31548fb98c29c789f149d9 Binary files /dev/null and b/vendor/zsks/demo/sample_kcf_track/pic/image-20221114150205685.png differ diff --git a/vendor/zsks/demo/sample_kcf_track/pic/image-20250210161601541.png b/vendor/zsks/demo/sample_kcf_track/pic/image-20250210161601541.png new file mode 100755 index 0000000000000000000000000000000000000000..80845ece8131933a4cd56ae3bd6119c38e0abd26 Binary files /dev/null and b/vendor/zsks/demo/sample_kcf_track/pic/image-20250210161601541.png differ diff --git a/vendor/zsks/demo/sample_kcf_track/pic/image-20250210170027454.png b/vendor/zsks/demo/sample_kcf_track/pic/image-20250210170027454.png new file mode 100755 index 0000000000000000000000000000000000000000..c06081fc2c963af4d1fa8af506804beedefe1b0b Binary files /dev/null and b/vendor/zsks/demo/sample_kcf_track/pic/image-20250210170027454.png differ diff --git a/vendor/zsks/demo/sample_kcf_track/pic/image-20250213112932380.png b/vendor/zsks/demo/sample_kcf_track/pic/image-20250213112932380.png new file mode 100755 index 0000000000000000000000000000000000000000..8b591b3437544859f4ec76b9ca03867cc401078c Binary files /dev/null and b/vendor/zsks/demo/sample_kcf_track/pic/image-20250213112932380.png differ diff --git a/vendor/zsks/demo/sample_kcf_track/pic/image-20250919150630870.png b/vendor/zsks/demo/sample_kcf_track/pic/image-20250919150630870.png new file mode 100755 index 0000000000000000000000000000000000000000..91f881e85b3292118336ce2c3d81968e4aee8c26 Binary files /dev/null and b/vendor/zsks/demo/sample_kcf_track/pic/image-20250919150630870.png differ diff --git a/vendor/zsks/demo/sample_kcf_track/pic/image-20250919151018659.png b/vendor/zsks/demo/sample_kcf_track/pic/image-20250919151018659.png new file mode 100755 index 0000000000000000000000000000000000000000..9b09507893b65d2142be49e6461daa31a81855c4 Binary files /dev/null and b/vendor/zsks/demo/sample_kcf_track/pic/image-20250919151018659.png differ diff --git a/vendor/zsks/demo/sample_kcf_track/pic/image-20250919151519310.png b/vendor/zsks/demo/sample_kcf_track/pic/image-20250919151519310.png new file mode 100755 index 0000000000000000000000000000000000000000..7f6d881a5d0e6d3f72b7a43d089b5c5a75d7f048 Binary files /dev/null and b/vendor/zsks/demo/sample_kcf_track/pic/image-20250919151519310.png differ diff --git a/vendor/zsks/demo/sample_kcf_track/pic/image-20250919184816530.png b/vendor/zsks/demo/sample_kcf_track/pic/image-20250919184816530.png new file mode 100755 index 0000000000000000000000000000000000000000..d9f0932b345fcfa7c140c5f21d352e7e5cff64c4 Binary files /dev/null and b/vendor/zsks/demo/sample_kcf_track/pic/image-20250919184816530.png differ diff --git a/vendor/zsks/demo/sample_kcf_track/pic/image-20250919192137187.png b/vendor/zsks/demo/sample_kcf_track/pic/image-20250919192137187.png new file mode 100755 index 0000000000000000000000000000000000000000..40ae9c3bd0c0aa31a35641ceb139f98b193505cf Binary files /dev/null and b/vendor/zsks/demo/sample_kcf_track/pic/image-20250919192137187.png differ diff --git a/vendor/zsks/demo/sample_kcf_track/pic/image-20250919192243361.png b/vendor/zsks/demo/sample_kcf_track/pic/image-20250919192243361.png new file mode 100755 index 0000000000000000000000000000000000000000..a2231c40a1430a678ed005050c743925f2ff02e1 Binary files /dev/null and b/vendor/zsks/demo/sample_kcf_track/pic/image-20250919192243361.png differ diff --git a/vendor/zsks/demo/sample_kcf_track/pic/image-20250919192349341.png b/vendor/zsks/demo/sample_kcf_track/pic/image-20250919192349341.png new file mode 100755 index 0000000000000000000000000000000000000000..e5ba33632d982e6806c584cc20fe487bb61389d6 Binary files /dev/null and b/vendor/zsks/demo/sample_kcf_track/pic/image-20250919192349341.png differ diff --git a/vendor/zsks/demo/sample_kcf_track/pic/image-20251023100505047.png b/vendor/zsks/demo/sample_kcf_track/pic/image-20251023100505047.png new file mode 100755 index 0000000000000000000000000000000000000000..25285f788edc601a27e2dadbc2cc4800d62c1a8a Binary files /dev/null and b/vendor/zsks/demo/sample_kcf_track/pic/image-20251023100505047.png differ diff --git a/vendor/zsks/demo/sample_kcf_track/pic/image-20251023100536744.png b/vendor/zsks/demo/sample_kcf_track/pic/image-20251023100536744.png new file mode 100755 index 0000000000000000000000000000000000000000..f399cbaa84a0f413594d2a082db25ab84a257084 Binary files /dev/null and b/vendor/zsks/demo/sample_kcf_track/pic/image-20251023100536744.png differ diff --git a/vendor/zsks/demo/sample_kcf_track/pic/image-20251023102053075.png b/vendor/zsks/demo/sample_kcf_track/pic/image-20251023102053075.png new file mode 100755 index 0000000000000000000000000000000000000000..6fc62dd35e3dcfeb36a6e3ba1e7cea3e008660ed Binary files /dev/null and b/vendor/zsks/demo/sample_kcf_track/pic/image-20251023102053075.png differ diff --git a/vendor/zsks/demo/sample_kcf_track/pic/image-20251023114014999.png b/vendor/zsks/demo/sample_kcf_track/pic/image-20251023114014999.png new file mode 100755 index 0000000000000000000000000000000000000000..eca5b8f8a17acda9378c05491b7357d6ae0fd7ce Binary files /dev/null and b/vendor/zsks/demo/sample_kcf_track/pic/image-20251023114014999.png differ diff --git a/vendor/zsks/demo/sample_kcf_track/pic/image-20251023114317633.png b/vendor/zsks/demo/sample_kcf_track/pic/image-20251023114317633.png new file mode 100755 index 0000000000000000000000000000000000000000..36789df9bbb5ebe76950defe29ed4e1ac4df6f2b Binary files /dev/null and b/vendor/zsks/demo/sample_kcf_track/pic/image-20251023114317633.png differ diff --git a/vendor/zsks/demo/sample_kcf_track/pic/image-20251023114345977.png b/vendor/zsks/demo/sample_kcf_track/pic/image-20251023114345977.png new file mode 100755 index 0000000000000000000000000000000000000000..4d258790884b235f6f7dce98c0d697f4d17bce88 Binary files /dev/null and b/vendor/zsks/demo/sample_kcf_track/pic/image-20251023114345977.png differ diff --git a/vendor/zsks/demo/sample_kcf_track/pic/image-20251023114430657.png b/vendor/zsks/demo/sample_kcf_track/pic/image-20251023114430657.png new file mode 100755 index 0000000000000000000000000000000000000000..921d330fc278312fb2c23e73fc2b3be5dd48bc29 Binary files /dev/null and b/vendor/zsks/demo/sample_kcf_track/pic/image-20251023114430657.png differ diff --git a/vendor/zsks/demo/sample_kcf_track/sample_kcf_track.c b/vendor/zsks/demo/sample_kcf_track/sample_kcf_track.c new file mode 100755 index 0000000000000000000000000000000000000000..066ecf22c1ead737fd4ecd2e259e3a73cf900bd0 --- /dev/null +++ b/vendor/zsks/demo/sample_kcf_track/sample_kcf_track.c @@ -0,0 +1,570 @@ +/* + * Copyright (c) 2025 HiSilicon (Shanghai) Technologies Co., Ltd. + * Copyright (c) 2025 Zhongshan Kuangshi Microelectronics Technology Co., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Modifications Author: yaohongtao (yht@cust.edu.cn) + */ +#include +#include +#include +#include +#include +#include "ss_mpi_ive.h" +#include "sample_common_svp.h" +#include "sample_common_ive.h" +#include "sample_kcf_track.h" + +#define KCF_QUEUE_LEN 4 /* 简化环形队列深度 */ + +/* **** 新增：KCF 用到的全局结构 **** */ +static sample_yolo_kcf_info g_kcf_info = {0}; /* 资源句柄、互斥锁等 */ +static ot_sample_svp_rect_info g_new_rect; /* NPU 推过来的检测框 */ +static pthread_mutex_t g_rect_mutex = PTHREAD_MUTEX_INITIALIZER; +static td_bool g_has_new_rect = TD_FALSE; /* 标记框是否更新 */ +static td_bool g_stop_flag = TD_FALSE; +static pthread_t g_get_frame_thd = 0; +static pthread_t g_track_thd = 0; +static int g_kcf_mode = 0; // 0=只画红框, 1=跟踪 + +// 新增：全局保存最后有效框和丢失计数（替代线程内的局部静态变量） +static ot_sample_svp_rect_info g_last_valid_rect = {0}; +static td_s32 g_lost_count = 0; + +#define KCF_BOX_WIDTH 200 // 统一宽度 +#define KCF_BOX_HEIGHT 200 // 统一高度 + +static void kcf_rect_to_roi_info(const ot_sample_svp_rect_info *rect, td_u32 base_w, td_u32 base_h) +{ + if (rect->num == 0) { + return; + } + // 只用第0个框，不做任何选择 + g_kcf_info.roi_num = 1; + g_kcf_info.roi_info[0].roi_id = 0; + g_kcf_info.roi_info[0].roi.x = rect->rect[0].point[0].x * OT_SAMPLE_QUARTER_OF_1M; + g_kcf_info.roi_info[0].roi.y = rect->rect[0].point[0].y * OT_SAMPLE_QUARTER_OF_1M; + g_kcf_info.roi_info[0].roi.width = rect->rect[0].point[1].x - rect->rect[0].point[0].x; + g_kcf_info.roi_info[0].roi.height = rect->rect[0].point[2].y - rect->rect[0].point[1].y; /* 2 index */ + // 后续宽高限制代码可保留 + if (g_kcf_info.roi_info[0].roi.width > 684) { /* 684 pic width */ + g_kcf_info.roi_info[0].roi.width = 684; /* 684 pic width */ + } + if (g_kcf_info.roi_info[0].roi.height > 684) { /* 684 pic height */ + g_kcf_info.roi_info[0].roi.height = 684; /* 684 pic height */ + } + if (g_kcf_info.roi_info[0].roi.width < KCF_BOX_WIDTH) { + g_kcf_info.roi_info[0].roi.width = KCF_BOX_WIDTH; + } + if (g_kcf_info.roi_info[0].roi.height < KCF_BOX_HEIGHT) { + g_kcf_info.roi_info[0].roi.height = KCF_BOX_HEIGHT; + } + if ((g_kcf_info.roi_info[0].roi.x / OT_SAMPLE_QUARTER_OF_1M) + g_kcf_info.roi_info[0].roi.width > base_w) { + g_kcf_info.roi_info[0].roi.x = (base_w - g_kcf_info.roi_info[0].roi.width) * OT_SAMPLE_QUARTER_OF_1M; + } + if ((g_kcf_info.roi_info[0].roi.y / OT_SAMPLE_QUARTER_OF_1M) + g_kcf_info.roi_info[0].roi.height > base_h) { + g_kcf_info.roi_info[0].roi.y = (base_h - g_kcf_info.roi_info[0].roi.height) * OT_SAMPLE_QUARTER_OF_1M; + } + if (g_kcf_info.roi_info[0].roi.x < 0) { + g_kcf_info.roi_info[0].roi.x = 0; + } + if (g_kcf_info.roi_info[0].roi.y < 0) { + g_kcf_info.roi_info[0].roi.y = 0; + } + if (g_kcf_info.roi_info[0].roi.width & 1) { + g_kcf_info.roi_info[0].roi.width--; + } + if (g_kcf_info.roi_info[0].roi.height & 1) { + g_kcf_info.roi_info[0].roi.height--; + } +} + +/* **** 简单把 video_frame 填到 ot_svp_img **** */ +static void kcf_fill_image(const ot_video_frame_info *frame, ot_svp_img *img) +{ + img->phys_addr[0] = frame->video_frame.phys_addr[0]; + img->phys_addr[1] = frame->video_frame.phys_addr[1]; + img->stride[0] = frame->video_frame.stride[0]; + img->stride[1] = frame->video_frame.stride[1]; + img->width = frame->video_frame.width; + img->height = frame->video_frame.height; + + /* 根据 VPSS 输出像素格式决定 IVE 输入类型 */ + switch (frame->video_frame.pixel_format) { + case OT_PIXEL_FORMAT_YVU_SEMIPLANAR_420: + case OT_PIXEL_FORMAT_YUV_SEMIPLANAR_420: + img->type = OT_SVP_IMG_TYPE_YUV420SP; + break; + case OT_PIXEL_FORMAT_YVU_SEMIPLANAR_422: + case OT_PIXEL_FORMAT_YUV_SEMIPLANAR_422: + img->type = OT_SVP_IMG_TYPE_YUV422SP; + break; + default: + /* 回退为 420，必要时可扩展 */ + img->type = OT_SVP_IMG_TYPE_YUV420SP; + break; + } +} + +typedef struct { + td_s32 x; + td_s32 y; + td_s32 w; + td_s32 h; +} BoxPosition; + +// 限制框在画面范围内 +static void clamp_box_to_frame(BoxPosition* box, const ot_video_frame_info* frame) +{ + const td_s32 max_w = frame->video_frame.width - 2; /* 2 pixel */ + const td_s32 max_h = frame->video_frame.height - 2; /* 2 pixel */ + + // 确保最小尺寸 + box->w = (box->w > KCF_BOX_WIDTH) ? box->w : KCF_BOX_WIDTH; + box->h = (box->h > KCF_BOX_HEIGHT) ? box->h : KCF_BOX_HEIGHT; + + /* 4 允许超出的最大范围（框宽的1/4） */ + const td_s32 max_overlap = box->w / 4; + if (box->x + box->w > max_w + max_overlap) { + box->x = max_w - box->w; + } + if (box->y + box->h > max_h + max_overlap) { + box->y = max_h - box->h; + } +} + +// 消除位置抖动 +static void smooth_box_position(BoxPosition* current, BoxPosition* last, td_bool* is_first_frame) +{ + if (*is_first_frame) { + *last = *current; + *is_first_frame = TD_FALSE; + return; + } + + // 微小变化时保持上一帧位置 + if (abs(current->x - last->x) <= 2 && abs(current->y - last->y) <= 2 && /* 2 pixel */ + abs(current->w - last->w) <= 2 && abs(current->h - last->h) <= 2) { /* 2 pixel */ + *current = *last; + } else { + *last = *current; + } +} + +// 确保坐标和尺寸为偶数 +static void align_to_even(BoxPosition* box) +{ + box->x &= ~1; // 清除最低位保证为偶数 + box->y &= ~1; + box->w &= ~1; + box->h &= ~1; +} + +// 设置矩形顶点坐标 +static void set_rect_points(ot_sample_svp_rect* rect, const BoxPosition* box) +{ + rect->point[0].x = box->x; + rect->point[0].y = box->y; + rect->point[1].x = box->x + box->w; + rect->point[1].y = box->y; + rect->point[2].x = box->x + box->w; /* 2 index of 2 */ + rect->point[2].y = box->y + box->h; /* 2 index of 2 */ + rect->point[3].x = box->x; /* 3 index of 3 */ + rect->point[3].y = box->y + box->h; /* 3 index of 3 */ +} + +static void kcf_bbox_to_rect(const ot_ive_kcf_bbox bbox[], td_u32 num, + const ot_video_frame_info* frame, ot_sample_svp_rect_info* rect) +{ + rect->num = (num > 0) ? 1 : 0; + if (rect->num == 0) { + return; + } + // 初始化转换参数 + static BoxPosition last_box = {0}; + static td_bool first_frame = TD_TRUE; + + BoxPosition current = { + .x = bbox[0].roi_info.roi.x / OT_SAMPLE_QUARTER_OF_1M, + .y = bbox[0].roi_info.roi.y / OT_SAMPLE_QUARTER_OF_1M, + .w = bbox[0].roi_info.roi.width, + .h = bbox[0].roi_info.roi.height + }; + + // 处理流程 + clamp_box_to_frame(¤t, frame); + smooth_box_position(¤t, &last_box, &first_frame); + align_to_even(¤t); + + // 设置输出矩形 + set_rect_points(&rect->rect[0], ¤t); +} +/* 边界检测辅助函数 */ +static td_bool is_touching_border(const ot_sample_svp_rect *rect, td_s32 screen_w, + td_s32 screen_h, td_s32 threshold) +{ + td_s32 rect_left = rect->point[0].x; + td_s32 rect_top = rect->point[0].y; + td_s32 rect_right = rect->point[1].x; + td_s32 rect_bottom = rect->point[2].y; + + return (rect_left <= threshold) || (rect_top <= threshold) || + ((screen_w - rect_right) <= threshold) || ((screen_h - rect_bottom) <= threshold); +} + +/* 绘制矩形辅助函数 */ +static void draw_tracking_rect(const ot_video_frame_info *frm, + const ot_sample_svp_rect_info *rect_info, td_bool is_touching_border) +{ + td_u32 color = is_touching_border ? 0x00FFFF00 : 0x0000FF00; + sample_common_svp_vgs_fill_rect(frm, rect_info, color); +} + +/* KCF训练辅助函数 */ +static void train_kcf_on_new_rect(const ot_video_frame_info *frm) +{ + pthread_mutex_lock(&g_rect_mutex); + kcf_rect_to_roi_info(&g_new_rect, frm->video_frame.width, frm->video_frame.height); + g_has_new_rect = TD_FALSE; + pthread_mutex_unlock(&g_rect_mutex); + + td_s32 ret = ss_mpi_ive_kcf_get_train_obj( + g_kcf_info.padding, g_kcf_info.roi_info, + g_kcf_info.roi_num, &g_kcf_info.cos_win_x, + &g_kcf_info.cos_win_y, &g_kcf_info.gauss_peak, + &g_kcf_info.obj_list); + if (ret != TD_SUCCESS) { + printf("[KCF] get_train_obj failed 0x%x, w=%u h=%u\n", ret, + g_kcf_info.roi_info[0].roi.width, + g_kcf_info.roi_info[0].roi.height); + } else { + printf("[KCF] train_obj_num=%u\n", g_kcf_info.obj_list.train_obj_num); + } +} + +/* 处理丢失目标情况 */ +static void handle_lost_target(const ot_video_frame_info *frm) +{ + g_lost_count++; + printf("[KCF] Lost target (count=%d), keep last rect\n", g_lost_count); + + if (g_last_valid_rect.num > 0) { + sample_common_svp_vgs_fill_rect(frm, &g_last_valid_rect, 0x00FFFF00); + } + + if (g_lost_count >= 10 && g_last_valid_rect.num > 0) { /* 10 lost count */ + pthread_mutex_lock(&g_rect_mutex); + g_new_rect = g_last_valid_rect; + g_has_new_rect = TD_TRUE; + pthread_mutex_unlock(&g_rect_mutex); + g_lost_count = 0; + } +} + +static void process_kcf_tracking(const ot_video_frame_info* frm, + ot_sample_svp_rect_info* draw_rect) +{ + td_s32 ret; + ot_ive_handle handle = 0; + ot_svp_img src_img; + + // 填充图像数据 + kcf_fill_image(frm, &src_img); + + // 执行KCF处理 + ret = ss_mpi_ive_kcf_proc(&handle, &src_img, &g_kcf_info.obj_list, + &g_kcf_info.kcf_proc_ctrl, TD_TRUE); + if (ret != TD_SUCCESS) { + printf("[KCF] kcf_proc failed 0x%x\n", ret); + g_kcf_info.bbox_obj_num = 0; + return; + } + + // 获取目标边界框 + ret = ss_mpi_ive_kcf_get_obj_bbox(&g_kcf_info.obj_list, g_kcf_info.bbox, + &g_kcf_info.bbox_obj_num, &g_kcf_info.kcf_bbox_ctrl); + if (ret != TD_SUCCESS) { + printf("[KCF] get_obj_bbox failed 0x%x\n", ret); + g_kcf_info.bbox_obj_num = 0; + return; + } + + // 处理有效目标 + if (g_kcf_info.bbox_obj_num > 0) { + kcf_bbox_to_rect(g_kcf_info.bbox, g_kcf_info.bbox_obj_num, frm, draw_rect); + + td_s32 screen_w = frm->video_frame.width - 2; + td_s32 screen_h = frm->video_frame.height - 2; + td_bool touching_border = is_touching_border( + &draw_rect->rect[0], screen_w, screen_h, 5); + + draw_tracking_rect(frm, draw_rect, touching_border); + g_last_valid_rect = *draw_rect; + g_lost_count = 0; + } +} + +/* 主线程函数（重构后） */ +static void *kcf_track_thread(void *args) +{ + (void)args; + td_s32 ret; + const ot_vo_layer vo_layer = 0; + const ot_vo_chn vo_chn = 0; + const td_s32 milli_sec = 20; + const td_s32 vpss_grp = 0; + const td_s32 vpss_chn = 0; + ot_sample_svp_rect_info draw_rect = {0}; + + while (!g_stop_flag) { + ot_video_frame_info frm; + ret = ss_mpi_vpss_get_chn_frame(vpss_grp, vpss_chn, &frm, milli_sec); + if (ret != TD_SUCCESS) { + usleep(1000); // 1000 sleep 1ms + continue; + } + + // 红框模式处理 + if (g_kcf_mode == 0) { + ot_ive_kcf_bbox initial_bbox[1]; + initial_bbox[0].roi_info = g_kcf_info.roi_info[0]; + kcf_bbox_to_rect(initial_bbox, 1, &frm, &draw_rect); + sample_common_svp_vgs_fill_rect(&frm, &draw_rect, 0x00FF0000); + g_kcf_info.bbox_obj_num = 0; + goto send_and_release; + } + + // 训练模式处理 + if (g_has_new_rect) { + train_kcf_on_new_rect(&frm); + } + + // 跟踪模式处理 + if (g_kcf_mode == 1 && (g_kcf_info.obj_list.train_obj_num != 0 || + g_kcf_info.obj_list.track_obj_num != 0)) { + process_kcf_tracking(&frm, &draw_rect); + } else if (g_kcf_mode == 1) { + g_kcf_info.bbox_obj_num = 0; + } + // 处理丢失目标 + if (g_kcf_mode == 1 && g_kcf_info.bbox_obj_num == 0) { + handle_lost_target(&frm); + } + + send_and_release: + ret = sample_common_svp_venc_vo_send_stream(&g_kcf_info.svp_switch, 0, + vo_layer, vo_chn, &frm); + if (ret != TD_SUCCESS) { + printf("[KCF] sample_common_svp_venc_vo_send_stream failed 0x%x\n", ret); + } + (td_void)ss_mpi_vpss_release_chn_frame(vpss_grp, vpss_chn, &frm); + usleep(8000); // 8000 sleep 8ms + } + return TD_NULL; +} + +static void sample_kcf_info_init(ot_svp_mem_info total_mem, td_u32 list_mem_size) +{ + g_kcf_info.list_mem = total_mem; + g_kcf_info.list_mem.size = list_mem_size; + + g_kcf_info.gauss_peak.phys_addr = g_kcf_info.list_mem.phys_addr + list_mem_size; + g_kcf_info.gauss_peak.virt_addr = g_kcf_info.list_mem.virt_addr + list_mem_size; + g_kcf_info.gauss_peak.size = SAMPLE_KCF_GAUSS_PEAK_SIZE; + + g_kcf_info.cos_win_x.phys_addr = g_kcf_info.gauss_peak.phys_addr + SAMPLE_KCF_GAUSS_PEAK_SIZE; + g_kcf_info.cos_win_x.virt_addr = g_kcf_info.gauss_peak.virt_addr + SAMPLE_KCF_GAUSS_PEAK_SIZE; + g_kcf_info.cos_win_x.size = SAMPLE_KCF_COS_WIN_SIZE; + + g_kcf_info.cos_win_y.phys_addr = g_kcf_info.cos_win_x.phys_addr + SAMPLE_KCF_COS_WIN_SIZE; + g_kcf_info.cos_win_y.virt_addr = g_kcf_info.cos_win_x.virt_addr + SAMPLE_KCF_COS_WIN_SIZE; + g_kcf_info.cos_win_y.size = SAMPLE_KCF_COS_WIN_SIZE; + + g_kcf_info.kcf_proc_ctrl.tmp_buf.phys_addr = + g_kcf_info.cos_win_y.phys_addr + SAMPLE_KCF_COS_WIN_SIZE; + g_kcf_info.kcf_proc_ctrl.tmp_buf.virt_addr = + g_kcf_info.cos_win_y.virt_addr + SAMPLE_KCF_COS_WIN_SIZE; + g_kcf_info.kcf_proc_ctrl.tmp_buf.size = SAMPLE_KCF_TEMP_BUF_SIZE; +} + +static void sample_kcf_proc_ctrl_init(void) +{ + g_kcf_info.kcf_proc_ctrl.csc_mode = OT_IVE_CSC_MODE_VIDEO_BT709_YUV_TO_RGB; + /* 0.001 learning rate 1024 Quantization scaling factor 32 Additional scaling factor */ + g_kcf_info.kcf_proc_ctrl.interp_factor = (td_u1q15)(0.001 * 1024 * 32); /* 降低学习率，减少抖动 ≈327 */ + g_kcf_info.kcf_proc_ctrl.lamda = 10; /* 10 lamda */ + g_kcf_info.kcf_proc_ctrl.sigma = (td_u0q8)(0.5 * 256); /* 0.5 * 256 降低sigma，提高稳定性 ≈77 */ + g_kcf_info.kcf_proc_ctrl.norm_trunc_alfa = (td_u4q12)(0.15 * 4096); /* 0.15 * 4096 降低截断因子 ≈614 */ + g_kcf_info.kcf_proc_ctrl.response_threshold = 5; /* 5 提高阈值，减少误检 */ + g_kcf_info.kcf_proc_ctrl.core_id = OT_IVE_KCF_CORE0; +} + +static void sample_kcf_bbox_ctrl_init(void) +{ + g_kcf_info.kcf_bbox_ctrl.max_bbox_num = SAMPLE_KCF_LIST_NODE_MAX; + g_kcf_info.kcf_bbox_ctrl.response_threshold = 12; /* 12 threshold */ + g_kcf_info.padding = 48; /* 48 padding */ +} + +static td_s32 sample_kcf_media_cfg(const ot_sample_svp_media_cfg *media_cfg) +{ + if (media_cfg != TD_NULL) { + g_kcf_info.svp_switch = media_cfg->svp_switch; /* {TD_FALSE, TD_TRUE} */ + } else { + g_kcf_info.svp_switch.is_venc_open = TD_FALSE; + g_kcf_info.svp_switch.is_vo_open = TD_TRUE; + } + + g_stop_flag = TD_FALSE; + // Removed g_get_frame_thd creation + if (pthread_create(&g_track_thd, NULL, kcf_track_thread, NULL) != 0) { + return TD_FAILURE; + } + // 在 sample_kcf_track_init 末尾添加 + g_kcf_info.roi_num = 1; + g_kcf_info.roi_info[0].roi_id = 0; + td_u32 safe_margin = 50; /* 50 Edge safety distance */ + td_u32 init_x = (1920 - KCF_BOX_WIDTH) / 2; /* 1920 PIC of width */ + td_u32 init_y = (1080 - KCF_BOX_HEIGHT) / 2; /* 1080 PIC of width */ + // 确保初始位置不靠近边缘（至少50像素缓冲） + if (init_x < safe_margin) init_x = safe_margin; + if (init_y < safe_margin) init_y = safe_margin; + g_kcf_info.roi_info[0].roi.x = init_x * OT_SAMPLE_QUARTER_OF_1M; + g_kcf_info.roi_info[0].roi.y = init_y * OT_SAMPLE_QUARTER_OF_1M; + g_kcf_info.roi_info[0].roi.width = KCF_BOX_WIDTH; + g_kcf_info.roi_info[0].roi.height = KCF_BOX_HEIGHT; + + return TD_SUCCESS; +} + +td_s32 sample_kcf_track_init(const ot_sample_svp_media_cfg *media_cfg, sample_vo_cfg *vo_cfg) +{ + memset(&g_kcf_info, 0, sizeof(g_kcf_info)); + pthread_mutex_init(&g_kcf_info.det_mutex, NULL); + + /* ---------------- KCF 内存初始化 ---------------- */ + td_s32 ret; + td_u32 list_mem_size; + td_u32 total_size; + + /* 1. 计算对象链表大小 */ + ret = ss_mpi_ive_kcf_get_mem_size(SAMPLE_KCF_LIST_NODE_MAX, &list_mem_size); + if (ret != TD_SUCCESS) { + printf("kcf get mem size failed 0x%x\n", ret); + return ret; + } + + /* 2. 申请一整块连续内存 (list + gauss + cosX + cosY + tmp) */ + total_size = list_mem_size + SAMPLE_KCF_GAUSS_PEAK_SIZE + + SAMPLE_KCF_COS_WIN_SIZE * 2 + SAMPLE_KCF_TEMP_BUF_SIZE; /* 2 is a number */ + + ot_svp_mem_info total_mem = {0}; + ret = sample_common_ive_create_mem_info(&total_mem, total_size); + if (ret != TD_SUCCESS) { + printf("kcf alloc total mem failed 0x%x\n", ret); + return ret; + } + + /* 3. 切分子区域 */ + sample_kcf_info_init(total_mem, list_mem_size); + + /* 4. 创建对象链表 */ + ret = ss_mpi_ive_kcf_create_obj_list(&g_kcf_info.list_mem, SAMPLE_KCF_LIST_NODE_MAX, &g_kcf_info.obj_list); + if (ret != TD_SUCCESS) { + printf("create obj list err 0x%x\n", ret); + return ret; + } + + /* 5. 生成 GaussPeak / CosWin 窗口 */ + g_kcf_info.padding = 48; /* 48 is 1.5*32 */ + ret = ss_mpi_ive_kcf_create_gauss_peak(g_kcf_info.padding, &g_kcf_info.gauss_peak); + if (ret != TD_SUCCESS) { + printf("create gauss peak err 0x%x\n", ret); + return ret; + } + ret = ss_mpi_ive_kcf_create_cos_win(&g_kcf_info.cos_win_x, &g_kcf_info.cos_win_y); + if (ret != TD_SUCCESS) { + printf("create cos win err 0x%x\n", ret); + return ret; + } + + /* 6. 填写 kcf_proc_ctrl 默认参数 */ + sample_kcf_proc_ctrl_init(); + + /* 7. 初始化 bbox ctrl */ + sample_kcf_bbox_ctrl_init(); + + /* 8. 继承媒体配置中的开关，使 KCF 线程能够把结果帧送到 VO/VENC */ + ret = sample_kcf_media_cfg(media_cfg); + if (ret != TD_SUCCESS) { + printf("kcf media cfg err 0x%x\n", ret); + return ret; + } + + return TD_SUCCESS; +} + +void sample_kcf_track_deinit(void) +{ + g_stop_flag = TD_TRUE; + // Removed g_get_frame_thd join + if (g_track_thd) { + pthread_join(g_track_thd, NULL); + } + /* 释放 KCF 内存与对象链表 */ + ss_mpi_ive_kcf_destroy_obj_list(&g_kcf_info.obj_list); + + /* 仅需释放一次整块内存，避免重复 free 导致 mmb not found 警告 */ + sample_svp_mmz_free(g_kcf_info.list_mem.phys_addr, g_kcf_info.list_mem.virt_addr); + + pthread_mutex_destroy(&g_kcf_info.det_mutex); +} + +// 修改 sample_kcf_set_det_rect，只在切换到跟踪模式时设置 g_has_new_rect +void sample_kcf_set_det_rect(const ot_sample_svp_rect_info *rect, int mode) +{ + pthread_mutex_lock(&g_rect_mutex); + g_new_rect = *rect; + if (mode == 1) { + ss_mpi_ive_kcf_destroy_obj_list(&g_kcf_info.obj_list); + ss_mpi_ive_kcf_create_obj_list(&g_kcf_info.list_mem, SAMPLE_KCF_LIST_NODE_MAX, &g_kcf_info.obj_list); + g_has_new_rect = TD_TRUE; + // 重置上一次的黄色框信息，避免切换时残留 + memset(&g_last_valid_rect, 0, sizeof(ot_sample_svp_rect_info)); + g_lost_count = 0; + } else if (mode == 0) { // 切换到红色框模式：强制红框回到中央 + // 计算屏幕中央坐标（基于1920x1080分辨率） + td_u32 center_x = (1920 - KCF_BOX_WIDTH) / 2; // 中央x坐标 + td_u32 center_y = (1080 - KCF_BOX_HEIGHT) / 2; // 中央y坐标 + // 重置红框的初始位置到中央（与初始化时一致） + g_kcf_info.roi_info[0].roi.x = center_x * OT_SAMPLE_QUARTER_OF_1M; + g_kcf_info.roi_info[0].roi.y = center_y * OT_SAMPLE_QUARTER_OF_1M; + // 保持宽高不变（KCF_BOX_WIDTH x KCF_BOX_HEIGHT） + g_kcf_info.roi_info[0].roi.width = KCF_BOX_WIDTH; + g_kcf_info.roi_info[0].roi.height = KCF_BOX_HEIGHT; + } + g_kcf_mode = mode; + + if (mode == 0) { + g_kcf_info.obj_list.train_obj_num = 0; + g_kcf_info.obj_list.track_obj_num = 0; + g_kcf_info.bbox_obj_num = 0; + } + pthread_mutex_unlock(&g_rect_mutex); +} + +td_u32 sample_kcf_get_bbox_num(void) +{ + return g_kcf_info.bbox_obj_num; +} diff --git a/vendor/zsks/demo/sample_kcf_track/sample_kcf_track.h b/vendor/zsks/demo/sample_kcf_track/sample_kcf_track.h new file mode 100755 index 0000000000000000000000000000000000000000..5ad8a3327f0f37a09f8c46cface5afa1b3d987f8 --- /dev/null +++ b/vendor/zsks/demo/sample_kcf_track/sample_kcf_track.h @@ -0,0 +1,81 @@ +/* + * Copyright (c) 2025 HiSilicon (Shanghai) Technologies Co., Ltd. + * Copyright (c) 2025 Zhongshan Kuangshi Microelectronics Technology Co., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Modifications Author: yaohongtao (yht@cust.edu.cn) + */ + +#ifndef SAMPLE_KCF_TRACK_H +#define SAMPLE_KCF_TRACK_H + +#include +#include "sample_comm.h" +#include "ot_common_svp.h" +#include "sample_common_svp.h" +#include "sample_common_ive.h" + +#ifndef SAMPLE_YOLO_KCF_INFO_T +#define SAMPLE_YOLO_KCF_INFO_T +#define SAMPLE_KCF_MAX_OBJ 4 + +typedef struct { + ot_ive_roi_info roi_info[SAMPLE_KCF_MAX_OBJ]; + td_u32 roi_num; + + /* 训练相关内存 */ + ot_svp_mem_info cos_win_x; + ot_svp_mem_info cos_win_y; + ot_svp_mem_info gauss_peak; + ot_svp_mem_info list_mem; /* KCF 对象链表内存 */ + ot_ive_kcf_obj_list obj_list; + ot_ive_kcf_proc_ctrl kcf_proc_ctrl; + ot_ive_kcf_bbox_ctrl kcf_bbox_ctrl; + + /* 跟踪结果 */ + ot_ive_kcf_bbox bbox[SAMPLE_KCF_MAX_OBJ]; + td_u32 bbox_obj_num; + + /* 帧缓存 */ + ot_video_frame_info frame_info_arr[1]; + + /* 互斥 */ + pthread_mutex_t det_mutex; + + /* 其它 */ + td_u3q5 padding; /* 调 API 需要的 padding 值 */ + ot_sample_svp_switch svp_switch; + sample_vo_cfg vo_cfg; /* VO configuration */ +} sample_yolo_kcf_info; +#endif + +#define SAMPLE_KCF_LIST_NODE_MAX 64 +#define SAMPLE_KCF_GAUSS_PEAK_SIZE 455680 +#define SAMPLE_KCF_COS_WIN_SIZE 832 +#define SAMPLE_KCF_TEMP_BUF_SIZE 47616 + +/* 说明： + * 1. sample_kcf_track_init 内部会创建获取帧+跟踪线程并一直运行； + * 2. 检测线程（NPU/RFCN 或 YOLO）每获得一次新检测框，调用 sample_kcf_set_det_rect + * 把第一只目标的 rect 信息推送给 KCF，KCF 会在下一帧里自动训练 / 跟踪； + * 3. sample_kcf_track_deinit 会安全地结束线程并释放内存。 + */ +td_s32 sample_kcf_track_init(const ot_sample_svp_media_cfg *media_cfg, sample_vo_cfg *vo_cfg); +void sample_kcf_track_deinit(void); + +void sample_kcf_set_det_rect(const ot_sample_svp_rect_info *rect, int mode); + +td_u32 sample_kcf_get_bbox_num(void); + +#endif /* SAMPLE_KCF_TRACK_H */ diff --git a/vendor/zsks/demo/sample_kcf_track/sample_svp_npu_process.c b/vendor/zsks/demo/sample_kcf_track/sample_svp_npu_process.c new file mode 100755 index 0000000000000000000000000000000000000000..b20b14d13e36993561a01f531a926f48a7269407 --- /dev/null +++ b/vendor/zsks/demo/sample_kcf_track/sample_svp_npu_process.c @@ -0,0 +1,233 @@ +/* + * Copyright (c) 2025 HiSilicon (Shanghai) Technologies Co., Ltd. + * Copyright (c) 2025 Zhongshan Kuangshi Microelectronics Technology Co., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Modifications Author: yaohongtao (yht@cust.edu.cn) + */ +#include +#include +#include +#include +#include +#include "sample_comm.h" +#include "sample_common_svp.h" +#include "sample_common_svp_npu.h" +#include "sample_common_svp_npu_model.h" +#include "sample_kcf_track.h" +#include "sample_svp_npu_process.h" + +#ifndef OT_SAMPLE_SYS_ALIGN_SIZE +#define OT_SAMPLE_SYS_ALIGN_SIZE 64 // 常见对齐值，根据 SDK 调整 +#endif +#ifndef CIF_WIDTH +#define CIF_WIDTH 200 +#endif +#ifndef CIF_HEIGHT +#define CIF_HEIGHT 200 +#endif + +// getch实现放在文件顶部 +static int getch(void) +{ + struct termios oldt; + struct termios newt; + int ch; + tcgetattr(STDIN_FILENO, &oldt); + newt = oldt; + newt.c_lflag &= ~(ICANON | ECHO); + tcsetattr(STDIN_FILENO, TCSANOW, &newt); + ch = getchar(); + tcsetattr(STDIN_FILENO, TCSANOW, &oldt); + return ch; +} + +td_s32 sample_common_svp_get_def_vo_cfg(sample_vo_cfg *vo_cfg); +static td_bool g_svp_npu_terminate_signal = TD_FALSE; +static sample_vi_cfg g_vi_config; + +static ot_sample_svp_rect_info g_svp_npu_rect_info = {0}; +static td_bool g_svp_npu_thread_stop = TD_FALSE; +static pthread_t g_svp_npu_thread = 0; +static sample_vo_cfg g_svp_npu_vo_cfg = { 0 }; +static pthread_t g_svp_npu_vdec_thread = 0; +static ot_vb_pool_info g_svp_npu_vb_pool_info; +static td_void *g_svp_npu_vb_virt_addr = TD_NULL; + +static ot_sample_svp_media_cfg g_svp_npu_media_cfg = { + .svp_switch = {TD_FALSE, TD_TRUE}, + .pic_type = {PIC_1080P, PIC_CIF}, // Changed to PIC_1080P for main stream + .chn_num = OT_SVP_MAX_VPSS_CHN_NUM, +}; + +static sample_vdec_attr g_svp_npu_vdec_cfg = { + .type = OT_PT_MJPEG, + .mode = OT_VDEC_SEND_MODE_FRAME, + .width = FHD_WIDTH, // Changed to FHD_WIDTH + .height = FHD_HEIGHT, // Changed to FHD_HEIGHT + .sample_vdec_picture.pixel_format = OT_PIXEL_FORMAT_YVU_SEMIPLANAR_420, /* 420:pixel_format */ + .sample_vdec_picture.alpha = 0, /* 0:alpha */ + .display_frame_num = 1, /* Keep 1 */ + .frame_buf_cnt = 3, /* Keep 3 */ +}; + +static vdec_thread_param g_svp_npu_vdec_param = { + .chn_id = 0, + + .v4l2_device = "/dev/video0", // 摄像头设备 + .width = FHD_WIDTH, // Changed to FHD_WIDTH + .height = FHD_HEIGHT, // Changed to FHD_HEIGHT + .pixel_format = V4L2_PIX_FMT_MJPEG, // MJPEG格式 + + .type = OT_PT_MJPEG, + .stream_mode = OT_VDEC_SEND_MODE_FRAME, + .interval_time = 1000, /* 1000:interval_time */ + .pts_init = 0, + .pts_increase = 0, + .e_thread_ctrl = THREAD_CTRL_START, + .circle_send = TD_TRUE, + .milli_sec = 0, + .min_buf_size = (FHD_WIDTH * FHD_HEIGHT * 3) >> 1, /* 3 Re-calculated for 1080P */ + .fps = 30, /* 30 fps */ +}; + +td_void sample_vb_cfg_init(ot_vb_cfg *vb_cfg, ot_size size_1080p, ot_size size_cif) +{ + td_u64 raw_size_1080p = 0; + td_u64 raw_size_cif = 0; + + raw_size_1080p = (td_u64)size_1080p.width * size_1080p.height * 3 / 2; /* 3 2 YUV420: 1.5 bytes per pixel */ + vb_cfg->max_pool_cnt = 2; /* 2 pool */ + vb_cfg->common_pool[0].blk_size = (raw_size_1080p + OT_SAMPLE_SYS_ALIGN_SIZE - 1) & ~(OT_SAMPLE_SYS_ALIGN_SIZE - 1); + vb_cfg->common_pool[0].blk_cnt = 10; // 块数，根据需要调整（对于1080P，建议10-20） + + raw_size_cif = (td_u64)size_cif.width * size_cif.height * 3 / 2; /* 3 2 raw size */ + vb_cfg->common_pool[1].blk_size = (raw_size_cif + OT_SAMPLE_SYS_ALIGN_SIZE - 1) & ~(OT_SAMPLE_SYS_ALIGN_SIZE - 1); + vb_cfg->common_pool[1].blk_cnt = 4; /* 4 pool */ +} + +td_void sample_center_rect_init(ot_sample_svp_rect_info *center_rect) +{ + int frame_w = FHD_WIDTH; + int frame_h = FHD_HEIGHT; // Use FHD resolution for initial box + int box_w = CIF_WIDTH; + int box_h = CIF_HEIGHT; // A reasonable box size for FHD + int x = (frame_w - box_w) / 2; /* 2 half of x */ + int y = (frame_h - box_h) / 2; /* 2 half of y */ + + center_rect->num = 1; + center_rect->rect[0].point[0].x = x; + center_rect->rect[0].point[0].y = y; + center_rect->rect[0].point[1].x = x + box_w; + center_rect->rect[0].point[1].y = y; + center_rect->rect[0].point[2].x = x + box_w; /* 2 index of 2 */ + center_rect->rect[0].point[2].y = y + box_h; /* 2 index of 2 */ + center_rect->rect[0].point[3].x = x; /* 3 index of 3 */ + center_rect->rect[0].point[3].y = y + box_h; /* 3 index of 3 */ + + printf("[MAIN-2] Setting input rect: (%d,%d) to (%d,%d)\n", + center_rect->rect[0].point[0].x, center_rect->rect[0].point[0].y, + center_rect->rect[0].point[2].x, center_rect->rect[0].point[2].y); /* 2 index of 2 */ +} + +// 1. 程序启动时，推送中心红框，KCF只画红框不训练 +static td_void sample_svp_npu_kcf_tracking_loop(td_void) +{ + ot_sample_svp_rect_info center_rect = {0}; + sample_center_rect_init(¢er_rect); + sample_kcf_set_det_rect(¢er_rect, 0); // 0=只画红框 + + int tracking_on = 0; // 0=红框, 1=跟踪 + // FIX: Use non-blocking input check to avoid blocking termination signal + struct termios oldt; + struct termios newt; + tcgetattr(STDIN_FILENO, &oldt); + newt = oldt; + newt.c_lflag &= ~(ICANON | ECHO); + tcsetattr(STDIN_FILENO, TCSANOW, &newt); + + while (!g_svp_npu_terminate_signal) { + int ch = getch(); + if (ch == ' ') { + tracking_on = !tracking_on; // 切换模式 + if (tracking_on) { + sample_kcf_set_det_rect(¢er_rect, 1); // 进入跟踪模式，重新训练KCF + printf("[MAIN] Set to tracking mode\n"); + } else { + sample_kcf_set_det_rect(¢er_rect, 0); // 回到红框模式 + printf("[MAIN] Set to red box mode\n"); + } + } else if (ch == 'q') { + break; + } + usleep(100000); // 100000 100ms poll to reduce CPU usage + } + // Restore terminal settings + tcsetattr(STDIN_FILENO, TCSANOW, &oldt); +} + +td_void sample_svp_npu_acl_v4l2_kcf(td_void) +{ + td_s32 ret; + ot_vb_cfg vb_cfg = {0}; + ot_size size_cif; + ot_size size_1080p; + + g_svp_npu_terminate_signal = TD_FALSE; /* 清除退出标志 */ + // 计算 1080P 的 blk_size + sample_comm_sys_get_pic_size(PIC_1080P, &size_1080p); // 从 SDK 获取标准分辨率 (1920x1080) + // 计算 CIF 的 blk_size + sample_comm_sys_get_pic_size(PIC_CIF, &size_cif); // 从 SDK 获取标准分辨率 (352x288) + sample_vb_cfg_init(&vb_cfg, size_1080p, size_cif); + ret = sample_comm_sys_init(&vb_cfg); // 初始化系统 VB + if (ret != TD_SUCCESS) { + printf("[SYS] init failed 0x%x\n", ret); + return; + } + /* ** 2. 新增：获取默认VO配置 ** */ + ret = sample_common_svp_get_def_vo_cfg(&g_svp_npu_vo_cfg); + if (ret != TD_SUCCESS) { + printf("[VO] get default config failed 0x%x\n", ret); + sample_comm_sys_exit(); // **** 新增：清理系统 VB **** + return; + } + /* 3. 根据采集分辨率重新设定 VB 图像类型，避免无谓的 4K 内存开销 */ + g_svp_npu_media_cfg.pic_type[0] = PIC_1080P; + g_svp_npu_media_cfg.pic_type[1] = PIC_CIF; /* 备用，不使用 */ + + /* 4. 启动 V4L2->VDEC->VPSS->VO 通路（内部自动绑定） */ + ret = sample_common_svp_create_vb_start_vdec_vpss_vo(&g_svp_npu_vdec_cfg, &g_svp_npu_vdec_param, + &g_svp_npu_vdec_thread, &g_svp_npu_media_cfg, &g_svp_npu_vo_cfg); + if (ret != TD_SUCCESS) { + printf("[PIPE] create pipeline failed 0x%x\n", ret); + sample_common_svp_destroy_vb_stop_vdec_vpss_vo(&g_svp_npu_vdec_param, &g_svp_npu_vdec_thread, + &g_svp_npu_media_cfg, &g_svp_npu_vo_cfg); + return; + } + /* 4. 启动 KCF 跟踪——内部线程会从 VPSS grp0 chn0 拉帧并画框 */ + ret = sample_kcf_track_init(&g_svp_npu_media_cfg, &g_svp_npu_vo_cfg); + if (ret != TD_SUCCESS) { + printf("[KCF] init failed 0x%x\n", ret); + sample_common_svp_destroy_vb_stop_vdec_vpss_vo(&g_svp_npu_vdec_param, &g_svp_npu_vdec_thread, + &g_svp_npu_media_cfg, &g_svp_npu_vo_cfg); + return; + } + + sample_svp_npu_kcf_tracking_loop(); + + sample_kcf_track_deinit(); + sample_common_svp_destroy_vb_stop_vdec_vpss_vo(&g_svp_npu_vdec_param, &g_svp_npu_vdec_thread, + &g_svp_npu_media_cfg, &g_svp_npu_vo_cfg); + sample_comm_sys_exit(); +} diff --git a/vendor/zsks/demo/sample_kcf_track/sample_svp_npu_process.h b/vendor/zsks/demo/sample_kcf_track/sample_svp_npu_process.h new file mode 100755 index 0000000000000000000000000000000000000000..567813211ca3b08fabc8cafda3d1610218d3007b --- /dev/null +++ b/vendor/zsks/demo/sample_kcf_track/sample_svp_npu_process.h @@ -0,0 +1,27 @@ +/* + * Copyright (c) 2025 HiSilicon (Shanghai) Technologies Co., Ltd. + * Copyright (c) 2025 Zhongshan Kuangshi Microelectronics Technology Co., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Modifications Author: yaohongtao (yht@cust.edu.cn) + */ + +#ifndef SAMPLE_SVP_NPU_PROCESS_H +#define SAMPLE_SVP_NPU_PROCESS_H +#include "ot_type.h" + +/* function : show the sample of v4l2 -> VDEC -> VPSS -> KCF -> VGS/VO */ +td_void sample_svp_npu_acl_v4l2_kcf(td_void); + +#endif diff --git a/vendor/zsks/demo/sdk_linux.mk b/vendor/zsks/demo/sdk_linux.mk new file mode 100755 index 0000000000000000000000000000000000000000..b9ce68fb47b0629f6df090e87372fb918e2449f5 --- /dev/null +++ b/vendor/zsks/demo/sdk_linux.mk @@ -0,0 +1,93 @@ +#==================================================== +# SDK libs +#==================================================== +MPI_LIBS += -L$(TARGET_DIR) +MPI_LIBS += -L$(SDK_LINUX_OUT_LIB) +MPI_LIBS += -L$(SYSROOT_PATH)/lib +#MPI_LIBS += -L$(SYSROOT_PATH)/usr/lib +#MPI_LIBS += -L$(OUT_UNSTRIPPED_LIB) +MPI_LIBS += -lot_osal -lot_base -lot_sys -lot_chnl -lot_rgn -lot_vpp -lot_vo -lot_vgs -lot_vpss -lsecurec +MPI_LIBS += -lot_vdec -lot_h265e -lot_h264e -lot_jpege -lot_venc -lot_vedu -lot_vi -lot_mpi_isp -lot_rc -lot_jpegd +MPI_LIBS += -lot_sensor_i2c -lot_sensor_spi -lot_aio -lot_ai -lot_ao -lot_aenc -lot_adec -lot_acodec -lot_irq +MPI_LIBS += -lss_mpi -lss_mau -lss_dpu_match -lss_ive -lss_dpu_rect -lss_pqp -lss_hdmi -lot_vfmw -lot_isp -lot_dis +MPI_LIBS += -lot_gdc -lss_ae -lss_isp -lot_isp -lss_awb -lss_dehaze -lss_extend_stats -lss_drc -lss_ldci -lss_crb -lss_bnr -lss_voice_engine -lss_upvqe -lss_dnvqe +MPI_LIBS += -lss_calcflicker -lss_ir_auto -lss_acs -lss_acs -lsns_hy_s0603 -laac_comm -laac_dec -laac_enc -laac_sbr_dec -laac_sbr_enc +#MPI_LIBS += -lsns_os08a20 -lsns_os05a10_2l_slave -lsns_imx347_slave -lsns_imx485 -lsns_os04a10 -lsns_os08b10 +MPI_LIBS += $(SDK_LINUX_OUT_LIB)/svp_npu/libsvp_acl.a +MPI_LIBS += -L$(SDK_LINUX_OUT_LIB)/svp_npu -lprotobuf-c + +# 设置浮点运算, cpu类型, 链接器的相关设置 +CFLAGS += -Dss928v100 -DOT_XXXX -lpthread -lm -ldl -DISP_V2 -lc++ -lc -Wall -fsigned-char -mcpu=cortex-a53 -ldl -ffunction-sections -fdata-sections -O2 -fstack-protector-strong -fPIC -D_FILE_OFFSET_BITS=64 -D_LARGEFILE_SOURCE -D_LARGEFILE64_SOURCE -fPIE -pie -s -Wall -fsigned-char -DOT_RELEASE -DVER_X=1 -DVER_Y=0 -DVER_Z=0 -DVER_P=0 -DVER_B=10 -DUSER_BIT_64 -DKERNEL_BIT_64 -Wno-date-time + +#==================================================== +# SDK include +#==================================================== +CFLAGS += -I$(SDK_LINUX_INCLUDE_DIR) +#CFLAGS += -I$(BOUNDS_CHECKING_DIR) +CFLAGS += -I$(SDK_SAMPLE_COMMON_DIR) +CFLAGS += -I$(SVP_COMMON_DIR) + +#==================================================== +# Source FileList +#==================================================== +SOURCE_TREE := $(ROOT_DIR) +BUILD_TREE := $(ROOT_DIR) + +#==================================================== +# Objects FileList +#==================================================== +SRC_ROOT := $(SOURCE_TREE) +OBJ_ROOT := $(OBJS_DIR) +OBJS := $(SMP_SRCS:$(SRC_ROOT)%.c=$(OBJ_ROOT)%.o) +OBJS := $(sort $(OBJS)) +OBJS_DIR := $(sort $(dir $(OBJS))) +CPP_OBJS := $(CPP_SRCS:$(SRC_ROOT)%.cpp=$(OBJ_ROOT)%.o) +OBJS_DIR += $(sort $(dir $(CPP_OBJS))) + +#==================================================== +# Create Directory +#==================================================== +CreateDir = $(shell [ -d $1 ] || mkdir -p $1 || echo ":mkdir '$1' fail") +CreateResult := +dummy := $(call CreateDir, $(OBJS_ROOT)) +dummy += $(foreach dir, $(OBJS_DIR), CreateResult += $(call CreateDir, $(dir))) +ifneq ($(strip CreateDir),) + err = $(error $(CreateResult)) +endif + +#==================================================== +# Compile rules +#==================================================== +MKDIRS := $(TARGET_DIR) + +.PHONY : clean all +all: $(TARGET) + +$(TARGET): $(OBJS) $(CPP_OBJS) | $(MKDIRS) + @echo ... ld $@ + @echo ... $(TARGET) + @$(CC) $(CFLAGS) $(FLAG_LD) -o $(TARGET_DIR)/$@ $^ -Wl,--start-group $(MPI_LIBS) -Wl,--end-group + +$(CPP_OBJS) : $(OBJ_ROOT)/%.o : $(SRC_ROOT)/%.cpp + @echo @@@ c++ $^ + $(CXXCPP) $(FLAG_LD) $(CFLAGS) -c "$<" -o "$@" -w + +$(OBJS) : $(OBJ_ROOT)/%.o : $(SRC_ROOT)/%.c + @echo @@@ cc $^ + @$(CC) $(FLAG_LD) $(CFLAGS) -c "$<" -o "$@" -w + +$(MKDIRS): + mkdir -p $@ + +echo =========== $(TARGET) build success ================= + +clean: + @rm -rf $(TARGET_DIR) + @rm -rf $(OBJ_ROOT) + +cleanstream: + @rm -f *.h264 + @rm -f *.h265 + @rm -f *.jpg + @rm -f *.mjp + @rm -f *.mp4 diff --git a/vendor/zsks/docs/alsa-lib/README.md b/vendor/zsks/docs/alsa-lib/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7840a0c6815f67a4d5e70ecf6cf6a7ee629c77eb --- /dev/null +++ b/vendor/zsks/docs/alsa-lib/README.md @@ -0,0 +1,230 @@ +# alsa-lib移植 + +* alsa-lib 是 Linux 系统中处理音频的核心组件。 + alsa-lib：音频功能的底层基础 + 它是 ALSA（Advanced Linux Sound Architecture，高级 Linux 声音架构）的核心库，主要作用是提供统一的音频硬件接口。 + 核心功能：封装了与声卡硬件交互的复杂逻辑，向上层应用程序提供标准化的 API（如 C 语言接口），让开发者无需直接操作硬件驱动，就能实现音频播放、录音等功能。 + 包含内容：主要生成动态链接库（如 libasound.so），是所有基于 ALSA 的程序的依赖基础；附带少量示例工具（如 aserver，一个简单的音频服务器 demo）。 + +## 1、软硬件环境 + +* 开发板：海鸥派 +* 交叉编译工具链：OHOS (dev) clang version 15.0.4 +* 编译链路径：pegasus/os/OpenHarmony/ohos/prebuilts/clang/ohos/linux-x86_64/llvm/bin +* python版本：Python-3.13.2 +* 移植的alsa-lib版本：alsa-lib-1.1.7 + +## 2、交叉编译alsa-lib + +### 步骤1：获取源码 + +* 在服务器的命令行，分步执行下面的命令，下载alsa-lib-1.1.7源码 + +~~~bash +# （可根据需要替换版本号） +wget https://www.alsa-project.org/files/pub/lib/alsa-lib-1.1.7.tar.bz2 + +# 解压源码包 +tar -xjvf alsa-lib-1.1.7.tar.bz2 +rm alsa-lib-1.1.7.tar.bz2 + +# 进入源码目录 +cd alsa-lib-1.1.7 +~~~ + +![image-20251017172451263](pic/image-20251017172451263.png) + +### 步骤2：配置环境变量 + +* 在服务器的命令行分步执行下面的命令，配置环境变量 + +~~~bash +export SYSROOT=/home/openharmony/pegasus/os/OpenHarmony/ohos/out/hispark_ss928v100/ipcamera_hispark_ss928v100_linux/sysroot +export CC=/home/openharmony/pegasus/os/OpenHarmony/ohos/prebuilts/clang/ohos/linux-x86_64/llvm/bin/aarch64-unknown-linux-ohos-clang +export CXX=/home/openharmony/pegasus/os/OpenHarmony/ohos/prebuilts/clang/ohos/linux-x86_64/llvm/bin/aarch64-unknown-linux-ohos-clang++ +export AR=/home/openharmony/pegasus/os/OpenHarmony/ohos/prebuilts/clang/ohos/linux-x86_64/llvm/bin/llvm-ar +export RANLIB=/home/openharmony/pegasus/os/OpenHarmony/ohos/prebuilts/clang/ohos/linux-x86_64/llvm/bin/llvm-ranlib +export CFLAGS="--sysroot=$SYSROOT" +export CXXFLAGS="--sysroot=$SYSROOT" +export LDFLAGS="--sysroot=$SYSROOT" +~~~ + +![image-20251017172745509](pic/image-20251017172745509.png) + +* 在服务器的命令执行下面的命令，更新config文件 + +```sh +#核心原因是config.sub和config.guess脚本版本过旧，先移除 +rm -rf config.sub config.guess + +# 下载 config.sub +wget -O config.sub https://git.savannah.gnu.org/cgit/config.git/plain/config.sub + +# 下载 config.guess +wget -O config.guess https://git.savannah.gnu.org/cgit/config.git/plain/config.guess + +#检查更新后是否有支持ohos +grep -i "ohos" config.sub +``` + +![image-20251017172824538](pic/image-20251017172824538.png) + +![image-20251017172849064](pic/image-20251017172849064.png) + +### 步骤3：配置命令 + +* 服务器的命令行执行下面的命令，进行meson的构建。 + +~~~bash +./configure \ + --host=aarch64-linux-ohos \ + --prefix=/home/openharmony/pegasus/vendor/opensource/alsa-lib-1.1.7/install \ + --enable-shared \ + --disable-static \ + --disable-python \ + --with-pic \ + --disable-alisp \ + --disable-old-symbols \ + CC=/home/openharmony/pegasus/os/OpenHarmony/ohos/prebuilts/clang/ohos/linux-x86_64/llvm/bin/aarch64-unknown-linux-ohos-clang +~~~ + +![image-20251017173155196](pic/image-20251017173155196.png) + +### 步骤4：编译安装 + +* 服务器的命令行执行下面的命令，进行alsa-lib的编译和安装 + +~~~bash +make -j$(nproc) 2>&1 | tee make_output.log +make install +~~~ + +![image-20250904165253761](./pic/image-20250904165253761.png) + +![image-20250904165432734](./pic/image-20250904165432734.png) + +* 编译成功后，会在install目录下生成下面的文件 + +![image-20251017173447490](pic/image-20251017173447490.png) + +## 3、板端测试 + +### 3.1 编译代码 + +* 将下面的代码复制到alsa_test.c中，在服务器上编译，得到可执行文件后放到板端执行。 + +~~~bash +#include +#include + +int main() { + int ret; + snd_pcm_t *handle; // PCM设备句柄 + snd_pcm_hw_params_t *params; // 硬件参数结构体 + + // 1. 打开PCM播放设备（默认设备） + ret = snd_pcm_open(&handle, "default", SND_PCM_STREAM_PLAYBACK, 0); + if (ret < 0) { + fprintf(stderr, "无法打开PCM设备: %s\n", snd_strerror(ret)); + return 1; + } + + // 2. 初始化硬件参数 + snd_pcm_hw_params_alloca(¶ms); + snd_pcm_hw_params_any(handle, params); + + // 设置参数： interleaved模式、采样格式（16位小端）、声道（双声道） + snd_pcm_hw_params_set_access(handle, params, SND_PCM_ACCESS_RW_INTERLEAVED); + snd_pcm_hw_params_set_format(handle, params, SND_PCM_FORMAT_S16_LE); + snd_pcm_hw_params_set_channels(handle, params, 2); + + // 设置采样率（44.1kHz） + unsigned int val = 44100; + snd_pcm_hw_params_set_rate_near(handle, params, &val, NULL); + + // 应用参数 + ret = snd_pcm_hw_params(handle, params); + if (ret < 0) { + fprintf(stderr, "无法设置PCM参数: %s\n", snd_strerror(ret)); + snd_pcm_close(handle); + return 1; + } + + // 3. 生成简单的正弦波音频数据（测试播放） + short buffer[1024]; + for (int i = 0; i < 1024; i++) { + buffer[i] = 32767 * sin(2 * 3.1415926 * 440 * i / 44100); // 440Hz正弦波 + } + + // 4. 播放音频（写入PCM设备） + ret = snd_pcm_writei(handle, buffer, 1024); + if (ret < 0) { + // 处理欠载错误（可尝试恢复） + ret = snd_pcm_recover(handle, ret, 0); + if (ret < 0) { + fprintf(stderr, "播放失败: %s\n", snd_strerror(ret)); + snd_pcm_close(handle); + return 1; + } + } + + // 5. 关闭设备 + snd_pcm_drain(handle); // 等待播放完成 + snd_pcm_close(handle); + printf("测试完成：正弦波播放成功\n"); + return 0; +} +~~~ + +* 在服务器的命令行执行下面的命令，编译该代码 +* 注意：这里的决定路径请根据自己服务器的实际情况进行填写 + +~~~bash +/home/openharmony/pegasus/os/OpenHarmony/ohos/prebuilts/clang/ohos/linux-x86_64/llvm/bin/aarch64-unknown-linux-ohos-clang alsa_test.c -o alsa_test -lasound \ +--sysroot=/home/openharmony/pegasus/os/OpenHarmony/ohos/out/hispark_ss928v100/ipcamera_hispark_ss928v100_linux/sysroot \ +-L /home/openharmony/pegasus/os/OpenHarmony/ohos/out/hispark_ss928v100/ipcamera_hispark_ss928v100_linux/sysroot/usr/lib/aarch64-linux-ohos \ + -lc \ +-I/home/openharmony/pegasus/vendor/opensource/alsa-lib-1.1.7/install/include \ +-L/home/openharmony/pegasus/vendor/opensource/alsa-lib-1.1.7/install/lib +~~~ + +![image-20251017174647517](pic/image-20251017174647517.png) + +### 3.2 NFS 共享 + +* nfs配置可参考参考链接： + +~~~bash +https://blog.csdn.net/weixin_34326429/article/details/92163791 +~~~ + +### 3.3 挂载NFS + +* 将第2章步骤4生成的install文件下载到本地（我这里是把install重命名为alsa_install了），然后拷贝到NFS的目录下 +* 再将3.1节编译生成的alsa_test下载到本地，然后拷贝到NFS的目录下 + +* 在开发板的命令行终端执行下面的命令，配置IP地址，并进行NFS挂载 + +~~~bash +ifconfig eth0 192.168.137.0 netmask 255.255.252.0 +route add default gw 192.168.137.1 +mount -o nolock,addr=192.168.137.1 -t nfs 192.168.137.1:/nfs /mnt +~~~ + +### 3.4 配置环境变量 + +* 在开发板的命令行执行下面的命令，配置alsa的环境变量 + +~~~bash +export PATH=/mnt/alsa_install/bin:$PATH +export LD_LIBRARY_PATH=/mnt/alsa_install/lib:$LD_LIBRARY_PATH +export ALSA_CONFIG_PATH=/mnt/alsa_install/share/alsa/alsa.conf:/mnt/alsa_install/share/alsa/cards/aliases.conf:/mnt/alsa_install/share/alsa/pcm/default.conf +~~~ + +![image-20251020152649127](pic/image-20251020152649127.png) + +~~~bash +chmod +x ./alsa_test +./alsa_test +~~~ + diff --git a/vendor/zsks/docs/alsa-lib/pic/image-20250903211122837.png b/vendor/zsks/docs/alsa-lib/pic/image-20250903211122837.png new file mode 100644 index 0000000000000000000000000000000000000000..e5e1a3283ff757d2867062c934003d84257ee3c8 Binary files /dev/null and b/vendor/zsks/docs/alsa-lib/pic/image-20250903211122837.png differ diff --git a/vendor/zsks/docs/alsa-lib/pic/image-20250904100830962.png b/vendor/zsks/docs/alsa-lib/pic/image-20250904100830962.png new file mode 100644 index 0000000000000000000000000000000000000000..5a60335f31e8348429e2fde87dca3662882414f7 Binary files /dev/null and b/vendor/zsks/docs/alsa-lib/pic/image-20250904100830962.png differ diff --git a/vendor/zsks/docs/alsa-lib/pic/image-20250904103350656.png b/vendor/zsks/docs/alsa-lib/pic/image-20250904103350656.png new file mode 100644 index 0000000000000000000000000000000000000000..9e06f2bd6467f730735c161a23cf81f07f1b8bfd Binary files /dev/null and b/vendor/zsks/docs/alsa-lib/pic/image-20250904103350656.png differ diff --git a/vendor/zsks/docs/alsa-lib/pic/image-20250904103533301.png b/vendor/zsks/docs/alsa-lib/pic/image-20250904103533301.png new file mode 100644 index 0000000000000000000000000000000000000000..b9c7489569f82a76e5c0ab2646361963d25fe950 Binary files /dev/null and b/vendor/zsks/docs/alsa-lib/pic/image-20250904103533301.png differ diff --git a/vendor/zsks/docs/alsa-lib/pic/image-20250904103636103.png b/vendor/zsks/docs/alsa-lib/pic/image-20250904103636103.png new file mode 100644 index 0000000000000000000000000000000000000000..6b61fd5a609a4bdaf278c963de73d64f2b3abe05 Binary files /dev/null and b/vendor/zsks/docs/alsa-lib/pic/image-20250904103636103.png differ diff --git a/vendor/zsks/docs/alsa-lib/pic/image-20250904103828253.png b/vendor/zsks/docs/alsa-lib/pic/image-20250904103828253.png new file mode 100644 index 0000000000000000000000000000000000000000..d1b84e9ea4a93c6639df14a197a2a6ed9ee0e775 Binary files /dev/null and b/vendor/zsks/docs/alsa-lib/pic/image-20250904103828253.png differ diff --git a/vendor/zsks/docs/alsa-lib/pic/image-20250904104156703.png b/vendor/zsks/docs/alsa-lib/pic/image-20250904104156703.png new file mode 100644 index 0000000000000000000000000000000000000000..198beecc6217c854f1c072651c5ccc6dd97720ee Binary files /dev/null and b/vendor/zsks/docs/alsa-lib/pic/image-20250904104156703.png differ diff --git a/vendor/zsks/docs/alsa-lib/pic/image-20250904113014566.png b/vendor/zsks/docs/alsa-lib/pic/image-20250904113014566.png new file mode 100644 index 0000000000000000000000000000000000000000..97d90f6510fdc89d6c4473ffecabe0caac74e0af Binary files /dev/null and b/vendor/zsks/docs/alsa-lib/pic/image-20250904113014566.png differ diff --git a/vendor/zsks/docs/alsa-lib/pic/image-20250904113329622.png b/vendor/zsks/docs/alsa-lib/pic/image-20250904113329622.png new file mode 100644 index 0000000000000000000000000000000000000000..494b08517939bb52dba3dddf4201b82e8a08acb6 Binary files /dev/null and b/vendor/zsks/docs/alsa-lib/pic/image-20250904113329622.png differ diff --git a/vendor/zsks/docs/alsa-lib/pic/image-20250904113515712.png b/vendor/zsks/docs/alsa-lib/pic/image-20250904113515712.png new file mode 100644 index 0000000000000000000000000000000000000000..6290f925c8bc9d0f7017d26320960e1820140189 Binary files /dev/null and b/vendor/zsks/docs/alsa-lib/pic/image-20250904113515712.png differ diff --git a/vendor/zsks/docs/alsa-lib/pic/image-20250904113639259.png b/vendor/zsks/docs/alsa-lib/pic/image-20250904113639259.png new file mode 100644 index 0000000000000000000000000000000000000000..827c90050c198fa02a3313360dca48be91edcbb6 Binary files /dev/null and b/vendor/zsks/docs/alsa-lib/pic/image-20250904113639259.png differ diff --git a/vendor/zsks/docs/alsa-lib/pic/image-20250904113704266.png b/vendor/zsks/docs/alsa-lib/pic/image-20250904113704266.png new file mode 100644 index 0000000000000000000000000000000000000000..cdbcf6e171b10ae0713f0217ce1c764b6e9c564a Binary files /dev/null and b/vendor/zsks/docs/alsa-lib/pic/image-20250904113704266.png differ diff --git a/vendor/zsks/docs/alsa-lib/pic/image-20250904143920431.png b/vendor/zsks/docs/alsa-lib/pic/image-20250904143920431.png new file mode 100644 index 0000000000000000000000000000000000000000..bbba9d4935ea43429a600ec64dfe14119f90a735 Binary files /dev/null and b/vendor/zsks/docs/alsa-lib/pic/image-20250904143920431.png differ diff --git a/vendor/zsks/docs/alsa-lib/pic/image-20250904144609428.png b/vendor/zsks/docs/alsa-lib/pic/image-20250904144609428.png new file mode 100644 index 0000000000000000000000000000000000000000..ff29749d89fab1166b46eee624e72297948d1b94 Binary files /dev/null and b/vendor/zsks/docs/alsa-lib/pic/image-20250904144609428.png differ diff --git a/vendor/zsks/docs/alsa-lib/pic/image-20250904144720470.png b/vendor/zsks/docs/alsa-lib/pic/image-20250904144720470.png new file mode 100644 index 0000000000000000000000000000000000000000..e0529247489c0470c3d147c268bb5d97e59565f1 Binary files /dev/null and b/vendor/zsks/docs/alsa-lib/pic/image-20250904144720470.png differ diff --git a/vendor/zsks/docs/alsa-lib/pic/image-20250904144926010.png b/vendor/zsks/docs/alsa-lib/pic/image-20250904144926010.png new file mode 100644 index 0000000000000000000000000000000000000000..8d0b234ad3f45ea454352a050b887c07d72e66b4 Binary files /dev/null and b/vendor/zsks/docs/alsa-lib/pic/image-20250904144926010.png differ diff --git a/vendor/zsks/docs/alsa-lib/pic/image-20250904145236942.png b/vendor/zsks/docs/alsa-lib/pic/image-20250904145236942.png new file mode 100644 index 0000000000000000000000000000000000000000..5a60335f31e8348429e2fde87dca3662882414f7 Binary files /dev/null and b/vendor/zsks/docs/alsa-lib/pic/image-20250904145236942.png differ diff --git a/vendor/zsks/docs/alsa-lib/pic/image-20250904145701210.png b/vendor/zsks/docs/alsa-lib/pic/image-20250904145701210.png new file mode 100644 index 0000000000000000000000000000000000000000..9e06f2bd6467f730735c161a23cf81f07f1b8bfd Binary files /dev/null and b/vendor/zsks/docs/alsa-lib/pic/image-20250904145701210.png differ diff --git a/vendor/zsks/docs/alsa-lib/pic/image-20250904145948379.png b/vendor/zsks/docs/alsa-lib/pic/image-20250904145948379.png new file mode 100644 index 0000000000000000000000000000000000000000..6b61fd5a609a4bdaf278c963de73d64f2b3abe05 Binary files /dev/null and b/vendor/zsks/docs/alsa-lib/pic/image-20250904145948379.png differ diff --git a/vendor/zsks/docs/alsa-lib/pic/image-20250904151336527.png b/vendor/zsks/docs/alsa-lib/pic/image-20250904151336527.png new file mode 100644 index 0000000000000000000000000000000000000000..795733fffe8f88aa19965ae0ab26d8c6c9663b22 Binary files /dev/null and b/vendor/zsks/docs/alsa-lib/pic/image-20250904151336527.png differ diff --git a/vendor/zsks/docs/alsa-lib/pic/image-20250904152000503.png b/vendor/zsks/docs/alsa-lib/pic/image-20250904152000503.png new file mode 100644 index 0000000000000000000000000000000000000000..a1fc3703b4a3bb495da282853fe62b782df1b6aa Binary files /dev/null and b/vendor/zsks/docs/alsa-lib/pic/image-20250904152000503.png differ diff --git a/vendor/zsks/docs/alsa-lib/pic/image-20250904165253761.png b/vendor/zsks/docs/alsa-lib/pic/image-20250904165253761.png new file mode 100644 index 0000000000000000000000000000000000000000..718827a13a2417025076b6dde68b08786980189b Binary files /dev/null and b/vendor/zsks/docs/alsa-lib/pic/image-20250904165253761.png differ diff --git a/vendor/zsks/docs/alsa-lib/pic/image-20250904165432734.png b/vendor/zsks/docs/alsa-lib/pic/image-20250904165432734.png new file mode 100644 index 0000000000000000000000000000000000000000..473263f18da1570a63375b6fc39227f9694a55b5 Binary files /dev/null and b/vendor/zsks/docs/alsa-lib/pic/image-20250904165432734.png differ diff --git a/vendor/zsks/docs/alsa-lib/pic/image-20250904173140314.png b/vendor/zsks/docs/alsa-lib/pic/image-20250904173140314.png new file mode 100644 index 0000000000000000000000000000000000000000..dee25e389a1a55565647ec4983c05599faf8229f Binary files /dev/null and b/vendor/zsks/docs/alsa-lib/pic/image-20250904173140314.png differ diff --git a/vendor/zsks/docs/alsa-lib/pic/image-20250904173633025.png b/vendor/zsks/docs/alsa-lib/pic/image-20250904173633025.png new file mode 100644 index 0000000000000000000000000000000000000000..b029515ebfbf810dec8116658a8c796d249dc6b6 Binary files /dev/null and b/vendor/zsks/docs/alsa-lib/pic/image-20250904173633025.png differ diff --git a/vendor/zsks/docs/alsa-lib/pic/image-20250904173809627.png b/vendor/zsks/docs/alsa-lib/pic/image-20250904173809627.png new file mode 100644 index 0000000000000000000000000000000000000000..6fa1f8881872628428ef718a10c3b5f871643b50 Binary files /dev/null and b/vendor/zsks/docs/alsa-lib/pic/image-20250904173809627.png differ diff --git a/vendor/zsks/docs/alsa-lib/pic/image-20250904173927739.png b/vendor/zsks/docs/alsa-lib/pic/image-20250904173927739.png new file mode 100644 index 0000000000000000000000000000000000000000..d8d4b29e3f02ae342edcd8b7d344ba1ba3733970 Binary files /dev/null and b/vendor/zsks/docs/alsa-lib/pic/image-20250904173927739.png differ diff --git a/vendor/zsks/docs/alsa-lib/pic/image-20250904174022615.png b/vendor/zsks/docs/alsa-lib/pic/image-20250904174022615.png new file mode 100644 index 0000000000000000000000000000000000000000..b707369cd7400cbb1ad8b4d2f61e05c05bd51afc Binary files /dev/null and b/vendor/zsks/docs/alsa-lib/pic/image-20250904174022615.png differ diff --git a/vendor/zsks/docs/alsa-lib/pic/image-20250904182859475.png b/vendor/zsks/docs/alsa-lib/pic/image-20250904182859475.png new file mode 100644 index 0000000000000000000000000000000000000000..9d58e6507f989dfac64d897d23362133d9056256 Binary files /dev/null and b/vendor/zsks/docs/alsa-lib/pic/image-20250904182859475.png differ diff --git a/vendor/zsks/docs/alsa-lib/pic/image-20250904191234866.png b/vendor/zsks/docs/alsa-lib/pic/image-20250904191234866.png new file mode 100644 index 0000000000000000000000000000000000000000..a4153ee3ac64fdb4d38326b7f589db62f6934321 Binary files /dev/null and b/vendor/zsks/docs/alsa-lib/pic/image-20250904191234866.png differ diff --git a/vendor/zsks/docs/alsa-lib/pic/image-20250904191947983.png b/vendor/zsks/docs/alsa-lib/pic/image-20250904191947983.png new file mode 100644 index 0000000000000000000000000000000000000000..aa149d5994b3281249c6b942b7478700c6e9bf42 Binary files /dev/null and b/vendor/zsks/docs/alsa-lib/pic/image-20250904191947983.png differ diff --git a/vendor/zsks/docs/alsa-lib/pic/image-20250904192346977.png b/vendor/zsks/docs/alsa-lib/pic/image-20250904192346977.png new file mode 100644 index 0000000000000000000000000000000000000000..38c0de5552375ef5db7e6abe9bc602cd85c028e1 Binary files /dev/null and b/vendor/zsks/docs/alsa-lib/pic/image-20250904192346977.png differ diff --git a/vendor/zsks/docs/alsa-lib/pic/image-20250904192419116.png b/vendor/zsks/docs/alsa-lib/pic/image-20250904192419116.png new file mode 100644 index 0000000000000000000000000000000000000000..6c523b6c5557261677700db84f7a7c7f443602d7 Binary files /dev/null and b/vendor/zsks/docs/alsa-lib/pic/image-20250904192419116.png differ diff --git a/vendor/zsks/docs/alsa-lib/pic/image-20250904193742624.png b/vendor/zsks/docs/alsa-lib/pic/image-20250904193742624.png new file mode 100644 index 0000000000000000000000000000000000000000..438d3875b0f9134152b8294c297c97b7ae5b630b Binary files /dev/null and b/vendor/zsks/docs/alsa-lib/pic/image-20250904193742624.png differ diff --git a/vendor/zsks/docs/alsa-lib/pic/image-20250905094629530.png b/vendor/zsks/docs/alsa-lib/pic/image-20250905094629530.png new file mode 100644 index 0000000000000000000000000000000000000000..5566985c274a58203ebf31aba03ade23957e2292 Binary files /dev/null and b/vendor/zsks/docs/alsa-lib/pic/image-20250905094629530.png differ diff --git a/vendor/zsks/docs/alsa-lib/pic/image-20250905101539245.png b/vendor/zsks/docs/alsa-lib/pic/image-20250905101539245.png new file mode 100644 index 0000000000000000000000000000000000000000..619c068e1d4315c52831997b9dbbe16463e56888 Binary files /dev/null and b/vendor/zsks/docs/alsa-lib/pic/image-20250905101539245.png differ diff --git a/vendor/zsks/docs/alsa-lib/pic/image-20250905101612375.png b/vendor/zsks/docs/alsa-lib/pic/image-20250905101612375.png new file mode 100644 index 0000000000000000000000000000000000000000..d8695cec0605742808d81edae00f728db220399c Binary files /dev/null and b/vendor/zsks/docs/alsa-lib/pic/image-20250905101612375.png differ diff --git a/vendor/zsks/docs/alsa-lib/pic/image-20250905101636280.png b/vendor/zsks/docs/alsa-lib/pic/image-20250905101636280.png new file mode 100644 index 0000000000000000000000000000000000000000..d885425e40dc8672fba8063d340808e04954f4cf Binary files /dev/null and b/vendor/zsks/docs/alsa-lib/pic/image-20250905101636280.png differ diff --git a/vendor/zsks/docs/alsa-lib/pic/image-20250905101752805.png b/vendor/zsks/docs/alsa-lib/pic/image-20250905101752805.png new file mode 100644 index 0000000000000000000000000000000000000000..2da44b646ed833201d91808f70a6dc23c5ec59fe Binary files /dev/null and b/vendor/zsks/docs/alsa-lib/pic/image-20250905101752805.png differ diff --git a/vendor/zsks/docs/alsa-lib/pic/image-20250905103524391.png b/vendor/zsks/docs/alsa-lib/pic/image-20250905103524391.png new file mode 100644 index 0000000000000000000000000000000000000000..dbf4aab601266fb38839d78bda2923857d6793dc Binary files /dev/null and b/vendor/zsks/docs/alsa-lib/pic/image-20250905103524391.png differ diff --git a/vendor/zsks/docs/alsa-lib/pic/image-20250905110002197.png b/vendor/zsks/docs/alsa-lib/pic/image-20250905110002197.png new file mode 100644 index 0000000000000000000000000000000000000000..506ffb6f4c10de7bc5c4437a236a265603656d61 Binary files /dev/null and b/vendor/zsks/docs/alsa-lib/pic/image-20250905110002197.png differ diff --git a/vendor/zsks/docs/alsa-lib/pic/image-20250908145127235.png b/vendor/zsks/docs/alsa-lib/pic/image-20250908145127235.png new file mode 100644 index 0000000000000000000000000000000000000000..930e0d2bc8d9283c755c45cf2b710ee05fdac5ca Binary files /dev/null and b/vendor/zsks/docs/alsa-lib/pic/image-20250908145127235.png differ diff --git a/vendor/zsks/docs/alsa-lib/pic/image-20250908152550633.png b/vendor/zsks/docs/alsa-lib/pic/image-20250908152550633.png new file mode 100644 index 0000000000000000000000000000000000000000..0782a0e2326097075d1a1effa4c4ebc9e1b9e78b Binary files /dev/null and b/vendor/zsks/docs/alsa-lib/pic/image-20250908152550633.png differ diff --git a/vendor/zsks/docs/alsa-lib/pic/image-20251017172451263.png b/vendor/zsks/docs/alsa-lib/pic/image-20251017172451263.png new file mode 100755 index 0000000000000000000000000000000000000000..13321c0c58829114a99aa63f3290832785c8730b Binary files /dev/null and b/vendor/zsks/docs/alsa-lib/pic/image-20251017172451263.png differ diff --git a/vendor/zsks/docs/alsa-lib/pic/image-20251017172745509.png b/vendor/zsks/docs/alsa-lib/pic/image-20251017172745509.png new file mode 100755 index 0000000000000000000000000000000000000000..acc80e4256784721b3a3ef3eb5e1c5882c78e467 Binary files /dev/null and b/vendor/zsks/docs/alsa-lib/pic/image-20251017172745509.png differ diff --git a/vendor/zsks/docs/alsa-lib/pic/image-20251017172824538.png b/vendor/zsks/docs/alsa-lib/pic/image-20251017172824538.png new file mode 100755 index 0000000000000000000000000000000000000000..f2ead8650a5a7d5e708d246186e3006d0f9709ed Binary files /dev/null and b/vendor/zsks/docs/alsa-lib/pic/image-20251017172824538.png differ diff --git a/vendor/zsks/docs/alsa-lib/pic/image-20251017172849064.png b/vendor/zsks/docs/alsa-lib/pic/image-20251017172849064.png new file mode 100755 index 0000000000000000000000000000000000000000..0575ccd1094df9f9311acbe384cbfc11aaa23b42 Binary files /dev/null and b/vendor/zsks/docs/alsa-lib/pic/image-20251017172849064.png differ diff --git a/vendor/zsks/docs/alsa-lib/pic/image-20251017173155196.png b/vendor/zsks/docs/alsa-lib/pic/image-20251017173155196.png new file mode 100755 index 0000000000000000000000000000000000000000..a57f1261c2e07d10c7e4f4d8f6d04162b3481206 Binary files /dev/null and b/vendor/zsks/docs/alsa-lib/pic/image-20251017173155196.png differ diff --git a/vendor/zsks/docs/alsa-lib/pic/image-20251017173447490.png b/vendor/zsks/docs/alsa-lib/pic/image-20251017173447490.png new file mode 100755 index 0000000000000000000000000000000000000000..8582e902cd1b387215cd094f3235b51be855c76c Binary files /dev/null and b/vendor/zsks/docs/alsa-lib/pic/image-20251017173447490.png differ diff --git a/vendor/zsks/docs/alsa-lib/pic/image-20251017174647517.png b/vendor/zsks/docs/alsa-lib/pic/image-20251017174647517.png new file mode 100755 index 0000000000000000000000000000000000000000..595ba99c60230d2228205ad813e050e096b9cb82 Binary files /dev/null and b/vendor/zsks/docs/alsa-lib/pic/image-20251017174647517.png differ diff --git a/vendor/zsks/docs/alsa-lib/pic/image-20251020152649127.png b/vendor/zsks/docs/alsa-lib/pic/image-20251020152649127.png new file mode 100755 index 0000000000000000000000000000000000000000..e25c700619eb17689e61258daccb752035464ccd Binary files /dev/null and b/vendor/zsks/docs/alsa-lib/pic/image-20251020152649127.png differ diff --git "a/vendor/zsks/docs/ffmpeg/FFmpeg_CPU\344\270\216\347\241\254\344\273\266\347\274\226\350\247\243\347\240\201\351\200\237\345\272\246\345\257\271\346\257\224.xlsx" "b/vendor/zsks/docs/ffmpeg/FFmpeg_CPU\344\270\216\347\241\254\344\273\266\347\274\226\350\247\243\347\240\201\351\200\237\345\272\246\345\257\271\346\257\224.xlsx" new file mode 100644 index 0000000000000000000000000000000000000000..86f42c39ddfb1fc9ef7241519e128f101f18c777 Binary files /dev/null and "b/vendor/zsks/docs/ffmpeg/FFmpeg_CPU\344\270\216\347\241\254\344\273\266\347\274\226\350\247\243\347\240\201\351\200\237\345\272\246\345\257\271\346\257\224.xlsx" differ diff --git a/vendor/zsks/docs/ffmpeg/README.md b/vendor/zsks/docs/ffmpeg/README.md new file mode 100644 index 0000000000000000000000000000000000000000..60466288d963aab62b9c9651f8744accbc1f1bf9 --- /dev/null +++ b/vendor/zsks/docs/ffmpeg/README.md @@ -0,0 +1,721 @@ +# FFmpeg移植 + +## 1、软硬件环境 + +* 开发板：海鸥派 +* 交叉编译工具链：OHOS (dev) clang version 15.0.4 + +* 编译链路径：pegasus/os/OpenHarmony/ohos/prebuilts/clang/ohos/linux-x86_64/llvm/bin +* Python版本：Python-3.13.2 +* 移植的ffmpeg版本：FFmpeg-6.0 + +## 2、安装依赖 + +* 由于在编译ffmpeg的时候，需要依赖其他第三方软件，因此在编译ffmpeg之前，我们先把它依赖的第三方软件全部交叉编译出来。 + +### 步骤1：交叉编译v4l2 + +* 可以参考[v4l2的移植文档](../libv4l2/README.md)的前两章节，完成v4l2的移植工作。 + +### 步骤2：交叉编译x264 + +* 在服务器的命令行执行下面的命令，下载源码，配置编译链 + +```sh +cd pegasus/vendor/opensource/ + +git clone https://code.videolan.org/videolan/x264.git + +cd x264 +``` + +* 修改config.sub文件，在125行添加 linux-ohos* | + +![image-20251021144741884](pic/image-20251021144741884.png) + +* 执行下面的命令，配置编译环境，对代码进行编译 + +```sh +CC="/home/openharmony/pegasus/os/OpenHarmony/ohos/prebuilts/clang/ohos/linux-x86_64/llvm/bin/aarch64-unknown-linux-ohos-clang \ + --sysroot=/home/openharmony/pegasus/os/OpenHarmony/ohos/out/hispark_ss928v100/ipcamera_hispark_ss928v100_linux/sysroot" \ +CXX="/home/openharmony/pegasus/os/OpenHarmony/ohos/prebuilts/clang/ohos/linux-x86_64/llvm/bin/aarch64-unknown-linux-ohos-clang++ \ + --sysroot=/home/openharmony/pegasus/os/OpenHarmony/ohos/out/hispark_ss928v100/ipcamera_hispark_ss928v100_linux/sysroot" \ +CFLAGS="-march=armv8-a -mfpu=neon" \ +./configure \ +--host=aarch64-linux-ohos \ +--prefix=$(pwd)/install \ +--enable-shared + +make -j$(nproc) && make install +``` + +![image-20251021150217681](pic/image-20251021150217681.png) + +* 编译成功后会在install目录下生成如下文件 + +![image-20251021154212147](pic/image-20251021154212147.png) + +### 步骤3：交叉编译x265 + +* 在服务器的命令行执行下面的命令，下载源码 + +```sh +cd ../ + +wget https://bitbucket.org/multicoreware/x265_git/downloads/x265_3.5.tar.gz + +tar xf x265_3.5.tar.gz + +rm x265_3.5.tar.gz + +cd x265_3.5 +``` + +![image-20251021154533551](pic/image-20251021154533551.png) + +![image-20251021154614257](pic/image-20251021154614257.png) + +* 将下面内容替换到x265_3.5/build/aarch64-linux/crosscompile.cmake文件中 +* 注意：里面的库、头文件等的绝对路径请根据自己服务器的实际情况进行修改。 + +```sh +# crosscompile.cmake for cross compiling x265 for aarch64 with OHOS toolchain +# This feature is only supported as experimental. Use with caution. +# Please report bugs on bitbucket +# Run cmake with: cmake -DCMAKE_TOOLCHAIN_FILE=crosscompile.cmake -G "Unix Makefiles" ../../source && ccmake ../../source + +# Enable ARM cross-compilation +set(CROSS_COMPILE_ARM 1) + +# Specify the target system +set(CMAKE_SYSTEM_NAME Linux) +set(CMAKE_SYSTEM_PROCESSOR aarch64) + +# Specify the OHOS target (emulate config.sub behavior) +set(CMAKE_SYSTEM_VERSION ohos) + +# Specify the cross compiler +set(CMAKE_C_COMPILER /home/openharmony/pegasus/os/OpenHarmony/ohos/prebuilts/clang/ohos/linux-x86_64/llvm/bin/aarch64-unknown-linux-ohos-clang) +set(CMAKE_CXX_COMPILER /home/openharmony/pegasus/os/OpenHarmony/ohos/prebuilts/clang/ohos/linux-x86_64/llvm/bin/aarch64-unknown-linux-ohos-clang++) +set(CMAKE_ASM_COMPILER /home/openharmony/pegasus/os/OpenHarmony/ohos/prebuilts/clang/ohos/linux-x86_64/llvm/bin/aarch64-unknown-linux-ohos-clang) +set(CMAKE_AR /home/openharmony/pegasus/os/OpenHarmony/ohos/prebuilts/clang/ohos/linux-x86_64/llvm/bin/llvm-ar) +set(CMAKE_LINKER /home/openharmony/pegasus/os/OpenHarmony/ohos/prebuilts/clang/ohos/linux-x86_64/llvm/bin/lld) +set(CMAKE_RANLIB /home/openharmony/pegasus/os/OpenHarmony/ohos/prebuilts/clang/ohos/linux-x86_64/llvm/bin/llvm-ranlib) +set(CMAKE_STRIP /home/openharmony/pegasus/os/OpenHarmony/ohos/prebuilts/clang/ohos/linux-x86_64/llvm/bin/llvm-strip) + +# Specify the target environment (sysroot) +set(CMAKE_SYSROOT /home/openharmony/pegasus/os/OpenHarmony/ohos/out/hispark_ss928v100/ipcamera_hispark_ss928v100_linux/sysroot) + +# Compiler and linker flags +set(CMAKE_C_FLAGS "-fPIC -target aarch64-unknown-linux-ohos" CACHE STRING "C flags") +set(CMAKE_CXX_FLAGS "-fPIC -target aarch64-unknown-linux-ohos" CACHE STRING "C++ flags") +set(CMAKE_ASM_FLAGS "-fPIC -target aarch64-unknown-linux-ohos" CACHE STRING "ASM flags") +set(CMAKE_EXE_LINKER_FLAGS "-fPIC" CACHE STRING "Linker flags") + +# Include and library paths for Hisi SDK and dependencies +set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -I/home/openharmony/pegasus/vendor/opensource/x264/install/include" CACHE STRING "C flags") +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -I/home/openharmony/pegasus/vendor/opensource/x264/install/include" CACHE STRING "C++ flags") +set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -L/home/openharmony/pegasus/vendor/opensource/x264/install/lib" CACHE STRING "Linker flags") + +# Search paths for libraries and headers +set(CMAKE_FIND_ROOT_PATH /home/openharmony/pegasus/os/OpenHarmony/ohos/out/hispark_ss928v100/ipcamera_hispark_ss928v100_linux/sysroot /home/openharmony/pegasus/vendor/opensource/x264/install /home/openharmony/pegasus/vendor/opensource/v4l-utils/install) +set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER) +set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY) +set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY) + +# PKG_CONFIG_PATH for finding dependencies +set(ENV{PKG_CONFIG_PATH} "/home/openharmony/pegasus/vendor/opensource/v4l-utils/install/lib/pkgconfig:/home/openharmony/pegasus/vendor/opensource/v4l-utils/install/lib/pkgconfig") + +# Debug output to verify configuration +message(STATUS "CMAKE_C_COMPILER: ${CMAKE_C_COMPILER}") +message(STATUS "CMAKE_CXX_COMPILER: ${CMAKE_CXX_COMPILER}") +message(STATUS "CMAKE_SYSROOT: ${CMAKE_SYSROOT}") +message(STATUS "CMAKE_FIND_ROOT_PATH: ${CMAKE_FIND_ROOT_PATH}") +message(STATUS "PKG_CONFIG_PATH: $ENV{PKG_CONFIG_PATH}") +``` + +* 在服务器的命令行执行下面的命令，配置编译选项 + +```sh +cd /home/openharmony/pegasus/vendor/opensource/x265_3.5 + +cmake ./source \ + -G "Unix Makefiles" \ + -DCMAKE_SYSTEM_NAME=Linux \ + -DCMAKE_SYSTEM_PROCESSOR=aarch64 \ + -DCMAKE_SYSTEM_VERSION=ohos \ + -DCMAKE_FIND_ROOT_PATH_MODE_PROGRAM=NEVER \ + -DCMAKE_FIND_ROOT_PATH_MODE_LIBRARY=ONLY \ + -DCMAKE_FIND_ROOT_PATH_MODE_INCLUDE=ONLY \ + -DCMAKE_INSTALL_PREFIX="$(pwd)/install" \ + -DENABLE_SHARED=ON \ + -DENABLE_PIC=ON \ + -DHIGH_BIT_DEPTH=ON \ + -DEXPORT_C_API=ON \ + -DENABLE_ASSEMBLY=OFF \ + -Wno-dev + +make -j$(nproc)&&make install +``` + +![image-20251021161044799](pic/image-20251021161044799.png) + +![image-20251021161134241](pic/image-20251021161134241.png) + +* 编译完成后，会在install目录下生成如下文件 + +![image-20251021161216433](pic/image-20251021161216433.png) + +## 3、交叉编译FFmpeg + +### 步骤1：下载源码 + +* 在服务器的命令行执行下面的命令，下载ffmpeg源码 + +```sh +cd ../ + +git clone -b release/6.0 https://gitee.com/zhongshankuangshi/ffmpeg.git + +cd ffmpeg +``` + +![image-20251021205649485](pic/image-20251021205649485.png) + +### 步骤2：环境变量配置 + +* 注意这里出现的决定路径的地址，请根据自己服务器的具体情况进行修改。 + +```sh +export CC=/home/openharmony/pegasus/os/OpenHarmony/ohos/prebuilts/clang/ohos/linux-x86_64/llvm/bin/aarch64-unknown-linux-ohos-clang +export CXX=/home/openharmony/pegasus/os/OpenHarmony/ohos/prebuilts/clang/ohos/linux-x86_64/llvm/bin/aarch64-unknown-linux-ohos-clang++ +export AR=/home/openharmony/pegasus/os/OpenHarmony/ohos/prebuilts/clang/ohos/linux-x86_64/llvm/bin/llvm-ar +export LD=/home/openharmony/pegasus/os/OpenHarmony/ohos/prebuilts/clang/ohos/linux-x86_64/llvm/bin/lld +export RANLIB=/home/openharmony/pegasus/os/OpenHarmony/ohos/prebuilts/clang/ohos/linux-x86_64/llvm/bin/llvm-ranlib +export STRIP=/home/openharmony/pegasus/os/OpenHarmony/ohos/prebuilts/clang/ohos/linux-x86_64/llvm/bin/llvm-strip + +export PKG_CONFIG_PATH="/home/openharmony/pegasus/vendor/opensource/x264/install/lib/pkgconfig:$PKG_CONFIG_PATH" +export CXXFLAGS="-I/home/openharmony/pegasus/vendor/opensource/x264/install/include $CXXFLAGS" +export LDFLAGS="-L/home/openharmony/pegasus/vendor/opensource/x264/install/lib $LDFLAGS" + +export PKG_CONFIG_PATH="/home/openharmony/pegasus/vendor/opensource/x265_3.5/install/lib/pkgconfig:$PKG_CONFIG_PATH" +export CXXFLAGS="-I/home/openharmony/pegasus/vendor/opensource/x265_3.5/install/include $CXXFLAGS" +export LDFLAGS="-L/home/openharmony/pegasus/vendor/opensource/x265_3.5/install/lib $LDFLAGS" + +export PKG_CONFIG_PATH="/home/openharmony/pegasus/vendor/opensource/v4l-utils/install/lib/pkgconfig:$PKG_CONFIG_PATH" +export CXXFLAGS="-I/home/openharmony/pegasus/vendor/opensource/v4l-utils/install/include $CXXFLAGS" +export LDFLAGS="-L/home/openharmony/pegasus/vendor/opensource/v4l-utils/install/lib $LDFLAGS" +``` + +![image-20251021205900311](pic/image-20251021205900311.png) + +### 步骤3：修改相关编译脚本 + +* 在pegasus/platform/ss928v100_clang/smp/a55_linux/mpp/sample/common/makefile的第4行，添加一个fPIC选项到CFLAGS，如下图所示： + +![image-20251022142837408](pic/image-20251022142837408.png) + +* 然后进入到pegasus/platform/ss928v100_clang/smp/a55_linux/mpp/sample/common目录下，执行 make clean && make 命令，重新生成.o文件 +* 注意：请根据自己服务器的实际情况修改里面的绝对路径 + +```sh +export PATH=/home/openharmony/pegasus/os/OpenHarmony/ohos/prebuilts/clang/ohos/linux-x86_64/llvm/bin:$PATH + +export SYSROOT_PATH=/home/openharmony/pegasus/os/OpenHarmony/ohos/out/hispark_ss928v100/ipcamera_hispark_ss928v100_linux/sysroot + +make clean && make +``` + +![image-20251022150813007](pic/image-20251022150813007.png) + +* 在ffmpeg目录下创建一个build_ffmpeg.sh脚本，把下面的内容复制进去 + +* 注意：请根据自己服务器的实际情况修改里面的绝对路径 + +![image-20251022150910879](pic/image-20251022150910879.png) + +```sh +#!/bin/bash + +# 设置交叉编译工具链的根目录 +TOOLCHAIN_ROOT="/home/openharmony/pegasus/os/OpenHarmony/ohos/prebuilts/clang/ohos/linux-x86_64/llvm" +# 交叉编译工具链前缀 +CROSS_PREFIX="aarch64-unknown-linux-ohos" +# 设置sysroot路径 +SYSROOT="/home/openharmony/pegasus/os/OpenHarmony/ohos/out/hispark_ss928v100/ipcamera_hispark_ss928v100_linux/sysroot" + +# C 和 C++ 编译器 +CC="${TOOLCHAIN_ROOT}/bin/${CROSS_PREFIX}-clang" +CXX="${TOOLCHAIN_ROOT}/bin/${CROSS_PREFIX}-clang++" +export AR="${TOOLCHAIN_ROOT}/bin/llvm-ar" +export LD="${TOOLCHAIN_ROOT}/bin/lld" +export RANLIB="${TOOLCHAIN_ROOT}/bin/llvm-ranlib" +export STRIP="${TOOLCHAIN_ROOT}/bin/llvm-strip" + +# 获取当前脚本目录 +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +WORK_DIR="$(pwd)" + +# 安装目录 +PREFIX="${WORK_DIR}/install" + +# 海思SDK路径 +HISI_SDK_BASE="/home/openharmony/pegasus/platform/ss928v100_clang/smp/a55_linux" +HISI_MPP_BASE="${HISI_SDK_BASE}/mpp" +HISI_COMMON_DIR="/home/openharmony/pegasus/platform/ss928v100_clang/smp/a55_linux/mpp/sample/common" + +# 配置PKG_CONFIG_PATH +export PKG_CONFIG_PATH="/home/openharmony/pegasus/vendor/opensource/x264/install/lib/pkgconfig:$PKG_CONFIG_PATH" +export PKG_CONFIG_PATH="/home/openharmony/pegasus/vendor/opensource/x265_3.5/install/lib/pkgconfig:$PKG_CONFIG_PATH" +export PKG_CONFIG_PATH="/home/openharmony/pegasus/vendor/opensource/v4l-utils/install/lib/pkgconfig:$PKG_CONFIG_PATH" + +# 查找FFmpeg源码目录 +FFMPEG_SRC="../ffmpeg" +# 创建SDK初始化库目录 +SDK_LIB_DIR="$(pwd)/hisi_sdk_lib" +mkdir -p "$SDK_LIB_DIR" +# 定义所有SDK库目录，添加更多可能的搜索路径 +ALL_SDK_LIB_DIRS=( + "${HISI_MPP_BASE}/out/lib" + "${HISI_MPP_BASE}/out/lib/svp_npu" + "${HISI_COMMON_DIR}" +) + +# 获取实际存在的海思SDK库列表 +echo "扫描实际存在的海思SDK库文件..." +AVAILABLE_LIBS=() +for dir in "${ALL_SDK_LIB_DIRS[@]}"; do + if [ -d "$dir" ]; then + echo "扫描库目录: $dir" + # 查找所有.so和.a库文件 + for lib_path in "${dir}/lib"*.so "${dir}/lib"*.a; do + if [ -f "$lib_path" ]; then + # 提取库名称（去除路径、lib前缀和扩展名） + lib_name=$(basename "$lib_path" .so) + lib_name=$(basename "$lib_name" .a) + lib_name=${lib_name#lib} + + # 仅添加新的库，避免重复 + if ! [[ " ${AVAILABLE_LIBS[@]} " =~ " ${lib_name} " ]]; then + AVAILABLE_LIBS+=("$lib_name") + echo "✓ 找到库: $lib_name ($lib_path)" + fi + fi + done + else + echo "⚠ 库目录不存在: $dir" + fi +done + +echo "总共找到 ${#AVAILABLE_LIBS[@]} 个海思SDK库" + +# 检查关键库是否存在，使用ss_mpi作为ot_mpi的替代 +CRITICAL_LIBS=("securec" "ot_osal" "ss_mpi" "ot_base") +for lib in "${CRITICAL_LIBS[@]}"; do + if [[ " ${AVAILABLE_LIBS[@]} " =~ " ${lib} " ]]; then + echo "✓ 关键库 ${lib} 存在" + else + echo "⚠ 警告: 关键库 ${lib} 缺失" + fi +done + +# 验证已编译的目标文件是否存在 +REQUIRED_OBJS=( + "${HISI_COMMON_DIR}/sdk_init.o" + "${HISI_COMMON_DIR}/sdk_exit.o" +) + +COMPILED_OBJECTS=() +for obj in "${REQUIRED_OBJS[@]}"; do + if [ -f "$obj" ]; then + COMPILED_OBJECTS+=("$obj") + echo "✓ 找到已编译目标文件: $(basename "$obj")" + else + echo "✗ 缺失已编译目标文件: $(basename "$obj")" + echo "请确保在 ${HISI_COMMON_DIR} 目录下存在该文件" + exit 1 + fi +done + +for obj_path in "${HISI_COMMON_DIR}"/*.o; do + # 跳过已经添加的sdk_init.o和sdk_exit.o + if [ -f "$obj_path" ] && [[ ! " ${COMPILED_OBJECTS[@]} " =~ " ${obj_path} " ]]; then + COMPILED_OBJECTS+=("$obj_path") + echo "✓ 添加额外目标文件: $(basename "$obj_path")" + fi +done + +# 构建链接参数 - 自动链接所有找到的库 +LINK_LIBS="" +LINKED_COUNT=0 + +echo "按顺序链接所有找到的海思SDK库:" +for lib in "${AVAILABLE_LIBS[@]}"; do + LINK_LIBS="${LINK_LIBS} -l${lib}" + echo "✓ 链接库: ${lib}" + ((LINKED_COUNT++)) +done + +# 构建动态库链接命令，添加处理非PIC代码的选项 +echo "构建动态库链接命令..." +LINK_CMD="$CC -shared -fPIC --sysroot=$SYSROOT" +LINK_CMD="${LINK_CMD} -o ${SDK_LIB_DIR}/libhisi_sdk_init.so" +LINK_CMD="${LINK_CMD} ${COMPILED_OBJECTS[*]}" + +# 添加所有SDK库目录到链接命令（包括common目录） +for dir in "${ALL_SDK_LIB_DIRS[@]}"; do + LINK_CMD="${LINK_CMD} -L${dir}" +done + +# 添加处理非PIC代码的链接选项 +LINK_CMD="${LINK_CMD} ${LINK_LIBS}" +LINK_CMD="${LINK_CMD} -lpthread -lm -lstdc++ -ldl -lrt" +LINK_CMD="${LINK_CMD} -Wl,--allow-shlib-undefined" +LINK_CMD="${LINK_CMD} -Wl,-allow-multiple-definition" # 允许多重定义 +LINK_CMD="${LINK_CMD} -Wl,-z,notext" # 允许非PIC代码在共享库中 + +echo "执行链接命令:" +echo "${LINK_CMD}" +echo "==========================================" + +eval "${LINK_CMD}" + +if [ $? -ne 0 ]; then + echo "✗ 动态库创建失败" + echo "尝试使用简化链接参数..." + + # 只链接关键库的简化版本，使用ss_mpi替代ot_mpi + SIMPLE_LINK_CMD="$CC -shared -fPIC --sysroot=$SYSROOT" + SIMPLE_LINK_CMD="${SIMPLE_LINK_CMD} -o ${SDK_LIB_DIR}/libhisi_sdk_init.so" + SIMPLE_LINK_CMD="${SIMPLE_LINK_CMD} ${COMPILED_OBJECTS[*]}" + + # 添加所有SDK库目录到简化链接命令 + for dir in "${ALL_SDK_LIB_DIRS[@]}"; do + SIMPLE_LINK_CMD="${SIMPLE_LINK_CMD} -L${dir}" + done + + # 使用ss_mpi替代ot_mpi + SIMPLE_LINK_CMD="${SIMPLE_LINK_CMD} -lsecurec -lot_osal -lss_mpi" + SIMPLE_LINK_CMD="${SIMPLE_LINK_CMD} -lpthread -lm -lstdc++ -ldl" + SIMPLE_LINK_CMD="${SIMPLE_LINK_CMD} -Wl,--allow-shlib-undefined" + SIMPLE_LINK_CMD="${SIMPLE_LINK_CMD} -Wl,-allow-multiple-definition" + SIMPLE_LINK_CMD="${SIMPLE_LINK_CMD} -Wl,-z,notext" + + echo "执行简化链接命令:" + echo "${SIMPLE_LINK_CMD}" + eval "${SIMPLE_LINK_CMD}" + + if [ $? -ne 0 ]; then + echo "✗ 动态库创建失败" + echo "建议尝试重新编译common目录下的目标文件，添加-fPIC选项:" + echo "cd ${HISI_COMMON_DIR}" + echo "${CC} -c -fPIC *.c -I${HISI_MPP_BASE}/include -I${HISI_SDK_BASE}/include" + exit 1 + fi +fi + +echo "✓ 海思SDK初始化动态库创建完成!" + +if [ -f "${SDK_LIB_DIR}/libhisi_sdk_init.so" ]; then + echo "✓ 动态库文件存在" + ls -la "${SDK_LIB_DIR}/libhisi_sdk_init.so" + + # 检查符号 + echo "" + echo "检查关键符号:" + REQUIRED_SYMBOLS=("SDK_init" "SDK_exit") + for symbol in "${REQUIRED_SYMBOLS[@]}"; do + if ${CROSS_PREFIX}-nm -D "${SDK_LIB_DIR}/libhisi_sdk_init.so" 2>/dev/null | grep -q "$symbol" || \ + ${CROSS_PREFIX}-objdump -T "${SDK_LIB_DIR}/libhisi_sdk_init.so" 2>/dev/null | grep -q "$symbol"; then + echo "✓ 符号 $symbol 存在" + else + echo "⚠ 符号 $symbol 可能缺失" + fi + done + +else + echo "✗ 动态库文件不存在" + exit 1 +fi + +# 设置编译标志，添加-fPIC +CFLAGS="--sysroot=$SYSROOT -fPIC" +CFLAGS="${CFLAGS} -I${HISI_COMMON_DIR}" # 添加common目录的头文件路径 +CFLAGS="${CFLAGS} -I${HISI_MPP_BASE}/out/include" +CFLAGS="${CFLAGS} -DHISI_SDK_ENABLED" +export CFLAGS + +# 设置C++编译标志，添加-fPIC +export CXXFLAGS="--sysroot=$SYSROOT -fPIC" + +# 设置链接标志 +LDFLAGS="--sysroot=$SYSROOT" +LDFLAGS="${LDFLAGS} -L${SDK_LIB_DIR}" + +# 添加所有SDK库目录到链接标志（包括common目录） +for dir in "${ALL_SDK_LIB_DIRS[@]}"; do + LDFLAGS="${LDFLAGS} -L${dir}" +done + +# 设置运行时库搜索路径 +LDFLAGS="${LDFLAGS} -Wl,--allow-shlib-undefined" +LDFLAGS="${LDFLAGS} -Wl,-allow-multiple-definition" + +# 链接库 +LDFLAGS="${LDFLAGS} -lhisi_sdk_init" +LDFLAGS="${LDFLAGS} ${LINK_LIBS}" +LDFLAGS="${LDFLAGS} -lpthread -lm -lstdc++ -ldl -lrt" +export LDFLAGS + +#echo "CFLAGS: ${CFLAGS}" +#echo "LDFLAGS: ${LDFLAGS}" + +# 构建configure命令 +function gen_cfg_cmd() { + printf "%s " \ + "${FFMPEG_SRC}/configure" \ + "--prefix=$PREFIX" \ + "--arch=aarch64" \ + "--target-os=linux" \ + "--enable-cross-compile" \ + "--disable-x86asm" \ + "--disable-static" \ + "--enable-shared" \ + "--cc=$CC" \ + "--cxx=$CXX" \ + "--strip=$STRIP" \ + "--ld=$CXX" \ + "--sysroot=$SYSROOT" \ + "--enable-libx264" \ + "--enable-libx265" \ + "--enable-gpl" \ + "--enable-encoder=h264_ss928" \ + "--enable-encoder=h265_ss928" \ + "--enable-encoder=mjpeg_ss928" \ + "--enable-decoder=h264_ss928" \ + "--enable-decoder=h265_ss928" \ + "--enable-decoder=mjpeg_ss928" \ + "--extra-cflags='$CFLAGS'" \ + "--extra-ldflags='$LDFLAGS'" + echo +} + +# 执行configure +echo "==========================================" +echo "开始配置FFmpeg..." +echo "==========================================" +cfg_cmd=$(gen_cfg_cmd) +echo "$cfg_cmd" +echo "==========================================" +eval "$cfg_cmd" +``` + +### 步骤4：编译ffmpeg + +* 在服务器的命令行，执行下面的命令，进行编译ffmpeg前的配置 + +```sh +chmod +x build_ffmpeg.sh + +./build_ffmpeg.sh +``` + +![image-20251022151107143](pic/image-20251022151107143.png) + +* 在服务器的命令行，执行下面的命令，进行ffmpeg的编译 + +```sh +make -j$(nproc) && make install +``` + +![image-20251022151357987](pic/image-20251022151357987.png) + +* 编译完成之后，会在install目录下生成下面的内容 + +![image-20251022151320279](pic/image-20251022151320279.png) + +* 并且在ffmpeg目录下生成一个hisi_sdk_lib的文件夹，里面有一个libhisi_sdk_init.so库 + +![image-20251022151540301](pic/image-20251022151540301.png) + +### 步骤5：编译sample + +* 在服务器的命令行，执行下面的命令，分别编译ffmpeg和hisi目录下的编解码案例 + +```sh +cd sample/ffmpeg + +make + +cd ../hisi + +make +``` + +![image-20251022155440903](pic/image-20251022155440903.png) + +![image-20251022155422517](pic/image-20251022155422517.png) + + + +## 4、板端验证 + +### 步骤1：配置板端环境 + +* 1、确保开发板已经烧录OpenHarmony操作系统 +* 2、使用网线将开发板与你的电脑进行连接，确保二者处于同一局域网内 +* 3、配置开发板的IP地址，并确保开发板与电脑能够互相ping通 + +```sh +# 注意：这里的eth0的IP地址，请根据自己的网络IP网段进行合理配置 +ifconfig eth0 192.168.100.100 + +# 添加权限 +echo 0 9999999 > /proc/sys/net/ipv4/ping_group_range +``` + +![image-20251010094441549](pic/image-20251010094441549.png) + +### 步骤2：准备ffmpeg依赖文件 + +* 1、将第2章交叉编译生成的v4l2、x264、x265的install目录，下载并拷贝到NFS挂载目录 +* 2、将第三章交叉编译ffmpeg生成的install目录，下载并拷贝到NFS挂载目录 +* 3、将第三章步骤5，编译好后的sample目录，下载并拷贝到NFS挂载目录 +* 4、将libhisi_sdk_init.so下载并拷贝到ffmpeg_install/lib/目录下 +* 5、将mpp/out/lib下载并拷贝到NFS挂载目录 + +![image-20251022162128690](pic/image-20251022162128690.png) + +![image-20251022164347161](pic/image-20251022164347161.png) + +* 6、在开发板的命令行终端执行下面的命令，将电脑的nfs目录挂载到开发板的/mnt目录下（注意：这里请根据自己的IP地址及NFS配置进行合理的修改） + +```sh +mount -o nolock,addr=192.168.100.10 -t nfs 192.168.100.10:/d/nfs /mnt +``` + +![image-20251022162847658](pic/image-20251022162847658.png) + +* 因为我这里对每个库进行了打包，所以当我们把NFS挂载上后，需要对这些库进行解压才能正常使用 + +![image-20251022163123950](pic/image-20251022163123950.png) + +* 7、在开发板的命令行终端执行下面的命令，配置各个库的环境变量 + +```sh +export PATH=/mnt/ffmpeg/ffmpeg_install/bin:$PATH +export LD_LIBRARY_PATH=/mnt/ffmpeg/ffmpeg_install/lib:/mnt/ffmpeg/lib:/mnt/ffmpeg/lib/svp_npu:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=/mnt/ffmpeg/v4l2_install/lib:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=/mnt/ffmpeg/x264_install/lib:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=/mnt/ffmpeg/x265_install/lib:$LD_LIBRARY_PATH + +chmod +x /mnt/ffmpeg/ffmpeg_install/bin/* +``` + +![image-20251022163146923](pic/image-20251022163146923.png) + +### 步骤3：测试ffmpeg是否正常工作 + +* 在开发板的命令行，执行下面的命令，获取ffmpeg的版本号 + +```sh +cd /mnt/ffmpeg/ffmpeg_install/bin/ + +ffmpeg -version +``` + +image-20251022170611820 + +### 步骤4：测试sample案例 + +* 注意：sample/hisi/目录下的是调用了海思硬件编解码模块，实现了硬件加速。sample/ffmpeg/目录下就是调用了ffmpeg的原始编解码接口。 +* 在开发板的命令行终端执行下面的命令，运行相关程序 + +```sh +cd /mnt/ffmpeg/sample/hisi + +chmod +x * + +# hisi编码 +./hisi_264venc /dev/video0 h264_ss928 640 480 30 + +cd /mnt/ffmpeg/sample/ffmpeg + +chmod +x * +# ffmpeg编码 +./264enc /dev/video0 640 480 30 +``` + +![image-20251022175239062](pic/image-20251022175239062.png) + +![image-20251022175457018](pic/image-20251022175457018.png) + +* 在开发板的命令行分步执行下面的命令，测试解码案例 + +```sh +cd /mnt/ffmpeg/sample/hisi + +# hisi解码 +./hisi_264vdec h264_ss928_output.h264 output + +# ffmpeg解码 +./264dec libx264_output.h264 output +``` + +![image-20251022175651926](pic/image-20251022175651926.png) + +![image-20251022175749466](pic/image-20251022175749466.png) + +* 如果想在自己的代码中使用ffmpeg调用hisi的硬件编解码模块，可以使用avcodec_find_decoder_by_name接口来实现 + +```c +avcodec_find_decoder_by_name("h264ss928"); + +avcodec_find_decoder_by_name("h265_ss928"); + +avcodec_find_decoder_by_name("mjpeg_ss928"); + +avcodec_find_encoder_by_name("h264_ss928"); + +avcodec_find_encoder_by_name("h265_ss928"); + +avcodec_find_encoder_by_name("mjpeg_ss928"); +``` + +## 5、海思硬件编解码与ffmpeg原生编解码对比 + +* decoder + +| 数据/解码器 | h264 | h264_ss928 | hevc | h265_ss928 | mjpeg | mjpeg_ss928 | +| ------------------- | --------------------- | --------------------- | --------------------- | --------------------- | --------------------- | --------------------- | +| 视频参数 | 1920x1080
@30FPS | 1920x1080
@30FPS | 1920x1080
@30FPS | 1920x1080
@30FPS | 1920x1080
@30FPS | 1920x1080
@30FPS | +| 总处理帧数 | 85 | 207 | 21 | 221 | 125 | 217 | +| 总解码时间(ms) | 4547.785 | 8592.903 | 4566.406 | 9147.614 | 3020.907 | 93398.829 | +| 总发送时间(ms) | | | 4566.203 | 8168.024(89.3%) | | 9007.92 | +| 总接收时间(ms) | | | 0.203 | 1004.696(10.7%) | | 335.712 | +| 平均解码时间(ms/帧) | 53.503 | 41.512 | 217.448 | 41.401 | 20.167 | 43.036 | +| 平均发送时间(ms/帧) | | | 217.438 | 36.959 | | | +| 平均接收时间(ms/帧) | | | 0.01 | 4.546 | | | +| 最短解码时间(ms) | 0.036 | 40.653 | 191.937 | 40.699 | 22.408 | 40.909 | +| 最长解码时间(ms) | 74.43 | 42.67 | 307.397 | 44.188 | 26.027 | 83.292 | +| 时间差值(ms) | 12.534 | | 115.46 | 3.489 | 3.599 | 42.383 | +| 解码帧率(FPS) | 18.69 | 24.09 | 4.6 | 24.15 | 41.38 | 23.24 | +| 每秒处理像素（MP/s) | | | 9.54 | 50.09 | 19.09 | 12.52 | + +* encoder + +| 数据/编码器 | libx264 | h264_ss928 | libx265 | h265_ss928 | mjpeg | mjpeg_ss928 | +| ----------------------------- | ------------------------ | --------------------- | --------------------- | --------------------- | --------------------- | --------------------- | +| 视频参数 | 1920x1080
@30FPS | 1920x1080
@30FPS | 1920x1080
@30FPS | 1920x1080
@30FPS | 1920x1080
@30FPS | 1920x1080
@30FPS | +| 总处理帧数 | 85 | 208 | 21 | 222 | 114 | 218 | +| 总耗时(s) | 30.45 | 14.05 | 41.62 | 15.56 | 16.36 | 15.1 | +| 平均FPS | 11.19 | 14.8 | 0.5 | 14.27 | 6.97 | 14.44 | +| 平均格式转换时间(ms/帧) | 3 | 65.22 | 11.52 | 66.91 | 51.11 | 66.17 | +| 平均编码时间(ms/帧) | 342.35 | 2.25 | 1957.39 | 3.07 | 89.4 | 2.99 | +| 编码效率(编码时间/理论帧间隔) | 1027.05% | 6.74% | 5872.18% | 9.20% | 268.21% | 8.97% | +| 总编码时间(s) | 29.01 | 0.47 | 41.11 | 0.68 | 10.19 | 0.65 | +| 总转换时间(s) | 0.95 | 13.57 | 0.24 | 14.85 | 5.83 | 14.43 | +| 输出文件大小(MB) | 1.26 | | 0.32 | 13.28 | 17.44 | 11.86 | +| 压缩比 | 39.29:1 | | 192.04：1 | 49.60：1 | 19.39：1 | 54.52：1 | +| 平均码率(kbps) | 347.13 | | 65.39 | 7158.57 | 8941.11 | 6589.71 | +| 性能评级 | 极高压缩率，编码需要优化 | 编码优秀 | 压缩良好，编码不足 | 编码优秀 | 需优化 | 优秀 | diff --git a/vendor/zsks/docs/ffmpeg/pic/image-20250915232849-tjfqh3n.png b/vendor/zsks/docs/ffmpeg/pic/image-20250915232849-tjfqh3n.png new file mode 100644 index 0000000000000000000000000000000000000000..21fda5fcf1869c2dcf558d468c9a2f86398af70f Binary files /dev/null and b/vendor/zsks/docs/ffmpeg/pic/image-20250915232849-tjfqh3n.png differ diff --git a/vendor/zsks/docs/ffmpeg/pic/image-20250915232913-7et18x6.png b/vendor/zsks/docs/ffmpeg/pic/image-20250915232913-7et18x6.png new file mode 100644 index 0000000000000000000000000000000000000000..fde025e7a494293a9f3e8bf900e3d1f3119a89e8 Binary files /dev/null and b/vendor/zsks/docs/ffmpeg/pic/image-20250915232913-7et18x6.png differ diff --git a/vendor/zsks/docs/ffmpeg/pic/image-20250915232951-jif09al.png b/vendor/zsks/docs/ffmpeg/pic/image-20250915232951-jif09al.png new file mode 100644 index 0000000000000000000000000000000000000000..31c380ac27cdb37a5e5c579297fb07dac19fd32c Binary files /dev/null and b/vendor/zsks/docs/ffmpeg/pic/image-20250915232951-jif09al.png differ diff --git a/vendor/zsks/docs/ffmpeg/pic/image-20250915233247-0qbuniv.png b/vendor/zsks/docs/ffmpeg/pic/image-20250915233247-0qbuniv.png new file mode 100644 index 0000000000000000000000000000000000000000..1e2554a359f21877ef8c2c0ddf477e75b2c31e05 Binary files /dev/null and b/vendor/zsks/docs/ffmpeg/pic/image-20250915233247-0qbuniv.png differ diff --git a/vendor/zsks/docs/ffmpeg/pic/image-20251010094441549.png b/vendor/zsks/docs/ffmpeg/pic/image-20251010094441549.png new file mode 100755 index 0000000000000000000000000000000000000000..e981001ec636db595d2766310bf95a49e58f3914 Binary files /dev/null and b/vendor/zsks/docs/ffmpeg/pic/image-20251010094441549.png differ diff --git a/vendor/zsks/docs/ffmpeg/pic/image-20251021144741884.png b/vendor/zsks/docs/ffmpeg/pic/image-20251021144741884.png new file mode 100755 index 0000000000000000000000000000000000000000..e481421c729bc4011b4cad2cf22518be016f4260 Binary files /dev/null and b/vendor/zsks/docs/ffmpeg/pic/image-20251021144741884.png differ diff --git a/vendor/zsks/docs/ffmpeg/pic/image-20251021150217681.png b/vendor/zsks/docs/ffmpeg/pic/image-20251021150217681.png new file mode 100755 index 0000000000000000000000000000000000000000..b1256530ef6e2b2bd83736558601545ff8b6c0b7 Binary files /dev/null and b/vendor/zsks/docs/ffmpeg/pic/image-20251021150217681.png differ diff --git a/vendor/zsks/docs/ffmpeg/pic/image-20251021154212147.png b/vendor/zsks/docs/ffmpeg/pic/image-20251021154212147.png new file mode 100755 index 0000000000000000000000000000000000000000..595e9639aa561f56eb2dd4155a6f69f19a91e910 Binary files /dev/null and b/vendor/zsks/docs/ffmpeg/pic/image-20251021154212147.png differ diff --git a/vendor/zsks/docs/ffmpeg/pic/image-20251021154533551.png b/vendor/zsks/docs/ffmpeg/pic/image-20251021154533551.png new file mode 100755 index 0000000000000000000000000000000000000000..b106a565c5e18e058c6d57a7aaba66d57220f0e4 Binary files /dev/null and b/vendor/zsks/docs/ffmpeg/pic/image-20251021154533551.png differ diff --git a/vendor/zsks/docs/ffmpeg/pic/image-20251021154614257.png b/vendor/zsks/docs/ffmpeg/pic/image-20251021154614257.png new file mode 100755 index 0000000000000000000000000000000000000000..ce0236fa509a1f00ee893369cc026769520406ae Binary files /dev/null and b/vendor/zsks/docs/ffmpeg/pic/image-20251021154614257.png differ diff --git a/vendor/zsks/docs/ffmpeg/pic/image-20251021161044799.png b/vendor/zsks/docs/ffmpeg/pic/image-20251021161044799.png new file mode 100755 index 0000000000000000000000000000000000000000..7a84e2650c691eeaf3e0da8fbaed35592e172075 Binary files /dev/null and b/vendor/zsks/docs/ffmpeg/pic/image-20251021161044799.png differ diff --git a/vendor/zsks/docs/ffmpeg/pic/image-20251021161134241.png b/vendor/zsks/docs/ffmpeg/pic/image-20251021161134241.png new file mode 100755 index 0000000000000000000000000000000000000000..16158c56b5ff0f57bac2ae30a4eaca37e25e17f9 Binary files /dev/null and b/vendor/zsks/docs/ffmpeg/pic/image-20251021161134241.png differ diff --git a/vendor/zsks/docs/ffmpeg/pic/image-20251021161216433.png b/vendor/zsks/docs/ffmpeg/pic/image-20251021161216433.png new file mode 100755 index 0000000000000000000000000000000000000000..a3739b57a6967609a503a01e901cb81957c8bfaf Binary files /dev/null and b/vendor/zsks/docs/ffmpeg/pic/image-20251021161216433.png differ diff --git a/vendor/zsks/docs/ffmpeg/pic/image-20251021205649485.png b/vendor/zsks/docs/ffmpeg/pic/image-20251021205649485.png new file mode 100755 index 0000000000000000000000000000000000000000..f5a12debb0aab543c5d0e327d1e10b1bee943dd9 Binary files /dev/null and b/vendor/zsks/docs/ffmpeg/pic/image-20251021205649485.png differ diff --git a/vendor/zsks/docs/ffmpeg/pic/image-20251021205900311.png b/vendor/zsks/docs/ffmpeg/pic/image-20251021205900311.png new file mode 100755 index 0000000000000000000000000000000000000000..995d820967e8a942a065956d51b38d48757d76a3 Binary files /dev/null and b/vendor/zsks/docs/ffmpeg/pic/image-20251021205900311.png differ diff --git a/vendor/zsks/docs/ffmpeg/pic/image-20251022142837408.png b/vendor/zsks/docs/ffmpeg/pic/image-20251022142837408.png new file mode 100755 index 0000000000000000000000000000000000000000..e249c0aede872c03525e5ac1992f16ce105f576d Binary files /dev/null and b/vendor/zsks/docs/ffmpeg/pic/image-20251022142837408.png differ diff --git a/vendor/zsks/docs/ffmpeg/pic/image-20251022150813007.png b/vendor/zsks/docs/ffmpeg/pic/image-20251022150813007.png new file mode 100755 index 0000000000000000000000000000000000000000..4d631cb10f69e307842107f01212b973d4ed81b8 Binary files /dev/null and b/vendor/zsks/docs/ffmpeg/pic/image-20251022150813007.png differ diff --git a/vendor/zsks/docs/ffmpeg/pic/image-20251022150910879.png b/vendor/zsks/docs/ffmpeg/pic/image-20251022150910879.png new file mode 100755 index 0000000000000000000000000000000000000000..c7b729ea84e50b92fbe266a27a1c163aa2c5762f Binary files /dev/null and b/vendor/zsks/docs/ffmpeg/pic/image-20251022150910879.png differ diff --git a/vendor/zsks/docs/ffmpeg/pic/image-20251022151107143.png b/vendor/zsks/docs/ffmpeg/pic/image-20251022151107143.png new file mode 100755 index 0000000000000000000000000000000000000000..eb5a5e8e0712b9e94a31929defb4765dd6ac08a6 Binary files /dev/null and b/vendor/zsks/docs/ffmpeg/pic/image-20251022151107143.png differ diff --git a/vendor/zsks/docs/ffmpeg/pic/image-20251022151320279.png b/vendor/zsks/docs/ffmpeg/pic/image-20251022151320279.png new file mode 100755 index 0000000000000000000000000000000000000000..a0af3d027c9e0b43333964a2706c3d449145f8aa Binary files /dev/null and b/vendor/zsks/docs/ffmpeg/pic/image-20251022151320279.png differ diff --git a/vendor/zsks/docs/ffmpeg/pic/image-20251022151357987.png b/vendor/zsks/docs/ffmpeg/pic/image-20251022151357987.png new file mode 100755 index 0000000000000000000000000000000000000000..3492af0c2d6a9a740dcd9e822879e3579d6b44ac Binary files /dev/null and b/vendor/zsks/docs/ffmpeg/pic/image-20251022151357987.png differ diff --git a/vendor/zsks/docs/ffmpeg/pic/image-20251022151540301.png b/vendor/zsks/docs/ffmpeg/pic/image-20251022151540301.png new file mode 100755 index 0000000000000000000000000000000000000000..f34bd883a0d5cd2637ba771560609ebe2d871195 Binary files /dev/null and b/vendor/zsks/docs/ffmpeg/pic/image-20251022151540301.png differ diff --git a/vendor/zsks/docs/ffmpeg/pic/image-20251022155422517.png b/vendor/zsks/docs/ffmpeg/pic/image-20251022155422517.png new file mode 100755 index 0000000000000000000000000000000000000000..e286666db8543a020f5f99b2857f8cca0f7ee2c4 Binary files /dev/null and b/vendor/zsks/docs/ffmpeg/pic/image-20251022155422517.png differ diff --git a/vendor/zsks/docs/ffmpeg/pic/image-20251022155440903.png b/vendor/zsks/docs/ffmpeg/pic/image-20251022155440903.png new file mode 100755 index 0000000000000000000000000000000000000000..6f579547291f2da2bb336c9e74e70abe82e9eb32 Binary files /dev/null and b/vendor/zsks/docs/ffmpeg/pic/image-20251022155440903.png differ diff --git a/vendor/zsks/docs/ffmpeg/pic/image-20251022162128690.png b/vendor/zsks/docs/ffmpeg/pic/image-20251022162128690.png new file mode 100755 index 0000000000000000000000000000000000000000..72f9513422d72a7141b187bd34f2e248b36e0a56 Binary files /dev/null and b/vendor/zsks/docs/ffmpeg/pic/image-20251022162128690.png differ diff --git a/vendor/zsks/docs/ffmpeg/pic/image-20251022162847658.png b/vendor/zsks/docs/ffmpeg/pic/image-20251022162847658.png new file mode 100755 index 0000000000000000000000000000000000000000..1213dd041032b7021581ae0c2d6d662a52e52ff0 Binary files /dev/null and b/vendor/zsks/docs/ffmpeg/pic/image-20251022162847658.png differ diff --git a/vendor/zsks/docs/ffmpeg/pic/image-20251022163123950.png b/vendor/zsks/docs/ffmpeg/pic/image-20251022163123950.png new file mode 100755 index 0000000000000000000000000000000000000000..cc97d736b87dd5d60d902df9b1c697259aa8566e Binary files /dev/null and b/vendor/zsks/docs/ffmpeg/pic/image-20251022163123950.png differ diff --git a/vendor/zsks/docs/ffmpeg/pic/image-20251022163146923.png b/vendor/zsks/docs/ffmpeg/pic/image-20251022163146923.png new file mode 100755 index 0000000000000000000000000000000000000000..73349a81e9600c6b33e5e5945f8ccde92d30fb5f Binary files /dev/null and b/vendor/zsks/docs/ffmpeg/pic/image-20251022163146923.png differ diff --git a/vendor/zsks/docs/ffmpeg/pic/image-20251022164347161.png b/vendor/zsks/docs/ffmpeg/pic/image-20251022164347161.png new file mode 100755 index 0000000000000000000000000000000000000000..c845bb9b545b91f51c6aa7509dfe695104a85d5c Binary files /dev/null and b/vendor/zsks/docs/ffmpeg/pic/image-20251022164347161.png differ diff --git a/vendor/zsks/docs/ffmpeg/pic/image-20251022170611820.png b/vendor/zsks/docs/ffmpeg/pic/image-20251022170611820.png new file mode 100755 index 0000000000000000000000000000000000000000..f8bb1cb8216cf9d9081fc7e11bffaaa6329452cb Binary files /dev/null and b/vendor/zsks/docs/ffmpeg/pic/image-20251022170611820.png differ diff --git a/vendor/zsks/docs/ffmpeg/pic/image-20251022175239062.png b/vendor/zsks/docs/ffmpeg/pic/image-20251022175239062.png new file mode 100755 index 0000000000000000000000000000000000000000..55d79571b64ab870c61a17e1ee9487cb635142e8 Binary files /dev/null and b/vendor/zsks/docs/ffmpeg/pic/image-20251022175239062.png differ diff --git a/vendor/zsks/docs/ffmpeg/pic/image-20251022175457018.png b/vendor/zsks/docs/ffmpeg/pic/image-20251022175457018.png new file mode 100755 index 0000000000000000000000000000000000000000..2206fcf88a1299ca6816af70637998036b59f4d0 Binary files /dev/null and b/vendor/zsks/docs/ffmpeg/pic/image-20251022175457018.png differ diff --git a/vendor/zsks/docs/ffmpeg/pic/image-20251022175651926.png b/vendor/zsks/docs/ffmpeg/pic/image-20251022175651926.png new file mode 100755 index 0000000000000000000000000000000000000000..740f3d30765246b6614355ac73dba47f8894e4e1 Binary files /dev/null and b/vendor/zsks/docs/ffmpeg/pic/image-20251022175651926.png differ diff --git a/vendor/zsks/docs/ffmpeg/pic/image-20251022175749466.png b/vendor/zsks/docs/ffmpeg/pic/image-20251022175749466.png new file mode 100755 index 0000000000000000000000000000000000000000..98e0cdd447690c49b46e766bfac8db37741d051e Binary files /dev/null and b/vendor/zsks/docs/ffmpeg/pic/image-20251022175749466.png differ diff --git a/vendor/zsks/docs/libcamera/README.md b/vendor/zsks/docs/libcamera/README.md new file mode 100755 index 0000000000000000000000000000000000000000..832a7ce9d7f6f5e7409dcacbca1760e18b783b84 --- /dev/null +++ b/vendor/zsks/docs/libcamera/README.md @@ -0,0 +1,466 @@ +# libcamera移植 + +## 1、软硬件环境 +* 开发板：海鸥派 +* 交叉编译工具链：OHOS (dev) clang version 15.0.4 + +* 编译链路径：pegasus/os/OpenHarmony/ohos/prebuilts/clang/ohos/linux-x86_64/llvm/bin +* Python版本：Python-3.13.2 +* 移植的libcamera版本：libcamera-0.5 + +## 2、配置python环境 + +* 步骤1：为了满足python3.13.2的移植要求，我们需要先把服务器的python版本改为3.13.2，且按照[Python的移植步骤](../python/README.md)，把Python-3.13.2交叉编译好。 +* 步骤2：按照[numpy的移植文档](../numpy/README.md)第2节的步骤3，把虚拟环境创建好。 + +```sh +cd pegasus/vendor/opensource/Python-3.13.2 + +# 激活环境 +. crossenv_aarch64/bin/activate +``` + +![image-20251020160523558](pic/image-20251020160523558.png) + +## 3.安装依赖 + +* 由于在编译libacamera的时候，需要依赖其他第三方软件，因此在编译libacamera之前，我们先把它依赖的第三方软件全部交叉编译出来。 + +### 步骤1： 配置依赖的环境变量 + +* 在服务器的命令行执行下面的命令，配置交叉编译时依赖的环境变量。这里的绝对路径请根据自己服务器实际情况进行修改。 + +```sh +export CC=/home/openharmony/pegasus/os/OpenHarmony/ohos/prebuilts/clang/ohos/linux-x86_64/llvm/bin/aarch64-unknown-linux-ohos-clang +export CXX=/home/openharmony/pegasus/os/OpenHarmony/ohos/prebuilts/clang/ohos/linux-x86_64/llvm/bin/aarch64-unknown-linux-ohos-clang++ +export AR=/home/openharmony/pegasus/os/OpenHarmony/ohos/prebuilts/clang/ohos/linux-x86_64/llvm/bin/llvm-ar +export LD=/home/openharmony/pegasus/os/OpenHarmony/ohos/prebuilts/clang/ohos/linux-x86_64/llvm/bin/ld.lld +export RANLIB=/home/openharmony/pegasus/os/OpenHarmony/ohos/prebuilts/clang/ohos/linux-x86_64/llvm/bin/llvm-ranlib +export STRIP=/home/openharmony/pegasus/os/OpenHarmony/ohos/prebuilts/clang/ohos/linux-x86_64/llvm/bin/llvm-strip +``` + +### 步骤2：交叉编译依赖第三方软件 + +**注意：**在交叉编译之前，请一定确保OpenHarmony的代码下载完成，且整编通过，具体可参考[ohos编译](https://gitee.com/HiSpark/pegasus/blob/master/docs/OpenHarmony%20Small%E7%89%88%E6%9C%AC%E4%BD%BF%E7%94%A8%E6%8C%87%E5%8D%97/OpenHarmony%20Small%E7%89%88%E6%9C%AC%E4%BD%BF%E7%94%A8%E6%8C%87%E5%8D%97.md#ohos%E7%BC%96%E8%AF%91)的内容 + +#### 1、libevent的交叉编译 + +* 在服务器的命令行执行下面的命令，下载源码，配置编译链 + +```sh +# 由于zlib也属于第三方软件，可以在opensource目录下进行移植 +cd pegasus/vendor/opensource/ + +wget https://github.com/libevent/libevent/releases/download/release-2.1.12-stable/libevent-2.1.12-stable.tar.gz + +tar -xzvf libevent-2.1.12-stable.tar.gz +rm libevent-2.1.12-stable.tar.gz + +cd libevent-2.1.12-stable + +# 注意：这里的路径请根据自己的Pegasus目录进行修改 +export CC="/home/openharmony/pegasus/os/OpenHarmony/ohos/prebuilts/clang/ohos/linux-x86_64/llvm/bin/aarch64-unknown-linux-ohos-clang --sysroot=/home/openharmony/pegasus/os/OpenHarmony/ohos/out/hispark_ss928v100/ipcamera_hispark_ss928v100_linux/sysroot" +``` + +![image-20251020162138430](pic/image-20251020162138430.png) + +![image-20251020162307165](pic/image-20251020162307165.png) + +* 在服务器的命令行执行下面的命令，修改配置脚本和部分代码，确保编译时不会报错 + +```sh +sed -i 's/| -kaos\*/| -kaos\* | -ohos\*/g' ./build-aux/config.sub +sed -i 's/linux-uclibc\*/linux-uclibc\* | linux-ohos\*/g' ./build-aux/config.sub + +sed -i 's/arc4random_buf/libevent_arc4random_buf/g' ./arc4random.c +sed -i 's/arc4random_buf/libevent_arc4random_buf/g' evutil_rand.c +``` + +![image-20251020162427982](pic/image-20251020162427982.png) + +* 在服务器的命令行执行下面的命令，编译libevent + +```sh +./configure --prefix=$PWD/install --host=aarch64-linux-ohos --disable-openssl + +make && make install +``` + +![image-20251020162558010](pic/image-20251020162558010.png) + +![image-20251020162712286](pic/image-20251020162712286.png) + +* 编译成功后，会在install目录下生成以下文件 + +![image-20251020163215884](pic/image-20251020163215884.png) + +#### 2、tiff的交叉编译 + +* 在服务器的命令行执行下面的命令，进行openssl的交叉编译 + +```sh +cd ../ + +wget http://download.osgeo.org/libtiff/tiff-4.5.1.tar.gz + +tar -xzf tiff-4.5.1.tar.gz +rm tiff-4.5.1.tar.gz + +cd tiff-4.5.1 +``` + +![image-20251020163840912](pic/image-20251020163840912.png) + +* 在./config/config.sub文件中添加OHOS的编译依赖，确保编译时不会报错 + * 在1761行添加 | ohos* + * 在1782行添加 | linux-ohos* + +![image-20251020164239531](pic/image-20251020164239531.png) + +* 在服务器的命令行执行下面的命令，编译ttif + +```sh +./configure --prefix=$PWD/install --host=aarch64-linux-ohos --with-sysroot=/home/openharmony/pegasus/os/OpenHarmony/ohos/out/hispark_ss928v100/ipcamera_hispark_ss928v100_linux/sysroot CFLAGS="--sysroot=/home/openharmony/pegasus/os/OpenHarmony/ohos/out/hispark_ss928v100/ipcamera_hispark_ss928v100_linux/sysroot" CXXFLAGS="--sysroot=/home/openharmony/pegasus/os/OpenHarmony/ohos/out/hispark_ss928v100/ipcamera_hispark_ss928v100_linux/sysroot" + +make && make install +``` + +![image-20251020164718454](pic/image-20251020164718454.png) + +![image-20251020164811557](pic/image-20251020164811557.png) + +* 编译完成后，会在install目录下生成如下的文件 + +![image-20251020165144726](pic/image-20251020165144726.png) + +#### 3、jpeg-9d的交叉编译 + +* 在服务器的命令行执行下面的命令，进行libffi的交叉编译 + +```sh +cd ../ + +wget https://www.ijg.org/files/jpegsrc.v9d.tar.gz +tar -zxvf jpegsrc.v9d.tar.gz + +rm jpegsrc.v9d.tar.gz + +cd jpegsrc.v9d +``` + +![image-20251020165513929](pic/image-20251020165513929.png) + +* 在服务器的命令行执行下面的命令，修改配置脚本和部分代码，确保编译时不会报错 + +```sh +sed -i 's/| -kaos\*/| -kaos\* | -ohos\*/g' config.sub +sed -i 's/linux-uclibc\*/linux-uclibc\* | linux-ohos\*/g' config.sub +``` + +![image-20251020165626871](pic/image-20251020165626871.png) + +* 执行下面的命令，交叉编译jpeg-9d + + +```sh +export CC="/home/openharmony/pegasus/os/OpenHarmony/ohos/prebuilts/clang/ohos/linux-x86_64/llvm/bin/aarch64-unknown-linux-ohos-clang --sysroot=/home/openharmony/pegasus/os/OpenHarmony/ohos/out/hispark_ss928v100/ipcamera_hispark_ss928v100_linux/sysroot" + +./configure --prefix=$PWD/install --host=aarch64-linux-ohos + +make && make install +``` + +![image-20251020165719960](pic/image-20251020165719960.png) + +![image-20251020165802562](pic/image-20251020165802562.png) + +* 编译完成后，会在install目录下生成如下的文件 + +![image-20251020165955883](pic/image-20251020165955883.png) + +#### 2、openssl的交叉编译 + +* 在服务器的命令行执行下面的命令，进行openssl的交叉编译 + +```sh +cd ../ + +wget https://github.com/openssl/openssl/archive/refs/tags/OpenSSL_1_1_1w.tar.gz + +tar -xvf OpenSSL_1_1_1w.tar.gz +rm OpenSSL_1_1_1w.tar.gz + +cd openssl-OpenSSL_1_1_1w + +perl Configure linux-aarch64 --prefix=$PWD/install + +make CC="/home/openharmony/pegasus/os/OpenHarmony/ohos/prebuilts/clang/ohos/linux-x86_64/llvm/bin/aarch64-unknown-linux-ohos-clang --sysroot=/home/openharmony/pegasus/os/OpenHarmony/ohos/out/hispark_ss928v100/ipcamera_hispark_ss928v100_linux/sysroot" LDFLAGS="--sysroot=/home/openharmony/pegasus/os/OpenHarmony/ohos/out/hispark_ss928v100/ipcamera_hispark_ss928v100_linux/sysroot -L/home/openharmony/pegasus/os/OpenHarmony/ohos/out/hispark_ss928v100/ipcamera_hispark_ss928v100_linux/sysroot/usr/lib" + +make install +``` + +![image-20251009173438071](pic/image-20251009173438071.png) + +![image-20251009173659012](pic/image-20251009173659012.png) + +![image-20251009174036856](pic/image-20251009174036856.png) + +![image-20251009174105488](pic/image-20251009174105488.png) + +### 步骤3：安装依赖软件 + +* 在服务器的命令行执行下面的命令，安装依赖软件 + +```sh +apt-get install ninja-build libevent-dev libjpeg-dev + +# 使用apt下载的meson版本太低不符合要求 +pip3 install meson==1.6 jinja2 pyyaml ply pybind11 -i https://pypi.tuna.tsinghua.edu.cn/simple +``` + +![image-20251020170606044](pic/image-20251020170606044.png) + +## 4、交叉编译libcamera + +### 步骤1：下载源码 + +* 在服务器的命令行执行下面的命令，进行libcamera的交叉编译 + +```sh +cd ../ + +git clone https://git.libcamera.org/libcamera/libcamera.git + +cd libcamera +``` + +![image-20251020172340245](pic/image-20251020172340245.png) + +### 步骤2：配置编译环境 + +* 在libcamera目录下创建一个cross_file.txt文件，然后把下面的内容复制进去。 +* 注意：下面内容中涉及到绝对路径的地方，请根据自己服务器的实际情况进行修改 + +```sh +[binaries] +c = '/home/openharmony/pegasus/os/OpenHarmony/ohos/prebuilts/clang/ohos/linux-x86_64/llvm/bin/aarch64-unknown-linux-ohos-clang' +cpp = '/home/openharmony/pegasus/os/OpenHarmony/ohos/prebuilts/clang/ohos/linux-x86_64/llvm/bin/aarch64-unknown-linux-ohos-clang++' +ar = '/home/openharmony/pegasus/os/OpenHarmony/ohos/prebuilts/clang/ohos/linux-x86_64/llvm/bin/llvm-ar' +strip = '/home/openharmony/pegasus/os/OpenHarmony/ohos/prebuilts/clang/ohos/linux-x86_64/llvm/bin/llvm-strip' +pkg-config = '/usr/bin/pkg-config' + +[host_machine] +system = 'linux' +cpu_family = 'aarch64' +cpu = 'aarch64' +endian = 'little' + +[properties] +sys_root = '/home/openharmony/pegasus/os/OpenHarmony/ohos/out/hispark_ss928v100/ipcamera_hispark_ss928v100_linux/sysroot' +libdir = '/home/openharmony/pegasus/os/OpenHarmony/ohos/out/hispark_ss928v100/ipcamera_hispark_ss928v100_linux/sysroot/usr/lib' +includedir = '/home/openharmony/pegasus/os/OpenHarmony/ohos/out/hispark_ss928v100/ipcamera_hispark_ss928v100_linux/sysroot/usr/include' + +[built-in options] +c_args = ['--sysroot=/home/openharmony/pegasus/os/OpenHarmony/ohos/out/hispark_ss928v100/ipcamera_hispark_ss928v100_linux/sysroot', '-isystem', '/home/openharmony/pegasus/os/OpenHarmony/ohos/out/hispark_ss928v100/ipcamera_hispark_ss928v100_linux/sysroot/usr/include'] +cpp_args = ['--sysroot=/home/openharmony/pegasus/os/OpenHarmony/ohos/out/hispark_ss928v100/ipcamera_hispark_ss928v100_linux/sysroot', '-isystem', '/home/openharmony/pegasus/os/OpenHarmony/ohos/prebuilts/clang/ohos/linux-x86_64/llvm/include/libcxx-ohos/include/c++/v1', '-D_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER', '-std=c++17', '-I/home/openharmony/pegasus/vendor/opensource/tiff-4.5.1/install/include', '-I/home/openharmony/pegasus/vendor/opensource/libevent-2.1.12-stable/install/include', '-I/home/openharmony/pegasus/vendor/opensource/openssl-OpenSSL_1_1_1w/install/include', '-I/home/openharmony/pegasus/vendor/opensource/jpeg-9d/install/include',] + +c_link_args = ['--sysroot=/home/openharmony/pegasus/os/OpenHarmony/ohos/out/hispark_ss928v100/ipcamera_hispark_ss928v100_linux/sysroot'] +cpp_link_args = ['--sysroot=/home/openharmony/pegasus/os/OpenHarmony/ohos/out/hispark_ss928v100/ipcamera_hispark_ss928v100_linux/sysroot', '-L/home/openharmony/pegasus/vendor/opensource/tiff-4.5.1/install/lib', '-L/home/openharmony/pegasus/vendor/opensource/libevent-2.1.12-stable/install/lib', '-L/home/openharmony/pegasus/vendor/opensource/openssl-OpenSSL_1_1_1w/install/lib', '-L/home/openharmony/pegasus/vendor/opensource/jpeg-9d/install/lib',] +``` + +* 结合第三章内容，配置依赖软件的环境变量。 +* 注意：下面内容中涉及到绝对路径的地方，请根据自己服务器的实际情况进行修改 + +```sh +export PKG_CONFIG_PATH=$PKG_CONFIG_PATH:/home/openharmony/pegasus/vendor/opensource/tiff-4.5.1/install/lib/pkgconfig:/home/openharmony/pegasus/vendor/opensource/libevent-2.1.12-stable/install/lib/pkgconfig:/home/openharmony/pegasus/vendor/opensource/openssl-OpenSSL_1_1_1w/install/lib/pkgconfig:/home/openharmony/pegasus/vendor/opensource/jpeg-9d/install/lib/pkgconfig +``` + +### 步骤3：修改代码 + +* /home/openharmony/pegasus/os/OpenHarmony/ohos/out/hispark_ss928v100/ipcamera_hispark_ss928v100_linux/sysroot/usr/include/aarch64-linux-ohos/linux/videodev2.h 文件中内容，如图所示，在文件的71行下添加14和15两个参数： + +```c + V4L2_BUF_TYPE_META_OUTPUT = 14, + V4L2_CAP_META_OUTPUT = 15, +``` + +![image-20251021093857925](pic/image-20251021093857925.png) + +* 修改libcamera/src/libcamera/process.cpp，在152行处添加下面内容。 + +```c +struct clone_args { + uint64_t flags; /* Flags bit mask */ + uint64_t pidfd; /* Where to store PID file descriptor*/ + uint64_t child_tid; /* Where to store child TID */ + uint64_t parent_tid; /* Where to store child TID */ + uint64_t exit_signal; /* Signal to deliver to parent on child termination */ + uint64_t stack; /* Pointer to lowest byte of stack */ + uint64_t stack_size; /* Size of stack */ + uint64_t tls; /* Location of new TLS */ +}; +``` + +![image-20251021110807481](pic/image-20251021110807481.png) + +### 步骤4：编译源码 + +* 执行下面的命令，使用meson进行编译前的配置 + +```sh +meson setup \ + --cross-file cross_file.txt \ + --prefix=$(pwd)/install \ + -Dcam=enabled \ + -Ddocumentation=disabled \ + -Dpycamera=enabled \ + build . 2>&1 | tee meson_output.log +``` + +![image-20251021094913442](pic/image-20251021094913442.png) + +* 执行下面的命令，进行源码的编译 + +```sh +cd build + +ninja +``` + +![image-20251021110457253](pic/image-20251021110457253.png) + +### 步骤5：修改报错 + +* 如果在编译的时候出现下面的错误 + +![image-20251021105919871](pic/image-20251021105919871.png) + +* 需要修改两个位置的代码 +* 在libcamera/subprojects/libpisp/src/helpers/media_device.cpp 文件的第行，添加一个头文件 #include + +![image-20251021110324282](pic/image-20251021110324282.png) + +* 在libcamera/subprojects/libpisp/src/helpers/v4l2_device.cpp文件的24~30行，添加下面的内容 + +```c +#ifndef V4L2_PIX_FMT_RGBX32 +#define V4L2_PIX_FMT_RGBX32 v4l2_fourcc('R', 'G', 'B', 'X') +#endif + +#ifndef V4L2_PIX_FMT_BGRX32 +#define V4L2_PIX_FMT_BGRX32 v4l2_fourcc('B', 'G', 'R', 'X') +#endif +``` + +![image-20251021110201136](pic/image-20251021110201136.png) + +* 然后删掉build目录，重新执行下面的命令，编译源码 + +```sh +cd /home/openharmony/pegasus/vendor/opensource/libcamera + +rm build + +meson setup \ + --cross-file cross_file.txt \ + --prefix=$(pwd)/install \ + -Dcam=enabled \ + -Ddocumentation=disabled \ + -Dpycamera=disabled \ + build . 2>&1 | tee meson_output.log + +ninja -C build +ninja -C build install + +cd build + +ninja + +ninja install +``` + +![image-20251021111005830](pic/image-20251021111005830.png) + +* 编译成功后，会在libcamera的install目录生成如下内容 + +![image-20251021111051030](pic/image-20251021111051030.png) + +## 5.板端测试 + +### 步骤1：配置板端环境 + +* 1、确保开发板已经烧录OpenHarmony操作系统 +* 2、使用网线将开发板与你的电脑进行连接，确保二者处于同一局域网内 +* 3、配置开发板的IP地址，并确保开发板与电脑能够互相ping通 + +```sh +# 注意：这里的eth0的IP地址，请根据自己的网络IP网段进行合理配置 +ifconfig eth0 192.168.100.100 + +# 添加权限 +echo 0 9999999 > /proc/sys/net/ipv4/ping_group_range +``` + +### 步骤2：准备libacamera依赖文件 + +* 1、将第4章交叉编译libcamera后，生成的install文件夹拷贝到你的NFS挂载目录（这里把install重命名为libcamera_install） +* 2、将第3章步骤2交叉编译的依赖软件中的libevent\_pthreads-2.1.so.7、libevent-2.1.so.7、libcrypto.so.1.1、libtiff.so.6都下载下来，并复制到libcamera_install的lib目录下 +* 3、如果你想使用Python调用libcamera的接口，需要把libcamera_install/lib/python3.13/site-packages中的 libcamera复制到Python的install/lib/python3.13/site-packages目录下，具体可参考[Python的移植文档](../python/README.md)。 +* 4、将下的这些库都复制到Python的install/lib/python3.13/lib-dynload目录下 + +![image-20251021112836269](pic/image-20251021112836269.png) + +* 5、在开发板的命令行执行下面的命令，将电脑的nfs目录挂载到开发板的/mnt目录下（注意：这里请根据自己的IP地址及NFS配置进行合理的修改） + +```sh +mount -o nolock,addr=192.168.100.10 -t nfs 192.168.100.10:/d/nfs /mnt +``` + +* 6、在开发板的命令行执行下面的命令，配置python的环境变量，确保python运行时能够找到依赖 + +```sh +export PATH=/mnt/install/bin:$PATH +export PYTHONPATH=/mnt/install/lib/python3.13:$PYTHONPATH +export LD_LIBRARY_PATH=/mnt/install/lib/python3.13/lib-dynload:$LD_LIBRARY_PATH +``` + +* 7、在开发板的命令行执行下面的命令，配置libcamera的环境变量，确保libcamera运行时能够找到依赖 + +```sh +export PATH=/mnt/libcamera_install/bin:$PATH +export LD_LIBRARY_PATH=/mnt/libcamera_install/lib:$LD_LIBRARY_PATH +``` + +### 步骤3：运行cam工具 + +* 在开发板的命令行执行下面的命令，给cam添加可执行权限 + +```sh +chmod +x /mnt/libcamera_install/bin/* +``` + +* 运行 cam -c 1 -I 可以查看当前摄像机支持的分辨率和格式： + +![image-20251022111145401](pic/image-20251022111145401.png) + +* 拉取图片： + + ```sh + # 可以通过cam参数 -s 设置图片分辨率、图片格式等等： + cam -c 1 --capture=10 --file=1.jpg + ``` + + ![image-20251022111205861](pic/image-20251022111205861.png) + +* 拉取视频流：（10帧，分辨率1920x1080，格式为YUYV） + + ```sh + cam -c 1 -C10 -s width=1920,height=1080,role=video,pixelformat=YUYV --file=/mnt/1.yuv + ``` + + ![image-20251022111249409](pic/image-20251022111249409.png) + + + + + diff --git a/vendor/zsks/docs/libcamera/pic/image-20251009173438071.png b/vendor/zsks/docs/libcamera/pic/image-20251009173438071.png new file mode 100755 index 0000000000000000000000000000000000000000..abd97d0442d4134130a4102c0bb5be8781218b64 Binary files /dev/null and b/vendor/zsks/docs/libcamera/pic/image-20251009173438071.png differ diff --git a/vendor/zsks/docs/libcamera/pic/image-20251009173659012.png b/vendor/zsks/docs/libcamera/pic/image-20251009173659012.png new file mode 100755 index 0000000000000000000000000000000000000000..d3c95af0bdef74808f59af6922f3c0dd9bdffc26 Binary files /dev/null and b/vendor/zsks/docs/libcamera/pic/image-20251009173659012.png differ diff --git a/vendor/zsks/docs/libcamera/pic/image-20251009174036856.png b/vendor/zsks/docs/libcamera/pic/image-20251009174036856.png new file mode 100755 index 0000000000000000000000000000000000000000..71cbbffc86935e2d55fcf0260092b5ccf46369ba Binary files /dev/null and b/vendor/zsks/docs/libcamera/pic/image-20251009174036856.png differ diff --git a/vendor/zsks/docs/libcamera/pic/image-20251009174105488.png b/vendor/zsks/docs/libcamera/pic/image-20251009174105488.png new file mode 100755 index 0000000000000000000000000000000000000000..1dca90f249e54c3ba3a54f64c14b8236de38642a Binary files /dev/null and b/vendor/zsks/docs/libcamera/pic/image-20251009174105488.png differ diff --git a/vendor/zsks/docs/libcamera/pic/image-20251020160523558.png b/vendor/zsks/docs/libcamera/pic/image-20251020160523558.png new file mode 100755 index 0000000000000000000000000000000000000000..8378f1a983dd85346cdbcdccd9ed1cc9d847c39e Binary files /dev/null and b/vendor/zsks/docs/libcamera/pic/image-20251020160523558.png differ diff --git a/vendor/zsks/docs/libcamera/pic/image-20251020162138430.png b/vendor/zsks/docs/libcamera/pic/image-20251020162138430.png new file mode 100755 index 0000000000000000000000000000000000000000..0050be6b1455dc457f2932004469736148a024b6 Binary files /dev/null and b/vendor/zsks/docs/libcamera/pic/image-20251020162138430.png differ diff --git a/vendor/zsks/docs/libcamera/pic/image-20251020162307165.png b/vendor/zsks/docs/libcamera/pic/image-20251020162307165.png new file mode 100755 index 0000000000000000000000000000000000000000..73247e11f11e955079b6f473a2c0c9fa0955ee2f Binary files /dev/null and b/vendor/zsks/docs/libcamera/pic/image-20251020162307165.png differ diff --git a/vendor/zsks/docs/libcamera/pic/image-20251020162427982.png b/vendor/zsks/docs/libcamera/pic/image-20251020162427982.png new file mode 100755 index 0000000000000000000000000000000000000000..b34ac6ba4f2da8f5aa9160f7a3bc5a36e9fb1791 Binary files /dev/null and b/vendor/zsks/docs/libcamera/pic/image-20251020162427982.png differ diff --git a/vendor/zsks/docs/libcamera/pic/image-20251020162558010.png b/vendor/zsks/docs/libcamera/pic/image-20251020162558010.png new file mode 100755 index 0000000000000000000000000000000000000000..d29fdb0ae0da169210c3db01af1026ea906c94e8 Binary files /dev/null and b/vendor/zsks/docs/libcamera/pic/image-20251020162558010.png differ diff --git a/vendor/zsks/docs/libcamera/pic/image-20251020162712286.png b/vendor/zsks/docs/libcamera/pic/image-20251020162712286.png new file mode 100755 index 0000000000000000000000000000000000000000..bf625e27b3dbc7b523d15ec8e6e494e11617dc78 Binary files /dev/null and b/vendor/zsks/docs/libcamera/pic/image-20251020162712286.png differ diff --git a/vendor/zsks/docs/libcamera/pic/image-20251020163215884.png b/vendor/zsks/docs/libcamera/pic/image-20251020163215884.png new file mode 100755 index 0000000000000000000000000000000000000000..a49d19cd1f2708d12aa72b85642f12804135e08a Binary files /dev/null and b/vendor/zsks/docs/libcamera/pic/image-20251020163215884.png differ diff --git a/vendor/zsks/docs/libcamera/pic/image-20251020163840912.png b/vendor/zsks/docs/libcamera/pic/image-20251020163840912.png new file mode 100755 index 0000000000000000000000000000000000000000..da32bad41287013834a96c2353b46e0c32a0995f Binary files /dev/null and b/vendor/zsks/docs/libcamera/pic/image-20251020163840912.png differ diff --git a/vendor/zsks/docs/libcamera/pic/image-20251020164239531.png b/vendor/zsks/docs/libcamera/pic/image-20251020164239531.png new file mode 100755 index 0000000000000000000000000000000000000000..3a63648e1dd0ee35169fb98f13c90cdbd32e80ee Binary files /dev/null and b/vendor/zsks/docs/libcamera/pic/image-20251020164239531.png differ diff --git a/vendor/zsks/docs/libcamera/pic/image-20251020164718454.png b/vendor/zsks/docs/libcamera/pic/image-20251020164718454.png new file mode 100755 index 0000000000000000000000000000000000000000..30f4bf59e56a131a2d261c0344830625ed73c9c8 Binary files /dev/null and b/vendor/zsks/docs/libcamera/pic/image-20251020164718454.png differ diff --git a/vendor/zsks/docs/libcamera/pic/image-20251020164811557.png b/vendor/zsks/docs/libcamera/pic/image-20251020164811557.png new file mode 100755 index 0000000000000000000000000000000000000000..c9b7c315a17421158dda8ae7d5081895c54793f1 Binary files /dev/null and b/vendor/zsks/docs/libcamera/pic/image-20251020164811557.png differ diff --git a/vendor/zsks/docs/libcamera/pic/image-20251020165144726.png b/vendor/zsks/docs/libcamera/pic/image-20251020165144726.png new file mode 100755 index 0000000000000000000000000000000000000000..668bb41f23e474c4be978ef3d86b40dea06150d9 Binary files /dev/null and b/vendor/zsks/docs/libcamera/pic/image-20251020165144726.png differ diff --git a/vendor/zsks/docs/libcamera/pic/image-20251020165513929.png b/vendor/zsks/docs/libcamera/pic/image-20251020165513929.png new file mode 100755 index 0000000000000000000000000000000000000000..d75be917e4a0cb09677fd4377a1ee0869192d181 Binary files /dev/null and b/vendor/zsks/docs/libcamera/pic/image-20251020165513929.png differ diff --git a/vendor/zsks/docs/libcamera/pic/image-20251020165626871.png b/vendor/zsks/docs/libcamera/pic/image-20251020165626871.png new file mode 100755 index 0000000000000000000000000000000000000000..7722c8825608d4ebb526f249239ba4a8a77d874b Binary files /dev/null and b/vendor/zsks/docs/libcamera/pic/image-20251020165626871.png differ diff --git a/vendor/zsks/docs/libcamera/pic/image-20251020165719960.png b/vendor/zsks/docs/libcamera/pic/image-20251020165719960.png new file mode 100755 index 0000000000000000000000000000000000000000..e0aafce1e2f46463854812033695d833a69b51b9 Binary files /dev/null and b/vendor/zsks/docs/libcamera/pic/image-20251020165719960.png differ diff --git a/vendor/zsks/docs/libcamera/pic/image-20251020165802562.png b/vendor/zsks/docs/libcamera/pic/image-20251020165802562.png new file mode 100755 index 0000000000000000000000000000000000000000..fc7098ed0fc3c16f9d63858f4e0212c20476e9fc Binary files /dev/null and b/vendor/zsks/docs/libcamera/pic/image-20251020165802562.png differ diff --git a/vendor/zsks/docs/libcamera/pic/image-20251020165955883.png b/vendor/zsks/docs/libcamera/pic/image-20251020165955883.png new file mode 100755 index 0000000000000000000000000000000000000000..fc20fcfb1db598b41190b01818e912e9d3574d5c Binary files /dev/null and b/vendor/zsks/docs/libcamera/pic/image-20251020165955883.png differ diff --git a/vendor/zsks/docs/libcamera/pic/image-20251020170606044.png b/vendor/zsks/docs/libcamera/pic/image-20251020170606044.png new file mode 100755 index 0000000000000000000000000000000000000000..3dbd9fd102246666f83409ca8ba17c3950174831 Binary files /dev/null and b/vendor/zsks/docs/libcamera/pic/image-20251020170606044.png differ diff --git a/vendor/zsks/docs/libcamera/pic/image-20251020172340245.png b/vendor/zsks/docs/libcamera/pic/image-20251020172340245.png new file mode 100755 index 0000000000000000000000000000000000000000..11e3393352686482431e1daf743a506f95536931 Binary files /dev/null and b/vendor/zsks/docs/libcamera/pic/image-20251020172340245.png differ diff --git a/vendor/zsks/docs/libcamera/pic/image-20251021093857925.png b/vendor/zsks/docs/libcamera/pic/image-20251021093857925.png new file mode 100755 index 0000000000000000000000000000000000000000..9de66c7aa73004601b9e954737571de19f40f913 Binary files /dev/null and b/vendor/zsks/docs/libcamera/pic/image-20251021093857925.png differ diff --git a/vendor/zsks/docs/libcamera/pic/image-20251021094913442.png b/vendor/zsks/docs/libcamera/pic/image-20251021094913442.png new file mode 100755 index 0000000000000000000000000000000000000000..7029a695cc1b2b9a4e7f180f534e8506fa1a62e3 Binary files /dev/null and b/vendor/zsks/docs/libcamera/pic/image-20251021094913442.png differ diff --git a/vendor/zsks/docs/libcamera/pic/image-20251021105919871.png b/vendor/zsks/docs/libcamera/pic/image-20251021105919871.png new file mode 100755 index 0000000000000000000000000000000000000000..6048787f578fa0534a0c728261b06812c4bdb0de Binary files /dev/null and b/vendor/zsks/docs/libcamera/pic/image-20251021105919871.png differ diff --git a/vendor/zsks/docs/libcamera/pic/image-20251021110201136.png b/vendor/zsks/docs/libcamera/pic/image-20251021110201136.png new file mode 100755 index 0000000000000000000000000000000000000000..0de72baccd3d30ae318b0415d01b4215410d4a64 Binary files /dev/null and b/vendor/zsks/docs/libcamera/pic/image-20251021110201136.png differ diff --git a/vendor/zsks/docs/libcamera/pic/image-20251021110324282.png b/vendor/zsks/docs/libcamera/pic/image-20251021110324282.png new file mode 100755 index 0000000000000000000000000000000000000000..e634d648146fadff8aa1858cb395fd1e6eb20c84 Binary files /dev/null and b/vendor/zsks/docs/libcamera/pic/image-20251021110324282.png differ diff --git a/vendor/zsks/docs/libcamera/pic/image-20251021110457253.png b/vendor/zsks/docs/libcamera/pic/image-20251021110457253.png new file mode 100755 index 0000000000000000000000000000000000000000..d2d6b7589918136834962f0e4c2abe5aed02b481 Binary files /dev/null and b/vendor/zsks/docs/libcamera/pic/image-20251021110457253.png differ diff --git a/vendor/zsks/docs/libcamera/pic/image-20251021110807481.png b/vendor/zsks/docs/libcamera/pic/image-20251021110807481.png new file mode 100755 index 0000000000000000000000000000000000000000..fb9fc5d06865ca44cfe44716f462add1532a3b33 Binary files /dev/null and b/vendor/zsks/docs/libcamera/pic/image-20251021110807481.png differ diff --git a/vendor/zsks/docs/libcamera/pic/image-20251021111005830.png b/vendor/zsks/docs/libcamera/pic/image-20251021111005830.png new file mode 100755 index 0000000000000000000000000000000000000000..0f58a54ac11af46742b5abd2afcd2e6b7551cbe0 Binary files /dev/null and b/vendor/zsks/docs/libcamera/pic/image-20251021111005830.png differ diff --git a/vendor/zsks/docs/libcamera/pic/image-20251021111051030.png b/vendor/zsks/docs/libcamera/pic/image-20251021111051030.png new file mode 100755 index 0000000000000000000000000000000000000000..bd03361a2062a1d81b46ca530ff298ff8b79b665 Binary files /dev/null and b/vendor/zsks/docs/libcamera/pic/image-20251021111051030.png differ diff --git a/vendor/zsks/docs/libcamera/pic/image-20251021112836269.png b/vendor/zsks/docs/libcamera/pic/image-20251021112836269.png new file mode 100755 index 0000000000000000000000000000000000000000..8e9f97abb2e6c1a4809d6cdf61f00c9473bb679b Binary files /dev/null and b/vendor/zsks/docs/libcamera/pic/image-20251021112836269.png differ diff --git a/vendor/zsks/docs/libcamera/pic/image-20251022111145401.png b/vendor/zsks/docs/libcamera/pic/image-20251022111145401.png new file mode 100755 index 0000000000000000000000000000000000000000..2bba43c28bdd4923ec9ed5fd596cea810817474f Binary files /dev/null and b/vendor/zsks/docs/libcamera/pic/image-20251022111145401.png differ diff --git a/vendor/zsks/docs/libcamera/pic/image-20251022111205861.png b/vendor/zsks/docs/libcamera/pic/image-20251022111205861.png new file mode 100755 index 0000000000000000000000000000000000000000..6bc3ea024acc72a132d886c57a5b5289e604ab01 Binary files /dev/null and b/vendor/zsks/docs/libcamera/pic/image-20251022111205861.png differ diff --git a/vendor/zsks/docs/libcamera/pic/image-20251022111249409.png b/vendor/zsks/docs/libcamera/pic/image-20251022111249409.png new file mode 100755 index 0000000000000000000000000000000000000000..bb7b51611cfa2a8a4a3326c7f6b3cf8c27977b87 Binary files /dev/null and b/vendor/zsks/docs/libcamera/pic/image-20251022111249409.png differ diff --git a/vendor/zsks/docs/libv4l2/README.md b/vendor/zsks/docs/libv4l2/README.md new file mode 100644 index 0000000000000000000000000000000000000000..22e4030a1c54f0277d67518f084b94571c228026 --- /dev/null +++ b/vendor/zsks/docs/libv4l2/README.md @@ -0,0 +1,441 @@ +# libv4l2移植 + +## 1、软硬件环境 + +* 开发板：海鸥派 +* 交叉编译工具链：OHOS (dev) clang version 15.0.4 +* 编译链路径：pegasus/os/OpenHarmony/ohos/prebuilts/clang/ohos/linux-x86_64/llvm/bin +* python版本：Python-3.13.2 +* 移植的libv4l2版本：OpenCV-4.13 + +## 2、交叉编译libv4l2 + +### 步骤1：获取源码 + +* 在服务器的命令行，分步执行下面的命令，下载v4l-utils源码 + +~~~bash +git clone https://git.linuxtv.org/v4l-utils.git + +cd v4l-utils +~~~ + +### 步骤2：创建交叉编译配置文件 + +[constants] +toolchain = '/home/openharmony/pegasus/os/OpenHarmony/ohos/prebuilts/clang/ohos/linux-x86_64/llvm/bin' +sysroot = '/home/openharmony/pegasus/os/OpenHarmony/ohos/out/hispark_ss928v100/ipcamera_hispark_ss928v100_linux/sysroot' + +* 在v4l-utils目录下创建一个hisi-cross.txt文件，把下面的内容复制进去并保存。 + +~~~bash +[host_machine] +system = 'linux' +cpu_family = 'aarch64' +cpu = 'aarch64' +endian = 'little' + +[binaries] +c = '/home/openharmony/pegasus/os/OpenHarmony/ohos/prebuilts/clang/ohos/linux-x86_64/llvm/bin/aarch64-unknown-linux-ohos-clang' +cpp = '/home/openharmony/pegasus/os/OpenHarmony/ohos/prebuilts/clang/ohos/linux-x86_64/llvm/bin/aarch64-unknown-linux-ohos-clang++' +ar = '/home/openharmony/pegasus/os/OpenHarmony/ohos/prebuilts/clang/ohos/linux-x86_64/llvm/bin/llvm-ar' +ranlib = '/home/openharmony/pegasus/os/OpenHarmony/ohos/prebuilts/clang/ohos/linux-x86_64/llvm/bin/llvm-ranlib' +strip = '/home/openharmony/pegasus/os/OpenHarmony/ohos/prebuilts/clang/ohos/linux-x86_64/llvm/bin/llvm-strip' +pkg-config = '/usr/bin/pkg-config' + +[properties] +sysroot = '/home/openharmony/pegasus/os/OpenHarmony/ohos/out/hispark_ss928v100/ipcamera_hispark_ss928v100_linux/sysroot' +c_args = ['--sysroot=' + sysroot, '--target=aarch64-linux-ohos', '-I' + sysroot + '/usr/include'] +c_link_args = ['--sysroot=' + sysroot, '--target=aarch64-linux-ohos', '-fuse-ld=lld', '-L' + sysroot + '/usr/lib/aarch64-linux-ohos', '-L' + sysroot + '/lib', '-lc'] +~~~ + +### 步骤3.加载配置并生成构建文件 + +#### argp库交叉编译 + +* 在服务器的命令行分步执行下面的命令，下载argp源码 + +```sh +#获取源码 +wget https://www.lysator.liu.se/~nisse/misc/argp-standalone-1.3.tar.gz +tar xf argp-standalone-1.3.tar.gz +rm argp-standalone-1.3.tar.gz +cd argp-standalone-1.3 +``` + +![image-20251017153708743](pic/image-20251017153708743.png) + +* 在服务器的命令行分步执行下面的命令，配置环境变量 + +```sh +#配置变量 +export SYSROOT=/home/openharmony/pegasus/os/OpenHarmony/ohos/out/hispark_ss928v100/ipcamera_hispark_ss928v100_linux/sysroot +export CC=/home/openharmony/pegasus/os/OpenHarmony/ohos/prebuilts/clang/ohos/linux-x86_64/llvm/bin/aarch64-unknown-linux-ohos-clang +export CXX=/home/openharmony/pegasus/os/OpenHarmony/ohos/prebuilts/clang/ohos/linux-x86_64/llvm/bin/aarch64-unknown-linux-ohos-clang++ +export AR=/home/openharmony/pegasus/os/OpenHarmony/ohos/prebuilts/clang/ohos/linux-x86_64/llvm/bin/llvm-ar +export RANLIB=/home/openharmony/pegasus/os/OpenHarmony/ohos/prebuilts/clang/ohos/linux-x86_64/llvm/bin/llvm-ranlib +export CFLAGS="--sysroot=$SYSROOT" +export CXXFLAGS="--sysroot=$SYSROOT" +export LDFLAGS="--sysroot=$SYSROOT" +``` + +![image-20251017154000326](pic/image-20251017154000326.png) + +* 在服务器的命令行分步执行下面的命令，对argp代码进行编译 + +~~~bash +#配置文件 +./configure \ + --host=aarch64-linux-ohos \ + --prefix=$SYSROOT/usr \ + --disable-shared \ + --enable-static + +#编译安装 +make -j$(nproc) && make install +~~~ + +![image-20251017154359756](pic/image-20251017154359756.png) + +![image-20251017154546402](pic/image-20251017154546402.png) + +* 在服务器的命令行分步执行下面的命令，复制argp.h和libargp.a到sysroot对应目录下 + +~~~bash +# 复制头文件 argp.h 到 sysroot 的 include 目录 +cp ./argp.h $SYSROOT/usr/include/ + +# 复制静态库 libargp.a 到 sysroot 的 lib 目录 +cp ./libargp.a $SYSROOT/usr/lib/ + +# 确保文件权限正确 +chmod 644 $SYSROOT/usr/include/argp.h +chmod 644 $SYSROOT/usr/lib/libargp.a +~~~ + +![image-20251017154814146](pic/image-20251017154814146.png) + +### 步骤4：meson构建 + +* 在服务器的命令行执行下面的命令，进行meson的构建。 +* 注意：如果你之前有参考过numpy或者opencv的移植文档，这里需要退出虚拟环境（source ~/.bashrc）。 + +~~~bash +cd ../ + +# 下载meson +python3 -m pip install meson -i https://pypi.tuna.tsinghua.edu.cn/simple + +cd v4l-utils +~~~ + +* 修改contrib/test/meson.build，将第89~102行代码注释掉 + +![image-20251017160603013](pic/image-20251017160603013.png) + +* 进入到v4l-utils的build目录下 + +~~~bash +# 使用meson进行构建 +meson setup \ + --cross-file hisi-cross.txt \ + --prefix /home/openharmony/pegasus/vendor/opensource/v4l-utils/install \ + -Ddoxygen-doc=disabled \ + -Dudevdir=lib/udev \ + -Djpeg=disabled \ + build . 2>&1 | tee meson_output.log + +cd build + +ninja && ninja install +~~~ + +![image-20251017155057453](pic/image-20251017155057453.png) + +![image-20251017163810406](pic/image-20251017163810406.png) + +* 执行成功后会在v4l-utils/install目录下生成如下文件 + +![image-20251017163852692](pic/image-20251017163852692.png) + +## 3.测试 + +### 3.1 NFS 共享 + +* nfs配置可参考参考链接： + +~~~bash +https://blog.csdn.net/weixin_34326429/article/details/92163791 +~~~ + +### 3.2 挂载NFS + +* 在开发板的命令行终端执行下面的命令，配置IP地址，并进行NFS挂载 + +~~~bash +ifconfig eth0 192.168.137.0 netmask 255.255.252.0 +route add default gw 192.168.137.1 +mount -o nolock,addr=192.168.137.1 -t nfs 192.168.137.1:/nfs /mnt +~~~ + +### 3.3 配置环境变量 + +* 将第2章步骤4生成的install文件下载到本地 + +![image-20250904143920431](pic/image-20250904143920431.png) + +* 为了区分，我将install文件名改为libv4l2_install +* 在开发板的命令行终端执行下面的命令，配置环境变量 + +~~~bash +cd mnt + +export PATH=/mnt/libv4l2_install/bin:$PATH +export LD_LIBRARY_PATH=/mnt/libv4l2_install/lib:$LD_LIBRARY_PATH +chmod +x /mnt/libv4l2_install/bin/v4l2-ctl +~~~ + +### 3.4测试 + +#### 3.4.1 测试v4l2-ctl工具 + +* 测试列出所有设备 + +~~~bash +v4l2-ctl --list-devices +~~~ + +![image-20250904144609428](pic/image-20250904144609428.png) + +* 测试查看usb摄像头支持的图像格式和分辨率 + +~~~bash +v4l2-ctl -d /dev/video0 --list-formats-ext +~~~ + +![image-20250904144720470](pic/image-20250904144720470.png) + +* 测试采集一帧图像 + +~~~bash +v4l2-ctl -d /dev/video0 \ + --set-fmt-video=width=1920,height=1080,pixelformat=MJPG \ + --stream-mmap \ + --stream-count=1 \ + --stream-to=v4l2test.jpg +~~~ + +![image-20250904144926010](pic/image-20250904144926010.png) + +#### 3.4.2 接口调用demo + +* 把下面的内容复制到test_libv4l2.c中，在服务器上编译，得到可执行文件后放到板端执行。 + +~~~bash +#include +#include +#include +#include +#include +#include +#include +#include +#include + +int main(int argc, char *argv[]) { + int fd; + char *dev_name = "/dev/video0"; // 默认视频设备 + struct v4l2_capability cap; + struct v4l2_format fmt; + struct v4l2_buffer buf; + struct v4l2_requestbuffers req; // 新增：用于请求缓冲区 + void *buffer; + + if (argc > 1) { + dev_name = argv[1]; + } + + printf("测试libv4l2是否正常工作...\n"); + printf("尝试打开视频设备: %s\n", dev_name); + + // 打开视频设备 + fd = v4l2_open(dev_name, O_RDWR | O_NONBLOCK, 0); + if (fd < 0) { + perror("无法打开视频设备"); + return EXIT_FAILURE; + } + printf("成功打开视频设备\n"); + + // 查询设备能力 + if (v4l2_ioctl(fd, VIDIOC_QUERYCAP, &cap) < 0) { + perror("无法查询设备能力"); + v4l2_close(fd); + return EXIT_FAILURE; + } + printf("设备信息:\n"); + printf(" 名称: %s\n", cap.card); + printf(" 驱动: %s\n", cap.driver); + printf(" 总线: %s\n", cap.bus_info); + + // 检查是否支持视频捕获和流式IO + if (!(cap.capabilities & V4L2_CAP_VIDEO_CAPTURE)) { + fprintf(stderr, "设备不支持视频捕获\n"); + v4l2_close(fd); + return EXIT_FAILURE; + } + if (!(cap.capabilities & V4L2_CAP_STREAMING)) { + fprintf(stderr, "设备不支持流式IO\n"); + v4l2_close(fd); + return EXIT_FAILURE; + } + + // 设置视频格式 + memset(&fmt, 0, sizeof(fmt)); + fmt.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; + fmt.fmt.pix.width = 640; + fmt.fmt.pix.height = 480; + fmt.fmt.pix.pixelformat = V4L2_PIX_FMT_YUYV; + fmt.fmt.pix.field = V4L2_FIELD_INTERLACED; + + if (v4l2_ioctl(fd, VIDIOC_S_FMT, &fmt) < 0) { + perror("无法设置视频格式"); + v4l2_close(fd); + return EXIT_FAILURE; + } + printf("成功设置视频格式: %dx%d, 格式: YUYV\n", + fmt.fmt.pix.width, fmt.fmt.pix.height); + + // 新增：请求缓冲区（关键步骤） + memset(&req, 0, sizeof(req)); + req.count = 1; // 请求1个缓冲区 + req.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; + req.memory = V4L2_MEMORY_MMAP; // 使用内存映射方式 + + if (v4l2_ioctl(fd, VIDIOC_REQBUFS, &req) < 0) { + perror("无法请求缓冲区"); + v4l2_close(fd); + return EXIT_FAILURE; + } + printf("成功请求 %d 个缓冲区\n", req.count); + + // 查询缓冲区信息 + memset(&buf, 0, sizeof(buf)); + buf.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; + buf.memory = V4L2_MEMORY_MMAP; + buf.index = 0; // 查询第0个缓冲区 + + if (v4l2_ioctl(fd, VIDIOC_QUERYBUF, &buf) < 0) { + perror("无法查询缓冲区"); + v4l2_close(fd); + return EXIT_FAILURE; + } + + // 映射缓冲区 + buffer = v4l2_mmap(NULL, buf.length, PROT_READ | PROT_WRITE, MAP_SHARED, fd, buf.m.offset); + if (buffer == MAP_FAILED) { + perror("无法映射缓冲区"); + v4l2_close(fd); + return EXIT_FAILURE; + } + printf("成功映射缓冲区，大小: %d bytes\n", buf.length); + + // 将缓冲区放入队列 + if (v4l2_ioctl(fd, VIDIOC_QBUF, &buf) < 0) { + perror("无法将缓冲区放入队列"); + v4l2_munmap(buffer, buf.length); + v4l2_close(fd); + return EXIT_FAILURE; + } + + // 开始捕获 + enum v4l2_buf_type type = V4L2_BUF_TYPE_VIDEO_CAPTURE; + if (v4l2_ioctl(fd, VIDIOC_STREAMON, &type) < 0) { + perror("无法开始捕获"); + v4l2_munmap(buffer, buf.length); + v4l2_close(fd); + return EXIT_FAILURE; + } + printf("开始视频捕获...\n"); + + // 等待帧就绪 + fd_set fds; + struct timeval tv; + int r; + + FD_ZERO(&fds); + FD_SET(fd, &fds); + tv.tv_sec = 2; + tv.tv_usec = 0; + + r = select(fd + 1, &fds, NULL, NULL, &tv); + if (r == -1) { + perror("select失败"); + v4l2_ioctl(fd, VIDIOC_STREAMOFF, &type); + v4l2_munmap(buffer, buf.length); + v4l2_close(fd); + return EXIT_FAILURE; + } + if (r == 0) { + fprintf(stderr, "捕获超时\n"); + v4l2_ioctl(fd, VIDIOC_STREAMOFF, &type); + v4l2_munmap(buffer, buf.length); + v4l2_close(fd); + return EXIT_FAILURE; + } + + // 从队列中取出缓冲区 + if (v4l2_ioctl(fd, VIDIOC_DQBUF, &buf) < 0) { + perror("无法从队列中取出缓冲区"); + v4l2_ioctl(fd, VIDIOC_STREAMOFF, &type); + v4l2_munmap(buffer, buf.length); + v4l2_close(fd); + return EXIT_FAILURE; + } + + printf("成功捕获一帧图像，大小: %d bytes\n", buf.bytesused); + + // 停止捕获 + if (v4l2_ioctl(fd, VIDIOC_STREAMOFF, &type) < 0) { + perror("无法停止捕获"); + v4l2_munmap(buffer, buf.length); + v4l2_close(fd); + return EXIT_FAILURE; + } + + // 清理资源 + v4l2_munmap(buffer, buf.length); + v4l2_close(fd); + + printf("所有测试操作完成，libv4l2工作正常!\n"); + return EXIT_SUCCESS; +} + +~~~ + +* 在服务器的命令行执行下面的命令，编译该代码 +* 注意：这里的决定路径请根据自己服务器的实际情况进行填写 + +~~~bash +/home/openharmony/pegasus/os/OpenHarmony/ohos/prebuilts/clang/ohos/linux-x86_64/llvm/bin/aarch64-unknown-linux-ohos-clang -o test_libv4l2 test_libv4l2.c \ +--sysroot=/home/openharmony/pegasus/os/OpenHarmony/ohos/out/hispark_ss928v100/ipcamera_hispark_ss928v100_linux/sysroot \ +-I/home/openharmony/pegasus/vendor/opensource/v4l-utils/install/include \ +-I/home/openharmony/pegasus/vendor/opensource/v4l-utils/lib/include \ +-L/home/openharmony/pegasus/vendor/opensource/v4l-utils/install/lib \ +-lv4l2 +~~~ + +![image-20251017170636644](pic/image-20251017170636644.png) + +~~~bash +#1.将可执行文件test_libv4l2下载并挂载上板端 +cd mnt +#2.添加v4l2的相关环境变量，参考5.3配置环境变量 +chmod +x test_libv4l2 +#3.运行测试 +./test_libv4l2 +~~~ + +![image-20251017170534770](pic/image-20251017170534770.png) + + + diff --git a/vendor/zsks/docs/libv4l2/pic/image-20250904143920431.png b/vendor/zsks/docs/libv4l2/pic/image-20250904143920431.png new file mode 100755 index 0000000000000000000000000000000000000000..bbba9d4935ea43429a600ec64dfe14119f90a735 Binary files /dev/null and b/vendor/zsks/docs/libv4l2/pic/image-20250904143920431.png differ diff --git a/vendor/zsks/docs/libv4l2/pic/image-20250904144609428.png b/vendor/zsks/docs/libv4l2/pic/image-20250904144609428.png new file mode 100755 index 0000000000000000000000000000000000000000..ff29749d89fab1166b46eee624e72297948d1b94 Binary files /dev/null and b/vendor/zsks/docs/libv4l2/pic/image-20250904144609428.png differ diff --git a/vendor/zsks/docs/libv4l2/pic/image-20250904144720470.png b/vendor/zsks/docs/libv4l2/pic/image-20250904144720470.png new file mode 100755 index 0000000000000000000000000000000000000000..e0529247489c0470c3d147c268bb5d97e59565f1 Binary files /dev/null and b/vendor/zsks/docs/libv4l2/pic/image-20250904144720470.png differ diff --git a/vendor/zsks/docs/libv4l2/pic/image-20250904144926010.png b/vendor/zsks/docs/libv4l2/pic/image-20250904144926010.png new file mode 100755 index 0000000000000000000000000000000000000000..8d0b234ad3f45ea454352a050b887c07d72e66b4 Binary files /dev/null and b/vendor/zsks/docs/libv4l2/pic/image-20250904144926010.png differ diff --git a/vendor/zsks/docs/libv4l2/pic/image-20251017153708743.png b/vendor/zsks/docs/libv4l2/pic/image-20251017153708743.png new file mode 100755 index 0000000000000000000000000000000000000000..8c893c832ee47793ac777c6f01adc6745217e3e5 Binary files /dev/null and b/vendor/zsks/docs/libv4l2/pic/image-20251017153708743.png differ diff --git a/vendor/zsks/docs/libv4l2/pic/image-20251017154000326.png b/vendor/zsks/docs/libv4l2/pic/image-20251017154000326.png new file mode 100755 index 0000000000000000000000000000000000000000..c19aaed11960beb276f707cd09d86f14afe9c68a Binary files /dev/null and b/vendor/zsks/docs/libv4l2/pic/image-20251017154000326.png differ diff --git a/vendor/zsks/docs/libv4l2/pic/image-20251017154359756.png b/vendor/zsks/docs/libv4l2/pic/image-20251017154359756.png new file mode 100755 index 0000000000000000000000000000000000000000..d6acb9d4d038ee0d099762a3df3e149ee8ba10b5 Binary files /dev/null and b/vendor/zsks/docs/libv4l2/pic/image-20251017154359756.png differ diff --git a/vendor/zsks/docs/libv4l2/pic/image-20251017154546402.png b/vendor/zsks/docs/libv4l2/pic/image-20251017154546402.png new file mode 100755 index 0000000000000000000000000000000000000000..cf70a7e1ea2cae27fb955e2876beef5e73d8224a Binary files /dev/null and b/vendor/zsks/docs/libv4l2/pic/image-20251017154546402.png differ diff --git a/vendor/zsks/docs/libv4l2/pic/image-20251017154814146.png b/vendor/zsks/docs/libv4l2/pic/image-20251017154814146.png new file mode 100755 index 0000000000000000000000000000000000000000..897aff759d0b5fe56a37a07e4a49b36435b3d644 Binary files /dev/null and b/vendor/zsks/docs/libv4l2/pic/image-20251017154814146.png differ diff --git a/vendor/zsks/docs/libv4l2/pic/image-20251017155057453.png b/vendor/zsks/docs/libv4l2/pic/image-20251017155057453.png new file mode 100755 index 0000000000000000000000000000000000000000..e238beb06f5522b7c3ed3e29a2be4188b5b74724 Binary files /dev/null and b/vendor/zsks/docs/libv4l2/pic/image-20251017155057453.png differ diff --git a/vendor/zsks/docs/libv4l2/pic/image-20251017160603013.png b/vendor/zsks/docs/libv4l2/pic/image-20251017160603013.png new file mode 100755 index 0000000000000000000000000000000000000000..9e43a76e1d7c5402a3688dcd3db9ddf1515f1961 Binary files /dev/null and b/vendor/zsks/docs/libv4l2/pic/image-20251017160603013.png differ diff --git a/vendor/zsks/docs/libv4l2/pic/image-20251017163810406.png b/vendor/zsks/docs/libv4l2/pic/image-20251017163810406.png new file mode 100755 index 0000000000000000000000000000000000000000..f758eb88dfeab772a2de52eb1f25bd30c0fa68c0 Binary files /dev/null and b/vendor/zsks/docs/libv4l2/pic/image-20251017163810406.png differ diff --git a/vendor/zsks/docs/libv4l2/pic/image-20251017163852692.png b/vendor/zsks/docs/libv4l2/pic/image-20251017163852692.png new file mode 100755 index 0000000000000000000000000000000000000000..8741b85f5f948bf38fce2ae59028cea3e79ee5ed Binary files /dev/null and b/vendor/zsks/docs/libv4l2/pic/image-20251017163852692.png differ diff --git a/vendor/zsks/docs/libv4l2/pic/image-20251017170534770.png b/vendor/zsks/docs/libv4l2/pic/image-20251017170534770.png new file mode 100755 index 0000000000000000000000000000000000000000..a2455a87b302ed14a2dfffa395db5ebf116dae08 Binary files /dev/null and b/vendor/zsks/docs/libv4l2/pic/image-20251017170534770.png differ diff --git a/vendor/zsks/docs/libv4l2/pic/image-20251017170636644.png b/vendor/zsks/docs/libv4l2/pic/image-20251017170636644.png new file mode 100755 index 0000000000000000000000000000000000000000..934bd3d01462445d61aafb08c7a24c8a4cedd9fe Binary files /dev/null and b/vendor/zsks/docs/libv4l2/pic/image-20251017170636644.png differ diff --git a/vendor/zsks/docs/numpy/README.md b/vendor/zsks/docs/numpy/README.md new file mode 100755 index 0000000000000000000000000000000000000000..785a885732605f5c8f0559e2079866679320897b --- /dev/null +++ b/vendor/zsks/docs/numpy/README.md @@ -0,0 +1,271 @@ +# numpy移植 + +## 1、软硬件环境 + +开发板：海鸥派 +交叉编译工具链：OHOS (dev) clang version 15.0.4 + +编译链路径：pegasus/os/OpenHarmony/ohos/prebuilts/clang/ohos/linux-x86_64/llvm/bin + +python版本：Python-3.13.2 + +移植的numpy版本：numpy-v2.2.5 + +## 2、交叉编译numpy + +### 步骤1：下载numpy源码 + +* 分步执行下面的命令，下载Numpy源码 + +```sh +cd pegasus/vendor/opensource + +git clone https://github.com/numpy/numpy.git + +cd numpy + +# 加载子模块 +git submodule update --init + +# 切到自己想要的版本，我这里选择v2.2.5 +git checkout v2.2.5 +``` + +![image-20251010105452576](pic/image-20251010105452576.png) + +![image-20251010105525630](pic/image-20251010105525630.png) + +### 步骤2：配置编译脚本 + +* 在numpy源码根目录下创建一个ohos-build.meson.ini 文件，然后把下面的内容复制进去并保存。 + +```ini +# toolchain和sysroot路径，请根据自己服务器的实际路径进行填写 + +[constants] +toolchain = '/home/openharmony/pegasus/os/OpenHarmony/ohos/prebuilts/clang/ohos/linux-x86_64/llvm/bin' +sysroot = '/home/openharmony/pegasus/os/OpenHarmony/ohos/out/hispark_ss928v100/ipcamera_hispark_ss928v100_linux/sysroot' +host_cpu = 'aarch64' +host_arch = 'aarch64' +common_flags = ['--sysroot=' + sysroot, '--target=' + host_cpu + '-linux-ohos'] + +[built-in options] +c_args = common_flags +cpp_args = common_flags +c_link_args = common_flags +cpp_link_args = common_flags + +[properties] +sizeof_long_double = 8 +longdouble_format = 'IEEE_DOUBLE_LE' + +[binaries] +c = toolchain / 'aarch64-unknown-linux-ohos-clang' +cpp = toolchain / 'aarch64-unknown-linux-ohos-clang++' +# 这里的python为虚拟环境的python路径 +python = '/home/openharmony/pegasus/vendor/opensource/Python-3.13.2/install/include/python3.13' +cython = '' +cython3 = cython +as = toolchain / 'llvm-as' +ld = toolchain / 'ld.lld' +c_ld = ld +cpp_ld = ld +lld = toolchain / 'ld.lld' +strip = toolchain / 'llvm-strip' +ranlib = toolchain / 'llvm-ranlib' +objdump = toolchain / 'llvm-objdump' +objcopy = toolchain / 'llvm-objcopy' +readelf = toolchain / 'llvm-readelf' +nm = toolchain / 'llvm-nm' +ar = toolchain / 'llvm-ar' +profdata = toolchain / 'llvm-profdata' + +[host_machine] +system = 'ohos' +kernel = 'linux' +cpu_family = host_cpu +cpu = host_cpu +endian = 'little' +``` + +### 步骤3：配置编译虚拟环境 + +* 创建crossenv虚拟环境，用于Numpy的编译 + +```sh +# 下载ninja +apt-get install ninja-build + +# 下载cpython +pip3 install cython -i https://pypi.tuna.tsinghua.edu.cn/simple + +# 下载crossenv +pip3 install crossenv + +# 在Python-3.13.2目录下创建crossenv虚拟环境 +# /home/openharmony/pegasus/vendor/opensource/Python-3.13.2/install 为交叉编译后的python路径 +cd Python-3.13.2 +python3 -m crossenv /home/openharmony/pegasus/vendor/opensource/Python-3.13.2/install/bin/python3 crossenv_aarch64 + +# 激活环境 +. crossenv_aarch64/bin/activate + +cd ../numpy + +# 配置 VENDORED_MESON,具体路径请根据自己numpy实际路径进行配置 + +VENDORED_MESON=/home/openharmony/pegasus/vendor/opensource/numpy/vendored-meson/meson/meson.py +python ${VENDORED_MESON} setup --reconfigure --prefix=$PWD/install --cross-file ./ohos-build.meson.ini build-ohos + +cd build-ohos + +# 编译Numpy +python ${VENDORED_MESON} compile + +# 安装 numpy +python ${VENDORED_MESON} install +``` + +![image-20251010154244943](pic/image-20251010154244943.png) + +![image-20251010155502489](pic/image-20251010155502489.png) + +![image-20251010155613680](pic/image-20251010155613680.png) + +![image-20251010155721678](pic/image-20251010155721678.png) + +![image-20251010155900494](pic/image-20251010155900494.png) + +* 生成的numpy安装包在numpy/install/lib/python3.13/site-packages/numpy/目录下 + +![image-20251010160053388](pic/image-20251010160053388.png) + + + +## 3、板端测试 + +### 步骤1：配置板端环境 + +* 1、确保开发板已经烧录OpenHarmony操作系统 +* 2、使用网线将开发板与你的电脑进行连接，确保二者处于同一局域网内 +* 3、配置开发板的IP地址，并确保开发板与电脑能够互相ping通 + +```sh +# 注意：这里的eth0的IP地址，请根据自己的网络IP网段进行合理配置 +ifconfig eth0 192.168.100.100 + +# 添加权限 +echo 0 9999999 > /proc/sys/net/ipv4/ping_group_range +``` + +![image-20251010094441549](pic/image-20251010094441549.png) + +### 步骤2：准备python依赖文件 + +* 1、将python移植第4章 交叉编译python3.13.2后，生成的install文件夹拷贝到你的NFS挂载目录 +* 2、再将numpy/install/lib/python3.13/site-packages下的numpy复制到install/lib/目录下。 +* 3、再根据python第三章移植的内容，将libz.so.1、 libssl.so.1.1 、 libcrypto.so.1.1文件复制到install/lib/python3.13/lib-dynload目录下 + +* 4、执行下面的，命令将电脑的nfs目录挂载到开发板的/mnt目录下（注意：这里请根据自己的IP地址及NFS配置进行合理的修改） + +```mk1.sh +mount -o nolock,addr=192.168.100.10 -t nfs 192.168.100.10:/d/nfs /mnt +``` + +![image-20251010163427786](pic/image-20251010163427786.png) + +![image-20251010161646562](pic/image-20251010161646562.png) + +* 3、配置python的环境变量，确保python运行时能够找到依赖 + +```mk1.sh +export PATH=/mnt/install/bin:$PATH +export PYTHONPATH=/mnt/install/lib/python3.13:$PYTHONPATH +export LD_LIBRARY_PATH=/mnt/install/lib/python3.13/lib-dynload:$LD_LIBRARY_PATH +``` + +image-20251010101732852 + +### 步骤3：使用python调用numpy接口: + +* 先输入python，进入板端python环境，然后输入import numpy 敲回车，如果没有任何报错，说明numpy移植成功 + +![image-20251010162222908](pic/image-20251010162222908.png) + +* 将下面的内容复制到numpy_test.py中 + +```python +import numpy as np +import time + +def test_numpy_functionality(): + """测试 NumPy 的基本功能""" + print("=" * 50) + print("NumPy 功能测试") + print("=" * 50) + + # 1. 创建数组 + arr1 = np.array([1, 2, 3, 4, 5]) + arr2 = np.arange(0, 10, 2) # 类似 range，但生成数组 + arr3 = np.zeros((3, 3)) # 3x3 零矩阵 + arr4 = np.random.rand(2, 2) # 2x2 随机矩阵 + + print("\n1. 数组创建:") + print(f"arr1: {arr1}") + print(f"arr2 (0到10步长2): {arr2}") + print(f"arr3 (3x3零矩阵):\n{arr3}") + print(f"arr4 (2x2随机矩阵):\n{arr4}") + + # 2. 数学运算 + print("\n2. 数学运算:") + print(f"arr1 + 10: {arr1 + 10}") # 广播机制 + print(f"arr1 * arr2 (前5项): {arr1 * arr2[:5]}") + print(f"sin(arr2): {np.sin(arr2)}") # 三角函数 + + # 3. 矩阵乘法 + matrix_a = np.array([[1, 2], [3, 4]]) + matrix_b = np.array([[5, 6], [7, 8]]) + dot_product = np.dot(matrix_a, matrix_b) # 矩阵乘法 + print("\n3. 矩阵乘法 (np.dot):") + print(f"matrix_a:\n{matrix_a}") + print(f"matrix_b:\n{matrix_b}") + print(f"结果:\n{dot_product}") + + # 4. 性能对比：NumPy vs 纯Python + print("\n4. 性能对比 (计算1百万个数的平方):") + size = 1_000_000 + + # 纯Python + py_list = list(range(size)) + start = time.time() + py_result = [x ** 2 for x in py_list] + py_time = time.time() - start + + # NumPy + np_arr = np.arange(size) + start = time.time() + np_result = np_arr ** 2 + np_time = time.time() - start + + print(f"纯Python耗时: {py_time:.5f} 秒") + print(f"NumPy耗时: {np_time:.5f} 秒") + print(f"NumPy比Python快 {py_time / np_time:.1f} 倍!") + + # 5. 高级功能：条件筛选 + print("\n5. 条件筛选 (找出arr2中大于5的数):") + filtered = arr2[arr2 > 5] + print(f"原始数组: {arr2}") + print(f"筛选结果: {filtered}") + +if __name__ == "__main__": + test_numpy_functionality() +``` + +* 然后执行下面的命令，运行numpy测试代码 + +```sh +python3 numpy_test.py +``` + +![image-20251010163638261](pic/image-20251010163638261.png) + diff --git a/vendor/zsks/docs/numpy/pic/image-20251010094441549.png b/vendor/zsks/docs/numpy/pic/image-20251010094441549.png new file mode 100755 index 0000000000000000000000000000000000000000..e981001ec636db595d2766310bf95a49e58f3914 Binary files /dev/null and b/vendor/zsks/docs/numpy/pic/image-20251010094441549.png differ diff --git a/vendor/zsks/docs/numpy/pic/image-20251010105452576.png b/vendor/zsks/docs/numpy/pic/image-20251010105452576.png new file mode 100755 index 0000000000000000000000000000000000000000..220d510fc77baff9461ee6909320d22b21c44fb1 Binary files /dev/null and b/vendor/zsks/docs/numpy/pic/image-20251010105452576.png differ diff --git a/vendor/zsks/docs/numpy/pic/image-20251010105525630.png b/vendor/zsks/docs/numpy/pic/image-20251010105525630.png new file mode 100755 index 0000000000000000000000000000000000000000..7823d256141b5acfaf95d2775898fb64b87594d6 Binary files /dev/null and b/vendor/zsks/docs/numpy/pic/image-20251010105525630.png differ diff --git a/vendor/zsks/docs/numpy/pic/image-20251010154244943.png b/vendor/zsks/docs/numpy/pic/image-20251010154244943.png new file mode 100755 index 0000000000000000000000000000000000000000..fb667f2474dd511d568e3a2a444911ad04ee7080 Binary files /dev/null and b/vendor/zsks/docs/numpy/pic/image-20251010154244943.png differ diff --git a/vendor/zsks/docs/numpy/pic/image-20251010155502489.png b/vendor/zsks/docs/numpy/pic/image-20251010155502489.png new file mode 100755 index 0000000000000000000000000000000000000000..3de9e193106f68677ae53a9ebc46d4183b0626b2 Binary files /dev/null and b/vendor/zsks/docs/numpy/pic/image-20251010155502489.png differ diff --git a/vendor/zsks/docs/numpy/pic/image-20251010155613680.png b/vendor/zsks/docs/numpy/pic/image-20251010155613680.png new file mode 100755 index 0000000000000000000000000000000000000000..e06097a6cf6445f85f16753a9db00dcd7f9f0c92 Binary files /dev/null and b/vendor/zsks/docs/numpy/pic/image-20251010155613680.png differ diff --git a/vendor/zsks/docs/numpy/pic/image-20251010155721678.png b/vendor/zsks/docs/numpy/pic/image-20251010155721678.png new file mode 100755 index 0000000000000000000000000000000000000000..ebce63b3c6256a1a9e1d9df76674ad6d87157ffa Binary files /dev/null and b/vendor/zsks/docs/numpy/pic/image-20251010155721678.png differ diff --git a/vendor/zsks/docs/numpy/pic/image-20251010155900494.png b/vendor/zsks/docs/numpy/pic/image-20251010155900494.png new file mode 100755 index 0000000000000000000000000000000000000000..2da7eb24782e381b745c5318c352671ff5a1a6a7 Binary files /dev/null and b/vendor/zsks/docs/numpy/pic/image-20251010155900494.png differ diff --git a/vendor/zsks/docs/numpy/pic/image-20251010160053388.png b/vendor/zsks/docs/numpy/pic/image-20251010160053388.png new file mode 100755 index 0000000000000000000000000000000000000000..18d4e661ba770e0d0399ee08c98150eab65cae15 Binary files /dev/null and b/vendor/zsks/docs/numpy/pic/image-20251010160053388.png differ diff --git a/vendor/zsks/docs/numpy/pic/image-20251010161646562.png b/vendor/zsks/docs/numpy/pic/image-20251010161646562.png new file mode 100755 index 0000000000000000000000000000000000000000..17728a71c2f2e11d52b0905b0aa2ae5a103d9f96 Binary files /dev/null and b/vendor/zsks/docs/numpy/pic/image-20251010161646562.png differ diff --git a/vendor/zsks/docs/numpy/pic/image-20251010162222908.png b/vendor/zsks/docs/numpy/pic/image-20251010162222908.png new file mode 100755 index 0000000000000000000000000000000000000000..21a9bb31eede98c9394c4b923c3b6fd7ca27b8fa Binary files /dev/null and b/vendor/zsks/docs/numpy/pic/image-20251010162222908.png differ diff --git a/vendor/zsks/docs/numpy/pic/image-20251010163427786.png b/vendor/zsks/docs/numpy/pic/image-20251010163427786.png new file mode 100755 index 0000000000000000000000000000000000000000..15763c53409aed093869168bc0c2916e81b9df5c Binary files /dev/null and b/vendor/zsks/docs/numpy/pic/image-20251010163427786.png differ diff --git a/vendor/zsks/docs/numpy/pic/image-20251010163638261.png b/vendor/zsks/docs/numpy/pic/image-20251010163638261.png new file mode 100755 index 0000000000000000000000000000000000000000..b6640ac996ba6fd27c69be5b56e6a19b6222cd0f Binary files /dev/null and b/vendor/zsks/docs/numpy/pic/image-20251010163638261.png differ diff --git a/vendor/zsks/docs/opencv/README.md b/vendor/zsks/docs/opencv/README.md new file mode 100755 index 0000000000000000000000000000000000000000..34950cfe49d680d2cc16b0f999b82ae3012ce96e --- /dev/null +++ b/vendor/zsks/docs/opencv/README.md @@ -0,0 +1,380 @@ +# OpenCV移植 + +## 1、软硬件环境 + +开发板：海鸥派 +交叉编译工具链：OHOS (dev) clang version 15.0.4 + +编译链路径：pegasus/os/OpenHarmony/ohos/prebuilts/clang/ohos/linux-x86_64/llvm/bin + +python版本：Python-3.13.2 + +移植的OpenCV版本：OpenCV-4.13 + +## 2、交叉编译OpenCV + +### 步骤1：安装依赖软件 + +* 在服务器的命令行执行下面的命令，安装OpenCV交叉编译是的依赖软件 + +```sh +apt-get install cmake libgtk2.0-dev pkg-config libavcodec-dev libavformat-dev libswscale-dev python3-dev python3-numpy libdc1394-dev libtbb2 libtbb-dev libjpeg-dev libpng-dev libtiff-dev -y +``` + +![image-20251013144111854](pic/image-20251013144111854.png) + +* 先进入到python目录下，确保虚拟环境已经开启，关于如何搭建虚拟环境，可参考[numpy移植文档第2章的步骤3](../numpy/README.md)。 + +```sh +cd opensource/Python-3.13.2 + +. crossenv_aarch64/bin/activate +``` + +![image-20251016145037570](pic/image-20251016145037570.png) + +### 步骤2：下载OpenCV源码 + +```sh +cd ../ + +git clone https://github.com/opencv/opencv.git + +cd opencv +``` + +### 步骤3：运行脚本 + +* 进入opencv源码并创建build文件夹： + +```sh +mkdir build + +cd build +``` + +* 把下面的内容复制到build_opencv.sh中： + +```sh +#!/bin/bash +set -e +BuildDir=. +ToolChain=/home/openharmony/pegasus/os/OpenHarmony/ohos/prebuilts/clang/ohos/linux-x86_64/llvm/bin +SYSROOT=/home/openharmony/pegasus/os/OpenHarmony/ohos/out/hispark_ss928v100/ipcamera_hispark_ss928v100_linux/sysroot +if [ ! -d "$BuildDir" ]; then + echo "create ${BuildDir}..." + mkdir -p ${BuildDir} +fi +cd ${BuildDir} + +echo "building OpenCV4" + +cmake -D CMAKE_BUILD_TYPE=RELEASE \ + -D BUILD_SHARED_LIBS=ON \ + -D CMAKE_FIND_ROOT_PATH=${ToolChain}/ \ + -D CMAKE_SYSROOT=${SYSROOT} \ + -D CMAKE_TOOLCHAIN_FILE=../platforms/linux/aarch64-gnu.toolchain.cmake \ + -D CMAKE_C_COMPILER=${ToolChain}/aarch64-unknown-linux-ohos-clang \ + -D CMAKE_CXX_COMPILER=${ToolChain}/aarch64-unknown-linux-ohos-clang++ \ + -D CMAKE_INSTALL_PREFIX=${BuildDir}/install \ + -D WITH_TBB=ON \ + -D WITH_EIGEN=ON \ + -D BUILD_ZLIB=ON \ + -D BUILD_TIFF=ON \ + -D BUILD_JASPER=ON \ + -D BUILD_JPEG=ON \ + -D BUILD_PNG=ON \ + -D WITH_LIBV4L=ON \ + -D BUILD_opencv_python=ON \ + # 这里的python为交叉编译成功后的python路径 + -D PYTHON3_INCLUDE_PATH=/home/openharmony/pegasus/vendor/opensource/Python-3.13.2/install/include/python3.13 \ + -D PYTHON3_NUMPY_INCLUDE_DIRS=/home/openharmony/pegasus/vendor/opensource/numpy/install/lib/python3.13/site-packages/numpy/_core/include \ + -D ENABLE_PRECOMPILED_HEADERS=OFF \ + -D BUILD_EXAMPLES=OFF \ + -D BUILD_TESTS=OFF \ + -D BUILD_PERF_TESTS=OFF \ + -D BUILD_WITH_DEBUG_INFO=OFF \ + -D BUILD_DOCS=OFF \ + -D WITH_OPENCL=OFF \ + -D WITH_1394=OFF \ + ../ + +make -j$(nproc) +``` + +![image-20251013150324179](pic/image-20251013150324179.png) + +* 出现下面的打印信息，说明OpenCV交叉编译成功： + +![image-20251013150632408](pic/image-20251013150632408.png) + +* 执行下面的命令，进行OpenCV的安装： + +```sh +make install +``` + +![image-20251013150714220](pic/image-20251013150714220.png) + +* 安装成功后，会在install目录下生成以下文件。 + +![image-20251013150935466](pic/image-20251013150935466.png) + +## 3、使用python调用OpenCV接口 + +* 1、将python移植第4章 交叉编译python3.13.2后，生成的install文件夹拷贝到你的NFS挂载目录 +* 2、再将opencv/build/install/lib下的库文件复制到install/lib/python3.13/lib-dynload目录下。 +* 3、再根据python第三章移植的内容，将libz.so.1、 libssl.so.1.1 、 libcrypto.so.1.1文件复制到install/lib/python3.13/lib-dynload目录下 + +![image-20251013164259362](pic/image-20251013164259362.png) + +* 4、将opencv/build/install/lib/python3.13/site-packages/cv2 复制到 install/lib/python3.13/site-packages目录下 + +![image-20251013181456311](pic/image-20251013181456311.png) + +* 5、执行下面的，命令将电脑的nfs目录挂载到开发板的/mnt目录下，然后配置环境变量 + +```sh +# 注意：这里的eth0的IP地址，请根据自己的网络IP网段进行合理配置 +ifconfig eth0 192.168.100.100 + +mount -o nolock,addr=192.168.100.10 -t nfs 192.168.100.10:/d/nfs /mnt + +export PATH=/mnt/install/bin:$PATH +export PYTHONPATH=/mnt/install/lib/python3.13:$PYTHONPATH +export LD_LIBRARY_PATH=/mnt/install/lib/python3.13/lib-dynload:$LD_LIBRARY_PATH +``` + +![image-20251013181928810](pic/image-20251013181928810.png) + +* 在开发板的命令行执行下面的命令，进入python环境，导入cv2 + +```sh +cd /mnt/install/bin + +python3 +``` + +* 在python3环境输入import cv2，如果没有任何报错，说明opencv接口调用成功 + +![image-20251016151507324](pic/image-20251016151507324.png) + +* 将下面的内容复制到opencv_test.py文件中，测试基本功能。 + +```mk1.sh +import cv2 +import numpy as np +import os + +def test_opencv_without_ui(): + """无UI的OpenCV功能测试（通过文件保存和日志输出验证）""" + print("=" * 50) + print("OpenCV 无UI测试") + print("=" * 50) + + # 创建临时测试目录 + os.makedirs("opencv_test_output", exist_ok=True) + print("[日志] 创建输出目录: opencv_test_output") + + # 1. 生成测试图像 + test_image = np.zeros((200, 200, 3), dtype=np.uint8) + cv2.rectangle(test_image, (50, 50), (150, 150), (0, 255, 0), 2) + cv2.putText(test_image, "OpenCV Test", (30, 30), + cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 255), 1) + cv2.imwrite("opencv_test_output/test_image.png", test_image) + print("[日志] 生成测试图像: test_image.png") + + # 2. 图像处理（灰度化+边缘检测） + gray_image = cv2.cvtColor(test_image, cv2.COLOR_BGR2GRAY) + edges = cv2.Canny(gray_image, 100, 200) + cv2.imwrite("opencv_test_output/edges.png", edges) + print("[日志] 生成边缘检测结果: edges.png") + + # 3. 矩阵运算测试（修复：转换为float32） + mat_a = np.random.rand(3, 3).astype(np.float32) # 改为浮点矩阵 + mat_b = np.random.rand(3, 3).astype(np.float32) + mat_mult = cv2.gemm(mat_a, mat_b, 1, None, 0) + print("[矩阵运算] A * B = \n", mat_mult) + + # 4. 摄像头测试（无UI模式） + cap = cv2.VideoCapture(0) + if not cap.isOpened(): + print("[警告] 未检测到摄像头，跳过摄像头测试") + else: + ret, frame = cap.read() + if ret: + cv2.imwrite("opencv_test_output/camera_capture.png", frame) + print("[日志] 摄像头捕获成功: camera_capture.png") + cap.release() + + # 5. 特征检测（ORB） + orb = cv2.ORB_create() + kp = orb.detect(gray_image, None) + kp_image = cv2.drawKeypoints(gray_image, kp, None, color=(0, 255, 0)) + cv2.imwrite("opencv_test_output/keypoints.png", kp_image) + print("[日志] 生成特征点检测结果: keypoints.png") + print(f"[特征检测] 共检测到 {len(kp)} 个特征点") + + # 6. 性能测试 + start_time = cv2.getTickCount() + for _ in range(100): + _ = cv2.blur(test_image, (5, 5)) + end_time = cv2.getTickCount() + print(f"[性能] 100次模糊运算耗时: {(end_time - start_time)/cv2.getTickFrequency():.4f}秒") + + print("\n测试完成！结果已保存至 opencv_test_output 目录") + +if __name__ == "__main__": + test_opencv_without_ui() +``` + +* 在开发板的命令行执行下面的命令，运行opencv_test.py + +```sh +python3 opencv_test.py +``` + +![image-20251016151851058](pic/image-20251016151851058.png) + +![image-20251016151915487](pic/image-20251016151915487.png) + +## 4、测试Opencv调用usb摄像头 + +* 将下面的内容复制到opencv_usb_camera_test.py文件中，测试opencv调用usb摄像头的基本功能。 + +```mk1.sh +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +import cv2 +import time + +def main(device_index=0, num_frames=100, backend=cv2.CAP_V4L2): + """ + 在无 UI 情况下从 USB 摄像头读取 num_frames 帧并打印每帧的尺寸。 + + :param device_index: 摄像头索引，0 表示第一个设备 + :param num_frames: 要读取的帧数 + :param backend: OpenCV 后端，可选 CAP_V4L2、CAP_ANY 等 + """ + # 打开摄像头 + cap = cv2.VideoCapture(device_index, backend) + if not cap.isOpened(): + print(f"无法打开摄像头 (index={device_index})") + return + + print("开始读取帧…按 Ctrl+C 可中断") + + try: + for i in range(num_frames): + ret, frame = cap.read() + if not ret: + print(f"第 {i} 帧读取失败，退出") + break + + # 示例处理：打印帧的分辨率 + h, w = frame.shape[:2] + print(f"帧 {i+1}/{num_frames} 大小：{w}x{h}") + + # 模拟处理时间 + time.sleep(0.01) + + except KeyboardInterrupt: + print("用户中断") + finally: + cap.release() + print("摄像头已释放，程序结束") + +if __name__ == "__main__": + # 可以通过修改以下参数来指定设备号或读取帧数 + main(device_index=0, num_frames=200) + +``` + +* 在开发板的命令行执行下面的命令，运行opencv_usb_camera_test.py + +```sh +python opencv_usb_camera_test.py +``` + +![image-20251016152114408](pic/image-20251016152114408.png) + + + +## 5、测试opencv中dnn模块 + +* 将下面的内容复制到opencv_dnn_test.py文件中，测试opencv调用dnn模块，实现人脸检测的基本功能。 + +```mk1.sh +import cv2 + +# 1. 读取图片 +image_path = "image.png" # 替换为你的图片路径 +img = cv2.imread(image_path) +if img is None: + print("无法加载图片") + exit(1) +orig_h, orig_w = img.shape[:2] + +# 2. 初始化 FaceDetectorYN，使用原图大小并调松参数 +detector = cv2.FaceDetectorYN.create( + model="face_detection_yunet_2023mar.onnx", + config="", + input_size=(orig_w, orig_h), # 与原图一致 + score_threshold=0.4, # 提高置信度阈值到 0.4 + nms_threshold=0.5, # NMS 阈值 + top_k=200 # 保留最多 200 个候选 +) + +# 3. 检测人脸 +_, faces = detector.detect(img) + +# 4. 后处理：剔除过小的框 +min_size = 50 # 小于 50 像素的宽高都视为噪声 +filtered = [] +if faces is not None: + for face in faces: + x, y, w, h, score = face[:5].astype(int) + # 保留宽高都不小于阈值的框 + if w >= min_size and h >= min_size: + filtered.append((x, y, w, h, score)) + +# 5. 在图像上绘制过滤后的人脸框 +for x, y, w, h, score in filtered: + cv2.rectangle(img, (x, y), (x + w, y + h), (0, 255, 0), 2) + cv2.putText( + img, + f"{score/100:.2f}", + (x, y - 5), + cv2.FONT_HERSHEY_SIMPLEX, + 0.5, + (0, 255, 0), + 1 + ) + +# 6. 保存结果并输出检测到的人脸数量 +output_path = "output_image.jpg" +cv2.imwrite(output_path, img) +print(f"检测完成，检测到 {len(filtered)} 张人脸，结果已保存到 {output_path}") +``` + +* 请自行下载一个带有人脸的图片放入install/bin/目录下，然后把图片重命名为image.png。 +* 然后访问链接下载[onnx模型](https://github.com/opencv/opencv_zoo/blob/main/models/face_detection_yunet/face_detection_yunet_2023mar.onnx)，并将该模型也放入install/bin/目录下。 + +![image-20251016153030674](pic/image-20251016153030674.png) + +* 在开发板的命令行执行下面的命令，运行opencv_dnn_test.py + +```sh +python3 opencv_dnn_test.py +``` + +![image-20251016153323187](pic/image-20251016153323187.png) + +检测前： + +![image-20251016153355942](pic/image-20251016153355942.png) + +检测后： + +![image-20251016153418154](pic/image-20251016153418154.png) diff --git a/vendor/zsks/docs/opencv/pic/image-20251013144111854.png b/vendor/zsks/docs/opencv/pic/image-20251013144111854.png new file mode 100755 index 0000000000000000000000000000000000000000..6b19a9f636c3d236c00a3044f73dffdc0d3f06fd Binary files /dev/null and b/vendor/zsks/docs/opencv/pic/image-20251013144111854.png differ diff --git a/vendor/zsks/docs/opencv/pic/image-20251013150324179.png b/vendor/zsks/docs/opencv/pic/image-20251013150324179.png new file mode 100755 index 0000000000000000000000000000000000000000..ea84672bd20544c2d8a9e481f8c92c8e93353d7b Binary files /dev/null and b/vendor/zsks/docs/opencv/pic/image-20251013150324179.png differ diff --git a/vendor/zsks/docs/opencv/pic/image-20251013150632408.png b/vendor/zsks/docs/opencv/pic/image-20251013150632408.png new file mode 100755 index 0000000000000000000000000000000000000000..8a388cfeb9b9c11c14cbc381728e54382c8cb187 Binary files /dev/null and b/vendor/zsks/docs/opencv/pic/image-20251013150632408.png differ diff --git a/vendor/zsks/docs/opencv/pic/image-20251013150714220.png b/vendor/zsks/docs/opencv/pic/image-20251013150714220.png new file mode 100755 index 0000000000000000000000000000000000000000..625eb55460f1be5c55d9c6e0aea66dc8091b3498 Binary files /dev/null and b/vendor/zsks/docs/opencv/pic/image-20251013150714220.png differ diff --git a/vendor/zsks/docs/opencv/pic/image-20251013150935466.png b/vendor/zsks/docs/opencv/pic/image-20251013150935466.png new file mode 100755 index 0000000000000000000000000000000000000000..ae0e2447108e472311f55f8b6d5eb32e61130c77 Binary files /dev/null and b/vendor/zsks/docs/opencv/pic/image-20251013150935466.png differ diff --git a/vendor/zsks/docs/opencv/pic/image-20251013164259362.png b/vendor/zsks/docs/opencv/pic/image-20251013164259362.png new file mode 100755 index 0000000000000000000000000000000000000000..77cbdce6858beb95660e98547ed11b4f50f7296f Binary files /dev/null and b/vendor/zsks/docs/opencv/pic/image-20251013164259362.png differ diff --git a/vendor/zsks/docs/opencv/pic/image-20251013181456311.png b/vendor/zsks/docs/opencv/pic/image-20251013181456311.png new file mode 100755 index 0000000000000000000000000000000000000000..ae6f1a8080a476ac036c1adda7d491c627b019b5 Binary files /dev/null and b/vendor/zsks/docs/opencv/pic/image-20251013181456311.png differ diff --git a/vendor/zsks/docs/opencv/pic/image-20251013181928810.png b/vendor/zsks/docs/opencv/pic/image-20251013181928810.png new file mode 100755 index 0000000000000000000000000000000000000000..5462015631c5b6c2dfaa6adbf3e718ee423c9e7b Binary files /dev/null and b/vendor/zsks/docs/opencv/pic/image-20251013181928810.png differ diff --git a/vendor/zsks/docs/opencv/pic/image-20251016145037570.png b/vendor/zsks/docs/opencv/pic/image-20251016145037570.png new file mode 100755 index 0000000000000000000000000000000000000000..d8396af5cd30d099c9e33297df7f72d47f87fc97 Binary files /dev/null and b/vendor/zsks/docs/opencv/pic/image-20251016145037570.png differ diff --git a/vendor/zsks/docs/opencv/pic/image-20251016151507324.png b/vendor/zsks/docs/opencv/pic/image-20251016151507324.png new file mode 100755 index 0000000000000000000000000000000000000000..f4f2e5a7a70127f12e0d8e1ad45e135577e3c715 Binary files /dev/null and b/vendor/zsks/docs/opencv/pic/image-20251016151507324.png differ diff --git a/vendor/zsks/docs/opencv/pic/image-20251016151851058.png b/vendor/zsks/docs/opencv/pic/image-20251016151851058.png new file mode 100755 index 0000000000000000000000000000000000000000..ba7dbc16eccffd7f4e3045984263489fc8f02c97 Binary files /dev/null and b/vendor/zsks/docs/opencv/pic/image-20251016151851058.png differ diff --git a/vendor/zsks/docs/opencv/pic/image-20251016151915487.png b/vendor/zsks/docs/opencv/pic/image-20251016151915487.png new file mode 100755 index 0000000000000000000000000000000000000000..ae019ea6575ab93b8472df4d0c8026b58b6e993a Binary files /dev/null and b/vendor/zsks/docs/opencv/pic/image-20251016151915487.png differ diff --git a/vendor/zsks/docs/opencv/pic/image-20251016152114408.png b/vendor/zsks/docs/opencv/pic/image-20251016152114408.png new file mode 100755 index 0000000000000000000000000000000000000000..76b62ea9ee2fcbf8a77d5271da52cee911e3704e Binary files /dev/null and b/vendor/zsks/docs/opencv/pic/image-20251016152114408.png differ diff --git a/vendor/zsks/docs/opencv/pic/image-20251016153030674.png b/vendor/zsks/docs/opencv/pic/image-20251016153030674.png new file mode 100755 index 0000000000000000000000000000000000000000..41d31eeefd1e31d8cdf2b9455288c9b316a834d9 Binary files /dev/null and b/vendor/zsks/docs/opencv/pic/image-20251016153030674.png differ diff --git a/vendor/zsks/docs/opencv/pic/image-20251016153323187.png b/vendor/zsks/docs/opencv/pic/image-20251016153323187.png new file mode 100755 index 0000000000000000000000000000000000000000..93050893423c439913e9fc35db3984c85617ecd9 Binary files /dev/null and b/vendor/zsks/docs/opencv/pic/image-20251016153323187.png differ diff --git a/vendor/zsks/docs/opencv/pic/image-20251016153355942.png b/vendor/zsks/docs/opencv/pic/image-20251016153355942.png new file mode 100755 index 0000000000000000000000000000000000000000..f5d5f95bb5c5732a4f3fa9782d9c8bf932ac4bb2 Binary files /dev/null and b/vendor/zsks/docs/opencv/pic/image-20251016153355942.png differ diff --git a/vendor/zsks/docs/opencv/pic/image-20251016153418154.png b/vendor/zsks/docs/opencv/pic/image-20251016153418154.png new file mode 100755 index 0000000000000000000000000000000000000000..764009360a2028ccb7ed14fe1d70d4e4604216f0 Binary files /dev/null and b/vendor/zsks/docs/opencv/pic/image-20251016153418154.png differ diff --git a/vendor/zsks/docs/python/README.md b/vendor/zsks/docs/python/README.md new file mode 100755 index 0000000000000000000000000000000000000000..30eea6b679de52d8e652a4c39adc4262e155b1f4 --- /dev/null +++ b/vendor/zsks/docs/python/README.md @@ -0,0 +1,487 @@ +# Python3移植 + +## 1、软硬件环境 +开发板：海鸥派 +交叉编译工具链：OHOS (dev) clang version 15.0.4 + +编译链路径：pegasus/os/OpenHarmony/ohos/prebuilts/clang/ohos/linux-x86_64/llvm/bin + +移植的Python版本：Python-3.13.2 + +## 2、配置python环境 + +* 为了满足python3.13.2的移植要求，我们需要先把服务器的python版本改为3.13.2 +* **步骤1**：在服务器的命令行终端执行下面的命令，下载python3.13.2并安装 + +```shell +wget https://www.python.org/ftp/python/3.13.2/Python-3.13.2.tgz + +tar -xvzf Python-3.13.2.tgz +rm Python-3.13.2.tgz + +cd Python-3.13.2 + +./configure --enable-optimizations --enable-shared + +make -j$(nproc) + +make altinstall +``` + +![image-20251010143023136](pic/image-20251010143023136.png) + +![image-20251010143136574](pic/image-20251010143136574.png) + +![image-20251010143226900](pic/image-20251010143226900.png) + +![image-20251010143640847](pic/image-20251010143640847.png) + +* **步骤2：**创建python环境，是python3.13.2生效 +* 在服务器的命令行执行下面的命令，配置python环境变量 + +```sh +rm /usr/bin/python3 + +ln -s /usr/local/bin/python3.13 /usr/bin/python3 + +# 在~/.bashrc文件末尾添加下面的内容 +export LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH + +# 执行下面的命令，让环境变量生效 +source ~/.bashrc +``` + +![image-20251010144051579](pic/image-20251010144051579.png) + +![image-20251010144137718](pic/image-20251010144137718.png) + +## 3.安装依赖软件 + +### 步骤1： 安装依赖软件包 + +* 在服务器的命令行执行下面的命令，下载移植python所需软件 +* 注意：由于本文档环境是root用户，请根据自己的实际服务器环境，选择是否需要在命令前加sudo + +``` +apt-get update +apt-get upgrade -y +apt-get install openssl libssl-dev gcc make cmake -y +``` + +### 步骤2：交叉编译依赖第三方软件 + +**注意：**在交叉编译之前，请一定确保OpenHarmony的代码下载完成，且整编通过，具体可参考[ohos编译](https://gitee.com/HiSpark/pegasus/blob/master/docs/OpenHarmony%20Small%E7%89%88%E6%9C%AC%E4%BD%BF%E7%94%A8%E6%8C%87%E5%8D%97/OpenHarmony%20Small%E7%89%88%E6%9C%AC%E4%BD%BF%E7%94%A8%E6%8C%87%E5%8D%97.md#ohos%E7%BC%96%E8%AF%91)的内容 + +#### 1、zlib的交叉编译 + +* 在服务器的命令行执行下面的命令，进行zlib的交叉编译 + +```sh +# 由于zlib也属于第三方软件，可以在opensource目录下进行移植 +cd pegasus/vendor/opensource/ + +wget http://zlib.net/zlib-1.3.1.tar.gz + +tar -xvf zlib-1.3.1.tar.gz +rm zlib-1.3.1.tar.gz + +cd zlib-1.3.1/ + +# 注意：这里的路径请根据自己的Pegasus目录进行修改 +export CC="/home/openharmony/pegasus/os/OpenHarmony/ohos/prebuilts/clang/ohos/linux-x86_64/llvm/bin/aarch64-unknown-linux-ohos-clang --sysroot=/home/openharmony/pegasus/os/OpenHarmony/ohos/out/hispark_ss928v100/ipcamera_hispark_ss928v100_linux/sysroot" + +./configure --prefix=$PWD/install + +# 注意：这里的路径请根据自己的Pegasus目录进行修改 +make LDFLAGS="--sysroot=/home/openharmony/pegasus/os/OpenHarmony/ohos/out/hispark_ss928v100/ipcamera_hispark_ss928v100_linux/sysroot -L/home/openharmony/pegasus/os/OpenHarmony/ohos/out/hispark_ss928v100/ipcamera_hispark_ss928v100_linux/sysroot/usr/lib" + +make install +``` + +![image-20251009170325172](pic/image-20251009170325172.png) + +![image-20251009171708407](pic/image-20251009171708407.png) + +![image-20251009171921919](pic/image-20251009171921919.png) + +![image-20251009172007367](pic/image-20251009172007367.png) + +#### 2、openssl的交叉编译 + +* 在服务器的命令行执行下面的命令，进行openssl的交叉编译 + +```sh +cd ../ + +wget https://github.com/openssl/openssl/archive/refs/tags/OpenSSL_1_1_1w.tar.gz + +tar -xvf OpenSSL_1_1_1w.tar.gz +rm OpenSSL_1_1_1w.tar.gz + +cd openssl-OpenSSL_1_1_1w + +perl Configure linux-aarch64 --prefix=$PWD/install + +make CC="/home/openharmony/pegasus/os/OpenHarmony/ohos/prebuilts/clang/ohos/linux-x86_64/llvm/bin/aarch64-unknown-linux-ohos-clang --sysroot=/home/openharmony/pegasus/os/OpenHarmony/ohos/out/hispark_ss928v100/ipcamera_hispark_ss928v100_linux/sysroot" LDFLAGS="--sysroot=/home/openharmony/pegasus/os/OpenHarmony/ohos/out/hispark_ss928v100/ipcamera_hispark_ss928v100_linux/sysroot -L/home/openharmony/pegasus/os/OpenHarmony/ohos/out/hispark_ss928v100/ipcamera_hispark_ss928v100_linux/sysroot/usr/lib" + +make install +``` + +![image-20251009173438071](pic/image-20251009173438071.png) + +![image-20251009173659012](pic/image-20251009173659012.png) + +![image-20251009174036856](pic/image-20251009174036856.png) + +![image-20251009174105488](pic/image-20251009174105488.png) + +#### 3、libffi的交叉编译 + +* 在服务器的命令行执行下面的命令，进行libffi的交叉编译 + +```sh +cd ../ + +wget https://github.com/libffi/libffi/archive/refs/tags/v3.4.7.tar.gz + +tar -xvf v3.4.7.tar.gz +rm v3.4.7.tar.gz + +cd libffi-3.4.7/ + +./autogen.sh +``` + +![image-20251009174843560](pic/image-20251009174843560.png) + +![image-20251009174953996](pic/image-20251009174953996.png) + +* 修改 config.sub 文件中两处内容 + + * 1、在1771行后面添加内容 | ohos* + + ![image-20251009175248076](pic/image-20251009175248076.png) + + * 2、在 1832行添加 | linux-ohos*- + + ![image-20251009175416655](pic/image-20251009175416655.png) + +* 保持config.sub文件之后，执行下面的命令，交叉编译libffi + +```sh + +export CC="/home/openharmony/pegasus/os/OpenHarmony/ohos/prebuilts/clang/ohos/linux-x86_64/llvm/bin/aarch64-unknown-linux-ohos-clang --sysroot=/home/openharmony/pegasus/os/OpenHarmony/ohos/out/hispark_ss928v100/ipcamera_hispark_ss928v100_linux/sysroot" + +./configure --prefix=$PWD/install --host=aarch64-linux-ohos + +make && make install +``` + +![image-20251009175706346](pic/image-20251009175706346.png) + +![image-20251009175825681](pic/image-20251009175825681.png) + +![image-20251009175854834](pic/image-20251009175854834.png) + +#### 4、libuuid的交叉编译 + +* 在服务器的命令行执行下面的命令，进行libuuid的交叉编译 + +```sh +cd ../ + +wget https://downloads.sourceforge.net/project/libuuid/libuuid-1.0.3.tar.gz + +tar -xvf libuuid-1.0.3.tar.gz + +rm libuuid-1.0.3.tar.gz + +cd libuuid-1.0.3 + +# 修改 config.sub 文件： + +sed -i 's/| -kaos\*/| -kaos\* | -ohos\*/g' config.sub + +sed -i 's/linux-uclibc\*/linux-uclibc\* | linux-ohos\*/g' config.sub + +export CC="/home/openharmony/pegasus/os/OpenHarmony/ohos/prebuilts/clang/ohos/linux-x86_64/llvm/bin/aarch64-unknown-linux-ohos-clang --sysroot=/home/openharmony/pegasus/os/OpenHarmony/ohos/out/hispark_ss928v100/ipcamera_hispark_ss928v100_linux/sysroot" + +./configure --prefix=$PWD/install --enable-shared --host=aarch64-linux-ohos + +make && make install +``` + +![image-20251009180156789](pic/image-20251009180156789.png) + +![image-20251009180424732](pic/image-20251009180424732.png) + +![image-20251009180446492](pic/image-20251009180446492.png) + +![image-20251009180509397](pic/image-20251009180509397.png) + +#### 5、xz的交叉编译 + +* 在服务器的命令行执行下面的命令，进行xz的交叉编译 + +```sh +cd ../ + +wget https://tukaani.org/xz/xz-5.2.5.tar.gz +tar -zxvf xz-5.2.5.tar.gz +rm xz-5.2.5.tar.gz +cd xz-5.2.5 + +sed -i 's/| -kaos\*/| -kaos\* | -ohos\*/g' ./build-aux/config.sub +sed -i 's/linux-uclibc\*/linux-uclibc\* | linux-ohos\*/g' ./build-aux/config.sub + +export CC="/home/openharmony/pegasus/os/OpenHarmony/ohos/prebuilts/clang/ohos/linux-x86_64/llvm/bin/aarch64-unknown-linux-ohos-clang --sysroot=/home/openharmony/pegasus/os/OpenHarmony/ohos/out/hispark_ss928v100/ipcamera_hispark_ss928v100_linux/sysroot" + +./configure --prefix=$PWD/install --enable-shared --host=aarch64-linux-ohos + +make && make install +``` + +![image-20251009181033404](pic/image-20251009181033404.png) + +![image-20251009181219093](pic/image-20251009181219093.png) + +![image-20251009181250942](pic/image-20251009181250942.png) + +![image-20251009181318999](pic/image-20251009181318999.png) + +#### 6、readline的交叉编译 + +* 在服务器的命令行执行下面的命令，进行readline的交叉编译 + +```sh +cd ../ + +wget https://mirrors.aliyun.com/gnu/readline/readline-8.2.tar.gz +tar xzf readline-8.2.tar.gz +rm readline-8.2.tar.gz +cd readline-8.2 +``` + +![image-20251009191238117](pic/image-20251009191238117.png) + +* 修改 support/config.sub 文件中一处内容 + +* 在1757行添加 | ohos* +* 在1775行添加 | linux-ohos* + +![image-20251009193532253](pic/image-20251009193532253.png) + +* support/config.sub 文件保存好后，在服务器的命令行终端，分别执行下面的命令 + +```sh +export CC="/home/openharmony/pegasus/os/OpenHarmony/ohos/prebuilts/clang/ohos/linux-x86_64/llvm/bin/aarch64-unknown-linux-ohos-clang --sysroot=/home/openharmony/pegasus/os/OpenHarmony/ohos/out/hispark_ss928v100/ipcamera_hispark_ss928v100_linux/sysroot" + +./configure --host=aarch64-linux-ohos --prefix=$PWD/install --disable-install-examples + +make && make install +``` + +![image-20251009195217103](pic/image-20251009195217103.png) + +![image-20251009191946638](pic/image-20251009191946638.png) + +![image-20251009200013989](pic/image-20251009200013989.png) + +## 4、交叉编译python3.13.2 + +* 在服务器的命令行执行下面的命令，进行python3.13.2的交叉编译 + +```sh +cd ../ + +wget https://www.python.org/ftp/python/3.13.2/Python-3.13.2.tgz + +tar -xvzf Python-3.13.2.tgz + +rm Python-3.13.2.tgz + +cd Python-3.13.2 +``` + +* 修改 config.sub 文件中两处内容 + * 在1772行 添加： | ohos* + * 在 1833行添加： | linux-ohos*- + +image-20251009200915614 + +![image-20251009201117880](pic/image-20251009201117880.png) + +* 修改configure文件，将7021~7027行的内容 改为下面的内容 + +```sh +if test x$PLATFORM_TRIPLET != x && test x$MULTIARCH != x; then + if test x$PLATFORM_TRIPLET != x && test x$MULTIARCH = x; then + MULTIARCH=$PLATFORM_TRIPLET + fi +fi +``` + +* 修改前 + +![image-20251009201657474](pic/image-20251009201657474.png) + +* 修改后 + +![image-20251009201624874](pic/image-20251009201624874.png) + +* 执行下面的命令，交叉编译python + +```sh +CROSS_TOOLCHAIN_DIR="/home/openharmony/pegasus/os/OpenHarmony/ohos/prebuilts/clang/ohos/linux-x86_64/llvm/bin" +SYSROOT="/home/openharmony/pegasus/os/OpenHarmony/ohos/out/hispark_ss928v100/ipcamera_hispark_ss928v100_linux/sysroot" + +CC="${CROSS_TOOLCHAIN_DIR}/aarch64-unknown-linux-ohos-clang" \ +CXX="${CROSS_TOOLCHAIN_DIR}/aarch64-unknown-linux-ohos-clang++" \ +AR="${CROSS_TOOLCHAIN_DIR}/llvm-ar" \ +LD="${CROSS_TOOLCHAIN_DIR}/ld.lld" \ +RANLIB="${CROSS_TOOLCHAIN_DIR}/llvm-ranlib" \ +STRIP="${CROSS_TOOLCHAIN_DIR}/llvm-strip" \ +CFLAGS="--sysroot=${SYSROOT} -fPIC" \ +CXXFLAGS="--sysroot=${SYSROOT} -fPIC" \ +LDFLAGS="--sysroot=${SYSROOT}" \ +./configure \ +--host=aarch64-linux-ohos \ +--build=x86_64-pc-linux-gnu \ +--target=aarch64-linux-ohos \ +--prefix="$PWD/install" \ +--disable-ipv6 \ +ac_cv_file__dev_ptmx="yes" \ +ac_cv_file__dev_ptc="yes" \ +--with-openssl=../openssl-OpenSSL_1_1_1w/install \ +--with-build-python=python3.13 2>&1 | tee config.log.txt + +CFLAGS="-I../zlib-1.3.1/install/include \ + -I../xz-5.2.5/install/include \ + -I../libffi-3.4.7/install/include \ + -I../readline-8.2/install/include \ + -I../libuuid-1.0.3/install/include/uuid" +CPPFLAGS="-I../openssl-OpenSSL_1_1_1w/install/include" +LDFLAGS="-L../zlib-1.3.1/install/lib \ + -L../xz-5.2.5/install/lib \ + -L../libffi-3.4.7/install/lib \ + -L../readline-8.2/install/lib \ + -L../libuuid-1.0.3/install/lib \ + -L../openssl-OpenSSL_1_1_1w/install/lib" + +CFLAGS="$CFLAGS" CPPFLAGS="$CPPFLAGS" LDFLAGS="$LDFLAGS" make 2>&1 | tee make.log.txt +``` + +![image-20251009202143477](pic/image-20251009202143477.png) + +![image-20251009202450693](pic/image-20251009202450693.png) + +```sh +make install 2>1 | tee install.log.txt +``` + +![image-20251009202940688](pic/image-20251009202940688.png) + +![image-20251009203011267](pic/image-20251009203011267.png) + +## 5.板端测试 + +### 步骤1：配置板端环境 + +* 1、确保开发板已经烧录OpenHarmony操作系统 +* 2、使用网线将开发板与你的电脑进行连接，确保二者处于同一局域网内 +* 3、配置开发板的IP地址，并确保开发板与电脑能够互相ping通 + +```sh +# 注意：这里的eth0的IP地址，请根据自己的网络IP网段进行合理配置 +ifconfig eth0 192.168.100.100 + +# 添加权限 +echo 0 9999999 > /proc/sys/net/ipv4/ping_group_range +``` + +![image-20251010094441549](pic/image-20251010094441549.png) + +### 步骤2：准备python依赖文件 + +* 1、将第4章交叉编译python3.13.2后，生成的install文件夹拷贝到你的NFS挂载目录（如果没有网线，SD卡也是可以挂载到开发板的） +* 2、执行下面的，命令将电脑的nfs目录挂载到开发板的/mnt目录下（注意：这里请根据自己的IP地址及NFS配置进行合理的修改） + +```mk1.sh +mount -o nolock,addr=192.168.100.10 -t nfs 192.168.100.10:/d/nfs /mnt +``` + +![image-20251010100758153](pic/image-20251010100758153.png) + +* 3、配置python的环境变量，确保python运行时能够找到依赖 + +```mk1.sh +export PATH=/mnt/install/bin:$PATH +export PYTHONPATH=/mnt/install/lib/python3.13:$PYTHONPATH +export LD_LIBRARY_PATH=/mnt/install/lib/python3.13/lib-dynload:$LD_LIBRARY_PATH +``` + +image-20251010101732852 + +### 步骤3：运行python代码 + +* 如果第一次移植python3，需要给python3添加可执行权限才能运行 + +```sh +chmod +x /mnt/install/bin/python3 +``` + +* 输入python3，查看结果 + +![image-20251010101851439](pic/image-20251010101851439.png) + +* 在板端测试python代码，将下面的内容全部复制到 python_test.py 文件中 + +```python +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +print("Hello, World!") + +import os +import sys +import datetime + +def main(): + if sys.version_info < (3, 6): + print("Error: Requires Python 3.6 or higher") + sys.exit(1) + + current_time = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") + user = os.getenv("USER", "unknown_user") + +def multi_language(): + languages = { + "Chinese": "ni hao shijie ", + "Spanish": "?Hola Mundo!", + "French": "Bonjour le monde!", + "Japanese": "shijie nihao" + } + + try: + for lang, greeting in languages.items(): + print(f"{lang}: {greeting}") + except UnicodeEncodeError: + print("Encoding error detected. Try setting environment variable:") + print("export PYTHONIOENCODING=utf-8") + +if __name__ == "__main__": + main() + multi_language() + + # basic I/O + input("\nPress Enter to exit...") +``` + +* 使用下面命令，运行python_test.py代码，查看现象，如果现象如下图所示，说明移植成功 + +```sh +python3 python_test.py +``` + +image-20251010102204475 + diff --git a/vendor/zsks/docs/python/pic/image-20251009170325172.png b/vendor/zsks/docs/python/pic/image-20251009170325172.png new file mode 100755 index 0000000000000000000000000000000000000000..2acbe6f36b1d75ea359aae5f8a7912264ce0751e Binary files /dev/null and b/vendor/zsks/docs/python/pic/image-20251009170325172.png differ diff --git a/vendor/zsks/docs/python/pic/image-20251009171708407.png b/vendor/zsks/docs/python/pic/image-20251009171708407.png new file mode 100755 index 0000000000000000000000000000000000000000..704d95da933fe0d18f33d651fb18b9e936e3430f Binary files /dev/null and b/vendor/zsks/docs/python/pic/image-20251009171708407.png differ diff --git a/vendor/zsks/docs/python/pic/image-20251009171921919.png b/vendor/zsks/docs/python/pic/image-20251009171921919.png new file mode 100755 index 0000000000000000000000000000000000000000..7896aea248c4b1f73eefa3fc963b2dee3780dc65 Binary files /dev/null and b/vendor/zsks/docs/python/pic/image-20251009171921919.png differ diff --git a/vendor/zsks/docs/python/pic/image-20251009172007367.png b/vendor/zsks/docs/python/pic/image-20251009172007367.png new file mode 100755 index 0000000000000000000000000000000000000000..48fc436f2e1b3fa0f9083d0202f4d9cdff368ec6 Binary files /dev/null and b/vendor/zsks/docs/python/pic/image-20251009172007367.png differ diff --git a/vendor/zsks/docs/python/pic/image-20251009173438071.png b/vendor/zsks/docs/python/pic/image-20251009173438071.png new file mode 100755 index 0000000000000000000000000000000000000000..abd97d0442d4134130a4102c0bb5be8781218b64 Binary files /dev/null and b/vendor/zsks/docs/python/pic/image-20251009173438071.png differ diff --git a/vendor/zsks/docs/python/pic/image-20251009173659012.png b/vendor/zsks/docs/python/pic/image-20251009173659012.png new file mode 100755 index 0000000000000000000000000000000000000000..d3c95af0bdef74808f59af6922f3c0dd9bdffc26 Binary files /dev/null and b/vendor/zsks/docs/python/pic/image-20251009173659012.png differ diff --git a/vendor/zsks/docs/python/pic/image-20251009174036856.png b/vendor/zsks/docs/python/pic/image-20251009174036856.png new file mode 100755 index 0000000000000000000000000000000000000000..71cbbffc86935e2d55fcf0260092b5ccf46369ba Binary files /dev/null and b/vendor/zsks/docs/python/pic/image-20251009174036856.png differ diff --git a/vendor/zsks/docs/python/pic/image-20251009174105488.png b/vendor/zsks/docs/python/pic/image-20251009174105488.png new file mode 100755 index 0000000000000000000000000000000000000000..1dca90f249e54c3ba3a54f64c14b8236de38642a Binary files /dev/null and b/vendor/zsks/docs/python/pic/image-20251009174105488.png differ diff --git a/vendor/zsks/docs/python/pic/image-20251009174843560.png b/vendor/zsks/docs/python/pic/image-20251009174843560.png new file mode 100755 index 0000000000000000000000000000000000000000..093e8dbca1958a97b0a1a924934a76ac70ef7a42 Binary files /dev/null and b/vendor/zsks/docs/python/pic/image-20251009174843560.png differ diff --git a/vendor/zsks/docs/python/pic/image-20251009174953996.png b/vendor/zsks/docs/python/pic/image-20251009174953996.png new file mode 100755 index 0000000000000000000000000000000000000000..54629a9e0f9d4aabf5a0e4f26b975c3ddfc0b209 Binary files /dev/null and b/vendor/zsks/docs/python/pic/image-20251009174953996.png differ diff --git a/vendor/zsks/docs/python/pic/image-20251009175248076.png b/vendor/zsks/docs/python/pic/image-20251009175248076.png new file mode 100755 index 0000000000000000000000000000000000000000..f54f82efa876c74fa32b61354ce31124aafa8411 Binary files /dev/null and b/vendor/zsks/docs/python/pic/image-20251009175248076.png differ diff --git a/vendor/zsks/docs/python/pic/image-20251009175416655.png b/vendor/zsks/docs/python/pic/image-20251009175416655.png new file mode 100755 index 0000000000000000000000000000000000000000..249a912c114736f43987c7481a4c1f701c903de0 Binary files /dev/null and b/vendor/zsks/docs/python/pic/image-20251009175416655.png differ diff --git a/vendor/zsks/docs/python/pic/image-20251009175706346.png b/vendor/zsks/docs/python/pic/image-20251009175706346.png new file mode 100755 index 0000000000000000000000000000000000000000..9a5a4ce0e605560bcf3c853d002dabe295593cc9 Binary files /dev/null and b/vendor/zsks/docs/python/pic/image-20251009175706346.png differ diff --git a/vendor/zsks/docs/python/pic/image-20251009175825681.png b/vendor/zsks/docs/python/pic/image-20251009175825681.png new file mode 100755 index 0000000000000000000000000000000000000000..a80c1f1487b94144385dac67027667510f35442f Binary files /dev/null and b/vendor/zsks/docs/python/pic/image-20251009175825681.png differ diff --git a/vendor/zsks/docs/python/pic/image-20251009175854834.png b/vendor/zsks/docs/python/pic/image-20251009175854834.png new file mode 100755 index 0000000000000000000000000000000000000000..98a944284bc9bed30571f197dfb8ff6744d5ddb8 Binary files /dev/null and b/vendor/zsks/docs/python/pic/image-20251009175854834.png differ diff --git a/vendor/zsks/docs/python/pic/image-20251009180156789.png b/vendor/zsks/docs/python/pic/image-20251009180156789.png new file mode 100755 index 0000000000000000000000000000000000000000..c821dcdebb6d3969e9bd07d63d47fae318ee8f02 Binary files /dev/null and b/vendor/zsks/docs/python/pic/image-20251009180156789.png differ diff --git a/vendor/zsks/docs/python/pic/image-20251009180424732.png b/vendor/zsks/docs/python/pic/image-20251009180424732.png new file mode 100755 index 0000000000000000000000000000000000000000..3e3e8b169d9ea66de61a28bfbced96c0b821f305 Binary files /dev/null and b/vendor/zsks/docs/python/pic/image-20251009180424732.png differ diff --git a/vendor/zsks/docs/python/pic/image-20251009180446492.png b/vendor/zsks/docs/python/pic/image-20251009180446492.png new file mode 100755 index 0000000000000000000000000000000000000000..82a33c448dd5b2656b348bc758e55fae1fce7a9e Binary files /dev/null and b/vendor/zsks/docs/python/pic/image-20251009180446492.png differ diff --git a/vendor/zsks/docs/python/pic/image-20251009180509397.png b/vendor/zsks/docs/python/pic/image-20251009180509397.png new file mode 100755 index 0000000000000000000000000000000000000000..089d35f9b1d107605bcff947cecf7b259a83fec9 Binary files /dev/null and b/vendor/zsks/docs/python/pic/image-20251009180509397.png differ diff --git a/vendor/zsks/docs/python/pic/image-20251009181033404.png b/vendor/zsks/docs/python/pic/image-20251009181033404.png new file mode 100755 index 0000000000000000000000000000000000000000..d250aaa0d7a0c36a7704cffa958e0f3f5347a161 Binary files /dev/null and b/vendor/zsks/docs/python/pic/image-20251009181033404.png differ diff --git a/vendor/zsks/docs/python/pic/image-20251009181219093.png b/vendor/zsks/docs/python/pic/image-20251009181219093.png new file mode 100755 index 0000000000000000000000000000000000000000..4ba2d44bd45aaf792db002519040e12827990381 Binary files /dev/null and b/vendor/zsks/docs/python/pic/image-20251009181219093.png differ diff --git a/vendor/zsks/docs/python/pic/image-20251009181250942.png b/vendor/zsks/docs/python/pic/image-20251009181250942.png new file mode 100755 index 0000000000000000000000000000000000000000..5afde18c86892741866744a0ad784183cb664a3e Binary files /dev/null and b/vendor/zsks/docs/python/pic/image-20251009181250942.png differ diff --git a/vendor/zsks/docs/python/pic/image-20251009181318999.png b/vendor/zsks/docs/python/pic/image-20251009181318999.png new file mode 100755 index 0000000000000000000000000000000000000000..6b54c6d96ea1576da55d790a0ffe4bf24b8954d4 Binary files /dev/null and b/vendor/zsks/docs/python/pic/image-20251009181318999.png differ diff --git a/vendor/zsks/docs/python/pic/image-20251009191238117.png b/vendor/zsks/docs/python/pic/image-20251009191238117.png new file mode 100755 index 0000000000000000000000000000000000000000..abe489d74208fb6fdf472c96b57eddc938353e58 Binary files /dev/null and b/vendor/zsks/docs/python/pic/image-20251009191238117.png differ diff --git a/vendor/zsks/docs/python/pic/image-20251009191946638.png b/vendor/zsks/docs/python/pic/image-20251009191946638.png new file mode 100755 index 0000000000000000000000000000000000000000..250a8349e41239e58e43688961617bbd320291c4 Binary files /dev/null and b/vendor/zsks/docs/python/pic/image-20251009191946638.png differ diff --git a/vendor/zsks/docs/python/pic/image-20251009193532253.png b/vendor/zsks/docs/python/pic/image-20251009193532253.png new file mode 100755 index 0000000000000000000000000000000000000000..8c5764606b13d42c5454c0c106404b35ed4000c3 Binary files /dev/null and b/vendor/zsks/docs/python/pic/image-20251009193532253.png differ diff --git a/vendor/zsks/docs/python/pic/image-20251009195217103.png b/vendor/zsks/docs/python/pic/image-20251009195217103.png new file mode 100755 index 0000000000000000000000000000000000000000..ede25b81fed5a4dbb470ee466eee3c02f2f211ff Binary files /dev/null and b/vendor/zsks/docs/python/pic/image-20251009195217103.png differ diff --git a/vendor/zsks/docs/python/pic/image-20251009200013989.png b/vendor/zsks/docs/python/pic/image-20251009200013989.png new file mode 100755 index 0000000000000000000000000000000000000000..1256aca756ed2b3604132215e1f12b37ce67cccc Binary files /dev/null and b/vendor/zsks/docs/python/pic/image-20251009200013989.png differ diff --git a/vendor/zsks/docs/python/pic/image-20251009200915614.png b/vendor/zsks/docs/python/pic/image-20251009200915614.png new file mode 100755 index 0000000000000000000000000000000000000000..ccca0579c41e1e2f42a48c1bcaafaf378a0de654 Binary files /dev/null and b/vendor/zsks/docs/python/pic/image-20251009200915614.png differ diff --git a/vendor/zsks/docs/python/pic/image-20251009201117880.png b/vendor/zsks/docs/python/pic/image-20251009201117880.png new file mode 100755 index 0000000000000000000000000000000000000000..7ab251224398ad9a736fead3312777334bd83295 Binary files /dev/null and b/vendor/zsks/docs/python/pic/image-20251009201117880.png differ diff --git a/vendor/zsks/docs/python/pic/image-20251009201624874.png b/vendor/zsks/docs/python/pic/image-20251009201624874.png new file mode 100755 index 0000000000000000000000000000000000000000..3604562217d5322d60a695d073753cbf4d9260e6 Binary files /dev/null and b/vendor/zsks/docs/python/pic/image-20251009201624874.png differ diff --git a/vendor/zsks/docs/python/pic/image-20251009201657474.png b/vendor/zsks/docs/python/pic/image-20251009201657474.png new file mode 100755 index 0000000000000000000000000000000000000000..6d6a52ff1da0a37fc15e73fd5f8df8fbb0502d1c Binary files /dev/null and b/vendor/zsks/docs/python/pic/image-20251009201657474.png differ diff --git a/vendor/zsks/docs/python/pic/image-20251009202143477.png b/vendor/zsks/docs/python/pic/image-20251009202143477.png new file mode 100755 index 0000000000000000000000000000000000000000..12d778905520fd6a1b734edf450d9977677c9c24 Binary files /dev/null and b/vendor/zsks/docs/python/pic/image-20251009202143477.png differ diff --git a/vendor/zsks/docs/python/pic/image-20251009202450693.png b/vendor/zsks/docs/python/pic/image-20251009202450693.png new file mode 100755 index 0000000000000000000000000000000000000000..08e1430c7f30533da40714205d12fd88a075e23f Binary files /dev/null and b/vendor/zsks/docs/python/pic/image-20251009202450693.png differ diff --git a/vendor/zsks/docs/python/pic/image-20251009202940688.png b/vendor/zsks/docs/python/pic/image-20251009202940688.png new file mode 100755 index 0000000000000000000000000000000000000000..c5385242c9f454412e5550fc9fc4d656d438c630 Binary files /dev/null and b/vendor/zsks/docs/python/pic/image-20251009202940688.png differ diff --git a/vendor/zsks/docs/python/pic/image-20251009203011267.png b/vendor/zsks/docs/python/pic/image-20251009203011267.png new file mode 100755 index 0000000000000000000000000000000000000000..bc9856619fb390f96226690c3659a30425bd0ec1 Binary files /dev/null and b/vendor/zsks/docs/python/pic/image-20251009203011267.png differ diff --git a/vendor/zsks/docs/python/pic/image-20251010094441549.png b/vendor/zsks/docs/python/pic/image-20251010094441549.png new file mode 100755 index 0000000000000000000000000000000000000000..e981001ec636db595d2766310bf95a49e58f3914 Binary files /dev/null and b/vendor/zsks/docs/python/pic/image-20251010094441549.png differ diff --git a/vendor/zsks/docs/python/pic/image-20251010100758153.png b/vendor/zsks/docs/python/pic/image-20251010100758153.png new file mode 100755 index 0000000000000000000000000000000000000000..736f301b3d0ea7959a6b1f1b7f23d3767d6945f8 Binary files /dev/null and b/vendor/zsks/docs/python/pic/image-20251010100758153.png differ diff --git a/vendor/zsks/docs/python/pic/image-20251010101732852.png b/vendor/zsks/docs/python/pic/image-20251010101732852.png new file mode 100755 index 0000000000000000000000000000000000000000..94055797bd0dd668e7618a2c5d31c064914198b1 Binary files /dev/null and b/vendor/zsks/docs/python/pic/image-20251010101732852.png differ diff --git a/vendor/zsks/docs/python/pic/image-20251010101851439.png b/vendor/zsks/docs/python/pic/image-20251010101851439.png new file mode 100755 index 0000000000000000000000000000000000000000..8921b22c17311aca77633404696f7a853cd557d5 Binary files /dev/null and b/vendor/zsks/docs/python/pic/image-20251010101851439.png differ diff --git a/vendor/zsks/docs/python/pic/image-20251010102204475.png b/vendor/zsks/docs/python/pic/image-20251010102204475.png new file mode 100755 index 0000000000000000000000000000000000000000..307037ddcfe30cc5560341d337c06f5ad876a2db Binary files /dev/null and b/vendor/zsks/docs/python/pic/image-20251010102204475.png differ diff --git a/vendor/zsks/docs/python/pic/image-20251010143023136.png b/vendor/zsks/docs/python/pic/image-20251010143023136.png new file mode 100755 index 0000000000000000000000000000000000000000..2432a1dd7f3002c59b3a311276aa596efa692839 Binary files /dev/null and b/vendor/zsks/docs/python/pic/image-20251010143023136.png differ diff --git a/vendor/zsks/docs/python/pic/image-20251010143136574.png b/vendor/zsks/docs/python/pic/image-20251010143136574.png new file mode 100755 index 0000000000000000000000000000000000000000..3e4887f3f65c92b3d964a3766ceab62b3ca80463 Binary files /dev/null and b/vendor/zsks/docs/python/pic/image-20251010143136574.png differ diff --git a/vendor/zsks/docs/python/pic/image-20251010143226900.png b/vendor/zsks/docs/python/pic/image-20251010143226900.png new file mode 100755 index 0000000000000000000000000000000000000000..cab5f395641fff439c2ca4d25781d954c7672526 Binary files /dev/null and b/vendor/zsks/docs/python/pic/image-20251010143226900.png differ diff --git a/vendor/zsks/docs/python/pic/image-20251010143640847.png b/vendor/zsks/docs/python/pic/image-20251010143640847.png new file mode 100755 index 0000000000000000000000000000000000000000..93009cf6ef951ba857d5e3609938138f911c678d Binary files /dev/null and b/vendor/zsks/docs/python/pic/image-20251010143640847.png differ diff --git a/vendor/zsks/docs/python/pic/image-20251010144051579.png b/vendor/zsks/docs/python/pic/image-20251010144051579.png new file mode 100755 index 0000000000000000000000000000000000000000..d22c6a78869110a3633e5acea33c7540464f7f4b Binary files /dev/null and b/vendor/zsks/docs/python/pic/image-20251010144051579.png differ diff --git a/vendor/zsks/docs/python/pic/image-20251010144137718.png b/vendor/zsks/docs/python/pic/image-20251010144137718.png new file mode 100755 index 0000000000000000000000000000000000000000..fc8d081784eb990a6c92c989b99faa5b13f8cd4c Binary files /dev/null and b/vendor/zsks/docs/python/pic/image-20251010144137718.png differ diff --git a/vendor/zsks/patch/cust_mpp_sample_patch.patch b/vendor/zsks/patch/cust_mpp_sample_patch.patch new file mode 100644 index 0000000000000000000000000000000000000000..9402885736c725316ce6505c6d663aa2cbec1f37 --- /dev/null +++ b/vendor/zsks/patch/cust_mpp_sample_patch.patch @@ -0,0 +1,381 @@ +diff -urN sample_back/common/sample_comm.h sample/common/sample_comm.h +--- sample_back/common/sample_comm.h 2025-09-15 16:34:35.381699231 +0800 ++++ sample/common/sample_comm.h 2025-09-16 16:53:26.946466207 +0800 +@@ -545,6 +545,12 @@ + td_u64 last_time; + td_u64 time_gap; + td_u64 fps; ++# ifdef USE_USB_CAMERA ++ td_char *v4l2_device; // v4l2设备路径，如"/dev/video0" ++ td_u32 width; // 摄像头采集分辨率宽 ++ td_u32 height; // 摄像头采集分辨率高 ++ td_u32 pixel_format; // 像素格式（如V4L2_PIX_FMT_MJPEG） ++# endif + } vdec_thread_param; + + typedef struct { +diff -urN sample_back/common/sample_comm_vdec.c sample/common/sample_comm_vdec.c +--- sample_back/common/sample_comm_vdec.c 2025-09-15 16:34:35.379699216 +0800 ++++ sample/common/sample_comm_vdec.c 2025-09-16 17:41:41.799826173 +0800 +@@ -32,6 +32,18 @@ + #include + #include "sample_comm.h" + ++#ifdef USE_USB_CAMERA ++/* 新增v4l2相关头文件 */ ++#include // 文件控制 ++#include // POSIX API ++#include // IO控制 ++#include // 内存映射 ++#include // select复用 ++#include // v4l2核心 ++#include // 错误处理 ++#include // 字符串操作 ++#endif ++ + #define SEND_STREAM_CNT 5 + #define VDEC_SECOND 1000000 + #define SAMPLE_VDEC_LOW_DELAY_LINE_CNT 16 +@@ -633,6 +645,244 @@ + return TD_SUCCESS; + } + ++#ifdef USE_USB_CAMERA ++ ++td_void *sample_comm_vdec_send_stream(td_void *args) ++{ ++ printf("vdec send stream thread start\n"); ++ ++ vdec_thread_param *thread_param = (vdec_thread_param *)args; ++ ot_vdec_stream stream; ++ printf("sample_comm_vdec_send_stream v4l2 \r\n"); ++ // v4l2专用变量 ++ td_s32 v4l2_fd = -1; ++ struct v4l2_capability cap; ++ struct v4l2_format fmt; ++ struct v4l2_requestbuffers req; ++ struct v4l2_buffer v4l2_buf; ++ enum v4l2_buf_type buf_type; ++ fd_set fds; ++ struct timeval tv; ++ td_u8 *v4l2_buffers[4] = {NULL}; ++ td_u32 buffer_count = 0; ++ td_u32 max_buffer_length = 0; ++ ++ prctl(PR_SET_NAME, "v4l2_to_vdec", 0, 0, 0); ++ ++ // 参数验证 ++ if (thread_param == NULL) { ++ sample_print("线程参数为空!"); ++ return (td_void *)TD_FAILURE; ++ } ++ ++ sample_print("启动v4l2摄像头采集: %s", thread_param->v4l2_device); ++ ++ // 1. 打开v4l2设备 ++ v4l2_fd = open(thread_param->v4l2_device, O_RDWR | O_NONBLOCK, 0); ++ if (v4l2_fd < 0) { ++ sample_print("无法打开摄像头 %s: %s", ++ thread_param->v4l2_device, strerror(errno)); ++ return (td_void *)(TD_FAILURE); ++ } ++ ++ // 2. 查询设备能力 ++ if (ioctl(v4l2_fd, VIDIOC_QUERYCAP, &cap) < 0) { ++ sample_print("VIDIOC_QUERYCAP错误: %s", strerror(errno)); ++ goto v4l2_cleanup; ++ } ++ ++ if (!(cap.capabilities & V4L2_CAP_STREAMING)) { ++ sample_print("设备不支持流式传输"); ++ goto v4l2_cleanup; ++ } ++ ++ // 3. 设置视频格式 ++ memset(&fmt, 0, sizeof(fmt)); ++ fmt.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; ++ fmt.fmt.pix.width = thread_param->width; ++ fmt.fmt.pix.height = thread_param->height; ++ fmt.fmt.pix.pixelformat = thread_param->pixel_format; ++ fmt.fmt.pix.field = V4L2_FIELD_ANY; ++ ++ if (ioctl(v4l2_fd, VIDIOC_S_FMT, &fmt) < 0) { ++ sample_print("VIDIOC_S_FMT错误: %s", strerror(errno)); ++ goto v4l2_cleanup; ++ } ++ ++ // 4. 设置帧率（可选） ++ struct v4l2_streamparm parm; ++ memset(&parm, 0, sizeof(parm)); ++ parm.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; ++ parm.parm.capture.timeperframe.numerator = 1; ++ parm.parm.capture.timeperframe.denominator = thread_param->fps; ++ if (ioctl(v4l2_fd, VIDIOC_S_PARM, &parm) < 0) { ++ sample_print("设置帧率失败，使用默认值: %s", strerror(errno)); ++ } ++ ++ // 5. 请求缓冲区 ++ memset(&req, 0, sizeof(req)); ++ req.count = 4; // 使用4个缓冲区 ++ req.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; ++ req.memory = V4L2_MEMORY_MMAP; ++ ++ if (ioctl(v4l2_fd, VIDIOC_REQBUFS, &req) < 0) { ++ sample_print("VIDIOC_REQBUFS错误: %s", strerror(errno)); ++ goto v4l2_cleanup; ++ } ++ ++ buffer_count = req.count; ++ max_buffer_length = 0; ++ ++ // 6. 映射缓冲区 ++ for (td_u32 i = 0; i < buffer_count; ++i) { ++ memset(&v4l2_buf, 0, sizeof(v4l2_buf)); ++ v4l2_buf.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; ++ v4l2_buf.memory = V4L2_MEMORY_MMAP; ++ v4l2_buf.index = i; ++ ++ if (ioctl(v4l2_fd, VIDIOC_QUERYBUF, &v4l2_buf) < 0) { ++ sample_print("VIDIOC_QUERYBUF错误: %s", strerror(errno)); ++ goto v4l2_cleanup; ++ } ++ ++ // 记录最大缓冲区长度 ++ if (v4l2_buf.length > max_buffer_length) { ++ max_buffer_length = v4l2_buf.length; ++ } ++ ++ v4l2_buffers[i] = mmap(NULL, v4l2_buf.length, ++ PROT_READ | PROT_WRITE, ++ MAP_SHARED, ++ v4l2_fd, ++ v4l2_buf.m.offset); ++ ++ if (v4l2_buffers[i] == MAP_FAILED) { ++ sample_print("mmap失败: %s", strerror(errno)); ++ v4l2_buffers[i] = NULL; ++ goto v4l2_cleanup; ++ } ++ } ++ ++ // 7. 将缓冲区放入队列 ++ for (td_u32 i = 0; i < buffer_count; ++i) { ++ memset(&v4l2_buf, 0, sizeof(v4l2_buf)); ++ v4l2_buf.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; ++ v4l2_buf.memory = V4L2_MEMORY_MMAP; ++ v4l2_buf.index = i; ++ ++ if (ioctl(v4l2_fd, VIDIOC_QBUF, &v4l2_buf) < 0) { ++ sample_print("VIDIOC_QBUF错误: %s", strerror(errno)); ++ goto v4l2_cleanup; ++ } ++ } ++ ++ // 8. 启动视频流 ++ buf_type = V4L2_BUF_TYPE_VIDEO_CAPTURE; ++ if (ioctl(v4l2_fd, VIDIOC_STREAMON, &buf_type) < 0) { ++ sample_print("VIDIOC_STREAMON错误: %s", strerror(errno)); ++ goto v4l2_cleanup; ++ } ++ ++ sample_print("v4l2摄像头已启动: %s, %dx%d@%llufps, 格式: %c%c%c%c", ++ thread_param->v4l2_device, ++ thread_param->width, ++ thread_param->height, ++ thread_param->fps, ++ thread_param->pixel_format & 0xFF, ++ (thread_param->pixel_format >> 8) & 0xFF, ++ (thread_param->pixel_format >> 16) & 0xFF, ++ (thread_param->pixel_format >> 24) & 0xFF); ++ ++ // 9. 主循环：采集并发送到VDEC ++ while (thread_param->e_thread_ctrl == THREAD_CTRL_START) { ++ FD_ZERO(&fds); ++ FD_SET(v4l2_fd, &fds); ++ ++ // 设置超时时间（2秒） ++ tv.tv_sec = 2; ++ tv.tv_usec = 0; ++ ++ td_s32 r = select(v4l2_fd + 1, &fds, NULL, NULL, &tv); ++ if (r == -1) { ++ if (errno == EINTR) continue; // 被信号中断 ++ sample_print("select错误: %s", strerror(errno)); ++ break; ++ } ++ ++ if (r == 0) { ++ sample_print("select超时"); ++ continue; ++ } ++ ++ // 从队列获取缓冲区 ++ memset(&v4l2_buf, 0, sizeof(v4l2_buf)); ++ v4l2_buf.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; ++ v4l2_buf.memory = V4L2_MEMORY_MMAP; ++ ++ if (ioctl(v4l2_fd, VIDIOC_DQBUF, &v4l2_buf) < 0) { ++ sample_print("VIDIOC_DQBUF错误: %s", strerror(errno)); ++ continue; ++ } ++ ++ // 检查有效数据长度 ++ if (v4l2_buf.bytesused == 0) { ++ sample_print("接收到空帧"); ++ goto requeue_buffer; ++ } ++ ++ // 封装VDEC流结构 ++ memset(&stream, 0, sizeof(ot_vdec_stream)); ++ stream.addr = v4l2_buffers[v4l2_buf.index]; ++ stream.len = v4l2_buf.bytesused; ++ stream.end_of_stream = TD_FALSE; ++ ++ // 使用系统时间作为PTS ++ struct timeval now; ++ gettimeofday(&now, NULL); ++ stream.pts = now.tv_sec * 1000000 + now.tv_usec; // 单位微秒 ++ ++ // 发送到VDEC ++ td_s32 s32Ret = ss_mpi_vdec_send_stream(thread_param->chn_id, &stream, 30); // 30ms超时 ++ if (s32Ret != TD_SUCCESS) { ++ sample_print("发送到VDEC失败, ret=%#x", s32Ret); ++ } ++ ++requeue_buffer: ++ // 将缓冲区重新放回队列 ++ if (ioctl(v4l2_fd, VIDIOC_QBUF, &v4l2_buf) < 0) { ++ sample_print("VIDIOC_QBUF错误: %s", strerror(errno)); ++ break; ++ } ++ } ++ ++ // 10. 发送流结束标志 ++ memset(&stream, 0, sizeof(ot_vdec_stream)); ++ stream.end_of_stream = TD_TRUE; ++ ss_mpi_vdec_send_stream(thread_param->chn_id, &stream, -1); ++ ++ sample_print("v4l2采集线程退出"); ++ ++v4l2_cleanup: ++ // 11. 停止视频流 ++ if (v4l2_fd >= 0) { ++ buf_type = V4L2_BUF_TYPE_VIDEO_CAPTURE; ++ ioctl(v4l2_fd, VIDIOC_STREAMOFF, &buf_type); ++ } ++ ++ // 12. 释放v4l2资源 ++ for (td_u32 i = 0; i < buffer_count; ++i) { ++ if (v4l2_buffers[i]) { ++ munmap(v4l2_buffers[i], max_buffer_length); ++ } ++ } ++ ++ if (v4l2_fd >= 0) { ++ close(v4l2_fd); ++ } ++ ++ return (td_void *)TD_SUCCESS; ++} ++#else + td_void *sample_comm_vdec_send_stream(td_void *args) + { + vdec_thread_param *thread_param = (vdec_thread_param *)args; +@@ -691,6 +941,7 @@ + path = TD_NULL; + return (td_void *)TD_SUCCESS; + } ++#endif + + td_void sample_comm_vdec_cmd_not_circle_send(td_u32 chn_num, vdec_thread_param *vdec_send, + pthread_t *vdec_thread, td_u32 send_arr_len, td_u32 thread_arr_len) +diff -urN sample_back/svp/common/sample_common_svp.c sample/svp/common/sample_common_svp.c +--- sample_back/svp/common/sample_common_svp.c 2025-09-15 16:34:34.655693673 +0800 ++++ sample/svp/common/sample_common_svp.c 2025-09-16 15:31:57.572977319 +0800 +@@ -428,7 +428,7 @@ + } + } + +-static td_s32 sample_common_svp_get_def_vo_cfg(sample_vo_cfg *vo_cfg) ++td_s32 sample_common_svp_get_def_vo_cfg(sample_vo_cfg *vo_cfg) + { + ot_rect def_disp_rect = {0, 0, 1920, 1080}; + ot_size def_img_size = {1920, 1080}; +diff -urN sample_back/svp/common/sample_common_svp.h sample/svp/common/sample_common_svp.h +--- sample_back/svp/common/sample_common_svp.h 2025-09-15 16:34:34.651693642 +0800 ++++ sample/svp/common/sample_common_svp.h 2025-09-16 19:50:04.386828575 +0800 +@@ -60,6 +60,8 @@ + + typedef struct { + ot_point point[OT_POINT_NUM]; ++ td_float score; ++ td_u16 class_id; + } ot_sample_svp_rect; + + typedef struct { +diff -urN sample_back/svp/common/sample_common_svp_npu_model.c sample/svp/common/sample_common_svp_npu_model.c +--- sample_back/svp/common/sample_common_svp_npu_model.c 2025-09-15 16:34:34.649693627 +0800 ++++ sample/svp/common/sample_common_svp_npu_model.c 2025-09-16 20:01:09.432093229 +0800 +@@ -999,6 +999,8 @@ + td_float *y_min = TD_NULL; + td_float *x_max = TD_NULL; + td_float *y_max = TD_NULL; ++ td_float *score = TD_NULL; ++ td_float *class_id = TD_NULL; + td_u32 roi_offset = info->roi_offset; + td_u32 i; + +@@ -1019,6 +1021,8 @@ + y_min = x_min + stride / sizeof(td_float); + x_max = y_min + stride / sizeof(td_float); + y_max = x_max + stride / sizeof(td_float); ++ score = y_max + stride / sizeof(td_float); ++ class_id = score + stride / sizeof(td_float); + + for (i = 0; i < rect_info->num; i++) { + rect_info->rect[i].point[SAMPLE_SVP_NPU_RECT_LEFT_TOP].x = (td_u32)((td_float)x_min[roi_offset] / +@@ -1040,6 +1044,10 @@ + rect_info->rect[i].point[SAMPLE_SVP_NPU_RECT_LEFT_TOP].x; + rect_info->rect[i].point[SAMPLE_SVP_NPU_RECT_LEFT_BOTTOM].y = + rect_info->rect[i].point[SAMPLE_SVP_NPU_RECT_RIGHT_BOTTOM].y; ++ ++ rect_info->rect[i].score = (td_float)((td_float)score[roi_offset]); ++ rect_info->rect[i].class_id = (td_u16)((td_float)class_id[roi_offset]); ++ + roi_offset++; + } + return TD_SUCCESS; +@@ -1319,6 +1327,9 @@ + const ot_video_frame_info *show_frame, ot_sample_svp_rect_info *rect_info) + { + td_s32 ret; ++ td_u32 i; ++ td_float max_confidence = 0.0f; ++ td_u32 max_index = 0; + + if (sample_svp_check_task_cfg(task) != TD_SUCCESS) { + sample_svp_trace_err("check task cfg failed!\n"); +@@ -1342,6 +1353,23 @@ + sample_svp_check_exps_return(ret != TD_SUCCESS, TD_FAILURE, SAMPLE_SVP_ERR_LEVEL_ERROR, + "get roi failed!\n"); + ++ // �������֣�ɸѡ���Ŷ���ߵľ��ο� ++ if (rect_info->num > 0) { ++ // 1. �������Ŷ���ߵľ��ο� ++ for (i = 0; i < rect_info->num; i++) { ++ if (rect_info->rect[i].score > max_confidence) { ++ max_confidence = rect_info->rect[i].score; ++ max_index = i; ++ } ++ } ++ ++ // 2. ֻ�������Ŷ���ߵľ��ο� ++ if (rect_info->num > 1) { ++ rect_info->rect[0] = rect_info->rect[max_index]; ++ rect_info->num = 1; ++ } ++ } ++ + return TD_SUCCESS; + } + diff --git a/vendor/zsks/pic/image-20250918185148122.png b/vendor/zsks/pic/image-20250918185148122.png new file mode 100755 index 0000000000000000000000000000000000000000..ececc626b71b0015ca5c43b3ff9b022b19a0a0be Binary files /dev/null and b/vendor/zsks/pic/image-20250918185148122.png differ