一.检测人脸并拍照
1.流程：
这个功能只需要用一个主线程就可以完成，首先要初始化 VI 模块、ROCKX 模块并启动 VI 模块采集摄像头数据。
初始化完成之后，则开始获取每一帧的 VI 码流数据，并且用 rockx 框架对每一帧视频数据进行人脸检测，若检测出人脸则用Opencv 对人脸图像进行截取，并用 opencv 的 API 把 NV12 图像转换为 BGR 图像，最后把转换后的 BGR 图像保存起来。
2.代码实现
/****************************************************************************
*
* Copyright (c) 2017 - 2019 by Rockchip Corp. All rights reserved.
*
* The material in this file is confidential and contains trade secrets
* of Rockchip Corporation. This is proprietary information owned by
* Rockchip Corporation. No part of this work may be disclosed,
* reproduced, copied, transmitted, or used in any way for any purpose,
* without the express written permission of Rockchip Corporation.
*
*****************************************************************************/
#include <assert.h>
#include <cstddef>
#include <fcntl.h>
#include <getopt.h>
#include <opencv2/imgproc.hpp>
#include <opencv2/imgproc/imgproc_c.h>
#include <pthread.h>
#include <signal.h>
#include <stdbool.h>
#include <stdio.h>
#include <stdlib.h>
#include <time.h>
#include <unistd.h>
// #include "common/sample_common.h"
#include "rkmedia_api.h"
#include "rknn_rockx_include/rockx_type.h"
#include "rockx.h"
#include <opencv2/core.hpp>
// #include <opencv2/imgoroc.hpp>
#include <opencv2/highgui.hpp>
#include <opencv2/opencv.hpp>
#define CAMERA_PATH "rkispp_scale0"
#define CAMERA_ID 0
#define CAMERA_CHN 0
#define WIDTH 1920
#define HEIGHT 1080
using namespace cv;
void nv12ToBGR(uint8_t *nv12_buf, int width, int height, uint8_t *bgr_buf)
{
Mat nv12_mat(height + height / 2, width, CV_8UC1, nv12_buf);//这里面height + height / 2是NV12单通道保存的格式
Mat bgr_mat(height, width, CV_8UC3, bgr_buf);
cvtColor(nv12_mat, bgr_mat, COLOR_YUV2BGR_NV12);
}
int main(int argc, char **argv)
{
//**********************VI模块初始化**********************************/
int ret;
VI_CHN_ATTR_S vi_chn_attr;
vi_chn_attr.pcVideoNode = CAMERA_PATH; // Path
vi_chn_attr.u32Width = 1920; // Width
vi_chn_attr.u32Height = 1080; // Height
vi_chn_attr.enPixFmt = IMAGE_TYPE_NV12; // ImageType
vi_chn_attr.enBufType = VI_CHN_BUF_TYPE_MMAP; // BufType
vi_chn_attr.u32BufCnt = 3; // Cnt
vi_chn_attr.enWorkMode = VI_WORK_MODE_NORMAL; // Mode
ret = RK_MPI_VI_SetChnAttr(CAMERA_ID, CAMERA_CHN, &vi_chn_attr);
if (ret)
{
printf("Vi Set Attr Failed.....\n");
return 0;
}
else
{
printf("Vi Set Attr Success.....\n");
}
ret = RK_MPI_VI_EnableChn(CAMERA_ID, CAMERA_CHN);
if (ret)
{
printf("Vi Enable Attr Failed.....\n");
return 0;
}
else
{
printf("Vi Enable Attr Success.....\n");
}
ret = RK_MPI_VI_StartStream(CAMERA_ID, CAMERA_CHN);
if (ret)
{
printf("RK_MPI_VI_StartStream Failed.....\n");
return 0;
}
else
{
printf("RK_MPI_VI_StartStream Success.....\n");
}
//******************************ROCKX模型初始化*************************/
rockx_config_t *face_detect_config = rockx_create_config();
rockx_add_config(face_detect_config, ROCKX_CONFIG_DATA_PATH, "/userdata/rockx_data");
rockx_handle_t face_detect_handle;
rockx_ret_t face_detect_ret;
rockx_module_t face_detect_module = ROCKX_MODULE_FACE_DETECTION_V3;
face_detect_ret = rockx_create(&face_detect_handle, face_detect_module, face_detect_config, 0);
if (face_detect_ret != ROCKX_RET_SUCCESS)
{
printf("rockx_create face_detect failed...\n");
return -1;
}
//******************************ROCKX图像初始化*************************/
rockx_image_t rv1126_rockx_image;//图像数据
rv1126_rockx_image.width = WIDTH;//图像宽度
rv1126_rockx_image.height = HEIGHT;//图像高度
rv1126_rockx_image.pixel_format = ROCKX_PIXEL_FORMAT_YUV420SP_NV12;//图像格式
//******************************图像数据获取*************************/
MEDIA_BUFFER mb;
uint8_t *bgr = new uint8_t[WIDTH * HEIGHT * 3];//用于存放单通道转换为三通道的图像数据
char face_id[64]; //用于存放人脸id
while (1)
{
//从VI模块获取数据
mb = RK_MPI_SYS_GetMediaBuffer(RK_ID_VI, CAMERA_CHN, -1);
if (!mb)
{
printf("Get Vi Stream break....\n");
break;
}
//VI模块数据放入rockx_image_t结构体里面
rv1126_rockx_image.data = (uint8_t *)RK_MPI_MB_GetPtr(mb);
rv1126_rockx_image.size = RK_MPI_MB_GetSize(mb);
//开启人脸检测
rockx_object_array_t face_detect_array;
face_detect_ret = rockx_face_detect(face_detect_handle, &rv1126_rockx_image, &face_detect_array, NULL);
if (face_detect_ret != ROCKX_RET_SUCCESS)
{
printf("face_detect failed....\n");
}
//获取人脸检测框
if(face_detect_array.count > 0)
{
printf("snap shot face detect success!\n");
Mat face_img = Mat(HEIGHT, WIDTH, CV_8UC1, rv1126_rockx_image.data);//将图像数据转换为Mat单通道格式(也就是灰色图像)
nv12ToBGR(face_img.data, WIDTH, HEIGHT, bgr);//将NV12格式转换为BGR格式,这个函数是自己定义的
Mat out_mat = Mat(HEIGHT, WIDTH, CV_8UC3, bgr);//将BGR格式图像数据转换为Mat三通道格式(也就是彩色图像)
srand(time(NULL));//设置随机数种子
sprintf(face_id, "face_%d.jpg", rand());//生成随机数作为人脸id
imwrite(face_id, out_mat);//将人脸图像保存到本地
}
RK_MPI_MB_ReleaseBuffer(mb);
}
delete[] bgr; //释放内存
return 0;
}
3.效果：亲测可以
二.ROCKX+RV1126 实现 1->N 人脸识别功能
1.流程：
首先要初始化 RV1126 模块初始化，包括 VI 模块、VENC 模块、人脸检测 rockx 模块、人脸识别rockx 模块，初始化模块之后，就要分两个线程做处理。
主流程是先读取单张图片的图像并提取人脸特征值，然后死循环获取 VI 的码流数据，然后用 rockx 的人脸检测模块 RV1126 的 VI数据是否有人脸，如果有人脸则调用 rockx 的人脸识别模块识别出 RV1126 视频流的人脸数据并且提取出来。然后对比两个人脸的阈值，如果<=1.0，则认定单张人脸图片和 RV1126 检测的人脸是同一个，否则就不是同一个人，并把数据通过 Opencv 显示到VI 数据，最后把识别后的 VI 数据传输到 VENC 编码器里面
get_rockx_face_recg_venc_thread 线程主要是获取每一帧的 VENC 码流数据，并且保存起来。
2.代码实现
/****************************************************************************
*
* Copyright (c) 2017 - 2019 by Rockchip Corp. All rights reserved.
*
* The material in this file is confidential and contains trade secrets
* of Rockchip Corporation. This is proprietary information owned by
* Rockchip Corporation. No part of this work may be disclosed,
* reproduced, copied, transmitted, or used in any way for any purpose,
* without the express written permission of Rockchip Corporation.
*
*****************************************************************************/
#include <assert.h>
#include <cstddef>
#include <fcntl.h>
#include <getopt.h>
#include <opencv2/imgproc.hpp>
#include <opencv2/imgproc/imgproc_c.h>
#include <pthread.h>
#include <signal.h>
#include <stdbool.h>
#include <stdio.h>
#include <stdlib.h>
#include <time.h>
#include <unistd.h>
// #include "common/sample_common.h"
#include "rkmedia_api.h"
#include "rknn_rockx_include/rockx_type.h"
#include "rockx.h"
#include <opencv2/core.hpp>
// #include <opencv2/imgoroc.hpp>
#include <opencv2/highgui.hpp>
#include <opencv2/opencv.hpp>
#define CAMERA_PATH "rkispp_scale0"
#define CAMERA_ID 0
#define CAMERA_CHN 0
#define VENC_CHN 0
#define WIDTH 1920
#define HEIGHT 1080
using namespace cv;
String name; //用于存放人脸名字
void* venc_thread_func(void *args)
{
pthread_detach(pthread_self());
FILE *fp = fopen("test.h264", "w+");
MEDIA_BUFFER mb = NULL;
while (1)
{
mb = RK_MPI_SYS_GetMediaBuffer(RK_ID_VENC,VENC_CHN, -1);
if (!mb)
{
printf("RK_MPI_SYS_GetMediaBuffer RK_ID_VENC failed\n");
break;
}
fwrite(RK_MPI_MB_GetPtr(mb), RK_MPI_MB_GetSize(mb),1 , fp);
RK_MPI_MB_ReleaseBuffer(mb);
}
return NULL;
}
/*void nv12ToBGR(uint8_t *nv12_buf, int width, int height, uint8_t *bgr_buf)
{
Mat nv12_mat(height + height / 2, width, CV_8UC1, nv12_buf);
Mat bgr_mat(height, width, CV_8UC3, bgr_buf);
cvtColor(nv12_mat, bgr_mat, COLOR_YUV2BGR_NV12);
}
*/
int main(int argc, char **argv)
{
//**********************VI模块初始化**********************************/
int ret;
VI_CHN_ATTR_S vi_chn_attr;
vi_chn_attr.pcVideoNode = CAMERA_PATH; // Path
vi_chn_attr.u32Width = 1920; // Width
vi_chn_attr.u32Height = 1080; // Height
vi_chn_attr.enPixFmt = IMAGE_TYPE_NV12; // ImageType
vi_chn_attr.enBufType = VI_CHN_BUF_TYPE_MMAP; // BufType
vi_chn_attr.u32BufCnt = 3; // Cnt
vi_chn_attr.enWorkMode = VI_WORK_MODE_NORMAL; // Mode
ret = RK_MPI_VI_SetChnAttr(CAMERA_ID, CAMERA_CHN, &vi_chn_attr);
if (ret)
{
printf("Vi Set Attr Failed.....\n");
return 0;
}
else
{
printf("Vi Set Attr Success.....\n");
}
ret = RK_MPI_VI_EnableChn(CAMERA_ID, CAMERA_CHN);
if (ret)
{
printf("Vi Enable Attr Failed.....\n");
return 0;
}
else
{
printf("Vi Enable Attr Success.....\n");
}
VENC_CHN_ATTR_S venc_chn_attr;
memset(&venc_chn_attr, 0, sizeof(VENC_CHN_ATTR_S));
venc_chn_attr.stVencAttr.u32PicWidth = 1920;
venc_chn_attr.stVencAttr.u32PicHeight = 1080;
venc_chn_attr.stVencAttr.u32VirWidth = 1920;
venc_chn_attr.stVencAttr.u32VirHeight = 1080;
venc_chn_attr.stVencAttr.imageType = IMAGE_TYPE_NV12;
venc_chn_attr.stVencAttr.enType = RK_CODEC_TYPE_H264;
venc_chn_attr.stVencAttr.u32Profile = 66;
venc_chn_attr.stRcAttr.enRcMode = VENC_RC_MODE_H264CBR;
venc_chn_attr.stRcAttr.stH264Cbr.u32Gop = 25;
venc_chn_attr.stRcAttr.stH264Cbr.u32BitRate = 1920 * 1080 * 3;
venc_chn_attr.stRcAttr.stH264Cbr.fr32DstFrameRateDen = 1;
venc_chn_attr.stRcAttr.stH264Cbr.fr32DstFrameRateNum = 25;
venc_chn_attr.stRcAttr.stH264Cbr.u32SrcFrameRateDen = 1;
venc_chn_attr.stRcAttr.stH264Cbr.u32SrcFrameRateNum = 25;
ret = RK_MPI_VENC_CreateChn(VENC_CHN, &venc_chn_attr);
if (ret)
{
printf("ERROR: Create venc failed!\n");
return 0;
}
ret = RK_MPI_VI_StartStream(CAMERA_ID, CAMERA_CHN);
if (ret)
{
printf("RK_MPI_VI_StartStream Failed.....\n");
return 0;
}
else
{
printf("RK_MPI_VI_StartStream Success.....\n");
}
//******************************ROCKX模型初始化*************************/
rockx_ret_t ret1;
rockx_config_t *face_detect_config = rockx_create_config();
ret1 = rockx_add_config(face_detect_config, ROCKX_CONFIG_DATA_PATH, "/userdata/rockx_data");
if (ret1 != ROCKX_RET_SUCCESS)
{
printf("ERROR: rockx_add_config failed!\n");
return -1;
}
//ROCKX检测人脸模块初始化
rockx_handle_t face_detect_handle;
rockx_ret_t face_detect_ret;
rockx_module_t face_detect_module = ROCKX_MODULE_FACE_DETECTION_V3;
face_detect_ret = rockx_create(&face_detect_handle, face_detect_module, face_detect_config, 0);
if (face_detect_ret != ROCKX_RET_SUCCESS)
{
printf("rockx_create face_detect failed...\n");
return -1;
}
//ROCKX人脸识别模块初始化
rockx_handle_t face_find_handle;
rockx_ret_t face_find_ret;
rockx_module_t face_find_module = ROCKX_MODULE_FACE_RECOGNIZE;
face_find_ret = rockx_create(&face_find_handle, face_find_module, face_detect_config, 0);
if (face_find_ret != ROCKX_RET_SUCCESS)
{
printf("rockx_create face_detect failed...\n");
return -1;
}
//******************************ROCKX读取单张脸特征*************************/
//读取人脸图片
rockx_ret_t read;
const char *image_path = argv[1];
rockx_image_t face_image;
read = rockx_image_read(image_path, &face_image, 1);
if (read != ROCKX_RET_SUCCESS)
{
printf("rockx_image_read failed...\n");
return -1;
}
//读取人脸特征
rockx_ret_t regnize;
rockx_face_feature_t pic_face_feature;
regnize = rockx_face_recognize(face_find_handle, &face_image, &pic_face_feature);
if (regnize != ROCKX_RET_SUCCESS)
{
printf("rockx_face_recognize failed...\n");
return -1;
}
//****************************线程获取VENC数据******************************/
pthread_t venc_thread;
pthread_create(&venc_thread, NULL, venc_thread_func, NULL);
//******************************ROCKX图像初始化*************************/
rockx_image_t rv1126_rockx_image;//图像数据
rv1126_rockx_image.width = WIDTH;//图像宽度
rv1126_rockx_image.height = HEIGHT;//图像高度
rv1126_rockx_image.pixel_format = ROCKX_PIXEL_FORMAT_YUV420SP_NV12;//图像格式
//******************************图像数据获取*************************/
MEDIA_BUFFER mb = NULL;
Point testPoint; //用于存放人脸位置
testPoint.x = 300;
testPoint.y = 300;
rockx_ret_t face1;
rockx_ret_t face2;
rockx_ret_t face3;
float similarity = 0;
while (1)
{
//从VI模块获取数据
mb = RK_MPI_SYS_GetMediaBuffer(RK_ID_VI, CAMERA_CHN, -1);
if (!mb)
{
printf("Get Vi Stream break....\n");
break;
}
//VI模块数据放入rockx_image_t结构体里面
rv1126_rockx_image.data = (uint8_t *)RK_MPI_MB_GetPtr(mb);
rv1126_rockx_image.size = RK_MPI_MB_GetSize(mb);
Mat face_rect = Mat(HEIGHT, WIDTH, CV_8UC1, rv1126_rockx_image.data);//将图像数据转换为Mat格式
//开启人脸检测
rockx_object_array_t face_detect_array;
face1 = rockx_face_detect(face_detect_handle, &rv1126_rockx_image, &face_detect_array, NULL);
if (face1 != ROCKX_RET_SUCCESS)
{
printf("face_detect failed....\n");
}
//获取VI人脸特征
rockx_face_feature_t out_facture;
if(face_detect_array.count > 0 )
{
face2 = rockx_face_recognize(face_find_handle, &rv1126_rockx_image, &out_facture);
if (face2 != ROCKX_RET_SUCCESS)
{
printf("face_recognize failed....\n");
}
//对比人脸特征
face3 = rockx_face_feature_similarity(&pic_face_feature, &out_facture, &similarity);
if (face3 != ROCKX_RET_SUCCESS)
{
printf("face_feature_similarity failed....\n");
}
if (similarity <= 1)
{
name = "rockx";
printf("This is rockx\n");
}
else
{
name = "";
printf("Can not recognize...\n");
}
//把名字放到框上面
putText(face_rect, name, testPoint, FONT_HERSHEY_SIMPLEX, 1, Scalar(0, 0, 255), 2);
}
//把上面处理好的VI数据传给VENC
RK_MPI_SYS_SendMediaBuffer( RK_ID_VENC, VENC_CHN, mb);
RK_MPI_MB_ReleaseBuffer(mb);
}
//释放内存
RK_MPI_VENC_DestroyChn(VENC_CHN); //释放VENC通道
RK_MPI_VI_DisableChn(CAMERA_ID,CAMERA_CHN); //释放VI通道
return 0;
}