【基于树莓派400的图像识别归类&运动检测&模拟信号处理系统第一帖】MJPEG

donatello1996 · 发表于2022-10-23 01:00

【基于树莓派400的图像识别归类&运动检测&模拟信号处理系统第一帖】MJPEG [复制链接]

【基于树莓派400的图像识别归类&运动检测&模拟信号处理系统第一帖】读取摄像头MJPEG流数据并发送到HTTP服务器上&TFLITE实现AI图像识别归类（image classify）

我的这个项目首先要用到的就是树莓派400与USB UVC摄像头的通信，这个非常简单，直接使用Linux系统自带的V4L2库就可以，绝大部分板子都支持UVC和V4L2驱动，不需要做啥设置或添加啥新库。稍微有难点的地方就是引用libjpeg库和tflite库，这两个库如果之前有相关项目经验积累的话也是很快上手的。首先是安装libjpeg库:

apt install libjpeg62-turbo libjpeg62-turbo-dev

然后是驱动代码，也非常简单：

void * Thread_V4l2_Grab_Mjpeg(void *arg)
{
    pic_data pic_temp;
    while(1)
    {
        int type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
        if(ioctl(fd_video , VIDIOC_STREAMON , &type) < 0)
        {
            printf("Unable to start capture.\n");
            break;
        }

        struct v4l2_buffer buff;
        buff.type   = V4L2_BUF_TYPE_VIDEO_CAPTURE;
        buff.memory = V4L2_MEMORY_MMAP;
        if(ioctl(fd_video , VIDIOC_DQBUF, &buff) < 0)
        {
            printf("camera VIDIOC_DQBUF Failed.\n");
            usleep(1000*1000);
            break;
        }

        pthread_mutex_lock(&pmt);

        memcpy(pic_tmpbuffer , pic.tmpbuffer , buff.bytesused);
        //pic_tmpbuffer = pic.tmpbuffer;
        pic.tmpbytesused = buff.bytesused;
        pic_tmpbytesused = pic.tmpbytesused;

        //if(build_file == true)
        //{
            int jpg_fd = open(MJPEG_FILE_NAME , O_RDWR | O_CREAT , 00700);
            if(jpg_fd == -1)
            {
                printf("open ipg Failed!\n ");
                break ;
            }
            int writesize = write(jpg_fd , pic.tmpbuffer , pic.tmpbytesused);
            printf("Write successfully size : %d\n" , writesize);
            close(jpg_fd);
        //}

        pthread_cond_broadcast(&pct);
        pthread_mutex_unlock(&pmt);

        printf("pic.tmpbytesused size : %d\n",pic.tmpbytesused);

        if(ioctl(fd_video , VIDIOC_QBUF, &buff) < 0)
        {
            printf("camera VIDIOC_QBUF Failed.\n");
            usleep(1000*1000);
            break;
        }
    }
}

抓取出的MJPEG流，可以以缓存形式直接调用，也可以保存成文件方便其它进程或同进程的其它线程引用，只是对MJPEG流的调用必须要加线程锁，不加锁的话帧数会提高，但是会导致画面撕裂或随机缺失。注意，使用V4L2驱动库之前必须先初始化，不然是无法使用的！

为了方便其它设备显示抓取出的MJPEG流，我这边又开辟了TCP接收线程和TCP发送线程搭建HTTP WEB服务器：

    pthread_create(&tid_tcp_web_recv , NULL , Thread_TCP_Web_Recv , NULL);
    pthread_create(&tid_tcp_web_send , NULL , Thread_TCP_Web_Send_Only_JPEG_File , NULL);

void HTTP_Send_Jpeg_File_Stream(int fd , const char filename[] , pthread_mutex_t *pmt , pthread_cond_t *pct)
{
    unsigned char *frame = NULL;
    int frame_size = 0;
    char buffer[BUFFER_SIZE] = {0};

    int fd_image_file = open(filename , O_RDONLY);

    printf("preparing header\n");
    sprintf(buffer, "HTTP/1.0 200 OK\r\n" \
            "Access-Control-Allow-Origin: *\r\n" \
            STD_HEADER \
            "Content-Type: multipart/x-mixed-replace;boundary=" BOUNDARY "\r\n" \
            "\r\n" \
            "--" BOUNDARY "\r\n");

    if(write(fd, buffer, strlen(buffer)) < 0) 
    {
        free(frame);
        return;
    }

    printf("Headers send, sending stream now\n");

    while(1) 
    {
        pthread_mutex_lock(pmt);
        pthread_cond_wait(pct , pmt);

        frame_size = lseek(fd_image_file , 0L , SEEK_END);
        frame = (unsigned char *)malloc(frame_size);
        lseek(fd_image_file , 0L , SEEK_SET);
        read(fd_image_file, frame , frame_size);

        pthread_mutex_unlock(pmt);

        printf("got image file size (size: %d kB)\n", frame_size / 1024);

        sprintf(buffer, "Content-Type: image/jpeg\r\n" \
                "Content-Length: %d\r\n" \
                "X-Timestamp: %d.%06d\r\n" \
                "\r\n", frame_size, 0, 0);

        printf("sending intemdiate header &  frame & boundary\n");
        if(write(fd , buffer , strlen(buffer)) < 0)
        {
            printf("write buffer1 break.\n");
            break;
        }

        if(write(fd , frame , frame_size) < 0)
        {
            printf("write frame break.\n");
            break;
        }

        sprintf(buffer, "\r\n--" BOUNDARY "\r\n");
        if(write(fd, buffer , strlen(buffer)) < 0)
        {
            printf("write buffer2 break.\n");
            break;
        }
    }

    free(frame);
}

void * Thread_TCP_Web_Send_Only_JPEG_File(void *arg)
{
    while(1)
    {
        if(flag_keep_alive && flag_post_once)
        {
            flag_post_once = 0;
            HTTP_Send_Jpeg_File_Stream(fd_socket_conn , "/home/proj/1.jpeg" , &pmt , &pct);
        }
    }
}

建立TCP服务器：

TCP_Server_Found(&socket_web_server , (char*)argv[2] , PORT_TCP);

int TCP_Server_Found(socklen_t* socket_found , char* ip , int port)
{
    struct sockaddr_in servaddr;
    socklen_t addrsize = sizeof(struct sockaddr);

    bzero(&servaddr , sizeof(servaddr));
    servaddr.sin_family = AF_INET;
    servaddr.sin_addr.s_addr = inet_addr(ip);
    servaddr.sin_port = htons(port);

    int ret;
    if( (*socket_found = socket(AF_INET , SOCK_STREAM , 0)) == -1) 
        {
            printf("Create socket error: %s (errno :%d)\n",strerror(errno),errno);
            return -1;
        }

    int on = 1;
    if(setsockopt(*socket_found , SOL_SOCKET, SO_REUSEADDR, &on, sizeof(on)) < 0) 
    {
        printf("setsockopt error\n");
    }
    
    ret = bind(*socket_found , (struct sockaddr *)&servaddr , addrsize);
    if(ret == -1)
    {
            printf("Tcp bind failed!\n");
            return -1;
    }

    if(listen(*socket_found , 5) == -1)
    {
            printf("Listen failed!\n");
            return -1;
    }
    return 0;
}

这样子操作之后，树莓派400就可以在HTTP服务器上显示摄像头采集到的图像了，任何访问该HTTP服务器的TCP客户端设备（不限于手机/电脑/开发板）都可以实时查看：

解决了显示问题，就看看怎么使用tflite库去做图像识别分类（classify）：

void * Thread_Tflite(void *arg)
{
	。。。
	while (1)
	{
		pthread_mutex_lock(&pmt);
        		pthread_cond_wait(&pct , &pmt);

		Mat frame = imread("/home/proj/1.jpeg");

		pthread_mutex_unlock(&pmt);

		start = clock();
		Size frame_size = frame.size();

		Size cropSize;
		if (frame_size.width / (float)frame_size.height > WHRatio)
		{
			cropSize = Size(static_cast<int>(frame_size.height * WHRatio),
				frame_size.height);
		}
		else
		{
			cropSize = Size(frame_size.width,
				static_cast<int>(frame_size.width / WHRatio));
		}

		Rect crop(Point((frame_size.width - cropSize.width) / 2,
			(frame_size.height - cropSize.height) / 2),
			cropSize);


		Mat blob = blobFromImage(frame, 1. / 255, Size(300, 300));
		//cout << "blob size: " << blob.size << endl;

		net.setInput(blob);
		Mat output = net.forward();
		//cout << "output size: " << output.size << endl;

		Mat detectionMat(output.size[2], output.size[3], CV_32F, output.ptr<float>());

		frame = frame(crop);
		float confidenceThreshold = 0.50;
		for (int i = 0; i < detectionMat.rows; i++)
		{
			float confidence = detectionMat.at<float>(i, 2);

			if (confidence > confidenceThreshold)
			{
				。。。

				ostringstream ss;
				ss << confidence;
				String conf(ss.str());

				Rect object((int)xLeftBottom, (int)yLeftBottom,
					(int)(xRightTop - xLeftBottom),
					(int)(yRightTop - yLeftBottom));

				rectangle(frame, object, Scalar(0, 255, 0), 2);
				//cout << "objectClass:" << objectClass << endl;
				String label = String(classNames[objectClass]) + ": " + conf;
				//cout << "label"<<label << endl;
				int baseLine = 0;
				。。。
			}
		}
		finish = clock();
		totaltime = finish - start;
		cout << "识别该帧图像所用的时间为:" << totaltime <<"ms"<< endl;
		
		pthread_mutex_lock(&pmt_tflite);

		imwrite("/home/proj/2.jpeg" , frame);
		
		pthread_cond_broadcast(&pct_tflite);
        pthread_mutex_unlock(&pmt_tflite);
	}
	。。。
}

代码过于冗长，影响观感体验，于是删减掉无关语句，后面直接跟随代码工程压缩包一起发送到得捷官方主办方处。

使用相同方式创建多个跟tflite相关的线程：

pthread_create(&tid_tcp_web_recv_tflite , NULL , Thread_TCP_Web_Recv_Tflite , NULL);
pthread_create(&tid_tflite , NULL , Thread_Tflite , NULL);
pthread_create(&tid_tcp_web_send_tflite , NULL , Thread_TCP_Web_Send_Only_JPEG_File_Tflite , NULL);

查看效果：

因为树莓派400的CPU计算神经网络算力有限，因此帧数很低，平均几秒刷新一帧。

ly87802541 · 发表于2022-10-28 11:41

厉害了我的哥！

ly87802541 · 发表于2022-10-28 11:41

都是干货鸭哈哈！

donatello1996 · 发表于2022-10-28 11:50

ly87802541 发表于 2022-10-28 11:41 厉害了我的哥！

看截图效果还行，但是实际上显示效果很差，树莓派的CPU算力去做classify并没有想象中的快，刷新频率几秒才一帧，太挫了，要商用更是扯淡

【基于树莓派400的图像识别归类&运动检测&模拟信号处理系统第一帖】MJPEG [复制链接]

最新回复

点评

浏览过的版块