|
本帖最后由 walker2048 于 2022-10-29 22:12 编辑
基于AI摄像头的场景重现项目
作者：李隆
一、作品简介
名称：基于AI摄像头的场景重现项目
照片：
项目背景和实现：目前AI摄像头可以识别的内容非常多，例如人脸识别，分类识别，姿态检测等等内容。在了解到得捷创新大赛的时候，就有了用AI摄像头实现通过人体姿态识别，在esp32开发板上播放奥特曼照片和攻击特效的想法。但是作为AI新手来说， 人体姿态识别想要在esp32芯片上实现，困难比较大。所以最后调整成通过M2dock套件实现奥特曼分类识别，并将识别信息发送到esp32主控板上，显示图片并播放攻击特效的功能。
主要功能：
- 通过AI摄像头识别奥特曼类型（M2dock套件的屏幕可预览识别结果）
- 在esp32开发套件的显示屏上显示奥特曼的图片
- 在esp32开发套件播放奥特曼攻击特效（开发中）
二、系统框图
硬件说明
用到的开发套件
- 控制板：ESP32-S2-Kaluga-1 套件
- AI识别套件：sipeed的m2dock套件
硬件框图
各扩展板通过排针和主控板相连，通过串口与m2dock套件相连：
ESP32-S2-Kaluga-1 套件的功能框图如下：
M2dock的引脚图
软件说明
软件分为以下功能：
- AI识别模型
- AI识别Python脚本
- LCD显示模块
- 音乐播放模块
- 串口命令识别模块
三、作品源码
Maixhub的AI识别方案为轻代码模式，而AI分类识别部分比较麻烦的地方是模型训练，此处展示的是增加了串口输出的AI识别代码。
# generated by maixhub, tested on maixpy3 v0.4.8
from maix import nn, camera, display, image
import serial
# 初始化串口
ser = serial.Serial("/dev/ttyS1", 115200)
model = "model-17814.awnn.mud"
labels = ['yinhe', 'Victory']
def main():
camera.config(size=(224, 224))
print("-- load model:", model)
m = nn.load(model)
print("-- load ok")
while True:
img = camera.capture()
out = m.forward(img)
out = nn.F.softmax(out)
msg = "{:.2f}: {}".format(out.max(), labels[out.argmax()])
display.show(img.draw_string(2, 2, msg, scale = 1.2, color = (255, 0, 0), thickness = 2))
# 若识别率高于0.9，则向串口输出识别到的分类
if(out.max() >= 0.9)
ser.write(labels[out.argmax()])
if __name__ == "__main__":
try:
main()
except Exception as e:
import traceback, time
msg = traceback.format_exc()
print(msg)
img = image.new(size = (240, 240), color = (255, 0, 0), mode = "RGB")
img.draw_string(0, 0, msg, scale = 0.8, color = (255, 255, 255), thickness = 1)
display.show(img)
time.sleep(20)
ESP32主控部分代码：
/* Camera Example
This example code is in the Public Domain (or CC0 licensed, at your option.)
Unless required by applicable law or agreed to in writing, this
software is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
CONDITIONS OF ANY KIND, either express or implied.
*/
#include <stdio.h>
#include <stdint.h>
#include <stddef.h>
#include <string.h>
#include "esp_wifi.h"
#include "esp_system.h"
#include "nvs_flash.h"
#include "esp_event.h"
#include "freertos/FreeRTOS.h"
#include "freertos/task.h"
#include "esp_heap_caps.h"
#include "esp_log.h"
#include "esp_spiffs.h"
#include "lcd.h"
#include "esp_camera.h"
#include "board.h"
#include "usart_pars.h"
#include "audio_play.h"
static const char *TAG = "main";
#define IMAGE_MAX_SIZE (100 * 1024)/**< The maximum size of a single picture in the boot animation */
#define IMAGE_WIDTH 320 /*!< width of jpeg file */
#define IMAGE_HIGHT 240 /*!< height of jpeg file */
/**
* [url=home.php?mod=space&uid=159083]@brief[/url] rgb -> rgb565
*
* @param r red (0~31)
* @param g green (0~63)
* @param b red (0~31)
*
* [url=home.php?mod=space&uid=784970]@return[/url] data about color565
*/
uint16_t color565(uint8_t r, uint8_t g, uint8_t b)
{
uint16_t color = ((r << 11) | (g << 6) | b);
return (color << 8) | (color >> 8);
}
void esp_photo_display(const char *name)
{
ESP_LOGI(TAG, "LCD photo test....");
esp_vfs_spiffs_conf_t conf = {
.base_path = "/spiffs",
.partition_label = NULL,
.max_files = 5,
.format_if_mount_failed = false
};
/*!< Use settings defined above to initialize and mount SPIFFS filesystem. */
/*!< Note: esp_vfs_spiffs_register is an all-in-one convenience function. */
ESP_ERROR_CHECK(esp_vfs_spiffs_register(&conf));
size_t total = 0, used = 0;
ESP_ERROR_CHECK(esp_spiffs_info(NULL, &total, &used));
uint8_t *rgb565 = malloc(IMAGE_WIDTH * IMAGE_HIGHT * 2);
if (NULL == rgb565) {
ESP_LOGE(TAG, "can't alloc memory for rgb565 buffer");
return;
}
uint8_t *buf = malloc(IMAGE_MAX_SIZE);
if (NULL == buf) {
free(rgb565);
ESP_LOGE(TAG, "can't alloc memory for jpeg file buffer");
return;
}
int read_bytes = 0;
if(strcmp(name, "Victory")){
FILE *fd = fopen("/spiffs/v15.jpg", "r");
}else if (strcmp(name, "yinhe")) {
FILE *fd = fopen("/spiffs/y12.jpg", "r");
}else {
FILE *fd = fopen("/spiffs/white.jpg", "r");
}
read_bytes = fread(buf, 1, IMAGE_MAX_SIZE, fd);
ESP_LOGI(TAG, "spiffs:read_bytes:%d fd: %p", read_bytes, fd);
fclose(fd);
jpg2rgb565(buf, read_bytes, rgb565, JPG_SCALE_NONE);
lcd_set_index(0, 0, IMAGE_WIDTH - 1, IMAGE_HIGHT - 1);
lcd_write_data(rgb565, IMAGE_WIDTH * IMAGE_HIGHT * sizeof(uint16_t));
free(buf);
free(rgb565);
vTaskDelay(2000 / portTICK_RATE_MS);
}
void esp_color_display(void)
{
ESP_LOGI(TAG, "LCD color test....");
uint16_t *data_buf = (uint16_t *)heap_caps_calloc(IMAGE_WIDTH * IMAGE_HIGHT, sizeof(uint16_t), MALLOC_CAP_SPIRAM);
while (1) {
uint16_t color = color565(0, 0, 0);
for (int r = 0, j = 0; j < IMAGE_HIGHT; j++) {
if (j % 8 == 0) {
color = color565(r++, 0, 0);
}
for (int i = 0; i < IMAGE_WIDTH; i++) {
data_buf[i + IMAGE_WIDTH * j] = color;
}
}
lcd_set_index(0, 0, IMAGE_WIDTH - 1, IMAGE_HIGHT - 1);
lcd_write_data((uint8_t *)data_buf, IMAGE_WIDTH * IMAGE_HIGHT * sizeof(uint16_t));
vTaskDelay(2000 / portTICK_RATE_MS);
for (int g = 0, j = 0; j < IMAGE_HIGHT; j++) {
if (j % 8 == 0) {
color = color565(0, g++, 0);
}
for (int i = 0; i < IMAGE_WIDTH; i++) {
data_buf[i + IMAGE_WIDTH * j] = color;
}
}
lcd_set_index(0, 0, IMAGE_WIDTH - 1, IMAGE_HIGHT - 1);
lcd_write_data((uint8_t *)data_buf, IMAGE_WIDTH * IMAGE_HIGHT * sizeof(uint16_t));
vTaskDelay(2000 / portTICK_RATE_MS);
for (int b = 0, j = 0; j < IMAGE_HIGHT; j++) {
if (j % 8 == 0) {
color = color565(0, 0, b++);
}
for (int i = 0; i < IMAGE_WIDTH; i++) {
data_buf[i + IMAGE_WIDTH * j] = color;
}
}
lcd_set_index(0, 0, IMAGE_WIDTH - 1, IMAGE_HIGHT - 1);
lcd_write_data((uint8_t *)data_buf, IMAGE_WIDTH * IMAGE_HIGHT * sizeof(uint16_t));
vTaskDelay(2000 / portTICK_RATE_MS);
}
}
void app_main()
{
lcd_config_t lcd_config = {
#ifdef CONFIG_LCD_ST7789
.clk_fre = 80 * 1000 * 1000, /*!< ILI9341 Stable frequency configuration */
#endif
#ifdef CONFIG_LCD_ILI9341
.clk_fre = 40 * 1000 * 1000, /*!< ILI9341 Stable frequency configuration */
#endif
.pin_clk = LCD_CLK,
.pin_mosi = LCD_MOSI,
.pin_dc = LCD_DC,
.pin_cs = LCD_CS,
.pin_rst = LCD_RST,
.pin_bk = LCD_BK,
.max_buffer_size = 2 * 1024,
.horizontal = 2, /*!< 2: UP, 3: DOWN */
.swap_data = 1,
};
lcd_init(&lcd_config);
char *pars_name = (char *)malloc(64);
while (1) {
uart_pars_name(pars_name);
if (strlen(pars_name) >= 0){
/*< Show a picture */
esp_photo_display(pars_name);
audio_play(pars_name);
}
}
}
四、作品功能演示视频
视频链接：https://training.eeworld.com.cn/video/34706
五、项目总结
经过本次项目的实践，我学习到了AI识别的一些入门知识，也加深了esp32开发的相关知识。同时也深刻意识到，真正要实现准确的奥特曼识别需要花费非常多的精力。简单的AI模型训练分类的正确识别率还是有限的。特别是在识别奥特曼视频这种特殊场景，以及奥特曼特征码较弱的情况下。
活动发帖汇总
1. 【基于esp32的姿态识别和场景重现】物料开箱—ESP32-S2-KALUGA-1以及k210
2. 【基于AI摄像头的场景重现项目】基于全志v831的AI识别方案
3. 【基于AI摄像头的场景重现项目】maixhub在线模型训练--奥特曼识别训练过程
4. 【基于AI摄像头的场景重现项目】ESP-IDE 环境 -- 命令行安装过程详细解析
2022_得捷创新设计大赛_基于AI摄像头的场景重现.docx
(534.03 KB, 下载次数: 2)
|
提升卡
变色卡
千斤顶
。
1/10 
京公网安备 11010802033920号
Copyright © 2005-2024 EEWORLD.com.cn, Inc. All rights reserved
