RV1106手把手教你：惊呆了！USB摄像头秒变AI助手，rknn轻松拍照做yolov5推理！

LitchiCheng · 发表于 2024-11-11 16:12

RV1106手把手教你：惊呆了！USB摄像头秒变AI助手，rknn轻松拍照做yolov5推理！ [复制链接]

邀请：@littleshrimp @w494143467 @kiverin @MianQi 参与回复

本帖最后由 LitchiCheng 于 2024-3-12 23:56 编辑

背景：官方的rknn_yolov5_demo是通过加载一张图片，然后进行推理，实际使用过程中往往需要使用摄像头capture图片后，然后再调用demo进行推理验证，很不方便。

故对官网demo进行改造，先将整个yolov5目录复制出来。偷懒的小伙伴可以直接clone我修改好的，github地址如下

LitchiCheng/RV1106_Linux: Linux test for RV1106 dev board (github.com)

https://github.com/LitchiCheng/RV1106_Linux

下面介绍几个需要特别修改的地方：

修改工程主目录下的CMakeLists.txt

指定交叉编译器

set (CMAKE_C_COMPILER "/mnt/d/luckfox-pico/tools/linux/toolchain/arm-rockchip830-linux-uclibcgnueabihf/bin/arm-rockchip830-linux-uclibcgnueabihf-gcc")
set (CMAKE_CXX_COMPILER "/mnt/d/luckfox-pico/tools/linux/toolchain/arm-rockchip830-linux-uclibcgnueabihf/bin/arm-rockchip830-linux-uclibcgnueabihf-g++")

增加预编译变量的声明

add_definitions(-DRV1106_1103)

增加我们上期分享的v4l2的工具类

add_executable(${PROJECT_NAME}
    main.cc
    postprocess.cc
    ../v4l2/tools/v4l2CapPicTool.cpp
    ${rknpu2_yolov5_file}
)

修改cmake安装路径，很多，这里只放一个，具体看整个工程

install(TARGETS ${PROJECT_NAME} DESTINATION ${OUTPUTPATH})

修改RV1106_Linux/yolov5/3rdparty下的CMakeLists.txt

直接设置架构类型为armhf_uclibc

if (CMAKE_C_COMPILER MATCHES "uclibc")
    set (TARGET_LIB_ARCH ${TARGET_LIB_ARCH}_uclibc)
endif()

修改runtime库，直接用armhf_uclibc版本

# rknn runtime
set(RKNN_PATH ${CMAKE_CURRENT_SOURCE_DIR}/rknpu2)
set(LIBRKNNRT ${RKNN_PATH}/${CMAKE_SYSTEM_NAME}/armhf-uclibc/librknnmrt.so)
set(LIBRKNNRT_INCLUDES ${RKNN_PATH}/include PARENT_SCOPE)
install(PROGRAMS ${LIBRKNNRT} DESTINATION ${OUTPUTPATH}/lib)
set(LIBRKNNRT ${LIBRKNNRT} PARENT_SCOPE)

然后修改main.cc，如下，增加对v4l2的调用

int main(int argc, char **argv)
{
    if (argc != 3)
    {
        printf("%s <model_path> <camera_path>\n", argv[0]);
        return -1;
    }

    const char *model_path = argv[1];
    const char *image_path = "tmp.jpg";

    std::string camera_path = std::string(argv[2]);
    v4l2CapPicTool vt(camera_path, 1080, 960, "jpg");
    vt.init();

    int ret;
    rknn_app_context_t rknn_app_ctx;
    memset(&rknn_app_ctx, 0, sizeof(rknn_app_context_t));

    init_post_process();

    ret = init_yolov5_model(model_path, &rknn_app_ctx);
    if (ret != 0)
    {
        printf("init_yolov5_model fail! ret=%d model_path=%s\n", ret, model_path);
        goto out;
    }
    vt.capture();
    vt.save("tmp.jpg");
    image_buffer_t src_image;
    memset(&src_image, 0, sizeof(image_buffer_t));
    ret = read_image("tmp.jpg", &src_image);

#if defined(RV1106_1103) 
    //RV1106 rga requires that input and output bufs are memory allocated by dma
    ret = dma_buf_alloc(RV1106_CMA_HEAP_PATH, src_image.size, &rknn_app_ctx.img_dma_buf.dma_buf_fd, 
                       (void **) & (rknn_app_ctx.img_dma_buf.dma_buf_virt_addr));
    memcpy(rknn_app_ctx.img_dma_buf.dma_buf_virt_addr, src_image.virt_addr, src_image.size);
    dma_sync_cpu_to_device(rknn_app_ctx.img_dma_buf.dma_buf_fd);
    free(src_image.virt_addr);
    src_image.virt_addr = (unsigned char *)rknn_app_ctx.img_dma_buf.dma_buf_virt_addr;
#endif

  									//.....

    return 0;
}

创建构建目录

mkdir build
cd build
cmake ..
make 
make install

生成的文件都放在主目录下的output

复制到开发板中，运行测试

pico@luckfox:~/output$ sudo ./rknn_yolov5_realtime model/yolov5s_relu.rknn /dev/video9
init success
load lable ./model/coco_80_labels_list.txt
model input num: 1, output num: 3
input tensors:
  index=0, name=images, n_dims=4, dims=[1, 640, 640, 3], n_elems=1228800, size=1228800, fmt=NHWC, type=INT8, qnt_type=AFFINE, zp=-128, scale=0.003922
output tensors:
  index=0, name=output, n_dims=4, dims=[1, 255, 80, 80], n_elems=1632000, size=1632000, fmt=NCHW, type=INT8, qnt_type=AFFINE, zp=-128, scale=0.003860
  index=1, name=283, n_dims=4, dims=[1, 255, 40, 40], n_elems=408000, size=408000, fmt=NCHW, type=INT8, qnt_type=AFFINE, zp=-128, scale=0.003922
  index=2, name=285, n_dims=4, dims=[1, 255, 20, 20], n_elems=102000, size=102000, fmt=NCHW, type=INT8, qnt_type=AFFINE, zp=-128, scale=0.003915
model is NHWC input fmt
model input height=640, width=640, channel=3
origin size=960x720 crop size=960x720
input image: 960 x 720, subsampling: 4:2:2, colorspace: YCbCr, orientation: 1
scale=0.666667 dst_box=(0 80 639 559) allow_slight_change=1 _left_offset=0 _top_offset=80 padding_w=0 padding_h=160
src width=960 height=720 fmt=0x1 virAddr=0x0x558b65c400 fd=0
dst width=640 height=640 fmt=0x1 virAddr=0x0x558b856810 fd=0
src_box=(0 0 959 719)
dst_box=(0 80 639 559)
color=0x72
fill dst image (x y w h)=(0 0 640 640) with color=0x72727272
rga_api version 1.10.0_[2]
rknn_run
book @ (268 189 544 457) 0.581
book @ (274 4 537 255) 0.522
book @ (238 372 555 637) 0.481
book @ (216 472 502 697) 0.477
tv @ (759 0 958 586) 0.357
suitcase @ (580 48 693 126) 0.223
write_image path: out.png width=960 height=720 channel=3 data=0x558b65c400

怀揣少年梦

对摄像头拍的实时画面进行推理吗？

tagetage

我这方面不懂，楼主，加载一张图片，然后进行推理，是什么意思？推理什么？？能详细讲一下吗？？？

LitchiCheng

怀揣少年梦发表于 2024-3-13 10:58 对摄像头拍的实时画面进行推理吗？

对，改成循环就可以了

LitchiCheng

tagetage 发表于 2024-3-13 10:58 我这方面不懂，楼主，加载一张图片，然后进行推理，是什么意思？推理什么？？能详细讲一下吗？？？

这个可以搜一下哈，DL方面的，一句两句讲不清楚

wangerxian

推理的时候用NPU了嘛。

怀揣少年梦

LitchiCheng 发表于 2024-3-13 11:19 对，改成循环就可以了

学习一下

LitchiCheng

wangerxian 发表于 2024-3-13 11:58 推理的时候用NPU了嘛。

rknn，可以看视频，里面有讲

freebsder

rknn的算力有多少？实时的话会不会卡？

cruise__tom123

先收下，感谢楼主

LitchiCheng

cruise__tom123 发表于 2024-3-13 16:59 先收下，感谢楼主

一键三连了吗，哈哈哈哈

LitchiCheng

freebsder 发表于 2024-3-13 15:45 rknn的算力有多少？实时的话会不会卡？

0.5 t，应该不会卡，推理没啥问题

li-ai-meng

lyh@lyh-pc:~/RV1106_Linux-main/yolov5/build$ cmake ..
-- The C compiler identification is unknown
-- The CXX compiler identification is unknown
CMake Error at CMakeLists.txt:6 (project):
The CMAKE_C_COMPILER:

/mnt/d/luckfox-pico/tools/linux/toolchain/arm-rockchip830-linux-uclibcgnueabihf/bin/arm-rockchip830-linux-uclibcgnueabihf-gcc

is not a full path to an existing compiler tool.

Tell CMake where to find the compiler by setting either the environment
variable "CC" or the CMake cache entry CMAKE_C_COMPILER to the full path to
the compiler, or to the compiler name if it is in the PATH.

CMake Error at CMakeLists.txt:6 (project):
The CMAKE_CXX_COMPILER:

/mnt/d/luckfox-pico/tools/linux/toolchain/arm-rockchip830-linux-uclibcgnueabihf/bin/arm-rockchip830-linux-uclibcgnueabihf-g++

is not a full path to an existing compiler tool.

Tell CMake where to find the compiler by setting either the environment
variable "CXX" or the CMake cache entry CMAKE_CXX_COMPILER to the full path
to the compiler, or to the compiler name if it is in the PATH.

-- Configuring incomplete, errors occurred!
See also "/home/lyh/RV1106_Linux-main/yolov5/build/CMakeFiles/CMakeOutput.log".
See also "/home/lyh/RV1106_Linux-main/yolov5/build/CMakeFiles/CMakeError.log".
CMake无法找到正确的C和C++编译器
请问这个怎么解决

LitchiCheng

li-ai-meng 发表于 2024-4-26 13:18 lyh@lyh-pc:~/RV1106_Linux-main/yolov5/build$ cmake .. -- The C compiler identification is unknown ...

/mnt/d/luckfox-pico/tools/linux/toolchain/arm-rockchip830-linux-uclibcgnueabihf/bin/arm-rockchip830-linux-uclibcgnueabihf-gcc

修改你对应sdk的位置，这个是我电脑的位置

fimai · 发表于2024-10-31 14:26

本帖最后由 fimai 于 2024-10-31 17:30 编辑

运行时报错，是什么问题

load lable ./model/coco_80_labels_list.txt
E RKNN: failed to decode config data!
Segmentation fault (core dumped)

LitchiCheng

fimai 发表于 2024-10-31 14:26 运行时报错，是什么问题 load lable ./model/coco_80_labels_list.txt E RKNN: failed to decode confi ...

越界之类的吧

h1mple

ubuntu@ubuntu:~/luckfox-pico/RV1106_Linux/yolov5/build$ make
[ 16%] Built target imageutils
[ 33%] Built target fileutils
[ 50%] Built target imagedrawing
make[2]: *** No rule to make target '/home/ubuntu/luckfox-pico/RV1106_Linux/sources/3rdparty/rknpu2/Linux/armhf-uclibc/librknnmrt.so', needed by 'rknn_yolov5_realtime'. Stop.
make[1]: *** [CMakeFiles/Makefile2:125: CMakeFiles/rknn_yolov5_realtime.dir/all] Error 2
make: *** [Makefile:136: all] Error 2
这是为啥呀

fimai · 发表于2024-11-11 16:12

LitchiCheng 发表于 2024-3-13 11:19 对，改成循环就可以了

改成循环的要怎么改，我改了之后，运行一会就报RK_DMA_HEAP_ALLOC_BUFFER failed:Cannot allocate memory错误，不知道哪里没释放

RV1106手把手教你：惊呆了！USB摄像头秒变AI助手，rknn轻松拍照做yolov5推理！ [复制链接]

最新回复

点评

点评

点评

点评

点评

点评

点评

点评