#AI挑战营终点站#RV1106手写数字识别，基于opencv-mobile和ncnn、rtsp部署

NNTK_NLY

#AI挑战营终点站#RV1106手写数字识别，基于opencv-mobile和ncnn、rtsp部署 [复制链接]

1.onnx模型转ncnn模型

    https://convertmodel.com/

2.下载opencv-mobile预编译包

    https://github.com/nihui/opencv-mobile/releases/download/v26/opencv-mobile-4.9.0-luckfox-pico.zip

3.为luckfox-pico编译ncnn

git clone https://github.com/Tencent/ncnn.git

cd ./ncnn./toolchains
vi luckfox-pico.cmake

填入

set(CMAKE_SYSTEM_NAME Linux)
set(CMAKE_SYSTEM_PROCESSOR arm)

if(DEFINED ENV{TOOLCHAIN_ROOT_PATH})
    file(TO_CMAKE_PATH $ENV{TOOLCHAIN_ROOT_PATH} TOOLCHAIN_ROOT_PATH)
else()
    message(FATAL_ERROR "TOOLCHAIN_ROOT_PATH env must be defined")
endif()

set(TOOLCHAIN_ROOT_PATH ${TOOLCHAIN_ROOT_PATH} CACHE STRING "root path to toolchain")

set(CMAKE_C_COMPILER "/mnt/sdd1/soc/toolchains/luckfox-pico/arm-rockchip830-linux-uclibcgnueabihf/bin/arm-rockchip830-linux-uclibcgnueabihf-gcc")
set(CMAKE_CXX_COMPILER "/mnt/sdd1/soc/toolchains/luckfox-pico/arm-rockchip830-linux-uclibcgnueabihf/bin/arm-rockchip830-linux-uclibcgnueabihf-g++")

set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER)
set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY)
set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY)

set(CMAKE_C_FLAGS "-march=armv7-a -mfloat-abi=hard -mfpu=neon")
set(CMAKE_CXX_FLAGS "-march=armv7-a -mfloat-abi=hard -mfpu=neon")

# cache flags
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS}" CACHE STRING "c flags")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}" CACHE STRING "c++ flags")

其中set(CMAKE_C_COMPILER "/mnt/sdd1/soc/toolchains/luckfox-pico/arm-rockchip830-linux-uclibcgnueabihf/bin/arm-rockchip830-linux-uclibcgnueabihf-gcc")
set(CMAKE_CXX_COMPILER "/mnt/sdd1/soc/toolchains/luckfox-pico/arm-rockchip830-linux-uclibcgnueabihf/bin/arm-rockchip830-linux-uclibcgnueabihf-g++")

按实际修改

 

回到ncnn根目录

mkdir -p build-luckfox-pico
cmake -DCMAKE_TOOLCHAIN_FILE=../toolchains/luckfox-pico.cmake ..
make -j4
make install

 

3.参考

git clone https://github.com/luckfox-eng29/luckfox_pico_rtsp_opencv

cd ./luckfox_pico_rtsp_opencv

cp ncnn/build-luckfox-pico/install/* ./ncnn_install

改CMakeLists.txt

set(CMAKE_C_COMPILER "/path/to/luckfox-pico/arm-rockchip830-linux-uclibcgnueabihf/bin/arm-rockchip830-linux-uclibcgnueabihf-gcc")
set(CMAKE_CXX_COMPILER "/path/to//luckfox-pico/arm-rockchip830-linux-uclibcgnueabihf/bin/arm-rockchip830-linux-uclibcgnueabihf-g++")

4.编写main.cpp

opencv获取摄像头帧

void *data = RK_MPI_MB_Handle2VirAddr(stVpssFrame.stVFrame.pMbBlk);

cv::Mat frame(height, width, CV_8UC3, data);

opencv框选数字

cv::Rect digit_rect = find_digit_contour(frame);
digit_rect.x = std::max(0, digit_rect.x - 10);
digit_rect.y = std::max(0, digit_rect.y - 50);
digit_rect.width = std::min(frame.cols - digit_rect.x, digit_rect.width + 20);
digit_rect.height = std::min(frame.rows - digit_rect.y, digit_rect.height + 100);

 

cv::Rect find_digit_contour(const cv::Mat &image)
{
	cv::Mat gray, blurred, edged;
	cv::cvtColor(image, gray, cv::COLOR_BGR2GRAY);
	cv::GaussianBlur(gray, blurred, cv::Size(5, 5), 0);
	cv::Canny(blurred, edged, 50, 150);

	std::vector<std::vector<cv::Point>> contours;
	cv::findContours(edged, contours, cv::RETR_EXTERNAL, cv::CHAIN_APPROX_SIMPLE);

	if (contours.empty())
	{
		return cv::Rect();
	}

	// 找到最大的轮廓
	auto largest_contour = std::max_element(contours.begin(), contours.end(),
											[](const std::vector<cv::Point> &a, const std::vector<cv::Point> &b)
											{
												return cv::contourArea(a) < cv::contourArea(b);
											});

	return cv::boundingRect(*largest_contour);
}

opencv截取数字的区域

cv::Mat digit_region = frame(digit_rect);

opencv转灰度图并resize到28*28

cv::cvtColor(digit_region, gray_1ch, cv::COLOR_BGR2GRAY);
threshold(gray_1ch, gray_1ch, atoi(argv[1]), 255, cv::THRESH_BINARY_INV);
cv::resize(gray_1ch, frame_resize, cv::Size(28, 28), 0, 0, cv::INTER_AREA);

ncnn推理

ncnn::Mat in = ncnn::Mat::from_pixels(frame_resize.data, ncnn::Mat::PIXEL_GRAY, frame_resize.cols, frame_resize.rows); // PIXEL_BGR2GRAY
ncnn::Mat out;

double total_latency = 0;
ncnn::Extractor ex = net.create_extractor();
ex.input("flatten_input", in);
ex.extract("dense_2", out);
const float *ptr = out.channel(0);
int gussed = -1;
float guss_exp = -10000000;
for (int i = 0; i < out.w * out.h; i++)
{
	printf("%d: %.2f\n", i, ptr[i]);
	if (guss_exp < ptr[i])
	{
		gussed = i;
		guss_exp = ptr[i];
	}
}
printf("I think it is number %d!\n", gussed);

在图像上显示预测结果

cv::rectangle(frame, digit_rect, cv::Scalar(0, 255, 0), 2);
sprintf(fps_text, "number:%d", gussed);
cv::putText(frame, fps_text,cv::Point(40, 40),
			cv::FONT_HERSHEY_SIMPLEX, 1,
			cv::Scalar(0, 255, 0), 2);

最后memcpy到rtsp帧中

memcpy(data, frame.data, width * height * 3);

编译

mkdir build
cd build
cmake ..
make && make install

生成可执行文件在luckfox_pico_rtsp_opencv-ncnn-mnist文件夹中

./luckfox_pico_rtsp_opencv-ncnn-mnist/luckfox_pico_rtsp_opencv-ncnn-mnist

 

5.识别效果

 

 

 

       

6.附工程开源链接

https://github.com/Ainit-NNTK/luckfox-pico-opencv-ncnn

7.总结

    opencv-mobile + ncnn推理速度还行，不过截取数字时框选有误差导致识别出错

    后续打算优化数字截取框算法和更换为rknn推理框架并对比推理效果

 

8.附实时推理视频

VID_20240528_182451

 

 

 

 

 

 

 

 

 

 

wangerxian

感觉你这个搞得很专业！