opencv rtsp 硬件解码

讨论使用opencv的reader

硬件解码的方案有太多种，如果使用ffmpeg硬件解码是最方便的，不方便的是把解码过后的GPU 拉到 CPU 上，再使用opencv的Mat 从cpu 上上载到gpu上，是不是多了两个过程，应该是直接从GPU mat 直接去处理，最后一步再从GPU mat 上下载到cpu，render显示。

GPU 硬件解码是nv12 格式，我们为了显示和cpu使用直接转成了RGB或者BGR，使用opencv再映射封装，最后又上载到cuda，这个过程很耗时间，而且不是必要的。

windows下使用cuda

经过实验，cv::cudacodec::createVideoReader 是可以拉取rtsp 流的，官方编译的可以读取rtsp，但是在文件流上出了问题，而且还有一个bug，就是在显示的时候，必须关闭一次窗口，才能显示后续的帧，而且还有一点，就是注意这个窗口必须是opengl 窗口，而且要打开这个窗口，而且在编译支持cuda的opencv时必须把opengl 勾选上，所以达不到产品化的要求，以下是测试代码：

#include <iostream>#include "opencv2/opencv_modules.hpp"#if defined(HAVE_OPENCV_CUDACODEC)#include <string>
#include <vector>
#include <algorithm>
#include <numeric>
#include <opencv2/opencv.hpp>
#include <opencv2/core.hpp>
#include <opencv2/core/opengl.hpp>
#include <opencv2/cudacodec.hpp>
#include <opencv2/highgui.hpp>#if _DEBUG
#pragma comment(lib,"opencv_world460.lib")
#else 
#pragma comment(lib,"opencv_world460.lib")
#endif
int main()
{cv::cuda::printCudaDeviceInfo(cv::cuda::getDevice());int count = cv::cuda::getCudaEnabledDeviceCount();printf("GPU Device Count : %d \n", count);const std::string fname("rtsp://127.0.0.1/101-640.mkv"); //视频文件// const std::string fname("test_222.mp4"); //视频文件// cv::namedWindow("CPU", cv::WINDOW_NORMAL);cv::namedWindow("GPU", cv::WINDOW_OPENGL);cv::cuda::setGlDevice();cv::Mat frame;cv::VideoCapture reader(fname);cv::cuda::GpuMat d_frame;cv::Ptr<cv::cudacodec::VideoReader> d_reader = cv::cudacodec::createVideoReader(fname);cv::TickMeter tm;std::vector<double> cpu_times;std::vector<double> gpu_times;int gpu_frame_count = 0, cpu_frame_count = 0;
#if 0for (;;){tm.reset(); tm.start();if (!reader.read(frame))break;tm.stop();cpu_times.push_back(tm.getTimeMilli());cpu_frame_count++;cv::imshow("CPU", frame);if (cv::waitKey(1) > 0)break;}
#endiffor (;;){tm.reset();tm.start();if (!d_reader->nextFrame(d_frame))break;tm.stop();//d_frame.step = d_frame.cols * d_frame.channels();//cv::cuda::GpuMat gpuMat_Temp = d_frame.clone();gpu_times.push_back(tm.getTimeMilli());gpu_frame_count++;if (gpu_frame_count > 2){cv::Mat test;d_frame.download(test);d_frame.release();// cv::cvtColor(test, test, cv::COLOR_BGRA2BGR);//cv::imwrite("./test1.jpg", test);cv::imshow("GPU", test);}if (cv::waitKey(1) > 0)break;}if (!cpu_times.empty() && !gpu_times.empty()){std::cout << std::endl << "Results:" << std::endl;std::sort(cpu_times.begin(), cpu_times.end());std::sort(gpu_times.begin(), gpu_times.end());double cpu_avg = std::accumulate(cpu_times.begin(), cpu_times.end(), 0.0) / cpu_times.size();double gpu_avg = std::accumulate(gpu_times.begin(), gpu_times.end(), 0.0) / gpu_times.size();std::cout << "CPU : Avg : " << cpu_avg << " ms FPS : " << 1000.0 / cpu_avg << " Frames " << cpu_frame_count << std::endl;std::cout << "GPU : Avg : " << gpu_avg << " ms FPS : " << 1000.0 / gpu_avg << " Frames " << gpu_frame_count << std::endl;}return 0;
}

经过release版本的测试，cuda硬件解码比cpu慢很多，我cpu是intel 13代 13700，速度很快，gpu是3060ti，实际测试就是如此。
说明在windows下实际类里面解码的时候在cpu和gpu上转换的时间太多
在这里插入图片描述
综上所述，必须使用更为简单的方法，放弃windows上的做法，放到linux上， ffmpeg硬件解码然后映射到gpu mat上，至于解码ffmpeg 可以看我的其他文章，至于ffmpeg 编解码 nvidia 上官网也是有介绍的：
编译ffmpeg
使用python和linux，使用python的作用是取消c++ 到python之间的内存共享，在windows上编译pynvcodec 会遇到各种问题，建议在linux 编译 pynvcodec，为什么不使用ffmpeg直接解码，因为：我们使用ffmpeg解码得到的YUV格式，我们只能在CPU下转化到RGB的色彩空间，缺少在GPU上进行全部转化的流程，因此我们使用vpf 来进行python上的视频处理，同时结束时可以直接转化成pytorch的张量来处理。

VideoProcessingFramework（VPF）是NVIDIA开源的适用于Python的视频处理框架，可用于硬件加速条件下的视频编解码等处理类任务。同时对于Pytorch比较友好，能够将解析出来的图像数据直接转化成Tensor()的格式。以下为例子：

import PyNvCodec as nvc
import PytorchNvCodec as pnvc  while True:# Read data.# Amount doesn't really matter, will be updated later on during decode.bits = proc.stdout.read(read_size)if not len(bits):print("Can't read data from pipe")breakelse:rt += len(bits)# Decodeenc_packet = np.frombuffer(buffer=bits, dtype=np.uint8)pkt_data = nvc.PacketData()try:surf = nvdec.DecodeSurfaceFromPacket(enc_packet, pkt_data)    # 获取流的数据# Convert to planar RGBrgb_pln = to_rgb.run(surf)   # 转换到rgb_plnif rgb_pln.Empty():break# PROCESS YOUR TENSOR HERE.# THIS DUMMY PROCESSING JUST ADDS RANDOM ROTATION.src_tensor = surface_to_tensor(rgb_pln)  # 转化为Tensor(),数据存储在GPU中dst_tensor = T.RandomRotation(degrees=(-1, 1))(src_tensor)surface_rgb = tensor_to_surface(dst_tensor, gpu_id)# Convert back to NV12dst_surface = to_nv12.run(surface_rgb) # 再转换回码流if src_surface.Empty():break# Handle HW exceptions in simplest possible way by decoder respawnexcept nvc.HwResetException:nvdec = nvc.PyNvDecoder(w, h, f, c, g)continue

使用gstreamer

近来来opencv的下载是一个问题，动不动就下载出错，使用gstreamer 在windows下和ffmpeg 差不离，编译也比较麻烦，我们尽量在linux下编译

sudo apt-get update 
sudo apt-get install build-essential cmake git pkg-config 
sudo apt-get install libjpeg8-dev libtiff4-dev libjasper-dev libpng12-dev 
sudo apt-get install libgtk2.0-dev 
sudo apt-get install libavcodec-dev libavformat-dev libswscale-dev libv4l-dev 
sudo apt-get install libatlas-base-dev gfortran 
//在opencv里面安装gstreamer插件 
sudo apt-get install gstreamer1.0-tools gstreamer1.0-alsa gstreamer1.0-plugins-base gstreamer1.0-plugins-good gstreamer1.0-plugins-bad gstreamer1.0-plugins-ugly gstreamer1.0-libav 
sudo apt-get install libgstreamer1.0-dev libgstreamer-plugins-base1.0-dev libgstreamer-plugins-good1.0-dev libgstreamer-plugins-bad1.0-dev cd /home/opencv 
git clone https://github.com/opencv.git 
cd opencv 
git checkout 4.7.0 
cd /home/opcv 
nkdir build 
cmake -D CMAKE_BUILD_TYPE=RELEASE -D CMAKE_INSTALL_PREFIX=/usr/local -D CUDA_GENERATION=Kepler .. 
make -j4 
sudo make install

int main()
{// std::cout << cv::getBuildInformation() << std::endl;using std::chrono::steady_clock;typedef std::chrono::milliseconds milliseconds_type;const int interval = 15;std::stringstream ss;std::string rtsp_url = "rtsp://127.0.0.1/101-640.mkv";size_t latency = 200;size_t frame_width = 1920;size_t frame_height = 1080;size_t framerate = 15;ss << "rtspsrc location=" << rtsp_url << " latency=" << latency << " ! application/x-rtp, media=video, encoding-name=H264 "<< "! rtph264depay ! video/x-h264, clock-rate=90000, width=" << frame_width << ", height=" << frame_height << ", framerate="<< framerate << "/1 ! nvv4l2decoder ! video/x-raw(memory:NVMM), width=" << frame_width << ", height=" << frame_height<< ", framerate=" << framerate << "/1 ! nvvideoconvert ! video/x-raw, format=BGRx ! videoconvert ! video/x-raw, format=BGR ! appsink";std::cout << ss.str() << std::endl;cv::VideoCapture cap = cv::VideoCapture(ss.str(), cv::CAP_GSTREAMER);if (!cap.isOpened()){std::cerr << "error to open camera." << std::endl;return -1;}std::cout << cv::getBuildInformation() << std::endl;cv::Mat frame;steady_clock::time_point start = steady_clock::now();size_t frame_idx = 0;while (1){bool ret = cap.read(frame);if (ret){// cv::imwrite("tmp.jpg", frame);++frame_idx;}if (frame_idx % interval == 0){steady_clock::time_point end = steady_clock::now();milliseconds_type span = std::chrono::duration_cast<milliseconds_type>(end - start);std::cout << "it took " << span.count() / frame_idx << " millisencods." << std::endl;start = end;}}return 0;
}