video_topic

使用qt5,ffmpeg6.0,opencv，os2来实现。qt并非必要，只是用惯了。

步骤是：

1.读取rtsp码流，转换成mat图像

2.发送ros::mat图像

项目结构如下：

videoplayer.h

#ifndef VIDEOPLAYER_H
#define VIDEOPLAYER_H#include <QThread>
#include <QImage>class VlcInstance;
class VlcMedia;
class VlcMediaPlayer;class VideoPlayer : public QThread
{Q_OBJECTpublic:explicit VideoPlayer();~VideoPlayer();void startPlay();void stopPlay();signals:void sig_GetOneFrame(QImage); //每获取到一帧图像 就发送此信号void sig_GetRFrame(QImage);   signals://void SigFinished(void);protected:void run();private:QString mFileName;VlcInstance *_instance;VlcMedia *_media;VlcMediaPlayer *_player;std::string rtspaddr;bool mStopFlag;//是否退出标志
public slots:void setrtsp(std::string addr);
};#endif // VIDEOPLAYER_H

videoplayer.cpp

#include "videoplayer.h"
extern "C"
{#include "libavcodec/avcodec.h"#include "libavformat/avformat.h"#include "libavutil/pixfmt.h"#include "libswscale/swscale.h"#include "libavutil/imgutils.h"}#include <stdio.h>
#include<iostream>
using namespace std;
VideoPlayer::VideoPlayer()
{rtspaddr="rtsp://admin:123456@192.168.123.104:554/stream1";
}VideoPlayer::~VideoPlayer()
{}void VideoPlayer::setrtsp(std::string addr){rtspaddr=addr;
}void VideoPlayer::startPlay()
{///调用 QThread 的start函数 将会自动执行下面的run函数 run函数是一个新的线程this->start();
}void VideoPlayer::stopPlay(){mStopFlag= true;
}void VideoPlayer::run()
{AVFormatContext *pFormatCtx;AVCodecContext *pCodecCtx;const AVCodec *pCodec;AVFrame *pFrame, *pFrameRGB;AVPacket *packet;uint8_t *out_buffer;static struct SwsContext *img_convert_ctx;int videoStream, i, numBytes;int ret, got_picture;avformat_network_init();//Allocate an AVFormatContext.pFormatCtx = avformat_alloc_context();AVDictionary *avdic=NULL;/*char option_key[]="rtsp_transport";char option_value[]="tcp";av_dict_set(&avdic,option_key,option_value,0);char option_key2[]="max_delay";char option_value2[]="100";av_dict_set(&avdic,option_key2,option_value2,0);*/av_dict_set(&avdic, "buffer_size", "1024000", 0); //设置最大缓存，1080可调到最大av_dict_set(&avdic, "rtsp_transport", "udp", 0); //以tcp的方式传送av_dict_set(&avdic, "stimeout", "5000000", 0); //设置超时断开链接时间，单位usav_dict_set(&avdic, "max_delay", "500000", 0); //设置最大时延av_dict_set(&avdic, "framerate", "5", 0);//av_dict_set(&avdic, "video_size","640x40",0);/*AVDictionary* options = NULL;av_dict_set(&options, "buffer_size", "1024000", 0); //设置最大缓存，1080可调到最大av_dict_set(&options, "rtsp_transport", "udp", 0); //以tcp的方式传送av_dict_set(&options, "stimeout", "5000000", 0); //设置超时断开链接时间，单位usav_dict_set(&options, "max_delay", "500000", 0); //设置最大时延av_dict_set(&options, "framerate", "20", 0);*////rtsp地址，可根据实际情况修改/// rtsp://127.0.0.1:8554/stream/// rtsp://admin:123456@192.168.123.104:554/stream1//char * tmp=(char*)rtspaddr.data();//char url[50];//strcpy(url, tmp);//char url[] ="rtsp://admin:123456@192.168.123.104:554/stream1";char url[100];for(int i=0;i<rtspaddr.length();i++){url[i] = rtspaddr[i];}url[rtspaddr.length()]='\0';if (avformat_open_input(&pFormatCtx, url, NULL, &avdic) != 0) {printf("can't open the file. \n");return;}if (avformat_find_stream_info(pFormatCtx, NULL) < 0) {printf("Could't find stream infomation.\n");return;}videoStream = -1;///循环查找视频中包含的流信息，直到找到视频类型的流///便将其记录下来 保存到videoStream变量中///这里我们现在只处理视频流  音频流先不管他for (i = 0; i < pFormatCtx->nb_streams; i++) {if (pFormatCtx->streams[i]->codecpar->codec_type == AVMEDIA_TYPE_VIDEO) {videoStream = i;break;}}///如果videoStream为-1 说明没有找到视频流if (videoStream == -1) {printf("Didn't find a video stream.\n");return;}printf("nb_stream:%d videoStream:%d\n",pFormatCtx->nb_streams,videoStream);pCodec = avcodec_find_decoder(pFormatCtx->streams[videoStream]->codecpar->codec_id);pCodecCtx = avcodec_alloc_context3(pCodec);avcodec_parameters_to_context(pCodecCtx, pFormatCtx->streams[videoStream]->codecpar);//printf("pCodecCtx->frame_number:%d\n", pCodecCtx->frame_number);//printf("pCodecCtx->time_base.num:%d\n", pCodecCtx->time_base.num);//printf("pCodecCtx->time_base.den:%d\n", pCodecCtx->time_base.den);//printf("pCodecCtx->bit_rate:%d\n", pCodecCtx->bit_rate);//printf("pCodecCtx->framerate:%d\n", pCodecCtx->framerate);// pCodecCtx->bit_rate =0;   //初始化为0// pCodecCtx->time_base.num=1;  //下面两行：一秒钟25帧// pCodecCtx->time_base.den=10;// pCodecCtx->frame_number=1;  //每包一个视频帧if (pCodec == NULL) {printf("Codec not found.\n");return;}///打开解码器if (avcodec_open2(pCodecCtx, pCodec, NULL) < 0) {printf("Could not open codec.\n");return;}pFrame = av_frame_alloc();pFrameRGB = av_frame_alloc();///这里我们改成了 将解码后的YUV数据转换成RGB32img_convert_ctx = sws_getContext(pCodecCtx->width, pCodecCtx->height,pCodecCtx->pix_fmt, pCodecCtx->width, pCodecCtx->height,AV_PIX_FMT_RGBA, SWS_BICUBIC, NULL, NULL, NULL);numBytes = av_image_get_buffer_size(AV_PIX_FMT_RGBA, pCodecCtx->width,pCodecCtx->height,1);out_buffer = (uint8_t *) av_malloc(numBytes * sizeof(uint8_t));av_image_fill_arrays(pFrameRGB->data,pFrameRGB->linesize,out_buffer,AV_PIX_FMT_RGBA,pCodecCtx->width,pCodecCtx->height,1);int y_size = pCodecCtx->width * pCodecCtx->height;packet = (AVPacket *) malloc(sizeof(AVPacket)); //分配一个packetav_new_packet(packet, y_size); //分配packet的数据mStopFlag = false;while (!mStopFlag){if (av_read_frame(pFormatCtx, packet) < 0){continue; //这里认为视频读取完了}if (packet->stream_index == videoStream) {ret = avcodec_send_packet(pCodecCtx,packet);if( 0 != ret){continue;}while (avcodec_receive_frame(pCodecCtx,pFrame) == 0){sws_scale(img_convert_ctx,(uint8_t const * const *) pFrame->data,pFrame->linesize, 0, pCodecCtx->height, pFrameRGB->data,pFrameRGB->linesize);//把这个RGB数据 用QImage加载QImage tmpImg((uchar *)out_buffer,pCodecCtx->width,pCodecCtx->height,QImage::Format_RGBA8888);//QImage tmpImg((uchar *)out_buffer,pCodecCtx->width,pCodecCtx->height,QImage::Format_RGB888);QImage image = tmpImg.copy(); //把图像复制一份 传递给界面显示emit sig_GetOneFrame(image);  //发送信号/*printf("pCodecCtx->width:%d\n", pCodecCtx->width);printf("pCodecCtx->height:%d\n", pCodecCtx->height);printf("pCodecCtx->frame_number:%d\n", pCodecCtx->frame_number);printf("pCodecCtx->time_base.num:%d\n", pCodecCtx->time_base.num);printf("pCodecCtx->time_base.den:%d\n", pCodecCtx->time_base.den);printf("pCodecCtx->bit_rate:%d\n", pCodecCtx->bit_rate);printf("pCodecCtx->framerate:%d\n", pCodecCtx->framerate);printf("pCodecCtx->frame_size:%d\n", pCodecCtx->frame_size);*/}}av_packet_unref(packet); //释放资源,否则内存会一直上升msleep(0.02); //停一停  不然放的太快了}av_free(out_buffer);av_free(pFrameRGB);avcodec_close(pCodecCtx);avformat_close_input(&pFormatCtx);//emit SigFinished();
}

widget.h

#ifndef WIDGET_H
#define WIDGET_H#include <QWidget>
#include <QLineEdit>
#include <QDir>
#include <QSettings>
#include <QDebug>
#include <QPushButton>
#include <QPainter>
#include <QInputDialog>
#include <QtMath>
#include <iostream>#include "videoplayer.h"
#include <iostream>
#include <csignal>
#include <opencv4/opencv2/opencv.hpp>#include <iostream>
#include <iomanip>
#include <ctime>
#include <opencv2/opencv.hpp>#include<algorithm>
#include<vector>
#include<iostream>#include "rclcpp/rclcpp.hpp"
#include "std_msgs/msg/string.hpp"#include <QTimer>#include <cv_bridge/cv_bridge.h>
#include <sensor_msgs/msg/image.hpp>
#include <std_msgs/msg/string.hpp>
using namespace std::chrono_literals;QT_BEGIN_NAMESPACE
namespace Ui { class Widget; }
QT_END_NAMESPACEclass Widget : public QWidget
{Q_OBJECTpublic:Widget(QWidget *parent = nullptr);~Widget();
signals:void sig_fame(QImage img);private:Ui::Widget *ui;
private:void readconfig();QString rtspaddr;void initWidget();void initconnect();
private slots:void slot_open_or_close();protected://void paintEvent(QPaintEvent *event);private:VideoPlayer *mPlayer; //播放线程QImage mImage; //记录当前的图像QString url;//QImage initimage;cv::Mat QImage2Mat(QImage image);QImage Mat2QImage(const cv::Mat &mat);//void readvideo();//std::vector<cv::Vec3b> colors(32);//cv::VideoWriter outputVideo;//int encode_type ;//= VideoWriter::fourcc('M', 'J', 'P', 'G');//std::vector<cv::Vec3b> colors;//cv::VideoWriter outputVideo;private slots:void slotGetOneFrame(QImage img);private:cv::Mat tempmat;// -------------------------------------// ros// -------------------------------------// noderclcpp::Node::SharedPtr node_;// pubrclcpp::Publisher<sensor_msgs::msg::CompressedImage>::SharedPtr publisher_;// sub//rclcpp::Subscription<std_msgs::msg::String>::SharedPtr subscriber_;// spinrclcpp::TimerBase::SharedPtr timer_;void initSpin(void);QTimer spin_timer_;void timer_callback();};
#endif // WIDGET_H

widget.cpp

#include "widget.h"
#include "ui_widget.h"Widget::Widget(QWidget *parent): QWidget(parent), ui(new Ui::Widget)
{mPlayer = new VideoPlayer;ui->setupUi(this);readconfig();initWidget();initconnect();// -------------------------------------/*// create topic pubthis->publisher_ = this->node_->create_publisher<std_msgs::msg::String>("pub_topic", 10);// create topic subthis->subscriber_ = node_->create_subscription<std_msgs::msg::String>("sub_topic", 10,[&](const std_msgs::msg::String::SharedPtr msg){// 處理訂閱到的消息QString receivedMsg = QString::fromStdString(msg->data);std::cout << msg->data << std::endl;//ui->textBrowser->append(receivedMsg);});this->initSpin();*/rclcpp::init(0, nullptr);// create nodethis->node_ = rclcpp::Node::make_shared("video");this->publisher_ = this->node_->create_publisher<sensor_msgs::msg::CompressedImage>("pubImageTopic", 10);this->timer_ = this->node_->create_wall_timer(33ms, std::bind(&Widget::timer_callback, this));this->initSpin();}Widget::~Widget()
{delete ui;// -------------------------------------// ROS 釋放// -------------------------------------this->spin_timer_.stop();rclcpp::shutdown();// -------------------------------------}void Widget::initSpin(void)
{this->spin_timer_.setInterval(1); // 1 msQObject::connect(&this->spin_timer_, &QTimer::timeout, [&](){ rclcpp::spin_some(node_); });this->spin_timer_.start();
}void Widget::timer_callback(){try{//ros_img_ = cv_bridge::CvImage(std_msgs::msg::Header(), "bgr8", tempmat).toImageMsg();//sensor_msgs::msg::Image::SharedPtr ros_img_ = cv_bridge::CvImage(std_msgs::msg::Header(), "bgr8", tempmat).toImageMsg();if(!tempmat.empty()){//rclcpp::Publisher<sensor_msgs::msg::CompressedImage>::SharedPtr video_compressed_publisher_;cv::Mat des1080;cv::resize(tempmat, des1080, cv::Size(1080, 720), 0, 0, cv::INTER_NEAREST);sensor_msgs::msg::CompressedImage::SharedPtr ros_img_compressed_ = cv_bridge::CvImage(std_msgs::msg::Header(), "bgr8", des1080).toCompressedImageMsg();//video_compressed_publisher_->publish(*ros_img_compressed_);this->publisher_ ->publish(*ros_img_compressed_);qDebug()<<"publisher";}else{qDebug()<<"empty image";}//RCLCPP_WARN(this->get_logger(), "empty image");// video_publisher_->publish(*ros_img_);}catch (cv_bridge::Exception &e){//RCLCPP_ERROR(this->get_logger(),ros_img_->encoding.c_str());qDebug()<<"Exception";}}void Widget::readconfig(){QSettings settingsread("./src/video_topic/conf/config.ini",QSettings::IniFormat);rtspaddr = settingsread.value("SetUpOption/camerartsp").toString();mPlayer->setrtsp(rtspaddr.toStdString());
}void Widget::initWidget(){qDebug()<<rtspaddr;ui->le_rtstspaddr->setText(rtspaddr);
}void Widget::slot_open_or_close(){if(ui->btn_openorclose->text()=="open"){ui->btn_openorclose->setText("close");mPlayer->startPlay();// ROS 初始化// -------------------------------------/*rclcpp::init(0, nullptr);// create nodethis->node_ = rclcpp::Node::make_shared("video");this->publisher_ = this->node_->create_publisher<sensor_msgs::msg::CompressedImage>("pubImageTopic", 10);this->timer_ = this->node_->create_wall_timer(500ms, std::bind(&Widget::timer_callback, this));//pub_img = this->create_publisher<sensor_msgs::msg::Image>("res_img", 10);rclcpp::spin(this->node_);*///readvideo();}else{ui->btn_openorclose->setText("open");mPlayer->stopPlay();}
}void Widget::initconnect(){connect(ui->btn_openorclose,&QPushButton::clicked,this,&Widget::slot_open_or_close);connect(mPlayer,SIGNAL(sig_GetOneFrame(QImage)),this,SLOT(slotGetOneFrame(QImage)));connect(this,&Widget::sig_fame,this,&Widget::slotGetOneFrame);//connect(mPlayer,&VideoPlayer::SigFinished, mPlayer,&VideoPlayer::deleteLater);//自动释放
}void Widget::slotGetOneFrame(QImage img)
{//cv::Mat tempmat = QImage2Mat(img);tempmat = QImage2Mat(img);if (tempmat.empty()) {printf("null img\n");}else {QImage outimg = Mat2QImage(tempmat);//printf("get img\n");mImage = outimg;QImage imageScale = mImage.scaled(QSize(ui->label->width(), ui->label->height()));QPixmap pixmap = QPixmap::fromImage(imageScale);ui->label->setPixmap(pixmap);}}cv::Mat Widget::QImage2Mat(QImage image)
{cv::Mat mat = cv::Mat::zeros(image.height(), image.width(),image.format()); //初始化Matswitch(image.format()) //判断image的类型{case QImage::QImage::Format_Grayscale8:  //灰度图mat = cv::Mat(image.height(), image.width(),CV_8UC1,(void*)image.constBits(),image.bytesPerLine());break;case QImage::Format_RGB888: //3通道彩色mat = cv::Mat(image.height(), image.width(),CV_8UC3,(void*)image.constBits(),image.bytesPerLine());break;case QImage::Format_ARGB32: //4通道彩色mat = cv::Mat(image.height(), image.width(),CV_8UC4,(void*)image.constBits(),image.bytesPerLine());break;case QImage::Format_RGBA8888:mat = cv::Mat(image.height(), image.width(),CV_8UC4,(void*)image.constBits(),image.bytesPerLine());break;default:return mat;}cv::cvtColor(mat, mat, cv::COLOR_BGR2RGB);return mat;}QImage Widget::Mat2QImage(const cv::Mat &mat)
{if(mat.type()==CV_8UC1 || mat.type()==CV_8U){QImage image((const uchar *)mat.data, mat.cols, mat.rows, mat.step, QImage::Format_Grayscale8);return image;}else if(mat.type()==CV_8UC3){QImage image((const uchar *)mat.data, mat.cols, mat.rows, mat.step, QImage::Format_RGB888);return image.rgbSwapped();  //r与b调换}
}

widget.ui

<?xml version="1.0" encoding="UTF-8"?>
<ui version="4.0"><class>Widget</class><widget class="QWidget" name="Widget"><property name="geometry"><rect><x>0</x><y>0</y><width>645</width><height>461</height></rect></property><property name="windowTitle"><string>Widget</string></property><layout class="QVBoxLayout" name="verticalLayout"><item><widget class="QStackedWidget" name="stackedWidget"><widget class="QWidget" name="page"><layout class="QVBoxLayout" name="verticalLayout_2"><item><widget class="QLabel" name="label"><property name="text"><string>TextLabel</string></property></widget></item><item><layout class="QHBoxLayout" name="horizontalLayout"><item><widget class="QLineEdit" name="le_rtstspaddr"><property name="readOnly"><bool>true</bool></property></widget></item><item><widget class="QPushButton" name="btn_openorclose"><property name="text"><string>open</string></property></widget></item></layout></item></layout></widget><widget class="QWidget" name="page_2"/></widget></item></layout></widget><resources/><connections/>
</ui>

main.cpp

#include "widget.h"#include <QApplication>int main(int argc, char *argv[])
{QApplication a(argc, argv);Widget w;w.show();//w.showMaximized();return a.exec();
}

package.xml

<?xml version="1.0"?>
<?xml-model href="http://download.ros.org/schema/package_format3.xsd" schematypens="http://www.w3.org/2001/XMLSchema"?>
<package format="3"><name>video_topic</name><version>0.0.0</version><description>TODO: Package description</description><maintainer email="cl@todo.todo">cl</maintainer><license>TODO: License declaration</license><buildtool_depend>ament_cmake</buildtool_depend><buildtool_depend>cv_bridge</buildtool_depend><depend>rclcpp</depend><depend>std_msgs</depend><depend>cv_bridge</depend><test_depend>ament_lint_auto</test_depend><test_depend>ament_lint_common</test_depend><export><build_type>ament_cmake</build_type></export>
</package>

CMakeLists.txt

cmake_minimum_required(VERSION 3.5)
project(video_topic)# Default to C++14
if(NOT CMAKE_CXX_STANDARD)set(CMAKE_CXX_STANDARD 14)
endif()if(CMAKE_COMPILER_IS_GNUCXX OR CMAKE_CXX_COMPILER_ID MATCHES "Clang")add_compile_options(-Wall -Wextra -Wpedantic)
endif()# find dependencies
find_package(ament_cmake REQUIRED)
find_package(rclcpp REQUIRED)
find_package(std_msgs REQUIRED)
# qt
find_package(Qt5 REQUIRED COMPONENTS Core Gui Widgets)
find_package(cv_bridge REQUIRED)
find_package(image_transport)
find_package(sensor_msgs REQUIRED)add_executable(videosrc/main.cppsrc/videoplayer.hsrc/videoplayer.cppsrc/widget.hsrc/widget.cppsrc/widget.ui
)#set(FFMPEG_LIBS_DIR /usr/local/ffmpeg/lib)
#set(FFMPEG_HEADERS_DIR /usr/local/ffmpeg/include)set(FFMPEG_LIBS_DIR /usr/lib/aarch64-linux-gnu)
set(FFMPEG_HEADERS_DIR /usr/include/aarch64-linux-gnu)
include_directories(${FFMPEG_HEADERS_DIR})
#link_directories(${FFMPEG_LIBS_DIR})
#set(FFMPEG_LIBS libavutil.so libavcodec.so libavdevice.so libavformat.so libavfilter.so libswresample.so libswscale.so  libavutil.so)
set(FFMPEG_LIBS libavcodec.so libavformat.so libswscale.so libavutil.so)find_package(OpenCV REQUIRED)
include_directories( ${OpenCV_INCLUDE_DIRS})
target_link_libraries(video ${OpenCV_LIBS})# ros
target_link_libraries(video${rclcpp_LIBRARIES} 
)
# qt
target_link_libraries(video Qt5::Core Qt5::GuiQt5::Widgets
)
#ffmpegtarget_link_libraries(video ${FFMPEG_LIBS})ament_target_dependencies(videorclcppstd_msgssensor_msgs cv_bridge OpenCV image_transport
)# ------------------------------------------
# 設置自動MOC、UIC和RCC (與QT相關)
# ------------------------------------------
set_target_properties(video PROPERTIES AUTOMOC ON)
set_target_properties(video PROPERTIES AUTOUIC ON)
set_target_properties(video PROPERTIES AUTORCC ON)
# ------------------------------------------# 安装可执行文件
install(TARGETS videoDESTINATION lib/${PROJECT_NAME}
)ament_package()