rtsp视频在使用unity三维融合播放后的修正

1 rtsp 接入

我们使用unity UE 等三维渲染引擎中使用c++编写插件来接入rtsp 视频。同时做融合的时候，和背景的三维颜色要一致，这就要使用视频融合修正技术。包括亮度，对比度，饱和度的修正。在单纯颜色上的修正可以简单使用rgb->hsv去修改，这里不做累赘说明了。

2 播放技术

使用unity 的纹理渲染来播放多路视频，视频接入最大可达到30路，同时在untiy ， UE5 中渲染。播放后有很多视频的颜色不一致，如何调整颜色成了一个问题

3 untiy 渲染过程

建立一个材质，如命名为m2
在这里插入图片描述
将m2 赋值给我们的面片，如pp

接下来创建一个shader，控制渲染的亮度，对比度，饱和度，分别是 Brightness, Saturation, Contrast, unity shader的基础不再啰嗦，各位可以稍稍看一下，不难。

Shader "self/BrightnessSaturationAndContrast"
{Properties{_MainTex ("Base(RGB", 2D) = "white" {}//从脚本传递更好，这里可以直接省略这些值的展示_Brightness ("Brightness", float) = 1_Saturation ("Saturation", float) = 1_Contrast ("Contrast", float) = 1}SubShader{Pass{//关闭深度写入//ZTest Always Cull Off Zwrite OffCGPROGRAM#pragma vertex vert#pragma fragment frag#include "UnityCG.cginc"//propertiessampler2D _MainTex;half _Brightness;half _Saturation;half _Contrast;struct v2f {float4 pos : SV_POSITION;half2 uv : TEXCOORD0; };//使用了内置的appdata_img结构体作为顶点着色器的输入v2f vert(appdata_img  v) {v2f o;o.pos = UnityObjectToClipPos(v.vertex);o.uv = v.texcoord;return o;}fixed4 frag(v2f i) : SV_Target {//获得屏幕图像的采样fixed4 renderTex = tex2D(_MainTex, i.uv);//亮度fixed3 finalColor = renderTex.rgb * _Brightness;//饱和度fixed luminance = 0.2125 * renderTex.r + 0.7154 * renderTex.g + 0.0721 * renderTex.b;  //计算该像素的亮度值fixed3 luminanceColor = fixed3(luminance, luminance, luminance);  //创建饱和度为0的颜色finalColor = lerp(luminanceColor, finalColor, _Saturation);//contrastfixed3 avgColor = fixed3(0.5, 0.5, 0.5);finalColor = lerp(avgColor, finalColor, _Contrast);return fixed4(finalColor, renderTex.a);}ENDCG}}FallBack  Off
}

写好shader以后，写一个脚本，调入我们的c#脚本，脚本一为控制我们的c++插件，二是可以修改,以下的插件脚本负责调入c++ 的插件，插件会在线程里面启动，获取图像，赋值给面片。插件本身可以接入实时视频，播放视频，倒放视频，支持国标gb28181和onvif协议，可以支持矫正视频，支持多组矫正参数，同时增加三个变量，改变亮度，对比度，饱和度，便于在里面实时修改观察。

//author: 钱波
using System;
using System.Text;
using System.Threading;
using System.Collections;
using UnityEngine;
using System.Runtime.InteropServices;
using UnityEngine.UI;
using System.Linq;
using System.Collections.Generic;
using Unity.VisualScripting;
using System.IO;[StructLayout(LayoutKind.Sequential)]
public struct FRAME
{public int width;public int height;public int len;public IntPtr Frame;public IntPtr data;//public byte[] data;//[MarshalAs(UnmanagedType.LPArray)]
}
[StructLayout(LayoutKind.Sequential)]
public struct PARAM
{public double p1;public double p2;public double p3;public double p4;public double p5;public double p6;public double p7;public double p8;public double p9;public double c1;public double c2;public double c3;public double c4;public double c5;
}public class rtspin : MonoBehaviour
{[DllImport("rtspPlugin")]public static extern bool rtsp_test([MarshalAs(UnmanagedType.LPStr)] string url, int isnv12);[DllImport("rtspPlugin")]public static extern bool rtsp_test_data([MarshalAs(UnmanagedType.LPStr)] string url,[MarshalAs(UnmanagedType.LPArray)] byte[] data, ref FRAME frame);[DllImport("rtspPlugin")]public static extern bool rtsp_test_data_nv12([MarshalAs(UnmanagedType.LPStr)] string url,[MarshalAs(UnmanagedType.LPArray)] byte[] data0, [MarshalAs(UnmanagedType.LPArray)] byte[] data1);[DllImport("rtspPlugin")]public static extern void rtsp_test_stop([MarshalAs(UnmanagedType.LPStr)] string url);[DllImport("rtspPlugin")]public static extern void rtsp_test_setparam([MarshalAs(UnmanagedType.LPStr)] string url, ref PARAM param);static string[] stringArray = {"rtsp://127.0.0.1/front.mkv" };static string[] stringPlanes = { "pp"};//Texture2D[] texture2Ds;int w1 = 1920;int h1 = 1080;int number = 1;[Range(0.0f, 3.0f)]public float brightness = 1.0f;[Range(0.0f, 3.0f)]public float saturation = 1.0f;[Range(0.0f, 3.0f)]public float contrast = 1.0f;public Material briSatConMaterial;class cmd{public int v_w1 = 1920;public int v_h1 = 1080;public byte[] v_data;public Texture2D v_texture2Ds;public cmd(){v_data = new byte[v_w1 * v_h1 * 3];v_texture2Ds = new Texture2D(v_w1, v_h1, TextureFormat.RGB24, false);//RGB24}}PARAM v_param;//byte[][] v_datas;Dictionary<int, cmd> hashMap_datas = new Dictionary<int, cmd>();//byte[][] hashMap_datas;void rtspThreading(string url){Debug.Log(url);rtsp_test(url, 1);}void Start(){//string path = Application.dataPath + "/rtsp.txt";//path = path.Replace("/", "\\");//if (File.Exists(path))//{//    Debug.Log("FileExists");//    stringArray = File.ReadAllLines(path);//}//else//{//    Debug.Log("FileNotExists");//    File.CreateText(path);//}stringArray[0] = "rtsp://127.0.0.1/front.mkv";for (int i = 0; i < number; i++){cmd c1 = new cmd();hashMap_datas.Add(i, c1);}v_param = new PARAM();//v_param.p1 = 6.5746697810243404e+002;//v_param.p2 = 0.0;//v_param.p3 = 3.1950000000000000e+002;//v_param.p4 = 0.0;//v_param.p5 = 6.5746697810243404e+002;//v_param.p6 = 2.3950000000000000e+002;//v_param.p7 = 0.0;//v_param.p8 = 0.0;//v_param.p9 = 1.0;//v_param.c1 = -0.5180232701824102559;//v_param.c2 = 0.5071524380583312119;//v_param.c3 = 0.0;//v_param.c4 = 0.0;//v_param.c5 = -0.5784359684793970446;//1281.48 0 975.5 0 1997.48 0 0 0 1 -0.6 0.4 0.1 0 -0.198v_param.p1 = 1281.48;v_param.p2 = 0.0;v_param.p3 = 975.5;v_param.p4 = 0.0;v_param.p5 = 1997.48;v_param.p6 = 0.0;v_param.p7 = 0.0;v_param.p8 = 0.0;v_param.p9 = 1.0;v_param.c1 = -0.6;v_param.c2 = 0.4;v_param.c3 = 0.1;v_param.c4 = 0.0;v_param.c5 = -0.198;GameObject go = GameObject.Find("pp");briSatConMaterial = go.GetComponent<MeshRenderer>().material;}private void OnGUI(){if (GUI.Button(new Rect(120, 10, 80, 30), "开始线程")){Debug.Log("开始rtsp......");for (int i = 0; i < number; i++){int currentIndex = i;Thread rtspthread1 = new Thread(() => rtspThreading(stringArray[currentIndex]));rtspthread1.Start();Thread.Sleep(1);}}//绘制按钮，以及按下断开连接按钮，发送断开连接请求if (GUI.Button(new Rect(210, 10, 80, 30), "结束线程")){Debug.Log("结束rtsp......");for (int i = 0; i < number; i++){int currentIndex = i;rtsp_test_stop(stringArray[currentIndex]);}}}//float delta_x, delta_y, delta_z;            //计算移动量//float distance = 5;//float ZoomSpeed = 5f;                  //拉近拉远速度//public bool isFar = true;void Update(){FRAME frame = new FRAME();if (briSatConMaterial != null){briSatConMaterial.SetFloat("_Brightness", brightness);briSatConMaterial.SetFloat("_Saturation", saturation);briSatConMaterial.SetFloat("_Contrast", contrast);}for (int i = 0; i < number; i++){if (rtsp_test_data(stringArray[i], hashMap_datas[i].v_data, ref frame)){rtsp_test_setparam(stringArray[i], ref v_param);//Texture2D original = new Texture2D(w1, h1, TextureFormat.RGB24, false);//original.LoadRawTextureData(hashMap_datas[i].v_data);//FlipTexture(original, hashMap_datas[i].v_texture2Ds);hashMap_datas[i].v_texture2Ds.LoadRawTextureData(hashMap_datas[i].v_data);hashMap_datas[i].v_texture2Ds.Apply();GameObject go = GameObject.Find(stringPlanes[i]);go.GetComponent<MeshRenderer>().material.mainTexture = hashMap_datas[i].v_texture2Ds;}}}void OnDestroy(){Debug.Log("Destory, 结束rtsp......");for (int i = 0; i < number; i++){int currentIndex = i;rtsp_test_stop(stringArray[currentIndex]);}}
}