Windows平台Unity下实现camera场景推送RTMP|轻量级RTSP服务|实时录像

技术背景

我们在对接Unity平台camera场景采集的时候，除了常规的RTMP推送、录像外，还有一些开发者，需要能实现轻量级RTSP服务，对外提供个拉流的RTSP URL。

目前我们在Windows平台Unity下数据源可采集到以下部分：

采集Unity camera场景；
采集摄像头；
采集屏幕；
采集Unity声音；
采集麦克风；
采集扬声器；
Unity PCM混音；

对外提供的技术能力有：

RTMP直播推送；
轻量级RTSP服务；
实时录像、暂停|恢复录像；
实时预览。

以下录制下来的MP4文件是采集Unity camera场景，音频是unity声音。

Unity平台实现camera场景实时录像

技术实现

实际上，在实现Unity平台音视频能力之前，我们原生模块已经有非常成熟的技术积累，Unity下还是调用的原生的推送模块，不同的是，数据源需要采集Unity的audio、video，然后高效的投递到底层模块，底层模块负责编码打包，并投递到RTMP或RTSP服务。

先说支持的音视频类型：

    public void SelVideoPushType(int type){switch (type){case 0:video_push_type_ = (uint)NTSmartPublisherDefine.NT_PB_E_VIDEO_OPTION.NT_PB_E_VIDEO_OPTION_LAYER;    //采集Unity窗体break;case 1:video_push_type_ = (uint)NTSmartPublisherDefine.NT_PB_E_VIDEO_OPTION.NT_PB_E_VIDEO_OPTION_CAMERA;   //采集摄像头break;case 2:video_push_type_ = (uint)NTSmartPublisherDefine.NT_PB_E_VIDEO_OPTION.NT_PB_E_VIDEO_OPTION_SCREEN;   //采集屏幕break;case 3:video_push_type_ = (uint)NTSmartPublisherDefine.NT_PB_E_VIDEO_OPTION.NT_PB_E_VIDEO_OPTION_NO_VIDEO; //不采集视频break;}Debug.Log("SelVideoPushType type: " + type + " video_push_type: " + video_push_type_);}public void SelAudioPushType(int type){switch (type){case 0:audio_push_type_ = (uint)NTSmartPublisherDefine.NT_PB_E_AUDIO_OPTION.NT_PB_E_AUDIO_OPTION_EXTERNAL_PCM_DATA;    //采集Unity声音break;case 1:audio_push_type_ = (uint)NTSmartPublisherDefine.NT_PB_E_AUDIO_OPTION.NT_PB_E_AUDIO_OPTION_CAPTURE_MIC;  //采集麦克风break;case 2:audio_push_type_ = (uint)NTSmartPublisherDefine.NT_PB_E_AUDIO_OPTION.NT_PB_E_AUDIO_OPTION_CAPTURE_SPEAKER;  //采集扬声器break;case 3:audio_push_type_ = (uint)NTSmartPublisherDefine.NT_PB_E_AUDIO_OPTION.NT_PB_E_AUDIO_OPTION_TWO_EXTERNAL_PCM_MIXER;  //两路Unity AudioClip混音测试break;case 4:audio_push_type_ = (uint)NTSmartPublisherDefine.NT_PB_E_AUDIO_OPTION.NT_PB_E_AUDIO_OPTION_NO_AUDIO;   //不采集音频break;}Debug.Log("SelAudioPushType type: " + type + " audio_push_type: " + audio_push_type_);}

采集音视频数据：

    private void StartCaptureAvData(){if (audio_push_type_ == (uint)NTSmartPublisherDefine.NT_PB_E_AUDIO_OPTION.NT_PB_E_AUDIO_OPTION_EXTERNAL_PCM_DATA|| audio_push_type_ == (uint)NTSmartPublisherDefine.NT_PB_E_AUDIO_OPTION.NT_PB_E_AUDIO_OPTION_TWO_EXTERNAL_PCM_MIXER){PostUnityAudioClipData();}textures_poll_ = new TexturesPool();post_image_worker_ = new PostImageWorker(textures_poll_, publisher_wrapper_);post_worker_thread_ = new Thread(post_image_worker_.run);post_worker_thread_.Start();}private void StopCaptureAvData(){if (post_image_worker_ != null){post_image_worker_.requestStop();post_image_worker_ = null;}if (post_worker_thread_ != null){post_worker_thread_.Join();post_worker_thread_ = null;}if (textures_poll_ != null){textures_poll_.clear();textures_poll_ = null;}StopAudioSource();}

RTMP推送：

    public void btn_start_rtmp_pusher_Click(){if (publisher_wrapper_.IsPushingRtmp()){StopPushRTMP();btn_rtmp_pusher_.GetComponentInChildren<Text>().text = "推送RTMP";return;}String url = rtmp_pusher_url_.text;if (url.Length < 8){publisher_wrapper_.Close();Debug.LogError("请输入RTMP推送地址");return;}if (!publisher_wrapper_.IsPreviewing() && !publisher_wrapper_.IsRtspPublisherRunning() && !publisher_wrapper_.IsRecording()){publisher_wrapper_.SetVideoPushType(video_push_type_);publisher_wrapper_.SetAudioPushType(audio_push_type_);}if (!publisher_wrapper_.StartRtmpPusher(url)){Debug.LogError("调用StartPublisher失败..");return;}btn_rtmp_pusher_.GetComponentInChildren<Text>().text = "停止推送";if (!publisher_wrapper_.IsPreviewing() && !publisher_wrapper_.IsRtspPublisherRunning() && !publisher_wrapper_.IsRecording()){StartCaptureAvData();coroutine_ = StartCoroutine(OnPostVideo());}}

轻量级RTSP服务相关调用：

    public void btn_rtsp_service_Click(){if (publisher_wrapper_.IsRtspServiceRunning()){publisher_wrapper_.StopRtspService();btn_rtsp_service_.GetComponentInChildren<Text>().text = "启动RTSP服务";btn_rtsp_publisher_.interactable = false;return;}if (!publisher_wrapper_.StartRtspService()){Debug.LogError("调用StartRtspService失败..");return;}btn_rtsp_publisher_.interactable = true;btn_rtsp_service_.GetComponentInChildren<Text>().text = "停止RTSP服务";}public void btn_rtsp_publisher_Click(){if (publisher_wrapper_.IsRtspPublisherRunning()){publisher_wrapper_.StopRtspStream();if (!publisher_wrapper_.IsPreviewing() && !publisher_wrapper_.IsPushingRtmp() && !publisher_wrapper_.IsRecording()){StopCaptureAvData();if (coroutine_ != null){StopCoroutine(coroutine_);coroutine_ = null;}}btn_rtsp_service_.interactable = true;btn_rtsp_publisher_.GetComponentInChildren<Text>().text = "发布RTSP";}else{if (!publisher_wrapper_.IsRtspServiceRunning()){Debug.LogError("RTSP service is not running..");return;}if (!publisher_wrapper_.IsPreviewing() && !publisher_wrapper_.IsPushingRtmp() && !publisher_wrapper_.IsRecording()){publisher_wrapper_.SetVideoPushType(video_push_type_);publisher_wrapper_.SetAudioPushType(audio_push_type_);}publisher_wrapper_.StartRtspStream();if (!publisher_wrapper_.IsPreviewing() && !publisher_wrapper_.IsPushingRtmp() && !publisher_wrapper_.IsRecording()){StartCaptureAvData();coroutine_ = StartCoroutine(OnPostVideo());}btn_rtsp_publisher_.GetComponentInChildren<Text>().text = "停止RTSP";btn_rtsp_service_.interactable = false;}}public void btn_get_rtsp_session_numbers_Click(){if (publisher_wrapper_.IsRtspServiceRunning()){btn_get_rtsp_session_numbers_.GetComponentInChildren<Text>().text = "RTSP会话数:" + publisher_wrapper_.GetRtspSessionNumbers();}}

实时录像调用：

    public void btn_record_Click(){if (publisher_wrapper_.IsRecording()){StopRecord();btn_record_.GetComponentInChildren<Text>().text = "开始录像";return;}if (!publisher_wrapper_.IsPreviewing() && !publisher_wrapper_.IsRtspPublisherRunning() && !publisher_wrapper_.IsPushingRtmp()){publisher_wrapper_.SetVideoPushType(video_push_type_);publisher_wrapper_.SetAudioPushType(audio_push_type_);}if (!publisher_wrapper_.StartRecorder()){Debug.LogError("调用StartRecorder失败..");return;}btn_record_.GetComponentInChildren<Text>().text = "停止录像";if (!publisher_wrapper_.IsPreviewing() && !publisher_wrapper_.IsRtspPublisherRunning() && !publisher_wrapper_.IsPushingRtmp()){StartCaptureAvData();coroutine_ = StartCoroutine(OnPostVideo());}}public void StopRecord(){if (!publisher_wrapper_.IsPreviewing() && !publisher_wrapper_.IsRtspPublisherRunning() && !publisher_wrapper_.IsPushingRtmp()){StopCaptureAvData();if (coroutine_ != null){StopCoroutine(coroutine_);coroutine_ = null;}}publisher_wrapper_.StopRecorder();}private void btn_pause_record_Click(){if (!publisher_wrapper_.IsPublisherHandleAvailable()){return;}String btn_pause_rec_text = btn_pause_record_.GetComponentInChildren<Text>().text;if ("暂停录像" == btn_pause_rec_text){UInt32 ret = publisher_wrapper_.PauseRecorder(true);if ((UInt32)NT.NTSmartPublisherDefine.NT_PB_E_ERROR_CODE.NT_ERC_PB_NEED_RETRY == ret){Debug.LogError("暂停录像失败, 请重新尝试!");return;}else if (NTBaseCodeDefine.NT_ERC_OK == ret){btn_pause_record_.GetComponentInChildren<Text>().text = "恢复录像";}}else{UInt32 ret = publisher_wrapper_.PauseRecorder(false);if ((UInt32)NT.NTSmartPublisherDefine.NT_PB_E_ERROR_CODE.NT_ERC_PB_NEED_RETRY == ret){Debug.LogError("恢复录像失败, 请重新尝试!");return;}else if (NTBaseCodeDefine.NT_ERC_OK == ret){btn_pause_record_.GetComponentInChildren<Text>().text = "暂停录像";}}}

实时预览相关：

    public void btn_preview_Click(){if (btn_preview_.GetComponentInChildren<Text>().text.Equals("本地预览")){if (!publisher_wrapper_.IsPushingRtmp() && !publisher_wrapper_.IsRtspPublisherRunning() && !publisher_wrapper_.IsRecording()){publisher_wrapper_.SetVideoPushType(video_push_type_);}if (publisher_wrapper_.StartPreview()){btn_preview_.GetComponentInChildren<Text>().text = "停止预览";}if (!publisher_wrapper_.IsPushingRtmp() && !publisher_wrapper_.IsRtspPublisherRunning() && !publisher_wrapper_.IsRecording()){StartCaptureAvData();coroutine_ = StartCoroutine(OnPostVideo());}}else{if (!publisher_wrapper_.IsPushingRtmp() && !publisher_wrapper_.IsRtspPublisherRunning() && !publisher_wrapper_.IsRecording()){StopCaptureAvData();if (coroutine_ != null){StopCoroutine(coroutine_);coroutine_ = null;}}publisher_wrapper_.StopPreview();btn_preview_.GetComponentInChildren<Text>().text = "本地预览";}}