Skip to content

Commit

Permalink
fix: Optimize VideoSource capture and add WebCameraSource. (#59)
Browse files Browse the repository at this point in the history
  • Loading branch information
@cloudwebrtc
cloudwebrtc authored Nov 19, 2024
1 parent 9f9bfdf commit 47692c7
Show file tree
Hide file tree
Showing 6 changed files with 163 additions and 76 deletions.
39 changes: 32 additions & 7 deletions Runtime/Scripts/CameraVideoSource.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,10 +8,14 @@

namespace LiveKit
{

// VideoSource for Unity Camera
public class CameraVideoSource : RtcVideoSource
{
private TextureFormat _textureFormat;

private RenderTexture _renderTexture;

public Camera Camera { get; }

public override int GetWidth()
Expand All @@ -24,6 +28,11 @@ public override int GetHeight()
return Camera.pixelHeight;
}

protected override VideoRotation GetVideoRotation()
{
return VideoRotation._0;
}

public CameraVideoSource(Camera camera, VideoBufferType bufferType = VideoBufferType.Rgba) : base(VideoStreamSource.Screen, bufferType)
{
Camera = camera;
Expand All @@ -44,9 +53,9 @@ public override void Stop()

private void ClearRenderTexture()
{
if (_dest)
if (_renderTexture)
{
var renderText = _dest as RenderTexture;
var renderText = _renderTexture as RenderTexture;
renderText.Release(); // can only be done on main thread
}
}
Expand All @@ -60,19 +69,35 @@ protected override bool ReadBuffer()
var textureChanged = false;
try
{
if (_dest == null || _dest.width != GetWidth() || _dest.height != GetHeight())
if (_renderTexture == null || _renderTexture.width != GetWidth() || _renderTexture.height != GetHeight())
{
var targetFormat = Utils.GetSupportedGraphicsFormat(SystemInfo.graphicsDeviceType);
var compatibleFormat = SystemInfo.GetCompatibleFormat(targetFormat, FormatUsage.ReadPixels);
_textureFormat = GraphicsFormatUtility.GetTextureFormat(compatibleFormat);
_bufferType = GetVideoBufferType(_textureFormat);
_dest = new RenderTexture(GetWidth(), GetHeight(), 0, compatibleFormat);
Camera.targetTexture = _dest as RenderTexture;
_renderTexture = new RenderTexture(GetWidth(), GetHeight(), 0, compatibleFormat);
Camera.targetTexture = _renderTexture as RenderTexture;
_data = new NativeArray<byte>(GetWidth() * GetHeight() * GetStrideForBuffer(_bufferType), Allocator.Persistent);
_dest = new Texture2D(GetWidth(), GetHeight(), _textureFormat, false);
textureChanged = true;
}
ScreenCapture.CaptureScreenshotIntoRenderTexture(_dest as RenderTexture);
AsyncGPUReadback.RequestIntoNativeArray(ref _data, _dest, 0, _textureFormat, OnReadback);
ScreenCapture.CaptureScreenshotIntoRenderTexture(_renderTexture);

#if UNITY_EDITOR_OSX || UNITY_STANDALONE_OSX
// Flip the texture for OSX
Graphics.CopyTexture(_renderTexture, _dest);
var pixels = _dest.GetPixels();
var flippedPixels = new Color[pixels.Length];
for (int i = 0; i < _dest.height; i++)
{
Array.Copy(pixels, i * _dest.width, flippedPixels, (_dest.height - i - 1) * _dest.width, _dest.width);
}
_dest.SetPixels(flippedPixels);
#else
Graphics.CopyTexture(_renderTexture, _dest);
#endif

AsyncGPUReadback.RequestIntoNativeArray(ref _data, _renderTexture, 0, _textureFormat, OnReadback);
}
catch (Exception e)
{
Expand Down
47 changes: 6 additions & 41 deletions Runtime/Scripts/RtcVideoSource.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,11 +26,13 @@ public enum VideoStreamSource
public abstract int GetWidth();
public abstract int GetHeight();

protected abstract VideoRotation GetVideoRotation();

public delegate void TextureReceiveDelegate(Texture2D tex2d);
/// Called when we receive a new texture (first texture or the resolution changed)
public event TextureReceiveDelegate TextureReceived;

protected Texture _dest;
protected Texture2D _dest;
protected NativeArray<byte> _data;
protected VideoStreamSource _sourceType;
protected VideoBufferType _bufferType;
Expand All @@ -39,7 +41,6 @@ public enum VideoStreamSource
protected bool _requestPending = false;
protected bool isDisposed = true;
protected bool _playing = false;
private Texture2D _texture2D = null;
private bool _muted = false;
public override bool Muted => _muted;

Expand Down Expand Up @@ -128,31 +129,6 @@ public virtual void Stop()
_playing = false;
}

private void LoadToTexture2D(Texture2D tex, RenderTexture rTex)
{
if (tex == null || rTex == null)
{
return;
}

var old_rt = RenderTexture.active;
RenderTexture.active = rTex;
tex.ReadPixels(new Rect(0, 0, rTex.width, rTex.height), 0, 0);

#if UNITY_EDITOR_OSX || UNITY_STANDALONE_OSX
// Flip the texture for OSX
var pixels = tex.GetPixels();
var flippedPixels = new Color[pixels.Length];
for (int i = 0; i < rTex.height; i++)
{
Array.Copy(pixels, i * rTex.width, flippedPixels, (rTex.height - i - 1) * rTex.width, rTex.width);
}
tex.SetPixels(flippedPixels);
#endif
tex.Apply();
RenderTexture.active = old_rt;
}

public IEnumerator Update()
{
while (_playing)
Expand All @@ -162,20 +138,9 @@ public IEnumerator Update()

if(textureChanged)
{
if (_texture2D == null)
{
_texture2D = new Texture2D(_dest.width, _dest.height, TextureFormat.RGB24, false);
} else
{
_texture2D.Reinitialize(_dest.width, _dest.height);
}
TextureReceived?.Invoke(_texture2D);
TextureReceived?.Invoke(_dest);
}

if(TextureReceived != null && TextureReceived.GetInvocationList().Length > 0)
{
LoadToTexture2D(_texture2D, _dest as RenderTexture);
}
if(_muted)
{
continue;
Expand All @@ -195,7 +160,7 @@ public virtual void Dispose()
{
if (!isDisposed)
{
if (_texture2D != null) UnityEngine.Object.Destroy(_texture2D);
if (_dest != null) UnityEngine.Object.Destroy(_dest);
isDisposed = true;
}
}
Expand All @@ -222,7 +187,7 @@ protected virtual bool SendFrame()
using var request = FFIBridge.Instance.NewRequest<CaptureVideoFrameRequest>();
var capture = request.request;
capture.SourceHandle = (ulong)Handle.DangerousGetHandle();
capture.Rotation = VideoRotation._0;
capture.Rotation = GetVideoRotation();
capture.TimestampUs = DateTimeOffset.Now.ToUnixTimeMilliseconds();
capture.Buffer = buffer;
using var response = request.Send();
Expand Down
34 changes: 28 additions & 6 deletions Runtime/Scripts/ScreenVideoSource.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ namespace LiveKit
public class ScreenVideoSource : RtcVideoSource
{
private TextureFormat _textureFormat;
private RenderTexture _renderTexture;

public override int GetWidth()
{
Expand All @@ -22,6 +23,11 @@ public override int GetHeight()
return Screen.height;
}

protected override VideoRotation GetVideoRotation()
{
return VideoRotation._0;
}

public ScreenVideoSource(VideoBufferType bufferType = VideoBufferType.Rgba) : base(VideoStreamSource.Screen, bufferType)
{
base.Init();
Expand All @@ -41,9 +47,9 @@ public override void Stop()

private void ClearRenderTexture()
{
if (_dest)
if (_renderTexture)
{
var renderText = _dest as RenderTexture;
var renderText = _renderTexture as RenderTexture;
renderText.Release(); // can only be done on main thread
}
}
Expand All @@ -57,18 +63,34 @@ protected override bool ReadBuffer()
var textureChanged = false;
try
{
if (_dest == null || _dest.width != GetWidth() || _dest.height != GetHeight())
if (_renderTexture == null || _renderTexture.width != GetWidth() || _renderTexture.height != GetHeight())
{
var targetFormat = Utils.GetSupportedGraphicsFormat(SystemInfo.graphicsDeviceType);
var compatibleFormat = SystemInfo.GetCompatibleFormat(targetFormat, FormatUsage.ReadPixels);
_textureFormat = GraphicsFormatUtility.GetTextureFormat(compatibleFormat);
_bufferType = GetVideoBufferType(_textureFormat);
_dest = new RenderTexture(GetWidth(), GetHeight(), 0, compatibleFormat);
_renderTexture = new RenderTexture(GetWidth(), GetHeight(), 0, compatibleFormat);
_data = new NativeArray<byte>(GetWidth() * GetHeight() * GetStrideForBuffer(_bufferType), Allocator.Persistent);
_dest = new Texture2D(GetWidth(), GetHeight(), _textureFormat, false);
textureChanged = true;
}
ScreenCapture.CaptureScreenshotIntoRenderTexture(_dest as RenderTexture);
AsyncGPUReadback.RequestIntoNativeArray(ref _data, _dest, 0, _textureFormat, OnReadback);
ScreenCapture.CaptureScreenshotIntoRenderTexture(_renderTexture);

#if UNITY_EDITOR_OSX || UNITY_STANDALONE_OSX
// Flip the texture for OSX
Graphics.CopyTexture(_renderTexture, _dest);
var pixels = _dest.GetPixels();
var flippedPixels = new Color[pixels.Length];
for (int i = 0; i < _dest.height; i++)
{
Array.Copy(pixels, i * _dest.width, flippedPixels, (_dest.height - i - 1) * _dest.width, _dest.width);
}
_dest.SetPixels(flippedPixels);
#else
Graphics.CopyTexture(_renderTexture, _dest);
#endif

AsyncGPUReadback.RequestIntoNativeArray(ref _data, _renderTexture, 0, _textureFormat, OnReadback);
}
catch (Exception e)
{
Expand Down
35 changes: 13 additions & 22 deletions Runtime/Scripts/TextureVideoSource.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
using UnityEngine.Rendering;
using Unity.Collections;
using UnityEngine.Experimental.Rendering;
using System;

namespace LiveKit
{
Expand All @@ -22,6 +23,11 @@ public override int GetHeight()
return Texture.height;
}

protected override VideoRotation GetVideoRotation()
{
return VideoRotation._0;
}

public TextureVideoSource(Texture texture, VideoBufferType bufferType = VideoBufferType.Rgba) : base(VideoStreamSource.Texture, bufferType)
{
Texture = texture;
Expand All @@ -40,31 +46,16 @@ protected override bool ReadBuffer()
return false;
_reading = true;
var textureChanged = false;
if (!SystemInfo.IsFormatSupported(Texture.graphicsFormat, FormatUsage.ReadPixels))
{
if (_dest == null || _dest.width != GetWidth() || _dest.height != GetHeight())
{
var compatibleFormat = SystemInfo.GetCompatibleFormat(Texture.graphicsFormat, FormatUsage.ReadPixels);
_textureFormat = GraphicsFormatUtility.GetTextureFormat(compatibleFormat);
_bufferType = GetVideoBufferType(_textureFormat);
_data = new NativeArray<byte>(GetWidth() * GetHeight() * GetStrideForBuffer(_bufferType), Allocator.Persistent);
_dest = new Texture2D(GetWidth(), GetHeight(), _textureFormat, false);
textureChanged = true;
}
Graphics.CopyTexture(Texture, _dest);
}
else
{
if(_dest == null || _dest != Texture)
{
textureChanged = true;
}

_dest = Texture;
_textureFormat = GraphicsFormatUtility.GetTextureFormat(Texture.graphicsFormat);
if (_dest == null || _dest.width != GetWidth() || _dest.height != GetHeight()) {
var compatibleFormat = SystemInfo.GetCompatibleFormat(Texture.graphicsFormat, FormatUsage.ReadPixels);
_textureFormat = GraphicsFormatUtility.GetTextureFormat(compatibleFormat);
_bufferType = GetVideoBufferType(_textureFormat);
_data = new NativeArray<byte>(GetWidth() * GetHeight() * GetStrideForBuffer(_bufferType), Allocator.Persistent);
_dest = new Texture2D(GetWidth(), GetHeight(), _textureFormat, false);
textureChanged = true;
}

Graphics.CopyTexture(Texture, _dest);
AsyncGPUReadback.RequestIntoNativeArray(ref _data, _dest, 0, _textureFormat, OnReadback);
return textureChanged;
}
Expand Down
73 changes: 73 additions & 0 deletions Runtime/Scripts/WebCameraSource.cs
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
using UnityEngine;
using LiveKit.Proto;
using UnityEngine.Rendering;
using Unity.Collections;
using UnityEngine.Experimental.Rendering;
using System;
using System.Runtime.InteropServices;

namespace LiveKit
{
// VideoSource for Unity WebCamTexture
public class WebCameraSource : RtcVideoSource
{
TextureFormat _textureFormat;

public WebCamTexture Texture { get; }

public override int GetWidth()
{
return Texture.width;
}

public override int GetHeight()
{
return Texture.height;
}

protected override VideoRotation GetVideoRotation()
{
return VideoRotation._180;
}

public WebCameraSource(WebCamTexture texture, VideoBufferType bufferType = VideoBufferType.Rgba) : base(VideoStreamSource.Texture, bufferType)
{
Texture = texture;
base.Init();
}

~WebCameraSource()
{
Dispose();
}

// Read the texture data into a native array asynchronously
protected override bool ReadBuffer()
{
if (_reading && !Texture.isPlaying)
return false;
_reading = true;
var textureChanged = false;

if (_dest == null || _dest.width != GetWidth() || _dest.height != GetHeight())
{
var compatibleFormat = SystemInfo.GetCompatibleFormat(Texture.graphicsFormat, FormatUsage.ReadPixels);
_textureFormat = GraphicsFormatUtility.GetTextureFormat(compatibleFormat);
_bufferType = GetVideoBufferType(_textureFormat);
_data = new NativeArray<byte>(GetWidth() * GetHeight() * GetStrideForBuffer(_bufferType), Allocator.Persistent);
_dest = new Texture2D(GetWidth(), GetHeight(), TextureFormat.BGRA32, false);
textureChanged = true;
}

Color32[] pixels = new Color32[GetWidth() * GetHeight()];
Texture.GetPixels32(pixels);
var bytes = MemoryMarshal.Cast<Color32, byte>(pixels);
_data.CopyFrom(bytes.ToArray());
_requestPending = true;
Graphics.CopyTexture(Texture, _dest);

return textureChanged;
}
}
}

11 changes: 11 additions & 0 deletions Runtime/Scripts/WebCameraSource.cs.meta
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

0 comments on commit 47692c7

Please sign in to comment.