Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add HDR support to DX11 Capture #61

Open
wants to merge 8 commits into
base: master
Choose a base branch
from
Open

Add HDR support to DX11 Capture #61

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
13a85b5
Replace SharpDX by Vortice and add HDR support
ktaeleman Jan 12, 2023
d95e294
Add texture format detection and cleanup
ktaeleman Jan 12, 2023
a26cb79
Go back to Vortice as it isn't needed
ktaeleman Jan 13, 2023
70c22d3
Some extra comments
ktaeleman Jan 13, 2023
74951e9
Some extra comments
ktaeleman Jan 13, 2023
fa3051b
Add support for R10G10B10A2_UNorm
ktaeleman Mar 7, 2023
534012b
Add R10G10B10A2 to dxgi desktopformats
ktaeleman Mar 7, 2023
38775af
Fix for color swap with R10G10B10A2 and remove endian check as our ta…
ktaeleman Mar 8, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
225 changes: 176 additions & 49 deletions HyperionScreenCap/Capture/Dx11ScreenCapture.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,21 @@
using System.Text;
using System.Threading;



namespace HyperionScreenCap
{

static class MathExt
{
public static T Clamp<T>(this T val, T min, T max) where T : IComparable<T>
{
if (val.CompareTo(min) < 0) return min;
else if (val.CompareTo(max) > 0) return max;
else return val;
}
}

class DX11ScreenCapture : IScreenCapture
{
private int _adapterIndex;
Expand All @@ -26,12 +39,21 @@ class DX11ScreenCapture : IScreenCapture
private Factory1 _factory;
private Adapter _adapter;
private Output _output;
private Output1 _output1;
private Output5 _output5;
private SharpDX.Direct3D11.Device _device;
private Texture2D _stagingTexture;
private Texture2D _smallerTexture;
private ShaderResourceView _smallerTextureView;
private OutputDuplication _duplicatedOutput;
//private IDXGIFactory2 _factory;
//private IDXGIAdapter1 _adapter;
//private IDXGIOutput _output;
//private IDXGIOutput5 _output5;
//private ID3D11Device _device;
//private ID3D11Texture2D _stagingTexture;
//private ID3D11Texture2D _smallerTexture;
//private ID3D11ShaderResourceView _smallerTextureView;
//private IDXGIOutputDuplication _duplicatedOutput;
private int _scalingFactorLog2;
private int _width;
private int _height;
Expand All @@ -40,6 +62,10 @@ class DX11ScreenCapture : IScreenCapture
private Stopwatch _captureTimer;
private bool _desktopDuplicatorInvalid;
private bool _disposed;
//private static readonly FeatureLevel[] s_featureLevels = new[]
//{
//FeatureLevel.Level_11_0
//};

public int CaptureWidth { get; private set; }
public int CaptureHeight { get; private set; }
Expand Down Expand Up @@ -78,18 +104,6 @@ public DX11ScreenCapture(int adapterIndex, int monitorIndex, int scalingFactor,

public void Initialize()
{
int mipLevels;
if ( _scalingFactor == 1 )
mipLevels = 1;
else if ( _scalingFactor > 0 && _scalingFactor % 2 == 0 )
{
/// Mip level for a scaling factor other than one is computed as follows:
/// 2^n = 2 + n - 1 where LHS is the scaling factor and RHS is the MipLevels value.
_scalingFactorLog2 = Convert.ToInt32(Math.Log(_scalingFactor, 2));
mipLevels = 2 + _scalingFactorLog2 - 1;
}
else
throw new Exception("Invalid scaling factor. Allowed valued are 1, 2, 4, etc.");

// Create DXGI Factory1
_factory = new Factory1();
Expand All @@ -100,7 +114,7 @@ public void Initialize()

// Get DXGI.Output
_output = _adapter.GetOutput(_monitorIndex);
_output1 = _output.QueryInterface<Output1>();
_output5 = _output.QueryInterface<Output5>();

// Width/Height of desktop to capture
var desktopBounds = _output.Description.DesktopBounds;
Expand All @@ -109,13 +123,48 @@ public void Initialize()

CaptureWidth = _width / _scalingFactor;
CaptureHeight = _height / _scalingFactor;

// Initialize duplicator so we can see what the output format is
InitDesktopDuplicator();


_minCaptureTime = 1000 / _maxFps;
_captureTimer = new Stopwatch();
_disposed = false;
}

private void InitDesktopDuplicator()
{
// We're potentially reinitializing the duplicator, which could change output format
// So make sure we reinitialize our textures
_stagingTexture?.Dispose();
_smallerTexture?.Dispose();
_smallerTextureView?.Dispose();

// Duplicate the output
Format[] DesktopFormats = { Format.R16G16B16A16_Float, Format.B8G8R8A8_UNorm, Format.R10G10B10A2_UNorm};
_duplicatedOutput = _output5.DuplicateOutput1(_device, 0, DesktopFormats.Count(), DesktopFormats);

// Calculate miplevels
int mipLevels;
if ( _scalingFactor == 1 )
mipLevels = 1;
else if ( _scalingFactor > 0 && _scalingFactor % 2 == 0 )
{
/// Mip level for a scaling factor other than one is computed as follows:
/// 2^n = 2 + n - 1 where LHS is the scaling factor and RHS is the MipLevels value.
_scalingFactorLog2 = Convert.ToInt32(Math.Log(_scalingFactor, 2));
mipLevels = 2 + _scalingFactorLog2 - 1;
}
else
throw new Exception("Invalid scaling factor. Allowed valued are 1, 2, 4, etc.");

// Create Staging texture CPU-accessible
var stagingTextureDesc = new Texture2DDescription
{
CpuAccessFlags = CpuAccessFlags.Read,
BindFlags = BindFlags.None,
Format = Format.B8G8R8A8_UNorm,
Format = _duplicatedOutput.Description.ModeDescription.Format,
Width = CaptureWidth,
Height = CaptureHeight,
OptionFlags = ResourceOptionFlags.None,
Expand All @@ -131,7 +180,7 @@ public void Initialize()
{
CpuAccessFlags = CpuAccessFlags.None,
BindFlags = BindFlags.RenderTarget | BindFlags.ShaderResource,
Format = Format.B8G8R8A8_UNorm,
Format = _duplicatedOutput.Description.ModeDescription.Format,
Width = _width,
Height = _height,
OptionFlags = ResourceOptionFlags.GenerateMipMaps,
Expand All @@ -143,18 +192,6 @@ public void Initialize()
_smallerTexture = new Texture2D(_device, smallerTextureDesc);
_smallerTextureView = new ShaderResourceView(_device, _smallerTexture);

_minCaptureTime = 1000 / _maxFps;
_captureTimer = new Stopwatch();
_disposed = false;

InitDesktopDuplicator();
}

private void InitDesktopDuplicator()
{
// Duplicate the output
_duplicatedOutput = _output1.DuplicateOutput(_device);

_desktopDuplicatorInvalid = false;
}

Expand Down Expand Up @@ -221,7 +258,7 @@ private byte[] ManagedCapture()

// Get the desktop capture texture
var mapSource = _device.ImmediateContext.MapSubresource(_stagingTexture, 0, MapMode.Read, SharpDX.Direct3D11.MapFlags.None);
_lastCapturedFrame = ToRGBArray(mapSource);
_lastCapturedFrame = ToRGBArray(mapSource, _stagingTexture.Description.Format);
return _lastCapturedFrame;
}
finally
Expand All @@ -234,42 +271,132 @@ private byte[] ManagedCapture()
}
}

public static float Parse16BitFloat(byte Hi, byte Lo)
{
// From https://stackoverflow.com/questions/6162651/half-precision-floating-point-in-java/6162687#6162687

int fullFloat = ((Hi << 8) | Lo);
int mant = fullFloat & 0x03ff; // 10 bits mantissa
int exp = fullFloat & 0x7c00; // 5 bits exponent
if( exp == 0x7c00 ) // NaN/Inf
exp = 0x3fc00; // -> NaN/Inf
else if( exp != 0 ) // normalized value
{
exp += 0x1c000; // exp - 15 + 127
if( mant == 0 && exp > 0x1c400 ) // smooth transition
return BitConverter.ToSingle(BitConverter.GetBytes( ( fullFloat & 0x8000 ) << 16
| exp << 13 | 0x3ff ), 0);
}
else if( mant != 0 ) // && exp==0 -> subnormal
{
exp = 0x1c400; // make it normal
do {
mant <<= 1; // mantissa * 2
exp -= 0x400; // decrease exp by 1
} while( ( mant & 0x400 ) == 0 ); // while not normal
mant &= 0x3ff; // discard subnormal bit
} // else +/-0 -> +/-0
return BitConverter.ToSingle(BitConverter.GetBytes( // combine all parts
( fullFloat & 0x8000 ) << 16 // sign << ( 31 - 15 )
| ( exp | mant ) << 13 ), 0); // value << ( 23 - 10 )
}
/// <summary>
/// Reads from the memory locations pointed to by the DataBox and saves it into a byte array
/// ignoring the alpha component of each pixel.
/// </summary>
/// <param name="mapSource"></param>
/// <returns></returns>
private byte[] ToRGBArray(DataBox mapSource)
private byte[] ToRGBArray(DataBox mapSource, Format format)
{
var sourcePtr = mapSource.DataPointer;
byte[] bytes = new byte[CaptureWidth * 3 * CaptureHeight];
int byteIndex = 0;
for ( int y = 0; y < CaptureHeight; y++ )
{
Int32[] rowData = new Int32[CaptureWidth];
Marshal.Copy(sourcePtr, rowData, 0, CaptureWidth);

foreach ( Int32 pixelData in rowData )
if (format == Format.R16G16B16A16_Float)
{
unsafe
{
byte[] values = BitConverter.GetBytes(pixelData);
if ( BitConverter.IsLittleEndian )
byte* ptr = (byte*)mapSource.DataPointer;
byte* rowptr = ptr;
for (int y = 0; y < CaptureHeight; y++)
{
// Byte order : bgra
bytes[byteIndex++] = values[2];
bytes[byteIndex++] = values[1];
bytes[byteIndex++] = values[0];
byte* pixelptr = rowptr;
for (int x = 0; x < CaptureWidth; x++)
{
for (int comp = 0; comp < 3; comp++)
{
byte lo = *pixelptr++;
byte hi = *pixelptr++;

// No idea why these values range from 4.6 to 0 instead of 1 to 0
// f(x) = sqrt(x/4.6) seems to approximate what the values should be.
bytes[byteIndex++] = (byte)(MathExt.Clamp(Math.Sqrt(Parse16BitFloat(hi, lo) / 4.6), 0, 1) * 255);
}
pixelptr += 2; //skip alpha
}
rowptr += mapSource.RowPitch;
}
else
}
}
else if (format == Format.B8G8R8A8_UNorm)
{
IntPtr sourcePtr = mapSource.DataPointer;

for ( int y = 0; y < CaptureHeight; y++ )
{
Int32[] rowData = new Int32[CaptureWidth];
Marshal.Copy(sourcePtr, rowData, 0, CaptureWidth);

foreach ( Int32 pixelData in rowData )
{
// Byte order : argb
bytes[byteIndex++] = values[1];
bytes[byteIndex++] = values[2];
bytes[byteIndex++] = values[3];
byte[] values = BitConverter.GetBytes(pixelData);
if ( BitConverter.IsLittleEndian )
{
// Byte order : bgra
bytes[byteIndex++] = values[2];
bytes[byteIndex++] = values[1];
bytes[byteIndex++] = values[0];
}
else
{
// Byte order : argb
bytes[byteIndex++] = values[1];
bytes[byteIndex++] = values[2];
bytes[byteIndex++] = values[3];
}
}

sourcePtr = IntPtr.Add(sourcePtr, mapSource.RowPitch);
}
}
else if (format == Format.R10G10B10A2_UNorm)
{
IntPtr sourcePtr = mapSource.DataPointer;

sourcePtr = IntPtr.Add(sourcePtr, mapSource.RowPitch);
for ( int y = 0; y < CaptureHeight; y++ )
{
Int32[] rowData = new Int32[CaptureWidth];
Marshal.Copy(sourcePtr, rowData, 0, CaptureWidth);

// From http://threadlocalmutex.com/?page_id=60, 10bit to 8bit conversion: (x * 1021 + 2048) >> 12
Func<Int32, byte> convert10bitTo8bit = (x) => (byte)MathExt.Clamp((x * 1021 + 2048) >> 12, 0, 255);

foreach ( Int32 pixelData in rowData )
{
Int32 r = pixelData & 0x3FF;
Int32 g = (pixelData >> 10) & 0x3FF;
Int32 b = (pixelData >> 20) & 0x3FF;

bytes[byteIndex++] = convert10bitTo8bit(r);
bytes[byteIndex++] = convert10bitTo8bit(g);
bytes[byteIndex++] = convert10bitTo8bit(b);
}

sourcePtr = IntPtr.Add(sourcePtr, mapSource.RowPitch);
}
}
else
{
throw new NotImplementedException($"Texture format {format.ToString()} is not supported.");
}
return bytes;
}
Expand All @@ -286,7 +413,7 @@ public void DelayNextCapture()
public void Dispose()
{
_duplicatedOutput?.Dispose();
_output1?.Dispose();
_output5?.Dispose();
_output?.Dispose();
_stagingTexture?.Dispose();
_smallerTexture?.Dispose();
Expand Down
6 changes: 4 additions & 2 deletions HyperionScreenCap/Helper/HyperionTask.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -139,8 +139,10 @@ private void TransmitNextFrame()
try
{
byte[] imageData = _screenCapture.Capture();
hyperionClient.SendImageData(imageData, _screenCapture.CaptureWidth, _screenCapture.CaptureHeight);

if (imageData != null)
{
hyperionClient.SendImageData(imageData, _screenCapture.CaptureWidth, _screenCapture.CaptureHeight);
}
// Uncomment the following to enable debugging
// MiscUtils.SaveRGBArrayToImageFile(imageData, _screenCapture.CaptureWidth, _screenCapture.CaptureHeight, AppConstants.DEBUG_IMAGE_FILE_NAME);
}
Expand Down
11 changes: 9 additions & 2 deletions HyperionScreenCap/HyperionScreenCap.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,7 @@
<ErrorReport>prompt</ErrorReport>
<CodeAnalysisRuleSet>MinimumRecommendedRules.ruleset</CodeAnalysisRuleSet>
<Prefer32Bit>true</Prefer32Bit>
<AllowUnsafeBlocks>true</AllowUnsafeBlocks>
</PropertyGroup>
<ItemGroup>
<Reference Include="Costura, Version=4.1.0.0, Culture=neutral, PublicKeyToken=9919ef960d84173d, processorArchitecture=MSIL">
Expand Down Expand Up @@ -122,6 +123,7 @@
<Reference Include="Newtonsoft.Json, Version=12.0.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed, processorArchitecture=MSIL">
<HintPath>..\packages\Newtonsoft.Json.12.0.3\lib\net45\Newtonsoft.Json.dll</HintPath>
</Reference>
<Reference Include="PresentationCore" />
<Reference Include="RestSharp, Version=106.11.7.0, Culture=neutral, PublicKeyToken=598062e77f915f75, processorArchitecture=MSIL">
<HintPath>..\packages\RestSharp.106.11.7\lib\net452\RestSharp.dll</HintPath>
</Reference>
Expand Down Expand Up @@ -163,8 +165,9 @@
<Reference Include="System.configuration" />
<Reference Include="System.Core" />
<Reference Include="System.Drawing" />
<Reference Include="System.Memory, Version=4.0.1.1, Culture=neutral, PublicKeyToken=cc7b13ffcd2ddd51, processorArchitecture=MSIL">
<HintPath>..\packages\System.Memory.4.5.4\lib\net461\System.Memory.dll</HintPath>
<Reference Include="System.Management" />
<Reference Include="System.Memory, Version=4.0.1.2, Culture=neutral, PublicKeyToken=cc7b13ffcd2ddd51, processorArchitecture=MSIL">
<HintPath>..\packages\System.Memory.4.5.5\lib\net461\System.Memory.dll</HintPath>
</Reference>
<Reference Include="System.Net" />
<Reference Include="System.Numerics" />
Expand All @@ -190,6 +193,7 @@
<Reference Include="Microsoft.CSharp" />
<Reference Include="System.Data" />
<Reference Include="System.Xml" />
<Reference Include="WindowsBase" />
</ItemGroup>
<ItemGroup>
<Compile Include="ApiServer.cs" />
Expand Down Expand Up @@ -382,6 +386,9 @@
<Analyzer Include="..\packages\Microsoft.CodeAnalysis.Analyzers.3.3.1\analyzers\dotnet\cs\Microsoft.CodeAnalysis.Analyzers.dll" />
<Analyzer Include="..\packages\Microsoft.CodeAnalysis.Analyzers.3.3.1\analyzers\dotnet\cs\Microsoft.CodeAnalysis.CSharp.Analyzers.dll" />
</ItemGroup>
<ItemGroup>
<WCFMetadata Include="Connected Services\" />
</ItemGroup>
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
<Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
<PropertyGroup>
Expand Down
Loading