Add HDR support

alvr/client_core/cpp/bindings.h

@@ -23,6 +23,7 @@ struct FfiStreamConfig {
     float foveationEdgeRatioX;
     float foveationEdgeRatioY;
     unsigned int enableSrgbCorrection;
+    unsigned int fixLimitedRange;
 };
 
 // gltf_model.h
@@ -37,7 +38,8 @@ extern "C" void destroyGraphicsNative();
 extern "C" void prepareLobbyRoom(int viewWidth,
                                  int viewHeight,
                                  const unsigned int *swapchainTextures[2],
-                                 int swapchainLength);
+                                 int swapchainLength,
+                                 bool enable_srgb_correction);
 extern "C" void destroyRenderers();
 extern "C" void streamStartNative(FfiStreamConfig config);
 extern "C" void updateLobbyHudTexture(const unsigned char *data);

alvr/client_core/cpp/graphics.cpp

@@ -322,7 +322,7 @@ void ovrFramebuffer_Create(ovrFramebuffer *frameBuffer,
     for (int i = 0; i < textures.size(); i++) {
         auto glRenderTarget = textures[i];
         frameBuffer->renderTargets.push_back(std::make_unique<gl_render_utils::Texture>(
-            true, glRenderTarget, false, width, height, GL_SRGB8_ALPHA8, GL_RGBA));
+            true, glRenderTarget, false, width, height, GL_RGBA16F, GL_RGBA));
         frameBuffer->renderStates.push_back(
             std::make_unique<gl_render_utils::RenderState>(frameBuffer->renderTargets[i].get()));
     }
@@ -580,19 +580,20 @@ void ovrRenderer_Create(ovrRenderer *renderer,
                         std::vector<GLuint> textures[2],
                         FFRData ffrData,
                         bool isLobby,
-                        bool enableSrgbCorrection) {
+                        bool enableSrgbCorrection,
+                        bool fixLimitedRange) {
     if (!isLobby) {
         renderer->srgbCorrectionPass = std::make_unique<SrgbCorrectionPass>(streamTexture);
         renderer->enableFFE = ffrData.enabled;
         if (renderer->enableFFE) {
             FoveationVars fv = CalculateFoveationVars(ffrData);
             renderer->srgbCorrectionPass->Initialize(
-                fv.optimizedEyeWidth, fv.optimizedEyeHeight, !enableSrgbCorrection);
+                fv.optimizedEyeWidth, fv.optimizedEyeHeight, !enableSrgbCorrection, fixLimitedRange);
             renderer->ffr = std::make_unique<FFR>(renderer->srgbCorrectionPass->GetOutputTexture());
             renderer->ffr->Initialize(fv);
             renderer->streamRenderTexture = renderer->ffr->GetOutputTexture()->GetGLTexture();
         } else {
-            renderer->srgbCorrectionPass->Initialize(width, height, !enableSrgbCorrection);
+            renderer->srgbCorrectionPass->Initialize(width, height, !enableSrgbCorrection, fixLimitedRange);
             renderer->streamRenderTexture =
                 renderer->srgbCorrectionPass->GetOutputTexture()->GetGLTexture();
         }
@@ -793,7 +794,8 @@ void destroyGraphicsNative() {
 void prepareLobbyRoom(int viewWidth,
                       int viewHeight,
                       const unsigned int *swapchainTextures[2],
-                      int swapchainLength) {
+                      int swapchainLength,
+                      bool enable_srgb_correction) {
     for (int eye = 0; eye < 2; eye++) {
         g_ctx.lobbySwapchainTextures[eye].clear();
 
@@ -811,6 +813,7 @@ void prepareLobbyRoom(int viewWidth,
                        g_ctx.lobbySwapchainTextures,
                        {false},
                        true,
+                       enable_srgb_correction,
                        false);
 }
 
@@ -857,7 +860,8 @@ void streamStartNative(FfiStreamConfig config) {
                         config.foveationEdgeRatioX,
                         config.foveationEdgeRatioY},
                        false,
-                       config.enableSrgbCorrection);
+                       config.enableSrgbCorrection,
+                       config.fixLimitedRange);
 }
 
 void updateLobbyHudTexture(const unsigned char *data) {

alvr/client_core/cpp/srgb_correction_pass.cpp

@@ -31,8 +31,10 @@ const string CORRECTION_FRAGMENT_SHADER = R"glsl(
         {
             color = texture(tex0, uv);
 
+#ifdef LIMITED_RANGE_BUG
             // For some reason, the encoder shifts full-range color into the negatives and over one.
             color.rgb = LIMITED_MIN + ((LIMITED_MAX - LIMITED_MIN) * color.rgb);
+#endif
 #ifdef SRGB_CORRECTION
             vec3 condition = vec3(color.r < THRESHOLD, color.g < THRESHOLD, color.b < THRESHOLD);
             vec3 lowValues = color.rgb * DIV12;
@@ -45,11 +47,14 @@ const string CORRECTION_FRAGMENT_SHADER = R"glsl(
 
 SrgbCorrectionPass::SrgbCorrectionPass(Texture *inputSurface) : mInputSurface(inputSurface) {}
 
-void SrgbCorrectionPass::Initialize(uint32_t width, uint32_t height, bool passthrough) {
+void SrgbCorrectionPass::Initialize(uint32_t width, uint32_t height, bool passthrough, bool fix_limited_range) {
     mOutputTexture.reset(new Texture(false, 0, false, width * 2, height));
     mOutputTextureState = make_unique<RenderState>(mOutputTexture.get());
 
     string defines = passthrough ? "" : "#define SRGB_CORRECTION";
+    if (fix_limited_range) {
+        defines += "\n#define LIMITED_RANGE_BUG";
+    }
 
     auto fragmentShader = CORRECTION_FRAGMENT_SHADER_HEADER + "\n" + defines + "\n" + CORRECTION_FRAGMENT_SHADER;
     mStagingPipeline = unique_ptr<RenderPipeline>(

alvr/client_core/cpp/srgb_correction_pass.h

@@ -9,7 +9,7 @@ class SrgbCorrectionPass {
   public:
     SrgbCorrectionPass(gl_render_utils::Texture *inputSurface);
 
-    void Initialize(uint32_t width, uint32_t height, bool passthrough);
+    void Initialize(uint32_t width, uint32_t height, bool passthrough, bool fix_limited_range);
 
     void Render() const;
 

alvr/client_core/src/c_api.rs

@@ -655,10 +655,12 @@ pub unsafe extern "C" fn alvr_resume_opengl(
     preferred_view_height: u32,
     swapchain_textures: *mut *const u32,
     swapchain_length: u32,
+    enable_srgb_correction: bool,
 ) {
     opengl::initialize_lobby(
         UVec2::new(preferred_view_width, preferred_view_height),
         convert_swapchain_array(swapchain_textures, swapchain_length),
+        enable_srgb_correction,
     );
 }
 
@@ -687,7 +689,13 @@ pub unsafe extern "C" fn alvr_start_stream_opengl(config: AlvrStreamConfig) {
         edge_ratio_y: config.foveation_edge_ratio_y,
     });
 
-    opengl::start_stream(view_resolution, swapchain_textures, foveated_encoding, true);
+    opengl::start_stream(
+        view_resolution,
+        swapchain_textures,
+        foveated_encoding,
+        true,
+        false,
+    );
 }
 
 #[no_mangle]

alvr/client_core/src/opengl.rs

@@ -42,7 +42,11 @@ pub fn destroy() {
     }
 }
 
-pub fn initialize_lobby(preferred_view_resolution: UVec2, swapchain_textures: [Vec<u32>; 2]) {
+pub fn initialize_lobby(
+    preferred_view_resolution: UVec2,
+    swapchain_textures: [Vec<u32>; 2],
+    enable_srgb_correction: bool,
+) {
     #[cfg(target_os = "android")]
     unsafe {
         let swapchain_length = swapchain_textures[0].len();
@@ -56,6 +60,7 @@ pub fn initialize_lobby(preferred_view_resolution: UVec2, swapchain_textures: [V
             preferred_view_resolution.y as _,
             swapchain_textures.as_mut_ptr(),
             swapchain_length as _,
+            enable_srgb_correction,
         );
     }
 }
@@ -72,6 +77,7 @@ pub fn start_stream(
     swapchain_textures: [Vec<u32>; 2],
     foveated_encoding: Option<FoveatedEncodingConfig>,
     enable_srgb_correction: bool,
+    fix_limited_range: bool,
 ) {
     #[cfg(target_os = "android")]
     unsafe {
@@ -109,6 +115,7 @@ pub fn start_stream(
                 .map(|f| f.edge_ratio_y)
                 .unwrap_or_default(),
             enableSrgbCorrection: enable_srgb_correction as u32,
+            fixLimitedRange: fix_limited_range as u32,
         };
 
         streamStartNative(config);

alvr/client_openxr/src/graphics.rs

@@ -101,7 +101,7 @@ pub fn create_swapchain(
     let swapchain_info = xr::SwapchainCreateInfo {
         create_flags: xr::SwapchainCreateFlags::EMPTY,
         usage_flags: xr::SwapchainUsageFlags::COLOR_ATTACHMENT | xr::SwapchainUsageFlags::SAMPLED,
-        format: glow::SRGB8_ALPHA8,
+        format: glow::RGBA16F,
         sample_count: 1,
         width: resolution.x,
         height: resolution.y,

alvr/client_openxr/src/lib.rs

@@ -20,6 +20,7 @@ use std::{
     time::{Duration, Instant},
 };
 use stream::StreamContext;
+use xr::ColorSpaceFB;
 
 const DECODER_MAX_TIMEOUT_MULTIPLIER: f32 = 0.8;
 
@@ -188,6 +189,10 @@ pub fn entry_point() {
             vec![90.0]
         };
 
+        if exts.fb_color_space {
+            xr_session.set_color_space(ColorSpaceFB::P3).unwrap();
+        }
+
         let capabilities = ClientCapabilities {
             default_view_resolution,
             external_decoder: false,

alvr/client_openxr/src/lobby.rs

@@ -39,6 +39,7 @@ impl Lobby {
                     .map(|i| *i as _)
                     .collect(),
             ],
+            false, // TODO: correct lobby sRGB for some headsets
         );
 
         Self {

alvr/client_openxr/src/stream.rs

@@ -11,7 +11,7 @@ use alvr_common::{
 };
 use alvr_packets::{FaceData, NegotiatedStreamingConfig, Tracking};
 use alvr_session::{
-    BodyTrackingSourcesConfig, ClientsideFoveationConfig, ClientsideFoveationMode,
+    BodyTrackingSourcesConfig, ClientsideFoveationConfig, ClientsideFoveationMode, EncoderConfig,
     FaceTrackingSourcesConfig, FoveatedEncodingConfig, Settings,
 };
 use openxr as xr;
@@ -33,6 +33,7 @@ pub struct StreamConfig {
     pub refresh_rate_hint: f32,
     pub foveated_encoding_config: Option<FoveatedEncodingConfig>,
     pub clientside_foveation_config: Option<ClientsideFoveationConfig>,
+    pub encoder_config: EncoderConfig,
     pub face_sources_config: Option<FaceTrackingSourcesConfig>,
     pub body_sources_config: Option<BodyTrackingSourcesConfig>,
 }
@@ -47,6 +48,7 @@ impl StreamConfig {
                 .then(|| settings.video.foveated_encoding.as_option().cloned())
                 .flatten(),
             clientside_foveation_config: settings.video.clientside_foveation.as_option().cloned(),
+            encoder_config: settings.video.encoder_config.clone(),
             face_sources_config: settings
                 .headset
                 .face_tracking
@@ -187,7 +189,10 @@ impl StreamContext {
                     .collect(),
             ],
             config.foveated_encoding_config.clone(),
-            platform != Platform::Lynx,
+            platform != Platform::Lynx
+                && platform != Platform::Pico4
+                && platform != Platform::PicoNeo3,
+            config.encoder_config.enable_hdr != true,
         );
 
         core_ctx.send_playspace(

alvr/server/cpp/alvr_server/Settings.cpp

@@ -66,6 +66,9 @@ void Settings::Load() {
         m_entropyCoding = (uint32_t)config.get("entropy_coding").get<int64_t>();
         m_use10bitEncoder = config.get("use_10bit_encoder").get<bool>();
         m_useFullRangeEncoding = config.get("use_full_range_encoding").get<bool>();
+        m_enableHdr = config.get("enable_hdr").get<bool>();
+        m_forceHdrSrgbCorrection = config.get("force_hdr_srgb_correction").get<bool>();
+        m_clampHdrExtendedRange = config.get("clamp_hdr_extended_range").get<bool>();
         m_enablePreAnalysis = config.get("enable_pre_analysis").get<bool>();
         m_enableVbaq = config.get("enable_vbaq").get<bool>();
         m_enableHmqb = config.get("enable_hmqb").get<bool>();

alvr/server/cpp/alvr_server/Settings.h

@@ -43,6 +43,9 @@ class Settings {
     int m_h264Profile;
     bool m_use10bitEncoder;
     bool m_useFullRangeEncoding;
+    bool m_enableHdr;
+    bool m_forceHdrSrgbCorrection;
+    bool m_clampHdrExtendedRange;
     bool m_enablePreAnalysis;
     bool m_enableVbaq;
     bool m_enableHmqb;

alvr/server/cpp/alvr_server/alvr_server.cpp

@@ -255,6 +255,8 @@ const unsigned char *COMPRESS_AXIS_ALIGNED_CSO_PTR;
 unsigned int COMPRESS_AXIS_ALIGNED_CSO_LEN;
 const unsigned char *COLOR_CORRECTION_CSO_PTR;
 unsigned int COLOR_CORRECTION_CSO_LEN;
+const unsigned char *RGBTOYUV420_CSO_PTR;
+unsigned int RGBTOYUV420_CSO_LEN;
 
 const unsigned char *QUAD_SHADER_COMP_SPV_PTR;
 unsigned int QUAD_SHADER_COMP_SPV_LEN;

alvr/server/cpp/alvr_server/bindings.h

@@ -94,6 +94,9 @@ extern "C" const unsigned char *COMPRESS_AXIS_ALIGNED_CSO_PTR;
 extern "C" unsigned int COMPRESS_AXIS_ALIGNED_CSO_LEN;
 extern "C" const unsigned char *COLOR_CORRECTION_CSO_PTR;
 extern "C" unsigned int COLOR_CORRECTION_CSO_LEN;
+extern "C" const unsigned char *RGBTOYUV420_CSO_PTR;
+extern "C" unsigned int RGBTOYUV420_CSO_LEN;
+
 
 extern "C" const unsigned char *QUAD_SHADER_COMP_SPV_PTR;
 extern "C" unsigned int QUAD_SHADER_COMP_SPV_LEN;

alvr/server/cpp/alvr_server/shader/FrameRender.fx

@@ -15,6 +15,32 @@ struct PS_INPUT
 	float2 Tex : TEXCOORD;
 	uint    View : VIEW;
 };
+
+float4 LinearToSRGB(float4 linearRGB)
+{
+	float4 sRGB;
+
+	// Apply sRGB transfer function to each channel
+	sRGB.r = (linearRGB.r <= 0.0031308) ? (linearRGB.r * 12.92) : (1.055 * pow(linearRGB.r, 1.0 / 2.4) - 0.055);
+	sRGB.g = (linearRGB.g <= 0.0031308) ? (linearRGB.g * 12.92) : (1.055 * pow(linearRGB.g, 1.0 / 2.4) - 0.055);
+	sRGB.b = (linearRGB.b <= 0.0031308) ? (linearRGB.b * 12.92) : (1.055 * pow(linearRGB.b, 1.0 / 2.4) - 0.055);
+	sRGB.a = (linearRGB.a <= 0.0031308) ? (linearRGB.a * 12.92) : (1.055 * pow(linearRGB.a, 1.0 / 2.4) - 0.055);
+
+	return sRGB;
+}
+
+float4 sRGBToLinear(float4 color)
+{
+    float4 linearColor;
+
+    linearColor.r = (color.r <= 0.04045) ? (color.r / 12.92) : pow((color.r + 0.055) / 1.055, 2.4);
+    linearColor.g = (color.g <= 0.04045) ? (color.g / 12.92) : pow((color.g + 0.055) / 1.055, 2.4);
+    linearColor.b = (color.b <= 0.04045) ? (color.b / 12.92) : pow((color.b + 0.055) / 1.055, 2.4);
+	linearColor.a = (color.a <= 0.04045) ? (color.a / 12.92) : pow((color.a + 0.055) / 1.055, 2.4);
+
+    return linearColor;
+}
+
 PS_INPUT VS(VS_INPUT input)
 {
 	PS_INPUT output = (PS_INPUT)0;
@@ -26,10 +52,29 @@ PS_INPUT VS(VS_INPUT input)
 }
 float4 PS(PS_INPUT input) : SV_Target
 {
-	if (input.View == (uint)0) { // Left View
-		return txLeft.Sample(samLinear, input.Tex);
+	float4 color = float4(1.0, 0.0, 0.0, 1.0);
+	uint correctionType = (input.View >> 1) & 0xF;
+	uint shouldClamp = (input.View >> 5);
+
+	if ((input.View & 1) == 1) {
+		color = txRight.Sample(samLinear, input.Tex);
+	}
+	else {
+		color = txLeft.Sample(samLinear, input.Tex);
 	}
-	else { // Right View
-		return txRight.Sample(samLinear, input.Tex);
+
+	if (shouldClamp == (uint)1) {
+		color = clamp(color, 0.0, 1.0);
 	}
+	if (correctionType == (uint)1) { // Left View sRGB
+		color = LinearToSRGB(color);
+	}
+	else if (correctionType == (uint)2) { // Left View non-HDR linear
+		color = sRGBToLinear(color);
+	}
+	if (shouldClamp == (uint)2) {
+		color = clamp(color, 0.0, 1.0);
+	}
+
+	return color;
 };