feat: ✨ Add chroma key support; make passthrough settings real time

alvr/client_core/src/c_api.rs

@@ -81,6 +81,8 @@ pub enum AlvrEvent {
     DecoderConfig {
         codec: AlvrCodec,
     },
+    // Unimplemented
+    RealTimeConfig {},
 }
 
 #[repr(C)]
@@ -373,6 +375,7 @@ pub extern "C" fn alvr_poll_event(out_event: *mut AlvrEvent) -> bool {
                         },
                     }
                 }
+                ClientCoreEvent::RealTimeConfig(_) => AlvrEvent::RealTimeConfig {},
             };
 
             unsafe { *out_event = event };
@@ -792,7 +795,6 @@ pub unsafe extern "C" fn alvr_start_stream_opengl(config: AlvrStreamConfig) {
         true,
         false, // TODO: limited range fix config
         1.0,   // TODO: encoding gamma config
-        None,  // TODO: passthrough config
     )));
 }
 
@@ -852,6 +854,7 @@ pub unsafe extern "C" fn alvr_render_stream_opengl(
                         fov: from_capi_fov(right_params.fov),
                     },
                 ],
+                None,
             );
         }
     });

alvr/client_core/src/connection.rs

@@ -492,6 +492,15 @@ fn connection_pipeline(
                         set_hud_message(&event_queue, SERVER_RESTART_MESSAGE);
                         disconnect_notif.notify_one();
                     }
+                    Ok(ServerControlPacket::ReservedBuffer(buffer)) => {
+                        // NB: it's nrmal for deserialization to fail if server has different
+                        // version
+                        if let Ok(config) = alvr_packets::decode_real_time_config(&buffer) {
+                            event_queue
+                                .lock()
+                                .push_back(ClientCoreEvent::RealTimeConfig(config));
+                        }
+                    }
                     Ok(_) => (),
                     Err(ConnectionError::TryAgain(_)) => {
                         if Instant::now() > disconnection_deadline {

alvr/client_core/src/lib.rs

@@ -25,8 +25,8 @@ use alvr_common::{
     HEAD_ID,
 };
 use alvr_packets::{
-    BatteryInfo, ButtonEntry, ClientControlPacket, FaceData, ReservedClientControlPacket,
-    StreamConfig, Tracking, ViewParams, ViewsConfig,
+    BatteryInfo, ButtonEntry, ClientControlPacket, FaceData, RealTimeConfig,
+    ReservedClientControlPacket, StreamConfig, Tracking, ViewParams, ViewsConfig,
 };
 use alvr_session::CodecType;
 use connection::{ConnectionContext, DecoderCallback};
@@ -55,6 +55,7 @@ pub enum ClientCoreEvent {
         codec: CodecType,
         config_nal: Vec<u8>,
     },
+    RealTimeConfig(RealTimeConfig),
 }
 
 // Note: this struct may change without breaking network protocol changes

alvr/client_mock/src/main.rs

@@ -273,6 +273,7 @@ fn client_thread(
 
                     window_output.decoder_codec = Some(codec);
                 }
+                ClientCoreEvent::RealTimeConfig(_) => (),
             }
 
             output_sender.send(window_output.clone()).ok();

alvr/client_openxr/src/lib.rs

@@ -423,6 +423,11 @@ pub fn entry_point() {
                             stream.maybe_initialize_decoder(codec, config_nal);
                         }
                     }
+                    ClientCoreEvent::RealTimeConfig(config) => {
+                        if let Some(stream) = &mut stream_context {
+                            stream.update_real_time_config(&config);
+                        }
+                    }
                 }
             }
 

alvr/client_openxr/src/stream.rs

@@ -14,7 +14,7 @@ use alvr_common::{
     Pose, RelaxedAtomic, HAND_LEFT_ID, HAND_RIGHT_ID, HEAD_ID,
 };
 use alvr_graphics::{GraphicsContext, StreamRenderer, StreamViewParams};
-use alvr_packets::{FaceData, StreamConfig, ViewParams};
+use alvr_packets::{FaceData, RealTimeConfig, StreamConfig, ViewParams};
 use alvr_session::{
     ClientsideFoveationConfig, ClientsideFoveationMode, CodecType, FoveatedEncodingConfig,
     MediacodecProperty, PassthroughMode,
@@ -185,7 +185,6 @@ impl StreamContext {
             platform != Platform::Lynx && !((platform.is_pico()) && config.enable_hdr),
             config.use_full_range && !config.enable_hdr, // TODO: figure out why HDR doesn't need the limited range hackfix in staging?
             config.encoding_gamma,
-            config.passthrough.clone(),
         );
 
         core_ctx.send_active_interaction_profile(
@@ -311,6 +310,10 @@ impl StreamContext {
         }
     }
 
+    pub fn update_real_time_config(&mut self, config: &RealTimeConfig) {
+        self.config.passthrough = config.passthrough.clone();
+    }
+
     pub fn render(
         &mut self,
         frame_interval: Duration,
@@ -368,6 +371,7 @@ impl StreamContext {
                         fov: view_params[1].fov,
                     },
                 ],
+                self.config.passthrough.as_ref(),
             )
         };
 
@@ -417,7 +421,10 @@ impl StreamContext {
                 .passthrough
                 .clone()
                 .map(|mode| ProjectionLayerAlphaConfig {
-                    premultiplied: !matches!(mode, PassthroughMode::Blend { .. }),
+                    premultiplied: matches!(
+                        mode,
+                        PassthroughMode::AugmentedReality { .. } | PassthroughMode::ChromaKey(_)
+                    ),
                 }),
         );
 

alvr/graphics/resources/stream.wgsl

@@ -9,37 +9,43 @@ override ENCODING_GAMMA: f32;
 
 override ENABLE_FFE: bool = false;
 
-override VIEW_WIDTH_RATIO: f32 = 0.;
-override VIEW_HEIGHT_RATIO: f32 = 0.;
-override EDGE_X_RATIO: f32 = 0.;
-override EDGE_Y_RATIO: f32 = 0.;
-
-override C1_X: f32 = 0.;
-override C1_Y: f32 = 0.;
-override C2_X: f32 = 0.;
-override C2_Y: f32 = 0.;
-override LO_BOUND_X: f32 = 0.;
-override LO_BOUND_Y: f32 = 0.;
-override HI_BOUND_X: f32 = 0.;
-override HI_BOUND_Y: f32 = 0.;
-
-override A_LEFT_X: f32 = 0.;
-override A_LEFT_Y: f32 = 0.;
-override B_LEFT_X: f32 = 0.;
-override B_LEFT_Y: f32 = 0.;
-
-override A_RIGHT_X: f32 = 0.;
-override A_RIGHT_Y: f32 = 0.;
-override B_RIGHT_X: f32 = 0.;
-override B_RIGHT_Y: f32 = 0.;
-override C_RIGHT_X: f32 = 0.;
-override C_RIGHT_Y: f32 = 0.;
-
-override COLOR_ALPHA: f32 = 1.0;
+override VIEW_WIDTH_RATIO: f32 = 0.0;
+override VIEW_HEIGHT_RATIO: f32 = 0.0;
+override EDGE_X_RATIO: f32 = 0.0;
+override EDGE_Y_RATIO: f32 = 0.0;
+
+override C1_X: f32 = 0.0;
+override C1_Y: f32 = 0.0;
+override C2_X: f32 = 0.0;
+override C2_Y: f32 = 0.0;
+override LO_BOUND_X: f32 = 0.0;
+override LO_BOUND_Y: f32 = 0.0;
+override HI_BOUND_X: f32 = 0.0;
+override HI_BOUND_Y: f32 = 0.0;
+
+override A_LEFT_X: f32 = 0.0;
+override A_LEFT_Y: f32 = 0.0;
+override B_LEFT_X: f32 = 0.0;
+override B_LEFT_Y: f32 = 0.0;
+
+override A_RIGHT_X: f32 = 0.0;
+override A_RIGHT_Y: f32 = 0.0;
+override B_RIGHT_X: f32 = 0.0;
+override B_RIGHT_Y: f32 = 0.0;
+override C_RIGHT_X: f32 = 0.0;
+override C_RIGHT_Y: f32 = 0.0;
 
 struct PushConstant {
     reprojection_transform: mat4x4f,
     view_idx: u32,
+    alpha: f32,
+    enable_chroma_key: u32,
+    _pad1: u32,
+    ck_target_hsv: vec3f,
+    _pad2: u32,
+    ck_weights: vec3f,
+    ck_feather_max: f32,
+    ck_feather_min: f32,
 }
 var<push_constant> pc: PushConstant;
 
@@ -127,5 +133,82 @@ fn fragment_main(@location(0) uv: vec2f) -> @location(0) vec4f {
         color = enc_condition * enc_lowValues + (1.0 - enc_condition) * enc_highValues;
     }
 
-    return vec4f(color, COLOR_ALPHA);
+    var alpha = pc.alpha;
+    if pc.enable_chroma_key == 1 {
+        let mask = chroma_key_alpha(rgb_to_hsv(color));
+        let target_rgb = hsv_to_rgb(pc.ck_target_hsv);
+
+        // Note: because of this calculation, we require premultiplied alpha option in the XR layer
+        color = max(color * mask - (target_rgb * (1.0 - mask)), vec3f(0.0));
+        alpha = mask;
+
+        // color = target_rgb;//vec3f(1.0, alpha, 0.0);
+    }
+
+    return vec4f(color, alpha);
+}
+
+fn chroma_key_alpha(hsv: vec3f) -> f32 {
+    let weighted_distance = length(pc.ck_weights * hsv - pc.ck_target_hsv);
+
+    if weighted_distance < pc.ck_feather_min {
+        return 0.0;
+    } else if weighted_distance < pc.ck_feather_max {
+        return (weighted_distance - pc.ck_feather_min) / (pc.ck_feather_max - pc.ck_feather_min);
+    } else {
+        return 1.0;
+    }
+}
+
+fn rgb_to_hsv(rgb: vec3f) -> vec3f {
+    let cmax = max(rgb.r, max(rgb.g, rgb.b));
+    let cmin = min(rgb.r, min(rgb.g, rgb.b));
+    let delta = cmax - cmin;
+
+    var h = 0.0;
+    var s = 0.0;
+    let v = cmax;
+
+    if cmax > cmin {
+        s = delta / cmax;
+
+        if rgb.r == cmax {
+            h = (rgb.g - rgb.b) / delta;
+        } else if rgb.g == cmax {
+            h = 2.0 + (rgb.b - rgb.r) / delta;
+        } else {
+            h = 4.0 + (rgb.r - rgb.g) / delta;
+        }
+        h = fract(h / 6.0);
+    }
+
+    return vec3f(h, s, v);
+}
+
+// https://stackoverflow.com/questions/24852345/hsv-to-rgb-color-conversion
+fn hsv_to_rgb(hsv: vec3f) -> vec3f {
+    var h = hsv.x;
+    let s = hsv.y;
+    let v = hsv.z;
+
+    let i = i32(h * 6.0);
+    let f = fract(h * 6.0);
+
+    let w = v * (1.0 - s);
+    let q = v * (1.0 - s * f);
+    let t = v * (1.0 - s * (1.0 - f));
+
+    if i == 0 {
+        return vec3f(v, t, w);
+    } else if i == 1 {
+        return vec3f(q, v, w);
+    } else if i == 2 {
+        return vec3f(w, v, t);
+    } else if i == 3 {
+        return vec3f(w, q, v);
+    } else if i == 4 {
+        return vec3f(t, w, v);
+    } else {
+        return vec3f(v, w, q);
+    }
 }

alvr/graphics/src/lib.rs

@@ -21,7 +21,7 @@ use wgpu::{
 pub const SDR_FORMAT: TextureFormat = TextureFormat::Rgba8Unorm;
 pub const SDR_FORMAT_GL: u32 = gl::RGBA8;
 pub const GL_TEXTURE_EXTERNAL_OES: u32 = 0x8D65;
-pub const MAX_PUSH_CONSTANTS_SIZE: u32 = 72;
+pub const MAX_PUSH_CONSTANTS_SIZE: u32 = 120;
 
 type CreateImageFn = unsafe extern "C" fn(
     egl::EGLDisplay,