From 4693502b2d3bf9601b0acdb09309057bc8bcf1f7 Mon Sep 17 00:00:00 2001 From: Riccardo Zaglia Date: Sat, 25 Jan 2025 18:38:53 +0100 Subject: [PATCH] feat: :sparkles: Add chroma key support; make passthrough settings real time --- Cargo.lock | 1 + alvr/client_core/src/c_api.rs | 5 +- alvr/client_core/src/connection.rs | 9 ++ alvr/client_core/src/lib.rs | 5 +- alvr/client_mock/src/main.rs | 1 + alvr/client_openxr/src/lib.rs | 5 + alvr/client_openxr/src/stream.rs | 13 ++- alvr/graphics/resources/stream.wgsl | 139 +++++++++++++++++++++------ alvr/graphics/src/lib.rs | 2 +- alvr/graphics/src/stream.rs | 143 ++++++++++++++++++++++++---- alvr/packets/Cargo.toml | 1 + alvr/packets/src/lib.rs | 19 +++- alvr/server_core/src/connection.rs | 29 +++++- alvr/session/src/settings.rs | 51 +++++++++- 14 files changed, 364 insertions(+), 59 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 653e395f98..9283fa4997 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -365,6 +365,7 @@ version = "21.0.0-dev01" dependencies = [ "alvr_common", "alvr_session", + "bincode", "serde", "serde_json", ] diff --git a/alvr/client_core/src/c_api.rs b/alvr/client_core/src/c_api.rs index 0619908a49..b20e5fca0c 100644 --- a/alvr/client_core/src/c_api.rs +++ b/alvr/client_core/src/c_api.rs @@ -81,6 +81,8 @@ pub enum AlvrEvent { DecoderConfig { codec: AlvrCodec, }, + // Unimplemented + RealTimeConfig {}, } #[repr(C)] @@ -373,6 +375,7 @@ pub extern "C" fn alvr_poll_event(out_event: *mut AlvrEvent) -> bool { }, } } + ClientCoreEvent::RealTimeConfig(_) => AlvrEvent::RealTimeConfig {}, }; unsafe { *out_event = event }; @@ -792,7 +795,6 @@ pub unsafe extern "C" fn alvr_start_stream_opengl(config: AlvrStreamConfig) { true, false, // TODO: limited range fix config 1.0, // TODO: encoding gamma config - None, // TODO: passthrough config ))); } @@ -852,6 +854,7 @@ pub unsafe extern "C" fn alvr_render_stream_opengl( fov: from_capi_fov(right_params.fov), }, ], + None, ); } }); diff --git a/alvr/client_core/src/connection.rs b/alvr/client_core/src/connection.rs index 3c3e8db1a9..9ac914e979 100644 --- a/alvr/client_core/src/connection.rs +++ b/alvr/client_core/src/connection.rs @@ -492,6 +492,15 @@ fn connection_pipeline( set_hud_message(&event_queue, SERVER_RESTART_MESSAGE); disconnect_notif.notify_one(); } + Ok(ServerControlPacket::ReservedBuffer(buffer)) => { + // NB: it's nrmal for deserialization to fail if server has different + // version + if let Ok(config) = alvr_packets::decode_real_time_config(&buffer) { + event_queue + .lock() + .push_back(ClientCoreEvent::RealTimeConfig(config)); + } + } Ok(_) => (), Err(ConnectionError::TryAgain(_)) => { if Instant::now() > disconnection_deadline { diff --git a/alvr/client_core/src/lib.rs b/alvr/client_core/src/lib.rs index d687355b22..946ec14102 100644 --- a/alvr/client_core/src/lib.rs +++ b/alvr/client_core/src/lib.rs @@ -25,8 +25,8 @@ use alvr_common::{ HEAD_ID, }; use alvr_packets::{ - BatteryInfo, ButtonEntry, ClientControlPacket, FaceData, ReservedClientControlPacket, - StreamConfig, Tracking, ViewParams, ViewsConfig, + BatteryInfo, ButtonEntry, ClientControlPacket, FaceData, RealTimeConfig, + ReservedClientControlPacket, StreamConfig, Tracking, ViewParams, ViewsConfig, }; use alvr_session::CodecType; use connection::{ConnectionContext, DecoderCallback}; @@ -55,6 +55,7 @@ pub enum ClientCoreEvent { codec: CodecType, config_nal: Vec, }, + RealTimeConfig(RealTimeConfig), } // Note: this struct may change without breaking network protocol changes diff --git a/alvr/client_mock/src/main.rs b/alvr/client_mock/src/main.rs index 4aa714eb0f..fbc47ca90e 100644 --- a/alvr/client_mock/src/main.rs +++ b/alvr/client_mock/src/main.rs @@ -273,6 +273,7 @@ fn client_thread( window_output.decoder_codec = Some(codec); } + ClientCoreEvent::RealTimeConfig(_) => (), } output_sender.send(window_output.clone()).ok(); diff --git a/alvr/client_openxr/src/lib.rs b/alvr/client_openxr/src/lib.rs index 719be4256d..639d92b528 100644 --- a/alvr/client_openxr/src/lib.rs +++ b/alvr/client_openxr/src/lib.rs @@ -423,6 +423,11 @@ pub fn entry_point() { stream.maybe_initialize_decoder(codec, config_nal); } } + ClientCoreEvent::RealTimeConfig(config) => { + if let Some(stream) = &mut stream_context { + stream.update_real_time_config(&config); + } + } } } diff --git a/alvr/client_openxr/src/stream.rs b/alvr/client_openxr/src/stream.rs index e789b886ec..f61e3e4ee9 100644 --- a/alvr/client_openxr/src/stream.rs +++ b/alvr/client_openxr/src/stream.rs @@ -14,7 +14,7 @@ use alvr_common::{ Pose, RelaxedAtomic, HAND_LEFT_ID, HAND_RIGHT_ID, HEAD_ID, }; use alvr_graphics::{GraphicsContext, StreamRenderer, StreamViewParams}; -use alvr_packets::{FaceData, StreamConfig, ViewParams}; +use alvr_packets::{FaceData, RealTimeConfig, StreamConfig, ViewParams}; use alvr_session::{ ClientsideFoveationConfig, ClientsideFoveationMode, CodecType, FoveatedEncodingConfig, MediacodecProperty, PassthroughMode, @@ -185,7 +185,6 @@ impl StreamContext { platform != Platform::Lynx && !((platform.is_pico()) && config.enable_hdr), config.use_full_range && !config.enable_hdr, // TODO: figure out why HDR doesn't need the limited range hackfix in staging? config.encoding_gamma, - config.passthrough.clone(), ); core_ctx.send_active_interaction_profile( @@ -311,6 +310,10 @@ impl StreamContext { } } + pub fn update_real_time_config(&mut self, config: &RealTimeConfig) { + self.config.passthrough = config.passthrough.clone(); + } + pub fn render( &mut self, frame_interval: Duration, @@ -368,6 +371,7 @@ impl StreamContext { fov: view_params[1].fov, }, ], + self.config.passthrough.as_ref(), ) }; @@ -417,7 +421,10 @@ impl StreamContext { .passthrough .clone() .map(|mode| ProjectionLayerAlphaConfig { - premultiplied: !matches!(mode, PassthroughMode::Blend { .. }), + premultiplied: matches!( + mode, + PassthroughMode::AugmentedReality { .. } | PassthroughMode::ChromaKey(_) + ), }), ); diff --git a/alvr/graphics/resources/stream.wgsl b/alvr/graphics/resources/stream.wgsl index f5f3d99c64..7be3c062ce 100644 --- a/alvr/graphics/resources/stream.wgsl +++ b/alvr/graphics/resources/stream.wgsl @@ -9,37 +9,43 @@ override ENCODING_GAMMA: f32; override ENABLE_FFE: bool = false; -override VIEW_WIDTH_RATIO: f32 = 0.; -override VIEW_HEIGHT_RATIO: f32 = 0.; -override EDGE_X_RATIO: f32 = 0.; -override EDGE_Y_RATIO: f32 = 0.; - -override C1_X: f32 = 0.; -override C1_Y: f32 = 0.; -override C2_X: f32 = 0.; -override C2_Y: f32 = 0.; -override LO_BOUND_X: f32 = 0.; -override LO_BOUND_Y: f32 = 0.; -override HI_BOUND_X: f32 = 0.; -override HI_BOUND_Y: f32 = 0.; - -override A_LEFT_X: f32 = 0.; -override A_LEFT_Y: f32 = 0.; -override B_LEFT_X: f32 = 0.; -override B_LEFT_Y: f32 = 0.; - -override A_RIGHT_X: f32 = 0.; -override A_RIGHT_Y: f32 = 0.; -override B_RIGHT_X: f32 = 0.; -override B_RIGHT_Y: f32 = 0.; -override C_RIGHT_X: f32 = 0.; -override C_RIGHT_Y: f32 = 0.; - -override COLOR_ALPHA: f32 = 1.0; +override VIEW_WIDTH_RATIO: f32 = 0.0; +override VIEW_HEIGHT_RATIO: f32 = 0.0; +override EDGE_X_RATIO: f32 = 0.0; +override EDGE_Y_RATIO: f32 = 0.0; + +override C1_X: f32 = 0.0; +override C1_Y: f32 = 0.0; +override C2_X: f32 = 0.0; +override C2_Y: f32 = 0.0; +override LO_BOUND_X: f32 = 0.0; +override LO_BOUND_Y: f32 = 0.0; +override HI_BOUND_X: f32 = 0.0; +override HI_BOUND_Y: f32 = 0.0; + +override A_LEFT_X: f32 = 0.0; +override A_LEFT_Y: f32 = 0.0; +override B_LEFT_X: f32 = 0.0; +override B_LEFT_Y: f32 = 0.0; + +override A_RIGHT_X: f32 = 0.0; +override A_RIGHT_Y: f32 = 0.0; +override B_RIGHT_X: f32 = 0.0; +override B_RIGHT_Y: f32 = 0.0; +override C_RIGHT_X: f32 = 0.0; +override C_RIGHT_Y: f32 = 0.0; struct PushConstant { reprojection_transform: mat4x4f, view_idx: u32, + alpha: f32, + enable_chroma_key: u32, + _pad1: u32, + ck_target_hsv: vec3f, + _pad2: u32, + ck_weights: vec3f, + ck_feather_max: f32, + ck_feather_min: f32, } var pc: PushConstant; @@ -127,5 +133,82 @@ fn fragment_main(@location(0) uv: vec2f) -> @location(0) vec4f { color = enc_condition * enc_lowValues + (1.0 - enc_condition) * enc_highValues; } - return vec4f(color, COLOR_ALPHA); + var alpha = pc.alpha; + if pc.enable_chroma_key == 1 { + let mask = chroma_key_alpha(rgb_to_hsv(color)); + let target_rgb = hsv_to_rgb(pc.ck_target_hsv); + + // Note: because of this calculation, we require premultiplied alpha option in the XR layer + color = max(color * mask - (target_rgb * (1.0 - mask)), vec3f(0.0)); + alpha = mask; + + // color = target_rgb;//vec3f(1.0, alpha, 0.0); + } + + return vec4f(color, alpha); +} + +fn chroma_key_alpha(hsv: vec3f) -> f32 { + let weighted_distance = length(pc.ck_weights * hsv - pc.ck_target_hsv); + + if weighted_distance < pc.ck_feather_min { + return 0.0; + } else if weighted_distance < pc.ck_feather_max { + return (weighted_distance - pc.ck_feather_min) / (pc.ck_feather_max - pc.ck_feather_min); + } else { + return 1.0; + } +} + +fn rgb_to_hsv(rgb: vec3f) -> vec3f { + let cmax = max(rgb.r, max(rgb.g, rgb.b)); + let cmin = min(rgb.r, min(rgb.g, rgb.b)); + let delta = cmax - cmin; + + var h = 0.0; + var s = 0.0; + let v = cmax; + + if cmax > cmin { + s = delta / cmax; + + if rgb.r == cmax { + h = (rgb.g - rgb.b) / delta; + } else if rgb.g == cmax { + h = 2.0 + (rgb.b - rgb.r) / delta; + } else { + h = 4.0 + (rgb.r - rgb.g) / delta; + } + h = fract(h / 6.0); + } + + return vec3f(h, s, v); +} + +// https://stackoverflow.com/questions/24852345/hsv-to-rgb-color-conversion +fn hsv_to_rgb(hsv: vec3f) -> vec3f { + var h = hsv.x; + let s = hsv.y; + let v = hsv.z; + + let i = i32(h * 6.0); + let f = fract(h * 6.0); + + let w = v * (1.0 - s); + let q = v * (1.0 - s * f); + let t = v * (1.0 - s * (1.0 - f)); + + if i == 0 { + return vec3f(v, t, w); + } else if i == 1 { + return vec3f(q, v, w); + } else if i == 2 { + return vec3f(w, v, t); + } else if i == 3 { + return vec3f(w, q, v); + } else if i == 4 { + return vec3f(t, w, v); + } else { + return vec3f(v, w, q); + } } diff --git a/alvr/graphics/src/lib.rs b/alvr/graphics/src/lib.rs index 7c2c122034..3210be05f4 100644 --- a/alvr/graphics/src/lib.rs +++ b/alvr/graphics/src/lib.rs @@ -21,7 +21,7 @@ use wgpu::{ pub const SDR_FORMAT: TextureFormat = TextureFormat::Rgba8Unorm; pub const SDR_FORMAT_GL: u32 = gl::RGBA8; pub const GL_TEXTURE_EXTERNAL_OES: u32 = 0x8D65; -pub const MAX_PUSH_CONSTANTS_SIZE: u32 = 72; +pub const MAX_PUSH_CONSTANTS_SIZE: u32 = 120; type CreateImageFn = unsafe extern "C" fn( egl::EGLDisplay, diff --git a/alvr/graphics/src/stream.rs b/alvr/graphics/src/stream.rs index 3ab8853da8..ae8a14283b 100644 --- a/alvr/graphics/src/stream.rs +++ b/alvr/graphics/src/stream.rs @@ -10,24 +10,42 @@ use wgpu::{ include_wgsl, BindGroup, BindGroupDescriptor, BindGroupEntry, BindGroupLayoutDescriptor, BindGroupLayoutEntry, BindingResource, BindingType, Color, ColorTargetState, ColorWrites, FragmentState, LoadOp, PipelineCompilationOptions, PipelineLayoutDescriptor, PrimitiveState, - PrimitiveTopology, PushConstantRange, RenderPassColorAttachment, RenderPassDescriptor, - RenderPipeline, RenderPipelineDescriptor, SamplerBindingType, SamplerDescriptor, ShaderStages, - StoreOp, TextureSampleType, TextureView, TextureViewDescriptor, TextureViewDimension, - VertexState, + PrimitiveTopology, PushConstantRange, RenderPass, RenderPassColorAttachment, + RenderPassDescriptor, RenderPipeline, RenderPipelineDescriptor, SamplerBindingType, + SamplerDescriptor, ShaderStages, StoreOp, TextureSampleType, TextureView, + TextureViewDescriptor, TextureViewDimension, VertexState, }; -const TRANSFORM_CONST_SIZE: u32 = mem::size_of::() as u32; -const VIEW_INDEX_CONST_SIZE: u32 = mem::size_of::() as u32; +const FLOAT_SIZE: u32 = mem::size_of::() as u32; +const U32_SIZE: u32 = mem::size_of::() as u32; +const PAD4_SIZE: u32 = 4; +const VEC3_SIZE: u32 = mem::size_of::() as u32; +const TRANSFORM_SIZE: u32 = mem::size_of::() as u32; -const PUSH_CONSTANTS_SIZE: u32 = TRANSFORM_CONST_SIZE + VIEW_INDEX_CONST_SIZE; +const TRANSFORM_CONST_OFFSET: u32 = 0; +const VIEW_INDEX_CONST_OFFSET: u32 = TRANSFORM_SIZE; +const ALPHA_CONST_OFFSET: u32 = VIEW_INDEX_CONST_OFFSET + U32_SIZE; +const ENABLE_CHROMA_KEY_CONST_OFFSET: u32 = ALPHA_CONST_OFFSET + FLOAT_SIZE; +const CK_TARGET_CONST_OFFSET: u32 = ENABLE_CHROMA_KEY_CONST_OFFSET + U32_SIZE + PAD4_SIZE; +const CK_WEIGHT_CONST_OFFSET: u32 = CK_TARGET_CONST_OFFSET + VEC3_SIZE + PAD4_SIZE; +const CK_FEATHER_MIN_CONST_OFFSET: u32 = CK_WEIGHT_CONST_OFFSET + VEC3_SIZE; +const CK_FEATHER_MAX_CONST_OFFSET: u32 = CK_FEATHER_MIN_CONST_OFFSET + FLOAT_SIZE; + +const PUSH_CONSTANTS_SIZE: u32 = TRANSFORM_SIZE + + U32_SIZE + + FLOAT_SIZE + + U32_SIZE + + PAD4_SIZE + + VEC3_SIZE + + PAD4_SIZE + + VEC3_SIZE + + FLOAT_SIZE + + FLOAT_SIZE; const _: () = assert!( PUSH_CONSTANTS_SIZE <= MAX_PUSH_CONSTANTS_SIZE, "Push constants size exceeds the maximum size" ); -const TRANSFORM_CONST_OFFSET: u32 = 0; -const VIEW_INDEX_CONST_OFFSET: u32 = TRANSFORM_CONST_SIZE; - pub struct StreamViewParams { pub swapchain_index: u32, pub reprojection_rotation: Quat, @@ -48,7 +66,6 @@ pub struct StreamRenderer { } impl StreamRenderer { - #[allow(clippy::too_many_arguments)] pub fn new( context: Rc, view_resolution: UVec2, @@ -58,7 +75,6 @@ impl StreamRenderer { enable_srgb_correction: bool, fix_limited_range: bool, encoding_gamma: f32, - passthrough: Option, ) -> Self { let device = &context.device; @@ -98,13 +114,19 @@ impl StreamRenderer { ("ENCODING_GAMMA".into(), encoding_gamma.into()), ]); - if let Some(mode) = passthrough { - let ps_alpha = match mode { - PassthroughMode::AugmentedReality { brightness } => brightness, - PassthroughMode::Blend { opacity } => opacity, - }; - constants.extend([("COLOR_ALPHA".into(), (1. - ps_alpha).into())]); - } + // if let Some(mode) = passthrough { + // match mode { + // PassthroughMode::AugmentedReality { brightness } => { + // constants.extend([("COLOR_ALPHA".into(), (1. - brightness).into())]) + // } + // PassthroughMode::Blend { opacity } => { + // constants.extend([("COLOR_ALPHA".into(), (1. - opacity).into())]) + // } + // PassthroughMode::ChromaKey(config) => { + // constants.extend(chroma_key_constants(config)); + // } + // } + // } let staging_resolution = if let Some(foveated_encoding) = foveated_encoding { let (staging_resolution, ffe_constants) = @@ -224,7 +246,12 @@ impl StreamRenderer { } } - pub unsafe fn render(&self, hardware_buffer: *mut c_void, view_params: [StreamViewParams; 2]) { + pub unsafe fn render( + &self, + hardware_buffer: *mut c_void, + view_params: [StreamViewParams; 2], + passthrough: Option<&PassthroughMode>, + ) { // if hardware_buffer is available copy stream to staging texture if !hardware_buffer.is_null() { self.staging_renderer.render(hardware_buffer); @@ -287,6 +314,7 @@ impl StreamRenderer { &(view_idx as u32).to_le_bytes(), ); render_pass.set_bind_group(0, &self.views_objects[view_idx].bind_group, &[]); + set_passthrough_push_constants(&mut render_pass, passthrough); render_pass.draw(0..4, 0..1); } @@ -294,6 +322,81 @@ impl StreamRenderer { } } +fn set_passthrough_push_constants(render_pass: &mut RenderPass, config: Option<&PassthroughMode>) { + const DEG_TO_NORM: f32 = 1. / 360.; + + alvr_common::error!("{:?}", config); + + fn set_float(render_pass: &mut RenderPass, offset: u32, value: f32) { + render_pass.set_push_constants(ShaderStages::VERTEX_FRAGMENT, offset, &value.to_le_bytes()); + } + + match config { + Some(PassthroughMode::AugmentedReality { brightness }) => { + set_float(render_pass, ALPHA_CONST_OFFSET, 1. - brightness); + set_float(render_pass, ENABLE_CHROMA_KEY_CONST_OFFSET, 0.); + } + Some(PassthroughMode::Blend { opacity }) => { + set_float(render_pass, ALPHA_CONST_OFFSET, 1. - opacity); + set_float(render_pass, ENABLE_CHROMA_KEY_CONST_OFFSET, 0.); + } + Some(PassthroughMode::ChromaKey(config)) => { + render_pass.set_push_constants( + ShaderStages::VERTEX_FRAGMENT, + ENABLE_CHROMA_KEY_CONST_OFFSET, + &1_u32.to_le_bytes(), + ); + + set_float( + render_pass, + CK_TARGET_CONST_OFFSET, + config.hue_deg * DEG_TO_NORM, + ); + set_float( + render_pass, + CK_TARGET_CONST_OFFSET + FLOAT_SIZE, + config.saturation, + ); + set_float( + render_pass, + CK_TARGET_CONST_OFFSET + 2 * FLOAT_SIZE, + config.value, + ); + + set_float( + render_pass, + CK_WEIGHT_CONST_OFFSET, + 1. / (config.hue_range_deg * DEG_TO_NORM), + ); + set_float( + render_pass, + CK_WEIGHT_CONST_OFFSET + FLOAT_SIZE, + 1. / config.saturation_range, + ); + set_float( + render_pass, + CK_WEIGHT_CONST_OFFSET + 2 * FLOAT_SIZE, + 1. / config.value_range, + ); + + set_float( + render_pass, + CK_FEATHER_MIN_CONST_OFFSET, + 1. - config.feathering_range, + ); + set_float( + render_pass, + CK_FEATHER_MAX_CONST_OFFSET, + 1. + config.feathering_range, + ); + } + None => { + set_float(render_pass, ALPHA_CONST_OFFSET, 1.0); + set_float(render_pass, ENABLE_CHROMA_KEY_CONST_OFFSET, 0.); + } + } +} + pub fn foveated_encoding_shader_constants( expanded_view_resolution: UVec2, config: FoveatedEncodingConfig, diff --git a/alvr/packets/Cargo.toml b/alvr/packets/Cargo.toml index 1b7a538e37..1427175af9 100644 --- a/alvr/packets/Cargo.toml +++ b/alvr/packets/Cargo.toml @@ -10,5 +10,6 @@ license.workspace = true alvr_common.workspace = true alvr_session.workspace = true +bincode = "1" serde = { version = "1", features = ["derive"] } serde_json = "1" diff --git a/alvr/packets/src/lib.rs b/alvr/packets/src/lib.rs index 5c492f5bc9..e03bd49606 100644 --- a/alvr/packets/src/lib.rs +++ b/alvr/packets/src/lib.rs @@ -4,7 +4,7 @@ use alvr_common::{ semver::Version, ConnectionState, DeviceMotion, Fov, LogEntry, LogSeverity, Pose, ToAny, }; -use alvr_session::{CodecType, SessionConfig, Settings}; +use alvr_session::{CodecType, PassthroughMode, SessionConfig, Settings}; use serde::{Deserialize, Serialize}; use serde_json as json; use std::{ @@ -406,6 +406,23 @@ pub enum ServerRequest { ShutdownSteamvr, } +// Note: server sends a packet to the client at low frequency, binary encoding, without ensuring +// compatibility between different versions, even if within the same major version. +#[derive(Serialize, Deserialize)] +pub struct RealTimeConfig { + pub passthrough: Option, +} + +pub fn encode_real_time_config(config: &RealTimeConfig) -> Result { + Ok(ServerControlPacket::ReservedBuffer(bincode::serialize( + config, + )?)) +} + +pub fn decode_real_time_config(buffer: &[u8]) -> Result { + Ok(bincode::deserialize(buffer)?) +} + // Per eye view parameters // todo: send together with video frame #[derive(Serialize, Deserialize, Clone, Copy, Default)] diff --git a/alvr/server_core/src/connection.rs b/alvr/server_core/src/connection.rs index 9c022629ba..db2f9c03c4 100644 --- a/alvr/server_core/src/connection.rs +++ b/alvr/server_core/src/connection.rs @@ -20,8 +20,8 @@ use alvr_common::{ use alvr_events::{AdbEvent, ButtonEvent, EventType}; use alvr_packets::{ BatteryInfo, ClientConnectionResult, ClientControlPacket, ClientListAction, ClientStatistics, - NegotiatedStreamingConfig, ReservedClientControlPacket, ServerControlPacket, Tracking, - VideoPacketHeader, AUDIO, HAPTICS, STATISTICS, TRACKING, VIDEO, + NegotiatedStreamingConfig, RealTimeConfig, ReservedClientControlPacket, ServerControlPacket, + Tracking, VideoPacketHeader, AUDIO, HAPTICS, STATISTICS, TRACKING, VIDEO, }; use alvr_session::{ BodyTrackingSinkConfig, CodecType, ControllersEmulationMode, FrameSize, H264Profile, @@ -43,6 +43,7 @@ use std::{ const RETRY_CONNECT_MIN_INTERVAL: Duration = Duration::from_secs(1); const HANDSHAKE_ACTION_TIMEOUT: Duration = Duration::from_secs(2); pub const STREAMING_RECV_TIMEOUT: Duration = Duration::from_millis(500); +const REAL_TIME_UPDATE_INTERVAL: Duration = Duration::from_secs(1); const MAX_UNREAD_PACKETS: usize = 10; // Applies per stream @@ -1069,6 +1070,29 @@ fn connection_pipeline( let control_sender = Arc::new(Mutex::new(control_sender)); + let real_time_update_thread = thread::spawn({ + let control_sender = Arc::clone(&control_sender); + let client_hostname = client_hostname.clone(); + move || { + while is_streaming(&client_hostname) { + let config = { + let session_manager_lock = SESSION_MANAGER.read(); + let settings = session_manager_lock.settings(); + + RealTimeConfig { + passthrough: settings.video.passthrough.clone().into_option(), + } + }; + + if let Ok(config) = alvr_packets::encode_real_time_config(&config) { + control_sender.lock().send(&config).ok(); + } + + thread::sleep(REAL_TIME_UPDATE_INTERVAL); + } + } + }); + let keepalive_thread = thread::spawn({ let control_sender = Arc::clone(&control_sender); let disconnect_notif = Arc::clone(&disconnect_notif); @@ -1436,6 +1460,7 @@ fn connection_pipeline( microphone_thread.join().ok(); tracking_receive_thread.join().ok(); statistics_thread.join().ok(); + real_time_update_thread.join().ok(); control_receive_thread.join().ok(); stream_receive_thread.join().ok(); keepalive_thread.join().ok(); diff --git a/alvr/session/src/settings.rs b/alvr/session/src/settings.rs index 8064ab4ef8..6a8b126b08 100644 --- a/alvr/session/src/settings.rs +++ b/alvr/session/src/settings.rs @@ -556,23 +556,63 @@ pub enum H264Profile { Baseline = 2, } -#[derive(SettingsSchema, Serialize, Deserialize, Clone, PartialEq)] +#[derive(SettingsSchema, Serialize, Deserialize, Clone, PartialEq, Debug)] +pub struct ChromaKeyConfig { + #[schema(strings(display_name = "Hue"), suffix = "°")] + #[schema(flag = "real-time")] + #[schema(gui(slider(min = 0.0, max = 360.0, step = 1.0)))] + pub hue_deg: f32, + + #[schema(strings(display_name = "Hue range"), suffix = "°")] + #[schema(flag = "real-time")] + #[schema(gui(slider(min = 0.0, max = 360.0, step = 1.0)))] + pub hue_range_deg: f32, + + #[schema(flag = "real-time")] + #[schema(gui(slider(min = 0.0, max = 1.0, step = 0.01)))] + pub saturation: f32, + + #[schema(flag = "real-time")] + #[schema(gui(slider(min = 0.0, max = 2.0, step = 0.01)))] + pub saturation_range: f32, + + #[schema(flag = "real-time")] + #[schema(gui(slider(min = 0.0, max = 1.0, step = 0.01)))] + pub value: f32, + + #[schema(flag = "real-time")] + #[schema(gui(slider(min = 0.0, max = 2.0, step = 0.01)))] + pub value_range: f32, + + #[schema(strings( + help = "Feathering applied to the combined normalized distance of every component" + ))] + #[schema(gui(slider(min = 0.0, max = 2.0, step = 0.01)))] + #[schema(flag = "real-time")] + pub feathering_range: f32, +} + +#[derive(SettingsSchema, Serialize, Deserialize, Clone, PartialEq, Debug)] #[schema(gui = "button_group")] pub enum PassthroughMode { AugmentedReality { + #[schema(flag = "real-time")] #[schema(gui(slider(min = 0.0, max = 1.0, step = 0.01)))] brightness: f32, }, Blend { + #[schema(flag = "real-time")] #[schema(gui(slider(min = 0.0, max = 1.0, step = 0.01)))] opacity: f32, }, + ChromaKey(#[schema(flag = "real-time")] ChromaKeyConfig), } #[derive(SettingsSchema, Serialize, Deserialize, Clone)] pub struct VideoConfig { #[schema(strings(help = r"Augmented reality: corresponds to premultiplied alpha Blend: corresponds to un-premultiplied alpha"))] + #[schema(flag = "real-time")] pub passthrough: Switch, pub bitrate: BitrateConfig, @@ -1422,6 +1462,15 @@ pub fn session_settings_default() -> SettingsDefault { variant: PassthroughModeDefaultVariant::AugmentedReality, AugmentedReality: PassthroughModeAugmentedRealityDefault { brightness: 0.4 }, Blend: PassthroughModeBlendDefault { opacity: 0.5 }, + ChromaKey: ChromaKeyConfigDefault { + hue_deg: 120.0, + hue_range_deg: 80.0, + saturation: 1.0, + saturation_range: 1.8, + value: 1.0, + value_range: 1.8, + feathering_range: 0.2, + }, }, }, adapter_index: 0,