From 92a5bba96804cf63466acb9daea1ca84b4d920ea Mon Sep 17 00:00:00 2001 From: zarik5 Date: Tue, 3 Sep 2024 00:24:06 +0200 Subject: [PATCH] Multimodal input (#2367) * feat: :sparkles: Multimodal input * Fix controllers and hands dropping to 0,0,0 when not visible * Actually fix multimodal input support * Address review comments --- Cargo.toml | 2 +- alvr/client_core/src/connection.rs | 7 +- alvr/client_core/src/graphics/lobby.rs | 4 +- alvr/client_core/src/lib.rs | 29 +++- .../src/extra_extensions/body_tracking_fb.rs | 4 +- .../client_openxr/src/extra_extensions/mod.rs | 19 ++- .../src/extra_extensions/multimodal_input.rs | 72 ++++++++ alvr/client_openxr/src/interaction.rs | 154 ++++++++++++------ alvr/client_openxr/src/lib.rs | 22 ++- alvr/client_openxr/src/lobby.rs | 8 +- alvr/client_openxr/src/stream.rs | 34 +++- alvr/common/src/inputs.rs | 2 + alvr/common/src/lib.rs | 1 + .../presets/builtin_schema.rs | 10 +- alvr/packets/src/lib.rs | 8 + alvr/server_core/src/connection.rs | 29 ++-- alvr/server_core/src/lib.rs | 1 - alvr/server_core/src/tracking.rs | 66 +------- .../cpp/alvr_server/Controller.cpp | 97 +++++++---- .../cpp/alvr_server/Controller.h | 7 +- .../cpp/alvr_server/alvr_server.cpp | 71 +++----- alvr/server_openvr/cpp/alvr_server/bindings.h | 17 +- alvr/server_openvr/src/lib.rs | 109 ++++++++----- alvr/server_openvr/src/tracking.rs | 57 ++++++- alvr/session/src/settings.rs | 15 +- 25 files changed, 556 insertions(+), 289 deletions(-) create mode 100644 alvr/client_openxr/src/extra_extensions/multimodal_input.rs diff --git a/Cargo.toml b/Cargo.toml index bd22830ee7..eedb6eef16 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,7 +5,7 @@ members = ["alvr/*"] [workspace.package] version = "21.0.0-dev01" edition = "2021" -rust-version = "1.76" +rust-version = "1.77" authors = ["alvr-org"] license = "MIT" diff --git a/alvr/client_core/src/connection.rs b/alvr/client_core/src/connection.rs index a184d72a47..2ec2b3b108 100644 --- a/alvr/client_core/src/connection.rs +++ b/alvr/client_core/src/connection.rs @@ -14,7 +14,7 @@ use alvr_common::{ dbg_connection, debug, error, info, parking_lot::{Condvar, Mutex, RwLock}, wait_rwlock, warn, AnyhowToCon, ConResult, ConnectionError, ConnectionState, LifecycleState, - ALVR_VERSION, + RelaxedAtomic, ALVR_VERSION, }; use alvr_packets::{ ClientConnectionResult, ClientControlPacket, ClientStatistics, Haptics, ServerControlPacket, @@ -69,6 +69,7 @@ pub struct ConnectionContext { // todo: the server is supposed to receive and send view configs for each frame pub view_params_queue: RwLock>, pub last_good_view_params: RwLock<[ViewParams; 2]>, + pub uses_multimodal_protocol: RelaxedAtomic, } fn set_hud_message(event_queue: &Mutex>, message: &str) { @@ -179,6 +180,7 @@ fn connection_pipeline( encoder_high_profile: capabilities.encoder_high_profile, encoder_10_bits: capabilities.encoder_10_bits, encoder_av1: capabilities.encoder_av1, + multimodal_protocol: true, }) .to_con()?, ), @@ -191,6 +193,9 @@ fn connection_pipeline( let (settings, negotiated_config) = alvr_packets::decode_stream_config(&config_packet).to_con()?; + ctx.uses_multimodal_protocol + .set(negotiated_config.use_multimodal_protocol); + let streaming_start_event = ClientCoreEvent::StreamingStarted { settings: Box::new(settings.clone()), negotiated_config: negotiated_config.clone(), diff --git a/alvr/client_core/src/graphics/lobby.rs b/alvr/client_core/src/graphics/lobby.rs index 4b63b3a765..905c33d3a3 100644 --- a/alvr/client_core/src/graphics/lobby.rs +++ b/alvr/client_core/src/graphics/lobby.rs @@ -428,7 +428,9 @@ impl LobbyRenderer { ); transform_draw(&mut pass, view_proj * transform, 2); } - } else if let Some(pose) = maybe_pose { + } + + if let Some(pose) = maybe_pose { let hand_transform = Mat4::from_scale_rotation_translation( Vec3::ONE * 0.2, pose.orientation, diff --git a/alvr/client_core/src/lib.rs b/alvr/client_core/src/lib.rs index 3a1e21cffa..7eb2844d5d 100644 --- a/alvr/client_core/src/lib.rs +++ b/alvr/client_core/src/lib.rs @@ -23,7 +23,8 @@ use alvr_common::{ dbg_client_core, error, glam::{UVec2, Vec2, Vec3}, parking_lot::{Mutex, RwLock}, - warn, ConnectionState, DeviceMotion, LifecycleState, Pose, HEAD_ID, + warn, ConnectionState, DeviceMotion, LifecycleState, Pose, HAND_LEFT_ID, HAND_RIGHT_ID, + HEAD_ID, }; use alvr_packets::{ BatteryInfo, ButtonEntry, ClientControlPacket, FaceData, NegotiatedStreamingConfig, @@ -273,6 +274,32 @@ impl ClientCoreContext { } } + // send_tracking() expects hand data in the multimodal protocol. In case multimodal protocol + // is not supported, convert back to legacy protocol. + if !self.connection_context.uses_multimodal_protocol.value() { + if hand_skeletons[0].is_some() { + device_motions.push(( + *HAND_LEFT_ID, + DeviceMotion { + pose: hand_skeletons[0].unwrap()[0], + linear_velocity: Vec3::ZERO, + angular_velocity: Vec3::ZERO, + }, + )); + } + + if hand_skeletons[1].is_some() { + device_motions.push(( + *HAND_RIGHT_ID, + DeviceMotion { + pose: hand_skeletons[1].unwrap()[0], + linear_velocity: Vec3::ZERO, + angular_velocity: Vec3::ZERO, + }, + )); + } + } + if let Some(sender) = &mut *self.connection_context.tracking_sender.lock() { device_motions.push(( *HEAD_ID, diff --git a/alvr/client_openxr/src/extra_extensions/body_tracking_fb.rs b/alvr/client_openxr/src/extra_extensions/body_tracking_fb.rs index f97c3eab7a..54aa23383b 100644 --- a/alvr/client_openxr/src/extra_extensions/body_tracking_fb.rs +++ b/alvr/client_openxr/src/extra_extensions/body_tracking_fb.rs @@ -5,9 +5,9 @@ use openxr::{self as xr, raw, sys}; use std::ptr; pub const META_BODY_TRACKING_FULL_BODY_EXTENSION_NAME: &str = "XR_META_body_tracking_full_body"; -pub const TYPE_SYSTEM_PROPERTIES_BODY_TRACKING_FULL_BODY_META: Lazy = +static TYPE_SYSTEM_PROPERTIES_BODY_TRACKING_FULL_BODY_META: Lazy = Lazy::new(|| xr::StructureType::from_raw(1000274000)); -pub const BODY_JOINT_SET_FULL_BODY_META: Lazy = +pub static BODY_JOINT_SET_FULL_BODY_META: Lazy = Lazy::new(|| xr::BodyJointSetFB::from_raw(1000274000)); pub const FULL_BODY_JOINT_LEFT_UPPER_LEG_META: usize = 70; diff --git a/alvr/client_openxr/src/extra_extensions/mod.rs b/alvr/client_openxr/src/extra_extensions/mod.rs index 21feefd42c..5150fb7d6f 100644 --- a/alvr/client_openxr/src/extra_extensions/mod.rs +++ b/alvr/client_openxr/src/extra_extensions/mod.rs @@ -2,15 +2,17 @@ mod body_tracking_fb; mod eye_tracking_social; mod face_tracking2_fb; mod facial_tracking_htc; +mod multimodal_input; pub use body_tracking_fb::*; pub use eye_tracking_social::*; pub use face_tracking2_fb::*; pub use facial_tracking_htc::*; +pub use multimodal_input::*; use alvr_common::anyhow::{anyhow, Result}; use openxr::{self as xr, sys}; -use std::ptr; +use std::{mem, ptr}; fn to_any(result: sys::Result) -> Result<()> { if result.into_raw() >= 0 { @@ -24,13 +26,28 @@ fn to_any(result: sys::Result) -> Result<()> { pub struct ExtraExtensions { base_function_ptrs: xr::raw::Instance, ext_functions_ptrs: xr::InstanceExtensions, + resume_simultaneous_hands_and_controllers_tracking_meta: + Option, } impl ExtraExtensions { pub fn new(instance: &xr::Instance) -> Self { + let resume_simultaneous_hands_and_controllers_tracking_meta = unsafe { + let mut resume_simultaneous_hands_and_controllers_tracking_meta = None; + let _ = (instance.fp().get_instance_proc_addr)( + instance.as_raw(), + c"xrResumeSimultaneousHandsAndControllersTrackingMETA".as_ptr(), + &mut resume_simultaneous_hands_and_controllers_tracking_meta, + ); + + resume_simultaneous_hands_and_controllers_tracking_meta + .map(|f| mem::transmute::<_, ResumeSimultaneousHandsAndControllersTrackingMETA>(f)) + }; + Self { base_function_ptrs: instance.fp().clone(), ext_functions_ptrs: *instance.exts(), + resume_simultaneous_hands_and_controllers_tracking_meta, } } diff --git a/alvr/client_openxr/src/extra_extensions/multimodal_input.rs b/alvr/client_openxr/src/extra_extensions/multimodal_input.rs new file mode 100644 index 0000000000..5042c03207 --- /dev/null +++ b/alvr/client_openxr/src/extra_extensions/multimodal_input.rs @@ -0,0 +1,72 @@ +// Code taken from: +// https://github.com/meta-quest/Meta-OpenXR-SDK/blob/main/OpenXR/meta_openxr_preview/meta_simultaneous_hands_and_controllers.h + +use alvr_common::{anyhow::Result, once_cell::sync::Lazy, ToAny}; +use openxr::{self as xr, sys}; +use std::ffi::c_void; + +pub const META_SIMULTANEOUS_HANDS_AND_CONTROLLERS_EXTENSION_NAME: &str = + "XR_META_simultaneous_hands_and_controllers"; +pub const META_DETACHED_CONTROLLERS_EXTENSION_NAME: &str = "XR_META_detached_controllers"; + +static TYPE_SYSTEM_SIMULTANEOUS_HANDS_AND_CONTROLLERS_PROPERTIES_META: Lazy = + Lazy::new(|| xr::StructureType::from_raw(1000532001)); +static TYPE_SIMULTANEOUS_HANDS_AND_CONTROLLERS_TRACKING_RESUME_INFO_META: Lazy = + Lazy::new(|| xr::StructureType::from_raw(1000532002)); + +#[repr(C)] +pub struct SystemSymultaneousHandsAndControllersPropertiesMETA { + ty: xr::StructureType, + next: *const c_void, + supports_simultaneous_hands_and_controllers: sys::Bool32, +} + +#[repr(C)] +pub struct SimultaneousHandsAndControllersTrackingResumeInfoMETA { + ty: xr::StructureType, + next: *const c_void, +} + +pub type ResumeSimultaneousHandsAndControllersTrackingMETA = + unsafe extern "system" fn( + sys::Session, + *const SimultaneousHandsAndControllersTrackingResumeInfoMETA, + ) -> sys::Result; + +impl super::ExtraExtensions { + pub fn supports_simultaneous_hands_and_controllers( + &self, + instance: &xr::Instance, + system: xr::SystemId, + ) -> bool { + self.get_props( + instance, + system, + SystemSymultaneousHandsAndControllersPropertiesMETA { + ty: *TYPE_SYSTEM_SIMULTANEOUS_HANDS_AND_CONTROLLERS_PROPERTIES_META, + next: std::ptr::null(), + supports_simultaneous_hands_and_controllers: xr::sys::FALSE, + }, + ) + .map(|props| props.supports_simultaneous_hands_and_controllers.into()) + .unwrap_or(false) + } + + pub fn resume_simultaneous_hands_and_controllers_tracking( + &self, + session: &xr::Session, + ) -> Result<()> { + let resume_info = SimultaneousHandsAndControllersTrackingResumeInfoMETA { + ty: *TYPE_SIMULTANEOUS_HANDS_AND_CONTROLLERS_TRACKING_RESUME_INFO_META, + next: std::ptr::null(), + }; + + unsafe { + super::to_any((self + .resume_simultaneous_hands_and_controllers_tracking_meta + .to_any()?)(session.as_raw(), &resume_info))?; + } + + Ok(()) + } +} diff --git a/alvr/client_openxr/src/interaction.rs b/alvr/client_openxr/src/interaction.rs index fbfc425bfa..8ac566ce4d 100644 --- a/alvr/client_openxr/src/interaction.rs +++ b/alvr/client_openxr/src/interaction.rs @@ -45,6 +45,7 @@ pub struct InteractionContext { pub action_set: xr::ActionSet, pub button_actions: HashMap, pub hands_interaction: [HandInteraction; 2], + pub uses_multimodal_hands: bool, pub face_sources: FaceSources, pub body_sources: BodySources, } @@ -52,6 +53,7 @@ pub struct InteractionContext { pub fn initialize_interaction( xr_ctx: &XrContext, platform: Platform, + prefer_multimodal_input: bool, face_tracking_sources: Option, body_tracking_sources: Option, ) -> InteractionContext { @@ -195,6 +197,54 @@ pub fn initialize_interaction( "/user/hand/right/output/haptic", )); + // Note: We cannot enable multimodal if fb body tracking is active. It would result in a + // ERROR_RUNTIME_FAILURE crash. + let uses_multimodal_hands = prefer_multimodal_input + && xr_ctx + .extra_extensions + .supports_simultaneous_hands_and_controllers(&xr_ctx.instance, xr_ctx.system) + && !body_tracking_sources + .as_ref() + .map(|s| s.body_tracking_fb.enabled()) + .unwrap_or(false); + + let left_detached_controller_pose_action; + let right_detached_controller_pose_action; + if uses_multimodal_hands { + xr_ctx + .extra_extensions + .resume_simultaneous_hands_and_controllers_tracking(&xr_ctx.session) + .ok(); + + // Note: when multimodal input is enabled, both controllers and hands will always be active. + // To be able to detect when controllers are actually held, we have to register detached + // controllers pose; the controller pose will be diverted to the detached controllers when + // they are not held. Currently the detached controllers pose is ignored + left_detached_controller_pose_action = action_set + .create_action::( + "left_detached_controller_pose", + "Left detached controller pose", + &[], + ) + .unwrap(); + right_detached_controller_pose_action = action_set + .create_action::( + "right_detached_controller_pose", + "Right detached controller pose", + &[], + ) + .unwrap(); + + bindings.push(binding( + &left_detached_controller_pose_action, + "/user/detached_controller_meta/left/input/grip/pose", + )); + bindings.push(binding( + &right_detached_controller_pose_action, + "/user/detached_controller_meta/right/input/grip/pose", + )); + } + // Apply bindings: xr_ctx .instance @@ -380,6 +430,7 @@ pub fn initialize_interaction( skeleton_tracker: right_hand_tracker, }, ], + uses_multimodal_hands, face_sources: FaceSources { combined_eyes_source, eye_tracker_fb, @@ -405,9 +456,42 @@ pub fn get_hand_data( reference_space: &xr::Space, time: xr::Time, hand_source: &HandInteraction, - last_position: &mut Vec3, + last_controller_pose: &mut Pose, + last_palm_pose: &mut Pose, ) -> (Option, Option<[Pose; 26]>) { - if let Some(tracker) = &hand_source.skeleton_tracker { + let controller_motion = if hand_source + .grip_action + .is_active(xr_session, xr::Path::NULL) + .unwrap_or(false) + { + if let Ok((location, velocity)) = hand_source.grip_space.relate(reference_space, time) { + if location + .location_flags + .contains(xr::SpaceLocationFlags::ORIENTATION_VALID) + { + last_controller_pose.orientation = crate::from_xr_quat(location.pose.orientation); + } + + if location + .location_flags + .contains(xr::SpaceLocationFlags::POSITION_VALID) + { + last_controller_pose.position = crate::from_xr_vec3(location.pose.position); + } + + Some(DeviceMotion { + pose: *last_controller_pose, + linear_velocity: crate::from_xr_vec3(velocity.linear_velocity), + angular_velocity: crate::from_xr_vec3(velocity.angular_velocity), + }) + } else { + None + } + } else { + None + }; + + let hand_joints = if let Some(tracker) = &hand_source.skeleton_tracker { if let Some(joint_locations) = reference_space .locate_hand_joints(tracker, time) .ok() @@ -415,67 +499,37 @@ pub fn get_hand_data( { if joint_locations[0] .location_flags - .contains(xr::SpaceLocationFlags::POSITION_VALID) + .contains(xr::SpaceLocationFlags::ORIENTATION_VALID) { - *last_position = crate::from_xr_vec3(joint_locations[0].pose.position); + last_palm_pose.orientation = + crate::from_xr_quat(joint_locations[0].pose.orientation); } - let root_motion = DeviceMotion { - pose: Pose { - orientation: crate::from_xr_quat(joint_locations[0].pose.orientation), - position: *last_position, - }, - linear_velocity: Vec3::ZERO, - angular_velocity: Vec3::ZERO, - }; + if joint_locations[0] + .location_flags + .contains(xr::SpaceLocationFlags::POSITION_VALID) + { + last_palm_pose.position = crate::from_xr_vec3(joint_locations[0].pose.position); + } - let joints = joint_locations + let mut joints: [_; 26] = joint_locations .iter() .map(|j| crate::from_xr_pose(j.pose)) .collect::>() .try_into() .unwrap(); - return (Some(root_motion), Some(joints)); - } - } - - if !hand_source - .grip_action - .is_active(xr_session, xr::Path::NULL) - .unwrap_or(false) - { - return (None, None); - } + joints[0] = *last_palm_pose; - let Ok((location, velocity)) = hand_source.grip_space.relate(reference_space, time) else { - return (None, None); - }; - - if !location - .location_flags - .contains(xr::SpaceLocationFlags::ORIENTATION_VALID) - { - return (None, None); - } - - if location - .location_flags - .contains(xr::SpaceLocationFlags::POSITION_VALID) - { - *last_position = crate::from_xr_vec3(location.pose.position); - } - - let hand_motion = DeviceMotion { - pose: Pose { - orientation: crate::from_xr_quat(location.pose.orientation), - position: *last_position, - }, - linear_velocity: crate::from_xr_vec3(velocity.linear_velocity), - angular_velocity: crate::from_xr_vec3(velocity.angular_velocity), + Some(joints) + } else { + None + } + } else { + None }; - (Some(hand_motion), None) + (controller_motion, hand_joints) } pub fn update_buttons( diff --git a/alvr/client_openxr/src/lib.rs b/alvr/client_openxr/src/lib.rs index a41b019961..0723b61424 100644 --- a/alvr/client_openxr/src/lib.rs +++ b/alvr/client_openxr/src/lib.rs @@ -14,7 +14,11 @@ use alvr_common::{ glam::{Quat, UVec2, Vec3}, info, Fov, Pose, HAND_LEFT_ID, }; -use extra_extensions::{ExtraExtensions, META_BODY_TRACKING_FULL_BODY_EXTENSION_NAME}; +use extra_extensions::{ + ExtraExtensions, META_BODY_TRACKING_FULL_BODY_EXTENSION_NAME, + META_DETACHED_CONTROLLERS_EXTENSION_NAME, + META_SIMULTANEOUS_HANDS_AND_CONTROLLERS_EXTENSION_NAME, +}; use lobby::Lobby; use openxr as xr; use std::{ @@ -25,7 +29,6 @@ use std::{ time::{Duration, Instant}, }; use stream::StreamContext; -use xr::ColorSpaceFB; const DECODER_MAX_TIMEOUT_MULTIPLIER: f32 = 0.8; @@ -168,7 +171,14 @@ pub fn entry_point() { exts.other = available_extensions .other .into_iter() - .filter(|ext| [META_BODY_TRACKING_FULL_BODY_EXTENSION_NAME].contains(&ext.as_str())) + .filter(|ext| { + [ + META_BODY_TRACKING_FULL_BODY_EXTENSION_NAME, + META_SIMULTANEOUS_HANDS_AND_CONTROLLERS_EXTENSION_NAME, + META_DETACHED_CONTROLLERS_EXTENSION_NAME, + ] + .contains(&ext.as_str()) + }) .collect(); let available_layers = xr_entry.enumerate_layers().unwrap(); @@ -236,7 +246,7 @@ pub fn entry_point() { }; if exts.fb_color_space { - xr_session.set_color_space(ColorSpaceFB::P3).unwrap(); + xr_session.set_color_space(xr::ColorSpaceFB::P3).unwrap(); } let capabilities = ClientCapabilities { @@ -253,6 +263,10 @@ pub fn entry_point() { let interaction_context = Arc::new(interaction::initialize_interaction( &xr_context, platform, + stream_config + .as_ref() + .map(|c| c.prefers_multimodal_input) + .unwrap_or(false), stream_config .as_ref() .and_then(|c| c.face_sources_config.clone()), diff --git a/alvr/client_openxr/src/lobby.rs b/alvr/client_openxr/src/lobby.rs index aa223fe760..35786c0281 100644 --- a/alvr/client_openxr/src/lobby.rs +++ b/alvr/client_openxr/src/lobby.rs @@ -4,7 +4,7 @@ use crate::{ XrContext, }; use alvr_client_core::graphics::{GraphicsContext, LobbyRenderer, RenderViewInput, SDR_FORMAT_GL}; -use alvr_common::glam::{UVec2, Vec3}; +use alvr_common::{glam::UVec2, Pose}; use openxr as xr; use std::{rc::Rc, sync::Arc}; @@ -119,14 +119,16 @@ impl Lobby { &self.reference_space, predicted_display_time, &self.interaction_ctx.hands_interaction[0], - &mut Vec3::new(0.0, 0.0, 0.0), + &mut Pose::default(), + &mut Pose::default(), ); let right_hand_data = interaction::get_hand_data( &self.xr_session, &self.reference_space, predicted_display_time, &self.interaction_ctx.hands_interaction[1], - &mut Vec3::new(0.0, 0.0, 0.0), + &mut Pose::default(), + &mut Pose::default(), ); let body_skeleton_fb = self diff --git a/alvr/client_openxr/src/stream.rs b/alvr/client_openxr/src/stream.rs index 037bf88a5b..23966c2a9b 100644 --- a/alvr/client_openxr/src/stream.rs +++ b/alvr/client_openxr/src/stream.rs @@ -10,8 +10,8 @@ use alvr_client_core::{ }; use alvr_common::{ error, - glam::{UVec2, Vec2, Vec3}, - RelaxedAtomic, HAND_LEFT_ID, HAND_RIGHT_ID, + glam::{UVec2, Vec2}, + Pose, RelaxedAtomic, HAND_LEFT_ID, HAND_RIGHT_ID, }; use alvr_packets::{FaceData, NegotiatedStreamingConfig, ViewParams}; use alvr_session::{ @@ -38,6 +38,7 @@ pub struct StreamConfig { pub encoder_config: EncoderConfig, pub face_sources_config: Option, pub body_sources_config: Option, + pub prefers_multimodal_input: bool, } impl StreamConfig { @@ -61,6 +62,12 @@ impl StreamConfig { .body_tracking .as_option() .map(|c| c.sources.clone()), + prefers_multimodal_input: settings + .headset + .controllers + .as_option() + .map(|c| c.multimodal_tracking) + .unwrap_or(false), } } } @@ -368,7 +375,8 @@ fn stream_input_loop( refresh_rate: f32, running: Arc, ) { - let mut last_hand_positions = [Vec3::ZERO; 2]; + let mut last_controller_poses = [Pose::default(); 2]; + let mut last_palm_poses = [Pose::default(); 2]; let mut deadline = Instant::now(); let frame_interval = Duration::from_secs_f32(1.0 / refresh_rate); @@ -428,21 +436,29 @@ fn stream_input_loop( &reference_space, tracker_time, &interaction_ctx.hands_interaction[0], - &mut last_hand_positions[0], + &mut last_controller_poses[0], + &mut last_palm_poses[0], ); let (right_hand_motion, right_hand_skeleton) = crate::interaction::get_hand_data( &xr_ctx.session, &reference_space, tracker_time, &interaction_ctx.hands_interaction[1], - &mut last_hand_positions[1], + &mut last_controller_poses[1], + &mut last_palm_poses[1], ); - if let Some(motion) = left_hand_motion { - device_motions.push((*HAND_LEFT_ID, motion)); + // Note: When multimodal input is enabled, we are sure that when free hands are used + // (not holding controllers) the controller data is None. + if interaction_ctx.uses_multimodal_hands || left_hand_skeleton.is_none() { + if let Some(motion) = left_hand_motion { + device_motions.push((*HAND_LEFT_ID, motion)); + } } - if let Some(motion) = right_hand_motion { - device_motions.push((*HAND_RIGHT_ID, motion)); + if interaction_ctx.uses_multimodal_hands || right_hand_skeleton.is_none() { + if let Some(motion) = right_hand_motion { + device_motions.push((*HAND_RIGHT_ID, motion)); + } } let face_data = FaceData { diff --git a/alvr/common/src/inputs.rs b/alvr/common/src/inputs.rs index 26ed34d6cb..062b47062d 100644 --- a/alvr/common/src/inputs.rs +++ b/alvr/common/src/inputs.rs @@ -54,6 +54,8 @@ devices! { (BODY_LEFT_FOOT, "/user/body/left_foot"), (BODY_RIGHT_KNEE, "/user/body/right_knee"), (BODY_RIGHT_FOOT, "/user/body/right_foot"), + (DETACHED_CONTROLLER_LEFT, "/user/detached_controller_meta/left"), + (DETACHED_CONTROLLER_RIGHT, "/user/detached_controller_meta/right"), } pub enum ButtonType { diff --git a/alvr/common/src/lib.rs b/alvr/common/src/lib.rs index 0628307ef8..db942c403a 100644 --- a/alvr/common/src/lib.rs +++ b/alvr/common/src/lib.rs @@ -35,6 +35,7 @@ pub const fn lazy_mut_none() -> OptLazy { // Simple wrapper for AtomicBool when using Ordering::Relaxed. Deref cannot be implemented (cannot // return local reference) +#[derive(Default)] pub struct RelaxedAtomic(AtomicBool); impl RelaxedAtomic { diff --git a/alvr/dashboard/src/dashboard/components/settings_controls/presets/builtin_schema.rs b/alvr/dashboard/src/dashboard/components/settings_controls/presets/builtin_schema.rs index 7149bac80c..248ac6b56e 100644 --- a/alvr/dashboard/src/dashboard/components/settings_controls/presets/builtin_schema.rs +++ b/alvr/dashboard/src/dashboard/components/settings_controls/presets/builtin_schema.rs @@ -321,7 +321,7 @@ pub fn microphone_schema(devices: Vec) -> PresetSchemaNode { pub fn hand_tracking_interaction_schema() -> PresetSchemaNode { const HELP: &str = r"Disabled: hands cannot emulate buttons. Useful for using Joy-Cons or other non-native controllers. -Separate trackers: create separate SteamVR devices for hand tracking. This is used for VRChat. +SteamVR Input 2.0: create separate SteamVR devices for hand tracking. ALVR bindings: use ALVR hand tracking button bindings. Check the wiki for help. "; @@ -337,7 +337,7 @@ ALVR bindings: use ALVR hand tracking button bindings. Check the wiki for help. modifiers: vec![ bool_modifier("session_settings.headset.controllers.enabled", true), bool_modifier( - &format!("{PREFIX}.hand_skeleton.content.use_separate_trackers"), + &format!("{PREFIX}.hand_skeleton.content.steamvr_input_2_0"), false, ), bool_modifier( @@ -348,12 +348,12 @@ ALVR bindings: use ALVR hand tracking button bindings. Check the wiki for help. content: None, }, HigherOrderChoiceOption { - display_name: "Separate trackers".into(), + display_name: "SteamVR Input 2.0".into(), modifiers: vec![ bool_modifier("session_settings.headset.controllers.enabled", true), bool_modifier(&format!("{PREFIX}.hand_skeleton.enabled"), true), bool_modifier( - &format!("{PREFIX}.hand_skeleton.content.use_separate_trackers"), + &format!("{PREFIX}.hand_skeleton.content.steamvr_input_2_0"), true, ), bool_modifier( @@ -368,7 +368,7 @@ ALVR bindings: use ALVR hand tracking button bindings. Check the wiki for help. modifiers: vec![ bool_modifier("session_settings.headset.controllers.enabled", true), bool_modifier( - &format!("{PREFIX}.hand_skeleton.content.use_separate_trackers"), + &format!("{PREFIX}.hand_skeleton.content.steamvr_input_2_0"), false, ), bool_modifier(&format!("{PREFIX}.hand_tracking_interaction.enabled"), true), diff --git a/alvr/packets/src/lib.rs b/alvr/packets/src/lib.rs index 098d15dcc2..9d2d0c911a 100644 --- a/alvr/packets/src/lib.rs +++ b/alvr/packets/src/lib.rs @@ -38,6 +38,7 @@ pub struct VideoStreamingCapabilities { pub encoder_high_profile: bool, pub encoder_10_bits: bool, pub encoder_av1: bool, + pub multimodal_protocol: bool, } // Nasty workaround to make the packet extensible, pushing the limits of protocol compatibility @@ -92,6 +93,7 @@ pub fn decode_video_streaming_capabilities( encoder_high_profile: caps_json["encoder_high_profile"].as_bool().unwrap_or(true), encoder_10_bits: caps_json["encoder_10_bits"].as_bool().unwrap_or(true), encoder_av1: caps_json["encoder_av1"].as_bool().unwrap_or(true), + multimodal_protocol: caps_json["multimodal_protocol"].as_bool().unwrap_or(false), }) } @@ -113,6 +115,9 @@ pub struct NegotiatedStreamingConfig { pub refresh_rate_hint: f32, pub game_audio_sample_rate: u32, pub enable_foveated_encoding: bool, + // This is needed to detect when to use SteamVR hand trackers. This does NOT imply if multimodal + // input is supported + pub use_multimodal_protocol: bool, } #[derive(Serialize, Deserialize)] @@ -147,6 +152,8 @@ pub fn decode_stream_config( let enable_foveated_encoding = json::from_value(negotiated_json["enable_foveated_encoding"].clone()) .unwrap_or_else(|_| settings.video.foveated_encoding.enabled()); + let use_multimodal_protocol = + json::from_value(negotiated_json["use_multimodal_protocol"].clone()).unwrap_or(false); Ok(( settings, @@ -155,6 +162,7 @@ pub fn decode_stream_config( refresh_rate_hint, game_audio_sample_rate, enable_foveated_encoding, + use_multimodal_protocol, }, )) } diff --git a/alvr/server_core/src/connection.rs b/alvr/server_core/src/connection.rs index 07260a18d8..224f2b7ff0 100644 --- a/alvr/server_core/src/connection.rs +++ b/alvr/server_core/src/connection.rs @@ -88,7 +88,7 @@ pub fn contruct_openvr_config(session: &SessionConfig) -> OpenvrConfig { use_separate_hand_trackers = config .hand_skeleton .as_option() - .map(|c| c.use_separate_trackers) + .map(|c| c.steamvr_input_2_0) .unwrap_or(false); true @@ -632,6 +632,7 @@ fn connection_pipeline( refresh_rate_hint: fps, game_audio_sample_rate, enable_foveated_encoding, + use_multimodal_protocol: streaming_caps.multimodal_protocol, }, ) .to_con()?; @@ -896,10 +897,24 @@ fn connection_pipeline( Err(ConnectionError::TryAgain(_)) => continue, Err(ConnectionError::Other(_)) => return, }; - let Ok(tracking) = data.get_header() else { + let Ok(mut tracking) = data.get_header() else { return; }; + if !streaming_caps.multimodal_protocol { + if tracking.hand_skeletons[0].is_some() { + tracking + .device_motions + .retain(|(id, _)| *id != *HAND_LEFT_ID); + } + + if tracking.hand_skeletons[1].is_some() { + tracking + .device_motions + .retain(|(id, _)| *id != *HAND_RIGHT_ID); + } + } + let controllers_config = { let data_lock = SESSION_MANAGER.read(); data_lock @@ -917,14 +932,8 @@ fn connection_pipeline( let session_manager_lock = SESSION_MANAGER.read(); let headset_config = &session_manager_lock.settings().headset; - motions = tracking_manager_lock.transform_motions( - headset_config, - &tracking.device_motions, - [ - tracking.hand_skeletons[0].is_some(), - tracking.hand_skeletons[1].is_some(), - ], - ); + motions = tracking_manager_lock + .transform_motions(headset_config, &tracking.device_motions); hand_skeletons = [ tracking.hand_skeletons[0] diff --git a/alvr/server_core/src/lib.rs b/alvr/server_core/src/lib.rs index 9b9e89dc2d..76655ad0d4 100644 --- a/alvr/server_core/src/lib.rs +++ b/alvr/server_core/src/lib.rs @@ -14,7 +14,6 @@ mod web_server; pub use c_api::*; pub use logging_backend::init_logging; -pub use tracking::get_hand_skeleton_offsets; use crate::connection::VideoPacket; use alvr_common::{ diff --git a/alvr/server_core/src/tracking.rs b/alvr/server_core/src/tracking.rs index 7687d48440..90c6f39ec2 100644 --- a/alvr/server_core/src/tracking.rs +++ b/alvr/server_core/src/tracking.rs @@ -11,39 +11,6 @@ use std::{collections::HashMap, f32::consts::PI}; const DEG_TO_RAD: f32 = PI / 180.0; -pub fn get_hand_skeleton_offsets(config: &HeadsetConfig) -> (Pose, Pose) { - let left_offset; - let right_offset; - if let Switch::Enabled(controllers) = &config.controllers { - let t = controllers.left_hand_tracking_position_offset; - let r = controllers.left_hand_tracking_rotation_offset; - - left_offset = Pose { - orientation: Quat::from_euler( - EulerRot::XYZ, - r[0] * DEG_TO_RAD, - r[1] * DEG_TO_RAD, - r[2] * DEG_TO_RAD, - ), - position: Vec3::new(t[0], t[1], t[2]), - }; - right_offset = Pose { - orientation: Quat::from_euler( - EulerRot::XYZ, - r[0] * DEG_TO_RAD, - -r[1] * DEG_TO_RAD, - -r[2] * DEG_TO_RAD, - ), - position: Vec3::new(-t[0], t[1], t[2]), - }; - } else { - left_offset = Pose::default(); - right_offset = Pose::default(); - } - - (left_offset, right_offset) -} - // todo: Move this struct to Settings and use it for every tracked device #[derive(Default)] struct MotionConfig { @@ -114,7 +81,6 @@ impl TrackingManager { &mut self, config: &HeadsetConfig, device_motions: &[(u64, DeviceMotion)], - hand_skeletons_enabled: [bool; 2], ) -> Vec<(u64, DeviceMotion)> { let mut device_motion_configs = HashMap::new(); device_motion_configs.insert(*HEAD_ID, MotionConfig::default()); @@ -168,9 +134,6 @@ impl TrackingManager { ); } - let (left_hand_skeleton_offset, right_hand_skeleton_offset) = - get_hand_skeleton_offsets(config); - let mut transformed_motions = vec![]; for &(device_id, mut motion) in device_motions { if device_id == *HEAD_ID { @@ -186,19 +149,12 @@ impl TrackingManager { motion.angular_velocity = inverse_origin_orientation * motion.angular_velocity; // Apply custom transform - let pose_offset = if device_id == *HAND_LEFT_ID && hand_skeletons_enabled[0] { - left_hand_skeleton_offset - } else if device_id == *HAND_RIGHT_ID && hand_skeletons_enabled[1] { - right_hand_skeleton_offset - } else { - config.pose_offset - }; - motion.pose.orientation *= pose_offset.orientation; - motion.pose.position += motion.pose.orientation * pose_offset.position; + motion.pose.orientation *= config.pose_offset.orientation; + motion.pose.position += motion.pose.orientation * config.pose_offset.position; motion.linear_velocity += motion .angular_velocity - .cross(motion.pose.orientation * pose_offset.position); + .cross(motion.pose.orientation * config.pose_offset.position); motion.angular_velocity = motion.pose.orientation.conjugate() * motion.angular_velocity; @@ -210,18 +166,10 @@ impl TrackingManager { } } - if (device_id == *HAND_LEFT_ID && hand_skeletons_enabled[0]) - || (device_id == *HAND_RIGHT_ID && hand_skeletons_enabled[1]) - { - // On hand tracking, velocities seem to make hands overly jittery - motion.linear_velocity = Vec3::ZERO; - motion.angular_velocity = Vec3::ZERO; - } else { - motion.linear_velocity = - cutoff(motion.linear_velocity, config.linear_velocity_cutoff); - motion.angular_velocity = - cutoff(motion.angular_velocity, config.angular_velocity_cutoff); - } + motion.linear_velocity = + cutoff(motion.linear_velocity, config.linear_velocity_cutoff); + motion.angular_velocity = + cutoff(motion.angular_velocity, config.angular_velocity_cutoff); transformed_motions.push((device_id, motion)); } diff --git a/alvr/server_openvr/cpp/alvr_server/Controller.cpp b/alvr/server_openvr/cpp/alvr_server/Controller.cpp index 6ab23822cb..6289fdbeca 100644 --- a/alvr/server_openvr/cpp/alvr_server/Controller.cpp +++ b/alvr/server_openvr/cpp/alvr_server/Controller.cpp @@ -220,45 +220,78 @@ void Controller::SetButton(uint64_t id, FfiButtonValue value) { } } -bool Controller::onPoseUpdate( - float predictionS, - FfiDeviceMotion motion, - const FfiHandSkeleton* handSkeleton, - unsigned int controllersTracked -) { +bool Controller::onPoseUpdate(float predictionS, FfiHandData handData) { if (this->object_id == vr::k_unTrackedDeviceIndexInvalid) { return false; } + auto controllerMotion = handData.controllerMotion; + auto handSkeleton = handData.handSkeleton; + + // Note: following the multimodal protocol, to make sure we want to use hand trackers we need to + // check controllerMotion == nullptr. handSkeleton != nullptr is not enough. + bool enabledAsHandTracker = handData.useHandTracker + && (device_id == HAND_TRACKER_LEFT_ID || device_id == HAND_TRACKER_RIGHT_ID) + && controllerMotion == nullptr; + bool enabledAsController = !handData.useHandTracker + && (device_id == HAND_LEFT_ID || device_id == HAND_RIGHT_ID) && controllerMotion != nullptr; + bool enabled = handData.tracked && (enabledAsHandTracker || enabledAsController); + + Debug( + "%s %s: enabled: %d, ctrl: %d, hand: %d", + (device_id == HAND_TRACKER_LEFT_ID || device_id == HAND_TRACKER_RIGHT_ID) ? "hand tracker" + : "controller", + (device_id == HAND_TRACKER_LEFT_ID || device_id == HAND_LEFT_ID) ? "left" : "right", + enabled, + handData.controllerMotion != nullptr, + handData.handSkeleton != nullptr + ); + auto vr_driver_input = vr::VRDriverInput(); auto pose = vr::DriverPose_t {}; - pose.poseIsValid = controllersTracked; - pose.deviceIsConnected = controllersTracked; - pose.result - = controllersTracked ? vr::TrackingResult_Running_OK : vr::TrackingResult_Uninitialized; + pose.poseIsValid = enabled; + pose.deviceIsConnected = enabled; + pose.result = enabled ? vr::TrackingResult_Running_OK : vr::TrackingResult_Uninitialized; pose.qDriverFromHeadRotation = HmdQuaternion_Init(1, 0, 0, 0); pose.qWorldFromDriverRotation = HmdQuaternion_Init(1, 0, 0, 0); - pose.qRotation = HmdQuaternion_Init( - motion.orientation.w, - motion.orientation.x, - motion.orientation.y, - motion.orientation.z - ); // controllerRotation; - - pose.vecPosition[0] = motion.position[0]; - pose.vecPosition[1] = motion.position[1]; - pose.vecPosition[2] = motion.position[2]; + if (controllerMotion != nullptr) { + auto m = controllerMotion; - pose.vecVelocity[0] = motion.linearVelocity[0]; - pose.vecVelocity[1] = motion.linearVelocity[1]; - pose.vecVelocity[2] = motion.linearVelocity[2]; + pose.qRotation = HmdQuaternion_Init( + m->orientation.w, m->orientation.x, m->orientation.y, m->orientation.z + ); - pose.vecAngularVelocity[0] = motion.angularVelocity[0]; - pose.vecAngularVelocity[1] = motion.angularVelocity[1]; - pose.vecAngularVelocity[2] = motion.angularVelocity[2]; + pose.vecPosition[0] = m->position[0]; + pose.vecPosition[1] = m->position[1]; + pose.vecPosition[2] = m->position[2]; + + pose.vecVelocity[0] = m->linearVelocity[0]; + pose.vecVelocity[1] = m->linearVelocity[1]; + pose.vecVelocity[2] = m->linearVelocity[2]; + + pose.vecAngularVelocity[0] = m->angularVelocity[0]; + pose.vecAngularVelocity[1] = m->angularVelocity[1]; + pose.vecAngularVelocity[2] = m->angularVelocity[2]; + } else if (handSkeleton != nullptr) { + auto r = handSkeleton->jointRotations[0]; + pose.qRotation = HmdQuaternion_Init(r.w, r.x, r.y, r.z); + + auto p = handSkeleton->jointPositions[0]; + pose.vecPosition[0] = p[0]; + pose.vecPosition[1] = p[1]; + pose.vecPosition[2] = p[2]; + + pose.vecVelocity[0] = 0; + pose.vecVelocity[1] = 0; + pose.vecVelocity[2] = 0; + + pose.vecAngularVelocity[0] = 0; + pose.vecAngularVelocity[1] = 0; + pose.vecAngularVelocity[2] = 0; + } pose.poseTimeOffset = predictionS; @@ -269,13 +302,13 @@ bool Controller::onPoseUpdate( ); // Early return to skip updating the skeleton - if (!this->isEnabled()) { + if (!enabled) { return false; - } - - if (handSkeleton != nullptr) { + } else if (handSkeleton != nullptr) { vr::VRBoneTransform_t boneTransform[SKELETON_BONE_COUNT] = {}; - for (int j = 0; j < 31; j++) { + + // NB: start from index 1 to skip the root bone + for (int j = 1; j < 31; j++) { boneTransform[j].orientation.w = handSkeleton->jointRotations[j].w; boneTransform[j].orientation.x = handSkeleton->jointRotations[j].x; boneTransform[j].orientation.y = handSkeleton->jointRotations[j].y; @@ -328,7 +361,7 @@ bool Controller::onPoseUpdate( vr_driver_input->UpdateScalarComponent( m_buttonHandles[ALVR_INPUT_FINGER_PINKY], rotPinky, 0.0 ); - } else { + } else if (controllerMotion != nullptr) { if (m_lastThumbTouch != m_currentThumbTouch) { m_thumbTouchAnimationProgress += 1.f / ANIMATION_FRAME_COUNT; if (m_thumbTouchAnimationProgress > 1.f) { diff --git a/alvr/server_openvr/cpp/alvr_server/Controller.h b/alvr/server_openvr/cpp/alvr_server/Controller.h index 21782a3b9c..fbc4794ef4 100644 --- a/alvr/server_openvr/cpp/alvr_server/Controller.h +++ b/alvr/server_openvr/cpp/alvr_server/Controller.h @@ -37,12 +37,7 @@ class Controller : public TrackedDevice, public vr::ITrackedDeviceServerDriver { void SetButton(uint64_t id, FfiButtonValue value); - bool onPoseUpdate( - float predictionS, - FfiDeviceMotion motion, - const FfiHandSkeleton* hand, - unsigned int controllersTracked - ); + bool onPoseUpdate(float predictionS, FfiHandData handData); void GetBoneTransform(bool withController, vr::VRBoneTransform_t outBoneTransform[]); diff --git a/alvr/server_openvr/cpp/alvr_server/alvr_server.cpp b/alvr/server_openvr/cpp/alvr_server/alvr_server.cpp index bc02a13648..38d6ccf1d8 100644 --- a/alvr/server_openvr/cpp/alvr_server/alvr_server.cpp +++ b/alvr/server_openvr/cpp/alvr_server/alvr_server.cpp @@ -406,57 +406,34 @@ void RequestIDR() { void SetTracking( unsigned long long targetTimestampNs, float controllerPoseTimeOffsetS, - const FfiDeviceMotion* deviceMotions, - int motionsCount, - unsigned int controllersTracked, - bool useLeftHandTracker, - bool useRightHandTracker, - const FfiHandSkeleton* leftHandSkeleton, - const FfiHandSkeleton* rightHandSkeleton, + FfiDeviceMotion headMotion, + FfiHandData leftHandData, + FfiHandData rightHandData, const FfiBodyTracker* bodyTrackers, int bodyTrackersCount ) { - for (int i = 0; i < motionsCount; i++) { - if (deviceMotions[i].deviceID == HEAD_ID && g_driver_provider.hmd) { - g_driver_provider.hmd->OnPoseUpdated(targetTimestampNs, deviceMotions[i]); - } else { - if (deviceMotions[i].deviceID == HAND_LEFT_ID) { - if (g_driver_provider.left_controller) { - g_driver_provider.left_controller->onPoseUpdate( - controllerPoseTimeOffsetS, - deviceMotions[i], - leftHandSkeleton, - controllersTracked && !useLeftHandTracker - ); - } - if (g_driver_provider.left_hand_tracker) { - g_driver_provider.left_hand_tracker->onPoseUpdate( - controllerPoseTimeOffsetS, - deviceMotions[i], - leftHandSkeleton, - controllersTracked && useLeftHandTracker - ); - } - } else if (deviceMotions[i].deviceID == HAND_RIGHT_ID) { - if (g_driver_provider.right_controller) { - g_driver_provider.right_controller->onPoseUpdate( - controllerPoseTimeOffsetS, - deviceMotions[i], - rightHandSkeleton, - controllersTracked && !useRightHandTracker - ); - } - if (g_driver_provider.right_hand_tracker) { - g_driver_provider.right_hand_tracker->onPoseUpdate( - controllerPoseTimeOffsetS, - deviceMotions[i], - rightHandSkeleton, - controllersTracked && useRightHandTracker - ); - } - } - } + if (g_driver_provider.hmd) { + g_driver_provider.hmd->OnPoseUpdated(targetTimestampNs, headMotion); + } + + if (g_driver_provider.left_hand_tracker) { + g_driver_provider.left_hand_tracker->onPoseUpdate(controllerPoseTimeOffsetS, leftHandData); + } + + if (g_driver_provider.left_controller) { + g_driver_provider.left_controller->onPoseUpdate(controllerPoseTimeOffsetS, leftHandData); } + + if (g_driver_provider.right_hand_tracker) { + g_driver_provider.right_hand_tracker->onPoseUpdate( + controllerPoseTimeOffsetS, rightHandData + ); + } + + if (g_driver_provider.right_controller) { + g_driver_provider.right_controller->onPoseUpdate(controllerPoseTimeOffsetS, rightHandData); + } + if (Settings::Instance().m_enableBodyTrackingFakeVive) { for (int i = 0; i < bodyTrackersCount; i++) { g_driver_provider.generic_trackers.at(bodyTrackers[i].trackerID) diff --git a/alvr/server_openvr/cpp/alvr_server/bindings.h b/alvr/server_openvr/cpp/alvr_server/bindings.h index 2db55b26c4..d89403c265 100644 --- a/alvr/server_openvr/cpp/alvr_server/bindings.h +++ b/alvr/server_openvr/cpp/alvr_server/bindings.h @@ -27,6 +27,13 @@ struct FfiDeviceMotion { float angularVelocity[3]; }; +struct FfiHandData { + unsigned int tracked; + const FfiDeviceMotion* controllerMotion; + const FfiHandSkeleton* handSkeleton; + bool useHandTracker; +}; + struct FfiBodyTracker { unsigned int trackerID; FfiQuat orientation; @@ -142,13 +149,9 @@ extern "C" void RequestIDR(); extern "C" void SetTracking( unsigned long long targetTimestampNs, float controllerPoseTimeOffsetS, - const FfiDeviceMotion* deviceMotions, - int motionsCount, - unsigned int controllersTracked, - bool useLeftHandTracker, - bool useRightHandTracker, - const FfiHandSkeleton* leftHand, - const FfiHandSkeleton* rightHand, + FfiDeviceMotion headMotion, + FfiHandData leftHandData, + FfiHandData rightHandData, const FfiBodyTracker* bodyTrackers, int bodyTrackersCount ); diff --git a/alvr/server_openvr/src/lib.rs b/alvr/server_openvr/src/lib.rs index 00de89cd74..caa2a386ee 100644 --- a/alvr/server_openvr/src/lib.rs +++ b/alvr/server_openvr/src/lib.rs @@ -20,6 +20,7 @@ use alvr_common::{ parking_lot::{Mutex, RwLock}, settings_schema::Switch, warn, BUTTON_INFO, HAND_LEFT_ID, HAND_RIGHT_ID, HAND_TRACKER_LEFT_ID, HAND_TRACKER_RIGHT_ID, + HEAD_ID, }; use alvr_filesystem as afs; use alvr_packets::{ButtonValue, Haptics}; @@ -112,32 +113,22 @@ extern "C" fn driver_ready_idle(set_default_chap: bool) { tracking, controllers_pose_time_offset, } => { - let controllers_config; - let track_body; - { - let headset_config = &alvr_server_core::settings().headset; + let headset_config = &alvr_server_core::settings().headset; - controllers_config = headset_config.controllers.clone().into_option(); - track_body = headset_config.body_tracking.enabled(); - }; + let controllers_config = headset_config.controllers.clone().into_option(); + let track_body = headset_config.body_tracking.enabled(); let track_controllers = controllers_config .as_ref() .map(|c| c.tracked) .unwrap_or(false); - let left_openvr_hand_skeleton; - let right_openvr_hand_skeleton; - { - let headset_config = &alvr_server_core::settings().headset; - - left_openvr_hand_skeleton = tracking.hand_skeletons[0].map(|s| { - tracking::to_openvr_hand_skeleton(headset_config, *HAND_LEFT_ID, s) - }); - right_openvr_hand_skeleton = tracking.hand_skeletons[1].map(|s| { - tracking::to_openvr_hand_skeleton(headset_config, *HAND_RIGHT_ID, s) - }); - } + let left_openvr_hand_skeleton = tracking.hand_skeletons[0].map(|s| { + tracking::to_openvr_hand_skeleton(headset_config, *HAND_LEFT_ID, s) + }); + let right_openvr_hand_skeleton = tracking.hand_skeletons[1].map(|s| { + tracking::to_openvr_hand_skeleton(headset_config, *HAND_RIGHT_ID, s) + }); let ( use_separate_hand_trackers, @@ -149,7 +140,7 @@ extern "C" fn driver_ready_idle(set_default_chap: bool) { }) = controllers_config { ( - hand_skeleton_config.use_separate_trackers, + hand_skeleton_config.steamvr_input_2_0, left_openvr_hand_skeleton.map(tracking::to_ffi_skeleton), right_openvr_hand_skeleton.map(tracking::to_ffi_skeleton), ) @@ -157,11 +148,65 @@ extern "C" fn driver_ready_idle(set_default_chap: bool) { (false, None, None) }; - let ffi_motions = tracking + let use_left_hand_tracker = use_separate_hand_trackers + && tracking.hand_skeletons[0].is_some() + && tracking + .device_motions + .iter() + .all(|(id, _)| *id != *HAND_LEFT_ID); + let use_right_hand_tracker = use_separate_hand_trackers + && tracking.hand_skeletons[1].is_some() + && tracking + .device_motions + .iter() + .all(|(id, _)| *id != *HAND_RIGHT_ID); + + let ffi_head_motion = tracking .device_motions .iter() - .map(|(id, motion)| tracking::to_ffi_motion(*id, *motion)) - .collect::>(); + .find_map(|(id, motion)| { + (*id == *HEAD_ID).then(|| tracking::to_ffi_motion(*HEAD_ID, *motion)) + }) + .unwrap_or_else(FfiDeviceMotion::default); + let ffi_left_controller_motion = + tracking.device_motions.iter().find_map(|(id, motion)| { + (*id == *HAND_LEFT_ID) + .then(|| tracking::to_ffi_motion(*HAND_LEFT_ID, *motion)) + }); + let ffi_right_controller_motion = + tracking.device_motions.iter().find_map(|(id, motion)| { + (*id == *HAND_RIGHT_ID) + .then(|| tracking::to_ffi_motion(*HAND_RIGHT_ID, *motion)) + }); + + let left_hand_data = FfiHandData { + tracked: track_controllers.into(), + controllerMotion: if let Some(motion) = &ffi_left_controller_motion { + motion + } else { + ptr::null() + }, + handSkeleton: if let Some(skeleton) = &ffi_left_hand_skeleton { + skeleton + } else { + ptr::null() + }, + useHandTracker: use_left_hand_tracker, + }; + let right_hand_data = FfiHandData { + tracked: track_controllers.into(), + controllerMotion: if let Some(motion) = &ffi_right_controller_motion { + motion + } else { + ptr::null() + }, + handSkeleton: if let Some(skeleton) = &ffi_right_hand_skeleton { + skeleton + } else { + ptr::null() + }, + useHandTracker: use_right_hand_tracker, + }; let ffi_body_trackers = tracking::to_ffi_body_trackers(&tracking.device_motions, track_body); @@ -174,21 +219,9 @@ extern "C" fn driver_ready_idle(set_default_chap: bool) { SetTracking( tracking.target_timestamp.as_nanos() as _, controllers_pose_time_offset.as_secs_f32(), - ffi_motions.as_ptr(), - ffi_motions.len() as _, - track_controllers.into(), - use_separate_hand_trackers && tracking.hand_skeletons[0].is_some(), - use_separate_hand_trackers && tracking.hand_skeletons[1].is_some(), - if let Some(skeleton) = &ffi_left_hand_skeleton { - skeleton - } else { - ptr::null() - }, - if let Some(skeleton) = &ffi_right_hand_skeleton { - skeleton - } else { - ptr::null() - }, + ffi_head_motion, + left_hand_data, + right_hand_data, if let Some(body_trackers) = &ffi_body_trackers { body_trackers.as_ptr() } else { diff --git a/alvr/server_openvr/src/tracking.rs b/alvr/server_openvr/src/tracking.rs index 64bc36a0bb..428e4514ed 100644 --- a/alvr/server_openvr/src/tracking.rs +++ b/alvr/server_openvr/src/tracking.rs @@ -2,6 +2,7 @@ use crate::{FfiBodyTracker, FfiDeviceMotion, FfiHandSkeleton, FfiQuat}; use alvr_common::{ glam::{EulerRot, Quat, Vec3}, once_cell::sync::Lazy, + settings_schema::Switch, DeviceMotion, Pose, BODY_CHEST_ID, BODY_HIPS_ID, BODY_LEFT_ELBOW_ID, BODY_LEFT_FOOT_ID, BODY_LEFT_KNEE_ID, BODY_RIGHT_ELBOW_ID, BODY_RIGHT_FOOT_ID, BODY_RIGHT_KNEE_ID, HAND_LEFT_ID, }; @@ -11,6 +12,8 @@ use std::{ f32::consts::{FRAC_PI_2, PI}, }; +const DEG_TO_RAD: f32 = PI / 180.0; + fn to_ffi_quat(quat: Quat) -> FfiQuat { FfiQuat { x: quat.x, @@ -20,15 +23,53 @@ fn to_ffi_quat(quat: Quat) -> FfiQuat { } } +fn get_hand_skeleton_offsets(config: &HeadsetConfig) -> (Pose, Pose) { + let left_offset; + let right_offset; + if let Switch::Enabled(controllers) = &config.controllers { + let t = controllers.left_hand_tracking_position_offset; + let r = controllers.left_hand_tracking_rotation_offset; + + left_offset = Pose { + orientation: Quat::from_euler( + EulerRot::XYZ, + r[0] * DEG_TO_RAD, + r[1] * DEG_TO_RAD, + r[2] * DEG_TO_RAD, + ), + position: Vec3::new(t[0], t[1], t[2]), + }; + right_offset = Pose { + orientation: Quat::from_euler( + EulerRot::XYZ, + r[0] * DEG_TO_RAD, + -r[1] * DEG_TO_RAD, + -r[2] * DEG_TO_RAD, + ), + position: Vec3::new(-t[0], t[1], t[2]), + }; + } else { + left_offset = Pose::default(); + right_offset = Pose::default(); + } + + (left_offset, right_offset) +} + pub fn to_openvr_hand_skeleton( config: &HeadsetConfig, device_id: u64, hand_skeleton: [Pose; 26], ) -> [Pose; 31] { - let (left_hand_skeleton_offset, right_hand_skeleton_offset) = - alvr_server_core::get_hand_skeleton_offsets(config); + let (left_hand_skeleton_offset, right_hand_skeleton_offset) = get_hand_skeleton_offsets(config); let id = device_id; + let pose_offset = if id == *HAND_LEFT_ID { + left_hand_skeleton_offset + } else { + right_hand_skeleton_offset + }; + // global joints let gj = hand_skeleton; @@ -86,12 +127,6 @@ pub fn to_openvr_hand_skeleton( // Adjust hand position based on the emulated controller for joints // parented to the root. let root_parented_pose = |pose: Pose| -> Pose { - let pose_offset = if id == *HAND_LEFT_ID { - left_hand_skeleton_offset - } else { - right_hand_skeleton_offset - }; - let sign = if id == *HAND_LEFT_ID { -1.0 } else { 1.0 }; let orientation = pose_offset.orientation.conjugate() * gj[0].orientation.conjugate() @@ -117,7 +152,11 @@ pub fn to_openvr_hand_skeleton( [ // Palm. NB: this is ignored by SteamVR - Pose::default(), + Pose { + orientation: gj[0].orientation * pose_offset.orientation, + position: gj[0].position + + gj[0].orientation * pose_offset.orientation * pose_offset.position, + }, // Wrist root_parented_pose(gj[1]), // Thumb diff --git a/alvr/session/src/settings.rs b/alvr/session/src/settings.rs index de439abc3a..ae48c91f20 100644 --- a/alvr/session/src/settings.rs +++ b/alvr/session/src/settings.rs @@ -683,6 +683,10 @@ pub struct FaceTrackingConfig { #[derive(SettingsSchema, Serialize, Deserialize, Clone, PartialEq)] pub struct BodyTrackingSourcesConfig { pub body_tracking_fb: Switch, + // todo: + // pub detached_controllers_as_feet: bool, + // unfortunately multimodal is incompatible with body tracking. To make this usable we need to + // at least add support for an android client as 3dof waist tracker. } #[derive(SettingsSchema, Serialize, Deserialize, Clone, PartialEq)] @@ -871,7 +875,7 @@ pub struct HandSkeletonConfig { #[schema(strings( help = r"Enabling this will use separate tracker objects with the full skeletal tracking level when hand tracking is detected. This is required for VRChat hand tracking." ))] - pub use_separate_trackers: bool, + pub steamvr_input_2_0: bool, } #[derive(SettingsSchema, Serialize, Deserialize, Clone)] @@ -887,6 +891,12 @@ pub struct ControllersConfig { ))] pub hand_skeleton: Switch, + #[schema(strings( + help = r"Track hand skeleton while holding controllers. This will reduce hand tracking frequency to 30Hz. +Because of runtime limitations, this option is ignored when body tracking is active." + ))] + pub multimodal_tracking: bool, + #[schema(flag = "real-time")] #[schema(strings( help = "Enabling this allows using hand gestures to emulate controller inputs." @@ -1532,9 +1542,10 @@ pub fn session_settings_default() -> SettingsDefault { hand_skeleton: SwitchDefault { enabled: true, content: HandSkeletonConfigDefault { - use_separate_trackers: true, + steamvr_input_2_0: true, }, }, + multimodal_tracking: false, emulation_mode: ControllersEmulationModeDefault { Custom: ControllersEmulationModeCustomDefault { serial_number: "ALVR Controller".into(),