Multimodal input (#2367)

* feat: ✨ Multimodal input

* Fix controllers and hands dropping to 0,0,0 when not visible

* Actually fix multimodal input support

* Address review comments

Cargo.toml

@@ -5,7 +5,7 @@ members = ["alvr/*"]
 [workspace.package]
 version = "21.0.0-dev01"
 edition = "2021"
-rust-version = "1.76"
+rust-version = "1.77"
 authors = ["alvr-org"]
 license = "MIT"
 

alvr/client_core/src/connection.rs

@@ -14,7 +14,7 @@ use alvr_common::{
     dbg_connection, debug, error, info,
     parking_lot::{Condvar, Mutex, RwLock},
     wait_rwlock, warn, AnyhowToCon, ConResult, ConnectionError, ConnectionState, LifecycleState,
-    ALVR_VERSION,
+    RelaxedAtomic, ALVR_VERSION,
 };
 use alvr_packets::{
     ClientConnectionResult, ClientControlPacket, ClientStatistics, Haptics, ServerControlPacket,
@@ -69,6 +69,7 @@ pub struct ConnectionContext {
     // todo: the server is supposed to receive and send view configs for each frame
     pub view_params_queue: RwLock<VecDeque<(Duration, [ViewParams; 2])>>,
     pub last_good_view_params: RwLock<[ViewParams; 2]>,
+    pub uses_multimodal_protocol: RelaxedAtomic,
 }
 
 fn set_hud_message(event_queue: &Mutex<VecDeque<ClientCoreEvent>>, message: &str) {
@@ -179,6 +180,7 @@ fn connection_pipeline(
                     encoder_high_profile: capabilities.encoder_high_profile,
                     encoder_10_bits: capabilities.encoder_10_bits,
                     encoder_av1: capabilities.encoder_av1,
+                    multimodal_protocol: true,
                 })
                 .to_con()?,
             ),
@@ -191,6 +193,9 @@ fn connection_pipeline(
     let (settings, negotiated_config) =
         alvr_packets::decode_stream_config(&config_packet).to_con()?;
 
+    ctx.uses_multimodal_protocol
+        .set(negotiated_config.use_multimodal_protocol);
+
     let streaming_start_event = ClientCoreEvent::StreamingStarted {
         settings: Box::new(settings.clone()),
         negotiated_config: negotiated_config.clone(),

alvr/client_core/src/graphics/lobby.rs

@@ -428,7 +428,9 @@ impl LobbyRenderer {
                         );
                         transform_draw(&mut pass, view_proj * transform, 2);
                     }
-                } else if let Some(pose) = maybe_pose {
+                }
+
+                if let Some(pose) = maybe_pose {
                     let hand_transform = Mat4::from_scale_rotation_translation(
                         Vec3::ONE * 0.2,
                         pose.orientation,

alvr/client_core/src/lib.rs

@@ -23,7 +23,8 @@ use alvr_common::{
     dbg_client_core, error,
     glam::{UVec2, Vec2, Vec3},
     parking_lot::{Mutex, RwLock},
-    warn, ConnectionState, DeviceMotion, LifecycleState, Pose, HEAD_ID,
+    warn, ConnectionState, DeviceMotion, LifecycleState, Pose, HAND_LEFT_ID, HAND_RIGHT_ID,
+    HEAD_ID,
 };
 use alvr_packets::{
     BatteryInfo, ButtonEntry, ClientControlPacket, FaceData, NegotiatedStreamingConfig,
@@ -273,6 +274,32 @@ impl ClientCoreContext {
             }
         }
 
+        // send_tracking() expects hand data in the multimodal protocol. In case multimodal protocol
+        // is not supported, convert back to legacy protocol.
+        if !self.connection_context.uses_multimodal_protocol.value() {
+            if hand_skeletons[0].is_some() {
+                device_motions.push((
+                    *HAND_LEFT_ID,
+                    DeviceMotion {
+                        pose: hand_skeletons[0].unwrap()[0],
+                        linear_velocity: Vec3::ZERO,
+                        angular_velocity: Vec3::ZERO,
+                    },
+                ));
+            }
+
+            if hand_skeletons[1].is_some() {
+                device_motions.push((
+                    *HAND_RIGHT_ID,
+                    DeviceMotion {
+                        pose: hand_skeletons[1].unwrap()[0],
+                        linear_velocity: Vec3::ZERO,
+                        angular_velocity: Vec3::ZERO,
+                    },
+                ));
+            }
+        }
+
         if let Some(sender) = &mut *self.connection_context.tracking_sender.lock() {
             device_motions.push((
                 *HEAD_ID,

alvr/client_openxr/src/extra_extensions/body_tracking_fb.rs

@@ -5,9 +5,9 @@ use openxr::{self as xr, raw, sys};
 use std::ptr;
 
 pub const META_BODY_TRACKING_FULL_BODY_EXTENSION_NAME: &str = "XR_META_body_tracking_full_body";
-pub const TYPE_SYSTEM_PROPERTIES_BODY_TRACKING_FULL_BODY_META: Lazy<xr::StructureType> =
+static TYPE_SYSTEM_PROPERTIES_BODY_TRACKING_FULL_BODY_META: Lazy<xr::StructureType> =
     Lazy::new(|| xr::StructureType::from_raw(1000274000));
-pub const BODY_JOINT_SET_FULL_BODY_META: Lazy<xr::BodyJointSetFB> =
+pub static BODY_JOINT_SET_FULL_BODY_META: Lazy<xr::BodyJointSetFB> =
     Lazy::new(|| xr::BodyJointSetFB::from_raw(1000274000));
 
 pub const FULL_BODY_JOINT_LEFT_UPPER_LEG_META: usize = 70;

alvr/client_openxr/src/extra_extensions/mod.rs

@@ -2,15 +2,17 @@ mod body_tracking_fb;
 mod eye_tracking_social;
 mod face_tracking2_fb;
 mod facial_tracking_htc;
+mod multimodal_input;
 
 pub use body_tracking_fb::*;
 pub use eye_tracking_social::*;
 pub use face_tracking2_fb::*;
 pub use facial_tracking_htc::*;
+pub use multimodal_input::*;
 
 use alvr_common::anyhow::{anyhow, Result};
 use openxr::{self as xr, sys};
-use std::ptr;
+use std::{mem, ptr};
 
 fn to_any(result: sys::Result) -> Result<()> {
     if result.into_raw() >= 0 {
@@ -24,13 +26,28 @@ fn to_any(result: sys::Result) -> Result<()> {
 pub struct ExtraExtensions {
     base_function_ptrs: xr::raw::Instance,
     ext_functions_ptrs: xr::InstanceExtensions,
+    resume_simultaneous_hands_and_controllers_tracking_meta:
+        Option<ResumeSimultaneousHandsAndControllersTrackingMETA>,
 }
 
 impl ExtraExtensions {
     pub fn new(instance: &xr::Instance) -> Self {
+        let resume_simultaneous_hands_and_controllers_tracking_meta = unsafe {
+            let mut resume_simultaneous_hands_and_controllers_tracking_meta = None;
+            let _ = (instance.fp().get_instance_proc_addr)(
+                instance.as_raw(),
+                c"xrResumeSimultaneousHandsAndControllersTrackingMETA".as_ptr(),
+                &mut resume_simultaneous_hands_and_controllers_tracking_meta,
+            );
+
+            resume_simultaneous_hands_and_controllers_tracking_meta
+                .map(|f| mem::transmute::<_, ResumeSimultaneousHandsAndControllersTrackingMETA>(f))
+        };
+
         Self {
             base_function_ptrs: instance.fp().clone(),
             ext_functions_ptrs: *instance.exts(),
+            resume_simultaneous_hands_and_controllers_tracking_meta,
         }
     }
 

alvr/client_openxr/src/extra_extensions/multimodal_input.rs

@@ -0,0 +1,72 @@
+// Code taken from:
+// https://github.com/meta-quest/Meta-OpenXR-SDK/blob/main/OpenXR/meta_openxr_preview/meta_simultaneous_hands_and_controllers.h
+
+use alvr_common::{anyhow::Result, once_cell::sync::Lazy, ToAny};
+use openxr::{self as xr, sys};
+use std::ffi::c_void;
+
+pub const META_SIMULTANEOUS_HANDS_AND_CONTROLLERS_EXTENSION_NAME: &str =
+    "XR_META_simultaneous_hands_and_controllers";
+pub const META_DETACHED_CONTROLLERS_EXTENSION_NAME: &str = "XR_META_detached_controllers";
+
+static TYPE_SYSTEM_SIMULTANEOUS_HANDS_AND_CONTROLLERS_PROPERTIES_META: Lazy<xr::StructureType> =
+    Lazy::new(|| xr::StructureType::from_raw(1000532001));
+static TYPE_SIMULTANEOUS_HANDS_AND_CONTROLLERS_TRACKING_RESUME_INFO_META: Lazy<xr::StructureType> =
+    Lazy::new(|| xr::StructureType::from_raw(1000532002));
+
+#[repr(C)]
+pub struct SystemSymultaneousHandsAndControllersPropertiesMETA {
+    ty: xr::StructureType,
+    next: *const c_void,
+    supports_simultaneous_hands_and_controllers: sys::Bool32,
+}
+
+#[repr(C)]
+pub struct SimultaneousHandsAndControllersTrackingResumeInfoMETA {
+    ty: xr::StructureType,
+    next: *const c_void,
+}
+
+pub type ResumeSimultaneousHandsAndControllersTrackingMETA =
+    unsafe extern "system" fn(
+        sys::Session,
+        *const SimultaneousHandsAndControllersTrackingResumeInfoMETA,
+    ) -> sys::Result;
+
+impl super::ExtraExtensions {
+    pub fn supports_simultaneous_hands_and_controllers(
+        &self,
+        instance: &xr::Instance,
+        system: xr::SystemId,
+    ) -> bool {
+        self.get_props(
+            instance,
+            system,
+            SystemSymultaneousHandsAndControllersPropertiesMETA {
+                ty: *TYPE_SYSTEM_SIMULTANEOUS_HANDS_AND_CONTROLLERS_PROPERTIES_META,
+                next: std::ptr::null(),
+                supports_simultaneous_hands_and_controllers: xr::sys::FALSE,
+            },
+        )
+        .map(|props| props.supports_simultaneous_hands_and_controllers.into())
+        .unwrap_or(false)
+    }
+
+    pub fn resume_simultaneous_hands_and_controllers_tracking(
+        &self,
+        session: &xr::Session<xr::OpenGlEs>,
+    ) -> Result<()> {
+        let resume_info = SimultaneousHandsAndControllersTrackingResumeInfoMETA {
+            ty: *TYPE_SIMULTANEOUS_HANDS_AND_CONTROLLERS_TRACKING_RESUME_INFO_META,
+            next: std::ptr::null(),
+        };
+
+        unsafe {
+            super::to_any((self
+                .resume_simultaneous_hands_and_controllers_tracking_meta
+                .to_any()?)(session.as_raw(), &resume_info))?;
+        }
+
+        Ok(())
+    }
+}