refactor(client_openxr): ♻️ Move extra xr ext wrappers in-tree; show …

…body skeleton in lobby (#2366)

alvr/client_core/src/c_api.rs

@@ -812,7 +812,7 @@ pub unsafe extern "C" fn alvr_render_lobby_opengl(view_inputs: *const AlvrViewIn
 
     LOBBY_RENDERER.with_borrow(|renderer| {
         if let Some(renderer) = renderer {
-            renderer.render(view_inputs, [(None, None), (None, None)]);
+            renderer.render(view_inputs, [(None, None), (None, None)], None);
         }
     });
 }

alvr/client_core/src/graphics/lobby.rs

@@ -1,6 +1,6 @@
 use super::{GraphicsContext, SDR_FORMAT};
 use alvr_common::{
-    glam::{Mat4, UVec2, Vec3, Vec4},
+    glam::{Mat4, Quat, UVec2, Vec3, Vec4},
     Fov, Pose,
 };
 use glyph_brush_layout::{
@@ -53,6 +53,44 @@ const HAND_SKELETON_BONES: [(usize, usize); 19] = [
     (24, 25),
 ];
 
+const BODY_SKELETON_BONES_FB: [(usize, usize); 30] = [
+    // Spine
+    (1, 2),
+    (2, 3),
+    (3, 4),
+    (4, 5),
+    (5, 6),
+    (6, 7),
+    // Left arm
+    (5, 8),
+    (8, 9),
+    (9, 10),
+    (10, 11),
+    (11, 12),
+    // Right arm
+    (5, 13),
+    (13, 14),
+    (14, 15),
+    (15, 16),
+    (16, 17),
+    // Left leg
+    (1, 70),
+    (70, 71),
+    (71, 72),
+    (72, 73),
+    (73, 74),
+    (74, 75),
+    (75, 76),
+    // Right leg
+    (1, 77),
+    (77, 78),
+    (78, 79),
+    (79, 80),
+    (80, 81),
+    (81, 82),
+    (82, 83),
+];
+
 fn projection_from_fov(fov: Fov) -> Mat4 {
     const NEAR: f32 = 0.1;
 
@@ -306,7 +344,8 @@ impl LobbyRenderer {
     pub fn render(
         &self,
         view_inputs: [RenderViewInput; 2],
-        hand_poses: [(Option<Pose>, Option<[Pose; 26]>); 2],
+        hand_data: [(Option<Pose>, Option<[Pose; 26]>); 2],
+        body_skeleton_fb: Option<Vec<Option<Pose>>>,
     ) {
         let mut encoder = self
             .context
@@ -376,7 +415,7 @@ impl LobbyRenderer {
 
             // Bind line pipeline and render hands
             pass.set_pipeline(&self.line_pipeline);
-            for (maybe_pose, maybe_skeleton) in &hand_poses {
+            for (maybe_pose, maybe_skeleton) in &hand_data {
                 if let Some(skeleton) = maybe_skeleton {
                     for (joint1_idx, joint2_idx) in HAND_SKELETON_BONES {
                         let j1_pose = skeleton[joint1_idx];
@@ -410,6 +449,23 @@ impl LobbyRenderer {
                     }
                 }
             }
+            if let Some(skeleton) = &body_skeleton_fb {
+                for (joint1_idx, joint2_idx) in BODY_SKELETON_BONES_FB {
+                    if let (Some(Some(j1_pose)), Some(Some(j2_pose))) =
+                        (skeleton.get(joint1_idx), skeleton.get(joint2_idx))
+                    {
+                        let transform = Mat4::from_scale_rotation_translation(
+                            Vec3::ONE * Vec3::distance(j1_pose.position, j2_pose.position),
+                            Quat::from_rotation_arc(
+                                -Vec3::Z,
+                                (j2_pose.position - j1_pose.position).normalize(),
+                            ),
+                            j1_pose.position,
+                        );
+                        transform_draw(&mut pass, view_proj * transform, 2);
+                    }
+                }
+            }
         }
 
         self.context.queue.submit(Some(encoder.finish()));

alvr/client_openxr/Cargo.toml

@@ -15,8 +15,7 @@ alvr_client_core.workspace = true
 alvr_packets.workspace = true
 alvr_session.workspace = true
 
-openxr = { git = "https://github.com/alvr-org/openxrs", rev = "f1f47ff" }
-# openxr = { path = "../../../../openxrs/openxr" }
+openxr = { git = "https://github.com/Ralith/openxrs" }
 
 [target.'cfg(target_os = "android")'.dependencies]
 android-activity = { version = "0.6", features = ["native-activity"] }
@@ -25,7 +24,7 @@ libc = "0.2"
 ndk-context = "0.1"
 
 [package.metadata.android]
-package = "alvr.client.dev" # Changed for Meta Store
+package = "alvr.client.dev"                # Changed for Meta Store
 install_location = "auto"
 build_targets = ["aarch64-linux-android"]
 runtime_libs = "../../deps/android_openxr"
@@ -125,7 +124,7 @@ value = "vr_only"
 name = "com.oculus.supportedDevices"
 # Note: value is changed for the Meta store, which requires an explicit list of platforms.
 # "all" is required to support Quest 1 which doesn't have newer platform names registered.
-value = "all" 
+value = "all"
 [[package.metadata.android.application.meta_data]]
 name = "com.oculus.vr.focusaware"
 value = "true"

alvr/client_openxr/src/extra_extensions/body_tracking_fb.rs

@@ -0,0 +1,147 @@
+#![allow(dead_code)]
+
+use alvr_common::{anyhow::Result, once_cell::sync::Lazy, ToAny};
+use openxr::{self as xr, raw, sys};
+use std::ptr;
+
+pub const META_BODY_TRACKING_FULL_BODY_EXTENSION_NAME: &str = "XR_META_body_tracking_full_body";
+pub const TYPE_SYSTEM_PROPERTIES_BODY_TRACKING_FULL_BODY_META: Lazy<xr::StructureType> =
+    Lazy::new(|| xr::StructureType::from_raw(1000274000));
+pub const BODY_JOINT_SET_FULL_BODY_META: Lazy<xr::BodyJointSetFB> =
+    Lazy::new(|| xr::BodyJointSetFB::from_raw(1000274000));
+
+pub const FULL_BODY_JOINT_LEFT_UPPER_LEG_META: usize = 70;
+pub const FULL_BODY_JOINT_LEFT_LOWER_LEG_META: usize = 71;
+pub const FULL_BODY_JOINT_LEFT_FOOT_ANKLE_TWIST_META: usize = 72;
+pub const FULL_BODY_JOINT_LEFT_FOOT_ANKLE_META: usize = 73;
+pub const FULL_BODY_JOINT_LEFT_FOOT_SUBTALAR_META: usize = 74;
+pub const FULL_BODY_JOINT_LEFT_FOOT_TRANSVERSE_META: usize = 75;
+pub const FULL_BODY_JOINT_LEFT_FOOT_BALL_META: usize = 76;
+pub const FULL_BODY_JOINT_RIGHT_UPPER_LEG_META: usize = 77;
+pub const FULL_BODY_JOINT_RIGHT_LOWER_LEG_META: usize = 78;
+pub const FULL_BODY_JOINT_RIGHT_FOOT_ANKLE_TWIST_META: usize = 79;
+pub const FULL_BODY_JOINT_RIGHT_FOOT_ANKLE_META: usize = 80;
+pub const FULL_BODY_JOINT_RIGHT_FOOT_SUBTALAR_META: usize = 81;
+pub const FULL_BODY_JOINT_RIGHT_FOOT_TRANSVERSE_META: usize = 82;
+pub const FULL_BODY_JOINT_RIGHT_FOOT_BALL_META: usize = 83;
+pub const FULL_BODY_JOINT_COUNT_META: usize = 84;
+
+#[repr(C)]
+struct SystemPropertiesBodyTrackingFullBodyMETA {
+    ty: xr::StructureType,
+    next: *mut std::ffi::c_void,
+    supports_full_body_tracking: sys::Bool32,
+}
+
+impl super::ExtraExtensions {
+    pub fn supports_body_tracking_fb(&self, instance: &xr::Instance, system: xr::SystemId) -> bool {
+        self.get_props(
+            instance,
+            system,
+            sys::SystemBodyTrackingPropertiesFB {
+                ty: sys::SystemBodyTrackingPropertiesFB::TYPE,
+                next: ptr::null_mut(),
+                supports_body_tracking: sys::FALSE,
+            },
+        )
+        .map(|props| props.supports_body_tracking.into())
+        .unwrap_or(false)
+    }
+
+    pub fn supports_full_body_tracking_meta(
+        &self,
+        instance: &xr::Instance,
+        system: xr::SystemId,
+    ) -> bool {
+        self.get_props(
+            instance,
+            system,
+            SystemPropertiesBodyTrackingFullBodyMETA {
+                ty: *TYPE_SYSTEM_PROPERTIES_BODY_TRACKING_FULL_BODY_META,
+                next: ptr::null_mut(),
+                supports_full_body_tracking: sys::FALSE,
+            },
+        )
+        .map(|props| props.supports_full_body_tracking.into())
+        .unwrap_or(false)
+    }
+
+    pub fn create_body_tracker_fb<G>(
+        &self,
+        session: &xr::Session<G>,
+        body_joint_set: xr::BodyJointSetFB,
+    ) -> Result<BodyTrackerFB> {
+        let ext_fns = self.ext_functions_ptrs.fb_body_tracking.to_any()?;
+
+        let mut handle = sys::BodyTrackerFB::NULL;
+        let info = sys::BodyTrackerCreateInfoFB {
+            ty: sys::BodyTrackerCreateInfoFB::TYPE,
+            next: ptr::null(),
+            body_joint_set,
+        };
+        unsafe {
+            super::to_any((ext_fns.create_body_tracker)(
+                session.as_raw(),
+                &info,
+                &mut handle,
+            ))?
+        };
+
+        Ok(BodyTrackerFB { handle, ext_fns })
+    }
+}
+
+pub struct BodyTrackerFB {
+    handle: sys::BodyTrackerFB,
+    ext_fns: raw::BodyTrackingFB,
+}
+
+impl BodyTrackerFB {
+    pub fn locate_body_joints(
+        &self,
+        time: xr::Time,
+        reference_space: &xr::Space,
+        joint_count: usize,
+    ) -> Result<Option<Vec<xr::BodyJointLocationFB>>> {
+        let locate_info = sys::BodyJointsLocateInfoFB {
+            ty: sys::BodyJointsLocateInfoFB::TYPE,
+            next: ptr::null(),
+            base_space: reference_space.as_raw(),
+            time,
+        };
+        let mut locations = Vec::with_capacity(joint_count);
+        let mut location_info = sys::BodyJointLocationsFB {
+            ty: sys::BodyJointLocationsFB::TYPE,
+            next: ptr::null_mut(),
+            is_active: sys::FALSE,
+            confidence: 0.0,
+            joint_count: joint_count as u32,
+            joint_locations: locations.as_mut_ptr() as _,
+            skeleton_changed_count: 0,
+            time: xr::Time::from_nanos(0),
+        };
+        unsafe {
+            super::to_any((self.ext_fns.locate_body_joints)(
+                self.handle,
+                &locate_info,
+                &mut location_info,
+            ))?;
+
+            Ok(if location_info.is_active.into() {
+                locations.set_len(joint_count);
+
+                Some(locations)
+            } else {
+                None
+            })
+        }
+    }
+}
+
+impl Drop for BodyTrackerFB {
+    fn drop(&mut self) {
+        unsafe {
+            (self.ext_fns.destroy_body_tracker)(self.handle);
+        }
+    }
+}

alvr/client_openxr/src/extra_extensions/eye_tracking_social.rs

@@ -0,0 +1,87 @@
+use alvr_common::{anyhow::Result, ToAny};
+use openxr::{self as xr, raw, sys};
+use std::ptr;
+
+impl super::ExtraExtensions {
+    pub fn supports_social_eye_tracking(
+        &self,
+        instance: &xr::Instance,
+        system: xr::SystemId,
+    ) -> bool {
+        self.get_props(instance, system, unsafe {
+            sys::SystemEyeTrackingPropertiesFB::out(ptr::null_mut()).assume_init()
+        })
+        .map(|props| props.supports_eye_tracking.into())
+        .unwrap_or(false)
+    }
+
+    pub fn create_eye_tracker_social<G>(
+        &self,
+        session: xr::Session<G>,
+    ) -> Result<EyeTrackerSocial> {
+        let ext_fns = self.ext_functions_ptrs.fb_eye_tracking_social.to_any()?;
+
+        let mut handle = sys::EyeTrackerFB::NULL;
+        let info = sys::EyeTrackerCreateInfoFB {
+            ty: sys::EyeTrackerCreateInfoFB::TYPE,
+            next: ptr::null(),
+        };
+        unsafe {
+            super::to_any((ext_fns.create_eye_tracker)(
+                session.as_raw(),
+                &info,
+                &mut handle,
+            ))?
+        };
+
+        Ok(EyeTrackerSocial { handle, ext_fns })
+    }
+}
+
+pub struct EyeTrackerSocial {
+    handle: sys::EyeTrackerFB,
+    ext_fns: raw::EyeTrackingSocialFB,
+}
+
+impl EyeTrackerSocial {
+    pub fn get_eye_gazes(
+        &self,
+        base: &xr::Space,
+        time: xr::Time,
+    ) -> Result<[Option<xr::Posef>; 2]> {
+        let gaze_info = sys::EyeGazesInfoFB {
+            ty: sys::EyeGazesInfoFB::TYPE,
+            next: ptr::null(),
+            base_space: base.as_raw(),
+            time,
+        };
+
+        let mut eye_gazes = sys::EyeGazesFB::out(ptr::null_mut());
+
+        let eye_gazes = unsafe {
+            super::to_any((self.ext_fns.get_eye_gazes)(
+                self.handle,
+                &gaze_info,
+                eye_gazes.as_mut_ptr(),
+            ))?;
+
+            eye_gazes.assume_init()
+        };
+
+        let left_valid: bool = eye_gazes.gaze[0].is_valid.into();
+        let right_valid: bool = eye_gazes.gaze[1].is_valid.into();
+
+        Ok([
+            left_valid.then(|| eye_gazes.gaze[0].gaze_pose),
+            right_valid.then(|| eye_gazes.gaze[1].gaze_pose),
+        ])
+    }
+}
+
+impl Drop for EyeTrackerSocial {
+    fn drop(&mut self) {
+        unsafe {
+            (self.ext_fns.destroy_eye_tracker)(self.handle);
+        }
+    }
+}