feat: Add support for Pico headsets face tracking

alvr/client_openxr/Cargo.toml

@@ -163,6 +163,8 @@ value = "3,6DoF"
 # Pico entries
 [[package.metadata.android.uses_permission]]
 name = "com.picovr.permission.EYE_TRACKING"
+[[package.metadata.android.uses_permission]]
+name = "com.picovr.permission.FACE_TRACKING"
 [[package.metadata.android.application.meta_data]]
 name = "eyetracking_calibration"
 value = "true"
@@ -173,6 +175,9 @@ value = "1"
 name = "picovr.software.eye_tracking"
 value = "1"
 [[package.metadata.android.application.meta_data]]
+name = "picovr.software.face_tracking"
+value = "true"
+[[package.metadata.android.application.meta_data]]
 name = "pvr.app.type"
 value = "vr"
 [[package.metadata.android.application.meta_data]]

alvr/client_openxr/src/extra_extensions/face_tracking_pico.rs

@@ -0,0 +1,161 @@
+use openxr::sys::pfn::VoidFunction;
+use openxr::{self as xr, sys};
+use std::mem;
+
+const TRACKING_MODE_FACE_BIT: u64 = 0x00000008;
+const TRACKING_MODE_FACE_LIPSYNC: u64 = 0x00002000;
+const TRACKING_MODE_FACE_LIPSYNC_BLEND_SHAPES: u64 = 0x00000100;
+
+#[repr(C)]
+struct FaceTrackingDataPICO {
+    time: sys::Time,
+    blend_shape_weight: [f32; 72],
+    is_video_input_valid: [f32; 10],
+    laughing_probability: f32,
+    emotion_probability: [f32; 10],
+    reserved: [f32; 128],
+}
+
+type StartEyeTrackingPICO = unsafe extern "system" fn(sys::Session) -> sys::Result;
+
+type StopEyeTrackingPICO = unsafe extern "system" fn(sys::Session, u64) -> sys::Result;
+
+type SetTrackingModePICO = unsafe extern "system" fn(sys::Session, u64) -> sys::Result;
+
+type GetFaceTrackingDataPICO = unsafe extern "system" fn(
+    sys::Session,
+    sys::Time,
+    i32,
+    *mut FaceTrackingDataPICO,
+) -> sys::Result;
+
+pub struct FaceTrackerPico {
+    session: xr::Session<xr::AnyGraphics>,
+    tracking_flags: u64,
+    start_eye_tracking: StartEyeTrackingPICO,
+    stop_eye_tracking: StopEyeTrackingPICO,
+    set_tracking_mode: SetTrackingModePICO,
+    get_face_tracking_data: GetFaceTrackingDataPICO,
+}
+
+impl FaceTrackerPico {
+    pub fn new<G>(session: xr::Session<G>, visual: bool, audio: bool) -> xr::Result<Self> {
+        session
+            .instance()
+            .exts()
+            .ext_eye_gaze_interaction
+            .ok_or(sys::Result::ERROR_EXTENSION_NOT_PRESENT)?;
+
+        let start_eye_tracking = unsafe {
+            let mut start_eye_tracking = None;
+            let _ = (session.instance().fp().get_instance_proc_addr)(
+                session.instance().as_raw(),
+                c"xrStartEyeTrackingPICO".as_ptr(),
+                &mut start_eye_tracking,
+            );
+
+            start_eye_tracking.map(|pfn| mem::transmute::<VoidFunction, StartEyeTrackingPICO>(pfn))
+        }
+        .ok_or(sys::Result::ERROR_EXTENSION_NOT_PRESENT)?;
+
+        let stop_eye_tracking = unsafe {
+            let mut stop_eye_tracking = None;
+            let _ = (session.instance().fp().get_instance_proc_addr)(
+                session.instance().as_raw(),
+                c"xrStopEyeTrackingPICO".as_ptr(),
+                &mut stop_eye_tracking,
+            );
+
+            stop_eye_tracking.map(|pfn| mem::transmute::<VoidFunction, StopEyeTrackingPICO>(pfn))
+        }
+        .ok_or(sys::Result::ERROR_EXTENSION_NOT_PRESENT)?;
+
+        let set_tracking_mode = unsafe {
+            let mut set_tracking_mode = None;
+            let _ = (session.instance().fp().get_instance_proc_addr)(
+                session.instance().as_raw(),
+                c"xrSetTrackingModePICO".as_ptr(),
+                &mut set_tracking_mode,
+            );
+
+            set_tracking_mode.map(|pfn| mem::transmute::<VoidFunction, SetTrackingModePICO>(pfn))
+        }
+        .ok_or(sys::Result::ERROR_EXTENSION_NOT_PRESENT)?;
+
+        let get_face_tracking_data = unsafe {
+            let mut get_face_tracking_data = None;
+            let _ = (session.instance().fp().get_instance_proc_addr)(
+                session.instance().as_raw(),
+                c"xrGetFaceTrackingDataPICO".as_ptr(),
+                &mut get_face_tracking_data,
+            );
+
+            get_face_tracking_data
+                .map(|pfn| mem::transmute::<VoidFunction, GetFaceTrackingDataPICO>(pfn))
+        }
+        .ok_or(sys::Result::ERROR_EXTENSION_NOT_PRESENT)?;
+
+        let mut tracking_flags = 0;
+
+        if visual {
+            tracking_flags |= TRACKING_MODE_FACE_BIT;
+        }
+        if audio {
+            tracking_flags |= TRACKING_MODE_FACE_LIPSYNC | TRACKING_MODE_FACE_LIPSYNC_BLEND_SHAPES;
+        }
+
+        Ok(Self {
+            session: session.into_any_graphics(),
+            tracking_flags,
+            start_eye_tracking,
+            stop_eye_tracking,
+            set_tracking_mode,
+            get_face_tracking_data,
+        })
+    }
+
+    pub fn get_face_tracking_data(&self, time: xr::Time) -> xr::Result<Option<Vec<f32>>> {
+        let mut face_tracking_data = FaceTrackingDataPICO {
+            time: xr::Time::from_nanos(0),
+            blend_shape_weight: [0.0; 72],
+            is_video_input_valid: [0.0; 10],
+            laughing_probability: 0.0,
+            emotion_probability: [0.0; 10],
+            reserved: [0.0; 128],
+        };
+
+        unsafe {
+            super::xr_res((self.get_face_tracking_data)(
+                self.session.as_raw(),
+                time,
+                0,
+                &mut face_tracking_data,
+            ))?;
+
+            if face_tracking_data.time.as_nanos() != 0 {
+                Ok(Some(face_tracking_data.blend_shape_weight.to_vec()))
+            } else {
+                Ok(None)
+            }
+        }
+    }
+
+    pub fn start_face_tracking(&self) -> xr::Result<()> {
+        unsafe {
+            super::xr_res((self.start_eye_tracking)(self.session.as_raw()))?;
+            super::xr_res((self.set_tracking_mode)(
+                self.session.as_raw(),
+                self.tracking_flags,
+            ))
+        }
+    }
+
+    pub fn stop_face_tracking(&self) -> xr::Result<()> {
+        unsafe {
+            super::xr_res((self.stop_eye_tracking)(
+                self.session.as_raw(),
+                self.tracking_flags,
+            ))
+        }
+    }
+}

alvr/client_openxr/src/extra_extensions/mod.rs

@@ -2,6 +2,7 @@ mod body_tracking_fb;
 mod eye_gaze_interaction;
 mod eye_tracking_social;
 mod face_tracking2_fb;
+mod face_tracking_pico;
 mod facial_tracking_htc;
 mod multimodal_input;
 mod passthrough_fb;
@@ -11,6 +12,7 @@ pub use body_tracking_fb::*;
 pub use eye_gaze_interaction::*;
 pub use eye_tracking_social::*;
 pub use face_tracking2_fb::*;
+pub use face_tracking_pico::*;
 pub use facial_tracking_htc::*;
 pub use multimodal_input::*;
 pub use passthrough_fb::*;

alvr/client_openxr/src/interaction.rs

@@ -1,7 +1,7 @@
 use crate::{
     extra_extensions::{
-        self, BodyTrackerFB, EyeTrackerSocial, FaceTracker2FB, FacialTrackerHTC, MultimodalMeta,
-        BODY_JOINT_SET_FULL_BODY_META, FULL_BODY_JOINT_COUNT_META,
+        self, BodyTrackerFB, EyeTrackerSocial, FaceTracker2FB, FaceTrackerPico, FacialTrackerHTC,
+        MultimodalMeta, BODY_JOINT_SET_FULL_BODY_META, FULL_BODY_JOINT_COUNT_META,
         FULL_BODY_JOINT_LEFT_FOOT_BALL_META, FULL_BODY_JOINT_LEFT_LOWER_LEG_META,
         FULL_BODY_JOINT_RIGHT_FOOT_BALL_META, FULL_BODY_JOINT_RIGHT_LOWER_LEG_META,
     },
@@ -63,6 +63,7 @@ pub struct FaceSources {
     pub face_tracker_fb: Option<FaceTracker2FB>,
     pub eye_tracker_htc: Option<FacialTrackerHTC>,
     pub lip_tracker_htc: Option<FacialTrackerHTC>,
+    pub face_tracker_pico: Option<FaceTrackerPico>,
 }
 
 pub struct BodySources {
@@ -368,6 +369,7 @@ impl InteractionContext {
                 face_tracker_fb: None,
                 eye_tracker_htc: None,
                 lip_tracker_htc: None,
+                face_tracker_pico: None,
             },
             body_sources: BodySources {
                 body_tracker_fb: None,
@@ -380,11 +382,15 @@ impl InteractionContext {
         if let Some(handle) = &mut self.multimodal_handle {
             handle.pause().ok();
         }
+        if let Some(face_tracker) = &self.face_sources.face_tracker_pico {
+            face_tracker.stop_face_tracking().ok();
+        }
         self.multimodal_hands_enabled = false;
         self.face_sources.eye_tracker_fb = None;
         self.face_sources.face_tracker_fb = None;
         self.face_sources.eye_tracker_htc = None;
         self.face_sources.lip_tracker_htc = None;
+        self.face_sources.face_tracker_pico = None;
         self.body_sources.body_tracker_fb = None;
 
         // todo: check which permissions are needed for htc
@@ -400,6 +406,13 @@ impl InteractionContext {
                     alvr_system_info::try_get_permission("com.oculus.permission.FACE_TRACKING")
                 }
             }
+            if config.face_tracking_pico && self.platform.is_pico() {
+                #[cfg(target_os = "android")]
+                {
+                    alvr_system_info::try_get_permission("android.permission.RECORD_AUDIO");
+                    alvr_system_info::try_get_permission("com.picovr.permission.FACE_TRACKING")
+                }
+            }
         }
 
         if let Some(config) = &config.body_tracking {
@@ -434,6 +447,12 @@ impl InteractionContext {
             || FaceTracker2FB::new(&self.xr_session, true, true),
         );
 
+        self.face_sources.face_tracker_pico = create_ext_object(
+            "FaceTrackerPico",
+            config.face_tracking.as_ref().map(|s| s.face_tracking_pico),
+            || FaceTrackerPico::new(self.xr_session.clone(), true, true),
+        );
+
         self.face_sources.eye_tracker_htc = create_ext_object(
             "FacialTrackerHTC (eyes)",
             config.face_tracking.as_ref().map(|s| s.eye_expressions_htc),
@@ -467,6 +486,10 @@ impl InteractionContext {
             )
             .map(|tracker| (tracker, xr::BodyJointFB::COUNT.into_raw() as usize))
         });
+
+        if let Some(face_tracker) = &self.face_sources.face_tracker_pico {
+            face_tracker.start_face_tracking().ok();
+        }
     }
 }
 
@@ -790,6 +813,16 @@ pub fn get_fb_face_expression(context: &FaceSources, time: Duration) -> Option<V
         .map(|weights| weights.into_iter().collect())
 }
 
+pub fn get_pico_face_expression(context: &FaceSources, time: Duration) -> Option<Vec<f32>> {
+    let xr_time = crate::to_xr_time(time);
+
+    context
+        .face_tracker_pico
+        .as_ref()
+        .and_then(|t| t.get_face_tracking_data(xr_time).ok().flatten())
+        .map(|weights| weights.into_iter().collect())
+}
+
 pub fn get_htc_eye_expression(context: &FaceSources) -> Option<Vec<f32>> {
     context
         .eye_tracker_htc

alvr/client_openxr/src/stream.rs

@@ -528,7 +528,9 @@ fn stream_input_loop(
                 stage_reference_space,
                 now,
             ),
-            fb_face_expression: interaction::get_fb_face_expression(&int_ctx.face_sources, now),
+            fb_face_expression: interaction::get_fb_face_expression(&int_ctx.face_sources, now).or(
+                interaction::get_pico_face_expression(&int_ctx.face_sources, now),
+            ),
             htc_eye_expression: interaction::get_htc_eye_expression(&int_ctx.face_sources),
             htc_lip_expression: interaction::get_htc_lip_expression(&int_ctx.face_sources),
         };

alvr/server_core/src/tracking/face.rs

@@ -8,6 +8,9 @@ const RAD_TO_DEG: f32 = 180.0 / PI;
 
 const VRCFT_PORT: u16 = 0xA1F7;
 
+const FB_FACE_EXPRESSION_COUNT: usize = 70;
+const PICO_FACE_EXPRESSION_COUNT: usize = 72;
+
 pub struct FaceTrackingSink {
     config: FaceTrackingSinkConfig,
     socket: UdpSocket,
@@ -62,6 +65,21 @@ impl FaceTrackingSink {
     }
 
     pub fn send_tracking(&mut self, face_data: FaceData) {
+        // todo: introduce pico_face_expression field in FaceData
+        let fb_face_expression = match &face_data.fb_face_expression {
+            Some(face_expression) if face_expression.len() == FB_FACE_EXPRESSION_COUNT => {
+                Some(face_expression)
+            }
+            _ => None,
+        };
+
+        let pico_face_expression = match &face_data.fb_face_expression {
+            Some(face_expression) if face_expression.len() == PICO_FACE_EXPRESSION_COUNT => {
+                Some(face_expression)
+            }
+            _ => None,
+        };
+
         match self.config {
             FaceTrackingSinkConfig::VrchatEyeOsc { .. } => {
                 if let [Some(left), Some(right)] = face_data.eye_gazes {
@@ -89,15 +107,14 @@ impl FaceTrackingSink {
                     );
                 }
 
-                let left_eye_blink = face_data
-                    .fb_face_expression
-                    .as_ref()
+                let left_eye_blink = fb_face_expression
                     .map(|v| v[12])
-                    .or_else(|| face_data.htc_eye_expression.as_ref().map(|v| v[0]));
-                let right_eye_blink = face_data
-                    .fb_face_expression
+                    .or_else(|| face_data.htc_eye_expression.as_ref().map(|v| v[0]))
+                    .or_else(|| pico_face_expression.map(|v| v[28]));
+                let right_eye_blink = fb_face_expression
                     .map(|v| v[13])
-                    .or_else(|| face_data.htc_eye_expression.map(|v| v[2]));
+                    .or_else(|| face_data.htc_eye_expression.as_ref().map(|v| v[2]))
+                    .or_else(|| pico_face_expression.map(|v| v[38]));
 
                 if let (Some(left), Some(right)) = (left_eye_blink, right_eye_blink) {
                     self.send_osc_message(
@@ -130,8 +147,12 @@ impl FaceTrackingSink {
                     _ => (),
                 }
 
-                if let Some(arr) = face_data.fb_face_expression {
-                    self.append_packet_vrcft(b"Face2Fb\0", &arr);
+                if let Some(arr) = fb_face_expression {
+                    self.append_packet_vrcft(b"Face2Fb\0", arr);
+                }
+
+                if let Some(arr) = pico_face_expression {
+                    self.append_packet_vrcft(b"FacePico", arr);
                 }
 
                 if let Some(arr) = face_data.htc_eye_expression {

alvr/session/src/settings.rs

@@ -737,6 +737,7 @@ pub struct FaceTrackingSourcesConfig {
     pub face_tracking_fb: bool,
     pub eye_expressions_htc: bool,
     pub lip_expressions_htc: bool,
+    pub face_tracking_pico: bool,
 }
 
 #[derive(SettingsSchema, Serialize, Deserialize, Clone)]
@@ -1672,6 +1673,7 @@ pub fn session_settings_default() -> SettingsDefault {
                         face_tracking_fb: true,
                         eye_expressions_htc: true,
                         lip_expressions_htc: true,
+                        face_tracking_pico: true,
                     },
                     sink: FaceTrackingSinkConfigDefault {
                         VrchatEyeOsc: FaceTrackingSinkConfigVrchatEyeOscDefault { port: 9000 },