Support separate OpenVR hand trackers for VRChat (#2295)

* refactoring: around button register and openvr prop * feat: full skeletal level toggle button. * fix: incorrect import * chore: format * fix: incorrect typing * feat: VRSkeletalTrackingLevel Auto Switch * fix: renamed config item to match functionality * fix: code formatting * fix: support for independent skeletal tracking settings per-hand * Revert "refactoring: around button register and openvr prop" This reverts commit 8704c99. * feat: implement a full skeletal virtual hand * chore: format * Small refactor * Disable gestures by default and remove setup wizard page * Send button inputs to the active hand device * Don't update skeleton if device is disabled * Fix buttons not registered for hand trackers * Add Hand tracking interaction preset * Add comment for device switching * Edit help for use_separate_trackers --------- Co-authored-by: Reina_Sakiria <[email protected]> Co-authored-by: Adalyn Black <[email protected]>
alvr-org · Aug 15, 2024 · 31885f8 · 31885f8
1 parent ff6225d
commit 31885f8
Show file tree

Hide file tree

Showing 15 changed files with 293 additions and 128 deletions.
diff --git a/alvr/common/src/inputs.rs b/alvr/common/src/inputs.rs
@@ -44,6 +44,8 @@ devices! {
     (HEAD, "/user/head"),
     (HAND_LEFT, "/user/hand/left"),
     (HAND_RIGHT, "/user/hand/right"),
+    (HAND_TRACKER_LEFT,"/user/hand_tracker/left"),
+    (HAND_TRACKER_RIGHT, "/user/hand_tracker/right"),
     (BODY_CHEST, "/user/body/chest"),
     (BODY_HIPS, "/user/body/waist"),
     (BODY_LEFT_ELBOW, "/user/body/left_elbow"),

diff --git a/alvr/dashboard/src/dashboard/components/settings.rs b/alvr/dashboard/src/dashboard/components/settings.rs
@@ -29,6 +29,7 @@ pub struct SettingsTab {
     encoder_preset: PresetControl,
     game_audio_preset: Option<PresetControl>,
     microphone_preset: Option<PresetControl>,
+    hand_tracking_interaction_preset: PresetControl,
     eye_face_tracking_preset: PresetControl,
     top_level_entries: Vec<TopLevelEntry>,
     session_settings_json: Option<json::Value>,
@@ -71,6 +72,9 @@ impl SettingsTab {
             encoder_preset: PresetControl::new(builtin_schema::encoder_preset_schema()),
             game_audio_preset: None,
             microphone_preset: None,
+            hand_tracking_interaction_preset: PresetControl::new(
+                builtin_schema::hand_tracking_interaction_schema(),
+            ),
             eye_face_tracking_preset: PresetControl::new(builtin_schema::eye_face_tracking_schema()),
             top_level_entries,
             session_settings_json: None,
@@ -92,6 +96,8 @@ impl SettingsTab {
         if let Some(preset) = self.microphone_preset.as_mut() {
             preset.update_session_settings(&settings_json)
         }
+        self.hand_tracking_interaction_preset
+            .update_session_settings(&settings_json);
         self.eye_face_tracking_preset
             .update_session_settings(&settings_json);
 
@@ -179,6 +185,9 @@ impl SettingsTab {
                                 ui.end_row();
                             }
 
+                            path_value_pairs.extend(self.hand_tracking_interaction_preset.ui(ui));
+                            ui.end_row();
+
                             path_value_pairs.extend(self.eye_face_tracking_preset.ui(ui));
                             ui.end_row();
                         })

diff --git a/alvr/dashboard/src/dashboard/components/settings_controls/presets/builtin_schema.rs b/alvr/dashboard/src/dashboard/components/settings_controls/presets/builtin_schema.rs
@@ -319,12 +319,74 @@ pub fn microphone_schema(devices: Vec<String>) -> PresetSchemaNode {
     })
 }
 
+pub fn hand_tracking_interaction_schema() -> PresetSchemaNode {
+    const HELP: &str = r"Disabled: hands cannot emulate buttons. Useful for using Joy-Cons or other non-native controllers.
+Separate trackers: create separate SteamVR devices for hand tracking. This is used for VRChat.
+ALVR bindings: use ALVR hand tracking button bindings. Check the wiki for help.
+";
+
+    const PREFIX: &str = "session_settings.headset.controllers.content";
+
+    PresetSchemaNode::HigherOrderChoice(HigherOrderChoiceSchema {
+        name: "Hand tracking interaction".into(),
+        strings: [("help".into(), HELP.into())].into_iter().collect(),
+        flags: HashSet::new(),
+        options: [
+            HigherOrderChoiceOption {
+                display_name: "Disabled".into(),
+                modifiers: vec![
+                    bool_modifier("session_settings.headset.controllers.enabled", true),
+                    bool_modifier(
+                        &format!("{PREFIX}.hand_skeleton.content.use_separate_trackers"),
+                        false,
+                    ),
+                    bool_modifier(
+                        &format!("{PREFIX}.hand_tracking_interaction.enabled"),
+                        false,
+                    ),
+                ],
+                content: None,
+            },
+            HigherOrderChoiceOption {
+                display_name: "Separate trackers".into(),
+                modifiers: vec![
+                    bool_modifier("session_settings.headset.controllers.enabled", true),
+                    bool_modifier(&format!("{PREFIX}.hand_skeleton.enabled"), true),
+                    bool_modifier(
+                        &format!("{PREFIX}.hand_skeleton.content.use_separate_trackers"),
+                        true,
+                    ),
+                    bool_modifier(
+                        &format!("{PREFIX}.hand_tracking_interaction.enabled"),
+                        false,
+                    ),
+                ],
+                content: None,
+            },
+            HigherOrderChoiceOption {
+                display_name: "ALVR bindings".into(),
+                modifiers: vec![
+                    bool_modifier("session_settings.headset.controllers.enabled", true),
+                    bool_modifier(
+                        &format!("{PREFIX}.hand_skeleton.content.use_separate_trackers"),
+                        false,
+                    ),
+                    bool_modifier(&format!("{PREFIX}.hand_tracking_interaction.enabled"), true),
+                ],
+                content: None,
+            },
+        ]
+        .into_iter()
+        .collect(),
+        default_option_index: 1,
+        gui: ChoiceControlType::ButtonGroup,
+    })
+}
+
 pub fn eye_face_tracking_schema() -> PresetSchemaNode {
     PresetSchemaNode::HigherOrderChoice(HigherOrderChoiceSchema {
-        name: "eye_face_tracking".into(),
-        strings: [("display_name".into(), "Eye and face tracking".into())]
-            .into_iter()
-            .collect(),
+        name: "Eye and face tracking".into(),
+        strings: HashMap::new(),
         flags: HashSet::new(),
         options: [
             HigherOrderChoiceOption {

diff --git a/alvr/dashboard/src/dashboard/components/setup_wizard.rs b/alvr/dashboard/src/dashboard/components/setup_wizard.rs
@@ -1,5 +1,4 @@
-use crate::dashboard::basic_components;
-use alvr_packets::{FirewallRulesAction, PathValuePair, ServerRequest};
+use alvr_packets::{FirewallRulesAction, ServerRequest};
 use eframe::{
     egui::{Button, Label, Layout, RichText, Ui},
     emath::Align,
@@ -16,10 +15,9 @@ enum Page {
     ResetSettings = 1,
     HardwareRequirements = 2,
     SoftwareRequirements = 3,
-    HandGestures = 4,
-    Firewall = 5,
-    Recommendations = 6,
-    Finished = 7,
+    Firewall = 4,
+    Recommendations = 5,
+    Finished = 6,
 }
 
 fn index_to_page(index: usize) -> Page {
@@ -28,11 +26,10 @@ fn index_to_page(index: usize) -> Page {
         1 => Page::ResetSettings,
         2 => Page::HardwareRequirements,
         3 => Page::SoftwareRequirements,
-        4 => Page::HandGestures,
-        5 => Page::Firewall,
-        6 => Page::Recommendations,
-        7 => Page::Finished,
-        _ => unreachable!(),
+        4 => Page::Firewall,
+        5 => Page::Recommendations,
+        6 => Page::Finished,
+        _ => panic!("Invalid page index"),
     }
 }
 
@@ -59,14 +56,12 @@ fn page_content(
 
 pub struct SetupWizard {
     page: Page,
-    only_touch: bool,
 }
 
 impl SetupWizard {
     pub fn new() -> Self {
         Self {
             page: Page::Welcome,
-            only_touch: true,
         }
     }
 
@@ -132,27 +127,6 @@ Make sure you have at least one output audio device.",
                     }
                 },
             ),
-
-            Page::HandGestures => page_content(
-                ui,
-                "Hand Gestures",
-                r"ALVR allows you to use Hand Tracking and emulate controller buttons using it.
-By default, controller button emulation is set to prevent accidental clicks. You can re-enable gestures by disabling slider bellow.",
-                |ui| {
-                    ui.label("Only touch");
-                    if basic_components::switch(ui, &mut self.only_touch).changed() {
-                        request = Some(SetupWizardRequest::ServerRequest(
-                            ServerRequest::SetValues(vec![PathValuePair {
-                                path: alvr_packets::parse_path(&format!(
-                                    "session_settings.headset.controllers.content.{}",
-                                    "gestures.content.only_touch"
-                                )),
-                                value: serde_json::Value::Bool(self.only_touch),
-                            }]),
-                        ));
-                    }
-                },
-            ),
             Page::Firewall => page_content(
                 ui,
                 "Firewall",

diff --git a/alvr/server_core/src/connection.rs b/alvr/server_core/src/connection.rs
@@ -946,7 +946,7 @@ fn connection_pipeline(
                 if let (Some(gestures_config), Some(gestures_button_mapping_manager)) = (
                     controllers_config
                         .as_ref()
-                        .and_then(|c| c.gestures.as_option()),
+                        .and_then(|c| c.hand_tracking_interaction.as_option()),
                     &mut gestures_button_mapping_manager,
                 ) {
                     let mut hand_gesture_manager_lock = hand_gesture_manager.lock();
@@ -991,7 +991,7 @@ fn connection_pipeline(
                                 device_motions: motions,
                                 hand_skeletons: if controllers_config
                                     .as_ref()
-                                    .map(|c| c.enable_skeleton)
+                                    .map(|c| c.hand_skeleton.enabled())
                                     .unwrap_or(false)
                                 {
                                     hand_skeletons

diff --git a/alvr/server_core/src/hand_gestures.rs b/alvr/server_core/src/hand_gestures.rs
@@ -11,7 +11,7 @@ use alvr_common::{
 };
 
 use alvr_packets::{ButtonEntry, ButtonValue};
-use alvr_session::HandGestureConfig;
+use alvr_session::HandTrackingInteractionConfig;
 
 use crate::input_mapping::ButtonMappingManager;
 
@@ -107,7 +107,7 @@ impl HandGestureManager {
     pub fn get_active_gestures(
         &mut self,
         hand_skeleton: [Pose; 26],
-        config: &HandGestureConfig,
+        config: &HandTrackingInteractionConfig,
         device_id: u64,
     ) -> Vec<HandGesture> {
         // global joints

diff --git a/alvr/server_openvr/cpp/alvr_server/Controller.cpp b/alvr/server_openvr/cpp/alvr_server/Controller.cpp
@@ -74,13 +74,15 @@ vr::EVRInitError Controller::Activate(vr::TrackedDeviceIndex_t unObjectId) {
         vr::VRScalarUnits_NormalizedOneSided
     );
 
-    if (this->device_id == HAND_LEFT_ID) {
+    if (this->device_id == HAND_LEFT_ID || this->device_id == HAND_TRACKER_LEFT_ID) {
         vr_driver_input->CreateSkeletonComponent(
             this->prop_container,
             "/input/skeleton/left",
             "/skeleton/hand/left",
             "/pose/raw",
-            vr::EVRSkeletalTrackingLevel::VRSkeletalTracking_Partial,
+            this->device_id == HAND_LEFT_ID
+                ? vr::EVRSkeletalTrackingLevel::VRSkeletalTracking_Partial
+                : vr::EVRSkeletalTrackingLevel::VRSkeletalTracking_Full,
             nullptr,
             0U,
             &m_compSkeleton
@@ -91,7 +93,9 @@ vr::EVRInitError Controller::Activate(vr::TrackedDeviceIndex_t unObjectId) {
             "/input/skeleton/right",
             "/skeleton/hand/right",
             "/pose/raw",
-            vr::EVRSkeletalTrackingLevel::VRSkeletalTracking_Partial,
+            this->device_id == HAND_RIGHT_ID
+                ? vr::EVRSkeletalTrackingLevel::VRSkeletalTracking_Partial
+                : vr::EVRSkeletalTrackingLevel::VRSkeletalTracking_Full,
             nullptr,
             0U,
             &m_compSkeleton
@@ -179,6 +183,10 @@ void Controller::RegisterButton(uint64_t id) {
 }
 
 void Controller::SetButton(uint64_t id, FfiButtonValue value) {
+    if (!this->isEnabled()) {
+        return;
+    }
+
     for (auto id : ALVR_TO_STEAMVR_PATH_IDS[id]) {
         if (value.type == BUTTON_TYPE_BINARY) {
             vr::VRDriverInput()->UpdateBooleanComponent(
@@ -249,6 +257,15 @@ bool Controller::onPoseUpdate(
 
     m_pose = pose;
 
+    vr::VRServerDriverHost()->TrackedDevicePoseUpdated(
+        this->object_id, pose, sizeof(vr::DriverPose_t)
+    );
+
+    // Early return to skip updating the skeleton
+    if (!this->isEnabled()) {
+        return false;
+    }
+
     if (handSkeleton != nullptr) {
         vr::VRBoneTransform_t boneTransform[SKELETON_BONE_COUNT] = {};
         for (int j = 0; j < 31; j++) {
@@ -390,10 +407,6 @@ bool Controller::onPoseUpdate(
         }
     }
 
-    vr::VRServerDriverHost()->TrackedDevicePoseUpdated(
-        this->object_id, pose, sizeof(vr::DriverPose_t)
-    );
-
     return false;
 }
 

diff --git a/alvr/server_openvr/cpp/alvr_server/Controller.h b/alvr/server_openvr/cpp/alvr_server/Controller.h
@@ -69,4 +69,6 @@ class Controller : public TrackedDevice, public vr::ITrackedDeviceServerDriver {
     bool m_lastTriggerTouch = false;
     float m_triggerValue = 0;
     float m_gripValue = 0;
+
+    bool isEnabled() { return m_pose.deviceIsConnected; }
 };
diff --git a/alvr/server_openvr/cpp/alvr_server/Paths.cpp b/alvr/server_openvr/cpp/alvr_server/Paths.cpp
@@ -4,6 +4,8 @@
 uint64_t HEAD_ID;
 uint64_t HAND_LEFT_ID;
 uint64_t HAND_RIGHT_ID;
+uint64_t HAND_TRACKER_LEFT_ID;
+uint64_t HAND_TRACKER_RIGHT_ID;
 
 std::map<uint64_t, ButtonInfo> LEFT_CONTROLLER_BUTTON_MAPPING;
 std::map<uint64_t, ButtonInfo> RIGHT_CONTROLLER_BUTTON_MAPPING;
@@ -32,10 +34,8 @@ void init_paths() {
     HEAD_ID = PathStringToHash("/user/head");
     HAND_LEFT_ID = PathStringToHash("/user/hand/left");
     HAND_RIGHT_ID = PathStringToHash("/user/hand/right");
-
-    HEAD_ID = PathStringToHash("/user/head");
-    HAND_LEFT_ID = PathStringToHash("/user/hand/left");
-    HAND_RIGHT_ID = PathStringToHash("/user/hand/right");
+    HAND_TRACKER_LEFT_ID = PathStringToHash("/user/hand_tracker/left");
+    HAND_TRACKER_RIGHT_ID = PathStringToHash("/user/hand_tracker/right");
 
     LEFT_CONTROLLER_BUTTON_MAPPING.insert({ PathStringToHash("/user/hand/left/input/system/click"),
                                             { { "/input/system/click" }, ButtonType::Binary } });

diff --git a/alvr/server_openvr/cpp/alvr_server/Paths.h b/alvr/server_openvr/cpp/alvr_server/Paths.h
@@ -9,6 +9,8 @@
 extern uint64_t HEAD_ID;
 extern uint64_t HAND_LEFT_ID;
 extern uint64_t HAND_RIGHT_ID;
+extern uint64_t HAND_TRACKER_LEFT_ID;
+extern uint64_t HAND_TRACKER_RIGHT_ID;
 
 enum class ButtonType {
     Binary,