Merge pull request #60313 from BastiaanOlij/xr_hand_tracking

Adding OpenXR hand tracking support

modules/openxr/SCsub

@@ -83,6 +83,7 @@ module_obj = []
 
 env_openxr.add_source_files(module_obj, "*.cpp")
 env_openxr.add_source_files(module_obj, "action_map/*.cpp")
+env_openxr.add_source_files(module_obj, "scene/*.cpp")
 
 # We're a little more targeted with our extensions
 if env["platform"] == "android":
@@ -91,6 +92,7 @@ if env["vulkan"]:
     env_openxr.add_source_files(module_obj, "extensions/openxr_vulkan_extension.cpp")
 
 env_openxr.add_source_files(module_obj, "extensions/openxr_htc_vive_tracker_extension.cpp")
+env_openxr.add_source_files(module_obj, "extensions/openxr_hand_tracking_extension.cpp")
 
 env.modules_sources += module_obj
 

modules/openxr/config.py

@@ -18,6 +18,7 @@ def get_doc_classes():
         "OpenXRActionMap",
         "OpenXRInteractionProfile",
         "OpenXRIPBinding",
+        "OpenXRHand",
     ]
 
 

modules/openxr/doc_classes/OpenXRHand.xml

@@ -0,0 +1,42 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<class name="OpenXRHand" inherits="Node3D" version="4.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="../../../doc/class.xsd">
+	<brief_description>
+		Node supporting finger tracking in OpenXR.
+	</brief_description>
+	<description>
+		This node enables OpenXR's hand tracking functionality. The node should be a child node of an [XROrigin3D] node, tracking will update its position to where the player's actual hand is positioned. This node also updates the skeleton of a properly skinned hand model. The hand mesh should be a child node of this node.
+	</description>
+	<tutorials>
+	</tutorials>
+	<members>
+		<member name="hand" type="int" setter="set_hand" getter="get_hand" enum="OpenXRHand.Hands" default="0">
+			Specifies whether this node tracks the left or right hand of the player.
+		</member>
+		<member name="hand_skeleton" type="NodePath" setter="set_hand_skeleton" getter="get_hand_skeleton" default="NodePath(&quot;&quot;)">
+			Set a [Skeleton3D] node for which the pose positions will be updated.
+		</member>
+		<member name="motion_range" type="int" setter="set_motion_range" getter="get_motion_range" enum="OpenXRHand.MotionRange" default="0">
+			Set the motion range (if supported) limiting the hand motion.
+		</member>
+	</members>
+	<constants>
+		<constant name="HAND_LEFT" value="0" enum="Hands">
+			Tracking the player's left hand.
+		</constant>
+		<constant name="HAND_RIGHT" value="1" enum="Hands">
+			Tracking the player's right hand.
+		</constant>
+		<constant name="HAND_MAX" value="2" enum="Hands">
+			Maximum supported hands.
+		</constant>
+		<constant name="MOTION_RANGE_UNOBSTRUCTED" value="0" enum="MotionRange">
+			When player grips, hand skeleton will form a full fist.
+		</constant>
+		<constant name="MOTION_RANGE_CONFORM_TO_CONTROLLER" value="1" enum="MotionRange">
+			When player grips, hand skeleton conforms to the controller the player is holding.
+		</constant>
+		<constant name="MOTION_RANGE_MAX" value="2" enum="MotionRange">
+			Maximum supported motion ranges.
+		</constant>
+	</constants>
+</class>

modules/openxr/extensions/openxr_hand_tracking_extension.cpp

@@ -0,0 +1,268 @@
+/*************************************************************************/
+/*  openxr_hand_tracking_extension.cpp                                   */
+/*************************************************************************/
+/*                       This file is part of:                           */
+/*                           GODOT ENGINE                                */
+/*                      https://godotengine.org                          */
+/*************************************************************************/
+/* Copyright (c) 2007-2022 Juan Linietsky, Ariel Manzur.                 */
+/* Copyright (c) 2014-2022 Godot Engine contributors (cf. AUTHORS.md).   */
+/*                                                                       */
+/* Permission is hereby granted, free of charge, to any person obtaining */
+/* a copy of this software and associated documentation files (the       */
+/* "Software"), to deal in the Software without restriction, including   */
+/* without limitation the rights to use, copy, modify, merge, publish,   */
+/* distribute, sublicense, and/or sell copies of the Software, and to    */
+/* permit persons to whom the Software is furnished to do so, subject to */
+/* the following conditions:                                             */
+/*                                                                       */
+/* The above copyright notice and this permission notice shall be        */
+/* included in all copies or substantial portions of the Software.       */
+/*                                                                       */
+/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,       */
+/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF    */
+/* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.*/
+/* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY  */
+/* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,  */
+/* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE     */
+/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.                */
+/*************************************************************************/
+
+#include "openxr_hand_tracking_extension.h"
+#include "../openxr_api.h"
+#include "core/string/print_string.h"
+#include "servers/xr_server.h"
+
+#include <openxr/openxr.h>
+
+OpenXRHandTrackingExtension *OpenXRHandTrackingExtension::singleton = nullptr;
+
+OpenXRHandTrackingExtension *OpenXRHandTrackingExtension::get_singleton() {
+	return singleton;
+}
+
+OpenXRHandTrackingExtension::OpenXRHandTrackingExtension(OpenXRAPI *p_openxr_api) :
+		OpenXRExtensionWrapper(p_openxr_api) {
+	singleton = this;
+
+	// Extensions we use for our hand tracking.
+	request_extensions[XR_EXT_HAND_TRACKING_EXTENSION_NAME] = &hand_tracking_ext;
+	request_extensions[XR_EXT_HAND_JOINTS_MOTION_RANGE_EXTENSION_NAME] = &hand_motion_range_ext;
+	request_extensions[XR_FB_HAND_TRACKING_AIM_EXTENSION_NAME] = &hand_tracking_aim_state_ext;
+
+	// Make sure this is cleared until we actually request it
+	handTrackingSystemProperties.supportsHandTracking = false;
+}
+
+OpenXRHandTrackingExtension::~OpenXRHandTrackingExtension() {
+	singleton = nullptr;
+}
+
+void OpenXRHandTrackingExtension::on_instance_created(const XrInstance p_instance) {
+	if (hand_tracking_ext) {
+		EXT_INIT_XR_FUNC(xrCreateHandTrackerEXT);
+		EXT_INIT_XR_FUNC(xrDestroyHandTrackerEXT);
+		EXT_INIT_XR_FUNC(xrLocateHandJointsEXT);
+
+		hand_tracking_ext = xrCreateHandTrackerEXT_ptr && xrDestroyHandTrackerEXT_ptr && xrLocateHandJointsEXT_ptr;
+	}
+}
+
+void OpenXRHandTrackingExtension::on_session_destroyed() {
+	cleanup_hand_tracking();
+}
+
+void OpenXRHandTrackingExtension::on_instance_destroyed() {
+	xrCreateHandTrackerEXT_ptr = nullptr;
+	xrDestroyHandTrackerEXT_ptr = nullptr;
+	xrLocateHandJointsEXT_ptr = nullptr;
+}
+
+void *OpenXRHandTrackingExtension::set_system_properties_and_get_next_pointer(void *p_next_pointer) {
+	if (!hand_tracking_ext) {
+		// not supported...
+		return p_next_pointer;
+	}
+
+	handTrackingSystemProperties = {
+		XR_TYPE_SYSTEM_HAND_TRACKING_PROPERTIES_EXT, // type
+		p_next_pointer, // next
+		false, // supportsHandTracking
+	};
+
+	return &handTrackingSystemProperties;
+}
+
+void OpenXRHandTrackingExtension::on_state_ready() {
+	if (!handTrackingSystemProperties.supportsHandTracking) {
+		// not supported...
+		return;
+	}
+
+	// Setup our hands and reset data
+	for (int i = 0; i < MAX_OPENXR_TRACKED_HANDS; i++) {
+		// we'll do this later
+		hand_trackers[i].is_initialised = false;
+		hand_trackers[i].hand_tracker = XR_NULL_HANDLE;
+
+		hand_trackers[i].aimState.aimPose = { { 0.0, 0.0, 0.0, 0.0 }, { 0.0, 0.0, 0.0 } };
+		hand_trackers[i].aimState.pinchStrengthIndex = 0.0;
+		hand_trackers[i].aimState.pinchStrengthMiddle = 0.0;
+		hand_trackers[i].aimState.pinchStrengthRing = 0.0;
+		hand_trackers[i].aimState.pinchStrengthLittle = 0.0;
+
+		hand_trackers[i].locations.isActive = false;
+
+		for (int j = 0; j < XR_HAND_JOINT_COUNT_EXT; j++) {
+			hand_trackers[i].joint_locations[j] = { 0, { { 0.0, 0.0, 0.0, 0.0 }, { 0.0, 0.0, 0.0 } }, 0.0 };
+			hand_trackers[i].joint_velocities[j] = { 0, { 0.0, 0.0, 0.0 }, { 0.0, 0.0, 0.0 } };
+		}
+	}
+}
+
+void OpenXRHandTrackingExtension::on_process() {
+	if (!handTrackingSystemProperties.supportsHandTracking) {
+		// not supported...
+		return;
+	}
+
+	// process our hands
+	const XrTime time = openxr_api->get_next_frame_time(); // This data will be used for the next frame we render
+
+	XrResult result;
+
+	for (int i = 0; i < MAX_OPENXR_TRACKED_HANDS; i++) {
+		if (hand_trackers[i].hand_tracker == XR_NULL_HANDLE) {
+			XrHandTrackerCreateInfoEXT createInfo = {
+				XR_TYPE_HAND_TRACKER_CREATE_INFO_EXT, // type
+				nullptr, // next
+				i == 0 ? XR_HAND_LEFT_EXT : XR_HAND_RIGHT_EXT, // hand
+				XR_HAND_JOINT_SET_DEFAULT_EXT, // handJointSet
+			};
+
+			result = xrCreateHandTrackerEXT(openxr_api->get_session(), &createInfo, &hand_trackers[i].hand_tracker);
+			if (XR_FAILED(result)) {
+				// not successful? then we do nothing.
+				print_line("OpenXR: Failed to obtain hand tracking information [", openxr_api->get_error_string(result), "]");
+				hand_trackers[i].is_initialised = false;
+			} else {
+				void *next_pointer = nullptr;
+				if (hand_tracking_aim_state_ext) {
+					hand_trackers[i].aimState.type = XR_TYPE_HAND_TRACKING_AIM_STATE_FB;
+					hand_trackers[i].aimState.next = next_pointer;
+					hand_trackers[i].aimState.status = 0;
+					hand_trackers[i].aimState.aimPose = { { 0.0, 0.0, 0.0, 0.0 }, { 0.0, 0.0, 0.0 } };
+					hand_trackers[i].aimState.pinchStrengthIndex = 0.0;
+					hand_trackers[i].aimState.pinchStrengthMiddle = 0.0;
+					hand_trackers[i].aimState.pinchStrengthRing = 0.0;
+					hand_trackers[i].aimState.pinchStrengthLittle = 0.0;
+
+					next_pointer = &hand_trackers[i].aimState;
+				}
+
+				hand_trackers[i].velocities.type = XR_TYPE_HAND_JOINT_VELOCITIES_EXT;
+				hand_trackers[i].velocities.next = next_pointer;
+				hand_trackers[i].velocities.jointCount = XR_HAND_JOINT_COUNT_EXT;
+				hand_trackers[i].velocities.jointVelocities = hand_trackers[i].joint_velocities;
+				next_pointer = &hand_trackers[i].velocities;
+
+				hand_trackers[i].locations.type = XR_TYPE_HAND_JOINT_LOCATIONS_EXT;
+				hand_trackers[i].locations.next = next_pointer;
+				hand_trackers[i].locations.isActive = false;
+				hand_trackers[i].locations.jointCount = XR_HAND_JOINT_COUNT_EXT;
+				hand_trackers[i].locations.jointLocations = hand_trackers[i].joint_locations;
+
+				hand_trackers[i].is_initialised = true;
+			}
+		}
+
+		if (hand_trackers[i].is_initialised) {
+			void *next_pointer = nullptr;
+
+			XrHandJointsMotionRangeInfoEXT motionRangeInfo;
+
+			if (hand_motion_range_ext) {
+				motionRangeInfo.type = XR_TYPE_HAND_JOINTS_MOTION_RANGE_INFO_EXT;
+				motionRangeInfo.next = next_pointer;
+				motionRangeInfo.handJointsMotionRange = hand_trackers[i].motion_range;
+
+				next_pointer = &motionRangeInfo;
+			}
+
+			XrHandJointsLocateInfoEXT locateInfo = {
+				XR_TYPE_HAND_JOINTS_LOCATE_INFO_EXT, // type
+				next_pointer, // next
+				openxr_api->get_play_space(), // baseSpace
+				time, // time
+			};
+
+			result = xrLocateHandJointsEXT(hand_trackers[i].hand_tracker, &locateInfo, &hand_trackers[i].locations);
+			if (XR_FAILED(result)) {
+				// not successful? then we do nothing.
+				print_line("OpenXR: Failed to get tracking for hand", i, "[", openxr_api->get_error_string(result), "]");
+				continue;
+			}
+
+			// For some reason an inactive controller isn't coming back as inactive but has coordinates either as NAN or very large
+			const XrPosef &palm = hand_trackers[i].joint_locations[XR_HAND_JOINT_PALM_EXT].pose;
+			if (
+					!hand_trackers[i].locations.isActive || isnan(palm.position.x) || palm.position.x < -1000000.00 || palm.position.x > 1000000.00) {
+				hand_trackers[i].locations.isActive = false; // workaround, make sure its inactive
+			}
+
+			/* TODO change this to managing the controller from openxr_interface
+			if (hand_tracking_aim_state_ext && hand_trackers[i].locations.isActive && check_bit(XR_HAND_TRACKING_AIM_VALID_BIT_FB, hand_trackers[i].aimState.status)) {
+				// Controllers are updated based on the aim state's pose and pinches' strength
+				if (hand_trackers[i].aim_state_godot_controller == -1) {
+					hand_trackers[i].aim_state_godot_controller =
+							arvr_api->godot_arvr_add_controller(
+									const_cast<char *>(hand_controller_names[i]),
+									i + HAND_CONTROLLER_ID_OFFSET,
+									true,
+									true);
+				}
+			}
+			*/
+		}
+	}
+}
+
+void OpenXRHandTrackingExtension::on_state_stopping() {
+	// cleanup
+	cleanup_hand_tracking();
+}
+
+void OpenXRHandTrackingExtension::cleanup_hand_tracking() {
+	XRServer *xr_server = XRServer::get_singleton();
+	ERR_FAIL_NULL(xr_server);
+
+	for (int i = 0; i < MAX_OPENXR_TRACKED_HANDS; i++) {
+		if (hand_trackers[i].hand_tracker != XR_NULL_HANDLE) {
+			xrDestroyHandTrackerEXT(hand_trackers[i].hand_tracker);
+
+			hand_trackers[i].is_initialised = false;
+			hand_trackers[i].hand_tracker = XR_NULL_HANDLE;
+		}
+	}
+}
+
+bool OpenXRHandTrackingExtension::get_active() {
+	return handTrackingSystemProperties.supportsHandTracking;
+}
+
+const OpenXRHandTrackingExtension::HandTracker *OpenXRHandTrackingExtension::get_hand_tracker(uint32_t p_hand) const {
+	ERR_FAIL_UNSIGNED_INDEX_V(p_hand, MAX_OPENXR_TRACKED_HANDS, nullptr);
+
+	return &hand_trackers[p_hand];
+}
+
+XrHandJointsMotionRangeEXT OpenXRHandTrackingExtension::get_motion_range(uint32_t p_hand) const {
+	ERR_FAIL_UNSIGNED_INDEX_V(p_hand, MAX_OPENXR_TRACKED_HANDS, XR_HAND_JOINTS_MOTION_RANGE_MAX_ENUM_EXT);
+
+	return hand_trackers[p_hand].motion_range;
+}
+
+void OpenXRHandTrackingExtension::set_motion_range(uint32_t p_hand, XrHandJointsMotionRangeEXT p_motion_range) {
+	ERR_FAIL_UNSIGNED_INDEX(p_hand, MAX_OPENXR_TRACKED_HANDS);
+	hand_trackers[p_hand].motion_range = p_motion_range;
+}