From 7fa8ccd1ed6d9c0b8e979d30e9af1d12dbf9ed48 Mon Sep 17 00:00:00 2001 From: David Snopek Date: Thu, 15 Feb 2024 14:58:38 -0600 Subject: [PATCH] WebXR: Add support for hand tracking --- modules/webxr/doc_classes/WebXRInterface.xml | 4 + modules/webxr/godot_webxr.h | 7 +- modules/webxr/native/library_godot_webxr.js | 25 ++++- modules/webxr/native/webxr.externs.js | 56 +++++++++- modules/webxr/webxr_interface.cpp | 2 + modules/webxr/webxr_interface.h | 1 + modules/webxr/webxr_interface_js.cpp | 103 ++++++++++++++++--- modules/webxr/webxr_interface_js.h | 10 +- 8 files changed, 182 insertions(+), 26 deletions(-) diff --git a/modules/webxr/doc_classes/WebXRInterface.xml b/modules/webxr/doc_classes/WebXRInterface.xml index 9e1167c09f1..caf7958f6bf 100644 --- a/modules/webxr/doc_classes/WebXRInterface.xml +++ b/modules/webxr/doc_classes/WebXRInterface.xml @@ -153,6 +153,10 @@ + + A comma-separated list of features that were successfully enabled by [method XRInterface.initialize] when setting up the WebXR session. + This may include features requested by setting [member required_features] and [member optional_features]. + A comma-seperated list of optional features used by [method XRInterface.initialize] when setting up the WebXR session. If a user's browser or device doesn't support one of the given features, initialization will continue, but you won't be able to use the requested feature. diff --git a/modules/webxr/godot_webxr.h b/modules/webxr/godot_webxr.h index a7ce7e45d25..caa7f217af7 100644 --- a/modules/webxr/godot_webxr.h +++ b/modules/webxr/godot_webxr.h @@ -45,7 +45,7 @@ enum WebXRInputEvent { }; typedef void (*GodotWebXRSupportedCallback)(char *p_session_mode, int p_supported); -typedef void (*GodotWebXRStartedCallback)(char *p_reference_space_type); +typedef void (*GodotWebXRStartedCallback)(char *p_reference_space_type, char *p_enabled_features); typedef void (*GodotWebXREndedCallback)(); typedef void (*GodotWebXRFailedCallback)(char *p_message); typedef void (*GodotWebXRInputEventCallback)(int p_event_type, int p_input_source_id); @@ -85,7 +85,10 @@ extern bool godot_webxr_update_input_source( int *r_button_count, float *r_buttons, int *r_axes_count, - float *r_axes); + float *r_axes, + int *r_has_hand_data, + float *r_hand_joints, + float *r_hand_radii); extern char *godot_webxr_get_visibility_state(); extern int godot_webxr_get_bounds_geometry(float **r_points); diff --git a/modules/webxr/native/library_godot_webxr.js b/modules/webxr/native/library_godot_webxr.js index fe47de02b05..c048bb21972 100644 --- a/modules/webxr/native/library_godot_webxr.js +++ b/modules/webxr/native/library_godot_webxr.js @@ -318,9 +318,11 @@ const GodotWebXR = { // callback don't bubble up here and cause Godot to try the // next reference space. window.setTimeout(function () { - const c_str = GodotRuntime.allocString(reference_space_type); - onstarted(c_str); - GodotRuntime.free(c_str); + const reference_space_c_str = GodotRuntime.allocString(reference_space_type); + const enabled_features_c_str = GodotRuntime.allocString(Array.from(session.enabledFeatures).join(",")); + onstarted(reference_space_c_str, enabled_features_c_str); + GodotRuntime.free(reference_space_c_str); + GodotRuntime.free(enabled_features_c_str); }, 0); } @@ -479,8 +481,8 @@ const GodotWebXR = { }, godot_webxr_update_input_source__proxy: 'sync', - godot_webxr_update_input_source__sig: 'iiiiiiiiiiii', - godot_webxr_update_input_source: function (p_input_source_id, r_target_pose, r_target_ray_mode, r_touch_index, r_has_grip_pose, r_grip_pose, r_has_standard_mapping, r_button_count, r_buttons, r_axes_count, r_axes) { + godot_webxr_update_input_source__sig: 'iiiiiiiiiiiiiii', + godot_webxr_update_input_source: function (p_input_source_id, r_target_pose, r_target_ray_mode, r_touch_index, r_has_grip_pose, r_grip_pose, r_has_standard_mapping, r_button_count, r_buttons, r_axes_count, r_axes, r_has_hand_data, r_hand_joints, r_hand_radii) { if (!GodotWebXR.session || !GodotWebXR.frame) { return 0; } @@ -563,6 +565,19 @@ const GodotWebXR = { GodotRuntime.setHeapValue(r_button_count, button_count, 'i32'); GodotRuntime.setHeapValue(r_axes_count, axes_count, 'i32'); + // Hand tracking data. + let has_hand_data = false; + if (input_source.hand && r_hand_joints != 0 && r_hand_radii != 0) { + const hand_joint_array = new Float32Array(25 * 16); + const hand_radii_array = new Float32Array(25); + if (frame.fillPoses(input_source.hand.values(), space, hand_joint_array) && frame.fillJointRadii(input_source.hand.values(), hand_radii_array)) { + GodotRuntime.heapCopy(HEAPF32, hand_joint_array, r_hand_joints); + GodotRuntime.heapCopy(HEAPF32, hand_radii_array, r_hand_radii); + has_hand_data = true; + } + } + GodotRuntime.setHeapValue(r_has_hand_data, has_hand_data ? 1 : 0, 'i32'); + return true; }, diff --git a/modules/webxr/native/webxr.externs.js b/modules/webxr/native/webxr.externs.js index 7f7c297acc7..35ad33fa933 100644 --- a/modules/webxr/native/webxr.externs.js +++ b/modules/webxr/native/webxr.externs.js @@ -229,13 +229,27 @@ XRFrame.prototype.session; XRFrame.prototype.getViewerPose = function (referenceSpace) {}; /** - * * @param {XRSpace} space * @param {XRSpace} baseSpace * @return {XRPose} */ XRFrame.prototype.getPose = function (space, baseSpace) {}; +/** + * @param {Array} spaces + * @param {XRSpace} baseSpace + * @param {Float32Array} transforms + * @return {boolean} + */ +XRFrame.prototype.fillPoses = function (spaces, baseSpace, transforms) {}; + +/** + * @param {Array} jointSpaces + * @param {Float32Array} radii + * @return {boolean} + */ +XRFrame.prototype.fillJointRadii = function (jointSpaces, radii) {}; + /** * @constructor */ @@ -498,11 +512,51 @@ XRInputSource.prototype.targetRayMode; */ XRInputSource.prototype.targetRaySpace; +/** + * @type {?XRHand} + */ +XRInputSource.prototype.hand; + +/** + * @constructor + */ +function XRHand() {}; + +/** + * Note: In fact, XRHand acts like a Map, but I don't know + * how to represent that here. So, we're just giving the one method we call. + * + * @return {Array} + */ +XRHand.prototype.values = function () {}; + +/** + * @type {number} + */ +XRHand.prototype.size; + +/** + * @param {string} key + * @return {XRJointSpace} + */ +XRHand.prototype.get = function (key) {}; + /** * @constructor */ function XRSpace() {}; +/** + * @constructor + * @extends {XRSpace} + */ +function XRJointSpace() {}; + +/** + * @type {string} + */ +XRJointSpace.prototype.jointName; + /** * @constructor */ diff --git a/modules/webxr/webxr_interface.cpp b/modules/webxr/webxr_interface.cpp index 85ed9f472e3..c3efebef0f1 100644 --- a/modules/webxr/webxr_interface.cpp +++ b/modules/webxr/webxr_interface.cpp @@ -41,6 +41,7 @@ void WebXRInterface::_bind_methods() { ClassDB::bind_method(D_METHOD("set_optional_features", "optional_features"), &WebXRInterface::set_optional_features); ClassDB::bind_method(D_METHOD("get_optional_features"), &WebXRInterface::get_optional_features); ClassDB::bind_method(D_METHOD("get_reference_space_type"), &WebXRInterface::get_reference_space_type); + ClassDB::bind_method(D_METHOD("get_enabled_features"), &WebXRInterface::get_enabled_features); ClassDB::bind_method(D_METHOD("set_requested_reference_space_types", "requested_reference_space_types"), &WebXRInterface::set_requested_reference_space_types); ClassDB::bind_method(D_METHOD("get_requested_reference_space_types"), &WebXRInterface::get_requested_reference_space_types); ClassDB::bind_method(D_METHOD("is_input_source_active", "input_source_id"), &WebXRInterface::is_input_source_active); @@ -56,6 +57,7 @@ void WebXRInterface::_bind_methods() { ADD_PROPERTY(PropertyInfo(Variant::STRING, "optional_features", PROPERTY_HINT_NONE), "set_optional_features", "get_optional_features"); ADD_PROPERTY(PropertyInfo(Variant::STRING, "requested_reference_space_types", PROPERTY_HINT_NONE), "set_requested_reference_space_types", "get_requested_reference_space_types"); ADD_PROPERTY(PropertyInfo(Variant::STRING, "reference_space_type", PROPERTY_HINT_NONE), "", "get_reference_space_type"); + ADD_PROPERTY(PropertyInfo(Variant::STRING, "enabled_features", PROPERTY_HINT_NONE), "", "get_enabled_features"); ADD_PROPERTY(PropertyInfo(Variant::STRING, "visibility_state", PROPERTY_HINT_NONE), "", "get_visibility_state"); ADD_SIGNAL(MethodInfo("session_supported", PropertyInfo(Variant::STRING, "session_mode"), PropertyInfo(Variant::BOOL, "supported"))); diff --git a/modules/webxr/webxr_interface.h b/modules/webxr/webxr_interface.h index abaa8c01f8a..06c18d0486e 100644 --- a/modules/webxr/webxr_interface.h +++ b/modules/webxr/webxr_interface.h @@ -62,6 +62,7 @@ public: virtual void set_requested_reference_space_types(String p_requested_reference_space_types) = 0; virtual String get_requested_reference_space_types() const = 0; virtual String get_reference_space_type() const = 0; + virtual String get_enabled_features() const = 0; virtual bool is_input_source_active(int p_input_source_id) const = 0; virtual Ref get_input_source_tracker(int p_input_source_id) const = 0; virtual TargetRayMode get_input_source_target_ray_mode(int p_input_source_id) const = 0; diff --git a/modules/webxr/webxr_interface_js.cpp b/modules/webxr/webxr_interface_js.cpp index 5415423fb84..c6213d1aae1 100644 --- a/modules/webxr/webxr_interface_js.cpp +++ b/modules/webxr/webxr_interface_js.cpp @@ -41,6 +41,7 @@ #include "scene/main/window.h" #include "servers/rendering/renderer_compositor.h" #include "servers/rendering/rendering_server_globals.h" +#include "servers/xr/xr_hand_tracker.h" #include #include @@ -49,22 +50,23 @@ void _emwebxr_on_session_supported(char *p_session_mode, int p_supported) { XRServer *xr_server = XRServer::get_singleton(); ERR_FAIL_NULL(xr_server); - Ref interface = xr_server->find_interface("WebXR"); + Ref interface = xr_server->find_interface("WebXR"); ERR_FAIL_COND(interface.is_null()); String session_mode = String(p_session_mode); interface->emit_signal(SNAME("session_supported"), session_mode, p_supported ? true : false); } -void _emwebxr_on_session_started(char *p_reference_space_type) { +void _emwebxr_on_session_started(char *p_reference_space_type, char *p_enabled_features) { XRServer *xr_server = XRServer::get_singleton(); ERR_FAIL_NULL(xr_server); - Ref interface = xr_server->find_interface("WebXR"); + Ref interface = xr_server->find_interface("WebXR"); ERR_FAIL_COND(interface.is_null()); String reference_space_type = String(p_reference_space_type); - static_cast(interface.ptr())->_set_reference_space_type(reference_space_type); + interface->_set_reference_space_type(reference_space_type); + interface->_set_enabled_features(p_enabled_features); interface->emit_signal(SNAME("session_started")); } @@ -72,7 +74,7 @@ void _emwebxr_on_session_ended() { XRServer *xr_server = XRServer::get_singleton(); ERR_FAIL_NULL(xr_server); - Ref interface = xr_server->find_interface("WebXR"); + Ref interface = xr_server->find_interface("WebXR"); ERR_FAIL_COND(interface.is_null()); interface->uninitialize(); @@ -83,7 +85,7 @@ void _emwebxr_on_session_failed(char *p_message) { XRServer *xr_server = XRServer::get_singleton(); ERR_FAIL_NULL(xr_server); - Ref interface = xr_server->find_interface("WebXR"); + Ref interface = xr_server->find_interface("WebXR"); ERR_FAIL_COND(interface.is_null()); interface->uninitialize(); @@ -96,17 +98,17 @@ extern "C" EMSCRIPTEN_KEEPALIVE void _emwebxr_on_input_event(int p_event_type, i XRServer *xr_server = XRServer::get_singleton(); ERR_FAIL_NULL(xr_server); - Ref interface = xr_server->find_interface("WebXR"); + Ref interface = xr_server->find_interface("WebXR"); ERR_FAIL_COND(interface.is_null()); - ((WebXRInterfaceJS *)interface.ptr())->_on_input_event(p_event_type, p_input_source_id); + interface->_on_input_event(p_event_type, p_input_source_id); } extern "C" EMSCRIPTEN_KEEPALIVE void _emwebxr_on_simple_event(char *p_signal_name) { XRServer *xr_server = XRServer::get_singleton(); ERR_FAIL_NULL(xr_server); - Ref interface = xr_server->find_interface("WebXR"); + Ref interface = xr_server->find_interface("WebXR"); ERR_FAIL_COND(interface.is_null()); StringName signal_name = StringName(p_signal_name); @@ -149,14 +151,14 @@ String WebXRInterfaceJS::get_requested_reference_space_types() const { return requested_reference_space_types; } -void WebXRInterfaceJS::_set_reference_space_type(String p_reference_space_type) { - reference_space_type = p_reference_space_type; -} - String WebXRInterfaceJS::get_reference_space_type() const { return reference_space_type; } +String WebXRInterfaceJS::get_enabled_features() const { + return enabled_features; +} + bool WebXRInterfaceJS::is_input_source_active(int p_input_source_id) const { ERR_FAIL_INDEX_V(p_input_source_id, input_source_count, false); return input_sources[p_input_source_id].active; @@ -256,7 +258,9 @@ bool WebXRInterfaceJS::initialize() { return false; } - // we must create a tracker for our head + enabled_features.clear(); + + // We must create a tracker for our head. head_transform.basis = Basis(); head_transform.origin = Vector3(); head_tracker.instantiate(); @@ -265,7 +269,7 @@ bool WebXRInterfaceJS::initialize() { head_tracker->set_tracker_desc("Players head"); xr_server->add_tracker(head_tracker); - // make this our primary interface + // Make this our primary interface. xr_server->set_primary_interface(this); // Clear render_targetsize to make sure it gets reset to the new size. @@ -301,6 +305,14 @@ void WebXRInterfaceJS::uninitialize() { head_tracker.unref(); } + for (int i = 0; i < HAND_MAX; i++) { + if (hand_trackers[i].is_valid()) { + xr_server->remove_hand_tracker(i == 0 ? "/user/left" : "/user/right"); + + hand_trackers[i].unref(); + } + } + if (xr_server->get_primary_interface() == this) { // no longer our primary interface xr_server->set_primary_interface(nullptr); @@ -321,7 +333,8 @@ void WebXRInterfaceJS::uninitialize() { } texture_cache.clear(); - reference_space_type = ""; + reference_space_type.clear(); + enabled_features.clear(); initialized = false; }; }; @@ -572,6 +585,9 @@ void WebXRInterfaceJS::_update_input_source(int p_input_source_id) { float buttons[10]; int axes_count; float axes[10]; + int has_hand_data; + float hand_joints[WEBXR_HAND_JOINT_MAX * 16]; + float hand_radii[WEBXR_HAND_JOINT_MAX]; input_source.active = godot_webxr_update_input_source( p_input_source_id, @@ -584,7 +600,10 @@ void WebXRInterfaceJS::_update_input_source(int p_input_source_id) { &button_count, buttons, &axes_count, - axes); + axes, + &has_hand_data, + hand_joints, + hand_radii); if (!input_source.active) { if (input_source.tracker.is_valid()) { @@ -683,6 +702,56 @@ void WebXRInterfaceJS::_update_input_source(int p_input_source_id) { touches[touch_index].position = position; } } + + if (p_input_source_id < 2) { + Ref hand_tracker = hand_trackers[p_input_source_id]; + if (has_hand_data) { + // Transform orientations to match Godot Humanoid skeleton. + const Basis bone_adjustment( + Vector3(-1.0, 0.0, 0.0), + Vector3(0.0, 0.0, -1.0), + Vector3(0.0, -1.0, 0.0)); + + if (unlikely(hand_tracker.is_null())) { + hand_tracker.instantiate(); + hand_tracker->set_hand(p_input_source_id == 0 ? XRHandTracker::HAND_LEFT : XRHandTracker::HAND_RIGHT); + + // These flags always apply, since WebXR doesn't give us enough insight to be more fine grained. + BitField joint_flags(XRHandTracker::HAND_JOINT_FLAG_POSITION_VALID | XRHandTracker::HAND_JOINT_FLAG_ORIENTATION_VALID | XRHandTracker::HAND_JOINT_FLAG_POSITION_TRACKED | XRHandTracker::HAND_JOINT_FLAG_ORIENTATION_TRACKED); + for (int godot_joint = 0; godot_joint < XRHandTracker::HAND_JOINT_MAX; godot_joint++) { + hand_tracker->set_hand_joint_flags((XRHandTracker::HandJoint)godot_joint, joint_flags); + } + + hand_trackers[p_input_source_id] = hand_tracker; + xr_server->add_hand_tracker(p_input_source_id == 0 ? "/user/left" : "/user/right", hand_tracker); + } + + hand_tracker->set_has_tracking_data(true); + for (int webxr_joint = 0; webxr_joint < WEBXR_HAND_JOINT_MAX; webxr_joint++) { + XRHandTracker::HandJoint godot_joint = (XRHandTracker::HandJoint)(webxr_joint + 1); + + Transform3D joint_transform = _js_matrix_to_transform(hand_joints + (16 * webxr_joint)); + joint_transform.basis *= bone_adjustment; + hand_tracker->set_hand_joint_transform(godot_joint, joint_transform); + + hand_tracker->set_hand_joint_radius(godot_joint, hand_radii[webxr_joint]); + } + + // WebXR doesn't have a palm joint, so we calculate it by finding the middle of the middle finger metacarpal bone. + { + // 10 is the WebXR middle finger metacarpal joint, and 12 is the offset to the transform origin. + const float *start_pos = hand_joints + (10 * 16) + 12; + // 11 is the WebXR middle finger phalanx proximal joint, and 12 is the offset to the transform origin. + const float *end_pos = hand_joints + (11 * 16) + 12; + Transform3D palm_transform; + palm_transform.origin = (Vector3(start_pos[0], start_pos[1], start_pos[2]) + Vector3(end_pos[0], end_pos[1], end_pos[2])) / 2.0; + hand_tracker->set_hand_joint_transform(XRHandTracker::HAND_JOINT_PALM, palm_transform); + } + + } else if (hand_tracker.is_valid()) { + hand_tracker->set_has_tracking_data(false); + } + } } void WebXRInterfaceJS::_on_input_event(int p_event_type, int p_input_source_id) { diff --git a/modules/webxr/webxr_interface_js.h b/modules/webxr/webxr_interface_js.h index d85eb8cad74..fc5df3a59bf 100644 --- a/modules/webxr/webxr_interface_js.h +++ b/modules/webxr/webxr_interface_js.h @@ -56,6 +56,7 @@ private: String optional_features; String requested_reference_space_types; String reference_space_type; + String enabled_features; Size2 render_targetsize; RBMap texture_cache; @@ -73,6 +74,10 @@ private: int touch_index = -1; } input_sources[input_source_count]; + static const int WEBXR_HAND_JOINT_MAX = 25; + static const int HAND_MAX = 2; + Ref hand_trackers[HAND_MAX]; + RID color_texture; RID depth_texture; @@ -94,8 +99,8 @@ public: virtual String get_optional_features() const override; virtual void set_requested_reference_space_types(String p_requested_reference_space_types) override; virtual String get_requested_reference_space_types() const override; - void _set_reference_space_type(String p_reference_space_type); virtual String get_reference_space_type() const override; + virtual String get_enabled_features() const override; virtual bool is_input_source_active(int p_input_source_id) const override; virtual Ref get_input_source_tracker(int p_input_source_id) const override; virtual TargetRayMode get_input_source_target_ray_mode(int p_input_source_id) const override; @@ -129,6 +134,9 @@ public: void _on_input_event(int p_event_type, int p_input_source_id); + inline void _set_reference_space_type(String p_reference_space_type) { reference_space_type = p_reference_space_type; } + inline void _set_enabled_features(String p_enabled_features) { enabled_features = p_enabled_features; } + WebXRInterfaceJS(); ~WebXRInterfaceJS();