asgardius/virtualx-engine
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge pull request #88312 from Malcolmnixon/face-tracker-provider

Browse source 
Add XR Face Tracking support
This commit is contained in:
Rémi Verschelde 2024-02-19 10:43:45 +01:00
commit aa7ac130c0
No known key found for this signature in database
GPG key ID: C3336907360768E1
11 changed files with 1725 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

22
doc/classes/XRFaceModifier3D.xml Normal file
View file

@ -0,0 +1,22 @@
<?xml version="1.0" encoding="UTF-8" ?>
<class name="XRFaceModifier3D" inherits="Node3D" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="../class.xsd">
<brief_description>
A node for driving standard face meshes from [XRFaceTracker] weights.
</brief_description>
<description>
This node applies weights from a [XRFaceTracker] to a mesh with supporting face blend shapes.
The [url=https://docs.vrcft.io/docs/tutorial-avatars/tutorial-avatars-extras/unified-blendshapes]Unified Expressions[/url] blend shapes are supported, as well as ARKit and SRanipal blend shapes.
The node attempts to identify blend shapes based on name matching. Blend shapes should match the names listed in the [url=https://docs.vrcft.io/docs/tutorial-avatars/tutorial-avatars-extras/compatibility/overview]Unified Expressions Compatibility[/url] chart.
</description>
<tutorials>
<link title="XR documentation index">$DOCS_URL/tutorials/xr/index.html</link>
</tutorials>
<members>
<member name="face_tracker" type="StringName" setter="set_face_tracker" getter="get_face_tracker" default="&amp;&quot;/user/head&quot;">
The [XRFaceTracker] path.
</member>
<member name="target" type="NodePath" setter="set_target" getter="get_target" default="NodePath(&quot;&quot;)">
The [NodePath] of the face [MeshInstance3D].
</member>
</members>
</class>

469
doc/classes/XRFaceTracker.xml Normal file
View file

@ -0,0 +1,469 @@
<?xml version="1.0" encoding="UTF-8" ?>
<class name="XRFaceTracker" inherits="RefCounted" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="../class.xsd">
<brief_description>
A tracked face.
</brief_description>
<description>
An instance of this object represents a tracked face and its corresponding blend shapes. The blend shapes come from the [url=https://docs.vrcft.io/docs/tutorial-avatars/tutorial-avatars-extras/unified-blendshapes]Unified Expressions[/url] standard, and contain extended details and visuals for each blend shape. Additionally the [url=https://docs.vrcft.io/docs/tutorial-avatars/tutorial-avatars-extras/compatibility/overview]Tracking Standard Comparison[/url] page documents the relationship between Unified Expressions and other standards.
As face trackers are turned on they are registered with the [XRServer].
</description>
<tutorials>
<link title="XR documentation index">$DOCS_URL/tutorials/xr/index.html</link>
</tutorials>
<methods>
<method name="get_blend_shape" qualifiers="const">
<return type="float" />
<param index="0" name="blend_shape" type="int" enum="XRFaceTracker.BlendShapeEntry" />
<description>
Returns the requested face blend shape weight.
</description>
</method>
<method name="set_blend_shape">
<return type="void" />
<param index="0" name="blend_shape" type="int" enum="XRFaceTracker.BlendShapeEntry" />
<param index="1" name="weight" type="float" />
<description>
Sets a face blend shape weight.
</description>
</method>
</methods>
<members>
<member name="blend_shapes" type="PackedFloat32Array" setter="set_blend_shapes" getter="get_blend_shapes" default="PackedFloat32Array()">
The array of face blend shape weights with indices corresponding to the [enum BlendShapeEntry] enum.
</member>
</members>
<constants>
<constant name="FT_EYE_LOOK_OUT_RIGHT" value="0" enum="BlendShapeEntry">
Right eye looks outwards.
</constant>
<constant name="FT_EYE_LOOK_IN_RIGHT" value="1" enum="BlendShapeEntry">
Right eye looks inwards.
</constant>
<constant name="FT_EYE_LOOK_UP_RIGHT" value="2" enum="BlendShapeEntry">
Right eye looks upwards.
</constant>
<constant name="FT_EYE_LOOK_DOWN_RIGHT" value="3" enum="BlendShapeEntry">
Right eye looks downwards.
</constant>
<constant name="FT_EYE_LOOK_OUT_LEFT" value="4" enum="BlendShapeEntry">
Left eye looks outwards.
</constant>
<constant name="FT_EYE_LOOK_IN_LEFT" value="5" enum="BlendShapeEntry">
Left eye looks inwards.
</constant>
<constant name="FT_EYE_LOOK_UP_LEFT" value="6" enum="BlendShapeEntry">
Left eye looks upwards.
</constant>
<constant name="FT_EYE_LOOK_DOWN_LEFT" value="7" enum="BlendShapeEntry">
Left eye looks downwards.
</constant>
<constant name="FT_EYE_CLOSED_RIGHT" value="8" enum="BlendShapeEntry">
Closes the right eyelid.
</constant>
<constant name="FT_EYE_CLOSED_LEFT" value="9" enum="BlendShapeEntry">
Closes the left eyelid.
</constant>
<constant name="FT_EYE_SQUINT_RIGHT" value="10" enum="BlendShapeEntry">
Squeezes the right eye socket muscles.
</constant>
<constant name="FT_EYE_SQUINT_LEFT" value="11" enum="BlendShapeEntry">
Squeezes the left eye socket muscles.
</constant>
<constant name="FT_EYE_WIDE_RIGHT" value="12" enum="BlendShapeEntry">
Right eyelid widens beyond relaxed.
</constant>
<constant name="FT_EYE_WIDE_LEFT" value="13" enum="BlendShapeEntry">
Left eyelid widens beyond relaxed.
</constant>
<constant name="FT_EYE_DILATION_RIGHT" value="14" enum="BlendShapeEntry">
Dilates the right eye pupil.
</constant>
<constant name="FT_EYE_DILATION_LEFT" value="15" enum="BlendShapeEntry">
Dilates the left eye pupil.
</constant>
<constant name="FT_EYE_CONSTRICT_RIGHT" value="16" enum="BlendShapeEntry">
Constricts the right eye pupil.
</constant>
<constant name="FT_EYE_CONSTRICT_LEFT" value="17" enum="BlendShapeEntry">
Constricts the left eye pupil.
</constant>
<constant name="FT_BROW_PINCH_RIGHT" value="18" enum="BlendShapeEntry">
Right eyebrow pinches in.
</constant>
<constant name="FT_BROW_PINCH_LEFT" value="19" enum="BlendShapeEntry">
Left eyebrow pinches in.
</constant>
<constant name="FT_BROW_LOWERER_RIGHT" value="20" enum="BlendShapeEntry">
Outer right eyebrow pulls down.
</constant>
<constant name="FT_BROW_LOWERER_LEFT" value="21" enum="BlendShapeEntry">
Outer left eyebrow pulls down.
</constant>
<constant name="FT_BROW_INNER_UP_RIGHT" value="22" enum="BlendShapeEntry">
Inner right eyebrow pulls up.
</constant>
<constant name="FT_BROW_INNER_UP_LEFT" value="23" enum="BlendShapeEntry">
Inner left eyebrow pulls up.
</constant>
<constant name="FT_BROW_OUTER_UP_RIGHT" value="24" enum="BlendShapeEntry">
Outer right eyebrow pulls up.
</constant>
<constant name="FT_BROW_OUTER_UP_LEFT" value="25" enum="BlendShapeEntry">
Outer left eyebrow pulls up.
</constant>
<constant name="FT_NOSE_SNEER_RIGHT" value="26" enum="BlendShapeEntry">
Right side face sneers.
</constant>
<constant name="FT_NOSE_SNEER_LEFT" value="27" enum="BlendShapeEntry">
Left side face sneers.
</constant>
<constant name="FT_NASAL_DILATION_RIGHT" value="28" enum="BlendShapeEntry">
Right side nose canal dilates.
</constant>
<constant name="FT_NASAL_DILATION_LEFT" value="29" enum="BlendShapeEntry">
Left side nose canal dilates.
</constant>
<constant name="FT_NASAL_CONSTRICT_RIGHT" value="30" enum="BlendShapeEntry">
Right side nose canal constricts.
</constant>
<constant name="FT_NASAL_CONSTRICT_LEFT" value="31" enum="BlendShapeEntry">
Left side nose canal constricts.
</constant>
<constant name="FT_CHEEK_SQUINT_RIGHT" value="32" enum="BlendShapeEntry">
Raises the right side cheek.
</constant>
<constant name="FT_CHEEK_SQUINT_LEFT" value="33" enum="BlendShapeEntry">
Raises the left side cheek.
</constant>
<constant name="FT_CHEEK_PUFF_RIGHT" value="34" enum="BlendShapeEntry">
Puffs the right side cheek.
</constant>
<constant name="FT_CHEEK_PUFF_LEFT" value="35" enum="BlendShapeEntry">
Puffs the left side cheek.
</constant>
<constant name="FT_CHEEK_SUCK_RIGHT" value="36" enum="BlendShapeEntry">
Sucks in the right side cheek.
</constant>
<constant name="FT_CHEEK_SUCK_LEFT" value="37" enum="BlendShapeEntry">
Sucks in the left side cheek.
</constant>
<constant name="FT_JAW_OPEN" value="38" enum="BlendShapeEntry">
Opens jawbone.
</constant>
<constant name="FT_MOUTH_CLOSED" value="39" enum="BlendShapeEntry">
Closes the mouth.
</constant>
<constant name="FT_JAW_RIGHT" value="40" enum="BlendShapeEntry">
Pushes jawbone right.
</constant>
<constant name="FT_JAW_LEFT" value="41" enum="BlendShapeEntry">
Pushes jawbone left.
</constant>
<constant name="FT_JAW_FORWARD" value="42" enum="BlendShapeEntry">
Pushes jawbone forward.
</constant>
<constant name="FT_JAW_BACKWARD" value="43" enum="BlendShapeEntry">
Pushes jawbone backward.
</constant>
<constant name="FT_JAW_CLENCH" value="44" enum="BlendShapeEntry">
Flexes jaw muscles.
</constant>
<constant name="FT_JAW_MANDIBLE_RAISE" value="45" enum="BlendShapeEntry">
Raises the jawbone.
</constant>
<constant name="FT_LIP_SUCK_UPPER_RIGHT" value="46" enum="BlendShapeEntry">
Upper right lip part tucks in the mouth.
</constant>
<constant name="FT_LIP_SUCK_UPPER_LEFT" value="47" enum="BlendShapeEntry">
Upper left lip part tucks in the mouth.
</constant>
<constant name="FT_LIP_SUCK_LOWER_RIGHT" value="48" enum="BlendShapeEntry">
Lower right lip part tucks in the mouth.
</constant>
<constant name="FT_LIP_SUCK_LOWER_LEFT" value="49" enum="BlendShapeEntry">
Lower left lip part tucks in the mouth.
</constant>
<constant name="FT_LIP_SUCK_CORNER_RIGHT" value="50" enum="BlendShapeEntry">
Right lip corner folds into the mouth.
</constant>
<constant name="FT_LIP_SUCK_CORNER_LEFT" value="51" enum="BlendShapeEntry">
Left lip corner folds into the mouth.
</constant>
<constant name="FT_LIP_FUNNEL_UPPER_RIGHT" value="52" enum="BlendShapeEntry">
Upper right lip part pushes into a funnel.
</constant>
<constant name="FT_LIP_FUNNEL_UPPER_LEFT" value="53" enum="BlendShapeEntry">
Upper left lip part pushes into a funnel.
</constant>
<constant name="FT_LIP_FUNNEL_LOWER_RIGHT" value="54" enum="BlendShapeEntry">
Lower right lip part pushes into a funnel.
</constant>
<constant name="FT_LIP_FUNNEL_LOWER_LEFT" value="55" enum="BlendShapeEntry">
Lower left lip part pushes into a funnel.
</constant>
<constant name="FT_LIP_PUCKER_UPPER_RIGHT" value="56" enum="BlendShapeEntry">
Upper right lip part pushes outwards.
</constant>
<constant name="FT_LIP_PUCKER_UPPER_LEFT" value="57" enum="BlendShapeEntry">
Upper left lip part pushes outwards.
</constant>
<constant name="FT_LIP_PUCKER_LOWER_RIGHT" value="58" enum="BlendShapeEntry">
Lower right lip part pushes outwards.
</constant>
<constant name="FT_LIP_PUCKER_LOWER_LEFT" value="59" enum="BlendShapeEntry">
Lower left lip part pushes outwards.
</constant>
<constant name="FT_MOUTH_UPPER_UP_RIGHT" value="60" enum="BlendShapeEntry">
Upper right part of the lip pulls up.
</constant>
<constant name="FT_MOUTH_UPPER_UP_LEFT" value="61" enum="BlendShapeEntry">
Upper left part of the lip pulls up.
</constant>
<constant name="FT_MOUTH_LOWER_DOWN_RIGHT" value="62" enum="BlendShapeEntry">
Lower right part of the lip pulls up.
</constant>
<constant name="FT_MOUTH_LOWER_DOWN_LEFT" value="63" enum="BlendShapeEntry">
Lower left part of the lip pulls up.
</constant>
<constant name="FT_MOUTH_UPPER_DEEPEN_RIGHT" value="64" enum="BlendShapeEntry">
Upper right lip part pushes in the cheek.
</constant>
<constant name="FT_MOUTH_UPPER_DEEPEN_LEFT" value="65" enum="BlendShapeEntry">
Upper left lip part pushes in the cheek.
</constant>
<constant name="FT_MOUTH_UPPER_RIGHT" value="66" enum="BlendShapeEntry">
Moves upper lip right.
</constant>
<constant name="FT_MOUTH_UPPER_LEFT" value="67" enum="BlendShapeEntry">
Moves upper lip left.
</constant>
<constant name="FT_MOUTH_LOWER_RIGHT" value="68" enum="BlendShapeEntry">
Moves lower lip right.
</constant>
<constant name="FT_MOUTH_LOWER_LEFT" value="69" enum="BlendShapeEntry">
Moves lower lip left.
</constant>
<constant name="FT_MOUTH_CORNER_PULL_RIGHT" value="70" enum="BlendShapeEntry">
Right lip corner pulls diagonally up and out.
</constant>
<constant name="FT_MOUTH_CORNER_PULL_LEFT" value="71" enum="BlendShapeEntry">
Left lip corner pulls diagonally up and out.
</constant>
<constant name="FT_MOUTH_CORNER_SLANT_RIGHT" value="72" enum="BlendShapeEntry">
Right corner lip slants up.
</constant>
<constant name="FT_MOUTH_CORNER_SLANT_LEFT" value="73" enum="BlendShapeEntry">
Left corner lip slants up.
</constant>
<constant name="FT_MOUTH_FROWN_RIGHT" value="74" enum="BlendShapeEntry">
Right corner lip pulls down.
</constant>
<constant name="FT_MOUTH_FROWN_LEFT" value="75" enum="BlendShapeEntry">
Left corner lip pulls down.
</constant>
<constant name="FT_MOUTH_STRETCH_RIGHT" value="76" enum="BlendShapeEntry">
Mouth corner lip pulls out and down.
</constant>
<constant name="FT_MOUTH_STRETCH_LEFT" value="77" enum="BlendShapeEntry">
Mouth corner lip pulls out and down.
</constant>
<constant name="FT_MOUTH_DIMPLE_RIGHT" value="78" enum="BlendShapeEntry">
Right lip corner is pushed backwards.
</constant>
<constant name="FT_MOUTH_DIMPLE_LEFT" value="79" enum="BlendShapeEntry">
Left lip corner is pushed backwards.
</constant>
<constant name="FT_MOUTH_RAISER_UPPER" value="80" enum="BlendShapeEntry">
Raises and slightly pushes out the upper mouth.
</constant>
<constant name="FT_MOUTH_RAISER_LOWER" value="81" enum="BlendShapeEntry">
Raises and slightly pushes out the lower mouth.
</constant>
<constant name="FT_MOUTH_PRESS_RIGHT" value="82" enum="BlendShapeEntry">
Right side lips press and flatten together vertically.
</constant>
<constant name="FT_MOUTH_PRESS_LEFT" value="83" enum="BlendShapeEntry">
Left side lips press and flatten together vertically.
</constant>
<constant name="FT_MOUTH_TIGHTENER_RIGHT" value="84" enum="BlendShapeEntry">
Right side lips squeeze together horizontally.
</constant>
<constant name="FT_MOUTH_TIGHTENER_LEFT" value="85" enum="BlendShapeEntry">
Left side lips squeeze together horizontally.
</constant>
<constant name="FT_TONGUE_OUT" value="86" enum="BlendShapeEntry">
Tongue visibly sticks out of the mouth.
</constant>
<constant name="FT_TONGUE_UP" value="87" enum="BlendShapeEntry">
Tongue points upwards.
</constant>
<constant name="FT_TONGUE_DOWN" value="88" enum="BlendShapeEntry">
Tongue points downwards.
</constant>
<constant name="FT_TONGUE_RIGHT" value="89" enum="BlendShapeEntry">
Tongue points right.
</constant>
<constant name="FT_TONGUE_LEFT" value="90" enum="BlendShapeEntry">
Tongue points left.
</constant>
<constant name="FT_TONGUE_ROLL" value="91" enum="BlendShapeEntry">
Sides of the tongue funnel, creating a roll.
</constant>
<constant name="FT_TONGUE_BLEND_DOWN" value="92" enum="BlendShapeEntry">
Tongue arches up then down inside the mouth.
</constant>
<constant name="FT_TONGUE_CURL_UP" value="93" enum="BlendShapeEntry">
Tongue arches down then up inside the mouth.
</constant>
<constant name="FT_TONGUE_SQUISH" value="94" enum="BlendShapeEntry">
Tongue squishes together and thickens.
</constant>
<constant name="FT_TONGUE_FLAT" value="95" enum="BlendShapeEntry">
Tongue flattens and thins out.
</constant>
<constant name="FT_TONGUE_TWIST_RIGHT" value="96" enum="BlendShapeEntry">
Tongue tip rotates clockwise, with the rest following gradually.
</constant>
<constant name="FT_TONGUE_TWIST_LEFT" value="97" enum="BlendShapeEntry">
Tongue tip rotates counter-clockwise, with the rest following gradually.
</constant>
<constant name="FT_SOFT_PALATE_CLOSE" value="98" enum="BlendShapeEntry">
Inner mouth throat closes.
</constant>
<constant name="FT_THROAT_SWALLOW" value="99" enum="BlendShapeEntry">
The Adam's apple visibly swallows.
</constant>
<constant name="FT_NECK_FLEX_RIGHT" value="100" enum="BlendShapeEntry">
Right side neck visibly flexes.
</constant>
<constant name="FT_NECK_FLEX_LEFT" value="101" enum="BlendShapeEntry">
Left side neck visibly flexes.
</constant>
<constant name="FT_EYE_CLOSED" value="102" enum="BlendShapeEntry">
Closes both eye lids.
</constant>
<constant name="FT_EYE_WIDE" value="103" enum="BlendShapeEntry">
Widens both eye lids.
</constant>
<constant name="FT_EYE_SQUINT" value="104" enum="BlendShapeEntry">
Squints both eye lids.
</constant>
<constant name="FT_EYE_DILATION" value="105" enum="BlendShapeEntry">
Dilates both pupils.
</constant>
<constant name="FT_EYE_CONSTRICT" value="106" enum="BlendShapeEntry">
Constricts both pupils.
</constant>
<constant name="FT_BROW_DOWN_RIGHT" value="107" enum="BlendShapeEntry">
Pulls the right eyebrow down and in.
</constant>
<constant name="FT_BROW_DOWN_LEFT" value="108" enum="BlendShapeEntry">
Pulls the left eyebrow down and in.
</constant>
<constant name="FT_BROW_DOWN" value="109" enum="BlendShapeEntry">
Pulls both eyebrows down and in.
</constant>
<constant name="FT_BROW_UP_RIGHT" value="110" enum="BlendShapeEntry">
Right brow appears worried.
</constant>
<constant name="FT_BROW_UP_LEFT" value="111" enum="BlendShapeEntry">
Left brow appears worried.
</constant>
<constant name="FT_BROW_UP" value="112" enum="BlendShapeEntry">
Both brows appear worried.
</constant>
<constant name="FT_NOSE_SNEER" value="113" enum="BlendShapeEntry">
Entire face sneers.
</constant>
<constant name="FT_NASAL_DILATION" value="114" enum="BlendShapeEntry">
Both nose canals dilate.
</constant>
<constant name="FT_NASAL_CONSTRICT" value="115" enum="BlendShapeEntry">
Both nose canals constrict.
</constant>
<constant name="FT_CHEEK_PUFF" value="116" enum="BlendShapeEntry">
Puffs both cheeks.
</constant>
<constant name="FT_CHEEK_SUCK" value="117" enum="BlendShapeEntry">
Sucks in both cheeks.
</constant>
<constant name="FT_CHEEK_SQUINT" value="118" enum="BlendShapeEntry">
Raises both cheeks.
</constant>
<constant name="FT_LIP_SUCK_UPPER" value="119" enum="BlendShapeEntry">
Tucks in the upper lips.
</constant>
<constant name="FT_LIP_SUCK_LOWER" value="120" enum="BlendShapeEntry">
Tucks in the lower lips.
</constant>
<constant name="FT_LIP_SUCK" value="121" enum="BlendShapeEntry">
Tucks in both lips.
</constant>
<constant name="FT_LIP_FUNNEL_UPPER" value="122" enum="BlendShapeEntry">
Funnels in the upper lips.
</constant>
<constant name="FT_LIP_FUNNEL_LOWER" value="123" enum="BlendShapeEntry">
Funnels in the lower lips.
</constant>
<constant name="FT_LIP_FUNNEL" value="124" enum="BlendShapeEntry">
Funnels in both lips.
</constant>
<constant name="FT_LIP_PUCKER_UPPER" value="125" enum="BlendShapeEntry">
Upper lip part pushes outwards.
</constant>
<constant name="FT_LIP_PUCKER_LOWER" value="126" enum="BlendShapeEntry">
Lower lip part pushes outwards.
</constant>
<constant name="FT_LIP_PUCKER" value="127" enum="BlendShapeEntry">
Lips push outwards.
</constant>
<constant name="FT_MOUTH_UPPER_UP" value="128" enum="BlendShapeEntry">
Raises the upper lips.
</constant>
<constant name="FT_MOUTH_LOWER_DOWN" value="129" enum="BlendShapeEntry">
Lowers the lower lips.
</constant>
<constant name="FT_MOUTH_OPEN" value="130" enum="BlendShapeEntry">
Mouth opens, revealing teeth.
</constant>
<constant name="FT_MOUTH_RIGHT" value="131" enum="BlendShapeEntry">
Moves mouth right.
</constant>
<constant name="FT_MOUTH_LEFT" value="132" enum="BlendShapeEntry">
Moves mouth left.
</constant>
<constant name="FT_MOUTH_SMILE_RIGHT" value="133" enum="BlendShapeEntry">
Right side of the mouth smiles.
</constant>
<constant name="FT_MOUTH_SMILE_LEFT" value="134" enum="BlendShapeEntry">
Left side of the mouth smiles.
</constant>
<constant name="FT_MOUTH_SMILE" value="135" enum="BlendShapeEntry">
Mouth expresses a smile.
</constant>
<constant name="FT_MOUTH_SAD_RIGHT" value="136" enum="BlendShapeEntry">
Right side of the mouth expresses sadness.
</constant>
<constant name="FT_MOUTH_SAD_LEFT" value="137" enum="BlendShapeEntry">
Left side of the mouth expresses sadness.
</constant>
<constant name="FT_MOUTH_SAD" value="138" enum="BlendShapeEntry">
Mouth expresses sadness.
</constant>
<constant name="FT_MOUTH_STRETCH" value="139" enum="BlendShapeEntry">
Mouth stretches.
</constant>
<constant name="FT_MOUTH_DIMPLE" value="140" enum="BlendShapeEntry">
Lip corners dimple.
</constant>
<constant name="FT_MOUTH_TIGHTENER" value="141" enum="BlendShapeEntry">
Mouth tightens.
</constant>
<constant name="FT_MOUTH_PRESS" value="142" enum="BlendShapeEntry">
Mouth presses together.
</constant>
<constant name="FT_MAX" value="143" enum="BlendShapeEntry">
Represents the size of the [enum BlendShapeEntry] enum.
</constant>
</constants>
</class>

48
doc/classes/XRServer.xml
View file

@ -10,6 +10,14 @@
<link title="XR documentation index">$DOCS_URL/tutorials/xr/index.html</link>
</tutorials>
<methods>
<method name="add_face_tracker">
<return type="void" />
<param index="0" name="tracker_name" type="StringName" />
<param index="1" name="face_tracker" type="XRFaceTracker" />
<description>
Registers a new [XRFaceTracker] that tracks the blend shapes of a face.
</description>
</method>
<method name="add_interface">
<return type="void" />
<param index="0" name="interface" type="XRInterface" />
@ -50,6 +58,19 @@
Finds an interface by its [param name]. For example, if your project uses capabilities of an AR/VR platform, you can find the interface for that platform by name and initialize it.
</description>
</method>
<method name="get_face_tracker" qualifiers="const">
<return type="XRFaceTracker" />
<param index="0" name="tracker_name" type="StringName" />
<description>
Returns the [XRFaceTracker] with the given tracker name.
</description>
</method>
<method name="get_face_trackers" qualifiers="const">
<return type="Dictionary" />
<description>
Returns a dictionary of the registered face trackers. Each element of the dictionary is a tracker name mapping to the [XRFaceTracker] instance.
</description>
</method>
<method name="get_hmd_transform">
<return type="Transform3D" />
<description>
@ -95,6 +116,13 @@
Returns a dictionary of trackers for [param tracker_types].
</description>
</method>
<method name="remove_face_tracker">
<return type="void" />
<param index="0" name="tracker_name" type="StringName" />
<description>
Removes a registered [XRFaceTracker].
</description>
</method>
<method name="remove_interface">
<return type="void" />
<param index="0" name="interface" type="XRInterface" />
@ -123,6 +151,26 @@
</member>
</members>
<signals>
<signal name="face_tracker_added">
<param index="0" name="tracker_name" type="StringName" />
<param index="1" name="face_tracker" type="XRFaceTracker" />
<description>
Emitted when a new face tracker is added.
</description>
</signal>
<signal name="face_tracker_removed">
<param index="0" name="tracker_name" type="StringName" />
<description>
Emitted when a face tracker is removed.
</description>
</signal>
<signal name="face_tracker_updated">
<param index="0" name="tracker_name" type="StringName" />
<param index="1" name="face_tracker" type="XRFaceTracker" />
<description>
Emitted when an existing face tracker is updated.
</description>
</signal>
<signal name="interface_added">
<param index="0" name="interface_name" type="StringName" />
<description>

615
scene/3d/xr_face_modifier_3d.cpp Normal file
View file

@ -0,0 +1,615 @@
/**************************************************************************/
/* xr_face_modifier_3d.cpp */
/**************************************************************************/
/* This file is part of: */
/* GODOT ENGINE */
/* https://godotengine.org */
/**************************************************************************/
/* Copyright (c) 2014-present Godot Engine contributors (see AUTHORS.md). */
/* Copyright (c) 2007-2014 Juan Linietsky, Ariel Manzur. */
/* */
/* Permission is hereby granted, free of charge, to any person obtaining */
/* a copy of this software and associated documentation files (the */
/* "Software"), to deal in the Software without restriction, including */
/* without limitation the rights to use, copy, modify, merge, publish, */
/* distribute, sublicense, and/or sell copies of the Software, and to */
/* permit persons to whom the Software is furnished to do so, subject to */
/* the following conditions: */
/* */
/* The above copyright notice and this permission notice shall be */
/* included in all copies or substantial portions of the Software. */
/* */
/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, */
/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF */
/* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. */
/* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY */
/* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, */
/* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE */
/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */
/**************************************************************************/
#include "xr_face_modifier_3d.h"
#include "servers/xr/xr_face_tracker.h"
#include "servers/xr_server.h"
// This method takes the name of a mesh blend shape and returns the
// corresponding XRFaceTracker blend shape. If no match is
// found then the function returns -1.
static int find_face_blend_shape(const StringName &p_name) {
// Entry for blend shape name table.
struct blend_map_entry {
int blend;
const char *name[4];
};
// Table of blend shape names.
//
// This table consists of the XRFaceTracker blend shape and
// the corresponding names (lowercase and no underscore) of:
// - The Unified Expression blend shape name.
// - The ARKit blend shape name (if present and different).
// - The SRanipal blend shape name (if present and different).
// - The Meta blend shape name (if present and different).
static constexpr blend_map_entry blend_map[] = {
{ XRFaceTracker::FT_EYE_LOOK_OUT_RIGHT,
{ "eyelookoutright", "eyerightright", "eyeslookoutr" } },
{ XRFaceTracker::FT_EYE_LOOK_IN_RIGHT,
{ "eyelookinright", "eyerightleft", "eyeslookinr" } },
{ XRFaceTracker::FT_EYE_LOOK_UP_RIGHT,
{ "eyelookupright", "eyerightlookup", "eyeslookupr" } },
{ XRFaceTracker::FT_EYE_LOOK_DOWN_RIGHT,
{ "eyelookdownright", "eyerightlookdown", "eyeslookdownr" } },
{ XRFaceTracker::FT_EYE_LOOK_OUT_LEFT,
{ "eyelookoutleft", "eyeleftleft", "eyeslookoutl" } },
{ XRFaceTracker::FT_EYE_LOOK_IN_LEFT,
{ "eyelookinleft", "eyeleftright", "eyeslookinl" } },
{ XRFaceTracker::FT_EYE_LOOK_UP_LEFT,
{ "eyelookupleft", "eyeleftlookup", "eyeslookupl" } },
{ XRFaceTracker::FT_EYE_LOOK_DOWN_LEFT,
{ "eyelookdownleft", "eyeleftlookdown", "eyeslookdownl" } },
{ XRFaceTracker::FT_EYE_CLOSED_RIGHT,
{ "eyeclosedright", "eyeblinkright", "eyerightblink", "eyesclosedr" } },
{ XRFaceTracker::FT_EYE_CLOSED_LEFT,
{ "eyeclosedleft", "eyeblinkleft", "eyeleftblink", "eyesclosedl" } },
{ XRFaceTracker::FT_EYE_SQUINT_RIGHT,
{ "eyesquintright", "eyessquintr" } },
{ XRFaceTracker::FT_EYE_SQUINT_LEFT,
{ "eyesquintleft", "eyessquintl" } },
{ XRFaceTracker::FT_EYE_WIDE_RIGHT,
{ "eyewideright", "eyerightwide", "eyeswidenr" } },
{ XRFaceTracker::FT_EYE_WIDE_LEFT,
{ "eyewideleft", "eyeleftwide", "eyeswidenl" } },
{ XRFaceTracker::FT_EYE_DILATION_RIGHT,
{ "eyedilationright", "eyerightdilation" } },
{ XRFaceTracker::FT_EYE_DILATION_LEFT,
{ "eyedilationleft", "eyeleftdilation" } },
{ XRFaceTracker::FT_EYE_CONSTRICT_RIGHT,
{ "eyeconstrictright", "eyerightconstrict" } },
{ XRFaceTracker::FT_EYE_CONSTRICT_LEFT,
{ "eyeconstrictleft", "eyeleftconstrict" } },
{ XRFaceTracker::FT_BROW_PINCH_RIGHT,
{ "browpinchright" } },
{ XRFaceTracker::FT_BROW_PINCH_LEFT,
{ "browpinchleft" } },
{ XRFaceTracker::FT_BROW_LOWERER_RIGHT,
{ "browlowererright" } },
{ XRFaceTracker::FT_BROW_LOWERER_LEFT,
{ "browlowererleft" } },
{ XRFaceTracker::FT_BROW_INNER_UP_RIGHT,
{ "browinnerupright", "innerbrowraiserr" } },
{ XRFaceTracker::FT_BROW_INNER_UP_LEFT,
{ "browinnerupleft", "innerbrowraiserl" } },
{ XRFaceTracker::FT_BROW_OUTER_UP_RIGHT,
{ "browouterupright", "outerbrowraiserr" } },
{ XRFaceTracker::FT_BROW_OUTER_UP_LEFT,
{ "browouterupleft", "outerbrowraiserl" } },
{ XRFaceTracker::FT_NOSE_SNEER_RIGHT,
{ "nosesneerright", "nosewrinklerr" } },
{ XRFaceTracker::FT_NOSE_SNEER_LEFT,
{ "nosesneerleft", "nosewrinklerl" } },
{ XRFaceTracker::FT_NASAL_DILATION_RIGHT,
{ "nasaldilationright" } },
{ XRFaceTracker::FT_NASAL_DILATION_LEFT,
{ "nasaldilationleft" } },
{ XRFaceTracker::FT_NASAL_CONSTRICT_RIGHT,
{ "nasalconstrictright" } },
{ XRFaceTracker::FT_NASAL_CONSTRICT_LEFT,
{ "nasalconstrictleft" } },
{ XRFaceTracker::FT_CHEEK_SQUINT_RIGHT,
{ "cheeksquintright", "cheekraiserr" } },
{ XRFaceTracker::FT_CHEEK_SQUINT_LEFT,
{ "cheeksquintleft", "cheekraiserl" } },
{ XRFaceTracker::FT_CHEEK_PUFF_RIGHT,
{ "cheekpuffright", "cheekpuffr" } },
{ XRFaceTracker::FT_CHEEK_PUFF_LEFT,
{ "cheekpuffleft", "cheekpuffl" } },
{ XRFaceTracker::FT_CHEEK_SUCK_RIGHT,
{ "cheeksuckright", "cheeksuckr" } },
{ XRFaceTracker::FT_CHEEK_SUCK_LEFT,
{ "cheeksuckleft", "cheeksuckl" } },
{ XRFaceTracker::FT_JAW_OPEN,
{ "jawopen", "jawdrop" } },
{ XRFaceTracker::FT_MOUTH_CLOSED,
{ "mouthclosed", "mouthclose", "mouthapeshape", "lipstoward" } },
{ XRFaceTracker::FT_JAW_RIGHT,
{ "jawright", "jawsidewaysright" } },
{ XRFaceTracker::FT_JAW_LEFT,
{ "jawleft", "jawsidewaysleft" } },
{ XRFaceTracker::FT_JAW_FORWARD,
{ "jawforward", "jawthrust" } },
{ XRFaceTracker::FT_JAW_BACKWARD,
{ "jawbackward" } },
{ XRFaceTracker::FT_JAW_CLENCH,
{ "jawclench" } },
{ XRFaceTracker::FT_JAW_MANDIBLE_RAISE,
{ "jawmandibleraise" } },
{ XRFaceTracker::FT_LIP_SUCK_UPPER_RIGHT,
{ "lipsuckupperright", "lipsuckrt" } },
{ XRFaceTracker::FT_LIP_SUCK_UPPER_LEFT,
{ "lipsuckupperleft", "lipsucklt" } },
{ XRFaceTracker::FT_LIP_SUCK_LOWER_RIGHT,
{ "lipsucklowerright", "lipsuckrb" } },
{ XRFaceTracker::FT_LIP_SUCK_LOWER_LEFT,
{ "lipsucklowerleft", "lipsucklb" } },
{ XRFaceTracker::FT_LIP_SUCK_CORNER_RIGHT,
{ "lipsuckcornerright" } },
{ XRFaceTracker::FT_LIP_SUCK_CORNER_LEFT,
{ "lipsuckcornerleft" } },
{ XRFaceTracker::FT_LIP_FUNNEL_UPPER_RIGHT,
{ "lipfunnelupperright", "lipfunnelerrt" } },
{ XRFaceTracker::FT_LIP_FUNNEL_UPPER_LEFT,
{ "lipfunnelupperleft", "lipfunnelerlt" } },
{ XRFaceTracker::FT_LIP_FUNNEL_LOWER_RIGHT,
{ "lipfunnellowerright", "lipsuckrb" } },
{ XRFaceTracker::FT_LIP_FUNNEL_LOWER_LEFT,
{ "lipfunnellowerleft", "lipsucklb" } },
{ XRFaceTracker::FT_LIP_PUCKER_UPPER_RIGHT,
{ "lippuckerupperright" } },
{ XRFaceTracker::FT_LIP_PUCKER_UPPER_LEFT,
{ "lippuckerupperleft" } },
{ XRFaceTracker::FT_LIP_PUCKER_LOWER_RIGHT,
{ "lippuckerlowerright" } },
{ XRFaceTracker::FT_LIP_PUCKER_LOWER_LEFT,
{ "lippuckerlowerleft" } },
{ XRFaceTracker::FT_MOUTH_UPPER_UP_RIGHT,
{ "mouthupperupright", "upperlipraiserr" } },
{ XRFaceTracker::FT_MOUTH_UPPER_UP_LEFT,
{ "mouthupperupleft", "upperlipraiserl" } },
{ XRFaceTracker::FT_MOUTH_LOWER_DOWN_RIGHT,
{ "mouthlowerdownright", "mouthlowerupright", "lowerlipdepressorr" } },
{ XRFaceTracker::FT_MOUTH_LOWER_DOWN_LEFT,
{ "mouthlowerdownleft", "mouthlowerupleft", "lowerlipdepressorl" } },
{ XRFaceTracker::FT_MOUTH_UPPER_DEEPEN_RIGHT,
{ "mouthupperdeepenright" } },
{ XRFaceTracker::FT_MOUTH_UPPER_DEEPEN_LEFT,
{ "mouthupperdeepenleft" } },
{ XRFaceTracker::FT_MOUTH_UPPER_RIGHT,
{ "mouthupperright" } },
{ XRFaceTracker::FT_MOUTH_UPPER_LEFT,
{ "mouthupperleft" } },
{ XRFaceTracker::FT_MOUTH_LOWER_RIGHT,
{ "mouthlowerright" } },
{ XRFaceTracker::FT_MOUTH_LOWER_LEFT,
{ "mouthlowerleft" } },
{ XRFaceTracker::FT_MOUTH_CORNER_PULL_RIGHT,
{ "mouthcornerpullright" } },
{ XRFaceTracker::FT_MOUTH_CORNER_PULL_LEFT,
{ "mouthcornerpullleft" } },
{ XRFaceTracker::FT_MOUTH_CORNER_SLANT_RIGHT,
{ "mouthcornerslantright" } },
{ XRFaceTracker::FT_MOUTH_CORNER_SLANT_LEFT,
{ "mouthcornerslantleft" } },
{ XRFaceTracker::FT_MOUTH_FROWN_RIGHT,
{ "mouthfrownright", "lipcornerdepressorr" } },
{ XRFaceTracker::FT_MOUTH_FROWN_LEFT,
{ "mouthfrownleft", "lipcornerdepressorl" } },
{ XRFaceTracker::FT_MOUTH_STRETCH_RIGHT,
{ "mouthstretchright", "lipstretcherr" } },
{ XRFaceTracker::FT_MOUTH_STRETCH_LEFT,
{ "mouthstretchleft", "lipstretcherl" } },
{ XRFaceTracker::FT_MOUTH_DIMPLE_RIGHT,
{ "mouthdimplerright", "mouthdimpleright", "dimplerr" } },
{ XRFaceTracker::FT_MOUTH_DIMPLE_LEFT,
{ "mouthdimplerleft", "mouthdimpleleft", "dimplerl" } },
{ XRFaceTracker::FT_MOUTH_RAISER_UPPER,
{ "mouthraiserupper", "mouthshrugupper", "chinraisert" } },
{ XRFaceTracker::FT_MOUTH_RAISER_LOWER,
{ "mouthraiserlower", "mouthshruglower", "mouthloweroverlay", "chinraiserb" } },
{ XRFaceTracker::FT_MOUTH_PRESS_RIGHT,
{ "mouthpressright", "lippressorr" } },
{ XRFaceTracker::FT_MOUTH_PRESS_LEFT,
{ "mouthpressleft", "lippressorl" } },
{ XRFaceTracker::FT_MOUTH_TIGHTENER_RIGHT,
{ "mouthtightenerright", "liptightenerr" } },
{ XRFaceTracker::FT_MOUTH_TIGHTENER_LEFT,
{ "mouthtightenerleft", "liptightenerl" } },
{ XRFaceTracker::FT_TONGUE_OUT,
{ "tongueout", "tonguelongstep2" } },
{ XRFaceTracker::FT_TONGUE_UP,
{ "tongueup" } },
{ XRFaceTracker::FT_TONGUE_DOWN,
{ "tonguedown" } },
{ XRFaceTracker::FT_TONGUE_RIGHT,
{ "tongueright" } },
{ XRFaceTracker::FT_TONGUE_LEFT,
{ "tongueleft" } },
{ XRFaceTracker::FT_TONGUE_ROLL,
{ "tongueroll" } },
{ XRFaceTracker::FT_TONGUE_BLEND_DOWN,
{ "tongueblenddown" } },
{ XRFaceTracker::FT_TONGUE_CURL_UP,
{ "tonguecurlup" } },
{ XRFaceTracker::FT_TONGUE_SQUISH,
{ "tonguesquish" } },
{ XRFaceTracker::FT_TONGUE_FLAT,
{ "tongueflat" } },
{ XRFaceTracker::FT_TONGUE_TWIST_RIGHT,
{ "tonguetwistright" } },
{ XRFaceTracker::FT_TONGUE_TWIST_LEFT,
{ "tonguetwistleft" } },
{ XRFaceTracker::FT_SOFT_PALATE_CLOSE,
{ "softpalateclose" } },
{ XRFaceTracker::FT_THROAT_SWALLOW,
{ "throatswallow" } },
{ XRFaceTracker::FT_NECK_FLEX_RIGHT,
{ "neckflexright" } },
{ XRFaceTracker::FT_NECK_FLEX_LEFT,
{ "neckflexleft" } },
{ XRFaceTracker::FT_EYE_CLOSED,
{ "eyeclosed" } },
{ XRFaceTracker::FT_EYE_WIDE,
{ "eyewide" } },
{ XRFaceTracker::FT_EYE_SQUINT,
{ "eyesquint" } },
{ XRFaceTracker::FT_EYE_DILATION,
{ "eyedilation" } },
{ XRFaceTracker::FT_EYE_CONSTRICT,
{ "eyeconstrict" } },
{ XRFaceTracker::FT_BROW_DOWN_RIGHT,
{ "browdownright", "browlowererr" } },
{ XRFaceTracker::FT_BROW_DOWN_LEFT,
{ "browdownleft", "browlowererl" } },
{ XRFaceTracker::FT_BROW_DOWN,
{ "browdown" } },
{ XRFaceTracker::FT_BROW_UP_RIGHT,
{ "browupright" } },
{ XRFaceTracker::FT_BROW_UP_LEFT,
{ "browupleft" } },
{ XRFaceTracker::FT_BROW_UP,
{ "browup" } },
{ XRFaceTracker::FT_NOSE_SNEER,
{ "nosesneer" } },
{ XRFaceTracker::FT_NASAL_DILATION,
{ "nasaldilation" } },
{ XRFaceTracker::FT_NASAL_CONSTRICT,
{ "nasalconstrict" } },
{ XRFaceTracker::FT_CHEEK_PUFF,
{ "cheekpuff" } },
{ XRFaceTracker::FT_CHEEK_SUCK,
{ "cheeksuck" } },
{ XRFaceTracker::FT_CHEEK_SQUINT,
{ "cheeksquint" } },
{ XRFaceTracker::FT_LIP_SUCK_UPPER,
{ "lipsuckupper", "mouthrollupper", "mouthupperinside" } },
{ XRFaceTracker::FT_LIP_SUCK_LOWER,
{ "lipsucklower", "mouthrolllower", "mouthlowerinside" } },
{ XRFaceTracker::FT_LIP_SUCK,
{ "lipsuck" } },
{ XRFaceTracker::FT_LIP_FUNNEL_UPPER,
{ "lipfunnelupper", "mouthupperoverturn" } },
{ XRFaceTracker::FT_LIP_FUNNEL_LOWER,
{ "lipfunnellower", "mouthloweroverturn" } },
{ XRFaceTracker::FT_LIP_FUNNEL,
{ "lipfunnel", "mouthfunnel" } },
{ XRFaceTracker::FT_LIP_PUCKER_UPPER,
{ "lippuckerupper" } },
{ XRFaceTracker::FT_LIP_PUCKER_LOWER,
{ "lippuckerlower" } },
{ XRFaceTracker::FT_LIP_PUCKER,
{ "lippucker", "mouthpucker", "mouthpout" } },
{ XRFaceTracker::FT_MOUTH_UPPER_UP,
{ "mouthupperup" } },
{ XRFaceTracker::FT_MOUTH_LOWER_DOWN,
{ "mouthlowerdown" } },
{ XRFaceTracker::FT_MOUTH_OPEN,
{ "mouthopen" } },
{ XRFaceTracker::FT_MOUTH_RIGHT,
{ "mouthright" } },
{ XRFaceTracker::FT_MOUTH_LEFT,
{ "mouthleft" } },
{ XRFaceTracker::FT_MOUTH_SMILE_RIGHT,
{ "mouthsmileright", "lipcornerpullerr" } },
{ XRFaceTracker::FT_MOUTH_SMILE_LEFT,
{ "mouthsmileleft", "lipcornerpullerl" } },
{ XRFaceTracker::FT_MOUTH_SMILE,
{ "mouthsmile" } },
{ XRFaceTracker::FT_MOUTH_SAD_RIGHT,
{ "mouthsadright" } },
{ XRFaceTracker::FT_MOUTH_SAD_LEFT,
{ "mouthsadleft" } },
{ XRFaceTracker::FT_MOUTH_SAD,
{ "mouthsad" } },
{ XRFaceTracker::FT_MOUTH_STRETCH,
{ "mouthstretch" } },
{ XRFaceTracker::FT_MOUTH_DIMPLE,
{ "mouthdimple" } },
{ XRFaceTracker::FT_MOUTH_TIGHTENER,
{ "mouthtightener" } },
{ XRFaceTracker::FT_MOUTH_PRESS,
{ "mouthpress" } }
};
// Convert the name to lower-case and strip non-alphanumeric characters.
const String name = String(p_name).to_lower().replace("_", "");
// Iterate through the blend map.
for (const blend_map_entry &entry : blend_map) {
for (const char *n : entry.name) {
if (n == nullptr) {
break;
}
if (name == n) {
return entry.blend;
}
}
}
// Blend shape not found.
return -1;
}
// This method adds all the identified XRFaceTracker blend shapes of
// the mesh to the p_blend_mapping map. The map is indexed by the
// XRFaceTracker blend shape, and the value is the index of the mesh
// blend shape.
static void identify_face_blend_shapes(RBMap<int, int> &p_blend_mapping, const Ref<Mesh> &mesh) {
// Find all blend shapes.
const int count = mesh->get_blend_shape_count();
for (int i = 0; i < count; i++) {
const int blend = find_face_blend_shape(mesh->get_blend_shape_name(i));
if (blend >= 0) {
p_blend_mapping[blend] = i;
}
}
}
// This method removes any unified blend shapes from the p_blend_mapping map
// if all the individual blend shapes are found and going to be driven.
static void remove_driven_unified_blend_shapes(RBMap<int, int> &p_blend_mapping) {
// Entry for unified blend table.
struct unified_blend_entry {
int unified;
int individual[4];
};
// Table of unified blend shapes.
//
// This table consists of:
// - The XRFaceTracker unified blend shape.
// - The individual blend shapes that make up the unified blend shape.
static constexpr unified_blend_entry unified_blends[] = {
{ XRFaceTracker::FT_EYE_CLOSED,
{ XRFaceTracker::FT_EYE_CLOSED_RIGHT, XRFaceTracker::FT_EYE_CLOSED_LEFT, -1, -1 } },
{ XRFaceTracker::FT_EYE_WIDE,
{ XRFaceTracker::FT_EYE_WIDE_RIGHT, XRFaceTracker::FT_EYE_WIDE_LEFT, -1, -1 } },
{ XRFaceTracker::FT_EYE_SQUINT,
{ XRFaceTracker::FT_EYE_SQUINT_RIGHT, XRFaceTracker::FT_EYE_SQUINT_LEFT, -1, -1 } },
{ XRFaceTracker::FT_EYE_DILATION,
{ XRFaceTracker::FT_EYE_DILATION_RIGHT, XRFaceTracker::FT_EYE_DILATION_LEFT, -1, -1 } },
{ XRFaceTracker::FT_EYE_CONSTRICT,
{ XRFaceTracker::FT_EYE_CONSTRICT_RIGHT, XRFaceTracker::FT_EYE_CONSTRICT_LEFT, -1, -1 } },
{ XRFaceTracker::FT_BROW_DOWN_RIGHT,
{ XRFaceTracker::FT_BROW_LOWERER_RIGHT, XRFaceTracker::FT_BROW_PINCH_RIGHT, -1, -1 } },
{ XRFaceTracker::FT_BROW_DOWN_LEFT,
{ XRFaceTracker::FT_BROW_LOWERER_LEFT, XRFaceTracker::FT_BROW_PINCH_LEFT, -1, -1 } },
{ XRFaceTracker::FT_BROW_DOWN,
{ XRFaceTracker::FT_BROW_LOWERER_RIGHT, XRFaceTracker::FT_BROW_PINCH_RIGHT, XRFaceTracker::FT_BROW_LOWERER_LEFT, XRFaceTracker::FT_BROW_PINCH_LEFT } },
{ XRFaceTracker::FT_BROW_UP_RIGHT,
{ XRFaceTracker::FT_BROW_INNER_UP_RIGHT, XRFaceTracker::FT_BROW_OUTER_UP_RIGHT, -1, -1 } },
{ XRFaceTracker::FT_BROW_UP_LEFT,
{ XRFaceTracker::FT_BROW_INNER_UP_LEFT, XRFaceTracker::FT_BROW_OUTER_UP_LEFT, -1, -1 } },
{ XRFaceTracker::FT_BROW_UP,
{ XRFaceTracker::FT_BROW_INNER_UP_RIGHT, XRFaceTracker::FT_BROW_OUTER_UP_RIGHT, XRFaceTracker::FT_BROW_INNER_UP_LEFT, XRFaceTracker::FT_BROW_OUTER_UP_LEFT } },
{ XRFaceTracker::FT_NOSE_SNEER,
{ XRFaceTracker::FT_NOSE_SNEER_RIGHT, XRFaceTracker::FT_NOSE_SNEER_LEFT, -1, -1 } },
{ XRFaceTracker::FT_NASAL_DILATION,
{ XRFaceTracker::FT_NASAL_DILATION_RIGHT, XRFaceTracker::FT_NASAL_DILATION_LEFT, -1, -1 } },
{ XRFaceTracker::FT_NASAL_CONSTRICT,
{ XRFaceTracker::FT_NASAL_CONSTRICT_RIGHT, XRFaceTracker::FT_NASAL_CONSTRICT_LEFT, -1, -1 } },
{ XRFaceTracker::FT_CHEEK_PUFF,
{ XRFaceTracker::FT_CHEEK_PUFF_RIGHT, XRFaceTracker::FT_CHEEK_PUFF_LEFT, -1, -1 } },
{ XRFaceTracker::FT_CHEEK_SUCK,
{ XRFaceTracker::FT_CHEEK_SUCK_RIGHT, XRFaceTracker::FT_CHEEK_SUCK_LEFT, -1, -1 } },
{ XRFaceTracker::FT_CHEEK_SQUINT,
{ XRFaceTracker::FT_CHEEK_SQUINT_RIGHT, XRFaceTracker::FT_CHEEK_SQUINT_LEFT, -1, -1 } },
{ XRFaceTracker::FT_LIP_SUCK_UPPER,
{ XRFaceTracker::FT_LIP_SUCK_UPPER_RIGHT, XRFaceTracker::FT_LIP_SUCK_UPPER_LEFT, -1, -1 } },
{ XRFaceTracker::FT_LIP_SUCK_LOWER,
{ XRFaceTracker::FT_LIP_SUCK_LOWER_RIGHT, XRFaceTracker::FT_LIP_SUCK_LOWER_LEFT, -1, -1 } },
{ XRFaceTracker::FT_LIP_SUCK,
{ XRFaceTracker::FT_LIP_SUCK_UPPER_RIGHT, XRFaceTracker::FT_LIP_SUCK_UPPER_LEFT, XRFaceTracker::FT_LIP_SUCK_LOWER_RIGHT, XRFaceTracker::FT_LIP_SUCK_LOWER_LEFT } },
{ XRFaceTracker::FT_LIP_FUNNEL_UPPER,
{ XRFaceTracker::FT_LIP_FUNNEL_UPPER_RIGHT, XRFaceTracker::FT_LIP_FUNNEL_UPPER_LEFT, -1, -1 } },
{ XRFaceTracker::FT_LIP_FUNNEL_LOWER,
{ XRFaceTracker::FT_LIP_FUNNEL_LOWER_RIGHT, XRFaceTracker::FT_LIP_FUNNEL_LOWER_LEFT, -1, -1 } },
{ XRFaceTracker::FT_LIP_FUNNEL,
{ XRFaceTracker::FT_LIP_FUNNEL_UPPER_RIGHT, XRFaceTracker::FT_LIP_FUNNEL_UPPER_LEFT, XRFaceTracker::FT_LIP_FUNNEL_LOWER_RIGHT, XRFaceTracker::FT_LIP_FUNNEL_LOWER_LEFT } },
{ XRFaceTracker::FT_LIP_PUCKER_UPPER,
{ XRFaceTracker::FT_LIP_PUCKER_UPPER_RIGHT, XRFaceTracker::FT_LIP_PUCKER_UPPER_LEFT, -1, -1 } },
{ XRFaceTracker::FT_LIP_PUCKER_LOWER,
{ XRFaceTracker::FT_LIP_PUCKER_LOWER_RIGHT, XRFaceTracker::FT_LIP_PUCKER_LOWER_LEFT, -1, -1 } },
{ XRFaceTracker::FT_LIP_PUCKER,
{ XRFaceTracker::FT_LIP_PUCKER_UPPER_RIGHT, XRFaceTracker::FT_LIP_PUCKER_UPPER_LEFT, XRFaceTracker::FT_LIP_PUCKER_LOWER_RIGHT, XRFaceTracker::FT_LIP_PUCKER_LOWER_LEFT } },
{ XRFaceTracker::FT_MOUTH_UPPER_UP,
{ XRFaceTracker::FT_MOUTH_UPPER_UP_RIGHT, XRFaceTracker::FT_MOUTH_UPPER_UP_LEFT, -1, -1 } },
{ XRFaceTracker::FT_MOUTH_LOWER_DOWN,
{ XRFaceTracker::FT_MOUTH_LOWER_DOWN_RIGHT, XRFaceTracker::FT_MOUTH_LOWER_DOWN_LEFT, -1, -1 } },
{ XRFaceTracker::FT_MOUTH_OPEN,
{ XRFaceTracker::FT_MOUTH_UPPER_UP_RIGHT, XRFaceTracker::FT_MOUTH_UPPER_UP_LEFT, XRFaceTracker::FT_MOUTH_LOWER_DOWN_RIGHT, XRFaceTracker::FT_MOUTH_LOWER_DOWN_LEFT } },
{ XRFaceTracker::FT_MOUTH_RIGHT,
{ XRFaceTracker::FT_MOUTH_UPPER_RIGHT, XRFaceTracker::FT_MOUTH_LOWER_RIGHT, -1, -1 } },
{ XRFaceTracker::FT_MOUTH_LEFT,
{ XRFaceTracker::FT_MOUTH_UPPER_LEFT, XRFaceTracker::FT_MOUTH_LOWER_LEFT, -1, -1 } },
{ XRFaceTracker::FT_MOUTH_SMILE_RIGHT,
{ XRFaceTracker::FT_MOUTH_CORNER_PULL_RIGHT, XRFaceTracker::FT_MOUTH_CORNER_SLANT_RIGHT, -1, -1 } },
{ XRFaceTracker::FT_MOUTH_SMILE_LEFT,
{ XRFaceTracker::FT_MOUTH_CORNER_PULL_LEFT, XRFaceTracker::FT_MOUTH_CORNER_SLANT_LEFT, -1, -1 } },
{ XRFaceTracker::FT_MOUTH_SMILE,
{ XRFaceTracker::FT_MOUTH_CORNER_PULL_RIGHT, XRFaceTracker::FT_MOUTH_CORNER_SLANT_RIGHT, XRFaceTracker::FT_MOUTH_CORNER_PULL_LEFT, XRFaceTracker::FT_MOUTH_CORNER_SLANT_LEFT } },
{ XRFaceTracker::FT_MOUTH_SAD_RIGHT,
{ XRFaceTracker::FT_MOUTH_FROWN_RIGHT, XRFaceTracker::FT_MOUTH_STRETCH_RIGHT, -1, -1 } },
{ XRFaceTracker::FT_MOUTH_SAD_LEFT,
{ XRFaceTracker::FT_MOUTH_FROWN_LEFT, XRFaceTracker::FT_MOUTH_STRETCH_LEFT, -1, -1 } },
{ XRFaceTracker::FT_MOUTH_SAD,
{ XRFaceTracker::FT_MOUTH_FROWN_RIGHT, XRFaceTracker::FT_MOUTH_STRETCH_RIGHT, XRFaceTracker::FT_MOUTH_FROWN_LEFT, XRFaceTracker::FT_MOUTH_STRETCH_LEFT } },
{ XRFaceTracker::FT_MOUTH_STRETCH,
{ XRFaceTracker::FT_MOUTH_STRETCH_RIGHT, XRFaceTracker::FT_MOUTH_STRETCH_LEFT, -1, -1 } },
{ XRFaceTracker::FT_MOUTH_DIMPLE,
{ XRFaceTracker::FT_MOUTH_DIMPLE_RIGHT, XRFaceTracker::FT_MOUTH_DIMPLE_LEFT, -1, -1 } },
{ XRFaceTracker::FT_MOUTH_TIGHTENER,
{ XRFaceTracker::FT_MOUTH_TIGHTENER_RIGHT, XRFaceTracker::FT_MOUTH_TIGHTENER_LEFT, -1, -1 } },
{ XRFaceTracker::FT_MOUTH_PRESS,
{ XRFaceTracker::FT_MOUTH_PRESS_RIGHT, XRFaceTracker::FT_MOUTH_PRESS_LEFT, -1, -1 } }
};
// Remove unified blend shapes if individual blend shapes are found.
for (const unified_blend_entry &entry : unified_blends) {
// Check if all individual blend shapes are found.
bool found = true;
for (const int i : entry.individual) {
if (i >= 0 && !p_blend_mapping.find(i)) {
found = false;
break;
}
}
// If all individual blend shapes are found then remove the unified blend shape.
if (found) {
p_blend_mapping.erase(entry.unified);
}
}
}
void XRFaceModifier3D::_bind_methods() {
ClassDB::bind_method(D_METHOD("set_face_tracker", "tracker_name"), &XRFaceModifier3D::set_face_tracker);
ClassDB::bind_method(D_METHOD("get_face_tracker"), &XRFaceModifier3D::get_face_tracker);
ADD_PROPERTY(PropertyInfo(Variant::STRING, "face_tracker", PROPERTY_HINT_ENUM_SUGGESTION, "/user/head"), "set_face_tracker", "get_face_tracker");
ClassDB::bind_method(D_METHOD("set_target", "target"), &XRFaceModifier3D::set_target);
ClassDB::bind_method(D_METHOD("get_target"), &XRFaceModifier3D::get_target);
ADD_PROPERTY(PropertyInfo(Variant::NODE_PATH, "target", PROPERTY_HINT_NODE_PATH_VALID_TYPES, "MeshInstance3D"), "set_target", "get_target");
}
void XRFaceModifier3D::set_face_tracker(const StringName &p_tracker_name) {
tracker_name = p_tracker_name;
}
StringName XRFaceModifier3D::get_face_tracker() const {
return tracker_name;
}
void XRFaceModifier3D::set_target(const NodePath &p_target) {
target = p_target;
if (is_inside_tree()) {
_get_blend_data();
}
}
NodePath XRFaceModifier3D::get_target() const {
return target;
}
MeshInstance3D *XRFaceModifier3D::get_mesh_instance() const {
if (!has_node(target)) {
return nullptr;
}
Node *node = get_node(target);
if (!node) {
return nullptr;
}
return Object::cast_to<MeshInstance3D>(node);
}
void XRFaceModifier3D::_get_blend_data() {
// This method constructs the blend mapping from the XRFaceTracker
// blend shapes to the available blend shapes of the target mesh. It does this
// by:
//
// 1. Identifying the blend shapes of the target mesh and identifying what
// XRFaceTracker blend shape they correspond to. The results are
// placed in the blend_mapping map.
// 2. Prevent over-driving facial blend-shapes by removing any unified blend
// shapes from the map if all the individual blend shapes are already
// found and going to be driven.
blend_mapping.clear();
// Get the target MeshInstance3D.
const MeshInstance3D *mesh_instance = get_mesh_instance();
if (!mesh_instance) {
return;
}
// Get the mesh.
const Ref<Mesh> mesh = mesh_instance->get_mesh();
if (mesh.is_null()) {
return;
}
// Identify all face blend shapes and populate the map.
identify_face_blend_shapes(blend_mapping, mesh);
// Remove the unified blend shapes if all the individual blend shapes are found.
remove_driven_unified_blend_shapes(blend_mapping);
}
void XRFaceModifier3D::_update_face_blends() const {
// Get the XR Server.
const XRServer *xr_server = XRServer::get_singleton();
if (!xr_server) {
return;
}
// Get the face tracker.
const Ref<XRFaceTracker> p = xr_server->get_face_tracker(tracker_name);
if (!p.is_valid()) {
return;
}
// Get the face mesh.
MeshInstance3D *mesh_instance = get_mesh_instance();
if (!mesh_instance) {
return;
}
// Get the blend weights.
const PackedFloat32Array weights = p->get_blend_shapes();
// Apply all the face blend weights to the mesh.
for (const KeyValue<int, int> &it : blend_mapping) {
mesh_instance->set_blend_shape_value(it.value, weights[it.key]);
}
}
void XRFaceModifier3D::_notification(int p_what) {
switch (p_what) {
case NOTIFICATION_ENTER_TREE: {
_get_blend_data();
set_process_internal(true);
} break;
case NOTIFICATION_EXIT_TREE: {
set_process_internal(false);
blend_mapping.clear();
} break;
case NOTIFICATION_INTERNAL_PROCESS: {
_update_face_blends();
} break;
default: {
} break;
}
}

73
scene/3d/xr_face_modifier_3d.h Normal file
View file

@ -0,0 +1,73 @@
/**************************************************************************/
/* xr_face_modifier_3d.h */
/**************************************************************************/
/* This file is part of: */
/* GODOT ENGINE */
/* https://godotengine.org */
/**************************************************************************/
/* Copyright (c) 2014-present Godot Engine contributors (see AUTHORS.md). */
/* Copyright (c) 2007-2014 Juan Linietsky, Ariel Manzur. */
/* */
/* Permission is hereby granted, free of charge, to any person obtaining */
/* a copy of this software and associated documentation files (the */
/* "Software"), to deal in the Software without restriction, including */
/* without limitation the rights to use, copy, modify, merge, publish, */
/* distribute, sublicense, and/or sell copies of the Software, and to */
/* permit persons to whom the Software is furnished to do so, subject to */
/* the following conditions: */
/* */
/* The above copyright notice and this permission notice shall be */
/* included in all copies or substantial portions of the Software. */
/* */
/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, */
/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF */
/* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. */
/* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY */
/* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, */
/* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE */
/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */
/**************************************************************************/
#ifndef XR_FACE_MODIFIER_3D_H
#define XR_FACE_MODIFIER_3D_H
#include "mesh_instance_3d.h"
#include "scene/3d/node_3d.h"
/**
The XRFaceModifier3D node drives the blend shapes of a MeshInstance3D
with facial expressions from an XRFaceTracking instance.
The blend shapes provided by the mesh are interrogated, and used to
deduce an optimal mapping from the Unified Expressions blend shapes
provided by the XRFaceTracking instance to drive the face.
*/
class XRFaceModifier3D : public Node3D {
GDCLASS(XRFaceModifier3D, Node3D);
private:
StringName tracker_name = "/user/head";
NodePath target;
// Map from XRFaceTracker blend shape index to mesh blend shape index.
RBMap<int, int> blend_mapping;
MeshInstance3D *get_mesh_instance() const;
void _get_blend_data();
void _update_face_blends() const;
protected:
static void _bind_methods();
public:
void set_face_tracker(const StringName &p_tracker_name);
StringName get_face_tracker() const;
void set_target(const NodePath &p_target);
NodePath get_target() const;
void _notification(int p_what);
};
#endif // XR_FACE_MODIFIER_3D_H

2
scene/register_scene_types.cpp
View file

@ -272,6 +272,7 @@
#include "scene/3d/visible_on_screen_notifier_3d.h"
#include "scene/3d/voxel_gi.h"
#include "scene/3d/world_environment.h"
#include "scene/3d/xr_face_modifier_3d.h"
#include "scene/3d/xr_nodes.h"
#include "scene/animation/root_motion_view.h"
#include "scene/resources/environment.h"
@ -519,6 +520,7 @@ void register_scene_types() {
GDREGISTER_CLASS(XRController3D);
GDREGISTER_CLASS(XRAnchor3D);
GDREGISTER_CLASS(XROrigin3D);
GDREGISTER_CLASS(XRFaceModifier3D);
GDREGISTER_CLASS(MeshInstance3D);
GDREGISTER_CLASS(OccluderInstance3D);
GDREGISTER_ABSTRACT_CLASS(Occluder3D);

2
servers/register_server_types.cpp
View file

@ -80,6 +80,7 @@
#include "text/text_server_dummy.h"
#include "text/text_server_extension.h"
#include "text_server.h"
#include "xr/xr_face_tracker.h"
#include "xr/xr_interface.h"
#include "xr/xr_interface_extension.h"
#include "xr/xr_positional_tracker.h"
@ -189,6 +190,7 @@ void register_server_types() {
GDREGISTER_CLASS(XRInterfaceExtension); // can't register this as virtual because we need a creation function for our extensions.
GDREGISTER_CLASS(XRPose);
GDREGISTER_CLASS(XRPositionalTracker);
GDREGISTER_CLASS(XRFaceTracker);
GDREGISTER_CLASS(AudioStream);
GDREGISTER_CLASS(AudioStreamPlayback);

222
servers/xr/xr_face_tracker.cpp Normal file
View file

@ -0,0 +1,222 @@
/**************************************************************************/
/* xr_face_tracker.cpp */
/**************************************************************************/
/* This file is part of: */
/* GODOT ENGINE */
/* https://godotengine.org */
/**************************************************************************/
/* Copyright (c) 2014-present Godot Engine contributors (see AUTHORS.md). */
/* Copyright (c) 2007-2014 Juan Linietsky, Ariel Manzur. */
/* */
/* Permission is hereby granted, free of charge, to any person obtaining */
/* a copy of this software and associated documentation files (the */
/* "Software"), to deal in the Software without restriction, including */
/* without limitation the rights to use, copy, modify, merge, publish, */
/* distribute, sublicense, and/or sell copies of the Software, and to */
/* permit persons to whom the Software is furnished to do so, subject to */
/* the following conditions: */
/* */
/* The above copyright notice and this permission notice shall be */
/* included in all copies or substantial portions of the Software. */
/* */
/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, */
/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF */
/* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. */
/* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY */
/* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, */
/* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE */
/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */
/**************************************************************************/
#include "xr_face_tracker.h"
void XRFaceTracker::_bind_methods() {
// Base Shapes
BIND_ENUM_CONSTANT(FT_EYE_LOOK_OUT_RIGHT);
BIND_ENUM_CONSTANT(FT_EYE_LOOK_IN_RIGHT);
BIND_ENUM_CONSTANT(FT_EYE_LOOK_UP_RIGHT);
BIND_ENUM_CONSTANT(FT_EYE_LOOK_DOWN_RIGHT);
BIND_ENUM_CONSTANT(FT_EYE_LOOK_OUT_LEFT);
BIND_ENUM_CONSTANT(FT_EYE_LOOK_IN_LEFT);
BIND_ENUM_CONSTANT(FT_EYE_LOOK_UP_LEFT);
BIND_ENUM_CONSTANT(FT_EYE_LOOK_DOWN_LEFT);
BIND_ENUM_CONSTANT(FT_EYE_CLOSED_RIGHT);
BIND_ENUM_CONSTANT(FT_EYE_CLOSED_LEFT);
BIND_ENUM_CONSTANT(FT_EYE_SQUINT_RIGHT);
BIND_ENUM_CONSTANT(FT_EYE_SQUINT_LEFT);
BIND_ENUM_CONSTANT(FT_EYE_WIDE_RIGHT);
BIND_ENUM_CONSTANT(FT_EYE_WIDE_LEFT);
BIND_ENUM_CONSTANT(FT_EYE_DILATION_RIGHT);
BIND_ENUM_CONSTANT(FT_EYE_DILATION_LEFT);
BIND_ENUM_CONSTANT(FT_EYE_CONSTRICT_RIGHT);
BIND_ENUM_CONSTANT(FT_EYE_CONSTRICT_LEFT);
BIND_ENUM_CONSTANT(FT_BROW_PINCH_RIGHT);
BIND_ENUM_CONSTANT(FT_BROW_PINCH_LEFT);
BIND_ENUM_CONSTANT(FT_BROW_LOWERER_RIGHT);
BIND_ENUM_CONSTANT(FT_BROW_LOWERER_LEFT);
BIND_ENUM_CONSTANT(FT_BROW_INNER_UP_RIGHT);
BIND_ENUM_CONSTANT(FT_BROW_INNER_UP_LEFT);
BIND_ENUM_CONSTANT(FT_BROW_OUTER_UP_RIGHT);
BIND_ENUM_CONSTANT(FT_BROW_OUTER_UP_LEFT);
BIND_ENUM_CONSTANT(FT_NOSE_SNEER_RIGHT);
BIND_ENUM_CONSTANT(FT_NOSE_SNEER_LEFT);
BIND_ENUM_CONSTANT(FT_NASAL_DILATION_RIGHT);
BIND_ENUM_CONSTANT(FT_NASAL_DILATION_LEFT);
BIND_ENUM_CONSTANT(FT_NASAL_CONSTRICT_RIGHT);
BIND_ENUM_CONSTANT(FT_NASAL_CONSTRICT_LEFT);
BIND_ENUM_CONSTANT(FT_CHEEK_SQUINT_RIGHT);
BIND_ENUM_CONSTANT(FT_CHEEK_SQUINT_LEFT);
BIND_ENUM_CONSTANT(FT_CHEEK_PUFF_RIGHT);
BIND_ENUM_CONSTANT(FT_CHEEK_PUFF_LEFT);
BIND_ENUM_CONSTANT(FT_CHEEK_SUCK_RIGHT);
BIND_ENUM_CONSTANT(FT_CHEEK_SUCK_LEFT);
BIND_ENUM_CONSTANT(FT_JAW_OPEN);
BIND_ENUM_CONSTANT(FT_MOUTH_CLOSED);
BIND_ENUM_CONSTANT(FT_JAW_RIGHT);
BIND_ENUM_CONSTANT(FT_JAW_LEFT);
BIND_ENUM_CONSTANT(FT_JAW_FORWARD);
BIND_ENUM_CONSTANT(FT_JAW_BACKWARD);
BIND_ENUM_CONSTANT(FT_JAW_CLENCH);
BIND_ENUM_CONSTANT(FT_JAW_MANDIBLE_RAISE);
BIND_ENUM_CONSTANT(FT_LIP_SUCK_UPPER_RIGHT);
BIND_ENUM_CONSTANT(FT_LIP_SUCK_UPPER_LEFT);
BIND_ENUM_CONSTANT(FT_LIP_SUCK_LOWER_RIGHT);
BIND_ENUM_CONSTANT(FT_LIP_SUCK_LOWER_LEFT);
BIND_ENUM_CONSTANT(FT_LIP_SUCK_CORNER_RIGHT);
BIND_ENUM_CONSTANT(FT_LIP_SUCK_CORNER_LEFT);
BIND_ENUM_CONSTANT(FT_LIP_FUNNEL_UPPER_RIGHT);
BIND_ENUM_CONSTANT(FT_LIP_FUNNEL_UPPER_LEFT);
BIND_ENUM_CONSTANT(FT_LIP_FUNNEL_LOWER_RIGHT);
BIND_ENUM_CONSTANT(FT_LIP_FUNNEL_LOWER_LEFT);
BIND_ENUM_CONSTANT(FT_LIP_PUCKER_UPPER_RIGHT);
BIND_ENUM_CONSTANT(FT_LIP_PUCKER_UPPER_LEFT);
BIND_ENUM_CONSTANT(FT_LIP_PUCKER_LOWER_RIGHT);
BIND_ENUM_CONSTANT(FT_LIP_PUCKER_LOWER_LEFT);
BIND_ENUM_CONSTANT(FT_MOUTH_UPPER_UP_RIGHT);
BIND_ENUM_CONSTANT(FT_MOUTH_UPPER_UP_LEFT);
BIND_ENUM_CONSTANT(FT_MOUTH_LOWER_DOWN_RIGHT);
BIND_ENUM_CONSTANT(FT_MOUTH_LOWER_DOWN_LEFT);
BIND_ENUM_CONSTANT(FT_MOUTH_UPPER_DEEPEN_RIGHT);
BIND_ENUM_CONSTANT(FT_MOUTH_UPPER_DEEPEN_LEFT);
BIND_ENUM_CONSTANT(FT_MOUTH_UPPER_RIGHT);
BIND_ENUM_CONSTANT(FT_MOUTH_UPPER_LEFT);
BIND_ENUM_CONSTANT(FT_MOUTH_LOWER_RIGHT);
BIND_ENUM_CONSTANT(FT_MOUTH_LOWER_LEFT);
BIND_ENUM_CONSTANT(FT_MOUTH_CORNER_PULL_RIGHT);
BIND_ENUM_CONSTANT(FT_MOUTH_CORNER_PULL_LEFT);
BIND_ENUM_CONSTANT(FT_MOUTH_CORNER_SLANT_RIGHT);
BIND_ENUM_CONSTANT(FT_MOUTH_CORNER_SLANT_LEFT);
BIND_ENUM_CONSTANT(FT_MOUTH_FROWN_RIGHT);
BIND_ENUM_CONSTANT(FT_MOUTH_FROWN_LEFT);
BIND_ENUM_CONSTANT(FT_MOUTH_STRETCH_RIGHT);
BIND_ENUM_CONSTANT(FT_MOUTH_STRETCH_LEFT);
BIND_ENUM_CONSTANT(FT_MOUTH_DIMPLE_RIGHT);
BIND_ENUM_CONSTANT(FT_MOUTH_DIMPLE_LEFT);
BIND_ENUM_CONSTANT(FT_MOUTH_RAISER_UPPER);
BIND_ENUM_CONSTANT(FT_MOUTH_RAISER_LOWER);
BIND_ENUM_CONSTANT(FT_MOUTH_PRESS_RIGHT);
BIND_ENUM_CONSTANT(FT_MOUTH_PRESS_LEFT);
BIND_ENUM_CONSTANT(FT_MOUTH_TIGHTENER_RIGHT);
BIND_ENUM_CONSTANT(FT_MOUTH_TIGHTENER_LEFT);
BIND_ENUM_CONSTANT(FT_TONGUE_OUT);
BIND_ENUM_CONSTANT(FT_TONGUE_UP);
BIND_ENUM_CONSTANT(FT_TONGUE_DOWN);
BIND_ENUM_CONSTANT(FT_TONGUE_RIGHT);
BIND_ENUM_CONSTANT(FT_TONGUE_LEFT);
BIND_ENUM_CONSTANT(FT_TONGUE_ROLL);
BIND_ENUM_CONSTANT(FT_TONGUE_BLEND_DOWN);
BIND_ENUM_CONSTANT(FT_TONGUE_CURL_UP);
BIND_ENUM_CONSTANT(FT_TONGUE_SQUISH);
BIND_ENUM_CONSTANT(FT_TONGUE_FLAT);
BIND_ENUM_CONSTANT(FT_TONGUE_TWIST_RIGHT);
BIND_ENUM_CONSTANT(FT_TONGUE_TWIST_LEFT);
BIND_ENUM_CONSTANT(FT_SOFT_PALATE_CLOSE);
BIND_ENUM_CONSTANT(FT_THROAT_SWALLOW);
BIND_ENUM_CONSTANT(FT_NECK_FLEX_RIGHT);
BIND_ENUM_CONSTANT(FT_NECK_FLEX_LEFT);
// Blended Shapes
BIND_ENUM_CONSTANT(FT_EYE_CLOSED);
BIND_ENUM_CONSTANT(FT_EYE_WIDE);
BIND_ENUM_CONSTANT(FT_EYE_SQUINT);
BIND_ENUM_CONSTANT(FT_EYE_DILATION);
BIND_ENUM_CONSTANT(FT_EYE_CONSTRICT);
BIND_ENUM_CONSTANT(FT_BROW_DOWN_RIGHT);
BIND_ENUM_CONSTANT(FT_BROW_DOWN_LEFT);
BIND_ENUM_CONSTANT(FT_BROW_DOWN);
BIND_ENUM_CONSTANT(FT_BROW_UP_RIGHT);
BIND_ENUM_CONSTANT(FT_BROW_UP_LEFT);
BIND_ENUM_CONSTANT(FT_BROW_UP);
BIND_ENUM_CONSTANT(FT_NOSE_SNEER);
BIND_ENUM_CONSTANT(FT_NASAL_DILATION);
BIND_ENUM_CONSTANT(FT_NASAL_CONSTRICT);
BIND_ENUM_CONSTANT(FT_CHEEK_PUFF);
BIND_ENUM_CONSTANT(FT_CHEEK_SUCK);
BIND_ENUM_CONSTANT(FT_CHEEK_SQUINT);
BIND_ENUM_CONSTANT(FT_LIP_SUCK_UPPER);
BIND_ENUM_CONSTANT(FT_LIP_SUCK_LOWER);
BIND_ENUM_CONSTANT(FT_LIP_SUCK);
BIND_ENUM_CONSTANT(FT_LIP_FUNNEL_UPPER);
BIND_ENUM_CONSTANT(FT_LIP_FUNNEL_LOWER);
BIND_ENUM_CONSTANT(FT_LIP_FUNNEL);
BIND_ENUM_CONSTANT(FT_LIP_PUCKER_UPPER);
BIND_ENUM_CONSTANT(FT_LIP_PUCKER_LOWER);
BIND_ENUM_CONSTANT(FT_LIP_PUCKER);
BIND_ENUM_CONSTANT(FT_MOUTH_UPPER_UP);
BIND_ENUM_CONSTANT(FT_MOUTH_LOWER_DOWN);
BIND_ENUM_CONSTANT(FT_MOUTH_OPEN);
BIND_ENUM_CONSTANT(FT_MOUTH_RIGHT);
BIND_ENUM_CONSTANT(FT_MOUTH_LEFT);
BIND_ENUM_CONSTANT(FT_MOUTH_SMILE_RIGHT);
BIND_ENUM_CONSTANT(FT_MOUTH_SMILE_LEFT);
BIND_ENUM_CONSTANT(FT_MOUTH_SMILE);
BIND_ENUM_CONSTANT(FT_MOUTH_SAD_RIGHT);
BIND_ENUM_CONSTANT(FT_MOUTH_SAD_LEFT);
BIND_ENUM_CONSTANT(FT_MOUTH_SAD);
BIND_ENUM_CONSTANT(FT_MOUTH_STRETCH);
BIND_ENUM_CONSTANT(FT_MOUTH_DIMPLE);
BIND_ENUM_CONSTANT(FT_MOUTH_TIGHTENER);
BIND_ENUM_CONSTANT(FT_MOUTH_PRESS);
BIND_ENUM_CONSTANT(FT_MAX);
ClassDB::bind_method(D_METHOD("get_blend_shape", "blend_shape"), &XRFaceTracker::get_blend_shape);
ClassDB::bind_method(D_METHOD("set_blend_shape", "blend_shape", "weight"), &XRFaceTracker::set_blend_shape);
ClassDB::bind_method(D_METHOD("get_blend_shapes"), &XRFaceTracker::get_blend_shapes);
ClassDB::bind_method(D_METHOD("set_blend_shapes", "weights"), &XRFaceTracker::set_blend_shapes);
ADD_PROPERTY(PropertyInfo(Variant::PACKED_FLOAT32_ARRAY, "blend_shapes"), "set_blend_shapes", "get_blend_shapes");
ADD_PROPERTY_DEFAULT("blend_shapes", PackedFloat32Array()); // To prevent ludicrously large default values.
}
float XRFaceTracker::get_blend_shape(BlendShapeEntry p_blend_shape) const {
// Fail if the blend shape index is out of range.
ERR_FAIL_INDEX_V(p_blend_shape, FT_MAX, 0.0f);
// Return the blend shape value.
return blend_shape_values[p_blend_shape];
}
void XRFaceTracker::set_blend_shape(BlendShapeEntry p_blend_shape, float p_value) {
// Fail if the blend shape index is out of range.
ERR_FAIL_INDEX(p_blend_shape, FT_MAX);
// Save the new blend shape value.
blend_shape_values[p_blend_shape] = p_value;
}
PackedFloat32Array XRFaceTracker::get_blend_shapes() const {
// Create a packed float32 array and copy the blend shape values into it.
PackedFloat32Array data;
data.resize(FT_MAX);
memcpy(data.ptrw(), blend_shape_values, sizeof(blend_shape_values));
// Return the blend shape array.
return data;
}
void XRFaceTracker::set_blend_shapes(const PackedFloat32Array &p_blend_shapes) {
// Fail if the blend shape array is not the correct size.
ERR_FAIL_COND(p_blend_shapes.size() != FT_MAX);
// Copy the blend shape values into the blend shape array.
memcpy(blend_shape_values, p_blend_shapes.ptr(), sizeof(blend_shape_values));
}

213
servers/xr/xr_face_tracker.h Normal file
View file

@ -0,0 +1,213 @@
/**************************************************************************/
/* xr_face_tracker.h */
/**************************************************************************/
/* This file is part of: */
/* GODOT ENGINE */
/* https://godotengine.org */
/**************************************************************************/
/* Copyright (c) 2014-present Godot Engine contributors (see AUTHORS.md). */
/* Copyright (c) 2007-2014 Juan Linietsky, Ariel Manzur. */
/* */
/* Permission is hereby granted, free of charge, to any person obtaining */
/* a copy of this software and associated documentation files (the */
/* "Software"), to deal in the Software without restriction, including */
/* without limitation the rights to use, copy, modify, merge, publish, */
/* distribute, sublicense, and/or sell copies of the Software, and to */
/* permit persons to whom the Software is furnished to do so, subject to */
/* the following conditions: */
/* */
/* The above copyright notice and this permission notice shall be */
/* included in all copies or substantial portions of the Software. */
/* */
/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, */
/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF */
/* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. */
/* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY */
/* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, */
/* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE */
/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */
/**************************************************************************/
#ifndef XR_FACE_TRACKER_H
#define XR_FACE_TRACKER_H
#include "core/object/ref_counted.h"
/**
The XRFaceTracker class provides face blend shape weights.
The supported blend shapes are based on the Unified Expressions
standard, and as such have a well defined mapping to ARKit, SRanipal,
and Meta Movement standards.
*/
class XRFaceTracker : public RefCounted {
GDCLASS(XRFaceTracker, RefCounted);
_THREAD_SAFE_CLASS_
public:
enum BlendShapeEntry {
// Base Shapes
FT_EYE_LOOK_OUT_RIGHT, // Right eye looks outwards.
FT_EYE_LOOK_IN_RIGHT, // Right eye looks inwards.
FT_EYE_LOOK_UP_RIGHT, // Right eye looks upwards.
FT_EYE_LOOK_DOWN_RIGHT, // Right eye looks downwards.
FT_EYE_LOOK_OUT_LEFT, // Left eye looks outwards.
FT_EYE_LOOK_IN_LEFT, // Left eye looks inwards.
FT_EYE_LOOK_UP_LEFT, // Left eye looks upwards.
FT_EYE_LOOK_DOWN_LEFT, // Left eye looks downwards.
FT_EYE_CLOSED_RIGHT, // Closes the right eyelid.
FT_EYE_CLOSED_LEFT, // Closes the left eyelid.
FT_EYE_SQUINT_RIGHT, // Squeezes the right eye socket muscles.
FT_EYE_SQUINT_LEFT, // Squeezes the left eye socket muscles.
FT_EYE_WIDE_RIGHT, // Right eyelid widens beyond relaxed.
FT_EYE_WIDE_LEFT, // Left eyelid widens beyond relaxed.
FT_EYE_DILATION_RIGHT, // Dilates the right eye pupil.
FT_EYE_DILATION_LEFT, // Dilates the left eye pupil.
FT_EYE_CONSTRICT_RIGHT, // Constricts the right eye pupil.
FT_EYE_CONSTRICT_LEFT, // Constricts the left eye pupil.
FT_BROW_PINCH_RIGHT, // Right eyebrow pinches in.
FT_BROW_PINCH_LEFT, // Left eyebrow pinches in.
FT_BROW_LOWERER_RIGHT, // Outer right eyebrow pulls down.
FT_BROW_LOWERER_LEFT, // Outer left eyebrow pulls down.
FT_BROW_INNER_UP_RIGHT, // Inner right eyebrow pulls up.
FT_BROW_INNER_UP_LEFT, // Inner left eyebrow pulls up.
FT_BROW_OUTER_UP_RIGHT, // Outer right eyebrow pulls up.
FT_BROW_OUTER_UP_LEFT, // Outer left eyebrow pulls up.
FT_NOSE_SNEER_RIGHT, // Right side face sneers.
FT_NOSE_SNEER_LEFT, // Left side face sneers.
FT_NASAL_DILATION_RIGHT, // Right side nose canal dilates.
FT_NASAL_DILATION_LEFT, // Left side nose canal dilates.
FT_NASAL_CONSTRICT_RIGHT, // Right side nose canal constricts.
FT_NASAL_CONSTRICT_LEFT, // Left side nose canal constricts.
FT_CHEEK_SQUINT_RIGHT, // Raises the right side cheek.
FT_CHEEK_SQUINT_LEFT, // Raises the left side cheek.
FT_CHEEK_PUFF_RIGHT, // Puffs the right side cheek.
FT_CHEEK_PUFF_LEFT, // Puffs the left side cheek.
FT_CHEEK_SUCK_RIGHT, // Sucks in the right side cheek.
FT_CHEEK_SUCK_LEFT, // Sucks in the left side cheek.
FT_JAW_OPEN, // Opens jawbone.
FT_MOUTH_CLOSED, // Closes the mouth.
FT_JAW_RIGHT, // Pushes jawbone right.
FT_JAW_LEFT, // Pushes jawbone left.
FT_JAW_FORWARD, // Pushes jawbone forward.
FT_JAW_BACKWARD, // Pushes jawbone backward.
FT_JAW_CLENCH, // Flexes jaw muscles.
FT_JAW_MANDIBLE_RAISE, // Raises the jawbone.
FT_LIP_SUCK_UPPER_RIGHT, // Upper right lip part tucks in the mouth.
FT_LIP_SUCK_UPPER_LEFT, // Upper left lip part tucks in the mouth.
FT_LIP_SUCK_LOWER_RIGHT, // Lower right lip part tucks in the mouth.
FT_LIP_SUCK_LOWER_LEFT, // Lower left lip part tucks in the mouth.
FT_LIP_SUCK_CORNER_RIGHT, // Right lip corner folds into the mouth.
FT_LIP_SUCK_CORNER_LEFT, // Left lip corner folds into the mouth.
FT_LIP_FUNNEL_UPPER_RIGHT, // Upper right lip part pushes into a funnel.
FT_LIP_FUNNEL_UPPER_LEFT, // Upper left lip part pushes into a funnel.
FT_LIP_FUNNEL_LOWER_RIGHT, // Lower right lip part pushes into a funnel.
FT_LIP_FUNNEL_LOWER_LEFT, // Lower left lip part pushes into a funnel.
FT_LIP_PUCKER_UPPER_RIGHT, // Upper right lip part pushes outwards.
FT_LIP_PUCKER_UPPER_LEFT, // Upper left lip part pushes outwards.
FT_LIP_PUCKER_LOWER_RIGHT, // Lower right lip part pushes outwards.
FT_LIP_PUCKER_LOWER_LEFT, // Lower left lip part pushes outwards.
FT_MOUTH_UPPER_UP_RIGHT, // Upper right part of the lip pulls up.
FT_MOUTH_UPPER_UP_LEFT, // Upper left part of the lip pulls up.
FT_MOUTH_LOWER_DOWN_RIGHT, // Lower right part of the lip pulls up.
FT_MOUTH_LOWER_DOWN_LEFT, // Lower left part of the lip pulls up.
FT_MOUTH_UPPER_DEEPEN_RIGHT, // Upper right lip part pushes in the cheek.
FT_MOUTH_UPPER_DEEPEN_LEFT, // Upper left lip part pushes in the cheek.
FT_MOUTH_UPPER_RIGHT, // Moves upper lip right.
FT_MOUTH_UPPER_LEFT, // Moves upper lip left.
FT_MOUTH_LOWER_RIGHT, // Moves lower lip right.
FT_MOUTH_LOWER_LEFT, // Moves lower lip left.
FT_MOUTH_CORNER_PULL_RIGHT, // Right lip corner pulls diagonally up and out.
FT_MOUTH_CORNER_PULL_LEFT, // Left lip corner pulls diagonally up and out.
FT_MOUTH_CORNER_SLANT_RIGHT, // Right corner lip slants up.
FT_MOUTH_CORNER_SLANT_LEFT, // Left corner lip slants up.
FT_MOUTH_FROWN_RIGHT, // Right corner lip pulls down.
FT_MOUTH_FROWN_LEFT, // Left corner lip pulls down.
FT_MOUTH_STRETCH_RIGHT, // Mouth corner lip pulls out and down.
FT_MOUTH_STRETCH_LEFT, // Mouth corner lip pulls out and down.
FT_MOUTH_DIMPLE_RIGHT, // Right lip corner is pushed backwards.
FT_MOUTH_DIMPLE_LEFT, // Left lip corner is pushed backwards.
FT_MOUTH_RAISER_UPPER, // Raises and slightly pushes out the upper mouth.
FT_MOUTH_RAISER_LOWER, // Raises and slightly pushes out the lower mouth.
FT_MOUTH_PRESS_RIGHT, // Right side lips press and flatten together vertically.
FT_MOUTH_PRESS_LEFT, // Left side lips press and flatten together vertically.
FT_MOUTH_TIGHTENER_RIGHT, // Right side lips squeeze together horizontally.
FT_MOUTH_TIGHTENER_LEFT, // Left side lips squeeze together horizontally.
FT_TONGUE_OUT, // Tongue visibly sticks out of the mouth.
FT_TONGUE_UP, // Tongue points upwards.
FT_TONGUE_DOWN, // Tongue points downwards.
FT_TONGUE_RIGHT, // Tongue points right.
FT_TONGUE_LEFT, // Tongue points left.
FT_TONGUE_ROLL, // Sides of the tongue funnel, creating a roll.
FT_TONGUE_BLEND_DOWN, // Tongue arches up then down inside the mouth.
FT_TONGUE_CURL_UP, // Tongue arches down then up inside the mouth.
FT_TONGUE_SQUISH, // Tongue squishes together and thickens.
FT_TONGUE_FLAT, // Tongue flattens and thins out.
FT_TONGUE_TWIST_RIGHT, // Tongue tip rotates clockwise, with the rest following gradually.
FT_TONGUE_TWIST_LEFT, // Tongue tip rotates counter-clockwise, with the rest following gradually.
FT_SOFT_PALATE_CLOSE, // Inner mouth throat closes.
FT_THROAT_SWALLOW, // The Adam's apple visibly swallows.
FT_NECK_FLEX_RIGHT, // Right side neck visibly flexes.
FT_NECK_FLEX_LEFT, // Left side neck visibly flexes.
// Blended Shapes
FT_EYE_CLOSED, // Closes both eye lids.
FT_EYE_WIDE, // Widens both eye lids.
FT_EYE_SQUINT, // Squints both eye lids.
FT_EYE_DILATION, // Dilates both pupils.
FT_EYE_CONSTRICT, // Constricts both pupils.
FT_BROW_DOWN_RIGHT, // Pulls the right eyebrow down and in.
FT_BROW_DOWN_LEFT, // Pulls the left eyebrow down and in.
FT_BROW_DOWN, // Pulls both eyebrows down and in.
FT_BROW_UP_RIGHT, // Right brow appears worried.
FT_BROW_UP_LEFT, // Left brow appears worried.
FT_BROW_UP, // Both brows appear worried.
FT_NOSE_SNEER, // Entire face sneers.
FT_NASAL_DILATION, // Both nose canals dilate.
FT_NASAL_CONSTRICT, // Both nose canals constrict.
FT_CHEEK_PUFF, // Puffs both cheeks.
FT_CHEEK_SUCK, // Sucks in both cheeks.
FT_CHEEK_SQUINT, // Raises both cheeks.
FT_LIP_SUCK_UPPER, // Tucks in the upper lips.
FT_LIP_SUCK_LOWER, // Tucks in the lower lips.
FT_LIP_SUCK, // Tucks in both lips.
FT_LIP_FUNNEL_UPPER, // Funnels in the upper lips.
FT_LIP_FUNNEL_LOWER, // Funnels in the lower lips.
FT_LIP_FUNNEL, // Funnels in both lips.
FT_LIP_PUCKER_UPPER, // Upper lip part pushes outwards.
FT_LIP_PUCKER_LOWER, // Lower lip part pushes outwards.
FT_LIP_PUCKER, // Lips push outwards.
FT_MOUTH_UPPER_UP, // Raises the upper lips.
FT_MOUTH_LOWER_DOWN, // Lowers the lower lips.
FT_MOUTH_OPEN, // Mouth opens, revealing teeth.
FT_MOUTH_RIGHT, // Moves mouth right.
FT_MOUTH_LEFT, // Moves mouth left.
FT_MOUTH_SMILE_RIGHT, // Right side of the mouth smiles.
FT_MOUTH_SMILE_LEFT, // Left side of the mouth smiles.
FT_MOUTH_SMILE, // Mouth expresses a smile.
FT_MOUTH_SAD_RIGHT, // Right side of the mouth expresses sadness.
FT_MOUTH_SAD_LEFT, // Left side of the mouth expresses sadness.
FT_MOUTH_SAD, // Mouth expresses sadness.
FT_MOUTH_STRETCH, // Mouth stretches.
FT_MOUTH_DIMPLE, // Lip corners dimple.
FT_MOUTH_TIGHTENER, // Mouth tightens.
FT_MOUTH_PRESS, // Mouth presses together.
FT_MAX // Maximum blend shape.
};
float get_blend_shape(BlendShapeEntry p_blend_shape) const;
void set_blend_shape(BlendShapeEntry p_blend_shape, float p_value);
PackedFloat32Array get_blend_shapes() const;
void set_blend_shapes(const PackedFloat32Array &p_blend_shapes);
protected:
static void _bind_methods();
private:
float blend_shape_values[FT_MAX] = {};
};
VARIANT_ENUM_CAST(XRFaceTracker::BlendShapeEntry);
#endif // XR_FACE_TRACKER_H

48
servers/xr_server.cpp
View file

@ -30,6 +30,7 @@
#include "xr_server.h"
#include "core/config/project_settings.h"
#include "xr/xr_face_tracker.h"
#include "xr/xr_interface.h"
#include "xr/xr_positional_tracker.h"
@ -74,6 +75,11 @@ void XRServer::_bind_methods() {
ClassDB::bind_method(D_METHOD("get_trackers", "tracker_types"), &XRServer::get_trackers);
ClassDB::bind_method(D_METHOD("get_tracker", "tracker_name"), &XRServer::get_tracker);
ClassDB::bind_method(D_METHOD("add_face_tracker", "tracker_name", "face_tracker"), &XRServer::add_face_tracker);
ClassDB::bind_method(D_METHOD("remove_face_tracker", "tracker_name"), &XRServer::remove_face_tracker);
ClassDB::bind_method(D_METHOD("get_face_trackers"), &XRServer::get_face_trackers);
ClassDB::bind_method(D_METHOD("get_face_tracker", "tracker_name"), &XRServer::get_face_tracker);
ClassDB::bind_method(D_METHOD("get_primary_interface"), &XRServer::get_primary_interface);
ClassDB::bind_method(D_METHOD("set_primary_interface", "interface"), &XRServer::set_primary_interface);
@ -97,6 +103,10 @@ void XRServer::_bind_methods() {
ADD_SIGNAL(MethodInfo("tracker_added", PropertyInfo(Variant::STRING_NAME, "tracker_name"), PropertyInfo(Variant::INT, "type")));
ADD_SIGNAL(MethodInfo("tracker_updated", PropertyInfo(Variant::STRING_NAME, "tracker_name"), PropertyInfo(Variant::INT, "type")));
ADD_SIGNAL(MethodInfo("tracker_removed", PropertyInfo(Variant::STRING_NAME, "tracker_name"), PropertyInfo(Variant::INT, "type")));
ADD_SIGNAL(MethodInfo("face_tracker_added", PropertyInfo(Variant::STRING_NAME, "tracker_name"), PropertyInfo(Variant::OBJECT, "face_tracker", PROPERTY_HINT_RESOURCE_TYPE, "XRFaceTracker")));
ADD_SIGNAL(MethodInfo("face_tracker_updated", PropertyInfo(Variant::STRING_NAME, "tracker_name"), PropertyInfo(Variant::OBJECT, "face_tracker", PROPERTY_HINT_RESOURCE_TYPE, "XRFaceTracker")));
ADD_SIGNAL(MethodInfo("face_tracker_removed", PropertyInfo(Variant::STRING_NAME, "tracker_name")));
};
double XRServer::get_world_scale() const {
@ -352,6 +362,44 @@ PackedStringArray XRServer::get_suggested_pose_names(const StringName &p_tracker
return arr;
}
void XRServer::add_face_tracker(const StringName &p_tracker_name, Ref<XRFaceTracker> p_face_tracker) {
ERR_FAIL_COND(p_face_tracker.is_null());
if (!face_trackers.has(p_tracker_name)) {
// We don't have a tracker with this name, we're going to add it.
face_trackers[p_tracker_name] = p_face_tracker;
emit_signal(SNAME("face_tracker_added"), p_tracker_name, p_face_tracker);
} else if (face_trackers[p_tracker_name] != p_face_tracker) {
// We already have a tracker with this name, we're going to replace it.
face_trackers[p_tracker_name] = p_face_tracker;
emit_signal(SNAME("face_tracker_updated"), p_tracker_name, p_face_tracker);
}
}
void XRServer::remove_face_tracker(const StringName &p_tracker_name) {
// Skip if no face tracker is found.
if (!face_trackers.has(p_tracker_name)) {
return;
}
// Send the removed signal, then remove the face tracker.
emit_signal(SNAME("face_tracker_removed"), p_tracker_name);
face_trackers.erase(p_tracker_name);
}
Dictionary XRServer::get_face_trackers() const {
return face_trackers;
}
Ref<XRFaceTracker> XRServer::get_face_tracker(const StringName &p_tracker_name) const {
// Skip if no tracker is found.
if (!face_trackers.has(p_tracker_name)) {
return Ref<XRFaceTracker>();
}
return face_trackers[p_tracker_name];
}
void XRServer::_process() {
// called from our main game loop before we handle physics and game logic
// note that we can have multiple interfaces active if we have interfaces that purely handle tracking

11
servers/xr_server.h
View file

@ -39,6 +39,7 @@
class XRInterface;
class XRPositionalTracker;
class XRFaceTracker;
/**
The XR server is a singleton object that gives access to the various
@ -86,6 +87,8 @@ private:
Vector<Ref<XRInterface>> interfaces;
Dictionary trackers;
Dictionary face_trackers;
Ref<XRInterface> primary_interface; /* we'll identify one interface as primary, this will be used by our viewports */
double world_scale; /* scale by which we multiply our tracker positions */
@ -183,6 +186,14 @@ public:
PackedStringArray get_suggested_pose_names(const StringName &p_tracker_name) const;
// Q: Should we add get_suggested_input_names and get_suggested_haptic_names even though we don't use them for the IDE?
/*
Face trackers are objects that expose the tracked blend shapes of a face.
*/
void add_face_tracker(const StringName &p_tracker_name, Ref<XRFaceTracker> p_face_tracker);
void remove_face_tracker(const StringName &p_tracker_name);
Dictionary get_face_trackers() const;
Ref<XRFaceTracker> get_face_tracker(const StringName &p_tracker_name) const;
// Process is called before we handle our physics process and game process. This is where our interfaces will update controller data and such.
void _process();