Pose the cockpit hands from the headset's hand tracking

- New [vr] hand_tracking (default off, Quest only for now): the Quest launcher's Settings > VR and
  the headset panel's VR tab, under hand steering. Two XR_EXT_hand_tracking trackers are located
  every XR frame: with the controllers held the Quest builds the joints from their touch sensors
  (XR_EXT_hand_tracking_data_source's controller source), once they are put down from its
  cameras. The trackers exist only while the option and hand steering are on and also serve the
  runtime hand mesh; the extensions (plus XR_FB_hand_tracking_aim) are asked for when either is
  on at launch.
- Aurora skins the runtime mesh with the joints themselves (tracked pose times inverse bind pose,
  no curl, no grip); runtimes with joints but no mesh get a skeleton; non-finite joints put only
  that hand back on its grip curl. AuroraCockpitHand carries the 26 seated-frame joints and radii.
- The manifest declares horizonos.permission.HAND_TRACKING (and the deprecated
  com.oculus.permission.HAND_TRACKING), both normal permissions with no prompt on a Quest 3, and
  oculus.software.handtracking as optional; without it Horizon OS keeps the app controllers-only.
  Bare hands then drive khr/simple_controller, so on Android a hand whose squeeze action is
  inactive and select active is treated as bare: with the option off it presses nothing but the
  menu gesture, is not drawn and feeds no Wii Remote motion.
- Interaction-profile changes and tracker sources are logged. The pure rules (grasp from finger
  flexion, bare latch, pinch gate, bare-hand buttons, flick) live in the OpenXR-free
  vr/openxr_hand_tracking.h with mkw_vr_hand_tracking_tests; the driving and flick parts are
  wired in the next commits. Docs: OPENXR.md, docs/quest-port.md, README.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

aurora-main/include/aurora/aurora.h

@@ -123,11 +123,20 @@ typedef enum {
  * immersive base position. Aurora draws it per eye after the scene, depth-tested
  * against the scene with the scene's own depth mapping.
  */
+enum { AURORA_VR_HAND_JOINT_COUNT = 26 };
+
 typedef struct {
   bool tracked;
   bool held;
+  // The hand-tracking joints below are valid: the hand is drawn from them
+  // instead of curling from squeeze at seatFromGrip.
+  bool jointsValid;
   float squeeze;
   float seatFromGrip[12];
+  // XR_EXT_hand_tracking joints in XR_HAND_JOINT_* order, each a row-major 3x4
+  // in the seated frame, and their radii in metres.
+  float seatFromJoint[AURORA_VR_HAND_JOINT_COUNT][12];
+  float jointRadii[AURORA_VR_HAND_JOINT_COUNT];
 } AuroraCockpitHand;
 
 typedef struct {
@@ -159,7 +168,9 @@ typedef struct {
 void aurora_set_stereo_panel_layer(bool enabled);
 
 // Copies optional runtime-provided hand meshes (XR_FB_hand_tracking_mesh, 26
-// joints). Null clears to the procedural glove. Bind poses: x,y,z,w,px,py,pz.
+// joints). Null clears to the procedural glove. Bind poses: x,y,z,w,px,py,pz,
+// in the space of the mesh's vertices, as xrLocateHandJointsEXT reports poses:
+// a hand with tracked joints is skinned with seatFromJoint * inverse(bind).
 void aurora_set_vr_hand_mesh(uint32_t hand, const AuroraVRHandVertex* vertices, uint32_t vertexCount,
                              const uint16_t* indices, uint32_t indexCount, const float* bindPoses,
                              const int32_t* parents, uint32_t jointCount);

aurora-main/lib/aurora.cpp

@@ -788,6 +788,20 @@ std::optional<AuroraStereoFrame> request_stereo_frame(uint32_t logicalFrame, uin
   }
   // The cockpit overlay is optional: a bad one is dropped, never the frame.
   if (frame.cockpit.active) {
+    // A hand's tracked joints are optional too: non-finite ones only put that
+    // hand back on its grip.
+    for (auto& hand : frame.cockpit.hands) {
+      if (hand.jointsValid && !(finite(&hand.seatFromJoint[0][0], AURORA_VR_HAND_JOINT_COUNT * 12) &&
+                                finite(hand.jointRadii, AURORA_VR_HAND_JOINT_COUNT))) {
+        hand.jointsValid = false;
+        static bool jointRejectionLogged = false;
+        if (!jointRejectionLogged) {
+          jointRejectionLogged = true;
+          Log.warn("Stereo frame {} carries non-finite VR hand joints; drawing that hand at its grip",
+                   logicalFrame);
+        }
+      }
+    }
     const auto& cockpit = frame.cockpit;
     bool valid = finite(&cockpit.wheelAngle, 1) && finite(&cockpit.handlebarRadius, 1) &&
                  finite(&cockpit.unitsPerMeter, 1) && cockpit.unitsPerMeter > 0.f &&

aurora-main/lib/gfx/cockpit.hpp

@@ -128,6 +128,25 @@ inline void glove(std::vector<Vertex>& v, const AuroraCockpitHand& hand, int sid
   tube(v,thumbKnuckle,{palm*(0.030f+0.014f*curl),-(0.052f-0.016f*curl),-0.020f},0.009f,white);
   for(size_t i=start;i<v.size();++i) v[i].position=point(hand.seatFromGrip,v[i].position);
 }
+// Blends the runtime mesh with one skinning matrix per joint (joints left out
+// by `valid` do not pull), then places it: shifted by `offset` and carried by
+// `seatFromMesh` when there is one.
+inline void skin_mesh(std::vector<Vertex>& out, const HandMesh& mesh, const std::array<M,26>& skin,
+                      const std::array<bool,26>& valid, V offset, const float* seatFromMesh) {
+  std::vector<V> points(mesh.vertices.size());
+  for(size_t i=0;i<points.size();++i) {
+    const auto& v=mesh.vertices[i]; V p{}; float total=0;
+    for(int w=0;w<4;++w) if(v.joints[w]>=0&&v.joints[w]<26&&valid[v.joints[w]]&&v.weights[w]>0) {
+      p=add(p,mul(point(skin[v.joints[w]].data(),{v.position[0],v.position[1],v.position[2]}),v.weights[w]));
+      total+=v.weights[w];
+    }
+    if(total>0) p=mul(p,1/total);
+    p=add(p,offset);
+    points[i]=seatFromMesh?point(seatFromMesh,p):p;
+  }
+  for(size_t i=0;i+2<mesh.indices.size();i+=3)
+    triangle(out,points[mesh.indices[i]],points[mesh.indices[i+1]],points[mesh.indices[i+2]],{0.91f,0.95f,1.0f});
+}
 inline void runtime_hand(std::vector<Vertex>& out, const AuroraCockpitHand& hand, const HandMesh& mesh) {
   std::array<M,26> posed{}, skin{};
   std::array<bool,26> done{};
@@ -151,19 +170,52 @@ inline void runtime_hand(std::vector<Vertex>& out, const AuroraCockpitHand& hand
     skin[j]=compose(mesh.inverseBind[1],compose(posed[j],mesh.inverseBind[j]));
     done[j]=true;
   }
-  std::vector<V> points(mesh.vertices.size());
-  for(size_t i=0;i<points.size();++i) {
-    const auto& v=mesh.vertices[i]; V p{}; float total=0;
-    for(int w=0;w<4;++w) if(v.joints[w]>=0&&v.joints[w]<26&&done[v.joints[w]]&&v.weights[w]>0) {
-      p=add(p,mul(point(skin[v.joints[w]].data(),{v.position[0],v.position[1],v.position[2]}),v.weights[w]));
-      total+=v.weights[w];
-    }
-    if(total>0) p=mul(p,1/total);
-    p=add(p,{0,0,0.04f}); // wrist behind the controller grip/palm origin.
-    points[i]=point(hand.seatFromGrip,p);
+  // The wrist sits behind the controller grip/palm origin.
+  skin_mesh(out,mesh,skin,done,{0,0,0.04f},hand.seatFromGrip);
+}
+// The hand-tracking joints are all finite: the hand is drawn from them.
+inline bool joints_finite(const AuroraCockpitHand& hand) {
+  const auto finite=[](float value) {
+    uint32_t bits; std::memcpy(&bits,&value,sizeof(bits)); return (bits&0x7f800000u)!=0x7f800000u;
+  };
+  for(int j=0;j<AURORA_VR_HAND_JOINT_COUNT;++j) {
+    if(!finite(hand.jointRadii[j])) return false;
+    for(float value : hand.seatFromJoint[j]) if(!finite(value)) return false;
   }
-  for(size_t i=0;i+2<mesh.indices.size();i+=3)
-    triangle(out,points[mesh.indices[i]],points[mesh.indices[i+1]],points[mesh.indices[i+2]],{0.91f,0.95f,1.0f});
+  return true;
+}
+inline M joint_matrix(const AuroraCockpitHand& hand, int joint) {
+  M m; std::memcpy(m.data(),hand.seatFromJoint[joint],sizeof(m)); return m;
+}
+// The runtime mesh posed by the tracked joints themselves: the bind poses are
+// in the mesh's own space, as xrLocateHandJointsEXT reports poses, so each
+// joint's skinning matrix is its tracked pose times its inverse bind pose,
+// already in the seated frame. Nothing curls and the grip plays no part.
+inline void tracked_hand(std::vector<Vertex>& out, const AuroraCockpitHand& hand, const HandMesh& mesh) {
+  std::array<M,26> skin{};
+  std::array<bool,26> valid{};
+  for(int j=0;j<26;++j) { skin[j]=compose(joint_matrix(hand,j),mesh.inverseBind[j]); valid[j]=true; }
+  skin_mesh(out,mesh,skin,valid,{0,0,0},nullptr);
+}
+// Without a runtime mesh (PC runtimes report joints but no XR_FB mesh), the
+// tracked joints as a skeleton: each finger a chain of tubes from the wrist to
+// its tip, the joints' own radii, and a ball in the palm.
+inline void joint_skeleton(std::vector<Vertex>& v, const AuroraCockpitHand& hand) {
+  const V white{0.91f,0.95f,1.0f};
+  const auto at=[&](int joint) -> V {
+    return {hand.seatFromJoint[joint][3],hand.seatFromJoint[joint][7],hand.seatFromJoint[joint][11]};
+  };
+  const auto radius=[&](int joint) { return std::clamp(hand.jointRadii[joint],0.004f,0.02f); };
+  constexpr int metacarpal[5]{2,6,11,16,21}, tip[5]{5,10,15,20,25};
+  for(int finger=0;finger<5;++finger) {
+    tube(v,at(1),at(metacarpal[finger]),radius(metacarpal[finger]),white);
+    for(int joint=metacarpal[finger];joint<tip[finger];++joint) {
+      tube(v,at(joint),at(joint+1),radius(joint+1),white);
+      ellipsoid(v,at(joint+1),{radius(joint+1),radius(joint+1),radius(joint+1)},white);
+    }
+  }
+  const float palm=std::clamp(hand.jointRadii[0],0.015f,0.03f);
+  ellipsoid(v,at(0),{palm,palm,palm},white);
 }
 inline void build_geometry(const AuroraCockpit& cockpit, std::vector<Vertex>& vertices) {
   vertices.clear();vertices.reserve(12000);
@@ -192,8 +244,11 @@ inline void build_geometry(const AuroraCockpit& cockpit, std::vector<Vertex>& ve
   std::array<std::shared_ptr<const HandMesh>,2> current;
   { std::lock_guard lock(meshMutex);current=meshes; }
   for(int side=0;side<2;++side) if(cockpit.hands[side].tracked) {
-    if(current[side]) runtime_hand(vertices,cockpit.hands[side],*current[side]);
-    else glove(vertices,cockpit.hands[side],side);
+    const auto& hand=cockpit.hands[side];
+    const bool joints=hand.jointsValid && joints_finite(hand);
+    if(current[side]) joints ? tracked_hand(vertices,hand,*current[side]) : runtime_hand(vertices,hand,*current[side]);
+    else if(joints) joint_skeleton(vertices,hand);
+    else glove(vertices,hand,side);
   }
 }
 inline std::vector<Vertex> geometry(const AuroraCockpit& cockpit) {

aurora-main/tests/cockpit_geometry_test.cpp

@@ -3,7 +3,11 @@
 // build in the seated frame, without a GPU.
 #include <gtest/gtest.h>
 
+#include <algorithm>
+#include <cmath>
 #include <cstring>
+#include <limits>
+#include <memory>
 
 #include "gfx/cockpit.hpp"
 
@@ -257,4 +261,145 @@ TEST_F(CockpitGeometry, RuntimeHandMeshIsSkinnedWithoutNans) {
   EXPECT_TRUE(all_finite(vertices));
 }
 
+// A 26-joint mesh with its bind poses turned and spread out, and a tiny
+// triangle rigidly on each joint, like the runtime's hand mesh.
+std::shared_ptr<aurora::gfx::cockpit::HandMesh> jointed_mesh() {
+  using namespace aurora::gfx::cockpit;
+  auto mesh = std::make_shared<HandMesh>();
+  for (int j = 0; j < 26; ++j) {
+    const float angle = 0.2f * float(j);
+    const float pose[7]{0, std::sin(angle * 0.5f), 0, std::cos(angle * 0.5f), 0.01f * float(j % 5),
+                        -0.02f * float(j / 5), -0.015f * float(j)};
+    mesh->bind[j] = from_pose(pose);
+    mesh->inverseBind[j] = inverse(mesh->bind[j]);
+    mesh->parents[j] = j - 1;
+    for (V offset : {V{0, 0, 0}, V{0.002f, 0, 0}, V{0, 0, 0.002f}}) {
+      AuroraVRHandVertex vertex{};
+      const V p = point(mesh->bind[j].data(), offset);
+      std::memcpy(vertex.position, p.data(), sizeof(vertex.position));
+      vertex.joints[0] = int16_t(j);
+      vertex.joints[1] = vertex.joints[2] = vertex.joints[3] = -1;
+      vertex.weights[0] = 1;
+      mesh->indices.push_back(uint16_t(mesh->vertices.size()));
+      mesh->vertices.push_back(vertex);
+    }
+  }
+  return mesh;
+}
+
+// Tracked joints: each at `seatFromBind` times its bind pose, so the whole
+// hand is that rigid motion of the mesh.
+void set_tracked_joints(AuroraCockpitHand& hand, const aurora::gfx::cockpit::HandMesh& mesh,
+                        const aurora::gfx::cockpit::M& seatFromBind) {
+  using namespace aurora::gfx::cockpit;
+  hand.tracked = true;
+  hand.jointsValid = true;
+  for (int j = 0; j < 26; ++j) {
+    const M joint = compose(seatFromBind, mesh.bind[j]);
+    std::memcpy(hand.seatFromJoint[j], joint.data(), sizeof(hand.seatFromJoint[j]));
+    hand.jointRadii[j] = 0.008f;
+  }
+}
+
+aurora::gfx::cockpit::M rigid_motion() {
+  using namespace aurora::gfx::cockpit;
+  const float pose[7]{0.2f, -0.3f, 0.1f, 0.9273618f, 0.15f, -0.25f, -0.35f};
+  return from_pose(pose);
+}
+
+TEST_F(CockpitGeometry, TrackedJointsSkinTheRuntimeMeshRigidly) {
+  using namespace aurora::gfx::cockpit;
+  const auto mesh = jointed_mesh();
+  {
+    std::lock_guard lock(meshMutex);
+    meshes[1] = mesh;
+  }
+  AuroraCockpit cockpit{};
+  cockpit.nativeWheel = true;
+  const M motion = rigid_motion();
+  set_tracked_joints(cockpit.hands[1], *mesh, motion);
+  const auto vertices = geometry(cockpit);
+  ASSERT_EQ(vertices.size(), mesh->vertices.size());
+  for (size_t i = 0; i < vertices.size(); ++i) {
+    const auto& bindVertex = mesh->vertices[mesh->indices[i]].position;
+    const V expected = point(motion.data(), {bindVertex[0], bindVertex[1], bindVertex[2]});
+    for (int axis = 0; axis < 3; ++axis)
+      EXPECT_NEAR(vertices[i].position[axis], expected[axis], 1e-5f) << "vertex " << i << " axis " << axis;
+  }
+}
+
+TEST_F(CockpitGeometry, TrackedJointsIgnoreSqueezeHeldAndGrip) {
+  using namespace aurora::gfx::cockpit;
+  const auto mesh = jointed_mesh();
+  {
+    std::lock_guard lock(meshMutex);
+    meshes[0] = mesh;
+  }
+  AuroraCockpit cockpit{};
+  cockpit.nativeWheel = true;
+  set_tracked_joints(cockpit.hands[0], *mesh, rigid_motion());
+  set_identity(cockpit.hands[0].seatFromGrip, {0, 0, 0});
+  const auto reference = geometry(cockpit);
+  cockpit.hands[0].squeeze = 1.0f;
+  cockpit.hands[0].held = true;
+  set_identity(cockpit.hands[0].seatFromGrip, {0.5f, 0.5f, 0.5f});
+  const auto moved = geometry(cockpit);
+  ASSERT_EQ(moved.size(), reference.size());
+  for (size_t i = 0; i < moved.size(); ++i)
+    for (int axis = 0; axis < 3; ++axis)
+      EXPECT_EQ(moved[i].position[axis], reference[i].position[axis]);
+}
+
+float distance_to_segment(V p, V a, V b) {
+  using namespace aurora::gfx::cockpit;
+  const V ab = sub(b, a);
+  const float length = dot(ab, ab);
+  const float t = length > 0 ? std::clamp(dot(sub(p, a), ab) / length, 0.0f, 1.0f) : 0.0f;
+  const V d = sub(p, add(a, mul(ab, t)));
+  return std::sqrt(dot(d, d));
+}
+
+TEST_F(CockpitGeometry, TrackedJointsWithoutAMeshDrawASkeletonOnTheBones) {
+  using namespace aurora::gfx::cockpit;
+  const auto mesh = jointed_mesh(); // only for joint poses; no runtime mesh is set
+  AuroraCockpit cockpit{};
+  cockpit.nativeWheel = true;
+  set_tracked_joints(cockpit.hands[0], *mesh, rigid_motion());
+  const auto& hand = cockpit.hands[0];
+  const auto at = [&](int j) { return V{hand.seatFromJoint[j][3], hand.seatFromJoint[j][7], hand.seatFromJoint[j][11]}; };
+  const auto vertices = geometry(cockpit);
+  ASSERT_FALSE(vertices.empty());
+  ASSERT_TRUE(all_finite(vertices));
+  constexpr int metacarpal[5]{2, 6, 11, 16, 21}, tip[5]{5, 10, 15, 20, 25};
+  for (const auto& vertex : vertices) {
+    float nearest = 1e9f;
+    for (int finger = 0; finger < 5; ++finger) {
+      nearest = std::min(nearest, distance_to_segment(vertex.position, at(1), at(metacarpal[finger])));
+      for (int j = metacarpal[finger]; j < tip[finger]; ++j)
+        nearest = std::min(nearest, distance_to_segment(vertex.position, at(j), at(j + 1)));
+    }
+    const V fromPalm = sub(vertex.position, at(0));
+    nearest = std::min(nearest, std::sqrt(dot(fromPalm, fromPalm)));
+    EXPECT_LT(nearest, 0.031f) << "every skeleton vertex lies on a bone, a joint or the palm";
+  }
+}
+
+TEST_F(CockpitGeometry, NonFiniteJointsFallBackToTheGrip) {
+  using namespace aurora::gfx::cockpit;
+  const auto mesh = jointed_mesh();
+  AuroraCockpit cockpit{};
+  cockpit.nativeWheel = true;
+  set_tracked_joints(cockpit.hands[1], *mesh, rigid_motion());
+  set_identity(cockpit.hands[1].seatFromGrip, {0.2f, -0.3f, -0.4f});
+  cockpit.hands[1].seatFromJoint[7][5] = std::numeric_limits<float>::quiet_NaN();
+  const auto broken = geometry(cockpit);
+  cockpit.hands[1].jointsValid = false;
+  const auto grip = geometry(cockpit);
+  ASSERT_TRUE(all_finite(broken));
+  ASSERT_EQ(broken.size(), grip.size()) << "a non-finite joint draws the glove at the grip";
+  for (size_t i = 0; i < broken.size(); ++i)
+    for (int axis = 0; axis < 3; ++axis)
+      EXPECT_EQ(broken[i].position[axis], grip[i].position[axis]);
+}
+
 } // namespace