Pose the cockpit hands from the headset's hand tracking

- New [vr] hand_tracking (default off, Quest only for now): the Quest launcher's Settings > VR and
  the headset panel's VR tab, under hand steering. Two XR_EXT_hand_tracking trackers are located
  every XR frame: with the controllers held the Quest builds the joints from their touch sensors
  (XR_EXT_hand_tracking_data_source's controller source), once they are put down from its
  cameras. The trackers exist only while the option and hand steering are on and also serve the
  runtime hand mesh; the extensions (plus XR_FB_hand_tracking_aim) are asked for when either is
  on at launch.
- Aurora skins the runtime mesh with the joints themselves (tracked pose times inverse bind pose,
  no curl, no grip); runtimes with joints but no mesh get a skeleton; non-finite joints put only
  that hand back on its grip curl. AuroraCockpitHand carries the 26 seated-frame joints and radii.
- The manifest declares horizonos.permission.HAND_TRACKING (and the deprecated
  com.oculus.permission.HAND_TRACKING), both normal permissions with no prompt on a Quest 3, and
  oculus.software.handtracking as optional; without it Horizon OS keeps the app controllers-only.
  Bare hands then drive khr/simple_controller, so on Android a hand whose squeeze action is
  inactive and select active is treated as bare: with the option off it presses nothing but the
  menu gesture, is not drawn and feeds no Wii Remote motion.
- Interaction-profile changes and tracker sources are logged. The pure rules (grasp from finger
  flexion, bare latch, pinch gate, bare-hand buttons, flick) live in the OpenXR-free
  vr/openxr_hand_tracking.h with mkw_vr_hand_tracking_tests; the driving and flick parts are
  wired in the next commits. Docs: OPENXR.md, docs/quest-port.md, README.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

runtime/include/runtime_config.h

@@ -80,6 +80,7 @@ struct RuntimeUserConfig {
     std::optional<bool> vrSteeringWheel;
     std::optional<bool> vrNativeSteeringWheel;
     std::optional<bool> vrHandSteering;
+    std::optional<bool> vrHandTracking;
     std::optional<float> vrWheelKartDegrees;
     std::optional<float> vrWheelBikeDegrees;
     std::optional<float> vrWheelGrabDistance;
@@ -233,6 +234,11 @@ inline constexpr float kVrCockpitUnitsPerMeterMax = 400.0f;
 inline constexpr bool kVrSteeringWheelDefault = true;
 inline constexpr bool kVrNativeSteeringWheelDefault = true;
 inline constexpr bool kVrHandSteeringDefault = true;
+// The cockpit hands follow the headset's hand tracking (the controllers' touch
+// sensors while they are held, the cameras once they are put down, when bare
+// hands also drive). Opt-in, and only offered on the Quest for now; the
+// launcher's Settings page shows the same default.
+inline constexpr bool kVrHandTrackingDefault = false;
 // Hand steering tuning ranges; the defaults are mkw::vr::WheelTuning's.
 inline constexpr float kVrWheelDegreesMin = 20.0f, kVrWheelDegreesMax = 180.0f;
 inline constexpr float kVrWheelGrabDistanceMin = 0.15f, kVrWheelGrabDistanceMax = 0.8f;
@@ -862,6 +868,7 @@ inline RuntimeUserConfig ParseConfigDocument(const toml::value& document) {
     config.vrSteeringWheel = FindConfigValue<bool>(document, "vr", "steering_wheel");
     config.vrNativeSteeringWheel = FindConfigValue<bool>(document, "vr", "native_steering_wheel");
     config.vrHandSteering = FindConfigValue<bool>(document, "vr", "hand_steering");
+    config.vrHandTracking = FindConfigValue<bool>(document, "vr", "hand_tracking");
     config.vrWheelKartDegrees = readRangedFloat("wheel_kart_degrees", kVrWheelDegreesMin, kVrWheelDegreesMax);
     config.vrWheelBikeDegrees = readRangedFloat("wheel_bike_degrees", kVrWheelDegreesMin, kVrWheelDegreesMax);
     config.vrWheelGrabDistance =
@@ -1263,6 +1270,11 @@ inline bool SetVrHandSteering(bool value) {
     return WriteSetting("vr", "hand_steering", value ? "true" : "false");
 }
 
+inline bool SetVrHandTracking(bool value) {
+    Mutable().vrHandTracking = value;
+    return WriteSetting("vr", "hand_tracking", value ? "true" : "false");
+}
+
 inline bool SetVrWheelTuning(const mkw::vr::WheelTuning& tuning) {
     auto& config = Mutable();
     const auto write = [](const char* key, std::optional<float>& slot, float value, float low, float high) {
@@ -1746,6 +1758,10 @@ inline bool VrHandSteering(bool fallback = kVrHandSteeringDefault) {
     return Get().vrHandSteering.value_or(fallback);
 }
 
+inline bool VrHandTracking(bool fallback = kVrHandTrackingDefault) {
+    return Get().vrHandTracking.value_or(fallback);
+}
+
 inline mkw::vr::WheelTuning VrWheelTuning() {
     const auto& config = Get();
     mkw::vr::WheelTuning tuning{};

runtime/include/vr/openxr_driving.h

@@ -17,6 +17,7 @@
 // forward. It is the frame the first-person anchor places the vehicle in, so
 // hands, wheel geometry and eye transforms all meet there.
 
+#include "vr/openxr_hand_tracking.h"
 #include "vr/openxr_wii_remote.h"
 #include "vr/steering_wheel.h"
 
@@ -35,6 +36,17 @@ struct DrivingHand {
     float squeeze = 0.0f;
     // Row-major 3x4 from the controller's grip space into the seated frame.
     std::array<float, 12> seat_from_grip{1.0f, 0.0f, 0.0f, 0.0f, 0.0f, 1.0f, 0.0f, 0.0f, 0.0f, 0.0f, 1.0f, 0.0f};
+    // Tracked hands ([vr] hand_tracking, openxr_hand_tracking.h). The joints
+    // themselves stay with OpenXRInput::HandJoints(), on the pacing thread;
+    // this says whether the cockpit draws the hand from them, and carries what
+    // the settings panel reads out: where they came from, whether the hand is
+    // bare (camera-tracked, no controller), how closed it is and whether it
+    // pinches.
+    bool joints_valid = false;
+    bool bare = false;
+    hand_tracking::Source source = hand_tracking::Source::None;
+    float grasp = 0.0f;
+    bool pinch = false;
 };
 
 struct DrivingSnapshot {

runtime/include/vr/openxr_hand_mesh.h

@@ -13,7 +13,9 @@ namespace mkw::vr {
 // AnimalCrossing-VR-MR-Standalone obtains its white hands from this Meta
 // runtime extension, not from a distributable model asset. Use the same API,
 // with a procedural fallback on PC runtimes that do not expose Meta meshes.
-inline bool LoadRuntimeHandMeshes(OpenXRRuntime& runtime) {
+// `existing` are the tracked hands' trackers (OpenXRInput::HandTracker), used
+// where there is one; otherwise a tracker is made just for the mesh.
+inline bool LoadRuntimeHandMeshes(OpenXRRuntime& runtime, const XrHandTrackerEXT* existing = nullptr) {
     for (uint32_t h = 0; h < 2; ++h)
         aurora_set_vr_hand_mesh(h, nullptr, 0, nullptr, 0, nullptr, nullptr, 0);
     const auto& extensions = runtime.EnabledExtensions();
@@ -28,12 +30,18 @@ inline bool LoadRuntimeHandMeshes(OpenXRRuntime& runtime) {
     if (!create || !destroy || !meshFn) return false;
     bool any = false;
     for (uint32_t h = 0; h < 2; ++h) {
-        XrHandTrackerCreateInfoEXT info{XR_TYPE_HAND_TRACKER_CREATE_INFO_EXT};
-        info.hand = h ? XR_HAND_RIGHT_EXT : XR_HAND_LEFT_EXT;
-        info.handJointSet = XR_HAND_JOINT_SET_DEFAULT_EXT;
-        XrHandTrackerEXT tracker = XR_NULL_HANDLE;
-        if (XR_FAILED(create(runtime.Session(), &info, &tracker))) continue;
-        struct Guard { XrHandTrackerEXT tracker; PFN_xrDestroyHandTrackerEXT destroy; ~Guard() { destroy(tracker); } } guard{tracker, destroy};
+        XrHandTrackerEXT tracker = existing != nullptr ? existing[h] : XR_NULL_HANDLE;
+        const bool owned = tracker == XR_NULL_HANDLE;
+        if (owned) {
+            XrHandTrackerCreateInfoEXT info{XR_TYPE_HAND_TRACKER_CREATE_INFO_EXT};
+            info.hand = h ? XR_HAND_RIGHT_EXT : XR_HAND_LEFT_EXT;
+            info.handJointSet = XR_HAND_JOINT_SET_DEFAULT_EXT;
+            if (XR_FAILED(create(runtime.Session(), &info, &tracker))) continue;
+        }
+        struct Guard {
+            XrHandTrackerEXT tracker; PFN_xrDestroyHandTrackerEXT destroy; bool owned;
+            ~Guard() { if (owned) destroy(tracker); }
+        } guard{tracker, destroy, owned};
         XrHandTrackingMeshFB mesh{XR_TYPE_HAND_TRACKING_MESH_FB};
         if (XR_FAILED(meshFn(tracker, &mesh)) || mesh.jointCountOutput != 26 ||
             !mesh.vertexCountOutput || mesh.vertexCountOutput > 65535 || !mesh.indexCountOutput ||

runtime/include/vr/openxr_hand_tracking.h

@@ -0,0 +1,426 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#pragma once
+
+// The cockpit hands from the headset's hand tracking ([vr] hand_tracking), and
+// what bare hands do once the controllers are put down.
+//
+// With the option on, the OpenXR pacing thread locates both hands' 26 joints
+// (XR_EXT_hand_tracking) every XR frame. Where a hand holds a controller, the
+// Quest builds them from the controller's touch sensors
+// (XR_EXT_hand_tracking_data_source's controller source); once the controllers
+// are put down, from its cameras, and the runtime then drives
+// /interaction_profiles/khr/simple_controller from the hand: select is an index
+// pinch, the left menu the palm-up menu gesture. The rules that turn all that
+// into the game's input live here, free of OpenXR, so they are tested headlessly
+// (tests/vr_hand_tracking_tests.cpp):
+//
+// - A hand is hand-driven when its squeeze action is inactive and its select
+//   action active: the Touch profile binds squeeze, simple_controller does not.
+//   That decides what its buttons mean, and needs no tracker, so the manifest's
+//   hand-tracking permission can never let resting hands press anything.
+// - It is bare while it is hand-driven with camera-tracked joints, latched
+//   through the wheel's tracking grace. That decides the wheel grab (the palm's
+//   position, a grasp from the fingers' flexion) and the flick.
+//
+// Only the Android build applies these: a PC runtime can drive real controllers
+// through simple_controller and synthesize joints for them.
+
+#include "vr/openxr_wii_remote.h"
+
+#include <algorithm>
+#include <array>
+#include <cmath>
+#include <cstddef>
+#include <cstdint>
+#include <cstring>
+
+namespace mkw::vr::hand_tracking {
+
+// XR_HAND_JOINT_*_EXT order.
+inline constexpr size_t kJointCount = 26;
+inline constexpr size_t kPalm = 0;
+inline constexpr size_t kWrist = 1;
+// Each finger's metacarpal joint, thumb first. A finger's chain runs from there
+// to its tip; the thumb's has no intermediate joint, so it is one shorter.
+inline constexpr std::array<size_t, 5> kMetacarpal{2, 6, 11, 16, 21};
+inline constexpr std::array<size_t, 5> kTip{5, 10, 15, 20, 25};
+
+using Vec3 = std::array<float, 3>;
+using JointPositions = std::array<Vec3, kJointCount>;
+
+// Where a hand's joints came from this frame.
+enum class Source : uint8_t {
+    None,       // not located
+    Controller, // built from a held controller's touch sensors
+    Camera,     // the headset's cameras
+    Unknown,    // located, but the runtime does not say how
+};
+
+inline const char* SourceLabel(Source source) noexcept {
+    switch (source) {
+    case Source::Controller: return "controller";
+    case Source::Camera: return "camera";
+    case Source::Unknown: return "tracked";
+    default: return "none";
+    }
+}
+
+// Both hands' joints in the seated frame (openxr_driving.h), for the cockpit
+// overlay: row-major 3x4 per joint, and each joint's radius in metres.
+struct HandJointFrame {
+    std::array<bool, 2> valid{};
+    std::array<std::array<std::array<float, 12>, kJointCount>, 2> seat_from_joint{};
+    std::array<std::array<float, kJointCount>, 2> radius{};
+};
+
+inline bool IsFinite(float value) noexcept {
+    // Bit test: the runtime may be built with -ffast-math.
+    uint32_t bits = 0;
+    std::memcpy(&bits, &value, sizeof(bits));
+    return (bits & 0x7F800000u) != 0x7F800000u;
+}
+
+// The fingers' summed flexion that reads as an open hand, and as a hand closed
+// on the wheel's rim. A relaxed hand bends its three finger joints by roughly
+// 1.0 to 1.6 radians in all, which must stay under the wheel's 0.15 release;
+// a grip round a rim bends them well past the 0.55 press. Starting values for
+// tuning with the headset panel's readout.
+inline constexpr float kGraspOpenRadians = 1.2f;
+inline constexpr float kGraspClosedRadians = 3.0f;
+
+// The angle between two successive bones a->b and b->c, 0 for a straight
+// finger. Negative when a bone is too short to have a direction.
+inline float BendRadians(const Vec3& a, const Vec3& b, const Vec3& c) noexcept {
+    const Vec3 u{b[0] - a[0], b[1] - a[1], b[2] - a[2]};
+    const Vec3 v{c[0] - b[0], c[1] - b[1], c[2] - b[2]};
+    const float uu = u[0] * u[0] + u[1] * u[1] + u[2] * u[2];
+    const float vv = v[0] * v[0] + v[1] * v[1] + v[2] * v[2];
+    if (!(uu > 1.0e-8f) || !(vv > 1.0e-8f)) {
+        return -1.0f;
+    }
+    const Vec3 cross{u[1] * v[2] - u[2] * v[1], u[2] * v[0] - u[0] * v[2], u[0] * v[1] - u[1] * v[0]};
+    const float sine = std::sqrt(cross[0] * cross[0] + cross[1] * cross[1] + cross[2] * cross[2]);
+    const float cosine = u[0] * v[0] + u[1] * v[1] + u[2] * v[2];
+    return std::atan2(sine, cosine);
+}
+
+// How far a hand is closed, 0 (open) to 1 (closed on a rim or in a fist): the
+// middle, ring and little fingers' flexion, each summed over their three
+// joints. The index finger and thumb are left out so pinching does not grab.
+// Angles only, so it is the same for either hand, at any size and in any
+// orientation. 0 for joints that are not finite or collapse onto each other.
+inline float GraspFromJoints(const JointPositions& joints) noexcept {
+    for (const Vec3& joint : joints) {
+        if (!IsFinite(joint[0]) || !IsFinite(joint[1]) || !IsFinite(joint[2])) {
+            return 0.0f;
+        }
+    }
+    float sum = 0.0f;
+    for (size_t finger = 2; finger < 5; ++finger) {
+        const size_t base = kMetacarpal[finger];
+        float flex = 0.0f;
+        for (size_t joint = base; joint + 2 <= base + 4; ++joint) {
+            const float bend = BendRadians(joints[joint], joints[joint + 1], joints[joint + 2]);
+            if (!(bend >= 0.0f)) {
+                return 0.0f;
+            }
+            flex += bend;
+        }
+        sum += std::clamp((flex - kGraspOpenRadians) / (kGraspClosedRadians - kGraspOpenRadians), 0.0f, 1.0f);
+    }
+    return sum / 3.0f;
+}
+
+// Keeps a camera-tracked hand bare, with its last grasp, through a short loss
+// of tracking (fingers hidden behind the other hand, a hand turned edge-on):
+// long enough for the wheel's own grace to keep hold. A controller's squeeze
+// clears it at once, since a hand that picked one up is no longer bare.
+class BareLatch {
+public:
+    // `hand_driven`: the hand drives simple_controller this frame.
+    // `camera_joints`: its joints were located this frame, not from a controller.
+    // `grace`: the wheel's tracking grace in seconds.
+    bool Update(bool hand_driven, bool squeeze_active, bool camera_joints, float grasp, float dt,
+                float grace) noexcept {
+        if (squeeze_active) {
+            Reset();
+            return false;
+        }
+        if (hand_driven && camera_joints) {
+            m_bare = true;
+            m_tracked = true;
+            m_lost = 0.0f;
+            m_grasp = IsFinite(grasp) ? std::clamp(grasp, 0.0f, 1.0f) : 0.0f;
+            return true;
+        }
+        if (m_bare) {
+            m_tracked = false;
+            m_lost += IsFinite(dt) ? std::clamp(dt, 0.0f, 0.1f) : 0.0f;
+            if (m_lost <= grace) {
+                return true;
+            }
+        }
+        Reset();
+        return false;
+    }
+
+    bool Bare() const noexcept { return m_bare; }
+    // False while the latch is only bridging a loss of tracking.
+    bool Tracked() const noexcept { return m_bare && m_tracked; }
+    float Grasp() const noexcept { return m_bare ? m_grasp : 0.0f; }
+    void Reset() noexcept { *this = BareLatch{}; }
+
+private:
+    bool m_bare = false;
+    bool m_tracked = false;
+    float m_lost = 0.0f;
+    float m_grasp = 0.0f;
+};
+
+// A pinch uses an item only when it starts on a hand that has been off the
+// wheel for a moment: a hand opening off the rim often reads as a pinch for a
+// frame or two. Holding the pinch holds the button, which trails an item.
+class PinchGate {
+public:
+    static constexpr float kFreeSeconds = 0.15f;
+
+    bool Update(bool pinch, bool held, float dt) noexcept {
+        dt = IsFinite(dt) ? std::clamp(dt, 0.0f, 0.1f) : 0.0f;
+        if (held) {
+            m_free = 0.0f;
+            m_firing = false;
+            m_blocked = pinch;
+            return false;
+        }
+        if (!pinch) {
+            m_firing = false;
+            m_blocked = false;
+            m_free = std::min(m_free + dt, 1.0f);
+            return false;
+        }
+        if (!m_firing && !m_blocked) {
+            (m_free >= kFreeSeconds ? m_firing : m_blocked) = true;
+        }
+        m_free = std::min(m_free + dt, 1.0f);
+        return m_firing;
+    }
+
+    void Reset() noexcept { *this = PinchGate{}; }
+
+private:
+    float m_free = 0.0f;
+    bool m_firing = false;
+    bool m_blocked = false;
+};
+
+// A hand's pinch and menu gesture this frame. The runtime's own recognition
+// (XR_FB_hand_tracking_aim) is preferred when it is valid, and a pinch made
+// while the hand is in the system gesture (palm towards the face) is the menu
+// gesture, not a pinch. Otherwise the select and menu actions, as
+// simple_controller delivers them.
+struct Gestures {
+    bool pinch = false;
+    bool menu = false;
+};
+
+inline Gestures GesturesOf(bool aim_valid, bool aim_pinching, bool aim_menu, bool aim_system_gesture,
+                           bool action_select, bool action_menu) noexcept {
+    if (aim_valid) {
+        return {aim_pinching && !aim_system_gesture, aim_menu || action_menu};
+    }
+    return {action_select, action_menu};
+}
+
+// The select action as simple_controller delivers it: right select is bound to
+// the primary action, left select to the secondary one (openxr_input.cpp).
+inline bool SelectOf(const wii_remote::HandInputs& inputs, size_t hand) noexcept {
+    return hand == 1 ? inputs.primary : inputs.secondary;
+}
+
+// Before anything reads the hands (the settings panel, then the game). A
+// hand-driven hand's select is replaced: with tracked hands on, a right pinch
+// stays A, for the menus, and a left one no longer toggles the settings panel;
+// with them off, the hand presses nothing but the menu gesture, so resting hands
+// with the controllers put down never press anything. Controller hands are left
+// alone.
+inline void ApplyHandDrivenButtons(std::array<wii_remote::HandInputs, 2>& hands,
+                                   const std::array<bool, 2>& hand_driven, const std::array<bool, 2>& pinch,
+                                   bool option_on) noexcept {
+    for (size_t hand = 0; hand < hands.size(); ++hand) {
+        if (!hand_driven[hand]) {
+            continue;
+        }
+        hands[hand].primary = option_on && hand == 1 && pinch[hand];
+        hands[hand].secondary = false;
+    }
+}
+
+// In a cockpit race, after the wheel: a bare hand on the wheel holds the gas
+// (A: the right primary button), and an item pinch uses an item (the left
+// trigger: the Wii Remote's Z, the GameCube's L). A hand-driven right hand's
+// pinch stops meaning A there. The grasp itself never becomes a squeeze, which
+// would press the gamepad's shoulders (GameCube R drifts).
+inline void ApplyBareHandRace(std::array<wii_remote::HandInputs, 2>& hands,
+                              const std::array<bool, 2>& hand_driven, const std::array<bool, 2>& bare_held,
+                              const std::array<bool, 2>& item_pinch) noexcept {
+    if (hand_driven[1]) {
+        hands[1].primary = false;
+    }
+    if (bare_held[0] || bare_held[1]) {
+        hands[1].primary = true;
+    }
+    if (item_pinch[0] || item_pinch[1]) {
+        hands[0].trigger = 1.0f;
+    }
+}
+
+// One bare hand for the flick detector: its palm's height in the seated frame.
+struct FlickHand {
+    bool tracked = false;
+    bool held = false;
+    float height = 0.0f;
+};
+
+// A quick upward flick of the hands, the bare-hand shake that does a trick off
+// a ramp or pulls a wheelie. Both hands on the wheel must rise together at a
+// similar speed, so a turn, where one hand rises as the other drops, never
+// counts; a free hand counts alone, but a lone hand on the wheel does not
+// (that is a turn too). A rise must last a few samples and cover some height
+// soon enough, so tracking noise and a pose jumping when tracking comes back
+// never count; the second is also caught as an impossible speed.
+class FlickDetector {
+public:
+    static constexpr float kRisingSpeed = 0.3f;        // m/s: the hand is going up
+    static constexpr float kJumpSpeed = 5.0f;          // m/s: faster than a hand, a tracking jump
+    static constexpr float kMinRise = 0.05f;           // m
+    static constexpr float kWindowSeconds = 0.15f;     // to cover kMinRise
+    static constexpr int kMinSamples = 3;
+    static constexpr float kPairSpeed = 1.2f;          // mean of both hands, m/s
+    static constexpr float kPairEachSpeed = 0.6f;
+    static constexpr float kPairSpeedDifference = 0.6f;
+    static constexpr float kFreeSpeed = 1.5f;
+    static constexpr float kCooldownSeconds = 0.5f;
+
+    bool Update(const std::array<FlickHand, 2>& hands, float dt) noexcept {
+        if (!IsFinite(dt) || dt <= 0.0f) {
+            return false;
+        }
+        dt = std::min(dt, 0.1f);
+        m_cooldown = std::max(m_cooldown - dt, 0.0f);
+        std::array<float, 2> speed{};
+        std::array<bool, 2> rising{};
+        for (size_t hand = 0; hand < 2; ++hand) {
+            rising[hand] = Step(m_tracks[hand], hands[hand], dt, speed[hand]);
+        }
+        if (m_cooldown > 0.0f) {
+            return false;
+        }
+        bool fire = false;
+        if (hands[0].held && hands[1].held) {
+            fire = rising[0] && rising[1] && speed[0] >= kPairEachSpeed && speed[1] >= kPairEachSpeed &&
+                   0.5f * (speed[0] + speed[1]) >= kPairSpeed &&
+                   std::fabs(speed[0] - speed[1]) <= kPairSpeedDifference;
+        }
+        for (size_t hand = 0; hand < 2 && !fire; ++hand) {
+            fire = hands[hand].tracked && !hands[hand].held && rising[hand] && speed[hand] >= kFreeSpeed;
+        }
+        if (fire) {
+            m_cooldown = kCooldownSeconds;
+            for (Track& track : m_tracks) {
+                track.spent = true;
+            }
+        }
+        return fire;
+    }
+
+    void Reset() noexcept { *this = FlickDetector{}; }
+
+private:
+    struct Track {
+        bool has_last = false;
+        float last = 0.0f;
+        bool rising = false;
+        bool spent = false; // this rise already flicked, or took too long
+        float start = 0.0f;
+        float time = 0.0f;
+        int samples = 0;
+    };
+
+    // Whether the hand is in a rise that qualifies, and that rise's mean speed.
+    static bool Step(Track& track, const FlickHand& hand, float dt, float& speed) noexcept {
+        if (!hand.tracked || !IsFinite(hand.height)) {
+            track = {};
+            return false;
+        }
+        if (!track.has_last) {
+            track.has_last = true;
+            track.last = hand.height;
+            return false;
+        }
+        const float velocity = (hand.height - track.last) / dt;
+        if (std::fabs(velocity) > kJumpSpeed) {
+            track = {};
+            track.has_last = true;
+            track.last = hand.height;
+            return false;
+        }
+        if (velocity >= kRisingSpeed) {
+            if (!track.rising) {
+                track.rising = true;
+                track.spent = false;
+                track.start = track.last;
+                track.time = 0.0f;
+                track.samples = 0;
+            }
+            track.time += dt;
+            ++track.samples;
+        } else {
+            track.rising = false;
+        }
+        track.last = hand.height;
+        if (!track.rising || track.spent) {
+            return false;
+        }
+        const float rise = hand.height - track.start;
+        if (rise < kMinRise) {
+            if (track.time > kWindowSeconds) {
+                track.spent = true; // a slow lift, not a flick
+            }
+            return false;
+        }
+        if (track.samples < kMinSamples) {
+            return false;
+        }
+        speed = rise / track.time;
+        return true;
+    }
+
+    std::array<Track, 2> m_tracks{};
+    float m_cooldown = 0.0f;
+};
+
+// The remote's accelerometer through one flick: up, then down, one cycle of a
+// shake, as Dolphin's emulated shake produces them. It lasts long enough that
+// the guest, which reads only the latest sample, sees it on at least three of
+// its frames; KPAD derives acc_speed from the change between them.
+inline constexpr int64_t kFlickPulseNs = 150'000'000;
+inline constexpr float kFlickPulseG = 3.0f;
+
+inline wii_remote::Vec3 FlickPulse(int64_t elapsed_ns, bool* active) noexcept {
+    const wii_remote::Vec3 rest{0.0f, -1.0f, 0.0f};
+    if (elapsed_ns < 0 || elapsed_ns >= kFlickPulseNs) {
+        if (active != nullptr) {
+            *active = false;
+        }
+        return rest;
+    }
+    if (active != nullptr) {
+        *active = true;
+    }
+    const float phase = static_cast<float>(elapsed_ns) / static_cast<float>(kFlickPulseNs);
+    const float swing = kFlickPulseG * std::sin(phase * 6.2831853f);
+    return {0.0f, std::clamp(rest[1] + swing, -wii_remote::kAccelRangeG, wii_remote::kAccelRangeG), 0.0f};
+}
+
+} // namespace mkw::vr::hand_tracking

runtime/include/vr/openxr_input.h

@@ -6,6 +6,7 @@
 
 #include "vr/camera_toggle.h"
 #include "vr/openxr_driving.h"
+#include "vr/openxr_hand_tracking.h"
 #include "vr/openxr_runtime.h"
 #include "vr/openxr_settings_panel.h"
 #include "vr/openxr_wii_remote.h"
@@ -66,6 +67,15 @@ struct OpenXRPointerScreen {
 // the game (a shoulder on the gamepad; as a Wii Remote the grips are unbound,
 // so a hand on the wheel cannot hold down a button).
 //
+// Tracked hands ([vr] hand_tracking, with hand steering): two hand trackers
+// (XR_EXT_hand_tracking) located every frame give the cockpit the hands' own
+// joints, from the controllers' touch sensors while they are held and from the
+// cameras once they are put down. On the Quest a hand that drives
+// khr/simple_controller instead of the Touch profile has bare-hand buttons
+// (openxr_hand_tracking.h): with the option off it presses nothing but the menu
+// gesture, so the manifest's hand-tracking permission changes nothing for
+// players who leave it off.
+//
 // A right-thumbstick click on its own toggles the first-person camera, as its
 // F10 checkbox does (first_person_toggle_click).
 //
@@ -105,6 +115,13 @@ public:
 
     // The cockpit state the last Sync published (also OpenXRReadDriving()).
     const DrivingSnapshot& Driving() const noexcept { return m_driving; }
+    // The tracked hands' joints in the seated frame, as the last Sync located
+    // them; read on the pacing thread only.
+    const hand_tracking::HandJointFrame& HandJoints() const noexcept { return m_joint_frame; }
+    // A hand's tracker while tracked hands keep one, else XR_NULL_HANDLE.
+    XrHandTrackerEXT HandTracker(uint32_t hand) const noexcept {
+        return hand < kHands ? m_hand_trackers[hand] : XR_NULL_HANDLE;
+    }
 
     // Publishes a remote with nothing held, at rest and not pointing, and stops
     // the haptics, for frames without focused input.
@@ -140,6 +157,14 @@ private:
     void UpdateDriving(XrTime display_time, const driving::SeatFrame& seat,
                        std::array<wii_remote::HandInputs, kHands>& hands, bool withheld);
     void ResetDriving();
+    // Tracked hands: the extension's functions at Create, the trackers as the
+    // settings ask for them, and both hands located for `time`, their joints
+    // in `seat` when it is valid.
+    void LoadHandTracking();
+    void UpdateHandTrackers();
+    void DestroyHandTrackers();
+    void LocateHands(XrTime time, const driving::SeatFrame& seat);
+    void LogInteractionProfiles();
     void UpdateRumble();
     void StopRumble();
     bool Check(XrResult result, const char* operation);
@@ -182,6 +207,38 @@ private:
     bool m_wheel_uses_geometry = false;
     bool m_wheel_bike = false;
     DrivingSnapshot m_driving{};
+
+    // Tracked hands.
+    struct TrackedHand {
+        bool active = false; // joints located this frame
+        bool seated = false; // and written into m_joint_frame, in the seated frame
+        hand_tracking::Source source = hand_tracking::Source::None;
+        hand_tracking::JointPositions positions{}; // application space
+        // XR_FB_hand_tracking_aim: the runtime's own pinch and menu gesture.
+        bool aim_valid = false;
+        bool aim_pinching = false;
+        bool aim_menu = false;
+        bool aim_system_gesture = false;
+    };
+    PFN_xrCreateHandTrackerEXT m_create_hand_tracker = nullptr;
+    PFN_xrDestroyHandTrackerEXT m_destroy_hand_tracker = nullptr;
+    PFN_xrLocateHandJointsEXT m_locate_hand_joints = nullptr;
+    bool m_hand_data_source = false; // XR_EXT_hand_tracking_data_source
+    bool m_hand_aim = false;         // XR_FB_hand_tracking_aim
+    XrHandTrackerEXT m_hand_trackers[kHands]{};
+    bool m_hand_trackers_failed = false;
+    bool m_logged_hand_restart = false;
+    std::array<TrackedHand, kHands> m_tracked_hands{};
+    hand_tracking::HandJointFrame m_joint_frame{};
+    std::array<hand_tracking::Source, kHands> m_logged_sources{hand_tracking::Source::None,
+                                                               hand_tracking::Source::None};
+    // Per frame, from the actions: a controller is in the hand (its squeeze is
+    // bound), and the hand drives simple_controller (Android only).
+    std::array<bool, kHands> m_squeeze_active{};
+    std::array<bool, kHands> m_hand_driven{};
+    std::array<bool, kHands> m_pinch{};
+    uint64_t m_profile_serial = 0;
+
     bool m_created = false;
     bool m_logged_sync_failure = false;
     bool m_logged_pointer = false;

runtime/include/vr/openxr_integration.h

@@ -78,6 +78,10 @@ void OpenXRSetImmersiveWindow(bool enabled) noexcept;
 // 0 = Off, 1 = Auto, otherwise 72/90/120 as a rendering-rate ceiling.
 void OpenXRSetFrameInterpolationFps(uint32_t target) noexcept;
 bool OpenXRFrameInterpolationAvailable() noexcept;
+
+// This session started with XR_EXT_hand_tracking, so [vr] hand_tracking applies
+// live (it is asked for when hand steering or tracked hands are on at launch).
+bool OpenXRHandTrackingAvailable() noexcept;
 struct OpenXRFrameTiming {
     float headset_hz = 0;
     float rendered_fps = 0; // Newly rendered pairs; excludes retained-layer repeats.

runtime/include/vr/openxr_runtime.h

@@ -209,6 +209,9 @@ public:
     XrSpace ViewSpace() const { return m_view_space; }
     XrSessionState SessionState() const { return m_session_state; }
     uint64_t SessionRunSerial() const { return m_session_run_serial; }
+    // Advances each time the runtime reports that a hand's interaction profile
+    // changed (a controller picked up or put down).
+    uint64_t InteractionProfileSerial() const { return m_interaction_profile_serial; }
     XrReferenceSpaceType AppSpaceType() const { return m_app_space_type; }
     XrEnvironmentBlendMode EnvironmentBlendMode() const { return m_blend_mode; }
 
@@ -281,6 +284,7 @@ private:
 
     FramePhase m_frame_phase = FramePhase::Idle;
     uint64_t m_session_run_serial = 0;
+    uint64_t m_interaction_profile_serial = 0;
     uint64_t m_next_frame_serial = 1;
     uint64_t m_active_frame_serial = 0;
     XrTime m_active_frame_display_time = 0;