Turn the cockpit wheel with the steering, and add hand steering

Every XR frame in the cockpit seat the pacing thread locates both grips in
the seated frame (the immersive base turned by the lean-back angle, the
frame the eye transforms place the vehicle in) and publishes a driving
snapshot. The wheel or handlebar shows the left stick's steering at the
configured full-lock angle, eased; the vehicle's own wheel reads that angle
on the guest thread.

With hand_steering on (off by default), squeezing a grip near the wheel or
handlebar takes hold of it (a short pulse on grab and release); one or two
hands turn it through heurazy's SteeringWheel, and while it is held the
wheel replaces the left stick's X axis in both the Wii Remote and gamepad
presentations, the stick's Y still aims items, and a holding grip no
longer presses C or a shoulder. The settings panel withholds it like any
other input.

The stereo packet carries the cockpit overlay: the hands, in the runtime's
hand mesh (XR_EXT_hand_tracking + XR_FB_hand_tracking_mesh, requested only
when hand steering is on at launch) or procedural gloves, and the separate
VR wheel or handlebar whenever the vehicle's own is not the one turning.

runtime/include/vr/openxr_driving.h

@@ -50,6 +50,12 @@ struct DrivingSnapshot {
     // clockwise as the driver sees it, i.e. to the right.
     float visual_angle = 0.0f;
     std::array<DrivingHand, 2> hands{};
+    // What the cockpit overlay draws: a separate VR wheel or handlebar when the
+    // vehicle's own is not the one turning. `control` places the handlebar
+    // (and, when its geometry is valid, is what the hands reach for).
+    bool synthetic_control = false;
+    bool bike = false;
+    WheelGeometry control{};
 };
 
 // Pacing thread publishes; any thread reads the latest. A default snapshot

runtime/include/vr/openxr_input.h

@@ -4,9 +4,11 @@
 
 #if defined(MKW_ENABLE_OPENXR)
 
+#include "vr/openxr_driving.h"
 #include "vr/openxr_runtime.h"
 #include "vr/openxr_settings_panel.h"
 #include "vr/openxr_wii_remote.h"
+#include "vr/steering_wheel.h"
 
 #include <array>
 #include <cstdint>
@@ -56,6 +58,12 @@ struct OpenXRPointerScreen {
 // Both are bound for the Oculus Touch profile; khr/simple_controller gets
 // select/menu and the poses so an unknown runtime still offers something.
 //
+// In the first-person cockpit (openxr_driving.h) the grip poses are located in
+// the seated frame every frame. With hand steering on, a squeezed grip near the
+// steering wheel or handlebar takes hold of it; while held, the wheel replaces
+// the left stick's X axis in both presentations and that grip no longer reaches
+// the game (C on the Nunchuk, a shoulder on the gamepad).
+//
 // Left Y (both thumbsticks clicked together as a gamepad) opens the in-headset
 // settings panel (openxr_settings_panel.h). While it is open, and until every
 // button has been released after it closes, the game sees idle controllers: the
@@ -83,9 +91,13 @@ public:
     // xrSyncActions + state reads, then publishes to the virtual gamepad and
     // the Wii Remote bridge. predicted_display_time is the frame's XrTime;
     // screen is where the Wii Remote pointer can land this frame, and
-    // settings_panel where the settings panel is (its whole rectangle).
+    // settings_panel where the settings panel is (its whole rectangle). seat is
+    // the immersive seated frame, invalid outside an immersive race.
     void Sync(XrTime predicted_display_time, const OpenXRPointerScreen& screen,
-              const OpenXRPointerScreen& settings_panel);
+              const OpenXRPointerScreen& settings_panel, const driving::SeatFrame& seat);
+
+    // The cockpit state the last Sync published (also OpenXRReadDriving()).
+    const DrivingSnapshot& Driving() const noexcept { return m_driving; }
 
     // Publishes a remote with nothing held, at rest and not pointing, and stops
     // the haptics, for frames without focused input.
@@ -116,6 +128,11 @@ private:
                           bool withheld);
     void PublishSettingsPanel(XrTime input_time, const OpenXRPointerScreen& panel,
                               const settings_panel::Frame& frame);
+    // Hand steering: locates the grips in the seated frame, runs the wheel and
+    // hands its steering to `hands` before the game sees them.
+    void UpdateDriving(XrTime display_time, const driving::SeatFrame& seat,
+                       std::array<wii_remote::HandInputs, kHands>& hands, bool withheld);
+    void ResetDriving();
     void UpdateRumble();
     void StopRumble();
     bool Check(XrResult result, const char* operation);
@@ -149,6 +166,14 @@ private:
     bool m_haptics_active[kHands]{};
     uint32_t m_joystick_id = 0; // SDL_JoystickID; 0 when detached
     void* m_joystick = nullptr; // SDL_Joystick*
+    SteeringWheel m_wheel;
+    WheelReferenceLatch m_wheel_reference;
+    driving::WheelVisual m_wheel_visual;
+    std::array<bool, kHands> m_wheel_held{};
+    XrTime m_wheel_time = 0;
+    bool m_wheel_uses_geometry = false;
+    bool m_wheel_bike = false;
+    DrivingSnapshot m_driving{};
     bool m_created = false;
     bool m_logged_sync_failure = false;
     bool m_logged_pointer = false;

runtime/src/vr/openxr_input.cpp

@@ -10,6 +10,8 @@
 #endif
 
 #include "vr/openxr_input.h"
+#include "runtime_config.h"
+#include "vr/mkw_vr_first_person.h"
 #include "vr/openxr_diagnostics.h"
 
 #include <SDL3/SDL_gamepad.h>
@@ -451,6 +453,7 @@ void OpenXRInput::DetachVirtualGamepad() {
 void OpenXRInput::Destroy() {
     // The game must stop reading a remote whose controllers are going away.
     OpenXRWithdrawWiiRemote();
+    ResetDriving();
     if (m_created) {
         StopRumble();
     }
@@ -493,6 +496,7 @@ void OpenXRInput::Idle() {
     m_last_input_time = 0;
     m_panel_select_held = false;
     OpenXRPublishSettingsPanelPointer(false, 0.0f, 0.0f, false, 0.0f);
+    ResetDriving();
     StopRumble();
     // Nothing stays held on the gamepad either while input is away.
     if (m_joystick != nullptr) {
@@ -507,7 +511,7 @@ void OpenXRInput::Idle() {
 }
 
 void OpenXRInput::Sync(XrTime predicted_display_time, const OpenXRPointerScreen& screen,
-                       const OpenXRPointerScreen& settings_panel) {
+                       const OpenXRPointerScreen& settings_panel, const driving::SeatFrame& seat) {
     if (!m_created || m_runtime == nullptr) {
         return;
     }
@@ -618,6 +622,10 @@ void OpenXRInput::Sync(XrTime predicted_display_time, const OpenXRPointerScreen&
         OpenXRSetSettingsPanelOpen(open);
     }
 
+    // The cockpit's wheel before the game reads the controllers: a held wheel
+    // steers through the left stick and keeps its grips from the game.
+    UpdateDriving(predicted_display_time, seat, hands, panel.withheld);
+
     // While the panel has the controllers, the game sees them idle.
     static const std::array<wii_remote::HandInputs, kHands> kIdleHands{};
     const auto& game_hands = panel.withheld ? kIdleHands : hands;
@@ -768,6 +776,117 @@ void OpenXRInput::PublishWiiRemote(XrTime input_time, const OpenXRPointerScreen&
     OpenXRPublishWiiRemote(m_joystick_id, sample);
 }
 
+void OpenXRInput::ResetDriving() {
+    m_wheel = {};
+    WheelGeometry unused{};
+    m_wheel_reference.Resolve(unused, false, false, false, false, 0, 0.0f);
+    m_wheel_visual.Reset();
+    m_wheel_held = {};
+    m_wheel_time = 0;
+    m_driving = {};
+    OpenXRPublishDriving(m_driving);
+}
+
+void OpenXRInput::UpdateDriving(XrTime display_time, const driving::SeatFrame& seat,
+                                std::array<wii_remote::HandInputs, kHands>& hands, bool withheld) {
+    const FirstPersonAnchor anchor = MkwVRFirstPersonGetAnchor();
+    if (!seat.valid || !anchor.valid || !anchor.cockpit) {
+        if (m_driving.cockpit_active || m_wheel_time != 0) {
+            ResetDriving();
+        }
+        return;
+    }
+    const WheelTuning tuning = RuntimeConfigFile::VrWheelTuning();
+    const bool hand_steering = RuntimeConfigFile::VrHandSteering();
+    const bool steering_wheel = RuntimeConfigFile::VrSteeringWheel();
+    const bool native_steering_wheel = RuntimeConfigFile::VrNativeSteeringWheel();
+    const float dt = m_wheel_time != 0 && display_time > m_wheel_time
+                         ? static_cast<float>(display_time - m_wheel_time) * 1.0e-9f
+                         : 1.0f / 90.0f;
+    m_wheel_time = display_time;
+
+    DrivingSnapshot snapshot{};
+    snapshot.cockpit_active = true;
+    snapshot.hand_steering = hand_steering;
+    snapshot.bike = anchor.bike;
+    // The vehicle's own control is the one turning (or none is shown at all),
+    // so the overlay adds no separate wheel.
+    snapshot.synthetic_control =
+        steering_wheel && !(native_steering_wheel && anchor.native_mesh_prepared);
+
+    // Which control the hands reach for: the vehicle's own wherever its
+    // geometry is known and no separate wheel is drawn, a handlebar always
+    // (held over a brief gap while gripped), otherwise the VR wheel in front
+    // of the seat.
+    WheelGeometry geometry = anchor.native_wheel;
+    const bool geometry_valid = m_wheel_reference.Resolve(geometry, true, geometry.valid,
+                                                          m_wheel_held[0] || m_wheel_held[1], anchor.bike,
+                                                          anchor.vehicle_identity, dt);
+    if (anchor.bike && !geometry_valid) {
+        geometry = {};
+        geometry.center = {0.0f, SteeringWheel::Height, SteeringWheel::Depth};
+        geometry.right = {1.0f, 0.0f, 0.0f};
+        geometry.up = {0.0f, 0.0f, -1.0f};
+        geometry.normal = {0.0f, 1.0f, 0.0f};
+        geometry.radius = 0.25f;
+        geometry.valid = true;
+    }
+    const bool uses_geometry = anchor.bike || (geometry_valid && !snapshot.synthetic_control);
+    if (uses_geometry != m_wheel_uses_geometry || anchor.bike != m_wheel_bike) {
+        m_wheel = {};
+        m_wheel_uses_geometry = uses_geometry;
+        m_wheel_bike = anchor.bike;
+    }
+    snapshot.control = geometry;
+
+    constexpr XrSpaceLocationFlags kPoseValid =
+        XR_SPACE_LOCATION_POSITION_VALID_BIT | XR_SPACE_LOCATION_ORIENTATION_VALID_BIT;
+    std::array<WheelHand, kHands> wheel_hands{};
+    for (uint32_t hand = 0; hand < kHands; ++hand) {
+        bool tracked = false;
+        std::array<float, 12> seat_from_grip = snapshot.hands[hand].seat_from_grip;
+        if (m_grip_spaces[hand] != XR_NULL_HANDLE) {
+            XrSpaceLocation location{XR_TYPE_SPACE_LOCATION};
+            if (XR_SUCCEEDED(xrLocateSpace(m_grip_spaces[hand], m_runtime->AppSpace(), display_time, &location)) &&
+                (location.locationFlags & kPoseValid) == kPoseValid) {
+                tracked = true;
+                const auto& pose = location.pose;
+                seat_from_grip = driving::SeatFromApp(seat, {pose.position.x, pose.position.y, pose.position.z},
+                                                      {pose.orientation.x, pose.orientation.y,
+                                                       pose.orientation.z, pose.orientation.w});
+            }
+        }
+        const float squeeze = hands[hand].squeeze;
+        // Hands are shown only while they can steer.
+        snapshot.hands[hand] = {tracked && hand_steering, false, squeeze, seat_from_grip};
+        wheel_hands[hand] = {seat_from_grip[3], seat_from_grip[7], seat_from_grip[11], squeeze, tracked};
+        if (uses_geometry) {
+            wheel_hands[hand] = geometry.ToWheel(wheel_hands[hand]);
+        }
+    }
+    const bool active = hand_steering && !withheld;
+    const WheelState wheel = m_wheel.Update(wheel_hands, active, dt,
+                                            uses_geometry ? geometry.radius : SteeringWheel::Radius,
+                                            anchor.bike, tuning);
+    for (uint32_t hand = 0; hand < kHands; ++hand) {
+        if (wheel.held[hand] != m_wheel_held[hand] && active && tuning.haptics) {
+            constexpr XrDuration kGrabPulseNs = 25'000'000;
+            constexpr XrDuration kReleasePulseNs = 15'000'000;
+            ApplyHaptic(hand, wheel.held[hand] ? 0.25f : 0.12f, wheel.held[hand] ? kGrabPulseNs : kReleasePulseNs);
+        }
+        snapshot.hands[hand].held = wheel.held[hand];
+    }
+    m_wheel_held = wheel.held;
+    snapshot.held = wheel.held;
+    driving::ApplyHandSteering(hands, wheel);
+    snapshot.steering_input = withheld ? 0.0f : hands[0].stick_x;
+    snapshot.visual_angle =
+        m_wheel_visual.Update(wheel.held[0] || wheel.held[1], wheel.visualAngle, snapshot.steering_input,
+                              driving::MaxWheelAngle(anchor.bike, tuning), dt);
+    m_driving = snapshot;
+    OpenXRPublishDriving(snapshot);
+}
+
 void OpenXRInput::UpdateRumble() {
     if (!OpenXRWiiRemoteRumbleRequested() || !OpenXRWiiRemoteOwnsGamepad(m_joystick_id)) {
         StopRumble();

runtime/src/vr/openxr_integration.cpp

@@ -31,6 +31,7 @@
 
 #if defined(MKW_ENABLE_OPENXR)
 #include "vr/openxr_backend.h"
+#include "vr/openxr_hand_mesh.h"
 #include "vr/openxr_input.h"
 #include "vr/openxr_runtime.h"
 #if defined(_WIN32)
@@ -180,6 +181,16 @@ XrPosef ScreenPoseAhead(const OpenXRFrame& frame, float distance) noexcept {
     return pose;
 }
 
+// Hand steering draws the player's hands, in the runtime's own hand mesh where
+// it offers one (XR_FB_hand_tracking_mesh). Only asked for when hand steering
+// is on at launch; turning it on later uses the procedural gloves.
+void AddHandMeshExtensions(OpenXRConfig& config) {
+    if (RuntimeConfigFile::VrHandSteering()) {
+        config.optional_extensions.push_back("XR_EXT_hand_tracking");
+        config.optional_extensions.push_back("XR_FB_hand_tracking_mesh");
+    }
+}
+
 void IdentityEye(AuroraStereoEye& eye) noexcept {
     std::fill(std::begin(eye.projection), std::end(eye.projection), 0.0f);
     eye.projection[0] = 1.0f;
@@ -375,6 +386,7 @@ public:
         config.required_extensions = {kRequiredAuroraBackend == BACKEND_VULKAN ? "XR_KHR_vulkan_enable2" : "XR_KHR_D3D12_enable"};
         config.optional_extensions = {"XR_KHR_win32_convert_performance_counter_time",
                                       "XR_FB_display_refresh_rate", "XR_EXT_performance_settings"};
+        AddHandMeshExtensions(config);
 #else
         // Either Vulkan binding extension is acceptable; the backend picks
         // whichever the runtime enabled, preferring enable2.
@@ -383,6 +395,7 @@ public:
                                       "XR_KHR_convert_timespec_time",
                                       "XR_KHR_android_thread_settings",
                                       "XR_FB_display_refresh_rate", "XR_EXT_performance_settings"};
+        AddHandMeshExtensions(config);
         config.instance_create_next = OpenXRAndroidInstanceCreateNext();
 #endif
         if (!runtime_->Initialize(config)) {
@@ -880,7 +893,7 @@ private:
                 // After the screen is placed, so the pointer aims at this
                 // frame's screen rather than the previous one's.
                 input_->Sync(frame.xr_frame.predicted_display_time, PointerScreen(frame, policy, immersive),
-                             SettingsPanelScreen(frame, policy, immersive));
+                             SettingsPanelScreen(frame, policy, immersive), InputSeatFrame(immersive));
             }
 
             if (!frame.expects_gpu_submission) {
@@ -1061,7 +1074,7 @@ private:
         if (input_ != nullptr) {
             const diagnostics::ScopedStage input_timer(diagnostics::Stage::InputSync);
             input_->Sync(packet.xr_frame.predicted_display_time, PointerScreen(packet, policy, immersive),
-                         SettingsPanelScreen(packet, policy, immersive));
+                         SettingsPanelScreen(packet, policy, immersive), InputSeatFrame(immersive));
         }
         if (!packet.expects_gpu_submission) {
             // Nothing to render (no rendering requested or no tracking): keep the compositor fed.
@@ -1228,6 +1241,61 @@ private:
                          position_valid && base_position_valid_, units_per_meter,
                          lean_back_radians, destination.eyes[eye].viewFromCenter);
         }
+        BuildCockpit(source, position_valid, units_per_meter, lean_back_radians, destination.cockpit);
+    }
+
+    // The first-person cockpit's hands and separate wheel, in the seated frame
+    // the eye transforms place at base + lean * seat (metres). Always carries
+    // the packet's world scale, which Aurora rescales to the sealed frame's.
+    void BuildCockpit(const OpenXRBackendFrame& source, bool position_valid, float units_per_meter,
+                      float lean_back_radians, AuroraCockpit& cockpit) noexcept {
+        cockpit.unitsPerMeter = units_per_meter;
+        const DrivingSnapshot driving = input_ != nullptr ? input_->Driving() : DrivingSnapshot{};
+        if (driving.hand_steering && !hand_meshes_loaded_ && runtime_ != nullptr) {
+            hand_meshes_loaded_ = true;
+            const bool loaded = LoadRuntimeHandMeshes(*runtime_);
+            RT_LOG(RT_TAG_RUNTIME) << "[mkw-vr] cockpit hands: "
+                                   << (loaded ? "the runtime's hand mesh" : "procedural gloves (no runtime hand mesh)")
+                                   << std::endl;
+        }
+        cockpit.active = driving.cockpit_active && position_valid && base_position_valid_ &&
+                         (driving.synthetic_control || driving.hand_steering);
+        if (!cockpit.active) {
+            return;
+        }
+        cockpit.wheelAngle = driving.visual_angle;
+        cockpit.nativeWheel = !driving.synthetic_control;
+        cockpit.bike = driving.bike;
+        cockpit.handlebarRadius = driving.control.radius;
+        for (int row = 0; row < 3; ++row) {
+            cockpit.seatFromHandlebar[row * 4 + 0] = driving.control.right[row];
+            cockpit.seatFromHandlebar[row * 4 + 1] = driving.control.up[row];
+            cockpit.seatFromHandlebar[row * 4 + 2] = driving.control.normal[row];
+            cockpit.seatFromHandlebar[row * 4 + 3] = driving.control.center[row];
+        }
+        for (uint32_t eye = 0; eye < kOpenXREyeCount; ++eye) {
+            ViewFromBase(source.xr_frame.views[eye].pose, base_position_, true, 1.0f, lean_back_radians,
+                         cockpit.eyeFromSeat[eye]);
+        }
+        for (size_t hand = 0; hand < 2; ++hand) {
+            auto& target = cockpit.hands[hand];
+            const auto& from = driving.hands[hand];
+            target.tracked = from.tracked;
+            target.held = from.held;
+            target.squeeze = from.squeeze;
+            std::copy(from.seat_from_grip.begin(), from.seat_from_grip.end(), target.seatFromGrip);
+        }
+    }
+
+    // The seated frame the controllers are located in for hand steering: the
+    // immersive base the eye transforms use, from the previous frame (this
+    // frame's is latched after input).
+    driving::SeatFrame InputSeatFrame(bool immersive) const noexcept {
+        driving::SeatFrame seat;
+        seat.valid = immersive && base_position_valid_ && last_immersive_;
+        seat.base = base_position_;
+        seat.lean_back_radians = lean_back_degrees_.load(std::memory_order_relaxed) * kDegreesToRadians;
+        return seat;
     }
 
     // Runs once per located frame, before the virtual screen is placed and
@@ -1569,6 +1637,7 @@ private:
     XrPosef virtual_screen_pose_{{0.0f, 0.0f, 0.0f, 1.0f}, {0.0f, 0.0f, 0.0f}};
     bool virtual_screen_pose_valid_ = false;
     bool last_immersive_ = false;
+    bool hand_meshes_loaded_ = false;
     uint64_t applied_session_run_serial_ = 0;
     bool session_was_active_ = false;
     bool requested_ = false;