Implement OpenXR Settings Panel and Stereo Overlay

runtime/src/vr/openxr_input.cpp

@@ -40,8 +40,9 @@ namespace {
 // A new sequence number holds the button for kInjectHoldFrames XR frames.
 // Buttons: a, b, x, y, start, up, down, left, right, and for the Wii Remote
 // presentation also home, c and z (x/y/start press 1/2/+ there, and the
-// directions push the Nunchuk stick). The property is unset in normal use, so
-// this costs one property read every few frames.
+// directions push the Nunchuk stick). `panel` clicks both thumbsticks, opening
+// or closing the settings panel, where `a` then selects. The property is unset
+// in normal use, so this costs one property read every few frames.
 constexpr uint32_t kInjectHoldFrames = 12;
 constexpr uint32_t kInjectPollFrames = 4;
 
@@ -468,6 +469,9 @@ void OpenXRInput::Destroy() {
     }
     m_pointer.Reset();
     m_horizon = {1.0f, 0.0f};
+    m_panel_controls.Reset();
+    m_last_input_time = 0;
+    m_panel_select_held = false;
     m_created = false;
     m_runtime = nullptr;
 }
@@ -482,6 +486,11 @@ void OpenXRInput::Idle() {
     m_pointer.Reset();
     m_horizon = {1.0f, 0.0f};
     OpenXRPublishWiiRemote(m_joystick_id, OpenXRWiiRemoteSample{});
+    // The panel stays as it was; only what the controllers were holding is forgotten.
+    m_panel_controls.Reset();
+    m_last_input_time = 0;
+    m_panel_select_held = false;
+    OpenXRPublishSettingsPanelPointer(false, 0.0f, 0.0f, false, 0.0f);
     StopRumble();
     // Nothing stays held on the gamepad either while input is away.
     if (m_joystick != nullptr) {
@@ -495,7 +504,8 @@ void OpenXRInput::Idle() {
     }
 }
 
-void OpenXRInput::Sync(XrTime predicted_display_time, const OpenXRPointerScreen& screen) {
+void OpenXRInput::Sync(XrTime predicted_display_time, const OpenXRPointerScreen& screen,
+                       const OpenXRPointerScreen& settings_panel) {
     if (!m_created || m_runtime == nullptr) {
         return;
     }
@@ -566,18 +576,48 @@ void OpenXRInput::Sync(XrTime predicted_display_time, const OpenXRPointerScreen&
     }
 
     PollInjection();
-    wii_remote::HandInputs& left = hands[0];
-    const wii_remote::HandInputs& right = hands[1];
     if (Injected("up")) {
-        left.stick_y = 1.0f;
+        hands[0].stick_y = 1.0f;
     } else if (Injected("down")) {
-        left.stick_y = -1.0f;
+        hands[0].stick_y = -1.0f;
     } else if (Injected("left")) {
-        left.stick_x = -1.0f;
+        hands[0].stick_x = -1.0f;
     } else if (Injected("right")) {
-        left.stick_x = 1.0f;
+        hands[0].stick_x = 1.0f;
     }
 
+    const XrTime input_time = InputSampleTime(predicted_display_time);
+    const float dt_seconds =
+        m_last_input_time != 0 && input_time > m_last_input_time
+            ? static_cast<float>(input_time - m_last_input_time) * 1.0e-9f
+            : 0.0f;
+    m_last_input_time = input_time;
+    std::array<wii_remote::HandInputs, kHands> panel_hands = hands;
+    if (Injected("panel")) {
+        panel_hands[0].thumbstick_click = true;
+        panel_hands[1].thumbstick_click = true;
+    }
+    if (Injected("a")) {
+        panel_hands[1].primary = true;
+    }
+    // The game thread may open or close the panel too; only a change made here
+    // is written back.
+    const bool was_open = OpenXRSettingsPanelOpen();
+    bool open = was_open;
+    const settings_panel::Frame panel = m_panel_controls.Update(panel_hands, open, dt_seconds);
+    // Pointer first: the game thread reads it as soon as it sees the panel open.
+    PublishSettingsPanel(input_time, settings_panel, panel);
+    if (open != was_open) {
+        OpenXRSetSettingsPanelOpen(open);
+    }
+
+    // While the panel has the controllers, the game sees them idle.
+    static const std::array<wii_remote::HandInputs, kHands> kIdleHands{};
+    const auto& game_hands = panel.withheld ? kIdleHands : hands;
+    const wii_remote::HandInputs& left = game_hands[0];
+    const wii_remote::HandInputs& right = game_hands[1];
+    const auto injected = [&panel](const char* button) { return !panel.withheld && Injected(button); };
+
     if (m_joystick != nullptr) {
         auto* joystick = static_cast<SDL_Joystick*>(m_joystick);
         // OpenXR thumbsticks report +Y up; SDL gamepads report +Y down.
@@ -588,27 +628,65 @@ void OpenXRInput::Sync(XrTime predicted_display_time, const OpenXRPointerScreen&
         SDL_SetJoystickVirtualAxis(joystick, SDL_GAMEPAD_AXIS_LEFT_TRIGGER, ToTrigger(left.trigger));
         SDL_SetJoystickVirtualAxis(joystick, SDL_GAMEPAD_AXIS_RIGHT_TRIGGER, ToTrigger(right.trigger));
 
-        SDL_SetJoystickVirtualButton(joystick, SDL_GAMEPAD_BUTTON_SOUTH, right.primary || Injected("a"));
-        SDL_SetJoystickVirtualButton(joystick, SDL_GAMEPAD_BUTTON_EAST, right.secondary || Injected("b"));
-        SDL_SetJoystickVirtualButton(joystick, SDL_GAMEPAD_BUTTON_WEST, left.primary || Injected("x"));
-        SDL_SetJoystickVirtualButton(joystick, SDL_GAMEPAD_BUTTON_NORTH, left.secondary || Injected("y"));
-        SDL_SetJoystickVirtualButton(joystick, SDL_GAMEPAD_BUTTON_START, left.menu || Injected("start"));
+        SDL_SetJoystickVirtualButton(joystick, SDL_GAMEPAD_BUTTON_SOUTH, right.primary || injected("a"));
+        SDL_SetJoystickVirtualButton(joystick, SDL_GAMEPAD_BUTTON_EAST, right.secondary || injected("b"));
+        SDL_SetJoystickVirtualButton(joystick, SDL_GAMEPAD_BUTTON_WEST, left.primary || injected("x"));
+        SDL_SetJoystickVirtualButton(joystick, SDL_GAMEPAD_BUTTON_NORTH, left.secondary || injected("y"));
+        SDL_SetJoystickVirtualButton(joystick, SDL_GAMEPAD_BUTTON_START, left.menu || injected("start"));
         SDL_SetJoystickVirtualButton(joystick, SDL_GAMEPAD_BUTTON_LEFT_STICK, left.thumbstick_click);
         SDL_SetJoystickVirtualButton(joystick, SDL_GAMEPAD_BUTTON_RIGHT_STICK, right.thumbstick_click);
         SDL_SetJoystickVirtualButton(joystick, SDL_GAMEPAD_BUTTON_LEFT_SHOULDER, left.squeeze > 0.5f);
         SDL_SetJoystickVirtualButton(joystick, SDL_GAMEPAD_BUTTON_RIGHT_SHOULDER, right.squeeze > 0.5f);
     }
 
-    PublishWiiRemote(predicted_display_time, screen, hands, InjectedWiiRemoteButtons());
+    PublishWiiRemote(input_time, screen, game_hands, panel.withheld ? 0u : InjectedWiiRemoteButtons(),
+                     panel.withheld);
     UpdateRumble();
 }
 
-void OpenXRInput::PublishWiiRemote(XrTime predicted_display_time, const OpenXRPointerScreen& screen,
+// The pointing hand's aim ray against the whole panel, in canvas pixels, with a
+// short tick in that hand when a selection starts.
+void OpenXRInput::PublishSettingsPanel(XrTime input_time, const OpenXRPointerScreen& panel,
+                                       const settings_panel::Frame& frame) {
+    if (!frame.open) {
+        // Nothing may still read as held when the panel next opens.
+        m_panel_select_held = false;
+        OpenXRPublishSettingsPanelPointer(false, 0.0f, 0.0f, false, 0.0f);
+        return;
+    }
+    constexpr XrSpaceLocationFlags kPoseValid =
+        XR_SPACE_LOCATION_POSITION_VALID_BIT | XR_SPACE_LOCATION_ORIENTATION_VALID_BIT;
+    bool valid = false;
+    std::array<float, 2> point{};
+    const XrSpace aim_space = m_aim_spaces[frame.pointing_hand];
+    if (panel.valid && aim_space != XR_NULL_HANDLE) {
+        XrSpaceLocation location{XR_TYPE_SPACE_LOCATION};
+        if (XR_SUCCEEDED(xrLocateSpace(aim_space, m_runtime->AppSpace(), input_time, &location)) &&
+            (location.locationFlags & kPoseValid) == kPoseValid) {
+            wii_remote::Screen target{};
+            target.pose = ToWiiRemotePose(panel.pose);
+            target.half_width = panel.half_width_meters;
+            target.half_height = panel.half_height_meters;
+            const wii_remote::ScreenHit hit = wii_remote::RaycastScreen(ToWiiRemotePose(location.pose), target);
+            if (hit.valid) {
+                valid = true;
+                point = settings_panel::CanvasPoint(hit);
+            }
+        }
+    }
+    if (frame.select && !m_panel_select_held) {
+        constexpr XrDuration kTickNs = 15'000'000;
+        ApplyHaptic(frame.pointing_hand, 0.35f, kTickNs);
+    }
+    m_panel_select_held = frame.select;
+    OpenXRPublishSettingsPanelPointer(valid, point[0], point[1], frame.select, frame.wheel);
+}
+
+void OpenXRInput::PublishWiiRemote(XrTime input_time, const OpenXRPointerScreen& screen,
                                    const std::array<wii_remote::HandInputs, kHands>& hands,
-                                   uint32_t injected_buttons) {
+                                   uint32_t injected_buttons, bool withheld) {
     constexpr XrSpaceLocationFlags kPoseValid =
         XR_SPACE_LOCATION_POSITION_VALID_BIT | XR_SPACE_LOCATION_ORIENTATION_VALID_BIT;
-    const XrTime input_time = InputSampleTime(predicted_display_time);
 
     OpenXRWiiRemoteSample sample{};
     sample.hold = wii_remote::RemoteButtons(hands[0], hands[1]) | injected_buttons;
@@ -648,6 +726,15 @@ void OpenXRInput::PublishWiiRemote(XrTime predicted_display_time, const OpenXRPo
         (hand == 0 ? sample.nunchuk_acc : sample.acc) = acc;
     }
 
+    if (withheld) {
+        // Waving a controller around the panel must not trick or wheelie.
+        sample.acc = OpenXRWiiRemoteSample{}.acc;
+        sample.nunchuk_acc = OpenXRWiiRemoteSample{}.nunchuk_acc;
+        m_pointer.Reset();
+        OpenXRPublishWiiRemote(m_joystick_id, sample);
+        return;
+    }
+
     wii_remote::Screen target{};
     wii_remote::ScreenHit hit{};
     if (screen.valid && aim_valid[1]) {

runtime/src/vr/openxr_integration.cpp

@@ -741,7 +741,8 @@ private:
             if (input_ != nullptr) {
                 // After the screen is placed, so the pointer aims at this
                 // frame's screen rather than the previous one's.
-                input_->Sync(frame.xr_frame.predicted_display_time, PointerScreen(frame, policy, immersive));
+                input_->Sync(frame.xr_frame.predicted_display_time, PointerScreen(frame, policy, immersive),
+                             SettingsPanelScreen(frame, policy, immersive));
             }
 
             if (!frame.expects_gpu_submission) {
@@ -951,55 +952,19 @@ private:
         if (!aurora_get_stereo_screen_aspects(&picture_aspect, &snapshot_aspect)) {
             return screen;
         }
-        const OpenXRFrame& xr_frame = frame.xr_frame;
-        const bool views_usable = xr_frame.views_valid &&
-                                  (xr_frame.view_state_flags & XR_VIEW_STATE_ORIENTATION_VALID_BIT) != 0;
-        const bool position_usable =
-            views_usable && (xr_frame.view_state_flags & XR_VIEW_STATE_POSITION_VALID_BIT) != 0;
 
         if (immersive) {
-            const float distance = policy.config.hud_distance_meters;
-            const float width = policy.config.hud_width_meters;
-            if (!aurora_get_stereo_hud_screen_enabled() || !(distance > 0.0f) || !(width > 0.0f)) {
+            if (!aurora_get_stereo_hud_screen_enabled() || !(policy.config.hud_width_meters > 0.0f) ||
+                !RaceScreenPose(frame, policy, screen.pose)) {
                 return screen;
             }
-            std::array<float, 3> base{};
-            if (base_position_valid_ && last_immersive_) {
-                base = base_position_;
-            } else if (position_usable) {
-                // BuildPublishedFrame latches exactly this for the frame.
-                base = CenterPosition(xr_frame);
-            } else {
-                return screen;
-            }
-            const float half_angle =
-                0.5f * lean_back_degrees_.load(std::memory_order_relaxed) * kDegreesToRadians;
-            const Quaternion lean{std::sin(half_angle), 0.0f, 0.0f, std::cos(half_angle)};
-            const std::array<float, 3> ahead = Rotate(lean, {0.0f, 0.0f, -distance});
-            screen.pose.orientation = {lean.x, lean.y, lean.z, lean.w};
-            screen.pose.position = {base[0] + ahead[0], base[1] + ahead[1], base[2] + ahead[2]};
-            screen.half_width_meters = 0.5f * width;
+            screen.half_width_meters = 0.5f * policy.config.hud_width_meters;
             screen.half_height_meters = screen.half_width_meters / picture_aspect;
             screen.valid = true;
             return screen;
         }
 
-        if (frame.presentation.mode != OpenXRFrameMode::VirtualScreen || frame.render_width[0] == 0 ||
-            frame.render_height[0] == 0) {
-            return screen;
-        }
-        if (frame.presentation.quad_anchored) {
-            screen.pose = frame.presentation.quad_pose;
-        } else if (position_usable) {
-            // Head-locked in the view space: straight ahead of the head.
-            const auto& head = xr_frame.views[0].pose.orientation;
-            const Quaternion orientation = Normalize({head.x, head.y, head.z, head.w});
-            const std::array<float, 3> center = CenterPosition(xr_frame);
-            const std::array<float, 3> ahead = Rotate(
-                orientation, {0.0f, 0.0f, -std::max(0.25f, frame.presentation.quad_distance_meters)});
-            screen.pose.orientation = {orientation.x, orientation.y, orientation.z, orientation.w};
-            screen.pose.position = {center[0] + ahead[0], center[1] + ahead[1], center[2] + ahead[2]};
-        } else {
+        if (frame.render_width[0] == 0 || frame.render_height[0] == 0 || !MenuScreenPose(frame, screen.pose)) {
             return screen;
         }
         const float eye_aspect =
@@ -1012,6 +977,81 @@ private:
         return screen;
     }
 
+    // The settings panel's rectangle: centred on the same screen, a fixed
+    // fraction of its width, the way Aurora lays it over the eyes
+    // (aurora_imgui_set_stereo_overlay). Unlike the pointer's picture it is
+    // there in a race even when the 2D layer is not on the screen.
+    OpenXRPointerScreen SettingsPanelScreen(const OpenXRBackendFrame& frame, const MkwVRPolicySnapshot& policy,
+                                            bool immersive) const noexcept {
+        OpenXRPointerScreen screen{};
+        const float screen_width =
+            immersive ? policy.config.hud_width_meters : std::max(0.25f, frame.presentation.quad_width_meters);
+        if (!(screen_width > 0.0f) ||
+            !(immersive ? RaceScreenPose(frame, policy, screen.pose) : MenuScreenPose(frame, screen.pose))) {
+            return screen;
+        }
+        const std::array<float, 2> extents = settings_panel::HalfExtents(screen_width);
+        screen.half_width_meters = extents[0];
+        screen.half_height_meters = extents[1];
+        screen.valid = true;
+        return screen;
+    }
+
+    // Centre of the race's 2D screen. ViewFromBase maps a point p of the
+    // recorded centre-eye space (in metres) to base + lean * p in the
+    // application space, so the screen Aurora hangs hud_distance_meters ahead
+    // sits at base + lean * (0, 0, -distance), turned by the lean.
+    bool RaceScreenPose(const OpenXRBackendFrame& frame, const MkwVRPolicySnapshot& policy,
+                        XrPosef& pose) const noexcept {
+        const OpenXRFrame& xr_frame = frame.xr_frame;
+        const float distance = policy.config.hud_distance_meters;
+        if (!(distance > 0.0f)) {
+            return false;
+        }
+        std::array<float, 3> base{};
+        if (base_position_valid_ && last_immersive_) {
+            base = base_position_;
+        } else if (xr_frame.views_valid &&
+                   (xr_frame.view_state_flags & XR_VIEW_STATE_ORIENTATION_VALID_BIT) != 0 &&
+                   (xr_frame.view_state_flags & XR_VIEW_STATE_POSITION_VALID_BIT) != 0) {
+            // BuildPublishedFrame latches exactly this for the frame.
+            base = CenterPosition(xr_frame);
+        } else {
+            return false;
+        }
+        const float half_angle = 0.5f * lean_back_degrees_.load(std::memory_order_relaxed) * kDegreesToRadians;
+        const Quaternion lean{std::sin(half_angle), 0.0f, 0.0f, std::cos(half_angle)};
+        const std::array<float, 3> ahead = Rotate(lean, {0.0f, 0.0f, -distance});
+        pose.orientation = {lean.x, lean.y, lean.z, lean.w};
+        pose.position = {base[0] + ahead[0], base[1] + ahead[1], base[2] + ahead[2]};
+        return true;
+    }
+
+    // Centre of the menu quad: where UpdateVirtualScreenPose anchored it, or
+    // its head-locked fallback straight ahead of the head.
+    bool MenuScreenPose(const OpenXRBackendFrame& frame, XrPosef& pose) const noexcept {
+        if (frame.presentation.mode != OpenXRFrameMode::VirtualScreen) {
+            return false;
+        }
+        if (frame.presentation.quad_anchored) {
+            pose = frame.presentation.quad_pose;
+            return true;
+        }
+        const OpenXRFrame& xr_frame = frame.xr_frame;
+        if (!xr_frame.views_valid || (xr_frame.view_state_flags & XR_VIEW_STATE_ORIENTATION_VALID_BIT) == 0 ||
+            (xr_frame.view_state_flags & XR_VIEW_STATE_POSITION_VALID_BIT) == 0) {
+            return false;
+        }
+        const auto& head = xr_frame.views[0].pose.orientation;
+        const Quaternion orientation = Normalize({head.x, head.y, head.z, head.w});
+        const std::array<float, 3> center = CenterPosition(xr_frame);
+        const std::array<float, 3> ahead =
+            Rotate(orientation, {0.0f, 0.0f, -std::max(0.25f, frame.presentation.quad_distance_meters)});
+        pose.orientation = {orientation.x, orientation.y, orientation.z, orientation.w};
+        pose.position = {center[0] + ahead[0], center[1] + ahead[1], center[2] + ahead[2]};
+        return true;
+    }
+
     void ApplyPendingReferenceSpaceChange(const OpenXRFrame& frame) noexcept {
         if (runtime_->ConsumeAppSpaceChangesThrough(frame.predicted_display_time)) {
             ResetTrackingOrigin();

runtime/src/vr/openxr_settings_panel.cpp

@@ -0,0 +1,52 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "vr/openxr_settings_panel.h"
+
+#include <atomic>
+#include <mutex>
+
+namespace mkw::vr {
+// Named rather than anonymous: runtime sources are unity-built in groups.
+namespace settings_panel_bridge {
+
+std::atomic_bool g_open{false};
+
+struct PublishedPointer {
+    std::mutex mutex;
+    OpenXRSettingsPanelPointer pointer{};
+};
+
+PublishedPointer& Published() {
+    static PublishedPointer published;
+    return published;
+}
+
+} // namespace settings_panel_bridge
+
+void OpenXRSetSettingsPanelOpen(bool open) noexcept {
+    settings_panel_bridge::g_open.store(open, std::memory_order_release);
+}
+
+bool OpenXRSettingsPanelOpen() noexcept {
+    return settings_panel_bridge::g_open.load(std::memory_order_acquire);
+}
+
+void OpenXRPublishSettingsPanelPointer(bool valid, float x, float y, bool select, float wheel) noexcept {
+    auto& published = settings_panel_bridge::Published();
+    std::lock_guard lock(published.mutex);
+    published.pointer.valid = valid;
+    published.pointer.x = x;
+    published.pointer.y = y;
+    published.pointer.select = select;
+    published.pointer.wheel += wheel;
+}
+
+OpenXRSettingsPanelPointer OpenXRTakeSettingsPanelPointer() noexcept {
+    auto& published = settings_panel_bridge::Published();
+    std::lock_guard lock(published.mutex);
+    OpenXRSettingsPanelPointer pointer = published.pointer;
+    published.pointer.wheel = 0.0f;
+    return pointer;
+}
+
+} // namespace mkw::vr