Implement passthrough feature for menu screens in OpenXR

runtime/include/runtime_config.h

@@ -57,6 +57,7 @@ struct RuntimeUserConfig {
     std::optional<float> vrHudDistanceMeters;
     std::optional<float> vrHudWidthMeters;
     std::optional<bool> vrHudVirtualScreen;
+    std::optional<bool> vrPassthrough;
     std::optional<bool> vrStopAtDisplayCopy;
     std::optional<bool> vrSkipCopyClears;
     std::optional<std::string> vrMirrorView;
@@ -743,6 +744,7 @@ inline RuntimeUserConfig ParseConfigDocument(const toml::value& document) {
         config.vrHudWidthMeters = *value;
     }
     config.vrHudVirtualScreen = FindConfigValue<bool>(document, "vr", "hud_virtual_screen");
+    config.vrPassthrough = FindConfigValue<bool>(document, "vr", "passthrough");
     config.vrStopAtDisplayCopy = FindConfigValue<bool>(document, "vr", "stop_at_display_copy");
     config.vrSkipCopyClears = FindConfigValue<bool>(document, "vr", "skip_copy_clears");
     config.vrFirstPerson = FindConfigValue<bool>(document, "vr", "first_person");
@@ -1056,6 +1058,11 @@ inline bool SetVrHudVirtualScreen(bool value) {
     return WriteSetting("vr", "hud_virtual_screen", value ? "true" : "false");
 }
 
+inline bool SetVrPassthrough(bool value) {
+    Mutable().vrPassthrough = value;
+    return WriteSetting("vr", "passthrough", value ? "true" : "false");
+}
+
 inline bool SetVrStopAtDisplayCopy(bool value) {
     Mutable().vrStopAtDisplayCopy = value;
     return WriteSetting("vr", "stop_at_display_copy", value ? "true" : "false");
@@ -1508,6 +1515,13 @@ inline bool VrHudVirtualScreen(bool fallback = true) {
     return Get().vrHudVirtualScreen.value_or(fallback);
 }
 
+// The room, through the headset's cameras, around the menu screen and every
+// other virtual screen (never in a race). Only the Quest offers it; the
+// launcher's Settings page shows the same default.
+inline bool VrPassthrough(bool fallback = true) {
+    return Get().vrPassthrough.value_or(fallback);
+}
+
 inline bool VrStopAtDisplayCopy(bool fallback = true) {
     return Get().vrStopAtDisplayCopy.value_or(fallback);
 }

runtime/include/vr/openxr_backend.h

@@ -7,7 +7,9 @@
 #include "vr/openxr_runtime.h"
 #include "vr/openxr_settings_panel.h"
 
+#include <algorithm>
 #include <array>
+#include <cmath>
 #include <cstdint>
 
 namespace mkw::vr {
@@ -81,12 +83,46 @@ inline XrCompositionLayerQuad OpenXRPanelQuadLayer(const OpenXRPanelLayer& panel
     return quad;
 }
 
+// The part of the virtual screen's image that holds anything. Aurora
+// letterboxes the desktop snapshot into that eye-sized image exactly like this
+// (webgpu::calculate_present_viewport_for_aspect) and, when the settings panel
+// is drawn into the eyes, centres it at kSettingsPanelWidthFraction of the
+// width; the rest is black. An unknown content_aspect (0) keeps the whole image.
+inline XrRect2Di OpenXRVirtualScreenContentRect(uint32_t width, uint32_t height, float content_aspect) noexcept {
+    XrRect2Di rect{{0, 0}, {static_cast<int32_t>(width), static_cast<int32_t>(height)}};
+    if (width == 0 || height == 0 || !(content_aspect > 0.0f)) {
+        return rect;
+    }
+    uint32_t content_width = width;
+    uint32_t content_height = std::min<uint32_t>(
+        height, std::max<uint32_t>(1u, static_cast<uint32_t>(std::lround(
+                                           static_cast<double>(width) * static_cast<double>(1.0f / content_aspect)))));
+    if (content_height == height) {
+        content_width = std::min<uint32_t>(
+            width, std::max<uint32_t>(1u, static_cast<uint32_t>(std::lround(static_cast<double>(height) *
+                                                                            static_cast<double>(content_aspect)))));
+    }
+    const uint32_t panel_width = std::min<uint32_t>(
+        width, static_cast<uint32_t>(std::ceil(static_cast<double>(width) * kSettingsPanelWidthFraction)));
+    const uint32_t panel_height = std::min<uint32_t>(
+        height, static_cast<uint32_t>(std::ceil(static_cast<double>(panel_width) * kSettingsPanelHeightPixels /
+                                                kSettingsPanelWidthPixels)));
+    const uint32_t shown_width = std::max(content_width, panel_width);
+    const uint32_t shown_height = std::max(content_height, panel_height);
+    rect.offset = {static_cast<int32_t>((width - shown_width) / 2), static_cast<int32_t>((height - shown_height) / 2)};
+    rect.extent = {static_cast<int32_t>(shown_width), static_cast<int32_t>(shown_height)};
+    return rect;
+}
+
 struct OpenXRPresentation {
     OpenXRFrameMode mode = OpenXRFrameMode::ImmersiveProjection;
 
     // Used only by VirtualScreen.
     float quad_distance_meters = 2.0f;
     float quad_width_meters = 2.4f;
+    // The desktop snapshot's width over height, which Aurora letterboxes into
+    // the screen's image (see OpenXRVirtualScreenContentRect); 0 while unknown.
+    float quad_content_aspect = 0.0f;
 
     // When quad_anchored is set, the quad is placed at quad_pose in the
     // application reference space and stays put as the player looks around.
@@ -96,6 +132,11 @@ struct OpenXRPresentation {
     bool quad_anchored = false;
     XrPosef quad_pose{{0.0f, 0.0f, 0.0f, 1.0f}, {0.0f, 0.0f, 0.0f}};
 
+    // Show the room through the headset's cameras around the virtual screen
+    // (OpenXRPassthrough). Taken when the presentation is handed to the backend,
+    // which starts or pauses the view then; a backend without one ignores it.
+    bool passthrough = false;
+
     OpenXRPanelLayer panel;
 };
 

runtime/include/vr/openxr_integration.h

@@ -53,6 +53,12 @@ void OpenXRRequestRecenter() noexcept;
 // once per published frame.
 void OpenXRSetLeanBackDegrees(float degrees) noexcept;
 
+// Shows the room through the headset's cameras around the menu screen and every
+// other virtual screen, never during an immersive race. Only the standalone
+// (Quest) backend offers it; elsewhere this changes nothing. Callable from any
+// thread; applied on the XR pacing thread's next frame.
+void OpenXRSetPassthrough(bool enabled) noexcept;
+
 // Live scene interpolation at the headset's own display deadlines.
 // 0 = Off, 1 = Auto, otherwise 72/90/120 as a rendering-rate ceiling.
 void OpenXRSetFrameInterpolationFps(uint32_t target) noexcept;

runtime/include/vr/openxr_passthrough.h

@@ -0,0 +1,62 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#pragma once
+
+#if defined(MKW_ENABLE_OPENXR)
+
+#include "vr/openxr_runtime.h"
+
+#include <string_view>
+
+namespace mkw::vr {
+
+// The headset's camera view of the room (XR_FB_passthrough) around the virtual
+// screen, as PPSSPP VR and DolphinXR show it: one reconstruction layer,
+// submitted before every other layer so they all cover it, with the frame's
+// blend mode left OPAQUE. Its objects are created the first time the view is
+// wanted and paused whenever it is not, so a race never keeps the cameras
+// running. On a Quest the app must also declare com.oculus.feature.PASSTHROUGH
+// in its manifest, or the runtime accepts every call and composites nothing.
+//
+// Not synchronized: every call belongs to the thread that ends the session's
+// frames, while the session exists. The handles are the session's children:
+// Destroy() before xrDestroySession, which otherwise frees them itself.
+class OpenXRPassthrough final {
+public:
+    explicit OpenXRPassthrough(OpenXRLogCallback logger = {});
+
+    // Starts (creating it on first use) or pauses the view. A runtime without
+    // XR_FB_passthrough, or one that refuses it, is logged once and leaves the
+    // view off until Destroy().
+    void SetRunning(OpenXRRuntime& runtime, bool running);
+
+    // The layer to submit first while the view runs, otherwise null.
+    const XrCompositionLayerBaseHeader* Layer() const noexcept;
+
+    void Destroy() noexcept;
+
+private:
+    bool Create(OpenXRRuntime& runtime);
+    // gives_up: the failure turns the view off until Destroy().
+    void LogResult(const OpenXRRuntime& runtime, std::string_view operation, XrResult result, bool gives_up) const;
+    void Log(OpenXRLogLevel level, std::string_view message) const noexcept;
+
+    OpenXRLogCallback m_logger;
+    PFN_xrCreatePassthroughFB m_create_passthrough = nullptr;
+    PFN_xrDestroyPassthroughFB m_destroy_passthrough = nullptr;
+    PFN_xrPassthroughStartFB m_start_passthrough = nullptr;
+    PFN_xrPassthroughPauseFB m_pause_passthrough = nullptr;
+    PFN_xrCreatePassthroughLayerFB m_create_layer = nullptr;
+    PFN_xrDestroyPassthroughLayerFB m_destroy_layer = nullptr;
+    PFN_xrPassthroughLayerResumeFB m_resume_layer = nullptr;
+    PFN_xrPassthroughLayerPauseFB m_pause_layer = nullptr;
+    XrPassthroughFB m_passthrough = XR_NULL_HANDLE;
+    XrPassthroughLayerFB m_layer = XR_NULL_HANDLE;
+    XrCompositionLayerPassthroughFB m_composition{XR_TYPE_COMPOSITION_LAYER_PASSTHROUGH_FB};
+    bool m_running = false;
+    bool m_failed = false;
+};
+
+} // namespace mkw::vr
+
+#endif // defined(MKW_ENABLE_OPENXR)