Implement passthrough feature for menu screens in OpenXR

runtime/include/runtime_config.h

@@ -57,6 +57,7 @@ struct RuntimeUserConfig {
     std::optional<float> vrHudDistanceMeters;
     std::optional<float> vrHudWidthMeters;
     std::optional<bool> vrHudVirtualScreen;
+    std::optional<bool> vrPassthrough;
     std::optional<bool> vrStopAtDisplayCopy;
     std::optional<bool> vrSkipCopyClears;
     std::optional<std::string> vrMirrorView;
@@ -743,6 +744,7 @@ inline RuntimeUserConfig ParseConfigDocument(const toml::value& document) {
         config.vrHudWidthMeters = *value;
     }
     config.vrHudVirtualScreen = FindConfigValue<bool>(document, "vr", "hud_virtual_screen");
+    config.vrPassthrough = FindConfigValue<bool>(document, "vr", "passthrough");
     config.vrStopAtDisplayCopy = FindConfigValue<bool>(document, "vr", "stop_at_display_copy");
     config.vrSkipCopyClears = FindConfigValue<bool>(document, "vr", "skip_copy_clears");
     config.vrFirstPerson = FindConfigValue<bool>(document, "vr", "first_person");
@@ -1056,6 +1058,11 @@ inline bool SetVrHudVirtualScreen(bool value) {
     return WriteSetting("vr", "hud_virtual_screen", value ? "true" : "false");
 }
 
+inline bool SetVrPassthrough(bool value) {
+    Mutable().vrPassthrough = value;
+    return WriteSetting("vr", "passthrough", value ? "true" : "false");
+}
+
 inline bool SetVrStopAtDisplayCopy(bool value) {
     Mutable().vrStopAtDisplayCopy = value;
     return WriteSetting("vr", "stop_at_display_copy", value ? "true" : "false");
@@ -1508,6 +1515,13 @@ inline bool VrHudVirtualScreen(bool fallback = true) {
     return Get().vrHudVirtualScreen.value_or(fallback);
 }
 
+// The room, through the headset's cameras, around the menu screen and every
+// other virtual screen (never in a race). Only the Quest offers it; the
+// launcher's Settings page shows the same default.
+inline bool VrPassthrough(bool fallback = true) {
+    return Get().vrPassthrough.value_or(fallback);
+}
+
 inline bool VrStopAtDisplayCopy(bool fallback = true) {
     return Get().vrStopAtDisplayCopy.value_or(fallback);
 }

runtime/include/vr/openxr_backend.h

@@ -7,7 +7,9 @@
 #include "vr/openxr_runtime.h"
 #include "vr/openxr_settings_panel.h"
 
+#include <algorithm>
 #include <array>
+#include <cmath>
 #include <cstdint>
 
 namespace mkw::vr {
@@ -81,12 +83,46 @@ inline XrCompositionLayerQuad OpenXRPanelQuadLayer(const OpenXRPanelLayer& panel
     return quad;
 }
 
+// The part of the virtual screen's image that holds anything. Aurora
+// letterboxes the desktop snapshot into that eye-sized image exactly like this
+// (webgpu::calculate_present_viewport_for_aspect) and, when the settings panel
+// is drawn into the eyes, centres it at kSettingsPanelWidthFraction of the
+// width; the rest is black. An unknown content_aspect (0) keeps the whole image.
+inline XrRect2Di OpenXRVirtualScreenContentRect(uint32_t width, uint32_t height, float content_aspect) noexcept {
+    XrRect2Di rect{{0, 0}, {static_cast<int32_t>(width), static_cast<int32_t>(height)}};
+    if (width == 0 || height == 0 || !(content_aspect > 0.0f)) {
+        return rect;
+    }
+    uint32_t content_width = width;
+    uint32_t content_height = std::min<uint32_t>(
+        height, std::max<uint32_t>(1u, static_cast<uint32_t>(std::lround(
+                                           static_cast<double>(width) * static_cast<double>(1.0f / content_aspect)))));
+    if (content_height == height) {
+        content_width = std::min<uint32_t>(
+            width, std::max<uint32_t>(1u, static_cast<uint32_t>(std::lround(static_cast<double>(height) *
+                                                                            static_cast<double>(content_aspect)))));
+    }
+    const uint32_t panel_width = std::min<uint32_t>(
+        width, static_cast<uint32_t>(std::ceil(static_cast<double>(width) * kSettingsPanelWidthFraction)));
+    const uint32_t panel_height = std::min<uint32_t>(
+        height, static_cast<uint32_t>(std::ceil(static_cast<double>(panel_width) * kSettingsPanelHeightPixels /
+                                                kSettingsPanelWidthPixels)));
+    const uint32_t shown_width = std::max(content_width, panel_width);
+    const uint32_t shown_height = std::max(content_height, panel_height);
+    rect.offset = {static_cast<int32_t>((width - shown_width) / 2), static_cast<int32_t>((height - shown_height) / 2)};
+    rect.extent = {static_cast<int32_t>(shown_width), static_cast<int32_t>(shown_height)};
+    return rect;
+}
+
 struct OpenXRPresentation {
     OpenXRFrameMode mode = OpenXRFrameMode::ImmersiveProjection;
 
     // Used only by VirtualScreen.
     float quad_distance_meters = 2.0f;
     float quad_width_meters = 2.4f;
+    // The desktop snapshot's width over height, which Aurora letterboxes into
+    // the screen's image (see OpenXRVirtualScreenContentRect); 0 while unknown.
+    float quad_content_aspect = 0.0f;
 
     // When quad_anchored is set, the quad is placed at quad_pose in the
     // application reference space and stays put as the player looks around.
@@ -96,6 +132,11 @@ struct OpenXRPresentation {
     bool quad_anchored = false;
     XrPosef quad_pose{{0.0f, 0.0f, 0.0f, 1.0f}, {0.0f, 0.0f, 0.0f}};
 
+    // Show the room through the headset's cameras around the virtual screen
+    // (OpenXRPassthrough). Taken when the presentation is handed to the backend,
+    // which starts or pauses the view then; a backend without one ignores it.
+    bool passthrough = false;
+
     OpenXRPanelLayer panel;
 };
 

runtime/include/vr/openxr_integration.h

@@ -53,6 +53,12 @@ void OpenXRRequestRecenter() noexcept;
 // once per published frame.
 void OpenXRSetLeanBackDegrees(float degrees) noexcept;
 
+// Shows the room through the headset's cameras around the menu screen and every
+// other virtual screen, never during an immersive race. Only the standalone
+// (Quest) backend offers it; elsewhere this changes nothing. Callable from any
+// thread; applied on the XR pacing thread's next frame.
+void OpenXRSetPassthrough(bool enabled) noexcept;
+
 // Live scene interpolation at the headset's own display deadlines.
 // 0 = Off, 1 = Auto, otherwise 72/90/120 as a rendering-rate ceiling.
 void OpenXRSetFrameInterpolationFps(uint32_t target) noexcept;

runtime/include/vr/openxr_passthrough.h

@@ -0,0 +1,62 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#pragma once
+
+#if defined(MKW_ENABLE_OPENXR)
+
+#include "vr/openxr_runtime.h"
+
+#include <string_view>
+
+namespace mkw::vr {
+
+// The headset's camera view of the room (XR_FB_passthrough) around the virtual
+// screen, as PPSSPP VR and DolphinXR show it: one reconstruction layer,
+// submitted before every other layer so they all cover it, with the frame's
+// blend mode left OPAQUE. Its objects are created the first time the view is
+// wanted and paused whenever it is not, so a race never keeps the cameras
+// running. On a Quest the app must also declare com.oculus.feature.PASSTHROUGH
+// in its manifest, or the runtime accepts every call and composites nothing.
+//
+// Not synchronized: every call belongs to the thread that ends the session's
+// frames, while the session exists. The handles are the session's children:
+// Destroy() before xrDestroySession, which otherwise frees them itself.
+class OpenXRPassthrough final {
+public:
+    explicit OpenXRPassthrough(OpenXRLogCallback logger = {});
+
+    // Starts (creating it on first use) or pauses the view. A runtime without
+    // XR_FB_passthrough, or one that refuses it, is logged once and leaves the
+    // view off until Destroy().
+    void SetRunning(OpenXRRuntime& runtime, bool running);
+
+    // The layer to submit first while the view runs, otherwise null.
+    const XrCompositionLayerBaseHeader* Layer() const noexcept;
+
+    void Destroy() noexcept;
+
+private:
+    bool Create(OpenXRRuntime& runtime);
+    // gives_up: the failure turns the view off until Destroy().
+    void LogResult(const OpenXRRuntime& runtime, std::string_view operation, XrResult result, bool gives_up) const;
+    void Log(OpenXRLogLevel level, std::string_view message) const noexcept;
+
+    OpenXRLogCallback m_logger;
+    PFN_xrCreatePassthroughFB m_create_passthrough = nullptr;
+    PFN_xrDestroyPassthroughFB m_destroy_passthrough = nullptr;
+    PFN_xrPassthroughStartFB m_start_passthrough = nullptr;
+    PFN_xrPassthroughPauseFB m_pause_passthrough = nullptr;
+    PFN_xrCreatePassthroughLayerFB m_create_layer = nullptr;
+    PFN_xrDestroyPassthroughLayerFB m_destroy_layer = nullptr;
+    PFN_xrPassthroughLayerResumeFB m_resume_layer = nullptr;
+    PFN_xrPassthroughLayerPauseFB m_pause_layer = nullptr;
+    XrPassthroughFB m_passthrough = XR_NULL_HANDLE;
+    XrPassthroughLayerFB m_layer = XR_NULL_HANDLE;
+    XrCompositionLayerPassthroughFB m_composition{XR_TYPE_COMPOSITION_LAYER_PASSTHROUGH_FB};
+    bool m_running = false;
+    bool m_failed = false;
+};
+
+} // namespace mkw::vr
+
+#endif // defined(MKW_ENABLE_OPENXR)

runtime/src/settings_overlay.cpp

@@ -141,6 +141,9 @@ bool g_vrEnabled = RuntimeConfigFile::VrEnabled(true);
 bool g_vrStopAtDisplayCopy = RuntimeConfigFile::VrStopAtDisplayCopy(true);
 bool g_vrSkipCopyClears = RuntimeConfigFile::VrSkipCopyClears(true);
 bool g_vrHudVirtualScreen = RuntimeConfigFile::VrHudVirtualScreen(true);
+#if defined(__ANDROID__)
+bool g_vrPassthrough = RuntimeConfigFile::VrPassthrough();
+#endif
 bool g_vrFirstPerson = RuntimeConfigFile::VrFirstPerson(false);
 bool g_vrFirstPersonToggleClick = RuntimeConfigFile::VrFirstPersonToggleClick();
 // Set from any thread by the right-thumbstick click, applied on the game thread.
@@ -1478,6 +1481,18 @@ void DrawVrSettings() {
             "does pitch the view, and looking sideways while it is set will roll the "
             "horizon the way a real recline would.");
     }
+#if defined(__ANDROID__)
+    if (ImGui::Checkbox("Passthrough around the menu screen", &g_vrPassthrough)) {
+        mkw::vr::OpenXRSetPassthrough(g_vrPassthrough);
+        RuntimeConfigFile::SetVrPassthrough(g_vrPassthrough);
+    }
+    if (ImGui::IsItemHovered()) {
+        ImGui::SetTooltip(
+            "Shows your room through the headset's cameras around the menu screen and every "
+            "other screen outside a race, instead of black. Races stay fully virtual. "
+            "Applies immediately.");
+    }
+#endif
     ImGui::Separator();
     ImGui::Text("VR camera");
     if (ImGui::Checkbox("First-person camera", &g_vrFirstPerson)) {

runtime/src/vr/openxr_integration.cpp

@@ -392,10 +392,13 @@ public:
         // Either Vulkan binding extension is acceptable; the backend picks
         // whichever the runtime enabled, preferring enable2.
         config.required_extensions = {"XR_KHR_android_create_instance"};
+        // XR_FB_passthrough: the room around the virtual screen (OpenXRPassthrough), asked for
+        // whatever [vr] passthrough says, since the setting is live.
         config.optional_extensions = {"XR_KHR_vulkan_enable2", "XR_KHR_vulkan_enable",
                                       "XR_KHR_convert_timespec_time",
                                       "XR_KHR_android_thread_settings",
-                                      "XR_FB_display_refresh_rate", "XR_EXT_performance_settings"};
+                                      "XR_FB_display_refresh_rate", "XR_EXT_performance_settings",
+                                      "XR_FB_passthrough"};
         AddHandMeshExtensions(config);
         config.instance_create_next = OpenXRAndroidInstanceCreateNext();
 #endif
@@ -561,6 +564,10 @@ public:
         return interpolation_available_.load(std::memory_order_acquire);
     }
 
+    void SetPassthrough(bool enabled) noexcept {
+        passthrough_.store(enabled, std::memory_order_relaxed);
+    }
+
     void SetLeanBackDegrees(float degrees) noexcept {
         lean_back_degrees_.store(
             std::clamp(degrees, -RuntimeConfigFile::kVrLeanBackDegreesLimit,
@@ -829,6 +836,13 @@ private:
                                            : OpenXRFrameMode::VirtualScreen;
             presentation.quad_distance_meters = policy.config.hud_distance_meters;
             presentation.quad_width_meters = policy.config.hud_width_meters;
+            if (float picture_aspect = 0.0f, snapshot_aspect = 0.0f;
+                aurora_get_stereo_screen_aspects(&picture_aspect, &snapshot_aspect)) {
+                presentation.quad_content_aspect = snapshot_aspect;
+            }
+            // The room around the menu screen and every other virtual screen; a race is
+            // fully virtual, and the cameras are paused for it.
+            presentation.passthrough = !immersive && passthrough_.load(std::memory_order_relaxed);
             // The settings panel gets a compositor layer of its own while it is
             // open, and Aurora leaves it out of the eyes. A backend that could
             // not make that layer has the panel drawn into the eyes instead.
@@ -1660,6 +1674,7 @@ private:
     std::atomic_bool teardown_requested_{false};
     std::atomic_bool recenter_requested_{false};
     std::atomic<float> lean_back_degrees_{RuntimeConfigFile::VrLeanBackDegrees()};
+    std::atomic_bool passthrough_{RuntimeConfigFile::VrPassthrough()};
     std::atomic_uint32_t frame_interpolation_fps_{RuntimeConfigFile::VrFrameInterpolationFps()};
     std::atomic_bool interpolation_available_{false};
     std::mutex interpolation_mutex_;
@@ -1767,6 +1782,14 @@ void OpenXRSetLeanBackDegrees(float degrees) noexcept {
 #endif
 }
 
+void OpenXRSetPassthrough(bool enabled) noexcept {
+#if MKW_OPENXR_GRAPHICS_BACKEND
+    OpenXRIntegration::Get().SetPassthrough(enabled);
+#else
+    (void)enabled;
+#endif
+}
+
 void OpenXRSetFrameInterpolationFps(uint32_t target) noexcept {
 #if MKW_OPENXR_GRAPHICS_BACKEND
     OpenXRIntegration::Get().SetFrameInterpolationFps(target);

runtime/src/vr/openxr_passthrough.cpp

@@ -0,0 +1,154 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#if defined(MKW_ENABLE_OPENXR)
+
+#include "vr/openxr_passthrough.h"
+
+#include <algorithm>
+#include <sstream>
+#include <string>
+#include <utility>
+
+namespace mkw::vr {
+
+OpenXRPassthrough::OpenXRPassthrough(OpenXRLogCallback logger) : m_logger(std::move(logger)) {}
+
+void OpenXRPassthrough::SetRunning(OpenXRRuntime& runtime, bool running) {
+    if (running == m_running || m_failed || !runtime.HasSession()) {
+        return;
+    }
+    if (m_passthrough == XR_NULL_HANDLE) {
+        // Only a start reaches here: the view and its layer are created running.
+        m_failed = !Create(runtime);
+        m_running = !m_failed;
+        return;
+    }
+    if (running) {
+        XrResult result = m_start_passthrough(m_passthrough);
+        runtime.ObserveResult(result);
+        if (XR_SUCCEEDED(result)) {
+            result = m_resume_layer(m_layer);
+            runtime.ObserveResult(result);
+        }
+        if (XR_FAILED(result)) {
+            LogResult(runtime, "resuming the passthrough view", result, true);
+            m_failed = true;
+            return;
+        }
+        m_running = true;
+        Log(OpenXRLogLevel::Info, "OpenXR passthrough resumed");
+        return;
+    }
+    // The layer is no longer submitted from here on, whatever the runtime answers.
+    m_running = false;
+    const XrResult layer_result = m_pause_layer(m_layer);
+    runtime.ObserveResult(layer_result);
+    const XrResult result = m_pause_passthrough(m_passthrough);
+    runtime.ObserveResult(result);
+    if (XR_FAILED(layer_result) || XR_FAILED(result)) {
+        LogResult(runtime, "pausing the passthrough view", XR_FAILED(layer_result) ? layer_result : result,
+                  false);
+        return;
+    }
+    Log(OpenXRLogLevel::Info, "OpenXR passthrough paused");
+}
+
+const XrCompositionLayerBaseHeader* OpenXRPassthrough::Layer() const noexcept {
+    return m_running ? reinterpret_cast<const XrCompositionLayerBaseHeader*>(&m_composition) : nullptr;
+}
+
+void OpenXRPassthrough::Destroy() noexcept {
+    if (m_layer != XR_NULL_HANDLE) {
+        m_destroy_layer(m_layer);
+    }
+    if (m_passthrough != XR_NULL_HANDLE) {
+        m_destroy_passthrough(m_passthrough);
+    }
+    m_layer = XR_NULL_HANDLE;
+    m_passthrough = XR_NULL_HANDLE;
+    m_running = false;
+    m_failed = false;
+}
+
+bool OpenXRPassthrough::Create(OpenXRRuntime& runtime) {
+    const auto& extensions = runtime.EnabledExtensions();
+    if (std::find(extensions.begin(), extensions.end(), XR_FB_PASSTHROUGH_EXTENSION_NAME) == extensions.end()) {
+        Log(OpenXRLogLevel::Warning,
+            "OpenXR passthrough unavailable: the runtime does not offer " XR_FB_PASSTHROUGH_EXTENSION_NAME);
+        return false;
+    }
+    if (!runtime.LoadFunction("xrCreatePassthroughFB", &m_create_passthrough) ||
+        !runtime.LoadFunction("xrDestroyPassthroughFB", &m_destroy_passthrough) ||
+        !runtime.LoadFunction("xrPassthroughStartFB", &m_start_passthrough) ||
+        !runtime.LoadFunction("xrPassthroughPauseFB", &m_pause_passthrough) ||
+        !runtime.LoadFunction("xrCreatePassthroughLayerFB", &m_create_layer) ||
+        !runtime.LoadFunction("xrDestroyPassthroughLayerFB", &m_destroy_layer) ||
+        !runtime.LoadFunction("xrPassthroughLayerResumeFB", &m_resume_layer) ||
+        !runtime.LoadFunction("xrPassthroughLayerPauseFB", &m_pause_layer)) {
+        Log(OpenXRLogLevel::Warning, "OpenXR passthrough unavailable: " + runtime.LastError().message);
+        return false;
+    }
+
+    XrPassthroughCreateInfoFB create_info{XR_TYPE_PASSTHROUGH_CREATE_INFO_FB};
+    create_info.flags = XR_PASSTHROUGH_IS_RUNNING_AT_CREATION_BIT_FB;
+    XrResult result = m_create_passthrough(runtime.Session(), &create_info, &m_passthrough);
+    runtime.ObserveResult(result);
+    if (XR_FAILED(result)) {
+        m_passthrough = XR_NULL_HANDLE;
+        LogResult(runtime, "xrCreatePassthroughFB", result, true);
+        return false;
+    }
+
+    XrPassthroughLayerCreateInfoFB layer_info{XR_TYPE_PASSTHROUGH_LAYER_CREATE_INFO_FB};
+    layer_info.passthrough = m_passthrough;
+    layer_info.flags = XR_PASSTHROUGH_IS_RUNNING_AT_CREATION_BIT_FB;
+    layer_info.purpose = XR_PASSTHROUGH_LAYER_PURPOSE_RECONSTRUCTION_FB;
+    result = m_create_layer(runtime.Session(), &layer_info, &m_layer);
+    runtime.ObserveResult(result);
+    if (XR_FAILED(result)) {
+        m_layer = XR_NULL_HANDLE;
+        m_destroy_passthrough(m_passthrough);
+        m_passthrough = XR_NULL_HANDLE;
+        LogResult(runtime, "xrCreatePassthroughLayerFB", result, true);
+        return false;
+    }
+
+    // A reconstruction layer has no space of its own; the layers after it cover it.
+    m_composition = {XR_TYPE_COMPOSITION_LAYER_PASSTHROUGH_FB};
+    m_composition.flags = XR_COMPOSITION_LAYER_BLEND_TEXTURE_SOURCE_ALPHA_BIT;
+    m_composition.space = XR_NULL_HANDLE;
+    m_composition.layerHandle = m_layer;
+    Log(OpenXRLogLevel::Info, "OpenXR passthrough started");
+    return true;
+}
+
+void OpenXRPassthrough::LogResult(const OpenXRRuntime& runtime, std::string_view operation, XrResult result,
+                                  bool gives_up) const {
+    char name[XR_MAX_RESULT_STRING_SIZE]{};
+    std::ostringstream message;
+    message << "OpenXR passthrough: " << operation << " failed: ";
+    if (XR_SUCCEEDED(xrResultToString(runtime.Instance(), result, name))) {
+        message << name;
+    } else {
+        message << static_cast<int32_t>(result);
+    }
+    if (gives_up) {
+        message << "; the view stays off for this session";
+    }
+    Log(OpenXRLogLevel::Warning, message.str());
+}
+
+void OpenXRPassthrough::Log(OpenXRLogLevel level, std::string_view message) const noexcept {
+    if (!m_logger) {
+        return;
+    }
+    try {
+        m_logger(level, message);
+    } catch (...) {
+        // A diagnostic callback must never break frame submission.
+    }
+}
+
+} // namespace mkw::vr
+
+#endif // defined(MKW_ENABLE_OPENXR)

runtime/src/vr/openxr_vulkan.cpp

@@ -9,6 +9,7 @@
 
 #include "vr/openxr_vulkan.h"
 #include "vr/openxr_diagnostics.h"
+#include "vr/openxr_passthrough.h"
 
 #include <aurora/vulkan_interop.h>
 
@@ -356,6 +357,7 @@ public:
                 break;
             }
         }
+        passthrough_.SetRunning(*runtime_, presentation.passthrough);
         if (!runtime_->BeginFrame(frame.xr_frame)) {
             Fail("xrBeginFrame failed");
             return OpenXRBeginStatus::Error;
@@ -532,6 +534,8 @@ public:
         if (!runtime_->IsSessionRunning()) {
             return OpenXRBeginStatus::SessionNotRunning;
         }
+        // Before any frame ends on this presentation, the priming cycle below included.
+        passthrough_.SetRunning(*runtime_, presentation.passthrough);
         if (last_display_period_ <= 0) {
             // No display timing yet: one compositor cycle learns it.
             const OpenXRBeginStatus primed = KeepAliveCycle();
@@ -887,6 +891,11 @@ public:
         if (!have_retained_frame_ || !active_frame_.should_render) {
             diagnostics::OnEmptyFrame(!active_frame_.should_render ? diagnostics::EmptyFrameReason::ShouldRenderOff
                                                                     : diagnostics::EmptyFrameReason::NoRetainedLayer);
+            // Outside a race the room stays in view until there is an image to show (at start
+            // and after a recenter), rather than flashing black.
+            if (const XrCompositionLayerBaseHeader* passthrough = passthrough_.Layer()) {
+                return runtime_->EndFrame(active_frame_, &passthrough, 1);
+            }
             return runtime_->EndFrameWithoutLayers(active_frame_);
         }
         diagnostics::OnLayer(fresh);
@@ -896,9 +905,11 @@ public:
             quad.layerFlags = 0;
             quad.eyeVisibility = XR_EYE_VISIBILITY_BOTH;
             quad.subImage.swapchain = retained_swapchains_[0].handle;
-            quad.subImage.imageRect = {{0, 0},
-                                       {static_cast<int32_t>(retained_swapchains_[0].width),
-                                        static_cast<int32_t>(retained_swapchains_[0].height)}};
+            // Only the picture, not the black bands letterboxing it into the eye-sized image: they
+            // would frame it against the passthrough view.
+            const uint32_t image_width = retained_swapchains_[0].width;
+            quad.subImage.imageRect = OpenXRVirtualScreenContentRect(
+                image_width, retained_swapchains_[0].height, frame.presentation.quad_content_aspect);
             quad.subImage.imageArrayIndex = 0;
             if (frame.presentation.quad_anchored) {
                 quad.space = runtime_->AppSpace();
@@ -909,9 +920,12 @@ public:
                 quad.pose.position = {
                     0.0f, 0.0f, -std::max(0.25f, frame.presentation.quad_distance_meters)};
             }
-            quad.size.width = std::max(0.25f, frame.presentation.quad_width_meters);
-            quad.size.height = quad.size.width * static_cast<float>(retained_swapchains_[0].height) /
-                               static_cast<float>(retained_swapchains_[0].width);
+            // The whole image would be quad_width_meters across: the crop keeps that size per pixel,
+            // so the picture stays exactly where the pointer and the settings panel expect it.
+            const float meters_per_pixel =
+                std::max(0.25f, frame.presentation.quad_width_meters) / static_cast<float>(image_width);
+            quad.size.width = meters_per_pixel * static_cast<float>(quad.subImage.imageRect.extent.width);
+            quad.size.height = meters_per_pixel * static_cast<float>(quad.subImage.imageRect.extent.height);
             return EndFrameWithPanel(frame, reinterpret_cast<const XrCompositionLayerBaseHeader*>(&quad));
         }
         std::array<XrCompositionLayerProjectionView, kOpenXREyeCount> views{};
@@ -935,13 +949,20 @@ public:
         return EndFrameWithPanel(frame, reinterpret_cast<const XrCompositionLayerBaseHeader*>(&projection));
     }
 
-    // Ends the compositor frame with the scene's layer and, while the retained
-    // frame rendered it, the settings panel's layer over it.
+    // Ends the compositor frame with the scene's layer: over the room's camera
+    // view while that runs and the scene is the virtual screen (never under the
+    // race's projection, which covers the whole view), and, while the retained
+    // frame rendered it, under the settings panel's layer.
     bool EndFrameWithPanel(const OpenXRBackendFrame& frame, const XrCompositionLayerBaseHeader* scene) {
         const auto& panel = frame.presentation.panel;
         XrCompositionLayerQuad panel_quad{};
-        const XrCompositionLayerBaseHeader* layers[2] = {scene, nullptr};
-        uint32_t count = 1;
+        const XrCompositionLayerBaseHeader* layers[3] = {};
+        uint32_t count = 0;
+        if (const XrCompositionLayerBaseHeader* passthrough = passthrough_.Layer();
+            passthrough != nullptr && frame.presentation.mode == OpenXRFrameMode::VirtualScreen) {
+            layers[count++] = passthrough;
+        }
+        layers[count++] = scene;
         if (retained_panel_valid_ && panel.requested && panel.placed) {
             panel_quad = OpenXRPanelQuadLayer(panel, runtime_->AppSpace(), retained_panel_swapchain_.handle);
             layers[count++] = reinterpret_cast<const XrCompositionLayerBaseHeader*>(&panel_quad);
@@ -988,6 +1009,8 @@ public:
         }
         DestroySwapchains();
         DestroySlots();
+        // Its handles belong to the session.
+        passthrough_.Destroy();
         if (owns_session_ && runtime_ != nullptr) {
             runtime_->DestroySession();
             owns_session_ = false;
@@ -1995,6 +2018,8 @@ private:
 
     OpenXRRuntime* runtime_ = nullptr;
     OpenXRLogCallback logger_;
+    // The room around the virtual screen, started and paused as each presentation arrives.
+    OpenXRPassthrough passthrough_{logger_};
     OpenXRVulkanGraphicsRequirements requirements_{};
     std::array<EyeSwapchain, kOpenXREyeCount> eye_swapchains_{};
     std::array<EyeSwapchain, kOpenXREyeCount> retained_swapchains_{};

runtime/tests/openxr_d3d12_replay_tests.cpp

@@ -30,6 +30,9 @@
 #include "vr/openxr_d3d12.h"
 #endif
 
+#include "vr/openxr_wii_remote.h"
+
+#include <cmath>
 #include <cstdlib>
 #include <cstring>
 #include <iostream>
@@ -581,9 +584,33 @@ void TestPanelLayer() {
     display_time = 0;
 }
 
+bool SameRect(const XrRect2Di& rect, int32_t x, int32_t y, int32_t width, int32_t height) {
+    return rect.offset.x == x && rect.offset.y == y && rect.extent.width == width && rect.extent.height == height;
+}
+
+// The menu quad shows only the part of its eye-sized image that Aurora draws into.
+void TestVirtualScreenContentRect() {
+    // A Quest 3 eye at render_scale 0.8 holding the 1280x720 snapshot: the bands above and below
+    // go, and the in-eye settings panel (3/4 of the width, 4:3) is exactly as tall as what is left.
+    const XrRect2Di quest = OpenXRVirtualScreenContentRect(1344, 1408, 16.0f / 9.0f);
+    Require(SameRect(quest, 0, 326, 1344, 756));
+    // At the whole image's size per pixel, the cropped quad is as tall as the snapshot the pointer
+    // maps onto (MenuPictureHalfExtents, 2.4 m across).
+    const float quad_height = 2.4f * static_cast<float>(quest.extent.height) / 1344.0f;
+    const auto picture = wii_remote::MenuPictureHalfExtents(2.4f, 1344.0f / 1408.0f, 16.0f / 9.0f, 16.0f / 9.0f);
+    Require(std::fabs(quad_height - 2.0f * picture[1]) < 1e-4f);
+    // A snapshot narrower than the image is pillarboxed; the panel still has to fit.
+    Require(SameRect(OpenXRVirtualScreenContentRect(1000, 1000, 0.5f), 125, 0, 750, 1000));
+    // A wide snapshot leaves the panel taller than the picture: keep the panel whole.
+    Require(SameRect(OpenXRVirtualScreenContentRect(1344, 1408, 2.4f), 0, 326, 1344, 756));
+    // Before Aurora has published an aspect, the whole image.
+    Require(SameRect(OpenXRVirtualScreenContentRect(1344, 1408, 0.0f), 0, 0, 1344, 1408));
+}
+
 int main() {
     TestRenderFirst();
     TestPanelLayer();
+    TestVirtualScreenContentRect();
     OpenXRRuntime runtime;
     OpenXRD3D12Backend backend;
     Require(backend.QueryGraphicsRequirements(runtime) && backend.BindAurora(runtime));