Implement passthrough feature for menu screens in OpenXR

runtime/src/settings_overlay.cpp

@@ -141,6 +141,9 @@ bool g_vrEnabled = RuntimeConfigFile::VrEnabled(true);
 bool g_vrStopAtDisplayCopy = RuntimeConfigFile::VrStopAtDisplayCopy(true);
 bool g_vrSkipCopyClears = RuntimeConfigFile::VrSkipCopyClears(true);
 bool g_vrHudVirtualScreen = RuntimeConfigFile::VrHudVirtualScreen(true);
+#if defined(__ANDROID__)
+bool g_vrPassthrough = RuntimeConfigFile::VrPassthrough();
+#endif
 bool g_vrFirstPerson = RuntimeConfigFile::VrFirstPerson(false);
 bool g_vrFirstPersonToggleClick = RuntimeConfigFile::VrFirstPersonToggleClick();
 // Set from any thread by the right-thumbstick click, applied on the game thread.
@@ -1478,6 +1481,18 @@ void DrawVrSettings() {
             "does pitch the view, and looking sideways while it is set will roll the "
             "horizon the way a real recline would.");
     }
+#if defined(__ANDROID__)
+    if (ImGui::Checkbox("Passthrough around the menu screen", &g_vrPassthrough)) {
+        mkw::vr::OpenXRSetPassthrough(g_vrPassthrough);
+        RuntimeConfigFile::SetVrPassthrough(g_vrPassthrough);
+    }
+    if (ImGui::IsItemHovered()) {
+        ImGui::SetTooltip(
+            "Shows your room through the headset's cameras around the menu screen and every "
+            "other screen outside a race, instead of black. Races stay fully virtual. "
+            "Applies immediately.");
+    }
+#endif
     ImGui::Separator();
     ImGui::Text("VR camera");
     if (ImGui::Checkbox("First-person camera", &g_vrFirstPerson)) {

runtime/src/vr/openxr_integration.cpp

@@ -392,10 +392,13 @@ public:
         // Either Vulkan binding extension is acceptable; the backend picks
         // whichever the runtime enabled, preferring enable2.
         config.required_extensions = {"XR_KHR_android_create_instance"};
+        // XR_FB_passthrough: the room around the virtual screen (OpenXRPassthrough), asked for
+        // whatever [vr] passthrough says, since the setting is live.
         config.optional_extensions = {"XR_KHR_vulkan_enable2", "XR_KHR_vulkan_enable",
                                       "XR_KHR_convert_timespec_time",
                                       "XR_KHR_android_thread_settings",
-                                      "XR_FB_display_refresh_rate", "XR_EXT_performance_settings"};
+                                      "XR_FB_display_refresh_rate", "XR_EXT_performance_settings",
+                                      "XR_FB_passthrough"};
         AddHandMeshExtensions(config);
         config.instance_create_next = OpenXRAndroidInstanceCreateNext();
 #endif
@@ -561,6 +564,10 @@ public:
         return interpolation_available_.load(std::memory_order_acquire);
     }
 
+    void SetPassthrough(bool enabled) noexcept {
+        passthrough_.store(enabled, std::memory_order_relaxed);
+    }
+
     void SetLeanBackDegrees(float degrees) noexcept {
         lean_back_degrees_.store(
             std::clamp(degrees, -RuntimeConfigFile::kVrLeanBackDegreesLimit,
@@ -829,6 +836,13 @@ private:
                                            : OpenXRFrameMode::VirtualScreen;
             presentation.quad_distance_meters = policy.config.hud_distance_meters;
             presentation.quad_width_meters = policy.config.hud_width_meters;
+            if (float picture_aspect = 0.0f, snapshot_aspect = 0.0f;
+                aurora_get_stereo_screen_aspects(&picture_aspect, &snapshot_aspect)) {
+                presentation.quad_content_aspect = snapshot_aspect;
+            }
+            // The room around the menu screen and every other virtual screen; a race is
+            // fully virtual, and the cameras are paused for it.
+            presentation.passthrough = !immersive && passthrough_.load(std::memory_order_relaxed);
             // The settings panel gets a compositor layer of its own while it is
             // open, and Aurora leaves it out of the eyes. A backend that could
             // not make that layer has the panel drawn into the eyes instead.
@@ -1660,6 +1674,7 @@ private:
     std::atomic_bool teardown_requested_{false};
     std::atomic_bool recenter_requested_{false};
     std::atomic<float> lean_back_degrees_{RuntimeConfigFile::VrLeanBackDegrees()};
+    std::atomic_bool passthrough_{RuntimeConfigFile::VrPassthrough()};
     std::atomic_uint32_t frame_interpolation_fps_{RuntimeConfigFile::VrFrameInterpolationFps()};
     std::atomic_bool interpolation_available_{false};
     std::mutex interpolation_mutex_;
@@ -1767,6 +1782,14 @@ void OpenXRSetLeanBackDegrees(float degrees) noexcept {
 #endif
 }
 
+void OpenXRSetPassthrough(bool enabled) noexcept {
+#if MKW_OPENXR_GRAPHICS_BACKEND
+    OpenXRIntegration::Get().SetPassthrough(enabled);
+#else
+    (void)enabled;
+#endif
+}
+
 void OpenXRSetFrameInterpolationFps(uint32_t target) noexcept {
 #if MKW_OPENXR_GRAPHICS_BACKEND
     OpenXRIntegration::Get().SetFrameInterpolationFps(target);

runtime/src/vr/openxr_passthrough.cpp

@@ -0,0 +1,154 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#if defined(MKW_ENABLE_OPENXR)
+
+#include "vr/openxr_passthrough.h"
+
+#include <algorithm>
+#include <sstream>
+#include <string>
+#include <utility>
+
+namespace mkw::vr {
+
+OpenXRPassthrough::OpenXRPassthrough(OpenXRLogCallback logger) : m_logger(std::move(logger)) {}
+
+void OpenXRPassthrough::SetRunning(OpenXRRuntime& runtime, bool running) {
+    if (running == m_running || m_failed || !runtime.HasSession()) {
+        return;
+    }
+    if (m_passthrough == XR_NULL_HANDLE) {
+        // Only a start reaches here: the view and its layer are created running.
+        m_failed = !Create(runtime);
+        m_running = !m_failed;
+        return;
+    }
+    if (running) {
+        XrResult result = m_start_passthrough(m_passthrough);
+        runtime.ObserveResult(result);
+        if (XR_SUCCEEDED(result)) {
+            result = m_resume_layer(m_layer);
+            runtime.ObserveResult(result);
+        }
+        if (XR_FAILED(result)) {
+            LogResult(runtime, "resuming the passthrough view", result, true);
+            m_failed = true;
+            return;
+        }
+        m_running = true;
+        Log(OpenXRLogLevel::Info, "OpenXR passthrough resumed");
+        return;
+    }
+    // The layer is no longer submitted from here on, whatever the runtime answers.
+    m_running = false;
+    const XrResult layer_result = m_pause_layer(m_layer);
+    runtime.ObserveResult(layer_result);
+    const XrResult result = m_pause_passthrough(m_passthrough);
+    runtime.ObserveResult(result);
+    if (XR_FAILED(layer_result) || XR_FAILED(result)) {
+        LogResult(runtime, "pausing the passthrough view", XR_FAILED(layer_result) ? layer_result : result,
+                  false);
+        return;
+    }
+    Log(OpenXRLogLevel::Info, "OpenXR passthrough paused");
+}
+
+const XrCompositionLayerBaseHeader* OpenXRPassthrough::Layer() const noexcept {
+    return m_running ? reinterpret_cast<const XrCompositionLayerBaseHeader*>(&m_composition) : nullptr;
+}
+
+void OpenXRPassthrough::Destroy() noexcept {
+    if (m_layer != XR_NULL_HANDLE) {
+        m_destroy_layer(m_layer);
+    }
+    if (m_passthrough != XR_NULL_HANDLE) {
+        m_destroy_passthrough(m_passthrough);
+    }
+    m_layer = XR_NULL_HANDLE;
+    m_passthrough = XR_NULL_HANDLE;
+    m_running = false;
+    m_failed = false;
+}
+
+bool OpenXRPassthrough::Create(OpenXRRuntime& runtime) {
+    const auto& extensions = runtime.EnabledExtensions();
+    if (std::find(extensions.begin(), extensions.end(), XR_FB_PASSTHROUGH_EXTENSION_NAME) == extensions.end()) {
+        Log(OpenXRLogLevel::Warning,
+            "OpenXR passthrough unavailable: the runtime does not offer " XR_FB_PASSTHROUGH_EXTENSION_NAME);
+        return false;
+    }
+    if (!runtime.LoadFunction("xrCreatePassthroughFB", &m_create_passthrough) ||
+        !runtime.LoadFunction("xrDestroyPassthroughFB", &m_destroy_passthrough) ||
+        !runtime.LoadFunction("xrPassthroughStartFB", &m_start_passthrough) ||
+        !runtime.LoadFunction("xrPassthroughPauseFB", &m_pause_passthrough) ||
+        !runtime.LoadFunction("xrCreatePassthroughLayerFB", &m_create_layer) ||
+        !runtime.LoadFunction("xrDestroyPassthroughLayerFB", &m_destroy_layer) ||
+        !runtime.LoadFunction("xrPassthroughLayerResumeFB", &m_resume_layer) ||
+        !runtime.LoadFunction("xrPassthroughLayerPauseFB", &m_pause_layer)) {
+        Log(OpenXRLogLevel::Warning, "OpenXR passthrough unavailable: " + runtime.LastError().message);
+        return false;
+    }
+
+    XrPassthroughCreateInfoFB create_info{XR_TYPE_PASSTHROUGH_CREATE_INFO_FB};
+    create_info.flags = XR_PASSTHROUGH_IS_RUNNING_AT_CREATION_BIT_FB;
+    XrResult result = m_create_passthrough(runtime.Session(), &create_info, &m_passthrough);
+    runtime.ObserveResult(result);
+    if (XR_FAILED(result)) {
+        m_passthrough = XR_NULL_HANDLE;
+        LogResult(runtime, "xrCreatePassthroughFB", result, true);
+        return false;
+    }
+
+    XrPassthroughLayerCreateInfoFB layer_info{XR_TYPE_PASSTHROUGH_LAYER_CREATE_INFO_FB};
+    layer_info.passthrough = m_passthrough;
+    layer_info.flags = XR_PASSTHROUGH_IS_RUNNING_AT_CREATION_BIT_FB;
+    layer_info.purpose = XR_PASSTHROUGH_LAYER_PURPOSE_RECONSTRUCTION_FB;
+    result = m_create_layer(runtime.Session(), &layer_info, &m_layer);
+    runtime.ObserveResult(result);
+    if (XR_FAILED(result)) {
+        m_layer = XR_NULL_HANDLE;
+        m_destroy_passthrough(m_passthrough);
+        m_passthrough = XR_NULL_HANDLE;
+        LogResult(runtime, "xrCreatePassthroughLayerFB", result, true);
+        return false;
+    }
+
+    // A reconstruction layer has no space of its own; the layers after it cover it.
+    m_composition = {XR_TYPE_COMPOSITION_LAYER_PASSTHROUGH_FB};
+    m_composition.flags = XR_COMPOSITION_LAYER_BLEND_TEXTURE_SOURCE_ALPHA_BIT;
+    m_composition.space = XR_NULL_HANDLE;
+    m_composition.layerHandle = m_layer;
+    Log(OpenXRLogLevel::Info, "OpenXR passthrough started");
+    return true;
+}
+
+void OpenXRPassthrough::LogResult(const OpenXRRuntime& runtime, std::string_view operation, XrResult result,
+                                  bool gives_up) const {
+    char name[XR_MAX_RESULT_STRING_SIZE]{};
+    std::ostringstream message;
+    message << "OpenXR passthrough: " << operation << " failed: ";
+    if (XR_SUCCEEDED(xrResultToString(runtime.Instance(), result, name))) {
+        message << name;
+    } else {
+        message << static_cast<int32_t>(result);
+    }
+    if (gives_up) {
+        message << "; the view stays off for this session";
+    }
+    Log(OpenXRLogLevel::Warning, message.str());
+}
+
+void OpenXRPassthrough::Log(OpenXRLogLevel level, std::string_view message) const noexcept {
+    if (!m_logger) {
+        return;
+    }
+    try {
+        m_logger(level, message);
+    } catch (...) {
+        // A diagnostic callback must never break frame submission.
+    }
+}
+
+} // namespace mkw::vr
+
+#endif // defined(MKW_ENABLE_OPENXR)

runtime/src/vr/openxr_vulkan.cpp

@@ -9,6 +9,7 @@
 
 #include "vr/openxr_vulkan.h"
 #include "vr/openxr_diagnostics.h"
+#include "vr/openxr_passthrough.h"
 
 #include <aurora/vulkan_interop.h>
 
@@ -356,6 +357,7 @@ public:
                 break;
             }
         }
+        passthrough_.SetRunning(*runtime_, presentation.passthrough);
         if (!runtime_->BeginFrame(frame.xr_frame)) {
             Fail("xrBeginFrame failed");
             return OpenXRBeginStatus::Error;
@@ -532,6 +534,8 @@ public:
         if (!runtime_->IsSessionRunning()) {
             return OpenXRBeginStatus::SessionNotRunning;
         }
+        // Before any frame ends on this presentation, the priming cycle below included.
+        passthrough_.SetRunning(*runtime_, presentation.passthrough);
         if (last_display_period_ <= 0) {
             // No display timing yet: one compositor cycle learns it.
             const OpenXRBeginStatus primed = KeepAliveCycle();
@@ -887,6 +891,11 @@ public:
         if (!have_retained_frame_ || !active_frame_.should_render) {
             diagnostics::OnEmptyFrame(!active_frame_.should_render ? diagnostics::EmptyFrameReason::ShouldRenderOff
                                                                     : diagnostics::EmptyFrameReason::NoRetainedLayer);
+            // Outside a race the room stays in view until there is an image to show (at start
+            // and after a recenter), rather than flashing black.
+            if (const XrCompositionLayerBaseHeader* passthrough = passthrough_.Layer()) {
+                return runtime_->EndFrame(active_frame_, &passthrough, 1);
+            }
             return runtime_->EndFrameWithoutLayers(active_frame_);
         }
         diagnostics::OnLayer(fresh);
@@ -896,9 +905,11 @@ public:
             quad.layerFlags = 0;
             quad.eyeVisibility = XR_EYE_VISIBILITY_BOTH;
             quad.subImage.swapchain = retained_swapchains_[0].handle;
-            quad.subImage.imageRect = {{0, 0},
-                                       {static_cast<int32_t>(retained_swapchains_[0].width),
-                                        static_cast<int32_t>(retained_swapchains_[0].height)}};
+            // Only the picture, not the black bands letterboxing it into the eye-sized image: they
+            // would frame it against the passthrough view.
+            const uint32_t image_width = retained_swapchains_[0].width;
+            quad.subImage.imageRect = OpenXRVirtualScreenContentRect(
+                image_width, retained_swapchains_[0].height, frame.presentation.quad_content_aspect);
             quad.subImage.imageArrayIndex = 0;
             if (frame.presentation.quad_anchored) {
                 quad.space = runtime_->AppSpace();
@@ -909,9 +920,12 @@ public:
                 quad.pose.position = {
                     0.0f, 0.0f, -std::max(0.25f, frame.presentation.quad_distance_meters)};
             }
-            quad.size.width = std::max(0.25f, frame.presentation.quad_width_meters);
-            quad.size.height = quad.size.width * static_cast<float>(retained_swapchains_[0].height) /
-                               static_cast<float>(retained_swapchains_[0].width);
+            // The whole image would be quad_width_meters across: the crop keeps that size per pixel,
+            // so the picture stays exactly where the pointer and the settings panel expect it.
+            const float meters_per_pixel =
+                std::max(0.25f, frame.presentation.quad_width_meters) / static_cast<float>(image_width);
+            quad.size.width = meters_per_pixel * static_cast<float>(quad.subImage.imageRect.extent.width);
+            quad.size.height = meters_per_pixel * static_cast<float>(quad.subImage.imageRect.extent.height);
             return EndFrameWithPanel(frame, reinterpret_cast<const XrCompositionLayerBaseHeader*>(&quad));
         }
         std::array<XrCompositionLayerProjectionView, kOpenXREyeCount> views{};
@@ -935,13 +949,20 @@ public:
         return EndFrameWithPanel(frame, reinterpret_cast<const XrCompositionLayerBaseHeader*>(&projection));
     }
 
-    // Ends the compositor frame with the scene's layer and, while the retained
-    // frame rendered it, the settings panel's layer over it.
+    // Ends the compositor frame with the scene's layer: over the room's camera
+    // view while that runs and the scene is the virtual screen (never under the
+    // race's projection, which covers the whole view), and, while the retained
+    // frame rendered it, under the settings panel's layer.
     bool EndFrameWithPanel(const OpenXRBackendFrame& frame, const XrCompositionLayerBaseHeader* scene) {
         const auto& panel = frame.presentation.panel;
         XrCompositionLayerQuad panel_quad{};
-        const XrCompositionLayerBaseHeader* layers[2] = {scene, nullptr};
-        uint32_t count = 1;
+        const XrCompositionLayerBaseHeader* layers[3] = {};
+        uint32_t count = 0;
+        if (const XrCompositionLayerBaseHeader* passthrough = passthrough_.Layer();
+            passthrough != nullptr && frame.presentation.mode == OpenXRFrameMode::VirtualScreen) {
+            layers[count++] = passthrough;
+        }
+        layers[count++] = scene;
         if (retained_panel_valid_ && panel.requested && panel.placed) {
             panel_quad = OpenXRPanelQuadLayer(panel, runtime_->AppSpace(), retained_panel_swapchain_.handle);
             layers[count++] = reinterpret_cast<const XrCompositionLayerBaseHeader*>(&panel_quad);
@@ -988,6 +1009,8 @@ public:
         }
         DestroySwapchains();
         DestroySlots();
+        // Its handles belong to the session.
+        passthrough_.Destroy();
         if (owns_session_ && runtime_ != nullptr) {
             runtime_->DestroySession();
             owns_session_ = false;
@@ -1995,6 +2018,8 @@ private:
 
     OpenXRRuntime* runtime_ = nullptr;
     OpenXRLogCallback logger_;
+    // The room around the virtual screen, started and paused as each presentation arrives.
+    OpenXRPassthrough passthrough_{logger_};
     OpenXRVulkanGraphicsRequirements requirements_{};
     std::array<EyeSwapchain, kOpenXREyeCount> eye_swapchains_{};
     std::array<EyeSwapchain, kOpenXREyeCount> retained_swapchains_{};