feat: added immersive window support for VR race views

- Introduced a new configuration option for immersive window mode in runtime_config.h.
- Updated the parsing and setting functions to handle the immersive window state.
- Modified the OpenXR backend to support rendering with the immersive window, blending the race view with the surrounding environment.
- Enhanced the settings overlay to allow users to select between immersive, immersive window, and flat screen race views.
- Implemented GPU rendering logic for the immersive window mask, ensuring correct visual output in various rendering paths.
- Added tests to validate the immersive window functionality and its interaction with existing race view settings.

runtime/include/runtime_config.h

@@ -58,6 +58,7 @@ struct RuntimeUserConfig {
     std::optional<float> vrHudWidthMeters;
     std::optional<bool> vrHudVirtualScreen;
     std::optional<bool> vrFlatScreen;
+    std::optional<bool> vrImmersiveWindow;
     std::optional<bool> vrPassthrough;
     std::optional<bool> vrStopAtDisplayCopy;
     std::optional<bool> vrSkipCopyClears;
@@ -522,6 +523,11 @@ inline void EnsureConfigFile() {
               "# first-person camera or hand steering. Changeable live from the\n"
               "# F10 menu.\n"
               "flat_screen = false\n"
+              "# The immersive window keeps the stereo race view but shows it\n"
+              "# only through that screen, with the room around it on the\n"
+              "# Quest (black elsewhere). Flat Screen mode wins over it.\n"
+              "# Changeable live from the F10 menu.\n"
+              "immersive_window = false\n"
               "# EFB replay controls for the per-eye views, changeable live\n"
               "# from the F10 menu. stop_at_display_copy ends each eye at the\n"
               "# frame's final GXCopyDisp; skip_copy_clears drops the EFB\n"
@@ -778,6 +784,7 @@ inline RuntimeUserConfig ParseConfigDocument(const toml::value& document) {
     }
     config.vrHudVirtualScreen = FindConfigValue<bool>(document, "vr", "hud_virtual_screen");
     config.vrFlatScreen = FindConfigValue<bool>(document, "vr", "flat_screen");
+    config.vrImmersiveWindow = FindConfigValue<bool>(document, "vr", "immersive_window");
     config.vrPassthrough = FindConfigValue<bool>(document, "vr", "passthrough");
     config.vrStopAtDisplayCopy = FindConfigValue<bool>(document, "vr", "stop_at_display_copy");
     config.vrSkipCopyClears = FindConfigValue<bool>(document, "vr", "skip_copy_clears");
@@ -1102,6 +1109,11 @@ inline bool SetVrFlatScreen(bool value) {
     return WriteSetting("vr", "flat_screen", value ? "true" : "false");
 }
 
+inline bool SetVrImmersiveWindow(bool value) {
+    Mutable().vrImmersiveWindow = value;
+    return WriteSetting("vr", "immersive_window", value ? "true" : "false");
+}
+
 inline bool SetVrPassthrough(bool value) {
     Mutable().vrPassthrough = value;
     return WriteSetting("vr", "passthrough", value ? "true" : "false");
@@ -1579,10 +1591,41 @@ inline bool VrFlatScreen(bool fallback = false) {
     return Get().vrFlatScreen.value_or(fallback);
 }
 
+// Races in stereo as usual, but seen only through the screen the race's 2D
+// layer sits on, with the room around it. Flat Screen mode wins over it.
+inline bool VrImmersiveWindow(bool fallback = false) {
+    return Get().vrImmersiveWindow.value_or(fallback);
+}
+
+// The race view the settings present as one choice, kept in the two keys
+// above so that a file without immersive_window reads as it always did.
+enum class VrRaceView : int {
+    Immersive = 0,
+    ImmersiveWindow = 1,
+    FlatScreen = 2,
+};
+
+inline VrRaceView VrRaceViewOf(const RuntimeUserConfig& config) {
+    if (config.vrFlatScreen.value_or(false)) {
+        return VrRaceView::FlatScreen;
+    }
+    return config.vrImmersiveWindow.value_or(false) ? VrRaceView::ImmersiveWindow : VrRaceView::Immersive;
+}
+
+inline VrRaceView GetVrRaceView() {
+    return VrRaceViewOf(Get());
+}
+
+inline bool SetVrRaceView(VrRaceView view) {
+    const bool flat = SetVrFlatScreen(view == VrRaceView::FlatScreen);
+    const bool window = SetVrImmersiveWindow(view == VrRaceView::ImmersiveWindow);
+    return flat && window;
+}
+
 // The room, through the headset's cameras, around the menu screen and every
-// other virtual screen, a Flat Screen race included (never an immersive
-// race). Only the Quest offers it; the launcher's Settings page shows the same
-// default.
+// other virtual screen, a Flat Screen race included, and around the immersive
+// window (never a fully immersive race). Only the Quest offers it; the
+// launcher's Settings page shows the same default.
 inline bool VrPassthrough(bool fallback = true) {
     return Get().vrPassthrough.value_or(fallback);
 }

runtime/include/vr/openxr_backend.h

@@ -132,9 +132,15 @@ struct OpenXRPresentation {
     bool quad_anchored = false;
     XrPosef quad_pose{{0.0f, 0.0f, 0.0f, 1.0f}, {0.0f, 0.0f, 0.0f}};
 
-    // Show the room through the headset's cameras around the virtual screen
-    // (OpenXRPassthrough). Taken when the presentation is handed to the backend,
-    // which starts or pauses the view then; a backend without one ignores it.
+    // Used only by ImmersiveProjection: Aurora left each eye transparent outside
+    // the race's 2D-layer screen (AuroraStereoFrame::window), so the projection
+    // layer is blended by its alpha over whatever is under it.
+    bool immersive_window = false;
+
+    // Show the room through the headset's cameras around the virtual screen or
+    // the immersive window (OpenXRPassthrough). Taken when the presentation is
+    // handed to the backend, which starts or pauses the view then; a backend
+    // without one ignores it.
     bool passthrough = false;
 
     OpenXRPanelLayer panel;

runtime/include/vr/openxr_integration.h

@@ -61,11 +61,19 @@ void OpenXRRequestRecenter() noexcept;
 void OpenXRSetLeanBackDegrees(float degrees) noexcept;
 
 // Shows the room through the headset's cameras around the menu screen and every
-// other virtual screen, never during an immersive race. Only the standalone
-// (Quest) backend offers it; elsewhere this changes nothing. Callable from any
-// thread; applied on the XR pacing thread's next frame.
+// other virtual screen, and around the immersive window, never during a fully
+// immersive race. Only the standalone (Quest) backend offers it; elsewhere this
+// changes nothing. Callable from any thread; applied on the XR pacing thread's
+// next frame.
 void OpenXRSetPassthrough(bool enabled) noexcept;
 
+// The immersive window: an immersive race keeps its stereo view but is seen
+// only through the screen its 2D layer sits on, with the room (or, without
+// passthrough, black) around it. Callable from any thread; applied to the next
+// published frame. Flat Screen mode, which keeps races off the immersive path
+// altogether, makes it moot.
+void OpenXRSetImmersiveWindow(bool enabled) noexcept;
+
 // Live scene interpolation at the headset's own display deadlines.
 // 0 = Off, 1 = Auto, otherwise 72/90/120 as a rendering-rate ceiling.
 void OpenXRSetFrameInterpolationFps(uint32_t target) noexcept;

runtime/src/settings_overlay.cpp

@@ -142,7 +142,11 @@ bool g_vrStopAtDisplayCopy = RuntimeConfigFile::VrStopAtDisplayCopy(true);
 bool g_vrSkipCopyClears = RuntimeConfigFile::VrSkipCopyClears(true);
 bool g_vrSinglePassEyes = RuntimeConfigFile::VrSinglePassEyes(true);
 bool g_vrHudVirtualScreen = RuntimeConfigFile::VrHudVirtualScreen(true);
-bool g_vrFlatScreen = RuntimeConfigFile::VrFlatScreen();
+// Race view: Immersive, Immersive window or Flat screen (RuntimeConfigFile::VrRaceView), and
+// Flat Screen mode as the flag the race view rows below are disabled by.
+int g_vrRaceView = static_cast<int>(RuntimeConfigFile::GetVrRaceView());
+bool g_vrFlatScreen = g_vrRaceView == static_cast<int>(RuntimeConfigFile::VrRaceView::FlatScreen);
+constexpr std::array<const char*, 3> kVrRaceViewLabels{"Immersive", "Immersive window", "Flat screen"};
 #if defined(__ANDROID__)
 bool g_vrPassthrough = RuntimeConfigFile::VrPassthrough();
 // Menu labels for the foveation levels, index-matched to RuntimeConfigFile::kVrFoveationLevels and to
@@ -1444,7 +1448,8 @@ void DrawVrSettings() {
     }
     ImGui::Separator();
     ImGui::Text("VR 2D layer");
-    ImGui::BeginDisabled(g_vrFlatScreen);
+    ImGui::BeginDisabled(
+        g_vrFlatScreen || g_vrRaceView == static_cast<int>(RuntimeConfigFile::VrRaceView::ImmersiveWindow));
     if (ImGui::Checkbox("2D layer on a virtual screen", &g_vrHudVirtualScreen)) {
         ApplyVrHudVirtualScreen();
         RuntimeConfigFile::SetVrHudVirtualScreen(g_vrHudVirtualScreen);
@@ -1455,7 +1460,8 @@ void DrawVrSettings() {
             "Puts the minimap, race position, item roulette and the rest of the race HUD on a "
             "screen fixed ahead of the kart camera. Turn off to leave them stretched across "
             "the whole view. Its size and distance are the [vr] hud_width_meters and "
-            "hud_distance_meters read at launch.");
+            "hud_distance_meters read at launch. The immersive window is that screen, and "
+            "always carries them.");
     }
     ImGui::Separator();
     ImGui::Text("VR view");
@@ -1514,8 +1520,8 @@ void DrawVrSettings() {
         ImGui::SetTooltip(
             "Shows your room through the headset's cameras around the menu screen and every "
             "other screen outside an immersive race, instead of black. Immersive races stay "
-            "fully virtual; the Flat Screen race has the room around it too. "
-            "Applies immediately.");
+            "fully virtual; the immersive window and the Flat Screen race have the room "
+            "around them too. Applies immediately.");
     }
     // Shows the live level, which debug.wiicompiled.foveation can override.
     g_vrFoveation = static_cast<int>(aurora_get_stereo_foveation());
@@ -1539,16 +1545,23 @@ void DrawVrSettings() {
 #endif
     ImGui::Separator();
     ImGui::Text("VR camera");
-    if (ImGui::Checkbox("Flat Screen mode", &g_vrFlatScreen)) {
-        RuntimeConfigFile::SetVrFlatScreen(g_vrFlatScreen);
+    if (ImGui::Combo("Race view", &g_vrRaceView, kVrRaceViewLabels.data(),
+                     static_cast<int>(kVrRaceViewLabels.size()))) {
+        const auto view = static_cast<RuntimeConfigFile::VrRaceView>(g_vrRaceView);
+        g_vrFlatScreen = view == RuntimeConfigFile::VrRaceView::FlatScreen;
+        RuntimeConfigFile::SetVrRaceView(view);
         mkw::vr::MkwVRPolicySetImmersiveRaces(!g_vrFlatScreen);
+        mkw::vr::OpenXRSetImmersiveWindow(view == RuntimeConfigFile::VrRaceView::ImmersiveWindow);
         mkw::vr::MkwVRFirstPersonApplyConfiguredSettings();
     }
     if (ImGui::IsItemHovered()) {
         ImGui::SetTooltip(
-            "Plays races on the same flat screen as the menus, through the game's own camera, "
-            "instead of all around you in stereo. The first-person camera, hand steering and "
-            "the race view settings do not apply while it is on. Applies immediately.");
+            "Immersive plays races all around you in stereo. Immersive window keeps that "
+            "stereo view but shows it only through a window where the menu screen sits, with "
+            "your room around it on the Quest (black elsewhere); look through it from another "
+            "angle and the view shifts as through a real window. Flat screen plays races on "
+            "the menu screen through the game's own camera; the first-person camera, hand "
+            "steering and the race view settings do not apply to it. Applies immediately.");
     }
     // Everything below shapes the immersive race view, which Flat Screen mode replaces.
     ImGui::BeginDisabled(g_vrFlatScreen);
@@ -1704,7 +1717,8 @@ void DrawVrSettings() {
 }
 
 // The right-thumbstick click: flips the first-person camera exactly as its
-// checkbox does, so it does nothing in Flat Screen mode either. Game thread.
+// checkbox does, so it does nothing in Flat Screen mode either (it does in the
+// immersive window, which is still the stereo race view). Game thread.
 void ToggleFirstPersonCamera() {
     if (g_vrFlatScreen) {
         return;

runtime/src/vr/openxr_integration.cpp

@@ -568,6 +568,10 @@ public:
         passthrough_.store(enabled, std::memory_order_relaxed);
     }
 
+    void SetImmersiveWindow(bool enabled) noexcept {
+        immersive_window_.store(enabled, std::memory_order_relaxed);
+    }
+
     void SetLeanBackDegrees(float degrees) noexcept {
         lean_back_degrees_.store(
             std::clamp(degrees, -RuntimeConfigFile::kVrLeanBackDegreesLimit,
@@ -846,10 +850,15 @@ private:
                 aurora_get_stereo_screen_aspects(&picture_aspect, &snapshot_aspect)) {
                 presentation.quad_content_aspect = snapshot_aspect;
             }
-            // The room around the menu screen and every other virtual screen, a
-            // Flat Screen race included; an immersive race is fully virtual, and the
-            // cameras are paused for it.
-            presentation.passthrough = !immersive && passthrough_.load(std::memory_order_relaxed);
+            // The immersive window: the race's stereo view seen through its 2D layer's screen.
+            // The flag travels with the packet, so the eyes Aurora masks and the layer the
+            // backend blends always belong to the same frame.
+            presentation.immersive_window = immersive && immersive_window_.load(std::memory_order_relaxed);
+            // The room around the menu screen and every other virtual screen, a Flat Screen
+            // race included, and around the immersive window; a fully immersive race is
+            // virtual all round, and the cameras are paused for it.
+            presentation.passthrough =
+                (!immersive || presentation.immersive_window) && passthrough_.load(std::memory_order_relaxed);
             // The settings panel gets a compositor layer of its own while it is
             // open, and Aurora leaves it out of the eyes. A backend that could
             // not make that layer has the panel drawn into the eyes instead.
@@ -1247,6 +1256,7 @@ private:
         destination.displayTimeNanos = DisplayTimeNanos(source.xr_frame.predicted_display_time);
         destination.mode = immersive ? AURORA_STEREO_FRAME_IMMERSIVE_REPLAY
                                       : AURORA_STEREO_FRAME_VIRTUAL_SCREEN;
+        destination.window = immersive && source.presentation.immersive_window;
         for (uint32_t eye = 0; eye < kOpenXREyeCount; ++eye) {
             destination.eyes[eye].width = source.render_width[eye];
             destination.eyes[eye].height = source.render_height[eye];
@@ -1378,7 +1388,8 @@ private:
     // that space (in metres) to base + lean * p in the application space, so the
     // screen sits at base + lean * (0, 0, -distance), turned by the lean, its
     // height following the picture aspect as stereo_hud_screen's does. With the
-    // 2D layer stretched across the eyes there is no screen to point at.
+    // 2D layer stretched across the eyes there is no screen to point at, except
+    // in the immersive window, which is that screen and always carries the layer.
     OpenXRPointerScreen PointerScreen(const OpenXRBackendFrame& frame, const MkwVRPolicySnapshot& policy,
                                       bool immersive) const noexcept {
         OpenXRPointerScreen screen{};
@@ -1389,8 +1400,9 @@ private:
         }
 
         if (immersive) {
-            if (!aurora_get_stereo_hud_screen_enabled() || !(policy.config.hud_width_meters > 0.0f) ||
-                !RaceScreenPose(frame, policy, screen.pose)) {
+            // The immersive window always carries the 2D layer.
+            const bool on_screen = aurora_get_stereo_hud_screen_enabled() || frame.presentation.immersive_window;
+            if (!on_screen || !(policy.config.hud_width_meters > 0.0f) || !RaceScreenPose(frame, policy, screen.pose)) {
                 return screen;
             }
             screen.half_width_meters = 0.5f * policy.config.hud_width_meters;
@@ -1756,6 +1768,7 @@ private:
     std::atomic_bool recenter_requested_{false};
     std::atomic<float> lean_back_degrees_{RuntimeConfigFile::VrLeanBackDegrees()};
     std::atomic_bool passthrough_{RuntimeConfigFile::VrPassthrough()};
+    std::atomic_bool immersive_window_{RuntimeConfigFile::VrImmersiveWindow()};
     std::atomic_uint32_t frame_interpolation_fps_{RuntimeConfigFile::VrFrameInterpolationFps()};
     std::atomic_bool interpolation_available_{false};
     std::mutex interpolation_mutex_;
@@ -1877,6 +1890,14 @@ void OpenXRSetPassthrough(bool enabled) noexcept {
 #endif
 }
 
+void OpenXRSetImmersiveWindow(bool enabled) noexcept {
+#if MKW_OPENXR_GRAPHICS_BACKEND
+    OpenXRIntegration::Get().SetImmersiveWindow(enabled);
+#else
+    (void)enabled;
+#endif
+}
+
 void OpenXRSetFrameInterpolationFps(uint32_t target) noexcept {
 #if MKW_OPENXR_GRAPHICS_BACKEND
     OpenXRIntegration::Get().SetFrameInterpolationFps(target);

runtime/src/vr/openxr_vulkan.cpp

@@ -942,7 +942,10 @@ public:
             views[eye].subImage.imageArrayIndex = 0;
         }
         XrCompositionLayerProjection projection{XR_TYPE_COMPOSITION_LAYER_PROJECTION};
-        projection.layerFlags = 0;
+        // The immersive window's eyes are transparent outside the window (premultiplied alpha), so
+        // the room shows around it; otherwise the race covers the whole view and alpha is ignored.
+        projection.layerFlags =
+            frame.presentation.immersive_window ? XR_COMPOSITION_LAYER_BLEND_TEXTURE_SOURCE_ALPHA_BIT : 0;
         projection.space = runtime_->AppSpace();
         projection.viewCount = kOpenXREyeCount;
         projection.views = views.data();
@@ -950,16 +953,18 @@ public:
     }
 
     // Ends the compositor frame with the scene's layer: over the room's camera
-    // view while that runs and the scene is the virtual screen (never under the
-    // race's projection, which covers the whole view), and, while the retained
-    // frame rendered it, under the settings panel's layer.
+    // view while that runs and the scene is the virtual screen or the immersive
+    // window (never under a fully immersive race's projection, which covers the
+    // whole view), and, while the retained frame rendered it, under the settings
+    // panel's layer.
     bool EndFrameWithPanel(const OpenXRBackendFrame& frame, const XrCompositionLayerBaseHeader* scene) {
         const auto& panel = frame.presentation.panel;
         XrCompositionLayerQuad panel_quad{};
         const XrCompositionLayerBaseHeader* layers[3] = {};
         uint32_t count = 0;
         if (const XrCompositionLayerBaseHeader* passthrough = passthrough_.Layer();
-            passthrough != nullptr && frame.presentation.mode == OpenXRFrameMode::VirtualScreen) {
+            passthrough != nullptr && (frame.presentation.mode == OpenXRFrameMode::VirtualScreen ||
+                                       frame.presentation.immersive_window)) {
             layers[count++] = passthrough;
         }
         layers[count++] = scene;

runtime/tests/vr_config_tests.cpp

@@ -34,5 +34,18 @@ int main() {
     Require(Parse("[vr]\nsingle_pass_eyes = true\n").vrSinglePassEyes == true);
     Require(Parse("[vr]\nsingle_pass_eyes = false\n").vrSinglePassEyes == false);
     Require(!Parse("[vr]\n").vrSinglePassEyes.has_value());
+
+    // [vr] immersive_window and flat_screen: one race view in two keys, Flat
+    // Screen mode winning, so a file that predates the window reads as before.
+    using RuntimeConfigFile::VrRaceView;
+    using RuntimeConfigFile::VrRaceViewOf;
+    Require(Parse("[vr]\nimmersive_window = true\n").vrImmersiveWindow == true);
+    Require(!Parse("[vr]\n").vrImmersiveWindow.has_value());
+    Require(VrRaceViewOf(Parse("[vr]\n")) == VrRaceView::Immersive);
+    Require(VrRaceViewOf(Parse("[vr]\nflat_screen = false\n")) == VrRaceView::Immersive);
+    Require(VrRaceViewOf(Parse("[vr]\nflat_screen = true\n")) == VrRaceView::FlatScreen);
+    Require(VrRaceViewOf(Parse("[vr]\nimmersive_window = true\n")) == VrRaceView::ImmersiveWindow);
+    Require(VrRaceViewOf(Parse("[vr]\nflat_screen = true\nimmersive_window = true\n")) == VrRaceView::FlatScreen);
+    Require(VrRaceViewOf(Parse("[vr]\nflat_screen = false\nimmersive_window = false\n")) == VrRaceView::Immersive);
     return 0;
 }