Add adaptive resolution for the immersive eyes

[vr] adaptive_resolution (off by default, in the VR tab) lowers the
race's eye resolution by 10% steps, down to 70%, while new eye frames
fall below 85% of the game's 60 FPS, and raises it again after three
seconds on time. Adapted from heurazy/Wiicompiled_VR-PLUS.

The eyes render into the top-left corner of the full swapchain image
and the projection layer shows only that corner, so the compositor
upscales them and no swapchain is recreated, unlike render_scale. The
Linux same-device bridge now copies an eye smaller than its target;
the Quest's bridge and layer already did. The PC's D3D12 path copies
whole eyes, so the setting is not offered on Windows.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Sabhzu6VgoWcHtpSk4C9KZ

README.md

@@ -123,6 +123,7 @@ settings back when it closes.
 | `[vr] foveation` | `medium` | `off` shades every pixel and costs the most. See [Known issues](#known-issues) if images double. |
 | `[vr] repeat_frames` | `true` (default) | Without it SteamVR halves the game's rate and fills refreshes itself. |
 | `[vr] frame_interpolation_fps` | `0` | Rendering in-between frames needs 120 eye pairs a second, which made things worse on the Frame. |
+| `[vr] adaptive_resolution` | `false` (default) | Experimental and untested on the Frame. When on, races drop to as little as 70% of `render_scale` while new frames fall behind 60 FPS, and climb back once they keep up. Each step rebuilds the foveation maps; `console.log` records every change as `OpenXR: adaptive resolution`. |
 | `[video] resolution_multiplier` | `2` | The game's own frame, which the eyes are made from. 4x is far too heavy for the Frame's GPU. |
 
 Keep SteamVR's refresh rate at 120 Hz. Motion Smoothing makes no difference to this game.

aurora-main/lib/webgpu/vulkan_win32_interop.cpp

@@ -150,8 +150,11 @@ public:
     for (uint32_t i = 0; i < images; ++i) {
       const auto& eye = sources[i];
       const auto& target = targets[i];
-      if (!eye.texture || eye.size.width != target.width || eye.size.height != target.height ||
-          !CopyCompatible(VkFormat(eye.format), target.dxgiFormat)) return false;
+      // An eye smaller than its target goes in the target's top-left corner, which is all the
+      // projection layer shows of it; the panel and the virtual screen are always full size.
+      const bool fits = i < count ? eye.size.width <= target.width && eye.size.height <= target.height
+                                  : eye.size.width == target.width && eye.size.height == target.height;
+      if (!eye.texture || !fits || !CopyCompatible(VkFormat(eye.format), target.dxgiFormat)) return false;
       const uint64_t image = reinterpret_cast<uintptr_t>(target.resource);
       auto it = std::find_if(imports.begin(), imports.end(), [&](const Import& entry) { return entry.image == image; });
       if (it == imports.end()) {
@@ -179,7 +182,7 @@ public:
       wgpu::TexelCopyTextureInfo source, destination;
       source.texture = *sources[i].texture;
       destination.texture = active[i];
-      wgpu::Extent3D size{targets[i].width, targets[i].height, 1};
+      wgpu::Extent3D size{sources[i].size.width, sources[i].size.height, 1};
       encoder.CopyTextureToTexture(&source, &destination, &size);
     }
     encoded = true;

runtime/CMakeLists.txt

@@ -460,6 +460,10 @@ target_include_directories(mkw_vr_wii_remote_tests PRIVATE "${CMAKE_CURRENT_LIST
 target_compile_features(mkw_vr_wii_remote_tests PRIVATE cxx_std_17)
 add_test(NAME mkw_vr_wii_remote_tests COMMAND mkw_vr_wii_remote_tests)
 # Eye-tracked foveation: the gaze in each eye's view (vr/eye_gaze.h).
+add_executable(mkw_vr_adaptive_resolution_tests "${CMAKE_CURRENT_LIST_DIR}/tests/vr_adaptive_resolution_tests.cpp")
+target_include_directories(mkw_vr_adaptive_resolution_tests PRIVATE "${CMAKE_CURRENT_LIST_DIR}/include")
+target_compile_features(mkw_vr_adaptive_resolution_tests PRIVATE cxx_std_17)
+add_test(NAME mkw_vr_adaptive_resolution_tests COMMAND mkw_vr_adaptive_resolution_tests)
 add_executable(mkw_vr_eye_gaze_tests "${CMAKE_CURRENT_LIST_DIR}/tests/vr_eye_gaze_tests.cpp")
 target_include_directories(mkw_vr_eye_gaze_tests PRIVATE "${CMAKE_CURRENT_LIST_DIR}/include")
 target_compile_features(mkw_vr_eye_gaze_tests PRIVATE cxx_std_17)

runtime/include/runtime_config.h

@@ -101,6 +101,7 @@ struct RuntimeUserConfig {
     std::optional<std::string> vrFoveation;
     std::optional<bool> vrEyeTrackedFoveation;
     std::optional<bool> vrRepeatFrames;
+    std::optional<bool> vrAdaptiveResolution;
     std::optional<std::string> vrRecenterKey;
     std::optional<float> vrLeanBackDegrees;
     // F10 > Diagnostics: OpenXR pacing and presentation logging in console.log.
@@ -607,6 +608,10 @@ inline void EnsureConfigFile() {
               "# Submit the last frame again for each refresh the game has no new\n"
               "# frame for, so the runtime does not fill those refreshes itself. Live.\n"
               "repeat_frames = " MKW_VR_REPEAT_FRAMES_DEFAULT_TOML "\n"
+              "# Lower the race's eye resolution, down to 70% of render_scale, while\n"
+              "# new frames fall behind the game's 60 FPS, and raise it again once\n"
+              "# they keep up. Experimental; the Steam Frame and the Quest. Live.\n"
+              "adaptive_resolution = false\n"
               "render_scale = " MKW_VR_RENDER_SCALE_DEFAULT_TEXT "\n"
               "world_units_per_meter = 500.0\n"
               "hud_distance_meters = 2.0\n"
@@ -928,6 +933,7 @@ inline RuntimeUserConfig ParseConfigDocument(const toml::value& document) {
     }
     config.vrEyeTrackedFoveation = FindConfigValue<bool>(document, "vr", "eye_tracked_foveation");
     config.vrRepeatFrames = FindConfigValue<bool>(document, "vr", "repeat_frames");
+    config.vrAdaptiveResolution = FindConfigValue<bool>(document, "vr", "adaptive_resolution");
     if (auto value = FindConfigValue<std::string>(document, "vr", "mirror_view");
         value && IsSupportedVrMirrorView(*value)) {
         config.vrMirrorView = *value;
@@ -1345,6 +1351,11 @@ inline bool SetVrRepeatFrames(bool value) {
     return WriteSetting("vr", "repeat_frames", value ? "true" : "false");
 }
 
+inline bool SetVrAdaptiveResolution(bool value) {
+    Mutable().vrAdaptiveResolution = value;
+    return WriteSetting("vr", "adaptive_resolution", value ? "true" : "false");
+}
+
 inline bool SetVrEyeTrackedFoveation(bool value) {
     Mutable().vrEyeTrackedFoveation = value;
     return WriteSetting("vr", "eye_tracked_foveation", value ? "true" : "false");
@@ -1880,6 +1891,10 @@ inline bool VrRepeatFrames(bool fallback = kVrRepeatFramesDefault) {
     return Get().vrRepeatFrames.value_or(fallback);
 }
 
+inline bool VrAdaptiveResolution(bool fallback = false) {
+    return Get().vrAdaptiveResolution.value_or(fallback);
+}
+
 inline bool VrEyeTrackedFoveation(bool fallback = kVrEyeTrackedFoveationDefault) {
     return Get().vrEyeTrackedFoveation.value_or(fallback);
 }

runtime/include/vr/adaptive_resolution.h

@@ -0,0 +1,69 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#pragma once
+
+// Adaptive resolution for the immersive eyes ([vr] adaptive_resolution), adapted from
+// heurazy/Wiicompiled_VR-PLUS. Once a second the pacing thread reports how many new eye frames
+// reached the headset and how many it should have had. A second well short of the target lowers
+// the eyes' scale by a step; three seconds in a row at the target raise it again. The steps are
+// slow so a single spike never resizes the eyes, which rebuilds their foveation maps.
+//
+// The eyes are rendered at the scaled size into a corner of the full swapchain image, and the
+// projection layer shows that corner, so the compositor does the upscaling and the swapchains are
+// never recreated. Nothing here depends on OpenXR (tests/vr_adaptive_resolution_tests.cpp).
+
+#include <algorithm>
+
+namespace mkw::vr {
+
+class AdaptiveResolution {
+public:
+    static constexpr float kMinimumScale = 0.7f;
+    static constexpr float kStep = 0.1f;
+    // Below this share of the target a second counts as short, at or above the other as on time.
+    static constexpr float kShortShare = 0.85f;
+    static constexpr float kOnTimeShare = 0.97f;
+    static constexpr unsigned kOnTimeSecondsToRaise = 3;
+
+    float Scale() const noexcept { return scale_; }
+
+    // One second's measurement: `fps` new eye frames against `target_fps`. Off resets to full size.
+    float Observe(float fps, float target_fps, bool enabled) noexcept {
+        if (!enabled) {
+            scale_ = 1.0f;
+            on_time_seconds_ = 0;
+            return scale_;
+        }
+        if (!(target_fps > 0.0f) || !(fps > 0.0f)) {
+            return scale_;
+        }
+        if (fps < target_fps * kShortShare) {
+            scale_ = std::max(kMinimumScale, scale_ - kStep);
+            on_time_seconds_ = 0;
+        } else if (fps >= target_fps * kOnTimeShare) {
+            if (++on_time_seconds_ >= kOnTimeSecondsToRaise) {
+                scale_ = std::min(1.0f, scale_ + kStep);
+                on_time_seconds_ = 0;
+            }
+        } else {
+            on_time_seconds_ = 0;
+        }
+        // Whole steps only, so float drift never leaves the eyes a pixel off full size.
+        scale_ = static_cast<float>(static_cast<int>(scale_ / kStep + 0.5f)) * kStep;
+        return scale_;
+    }
+
+    // The scaled size of an eye `full` pixels across, never below 2.
+    static unsigned Scaled(unsigned full, float scale) noexcept {
+        if (scale >= 1.0f) {
+            return full;
+        }
+        return std::max(2u, static_cast<unsigned>(static_cast<float>(full) * scale));
+    }
+
+private:
+    float scale_ = 1.0f;
+    unsigned on_time_seconds_ = 0;
+};
+
+} // namespace mkw::vr

runtime/src/settings_overlay.cpp

@@ -159,6 +159,7 @@ int g_vrFoveation = static_cast<int>(RuntimeConfigFile::VrFoveationLevelIndex(Ru
 bool g_vrEyeTrackedFoveation = RuntimeConfigFile::VrEyeTrackedFoveation();
 #endif
 bool g_vrRepeatFrames = RuntimeConfigFile::VrRepeatFrames();
+bool g_vrAdaptiveResolution = RuntimeConfigFile::VrAdaptiveResolution();
 bool g_vrFirstPerson = RuntimeConfigFile::VrFirstPerson(false);
 bool g_vrFirstPersonToggleClick = RuntimeConfigFile::VrFirstPersonToggleClick();
 // Set from any thread by the right-thumbstick click, applied on the game thread.
@@ -1568,6 +1569,18 @@ void DrawVrSettings() {
             "fill the gaps itself, which doubles the HUD and the menu screen as you turn your head "
             "(SteamVR on the Steam Frame). Applies immediately.");
     }
+#if !defined(_WIN32)
+    // Only the Vulkan backends show part of an eye image; the PC's copy whole eyes.
+    if (ImGui::Checkbox("Adaptive resolution (experimental)", &g_vrAdaptiveResolution)) {
+        RuntimeConfigFile::SetVrAdaptiveResolution(g_vrAdaptiveResolution);
+    }
+    if (ImGui::IsItemHovered()) {
+        ImGui::SetTooltip(
+            "During races, lowers the eyes' resolution a step at a time, down to 70%%, while new "
+            "frames fall behind the game's 60 FPS, and raises it again once they keep up. Each change "
+            "rebuilds the foveation maps. Applies immediately.");
+    }
+#endif
     if (ImGui::Combo("VR frame interpolation (experimental)", &g_vrFrameInterpolationMode,
                      kVrInterpolationLabels.data(), static_cast<int>(kVrInterpolationLabels.size()))) {
         const auto target = kVrInterpolationFps[static_cast<size_t>(g_vrFrameInterpolationMode)];

runtime/src/vr/openxr_integration.cpp

@@ -9,6 +9,7 @@
 #include "runtime_config.h"
 #include "gx_thread.h"
 #include "runtime_log.h"
+#include "vr/adaptive_resolution.h"
 #include "vr/eye_gaze.h"
 #include "vr/mkw_vr_culling.h"
 #include "vr/mkw_vr_first_person.h"
@@ -111,6 +112,15 @@ inline constexpr bool kWindowShapedEyesSupported = true;
 inline constexpr bool kWindowShapedEyesSupported = false;
 #endif
 
+// Whether the immersive eyes can be rendered smaller than their swapchain images for
+// [vr] adaptive_resolution: the backend copies them into a corner of the image and its projection
+// layer shows that corner. The Vulkan backends do; the PC's D3D12 one copies whole eyes.
+#if defined(_WIN32)
+inline constexpr bool kScaledEyesSupported = false;
+#else
+inline constexpr bool kScaledEyesSupported = true;
+#endif
+
 struct Quaternion {
     float x = 0.0f;
     float y = 0.0f;
@@ -1506,6 +1516,15 @@ private:
                 destination.eyes[eye].height = source.render_height[eye];
             }
         }
+        // The backend shows the part of each image this size fills, so the field of view is kept.
+        if (const float scale = adaptive_scale_.load(std::memory_order_relaxed); kScaledEyesSupported && scale < 1.0f) {
+            for (uint32_t eye = 0; eye < kOpenXREyeCount; ++eye) {
+                source.render_width[eye] = AdaptiveResolution::Scaled(source.render_width[eye], scale);
+                source.render_height[eye] = AdaptiveResolution::Scaled(source.render_height[eye], scale);
+                destination.eyes[eye].width = source.render_width[eye];
+                destination.eyes[eye].height = source.render_height[eye];
+            }
+        }
         // Read once so both eyes are built from the same angle even if the
         // settings slider moves between them.
         const float lean_back_radians =
@@ -1879,7 +1898,21 @@ private:
         const auto now = std::chrono::steady_clock::now();
         const float elapsed = std::chrono::duration<float>(now - timing_start_).count();
         if (elapsed >= 1.0f) {
-            rendered_fps_.store(static_cast<float>(timing_submissions_) / elapsed, std::memory_order_relaxed);
+            const float fps = static_cast<float>(timing_submissions_) / elapsed;
+            rendered_fps_.store(fps, std::memory_order_relaxed);
+            // Measured during races only: menus and loading screens run below 60 on their own.
+            // The target is the game's 60 FPS, or the headset's rate when frames are interpolated.
+            const bool adaptive = kScaledEyesSupported && RuntimeConfigFile::VrAdaptiveResolution();
+            if (!adaptive || last_immersive_) {
+                const float target = aurora_get_stereo_frame_interpolation() ? hz : std::min(hz, 60.0f);
+                const float previous = adaptive_resolution_.Scale();
+                const float scale = adaptive_resolution_.Observe(fps, target, adaptive);
+                adaptive_scale_.store(scale, std::memory_order_relaxed);
+                if (scale != previous) {
+                    RT_LOG(RT_TAG_RUNTIME) << "OpenXR: adaptive resolution " << static_cast<int>(scale * 100.0f + 0.5f)
+                                           << "% (" << fps << " of " << target << " FPS)" << std::endl;
+                }
+            }
             timing_start_ = now;
             timing_submissions_ = 0;
         }
@@ -2031,6 +2064,9 @@ private:
     std::atomic<float> rendered_fps_{0};
     std::chrono::steady_clock::time_point timing_start_ = std::chrono::steady_clock::now();
     uint32_t timing_submissions_ = 0;
+    // [vr] adaptive_resolution: measured on the pacing thread, read where eyes are published.
+    AdaptiveResolution adaptive_resolution_;
+    std::atomic<float> adaptive_scale_{1.0f};
     PFN_xrGetDisplayRefreshRateFB get_display_refresh_rate_ = nullptr;
     PFN_xrEnumerateDisplayRefreshRatesFB enumerate_refresh_rates_ = nullptr;
     PFN_xrRequestDisplayRefreshRateFB request_refresh_rate_ = nullptr;

runtime/src/vr/openxr_vulkan_win32.cpp

@@ -744,10 +744,11 @@ public:
                 views[eye].pose.position = frame.xr_frame.views[eye].pose.position;
                 views[eye].fov = frame.xr_frame.views[eye].fov;
                 views[eye].subImage.swapchain = retained_swapchains_[eye].handle;
+                // The eyes may fill only a corner of the image ([vr] adaptive_resolution).
                 views[eye].subImage.imageRect = {
                     {0, 0},
-                    {static_cast<int32_t>(retained_swapchains_[eye].width),
-                     static_cast<int32_t>(retained_swapchains_[eye].height)}};
+                    {static_cast<int32_t>(std::min(frame.render_width[eye], retained_swapchains_[eye].width)),
+                     static_cast<int32_t>(std::min(frame.render_height[eye], retained_swapchains_[eye].height))}};
                 views[eye].subImage.imageArrayIndex = 0;
             }
             XrCompositionLayerProjection projection{XR_TYPE_COMPOSITION_LAYER_PROJECTION};

runtime/tests/vr_adaptive_resolution_tests.cpp

@@ -0,0 +1,56 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+#include "vr/adaptive_resolution.h"
+
+#include <cmath>
+#include <cstdio>
+#include <cstdlib>
+
+using mkw::vr::AdaptiveResolution;
+
+namespace {
+
+void Check(bool condition, const char* what) {
+    if (!condition) {
+        std::fprintf(stderr, "vr_adaptive_resolution_tests: %s\n", what);
+        std::exit(1);
+    }
+}
+
+void CheckNear(float value, float expected, const char* what) {
+    Check(std::fabs(value - expected) <= 1.0e-5f, what);
+}
+
+} // namespace
+
+int main() {
+    {
+        AdaptiveResolution adaptive;
+        CheckNear(adaptive.Observe(40.0f, 60.0f, false), 1.0f, "off stays at full size");
+        CheckNear(adaptive.Observe(60.0f, 60.0f, true), 1.0f, "on time at full size stays there");
+    }
+    {
+        AdaptiveResolution adaptive;
+        CheckNear(adaptive.Observe(45.0f, 60.0f, true), 0.9f, "a short second lowers a step");
+        CheckNear(adaptive.Observe(45.0f, 60.0f, true), 0.8f, "another lowers another");
+        CheckNear(adaptive.Observe(45.0f, 60.0f, true), 0.7f, "and another");
+        CheckNear(adaptive.Observe(10.0f, 60.0f, true), 0.7f, "never below the minimum");
+        CheckNear(adaptive.Observe(60.0f, 60.0f, true), 0.7f, "one second on time is not enough");
+        CheckNear(adaptive.Observe(60.0f, 60.0f, true), 0.7f, "two are not enough");
+        CheckNear(adaptive.Observe(60.0f, 60.0f, true), 0.8f, "three raise a step");
+        CheckNear(adaptive.Observe(55.0f, 60.0f, true), 0.8f, "between the thresholds holds");
+        CheckNear(adaptive.Observe(60.0f, 60.0f, true), 0.8f, "and restarts the on-time count");
+        CheckNear(adaptive.Observe(60.0f, 60.0f, true), 0.8f, "still counting");
+        CheckNear(adaptive.Observe(60.0f, 60.0f, true), 0.9f, "three more raise again");
+        for (int second = 0; second < 30; ++second) adaptive.Observe(60.0f, 60.0f, true);
+        Check(adaptive.Scale() == 1.0f, "back at exactly full size");
+        CheckNear(adaptive.Observe(0.0f, 60.0f, true), 1.0f, "a second with no measurement changes nothing");
+        CheckNear(adaptive.Observe(30.0f, 0.0f, true), 1.0f, "nor one with no target");
+        adaptive.Observe(30.0f, 60.0f, true);
+        CheckNear(adaptive.Observe(30.0f, 60.0f, false), 1.0f, "turning it off restores full size");
+    }
+    Check(AdaptiveResolution::Scaled(2064, 1.0f) == 2064, "full size is unchanged");
+    Check(AdaptiveResolution::Scaled(2000, 0.7f) == 1400, "scaled size");
+    Check(AdaptiveResolution::Scaled(2, 0.7f) == 2, "never below two pixels");
+    std::puts("vr_adaptive_resolution_tests: ok");
+    return 0;
+}

runtime/tests/vr_config_tests.cpp

@@ -84,6 +84,9 @@ int main() {
     Require(Parse("[vr]\nrepeat_frames = false\n").vrRepeatFrames == false);
     Require(!Parse("[vr]\nrepeat_frames = 1\n").vrRepeatFrames.has_value());
     Require(!Parse("[vr]\n").vrRepeatFrames.has_value());
+    // [vr] adaptive_resolution: off unless set.
+    Require(Parse("[vr]\nadaptive_resolution = true\n").vrAdaptiveResolution == true);
+    Require(!Parse("[vr]\n").vrAdaptiveResolution.has_value());
 #if defined(MKW_HEADSET_STEAM_FRAME)
     Require(RuntimeConfigFile::kVrRepeatFramesDefault);
     Require(std::string_view(MKW_VR_REPEAT_FRAMES_DEFAULT_TOML) == "true");