diff --git a/README.md b/README.md index 0259647..4904003 100644 --- a/README.md +++ b/README.md @@ -123,6 +123,7 @@ settings back when it closes. | `[vr] foveation` | `medium` | `off` shades every pixel and costs the most. See [Known issues](#known-issues) if images double. | | `[vr] repeat_frames` | `true` (default) | Without it SteamVR halves the game's rate and fills refreshes itself. | | `[vr] frame_interpolation_fps` | `0` | Rendering in-between frames needs 120 eye pairs a second, which made things worse on the Frame. | +| `[vr] adaptive_resolution` | `false` (default) | Experimental and untested on the Frame. When on, races drop to as little as 70% of `render_scale` while new frames fall behind 60 FPS, and climb back once they keep up. Each step rebuilds the foveation maps; `console.log` records every change as `OpenXR: adaptive resolution`. | | `[video] resolution_multiplier` | `2` | The game's own frame, which the eyes are made from. 4x is far too heavy for the Frame's GPU. | Keep SteamVR's refresh rate at 120 Hz. Motion Smoothing makes no difference to this game. diff --git a/aurora-main/lib/webgpu/vulkan_win32_interop.cpp b/aurora-main/lib/webgpu/vulkan_win32_interop.cpp index 93f44f6..0c4a2af 100644 --- a/aurora-main/lib/webgpu/vulkan_win32_interop.cpp +++ b/aurora-main/lib/webgpu/vulkan_win32_interop.cpp @@ -150,8 +150,11 @@ public: for (uint32_t i = 0; i < images; ++i) { const auto& eye = sources[i]; const auto& target = targets[i]; - if (!eye.texture || eye.size.width != target.width || eye.size.height != target.height || - !CopyCompatible(VkFormat(eye.format), target.dxgiFormat)) return false; + // An eye smaller than its target goes in the target's top-left corner, which is all the + // projection layer shows of it; the panel and the virtual screen are always full size. + const bool fits = i < count ? eye.size.width <= target.width && eye.size.height <= target.height + : eye.size.width == target.width && eye.size.height == target.height; + if (!eye.texture || !fits || !CopyCompatible(VkFormat(eye.format), target.dxgiFormat)) return false; const uint64_t image = reinterpret_cast(target.resource); auto it = std::find_if(imports.begin(), imports.end(), [&](const Import& entry) { return entry.image == image; }); if (it == imports.end()) { @@ -179,7 +182,7 @@ public: wgpu::TexelCopyTextureInfo source, destination; source.texture = *sources[i].texture; destination.texture = active[i]; - wgpu::Extent3D size{targets[i].width, targets[i].height, 1}; + wgpu::Extent3D size{sources[i].size.width, sources[i].size.height, 1}; encoder.CopyTextureToTexture(&source, &destination, &size); } encoded = true; diff --git a/runtime/CMakeLists.txt b/runtime/CMakeLists.txt index e977a35..44d15f9 100644 --- a/runtime/CMakeLists.txt +++ b/runtime/CMakeLists.txt @@ -460,6 +460,10 @@ target_include_directories(mkw_vr_wii_remote_tests PRIVATE "${CMAKE_CURRENT_LIST target_compile_features(mkw_vr_wii_remote_tests PRIVATE cxx_std_17) add_test(NAME mkw_vr_wii_remote_tests COMMAND mkw_vr_wii_remote_tests) # Eye-tracked foveation: the gaze in each eye's view (vr/eye_gaze.h). +add_executable(mkw_vr_adaptive_resolution_tests "${CMAKE_CURRENT_LIST_DIR}/tests/vr_adaptive_resolution_tests.cpp") +target_include_directories(mkw_vr_adaptive_resolution_tests PRIVATE "${CMAKE_CURRENT_LIST_DIR}/include") +target_compile_features(mkw_vr_adaptive_resolution_tests PRIVATE cxx_std_17) +add_test(NAME mkw_vr_adaptive_resolution_tests COMMAND mkw_vr_adaptive_resolution_tests) add_executable(mkw_vr_eye_gaze_tests "${CMAKE_CURRENT_LIST_DIR}/tests/vr_eye_gaze_tests.cpp") target_include_directories(mkw_vr_eye_gaze_tests PRIVATE "${CMAKE_CURRENT_LIST_DIR}/include") target_compile_features(mkw_vr_eye_gaze_tests PRIVATE cxx_std_17) diff --git a/runtime/include/runtime_config.h b/runtime/include/runtime_config.h index e524dab..76b7d96 100644 --- a/runtime/include/runtime_config.h +++ b/runtime/include/runtime_config.h @@ -101,6 +101,7 @@ struct RuntimeUserConfig { std::optional vrFoveation; std::optional vrEyeTrackedFoveation; std::optional vrRepeatFrames; + std::optional vrAdaptiveResolution; std::optional vrRecenterKey; std::optional vrLeanBackDegrees; // F10 > Diagnostics: OpenXR pacing and presentation logging in console.log. @@ -607,6 +608,10 @@ inline void EnsureConfigFile() { "# Submit the last frame again for each refresh the game has no new\n" "# frame for, so the runtime does not fill those refreshes itself. Live.\n" "repeat_frames = " MKW_VR_REPEAT_FRAMES_DEFAULT_TOML "\n" + "# Lower the race's eye resolution, down to 70% of render_scale, while\n" + "# new frames fall behind the game's 60 FPS, and raise it again once\n" + "# they keep up. Experimental; the Steam Frame and the Quest. Live.\n" + "adaptive_resolution = false\n" "render_scale = " MKW_VR_RENDER_SCALE_DEFAULT_TEXT "\n" "world_units_per_meter = 500.0\n" "hud_distance_meters = 2.0\n" @@ -928,6 +933,7 @@ inline RuntimeUserConfig ParseConfigDocument(const toml::value& document) { } config.vrEyeTrackedFoveation = FindConfigValue(document, "vr", "eye_tracked_foveation"); config.vrRepeatFrames = FindConfigValue(document, "vr", "repeat_frames"); + config.vrAdaptiveResolution = FindConfigValue(document, "vr", "adaptive_resolution"); if (auto value = FindConfigValue(document, "vr", "mirror_view"); value && IsSupportedVrMirrorView(*value)) { config.vrMirrorView = *value; @@ -1345,6 +1351,11 @@ inline bool SetVrRepeatFrames(bool value) { return WriteSetting("vr", "repeat_frames", value ? "true" : "false"); } +inline bool SetVrAdaptiveResolution(bool value) { + Mutable().vrAdaptiveResolution = value; + return WriteSetting("vr", "adaptive_resolution", value ? "true" : "false"); +} + inline bool SetVrEyeTrackedFoveation(bool value) { Mutable().vrEyeTrackedFoveation = value; return WriteSetting("vr", "eye_tracked_foveation", value ? "true" : "false"); @@ -1880,6 +1891,10 @@ inline bool VrRepeatFrames(bool fallback = kVrRepeatFramesDefault) { return Get().vrRepeatFrames.value_or(fallback); } +inline bool VrAdaptiveResolution(bool fallback = false) { + return Get().vrAdaptiveResolution.value_or(fallback); +} + inline bool VrEyeTrackedFoveation(bool fallback = kVrEyeTrackedFoveationDefault) { return Get().vrEyeTrackedFoveation.value_or(fallback); } diff --git a/runtime/include/vr/adaptive_resolution.h b/runtime/include/vr/adaptive_resolution.h new file mode 100644 index 0000000..891ed0a --- /dev/null +++ b/runtime/include/vr/adaptive_resolution.h @@ -0,0 +1,69 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#pragma once + +// Adaptive resolution for the immersive eyes ([vr] adaptive_resolution), adapted from +// heurazy/Wiicompiled_VR-PLUS. Once a second the pacing thread reports how many new eye frames +// reached the headset and how many it should have had. A second well short of the target lowers +// the eyes' scale by a step; three seconds in a row at the target raise it again. The steps are +// slow so a single spike never resizes the eyes, which rebuilds their foveation maps. +// +// The eyes are rendered at the scaled size into a corner of the full swapchain image, and the +// projection layer shows that corner, so the compositor does the upscaling and the swapchains are +// never recreated. Nothing here depends on OpenXR (tests/vr_adaptive_resolution_tests.cpp). + +#include + +namespace mkw::vr { + +class AdaptiveResolution { +public: + static constexpr float kMinimumScale = 0.7f; + static constexpr float kStep = 0.1f; + // Below this share of the target a second counts as short, at or above the other as on time. + static constexpr float kShortShare = 0.85f; + static constexpr float kOnTimeShare = 0.97f; + static constexpr unsigned kOnTimeSecondsToRaise = 3; + + float Scale() const noexcept { return scale_; } + + // One second's measurement: `fps` new eye frames against `target_fps`. Off resets to full size. + float Observe(float fps, float target_fps, bool enabled) noexcept { + if (!enabled) { + scale_ = 1.0f; + on_time_seconds_ = 0; + return scale_; + } + if (!(target_fps > 0.0f) || !(fps > 0.0f)) { + return scale_; + } + if (fps < target_fps * kShortShare) { + scale_ = std::max(kMinimumScale, scale_ - kStep); + on_time_seconds_ = 0; + } else if (fps >= target_fps * kOnTimeShare) { + if (++on_time_seconds_ >= kOnTimeSecondsToRaise) { + scale_ = std::min(1.0f, scale_ + kStep); + on_time_seconds_ = 0; + } + } else { + on_time_seconds_ = 0; + } + // Whole steps only, so float drift never leaves the eyes a pixel off full size. + scale_ = static_cast(static_cast(scale_ / kStep + 0.5f)) * kStep; + return scale_; + } + + // The scaled size of an eye `full` pixels across, never below 2. + static unsigned Scaled(unsigned full, float scale) noexcept { + if (scale >= 1.0f) { + return full; + } + return std::max(2u, static_cast(static_cast(full) * scale)); + } + +private: + float scale_ = 1.0f; + unsigned on_time_seconds_ = 0; +}; + +} // namespace mkw::vr diff --git a/runtime/src/settings_overlay.cpp b/runtime/src/settings_overlay.cpp index 30ac3c5..ecc5926 100644 --- a/runtime/src/settings_overlay.cpp +++ b/runtime/src/settings_overlay.cpp @@ -159,6 +159,7 @@ int g_vrFoveation = static_cast(RuntimeConfigFile::VrFoveationLevelIndex(Ru bool g_vrEyeTrackedFoveation = RuntimeConfigFile::VrEyeTrackedFoveation(); #endif bool g_vrRepeatFrames = RuntimeConfigFile::VrRepeatFrames(); +bool g_vrAdaptiveResolution = RuntimeConfigFile::VrAdaptiveResolution(); bool g_vrFirstPerson = RuntimeConfigFile::VrFirstPerson(false); bool g_vrFirstPersonToggleClick = RuntimeConfigFile::VrFirstPersonToggleClick(); // Set from any thread by the right-thumbstick click, applied on the game thread. @@ -1568,6 +1569,18 @@ void DrawVrSettings() { "fill the gaps itself, which doubles the HUD and the menu screen as you turn your head " "(SteamVR on the Steam Frame). Applies immediately."); } +#if !defined(_WIN32) + // Only the Vulkan backends show part of an eye image; the PC's copy whole eyes. + if (ImGui::Checkbox("Adaptive resolution (experimental)", &g_vrAdaptiveResolution)) { + RuntimeConfigFile::SetVrAdaptiveResolution(g_vrAdaptiveResolution); + } + if (ImGui::IsItemHovered()) { + ImGui::SetTooltip( + "During races, lowers the eyes' resolution a step at a time, down to 70%%, while new " + "frames fall behind the game's 60 FPS, and raises it again once they keep up. Each change " + "rebuilds the foveation maps. Applies immediately."); + } +#endif if (ImGui::Combo("VR frame interpolation (experimental)", &g_vrFrameInterpolationMode, kVrInterpolationLabels.data(), static_cast(kVrInterpolationLabels.size()))) { const auto target = kVrInterpolationFps[static_cast(g_vrFrameInterpolationMode)]; diff --git a/runtime/src/vr/openxr_integration.cpp b/runtime/src/vr/openxr_integration.cpp index 7497b59..a76d6a6 100644 --- a/runtime/src/vr/openxr_integration.cpp +++ b/runtime/src/vr/openxr_integration.cpp @@ -9,6 +9,7 @@ #include "runtime_config.h" #include "gx_thread.h" #include "runtime_log.h" +#include "vr/adaptive_resolution.h" #include "vr/eye_gaze.h" #include "vr/mkw_vr_culling.h" #include "vr/mkw_vr_first_person.h" @@ -111,6 +112,15 @@ inline constexpr bool kWindowShapedEyesSupported = true; inline constexpr bool kWindowShapedEyesSupported = false; #endif +// Whether the immersive eyes can be rendered smaller than their swapchain images for +// [vr] adaptive_resolution: the backend copies them into a corner of the image and its projection +// layer shows that corner. The Vulkan backends do; the PC's D3D12 one copies whole eyes. +#if defined(_WIN32) +inline constexpr bool kScaledEyesSupported = false; +#else +inline constexpr bool kScaledEyesSupported = true; +#endif + struct Quaternion { float x = 0.0f; float y = 0.0f; @@ -1506,6 +1516,15 @@ private: destination.eyes[eye].height = source.render_height[eye]; } } + // The backend shows the part of each image this size fills, so the field of view is kept. + if (const float scale = adaptive_scale_.load(std::memory_order_relaxed); kScaledEyesSupported && scale < 1.0f) { + for (uint32_t eye = 0; eye < kOpenXREyeCount; ++eye) { + source.render_width[eye] = AdaptiveResolution::Scaled(source.render_width[eye], scale); + source.render_height[eye] = AdaptiveResolution::Scaled(source.render_height[eye], scale); + destination.eyes[eye].width = source.render_width[eye]; + destination.eyes[eye].height = source.render_height[eye]; + } + } // Read once so both eyes are built from the same angle even if the // settings slider moves between them. const float lean_back_radians = @@ -1879,7 +1898,21 @@ private: const auto now = std::chrono::steady_clock::now(); const float elapsed = std::chrono::duration(now - timing_start_).count(); if (elapsed >= 1.0f) { - rendered_fps_.store(static_cast(timing_submissions_) / elapsed, std::memory_order_relaxed); + const float fps = static_cast(timing_submissions_) / elapsed; + rendered_fps_.store(fps, std::memory_order_relaxed); + // Measured during races only: menus and loading screens run below 60 on their own. + // The target is the game's 60 FPS, or the headset's rate when frames are interpolated. + const bool adaptive = kScaledEyesSupported && RuntimeConfigFile::VrAdaptiveResolution(); + if (!adaptive || last_immersive_) { + const float target = aurora_get_stereo_frame_interpolation() ? hz : std::min(hz, 60.0f); + const float previous = adaptive_resolution_.Scale(); + const float scale = adaptive_resolution_.Observe(fps, target, adaptive); + adaptive_scale_.store(scale, std::memory_order_relaxed); + if (scale != previous) { + RT_LOG(RT_TAG_RUNTIME) << "OpenXR: adaptive resolution " << static_cast(scale * 100.0f + 0.5f) + << "% (" << fps << " of " << target << " FPS)" << std::endl; + } + } timing_start_ = now; timing_submissions_ = 0; } @@ -2031,6 +2064,9 @@ private: std::atomic rendered_fps_{0}; std::chrono::steady_clock::time_point timing_start_ = std::chrono::steady_clock::now(); uint32_t timing_submissions_ = 0; + // [vr] adaptive_resolution: measured on the pacing thread, read where eyes are published. + AdaptiveResolution adaptive_resolution_; + std::atomic adaptive_scale_{1.0f}; PFN_xrGetDisplayRefreshRateFB get_display_refresh_rate_ = nullptr; PFN_xrEnumerateDisplayRefreshRatesFB enumerate_refresh_rates_ = nullptr; PFN_xrRequestDisplayRefreshRateFB request_refresh_rate_ = nullptr; diff --git a/runtime/src/vr/openxr_vulkan_win32.cpp b/runtime/src/vr/openxr_vulkan_win32.cpp index bd6da99..52f7387 100644 --- a/runtime/src/vr/openxr_vulkan_win32.cpp +++ b/runtime/src/vr/openxr_vulkan_win32.cpp @@ -744,10 +744,11 @@ public: views[eye].pose.position = frame.xr_frame.views[eye].pose.position; views[eye].fov = frame.xr_frame.views[eye].fov; views[eye].subImage.swapchain = retained_swapchains_[eye].handle; + // The eyes may fill only a corner of the image ([vr] adaptive_resolution). views[eye].subImage.imageRect = { {0, 0}, - {static_cast(retained_swapchains_[eye].width), - static_cast(retained_swapchains_[eye].height)}}; + {static_cast(std::min(frame.render_width[eye], retained_swapchains_[eye].width)), + static_cast(std::min(frame.render_height[eye], retained_swapchains_[eye].height))}}; views[eye].subImage.imageArrayIndex = 0; } XrCompositionLayerProjection projection{XR_TYPE_COMPOSITION_LAYER_PROJECTION}; diff --git a/runtime/tests/vr_adaptive_resolution_tests.cpp b/runtime/tests/vr_adaptive_resolution_tests.cpp new file mode 100644 index 0000000..3468440 --- /dev/null +++ b/runtime/tests/vr_adaptive_resolution_tests.cpp @@ -0,0 +1,56 @@ +// SPDX-License-Identifier: GPL-3.0-or-later +#include "vr/adaptive_resolution.h" + +#include +#include +#include + +using mkw::vr::AdaptiveResolution; + +namespace { + +void Check(bool condition, const char* what) { + if (!condition) { + std::fprintf(stderr, "vr_adaptive_resolution_tests: %s\n", what); + std::exit(1); + } +} + +void CheckNear(float value, float expected, const char* what) { + Check(std::fabs(value - expected) <= 1.0e-5f, what); +} + +} // namespace + +int main() { + { + AdaptiveResolution adaptive; + CheckNear(adaptive.Observe(40.0f, 60.0f, false), 1.0f, "off stays at full size"); + CheckNear(adaptive.Observe(60.0f, 60.0f, true), 1.0f, "on time at full size stays there"); + } + { + AdaptiveResolution adaptive; + CheckNear(adaptive.Observe(45.0f, 60.0f, true), 0.9f, "a short second lowers a step"); + CheckNear(adaptive.Observe(45.0f, 60.0f, true), 0.8f, "another lowers another"); + CheckNear(adaptive.Observe(45.0f, 60.0f, true), 0.7f, "and another"); + CheckNear(adaptive.Observe(10.0f, 60.0f, true), 0.7f, "never below the minimum"); + CheckNear(adaptive.Observe(60.0f, 60.0f, true), 0.7f, "one second on time is not enough"); + CheckNear(adaptive.Observe(60.0f, 60.0f, true), 0.7f, "two are not enough"); + CheckNear(adaptive.Observe(60.0f, 60.0f, true), 0.8f, "three raise a step"); + CheckNear(adaptive.Observe(55.0f, 60.0f, true), 0.8f, "between the thresholds holds"); + CheckNear(adaptive.Observe(60.0f, 60.0f, true), 0.8f, "and restarts the on-time count"); + CheckNear(adaptive.Observe(60.0f, 60.0f, true), 0.8f, "still counting"); + CheckNear(adaptive.Observe(60.0f, 60.0f, true), 0.9f, "three more raise again"); + for (int second = 0; second < 30; ++second) adaptive.Observe(60.0f, 60.0f, true); + Check(adaptive.Scale() == 1.0f, "back at exactly full size"); + CheckNear(adaptive.Observe(0.0f, 60.0f, true), 1.0f, "a second with no measurement changes nothing"); + CheckNear(adaptive.Observe(30.0f, 0.0f, true), 1.0f, "nor one with no target"); + adaptive.Observe(30.0f, 60.0f, true); + CheckNear(adaptive.Observe(30.0f, 60.0f, false), 1.0f, "turning it off restores full size"); + } + Check(AdaptiveResolution::Scaled(2064, 1.0f) == 2064, "full size is unchanged"); + Check(AdaptiveResolution::Scaled(2000, 0.7f) == 1400, "scaled size"); + Check(AdaptiveResolution::Scaled(2, 0.7f) == 2, "never below two pixels"); + std::puts("vr_adaptive_resolution_tests: ok"); + return 0; +} diff --git a/runtime/tests/vr_config_tests.cpp b/runtime/tests/vr_config_tests.cpp index 400b870..b0ae0d9 100644 --- a/runtime/tests/vr_config_tests.cpp +++ b/runtime/tests/vr_config_tests.cpp @@ -84,6 +84,9 @@ int main() { Require(Parse("[vr]\nrepeat_frames = false\n").vrRepeatFrames == false); Require(!Parse("[vr]\nrepeat_frames = 1\n").vrRepeatFrames.has_value()); Require(!Parse("[vr]\n").vrRepeatFrames.has_value()); + // [vr] adaptive_resolution: off unless set. + Require(Parse("[vr]\nadaptive_resolution = true\n").vrAdaptiveResolution == true); + Require(!Parse("[vr]\n").vrAdaptiveResolution.has_value()); #if defined(MKW_HEADSET_STEAM_FRAME) Require(RuntimeConfigFile::kVrRepeatFramesDefault); Require(std::string_view(MKW_VR_REPEAT_FRAMES_DEFAULT_TOML) == "true");