From 1346e53cd69eca0febd1f9f68131176cc0817f12 Mon Sep 17 00:00:00 2001 From: iChris4 Date: Thu, 17 Sep 2026 04:32:22 +0200 Subject: [PATCH] Implement OpenXR Settings Panel and Stereo Overlay --- OPENXR.md | 48 ++- aurora-main/cmake/aurora_core.cmake | 2 +- aurora-main/include/aurora/imgui.h | 13 + aurora-main/lib/aurora.cpp | 22 ++ aurora-main/lib/gfx/common.cpp | 4 + aurora-main/lib/gfx/common.hpp | 2 + aurora-main/lib/gfx/stereo_replay.hpp | 58 ++++ aurora-main/lib/imgui.cpp | 30 ++ aurora-main/lib/imgui.hpp | 12 + aurora-main/lib/stereo_overlay.cpp | 343 +++++++++++++++++++++ aurora-main/lib/stereo_overlay.hpp | 33 ++ aurora-main/tests/stereo_replay_test.cpp | 64 ++++ docs/quest-port.md | 2 +- runtime/CMakeLists.txt | 8 + runtime/include/vr/openxr_input.h | 24 +- runtime/include/vr/openxr_settings_panel.h | 165 ++++++++++ runtime/src/main.cpp | 50 ++- runtime/src/settings_overlay.cpp | 200 ++++++++++-- runtime/src/vr/openxr_input.cpp | 123 ++++++-- runtime/src/vr/openxr_integration.cpp | 122 +++++--- runtime/src/vr/openxr_settings_panel.cpp | 52 ++++ runtime/tests/vr_settings_panel_tests.cpp | 190 ++++++++++++ 22 files changed, 1466 insertions(+), 101 deletions(-) create mode 100644 aurora-main/lib/stereo_overlay.cpp create mode 100644 aurora-main/lib/stereo_overlay.hpp create mode 100644 runtime/include/vr/openxr_settings_panel.h create mode 100644 runtime/src/vr/openxr_settings_panel.cpp create mode 100644 runtime/tests/vr_settings_panel_tests.cpp diff --git a/OPENXR.md b/OPENXR.md index 2b2a395..d62cf2d 100644 --- a/OPENXR.md +++ b/OPENXR.md @@ -182,6 +182,9 @@ cursor for 100 ms before it disappears, so tracking spikes during fast motion do Raw IR camera dots in `KPADGetUnifiedWpadStatus` stay invalid; the game reads the pointer from `KPADStatus`. +**Settings in the headset.** Clicking both thumbsticks together opens the settings panel described +below; while it is open the controllers operate the panel and the game sees them idle. + `"gamepad"` keeps the controllers one ordinary gamepad read through PAD as a GameCube controller: A/B → South/East, X/Y → West/North, index triggers → trigger axes, grips → shoulders, thumbsticks → sticks (clicks → stick buttons), left menu → Start. Every binding in the F10 controller menu @@ -191,6 +194,45 @@ Bindings are suggested for `oculus/touch_controller` (Quest 2, 3 and Pro) and `khr/simple_controller`. `mkw_vr_wii_remote_tests` checks the accelerometer frame, the pointer raycast and debounce, the picture placement and the button profile without a headset. +## Settings in the headset + +The F10 settings bar is only visible on the desktop window, so the same settings are also offered on +a panel inside the headset, in menus and during an immersive race alike, including on the Quest. +**Click both thumbsticks together** to open it, and again to close it; the left controller's menu +button and the panel's *Close* button also close it. It can be opened from the desktop as well, with +**F10 → VR → Show these settings in the headset**. + +The panel has the F10 bar's menus as tabs (VR, Graphics, Controllers, Audio, Diagnostics) and a +*Recenter view* button. Aim a controller at it: the cursor goes where you aim, a trigger (or A / X) +selects and drags sliders, and a thumbstick scrolls. Whichever hand last pulled its trigger does the +pointing. Changes apply exactly as they do from the F10 bar, and the two stay in step. + +While the panel is open, and until every button has been released after it closes, the game sees +the VR controllers idle: no buttons, no pointer and a remote at rest. Nothing reaches the game from +the chord, the trigger that clicked *Close*, or the menu press that closed the panel. The game is +not paused, so a race carries on while you change settings. Other controllers (keyboard, desktop +gamepads, Bluetooth remotes) are not affected. + +The panel sits centred on the virtual screen, three quarters of its width across (1.8 m with the +default `hud_width_meters`). On a menu that is the anchored menu quad; in a race it is the 2D layer's +screen, `hud_distance_meters` ahead of the latched race origin and turned by the lean-back angle, +whether or not `hud_virtual_screen` places the HUD there. **Recenter view** brings both back in front +of you. + +How it is drawn: `settings_overlay.cpp` builds the panel with a second Dear ImGui context of its own, +a 1440 × 1080 canvas at twice the desktop menu's scale with its own font atlas, fed by the pointer +that `openxr_input.cpp` publishes through `vr/openxr_settings_panel.h`. Aurora renders that draw data +into a panel texture once per sealed frame and lays it over each eye after the eye is finished +(`aurora-main/lib/stereo_overlay.cpp`): through the eye's frustum and `viewFromCenter` onto the +screen rectangle for an immersive eye (including headset-rate interpolated eyes, which reuse the +texture), and as a centred rectangle on a virtual-screen eye image. The eye images the OpenXR +backends already submit carry it, so no extra swapchain or composition layer is involved. The +ImGui backend keeps a single projection uniform, so the panel's pass is submitted on its own command +buffer before the desktop's ImGui pass of the same frame is recorded. + +`mkw_vr_settings_panel_tests` covers the chord, the release latch, selection, scrolling and the +canvas mapping; `gx_fifo_tests` covers where the panel lands in each eye. + ## The first-person camera By default the headset sits where Mario Kart's own chase camera sits, and `world_units_per_meter` @@ -448,8 +490,10 @@ ends, including mid-frame flushes, so live setting changes cannot invalidate pen - The Quest build (`android/`, `docs/quest-port.md`) runs on a Quest 3 through menus and races. Lifecycle events and performance (about 43 game FPS) are still open. Apple visionOS packaging is not implemented. -- Scene-specific comfort options, culling fixes, replay/spectator classification, and a broader VR - settings UI beyond the current enable/replay controls are future work. +- Scene-specific comfort options, culling fixes and replay/spectator classification are future work. +- The headset settings panel is drawn into the eye images rather than submitted as its own quad + layer, so its text is resampled once more than a compositor layer's would be. It has no laser + beam, only the cursor on the panel itself, and text fields cannot be typed into without a keyboard. - The desktop window remains available as a mirror/fallback. OpenXR diagnostics are written to the normal run log under diff --git a/aurora-main/cmake/aurora_core.cmake b/aurora-main/cmake/aurora_core.cmake index 3811afc..2f47945 100644 --- a/aurora-main/cmake/aurora_core.cmake +++ b/aurora-main/cmake/aurora_core.cmake @@ -26,7 +26,7 @@ elseif (APPLE) endif () if (AURORA_ENABLE_GX) - target_sources(aurora_core PRIVATE lib/imgui.cpp) + target_sources(aurora_core PRIVATE lib/imgui.cpp lib/stereo_overlay.cpp) target_link_libraries(aurora_core PUBLIC imgui) endif () diff --git a/aurora-main/include/aurora/imgui.h b/aurora-main/include/aurora/imgui.h index 77eba9a..57fe838 100644 --- a/aurora-main/include/aurora/imgui.h +++ b/aurora-main/include/aurora/imgui.h @@ -13,6 +13,19 @@ extern "C" { ImTextureID aurora_imgui_add_texture(uint32_t width, uint32_t height, const void* rgba8); +// A panel shown only in the headset, drawn by the host with a Dear ImGui context of its own (for +// instance a settings menu reachable without the desktop window). It is laid over whatever the eyes +// carry: centred on the virtual screen, `widthFraction` of that screen's width, its height following +// the draw data's aspect. On a menu frame the screen is the quad the eye image is shown on; in an +// immersive race it is the 2D layer's screen (aurora_set_stereo_hud_screen's width and distance, +// whether or not the 2D layer is placed on it). +// +// The draw data belongs to the host's context and is rendered with Aurora's ImGui backend, so its +// texture ids must be views that backend can bind (aurora_imgui_add_texture). Call this from the +// producer before the frame is sealed, and leave the draw data untouched until the frame worker is +// done with that frame (aurora_wait_for_frame_worker). Null hides the panel. +void aurora_imgui_set_stereo_overlay(ImDrawData* drawData, float widthFraction); + #ifdef __cplusplus } #endif diff --git a/aurora-main/lib/aurora.cpp b/aurora-main/lib/aurora.cpp index 70537a2..eb2260e 100644 --- a/aurora-main/lib/aurora.cpp +++ b/aurora-main/lib/aurora.cpp @@ -10,6 +10,7 @@ #include "stereo.hpp" #include "stereo_mirror.hpp" #include "stereo_interpolation.hpp" +#include "stereo_overlay.hpp" #include "webgpu/gpu.hpp" #include #endif @@ -1613,6 +1614,7 @@ void shutdown() noexcept { g_stereoEyeTargets = {}; g_stereoMirrorState.Reset(); g_presentationImagePools = {}; + stereo_overlay::shutdown(); imgui::shutdown(); gfx::shutdown(); webgpu::shutdown(); @@ -1734,6 +1736,7 @@ struct SealedFrameContext { bool replayInterpolatedFrames = false; std::optional stereoInput; bool retainStereo = false; + imgui::StereoOverlay stereoOverlay; }; // Worker-owned scene state. A separate buffer generation check protects against @@ -1779,6 +1782,9 @@ void run_retained_stereo_frame(gfx::SealedFrame& sealedFrame) noexcept { return; for (uint32_t eye = 0; eye < AURORA_STEREO_EYE_COUNT; ++eye) { gfx::render_stereo_eye(sealedFrame, encoder, replay, eye, false); + // The panel texture from the last sealed frame; its draw data is not ours to touch here. + stereo_overlay::composite_immersive(encoder, g_stereoEyeTargets[eye].output().view, replay.eyes[eye].projection, + replay.eyes[eye].viewFromCenter, eye); } const auto sink = run_stereo_sink(encoder, input->frameToken, g_retainedStereo.logicalFrame, input->mode); const auto buffer = encoder.Finish(); @@ -1847,6 +1853,8 @@ void seal_frame_locked(gfx::SealedFrame& sealedFrame, SealedFrameContext& ctx, u // ImGui draw lists are built once per frame and replayed by each slot's ImGui pass, which is why // the next ImGui frame cannot start until the encode phase is done. imgui::render_frame_data(); + // The headset panel's draw data follows the same rule on the host's side. + ctx.stereoOverlay = imgui::latch_stereo_overlay(); // Drop the sealed frame's lazy RAM-readback requests while the producer is still excluded; it // starts registering the next frame's as soon as SEALED is published. gfx::efb_ram::cancel(); @@ -1923,6 +1931,15 @@ std::vector encode_sealed_frame(gfx::SealedFrame& sealedFrame, } }; + // Ahead of every other ImGui pass of this frame: the ImGui backend keeps one projection uniform, + // and the headset panel's canvas is not the desktop's size, so its pass has to reach the queue + // before a desktop pass rewrites that uniform. The eyes below sample the texture it fills. + if (const auto panel = stereo_overlay::prepare(stereo_frame_provider_active() ? ctx.stereoOverlay.drawData : nullptr, + ctx.stereoOverlay.widthFraction)) { + std::lock_guard submitLock(g_queueSubmitMutex); + g_queue.Submit(1, &panel); + } + if (ctx.replayInterpolatedFrames) { for (uint32_t interpolatedFrame = 0; interpolatedFrame < ctx.interpolatedFrameCount; ++interpolatedFrame) { gfx::render(sealedFrame, encoder, static_cast(interpolatedFrame), false); @@ -1976,6 +1993,9 @@ std::vector encode_sealed_frame(gfx::SealedFrame& sealedFrame, for (uint32_t eye = 0; eye < AURORA_STEREO_EYE_COUNT; ++eye) { gfx::render_stereo_eye(sealedFrame, encoder, *ctx.stereoReplay, eye, !ctx.retainStereo && eye + 1 == AURORA_STEREO_EYE_COUNT); + stereo_overlay::composite_immersive(encoder, g_stereoEyeTargets[eye].output().view, + ctx.stereoReplay->eyes[eye].projection, + ctx.stereoReplay->eyes[eye].viewFromCenter, eye); } } @@ -2003,6 +2023,8 @@ std::vector encode_sealed_frame(gfx::SealedFrame& sealedFrame, }; for (uint32_t eye = 0; eye < AURORA_STEREO_EYE_COUNT; ++eye) { encode_virtual_screen_eye(encoder, completedMono, eye); + const auto& output = g_stereoEyeTargets[eye].output(); + stereo_overlay::composite_flat(encoder, output.view, output.size, eye); } if (mirrorPlan == MirrorPlan::Black && !headsetOnly) { encode_presentation_snapshot(encoder, ctx.presentSource, *finalImage, true, MirrorPlan::Black); diff --git a/aurora-main/lib/gfx/common.cpp b/aurora-main/lib/gfx/common.cpp index 4acabf3..1d9e499 100644 --- a/aurora-main/lib/gfx/common.cpp +++ b/aurora-main/lib/gfx/common.cpp @@ -238,6 +238,10 @@ void set_stereo_hud_screen(bool enabled, float width, float distance) noexcept { g_stereoHudScreenEnabled.store(enabled, std::memory_order_relaxed); } bool get_stereo_hud_screen_enabled() noexcept { return g_stereoHudScreenEnabled.load(std::memory_order_relaxed); } +void get_stereo_hud_screen_size(float& width, float& distance) noexcept { + width = g_stereoHudScreenWidth.load(std::memory_order_relaxed); + distance = g_stereoHudScreenDistance.load(std::memory_order_relaxed); +} // The desktop mirror choice. Normal is the ordinary mono presentation, so a // build that never touches this setting presents exactly as it did before. diff --git a/aurora-main/lib/gfx/common.hpp b/aurora-main/lib/gfx/common.hpp index 493e9a5..613f3db 100644 --- a/aurora-main/lib/gfx/common.hpp +++ b/aurora-main/lib/gfx/common.hpp @@ -400,6 +400,8 @@ bool get_stereo_skip_copy_clears() noexcept; // the game's presented aspect ratio. Also live. void set_stereo_hud_screen(bool enabled, float width, float distance) noexcept; bool get_stereo_hud_screen_enabled() noexcept; +// The screen's width and distance in world units, kept while the 2D layer is off it. +void get_stereo_hud_screen_size(float& width, float& distance) noexcept; // What the desktop window presents while a stereo provider is feeding a // headset. Live, and read once per presentation group by the frame worker. diff --git a/aurora-main/lib/gfx/stereo_replay.hpp b/aurora-main/lib/gfx/stereo_replay.hpp index 24f69e3..45a97c8 100644 --- a/aurora-main/lib/gfx/stereo_replay.hpp +++ b/aurora-main/lib/gfx/stereo_replay.hpp @@ -306,4 +306,62 @@ inline Mat4x4 compose_hud_screen_projection(const Mat4x4& eyeFrust return out; } +// The headset settings panel (aurora_imgui_set_stereo_overlay): a rectangle +// centred on the virtual screen, in the same world units as the screen. +struct OverlayPanel { + float halfWidth = 0.0f; + float halfHeight = 0.0f; + float distance = 0.0f; + + [[nodiscard]] bool valid() const noexcept { return halfWidth > 0.0f && halfHeight > 0.0f && distance > 0.0f; } +}; + +// `widthFraction` of a virtual screen `screenWidth` across and `screenDistance` +// ahead, its height following the panel's own aspect. +inline OverlayPanel overlay_panel_on_screen(float screenWidth, float screenDistance, float widthFraction, + float panelAspect) noexcept { + if (!(panelAspect > 0.0f)) { + return {}; + } + const float halfWidth = 0.5f * screenWidth * widthFraction; + return {.halfWidth = halfWidth, .halfHeight = halfWidth / panelAspect, .distance = screenDistance}; +} + +// Clip position of a point on the panel for one eye of an immersive frame, as a +// matrix applied to (x, y, 0, 1) with x and y running -1..1 across the panel, +// +y up. It is the 2D layer's chain with the panel's corners in place of a +// game draw's NDC, so the panel sits exactly where the race HUD's screen does. +// The panel is drawn over the finished eye with no depth test, so its depth is +// simply parked mid-volume. +inline Mat4x4 compose_overlay_panel_projection(const Mat4x4& eyeFrustum, + const Mat3x4& viewFromCenter, + const OverlayPanel& panel) noexcept { + Mat4x4 corners{}; + corners.m0 = {1.0f, 0.0f, 0.0f, 0.0f}; + corners.m1 = {0.0f, 1.0f, 0.0f, 0.0f}; + corners.m3 = {0.0f, 0.0f, 0.0f, 1.0f}; + const HudScreen screen{.halfWidth = panel.halfWidth, .halfHeight = panel.halfHeight, .distance = panel.distance}; + auto out = compose_hud_screen_projection(eyeFrustum, viewFromCenter, screen, corners); + for (size_t i = 0; i < 4; ++i) { + out.m2[i] = 0.5f * out.m3[i]; + } + return out; +} + +// The same panel on a virtual-screen eye image, which the runtime shows as a +// quad layer the screen's width across: a centred rectangle `widthFraction` of +// the image's width, with the panel's aspect. +inline Mat4x4 overlay_panel_flat_projection(float widthFraction, float panelAspect, + float imageAspect) noexcept { + Mat4x4 out{}; + if (!(panelAspect > 0.0f) || !(imageAspect > 0.0f)) { + return out; + } + out.m0 = {widthFraction, 0.0f, 0.0f, 0.0f}; + out.m1 = {0.0f, widthFraction * imageAspect / panelAspect, 0.0f, 0.0f}; + out.m2 = {0.0f, 0.0f, 0.0f, 0.5f}; + out.m3 = {0.0f, 0.0f, 0.0f, 1.0f}; + return out; +} + } // namespace aurora::gfx::stereo_replay diff --git a/aurora-main/lib/imgui.cpp b/aurora-main/lib/imgui.cpp index 15a1fe1..08e6c5e 100644 --- a/aurora-main/lib/imgui.cpp +++ b/aurora-main/lib/imgui.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include @@ -32,6 +33,10 @@ static bool g_frameDataBuilt = false; static std::vector g_sdlTextures; static std::vector g_wgpuTextures; +// Set by the producer, latched by the seal. +static std::mutex g_stereoOverlayMutex; +static StereoOverlay g_stereoOverlay; + void remove_legacy_ini_file(const char* basePath) noexcept { if (basePath == nullptr || *basePath == '\0') { return; @@ -200,6 +205,23 @@ void render(const wgpu::RenderPassEncoder& pass) noexcept { } } +StereoOverlay latch_stereo_overlay() noexcept { + std::lock_guard lock(g_stereoOverlayMutex); + return g_stereoOverlay; +} + +bool render_draw_data(const wgpu::RenderPassEncoder& pass, ImDrawData* data) noexcept { + ZoneScoped; + // The SDL renderer fallback has no render passes to draw into. + if (g_useSdlRenderer || data == nullptr || ImGui::GetCurrentContext() == nullptr) { + return false; + } + pass.PushDebugGroup("Aurora: Dear Imgui headset panel"); + ImGui_ImplWGPU_RenderDrawData(data, pass.Get()); + pass.PopDebugGroup(); + return true; +} + ImTextureID add_texture(uint32_t width, uint32_t height, const uint8_t* data) noexcept { if (SDL_Renderer* renderer = window::get_sdl_renderer()) { SDL_Texture* texture = SDL_CreateTexture(renderer, SDL_PIXELFORMAT_RGBA32, SDL_TEXTUREACCESS_STATIC, width, height); @@ -251,4 +273,12 @@ extern "C" { ImTextureID aurora_imgui_add_texture(uint32_t width, uint32_t height, const void* rgba8) { return aurora::imgui::add_texture(width, height, static_cast(rgba8)); } + +void aurora_imgui_set_stereo_overlay(ImDrawData* drawData, float widthFraction) { + std::lock_guard lock(aurora::imgui::g_stereoOverlayMutex); + aurora::imgui::g_stereoOverlay = { + .drawData = drawData, + .widthFraction = drawData != nullptr ? widthFraction : 0.f, + }; +} } diff --git a/aurora-main/lib/imgui.hpp b/aurora-main/lib/imgui.hpp index 908a122..f2f3345 100644 --- a/aurora-main/lib/imgui.hpp +++ b/aurora-main/lib/imgui.hpp @@ -3,6 +3,7 @@ #include union SDL_Event; +struct ImDrawData; namespace wgpu { class RenderPassEncoder; @@ -20,4 +21,15 @@ void new_frame(const AuroraWindowSize& size) noexcept; // once and every presentation slot replays them. Reset by new_frame. void render_frame_data() noexcept; void render(const wgpu::RenderPassEncoder& pass) noexcept; + +// The headset panel as aurora_imgui_set_stereo_overlay last set it. +struct StereoOverlay { + ImDrawData* drawData = nullptr; + float widthFraction = 0.f; +}; +StereoOverlay latch_stereo_overlay() noexcept; +// Renders another context's draw data with this context's backend. The backend keeps one projection +// uniform for every pass, so a pass whose display size differs from the desktop's must be submitted +// before the next pass is recorded. +bool render_draw_data(const wgpu::RenderPassEncoder& pass, ImDrawData* data) noexcept; } // namespace aurora::imgui diff --git a/aurora-main/lib/stereo_overlay.cpp b/aurora-main/lib/stereo_overlay.cpp new file mode 100644 index 0000000..1f0a9f1 --- /dev/null +++ b/aurora-main/lib/stereo_overlay.cpp @@ -0,0 +1,343 @@ +#include "stereo_overlay.hpp" + +#include "gfx/common.hpp" +#include "gfx/stereo_replay.hpp" +#include "imgui.hpp" +#include "webgpu/gpu.hpp" + +#include +#include + +#include "tracy/Tracy.hpp" + +#include +#include + +namespace aurora::stereo_overlay { +namespace { + +using webgpu::g_device; +using webgpu::g_queue; + +// Uploaded as-is into a WGSL mat4x4, like the GX uniforms' matrices. +static_assert(sizeof(Mat4x4) == 64); + +// The panel's corners, (-1, 1) top left to (1, -1) bottom right, carried through +// one clip-from-panel matrix per eye. UVs are interpolated perspective-correct, +// so the texture stays straight on a panel seen at an angle. The texture holds +// premultiplied colour, which is what ImGui's blending leaves in a cleared target. +constexpr const char* kShader = R"""( +struct Panel { + clip_from_panel: mat4x4, +}; +@group(0) @binding(0) +var panel: Panel; +@group(0) @binding(1) +var panel_sampler: sampler; +@group(0) @binding(2) +var panel_texture: texture_2d; + +struct VertexOutput { + @builtin(position) pos: vec4, + @location(0) uv: vec2, +}; + +var corners: array, 4> = array, 4>( + vec2(-1.0, 1.0), + vec2(-1.0, -1.0), + vec2(1.0, 1.0), + vec2(1.0, -1.0), +); + +@vertex +fn vs_main(@builtin(vertex_index) vtxIdx: u32) -> VertexOutput { + let corner = corners[vtxIdx]; + var out: VertexOutput; + // Row-vector convention, like the GX shaders: m0..m3 are the clip x/y/z/w rows. + out.pos = vec4(corner, 0.0, 1.0) * panel.clip_from_panel; + out.uv = vec2(0.5 + 0.5 * corner.x, 0.5 - 0.5 * corner.y); + return out; +} + +@fragment +fn fs_main(in: VertexOutput) -> @location(0) vec4 { + return textureSample(panel_texture, panel_sampler, in.uv); +} +)"""; + +struct State { + webgpu::TextureWithSampler panel; + wgpu::RenderPipeline pipeline; + wgpu::BindGroupLayout bindGroupLayout; + wgpu::TextureFormat pipelineFormat = wgpu::TextureFormat::Undefined; + std::array uniforms; + std::array bindGroups; + float widthFraction = 0.f; + bool visible = false; +}; +State g_state; + +bool ensure_pipeline() { + auto& state = g_state; + const auto format = webgpu::g_graphicsConfig.surfaceConfiguration.format; + if (state.pipeline && state.pipelineFormat == format) { + return true; + } + state.pipeline = {}; + state.bindGroups = {}; + + wgpu::ShaderSourceWGSL source{}; + source.code = kShader; + const wgpu::ShaderModuleDescriptor moduleDescriptor{ + .nextInChain = &source, + .label = "Headset panel module", + }; + const auto module = g_device.CreateShaderModule(&moduleDescriptor); + + const std::array layoutEntries{ + wgpu::BindGroupLayoutEntry{ + .binding = 0, + .visibility = wgpu::ShaderStage::Vertex, + .buffer = + wgpu::BufferBindingLayout{ + .type = wgpu::BufferBindingType::Uniform, + .minBindingSize = sizeof(Mat4x4), + }, + }, + wgpu::BindGroupLayoutEntry{ + .binding = 1, + .visibility = wgpu::ShaderStage::Fragment, + .sampler = + wgpu::SamplerBindingLayout{ + .type = wgpu::SamplerBindingType::Filtering, + }, + }, + wgpu::BindGroupLayoutEntry{ + .binding = 2, + .visibility = wgpu::ShaderStage::Fragment, + .texture = + wgpu::TextureBindingLayout{ + .sampleType = wgpu::TextureSampleType::Float, + .viewDimension = wgpu::TextureViewDimension::e2D, + }, + }, + }; + const wgpu::BindGroupLayoutDescriptor layoutDescriptor{ + .label = "Headset panel bind group layout", + .entryCount = layoutEntries.size(), + .entries = layoutEntries.data(), + }; + state.bindGroupLayout = g_device.CreateBindGroupLayout(&layoutDescriptor); + const wgpu::PipelineLayoutDescriptor pipelineLayoutDescriptor{ + .label = "Headset panel pipeline layout", + .bindGroupLayoutCount = 1, + .bindGroupLayouts = &state.bindGroupLayout, + }; + const auto pipelineLayout = g_device.CreatePipelineLayout(&pipelineLayoutDescriptor); + + constexpr wgpu::BlendComponent kPremultipliedOver{ + .operation = wgpu::BlendOperation::Add, + .srcFactor = wgpu::BlendFactor::One, + .dstFactor = wgpu::BlendFactor::OneMinusSrcAlpha, + }; + const wgpu::BlendState blend{ + .color = kPremultipliedOver, + .alpha = kPremultipliedOver, + }; + const std::array colorTargets{wgpu::ColorTargetState{ + .format = format, + .blend = &blend, + .writeMask = wgpu::ColorWriteMask::All, + }}; + const wgpu::FragmentState fragmentState{ + .module = module, + .entryPoint = "fs_main", + .targetCount = colorTargets.size(), + .targets = colorTargets.data(), + }; + const wgpu::RenderPipelineDescriptor pipelineDescriptor{ + .label = "Headset panel pipeline", + .layout = pipelineLayout, + .vertex = + wgpu::VertexState{ + .module = module, + .entryPoint = "vs_main", + }, + .primitive = + wgpu::PrimitiveState{ + .topology = wgpu::PrimitiveTopology::TriangleStrip, + .cullMode = wgpu::CullMode::None, + }, + .multisample = + wgpu::MultisampleState{ + .count = 1, + .mask = UINT32_MAX, + }, + .fragment = &fragmentState, + }; + state.pipeline = g_device.CreateRenderPipeline(&pipelineDescriptor); + if (!state.pipeline) { + return false; + } + for (auto& uniform : state.uniforms) { + if (!uniform) { + const wgpu::BufferDescriptor bufferDescriptor{ + .label = "Headset panel uniform", + .usage = wgpu::BufferUsage::Uniform | wgpu::BufferUsage::CopyDst, + .size = sizeof(Mat4x4), + }; + uniform = g_device.CreateBuffer(&bufferDescriptor); + } + } + state.pipelineFormat = format; + return true; +} + +void composite(const wgpu::CommandEncoder& encoder, const wgpu::TextureView& target, const Mat4x4& clipFromPanel, + uint32_t eyeIndex) noexcept { + auto& state = g_state; + if (!state.visible || !state.pipeline || eyeIndex >= AURORA_STEREO_EYE_COUNT || !target) { + return; + } + auto& bindGroup = state.bindGroups[eyeIndex]; + if (!bindGroup) { + const std::array entries{ + wgpu::BindGroupEntry{ + .binding = 0, + .buffer = state.uniforms[eyeIndex], + .size = sizeof(Mat4x4), + }, + wgpu::BindGroupEntry{ + .binding = 1, + .sampler = state.panel.sampler, + }, + wgpu::BindGroupEntry{ + .binding = 2, + .textureView = state.panel.view, + }, + }; + const wgpu::BindGroupDescriptor descriptor{ + .label = "Headset panel bind group", + .layout = state.bindGroupLayout, + .entryCount = entries.size(), + .entries = entries.data(), + }; + bindGroup = g_device.CreateBindGroup(&descriptor); + } + // Each eye has its own uniform, and every pass that reads it is submitted + // before this worker writes it again. + g_queue.WriteBuffer(state.uniforms[eyeIndex], 0, &clipFromPanel, sizeof(clipFromPanel)); + + const std::array attachments{ + wgpu::RenderPassColorAttachment{ + .view = target, + .loadOp = wgpu::LoadOp::Load, + .storeOp = wgpu::StoreOp::Store, + }, + }; + const wgpu::RenderPassDescriptor descriptor{ + .label = eyeIndex == 0 ? "Headset panel left eye" : "Headset panel right eye", + .colorAttachmentCount = attachments.size(), + .colorAttachments = attachments.data(), + }; + const auto pass = encoder.BeginRenderPass(&descriptor); + pass.SetPipeline(state.pipeline); + pass.SetBindGroup(0, bindGroup, 0, nullptr); + pass.Draw(4); + pass.End(); +} + +float panel_aspect() noexcept { + const auto& size = g_state.panel.size; + return size.height != 0 ? static_cast(size.width) / static_cast(size.height) : 0.f; +} + +} // namespace + +wgpu::CommandBuffer prepare(ImDrawData* drawData, float widthFraction) noexcept { + ZoneScoped; + auto& state = g_state; + state.visible = false; + if (drawData == nullptr || !(widthFraction > 0.f)) { + return {}; + } + const auto width = static_cast(std::lround(drawData->DisplaySize.x * drawData->FramebufferScale.x)); + const auto height = static_cast(std::lround(drawData->DisplaySize.y * drawData->FramebufferScale.y)); + if (width == 0 || height == 0 || !ensure_pipeline()) { + return {}; + } + const auto format = webgpu::g_graphicsConfig.surfaceConfiguration.format; + if (!state.panel.texture || state.panel.size.width != width || state.panel.size.height != height || + state.panel.format != format) { + state.panel = webgpu::create_render_texture(width, height, false); + state.bindGroups = {}; + } + // The ImGui backend sets its viewport from the draw data, so a texture the + // device clamped to a smaller size cannot hold the pass. + if (state.panel.size.width != width || state.panel.size.height != height) { + return {}; + } + + const wgpu::CommandEncoderDescriptor encoderDescriptor{ + .label = "Headset panel encoder", + }; + auto encoder = g_device.CreateCommandEncoder(&encoderDescriptor); + const std::array attachments{ + wgpu::RenderPassColorAttachment{ + .view = state.panel.view, + .loadOp = wgpu::LoadOp::Clear, + .storeOp = wgpu::StoreOp::Store, + .clearValue = {.r = 0.0, .g = 0.0, .b = 0.0, .a = 0.0}, + }, + }; + const wgpu::RenderPassDescriptor passDescriptor{ + .label = "Headset panel ImGui pass", + .colorAttachmentCount = attachments.size(), + .colorAttachments = attachments.data(), + }; + bool drawn = false; + { + const auto pass = encoder.BeginRenderPass(&passDescriptor); + drawn = imgui::render_draw_data(pass, drawData); + pass.End(); + } + if (!drawn) { + return {}; + } + state.widthFraction = widthFraction; + state.visible = true; + return encoder.Finish(); +} + +void composite_immersive(const wgpu::CommandEncoder& encoder, const wgpu::TextureView& eye, + const Mat4x4& eyeFrustum, const Mat3x4& viewFromCenter, + uint32_t eyeIndex) noexcept { + if (!g_state.visible) { + return; + } + float screenWidth = 0.f; + float screenDistance = 0.f; + gfx::get_stereo_hud_screen_size(screenWidth, screenDistance); + const auto panel = gfx::stereo_replay::overlay_panel_on_screen(screenWidth, screenDistance, g_state.widthFraction, + panel_aspect()); + if (!panel.valid()) { + return; + } + composite(encoder, eye, gfx::stereo_replay::compose_overlay_panel_projection(eyeFrustum, viewFromCenter, panel), + eyeIndex); +} + +void composite_flat(const wgpu::CommandEncoder& encoder, const wgpu::TextureView& eye, const wgpu::Extent3D& size, + uint32_t eyeIndex) noexcept { + if (!g_state.visible || size.width == 0 || size.height == 0) { + return; + } + const float imageAspect = static_cast(size.width) / static_cast(size.height); + composite(encoder, eye, + gfx::stereo_replay::overlay_panel_flat_projection(g_state.widthFraction, panel_aspect(), imageAspect), + eyeIndex); +} + +void shutdown() noexcept { g_state = {}; } + +} // namespace aurora::stereo_overlay diff --git a/aurora-main/lib/stereo_overlay.hpp b/aurora-main/lib/stereo_overlay.hpp new file mode 100644 index 0000000..b21eb77 --- /dev/null +++ b/aurora-main/lib/stereo_overlay.hpp @@ -0,0 +1,33 @@ +#pragma once + +#include +#include + +#include + +struct ImDrawData; + +// The headset-only panel of aurora_imgui_set_stereo_overlay. Everything here +// belongs to the frame worker, like the eye targets it draws into. +namespace aurora::stereo_overlay { + +// Renders the panel's draw data into the panel texture and returns the command +// buffer holding that pass, which the caller submits before recording any other +// ImGui pass: the ImGui backend's projection uniform is shared by every pass, and +// the panel's canvas is not the desktop's size. Null draw data hides the panel, +// and so does any failure; both return a null command buffer. +wgpu::CommandBuffer prepare(ImDrawData* drawData, float widthFraction) noexcept; + +// Draws the most recently prepared panel over a finished immersive eye, on the +// 2D layer's virtual screen seen through that eye's frustum and view. +void composite_immersive(const wgpu::CommandEncoder& encoder, const wgpu::TextureView& eye, + const Mat4x4& eyeFrustum, const Mat3x4& viewFromCenter, + uint32_t eyeIndex) noexcept; + +// Draws it over a virtual-screen eye image, which is shown flat as a quad. +void composite_flat(const wgpu::CommandEncoder& encoder, const wgpu::TextureView& eye, const wgpu::Extent3D& size, + uint32_t eyeIndex) noexcept; + +void shutdown() noexcept; + +} // namespace aurora::stereo_overlay diff --git a/aurora-main/tests/stereo_replay_test.cpp b/aurora-main/tests/stereo_replay_test.cpp index 4ed7a2b..f84c54f 100644 --- a/aurora-main/tests/stereo_replay_test.cpp +++ b/aurora-main/tests/stereo_replay_test.cpp @@ -245,6 +245,70 @@ TEST(StereoReplayTest, HudScreenParksRasterDepthAtMidrangeUnderHeadMotion) { } } +TEST(StereoReplayTest, OverlayPanelIsCentredOnTheVirtualScreen) { + // Three quarters of a 1200-unit screen, 1000 ahead, with a 4:3 panel. + const auto panel = overlay_panel_on_screen(1200.0f, 1000.0f, 0.75f, 4.0f / 3.0f); + EXPECT_FLOAT_EQ(panel.halfWidth, 450.0f); + EXPECT_FLOAT_EQ(panel.halfHeight, 337.5f); + EXPECT_FLOAT_EQ(panel.distance, 1000.0f); + EXPECT_TRUE(panel.valid()); + EXPECT_FALSE(overlay_panel_on_screen(1200.0f, 1000.0f, 0.75f, 0.0f).valid()); + EXPECT_FALSE(overlay_panel_on_screen(1200.0f, 0.0f, 0.75f, 4.0f / 3.0f).valid()); +} + +TEST(StereoReplayTest, OverlayPanelCornersFollowTheEyeChain) { + Mat4x4 eyeFrustum{}; + eyeFrustum.m0 = {1.15f, 0.0f, 0.08f, 0.0f}; + eyeFrustum.m1 = {0.0f, 1.02f, -0.03f, 0.0f}; + const float angle = 0.3f; + const float c = std::cos(angle); + const float s = std::sin(angle); + Mat3x4 viewFromCenter{}; + viewFromCenter.m0 = {c, 0.0f, s, 15.0f}; + viewFromCenter.m1 = {0.0f, 1.0f, 0.0f, -4.0f}; + viewFromCenter.m2 = {-s, 0.0f, c, 7.0f}; + + const OverlayPanel panel{.halfWidth = 450.0f, .halfHeight = 337.5f, .distance = 1000.0f}; + const auto composed = compose_overlay_panel_projection(eyeFrustum, viewFromCenter, panel); + + const std::array, 5> corners{{ + {-1.0f, 1.0f, 0.0f, 1.0f}, + {1.0f, 1.0f, 0.0f, 1.0f}, + {-1.0f, -1.0f, 0.0f, 1.0f}, + {1.0f, -1.0f, 0.0f, 1.0f}, + {0.0f, 0.0f, 0.0f, 1.0f}, + }}; + for (const auto& corner : corners) { + // Top left is (-1, +1): the panel's +y is up, like the screen it sits on. + const Vec4 centerPoint{corner[0] * panel.halfWidth, corner[1] * panel.halfHeight, -panel.distance, 1.0f}; + const float eyeX = dot4(viewFromCenter.m0, centerPoint); + const float eyeY = dot4(viewFromCenter.m1, centerPoint); + const float eyeZ = dot4(viewFromCenter.m2, centerPoint); + const float w = dot4(composed.m3, corner); + EXPECT_NEAR(dot4(composed.m0, corner), eyeFrustum.m0[0] * eyeX + eyeFrustum.m0[2] * eyeZ, 1e-2f); + EXPECT_NEAR(dot4(composed.m1, corner), eyeFrustum.m1[1] * eyeY + eyeFrustum.m1[2] * eyeZ, 1e-2f); + EXPECT_NEAR(w, -eyeZ, 1e-2f); + ASSERT_GT(w, 0.0f); + EXPECT_NEAR(dot4(composed.m2, corner) / w, 0.5f, 1e-5f); + } +} + +TEST(StereoReplayTest, OverlayPanelOnAFlatEyeKeepsItsAspect) { + // A 4:3 panel three quarters across a 2064x2208 eye image. + const float imageAspect = 2064.0f / 2208.0f; + const auto flat = overlay_panel_flat_projection(0.75f, 4.0f / 3.0f, imageAspect); + const Vec4 topRight{1.0f, 1.0f, 0.0f, 1.0f}; + const float ndcX = dot4(flat.m0, topRight) / dot4(flat.m3, topRight); + const float ndcY = dot4(flat.m1, topRight) / dot4(flat.m3, topRight); + EXPECT_FLOAT_EQ(ndcX, 0.75f); + // In pixels: 0.75 * 2064 wide over ndcY * 2208 tall is the panel's 4:3. + EXPECT_NEAR((ndcX * 2064.0f) / (ndcY * 2208.0f), 4.0f / 3.0f, 1e-4f); + EXPECT_FLOAT_EQ(dot4(flat.m2, topRight), 0.5f); + const Vec4 centre{0.0f, 0.0f, 0.0f, 1.0f}; + EXPECT_FLOAT_EQ(dot4(flat.m0, centre), 0.0f); + EXPECT_FLOAT_EQ(dot4(flat.m1, centre), 0.0f); +} + Mat3x4 identity3x4() { Mat3x4 m{}; m.m0 = {1.0f, 0.0f, 0.0f, 0.0f}; diff --git a/docs/quest-port.md b/docs/quest-port.md index 7c3687c..87f162f 100644 --- a/docs/quest-port.md +++ b/docs/quest-port.md @@ -256,7 +256,7 @@ the app: | --- | --- | | `debug.wiicompiled.vtxpad 0` | Turns the stride padding off, to re-check a driver update | | `debug.wiicompiled.validation 1` | Keeps WebGPU validation and robustness on in release builds | -| `debug.wiicompiled.inject :