Implement OpenXR Settings Panel and Stereo Overlay

aurora-main/cmake/aurora_core.cmake

@@ -26,7 +26,7 @@ elseif (APPLE)
 endif ()
 
 if (AURORA_ENABLE_GX)
-    target_sources(aurora_core PRIVATE lib/imgui.cpp)
+    target_sources(aurora_core PRIVATE lib/imgui.cpp lib/stereo_overlay.cpp)
     target_link_libraries(aurora_core PUBLIC imgui)
 endif ()
 

aurora-main/include/aurora/imgui.h

@@ -13,6 +13,19 @@ extern "C" {
 
 ImTextureID aurora_imgui_add_texture(uint32_t width, uint32_t height, const void* rgba8);
 
+// A panel shown only in the headset, drawn by the host with a Dear ImGui context of its own (for
+// instance a settings menu reachable without the desktop window). It is laid over whatever the eyes
+// carry: centred on the virtual screen, `widthFraction` of that screen's width, its height following
+// the draw data's aspect. On a menu frame the screen is the quad the eye image is shown on; in an
+// immersive race it is the 2D layer's screen (aurora_set_stereo_hud_screen's width and distance,
+// whether or not the 2D layer is placed on it).
+//
+// The draw data belongs to the host's context and is rendered with Aurora's ImGui backend, so its
+// texture ids must be views that backend can bind (aurora_imgui_add_texture). Call this from the
+// producer before the frame is sealed, and leave the draw data untouched until the frame worker is
+// done with that frame (aurora_wait_for_frame_worker). Null hides the panel.
+void aurora_imgui_set_stereo_overlay(ImDrawData* drawData, float widthFraction);
+
 #ifdef __cplusplus
 }
 #endif

aurora-main/lib/aurora.cpp

@@ -10,6 +10,7 @@
 #include "stereo.hpp"
 #include "stereo_mirror.hpp"
 #include "stereo_interpolation.hpp"
+#include "stereo_overlay.hpp"
 #include "webgpu/gpu.hpp"
 #include <webgpu/webgpu_cpp.h>
 #endif
@@ -1613,6 +1614,7 @@ void shutdown() noexcept {
   g_stereoEyeTargets = {};
   g_stereoMirrorState.Reset();
   g_presentationImagePools = {};
+  stereo_overlay::shutdown();
   imgui::shutdown();
   gfx::shutdown();
   webgpu::shutdown();
@@ -1734,6 +1736,7 @@ struct SealedFrameContext {
   bool replayInterpolatedFrames = false;
   std::optional<AuroraStereoFrame> stereoInput;
   bool retainStereo = false;
+  imgui::StereoOverlay stereoOverlay;
 };
 
 // Worker-owned scene state. A separate buffer generation check protects against
@@ -1779,6 +1782,9 @@ void run_retained_stereo_frame(gfx::SealedFrame& sealedFrame) noexcept {
     return;
   for (uint32_t eye = 0; eye < AURORA_STEREO_EYE_COUNT; ++eye) {
     gfx::render_stereo_eye(sealedFrame, encoder, replay, eye, false);
+    // The panel texture from the last sealed frame; its draw data is not ours to touch here.
+    stereo_overlay::composite_immersive(encoder, g_stereoEyeTargets[eye].output().view, replay.eyes[eye].projection,
+                                        replay.eyes[eye].viewFromCenter, eye);
   }
   const auto sink = run_stereo_sink(encoder, input->frameToken, g_retainedStereo.logicalFrame, input->mode);
   const auto buffer = encoder.Finish();
@@ -1847,6 +1853,8 @@ void seal_frame_locked(gfx::SealedFrame& sealedFrame, SealedFrameContext& ctx, u
   // ImGui draw lists are built once per frame and replayed by each slot's ImGui pass, which is why
   // the next ImGui frame cannot start until the encode phase is done.
   imgui::render_frame_data();
+  // The headset panel's draw data follows the same rule on the host's side.
+  ctx.stereoOverlay = imgui::latch_stereo_overlay();
   // Drop the sealed frame's lazy RAM-readback requests while the producer is still excluded; it
   // starts registering the next frame's as soon as SEALED is published.
   gfx::efb_ram::cancel();
@@ -1923,6 +1931,15 @@ std::vector<PresentationJob> encode_sealed_frame(gfx::SealedFrame& sealedFrame,
     }
   };
 
+  // Ahead of every other ImGui pass of this frame: the ImGui backend keeps one projection uniform,
+  // and the headset panel's canvas is not the desktop's size, so its pass has to reach the queue
+  // before a desktop pass rewrites that uniform. The eyes below sample the texture it fills.
+  if (const auto panel = stereo_overlay::prepare(stereo_frame_provider_active() ? ctx.stereoOverlay.drawData : nullptr,
+                                                 ctx.stereoOverlay.widthFraction)) {
+    std::lock_guard submitLock(g_queueSubmitMutex);
+    g_queue.Submit(1, &panel);
+  }
+
   if (ctx.replayInterpolatedFrames) {
     for (uint32_t interpolatedFrame = 0; interpolatedFrame < ctx.interpolatedFrameCount; ++interpolatedFrame) {
       gfx::render(sealedFrame, encoder, static_cast<int32_t>(interpolatedFrame), false);
@@ -1976,6 +1993,9 @@ std::vector<PresentationJob> encode_sealed_frame(gfx::SealedFrame& sealedFrame,
     for (uint32_t eye = 0; eye < AURORA_STEREO_EYE_COUNT; ++eye) {
       gfx::render_stereo_eye(sealedFrame, encoder, *ctx.stereoReplay, eye,
                              !ctx.retainStereo && eye + 1 == AURORA_STEREO_EYE_COUNT);
+      stereo_overlay::composite_immersive(encoder, g_stereoEyeTargets[eye].output().view,
+                                          ctx.stereoReplay->eyes[eye].projection,
+                                          ctx.stereoReplay->eyes[eye].viewFromCenter, eye);
     }
   }
 
@@ -2003,6 +2023,8 @@ std::vector<PresentationJob> encode_sealed_frame(gfx::SealedFrame& sealedFrame,
     };
     for (uint32_t eye = 0; eye < AURORA_STEREO_EYE_COUNT; ++eye) {
       encode_virtual_screen_eye(encoder, completedMono, eye);
+      const auto& output = g_stereoEyeTargets[eye].output();
+      stereo_overlay::composite_flat(encoder, output.view, output.size, eye);
     }
     if (mirrorPlan == MirrorPlan::Black && !headsetOnly) {
       encode_presentation_snapshot(encoder, ctx.presentSource, *finalImage, true, MirrorPlan::Black);

aurora-main/lib/gfx/common.cpp

@@ -238,6 +238,10 @@ void set_stereo_hud_screen(bool enabled, float width, float distance) noexcept {
   g_stereoHudScreenEnabled.store(enabled, std::memory_order_relaxed);
 }
 bool get_stereo_hud_screen_enabled() noexcept { return g_stereoHudScreenEnabled.load(std::memory_order_relaxed); }
+void get_stereo_hud_screen_size(float& width, float& distance) noexcept {
+  width = g_stereoHudScreenWidth.load(std::memory_order_relaxed);
+  distance = g_stereoHudScreenDistance.load(std::memory_order_relaxed);
+}
 
 // The desktop mirror choice. Normal is the ordinary mono presentation, so a
 // build that never touches this setting presents exactly as it did before.

aurora-main/lib/gfx/common.hpp

@@ -400,6 +400,8 @@ bool get_stereo_skip_copy_clears() noexcept;
 // the game's presented aspect ratio. Also live.
 void set_stereo_hud_screen(bool enabled, float width, float distance) noexcept;
 bool get_stereo_hud_screen_enabled() noexcept;
+// The screen's width and distance in world units, kept while the 2D layer is off it.
+void get_stereo_hud_screen_size(float& width, float& distance) noexcept;
 
 // What the desktop window presents while a stereo provider is feeding a
 // headset. Live, and read once per presentation group by the frame worker.

aurora-main/lib/gfx/stereo_replay.hpp

@@ -306,4 +306,62 @@ inline Mat4x4<float> compose_hud_screen_projection(const Mat4x4<float>& eyeFrust
   return out;
 }
 
+// The headset settings panel (aurora_imgui_set_stereo_overlay): a rectangle
+// centred on the virtual screen, in the same world units as the screen.
+struct OverlayPanel {
+  float halfWidth = 0.0f;
+  float halfHeight = 0.0f;
+  float distance = 0.0f;
+
+  [[nodiscard]] bool valid() const noexcept { return halfWidth > 0.0f && halfHeight > 0.0f && distance > 0.0f; }
+};
+
+// `widthFraction` of a virtual screen `screenWidth` across and `screenDistance`
+// ahead, its height following the panel's own aspect.
+inline OverlayPanel overlay_panel_on_screen(float screenWidth, float screenDistance, float widthFraction,
+                                            float panelAspect) noexcept {
+  if (!(panelAspect > 0.0f)) {
+    return {};
+  }
+  const float halfWidth = 0.5f * screenWidth * widthFraction;
+  return {.halfWidth = halfWidth, .halfHeight = halfWidth / panelAspect, .distance = screenDistance};
+}
+
+// Clip position of a point on the panel for one eye of an immersive frame, as a
+// matrix applied to (x, y, 0, 1) with x and y running -1..1 across the panel,
+// +y up. It is the 2D layer's chain with the panel's corners in place of a
+// game draw's NDC, so the panel sits exactly where the race HUD's screen does.
+// The panel is drawn over the finished eye with no depth test, so its depth is
+// simply parked mid-volume.
+inline Mat4x4<float> compose_overlay_panel_projection(const Mat4x4<float>& eyeFrustum,
+                                                      const Mat3x4<float>& viewFromCenter,
+                                                      const OverlayPanel& panel) noexcept {
+  Mat4x4<float> corners{};
+  corners.m0 = {1.0f, 0.0f, 0.0f, 0.0f};
+  corners.m1 = {0.0f, 1.0f, 0.0f, 0.0f};
+  corners.m3 = {0.0f, 0.0f, 0.0f, 1.0f};
+  const HudScreen screen{.halfWidth = panel.halfWidth, .halfHeight = panel.halfHeight, .distance = panel.distance};
+  auto out = compose_hud_screen_projection(eyeFrustum, viewFromCenter, screen, corners);
+  for (size_t i = 0; i < 4; ++i) {
+    out.m2[i] = 0.5f * out.m3[i];
+  }
+  return out;
+}
+
+// The same panel on a virtual-screen eye image, which the runtime shows as a
+// quad layer the screen's width across: a centred rectangle `widthFraction` of
+// the image's width, with the panel's aspect.
+inline Mat4x4<float> overlay_panel_flat_projection(float widthFraction, float panelAspect,
+                                                   float imageAspect) noexcept {
+  Mat4x4<float> out{};
+  if (!(panelAspect > 0.0f) || !(imageAspect > 0.0f)) {
+    return out;
+  }
+  out.m0 = {widthFraction, 0.0f, 0.0f, 0.0f};
+  out.m1 = {0.0f, widthFraction * imageAspect / panelAspect, 0.0f, 0.0f};
+  out.m2 = {0.0f, 0.0f, 0.0f, 0.5f};
+  out.m3 = {0.0f, 0.0f, 0.0f, 1.0f};
+  return out;
+}
+
 } // namespace aurora::gfx::stereo_replay

aurora-main/lib/imgui.cpp

@@ -3,6 +3,7 @@
 #include <cstddef>
 #include <cmath>
 #include <filesystem>
+#include <mutex>
 #include <string>
 #include <vector>
 
@@ -32,6 +33,10 @@ static bool g_frameDataBuilt = false;
 static std::vector<SDL_Texture*> g_sdlTextures;
 static std::vector<wgpu::Texture> g_wgpuTextures;
 
+// Set by the producer, latched by the seal.
+static std::mutex g_stereoOverlayMutex;
+static StereoOverlay g_stereoOverlay;
+
 void remove_legacy_ini_file(const char* basePath) noexcept {
   if (basePath == nullptr || *basePath == '\0') {
     return;
@@ -200,6 +205,23 @@ void render(const wgpu::RenderPassEncoder& pass) noexcept {
   }
 }
 
+StereoOverlay latch_stereo_overlay() noexcept {
+  std::lock_guard lock(g_stereoOverlayMutex);
+  return g_stereoOverlay;
+}
+
+bool render_draw_data(const wgpu::RenderPassEncoder& pass, ImDrawData* data) noexcept {
+  ZoneScoped;
+  // The SDL renderer fallback has no render passes to draw into.
+  if (g_useSdlRenderer || data == nullptr || ImGui::GetCurrentContext() == nullptr) {
+    return false;
+  }
+  pass.PushDebugGroup("Aurora: Dear Imgui headset panel");
+  ImGui_ImplWGPU_RenderDrawData(data, pass.Get());
+  pass.PopDebugGroup();
+  return true;
+}
+
 ImTextureID add_texture(uint32_t width, uint32_t height, const uint8_t* data) noexcept {
   if (SDL_Renderer* renderer = window::get_sdl_renderer()) {
     SDL_Texture* texture = SDL_CreateTexture(renderer, SDL_PIXELFORMAT_RGBA32, SDL_TEXTUREACCESS_STATIC, width, height);
@@ -251,4 +273,12 @@ extern "C" {
 ImTextureID aurora_imgui_add_texture(uint32_t width, uint32_t height, const void* rgba8) {
   return aurora::imgui::add_texture(width, height, static_cast<const uint8_t*>(rgba8));
 }
+
+void aurora_imgui_set_stereo_overlay(ImDrawData* drawData, float widthFraction) {
+  std::lock_guard lock(aurora::imgui::g_stereoOverlayMutex);
+  aurora::imgui::g_stereoOverlay = {
+      .drawData = drawData,
+      .widthFraction = drawData != nullptr ? widthFraction : 0.f,
+  };
+}
 }

aurora-main/lib/imgui.hpp

@@ -3,6 +3,7 @@
 #include <aurora/event.h>
 
 union SDL_Event;
+struct ImDrawData;
 
 namespace wgpu {
 class RenderPassEncoder;
@@ -20,4 +21,15 @@ void new_frame(const AuroraWindowSize& size) noexcept;
 // once and every presentation slot replays them. Reset by new_frame.
 void render_frame_data() noexcept;
 void render(const wgpu::RenderPassEncoder& pass) noexcept;
+
+// The headset panel as aurora_imgui_set_stereo_overlay last set it.
+struct StereoOverlay {
+  ImDrawData* drawData = nullptr;
+  float widthFraction = 0.f;
+};
+StereoOverlay latch_stereo_overlay() noexcept;
+// Renders another context's draw data with this context's backend. The backend keeps one projection
+// uniform for every pass, so a pass whose display size differs from the desktop's must be submitted
+// before the next pass is recorded.
+bool render_draw_data(const wgpu::RenderPassEncoder& pass, ImDrawData* data) noexcept;
 } // namespace aurora::imgui

aurora-main/lib/stereo_overlay.cpp

@@ -0,0 +1,343 @@
+#include "stereo_overlay.hpp"
+
+#include "gfx/common.hpp"
+#include "gfx/stereo_replay.hpp"
+#include "imgui.hpp"
+#include "webgpu/gpu.hpp"
+
+#include <aurora/aurora.h>
+#include <imgui.h>
+
+#include "tracy/Tracy.hpp"
+
+#include <array>
+#include <cmath>
+
+namespace aurora::stereo_overlay {
+namespace {
+
+using webgpu::g_device;
+using webgpu::g_queue;
+
+// Uploaded as-is into a WGSL mat4x4<f32>, like the GX uniforms' matrices.
+static_assert(sizeof(Mat4x4<float>) == 64);
+
+// The panel's corners, (-1, 1) top left to (1, -1) bottom right, carried through
+// one clip-from-panel matrix per eye. UVs are interpolated perspective-correct,
+// so the texture stays straight on a panel seen at an angle. The texture holds
+// premultiplied colour, which is what ImGui's blending leaves in a cleared target.
+constexpr const char* kShader = R"""(
+struct Panel {
+    clip_from_panel: mat4x4<f32>,
+};
+@group(0) @binding(0)
+var<uniform> panel: Panel;
+@group(0) @binding(1)
+var panel_sampler: sampler;
+@group(0) @binding(2)
+var panel_texture: texture_2d<f32>;
+
+struct VertexOutput {
+    @builtin(position) pos: vec4<f32>,
+    @location(0) uv: vec2<f32>,
+};
+
+var<private> corners: array<vec2<f32>, 4> = array<vec2<f32>, 4>(
+    vec2(-1.0, 1.0),
+    vec2(-1.0, -1.0),
+    vec2(1.0, 1.0),
+    vec2(1.0, -1.0),
+);
+
+@vertex
+fn vs_main(@builtin(vertex_index) vtxIdx: u32) -> VertexOutput {
+    let corner = corners[vtxIdx];
+    var out: VertexOutput;
+    // Row-vector convention, like the GX shaders: m0..m3 are the clip x/y/z/w rows.
+    out.pos = vec4<f32>(corner, 0.0, 1.0) * panel.clip_from_panel;
+    out.uv = vec2<f32>(0.5 + 0.5 * corner.x, 0.5 - 0.5 * corner.y);
+    return out;
+}
+
+@fragment
+fn fs_main(in: VertexOutput) -> @location(0) vec4<f32> {
+    return textureSample(panel_texture, panel_sampler, in.uv);
+}
+)""";
+
+struct State {
+  webgpu::TextureWithSampler panel;
+  wgpu::RenderPipeline pipeline;
+  wgpu::BindGroupLayout bindGroupLayout;
+  wgpu::TextureFormat pipelineFormat = wgpu::TextureFormat::Undefined;
+  std::array<wgpu::Buffer, AURORA_STEREO_EYE_COUNT> uniforms;
+  std::array<wgpu::BindGroup, AURORA_STEREO_EYE_COUNT> bindGroups;
+  float widthFraction = 0.f;
+  bool visible = false;
+};
+State g_state;
+
+bool ensure_pipeline() {
+  auto& state = g_state;
+  const auto format = webgpu::g_graphicsConfig.surfaceConfiguration.format;
+  if (state.pipeline && state.pipelineFormat == format) {
+    return true;
+  }
+  state.pipeline = {};
+  state.bindGroups = {};
+
+  wgpu::ShaderSourceWGSL source{};
+  source.code = kShader;
+  const wgpu::ShaderModuleDescriptor moduleDescriptor{
+      .nextInChain = &source,
+      .label = "Headset panel module",
+  };
+  const auto module = g_device.CreateShaderModule(&moduleDescriptor);
+
+  const std::array layoutEntries{
+      wgpu::BindGroupLayoutEntry{
+          .binding = 0,
+          .visibility = wgpu::ShaderStage::Vertex,
+          .buffer =
+              wgpu::BufferBindingLayout{
+                  .type = wgpu::BufferBindingType::Uniform,
+                  .minBindingSize = sizeof(Mat4x4<float>),
+              },
+      },
+      wgpu::BindGroupLayoutEntry{
+          .binding = 1,
+          .visibility = wgpu::ShaderStage::Fragment,
+          .sampler =
+              wgpu::SamplerBindingLayout{
+                  .type = wgpu::SamplerBindingType::Filtering,
+              },
+      },
+      wgpu::BindGroupLayoutEntry{
+          .binding = 2,
+          .visibility = wgpu::ShaderStage::Fragment,
+          .texture =
+              wgpu::TextureBindingLayout{
+                  .sampleType = wgpu::TextureSampleType::Float,
+                  .viewDimension = wgpu::TextureViewDimension::e2D,
+              },
+      },
+  };
+  const wgpu::BindGroupLayoutDescriptor layoutDescriptor{
+      .label = "Headset panel bind group layout",
+      .entryCount = layoutEntries.size(),
+      .entries = layoutEntries.data(),
+  };
+  state.bindGroupLayout = g_device.CreateBindGroupLayout(&layoutDescriptor);
+  const wgpu::PipelineLayoutDescriptor pipelineLayoutDescriptor{
+      .label = "Headset panel pipeline layout",
+      .bindGroupLayoutCount = 1,
+      .bindGroupLayouts = &state.bindGroupLayout,
+  };
+  const auto pipelineLayout = g_device.CreatePipelineLayout(&pipelineLayoutDescriptor);
+
+  constexpr wgpu::BlendComponent kPremultipliedOver{
+      .operation = wgpu::BlendOperation::Add,
+      .srcFactor = wgpu::BlendFactor::One,
+      .dstFactor = wgpu::BlendFactor::OneMinusSrcAlpha,
+  };
+  const wgpu::BlendState blend{
+      .color = kPremultipliedOver,
+      .alpha = kPremultipliedOver,
+  };
+  const std::array colorTargets{wgpu::ColorTargetState{
+      .format = format,
+      .blend = &blend,
+      .writeMask = wgpu::ColorWriteMask::All,
+  }};
+  const wgpu::FragmentState fragmentState{
+      .module = module,
+      .entryPoint = "fs_main",
+      .targetCount = colorTargets.size(),
+      .targets = colorTargets.data(),
+  };
+  const wgpu::RenderPipelineDescriptor pipelineDescriptor{
+      .label = "Headset panel pipeline",
+      .layout = pipelineLayout,
+      .vertex =
+          wgpu::VertexState{
+              .module = module,
+              .entryPoint = "vs_main",
+          },
+      .primitive =
+          wgpu::PrimitiveState{
+              .topology = wgpu::PrimitiveTopology::TriangleStrip,
+              .cullMode = wgpu::CullMode::None,
+          },
+      .multisample =
+          wgpu::MultisampleState{
+              .count = 1,
+              .mask = UINT32_MAX,
+          },
+      .fragment = &fragmentState,
+  };
+  state.pipeline = g_device.CreateRenderPipeline(&pipelineDescriptor);
+  if (!state.pipeline) {
+    return false;
+  }
+  for (auto& uniform : state.uniforms) {
+    if (!uniform) {
+      const wgpu::BufferDescriptor bufferDescriptor{
+          .label = "Headset panel uniform",
+          .usage = wgpu::BufferUsage::Uniform | wgpu::BufferUsage::CopyDst,
+          .size = sizeof(Mat4x4<float>),
+      };
+      uniform = g_device.CreateBuffer(&bufferDescriptor);
+    }
+  }
+  state.pipelineFormat = format;
+  return true;
+}
+
+void composite(const wgpu::CommandEncoder& encoder, const wgpu::TextureView& target, const Mat4x4<float>& clipFromPanel,
+               uint32_t eyeIndex) noexcept {
+  auto& state = g_state;
+  if (!state.visible || !state.pipeline || eyeIndex >= AURORA_STEREO_EYE_COUNT || !target) {
+    return;
+  }
+  auto& bindGroup = state.bindGroups[eyeIndex];
+  if (!bindGroup) {
+    const std::array entries{
+        wgpu::BindGroupEntry{
+            .binding = 0,
+            .buffer = state.uniforms[eyeIndex],
+            .size = sizeof(Mat4x4<float>),
+        },
+        wgpu::BindGroupEntry{
+            .binding = 1,
+            .sampler = state.panel.sampler,
+        },
+        wgpu::BindGroupEntry{
+            .binding = 2,
+            .textureView = state.panel.view,
+        },
+    };
+    const wgpu::BindGroupDescriptor descriptor{
+        .label = "Headset panel bind group",
+        .layout = state.bindGroupLayout,
+        .entryCount = entries.size(),
+        .entries = entries.data(),
+    };
+    bindGroup = g_device.CreateBindGroup(&descriptor);
+  }
+  // Each eye has its own uniform, and every pass that reads it is submitted
+  // before this worker writes it again.
+  g_queue.WriteBuffer(state.uniforms[eyeIndex], 0, &clipFromPanel, sizeof(clipFromPanel));
+
+  const std::array attachments{
+      wgpu::RenderPassColorAttachment{
+          .view = target,
+          .loadOp = wgpu::LoadOp::Load,
+          .storeOp = wgpu::StoreOp::Store,
+      },
+  };
+  const wgpu::RenderPassDescriptor descriptor{
+      .label = eyeIndex == 0 ? "Headset panel left eye" : "Headset panel right eye",
+      .colorAttachmentCount = attachments.size(),
+      .colorAttachments = attachments.data(),
+  };
+  const auto pass = encoder.BeginRenderPass(&descriptor);
+  pass.SetPipeline(state.pipeline);
+  pass.SetBindGroup(0, bindGroup, 0, nullptr);
+  pass.Draw(4);
+  pass.End();
+}
+
+float panel_aspect() noexcept {
+  const auto& size = g_state.panel.size;
+  return size.height != 0 ? static_cast<float>(size.width) / static_cast<float>(size.height) : 0.f;
+}
+
+} // namespace
+
+wgpu::CommandBuffer prepare(ImDrawData* drawData, float widthFraction) noexcept {
+  ZoneScoped;
+  auto& state = g_state;
+  state.visible = false;
+  if (drawData == nullptr || !(widthFraction > 0.f)) {
+    return {};
+  }
+  const auto width = static_cast<uint32_t>(std::lround(drawData->DisplaySize.x * drawData->FramebufferScale.x));
+  const auto height = static_cast<uint32_t>(std::lround(drawData->DisplaySize.y * drawData->FramebufferScale.y));
+  if (width == 0 || height == 0 || !ensure_pipeline()) {
+    return {};
+  }
+  const auto format = webgpu::g_graphicsConfig.surfaceConfiguration.format;
+  if (!state.panel.texture || state.panel.size.width != width || state.panel.size.height != height ||
+      state.panel.format != format) {
+    state.panel = webgpu::create_render_texture(width, height, false);
+    state.bindGroups = {};
+  }
+  // The ImGui backend sets its viewport from the draw data, so a texture the
+  // device clamped to a smaller size cannot hold the pass.
+  if (state.panel.size.width != width || state.panel.size.height != height) {
+    return {};
+  }
+
+  const wgpu::CommandEncoderDescriptor encoderDescriptor{
+      .label = "Headset panel encoder",
+  };
+  auto encoder = g_device.CreateCommandEncoder(&encoderDescriptor);
+  const std::array attachments{
+      wgpu::RenderPassColorAttachment{
+          .view = state.panel.view,
+          .loadOp = wgpu::LoadOp::Clear,
+          .storeOp = wgpu::StoreOp::Store,
+          .clearValue = {.r = 0.0, .g = 0.0, .b = 0.0, .a = 0.0},
+      },
+  };
+  const wgpu::RenderPassDescriptor passDescriptor{
+      .label = "Headset panel ImGui pass",
+      .colorAttachmentCount = attachments.size(),
+      .colorAttachments = attachments.data(),
+  };
+  bool drawn = false;
+  {
+    const auto pass = encoder.BeginRenderPass(&passDescriptor);
+    drawn = imgui::render_draw_data(pass, drawData);
+    pass.End();
+  }
+  if (!drawn) {
+    return {};
+  }
+  state.widthFraction = widthFraction;
+  state.visible = true;
+  return encoder.Finish();
+}
+
+void composite_immersive(const wgpu::CommandEncoder& encoder, const wgpu::TextureView& eye,
+                         const Mat4x4<float>& eyeFrustum, const Mat3x4<float>& viewFromCenter,
+                         uint32_t eyeIndex) noexcept {
+  if (!g_state.visible) {
+    return;
+  }
+  float screenWidth = 0.f;
+  float screenDistance = 0.f;
+  gfx::get_stereo_hud_screen_size(screenWidth, screenDistance);
+  const auto panel = gfx::stereo_replay::overlay_panel_on_screen(screenWidth, screenDistance, g_state.widthFraction,
+                                                                 panel_aspect());
+  if (!panel.valid()) {
+    return;
+  }
+  composite(encoder, eye, gfx::stereo_replay::compose_overlay_panel_projection(eyeFrustum, viewFromCenter, panel),
+            eyeIndex);
+}
+
+void composite_flat(const wgpu::CommandEncoder& encoder, const wgpu::TextureView& eye, const wgpu::Extent3D& size,
+                    uint32_t eyeIndex) noexcept {
+  if (!g_state.visible || size.width == 0 || size.height == 0) {
+    return;
+  }
+  const float imageAspect = static_cast<float>(size.width) / static_cast<float>(size.height);
+  composite(encoder, eye,
+            gfx::stereo_replay::overlay_panel_flat_projection(g_state.widthFraction, panel_aspect(), imageAspect),
+            eyeIndex);
+}
+
+void shutdown() noexcept { g_state = {}; }
+
+} // namespace aurora::stereo_overlay

aurora-main/lib/stereo_overlay.hpp

@@ -0,0 +1,33 @@
+#pragma once
+
+#include <aurora/math.hpp>
+#include <webgpu/webgpu_cpp.h>
+
+#include <cstdint>
+
+struct ImDrawData;
+
+// The headset-only panel of aurora_imgui_set_stereo_overlay. Everything here
+// belongs to the frame worker, like the eye targets it draws into.
+namespace aurora::stereo_overlay {
+
+// Renders the panel's draw data into the panel texture and returns the command
+// buffer holding that pass, which the caller submits before recording any other
+// ImGui pass: the ImGui backend's projection uniform is shared by every pass, and
+// the panel's canvas is not the desktop's size. Null draw data hides the panel,
+// and so does any failure; both return a null command buffer.
+wgpu::CommandBuffer prepare(ImDrawData* drawData, float widthFraction) noexcept;
+
+// Draws the most recently prepared panel over a finished immersive eye, on the
+// 2D layer's virtual screen seen through that eye's frustum and view.
+void composite_immersive(const wgpu::CommandEncoder& encoder, const wgpu::TextureView& eye,
+                         const Mat4x4<float>& eyeFrustum, const Mat3x4<float>& viewFromCenter,
+                         uint32_t eyeIndex) noexcept;
+
+// Draws it over a virtual-screen eye image, which is shown flat as a quad.
+void composite_flat(const wgpu::CommandEncoder& encoder, const wgpu::TextureView& eye, const wgpu::Extent3D& size,
+                    uint32_t eyeIndex) noexcept;
+
+void shutdown() noexcept;
+
+} // namespace aurora::stereo_overlay

aurora-main/tests/stereo_replay_test.cpp

@@ -245,6 +245,70 @@ TEST(StereoReplayTest, HudScreenParksRasterDepthAtMidrangeUnderHeadMotion) {
   }
 }
 
+TEST(StereoReplayTest, OverlayPanelIsCentredOnTheVirtualScreen) {
+  // Three quarters of a 1200-unit screen, 1000 ahead, with a 4:3 panel.
+  const auto panel = overlay_panel_on_screen(1200.0f, 1000.0f, 0.75f, 4.0f / 3.0f);
+  EXPECT_FLOAT_EQ(panel.halfWidth, 450.0f);
+  EXPECT_FLOAT_EQ(panel.halfHeight, 337.5f);
+  EXPECT_FLOAT_EQ(panel.distance, 1000.0f);
+  EXPECT_TRUE(panel.valid());
+  EXPECT_FALSE(overlay_panel_on_screen(1200.0f, 1000.0f, 0.75f, 0.0f).valid());
+  EXPECT_FALSE(overlay_panel_on_screen(1200.0f, 0.0f, 0.75f, 4.0f / 3.0f).valid());
+}
+
+TEST(StereoReplayTest, OverlayPanelCornersFollowTheEyeChain) {
+  Mat4x4<float> eyeFrustum{};
+  eyeFrustum.m0 = {1.15f, 0.0f, 0.08f, 0.0f};
+  eyeFrustum.m1 = {0.0f, 1.02f, -0.03f, 0.0f};
+  const float angle = 0.3f;
+  const float c = std::cos(angle);
+  const float s = std::sin(angle);
+  Mat3x4<float> viewFromCenter{};
+  viewFromCenter.m0 = {c, 0.0f, s, 15.0f};
+  viewFromCenter.m1 = {0.0f, 1.0f, 0.0f, -4.0f};
+  viewFromCenter.m2 = {-s, 0.0f, c, 7.0f};
+
+  const OverlayPanel panel{.halfWidth = 450.0f, .halfHeight = 337.5f, .distance = 1000.0f};
+  const auto composed = compose_overlay_panel_projection(eyeFrustum, viewFromCenter, panel);
+
+  const std::array<Vec4<float>, 5> corners{{
+      {-1.0f, 1.0f, 0.0f, 1.0f},
+      {1.0f, 1.0f, 0.0f, 1.0f},
+      {-1.0f, -1.0f, 0.0f, 1.0f},
+      {1.0f, -1.0f, 0.0f, 1.0f},
+      {0.0f, 0.0f, 0.0f, 1.0f},
+  }};
+  for (const auto& corner : corners) {
+    // Top left is (-1, +1): the panel's +y is up, like the screen it sits on.
+    const Vec4<float> centerPoint{corner[0] * panel.halfWidth, corner[1] * panel.halfHeight, -panel.distance, 1.0f};
+    const float eyeX = dot4(viewFromCenter.m0, centerPoint);
+    const float eyeY = dot4(viewFromCenter.m1, centerPoint);
+    const float eyeZ = dot4(viewFromCenter.m2, centerPoint);
+    const float w = dot4(composed.m3, corner);
+    EXPECT_NEAR(dot4(composed.m0, corner), eyeFrustum.m0[0] * eyeX + eyeFrustum.m0[2] * eyeZ, 1e-2f);
+    EXPECT_NEAR(dot4(composed.m1, corner), eyeFrustum.m1[1] * eyeY + eyeFrustum.m1[2] * eyeZ, 1e-2f);
+    EXPECT_NEAR(w, -eyeZ, 1e-2f);
+    ASSERT_GT(w, 0.0f);
+    EXPECT_NEAR(dot4(composed.m2, corner) / w, 0.5f, 1e-5f);
+  }
+}
+
+TEST(StereoReplayTest, OverlayPanelOnAFlatEyeKeepsItsAspect) {
+  // A 4:3 panel three quarters across a 2064x2208 eye image.
+  const float imageAspect = 2064.0f / 2208.0f;
+  const auto flat = overlay_panel_flat_projection(0.75f, 4.0f / 3.0f, imageAspect);
+  const Vec4<float> topRight{1.0f, 1.0f, 0.0f, 1.0f};
+  const float ndcX = dot4(flat.m0, topRight) / dot4(flat.m3, topRight);
+  const float ndcY = dot4(flat.m1, topRight) / dot4(flat.m3, topRight);
+  EXPECT_FLOAT_EQ(ndcX, 0.75f);
+  // In pixels: 0.75 * 2064 wide over ndcY * 2208 tall is the panel's 4:3.
+  EXPECT_NEAR((ndcX * 2064.0f) / (ndcY * 2208.0f), 4.0f / 3.0f, 1e-4f);
+  EXPECT_FLOAT_EQ(dot4(flat.m2, topRight), 0.5f);
+  const Vec4<float> centre{0.0f, 0.0f, 0.0f, 1.0f};
+  EXPECT_FLOAT_EQ(dot4(flat.m0, centre), 0.0f);
+  EXPECT_FLOAT_EQ(dot4(flat.m1, centre), 0.0f);
+}
+
 Mat3x4<float> identity3x4() {
   Mat3x4<float> m{};
   m.m0 = {1.0f, 0.0f, 0.0f, 0.0f};