Add the VR cockpit overlay to Aurora's eye replay

A stereo packet can now carry an AuroraCockpit: tracked hands and, when the
vehicle's own wheel cannot be animated, a synthetic steering wheel or
handlebar, all in metres in the seated frame. Each eye draws it inside the
scene's pass just before the first virtual-screen draw, depth-tested with
the world's own depth mapping (captured from a full-view world draw), so
the kart and track occlude the hands and the 2D layer cannot hide them.
Hands use a runtime-provided hand mesh when one is supplied and a
procedural glove otherwise.

aurora_set_stereo_scene_anchor_scaled lets the sealed frame own its world
scale: each eye's head translation is rescaled from the packet's scale to
the frame's. A non-finite cockpit is dropped with one warning; the frame
still renders.

Ported from heurazy's mario-kart-wii-VR-port (GPL-3.0-or-later).

aurora-main/tests/CMakeLists.txt

@@ -27,6 +27,13 @@ if (AURORA_GPU_SMOKE_TESTS AND AURORA_ENABLE_GX AND WIN32)
   target_include_directories(efb_ram_lifetime_smoke PRIVATE ../lib)
   target_link_libraries(efb_ram_lifetime_smoke PRIVATE aurora::core aurora::gx aurora::main aurora::vi
                         dawn::dawncpp_headers)
+  # VR cockpit overlay (hands, synthetic wheel) against real scene depth. Standalone: it
+  # defines the GPU globals itself and needs only the header.
+  add_executable(cockpit_gpu_smoke cockpit_gpu_smoke.cpp)
+  target_include_directories(cockpit_gpu_smoke PRIVATE ../include ../lib)
+  target_compile_definitions(cockpit_gpu_smoke PRIVATE AURORA TARGET_PC WEBGPU_DAWN)
+  target_link_libraries(cockpit_gpu_smoke PRIVATE fmt::fmt xxhash absl::flat_hash_map absl::btree
+                        dawn::webgpu_dawn dawn::dawncpp_headers TracyClient ${AURORA_SDL3_TARGET})
 endif ()
 
 if (NOT TARGET gtest)
@@ -49,6 +56,7 @@ if (AURORA_ENABLE_GX)
     stereo_interpolation_test.cpp
     stereo_mirror_test.cpp
     native_wheel_test.cpp
+    cockpit_geometry_test.cpp
     texture_bind_group_cache_key_test.cpp
     ../lib/gfx/efb_ram_encoder.cpp
     # GX API implementations (encoders)

aurora-main/tests/cockpit_geometry_test.cpp

@@ -0,0 +1,149 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+// VR cockpit overlay geometry: what the synthetic wheel, handlebar and hands
+// build in the seated frame, without a GPU.
+#include <gtest/gtest.h>
+
+#include <cstring>
+
+#include "gfx/cockpit.hpp"
+
+namespace {
+using aurora::gfx::cockpit::V;
+using aurora::gfx::cockpit::Vertex;
+
+bool all_finite(const std::vector<Vertex>& vertices) {
+  for (const auto& vertex : vertices) {
+    for (float value : vertex.position) {
+      uint32_t bits = 0;
+      std::memcpy(&bits, &value, sizeof(bits));
+      if ((bits & 0x7f800000u) == 0x7f800000u) {
+        return false;
+      }
+    }
+  }
+  return true;
+}
+
+void set_identity(float (&matrix)[12], V translation) {
+  const auto identity = aurora::gfx::cockpit::identity();
+  std::memcpy(matrix, identity.data(), sizeof(matrix));
+  matrix[3] = translation[0];
+  matrix[7] = translation[1];
+  matrix[11] = translation[2];
+}
+
+class CockpitGeometry : public ::testing::Test {
+protected:
+  void SetUp() override { clear_meshes(); }
+  void TearDown() override { clear_meshes(); }
+  static void clear_meshes() {
+    std::lock_guard lock(aurora::gfx::cockpit::meshMutex);
+    aurora::gfx::cockpit::meshes = {};
+  }
+};
+
+TEST_F(CockpitGeometry, NativeWheelWithoutHandsDrawsNothing) {
+  AuroraCockpit cockpit{};
+  cockpit.nativeWheel = true;
+  EXPECT_TRUE(aurora::gfx::cockpit::geometry(cockpit).empty());
+}
+
+TEST_F(CockpitGeometry, SyntheticKartWheelSitsOnItsRim) {
+  AuroraCockpit cockpit{};
+  const auto vertices = aurora::gfx::cockpit::geometry(cockpit);
+  ASSERT_FALSE(vertices.empty());
+  ASSERT_TRUE(all_finite(vertices));
+  // The rim, spokes and hub stay within the 0.18 m wheel plus its tube, around
+  // the wheel centre the input side uses (steering_wheel.h).
+  for (const auto& vertex : vertices) {
+    const float x = vertex.position[0];
+    const float y = vertex.position[1] + 0.30f;
+    EXPECT_LE(std::hypot(x, y), 0.18f + 0.02f);
+    EXPECT_NEAR(vertex.position[2], -0.42f, 0.04f);
+  }
+}
+
+TEST_F(CockpitGeometry, SyntheticWheelTurnsWithTheAngle) {
+  AuroraCockpit cockpit{};
+  const auto straight = aurora::gfx::cockpit::geometry(cockpit);
+  cockpit.wheelAngle = 0.5f;
+  const auto turned = aurora::gfx::cockpit::geometry(cockpit);
+  ASSERT_EQ(straight.size(), turned.size());
+  bool moved = false;
+  for (size_t i = 0; i < straight.size() && !moved; ++i) {
+    moved = std::abs(straight[i].position[0] - turned[i].position[0]) > 1e-3f;
+  }
+  EXPECT_TRUE(moved);
+}
+
+TEST_F(CockpitGeometry, SyntheticHandlebarFollowsItsFrame) {
+  AuroraCockpit cockpit{};
+  cockpit.bike = true;
+  cockpit.handlebarRadius = 0.25f;
+  // Bar axis along seat +X, centred 0.3 m down and 0.42 m ahead.
+  const float pose[12]{1, 0, 0, 0, 0, 0, 1, -0.3f, 0, -1, 0, -0.42f};
+  std::memcpy(cockpit.seatFromHandlebar, pose, sizeof(pose));
+  const auto vertices = aurora::gfx::cockpit::geometry(cockpit);
+  ASSERT_FALSE(vertices.empty());
+  ASSERT_TRUE(all_finite(vertices));
+  float minX = 1e9f;
+  float maxX = -1e9f;
+  for (const auto& vertex : vertices) {
+    minX = std::min(minX, vertex.position[0]);
+    maxX = std::max(maxX, vertex.position[0]);
+  }
+  EXPECT_NEAR(minX, -0.25f, 0.03f);
+  EXPECT_NEAR(maxX, 0.25f, 0.03f);
+}
+
+TEST_F(CockpitGeometry, TrackedHandDrawsAGloveAtItsGrip) {
+  AuroraCockpit cockpit{};
+  cockpit.nativeWheel = true;
+  cockpit.hands[1].tracked = true;
+  cockpit.hands[1].squeeze = 1.0f;
+  set_identity(cockpit.hands[1].seatFromGrip, {0.2f, -0.3f, -0.4f});
+  const auto vertices = aurora::gfx::cockpit::geometry(cockpit);
+  ASSERT_FALSE(vertices.empty());
+  ASSERT_TRUE(all_finite(vertices));
+  for (const auto& vertex : vertices) {
+    EXPECT_LT(std::abs(vertex.position[0] - 0.2f), 0.15f);
+    EXPECT_LT(std::abs(vertex.position[1] + 0.3f), 0.15f);
+    EXPECT_LT(std::abs(vertex.position[2] + 0.4f), 0.15f);
+  }
+}
+
+TEST_F(CockpitGeometry, RuntimeHandMeshIsSkinnedWithoutNans) {
+  using namespace aurora::gfx::cockpit;
+  auto mesh = std::make_shared<HandMesh>();
+  // A 26-joint chain, each joint 1 cm past its parent; one triangle on the tip.
+  for (int j = 0; j < 26; ++j) {
+    mesh->bind[j] = identity();
+    mesh->bind[j][11] = -0.01f * float(j);
+    mesh->inverseBind[j] = inverse(mesh->bind[j]);
+    mesh->parents[j] = j - 1;
+  }
+  for (int i = 0; i < 3; ++i) {
+    AuroraVRHandVertex vertex{};
+    vertex.position[0] = 0.01f * float(i);
+    vertex.position[2] = -0.25f;
+    vertex.joints[0] = 25;
+    vertex.joints[1] = vertex.joints[2] = vertex.joints[3] = -1;
+    vertex.weights[0] = 1.0f;
+    mesh->vertices.push_back(vertex);
+    mesh->indices.push_back(uint16_t(i));
+  }
+  {
+    std::lock_guard lock(meshMutex);
+    meshes[0] = mesh;
+  }
+  AuroraCockpit cockpit{};
+  cockpit.nativeWheel = true;
+  cockpit.hands[0].tracked = true;
+  cockpit.hands[0].held = true;
+  set_identity(cockpit.hands[0].seatFromGrip, {-0.2f, -0.3f, -0.4f});
+  const auto vertices = geometry(cockpit);
+  ASSERT_EQ(vertices.size(), 3u) << "the runtime mesh replaces the glove";
+  EXPECT_TRUE(all_finite(vertices));
+}
+
+} // namespace

aurora-main/tests/cockpit_gpu_smoke.cpp

@@ -0,0 +1,130 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+// Ported from heurazy's mario-kart-wii-VR-port (GPL-3.0-or-later).
+// Renders the VR cockpit overlay on a real GPU against cleared, occluding and
+// partially occluding scene depth, forward and reversed, 1x and 4x MSAA.
+#include "../lib/gfx/cockpit.hpp"
+#include <fstream>
+#include <iostream>
+#include <atomic>
+namespace aurora::webgpu { wgpu::Device g_device; wgpu::Queue g_queue; GraphicsConfig g_graphicsConfig{}; }
+std::atomic<int> errors=0;
+int main() {
+  using namespace aurora;
+  using namespace webgpu;
+  wgpu::InstanceDescriptor id{};
+  const wgpu::InstanceFeatureName timed=wgpu::InstanceFeatureName::TimedWaitAny;
+  id.requiredFeatureCount=1;id.requiredFeatures=&timed;
+  auto instance=wgpu::CreateInstance(&id);
+  wgpu::Adapter adapter;
+  wgpu::RequestAdapterOptions options{.backendType=wgpu::BackendType::D3D12};
+  auto future=instance.RequestAdapter(&options,wgpu::CallbackMode::WaitAnyOnly,
+    [&](wgpu::RequestAdapterStatus status,wgpu::Adapter a,wgpu::StringView message) {
+      if(status==wgpu::RequestAdapterStatus::Success) adapter=std::move(a);
+      else std::cerr<<std::string_view(message)<<'\n';
+    });
+  if(instance.WaitAny(future,5000000000)!=wgpu::WaitStatus::Success||!adapter) return 1;
+  wgpu::DeviceDescriptor dd{};
+  dd.SetUncapturedErrorCallback([](const wgpu::Device&,wgpu::ErrorType,wgpu::StringView message) {
+    ++errors;std::cerr<<std::string_view(message)<<'\n';
+  });
+  future=adapter.RequestDevice(&dd,wgpu::CallbackMode::WaitAnyOnly,
+    [&](wgpu::RequestDeviceStatus status,wgpu::Device device,wgpu::StringView message) {
+      if(status==wgpu::RequestDeviceStatus::Success) g_device=std::move(device);
+      else std::cerr<<std::string_view(message)<<'\n';
+    });
+  if(instance.WaitAny(future,5000000000)!=wgpu::WaitStatus::Success||!g_device) return 1;
+  g_queue=g_device.GetQueue();
+  g_graphicsConfig.surfaceConfiguration.format=wgpu::TextureFormat::RGBA8Unorm;
+  g_graphicsConfig.depthFormat=wgpu::TextureFormat::Depth32Float;
+  AuroraCockpit native{}; native.nativeWheel=true;
+  if(!gfx::cockpit::geometry(native).empty()) return 1;
+  native.nativeWheel=false;
+  if(gfx::cockpit::geometry(native).empty()) return 1;
+  for(bool bike : {false,true}) for(bool original : {false,true}) for(uint32_t samples : {1u,4u})
+  for(int coverage : {0,1,2}) for(bool reversed : {false,true}) for(uint32_t eyeIndex : {0u,1u}) {
+    const bool occluded=coverage==1;
+    gfx::StereoReplayFrame frame{};
+    frame.cockpit.unitsPerMeter=100;
+    frame.cockpit.active=true;frame.cockpit.wheelAngle=0.35f;
+    frame.cockpit.nativeWheel=original;
+    frame.cockpit.bike=bike;frame.cockpit.handlebarRadius=0.25f;
+    const float handlePose[12]{1,0,0,0, 0,0,1,-0.3f, 0,-1,0,-0.42f};
+    std::memcpy(frame.cockpit.seatFromHandlebar,handlePose,sizeof(handlePose));
+    for(int hand=0;hand<2;++hand) {
+      auto& h=frame.cockpit.hands[hand];h.tracked=true;h.held=true;h.squeeze=1;
+      auto pose=gfx::cockpit::identity();pose[3]=hand?0.18f:-0.18f;pose[7]=-0.30f;pose[11]=-0.42f;
+      std::memcpy(h.seatFromGrip,pose.data(),sizeof(h.seatFromGrip));
+    }
+    wgpu::TextureDescriptor td{.usage=wgpu::TextureUsage::RenderAttachment|wgpu::TextureUsage::CopySrc,
+      .size={512,512,1},.format=wgpu::TextureFormat::RGBA8Unorm,.sampleCount=1};
+    auto output=g_device.CreateTexture(&td);
+    td.sampleCount=samples;td.usage=wgpu::TextureUsage::RenderAttachment;
+    auto color=g_device.CreateTexture(&td);
+    td.format=wgpu::TextureFormat::Depth32Float;auto depth=g_device.CreateTexture(&td);
+    auto& eye=frame.eyes[eyeIndex];eye.target.colorView=samples==1?output.CreateView():color.CreateView();
+    if(samples>1) eye.target.resolveView=output.CreateView();
+    eye.target.depthView=depth.CreateView();eye.target.size={512,512,1};eye.target.msaaSamples=samples;
+    eye.projection.m0[0]=1;eye.projection.m1[1]=1;
+    eye.projection.m0[2]=eyeIndex?0.06f:-0.06f;
+    auto view=gfx::cockpit::identity();view[7]=0.20f;
+    std::memcpy(frame.cockpit.eyeFromSeat[eyeIndex],view.data(),sizeof(frame.cockpit.eyeFromSeat[eyeIndex]));
+    auto encoder=g_device.CreateCommandEncoder();
+    const wgpu::RenderPassColorAttachment clear{.view=eye.target.colorView,.resolveTarget=eye.target.resolveView,
+      .loadOp=wgpu::LoadOp::Clear,.storeOp=wgpu::StoreOp::Store,.clearValue={0.06,0.09,0.13,1}};
+    const wgpu::RenderPassDepthStencilAttachment sceneDepth{.view=eye.target.depthView,
+      .depthLoadOp=wgpu::LoadOp::Clear,.depthStoreOp=wgpu::StoreOp::Store,.depthClearValue=reversed?(occluded?0.8f:0.0f):(occluded?0.2f:1.0f)};
+    const wgpu::RenderPassDescriptor pd{.colorAttachmentCount=1,.colorAttachments=&clear,.depthStencilAttachment=&sceneDepth};
+    auto pass=encoder.BeginRenderPass(&pd);
+    if(coverage==2) {
+      wgpu::ShaderSourceWGSL code{};
+      code.code=R"(
+        @vertex fn vs(@builtin(vertex_index) i:u32) -> @builtin(position) vec4f {
+          let p=array<vec2f,6>(vec2f(0,-1),vec2f(1,-1),vec2f(0,1),vec2f(0,1),vec2f(1,-1),vec2f(1,1));
+          return vec4f(p[i],0.5,1);
+        }
+        @fragment fn fs() -> @location(0) vec4f { return vec4f(0.06,0.09,0.13,1); }
+      )";
+      wgpu::ShaderModuleDescriptor md{};md.nextInChain=&code;
+      auto shader=g_device.CreateShaderModule(&md);
+      const wgpu::ColorTargetState colorState{.format=wgpu::TextureFormat::RGBA8Unorm};
+      const wgpu::FragmentState fragment{.module=shader,.entryPoint="fs",.targetCount=1,.targets=&colorState};
+      const wgpu::DepthStencilState ds{.format=wgpu::TextureFormat::Depth32Float,.depthWriteEnabled=true,.depthCompare=wgpu::CompareFunction::Always};
+      wgpu::RenderPipelineDescriptor desc{};desc.vertex={.module=shader,.entryPoint="vs"};
+      desc.fragment=&fragment;desc.depthStencil=&ds;desc.multisample.count=samples;
+      auto wall=g_device.CreateRenderPipeline(&desc);pass.SetPipeline(wall);pass.Draw(6);
+    }
+    if(coverage==2) gfx::cockpit::render(encoder,frame,eyeIndex,reversed?gfx::cockpit::SceneDepth{0,2,true}:gfx::cockpit::SceneDepth{-1,-2,true},&pass);
+    pass.End();
+    if(coverage!=2) gfx::cockpit::render(encoder,frame,eyeIndex,reversed?gfx::cockpit::SceneDepth{0,2,true}:gfx::cockpit::SceneDepth{-1,-2,true});
+    const wgpu::BufferDescriptor bd{.usage=wgpu::BufferUsage::CopyDst|wgpu::BufferUsage::MapRead,.size=512*512*4};
+    auto readback=g_device.CreateBuffer(&bd);
+    const wgpu::TexelCopyTextureInfo src{.texture=output};
+    const wgpu::TexelCopyBufferInfo dst{.layout={.bytesPerRow=2048,.rowsPerImage=512},.buffer=readback};
+    const wgpu::Extent3D extent{512,512,1};encoder.CopyTextureToBuffer(&src,&dst,&extent);
+    auto commands=encoder.Finish();g_device.GetQueue().Submit(1,&commands);
+    bool mapped=false;
+    future=readback.MapAsync(wgpu::MapMode::Read,0,512*512*4,wgpu::CallbackMode::WaitAnyOnly,
+      [&](wgpu::MapAsyncStatus status,wgpu::StringView) { mapped=status==wgpu::MapAsyncStatus::Success; });
+    if(instance.WaitAny(future,5000000000)!=wgpu::WaitStatus::Success||!mapped) return 1;
+    const auto* bytes=static_cast<const unsigned char*>(readback.GetConstMappedRange());
+    size_t bright=0;
+    for(size_t i=0;i<512*512;++i) if(bytes[4*i]>90&&bytes[4*i+1]>90&&bytes[4*i+2]>90) ++bright;
+    if(occluded ? bright!=0 : bright<1000) { std::cerr<<"Incorrect hands/wheel occlusion\n";++errors; }
+    if(coverage==2) {
+      size_t left=0,right=0;
+      for(size_t y=0;y<512;++y) for(size_t x=0;x<512;++x) {
+        const auto i=y*512+x;
+        if(bytes[4*i]>90&&bytes[4*i+1]>90&&bytes[4*i+2]>90) (x<256?left:right)++;
+      }
+      if(left<500||right>8) { std::cerr<<"Partial wall occlusion failed for eye "<<eyeIndex<<'\n';++errors; }
+    }
+    if(samples==4 && !original && !occluded) {
+      std::ofstream image("cockpit-preview.ppm",std::ios::binary);image<<"P6\n512 512\n255\n";
+      for(size_t i=0;i<512*512;++i) image.write(reinterpret_cast<const char*>(bytes+i*4),3);
+    }
+    readback.Unmap();
+    std::cout<<(bike?"Bike ":"Kart ")<<(original?"native hands: ":"VR controls: ")<<samples<<"x MSAA: "<<bright<<" visible geometry pixels\n";
+  }
+  gfx::cockpit::shutdown();g_queue=nullptr;g_device.Destroy();g_device=nullptr;
+  return errors?1:0;
+}