From 21b8d209dd74a2f3444eabb04adf7f2e4e0377e3 Mon Sep 17 00:00:00 2001 From: iChris4 Date: Tue, 22 Sep 2026 01:40:42 +0200 Subject: [PATCH] Add the VR cockpit overlay to Aurora's eye replay A stereo packet can now carry an AuroraCockpit: tracked hands and, when the vehicle's own wheel cannot be animated, a synthetic steering wheel or handlebar, all in metres in the seated frame. Each eye draws it inside the scene's pass just before the first virtual-screen draw, depth-tested with the world's own depth mapping (captured from a full-view world draw), so the kart and track occlude the hands and the 2D layer cannot hide them. Hands use a runtime-provided hand mesh when one is supplied and a procedural glove otherwise. aurora_set_stereo_scene_anchor_scaled lets the sealed frame own its world scale: each eye's head translation is rescaled from the packet's scale to the frame's. A non-finite cockpit is dropped with one warning; the frame still renders. Ported from heurazy's mario-kart-wii-VR-port (GPL-3.0-or-later). --- aurora-main/include/aurora/aurora.h | 46 ++++ aurora-main/lib/aurora.cpp | 47 ++++ aurora-main/lib/gfx/cockpit.hpp | 272 ++++++++++++++++++++ aurora-main/lib/gfx/common.cpp | 95 +++++++ aurora-main/lib/gfx/common.hpp | 2 + aurora-main/tests/CMakeLists.txt | 8 + aurora-main/tests/cockpit_geometry_test.cpp | 149 +++++++++++ aurora-main/tests/cockpit_gpu_smoke.cpp | 130 ++++++++++ 8 files changed, 749 insertions(+) create mode 100644 aurora-main/lib/gfx/cockpit.hpp create mode 100644 aurora-main/tests/cockpit_geometry_test.cpp create mode 100644 aurora-main/tests/cockpit_gpu_smoke.cpp diff --git a/aurora-main/include/aurora/aurora.h b/aurora-main/include/aurora/aurora.h index 73037fa..5ac54a7 100644 --- a/aurora-main/include/aurora/aurora.h +++ b/aurora-main/include/aurora/aurora.h @@ -116,6 +116,46 @@ typedef enum { // accepted through aurora_end_frame_tagged() with an exact matching tag. #define AURORA_STEREO_CONTENT_TAG_UNKNOWN UINT64_MAX +/** + * VR cockpit overlay: tracked hands and, when the vehicle's own wheel cannot be + * animated, a synthetic steering wheel or handlebar. Everything is in metres + * in a seated frame (+X right, +Y up, -Z forward) whose origin is the headset's + * immersive base position. Aurora draws it per eye after the scene, depth-tested + * against the scene with the scene's own depth mapping. + */ +typedef struct { + bool tracked; + bool held; + float squeeze; + float seatFromGrip[12]; +} AuroraCockpitHand; + +typedef struct { + bool active; + float wheelAngle; + // The vehicle's own wheel is animated in the scene, so no synthetic wheel is drawn. + bool nativeWheel; + bool bike; + float handlebarRadius; + // World units per metre used to build this packet's eye transforms. + float unitsPerMeter; + float seatFromHandlebar[12]; + float eyeFromSeat[AURORA_STEREO_EYE_COUNT][12]; + AuroraCockpitHand hands[2]; +} AuroraCockpit; + +typedef struct { + float position[3]; + int16_t joints[4]; + float weights[4]; +} AuroraVRHandVertex; + +// Copies optional runtime-provided hand meshes (XR_FB_hand_tracking_mesh, 26 +// joints). Null clears to the procedural glove. Bind poses: x,y,z,w,px,py,pz. +void aurora_set_vr_hand_mesh(uint32_t hand, const AuroraVRHandVertex* vertices, uint32_t vertexCount, + const uint16_t* indices, uint32_t indexCount, const float* bindPoses, + const int32_t* parents, uint32_t jointCount); + /** * Stereo data for one sealed GX frame. frameToken is opaque to Aurora and is * forwarded unchanged to the internal stereo output sink. contentTag must @@ -130,6 +170,8 @@ typedef struct { // Predicted display time converted to std::chrono::steady_clock nanoseconds. // Zero disables temporal interpolation for this packet. uint64_t displayTimeNanos; + // Optional; inactive when zero-initialised. + AuroraCockpit cockpit; } AuroraStereoFrame; /** @@ -247,6 +289,10 @@ void aurora_set_frame_log_callback(AuroraFrameLogCallback callback); * provider, which cannot know which frame will consume its packet. */ void aurora_set_stereo_scene_anchor(const float anchorFromScene[12]); +// As above, also naming the world units per metre the anchor was built with. +// The sealed frame then owns that scale: each eye's head/IPD translation is +// rescaled from the packet's AuroraCockpit::unitsPerMeter to it. +void aurora_set_stereo_scene_anchor_scaled(const float anchorFromScene[12], float unitsPerMeter); // Select Player 1's subview for immersive replay of 2-4 local screens. // Producer-thread, per-frame metadata, consumed by the next end_frame call. // One (the default) keeps full-frame replay. Desktop rendering is unaffected. diff --git a/aurora-main/lib/aurora.cpp b/aurora-main/lib/aurora.cpp index 98efd71..fea61d2 100644 --- a/aurora-main/lib/aurora.cpp +++ b/aurora-main/lib/aurora.cpp @@ -113,6 +113,9 @@ struct StereoSceneAnchor { }; bool active = false; uint32_t localPlayerCount = 1; + // World units per metre the anchor was built with, or zero when the packet's + // own scale applies (aurora_set_stereo_scene_anchor_scaled). + float unitsPerMeter = 0.f; }; // Producer thread only, between aurora_set_stereo_scene_anchor() and the seal // that consumes it. Cleared at every seal so a producer that stops publishing @@ -713,6 +716,27 @@ std::optional request_stereo_frame(uint32_t logicalFrame, uin return std::nullopt; } } + // The cockpit overlay is optional: a bad one is dropped, never the frame. + if (frame.cockpit.active) { + const auto& cockpit = frame.cockpit; + bool valid = finite(&cockpit.wheelAngle, 1) && finite(&cockpit.handlebarRadius, 1) && + finite(&cockpit.unitsPerMeter, 1) && cockpit.unitsPerMeter > 0.f && + finite(cockpit.seatFromHandlebar, 12); + for (uint32_t eye = 0; eye < AURORA_STEREO_EYE_COUNT; ++eye) { + valid = valid && finite(cockpit.eyeFromSeat[eye], 12); + } + for (const auto& hand : cockpit.hands) { + valid = valid && finite(&hand.squeeze, 1) && finite(hand.seatFromGrip, 12); + } + if (!valid) { + static bool cockpitRejectionLogged = false; + if (!cockpitRejectionLogged) { + cockpitRejectionLogged = true; + Log.warn("Stereo frame {} carries a non-finite VR cockpit; drawing it without the cockpit", logicalFrame); + } + frame.cockpit = {}; + } + } return frame; } @@ -720,6 +744,16 @@ gfx::StereoReplayFrame make_stereo_replay_frame(const AuroraStereoFrame& input, Mat3x4 anchorFromScene; std::memcpy(&anchorFromScene, sceneAnchor.anchorFromScene.data(), sizeof(anchorFromScene)); gfx::StereoReplayFrame replay{}; + replay.cockpit = input.cockpit; + // The sealed guest frame owns its scale. The packet may have been sampled + // just before a change of scale (a character swap, a lightning strike), so + // only its head/IPD translation is rescaled to the frame's. + const float frameUnits = sceneAnchor.active && sceneAnchor.unitsPerMeter > 0.f ? sceneAnchor.unitsPerMeter + : input.cockpit.unitsPerMeter; + const float unitRatio = input.cockpit.unitsPerMeter > 0.f && frameUnits > 0.f + ? frameUnits / input.cockpit.unitsPerMeter + : 1.f; + replay.cockpit.unitsPerMeter = frameUnits; for (uint32_t eye = 0; eye < AURORA_STEREO_EYE_COUNT; ++eye) { ensure_stereo_eye_target(eye, input.eyes[eye].width, input.eyes[eye].height); const auto& owned = g_stereoEyeTargets[eye]; @@ -737,6 +771,11 @@ gfx::StereoReplayFrame make_stereo_replay_frame(const AuroraStereoFrame& input, }; std::memcpy(&view.projection, input.eyes[eye].projection, sizeof(view.projection)); std::memcpy(&view.viewFromCenter, input.eyes[eye].viewFromCenter, sizeof(view.viewFromCenter)); + if (unitRatio != 1.f) { + view.viewFromCenter.m0[3] *= unitRatio; + view.viewFromCenter.m1[3] *= unitRatio; + view.viewFromCenter.m2[3] *= unitRatio; + } // World draws already carry the recorded camera, so they need the anchor // folded in; the virtual screen is authored in the anchored camera's space // and keeps viewFromCenter. @@ -2607,6 +2646,14 @@ extern "C" void aurora_imgui_host_frame_release(void* imguiFrame) { void aurora_set_stereo_scene_anchor(const float anchorFromScene[12]) { aurora::set_stereo_scene_anchor(anchorFromScene); } +void aurora_set_stereo_scene_anchor_scaled(const float anchorFromScene[12], float unitsPerMeter) { + aurora::set_stereo_scene_anchor(anchorFromScene); + uint32_t bits = 0; + std::memcpy(&bits, &unitsPerMeter, sizeof(bits)); + if (aurora::g_pendingSceneAnchor.active && (bits & 0x7f800000u) != 0x7f800000u && unitsPerMeter > 0.f) { + aurora::g_pendingSceneAnchor.unitsPerMeter = unitsPerMeter; + } +} void aurora_set_stereo_local_player_count(uint32_t count) { aurora::g_pendingStereoLocalPlayerCount = count >= 1 && count <= 4 ? count : 1; } diff --git a/aurora-main/lib/gfx/cockpit.hpp b/aurora-main/lib/gfx/cockpit.hpp new file mode 100644 index 0000000..ba81b39 --- /dev/null +++ b/aurora-main/lib/gfx/cockpit.hpp @@ -0,0 +1,272 @@ +// SPDX-License-Identifier: GPL-3.0-or-later +// Ported from heurazy's mario-kart-wii-VR-port (GPL-3.0-or-later). +// +// VR cockpit overlay: the synthetic steering wheel or handlebar (used when the +// vehicle's own wheel cannot be animated) and the tracked hands, drawn per eye +// in metres against the replayed scene's depth. See OPENXR.md, "Steering wheel +// and hand steering". +#pragma once +#include "common.hpp" +#include "../webgpu/gpu.hpp" +#include +#include +#include +#include +#include +#include +#include + +namespace aurora::gfx::cockpit { +using V = std::array; +using M = std::array; +inline V add(V a, V b) { return {a[0]+b[0], a[1]+b[1], a[2]+b[2]}; } +inline V sub(V a, V b) { return {a[0]-b[0], a[1]-b[1], a[2]-b[2]}; } +inline V mul(V a, float b) { return {a[0]*b, a[1]*b, a[2]*b}; } +inline float dot(V a, V b) { return a[0]*b[0]+a[1]*b[1]+a[2]*b[2]; } +inline V cross(V a, V b) { return {a[1]*b[2]-a[2]*b[1],a[2]*b[0]-a[0]*b[2],a[0]*b[1]-a[1]*b[0]}; } +inline V norm(V a) { return mul(a, 1/std::sqrt(std::max(dot(a,a), 1e-10f))); } +inline V point(const float* m, V p) { + return {m[0]*p[0]+m[1]*p[1]+m[2]*p[2]+m[3], m[4]*p[0]+m[5]*p[1]+m[6]*p[2]+m[7], + m[8]*p[0]+m[9]*p[1]+m[10]*p[2]+m[11]}; +} +inline M identity() { return {1,0,0,0,0,1,0,0,0,0,1,0}; } +inline M compose(const M& a, const M& b) { + M result{}; + for(int r=0;r<3;++r) { + for(int c=0;c<3;++c) for(int k=0;k<3;++k) result[r*4+c]+=a[r*4+k]*b[k*4+c]; + result[r*4+3]=a[r*4+3]; + for(int k=0;k<3;++k) result[r*4+3]+=a[r*4+k]*b[k*4+3]; + } + return result; +} +inline M inverse(const M& m) { + M out=identity(); + for(int r=0;r<3;++r) for(int c=0;c<3;++c) out[r*4+c]=m[c*4+r]; + const auto p=point(out.data(), {-m[3],-m[7],-m[11]}); + out[3]=p[0];out[7]=p[1];out[11]=p[2];return out; +} +inline M from_pose(const float* p) { + const float x=p[0],y=p[1],z=p[2],w=p[3]; + return {1-2*(y*y+z*z),2*(x*y-z*w),2*(x*z+y*w),p[4], + 2*(x*y+z*w),1-2*(x*x+z*z),2*(y*z-x*w),p[5], + 2*(x*z-y*w),2*(y*z+x*w),1-2*(x*x+y*y),p[6]}; +} +struct HandMesh { + std::vector vertices; + std::vector indices; + std::array bind{}, inverseBind{}; + std::array parents{}; +}; +inline std::mutex meshMutex; +inline std::array,2> meshes; +struct Vertex { V position, color; }; +inline void triangle(std::vector& vertices, V a, V b, V c, V color) { + const V normal=norm(cross(sub(b,a),sub(c,a))); + const float light=0.55f+0.45f*std::abs(dot(normal,norm({0.3f,0.8f,0.5f}))); + color=mul(color,light); + vertices.insert(vertices.end(),{{a,color},{b,color},{c,color}}); +} +inline void tube(std::vector& v, V a, V b, float radius, V color, int sides=8) { + const auto direction=norm(sub(b,a)); + const auto u=norm(cross(direction,std::abs(direction[1])<0.9f?V{0,1,0}:V{1,0,0})); + const auto w=cross(direction,u); + for(int i=0;i& vertices,V center,V radii,V color) { + const auto surface=[&](int ring,int segment) { + const float latitude=float(ring)*3.14159265f/6,longitude=float(segment)*6.2831853f/12; + return add(center,{radii[0]*std::sin(latitude)*std::cos(longitude),radii[1]*std::cos(latitude), + radii[2]*std::sin(latitude)*std::sin(longitude)}); + }; + for(int ring=0;ring<6;++ring) for(int segment=0;segment<12;++segment) { + const auto a=surface(ring,segment),b=surface(ring+1,segment),c=surface(ring+1,segment+1),d=surface(ring,segment+1); + if(ring>0) triangle(vertices,a,b,d,color); + if(ring<5) triangle(vertices,b,c,d,color); + } +} +inline void glove(std::vector& v, const AuroraCockpitHand& hand, int side) { + const size_t start=v.size(); + const V white{0.91f,0.95f,1.0f}; + // Rounded palm and individually articulated fingers in grip-local metres. + ellipsoid(v,{0,0,0},{0.041f,0.018f,0.043f},white); + const float curl=std::clamp(hand.held?0.85f:hand.squeeze,0.0f,1.0f); + for(int finger=0;finger<4;++finger) { + V a{-0.025f+finger*0.017f,0,-0.028f}; + const float length=finger==0||finger==3?0.021f:0.026f; + for(int joint=0;joint<3;++joint) { + const float angle=curl*(0.55f+joint*0.8f); + V b=add(a,{0,-std::sin(angle)*length,-std::cos(angle)*length}); + tube(v,a,b,0.008f,white); + ellipsoid(v,b,{0.008f,0.008f,0.008f},white);a=b; + } + } + const float sign=side? -1.0f:1.0f; + tube(v,{sign*0.025f,0,0.012f},{sign*0.048f,-0.012f,-0.012f},0.010f,white); + tube(v,{sign*0.048f,-0.012f,-0.012f},{sign*(0.055f-0.021f*curl),-0.018f,-0.041f},0.009f,white); + for(size_t i=start;i& out, const AuroraCockpitHand& hand, const HandMesh& mesh) { + std::array posed{}, skin{}; + std::array done{}; + const float curl=std::clamp(hand.held?0.85f:hand.squeeze,0.0f,1.0f); + // Bind hierarchy is supplied by the runtime. Root and wrist stay rigid; + // finger joints curl locally when controllers provide squeeze input. + for(int pass=0;pass<26;++pass) for(int j=0;j<26;++j) { + if(done[j]) continue; + const int parent=mesh.parents[j]; + if(parent>=0&&parent<26&&!done[parent]) continue; + M local=parent>=0&&parent<26?compose(mesh.inverseBind[parent],mesh.bind[j]):mesh.bind[j]; + const bool fingerJoint=j>=2 && j!=6 && j!=11 && j!=16 && j!=21; + if(fingerJoint) { + const float a=curl*(j<6?0.3f:0.75f),c=std::cos(a),s=std::sin(a); + local=compose(local,M{1,0,0,0,0,c,-s,0,0,s,c,0}); + } + posed[j]=parent>=0&&parent<26?compose(posed[parent],local):local; + skin[j]=compose(mesh.inverseBind[1],compose(posed[j],mesh.inverseBind[j])); + done[j]=true; + } + std::vector points(mesh.vertices.size()); + for(size_t i=0;i=0&&v.joints[w]<26&&done[v.joints[w]]&&v.weights[w]>0) { + p=add(p,mul(point(skin[v.joints[w]].data(),{v.position[0],v.position[1],v.position[2]}),v.weights[w])); + total+=v.weights[w]; + } + if(total>0) p=mul(p,1/total); + p=add(p,{0,0,0.04f}); // wrist behind the controller grip/palm origin. + points[i]=point(hand.seatFromGrip,p); + } + for(size_t i=0;i+2& vertices) { + vertices.clear();vertices.reserve(12000); + // The visible radius and position must match runtime/vr/steering_wheel.h. + if (!cockpit.nativeWheel && cockpit.bike) { + const float c=std::cos(cockpit.wheelAngle),s=std::sin(cockpit.wheelAngle); + const auto barPoint=[&](float x,float y,float z) { + return point(cockpit.seatFromHandlebar,{c*x+s*y,-s*x+c*y,z}); + }; + const float radius=cockpit.handlebarRadius; + tube(vertices,barPoint(-radius,0,0),barPoint(radius,0,0),0.013f,{0.45f,0.48f,0.52f}); + for(float side:{-1.0f,1.0f}) + tube(vertices,barPoint(side*std::max(radius-0.10f,0.0f),0,0),barPoint(side*radius,0,0),0.024f,{0.12f,0.18f,0.19f}); + tube(vertices,barPoint(0,0,-0.13f),barPoint(0,0,0),0.023f,{0.12f,0.65f,0.61f}); + } else if (!cockpit.nativeWheel) { + const auto rim=[&](float angle) -> V { return {0.18f*std::cos(angle),-0.30f+0.18f*std::sin(angle),-0.42f}; }; + for(int i=0;i<64;++i) { + const float angle=float(i)*6.2831853f/64-cockpit.wheelAngle; + const V color=i>=15&&i<=17?V{0.2f,0.9f,0.8f}:V{0.14f,0.17f,0.20f}; + tube(vertices,rim(angle),rim(angle+6.2831853f/64),0.016f,color,6); + } + for(float a : {0.0f,3.14159265f,4.71238898f}) + tube(vertices,{0,-0.30f,-0.42f},rim(a-cockpit.wheelAngle),0.011f,{0.45f,0.48f,0.52f}); + tube(vertices,{0,-0.30f,-0.445f},{0,-0.30f,-0.395f},0.035f,{0.12f,0.65f,0.61f},16); + } + std::array,2> current; + { std::lock_guard lock(meshMutex);current=meshes; } + for(int side=0;side<2;++side) if(cockpit.hands[side].tracked) { + if(current[side]) runtime_hand(vertices,cockpit.hands[side],*current[side]); + else glove(vertices,cockpit.hands[side],side); + } +} +inline std::vector geometry(const AuroraCockpit& cockpit) { + std::vector result;build_geometry(cockpit,result);return result; +} +inline std::atomic meshRevision{1}; +inline std::vector frameVertices; +inline AuroraCockpit cachedCockpit{}; +inline uint64_t cachedMeshRevision=0; +inline wgpu::RenderPipeline pipeline; +struct SceneDepth { + float z=0, constant=0; + bool valid=false; +}; +inline uint32_t pipelineSamples=0; +inline bool pipelineReversedDepth=false; +inline wgpu::TextureFormat pipelineFormat{}; +inline std::array vertexBuffers; +inline std::array vertexCapacity{}; +inline void shutdown() { pipeline=nullptr;pipelineSamples=0;vertexBuffers={};vertexCapacity={};cachedMeshRevision=0;frameVertices.clear(); } +inline void render(wgpu::CommandEncoder& cmd,const StereoReplayFrame& frame,uint32_t eye,SceneDepth sceneDepth={}, + const wgpu::RenderPassEncoder* existingPass=nullptr) { + if(!frame.cockpit.active || !sceneDepth.valid) return; + using namespace webgpu; + const auto& target=frame.eyes[eye].target; + const auto format=g_graphicsConfig.surfaceConfiguration.format; + // The guest can reverse its viewport depth independently of Aurora's + // global reversed-Z convention. The final 1/d coefficient is authoritative. + const bool reversedDepth=sceneDepth.constant>0; + if(!pipeline||pipelineSamples!=target.msaaSamples||pipelineFormat!=format||pipelineReversedDepth!=reversedDepth) { + wgpu::ShaderSourceWGSL source{}; + source.code=R"( + struct Out { @builtin(position) position: vec4f, @location(0) color: vec3f }; + @vertex fn vs(@location(0) position: vec4f, @location(1) color: vec3f) -> Out { + var o: Out; o.position=position; o.color=color; return o; + } + @fragment fn fs(i: Out) -> @location(0) vec4f { return vec4f(i.color,1); } + )"; + wgpu::ShaderModuleDescriptor md{};md.nextInChain=&source;md.label="VR cockpit hands and wheel"; + auto shader=g_device.CreateShaderModule(&md); + const wgpu::VertexAttribute attrs[]={{.format=wgpu::VertexFormat::Float32x4,.offset=0,.shaderLocation=0}, + {.format=wgpu::VertexFormat::Float32x3,.offset=16,.shaderLocation=1}}; + const wgpu::VertexBufferLayout layout{.arrayStride=28,.attributeCount=2,.attributes=attrs}; + const wgpu::ColorTargetState color{.format=format}; + const wgpu::FragmentState fragment{.module=shader,.entryPoint="fs",.targetCount=1,.targets=&color}; + const wgpu::DepthStencilState depth{.format=g_graphicsConfig.depthFormat,.depthWriteEnabled=true, + .depthCompare=reversedDepth?wgpu::CompareFunction::GreaterEqual:wgpu::CompareFunction::LessEqual}; + wgpu::RenderPipelineDescriptor desc{};desc.label="VR cockpit"; + desc.vertex={.module=shader,.entryPoint="vs",.bufferCount=1,.buffers=&layout}; + desc.fragment=&fragment;desc.depthStencil=&depth;desc.multisample.count=target.msaaSamples; + desc.primitive.topology=wgpu::PrimitiveTopology::TriangleList; + pipeline=g_device.CreateRenderPipeline(&desc);pipelineSamples=target.msaaSamples;pipelineFormat=format; + pipelineReversedDepth=reversedDepth; + } + const auto revision=meshRevision.load(); + if(cachedMeshRevision!=revision || std::memcmp(&cachedCockpit,&frame.cockpit,sizeof(AuroraCockpit))!=0) { + build_geometry(frame.cockpit,frameVertices); + cachedCockpit=frame.cockpit;cachedMeshRevision=revision; + } + const auto& vertices=frameVertices; + if(vertices.empty()) return; + struct ClipVertex { float p[4]; V color; }; + static std::vector clip; + clip.resize(vertices.size()); + const auto& projection=frame.eyes[eye].projection; + for(size_t i=0;i; struct RenderPass { + // The world depth mapping of this pass's last full-view perspective draw, for + // the VR cockpit overlay (set by prepare_stereo_replay_uniforms). + cockpit::SceneDepth cockpitDepth{}; wgpu::TextureView colorView; wgpu::TextureView resolveView; // MSAA resolve target; null if msaaSamples == 1 wgpu::TextureView depthView; @@ -1057,6 +1061,7 @@ void initialize() { } void shutdown() { + cockpit::shutdown(); shutdown_pipeline_cache(); gx::clear_shader_module_cache(); efb_ram::shutdown(); @@ -1512,6 +1517,7 @@ static bool prepare_stereo_replay_uniforms(const StereoReplayFrame& stereoFrame, std::array sourceUniform; std::array eyeUniform; for (auto& pass : g_renderPasses) { + pass.cockpitDepth = {}; if (!pass.efbTarget) { continue; } @@ -1541,6 +1547,19 @@ static bool prepare_stereo_replay_uniforms(const StereoReplayFrame& stereoFrame, std::memcpy(sourceUniform.data(), g_uniforms.data() + draw.uniformRange.offset, draw.uniformRange.size); Mat4x4 gameProjection; std::memcpy(&gameProjection, sourceUniform.data() + layout.projectionOffset, sizeof(gameProjection)); + // The VR cockpit overlay (hands, synthetic wheel) is drawn in metres and + // depth-tested against the world, so it needs the world's own depth + // mapping: the backend depth row of a full-view world draw, with this + // viewport's depth range folded in because the overlay draws with 0..1. + // Camera-attached effects share the camera's projection, so any full-view + // perspective draw describes the same mapping. + if (layout.perspective && !layout.nativeEfbEffect && gameProjection.m2[3] != 0.0f && + drawViewport.width >= displayRegion.width * 0.9f && drawViewport.height >= displayRegion.height * 0.9f) { + const auto row = stereo_replay::backend_ndc_depth_row(gameProjection); + const float low = std::clamp(std::min(drawViewport.znear, drawViewport.zfar), 0.f, 1.f); + const float high = std::clamp(std::max(drawViewport.znear, drawViewport.zfar), 0.f, 1.f); + pass.cockpitDepth = {row[2] * (high - low) - low, row[3] * (high - low), true}; + } // Only a genuinely affine projection carries its NDC position in its clip // position, which is what the virtual screen reprojection consumes. GX // tracks the projection type separately from the matrix, so a 2D draw @@ -1733,6 +1752,13 @@ struct RenderInvocation { bool encodeTextureBakes = true; bool encodeResolves = true; bool captureDepth = true; + // VR cockpit overlay, drawn inside the scene's pass just before the first + // virtual-screen draw so the 2D layer's depth cannot hide it (see render_stereo_eye). + const StereoReplayFrame* cockpitFrame = nullptr; + wgpu::CommandEncoder* cockpitEncoder = nullptr; + cockpit::SceneDepth cockpitDepth{}; + bool* cockpitDrawn = nullptr; + bool* sceneDrawn = nullptr; }; static void render_pass_impl(const wgpu::RenderPassEncoder& pass, const std::vector& passes, u32 idx, @@ -2025,6 +2051,18 @@ void render_stereo_eye(SealedFrame& frame, wgpu::CommandEncoder& cmd, const Ster // The eye is a fresh per-frame attachment, not the reused EFB, so replaying // past that copy blanks the very image the game presented. const int32_t lastPass = get_stereo_stop_at_display_copy() ? displaySource.lastDisplayCopyPass : -1; + cockpit::SceneDepth cockpitDepth{}; + for (size_t i = 0; i < frame.data().passes.size(); ++i) { + if (lastPass >= 0 && i > static_cast(lastPass)) { + break; + } + if (frame.data().passes[i].cockpitDepth.valid) { + cockpitDepth = frame.data().passes[i].cockpitDepth; + } + } + bool cockpitDrawn = false; + bool sceneDrawn = false; + const bool cockpitActive = stereoFrame.cockpit.active && cockpitDepth.valid; render_impl(frame.data().passes, cmd, RenderInvocation{ .stereoEye = eye, @@ -2038,7 +2076,17 @@ void render_stereo_eye(SealedFrame& frame, wgpu::CommandEncoder& cmd, const Ster .encodeTextureBakes = false, .encodeResolves = false, .captureDepth = false, + .cockpitFrame = cockpitActive ? &stereoFrame : nullptr, + .cockpitEncoder = &cmd, + .cockpitDepth = cockpitDepth, + .cockpitDrawn = &cockpitDrawn, + .sceneDrawn = &sceneDrawn, }); + // A frame without a virtual-screen draw after its world still gets the + // overlay, in a pass of its own over the finished eye. + if (cockpitActive && !cockpitDrawn) { + cockpit::render(cmd, stereoFrame, eye, cockpitDepth); + } } void render(wgpu::CommandEncoder& cmd, int32_t interpolatedFrame, bool finalize) { @@ -2452,6 +2500,24 @@ static void render_pass_impl(const wgpu::RenderPassEncoder& pass, const std::vec draw.gx.interpolatedUniformRanges[invocation.interpolatedFrame].size != 0) { uniformOverride = &draw.gx.interpolatedUniformRanges[invocation.interpolatedFrame]; } + // Draw the VR cockpit against the world's depth before the first HUD + // draw can write a screen-plane depth over it, inside this open pass. + if (invocation.cockpitFrame != nullptr && overrideTarget) { + if (draw.gx.uniformReplayLayout.perspective) { + *invocation.sceneDrawn = true; + } + if (virtualScreenDraw && *invocation.sceneDrawn && !*invocation.cockpitDrawn) { + cockpit::render(*invocation.cockpitEncoder, *invocation.cockpitFrame, invocation.stereoEye, + invocation.cockpitDepth, &pass); + *invocation.cockpitDrawn = true; + encodeState = {}; + encodeState.boundTextureBindGroup = gx::g_emptyTextureBindGroup.Get(); + pass.SetBindGroup(0, g_staticBindGroup); + pass.SetBindGroup(2, gx::g_emptyTextureBindGroup); + scissorStateKnown = false; + viewportStateKnown = false; + } + } // Such a draw no longer lands where the game aimed it, while the // recorded scissor still describes the rectangle it occupied on the flat // frame (Mario Kart clips the item roulette that way). Honouring that @@ -2707,3 +2773,32 @@ void aurora_pop_debug_group() { } const AuroraStats* aurora_get_stats() { return &aurora::gfx::g_stats; } + +void aurora_set_vr_hand_mesh(uint32_t hand, const AuroraVRHandVertex* vertices, uint32_t vertexCount, + const uint16_t* indices, uint32_t indexCount, const float* bindPoses, + const int32_t* parents, uint32_t jointCount) { + using namespace aurora::gfx::cockpit; + if (hand >= 2) { + return; + } + std::shared_ptr mesh; + if (vertices && indices && bindPoses && parents && jointCount == 26 && vertexCount > 0 && vertexCount <= 65535 && + indexCount <= 100000 && indexCount % 3 == 0) { + for (uint32_t i = 0; i < indexCount; ++i) { + if (indices[i] >= vertexCount) { + return; + } + } + mesh = std::make_shared(); + mesh->vertices.assign(vertices, vertices + vertexCount); + mesh->indices.assign(indices, indices + indexCount); + for (int j = 0; j < 26; ++j) { + mesh->bind[j] = from_pose(bindPoses + j * 7); + mesh->inverseBind[j] = inverse(mesh->bind[j]); + mesh->parents[j] = parents[j]; + } + } + std::lock_guard lock(meshMutex); + meshes[hand] = std::move(mesh); + ++meshRevision; +} diff --git a/aurora-main/lib/gfx/common.hpp b/aurora-main/lib/gfx/common.hpp index 4612b68..b86c6c8 100644 --- a/aurora-main/lib/gfx/common.hpp +++ b/aurora-main/lib/gfx/common.hpp @@ -314,6 +314,8 @@ struct StereoReplayEye { struct StereoReplayFrame { std::array eyes; + // VR hands and synthetic wheel, drawn per eye after the world (gfx/cockpit.hpp). + AuroraCockpit cockpit{}; }; void end_frame(const wgpu::CommandEncoder& cmd); diff --git a/aurora-main/tests/CMakeLists.txt b/aurora-main/tests/CMakeLists.txt index dc6e660..fdc8147 100644 --- a/aurora-main/tests/CMakeLists.txt +++ b/aurora-main/tests/CMakeLists.txt @@ -27,6 +27,13 @@ if (AURORA_GPU_SMOKE_TESTS AND AURORA_ENABLE_GX AND WIN32) target_include_directories(efb_ram_lifetime_smoke PRIVATE ../lib) target_link_libraries(efb_ram_lifetime_smoke PRIVATE aurora::core aurora::gx aurora::main aurora::vi dawn::dawncpp_headers) + # VR cockpit overlay (hands, synthetic wheel) against real scene depth. Standalone: it + # defines the GPU globals itself and needs only the header. + add_executable(cockpit_gpu_smoke cockpit_gpu_smoke.cpp) + target_include_directories(cockpit_gpu_smoke PRIVATE ../include ../lib) + target_compile_definitions(cockpit_gpu_smoke PRIVATE AURORA TARGET_PC WEBGPU_DAWN) + target_link_libraries(cockpit_gpu_smoke PRIVATE fmt::fmt xxhash absl::flat_hash_map absl::btree + dawn::webgpu_dawn dawn::dawncpp_headers TracyClient ${AURORA_SDL3_TARGET}) endif () if (NOT TARGET gtest) @@ -49,6 +56,7 @@ if (AURORA_ENABLE_GX) stereo_interpolation_test.cpp stereo_mirror_test.cpp native_wheel_test.cpp + cockpit_geometry_test.cpp texture_bind_group_cache_key_test.cpp ../lib/gfx/efb_ram_encoder.cpp # GX API implementations (encoders) diff --git a/aurora-main/tests/cockpit_geometry_test.cpp b/aurora-main/tests/cockpit_geometry_test.cpp new file mode 100644 index 0000000..94f1ad7 --- /dev/null +++ b/aurora-main/tests/cockpit_geometry_test.cpp @@ -0,0 +1,149 @@ +// SPDX-License-Identifier: GPL-3.0-or-later +// VR cockpit overlay geometry: what the synthetic wheel, handlebar and hands +// build in the seated frame, without a GPU. +#include + +#include + +#include "gfx/cockpit.hpp" + +namespace { +using aurora::gfx::cockpit::V; +using aurora::gfx::cockpit::Vertex; + +bool all_finite(const std::vector& vertices) { + for (const auto& vertex : vertices) { + for (float value : vertex.position) { + uint32_t bits = 0; + std::memcpy(&bits, &value, sizeof(bits)); + if ((bits & 0x7f800000u) == 0x7f800000u) { + return false; + } + } + } + return true; +} + +void set_identity(float (&matrix)[12], V translation) { + const auto identity = aurora::gfx::cockpit::identity(); + std::memcpy(matrix, identity.data(), sizeof(matrix)); + matrix[3] = translation[0]; + matrix[7] = translation[1]; + matrix[11] = translation[2]; +} + +class CockpitGeometry : public ::testing::Test { +protected: + void SetUp() override { clear_meshes(); } + void TearDown() override { clear_meshes(); } + static void clear_meshes() { + std::lock_guard lock(aurora::gfx::cockpit::meshMutex); + aurora::gfx::cockpit::meshes = {}; + } +}; + +TEST_F(CockpitGeometry, NativeWheelWithoutHandsDrawsNothing) { + AuroraCockpit cockpit{}; + cockpit.nativeWheel = true; + EXPECT_TRUE(aurora::gfx::cockpit::geometry(cockpit).empty()); +} + +TEST_F(CockpitGeometry, SyntheticKartWheelSitsOnItsRim) { + AuroraCockpit cockpit{}; + const auto vertices = aurora::gfx::cockpit::geometry(cockpit); + ASSERT_FALSE(vertices.empty()); + ASSERT_TRUE(all_finite(vertices)); + // The rim, spokes and hub stay within the 0.18 m wheel plus its tube, around + // the wheel centre the input side uses (steering_wheel.h). + for (const auto& vertex : vertices) { + const float x = vertex.position[0]; + const float y = vertex.position[1] + 0.30f; + EXPECT_LE(std::hypot(x, y), 0.18f + 0.02f); + EXPECT_NEAR(vertex.position[2], -0.42f, 0.04f); + } +} + +TEST_F(CockpitGeometry, SyntheticWheelTurnsWithTheAngle) { + AuroraCockpit cockpit{}; + const auto straight = aurora::gfx::cockpit::geometry(cockpit); + cockpit.wheelAngle = 0.5f; + const auto turned = aurora::gfx::cockpit::geometry(cockpit); + ASSERT_EQ(straight.size(), turned.size()); + bool moved = false; + for (size_t i = 0; i < straight.size() && !moved; ++i) { + moved = std::abs(straight[i].position[0] - turned[i].position[0]) > 1e-3f; + } + EXPECT_TRUE(moved); +} + +TEST_F(CockpitGeometry, SyntheticHandlebarFollowsItsFrame) { + AuroraCockpit cockpit{}; + cockpit.bike = true; + cockpit.handlebarRadius = 0.25f; + // Bar axis along seat +X, centred 0.3 m down and 0.42 m ahead. + const float pose[12]{1, 0, 0, 0, 0, 0, 1, -0.3f, 0, -1, 0, -0.42f}; + std::memcpy(cockpit.seatFromHandlebar, pose, sizeof(pose)); + const auto vertices = aurora::gfx::cockpit::geometry(cockpit); + ASSERT_FALSE(vertices.empty()); + ASSERT_TRUE(all_finite(vertices)); + float minX = 1e9f; + float maxX = -1e9f; + for (const auto& vertex : vertices) { + minX = std::min(minX, vertex.position[0]); + maxX = std::max(maxX, vertex.position[0]); + } + EXPECT_NEAR(minX, -0.25f, 0.03f); + EXPECT_NEAR(maxX, 0.25f, 0.03f); +} + +TEST_F(CockpitGeometry, TrackedHandDrawsAGloveAtItsGrip) { + AuroraCockpit cockpit{}; + cockpit.nativeWheel = true; + cockpit.hands[1].tracked = true; + cockpit.hands[1].squeeze = 1.0f; + set_identity(cockpit.hands[1].seatFromGrip, {0.2f, -0.3f, -0.4f}); + const auto vertices = aurora::gfx::cockpit::geometry(cockpit); + ASSERT_FALSE(vertices.empty()); + ASSERT_TRUE(all_finite(vertices)); + for (const auto& vertex : vertices) { + EXPECT_LT(std::abs(vertex.position[0] - 0.2f), 0.15f); + EXPECT_LT(std::abs(vertex.position[1] + 0.3f), 0.15f); + EXPECT_LT(std::abs(vertex.position[2] + 0.4f), 0.15f); + } +} + +TEST_F(CockpitGeometry, RuntimeHandMeshIsSkinnedWithoutNans) { + using namespace aurora::gfx::cockpit; + auto mesh = std::make_shared(); + // A 26-joint chain, each joint 1 cm past its parent; one triangle on the tip. + for (int j = 0; j < 26; ++j) { + mesh->bind[j] = identity(); + mesh->bind[j][11] = -0.01f * float(j); + mesh->inverseBind[j] = inverse(mesh->bind[j]); + mesh->parents[j] = j - 1; + } + for (int i = 0; i < 3; ++i) { + AuroraVRHandVertex vertex{}; + vertex.position[0] = 0.01f * float(i); + vertex.position[2] = -0.25f; + vertex.joints[0] = 25; + vertex.joints[1] = vertex.joints[2] = vertex.joints[3] = -1; + vertex.weights[0] = 1.0f; + mesh->vertices.push_back(vertex); + mesh->indices.push_back(uint16_t(i)); + } + { + std::lock_guard lock(meshMutex); + meshes[0] = mesh; + } + AuroraCockpit cockpit{}; + cockpit.nativeWheel = true; + cockpit.hands[0].tracked = true; + cockpit.hands[0].held = true; + set_identity(cockpit.hands[0].seatFromGrip, {-0.2f, -0.3f, -0.4f}); + const auto vertices = geometry(cockpit); + ASSERT_EQ(vertices.size(), 3u) << "the runtime mesh replaces the glove"; + EXPECT_TRUE(all_finite(vertices)); +} + +} // namespace diff --git a/aurora-main/tests/cockpit_gpu_smoke.cpp b/aurora-main/tests/cockpit_gpu_smoke.cpp new file mode 100644 index 0000000..a8189c2 --- /dev/null +++ b/aurora-main/tests/cockpit_gpu_smoke.cpp @@ -0,0 +1,130 @@ +// SPDX-License-Identifier: GPL-3.0-or-later +// Ported from heurazy's mario-kart-wii-VR-port (GPL-3.0-or-later). +// Renders the VR cockpit overlay on a real GPU against cleared, occluding and +// partially occluding scene depth, forward and reversed, 1x and 4x MSAA. +#include "../lib/gfx/cockpit.hpp" +#include +#include +#include +namespace aurora::webgpu { wgpu::Device g_device; wgpu::Queue g_queue; GraphicsConfig g_graphicsConfig{}; } +std::atomic errors=0; +int main() { + using namespace aurora; + using namespace webgpu; + wgpu::InstanceDescriptor id{}; + const wgpu::InstanceFeatureName timed=wgpu::InstanceFeatureName::TimedWaitAny; + id.requiredFeatureCount=1;id.requiredFeatures=&timed; + auto instance=wgpu::CreateInstance(&id); + wgpu::Adapter adapter; + wgpu::RequestAdapterOptions options{.backendType=wgpu::BackendType::D3D12}; + auto future=instance.RequestAdapter(&options,wgpu::CallbackMode::WaitAnyOnly, + [&](wgpu::RequestAdapterStatus status,wgpu::Adapter a,wgpu::StringView message) { + if(status==wgpu::RequestAdapterStatus::Success) adapter=std::move(a); + else std::cerr<1) eye.target.resolveView=output.CreateView(); + eye.target.depthView=depth.CreateView();eye.target.size={512,512,1};eye.target.msaaSamples=samples; + eye.projection.m0[0]=1;eye.projection.m1[1]=1; + eye.projection.m0[2]=eyeIndex?0.06f:-0.06f; + auto view=gfx::cockpit::identity();view[7]=0.20f; + std::memcpy(frame.cockpit.eyeFromSeat[eyeIndex],view.data(),sizeof(frame.cockpit.eyeFromSeat[eyeIndex])); + auto encoder=g_device.CreateCommandEncoder(); + const wgpu::RenderPassColorAttachment clear{.view=eye.target.colorView,.resolveTarget=eye.target.resolveView, + .loadOp=wgpu::LoadOp::Clear,.storeOp=wgpu::StoreOp::Store,.clearValue={0.06,0.09,0.13,1}}; + const wgpu::RenderPassDepthStencilAttachment sceneDepth{.view=eye.target.depthView, + .depthLoadOp=wgpu::LoadOp::Clear,.depthStoreOp=wgpu::StoreOp::Store,.depthClearValue=reversed?(occluded?0.8f:0.0f):(occluded?0.2f:1.0f)}; + const wgpu::RenderPassDescriptor pd{.colorAttachmentCount=1,.colorAttachments=&clear,.depthStencilAttachment=&sceneDepth}; + auto pass=encoder.BeginRenderPass(&pd); + if(coverage==2) { + wgpu::ShaderSourceWGSL code{}; + code.code=R"( + @vertex fn vs(@builtin(vertex_index) i:u32) -> @builtin(position) vec4f { + let p=array(vec2f(0,-1),vec2f(1,-1),vec2f(0,1),vec2f(0,1),vec2f(1,-1),vec2f(1,1)); + return vec4f(p[i],0.5,1); + } + @fragment fn fs() -> @location(0) vec4f { return vec4f(0.06,0.09,0.13,1); } + )"; + wgpu::ShaderModuleDescriptor md{};md.nextInChain=&code; + auto shader=g_device.CreateShaderModule(&md); + const wgpu::ColorTargetState colorState{.format=wgpu::TextureFormat::RGBA8Unorm}; + const wgpu::FragmentState fragment{.module=shader,.entryPoint="fs",.targetCount=1,.targets=&colorState}; + const wgpu::DepthStencilState ds{.format=wgpu::TextureFormat::Depth32Float,.depthWriteEnabled=true,.depthCompare=wgpu::CompareFunction::Always}; + wgpu::RenderPipelineDescriptor desc{};desc.vertex={.module=shader,.entryPoint="vs"}; + desc.fragment=&fragment;desc.depthStencil=&ds;desc.multisample.count=samples; + auto wall=g_device.CreateRenderPipeline(&desc);pass.SetPipeline(wall);pass.Draw(6); + } + if(coverage==2) gfx::cockpit::render(encoder,frame,eyeIndex,reversed?gfx::cockpit::SceneDepth{0,2,true}:gfx::cockpit::SceneDepth{-1,-2,true},&pass); + pass.End(); + if(coverage!=2) gfx::cockpit::render(encoder,frame,eyeIndex,reversed?gfx::cockpit::SceneDepth{0,2,true}:gfx::cockpit::SceneDepth{-1,-2,true}); + const wgpu::BufferDescriptor bd{.usage=wgpu::BufferUsage::CopyDst|wgpu::BufferUsage::MapRead,.size=512*512*4}; + auto readback=g_device.CreateBuffer(&bd); + const wgpu::TexelCopyTextureInfo src{.texture=output}; + const wgpu::TexelCopyBufferInfo dst{.layout={.bytesPerRow=2048,.rowsPerImage=512},.buffer=readback}; + const wgpu::Extent3D extent{512,512,1};encoder.CopyTextureToBuffer(&src,&dst,&extent); + auto commands=encoder.Finish();g_device.GetQueue().Submit(1,&commands); + bool mapped=false; + future=readback.MapAsync(wgpu::MapMode::Read,0,512*512*4,wgpu::CallbackMode::WaitAnyOnly, + [&](wgpu::MapAsyncStatus status,wgpu::StringView) { mapped=status==wgpu::MapAsyncStatus::Success; }); + if(instance.WaitAny(future,5000000000)!=wgpu::WaitStatus::Success||!mapped) return 1; + const auto* bytes=static_cast(readback.GetConstMappedRange()); + size_t bright=0; + for(size_t i=0;i<512*512;++i) if(bytes[4*i]>90&&bytes[4*i+1]>90&&bytes[4*i+2]>90) ++bright; + if(occluded ? bright!=0 : bright<1000) { std::cerr<<"Incorrect hands/wheel occlusion\n";++errors; } + if(coverage==2) { + size_t left=0,right=0; + for(size_t y=0;y<512;++y) for(size_t x=0;x<512;++x) { + const auto i=y*512+x; + if(bytes[4*i]>90&&bytes[4*i+1]>90&&bytes[4*i+2]>90) (x<256?left:right)++; + } + if(left<500||right>8) { std::cerr<<"Partial wall occlusion failed for eye "<(bytes+i*4),3); + } + readback.Unmap(); + std::cout<<(bike?"Bike ":"Kart ")<<(original?"native hands: ":"VR controls: ")<