diff --git a/runtime/include/vr/openxr_driving.h b/runtime/include/vr/openxr_driving.h index 8deb31e..c067558 100644 --- a/runtime/include/vr/openxr_driving.h +++ b/runtime/include/vr/openxr_driving.h @@ -50,6 +50,12 @@ struct DrivingSnapshot { // clockwise as the driver sees it, i.e. to the right. float visual_angle = 0.0f; std::array hands{}; + // What the cockpit overlay draws: a separate VR wheel or handlebar when the + // vehicle's own is not the one turning. `control` places the handlebar + // (and, when its geometry is valid, is what the hands reach for). + bool synthetic_control = false; + bool bike = false; + WheelGeometry control{}; }; // Pacing thread publishes; any thread reads the latest. A default snapshot diff --git a/runtime/include/vr/openxr_input.h b/runtime/include/vr/openxr_input.h index fe51886..97b70a3 100644 --- a/runtime/include/vr/openxr_input.h +++ b/runtime/include/vr/openxr_input.h @@ -4,9 +4,11 @@ #if defined(MKW_ENABLE_OPENXR) +#include "vr/openxr_driving.h" #include "vr/openxr_runtime.h" #include "vr/openxr_settings_panel.h" #include "vr/openxr_wii_remote.h" +#include "vr/steering_wheel.h" #include #include @@ -56,6 +58,12 @@ struct OpenXRPointerScreen { // Both are bound for the Oculus Touch profile; khr/simple_controller gets // select/menu and the poses so an unknown runtime still offers something. // +// In the first-person cockpit (openxr_driving.h) the grip poses are located in +// the seated frame every frame. With hand steering on, a squeezed grip near the +// steering wheel or handlebar takes hold of it; while held, the wheel replaces +// the left stick's X axis in both presentations and that grip no longer reaches +// the game (C on the Nunchuk, a shoulder on the gamepad). +// // Left Y (both thumbsticks clicked together as a gamepad) opens the in-headset // settings panel (openxr_settings_panel.h). While it is open, and until every // button has been released after it closes, the game sees idle controllers: the @@ -83,9 +91,13 @@ public: // xrSyncActions + state reads, then publishes to the virtual gamepad and // the Wii Remote bridge. predicted_display_time is the frame's XrTime; // screen is where the Wii Remote pointer can land this frame, and - // settings_panel where the settings panel is (its whole rectangle). + // settings_panel where the settings panel is (its whole rectangle). seat is + // the immersive seated frame, invalid outside an immersive race. void Sync(XrTime predicted_display_time, const OpenXRPointerScreen& screen, - const OpenXRPointerScreen& settings_panel); + const OpenXRPointerScreen& settings_panel, const driving::SeatFrame& seat); + + // The cockpit state the last Sync published (also OpenXRReadDriving()). + const DrivingSnapshot& Driving() const noexcept { return m_driving; } // Publishes a remote with nothing held, at rest and not pointing, and stops // the haptics, for frames without focused input. @@ -116,6 +128,11 @@ private: bool withheld); void PublishSettingsPanel(XrTime input_time, const OpenXRPointerScreen& panel, const settings_panel::Frame& frame); + // Hand steering: locates the grips in the seated frame, runs the wheel and + // hands its steering to `hands` before the game sees them. + void UpdateDriving(XrTime display_time, const driving::SeatFrame& seat, + std::array& hands, bool withheld); + void ResetDriving(); void UpdateRumble(); void StopRumble(); bool Check(XrResult result, const char* operation); @@ -149,6 +166,14 @@ private: bool m_haptics_active[kHands]{}; uint32_t m_joystick_id = 0; // SDL_JoystickID; 0 when detached void* m_joystick = nullptr; // SDL_Joystick* + SteeringWheel m_wheel; + WheelReferenceLatch m_wheel_reference; + driving::WheelVisual m_wheel_visual; + std::array m_wheel_held{}; + XrTime m_wheel_time = 0; + bool m_wheel_uses_geometry = false; + bool m_wheel_bike = false; + DrivingSnapshot m_driving{}; bool m_created = false; bool m_logged_sync_failure = false; bool m_logged_pointer = false; diff --git a/runtime/src/vr/openxr_input.cpp b/runtime/src/vr/openxr_input.cpp index 558aac1..6299a03 100644 --- a/runtime/src/vr/openxr_input.cpp +++ b/runtime/src/vr/openxr_input.cpp @@ -10,6 +10,8 @@ #endif #include "vr/openxr_input.h" +#include "runtime_config.h" +#include "vr/mkw_vr_first_person.h" #include "vr/openxr_diagnostics.h" #include @@ -451,6 +453,7 @@ void OpenXRInput::DetachVirtualGamepad() { void OpenXRInput::Destroy() { // The game must stop reading a remote whose controllers are going away. OpenXRWithdrawWiiRemote(); + ResetDriving(); if (m_created) { StopRumble(); } @@ -493,6 +496,7 @@ void OpenXRInput::Idle() { m_last_input_time = 0; m_panel_select_held = false; OpenXRPublishSettingsPanelPointer(false, 0.0f, 0.0f, false, 0.0f); + ResetDriving(); StopRumble(); // Nothing stays held on the gamepad either while input is away. if (m_joystick != nullptr) { @@ -507,7 +511,7 @@ void OpenXRInput::Idle() { } void OpenXRInput::Sync(XrTime predicted_display_time, const OpenXRPointerScreen& screen, - const OpenXRPointerScreen& settings_panel) { + const OpenXRPointerScreen& settings_panel, const driving::SeatFrame& seat) { if (!m_created || m_runtime == nullptr) { return; } @@ -618,6 +622,10 @@ void OpenXRInput::Sync(XrTime predicted_display_time, const OpenXRPointerScreen& OpenXRSetSettingsPanelOpen(open); } + // The cockpit's wheel before the game reads the controllers: a held wheel + // steers through the left stick and keeps its grips from the game. + UpdateDriving(predicted_display_time, seat, hands, panel.withheld); + // While the panel has the controllers, the game sees them idle. static const std::array kIdleHands{}; const auto& game_hands = panel.withheld ? kIdleHands : hands; @@ -768,6 +776,117 @@ void OpenXRInput::PublishWiiRemote(XrTime input_time, const OpenXRPointerScreen& OpenXRPublishWiiRemote(m_joystick_id, sample); } +void OpenXRInput::ResetDriving() { + m_wheel = {}; + WheelGeometry unused{}; + m_wheel_reference.Resolve(unused, false, false, false, false, 0, 0.0f); + m_wheel_visual.Reset(); + m_wheel_held = {}; + m_wheel_time = 0; + m_driving = {}; + OpenXRPublishDriving(m_driving); +} + +void OpenXRInput::UpdateDriving(XrTime display_time, const driving::SeatFrame& seat, + std::array& hands, bool withheld) { + const FirstPersonAnchor anchor = MkwVRFirstPersonGetAnchor(); + if (!seat.valid || !anchor.valid || !anchor.cockpit) { + if (m_driving.cockpit_active || m_wheel_time != 0) { + ResetDriving(); + } + return; + } + const WheelTuning tuning = RuntimeConfigFile::VrWheelTuning(); + const bool hand_steering = RuntimeConfigFile::VrHandSteering(); + const bool steering_wheel = RuntimeConfigFile::VrSteeringWheel(); + const bool native_steering_wheel = RuntimeConfigFile::VrNativeSteeringWheel(); + const float dt = m_wheel_time != 0 && display_time > m_wheel_time + ? static_cast(display_time - m_wheel_time) * 1.0e-9f + : 1.0f / 90.0f; + m_wheel_time = display_time; + + DrivingSnapshot snapshot{}; + snapshot.cockpit_active = true; + snapshot.hand_steering = hand_steering; + snapshot.bike = anchor.bike; + // The vehicle's own control is the one turning (or none is shown at all), + // so the overlay adds no separate wheel. + snapshot.synthetic_control = + steering_wheel && !(native_steering_wheel && anchor.native_mesh_prepared); + + // Which control the hands reach for: the vehicle's own wherever its + // geometry is known and no separate wheel is drawn, a handlebar always + // (held over a brief gap while gripped), otherwise the VR wheel in front + // of the seat. + WheelGeometry geometry = anchor.native_wheel; + const bool geometry_valid = m_wheel_reference.Resolve(geometry, true, geometry.valid, + m_wheel_held[0] || m_wheel_held[1], anchor.bike, + anchor.vehicle_identity, dt); + if (anchor.bike && !geometry_valid) { + geometry = {}; + geometry.center = {0.0f, SteeringWheel::Height, SteeringWheel::Depth}; + geometry.right = {1.0f, 0.0f, 0.0f}; + geometry.up = {0.0f, 0.0f, -1.0f}; + geometry.normal = {0.0f, 1.0f, 0.0f}; + geometry.radius = 0.25f; + geometry.valid = true; + } + const bool uses_geometry = anchor.bike || (geometry_valid && !snapshot.synthetic_control); + if (uses_geometry != m_wheel_uses_geometry || anchor.bike != m_wheel_bike) { + m_wheel = {}; + m_wheel_uses_geometry = uses_geometry; + m_wheel_bike = anchor.bike; + } + snapshot.control = geometry; + + constexpr XrSpaceLocationFlags kPoseValid = + XR_SPACE_LOCATION_POSITION_VALID_BIT | XR_SPACE_LOCATION_ORIENTATION_VALID_BIT; + std::array wheel_hands{}; + for (uint32_t hand = 0; hand < kHands; ++hand) { + bool tracked = false; + std::array seat_from_grip = snapshot.hands[hand].seat_from_grip; + if (m_grip_spaces[hand] != XR_NULL_HANDLE) { + XrSpaceLocation location{XR_TYPE_SPACE_LOCATION}; + if (XR_SUCCEEDED(xrLocateSpace(m_grip_spaces[hand], m_runtime->AppSpace(), display_time, &location)) && + (location.locationFlags & kPoseValid) == kPoseValid) { + tracked = true; + const auto& pose = location.pose; + seat_from_grip = driving::SeatFromApp(seat, {pose.position.x, pose.position.y, pose.position.z}, + {pose.orientation.x, pose.orientation.y, + pose.orientation.z, pose.orientation.w}); + } + } + const float squeeze = hands[hand].squeeze; + // Hands are shown only while they can steer. + snapshot.hands[hand] = {tracked && hand_steering, false, squeeze, seat_from_grip}; + wheel_hands[hand] = {seat_from_grip[3], seat_from_grip[7], seat_from_grip[11], squeeze, tracked}; + if (uses_geometry) { + wheel_hands[hand] = geometry.ToWheel(wheel_hands[hand]); + } + } + const bool active = hand_steering && !withheld; + const WheelState wheel = m_wheel.Update(wheel_hands, active, dt, + uses_geometry ? geometry.radius : SteeringWheel::Radius, + anchor.bike, tuning); + for (uint32_t hand = 0; hand < kHands; ++hand) { + if (wheel.held[hand] != m_wheel_held[hand] && active && tuning.haptics) { + constexpr XrDuration kGrabPulseNs = 25'000'000; + constexpr XrDuration kReleasePulseNs = 15'000'000; + ApplyHaptic(hand, wheel.held[hand] ? 0.25f : 0.12f, wheel.held[hand] ? kGrabPulseNs : kReleasePulseNs); + } + snapshot.hands[hand].held = wheel.held[hand]; + } + m_wheel_held = wheel.held; + snapshot.held = wheel.held; + driving::ApplyHandSteering(hands, wheel); + snapshot.steering_input = withheld ? 0.0f : hands[0].stick_x; + snapshot.visual_angle = + m_wheel_visual.Update(wheel.held[0] || wheel.held[1], wheel.visualAngle, snapshot.steering_input, + driving::MaxWheelAngle(anchor.bike, tuning), dt); + m_driving = snapshot; + OpenXRPublishDriving(snapshot); +} + void OpenXRInput::UpdateRumble() { if (!OpenXRWiiRemoteRumbleRequested() || !OpenXRWiiRemoteOwnsGamepad(m_joystick_id)) { StopRumble(); diff --git a/runtime/src/vr/openxr_integration.cpp b/runtime/src/vr/openxr_integration.cpp index 9d92b44..05784f8 100644 --- a/runtime/src/vr/openxr_integration.cpp +++ b/runtime/src/vr/openxr_integration.cpp @@ -31,6 +31,7 @@ #if defined(MKW_ENABLE_OPENXR) #include "vr/openxr_backend.h" +#include "vr/openxr_hand_mesh.h" #include "vr/openxr_input.h" #include "vr/openxr_runtime.h" #if defined(_WIN32) @@ -180,6 +181,16 @@ XrPosef ScreenPoseAhead(const OpenXRFrame& frame, float distance) noexcept { return pose; } +// Hand steering draws the player's hands, in the runtime's own hand mesh where +// it offers one (XR_FB_hand_tracking_mesh). Only asked for when hand steering +// is on at launch; turning it on later uses the procedural gloves. +void AddHandMeshExtensions(OpenXRConfig& config) { + if (RuntimeConfigFile::VrHandSteering()) { + config.optional_extensions.push_back("XR_EXT_hand_tracking"); + config.optional_extensions.push_back("XR_FB_hand_tracking_mesh"); + } +} + void IdentityEye(AuroraStereoEye& eye) noexcept { std::fill(std::begin(eye.projection), std::end(eye.projection), 0.0f); eye.projection[0] = 1.0f; @@ -375,6 +386,7 @@ public: config.required_extensions = {kRequiredAuroraBackend == BACKEND_VULKAN ? "XR_KHR_vulkan_enable2" : "XR_KHR_D3D12_enable"}; config.optional_extensions = {"XR_KHR_win32_convert_performance_counter_time", "XR_FB_display_refresh_rate", "XR_EXT_performance_settings"}; + AddHandMeshExtensions(config); #else // Either Vulkan binding extension is acceptable; the backend picks // whichever the runtime enabled, preferring enable2. @@ -383,6 +395,7 @@ public: "XR_KHR_convert_timespec_time", "XR_KHR_android_thread_settings", "XR_FB_display_refresh_rate", "XR_EXT_performance_settings"}; + AddHandMeshExtensions(config); config.instance_create_next = OpenXRAndroidInstanceCreateNext(); #endif if (!runtime_->Initialize(config)) { @@ -880,7 +893,7 @@ private: // After the screen is placed, so the pointer aims at this // frame's screen rather than the previous one's. input_->Sync(frame.xr_frame.predicted_display_time, PointerScreen(frame, policy, immersive), - SettingsPanelScreen(frame, policy, immersive)); + SettingsPanelScreen(frame, policy, immersive), InputSeatFrame(immersive)); } if (!frame.expects_gpu_submission) { @@ -1061,7 +1074,7 @@ private: if (input_ != nullptr) { const diagnostics::ScopedStage input_timer(diagnostics::Stage::InputSync); input_->Sync(packet.xr_frame.predicted_display_time, PointerScreen(packet, policy, immersive), - SettingsPanelScreen(packet, policy, immersive)); + SettingsPanelScreen(packet, policy, immersive), InputSeatFrame(immersive)); } if (!packet.expects_gpu_submission) { // Nothing to render (no rendering requested or no tracking): keep the compositor fed. @@ -1228,6 +1241,61 @@ private: position_valid && base_position_valid_, units_per_meter, lean_back_radians, destination.eyes[eye].viewFromCenter); } + BuildCockpit(source, position_valid, units_per_meter, lean_back_radians, destination.cockpit); + } + + // The first-person cockpit's hands and separate wheel, in the seated frame + // the eye transforms place at base + lean * seat (metres). Always carries + // the packet's world scale, which Aurora rescales to the sealed frame's. + void BuildCockpit(const OpenXRBackendFrame& source, bool position_valid, float units_per_meter, + float lean_back_radians, AuroraCockpit& cockpit) noexcept { + cockpit.unitsPerMeter = units_per_meter; + const DrivingSnapshot driving = input_ != nullptr ? input_->Driving() : DrivingSnapshot{}; + if (driving.hand_steering && !hand_meshes_loaded_ && runtime_ != nullptr) { + hand_meshes_loaded_ = true; + const bool loaded = LoadRuntimeHandMeshes(*runtime_); + RT_LOG(RT_TAG_RUNTIME) << "[mkw-vr] cockpit hands: " + << (loaded ? "the runtime's hand mesh" : "procedural gloves (no runtime hand mesh)") + << std::endl; + } + cockpit.active = driving.cockpit_active && position_valid && base_position_valid_ && + (driving.synthetic_control || driving.hand_steering); + if (!cockpit.active) { + return; + } + cockpit.wheelAngle = driving.visual_angle; + cockpit.nativeWheel = !driving.synthetic_control; + cockpit.bike = driving.bike; + cockpit.handlebarRadius = driving.control.radius; + for (int row = 0; row < 3; ++row) { + cockpit.seatFromHandlebar[row * 4 + 0] = driving.control.right[row]; + cockpit.seatFromHandlebar[row * 4 + 1] = driving.control.up[row]; + cockpit.seatFromHandlebar[row * 4 + 2] = driving.control.normal[row]; + cockpit.seatFromHandlebar[row * 4 + 3] = driving.control.center[row]; + } + for (uint32_t eye = 0; eye < kOpenXREyeCount; ++eye) { + ViewFromBase(source.xr_frame.views[eye].pose, base_position_, true, 1.0f, lean_back_radians, + cockpit.eyeFromSeat[eye]); + } + for (size_t hand = 0; hand < 2; ++hand) { + auto& target = cockpit.hands[hand]; + const auto& from = driving.hands[hand]; + target.tracked = from.tracked; + target.held = from.held; + target.squeeze = from.squeeze; + std::copy(from.seat_from_grip.begin(), from.seat_from_grip.end(), target.seatFromGrip); + } + } + + // The seated frame the controllers are located in for hand steering: the + // immersive base the eye transforms use, from the previous frame (this + // frame's is latched after input). + driving::SeatFrame InputSeatFrame(bool immersive) const noexcept { + driving::SeatFrame seat; + seat.valid = immersive && base_position_valid_ && last_immersive_; + seat.base = base_position_; + seat.lean_back_radians = lean_back_degrees_.load(std::memory_order_relaxed) * kDegreesToRadians; + return seat; } // Runs once per located frame, before the virtual screen is placed and @@ -1569,6 +1637,7 @@ private: XrPosef virtual_screen_pose_{{0.0f, 0.0f, 0.0f, 1.0f}, {0.0f, 0.0f, 0.0f}}; bool virtual_screen_pose_valid_ = false; bool last_immersive_ = false; + bool hand_meshes_loaded_ = false; uint64_t applied_session_run_serial_ = 0; bool session_was_active_ = false; bool requested_ = false;