mirror of
https://github.com/mitch030504/Wiicompiled_VR_Frame.git
synced 2026-10-06 05:00:27 +02:00
Pose the cockpit hands from the headset's hand tracking
- New [vr] hand_tracking (default off, Quest only for now): the Quest launcher's Settings > VR and the headset panel's VR tab, under hand steering. Two XR_EXT_hand_tracking trackers are located every XR frame: with the controllers held the Quest builds the joints from their touch sensors (XR_EXT_hand_tracking_data_source's controller source), once they are put down from its cameras. The trackers exist only while the option and hand steering are on and also serve the runtime hand mesh; the extensions (plus XR_FB_hand_tracking_aim) are asked for when either is on at launch. - Aurora skins the runtime mesh with the joints themselves (tracked pose times inverse bind pose, no curl, no grip); runtimes with joints but no mesh get a skeleton; non-finite joints put only that hand back on its grip curl. AuroraCockpitHand carries the 26 seated-frame joints and radii. - The manifest declares horizonos.permission.HAND_TRACKING (and the deprecated com.oculus.permission.HAND_TRACKING), both normal permissions with no prompt on a Quest 3, and oculus.software.handtracking as optional; without it Horizon OS keeps the app controllers-only. Bare hands then drive khr/simple_controller, so on Android a hand whose squeeze action is inactive and select active is treated as bare: with the option off it presses nothing but the menu gesture, is not drawn and feeds no Wii Remote motion. - Interaction-profile changes and tracker sources are logged. The pure rules (grasp from finger flexion, bare latch, pinch gate, bare-hand buttons, flick) live in the OpenXR-free vr/openxr_hand_tracking.h with mkw_vr_hand_tracking_tests; the driving and flick parts are wired in the next commits. Docs: OPENXR.md, docs/quest-port.md, README. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
iChris4andClaude Opus 5.5 committed
1 parent
c990c595f0
commit
a21e22b605
24 files changed
+1549
-49
No files matched your search
@@ -415,8 +415,10 @@ add_test(NAME mkw_vr_first_person_tests COMMAND mkw_vr_first_person_tests)
|
||||
|
||||
# The first-person cockpit (seated eye, wheel and handlebar geometry, level seat,
|
||||
# native wheel vertices) and hand steering (grab, turn, hand-off to the game),
|
||||
# ported from heurazy's mario-kart-wii-VR-port. All header-only.
|
||||
foreach(test_name mkw_steering_wheel_tests mkw_vr_cockpit_tests mkw_vr_hand_steering_tests mkw_vr_camera_toggle_tests)
|
||||
# ported from heurazy's mario-kart-wii-VR-port, and tracked hands (grasp, bare-hand
|
||||
# buttons, flick). All header-only.
|
||||
foreach(test_name mkw_steering_wheel_tests mkw_vr_cockpit_tests mkw_vr_hand_steering_tests mkw_vr_camera_toggle_tests
|
||||
mkw_vr_hand_tracking_tests)
|
||||
string(REGEX REPLACE "^mkw_" "" test_source "${test_name}")
|
||||
add_executable(${test_name} "${CMAKE_CURRENT_LIST_DIR}/tests/${test_source}.cpp")
|
||||
target_include_directories(${test_name} PRIVATE "${CMAKE_CURRENT_LIST_DIR}/include")
|
||||
|
||||
@@ -80,6 +80,7 @@ struct RuntimeUserConfig {
|
||||
std::optional<bool> vrSteeringWheel;
|
||||
std::optional<bool> vrNativeSteeringWheel;
|
||||
std::optional<bool> vrHandSteering;
|
||||
std::optional<bool> vrHandTracking;
|
||||
std::optional<float> vrWheelKartDegrees;
|
||||
std::optional<float> vrWheelBikeDegrees;
|
||||
std::optional<float> vrWheelGrabDistance;
|
||||
@@ -233,6 +234,11 @@ inline constexpr float kVrCockpitUnitsPerMeterMax = 400.0f;
|
||||
inline constexpr bool kVrSteeringWheelDefault = true;
|
||||
inline constexpr bool kVrNativeSteeringWheelDefault = true;
|
||||
inline constexpr bool kVrHandSteeringDefault = true;
|
||||
// The cockpit hands follow the headset's hand tracking (the controllers' touch
|
||||
// sensors while they are held, the cameras once they are put down, when bare
|
||||
// hands also drive). Opt-in, and only offered on the Quest for now; the
|
||||
// launcher's Settings page shows the same default.
|
||||
inline constexpr bool kVrHandTrackingDefault = false;
|
||||
// Hand steering tuning ranges; the defaults are mkw::vr::WheelTuning's.
|
||||
inline constexpr float kVrWheelDegreesMin = 20.0f, kVrWheelDegreesMax = 180.0f;
|
||||
inline constexpr float kVrWheelGrabDistanceMin = 0.15f, kVrWheelGrabDistanceMax = 0.8f;
|
||||
@@ -862,6 +868,7 @@ inline RuntimeUserConfig ParseConfigDocument(const toml::value& document) {
|
||||
config.vrSteeringWheel = FindConfigValue<bool>(document, "vr", "steering_wheel");
|
||||
config.vrNativeSteeringWheel = FindConfigValue<bool>(document, "vr", "native_steering_wheel");
|
||||
config.vrHandSteering = FindConfigValue<bool>(document, "vr", "hand_steering");
|
||||
config.vrHandTracking = FindConfigValue<bool>(document, "vr", "hand_tracking");
|
||||
config.vrWheelKartDegrees = readRangedFloat("wheel_kart_degrees", kVrWheelDegreesMin, kVrWheelDegreesMax);
|
||||
config.vrWheelBikeDegrees = readRangedFloat("wheel_bike_degrees", kVrWheelDegreesMin, kVrWheelDegreesMax);
|
||||
config.vrWheelGrabDistance =
|
||||
@@ -1263,6 +1270,11 @@ inline bool SetVrHandSteering(bool value) {
|
||||
return WriteSetting("vr", "hand_steering", value ? "true" : "false");
|
||||
}
|
||||
|
||||
inline bool SetVrHandTracking(bool value) {
|
||||
Mutable().vrHandTracking = value;
|
||||
return WriteSetting("vr", "hand_tracking", value ? "true" : "false");
|
||||
}
|
||||
|
||||
inline bool SetVrWheelTuning(const mkw::vr::WheelTuning& tuning) {
|
||||
auto& config = Mutable();
|
||||
const auto write = [](const char* key, std::optional<float>& slot, float value, float low, float high) {
|
||||
@@ -1746,6 +1758,10 @@ inline bool VrHandSteering(bool fallback = kVrHandSteeringDefault) {
|
||||
return Get().vrHandSteering.value_or(fallback);
|
||||
}
|
||||
|
||||
inline bool VrHandTracking(bool fallback = kVrHandTrackingDefault) {
|
||||
return Get().vrHandTracking.value_or(fallback);
|
||||
}
|
||||
|
||||
inline mkw::vr::WheelTuning VrWheelTuning() {
|
||||
const auto& config = Get();
|
||||
mkw::vr::WheelTuning tuning{};
|
||||
|
||||
@@ -17,6 +17,7 @@
|
||||
// forward. It is the frame the first-person anchor places the vehicle in, so
|
||||
// hands, wheel geometry and eye transforms all meet there.
|
||||
|
||||
#include "vr/openxr_hand_tracking.h"
|
||||
#include "vr/openxr_wii_remote.h"
|
||||
#include "vr/steering_wheel.h"
|
||||
|
||||
@@ -35,6 +36,17 @@ struct DrivingHand {
|
||||
float squeeze = 0.0f;
|
||||
// Row-major 3x4 from the controller's grip space into the seated frame.
|
||||
std::array<float, 12> seat_from_grip{1.0f, 0.0f, 0.0f, 0.0f, 0.0f, 1.0f, 0.0f, 0.0f, 0.0f, 0.0f, 1.0f, 0.0f};
|
||||
// Tracked hands ([vr] hand_tracking, openxr_hand_tracking.h). The joints
|
||||
// themselves stay with OpenXRInput::HandJoints(), on the pacing thread;
|
||||
// this says whether the cockpit draws the hand from them, and carries what
|
||||
// the settings panel reads out: where they came from, whether the hand is
|
||||
// bare (camera-tracked, no controller), how closed it is and whether it
|
||||
// pinches.
|
||||
bool joints_valid = false;
|
||||
bool bare = false;
|
||||
hand_tracking::Source source = hand_tracking::Source::None;
|
||||
float grasp = 0.0f;
|
||||
bool pinch = false;
|
||||
};
|
||||
|
||||
struct DrivingSnapshot {
|
||||
|
||||
@@ -13,7 +13,9 @@ namespace mkw::vr {
|
||||
// AnimalCrossing-VR-MR-Standalone obtains its white hands from this Meta
|
||||
// runtime extension, not from a distributable model asset. Use the same API,
|
||||
// with a procedural fallback on PC runtimes that do not expose Meta meshes.
|
||||
inline bool LoadRuntimeHandMeshes(OpenXRRuntime& runtime) {
|
||||
// `existing` are the tracked hands' trackers (OpenXRInput::HandTracker), used
|
||||
// where there is one; otherwise a tracker is made just for the mesh.
|
||||
inline bool LoadRuntimeHandMeshes(OpenXRRuntime& runtime, const XrHandTrackerEXT* existing = nullptr) {
|
||||
for (uint32_t h = 0; h < 2; ++h)
|
||||
aurora_set_vr_hand_mesh(h, nullptr, 0, nullptr, 0, nullptr, nullptr, 0);
|
||||
const auto& extensions = runtime.EnabledExtensions();
|
||||
@@ -28,12 +30,18 @@ inline bool LoadRuntimeHandMeshes(OpenXRRuntime& runtime) {
|
||||
if (!create || !destroy || !meshFn) return false;
|
||||
bool any = false;
|
||||
for (uint32_t h = 0; h < 2; ++h) {
|
||||
XrHandTrackerCreateInfoEXT info{XR_TYPE_HAND_TRACKER_CREATE_INFO_EXT};
|
||||
info.hand = h ? XR_HAND_RIGHT_EXT : XR_HAND_LEFT_EXT;
|
||||
info.handJointSet = XR_HAND_JOINT_SET_DEFAULT_EXT;
|
||||
XrHandTrackerEXT tracker = XR_NULL_HANDLE;
|
||||
if (XR_FAILED(create(runtime.Session(), &info, &tracker))) continue;
|
||||
struct Guard { XrHandTrackerEXT tracker; PFN_xrDestroyHandTrackerEXT destroy; ~Guard() { destroy(tracker); } } guard{tracker, destroy};
|
||||
XrHandTrackerEXT tracker = existing != nullptr ? existing[h] : XR_NULL_HANDLE;
|
||||
const bool owned = tracker == XR_NULL_HANDLE;
|
||||
if (owned) {
|
||||
XrHandTrackerCreateInfoEXT info{XR_TYPE_HAND_TRACKER_CREATE_INFO_EXT};
|
||||
info.hand = h ? XR_HAND_RIGHT_EXT : XR_HAND_LEFT_EXT;
|
||||
info.handJointSet = XR_HAND_JOINT_SET_DEFAULT_EXT;
|
||||
if (XR_FAILED(create(runtime.Session(), &info, &tracker))) continue;
|
||||
}
|
||||
struct Guard {
|
||||
XrHandTrackerEXT tracker; PFN_xrDestroyHandTrackerEXT destroy; bool owned;
|
||||
~Guard() { if (owned) destroy(tracker); }
|
||||
} guard{tracker, destroy, owned};
|
||||
XrHandTrackingMeshFB mesh{XR_TYPE_HAND_TRACKING_MESH_FB};
|
||||
if (XR_FAILED(meshFn(tracker, &mesh)) || mesh.jointCountOutput != 26 ||
|
||||
!mesh.vertexCountOutput || mesh.vertexCountOutput > 65535 || !mesh.indexCountOutput ||
|
||||
|
||||
@@ -0,0 +1,426 @@
|
||||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
#pragma once
|
||||
|
||||
// The cockpit hands from the headset's hand tracking ([vr] hand_tracking), and
|
||||
// what bare hands do once the controllers are put down.
|
||||
//
|
||||
// With the option on, the OpenXR pacing thread locates both hands' 26 joints
|
||||
// (XR_EXT_hand_tracking) every XR frame. Where a hand holds a controller, the
|
||||
// Quest builds them from the controller's touch sensors
|
||||
// (XR_EXT_hand_tracking_data_source's controller source); once the controllers
|
||||
// are put down, from its cameras, and the runtime then drives
|
||||
// /interaction_profiles/khr/simple_controller from the hand: select is an index
|
||||
// pinch, the left menu the palm-up menu gesture. The rules that turn all that
|
||||
// into the game's input live here, free of OpenXR, so they are tested headlessly
|
||||
// (tests/vr_hand_tracking_tests.cpp):
|
||||
//
|
||||
// - A hand is hand-driven when its squeeze action is inactive and its select
|
||||
// action active: the Touch profile binds squeeze, simple_controller does not.
|
||||
// That decides what its buttons mean, and needs no tracker, so the manifest's
|
||||
// hand-tracking permission can never let resting hands press anything.
|
||||
// - It is bare while it is hand-driven with camera-tracked joints, latched
|
||||
// through the wheel's tracking grace. That decides the wheel grab (the palm's
|
||||
// position, a grasp from the fingers' flexion) and the flick.
|
||||
//
|
||||
// Only the Android build applies these: a PC runtime can drive real controllers
|
||||
// through simple_controller and synthesize joints for them.
|
||||
|
||||
#include "vr/openxr_wii_remote.h"
|
||||
|
||||
#include <algorithm>
|
||||
#include <array>
|
||||
#include <cmath>
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
#include <cstring>
|
||||
|
||||
namespace mkw::vr::hand_tracking {
|
||||
|
||||
// XR_HAND_JOINT_*_EXT order.
|
||||
inline constexpr size_t kJointCount = 26;
|
||||
inline constexpr size_t kPalm = 0;
|
||||
inline constexpr size_t kWrist = 1;
|
||||
// Each finger's metacarpal joint, thumb first. A finger's chain runs from there
|
||||
// to its tip; the thumb's has no intermediate joint, so it is one shorter.
|
||||
inline constexpr std::array<size_t, 5> kMetacarpal{2, 6, 11, 16, 21};
|
||||
inline constexpr std::array<size_t, 5> kTip{5, 10, 15, 20, 25};
|
||||
|
||||
using Vec3 = std::array<float, 3>;
|
||||
using JointPositions = std::array<Vec3, kJointCount>;
|
||||
|
||||
// Where a hand's joints came from this frame.
|
||||
enum class Source : uint8_t {
|
||||
None, // not located
|
||||
Controller, // built from a held controller's touch sensors
|
||||
Camera, // the headset's cameras
|
||||
Unknown, // located, but the runtime does not say how
|
||||
};
|
||||
|
||||
inline const char* SourceLabel(Source source) noexcept {
|
||||
switch (source) {
|
||||
case Source::Controller: return "controller";
|
||||
case Source::Camera: return "camera";
|
||||
case Source::Unknown: return "tracked";
|
||||
default: return "none";
|
||||
}
|
||||
}
|
||||
|
||||
// Both hands' joints in the seated frame (openxr_driving.h), for the cockpit
|
||||
// overlay: row-major 3x4 per joint, and each joint's radius in metres.
|
||||
struct HandJointFrame {
|
||||
std::array<bool, 2> valid{};
|
||||
std::array<std::array<std::array<float, 12>, kJointCount>, 2> seat_from_joint{};
|
||||
std::array<std::array<float, kJointCount>, 2> radius{};
|
||||
};
|
||||
|
||||
inline bool IsFinite(float value) noexcept {
|
||||
// Bit test: the runtime may be built with -ffast-math.
|
||||
uint32_t bits = 0;
|
||||
std::memcpy(&bits, &value, sizeof(bits));
|
||||
return (bits & 0x7F800000u) != 0x7F800000u;
|
||||
}
|
||||
|
||||
// The fingers' summed flexion that reads as an open hand, and as a hand closed
|
||||
// on the wheel's rim. A relaxed hand bends its three finger joints by roughly
|
||||
// 1.0 to 1.6 radians in all, which must stay under the wheel's 0.15 release;
|
||||
// a grip round a rim bends them well past the 0.55 press. Starting values for
|
||||
// tuning with the headset panel's readout.
|
||||
inline constexpr float kGraspOpenRadians = 1.2f;
|
||||
inline constexpr float kGraspClosedRadians = 3.0f;
|
||||
|
||||
// The angle between two successive bones a->b and b->c, 0 for a straight
|
||||
// finger. Negative when a bone is too short to have a direction.
|
||||
inline float BendRadians(const Vec3& a, const Vec3& b, const Vec3& c) noexcept {
|
||||
const Vec3 u{b[0] - a[0], b[1] - a[1], b[2] - a[2]};
|
||||
const Vec3 v{c[0] - b[0], c[1] - b[1], c[2] - b[2]};
|
||||
const float uu = u[0] * u[0] + u[1] * u[1] + u[2] * u[2];
|
||||
const float vv = v[0] * v[0] + v[1] * v[1] + v[2] * v[2];
|
||||
if (!(uu > 1.0e-8f) || !(vv > 1.0e-8f)) {
|
||||
return -1.0f;
|
||||
}
|
||||
const Vec3 cross{u[1] * v[2] - u[2] * v[1], u[2] * v[0] - u[0] * v[2], u[0] * v[1] - u[1] * v[0]};
|
||||
const float sine = std::sqrt(cross[0] * cross[0] + cross[1] * cross[1] + cross[2] * cross[2]);
|
||||
const float cosine = u[0] * v[0] + u[1] * v[1] + u[2] * v[2];
|
||||
return std::atan2(sine, cosine);
|
||||
}
|
||||
|
||||
// How far a hand is closed, 0 (open) to 1 (closed on a rim or in a fist): the
|
||||
// middle, ring and little fingers' flexion, each summed over their three
|
||||
// joints. The index finger and thumb are left out so pinching does not grab.
|
||||
// Angles only, so it is the same for either hand, at any size and in any
|
||||
// orientation. 0 for joints that are not finite or collapse onto each other.
|
||||
inline float GraspFromJoints(const JointPositions& joints) noexcept {
|
||||
for (const Vec3& joint : joints) {
|
||||
if (!IsFinite(joint[0]) || !IsFinite(joint[1]) || !IsFinite(joint[2])) {
|
||||
return 0.0f;
|
||||
}
|
||||
}
|
||||
float sum = 0.0f;
|
||||
for (size_t finger = 2; finger < 5; ++finger) {
|
||||
const size_t base = kMetacarpal[finger];
|
||||
float flex = 0.0f;
|
||||
for (size_t joint = base; joint + 2 <= base + 4; ++joint) {
|
||||
const float bend = BendRadians(joints[joint], joints[joint + 1], joints[joint + 2]);
|
||||
if (!(bend >= 0.0f)) {
|
||||
return 0.0f;
|
||||
}
|
||||
flex += bend;
|
||||
}
|
||||
sum += std::clamp((flex - kGraspOpenRadians) / (kGraspClosedRadians - kGraspOpenRadians), 0.0f, 1.0f);
|
||||
}
|
||||
return sum / 3.0f;
|
||||
}
|
||||
|
||||
// Keeps a camera-tracked hand bare, with its last grasp, through a short loss
|
||||
// of tracking (fingers hidden behind the other hand, a hand turned edge-on):
|
||||
// long enough for the wheel's own grace to keep hold. A controller's squeeze
|
||||
// clears it at once, since a hand that picked one up is no longer bare.
|
||||
class BareLatch {
|
||||
public:
|
||||
// `hand_driven`: the hand drives simple_controller this frame.
|
||||
// `camera_joints`: its joints were located this frame, not from a controller.
|
||||
// `grace`: the wheel's tracking grace in seconds.
|
||||
bool Update(bool hand_driven, bool squeeze_active, bool camera_joints, float grasp, float dt,
|
||||
float grace) noexcept {
|
||||
if (squeeze_active) {
|
||||
Reset();
|
||||
return false;
|
||||
}
|
||||
if (hand_driven && camera_joints) {
|
||||
m_bare = true;
|
||||
m_tracked = true;
|
||||
m_lost = 0.0f;
|
||||
m_grasp = IsFinite(grasp) ? std::clamp(grasp, 0.0f, 1.0f) : 0.0f;
|
||||
return true;
|
||||
}
|
||||
if (m_bare) {
|
||||
m_tracked = false;
|
||||
m_lost += IsFinite(dt) ? std::clamp(dt, 0.0f, 0.1f) : 0.0f;
|
||||
if (m_lost <= grace) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
Reset();
|
||||
return false;
|
||||
}
|
||||
|
||||
bool Bare() const noexcept { return m_bare; }
|
||||
// False while the latch is only bridging a loss of tracking.
|
||||
bool Tracked() const noexcept { return m_bare && m_tracked; }
|
||||
float Grasp() const noexcept { return m_bare ? m_grasp : 0.0f; }
|
||||
void Reset() noexcept { *this = BareLatch{}; }
|
||||
|
||||
private:
|
||||
bool m_bare = false;
|
||||
bool m_tracked = false;
|
||||
float m_lost = 0.0f;
|
||||
float m_grasp = 0.0f;
|
||||
};
|
||||
|
||||
// A pinch uses an item only when it starts on a hand that has been off the
|
||||
// wheel for a moment: a hand opening off the rim often reads as a pinch for a
|
||||
// frame or two. Holding the pinch holds the button, which trails an item.
|
||||
class PinchGate {
|
||||
public:
|
||||
static constexpr float kFreeSeconds = 0.15f;
|
||||
|
||||
bool Update(bool pinch, bool held, float dt) noexcept {
|
||||
dt = IsFinite(dt) ? std::clamp(dt, 0.0f, 0.1f) : 0.0f;
|
||||
if (held) {
|
||||
m_free = 0.0f;
|
||||
m_firing = false;
|
||||
m_blocked = pinch;
|
||||
return false;
|
||||
}
|
||||
if (!pinch) {
|
||||
m_firing = false;
|
||||
m_blocked = false;
|
||||
m_free = std::min(m_free + dt, 1.0f);
|
||||
return false;
|
||||
}
|
||||
if (!m_firing && !m_blocked) {
|
||||
(m_free >= kFreeSeconds ? m_firing : m_blocked) = true;
|
||||
}
|
||||
m_free = std::min(m_free + dt, 1.0f);
|
||||
return m_firing;
|
||||
}
|
||||
|
||||
void Reset() noexcept { *this = PinchGate{}; }
|
||||
|
||||
private:
|
||||
float m_free = 0.0f;
|
||||
bool m_firing = false;
|
||||
bool m_blocked = false;
|
||||
};
|
||||
|
||||
// A hand's pinch and menu gesture this frame. The runtime's own recognition
|
||||
// (XR_FB_hand_tracking_aim) is preferred when it is valid, and a pinch made
|
||||
// while the hand is in the system gesture (palm towards the face) is the menu
|
||||
// gesture, not a pinch. Otherwise the select and menu actions, as
|
||||
// simple_controller delivers them.
|
||||
struct Gestures {
|
||||
bool pinch = false;
|
||||
bool menu = false;
|
||||
};
|
||||
|
||||
inline Gestures GesturesOf(bool aim_valid, bool aim_pinching, bool aim_menu, bool aim_system_gesture,
|
||||
bool action_select, bool action_menu) noexcept {
|
||||
if (aim_valid) {
|
||||
return {aim_pinching && !aim_system_gesture, aim_menu || action_menu};
|
||||
}
|
||||
return {action_select, action_menu};
|
||||
}
|
||||
|
||||
// The select action as simple_controller delivers it: right select is bound to
|
||||
// the primary action, left select to the secondary one (openxr_input.cpp).
|
||||
inline bool SelectOf(const wii_remote::HandInputs& inputs, size_t hand) noexcept {
|
||||
return hand == 1 ? inputs.primary : inputs.secondary;
|
||||
}
|
||||
|
||||
// Before anything reads the hands (the settings panel, then the game). A
|
||||
// hand-driven hand's select is replaced: with tracked hands on, a right pinch
|
||||
// stays A, for the menus, and a left one no longer toggles the settings panel;
|
||||
// with them off, the hand presses nothing but the menu gesture, so resting hands
|
||||
// with the controllers put down never press anything. Controller hands are left
|
||||
// alone.
|
||||
inline void ApplyHandDrivenButtons(std::array<wii_remote::HandInputs, 2>& hands,
|
||||
const std::array<bool, 2>& hand_driven, const std::array<bool, 2>& pinch,
|
||||
bool option_on) noexcept {
|
||||
for (size_t hand = 0; hand < hands.size(); ++hand) {
|
||||
if (!hand_driven[hand]) {
|
||||
continue;
|
||||
}
|
||||
hands[hand].primary = option_on && hand == 1 && pinch[hand];
|
||||
hands[hand].secondary = false;
|
||||
}
|
||||
}
|
||||
|
||||
// In a cockpit race, after the wheel: a bare hand on the wheel holds the gas
|
||||
// (A: the right primary button), and an item pinch uses an item (the left
|
||||
// trigger: the Wii Remote's Z, the GameCube's L). A hand-driven right hand's
|
||||
// pinch stops meaning A there. The grasp itself never becomes a squeeze, which
|
||||
// would press the gamepad's shoulders (GameCube R drifts).
|
||||
inline void ApplyBareHandRace(std::array<wii_remote::HandInputs, 2>& hands,
|
||||
const std::array<bool, 2>& hand_driven, const std::array<bool, 2>& bare_held,
|
||||
const std::array<bool, 2>& item_pinch) noexcept {
|
||||
if (hand_driven[1]) {
|
||||
hands[1].primary = false;
|
||||
}
|
||||
if (bare_held[0] || bare_held[1]) {
|
||||
hands[1].primary = true;
|
||||
}
|
||||
if (item_pinch[0] || item_pinch[1]) {
|
||||
hands[0].trigger = 1.0f;
|
||||
}
|
||||
}
|
||||
|
||||
// One bare hand for the flick detector: its palm's height in the seated frame.
|
||||
struct FlickHand {
|
||||
bool tracked = false;
|
||||
bool held = false;
|
||||
float height = 0.0f;
|
||||
};
|
||||
|
||||
// A quick upward flick of the hands, the bare-hand shake that does a trick off
|
||||
// a ramp or pulls a wheelie. Both hands on the wheel must rise together at a
|
||||
// similar speed, so a turn, where one hand rises as the other drops, never
|
||||
// counts; a free hand counts alone, but a lone hand on the wheel does not
|
||||
// (that is a turn too). A rise must last a few samples and cover some height
|
||||
// soon enough, so tracking noise and a pose jumping when tracking comes back
|
||||
// never count; the second is also caught as an impossible speed.
|
||||
class FlickDetector {
|
||||
public:
|
||||
static constexpr float kRisingSpeed = 0.3f; // m/s: the hand is going up
|
||||
static constexpr float kJumpSpeed = 5.0f; // m/s: faster than a hand, a tracking jump
|
||||
static constexpr float kMinRise = 0.05f; // m
|
||||
static constexpr float kWindowSeconds = 0.15f; // to cover kMinRise
|
||||
static constexpr int kMinSamples = 3;
|
||||
static constexpr float kPairSpeed = 1.2f; // mean of both hands, m/s
|
||||
static constexpr float kPairEachSpeed = 0.6f;
|
||||
static constexpr float kPairSpeedDifference = 0.6f;
|
||||
static constexpr float kFreeSpeed = 1.5f;
|
||||
static constexpr float kCooldownSeconds = 0.5f;
|
||||
|
||||
bool Update(const std::array<FlickHand, 2>& hands, float dt) noexcept {
|
||||
if (!IsFinite(dt) || dt <= 0.0f) {
|
||||
return false;
|
||||
}
|
||||
dt = std::min(dt, 0.1f);
|
||||
m_cooldown = std::max(m_cooldown - dt, 0.0f);
|
||||
std::array<float, 2> speed{};
|
||||
std::array<bool, 2> rising{};
|
||||
for (size_t hand = 0; hand < 2; ++hand) {
|
||||
rising[hand] = Step(m_tracks[hand], hands[hand], dt, speed[hand]);
|
||||
}
|
||||
if (m_cooldown > 0.0f) {
|
||||
return false;
|
||||
}
|
||||
bool fire = false;
|
||||
if (hands[0].held && hands[1].held) {
|
||||
fire = rising[0] && rising[1] && speed[0] >= kPairEachSpeed && speed[1] >= kPairEachSpeed &&
|
||||
0.5f * (speed[0] + speed[1]) >= kPairSpeed &&
|
||||
std::fabs(speed[0] - speed[1]) <= kPairSpeedDifference;
|
||||
}
|
||||
for (size_t hand = 0; hand < 2 && !fire; ++hand) {
|
||||
fire = hands[hand].tracked && !hands[hand].held && rising[hand] && speed[hand] >= kFreeSpeed;
|
||||
}
|
||||
if (fire) {
|
||||
m_cooldown = kCooldownSeconds;
|
||||
for (Track& track : m_tracks) {
|
||||
track.spent = true;
|
||||
}
|
||||
}
|
||||
return fire;
|
||||
}
|
||||
|
||||
void Reset() noexcept { *this = FlickDetector{}; }
|
||||
|
||||
private:
|
||||
struct Track {
|
||||
bool has_last = false;
|
||||
float last = 0.0f;
|
||||
bool rising = false;
|
||||
bool spent = false; // this rise already flicked, or took too long
|
||||
float start = 0.0f;
|
||||
float time = 0.0f;
|
||||
int samples = 0;
|
||||
};
|
||||
|
||||
// Whether the hand is in a rise that qualifies, and that rise's mean speed.
|
||||
static bool Step(Track& track, const FlickHand& hand, float dt, float& speed) noexcept {
|
||||
if (!hand.tracked || !IsFinite(hand.height)) {
|
||||
track = {};
|
||||
return false;
|
||||
}
|
||||
if (!track.has_last) {
|
||||
track.has_last = true;
|
||||
track.last = hand.height;
|
||||
return false;
|
||||
}
|
||||
const float velocity = (hand.height - track.last) / dt;
|
||||
if (std::fabs(velocity) > kJumpSpeed) {
|
||||
track = {};
|
||||
track.has_last = true;
|
||||
track.last = hand.height;
|
||||
return false;
|
||||
}
|
||||
if (velocity >= kRisingSpeed) {
|
||||
if (!track.rising) {
|
||||
track.rising = true;
|
||||
track.spent = false;
|
||||
track.start = track.last;
|
||||
track.time = 0.0f;
|
||||
track.samples = 0;
|
||||
}
|
||||
track.time += dt;
|
||||
++track.samples;
|
||||
} else {
|
||||
track.rising = false;
|
||||
}
|
||||
track.last = hand.height;
|
||||
if (!track.rising || track.spent) {
|
||||
return false;
|
||||
}
|
||||
const float rise = hand.height - track.start;
|
||||
if (rise < kMinRise) {
|
||||
if (track.time > kWindowSeconds) {
|
||||
track.spent = true; // a slow lift, not a flick
|
||||
}
|
||||
return false;
|
||||
}
|
||||
if (track.samples < kMinSamples) {
|
||||
return false;
|
||||
}
|
||||
speed = rise / track.time;
|
||||
return true;
|
||||
}
|
||||
|
||||
std::array<Track, 2> m_tracks{};
|
||||
float m_cooldown = 0.0f;
|
||||
};
|
||||
|
||||
// The remote's accelerometer through one flick: up, then down, one cycle of a
|
||||
// shake, as Dolphin's emulated shake produces them. It lasts long enough that
|
||||
// the guest, which reads only the latest sample, sees it on at least three of
|
||||
// its frames; KPAD derives acc_speed from the change between them.
|
||||
inline constexpr int64_t kFlickPulseNs = 150'000'000;
|
||||
inline constexpr float kFlickPulseG = 3.0f;
|
||||
|
||||
inline wii_remote::Vec3 FlickPulse(int64_t elapsed_ns, bool* active) noexcept {
|
||||
const wii_remote::Vec3 rest{0.0f, -1.0f, 0.0f};
|
||||
if (elapsed_ns < 0 || elapsed_ns >= kFlickPulseNs) {
|
||||
if (active != nullptr) {
|
||||
*active = false;
|
||||
}
|
||||
return rest;
|
||||
}
|
||||
if (active != nullptr) {
|
||||
*active = true;
|
||||
}
|
||||
const float phase = static_cast<float>(elapsed_ns) / static_cast<float>(kFlickPulseNs);
|
||||
const float swing = kFlickPulseG * std::sin(phase * 6.2831853f);
|
||||
return {0.0f, std::clamp(rest[1] + swing, -wii_remote::kAccelRangeG, wii_remote::kAccelRangeG), 0.0f};
|
||||
}
|
||||
|
||||
} // namespace mkw::vr::hand_tracking
|
||||
@@ -6,6 +6,7 @@
|
||||
|
||||
#include "vr/camera_toggle.h"
|
||||
#include "vr/openxr_driving.h"
|
||||
#include "vr/openxr_hand_tracking.h"
|
||||
#include "vr/openxr_runtime.h"
|
||||
#include "vr/openxr_settings_panel.h"
|
||||
#include "vr/openxr_wii_remote.h"
|
||||
@@ -66,6 +67,15 @@ struct OpenXRPointerScreen {
|
||||
// the game (a shoulder on the gamepad; as a Wii Remote the grips are unbound,
|
||||
// so a hand on the wheel cannot hold down a button).
|
||||
//
|
||||
// Tracked hands ([vr] hand_tracking, with hand steering): two hand trackers
|
||||
// (XR_EXT_hand_tracking) located every frame give the cockpit the hands' own
|
||||
// joints, from the controllers' touch sensors while they are held and from the
|
||||
// cameras once they are put down. On the Quest a hand that drives
|
||||
// khr/simple_controller instead of the Touch profile has bare-hand buttons
|
||||
// (openxr_hand_tracking.h): with the option off it presses nothing but the menu
|
||||
// gesture, so the manifest's hand-tracking permission changes nothing for
|
||||
// players who leave it off.
|
||||
//
|
||||
// A right-thumbstick click on its own toggles the first-person camera, as its
|
||||
// F10 checkbox does (first_person_toggle_click).
|
||||
//
|
||||
@@ -105,6 +115,13 @@ public:
|
||||
|
||||
// The cockpit state the last Sync published (also OpenXRReadDriving()).
|
||||
const DrivingSnapshot& Driving() const noexcept { return m_driving; }
|
||||
// The tracked hands' joints in the seated frame, as the last Sync located
|
||||
// them; read on the pacing thread only.
|
||||
const hand_tracking::HandJointFrame& HandJoints() const noexcept { return m_joint_frame; }
|
||||
// A hand's tracker while tracked hands keep one, else XR_NULL_HANDLE.
|
||||
XrHandTrackerEXT HandTracker(uint32_t hand) const noexcept {
|
||||
return hand < kHands ? m_hand_trackers[hand] : XR_NULL_HANDLE;
|
||||
}
|
||||
|
||||
// Publishes a remote with nothing held, at rest and not pointing, and stops
|
||||
// the haptics, for frames without focused input.
|
||||
@@ -140,6 +157,14 @@ private:
|
||||
void UpdateDriving(XrTime display_time, const driving::SeatFrame& seat,
|
||||
std::array<wii_remote::HandInputs, kHands>& hands, bool withheld);
|
||||
void ResetDriving();
|
||||
// Tracked hands: the extension's functions at Create, the trackers as the
|
||||
// settings ask for them, and both hands located for `time`, their joints
|
||||
// in `seat` when it is valid.
|
||||
void LoadHandTracking();
|
||||
void UpdateHandTrackers();
|
||||
void DestroyHandTrackers();
|
||||
void LocateHands(XrTime time, const driving::SeatFrame& seat);
|
||||
void LogInteractionProfiles();
|
||||
void UpdateRumble();
|
||||
void StopRumble();
|
||||
bool Check(XrResult result, const char* operation);
|
||||
@@ -182,6 +207,38 @@ private:
|
||||
bool m_wheel_uses_geometry = false;
|
||||
bool m_wheel_bike = false;
|
||||
DrivingSnapshot m_driving{};
|
||||
|
||||
// Tracked hands.
|
||||
struct TrackedHand {
|
||||
bool active = false; // joints located this frame
|
||||
bool seated = false; // and written into m_joint_frame, in the seated frame
|
||||
hand_tracking::Source source = hand_tracking::Source::None;
|
||||
hand_tracking::JointPositions positions{}; // application space
|
||||
// XR_FB_hand_tracking_aim: the runtime's own pinch and menu gesture.
|
||||
bool aim_valid = false;
|
||||
bool aim_pinching = false;
|
||||
bool aim_menu = false;
|
||||
bool aim_system_gesture = false;
|
||||
};
|
||||
PFN_xrCreateHandTrackerEXT m_create_hand_tracker = nullptr;
|
||||
PFN_xrDestroyHandTrackerEXT m_destroy_hand_tracker = nullptr;
|
||||
PFN_xrLocateHandJointsEXT m_locate_hand_joints = nullptr;
|
||||
bool m_hand_data_source = false; // XR_EXT_hand_tracking_data_source
|
||||
bool m_hand_aim = false; // XR_FB_hand_tracking_aim
|
||||
XrHandTrackerEXT m_hand_trackers[kHands]{};
|
||||
bool m_hand_trackers_failed = false;
|
||||
bool m_logged_hand_restart = false;
|
||||
std::array<TrackedHand, kHands> m_tracked_hands{};
|
||||
hand_tracking::HandJointFrame m_joint_frame{};
|
||||
std::array<hand_tracking::Source, kHands> m_logged_sources{hand_tracking::Source::None,
|
||||
hand_tracking::Source::None};
|
||||
// Per frame, from the actions: a controller is in the hand (its squeeze is
|
||||
// bound), and the hand drives simple_controller (Android only).
|
||||
std::array<bool, kHands> m_squeeze_active{};
|
||||
std::array<bool, kHands> m_hand_driven{};
|
||||
std::array<bool, kHands> m_pinch{};
|
||||
uint64_t m_profile_serial = 0;
|
||||
|
||||
bool m_created = false;
|
||||
bool m_logged_sync_failure = false;
|
||||
bool m_logged_pointer = false;
|
||||
|
||||
@@ -78,6 +78,10 @@ void OpenXRSetImmersiveWindow(bool enabled) noexcept;
|
||||
// 0 = Off, 1 = Auto, otherwise 72/90/120 as a rendering-rate ceiling.
|
||||
void OpenXRSetFrameInterpolationFps(uint32_t target) noexcept;
|
||||
bool OpenXRFrameInterpolationAvailable() noexcept;
|
||||
|
||||
// This session started with XR_EXT_hand_tracking, so [vr] hand_tracking applies
|
||||
// live (it is asked for when hand steering or tracked hands are on at launch).
|
||||
bool OpenXRHandTrackingAvailable() noexcept;
|
||||
struct OpenXRFrameTiming {
|
||||
float headset_hz = 0;
|
||||
float rendered_fps = 0; // Newly rendered pairs; excludes retained-layer repeats.
|
||||
|
||||
@@ -209,6 +209,9 @@ public:
|
||||
XrSpace ViewSpace() const { return m_view_space; }
|
||||
XrSessionState SessionState() const { return m_session_state; }
|
||||
uint64_t SessionRunSerial() const { return m_session_run_serial; }
|
||||
// Advances each time the runtime reports that a hand's interaction profile
|
||||
// changed (a controller picked up or put down).
|
||||
uint64_t InteractionProfileSerial() const { return m_interaction_profile_serial; }
|
||||
XrReferenceSpaceType AppSpaceType() const { return m_app_space_type; }
|
||||
XrEnvironmentBlendMode EnvironmentBlendMode() const { return m_blend_mode; }
|
||||
|
||||
@@ -281,6 +284,7 @@ private:
|
||||
|
||||
FramePhase m_frame_phase = FramePhase::Idle;
|
||||
uint64_t m_session_run_serial = 0;
|
||||
uint64_t m_interaction_profile_serial = 0;
|
||||
uint64_t m_next_frame_serial = 1;
|
||||
uint64_t m_active_frame_serial = 0;
|
||||
XrTime m_active_frame_display_time = 0;
|
||||
|
||||
@@ -15,6 +15,7 @@
|
||||
#include "vr/mkw_vr_first_person.h"
|
||||
#include "vr/mkw_vr_policy.h"
|
||||
#include "vr/openxr_diagnostics.h"
|
||||
#include "vr/openxr_driving.h"
|
||||
#include "vr/openxr_integration.h"
|
||||
#include "vr/openxr_settings_panel.h"
|
||||
#include "vr/openxr_wii_remote.h"
|
||||
@@ -149,6 +150,7 @@ bool g_vrFlatScreen = g_vrRaceView == static_cast<int>(RuntimeConfigFile::VrRace
|
||||
constexpr std::array<const char*, 3> kVrRaceViewLabels{"Immersive", "Immersive window", "Flat screen"};
|
||||
#if defined(__ANDROID__)
|
||||
bool g_vrPassthrough = RuntimeConfigFile::VrPassthrough();
|
||||
bool g_vrHandTracking = RuntimeConfigFile::VrHandTracking();
|
||||
// Menu labels for the foveation levels, index-matched to RuntimeConfigFile::kVrFoveationLevels and to
|
||||
// aurora_set_stereo_foveation.
|
||||
constexpr std::array<const char*, 4> kVrFoveationLabels{"Off", "Low", "Medium", "High"};
|
||||
@@ -1181,6 +1183,34 @@ void DrawVrSteeringWheelSettings() {
|
||||
"back to the stick, which still aims items. The runtime's hand mesh is "
|
||||
"used when hand steering was on at launch.");
|
||||
}
|
||||
#if defined(__ANDROID__)
|
||||
ImGui::BeginDisabled(!g_vrHandSteering);
|
||||
if (ImGui::Checkbox("Tracked hands", &g_vrHandTracking)) {
|
||||
RuntimeConfigFile::SetVrHandTracking(g_vrHandTracking);
|
||||
}
|
||||
if (ImGui::IsItemHovered()) {
|
||||
ImGui::SetTooltip(
|
||||
"%s", mkw::vr::OpenXRHandTrackingAvailable()
|
||||
? "The cockpit hands follow your own. Holding the controllers, the fingers follow "
|
||||
"their touch sensors; put the controllers down and the headset's cameras track "
|
||||
"your hands. Needs hand tracking on in the headset's settings."
|
||||
: "The cockpit hands follow your own, from the controllers' touch sensors or, "
|
||||
"with the controllers put down, the headset's cameras. This session started "
|
||||
"with hand steering and tracked hands off, so it applies after a restart.");
|
||||
}
|
||||
ImGui::EndDisabled();
|
||||
if (g_vrHandSteering && g_vrHandTracking) {
|
||||
const mkw::vr::DrivingSnapshot driving = mkw::vr::OpenXRReadDriving();
|
||||
const auto hand = [&](size_t side) {
|
||||
const auto& state = driving.hands[side];
|
||||
return std::string(mkw::vr::hand_tracking::SourceLabel(state.source)) +
|
||||
(state.bare ? " (bare)" : "") + (state.held ? ", holding" : "") +
|
||||
(state.pinch ? ", pinch" : "");
|
||||
};
|
||||
ImGui::TextDisabled("Hands: left %s, grasp %.2f | right %s, grasp %.2f", hand(0).c_str(),
|
||||
driving.hands[0].grasp, hand(1).c_str(), driving.hands[1].grasp);
|
||||
}
|
||||
#endif
|
||||
if (g_vrHandSteering && ImGui::TreeNode("Hand steering tuning")) {
|
||||
bool changed = false;
|
||||
changed |= ImGui::SliderFloat("Kart full lock (degrees)", &g_vrWheelTuning.kartDegrees,
|
||||
@@ -1694,6 +1724,10 @@ void DrawVrSettings() {
|
||||
RuntimeConfigFile::SetVrSteeringWheel(g_vrSteeringWheel);
|
||||
RuntimeConfigFile::SetVrNativeSteeringWheel(g_vrNativeSteeringWheel);
|
||||
RuntimeConfigFile::SetVrHandSteering(g_vrHandSteering);
|
||||
#if defined(__ANDROID__)
|
||||
g_vrHandTracking = RuntimeConfigFile::kVrHandTrackingDefault;
|
||||
RuntimeConfigFile::SetVrHandTracking(g_vrHandTracking);
|
||||
#endif
|
||||
g_vrFirstPersonUnitsPerMeter = RuntimeConfigFile::kVrFirstPersonUnitsPerMeterDefault;
|
||||
g_vrFirstPersonHeadUp = RuntimeConfigFile::kVrFirstPersonHeadUpDefault;
|
||||
g_vrFirstPersonHeadForward = RuntimeConfigFile::kVrFirstPersonHeadForwardDefault;
|
||||
|
||||
+293
-13
@@ -271,6 +271,7 @@ bool OpenXRInput::Create(OpenXRRuntime& runtime) {
|
||||
m_created = true;
|
||||
CreatePoseSpaces();
|
||||
LoadInputClock();
|
||||
LoadHandTracking();
|
||||
if (!AttachVirtualGamepad()) {
|
||||
Log(OpenXRLogLevel::Warning,
|
||||
"SDL refused the virtual gamepad; OpenXR controllers will not reach the game");
|
||||
@@ -484,6 +485,197 @@ XrTime OpenXRInput::InputSampleTime(XrTime predicted_display_time) const {
|
||||
return now > 0 ? (std::min)(predicted_display_time, now) : predicted_display_time;
|
||||
}
|
||||
|
||||
// The hand-tracking extensions are asked for at launch when hand steering or
|
||||
// tracked hands are on (openxr_integration.cpp), so the option itself is live.
|
||||
void OpenXRInput::LoadHandTracking() {
|
||||
const auto& extensions = m_runtime->EnabledExtensions();
|
||||
const auto enabled = [&](const char* name) {
|
||||
return std::find(extensions.begin(), extensions.end(), name) != extensions.end();
|
||||
};
|
||||
if (!enabled(XR_EXT_HAND_TRACKING_EXTENSION_NAME)) {
|
||||
return;
|
||||
}
|
||||
PFN_xrVoidFunction create = nullptr, destroy = nullptr, locate = nullptr;
|
||||
if (!m_runtime->GetInstanceProcAddress("xrCreateHandTrackerEXT", &create) ||
|
||||
!m_runtime->GetInstanceProcAddress("xrDestroyHandTrackerEXT", &destroy) ||
|
||||
!m_runtime->GetInstanceProcAddress("xrLocateHandJointsEXT", &locate) || create == nullptr ||
|
||||
destroy == nullptr || locate == nullptr) {
|
||||
return;
|
||||
}
|
||||
m_create_hand_tracker = reinterpret_cast<PFN_xrCreateHandTrackerEXT>(create);
|
||||
m_destroy_hand_tracker = reinterpret_cast<PFN_xrDestroyHandTrackerEXT>(destroy);
|
||||
m_locate_hand_joints = reinterpret_cast<PFN_xrLocateHandJointsEXT>(locate);
|
||||
m_hand_data_source = enabled(XR_EXT_HAND_TRACKING_DATA_SOURCE_EXTENSION_NAME);
|
||||
m_hand_aim = enabled(XR_FB_HAND_TRACKING_AIM_EXTENSION_NAME);
|
||||
}
|
||||
|
||||
// Trackers only exist while tracked hands and hand steering are both on: the
|
||||
// hands are only drawn while they can steer. They live as long as the session
|
||||
// otherwise (Idle and the cockpit's reset keep them); a runtime that refuses
|
||||
// them is not asked again until the option is turned off and on.
|
||||
void OpenXRInput::UpdateHandTrackers() {
|
||||
if (!RuntimeConfigFile::VrHandTracking() || !RuntimeConfigFile::VrHandSteering()) {
|
||||
DestroyHandTrackers();
|
||||
m_hand_trackers_failed = false;
|
||||
m_logged_hand_restart = false;
|
||||
return;
|
||||
}
|
||||
if (m_hand_trackers[0] != XR_NULL_HANDLE || m_hand_trackers_failed) {
|
||||
return;
|
||||
}
|
||||
if (m_create_hand_tracker == nullptr) {
|
||||
if (!m_logged_hand_restart) {
|
||||
m_logged_hand_restart = true;
|
||||
Log(OpenXRLogLevel::Info, "OpenXR tracked hands apply after a restart: this session started "
|
||||
"without XR_EXT_hand_tracking");
|
||||
}
|
||||
return;
|
||||
}
|
||||
bool controller_hands = m_hand_data_source;
|
||||
for (uint32_t hand = 0; hand < kHands; ++hand) {
|
||||
XrHandTrackerCreateInfoEXT info{XR_TYPE_HAND_TRACKER_CREATE_INFO_EXT};
|
||||
info.hand = hand == 0 ? XR_HAND_LEFT_EXT : XR_HAND_RIGHT_EXT;
|
||||
info.handJointSet = XR_HAND_JOINT_SET_DEFAULT_EXT;
|
||||
// Both sources: the cameras once the controllers are put down, the
|
||||
// controllers' touch sensors while they are held.
|
||||
XrHandTrackingDataSourceEXT sources[]{XR_HAND_TRACKING_DATA_SOURCE_UNOBSTRUCTED_EXT,
|
||||
XR_HAND_TRACKING_DATA_SOURCE_CONTROLLER_EXT};
|
||||
XrHandTrackingDataSourceInfoEXT source_info{XR_TYPE_HAND_TRACKING_DATA_SOURCE_INFO_EXT};
|
||||
source_info.requestedDataSourceCount = 2;
|
||||
source_info.requestedDataSources = sources;
|
||||
info.next = controller_hands ? &source_info : nullptr;
|
||||
XrResult result = m_create_hand_tracker(m_runtime->Session(), &info, &m_hand_trackers[hand]);
|
||||
if (XR_FAILED(result) && info.next != nullptr) {
|
||||
// The cameras still work without the controller source.
|
||||
controller_hands = false;
|
||||
info.next = nullptr;
|
||||
result = m_create_hand_tracker(m_runtime->Session(), &info, &m_hand_trackers[hand]);
|
||||
}
|
||||
m_runtime->ObserveResult(result);
|
||||
if (XR_FAILED(result)) {
|
||||
m_hand_trackers[hand] = XR_NULL_HANDLE;
|
||||
DestroyHandTrackers();
|
||||
m_hand_trackers_failed = true;
|
||||
std::ostringstream message;
|
||||
message << "xrCreateHandTrackerEXT failed (" << result
|
||||
<< "); the cockpit hands keep following the controllers";
|
||||
Log(OpenXRLogLevel::Warning, message.str());
|
||||
return;
|
||||
}
|
||||
}
|
||||
Log(OpenXRLogLevel::Info, controller_hands ? "OpenXR tracked hands ready (controller-driven hands: yes)"
|
||||
: "OpenXR tracked hands ready (controller-driven hands: no)");
|
||||
}
|
||||
|
||||
void OpenXRInput::DestroyHandTrackers() {
|
||||
for (XrHandTrackerEXT& tracker : m_hand_trackers) {
|
||||
if (tracker != XR_NULL_HANDLE) {
|
||||
if (m_destroy_hand_tracker != nullptr) {
|
||||
m_destroy_hand_tracker(tracker);
|
||||
}
|
||||
tracker = XR_NULL_HANDLE;
|
||||
}
|
||||
}
|
||||
m_tracked_hands = {};
|
||||
m_joint_frame = {};
|
||||
}
|
||||
|
||||
// Both hands' joints for `time` in the application space, and, when the
|
||||
// seated frame is valid, in it for the cockpit (m_joint_frame keeps a hand's
|
||||
// last located joints until new ones arrive; UpdateDriving decides what is
|
||||
// drawn). A hand counts only when every joint is located.
|
||||
void OpenXRInput::LocateHands(XrTime time, const driving::SeatFrame& seat) {
|
||||
using hand_tracking::Source;
|
||||
for (uint32_t hand = 0; hand < kHands; ++hand) {
|
||||
TrackedHand& tracked = m_tracked_hands[hand];
|
||||
tracked = {};
|
||||
if (m_hand_trackers[hand] == XR_NULL_HANDLE || m_locate_hand_joints == nullptr) {
|
||||
continue;
|
||||
}
|
||||
std::array<XrHandJointLocationEXT, XR_HAND_JOINT_COUNT_EXT> joints{};
|
||||
XrHandJointLocationsEXT locations{XR_TYPE_HAND_JOINT_LOCATIONS_EXT};
|
||||
locations.jointCount = XR_HAND_JOINT_COUNT_EXT;
|
||||
locations.jointLocations = joints.data();
|
||||
XrHandTrackingDataSourceStateEXT source{XR_TYPE_HAND_TRACKING_DATA_SOURCE_STATE_EXT};
|
||||
XrHandTrackingAimStateFB aim{XR_TYPE_HAND_TRACKING_AIM_STATE_FB};
|
||||
void* next = nullptr;
|
||||
if (m_hand_data_source) {
|
||||
source.next = next;
|
||||
next = &source;
|
||||
}
|
||||
if (m_hand_aim) {
|
||||
aim.next = next;
|
||||
next = &aim;
|
||||
}
|
||||
locations.next = next;
|
||||
XrHandJointsLocateInfoEXT info{XR_TYPE_HAND_JOINTS_LOCATE_INFO_EXT};
|
||||
info.baseSpace = m_runtime->AppSpace();
|
||||
info.time = time;
|
||||
if (XR_FAILED(m_locate_hand_joints(m_hand_trackers[hand], &info, &locations)) ||
|
||||
locations.isActive != XR_TRUE) {
|
||||
continue;
|
||||
}
|
||||
constexpr XrSpaceLocationFlags kValid =
|
||||
XR_SPACE_LOCATION_POSITION_VALID_BIT | XR_SPACE_LOCATION_ORIENTATION_VALID_BIT;
|
||||
if (!std::all_of(joints.begin(), joints.end(),
|
||||
[](const XrHandJointLocationEXT& joint) { return (joint.locationFlags & kValid) == kValid; })) {
|
||||
continue;
|
||||
}
|
||||
tracked.active = true;
|
||||
tracked.source = !m_hand_data_source || source.isActive != XR_TRUE ? Source::Unknown
|
||||
: source.dataSource == XR_HAND_TRACKING_DATA_SOURCE_CONTROLLER_EXT ? Source::Controller
|
||||
: Source::Camera;
|
||||
if (m_hand_aim) {
|
||||
tracked.aim_valid = (aim.status & XR_HAND_TRACKING_AIM_VALID_BIT_FB) != 0;
|
||||
tracked.aim_pinching = (aim.status & XR_HAND_TRACKING_AIM_INDEX_PINCHING_BIT_FB) != 0;
|
||||
tracked.aim_menu = (aim.status & XR_HAND_TRACKING_AIM_MENU_PRESSED_BIT_FB) != 0;
|
||||
tracked.aim_system_gesture = (aim.status & XR_HAND_TRACKING_AIM_SYSTEM_GESTURE_BIT_FB) != 0;
|
||||
}
|
||||
for (size_t joint = 0; joint < hand_tracking::kJointCount; ++joint) {
|
||||
const XrPosef& pose = joints[joint].pose;
|
||||
tracked.positions[joint] = {pose.position.x, pose.position.y, pose.position.z};
|
||||
if (seat.valid) {
|
||||
m_joint_frame.seat_from_joint[hand][joint] =
|
||||
driving::SeatFromApp(seat, {pose.position.x, pose.position.y, pose.position.z},
|
||||
{pose.orientation.x, pose.orientation.y, pose.orientation.z,
|
||||
pose.orientation.w});
|
||||
m_joint_frame.radius[hand][joint] = joints[joint].radius;
|
||||
}
|
||||
}
|
||||
tracked.seated = seat.valid;
|
||||
}
|
||||
if (m_tracked_hands[0].source != m_logged_sources[0] || m_tracked_hands[1].source != m_logged_sources[1]) {
|
||||
m_logged_sources = {m_tracked_hands[0].source, m_tracked_hands[1].source};
|
||||
std::ostringstream message;
|
||||
message << "OpenXR tracked hands: left " << hand_tracking::SourceLabel(m_logged_sources[0]) << ", right "
|
||||
<< hand_tracking::SourceLabel(m_logged_sources[1]);
|
||||
Log(OpenXRLogLevel::Info, message.str());
|
||||
}
|
||||
}
|
||||
|
||||
// Which interaction profile each hand moved to, whenever the runtime reports a
|
||||
// change: on the Quest, the Touch profile with controllers and
|
||||
// khr/simple_controller for bare hands.
|
||||
void OpenXRInput::LogInteractionProfiles() {
|
||||
std::ostringstream message;
|
||||
message << "OpenXR interaction profiles:";
|
||||
for (uint32_t hand = 0; hand < kHands; ++hand) {
|
||||
message << (hand == 0 ? " left " : ", right ");
|
||||
XrInteractionProfileState state{XR_TYPE_INTERACTION_PROFILE_STATE};
|
||||
char path[XR_MAX_PATH_LENGTH]{};
|
||||
uint32_t length = 0;
|
||||
if (XR_SUCCEEDED(xrGetCurrentInteractionProfile(m_runtime->Session(), m_hand_paths[hand], &state)) &&
|
||||
state.interactionProfile != XR_NULL_PATH &&
|
||||
XR_SUCCEEDED(xrPathToString(m_runtime->Instance(), state.interactionProfile, sizeof(path), &length,
|
||||
path))) {
|
||||
message << path;
|
||||
} else {
|
||||
message << "none";
|
||||
}
|
||||
}
|
||||
Log(OpenXRLogLevel::Info, message.str());
|
||||
}
|
||||
|
||||
void OpenXRApplyVirtualGamepad() noexcept {
|
||||
Relay().Apply();
|
||||
}
|
||||
@@ -544,6 +736,15 @@ void OpenXRInput::Destroy() {
|
||||
StopRumble();
|
||||
}
|
||||
DetachVirtualGamepad();
|
||||
DestroyHandTrackers();
|
||||
m_create_hand_tracker = nullptr;
|
||||
m_destroy_hand_tracker = nullptr;
|
||||
m_locate_hand_joints = nullptr;
|
||||
m_hand_data_source = m_hand_aim = false;
|
||||
m_hand_trackers_failed = m_logged_hand_restart = false;
|
||||
m_logged_sources = {};
|
||||
m_squeeze_active = m_hand_driven = m_pinch = {};
|
||||
m_profile_serial = 0;
|
||||
DestroyPoseSpaces();
|
||||
if (m_action_set != XR_NULL_HANDLE) {
|
||||
// Destroying the set destroys every action created from it.
|
||||
@@ -583,6 +784,9 @@ void OpenXRInput::Idle() {
|
||||
m_panel_select_held = false;
|
||||
OpenXRPublishSettingsPanelPointer(false, 0.0f, 0.0f, false, 0.0f);
|
||||
m_first_person_click.Reset();
|
||||
// The trackers stay with the session; only this frame's hands are forgotten.
|
||||
m_tracked_hands = {};
|
||||
m_squeeze_active = m_hand_driven = m_pinch = {};
|
||||
ResetDriving();
|
||||
StopRumble();
|
||||
// Nothing stays held on the gamepad either while input is away.
|
||||
@@ -619,24 +823,31 @@ void OpenXRInput::Sync(XrTime predicted_display_time, const OpenXRPointerScreen&
|
||||
return;
|
||||
}
|
||||
|
||||
const auto boolean = [&](XrAction action, uint32_t hand) {
|
||||
// `active`, when given, says whether the action is bound to a source the
|
||||
// runtime has right now (a controller, or a tracked hand).
|
||||
const auto boolean = [&](XrAction action, uint32_t hand, bool* active = nullptr) {
|
||||
XrActionStateGetInfo info{XR_TYPE_ACTION_STATE_GET_INFO};
|
||||
info.action = action;
|
||||
info.subactionPath = m_hand_paths[hand];
|
||||
XrActionStateBoolean state{XR_TYPE_ACTION_STATE_BOOLEAN};
|
||||
return XR_SUCCEEDED(xrGetActionStateBoolean(m_runtime->Session(), &info, &state)) &&
|
||||
state.isActive == XR_TRUE && state.currentState == XR_TRUE;
|
||||
const bool bound = XR_SUCCEEDED(xrGetActionStateBoolean(m_runtime->Session(), &info, &state)) &&
|
||||
state.isActive == XR_TRUE;
|
||||
if (active != nullptr) {
|
||||
*active = bound;
|
||||
}
|
||||
return bound && state.currentState == XR_TRUE;
|
||||
};
|
||||
const auto scalar = [&](XrAction action, uint32_t hand) {
|
||||
const auto scalar = [&](XrAction action, uint32_t hand, bool* active = nullptr) {
|
||||
XrActionStateGetInfo info{XR_TYPE_ACTION_STATE_GET_INFO};
|
||||
info.action = action;
|
||||
info.subactionPath = m_hand_paths[hand];
|
||||
XrActionStateFloat state{XR_TYPE_ACTION_STATE_FLOAT};
|
||||
if (XR_FAILED(xrGetActionStateFloat(m_runtime->Session(), &info, &state)) ||
|
||||
state.isActive != XR_TRUE) {
|
||||
return 0.0f;
|
||||
const bool bound = XR_SUCCEEDED(xrGetActionStateFloat(m_runtime->Session(), &info, &state)) &&
|
||||
state.isActive == XR_TRUE;
|
||||
if (active != nullptr) {
|
||||
*active = bound;
|
||||
}
|
||||
return state.currentState;
|
||||
return bound ? state.currentState : 0.0f;
|
||||
};
|
||||
const auto vector = [&](XrAction action, uint32_t hand) {
|
||||
XrActionStateGetInfo info{XR_TYPE_ACTION_STATE_GET_INFO};
|
||||
@@ -651,17 +862,22 @@ void OpenXRInput::Sync(XrTime predicted_display_time, const OpenXRPointerScreen&
|
||||
};
|
||||
|
||||
std::array<wii_remote::HandInputs, kHands> hands{};
|
||||
// simple_controller binds select to the right primary and the left secondary action.
|
||||
std::array<bool, kHands> select_active{};
|
||||
for (uint32_t hand = 0; hand < kHands; ++hand) {
|
||||
wii_remote::HandInputs& inputs = hands[hand];
|
||||
inputs.primary = boolean(m_button_primary, hand);
|
||||
inputs.secondary = boolean(m_button_secondary, hand);
|
||||
bool primary_active = false, secondary_active = false, squeeze_active = false;
|
||||
inputs.primary = boolean(m_button_primary, hand, &primary_active);
|
||||
inputs.secondary = boolean(m_button_secondary, hand, &secondary_active);
|
||||
inputs.menu = boolean(m_menu, hand);
|
||||
inputs.thumbstick_click = boolean(m_thumbstick_click, hand);
|
||||
inputs.trigger = scalar(m_trigger, hand);
|
||||
inputs.squeeze = scalar(m_squeeze, hand);
|
||||
inputs.squeeze = scalar(m_squeeze, hand, &squeeze_active);
|
||||
const XrVector2f stick = vector(m_thumbstick, hand);
|
||||
inputs.stick_x = stick.x;
|
||||
inputs.stick_y = stick.y;
|
||||
m_squeeze_active[hand] = squeeze_active;
|
||||
select_active[hand] = hand == 1 ? primary_active : secondary_active;
|
||||
}
|
||||
|
||||
PollInjection();
|
||||
@@ -675,6 +891,42 @@ void OpenXRInput::Sync(XrTime predicted_display_time, const OpenXRPointerScreen&
|
||||
hands[0].stick_x = 1.0f;
|
||||
}
|
||||
|
||||
if (m_runtime->InteractionProfileSerial() != m_profile_serial) {
|
||||
m_profile_serial = m_runtime->InteractionProfileSerial();
|
||||
LogInteractionProfiles();
|
||||
}
|
||||
// Tracked hands, before anything reads the hands (the settings panel, the
|
||||
// wheel, the game).
|
||||
const bool hand_tracking_on = RuntimeConfigFile::VrHandTracking();
|
||||
UpdateHandTrackers();
|
||||
LocateHands(predicted_display_time, seat);
|
||||
#if defined(__ANDROID__)
|
||||
// A hand driving simple_controller is a bare hand: its select is a pinch
|
||||
// and its menu the palm-up gesture, which pauses from either hand.
|
||||
bool menu_gesture = false;
|
||||
for (uint32_t hand = 0; hand < kHands; ++hand) {
|
||||
m_hand_driven[hand] = !m_squeeze_active[hand] && select_active[hand];
|
||||
if (!m_hand_driven[hand]) {
|
||||
m_pinch[hand] = false;
|
||||
continue;
|
||||
}
|
||||
const TrackedHand& tracked = m_tracked_hands[hand];
|
||||
const hand_tracking::Gestures gestures = hand_tracking::GesturesOf(
|
||||
tracked.aim_valid, tracked.aim_pinching, tracked.aim_menu, tracked.aim_system_gesture,
|
||||
hand_tracking::SelectOf(hands[hand], hand), hands[hand].menu);
|
||||
m_pinch[hand] = gestures.pinch;
|
||||
menu_gesture = menu_gesture || gestures.menu;
|
||||
hands[hand].menu = false;
|
||||
}
|
||||
hand_tracking::ApplyHandDrivenButtons(hands, m_hand_driven, m_pinch, hand_tracking_on);
|
||||
if (menu_gesture) {
|
||||
hands[0].menu = true;
|
||||
}
|
||||
#else
|
||||
(void)select_active;
|
||||
(void)hand_tracking_on;
|
||||
#endif
|
||||
|
||||
const XrTime input_time = InputSampleTime(predicted_display_time);
|
||||
const float dt_seconds =
|
||||
m_last_input_time != 0 && input_time > m_last_input_time
|
||||
@@ -812,6 +1064,15 @@ void OpenXRInput::PublishWiiRemote(XrTime input_time, const OpenXRPointerScreen&
|
||||
aim_valid[hand] = true;
|
||||
}
|
||||
}
|
||||
if (m_hand_driven[hand]) {
|
||||
// A bare hand keeps its pointer but is a still remote: camera-tracked
|
||||
// poses are too noisy to differentiate twice (turning the wheel would
|
||||
// trick and wheelie), and resting every frame clears the history, so
|
||||
// picking a controller back up cannot read as a jolt.
|
||||
m_motion[hand].Rest();
|
||||
(hand == 0 ? sample.nunchuk_acc : sample.acc) = OpenXRWiiRemoteSample{}.acc;
|
||||
continue;
|
||||
}
|
||||
wii_remote::Vec3 grip_position{};
|
||||
wii_remote::Vec3 grip_velocity{};
|
||||
bool position_valid = false;
|
||||
@@ -877,6 +1138,7 @@ void OpenXRInput::ResetDriving() {
|
||||
m_wheel_held = {};
|
||||
m_wheel_time = 0;
|
||||
m_driving = {};
|
||||
m_joint_frame.valid = {};
|
||||
OpenXRPublishDriving(m_driving);
|
||||
}
|
||||
|
||||
@@ -934,6 +1196,7 @@ void OpenXRInput::UpdateDriving(XrTime display_time, const driving::SeatFrame& s
|
||||
|
||||
constexpr XrSpaceLocationFlags kPoseValid =
|
||||
XR_SPACE_LOCATION_POSITION_VALID_BIT | XR_SPACE_LOCATION_ORIENTATION_VALID_BIT;
|
||||
const bool hand_tracking_on = RuntimeConfigFile::VrHandTracking();
|
||||
std::array<WheelHand, kHands> wheel_hands{};
|
||||
for (uint32_t hand = 0; hand < kHands; ++hand) {
|
||||
bool tracked = false;
|
||||
@@ -950,8 +1213,25 @@ void OpenXRInput::UpdateDriving(XrTime display_time, const driving::SeatFrame& s
|
||||
}
|
||||
}
|
||||
const float squeeze = hands[hand].squeeze;
|
||||
const TrackedHand& located = m_tracked_hands[hand];
|
||||
// With tracked hands on, a hand whose joints were located is drawn from
|
||||
// them (the controller's touch sensors, or the cameras). A bare hand is
|
||||
// never drawn or steered from its grip pose, which would show an open
|
||||
// hand wherever it rests.
|
||||
const bool joints = hand_tracking_on && located.active && located.seated;
|
||||
if (m_hand_driven[hand]) {
|
||||
tracked = false;
|
||||
}
|
||||
DrivingHand& out = snapshot.hands[hand];
|
||||
// Hands are shown only while they can steer.
|
||||
snapshot.hands[hand] = {tracked && hand_steering, false, squeeze, seat_from_grip};
|
||||
out.tracked = hand_steering && (tracked || joints);
|
||||
out.held = false;
|
||||
out.squeeze = squeeze;
|
||||
out.seat_from_grip = seat_from_grip;
|
||||
out.joints_valid = joints;
|
||||
out.source = located.source;
|
||||
out.pinch = m_pinch[hand];
|
||||
m_joint_frame.valid[hand] = joints;
|
||||
wheel_hands[hand] = {seat_from_grip[3], seat_from_grip[7], seat_from_grip[11], squeeze, tracked};
|
||||
if (uses_geometry) {
|
||||
wheel_hands[hand] = geometry.ToWheel(wheel_hands[hand]);
|
||||
@@ -966,7 +1246,7 @@ void OpenXRInput::UpdateDriving(XrTime display_time, const driving::SeatFrame& s
|
||||
uses_geometry ? geometry.radius : SteeringWheel::Radius,
|
||||
anchor.bike, tuning);
|
||||
for (uint32_t hand = 0; hand < kHands; ++hand) {
|
||||
if (wheel.held[hand] != m_wheel_held[hand] && active && tuning.haptics) {
|
||||
if (wheel.held[hand] != m_wheel_held[hand] && active && tuning.haptics && !m_hand_driven[hand]) {
|
||||
constexpr XrDuration kGrabPulseNs = 25'000'000;
|
||||
constexpr XrDuration kReleasePulseNs = 15'000'000;
|
||||
ApplyHaptic(hand, wheel.held[hand] ? 0.25f : 0.12f, wheel.held[hand] ? kGrabPulseNs : kReleasePulseNs);
|
||||
|
||||
@@ -193,12 +193,19 @@ XrPosef ScreenPoseAhead(const OpenXRFrame& frame, float distance) noexcept {
|
||||
}
|
||||
|
||||
// Hand steering draws the player's hands, in the runtime's own hand mesh where
|
||||
// it offers one (XR_FB_hand_tracking_mesh). Only asked for when hand steering
|
||||
// is on at launch; turning it on later uses the procedural gloves.
|
||||
// it offers one (XR_FB_hand_tracking_mesh), and tracked hands pose them from
|
||||
// the hand trackers: the controllers' touch sensors while they are held
|
||||
// (XR_EXT_hand_tracking_data_source) and the cameras once they are put down,
|
||||
// with the runtime's own pinch and menu gesture (XR_FB_hand_tracking_aim).
|
||||
// Asked for when either is on at launch, since none costs anything until a
|
||||
// tracker exists; turning both on later needs a restart for the mesh and the
|
||||
// tracked hands (until then the procedural gloves are drawn).
|
||||
void AddHandMeshExtensions(OpenXRConfig& config) {
|
||||
if (RuntimeConfigFile::VrHandSteering()) {
|
||||
if (RuntimeConfigFile::VrHandSteering() || RuntimeConfigFile::VrHandTracking()) {
|
||||
config.optional_extensions.push_back("XR_EXT_hand_tracking");
|
||||
config.optional_extensions.push_back("XR_FB_hand_tracking_mesh");
|
||||
config.optional_extensions.push_back("XR_EXT_hand_tracking_data_source");
|
||||
config.optional_extensions.push_back("XR_FB_hand_tracking_aim");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -437,6 +444,7 @@ public:
|
||||
runtime_->LoadFunction("xrPerfSettingsSetPerformanceLevelEXT", &set_performance_level_);
|
||||
}
|
||||
interpolation_available_.store(convert_display_time_ != nullptr, std::memory_order_release);
|
||||
hand_tracking_available_.store(has_extension("XR_EXT_hand_tracking"), std::memory_order_release);
|
||||
if (!backend_->QueryGraphicsRequirements(*runtime_)) {
|
||||
SetError(backend_->LastError());
|
||||
ResetPreparedObjects();
|
||||
@@ -551,6 +559,7 @@ public:
|
||||
headset_hz_.store(0, std::memory_order_relaxed);
|
||||
rendered_fps_.store(0, std::memory_order_relaxed);
|
||||
interpolation_available_.store(false, std::memory_order_release);
|
||||
hand_tracking_available_.store(false, std::memory_order_release);
|
||||
ResetTrackingOrigin();
|
||||
applied_session_run_serial_ = 0;
|
||||
session_was_active_ = false;
|
||||
@@ -574,6 +583,10 @@ public:
|
||||
return interpolation_available_.load(std::memory_order_acquire);
|
||||
}
|
||||
|
||||
bool HandTrackingAvailable() const noexcept {
|
||||
return hand_tracking_available_.load(std::memory_order_acquire);
|
||||
}
|
||||
|
||||
void SetPassthrough(bool enabled) noexcept {
|
||||
passthrough_.store(enabled, std::memory_order_relaxed);
|
||||
}
|
||||
@@ -1322,7 +1335,10 @@ private:
|
||||
const DrivingSnapshot driving = input_ != nullptr ? input_->Driving() : DrivingSnapshot{};
|
||||
if (driving.hand_steering && !hand_meshes_loaded_ && runtime_ != nullptr) {
|
||||
hand_meshes_loaded_ = true;
|
||||
const bool loaded = LoadRuntimeHandMeshes(*runtime_);
|
||||
// Tracked hands' trackers, when they exist, serve the mesh too.
|
||||
const XrHandTrackerEXT trackers[2]{input_ != nullptr ? input_->HandTracker(0) : XR_NULL_HANDLE,
|
||||
input_ != nullptr ? input_->HandTracker(1) : XR_NULL_HANDLE};
|
||||
const bool loaded = LoadRuntimeHandMeshes(*runtime_, trackers);
|
||||
RT_LOG(RT_TAG_RUNTIME) << "[mkw-vr] cockpit hands: "
|
||||
<< (loaded ? "the runtime's hand mesh" : "procedural gloves (no runtime hand mesh)")
|
||||
<< std::endl;
|
||||
@@ -1346,6 +1362,7 @@ private:
|
||||
ViewFromBase(source.xr_frame.views[eye].pose, base_position_, true, 1.0f, lean_back_radians,
|
||||
cockpit.eyeFromSeat[eye]);
|
||||
}
|
||||
const hand_tracking::HandJointFrame* joints = input_ != nullptr ? &input_->HandJoints() : nullptr;
|
||||
for (size_t hand = 0; hand < 2; ++hand) {
|
||||
auto& target = cockpit.hands[hand];
|
||||
const auto& from = driving.hands[hand];
|
||||
@@ -1353,6 +1370,14 @@ private:
|
||||
target.held = from.held;
|
||||
target.squeeze = from.squeeze;
|
||||
std::copy(from.seat_from_grip.begin(), from.seat_from_grip.end(), target.seatFromGrip);
|
||||
target.jointsValid = from.joints_valid && joints != nullptr && joints->valid[hand];
|
||||
if (target.jointsValid) {
|
||||
for (size_t joint = 0; joint < hand_tracking::kJointCount; ++joint) {
|
||||
const auto& matrix = joints->seat_from_joint[hand][joint];
|
||||
std::copy(matrix.begin(), matrix.end(), target.seatFromJoint[joint]);
|
||||
target.jointRadii[joint] = joints->radius[hand][joint];
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1893,6 +1918,7 @@ private:
|
||||
std::atomic_bool immersive_window_{RuntimeConfigFile::VrImmersiveWindow()};
|
||||
std::atomic_uint32_t frame_interpolation_fps_{RuntimeConfigFile::VrFrameInterpolationFps()};
|
||||
std::atomic_bool interpolation_available_{false};
|
||||
std::atomic_bool hand_tracking_available_{false};
|
||||
std::mutex interpolation_mutex_;
|
||||
bool interpolation_stopping_ = true;
|
||||
FrameInterpolationPacing interpolation_pacing_;
|
||||
@@ -2044,6 +2070,14 @@ bool OpenXRFrameInterpolationAvailable() noexcept {
|
||||
#endif
|
||||
}
|
||||
|
||||
bool OpenXRHandTrackingAvailable() noexcept {
|
||||
#if MKW_OPENXR_GRAPHICS_BACKEND
|
||||
return OpenXRIntegration::Get().HandTrackingAvailable();
|
||||
#else
|
||||
return false;
|
||||
#endif
|
||||
}
|
||||
|
||||
std::string OpenXRLastError() {
|
||||
#if !defined(MKW_ENABLE_OPENXR)
|
||||
return "this build was compiled without OpenXR support";
|
||||
|
||||
@@ -557,6 +557,16 @@ OpenXREventStatus OpenXRRuntime::PollEvents() {
|
||||
}
|
||||
break;
|
||||
}
|
||||
case XR_TYPE_EVENT_DATA_INTERACTION_PROFILE_CHANGED: {
|
||||
// A controller picked up or put down; the input logs the profiles
|
||||
// the hands moved to.
|
||||
const auto& profile_event =
|
||||
*reinterpret_cast<const XrEventDataInteractionProfileChanged*>(&event);
|
||||
if (profile_event.session == m_session) {
|
||||
++m_interaction_profile_serial;
|
||||
}
|
||||
break;
|
||||
}
|
||||
case XR_TYPE_EVENT_DATA_EVENTS_LOST: {
|
||||
const auto& lost_event =
|
||||
*reinterpret_cast<const XrEventDataEventsLost*>(&event);
|
||||
|
||||
@@ -35,6 +35,13 @@ int main() {
|
||||
Require(Parse("[vr]\nsingle_pass_eyes = false\n").vrSinglePassEyes == false);
|
||||
Require(!Parse("[vr]\n").vrSinglePassEyes.has_value());
|
||||
|
||||
// [vr] hand_tracking: the cockpit hands follow the headset's hand tracking.
|
||||
Require(Parse("[vr]\nhand_tracking = true\n").vrHandTracking == true);
|
||||
Require(Parse("[vr]\nhand_tracking = false\n").vrHandTracking == false);
|
||||
Require(!Parse("[vr]\n").vrHandTracking.has_value());
|
||||
Require(!Parse("[vr]\nhand_tracking = 1\n").vrHandTracking.has_value());
|
||||
Require(!RuntimeConfigFile::kVrHandTrackingDefault);
|
||||
|
||||
// [vr] immersive_window and flat_screen: one race view in two keys, Flat
|
||||
// Screen mode winning, so a file that predates the window reads as before.
|
||||
using RuntimeConfigFile::VrRaceView;
|
||||
|
||||
@@ -0,0 +1,312 @@
|
||||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
//
|
||||
// Tracked hands: the grasp read from the fingers, the bare-hand latch, pinch
|
||||
// gating, the bare-hand buttons and the flick, tested without a headset
|
||||
// (vr/openxr_hand_tracking.h).
|
||||
|
||||
#include "vr/openxr_hand_tracking.h"
|
||||
#include "vr/steering_wheel.h"
|
||||
|
||||
#include <cmath>
|
||||
#include <iostream>
|
||||
#include <limits>
|
||||
|
||||
namespace {
|
||||
|
||||
using namespace mkw::vr;
|
||||
using namespace mkw::vr::hand_tracking;
|
||||
|
||||
int g_failures = 0;
|
||||
|
||||
void Check(bool condition, const char* what) {
|
||||
if (!condition) {
|
||||
++g_failures;
|
||||
std::cerr << "FAILED: " << what << '\n';
|
||||
}
|
||||
}
|
||||
|
||||
void CheckNear(float actual, float expected, const char* what, float tolerance = 1.0e-3f) {
|
||||
if (!(std::fabs(actual - expected) <= tolerance)) {
|
||||
++g_failures;
|
||||
std::cerr << "FAILED: " << what << " (expected " << expected << ", got " << actual << ")\n";
|
||||
}
|
||||
}
|
||||
|
||||
// A hand with its fingers pointing along -Z and bending towards -Y (the palm
|
||||
// side), each finger bent by the same three angles at its knuckle, middle and
|
||||
// end joints. `side` mirrors it across X for the other hand.
|
||||
JointPositions Hand(float knuckle, float middle, float end, float side = 1.0f) {
|
||||
JointPositions joints{};
|
||||
joints[kPalm] = {0.0f, 0.0f, 0.0f};
|
||||
joints[kWrist] = {0.0f, 0.0f, 0.08f};
|
||||
for (size_t finger = 0; finger < 5; ++finger) {
|
||||
const size_t base = kMetacarpal[finger];
|
||||
const size_t count = finger == 0 ? 4 : 5;
|
||||
const float x = side * (finger == 0 ? -0.04f : -0.03f + 0.02f * static_cast<float>(finger - 1));
|
||||
joints[base] = {x, 0.0f, 0.06f};
|
||||
joints[base + 1] = {x, 0.0f, -0.01f};
|
||||
const float lengths[3]{0.045f, 0.025f, 0.02f};
|
||||
const float bends[3]{knuckle, middle, end};
|
||||
float angle = 0.0f;
|
||||
// The phalanges after the knuckle: three for a finger, two for the thumb.
|
||||
for (size_t bone = 0; bone + 2 < count; ++bone) {
|
||||
angle += bends[bone];
|
||||
const Vec3& from = joints[base + 1 + bone];
|
||||
joints[base + 2 + bone] = {from[0], from[1] - std::sin(angle) * lengths[bone],
|
||||
from[2] - std::cos(angle) * lengths[bone]};
|
||||
}
|
||||
}
|
||||
return joints;
|
||||
}
|
||||
|
||||
JointPositions Transformed(const JointPositions& joints, float scale, float yaw, const Vec3& offset) {
|
||||
JointPositions out{};
|
||||
const float c = std::cos(yaw), s = std::sin(yaw);
|
||||
for (size_t j = 0; j < kJointCount; ++j) {
|
||||
const Vec3& p = joints[j];
|
||||
out[j] = {scale * (c * p[0] + s * p[2]) + offset[0], scale * p[1] + offset[1],
|
||||
scale * (-s * p[0] + c * p[2]) + offset[2]};
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
void TestGrasp() {
|
||||
CheckNear(GraspFromJoints(Hand(0.05f, 0.05f, 0.05f)), 0.0f, "an open hand does not grasp");
|
||||
Check(GraspFromJoints(Hand(0.3f, 0.5f, 0.3f)) < 0.15f, "a relaxed hand is under the wheel's release");
|
||||
const float grip = GraspFromJoints(Hand(1.0f, 1.3f, 0.6f));
|
||||
Check(grip > 0.55f, "a hand closed on a rim is over the wheel's press");
|
||||
CheckNear(GraspFromJoints(Hand(1.4f, 1.6f, 1.0f)), 1.0f, "a fist grasps fully");
|
||||
CheckNear(GraspFromJoints(Hand(1.0f, 1.3f, 0.6f, -1.0f)), grip, "the other hand grasps the same");
|
||||
CheckNear(GraspFromJoints(Transformed(Hand(1.0f, 1.3f, 0.6f), 1.3f, 0.7f, {0.2f, -0.3f, -0.4f})), grip,
|
||||
"a bigger, turned hand elsewhere grasps the same");
|
||||
JointPositions broken = Hand(1.0f, 1.3f, 0.6f);
|
||||
broken[13][1] = std::numeric_limits<float>::quiet_NaN();
|
||||
CheckNear(GraspFromJoints(broken), 0.0f, "a NaN joint reads as open");
|
||||
JointPositions collapsed = Hand(1.0f, 1.3f, 0.6f);
|
||||
collapsed[14] = collapsed[13];
|
||||
CheckNear(GraspFromJoints(collapsed), 0.0f, "a collapsed bone reads as open");
|
||||
}
|
||||
|
||||
void TestLatch() {
|
||||
constexpr float dt = 1.0f / 72.0f, grace = 0.2f;
|
||||
BareLatch latch;
|
||||
Check(!latch.Update(false, true, false, 0.0f, dt, grace), "a controller hand is not bare");
|
||||
Check(!latch.Update(true, false, false, 0.9f, dt, grace), "a hand-driven hand needs camera joints");
|
||||
Check(latch.Update(true, false, true, 0.9f, dt, grace) && latch.Tracked(), "camera joints make it bare");
|
||||
CheckNear(latch.Grasp(), 0.9f, "the latch keeps the grasp");
|
||||
bool held = true;
|
||||
for (int frame = 0; frame < 10; ++frame) {
|
||||
held = held && latch.Update(false, false, false, 0.0f, dt, grace);
|
||||
}
|
||||
Check(held && !latch.Tracked(), "a short loss keeps it bare but untracked");
|
||||
CheckNear(latch.Grasp(), 0.9f, "the last grasp is kept through the loss");
|
||||
for (int frame = 0; frame < 10; ++frame) {
|
||||
latch.Update(false, false, false, 0.0f, dt, grace);
|
||||
}
|
||||
Check(!latch.Bare(), "a loss past the grace lets go");
|
||||
latch.Update(true, false, true, 0.9f, dt, grace);
|
||||
Check(!latch.Update(false, true, true, 0.9f, dt, grace), "picking a controller up clears it at once");
|
||||
Check(!latch.Update(false, false, true, 0.9f, dt, grace), "joints alone, not hand-driven, are not bare");
|
||||
}
|
||||
|
||||
void TestPinchGate() {
|
||||
constexpr float dt = 1.0f / 72.0f;
|
||||
PinchGate gate;
|
||||
Check(!gate.Update(true, true, dt), "a hand on the wheel never uses an item");
|
||||
bool fired = false;
|
||||
for (int frame = 0; frame < 5; ++frame) {
|
||||
fired = gate.Update(true, false, dt) || fired;
|
||||
}
|
||||
Check(!fired, "a pinch carried off the rim does not use an item");
|
||||
gate.Update(false, false, dt);
|
||||
for (int frame = 0; frame < 12; ++frame) {
|
||||
gate.Update(false, false, dt);
|
||||
}
|
||||
Check(gate.Update(true, false, dt), "a pinch from a hand free for a moment uses an item");
|
||||
Check(gate.Update(true, false, dt), "holding the pinch holds the button");
|
||||
Check(!gate.Update(false, false, dt), "letting go releases it");
|
||||
Check(!gate.Update(true, true, dt), "grabbing the wheel stops it");
|
||||
}
|
||||
|
||||
void TestGestures() {
|
||||
Check(GesturesOf(true, true, false, false, false, false).pinch, "the runtime's pinch counts");
|
||||
Check(!GesturesOf(true, true, false, true, true, false).pinch, "a pinch in the system gesture is the menu");
|
||||
Check(GesturesOf(true, false, true, true, false, false).menu, "the runtime's menu gesture counts");
|
||||
Check(GesturesOf(false, false, false, false, true, false).pinch, "without the aim state, select is the pinch");
|
||||
Check(GesturesOf(false, false, false, false, false, true).menu, "without the aim state, the menu action");
|
||||
wii_remote::HandInputs left{}, right{};
|
||||
left.secondary = true;
|
||||
right.primary = true;
|
||||
Check(SelectOf(left, 0) && SelectOf(right, 1), "simple_controller's select per hand");
|
||||
}
|
||||
|
||||
std::array<wii_remote::HandInputs, 2> PinchingHands() {
|
||||
std::array<wii_remote::HandInputs, 2> hands{};
|
||||
hands[0].secondary = true; // left select
|
||||
hands[0].menu = true;
|
||||
hands[1].primary = true; // right select
|
||||
return hands;
|
||||
}
|
||||
|
||||
void TestHandDrivenButtons() {
|
||||
auto hands = PinchingHands();
|
||||
ApplyHandDrivenButtons(hands, {true, true}, {true, true}, true);
|
||||
Check(hands[1].primary, "option on: a right pinch is A");
|
||||
Check(!hands[0].secondary && !hands[0].primary, "option on: a left pinch no longer toggles the panel");
|
||||
Check(hands[0].menu, "the menu gesture is kept");
|
||||
|
||||
hands = PinchingHands();
|
||||
ApplyHandDrivenButtons(hands, {true, true}, {true, true}, false);
|
||||
Check(!hands[1].primary && !hands[0].secondary, "option off: hand-driven hands press nothing");
|
||||
Check(hands[0].menu, "option off: the menu gesture still pauses");
|
||||
|
||||
std::array<wii_remote::HandInputs, 2> controllers{};
|
||||
controllers[0].secondary = true;
|
||||
controllers[1].primary = true;
|
||||
controllers[1].squeeze = 0.8f;
|
||||
const auto before = controllers;
|
||||
ApplyHandDrivenButtons(controllers, {false, false}, {false, false}, true);
|
||||
Check(controllers[0].secondary == before[0].secondary && controllers[1].primary == before[1].primary &&
|
||||
controllers[1].squeeze == before[1].squeeze,
|
||||
"controller hands are left alone");
|
||||
}
|
||||
|
||||
void TestBareHandRace() {
|
||||
std::array<wii_remote::HandInputs, 2> hands{};
|
||||
hands[1].primary = true; // a right pinch, already A from the menus' mapping
|
||||
ApplyBareHandRace(hands, {true, true}, {false, false}, {false, false});
|
||||
Check(!hands[1].primary, "in a race a bare right pinch is not A");
|
||||
ApplyBareHandRace(hands, {true, true}, {true, false}, {false, false});
|
||||
Check(hands[1].primary, "a bare hand on the wheel holds the gas");
|
||||
Check(hands[0].trigger == 0.0f && hands[0].squeeze == 0.0f && hands[1].squeeze == 0.0f,
|
||||
"holding presses no item and no shoulder");
|
||||
hands = {};
|
||||
ApplyBareHandRace(hands, {true, true}, {false, true}, {true, false});
|
||||
Check(hands[1].primary && hands[0].trigger == 1.0f, "a free hand's pinch uses an item while the other drives");
|
||||
std::array<wii_remote::HandInputs, 2> controllers{};
|
||||
controllers[1].primary = true;
|
||||
controllers[0].trigger = 0.3f;
|
||||
ApplyBareHandRace(controllers, {false, false}, {false, false}, {false, false});
|
||||
Check(controllers[1].primary && controllers[0].trigger == 0.3f, "controller hands keep their own buttons");
|
||||
}
|
||||
|
||||
// Runs the detector over `frames` frames of both hands moving at the given
|
||||
// vertical speeds; returns how many flicks fired.
|
||||
int Flicks(FlickDetector& detector, std::array<FlickHand, 2> hands, float left_speed, float right_speed,
|
||||
int frames, float dt = 1.0f / 72.0f) {
|
||||
int fired = 0;
|
||||
for (int frame = 0; frame < frames; ++frame) {
|
||||
fired += detector.Update(hands, dt) ? 1 : 0;
|
||||
hands[0].height += left_speed * dt;
|
||||
hands[1].height += right_speed * dt;
|
||||
}
|
||||
return fired;
|
||||
}
|
||||
|
||||
void TestFlick() {
|
||||
const std::array<FlickHand, 2> holding{{{true, true, -0.3f}, {true, true, -0.3f}}};
|
||||
FlickDetector detector;
|
||||
Check(Flicks(detector, holding, 0.0f, 0.0f, 20) == 0, "still hands do not flick");
|
||||
Check(Flicks(detector, holding, 2.0f, 2.0f, 12) == 1, "both hands rising together flick once");
|
||||
|
||||
detector.Reset();
|
||||
Check(Flicks(detector, holding, 2.0f, -2.0f, 20) == 0, "a turn (one up, one down) never flicks");
|
||||
detector.Reset();
|
||||
Check(Flicks(detector, holding, 2.5f, 0.2f, 20) == 0, "one hand rising on the wheel is a turn");
|
||||
|
||||
detector.Reset();
|
||||
const std::array<FlickHand, 2> free_right{{{true, true, -0.3f}, {true, false, -0.2f}}};
|
||||
Check(Flicks(detector, free_right, 0.0f, 2.0f, 12) == 1, "a free hand rising fast flicks");
|
||||
detector.Reset();
|
||||
Check(Flicks(detector, free_right, 0.0f, 1.0f, 30) == 0, "a free hand lifted slowly does not");
|
||||
|
||||
detector.Reset();
|
||||
const std::array<FlickHand, 2> lone{{{false, false, 0.0f}, {true, true, -0.3f}}};
|
||||
Check(Flicks(detector, lone, 0.0f, 2.5f, 20) == 0, "a lone hand on the wheel does not flick");
|
||||
|
||||
detector.Reset();
|
||||
std::array<FlickHand, 2> jump = holding;
|
||||
Flicks(detector, jump, 0.0f, 0.0f, 5);
|
||||
jump[0].height += 0.2f;
|
||||
jump[1].height += 0.2f;
|
||||
Check(Flicks(detector, jump, 0.0f, 0.0f, 10) == 0, "a pose jumping back into tracking does not flick");
|
||||
|
||||
detector.Reset();
|
||||
Check(Flicks(detector, holding, 2.0f, 2.0f, 12) == 1, "a flick");
|
||||
Check(Flicks(detector, holding, 0.0f, 0.0f, 5) + Flicks(detector, holding, 2.0f, 2.0f, 12) == 0,
|
||||
"a second flick inside the cooldown does not fire");
|
||||
Check(Flicks(detector, holding, 0.0f, 0.0f, 40) + Flicks(detector, holding, 2.0f, 2.0f, 12) == 1,
|
||||
"after the cooldown it fires again");
|
||||
|
||||
detector.Reset();
|
||||
std::array<FlickHand, 2> nan = holding;
|
||||
nan[0].height = std::numeric_limits<float>::quiet_NaN();
|
||||
Check(Flicks(detector, nan, 2.0f, 2.0f, 12) == 0, "a NaN height never flicks");
|
||||
Check(!detector.Update(holding, std::numeric_limits<float>::quiet_NaN()), "a NaN step never flicks");
|
||||
}
|
||||
|
||||
void TestPulse() {
|
||||
bool active = false;
|
||||
float peak = 0.0f, low = 0.0f;
|
||||
int64_t last_active = -1;
|
||||
for (int64_t t = 0; t <= 200'000'000; t += 1'000'000) {
|
||||
const auto acc = FlickPulse(t, &active);
|
||||
Check(std::fabs(acc[1]) <= wii_remote::kAccelRangeG, "the pulse stays within the accelerometer's range");
|
||||
peak = std::max(peak, acc[1]);
|
||||
low = std::min(low, acc[1]);
|
||||
if (active) {
|
||||
last_active = t;
|
||||
}
|
||||
}
|
||||
Check(last_active >= 50'000'000, "the pulse lasts at least three guest frames");
|
||||
Check(!active, "the pulse ends");
|
||||
Check(peak > 1.0f && low < -3.0f, "the pulse swings up then down, well past rest");
|
||||
const auto rest = FlickPulse(-1, &active);
|
||||
Check(!active && rest[1] == -1.0f, "before it starts the remote is at rest");
|
||||
}
|
||||
|
||||
// A bare hand on the canonical VR wheel: palm position and grasp, as
|
||||
// OpenXRInput::UpdateDriving feeds them.
|
||||
void TestBareHandSteers() {
|
||||
constexpr float dt = 1.0f / 72.0f;
|
||||
SteeringWheel wheel;
|
||||
const float x = SteeringWheel::Radius, y = SteeringWheel::Height, z = SteeringWheel::Depth;
|
||||
const float open = GraspFromJoints(Hand(0.2f, 0.3f, 0.2f));
|
||||
const float closed = GraspFromJoints(Hand(1.0f, 1.3f, 0.6f));
|
||||
WheelState state = wheel.Update({WheelHand{}, WheelHand{x, y, z, open, true}}, true, dt);
|
||||
Check(!state.held[1], "an open hand at the rim does not hold");
|
||||
state = wheel.Update({WheelHand{}, WheelHand{x, y, z, closed, true}}, true, dt);
|
||||
Check(state.held[1], "closing the hand on the rim takes hold");
|
||||
for (int frame = 0; frame < 30; ++frame) {
|
||||
const float angle = -0.02f * static_cast<float>(frame + 1);
|
||||
state = wheel.Update({WheelHand{}, WheelHand{x * std::cos(angle), y + x * std::sin(angle), z, closed, true}},
|
||||
true, dt);
|
||||
}
|
||||
Check(state.held[1] && state.steering > 0.1f, "turning the hand down the right of the rim steers right");
|
||||
for (int frame = 0; frame < 7; ++frame) {
|
||||
state = wheel.Update({WheelHand{}, WheelHand{0.0f, 0.0f, 0.0f, closed, false}}, true, dt);
|
||||
}
|
||||
Check(state.held[1], "a short loss with the latched grasp keeps hold");
|
||||
state = wheel.Update({WheelHand{}, WheelHand{x, y, z, open, true}}, true, dt);
|
||||
Check(!state.held[1], "opening the hand lets go");
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
int main() {
|
||||
TestGrasp();
|
||||
TestLatch();
|
||||
TestPinchGate();
|
||||
TestGestures();
|
||||
TestHandDrivenButtons();
|
||||
TestBareHandRace();
|
||||
TestFlick();
|
||||
TestPulse();
|
||||
TestBareHandSteers();
|
||||
if (g_failures != 0) {
|
||||
std::cerr << g_failures << " check(s) failed\n";
|
||||
return 1;
|
||||
}
|
||||
std::cout << "mkw_vr_hand_tracking_tests passed\n";
|
||||
return 0;
|
||||
}
|
||||
Reference in new issue
Block a user