From 9e7c12e0d458eb725aa3b033a28c1a69d015c50c Mon Sep 17 00:00:00 2001 From: DeeJanuz <45082401+DeeJanuz@users.noreply.github.com> Date: Fri, 2 Oct 2026 11:48:43 -0600 Subject: [PATCH] Pointer helper: split the member functions into files by topic Structure only, no behaviour change; the code moves as it is (git's --color-moved shows it). build.sh compiles ft-pointer.cpp and pointer_*.cpp. - pointer.h: the includes, the helpers more than one file uses (HandGestures, PoseHistory, FramePanel, SendTo, JsonQuote, OverlayList; the three functions are now inline), Clock, Frame, and Pointer. - ft-pointer.cpp: the header comment, setup (Init, ApplyConfig, and the helpers only it uses), the short per-frame sections, and main(). - pointer_input.cpp: waking, the mouse's buttons, the relay's commands. - pointer_place.cpp: panel placement for layouts. - pointer_gaze.cpp: gaze mode, the held-back press, the click. - pointer_holds.cpp: holds (hands, gaze precision buttons). - pointer_keys.cpp: keyboard clicks. - pointer_cursor.cpp: recenter, head follow, slow work, the cursor. Helpers only one file uses stay in its anonymous namespace. Each new file says that "the top" in its comments is ft-pointer.cpp's header comment. hands/README.md and update-check.py now name the files that hold the code they point at. Co-Authored-By: Claude Opus 5.5 --- hands/README.md | 4 +- pointer/helper/build.sh | 2 +- pointer/helper/ft-pointer.cpp | 1885 +---------------------------- pointer/helper/pointer.h | 478 ++++++++ pointer/helper/pointer_cursor.cpp | 474 ++++++++ pointer/helper/pointer_gaze.cpp | 91 ++ pointer/helper/pointer_holds.cpp | 202 ++++ pointer/helper/pointer_input.cpp | 390 ++++++ pointer/helper/pointer_keys.cpp | 113 ++ pointer/helper/pointer_place.cpp | 168 +++ scripts/update-check.py | 4 +- 11 files changed, 1922 insertions(+), 1889 deletions(-) create mode 100644 pointer/helper/pointer.h create mode 100644 pointer/helper/pointer_cursor.cpp create mode 100644 pointer/helper/pointer_gaze.cpp create mode 100644 pointer/helper/pointer_holds.cpp create mode 100644 pointer/helper/pointer_input.cpp create mode 100644 pointer/helper/pointer_keys.cpp create mode 100644 pointer/helper/pointer_place.cpp diff --git a/hands/README.md b/hands/README.md index cfbca6d..42f322f 100644 --- a/hands/README.md +++ b/hands/README.md @@ -49,7 +49,7 @@ Files, all in `/run/user/UID/frametop-hands/` (private to the user; not `/run/us | --- | --- | --- | --- | | `cam-ring` | ft-camd | `camd/fhring.h` | ft-hands, `tools/ring.py` | | `hands` | ft-hands | `include/fh_hands.h` | ft-screens (`screens/handcut.cpp`) | -| `gestures` | ft-hands | `include/fh_gestures.h` | the pointer helper (`pointer/helper/ft-pointer.cpp`), `tools/watch_gestures.py` | +| `gestures` | ft-hands | `include/fh_gestures.h` | the pointer helper (`pointer/helper/pointer_holds.cpp`), `tools/watch_gestures.py` | The source keeps the `fh_` names and magic strings of frame-hands, the project it started as, so recordings made with it still work. @@ -156,7 +156,7 @@ ft-hands detects a pinch per hand (`track/pinch.h`) and publishes it to the gest ## Pinches and grips in the pointer -With `POINTER_HANDS=1`, the pointer helper (`pointer/helper/ft-pointer.cpp`) reads the gestures file every frame. It's off by default. +With `POINTER_HANDS=1`, the pointer helper (`pointer/helper/pointer_holds.cpp`) reads the gestures file every frame. It's off by default. - **Pinch to click.** In gaze mode a pinch works like the mouse's press: the pointer stops where the gaze put it, and the click comes when the pinch opens, where the pointer is then. A quick tap clicks where you looked. Held, the pinching hand moves the pointer to correct the gaze, and the correction is a lesson for the gaze tracker, as with the mouse. - **Without gaze mode,** a pinch is a real press, like the mouse's button: pressed when it closes, released when it opens, and while it's held the hand drags the pointer. A tap is still a click where the pointer is. diff --git a/pointer/helper/build.sh b/pointer/helper/build.sh index 59ff91f..2a9d2a4 100755 --- a/pointer/helper/build.sh +++ b/pointer/helper/build.sh @@ -5,5 +5,5 @@ root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd) "$root/scripts/sync.sh" >/dev/null exec "$root/scripts/frame.sh" -C pointer/helper 'set -e; mkdir -p build g++ -std=c++17 -O2 -Wall -Wno-unused-parameter -I/opt/steamvr/tools/hellovr_vulkan_linux/src/openvr/headers -I../common \ - -o build/ft-pointer ft-pointer.cpp -L/opt/steamvr/bin/linuxarm64 -lopenvr_api -Wl,-rpath,/opt/steamvr/bin/linuxarm64 -lpthread + -o build/ft-pointer ft-pointer.cpp pointer_*.cpp -L/opt/steamvr/bin/linuxarm64 -lopenvr_api -Wl,-rpath,/opt/steamvr/bin/linuxarm64 -lpthread echo "built build/ft-pointer"' diff --git a/pointer/helper/ft-pointer.cpp b/pointer/helper/ft-pointer.cpp index e29d542..90f586c 100644 --- a/pointer/helper/ft-pointer.cpp +++ b/pointer/helper/ft-pointer.cpp @@ -297,122 +297,10 @@ // POINTER_GRIP_GAIN (1), POINTER_GRIP_BELOW (0.35 m), POINTER_PINCH_TYPING (1 s): hands, above. // POINTER_GAZE_MOUSE (precision), POINTER_HEAD_DEADZONE (0.5 deg), POINTER_KEY_TAP (0.25 s), // POINTER_ROLE (right): gaze precision and keyboard clicks, above. -#include - -#include "pointer_config.h" -#include "vrbuttons.h" -#include "vrmath.h" - -extern "C" { -#include "../../hands/include/fh_gestures.h" -} - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -#include -#include - -#include -#include -#include -#include -#include +#include "pointer.h" namespace { -using namespace md; - -// Hand gestures from ft-hands (see "Hands" at the top): /run/user/UID/frametop-hands/gestures, -// mapped read-only. Version 1 files have pinches only; their grips read as zero. -class HandGestures { -public: - ~HandGestures() { Close(); } - // A consistent copy of the file (its sequence lock), if it's there. Opens it, and - // checks it's still the same file, at most once a second. - bool Read(fh_gestures_t &out) { - const auto now = std::chrono::steady_clock::now(); - if (now - checked_ > std::chrono::seconds(1)) { - checked_ = now; - const std::string path = "/run/user/" + std::to_string(getuid()) + "/frametop-hands/gestures"; - struct stat st; - if (stat(path.c_str(), &st) != 0 || size_t(st.st_size) != len_ || st.st_ino != ino_) { - Close(); - const int fd = open(path.c_str(), O_RDONLY | O_CLOEXEC | O_NOFOLLOW); - if (fd >= 0 && fstat(fd, &st) == 0 && size_t(st.st_size) >= offsetof(fh_gestures_t, grip)) { - void *m = mmap(nullptr, size_t(st.st_size), PROT_READ, MAP_SHARED, fd, 0); - if (m != MAP_FAILED) map_ = m, len_ = size_t(st.st_size), ino_ = st.st_ino, ++opens; - } - if (fd >= 0) close(fd); - } - } - if (!map_) return false; - const auto *g = static_cast(map_); - if (std::memcmp(g->magic, FH_GESTURES_MAGIC, 8) != 0) return false; - const size_t n = std::min(len_, sizeof out); - for (int tries = 0; tries < 3; ++tries) { - const uint64_t seq = __atomic_load_n(&g->seq, __ATOMIC_ACQUIRE); - if (seq & 1) continue; - std::memset(&out, 0, sizeof out); - std::memcpy(&out, map_, n); - __atomic_thread_fence(__ATOMIC_ACQUIRE); - if (__atomic_load_n(&g->seq, __ATOMIC_RELAXED) != seq) continue; - if (out.version < 2) std::memset(out.grip, 0, sizeof out.grip); - return true; - } - return false; - } - int opens = 0; // a new file: the counters start over - -private: - void Close() { - if (map_) munmap(map_, len_); - map_ = nullptr, len_ = 0, ino_ = 0; - } - void *map_ = nullptr; - size_t len_ = 0; - ino_t ino_ = 0; - std::chrono::steady_clock::time_point checked_{}; -}; - -// The HMD's recent poses (standing universe), to turn the gestures' head-frame points into -// the room as the head was when the cameras took them. -class PoseHistory { -public: - void Add(std::chrono::steady_clock::time_point t, const vr::HmdMatrix34_t &m) { - poses_.push_back({t, m}); - while (poses_.size() > 128) poses_.pop_front(); // about a second - } - // The pose at CLOCK_MONOTONIC time t_ns (steady_clock's), or the nearest kept. - bool At(uint64_t t_ns, vr::HmdMatrix34_t &out) const { - if (poses_.empty()) return false; - const std::chrono::steady_clock::time_point t{std::chrono::nanoseconds(t_ns)}; - const auto *best = &poses_.front(); - for (const auto &p : poses_) - if (std::chrono::abs(p.first - t) < std::chrono::abs(best->first - t)) best = &p; - out = best->second; - return true; - } - -private: - std::deque> poses_; -}; - int AbstractSocket(const char *name, bool bindIt) { const int fd = socket(AF_UNIX, SOCK_DGRAM | SOCK_CLOEXEC | SOCK_NONBLOCK, 0); if (bindIt) { @@ -437,166 +325,6 @@ std::string ExeDir() { return p.substr(0, p.rfind('/')); } -// One of ft-screens' panels showing a desktop: a screen (frametop.screen.N), a floating -// window (frametop.float.N), or a floating window's popup (frametop.float.N.sub.K), not a -// control of theirs. -bool FramePanel(const std::string &key) { - for (const char *prefix : {"frametop.screen.", "frametop.float."}) { - if (key.rfind(prefix, 0) != 0) continue; - const std::string rest = key.substr(std::strlen(prefix)); - const size_t dot = rest.find('.'); - return dot == std::string::npos || rest.compare(dot, 5, ".sub.") == 0; - } - return false; -} - -void SendTo(int fd, const char *name, const std::string &msg) { - sockaddr_un addr{}; - addr.sun_family = AF_UNIX; - std::memcpy(addr.sun_path + 1, name, std::strlen(name)); - const socklen_t len = offsetof(sockaddr_un, sun_path) + 1 + std::strlen(name); - sendto(fd, msg.data(), msg.size(), 0, reinterpret_cast(&addr), len); -} - -// JSON string literal (names come from other apps). -std::string JsonQuote(const std::string &s) { - std::string out = "\""; - for (const unsigned char c : s) { - if (c == '"' || c == '\\') out += '\\', out += char(c); - else if (c < 0x20) { - char esc[8]; - std::snprintf(esc, sizeof esc, "\\u%04x", c); - out += esc; - } else out += char(c); - } - return out + "\""; -} - -// Overlay keys, refreshed in the background from `vrcmd --overlays` (OpenVR has no -// public call to enumerate other apps' overlays). Hidden ones are listed too: the -// window controls under a floating panel only appear while something hovers the -// panel, and the cursor has to find them the moment they do, not a second later. -// Paused while the pointer is off: each vrcmd run connects to SteamVR as a new app, and a new -// app every second kept SteamVR (and the headset's displays) from going to standby. -// "overlays" requests (Frametop Input Settings' Ignored panels page) refresh the list even -// while paused, and are answered from this thread once it's fresh. -class OverlayList { -public: - void Start() { - out_ = socket(AF_UNIX, SOCK_DGRAM | SOCK_CLOEXEC, 0); - thread_ = std::thread([this] { - while (running_) { - if (!paused_ || requested_) Refresh(); - // Wait a second, or less when the pointer wakes (refresh right away then). - for (int i = 0; i < 10 && running_; ++i) { - const bool wasPaused = paused_; - std::this_thread::sleep_for(std::chrono::milliseconds(100)); - if ((wasPaused && !paused_) || requested_) break; - } - } - }); - } - void SetPaused(bool paused) { paused_ = paused; } - void Stop() { - running_ = false; - if (thread_.joinable()) thread_.join(); - } - std::vector Keys() { - std::lock_guard guard(lock_); - std::vector keys; - for (const auto &e : entries_) keys.push_back(e.key); - return keys; - } - // Answer `to` with {"t":"overlays","list":[{"key","name","visible"}...]} after the next refresh. - void Request(const sockaddr_un &to, socklen_t len) { - if (len <= offsetof(sockaddr_un, sun_path)) return; - std::lock_guard guard(lock_); - if (waiting_.size() < 8) waiting_.push_back({to, len}); - requested_ = true; - } - -private: - struct Entry { - std::string key, name; - bool visible; - }; - void Refresh() { - requested_ = false; - FILE *p = popen("LD_LIBRARY_PATH=/opt/steamvr/bin/linuxarm64 /opt/steamvr/bin/linuxarm64/vrcmd --overlays 2>/dev/null", "r"); - if (!p) return; - std::vector entries; - char line[1024]; - while (std::fgets(line, sizeof line, p)) { - // 'key' -- 'name', WxH visible VROverlayType_... - if (line[0] != '\'') continue; - const char *end = std::strchr(line + 1, '\''); - if (!end) continue; - const std::string key(line + 1, size_t(end - (line + 1))); - const std::string rest(end); - if (rest.find("Thumbnail") != std::string::npos || rest.find("Subview") != std::string::npos) continue; - if (key.rfind("system.pointer", 0) == 0 || key.rfind("system.cursor", 0) == 0 || - key.rfind("frametop.pointer", 0) == 0 || key.rfind("frametop.guide", 0) == 0 || - key == "frametop.gazepanel" || // the gaze calibration panel, fixed to the headset - key == "frametop.catcher" || // ft-screens' release catcher: only on a laser mid-drag - key == "system.HeadsetView" || key == "system.toast") - continue; - // The name can hold quotes; it ends at the last "', " (the size and state follow). - const auto nameAt = rest.find("-- '"), nameEnd = rest.rfind("', "); - const std::string name = - nameAt != std::string::npos && nameEnd > nameAt + 3 ? rest.substr(nameAt + 4, nameEnd - nameAt - 4) : key; - entries.push_back({key, name, rest.find(" not_visible ") == std::string::npos}); - } - pclose(p); - std::vector> waiting; - { - std::lock_guard guard(lock_); - entries_ = std::move(entries); - waiting.swap(waiting_); - } - if (waiting.empty()) return; - std::string msg = "{\"t\":\"overlays\",\"list\":["; - for (size_t i = 0; i < entries_.size(); ++i) - msg += std::string(i ? "," : "") + "{\"key\":" + JsonQuote(entries_[i].key) + ",\"name\":" + - JsonQuote(entries_[i].name) + ",\"visible\":" + (entries_[i].visible ? "true" : "false") + "}"; - msg += "]}"; - for (const auto &[to, len] : waiting) - sendto(out_, msg.data(), msg.size(), MSG_DONTWAIT, reinterpret_cast(&to), len); - } - - std::thread thread_; - int out_ = -1; - std::atomic paused_{false}; - std::atomic running_{true}; - std::atomic requested_{false}; - std::mutex lock_; - std::vector entries_; // written only by the thread; the lock guards readers - std::vector> waiting_; -}; - -bool Ignored(const std::vector &patterns, const std::string &key) { - for (const auto &p : patterns) - if (fnmatch(p.c_str(), key.c_str(), FNM_NOESCAPE) == 0) return true; - return false; -} - -vr::HmdMatrix34_t Billboard(Vec3 at, Vec3 eye) { - // Overlay faces +Z; point +Z at the eye, keep +Y roughly up. - const Vec3 z = Normalize(eye - at); - const Vec3 x = Normalize(Cross({0, 1, 0}, z)); - const Vec3 y = Cross(z, x); - vr::HmdMatrix34_t m{}; - const Vec3 cols[3] = {x, y, z}; - for (int c = 0; c < 3; ++c) { - m.m[0][c] = float(cols[c].x); - m.m[1][c] = float(cols[c].y); - m.m[2][c] = float(cols[c].z); - } - m.m[0][3] = float(at.x); - m.m[1][3] = float(at.y); - m.m[2][3] = float(at.z); - return m; -} - std::vector DotTexture(int size) { // White dot with a dark rim, soft edge, transparent outside. std::vector px(size * size * 4, 0); @@ -614,273 +342,8 @@ std::vector DotTexture(int size) { return px; } -// Unit vector v, turned toward unit vector `center` until it's at most maxRad from it. -Vec3 PullWithin(Vec3 v, Vec3 center, double maxRad) { - if (std::acos(std::clamp(Dot(v, center), -1.0, 1.0)) <= maxRad) return v; - Vec3 axis = Cross(center, v); - if (Length(axis) < 1e-9) axis = Cross(center, {0, 1, 0}); // opposite: any perpendicular - return RotateAbout(center, Normalize(axis), maxRad); -} - -// Unit direction with its pitch limited to +-maxDeg (AimBasis needs it off vertical). -Vec3 LimitPitch(Vec3 d, double maxDeg) { - const double pitch = std::clamp(std::asin(std::clamp(d.y, -1.0, 1.0)) * 180 / M_PI, -maxDeg, maxDeg); - return Direction(std::atan2(-d.x, -d.z) * 180 / M_PI, pitch); -} - -// SteamVR Settings page (see the top): the laser starts this far from the eye, the dot is -// drawn this far out, and the laser-catching dot sits this far out (metres). -constexpr double SETTINGS_ORIGIN = 0.25, SETTINGS_DOT = 0.6, SETTINGS_CATCHER = 8.0; - -// Whether the line of sight from `eye` along `d` crosses the SteamVR Settings page, drawn by -// the dashboard's scene-graph panel (valve.steam.gamepadui.frame.menu.N, transform t); see -// "SteamVR Settings" at the top. The page has no size in OpenVR, so this is its area as -// measured on the Frame, generously: in metres from the panel's origin, which is near the -// page's left edge, it ran from about -0.35 (the sidebar) to 1.15 across and +-0.4 up and -// down, with the transform scaled 0.369. Kept in the transform's units so it scales with it. -bool OnSettingsPage(const vr::HmdMatrix34_t &t, Vec3 eye, Vec3 d) { - const Vec3 c = Position(t), x{t.m[0][0], t.m[1][0], t.m[2][0]}, y{t.m[0][1], t.m[1][1], t.m[2][1]}, - z{t.m[0][2], t.m[1][2], t.m[2][2]}; - const double sx = Dot(x, x), sy = Dot(y, y), denom = Dot(d, z); - if (sx < 1e-9 || sy < 1e-9 || std::fabs(denom) < 1e-6) return false; - const double along = Dot(c - eye, z) / denom; - if (along <= 0) return false; - const Vec3 off = eye + d * along - c; - const double u = Dot(off, x) / sx, v = Dot(off, y) / sy; // in the transform's units - return u >= -1.35 && u <= 3.4 && std::fabs(v) <= 1.25; -} - -void SleepMs(int ms) { std::this_thread::sleep_for(std::chrono::milliseconds(ms)); } - } // namespace -// Placement speeds (see "Placement" at the top). -constexpr double kPlaceDegPerSec = 60; // tested: 40 deg/s is applied exactly - -using Clock = std::chrono::steady_clock; - -// One frame's poses and time: ReadFrame reads them after the relay's commands, and the sections -// after it share them. handOk: PinchesAndGrips read the hand gestures (for HandsStopped). -struct Frame { - vr::TrackedDevicePose_t all[vr::k_unMaxTrackedDeviceCount]; - vr::TrackedDevicePose_t hmdRaw; - Clock::time_point tnow; - Vec3 eye; - bool handOk = false; - const vr::TrackedDevicePose_t &Hmd() const { return all[0]; } -}; - -// Everything the pointer keeps: its settings, what Init sets up, and the state the main loop -// carries from one frame to the next. -struct Pointer { - PointerConfig cfg; - // Head follow (see the top). followConf is POINTER_FOLLOW as last read: a reload only - // overrides a "follow" command when the setting itself changed. - bool follow = false, followConf = false, followReset = true; - // Gaze mode (see the top); gazeConf is POINTER_GAZE as last read, like followConf. - bool gazeOn = false, gazeConf = false; - // Set in Init, after VR_Init. - vr::IVRSystem *sys = nullptr; - vr::IVROverlay *overlay = nullptr; - // The dot, the marker, and laser mode's overlay, created in Init. - vr::VROverlayHandle_t cursor = vr::k_ulOverlayHandleInvalid; - vr::VROverlayHandle_t marker = vr::k_ulOverlayHandleInvalid; - vr::VROverlayHandle_t laserMode = vr::k_ulOverlayHandleInvalid; - bool laserModeShown = false; - int in = -1, out = -1; // @ft_pointer_helper, and the socket we send from (opened in Init) - ControllerButtons controllerButtons; // Frame controller buttons (vrbuttons.h) - OverlayList overlays; - std::map handles; - std::map sceneGraph; // no texture: plane test instead of ComputeOverlayIntersection - std::map visible; // refreshed every 50 ms - Clock::time_point lastVisible{}; // set in Init - // The plane of the last panel the cursor was on, and the last point on it (panel edges). - Vec3 edgePoint, edgeNormal, edgeLast; - std::string edgeKey; - - bool active = false, recenter = false, anchored = false; - Clock::time_point lastMouse{}, claimAt{}, claimRelease{}, wokeAt{}, noWakeUntil{}; - bool claimPending = false, claimHeld = false; - // Last used wins: since when each controller has been moving (zero: it isn't). - Clock::time_point movingSince[vr::k_unMaxTrackedDeviceCount] = {}; - // Tilt mode (see top of file). - bool leftHeld = false, tilting = false, tiltStart = false, swallowedRight = false; - // The mouse's buttons in gaze mode (see the top): leftDown, rightDown as the relay last said; - // aimRight, the held-back press is the right button's; chordDrag, a drag the right button - // began during the left's held-back press (the first release drops it, the other's is nothing). - bool leftDown = false, rightDown = false, aimRight = false, chordDrag = false; - bool ignoreLeftUp = false; - // The keyboard clicks' keys as the relay last said, and a tilt gaze_right holds during a - // keyboard drag (see the top): the head turns the panel, from where it was (keyTiltYaw, keyTiltPitch). - bool keyLeftDown = false, keyRightDown = false, keyTilting = false; - double keyTiltYaw = 0, keyTiltPitch = 0; - double tiltYaw = 0, tiltPitch = 0; - double dragDistance = 0, lastDistance = 1.5; // drag lock: distance from the anchor at the press - bool onVrSettings = false; // the cursor is on the SteamVR Settings page (kept while dragging) - bool catcherHidesHit = false; // the laser-catching dot hides SteamVR's hit dot (Settings page) - Clock::time_point dropHoldUntil{}; // after a left release: keep the drag pose this long - bool debug = false; - std::string lastHit; - // The ft-screens panel the left button was pressed on, and where it was then (see the top). - std::string pressKey; - vr::HmdMatrix34_t pressPose{}; - Clock::time_point lastDebug{}; // set in Init - vr::VROverlayHandle_t systemPointer = vr::k_ulOverlayHandleInvalid; // found in Init - Vec3 pivot, tiltOrigin, lastPoint, lastOrigin, lastAim{0, 0, -1}; - Basis tiltBasis{}; - bool headsetOff = false; // nobody is wearing the headset (see the main loop) - Vec3 anchor; - double yaw = 0, pitch = 0; - Vec3 followRef{0, 0, -1}; // head follow's reference direction (see the top) - Clock::time_point followAt{}; // its last update, for the easing (set in Init) - bool following = false; // past the leash: easing toward the head - double followLag = 0; // radians the reference trails the head - std::chrono::steady_clock::time_point leashOutSince{}; // head past the leash since (delay) - // Gaze mode (see the top). - struct Gaze { - double hy = 0, hp = 0, rhy = 0, rhp = 0; // corrected, and raw - Clock::time_point at{}; - } gz; - bool gazeOwns = true; // the pointer follows the gaze; false: the mouse has it - bool nudging = false; // the mouse took it from the gaze: the next click may be a lesson - double nudgeRawHy = 0, nudgeRawHp = 0, nudgeMoved = 0; - vr::HmdMatrix34_t nudgeHead{}, lastHead{}; - bool haveHead = false, havePoint = false; - Clock::time_point nudgeAt{}, retakeSince{}; - // A held-back press (see the top): aimHeld while the button is down; then the click - // (clickPress at the end of the next frame, clickRelease 40 ms later). gazeBack: give - // the gaze the pointer again when the press or click is over. - vr::TrackedDeviceIndex_t ours = vr::k_unTrackedDeviceIndexInvalid; - bool aimHeld = false, clickPress = false, clickRelease = false, gazeBack = false; - bool aimHand = false; // the held-back press is a hold's (below): it never turns into a real press - Clock::time_point aimSince{}, clickReleaseAt{}; - // Holds (see "Gaze precision" and "Hands" at the top): a pinch or grip, or a gaze - // precision or gaze drag button, held now. What steers the pointer meanwhile (a hand, or - // the mouse through its own moves), from where it pointed when the hold began (for a - // hand: seen from the eye then, in the room), and the pointer then. - HandGestures handFile; - PoseHistory poses; - enum class Src { None, Hand, Mouse, Head }; - struct Hold { - Src src = Src::None; - int side = -1; // a hand's side (0 left, 1 right) - bool grip = false; // pressed at once and dragging (a grip, gaze drag), not a click on release - bool engaged = false; // past the dead zone - bool pressed = false; // a real press went out (a grip or gaze drag, or a pinch without gaze mode) - bool promote = false; // held still for POINTER_GAZE_HOLD, it becomes a real press (keyboard clicks) - bool right = false; // the right button (gaze_right) - bool keyDrag = false; // gaze_right pressed while gaze_left aimed: a left press, either key ends it - double pressYaw = 0, pressPitch = 0; // the pointer at the press (a quick tap clicks there) - Vec3 origin; - double refYaw = 0, refPitch = 0, startYaw = 0, startPitch = 0, lastYaw = 0, lastPitch = 0; - } hold; - // "precision|gazedrag 1|0" from the relay, done in the frame (see Holds). - struct DevicePress { - std::string source; - bool drag, down; - }; - std::vector devicePresses; - // "gazekey left|right 1|0" from the relay (keyboard clicks), done in the frame. - struct KeyPress { - bool right, down; - }; - std::vector keyPresses; - uint32_t seenBegins[2][2] = {}, seenEnds[2][2] = {}; // [pinch, grip][side], as last read - bool handBaseline = false; - int handOpens = 0; - uint64_t handSeq = 0, handPublished = 0; - Clock::time_point handUsed{}; // a gesture began then (keeps the pointer, like gaze mode) - Clock::time_point lastTyping{}; // the relay's last "typing": a key on a keyboard - // Gaze mode outside games, and its dot (see the top): lastMove/lastHeld/pulseAt. - bool inGame = false, gazeAwake = false; - Clock::time_point inGameAt{}, gazeAwakeAt{}; - Clock::time_point lastMove{}, lastHeld{}, pulseAt{}; - // The left button, as sent to the driver; pressRight: the next press is the right button - // instead (gaze_right), heldButton: the one pressed. - bool pressRight = false; - std::string heldButton = "trigger"; - bool confirmLesson = false; // a keyboard click's quick tap: a lesson with no correction (see the top) - // The gaze calibration panel is up until then ("calpanel 1"; see the top); calOpened: it just - // came up, so a press in progress ends without a click. - Clock::time_point calPanelUntil{}; - bool calOpened = false; - Clock::time_point lastSlow{}; // set in Init - - void ApplyConfig(); - void Init(); - void Wake(Clock::time_point t); - void PressLeft(); - void ReleaseLeft(); - bool CanAim(); - void AimStart(bool right); - void ChordDrop(); - void LeftButton(bool down); - bool RightButton(bool down); - bool MouseMoveHeld(); - void SendPose(Vec3 originStanding, const Basis &b); - std::pair HeadPos(); - void Borrow(); - void GiveBack(); - std::string FindPanel(const char *key, Panel &p, Vec3 &eye); - void GrabProbe(const char *key); - std::string Place(const char *key, Vec3 target, const Basis &bt, double grabBelow); - // Nearest's answer: the nearest overlay along a ray. - struct Hit { - double along = 1e9; - std::string key; - bool scene = false; - Vec3 point, normal; - }; - // Where the cursor is this frame (FindCursor), for DrawDot and SendRay. - struct Spot { - bool dragging = false; - Vec3 dir, point; - double best = 1e9, distance = 0; - bool onEdge = false, occluded = false, onScene = false, onPanel = false; - }; - - // The main loop's sections, in the order main calls them, and what they share. - void HeadsetOff(); - void GazeAwake(); - void RelayCommands(); - void Command(const char *buf, const sockaddr_un &sender, socklen_t senderLen); - void ReadFrame(Frame &frame); - void ClaimPulse(const Frame &frame); - void PauseOverlayList(); - void LaserMode(); - void HandRole(const Frame &frame); - void LastUsedWins(const Frame &frame); - void Recenter(const Frame &frame); - void GazeMode(const Frame &frame); - void Hands(const Frame &frame); - bool HandAngles(const float p[3], uint64_t t_ns, const Vec3 &from, double &hy, double &hp); - void EndHold(bool lost); - void StartHold(bool press, Clock::time_point tnow); - void Steer(double hy, double hp, double deadzone, double gain, Clock::time_point tnow); - void BeginHold(int side, bool grip, const fh_pinch_t &g, Clock::time_point tnow); - void GazePrecisionButtons(const Frame &frame); - bool HeadAngles(const vr::TrackedDevicePose_t &hmd, double &hy, double &hp); - void KeyboardClicks(const Frame &frame); - void CalPanelOpened(); - void HeadSteer(const Frame &frame); - void PinchesAndGrips(Frame &frame); - void HandsStopped(const Frame &frame); - void MarkHeld(const Frame &frame); - void HeadFollow(const Frame &frame); - void SlowWork(); - void Cursor(const Frame &frame); - void Tilt(const Frame &frame); - Hit Nearest(Vec3 from, Vec3 d); - Spot FindCursor(const Frame &frame); - void DrawDot(const Frame &frame, const Spot &spot); - void SendRay(const Frame &frame, const Spot &spot); - void HeldBackPress(const Frame &frame); - void KeyboardClickHold(const Frame &frame); - void Click(const Frame &frame); - void PollControllerButtons(); - bool SteamVRQuit(); -}; - // After cfg.Load: a changed POINTER_FOLLOW or POINTER_GAZE turns head follow or gaze mode on or off. void Pointer::ApplyConfig() { if (cfg.follow != followConf) follow = followConf = cfg.follow, followReset = true; @@ -951,326 +414,6 @@ void Pointer::Init() { std::fflush(stdout); } -void Pointer::Wake(Clock::time_point t) { - if (t < noWakeUntil || headsetOff) return; - wokeAt = t; - active = true; - recenter = true; - SendTo(out, "ft_pointer", "role " + cfg.role); // POINTER_ROLE, before it takes it - SendTo(out, "ft_pointer", "show"); - claimPending = true; // take the laser without clicking, once SteamVR has bound the device - claimAt = t + std::chrono::milliseconds(300); -} - -void Pointer::PressLeft() { - // ft-screens sends the keyboard to the panel clicked last; it sees clicks on - // its own screens, but only we know when one lands on another panel. - SendTo(out, "ft_screens", "click " + (lastHit.empty() ? std::string("-") : lastHit)); - // A click after nudging the gaze-placed pointer: the nudge is a lesson, or past - // POINTER_GAZE_NUDGE_MAX, a quick check (see the top). - if (gazeOn && nudging && !gazeOwns && havePoint && Clock::now() - nudgeAt < std::chrono::seconds(10) && - (confirmLesson || nudgeMoved >= 0.2)) { - const Vec3 d = RotateInverse(nudgeHead, Normalize(lastPoint - Position(nudgeHead))); - const double ty = std::atan2(-d.x, -d.z) * 180 / M_PI, tp = std::asin(std::clamp(d.y, -1.0, 1.0)) * 180 / M_PI; - const double off = std::hypot(std::remainder(ty - nudgeRawHy, 360.0), tp - nudgeRawHp); - char msg[160]; - if (off <= cfg.gazeNudgeMax) - std::snprintf(msg, sizeof msg, "lesson %.3f %.3f %.3f %.3f", nudgeRawHy, nudgeRawHp, ty, tp); - else - std::snprintf(msg, sizeof msg, "recheck %.0f", off); - SendTo(out, "ft_gazed", msg); - if (debug) std::printf("gaze %s (nudged %.2f deg, off %.2f)\n", msg, nudgeMoved, off); - if (debug) std::fflush(stdout); - } - nudging = confirmLesson = false; - leftHeld = true; - dragDistance = lastDistance; - pressKey.clear(); - if (FramePanel(lastHit)) { - vr::ETrackingUniverseOrigin uo; - auto it = handles.find(lastHit); - if (it != handles.end() && - overlay->GetOverlayTransformAbsolute(it->second, &uo, &pressPose) == vr::VROverlayError_None) - pressKey = lastHit; - } - tiltYaw = tiltPitch = 0; // a new drag starts untilted - dropHoldUntil = {}; - pulseAt = Clock::now(); - heldButton = pressRight ? "b" : "trigger"; - pressRight = false; - SendTo(out, "ft_pointer", "btn " + heldButton + " 1"); -} - -void Pointer::ReleaseLeft() { - leftHeld = false; - tilting = false; - // Hold the drag pose (tilt, frozen distance) while SteamVR finishes the drop. - dropHoldUntil = Clock::now() + std::chrono::milliseconds(500); - SendTo(out, "ft_pointer", "btn " + heldButton + " 0"); - // ft-screens releases a button held on its screens in KWin even when SteamVR hands - // the release to some other overlay (its catcher usually gets it; this is the backstop). - SendTo(out, "ft_screens", "up"); - if (gazeBack) gazeOwns = true, gazeBack = false; -} - -// The left button, from the relay's "btn trigger", with gaze mode's held-back press (see -// the top). -bool Pointer::CanAim() { - return gazeOn && cfg.gazeMousePrecision && gazeOwns && !aimHeld && !clickPress && !clickRelease && - hold.src == Src::None; -} - -// Hold the press back: the pointer stops where the gaze put it. -void Pointer::AimStart(bool right) { - gazeOwns = false; - nudging = haveHead && Clock::now() - gz.at < std::chrono::milliseconds(200); - nudgeRawHy = gz.rhy, nudgeRawHp = gz.rhp, nudgeHead = lastHead; - nudgeAt = aimSince = Clock::now(), nudgeMoved = 0; - aimHeld = true, aimRight = right; -} - -// A drag the right button began (chordDrag): it lasts while either button is held. -void Pointer::ChordDrop() { - chordDrag = false; - if (leftHeld) ReleaseLeft(); - if (debug) std::printf("mouse drag dropped\n"); - if (debug) std::fflush(stdout); -} - -void Pointer::LeftButton(bool down) { - leftDown = down; - if (down) { - if (aimHeld && aimRight && !aimHand) { - ignoreLeftUp = true; // the right's press is held back: the left does nothing - return; - } - if (CanAim()) { - AimStart(false); - return; - } - PressLeft(); - return; - } - if (ignoreLeftUp) { - ignoreLeftUp = false; - return; - } - if (chordDrag) { - if (!rightDown) ChordDrop(); // otherwise the right holds it (tilting, maybe) - return; - } - if (aimHeld && !aimHand && !aimRight) { - aimHeld = false; - clickPress = true; // after this frame's pose, so it lands where the pointer was moved to - gazeBack = nudgeMoved < 0.2; - return; - } - if (leftHeld) ReleaseLeft(); -} - -// The right button (see the top); false: not taken here (a tilt, or passed on as it is). -bool Pointer::RightButton(bool down) { - rightDown = down; - if (down) { - if (aimHeld && !aimHand && !aimRight) { - // During the left's held-back press: press the left where the pointer is now (the - // correction is a lesson), and the mouse drags. - aimHeld = false; - chordDrag = gazeBack = true; - PressLeft(); - if (debug) std::printf("mouse drag began (right during left)\n"); - if (debug) std::fflush(stdout); - return true; - } - if (CanAim() && !leftHeld) { - AimStart(true); - return true; - } - return false; - } - if (chordDrag) { - if (leftDown) return !swallowedRight; // the left holds it; a tilt's release ends the tilt - tilting = swallowedRight = false; - ChordDrop(); - return true; - } - if (aimHeld && !aimHand && aimRight) { - aimHeld = aimRight = false; - pressRight = clickPress = true; // the right click, where the pointer was moved to - gazeBack = nudgeMoved < 0.2; - return true; - } - if (leftHeld && heldButton == "b") { // its press, held still into a real one - ReleaseLeft(); - return true; - } - return false; -} - -// POINTER_GAZE_MOUSE_MOVE=held (see the top): the gaze is fresh and nothing is pressed, so a -// mouse move doesn't move the pointer. -bool Pointer::MouseMoveHeld() { - return gazeOn && cfg.gazeMouseHeld && !inGame && !headsetOff && Clock::now() - gz.at < std::chrono::seconds(1) && - !aimHeld && !leftHeld && !tilting && hold.src == Src::None && !clickPress && !clickRelease; -} - -// --- Panel placement (see "Placement" at the top of the file) --- -// Device pose, given in the standing universe, sent to the driver in raw space. -void Pointer::SendPose(Vec3 originStanding, const Basis &b) { - vr::TrackedDevicePose_t s, r; - sys->GetDeviceToAbsoluteTrackingPose(vr::TrackingUniverseStanding, 0, &s, 1); - sys->GetDeviceToAbsoluteTrackingPose(vr::TrackingUniverseRawAndUncalibrated, 0, &r, 1); - const auto &S = s.mDeviceToAbsoluteTracking, &R = r.mDeviceToAbsoluteTracking; - auto toRaw = [&](Vec3 v) { return Rotate(R, RotateInverse(S, v)); }; - const Vec3 o = Position(R) + toRaw(originStanding - Position(S)); - double q[4]; - BasisQuat({toRaw(b.x), toRaw(b.y), toRaw(b.z)}, q); - char msg[200]; - std::snprintf(msg, sizeof msg, "posq %.5f %.5f %.5f %.6f %.6f %.6f %.6f", o.x, o.y, o.z, q[0], q[1], q[2], q[3]); - SendTo(out, "ft_pointer", msg); -} - -std::pair Pointer::HeadPos() { - vr::TrackedDevicePose_t s; - sys->GetDeviceToAbsoluteTrackingPose(vr::TrackingUniverseStanding, 0, &s, 1); - return std::make_pair(s.bPoseIsValid, Position(s.mDeviceToAbsoluteTracking)); -} - -// Borrow the device and the laser for a placement: connect, claim, laser mode on. -void Pointer::Borrow() { - SendTo(out, "ft_pointer", "show"); - overlay->ShowOverlay(laserMode); - overlay->HideOverlay(cursor); - overlay->HideOverlay(marker); - SleepMs(active ? 50 : 400); // a fresh connect needs SteamVR to bind the device - SendTo(out, "ft_pointer", "btn a 1"); - SleepMs(60); - SendTo(out, "ft_pointer", "btn a 0"); -} - -void Pointer::GiveBack() { - if (!active) { - SendTo(out, "ft_pointer", "hide"); - overlay->HideOverlay(laserMode); - laserModeShown = false; - } -} - -std::string Pointer::FindPanel(const char *key, Panel &p, Vec3 &eye) { - vr::VROverlayHandle_t h; - if (overlay->FindOverlay(key, &h) != vr::VROverlayError_None) return std::string("no overlay ") + key; - bool valid; - std::tie(valid, eye) = HeadPos(); - if (!valid) return "no head pose (headset off?)"; - p = ScanPanel(h, eye, 0.5); - if (!p.found) return std::string("panel not visible: ") + key; - return ""; -} - -// grabprobe (maintenance): aim down from the panel's bottom edge, 1 cm a step, and log where -// SteamVR's laser hits something (the hit dot shows) and whether the window controls are up. -void Pointer::GrabProbe(const char *key) { - Panel p; - Vec3 eye; - const std::string err = FindPanel(key, p, eye); - if (!err.empty()) { - std::printf("grabprobe: %s\n", err.c_str()); - std::fflush(stdout); - return; - } - Borrow(); - vr::VROverlayHandle_t footer = vr::k_ulOverlayHandleInvalid; - overlay->FindOverlay("valve.steam.gamepadui.floatingfooter", &footer); - const Vec3 bottom = p.center - p.basis.y * (p.height / 2); - std::printf("grabprobe %s: center (%.3f %.3f %.3f) %.3f x %.3f m\n", key, p.center.x, p.center.y, p.center.z, - p.width, p.height); - for (int cm = -5; cm <= 45; ++cm) { - const Vec3 target = bottom - p.basis.y * (cm / 100.0); - SendPose(eye, AimBasis(target - eye)); - SleepMs(90); - const bool dot = systemPointer != vr::k_ulOverlayHandleInvalid && overlay->IsOverlayVisible(systemPointer); - const bool foot = footer != vr::k_ulOverlayHandleInvalid && overlay->IsOverlayVisible(footer); - std::printf(" %+3d cm below the bottom edge: steamvr_dot=%d footer=%d", cm, dot, foot); - if (foot) { - vr::ETrackingUniverseOrigin uo; - vr::HmdMatrix34_t t{}; - if (overlay->GetOverlayTransformAbsolute(footer, &uo, &t) == vr::VROverlayError_None) { - const Vec3 f = Position(t) - p.center; - std::printf(" footer at panel (%.3f %.3f %.3f)", Dot(f, p.basis.x), Dot(f, p.basis.y), - Dot(f, p.basis.z)); - } - } - std::printf("\n"); - } - std::fflush(stdout); - GiveBack(); -} - -// Carry a floating panel so its centre lands on `target` with frame `bt` (see -// "Placement" at the top). Returns "ok ..." or "error ...". -std::string Pointer::Place(const char *key, Vec3 target, const Basis &bt, double grabBelow) { - Panel p; - Vec3 eye; - std::string err = FindPanel(key, p, eye); - if (!err.empty()) return "error " + err; - auto offBy = [&](const Panel &q, double &cm, double °) { - cm = Length(q.center - target) * 100; - const double c = (Dot(q.basis.x, bt.x) + Dot(q.basis.y, bt.y) + Dot(q.basis.z, bt.z) - 1) / 2; - deg = std::acos(std::clamp(c, -1.0, 1.0)) * 180 / M_PI; - }; - double cm, deg; - int moves = 0; - Borrow(); - for (int attempt = 0; attempt < 3; ++attempt) { - offBy(p, cm, deg); - if (cm < 1.5 && deg < 1.0) break; - // The rigid motion that takes the panel to the target: rotate by R, then move. - auto turn = [&](Vec3 v) { return FromBasis(bt, ToBasis(p.basis, v)); }; - double q[4]; - BasisQuat({turn({1, 0, 0}), turn({0, 1, 0}), turn({0, 0, 1})}, q); - const double angle = 2 * std::acos(std::clamp(q[0], -1.0, 1.0)); - const Vec3 axis = std::sin(angle / 2) > 1e-6 ? Normalize({q[1], q[2], q[3]}) : Vec3{0, 1, 0}; - const Vec3 grab = p.center - p.basis.y * (p.height / 2 + grabBelow); - const Basis d0 = AimBasis(grab - eye); - const Vec3 o1 = target + turn(eye - p.center); // device origin at the end - ++moves; - SendPose(eye, d0); - SleepMs(150); // hover: the window controls come up - SendTo(out, "ft_pointer", "btn trigger 1"); - SleepMs(150); - // 1. Rotate about the device origin (the eye): the dashboard applies it exactly. - const int rsteps = std::max(4, int(angle * 180 / M_PI / kPlaceDegPerSec * 60)); - for (int i = 1; i <= rsteps; ++i) { - const double a = angle * i / rsteps; - auto r = [&](Vec3 v) { return RotateAbout(v, axis, a); }; - SendPose(eye, {r(d0.x), r(d0.y), r(d0.z)}); - SleepMs(16); - } - // 2. Slide the device slowly: fast moves are accelerated by the dashboard. - const Basis d1{turn(d0.x), turn(d0.y), turn(d0.z)}; - const int tsteps = std::max(4, int(Length(o1 - eye) / cfg.slideSpeed * 60)); - for (int i = 1; i <= tsteps; ++i) { - SendPose(eye + (o1 - eye) * (double(i) / tsteps), d1); - SleepMs(16); - } - SleepMs(100); - SendTo(out, "ft_pointer", "btn trigger 0"); - SleepMs(600); // the dashboard re-reads the pose up to 150 ms after the release - err = FindPanel(key, p, eye); - if (!err.empty()) break; - } - GiveBack(); - if (!err.empty()) return "error after the move: " + err; - offBy(p, cm, deg); - char msg[160]; - std::snprintf(msg, sizeof msg, "ok %s off by %.1f cm, %.1f deg after %d move%s", key, cm, deg, moves, - moves == 1 ? "" : "s"); - std::printf("place: %s\n", msg); - std::fflush(stdout); - return msg; -} - - // The headset off: SteamVR drops the HMD's activity to idle as soon as it comes off. // An awake pointer (a connected controller, SteamVR's laser mode forced on) kept the // displays from sleeping, so it's released at once, and the mouse can't wake it until @@ -1291,251 +434,6 @@ void Pointer::HeadsetOff() { } } -// Outside games, gaze mode keeps the pointer (see the top); the relay needs to know. -void Pointer::GazeAwake() { - const auto t = Clock::now(); - if (t - inGameAt > std::chrono::milliseconds(500)) { - inGameAt = t; - inGame = vr::VRApplications()->GetCurrentSceneProcessId() != 0; - } - // Hand gestures in the last two minutes keep it the same way. - const bool handsRecent = cfg.handsOn && handUsed != Clock::time_point{} && t - handUsed < std::chrono::minutes(2); - const bool awake = (gazeOn || handsRecent) && !inGame && !headsetOff; - if (awake != gazeAwake || t - gazeAwakeAt > std::chrono::seconds(5)) { - if (awake != gazeAwake) - std::printf("%s keeps the pointer: %s\n", gazeOn ? "gaze" : "hand use", - awake ? "yes" : "no (off, in a game, or headset off)"); - if (awake != gazeAwake) std::fflush(stdout); - gazeAwake = awake; - gazeAwakeAt = t; - SendTo(out, "frametop_relay", awake ? "gazeawake 1" : "gazeawake 0"); - } -} - -// Commands from the relay. -void Pointer::RelayCommands() { - char buf[256]; - ssize_t n; - sockaddr_un from{}; - socklen_t fromLen = sizeof from; - while ((n = recvfrom(in, buf, sizeof buf - 1, 0, reinterpret_cast(&from), &fromLen)) > 0) { - buf[n] = 0; - // Reply to the sender (the layout tool binds an abstract address to get answers). - const sockaddr_un sender = from; - const socklen_t senderLen = fromLen; - fromLen = sizeof from; - Command(buf, sender, senderLen); - } -} - -// One command from the relay (or another sender: reply answers it). -void Pointer::Command(const char *buf, const sockaddr_un &sender, socklen_t senderLen) { - auto reply = [&](const std::string &msg) { - if (senderLen > offsetof(sockaddr_un, sun_path)) - sendto(out, msg.data(), msg.size(), 0, reinterpret_cast(&sender), senderLen); - }; - // The gaze, from ft-gazed: not mouse input, it never wakes the pointer. - double g[4]; - if (std::sscanf(buf, "gz %lf %lf %lf %lf", &g[0], &g[1], &g[2], &g[3]) == 4) { - gz = {g[0], g[1], g[2], g[3], Clock::now()}; - return; - } - // The gaze calibration panel (see the top): presses answer it instead of clicking. - { - int on; - if (std::sscanf(buf, "calpanel %d", &on) == 1) { - const bool was = Clock::now() < calPanelUntil; - calPanelUntil = on ? Clock::now() + std::chrono::seconds(3) : Clock::time_point{}; - if (on && !was) calOpened = true; - return; - } - } - if (Clock::now() < calPanelUntil) { - const bool accept = !std::strncmp(buf, "btn trigger 1", 13) || !std::strncmp(buf, "gazekey left 1", 14); - const bool quit = !std::strncmp(buf, "btn b 1", 7) || !std::strncmp(buf, "gazekey right 1", 15); - if (accept || quit) { - SendTo(out, "ft_gazed", accept ? "calaccept" : "calquit"); - return; - } - if (!std::strncmp(buf, "btn ", 4) || !std::strncmp(buf, "gazekey ", 8) || - !std::strncmp(buf, "precision ", 10) || !std::strncmp(buf, "gazedrag ", 9)) - return; // their releases, and the other buttons: nothing to click now - } - { - char kind[16], source[16]; - int v; - if (std::sscanf(buf, "%15s %15s %d", kind, source, &v) == 3 && - (!std::strcmp(kind, "precision") || !std::strcmp(kind, "gazedrag"))) { - lastMouse = Clock::now(); - if (!active) Wake(Clock::now()); - devicePresses.push_back({source, !std::strcmp(kind, "gazedrag"), v != 0}); - return; - } - if (std::sscanf(buf, "gazekey %15s %d", source, &v) == 2 && - (!std::strcmp(source, "left") || !std::strcmp(source, "right"))) { - lastMouse = Clock::now(); - if (!active) Wake(Clock::now()); - keyPresses.push_back({!std::strcmp(source, "right"), v != 0}); - return; - } - } - if (std::strcmp(buf, "typing") == 0) { // not mouse input: it never wakes the pointer - lastTyping = Clock::now(); - return; - } - if (std::strncmp(buf, "vrbind", 6) == 0) { - std::printf("controller buttons: %s\n", controllerButtons.Bind(buf + 6).c_str()); - std::fflush(stdout); - return; - } - if (std::strncmp(buf, "vrglobal", 8) == 0) { - const char *arg = buf + 8; - while (*arg == ' ') ++arg; - ControllerButtons::SetGlobal(std::strncmp(arg, "off", 3) != 0); - reply(controllerButtons.Status()); - return; - } - if (std::strncmp(buf, "vrstatus", 8) == 0) { - reply(controllerButtons.Status()); - return; - } - if (std::strncmp(buf, "overlays", 8) == 0) { - overlays.Request(sender, senderLen); // answered from the list's thread - return; - } - if (std::strncmp(buf, "gaze", 4) == 0) { - const char *arg = buf + 4; - while (*arg == ' ') ++arg; - if (*arg != '?') { // "gaze ?" only asks - gazeOn = std::strncmp(arg, "on", 2) == 0 ? true - : std::strncmp(arg, "off", 3) == 0 ? false - : !gazeOn; - gazeOwns = true, nudging = false; - std::printf("gaze mode %s\n", gazeOn ? "on" : "off"); - std::fflush(stdout); - } - reply(gazeOn ? "ok on" : "ok off"); - return; - } - const bool mouseInput = std::strncmp(buf, "move", 4) == 0 || std::strncmp(buf, "btn", 3) == 0 || - std::strncmp(buf, "scroll", 6) == 0; - // A move held back (POINTER_GAZE_MOUSE_MOVE=held) only wakes the pointer: it isn't using - // the mouse, so a drifting mouse doesn't keep the gaze from taking the pointer back. - const bool moveHeldBack = std::strncmp(buf, "move", 4) == 0 && MouseMoveHeld(); - if (mouseInput && !moveHeldBack) lastMouse = Clock::now(); - // Any mouse input wakes the pointer (after a controller took over, or a helper restart). - if (!active && mouseInput) Wake(Clock::now()); - if (moveHeldBack) return; - double a, b; - char key[128]; - double px, py, pz, pyaw, ppitch, proll = 0, pgrab = -1; - if (std::sscanf(buf, "grabprobe %127s", key) == 1) { - GrabProbe(key); - return; - } - if (std::sscanf(buf, "place %127s %lf %lf %lf %lf %lf %lf %lf", key, &px, &py, &pz, &pyaw, &ppitch, &proll, - &pgrab) >= 6) { - reply(Place(key, {px, py, pz}, PanelBasis(pyaw, ppitch, proll), pgrab >= 0 ? pgrab : cfg.grabOffset)); - return; - } - if (std::sscanf(buf, "measure %127s", key) == 1) { - Panel p; - Vec3 eye; - const std::string err = FindPanel(key, p, eye); - char msg[400] = ""; - if (err.empty()) - std::snprintf(msg, sizeof msg, "ok %.4f %.4f %.4f %.4f %.4f %.4f %.4f %.4f %.4f %.4f %.4f %.4f %.4f %.4f", - p.center.x, p.center.y, p.center.z, p.width, p.height, p.basis.x.x, p.basis.x.y, - p.basis.x.z, p.basis.y.x, p.basis.y.y, p.basis.y.z, p.basis.z.x, p.basis.z.y, - p.basis.z.z); - reply(err.empty() ? msg : "error " + err); - return; - } - if (std::strncmp(buf, "head", 4) == 0) { - vr::TrackedDevicePose_t h; - sys->GetDeviceToAbsoluteTrackingPose(vr::TrackingUniverseStanding, 0, &h, 1); - const Vec3 e = Position(h.mDeviceToAbsoluteTracking); - const Vec3 f = Rotate(h.mDeviceToAbsoluteTracking, {0, 0, -1}); - char msg[200]; - std::snprintf(msg, sizeof msg, "ok %.4f %.4f %.4f %.2f %.2f", e.x, e.y, e.z, - std::atan2(-f.x, -f.z) * 180 / M_PI, std::asin(std::clamp(f.y, -1.0, 1.0)) * 180 / M_PI); - reply(h.bPoseIsValid ? msg : "error no head pose (headset off?)"); - return; - } - if (std::strncmp(buf, "debug", 5) == 0) { - debug = !debug; - std::printf("debug %s\n", debug ? "on" : "off"); - std::fflush(stdout); - return; - } - if (std::strncmp(buf, "btn trigger 1", 13) == 0) { - LeftButton(true); - return; - } else if (std::strncmp(buf, "btn trigger 0", 13) == 0) { - LeftButton(false); - return; - } else if (std::strncmp(buf, "btn b 1", 7) == 0 && RightButton(true)) { - return; - } else if (std::strncmp(buf, "btn b 0", 7) == 0 && RightButton(false)) { - return; - } else if (std::strncmp(buf, "btn b 1", 7) == 0 && leftHeld) { - tilting = tiltStart = swallowedRight = true; // right press while dragging: tilt, no right-click - return; - } else if (std::strncmp(buf, "btn b 0", 7) == 0 && swallowedRight) { - tilting = swallowedRight = false; - return; - } - if (tilting && std::sscanf(buf, "move %lf %lf", &a, &b) == 2) { - tiltYaw += a; - tiltPitch = std::clamp(tiltPitch + b, -80.0, 80.0); - return; - } - if (std::sscanf(buf, "move %lf %lf", &a, &b) == 2) { - lastMove = Clock::now(); // the dot shows while the mouse moves it (gaze mode) - if (gazeOn && gazeOwns) { - // The mouse takes the pointer from the gaze, from where the gaze left it. - gazeOwns = false; - nudging = haveHead && Clock::now() - gz.at < std::chrono::milliseconds(200); - nudgeRawHy = gz.rhy, nudgeRawHp = gz.rhp, nudgeHead = lastHead; - nudgeAt = Clock::now(), nudgeMoved = 0; - } - if (nudging || aimHeld) nudgeMoved += std::hypot(a, b); - if (!anchored) recenter = true; - yaw += a; - while (yaw > 180) yaw -= 360; - while (yaw < -180) yaw += 360; - pitch = std::clamp(pitch + b, -85.0, 85.0); - } else if (std::strncmp(buf, "recenter", 8) == 0) { - recenter = true; - } else if (std::strncmp(buf, "reload", 6) == 0) { - cfg.Load(ReadConfig()); - ApplyConfig(); - lastSlow = Clock::now() - std::chrono::seconds(10); // apply POINTER_IGNORE now - std::printf("reloaded: free distance %.2f m, dot %.2f deg, origin %.2f, head follow %s, leash %.0f deg, " - "controller pickup %.1fx, %zu ignored\n", - cfg.freeDistance, cfg.cursorDeg, cfg.originFraction, follow ? "on" : "off", cfg.leashDeg, cfg.pickupScale, - cfg.ignore.size()); - std::fflush(stdout); - } else if (std::strncmp(buf, "follow", 6) == 0) { - const char *arg = buf + 6; - while (*arg == ' ') ++arg; - const bool was = follow; - follow = std::strncmp(arg, "on", 2) == 0 ? true : std::strncmp(arg, "off", 3) == 0 ? false : !follow; - if (follow && !was) followReset = true; - std::printf("head follow %s (leash %.0f deg)\n", follow ? "on" : "off", cfg.leashDeg); - std::fflush(stdout); - } else if (std::strncmp(buf, "show", 4) == 0) { - if (!active) Wake(Clock::now()); - } else if (std::strncmp(buf, "hide", 4) == 0) { - active = false; - overlay->HideOverlay(cursor); - overlay->HideOverlay(marker); - SendTo(out, "ft_pointer", "hide"); - } else { - SendTo(out, "ft_pointer", buf); // btn, scroll - } -} - // This frame's poses and time (see Frame). void Pointer::ReadFrame(Frame &frame) { sys->GetDeviceToAbsoluteTrackingPose(vr::TrackingUniverseStanding, 0.011f, frame.all, vr::k_unMaxTrackedDeviceCount); @@ -1629,793 +527,12 @@ void Pointer::LastUsedWins(const Frame &frame) { } } -// A pending recenter (the command, a wake, a first move): the anchor goes to the eye, aimed where the -// head faces. -void Pointer::Recenter(const Frame &frame) { - const vr::TrackedDevicePose_t &hmd = frame.Hmd(); - const auto &hm = hmd.mDeviceToAbsoluteTracking.m; - const Vec3 &eye = frame.eye; - if (recenter && hmd.bPoseIsValid) { - anchor = eye; - const Vec3 f{-hm[0][2], -hm[1][2], -hm[2][2]}; - yaw = std::atan2(-f.x, -f.z) * 180 / M_PI; - pitch = std::asin(std::clamp(f.y, -1.0, 1.0)) * 180 / M_PI; - anchored = true; - recenter = false; - followReset = true; - } -} - -// Gaze mode (see the top): the gaze has the pointer, or takes it back when you look -// well away from it. Not while a press holds the pointer, and only on fresh gaze. -void Pointer::GazeMode(const Frame &frame) { - const vr::TrackedDevicePose_t &hmd = frame.Hmd(); - const auto tnow = frame.tnow; - const Vec3 &eye = frame.eye; - if (hmd.bPoseIsValid) lastHead = hmd.mDeviceToAbsoluteTracking, haveHead = true; - if (gazeOn && active && hmd.bPoseIsValid && !tilting && !leftHeld && !aimHeld && !clickPress && !clickRelease && - tnow >= dropHoldUntil && - tnow - gz.at < std::chrono::milliseconds(150)) { - const Vec3 g = Rotate(hmd.mDeviceToAbsoluteTracking, Direction(gz.hy, gz.hp)); - if (!gazeOwns && havePoint) { - const double off = std::acos(std::clamp(Dot(g, Normalize(lastPoint - eye)), -1.0, 1.0)) * 180 / M_PI; - if (off > cfg.gazeRetake && tnow - lastMouse > std::chrono::milliseconds(300)) { - if (retakeSince == Clock::time_point{}) retakeSince = tnow; - if (tnow - retakeSince >= std::chrono::milliseconds(120)) gazeOwns = true, nudging = false; - } else { - retakeSince = {}; - } - } - if (gazeOwns) { - retakeSince = {}; - anchor = eye; - anchored = true; - yaw = std::atan2(-g.x, -g.z) * 180 / M_PI; - pitch = std::clamp(std::asin(std::clamp(g.y, -1.0, 1.0)) * 180 / M_PI, -85.0, 85.0); - } - } -} - -// Hands (see the top): pinches and grips from ft-hands. -void Pointer::Hands(const Frame &frame) { - const auto tnow = frame.tnow; - vr::TrackedDevicePose_t h0; - sys->GetDeviceToAbsoluteTrackingPose(vr::TrackingUniverseStanding, 0, &h0, 1); - if (h0.bPoseIsValid) poses.Add(tnow, h0.mDeviceToAbsoluteTracking); -} - -// Where a gesture's point is, seen from the hold's origin: yaw and pitch, degrees. -bool Pointer::HandAngles(const float p[3], uint64_t t_ns, const Vec3 &from, double &hy, double &hp) { - vr::HmdMatrix34_t head; - if (!poses.At(t_ns, head)) return false; - const Vec3 d = Normalize(Position(head) + Rotate(head, Vec3{p[0], p[1], p[2]}) - from); - hy = std::atan2(-d.x, -d.z) * 180 / M_PI; - hp = std::asin(std::clamp(d.y, -1.0, 1.0)) * 180 / M_PI; - return true; -} - -void Pointer::EndHold(bool lost) { - if (hold.pressed) { - if (leftHeld) ReleaseLeft(); - } else if (aimHeld) { - aimHeld = aimHand = false; - if (lost) { - if (gazeOn) gazeOwns = true, nudging = false; // no click: the pointer goes back to the gaze - } else { - clickPress = true; // the click is on the release, where the pointer is now - pressRight = hold.right; - gazeBack = nudgeMoved < 0.2; - } - } - if (debug) - std::printf("hold %s %s%s\n", hold.src == Src::Hand ? (hold.grip ? "grip" : "pinch") - : hold.src == Src::Head ? (hold.right ? "key right" : "key left") - : hold.grip ? "gaze drag" : "gaze precision", - lost ? "lost" : "released", hold.pressed ? "" : lost ? " (no click)" : " (click)"); - if (debug) std::fflush(stdout); - hold = {}; -} - -// The press, once a hold's source is set: a real press (dragging until the release), or -// held back like gaze mode's mouse press (see the top): the click comes on the release. -void Pointer::StartHold(bool press, Clock::time_point tnow) { - hold.startYaw = hold.lastYaw = yaw, hold.startPitch = hold.lastPitch = pitch; - hold.pressed = press; - if (press) { - gazeBack = gazeOn && gazeOwns; - PressLeft(); - } else { - gazeOwns = false; - nudging = gazeOn && haveHead && tnow - gz.at < std::chrono::milliseconds(200); - nudgeRawHy = gz.rhy, nudgeRawHp = gz.rhp, nudgeHead = lastHead; - nudgeAt = aimSince = tnow, nudgeMoved = 0; - aimHeld = aimHand = true; - pulseAt = tnow; - } -} - -// The hold's source moved to (hy, hp): past the dead zone, the pointer follows at the gain, -// from where it was when it got past. -void Pointer::Steer(double hy, double hp, double deadzone, double gain, Clock::time_point tnow) { - const double dy = std::remainder(hy - hold.refYaw, 360.0), dp = hp - hold.refPitch; - if (!hold.engaged) { - if (std::hypot(dy, dp) <= deadzone) return; - hold.engaged = true; - hold.refYaw = hy, hold.refPitch = hp; - hold.startYaw = hold.lastYaw = yaw, hold.startPitch = hold.lastPitch = pitch; - return; - } - yaw = std::remainder(hold.startYaw + gain * dy, 360.0); - pitch = std::clamp(hold.startPitch + gain * dp, -85.0, 85.0); - // How far the nudge went: for a keyboard click, from where the dot was at the press - // (a head wobbles on the way); otherwise the path, as with the mouse. - if (!hold.pressed && hold.src == Src::Head) - nudgeMoved = std::hypot(std::remainder(yaw - hold.pressYaw, 360.0), pitch - hold.pressPitch); - else if (!hold.pressed) - nudgeMoved += std::hypot(std::remainder(yaw - hold.lastYaw, 360.0), pitch - hold.lastPitch); - hold.lastYaw = yaw, hold.lastPitch = pitch; - lastMove = tnow; // the dot shows while it moves (gaze mode) -} - -void Pointer::BeginHold(int side, bool grip, const fh_pinch_t &g, Clock::time_point tnow) { - handUsed = tnow; - if (hold.src != Src::None) { - // A grip takes over a pinch on the way to it (closing the hand passes through - // one); otherwise one hold at a time. - if (hold.src != Src::Hand || !grip || hold.grip) return; - aimHeld = aimHand = false; - hold = {}; - } - if (leftHeld || aimHeld || clickPress || clickRelease) return; // the mouse's button is busy - if (!active) { - Wake(tnow); // the first gesture only wakes the pointer, like the first mouse move - return; - } - vr::HmdMatrix34_t head; - if (!poses.At(g.begin_ns, head)) return; - // No pinch just after a key (typing touches thumb to index), and a grip only with - // the hand held up (hands on a desk curl like a loose fist; looking down at them - // puts them straight ahead in the head's frame, where ft-hands can't tell). - const Vec3 at = Position(head) + Rotate(head, Vec3{g.begin_point[0], g.begin_point[1], g.begin_point[2]}); - if (!grip && tnow - lastTyping < std::chrono::duration(cfg.typingHold)) { - if (debug) std::printf("hand pinch ignored: typing\n"); - return; - } - if (grip && Position(head).y - at.y > cfg.handBelow) { - if (debug) std::printf("hand grip ignored: %.2f m below the eyes\n", Position(head).y - at.y); - return; - } - Hold h; - h.src = Src::Hand, h.side = side, h.grip = grip, h.origin = Position(head); - if (!HandAngles(g.begin_point, g.begin_ns, h.origin, h.refYaw, h.refPitch)) return; - hold = h; - // A grip, or a pinch without gaze mode: a real press where the pointer is (where - // you look, in gaze mode), dragging with the hand until it opens. - StartHold(grip || !gazeOn, tnow); - if (debug) std::printf("hand %s %s began\n", side ? "right" : "left", grip ? "grip" : "pinch"); - if (debug) std::fflush(stdout); -} - -// Gaze precision and gaze drag buttons (see the top): the mouse steers. -void Pointer::GazePrecisionButtons(const Frame &frame) { - const auto tnow = frame.tnow; - for (const DevicePress &p : devicePresses) { - if (p.source == "left" || p.source == "right") continue; // no controllers in gaze mode - if (!p.down) { - if (hold.src == Src::Mouse) EndHold(false); - continue; - } - if (hold.src != Src::None || leftHeld || aimHeld || clickPress || clickRelease) continue; - Hold h; - h.src = Src::Mouse, h.grip = p.drag; - hold = h; - StartHold(p.drag, tnow); - if (debug) std::printf("hold gaze %s began (%s)\n", p.drag ? "drag" : "precision", p.source.c_str()); - if (debug) std::fflush(stdout); - } - devicePresses.clear(); -} - -// Where the head faces: yaw and pitch, degrees. -bool Pointer::HeadAngles(const vr::TrackedDevicePose_t &hmd, double &hy, double &hp) { - const auto &hm = hmd.mDeviceToAbsoluteTracking.m; - if (!hmd.bPoseIsValid) return false; - const Vec3 f = Normalize({-hm[0][2], -hm[1][2], -hm[2][2]}); - hy = std::atan2(-f.x, -f.z) * 180 / M_PI; - hp = std::asin(std::clamp(f.y, -1.0, 1.0)) * 180 / M_PI; - return true; -} - -// Keyboard clicks (see the top): held back at the gaze, steered by the head. -void Pointer::KeyboardClicks(const Frame &frame) { - const vr::TrackedDevicePose_t &hmd = frame.Hmd(); - const auto tnow = frame.tnow; - for (const KeyPress &k : keyPresses) { - (k.right ? keyRightDown : keyLeftDown) = k.down; - if (!k.down) { - if (keyTilting && k.right) { // the tilt ends; the head drags again from here - keyTilting = tilting = false; - hold.engaged = false; - if (debug) std::printf("hold key left: tilt ended\n"); - if (debug) std::fflush(stdout); - } - // The keys holding it: a gaze_left then gaze_right drag, either; otherwise its own. - const bool held = hold.keyDrag ? keyLeftDown || keyRightDown : hold.right ? keyRightDown : keyLeftDown; - if (hold.src == Src::Head && !held) { - if (!hold.pressed && aimHeld && tnow - aimSince < std::chrono::duration(cfg.keyTap)) { - // A quick tap: a click where the dot was at the press, and the gaze was right. - yaw = hold.pressYaw, pitch = hold.pressPitch; - nudgeMoved = 0; - confirmLesson = true; - } - EndHold(false); - keyTilting = false; - } - continue; - } - if (k.right && hold.src == Src::Head && !hold.right && hold.pressed && leftHeld) { - // gaze_right during a gaze_left drag: tilt while it's held (see the top). - tilting = tiltStart = keyTilting = true; - HeadAngles(hmd, keyTiltYaw, keyTiltPitch); - if (debug) std::printf("hold key left: tilt began\n"); - if (debug) std::fflush(stdout); - continue; - } - if (k.right && hold.src == Src::Head && !hold.right && !hold.pressed && aimHeld) { - // gaze_right while gaze_left aims: press the left button where the dot is now - // (the correction is a lesson), then the head drags. - aimHeld = aimHand = false; - hold.pressed = hold.grip = hold.keyDrag = true, hold.promote = false, hold.engaged = false; - HeadAngles(hmd, hold.refYaw, hold.refPitch); - gazeBack = gazeOn; - pressRight = false; - PressLeft(); - if (debug) std::printf("hold key left: pressed (right key)\n"); - if (debug) std::fflush(stdout); - continue; - } - if (hold.src != Src::None || leftHeld || aimHeld || clickPress || clickRelease) continue; - Hold h; - h.src = Src::Head, h.promote = true, h.right = k.right, h.pressYaw = yaw, h.pressPitch = pitch; - if (!HeadAngles(hmd, h.refYaw, h.refPitch)) continue; - hold = h; - StartHold(false, tnow); - if (debug) std::printf("hold key %s began\n", k.right ? "right" : "left"); - if (debug) std::fflush(stdout); - } - keyPresses.clear(); -} - -void Pointer::CalPanelOpened() { - if (calOpened) { // the calibration panel came up: a press in progress ends, no click - calOpened = false; - if (hold.src != Src::None) EndHold(true); - if (leftHeld) ReleaseLeft(); - aimHeld = aimHand = clickPress = false; - } -} - -// A keyboard click's hold: the head steers the pointer, or turns the panel in a tilt. -void Pointer::HeadSteer(const Frame &frame) { - const vr::TrackedDevicePose_t &hmd = frame.Hmd(); - const auto tnow = frame.tnow; - if (hold.src == Src::Head) { - double hy, hp; - if (HeadAngles(hmd, hy, hp) && keyTilting) { - // The head turns the panel, as the mouse does in a tilt; the pointer stays put. - tiltYaw += std::remainder(hy - keyTiltYaw, 360.0); - tiltPitch = std::clamp(tiltPitch + hp - keyTiltPitch, -80.0, 80.0); - keyTiltYaw = hy, keyTiltPitch = hp; - } else if (HeadAngles(hmd, hy, hp)) { - Steer(hy, hp, hold.pressed ? 0.3 : cfg.headDeadzone, 1.0, tnow); - } - } -} - -// Pinches and grips from ft-hands (see "Hands" at the top), read every frame. -void Pointer::PinchesAndGrips(Frame &frame) { - const auto tnow = frame.tnow; - fh_gestures_t hg; - const bool handOk = cfg.handsOn && handFile.Read(hg); - frame.handOk = handOk; - if (handOk && (hg.seq != handSeq || handFile.opens != handOpens)) { - handSeq = hg.seq; - handPublished = hg.publish_ns; - const fh_pinch_t *slots[2] = {hg.pinch, hg.grip}; - // A new file, or a tracker whose counters went back: start counting from here. - bool rebase = !handBaseline || handFile.opens != handOpens; - for (int k = 0; k < 2; ++k) - for (int s = 0; s < 2; ++s) - rebase = rebase || slots[k][s].begins < seenBegins[k][s] || slots[k][s].ends < seenEnds[k][s]; - if (rebase) { - if (hold.src == Src::Hand) EndHold(true); - for (int k = 0; k < 2; ++k) - for (int s = 0; s < 2; ++s) seenBegins[k][s] = slots[k][s].begins, seenEnds[k][s] = slots[k][s].ends; - handBaseline = true, handOpens = handFile.opens; - } - // Not over a VR game (unless the dashboard is up), and not with the headset off. - const bool allowed = !headsetOff && (!inGame || overlay->IsDashboardVisible()); - for (int k = 1; k >= 0; --k) // grips first: one takes over a pinch - for (int s = 0; s < 2; ++s) { - const fh_pinch_t &g = slots[k][s]; - const bool began = g.begins != seenBegins[k][s], ended = g.ends != seenEnds[k][s]; - const bool lost = g.flags & FH_PINCH_LOST; - seenBegins[k][s] = g.begins, seenEnds[k][s] = g.ends; - auto holding = [&] { return hold.src == Src::Hand && hold.side == s && hold.grip == (k == 1); }; - if (holding() && ended) EndHold(lost); // the one held ended (another may have begun) - if (began && allowed) { - BeginHold(s, k == 1, g, tnow); - // begun and ended since the last read (a quick tap): its release too - if (!(g.flags & FH_PINCH_DOWN) && holding()) EndHold(lost); - } - } - // The held gesture's hand moves the pointer: past the dead zone (a tap's jitter, - // the pinch point shifting as the fingers close), then at the gain, from there. - if (hold.src == Src::Hand) { - const fh_pinch_t &g = hold.grip ? hg.grip[hold.side] : hg.pinch[hold.side]; - double hy, hp; - if ((g.flags & FH_PINCH_TRACKED) && HandAngles(g.point, hg.capture_ns, hold.origin, hy, hp)) - Steer(hy, hp, hold.grip ? 0.5 : cfg.pinchDeadzone, hold.grip ? cfg.gripGain : cfg.pinchGain, tnow); - } - } -} - -// ft-hands stopped (or hung) with a gesture held: it's over, as lost. -void Pointer::HandsStopped(const Frame &frame) { - const auto tnow = frame.tnow; - const bool handOk = frame.handOk; - const uint64_t nowNs = - uint64_t(std::chrono::duration_cast(tnow.time_since_epoch()).count()); - if (hold.src == Src::Hand && (!handOk || !active || nowNs - handPublished > 1'500'000'000ull)) EndHold(true); - if (hold.src != Src::None && hold.src != Src::Hand && !active) EndHold(true); -} - // A press holds the pointer (gaze mode's dot shows meanwhile, see the top). void Pointer::MarkHeld(const Frame &frame) { const auto tnow = frame.tnow; if (aimHeld || leftHeld || clickPress || clickRelease) lastHeld = tnow; } -// Head follow (see the top): past the leash (for the delay), ease the reference to the -// head's facing, and turn the cursor with it. Not in gaze mode: the gaze places it. -void Pointer::HeadFollow(const Frame &frame) { - const vr::TrackedDevicePose_t &hmd = frame.Hmd(); - const auto &hm = hmd.mDeviceToAbsoluteTracking.m; - const auto tnow = frame.tnow; - const Vec3 &eye = frame.eye; - if (follow && !gazeOn && active && anchored && hmd.bPoseIsValid) { - const Vec3 head = LimitPitch(Vec3{-hm[0][2], -hm[1][2], -hm[2][2]}, 85); - if (followReset) { - followRef = head, followLag = 0, leashOutSince = {}; - followReset = following = false; - } - const double dt = std::min(0.1, std::chrono::duration(tnow - followAt).count()); - const double leash = cfg.leashDeg * M_PI / 180; - const double lag = std::acos(std::clamp(Dot(followRef, head), -1.0, 1.0)); - double keep = lag; // how far the reference stays behind the head after this frame - if (leash <= 0) { - keep = 0; // head-locked - } else if (following) { - keep = lag * (cfg.leashReturn > 0 ? std::exp(-dt / cfg.leashReturn) : 0.0); - // A fast turn drags it at the leash's end; past it already (after the delay), it - // can't fall further behind, and it closes in from there without a jump. - keep = std::min(keep, std::max(leash, followLag)); - if (keep < 0.05 * M_PI / 180) keep = 0, following = false; // landed on the facing - } else if (lag > leash) { - if (leashOutSince == decltype(leashOutSince){}) leashOutSince = tnow; - if (std::chrono::duration(tnow - leashOutSince).count() >= cfg.leashDelay) - following = true, leashOutSince = {}; - } else { - leashOutSince = {}; // back inside before the delay: a glance - } - followLag = keep; - const Vec3 ref = LimitPitch(PullWithin(followRef, head, keep), 85); - if (!leftHeld && tnow >= dropHoldUntil) { - // The cursor keeps its offset from the reference (frames without roll). - Vec3 d = FromBasis(AimBasis(ref), ToBasis(AimBasis(followRef), Direction(yaw, pitch))); - d = PullWithin(Normalize(d), ref, cfg.followReach * M_PI / 180); - yaw = std::atan2(-d.x, -d.z) * 180 / M_PI; - pitch = std::clamp(std::asin(std::clamp(d.y, -1.0, 1.0)) * 180 / M_PI, -85.0, 85.0); - // The ray origin closes in on the eye as the reference does on the facing. - anchor = eye + (anchor - eye) * (leash <= 0 ? 0.0 : lag > 1e-6 ? keep / lag : following ? 0.0 : 1.0); - } - followRef = ref; - } - followAt = tnow; -} - -// Slow work, once a second: overlay handles, our device index, laser width. -void Pointer::SlowWork() { - const auto now = std::chrono::steady_clock::now(); - if (now - lastSlow > std::chrono::seconds(1)) { - lastSlow = now; - handles.clear(); - for (const auto &key : overlays.Keys()) { - if (Ignored(cfg.ignore, key)) continue; - vr::VROverlayHandle_t h; - if (overlay->FindOverlay(key.c_str(), &h) != vr::VROverlayError_None) continue; - handles[key] = h; - // Scene-graph overlays are placed absolutely. Other overlays can report no - // texture too (gamescope's app panels, placed as dashboard tabs, share theirs - // from another process), and ComputeOverlayIntersection handles those. - uint32_t tw = 0, th = 0; - overlay->GetOverlayTextureSize(h, &tw, &th); - vr::VROverlayTransformType tt = vr::VROverlayTransform_Invalid; - overlay->GetOverlayTransformType(h, &tt); - // ft-screens' panels (frametop.screen.N, floating windows with their popups, - // frametop.float.N..., the keyboard) are 0x0 and absolute too (a shared - // texture), but they're real panels of any size. - sceneGraph[key] = (tw == 0 || th == 0) && tt == vr::VROverlayTransform_Absolute && - key.rfind("frametop.", 0) != 0; - } - ours = vr::k_unTrackedDeviceIndexInvalid; - for (vr::TrackedDeviceIndex_t i = 0; i < vr::k_unMaxTrackedDeviceCount; ++i) { - char type[64] = ""; - sys->GetStringTrackedDeviceProperty(i, vr::Prop_ControllerType_String, type, sizeof type); - if (std::strcmp(type, "ft_pointer") == 0) ours = i; - } - - } - - if (now - lastVisible > std::chrono::milliseconds(50) || visible.size() != handles.size()) { - lastVisible = now; - visible.clear(); - for (const auto &[key, h] : handles) visible[key] = overlay->IsOverlayVisible(h); - } -} - -// The cursor: tilting a grabbed panel, or pointing (find the spot, draw the dot, send the ray). -void Pointer::Cursor(const Frame &frame) { - const vr::TrackedDevicePose_t &hmd = frame.Hmd(); - if (active && anchored && hmd.bPoseIsValid && tilting) { - Tilt(frame); - } else if (active && anchored && hmd.bPoseIsValid) { - const Spot spot = FindCursor(frame); - DrawDot(frame, spot); - SendRay(frame, spot); - } -} - -void Pointer::Tilt(const Frame &frame) { - const vr::TrackedDevicePose_t &hmd = frame.Hmd(); - const Vec3 &eye = frame.eye; - const vr::TrackedDevicePose_t &hmdRaw = frame.hmdRaw; - // Rotate the device around the grab point; the grabbed panel turns with it. - if (tiltStart) { - pivot = lastPoint; - tiltOrigin = lastOrigin; - tiltBasis = AimBasis(lastAim); - tiltStart = false; // angles carry on from any earlier tilt in this drag - overlay->HideOverlay(cursor); - overlay->HideOverlay(marker); - } - const double yr = tiltYaw * M_PI / 180, pr = tiltPitch * M_PI / 180; - const Vec3 up{0, 1, 0}, side = tiltBasis.x; - auto turn = [&](Vec3 v) { return RotateAbout(RotateAbout(v, side, pr), up, yr); }; - const Vec3 originStanding = pivot + turn(tiltOrigin - pivot); - const Basis b{turn(tiltBasis.x), turn(tiltBasis.y), turn(tiltBasis.z)}; - const auto &S = hmd.mDeviceToAbsoluteTracking, &R = hmdRaw.mDeviceToAbsoluteTracking; - auto toRaw = [&](Vec3 v) { return Rotate(R, RotateInverse(S, v)); }; // directions: raw <- standing - const Vec3 originRaw = Position(R) + toRaw(originStanding - eye); - double q[4]; - BasisQuat({toRaw(b.x), toRaw(b.y), toRaw(b.z)}, q); - char msg[200]; - std::snprintf(msg, sizeof msg, "posq %.5f %.5f %.5f %.6f %.6f %.6f %.6f", originRaw.x, originRaw.y, - originRaw.z, q[0], q[1], q[2], q[3]); - SendTo(out, "ft_pointer", msg); -} - -// Nearest visible overlay along a ray (frozen while dragging). -Pointer::Hit Pointer::Nearest(Vec3 from, Vec3 d) { - Hit h; - for (const auto &[key, handle] : handles) { - if (!visible[key]) continue; - if (sceneGraph[key]) { - // Plane test: overlay origin and its +Z normal, within sceneRadius of the origin. - vr::ETrackingUniverseOrigin uo; - vr::HmdMatrix34_t t{}; - if (overlay->GetOverlayTransformAbsolute(handle, &uo, &t) != vr::VROverlayError_None) continue; - const Vec3 center = Position(t), normal{t.m[0][2], t.m[1][2], t.m[2][2]}; - const double denom = Dot(d, normal); - if (std::fabs(denom) < 1e-4) continue; - const double along = Dot(center - from, normal) / denom; - const Vec3 at = from + d * along; - if (along > 0.05 && along < h.along && std::sqrt(Dot(at - center, at - center)) <= cfg.sceneRadius) - h.along = along, h.key = key, h.scene = true, h.point = at, h.normal = normal; - continue; - } - vr::VROverlayIntersectionParams_t params{}; - params.vSource = {float(from.x), float(from.y), float(from.z)}; - params.vDirection = {float(d.x), float(d.y), float(d.z)}; - params.eOrigin = vr::TrackingUniverseStanding; - vr::VROverlayIntersectionResults_t r{}; - if (overlay->ComputeOverlayIntersection(handle, ¶ms, &r) && r.fDistance > 0.05f && - r.fDistance < h.along) { - h.along = r.fDistance, h.key = key, h.scene = false; - h.point = {r.vPoint.v[0], r.vPoint.v[1], r.vPoint.v[2]}; - h.normal = {r.vNormal.v[0], r.vNormal.v[1], r.vNormal.v[2]}; - } - } - return h; -} - -// Where the cursor is this frame: on a panel, a scene-graph plane, or a panel's edge, or out in free -// space. -Pointer::Spot Pointer::FindCursor(const Frame &frame) { - const auto tnow = frame.tnow; - const Vec3 &eye = frame.eye; - const bool dragging = leftHeld || tnow < dropHoldUntil; - if (!dragging) tiltYaw = tiltPitch = 0; // drop finished: back to plain pointing - const Vec3 dir = Direction(yaw, pitch); - Hit first; - if (!dragging) first = Nearest(anchor, dir); - double best = first.along; - std::string bestKey = first.key; - bool bestScene = first.scene; - Vec3 bestPoint = first.point, bestNormal = first.normal; - bool onEdge = false; - if (!dragging && best < 1e8 && !bestScene) { - edgeKey = bestKey, edgePoint = bestPoint, edgeNormal = Normalize(bestNormal), edgeLast = bestPoint; - } else if (!dragging && best >= 1e8 && !edgeKey.empty() && visible[edgeKey]) { - // Just off a panel: stay on its plane (see "Panel edges" at the top). - const double denom = Dot(dir, edgeNormal); - if (std::fabs(denom) > 1e-4) { - const double along = Dot(edgePoint - anchor, edgeNormal) / denom; - const Vec3 at = anchor + dir * along; - if (along > 0.05 && std::sqrt(Dot(at - edgeLast, at - edgeLast)) <= cfg.edgeReach) { - best = along; - bestKey = edgeKey; - onEdge = true; - } - } - } - // Held on one of ft-screens' panels that stays where it is: onto whichever of them - // the ray meets (see the top). - if (leftHeld && !pressKey.empty()) { - vr::ETrackingUniverseOrigin uo; - vr::HmdMatrix34_t now{}; - auto it = handles.find(pressKey); - bool still = it != handles.end() && - overlay->GetOverlayTransformAbsolute(it->second, &uo, &now) == vr::VROverlayError_None; - for (int i = 0; still && i < 3; ++i) - for (int j = 0; j < 4; ++j) - if (std::fabs(now.m[i][j] - pressPose.m[i][j]) > 0.001f) still = false; - if (still) { - Hit h; - for (const auto &[key, handle] : handles) { - if (!visible[key] || !FramePanel(key)) continue; - vr::VROverlayIntersectionParams_t params{}; - params.vSource = {float(anchor.x), float(anchor.y), float(anchor.z)}; - params.vDirection = {float(dir.x), float(dir.y), float(dir.z)}; - params.eOrigin = vr::TrackingUniverseStanding; - vr::VROverlayIntersectionResults_t r{}; - if (overlay->ComputeOverlayIntersection(handle, ¶ms, &r) && r.fDistance > 0.05f && - r.fDistance < h.along) - h.along = r.fDistance, h.key = key; - } - if (h.along < 1e8) dragDistance = h.along, lastHit = h.key; - } else { - pressKey.clear(); // carried: the lock holds for the rest of this press - } - } - // While dragging: keep the press-time distance and show the non-interactive marker. - // On a scene-graph plane or a panel's edge: the laser-catching dot goes 5 cm behind it. - double distance = dragging ? dragDistance : (best < 1e8 ? best : cfg.freeDistance); - Vec3 point = anchor + dir * distance; - bool occluded = false; - if (!dragging) { - // The cursor lands on what you see under it: the ray above starts at the anchor, - // not the eye, so after leaning it can pick a panel that something nearer - // covers from where you are now (panels close together in view, at different - // depths). Anything in front of the point on the eye's line of sight wins. - const double toPoint = std::sqrt(Dot(point - eye, point - eye)); - const Hit front = Nearest(eye, Normalize(point - eye)); - if (front.along < toPoint - 0.02) { - occluded = true; - bestKey = front.key, bestScene = front.scene; - point = front.point; - if (!front.scene) edgeKey = front.key, edgePoint = front.point, edgeNormal = Normalize(front.normal), - edgeLast = front.point; - distance = std::sqrt(Dot(point - anchor, point - anchor)); - best = distance; - onEdge = false; - } - lastDistance = distance, lastHit = bestKey; - } - const bool onScene = !dragging && ((bestScene && best < 1e8) || onEdge); - const bool onPanel = dragging || (best < 1e8 && !onScene); - Spot s; - s.dragging = dragging, s.dir = dir, s.point = point, s.best = best, s.distance = distance; - s.onEdge = onEdge, s.occluded = occluded, s.onScene = onScene, s.onPanel = onPanel; - return s; -} - -// Our dot where the cursor is (see "Looks" and "SteamVR Settings" at the top). -void Pointer::DrawDot(const Frame &frame, const Spot &spot) { - const auto tnow = frame.tnow; - const Vec3 &eye = frame.eye; - const bool dragging = spot.dragging, onScene = spot.onScene, onPanel = spot.onPanel; - const Vec3 dir = spot.dir, point = spot.point; - const Vec3 sight = Normalize(point - eye); - if (!dragging) { - // SteamVR Settings (see the top): the dashboard's main panel is hidden and its - // scene-graph panel shows the page. - onVrSettings = false; - const auto mainIt = visible.find("valve.steam.gamepadui.main"); - if (overlay->IsDashboardVisible() && !(mainIt != visible.end() && mainIt->second)) { - for (const auto &[key, handle] : handles) { - if (!visible[key] || key.rfind("valve.steam.gamepadui.frame.menu.", 0) != 0) continue; - vr::ETrackingUniverseOrigin uo; - vr::HmdMatrix34_t t{}; - if (overlay->GetOverlayTransformAbsolute(handle, &uo, &t) == vr::VROverlayError_None && - OnSettingsPage(t, eye, sight)) - onVrSettings = true; - } - } - } - if (onVrSettings != catcherHidesHit) { - overlay->SetOverlayFlag(cursor, vr::VROverlayFlags_HideLaserIntersection, onVrSettings); - catcherHidesHit = onVrSettings; - } - - if (onVrSettings) { - // The dot close in front of the page, which is nearer than any guess of ours; - // the laser-catching dot far behind everything, invisible, so it never covers - // the page, with SteamVR's hit dot hidden on it. - const Vec3 near = eye + sight * SETTINGS_DOT, far = eye + sight * SETTINGS_CATCHER; - double alpha = 1; - if (gazeOn && !cfg.gazeDotAlways) { - auto secs = [&](Clock::time_point t) { return std::chrono::duration(tnow - t).count(); }; - alpha = std::clamp(1 - std::min(secs(lastMove) - cfg.gazeShow, secs(lastHeld)) / 0.25, 0.0, 1.0); - } - if (tnow < calPanelUntil) alpha = 0; - overlay->SetOverlayAlpha(marker, float(alpha)); - overlay->SetOverlayWidthInMeters(marker, float(2 * SETTINGS_DOT * std::tan(cfg.cursorDeg * M_PI / 360))); - auto mm = Billboard(near, eye); - overlay->SetOverlayTransformAbsolute(marker, vr::TrackingUniverseStanding, &mm); - overlay->SetOverlayAlpha(cursor, 0); - overlay->SetOverlayWidthInMeters(cursor, float(2 * SETTINGS_CATCHER * std::tan(cfg.cursorDeg * M_PI / 360))); - auto mc = Billboard(far, eye); - overlay->SetOverlayTransformAbsolute(cursor, vr::TrackingUniverseStanding, &mc); - overlay->ShowOverlay(marker); - overlay->ShowOverlay(cursor); - } else { - // On a panel: the non-interactive marker, pulled 5 mm toward the eye so it - // draws on top. In free space: the interactive dot the laser lands on. - const vr::VROverlayHandle_t show = onPanel ? marker : cursor, hide = onPanel ? cursor : marker; - const Vec3 at = onPanel ? point + Normalize(eye - point) * 0.005 : onScene ? point + dir * 0.05 : point; - const double dist = std::sqrt(Dot(at - eye, at - eye)); - // Gaze mode: a pulse for each click, and with POINTER_GAZE_DOT=moving, shown only - // while something moves it or a press holds it (see the top); transparent - // otherwise, the laser still lands on it. - double scale = 1, alpha = 1; - if (gazeOn) { - auto secs = [&](Clock::time_point t) { return std::chrono::duration(tnow - t).count(); }; - alpha = cfg.gazeDotAlways ? 1.0 : std::clamp(1 - std::min(secs(lastMove) - cfg.gazeShow, secs(lastHeld)) / 0.25, 0.0, 1.0); - const double pulse = secs(pulseAt); - if (pulse < 0.6) { - scale = 1 + 1.5 * std::max(0.0, 1 - pulse / 0.3); - alpha = std::max(alpha, std::clamp((0.6 - pulse) / 0.3, 0.0, 1.0)); - } - } - if (tnow < calPanelUntil) alpha = 0; // the calibration panel is up (see the top) - overlay->SetOverlayAlpha(show, float(alpha)); - overlay->SetOverlayWidthInMeters(show, float(2 * dist * std::tan(scale * cfg.cursorDeg * M_PI / 360))); - auto m = Billboard(at, eye); - overlay->SetOverlayTransformAbsolute(show, vr::TrackingUniverseStanding, &m); - overlay->ShowOverlay(show); - overlay->HideOverlay(hide); - } -} - -void Pointer::SendRay(const Frame &frame, const Spot &spot) { - const vr::TrackedDevicePose_t &hmd = frame.Hmd(); - const auto tnow = frame.tnow; - const Vec3 &eye = frame.eye; - const vr::TrackedDevicePose_t &hmdRaw = frame.hmdRaw; - const bool dragging = spot.dragging, occluded = spot.occluded, onEdge = spot.onEdge, onScene = spot.onScene; - const double best = spot.best, distance = spot.distance; - const Vec3 point = spot.point; - // Controller ray: from the eye, aimed at the cursor point, converted from the - // standing universe to raw tracking space via the HMD's pose in both. - const Vec3 aimStanding = Normalize(point - eye); - const auto &S = hmd.mDeviceToAbsoluteTracking, &R = hmdRaw.mDeviceToAbsoluteTracking; - const Vec3 aim = Rotate(R, RotateInverse(S, aimStanding)); // raw <- head <- standing - // Origin partway along the line of sight to the cursor (smaller hit dot). - const double toPoint = std::sqrt(Dot(point - eye, point - eye)); - double originDist = std::max(0.0, std::min(toPoint * cfg.originFraction, toPoint - cfg.originMargin)); - if (onVrSettings) originDist = std::min(originDist, SETTINGS_ORIGIN); // SteamVR finds the page - const Vec3 originStanding = eye + Normalize(point - eye) * originDist; - const Vec3 eyeRaw = Position(R) + Rotate(R, RotateInverse(S, originStanding - eye)); - lastPoint = point, lastOrigin = originStanding, lastAim = aimStanding; // tilt starts from here - havePoint = true; - if (debug && tnow - lastDebug > std::chrono::milliseconds(500)) { - lastDebug = tnow; - if (systemPointer == vr::k_ulOverlayHandleInvalid) overlay->FindOverlay("system.pointer", &systemPointer); - std::printf("dbg %s hit=%s dist=%.2f eye->point=%.2f origin=%.2f vrsettings=%d yaw=%.1f pitch=%.1f gaze=%s steamvr_dot=%d primary=%u\n", - dragging ? "DRAG" : occluded ? "INFRONT" : onEdge ? "EDGE" : onScene ? "SCENE" : (best < 1e8 ? "PANEL" : "FREE"), lastHit.empty() ? "-" : lastHit.c_str(), - distance, toPoint, originDist, onVrSettings, yaw, pitch, - !gazeOn ? "off" : tnow - gz.at > std::chrono::milliseconds(150) ? "stale" : gazeOwns ? "owns" : "mouse", - systemPointer != vr::k_ulOverlayHandleInvalid && overlay->IsOverlayVisible(systemPointer), - overlay->GetPrimaryDashboardDevice()); - std::fflush(stdout); - } - const double ayaw = std::atan2(-aim.x, -aim.z) * 180 / M_PI; - const double apitch = std::asin(std::clamp(aim.y, -1.0, 1.0)) * 180 / M_PI; - if (dragging && (tiltYaw != 0 || tiltPitch != 0)) { - // Keep this drag's tilt applied, about the current cursor point. - const double yr = tiltYaw * M_PI / 180, pr = tiltPitch * M_PI / 180; - const Basis base = AimBasis(aimStanding); - const Vec3 up{0, 1, 0}, side = base.x; - auto turn = [&](Vec3 v) { return RotateAbout(RotateAbout(v, side, pr), up, yr); }; - const Vec3 o = point + turn(originStanding - point); - const Basis b{turn(base.x), turn(base.y), turn(base.z)}; - auto toRaw = [&](Vec3 v) { return Rotate(R, RotateInverse(S, v)); }; - const Vec3 oRaw = Position(R) + toRaw(o - eye); - double q[4]; - BasisQuat({toRaw(b.x), toRaw(b.y), toRaw(b.z)}, q); - char msg[200]; - std::snprintf(msg, sizeof msg, "posq %.5f %.5f %.5f %.6f %.6f %.6f %.6f", oRaw.x, oRaw.y, oRaw.z, q[0], - q[1], q[2], q[3]); - SendTo(out, "ft_pointer", msg); - } else { - char msg[160]; - std::snprintf(msg, sizeof msg, "pose %.5f %.5f %.5f %.4f %.4f", eyeRaw.x, eyeRaw.y, eyeRaw.z, ayaw, apitch); - SendTo(out, "ft_pointer", msg); - } -} - -// A held-back press (see the top): held still long enough, it's a real press (a drag); -// released, it's a click where the pointer is now (this frame's pose has gone out). -void Pointer::HeldBackPress(const Frame &frame) { - const auto tnow = frame.tnow; - if (!active) aimHeld = aimRight = clickPress = aimHand = confirmLesson = false; // released meanwhile: nothing to click - if (aimHeld && !aimHand && nudgeMoved < 0.2 && tnow - aimSince >= std::chrono::duration(cfg.gazeHold)) { - aimHeld = false; - gazeBack = true; - pressRight = aimRight; // the right button's: a real right press (see the top) - aimRight = false; - PressLeft(); - } -} - -// A keyboard click held still for POINTER_GAZE_HOLD: a real press, then the head drags -// (from where it is now). -void Pointer::KeyboardClickHold(const Frame &frame) { - const vr::TrackedDevicePose_t &hmd = frame.Hmd(); - const auto &hm = hmd.mDeviceToAbsoluteTracking.m; - const auto tnow = frame.tnow; - if (aimHeld && aimHand && hold.promote && !hold.engaged && - tnow - aimSince >= std::chrono::duration(cfg.gazeHold)) { - aimHeld = aimHand = false; - hold.pressed = hold.grip = true; - if (hmd.bPoseIsValid) { - const Vec3 f = Normalize({-hm[0][2], -hm[1][2], -hm[2][2]}); - hold.refYaw = std::atan2(-f.x, -f.z) * 180 / M_PI; - hold.refPitch = std::asin(std::clamp(f.y, -1.0, 1.0)) * 180 / M_PI; - } - gazeBack = gazeOn; - pressRight = hold.right; - PressLeft(); - if (debug) std::printf("hold key %s: pressed (held still)\n", hold.right ? "right" : "left"); - if (debug) std::fflush(stdout); - } -} - -// The click a held-back press ends in: the press now, the release 40 ms later. -void Pointer::Click(const Frame &frame) { - const auto tnow = frame.tnow; - if (clickPress) { - clickPress = false; - PressLeft(); - clickRelease = true; - clickReleaseAt = tnow + std::chrono::milliseconds(40); - } else if (clickRelease && tnow >= clickReleaseAt) { - clickRelease = false; - ReleaseLeft(); - } -} - // Frame controller buttons (vrbuttons.h), to the relay. void Pointer::PollControllerButtons() { controllerButtons.Poll( diff --git a/pointer/helper/pointer.h b/pointer/helper/pointer.h new file mode 100644 index 0000000..07524ec --- /dev/null +++ b/pointer/helper/pointer.h @@ -0,0 +1,478 @@ +// ft-pointer's shared code: helpers, the Frame, and the Pointer struct, whose member functions are +// in ft-pointer.cpp and pointer_*.cpp. +// "The top" in comments here is the header comment of ft-pointer.cpp. +#pragma once + +#include + +#include "pointer_config.h" +#include "vrbuttons.h" +#include "vrmath.h" + +extern "C" { +#include "../../hands/include/fh_gestures.h" +} + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include + +#include +#include +#include +#include +#include + +using namespace md; + +// Hand gestures from ft-hands (see "Hands" at the top): /run/user/UID/frametop-hands/gestures, +// mapped read-only. Version 1 files have pinches only; their grips read as zero. +class HandGestures { +public: + ~HandGestures() { Close(); } + // A consistent copy of the file (its sequence lock), if it's there. Opens it, and + // checks it's still the same file, at most once a second. + bool Read(fh_gestures_t &out) { + const auto now = std::chrono::steady_clock::now(); + if (now - checked_ > std::chrono::seconds(1)) { + checked_ = now; + const std::string path = "/run/user/" + std::to_string(getuid()) + "/frametop-hands/gestures"; + struct stat st; + if (stat(path.c_str(), &st) != 0 || size_t(st.st_size) != len_ || st.st_ino != ino_) { + Close(); + const int fd = open(path.c_str(), O_RDONLY | O_CLOEXEC | O_NOFOLLOW); + if (fd >= 0 && fstat(fd, &st) == 0 && size_t(st.st_size) >= offsetof(fh_gestures_t, grip)) { + void *m = mmap(nullptr, size_t(st.st_size), PROT_READ, MAP_SHARED, fd, 0); + if (m != MAP_FAILED) map_ = m, len_ = size_t(st.st_size), ino_ = st.st_ino, ++opens; + } + if (fd >= 0) close(fd); + } + } + if (!map_) return false; + const auto *g = static_cast(map_); + if (std::memcmp(g->magic, FH_GESTURES_MAGIC, 8) != 0) return false; + const size_t n = std::min(len_, sizeof out); + for (int tries = 0; tries < 3; ++tries) { + const uint64_t seq = __atomic_load_n(&g->seq, __ATOMIC_ACQUIRE); + if (seq & 1) continue; + std::memset(&out, 0, sizeof out); + std::memcpy(&out, map_, n); + __atomic_thread_fence(__ATOMIC_ACQUIRE); + if (__atomic_load_n(&g->seq, __ATOMIC_RELAXED) != seq) continue; + if (out.version < 2) std::memset(out.grip, 0, sizeof out.grip); + return true; + } + return false; + } + int opens = 0; // a new file: the counters start over + +private: + void Close() { + if (map_) munmap(map_, len_); + map_ = nullptr, len_ = 0, ino_ = 0; + } + void *map_ = nullptr; + size_t len_ = 0; + ino_t ino_ = 0; + std::chrono::steady_clock::time_point checked_{}; +}; + +// The HMD's recent poses (standing universe), to turn the gestures' head-frame points into +// the room as the head was when the cameras took them. +class PoseHistory { +public: + void Add(std::chrono::steady_clock::time_point t, const vr::HmdMatrix34_t &m) { + poses_.push_back({t, m}); + while (poses_.size() > 128) poses_.pop_front(); // about a second + } + // The pose at CLOCK_MONOTONIC time t_ns (steady_clock's), or the nearest kept. + bool At(uint64_t t_ns, vr::HmdMatrix34_t &out) const { + if (poses_.empty()) return false; + const std::chrono::steady_clock::time_point t{std::chrono::nanoseconds(t_ns)}; + const auto *best = &poses_.front(); + for (const auto &p : poses_) + if (std::chrono::abs(p.first - t) < std::chrono::abs(best->first - t)) best = &p; + out = best->second; + return true; + } + +private: + std::deque> poses_; +}; + +// One of ft-screens' panels showing a desktop: a screen (frametop.screen.N), a floating +// window (frametop.float.N), or a floating window's popup (frametop.float.N.sub.K), not a +// control of theirs. +inline bool FramePanel(const std::string &key) { + for (const char *prefix : {"frametop.screen.", "frametop.float."}) { + if (key.rfind(prefix, 0) != 0) continue; + const std::string rest = key.substr(std::strlen(prefix)); + const size_t dot = rest.find('.'); + return dot == std::string::npos || rest.compare(dot, 5, ".sub.") == 0; + } + return false; +} + +inline void SendTo(int fd, const char *name, const std::string &msg) { + sockaddr_un addr{}; + addr.sun_family = AF_UNIX; + std::memcpy(addr.sun_path + 1, name, std::strlen(name)); + const socklen_t len = offsetof(sockaddr_un, sun_path) + 1 + std::strlen(name); + sendto(fd, msg.data(), msg.size(), 0, reinterpret_cast(&addr), len); +} + +// JSON string literal (names come from other apps). +inline std::string JsonQuote(const std::string &s) { + std::string out = "\""; + for (const unsigned char c : s) { + if (c == '"' || c == '\\') out += '\\', out += char(c); + else if (c < 0x20) { + char esc[8]; + std::snprintf(esc, sizeof esc, "\\u%04x", c); + out += esc; + } else out += char(c); + } + return out + "\""; +} + +// Overlay keys, refreshed in the background from `vrcmd --overlays` (OpenVR has no +// public call to enumerate other apps' overlays). Hidden ones are listed too: the +// window controls under a floating panel only appear while something hovers the +// panel, and the cursor has to find them the moment they do, not a second later. +// Paused while the pointer is off: each vrcmd run connects to SteamVR as a new app, and a new +// app every second kept SteamVR (and the headset's displays) from going to standby. +// "overlays" requests (Frametop Input Settings' Ignored panels page) refresh the list even +// while paused, and are answered from this thread once it's fresh. +class OverlayList { +public: + void Start() { + out_ = socket(AF_UNIX, SOCK_DGRAM | SOCK_CLOEXEC, 0); + thread_ = std::thread([this] { + while (running_) { + if (!paused_ || requested_) Refresh(); + // Wait a second, or less when the pointer wakes (refresh right away then). + for (int i = 0; i < 10 && running_; ++i) { + const bool wasPaused = paused_; + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + if ((wasPaused && !paused_) || requested_) break; + } + } + }); + } + void SetPaused(bool paused) { paused_ = paused; } + void Stop() { + running_ = false; + if (thread_.joinable()) thread_.join(); + } + std::vector Keys() { + std::lock_guard guard(lock_); + std::vector keys; + for (const auto &e : entries_) keys.push_back(e.key); + return keys; + } + // Answer `to` with {"t":"overlays","list":[{"key","name","visible"}...]} after the next refresh. + void Request(const sockaddr_un &to, socklen_t len) { + if (len <= offsetof(sockaddr_un, sun_path)) return; + std::lock_guard guard(lock_); + if (waiting_.size() < 8) waiting_.push_back({to, len}); + requested_ = true; + } + +private: + struct Entry { + std::string key, name; + bool visible; + }; + void Refresh() { + requested_ = false; + FILE *p = popen("LD_LIBRARY_PATH=/opt/steamvr/bin/linuxarm64 /opt/steamvr/bin/linuxarm64/vrcmd --overlays 2>/dev/null", "r"); + if (!p) return; + std::vector entries; + char line[1024]; + while (std::fgets(line, sizeof line, p)) { + // 'key' -- 'name', WxH visible VROverlayType_... + if (line[0] != '\'') continue; + const char *end = std::strchr(line + 1, '\''); + if (!end) continue; + const std::string key(line + 1, size_t(end - (line + 1))); + const std::string rest(end); + if (rest.find("Thumbnail") != std::string::npos || rest.find("Subview") != std::string::npos) continue; + if (key.rfind("system.pointer", 0) == 0 || key.rfind("system.cursor", 0) == 0 || + key.rfind("frametop.pointer", 0) == 0 || key.rfind("frametop.guide", 0) == 0 || + key == "frametop.gazepanel" || // the gaze calibration panel, fixed to the headset + key == "frametop.catcher" || // ft-screens' release catcher: only on a laser mid-drag + key == "system.HeadsetView" || key == "system.toast") + continue; + // The name can hold quotes; it ends at the last "', " (the size and state follow). + const auto nameAt = rest.find("-- '"), nameEnd = rest.rfind("', "); + const std::string name = + nameAt != std::string::npos && nameEnd > nameAt + 3 ? rest.substr(nameAt + 4, nameEnd - nameAt - 4) : key; + entries.push_back({key, name, rest.find(" not_visible ") == std::string::npos}); + } + pclose(p); + std::vector> waiting; + { + std::lock_guard guard(lock_); + entries_ = std::move(entries); + waiting.swap(waiting_); + } + if (waiting.empty()) return; + std::string msg = "{\"t\":\"overlays\",\"list\":["; + for (size_t i = 0; i < entries_.size(); ++i) + msg += std::string(i ? "," : "") + "{\"key\":" + JsonQuote(entries_[i].key) + ",\"name\":" + + JsonQuote(entries_[i].name) + ",\"visible\":" + (entries_[i].visible ? "true" : "false") + "}"; + msg += "]}"; + for (const auto &[to, len] : waiting) + sendto(out_, msg.data(), msg.size(), MSG_DONTWAIT, reinterpret_cast(&to), len); + } + + std::thread thread_; + int out_ = -1; + std::atomic paused_{false}; + std::atomic running_{true}; + std::atomic requested_{false}; + std::mutex lock_; + std::vector entries_; // written only by the thread; the lock guards readers + std::vector> waiting_; +}; + +using Clock = std::chrono::steady_clock; + +// One frame's poses and time: ReadFrame reads them after the relay's commands, and the sections +// after it share them. handOk: PinchesAndGrips read the hand gestures (for HandsStopped). +struct Frame { + vr::TrackedDevicePose_t all[vr::k_unMaxTrackedDeviceCount]; + vr::TrackedDevicePose_t hmdRaw; + Clock::time_point tnow; + Vec3 eye; + bool handOk = false; + const vr::TrackedDevicePose_t &Hmd() const { return all[0]; } +}; + +// Everything the pointer keeps: its settings, what Init sets up, and the state the main loop +// carries from one frame to the next. +struct Pointer { + PointerConfig cfg; + // Head follow (see the top). followConf is POINTER_FOLLOW as last read: a reload only + // overrides a "follow" command when the setting itself changed. + bool follow = false, followConf = false, followReset = true; + // Gaze mode (see the top); gazeConf is POINTER_GAZE as last read, like followConf. + bool gazeOn = false, gazeConf = false; + // Set in Init, after VR_Init. + vr::IVRSystem *sys = nullptr; + vr::IVROverlay *overlay = nullptr; + // The dot, the marker, and laser mode's overlay, created in Init. + vr::VROverlayHandle_t cursor = vr::k_ulOverlayHandleInvalid; + vr::VROverlayHandle_t marker = vr::k_ulOverlayHandleInvalid; + vr::VROverlayHandle_t laserMode = vr::k_ulOverlayHandleInvalid; + bool laserModeShown = false; + int in = -1, out = -1; // @ft_pointer_helper, and the socket we send from (opened in Init) + ControllerButtons controllerButtons; // Frame controller buttons (vrbuttons.h) + OverlayList overlays; + std::map handles; + std::map sceneGraph; // no texture: plane test instead of ComputeOverlayIntersection + std::map visible; // refreshed every 50 ms + Clock::time_point lastVisible{}; // set in Init + // The plane of the last panel the cursor was on, and the last point on it (panel edges). + Vec3 edgePoint, edgeNormal, edgeLast; + std::string edgeKey; + + bool active = false, recenter = false, anchored = false; + Clock::time_point lastMouse{}, claimAt{}, claimRelease{}, wokeAt{}, noWakeUntil{}; + bool claimPending = false, claimHeld = false; + // Last used wins: since when each controller has been moving (zero: it isn't). + Clock::time_point movingSince[vr::k_unMaxTrackedDeviceCount] = {}; + // Tilt mode (see top of file). + bool leftHeld = false, tilting = false, tiltStart = false, swallowedRight = false; + // The mouse's buttons in gaze mode (see the top): leftDown, rightDown as the relay last said; + // aimRight, the held-back press is the right button's; chordDrag, a drag the right button + // began during the left's held-back press (the first release drops it, the other's is nothing). + bool leftDown = false, rightDown = false, aimRight = false, chordDrag = false; + bool ignoreLeftUp = false; + // The keyboard clicks' keys as the relay last said, and a tilt gaze_right holds during a + // keyboard drag (see the top): the head turns the panel, from where it was (keyTiltYaw, keyTiltPitch). + bool keyLeftDown = false, keyRightDown = false, keyTilting = false; + double keyTiltYaw = 0, keyTiltPitch = 0; + double tiltYaw = 0, tiltPitch = 0; + double dragDistance = 0, lastDistance = 1.5; // drag lock: distance from the anchor at the press + bool onVrSettings = false; // the cursor is on the SteamVR Settings page (kept while dragging) + bool catcherHidesHit = false; // the laser-catching dot hides SteamVR's hit dot (Settings page) + Clock::time_point dropHoldUntil{}; // after a left release: keep the drag pose this long + bool debug = false; + std::string lastHit; + // The ft-screens panel the left button was pressed on, and where it was then (see the top). + std::string pressKey; + vr::HmdMatrix34_t pressPose{}; + Clock::time_point lastDebug{}; // set in Init + vr::VROverlayHandle_t systemPointer = vr::k_ulOverlayHandleInvalid; // found in Init + Vec3 pivot, tiltOrigin, lastPoint, lastOrigin, lastAim{0, 0, -1}; + Basis tiltBasis{}; + bool headsetOff = false; // nobody is wearing the headset (see the main loop) + Vec3 anchor; + double yaw = 0, pitch = 0; + Vec3 followRef{0, 0, -1}; // head follow's reference direction (see the top) + Clock::time_point followAt{}; // its last update, for the easing (set in Init) + bool following = false; // past the leash: easing toward the head + double followLag = 0; // radians the reference trails the head + std::chrono::steady_clock::time_point leashOutSince{}; // head past the leash since (delay) + // Gaze mode (see the top). + struct Gaze { + double hy = 0, hp = 0, rhy = 0, rhp = 0; // corrected, and raw + Clock::time_point at{}; + } gz; + bool gazeOwns = true; // the pointer follows the gaze; false: the mouse has it + bool nudging = false; // the mouse took it from the gaze: the next click may be a lesson + double nudgeRawHy = 0, nudgeRawHp = 0, nudgeMoved = 0; + vr::HmdMatrix34_t nudgeHead{}, lastHead{}; + bool haveHead = false, havePoint = false; + Clock::time_point nudgeAt{}, retakeSince{}; + // A held-back press (see the top): aimHeld while the button is down; then the click + // (clickPress at the end of the next frame, clickRelease 40 ms later). gazeBack: give + // the gaze the pointer again when the press or click is over. + vr::TrackedDeviceIndex_t ours = vr::k_unTrackedDeviceIndexInvalid; + bool aimHeld = false, clickPress = false, clickRelease = false, gazeBack = false; + bool aimHand = false; // the held-back press is a hold's (below): it never turns into a real press + Clock::time_point aimSince{}, clickReleaseAt{}; + // Holds (see "Gaze precision" and "Hands" at the top): a pinch or grip, or a gaze + // precision or gaze drag button, held now. What steers the pointer meanwhile (a hand, or + // the mouse through its own moves), from where it pointed when the hold began (for a + // hand: seen from the eye then, in the room), and the pointer then. + HandGestures handFile; + PoseHistory poses; + enum class Src { None, Hand, Mouse, Head }; + struct Hold { + Src src = Src::None; + int side = -1; // a hand's side (0 left, 1 right) + bool grip = false; // pressed at once and dragging (a grip, gaze drag), not a click on release + bool engaged = false; // past the dead zone + bool pressed = false; // a real press went out (a grip or gaze drag, or a pinch without gaze mode) + bool promote = false; // held still for POINTER_GAZE_HOLD, it becomes a real press (keyboard clicks) + bool right = false; // the right button (gaze_right) + bool keyDrag = false; // gaze_right pressed while gaze_left aimed: a left press, either key ends it + double pressYaw = 0, pressPitch = 0; // the pointer at the press (a quick tap clicks there) + Vec3 origin; + double refYaw = 0, refPitch = 0, startYaw = 0, startPitch = 0, lastYaw = 0, lastPitch = 0; + } hold; + // "precision|gazedrag 1|0" from the relay, done in the frame (see Holds). + struct DevicePress { + std::string source; + bool drag, down; + }; + std::vector devicePresses; + // "gazekey left|right 1|0" from the relay (keyboard clicks), done in the frame. + struct KeyPress { + bool right, down; + }; + std::vector keyPresses; + uint32_t seenBegins[2][2] = {}, seenEnds[2][2] = {}; // [pinch, grip][side], as last read + bool handBaseline = false; + int handOpens = 0; + uint64_t handSeq = 0, handPublished = 0; + Clock::time_point handUsed{}; // a gesture began then (keeps the pointer, like gaze mode) + Clock::time_point lastTyping{}; // the relay's last "typing": a key on a keyboard + // Gaze mode outside games, and its dot (see the top): lastMove/lastHeld/pulseAt. + bool inGame = false, gazeAwake = false; + Clock::time_point inGameAt{}, gazeAwakeAt{}; + Clock::time_point lastMove{}, lastHeld{}, pulseAt{}; + // The left button, as sent to the driver; pressRight: the next press is the right button + // instead (gaze_right), heldButton: the one pressed. + bool pressRight = false; + std::string heldButton = "trigger"; + bool confirmLesson = false; // a keyboard click's quick tap: a lesson with no correction (see the top) + // The gaze calibration panel is up until then ("calpanel 1"; see the top); calOpened: it just + // came up, so a press in progress ends without a click. + Clock::time_point calPanelUntil{}; + bool calOpened = false; + Clock::time_point lastSlow{}; // set in Init + + void ApplyConfig(); + void Init(); + void Wake(Clock::time_point t); + void PressLeft(); + void ReleaseLeft(); + bool CanAim(); + void AimStart(bool right); + void ChordDrop(); + void LeftButton(bool down); + bool RightButton(bool down); + bool MouseMoveHeld(); + void SendPose(Vec3 originStanding, const Basis &b); + std::pair HeadPos(); + void Borrow(); + void GiveBack(); + std::string FindPanel(const char *key, Panel &p, Vec3 &eye); + void GrabProbe(const char *key); + std::string Place(const char *key, Vec3 target, const Basis &bt, double grabBelow); + // Nearest's answer: the nearest overlay along a ray. + struct Hit { + double along = 1e9; + std::string key; + bool scene = false; + Vec3 point, normal; + }; + // Where the cursor is this frame (FindCursor), for DrawDot and SendRay. + struct Spot { + bool dragging = false; + Vec3 dir, point; + double best = 1e9, distance = 0; + bool onEdge = false, occluded = false, onScene = false, onPanel = false; + }; + + // The main loop's sections, in the order main calls them, and what they share. + void HeadsetOff(); + void GazeAwake(); + void RelayCommands(); + void Command(const char *buf, const sockaddr_un &sender, socklen_t senderLen); + void ReadFrame(Frame &frame); + void ClaimPulse(const Frame &frame); + void PauseOverlayList(); + void LaserMode(); + void HandRole(const Frame &frame); + void LastUsedWins(const Frame &frame); + void Recenter(const Frame &frame); + void GazeMode(const Frame &frame); + void Hands(const Frame &frame); + bool HandAngles(const float p[3], uint64_t t_ns, const Vec3 &from, double &hy, double &hp); + void EndHold(bool lost); + void StartHold(bool press, Clock::time_point tnow); + void Steer(double hy, double hp, double deadzone, double gain, Clock::time_point tnow); + void BeginHold(int side, bool grip, const fh_pinch_t &g, Clock::time_point tnow); + void GazePrecisionButtons(const Frame &frame); + bool HeadAngles(const vr::TrackedDevicePose_t &hmd, double &hy, double &hp); + void KeyboardClicks(const Frame &frame); + void CalPanelOpened(); + void HeadSteer(const Frame &frame); + void PinchesAndGrips(Frame &frame); + void HandsStopped(const Frame &frame); + void MarkHeld(const Frame &frame); + void HeadFollow(const Frame &frame); + void SlowWork(); + void Cursor(const Frame &frame); + void Tilt(const Frame &frame); + Hit Nearest(Vec3 from, Vec3 d); + Spot FindCursor(const Frame &frame); + void DrawDot(const Frame &frame, const Spot &spot); + void SendRay(const Frame &frame, const Spot &spot); + void HeldBackPress(const Frame &frame); + void KeyboardClickHold(const Frame &frame); + void Click(const Frame &frame); + void PollControllerButtons(); + bool SteamVRQuit(); +}; diff --git a/pointer/helper/pointer_cursor.cpp b/pointer/helper/pointer_cursor.cpp new file mode 100644 index 0000000..0491103 --- /dev/null +++ b/pointer/helper/pointer_cursor.cpp @@ -0,0 +1,474 @@ +// ft-pointer: recenter, head follow, the overlay handles, and the cursor (see "Cursor model" at +// the top). +// "The top" in comments here is the header comment of ft-pointer.cpp. +#include "pointer.h" + +namespace { + +bool Ignored(const std::vector &patterns, const std::string &key) { + for (const auto &p : patterns) + if (fnmatch(p.c_str(), key.c_str(), FNM_NOESCAPE) == 0) return true; + return false; +} + +vr::HmdMatrix34_t Billboard(Vec3 at, Vec3 eye) { + // Overlay faces +Z; point +Z at the eye, keep +Y roughly up. + const Vec3 z = Normalize(eye - at); + const Vec3 x = Normalize(Cross({0, 1, 0}, z)); + const Vec3 y = Cross(z, x); + vr::HmdMatrix34_t m{}; + const Vec3 cols[3] = {x, y, z}; + for (int c = 0; c < 3; ++c) { + m.m[0][c] = float(cols[c].x); + m.m[1][c] = float(cols[c].y); + m.m[2][c] = float(cols[c].z); + } + m.m[0][3] = float(at.x); + m.m[1][3] = float(at.y); + m.m[2][3] = float(at.z); + return m; +} + +// Unit vector v, turned toward unit vector `center` until it's at most maxRad from it. +Vec3 PullWithin(Vec3 v, Vec3 center, double maxRad) { + if (std::acos(std::clamp(Dot(v, center), -1.0, 1.0)) <= maxRad) return v; + Vec3 axis = Cross(center, v); + if (Length(axis) < 1e-9) axis = Cross(center, {0, 1, 0}); // opposite: any perpendicular + return RotateAbout(center, Normalize(axis), maxRad); +} + +// Unit direction with its pitch limited to +-maxDeg (AimBasis needs it off vertical). +Vec3 LimitPitch(Vec3 d, double maxDeg) { + const double pitch = std::clamp(std::asin(std::clamp(d.y, -1.0, 1.0)) * 180 / M_PI, -maxDeg, maxDeg); + return Direction(std::atan2(-d.x, -d.z) * 180 / M_PI, pitch); +} + +// SteamVR Settings page (see the top): the laser starts this far from the eye, the dot is +// drawn this far out, and the laser-catching dot sits this far out (metres). +constexpr double SETTINGS_ORIGIN = 0.25, SETTINGS_DOT = 0.6, SETTINGS_CATCHER = 8.0; + +// Whether the line of sight from `eye` along `d` crosses the SteamVR Settings page, drawn by +// the dashboard's scene-graph panel (valve.steam.gamepadui.frame.menu.N, transform t); see +// "SteamVR Settings" at the top. The page has no size in OpenVR, so this is its area as +// measured on the Frame, generously: in metres from the panel's origin, which is near the +// page's left edge, it ran from about -0.35 (the sidebar) to 1.15 across and +-0.4 up and +// down, with the transform scaled 0.369. Kept in the transform's units so it scales with it. +bool OnSettingsPage(const vr::HmdMatrix34_t &t, Vec3 eye, Vec3 d) { + const Vec3 c = Position(t), x{t.m[0][0], t.m[1][0], t.m[2][0]}, y{t.m[0][1], t.m[1][1], t.m[2][1]}, + z{t.m[0][2], t.m[1][2], t.m[2][2]}; + const double sx = Dot(x, x), sy = Dot(y, y), denom = Dot(d, z); + if (sx < 1e-9 || sy < 1e-9 || std::fabs(denom) < 1e-6) return false; + const double along = Dot(c - eye, z) / denom; + if (along <= 0) return false; + const Vec3 off = eye + d * along - c; + const double u = Dot(off, x) / sx, v = Dot(off, y) / sy; // in the transform's units + return u >= -1.35 && u <= 3.4 && std::fabs(v) <= 1.25; +} + +} // namespace + +// A pending recenter (the command, a wake, a first move): the anchor goes to the eye, aimed where the +// head faces. +void Pointer::Recenter(const Frame &frame) { + const vr::TrackedDevicePose_t &hmd = frame.Hmd(); + const auto &hm = hmd.mDeviceToAbsoluteTracking.m; + const Vec3 &eye = frame.eye; + if (recenter && hmd.bPoseIsValid) { + anchor = eye; + const Vec3 f{-hm[0][2], -hm[1][2], -hm[2][2]}; + yaw = std::atan2(-f.x, -f.z) * 180 / M_PI; + pitch = std::asin(std::clamp(f.y, -1.0, 1.0)) * 180 / M_PI; + anchored = true; + recenter = false; + followReset = true; + } +} + +// Head follow (see the top): past the leash (for the delay), ease the reference to the +// head's facing, and turn the cursor with it. Not in gaze mode: the gaze places it. +void Pointer::HeadFollow(const Frame &frame) { + const vr::TrackedDevicePose_t &hmd = frame.Hmd(); + const auto &hm = hmd.mDeviceToAbsoluteTracking.m; + const auto tnow = frame.tnow; + const Vec3 &eye = frame.eye; + if (follow && !gazeOn && active && anchored && hmd.bPoseIsValid) { + const Vec3 head = LimitPitch(Vec3{-hm[0][2], -hm[1][2], -hm[2][2]}, 85); + if (followReset) { + followRef = head, followLag = 0, leashOutSince = {}; + followReset = following = false; + } + const double dt = std::min(0.1, std::chrono::duration(tnow - followAt).count()); + const double leash = cfg.leashDeg * M_PI / 180; + const double lag = std::acos(std::clamp(Dot(followRef, head), -1.0, 1.0)); + double keep = lag; // how far the reference stays behind the head after this frame + if (leash <= 0) { + keep = 0; // head-locked + } else if (following) { + keep = lag * (cfg.leashReturn > 0 ? std::exp(-dt / cfg.leashReturn) : 0.0); + // A fast turn drags it at the leash's end; past it already (after the delay), it + // can't fall further behind, and it closes in from there without a jump. + keep = std::min(keep, std::max(leash, followLag)); + if (keep < 0.05 * M_PI / 180) keep = 0, following = false; // landed on the facing + } else if (lag > leash) { + if (leashOutSince == decltype(leashOutSince){}) leashOutSince = tnow; + if (std::chrono::duration(tnow - leashOutSince).count() >= cfg.leashDelay) + following = true, leashOutSince = {}; + } else { + leashOutSince = {}; // back inside before the delay: a glance + } + followLag = keep; + const Vec3 ref = LimitPitch(PullWithin(followRef, head, keep), 85); + if (!leftHeld && tnow >= dropHoldUntil) { + // The cursor keeps its offset from the reference (frames without roll). + Vec3 d = FromBasis(AimBasis(ref), ToBasis(AimBasis(followRef), Direction(yaw, pitch))); + d = PullWithin(Normalize(d), ref, cfg.followReach * M_PI / 180); + yaw = std::atan2(-d.x, -d.z) * 180 / M_PI; + pitch = std::clamp(std::asin(std::clamp(d.y, -1.0, 1.0)) * 180 / M_PI, -85.0, 85.0); + // The ray origin closes in on the eye as the reference does on the facing. + anchor = eye + (anchor - eye) * (leash <= 0 ? 0.0 : lag > 1e-6 ? keep / lag : following ? 0.0 : 1.0); + } + followRef = ref; + } + followAt = tnow; +} + +// Slow work, once a second: overlay handles, our device index, laser width. +void Pointer::SlowWork() { + const auto now = std::chrono::steady_clock::now(); + if (now - lastSlow > std::chrono::seconds(1)) { + lastSlow = now; + handles.clear(); + for (const auto &key : overlays.Keys()) { + if (Ignored(cfg.ignore, key)) continue; + vr::VROverlayHandle_t h; + if (overlay->FindOverlay(key.c_str(), &h) != vr::VROverlayError_None) continue; + handles[key] = h; + // Scene-graph overlays are placed absolutely. Other overlays can report no + // texture too (gamescope's app panels, placed as dashboard tabs, share theirs + // from another process), and ComputeOverlayIntersection handles those. + uint32_t tw = 0, th = 0; + overlay->GetOverlayTextureSize(h, &tw, &th); + vr::VROverlayTransformType tt = vr::VROverlayTransform_Invalid; + overlay->GetOverlayTransformType(h, &tt); + // ft-screens' panels (frametop.screen.N, floating windows with their popups, + // frametop.float.N..., the keyboard) are 0x0 and absolute too (a shared + // texture), but they're real panels of any size. + sceneGraph[key] = (tw == 0 || th == 0) && tt == vr::VROverlayTransform_Absolute && + key.rfind("frametop.", 0) != 0; + } + ours = vr::k_unTrackedDeviceIndexInvalid; + for (vr::TrackedDeviceIndex_t i = 0; i < vr::k_unMaxTrackedDeviceCount; ++i) { + char type[64] = ""; + sys->GetStringTrackedDeviceProperty(i, vr::Prop_ControllerType_String, type, sizeof type); + if (std::strcmp(type, "ft_pointer") == 0) ours = i; + } + + } + + if (now - lastVisible > std::chrono::milliseconds(50) || visible.size() != handles.size()) { + lastVisible = now; + visible.clear(); + for (const auto &[key, h] : handles) visible[key] = overlay->IsOverlayVisible(h); + } +} + +// The cursor: tilting a grabbed panel, or pointing (find the spot, draw the dot, send the ray). +void Pointer::Cursor(const Frame &frame) { + const vr::TrackedDevicePose_t &hmd = frame.Hmd(); + if (active && anchored && hmd.bPoseIsValid && tilting) { + Tilt(frame); + } else if (active && anchored && hmd.bPoseIsValid) { + const Spot spot = FindCursor(frame); + DrawDot(frame, spot); + SendRay(frame, spot); + } +} + +void Pointer::Tilt(const Frame &frame) { + const vr::TrackedDevicePose_t &hmd = frame.Hmd(); + const Vec3 &eye = frame.eye; + const vr::TrackedDevicePose_t &hmdRaw = frame.hmdRaw; + // Rotate the device around the grab point; the grabbed panel turns with it. + if (tiltStart) { + pivot = lastPoint; + tiltOrigin = lastOrigin; + tiltBasis = AimBasis(lastAim); + tiltStart = false; // angles carry on from any earlier tilt in this drag + overlay->HideOverlay(cursor); + overlay->HideOverlay(marker); + } + const double yr = tiltYaw * M_PI / 180, pr = tiltPitch * M_PI / 180; + const Vec3 up{0, 1, 0}, side = tiltBasis.x; + auto turn = [&](Vec3 v) { return RotateAbout(RotateAbout(v, side, pr), up, yr); }; + const Vec3 originStanding = pivot + turn(tiltOrigin - pivot); + const Basis b{turn(tiltBasis.x), turn(tiltBasis.y), turn(tiltBasis.z)}; + const auto &S = hmd.mDeviceToAbsoluteTracking, &R = hmdRaw.mDeviceToAbsoluteTracking; + auto toRaw = [&](Vec3 v) { return Rotate(R, RotateInverse(S, v)); }; // directions: raw <- standing + const Vec3 originRaw = Position(R) + toRaw(originStanding - eye); + double q[4]; + BasisQuat({toRaw(b.x), toRaw(b.y), toRaw(b.z)}, q); + char msg[200]; + std::snprintf(msg, sizeof msg, "posq %.5f %.5f %.5f %.6f %.6f %.6f %.6f", originRaw.x, originRaw.y, + originRaw.z, q[0], q[1], q[2], q[3]); + SendTo(out, "ft_pointer", msg); +} + +// Nearest visible overlay along a ray (frozen while dragging). +Pointer::Hit Pointer::Nearest(Vec3 from, Vec3 d) { + Hit h; + for (const auto &[key, handle] : handles) { + if (!visible[key]) continue; + if (sceneGraph[key]) { + // Plane test: overlay origin and its +Z normal, within sceneRadius of the origin. + vr::ETrackingUniverseOrigin uo; + vr::HmdMatrix34_t t{}; + if (overlay->GetOverlayTransformAbsolute(handle, &uo, &t) != vr::VROverlayError_None) continue; + const Vec3 center = Position(t), normal{t.m[0][2], t.m[1][2], t.m[2][2]}; + const double denom = Dot(d, normal); + if (std::fabs(denom) < 1e-4) continue; + const double along = Dot(center - from, normal) / denom; + const Vec3 at = from + d * along; + if (along > 0.05 && along < h.along && std::sqrt(Dot(at - center, at - center)) <= cfg.sceneRadius) + h.along = along, h.key = key, h.scene = true, h.point = at, h.normal = normal; + continue; + } + vr::VROverlayIntersectionParams_t params{}; + params.vSource = {float(from.x), float(from.y), float(from.z)}; + params.vDirection = {float(d.x), float(d.y), float(d.z)}; + params.eOrigin = vr::TrackingUniverseStanding; + vr::VROverlayIntersectionResults_t r{}; + if (overlay->ComputeOverlayIntersection(handle, ¶ms, &r) && r.fDistance > 0.05f && + r.fDistance < h.along) { + h.along = r.fDistance, h.key = key, h.scene = false; + h.point = {r.vPoint.v[0], r.vPoint.v[1], r.vPoint.v[2]}; + h.normal = {r.vNormal.v[0], r.vNormal.v[1], r.vNormal.v[2]}; + } + } + return h; +} + +// Where the cursor is this frame: on a panel, a scene-graph plane, or a panel's edge, or out in free +// space. +Pointer::Spot Pointer::FindCursor(const Frame &frame) { + const auto tnow = frame.tnow; + const Vec3 &eye = frame.eye; + const bool dragging = leftHeld || tnow < dropHoldUntil; + if (!dragging) tiltYaw = tiltPitch = 0; // drop finished: back to plain pointing + const Vec3 dir = Direction(yaw, pitch); + Hit first; + if (!dragging) first = Nearest(anchor, dir); + double best = first.along; + std::string bestKey = first.key; + bool bestScene = first.scene; + Vec3 bestPoint = first.point, bestNormal = first.normal; + bool onEdge = false; + if (!dragging && best < 1e8 && !bestScene) { + edgeKey = bestKey, edgePoint = bestPoint, edgeNormal = Normalize(bestNormal), edgeLast = bestPoint; + } else if (!dragging && best >= 1e8 && !edgeKey.empty() && visible[edgeKey]) { + // Just off a panel: stay on its plane (see "Panel edges" at the top). + const double denom = Dot(dir, edgeNormal); + if (std::fabs(denom) > 1e-4) { + const double along = Dot(edgePoint - anchor, edgeNormal) / denom; + const Vec3 at = anchor + dir * along; + if (along > 0.05 && std::sqrt(Dot(at - edgeLast, at - edgeLast)) <= cfg.edgeReach) { + best = along; + bestKey = edgeKey; + onEdge = true; + } + } + } + // Held on one of ft-screens' panels that stays where it is: onto whichever of them + // the ray meets (see the top). + if (leftHeld && !pressKey.empty()) { + vr::ETrackingUniverseOrigin uo; + vr::HmdMatrix34_t now{}; + auto it = handles.find(pressKey); + bool still = it != handles.end() && + overlay->GetOverlayTransformAbsolute(it->second, &uo, &now) == vr::VROverlayError_None; + for (int i = 0; still && i < 3; ++i) + for (int j = 0; j < 4; ++j) + if (std::fabs(now.m[i][j] - pressPose.m[i][j]) > 0.001f) still = false; + if (still) { + Hit h; + for (const auto &[key, handle] : handles) { + if (!visible[key] || !FramePanel(key)) continue; + vr::VROverlayIntersectionParams_t params{}; + params.vSource = {float(anchor.x), float(anchor.y), float(anchor.z)}; + params.vDirection = {float(dir.x), float(dir.y), float(dir.z)}; + params.eOrigin = vr::TrackingUniverseStanding; + vr::VROverlayIntersectionResults_t r{}; + if (overlay->ComputeOverlayIntersection(handle, ¶ms, &r) && r.fDistance > 0.05f && + r.fDistance < h.along) + h.along = r.fDistance, h.key = key; + } + if (h.along < 1e8) dragDistance = h.along, lastHit = h.key; + } else { + pressKey.clear(); // carried: the lock holds for the rest of this press + } + } + // While dragging: keep the press-time distance and show the non-interactive marker. + // On a scene-graph plane or a panel's edge: the laser-catching dot goes 5 cm behind it. + double distance = dragging ? dragDistance : (best < 1e8 ? best : cfg.freeDistance); + Vec3 point = anchor + dir * distance; + bool occluded = false; + if (!dragging) { + // The cursor lands on what you see under it: the ray above starts at the anchor, + // not the eye, so after leaning it can pick a panel that something nearer + // covers from where you are now (panels close together in view, at different + // depths). Anything in front of the point on the eye's line of sight wins. + const double toPoint = std::sqrt(Dot(point - eye, point - eye)); + const Hit front = Nearest(eye, Normalize(point - eye)); + if (front.along < toPoint - 0.02) { + occluded = true; + bestKey = front.key, bestScene = front.scene; + point = front.point; + if (!front.scene) edgeKey = front.key, edgePoint = front.point, edgeNormal = Normalize(front.normal), + edgeLast = front.point; + distance = std::sqrt(Dot(point - anchor, point - anchor)); + best = distance; + onEdge = false; + } + lastDistance = distance, lastHit = bestKey; + } + const bool onScene = !dragging && ((bestScene && best < 1e8) || onEdge); + const bool onPanel = dragging || (best < 1e8 && !onScene); + Spot s; + s.dragging = dragging, s.dir = dir, s.point = point, s.best = best, s.distance = distance; + s.onEdge = onEdge, s.occluded = occluded, s.onScene = onScene, s.onPanel = onPanel; + return s; +} + +// Our dot where the cursor is (see "Looks" and "SteamVR Settings" at the top). +void Pointer::DrawDot(const Frame &frame, const Spot &spot) { + const auto tnow = frame.tnow; + const Vec3 &eye = frame.eye; + const bool dragging = spot.dragging, onScene = spot.onScene, onPanel = spot.onPanel; + const Vec3 dir = spot.dir, point = spot.point; + const Vec3 sight = Normalize(point - eye); + if (!dragging) { + // SteamVR Settings (see the top): the dashboard's main panel is hidden and its + // scene-graph panel shows the page. + onVrSettings = false; + const auto mainIt = visible.find("valve.steam.gamepadui.main"); + if (overlay->IsDashboardVisible() && !(mainIt != visible.end() && mainIt->second)) { + for (const auto &[key, handle] : handles) { + if (!visible[key] || key.rfind("valve.steam.gamepadui.frame.menu.", 0) != 0) continue; + vr::ETrackingUniverseOrigin uo; + vr::HmdMatrix34_t t{}; + if (overlay->GetOverlayTransformAbsolute(handle, &uo, &t) == vr::VROverlayError_None && + OnSettingsPage(t, eye, sight)) + onVrSettings = true; + } + } + } + if (onVrSettings != catcherHidesHit) { + overlay->SetOverlayFlag(cursor, vr::VROverlayFlags_HideLaserIntersection, onVrSettings); + catcherHidesHit = onVrSettings; + } + + if (onVrSettings) { + // The dot close in front of the page, which is nearer than any guess of ours; + // the laser-catching dot far behind everything, invisible, so it never covers + // the page, with SteamVR's hit dot hidden on it. + const Vec3 near = eye + sight * SETTINGS_DOT, far = eye + sight * SETTINGS_CATCHER; + double alpha = 1; + if (gazeOn && !cfg.gazeDotAlways) { + auto secs = [&](Clock::time_point t) { return std::chrono::duration(tnow - t).count(); }; + alpha = std::clamp(1 - std::min(secs(lastMove) - cfg.gazeShow, secs(lastHeld)) / 0.25, 0.0, 1.0); + } + if (tnow < calPanelUntil) alpha = 0; + overlay->SetOverlayAlpha(marker, float(alpha)); + overlay->SetOverlayWidthInMeters(marker, float(2 * SETTINGS_DOT * std::tan(cfg.cursorDeg * M_PI / 360))); + auto mm = Billboard(near, eye); + overlay->SetOverlayTransformAbsolute(marker, vr::TrackingUniverseStanding, &mm); + overlay->SetOverlayAlpha(cursor, 0); + overlay->SetOverlayWidthInMeters(cursor, float(2 * SETTINGS_CATCHER * std::tan(cfg.cursorDeg * M_PI / 360))); + auto mc = Billboard(far, eye); + overlay->SetOverlayTransformAbsolute(cursor, vr::TrackingUniverseStanding, &mc); + overlay->ShowOverlay(marker); + overlay->ShowOverlay(cursor); + } else { + // On a panel: the non-interactive marker, pulled 5 mm toward the eye so it + // draws on top. In free space: the interactive dot the laser lands on. + const vr::VROverlayHandle_t show = onPanel ? marker : cursor, hide = onPanel ? cursor : marker; + const Vec3 at = onPanel ? point + Normalize(eye - point) * 0.005 : onScene ? point + dir * 0.05 : point; + const double dist = std::sqrt(Dot(at - eye, at - eye)); + // Gaze mode: a pulse for each click, and with POINTER_GAZE_DOT=moving, shown only + // while something moves it or a press holds it (see the top); transparent + // otherwise, the laser still lands on it. + double scale = 1, alpha = 1; + if (gazeOn) { + auto secs = [&](Clock::time_point t) { return std::chrono::duration(tnow - t).count(); }; + alpha = cfg.gazeDotAlways ? 1.0 : std::clamp(1 - std::min(secs(lastMove) - cfg.gazeShow, secs(lastHeld)) / 0.25, 0.0, 1.0); + const double pulse = secs(pulseAt); + if (pulse < 0.6) { + scale = 1 + 1.5 * std::max(0.0, 1 - pulse / 0.3); + alpha = std::max(alpha, std::clamp((0.6 - pulse) / 0.3, 0.0, 1.0)); + } + } + if (tnow < calPanelUntil) alpha = 0; // the calibration panel is up (see the top) + overlay->SetOverlayAlpha(show, float(alpha)); + overlay->SetOverlayWidthInMeters(show, float(2 * dist * std::tan(scale * cfg.cursorDeg * M_PI / 360))); + auto m = Billboard(at, eye); + overlay->SetOverlayTransformAbsolute(show, vr::TrackingUniverseStanding, &m); + overlay->ShowOverlay(show); + overlay->HideOverlay(hide); + } +} + +void Pointer::SendRay(const Frame &frame, const Spot &spot) { + const vr::TrackedDevicePose_t &hmd = frame.Hmd(); + const auto tnow = frame.tnow; + const Vec3 &eye = frame.eye; + const vr::TrackedDevicePose_t &hmdRaw = frame.hmdRaw; + const bool dragging = spot.dragging, occluded = spot.occluded, onEdge = spot.onEdge, onScene = spot.onScene; + const double best = spot.best, distance = spot.distance; + const Vec3 point = spot.point; + // Controller ray: from the eye, aimed at the cursor point, converted from the + // standing universe to raw tracking space via the HMD's pose in both. + const Vec3 aimStanding = Normalize(point - eye); + const auto &S = hmd.mDeviceToAbsoluteTracking, &R = hmdRaw.mDeviceToAbsoluteTracking; + const Vec3 aim = Rotate(R, RotateInverse(S, aimStanding)); // raw <- head <- standing + // Origin partway along the line of sight to the cursor (smaller hit dot). + const double toPoint = std::sqrt(Dot(point - eye, point - eye)); + double originDist = std::max(0.0, std::min(toPoint * cfg.originFraction, toPoint - cfg.originMargin)); + if (onVrSettings) originDist = std::min(originDist, SETTINGS_ORIGIN); // SteamVR finds the page + const Vec3 originStanding = eye + Normalize(point - eye) * originDist; + const Vec3 eyeRaw = Position(R) + Rotate(R, RotateInverse(S, originStanding - eye)); + lastPoint = point, lastOrigin = originStanding, lastAim = aimStanding; // tilt starts from here + havePoint = true; + if (debug && tnow - lastDebug > std::chrono::milliseconds(500)) { + lastDebug = tnow; + if (systemPointer == vr::k_ulOverlayHandleInvalid) overlay->FindOverlay("system.pointer", &systemPointer); + std::printf("dbg %s hit=%s dist=%.2f eye->point=%.2f origin=%.2f vrsettings=%d yaw=%.1f pitch=%.1f gaze=%s steamvr_dot=%d primary=%u\n", + dragging ? "DRAG" : occluded ? "INFRONT" : onEdge ? "EDGE" : onScene ? "SCENE" : (best < 1e8 ? "PANEL" : "FREE"), lastHit.empty() ? "-" : lastHit.c_str(), + distance, toPoint, originDist, onVrSettings, yaw, pitch, + !gazeOn ? "off" : tnow - gz.at > std::chrono::milliseconds(150) ? "stale" : gazeOwns ? "owns" : "mouse", + systemPointer != vr::k_ulOverlayHandleInvalid && overlay->IsOverlayVisible(systemPointer), + overlay->GetPrimaryDashboardDevice()); + std::fflush(stdout); + } + const double ayaw = std::atan2(-aim.x, -aim.z) * 180 / M_PI; + const double apitch = std::asin(std::clamp(aim.y, -1.0, 1.0)) * 180 / M_PI; + if (dragging && (tiltYaw != 0 || tiltPitch != 0)) { + // Keep this drag's tilt applied, about the current cursor point. + const double yr = tiltYaw * M_PI / 180, pr = tiltPitch * M_PI / 180; + const Basis base = AimBasis(aimStanding); + const Vec3 up{0, 1, 0}, side = base.x; + auto turn = [&](Vec3 v) { return RotateAbout(RotateAbout(v, side, pr), up, yr); }; + const Vec3 o = point + turn(originStanding - point); + const Basis b{turn(base.x), turn(base.y), turn(base.z)}; + auto toRaw = [&](Vec3 v) { return Rotate(R, RotateInverse(S, v)); }; + const Vec3 oRaw = Position(R) + toRaw(o - eye); + double q[4]; + BasisQuat({toRaw(b.x), toRaw(b.y), toRaw(b.z)}, q); + char msg[200]; + std::snprintf(msg, sizeof msg, "posq %.5f %.5f %.5f %.6f %.6f %.6f %.6f", oRaw.x, oRaw.y, oRaw.z, q[0], + q[1], q[2], q[3]); + SendTo(out, "ft_pointer", msg); + } else { + char msg[160]; + std::snprintf(msg, sizeof msg, "pose %.5f %.5f %.5f %.4f %.4f", eyeRaw.x, eyeRaw.y, eyeRaw.z, ayaw, apitch); + SendTo(out, "ft_pointer", msg); + } +} diff --git a/pointer/helper/pointer_gaze.cpp b/pointer/helper/pointer_gaze.cpp new file mode 100644 index 0000000..fb1ee7b --- /dev/null +++ b/pointer/helper/pointer_gaze.cpp @@ -0,0 +1,91 @@ +// ft-pointer: gaze mode and its held-back press (see "Gaze mode" at the top). +// "The top" in comments here is the header comment of ft-pointer.cpp. +#include "pointer.h" + +// Outside games, gaze mode keeps the pointer (see the top); the relay needs to know. +void Pointer::GazeAwake() { + const auto t = Clock::now(); + if (t - inGameAt > std::chrono::milliseconds(500)) { + inGameAt = t; + inGame = vr::VRApplications()->GetCurrentSceneProcessId() != 0; + } + // Hand gestures in the last two minutes keep it the same way. + const bool handsRecent = cfg.handsOn && handUsed != Clock::time_point{} && t - handUsed < std::chrono::minutes(2); + const bool awake = (gazeOn || handsRecent) && !inGame && !headsetOff; + if (awake != gazeAwake || t - gazeAwakeAt > std::chrono::seconds(5)) { + if (awake != gazeAwake) + std::printf("%s keeps the pointer: %s\n", gazeOn ? "gaze" : "hand use", + awake ? "yes" : "no (off, in a game, or headset off)"); + if (awake != gazeAwake) std::fflush(stdout); + gazeAwake = awake; + gazeAwakeAt = t; + SendTo(out, "frametop_relay", awake ? "gazeawake 1" : "gazeawake 0"); + } +} + +// Gaze mode (see the top): the gaze has the pointer, or takes it back when you look +// well away from it. Not while a press holds the pointer, and only on fresh gaze. +void Pointer::GazeMode(const Frame &frame) { + const vr::TrackedDevicePose_t &hmd = frame.Hmd(); + const auto tnow = frame.tnow; + const Vec3 &eye = frame.eye; + if (hmd.bPoseIsValid) lastHead = hmd.mDeviceToAbsoluteTracking, haveHead = true; + if (gazeOn && active && hmd.bPoseIsValid && !tilting && !leftHeld && !aimHeld && !clickPress && !clickRelease && + tnow >= dropHoldUntil && + tnow - gz.at < std::chrono::milliseconds(150)) { + const Vec3 g = Rotate(hmd.mDeviceToAbsoluteTracking, Direction(gz.hy, gz.hp)); + if (!gazeOwns && havePoint) { + const double off = std::acos(std::clamp(Dot(g, Normalize(lastPoint - eye)), -1.0, 1.0)) * 180 / M_PI; + if (off > cfg.gazeRetake && tnow - lastMouse > std::chrono::milliseconds(300)) { + if (retakeSince == Clock::time_point{}) retakeSince = tnow; + if (tnow - retakeSince >= std::chrono::milliseconds(120)) gazeOwns = true, nudging = false; + } else { + retakeSince = {}; + } + } + if (gazeOwns) { + retakeSince = {}; + anchor = eye; + anchored = true; + yaw = std::atan2(-g.x, -g.z) * 180 / M_PI; + pitch = std::clamp(std::asin(std::clamp(g.y, -1.0, 1.0)) * 180 / M_PI, -85.0, 85.0); + } + } +} + +void Pointer::CalPanelOpened() { + if (calOpened) { // the calibration panel came up: a press in progress ends, no click + calOpened = false; + if (hold.src != Src::None) EndHold(true); + if (leftHeld) ReleaseLeft(); + aimHeld = aimHand = clickPress = false; + } +} + +// A held-back press (see the top): held still long enough, it's a real press (a drag); +// released, it's a click where the pointer is now (this frame's pose has gone out). +void Pointer::HeldBackPress(const Frame &frame) { + const auto tnow = frame.tnow; + if (!active) aimHeld = aimRight = clickPress = aimHand = confirmLesson = false; // released meanwhile: nothing to click + if (aimHeld && !aimHand && nudgeMoved < 0.2 && tnow - aimSince >= std::chrono::duration(cfg.gazeHold)) { + aimHeld = false; + gazeBack = true; + pressRight = aimRight; // the right button's: a real right press (see the top) + aimRight = false; + PressLeft(); + } +} + +// The click a held-back press ends in: the press now, the release 40 ms later. +void Pointer::Click(const Frame &frame) { + const auto tnow = frame.tnow; + if (clickPress) { + clickPress = false; + PressLeft(); + clickRelease = true; + clickReleaseAt = tnow + std::chrono::milliseconds(40); + } else if (clickRelease && tnow >= clickReleaseAt) { + clickRelease = false; + ReleaseLeft(); + } +} diff --git a/pointer/helper/pointer_holds.cpp b/pointer/helper/pointer_holds.cpp new file mode 100644 index 0000000..9a6c09c --- /dev/null +++ b/pointer/helper/pointer_holds.cpp @@ -0,0 +1,202 @@ +// ft-pointer: holds, hands, and the gaze precision buttons (see "Gaze precision" and "Hands" at +// the top). +// "The top" in comments here is the header comment of ft-pointer.cpp. +#include "pointer.h" + +// Hands (see the top): pinches and grips from ft-hands. +void Pointer::Hands(const Frame &frame) { + const auto tnow = frame.tnow; + vr::TrackedDevicePose_t h0; + sys->GetDeviceToAbsoluteTrackingPose(vr::TrackingUniverseStanding, 0, &h0, 1); + if (h0.bPoseIsValid) poses.Add(tnow, h0.mDeviceToAbsoluteTracking); +} + +// Where a gesture's point is, seen from the hold's origin: yaw and pitch, degrees. +bool Pointer::HandAngles(const float p[3], uint64_t t_ns, const Vec3 &from, double &hy, double &hp) { + vr::HmdMatrix34_t head; + if (!poses.At(t_ns, head)) return false; + const Vec3 d = Normalize(Position(head) + Rotate(head, Vec3{p[0], p[1], p[2]}) - from); + hy = std::atan2(-d.x, -d.z) * 180 / M_PI; + hp = std::asin(std::clamp(d.y, -1.0, 1.0)) * 180 / M_PI; + return true; +} + +void Pointer::EndHold(bool lost) { + if (hold.pressed) { + if (leftHeld) ReleaseLeft(); + } else if (aimHeld) { + aimHeld = aimHand = false; + if (lost) { + if (gazeOn) gazeOwns = true, nudging = false; // no click: the pointer goes back to the gaze + } else { + clickPress = true; // the click is on the release, where the pointer is now + pressRight = hold.right; + gazeBack = nudgeMoved < 0.2; + } + } + if (debug) + std::printf("hold %s %s%s\n", hold.src == Src::Hand ? (hold.grip ? "grip" : "pinch") + : hold.src == Src::Head ? (hold.right ? "key right" : "key left") + : hold.grip ? "gaze drag" : "gaze precision", + lost ? "lost" : "released", hold.pressed ? "" : lost ? " (no click)" : " (click)"); + if (debug) std::fflush(stdout); + hold = {}; +} + +// The press, once a hold's source is set: a real press (dragging until the release), or +// held back like gaze mode's mouse press (see the top): the click comes on the release. +void Pointer::StartHold(bool press, Clock::time_point tnow) { + hold.startYaw = hold.lastYaw = yaw, hold.startPitch = hold.lastPitch = pitch; + hold.pressed = press; + if (press) { + gazeBack = gazeOn && gazeOwns; + PressLeft(); + } else { + gazeOwns = false; + nudging = gazeOn && haveHead && tnow - gz.at < std::chrono::milliseconds(200); + nudgeRawHy = gz.rhy, nudgeRawHp = gz.rhp, nudgeHead = lastHead; + nudgeAt = aimSince = tnow, nudgeMoved = 0; + aimHeld = aimHand = true; + pulseAt = tnow; + } +} + +// The hold's source moved to (hy, hp): past the dead zone, the pointer follows at the gain, +// from where it was when it got past. +void Pointer::Steer(double hy, double hp, double deadzone, double gain, Clock::time_point tnow) { + const double dy = std::remainder(hy - hold.refYaw, 360.0), dp = hp - hold.refPitch; + if (!hold.engaged) { + if (std::hypot(dy, dp) <= deadzone) return; + hold.engaged = true; + hold.refYaw = hy, hold.refPitch = hp; + hold.startYaw = hold.lastYaw = yaw, hold.startPitch = hold.lastPitch = pitch; + return; + } + yaw = std::remainder(hold.startYaw + gain * dy, 360.0); + pitch = std::clamp(hold.startPitch + gain * dp, -85.0, 85.0); + // How far the nudge went: for a keyboard click, from where the dot was at the press + // (a head wobbles on the way); otherwise the path, as with the mouse. + if (!hold.pressed && hold.src == Src::Head) + nudgeMoved = std::hypot(std::remainder(yaw - hold.pressYaw, 360.0), pitch - hold.pressPitch); + else if (!hold.pressed) + nudgeMoved += std::hypot(std::remainder(yaw - hold.lastYaw, 360.0), pitch - hold.lastPitch); + hold.lastYaw = yaw, hold.lastPitch = pitch; + lastMove = tnow; // the dot shows while it moves (gaze mode) +} + +void Pointer::BeginHold(int side, bool grip, const fh_pinch_t &g, Clock::time_point tnow) { + handUsed = tnow; + if (hold.src != Src::None) { + // A grip takes over a pinch on the way to it (closing the hand passes through + // one); otherwise one hold at a time. + if (hold.src != Src::Hand || !grip || hold.grip) return; + aimHeld = aimHand = false; + hold = {}; + } + if (leftHeld || aimHeld || clickPress || clickRelease) return; // the mouse's button is busy + if (!active) { + Wake(tnow); // the first gesture only wakes the pointer, like the first mouse move + return; + } + vr::HmdMatrix34_t head; + if (!poses.At(g.begin_ns, head)) return; + // No pinch just after a key (typing touches thumb to index), and a grip only with + // the hand held up (hands on a desk curl like a loose fist; looking down at them + // puts them straight ahead in the head's frame, where ft-hands can't tell). + const Vec3 at = Position(head) + Rotate(head, Vec3{g.begin_point[0], g.begin_point[1], g.begin_point[2]}); + if (!grip && tnow - lastTyping < std::chrono::duration(cfg.typingHold)) { + if (debug) std::printf("hand pinch ignored: typing\n"); + return; + } + if (grip && Position(head).y - at.y > cfg.handBelow) { + if (debug) std::printf("hand grip ignored: %.2f m below the eyes\n", Position(head).y - at.y); + return; + } + Hold h; + h.src = Src::Hand, h.side = side, h.grip = grip, h.origin = Position(head); + if (!HandAngles(g.begin_point, g.begin_ns, h.origin, h.refYaw, h.refPitch)) return; + hold = h; + // A grip, or a pinch without gaze mode: a real press where the pointer is (where + // you look, in gaze mode), dragging with the hand until it opens. + StartHold(grip || !gazeOn, tnow); + if (debug) std::printf("hand %s %s began\n", side ? "right" : "left", grip ? "grip" : "pinch"); + if (debug) std::fflush(stdout); +} + +// Gaze precision and gaze drag buttons (see the top): the mouse steers. +void Pointer::GazePrecisionButtons(const Frame &frame) { + const auto tnow = frame.tnow; + for (const DevicePress &p : devicePresses) { + if (p.source == "left" || p.source == "right") continue; // no controllers in gaze mode + if (!p.down) { + if (hold.src == Src::Mouse) EndHold(false); + continue; + } + if (hold.src != Src::None || leftHeld || aimHeld || clickPress || clickRelease) continue; + Hold h; + h.src = Src::Mouse, h.grip = p.drag; + hold = h; + StartHold(p.drag, tnow); + if (debug) std::printf("hold gaze %s began (%s)\n", p.drag ? "drag" : "precision", p.source.c_str()); + if (debug) std::fflush(stdout); + } + devicePresses.clear(); +} + +// Pinches and grips from ft-hands (see "Hands" at the top), read every frame. +void Pointer::PinchesAndGrips(Frame &frame) { + const auto tnow = frame.tnow; + fh_gestures_t hg; + const bool handOk = cfg.handsOn && handFile.Read(hg); + frame.handOk = handOk; + if (handOk && (hg.seq != handSeq || handFile.opens != handOpens)) { + handSeq = hg.seq; + handPublished = hg.publish_ns; + const fh_pinch_t *slots[2] = {hg.pinch, hg.grip}; + // A new file, or a tracker whose counters went back: start counting from here. + bool rebase = !handBaseline || handFile.opens != handOpens; + for (int k = 0; k < 2; ++k) + for (int s = 0; s < 2; ++s) + rebase = rebase || slots[k][s].begins < seenBegins[k][s] || slots[k][s].ends < seenEnds[k][s]; + if (rebase) { + if (hold.src == Src::Hand) EndHold(true); + for (int k = 0; k < 2; ++k) + for (int s = 0; s < 2; ++s) seenBegins[k][s] = slots[k][s].begins, seenEnds[k][s] = slots[k][s].ends; + handBaseline = true, handOpens = handFile.opens; + } + // Not over a VR game (unless the dashboard is up), and not with the headset off. + const bool allowed = !headsetOff && (!inGame || overlay->IsDashboardVisible()); + for (int k = 1; k >= 0; --k) // grips first: one takes over a pinch + for (int s = 0; s < 2; ++s) { + const fh_pinch_t &g = slots[k][s]; + const bool began = g.begins != seenBegins[k][s], ended = g.ends != seenEnds[k][s]; + const bool lost = g.flags & FH_PINCH_LOST; + seenBegins[k][s] = g.begins, seenEnds[k][s] = g.ends; + auto holding = [&] { return hold.src == Src::Hand && hold.side == s && hold.grip == (k == 1); }; + if (holding() && ended) EndHold(lost); // the one held ended (another may have begun) + if (began && allowed) { + BeginHold(s, k == 1, g, tnow); + // begun and ended since the last read (a quick tap): its release too + if (!(g.flags & FH_PINCH_DOWN) && holding()) EndHold(lost); + } + } + // The held gesture's hand moves the pointer: past the dead zone (a tap's jitter, + // the pinch point shifting as the fingers close), then at the gain, from there. + if (hold.src == Src::Hand) { + const fh_pinch_t &g = hold.grip ? hg.grip[hold.side] : hg.pinch[hold.side]; + double hy, hp; + if ((g.flags & FH_PINCH_TRACKED) && HandAngles(g.point, hg.capture_ns, hold.origin, hy, hp)) + Steer(hy, hp, hold.grip ? 0.5 : cfg.pinchDeadzone, hold.grip ? cfg.gripGain : cfg.pinchGain, tnow); + } + } +} + +// ft-hands stopped (or hung) with a gesture held: it's over, as lost. +void Pointer::HandsStopped(const Frame &frame) { + const auto tnow = frame.tnow; + const bool handOk = frame.handOk; + const uint64_t nowNs = + uint64_t(std::chrono::duration_cast(tnow.time_since_epoch()).count()); + if (hold.src == Src::Hand && (!handOk || !active || nowNs - handPublished > 1'500'000'000ull)) EndHold(true); + if (hold.src != Src::None && hold.src != Src::Hand && !active) EndHold(true); +} diff --git a/pointer/helper/pointer_input.cpp b/pointer/helper/pointer_input.cpp new file mode 100644 index 0000000..830b545 --- /dev/null +++ b/pointer/helper/pointer_input.cpp @@ -0,0 +1,390 @@ +// ft-pointer: waking the pointer, its buttons, and the commands from the relay. +// "The top" in comments here is the header comment of ft-pointer.cpp. +#include "pointer.h" + +void Pointer::Wake(Clock::time_point t) { + if (t < noWakeUntil || headsetOff) return; + wokeAt = t; + active = true; + recenter = true; + SendTo(out, "ft_pointer", "role " + cfg.role); // POINTER_ROLE, before it takes it + SendTo(out, "ft_pointer", "show"); + claimPending = true; // take the laser without clicking, once SteamVR has bound the device + claimAt = t + std::chrono::milliseconds(300); +} + +void Pointer::PressLeft() { + // ft-screens sends the keyboard to the panel clicked last; it sees clicks on + // its own screens, but only we know when one lands on another panel. + SendTo(out, "ft_screens", "click " + (lastHit.empty() ? std::string("-") : lastHit)); + // A click after nudging the gaze-placed pointer: the nudge is a lesson, or past + // POINTER_GAZE_NUDGE_MAX, a quick check (see the top). + if (gazeOn && nudging && !gazeOwns && havePoint && Clock::now() - nudgeAt < std::chrono::seconds(10) && + (confirmLesson || nudgeMoved >= 0.2)) { + const Vec3 d = RotateInverse(nudgeHead, Normalize(lastPoint - Position(nudgeHead))); + const double ty = std::atan2(-d.x, -d.z) * 180 / M_PI, tp = std::asin(std::clamp(d.y, -1.0, 1.0)) * 180 / M_PI; + const double off = std::hypot(std::remainder(ty - nudgeRawHy, 360.0), tp - nudgeRawHp); + char msg[160]; + if (off <= cfg.gazeNudgeMax) + std::snprintf(msg, sizeof msg, "lesson %.3f %.3f %.3f %.3f", nudgeRawHy, nudgeRawHp, ty, tp); + else + std::snprintf(msg, sizeof msg, "recheck %.0f", off); + SendTo(out, "ft_gazed", msg); + if (debug) std::printf("gaze %s (nudged %.2f deg, off %.2f)\n", msg, nudgeMoved, off); + if (debug) std::fflush(stdout); + } + nudging = confirmLesson = false; + leftHeld = true; + dragDistance = lastDistance; + pressKey.clear(); + if (FramePanel(lastHit)) { + vr::ETrackingUniverseOrigin uo; + auto it = handles.find(lastHit); + if (it != handles.end() && + overlay->GetOverlayTransformAbsolute(it->second, &uo, &pressPose) == vr::VROverlayError_None) + pressKey = lastHit; + } + tiltYaw = tiltPitch = 0; // a new drag starts untilted + dropHoldUntil = {}; + pulseAt = Clock::now(); + heldButton = pressRight ? "b" : "trigger"; + pressRight = false; + SendTo(out, "ft_pointer", "btn " + heldButton + " 1"); +} + +void Pointer::ReleaseLeft() { + leftHeld = false; + tilting = false; + // Hold the drag pose (tilt, frozen distance) while SteamVR finishes the drop. + dropHoldUntil = Clock::now() + std::chrono::milliseconds(500); + SendTo(out, "ft_pointer", "btn " + heldButton + " 0"); + // ft-screens releases a button held on its screens in KWin even when SteamVR hands + // the release to some other overlay (its catcher usually gets it; this is the backstop). + SendTo(out, "ft_screens", "up"); + if (gazeBack) gazeOwns = true, gazeBack = false; +} + +// The left button, from the relay's "btn trigger", with gaze mode's held-back press (see +// the top). +bool Pointer::CanAim() { + return gazeOn && cfg.gazeMousePrecision && gazeOwns && !aimHeld && !clickPress && !clickRelease && + hold.src == Src::None; +} + +// Hold the press back: the pointer stops where the gaze put it. +void Pointer::AimStart(bool right) { + gazeOwns = false; + nudging = haveHead && Clock::now() - gz.at < std::chrono::milliseconds(200); + nudgeRawHy = gz.rhy, nudgeRawHp = gz.rhp, nudgeHead = lastHead; + nudgeAt = aimSince = Clock::now(), nudgeMoved = 0; + aimHeld = true, aimRight = right; +} + +// A drag the right button began (chordDrag): it lasts while either button is held. +void Pointer::ChordDrop() { + chordDrag = false; + if (leftHeld) ReleaseLeft(); + if (debug) std::printf("mouse drag dropped\n"); + if (debug) std::fflush(stdout); +} + +void Pointer::LeftButton(bool down) { + leftDown = down; + if (down) { + if (aimHeld && aimRight && !aimHand) { + ignoreLeftUp = true; // the right's press is held back: the left does nothing + return; + } + if (CanAim()) { + AimStart(false); + return; + } + PressLeft(); + return; + } + if (ignoreLeftUp) { + ignoreLeftUp = false; + return; + } + if (chordDrag) { + if (!rightDown) ChordDrop(); // otherwise the right holds it (tilting, maybe) + return; + } + if (aimHeld && !aimHand && !aimRight) { + aimHeld = false; + clickPress = true; // after this frame's pose, so it lands where the pointer was moved to + gazeBack = nudgeMoved < 0.2; + return; + } + if (leftHeld) ReleaseLeft(); +} + +// The right button (see the top); false: not taken here (a tilt, or passed on as it is). +bool Pointer::RightButton(bool down) { + rightDown = down; + if (down) { + if (aimHeld && !aimHand && !aimRight) { + // During the left's held-back press: press the left where the pointer is now (the + // correction is a lesson), and the mouse drags. + aimHeld = false; + chordDrag = gazeBack = true; + PressLeft(); + if (debug) std::printf("mouse drag began (right during left)\n"); + if (debug) std::fflush(stdout); + return true; + } + if (CanAim() && !leftHeld) { + AimStart(true); + return true; + } + return false; + } + if (chordDrag) { + if (leftDown) return !swallowedRight; // the left holds it; a tilt's release ends the tilt + tilting = swallowedRight = false; + ChordDrop(); + return true; + } + if (aimHeld && !aimHand && aimRight) { + aimHeld = aimRight = false; + pressRight = clickPress = true; // the right click, where the pointer was moved to + gazeBack = nudgeMoved < 0.2; + return true; + } + if (leftHeld && heldButton == "b") { // its press, held still into a real one + ReleaseLeft(); + return true; + } + return false; +} + +// POINTER_GAZE_MOUSE_MOVE=held (see the top): the gaze is fresh and nothing is pressed, so a +// mouse move doesn't move the pointer. +bool Pointer::MouseMoveHeld() { + return gazeOn && cfg.gazeMouseHeld && !inGame && !headsetOff && Clock::now() - gz.at < std::chrono::seconds(1) && + !aimHeld && !leftHeld && !tilting && hold.src == Src::None && !clickPress && !clickRelease; +} + +// Commands from the relay. +void Pointer::RelayCommands() { + char buf[256]; + ssize_t n; + sockaddr_un from{}; + socklen_t fromLen = sizeof from; + while ((n = recvfrom(in, buf, sizeof buf - 1, 0, reinterpret_cast(&from), &fromLen)) > 0) { + buf[n] = 0; + // Reply to the sender (the layout tool binds an abstract address to get answers). + const sockaddr_un sender = from; + const socklen_t senderLen = fromLen; + fromLen = sizeof from; + Command(buf, sender, senderLen); + } +} + +// One command from the relay (or another sender: reply answers it). +void Pointer::Command(const char *buf, const sockaddr_un &sender, socklen_t senderLen) { + auto reply = [&](const std::string &msg) { + if (senderLen > offsetof(sockaddr_un, sun_path)) + sendto(out, msg.data(), msg.size(), 0, reinterpret_cast(&sender), senderLen); + }; + // The gaze, from ft-gazed: not mouse input, it never wakes the pointer. + double g[4]; + if (std::sscanf(buf, "gz %lf %lf %lf %lf", &g[0], &g[1], &g[2], &g[3]) == 4) { + gz = {g[0], g[1], g[2], g[3], Clock::now()}; + return; + } + // The gaze calibration panel (see the top): presses answer it instead of clicking. + { + int on; + if (std::sscanf(buf, "calpanel %d", &on) == 1) { + const bool was = Clock::now() < calPanelUntil; + calPanelUntil = on ? Clock::now() + std::chrono::seconds(3) : Clock::time_point{}; + if (on && !was) calOpened = true; + return; + } + } + if (Clock::now() < calPanelUntil) { + const bool accept = !std::strncmp(buf, "btn trigger 1", 13) || !std::strncmp(buf, "gazekey left 1", 14); + const bool quit = !std::strncmp(buf, "btn b 1", 7) || !std::strncmp(buf, "gazekey right 1", 15); + if (accept || quit) { + SendTo(out, "ft_gazed", accept ? "calaccept" : "calquit"); + return; + } + if (!std::strncmp(buf, "btn ", 4) || !std::strncmp(buf, "gazekey ", 8) || + !std::strncmp(buf, "precision ", 10) || !std::strncmp(buf, "gazedrag ", 9)) + return; // their releases, and the other buttons: nothing to click now + } + { + char kind[16], source[16]; + int v; + if (std::sscanf(buf, "%15s %15s %d", kind, source, &v) == 3 && + (!std::strcmp(kind, "precision") || !std::strcmp(kind, "gazedrag"))) { + lastMouse = Clock::now(); + if (!active) Wake(Clock::now()); + devicePresses.push_back({source, !std::strcmp(kind, "gazedrag"), v != 0}); + return; + } + if (std::sscanf(buf, "gazekey %15s %d", source, &v) == 2 && + (!std::strcmp(source, "left") || !std::strcmp(source, "right"))) { + lastMouse = Clock::now(); + if (!active) Wake(Clock::now()); + keyPresses.push_back({!std::strcmp(source, "right"), v != 0}); + return; + } + } + if (std::strcmp(buf, "typing") == 0) { // not mouse input: it never wakes the pointer + lastTyping = Clock::now(); + return; + } + if (std::strncmp(buf, "vrbind", 6) == 0) { + std::printf("controller buttons: %s\n", controllerButtons.Bind(buf + 6).c_str()); + std::fflush(stdout); + return; + } + if (std::strncmp(buf, "vrglobal", 8) == 0) { + const char *arg = buf + 8; + while (*arg == ' ') ++arg; + ControllerButtons::SetGlobal(std::strncmp(arg, "off", 3) != 0); + reply(controllerButtons.Status()); + return; + } + if (std::strncmp(buf, "vrstatus", 8) == 0) { + reply(controllerButtons.Status()); + return; + } + if (std::strncmp(buf, "overlays", 8) == 0) { + overlays.Request(sender, senderLen); // answered from the list's thread + return; + } + if (std::strncmp(buf, "gaze", 4) == 0) { + const char *arg = buf + 4; + while (*arg == ' ') ++arg; + if (*arg != '?') { // "gaze ?" only asks + gazeOn = std::strncmp(arg, "on", 2) == 0 ? true + : std::strncmp(arg, "off", 3) == 0 ? false + : !gazeOn; + gazeOwns = true, nudging = false; + std::printf("gaze mode %s\n", gazeOn ? "on" : "off"); + std::fflush(stdout); + } + reply(gazeOn ? "ok on" : "ok off"); + return; + } + const bool mouseInput = std::strncmp(buf, "move", 4) == 0 || std::strncmp(buf, "btn", 3) == 0 || + std::strncmp(buf, "scroll", 6) == 0; + // A move held back (POINTER_GAZE_MOUSE_MOVE=held) only wakes the pointer: it isn't using + // the mouse, so a drifting mouse doesn't keep the gaze from taking the pointer back. + const bool moveHeldBack = std::strncmp(buf, "move", 4) == 0 && MouseMoveHeld(); + if (mouseInput && !moveHeldBack) lastMouse = Clock::now(); + // Any mouse input wakes the pointer (after a controller took over, or a helper restart). + if (!active && mouseInput) Wake(Clock::now()); + if (moveHeldBack) return; + double a, b; + char key[128]; + double px, py, pz, pyaw, ppitch, proll = 0, pgrab = -1; + if (std::sscanf(buf, "grabprobe %127s", key) == 1) { + GrabProbe(key); + return; + } + if (std::sscanf(buf, "place %127s %lf %lf %lf %lf %lf %lf %lf", key, &px, &py, &pz, &pyaw, &ppitch, &proll, + &pgrab) >= 6) { + reply(Place(key, {px, py, pz}, PanelBasis(pyaw, ppitch, proll), pgrab >= 0 ? pgrab : cfg.grabOffset)); + return; + } + if (std::sscanf(buf, "measure %127s", key) == 1) { + Panel p; + Vec3 eye; + const std::string err = FindPanel(key, p, eye); + char msg[400] = ""; + if (err.empty()) + std::snprintf(msg, sizeof msg, "ok %.4f %.4f %.4f %.4f %.4f %.4f %.4f %.4f %.4f %.4f %.4f %.4f %.4f %.4f", + p.center.x, p.center.y, p.center.z, p.width, p.height, p.basis.x.x, p.basis.x.y, + p.basis.x.z, p.basis.y.x, p.basis.y.y, p.basis.y.z, p.basis.z.x, p.basis.z.y, + p.basis.z.z); + reply(err.empty() ? msg : "error " + err); + return; + } + if (std::strncmp(buf, "head", 4) == 0) { + vr::TrackedDevicePose_t h; + sys->GetDeviceToAbsoluteTrackingPose(vr::TrackingUniverseStanding, 0, &h, 1); + const Vec3 e = Position(h.mDeviceToAbsoluteTracking); + const Vec3 f = Rotate(h.mDeviceToAbsoluteTracking, {0, 0, -1}); + char msg[200]; + std::snprintf(msg, sizeof msg, "ok %.4f %.4f %.4f %.2f %.2f", e.x, e.y, e.z, + std::atan2(-f.x, -f.z) * 180 / M_PI, std::asin(std::clamp(f.y, -1.0, 1.0)) * 180 / M_PI); + reply(h.bPoseIsValid ? msg : "error no head pose (headset off?)"); + return; + } + if (std::strncmp(buf, "debug", 5) == 0) { + debug = !debug; + std::printf("debug %s\n", debug ? "on" : "off"); + std::fflush(stdout); + return; + } + if (std::strncmp(buf, "btn trigger 1", 13) == 0) { + LeftButton(true); + return; + } else if (std::strncmp(buf, "btn trigger 0", 13) == 0) { + LeftButton(false); + return; + } else if (std::strncmp(buf, "btn b 1", 7) == 0 && RightButton(true)) { + return; + } else if (std::strncmp(buf, "btn b 0", 7) == 0 && RightButton(false)) { + return; + } else if (std::strncmp(buf, "btn b 1", 7) == 0 && leftHeld) { + tilting = tiltStart = swallowedRight = true; // right press while dragging: tilt, no right-click + return; + } else if (std::strncmp(buf, "btn b 0", 7) == 0 && swallowedRight) { + tilting = swallowedRight = false; + return; + } + if (tilting && std::sscanf(buf, "move %lf %lf", &a, &b) == 2) { + tiltYaw += a; + tiltPitch = std::clamp(tiltPitch + b, -80.0, 80.0); + return; + } + if (std::sscanf(buf, "move %lf %lf", &a, &b) == 2) { + lastMove = Clock::now(); // the dot shows while the mouse moves it (gaze mode) + if (gazeOn && gazeOwns) { + // The mouse takes the pointer from the gaze, from where the gaze left it. + gazeOwns = false; + nudging = haveHead && Clock::now() - gz.at < std::chrono::milliseconds(200); + nudgeRawHy = gz.rhy, nudgeRawHp = gz.rhp, nudgeHead = lastHead; + nudgeAt = Clock::now(), nudgeMoved = 0; + } + if (nudging || aimHeld) nudgeMoved += std::hypot(a, b); + if (!anchored) recenter = true; + yaw += a; + while (yaw > 180) yaw -= 360; + while (yaw < -180) yaw += 360; + pitch = std::clamp(pitch + b, -85.0, 85.0); + } else if (std::strncmp(buf, "recenter", 8) == 0) { + recenter = true; + } else if (std::strncmp(buf, "reload", 6) == 0) { + cfg.Load(ReadConfig()); + ApplyConfig(); + lastSlow = Clock::now() - std::chrono::seconds(10); // apply POINTER_IGNORE now + std::printf("reloaded: free distance %.2f m, dot %.2f deg, origin %.2f, head follow %s, leash %.0f deg, " + "controller pickup %.1fx, %zu ignored\n", + cfg.freeDistance, cfg.cursorDeg, cfg.originFraction, follow ? "on" : "off", cfg.leashDeg, cfg.pickupScale, + cfg.ignore.size()); + std::fflush(stdout); + } else if (std::strncmp(buf, "follow", 6) == 0) { + const char *arg = buf + 6; + while (*arg == ' ') ++arg; + const bool was = follow; + follow = std::strncmp(arg, "on", 2) == 0 ? true : std::strncmp(arg, "off", 3) == 0 ? false : !follow; + if (follow && !was) followReset = true; + std::printf("head follow %s (leash %.0f deg)\n", follow ? "on" : "off", cfg.leashDeg); + std::fflush(stdout); + } else if (std::strncmp(buf, "show", 4) == 0) { + if (!active) Wake(Clock::now()); + } else if (std::strncmp(buf, "hide", 4) == 0) { + active = false; + overlay->HideOverlay(cursor); + overlay->HideOverlay(marker); + SendTo(out, "ft_pointer", "hide"); + } else { + SendTo(out, "ft_pointer", buf); // btn, scroll + } +} diff --git a/pointer/helper/pointer_keys.cpp b/pointer/helper/pointer_keys.cpp new file mode 100644 index 0000000..67565be --- /dev/null +++ b/pointer/helper/pointer_keys.cpp @@ -0,0 +1,113 @@ +// ft-pointer: keyboard clicks (see "Keyboard clicks" under "Gaze precision" at the top). +// "The top" in comments here is the header comment of ft-pointer.cpp. +#include "pointer.h" + +// Where the head faces: yaw and pitch, degrees. +bool Pointer::HeadAngles(const vr::TrackedDevicePose_t &hmd, double &hy, double &hp) { + const auto &hm = hmd.mDeviceToAbsoluteTracking.m; + if (!hmd.bPoseIsValid) return false; + const Vec3 f = Normalize({-hm[0][2], -hm[1][2], -hm[2][2]}); + hy = std::atan2(-f.x, -f.z) * 180 / M_PI; + hp = std::asin(std::clamp(f.y, -1.0, 1.0)) * 180 / M_PI; + return true; +} + +// Keyboard clicks (see the top): held back at the gaze, steered by the head. +void Pointer::KeyboardClicks(const Frame &frame) { + const vr::TrackedDevicePose_t &hmd = frame.Hmd(); + const auto tnow = frame.tnow; + for (const KeyPress &k : keyPresses) { + (k.right ? keyRightDown : keyLeftDown) = k.down; + if (!k.down) { + if (keyTilting && k.right) { // the tilt ends; the head drags again from here + keyTilting = tilting = false; + hold.engaged = false; + if (debug) std::printf("hold key left: tilt ended\n"); + if (debug) std::fflush(stdout); + } + // The keys holding it: a gaze_left then gaze_right drag, either; otherwise its own. + const bool held = hold.keyDrag ? keyLeftDown || keyRightDown : hold.right ? keyRightDown : keyLeftDown; + if (hold.src == Src::Head && !held) { + if (!hold.pressed && aimHeld && tnow - aimSince < std::chrono::duration(cfg.keyTap)) { + // A quick tap: a click where the dot was at the press, and the gaze was right. + yaw = hold.pressYaw, pitch = hold.pressPitch; + nudgeMoved = 0; + confirmLesson = true; + } + EndHold(false); + keyTilting = false; + } + continue; + } + if (k.right && hold.src == Src::Head && !hold.right && hold.pressed && leftHeld) { + // gaze_right during a gaze_left drag: tilt while it's held (see the top). + tilting = tiltStart = keyTilting = true; + HeadAngles(hmd, keyTiltYaw, keyTiltPitch); + if (debug) std::printf("hold key left: tilt began\n"); + if (debug) std::fflush(stdout); + continue; + } + if (k.right && hold.src == Src::Head && !hold.right && !hold.pressed && aimHeld) { + // gaze_right while gaze_left aims: press the left button where the dot is now + // (the correction is a lesson), then the head drags. + aimHeld = aimHand = false; + hold.pressed = hold.grip = hold.keyDrag = true, hold.promote = false, hold.engaged = false; + HeadAngles(hmd, hold.refYaw, hold.refPitch); + gazeBack = gazeOn; + pressRight = false; + PressLeft(); + if (debug) std::printf("hold key left: pressed (right key)\n"); + if (debug) std::fflush(stdout); + continue; + } + if (hold.src != Src::None || leftHeld || aimHeld || clickPress || clickRelease) continue; + Hold h; + h.src = Src::Head, h.promote = true, h.right = k.right, h.pressYaw = yaw, h.pressPitch = pitch; + if (!HeadAngles(hmd, h.refYaw, h.refPitch)) continue; + hold = h; + StartHold(false, tnow); + if (debug) std::printf("hold key %s began\n", k.right ? "right" : "left"); + if (debug) std::fflush(stdout); + } + keyPresses.clear(); +} + +// A keyboard click's hold: the head steers the pointer, or turns the panel in a tilt. +void Pointer::HeadSteer(const Frame &frame) { + const vr::TrackedDevicePose_t &hmd = frame.Hmd(); + const auto tnow = frame.tnow; + if (hold.src == Src::Head) { + double hy, hp; + if (HeadAngles(hmd, hy, hp) && keyTilting) { + // The head turns the panel, as the mouse does in a tilt; the pointer stays put. + tiltYaw += std::remainder(hy - keyTiltYaw, 360.0); + tiltPitch = std::clamp(tiltPitch + hp - keyTiltPitch, -80.0, 80.0); + keyTiltYaw = hy, keyTiltPitch = hp; + } else if (HeadAngles(hmd, hy, hp)) { + Steer(hy, hp, hold.pressed ? 0.3 : cfg.headDeadzone, 1.0, tnow); + } + } +} + +// A keyboard click held still for POINTER_GAZE_HOLD: a real press, then the head drags +// (from where it is now). +void Pointer::KeyboardClickHold(const Frame &frame) { + const vr::TrackedDevicePose_t &hmd = frame.Hmd(); + const auto &hm = hmd.mDeviceToAbsoluteTracking.m; + const auto tnow = frame.tnow; + if (aimHeld && aimHand && hold.promote && !hold.engaged && + tnow - aimSince >= std::chrono::duration(cfg.gazeHold)) { + aimHeld = aimHand = false; + hold.pressed = hold.grip = true; + if (hmd.bPoseIsValid) { + const Vec3 f = Normalize({-hm[0][2], -hm[1][2], -hm[2][2]}); + hold.refYaw = std::atan2(-f.x, -f.z) * 180 / M_PI; + hold.refPitch = std::asin(std::clamp(f.y, -1.0, 1.0)) * 180 / M_PI; + } + gazeBack = gazeOn; + pressRight = hold.right; + PressLeft(); + if (debug) std::printf("hold key %s: pressed (held still)\n", hold.right ? "right" : "left"); + if (debug) std::fflush(stdout); + } +} diff --git a/pointer/helper/pointer_place.cpp b/pointer/helper/pointer_place.cpp new file mode 100644 index 0000000..1bae9ce --- /dev/null +++ b/pointer/helper/pointer_place.cpp @@ -0,0 +1,168 @@ +// ft-pointer: panel placement for layouts (see "Placement" at the top). +// "The top" in comments here is the header comment of ft-pointer.cpp. +#include "pointer.h" + +namespace { + +void SleepMs(int ms) { std::this_thread::sleep_for(std::chrono::milliseconds(ms)); } + +} // namespace + +// Placement speeds (see "Placement" at the top). +constexpr double kPlaceDegPerSec = 60; // tested: 40 deg/s is applied exactly + +// --- Panel placement (see "Placement" at the top of the file) --- +// Device pose, given in the standing universe, sent to the driver in raw space. +void Pointer::SendPose(Vec3 originStanding, const Basis &b) { + vr::TrackedDevicePose_t s, r; + sys->GetDeviceToAbsoluteTrackingPose(vr::TrackingUniverseStanding, 0, &s, 1); + sys->GetDeviceToAbsoluteTrackingPose(vr::TrackingUniverseRawAndUncalibrated, 0, &r, 1); + const auto &S = s.mDeviceToAbsoluteTracking, &R = r.mDeviceToAbsoluteTracking; + auto toRaw = [&](Vec3 v) { return Rotate(R, RotateInverse(S, v)); }; + const Vec3 o = Position(R) + toRaw(originStanding - Position(S)); + double q[4]; + BasisQuat({toRaw(b.x), toRaw(b.y), toRaw(b.z)}, q); + char msg[200]; + std::snprintf(msg, sizeof msg, "posq %.5f %.5f %.5f %.6f %.6f %.6f %.6f", o.x, o.y, o.z, q[0], q[1], q[2], q[3]); + SendTo(out, "ft_pointer", msg); +} + +std::pair Pointer::HeadPos() { + vr::TrackedDevicePose_t s; + sys->GetDeviceToAbsoluteTrackingPose(vr::TrackingUniverseStanding, 0, &s, 1); + return std::make_pair(s.bPoseIsValid, Position(s.mDeviceToAbsoluteTracking)); +} + +// Borrow the device and the laser for a placement: connect, claim, laser mode on. +void Pointer::Borrow() { + SendTo(out, "ft_pointer", "show"); + overlay->ShowOverlay(laserMode); + overlay->HideOverlay(cursor); + overlay->HideOverlay(marker); + SleepMs(active ? 50 : 400); // a fresh connect needs SteamVR to bind the device + SendTo(out, "ft_pointer", "btn a 1"); + SleepMs(60); + SendTo(out, "ft_pointer", "btn a 0"); +} + +void Pointer::GiveBack() { + if (!active) { + SendTo(out, "ft_pointer", "hide"); + overlay->HideOverlay(laserMode); + laserModeShown = false; + } +} + +std::string Pointer::FindPanel(const char *key, Panel &p, Vec3 &eye) { + vr::VROverlayHandle_t h; + if (overlay->FindOverlay(key, &h) != vr::VROverlayError_None) return std::string("no overlay ") + key; + bool valid; + std::tie(valid, eye) = HeadPos(); + if (!valid) return "no head pose (headset off?)"; + p = ScanPanel(h, eye, 0.5); + if (!p.found) return std::string("panel not visible: ") + key; + return ""; +} + +// grabprobe (maintenance): aim down from the panel's bottom edge, 1 cm a step, and log where +// SteamVR's laser hits something (the hit dot shows) and whether the window controls are up. +void Pointer::GrabProbe(const char *key) { + Panel p; + Vec3 eye; + const std::string err = FindPanel(key, p, eye); + if (!err.empty()) { + std::printf("grabprobe: %s\n", err.c_str()); + std::fflush(stdout); + return; + } + Borrow(); + vr::VROverlayHandle_t footer = vr::k_ulOverlayHandleInvalid; + overlay->FindOverlay("valve.steam.gamepadui.floatingfooter", &footer); + const Vec3 bottom = p.center - p.basis.y * (p.height / 2); + std::printf("grabprobe %s: center (%.3f %.3f %.3f) %.3f x %.3f m\n", key, p.center.x, p.center.y, p.center.z, + p.width, p.height); + for (int cm = -5; cm <= 45; ++cm) { + const Vec3 target = bottom - p.basis.y * (cm / 100.0); + SendPose(eye, AimBasis(target - eye)); + SleepMs(90); + const bool dot = systemPointer != vr::k_ulOverlayHandleInvalid && overlay->IsOverlayVisible(systemPointer); + const bool foot = footer != vr::k_ulOverlayHandleInvalid && overlay->IsOverlayVisible(footer); + std::printf(" %+3d cm below the bottom edge: steamvr_dot=%d footer=%d", cm, dot, foot); + if (foot) { + vr::ETrackingUniverseOrigin uo; + vr::HmdMatrix34_t t{}; + if (overlay->GetOverlayTransformAbsolute(footer, &uo, &t) == vr::VROverlayError_None) { + const Vec3 f = Position(t) - p.center; + std::printf(" footer at panel (%.3f %.3f %.3f)", Dot(f, p.basis.x), Dot(f, p.basis.y), + Dot(f, p.basis.z)); + } + } + std::printf("\n"); + } + std::fflush(stdout); + GiveBack(); +} + +// Carry a floating panel so its centre lands on `target` with frame `bt` (see +// "Placement" at the top). Returns "ok ..." or "error ...". +std::string Pointer::Place(const char *key, Vec3 target, const Basis &bt, double grabBelow) { + Panel p; + Vec3 eye; + std::string err = FindPanel(key, p, eye); + if (!err.empty()) return "error " + err; + auto offBy = [&](const Panel &q, double &cm, double °) { + cm = Length(q.center - target) * 100; + const double c = (Dot(q.basis.x, bt.x) + Dot(q.basis.y, bt.y) + Dot(q.basis.z, bt.z) - 1) / 2; + deg = std::acos(std::clamp(c, -1.0, 1.0)) * 180 / M_PI; + }; + double cm, deg; + int moves = 0; + Borrow(); + for (int attempt = 0; attempt < 3; ++attempt) { + offBy(p, cm, deg); + if (cm < 1.5 && deg < 1.0) break; + // The rigid motion that takes the panel to the target: rotate by R, then move. + auto turn = [&](Vec3 v) { return FromBasis(bt, ToBasis(p.basis, v)); }; + double q[4]; + BasisQuat({turn({1, 0, 0}), turn({0, 1, 0}), turn({0, 0, 1})}, q); + const double angle = 2 * std::acos(std::clamp(q[0], -1.0, 1.0)); + const Vec3 axis = std::sin(angle / 2) > 1e-6 ? Normalize({q[1], q[2], q[3]}) : Vec3{0, 1, 0}; + const Vec3 grab = p.center - p.basis.y * (p.height / 2 + grabBelow); + const Basis d0 = AimBasis(grab - eye); + const Vec3 o1 = target + turn(eye - p.center); // device origin at the end + ++moves; + SendPose(eye, d0); + SleepMs(150); // hover: the window controls come up + SendTo(out, "ft_pointer", "btn trigger 1"); + SleepMs(150); + // 1. Rotate about the device origin (the eye): the dashboard applies it exactly. + const int rsteps = std::max(4, int(angle * 180 / M_PI / kPlaceDegPerSec * 60)); + for (int i = 1; i <= rsteps; ++i) { + const double a = angle * i / rsteps; + auto r = [&](Vec3 v) { return RotateAbout(v, axis, a); }; + SendPose(eye, {r(d0.x), r(d0.y), r(d0.z)}); + SleepMs(16); + } + // 2. Slide the device slowly: fast moves are accelerated by the dashboard. + const Basis d1{turn(d0.x), turn(d0.y), turn(d0.z)}; + const int tsteps = std::max(4, int(Length(o1 - eye) / cfg.slideSpeed * 60)); + for (int i = 1; i <= tsteps; ++i) { + SendPose(eye + (o1 - eye) * (double(i) / tsteps), d1); + SleepMs(16); + } + SleepMs(100); + SendTo(out, "ft_pointer", "btn trigger 0"); + SleepMs(600); // the dashboard re-reads the pose up to 150 ms after the release + err = FindPanel(key, p, eye); + if (!err.empty()) break; + } + GiveBack(); + if (!err.empty()) return "error after the move: " + err; + offBy(p, cm, deg); + char msg[160]; + std::snprintf(msg, sizeof msg, "ok %s off by %.1f cm, %.1f deg after %d move%s", key, cm, deg, moves, + moves == 1 ? "" : "s"); + std::printf("place: %s\n", msg); + std::fflush(stdout); + return msg; +} diff --git a/scripts/update-check.py b/scripts/update-check.py index e16a32b..3c23ece 100755 --- a/scripts/update-check.py +++ b/scripts/update-check.py @@ -285,11 +285,11 @@ def check_openvr(): def check_overlays(desktop_up): code, out = run(f"{STEAMVR_BIN}/vrcmd", "--overlays", env=dict(os.environ, LD_LIBRARY_PATH=STEAMVR_BIN)) - # The format ft-pointer.cpp and ft_layout.py parse: 'key' -- 'name', WxH visible VROverlayType_... + # The format the pointer helper (pointer.h) and ft_layout.py parse: 'key' -- 'name', WxH visible VROverlayType_... keys = re.findall(r"^'([^']+)' -- '.*VROverlayType_", out, re.M) if not keys: report("FAIL", "vrcmd --overlays", "lists no overlays in the expected format, so the 3D mouse can't " - "find panels (pointer/helper/ft-pointer.cpp and layout/ft_layout.py parse it)") + "find panels (pointer/helper/pointer.h and layout/ft_layout.py parse it)") elif desktop_up and not any(re.fullmatch(r"frametop\.screen\.\d+", k) for k in keys): report("FAIL", "vrcmd --overlays", f"lists {len(keys)} overlays, but none of the desktop's screens") else: