From 5ca4160a7e1edca6ada8470bf78965e83c4a12ea Mon Sep 17 00:00:00 2001
From: DeeJanuz <45082401+DeeJanuz@users.noreply.github.com>
Date: Wed, 30 Sep 2026 15:07:58 -0600
Subject: [PATCH] Hands: fixes from the second headset test (pinches)

- The palm-down filter is off by default: the user's deliberate pinches,
  hand raised, read 0.90-0.99, like typing.
- Typing is told apart by the keyboard instead: the input relay sends the
  pointer helper "typing" on key presses, and it takes no pinch within
  POINTER_PINCH_TYPING (1 s) of one.
- Grips are still held to hands raised (POINTER_GRIP_BELOW, 0.35 m below
  the eyes); pinches aren't, since the user's own sat 0.35-0.45 m below,
  elbow resting.
- A grip doesn't begin with the thumb on the index tip: that's a pinch
  with the other fingers curled, which was taken for a grip.
- A pinch's point is the index and middle knuckles: the tips' midpoint
  moved 1-2 cm as the pinch opened, dragging every release off its press.
- ft-hands --gesture-log prints what the detectors measure, 10 times a
  second.

Pinches still aren't reliable enough to use; hand tracking is parked for
now in favour of the controllers.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 hands/include/fh_gestures.h   |  7 ++++---
 hands/track/main.cpp          | 14 ++++++++++++--
 hands/track/pinch.cpp         | 23 +++++++++++++++--------
 hands/track/pinch.h           |  9 ++++++++-
 input/input-relay.py          |  6 ++++++
 pointer/helper/ft-pointer.cpp | 33 +++++++++++++++++++++++----------
 session/frametop.conf.example |  3 ++-
 7 files changed, 70 insertions(+), 25 deletions(-)

diff --git a/hands/include/fh_gestures.h b/hands/include/fh_gestures.h
index 5a843fa..445e1ce 100644
--- a/hands/include/fh_gestures.h
+++ b/hands/include/fh_gestures.h
@@ -10,7 +10,8 @@
  * One slot per side and gesture: pinch[0] and grip[0] are the left hand, [1] the right.
  * A gesture follows the hand it began on until it ends.
  *   Pinch: begins when the thumb and index tips close within begin_m and ends when they
- * open past end_m (the gap between keeps it from flickering). point is between the tips.
+ * open past end_m (the gap between keeps it from flickering). point is the index and middle
+ * knuckles, which don't move as the fingers open and close (the tips' midpoint did).
  *   Grip: a closed hand. It begins when all four fingers are curled in (each fingertip
  * nearer the wrist than grip_begin times its knuckle is) and ends when they open past
  * grip_end on average. distance is that average (about 2 open, under 1.2 closed), strength
@@ -55,8 +56,8 @@ typedef struct {
     float    distance;          /* pinch: thumb tip to index tip, m, at this user's */
                                 /* hand size. grip: the fingers' mean curl (above)  */
     float    strength;          /* 0 open .. 1 closed                               */
-    float    point[3];          /* pinch: between the thumb and index tips; grip:   */
-                                /* the palm's centre                                */
+    float    point[3];          /* pinch: the index and middle knuckles; grip: the  */
+                                /* palm's centre                                    */
     float    begin_point[3];    /* point when the current or last one began         */
 } fh_pinch_t;                   /* 64 bytes */
 
diff --git a/hands/track/main.cpp b/hands/track/main.cpp
index 40aedf0..af8a029 100644
--- a/hands/track/main.cpp
+++ b/hands/track/main.cpp
@@ -177,6 +177,7 @@ int main(int argc, char **argv) {
     double keep_presence = 0.5;   // landmark presence a tracked view needs to stay
     PinchParams pinch_params;
     GripParams grip_params;
+    bool gesture_log = false;   // what the pinch and grip detectors measure, 10 times a second
     double record_for = 120;
     for (int i = 1; i < argc; ++i) {
         const std::string a = argv[i];
@@ -194,6 +195,7 @@ int main(int argc, char **argv) {
         else if (a == "--pinch-palm-down" && more) pinch_params.palm_down_max = std::atof(argv[++i]);
         else if (a == "--grip-begin" && more) grip_params.begin = std::atof(argv[++i]);
         else if (a == "--grip-end" && more) grip_params.end = std::atof(argv[++i]);
+        else if (a == "--gesture-log") gesture_log = true;
         else if (a == "--swap-sides") swap_sides = true;
         else if (a == "--record-only") track = publish = false;
         else if (a == "--ring" && more) ring_path = argv[++i];
@@ -220,7 +222,7 @@ int main(int argc, char **argv) {
                         "          [--cams auto|mono|color|all] (auto) [--bright all|color] (all) [--bright-on L] (40) [--bright-off L] (25)\n"
                         "          [--color-left color_video0|color_video3] [--color-crop subtract|none]\n"
                         "          [--pinch-begin M] (0.020) [--pinch-end M] (0.035) [--pinch-triangulated] [--pinch-palm-down MAX] (0.6)\n"
-                        "          [--grip-begin R] (1.2) [--grip-end R] (1.45)\n"
+                        "          [--grip-begin R] (1.2) [--grip-end R] (1.45) [--gesture-log]\n"
                         "          [--contrast MODE|PALM/HAND] (clahe[:CLIP], none, stretch; default clahe:2/none)\n"
                         "Recording saves every frame set for S seconds (120) to DIR/sets.bin, for ft-handreplay; SIGUSR1\n"
                         "starts one in ~/.local/share/frametop/hands/rec-<time>. --record-only records without tracking, so it\n"
@@ -339,7 +341,7 @@ int main(int argc, char **argv) {
     std::map<std::string, uint64_t> last;       // per camera: the frame last used
     std::map<std::string, uint64_t> lit_seen;   // per colour camera: the frame last counted for the light
     const uint64_t start = mono_ns();
-    uint64_t t_status = start, next_ns = 0, t_want = 0;
+    uint64_t t_status = start, next_ns = 0, t_want = 0, t_glog = 0;
     double cpu0 = cpu_seconds();
     std::vector<double> lat;
     double hands_sum = 0, resid_sum = 0;
@@ -541,6 +543,14 @@ int main(int argc, char **argv) {
         const std::vector<Seen> views = tracker.views_now();
         grip.update(hands, views, int64_t(capture));
         pinch.update(hands, views, int64_t(capture), grip.gripping());
+        if (gesture_log && capture - t_glog >= 100'000'000) {
+            t_glog = capture;
+            for (int k = 0; k < 2; ++k)
+                if (pinch.world_d[k] >= 0)
+                    std::printf("gesture %s d world %.3f tri %.3f palm-down %.2f curl %.2f%s\n", k ? "right" : "left ",
+                                pinch.world_d[k], pinch.tri_d[k], pinch.palm_down[k], grip.curl[k],
+                                pinch.side(k).flags & FH_PINCH_DOWN ? " PINCH" : grip.side(k).flags & FH_PINCH_DOWN ? " GRIP" : "");
+        }
         // a gesture down or closing gets the full rate, even while the palm holds still
         next_ns = tmin + uint64_t((std::min(tracker.interval(), pinch.engaged() || grip.engaged() ? 1 / 30.0 : 1.0) -
                                    0.005) * 1e9);
diff --git a/hands/track/pinch.cpp b/hands/track/pinch.cpp
index afc0859..665580d 100644
--- a/hands/track/pinch.cpp
+++ b/hands/track/pinch.cpp
@@ -65,7 +65,11 @@ void Pinch::update(const std::vector<const Hand *> &hands, const std::vector<See
         }
         if (n) world_d[s] = sum / n * h->scale;
         const double d = p_.triangulated || world_d[s] < 0 ? tri_d[s] : world_d[s];
-        const V3 point = (h->smooth[kThumbTip] + h->smooth[kIndexTip]) * 0.5;
+        // Where the pinch is, for drags: the index and middle knuckles, which hold still while
+        // the fingers open and close. The point between the tips moved 1-2 cm as a pinch
+        // opened, so every release dragged the pointer off what it pressed (headset test,
+        // 2026-09-30).
+        const V3 point = (h->smooth[5] + h->smooth[9]) * 0.5;
         o.flags |= FH_PINCH_TRACKED;
         o.hand_id = uint32_t(h->id);
         o.distance = float(d);
@@ -126,10 +130,11 @@ namespace {
 
 constexpr int kWrist = 0, kFingers[4][2] = {{5, 8}, {9, 12}, {13, 16}, {17, 20}};   // knuckle, tip
 
-// Each finger's curl (see GripParams): from the model's world landmarks averaged over the
-// hand's views this step, or the tracker's 3D points without any.
-bool curls(const Hand &h, const std::vector<Seen> &views, double out[4]) {
-    double sum[4] = {};
+// Each finger's curl (see GripParams), and the thumb tip's distance from the index tip (m):
+// from the model's world landmarks averaged over the hand's views this step, or the
+// tracker's 3D points without any.
+bool curls(const Hand &h, const std::vector<Seen> &views, double out[4], double *thumb) {
+    double sum[4] = {}, gap = 0;
     int n = 0;
     for (const Seen &v : views) {
         if (v.hand != h.id) continue;
@@ -138,8 +143,10 @@ bool curls(const Hand &h, const std::vector<Seen> &views, double out[4]) {
             const double k = norm(p(kFingers[f][0]) - p(kWrist));
             sum[f] += k > 1e-4 ? norm(p(kFingers[f][1]) - p(kWrist)) / k : 2;
         }
+        gap += norm(p(4) - p(8)) * h.scale;
         ++n;
     }
+    *thumb = n ? gap / n : norm(h.smooth[4] - h.smooth[8]);
     for (int f = 0; f < 4; ++f) {
         if (n) {
             out[f] = sum[f] / n;
@@ -172,8 +179,8 @@ void Grip::update(const std::vector<const Hand *> &hands, const std::vector<Seen
             if (!h || c->frames > h->frames) h = c;
         }
         curl[s] = -1;
-        double f[4];
-        if (!h || !curls(*h, views, f)) {
+        double f[4], thumb = 1;
+        if (!h || !curls(*h, views, f, &thumb)) {
             o.flags &= ~FH_PINCH_TRACKED;
             if (down && (t_ns - seen_ns_[s]) / 1e9 > p_.grace_s) end(s, t_ns, true);
             continue;
@@ -191,7 +198,7 @@ void Grip::update(const std::vector<const Hand *> &hands, const std::vector<Seen
             if (mean > p_.end) open_ns_[s] = t_ns;
             const bool ahead = -point[2] >= p_.min_ahead_m &&
                                std::atan2(-point[1], -point[2]) * 180 / M_PI <= p_.max_down_deg;
-            if (most < p_.begin && ahead && open_ns_[s] && (t_ns - open_ns_[s]) / 1e9 <= p_.armed_s) {
+            if (most < p_.begin && ahead && thumb >= p_.thumb_off_m && open_ns_[s] && (t_ns - open_ns_[s]) / 1e9 <= p_.armed_s) {
                 o.flags = (o.flags | FH_PINCH_DOWN) & ~FH_PINCH_LOST;
                 ++o.begins;
                 o.begin_ns = uint64_t(t_ns);
diff --git a/hands/track/pinch.h b/hands/track/pinch.h
index eadbecb..0d47e41 100644
--- a/hands/track/pinch.h
+++ b/hands/track/pinch.h
@@ -27,7 +27,10 @@ struct PinchParams {
     // head frame; 1 turns it off). Typing curls the thumb onto the index: in the 2026-09-30
     // lit recording, pinches that began while typing had 0.69-1.00, deliberate ones 0.00-0.50.
     // Looking down tilts the head frame, which lowers the reading for a hand on a keyboard.
-    double palm_down_max = 0.6;
+    // Off by default since the first headset test (2026-09-30 14:55): the user's deliberate
+    // pinches, hand raised in front, read 0.90-0.99 too. The pointer helper now leaves out
+    // gestures that begin low (hands on a desk), which it can tell with the head's pose.
+    double palm_down_max = 1.0;
 };
 
 class Pinch {
@@ -87,6 +90,10 @@ struct GripParams {
     // deliberate pinches 5-15.
     double max_down_deg = 35;
     double min_ahead_m = 0.15;
+    // ...and not with the thumb on the index fingertip, closer than this (m, the pinch's
+    // measure): that's a pinch with the other fingers curled, which the first headset test
+    // took for a grip.
+    double thumb_off_m = 0.03;
 };
 
 // Grip (a closed hand) detection, per side like Pinch: press and drag.
diff --git a/input/input-relay.py b/input/input-relay.py
index b3de670..99ddc89 100755
--- a/input/input-relay.py
+++ b/input/input-relay.py
@@ -34,6 +34,8 @@ reaches games; see pointer/helper/vrbuttons.h).
 
 In gaze mode outside games, the helper keeps the pointer ("gazeawake 1", repeated every 5
 seconds; "gazeawake 0" or silence ends it): the pointer isn't released when the mouse is idle.
+Typing on a keyboard sends the helper "typing" (at most 4 times a second): it takes no hand
+pinches right after a key, since typing touches thumb to index like a pinch.
 
 Keys also go to ft-screens (@ft_screens, the Frametop desktop's compositor), which
 types them into the desktop screen that has focus: from pass-through keyboards, and
@@ -614,6 +616,7 @@ def main():
     load_config()
     meta_down = False  # Meta pressed with no other key yet: a tap toggles the dashboard
     screens_sock = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM | socket.SOCK_NONBLOCK)
+    last_typing = 0.0  # the helper was last told of a key then (see "typing" at the top)
 
     def vr_bind(now):
         """Tell the pointer helper which controller buttons to take from games."""
@@ -957,6 +960,9 @@ def main():
                     # Observed only, unless typing goes to the desktop. With META_DASHBOARD=1,
                     # a Meta tap on any keyboard toggles the dashboard.
                     if node.role == "passthrough" and etype == EV_KEY:
+                        if value == 1 and code < BTN_MISC and now - last_typing >= 0.25:
+                            last_typing = now
+                            screens_sock.sendto(b"typing", HELPER)
                         to_screens(code, value)
                         if node.grabbed and code < BTN_MISC and value in (0, 1):
                             share_key(node, code, value)
diff --git a/pointer/helper/ft-pointer.cpp b/pointer/helper/ft-pointer.cpp
index e985c5c..e061205 100644
--- a/pointer/helper/ft-pointer.cpp
+++ b/pointer/helper/ft-pointer.cpp
@@ -180,8 +180,11 @@
 //   A grip (closing the hand) is a press and drag: the press where the pointer is, then the
 // hand moves the pointer at POINTER_GRIP_GAIN (1: as far as it moves, seen from the eye), and
 // opening the hand releases. So it drags whatever the pointer is on: a title bar moves the
-// window, a panel's grab bar carries the panel, text is selected. Only with the hand held up,
-// at most POINTER_GRIP_BELOW (0.3 m) below the eyes: hands on a desk curl like a loose fist.
+// window, a panel's grab bar carries the panel, text is selected.
+//   Typing touches thumb to index like a pinch: no pinch begins within POINTER_PINCH_TYPING
+// (1 s) of a key (the relay says "typing"). A grip begins only with the hand held up, at most
+// POINTER_GRIP_BELOW (0.35 m) below the eyes: hands on a desk curl like a loose fist. (The
+// user's own pinches sat 0.35-0.45 m below the eyes, elbow resting, so pinches have no such limit.)
 //   The hand's movement is taken in the room, from where the eye was when the gesture began,
 // with the head pose at each frame's capture time, so turning your head doesn't move it.
 // The first gesture while the pointer is off only wakes it. Gestures are ignored in a VR game
@@ -236,7 +239,7 @@
 // gaze mode, above. POINTER_CONTROLLER_PICKUP (1, 0.5 to 5): how hard a controller must
 // move to take the laser back, above. POINTER_IGNORE (empty): ignored panels, above.
 // POINTER_HANDS (1), POINTER_PINCH_GAIN (0.5), POINTER_PINCH_DEADZONE (1.5 deg),
-// POINTER_GRIP_GAIN (1), POINTER_GRIP_BELOW (0.3 m): hands, above.
+// POINTER_GRIP_GAIN (1), POINTER_GRIP_BELOW (0.35 m), POINTER_PINCH_TYPING (1 s): hands, above.
 #include <openvr.h>
 
 #include "vrbuttons.h"
@@ -646,7 +649,7 @@ int main() {
     double gazeRetake = 5, gazeNudgeMax = 8, gazeHold = 0.5, gazeShow = 1;
     // Hands (see the top): POINTER_HANDS, POINTER_PINCH_GAIN, POINTER_PINCH_DEADZONE, POINTER_GRIP_GAIN.
     bool handsOn = true;
-    double pinchGain = 0.5, pinchDeadzone = 1.5, gripGain = 1.0, gripBelow = 0.3;
+    double pinchGain = 0.5, pinchDeadzone = 1.5, gripGain = 1.0, handBelow = 0.35, typingHold = 1.0;
     double pickupScale = 1;  // POINTER_CONTROLLER_PICKUP: scales the controller-moved limits
     std::vector<std::string> ignore;  // POINTER_IGNORE (see ParseIgnore)
     auto loadConfig = [&] {
@@ -675,7 +678,8 @@ int main() {
         pinchGain = std::clamp(ConfDouble(conf, "POINTER_PINCH_GAIN", 0.5), 0.05, 3.0);
         pinchDeadzone = std::clamp(ConfDouble(conf, "POINTER_PINCH_DEADZONE", 1.5), 0.0, 10.0);
         gripGain = std::clamp(ConfDouble(conf, "POINTER_GRIP_GAIN", 1.0), 0.05, 3.0);
-        gripBelow = std::clamp(ConfDouble(conf, "POINTER_GRIP_BELOW", 0.3), 0.05, 1.0);
+        handBelow = std::clamp(ConfDouble(conf, "POINTER_GRIP_BELOW", 0.35), 0.05, 1.0);
+        typingHold = std::clamp(ConfDouble(conf, "POINTER_PINCH_TYPING", 1.0), 0.0, 5.0);
         pickupScale = std::clamp(ConfDouble(conf, "POINTER_CONTROLLER_PICKUP", 1), 0.5, 5.0);
         const auto ig = conf.find("POINTER_IGNORE");
         ignore = ParseIgnore(ig == conf.end() ? "" : ig->second);
@@ -820,6 +824,7 @@ int main() {
     int handOpens = 0;
     uint64_t handSeq = 0, handPublished = 0;
     Clock::time_point handUsed{};  // a gesture began then (keeps the pointer, like gaze mode)
+    Clock::time_point lastTyping{};  // the relay's last "typing": a key on a keyboard
     // Gaze mode outside games, and its dot (see the top): lastMove/lastHeld/pulseAt.
     bool inGame = false, gazeAwake = false;
     Clock::time_point inGameAt{}, gazeAwakeAt{};
@@ -1110,6 +1115,10 @@ int main() {
                 gz = {g[0], g[1], g[2], g[3], Clock::now()};
                 continue;
             }
+            if (std::strcmp(buf, "typing") == 0) {  // not mouse input: it never wakes the pointer
+                lastTyping = Clock::now();
+                continue;
+            }
             if (std::strncmp(buf, "vrbind", 6) == 0) {
                 std::printf("controller buttons: %s\n", controllerButtons.Bind(buf + 6).c_str());
                 std::fflush(stdout);
@@ -1415,12 +1424,16 @@ int main() {
             }
             vr::HmdMatrix34_t head;
             if (!poses.At(g.begin_ns, head)) return;
-            // A grip only with the hand held up: hands on a desk (typing, on the mouse) curl
-            // like a loose fist, and looking down at them puts them straight ahead in the
-            // head's frame, where ft-hands' own check can't tell.
+            // No pinch just after a key (typing touches thumb to index), and a grip only with
+            // the hand held up (hands on a desk curl like a loose fist; looking down at them
+            // puts them straight ahead in the head's frame, where ft-hands can't tell).
             const Vec3 at = Position(head) + Rotate(head, Vec3{g.begin_point[0], g.begin_point[1], g.begin_point[2]});
-            if (grip && Position(head).y - at.y > gripBelow) {
-                if (debug) std::printf("hand grip ignored: %.2f m below the eyes\n", Position(head).y - at.y);
+            if (!grip && tnow - lastTyping < std::chrono::duration<double>(typingHold)) {
+                if (debug) std::printf("hand pinch ignored: typing\n");
+                return;
+            }
+            if (grip && Position(head).y - at.y > handBelow) {
+                if (debug) std::printf("hand %s ignored: %.2f m below the eyes\n", grip ? "grip" : "pinch", Position(head).y - at.y);
                 return;
             }
             hold = {side, grip, false, Position(head)};
diff --git a/session/frametop.conf.example b/session/frametop.conf.example
index 3358bf6..58bc5b7 100644
--- a/session/frametop.conf.example
+++ b/session/frametop.conf.example
@@ -46,7 +46,8 @@ POINTER_HANDS=1            # 1 = pinches click (on the release; hold and move to
 POINTER_PINCH_GAIN=0.5     # hands: the pointer moves this times the pinching hand's angle (precision)
 POINTER_PINCH_DEADZONE=1.5 # hands: degrees the pinching hand moves before the pointer does (a tap's jitter)
 POINTER_GRIP_GAIN=1        # hands: the pointer moves this times the gripping hand's angle
-POINTER_GRIP_BELOW=0.3     # hands: grips beginning more than this (m) below the eyes are ignored (hands on a desk)
+POINTER_GRIP_BELOW=0.35    # hands: grips beginning more than this (m) below the eyes are ignored (hands on a desk)
+POINTER_PINCH_TYPING=1     # hands: no pinch within this many seconds of a key (typing touches thumb to index)
 META_DASHBOARD=0   # 1 = a Meta tap on a pass-through keyboard toggles the SteamVR dashboard (pointer mode only)
 SHARE_KEYS=0       # 1 = keys of keyboards grabbed for the desktop also go to @frametop_keys, for hotkey tools (any local process can listen)
 DISPLAY_OFF_MIN=0  # minutes without use before frametop-power turns the displays off, even if the headset seems worn (0 = never)