// Gesture detection for input: look at something and pinch to click it (pinch and move to // nudge the pointer first), or close the hand to press and drag it. Per side, from the // tracker's hands after each step; published as fh_gestures.h. #pragma once #include "tracker.h" #include #include extern "C" { #include "../include/fh_gestures.h" } struct PinchParams { double begin_m = 0.020; // thumb and index tips closer than this: the pinch begins double end_m = 0.035; // further apart than this: it ends (the gap keeps it from flickering) int end_frames = 2; // processed frames in a row past end_m before it ends, so one // noisy frame doesn't drop a drag double grace_s = 0.25; // a pinching hand lost this long ends its pinch (FH_PINCH_LOST) // Where the distance comes from: MediaPipe's world landmarks (the model's own 3D hand // pose, averaged over the hand's views, at the user's hand size), or the tracker's // triangulated tips. The model's pose should hold up better when the fingers hide each // other; tomorrow's recordings will tell. bool triangulated = false; // No pinch begins while the palm faces down more than this (|palm normal . up| in the // head frame; 1 turns it off). Typing curls the thumb onto the index: in the 2026-09-30 // lit recording, pinches that began while typing had 0.69-1.00, deliberate ones 0.00-0.50. // Looking down tilts the head frame, which lowers the reading for a hand on a keyboard. double palm_down_max = 0.6; }; class Pinch { public: explicit Pinch(const PinchParams &p = {}) : p_(p) {} const PinchParams ¶ms() const { return p_; } // After each processed set: the hands out of Tracker::step, the tracker's views (for // the world landmarks) and the capture time. Hands in `gripping` (their ids) are closed: // no pinch begins on them, and one that's down on them ends, as lost. void update(const std::vector &hands, const std::vector &views, int64_t t_ns, const std::vector &gripping = {}); // Ends any pinch that's down (as lost), e.g. when the tracker stops. void release(int64_t t_ns); const fh_pinch_t &side(int s) const { return side_[s]; } // 0 left, 1 right // A pinch is down or closing: worth tracking at the full rate. bool engaged() const; // What changed in the last update, for logs. struct Event { int side; const char *what; // "begin", "end", "lost" int64_t t_ns; double distance; V3 point; }; std::vector events; // Both distance measures for the last update, per side (-1: no hand), for logs. double world_d[2] = {-1, -1}, tri_d[2] = {-1, -1}; double palm_down[2] = {-1, -1}; // |palm normal . up| of each side's hand int held_back[2] = {0, 0}; // pinches that didn't begin because the palm faced down private: void end(int s, int64_t t_ns, bool lost); PinchParams p_; fh_pinch_t side_[2]{}; int follow_[2] = {0, 0}; // the hand id a pinch follows while down int open_frames_[2] = {0, 0}; int64_t seen_ns_[2] = {0, 0}; bool held_[2] = {false, false}; // a close held back (palm down) that hasn't opened yet }; struct GripParams { // How curled a finger is: its tip's distance from the wrist over its knuckle's, from the // model's world landmarks (so the hand's size doesn't matter). About 1.8-2.0 straight, // 0.8-1.0 curled into a fist. double begin = 1.2; // every finger under this: the grip begins double end = 1.45; // their mean over this: it ends int end_frames = 2; // processed frames in a row past end before it ends double grace_s = 0.25; // a gripping hand lost this long ends its grip (FH_PINCH_LOST) // A grip begins only on a hand seen open (mean over end) within this long: closing the // hand is the gesture. A hand resting closed (in your lap, on a mouse) never grips. double armed_s = 1.0; // ...and only in front of you, where you'd hold a hand up to grab something: the palm no // more than max_down_deg below straight ahead (head frame) and at least min_ahead_m in // front of the eyes. Typing curls the fingers like a loose fist: in the 2026-09-30 lit // recording, typing hands sat about 47 degrees down (14 false grips without this), // deliberate pinches 5-15. double max_down_deg = 35; double min_ahead_m = 0.15; }; // Grip (a closed hand) detection, per side like Pinch: press and drag. class Grip { public: explicit Grip(const GripParams &p = {}) : p_(p) {} const GripParams ¶ms() const { return p_; } void update(const std::vector &hands, const std::vector &views, int64_t t_ns); void release(int64_t t_ns); const fh_pinch_t &side(int s) const { return side_[s]; } // The hands gripping now (for Pinch::update). std::vector gripping() const; bool engaged() const; // a grip is down or closing: worth tracking at the full rate std::vector events; double curl[2] = {-1, -1}; // each side's hand: mean curl, for logs private: void end(int s, int64_t t_ns, bool lost); GripParams p_; fh_pinch_t side_[2]{}; int follow_[2] = {0, 0}; int open_frames_[2] = {0, 0}; int64_t seen_ns_[2] = {0, 0}; int64_t open_ns_[2] = {0, 0}; // the side's hand was last seen open then }; // Writes /run/user/UID/frametop-hands/gestures. class GesturePublisher { public: bool open(const Pinch &pinch, const Grip &grip, std::string &err); void write(const Pinch &pinch, const Grip &grip, uint64_t capture_ns); private: fh_gestures_t *out_ = nullptr; uint64_t seq_ = 0; // the counters last written, per gesture (0 pinch, 1 grip) and side uint32_t last_begins_[2][2] = {}, last_ends_[2][2] = {}; };