steam-frame/DeeJanuz--frametop
1
0 Fork 0
mirror of https://github.com/DeeJanuz/frametop.git synced 2026-10-06 01:00:06 +02:00
Code Issues Packages Projects Releases 3 Wiki Activity

Hands: fixes from the second headset test (pinches)

Browse Source 
- The palm-down filter is off by default: the user's deliberate pinches,
  hand raised, read 0.90-0.99, like typing.
- Typing is told apart by the keyboard instead: the input relay sends the
  pointer helper "typing" on key presses, and it takes no pinch within
  POINTER_PINCH_TYPING (1 s) of one.
- Grips are still held to hands raised (POINTER_GRIP_BELOW, 0.35 m below
  the eyes); pinches aren't, since the user's own sat 0.35-0.45 m below,
  elbow resting.
- A grip doesn't begin with the thumb on the index tip: that's a pinch
  with the other fingers curled, which was taken for a grip.
- A pinch's point is the index and middle knuckles: the tips' midpoint
  moved 1-2 cm as the pinch opened, dragging every release off its press.
- ft-hands --gesture-log prints what the detectors measure, 10 times a
  second.

Pinches still aren't reliable enough to use; hand tracking is parked for
now in favour of the controllers.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
DeeJanuzandClaude Opus 5.5 committed 2026-09-30 15:07:58 -06:00
1 parent fbd188f5f9
commit 5ca4160a7e
7 files changed
+70 -25
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

hands/include/fh_gestures.h
+4 -3
View File
@@ -10,7 +10,8 @@
* One slot per side and gesture: pinch[0] and grip[0] are the left hand, [1] the right.
* A gesture follows the hand it began on until it ends.
* Pinch: begins when the thumb and index tips close within begin_m and ends when they
* open past end_m (the gap between keeps it from flickering). point is between the tips.
* open past end_m (the gap between keeps it from flickering). point is the index and middle
* knuckles, which don't move as the fingers open and close (the tips' midpoint did).
* Grip: a closed hand. It begins when all four fingers are curled in (each fingertip
* nearer the wrist than grip_begin times its knuckle is) and ends when they open past
* grip_end on average. distance is that average (about 2 open, under 1.2 closed), strength
@@ -55,8 +56,8 @@ typedef struct {
float distance; /* pinch: thumb tip to index tip, m, at this user's */
/* hand size. grip: the fingers' mean curl (above) */
float strength; /* 0 open .. 1 closed */
float point[3]; /* pinch: between the thumb and index tips; grip: */
/* the palm's centre */
float point[3]; /* pinch: the index and middle knuckles; grip: the */
/* palm's centre */
float begin_point[3]; /* point when the current or last one began */
} fh_pinch_t; /* 64 bytes */
hands/track/main.cpp
+12 -2
View File
@@ -177,6 +177,7 @@ int main(int argc, char **argv) {
double keep_presence = 0.5; // landmark presence a tracked view needs to stay
PinchParams pinch_params;
GripParams grip_params;
bool gesture_log = false; // what the pinch and grip detectors measure, 10 times a second
double record_for = 120;
for (int i = 1; i < argc; ++i) {
const std::string a = argv[i];
@@ -194,6 +195,7 @@ int main(int argc, char **argv) {
else if (a == "--pinch-palm-down" && more) pinch_params.palm_down_max = std::atof(argv[++i]);
else if (a == "--grip-begin" && more) grip_params.begin = std::atof(argv[++i]);
else if (a == "--grip-end" && more) grip_params.end = std::atof(argv[++i]);
else if (a == "--gesture-log") gesture_log = true;
else if (a == "--swap-sides") swap_sides = true;
else if (a == "--record-only") track = publish = false;
else if (a == "--ring" && more) ring_path = argv[++i];
@@ -220,7 +222,7 @@ int main(int argc, char **argv) {
" [--cams auto|mono|color|all] (auto) [--bright all|color] (all) [--bright-on L] (40) [--bright-off L] (25)\n"
" [--color-left color_video0|color_video3] [--color-crop subtract|none]\n"
" [--pinch-begin M] (0.020) [--pinch-end M] (0.035) [--pinch-triangulated] [--pinch-palm-down MAX] (0.6)\n"
" [--grip-begin R] (1.2) [--grip-end R] (1.45)\n"
" [--grip-begin R] (1.2) [--grip-end R] (1.45) [--gesture-log]\n"
" [--contrast MODE|PALM/HAND] (clahe[:CLIP], none, stretch; default clahe:2/none)\n"
"Recording saves every frame set for S seconds (120) to DIR/sets.bin, for ft-handreplay; SIGUSR1\n"
"starts one in ~/.local/share/frametop/hands/rec-<time>. --record-only records without tracking, so it\n"
@@ -339,7 +341,7 @@ int main(int argc, char **argv) {
std::map<std::string, uint64_t> last; // per camera: the frame last used
std::map<std::string, uint64_t> lit_seen; // per colour camera: the frame last counted for the light
const uint64_t start = mono_ns();
uint64_t t_status = start, next_ns = 0, t_want = 0;
uint64_t t_status = start, next_ns = 0, t_want = 0, t_glog = 0;
double cpu0 = cpu_seconds();
std::vector<double> lat;
double hands_sum = 0, resid_sum = 0;
@@ -541,6 +543,14 @@ int main(int argc, char **argv) {
const std::vector<Seen> views = tracker.views_now();
grip.update(hands, views, int64_t(capture));
pinch.update(hands, views, int64_t(capture), grip.gripping());
if (gesture_log && capture - t_glog >= 100'000'000) {
t_glog = capture;
for (int k = 0; k < 2; ++k)
if (pinch.world_d[k] >= 0)
std::printf("gesture %s d world %.3f tri %.3f palm-down %.2f curl %.2f%s\n", k ? "right" : "left ",
pinch.world_d[k], pinch.tri_d[k], pinch.palm_down[k], grip.curl[k],
pinch.side(k).flags & FH_PINCH_DOWN ? " PINCH" : grip.side(k).flags & FH_PINCH_DOWN ? " GRIP" : "");
}
// a gesture down or closing gets the full rate, even while the palm holds still
next_ns = tmin + uint64_t((std::min(tracker.interval(), pinch.engaged() || grip.engaged() ? 1 / 30.0 : 1.0) -
0.005) * 1e9);
hands/track/pinch.cpp
+15 -8
View File
@@ -65,7 +65,11 @@ void Pinch::update(const std::vector<const Hand *> &hands, const std::vector<See
}
if (n) world_d[s] = sum / n * h->scale;
const double d = p_.triangulated || world_d[s] < 0 ? tri_d[s] : world_d[s];
const V3 point = (h->smooth[kThumbTip] + h->smooth[kIndexTip]) * 0.5;
// Where the pinch is, for drags: the index and middle knuckles, which hold still while
// the fingers open and close. The point between the tips moved 1-2 cm as a pinch
// opened, so every release dragged the pointer off what it pressed (headset test,
// 2026-09-30).
const V3 point = (h->smooth[5] + h->smooth[9]) * 0.5;
o.flags |= FH_PINCH_TRACKED;
o.hand_id = uint32_t(h->id);
o.distance = float(d);
@@ -126,10 +130,11 @@ namespace {
constexpr int kWrist = 0, kFingers[4][2] = {{5, 8}, {9, 12}, {13, 16}, {17, 20}}; // knuckle, tip
// Each finger's curl (see GripParams): from the model's world landmarks averaged over the
// hand's views this step, or the tracker's 3D points without any.
bool curls(const Hand &h, const std::vector<Seen> &views, double out[4]) {
double sum[4] = {};
// Each finger's curl (see GripParams), and the thumb tip's distance from the index tip (m):
// from the model's world landmarks averaged over the hand's views this step, or the
// tracker's 3D points without any.
bool curls(const Hand &h, const std::vector<Seen> &views, double out[4], double *thumb) {
double sum[4] = {}, gap = 0;
int n = 0;
for (const Seen &v : views) {
if (v.hand != h.id) continue;
@@ -138,8 +143,10 @@ bool curls(const Hand &h, const std::vector<Seen> &views, double out[4]) {
const double k = norm(p(kFingers[f][0]) - p(kWrist));
sum[f] += k > 1e-4 ? norm(p(kFingers[f][1]) - p(kWrist)) / k : 2;
}
gap += norm(p(4) - p(8)) * h.scale;
++n;
}
*thumb = n ? gap / n : norm(h.smooth[4] - h.smooth[8]);
for (int f = 0; f < 4; ++f) {
if (n) {
out[f] = sum[f] / n;
@@ -172,8 +179,8 @@ void Grip::update(const std::vector<const Hand *> &hands, const std::vector<Seen
if (!h || c->frames > h->frames) h = c;
}
curl[s] = -1;
double f[4];
if (!h || !curls(*h, views, f)) {
double f[4], thumb = 1;
if (!h || !curls(*h, views, f, &thumb)) {
o.flags &= ~FH_PINCH_TRACKED;
if (down && (t_ns - seen_ns_[s]) / 1e9 > p_.grace_s) end(s, t_ns, true);
continue;
@@ -191,7 +198,7 @@ void Grip::update(const std::vector<const Hand *> &hands, const std::vector<Seen
if (mean > p_.end) open_ns_[s] = t_ns;
const bool ahead = -point[2] >= p_.min_ahead_m &&
std::atan2(-point[1], -point[2]) * 180 / M_PI <= p_.max_down_deg;
if (most < p_.begin && ahead && open_ns_[s] && (t_ns - open_ns_[s]) / 1e9 <= p_.armed_s) {
if (most < p_.begin && ahead && thumb >= p_.thumb_off_m && open_ns_[s] && (t_ns - open_ns_[s]) / 1e9 <= p_.armed_s) {
o.flags = (o.flags | FH_PINCH_DOWN) & ~FH_PINCH_LOST;
++o.begins;
o.begin_ns = uint64_t(t_ns);
hands/track/pinch.h
+8 -1
View File
@@ -27,7 +27,10 @@ struct PinchParams {
// head frame; 1 turns it off). Typing curls the thumb onto the index: in the 2026-09-30
// lit recording, pinches that began while typing had 0.69-1.00, deliberate ones 0.00-0.50.
// Looking down tilts the head frame, which lowers the reading for a hand on a keyboard.
double palm_down_max = 0.6;
// Off by default since the first headset test (2026-09-30 14:55): the user's deliberate
// pinches, hand raised in front, read 0.90-0.99 too. The pointer helper now leaves out
// gestures that begin low (hands on a desk), which it can tell with the head's pose.
double palm_down_max = 1.0;
};
class Pinch {
@@ -87,6 +90,10 @@ struct GripParams {
// deliberate pinches 5-15.
double max_down_deg = 35;
double min_ahead_m = 0.15;
// ...and not with the thumb on the index fingertip, closer than this (m, the pinch's
// measure): that's a pinch with the other fingers curled, which the first headset test
// took for a grip.
double thumb_off_m = 0.03;
};
// Grip (a closed hand) detection, per side like Pinch: press and drag.
input/input-relay.py
+6
View File
@@ -34,6 +34,8 @@ reaches games; see pointer/helper/vrbuttons.h).
In gaze mode outside games, the helper keeps the pointer ("gazeawake 1", repeated every 5
seconds; "gazeawake 0" or silence ends it): the pointer isn't released when the mouse is idle.
Typing on a keyboard sends the helper "typing" (at most 4 times a second): it takes no hand
pinches right after a key, since typing touches thumb to index like a pinch.
Keys also go to ft-screens (@ft_screens, the Frametop desktop's compositor), which
types them into the desktop screen that has focus: from pass-through keyboards, and
@@ -614,6 +616,7 @@ def main():
load_config()
meta_down = False # Meta pressed with no other key yet: a tap toggles the dashboard
screens_sock = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM | socket.SOCK_NONBLOCK)
last_typing = 0.0 # the helper was last told of a key then (see "typing" at the top)
def vr_bind(now):
"""Tell the pointer helper which controller buttons to take from games."""
@@ -957,6 +960,9 @@ def main():
# Observed only, unless typing goes to the desktop. With META_DASHBOARD=1,
# a Meta tap on any keyboard toggles the dashboard.
if node.role == "passthrough" and etype == EV_KEY:
if value == 1 and code < BTN_MISC and now - last_typing >= 0.25:
last_typing = now
screens_sock.sendto(b"typing", HELPER)
to_screens(code, value)
if node.grabbed and code < BTN_MISC and value in (0, 1):
share_key(node, code, value)
pointer/helper/ft-pointer.cpp
+23 -10
View File
@@ -180,8 +180,11 @@
// A grip (closing the hand) is a press and drag: the press where the pointer is, then the
// hand moves the pointer at POINTER_GRIP_GAIN (1: as far as it moves, seen from the eye), and
// opening the hand releases. So it drags whatever the pointer is on: a title bar moves the
// window, a panel's grab bar carries the panel, text is selected. Only with the hand held up,
// at most POINTER_GRIP_BELOW (0.3 m) below the eyes: hands on a desk curl like a loose fist.
// window, a panel's grab bar carries the panel, text is selected.
// Typing touches thumb to index like a pinch: no pinch begins within POINTER_PINCH_TYPING
// (1 s) of a key (the relay says "typing"). A grip begins only with the hand held up, at most
// POINTER_GRIP_BELOW (0.35 m) below the eyes: hands on a desk curl like a loose fist. (The
// user's own pinches sat 0.35-0.45 m below the eyes, elbow resting, so pinches have no such limit.)
// The hand's movement is taken in the room, from where the eye was when the gesture began,
// with the head pose at each frame's capture time, so turning your head doesn't move it.
// The first gesture while the pointer is off only wakes it. Gestures are ignored in a VR game
@@ -236,7 +239,7 @@
// gaze mode, above. POINTER_CONTROLLER_PICKUP (1, 0.5 to 5): how hard a controller must
// move to take the laser back, above. POINTER_IGNORE (empty): ignored panels, above.
// POINTER_HANDS (1), POINTER_PINCH_GAIN (0.5), POINTER_PINCH_DEADZONE (1.5 deg),
// POINTER_GRIP_GAIN (1), POINTER_GRIP_BELOW (0.3 m): hands, above.
// POINTER_GRIP_GAIN (1), POINTER_GRIP_BELOW (0.35 m), POINTER_PINCH_TYPING (1 s): hands, above.
#include <openvr.h>
#include "vrbuttons.h"
@@ -646,7 +649,7 @@ int main() {
double gazeRetake = 5, gazeNudgeMax = 8, gazeHold = 0.5, gazeShow = 1;
// Hands (see the top): POINTER_HANDS, POINTER_PINCH_GAIN, POINTER_PINCH_DEADZONE, POINTER_GRIP_GAIN.
bool handsOn = true;
double pinchGain = 0.5, pinchDeadzone = 1.5, gripGain = 1.0, gripBelow = 0.3;
double pinchGain = 0.5, pinchDeadzone = 1.5, gripGain = 1.0, handBelow = 0.35, typingHold = 1.0;
double pickupScale = 1; // POINTER_CONTROLLER_PICKUP: scales the controller-moved limits
std::vector<std::string> ignore; // POINTER_IGNORE (see ParseIgnore)
auto loadConfig = [&] {
@@ -675,7 +678,8 @@ int main() {
pinchGain = std::clamp(ConfDouble(conf, "POINTER_PINCH_GAIN", 0.5), 0.05, 3.0);
pinchDeadzone = std::clamp(ConfDouble(conf, "POINTER_PINCH_DEADZONE", 1.5), 0.0, 10.0);
gripGain = std::clamp(ConfDouble(conf, "POINTER_GRIP_GAIN", 1.0), 0.05, 3.0);
gripBelow = std::clamp(ConfDouble(conf, "POINTER_GRIP_BELOW", 0.3), 0.05, 1.0);
handBelow = std::clamp(ConfDouble(conf, "POINTER_GRIP_BELOW", 0.35), 0.05, 1.0);
typingHold = std::clamp(ConfDouble(conf, "POINTER_PINCH_TYPING", 1.0), 0.0, 5.0);
pickupScale = std::clamp(ConfDouble(conf, "POINTER_CONTROLLER_PICKUP", 1), 0.5, 5.0);
const auto ig = conf.find("POINTER_IGNORE");
ignore = ParseIgnore(ig == conf.end() ? "" : ig->second);
@@ -820,6 +824,7 @@ int main() {
int handOpens = 0;
uint64_t handSeq = 0, handPublished = 0;
Clock::time_point handUsed{}; // a gesture began then (keeps the pointer, like gaze mode)
Clock::time_point lastTyping{}; // the relay's last "typing": a key on a keyboard
// Gaze mode outside games, and its dot (see the top): lastMove/lastHeld/pulseAt.
bool inGame = false, gazeAwake = false;
Clock::time_point inGameAt{}, gazeAwakeAt{};
@@ -1110,6 +1115,10 @@ int main() {
gz = {g[0], g[1], g[2], g[3], Clock::now()};
continue;
}
if (std::strcmp(buf, "typing") == 0) { // not mouse input: it never wakes the pointer
lastTyping = Clock::now();
continue;
}
if (std::strncmp(buf, "vrbind", 6) == 0) {
std::printf("controller buttons: %s\n", controllerButtons.Bind(buf + 6).c_str());
std::fflush(stdout);
@@ -1415,12 +1424,16 @@ int main() {
}
vr::HmdMatrix34_t head;
if (!poses.At(g.begin_ns, head)) return;
// A grip only with the hand held up: hands on a desk (typing, on the mouse) curl
// like a loose fist, and looking down at them puts them straight ahead in the
// head's frame, where ft-hands' own check can't tell.
// No pinch just after a key (typing touches thumb to index), and a grip only with
// the hand held up (hands on a desk curl like a loose fist; looking down at them
// puts them straight ahead in the head's frame, where ft-hands can't tell).
const Vec3 at = Position(head) + Rotate(head, Vec3{g.begin_point[0], g.begin_point[1], g.begin_point[2]});
if (grip && Position(head).y - at.y > gripBelow) {
if (debug) std::printf("hand grip ignored: %.2f m below the eyes\n", Position(head).y - at.y);
if (!grip && tnow - lastTyping < std::chrono::duration<double>(typingHold)) {
if (debug) std::printf("hand pinch ignored: typing\n");
return;
}
if (grip && Position(head).y - at.y > handBelow) {
if (debug) std::printf("hand %s ignored: %.2f m below the eyes\n", grip ? "grip" : "pinch", Position(head).y - at.y);
return;
}
hold = {side, grip, false, Position(head)};
session/frametop.conf.example
+2 -1
View File
@@ -46,7 +46,8 @@ POINTER_HANDS=1 # 1 = pinches click (on the release; hold and move to
POINTER_PINCH_GAIN=0.5 # hands: the pointer moves this times the pinching hand's angle (precision)
POINTER_PINCH_DEADZONE=1.5 # hands: degrees the pinching hand moves before the pointer does (a tap's jitter)
POINTER_GRIP_GAIN=1 # hands: the pointer moves this times the gripping hand's angle
POINTER_GRIP_BELOW=0.3 # hands: grips beginning more than this (m) below the eyes are ignored (hands on a desk)
POINTER_GRIP_BELOW=0.35 # hands: grips beginning more than this (m) below the eyes are ignored (hands on a desk)
POINTER_PINCH_TYPING=1 # hands: no pinch within this many seconds of a key (typing touches thumb to index)
META_DASHBOARD=0 # 1 = a Meta tap on a pass-through keyboard toggles the SteamVR dashboard (pointer mode only)
SHARE_KEYS=0 # 1 = keys of keyboards grabbed for the desktop also go to @frametop_keys, for hotkey tools (any local process can listen)
DISPLAY_OFF_MIN=0 # minutes without use before frametop-power turns the displays off, even if the headset seems worn (0 = never)