From c2739cb0df03c1b1221c5f6226385a15c4db3ee9 Mon Sep 17 00:00:00 2001 From: DeeJanuz <45082401+DeeJanuz@users.noreply.github.com> Date: Wed, 30 Sep 2026 15:08:54 -0600 Subject: [PATCH] Hands: park hand tracking behind ft-handsctl Hand tracking no longer starts with SteamVR: hands/run.sh install leaves the units disabled and links hands/ft-handsctl into ~/.local/bin, which turns it on and off (on | off | status | log | cutouts on|off | gestures). With it on, hands show through the screens; pinches and grips move the pointer only with POINTER_HANDS=1, now off by default. ft-camd's service runs the mono cameras only: while the headset is worn, the colour module writes just a half-size image into the top-left quarter of its buffers, which ft-camd can't use yet. Co-Authored-By: Claude Opus 5.5 --- hands/frametop-camd.service | 7 ++--- hands/ft-handsctl | 48 +++++++++++++++++++++++++++++++++++ hands/run.sh | 10 +++++--- pointer/helper/ft-pointer.cpp | 8 +++--- session/frametop.conf.example | 2 +- 5 files changed, 63 insertions(+), 12 deletions(-) create mode 100755 hands/ft-handsctl diff --git a/hands/frametop-camd.service b/hands/frametop-camd.service index b653d8c..f5abf49 100644 --- a/hands/frametop-camd.service +++ b/hands/frametop-camd.service @@ -10,9 +10,10 @@ PartOf=steamvr.service # On the host: the dev container can't reach XRService. Its file capabilities (set by # hands/run.sh install) let it borrow the buffers; it drops them once set up. It exits when # XRService restarts, and comes back to attach to the new one. -# The colour cameras run at 2 fps (their brightness picks the tracking cameras) until -# ft-hands asks for more; HANDS_CAMERAS=mono leaves them out. -ExecStart=@REPO@/hands/build/ft-camd --with-color --status 60 +# Mono cameras only: while the headset is worn the colour module writes only a half-size +# image into the top-left quarter of its buffers (2026-09-30), which ft-camd can't use yet. +# Add --with-color to try the colour cameras (ft-hands then picks them by the light). +ExecStart=@REPO@/hands/build/ft-camd --status 60 Restart=always RestartSec=5 TimeoutStopSec=5 diff --git a/hands/ft-handsctl b/hands/ft-handsctl new file mode 100755 index 0000000..8de5f66 --- /dev/null +++ b/hands/ft-handsctl @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +# ft-handsctl: turn hand tracking on and off by hand, on the Frame. With it on, your hands show +# through Frametop's screens (the hand cutouts); pinches and grips only move the pointer with +# POINTER_HANDS=1. It doesn't start with SteamVR (hands/run.sh install leaves it off), and it +# stops when SteamVR does. +# +# ft-handsctl on | off | status | log [lines] +# ft-handsctl cutouts on|off|state ft-screens' hand cutouts, without stopping tracking +# ft-handsctl gestures watch pinches and grips live (Ctrl+C to stop) +# +# Needs the services installed once: hands/run.sh install (it sets ft-camd's capabilities). +set -euo pipefail +here=$(cd "$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")" && pwd) +units="frametop-camd.service frametop-hands.service" + +ask_screens() { # a command to ft-screens' control socket, and its reply + python3 - "$1" <<'EOF' +import socket, sys +s = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) +s.bind("") +s.settimeout(2) +try: + s.sendto(sys.argv[1].encode(), "\0ft_screens") + print(s.recv(512).decode()) +except OSError as e: + sys.exit("ft-screens didn't answer (is the Frametop desktop running?): %s" % e) +EOF +} + +case ${1:-status} in + on) + systemctl --user -q is-active steamvr.service || { echo "SteamVR isn't running" >&2; exit 1; } + systemctl --user start $units + sleep 4 + "$0" status ;; + off) + systemctl --user stop $units + echo "hand tracking off" ;; + status) + for u in $units; do echo "$u: $(systemctl --user is-active $u || true)"; done + journalctl --user -u frametop-hands.service --no-pager -o cat -n 40 | + grep -E '^ *[0-9.]+s |sets with a hand|cameras:' | tail -2 | cut -c1-160 || true ;; + log) journalctl --user -u frametop-camd.service -u frametop-hands.service --no-pager -o short -n "${2:-30}" ;; + cutouts) + ask_screens "cutouts ${2:-state}" ;; + gestures) exec python3 "$here/tools/watch_gestures.py" --distance ;; + *) sed -n '2,11p' "$0" | sed 's/^# \{0,1\}//'; exit 2 ;; +esac diff --git a/hands/run.sh b/hands/run.sh index 45d11f5..0be1575 100755 --- a/hands/run.sh +++ b/hands/run.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash # Install, start, stop, or inspect hand tracking on the Frame: ft-camd (the camera broker) and -# ft-hands (the tracker), user services that start and stop with SteamVR. +# ft-hands (the tracker), user services that stop with SteamVR. They don't start with it: +# ft-handsctl on|off (on the Frame) or hands/run.sh start|stop. # Usage: hands/run.sh install|uninstall # hands/run.sh caps # give ft-camd its capabilities again (a rebuild clears them) # hands/run.sh start|stop|restart|status|log [lines] @@ -47,9 +48,10 @@ case ${1:-status} in for u in $units; do fill_template "$root/hands/$u" | on_frame "mkdir -p ~/.config/systemd/user && cat > ~/.config/systemd/user/$u" done - "$frame" --host "set -e; systemctl --user daemon-reload; systemctl --user enable $units -if systemctl --user -q is-active steamvr.service; then systemctl --user restart $units; sleep 5; fi -$states" ;; + # Installed but not started with SteamVR: ft-handsctl on|off (linked into ~/.local/bin). + "$frame" --host "set -e; systemctl --user daemon-reload; systemctl --user disable $units 2>/dev/null || true +mkdir -p ~/.local/bin && ln -sfn $(printf %q "$FRAME_REPO/hands/ft-handsctl") ~/.local/bin/ft-handsctl +$states; echo 'start it with: ft-handsctl on'" ;; caps) set_caps ;; uninstall) "$frame" --host "systemctl --user disable --now $units 2>/dev/null for u in $units; do rm -f ~/.config/systemd/user/\$u; done; systemctl --user daemon-reload; echo removed" ;; diff --git a/pointer/helper/ft-pointer.cpp b/pointer/helper/ft-pointer.cpp index e061205..bb79c1f 100644 --- a/pointer/helper/ft-pointer.cpp +++ b/pointer/helper/ft-pointer.cpp @@ -163,7 +163,7 @@ // press dragged onto the target is the same: from the raw gaze at the press to the release. With no // fresh gaze (a blink, the service stopped, the headset off), the pointer stays put. // -// Hands (POINTER_HANDS, on by default; needs hand tracking, hands/): ft-hands publishes +// Hands (POINTER_HANDS, off by default; needs hand tracking, hands/): ft-hands publishes // pinches and grips (hands/include/fh_gestures.h), read here every frame. // A pinch works like gaze mode's mouse press: the pointer stops (where the gaze put it), and // the click comes when the pinch opens, where the pointer is then. A quick tap clicks where @@ -238,7 +238,7 @@ // (5 deg), POINTER_GAZE_NUDGE_MAX (8 deg), POINTER_GAZE_HOLD (0.5 s), POINTER_GAZE_SHOW (1 s): // gaze mode, above. POINTER_CONTROLLER_PICKUP (1, 0.5 to 5): how hard a controller must // move to take the laser back, above. POINTER_IGNORE (empty): ignored panels, above. -// POINTER_HANDS (1), POINTER_PINCH_GAIN (0.5), POINTER_PINCH_DEADZONE (1.5 deg), +// POINTER_HANDS (0), POINTER_PINCH_GAIN (0.5), POINTER_PINCH_DEADZONE (1.5 deg), // POINTER_GRIP_GAIN (1), POINTER_GRIP_BELOW (0.35 m), POINTER_PINCH_TYPING (1 s): hands, above. #include @@ -648,7 +648,7 @@ int main() { bool gazeOn = false, gazeConf = false; double gazeRetake = 5, gazeNudgeMax = 8, gazeHold = 0.5, gazeShow = 1; // Hands (see the top): POINTER_HANDS, POINTER_PINCH_GAIN, POINTER_PINCH_DEADZONE, POINTER_GRIP_GAIN. - bool handsOn = true; + bool handsOn = false; double pinchGain = 0.5, pinchDeadzone = 1.5, gripGain = 1.0, handBelow = 0.35, typingHold = 1.0; double pickupScale = 1; // POINTER_CONTROLLER_PICKUP: scales the controller-moved limits std::vector ignore; // POINTER_IGNORE (see ParseIgnore) @@ -674,7 +674,7 @@ int main() { gazeShow = std::clamp(ConfDouble(conf, "POINTER_GAZE_SHOW", 1), 0.0, 30.0); const bool wantGaze = ConfDouble(conf, "POINTER_GAZE", 0) != 0; if (wantGaze != gazeConf) gazeOn = gazeConf = wantGaze; - handsOn = ConfDouble(conf, "POINTER_HANDS", 1) != 0; + handsOn = ConfDouble(conf, "POINTER_HANDS", 0) != 0; pinchGain = std::clamp(ConfDouble(conf, "POINTER_PINCH_GAIN", 0.5), 0.05, 3.0); pinchDeadzone = std::clamp(ConfDouble(conf, "POINTER_PINCH_DEADZONE", 1.5), 0.0, 10.0); gripGain = std::clamp(ConfDouble(conf, "POINTER_GRIP_GAIN", 1.0), 0.05, 3.0); diff --git a/session/frametop.conf.example b/session/frametop.conf.example index 58bc5b7..a951cb9 100644 --- a/session/frametop.conf.example +++ b/session/frametop.conf.example @@ -42,7 +42,7 @@ HANDS_BRIGHT_ON=40 # hand tracking, auto: the colour frames' mean bright HANDS_BRIGHT_OFF=25 # ... and back to the mono cameras under this HANDS_COLOR_LEFT=color_video0 # hand tracking: which colour camera is passthrough_left (hands/tools/check_color.py tells) HANDS_COLOR_CROP=subtract # hand tracking: how the colour calibration's crop applies, subtract | none (check_color.py tells) -POINTER_HANDS=1 # 1 = pinches click (on the release; hold and move to nudge the pointer first) and grips press and drag, with hand tracking +POINTER_HANDS=0 # 1 = pinches click (on the release; hold and move to nudge the pointer first) and grips press and drag, with hand tracking POINTER_PINCH_GAIN=0.5 # hands: the pointer moves this times the pinching hand's angle (precision) POINTER_PINCH_DEADZONE=1.5 # hands: degrees the pinching hand moves before the pointer does (a tap's jitter) POINTER_GRIP_GAIN=1 # hands: the pointer moves this times the gripping hand's angle