Make hand tracking a Frametop component

- Programs: ft-camd (the camera broker), ft-hands (the tracker), and
  ft-handreplay and ft-ringplay for recordings, built by hands/build.sh
  into hands/build/ with one Makefile. The first build fetches ncnn at
  frame-hands' pinned tag and builds it with the same options.
- ft-camd gets its privileges from file capabilities (CAP_SYS_PTRACE,
  CAP_PERFMON, CAP_DAC_READ_SEARCH) that hands/run.sh install sets with
  sudo, and drops them once set up. It still works under sudo. It runs
  on the host, linked statically, as frametop-camd.service. ft-hands
  runs in the dev container as frametop-hands.service. Both start and
  stop with SteamVR.
- Files move to /run/user/UID/frametop/ (cam-ring, hands, gestures),
  not $XDG_RUNTIME_DIR, which a terminal in the Frametop desktop has
  its own of. SIGUSR1 recordings go to ~/.local/share/frametop/hands.
- The calibration is read through /run/host in the container.
- Settings: HANDS_SWAP_SIDES and HANDS_CPUS in frametop.conf.
- install.sh offers hand tracking as an optional last step.
- The container gets jsoncpp-devel, glibc-static, and NumPy and OpenCV
  for the Python tools.
- tools/ring.py reads the ring, and models/NOTICE credits the
  Apache-2.0 models.

Checked: ft-handreplay gives identical summaries and byte-identical
depth dumps to frame-hands' fh-replay on both 2026-09-29 recordings.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

hands/tools/calib.py

@@ -13,6 +13,7 @@ the image.
 Everything here returns metres in the head frame.
 """
 import json
+import os
 
 import numpy as np
 
@@ -91,11 +92,23 @@ class Camera:
         return np.degrees(np.arccos(np.clip(self.unproject(uv)[:, 2], -1, 1)))
 
 
+def device_path(path):
+    """A headset file such as /persist/xrservice.json. In the dev container the host's / is
+    at /run/host (distrobox doesn't mount /persist); off the Frame, FRAME_JOB_DEVICE_ROOT can
+    point at a folder with copies of them."""
+    root = os.environ.get('FRAME_JOB_DEVICE_ROOT')
+    if root:
+        return root + path
+    if not os.access(path, os.R_OK) and os.access('/run/host' + path, os.R_OK):
+        return '/run/host' + path
+    return path
+
+
 def load(xrservice=XRSERVICE_JSON, device=DEVICE_JSON):
     """{calibration name: Camera} for the tracking cameras, posed in the head frame."""
-    with open(xrservice) as f:
+    with open(device_path(xrservice)) as f:
         rig = json.load(f)
-    with open(device) as f:
+    with open(device_path(device)) as f:
         dev = json.load(f)
     cad_from_cam0 = _pose(dev['cv']['cad_from_cal'])
     head_from_cad = np.linalg.inv(_pose(dev['head']))
@@ -110,18 +123,18 @@ def load(xrservice=XRSERVICE_JSON, device=DEVICE_JSON):
 
 def load_color(eeprom=ARCTURUS_EEPROM, device=DEVICE_JSON, scale=2, crop='subtract'):
     """{"passthrough_left"/"passthrough_right": Camera} for the Arcturus color cameras, posed in
-    the head frame, for fh-camd --with-color's images (luma at 1/scale size).
+    the head frame, for ft-camd --with-color's images (luma at 1/scale size).
 
     Their calibration is in the CAD frame (mm) with pixel coordinates on the full 2464x2464
     sensor; each camera also has a cropRegion. crop says how that maps to the delivered
     image: 'subtract' (image x = sensor x - cropRegion.x) or 'none'. tools/check_color.py
     tells which fits.
     """
-    with open(eeprom, 'rb') as f:
+    with open(device_path(eeprom), 'rb') as f:
         raw = f.read()
     i = raw.rfind(b'{', 0, raw.find(b'"alignment_method"'))
     rig, _ = json.JSONDecoder().raw_decode(raw[i:].decode('latin1'))
-    with open(device) as f:
+    with open(device_path(device)) as f:
         dev = json.load(f)
     head_from_cad = np.linalg.inv(_pose(dev['head']))
     cams = {}

hands/tools/check_color.py

@@ -1,8 +1,8 @@
-"""Which color camera is which, and how their calibration maps onto fh-camd's images.
+"""Which color camera is which, and how their calibration maps onto ft-camd's images.
 
 usage: python tools/check_color.py REC_DIR [--sets N]
 
-A recording made with fh-camd --with-color holds color_video<N> frames with each set.
+A recording made with ft-camd --with-color holds color_video<N> frames with each set.
 This matches features between the two color images and scores every reading of the
 calibration: which video node is passthrough_left, and whether the calibration's
 cropRegion is subtracted from x ('subtract') or not ('none'). Only the right reading
@@ -19,12 +19,11 @@ import numpy as np
 sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), '..'))
 from tools.check_sides import load_cams, matches, score  # noqa: E402
 from tools.show_set import index, read_set  # noqa: E402
-from tracker import calib  # noqa: E402
+from tools import calib  # noqa: E402
 
 
 def load_color(crop):
-    root = os.environ.get('FRAME_JOB_DEVICE_ROOT', '')   # frame-job's copy of the device files
-    return calib.load_color(root + calib.ARCTURUS_EEPROM, root + calib.DEVICE_JSON, crop=crop)
+    return calib.load_color(crop=crop)
 
 
 def main():
@@ -37,7 +36,7 @@ def main():
     sets = [read_set(path, offs[n]) for n in np.linspace(0, len(offs) - 1, a.sets).astype(int)]
     nodes = sorted(k for k in sets[0] if k.startswith('color_video'))
     if len(nodes) != 2:
-        sys.exit('need two color_video<N> cameras in the recording (fh-camd --with-color); found %s' % nodes)
+        sys.exit('need two color_video<N> cameras in the recording (ft-camd --with-color); found %s' % nodes)
     pairs = [matches(s[nodes[0]][0], s[nodes[1]][0]) for s in sets]
     print('%d sets, %d matches between %s and %s' % (len(sets), sum(len(p[0]) for p in pairs), *nodes))
 

hands/tools/check_sides.py

@@ -1,13 +1,13 @@
 """Check that the side cameras' images carry the right names (slam_left vs slam_right).
 
 usage: python tools/check_sides.py REC_DIR [--sets N]
-       python tools/check_sides.py --ring [--sets N]    (live, from fh-camd's ring)
+       python tools/check_sides.py --ring [--sets N]    (live, from ft-camd's ring)
 
-With --ring it exits 0 when the names are right, 3 when they're swapped (run fh-tracker
+With --ring it exits 0 when the names are right, 3 when they're swapped (run ft-hands
 with --swap-sides), and 2 when it can't tell (too little texture in view, or the headset
 isn't worn).
 
-fh-camd tells the two side cameras' buffers apart by the order XRService allocated them,
+ft-camd tells the two side cameras' buffers apart by the order XRService allocated them,
 and after some XRService restarts that order puts each camera's images under the other's
 name. The tracker then sees every hand in one camera only, at the wrong depth. This
 matches features between the two images and measures how close each pair's rays pass
@@ -23,15 +23,14 @@ import numpy as np
 
 sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), '..'))
 from tools.show_set import index, read_set  # noqa: E402
-from tracker import calib  # noqa: E402
+from tools import calib  # noqa: E402
 
 
-PIPES = {'msm_vfe3_video0': 'slam_left', 'msm_vfe4_video0': 'slam_right'}   # as fh-tracker maps them
+PIPES = {'msm_vfe3_video0': 'slam_left', 'msm_vfe4_video0': 'slam_right'}   # as ft-hands maps them
 
 
 def load_cams():
-    root = os.environ.get('FRAME_JOB_DEVICE_ROOT', '')   # frame-job's copy of /persist off the Frame
-    return calib.load(root + calib.XRSERVICE_JSON, root + calib.DEVICE_JSON)
+    return calib.load()
 
 
 def matches(a, b):
@@ -80,12 +79,12 @@ def recorded_pairs(rec, count):
 
 
 def live_pairs(count):
-    """(label, slam_left image, slam_right image) from fh-camd's ring, half a second apart."""
+    """(label, slam_left image, slam_right image) from ft-camd's ring, half a second apart."""
     import time
-    from tracker.ring import Ring
+    from tools.ring import Ring
     ring = Ring()
     if not ring.alive():
-        sys.exit('fh-camd isn\'t running (no heartbeat)')
+        sys.exit('ft-camd isn\'t running (no heartbeat)')
     cams = {}
     for c in ring.cams:
         name = PIPES.get(open('/sys/class/video4linux/video%d/name' % c.node).read().strip())

hands/tools/depth_report.py

@@ -2,13 +2,13 @@
 
 usage: python3 tools/depth_report.py DEPTH [DEPTH...] [--still M/S]
 
-DEPTH comes from `trackd/fh-replay DIR --depth DEPTH`. Every measure is split by how the
+DEPTH comes from `hands/build/ft-handreplay DIR --depth DEPTH`. Every measure is split by how the
 hand was seen: by the two lower cameras ("lower pair"), by a lower and an upper camera on
 one side ("lower+upper"), or by one camera. Distances are from the head (between the eyes).
 
 1. How the hands were seen: the share of hand updates in each way, by distance.
 2. Noise along the line of sight against across it. Each update's palm is compared with a
-   straight line through the two updates before it (fh-replay's jitter measure), and the
+   straight line through the two updates before it (ft-handreplay's jitter measure), and the
    miss is split along the line from the hand's cameras to the palm and across it. Given
    as a robust sigma per axis, measured (as triangulated) and published (after the One Euro
    filter), on updates where the published palm moved slower than --still (default 0.15
@@ -20,7 +20,7 @@ one side ("lower+upper"), or by one camera. Distances are from the head (between
    distance from that camera.
 4. A camera lost: from two-camera updates, what the tracker would have had if one of the
    two cameras dropped out there. It keeps the last distance and moves a share of the way
-   to the one-view guess each update (0.1 now, kMonoDepthGain in trackd/tracker.cpp);
+   to the one-view guess each update (0.1 now, kMonoDepthGain in track/tracker.cpp);
    also shown with other shares, 0 (keep the distance) and 1 (take each guess), and with
    the guess first scaled by how far off it was while both cameras saw the hand. Compared with the
    triangulated distance from that camera, 0.1-2 s after the loss.

hands/tools/ring.py

@@ -0,0 +1,80 @@
+"""Read frames from ft-camd's shared-memory ring (layout: camd/fhring.h)."""
+import mmap
+import os
+import struct
+import time
+
+import numpy as np
+
+RING_FILE = '/run/user/%d/frametop/cam-ring' % os.getuid()
+MAGIC = b'FHRING01'
+HDR = struct.Struct('<8sIIIIQqQ16x')                     # 64 bytes
+CAM = struct.Struct('<32s32siIIIIIQQQQQ32x')             # 160 bytes
+SLOT = struct.Struct('<QQQQQIf16x')                       # 64 bytes
+MAX_CAMS = 8
+LATEST_OFF = 32 + 32 + 4 * 6 + 8 * 2                      # cam.latest within fh_ring_cam_t
+HEARTBEAT_OFF = 40
+
+
+class Frame:
+    __slots__ = ('cam', 'frame', 'capture_ns', 'dqbuf_ns', 'publish_ns', 'v4l2_seq', 'mean', 'image')
+
+    def __init__(self, cam, fields, image):
+        self.cam = cam
+        (_, self.frame, self.capture_ns, self.dqbuf_ns, self.publish_ns, self.v4l2_seq, self.mean) = fields
+        self.image = image
+
+
+class RingCamera:
+    def __init__(self, index, fields):
+        (sensor, name, self.node, self.format, self.width, self.height, self.stride, self.nslots,
+         self.slot_offset, self.slot_bytes, _latest, _pub, _drop) = fields
+        self.index = index
+        self.sensor = sensor.split(b'\0', 1)[0].decode()
+        self.name = name.split(b'\0', 1)[0].decode()
+        self.latest_off = HDR.size + index * CAM.size + LATEST_OFF
+
+    def __repr__(self):
+        return 'RingCamera(video%d %s %dx%d)' % (self.node, self.sensor, self.width, self.height)
+
+
+class Ring:
+    def __init__(self, path=RING_FILE):
+        fd = os.open(path, os.O_RDONLY)
+        try:
+            self.map = mmap.mmap(fd, 0, mmap.MAP_SHARED, mmap.PROT_READ)
+        finally:
+            os.close(fd)
+        magic, version, hdr_bytes, ncams, _, file_bytes, self.writer_pid, _ = HDR.unpack_from(self.map, 0)
+        if magic != MAGIC or version != 1:
+            raise RuntimeError('%s is not an ft-camd ring (magic %r version %d)' % (path, magic, version))
+        self.cams = [RingCamera(i, CAM.unpack_from(self.map, HDR.size + i * CAM.size)) for i in range(ncams)]
+
+    def heartbeat_ns(self):
+        return struct.unpack_from('<Q', self.map, HEARTBEAT_OFF)[0]
+
+    def alive(self, max_age=1.0):
+        hb = self.heartbeat_ns()
+        return hb != 0 and (time.clock_gettime_ns(time.CLOCK_MONOTONIC) - hb) / 1e9 < max_age
+
+    def latest(self, cam):
+        return struct.unpack_from('<Q', self.map, cam.latest_off)[0]
+
+    def read(self, cam, n=None):
+        """Copy frame n (default: the newest) of a camera, or None if it's gone or being written."""
+        for _ in range(3):
+            if n is None or n == 0:
+                n = self.latest(cam)
+            if n == 0:
+                return None
+            off = cam.slot_offset + (n % cam.nslots) * cam.slot_bytes
+            fields = SLOT.unpack_from(self.map, off)
+            if fields[0] != 2 * n + 2:
+                return None
+            start = off + SLOT.size
+            image = np.frombuffer(self.map, np.uint8, cam.stride * cam.height, start).reshape(cam.height, cam.stride)
+            image = image[:, :cam.width].copy()
+            if struct.unpack_from('<Q', self.map, off)[0] == fields[0]:
+                return Frame(cam, fields, image)
+            n = None                                      # overwritten while copying: take the newest
+        return None

hands/tools/show_set.py

@@ -1,12 +1,12 @@
-"""Draw frame sets from a recording (fh-tracker --record) with what the tracker saw.
+"""Draw frame sets from a recording (ft-hands --record) with what the tracker saw.
 
 usage: python tools/show_set.py REC_DIR SET [SET...] [--timeline TL] [--out DIR]
 
-SET is a set index (fh-replay's timeline gives them). With --timeline (fh-replay
+SET is a set index (ft-handreplay's timeline gives them). With --timeline (ft-handreplay
 --timeline), each camera shows the tracker's views at that set: the crop for the next
-frame, labelled with the hand and presence. Recordings made with fh-camd --with-dark get
+frame, labelled with the hand and presence. Recordings made with ft-camd --with-dark get
 a second row: each camera's latest dark frame (<name>_dk), stretched to be visible and
-labelled with its mean brightness. Recordings made with fh-camd --with-color get a row of
+labelled with its mean brightness. Recordings made with ft-camd --with-color get a row of
 the color cameras (color_video<N>). Writes OUT/set_<n>.jpg (default /tmp).
 """
 import argparse

hands/tools/watch_gestures.py

@@ -1,4 +1,4 @@
-"""Watch the pinch gestures fh-tracker publishes, live: begins, ends, and drags.
+"""Watch the pinch gestures ft-hands publishes, live: begins, ends, and drags.
 
 usage: python3 tools/watch_gestures.py [--every S] [--distance]
 
@@ -22,8 +22,7 @@ SIDES = ('left ', 'right')
 
 
 def path():
-    run = os.environ.get('XDG_RUNTIME_DIR', '/run/user/%d' % os.getuid())
-    return os.path.join(run, 'frame-hands', 'gestures')
+    return '/run/user/%d/frametop/gestures' % os.getuid()
 
 
 def read(m):
@@ -48,7 +47,7 @@ def main():
         m = mmap.mmap(f.fileno(), 0, prot=mmap.PROT_READ)
     first = read(m)
     if first is None or first[0][0] != b'FHGEST01':
-        raise SystemExit('%s is not an fh-tracker gestures file' % path())
+        raise SystemExit('%s is not an ft-hands gestures file' % path())
     h, p = first
     print('thresholds: pinch begins under %.3f m, ends over %.3f m' % (h[6], h[7]))
     seen = [(q[2], q[3]) for q in p]   # begins, ends