mirror of
https://github.com/DeeJanuz/frametop.git
synced 2026-10-15 23:00:38 +02:00
Compare commits
44
Commits
No files matched your search
@@ -18,3 +18,4 @@ frametop-report-*.txt
|
||||
.frame-job.d/
|
||||
# the fine-tuned hand models: the image fetches its own pinned copy (hands/models.sh)
|
||||
hands/models/student/
|
||||
hands/models/.student-*
|
||||
@@ -59,11 +59,13 @@ jobs:
|
||||
test -f /src/frametop/pointer/driver/build/driver_ft_pointer.so
|
||||
test -x /src/frametop/hands/build/ft-camd
|
||||
test -x /src/frametop/hands/build/ft-hands
|
||||
test -x /src/frametop/hands/rec/build/ft-handpanel
|
||||
/src/frametop/hands/models.sh check
|
||||
test -x /src/frametop/pack/build/distrobox/install
|
||||
/src/frametop/gaze/tracker/build/venv/bin/python -c "import numpy, cv2"
|
||||
/src/frametop/voice/build/venv/bin/python -c "import numpy, sentencepiece, sherpa_onnx"
|
||||
python3 -c "import PySide6"'
|
||||
python3 -c "import PySide6, huggingface_hub"
|
||||
command -v zstd >/dev/null'
|
||||
|
||||
# The FrameDrop manifest names the zip's URL in this repo's release (a fork's, in a fork).
|
||||
- name: Frametop.zip
|
||||
|
||||
@@ -197,9 +197,19 @@ cd ~/frametop && scripts/report.sh
|
||||
|
||||
From a release, start in `~/.local/share/frametop/releases/current` instead of `~/frametop`.
|
||||
|
||||
This writes `frametop-report-<date>.txt` with version numbers, service states, settings, Frametop's keyboard and the Steam menu, and recent logs. Bluetooth addresses and the headset's serial number are masked.
|
||||
This writes `frametop-report-<date>.txt` with version numbers, settings, service states, CPU, memory and heat, and the state and recent logs of every part: the desktop (Frametop's keyboard, the Steam menu, the screens), gaze, hand tracking (its models, cameras, and how often it sees each hand), and voice (its models, the microphone, and whether whisper-server transcribes). It takes up to half a minute, because it starts the gaze service for a moment to see whether the eye tracker sends. Bluetooth addresses and the headset's serial number are masked, and what you said to Frametop Voice is left out.
|
||||
|
||||
If the problem is something you can make happen, like a window that won't drag or a keyboard that doesn't open, run `scripts/report.sh --watch` instead. After the usual report it records for 60 seconds (`--watch 120` for longer) while you make it happen in the headset. It notes when the Steam menu opens and closes, which laser drags what, where typing goes, and when Frametop's keyboard opens or why it doesn't. It takes up to half a minute, because it also checks gaze mode: it starts the gaze service for a moment to see whether the eye tracker sends. If gaze or its calibration doesn't work, run it while you wear the headset. `scripts/gaze-report.py` prints only the gaze part, with what looks wrong first. Then [open an issue](https://github.com/Frametop/frametop/issues), describe what you did, what you expected, and what happened, and attach the file. Quick questions can go to [Discord](https://discord.gg/W3X9f7z3Bc) instead.
|
||||
To report on one part only, name it: `--system`, `--desktop`, `--gaze`, `--hands` or `--voice` (several are fine, for example `scripts/report.sh --hands --voice`).
|
||||
|
||||
If the problem is something you can make happen, like a window that won't drag, a pinch that doesn't register, or dictation that types nothing, add `--watch`. After the report it records for 60 seconds (`--watch 120` for longer) while you make it happen in the headset, and keeps the logs from that time:
|
||||
|
||||
- desktop: when the Steam menu opens and closes, which laser drags what, where typing goes, and when Frametop's keyboard opens or why it doesn't
|
||||
- gaze: which eye tracker the gaze service uses, and whether its samples come in
|
||||
- hands: each second, how many frames hand tracking made, how often it saw each hand, and how close the fingertips came
|
||||
- voice: what Frametop Voice did (the number of words, not the words) and the microphone's level each second (no sound is kept)
|
||||
- system: every 5 seconds, the CPU's load, clocks and heat, and the busiest programs
|
||||
|
||||
If gaze or its calibration doesn't work, run the report while you wear the headset. `scripts/gaze-report.py` prints only the gaze part, with what looks wrong first. Then [open an issue](https://github.com/Frametop/frametop/issues), describe what you did, what you expected, and what happened, and attach the file. Quick questions can go to [Discord](https://discord.gg/W3X9f7z3Bc) instead.
|
||||
|
||||
## Update
|
||||
|
||||
|
||||
+1
-1
@@ -137,7 +137,7 @@ pointer/driver/install.sh probe # devices, hand roles, activity, charging, w
|
||||
pointer/probe/build.sh # probes: vrprobe, lasertest (who has the laser), ctlnoise (controller noise)
|
||||
```
|
||||
|
||||
The pointer settings are in `~/.config/frametop.conf`: `POINTER_SENSITIVITY`, `POINTER_IDLE`, `POINTER_WAKE_COUNTS`, `POINTER_CONTROLLER_PICKUP`, `POINTER_DISTANCE`, `POINTER_CURSOR_DEG`, `POINTER_ORIGIN_FRACTION`, `POINTER_ORIGIN_MARGIN`, `POINTER_SCENE_RADIUS`, `POINTER_EDGE_REACH`, `POINTER_LASER_WIDTH`, `POINTER_IGNORE`, the head follow settings `POINTER_FOLLOW`, `POINTER_LEASH_DEG`, `POINTER_LEASH_DELAY`, `POINTER_LEASH_RETURN`, and `POINTER_FOLLOW_REACH`, and the gaze mode settings `POINTER_GAZE`, `POINTER_GAZE_RETAKE`, `POINTER_GAZE_NUDGE_MAX`, `POINTER_GAZE_HOLD`, `POINTER_GAZE_CONTROL_HOLD`, `POINTER_GAZE_DOT`, `POINTER_GAZE_SHOW`, `POINTER_GAZE_MOUSE`, `POINTER_GAZE_MOUSE_MOVE`, and the keyboard clicks' `POINTER_HEAD_DEADZONE` and `POINTER_KEY_TAP`, and the gaze service's `GAZE_TRACKER` (`auto`, the default: our own eye tracker when it's installed, else SteamVR's; or `own` or `steam`) and `GAZE_EYE` (the eye bias). The example config explains each. Frametop Input Settings changes them live; after editing the file by hand, restart the relay or the helper (the gaze service reads its two again when the file changes).
|
||||
The pointer settings are in `~/.config/frametop.conf`: `POINTER_SENSITIVITY`, `POINTER_IDLE`, `POINTER_WAKE_COUNTS`, `POINTER_CONTROLLER_PICKUP`, `POINTER_DISTANCE`, `POINTER_CURSOR_DEG`, `POINTER_ORIGIN_FRACTION`, `POINTER_ORIGIN_MARGIN`, `POINTER_SCENE_RADIUS`, `POINTER_EDGE_REACH`, `POINTER_LASER_WIDTH`, `POINTER_IGNORE`, the head follow settings `POINTER_FOLLOW`, `POINTER_LEASH_DEG`, `POINTER_LEASH_DELAY`, `POINTER_LEASH_RETURN`, and `POINTER_FOLLOW_REACH`, and the gaze mode settings `POINTER_GAZE`, `POINTER_GAZE_RETAKE`, `POINTER_GAZE_NUDGE_MAX`, `POINTER_GAZE_HOLD`, `POINTER_GAZE_CONTROL_HOLD`, `POINTER_GAZE_DOT`, `POINTER_GAZE_SHOW`, `POINTER_GAZE_MOUSE`, `POINTER_GAZE_MOUSE_MOVE`, and the keyboard clicks' `POINTER_HEAD_DEADZONE` and `POINTER_KEY_TAP`, and the gaze service's `GAZE_TRACKER` (`auto`, the default: our own eye tracker when it's installed, else SteamVR's; or `own` or `steam`), `GAZE_EYE` (the eye bias), and `GAZE_FIT_WATCH` (the put-on check and the slip note, off by default: [gaze/README.md](../gaze/README.md), "Headset fit"). The example config explains each. Frametop Input Settings changes them live; after editing the file by hand, restart the relay or the helper (the gaze service reads its three again when the file changes).
|
||||
|
||||
## Frametop Input Settings
|
||||
|
||||
|
||||
+2
-2
@@ -1,6 +1,6 @@
|
||||
# Install with FrameDrop (proof of concept)
|
||||
|
||||
[FrameDrop](https://framedropvr.com) is a Windows app that sideloads onto a Steam Frame: it copies a build to the headset and adds it to the Steam library. Issue #25 asks for an "Install with FrameDrop" button. Frametop isn't an app FrameDrop can copy over as is: it installs user services, a SteamVR driver, and a container, and three optional parts need sudo. So FrameDrop installs a small installer instead. Playing "Frametop" from the library opens a window that asks what to install, and your password for the parts that need it, then installs and shows its progress. A release's Frametop.zip (about 1.1 GB, built by CI on a tag, see [pack/README.md](../pack/README.md), Releases) carries Frametop built, as an image, and installs it with `install-release.sh`: nothing compiles on the headset and nothing else downloads. The same zip works unpacked on the headset. A test zip (a few KB) clones Frametop with `get.sh` instead.
|
||||
[FrameDrop](https://framedropvr.com) is a Windows app that sideloads onto a Steam Frame: it copies a build to the headset and adds it to the Steam library. Issue #25 asks for an "Install with FrameDrop" button. Frametop isn't an app FrameDrop can copy over as is: it installs user services, a SteamVR driver, and a container, and four optional parts need sudo. So FrameDrop installs a small installer instead. Playing "Frametop" from the library opens a window that asks what to install, and your password for the parts that need it, then installs and shows its progress. A release's Frametop.zip (about 1.1 GB, built by CI on a tag, see [pack/README.md](../pack/README.md), Releases) carries Frametop built, as an image, and installs it with `install-release.sh`: nothing compiles on the headset and nothing else downloads. The same zip works unpacked on the headset. A test zip (a few KB) clones Frametop with `get.sh` instead.
|
||||
|
||||
Nothing here is published yet: no release, no button.
|
||||
|
||||
@@ -26,7 +26,7 @@ It uses Valve's SteamOS Devkit path: pair once with the headset's devkit service
|
||||
|
||||
1. In the container, it starts itself again on the host with `flatpak-spawn --host`.
|
||||
2. It clears Steam's preload and library paths, and opens `installer/progress.py` (GTK 4 and libadwaita).
|
||||
3. The window asks which optional parts to install: voice dictation (on by default), hand tracking (on by default), our own eye tracker (on by default), and the Bluetooth fixes. All but voice dictation need sudo, so it asks for your SteamOS password and checks it with `sudo -v`. If your user has no password (SteamOS starts without one), it says how to set one and leaves those three out.
|
||||
3. The window asks which optional parts to install: voice dictation (on by default), hand tracking (on by default), the Hand Recorder (on by default; a release's zip only, since a clone builds it with `hands/rec/install.sh`), our own eye tracker (on by default), and the Bluetooth fixes. All but voice dictation need sudo, so it asks for your SteamOS password and checks it with `sudo -v`. If your user has no password (SteamOS starts without one), it says how to set one and leaves those out.
|
||||
4. It runs the zip's `install-release.sh --yes` (a test zip: `get.sh --yes`) in a transient user service, `frametop-framedrop-install`. The service is used because Steam ends the title's whole process tree when it's quit, and starts it with an OOM score of 900. Opened from a VR desktop (the zip unpacked by hand), it still uses the user's real bus and runtime folder for the service.
|
||||
5. It follows the service's log, shows the steps as a progress bar, and reports the result. Closing it leaves the install running. Playing the title again reattaches.
|
||||
|
||||
|
||||
@@ -13,7 +13,7 @@ The install runs in a user service of its own (UNIT): Steam ends the title's who
|
||||
tree when it's quit, and starts it with a high OOM score. Closing the window doesn't stop the
|
||||
install; playing the title again reattaches to it.
|
||||
|
||||
Our eye tracker and the Bluetooth fixes need sudo. The window asks for your password on the
|
||||
Hand tracking, the Hand Recorder, our eye tracker and the Bluetooth fixes need sudo. The window asks for your password on the
|
||||
headset, checks it with sudo, and keeps it in this process's memory until the install ends.
|
||||
sudo in the install gets it through askpass (SUDO_ASKPASS), from a socket in a folder only you
|
||||
can open, and the window answers only programs in the install's service. It's never written to
|
||||
@@ -101,7 +101,7 @@ def in_unit(pid):
|
||||
return False
|
||||
|
||||
|
||||
def install_command(eye_tracker, bluetooth, voice=True, hands=True):
|
||||
def install_command(eye_tracker, bluetooth, voice=True, hands=True, hand_recorder=True):
|
||||
if RELEASE:
|
||||
args = [str(HERE / "install-release.sh"), "--yes"]
|
||||
if DRY_RUN:
|
||||
@@ -114,6 +114,8 @@ def install_command(eye_tracker, bluetooth, voice=True, hands=True):
|
||||
args.append("--no-voice")
|
||||
if not hands:
|
||||
args.append("--no-hands")
|
||||
if RELEASE and not hand_recorder: # a clone (get.sh) doesn't offer it: hands/rec/install.sh builds it
|
||||
args.append("--no-hand-recorder")
|
||||
if not eye_tracker:
|
||||
args.append("--no-eye-tracker")
|
||||
if bluetooth:
|
||||
@@ -306,12 +308,16 @@ class Window(Adw.ApplicationWindow):
|
||||
def show_choices(self):
|
||||
what = f"Frametop {release_version()}" if RELEASE else "Frametop from GitHub"
|
||||
self.detail.set_label(f"This installs {what}: the multi-screen desktop, the 3D mouse, gaze "
|
||||
"mode, and Frametop's settings apps. Optional: voice dictation, and three "
|
||||
"mode, and Frametop's settings apps. Optional: voice dictation, and "
|
||||
"parts that need your SteamOS password (sudo):")
|
||||
self.voice = Gtk.CheckButton(label="Voice dictation: a Right Ctrl tap or holding the left d-pad down, and "
|
||||
"what you say is typed (downloads about 320 MB)", active=True)
|
||||
self.hands = Gtk.CheckButton(label="Hand tracking (experimental): poke, pinch and grab the screens with "
|
||||
"your hands. It uses a lot of the headset's CPU", active=True)
|
||||
# A release brings it built; a clone builds it with hands/rec/install.sh, so isn't offered it
|
||||
self.recorder = Gtk.CheckButton(label="Hand Recorder: record your hands for Frametop's open hand "
|
||||
"dataset, which trains hand tracking (nothing is uploaded "
|
||||
"until you press Upload)", active=True, visible=RELEASE)
|
||||
self.eye = Gtk.CheckButton(label="Our own eye tracker for gaze mode (more accurate than SteamVR's)",
|
||||
active=True)
|
||||
self.bt = Gtk.CheckButton(label="Bluetooth fixes (LE mice and keyboards, like the Swiftpoint Z3, "
|
||||
@@ -327,18 +333,20 @@ class Window(Adw.ApplicationWindow):
|
||||
self.go.connect("clicked", self.install)
|
||||
self.pw.connect("activate", self.install)
|
||||
pw_row, keypad = keypad_row(self.pw)
|
||||
self.choices = [self.voice, self.hands, self.eye, self.bt, pw_row, keypad, pw_note, self.error, self.go]
|
||||
self.choices = [self.voice, self.hands, self.recorder, self.eye, self.bt, pw_row, keypad, pw_note,
|
||||
self.error, self.go]
|
||||
if not password_set():
|
||||
for c in (self.hands, self.eye, self.bt):
|
||||
for c in (self.hands, self.recorder, self.eye, self.bt):
|
||||
c.set_active(False)
|
||||
c.set_sensitive(False)
|
||||
pw_row.set_visible(False)
|
||||
keypad.set_visible(False)
|
||||
pw_note.set_label("Your user has no password, so sudo can't run and these three can't be "
|
||||
pw_note.set_label("Your user has no password, so sudo can't run and these can't be "
|
||||
"installed from here. To set one, run passwd in Konsole; then play "
|
||||
"Frametop again, or install them later from a terminal "
|
||||
"(hands/run.sh install, gaze/tracker/install.sh, setup/bluetooth/install.sh).")
|
||||
for c in (self.hands, self.eye, self.bt):
|
||||
"(hands/run.sh install, hands/rec/install.sh, gaze/tracker/install.sh, "
|
||||
"setup/bluetooth/install.sh).")
|
||||
for c in (self.hands, self.recorder, self.eye, self.bt):
|
||||
c.connect("toggled", lambda *_: pw_row.set_sensitive(self.needs_password()))
|
||||
for w in self.choices:
|
||||
self.box.append(w)
|
||||
@@ -374,7 +382,8 @@ class Window(Adw.ApplicationWindow):
|
||||
threading.Thread(target=check, daemon=True).start()
|
||||
|
||||
def needs_password(self):
|
||||
return self.hands.get_active() or self.eye.get_active() or self.bt.get_active()
|
||||
return (self.hands.get_active() or (RELEASE and self.recorder.get_active()) or self.eye.get_active()
|
||||
or self.bt.get_active())
|
||||
|
||||
def say(self, text):
|
||||
self.error.set_label(text)
|
||||
@@ -382,7 +391,7 @@ class Window(Adw.ApplicationWindow):
|
||||
|
||||
def begin(self, pw):
|
||||
args = install_command(self.eye.get_active(), self.bt.get_active(), self.voice.get_active(),
|
||||
self.hands.get_active())
|
||||
self.hands.get_active(), self.recorder.get_active())
|
||||
if pw is not None:
|
||||
try:
|
||||
self.askpass = Askpass(self.ask_again)
|
||||
|
||||
@@ -74,6 +74,11 @@ Ground rules, for anyone changing it:
|
||||
|
||||
Check headset fit on the Gaze page opens it in the headset panel: a card per eye (tracked or lost, the tracker's signal, how much of the last 10 s it was seen) and the hints, live while you adjust the headset. A left click or Meta+J runs the guided check (dots, then looks down, up, left and right), and a right click or Meta+K closes it. The probe's Headset fit mode (`ft-gazeprobe --mode fit`, for development) has the same check with maps: it shows, for each eye, whether the tracker has it, how open it is, and the tracker's confidence in it, and a map of where you looked coloured by how often it lost that eye there. Hints under the maps say which eye gets lost where, and what to try. Enter runs a guided check: dots around the screen, then looking down at the keyboard, up, left and right. R starts over. Adjust the headset while you watch it.
|
||||
|
||||
**The fit watch** (`GAZE_FIT_WATCH=1` in `~/.config/frametop.conf`; off by default until it's been tried in the headset). The fit check keeps a good fit as yours: when its coach (`gaze/fitcoach.py`, from ft-eyes' pupils and glints, with the shipped reference) last read it as good while you looked straight ahead, closing the check saves that reading to `~/.local/state/frametop/gaze/fit-baseline.json`, with the IPD setting. The eye recorder's fitment step saves it too. Then, with the fit watch on:
|
||||
- **Put-on check:** when the headset goes on, the quick check first holds its dot for up to 4 s while the coach reads the fit against your saved one. If it sits the same, the dot is taken as usual. If it moved (5 degrees of gaze or more), the panel widens and says which way to move the headset, and a changed IPD over that, live, until it's back ("Good: it's back where it was"); then the dot is taken. A click takes the dot as the headset sits, and after 30 s it's taken anyway. It needs our tracker installed (ft-eyes runs for it with either tracker in use), and without a saved fit it's the plain quick check.
|
||||
- **Slip note:** with our tracker in use and gaze mode on, ft-eyes' glint slip (its `jump_px`, how far each pupil moved in its image since the last click) is turned into degrees through the reference. Past 5 degrees for 2 s, a small strip low in your view says "Your headset slipped" and which way to move it, for 8 s or until the slip is gone, at most once every 2 minutes. It isn't a check: the pointer stays, and a click or the quick check still teaches the tracker the new seating.
|
||||
`gaze/test/fit-watch-test.py` tests both offline.
|
||||
|
||||
Losing an eye is usually about where you look, not the tracker. On this Frame the left eye was lost 57 to 64 % of the time looking 30 to 50 degrees down (at the keyboard) and the right eye never; at screen height both were seen over 98 % of the time. Looking down, the lids come down over the eyes. That's harmless, since the gaze service ignores looks down past the screens: they show on the maps, but not in the counts or as a problem.
|
||||
|
||||
## Probe
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
{
|
||||
"schema": 1,
|
||||
"made": "2026-10-14",
|
||||
"source": "the maintainer's headset: 2 of 3 own-tracker calibrations (2026-10-08, 2026-10-09), the third fitted worse than 4 px",
|
||||
"faces": 1,
|
||||
"ipd": 0.0651,
|
||||
"cameras_follow_lenses": null,
|
||||
"eyes": {
|
||||
"0": {
|
||||
"centre": [
|
||||
341.04,
|
||||
221.4
|
||||
],
|
||||
"jacobian": [
|
||||
[
|
||||
1.2749,
|
||||
-0.4221
|
||||
],
|
||||
[
|
||||
-0.4783,
|
||||
-1.3388
|
||||
]
|
||||
],
|
||||
"fits": 2,
|
||||
"fit_err": 2.339
|
||||
},
|
||||
"1": {
|
||||
"centre": [
|
||||
334.74,
|
||||
173.53
|
||||
],
|
||||
"jacobian": [
|
||||
[
|
||||
-1.3415,
|
||||
-0.5307
|
||||
],
|
||||
[
|
||||
-0.4822,
|
||||
1.4794
|
||||
]
|
||||
],
|
||||
"fits": 2,
|
||||
"fit_err": 2.146
|
||||
}
|
||||
}
|
||||
}
|
||||
Executable
+521
@@ -0,0 +1,521 @@
|
||||
#!/usr/bin/env python3
|
||||
"""fitcoach: how the headset sits on this face, from where our tracker sees each eye, and what
|
||||
to do about it (gaze/rec/DESIGN.md, "Fit coach"). The eye recorder runs it before each round;
|
||||
the gaze service's fit check, put-on check and slip notice use it too (gaze/gazecheck.py).
|
||||
|
||||
It takes per-frame pupil finds (gaze/tracker/eyes_pupil.find_pupil: centre, axes, glints) for
|
||||
each camera (0 the right eye, 1 the left), keeps the last second, and compares the medians
|
||||
with a reference: where a well-fitted eye sits in that camera's image while looking straight
|
||||
ahead, and how the pupil moves per degree of gaze (the reference's Jacobian, fitted on
|
||||
calibration dots). Offsets go through that Jacobian into degrees as if the eye had turned, so
|
||||
the cameras' orientations don't matter (camera 1's image is turned 180 degrees from camera 0's:
|
||||
looking left moves its pupil the other way) and both eyes' readings can be added up:
|
||||
|
||||
both eyes appear to look up the eyes sit high in the headset: it's too low -> push it up
|
||||
both appear to look left the headset sits off to the right -> slide it left
|
||||
the eyes appear to look apart the eyes are wider apart than the lenses -> IPD wider
|
||||
one appears to look higher the headset is tilted -> level it
|
||||
|
||||
The IPD reading only becomes a hint once we know the eye cameras move with the lenses
|
||||
(HEADSET-TESTS, 2026-10-14): if they're fixed to the housing, the offset also depends on the IPD
|
||||
setting, and that needs a reference per setting. Until then it's reported, not advised.
|
||||
|
||||
A baseline is the same reading saved from one person's good fit. Compared with a baseline the
|
||||
glint pair's spacing says whether the eye is nearer the camera than then (the eye relief: on
|
||||
the maintainer's headset it moved 18 % between two seatings, and 4 % within one), and the put-on
|
||||
check says how far the headset moved since. The reference starts as the maintainer's
|
||||
(gaze/fit-reference.json); its hints are marked provisional until it's built from more faces.
|
||||
|
||||
Measured on the maintainer's headset (2026-10-14, three calibrations of 27 dots): straight-ahead
|
||||
pupils moved 15-20 px between seatings, at about 1.3 px per degree; so OK within OK_DEG and
|
||||
adjust past ADJUST_DEG, both provisional.
|
||||
|
||||
Usage (on the Frame, in the dev container, with frametop-eyegrab running):
|
||||
python3 gaze/fitcoach.py live [--baseline] print the reading twice a second; --baseline saves
|
||||
the last one as your baseline on Ctrl-C
|
||||
python3 gaze/fitcoach.py reference [DOTS...] fit a reference from calibration-dots.jsonl files
|
||||
"""
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import statistics
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from collections import deque
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
REFERENCE = os.path.join(HERE, "fit-reference.json")
|
||||
BASELINE = os.path.expanduser("~/.local/state/frametop/gaze/fit-baseline.json")
|
||||
DOTS = os.path.expanduser("~/.local/state/frametop/gaze/eyes/calibration-dots.jsonl")
|
||||
EYES = ("right", "left") # camera 0, camera 1
|
||||
WINDOW = 1.0 # s of finds per reading
|
||||
MIN_FINDS = 10 # finds in the window before an eye is judged
|
||||
MIN_FOUND = 0.5 # share of frames with a pupil: under this, the eye is lost
|
||||
OK_DEG = 5.0 # every term within this: the fit is good (provisional)
|
||||
ADJUST_DEG = 8.0 # a term past this: say what to do (between the two: keep the last state)
|
||||
TILT_DEG = 4.0 # half the eyes' vertical difference
|
||||
RELIEF_TOL = 0.08 # glint spacing this much off the baseline: nearer or further
|
||||
MANY_GLINTS = 5 # a find with this many glints has stray reflections. Without glasses
|
||||
# (three maintainer captures, 2026-10-14) 3+ glints came in up to 37 % of
|
||||
# finds and 5+ in up to 12 %: a guess until the Glasses bounty's data
|
||||
EXTRA_GLINTS = 0.5 # share of finds with MANY_GLINTS: reflections (glasses, a smudge)
|
||||
MOVED_DEG = 5.0 # put-on check: this far from the baseline, the headset moved; the slip
|
||||
# notice: ft-eyes' glint jump this far
|
||||
|
||||
|
||||
# ------------------------------------------------------------------------------------------
|
||||
# Readings
|
||||
|
||||
def glint_spacing(p):
|
||||
"""The LED glint pair's spacing in pixels, or None (eyes_pupil.glint_pair's rule)."""
|
||||
pair = _glint_pair(p)
|
||||
if not pair:
|
||||
return None
|
||||
(x0, y0), (x1, y1) = pair
|
||||
return math.hypot(x1 - x0, y1 - y0)
|
||||
|
||||
|
||||
def _glint_pair(p):
|
||||
"""eyes_pupil.glint_pair without numpy: the vertical-ish pair nearest the pupil centre."""
|
||||
g, best = p.get("glints") or [], None
|
||||
for i in range(len(g)):
|
||||
for j in range(i + 1, len(g)):
|
||||
dx, dy = g[j][0] - g[i][0], g[j][1] - g[i][1]
|
||||
d = math.hypot(dx, dy)
|
||||
if not 5 <= d <= 45 or abs(dy) < 2 * abs(dx):
|
||||
continue
|
||||
cost = math.hypot((g[i][0] + g[j][0]) / 2 - p["x"], (g[i][1] + g[j][1]) / 2 - p["y"])
|
||||
if best is None or cost < best[0]:
|
||||
best = (cost, (g[i], g[j]) if dy > 0 else (g[j], g[i]))
|
||||
return best[1] if best else None
|
||||
|
||||
|
||||
class Observations:
|
||||
"""The last WINDOW seconds of pupil finds per camera. add() from any thread."""
|
||||
|
||||
def __init__(self, window=WINDOW):
|
||||
self.window = window
|
||||
self._rows = {0: deque(), 1: deque()} # (t, x, y, a, spacing or None, extra glints) or (t, None)
|
||||
self._lock = threading.Lock()
|
||||
|
||||
def add(self, cam, t, p):
|
||||
if p is None:
|
||||
row = (t, None)
|
||||
else:
|
||||
row = (t, float(p["x"]), float(p["y"]), float(p["a"]), glint_spacing(p), len(p.get("glints") or []) >= MANY_GLINTS)
|
||||
with self._lock:
|
||||
rows = self._rows[cam]
|
||||
rows.append(row)
|
||||
while rows and rows[0][0] < t - self.window:
|
||||
rows.popleft()
|
||||
|
||||
def summary(self, now=None):
|
||||
"""{camera: {"n", "found", "x", "y", "a", "spacing", "extra"}}: medians of the finds,
|
||||
found the share of frames with a pupil, extra the share with MANY_GLINTS or more. A camera with
|
||||
no frames in the window: {"n": 0}."""
|
||||
out = {}
|
||||
with self._lock:
|
||||
for cam, rows in self._rows.items():
|
||||
rs = [r for r in rows if now is None or r[0] >= now - self.window]
|
||||
finds = [r for r in rs if r[1] is not None]
|
||||
if not rs:
|
||||
out[cam] = {"n": 0}
|
||||
continue
|
||||
sp = [r[4] for r in finds if r[4] is not None]
|
||||
out[cam] = {"n": len(rs), "finds": len(finds), "found": len(finds) / len(rs),
|
||||
"x": statistics.median(r[1] for r in finds) if finds else None,
|
||||
"y": statistics.median(r[2] for r in finds) if finds else None,
|
||||
"a": statistics.median(r[3] for r in finds) if finds else None,
|
||||
"spacing": statistics.median(sp) if len(sp) >= 3 else None,
|
||||
"extra": sum(r[5] for r in finds) / len(finds) if finds else 0.0}
|
||||
return out
|
||||
|
||||
|
||||
# ------------------------------------------------------------------------------------------
|
||||
# The reference and the baseline
|
||||
|
||||
def _solve3(a, b):
|
||||
"""x for a 3x3 system a x = b (Gaussian elimination with pivoting), or None if singular."""
|
||||
m = [row[:] + [v] for row, v in zip(a, b)]
|
||||
for i in range(3):
|
||||
k = max(range(i, 3), key=lambda r: abs(m[r][i]))
|
||||
if abs(m[k][i]) < 1e-12:
|
||||
return None
|
||||
m[i], m[k] = m[k], m[i]
|
||||
for r in range(3):
|
||||
if r != i:
|
||||
f = m[r][i] / m[i][i]
|
||||
m[r] = [x - f * y for x, y in zip(m[r], m[i])]
|
||||
return [m[i][3] / m[i][i] for i in range(3)]
|
||||
|
||||
|
||||
def fit_linear(truths, values):
|
||||
"""value = c0 + c1 yaw + c2 pitch by least squares: [c0, c1, c2], or None."""
|
||||
rows = [(1.0, float(t[0]), float(t[1])) for t in truths]
|
||||
ata = [[sum(r[i] * r[j] for r in rows) for j in range(3)] for i in range(3)]
|
||||
atb = [sum(r[i] * v for r, v in zip(rows, values)) for i in range(3)]
|
||||
return _solve3(ata, atb)
|
||||
|
||||
|
||||
class Reference:
|
||||
"""Per camera: the straight-ahead pupil centre (px) and the Jacobian [[dx/dyaw, dx/dpitch],
|
||||
[dy/dyaw, dy/dpitch]] (px per degree); optionally the glint spacing, the IPD it was made
|
||||
at (m), whether the eye cameras move with the lenses, where it came from and from how many faces."""
|
||||
|
||||
def __init__(self, eyes, ipd=None, source="", faces=1, cameras_follow_lenses=None, made=""):
|
||||
self.eyes, self.ipd, self.source, self.faces = eyes, ipd, source, faces
|
||||
self.cameras_follow_lenses, self.made = cameras_follow_lenses, made
|
||||
|
||||
@property
|
||||
def provisional(self):
|
||||
return self.faces < 3
|
||||
|
||||
@classmethod
|
||||
def from_dots(cls, rows, source="", ipd=None, max_err=4.0):
|
||||
"""From calibration-dots.jsonl rows ({"made", "truth": [yaw, pitch], "eye": {"0": {"pupil": [x, y]}}}).
|
||||
Each calibration ("made") is fitted on its own; those that fit worse than max_err px
|
||||
(median) are left out, and the rest averaged."""
|
||||
fits = {0: [], 1: []}
|
||||
for made in sorted({r.get("made", "") for r in rows}):
|
||||
for cam in (0, 1):
|
||||
rs = [r for r in rows if r.get("made", "") == made and (r.get("eye") or {}).get(str(cam))]
|
||||
if len(rs) < 6:
|
||||
continue
|
||||
truths = [r["truth"] for r in rs]
|
||||
px = [r["eye"][str(cam)]["pupil"] for r in rs]
|
||||
cx, cy = fit_linear(truths, [p[0] for p in px]), fit_linear(truths, [p[1] for p in px])
|
||||
if cx is None or cy is None:
|
||||
continue
|
||||
err = statistics.median(math.hypot(cx[0] + cx[1] * t[0] + cx[2] * t[1] - p[0],
|
||||
cy[0] + cy[1] * t[0] + cy[2] * t[1] - p[1]) for t, p in zip(truths, px))
|
||||
if err <= max_err:
|
||||
fits[cam].append((cx, cy, err))
|
||||
eyes = {}
|
||||
for cam, fs in fits.items():
|
||||
if not fs:
|
||||
continue
|
||||
avg = lambda k, i: sum(f[k][i] for f in fs) / len(fs) # noqa: E731
|
||||
eyes[cam] = {"centre": [avg(0, 0), avg(1, 0)], "jacobian": [[avg(0, 1), avg(0, 2)], [avg(1, 1), avg(1, 2)]],
|
||||
"fits": len(fs), "fit_err": statistics.median(f[2] for f in fs)}
|
||||
return cls(eyes, ipd=ipd, source=source, made=time.strftime("%Y-%m-%d"))
|
||||
|
||||
def degrees(self, cam, dx, dy):
|
||||
"""A pupil offset in px as the gaze change it would be: (yaw, pitch) degrees."""
|
||||
(a, b), (c, d) = self.eyes[cam]["jacobian"]
|
||||
det = a * d - b * c
|
||||
if abs(det) < 1e-9:
|
||||
return 0.0, 0.0
|
||||
return (d * dx - b * dy) / det, (-c * dx + a * dy) / det
|
||||
|
||||
def to_json(self):
|
||||
eyes = {str(k): {"centre": [round(v, 2) for v in e["centre"]],
|
||||
"jacobian": [[round(v, 4) for v in row] for row in e["jacobian"]],
|
||||
**{k2: (round(v2, 3) if isinstance(v2, float) else v2) for k2, v2 in e.items()
|
||||
if k2 not in ("centre", "jacobian")}}
|
||||
for k, e in sorted(self.eyes.items())}
|
||||
return {"schema": 1, "made": self.made, "source": self.source, "faces": self.faces, "ipd": self.ipd,
|
||||
"cameras_follow_lenses": self.cameras_follow_lenses,
|
||||
"eyes": eyes}
|
||||
|
||||
@classmethod
|
||||
def from_json(cls, d):
|
||||
return cls({int(k): v for k, v in d.get("eyes", {}).items()}, ipd=d.get("ipd"), source=d.get("source", ""),
|
||||
faces=d.get("faces", 1), cameras_follow_lenses=d.get("cameras_follow_lenses"), made=d.get("made", ""))
|
||||
|
||||
@classmethod
|
||||
def load(cls, path=REFERENCE):
|
||||
try:
|
||||
with open(path) as f:
|
||||
return cls.from_json(json.load(f))
|
||||
except (OSError, ValueError):
|
||||
return None
|
||||
|
||||
def save(self, path=REFERENCE):
|
||||
_write_json(path, self.to_json())
|
||||
|
||||
|
||||
def make_baseline(summary, ipd=None):
|
||||
"""A baseline from a good reading: per camera the centre and glint spacing, and the IPD."""
|
||||
eyes = {str(c): {"centre": [s["x"], s["y"]], "spacing": s.get("spacing"), "a": s.get("a")}
|
||||
for c, s in summary.items() if s.get("x") is not None}
|
||||
return {"schema": 1, "made": time.strftime("%Y-%m-%d %H:%M:%S"), "ipd": ipd, "eyes": eyes}
|
||||
|
||||
|
||||
def load_baseline(path=BASELINE):
|
||||
try:
|
||||
with open(path) as f:
|
||||
return json.load(f)
|
||||
except (OSError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def save_baseline(baseline, path=None):
|
||||
"""Keep a good fit as the person's (the gaze service's fit check, the eye recorder's fitment)."""
|
||||
_write_json(path or BASELINE, baseline)
|
||||
|
||||
|
||||
def _write_json(path, obj):
|
||||
os.makedirs(os.path.dirname(path), exist_ok=True)
|
||||
tmp = path + ".tmp"
|
||||
with open(tmp, "w") as f:
|
||||
json.dump(obj, f, indent=1)
|
||||
f.write("\n")
|
||||
os.replace(tmp, path)
|
||||
|
||||
|
||||
# ------------------------------------------------------------------------------------------
|
||||
# The assessment
|
||||
|
||||
HINTS = {
|
||||
"up": "Push the headset up a little (the top strap).",
|
||||
"down": "Let the headset down a little.",
|
||||
"left": "Slide the headset a little to your left.",
|
||||
"right": "Slide the headset a little to your right.",
|
||||
"ipd-wider": "Turn the IPD wheel a little wider.",
|
||||
"ipd-narrower": "Turn the IPD wheel a little narrower.",
|
||||
"tilt-left-up": "Level the headset: lift its left side a little.",
|
||||
"tilt-right-up": "Level the headset: lift its right side a little.",
|
||||
"nearer": "The headset sits closer to your eyes than your saved fit: loosen the back strap a little.",
|
||||
"further": "The headset sits further from your eyes than your saved fit: tighten the back strap a little.",
|
||||
"reflections": "Stray reflections in the {eye} eye's camera: tilt your glasses a little, or wipe the lens.",
|
||||
"lost": "The {eye} eye is hard to see: lashes touching the lens, or the headset too low on that side?",
|
||||
"ipd-changed": "The IPD setting changed since your saved fit ({was:.1f} mm, now {now:.1f} mm).",
|
||||
}
|
||||
|
||||
|
||||
def assess(summary, ref, ipd=None, baseline=None, previous=None):
|
||||
"""The reading: {"state": "ok" | "adjust" | "lost" | "waiting", "eyes": {camera: {...}},
|
||||
"terms": {"up", "left", "ipd", "tilt"} in degrees, "hints": [{"key", "text", "amount"}],
|
||||
"provisional", "against": "baseline" | "reference"}.
|
||||
|
||||
Against a baseline (the person's own good fit) when there is one, else the reference.
|
||||
previous (an earlier assess() result) keeps the state between OK_DEG and ADJUST_DEG."""
|
||||
eyes, hints, offsets = {}, [], {}
|
||||
centres = {}
|
||||
if baseline:
|
||||
centres = {int(k): v["centre"] for k, v in baseline.get("eyes", {}).items()}
|
||||
for cam, s in summary.items():
|
||||
e = {"name": EYES[cam], "found": round(s.get("found", 0.0), 3), "seen": bool(s.get("n"))}
|
||||
if not s.get("n") or s.get("finds", 0) < MIN_FINDS:
|
||||
e["state"] = "waiting" if not s.get("n") else "lost"
|
||||
elif s["found"] < MIN_FOUND:
|
||||
e["state"] = "lost"
|
||||
else:
|
||||
e["state"] = "found"
|
||||
centre = centres.get(cam) or (ref.eyes.get(cam) or {}).get("centre") if ref else centres.get(cam)
|
||||
if centre and ref and cam in ref.eyes:
|
||||
dx, dy = s["x"] - centre[0], s["y"] - centre[1]
|
||||
offsets[cam] = ref.degrees(cam, dx, dy)
|
||||
e["offset_px"] = [round(dx, 1), round(dy, 1)]
|
||||
e["offset_deg"] = [round(v, 2) for v in offsets[cam]]
|
||||
if s.get("extra", 0.0) > EXTRA_GLINTS:
|
||||
hints.append({"key": "reflections", "text": HINTS["reflections"].format(eye=EYES[cam]), "amount": s["extra"]})
|
||||
if baseline:
|
||||
b = baseline.get("eyes", {}).get(str(cam)) or {}
|
||||
if b.get("spacing") and s.get("spacing"):
|
||||
rel = s["spacing"] / b["spacing"] - 1.0
|
||||
e["relief"] = round(rel, 3)
|
||||
eyes[cam] = e
|
||||
lost = [c for c, e in eyes.items() if e["state"] == "lost"]
|
||||
for c in lost:
|
||||
hints.append({"key": "lost", "text": HINTS["lost"].format(eye=EYES[c]), "amount": 1.0 - eyes[c]["found"]})
|
||||
terms = {}
|
||||
if offsets:
|
||||
yaws = {c: v[0] for c, v in offsets.items()}
|
||||
pitches = {c: v[1] for c, v in offsets.items()}
|
||||
terms["up"] = sum(pitches.values()) / len(pitches) # + : the eyes sit high: the headset is low
|
||||
terms["left"] = sum(yaws.values()) / len(yaws) # + : the eyes sit left: the headset is right
|
||||
if len(offsets) == 2:
|
||||
terms["ipd"] = (yaws[1] - yaws[0]) / 2 # + : the eyes are wider apart than the lenses
|
||||
terms["tilt"] = (pitches[1] - pitches[0]) / 2 # + : the left eye sits higher: its side is low
|
||||
t = terms
|
||||
if abs(t.get("up", 0)) > ADJUST_DEG:
|
||||
hints.append(_hint("up" if t["up"] > 0 else "down", t["up"]))
|
||||
if abs(t.get("left", 0)) > ADJUST_DEG:
|
||||
hints.append(_hint("left" if t["left"] > 0 else "right", t["left"]))
|
||||
ipd_advice = ref is not None and ref.cameras_follow_lenses is True
|
||||
if ipd_advice and abs(t.get("ipd", 0)) > ADJUST_DEG:
|
||||
hints.append(_hint("ipd-wider" if t["ipd"] > 0 else "ipd-narrower", t["ipd"]))
|
||||
if abs(t.get("tilt", 0)) > TILT_DEG:
|
||||
hints.append(_hint("tilt-left-up" if t["tilt"] > 0 else "tilt-right-up", t["tilt"]))
|
||||
reliefs = [e["relief"] for e in eyes.values() if "relief" in e]
|
||||
if reliefs:
|
||||
rel = sum(reliefs) / len(reliefs)
|
||||
terms["relief"] = rel
|
||||
if abs(rel) > RELIEF_TOL:
|
||||
hints.append({"key": "nearer" if rel > 0 else "further", "text": HINTS["nearer" if rel > 0 else "further"],
|
||||
"amount": rel})
|
||||
if baseline and ipd and baseline.get("ipd") and abs(ipd - baseline["ipd"]) >= 0.001:
|
||||
hints.insert(0, {"key": "ipd-changed", "amount": ipd - baseline["ipd"],
|
||||
"text": HINTS["ipd-changed"].format(was=baseline["ipd"] * 1000, now=ipd * 1000)})
|
||||
# The state, with hysteresis on the position terms.
|
||||
big = max([abs(t.get(k, 0.0)) for k in ("up", "left")] + ([abs(t.get("ipd", 0.0))] if ipd_advice else []), default=0.0)
|
||||
if any(e["state"] == "waiting" for e in eyes.values()) or not eyes:
|
||||
state = "waiting"
|
||||
elif lost:
|
||||
state = "lost"
|
||||
elif not offsets:
|
||||
state = "waiting"
|
||||
elif big > ADJUST_DEG or abs(t.get("tilt", 0)) > TILT_DEG or abs(t.get("relief", 0)) > RELIEF_TOL:
|
||||
state = "adjust"
|
||||
elif big <= OK_DEG:
|
||||
state = "ok"
|
||||
else:
|
||||
state = (previous or {}).get("state") if (previous or {}).get("state") in ("ok", "adjust") else "adjust"
|
||||
if state == "adjust" and not hints:
|
||||
# Between OK_DEG and ADJUST_DEG, and not OK before: name the biggest term.
|
||||
names = {"up": ("up", "down"), "left": ("left", "right")}
|
||||
if ipd_advice:
|
||||
names["ipd"] = ("ipd-wider", "ipd-narrower")
|
||||
term = max((k for k in names if k in t), key=lambda k: abs(t[k]), default=None)
|
||||
if term:
|
||||
hints.append(_hint(names[term][0] if t[term] > 0 else names[term][1], t[term]))
|
||||
hints.sort(key=lambda h: (h["key"] not in ("ipd-changed", "lost"), -abs(h["amount"])))
|
||||
return {"state": state, "eyes": eyes, "terms": {k: round(v, 3) for k, v in terms.items()}, "hints": hints,
|
||||
"provisional": bool(ref is None or ref.provisional), "against": "baseline" if centres else "reference",
|
||||
"ipd": ipd, "ipd_advice": ipd_advice}
|
||||
|
||||
|
||||
def _hint(key, amount):
|
||||
return {"key": key, "text": HINTS[key], "amount": amount}
|
||||
|
||||
|
||||
def moved(reading):
|
||||
"""Put-on check: the headset sits more than MOVED_DEG from the baseline (or changed IPD)."""
|
||||
t = reading.get("terms", {})
|
||||
return reading.get("against") == "baseline" and (
|
||||
max(abs(t.get("up", 0.0)), abs(t.get("left", 0.0))) > MOVED_DEG or
|
||||
any(h["key"] == "ipd-changed" for h in reading.get("hints", [])))
|
||||
|
||||
|
||||
def position_hint(terms, limit=MOVED_DEG):
|
||||
"""The biggest position term (up, left) past limit as a hint ("up", "down", "left" or
|
||||
"right"), or None."""
|
||||
names = {"up": ("up", "down"), "left": ("left", "right")}
|
||||
term = max((k for k in names if abs(terms.get(k, 0.0)) > limit), key=lambda k: abs(terms[k]), default=None)
|
||||
if term is None:
|
||||
return None
|
||||
return _hint(names[term][0] if terms[term] > 0 else names[term][1], terms[term])
|
||||
|
||||
|
||||
def slip_terms(jumps, ref):
|
||||
"""ft-eyes' glint jumps ({camera: (dx, dy)}, pixels: how far each pupil moved in its image
|
||||
since the last click, the headset having slipped) as the coach's position terms, {"up",
|
||||
"left"} in degrees, through the reference's Jacobian as assess() does; {} without any."""
|
||||
offsets = [ref.degrees(cam, dx, dy) for cam, (dx, dy) in jumps.items() if ref and cam in ref.eyes]
|
||||
if not offsets:
|
||||
return {}
|
||||
return {"up": sum(o[1] for o in offsets) / len(offsets), "left": sum(o[0] for o in offsets) / len(offsets)}
|
||||
|
||||
|
||||
# ------------------------------------------------------------------------------------------
|
||||
# Live finds from the shared frames (numpy and OpenCV: the dev container, or a venv with them)
|
||||
|
||||
class LiveFinder:
|
||||
"""Runs the pupil finder on each camera's newest frame, at most rate times a second, and adds
|
||||
the finds to obs. About 0.4 ms a frame while it follows the pupil, 2 ms when it searches."""
|
||||
|
||||
def __init__(self, obs, rate=30.0, share_path=None, want_path=None, log=None):
|
||||
sys.path.insert(0, os.path.join(HERE, "tracker"))
|
||||
import eyes_pupil # noqa: E402 (numpy, OpenCV)
|
||||
import eyes_share # noqa: E402
|
||||
import numpy as np # noqa: E402
|
||||
self._pupil, self._share_mod, self._np = eyes_pupil, eyes_share, np
|
||||
self.obs, self.period = obs, 1.0 / rate
|
||||
self.share_path, self.want_path = share_path, want_path
|
||||
self.log = log or (lambda line: None)
|
||||
self.error = None
|
||||
self._stop = threading.Event()
|
||||
self._thread = threading.Thread(target=self._run, name="fitcoach-finder", daemon=True)
|
||||
|
||||
def start(self):
|
||||
self._thread.start()
|
||||
return self
|
||||
|
||||
def stop(self):
|
||||
self._stop.set()
|
||||
self._thread.join(2)
|
||||
|
||||
def _run(self):
|
||||
share = self._share_mod.open_share(10, self.share_path, self.want_path)
|
||||
if share is None:
|
||||
self.error = "no eye camera frames: is frametop-eyegrab.service running?"
|
||||
self.log("fit coach: " + self.error)
|
||||
return
|
||||
last, near, touched = {0: -1, 1: -1}, {0: None, 1: None}, 0.0
|
||||
np = self._np
|
||||
while not self._stop.is_set():
|
||||
t0 = time.monotonic()
|
||||
for cam in (0, 1):
|
||||
n = share.count(cam)
|
||||
if n == last[cam] or n == 0:
|
||||
continue
|
||||
last[cam] = n
|
||||
f = share.frame(cam, n - 1)
|
||||
if f is None:
|
||||
continue
|
||||
img = np.frombuffer(f[2], np.uint8).reshape(self._share_mod.H, self._share_mod.W)
|
||||
p = self._pupil.find_pupil(img, near[cam])
|
||||
near[cam] = p
|
||||
self.obs.add(cam, f[0], p)
|
||||
if t0 - touched >= 1.0:
|
||||
touched = t0
|
||||
self._share_mod.touch_want(self.want_path)
|
||||
if share.replaced():
|
||||
share.close()
|
||||
share = self._share_mod.open_share(10, self.share_path, self.want_path)
|
||||
if share is None:
|
||||
self.error = "the eye camera frames stopped"
|
||||
return
|
||||
self._stop.wait(max(0.0, self.period - (time.monotonic() - t0)))
|
||||
share.close()
|
||||
|
||||
|
||||
# ------------------------------------------------------------------------------------------
|
||||
|
||||
def _print_reading(r):
|
||||
eyes = "; ".join("%s %s found %.0f%%%s" % (e["name"], e["state"], 100 * e["found"],
|
||||
" off %+.1f,%+.1f deg" % tuple(e["offset_deg"]) if "offset_deg" in e else "")
|
||||
for _c, e in sorted(r["eyes"].items()))
|
||||
terms = " ".join("%s %+.1f" % kv for kv in sorted(r["terms"].items()))
|
||||
print("%-7s %s | %s%s" % (r["state"], eyes, terms, " (provisional)" if r["provisional"] else ""))
|
||||
for h in r["hints"]:
|
||||
print(" -> " + h["text"])
|
||||
|
||||
|
||||
def main(argv):
|
||||
if not argv or argv[0] in ("-h", "--help"):
|
||||
sys.exit(__doc__)
|
||||
if argv[0] == "reference":
|
||||
paths = argv[1:] or [DOTS]
|
||||
rows = []
|
||||
for p in paths:
|
||||
with open(p) as f:
|
||||
rows += [json.loads(line) for line in f if line.strip()]
|
||||
ref = Reference.from_dots(rows, source="calibration dots: " + ", ".join(os.path.basename(p) for p in paths))
|
||||
print(json.dumps(ref.to_json(), indent=1))
|
||||
return
|
||||
if argv[0] == "live":
|
||||
ref = Reference.load()
|
||||
baseline = load_baseline()
|
||||
obs = Observations()
|
||||
finder = LiveFinder(obs, log=print).start()
|
||||
last = None
|
||||
try:
|
||||
while finder.error is None:
|
||||
time.sleep(0.5)
|
||||
last = assess(obs.summary(), ref, baseline=baseline, previous=last)
|
||||
_print_reading(last)
|
||||
except KeyboardInterrupt:
|
||||
pass
|
||||
finder.stop()
|
||||
if "--baseline" in argv and last and last["state"] in ("ok", "adjust"):
|
||||
_write_json(BASELINE, make_baseline(obs.summary()))
|
||||
print("saved the baseline: " + BASELINE)
|
||||
return
|
||||
sys.exit(__doc__)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main(sys.argv[1:])
|
||||
@@ -545,6 +545,9 @@ class Service:
|
||||
self.proc = None
|
||||
self.mmap_unknown = False
|
||||
|
||||
def own_installed(self):
|
||||
return own_installed() # (for the checks: the fit coach needs ft-eyes)
|
||||
|
||||
def eyes_wanted(self):
|
||||
return (self.tracker == "own" and not self.override and self.awake) or time.monotonic() < self.eyes_until
|
||||
|
||||
|
||||
+299
-6
@@ -32,6 +32,28 @@ directions: look at each one.
|
||||
seen) and hints, from the gaze probe's Headset fit (gaze/fitcheck.py), while you
|
||||
adjust the headset. A left click or Meta+J runs its guided check (dots, then looks
|
||||
down, up, left and right); a right click or Meta+K closes it, as does FIT_TIMEOUT.
|
||||
With our tracker installed, its first hint line is the fit coach's (gaze/fitcoach.py):
|
||||
where the headset sits, from where ft-eyes sees each pupil and glint (ft-eyes "fit",
|
||||
run for the check whichever tracker is in use). It's judged only while you look
|
||||
straight ahead (COACH_CENTRE), since the pupils move with the gaze. A good reading
|
||||
(the coach's "ok") is kept as your saved fit when the check closes (fitcoach.BASELINE,
|
||||
with SteamVR's IPD setting from the panel): the put-on check compares with it.
|
||||
|
||||
The fit watch (GAZE_FIT_WATCH=1 in frametop.conf, off by default until it's been tried; read
|
||||
again when the file changes) adds two things, with our tracker installed, its reference
|
||||
(gaze/fit-reference.json) and a saved fit:
|
||||
put-on the quick check that opens when the headset goes on first reads the fit coach while
|
||||
you look at its dot, against your saved fit, and holds the dot meanwhile (at most
|
||||
PUTON_READ). If the headset sits more than fitcoach.MOVED_DEG from it, or the IPD
|
||||
setting changed, the panel widens (five's size) and says which way to move it, live,
|
||||
until it's back; then the dot is taken as usual. A left click or Meta+J takes it as
|
||||
it sits, and after PUTON_TIMEOUT it's taken anyway. So the click teaches our tracker
|
||||
the headset where you meant it to be, not where it landed.
|
||||
slip while gaze mode is on with our tracker, ft-eyes' glint jump (the headset slipped
|
||||
since the last click; "jump_px" in its status) past fitcoach.MOVED_DEG for SLIP_HOLD
|
||||
shows a small note low in view for NOTE_SECONDS (the panel's "note" strip: "Your
|
||||
headset slipped" and which way to push it), at most once every NOTE_COOLDOWN, and
|
||||
never over a check. It goes sooner once the slip is gone.
|
||||
|
||||
A check asked for while the gaze service idles (quickcal, calibrate, fitcheck) wakes it and
|
||||
waits until the tracker sends, at most ft-gazed's WAKE_SETTLE; then it opens, or logs why not.
|
||||
@@ -83,8 +105,10 @@ import statistics
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from collections import deque
|
||||
from pathlib import Path
|
||||
|
||||
import fitcoach
|
||||
from fitcheck import MIN_REGION, FitCheck, wrap
|
||||
from gazecal import DEFAULT_MODEL, EYE_LOST, STATE, steady_samples
|
||||
|
||||
@@ -121,6 +145,15 @@ FULL_RETRY = 10.0 # seconds before an automatic calibration that failed to
|
||||
FIT_TIMEOUT = 300.0 # seconds the fit check stays up
|
||||
FIT_EVERY = 0.5 # seconds between its cards' updates (each is a new picture for the panel)
|
||||
FIT_HINT_WIDTH = 95 # characters a hint line holds in the panel
|
||||
COACH_CENTRE = 8.0 # degrees: the fit coach judges only while the gaze stays this near straight ahead
|
||||
COACH_LEASE = 5.0 # seconds of ft-eyes asked for at a time, while the fit check is up
|
||||
PUTON_REASON = "the headset went on"
|
||||
PUTON_READ = 4.0 # seconds the put-on check holds the dot for a straight-ahead reading
|
||||
PUTON_TIMEOUT = 30.0 # seconds it says what to move before the dot is taken as it sits
|
||||
PUTON_GONE = 5.0 # seconds without eyes that close it meanwhile (adjusting loses them)
|
||||
SLIP_HOLD = 2.0 # seconds a slip lasts before its note
|
||||
NOTE_SECONDS = 8.0
|
||||
NOTE_COOLDOWN = 120.0
|
||||
|
||||
|
||||
def log(msg):
|
||||
@@ -269,6 +302,11 @@ class Checks:
|
||||
self.screens_shown = None
|
||||
self.last_progress = 0.0
|
||||
self.asking = None # (socket, deadline, done): a command to ft-eyes waiting for its reply
|
||||
self.watch = (None, False) # (frametop.conf's mtime, GAZE_FIT_WATCH on)
|
||||
self.slip_ref = None # the fit coach's reference, for the slip note (loaded once)
|
||||
self.slip_since = None # a slip past MOVED_DEG since then
|
||||
self.note_until = None # the slip note shows until then
|
||||
self.note_next = 0.0 # and not again before
|
||||
|
||||
@property
|
||||
def active(self):
|
||||
@@ -490,6 +528,34 @@ class Checks:
|
||||
return
|
||||
self.start("quick", reason)
|
||||
|
||||
def fit_watch(self):
|
||||
"""GAZE_FIT_WATCH=1 in frametop.conf: the put-on check and the slip note (see the top)."""
|
||||
try:
|
||||
m = CONF.stat().st_mtime_ns
|
||||
except OSError:
|
||||
m = None
|
||||
if m != self.watch[0]:
|
||||
on = False
|
||||
try:
|
||||
for line in CONF.read_text().splitlines():
|
||||
k, _, v = line.split("#", 1)[0].partition("=")
|
||||
if k.strip() == "GAZE_FIT_WATCH":
|
||||
on = v.strip() == "1"
|
||||
except OSError:
|
||||
pass
|
||||
if on != self.watch[1]:
|
||||
log(f"fit watch {'on' if on else 'off'}")
|
||||
self.watch = (m, on)
|
||||
return self.watch[1]
|
||||
|
||||
def panel_ipd(self):
|
||||
"""SteamVR's IPD setting (metres) from the panel, or None."""
|
||||
words = ask(PANEL, "ipd", 0.3).split()
|
||||
try:
|
||||
return float(words[1]) if words[:1] == ["ok"] and float(words[1]) > 0 else None
|
||||
except (IndexError, ValueError):
|
||||
return None
|
||||
|
||||
# --- Running a check ---
|
||||
|
||||
def start(self, kind, reason):
|
||||
@@ -518,6 +584,9 @@ class Checks:
|
||||
"blind": blind}
|
||||
log(f"{kind} check: {reason}" + (" (our tracker's first: no gaze yet, so each click takes the look "
|
||||
"up to it)" if blind else ""))
|
||||
self.note_until = None # the check's panel replaces the slip note
|
||||
if kind == "quick" and reason == PUTON_REASON:
|
||||
self.start_puton(now)
|
||||
if kind == "full":
|
||||
st = ask(SCREENS, "state", 0.5).split()
|
||||
if len(st) >= 3 and st[0] == "ok":
|
||||
@@ -541,17 +610,216 @@ class Checks:
|
||||
ignore = None if self.svc.one_eye is None else 1 - self.svc.one_eye # the eye SteamVR ignores
|
||||
self.check = {"kind": "fit", "reason": reason, "own": False, "dots": [], "i": 0, "started": now, "shown": now,
|
||||
"run": [], "accept": False, "done_at": None, "tries": 0, "skipped": 0, "captured": 0,
|
||||
"points": {}, "fit": FitCheck(ignore=ignore), "drawn": {}, "drawn_at": 0.0, "step": None}
|
||||
"points": {}, "fit": FitCheck(ignore=ignore), "drawn": {}, "drawn_at": 0.0, "step": None,
|
||||
"coach": self.new_coach()}
|
||||
log(f"fit check: {reason}")
|
||||
self.note_until = None
|
||||
if self.check["coach"]:
|
||||
self.check["coach"]["ipd"] = self.panel_ipd()
|
||||
self.to_helper("calpanel 1")
|
||||
self.to_panel("show fit")
|
||||
self.to_panel("title Headset fit: adjust the headset while you watch")
|
||||
self.to_panel("text Left click or Meta+J: guided check · Right click or Meta+K: done")
|
||||
return "ok"
|
||||
|
||||
def new_coach(self):
|
||||
"""The fit coach's part of the fit check, or None without our tracker (or its reference)."""
|
||||
if not self.svc.own_installed():
|
||||
return None
|
||||
ref = fitcoach.Reference.load()
|
||||
if ref is None:
|
||||
log(f"fit coach: no reference ({fitcoach.REFERENCE})")
|
||||
return None
|
||||
return {"ref": ref, "baseline": fitcoach.load_baseline(), "reading": None, "summary": None,
|
||||
"looks": deque(), "centred": False, "logged": None, "ipd": None, "good": None}
|
||||
|
||||
def coach_feed(self, s, now):
|
||||
k = self.check["coach"]
|
||||
m1 = s["src"].get("mmap1") or {}
|
||||
if k is None or "hy" not in m1:
|
||||
return
|
||||
k["looks"].append((now, m1["hy"], m1["hp"]))
|
||||
while k["looks"] and k["looks"][0][0] < now - fitcoach.WINDOW:
|
||||
k["looks"].popleft()
|
||||
|
||||
def coach_tick(self, now):
|
||||
"""Keeps ft-eyes running and asks it for the last second of finds; judged when centred."""
|
||||
k = self.check["coach"]
|
||||
if k is None:
|
||||
return
|
||||
svc = self.svc
|
||||
svc.eyes_until = max(svc.eyes_until, now + COACH_LEASE)
|
||||
looks = k["looks"]
|
||||
k["centred"] = (len(looks) >= 10 and looks[-1][0] - looks[0][0] > 0.8 * fitcoach.WINDOW
|
||||
and all(math.hypot(y, p) <= COACH_CENTRE for _, y, p in looks))
|
||||
if self.asking:
|
||||
if now >= self.asking[1]:
|
||||
self.drop_ask()
|
||||
return
|
||||
if not svc.eyes_proc:
|
||||
k["summary"] = None
|
||||
return
|
||||
centred = k["centred"]
|
||||
|
||||
def got(reply):
|
||||
try:
|
||||
summary = {int(cam): v for cam, v in json.loads(reply).items()}
|
||||
except (ValueError, AttributeError):
|
||||
k["summary"] = None
|
||||
return
|
||||
k["summary"] = summary
|
||||
if centred and self.check and self.check.get("coach") is k:
|
||||
k["reading"] = fitcoach.assess(summary, k["ref"], ipd=k["ipd"], baseline=k["baseline"],
|
||||
previous=k["reading"])
|
||||
if k["reading"]["state"] == "ok":
|
||||
k["good"] = summary # (the fit check keeps the last as your saved fit)
|
||||
self.ask_eyes("fit", 0.4, got)
|
||||
|
||||
def coach_lines(self):
|
||||
"""The fit coach's hints for the panel (first), or [] without it."""
|
||||
c = self.check
|
||||
k = c.get("coach")
|
||||
if k is None:
|
||||
return []
|
||||
if k["summary"] is None:
|
||||
return ["Headset position: starting our eye tracker..."]
|
||||
if not k["centred"]:
|
||||
if k["reading"] is None:
|
||||
return ["Headset position: look straight ahead for a moment."]
|
||||
return [f"Headset position (when you last looked straight ahead): {self.coach_words(k['reading'])}"]
|
||||
return [f"Headset position: {self.coach_words(k['reading'])}"] if k["reading"] else []
|
||||
|
||||
@staticmethod
|
||||
def coach_words(r):
|
||||
if r["state"] == "waiting":
|
||||
return "waiting for both eyes."
|
||||
hints = " ".join(h["text"] for h in r["hints"][:2])
|
||||
if r["state"] == "ok":
|
||||
words = "good." + (f" {hints}" if hints else "")
|
||||
else:
|
||||
words = hints or "a little off."
|
||||
if r["provisional"] and r["against"] == "reference":
|
||||
words += " (A first guess: measured on one face so far.)"
|
||||
return words
|
||||
|
||||
# --- The put-on check (the fit watch: see the top) ---
|
||||
|
||||
def start_puton(self, now):
|
||||
c = self.check
|
||||
if not self.fit_watch():
|
||||
return
|
||||
k = self.new_coach()
|
||||
if k is None or not k["baseline"]:
|
||||
if k is not None:
|
||||
log("put-on check: no saved fit yet (Check headset fit saves a good one)")
|
||||
return
|
||||
k["ipd"] = self.panel_ipd()
|
||||
c["coach"] = k
|
||||
c["puton"] = {"state": "reading", "until": now + PUTON_READ, "since": now, "lines": None}
|
||||
log("put-on check: reading the fit against your saved one")
|
||||
|
||||
def puton_holds(self, c):
|
||||
"""The put-on check holds the quick check's dot (reading, or the headset moved)."""
|
||||
return (c.get("puton") or {}).get("state") in ("reading", "moved")
|
||||
|
||||
def puton_tick(self, now):
|
||||
c = self.check
|
||||
pu, k = c["puton"], c["coach"]
|
||||
if not self.puton_holds(c):
|
||||
return # (no more asks: the quick check skips samples while one waits)
|
||||
self.coach_tick(now)
|
||||
if c["accept"]:
|
||||
self.puton_done("the dot taken as the headset sits (a click)")
|
||||
return
|
||||
r = k["reading"] if k["centred"] and k["summary"] is not None else None
|
||||
if r is not None and r["state"] == "waiting":
|
||||
r = None
|
||||
if pu["state"] == "reading":
|
||||
if r is None:
|
||||
if now > pu["until"]:
|
||||
self.puton_done("no straight-ahead reading in time; the dot is taken as usual")
|
||||
return
|
||||
if not fitcoach.moved(r):
|
||||
self.puton_done("the headset sits as your saved fit")
|
||||
return
|
||||
pu["state"], pu["since"] = "moved", now
|
||||
log("put-on check: the headset moved since your saved fit: terms "
|
||||
+ ", ".join(f"{n} {v:+.1f}" for n, v in r["terms"].items()))
|
||||
self.to_panel("show five") # wider: the hints don't fit quick's square
|
||||
self.to_panel("title The headset sits differently from your saved fit")
|
||||
self.show_dot()
|
||||
self.puton_lines(r)
|
||||
return
|
||||
if r is not None and not fitcoach.moved(r):
|
||||
self.note("Good: it's back where it was")
|
||||
self.puton_done("the headset is back as your saved fit")
|
||||
return
|
||||
if r is not None:
|
||||
self.puton_lines(r)
|
||||
if now - pu["since"] > PUTON_TIMEOUT:
|
||||
self.puton_done(f"not back after {PUTON_TIMEOUT:.0f} s: the dot is taken as the headset sits")
|
||||
|
||||
def puton_lines(self, r):
|
||||
"""What to move, in the panel: the position hint at the bottom, a changed IPD over it."""
|
||||
pu = self.check["puton"]
|
||||
ipd = next((h["text"] for h in r["hints"] if h["key"] == "ipd-changed"), "")
|
||||
pos = fitcoach.position_hint(r["terms"])
|
||||
lines = (pos["text"] if pos else "Move the headset back where it sat, then look at the dot", ipd)
|
||||
if lines != pu["lines"]:
|
||||
pu["lines"] = lines
|
||||
self.to_panel(f"text {lines[0]}")
|
||||
self.note(lines[1])
|
||||
|
||||
def puton_done(self, why):
|
||||
c = self.check
|
||||
now = time.monotonic()
|
||||
c["puton"]["state"], c["puton"]["why"] = "done", why
|
||||
log(f"put-on check: {why}")
|
||||
c["run"], c["started"], c["shown"] = [], now, now # the quick check's own time starts now
|
||||
if c["puton"]["lines"]:
|
||||
self.to_panel("text Look at the dot")
|
||||
|
||||
# --- The slip note (the fit watch: see the top) ---
|
||||
|
||||
def slip_watch(self, now):
|
||||
svc = self.svc
|
||||
own = svc.own if now - svc.own_at < 3 else {}
|
||||
on = (svc.kind == "own" and svc.eyes_proc is not None and own and self.gaze_on
|
||||
and self.headset is not False and not self.check and self.fit_watch())
|
||||
hint = None
|
||||
if on:
|
||||
if self.slip_ref is None:
|
||||
self.slip_ref = fitcoach.Reference.load() or False
|
||||
jumps = {}
|
||||
for cam, name in enumerate(fitcoach.EYES):
|
||||
j = (own.get("eyes") or {}).get(name, {}).get("jump_px")
|
||||
if j and any(j):
|
||||
jumps[cam] = j
|
||||
hint = fitcoach.position_hint(fitcoach.slip_terms(jumps, self.slip_ref or None))
|
||||
if hint is None:
|
||||
self.slip_since = None
|
||||
if self.note_until is not None and not self.check:
|
||||
self.to_panel("hide")
|
||||
self.note_until = None
|
||||
return
|
||||
if self.note_until is not None:
|
||||
if now >= self.note_until:
|
||||
self.to_panel("hide")
|
||||
self.note_until = None
|
||||
return
|
||||
if self.slip_since is None:
|
||||
self.slip_since = now
|
||||
if now - self.slip_since >= SLIP_HOLD and now >= self.note_next:
|
||||
self.to_panel("show note")
|
||||
self.to_panel("title Your headset slipped")
|
||||
self.to_panel(f"text {hint['text']}")
|
||||
self.note_until, self.note_next = now + NOTE_SECONDS, now + NOTE_COOLDOWN
|
||||
log(f"slip note: {hint['text']} ({hint['amount']:+.1f} deg)")
|
||||
|
||||
def fit_tick(self, now):
|
||||
c = self.check
|
||||
fit = c["fit"]
|
||||
self.coach_tick(now)
|
||||
# The guided check: its dots at head-relative directions in the panel (the probe's
|
||||
# screen fractions, spread over the middle of the panel), and its looks as the title.
|
||||
step = fit.guide_step(now)
|
||||
@@ -584,6 +852,7 @@ class Checks:
|
||||
hints = ["Look around slowly: up, down, left and right. Or left click (Meta+J) for a guided check."]
|
||||
else:
|
||||
hints = fit.hints()
|
||||
hints = self.coach_lines() + hints
|
||||
lines = [line for h in hints for line in wrap(h, FIT_HINT_WIDTH)][:8]
|
||||
cmd = "hints " + "|".join(lines)
|
||||
if drawn.get("hints") != cmd:
|
||||
@@ -619,7 +888,10 @@ class Checks:
|
||||
c = self.check
|
||||
if c and c["kind"] == "fit":
|
||||
c["fit"].feed(s, self.sample_at)
|
||||
self.coach_feed(s, self.sample_at)
|
||||
return
|
||||
if c and self.puton_holds(c):
|
||||
self.coach_feed(s, self.sample_at)
|
||||
if not c or c["done_at"] or self.asking:
|
||||
return # (asking: ft-eyes has this dot's look, or the fit)
|
||||
if c["own"]:
|
||||
@@ -640,7 +912,7 @@ class Checks:
|
||||
return
|
||||
now = time.monotonic()
|
||||
c["gaze_at"] = now
|
||||
if now < c["shown"] + CHECK_SETTLE:
|
||||
if now < c["shown"] + CHECK_SETTLE or self.puton_holds(c):
|
||||
return
|
||||
g = (src["hy"], src["hp"])
|
||||
run = c["run"]
|
||||
@@ -872,6 +1144,17 @@ class Checks:
|
||||
return
|
||||
if why:
|
||||
log(f"{self.check['kind']} check closed: {why}")
|
||||
k = self.check.get("coach")
|
||||
if k and k["reading"]:
|
||||
r = k["reading"]
|
||||
log(f"fit coach, last reading: {r['state']} against the {r['against']}, terms "
|
||||
+ ", ".join(f"{n} {v:+.1f}" for n, v in r["terms"].items()))
|
||||
if k and k["good"] and self.check["kind"] == "fit":
|
||||
try:
|
||||
fitcoach.save_baseline(fitcoach.make_baseline(k["good"], k["ipd"]))
|
||||
log(f"fit coach: a good fit, saved as yours ({fitcoach.BASELINE})")
|
||||
except OSError as e:
|
||||
log(f"fit coach: can't save the fit: {e}")
|
||||
self.drop_ask()
|
||||
self.to_panel("hide")
|
||||
self.to_helper("calpanel 0")
|
||||
@@ -937,7 +1220,8 @@ class Checks:
|
||||
if cmd == "calaccept":
|
||||
if self.check and self.check["kind"] == "fit":
|
||||
self.check["fit"].toggle_guide(time.monotonic())
|
||||
elif self.check and not self.asking: # (asking: this dot's click landed already)
|
||||
elif self.check and (not self.asking or self.puton_holds(self.check)):
|
||||
# (asking: this dot's click landed already; the put-on check's asks are the coach's)
|
||||
if not self.check["accept"]:
|
||||
self.check["accept_at"] = time.monotonic()
|
||||
self.check["accept"] = True
|
||||
@@ -975,6 +1259,11 @@ class Checks:
|
||||
else:
|
||||
self.fit_tick(now)
|
||||
return
|
||||
if c.get("puton"):
|
||||
self.puton_tick(now)
|
||||
c = self.check
|
||||
if not c:
|
||||
return
|
||||
if self.asking:
|
||||
# ft-eyes has a dot's look or the fit: wait for its answer, at most to the deadline.
|
||||
if now >= self.asking[1]:
|
||||
@@ -988,7 +1277,7 @@ class Checks:
|
||||
else:
|
||||
self.advance()
|
||||
return
|
||||
if not self.eyes_seen(EYES_GONE):
|
||||
if not self.eyes_seen(PUTON_GONE if self.puton_holds(c) else EYES_GONE):
|
||||
self.close("the headset came off")
|
||||
return
|
||||
if c["done_at"]:
|
||||
@@ -1000,7 +1289,7 @@ class Checks:
|
||||
self.note("Waiting: the eye tracker isn't sending a gaze" if big else "Waiting: no gaze")
|
||||
else:
|
||||
self.note("Waiting for your gaze to hold still on the dot" if big else "Hold your look still")
|
||||
if c["kind"] == "quick" and now - c["started"] > QUICK_TIMEOUT:
|
||||
if c["kind"] == "quick" and now - c["started"] > QUICK_TIMEOUT and not self.puton_holds(c):
|
||||
self.close("ignored")
|
||||
elif c["kind"] != "quick" and now - c["shown"] > CLICK_IDLE:
|
||||
self.close(f"no click on dot {c['i'] + 1} for {CLICK_IDLE:.0f} s")
|
||||
@@ -1037,6 +1326,7 @@ class Checks:
|
||||
self.reseat_seen = True
|
||||
self.auto_quick("our tracker asked for a click")
|
||||
self.need_full("gaze mode is on without a calibration")
|
||||
self.slip_watch(now)
|
||||
|
||||
def status(self):
|
||||
c = self.check
|
||||
@@ -1044,10 +1334,13 @@ class Checks:
|
||||
if self.pending else None, "calibrated": self.calibrated(), "eyes": self.eyes_seen(),
|
||||
"problem": self.problem(),
|
||||
"panel": self.panel_proc is not None,
|
||||
"last_quick_s": round(time.monotonic() - self.last_quick) if self.last_quick else None}
|
||||
"last_quick_s": round(time.monotonic() - self.last_quick) if self.last_quick else None,
|
||||
"fit_watch": self.watch[1], "slip_note": self.note_until is not None}
|
||||
if c:
|
||||
st["check"] = {"kind": c["kind"], "dot": c["i"] + 1, "dots": len(c["dots"]), "captured": c["captured"],
|
||||
"skipped": c["skipped"], "reason": c["reason"]}
|
||||
if c.get("puton"):
|
||||
st["check"]["puton"] = c["puton"]["state"]
|
||||
return st
|
||||
|
||||
def stop(self):
|
||||
|
||||
+700
-65
@@ -13,11 +13,13 @@
|
||||
// size changes with it, and the tracker's error with it); "fit" is FIT_DEG across (4:3),
|
||||
// see-through like quick, for the headset fit check: a card per eye (tracked or lost, the
|
||||
// tracker's signal, how much of the last 10 s it was seen) and hints. Every dot a check shows
|
||||
// must fit its panel: gazecheck.py's PANEL_DEG mirrors these sizes and checks it.
|
||||
// must fit its panel: gazecheck.py's PANEL_DEG mirrors these sizes and checks it. "note" is a
|
||||
// strip NOTE_DEG across (4:1), see-through, NOTE_PITCH degrees below straight ahead, for the gaze
|
||||
// service's slip notice: its title and text lines only, no dot.
|
||||
//
|
||||
// Control socket: abstract unix datagram "@ft_gazepanel" (--socket NAME); a sender with an
|
||||
// address gets "ok" or "error ...":
|
||||
// show quick|five|full|fit the panel, empty, in front of you
|
||||
// show quick|five|full|fit|note the panel, empty, in front of you (note: below it)
|
||||
// hide
|
||||
// bg <0..1> the background's brightness (full)
|
||||
// dot <yaw> <pitch> <state> [<progress 0..1>]
|
||||
@@ -33,6 +35,48 @@
|
||||
// hints <line>|<line>|... fit: lines under the cards (empty to clear)
|
||||
// ping
|
||||
//
|
||||
// The eye recorder's mode (gaze/rec/DESIGN.md, "Panel"): "show rec" is FULL_DEG across with
|
||||
// the full calibration's solid background (bg), and the hand panel's lines (hands/rec/panel):
|
||||
// title <text> the big line at the top
|
||||
// step <text> a small line at the top right, "Round 2 of 5"; rec on|off puts a
|
||||
// red "Rec" dot by it
|
||||
// text <text> the instruction, under the title, wrapped ("|" starts a line)
|
||||
// big <text> / action <text> large in the middle (the countdown) / a cyan line under it
|
||||
// note <text> / keys <text> the orange warning line / a faint line along the bottom
|
||||
// countdown <0..1>|off a thin bar along the bottom: the share of the step's time left
|
||||
// paused on|off "Paused" over the picture
|
||||
// Its dot is a second overlay (OVERLAY.dot), drawn once per state and only moved, so a moving
|
||||
// dot is smooth at the headset's frame rate and costs no pictures:
|
||||
// dot <yaw> <pitch> <state> [<progress>] as above, at the panel's distance
|
||||
// path circle <yaw> <pitch> <radius> <seconds>
|
||||
// path line <yaw0> <pitch0> <yaw1> <pitch1> <seconds>
|
||||
// path eight <yaw> <pitch> <yaw radius> <pitch radius> <seconds>
|
||||
// the dot moves by itself: round the circle once per seconds
|
||||
// (from the top, going left), along the line and back (seconds
|
||||
// each way, at a steady speed), or a figure eight (seconds a
|
||||
// loop; up and down twice per side to side), until the next dot,
|
||||
// path, world or "dot 0 0 off". Reply: "ok"
|
||||
// world <x> <y> <z> [<state> [<progress>]]
|
||||
// a dot fixed in the room: the point is in the head frame now
|
||||
// (metres, +x right, +y up, -z forward), placed in the room with
|
||||
// the headset's pose, facing the head as it moves; the same point
|
||||
// again changes only the state. Reply "ok <room x> <y> <z>"
|
||||
// (standing universe)
|
||||
// distance <metres> the panel and its dot that far away (0.25 to 10), the same size
|
||||
// in degrees (the Depth bounty)
|
||||
// ipd "ok <metres>": SteamVR's IPD setting now (--no-vr: 0.0630)
|
||||
// log start <dir> / log stop append to <dir>/targets.jsonl, poses.jsonl and ipd.jsonl (below)
|
||||
// until stopped; "log stop" replies "ok <targets> <poses> <ipd>"
|
||||
// targets.jsonl, the ground truth: what the dot did, when. A row for each frame the dot moves in
|
||||
// (path, world) and for each change (dot, state, background, distance): {"t": when the frame
|
||||
// reaches the eyes (estimated: the next vsync after the change, plus SteamVR's vsync to photons),
|
||||
// "set": when the panel set it, both CLOCK_MONOTONIC_RAW seconds, the eye frames' clock;
|
||||
// "frame": SteamVR's vsync count; "dot": "fixed|path|world|off", "yaw", "pitch" (head-relative
|
||||
// degrees, for world from the head's predicted pose), "dist" (metres), "state", "progress",
|
||||
// "bg"}. poses.jsonl: the headset's pose 250 times a second, {"t", "hmd": {"m": 3x4 row-major,
|
||||
// standing universe, "r": ETrackingResult, "ok"}}. ipd.jsonl: SteamVR's IPD setting at the start
|
||||
// and when it changes, {"t", "ipd": metres}, looked at 30 times a second as well as on its event.
|
||||
//
|
||||
// Each picture goes into the next of three shared buffers (linear DMA-BUFs SteamVR imported
|
||||
// once, the size of the biggest panel; the texture bounds show the part in use), and the panel
|
||||
// switches to it, as screens/keyboard.cpp does. SetOverlayRaw, which uploads a new texture each
|
||||
@@ -41,7 +85,10 @@
|
||||
// It's only the fallback. A "show" makes the panel visible once its first picture is in.
|
||||
//
|
||||
// Options: --watch-stdin (quit when stdin closes: the service runs it), --socket NAME,
|
||||
// --distance METRES. Runs in the dev container (gaze/build.sh builds it into gaze/build).
|
||||
// --overlay KEY (frametop.gazepanel; the eye recorder runs a copy of its own next to the gaze
|
||||
// service's), --distance METRES, --no-vr (no SteamVR: for tests, at a made-up 90 Hz vsync; with
|
||||
// --dump DIR each picture goes there as a PAM file). Runs in the dev container (gaze/build.sh
|
||||
// builds it into gaze/build).
|
||||
#include <openvr.h>
|
||||
|
||||
#define STB_TRUETYPE_IMPLEMENTATION
|
||||
@@ -51,6 +98,7 @@
|
||||
#include <fcntl.h>
|
||||
#include <gbm.h>
|
||||
#include <poll.h>
|
||||
#include <sys/stat.h>
|
||||
#include <sys/socket.h>
|
||||
#include <sys/un.h>
|
||||
#include <unistd.h>
|
||||
@@ -64,6 +112,7 @@
|
||||
#include <cstdlib>
|
||||
#include <cstring>
|
||||
#include <map>
|
||||
#include <mutex>
|
||||
#include <string>
|
||||
#include <thread>
|
||||
#include <vector>
|
||||
@@ -75,8 +124,22 @@ constexpr double kQuickDeg = 16; // QUICK_DEG: the one-dot check's square
|
||||
constexpr double kFullDeg = 64; // FULL_DEG: the full calibration's width (4:3)
|
||||
constexpr double kFitDeg = 40; // FIT_DEG: the headset fit check's width (4:3)
|
||||
constexpr double kFiveDeg = 40; // FIVE_DEG: the five-dot check's width (4:3), past its dots
|
||||
constexpr double kNoteDeg = 30; // NOTE_DEG: the slip notice's strip (4:1)
|
||||
constexpr double kNotePitch = -16; // NOTE_PITCH: the strip's centre, degrees below straight ahead
|
||||
constexpr int kQuickPx = 320, kFullW = 1024, kFullH = 768, kFitW = 800, kFitH = 600, kFiveW = 800, kFiveH = 600;
|
||||
constexpr int kNoteW = 600, kNoteH = 150;
|
||||
constexpr double kDotDeg = 1.8; // the rec mode's dot overlay: the ring's 1.5 degrees, and a margin
|
||||
constexpr int kDotPx = 96;
|
||||
constexpr int64_t kPosePeriodNs = 4'000'000; // 250 Hz, as the hand panel
|
||||
constexpr int64_t kIpdPeriodNs = 33'000'000;
|
||||
std::atomic<bool> g_stop{false};
|
||||
std::mutex g_vr; // around OpenVR calls once the pose log's thread makes them too
|
||||
|
||||
int64_t RawNs() {
|
||||
timespec ts;
|
||||
clock_gettime(CLOCK_MONOTONIC_RAW, &ts);
|
||||
return int64_t(ts.tv_sec) * 1'000'000'000 + ts.tv_nsec;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- text (as screens/keyboard.cpp)
|
||||
|
||||
@@ -135,6 +198,41 @@ std::vector<uint32_t> Codepoints(const std::string &s) {
|
||||
return out;
|
||||
}
|
||||
|
||||
int TextWidth(const std::string &s, int size) {
|
||||
if (!g_fontOk) return 0;
|
||||
int width = 0;
|
||||
for (uint32_t cp : Codepoints(s)) width += GetGlyph(cp, size).advance;
|
||||
return width;
|
||||
}
|
||||
|
||||
// s as lines no wider than width (the hand panel's): "|" starts a new line, and lines break
|
||||
// between words.
|
||||
std::vector<std::string> Wrap(const std::string &s, int size, int width) {
|
||||
std::vector<std::string> out;
|
||||
if (s.empty()) return out;
|
||||
for (size_t at = 0; at <= s.size();) {
|
||||
const size_t bar = std::min(s.find('|', at), s.size());
|
||||
const std::string para = s.substr(at, bar - at);
|
||||
std::string line;
|
||||
for (size_t w = 0; w < para.size();) {
|
||||
const size_t space = std::min(para.find(' ', w), para.size());
|
||||
const std::string word = para.substr(w, space - w);
|
||||
w = space + 1;
|
||||
if (word.empty()) continue;
|
||||
const std::string longer = line.empty() ? word : line + " " + word;
|
||||
if (line.empty() || TextWidth(longer, size) <= width) {
|
||||
line = longer;
|
||||
} else {
|
||||
out.push_back(line);
|
||||
line = word;
|
||||
}
|
||||
}
|
||||
out.push_back(line);
|
||||
at = bar + 1;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- the picture
|
||||
|
||||
struct EyeCard {
|
||||
@@ -143,7 +241,10 @@ struct EyeCard {
|
||||
};
|
||||
|
||||
struct Panel {
|
||||
bool full = false, fit = false;
|
||||
bool full = false, fit = false, rec = false, strip = false; // strip: "show note"
|
||||
std::string step, big, action, keys; // rec
|
||||
bool recOn = false, paused = false;
|
||||
double countdown = -1;
|
||||
EyeCard eyes[2];
|
||||
std::vector<std::string> hints;
|
||||
int w = kQuickPx, h = kQuickPx;
|
||||
@@ -253,9 +354,58 @@ void ToPixel(const Panel &p, double yaw, double pitch, double &x, double &y) {
|
||||
y = (0.5 - Y / (2 * half) * p.w / p.h) * p.h;
|
||||
}
|
||||
|
||||
void DrawDot(Panel &p, double x, double y, double pxPerDeg, const std::string &state, double progress, double ink);
|
||||
|
||||
// The eye recorder's panel (see the top). Its dot is the other overlay's.
|
||||
void DrawRec(Panel &p, double pxPerDeg) {
|
||||
const bool light = p.bg > 0.5;
|
||||
const double faint = light ? 0.2 : 0.75, ink = light ? 0.05 : 0.95;
|
||||
const int titleSize = int(pxPerDeg * 1.5), textSize = int(pxPerDeg * 1.2), small = int(pxPerDeg * 0.9);
|
||||
const int bigSize = int(pxPerDeg * 4.0), margin = int(p.w * 0.05);
|
||||
const int titleY = int(titleSize * 1.2);
|
||||
Text(p, p.title, titleSize, p.w / 2, titleY, ink);
|
||||
if (!p.step.empty()) {
|
||||
const int sw = TextWidth(p.step, small);
|
||||
Text(p, p.step, small, p.w - margin - sw, titleY, faint, true);
|
||||
if (p.recOn) {
|
||||
const int rx = p.w - margin - sw - small;
|
||||
Disc(p, rx, titleY, small * 0.35, 0, 0.9, 0.15, 0.1, 1);
|
||||
Text(p, "Rec", small, rx - small - TextWidth("Rec", small), titleY, 1.0, 0.35, 0.3, true);
|
||||
}
|
||||
} else if (p.recOn) {
|
||||
Disc(p, p.w - margin - small * 0.35, titleY, small * 0.35, 0, 0.9, 0.15, 0.1, 1);
|
||||
}
|
||||
int y = titleY + int(titleSize * 1.4);
|
||||
for (const std::string &line : Wrap(p.text, textSize, int(p.w * 0.8))) {
|
||||
Text(p, line, textSize, p.w / 2, y, faint);
|
||||
y += int(textSize * 1.35);
|
||||
}
|
||||
Text(p, p.big, bigSize, p.w / 2, p.h / 2, ink);
|
||||
if (light) Text(p, p.action, textSize, p.w / 2, int(p.h * 0.68), 0.0, 0.35, 0.45);
|
||||
else Text(p, p.action, textSize, p.w / 2, int(p.h * 0.68), 0.35, 0.85, 1.0);
|
||||
if (light) Text(p, p.note, textSize, p.w / 2, p.h - int(textSize * 2.8), 0.7, 0.12, 0.05);
|
||||
else Text(p, p.note, textSize, p.w / 2, p.h - int(textSize * 2.8), 1.0, 0.62, 0.3);
|
||||
Text(p, p.keys, small, p.w / 2, p.h - int(small * 1.6), faint);
|
||||
if (p.countdown >= 0) {
|
||||
const int bh = std::max(3, int(pxPerDeg * 0.2));
|
||||
Rect(p, 0, p.h - bh, p.w, p.h, ink, ink, ink, 0.12);
|
||||
Rect(p, 0, p.h - bh, int(p.w * std::clamp(p.countdown, 0.0, 1.0)), p.h, 0.35, 0.85, 1.0, 0.9);
|
||||
}
|
||||
if (p.paused) {
|
||||
Rect(p, 0, 0, p.w, p.h, 0, 0, 0, 0.55);
|
||||
Text(p, "Paused", bigSize, p.w / 2, p.h / 2, 1.0);
|
||||
}
|
||||
}
|
||||
|
||||
void Draw(Panel &p) {
|
||||
p.px.assign(size_t(p.w) * p.h * 4, 0);
|
||||
const double pxPerDeg = p.w / p.wDeg;
|
||||
if (p.rec) {
|
||||
for (size_t i = 0; i < p.px.size(); i += 4)
|
||||
p.px[i] = p.px[i + 1] = p.px[i + 2] = uint8_t(std::lround(p.bg * 255)), p.px[i + 3] = 255;
|
||||
DrawRec(p, pxPerDeg);
|
||||
return;
|
||||
}
|
||||
if (p.full) {
|
||||
for (size_t i = 0; i < p.px.size(); i += 4)
|
||||
p.px[i] = p.px[i + 1] = p.px[i + 2] = uint8_t(std::lround(p.bg * 255)), p.px[i + 3] = 255;
|
||||
@@ -272,6 +422,12 @@ void Draw(Panel &p) {
|
||||
}
|
||||
const bool light = p.full && p.bg > 0.5; // a dark dot on the bright round
|
||||
const double ink = light ? 0.0 : 1.0, faint = light ? 0.2 : 0.75;
|
||||
if (p.strip) {
|
||||
// The slip notice: the title and the hint, together in the middle of the strip.
|
||||
Text(p, p.title, int(pxPerDeg * 1.4), p.w / 2, int(p.h * 0.36), ink);
|
||||
Text(p, p.text, int(pxPerDeg * 1.05), p.w / 2, int(p.h * 0.68), faint);
|
||||
return;
|
||||
}
|
||||
const int titleSize = int(pxPerDeg * (p.full ? 1.5 : 1.1)), textSize = int(pxPerDeg * (p.full ? 1.2 : 0.9));
|
||||
Text(p, p.title, titleSize, p.w / 2, int(titleSize * 1.2), faint);
|
||||
Text(p, p.text, textSize, p.w / 2, p.h - int(textSize * 1.3), faint);
|
||||
@@ -282,21 +438,26 @@ void Draw(Panel &p) {
|
||||
if (!p.dotOn || p.state == "off") return;
|
||||
double x, y;
|
||||
ToPixel(p, p.dotYaw, p.dotPitch, x, y);
|
||||
DrawDot(p, x, y, pxPerDeg, p.state, p.progress, ink);
|
||||
}
|
||||
|
||||
// The dot at (x, y): its look for each state.
|
||||
void DrawDot(Panel &p, double x, double y, double pxPerDeg, const std::string &state, double progress, double ink) {
|
||||
const double core = 0.22 * pxPerDeg;
|
||||
if (p.state == "look") {
|
||||
if (state == "look") {
|
||||
// Still, so the panel looks solid and is drawn again only when something changes.
|
||||
const double rr = 0.75 * pxPerDeg;
|
||||
Disc(p, x, y, rr, rr - 0.12 * pxPerDeg, ink, ink, ink, 0.6);
|
||||
Disc(p, x, y, core, 0, ink, ink, ink, 1);
|
||||
} else if (p.state == "capture") {
|
||||
} else if (state == "capture") {
|
||||
const double rr = 0.75 * pxPerDeg;
|
||||
Disc(p, x, y, rr, rr - 0.12 * pxPerDeg, ink, ink, ink, 0.25);
|
||||
Disc(p, x, y, rr, rr - 0.12 * pxPerDeg, 0.3, 0.85, 1.0, 1, std::clamp(p.progress, 0.0, 1.0));
|
||||
Disc(p, x, y, rr, rr - 0.12 * pxPerDeg, 0.3, 0.85, 1.0, 1, std::clamp(progress, 0.0, 1.0));
|
||||
Disc(p, x, y, core, 0, ink, ink, ink, 1);
|
||||
} else if (p.state == "done") {
|
||||
} else if (state == "done") {
|
||||
Disc(p, x, y, 0.75 * pxPerDeg, 0, 0.25, 0.85, 0.4, 0.9);
|
||||
Disc(p, x, y, core, 0, 1, 1, 1, 1);
|
||||
} else if (p.state == "fail") {
|
||||
} else if (state == "fail") {
|
||||
Disc(p, x, y, 0.75 * pxPerDeg, 0.6 * pxPerDeg, 0.95, 0.35, 0.3, 0.9);
|
||||
Disc(p, x, y, core, 0, ink, ink, ink, 1);
|
||||
}
|
||||
@@ -312,6 +473,7 @@ struct Buffer {
|
||||
};
|
||||
|
||||
struct Buffers {
|
||||
int w = kMaxW, h = kMaxH; // the biggest picture they take
|
||||
int drm = -1;
|
||||
gbm_device *gbm = nullptr;
|
||||
Buffer b[3];
|
||||
@@ -319,14 +481,15 @@ struct Buffers {
|
||||
bool ok = false;
|
||||
|
||||
bool Make() {
|
||||
std::lock_guard<std::mutex> lk(g_vr);
|
||||
drm = open("/dev/dri/renderD128", O_RDWR | O_CLOEXEC);
|
||||
if (drm >= 0) gbm = gbm_create_device(drm);
|
||||
for (Buffer &x : b) {
|
||||
// ABGR8888 is R, G, B, A in memory, like Panel::px.
|
||||
if (gbm) x.bo = gbm_bo_create(gbm, kMaxW, kMaxH, GBM_FORMAT_ABGR8888, GBM_BO_USE_RENDERING | GBM_BO_USE_LINEAR);
|
||||
if (gbm) x.bo = gbm_bo_create(gbm, w, h, GBM_FORMAT_ABGR8888, GBM_BO_USE_RENDERING | GBM_BO_USE_LINEAR);
|
||||
if (!x.bo || (x.fd = gbm_bo_get_fd(x.bo)) < 0) break;
|
||||
vr::DmabufAttributes_t a{};
|
||||
a.unWidth = kMaxW, a.unHeight = kMaxH;
|
||||
a.unWidth = w, a.unHeight = h;
|
||||
a.unDepth = a.unMipLevels = a.unArrayLayers = a.unSampleCount = 1;
|
||||
a.unFormat = DRM_FORMAT_ABGR8888;
|
||||
a.ulModifier = DRM_FORMAT_MOD_LINEAR;
|
||||
@@ -340,12 +503,17 @@ struct Buffers {
|
||||
ok = b[2].handle != 0;
|
||||
if (!ok) {
|
||||
std::fprintf(stderr, "ft-gazepanel: no shared buffers; falling back to SetOverlayRaw (it flickers)\n");
|
||||
Drop();
|
||||
DropLocked();
|
||||
}
|
||||
return ok;
|
||||
}
|
||||
|
||||
void Drop() {
|
||||
std::lock_guard<std::mutex> lk(g_vr);
|
||||
DropLocked();
|
||||
}
|
||||
|
||||
void DropLocked() {
|
||||
for (Buffer &x : b) {
|
||||
if (x.handle) vr::VRIPCResourceManager()->UnrefResource(x.handle);
|
||||
if (x.fd >= 0) close(x.fd);
|
||||
@@ -362,6 +530,7 @@ struct Buffers {
|
||||
void Present(vr::IVROverlay *ov, vr::VROverlayHandle_t h, const Panel &p) {
|
||||
vr::EVROverlayError e;
|
||||
if (!ok) {
|
||||
std::lock_guard<std::mutex> lk(g_vr);
|
||||
e = ov->SetOverlayRaw(h, const_cast<uint8_t *>(p.px.data()), uint32_t(p.w), uint32_t(p.h), 4);
|
||||
} else {
|
||||
Buffer &x = b[next];
|
||||
@@ -383,7 +552,8 @@ struct Buffers {
|
||||
}
|
||||
}
|
||||
gbm_bo_unmap(x.bo, mapping);
|
||||
const vr::VRTextureBounds_t bounds{0, 0, float(p.w) / kMaxW, float(p.h) / kMaxH};
|
||||
std::lock_guard<std::mutex> lk(g_vr);
|
||||
const vr::VRTextureBounds_t bounds{0, 0, float(p.w) / w, float(p.h) / this->h};
|
||||
ov->SetOverlayTextureBounds(h, &bounds);
|
||||
vr::Texture_t tex = {&x.handle, vr::TextureType_SharedTextureHandle, vr::ColorSpace_Gamma};
|
||||
e = ov->SetOverlayTexture(h, &tex);
|
||||
@@ -393,18 +563,213 @@ struct Buffers {
|
||||
}
|
||||
};
|
||||
|
||||
// ---------------------------------------------------------------- the rec mode's logs (see the top)
|
||||
|
||||
void PutFloat(FILE *f, double v) { std::fprintf(f, "%.6g", std::isfinite(v) ? v : 0.0); }
|
||||
double Sec(int64_t ns) { return double(ns) / 1e9; }
|
||||
|
||||
// A JSON-lines file the main thread writes.
|
||||
struct Lines {
|
||||
FILE *f = nullptr;
|
||||
long rows = 0;
|
||||
bool Open(const std::string &path) {
|
||||
Close();
|
||||
if (!(f = std::fopen(path.c_str(), "ae"))) return false;
|
||||
std::setvbuf(f, nullptr, _IOFBF, 1 << 16);
|
||||
rows = 0;
|
||||
return true;
|
||||
}
|
||||
long Close() {
|
||||
if (!f) return rows;
|
||||
if (std::fflush(f) != 0 || std::ferror(f)) std::fprintf(stderr, "ft-gazepanel: writing a log failed\n");
|
||||
std::fclose(f);
|
||||
f = nullptr;
|
||||
return rows;
|
||||
}
|
||||
};
|
||||
|
||||
// The headset's pose 250 times a second (as the hand panel's, head only), on a thread of its
|
||||
// own, so a picture or a command never delays a sample.
|
||||
struct PoseLog {
|
||||
FILE *f = nullptr;
|
||||
std::thread th;
|
||||
std::atomic<bool> run{false};
|
||||
std::atomic<long> samples{0};
|
||||
|
||||
bool Start(const std::string &path, bool vr) {
|
||||
Stop();
|
||||
if (!(f = std::fopen(path.c_str(), "ae"))) return false;
|
||||
std::setvbuf(f, nullptr, _IOFBF, 1 << 16);
|
||||
samples = 0;
|
||||
if (vr) run = true, th = std::thread([this] { Run(); });
|
||||
return true;
|
||||
}
|
||||
|
||||
long Stop() {
|
||||
if (!f) return 0;
|
||||
run = false;
|
||||
if (th.joinable()) th.join();
|
||||
if (std::fflush(f) != 0 || std::ferror(f)) std::fprintf(stderr, "ft-gazepanel: writing the pose log failed\n");
|
||||
std::fclose(f);
|
||||
f = nullptr;
|
||||
return samples;
|
||||
}
|
||||
|
||||
void Run() {
|
||||
auto *sys = vr::VRSystem();
|
||||
vr::TrackedDevicePose_t pose{};
|
||||
int64_t next = RawNs(), lastFlush = next;
|
||||
while (run) {
|
||||
int64_t before, after;
|
||||
{
|
||||
std::lock_guard<std::mutex> lk(g_vr);
|
||||
before = RawNs();
|
||||
sys->GetDeviceToAbsoluteTrackingPose(vr::TrackingUniverseStanding, 0, &pose, 1);
|
||||
after = RawNs();
|
||||
}
|
||||
const int64_t t = before + (after - before) / 2;
|
||||
std::fprintf(f, "{\"t\": %.6f, \"hmd\": ", Sec(t));
|
||||
if (!pose.bDeviceIsConnected) {
|
||||
std::fputs("null}\n", f);
|
||||
} else {
|
||||
std::fputs("{\"m\": [", f);
|
||||
for (int k = 0; k < 12; ++k) {
|
||||
if (k) std::fputs(", ", f);
|
||||
PutFloat(f, pose.mDeviceToAbsoluteTracking.m[k / 4][k % 4]);
|
||||
}
|
||||
std::fprintf(f, "], \"r\": %d, \"ok\": %s}}\n", int(pose.eTrackingResult), pose.bPoseIsValid ? "true" : "false");
|
||||
}
|
||||
++samples;
|
||||
if (t - lastFlush >= 1'000'000'000) std::fflush(f), lastFlush = t;
|
||||
next += kPosePeriodNs;
|
||||
const int64_t now = RawNs();
|
||||
if (next < now - kPosePeriodNs) next = now;
|
||||
const timespec ts{time_t(next / 1'000'000'000), long(next % 1'000'000'000)};
|
||||
clock_nanosleep(CLOCK_MONOTONIC_RAW, TIMER_ABSTIME, &ts, nullptr);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
// ---------------------------------------------------------------- the rec mode's dot
|
||||
|
||||
// A head-relative direction (yaw +left, pitch +up; see ToPixel) as a unit vector in the head
|
||||
// frame (+x right, +y up, -z forward), and back.
|
||||
void Direction(double yaw, double pitch, double v[3]) {
|
||||
const double yr = yaw * M_PI / 180, pr = pitch * M_PI / 180;
|
||||
v[0] = -std::sin(yr) * std::cos(pr), v[1] = std::sin(pr), v[2] = -std::cos(yr) * std::cos(pr);
|
||||
}
|
||||
|
||||
void Angles(const double v[3], double &yaw, double &pitch) {
|
||||
yaw = std::atan2(-v[0], -v[2]) * 180 / M_PI;
|
||||
pitch = std::atan2(v[1], std::hypot(v[0], v[2])) * 180 / M_PI;
|
||||
}
|
||||
|
||||
// An overlay's transform at `at`, its face (+z) turned toward `eye`, upright (the hand panel's).
|
||||
vr::HmdMatrix34_t Facing(const double at[3], const double eye[3]) {
|
||||
auto norm = [](double v[3]) {
|
||||
const double n = std::sqrt(v[0] * v[0] + v[1] * v[1] + v[2] * v[2]);
|
||||
if (n < 1e-9) return false;
|
||||
v[0] /= n, v[1] /= n, v[2] /= n;
|
||||
return true;
|
||||
};
|
||||
double z[3] = {eye[0] - at[0], eye[1] - at[1], eye[2] - at[2]};
|
||||
if (!norm(z)) z[0] = z[1] = 0, z[2] = 1;
|
||||
double x[3] = {z[2], 0, -z[0]};
|
||||
if (!norm(x)) x[0] = 1, x[1] = x[2] = 0;
|
||||
const double y[3] = {z[1] * x[2] - z[2] * x[1], z[2] * x[0] - z[0] * x[2], z[0] * x[1] - z[1] * x[0]};
|
||||
vr::HmdMatrix34_t m{};
|
||||
for (int r = 0; r < 3; ++r)
|
||||
m.m[r][0] = float(x[r]), m.m[r][1] = float(y[r]), m.m[r][2] = float(z[r]), m.m[r][3] = float(at[r]);
|
||||
return m;
|
||||
}
|
||||
|
||||
// m's point p in the room -> the device frame (m is device to room, rotation orthonormal).
|
||||
void ToDevice(const vr::HmdMatrix34_t &m, const double p[3], double out[3]) {
|
||||
const double d[3] = {p[0] - m.m[0][3], p[1] - m.m[1][3], p[2] - m.m[2][3]};
|
||||
for (int c = 0; c < 3; ++c) out[c] = m.m[0][c] * d[0] + m.m[1][c] * d[1] + m.m[2][c] * d[2];
|
||||
}
|
||||
|
||||
void ToRoom(const vr::HmdMatrix34_t &m, const double p[3], double out[3]) {
|
||||
for (int r = 0; r < 3; ++r) out[r] = m.m[r][0] * p[0] + m.m[r][1] * p[1] + m.m[r][2] * p[2] + m.m[r][3];
|
||||
}
|
||||
|
||||
struct Dot {
|
||||
std::string mode = "off"; // off, fixed, path, world
|
||||
double yaw = 0, pitch = 0;
|
||||
std::string state = "look";
|
||||
double progress = 0;
|
||||
// path
|
||||
std::string kind;
|
||||
double a[4] = {0, 0, 0, 0}, seconds = 1;
|
||||
int64_t start = -1; // the path's first frame's photon time (ns), set on that frame
|
||||
// world
|
||||
double room[3] = {0, 0, 0}, head[3] = {0, 0, 0}; // where it is, and the head-frame point it came from
|
||||
// drawn: what the overlay's picture shows, and whether it's visible
|
||||
std::string drawnState;
|
||||
double drawnProgress = -1, drawnBg = -1;
|
||||
bool shown = false;
|
||||
bool changed = true; // log a row on the next frame
|
||||
};
|
||||
|
||||
// Where a path's dot is t seconds after it started.
|
||||
void PathAt(const Dot &d, double t, double &yaw, double &pitch) {
|
||||
const double ph = t / std::max(d.seconds, 0.05);
|
||||
if (d.kind == "circle") {
|
||||
yaw = d.a[0] + d.a[2] * std::sin(2 * M_PI * ph), pitch = d.a[1] + d.a[2] * std::cos(2 * M_PI * ph);
|
||||
} else if (d.kind == "line") {
|
||||
double f = std::fmod(ph, 2.0);
|
||||
if (f > 1) f = 2 - f;
|
||||
yaw = d.a[0] + (d.a[2] - d.a[0]) * f, pitch = d.a[1] + (d.a[3] - d.a[1]) * f;
|
||||
} else { // eight
|
||||
yaw = d.a[0] + d.a[2] * std::sin(2 * M_PI * ph), pitch = d.a[1] + d.a[3] * std::sin(4 * M_PI * ph);
|
||||
}
|
||||
}
|
||||
|
||||
// --no-vr: a picture as a PAM file (RGBA, straight alpha), as the hand panel's.
|
||||
void Dump(const Panel &p, const std::string &path) {
|
||||
if (FILE *f = std::fopen(path.c_str(), "wb")) {
|
||||
std::fprintf(f, "P7\nWIDTH %d\nHEIGHT %d\nDEPTH 4\nMAXVAL 255\nTUPLTYPE RGB_ALPHA\nENDHDR\n", p.w, p.h);
|
||||
std::fwrite(p.px.data(), 1, p.px.size(), f);
|
||||
std::fclose(f);
|
||||
}
|
||||
}
|
||||
|
||||
// The headset's pose `ahead` seconds from now; --no-vr's is at the room's origin, looking down -z.
|
||||
vr::TrackedDevicePose_t HeadPose(bool noVr, float ahead) {
|
||||
vr::TrackedDevicePose_t pose{};
|
||||
if (noVr) {
|
||||
pose.bPoseIsValid = pose.bDeviceIsConnected = true;
|
||||
pose.mDeviceToAbsoluteTracking.m[0][0] = pose.mDeviceToAbsoluteTracking.m[1][1] = pose.mDeviceToAbsoluteTracking.m[2][2] = 1;
|
||||
} else {
|
||||
std::lock_guard<std::mutex> lk(g_vr);
|
||||
vr::VRSystem()->GetDeviceToAbsoluteTrackingPose(vr::TrackingUniverseStanding, ahead, &pose, 1);
|
||||
}
|
||||
return pose;
|
||||
}
|
||||
|
||||
bool Is(const char *buf, const char *word, const char **rest = nullptr) {
|
||||
const size_t n = std::strlen(word);
|
||||
if (std::strncmp(buf, word, n) || (buf[n] && buf[n] != ' ')) return false;
|
||||
if (rest) *rest = buf[n] ? buf + n + 1 : buf + n;
|
||||
return true;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
int main(int argc, char **argv) {
|
||||
bool watchStdin = false;
|
||||
std::string sockName = "ft_gazepanel";
|
||||
bool watchStdin = false, noVr = false;
|
||||
std::string sockName = "ft_gazepanel", overlayKey = "frametop.gazepanel", dumpDir;
|
||||
double distance = 1.5;
|
||||
for (int i = 1; i < argc; ++i) {
|
||||
if (!std::strcmp(argv[i], "--watch-stdin")) watchStdin = true;
|
||||
else if (!std::strcmp(argv[i], "--no-vr")) noVr = true;
|
||||
else if (!std::strcmp(argv[i], "--socket") && i + 1 < argc) sockName = argv[++i];
|
||||
else if (!std::strcmp(argv[i], "--overlay") && i + 1 < argc) overlayKey = argv[++i];
|
||||
else if (!std::strcmp(argv[i], "--dump") && i + 1 < argc) dumpDir = argv[++i];
|
||||
else if (!std::strcmp(argv[i], "--distance") && i + 1 < argc) distance = std::clamp(std::atof(argv[++i]), 0.5, 5.0);
|
||||
else {
|
||||
std::fprintf(stderr, "usage: %s [--watch-stdin] [--socket NAME] [--distance METRES]\n", argv[0]);
|
||||
std::fprintf(stderr, "usage: %s [--watch-stdin] [--socket NAME] [--overlay KEY] [--distance METRES] "
|
||||
"[--no-vr [--dump DIR]]\n", argv[0]);
|
||||
return 2;
|
||||
}
|
||||
}
|
||||
@@ -427,39 +792,176 @@ int main(int argc, char **argv) {
|
||||
return 1;
|
||||
}
|
||||
|
||||
// As Frametop's other SteamVR clients: background first, so we never start vrserver.
|
||||
vr::EVRInitError err = vr::VRInitError_None;
|
||||
vr::VR_Init(&err, vr::VRApplication_Background);
|
||||
if (err == vr::VRInitError_None) {
|
||||
vr::VR_Shutdown();
|
||||
vr::VR_Init(&err, vr::VRApplication_Overlay);
|
||||
vr::IVROverlay *ov = nullptr;
|
||||
vr::VROverlayHandle_t h = vr::k_ulOverlayHandleInvalid, dotH = vr::k_ulOverlayHandleInvalid;
|
||||
double period = 1 / 90.0, toPhotons = 0;
|
||||
if (!noVr) {
|
||||
// As Frametop's other SteamVR clients: background first, so we never start vrserver.
|
||||
vr::EVRInitError err = vr::VRInitError_None;
|
||||
vr::VR_Init(&err, vr::VRApplication_Background);
|
||||
if (err == vr::VRInitError_None) {
|
||||
vr::VR_Shutdown();
|
||||
vr::VR_Init(&err, vr::VRApplication_Overlay);
|
||||
}
|
||||
if (err != vr::VRInitError_None) {
|
||||
std::fprintf(stderr, "ft-gazepanel: SteamVR: %s\n", vr::VR_GetVRInitErrorAsEnglishDescription(err));
|
||||
return 1;
|
||||
}
|
||||
ov = vr::VROverlay();
|
||||
if (ov->CreateOverlay(overlayKey.c_str(), "Frametop gaze calibration", &h) != vr::VROverlayError_None ||
|
||||
ov->CreateOverlay((overlayKey + ".dot").c_str(), "Frametop gaze dot", &dotH) != vr::VROverlayError_None) {
|
||||
std::fprintf(stderr, "ft-gazepanel: can't create the overlay (another copy running?)\n");
|
||||
return 1;
|
||||
}
|
||||
ov->SetOverlaySortOrder(h, 250); // in front of Frametop's screens and the pointer's dot
|
||||
ov->SetOverlaySortOrder(dotH, 251);
|
||||
auto *sys = vr::VRSystem();
|
||||
vr::ETrackedPropertyError pe;
|
||||
const float hz = sys->GetFloatTrackedDeviceProperty(vr::k_unTrackedDeviceIndex_Hmd, vr::Prop_DisplayFrequency_Float, &pe);
|
||||
if (pe == vr::TrackedProp_Success && hz > 30) period = 1.0 / hz;
|
||||
const float ph = sys->GetFloatTrackedDeviceProperty(vr::k_unTrackedDeviceIndex_Hmd, vr::Prop_SecondsFromVsyncToPhotons_Float, &pe);
|
||||
if (pe == vr::TrackedProp_Success && ph >= 0 && ph < 0.1) toPhotons = ph;
|
||||
}
|
||||
if (err != vr::VRInitError_None) {
|
||||
std::fprintf(stderr, "ft-gazepanel: SteamVR: %s\n", vr::VR_GetVRInitErrorAsEnglishDescription(err));
|
||||
return 1;
|
||||
}
|
||||
vr::IVROverlay *ov = vr::VROverlay();
|
||||
vr::VROverlayHandle_t h = vr::k_ulOverlayHandleInvalid;
|
||||
if (ov->CreateOverlay("frametop.gazepanel", "Frametop gaze calibration", &h) != vr::VROverlayError_None) {
|
||||
std::fprintf(stderr, "ft-gazepanel: can't create the overlay (another copy running?)\n");
|
||||
return 1;
|
||||
}
|
||||
ov->SetOverlaySortOrder(h, 250); // in front of Frametop's screens and the pointer's dot
|
||||
LoadFont();
|
||||
Buffers buffers;
|
||||
if (buffers.Make()) ov->SetOverlayFlag(h, vr::VROverlayFlags_IsPremultiplied, true);
|
||||
Buffers buffers, dotBuffers;
|
||||
dotBuffers.w = dotBuffers.h = kDotPx;
|
||||
if (!noVr && buffers.Make()) ov->SetOverlayFlag(h, vr::VROverlayFlags_IsPremultiplied, true);
|
||||
if (!noVr && dotBuffers.Make()) ov->SetOverlayFlag(dotH, vr::VROverlayFlags_IsPremultiplied, true);
|
||||
if (!dumpDir.empty()) mkdir(dumpDir.c_str(), 0755);
|
||||
|
||||
Panel p;
|
||||
Panel dotPic; // the rec mode's dot overlay's picture
|
||||
dotPic.w = dotPic.h = kDotPx, dotPic.wDeg = kDotDeg;
|
||||
Dot dot;
|
||||
Lines targets, ipdLog;
|
||||
PoseLog poses;
|
||||
float ipdNow = -1;
|
||||
int64_t ipdAt = 0;
|
||||
int pictures = 0;
|
||||
bool visible = false, dirty = false, shown = false; // shown: SteamVR shows it (after its first picture)
|
||||
auto place = [&] {
|
||||
if (noVr) return;
|
||||
const double half = std::tan(p.wDeg * M_PI / 360);
|
||||
vr::HmdMatrix34_t m{};
|
||||
m.m[0][0] = m.m[1][1] = m.m[2][2] = 1;
|
||||
m.m[2][3] = float(-distance);
|
||||
// Turned down about x by the strip's pitch (none for the others), facing the eyes.
|
||||
const double tilt = p.strip ? kNotePitch * M_PI / 180 : 0, c = std::cos(tilt), sn = std::sin(tilt);
|
||||
m.m[0][0] = 1;
|
||||
m.m[1][1] = float(c), m.m[1][2] = float(-sn), m.m[2][1] = float(sn), m.m[2][2] = float(c);
|
||||
m.m[1][3] = float(distance * sn), m.m[2][3] = float(-distance * c);
|
||||
std::lock_guard<std::mutex> lk(g_vr);
|
||||
ov->SetOverlayTransformTrackedDeviceRelative(h, vr::k_unTrackedDeviceIndex_Hmd, &m);
|
||||
ov->SetOverlayWidthInMeters(h, float(2 * distance * half));
|
||||
};
|
||||
std::fprintf(stderr, "ft-gazepanel running: @%s, %.2f m\n", sockName.c_str(), distance);
|
||||
auto hideDot = [&] {
|
||||
if (dot.shown && !noVr) {
|
||||
std::lock_guard<std::mutex> lk(g_vr);
|
||||
ov->HideOverlay(dotH);
|
||||
}
|
||||
dot.shown = false;
|
||||
};
|
||||
// The next frame's vsync count and when it reaches the eyes (RAW ns).
|
||||
auto nextFrame = [&](int64_t now, uint64_t &frame) {
|
||||
double since = 0;
|
||||
frame = 0;
|
||||
if (!noVr) {
|
||||
float s = 0;
|
||||
std::lock_guard<std::mutex> lk(g_vr);
|
||||
if (vr::VRSystem()->GetTimeSinceLastVsync(&s, &frame)) since = s;
|
||||
} else {
|
||||
const int64_t per = int64_t(period * 1e9);
|
||||
since = double(now % per) / 1e9, frame = uint64_t(now / per);
|
||||
}
|
||||
frame += 1;
|
||||
return now + int64_t((period - std::min(since, period) + toPhotons) * 1e9);
|
||||
};
|
||||
auto ipdLook = [&](int64_t now, const char *why) {
|
||||
if (noVr || !ipdLog.f) return;
|
||||
vr::ETrackedPropertyError pe;
|
||||
float v;
|
||||
{
|
||||
std::lock_guard<std::mutex> lk(g_vr);
|
||||
v = vr::VRSystem()->GetFloatTrackedDeviceProperty(vr::k_unTrackedDeviceIndex_Hmd, vr::Prop_UserIpdMeters_Float, &pe);
|
||||
}
|
||||
ipdAt = now;
|
||||
if (pe != vr::TrackedProp_Success || v == ipdNow) return;
|
||||
ipdNow = v;
|
||||
std::fprintf(ipdLog.f, "{\"t\": %.6f, \"ipd\": %.5f, \"from\": \"%s\"}\n", Sec(now), v, why);
|
||||
++ipdLog.rows;
|
||||
};
|
||||
// The rec mode's dot, once a frame: where it is at the next frame's photons, the overlay
|
||||
// moved there, and a targets row when it moved or changed.
|
||||
auto dotTick = [&](int64_t now) {
|
||||
if (!p.rec || !visible || dot.mode == "off") {
|
||||
if (dot.changed && targets.f && p.rec) {
|
||||
uint64_t frame;
|
||||
const int64_t t = nextFrame(now, frame);
|
||||
std::fprintf(targets.f, "{\"t\": %.6f, \"set\": %.6f, \"frame\": %llu, \"dot\": \"off\", \"dist\": %.3f, \"bg\": %.3f}\n",
|
||||
Sec(t), Sec(now), (unsigned long long)frame, distance, p.bg);
|
||||
++targets.rows;
|
||||
}
|
||||
dot.changed = false;
|
||||
hideDot();
|
||||
return;
|
||||
}
|
||||
uint64_t frame;
|
||||
const int64_t t = nextFrame(now, frame);
|
||||
double yaw = dot.yaw, pitch = dot.pitch, dist = distance;
|
||||
bool moved = false;
|
||||
vr::HmdMatrix34_t m{};
|
||||
bool absolute = false;
|
||||
if (dot.mode == "path") {
|
||||
if (dot.start < 0) dot.start = t;
|
||||
PathAt(dot, Sec(t - dot.start), yaw, pitch);
|
||||
moved = true;
|
||||
}
|
||||
if (dot.mode == "world") {
|
||||
// The head where it will be at the photons, and the dot's direction from it.
|
||||
const vr::TrackedDevicePose_t pose = HeadPose(noVr, float(Sec(t - now)));
|
||||
if (pose.bPoseIsValid) {
|
||||
double v[3];
|
||||
ToDevice(pose.mDeviceToAbsoluteTracking, dot.room, v);
|
||||
dist = std::sqrt(v[0] * v[0] + v[1] * v[1] + v[2] * v[2]);
|
||||
if (dist > 1e-6) {
|
||||
for (double &c : v) c /= dist;
|
||||
Angles(v, yaw, pitch);
|
||||
}
|
||||
const double eye[3] = {pose.mDeviceToAbsoluteTracking.m[0][3], pose.mDeviceToAbsoluteTracking.m[1][3],
|
||||
pose.mDeviceToAbsoluteTracking.m[2][3]};
|
||||
m = Facing(dot.room, eye);
|
||||
absolute = true;
|
||||
}
|
||||
moved = true;
|
||||
} else {
|
||||
double v[3];
|
||||
Direction(yaw, pitch, v);
|
||||
const double at[3] = {v[0] * dist, v[1] * dist, v[2] * dist}, eye[3] = {0, 0, 0};
|
||||
m = Facing(at, eye);
|
||||
}
|
||||
const bool redraw = dot.state != dot.drawnState || dot.progress != dot.drawnProgress || p.bg != dot.drawnBg;
|
||||
if (redraw) {
|
||||
dotPic.px.assign(size_t(dotPic.w) * dotPic.h * 4, 0);
|
||||
DrawDot(dotPic, dotPic.w / 2.0, dotPic.h / 2.0, dotPic.w / dotPic.wDeg, dot.state, dot.progress, p.bg > 0.5 ? 0.0 : 1.0);
|
||||
dot.drawnState = dot.state, dot.drawnProgress = dot.progress, dot.drawnBg = p.bg;
|
||||
if (!noVr) dotBuffers.Present(ov, dotH, dotPic);
|
||||
}
|
||||
if (!noVr && (moved || dot.changed || redraw)) {
|
||||
std::lock_guard<std::mutex> lk(g_vr);
|
||||
if (absolute) ov->SetOverlayTransformAbsolute(dotH, vr::TrackingUniverseStanding, &m);
|
||||
else ov->SetOverlayTransformTrackedDeviceRelative(dotH, vr::k_unTrackedDeviceIndex_Hmd, &m);
|
||||
ov->SetOverlayWidthInMeters(dotH, float(2 * dist * std::tan(kDotDeg * M_PI / 360)));
|
||||
if (!dot.shown) ov->ShowOverlay(dotH);
|
||||
}
|
||||
dot.shown = true;
|
||||
if (targets.f && (moved || dot.changed || redraw)) {
|
||||
std::fprintf(targets.f, "{\"t\": %.6f, \"set\": %.6f, \"frame\": %llu, \"dot\": \"%s\", \"yaw\": %.4f, \"pitch\": %.4f, "
|
||||
"\"dist\": %.3f, \"state\": \"%s\", \"progress\": %.3f, \"bg\": %.3f}\n",
|
||||
Sec(t), Sec(now), (unsigned long long)frame, dot.mode.c_str(), yaw, pitch, dist, dot.state.c_str(),
|
||||
dot.progress, p.bg);
|
||||
++targets.rows;
|
||||
}
|
||||
dot.changed = false;
|
||||
};
|
||||
std::fprintf(stderr, "ft-gazepanel running: @%s, %.2f m%s\n", sockName.c_str(), distance, noVr ? ", no SteamVR" : "");
|
||||
|
||||
while (!g_stop) {
|
||||
char buf[512];
|
||||
@@ -467,30 +969,118 @@ int main(int argc, char **argv) {
|
||||
socklen_t fromLen = sizeof from;
|
||||
ssize_t n;
|
||||
while ((n = recvfrom(sock, buf, sizeof buf - 1, 0, reinterpret_cast<sockaddr *>(&from), &fromLen)) > 0) {
|
||||
while (n > 0 && (buf[n - 1] == '\n' || buf[n - 1] == '\r')) --n; // from echo or socat
|
||||
buf[n] = 0;
|
||||
std::string reply = "ok";
|
||||
char word[16] = "", state[16] = "";
|
||||
double a = 0, b = 0, c = 0, d = 0, e = 0;
|
||||
int rest = 0;
|
||||
const char *arg = nullptr;
|
||||
if (!std::strncmp(buf, "show ", 5)) {
|
||||
p.rec = !std::strcmp(buf + 5, "rec");
|
||||
p.full = !std::strcmp(buf + 5, "full");
|
||||
p.fit = !std::strcmp(buf + 5, "fit");
|
||||
const bool five = !std::strcmp(buf + 5, "five");
|
||||
p.w = p.full ? kFullW : p.fit ? kFitW : five ? kFiveW : kQuickPx;
|
||||
p.h = p.full ? kFullH : p.fit ? kFitH : five ? kFiveH : kQuickPx;
|
||||
p.wDeg = p.full ? kFullDeg : p.fit ? kFitDeg : five ? kFiveDeg : kQuickDeg;
|
||||
p.strip = !std::strcmp(buf + 5, "note");
|
||||
const bool five = !std::strcmp(buf + 5, "five"), big = p.full || p.rec;
|
||||
p.w = big ? kFullW : p.fit ? kFitW : five ? kFiveW : p.strip ? kNoteW : kQuickPx;
|
||||
p.h = big ? kFullH : p.fit ? kFitH : five ? kFiveH : p.strip ? kNoteH : kQuickPx;
|
||||
p.wDeg = big ? kFullDeg : p.fit ? kFitDeg : five ? kFiveDeg : p.strip ? kNoteDeg : kQuickDeg;
|
||||
p.title.clear(), p.text.clear(), p.note.clear(), p.dotOn = false, p.state = "off";
|
||||
p.eyes[0] = p.eyes[1] = EyeCard{}, p.hints.clear();
|
||||
p.step.clear(), p.big.clear(), p.action.clear(), p.keys.clear();
|
||||
p.recOn = p.paused = false, p.countdown = -1;
|
||||
dot.mode = "off", dot.changed = true;
|
||||
place();
|
||||
visible = dirty = true; // shown with its first picture
|
||||
} else if (!std::strcmp(buf, "hide")) {
|
||||
ov->HideOverlay(h);
|
||||
if (!noVr) {
|
||||
std::lock_guard<std::mutex> lk(g_vr);
|
||||
ov->HideOverlay(h);
|
||||
}
|
||||
hideDot();
|
||||
visible = shown = false;
|
||||
dot.mode = "off", dot.changed = true;
|
||||
} else if (std::sscanf(buf, "bg %lf", &a) == 1) {
|
||||
p.bg = std::clamp(a, 0.0, 1.0), dirty = true;
|
||||
p.bg = std::clamp(a, 0.0, 1.0), dirty = true, dot.changed = true;
|
||||
} else if (std::sscanf(buf, "dot %lf %lf %15s %lf", &a, &b, state, &c) >= 3) {
|
||||
p.dotYaw = a, p.dotPitch = b, p.state = state, p.progress = c;
|
||||
p.dotOn = std::strcmp(state, "off") != 0, dirty = true;
|
||||
p.dotOn = std::strcmp(state, "off") != 0;
|
||||
if (p.rec) {
|
||||
dot.mode = p.dotOn ? "fixed" : "off", dot.yaw = a, dot.pitch = b, dot.state = state, dot.progress = c;
|
||||
dot.changed = true;
|
||||
} else {
|
||||
dirty = true;
|
||||
}
|
||||
} else if (Is(buf, "path", &arg)) {
|
||||
char kind[16] = "";
|
||||
double v[5] = {0, 0, 0, 0, 0};
|
||||
const int got = std::sscanf(arg, "%15s %lf %lf %lf %lf %lf", kind, &v[0], &v[1], &v[2], &v[3], &v[4]);
|
||||
const int want = !std::strcmp(kind, "circle") ? 5 : (!std::strcmp(kind, "line") || !std::strcmp(kind, "eight")) ? 6 : 0;
|
||||
if (!p.rec) {
|
||||
reply = "error path is for show rec";
|
||||
} else if (!want || got != want || v[want - 2] <= 0) {
|
||||
reply = "error path circle|line|eight <numbers> <seconds>";
|
||||
} else {
|
||||
dot.mode = "path", dot.kind = kind, dot.seconds = v[want - 2], dot.start = -1;
|
||||
for (int k = 0; k < 4; ++k) dot.a[k] = k < want - 2 ? v[k] : 0;
|
||||
dot.state = "look", dot.progress = 0, dot.changed = true;
|
||||
}
|
||||
} else if (Is(buf, "world", &arg)) {
|
||||
double v[3];
|
||||
char st[16] = "look";
|
||||
double prog = 0;
|
||||
if (!std::strcmp(arg, "off")) {
|
||||
if (dot.mode == "world") dot.mode = "off", dot.changed = true;
|
||||
} else if (!p.rec) {
|
||||
reply = "error world is for show rec";
|
||||
} else if (std::sscanf(arg, "%lf %lf %lf %15s %lf", &v[0], &v[1], &v[2], st, &prog) < 3) {
|
||||
reply = "error world <x> <y> <z> [<state> [<progress>]]";
|
||||
} else {
|
||||
const vr::TrackedDevicePose_t pose = HeadPose(noVr, 0);
|
||||
if (!pose.bPoseIsValid) {
|
||||
reply = "error no headset pose";
|
||||
} else {
|
||||
if (dot.mode != "world" || v[0] != dot.head[0] || v[1] != dot.head[1] || v[2] != dot.head[2])
|
||||
ToRoom(pose.mDeviceToAbsoluteTracking, v, dot.room);
|
||||
std::copy(v, v + 3, dot.head);
|
||||
dot.mode = "world", dot.state = st, dot.progress = prog, dot.changed = true;
|
||||
char r[96];
|
||||
std::snprintf(r, sizeof r, "ok %.4f %.4f %.4f", dot.room[0], dot.room[1], dot.room[2]);
|
||||
reply = r;
|
||||
}
|
||||
}
|
||||
} else if (std::sscanf(buf, "distance %lf", &a) == 1) {
|
||||
distance = std::clamp(a, 0.25, 10.0), dot.changed = true;
|
||||
place();
|
||||
} else if (Is(buf, "ipd")) {
|
||||
float v = 0.063f;
|
||||
vr::ETrackedPropertyError pe = vr::TrackedProp_Success;
|
||||
if (!noVr) {
|
||||
std::lock_guard<std::mutex> lk(g_vr);
|
||||
v = vr::VRSystem()->GetFloatTrackedDeviceProperty(vr::k_unTrackedDeviceIndex_Hmd, vr::Prop_UserIpdMeters_Float, &pe);
|
||||
}
|
||||
char r[48];
|
||||
std::snprintf(r, sizeof r, "ok %.5f", v);
|
||||
reply = pe == vr::TrackedProp_Success ? r : "error no IPD from SteamVR";
|
||||
} else if (Is(buf, "log", &arg)) {
|
||||
if (!std::strncmp(arg, "start ", 6)) {
|
||||
const std::string dir = arg + 6;
|
||||
if (!targets.Open(dir + "/targets.jsonl") || !ipdLog.Open(dir + "/ipd.jsonl") ||
|
||||
!poses.Start(dir + "/poses.jsonl", !noVr)) {
|
||||
targets.Close(), ipdLog.Close(), poses.Stop();
|
||||
reply = "error can't write in " + dir;
|
||||
} else {
|
||||
ipdNow = -1, dot.changed = true;
|
||||
ipdLook(RawNs(), "start");
|
||||
}
|
||||
} else if (!std::strcmp(arg, "stop")) {
|
||||
char r[96];
|
||||
const long tr = targets.Close(), ir = ipdLog.Close(), pr = poses.Stop();
|
||||
std::snprintf(r, sizeof r, "ok %ld %ld %ld", tr, pr, ir);
|
||||
reply = r;
|
||||
} else {
|
||||
reply = "error log start <dir> | log stop";
|
||||
}
|
||||
} else if (int k; std::sscanf(buf, "eye %d %lf %lf %lf %lf %lf %n", &k, &a, &b, &c, &d, &e, &rest) >= 6 &&
|
||||
rest > 0 && (k == 0 || k == 1)) {
|
||||
p.eyes[k] = EyeCard{buf + rest, a, b, c, d, e}, dirty = true;
|
||||
@@ -503,12 +1093,26 @@ int main(int argc, char **argv) {
|
||||
at = bar + 1;
|
||||
}
|
||||
dirty = true;
|
||||
} else if (!std::strncmp(buf, "title", 5)) {
|
||||
p.title = buf[5] == ' ' ? buf + 6 : "", dirty = true;
|
||||
} else if (!std::strncmp(buf, "text", 4)) {
|
||||
p.text = buf[4] == ' ' ? buf + 5 : "", dirty = true;
|
||||
} else if (!std::strncmp(buf, "note", 4)) {
|
||||
p.note = buf[4] == ' ' ? buf + 5 : "", dirty = true;
|
||||
} else if (Is(buf, "title", &arg)) {
|
||||
p.title = arg, dirty = true;
|
||||
} else if (Is(buf, "text", &arg)) {
|
||||
p.text = arg, dirty = true;
|
||||
} else if (Is(buf, "note", &arg)) {
|
||||
p.note = arg, dirty = true;
|
||||
} else if (Is(buf, "step", &arg)) {
|
||||
p.step = arg, dirty = true;
|
||||
} else if (Is(buf, "big", &arg)) {
|
||||
p.big = arg, dirty = true;
|
||||
} else if (Is(buf, "action", &arg)) {
|
||||
p.action = arg, dirty = true;
|
||||
} else if (Is(buf, "keys", &arg)) {
|
||||
p.keys = arg, dirty = true;
|
||||
} else if (Is(buf, "rec", &arg) && (!std::strcmp(arg, "on") || !std::strcmp(arg, "off"))) {
|
||||
p.recOn = !std::strcmp(arg, "on"), dirty = true;
|
||||
} else if (Is(buf, "paused", &arg) && (!std::strcmp(arg, "on") || !std::strcmp(arg, "off"))) {
|
||||
p.paused = !std::strcmp(arg, "on"), dirty = true;
|
||||
} else if (Is(buf, "countdown", &arg)) {
|
||||
p.countdown = !std::strcmp(arg, "off") ? -1 : std::clamp(std::atof(arg), 0.0, 1.0), dirty = true;
|
||||
} else if (std::sscanf(buf, "%15s", word) == 1 && !std::strcmp(word, "ping")) {
|
||||
reply = visible ? "ok shown" : "ok hidden";
|
||||
} else {
|
||||
@@ -518,26 +1122,57 @@ int main(int argc, char **argv) {
|
||||
sendto(sock, reply.data(), reply.size(), 0, reinterpret_cast<sockaddr *>(&from), fromLen);
|
||||
fromLen = sizeof from;
|
||||
}
|
||||
vr::VREvent_t ev;
|
||||
while (vr::VRSystem()->PollNextEvent(&ev, sizeof ev))
|
||||
if (ev.eventType == vr::VREvent_Quit) {
|
||||
vr::VRSystem()->AcknowledgeQuit_Exiting();
|
||||
g_stop = true;
|
||||
if (!noVr) {
|
||||
vr::VREvent_t ev;
|
||||
std::unique_lock<std::mutex> lk(g_vr);
|
||||
while (vr::VRSystem()->PollNextEvent(&ev, sizeof ev)) {
|
||||
if (ev.eventType == vr::VREvent_Quit) {
|
||||
vr::VRSystem()->AcknowledgeQuit_Exiting();
|
||||
g_stop = true;
|
||||
} else if (ev.eventType == vr::VREvent_IpdChanged && ipdLog.f) {
|
||||
ipdNow = ev.data.ipd.ipdMeters;
|
||||
std::fprintf(ipdLog.f, "{\"t\": %.6f, \"ipd\": %.5f, \"from\": \"event\"}\n", Sec(RawNs()), ipdNow);
|
||||
++ipdLog.rows;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (visible && dirty) {
|
||||
Draw(p);
|
||||
buffers.Present(ov, h, p);
|
||||
if (!shown) ov->ShowOverlay(h), shown = true;
|
||||
++pictures;
|
||||
if (!dumpDir.empty()) Dump(p, dumpDir + "/" + std::to_string(pictures) + ".pam");
|
||||
if (!noVr) {
|
||||
buffers.Present(ov, h, p);
|
||||
std::lock_guard<std::mutex> lk(g_vr);
|
||||
if (!shown) ov->ShowOverlay(h);
|
||||
}
|
||||
shown = true;
|
||||
dirty = false;
|
||||
}
|
||||
// Until a command comes, or 10 ms while shown (SteamVR's events). Hidden, it waits up to a
|
||||
// second: it woke 20 to 30 times a second for nothing, the main cost left with gaze idle.
|
||||
// A closed stdin (--watch-stdin) wakes it too, so quitting doesn't wait.
|
||||
const int64_t now = RawNs();
|
||||
dotTick(now);
|
||||
if (ipdLog.f && now - ipdAt >= kIpdPeriodNs) ipdLook(now, "look");
|
||||
// Until a command comes, or the next frame while the rec mode's dot moves, or 10 ms while
|
||||
// shown (SteamVR's events). Hidden, it waits up to a second: it woke 20 to 30 times a
|
||||
// second for nothing, the main cost left with gaze idle. A closed stdin (--watch-stdin)
|
||||
// wakes it too, so quitting doesn't wait.
|
||||
double wait = visible ? 0.010 : 1.0;
|
||||
if (visible && p.rec && (dot.mode == "path" || dot.mode == "world")) {
|
||||
uint64_t frame;
|
||||
// Just after the next vsync: the compositor has taken this frame's transform.
|
||||
wait = std::max(0.001, Sec(nextFrame(now, frame) - now) - toPhotons + 0.001);
|
||||
}
|
||||
if (ipdLog.f) wait = std::min(wait, Sec(kIpdPeriodNs));
|
||||
pollfd fds[2] = {{sock, POLLIN, 0}, {0, POLLIN, 0}};
|
||||
poll(fds, watchStdin ? 2 : 1, visible ? 10 : 1000);
|
||||
const timespec ts{time_t(wait), long((wait - std::floor(wait)) * 1e9)};
|
||||
ppoll(fds, watchStdin ? 2 : 1, &ts, nullptr);
|
||||
}
|
||||
ov->DestroyOverlay(h);
|
||||
buffers.Drop();
|
||||
vr::VR_Shutdown();
|
||||
targets.Close(), ipdLog.Close(), poses.Stop();
|
||||
if (!noVr) {
|
||||
std::lock_guard<std::mutex> lk(g_vr);
|
||||
ov->DestroyOverlay(dotH);
|
||||
ov->DestroyOverlay(h);
|
||||
}
|
||||
buffers.Drop(), dotBuffers.Drop();
|
||||
if (!noVr) vr::VR_Shutdown();
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,96 @@
|
||||
# Recording your eyes for the Frametop eye dataset (DRAFT, not reviewed yet)
|
||||
|
||||
Version: 2026-10-14-draft
|
||||
|
||||
Frametop's eye tracking needs a better model than it has: one that keeps working when you look far to the side, blink, squint, wear glasses or makeup, or when the headset moves on your face. To train it, we're collecting recordings of many people's eyes into a dataset that only the maintainers can read. This page explains what the eye recorder records, where it goes, who can see it, and what you agree to if you take part. Please read all of it.
|
||||
|
||||
## Who runs this
|
||||
|
||||
Frametop is an independent open-source project, maintained by DeeJanuz (<https://github.com/DeeJanuz>). It isn't affiliated with or endorsed by Valve or Hugging Face. Steam and Steam Frame are trademarks of Valve Corporation. This text was written without a lawyer, in good faith; if something in it seems wrong or unclear, please say so before you take part.
|
||||
|
||||
You can reach the maintainer through the Frametop repository's issues (<https://github.com/Frametop/frametop/issues>) or the dataset's discussion page on Hugging Face. Both are public.
|
||||
|
||||
## Who can take part
|
||||
|
||||
- You must be **18 or older**.
|
||||
- For now, you can't take part if you live in **Illinois, Texas or Washington** (USA). Those states have laws about biometric data, which includes images of the iris, that need more than this project can provide yet.
|
||||
- Take part only if you're comfortable with the maintainers seeing close-up images of your eyes.
|
||||
|
||||
## Your eyes are biometric data
|
||||
|
||||
The headset's eye cameras film each eye from a few centimetres away. The iris in those images can identify a person, the way a fingerprint can. That's why this dataset works differently from Frametop's hand dataset:
|
||||
|
||||
- **The eye images are never public.** Your headset encrypts them before they're uploaded, so that only the maintainers' keys can read them. Hugging Face, other contributors and people who download the dataset get only the encrypted files.
|
||||
- **What's trained from them can be public.** Models trained on the dataset can be published and shared. They don't contain your images, though no one can rule out that a model remembers something about the eyes it was trained on.
|
||||
|
||||
## What is recorded
|
||||
|
||||
While a round runs, the eye recorder saves:
|
||||
|
||||
- **Eye images.** Infrared images from the headset's 2 eye cameras, about 90 a second each. They show your eyes, eyelids, eyelashes and the skin around them, and your glasses or contact lenses if you wear them. Encrypted in the export.
|
||||
- **Where you looked.** Where each dot was when you looked at it, what Frametop's eye tracker and SteamVR's found in each image (pupil and glint positions, gaze directions), and how the fit check rated the headset's position. These stay readable in the export, so a pull request can be checked before anyone decrypts it.
|
||||
- **Motion and settings.** The headset's position and rotation, many times a second, and its IPD setting (the distance between the lenses).
|
||||
- **Prompts.** What you were asked to do and when.
|
||||
- **Your answers.** Glasses, contacts or inserts, eye makeup, the face pad and strap, and your measured IPD if you gave it. Readable in the export.
|
||||
- **Health answers, only if you tick their box.** Eye surgery and conditions such as a turned eye, a droopy eyelid or nystagmus. They're health data, so they're kept in a separate file and encrypted like the images. You can leave them out, and still take part.
|
||||
- **Times.** When each round was recorded, with your time zone.
|
||||
- **A contributor id.** A random number made on your headset the first time you agree to this page. It isn't made from your name, your Steam account or your headset. It keeps your rounds together and lets you withdraw them.
|
||||
|
||||
The recorder doesn't record sound, the headset's outward cameras, your name, your email address or your Steam account.
|
||||
|
||||
**Your Hugging Face account.** You upload with your own Hugging Face account, and your pull request shows its username next to your contributor id, publicly. The images in it are encrypted, but anyone can see that this account contributed eye recordings, and read the readable parts listed above. Use an account you're happy to have linked to them.
|
||||
|
||||
## What it's used for
|
||||
|
||||
- Training and testing eye-tracking models: finding the pupil, the glints and the eyelids, and working out where you're looking, including when the headset sits badly. Mainly for Frametop, and possibly elsewhere (below, "The license").
|
||||
- Checking and improving the recorder's fit coach, which helps people put the headset on well.
|
||||
- It isn't used to recognise or identify people, and the maintainers won't try.
|
||||
|
||||
## Nothing leaves your headset unless you send it
|
||||
|
||||
- Recordings stay on your headset, in `~/.local/share/frametop/eyes/contrib`. Nothing is uploaded automatically.
|
||||
- Before you share anything, you can look through every recording in the Review page. You can delete any stretch of a take, a whole take or a whole round. Export leaves out what you deleted.
|
||||
- You upload the export yourself, from the Upload page. That opens a pull request on the dataset. The maintainer checks it before it becomes part of the dataset, and may decline it. Until it's merged, you can close the pull request yourself.
|
||||
|
||||
## Where it goes, and who can read it
|
||||
|
||||
- The dataset is hosted on Hugging Face (<https://huggingface.co/datasets/DeeJanuz/frametop-eyes>), whose servers may be outside your country, for example in the USA. It's gated: people must accept its terms to download it, and what they get has the eye images and health answers encrypted.
|
||||
- Only the maintainers hold the keys that decrypt them. Today that's DeeJanuz, with one key on their own computer and a backup copy stored offline. The keys are never uploaded. If a key ever leaked, uploads made to it could be read: the maintainers would say so on the dataset's page and stop using it.
|
||||
- Decrypted images are kept only on the maintainers' own computers, for training and checking models, and aren't shared with anyone else or uploaded to other services.
|
||||
- Recordings stay in the dataset until they're withdrawn or the dataset is taken down.
|
||||
|
||||
## Safety
|
||||
|
||||
The rounds ask you to look at dots, including far to the sides, to blink and squint, and sometimes to turn your head or move the headset on your face. Sit where you normally use the headset and stop whenever anything is uncomfortable, or if your eyes get tired or sore. If you have a condition that bright or moving lights affect, such as photosensitive epilepsy, don't take part. You take part at your own risk.
|
||||
|
||||
## The license
|
||||
|
||||
- **You give the maintainer of Frametop a non-exclusive, worldwide, royalty-free license to use your contribution for any purpose, including commercially.** That includes copying it, changing it, and training, publishing and selling models made from it, in Frametop and elsewhere. It doesn't include publishing your eye images or health answers, or giving them to anyone outside the maintainers. The maintainer today is DeeJanuz. If someone else, or an organization, takes over maintaining Frametop, this license passes to them, with the same limits. It's non-exclusive: you keep any rights you have in your recordings and can do what you like with your own copies. It ends for a recording when you withdraw it, except for models already trained with it.
|
||||
- You confirm that you have the right to give this license: the recordings are of your own eyes.
|
||||
- There is no payment. The dataset and the eye recorder come with no warranty, and as far as the law allows, the maintainer isn't liable for any loss or damage from taking part.
|
||||
|
||||
## Withdrawing
|
||||
|
||||
You can withdraw a contribution at any time, without giving a reason:
|
||||
|
||||
- **Before it's merged:** close your pull request on Hugging Face. The maintainer deletes its files.
|
||||
- **After it's merged:** post on the dataset's discussion page, or in the Frametop repository's issues, with your contributor id (shown in the eye recorder) and which rounds, or "all". Post from the Hugging Face account that uploaded them if you can, so the maintainer can tell it's you; otherwise, say how to check. These requests are public, so don't add anything else that identifies you.
|
||||
|
||||
Then, usually within 30 days:
|
||||
|
||||
- your recordings are deleted from the dataset and purged from its repository's history, and the maintainers delete their decrypted copies;
|
||||
- they're left out of anything trained after that.
|
||||
|
||||
What can't be undone: models already trained with them.
|
||||
|
||||
## Your rights
|
||||
|
||||
Depending on where you live (for example in the EU or the UK, under the GDPR), the law may give you more rights over this data: to get a copy of it, to have it corrected or deleted, to object to its use, and to complain to your data protection authority. Eye images that can identify you and health answers are special kinds of data there, and they're used only because you explicitly agreed to it, here and with the health answers' own tick. You can withdraw that agreement at any time, as above. Withdrawing doesn't make earlier use unlawful. To use any of these rights, contact the maintainer as above.
|
||||
|
||||
## Changes to this text
|
||||
|
||||
If this text changes, the eye recorder shows the new version and asks you again before your next round. Each upload records the version you agreed to, and recordings you already uploaded stay under that version, unless you agree to a newer one or withdraw them.
|
||||
|
||||
## Agreeing
|
||||
|
||||
By ticking the three boxes and "Agree and continue", you confirm that you're 18 or older, that you don't live in Illinois, Texas or Washington, and that you agree to the above, including the use of images of your eyes as described. You can still decide not to upload anything.
|
||||
@@ -0,0 +1,176 @@
|
||||
# Eye recorder: design (2026-10-14)
|
||||
|
||||
The eye recorder guides a person through recording their eyes with the headset's two eye cameras, at known gaze targets and through the cases where eye tracking breaks. It also fits the headset: it shows whether each eye sits where the cameras see it best, helps set the IPD, and catches the headset shifting on the face, with instructions to put it back. Recordings stay on the headset until the person uploads them. Only the maintainers can read what's uploaded (below, "Privacy").
|
||||
|
||||
It's the hand recorder's sibling (`hands/rec/DESIGN.md`): the same session runner pattern, headset button, review, export and Hugging Face upload, with eye cameras instead of hand cameras. Like the hand recorder it ships in Frametop and, later, as a standalone app for people without Frametop (Frametop/frametop-eye-recorder, Frametop as a submodule).
|
||||
|
||||
The data is for training a better eye tracker than `gaze/tracker/` (ft-eyes, a classic pupil and glint finder) and SteamVR's, one that holds up at the edges of view, through blinks and squints, with glasses and makeup, and when the headset moves.
|
||||
|
||||
## Decisions (2026-10-14)
|
||||
|
||||
| | Decided |
|
||||
|---|---|
|
||||
| Who reads the recordings | Maintainers only. The eye cameras see the iris, which identifies a person. Trained models can be public; the images never are. |
|
||||
| How | Encrypted frames in a gated dataset, `DeeJanuz/frametop-eyes` (below, "Privacy"). |
|
||||
| Keys | Two maintainer keys, both recipients of every upload: key 1 on the maintainer's training machine (the 7i), key 2 kept offline in the maintainer's password manager. |
|
||||
| Packaging | Backend here in `gaze/rec/`, then a standalone repo with Frametop as a submodule, as the hand recorder does. |
|
||||
| Sessions | Short rounds picked from a bounty board (below, "Bounty board"), not one long session. Each round starts with a fit check and 9 dots. |
|
||||
| Bounty list | `bounties.json` in the dataset repo, fetched by the app (a copy ships for offline use). The board shows your own done marks and a maintainer-set priority tag per bounty, not community counts. |
|
||||
| Fit and slip source | Our tracker, ft-eyes: pupil and glint positions in the camera image. It needs ft-eyegrab, the root service `gaze/tracker/install.sh` installs. |
|
||||
| Where the fit coach runs | A fitment step in the recorder before each round, for better recordings, and the put-on and slip check in Frametop's gaze service. |
|
||||
| Shared code | The hand recorder's non-hand parts move into a shared module first, and the bounty board is built there too, so the hand recorder can use it next. |
|
||||
| License | As hands: contributors give the maintainer a license for any use, including commercial models; the eye images and health answers are never published or shared outside the maintainers (user, 2026-10-14). |
|
||||
| Standalone repo | Created after the window and a full round pass in the headset; step 9 comes first (user, 2026-10-14). |
|
||||
| Checklist | Glasses/inserts/contacts, eye makeup, measured IPD, and eye surgery or conditions. That last one is health data: a separate opt-in, and its answers are encrypted like the images. |
|
||||
|
||||
## Privacy
|
||||
|
||||
A private Hugging Face repo can't take contributions: people outside the org can't see it, so they can't open a pull request, and org membership would let every member read every contribution. So the dataset is gated like `DeeJanuz/frametop-hands`, and the export encrypts what identifies a person before upload:
|
||||
- Every frame chunk is encrypted with [age](https://age-encryption.org) (pyrage, which has aarch64 wheels: `pyrage-1.4.0-cp310-abi3-manylinux_2_17_aarch64`) to the recipients in `recipients.txt`: the maintainers' public keys: key 1 on the 7i, key 2 offline. The private keys never go in a repo.
|
||||
- The health answers go in `health.json.age`, encrypted the same way.
|
||||
- Labels, timings, fit readings, the IPD and the rest of session.json stay readable, so `validate.py` and a review can check a pull request before anyone decrypts it.
|
||||
- Contributors open pull requests from their own accounts, so `hub.py` and the review tools carry over from hands.
|
||||
|
||||
The risk: if a private key leaks, every upload made to it can be read. A new key is one more recipient for new uploads; old uploads stay readable to the old key until they're re-encrypted. Withdrawal means deleting the files and squashing history, as with hands.
|
||||
|
||||
## What's measured already
|
||||
|
||||
- **The eye cameras** (`gaze/tracker/ft-eyegrab.c`): 512x400 8-bit IR, about 90 fps each when SteamVR runs them at 90 (`gaze/eye-rate.py`; at its default 15 fps a recording is worth little, so the recorder checks the rate first). Camera 0 is the right eye (steep angle), camera 1 the left (head-on). ft-eyegrab copies the frames read-only into `/dev/shm/frametop-eyes-cams` while someone touches `/dev/shm/frametop-eyes-want`.
|
||||
- **Size.** Raw, both cameras are 36.9 MB/s, 2.2 GB a minute. zstd level 1 on 200 MB of the `fit1` capture (2026-10-14): 56 % of the size at about 250 MB/s on one core, so compressing while recording costs about 15 % of a core. Level 3: 53 %, 2.3x slower. Level 9: 49 %, 9.5x slower. The recorder uses level 1 while recording: about 1.2 GB per recorded minute, so a session of about 10 recorded minutes (below) is about 12 GB, close to a hand session.
|
||||
- **IPD.** SteamVR reports it (2026-10-14, a probe against the live vrserver): `Prop_UserIpdMeters_Float` 0.0651, `Prop_IpdUIRangeMinMeters` 0.057, `Max` 0.071, `Prop_MinimumIpdStepMeters` 0.0015, and the eye-to-head transforms are ±IPD/2. `driver_cv.so` reads the range from the headset's EEPROM. So the recorder can read the IPD setting live and record it with every take. Not yet known: whether the eye cameras move with the lenses when the IPD changes. That decides how the IPD guidance works (below), and it's the first headset test.
|
||||
- **ft-eyes** already finds each pupil (centre, ellipse) and glint pair per frame, keeps a per-eye slip estimate from the glints (`eyes_model.SlipTracker`), and resets its shift when frames stop for 3 s (the headset was off). `gaze/fitcheck.py` already keeps per-eye tracked and lost by gaze direction, with hints.
|
||||
|
||||
## Parts
|
||||
|
||||
| Part | What it does |
|
||||
|---|---|
|
||||
| `gaze/rec/session.py` (Python, no Qt) | The session runner, like `hands/rec/session.py`: reads a round's script, drives the panel, starts and stops the frame writer per step, reads the live fit for feedback, writes each take. `--dry-run`, `--next-after`, stdin steering and `--button-device` as in hands. |
|
||||
| `gaze/rec/rounds/<id>.json` | Each kind of round's steps, targets and timings (below, "Bounty board"). |
|
||||
| `gaze/rec/eyerec.py` | The frame writer: reads the shared frames (as `lab/ft-eyes-record` does) and writes zstd-1 chunks, one per second per camera, with an index (below, "Files"). A thread of the session, not a process: ft-eyes-record shows the copy needs no numpy. |
|
||||
| `gaze/fitcoach.py` (shared) | The fit coach: from ft-eyes' pupil and glint positions, how the headset sits on this face, and what to do about it (below, "Fit coach"). The recorder, the Gaze page's fit check and the put-on check all use it. |
|
||||
| ft-gazepanel `rec` mode | The headset panel. The gaze panel already draws dots at exact head-relative directions, which the ground truth needs, so it gains a `rec` mode with the hand panel's text, step, countdown and paused lines rather than a second panel (below, "Panel"). |
|
||||
| `gaze/rec/takes.py`, `validate.py`, `hub.py` | Review, export, checks and upload, from `hands/rec/`'s. The shared parts move into a common module rather than being copied (below, "Shared with the hand recorder"). |
|
||||
| `gaze/rec/ft_eyerec.py` + `main.qml` | The desktop window: consent, the bounty board, the checklist and eye camera check, the round with its fit card, review, export, upload. Launched by `ft-eyerec` (the dev container), installed to the app menu by `install.sh`. The standalone app gets a Qt Quick Controls window, as the hand recorder's does. |
|
||||
| `gaze/rec/CONSENT.md`, `UPLOAD.md` | The texts. Eye images need a stronger consent than hands (biometric data, who can read it, how to withdraw). They need a review like `frame-hands/notes/legal-review-2026-10-03.md` before uploads open. |
|
||||
|
||||
## Fit coach
|
||||
|
||||
It reads, per eye, from ft-eyes: the pupil centre, its ellipse, the glint pair (both glints and their spacing), and how often the eye was found over the last second. It compares them with a reference: where a well-fitted eye sits in that camera's image. At first the reference comes from the maintainer's headset; once contributions come in, it comes from their rounds with good nine-dot fits (every round's fitment step records exactly that), per headset model. The coach says which reference it used, and until it's built from more than one face its hints are marked provisional.
|
||||
|
||||
What it can tell, and the instruction it gives:
|
||||
|
||||
| Reading | Means | Instruction |
|
||||
|---|---|---|
|
||||
| Both pupils high or low in their images | The headset sits low or high on the face | "Push the headset up" / "Let it down a little", with the top strap |
|
||||
| Both pupils left or right | The headset sits off to one side | "Slide the headset right" |
|
||||
| Pupils apart or together, compared with the reference at this IPD | IPD set too narrow or wide | "Turn the IPD wheel wider", until both are centred |
|
||||
| One pupil higher than the other | The headset is tilted | "Level the headset" |
|
||||
| Glint pair spacing larger or smaller than the reference | The eye is nearer or further than usual (eye relief: face pad, straps, glasses) | "Tighten the back strap" / "The headset is pressing close: loosen it" |
|
||||
| A glint missing, or extra bright spots | Glasses reflections, or the lens fogged or smudged | "Tilt your glasses" / "Wipe the lenses" |
|
||||
| An eye found in under half the frames, looking straight ahead | Lashes on the lens, a droopy lid, or the eye outside the camera's view | The per-eye card says which |
|
||||
|
||||
The IPD row depends on the first headset test. If the cameras move with the lenses, a pupil's sideways offset in its image directly says how far that eye is from its lens axis, and the coach turns the IPD wheel's direction into "wider" or "narrower" until both offsets are about zero, at any IPD. If the cameras are fixed, the offset depends on the IPD setting too, and the coach needs the reference at each IPD step: the fitment step's IPD sweep (below, "Bounty board") builds that.
|
||||
|
||||
**Put-on and shift check.** After a good fit, the coach keeps that fit as the person's baseline (`~/.local/state/frametop/gaze/fit-baseline.json`: per eye, the pupil looking straight ahead and the glint pair, with the IPD setting). Then:
|
||||
- When the headset goes back on (frames resume after ft-eyes' 3 s gap), it shows the fit card for a few seconds while the person looks ahead. If the pupils sit more than a threshold (start with 1.5 mm equivalent, tuned from data) from the baseline, it says which way to move the headset, live, until it's back. If the IPD setting changed, it says so first.
|
||||
- While in use, ft-eyes' glint slip estimate already tracks small movement. A shift past the threshold that stays for 2 s gets a quiet notice ("Your headset slipped down: push it up, or look at the dot to recalibrate"), not a modal.
|
||||
In the recorder the fit coach runs as each round's fitment step, and the shift warning runs during the round. In Frametop the put-on and slip check becomes part of the gaze service, off by default until it's been tried.
|
||||
|
||||
## Bounty board
|
||||
|
||||
People pick what to record from a board of bounties. Each bounty is one round of about 5 minutes, in step mode: the headset's right button starts each step (press = next or pause, two presses = redo, hold = stop), as in the hand recorder. Every round starts the same way:
|
||||
1. **Fitment** (not recorded as a take, but its readings go into fit.json): the fit coach until the headset sits well (below, "Fit coach"). Skippable after a warning, and the round records that it was skipped. On a contributor's first round, and whenever the IPD setting has changed since their last, it ends with an IPD sweep: turn the wheel slowly from narrow to wide and back while looking at a centre dot, with the IPD logged as it changes (the panel looks 30 times a second, and on SteamVR's IPD event). Its own take, `NN-ipd`, before the nine dots.
|
||||
2. **Nine dots** (recorded): a 3x3 grid to 20 degrees on the mid background. It's the round's own reference, so each round can be labelled alone, even after a re-seat.
|
||||
3. **The bounty's steps.**
|
||||
|
||||
Every round uploads on its own. The board marks the bounties you've done, by your contributor id in `profile.json`, with how many times. You can do one again.
|
||||
|
||||
The launch bounties:
|
||||
|
||||
| Bounty | What | Edge case it covers |
|
||||
|---|---|---|
|
||||
| View edges | Dots at the panel's edges and corners: 30 degrees to the sides, 23 up and down, about 35 in the corners (the rec panel is 64 degrees across; every round's dots are checked against it) | Eyes turned far: the lid, the glints lost, the pupil foreshortened |
|
||||
| Blinks and lids | Blink on cue, close both for 2 s, wink each eye, squint, eyes wide, half-closed, looking down with lids lowered | Closed and partly closed eyes; one eye missing |
|
||||
| Headset slips | Nudge the headset up, down, left, right, tilt it; then take it off and put it back on; dots after each | Headset shifting on the face: the put-on check's data |
|
||||
| Glasses | A 5x5 grid and the edges with glasses or inserts on (offered when the checklist says so) | Lens reflections, the glasses' frame in view |
|
||||
| Jumps and pursuit | A dot jumping between random spots (0.5 to 1.5 s each), then moving on circles, lines and a figure eight at two speeds | Saccades, overshoot, smooth pursuit, the tracker's lag |
|
||||
| Depth | Dots at 0.5 m, 1.5 m and 5 m in turn (the panel's distance changes) | Vergence; the eyes' angle with distance |
|
||||
| Light changes | The background steps dark to bright and back while looking at a dot, then a 5x5 grid at each of three backgrounds | Pupil dilation and constriction; error against pupil size |
|
||||
| Face moves | Smile, raise eyebrows, frown, yawn, talk (read a line aloud: the microphone isn't recorded), chew | The face moving the headset and the lids |
|
||||
| Head turning | A dot fixed in the room while turning the head (head poses logged) | The eyes counter-rotating (VOR), headset jiggle |
|
||||
| Reading | Lines of text to read, then a page of icons to look over | Natural use |
|
||||
|
||||
Later candidates, not at launch: eye makeup (the grid and blinks with mascara or eyeliner; the checklist still records makeup on every round), long wear (the grid after 30+ minutes in the headset).
|
||||
|
||||
### bounties.json
|
||||
|
||||
In the dataset repo, fetched when the board opens (a copy in the app for offline use, used when the fetch fails):
|
||||
|
||||
```json
|
||||
{"schema": 1, "updated": "2026-10-14",
|
||||
"bounties": [{"id": "edges", "title": "View edges", "blurb": "...", "minutes": 5,
|
||||
"priority": "high|normal|low|retired", "script": "edges", "needs": {"glasses": true}}]}
|
||||
```
|
||||
|
||||
`script` names a round in the app's `rounds/` folder: a new kind of round needs a release, but the list, wording and priorities don't. `needs` hides a bounty unless the checklist matches. A bounty id the app doesn't know is shown as "needs an update".
|
||||
|
||||
The board is in the shared recorder module, so the hand recorder's v2 sections can become bounties later.
|
||||
|
||||
### Checklist
|
||||
|
||||
Asked once and kept in profile.json, editable before each round, all optional, none identifying: glasses (none, glasses, prescription inserts, contacts), eye makeup today (none, mascara, eyeliner, lash extensions), the face pad and strap in use, and a measured IPD if they know it (from an optician). Separately, behind its own consent tick: eye surgery or conditions (LASIK or other surgery, a lazy or turned eye, a droopy lid, nystagmus, other). Those answers go into `health.json`, which the export encrypts. Eye colour isn't asked: the IR images show what matters about it.
|
||||
|
||||
## Panel
|
||||
|
||||
ft-gazepanel gains `show rec`: a large head-locked panel (FULL_DEG, 64 degrees, with a solid background whose brightness the script sets) that keeps every existing command (`dot`, `bg`, `title`, `text`, `note`, `eye`, `hints`) and adds what the hand panel has: `step`, `big` (countdown), `action`, `keys`, `rec on|off`, `paused on|off`. New for the ground truth:
|
||||
- `path <kind> <params> <seconds>`: the panel moves a dot itself (circle, line, figure eight), so it's smooth at the headset's frame rate, not at the socket's.
|
||||
- `distance <metres>`: moves the panel for the Depth bounty.
|
||||
- `world <x> <y> <z>`: a dot fixed in the room (the Head turning bounty), as the hand panel's touch target.
|
||||
- Every picture the panel shows with a dot logs that dot's direction and the frame's time to `targets.jsonl` (CLOCK_MONOTONIC_RAW, the eye frames' clock). The ground truth is what was on screen when, not what was asked.
|
||||
|
||||
## Files
|
||||
|
||||
```
|
||||
~/.local/share/frametop/eyes/contrib/
|
||||
profile.json consent, contributor id, checklist, bounties done
|
||||
rounds/<YYYYMMDD-HHMMSS>-<bounty>/
|
||||
session.json the bounty, checklist, versions, takes, eye camera rate, IPD at start
|
||||
fit.json the fit coach's readings: the fitment step, and any slip warnings
|
||||
health.json the opt-in health answers (encrypted in the export)
|
||||
takes/<NN>-<step group>/ numbered in order: NN-ipd (when there's a sweep), NN-dots (the nine dots), then the bounty's
|
||||
cam0/000000.zst ... right eye: one zstd chunk per second, frames back to back (512x400 u8)
|
||||
cam1/000000.zst ... left eye
|
||||
index.tsv per frame: camera, chunk, offset in it, slot, CLOCK_MONOTONIC_RAW time
|
||||
targets.jsonl what the panel showed, when (direction, distance, state, background)
|
||||
prompts.jsonl what the person was asked to do, when (as hands)
|
||||
poses.jsonl head poses, 250 Hz (as hands; needed for Head turning)
|
||||
ipd.jsonl the IPD setting at the start and when it changes
|
||||
steamvr.jsonl SteamVR's gaze from ft-gaze, per sample (a baseline, not a label)
|
||||
own.jsonl ft-eyes' pupils, glints and gaze (a baseline, not a label)
|
||||
take.json status, deleted ranges, parts, clock samples, as hands
|
||||
exports/<session>/
|
||||
```
|
||||
|
||||
The export recompresses nothing (the chunks are zstd already), drops deleted ranges by rewriting the chunks they touch, encrypts each chunk and `health.json` to `recipients.txt` (`.age`), and writes the manifest and `SHA256SUMS` over the encrypted files. It never includes `/persist` files: unlike hands, nothing here needs the unit's calibration, and the eye cameras' isn't in `/persist` anyway (2026-10-14: `device_config.json` has the displays' transforms only).
|
||||
|
||||
## Shared with the hand recorder
|
||||
|
||||
Much of `hands/rec/` isn't about hands: the headset button reader, profile and consent, take bookkeeping (deleted ranges, redo, parts, clock samples), the Hugging Face login and upload, validate's manifest and checksum checks, and most of the window. These live in `recorder/` at the top of the tree, with the bounty board, and both recorders use them: `common.py`, `units.py`, `button.py`, `bounties.py`, `files.py` (ranges, checksums, sizes), `checks.py` (validate's report, SHA256SUMS, typed manifest lookups), `hub.py` (login, errors, upload records, the pull request flow), `age.py` (the export's encryption) and `window.py` (the window backend's shared half: consent, disk, session controls, export, login and upload; each recorder's `Backend` subclasses it). The hand recorder's modules keep their names on top of them, and its tests pass.
|
||||
|
||||
Uploads: `gaze/rec/hub.py` sends nothing to the real dataset until `CONSENT.md` and `UPLOAD.md` exist and aren't drafts. `FT_EYEREC_ALLOW_UPLOAD=1` opens rehearsals, and only to another dataset (`FT_EYEREC_DATASET`). Rehearsed 2026-10-14 against the private `DeeJanuz/frametop-eyes-test` with a round of random bytes: uploaded, downloaded, checked, and decrypted with both keys by age 1.1.1 on the 7i.
|
||||
|
||||
## Build order
|
||||
|
||||
1. **Headset test 1:** do the eye cameras move with the IPD wheel? (Look at one dot, turn the wheel end to end, watch ft-eyes' pupil x.) Decides the IPD guidance.
|
||||
2. Pull the shared recorder code out of `hands/rec/` (hand recorder tests pass).
|
||||
3. `eyerec.py` (frame writer) and its replay test against a capture.
|
||||
4. `fitcoach.py` with readings from a capture, and the fit card on the existing fit panel.
|
||||
5. ft-gazepanel `rec` mode, `path`, `distance`, target logging.
|
||||
6. `session.py`, the nine-dot opener and the first rounds, run with `--dry-run`, then on a replay. The bounty board and `bounties.json`.
|
||||
7. Review, export with encryption (two recipients), `validate.py`, `hub.py` against a test dataset.
|
||||
8. The window, then the standalone repo.
|
||||
9. The put-on check in the gaze service. Built 2026-10-14 behind `GAZE_FIT_WATCH=1`: the quick check holds its dot while the coach reads the fit against the saved one (the fit check and the recorder's fitment save a good fit), and the slip note is a strip the gaze panel draws (`show note`) from ft-eyes' `jump_px`. Thresholds are the coach's 5 degrees, not 1.5 mm yet; tuned from the headset tests.
|
||||
|
||||
Before uploads open: the consent texts and a review like `frame-hands/notes/legal-review-2026-10-03.md`, with the health opt-in in it.
|
||||
|
||||
Not in scope here: the training. The dataset format is designed so that frame-eyes or a later trainer can label pupils from the targets and `own.jsonl`, but that's its own project.
|
||||
@@ -0,0 +1,11 @@
|
||||
DRAFT: this text waits for its review, with CONSENT.md.
|
||||
### About uploading
|
||||
|
||||
- **What's sent:** the export in `@EXPORT_PATH@` (@EXPORT_SIZE@), into `contributions/@CONTRIBUTOR@/@ROUND@` in [@DATASET@](https://huggingface.co/datasets/@DATASET@). Upload first checks that every file is complete and matches its checksum, and that the eye images and health answers are encrypted.
|
||||
- **Who can read it:** the eye images and health answers are encrypted on this headset to the maintainers' keys. Hugging Face and anyone who downloads the dataset get only the encrypted files. The rest (where the dots were, what the trackers found, the fit readings and your checklist answers) stays readable.
|
||||
- **Your account:** the pull request comes from your Hugging Face account, and anyone can see its username next to your contributor id, `@CONTRIBUTOR@`. The login stays saved on this headset, so you only log in once.
|
||||
- **The dataset's terms:** the first time, open the dataset's page and accept its terms. Until you have, Upload stops with "accept the dataset's terms first".
|
||||
- **Once your pull request's link shows, plug in the headset and leave it plugged in until this page says Uploaded.** A round is several gigabytes, so this can take a while. You can take the headset off: the Eye Recorder keeps it awake until the upload is done. Keep the Eye Recorder open, since closing it stops the upload.
|
||||
- **If it stops partway** (Cancel, or the network drops), press Upload again. It carries on in the same pull request, and files already sent aren't sent twice.
|
||||
- **The maintainer's review:** they check that the files are complete and decrypt correctly before merging. You can follow it and answer questions on the pull request's page. Once it's merged, you can delete the round and its export here to free the space.
|
||||
- **Withdrawing:** see "Withdrawing" in the consent text. You'll need your contributor id, `@CONTRIBUTOR@`.
|
||||
@@ -0,0 +1,13 @@
|
||||
{"schema": 1, "updated": "2026-10-14",
|
||||
"bounties": [
|
||||
{"id": "edges", "title": "View edges", "blurb": "Dots far out to the sides and corners: eyes turned as far as they go.", "minutes": 5, "priority": "high", "script": "edges"},
|
||||
{"id": "blinks", "title": "Blinks and lids", "blurb": "Blinks, winks, squints and half-closed eyes.", "minutes": 5, "priority": "high", "script": "blinks"},
|
||||
{"id": "slips", "title": "Headset slips", "blurb": "Nudge the headset on your face, take it off and on: dots after each.", "minutes": 6, "priority": "high", "script": "slips"},
|
||||
{"id": "glasses", "title": "Glasses", "blurb": "The grid and the edges with glasses or inserts on.", "minutes": 5, "priority": "high", "script": "glasses", "needs": {"glasses": true}},
|
||||
{"id": "jumps", "title": "Jumps and pursuit", "blurb": "A jumping dot, then one gliding on circles, lines and a figure eight.", "minutes": 5, "priority": "normal", "script": "jumps"},
|
||||
{"id": "depth", "title": "Depth", "blurb": "Dots near, mid and far.", "minutes": 4, "priority": "normal", "script": "depth"},
|
||||
{"id": "light", "title": "Light changes", "blurb": "Dark to bright and back: pupils changing size.", "minutes": 5, "priority": "normal", "script": "light"},
|
||||
{"id": "face", "title": "Face moves", "blurb": "Smile, frown, yawn, talk and chew while looking at a dot.", "minutes": 4, "priority": "normal", "script": "face"},
|
||||
{"id": "head", "title": "Head turning", "blurb": "A dot fixed in the room while you turn your head.", "minutes": 4, "priority": "normal", "script": "head"},
|
||||
{"id": "reading", "title": "Reading", "blurb": "Read a few paragraphs and look around freely.", "minutes": 4, "priority": "low", "script": "reading"}
|
||||
]}
|
||||
Executable
+324
@@ -0,0 +1,324 @@
|
||||
#!/usr/bin/env python3
|
||||
"""The eye recorder's frame writer (DESIGN.md: "Files"): every frame of both eye cameras, from
|
||||
the frames ft-eyegrab shares (gaze/tracker/eyes_share.py), into a take's folder as zstd chunks.
|
||||
|
||||
cam0/000000.zst, 000001.zst, ... the right eye (camera 0): up to CHUNK_FRAMES frames each
|
||||
cam1/... the left eye (camera 1)
|
||||
index.tsv a header line, then one line per frame:
|
||||
camera, chunk, frame in the chunk, the tracker's slot, time
|
||||
(CLOCK_MONOTONIC_RAW seconds, the share's)
|
||||
|
||||
A chunk is the frames back to back, 512x400 bytes each, compressed with zstd level 1 (56 % of the
|
||||
size at about 250 MB/s on one core, measured 2026-10-14; both cameras at 90 fps are 36.9 MB/s
|
||||
raw). A chunk ends when it's full or the recording pauses, so a pause or a deleted range never
|
||||
splits one. The index lines of a chunk are written once the chunk is on disk, so the index never
|
||||
names a chunk that isn't there; they're in chunk order, not time order across the cameras.
|
||||
|
||||
One thread copies frames out of the share as they come (it holds only 8 per camera, 89 ms);
|
||||
another compresses and writes, so a slow chunk never makes the copy fall behind. If the writer
|
||||
falls MAX_QUEUED chunks behind, the next chunk is dropped and counted rather than held in memory.
|
||||
|
||||
Plain Python, standard library: zstd is Python 3.14's compression.zstd (the dev container's), or
|
||||
zstandard, or the zstd command (SteamOS's host Python 3.12 has neither module).
|
||||
|
||||
From the command line, for testing (FT_EYES_CAMS points it at a replay, lab/ft-eyes-replay):
|
||||
python3 gaze/rec/eyerec.py DIR [SECONDS]
|
||||
python3 gaze/rec/eyerec.py --read DIR the frames' count and times, decompressed
|
||||
"""
|
||||
import os
|
||||
import queue
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from collections import deque
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
sys.path.insert(0, os.path.join(os.path.dirname(HERE), "tracker"))
|
||||
import eyes_share # noqa: E402 (gaze/tracker/eyes_share.py)
|
||||
|
||||
W, H = eyes_share.W, eyes_share.H
|
||||
FRAME = W * H
|
||||
CAMERAS = (0, 1) # 0 the right eye, 1 the left (gaze/tracker/README: camera 0 is the steep one)
|
||||
CHUNK_FRAMES = 90 # a second at 90 fps
|
||||
LEVEL = 1
|
||||
MAX_QUEUED = 6 # chunks waiting for the writer (about 18 MB each) before one is dropped
|
||||
MIN_FREE = 3e9 # bytes: stop before the disk fills
|
||||
RATE_WINDOW = 2.0 # s of frame times for each camera's rate
|
||||
SHARE_WAIT = 10.0 # s to wait for the share at the start, or after it was replaced
|
||||
POLL = 0.002
|
||||
INDEX_HEADER = "# camera\tchunk\tframe\tslot\tt_raw\n"
|
||||
|
||||
|
||||
def _compressor():
|
||||
"""(compress(bytes) -> bytes, decompress(bytes) -> bytes, name)."""
|
||||
try:
|
||||
from compression import zstd # Python 3.14
|
||||
return (lambda b: zstd.compress(b, level=LEVEL)), zstd.decompress, "compression.zstd"
|
||||
except ImportError:
|
||||
pass
|
||||
try:
|
||||
import zstandard
|
||||
c, d = zstandard.ZstdCompressor(level=LEVEL), zstandard.ZstdDecompressor()
|
||||
return c.compress, (lambda b: d.decompress(b, max_output_size=CHUNK_FRAMES * FRAME)), "zstandard"
|
||||
except ImportError:
|
||||
pass
|
||||
exe = shutil.which("zstd") or ("/usr/bin/zstd" if os.path.exists("/usr/bin/zstd") else None)
|
||||
if not exe:
|
||||
return None, None, None
|
||||
|
||||
def run(args, data):
|
||||
return subprocess.run([exe, *args], input=data, capture_output=True, check=True).stdout
|
||||
return (lambda b: run(["-q", "-%d" % LEVEL, "-c"], b)), (lambda b: run(["-q", "-d", "-c"], b)), exe
|
||||
|
||||
|
||||
compress, decompress, ZSTD = _compressor()
|
||||
|
||||
|
||||
class Writer:
|
||||
"""Records the eye cameras into take_dir while recording is on.
|
||||
|
||||
start() starts the threads (not recording yet); record(True) and record(False) start and
|
||||
pause; stop() ends the open chunks, waits for the writer and returns stats(). error is set
|
||||
(and recording stops) when the disk gets full or the share is gone for good."""
|
||||
|
||||
def __init__(self, take_dir, share_path=None, want_path=None, log=None, min_free=MIN_FREE,
|
||||
chunk_frames=CHUNK_FRAMES):
|
||||
if compress is None:
|
||||
raise RuntimeError("no zstd: neither compression.zstd, zstandard nor the zstd command")
|
||||
self.dir, self.share_path, self.want_path = take_dir, share_path, want_path
|
||||
self.log = log or (lambda line: None)
|
||||
self.min_free, self.chunk_frames = min_free, chunk_frames
|
||||
self.error = None
|
||||
self._on = False
|
||||
self._lock = threading.Lock()
|
||||
self._stop = threading.Event()
|
||||
self._queue = queue.Queue()
|
||||
self._open = {c: [] for c in CAMERAS} # the chunk being filled: [(t, slot, bytes)]
|
||||
self._next = {c: 0 for c in CAMERAS} # the next chunk's number
|
||||
self._frames = {c: 0 for c in CAMERAS} # frames written (in chunks on disk)
|
||||
self._dropped = {c: 0 for c in CAMERAS} # frames lost: overwritten in the share, or a dropped chunk
|
||||
self._times = {c: deque() for c in CAMERAS}
|
||||
self._last = {c: None for c in CAMERAS}
|
||||
self._bytes = 0
|
||||
self._busy = 0.0 # s spent compressing and writing
|
||||
for c in CAMERAS:
|
||||
os.makedirs(os.path.join(take_dir, "cam%d" % c), exist_ok=True)
|
||||
self._index = open(os.path.join(take_dir, "index.tsv"), "a")
|
||||
if self._index.tell() == 0:
|
||||
self._index.write(INDEX_HEADER)
|
||||
# Numbering goes on after any chunks already there (a take resumed after a pause in another run).
|
||||
for c in CAMERAS:
|
||||
names = [n for n in os.listdir(os.path.join(take_dir, "cam%d" % c)) if n.endswith(".zst")]
|
||||
self._next[c] = max((int(n[:-4]) + 1 for n in names if n[:-4].isdigit()), default=0)
|
||||
self._reader = threading.Thread(target=self._read, name="eyerec-read", daemon=True)
|
||||
self._writer = threading.Thread(target=self._write, name="eyerec-write", daemon=True)
|
||||
|
||||
def start(self):
|
||||
self._reader.start()
|
||||
self._writer.start()
|
||||
return self
|
||||
|
||||
def record(self, on):
|
||||
with self._lock:
|
||||
if self._on and not on:
|
||||
for c in CAMERAS:
|
||||
self._end_chunk(c)
|
||||
self._on = bool(on) and self.error is None
|
||||
|
||||
@property
|
||||
def recording(self):
|
||||
return self._on
|
||||
|
||||
def stop(self):
|
||||
self.record(False)
|
||||
self._stop.set()
|
||||
self._reader.join(5)
|
||||
self._queue.put(None)
|
||||
self._writer.join(60)
|
||||
self._index.close()
|
||||
return self.stats()
|
||||
|
||||
def stats(self):
|
||||
"""{"frames": [right, left], "dropped": [...], "fps": [...], "bytes": on disk, "busy": s,
|
||||
"queued": chunks waiting, "last_t": [the newest frame's time or None], "error"}"""
|
||||
with self._lock:
|
||||
fps = []
|
||||
for c in CAMERAS:
|
||||
ts = self._times[c]
|
||||
fps.append(round((len(ts) - 1) / (ts[-1] - ts[0]), 1) if len(ts) > 1 and ts[-1] > ts[0] else 0.0)
|
||||
return {"frames": [self._frames[c] for c in CAMERAS], "dropped": [self._dropped[c] for c in CAMERAS],
|
||||
"fps": fps, "bytes": self._bytes, "busy": round(self._busy, 3), "queued": self._queue.qsize(),
|
||||
"last_t": [self._last[c] for c in CAMERAS], "error": self.error}
|
||||
|
||||
# -- the copy
|
||||
|
||||
def _read(self):
|
||||
share = eyes_share.open_share(SHARE_WAIT, self.share_path, self.want_path)
|
||||
if share is None:
|
||||
self._fail("no eye camera frames (%s): is frametop-eyegrab.service running?" % (self.share_path or eyes_share.CAMS))
|
||||
return
|
||||
seen = {c: share.count(c) for c in CAMERAS}
|
||||
touched = checked = time.monotonic()
|
||||
while not self._stop.is_set():
|
||||
got = False
|
||||
for c in CAMERAS:
|
||||
n = share.count(c)
|
||||
if n - seen[c] > share.slots: # fell behind: those are gone
|
||||
with self._lock:
|
||||
if self._on:
|
||||
self._dropped[c] += n - seen[c] - share.slots
|
||||
seen[c] = n - share.slots
|
||||
for i in range(seen[c], n):
|
||||
f = share.frame(c, i)
|
||||
got = True
|
||||
with self._lock:
|
||||
if f is None:
|
||||
if self._on:
|
||||
self._dropped[c] += 1
|
||||
continue
|
||||
self._seen(c, f[0])
|
||||
if self._on:
|
||||
self._open[c].append(f)
|
||||
if len(self._open[c]) >= self.chunk_frames:
|
||||
self._end_chunk(c)
|
||||
seen[c] = n
|
||||
now = time.monotonic()
|
||||
if now - touched >= 1.0:
|
||||
touched = now
|
||||
eyes_share.touch_want(self.want_path)
|
||||
if now - checked >= 2.0:
|
||||
checked = now
|
||||
if self._on and shutil.disk_usage(self.dir).free < self.min_free:
|
||||
self._fail("under %.1f GB free on the disk" % (self.min_free / 1e9))
|
||||
if share.replaced(): # ft-eyegrab restarted (SteamVR did): follow it
|
||||
self.log("eye frames: the share was replaced; following it")
|
||||
share.close()
|
||||
share = eyes_share.open_share(SHARE_WAIT, self.share_path, self.want_path)
|
||||
if share is None:
|
||||
self._fail("the eye camera frames stopped (frametop-eyegrab.service)")
|
||||
return
|
||||
seen = {c: share.count(c) for c in CAMERAS}
|
||||
if not got:
|
||||
time.sleep(POLL)
|
||||
share.close()
|
||||
|
||||
def _seen(self, c, t):
|
||||
ts = self._times[c]
|
||||
ts.append(t)
|
||||
while ts and ts[0] < t - RATE_WINDOW:
|
||||
ts.popleft()
|
||||
self._last[c] = t
|
||||
|
||||
def _end_chunk(self, c):
|
||||
"""Hand the open chunk to the writer (with the lock held)."""
|
||||
frames = self._open[c]
|
||||
if not frames:
|
||||
return
|
||||
self._open[c] = []
|
||||
if self._queue.qsize() >= MAX_QUEUED:
|
||||
self._dropped[c] += len(frames)
|
||||
self.log("eye frames: the writer is behind; dropped %d frames of camera %d" % (len(frames), c))
|
||||
return
|
||||
self._queue.put((c, self._next[c], frames))
|
||||
self._next[c] += 1
|
||||
|
||||
def _fail(self, why):
|
||||
with self._lock:
|
||||
if self.error is None:
|
||||
self.error = why
|
||||
self.log("eye frames: " + why)
|
||||
if self._on:
|
||||
for c in CAMERAS:
|
||||
self._end_chunk(c)
|
||||
self._on = False
|
||||
|
||||
# -- the writer
|
||||
|
||||
def _write(self):
|
||||
while True:
|
||||
item = self._queue.get()
|
||||
if item is None:
|
||||
return
|
||||
c, chunk, frames = item
|
||||
t0 = time.perf_counter()
|
||||
path = os.path.join(self.dir, "cam%d" % c, "%06d.zst" % chunk)
|
||||
try:
|
||||
data = compress(b"".join(f[2] for f in frames))
|
||||
with open(path + ".tmp", "wb") as f:
|
||||
f.write(data)
|
||||
os.replace(path + ".tmp", path)
|
||||
except (OSError, subprocess.CalledProcessError) as e:
|
||||
self._fail("can't write %s: %s" % (path, e))
|
||||
with self._lock:
|
||||
self._dropped[c] += len(frames)
|
||||
continue
|
||||
self._index.write("".join("%d\t%d\t%d\t%d\t%.6f\n" % (c, chunk, i, f[1], f[0]) for i, f in enumerate(frames)))
|
||||
self._index.flush()
|
||||
with self._lock:
|
||||
self._frames[c] += len(frames)
|
||||
self._bytes += len(data)
|
||||
self._busy += time.perf_counter() - t0
|
||||
|
||||
|
||||
def read_index(take_dir):
|
||||
"""index.tsv as [(camera, chunk, frame, slot, t), ...] in its order."""
|
||||
out = []
|
||||
try:
|
||||
with open(os.path.join(take_dir, "index.tsv")) as f:
|
||||
for line in f:
|
||||
if line.startswith("#") or not line.strip():
|
||||
continue
|
||||
c, chunk, i, slot, t = line.split("\t")
|
||||
out.append((int(c), int(chunk), int(i), int(slot), float(t)))
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
return out
|
||||
|
||||
|
||||
def read_chunk(take_dir, cam, chunk):
|
||||
"""The frames of a chunk, as a list of 512x400 byte strings."""
|
||||
with open(os.path.join(take_dir, "cam%d" % cam, "%06d.zst" % chunk), "rb") as f:
|
||||
raw = decompress(f.read())
|
||||
if len(raw) % FRAME:
|
||||
raise ValueError("cam%d/%06d.zst: %d bytes isn't whole frames" % (cam, chunk, len(raw)))
|
||||
return [raw[i:i + FRAME] for i in range(0, len(raw), FRAME)]
|
||||
|
||||
|
||||
def iter_frames(take_dir):
|
||||
"""Every frame in index order: (camera, t, slot, bytes)."""
|
||||
cache = (None, None)
|
||||
for c, chunk, i, slot, t in read_index(take_dir):
|
||||
if cache[0] != (c, chunk):
|
||||
cache = ((c, chunk), read_chunk(take_dir, c, chunk))
|
||||
yield c, t, slot, cache[1][i]
|
||||
|
||||
|
||||
def main(argv):
|
||||
if not argv or argv[0] in ("-h", "--help"):
|
||||
sys.exit(__doc__)
|
||||
if argv[0] == "--read":
|
||||
n, ts = [0, 0], [[], []]
|
||||
for c, t, _slot, _data in iter_frames(argv[1]):
|
||||
n[c] += 1
|
||||
ts[c].append(t)
|
||||
for c in CAMERAS:
|
||||
span = max(ts[c]) - min(ts[c]) if len(ts[c]) > 1 else 0.0
|
||||
print("camera %d: %d frames over %.2f s" % (c, n[c], span))
|
||||
return
|
||||
secs = float(argv[1]) if len(argv) > 1 else 10.0
|
||||
w = Writer(argv[0], log=lambda line: print(line, file=sys.stderr)).start()
|
||||
w.record(True)
|
||||
end = time.monotonic() + secs
|
||||
try:
|
||||
while time.monotonic() < end and w.error is None:
|
||||
time.sleep(1)
|
||||
print(w.stats(), file=sys.stderr)
|
||||
except KeyboardInterrupt:
|
||||
pass
|
||||
print(w.stop())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main(sys.argv[1:])
|
||||
Executable
+21
@@ -0,0 +1,21 @@
|
||||
#!/bin/bash
|
||||
# Launch the Frametop Eye Recorder from a Plasma session on the Frame host.
|
||||
# The app runs in the dev container (PySide6, Kirigami and zstd come from Fedora there).
|
||||
# podman needs the real XDG_RUNTIME_DIR and the real user bus (to reach systemd for
|
||||
# the container's cgroup; the Frametop session runs on a private bus from
|
||||
# dbus-run-session). The session's Wayland socket and bus go to the app itself.
|
||||
# Options go to ft_eyerec.py: --base DIR, --page NAME, and --dry-run (--hub-dry-run, --ignore-eyes) for testing.
|
||||
here=$(cd "$(dirname "$(readlink -f "$0")")" && pwd)
|
||||
wl=${WAYLAND_DISPLAY:-wayland-0}
|
||||
case $wl in /*) ;; *) wl="${XDG_RUNTIME_DIR:-/run/user/$(id -u)}/$wl" ;; esac
|
||||
session_bus=${DBUS_SESSION_BUS_ADDRESS:-}
|
||||
export XDG_RUNTIME_DIR=/run/user/$(id -u)
|
||||
export DBUS_SESSION_BUS_ADDRESS=unix:path=$XDG_RUNTIME_DIR/bus
|
||||
# From the home folder: the app's host commands (distrobox-host-exec) run in its folder on the
|
||||
# host, and a folder only the container has, such as /run/host/tmp, makes them all fail.
|
||||
cd "$HOME" || exit 1
|
||||
"$here/../../scripts/container-up.sh"
|
||||
exec "$HOME/.local/bin/distrobox" enter dev -- env WAYLAND_DISPLAY="$wl" DISPLAY="${DISPLAY:-}" \
|
||||
XAUTHORITY="${XAUTHORITY:-}" DBUS_SESSION_BUS_ADDRESS="$session_bus" \
|
||||
QT_QPA_PLATFORM="wayland;xcb" \
|
||||
python3 "$here/ft_eyerec.py" "$@"
|
||||
@@ -0,0 +1,9 @@
|
||||
[Desktop Entry]
|
||||
Type=Application
|
||||
Name=Frametop Eye Recorder
|
||||
GenericName=Record your eyes for the eye dataset
|
||||
Comment=Fit the headset, then record, review and export eye recordings for Frametop's eye dataset
|
||||
Exec=@REPO@/gaze/rec/ft-eyerec
|
||||
Icon=camera-video
|
||||
Categories=Utility;
|
||||
Keywords=eyes;eye tracking;gaze;dataset;record;frametop;
|
||||
@@ -0,0 +1,827 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Frametop Eye Recorder: record your eyes for the eye dataset (gaze/rec/DESIGN.md).
|
||||
|
||||
A Kirigami (QML) app with a Python backend, in the dev container, as the hand recorder
|
||||
(hands/rec/ft_handrec.py); what the two windows share is recorder/window.py:
|
||||
- Welcome: the consent text (CONSENT.md), the first time and again when its version changes.
|
||||
Agreeing writes profile.json with a random contributor id.
|
||||
- Bounties: the bounty board (recorder/bounties.py): what to record, the priority the
|
||||
maintainers gave it, and how often you've done it. Refresh fetches the dataset's board
|
||||
(hub.py board). Picking one goes on to the checklist.
|
||||
- Before you start: the checklist (glasses, eye makeup, face pad and strap, a measured IPD),
|
||||
kept in profile.json; the health questions behind their own tick, kept in BASE/health.json
|
||||
and encrypted in the export; free space; the eye camera check (frames from ft-eyegrab at
|
||||
90 fps, with the fix when SteamVR runs them at 15). Start hands the round to session.py.
|
||||
- Round: the runner's live status, the fit card while the headset is fitted, Next, Pause,
|
||||
Redo, Skip and Stop (Space, P, R, S and Esc while the window has focus; the button on the
|
||||
headset's right side too). The prompts and dots appear in the headset.
|
||||
- Review: rounds, their takes, and a viewer of both eye cameras a frame at a time, where
|
||||
ranges, takes and rounds can be deleted (takes.py).
|
||||
- Export: encrypt what's kept into exports/<round>/ at nice 19 (takes.py).
|
||||
- Upload: the Hugging Face login, and Upload: hub.py checks the export (validate.py) and
|
||||
uploads it as a pull request. Nothing reaches the real dataset until CONSENT.md and
|
||||
UPLOAD.md are reviewed (hub.py's gate); --hub-dry-run does everything but the network calls.
|
||||
Everything lives under ~/.local/share/frametop/eyes/contrib (--base). Launch with
|
||||
gaze/rec/ft-eyerec (host wrapper).
|
||||
"""
|
||||
import argparse
|
||||
import bisect
|
||||
import datetime
|
||||
import json
|
||||
import os
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
|
||||
from PySide6.QtCore import Property, Qt, QTimer, QUrl, Signal, Slot
|
||||
from PySide6.QtGui import QColor, QFont, QGuiApplication, QIcon, QImage, QPainter
|
||||
from PySide6.QtQml import QQmlApplicationEngine
|
||||
from PySide6.QtQuick import QQuickImageProvider
|
||||
from PySide6.QtQuickControls2 import QQuickStyle
|
||||
import shiboken6
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
GAZE = os.path.dirname(HERE)
|
||||
REPO = os.path.dirname(GAZE)
|
||||
sys.path.insert(0, HERE)
|
||||
sys.path.insert(0, REPO)
|
||||
import eyerec # noqa: E402 (next to this file; it imports gaze/tracker/eyes_share.py)
|
||||
import eyes_share # noqa: E402
|
||||
import hub # noqa: E402
|
||||
import takes # noqa: E402
|
||||
from recorder import bounties # noqa: E402
|
||||
from recorder.common import host_command, write_json # noqa: E402
|
||||
from recorder.files import human_bytes, read_json # noqa: E402
|
||||
from recorder.window import RecorderBackend # noqa: E402
|
||||
|
||||
HUB_PATH = os.path.join(HERE, "hub.py")
|
||||
EYE_RATE = os.path.join(GAZE, "eye-rate.py")
|
||||
AWAKE_UNIT = "frametop-eyerec-awake.service"
|
||||
# A round of about 5 recorded minutes is about 6 GB (1.2 GB a minute: DESIGN.md, "Size"), and
|
||||
# its export as much again.
|
||||
ROUND_BYTES = 12 * 1000 ** 3
|
||||
ACTIVE_STATES = ("starting", "fitment", "ready", "countdown", "running", "paused")
|
||||
EYE_FPS_OK = 60 # under this the cameras run at SteamVR's 15 fps: recordings are worth little
|
||||
EYE_CHECK_S = 1.5 # how long the eye camera check counts frames
|
||||
# After the eye rate fix's SteamVR restart: check the cameras this often, for at most this long.
|
||||
RECHECK_S = 3
|
||||
RECHECK_FOR_S = 120
|
||||
FRAME_HEIGHT = 400 # the viewer's height for each camera (the cameras' own)
|
||||
THUMB_HEIGHT = 96
|
||||
# The checklist's choices (DESIGN.md, "Checklist"); the keys are what profile.json keeps.
|
||||
GLASSES = [("none", "No glasses"), ("glasses", "Glasses"), ("inserts", "Prescription lens inserts"),
|
||||
("contacts", "Contact lenses")]
|
||||
MAKEUP = [("mascara", "Mascara"), ("eyeliner", "Eyeliner"), ("lashes", "Lash extensions")]
|
||||
PADS = [("", "Rather not say"), ("stock", "The face pad it came with"), ("other", "Another face pad")]
|
||||
STRAPS = [("", "Rather not say"), ("stock", "The strap it came with"), ("other", "Another strap")]
|
||||
# The health questions: only with their own tick, and only ever in health.json(.age).
|
||||
HEALTH = [("surgery", "LASIK or other eye surgery"), ("strabismus", "A lazy or turned eye"),
|
||||
("ptosis", "A droopy eyelid"), ("nystagmus", "Nystagmus (eyes that move by themselves)"),
|
||||
("other", "Another eye condition")]
|
||||
GROUP_TITLES = {"ipd": "IPD sweep", "dots": "Nine dots"}
|
||||
STATUS_TEXT = {"running": "interrupted", "error": "ended with an error", "stopped": "stopped early"}
|
||||
|
||||
|
||||
def round_label(rid, title=""):
|
||||
"""20261014-120000-edges -> 2026-10-14 12:00 · View edges."""
|
||||
try:
|
||||
when = datetime.datetime.strptime(rid[:15], "%Y%m%d-%H%M%S").strftime("%Y-%m-%d %H:%M")
|
||||
except ValueError:
|
||||
return rid
|
||||
rest = rid[16:]
|
||||
n = rest.rsplit("-", 1)[1] if "-" in rest and rest.rsplit("-", 1)[1].isdigit() else ""
|
||||
return f"{when} · {title or rest}" + (f" ({n})" if n else "")
|
||||
|
||||
|
||||
def take_title(group):
|
||||
return GROUP_TITLES.get(group) or group.replace("-", " ").replace("_", " ").capitalize()
|
||||
|
||||
|
||||
def clock_text(seconds):
|
||||
return f"{int(seconds // 60)}:{seconds % 60:04.1f}"
|
||||
|
||||
|
||||
def eye_check(share_path=None, seconds=EYE_CHECK_S):
|
||||
"""Are the eye cameras' frames coming, and how fast? {"state": ok|slow|none, "fps": [right,
|
||||
left], "text"}. Counts frames in ft-eyegrab's share (read-only; it touches the want file,
|
||||
which wakes ft-eyegrab, as every reader does)."""
|
||||
share = eyes_share.open_share(3.0, share_path)
|
||||
if share is None:
|
||||
return {"state": "none", "fps": [0, 0],
|
||||
"text": "No eye camera frames: frametop-eyegrab.service isn't running. Frametop's gaze setup "
|
||||
"installs it (gaze/tracker/install.sh), and SteamVR has to be running."}
|
||||
try:
|
||||
time.sleep(0.5) # ft-eyegrab starts copying once it sees the want file
|
||||
eyes_share.touch_want()
|
||||
a, t0 = [share.count(c) for c in eyerec.CAMERAS], time.monotonic()
|
||||
time.sleep(seconds)
|
||||
b, t1 = [share.count(c) for c in eyerec.CAMERAS], time.monotonic()
|
||||
finally:
|
||||
share.close()
|
||||
fps = [round((y - x) / (t1 - t0), 1) for x, y in zip(a, b)]
|
||||
if max(fps) <= 0:
|
||||
return {"state": "none", "fps": fps,
|
||||
"text": "The eye cameras aren't sending frames. Is SteamVR running? If it is, restarting it "
|
||||
"starts the eye cameras again."}
|
||||
if min(fps) < EYE_FPS_OK:
|
||||
return {"state": "slow", "fps": fps,
|
||||
"text": f"The eye cameras run at {min(fps):.0f} frames a second; recordings need 90. SteamVR runs "
|
||||
"them at 90 with a setting Frametop can turn on, from SteamVR's next start."}
|
||||
return {"state": "ok", "fps": fps, "text": ""}
|
||||
|
||||
|
||||
class Timeline:
|
||||
"""A take's frames for the viewer: the camera with more frames sets the timeline, and the
|
||||
other camera's frame nearest in time goes beside it. Chunks are decompressed when needed
|
||||
and the last few kept."""
|
||||
KEEP = 4
|
||||
|
||||
def __init__(self, take_dir, clock):
|
||||
self.dir = take_dir
|
||||
self.clock = clock or []
|
||||
frames = {c: [] for c in eyerec.CAMERAS}
|
||||
for c, chunk, i, _slot, t in eyerec.read_index(take_dir):
|
||||
if c in frames:
|
||||
frames[c].append((t, chunk, i))
|
||||
for c in frames:
|
||||
frames[c].sort()
|
||||
self.main = max(eyerec.CAMERAS, key=lambda c: (len(frames[c]), -c))
|
||||
self.frames = frames
|
||||
self.times = {c: [f[0] for f in frames[c]] for c in frames}
|
||||
self._chunks = {}
|
||||
self._lock = threading.Lock()
|
||||
|
||||
def __len__(self):
|
||||
return len(self.frames[self.main])
|
||||
|
||||
def time(self, i):
|
||||
return self.frames[self.main][i][0]
|
||||
|
||||
def mono_ns(self, t_raw):
|
||||
"""A frame's RAW seconds as CLOCK_MONOTONIC ns (take.json's clock samples, turned round)."""
|
||||
raw = int(round(t_raw * 1e9))
|
||||
guess = raw - takes.raw_offset_ns(raw, self.clock)
|
||||
return raw - takes.raw_offset_ns(guess, self.clock)
|
||||
|
||||
def _chunk(self, cam, chunk):
|
||||
with self._lock:
|
||||
key = (cam, chunk)
|
||||
if key in self._chunks:
|
||||
self._chunks[key] = self._chunks.pop(key) # the newest at the end
|
||||
return self._chunks[key]
|
||||
frames = eyerec.read_chunk(self.dir, cam, chunk)
|
||||
with self._lock:
|
||||
self._chunks[key] = frames
|
||||
while len(self._chunks) > self.KEEP:
|
||||
self._chunks.pop(next(iter(self._chunks)))
|
||||
return frames
|
||||
|
||||
def pair(self, i):
|
||||
"""{camera: bytes} at timeline index i: the main camera's frame and the other's nearest."""
|
||||
out = {}
|
||||
t, chunk, k = self.frames[self.main][i]
|
||||
out[self.main] = self._chunk(self.main, chunk)[k]
|
||||
for c in eyerec.CAMERAS:
|
||||
if c == self.main or not self.frames[c]:
|
||||
continue
|
||||
j = bisect.bisect_left(self.times[c], t)
|
||||
j = min(range(max(0, j - 1), min(len(self.times[c]), j + 1)), key=lambda n: abs(self.times[c][n] - t))
|
||||
_, chunk, k = self.frames[c][j]
|
||||
out[c] = self._chunk(c, chunk)[k]
|
||||
return out
|
||||
|
||||
|
||||
class Timelines:
|
||||
"""Timelines by take, made again when the take's index.tsv or take.json changes."""
|
||||
|
||||
def __init__(self, store):
|
||||
self.store = store
|
||||
self._made = {}
|
||||
self._lock = threading.Lock()
|
||||
|
||||
def get(self, rnd, take):
|
||||
d = self.store.take_dir(rnd, take)
|
||||
stamp = tuple(os.path.getmtime(os.path.join(d, n)) if os.path.exists(os.path.join(d, n)) else 0
|
||||
for n in ("index.tsv", "take.json"))
|
||||
with self._lock:
|
||||
got = self._made.get((rnd, take))
|
||||
if got and got[0] == stamp:
|
||||
return got[1]
|
||||
tl = Timeline(d, read_json(os.path.join(d, "take.json")).get("clock"))
|
||||
with self._lock:
|
||||
self._made[(rnd, take)] = (stamp, tl)
|
||||
if len(self._made) > 8:
|
||||
self._made.pop(next(iter(self._made)))
|
||||
return tl
|
||||
|
||||
|
||||
def grey_image(data):
|
||||
img = QImage(data, eyerec.W, eyerec.H, eyerec.W, QImage.Format_Grayscale8)
|
||||
return img.copy()
|
||||
|
||||
|
||||
class EyeProvider(QQuickImageProvider):
|
||||
"""image://eyes/pair/ROUND/TAKE/INDEX: both eye cameras at a timeline index, the right eye
|
||||
(camera 0) first. image://eyes/thumb/ROUND/TAKE: the main camera mid-take, small. Anything
|
||||
after "?" is ignored (it makes QML load the image again)."""
|
||||
|
||||
def __init__(self, timelines):
|
||||
super().__init__(QQuickImageProvider.ImageType.Image)
|
||||
self.timelines = timelines
|
||||
|
||||
def requestImage(self, ident, size, requested):
|
||||
parts = ident.split("?", 1)[0].split("/")
|
||||
try:
|
||||
if parts[0] == "thumb" and len(parts) == 3:
|
||||
tl = self.timelines.get(parts[1], parts[2])
|
||||
if not len(tl):
|
||||
raise ValueError("no frames")
|
||||
data = tl.pair(len(tl) // 2)[tl.main]
|
||||
return grey_image(data).scaledToHeight(THUMB_HEIGHT, Qt.SmoothTransformation)
|
||||
if parts[0] == "pair" and len(parts) == 4:
|
||||
return self._pair(self.timelines.get(parts[1], parts[2]), int(parts[3]),
|
||||
requested.height() if requested.height() > 0 else FRAME_HEIGHT)
|
||||
except (OSError, ValueError, IndexError, KeyError) as e:
|
||||
print(f"ft-eyerec: image {ident}: {e}", file=sys.stderr)
|
||||
img = QImage(4, 3, QImage.Format_Grayscale8)
|
||||
img.fill(0)
|
||||
return img
|
||||
|
||||
def _pair(self, tl, i, height):
|
||||
frames = tl.pair(i)
|
||||
gap = 8
|
||||
scaled = [(c, grey_image(frames[c]).scaledToHeight(height, Qt.SmoothTransformation))
|
||||
for c in eyerec.CAMERAS if c in frames]
|
||||
out = QImage(sum(s.width() for _, s in scaled) + gap * (len(scaled) - 1), height, QImage.Format_RGB32)
|
||||
out.fill(QColor(30, 30, 30))
|
||||
p = QPainter(out)
|
||||
font = QFont()
|
||||
font.setPixelSize(max(12, height // 22))
|
||||
p.setFont(font)
|
||||
x = 0
|
||||
for c, img in scaled:
|
||||
p.drawImage(x, 0, img)
|
||||
p.setPen(QColor(255, 200, 80))
|
||||
p.drawText(x + 6, 6 + font.pixelSize(), ("Right eye" if c == 0 else "Left eye") + f" (camera {c})")
|
||||
x += img.width() + gap
|
||||
p.end()
|
||||
return out
|
||||
|
||||
|
||||
class Backend(RecorderBackend):
|
||||
"""The eye recorder's window backend: recorder/window.py's, plus the bounty board, the
|
||||
checklist and health answers, the eye camera check, the round and the review."""
|
||||
APP = "Frametop Eye Recorder"
|
||||
AWAKE_UNIT = AWAKE_UNIT
|
||||
AWAKE_WHY = "Exporting or uploading eye recordings"
|
||||
HUB_PATH = HUB_PATH
|
||||
CONSENT_PATH = hub.CONSENT_PATH
|
||||
UPLOAD_PATH = hub.UPLOAD_PATH
|
||||
ACTIVE_STATES = ACTIVE_STATES
|
||||
ROUND_BYTES = ROUND_BYTES
|
||||
PROFILE_KEEP = ("checklist", "bounties", "eyes_ipd")
|
||||
|
||||
# A property here can't notify with the base's signals (PySide6 6.11 leaves it without
|
||||
# one), so these relay them.
|
||||
profileRelay = Signal()
|
||||
sessionsRelay = Signal()
|
||||
boardChanged = Signal()
|
||||
healthChanged = Signal()
|
||||
eyesChanged = Signal()
|
||||
_eyesArrived = Signal(dict)
|
||||
_boardArrived = Signal(dict)
|
||||
|
||||
def __init__(self, store, session_options=None, hub_dry_run=False, ignore_eyes=False, timelines=None):
|
||||
super().__init__(store, hub, hub_dry_run)
|
||||
self.profileChanged.connect(self.profileRelay)
|
||||
self.sessionsChanged.connect(self.sessionsRelay)
|
||||
self.profileChanged.connect(self.boardChanged) # the board's done marks and needs
|
||||
self._session_options = session_options or {} # Session options: dry_run, speed, cams, button
|
||||
self._session_mod = None
|
||||
self._session_error = ""
|
||||
self._timelines = timelines or Timelines(store)
|
||||
self._ignore_eyes = ignore_eyes
|
||||
self._eyes = {}
|
||||
self._eyes_busy = False
|
||||
self._restarting = False
|
||||
self._board = {"state": "", "text": ""}
|
||||
self._board_busy = False
|
||||
self._eyesArrived.connect(self._on_eyes)
|
||||
self._boardArrived.connect(self._on_board)
|
||||
|
||||
# --- the session runner, imported when first needed
|
||||
def _runner(self):
|
||||
if self._session_mod is None and not self._session_error:
|
||||
try:
|
||||
import session as mod
|
||||
self._session_mod = mod
|
||||
except Exception as e: # missing, or broken: say so, the rest of the app still works
|
||||
self._session_error = f"The session runner (gaze/rec/session.py) can't be loaded: {e}"
|
||||
return self._session_mod
|
||||
|
||||
def _runner_error(self):
|
||||
self._runner()
|
||||
return self._session_error
|
||||
|
||||
def _texts_draft(self):
|
||||
return not hub.texts_ready()
|
||||
|
||||
def _upload_allowed(self):
|
||||
try:
|
||||
return hub.upload_allowed(hub.dataset_id())
|
||||
except hub.HubError:
|
||||
return False
|
||||
|
||||
def _upload_values(self, session, path):
|
||||
return {"ROUND": session}
|
||||
|
||||
@Property(str, constant=True)
|
||||
def toolVersion(self):
|
||||
return takes.tool_version()
|
||||
|
||||
# --- consent
|
||||
@Slot(bool, bool, bool)
|
||||
def acceptConsent(self, adult, region, agree):
|
||||
"""Write profile.json (recorder/window.py); its checklist and bounties stay."""
|
||||
optional = self.store.profile().get("optional")
|
||||
self._write_consent(adult, region, agree, optional if isinstance(optional, dict) else {})
|
||||
|
||||
# --- the bounty board
|
||||
def _checklist(self):
|
||||
c = self.store.profile().get("checklist")
|
||||
return c if isinstance(c, dict) else {}
|
||||
|
||||
def _board_now(self):
|
||||
mod = self._runner()
|
||||
return mod.load_board(self.store.base) if mod else None
|
||||
|
||||
@Property("QVariantList", notify=boardChanged)
|
||||
def bounties(self):
|
||||
"""The board's bounties (bounties.view), with "lastText" for the last round."""
|
||||
mod = self._runner()
|
||||
if not mod:
|
||||
return []
|
||||
rows = bounties.view(self._board_now(), self.store.profile(), self._checklist(), mod.have_round)
|
||||
for b in rows:
|
||||
b["lastText"] = round_label(b["last"]).split(" · ")[0] if b["last"] else ""
|
||||
b["minutesText"] = f"about {b['minutes']:g} min" if b["minutes"] else ""
|
||||
return rows
|
||||
|
||||
@Property(str, notify=boardChanged)
|
||||
def boardUpdated(self):
|
||||
board = self._board_now()
|
||||
return board["updated"] if board else ""
|
||||
|
||||
@Property("QVariantMap", notify=boardChanged)
|
||||
def boardFetch(self):
|
||||
"""The last fetch: {"state": ""|fetching|ok|error, "text"}."""
|
||||
return self._board
|
||||
|
||||
@Slot()
|
||||
def refreshBoard(self):
|
||||
"""hub.py board in a child process: the dataset's board into board-cache.json."""
|
||||
if self._board_busy or self._hub_dry_run:
|
||||
return
|
||||
self._board_busy = True
|
||||
self._board = {"state": "fetching", "text": "Getting the newest board"}
|
||||
self.boardChanged.emit()
|
||||
argv = self._hub_argv("board", "--json")
|
||||
|
||||
def run():
|
||||
try:
|
||||
r = subprocess.run(argv, capture_output=True, text=True, timeout=60)
|
||||
lines = [json.loads(x) for x in r.stdout.splitlines() if x.startswith("{")]
|
||||
result = lines[-1] if lines else {"error": {"text": (r.stderr.strip().splitlines() or ["hub.py failed"])[-1]}}
|
||||
except (OSError, ValueError, subprocess.TimeoutExpired) as e:
|
||||
result = {"error": {"text": f"The board didn't come: {e}"}}
|
||||
self._boardArrived.emit(result)
|
||||
self._thread(run)
|
||||
|
||||
def _on_board(self, result):
|
||||
self._board_busy = False
|
||||
if "done" in result:
|
||||
self._board = {"state": "ok", "text": f"The newest board, of {result['done'].get('updated', '?')}"}
|
||||
else:
|
||||
self._board = {"state": "error", "text": (result.get("error") or {}).get("text", "The board didn't come")
|
||||
+ " The board that came with the recorder is used."}
|
||||
self.boardChanged.emit()
|
||||
|
||||
@Slot(str, result=str)
|
||||
def planText(self, bounty_id):
|
||||
"""How long a bounty's round takes (session.plan_seconds), with the fit check before it."""
|
||||
mod = self._runner()
|
||||
b = next((b for b in self.bounties if b["id"] == bounty_id), None)
|
||||
if not mod or not b:
|
||||
return ""
|
||||
try:
|
||||
plan = mod.build_plan(mod.load_round(mod.round_path(b["script"])), 0, ipd_sweep=False)
|
||||
except (OSError, ValueError) as e:
|
||||
return f"Couldn't read the round: {e}"
|
||||
return (f"{len(plan)} steps, about {mod.plan_seconds(plan) / 60:.0f} min recorded, after the headset's fit "
|
||||
"check (and an IPD sweep on your first round, or when the IPD setting changed)")
|
||||
|
||||
# --- the checklist and health answers
|
||||
@Property("QVariantMap", notify=profileRelay)
|
||||
def checklist(self):
|
||||
c = self._checklist()
|
||||
return {"glasses": c.get("glasses") if c.get("glasses") in dict(GLASSES) else "none",
|
||||
"makeup": [m for m in c.get("makeup") or [] if m in dict(MAKEUP)],
|
||||
"pad": c.get("pad") if c.get("pad") in dict(PADS) else "",
|
||||
"strap": c.get("strap") if c.get("strap") in dict(STRAPS) else "",
|
||||
"ipd_mm": c.get("ipd_mm") if isinstance(c.get("ipd_mm"), (int, float)) else 0}
|
||||
|
||||
@Property("QVariantList", constant=True)
|
||||
def glassesChoices(self):
|
||||
return [{"value": k, "text": v} for k, v in GLASSES]
|
||||
|
||||
@Property("QVariantList", constant=True)
|
||||
def makeupChoices(self):
|
||||
return [{"value": k, "text": v} for k, v in MAKEUP]
|
||||
|
||||
@Property("QVariantList", constant=True)
|
||||
def padChoices(self):
|
||||
return [{"value": k, "text": v} for k, v in PADS]
|
||||
|
||||
@Property("QVariantList", constant=True)
|
||||
def strapChoices(self):
|
||||
return [{"value": k, "text": v} for k, v in STRAPS]
|
||||
|
||||
@Property("QVariantList", constant=True)
|
||||
def healthChoices(self):
|
||||
return [{"value": k, "text": v} for k, v in HEALTH]
|
||||
|
||||
@Slot("QVariantMap")
|
||||
def saveChecklist(self, answers):
|
||||
"""Keep the checklist in profile.json: only the known answers. ipd_mm: 0 when not known,
|
||||
else 50 to 80 mm."""
|
||||
a = dict(answers)
|
||||
try:
|
||||
ipd = round(float(a.get("ipd_mm") or 0), 1)
|
||||
except (TypeError, ValueError):
|
||||
ipd = 0.0
|
||||
checklist = {"glasses": a.get("glasses") if a.get("glasses") in dict(GLASSES) else "none",
|
||||
"makeup": sorted({m for m in a.get("makeup") or [] if m in dict(MAKEUP)}),
|
||||
"pad": a.get("pad") if a.get("pad") in dict(PADS) else "",
|
||||
"strap": a.get("strap") if a.get("strap") in dict(STRAPS) else "",
|
||||
"ipd_mm": ipd if 50 <= ipd <= 80 else None}
|
||||
profile = self.store.profile()
|
||||
if not profile or profile.get("checklist") == checklist:
|
||||
return
|
||||
profile["checklist"] = checklist
|
||||
write_json(self.store.profile_path, profile)
|
||||
self.profileChanged.emit()
|
||||
|
||||
def _health_path(self):
|
||||
return os.path.join(self.store.base, "health.json")
|
||||
|
||||
def _health(self):
|
||||
h = read_json(self._health_path())
|
||||
return h if h.get("opt_in") is True else {}
|
||||
|
||||
@Property(bool, notify=healthChanged)
|
||||
def healthOptIn(self):
|
||||
return bool(self._health())
|
||||
|
||||
@Property("QVariantList", notify=healthChanged)
|
||||
def healthAnswers(self):
|
||||
return [k for k in self._health().get("conditions") or [] if k in dict(HEALTH)]
|
||||
|
||||
@Slot(bool, "QVariantList")
|
||||
def saveHealth(self, opt_in, conditions):
|
||||
"""The health answers, in BASE/health.json (only this user can read it), never in
|
||||
profile.json; each round gets a copy, which its export encrypts. Opting out deletes them."""
|
||||
path = self._health_path()
|
||||
if not opt_in:
|
||||
try:
|
||||
os.remove(path)
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
else:
|
||||
data = {"schema": 1, "opt_in": True, "conditions": sorted({c for c in conditions if c in dict(HEALTH)})}
|
||||
if read_json(path) == data:
|
||||
return
|
||||
fd = os.open(path + ".tmp", os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
|
||||
with os.fdopen(fd, "w") as f:
|
||||
json.dump(data, f, indent=1)
|
||||
f.write("\n")
|
||||
os.replace(path + ".tmp", path)
|
||||
self.healthChanged.emit()
|
||||
|
||||
# --- the eye camera check
|
||||
@Property(str, notify=eyesChanged)
|
||||
def eyeState(self):
|
||||
"""ok, slow, none; checking while it runs; "" before the first check."""
|
||||
if self._eyes_busy and not self._eyes:
|
||||
return "checking"
|
||||
return self._eyes.get("state", "")
|
||||
|
||||
@Property(str, notify=eyesChanged)
|
||||
def eyeText(self):
|
||||
return self._eyes.get("text", "")
|
||||
|
||||
@Property(str, notify=eyesChanged)
|
||||
def eyeFpsText(self):
|
||||
fps = self._eyes.get("fps") or []
|
||||
return " · ".join(f"{name} {f:.0f} fps" for name, f in zip(("right", "left"), fps))
|
||||
|
||||
@Property(bool, notify=eyesChanged)
|
||||
def eyeBusy(self):
|
||||
return self._eyes_busy
|
||||
|
||||
@Property(bool, notify=eyesChanged)
|
||||
def restartingSteamVR(self):
|
||||
return self._restarting
|
||||
|
||||
@Property(bool, constant=True)
|
||||
def eyesIgnored(self):
|
||||
return self._ignore_eyes or bool(self._session_options.get("dry_run"))
|
||||
|
||||
@Property(bool, notify=eyesChanged)
|
||||
def eyesBlockStart(self):
|
||||
return not self.eyesIgnored and self._eyes.get("state") != "ok"
|
||||
|
||||
def _check_now(self):
|
||||
if self.eyesIgnored:
|
||||
return {"state": "ok", "fps": [], "text": ""}
|
||||
try:
|
||||
return eye_check(self._session_options.get("cams"))
|
||||
except Exception as e:
|
||||
return {"state": "none", "fps": [], "text": f"The eye camera check failed: {e}"}
|
||||
|
||||
@Slot()
|
||||
def checkEyes(self):
|
||||
if self._eyes_busy or self.sessionActive:
|
||||
return
|
||||
self._eyes_busy = True
|
||||
self.eyesChanged.emit()
|
||||
self._thread(lambda: self._eyesArrived.emit(self._check_now()))
|
||||
|
||||
def _on_eyes(self, result):
|
||||
self._eyes = result
|
||||
self._eyes_busy = self._restarting
|
||||
self.eyesChanged.emit()
|
||||
|
||||
@Slot(bool)
|
||||
def restartSteamVR(self, fast):
|
||||
"""fast: first turn on SteamVR's 90 fps eye cameras (gaze/eye-rate.py on, on the host).
|
||||
Then systemctl --user restart --no-block steamvr.service (the job runs in systemd, so it
|
||||
finishes even when this window closes with the Frametop desktop), and the eye check every
|
||||
few seconds until the frames come at 90."""
|
||||
if self._restarting or self.sessionActive:
|
||||
return
|
||||
self._restarting = self._eyes_busy = True
|
||||
self.eyesChanged.emit()
|
||||
|
||||
def run():
|
||||
result = None
|
||||
try:
|
||||
if fast:
|
||||
r = subprocess.run(host_command("python3", EYE_RATE, "on"), capture_output=True, text=True,
|
||||
timeout=60)
|
||||
if r.returncode != 0:
|
||||
raise RuntimeError((r.stderr or r.stdout).strip() or "eye-rate.py: exit %d" % r.returncode)
|
||||
r = subprocess.run(host_command("systemctl", "--user", "restart", "--no-block", "steamvr.service"),
|
||||
capture_output=True, text=True, timeout=60)
|
||||
if r.returncode != 0:
|
||||
raise RuntimeError((r.stderr or r.stdout).strip() or "exit %d" % r.returncode)
|
||||
end = time.monotonic() + RECHECK_FOR_S
|
||||
time.sleep(RECHECK_S * 3)
|
||||
while time.monotonic() < end:
|
||||
result = self._check_now()
|
||||
if result.get("state") == "ok":
|
||||
break
|
||||
time.sleep(RECHECK_S)
|
||||
except Exception as e:
|
||||
result = {"state": "none", "fps": [], "text": f"Couldn't restart SteamVR: {e}"}
|
||||
self._restarting = False
|
||||
self._eyesArrived.emit(result or self._check_now())
|
||||
self._thread(run)
|
||||
|
||||
# --- the round
|
||||
@Slot(str, result=bool)
|
||||
def startSession(self, bounty_id):
|
||||
if self.sessionActive:
|
||||
return False
|
||||
mod = self._runner()
|
||||
if not mod:
|
||||
self.message.emit(self._session_error, True)
|
||||
return False
|
||||
if self.needsConsent:
|
||||
self.message.emit("Agree to the consent text first (Welcome page)", True)
|
||||
return False
|
||||
entry = next((b for b in self.bounties if b["id"] == bounty_id), None)
|
||||
if not entry:
|
||||
self.message.emit("Pick a bounty first", True)
|
||||
return False
|
||||
if entry["available"]:
|
||||
self.message.emit(entry["available"], True)
|
||||
return False
|
||||
if not self.eyesIgnored:
|
||||
self._eyes = self._check_now() # fresh: it takes two seconds
|
||||
self.eyesChanged.emit()
|
||||
if self._eyes.get("state") != "ok":
|
||||
self.message.emit(self._eyes.get("text", "The eye cameras aren't ready"), True)
|
||||
return False
|
||||
board = self._board_now() or {}
|
||||
try:
|
||||
self._session = mod.Session(self.store.base, entry["script"], bounty=dict(entry, updated=board.get("updated")),
|
||||
profile=self.store.profile(), checklist=self._checklist(), health=self._health(),
|
||||
on_status=lambda s: self._statusArrived.emit(dict(s)),
|
||||
profile_path=self.store.profile_path, **self._session_options)
|
||||
except Exception as e:
|
||||
self.message.emit(f"Couldn't set up the round: {e}", True)
|
||||
return False
|
||||
self._session_id = ""
|
||||
self._status = {"state": "starting", "bounty": bounty_id}
|
||||
self.statusChanged.emit()
|
||||
session = self._session
|
||||
|
||||
def run():
|
||||
try:
|
||||
session.start()
|
||||
except Exception as e:
|
||||
self._statusArrived.emit({"state": "error", "error": str(e)})
|
||||
self._thread(run)
|
||||
return True
|
||||
|
||||
def _on_status(self, status):
|
||||
super()._on_status(status)
|
||||
if status.get("state") == "done":
|
||||
self.profileChanged.emit() # the bounty is marked done, and the IPD kept
|
||||
|
||||
# --- review
|
||||
def _session_rows(self):
|
||||
out = []
|
||||
for r in self.store.rounds():
|
||||
r["label"] = round_label(r["id"], r["title"])
|
||||
r["sizeText"] = human_bytes(r["bytes"])
|
||||
r["exportText"] = human_bytes(r["export_bytes"]) if r["exported"] else ""
|
||||
r["active"] = self.sessionActive and r["id"] == self._session_id
|
||||
r["statusText"] = "" if r["active"] else STATUS_TEXT.get(r["status"], "")
|
||||
out.append(r)
|
||||
return out
|
||||
|
||||
@Slot(str, result="QVariantList")
|
||||
def takeList(self, rnd):
|
||||
try:
|
||||
rows = self.store.takes(rnd)
|
||||
except ValueError:
|
||||
return []
|
||||
for t in rows:
|
||||
t["title"] = take_title(t["group"])
|
||||
t["durationText"] = f"{int(t['duration_s'] // 60)}:{int(t['duration_s'] % 60):02d}"
|
||||
t["sizeText"] = human_bytes(t["bytes"])
|
||||
# Per eye, as the viewer steps (the camera with more frames)
|
||||
t["frameCount"] = max(t["frames"], default=0)
|
||||
t["deletedCount"] = max(t["deleted_frames"], default=0)
|
||||
t.pop("ranges")
|
||||
return rows
|
||||
|
||||
@Slot(str, str, result="QVariantMap")
|
||||
def takeInfo(self, rnd, take):
|
||||
"""For the viewer: {count, title, ranges: [[first, last] timeline indexes]}."""
|
||||
try:
|
||||
tl = self._timelines.get(rnd, take)
|
||||
ranges = self.store.ranges(rnd, take)
|
||||
meta = self.store.take_meta(rnd, take)
|
||||
except (OSError, ValueError):
|
||||
return {"count": 0, "title": take, "ranges": []}
|
||||
marks = []
|
||||
monos = [tl.mono_ns(t) for t in tl.times[tl.main]]
|
||||
for a, b in ranges:
|
||||
first, last = bisect.bisect_left(monos, a), bisect.bisect_right(monos, b) - 1
|
||||
marks.append([first, last] if first <= last else [-1, -1])
|
||||
return {"count": len(tl), "title": take_title(meta.get("group", take.split("-", 1)[-1])), "ranges": marks}
|
||||
|
||||
@Slot(str, str, int, result=str)
|
||||
def frameTime(self, rnd, take, i):
|
||||
"""Frame i's time from the take's first, m:ss.s."""
|
||||
try:
|
||||
tl = self._timelines.get(rnd, take)
|
||||
return clock_text(tl.time(i) - tl.time(0))
|
||||
except (OSError, ValueError, IndexError):
|
||||
return ""
|
||||
|
||||
@Slot(str, str, int, int)
|
||||
def deleteRange(self, rnd, take, first, last):
|
||||
"""Leave frames first..last (timeline indexes, either order) out of the export: both
|
||||
cameras', by time."""
|
||||
if self._active_guard(rnd):
|
||||
return
|
||||
tl = self._timelines.get(rnd, take)
|
||||
first, last = sorted((max(0, first), min(len(tl) - 1, last)))
|
||||
if first > last:
|
||||
return
|
||||
# Half a millisecond either side: index.tsv keeps whole microseconds, and the clocks'
|
||||
# conversion rounds; the frames are 11 ms apart.
|
||||
self.store.delete_range(rnd, take, tl.mono_ns(tl.time(first)) - 500000, tl.mono_ns(tl.time(last)) + 500000)
|
||||
self.message.emit(f"Frames {first + 1} to {last + 1} won't be exported", False)
|
||||
self.sessionsChanged.emit()
|
||||
|
||||
@Slot(str, str, int)
|
||||
def restoreRange(self, rnd, take, k):
|
||||
self.store.restore_range(rnd, take, k)
|
||||
self.message.emit("Range restored", False)
|
||||
self.sessionsChanged.emit()
|
||||
|
||||
@Slot(str, str)
|
||||
def deleteTake(self, rnd, take):
|
||||
if self._active_guard(rnd):
|
||||
return
|
||||
try:
|
||||
self.store.delete_take(rnd, take)
|
||||
self.message.emit(f"Deleted {take}", False)
|
||||
except (OSError, ValueError) as e:
|
||||
self.message.emit(f"Couldn't delete {take}: {e}", True)
|
||||
self.refreshSessions()
|
||||
|
||||
@Slot(str)
|
||||
def deleteSession(self, rnd):
|
||||
if self._active_guard(rnd):
|
||||
return
|
||||
try:
|
||||
self.store.delete_round(rnd)
|
||||
self.message.emit(f"Deleted the round of {round_label(rnd)}", False)
|
||||
except (OSError, ValueError) as e:
|
||||
self.message.emit(f"Couldn't delete the round: {e}", True)
|
||||
self.refreshSessions()
|
||||
|
||||
# --- export
|
||||
@Property(str, constant=True)
|
||||
def exportProblem(self):
|
||||
"""Why nothing can be exported here, or ""."""
|
||||
if eyerec.compress is None:
|
||||
return "zstd isn't installed in the dev container, so nothing can be exported. Run setup/dev-container.sh."
|
||||
try:
|
||||
import cryptography # noqa: F401 (recorder/age.py's)
|
||||
except ImportError:
|
||||
return ("The cryptography library isn't installed in the dev container, so nothing can be encrypted "
|
||||
"for export. Run setup/dev-container.sh.")
|
||||
return ""
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description="Frametop Eye Recorder")
|
||||
ap.add_argument("--base", default=takes.DEFAULT_BASE, help="where profile.json, rounds/ and exports/ go")
|
||||
ap.add_argument("--page", default="",
|
||||
help="open on this page: welcome, bounties, checklist, session, review, export, upload")
|
||||
ap.add_argument("--dry-run", action="store_true",
|
||||
help="test: rounds start no processes, make up the panel's replies and print its commands")
|
||||
ap.add_argument("--speed", type=float, default=1.0, help="test, with --dry-run: run rounds this much faster")
|
||||
ap.add_argument("--cams", help="test: read eye frames from this share (lab/ft-eyes-replay's)")
|
||||
ap.add_argument("--no-headset-button", action="store_true",
|
||||
help="rounds don't read the headset's button (Next, pause, resume)")
|
||||
ap.add_argument("--hub-dry-run", action="store_true",
|
||||
help="test: Upload checks the export and says what it would send, with no network calls")
|
||||
ap.add_argument("--ignore-eyes", action="store_true",
|
||||
help="start rounds even if the eye camera check finds no frames, or slow ones")
|
||||
ap.add_argument("--qml", default=os.path.join(HERE, "main.qml"),
|
||||
help="the window (default: this folder's main.qml, which needs Kirigami)")
|
||||
ap.add_argument("--style", default="org.kde.desktop", help="the Qt Quick Controls style")
|
||||
a, qt_args = ap.parse_known_args()
|
||||
app = QGuiApplication([sys.argv[0]] + qt_args)
|
||||
app.setApplicationName("ft-eyerec")
|
||||
app.setApplicationDisplayName("Frametop Eye Recorder")
|
||||
app.setDesktopFileName("ft-eyerec")
|
||||
if not QIcon.themeName():
|
||||
QIcon.setThemeName("breeze")
|
||||
QQuickStyle.setStyle(a.style)
|
||||
store = takes.Store(a.base)
|
||||
timelines = Timelines(store)
|
||||
engine = QQmlApplicationEngine()
|
||||
engine.addImageProvider("eyes", EyeProvider(timelines))
|
||||
options = {"dry_run": True, "speed": a.speed} if a.dry_run else {}
|
||||
if a.cams:
|
||||
options["cams"] = a.cams
|
||||
if a.no_headset_button or a.dry_run:
|
||||
options["button"] = False
|
||||
backend = Backend(store, options, hub_dry_run=a.hub_dry_run, ignore_eyes=a.ignore_eyes, timelines=timelines)
|
||||
app.aboutToQuit.connect(backend.shutdown)
|
||||
mod = None if a.dry_run else backend._runner()
|
||||
if mod: # Frametop's gaze service, if a round that was killed left it paused
|
||||
backend._thread(lambda: mod.resume_units(os.path.join(mod.run_dir(), mod.PAUSED_UNITS_FILE)))
|
||||
engine.rootContext().setContextProperty("backend", backend)
|
||||
engine.rootContext().setContextProperty("startPage", a.page)
|
||||
engine.load(QUrl.fromLocalFile(os.path.abspath(a.qml)))
|
||||
if not engine.rootObjects():
|
||||
sys.exit(1)
|
||||
|
||||
# SIGTERM and Ctrl+C quit as closing does, so a running round still stops cleanly. Python
|
||||
# runs signal handlers between bytecodes: the timer gives it some while Qt waits.
|
||||
def on_signal(*_):
|
||||
engine.rootObjects()[0].setProperty("quitting", True)
|
||||
app.quit()
|
||||
signal.signal(signal.SIGTERM, on_signal)
|
||||
signal.signal(signal.SIGINT, on_signal)
|
||||
tick = QTimer(interval=500, timeout=lambda: None)
|
||||
tick.start()
|
||||
code = app.exec()
|
||||
# The window goes before the backend (as the hand recorder: bindings would run against it).
|
||||
shiboken6.delete(engine)
|
||||
sys.exit(code)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+276
@@ -0,0 +1,276 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Upload an eye recorder export to the eye dataset on Hugging Face.
|
||||
|
||||
As the hand recorder's hub.py (the shared part is recorder/hub.py): the window runs it as a child
|
||||
process, it uses huggingface_hub's saved login (`hub.py login` is the browser's device code), and
|
||||
nobody types a token anywhere. An upload:
|
||||
1. checks the export with validate.py (against recipients.txt's keys), and stops on errors;
|
||||
2. stops if this export (same SHA256SUMS) was uploaded before, unless --again;
|
||||
3. stops unless CONSENT.md and UPLOAD.md are here and neither is a draft. FT_EYEREC_ALLOW_UPLOAD=1
|
||||
opens it for a rehearsal, and only to another dataset than the real one (FT_EYEREC_DATASET):
|
||||
the real one takes nothing before the texts are reviewed;
|
||||
4. checks the login and access to the dataset, opens the pull request first, uploads the folder
|
||||
to contributions/<contributor>/<round> in it, and records it under "uploads" in session.json.
|
||||
--dry-run does all of it except the network calls and the records, and says what it would upload.
|
||||
|
||||
The images and health answers are encrypted already (takes.py's export): nothing here can read
|
||||
them, and the dataset's readers need one of its keys.
|
||||
|
||||
`hub.py board` fetches the bounty board (bounties.json) from the dataset into BASE/board-cache.json,
|
||||
with the saved login when there is one (the dataset is gated): the window's board then uses it
|
||||
when it's newer than the one shipped here (recorder/bounties.py checks what it may say).
|
||||
|
||||
usage: hub.py [--base DIR] whoami [--json]
|
||||
hub.py [--base DIR] login [--json]
|
||||
hub.py [--base DIR] board [--json]
|
||||
hub.py [--base DIR] upload ROUND [--dry-run] [--again] [--json]
|
||||
With --json, each line of output is one JSON object, as the hand recorder's hub.py.
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
sys.path.insert(0, HERE)
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(HERE)))
|
||||
import takes # noqa: E402 (next to this file)
|
||||
from recorder import bounties # noqa: E402
|
||||
from recorder import hub as rhub # noqa: E402
|
||||
from recorder.common import write_json # noqa: E402
|
||||
from recorder.hub import HubError, export_sha, is_draft, now_iso # noqa: E402,F401
|
||||
|
||||
HF_DATASET = "DeeJanuz/frametop-eyes"
|
||||
DATASET_ENV = "FT_EYEREC_DATASET"
|
||||
ALLOW_ENV = "FT_EYEREC_ALLOW_UPLOAD"
|
||||
CONSENT_PATH = os.path.join(HERE, "CONSENT.md")
|
||||
UPLOAD_PATH = os.path.join(HERE, "UPLOAD.md")
|
||||
|
||||
|
||||
def dataset_id():
|
||||
"""HF_DATASET, or FT_EYEREC_DATASET when set."""
|
||||
return rhub.dataset_id(DATASET_ENV, HF_DATASET)
|
||||
|
||||
|
||||
def texts_ready():
|
||||
return all(os.path.isfile(p) and not is_draft(p) for p in (CONSENT_PATH, UPLOAD_PATH))
|
||||
|
||||
|
||||
def upload_allowed(repo):
|
||||
"""Real uploads: once the texts are reviewed, or for a rehearsal against another dataset."""
|
||||
return texts_ready() or (os.environ.get(ALLOW_ENV) == "1" and repo != HF_DATASET)
|
||||
|
||||
|
||||
def meta_path(store, rnd):
|
||||
return os.path.join(store.round_dir(rnd), "session.json")
|
||||
|
||||
|
||||
def describe(summary, report, sha, nbytes):
|
||||
"""(commit message, commit description) from validate's summary of the manifest."""
|
||||
s = summary
|
||||
message = f"Eyes: {s.get('bounty')} round {s.get('round')} from {s.get('contributor')}"
|
||||
lines = ["Contribution to the Frametop eye dataset, uploaded from the eye recorder. The eye images "
|
||||
"and any health answers are encrypted to the dataset's keys.", "",
|
||||
f"- Round: {s.get('round')}",
|
||||
f"- Bounty: {s.get('bounty')}",
|
||||
f"- Contributor: {s.get('contributor')}",
|
||||
f"- Takes: {s.get('takes')} ({s.get('minutes')} minutes, {s.get('frames')} frames)",
|
||||
f"- Glasses: {s.get('glasses') or 'not given'}",
|
||||
f"- Health answers: {'included, encrypted' if s.get('health') else 'none'}",
|
||||
f"- Consent version: {s.get('consent_version')}",
|
||||
f"- Tool: {s.get('tool')}",
|
||||
f"- Size: {takes.human_bytes(nbytes)}",
|
||||
f"- SHA256 of SHA256SUMS: {sha}", "",
|
||||
"Checked with gaze/rec/validate.py: no errors" + (f", {len(report.warnings)} warnings:"
|
||||
if report.warnings else ".")]
|
||||
lines += [f"- {w}" for w in report.warnings]
|
||||
return message, "\n".join(lines) + "\n"
|
||||
|
||||
|
||||
def _hf():
|
||||
return rhub.hf_module("run setup/dev-container.sh (or: pip install --user huggingface_hub)")
|
||||
|
||||
|
||||
def whoami():
|
||||
return rhub.whoami(_hf(), dataset_id())
|
||||
|
||||
|
||||
def login(progress=None):
|
||||
_hf()
|
||||
return rhub.login(dataset_id(), progress, lambda: whoami(), "update the dev container (setup/dev-container.sh).")
|
||||
|
||||
|
||||
def board_url(repo):
|
||||
return f"{rhub.dataset_url(repo)}/resolve/main/bounties.json"
|
||||
|
||||
|
||||
def fetch_board(base):
|
||||
"""The dataset's bounties.json into base/board-cache.json (session.load_board reads it).
|
||||
Returns {"updated", "bounties"}; raises HubError."""
|
||||
repo = dataset_id()
|
||||
token = None
|
||||
try:
|
||||
token = _hf().get_token()
|
||||
except (HubError, AttributeError):
|
||||
pass # no huggingface_hub, or an old one: try without a login
|
||||
try:
|
||||
board = bounties.fetch(board_url(repo), os.path.join(base, "board-cache.json"), token)
|
||||
except ValueError as e:
|
||||
raise HubError("hub", f"The dataset's bounty board isn't valid: {e}") from None
|
||||
except OSError as e:
|
||||
status = getattr(e, "code", None)
|
||||
if status in (401, 403):
|
||||
raise HubError("terms", "Log in and accept the dataset's terms to get its newest bounty board.",
|
||||
rhub.dataset_url(repo)) from None
|
||||
if status == 404:
|
||||
raise HubError("not-found", f"{repo} has no bounty board yet.") from None
|
||||
raise HubError("network", f"Couldn't fetch the bounty board ({type(e).__name__}: {e}).") from None
|
||||
return {"updated": board["updated"], "bounties": len(board["bounties"])}
|
||||
|
||||
|
||||
def upload(store, rnd, dry_run=False, again=False, progress=None, log=None):
|
||||
"""Upload exports/<round> (the steps in this file's docstring). Returns the result; raises HubError."""
|
||||
tell = progress or (lambda phase, text, fraction=None, **extra: None)
|
||||
say = log or (lambda text: None)
|
||||
import validate
|
||||
repo = dataset_id()
|
||||
try:
|
||||
export = store.export_dir(rnd)
|
||||
except ValueError as e:
|
||||
raise HubError("no-export", str(e)) from None
|
||||
if not os.path.isfile(os.path.join(export, "manifest.json")):
|
||||
raise HubError("no-export", f"No export of round {rnd}: export it first")
|
||||
|
||||
tell("check", "Checking the export", 0.0)
|
||||
report = validate.validate(export, progress=lambda f, text: tell("check", text, f))
|
||||
if not report.ok:
|
||||
raise HubError("invalid", f"The export has {len(report.errors)} problems, so it can't be uploaded. Export "
|
||||
"the round again; if that doesn't help, report it.", errors=report.errors)
|
||||
summary = report.summary
|
||||
sha = export_sha(export)
|
||||
before = rhub.previous_upload(meta_path(store, rnd), sha)
|
||||
if before and not again:
|
||||
raise HubError("duplicate", f"This export was uploaded already, on {before.get('uploaded', '?')}: "
|
||||
f"{before.get('pr_url', '')}", before.get("pr_url", ""), extra={"previous": before})
|
||||
if not dry_run and not upload_allowed(repo):
|
||||
raise HubError("closed", "Eye contributions aren't open yet: the consent and upload texts are waiting for a "
|
||||
f"legal review. (A rehearsal: {ALLOW_ENV}=1 with {DATASET_ENV} set to a test dataset.)")
|
||||
path_in_repo = f"contributions/{summary.get('contributor', '')}/{rnd}"
|
||||
message, description = describe(summary, report, sha, report.bytes)
|
||||
files = []
|
||||
for root, _, names in os.walk(export):
|
||||
files += [os.path.relpath(os.path.join(root, n), export) for n in names]
|
||||
plan = {"repo": repo, "repo_type": "dataset", "folder_path": export, "path_in_repo": path_in_repo,
|
||||
"create_pr": True, "commit_message": message, "commit_description": description,
|
||||
"files": len(files), "bytes": report.bytes, "warnings": report.warnings}
|
||||
|
||||
if dry_run:
|
||||
say(f"dry run: would check the login (whoami) and access to {repo} (auth_check)")
|
||||
say(f"dry run: would upload_folder {len(files)} files, {takes.human_bytes(report.bytes)}, "
|
||||
f"from {export} to datasets/{repo}/{path_in_repo}, as a pull request")
|
||||
for rel in sorted(files):
|
||||
say(f" {rel} {takes.human_bytes(os.path.getsize(os.path.join(export, rel)))}")
|
||||
say(f"dry run: commit message: {message}")
|
||||
say("dry run: commit description:\n" + description.rstrip())
|
||||
record = {"repo": repo, "pr_url": "", "uploaded": now_iso(), "export_sha": sha, "status": "done"}
|
||||
say(f"dry run: would record in session.json's uploads: {json.dumps(record)}")
|
||||
tell("done", "Dry run: nothing was uploaded", 1.0)
|
||||
return dict(plan, dry_run=True, pr_url="", record=record)
|
||||
|
||||
pr, who, record = rhub.push(_hf(), repo, export, path_in_repo, message, description, meta_path(store, rnd), sha,
|
||||
len(files), report.bytes, tell, write_json, takes.human_bytes)
|
||||
tell("done", "Uploaded", 1.0)
|
||||
return dict(plan, dry_run=False, pr_url=record["pr_url"], pr_num=pr.num, user=who.get("name", ""), record=record)
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description="Upload an eye recorder export to the eye dataset on Hugging Face.")
|
||||
ap.add_argument("--base", default=takes.DEFAULT_BASE)
|
||||
sub = ap.add_subparsers(dest="cmd", required=True)
|
||||
w = sub.add_parser("whoami", help="show the saved Hugging Face login")
|
||||
w.add_argument("--json", action="store_true")
|
||||
li = sub.add_parser("login", help="log in to Hugging Face in the browser")
|
||||
li.add_argument("--json", action="store_true")
|
||||
b = sub.add_parser("board", help="fetch the dataset's bounty board")
|
||||
b.add_argument("--json", action="store_true")
|
||||
u = sub.add_parser("upload", help="upload exports/ROUND as a pull request")
|
||||
u.add_argument("round")
|
||||
u.add_argument("--dry-run", action="store_true", help="no network: say what would be uploaded")
|
||||
u.add_argument("--again", action="store_true", help="upload even if this export was uploaded before")
|
||||
u.add_argument("--json", action="store_true", help="JSON lines, for the window")
|
||||
a = ap.parse_args()
|
||||
|
||||
def emit(obj):
|
||||
print(json.dumps(obj), flush=True)
|
||||
|
||||
try:
|
||||
if a.cmd == "whoami":
|
||||
who = whoami()
|
||||
if a.json:
|
||||
emit({"done": who})
|
||||
else:
|
||||
print(f"logged in as {who['name']} (token role: {who['role'] or 'unknown'})")
|
||||
return 0
|
||||
if a.cmd == "board":
|
||||
got = fetch_board(os.path.abspath(os.path.expanduser(a.base)))
|
||||
if a.json:
|
||||
emit({"done": got})
|
||||
else:
|
||||
print(f"bounty board of {got['updated']}: {got['bounties']} bounties")
|
||||
return 0
|
||||
if a.cmd == "login":
|
||||
def code(phase, text, **extra):
|
||||
if a.json:
|
||||
emit(dict(extra, phase=phase, text=text))
|
||||
else:
|
||||
print(f"Open {extra['url']} and approve the code {extra['code']}. Waiting...", flush=True)
|
||||
who = login(code)
|
||||
if a.json:
|
||||
emit({"done": who})
|
||||
else:
|
||||
print(f"logged in as {who['name']}")
|
||||
return 0
|
||||
if a.json:
|
||||
def progress(phase, text, fraction=None, **extra):
|
||||
emit(dict(extra, phase=phase, text=text, fraction=fraction))
|
||||
|
||||
def log(text):
|
||||
emit({"log": text})
|
||||
else:
|
||||
last = {}
|
||||
|
||||
def progress(phase, text, fraction=None, **extra):
|
||||
if extra.get("pr_url"):
|
||||
print(f"pull request: {extra['pr_url']}", flush=True)
|
||||
line = text if fraction is None else f"{fraction * 100:5.1f}% {text}"
|
||||
if (phase, text) != last.get("key") or fraction in (0.0, 1.0):
|
||||
print(line, file=sys.stderr, flush=True)
|
||||
last["key"] = (phase, text)
|
||||
|
||||
def log(text):
|
||||
print(text, flush=True)
|
||||
if hasattr(os, "setpriority"):
|
||||
try:
|
||||
os.setpriority(os.PRIO_PROCESS, 0, 19) # validation reads and hashes everything: stay out of VR's way
|
||||
except OSError:
|
||||
pass
|
||||
result = upload(takes.Store(a.base), a.round, dry_run=a.dry_run, again=a.again, progress=progress, log=log)
|
||||
if a.json:
|
||||
emit({"done": result})
|
||||
elif result["dry_run"]:
|
||||
print(f"dry run done: {result['files']} files would go to datasets/{result['repo']}/{result['path_in_repo']}")
|
||||
else:
|
||||
print(f"uploaded: {result['pr_url']}")
|
||||
return 0
|
||||
except HubError as e:
|
||||
if a.json:
|
||||
emit({"error": e.as_dict()})
|
||||
else:
|
||||
print(f"error ({e.kind}): {e.text}", file=sys.stderr)
|
||||
for line in e.errors:
|
||||
print(f" {line}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Executable
+36
@@ -0,0 +1,36 @@
|
||||
#!/usr/bin/env bash
|
||||
# Install the eye recorder on a Frame that has Frametop (get.sh --experimental): bring the dev
|
||||
# container's packages up to date, build the gaze tools the round uses (gaze/build.sh: ft-gaze and
|
||||
# the headset panel ft-gazepanel; gaze/tracker/build.sh: our tracker ft-eyes, which the fit coach
|
||||
# runs) and add "Frametop Eye Recorder" to the app menu (for Frametop's desktop, as the hand
|
||||
# recorder's). It doesn't change Frametop's gaze setup; the window offers SteamVR's 90 fps eye
|
||||
# cameras (gaze/eye-rate.py) when they're slower.
|
||||
# Usage: gaze/rec/install.sh install or update
|
||||
# gaze/rec/install.sh uninstall remove the menu entry. Recordings stay where they are.
|
||||
set -euo pipefail
|
||||
root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)
|
||||
. "$root/scripts/_env.sh"
|
||||
entry='~/.local/share/applications/frametop-eyerec.desktop'
|
||||
|
||||
case ${1:-install} in
|
||||
install)
|
||||
echo "== 1/3 dev container packages"
|
||||
"$root/setup/dev-container.sh"
|
||||
echo "== 2/3 the gaze tools: ft-gaze, ft-gazepanel and ft-eyes"
|
||||
"$root/gaze/build.sh"
|
||||
"$root/gaze/tracker/build.sh"
|
||||
echo "== 3/3 the menu entry"
|
||||
fill_template "$root/gaze/rec/ft-eyerec.desktop" |
|
||||
on_frame "mkdir -p ~/.local/share/applications && cat > $entry"
|
||||
echo
|
||||
echo "Installed. On Frametop's desktop, open \"Frametop Eye Recorder\" from the app menu."
|
||||
echo "Recordings go to ~/.local/share/frametop/eyes/contrib."
|
||||
;;
|
||||
uninstall)
|
||||
on_frame "rm -f $entry"
|
||||
echo "Removed the menu entry."
|
||||
echo "Your recordings are still in ~/.local/share/frametop/eyes/contrib:"
|
||||
echo "delete that folder to remove them."
|
||||
;;
|
||||
*) echo "usage: $0 [install|uninstall]" >&2; exit 2 ;;
|
||||
esac
|
||||
+1805
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,4 @@
|
||||
# The eye dataset's recipients (gaze/rec/DESIGN.md "Files"): exports encrypt each chunk and health.json
|
||||
# to all of them (recorder/age.py). Key 1 is kept on the maintainer's training machine, key 2 offline.
|
||||
age1a0tc98pj2s7gdfgrv4gam6ah0qv69ymf3t2cc25hcwepmats69hst35z2e
|
||||
age1r0zlq7kcyg4dz4y3vkl9j36jw3zc6ly6reddh9ag5kfy6aw0fy6s5hepat
|
||||
@@ -0,0 +1,233 @@
|
||||
{
|
||||
"schema": 1,
|
||||
"id": "blinks",
|
||||
"title": "Blinks and lids",
|
||||
"intro": "Blinks, closed and half-closed eyes. Keep looking at the dot whenever your eyes are open.",
|
||||
"steps": [
|
||||
{
|
||||
"id": "blink-cue",
|
||||
"kind": "prompt",
|
||||
"text": "Look at the dot and blink each time the word says so.",
|
||||
"dot": [
|
||||
0,
|
||||
0
|
||||
],
|
||||
"cues": [
|
||||
[
|
||||
"Blink",
|
||||
1.5
|
||||
],
|
||||
[
|
||||
"Blink",
|
||||
1.5
|
||||
],
|
||||
[
|
||||
"Blink",
|
||||
1.5
|
||||
],
|
||||
[
|
||||
"Blink",
|
||||
1.5
|
||||
],
|
||||
[
|
||||
"Blink",
|
||||
1.5
|
||||
],
|
||||
[
|
||||
"Blink",
|
||||
1.5
|
||||
],
|
||||
[
|
||||
"Blink",
|
||||
1.5
|
||||
],
|
||||
[
|
||||
"Blink",
|
||||
1.5
|
||||
]
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "closed",
|
||||
"kind": "prompt",
|
||||
"text": "Look at the dot, then close both eyes when it says so.",
|
||||
"dot": [
|
||||
0,
|
||||
0
|
||||
],
|
||||
"cues": [
|
||||
[
|
||||
"Look",
|
||||
2
|
||||
],
|
||||
[
|
||||
"Close both eyes",
|
||||
2.5
|
||||
],
|
||||
[
|
||||
"Open",
|
||||
2
|
||||
],
|
||||
[
|
||||
"Close both eyes",
|
||||
2.5
|
||||
],
|
||||
[
|
||||
"Open",
|
||||
2
|
||||
]
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "winks",
|
||||
"kind": "prompt",
|
||||
"text": "Wink each eye in turn when it says so (as well as you can).",
|
||||
"dot": [
|
||||
0,
|
||||
0
|
||||
],
|
||||
"cues": [
|
||||
[
|
||||
"Look",
|
||||
1.5
|
||||
],
|
||||
[
|
||||
"Close your left eye",
|
||||
2
|
||||
],
|
||||
[
|
||||
"Open",
|
||||
1.5
|
||||
],
|
||||
[
|
||||
"Close your right eye",
|
||||
2
|
||||
],
|
||||
[
|
||||
"Open",
|
||||
1.5
|
||||
],
|
||||
[
|
||||
"Close your left eye",
|
||||
2
|
||||
],
|
||||
[
|
||||
"Open",
|
||||
1.5
|
||||
],
|
||||
[
|
||||
"Close your right eye",
|
||||
2
|
||||
],
|
||||
[
|
||||
"Open",
|
||||
1.5
|
||||
]
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "squint",
|
||||
"kind": "prompt",
|
||||
"text": "Keep looking at the dot while you change how open your eyes are.",
|
||||
"dot": [
|
||||
0,
|
||||
0
|
||||
],
|
||||
"cues": [
|
||||
[
|
||||
"Normal",
|
||||
2
|
||||
],
|
||||
[
|
||||
"Squint",
|
||||
3
|
||||
],
|
||||
[
|
||||
"Normal",
|
||||
2
|
||||
],
|
||||
[
|
||||
"Eyes wide open",
|
||||
3
|
||||
],
|
||||
[
|
||||
"Normal",
|
||||
2
|
||||
],
|
||||
[
|
||||
"Half closed",
|
||||
3
|
||||
],
|
||||
[
|
||||
"Normal",
|
||||
2
|
||||
]
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "lids-down",
|
||||
"kind": "dots",
|
||||
"text": "Look down at each dot. Let your lids drop as they do when you look down.",
|
||||
"dots": [
|
||||
[
|
||||
0,
|
||||
-12
|
||||
],
|
||||
[
|
||||
0,
|
||||
-18
|
||||
],
|
||||
[
|
||||
12,
|
||||
-18
|
||||
],
|
||||
[
|
||||
-12,
|
||||
-18
|
||||
],
|
||||
[
|
||||
0,
|
||||
-23
|
||||
],
|
||||
[
|
||||
18,
|
||||
-20
|
||||
],
|
||||
[
|
||||
-18,
|
||||
-20
|
||||
]
|
||||
],
|
||||
"hold": 1.5,
|
||||
"shuffle": true
|
||||
},
|
||||
{
|
||||
"id": "blink-dots",
|
||||
"kind": "dots",
|
||||
"text": "Look at each dot and blink once while its ring fills.",
|
||||
"dots": [
|
||||
[
|
||||
0,
|
||||
0
|
||||
],
|
||||
[
|
||||
15,
|
||||
10
|
||||
],
|
||||
[
|
||||
-15,
|
||||
10
|
||||
],
|
||||
[
|
||||
15,
|
||||
-10
|
||||
],
|
||||
[
|
||||
-15,
|
||||
-10
|
||||
]
|
||||
],
|
||||
"hold": 2.0
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,89 @@
|
||||
{
|
||||
"schema": 1,
|
||||
"id": "depth",
|
||||
"title": "Depth",
|
||||
"intro": "Dots near and far. The panel moves closer and further away; keep the dot sharp.",
|
||||
"steps": [
|
||||
{
|
||||
"id": "near",
|
||||
"kind": "dots",
|
||||
"text": "The panel is close now. Look at each dot until its ring fills. Keep your head still.",
|
||||
"grid": [
|
||||
3,
|
||||
3,
|
||||
15,
|
||||
15
|
||||
],
|
||||
"hold": 1.5,
|
||||
"shuffle": true,
|
||||
"distance": 0.5
|
||||
},
|
||||
{
|
||||
"id": "mid",
|
||||
"kind": "dots",
|
||||
"text": "The panel is at arm's length and a bit. Look at each dot until its ring fills. Keep your head still.",
|
||||
"grid": [
|
||||
3,
|
||||
3,
|
||||
15,
|
||||
15
|
||||
],
|
||||
"hold": 1.5,
|
||||
"shuffle": true,
|
||||
"distance": 1.5
|
||||
},
|
||||
{
|
||||
"id": "far",
|
||||
"kind": "dots",
|
||||
"text": "The panel is far now. Look at each dot until its ring fills. Keep your head still.",
|
||||
"grid": [
|
||||
3,
|
||||
3,
|
||||
15,
|
||||
15
|
||||
],
|
||||
"hold": 1.5,
|
||||
"shuffle": true,
|
||||
"distance": 5.0
|
||||
},
|
||||
{
|
||||
"id": "near-far",
|
||||
"kind": "prompt",
|
||||
"text": "Keep looking at the dot as it comes near and goes far.",
|
||||
"dot": [
|
||||
0,
|
||||
0
|
||||
],
|
||||
"cues": [
|
||||
[
|
||||
"Far",
|
||||
2.5
|
||||
],
|
||||
[
|
||||
"Near",
|
||||
2.5
|
||||
],
|
||||
[
|
||||
"Far",
|
||||
2.5
|
||||
],
|
||||
[
|
||||
"Near",
|
||||
2.5
|
||||
],
|
||||
[
|
||||
"Far",
|
||||
2.5
|
||||
],
|
||||
[
|
||||
"Near",
|
||||
2.5
|
||||
]
|
||||
],
|
||||
"cue_distance": {
|
||||
"Far": 5.0,
|
||||
"Near": 0.5
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,129 @@
|
||||
{
|
||||
"schema": 1,
|
||||
"id": "edges",
|
||||
"title": "View edges",
|
||||
"intro": "Dots out at the edges of the panel and in its corners: turn only your eyes to them, as far as is comfortable.",
|
||||
"steps": [
|
||||
{
|
||||
"id": "edges",
|
||||
"kind": "dots",
|
||||
"text": "Look at each dot until its ring fills. Keep your head still.|Some are far out: turn your eyes, not your head.",
|
||||
"dots": [
|
||||
[
|
||||
24,
|
||||
0
|
||||
],
|
||||
[
|
||||
-24,
|
||||
0
|
||||
],
|
||||
[
|
||||
30,
|
||||
0
|
||||
],
|
||||
[
|
||||
-30,
|
||||
0
|
||||
],
|
||||
[
|
||||
0,
|
||||
18
|
||||
],
|
||||
[
|
||||
0,
|
||||
-18
|
||||
],
|
||||
[
|
||||
0,
|
||||
23
|
||||
],
|
||||
[
|
||||
0,
|
||||
-23
|
||||
],
|
||||
[
|
||||
30,
|
||||
20
|
||||
],
|
||||
[
|
||||
-30,
|
||||
20
|
||||
],
|
||||
[
|
||||
30,
|
||||
-20
|
||||
],
|
||||
[
|
||||
-30,
|
||||
-20
|
||||
],
|
||||
[
|
||||
22,
|
||||
16
|
||||
],
|
||||
[
|
||||
-22,
|
||||
16
|
||||
],
|
||||
[
|
||||
22,
|
||||
-16
|
||||
],
|
||||
[
|
||||
-22,
|
||||
-16
|
||||
]
|
||||
],
|
||||
"hold": 1.2,
|
||||
"shuffle": true
|
||||
},
|
||||
{
|
||||
"id": "edges-sweep",
|
||||
"kind": "path",
|
||||
"text": "Follow the dot with your eyes as it goes round, far out.",
|
||||
"path": "circle 0 0 22 8",
|
||||
"seconds": 16
|
||||
},
|
||||
{
|
||||
"id": "edges-again",
|
||||
"kind": "dots",
|
||||
"text": "Look at each dot until its ring fills. Keep your head still.",
|
||||
"dots": [
|
||||
[
|
||||
30,
|
||||
0
|
||||
],
|
||||
[
|
||||
-30,
|
||||
0
|
||||
],
|
||||
[
|
||||
0,
|
||||
23
|
||||
],
|
||||
[
|
||||
0,
|
||||
-23
|
||||
],
|
||||
[
|
||||
30,
|
||||
20
|
||||
],
|
||||
[
|
||||
-30,
|
||||
20
|
||||
],
|
||||
[
|
||||
30,
|
||||
-20
|
||||
],
|
||||
[
|
||||
-30,
|
||||
-20
|
||||
]
|
||||
],
|
||||
"hold": 1.2,
|
||||
"shuffle": true
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,81 @@
|
||||
{
|
||||
"schema": 1,
|
||||
"id": "face",
|
||||
"title": "Face moves",
|
||||
"intro": "Faces move the headset and the lids. Keep looking at the dot while you make each face. The microphone isn't recorded.",
|
||||
"steps": [
|
||||
{
|
||||
"id": "faces",
|
||||
"kind": "prompt",
|
||||
"text": "Keep looking at the dot and do what it says.",
|
||||
"dot": [
|
||||
0,
|
||||
0
|
||||
],
|
||||
"cues": [
|
||||
[
|
||||
"Smile",
|
||||
3
|
||||
],
|
||||
[
|
||||
"Relax",
|
||||
2
|
||||
],
|
||||
[
|
||||
"Raise your eyebrows",
|
||||
3
|
||||
],
|
||||
[
|
||||
"Relax",
|
||||
2
|
||||
],
|
||||
[
|
||||
"Frown",
|
||||
3
|
||||
],
|
||||
[
|
||||
"Relax",
|
||||
2
|
||||
],
|
||||
[
|
||||
"Yawn (or pretend to)",
|
||||
4
|
||||
],
|
||||
[
|
||||
"Relax",
|
||||
2
|
||||
],
|
||||
[
|
||||
"Chew",
|
||||
4
|
||||
],
|
||||
[
|
||||
"Relax",
|
||||
2
|
||||
]
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "talk",
|
||||
"kind": "prompt",
|
||||
"text": "Read this aloud while looking at the dot as much as you can:|\"The quick brown fox jumps over the lazy dog, then naps in the warm afternoon sun.\"",
|
||||
"dot": [
|
||||
0,
|
||||
10
|
||||
],
|
||||
"seconds": 10
|
||||
},
|
||||
{
|
||||
"id": "faces-dots",
|
||||
"kind": "dots",
|
||||
"text": "Smile and keep smiling. Look at each dot until its ring fills. Keep your head still.",
|
||||
"grid": [
|
||||
3,
|
||||
3,
|
||||
15,
|
||||
15
|
||||
],
|
||||
"hold": 1.0
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,69 @@
|
||||
{
|
||||
"schema": 1,
|
||||
"id": "glasses",
|
||||
"title": "Glasses",
|
||||
"intro": "With your glasses or inserts on. Wear them as you usually do in the headset.",
|
||||
"steps": [
|
||||
{
|
||||
"id": "grid",
|
||||
"kind": "dots",
|
||||
"text": "Look at each dot until its ring fills. Keep your head still.",
|
||||
"grid": [
|
||||
5,
|
||||
5,
|
||||
24,
|
||||
18
|
||||
],
|
||||
"hold": 1.2,
|
||||
"shuffle": true
|
||||
},
|
||||
{
|
||||
"id": "edges",
|
||||
"kind": "dots",
|
||||
"text": "Look at each dot until its ring fills. Keep your head still.|Some are far out: turn your eyes, not your head.",
|
||||
"dots": [
|
||||
[
|
||||
30,
|
||||
0
|
||||
],
|
||||
[
|
||||
-30,
|
||||
0
|
||||
],
|
||||
[
|
||||
0,
|
||||
23
|
||||
],
|
||||
[
|
||||
0,
|
||||
-23
|
||||
],
|
||||
[
|
||||
30,
|
||||
20
|
||||
],
|
||||
[
|
||||
-30,
|
||||
20
|
||||
],
|
||||
[
|
||||
30,
|
||||
-20
|
||||
],
|
||||
[
|
||||
-30,
|
||||
-20
|
||||
]
|
||||
],
|
||||
"hold": 1.2,
|
||||
"shuffle": true
|
||||
},
|
||||
{
|
||||
"id": "follow",
|
||||
"kind": "path",
|
||||
"text": "Follow the dot with your eyes.",
|
||||
"path": "eight 0 0 22 14 6",
|
||||
"seconds": 12
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
{
|
||||
"schema": 1,
|
||||
"id": "head",
|
||||
"title": "Head turning",
|
||||
"intro": "A dot that stays put in the room while you turn your head. Keep your eyes on it.",
|
||||
"steps": [
|
||||
{
|
||||
"id": "turn-sides",
|
||||
"kind": "world",
|
||||
"group": "head",
|
||||
"text": "Keep looking at the dot. Turn your head slowly left and right.",
|
||||
"point": [
|
||||
0,
|
||||
0,
|
||||
-1.5
|
||||
],
|
||||
"seconds": 15
|
||||
},
|
||||
{
|
||||
"id": "turn-updown",
|
||||
"kind": "world",
|
||||
"group": "head",
|
||||
"text": "Keep looking at the dot. Nod slowly up and down.",
|
||||
"point": [
|
||||
0,
|
||||
0,
|
||||
-1.5
|
||||
],
|
||||
"seconds": 12
|
||||
},
|
||||
{
|
||||
"id": "tilt",
|
||||
"kind": "world",
|
||||
"group": "head",
|
||||
"text": "Keep looking at the dot. Tilt your head side to side.",
|
||||
"point": [
|
||||
0,
|
||||
0,
|
||||
-1.5
|
||||
],
|
||||
"seconds": 10
|
||||
},
|
||||
{
|
||||
"id": "shake",
|
||||
"kind": "world",
|
||||
"group": "head",
|
||||
"text": "Keep looking at the dot. Shake your head a little, quickly.",
|
||||
"point": [
|
||||
0,
|
||||
0,
|
||||
-1.5
|
||||
],
|
||||
"seconds": 8
|
||||
},
|
||||
{
|
||||
"id": "walk",
|
||||
"kind": "world",
|
||||
"group": "head",
|
||||
"text": "If you have room: take a step to each side while looking at the dot.",
|
||||
"point": [
|
||||
0,
|
||||
0,
|
||||
-2.0
|
||||
],
|
||||
"seconds": 12
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,78 @@
|
||||
{
|
||||
"schema": 1,
|
||||
"id": "jumps",
|
||||
"title": "Jumps and pursuit",
|
||||
"intro": "A dot that jumps around, then one that glides. Follow it with your eyes, as quickly as you can, without moving your head.",
|
||||
"steps": [
|
||||
{
|
||||
"id": "jumps",
|
||||
"kind": "jumps",
|
||||
"text": "Look at the dot wherever it jumps.",
|
||||
"count": 40,
|
||||
"range": [
|
||||
24,
|
||||
18
|
||||
],
|
||||
"each": [
|
||||
0.5,
|
||||
1.5
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "circle-slow",
|
||||
"kind": "path",
|
||||
"group": "pursuit",
|
||||
"text": "Follow the dot with your eyes.",
|
||||
"path": "circle 0 0 15 6",
|
||||
"seconds": 12
|
||||
},
|
||||
{
|
||||
"id": "circle-fast",
|
||||
"kind": "path",
|
||||
"group": "pursuit",
|
||||
"text": "Follow the dot with your eyes. It's faster now.",
|
||||
"path": "circle 0 0 15 2.5",
|
||||
"seconds": 10
|
||||
},
|
||||
{
|
||||
"id": "line-slow",
|
||||
"kind": "path",
|
||||
"group": "pursuit",
|
||||
"text": "Follow the dot from side to side.",
|
||||
"path": "line -24 0 24 0 3",
|
||||
"seconds": 12
|
||||
},
|
||||
{
|
||||
"id": "line-fast",
|
||||
"kind": "path",
|
||||
"group": "pursuit",
|
||||
"text": "Follow the dot from side to side, faster.",
|
||||
"path": "line -24 0 24 0 1.2",
|
||||
"seconds": 8
|
||||
},
|
||||
{
|
||||
"id": "line-up",
|
||||
"kind": "path",
|
||||
"group": "pursuit",
|
||||
"text": "Follow the dot up and down.",
|
||||
"path": "line 0 -18 0 18 2.5",
|
||||
"seconds": 10
|
||||
},
|
||||
{
|
||||
"id": "eight",
|
||||
"kind": "path",
|
||||
"group": "pursuit",
|
||||
"text": "Follow the dot along the figure eight.",
|
||||
"path": "eight 0 0 22 12 6",
|
||||
"seconds": 12
|
||||
},
|
||||
{
|
||||
"id": "eight-fast",
|
||||
"kind": "path",
|
||||
"group": "pursuit",
|
||||
"text": "Follow the dot along the figure eight, faster.",
|
||||
"path": "eight 0 0 22 12 3",
|
||||
"seconds": 9
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,89 @@
|
||||
{
|
||||
"schema": 1,
|
||||
"id": "light",
|
||||
"title": "Light changes",
|
||||
"intro": "The panel goes dark and bright. Your pupils change size with it: that's what this records.",
|
||||
"steps": [
|
||||
{
|
||||
"id": "steps",
|
||||
"kind": "light",
|
||||
"text": "Keep looking at the dot while the light changes.",
|
||||
"dot": [
|
||||
0,
|
||||
0
|
||||
],
|
||||
"levels": [
|
||||
[
|
||||
0.03,
|
||||
4
|
||||
],
|
||||
[
|
||||
0.9,
|
||||
4
|
||||
],
|
||||
[
|
||||
0.03,
|
||||
4
|
||||
],
|
||||
[
|
||||
0.33,
|
||||
4
|
||||
],
|
||||
[
|
||||
0.9,
|
||||
4
|
||||
],
|
||||
[
|
||||
0.33,
|
||||
4
|
||||
],
|
||||
[
|
||||
0.03,
|
||||
4
|
||||
]
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "grid-dark",
|
||||
"kind": "dots",
|
||||
"text": "Look at each dot until its ring fills. Keep your head still.",
|
||||
"grid": [
|
||||
5,
|
||||
5,
|
||||
24,
|
||||
18
|
||||
],
|
||||
"hold": 1.0,
|
||||
"shuffle": true,
|
||||
"bg": 0.03
|
||||
},
|
||||
{
|
||||
"id": "grid-mid",
|
||||
"kind": "dots",
|
||||
"text": "Look at each dot until its ring fills. Keep your head still.",
|
||||
"grid": [
|
||||
5,
|
||||
5,
|
||||
24,
|
||||
18
|
||||
],
|
||||
"hold": 1.0,
|
||||
"shuffle": true,
|
||||
"bg": 0.33
|
||||
},
|
||||
{
|
||||
"id": "grid-bright",
|
||||
"kind": "dots",
|
||||
"text": "Look at each dot until its ring fills. Keep your head still.",
|
||||
"grid": [
|
||||
5,
|
||||
5,
|
||||
24,
|
||||
18
|
||||
],
|
||||
"hold": 1.0,
|
||||
"shuffle": true,
|
||||
"bg": 0.9
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
{
|
||||
"schema": 1,
|
||||
"id": "reading",
|
||||
"title": "Reading",
|
||||
"intro": "Reading and looking around, as in everyday use.",
|
||||
"steps": [
|
||||
{
|
||||
"id": "read-1",
|
||||
"kind": "prompt",
|
||||
"group": "reading",
|
||||
"dot": null,
|
||||
"seconds": 20,
|
||||
"text": "Read this at your own pace.||The lighthouse keeper climbed the spiral stairs every evening, counting each of the one hundred and twelve steps. At the top, he polished the great lens until it shone, then lit the lamp and watched its beam sweep across the dark water. Ships far out at sea would see the flash and know where the rocks were."
|
||||
},
|
||||
{
|
||||
"id": "read-2",
|
||||
"kind": "prompt",
|
||||
"group": "reading",
|
||||
"dot": null,
|
||||
"seconds": 20,
|
||||
"text": "And this one.||In the morning the market filled with stalls of oranges, peppers and fresh bread. A boy on a bicycle wove between the crates, ringing his bell. Somewhere a radio played an old song, and an old woman hummed along while she weighed out a bag of cherries for a customer who couldn't decide."
|
||||
},
|
||||
{
|
||||
"id": "free",
|
||||
"kind": "prompt",
|
||||
"dot": null,
|
||||
"seconds": 15,
|
||||
"text": "Look around the panel freely: the corners, the edges, wherever you like."
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,231 @@
|
||||
{
|
||||
"schema": 1,
|
||||
"id": "slips",
|
||||
"title": "Headset slips",
|
||||
"intro": "You'll nudge the headset on your face on purpose, then put it back. Use one hand, gently: a few millimetres is enough.",
|
||||
"steps": [
|
||||
{
|
||||
"id": "nudge-up",
|
||||
"kind": "prompt",
|
||||
"group": "nudges",
|
||||
"text": "Keep looking at the dot and push the headset up a little. Hold it there.",
|
||||
"dot": [
|
||||
0,
|
||||
0
|
||||
],
|
||||
"seconds": 5
|
||||
},
|
||||
{
|
||||
"id": "after-up",
|
||||
"kind": "dots",
|
||||
"group": "nudges",
|
||||
"ready": false,
|
||||
"text": "Hold the headset there. Look at each dot until its ring fills. Keep your head still.",
|
||||
"dots": [
|
||||
[
|
||||
0,
|
||||
0
|
||||
],
|
||||
[
|
||||
15,
|
||||
12
|
||||
],
|
||||
[
|
||||
-15,
|
||||
-12
|
||||
],
|
||||
[
|
||||
15,
|
||||
-12
|
||||
],
|
||||
[
|
||||
-15,
|
||||
12
|
||||
]
|
||||
],
|
||||
"hold": 1.0
|
||||
},
|
||||
{
|
||||
"id": "nudge-down",
|
||||
"kind": "prompt",
|
||||
"group": "nudges",
|
||||
"text": "Let go, then pull the headset down a little. Hold it there.",
|
||||
"dot": [
|
||||
0,
|
||||
0
|
||||
],
|
||||
"seconds": 5
|
||||
},
|
||||
{
|
||||
"id": "after-down",
|
||||
"kind": "dots",
|
||||
"group": "nudges",
|
||||
"ready": false,
|
||||
"text": "Hold it there. Look at each dot until its ring fills. Keep your head still.",
|
||||
"dots": [
|
||||
[
|
||||
0,
|
||||
0
|
||||
],
|
||||
[
|
||||
15,
|
||||
12
|
||||
],
|
||||
[
|
||||
-15,
|
||||
-12
|
||||
],
|
||||
[
|
||||
15,
|
||||
-12
|
||||
],
|
||||
[
|
||||
-15,
|
||||
12
|
||||
]
|
||||
],
|
||||
"hold": 1.0
|
||||
},
|
||||
{
|
||||
"id": "nudge-left",
|
||||
"kind": "prompt",
|
||||
"group": "nudges",
|
||||
"text": "Let go, then slide the headset a little to your left.",
|
||||
"dot": [
|
||||
0,
|
||||
0
|
||||
],
|
||||
"seconds": 5
|
||||
},
|
||||
{
|
||||
"id": "after-left",
|
||||
"kind": "dots",
|
||||
"group": "nudges",
|
||||
"ready": false,
|
||||
"text": "Hold it there. Look at each dot until its ring fills. Keep your head still.",
|
||||
"dots": [
|
||||
[
|
||||
0,
|
||||
0
|
||||
],
|
||||
[
|
||||
15,
|
||||
12
|
||||
],
|
||||
[
|
||||
-15,
|
||||
-12
|
||||
],
|
||||
[
|
||||
15,
|
||||
-12
|
||||
],
|
||||
[
|
||||
-15,
|
||||
12
|
||||
]
|
||||
],
|
||||
"hold": 1.0
|
||||
},
|
||||
{
|
||||
"id": "nudge-right",
|
||||
"kind": "prompt",
|
||||
"group": "nudges",
|
||||
"text": "Let go, then slide it a little to your right.",
|
||||
"dot": [
|
||||
0,
|
||||
0
|
||||
],
|
||||
"seconds": 5
|
||||
},
|
||||
{
|
||||
"id": "after-right",
|
||||
"kind": "dots",
|
||||
"group": "nudges",
|
||||
"ready": false,
|
||||
"text": "Hold it there. Look at each dot until its ring fills. Keep your head still.",
|
||||
"dots": [
|
||||
[
|
||||
0,
|
||||
0
|
||||
],
|
||||
[
|
||||
15,
|
||||
12
|
||||
],
|
||||
[
|
||||
-15,
|
||||
-12
|
||||
],
|
||||
[
|
||||
15,
|
||||
-12
|
||||
],
|
||||
[
|
||||
-15,
|
||||
12
|
||||
]
|
||||
],
|
||||
"hold": 1.0
|
||||
},
|
||||
{
|
||||
"id": "nudge-tilt",
|
||||
"kind": "prompt",
|
||||
"group": "nudges",
|
||||
"text": "Let go, then tilt the headset a little: one side up.",
|
||||
"dot": [
|
||||
0,
|
||||
0
|
||||
],
|
||||
"seconds": 5
|
||||
},
|
||||
{
|
||||
"id": "after-tilt",
|
||||
"kind": "dots",
|
||||
"group": "nudges",
|
||||
"ready": false,
|
||||
"text": "Hold it there. Look at each dot until its ring fills. Keep your head still.",
|
||||
"dots": [
|
||||
[
|
||||
0,
|
||||
0
|
||||
],
|
||||
[
|
||||
15,
|
||||
12
|
||||
],
|
||||
[
|
||||
-15,
|
||||
-12
|
||||
],
|
||||
[
|
||||
15,
|
||||
-12
|
||||
],
|
||||
[
|
||||
-15,
|
||||
12
|
||||
]
|
||||
],
|
||||
"hold": 1.0
|
||||
},
|
||||
{
|
||||
"id": "reseat",
|
||||
"kind": "reseat",
|
||||
"text": "Take the headset off, put it back on as you usually do, then press the button."
|
||||
},
|
||||
{
|
||||
"id": "after-reseat",
|
||||
"kind": "dots",
|
||||
"text": "Look at each dot until its ring fills. Keep your head still.",
|
||||
"grid": [
|
||||
3,
|
||||
3,
|
||||
20,
|
||||
20
|
||||
],
|
||||
"hold": 1.2,
|
||||
"shuffle": true
|
||||
}
|
||||
]
|
||||
}
|
||||
+1572
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,479 @@
|
||||
#!/usr/bin/env python3
|
||||
"""The eye recorder's rounds and takes on disk: listing, review, deleting, and the encrypted export.
|
||||
|
||||
The layout is DESIGN.md's "Files":
|
||||
BASE/profile.json
|
||||
BASE/rounds/<YYYYMMDD-HHMMSS>-<bounty>/session.json, fit.json, health.json, session.log
|
||||
BASE/rounds/<round>/takes/<NN>-<group>/cam0/*.zst, cam1/*.zst, index.tsv (eyerec.py),
|
||||
targets.jsonl, poses.jsonl, ipd.jsonl, own.jsonl, steamvr.jsonl, prompts.jsonl, take.json
|
||||
BASE/exports/<round>/
|
||||
|
||||
Deleted ranges live in take.json ("deleted": [[from_ns, to_ns], ...]) on CLOCK_MONOTONIC, the
|
||||
clock of prompts.jsonl, which the window's timeline shows. The frames and the other logs are on
|
||||
CLOCK_MONOTONIC_RAW seconds; take.json's clock samples ([mono_ns, raw - mono ns]) turn one into
|
||||
the other. The files keep every frame until export leaves the deleted ones out.
|
||||
|
||||
The export (exports/<round>/, built in <round>.partial and moved when it's whole):
|
||||
manifest.json the round, its profile and session, the recipients, each take
|
||||
fit.json the fit coach's readings, as recorded
|
||||
health.json.age the opt-in health answers, encrypted (when there are any)
|
||||
takes/<id>/take.json as recorded, plus "export": the deleted ranges in RAW seconds
|
||||
takes/<id>/index.tsv the kept frames; a chunk keeps its number, frames renumbered in it
|
||||
takes/<id>/cam0/NNNNNN.zst.age ... each chunk encrypted to every recipient (recorder/age.py)
|
||||
takes/<id>/targets.jsonl, ipd.jsonl, prompts.jsonl whole: they say what was shown and asked
|
||||
takes/<id>/own.jsonl, steamvr.jsonl, poses.jsonl rows in deleted ranges left out
|
||||
SHA256SUMS over every file above, the encrypted ones as they are
|
||||
A chunk none of whose frames are deleted is encrypted as it is; one with some deleted frames is
|
||||
decompressed, the rest compressed again (zstd 1, as eyerec.py) and encrypted; one with none left
|
||||
is dropped. Every chunk is decompressed once to check it holds the frames index.tsv says; a
|
||||
chunk that doesn't, or one index.tsv doesn't name (the recorder was killed before the index
|
||||
line), is left out and counted in the manifest. session.log isn't exported (it names paths).
|
||||
|
||||
Needs the cryptography library to export (recorder/age.py). The rest is standard library.
|
||||
|
||||
usage: takes.py [--base DIR] list | takes ROUND | export ROUND
|
||||
takes.py unpack EXPORT --identity KEYFILE --out DIR decrypt an export (the dataset's side)
|
||||
"""
|
||||
import argparse
|
||||
import datetime
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
REPO = os.path.dirname(os.path.dirname(HERE))
|
||||
sys.path.insert(0, REPO)
|
||||
sys.path.insert(0, HERE)
|
||||
import eyerec # noqa: E402 (gaze/rec/eyerec.py: the chunks' format)
|
||||
from recorder import age # noqa: E402
|
||||
from recorder.common import write_json # noqa: E402
|
||||
from recorder.files import (Cancelled, human_bytes, in_ranges, merge_ranges, read_json, sha256_file, # noqa: E402,F401
|
||||
tree_bytes, tree_stat, write_sums)
|
||||
|
||||
DEFAULT_BASE = os.path.expanduser("~/.local/share/frametop/eyes/contrib")
|
||||
RECIPIENTS = os.path.join(HERE, "recipients.txt")
|
||||
ROUND_RE = re.compile(r"^\d{8}-\d{6}-[a-z0-9][a-z0-9-]{0,39}$") # session.py: <time>-<bounty>[-N]
|
||||
TAKE_RE = re.compile(r"^\d{2,}-[a-z0-9][a-z0-9_-]{0,63}$")
|
||||
CHUNK_RE = re.compile(r"^(\d{6})\.zst$")
|
||||
EXPORT_SCHEMA = 1
|
||||
WHOLE_LOGS = ("targets.jsonl", "ipd.jsonl", "prompts.jsonl")
|
||||
CUT_LOGS = ("own.jsonl", "steamvr.jsonl", "poses.jsonl") # "t" in RAW seconds
|
||||
GAP_S = 0.5 # frames further apart than this are a pause, not recorded time
|
||||
|
||||
|
||||
def tool_version():
|
||||
try:
|
||||
out = subprocess.run(["git", "-C", REPO, "describe", "--always", "--dirty", "--tags"],
|
||||
capture_output=True, text=True, timeout=5)
|
||||
described = out.stdout.strip()
|
||||
except (OSError, subprocess.TimeoutExpired):
|
||||
described = ""
|
||||
return "ft-eyerec " + (described or "unknown")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- clocks
|
||||
|
||||
def raw_offset_ns(t_ns, clock):
|
||||
"""RAW - MONOTONIC at t_ns (MONOTONIC), from take.json's clock samples: linear between the
|
||||
two around it, the nearest outside them, 0 without any."""
|
||||
pts = sorted((int(m), int(d)) for m, d in clock or [] if isinstance(m, (int, float)) and isinstance(d, (int, float)))
|
||||
if not pts:
|
||||
return 0
|
||||
if t_ns <= pts[0][0]:
|
||||
return pts[0][1]
|
||||
for (m0, d0), (m1, d1) in zip(pts, pts[1:]):
|
||||
if t_ns <= m1:
|
||||
return d0 + (d1 - d0) * (t_ns - m0) // max(1, m1 - m0)
|
||||
return pts[-1][1]
|
||||
|
||||
|
||||
def raw_ranges(ranges, clock):
|
||||
"""Deleted ranges (MONOTONIC ns) as [[from, to], ...] in RAW seconds."""
|
||||
return [[(a + raw_offset_ns(a, clock)) / 1e9, (b + raw_offset_ns(b, clock)) / 1e9] for a, b in merge_ranges(ranges)]
|
||||
|
||||
|
||||
def recorded_seconds(times):
|
||||
"""Time spent recording: the gaps under GAP_S between sorted frame times."""
|
||||
total = 0.0
|
||||
for a, b in zip(times, times[1:]):
|
||||
if b - a < GAP_S:
|
||||
total += b - a
|
||||
return total
|
||||
|
||||
|
||||
def cut_jsonl(src, dst, ranges):
|
||||
"""Copy a jsonl log, leaving out rows whose "t" (RAW seconds) is in ranges. Lines that don't
|
||||
parse are copied as they are (validate.py reports them)."""
|
||||
kept = cut = 0
|
||||
with open(src, encoding="utf-8", errors="replace") as f, open(dst, "w", encoding="utf-8", newline="\n") as out:
|
||||
for line in f:
|
||||
if not line.strip():
|
||||
continue
|
||||
try:
|
||||
t = json.loads(line).get("t")
|
||||
except (ValueError, AttributeError):
|
||||
t = None
|
||||
if isinstance(t, (int, float)) and in_ranges(t, ranges):
|
||||
cut += 1
|
||||
continue
|
||||
out.write(line if line.endswith("\n") else line + "\n")
|
||||
kept += 1
|
||||
return kept, cut
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- the store
|
||||
|
||||
class Store:
|
||||
def __init__(self, base=DEFAULT_BASE):
|
||||
self.base = os.path.abspath(os.path.expanduser(base))
|
||||
self.rounds_dir = os.path.join(self.base, "rounds")
|
||||
self.exports_dir = os.path.join(self.base, "exports")
|
||||
self.profile_path = os.path.join(self.base, "profile.json")
|
||||
|
||||
# --- paths, checked: ids come from the window, so nothing may climb out of the base
|
||||
def round_dir(self, rnd):
|
||||
if not ROUND_RE.match(rnd or ""):
|
||||
raise ValueError(f"not a round id: {rnd!r}")
|
||||
return os.path.join(self.rounds_dir, rnd)
|
||||
|
||||
def take_dir(self, rnd, take):
|
||||
if not TAKE_RE.match(take or ""):
|
||||
raise ValueError(f"not a take id: {take!r}")
|
||||
return os.path.join(self.round_dir(rnd), "takes", take)
|
||||
|
||||
def export_dir(self, rnd):
|
||||
self.round_dir(rnd) # checks the id
|
||||
return os.path.join(self.exports_dir, rnd)
|
||||
|
||||
def profile(self):
|
||||
return read_json(self.profile_path)
|
||||
|
||||
# --- listing
|
||||
def rounds(self):
|
||||
"""Newest first: [{id, bounty, title, started, status, dry_run, takes, bytes, exported,
|
||||
export_stale (changed since), export_bytes}]."""
|
||||
out = []
|
||||
try:
|
||||
names = os.listdir(self.rounds_dir)
|
||||
except OSError:
|
||||
return out
|
||||
for rid in names:
|
||||
path = os.path.join(self.rounds_dir, rid)
|
||||
if not ROUND_RE.match(rid) or not os.path.isdir(path):
|
||||
continue
|
||||
meta = read_json(os.path.join(path, "session.json"))
|
||||
size, changed = tree_stat(path)
|
||||
manifest = os.path.join(self.export_dir(rid), "manifest.json")
|
||||
exported = os.path.getmtime(manifest) if os.path.isfile(manifest) else 0.0
|
||||
out.append({"id": rid, "bounty": meta.get("bounty", ""), "title": meta.get("title", ""),
|
||||
"started": meta.get("started", ""), "status": meta.get("status", ""),
|
||||
"dry_run": bool(meta.get("dry_run")), "takes": len(self.take_ids(rid)), "bytes": size,
|
||||
"exported": bool(exported), "export_stale": bool(exported) and changed > exported,
|
||||
"export_bytes": tree_bytes(self.export_dir(rid)) if exported else 0})
|
||||
return sorted(out, key=lambda r: r["id"], reverse=True)
|
||||
|
||||
def take_ids(self, rnd):
|
||||
d = os.path.join(self.round_dir(rnd), "takes")
|
||||
try:
|
||||
names = os.listdir(d)
|
||||
except OSError:
|
||||
return []
|
||||
return sorted(n for n in names if TAKE_RE.match(n) and os.path.isdir(os.path.join(d, n)))
|
||||
|
||||
def takes(self, rnd):
|
||||
"""[{id, group, status, frames: [right, left], deleted_frames: [..], ranges, duration_s,
|
||||
bytes, started_ns, ended_ns}]."""
|
||||
out = []
|
||||
for tid in self.take_ids(rnd):
|
||||
path = self.take_dir(rnd, tid)
|
||||
meta = read_json(os.path.join(path, "take.json"))
|
||||
ranges = merge_ranges(meta.get("deleted") or [])
|
||||
raw = raw_ranges(ranges, meta.get("clock"))
|
||||
times = {c: [] for c in eyerec.CAMERAS}
|
||||
for c, _chunk, _i, _slot, t in eyerec.read_index(path):
|
||||
if c in times:
|
||||
times[c].append(t)
|
||||
for c in times:
|
||||
times[c].sort()
|
||||
out.append({"id": tid, "group": meta.get("group", tid.split("-", 1)[-1]),
|
||||
"status": meta.get("status", "recording"),
|
||||
"frames": [len(times[c]) for c in eyerec.CAMERAS],
|
||||
"deleted_frames": [sum(1 for t in times[c] if in_ranges(t, raw)) for c in eyerec.CAMERAS],
|
||||
"ranges": ranges, "duration_s": max(recorded_seconds(times[c]) for c in eyerec.CAMERAS),
|
||||
"bytes": sum(tree_bytes(os.path.join(path, "cam%d" % c)) for c in eyerec.CAMERAS),
|
||||
"started_ns": meta.get("started_ns"), "ended_ns": meta.get("ended_ns")})
|
||||
return out
|
||||
|
||||
def take_meta(self, rnd, take):
|
||||
return read_json(os.path.join(self.take_dir(rnd, take), "take.json"))
|
||||
|
||||
# --- deleted ranges
|
||||
def ranges(self, rnd, take):
|
||||
return merge_ranges(self.take_meta(rnd, take).get("deleted") or [])
|
||||
|
||||
def _set_ranges(self, rnd, take, ranges):
|
||||
path = os.path.join(self.take_dir(rnd, take), "take.json")
|
||||
meta = read_json(path)
|
||||
if not meta:
|
||||
raise RuntimeError(f"no take {rnd}/{take}")
|
||||
meta["deleted"] = merge_ranges(ranges)
|
||||
write_json(path, meta)
|
||||
return meta["deleted"]
|
||||
|
||||
def delete_range(self, rnd, take, from_ns, to_ns):
|
||||
"""Leave the frames from from_ns to to_ns (CLOCK_MONOTONIC, inclusive) out of the export."""
|
||||
return self._set_ranges(rnd, take, self.ranges(rnd, take) + [[int(from_ns), int(to_ns)]])
|
||||
|
||||
def restore_range(self, rnd, take, i):
|
||||
"""Take deleted range i (in ranges()' order) back."""
|
||||
ranges = self.ranges(rnd, take)
|
||||
if 0 <= i < len(ranges):
|
||||
del ranges[i]
|
||||
return self._set_ranges(rnd, take, ranges)
|
||||
|
||||
# --- deleting files
|
||||
def delete_take(self, rnd, take):
|
||||
path = self.take_dir(rnd, take)
|
||||
if os.path.isdir(path):
|
||||
shutil.rmtree(path)
|
||||
meta_path = os.path.join(self.round_dir(rnd), "session.json")
|
||||
meta = read_json(meta_path)
|
||||
if take in (meta.get("takes") or []):
|
||||
meta["takes"] = [t for t in meta["takes"] if t != take]
|
||||
meta.setdefault("deleted_takes", []).append(take)
|
||||
write_json(meta_path, meta)
|
||||
|
||||
def delete_round(self, rnd):
|
||||
"""The round and its export, if any."""
|
||||
for path in (self.round_dir(rnd), self.export_dir(rnd)):
|
||||
if os.path.isdir(path):
|
||||
shutil.rmtree(path)
|
||||
|
||||
def delete_export(self, rnd):
|
||||
path = self.export_dir(rnd)
|
||||
if os.path.isdir(path):
|
||||
shutil.rmtree(path)
|
||||
|
||||
# --- export
|
||||
def export(self, rnd, progress=None, cancel=None, keep_notes=False, low_priority=True, recipients=None):
|
||||
"""Write exports/<round>/ (the module's docstring). progress(fraction 0..1, text) is
|
||||
called as it goes; cancel() (or cancel.is_set()) returning true stops it, raising
|
||||
Cancelled. Nothing is left behind on failure or cancel. Returns the export's path.
|
||||
recipients: "age1..." keys, default recipients.txt's. low_priority: run this thread at
|
||||
nice 19 (Linux nice is per thread)."""
|
||||
recipients = list(recipients) if recipients else age.read_recipients(RECIPIENTS)
|
||||
age._crypto() # fails now, not half way, without the cryptography library
|
||||
cancelled = getattr(cancel, "is_set", cancel) or (lambda: False)
|
||||
report = progress or (lambda fraction, text: None)
|
||||
src = self.round_dir(rnd)
|
||||
if not os.path.isdir(src):
|
||||
raise RuntimeError(f"no round {rnd}")
|
||||
if low_priority:
|
||||
try:
|
||||
os.setpriority(os.PRIO_PROCESS, threading.get_native_id(), 19)
|
||||
except OSError:
|
||||
pass
|
||||
final = self.export_dir(rnd)
|
||||
work = final + ".partial"
|
||||
if os.path.isdir(work):
|
||||
shutil.rmtree(work)
|
||||
os.makedirs(work)
|
||||
try:
|
||||
self._export(rnd, src, work, recipients, report, cancelled, keep_notes)
|
||||
if os.path.isdir(final):
|
||||
shutil.rmtree(final)
|
||||
os.replace(work, final)
|
||||
except BaseException:
|
||||
shutil.rmtree(work, ignore_errors=True)
|
||||
raise
|
||||
report(1.0, "Done")
|
||||
return final
|
||||
|
||||
def _export(self, rnd, src, work, recipients, report, cancelled, keep_notes):
|
||||
profile = self.profile()
|
||||
meta = read_json(os.path.join(src, "session.json"))
|
||||
meta.pop("uploads", None) # earlier uploads' records (hub.py) aren't part of the contribution
|
||||
takes = self.takes(rnd)
|
||||
total = sum(t["bytes"] for t in takes) or 1
|
||||
done = 0
|
||||
entries = []
|
||||
for t in takes:
|
||||
if cancelled():
|
||||
raise Cancelled()
|
||||
tdir = self.take_dir(rnd, t["id"])
|
||||
out = os.path.join(work, "takes", t["id"])
|
||||
os.makedirs(out)
|
||||
take_meta = read_json(os.path.join(tdir, "take.json"))
|
||||
raw = raw_ranges(t["ranges"], take_meta.get("clock"))
|
||||
logs = {}
|
||||
for name in WHOLE_LOGS:
|
||||
if os.path.isfile(os.path.join(tdir, name)):
|
||||
shutil.copyfile(os.path.join(tdir, name), os.path.join(out, name))
|
||||
for name in CUT_LOGS:
|
||||
if os.path.isfile(os.path.join(tdir, name)):
|
||||
kept, cut = cut_jsonl(os.path.join(tdir, name), os.path.join(out, name), raw)
|
||||
logs[name] = {"rows": kept, "deleted": cut}
|
||||
|
||||
def step(nbytes, title=t["id"]):
|
||||
nonlocal done
|
||||
done += nbytes
|
||||
report(min(done / total, 0.99), f"Encrypting {title}")
|
||||
chunks = self._chunks(tdir, out, raw, recipients, step, cancelled)
|
||||
if take_meta:
|
||||
take_meta["export"] = {"deleted_raw_s": raw}
|
||||
write_json(os.path.join(out, "take.json"), take_meta)
|
||||
entries.append({"id": t["id"], "group": t["group"], "status": t["status"],
|
||||
"frames": chunks["frames"], "frames_deleted": chunks["deleted"],
|
||||
"duration_s": round(t["duration_s"], 2), "chunks": chunks["chunks"],
|
||||
"raw_bytes": chunks["raw_bytes"], "left_out": chunks["left_out"], "logs": logs})
|
||||
if cancelled():
|
||||
raise Cancelled()
|
||||
if os.path.isfile(os.path.join(src, "fit.json")):
|
||||
shutil.copyfile(os.path.join(src, "fit.json"), os.path.join(work, "fit.json"))
|
||||
health = os.path.join(src, "health.json")
|
||||
if os.path.isfile(health):
|
||||
age.encrypt_file(health, os.path.join(work, "health.json.age"), recipients)
|
||||
shown = json.loads(json.dumps(profile))
|
||||
shown.pop("health", None) # only ever encrypted, in health.json.age
|
||||
if not keep_notes and isinstance(shown.get("optional"), dict):
|
||||
shown["optional"].pop("notes", None)
|
||||
manifest = {"schema": EXPORT_SCHEMA, "tool": tool_version(),
|
||||
"exported": datetime.datetime.now().astimezone().isoformat(timespec="seconds"),
|
||||
"round_id": rnd, "bounty": meta.get("bounty", ""),
|
||||
"contributor": profile.get("contributor", ""),
|
||||
"consent_version": (profile.get("consent") or {}).get("version", ""),
|
||||
"encryption": {"format": "age-encryption.org/v1", "recipients": recipients,
|
||||
"files": ["takes/*/cam*/*.zst.age"] + (["health.json.age"] if os.path.isfile(health) else [])},
|
||||
"frame": {"width": eyerec.W, "height": eyerec.H, "format": "u8 grey",
|
||||
"cameras": {"0": "right eye", "1": "left eye"}},
|
||||
"profile": shown, "session": meta, "takes": entries}
|
||||
write_json(os.path.join(work, "manifest.json"), manifest)
|
||||
report(0.995, "Writing checksums")
|
||||
write_sums(work, cancelled)
|
||||
|
||||
@staticmethod
|
||||
def _chunks(tdir, out, ranges, recipients, step, cancelled):
|
||||
"""Each camera's chunks into out/camN/*.zst.age and the kept frames into out/index.tsv."""
|
||||
rows = {}
|
||||
for c, chunk, i, slot, t in eyerec.read_index(tdir):
|
||||
rows.setdefault((c, chunk), []).append((i, slot, t))
|
||||
on_disk = set()
|
||||
for c in eyerec.CAMERAS:
|
||||
try:
|
||||
names = os.listdir(os.path.join(tdir, "cam%d" % c))
|
||||
except OSError:
|
||||
continue
|
||||
on_disk.update((c, int(m.group(1))) for m in map(CHUNK_RE.match, names) if m)
|
||||
res = {"frames": [0] * len(eyerec.CAMERAS), "deleted": [0] * len(eyerec.CAMERAS), "chunks": 0,
|
||||
"raw_bytes": 0, "left_out": {"not_indexed": 0, "missing": 0, "bad": 0}}
|
||||
res["left_out"]["not_indexed"] = len(on_disk - set(rows))
|
||||
lines = [eyerec.INDEX_HEADER]
|
||||
for c, chunk in sorted(rows):
|
||||
if cancelled():
|
||||
raise Cancelled()
|
||||
frames = sorted(rows[(c, chunk)])
|
||||
if c not in eyerec.CAMERAS:
|
||||
continue
|
||||
if (c, chunk) not in on_disk:
|
||||
res["left_out"]["missing"] += 1
|
||||
continue
|
||||
path = os.path.join(tdir, "cam%d" % c, "%06d.zst" % chunk)
|
||||
with open(path, "rb") as f:
|
||||
packed = f.read()
|
||||
step(len(packed))
|
||||
try:
|
||||
data = eyerec.decompress(packed)
|
||||
except Exception:
|
||||
data = b""
|
||||
if len(data) != len(frames) * eyerec.FRAME or [i for i, _, _ in frames] != list(range(len(frames))):
|
||||
res["left_out"]["bad"] += 1
|
||||
continue
|
||||
kept = [(i, slot, t) for i, slot, t in frames if not in_ranges(t, ranges)]
|
||||
res["deleted"][c] += len(frames) - len(kept)
|
||||
if not kept:
|
||||
continue
|
||||
if len(kept) < len(frames):
|
||||
packed = eyerec.compress(b"".join(data[i * eyerec.FRAME:(i + 1) * eyerec.FRAME] for i, _, _ in kept))
|
||||
os.makedirs(os.path.join(out, "cam%d" % c), exist_ok=True)
|
||||
with open(os.path.join(out, "cam%d" % c, "%06d.zst.age" % chunk), "wb") as f:
|
||||
f.write(age.encrypt_bytes(packed, recipients))
|
||||
lines.extend("%d\t%d\t%d\t%d\t%.6f\n" % (c, chunk, k, slot, t) for k, (_, slot, t) in enumerate(kept))
|
||||
res["frames"][c] += len(kept)
|
||||
res["chunks"] += 1
|
||||
res["raw_bytes"] += len(kept) * eyerec.FRAME
|
||||
with open(os.path.join(out, "index.tsv"), "w") as f:
|
||||
f.writelines(lines)
|
||||
return res
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- the dataset's side
|
||||
|
||||
def read_identity(path):
|
||||
with open(path) as f:
|
||||
for line in f:
|
||||
if line.strip().startswith("AGE-SECRET-KEY-1"):
|
||||
return line.strip()
|
||||
raise ValueError(f"{path} has no age identity")
|
||||
|
||||
|
||||
def unpack(export, identity_path, dest, progress=None):
|
||||
"""Check an export's SHA256SUMS, then copy it to dest with every .age file decrypted (its
|
||||
name without .age). The output is a take folder eyerec.py can read. Returns dest."""
|
||||
identity = read_identity(identity_path)
|
||||
report = progress or (lambda fraction, text: None)
|
||||
sums = {}
|
||||
with open(os.path.join(export, "SHA256SUMS")) as f:
|
||||
for line in f:
|
||||
digest, rel = line.rstrip("\n").split(" ", 1)
|
||||
if rel.startswith("/") or ".." in rel.split("/"):
|
||||
raise ValueError(f"SHA256SUMS names {rel!r}")
|
||||
sums[rel] = digest
|
||||
for k, (rel, digest) in enumerate(sorted(sums.items())):
|
||||
src = os.path.join(export, rel)
|
||||
if sha256_file(src) != digest:
|
||||
raise ValueError(f"{rel}: checksum doesn't match")
|
||||
out = os.path.join(dest, rel[:-4] if rel.endswith(".age") else rel)
|
||||
os.makedirs(os.path.dirname(out), exist_ok=True)
|
||||
if rel.endswith(".age"):
|
||||
age.decrypt_file(src, out, identity)
|
||||
else:
|
||||
shutil.copyfile(src, out)
|
||||
report((k + 1) / len(sums), rel)
|
||||
return dest
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description="List, export and unpack eye recorder rounds.")
|
||||
ap.add_argument("--base", default=DEFAULT_BASE)
|
||||
sub = ap.add_subparsers(dest="cmd", required=True)
|
||||
sub.add_parser("list")
|
||||
sub.add_parser("takes").add_argument("round")
|
||||
sub.add_parser("export").add_argument("round")
|
||||
up = sub.add_parser("unpack", help="check and decrypt an export (needs one of the dataset's keys)")
|
||||
up.add_argument("export")
|
||||
up.add_argument("--identity", required=True, help="an age identity file")
|
||||
up.add_argument("--out", required=True)
|
||||
a = ap.parse_args()
|
||||
store = Store(a.base)
|
||||
if a.cmd == "list":
|
||||
for r in store.rounds():
|
||||
print(f"{r['id']} {r['status'] or '-'} {r['takes']} takes {human_bytes(r['bytes'])}"
|
||||
+ (" dry run" if r["dry_run"] else "") + (" exported" if r["exported"] else ""))
|
||||
elif a.cmd == "takes":
|
||||
for t in store.takes(a.round):
|
||||
print(f"{t['id']} {t['status']} frames {t['frames']} ({t['deleted_frames']} deleted) "
|
||||
f"{t['duration_s']:.1f} s {human_bytes(t['bytes'])}")
|
||||
elif a.cmd == "unpack":
|
||||
print(unpack(a.export, a.identity, a.out))
|
||||
else:
|
||||
path = store.export(a.round, progress=lambda f, text: print(f"\r{f * 100:5.1f}% {text:40.40}", end="",
|
||||
flush=True))
|
||||
print(f"\n{path} {human_bytes(tree_bytes(path))}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+253
@@ -0,0 +1,253 @@
|
||||
#!/usr/bin/env python3
|
||||
"""The eye recorder's frame writer (gaze/rec/eyerec.py) against a made-up share, laid out as
|
||||
ft-eyegrab's (gaze/tracker/eyes_share.py). No eye images: each frame is one byte value
|
||||
repeated, from its camera and number, so the round trip can be checked.
|
||||
|
||||
python3 gaze/rec/tests/test_eyerec.py
|
||||
"""
|
||||
import mmap
|
||||
import os
|
||||
import shutil
|
||||
import struct
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
import unittest
|
||||
from unittest import mock
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
sys.path.insert(0, os.path.dirname(HERE))
|
||||
import eyerec # noqa: E402
|
||||
|
||||
W, H, SLOTS = eyerec.W, eyerec.H, 8
|
||||
ESIZE = 64 + W * H
|
||||
|
||||
|
||||
def frame_bytes(cam, n):
|
||||
return bytes([(cam * 128 + n) % 256]) * (W * H)
|
||||
|
||||
|
||||
class FakeShare:
|
||||
"""Writes frames into a share file at fps per camera, on a thread, as ft-eyegrab --share does."""
|
||||
|
||||
def __init__(self, path, fps=90.0):
|
||||
self.path, self.fps = path, fps
|
||||
self.count = [0, 0]
|
||||
self._stop = threading.Event()
|
||||
self._make()
|
||||
self._thread = threading.Thread(target=self._run, daemon=True)
|
||||
|
||||
def _make(self):
|
||||
size = 64 + 2 * SLOTS * ESIZE
|
||||
tmp = self.path + ".new"
|
||||
fd = os.open(tmp, os.O_RDWR | os.O_CREAT | os.O_TRUNC, 0o600)
|
||||
os.ftruncate(fd, size)
|
||||
self.mm = mmap.mmap(fd, size)
|
||||
os.close(fd)
|
||||
struct.pack_into("<6I2QI", self.mm, 0, 0x31434546, 1, W, H, SLOTS, ESIZE, 0, 0, os.getpid())
|
||||
os.replace(tmp, self.path)
|
||||
self.count = [0, 0]
|
||||
|
||||
def replace(self):
|
||||
"""As ft-eyegrab restarting: a new file, the counts from 0."""
|
||||
self._make()
|
||||
|
||||
def start(self):
|
||||
self._thread.start()
|
||||
return self
|
||||
|
||||
def stop(self):
|
||||
self._stop.set()
|
||||
self._thread.join(2)
|
||||
|
||||
def put(self, cam):
|
||||
n = self.count[cam]
|
||||
off = 64 + (cam * SLOTS + n % SLOTS) * ESIZE
|
||||
seq = struct.unpack_from("<Q", self.mm, off)[0]
|
||||
struct.pack_into("<Q", self.mm, off, seq + 1)
|
||||
self.mm[off + 64:off + 64 + W * H] = frame_bytes(cam, n)
|
||||
struct.pack_into("<dQII", self.mm, off + 8, time.clock_gettime(time.CLOCK_MONOTONIC_RAW), n, cam, n % 4 + 4 * cam)
|
||||
struct.pack_into("<Q", self.mm, off, seq + 2)
|
||||
self.count[cam] = n + 1
|
||||
struct.pack_into("<Q", self.mm, 24 + 8 * cam, n + 1)
|
||||
|
||||
def _run(self):
|
||||
period = 1.0 / self.fps
|
||||
nxt = time.monotonic()
|
||||
while not self._stop.is_set():
|
||||
for c in (0, 1):
|
||||
self.put(c)
|
||||
nxt += period
|
||||
time.sleep(max(0.0, nxt - time.monotonic()))
|
||||
|
||||
|
||||
class WriterTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.mkdtemp(prefix="eyerec-test-")
|
||||
self.share_path = os.path.join(self.tmp, "cams")
|
||||
self.want = os.path.join(self.tmp, "want")
|
||||
self.take = os.path.join(self.tmp, "take")
|
||||
self.lines = []
|
||||
|
||||
def tearDown(self):
|
||||
shutil.rmtree(self.tmp, ignore_errors=True)
|
||||
|
||||
def writer(self, **kw):
|
||||
kw.setdefault("min_free", 0)
|
||||
return eyerec.Writer(self.take, share_path=self.share_path, want_path=self.want, log=self.lines.append, **kw)
|
||||
|
||||
def wait_frames(self, w, n, timeout=10):
|
||||
end = time.monotonic() + timeout
|
||||
while time.monotonic() < end:
|
||||
with w._lock:
|
||||
if all(len(w._times[c]) and w._last[c] is not None for c in (0, 1)) and \
|
||||
sum(len(w._open[c]) for c in (0, 1)) + sum(w._frames.values()) >= n:
|
||||
return
|
||||
time.sleep(0.02)
|
||||
self.fail("not enough frames")
|
||||
|
||||
def test_round_trip_with_a_pause(self):
|
||||
share = FakeShare(self.share_path).start()
|
||||
try:
|
||||
w = self.writer(chunk_frames=20).start()
|
||||
time.sleep(0.2) # frames before recording aren't kept
|
||||
w.record(True)
|
||||
time.sleep(0.6)
|
||||
w.record(False)
|
||||
time.sleep(0.3) # nor while paused
|
||||
w.record(True)
|
||||
time.sleep(0.4)
|
||||
stats = w.stop()
|
||||
finally:
|
||||
share.stop()
|
||||
self.assertIsNone(stats["error"])
|
||||
self.assertEqual(stats["dropped"], [0, 0])
|
||||
self.assertGreater(min(stats["frames"]), 50)
|
||||
self.assertGreater(min(stats["fps"]), 60)
|
||||
rows = eyerec.read_index(self.take)
|
||||
self.assertEqual(len(rows), sum(stats["frames"]))
|
||||
got = list(eyerec.iter_frames(self.take))
|
||||
for c in (0, 1):
|
||||
ts = [t for cam, t, _s, _d in got if cam == c]
|
||||
gaps = [b - a for a, b in zip(ts, ts[1:])]
|
||||
self.assertGreater(max(gaps), 0.25, "the pause leaves a gap")
|
||||
self.assertEqual(sorted(ts), ts)
|
||||
# Each frame is what the share had: a byte value from its camera and number, so
|
||||
# consecutive frames of a camera go up by one (mod 256) outside the gap.
|
||||
vals = [d[0] for cam, _t, _s, d in got if cam == c]
|
||||
steps = {(b - a) % 256 for a, b, g in zip(vals, vals[1:], gaps) if g < 0.05}
|
||||
self.assertEqual(steps, {1})
|
||||
self.assertTrue(all(d == bytes([d[0]]) * len(d) for cam, _t, _s, d in got if cam == c))
|
||||
# The pause ended the chunks: none spans it.
|
||||
for c in (0, 1):
|
||||
chunks = {}
|
||||
for cam, chunk, i, _slot, t in rows:
|
||||
if cam == c:
|
||||
chunks.setdefault(chunk, []).append(t)
|
||||
self.assertTrue(all(max(ts) - min(ts) < 0.25 for ts in chunks.values()))
|
||||
self.assertTrue(os.path.exists(self.want), "ft-eyegrab's want file is touched")
|
||||
# It compresses: a repeated byte is tiny.
|
||||
self.assertLess(stats["bytes"], sum(stats["frames"]) * W * H / 100)
|
||||
|
||||
def test_a_slow_writer_drops_chunks_not_frames_in_the_share(self):
|
||||
share = FakeShare(self.share_path).start()
|
||||
real = eyerec.compress
|
||||
|
||||
def slow(b):
|
||||
time.sleep(0.3)
|
||||
return real(b)
|
||||
try:
|
||||
with mock.patch.object(eyerec, "compress", slow), mock.patch.object(eyerec, "MAX_QUEUED", 1):
|
||||
w = self.writer(chunk_frames=5).start()
|
||||
w.record(True)
|
||||
time.sleep(1.0)
|
||||
stats = w.stop()
|
||||
finally:
|
||||
share.stop()
|
||||
self.assertGreater(sum(stats["dropped"]), 0)
|
||||
self.assertTrue(any("behind" in line for line in self.lines))
|
||||
rows = eyerec.read_index(self.take)
|
||||
self.assertEqual(len(rows), sum(stats["frames"]))
|
||||
for c, chunk, _i, _s, _t in rows: # every chunk the index names is there
|
||||
self.assertTrue(os.path.exists(os.path.join(self.take, "cam%d" % c, "%06d.zst" % chunk)))
|
||||
|
||||
def test_follows_a_replaced_share(self):
|
||||
share = FakeShare(self.share_path).start()
|
||||
try:
|
||||
w = self.writer(chunk_frames=10).start()
|
||||
w.record(True)
|
||||
time.sleep(0.4)
|
||||
share.replace() # ft-eyegrab restarted with SteamVR
|
||||
time.sleep(2.8) # the reader looks every 2 s
|
||||
before = w.stats()["frames"]
|
||||
time.sleep(0.5)
|
||||
stats = w.stop()
|
||||
finally:
|
||||
share.stop()
|
||||
self.assertIsNone(stats["error"])
|
||||
self.assertTrue(any("replaced" in line for line in self.lines))
|
||||
self.assertGreater(min(stats["frames"]), min(before))
|
||||
|
||||
def test_no_share(self):
|
||||
with mock.patch.object(eyerec, "SHARE_WAIT", 0.2):
|
||||
w = self.writer().start()
|
||||
w.record(True)
|
||||
time.sleep(0.6)
|
||||
stats = w.stop()
|
||||
self.assertIn("frametop-eyegrab", stats["error"])
|
||||
self.assertFalse(w.recording)
|
||||
|
||||
def test_stops_before_the_disk_fills(self):
|
||||
share = FakeShare(self.share_path).start()
|
||||
try:
|
||||
w = self.writer(min_free=1e18).start()
|
||||
w.record(True)
|
||||
time.sleep(2.5)
|
||||
self.assertFalse(w.recording)
|
||||
stats = w.stop()
|
||||
finally:
|
||||
share.stop()
|
||||
self.assertIn("free on the disk", stats["error"])
|
||||
w.record(True)
|
||||
self.assertFalse(w.recording, "it doesn't start again after an error")
|
||||
|
||||
def test_numbering_goes_on_in_a_take_already_there(self):
|
||||
share = FakeShare(self.share_path).start()
|
||||
try:
|
||||
for _ in range(2):
|
||||
w = self.writer(chunk_frames=10).start()
|
||||
w.record(True)
|
||||
time.sleep(0.3)
|
||||
w.stop()
|
||||
finally:
|
||||
share.stop()
|
||||
rows = eyerec.read_index(self.take)
|
||||
chunks = [chunk for c, chunk, _i, _s, _t in rows if c == 0]
|
||||
self.assertEqual(len(set(chunks)), len(os.listdir(os.path.join(self.take, "cam0"))))
|
||||
with open(os.path.join(self.take, "index.tsv")) as f:
|
||||
self.assertEqual(sum(1 for line in f if line.startswith("#")), 1)
|
||||
|
||||
|
||||
@unittest.skipUnless(shutil.which("zstd") or os.path.exists("/usr/bin/zstd"), "no zstd command")
|
||||
class CommandFallbackTest(unittest.TestCase):
|
||||
"""The zstd command, as on the SteamOS host's Python 3.12."""
|
||||
|
||||
def test_round_trip(self):
|
||||
real_import = __import__
|
||||
|
||||
def no_modules(name, *a, **kw):
|
||||
if name in ("compression", "zstandard"):
|
||||
raise ImportError(name)
|
||||
return real_import(name, *a, **kw)
|
||||
with mock.patch("builtins.__import__", no_modules):
|
||||
c, d, name = eyerec._compressor()
|
||||
self.assertTrue(name.endswith("zstd"))
|
||||
data = frame_bytes(0, 3) + frame_bytes(1, 4)
|
||||
self.assertEqual(d(c(data)), data)
|
||||
if eyerec.decompress is not None: # the module's and the command's are the same format
|
||||
self.assertEqual(eyerec.decompress(c(data)), data)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,134 @@
|
||||
#!/usr/bin/env python3
|
||||
"""The eye recorder's upload (gaze/rec/hub.py) without the network: the dry run, the gate that
|
||||
keeps the real dataset closed until the texts are reviewed, upload records, and the pull request
|
||||
flow against a fake Hub. The round is test_takes.make_round's random frames.
|
||||
|
||||
python3 gaze/rec/tests/test_hub.py
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
import uuid
|
||||
from unittest import mock
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
sys.path.insert(0, HERE)
|
||||
sys.path.insert(0, os.path.join(HERE, ".."))
|
||||
import hub # noqa: E402
|
||||
import takes # noqa: E402
|
||||
from test_takes import HAVE, RID, make_round # noqa: E402
|
||||
|
||||
|
||||
class Pr:
|
||||
def __init__(self, num, status="draft"):
|
||||
self.num, self.status, self.is_pull_request = num, status, True
|
||||
self.url = f"https://huggingface.co/datasets/a/b/discussions/{num}"
|
||||
|
||||
|
||||
@unittest.skipUnless(HAVE, "needs zstd and the cryptography library (the dev container)")
|
||||
class HubTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.mkdtemp(prefix="eyerec-hub-")
|
||||
base = os.path.join(self.tmp, "contrib")
|
||||
make_round(base)
|
||||
self.contributor = str(uuid.uuid4())
|
||||
with open(os.path.join(base, "profile.json"), "w") as f:
|
||||
json.dump({"contributor": self.contributor,
|
||||
"consent": {"version": "test-1", "adult": True, "region_ok": True, "accepted": "2026-10-14"}}, f)
|
||||
self.store = takes.Store(base)
|
||||
meta = hub.meta_path(self.store, RID)
|
||||
with open(meta) as f:
|
||||
m = json.load(f)
|
||||
m.pop("uploads") # the fixture's, for the export's test
|
||||
with open(meta, "w") as f:
|
||||
json.dump(m, f)
|
||||
self.store.export(RID, low_priority=False) # to recipients.txt's keys, as validate wants
|
||||
|
||||
def tearDown(self):
|
||||
shutil.rmtree(self.tmp, ignore_errors=True)
|
||||
|
||||
def test_dry_run(self):
|
||||
logged = []
|
||||
with mock.patch.dict(os.environ, {hub.DATASET_ENV: "someone/test-eyes"}):
|
||||
result = hub.upload(self.store, RID, dry_run=True, log=logged.append)
|
||||
self.assertTrue(result["dry_run"])
|
||||
self.assertEqual(result["repo"], "someone/test-eyes")
|
||||
self.assertEqual(result["path_in_repo"], f"contributions/{self.contributor}/{RID}")
|
||||
self.assertIn("- Bounty: edges", result["commit_description"])
|
||||
self.assertIn("- Health answers: included, encrypted", result["commit_description"])
|
||||
self.assertTrue(any("health.json.age" in line for line in logged))
|
||||
self.assertEqual(hub.rhub.uploads(hub.meta_path(self.store, RID)), [])
|
||||
|
||||
def test_gate(self):
|
||||
self.assertFalse(hub.texts_ready(), "the texts are drafts")
|
||||
with mock.patch.dict(os.environ, {hub.ALLOW_ENV: "1", hub.DATASET_ENV: ""}):
|
||||
self.assertFalse(hub.upload_allowed(hub.dataset_id()), "never the real dataset before the review")
|
||||
with self.assertRaises(hub.HubError) as cm:
|
||||
hub.upload(self.store, RID)
|
||||
self.assertEqual(cm.exception.kind, "closed")
|
||||
with mock.patch.dict(os.environ, {hub.ALLOW_ENV: "1", hub.DATASET_ENV: "a/test"}):
|
||||
self.assertTrue(hub.upload_allowed(hub.dataset_id()))
|
||||
with mock.patch.dict(os.environ, {hub.ALLOW_ENV: "", hub.DATASET_ENV: "a/test"}):
|
||||
self.assertFalse(hub.upload_allowed(hub.dataset_id()))
|
||||
|
||||
def test_invalid_blocks(self):
|
||||
with open(os.path.join(self.store.export_dir(RID), "health.json"), "w") as f:
|
||||
f.write("{}")
|
||||
with self.assertRaises(hub.HubError) as cm:
|
||||
hub.upload(self.store, RID, dry_run=True)
|
||||
self.assertEqual(cm.exception.kind, "invalid")
|
||||
|
||||
def test_pull_request_first(self):
|
||||
calls = []
|
||||
|
||||
class Api:
|
||||
fail = True
|
||||
|
||||
def whoami(self):
|
||||
return {"name": "someone", "auth": {"accessToken": {"role": "write"}}}
|
||||
|
||||
def auth_check(self, repo, repo_type=None):
|
||||
pass
|
||||
|
||||
def create_pull_request(self, repo, title, description=None, repo_type=None):
|
||||
calls.append("create")
|
||||
return Pr(3)
|
||||
|
||||
def get_discussion_details(self, repo, num, repo_type=None):
|
||||
return Pr(num)
|
||||
|
||||
def upload_folder(self, **kw):
|
||||
calls.append(("upload", kw["revision"], kw["path_in_repo"]))
|
||||
if Api.fail:
|
||||
raise ConnectionResetError("reset")
|
||||
|
||||
def change_discussion_status(self, repo, num, status, repo_type=None):
|
||||
calls.append(("status", num, status))
|
||||
|
||||
env = {hub.DATASET_ENV: "a/b", hub.ALLOW_ENV: "1"}
|
||||
meta = hub.meta_path(self.store, RID)
|
||||
before = os.stat(meta).st_mtime_ns
|
||||
with mock.patch.dict(os.environ, env), mock.patch.object(hub, "_hf", return_value=mock.Mock(HfApi=Api)):
|
||||
with self.assertRaises(hub.HubError) as cm:
|
||||
hub.upload(self.store, RID)
|
||||
self.assertEqual(cm.exception.kind, "network")
|
||||
[rec] = hub.rhub.uploads(meta)
|
||||
self.assertEqual((rec["status"], rec["pr_num"]), ("started", 3))
|
||||
Api.fail = False
|
||||
result = hub.upload(self.store, RID)
|
||||
with self.assertRaises(hub.HubError) as cm:
|
||||
hub.upload(self.store, RID, dry_run=True)
|
||||
self.assertEqual(cm.exception.kind, "duplicate")
|
||||
where = f"contributions/{self.contributor}/{RID}"
|
||||
self.assertEqual(calls, ["create", ("upload", "refs/pr/3", where), ("upload", "refs/pr/3", where),
|
||||
("status", 3, "open")])
|
||||
self.assertEqual(result["pr_num"], 3)
|
||||
self.assertEqual(os.stat(meta).st_mtime_ns, before, "the record doesn't make the export look stale")
|
||||
self.assertFalse(self.store.rounds()[0]["export_stale"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,173 @@
|
||||
#!/usr/bin/env python3
|
||||
"""The gaze panel's rec mode (gaze/panel/ft-gazepanel.cpp, "show rec") without SteamVR (--no-vr,
|
||||
a made-up 90 Hz vsync): its commands, and targets.jsonl, the ground truth of where the dot was
|
||||
when. Needs the built panel (gaze/build.sh); skipped without it.
|
||||
|
||||
python3 gaze/rec/tests/test_panel.py
|
||||
"""
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import shutil
|
||||
import socket
|
||||
import subprocess
|
||||
import tempfile
|
||||
import time
|
||||
import unittest
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
PANEL = os.path.join(HERE, "..", "..", "build", "ft-gazepanel")
|
||||
IN_BOX = os.path.expanduser("~/.local/bin/distrobox")
|
||||
|
||||
|
||||
def command():
|
||||
"""The panel's argv: in the dev container on the Frame's host, where it was built."""
|
||||
if os.path.exists("/run/.containerenv") or not os.path.exists(IN_BOX):
|
||||
return [PANEL]
|
||||
return [IN_BOX, "enter", "dev", "--", PANEL]
|
||||
|
||||
|
||||
@unittest.skipUnless(os.path.exists(PANEL), "the panel isn't built (gaze/build.sh)")
|
||||
class RecPanelTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.mkdtemp(prefix="gazepanel-test-")
|
||||
self.name = f"ft_gazepanel_test_{os.getpid()}"
|
||||
self.proc = subprocess.Popen(command() + ["--no-vr", "--watch-stdin", "--socket", self.name, "--dump",
|
||||
os.path.join(self.tmp, "pics")],
|
||||
stdin=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||
self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM)
|
||||
self.sock.bind("")
|
||||
self.sock.settimeout(1.0)
|
||||
end = time.monotonic() + 20
|
||||
while True:
|
||||
try:
|
||||
if self.ask("ping").startswith("ok"):
|
||||
break
|
||||
except OSError:
|
||||
pass
|
||||
if time.monotonic() > end:
|
||||
self.fail("the panel didn't start")
|
||||
time.sleep(0.1)
|
||||
|
||||
def tearDown(self):
|
||||
self.proc.stdin.close()
|
||||
try:
|
||||
self.proc.wait(5)
|
||||
except subprocess.TimeoutExpired:
|
||||
self.proc.kill()
|
||||
self.proc.stderr.close()
|
||||
self.sock.close()
|
||||
shutil.rmtree(self.tmp, ignore_errors=True)
|
||||
|
||||
def ask(self, cmd):
|
||||
self.sock.sendto(cmd.encode(), "\0" + self.name)
|
||||
return self.sock.recv(4096).decode()
|
||||
|
||||
def rows(self):
|
||||
with open(os.path.join(self.tmp, "targets.jsonl")) as f:
|
||||
return [json.loads(line) for line in f]
|
||||
|
||||
def test_rec_lines_and_errors(self):
|
||||
for cmd in ("show rec", "bg 0.3", "title Nine dots", "step Round 1 of 5", "rec on", "text Look at each dot|as it comes",
|
||||
"big 3", "action Ready? Press the side button", "note Your left eye is hard to see",
|
||||
"keys Space: go · P: pause", "countdown 0.4", "paused on", "paused off", "countdown off"):
|
||||
self.assertEqual(self.ask(cmd), "ok", cmd)
|
||||
self.assertTrue(self.ask("path spiral 0 0 1 2").startswith("error"))
|
||||
self.assertTrue(self.ask("path circle 0 0 10 0").startswith("error"), "seconds must be positive")
|
||||
self.assertTrue(self.ask("log start /nonexistent/dir").startswith("error"))
|
||||
self.assertEqual(self.ask("ipd"), "ok 0.06300")
|
||||
time.sleep(0.2)
|
||||
self.assertTrue(os.listdir(os.path.join(self.tmp, "pics")), "pictures were drawn")
|
||||
self.ask("show quick")
|
||||
self.assertTrue(self.ask("path circle 0 0 10 2").startswith("error"), "only in rec")
|
||||
|
||||
def test_fixed_dot_logs_its_changes(self):
|
||||
self.ask("show rec")
|
||||
self.assertEqual(self.ask(f"log start {self.tmp}"), "ok")
|
||||
self.ask("dot 10 -5 look")
|
||||
time.sleep(0.15)
|
||||
self.ask("dot 10 -5 capture 0.5")
|
||||
time.sleep(0.15)
|
||||
self.ask("bg 0.8")
|
||||
time.sleep(0.15)
|
||||
self.ask("dot 0 0 off")
|
||||
time.sleep(0.15)
|
||||
reply = self.ask("log stop").split()
|
||||
rows = self.rows()
|
||||
self.assertEqual(int(reply[1]), len(rows))
|
||||
on = [r for r in rows if r["dot"] == "fixed"]
|
||||
self.assertEqual([(r["yaw"], r["pitch"], r["state"]) for r in on],
|
||||
[(10, -5, "look"), (10, -5, "capture"), (10, -5, "capture")])
|
||||
self.assertEqual(on[-1]["bg"], 0.8)
|
||||
self.assertEqual(rows[-1]["dot"], "off")
|
||||
for r in rows:
|
||||
self.assertGreater(r["t"], r["set"], "the photons come after it's set")
|
||||
self.assertLess(r["t"] - r["set"], 0.03)
|
||||
# The RAW clock, as the eye frames' (this process's CLOCK_MONOTONIC_RAW).
|
||||
self.assertLess(abs(rows[-1]["set"] - time.clock_gettime(time.CLOCK_MONOTONIC_RAW)), 5)
|
||||
|
||||
def test_path_every_frame(self):
|
||||
self.ask("show rec")
|
||||
self.ask(f"log start {self.tmp}")
|
||||
self.assertEqual(self.ask("path circle 2 1 10 1.0"), "ok")
|
||||
time.sleep(1.2)
|
||||
self.ask("dot 0 0 off")
|
||||
self.ask("log stop")
|
||||
rows = [r for r in self.rows() if r["dot"] == "path"]
|
||||
ts = [r["t"] for r in rows]
|
||||
self.assertGreater(len(rows), 80, "about one a frame")
|
||||
gaps = [b - a for a, b in zip(ts, ts[1:])]
|
||||
self.assertTrue(all(g > 0 for g in gaps))
|
||||
self.assertAlmostEqual(sorted(gaps)[len(gaps) // 2], 1 / 90, delta=0.002)
|
||||
self.assertEqual(len({r["frame"] for r in rows}), len(rows), "one row a frame")
|
||||
for r in rows:
|
||||
self.assertAlmostEqual(math.hypot(r["yaw"] - 2, r["pitch"] - 1), 10, delta=1e-3)
|
||||
first = rows[0]
|
||||
self.assertAlmostEqual((first["yaw"], first["pitch"])[1], 11, delta=0.5, msg="it starts at the top")
|
||||
# A lap a second: back near the top a second later, having gone left (yaw up) first.
|
||||
later = min(rows, key=lambda r: abs(r["t"] - first["t"] - 1.0))
|
||||
self.assertAlmostEqual(later["pitch"], 11, delta=0.5)
|
||||
self.assertGreater(rows[5]["yaw"], 2)
|
||||
|
||||
def test_line_and_eight(self):
|
||||
self.ask("show rec")
|
||||
self.ask(f"log start {self.tmp}")
|
||||
self.ask("path line -10 0 10 0 0.5")
|
||||
time.sleep(0.8)
|
||||
self.ask("path eight 0 0 15 8 1")
|
||||
time.sleep(0.6)
|
||||
self.ask("log stop")
|
||||
rows = self.rows()
|
||||
line = [r for r in rows if r["dot"] == "path" and abs(r["pitch"]) < 1e-6 and abs(r["yaw"]) <= 10 + 1e-6]
|
||||
self.assertGreater(len(line), 60)
|
||||
self.assertGreater(max(r["yaw"] for r in line), 9)
|
||||
self.assertLess(min(r["yaw"] for r in line), -9.5)
|
||||
eight = [r for r in rows if r["t"] > line[-1]["t"]]
|
||||
self.assertTrue(all(abs(r["yaw"]) <= 15 + 1e-6 and abs(r["pitch"]) <= 8 + 1e-6 for r in eight if r["dot"] == "path"))
|
||||
|
||||
def test_world_dot_and_distance(self):
|
||||
self.ask("show rec")
|
||||
self.ask(f"log start {self.tmp}")
|
||||
reply = self.ask("world 0.5 0 -1")
|
||||
self.assertTrue(reply.startswith("ok"), reply)
|
||||
self.assertEqual([round(float(v), 3) for v in reply.split()[1:]], [0.5, 0.0, -1.0])
|
||||
time.sleep(0.2)
|
||||
again = self.ask("world 0.5 0 -1 capture 0.5")
|
||||
self.assertEqual(again, reply, "the same point again: only the state changes")
|
||||
time.sleep(0.1)
|
||||
self.ask("dot 0 0 look")
|
||||
self.assertEqual(self.ask("distance 0.5"), "ok")
|
||||
time.sleep(0.1)
|
||||
self.ask("log stop")
|
||||
rows = self.rows()
|
||||
world = [r for r in rows if r["dot"] == "world"]
|
||||
self.assertGreater(len(world), 10)
|
||||
# --no-vr's head is at the origin looking down -z: half a metre right of a metre ahead.
|
||||
self.assertAlmostEqual(world[0]["yaw"], -math.degrees(math.atan2(0.5, 1)), delta=0.01)
|
||||
self.assertAlmostEqual(world[0]["dist"], math.hypot(0.5, 1), delta=0.001)
|
||||
self.assertEqual(rows[-1]["dist"], 0.5)
|
||||
self.assertEqual(rows[-1]["dot"], "fixed")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,72 @@
|
||||
#!/usr/bin/env python3
|
||||
"""main.qml against ft_eyerec.Backend: every backend.name(...) the window calls is a slot, and
|
||||
every backend.name it reads is a property or a slot. A method that lost its @Slot shows up in
|
||||
QML only as "is not a function" when its button is pressed (the hand recorder's Export, 2026-10-03).
|
||||
Needs PySide6 (the dev container); skipped without it. FT_EYEREC_QML=PATH checks another
|
||||
window against the same backend (the standalone Eye Recorder's, in its CI).
|
||||
|
||||
python3 gaze/rec/tests/test_qml_backend.py
|
||||
"""
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import unittest
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
REC = os.path.dirname(HERE)
|
||||
sys.path.insert(0, REC)
|
||||
QML = os.environ.get("FT_EYEREC_QML") or os.path.join(REC, "main.qml")
|
||||
|
||||
|
||||
class QmlBackendTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
try:
|
||||
import ft_eyerec
|
||||
except ImportError as e:
|
||||
self.skipTest(f"no PySide6: {e}")
|
||||
meta = self.meta = ft_eyerec.Backend.staticMetaObject
|
||||
self.slots = {bytes(meta.method(i).name()).decode() for i in range(meta.methodCount())}
|
||||
self.props = {meta.property(i).name() for i in range(meta.propertyCount())}
|
||||
with open(QML) as f:
|
||||
self.qml = f.read()
|
||||
|
||||
def test_calls_are_slots(self):
|
||||
called = set(re.findall(r"\bbackend\.(\w+)\s*\(", self.qml))
|
||||
self.assertTrue(called)
|
||||
self.assertEqual(sorted(called - self.slots), [], "called from main.qml but not a slot")
|
||||
|
||||
def test_call_arity(self):
|
||||
"""Each backend.name(a, b) call passes as many arguments as some slot of that name takes
|
||||
(a decorator left at the old count fails only when the button is pressed)."""
|
||||
arity = {}
|
||||
meta = self.meta
|
||||
for i in range(meta.methodCount()):
|
||||
mm = meta.method(i)
|
||||
arity.setdefault(bytes(mm.name()).decode(), set()).add(mm.parameterCount())
|
||||
bad = []
|
||||
for m in re.finditer(r"\bbackend\.(\w+)\s*\(", self.qml):
|
||||
depth, args, k, seen = 1, 0, m.end(), False
|
||||
while depth and k < len(self.qml):
|
||||
c = self.qml[k]
|
||||
if c in "([{":
|
||||
depth += 1
|
||||
elif c in ")]}":
|
||||
depth -= 1
|
||||
elif c == "," and depth == 1:
|
||||
args += 1
|
||||
elif not c.isspace() and depth >= 1:
|
||||
seen = True
|
||||
k += 1
|
||||
n = args + 1 if seen else 0
|
||||
if m.group(1) in arity and n not in arity[m.group(1)]:
|
||||
bad.append("%s: %d arguments, slots take %s" % (m.group(1), n, sorted(arity[m.group(1)])))
|
||||
self.assertEqual(bad, [])
|
||||
|
||||
def test_reads_exist(self):
|
||||
read = set(re.findall(r"\bbackend\.(\w+)\b(?!\s*\()", self.qml))
|
||||
self.assertTrue(read)
|
||||
self.assertEqual(sorted(read - self.props - self.slots), [], "read in main.qml but not on Backend")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,297 @@
|
||||
#!/usr/bin/env python3
|
||||
"""The eye recorder's session runner (gaze/rec/session.py) without the headset: the round
|
||||
scripts' checks, the plans, the control sockets' answers, and whole rounds in a dry run.
|
||||
|
||||
python3 gaze/rec/tests/test_session.py
|
||||
"""
|
||||
import glob
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import socket
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
import unittest
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
sys.path.insert(0, os.path.join(HERE, ".."))
|
||||
import session # noqa: E402
|
||||
|
||||
|
||||
def write_round(tmp, steps, **extra):
|
||||
d = dict({"schema": 1, "id": "t", "title": "Test", "steps": steps}, **extra)
|
||||
path = os.path.join(tmp, "t.json")
|
||||
with open(path, "w") as f:
|
||||
json.dump(d, f)
|
||||
return path
|
||||
|
||||
|
||||
class RoundTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.mkdtemp(prefix="eyerec-round-")
|
||||
|
||||
def tearDown(self):
|
||||
shutil.rmtree(self.tmp, ignore_errors=True)
|
||||
|
||||
def bad(self, steps, want):
|
||||
with self.assertRaises(ValueError) as e:
|
||||
session.load_round(write_round(self.tmp, steps))
|
||||
self.assertIn(want, str(e.exception))
|
||||
|
||||
def test_shipped_rounds_load(self):
|
||||
paths = sorted(glob.glob(os.path.join(session.ROUNDS_DIR, "*.json")))
|
||||
self.assertEqual(len(paths), 10)
|
||||
for p in paths:
|
||||
r = session.load_round(p)
|
||||
self.assertEqual(r["id"], os.path.basename(p)[:-5])
|
||||
self.assertTrue(r["steps"][0]["ready"], p)
|
||||
|
||||
def test_panel_reach(self):
|
||||
self.assertFalse(session.off_panel(0, 0))
|
||||
self.assertFalse(session.off_panel(30, 20))
|
||||
self.assertTrue(session.off_panel(38, 0))
|
||||
self.assertTrue(session.off_panel(0, 26))
|
||||
self.assertTrue(session.off_panel(30, 23), "corners reach less far up")
|
||||
self.bad([{"id": "a", "kind": "dots", "dots": [[40, 0]]}], "off the panel")
|
||||
self.bad([{"id": "a", "kind": "path", "path": "circle 0 0 26 4", "seconds": 4}], "off the panel")
|
||||
self.bad([{"id": "a", "kind": "jumps", "count": 3, "range": [40, 10], "each": [0.5, 1]}], "off the panel")
|
||||
|
||||
def test_bad_steps(self):
|
||||
self.bad([{"id": "a", "kind": "dance"}], "unknown kind")
|
||||
self.bad([{"id": "A b", "kind": "dots", "dots": [[0, 0]]}], "bad id")
|
||||
self.bad([{"id": "a", "kind": "path", "path": "spiral 0 0 1 2", "seconds": 2}], "bad path")
|
||||
self.bad([{"id": "a", "kind": "path", "path": "circle 0 0 10 0", "seconds": 2}], "positive")
|
||||
self.bad([{"id": "a", "kind": "prompt"}], "seconds")
|
||||
self.bad([{"id": "a", "kind": "world", "point": [0, 0, 1], "seconds": 3}], "ahead")
|
||||
self.bad([{"id": "a", "kind": "dots", "dots": [[0, 0]]}, {"id": "a", "kind": "dots", "dots": [[0, 0]]}], "repeat")
|
||||
self.bad([{"id": "a", "kind": "dots", "group": "g", "dots": [[0, 0]]}, {"id": "b", "kind": "dots", "dots": [[0, 0]]},
|
||||
{"id": "c", "kind": "dots", "group": "g", "dots": [[0, 0]]}], "comes back")
|
||||
self.bad([{"id": "a", "kind": "dots", "group": "dots", "dots": [[0, 0]]}], "opener")
|
||||
|
||||
def test_grid_and_defaults(self):
|
||||
r = session.load_round(write_round(self.tmp, [{"id": "a", "kind": "dots", "grid": [3, 2, 10, 5], "ready": False},
|
||||
{"id": "b", "kind": "reseat", "text": "off and on"}]))
|
||||
a = r["steps"][0]
|
||||
self.assertEqual(a["dots"], [[10, 5], [0, 5], [-10, 5], [10, -5], [0, -5], [-10, -5]])
|
||||
self.assertTrue(a["ready"], "the first step always waits")
|
||||
self.assertEqual((a["bg"], a["distance"], a["group"]), (session.MID_BG, session.PANEL_DISTANCE, "a"))
|
||||
self.assertFalse(r["shift_check"], "a round that reseats on purpose has no shift check")
|
||||
|
||||
def test_plan(self):
|
||||
rnd = session.load_round(os.path.join(session.ROUNDS_DIR, "jumps.json"))
|
||||
a = session.build_plan(rnd, 7, ipd_sweep=True)
|
||||
self.assertEqual([s["group"] for s in a[:3]], ["ipd", "dots", "jumps"])
|
||||
nine = a[1]["dots"]
|
||||
self.assertEqual((nine[0], nine[-1], len(nine)), ([0.0, 0.0], [0.0, 0.0], 10))
|
||||
self.assertEqual(sorted(map(tuple, nine[1:-1])), sorted(map(tuple, (d for d in session.grid(3, 3, 20, 20) if d != [0, 0]))))
|
||||
jumps = a[2]
|
||||
self.assertEqual(len(jumps["dots"]), 40)
|
||||
self.assertTrue(all(0.5 <= t <= 1.5 for t in jumps["times"]))
|
||||
self.assertTrue(all(not session.off_panel(*d) for d in jumps["dots"]))
|
||||
self.assertEqual(a, session.build_plan(rnd, 7, ipd_sweep=True), "the same seed, the same round")
|
||||
self.assertNotEqual(a[1]["dots"], session.build_plan(rnd, 8, ipd_sweep=True)[1]["dots"])
|
||||
self.assertEqual(session.build_plan(rnd, 7, ipd_sweep=False)[0]["group"], "dots")
|
||||
self.assertGreater(session.plan_seconds(a), 60)
|
||||
|
||||
|
||||
class SocketTest(unittest.TestCase):
|
||||
"""The panel and ft-eyes answer every command in order: an answer waited for is the one
|
||||
after the answers of commands sent before it."""
|
||||
|
||||
def setUp(self):
|
||||
self.name = "eyerec_socket_test_%d_%s" % (os.getpid(), self.id().rsplit(".", 1)[-1])
|
||||
self.server = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM)
|
||||
self.server.bind("\0" + self.name)
|
||||
self.server.settimeout(0.2)
|
||||
self.delay = 0.02
|
||||
self.drop = set()
|
||||
self.running = True
|
||||
threading.Thread(target=self.serve, daemon=True).start()
|
||||
|
||||
def tearDown(self):
|
||||
self.running = False
|
||||
self.server.close()
|
||||
|
||||
def serve(self):
|
||||
while self.running:
|
||||
try:
|
||||
data, addr = self.server.recvfrom(4096)
|
||||
except OSError:
|
||||
continue
|
||||
text = data.decode()
|
||||
time.sleep(self.delay)
|
||||
if text in self.drop:
|
||||
continue
|
||||
try:
|
||||
self.server.sendto(("ok " + text).encode(), addr)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
def test_answers_in_order(self):
|
||||
s = session.Socket(self.name)
|
||||
for n in range(5):
|
||||
s.cmd("dot %d" % n)
|
||||
self.assertEqual(s.cmd("ipd", reply=True), "ok ipd")
|
||||
self.assertEqual(s.pending, 0)
|
||||
s.cmd("bg 1")
|
||||
self.assertEqual(s.cmd("log stop", reply=True), "ok log stop")
|
||||
s.close()
|
||||
|
||||
def test_lost_answer(self):
|
||||
s = session.Socket(self.name)
|
||||
self.drop.add("fit")
|
||||
self.assertIsNone(s.cmd("fit", reply=True, timeout=0.2))
|
||||
self.assertEqual(s.cmd("ipd", reply=True), "ok ipd", "counting starts afresh")
|
||||
s.close()
|
||||
|
||||
def test_errors_logged(self):
|
||||
logs = []
|
||||
s = session.Socket(self.name, log=logs.append)
|
||||
s.cmd("x")
|
||||
time.sleep(0.1)
|
||||
s._got("error bad")
|
||||
self.assertEqual(logs, ["%s: error bad" % self.name])
|
||||
s.close()
|
||||
|
||||
|
||||
class DryRunTest(unittest.TestCase):
|
||||
"""Whole rounds with no processes: the files, the fitment, the controls."""
|
||||
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.mkdtemp(prefix="eyerec-dry-")
|
||||
self.out = []
|
||||
|
||||
def tearDown(self):
|
||||
shutil.rmtree(self.tmp, ignore_errors=True)
|
||||
|
||||
def run_round(self, script, steer=None, profile=None, **kw):
|
||||
path = os.path.join(self.tmp, "profile.json")
|
||||
with open(path, "w") as f:
|
||||
json.dump(profile or {}, f)
|
||||
states = []
|
||||
s = session.Session(self.tmp, script, profile=profile, dry_run=True, speed=kw.pop("speed", 40),
|
||||
next_after=kw.pop("next_after", 0.25), profile_path=path, button=False,
|
||||
on_status=lambda st: states.append(dict(st)), **kw)
|
||||
s.print = self.out.append
|
||||
s.start()
|
||||
if steer:
|
||||
steer(s, states)
|
||||
s.join(60)
|
||||
self.assertFalse(s._thread.is_alive())
|
||||
with open(path) as f:
|
||||
return s, states, json.load(f)
|
||||
|
||||
def files(self, s):
|
||||
with open(os.path.join(s.session_dir, "session.json")) as f:
|
||||
sj = json.load(f)
|
||||
with open(os.path.join(s.session_dir, "fit.json")) as f:
|
||||
fit = json.load(f)
|
||||
return sj, fit
|
||||
|
||||
def prompts(self, s, take):
|
||||
with open(os.path.join(s.session_dir, "takes", take, "prompts.jsonl")) as f:
|
||||
return [json.loads(line) for line in f]
|
||||
|
||||
def test_round_and_profile(self):
|
||||
s, states, profile = self.run_round("depth")
|
||||
self.assertEqual(s.state, "done")
|
||||
sj, fit = self.files(s)
|
||||
self.assertEqual(sj["takes"], ["01-ipd", "02-dots", "03-near", "04-mid", "05-far", "06-near-far"])
|
||||
self.assertEqual((sj["status"], sj["fit"], sj["ipd_sweep"], sj["ipd_start"]), ("done", "good", True, 0.063))
|
||||
self.assertEqual(fit["fitment"]["result"], "good")
|
||||
self.assertIsNotNone(fit["baseline"])
|
||||
self.assertEqual(profile["bounties"]["depth"]["rounds"], 1)
|
||||
self.assertEqual(profile["eyes_ipd"], 0.063)
|
||||
for take in sj["takes"]:
|
||||
with open(os.path.join(s.session_dir, "takes", take, "take.json")) as f:
|
||||
self.assertEqual(json.load(f)["status"], "complete", take)
|
||||
events = [e["event"] for e in self.prompts(s, "03-near")]
|
||||
self.assertEqual(events[:4], ["take", "ready", "rec", "step"])
|
||||
self.assertEqual(events[-3:], ["wait", "rec", "end"])
|
||||
self.assertIn("ft_eyerecpanel: distance 0.500", self.out)
|
||||
self.assertIn("ft_eyerecpanel: distance 5.000", self.out)
|
||||
# The IPD is the same next time: no sweep.
|
||||
s2, _, profile = self.run_round("depth", profile=profile)
|
||||
self.assertEqual(self.files(s2)[0]["takes"][0], "01-dots")
|
||||
self.assertEqual(profile["bounties"]["depth"]["rounds"], 2)
|
||||
|
||||
def test_fitment_warns_then_skips(self):
|
||||
# Next before the fit has been good for FIT_GOOD_S: a warning, then the round goes on.
|
||||
s, states, _ = self.run_round("reading", next_after=0.01, speed=20)
|
||||
sj, fit = self.files(s)
|
||||
self.assertEqual((sj["fit"], fit["fitment"]["result"]), ("skipped", "skipped"))
|
||||
self.assertTrue(any(st["note"] == session.FIT_SKIP_NOTE for st in states))
|
||||
self.assertIsNone(fit["baseline"])
|
||||
|
||||
def test_skip_redo_stop(self):
|
||||
def steer(s, states):
|
||||
def wait_for(cond):
|
||||
end = time.monotonic() + 30
|
||||
while time.monotonic() < end and not cond():
|
||||
time.sleep(0.01)
|
||||
# In the jumps take: redo the step while it runs, then skip the rest of the take.
|
||||
# (Its ready first: the step's name comes one emit before the dots take's "running" goes.)
|
||||
wait_for(lambda: s._status["step"] == "jumps" and s._status["state"] == "ready")
|
||||
wait_for(lambda: s._status["step"] == "jumps" and s._status["state"] == "running")
|
||||
s.redo()
|
||||
wait_for(lambda: s._status["step"] == "jumps" and s._status["state"] == "ready")
|
||||
wait_for(lambda: s._status["step"] == "jumps" and s._status["state"] == "running")
|
||||
s.skip()
|
||||
wait_for(lambda: s._status["step"] == "circle-slow" and s._status["state"] == "running")
|
||||
s.stop(wait=0)
|
||||
s, states, profile = self.run_round("jumps", steer=steer, speed=10, profile={"eyes_ipd": 0.063})
|
||||
self.assertEqual(s.state, "stopped")
|
||||
sj, _ = self.files(s)
|
||||
self.assertEqual(sj["takes"], ["01-dots", "02-jumps", "03-pursuit"])
|
||||
with open(os.path.join(s.session_dir, "takes", "02-jumps", "take.json")) as f:
|
||||
self.assertEqual(json.load(f)["status"], "skipped")
|
||||
with open(os.path.join(s.session_dir, "takes", "03-pursuit", "take.json")) as f:
|
||||
self.assertEqual(json.load(f)["status"], "stopped")
|
||||
redo = [e for e in self.prompts(s, "02-jumps") if e["event"] == "redo"]
|
||||
self.assertEqual(len(redo), 1)
|
||||
self.assertLess(redo[0]["from"], redo[0]["to"])
|
||||
self.assertNotIn("bounties", profile, "a stopped round isn't counted")
|
||||
|
||||
def test_pause_stops_the_recording(self):
|
||||
def steer(s, states):
|
||||
end = time.monotonic() + 30
|
||||
while time.monotonic() < end and not (s._status["step"] == "dots" and s._status["state"] == "running"):
|
||||
time.sleep(0.01)
|
||||
s.pause()
|
||||
time.sleep(0.3)
|
||||
self.assertFalse(s._recording)
|
||||
s.resume()
|
||||
s, _, _ = self.run_round("face", steer=steer, speed=10, profile={"eyes_ipd": 0.063})
|
||||
self.assertEqual(s.state, "done")
|
||||
events = [e["event"] for e in self.prompts(s, "01-dots")]
|
||||
i = events.index("pause")
|
||||
self.assertEqual(events[i - 1:i + 3], ["rec", "pause", "resume", "rec"])
|
||||
|
||||
def test_shift_check(self):
|
||||
moved = {"on": False}
|
||||
|
||||
def steer(s, states):
|
||||
# Once the fitment is done, the eyes appear 12 degrees up: the headset slipped down.
|
||||
end = time.monotonic() + 30
|
||||
while time.monotonic() < end and s._status["state"] in ("starting", "fitment"):
|
||||
time.sleep(0.01)
|
||||
ref = s.ref
|
||||
summary = {str(cam): {"n": 90, "finds": 90, "found": 1.0,
|
||||
"x": ref.eyes[cam]["centre"][0] + ref.eyes[cam]["jacobian"][0][1] * 12,
|
||||
"y": ref.eyes[cam]["centre"][1] + ref.eyes[cam]["jacobian"][1][1] * 12,
|
||||
"a": 28.0, "spacing": 15.0, "extra": 0.0} for cam in (0, 1)}
|
||||
s._eyes.fake = lambda text: json.dumps(summary) if text == "fit" else "ok"
|
||||
moved["on"] = True
|
||||
s, states, _ = self.run_round("glasses", steer=steer, profile={"eyes_ipd": 0.063})
|
||||
self.assertTrue(moved["on"])
|
||||
_, fit = self.files(s)
|
||||
self.assertGreaterEqual(len(fit["slips"]), 2, "the nine dots' middle, twice, and the grid's")
|
||||
self.assertIn("up", fit["slips"][0]["hints"])
|
||||
self.assertTrue(any(st["note"].startswith("The headset moved") for st in states))
|
||||
self.assertTrue(any(e["event"] == "slip" for e in self.prompts(s, "01-dots")))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,227 @@
|
||||
#!/usr/bin/env python3
|
||||
"""The eye recorder's store and encrypted export (gaze/rec/takes.py), on a made-up round of
|
||||
random frames (never real eye images): listing, deleted ranges across the two clocks, the export's
|
||||
chunks, logs and checksums, and unpacking it with a key. Needs zstd (Python 3.14's, as eyerec.py)
|
||||
and the cryptography library: the dev container has both.
|
||||
|
||||
python3 gaze/rec/tests/test_takes.py
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
sys.path.insert(0, os.path.join(HERE, ".."))
|
||||
import eyerec # noqa: E402
|
||||
import takes # noqa: E402
|
||||
from recorder import age # noqa: E402
|
||||
|
||||
try:
|
||||
age._crypto()
|
||||
HAVE = eyerec.compress is not None
|
||||
except RuntimeError:
|
||||
HAVE = False
|
||||
|
||||
RID = "20261014-120000-edges"
|
||||
TID = "01-dots"
|
||||
MONO0 = 1000 * 10**9 # the take starts at 1000 s MONOTONIC
|
||||
OFF = 5 * 10**9 + 123 # RAW is 5 s ahead
|
||||
FPS = 90.0
|
||||
PER_CHUNK = 10
|
||||
CHUNKS = 3
|
||||
|
||||
|
||||
def raw_s(mono_ns):
|
||||
return (mono_ns + OFF) / 1e9
|
||||
|
||||
|
||||
def make_round(base, health=True):
|
||||
"""A round with one take: CHUNKS chunks of PER_CHUNK random frames per camera, frame k of
|
||||
camera c at MONO0 + k / FPS (plus 1 ms for camera 1). Returns {(c, k): frame bytes}."""
|
||||
rdir = os.path.join(base, "rounds", RID)
|
||||
tdir = os.path.join(rdir, "takes", TID)
|
||||
frames = {}
|
||||
lines = [eyerec.INDEX_HEADER]
|
||||
for c in eyerec.CAMERAS:
|
||||
os.makedirs(os.path.join(tdir, "cam%d" % c))
|
||||
for chunk in range(CHUNKS):
|
||||
data = []
|
||||
for i in range(PER_CHUNK):
|
||||
k = chunk * PER_CHUNK + i
|
||||
frames[(c, k)] = os.urandom(eyerec.FRAME)
|
||||
data.append(frames[(c, k)])
|
||||
lines.append("%d\t%d\t%d\t%d\t%.6f\n" % (c, chunk, i, k % 8, raw_s(MONO0 + int(k / FPS * 1e9)) + c * 0.001))
|
||||
with open(os.path.join(tdir, "cam%d" % c, "%06d.zst" % chunk), "wb") as f:
|
||||
f.write(eyerec.compress(b"".join(data)))
|
||||
with open(os.path.join(tdir, "index.tsv"), "w") as f:
|
||||
f.writelines(lines)
|
||||
end = MONO0 + int(CHUNKS * PER_CHUNK / FPS * 1e9)
|
||||
with open(os.path.join(tdir, "take.json"), "w") as f:
|
||||
json.dump({"group": "dots", "started_ns": MONO0, "ended_ns": end, "status": "complete", "steps": [],
|
||||
"deleted": [], "clock": [[MONO0, OFF], [end, OFF]], "notes": ""}, f)
|
||||
with open(os.path.join(tdir, "own.jsonl"), "w") as f:
|
||||
for k in range(CHUNKS * PER_CHUNK):
|
||||
f.write(json.dumps({"t": raw_s(MONO0 + int(k / FPS * 1e9)), "cam": 0, "x": 1.0}) + "\n")
|
||||
with open(os.path.join(tdir, "prompts.jsonl"), "w") as f:
|
||||
f.write(json.dumps({"t": MONO0, "event": "take"}) + "\n")
|
||||
with open(os.path.join(tdir, "targets.jsonl"), "w") as f:
|
||||
f.write(json.dumps({"t": raw_s(MONO0), "dot": "on", "yaw": 0, "pitch": 0}) + "\n")
|
||||
with open(os.path.join(rdir, "session.json"), "w") as f:
|
||||
json.dump({"schema": 1, "round": RID, "bounty": "edges", "status": "done", "takes": [TID],
|
||||
"started": "2026-10-14T12:00:00+0000", "checklist": {"glasses": "none"},
|
||||
"uploads": [{"pr": 1}]}, f)
|
||||
with open(os.path.join(rdir, "fit.json"), "w") as f:
|
||||
json.dump({"fitment": {"state": "ok"}}, f)
|
||||
with open(os.path.join(rdir, "session.log"), "w") as f:
|
||||
f.write("a log that names /home/someone\n")
|
||||
if health:
|
||||
with open(os.path.join(rdir, "health.json"), "w") as f:
|
||||
json.dump({"conditions": ["test"]}, f)
|
||||
with open(os.path.join(base, "profile.json"), "w") as f:
|
||||
json.dump({"contributor": "c0ffee", "consent": {"version": "test-1"}, "health": {"x": 1},
|
||||
"optional": {"notes": "private"}}, f)
|
||||
return frames
|
||||
|
||||
|
||||
def load(path, mode="r"):
|
||||
with open(path, mode) as f:
|
||||
return json.load(f) if path.endswith(".json") else f.read()
|
||||
|
||||
|
||||
def frame_mono(k):
|
||||
return MONO0 + int(k / FPS * 1e9)
|
||||
|
||||
|
||||
@unittest.skipUnless(HAVE, "needs zstd and the cryptography library (the dev container)")
|
||||
class TakesTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.mkdtemp(prefix="eyerec-takes-")
|
||||
self.base = os.path.join(self.tmp, "contrib")
|
||||
self.frames = make_round(self.base)
|
||||
self.store = takes.Store(self.base)
|
||||
self.ident, self.rec = age.keygen()
|
||||
self.key = os.path.join(self.tmp, "key.txt")
|
||||
with open(self.key, "w") as f:
|
||||
f.write("# test key\n" + self.ident + "\n")
|
||||
|
||||
def tearDown(self):
|
||||
shutil.rmtree(self.tmp, ignore_errors=True)
|
||||
|
||||
def test_ids(self):
|
||||
for bad in ("../x", "20261014-120000-../x", "", "x"):
|
||||
with self.assertRaises(ValueError):
|
||||
self.store.round_dir(bad)
|
||||
for bad in ("..", "01-a/b", ".x"):
|
||||
with self.assertRaises(ValueError):
|
||||
self.store.take_dir(RID, bad)
|
||||
self.store.round_dir("20261014-120000-edges-2")
|
||||
|
||||
def test_clock(self):
|
||||
clock = [[0, 100], [1000, 200]]
|
||||
self.assertEqual(takes.raw_offset_ns(-5, clock), 100)
|
||||
self.assertEqual(takes.raw_offset_ns(500, clock), 150)
|
||||
self.assertEqual(takes.raw_offset_ns(5000, clock), 200)
|
||||
self.assertEqual(takes.raw_offset_ns(5, []), 0)
|
||||
|
||||
def test_listing(self):
|
||||
[r] = self.store.rounds()
|
||||
self.assertEqual((r["id"], r["bounty"], r["takes"], r["exported"]), (RID, "edges", 1, False))
|
||||
[t] = self.store.takes(RID)
|
||||
self.assertEqual(t["frames"], [30, 30])
|
||||
self.assertEqual(t["deleted_frames"], [0, 0])
|
||||
self.assertAlmostEqual(t["duration_s"], 29 / FPS, places=2)
|
||||
|
||||
def test_ranges(self):
|
||||
# index.tsv's times are whole microseconds: the ranges have a millisecond to spare
|
||||
self.store.delete_range(RID, TID, frame_mono(12) - 10**6, frame_mono(25))
|
||||
self.store.delete_range(RID, TID, frame_mono(20), frame_mono(29) + 10**6)
|
||||
self.assertEqual(self.store.ranges(RID, TID), [[frame_mono(12) - 10**6, frame_mono(29) + 10**6]])
|
||||
[t] = self.store.takes(RID)
|
||||
self.assertEqual(t["deleted_frames"][0], 18)
|
||||
self.store.restore_range(RID, TID, 0)
|
||||
self.assertEqual(self.store.ranges(RID, TID), [])
|
||||
|
||||
def test_export_and_unpack(self):
|
||||
# frames 15..29 deleted: chunk 0 whole, chunk 1 half (10..14 kept), chunk 2 gone
|
||||
self.store.delete_range(RID, TID, frame_mono(15) - 1000, frame_mono(29) + 5 * 10**6)
|
||||
path = self.store.export(RID, recipients=[self.rec])
|
||||
self.assertFalse(os.path.exists(path + ".partial"))
|
||||
m = load(os.path.join(path, "manifest.json"))
|
||||
[e] = m["takes"]
|
||||
self.assertEqual(e["frames_deleted"][0], 15)
|
||||
self.assertEqual(e["frames"][0], 15)
|
||||
self.assertEqual(e["chunks"], 4)
|
||||
self.assertEqual(e["logs"]["own.jsonl"], {"rows": 15, "deleted": 15})
|
||||
self.assertNotIn("uploads", m["session"])
|
||||
self.assertNotIn("health", m["profile"])
|
||||
self.assertNotIn("notes", m["profile"]["optional"])
|
||||
self.assertEqual(m["encryption"]["recipients"], [self.rec])
|
||||
files = sorted(os.path.relpath(os.path.join(r, n), path) for r, _, ns in os.walk(path) for n in ns)
|
||||
self.assertIn("health.json.age", files)
|
||||
self.assertNotIn("health.json", files)
|
||||
self.assertNotIn("session.log", files)
|
||||
self.assertFalse([f for f in files if f.endswith(".zst")], "every chunk is encrypted")
|
||||
self.assertIn("takes/%s/cam0/000001.zst.age" % TID, files)
|
||||
self.assertNotIn("takes/%s/cam0/000002.zst.age" % TID, files)
|
||||
# chunk 0 untouched: its encrypted copy decrypts to the recorded file
|
||||
src = load(os.path.join(self.base, "rounds", RID, "takes", TID, "cam0", "000000.zst"), "rb")
|
||||
enc = load(os.path.join(path, "takes", TID, "cam0", "000000.zst.age"), "rb")
|
||||
self.assertEqual(age.decrypt_bytes(enc, self.ident), src)
|
||||
with open(os.path.join(path, "SHA256SUMS")) as f:
|
||||
self.assertEqual(sorted(line.split(" ", 1)[1].strip() for line in f), [f for f in files if f != "SHA256SUMS"])
|
||||
|
||||
out = takes.unpack(path, self.key, os.path.join(self.tmp, "unpacked"))
|
||||
self.assertEqual(load(os.path.join(out, "health.json")), {"conditions": ["test"]})
|
||||
got = {}
|
||||
tdir = os.path.join(out, "takes", TID)
|
||||
for c, t, slot, data in eyerec.iter_frames(tdir):
|
||||
got.setdefault(c, []).append(data)
|
||||
for c in eyerec.CAMERAS:
|
||||
self.assertEqual(got[c], [self.frames[(c, k)] for k in range(15)])
|
||||
self.assertEqual(load(os.path.join(tdir, "take.json"))["export"]["deleted_raw_s"][0][0],
|
||||
raw_s(frame_mono(15) - 1000))
|
||||
|
||||
[r] = self.store.rounds()
|
||||
self.assertTrue(r["exported"] and not r["export_stale"])
|
||||
|
||||
def test_default_recipients(self):
|
||||
path = self.store.export(RID)
|
||||
m = load(os.path.join(path, "manifest.json"))
|
||||
self.assertEqual(m["encryption"]["recipients"], age.read_recipients(takes.RECIPIENTS))
|
||||
with self.assertRaises(ValueError): # the test key isn't one of them
|
||||
takes.unpack(path, self.key, os.path.join(self.tmp, "no"))
|
||||
|
||||
def test_bad_chunks(self):
|
||||
tdir = os.path.join(self.base, "rounds", RID, "takes", TID)
|
||||
shutil.copyfile(os.path.join(tdir, "cam0", "000000.zst"), os.path.join(tdir, "cam0", "000007.zst"))
|
||||
with open(os.path.join(tdir, "cam1", "000001.zst"), "r+b") as f:
|
||||
f.truncate(100)
|
||||
os.remove(os.path.join(tdir, "cam1", "000002.zst"))
|
||||
path = self.store.export(RID, recipients=[self.rec])
|
||||
[e] = load(os.path.join(path, "manifest.json"))["takes"]
|
||||
self.assertEqual(e["left_out"], {"not_indexed": 1, "missing": 1, "bad": 1})
|
||||
self.assertEqual(e["frames"], [30, 10])
|
||||
with open(os.path.join(path, "takes", TID, "index.tsv")) as f:
|
||||
self.assertEqual(sum(1 for line in f if line.startswith("1\t")), 10)
|
||||
|
||||
def test_cancel(self):
|
||||
with self.assertRaises(takes.Cancelled):
|
||||
self.store.export(RID, recipients=[self.rec], cancel=lambda: True)
|
||||
self.assertFalse(os.path.exists(self.store.export_dir(RID) + ".partial"))
|
||||
self.assertFalse(os.path.exists(self.store.export_dir(RID)))
|
||||
|
||||
def test_delete(self):
|
||||
self.store.export(RID, recipients=[self.rec])
|
||||
self.store.delete_take(RID, TID)
|
||||
meta = load(os.path.join(self.base, "rounds", RID, "session.json"))
|
||||
self.assertEqual((meta["takes"], meta["deleted_takes"]), ([], [TID]))
|
||||
self.store.delete_round(RID)
|
||||
self.assertEqual(self.store.rounds(), [])
|
||||
self.assertFalse(os.path.exists(self.store.export_dir(RID)))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,150 @@
|
||||
#!/usr/bin/env python3
|
||||
"""The eye recorder's export check (gaze/rec/validate.py) on takes.py's export of a made-up
|
||||
round of random frames (test_takes.make_round): a good one, with and without a key, and broken
|
||||
ones. Needs zstd and the cryptography library: the dev container has both.
|
||||
|
||||
python3 gaze/rec/tests/test_validate.py
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
import uuid
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
sys.path.insert(0, HERE)
|
||||
sys.path.insert(0, os.path.join(HERE, ".."))
|
||||
import takes # noqa: E402
|
||||
import validate # noqa: E402
|
||||
from recorder import age # noqa: E402
|
||||
from recorder.files import write_sums # noqa: E402
|
||||
from test_takes import HAVE, RID, TID, make_round # noqa: E402
|
||||
|
||||
|
||||
@unittest.skipUnless(HAVE, "needs zstd and the cryptography library (the dev container)")
|
||||
class ValidateTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.mkdtemp(prefix="eyerec-validate-")
|
||||
base = os.path.join(self.tmp, "contrib")
|
||||
make_round(base)
|
||||
self.contributor = str(uuid.uuid4())
|
||||
with open(os.path.join(base, "profile.json"), "w") as f:
|
||||
json.dump({"contributor": self.contributor,
|
||||
"consent": {"version": "test-1", "adult": True, "region_ok": True, "accepted": "2026-10-14"},
|
||||
"bounties": {}}, f)
|
||||
self.ident, self.rec = age.keygen()
|
||||
self.keys = os.path.join(self.tmp, "recipients.txt")
|
||||
with open(self.keys, "w") as f:
|
||||
f.write("# test\n%s\n" % self.rec)
|
||||
self.store = takes.Store(base)
|
||||
self.path = self.store.export(RID, recipients=[self.rec])
|
||||
self.take = os.path.join(self.path, "takes", TID)
|
||||
|
||||
def tearDown(self):
|
||||
shutil.rmtree(self.tmp, ignore_errors=True)
|
||||
|
||||
def check(self, identity=None, resum=False):
|
||||
if resum: # a change made on purpose, with checksums to match, so the other checks see it
|
||||
os.remove(os.path.join(self.path, "SHA256SUMS"))
|
||||
write_sums(self.path)
|
||||
return validate.validate(self.path, identity=identity, recipients_path=self.keys)
|
||||
|
||||
def has(self, report, text):
|
||||
self.assertTrue(any(text in e for e in report.errors), (text, report.errors))
|
||||
|
||||
def manifest(self, change):
|
||||
p = os.path.join(self.path, "manifest.json")
|
||||
with open(p) as f:
|
||||
m = json.load(f)
|
||||
change(m)
|
||||
with open(p, "w") as f:
|
||||
json.dump(m, f)
|
||||
|
||||
def test_good(self):
|
||||
r = self.check()
|
||||
self.assertEqual(r.errors, [])
|
||||
self.assertEqual(r.summary["frames"], 60)
|
||||
self.assertTrue(r.summary["health"])
|
||||
r = self.check(identity=self.ident)
|
||||
self.assertEqual(r.errors, [])
|
||||
|
||||
def test_shipped_keys(self):
|
||||
r = validate.validate(self.path) # recipients.txt's, not the test key
|
||||
self.has(r, "other keys than the dataset's")
|
||||
|
||||
def test_checksum(self):
|
||||
with open(os.path.join(self.take, "cam0", "000001.zst.age"), "r+b") as f:
|
||||
f.seek(200)
|
||||
b = f.read(1)
|
||||
f.seek(200)
|
||||
f.write(bytes([b[0] ^ 1]))
|
||||
self.has(self.check(), "checksum mismatch")
|
||||
|
||||
def test_unencrypted(self):
|
||||
shutil.copyfile(os.path.join(self.take, "index.tsv"), os.path.join(self.take, "cam0", "000009.zst"))
|
||||
with open(os.path.join(self.path, "health.json"), "w") as f:
|
||||
f.write("{}")
|
||||
r = self.check(resum=True)
|
||||
self.has(r, "cam0/000009.zst: not an allowed file (unencrypted")
|
||||
self.has(r, "health.json: not an allowed file (unencrypted")
|
||||
|
||||
def test_cut_short(self):
|
||||
# Without a key only some sizes show a cut (a last piece under a tag); the key finds any.
|
||||
p = os.path.join(self.take, "cam1", "000000.zst.age")
|
||||
with open(p, "rb") as f:
|
||||
data = f.read()
|
||||
body = data.index(b"\n", data.index(b"\n---") + 1) + 1 + 16
|
||||
whole = (len(data) - body) // (age.CHUNK + age.TAG)
|
||||
with open(p, "r+b") as f:
|
||||
f.truncate(body + whole * (age.CHUNK + age.TAG) + 5)
|
||||
self.has(self.check(resum=True), "cam1/000000.zst.age: a size no age file has")
|
||||
with open(p, "r+b") as f:
|
||||
f.truncate(len(data) - 65000)
|
||||
self.assertEqual(self.check(resum=True).errors, [])
|
||||
self.has(self.check(identity=self.ident, resum=True), "cam1/000000.zst.age: doesn't decrypt")
|
||||
|
||||
def test_wrong_content(self):
|
||||
with open(os.path.join(self.take, "cam0", "000000.zst.age"), "wb") as f:
|
||||
f.write(age.encrypt_bytes(os.urandom(1000), [self.rec]))
|
||||
self.assertEqual(self.check(resum=True).errors, [], "without a key it can't tell")
|
||||
self.has(self.check(identity=self.ident), "doesn't decompress")
|
||||
other, _ = age.keygen()
|
||||
self.has(self.check(identity=other), "doesn't decrypt")
|
||||
|
||||
def test_index(self):
|
||||
p = os.path.join(self.take, "index.tsv")
|
||||
with open(p) as f:
|
||||
lines = f.readlines()
|
||||
with open(p, "w") as f:
|
||||
f.writelines(lines[:3] + lines[4:]) # camera 0, chunk 0, frame 2 gone
|
||||
r = self.check(resum=True)
|
||||
self.has(r, "frames aren't numbered")
|
||||
self.has(r, "the manifest says [30, 30] frames, index.tsv has [29, 30]")
|
||||
os.remove(os.path.join(self.take, "cam1", "000001.zst.age"))
|
||||
self.has(self.check(resum=True), "cam1/000001.zst.age: in index.tsv but missing")
|
||||
|
||||
def test_manifest(self):
|
||||
def change(m):
|
||||
m["session"]["dry_run"] = True
|
||||
m["session"]["replay"] = "/dev/shm/x"
|
||||
m["profile"]["health"] = {"x": 1}
|
||||
m["profile"]["consent"]["adult"] = False
|
||||
m["contributor"] = m["profile"]["contributor"] = "c0ffee"
|
||||
self.manifest(change)
|
||||
r = self.check(resum=True)
|
||||
for text in ("dry-run round", "from a replay", "unexpected keys: health", "18 or older", "random uuid4"):
|
||||
self.has(r, text)
|
||||
|
||||
def test_two_recipients(self):
|
||||
_, second = age.keygen()
|
||||
with open(self.keys, "a") as f:
|
||||
f.write(second + "\n")
|
||||
self.has(self.check(), "other keys than the dataset's")
|
||||
self.manifest(lambda m: m["encryption"].update(recipients=[self.rec, second]))
|
||||
self.has(self.check(resum=True), "encrypted to 1 recipients, not 2")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,467 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Check an eye recorder export before it's uploaded, or when it's received (takes.py's
|
||||
exports/<round>/).
|
||||
|
||||
The window runs it before an upload and refuses on errors; the maintainer runs it on each
|
||||
submission. The images are encrypted, so without a key it checks everything around them:
|
||||
- SHA256SUMS: every file listed and matching, nothing missing from it;
|
||||
- an allow-list of files: anything else (an unencrypted chunk, health.json, a symlink) is an error;
|
||||
- every .age file is age v1 to as many X25519 recipients as the manifest names, and its size
|
||||
is one a whole age file can have; the manifest's recipients are recipients.txt's;
|
||||
- manifest.json: the schema, required keys and types, the consent, a random uuid4 contributor,
|
||||
no dry run or replay, no health answers outside health.json.age;
|
||||
- each take's index.tsv against its chunks (every chunk named is there, every one there is
|
||||
named, frames numbered from 0, times going forward) and against the manifest's counts;
|
||||
- every .jsonl line is a JSON object with a time "t", and take.json and fit.json parse;
|
||||
- the total size.
|
||||
With --identity KEYFILE (one of the dataset's keys, the maintainer's side) it also decrypts each
|
||||
chunk in memory and checks it decompresses to the frames index.tsv gives it.
|
||||
|
||||
Standard library only without --identity (zstd and the cryptography library with it). Python
|
||||
3.12 or later, Linux or Windows.
|
||||
|
||||
usage: validate.py DIR [--json] [--identity KEYFILE]
|
||||
exit status 0: no errors, 1: errors, 2: not an export
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(HERE)))
|
||||
from recorder import age # noqa: E402
|
||||
from recorder.checks import (Cancelled, Report, check_sums, is_uuid4, listed, need, # noqa: E402,F401
|
||||
read_json_file)
|
||||
|
||||
EXPORT_SCHEMA = 1
|
||||
RECIPIENTS = os.path.join(HERE, "recipients.txt")
|
||||
TOP_FILES = {"manifest.json", "fit.json", "health.json.age", "SHA256SUMS"}
|
||||
TAKE_FILES = {"take.json", "index.tsv", "targets.jsonl", "ipd.jsonl", "prompts.jsonl", "own.jsonl",
|
||||
"steamvr.jsonl", "poses.jsonl"}
|
||||
CAM_DIRS = ("cam0", "cam1")
|
||||
ROUND_RE = re.compile(r"^\d{8}-\d{6}-[a-z0-9][a-z0-9-]{0,39}$")
|
||||
TAKE_RE = re.compile(r"^\d{2,}-[a-z0-9][a-z0-9_-]{0,63}$")
|
||||
CHUNK_FILE_RE = re.compile(r"^(\d{6})\.zst\.age$")
|
||||
TAKE_STATUSES = ("complete", "stopped", "skipped")
|
||||
PROFILE_KEYS = {"schema", "contributor", "consent", "optional", "checklist", "bounties", "eyes_ipd"}
|
||||
INDEX_HEADER = "# camera\tchunk\tframe\tslot\tt_raw"
|
||||
FRAME = 512 * 400
|
||||
CHUNK_FRAMES_MAX = 1000 # eyerec.py writes 90 a chunk
|
||||
# One round is about 2.6 GB (2026-10-14 replay: 20 MB/s on disk); encryption adds next to nothing.
|
||||
WARN_BYTES = 6 * 1000 ** 3
|
||||
MAX_BYTES = 15 * 1000 ** 3
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- the files
|
||||
|
||||
def walk_files(root, report):
|
||||
"""The export's files as relative paths, and allow-list errors on the way."""
|
||||
files = []
|
||||
for dirpath, dirnames, filenames in os.walk(root):
|
||||
rel_dir = os.path.relpath(dirpath, root).replace(os.sep, "/")
|
||||
rel_dir = "" if rel_dir == "." else rel_dir
|
||||
parts = rel_dir.split("/") if rel_dir else []
|
||||
for d in list(dirnames):
|
||||
rel = f"{rel_dir}/{d}" if rel_dir else d
|
||||
ok = ((not parts and d == "takes") or (parts == ["takes"] and TAKE_RE.match(d))
|
||||
or (len(parts) == 2 and parts[0] == "takes" and d in CAM_DIRS))
|
||||
if os.path.islink(os.path.join(dirpath, d)):
|
||||
report.error(f"{rel}: a symlink, not allowed in an export")
|
||||
dirnames.remove(d)
|
||||
elif not ok:
|
||||
report.error(f"{rel}/: not an allowed folder")
|
||||
dirnames.remove(d)
|
||||
for name in filenames:
|
||||
rel = f"{rel_dir}/{name}" if rel_dir else name
|
||||
full = os.path.join(dirpath, name)
|
||||
if os.path.islink(full) or not os.path.isfile(full):
|
||||
report.error(f"{rel}: a symlink or special file, not allowed in an export")
|
||||
continue
|
||||
ok = ((not parts and name in TOP_FILES) or (len(parts) == 2 and name in TAKE_FILES)
|
||||
or (len(parts) == 3 and CHUNK_FILE_RE.match(name)))
|
||||
if not ok:
|
||||
hint = " (unencrypted: export it again)" if name.endswith(".zst") or name == "health.json" else ""
|
||||
report.error(f"{rel}: not an allowed file{hint}")
|
||||
files.append(rel)
|
||||
return sorted(files)
|
||||
|
||||
|
||||
age_shape = age.shape # recorder/age.py: shared with the hand recorder's debug exports
|
||||
|
||||
|
||||
def check_jsonl(path, rel, report, cancelled):
|
||||
"""Every line a JSON object with a finite time "t" (RAW seconds, or MONOTONIC ns in prompts)."""
|
||||
try:
|
||||
with open(path, encoding="utf-8") as f:
|
||||
for n, line in enumerate(f, 1):
|
||||
if n % 50000 == 0 and cancelled():
|
||||
raise Cancelled()
|
||||
if not line.strip():
|
||||
continue
|
||||
try:
|
||||
obj = json.loads(line)
|
||||
except ValueError as e:
|
||||
report.error(f"{rel} line {n}: not valid JSON: {e}")
|
||||
return
|
||||
t = obj.get("t") if isinstance(obj, dict) else None
|
||||
if isinstance(t, bool) or not isinstance(t, (int, float)) or not math.isfinite(t):
|
||||
report.error(f"{rel} line {n}: not an object with a time \"t\"")
|
||||
return
|
||||
except UnicodeDecodeError:
|
||||
report.error(f"{rel}: not UTF-8 text")
|
||||
except OSError as e:
|
||||
report.error(f"{rel}: can't read it: {e.strerror or e}")
|
||||
|
||||
|
||||
def read_index(path, rel, report):
|
||||
"""index.tsv as {(camera, chunk): [(frame, t)]}, or None when it doesn't parse."""
|
||||
out = {}
|
||||
with open(path, encoding="utf-8", errors="replace") as f:
|
||||
for n, line in enumerate(f, 1):
|
||||
line = line.rstrip("\n")
|
||||
if n == 1:
|
||||
if line != INDEX_HEADER:
|
||||
report.error(f"{rel}: the first line isn't the header")
|
||||
return None
|
||||
continue
|
||||
if not line:
|
||||
continue
|
||||
fields = line.split("\t")
|
||||
try:
|
||||
c, chunk, i, slot = (int(x) for x in fields[:4])
|
||||
t = float(fields[4])
|
||||
if len(fields) != 5 or not math.isfinite(t) or c not in (0, 1) or min(chunk, i, slot) < 0:
|
||||
raise ValueError
|
||||
except (ValueError, IndexError):
|
||||
report.error(f"{rel} line {n}: not camera, chunk, frame, slot, time")
|
||||
return None
|
||||
out.setdefault((c, chunk), []).append((i, t))
|
||||
return out
|
||||
|
||||
|
||||
def check_take(root, tid, entry, sizes, recipients, identity, report, step, cancelled):
|
||||
base = f"takes/{tid}/"
|
||||
for name in sorted(TAKE_FILES - {"take.json", "index.tsv"}):
|
||||
rel = base + name
|
||||
if rel in sizes:
|
||||
check_jsonl(os.path.join(root, rel), rel, report, cancelled)
|
||||
elif name in ("targets.jsonl", "prompts.jsonl", "own.jsonl"):
|
||||
report.warn(f"{rel} is missing")
|
||||
if base + "take.json" in sizes:
|
||||
tj = read_json_file(os.path.join(root, base + "take.json"), base + "take.json", report)
|
||||
if tj is not None and not isinstance(tj, dict):
|
||||
report.error(f"{base}take.json: not a JSON object")
|
||||
else:
|
||||
report.warn(f"{base}take.json is missing")
|
||||
chunks = {}
|
||||
for rel in sizes:
|
||||
if rel.startswith(base) and rel.count("/") == 3:
|
||||
m = CHUNK_FILE_RE.match(rel.rsplit("/", 1)[1])
|
||||
if m:
|
||||
chunks[(int(rel.split("/")[2][3:]), int(m.group(1)))] = rel
|
||||
if base + "index.tsv" not in sizes:
|
||||
if chunks:
|
||||
report.error(f"{base}index.tsv is missing, but there are {len(chunks)} chunks")
|
||||
return
|
||||
index = read_index(os.path.join(root, base + "index.tsv"), base + "index.tsv", report)
|
||||
if index is None:
|
||||
return
|
||||
frames = [0, 0]
|
||||
unnamed = sorted(set(chunks) - set(index))
|
||||
if unnamed:
|
||||
report.error(f"{base}: chunks index.tsv doesn't name: {listed(chunks[k] for k in unnamed)}")
|
||||
last = [-math.inf, -math.inf]
|
||||
for key in sorted(index):
|
||||
if cancelled():
|
||||
raise Cancelled()
|
||||
c, chunk = key
|
||||
rows = index[key]
|
||||
frames[c] += len(rows)
|
||||
name = f"{base}cam{c}/{chunk:06d}.zst.age"
|
||||
if key not in chunks:
|
||||
report.error(f"{name}: in index.tsv but missing")
|
||||
continue
|
||||
if [i for i, _ in rows] != list(range(len(rows))) or len(rows) > CHUNK_FRAMES_MAX:
|
||||
report.error(f"{base}index.tsv: chunk {c}/{chunk:06d}'s frames aren't numbered 0 to {len(rows) - 1}")
|
||||
times = [t for _, t in rows]
|
||||
if any(b < a for a, b in zip(times, times[1:])) or times[0] < last[c]:
|
||||
report.error(f"{base}index.tsv: camera {c}'s times go back at chunk {chunk:06d}")
|
||||
last[c] = times[-1]
|
||||
problem = age_shape(os.path.join(root, name), recipients)
|
||||
if problem:
|
||||
report.error(f"{name}: {problem}")
|
||||
elif identity:
|
||||
problem = decrypted_problem(os.path.join(root, name), identity, len(rows))
|
||||
if problem:
|
||||
report.error(f"{name}: {problem}")
|
||||
step(sizes[name])
|
||||
if entry is not None and isinstance(entry.get("frames"), list) and entry["frames"] != frames:
|
||||
report.error(f"take {tid}: the manifest says {entry['frames']} frames, index.tsv has {frames}")
|
||||
|
||||
|
||||
def decrypted_problem(path, identity, nframes):
|
||||
sys.path.insert(0, HERE)
|
||||
import eyerec # zstd, as the recorder has it
|
||||
try:
|
||||
with open(path, "rb") as f:
|
||||
data = eyerec.decompress(age.decrypt_bytes(f.read(), identity))
|
||||
except ValueError as e:
|
||||
return f"doesn't decrypt: {e}"
|
||||
except Exception as e:
|
||||
return f"doesn't decompress: {e}"
|
||||
if len(data) != nframes * FRAME:
|
||||
return f"{len(data)} bytes, but index.tsv gives it {nframes} frames"
|
||||
return None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- the manifest
|
||||
|
||||
def check_manifest(m, report, root, shipped):
|
||||
"""The manifest's keys and types. Returns ({take id: entry}, the recipients)."""
|
||||
if not isinstance(m, dict):
|
||||
report.error("manifest: not a JSON object")
|
||||
return {}, []
|
||||
if m.get("schema") != EXPORT_SCHEMA:
|
||||
report.error(f"manifest: schema {m.get('schema')!r}, expected {EXPORT_SCHEMA}")
|
||||
tool = need(m, "tool", str, "", report)
|
||||
if tool is not None and not tool.startswith("ft-eyerec"):
|
||||
report.warn(f"manifest: tool is {tool!r}")
|
||||
need(m, "exported", str, "", report)
|
||||
need(m, "bounty", str, "", report)
|
||||
rid = need(m, "round_id", str, "", report)
|
||||
if rid is not None and not ROUND_RE.match(rid):
|
||||
report.error(f"manifest: round_id {rid!r} isn't a round id")
|
||||
if rid and ROUND_RE.match(os.path.basename(root)) and os.path.basename(root) != rid:
|
||||
report.warn(f"the folder is named {os.path.basename(root)}, the manifest's round is {rid}")
|
||||
contributor = need(m, "contributor", str, "", report)
|
||||
if contributor is not None and not is_uuid4(contributor):
|
||||
report.error(f"manifest: the contributor id {contributor!r} isn't a random uuid4")
|
||||
parent = os.path.basename(os.path.dirname(root))
|
||||
if contributor and is_uuid4(parent) and parent != contributor:
|
||||
report.error(f"the export sits under contributor {parent}, but its manifest says {contributor}")
|
||||
consent = need(m, "consent_version", str, "", report)
|
||||
if consent is not None and not consent.strip():
|
||||
report.error("manifest: the consent version is empty")
|
||||
|
||||
recipients = []
|
||||
enc = need(m, "encryption", dict, "", report)
|
||||
if enc is not None:
|
||||
if enc.get("format") != "age-encryption.org/v1":
|
||||
report.error(f"manifest: encryption.format {enc.get('format')!r}")
|
||||
r = enc.get("recipients")
|
||||
if not isinstance(r, list) or not r or not all(isinstance(x, str) for x in r):
|
||||
report.error("manifest: encryption.recipients isn't a list of keys")
|
||||
else:
|
||||
recipients = r
|
||||
try:
|
||||
for x in r:
|
||||
age.parse_recipient(x)
|
||||
except ValueError as e:
|
||||
report.error(f"manifest: encryption.recipients: {e}")
|
||||
if shipped is not None and sorted(r) != sorted(shipped):
|
||||
report.error("manifest: encrypted to other keys than the dataset's (recipients.txt)")
|
||||
|
||||
profile = need(m, "profile", dict, "", report)
|
||||
if profile is not None:
|
||||
if profile.get("contributor") != contributor:
|
||||
report.error("manifest: profile.contributor differs from contributor")
|
||||
pc = need(profile, "consent", dict, "profile.", report)
|
||||
if pc is not None:
|
||||
if pc.get("version") != consent:
|
||||
report.error("manifest: profile.consent.version differs from consent_version")
|
||||
if pc.get("adult") is not True:
|
||||
report.error("manifest: the contributor didn't confirm being 18 or older")
|
||||
if pc.get("region_ok") is not True:
|
||||
report.error("manifest: the contributor didn't confirm not living in Illinois, Texas or Washington")
|
||||
need(pc, "accepted", str, "profile.consent.", report)
|
||||
optional = profile.get("optional")
|
||||
if isinstance(optional, dict) and str(optional.get("notes") or "").strip():
|
||||
report.warn("the profile's notes are included: check they hold nothing private")
|
||||
extra = set(profile) - PROFILE_KEYS
|
||||
if extra:
|
||||
report.error(f"manifest: the profile has unexpected keys: {listed(sorted(extra))}")
|
||||
|
||||
session = need(m, "session", dict, "", report)
|
||||
if session is not None:
|
||||
for key, t in (("started", str), ("checklist", dict), ("takes", list), ("bounty", str)):
|
||||
need(session, key, t, "session.", report)
|
||||
if session.get("dry_run"):
|
||||
report.error("this is a dry-run round: it has no real recordings")
|
||||
if session.get("replay"):
|
||||
report.error("this round was recorded from a replay, not the headset's cameras")
|
||||
if session.get("status") not in ("done", None):
|
||||
report.warn(f"the round's status is {session.get('status')!r}")
|
||||
if "uploads" in session:
|
||||
report.warn("manifest: session.uploads is included (earlier uploads' links)")
|
||||
if "health" in session:
|
||||
report.error("manifest: health answers outside health.json.age")
|
||||
checklist = session.get("checklist")
|
||||
if isinstance(checklist, dict) and str(checklist.get("notes") or "").strip():
|
||||
report.warn("the checklist's notes are included: check they hold nothing private")
|
||||
if "/home/" in json.dumps(m):
|
||||
report.warn("manifest: it mentions a home folder path (a username may show)")
|
||||
|
||||
entries = {}
|
||||
takes = need(m, "takes", list, "", report)
|
||||
for i, t in enumerate(takes or []):
|
||||
where = f"takes[{i}]."
|
||||
if not isinstance(t, dict):
|
||||
report.error(f"manifest: takes[{i}] isn't an object")
|
||||
continue
|
||||
tid = need(t, "id", str, where, report)
|
||||
if tid is None:
|
||||
continue
|
||||
if not TAKE_RE.match(tid):
|
||||
report.error(f"manifest: take id {tid!r} isn't one")
|
||||
continue
|
||||
if tid in entries:
|
||||
report.error(f"manifest: take {tid} is listed twice")
|
||||
entries[tid] = t
|
||||
for key, typ in (("group", str), ("status", str), ("frames", list), ("frames_deleted", list),
|
||||
("duration_s", (int, float)), ("chunks", int)):
|
||||
need(t, key, typ, where, report)
|
||||
if t.get("status") not in TAKE_STATUSES:
|
||||
report.warn(f"manifest: take {tid} has status {t.get('status')!r}")
|
||||
left = t.get("left_out")
|
||||
if isinstance(left, dict) and any(left.values()):
|
||||
report.warn(f"take {tid}: chunks the export left out: {left}")
|
||||
return entries, recipients
|
||||
|
||||
|
||||
def summarize(m):
|
||||
"""What a pull request's description says about the export."""
|
||||
if not isinstance(m, dict):
|
||||
return {}
|
||||
session = m.get("session") if isinstance(m.get("session"), dict) else {}
|
||||
checklist = session.get("checklist") if isinstance(session.get("checklist"), dict) else {}
|
||||
takes = [t for t in m.get("takes") or [] if isinstance(t, dict)]
|
||||
seconds = sum(t.get("duration_s") or 0 for t in takes if isinstance(t.get("duration_s"), (int, float)))
|
||||
frames = sum(sum(f for f in t.get("frames") or [] if isinstance(f, int)) for t in takes
|
||||
if isinstance(t.get("frames"), list))
|
||||
enc = m.get("encryption") if isinstance(m.get("encryption"), dict) else {}
|
||||
return {"round": m.get("round_id", ""), "bounty": m.get("bounty", ""), "contributor": m.get("contributor", ""),
|
||||
"takes": len(takes), "frames": frames, "minutes": round(seconds / 60, 1),
|
||||
"glasses": checklist.get("glasses", ""), "health": "health.json.age" in (enc.get("files") or []),
|
||||
"consent_version": m.get("consent_version", ""), "tool": m.get("tool", "")}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- all of it
|
||||
|
||||
def validate(root, progress=None, cancel=None, identity=None, recipients_path=RECIPIENTS):
|
||||
"""Check the export at root. progress(fraction 0..1, text) is called as it goes; cancel()
|
||||
(or cancel.is_set()) returning true raises Cancelled. identity: an "AGE-SECRET-KEY-1..."
|
||||
string, to check inside the chunks. recipients_path: the keys the export must be encrypted
|
||||
to (None: don't compare). Returns a Report."""
|
||||
root = os.path.abspath(root)
|
||||
report = Report(root)
|
||||
cancelled = getattr(cancel, "is_set", cancel) or (lambda: False)
|
||||
tell = progress or (lambda fraction, text: None)
|
||||
if not os.path.isdir(root):
|
||||
report.error(f"{root}: no such folder")
|
||||
return report
|
||||
shipped = None
|
||||
if recipients_path:
|
||||
try:
|
||||
shipped = age.read_recipients(recipients_path)
|
||||
except (OSError, ValueError) as e:
|
||||
report.error(f"the dataset's keys ({recipients_path}): {e}")
|
||||
files = walk_files(root, report)
|
||||
sizes = {rel: os.path.getsize(os.path.join(root, rel)) for rel in files}
|
||||
report.bytes = sum(sizes.values())
|
||||
total = max(1, 2 * report.bytes) # checksums, then the chunks again
|
||||
done = 0
|
||||
|
||||
def step(n, text):
|
||||
nonlocal done
|
||||
done += n
|
||||
tell(min(done / total, 0.999), text)
|
||||
|
||||
tell(0.0, "Checking checksums")
|
||||
check_sums(root, files, report, lambda n: step(n, "Checking checksums"), cancelled)
|
||||
|
||||
manifest = None
|
||||
if "manifest.json" in files:
|
||||
manifest = read_json_file(os.path.join(root, "manifest.json"), "manifest.json", report)
|
||||
else:
|
||||
report.error("manifest.json is missing")
|
||||
entries, recipients = check_manifest(manifest, report, root, shipped) if manifest is not None else ({}, [])
|
||||
report.summary = summarize(manifest)
|
||||
|
||||
if "fit.json" in files:
|
||||
fit = read_json_file(os.path.join(root, "fit.json"), "fit.json", report)
|
||||
if fit is not None and not isinstance(fit, dict):
|
||||
report.error("fit.json: not a JSON object")
|
||||
else:
|
||||
report.warn("no fit.json: the round's fit readings are missing")
|
||||
enc_files = ((manifest or {}).get("encryption") or {}).get("files") if isinstance(manifest, dict) else None
|
||||
listed_health = isinstance(enc_files, list) and "health.json.age" in enc_files
|
||||
if ("health.json.age" in files) != listed_health:
|
||||
report.error("health.json.age and the manifest's encryption.files don't agree")
|
||||
if "health.json.age" in files and recipients:
|
||||
problem = age_shape(os.path.join(root, "health.json.age"), recipients)
|
||||
if problem:
|
||||
report.error(f"health.json.age: {problem}")
|
||||
|
||||
take_dirs = sorted({rel.split("/")[1] for rel in files if rel.startswith("takes/") and rel.count("/") >= 2})
|
||||
for tid in take_dirs:
|
||||
if tid not in entries:
|
||||
report.error(f"takes/{tid}/ isn't in the manifest")
|
||||
for tid, entry in entries.items():
|
||||
if tid not in take_dirs:
|
||||
report.error(f"take {tid} is in the manifest but has no folder")
|
||||
for tid in take_dirs:
|
||||
if cancelled():
|
||||
raise Cancelled()
|
||||
check_take(root, tid, entries.get(tid), sizes, recipients, identity, report,
|
||||
lambda n, tid=tid: step(n, f"Checking {tid}"), cancelled)
|
||||
|
||||
if report.bytes > MAX_BYTES:
|
||||
report.error(f"the export is {report.bytes / 1000 ** 3:.1f} GB, over the {MAX_BYTES / 1000 ** 3:.0f} GB limit")
|
||||
elif report.bytes > WARN_BYTES:
|
||||
report.warn(f"the export is large: {report.bytes / 1000 ** 3:.1f} GB")
|
||||
tell(1.0, "Checked")
|
||||
return report
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description="Check an eye recorder export (an exports/<round>/ folder).")
|
||||
ap.add_argument("dir")
|
||||
ap.add_argument("--json", action="store_true", help="print the result as JSON")
|
||||
ap.add_argument("--identity", help="an age identity file: also decrypt and check every chunk")
|
||||
a = ap.parse_args()
|
||||
if not os.path.isdir(a.dir):
|
||||
print(f"{a.dir}: no such folder", file=sys.stderr)
|
||||
return 2
|
||||
if hasattr(os, "setpriority"): # not on Windows
|
||||
try:
|
||||
os.setpriority(os.PRIO_PROCESS, 0, 19) # stay out of VR's way
|
||||
except OSError:
|
||||
pass
|
||||
identity = None
|
||||
if a.identity:
|
||||
with open(a.identity) as f:
|
||||
identity = next((line.strip() for line in f if line.strip().startswith("AGE-SECRET-KEY-1")), None)
|
||||
if not identity:
|
||||
print(f"{a.identity}: no age identity in it", file=sys.stderr)
|
||||
return 2
|
||||
report = validate(a.dir, identity=identity)
|
||||
if a.json:
|
||||
print(json.dumps(report.as_dict(), indent=2))
|
||||
else:
|
||||
for e in report.errors:
|
||||
print(f"error: {e}")
|
||||
for w in report.warnings:
|
||||
print(f"warning: {w}")
|
||||
s = report.summary
|
||||
if s:
|
||||
print(f"{s.get('round')}: {s.get('takes')} takes, {s.get('frames')} frames, {s.get('minutes')} min, "
|
||||
f"{report.bytes / 1000 ** 2:.1f} MB, consent {s.get('consent_version') or '-'}")
|
||||
print("OK" if report.ok else f"{len(report.errors)} errors")
|
||||
return 0 if report.ok else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Executable
+247
@@ -0,0 +1,247 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Offline test of the fit coach in the headset fit check (gaze/gazecheck.py, "fit"): with
|
||||
SteamVR's tracker in use, the check keeps ft-eyes running (the lease), asks it for the last
|
||||
second of pupil finds ("fit"), judges them only while the gaze is straight ahead, and puts the
|
||||
coach's hint first in the panel's hints.
|
||||
|
||||
The harness is first-calibration-test.py's: ft-gazed's Service with its sockets renamed, HOME in
|
||||
a temp folder, a fake pointer helper, a fake ft-gaze (its gaze from a file the test writes), a
|
||||
fake ft-eyes control socket (its finds from the shipped reference, offset as the test says),
|
||||
and the panel's commands recorded instead of sent. Safe next to the live gaze service.
|
||||
|
||||
gaze/test/fit-coach-check-test.py
|
||||
"""
|
||||
import os
|
||||
import tempfile
|
||||
|
||||
HOME = tempfile.mkdtemp(prefix="ft-gaze-fit-coach-test-")
|
||||
os.environ["HOME"] = HOME # before gazecal: its STATE, and frametop.conf, follow HOME
|
||||
|
||||
import importlib.machinery # noqa: E402
|
||||
import importlib.util # noqa: E402
|
||||
import json # noqa: E402
|
||||
import selectors # noqa: E402
|
||||
import shutil # noqa: E402
|
||||
import socket # noqa: E402
|
||||
import subprocess # noqa: E402
|
||||
import sys # noqa: E402
|
||||
import threading # noqa: E402
|
||||
import time # noqa: E402
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
GAZE = os.path.join(HERE, "..")
|
||||
sys.path.insert(0, GAZE)
|
||||
loader = importlib.machinery.SourceFileLoader("ftgazed", os.path.join(GAZE, "ft-gazed"))
|
||||
gazed = importlib.util.module_from_spec(importlib.util.spec_from_loader("ftgazed", loader))
|
||||
loader.exec_module(gazed)
|
||||
import fitcoach # noqa: E402
|
||||
import gazecheck # noqa: E402 (the module ft-gazed imported)
|
||||
|
||||
tag = f"ft_gaze_fit_coach_test_{os.getpid()}"
|
||||
gazed.ME = f"\0{tag}_gazed"
|
||||
gazed.POINTER = gazecheck.POINTER = f"\0{tag}_helper"
|
||||
gazecheck.SCREENS = f"\0{tag}_screens"
|
||||
gazecheck.PANEL = f"\0{tag}_panel"
|
||||
gazed.EYES_SOCKET = gazecheck.EYES = f"\0{tag}_eyes"
|
||||
gazed.read_settings = lambda: ("steam", "auto", "auto", 55.0) # SteamVR's tracker in use
|
||||
gazed.own_installed = lambda: True
|
||||
gazed.TrackedEye = lambda: lambda now=None: None
|
||||
logs = []
|
||||
gazed.log = gazecheck.log = lambda msg: logs.append(msg)
|
||||
|
||||
# The fake ft-gaze: 90 samples a second, SteamVR sees both eyes, its gaze from LOOK.
|
||||
LOOK = os.path.join(HOME, "look.json")
|
||||
FAKE = os.path.join(HOME, "ft-gaze")
|
||||
|
||||
|
||||
def look(yaw, pitch):
|
||||
with open(LOOK + ".new", "w") as f:
|
||||
json.dump([yaw, pitch], f)
|
||||
os.replace(LOOK + ".new", LOOK)
|
||||
|
||||
|
||||
look(0.0, 0.0)
|
||||
with open(FAKE, "w") as f:
|
||||
f.write(f'''import json, os, select, sys, time
|
||||
while True:
|
||||
if select.select([sys.stdin], [], [], 1 / 90)[0]:
|
||||
if not os.read(0, 4096):
|
||||
break
|
||||
try:
|
||||
hy, hp = json.load(open({LOOK!r}))
|
||||
except (OSError, ValueError):
|
||||
continue
|
||||
print(json.dumps({{"t": time.monotonic(), "src": {{"mmap1": {{"hy": hy, "hp": hp, "unc": [0.001, 0.001],
|
||||
"open": [0.8, 0.8]}}}}}}), flush=True)
|
||||
''')
|
||||
SLEEPER = [sys.executable, "-c", "import sys; sys.stdin.read()"] # quits when its stdin closes
|
||||
started = {"eyes": 0}
|
||||
|
||||
|
||||
def start_helper(self):
|
||||
self.proc = subprocess.Popen([sys.executable, FAKE], stdin=subprocess.PIPE, stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE)
|
||||
self.proc_sources = self.wanted_sources()
|
||||
os.set_blocking(self.proc.stdout.fileno(), False)
|
||||
os.set_blocking(self.proc.stderr.fileno(), False)
|
||||
self.sel.register(self.proc.stdout, selectors.EVENT_READ, "stdout")
|
||||
self.sel.register(self.proc.stderr, selectors.EVENT_READ, "stderr")
|
||||
self.buf = b""
|
||||
|
||||
|
||||
def start_eyes(self):
|
||||
started["eyes"] += 1
|
||||
self.eyes_proc = subprocess.Popen(SLEEPER, stdin=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||
os.set_blocking(self.eyes_proc.stderr.fileno(), False)
|
||||
self.sel.register(self.eyes_proc.stderr, selectors.EVENT_READ, "eyes")
|
||||
|
||||
|
||||
def start_panel(self):
|
||||
self.panel_proc = subprocess.Popen(SLEEPER, stdin=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||
os.set_blocking(self.panel_proc.stderr.fileno(), False)
|
||||
self.sel.register(self.panel_proc.stderr, selectors.EVENT_READ, "panel")
|
||||
|
||||
|
||||
panel = []
|
||||
gazed.Service.start_helper = start_helper
|
||||
gazed.Service.start_eyes = start_eyes
|
||||
gazecheck.Checks.start_panel = start_panel
|
||||
gazecheck.Checks.to_panel = lambda self, command: panel.append(command)
|
||||
|
||||
# The fake ft-eyes: "fit" answers with finds where the shipped reference puts eyes offset by
|
||||
# eyes_state["offset"] (yaw, pitch degrees, both eyes alike).
|
||||
REF = fitcoach.Reference.load()
|
||||
eyes_state = {"offset": (0.0, 0.0), "asked": 0}
|
||||
eyes = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM)
|
||||
eyes.bind(gazed.EYES_SOCKET)
|
||||
eyes.settimeout(0.2)
|
||||
|
||||
|
||||
def finds():
|
||||
yaw, pitch = eyes_state["offset"]
|
||||
out = {}
|
||||
for cam in (0, 1):
|
||||
(cx, cy), ((a, b), (c, d)) = REF.eyes[cam]["centre"], REF.eyes[cam]["jacobian"]
|
||||
out[cam] = {"n": 90, "finds": 90, "found": 1.0, "x": cx + a * yaw + b * pitch, "y": cy + c * yaw + d * pitch,
|
||||
"a": 28.0, "spacing": 15.0, "extra": 0.0}
|
||||
return out
|
||||
|
||||
|
||||
def eyes_answer():
|
||||
while True:
|
||||
try:
|
||||
data, addr = eyes.recvfrom(512)
|
||||
except socket.timeout:
|
||||
continue
|
||||
except OSError:
|
||||
return
|
||||
w = data.decode().split()
|
||||
if w[0] == "status":
|
||||
reply = json.dumps({"calibration": None, "calibrating": False, "dots": 0,
|
||||
"eyes": {"right": {"reseat": False}, "left": {"reseat": False}}})
|
||||
elif w[0] == "fit":
|
||||
eyes_state["asked"] += 1
|
||||
reply = json.dumps(finds())
|
||||
else:
|
||||
reply = f"fail unknown command {w[0]}"
|
||||
if addr:
|
||||
try:
|
||||
eyes.sendto(reply.encode(), addr)
|
||||
except OSError:
|
||||
pass # the service gave up on it (as ft-eyes' serve() does)
|
||||
|
||||
|
||||
threading.Thread(target=eyes_answer, daemon=True).start()
|
||||
|
||||
helper = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM)
|
||||
helper.bind(gazed.POINTER)
|
||||
helper.settimeout(0.2)
|
||||
|
||||
|
||||
def helper_answer():
|
||||
while True:
|
||||
try:
|
||||
data, addr = helper.recvfrom(512)
|
||||
except socket.timeout:
|
||||
continue
|
||||
except OSError:
|
||||
return
|
||||
if data.startswith(b"gaze ?") and addr:
|
||||
helper.sendto(b"ok off worn", addr)
|
||||
|
||||
|
||||
threading.Thread(target=helper_answer, daemon=True).start()
|
||||
svc = gazed.Service(None, False, gazed.POINTER)
|
||||
threading.Thread(target=svc.run, daemon=True).start()
|
||||
ctl = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM)
|
||||
ctl.bind("")
|
||||
ctl.settimeout(3)
|
||||
|
||||
|
||||
def ask(cmd):
|
||||
ctl.sendto(cmd.encode(), gazed.ME)
|
||||
return ctl.recv(65536).decode()
|
||||
|
||||
|
||||
failures = []
|
||||
|
||||
|
||||
def check(label, got, want):
|
||||
ok = got == want
|
||||
print(("ok " if ok else "FAIL ") + label + ("" if ok else f": got {got!r}, want {want!r}"), flush=True)
|
||||
if not ok:
|
||||
failures.append(label)
|
||||
|
||||
|
||||
def wait(cond, seconds):
|
||||
end = time.monotonic() + seconds
|
||||
while time.monotonic() < end:
|
||||
if cond():
|
||||
return True
|
||||
time.sleep(0.05)
|
||||
return cond()
|
||||
|
||||
|
||||
def coach_line():
|
||||
"""The first hint line the panel got last, if it's the coach's."""
|
||||
for cmd in reversed(panel):
|
||||
if cmd.startswith("hints "):
|
||||
first = cmd[len("hints "):].split("|")[0]
|
||||
return first if first.startswith("Headset position") else None
|
||||
return None
|
||||
|
||||
|
||||
ask("wake 60")
|
||||
check("the tracker sends", wait(lambda: time.monotonic() - svc.checks.sample_at < 1, 6), True)
|
||||
check("SteamVR's tracker in use: ft-eyes isn't running", svc.eyes_proc is None, True)
|
||||
check("fitcheck is taken", ask("fitcheck").startswith("ok"), True)
|
||||
check("the fit check opens", wait(lambda: (svc.checks.check or {}).get("kind") == "fit", 6), True)
|
||||
check("the fit check starts ft-eyes for the coach", wait(lambda: svc.eyes_proc is not None, 3), True)
|
||||
check("and asks it for finds", wait(lambda: eyes_state["asked"] > 0, 3), True)
|
||||
check("looking ahead, well fitted: good",
|
||||
wait(lambda: (coach_line() or "").startswith("Headset position: good."), 4), True)
|
||||
check("marked a first guess (the reference is from one face)", "first guess" in (coach_line() or ""), True)
|
||||
|
||||
eyes_state["offset"] = (0.0, 12.0) # both eyes appear to look up: the headset is low
|
||||
check("the headset low: push it up",
|
||||
wait(lambda: "Push the headset up" in (coach_line() or ""), 4), True)
|
||||
check("it's the first hint line", (coach_line() or "").startswith("Headset position: Push"), True)
|
||||
|
||||
look(25.0, 0.0) # looking away: the pupils move with the gaze, so no new reading
|
||||
check("looking away: the last centred reading, marked so",
|
||||
wait(lambda: "when you last looked straight ahead" in (coach_line() or ""), 4), True)
|
||||
eyes_state["offset"] = (0.0, 0.0)
|
||||
time.sleep(1.5)
|
||||
check("and it isn't judged meanwhile", "Push the headset up" in (coach_line() or ""), True)
|
||||
look(0.0, 0.0)
|
||||
check("looking ahead again: judged again", wait(lambda: (coach_line() or "").startswith("Headset position: good."), 4), True)
|
||||
|
||||
ask("calquit")
|
||||
check("closing logs the last reading", wait(lambda: any(m.startswith("fit coach, last reading: ok") for m in logs), 2), True)
|
||||
check("the check closed", wait(lambda: svc.checks.check is None, 2), True)
|
||||
check("ft-eyes stops once its lease runs out",
|
||||
wait(lambda: svc.eyes_proc is None, gazecheck.COACH_LEASE + 4), True)
|
||||
|
||||
print("all passed" if not failures else f"{len(failures)} failed: {failures}", flush=True)
|
||||
shutil.rmtree(HOME, ignore_errors=True)
|
||||
os._exit(1 if failures else 0)
|
||||
Executable
+316
@@ -0,0 +1,316 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Offline test of the fit watch (gaze/gazecheck.py, GAZE_FIT_WATCH): the fit check keeping a
|
||||
good fit as yours, the put-on check in the quick check that opens when the headset goes on
|
||||
(holds the dot, says what to move, lets go once it's back or on a click), and the slip note
|
||||
with our tracker (ft-eyes' glint jump past the threshold for SLIP_HOLD; gone with the slip;
|
||||
not again within NOTE_COOLDOWN).
|
||||
|
||||
fit-coach-check-test.py's harness: ft-gazed's Service with its sockets renamed, HOME in a temp
|
||||
folder, a fake pointer helper, a fake ft-gaze, a fake ft-eyes control socket (its finds from
|
||||
the shipped reference, offset as the test says, and its status with the glint jump), and the
|
||||
panel's commands recorded instead of sent. Safe next to the live gaze service.
|
||||
|
||||
gaze/test/fit-watch-test.py
|
||||
"""
|
||||
import os
|
||||
import tempfile
|
||||
|
||||
HOME = tempfile.mkdtemp(prefix="ft-gaze-fit-watch-test-")
|
||||
os.environ["HOME"] = HOME # before gazecal: its STATE, and frametop.conf, follow HOME
|
||||
|
||||
import importlib.machinery # noqa: E402
|
||||
import importlib.util # noqa: E402
|
||||
import json # noqa: E402
|
||||
import selectors # noqa: E402
|
||||
import shutil # noqa: E402
|
||||
import socket # noqa: E402
|
||||
import subprocess # noqa: E402
|
||||
import sys # noqa: E402
|
||||
import threading # noqa: E402
|
||||
import time # noqa: E402
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
GAZE = os.path.join(HERE, "..")
|
||||
sys.path.insert(0, GAZE)
|
||||
loader = importlib.machinery.SourceFileLoader("ftgazed", os.path.join(GAZE, "ft-gazed"))
|
||||
gazed = importlib.util.module_from_spec(importlib.util.spec_from_loader("ftgazed", loader))
|
||||
loader.exec_module(gazed)
|
||||
import fitcoach # noqa: E402
|
||||
import gazecheck # noqa: E402 (the module ft-gazed imported)
|
||||
|
||||
tag = f"ft_gaze_fit_watch_test_{os.getpid()}"
|
||||
gazed.ME = f"\0{tag}_gazed"
|
||||
gazed.POINTER = gazecheck.POINTER = f"\0{tag}_helper"
|
||||
gazecheck.SCREENS = f"\0{tag}_screens"
|
||||
gazecheck.PANEL = f"\0{tag}_panel"
|
||||
gazed.EYES_SOCKET = gazecheck.EYES = f"\0{tag}_eyes"
|
||||
SETTINGS = {"tracker": "steam"} # SteamVR's tracker in use first, then ours
|
||||
gazed.read_settings = lambda: (SETTINGS["tracker"], "auto", "auto", 55.0)
|
||||
gazed.own_installed = lambda: True
|
||||
gazed.TrackedEye = lambda: lambda now=None: None
|
||||
logs = []
|
||||
gazed.log = gazecheck.log = lambda msg: logs.append(msg)
|
||||
|
||||
# The fake ft-gaze: 90 samples a second, SteamVR sees both eyes, its gaze from LOOK.
|
||||
LOOK = os.path.join(HOME, "look.json")
|
||||
FAKE = os.path.join(HOME, "ft-gaze")
|
||||
|
||||
|
||||
def look(yaw, pitch):
|
||||
with open(LOOK + ".new", "w") as f:
|
||||
json.dump([yaw, pitch], f)
|
||||
os.replace(LOOK + ".new", LOOK)
|
||||
|
||||
|
||||
look(0.0, 0.0)
|
||||
with open(FAKE, "w") as f:
|
||||
f.write(f'''import json, os, select, sys, time
|
||||
while True:
|
||||
if select.select([sys.stdin], [], [], 1 / 90)[0]:
|
||||
if not os.read(0, 4096):
|
||||
break
|
||||
try:
|
||||
hy, hp = json.load(open({LOOK!r}))
|
||||
except (OSError, ValueError):
|
||||
continue
|
||||
print(json.dumps({{"t": time.monotonic(), "src": {{"mmap1": {{"hy": hy, "hp": hp, "unc": [0.001, 0.001],
|
||||
"open": [0.8, 0.8]}}}}}}), flush=True)
|
||||
''')
|
||||
SLEEPER = [sys.executable, "-c", "import sys; sys.stdin.read()"] # quits when its stdin closes
|
||||
started = {"eyes": 0}
|
||||
|
||||
|
||||
def start_helper(self):
|
||||
self.proc = subprocess.Popen([sys.executable, FAKE], stdin=subprocess.PIPE, stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE)
|
||||
self.proc_sources = self.wanted_sources()
|
||||
os.set_blocking(self.proc.stdout.fileno(), False)
|
||||
os.set_blocking(self.proc.stderr.fileno(), False)
|
||||
self.sel.register(self.proc.stdout, selectors.EVENT_READ, "stdout")
|
||||
self.sel.register(self.proc.stderr, selectors.EVENT_READ, "stderr")
|
||||
self.buf = b""
|
||||
|
||||
|
||||
def start_eyes(self):
|
||||
started["eyes"] += 1
|
||||
self.eyes_proc = subprocess.Popen(SLEEPER, stdin=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||
os.set_blocking(self.eyes_proc.stderr.fileno(), False)
|
||||
self.sel.register(self.eyes_proc.stderr, selectors.EVENT_READ, "eyes")
|
||||
|
||||
|
||||
def start_panel(self):
|
||||
self.panel_proc = subprocess.Popen(SLEEPER, stdin=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||
os.set_blocking(self.panel_proc.stderr.fileno(), False)
|
||||
self.sel.register(self.panel_proc.stderr, selectors.EVENT_READ, "panel")
|
||||
|
||||
|
||||
panel = []
|
||||
gazed.Service.start_helper = start_helper
|
||||
gazed.Service.start_eyes = start_eyes
|
||||
gazecheck.Checks.start_panel = start_panel
|
||||
gazecheck.Checks.to_panel = lambda self, command: panel.append(command)
|
||||
|
||||
# The fake ft-eyes: "fit" answers with finds where the shipped reference puts eyes offset by
|
||||
# eyes_state["offset"] (yaw, pitch degrees, both eyes alike).
|
||||
REF = fitcoach.Reference.load()
|
||||
eyes_state = {"offset": (0.0, 0.0), "jump": (0.0, 0.0), "asked": 0}
|
||||
eyes = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM)
|
||||
eyes.bind(gazed.EYES_SOCKET)
|
||||
eyes.settimeout(0.2)
|
||||
|
||||
|
||||
def finds():
|
||||
yaw, pitch = eyes_state["offset"]
|
||||
out = {}
|
||||
for cam in (0, 1):
|
||||
(cx, cy), ((a, b), (c, d)) = REF.eyes[cam]["centre"], REF.eyes[cam]["jacobian"]
|
||||
out[cam] = {"n": 90, "finds": 90, "found": 1.0, "x": cx + a * yaw + b * pitch, "y": cy + c * yaw + d * pitch,
|
||||
"a": 28.0, "spacing": 15.0, "extra": 0.0}
|
||||
return out
|
||||
|
||||
|
||||
def eyes_answer():
|
||||
while True:
|
||||
try:
|
||||
data, addr = eyes.recvfrom(512)
|
||||
except socket.timeout:
|
||||
continue
|
||||
except OSError:
|
||||
return
|
||||
w = data.decode().split()
|
||||
if w[0] == "status":
|
||||
yaw, pitch = eyes_state["jump"] # the slip, as both pupils' move in degrees
|
||||
jump = {}
|
||||
for cam, name in ((0, "right"), (1, "left")):
|
||||
(a, b), (c, d) = REF.eyes[cam]["jacobian"]
|
||||
jump[name] = {"reseat": False, "jump": bool(yaw or pitch),
|
||||
"jump_px": [a * yaw + b * pitch, c * yaw + d * pitch]}
|
||||
reply = json.dumps({"calibration": {"dots": 27}, "calibrating": False, "dots": 27, "eyes": jump})
|
||||
elif w[0] == "fit":
|
||||
eyes_state["asked"] += 1
|
||||
reply = json.dumps(finds())
|
||||
else:
|
||||
reply = f"fail unknown command {w[0]}"
|
||||
if addr:
|
||||
try:
|
||||
eyes.sendto(reply.encode(), addr)
|
||||
except OSError:
|
||||
pass # the service gave up on it (as ft-eyes' serve() does)
|
||||
|
||||
|
||||
threading.Thread(target=eyes_answer, daemon=True).start()
|
||||
|
||||
HELPER = {"gaze": False}
|
||||
helper = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM)
|
||||
helper.bind(gazed.POINTER)
|
||||
helper.settimeout(0.2)
|
||||
|
||||
|
||||
def helper_answer():
|
||||
while True:
|
||||
try:
|
||||
data, addr = helper.recvfrom(512)
|
||||
except socket.timeout:
|
||||
continue
|
||||
except OSError:
|
||||
return
|
||||
if data.startswith(b"gaze ?") and addr:
|
||||
helper.sendto(b"ok on worn" if HELPER["gaze"] else b"ok off worn", addr)
|
||||
|
||||
|
||||
threading.Thread(target=helper_answer, daemon=True).start()
|
||||
svc = gazed.Service(None, False, gazed.POINTER)
|
||||
threading.Thread(target=svc.run, daemon=True).start()
|
||||
ctl = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM)
|
||||
ctl.bind("")
|
||||
ctl.settimeout(3)
|
||||
|
||||
|
||||
def ask(cmd):
|
||||
ctl.sendto(cmd.encode(), gazed.ME)
|
||||
return ctl.recv(65536).decode()
|
||||
|
||||
|
||||
failures = []
|
||||
|
||||
|
||||
def check(label, got, want):
|
||||
ok = got == want
|
||||
print(("ok " if ok else "FAIL ") + label + ("" if ok else f": got {got!r}, want {want!r}"), flush=True)
|
||||
if not ok:
|
||||
failures.append(label)
|
||||
|
||||
|
||||
def wait(cond, seconds):
|
||||
end = time.monotonic() + seconds
|
||||
while time.monotonic() < end:
|
||||
if cond():
|
||||
return True
|
||||
time.sleep(0.05)
|
||||
return cond()
|
||||
|
||||
|
||||
def since(n):
|
||||
"""The panel's commands from n on."""
|
||||
return panel[n:]
|
||||
|
||||
|
||||
def quick(label):
|
||||
"""Opens the quick check as the headset going on does; returns where its panel commands start."""
|
||||
n = len(panel)
|
||||
check(label, svc.checks.start("quick", gazecheck.PUTON_REASON), "ok")
|
||||
return n
|
||||
|
||||
|
||||
def puton():
|
||||
return ((svc.checks.check or {}).get("puton") or {}).get("state")
|
||||
|
||||
|
||||
def closed_with_capture(seconds=8):
|
||||
return wait(lambda: svc.checks.check is None and any(m.startswith("quick check done") for m in logs[mark:]), seconds)
|
||||
|
||||
|
||||
ask("wake 60")
|
||||
check("the tracker sends", wait(lambda: time.monotonic() - svc.checks.sample_at < 1, 6), True)
|
||||
check("the eyes count as seen", wait(lambda: svc.checks.eyes_seen(), 3), True)
|
||||
|
||||
# The fit watch off (the default): the quick check is as it was.
|
||||
mark = len(logs)
|
||||
quick("fit watch off: a quick check opens")
|
||||
check(" without a put-on check", puton(), None)
|
||||
check(" and takes its dot", closed_with_capture(), True)
|
||||
|
||||
# A good fit in the fit check is kept as yours.
|
||||
check("no saved fit yet", os.path.exists(fitcoach.BASELINE), False)
|
||||
check("fitcheck is taken", ask("fitcheck").startswith("ok"), True)
|
||||
check("the fit check opens", wait(lambda: (svc.checks.check or {}).get("kind") == "fit", 6), True)
|
||||
check("it reads a good fit", wait(lambda: ((svc.checks.check or {}).get("coach") or {}).get("good") is not None, 6), True)
|
||||
ask("calquit")
|
||||
check("closing it saves the fit as yours", wait(lambda: os.path.exists(fitcoach.BASELINE), 2), True)
|
||||
check("with both eyes", sorted(json.load(open(fitcoach.BASELINE))["eyes"]), ["0", "1"])
|
||||
|
||||
gazed.CONF.parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(gazed.CONF, "w") as f:
|
||||
f.write("GAZE_FIT_WATCH=1\n")
|
||||
|
||||
# Put on as saved: the dot is held for the reading, then taken.
|
||||
mark = len(logs)
|
||||
quick("put on as saved: the quick check opens")
|
||||
check(" reading the fit first", puton(), "reading")
|
||||
check(" sits as saved", wait(lambda: any("sits as your saved fit" in m for m in logs[mark:]), 5), True)
|
||||
check(" then the dot is taken", closed_with_capture(), True)
|
||||
|
||||
# Put on low: it says to push it up, holds the dot past the quick check's timeout, lets go once back.
|
||||
eyes_state["offset"] = (0.0, 12.0)
|
||||
mark = len(logs)
|
||||
n = quick("put on low: the quick check opens")
|
||||
check(" the headset moved", wait(lambda: puton() == "moved", 5), True)
|
||||
check(" the panel widens", "show five" in since(n), True)
|
||||
check(" and says to push it up", wait(lambda: any(c.startswith("text Push the headset up") for c in since(n)), 2), True)
|
||||
time.sleep(gazecheck.QUICK_TIMEOUT + 1)
|
||||
check(" the dot waits past the quick check's timeout", (svc.checks.check or {}).get("captured"), 0)
|
||||
eyes_state["offset"] = (0.0, 0.0)
|
||||
check(" pushed back: let go", wait(lambda: any("is back as your saved fit" in m for m in logs[mark:]), 5), True)
|
||||
check(" saying so", "note Good: it's back where it was" in since(n), True)
|
||||
check(" then the dot is taken", closed_with_capture(), True)
|
||||
|
||||
# Put on low, then a click: taken as it sits.
|
||||
eyes_state["offset"] = (0.0, 12.0)
|
||||
mark = len(logs)
|
||||
quick("put on low again")
|
||||
check(" the headset moved", wait(lambda: puton() == "moved", 5), True)
|
||||
ask("calaccept")
|
||||
check(" a click lets go", wait(lambda: any("a click" in m for m in logs[mark:]), 3), True)
|
||||
check(" and takes the dot", closed_with_capture(), True)
|
||||
eyes_state["offset"] = (0.0, 0.0)
|
||||
|
||||
# The slip note: our tracker, gaze mode on.
|
||||
SETTINGS["tracker"] = "own"
|
||||
HELPER["gaze"] = True
|
||||
with open(gazed.CONF, "w") as f:
|
||||
f.write("GAZE_FIT_WATCH=1\n# our tracker now\n")
|
||||
check("our tracker in use", wait(lambda: svc.kind == "own" and svc.eyes_proc is not None, 6), True)
|
||||
check("gaze mode on", wait(lambda: svc.checks.gaze_on is True, 4), True)
|
||||
check("ft-eyes' status comes", wait(lambda: time.monotonic() - svc.own_at < 2, 4), True)
|
||||
n = len(panel)
|
||||
time.sleep(2)
|
||||
check("no slip: no note", "show note" in since(n), False)
|
||||
eyes_state["jump"] = (0.0, -12.0) # the pupils dropped: the headset rode up
|
||||
start = time.monotonic()
|
||||
check("a slip: the note", wait(lambda: "show note" in since(n), gazecheck.SLIP_HOLD + 4), True)
|
||||
check(" not before SLIP_HOLD", time.monotonic() - start >= gazecheck.SLIP_HOLD, True)
|
||||
check(" saying so", "title Your headset slipped" in since(n), True)
|
||||
check(" and which way", any(c.startswith("text Let the headset down") for c in since(n)), True)
|
||||
check(" no pointer hiding for it", any(c == "calpanel 1" for c in since(n)), False)
|
||||
n = len(panel)
|
||||
eyes_state["jump"] = (0.0, 0.0)
|
||||
check("the slip gone: the note goes", wait(lambda: "hide" in since(n), 4), True)
|
||||
n = len(panel)
|
||||
eyes_state["jump"] = (0.0, -12.0)
|
||||
time.sleep(gazecheck.SLIP_HOLD + 3)
|
||||
check("again within NOTE_COOLDOWN: no note", "show note" in since(n), False)
|
||||
eyes_state["jump"] = (0.0, 0.0)
|
||||
|
||||
print("all passed" if not failures else f"{len(failures)} failed: {failures}", flush=True)
|
||||
shutil.rmtree(HOME, ignore_errors=True)
|
||||
os._exit(1 if failures else 0)
|
||||
Executable
+238
@@ -0,0 +1,238 @@
|
||||
#!/usr/bin/env python3
|
||||
"""The fit coach (gaze/fitcoach.py) on made-up readings: the reference fit, which way each hint
|
||||
points with both cameras' orientations, the state's hysteresis, lost eyes, and a baseline.
|
||||
|
||||
python3 gaze/test/fitcoach-test.py
|
||||
"""
|
||||
import math
|
||||
import os
|
||||
import random
|
||||
import sys
|
||||
import unittest
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
sys.path.insert(0, os.path.dirname(HERE))
|
||||
import fitcoach # noqa: E402
|
||||
|
||||
# About the maintainer's headset's (gaze/fit-reference.json): camera 1's image is turned 180
|
||||
# degrees from camera 0's, so its signs are the other way.
|
||||
J = {0: [[1.27, -0.42], [-0.48, -1.34]], 1: [[-1.34, -0.53], [-0.48, 1.48]]}
|
||||
CENTRE = {0: [341.0, 221.4], 1: [334.7, 173.5]}
|
||||
|
||||
|
||||
def ref(follow=None, faces=1):
|
||||
return fitcoach.Reference({c: {"centre": CENTRE[c], "jacobian": J[c]} for c in (0, 1)}, ipd=0.065,
|
||||
faces=faces, cameras_follow_lenses=follow)
|
||||
|
||||
|
||||
def pupil(cam, yaw, pitch, centre=None):
|
||||
"""Where camera cam sees the pupil when the eye appears to look (yaw, pitch) degrees."""
|
||||
cx, cy = centre or CENTRE[cam]
|
||||
(a, b), (c, d) = J[cam]
|
||||
return cx + a * yaw + b * pitch, cy + c * yaw + d * pitch
|
||||
|
||||
|
||||
def summary(right=(0, 0), left=(0, 0), found=(1.0, 1.0), spacing=(14.0, 16.0), extra=(0.0, 0.0)):
|
||||
out = {}
|
||||
for cam, (yaw, pitch) in ((0, right), (1, left)):
|
||||
x, y = pupil(cam, yaw, pitch)
|
||||
out[cam] = {"n": 90, "finds": int(90 * found[cam]), "found": found[cam], "x": x, "y": y, "a": 28.0,
|
||||
"spacing": spacing[cam], "extra": extra[cam]}
|
||||
return out
|
||||
|
||||
|
||||
def keys(reading):
|
||||
return [h["key"] for h in reading["hints"]]
|
||||
|
||||
|
||||
class ReferenceTest(unittest.TestCase):
|
||||
def test_fit_recovers_both_orientations(self):
|
||||
rng = random.Random(1)
|
||||
rows = []
|
||||
for made in ("a", "b"):
|
||||
for _ in range(27):
|
||||
yaw, pitch = rng.uniform(-20, 20), rng.uniform(-15, 15)
|
||||
rows.append({"made": made, "truth": [yaw, pitch],
|
||||
"eye": {str(c): {"pupil": [v + rng.gauss(0, 0.5) for v in pupil(c, yaw, pitch)]} for c in (0, 1)}})
|
||||
# A third calibration that fits badly is left out.
|
||||
for _ in range(27):
|
||||
yaw, pitch = rng.uniform(-20, 20), rng.uniform(-15, 15)
|
||||
rows.append({"made": "bad", "truth": [yaw, pitch],
|
||||
"eye": {str(c): {"pupil": [v + rng.gauss(0, 15) for v in pupil(c, yaw, pitch)]} for c in (0, 1)}})
|
||||
r = fitcoach.Reference.from_dots(rows)
|
||||
for c in (0, 1):
|
||||
self.assertEqual(r.eyes[c]["fits"], 2)
|
||||
for i in range(2):
|
||||
self.assertAlmostEqual(r.eyes[c]["centre"][i], CENTRE[c][i], delta=0.5)
|
||||
for j in range(2):
|
||||
self.assertAlmostEqual(r.eyes[c]["jacobian"][i][j], J[c][i][j], delta=0.05)
|
||||
# degrees() undoes the Jacobian.
|
||||
x, y = pupil(1, 7.0, -3.0)
|
||||
yaw, pitch = r.degrees(1, x - r.eyes[1]["centre"][0], y - r.eyes[1]["centre"][1])
|
||||
self.assertAlmostEqual(yaw, 7.0, delta=0.2)
|
||||
self.assertAlmostEqual(pitch, -3.0, delta=0.2)
|
||||
|
||||
def test_json_round_trip(self):
|
||||
r = ref(follow=True, faces=4)
|
||||
back = fitcoach.Reference.from_json(r.to_json())
|
||||
self.assertEqual(back.eyes[1]["jacobian"], J[1])
|
||||
self.assertTrue(back.cameras_follow_lenses)
|
||||
self.assertFalse(back.provisional)
|
||||
self.assertTrue(ref().provisional)
|
||||
|
||||
def test_the_shipped_reference_loads(self):
|
||||
r = fitcoach.Reference.load()
|
||||
self.assertIsNotNone(r, fitcoach.REFERENCE)
|
||||
self.assertEqual(sorted(r.eyes), [0, 1])
|
||||
self.assertTrue(r.provisional)
|
||||
|
||||
|
||||
class AssessTest(unittest.TestCase):
|
||||
def test_centred_is_ok(self):
|
||||
r = fitcoach.assess(summary(), ref())
|
||||
self.assertEqual(r["state"], "ok")
|
||||
self.assertEqual(r["hints"], [])
|
||||
self.assertTrue(r["provisional"])
|
||||
|
||||
def test_eyes_high_means_push_up(self):
|
||||
r = fitcoach.assess(summary(right=(0, 12), left=(0, 12)), ref())
|
||||
self.assertEqual(r["state"], "adjust")
|
||||
self.assertEqual(keys(r), ["up"])
|
||||
r = fitcoach.assess(summary(right=(0, -12), left=(0, -12)), ref())
|
||||
self.assertEqual(keys(r), ["down"])
|
||||
|
||||
def test_eyes_left_means_slide_left(self):
|
||||
r = fitcoach.assess(summary(right=(11, 0), left=(11, 0)), ref())
|
||||
self.assertEqual(keys(r), ["left"])
|
||||
self.assertAlmostEqual(r["terms"]["left"], 11, delta=0.1)
|
||||
r = fitcoach.assess(summary(right=(-11, 0), left=(-11, 0)), ref())
|
||||
self.assertEqual(keys(r), ["right"])
|
||||
|
||||
def test_ipd_only_advised_once_the_cameras_are_known_to_follow_the_lenses(self):
|
||||
apart = summary(right=(-10, 0), left=(10, 0)) # each eye outward of its lens
|
||||
r = fitcoach.assess(apart, ref(follow=None))
|
||||
self.assertAlmostEqual(r["terms"]["ipd"], 10, delta=0.1)
|
||||
self.assertNotIn("ipd-wider", keys(r))
|
||||
self.assertFalse(r["ipd_advice"])
|
||||
self.assertEqual(r["state"], "ok", "the common terms are centred")
|
||||
r = fitcoach.assess(apart, ref(follow=True))
|
||||
self.assertEqual(keys(r), ["ipd-wider"])
|
||||
self.assertEqual(r["state"], "adjust")
|
||||
r = fitcoach.assess(summary(right=(10, 0), left=(-10, 0)), ref(follow=True))
|
||||
self.assertEqual(keys(r), ["ipd-narrower"])
|
||||
|
||||
def test_tilt(self):
|
||||
r = fitcoach.assess(summary(right=(0, -5), left=(0, 5)), ref())
|
||||
self.assertEqual(keys(r), ["tilt-left-up"])
|
||||
r = fitcoach.assess(summary(right=(0, 5), left=(0, -5)), ref())
|
||||
self.assertEqual(keys(r), ["tilt-right-up"])
|
||||
|
||||
def test_hysteresis_between_ok_and_adjust(self):
|
||||
mid = summary(right=(0, 6.5), left=(0, 6.5))
|
||||
self.assertEqual(fitcoach.assess(mid, ref(), previous={"state": "ok"})["state"], "ok")
|
||||
self.assertEqual(fitcoach.assess(mid, ref(), previous={"state": "adjust"})["state"], "adjust")
|
||||
self.assertEqual(fitcoach.assess(mid, ref())["state"], "adjust")
|
||||
self.assertEqual(fitcoach.assess(mid, ref(), previous={"state": "lost"})["state"], "adjust")
|
||||
# Adjust without a term past ADJUST_DEG: the biggest one's hint, so there's something to do.
|
||||
self.assertEqual(keys(fitcoach.assess(mid, ref())), ["up"])
|
||||
self.assertEqual(keys(fitcoach.assess(mid, ref(), previous={"state": "ok"})), [])
|
||||
|
||||
def test_a_lost_eye_comes_first(self):
|
||||
r = fitcoach.assess(summary(right=(0, 12), left=(0, 12), found=(1.0, 0.3)), ref())
|
||||
self.assertEqual(r["state"], "lost")
|
||||
self.assertEqual(keys(r)[0], "lost")
|
||||
self.assertIn("left eye", r["hints"][0]["text"])
|
||||
|
||||
def test_waiting_without_frames(self):
|
||||
r = fitcoach.assess({0: {"n": 0}, 1: {"n": 0}}, ref())
|
||||
self.assertEqual(r["state"], "waiting")
|
||||
self.assertEqual(fitcoach.assess({}, ref())["state"], "waiting")
|
||||
|
||||
def test_reflections(self):
|
||||
r = fitcoach.assess(summary(extra=(0.0, 0.8)), ref())
|
||||
self.assertEqual(keys(r), ["reflections"])
|
||||
self.assertIn("left", r["hints"][0]["text"])
|
||||
|
||||
|
||||
class BaselineTest(unittest.TestCase):
|
||||
def test_against_the_baseline(self):
|
||||
good = summary(right=(4, -3), left=(4, -3)) # this face's good fit isn't the reference's
|
||||
base = fitcoach.make_baseline(good, ipd=0.065)
|
||||
r = fitcoach.assess(good, ref(), ipd=0.065, baseline=base)
|
||||
self.assertEqual((r["state"], r["against"]), ("ok", "baseline"))
|
||||
self.assertFalse(fitcoach.moved(r))
|
||||
# Back on, 7 degrees lower on the face: moved, and which way to put it back.
|
||||
slipped = summary(right=(4, 4), left=(4, 4))
|
||||
r = fitcoach.assess(slipped, ref(), ipd=0.065, baseline=base)
|
||||
self.assertTrue(fitcoach.moved(r))
|
||||
self.assertAlmostEqual(r["terms"]["up"], 7, delta=0.1)
|
||||
|
||||
def test_eye_relief_and_ipd_change(self):
|
||||
base = fitcoach.make_baseline(summary(), ipd=0.065)
|
||||
near = summary(spacing=(14.0 * 1.12, 16.0 * 1.12))
|
||||
r = fitcoach.assess(near, ref(), ipd=0.0665, baseline=base)
|
||||
self.assertEqual(keys(r), ["ipd-changed", "nearer"])
|
||||
self.assertIn("65.0 mm, now 66.5 mm", r["hints"][0]["text"])
|
||||
self.assertTrue(fitcoach.moved(r))
|
||||
far = summary(spacing=(14.0 * 0.9, 16.0 * 0.9))
|
||||
self.assertEqual(keys(fitcoach.assess(far, ref(), ipd=0.065, baseline=base)), ["further"])
|
||||
|
||||
|
||||
class ObservationsTest(unittest.TestCase):
|
||||
def test_window_and_medians(self):
|
||||
obs = fitcoach.Observations(window=1.0)
|
||||
# A pair 14 px apart and four stray ones: MANY_GLINTS.
|
||||
glints = [(300.0, 200.0), (301.0, 214.0), (250.0, 120.0), (260.0, 120.0), (270.0, 120.0), (280.0, 120.0)]
|
||||
for i in range(60):
|
||||
t = i / 30
|
||||
obs.add(0, t, {"x": 300 + (i % 3), "y": 210.0, "a": 25.0, "glints": glints})
|
||||
obs.add(1, t, None if i % 2 else {"x": 320.0, "y": 170.0, "a": 30.0, "glints": []})
|
||||
s = obs.summary()
|
||||
self.assertEqual(s[0]["n"], 31) # the last second only
|
||||
self.assertEqual(s[0]["x"], 301)
|
||||
self.assertAlmostEqual(s[0]["spacing"], math.hypot(1, 14), places=5)
|
||||
self.assertEqual(s[0]["extra"], 1.0)
|
||||
self.assertAlmostEqual(s[1]["found"], 15 / 31, places=3)
|
||||
self.assertIsNone(s[1]["spacing"])
|
||||
|
||||
def test_glint_spacing_matches_eyes_pupil(self):
|
||||
try:
|
||||
sys.path.insert(0, os.path.join(os.path.dirname(HERE), "tracker"))
|
||||
import eyes_pupil
|
||||
except ImportError:
|
||||
self.skipTest("no numpy or OpenCV here")
|
||||
rng = random.Random(3)
|
||||
for _ in range(200):
|
||||
p = {"x": 300.0, "y": 200.0, "glints": [(rng.uniform(270, 330), rng.uniform(170, 230)) for _ in range(rng.randint(0, 5))]}
|
||||
pair = eyes_pupil.glint_pair(p)
|
||||
want = None if pair is None else math.hypot(pair[1][0] - pair[0][0], pair[1][1] - pair[0][1])
|
||||
got = fitcoach.glint_spacing(p)
|
||||
if want is None:
|
||||
self.assertIsNone(got)
|
||||
else:
|
||||
self.assertAlmostEqual(got, want, places=6)
|
||||
|
||||
|
||||
class SlipTest(unittest.TestCase):
|
||||
"""The slip notice's reading: ft-eyes' glint jumps (pupil px) as position terms and a hint."""
|
||||
|
||||
def jumps(self, yaw, pitch):
|
||||
# Both pupils moved as if the eyes looked (yaw, pitch): each camera's own direction.
|
||||
return {c: (pupil(c, yaw, pitch)[0] - CENTRE[c][0], pupil(c, yaw, pitch)[1] - CENTRE[c][1]) for c in (0, 1)}
|
||||
|
||||
def test_terms(self):
|
||||
t = fitcoach.slip_terms(self.jumps(2.0, 9.0), ref())
|
||||
self.assertAlmostEqual(t["up"], 9.0, places=6)
|
||||
self.assertAlmostEqual(t["left"], 2.0, places=6)
|
||||
self.assertEqual(fitcoach.slip_terms({}, ref()), {})
|
||||
self.assertEqual(fitcoach.slip_terms({0: (3.0, 4.0)}, None), {})
|
||||
|
||||
def test_hint(self):
|
||||
self.assertEqual(fitcoach.position_hint(fitcoach.slip_terms(self.jumps(0, 9.0), ref()))["key"], "up")
|
||||
self.assertEqual(fitcoach.position_hint(fitcoach.slip_terms(self.jumps(0, -9.0), ref()))["key"], "down")
|
||||
self.assertEqual(fitcoach.position_hint(fitcoach.slip_terms(self.jumps(-7.0, 6.0), ref()))["key"], "right")
|
||||
self.assertIsNone(fitcoach.position_hint(fitcoach.slip_terms(self.jumps(3.0, -4.0), ref())))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,85 @@
|
||||
"""The shared eye-camera frames ft-eyegrab keeps in /dev/shm/frametop-eyes-cams, read without
|
||||
numpy: lab/ft-eyes-record and the eye recorder's frame writer (gaze/rec/eyerec.py) use it.
|
||||
|
||||
Layout (ft-eyegrab.c, share_head_t and share_entry_t): a 64-byte header (magic "FEC1", version,
|
||||
width, height, slots per camera, entry size, then each camera's frame count as u64 at 24 + 8 *
|
||||
camera, and the tracker's pid at 40), then SLOTS entries per camera. Frame n of camera c is in
|
||||
entry c * slots + n % slots: a 64-byte block (u64 seq, odd while it's written; f64 time,
|
||||
CLOCK_MONOTONIC_RAW seconds; u64 n; u32 camera; u32 the tracker's slot), then the image.
|
||||
"""
|
||||
import mmap
|
||||
import os
|
||||
import struct
|
||||
import time
|
||||
|
||||
CAMS = os.environ.get("FT_EYES_CAMS", "/dev/shm/frametop-eyes-cams") # overrides for replays
|
||||
WANT = "/dev/shm/frametop-eyes-want"
|
||||
W, H = 512, 400
|
||||
MAGIC = 0x31434546
|
||||
|
||||
|
||||
class Share:
|
||||
"""The shared frames, mapped read-only."""
|
||||
|
||||
def __init__(self, path=None):
|
||||
self.path = path or CAMS
|
||||
fd = os.open(self.path, os.O_RDONLY)
|
||||
try:
|
||||
self.ino = os.fstat(fd).st_ino
|
||||
self.mm = mmap.mmap(fd, 0, prot=mmap.PROT_READ)
|
||||
finally:
|
||||
os.close(fd)
|
||||
magic, version, w, h, self.slots, self.esize = struct.unpack_from("<6I", self.mm, 0)
|
||||
if magic != MAGIC or version != 1 or (w, h) != (W, H):
|
||||
raise RuntimeError(f"{self.path}: unexpected header")
|
||||
|
||||
def replaced(self):
|
||||
"""ft-eyegrab restarted: it makes a new file, and this one is stale."""
|
||||
try:
|
||||
return os.stat(self.path).st_ino != self.ino
|
||||
except OSError:
|
||||
return True
|
||||
|
||||
def count(self, cam):
|
||||
return struct.unpack_from("<Q", self.mm, 24 + 8 * cam)[0]
|
||||
|
||||
def frame(self, cam, n):
|
||||
"""(time, slot, bytes) of frame n of a camera, or None if it was overwritten."""
|
||||
off = 64 + (cam * self.slots + n % self.slots) * self.esize
|
||||
for _ in range(3):
|
||||
seq = struct.unpack_from("<Q", self.mm, off)[0]
|
||||
if seq & 1:
|
||||
continue
|
||||
data = self.mm[off + 64:off + 64 + W * H]
|
||||
t, got, _cam, slot = struct.unpack_from("<dQII", self.mm, off + 8)
|
||||
if struct.unpack_from("<Q", self.mm, off)[0] == seq and got == n:
|
||||
return t, slot, data
|
||||
return None
|
||||
|
||||
def close(self):
|
||||
self.mm.close()
|
||||
|
||||
|
||||
def touch_want(path=None):
|
||||
"""Tell ft-eyegrab someone wants frames (it idles otherwise). Not for replays."""
|
||||
if path is None and "FT_EYES_CAMS" in os.environ:
|
||||
return
|
||||
try:
|
||||
fd = os.open(path or WANT, os.O_WRONLY | os.O_CREAT | os.O_NOFOLLOW | os.O_CLOEXEC, 0o600)
|
||||
os.utime(fd)
|
||||
os.close(fd)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
def open_share(timeout, path=None, want=None):
|
||||
"""The share, waiting up to timeout seconds for it (touching the want file meanwhile), or None."""
|
||||
deadline = time.monotonic() + timeout
|
||||
while True:
|
||||
touch_want(want)
|
||||
try:
|
||||
return Share(path)
|
||||
except (OSError, ValueError, RuntimeError):
|
||||
if time.monotonic() >= deadline:
|
||||
return None
|
||||
time.sleep(0.5)
|
||||
+73
-3
@@ -21,8 +21,10 @@ off), the next click starts that eye's shift over, since the headset may sit dif
|
||||
|
||||
Control socket: abstract datagram "@ft_eyes"; each command gets one reply line.
|
||||
status JSON: calibration, and per eye its shift, clicks, whether a
|
||||
glint jump is applied, and "reseat" (the next click starts
|
||||
the shift over: the probe asks for a one-dot check then)
|
||||
glint jump is applied ("jump", and "jump_px": how far, pixels:
|
||||
the headset slipped since the last click; the gaze service's
|
||||
slip notice reads it), and "reseat" (the next click starts the
|
||||
shift over: the probe asks for a one-dot check then)
|
||||
calib-start a new calibration: collect dots from now on
|
||||
calib-point T0 T1 YAW PITCH a dot you looked at from T0 to T1 (CLOCK_MONOTONIC_RAW) in
|
||||
that direction (head-relative degrees): "ok N0 N1 SD0 SD1"
|
||||
@@ -30,6 +32,13 @@ Control socket: abstract datagram "@ft_eyes"; each command gets one reply line.
|
||||
calib-fit fit the dots, save, and start the shifts and clicks over
|
||||
click T YAW PITCH a click: you were looking there just before T; teaches the
|
||||
shift: "ok DX0 DY0 DX1 DY1" (the shift each eye measured)
|
||||
fit JSON: the last second of pupil finds per camera, for the fit
|
||||
coach (gaze/fitcoach.py, Observations.summary); works uncalibrated
|
||||
finds start PATH | finds stop
|
||||
append every frame looked at to PATH as JSON lines (the eye
|
||||
recorder's own.jsonl): {"t", "cam", "x", "y", "a", "b", "angle",
|
||||
"glints": [[x, y], ...], "gaze": [yaw, pitch] once calibrated},
|
||||
or {"t", "cam", "lost": true}. "finds stop" replies "ok ROWS"
|
||||
|
||||
Environment, for replays (lab/ft-eyes-e2e): FT_EYES_CAMS, FT_EYES_GAZE, FT_EYES_STATE (the
|
||||
state folder), FT_EYES_SOCKET (the socket's name). With FT_EYES_CAMS set, the want file isn't
|
||||
@@ -67,8 +76,10 @@ for _var in ("OPENBLAS_NUM_THREADS", "OMP_NUM_THREADS"):
|
||||
import numpy as np # noqa: E402
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
import eyes_model # noqa: E402
|
||||
import eyes_pupil # noqa: E402
|
||||
import fitcoach # noqa: E402
|
||||
|
||||
CAMS = os.environ.get("FT_EYES_CAMS", "/dev/shm/frametop-eyes-cams") # overrides for replays
|
||||
OUT = os.environ.get("FT_EYES_GAZE", "/dev/shm/frametop-eyes-gaze")
|
||||
@@ -173,6 +184,9 @@ class Eye:
|
||||
self.frames = self.found = 0
|
||||
self.work = 0.0
|
||||
self.last_t = None # the previous frame's time
|
||||
self.obs = None # the fit coach's window (fitcoach.Observations), shared by both eyes
|
||||
self.fed = None # (frame time, CLOCK_MONOTONIC_RAW then) of the newest frame fed
|
||||
self.finds = None # the finds log (Tracker.finds), shared by both eyes
|
||||
|
||||
def use(self, cal, shift=None):
|
||||
self.cal = cal if cal is not None and cal.has("pupil", self.eye) else None
|
||||
@@ -192,6 +206,9 @@ class Eye:
|
||||
t0 = time.perf_counter()
|
||||
p = eyes_pupil.find_pupil(img, self.last)
|
||||
self.last = p
|
||||
if self.obs is not None:
|
||||
self.obs.add(self.eye, t, p)
|
||||
self.fed = (t, time.clock_gettime(time.CLOCK_MONOTONIC_RAW))
|
||||
if p is not None:
|
||||
self.found += 1
|
||||
self.lost = 0
|
||||
@@ -206,6 +223,8 @@ class Eye:
|
||||
self.shift.glint(self.slip.get(), t)
|
||||
g = self.cal.gaze(self.eye, p["x"], p["y"], self.shift.value)
|
||||
self.gaze = (t, float(g[0]), float(g[1]), p["x"], p["y"])
|
||||
if self.finds is not None:
|
||||
self.finds.write(self, t, p)
|
||||
self.work += time.perf_counter() - t0
|
||||
|
||||
def window(self, t0, t1):
|
||||
@@ -220,12 +239,39 @@ class Eye:
|
||||
return dict(pupil=pupil, mid=mid, n=len(rows), spread=spread)
|
||||
|
||||
|
||||
class Finds:
|
||||
"""The finds log (the "finds" command): a JSON line per frame looked at."""
|
||||
|
||||
def __init__(self, path):
|
||||
self.f = open(path, "a", buffering=1 << 16)
|
||||
self.rows = 0
|
||||
|
||||
def write(self, eye, t, p):
|
||||
if p is None:
|
||||
row = {"t": round(t, 6), "cam": eye.eye, "lost": True}
|
||||
else:
|
||||
row = {"t": round(t, 6), "cam": eye.eye, "x": round(float(p["x"]), 2), "y": round(float(p["y"]), 2),
|
||||
"a": round(float(p["a"]), 2), "b": round(float(p["b"]), 2), "angle": round(float(p["angle"]), 1),
|
||||
"glints": [[round(float(x), 1), round(float(y), 1)] for x, y in p.get("glints") or []]}
|
||||
if eye.gaze and eye.gaze[0] == t:
|
||||
row["gaze"] = [round(eye.gaze[1], 3), round(eye.gaze[2], 3)]
|
||||
self.f.write(json.dumps(row) + "\n")
|
||||
self.rows += 1
|
||||
|
||||
def close(self):
|
||||
self.f.close()
|
||||
return self.rows
|
||||
|
||||
|
||||
class Tracker:
|
||||
def __init__(self):
|
||||
self.eyes = [Eye(0), Eye(1)]
|
||||
self.cal = None
|
||||
self.dots = [] # calibration dots so far, in ft-eyes-score's click form
|
||||
self.calibrating = False
|
||||
self.fit = fitcoach.Observations()
|
||||
for e in self.eyes:
|
||||
e.obs = self.fit
|
||||
if CALIBRATION.exists():
|
||||
self.cal = eyes_model.Calibration.load(CALIBRATION)
|
||||
if not self.cal.spread:
|
||||
@@ -256,6 +302,30 @@ class Tracker:
|
||||
return "fail empty"
|
||||
if w[0] == "status":
|
||||
return json.dumps(self.status())
|
||||
if w[0] == "finds" and len(w) >= 2:
|
||||
rows = None
|
||||
for e in self.eyes:
|
||||
if e.finds is not None:
|
||||
rows = e.finds.close()
|
||||
break
|
||||
for e in self.eyes:
|
||||
e.finds = None
|
||||
if w[1] == "stop":
|
||||
return f"ok {rows or 0}"
|
||||
if w[1] == "start" and len(w) == 3:
|
||||
log = Finds(w[2])
|
||||
for e in self.eyes:
|
||||
e.finds = log
|
||||
return "ok"
|
||||
return "fail finds start PATH | finds stop"
|
||||
if w[0] == "fit":
|
||||
# In frame time, which a replay's frames keep from the recording: the newest frame's,
|
||||
# plus how long ago it came (frames stopping, the headset off, empty the window).
|
||||
fed = [e.fed for e in self.eyes if e.fed]
|
||||
if not fed:
|
||||
return json.dumps(self.fit.summary(0.0))
|
||||
t, at = max(fed)
|
||||
return json.dumps(self.fit.summary(t + time.clock_gettime(time.CLOCK_MONOTONIC_RAW) - at))
|
||||
if w[0] == "calib-start":
|
||||
self.dots, self.calibrating = [], True
|
||||
return "ok"
|
||||
@@ -321,7 +391,7 @@ class Tracker:
|
||||
return {"calibration": self.cal.info if self.cal else None, "calibrating": self.calibrating,
|
||||
"dots": len(self.dots),
|
||||
"eyes": {EYES[e.eye]: {"shift": e.shift.value.tolist(), "clicks": len(e.shift.meas),
|
||||
"jump": bool(np.any(e.shift.jump)),
|
||||
"jump": bool(np.any(e.shift.jump)), "jump_px": e.shift.jump.tolist(),
|
||||
"reseat": e.shift.reseated} for e in self.eyes}}
|
||||
|
||||
|
||||
|
||||
@@ -12,81 +12,25 @@ clicks are added later, on the Frame: `lab/py lab/ft-eyes-score --clicks NAME`.
|
||||
|
||||
Usage: lab/ft-eyes-record NAME [SECONDS] (host Python is enough: no numpy)
|
||||
"""
|
||||
import mmap
|
||||
import os
|
||||
import shutil
|
||||
import signal
|
||||
import struct
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
from eyes_lab import new_capture # noqa: E402
|
||||
import eyes_share # noqa: E402 (gaze/tracker, on the path through eyes_lab)
|
||||
from eyes_share import touch_want # noqa: E402
|
||||
|
||||
CAMS = os.environ.get("FT_EYES_CAMS", "/dev/shm/frametop-eyes-cams")
|
||||
WANT = "/dev/shm/frametop-eyes-want"
|
||||
W, H = 512, 400
|
||||
MIN_FREE_GB = 20
|
||||
|
||||
|
||||
class Share:
|
||||
"""The shared frames. Layout: ft-eyegrab.c, share_head_t/share_entry_t."""
|
||||
|
||||
def __init__(self):
|
||||
fd = os.open(CAMS, os.O_RDONLY)
|
||||
try:
|
||||
self.ino = os.fstat(fd).st_ino
|
||||
self.mm = mmap.mmap(fd, 0, prot=mmap.PROT_READ)
|
||||
finally:
|
||||
os.close(fd)
|
||||
magic, version, w, h, self.slots, self.esize = struct.unpack_from("<6I", self.mm, 0)
|
||||
if magic != 0x31434546 or version != 1 or (w, h) != (W, H):
|
||||
raise RuntimeError(f"{CAMS}: unexpected header")
|
||||
|
||||
def replaced(self):
|
||||
try:
|
||||
return os.stat(CAMS).st_ino != self.ino
|
||||
except OSError:
|
||||
return True
|
||||
|
||||
def count(self, cam):
|
||||
return struct.unpack_from("<Q", self.mm, 24 + 8 * cam)[0]
|
||||
|
||||
def frame(self, cam, n):
|
||||
"""(time, slot, bytes) of frame n of a camera, or None if it was overwritten."""
|
||||
off = 64 + (cam * self.slots + n % self.slots) * self.esize
|
||||
for _ in range(3):
|
||||
seq = struct.unpack_from("<Q", self.mm, off)[0]
|
||||
if seq & 1:
|
||||
continue
|
||||
data = self.mm[off + 64:off + 64 + W * H]
|
||||
t, got, _cam, slot = struct.unpack_from("<dQII", self.mm, off + 8)
|
||||
if struct.unpack_from("<Q", self.mm, off)[0] == seq and got == n:
|
||||
return t, slot, data
|
||||
return None
|
||||
|
||||
|
||||
def touch_want():
|
||||
"""Tell ft-eyegrab someone wants frames (it idles otherwise)."""
|
||||
if "FT_EYES_CAMS" in os.environ:
|
||||
return
|
||||
try:
|
||||
fd = os.open(WANT, os.O_WRONLY | os.O_CREAT | os.O_NOFOLLOW | os.O_CLOEXEC, 0o600)
|
||||
os.utime(fd)
|
||||
os.close(fd)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
def open_share(deadline):
|
||||
while time.monotonic() < deadline:
|
||||
touch_want()
|
||||
try:
|
||||
return Share()
|
||||
except (OSError, ValueError, RuntimeError):
|
||||
time.sleep(0.5)
|
||||
sys.exit(f"ft-eyes-record: no {CAMS} (is frametop-eyegrab.service running? gaze/tracker/install.sh)")
|
||||
def open_share(timeout):
|
||||
share = eyes_share.open_share(timeout)
|
||||
if share is None:
|
||||
sys.exit(f"ft-eyes-record: no {eyes_share.CAMS} (is frametop-eyegrab.service running? gaze/tracker/install.sh)")
|
||||
return share
|
||||
|
||||
|
||||
def main(argv):
|
||||
@@ -97,7 +41,7 @@ def main(argv):
|
||||
stop = []
|
||||
for sig in (signal.SIGINT, signal.SIGTERM):
|
||||
signal.signal(sig, lambda *_: stop.append(1))
|
||||
share = open_share(time.monotonic() + 10)
|
||||
share = open_share(10)
|
||||
(out / "clocks.txt").write_text(f"{time.time()} {time.clock_gettime(time.CLOCK_MONOTONIC_RAW)}\n")
|
||||
seen = [share.count(0), share.count(1)]
|
||||
written = dropped = 0
|
||||
@@ -134,7 +78,7 @@ def main(argv):
|
||||
break
|
||||
if share.replaced():
|
||||
print("ft-eyes-record: the frame share was restarted; following it", file=sys.stderr)
|
||||
share = open_share(time.monotonic() + 10)
|
||||
share = open_share(10)
|
||||
seen = [share.count(0), share.count(1)]
|
||||
print(f"ft-eyes-record: {written} frames ({dropped} dropped) in {out}", file=sys.stderr)
|
||||
|
||||
|
||||
+3
-1
@@ -10,7 +10,9 @@ The Hand Recorder records your hands with the Steam Frame's tracking cameras for
|
||||
|
||||
Join the [Frametop Discord](https://discord.gg/W3X9f7z3Bc) for questions and help with recording.
|
||||
|
||||
For now the recorder runs inside Frametop's desktop, so these steps install Frametop first. A standalone recorder that runs from the SteamVR dashboard without Frametop is planned.
|
||||
These steps install Frametop first, and the recorder runs inside Frametop's desktop. To record **bounty rounds** (short rounds of taps, pinches and grabs, about 4 minutes each) without installing Frametop, use the standalone [Frametop Hand Recorder](https://github.com/Frametop/frametop-hand-recorder) instead: it runs from the SteamVR dashboard and installs with one command. Bounty rounds go to the same public dataset. In Frametop itself they need the experimental version (`--experimental` instead of `--stable` below).
|
||||
|
||||
Installing a Frametop release (Frametop.zip, FrameDrop, or `get.sh --release`) instead? Its installer offers the Hand Recorder, built, so there's nothing to compile: leave Hand Recorder ticked (or answer yes) and skip step 2. To add it to a release later, run `~/.local/share/frametop/releases/current/hands/rec/install.sh`.
|
||||
|
||||
## Before you start
|
||||
|
||||
|
||||
@@ -3,6 +3,8 @@ models/onnx/
|
||||
models/*.task
|
||||
# the fine-tuned models hands/models.sh fetches (their own licence, CC BY-NC 4.0: not in the repo)
|
||||
models/student/
|
||||
# hands/models.sh update's download folder, lock and last check
|
||||
models/.student-*
|
||||
# frame-job settings (where replays run, and the lab's capture folder): personal, not shipped
|
||||
.frame-job
|
||||
dumps/
|
||||
|
||||
+3
-2
@@ -51,6 +51,7 @@ Settings in `~/.config/frametop.conf` (`FT_<name>` in the environment overrides
|
||||
- `HANDS_CPUS=5,6,7`: the CPUs the model threads run on (below).
|
||||
- `HANDS_CAMERAS` (`auto`), `HANDS_BRIGHT` (`all`), `HANDS_BRIGHT_ON` (40), `HANDS_BRIGHT_OFF` (25): which cameras ft-hands tracks with, as `--cams`, `--bright`, `--bright-on` and `--bright-off` (see ft-hands). `HANDS_CAMERAS=mono` also keeps ft-camd off the colour cameras.
|
||||
- `HANDS_MODELS` (unset: `hands/models/student`, the fine-tuned models, when `hands/models.sh` fetched them, else `hands/models/ncnn`, the stock MediaPipe models): a folder holding `palm.ncnn.*` and `hand.ncnn.*`, as `--models`. Use it to try other models without passing options to every launcher. The fine-tuned folders lack the `*-int8` files, so `--int8` without `--models` uses the stock ones.
|
||||
- `HANDS_MODELS_UPDATE=1`: when hand tracking starts, fetch the fine-tuned models GitHub's branch pins if they changed, without a new release (`models.sh` below). `0` keeps the ones it came with.
|
||||
- `HANDS_COLOR_LEFT` (`color_video0`), `HANDS_COLOR_CROP` (`subtract`): how the colour module's calibration maps onto its images, as `--color-left` and `--color-crop`.
|
||||
- `HANDS_SKELETON=1`: fit a skeleton with your bone lengths and publish it (see "Skeleton" below); `0` publishes the filtered landmarks as before. As `--skeleton`.
|
||||
- `HANDS_PINCH_DIP=0`: `1` also starts a pinch on a quick dip of the thumb-index distance that never reaches 2 cm (see the pinch under "Gestures" below). As `--pinch-dip`.
|
||||
@@ -214,7 +215,7 @@ What it measured, `ft-handreplay` on three recordings and three takes of the 202
|
||||
|
||||
ft-hands detects a pinch per hand (`track/pinch.h`) and publishes it to the gestures file. The layout, and how to read it without missing quick taps, is in `include/fh_gestures.h`.
|
||||
|
||||
- A pinch begins when the thumb and index tips come within `--pinch-begin` (default 0.020 m). It ends when they open past `--pinch-end` (0.035 m) for 2 processed frames in a row, or when the hand stays lost for 0.25 s (flagged lost).
|
||||
- A pinch begins when the thumb and index tips come within `--pinch-begin` (default 0.020 m). It ends when they open past `--pinch-end` (0.045 m) for 2 processed frames in a row, or when the hand stays lost for 0.25 s (flagged lost).
|
||||
- A pinch held 0.3 s or longer is a drag, and holds on harder: it ends only once the tips are 5 cm apart (for the same 2 frames), or 0.12 s after they first went past 3.5 cm unless they closed back under 3 cm meanwhile. In the 2026-10-09 headset test drags ended with the tips 3.5-4.0 cm apart, a thumb hidden for a frame or fingers loosened a little mid-drag. On the lit recording's replayed distances, pinches held 0.3 s or more (typing among them) ended 33-134 ms later than before, 200 at most, and one of 25 ran into the next pinch's begin. A grip ends a pinch (as lost) only within 0.3 s of its begin, when closing the hand passes through a pinch; after that no grip begins on that hand, so curling the other fingers around a held pinch doesn't drop it (3 of 28 pinches in the same test). `tests/pinch_test.cpp` (`make check`).
|
||||
- The distance comes from MediaPipe's world landmarks: the model's own 3D hand pose, averaged over the hand's views, at the user's hand size. `--pinch-triangulated` uses the triangulated tips instead. On two recordings without deliberate pinches, the world landmarks came under 2 cm in 0.2-1% of frames, against 3.3-4.5% for the triangulated tips. In the dim recording, typing still gave 2 pinches a minute (see the next point).
|
||||
- `--pinch-palm-down MAX` holds back pinches begun with the palm facing down (MAX is the palm normal's share of the head's up axis). The default, 1, turns it off. A close held back that way has to open again before a pinch can begin. Typing curls the thumb onto the index: in the lit recording of 2026-09-30, typing on a keyboard in the lap began 23 pinches in about 2 minutes, all with the palm facing down (0.69-1.00), while the 26 deliberate ones read 0.00-0.50. But in the headset, deliberate pinches with the hand raised in front read 0.90-0.99 too, so the limit is off. Typing is caught by the pointer helper instead: the input relay tells it when you press a key, and the hands on the keys are held back (`POINTER_PINCH_TYPING` and the settings after it, below).
|
||||
@@ -298,7 +299,7 @@ Python, with NumPy and OpenCV. `setup/dev-container.sh` doesn't install them, be
|
||||
- `tools/depth_report.py DEPTH`: the depth measures above.
|
||||
- `tools/cut_sets.py REC OUT [--sets N | --at I,J,...]`: copies a few frame sets (by default 8, spread evenly) out of a recording into a small one, to look at or check elsewhere without moving gigabytes. Plain Python, so it also runs on the Frame's host.
|
||||
- `tools/convert_models.py`: how `models/ncnn` was made from the OpenCV Zoo ONNX ports of MediaPipe's models (see `models/NOTICE`).
|
||||
- `models.sh`: fetches the fine-tuned models into `models/student` (`hands/build.sh` runs it, so source installs and the release image both get them): MediaPipe's two models retrained on the Frame's infrared cameras with the [Frametop Hands](https://huggingface.co/datasets/DeeJanuz/frametop-hands) dataset, from a fixed commit of [DeeJanuz/frametop-hands-models](https://huggingface.co/DeeJanuz/frametop-hands-models), each file checked against its SHA-256. They're licensed CC BY-NC 4.0 (non-commercial), not MIT, so they aren't in this repository; `models/student/LICENSE` and `README.md` come with them. `hands/models.sh check` checks them. A new version means new pins in `models.sh` (`rev`, `model`, the sums).
|
||||
- `models.sh`: fetches the fine-tuned models into `models/student` (`hands/build.sh` runs it, so source installs and the release image both get them): MediaPipe's two models retrained on the Frame's infrared cameras with the [Frametop Hands](https://huggingface.co/datasets/DeeJanuz/frametop-hands) dataset, from a fixed commit of [DeeJanuz/frametop-hands-models](https://huggingface.co/DeeJanuz/frametop-hands-models), each file checked against its SHA-256. They're licensed CC BY-NC 4.0 (non-commercial), not MIT, so they aren't in this repository; `models/student/LICENSE` and `README.md` come with them. `hands/models.pin` pins them: the commit, the version folder and every file's sum (frame-hands' `model-repo/pin.py` writes it); `hands/models.sh check` checks them against it. Installs don't need a release for a new version: when hand tracking starts, `frametop-hands.service` runs `hands/models.sh auto`, which reads `hands/models.pin` from GitHub (main for a stable release, experimental for an experimental one or a clone on that branch; at most every 15 minutes, within a minute) and, if it pins other models, fetches them into `models/.student-new`, checks every sum, and swaps that folder in whole. Offline, or anything not matching, it keeps the models it has. A pin only names the files above, and its `format` says which ft-hands it needs: raise it when a model needs a newer one, and installs without it keep theirs. `HANDS_MODELS_UPDATE=0` turns it off; `hands/models.sh update` checks now.
|
||||
|
||||
To try the hand cutouts without restarting the desktop, `screens/build/ft-handtest [--distance m] [--width m] [--seconds s] [--matte circle|mesh|capsules] [--margin deg]` (built by `screens/build.sh`, run in the dev container, with hand tracking on) shows a test panel of its own, a light grid 1 m wide and 0.8 m ahead by default, and cuts your hands out of it the way ft-screens cuts them out of the screens. Every 2 s it prints the composite's cost.
|
||||
|
||||
|
||||
@@ -10,7 +10,10 @@ PartOf=steamvr.service
|
||||
# In the container it was built in (scripts/in-box: dev for a clone, the release's own for a
|
||||
# release; it's built against Fedora's libraries). It reads ft-camd's ring and
|
||||
# writes /run/user/UID/frametop-hands/hands and gestures. Settings: HANDS_* in ~/.config/frametop.conf.
|
||||
# Before it starts, newer fine-tuned models if its branch pins them (hands/models.sh auto: at most
|
||||
# every 15 minutes, never more than a minute, offline or failing it starts with the ones it has).
|
||||
ExecStartPre=-/usr/bin/pkill -x ft-hands
|
||||
ExecStartPre=-/usr/bin/timeout 60 @REPO@/hands/models.sh auto
|
||||
ExecStart=@REPO@/scripts/in-box @REPO@/hands/build/ft-hands --status 60
|
||||
ExecStopPost=-/usr/bin/pkill -x ft-hands
|
||||
Restart=always
|
||||
|
||||
@@ -0,0 +1,118 @@
|
||||
/*
|
||||
* fh_events - what Frametop did with the tracked hands, one JSON line per decision, for a
|
||||
* testing session: the Hand Recorder copies them beside the recording (hands/rec/eventlog.py),
|
||||
* so a mark or a spoken note can be lined up with what the panels and the pointer did just
|
||||
* before it, and a session can say how many pokes, pinches and strokes worked.
|
||||
*
|
||||
* /run/user/UID/frametop-hands/events.jsonl, appended only while events-want exists next to
|
||||
* it (the recorder makes it while a testing session records and removes it after), so normal
|
||||
* use writes nothing. A writer looks for events-want at most once a second, when it has
|
||||
* something to write, and opens the file again if it was removed.
|
||||
*
|
||||
* A line:
|
||||
* {"t": CLOCK_MONOTONIC ns, "src": "screens" | "pointer", "kind": "...", "outcome": "...",
|
||||
* "hand": "left" | "right" | "", "panel": the screen's number as the logs give it (0: none),
|
||||
* "text": the log line}
|
||||
* kind: what was tried. ft-screens: poke, pinch (a click by pinching), grab (a panel taken by
|
||||
* a pinch or fist), throw, push, flick, card, scroll, tap, lazy-susan, stretch, grab-that,
|
||||
* palm-menu, far-stretch, hold (whatever a hand held, when it was lost). The pointer
|
||||
* helper: ray (the hand's laser), click, drag, right-click, scroll, handoff.
|
||||
* outcome: done (it happened), refused (a rule held it back; text says which), missed
|
||||
* (nothing there to act on), end (a hold ended), lost (it ended because the hand was lost),
|
||||
* passed (handed to the other program: the pointer leaves a pinch at a panel to ft-screens),
|
||||
* info (a note: a hold going on, a stroke seen before the panels turned).
|
||||
* Each line is one write() to an O_APPEND file, so the two writers don't interleave.
|
||||
*/
|
||||
#ifndef FH_EVENTS_H
|
||||
#define FH_EVENTS_H
|
||||
|
||||
#include <fcntl.h>
|
||||
#include <stdarg.h>
|
||||
#include <stdio.h>
|
||||
#include <string.h>
|
||||
#include <sys/stat.h>
|
||||
#include <time.h>
|
||||
#include <unistd.h>
|
||||
|
||||
#define FH_EVENTS_FILE "events.jsonl"
|
||||
#define FH_EVENTS_WANT "events-want"
|
||||
|
||||
static inline long long fh_events_now_ns(void) {
|
||||
struct timespec ts;
|
||||
clock_gettime(CLOCK_MONOTONIC, &ts);
|
||||
return (long long)ts.tv_sec * 1000000000LL + ts.tv_nsec;
|
||||
}
|
||||
|
||||
/* The open file while a session wants events, else -1 (looked at again after a second). */
|
||||
static inline int fh_events_fd(void) {
|
||||
static int fd = -1;
|
||||
static long long checked = 0;
|
||||
static ino_t ino = 0;
|
||||
long long now = fh_events_now_ns();
|
||||
if (now - checked < 1000000000LL && checked) return fd;
|
||||
checked = now;
|
||||
char dir[64], path[96];
|
||||
snprintf(dir, sizeof dir, "/run/user/%u/frametop-hands", (unsigned)getuid());
|
||||
snprintf(path, sizeof path, "%s/" FH_EVENTS_WANT, dir);
|
||||
struct stat st;
|
||||
if (stat(path, &st) != 0) {
|
||||
if (fd >= 0) close(fd);
|
||||
fd = -1;
|
||||
return fd;
|
||||
}
|
||||
snprintf(path, sizeof path, "%s/" FH_EVENTS_FILE, dir);
|
||||
if (fd >= 0 && (stat(path, &st) != 0 || st.st_ino != ino)) { /* removed since: a new one */
|
||||
close(fd);
|
||||
fd = -1;
|
||||
}
|
||||
if (fd < 0) {
|
||||
fd = open(path, O_WRONLY | O_APPEND | O_CREAT | O_CLOEXEC, 0600);
|
||||
if (fd >= 0 && fstat(fd, &st) == 0) ino = st.st_ino;
|
||||
}
|
||||
return fd;
|
||||
}
|
||||
|
||||
static inline size_t fh_events_quote(char *out, size_t room, const char *s) {
|
||||
size_t n = 0;
|
||||
for (; *s && n + 7 < room; s++) {
|
||||
unsigned char c = (unsigned char)*s;
|
||||
if (c == '"' || c == '\\') { out[n++] = '\\'; out[n++] = (char)c; }
|
||||
else if (c == '\n') { out[n++] = '\\'; out[n++] = 'n'; }
|
||||
else if (c < 0x20) n += (size_t)snprintf(out + n, room - n, "\\u%04x", c);
|
||||
else out[n++] = (char)c;
|
||||
}
|
||||
out[n] = 0;
|
||||
return n;
|
||||
}
|
||||
|
||||
/* One decision. side: 0 left, 1 right, -1 neither; panel: the logs' screen number, 0 none. */
|
||||
static inline void fh_event_text(const char *src, const char *kind, const char *outcome, int side, int panel,
|
||||
const char *text) {
|
||||
int fd = fh_events_fd();
|
||||
if (fd < 0) return;
|
||||
char q[1024], line[1400];
|
||||
fh_events_quote(q, sizeof q, text ? text : "");
|
||||
int n = snprintf(line, sizeof line,
|
||||
"{\"t\": %lld, \"src\": \"%s\", \"kind\": \"%s\", \"outcome\": \"%s\", \"hand\": \"%s\", "
|
||||
"\"panel\": %d, \"text\": \"%s\"}\n",
|
||||
fh_events_now_ns(), src, kind, outcome, side == 0 ? "left" : side == 1 ? "right" : "", panel, q);
|
||||
if (n > 0 && n < (int)sizeof line) {
|
||||
ssize_t w = write(fd, line, (size_t)n);
|
||||
(void)w;
|
||||
}
|
||||
}
|
||||
|
||||
static inline void fh_event(const char *src, const char *kind, const char *outcome, int side, int panel,
|
||||
const char *fmt, ...) __attribute__((format(printf, 6, 7)));
|
||||
static inline void fh_event(const char *src, const char *kind, const char *outcome, int side, int panel,
|
||||
const char *fmt, ...) {
|
||||
if (fh_events_fd() < 0) return;
|
||||
char text[1024];
|
||||
va_list ap;
|
||||
va_start(ap, fmt);
|
||||
vsnprintf(text, sizeof text, fmt, ap);
|
||||
va_end(ap);
|
||||
fh_event_text(src, kind, outcome, side, panel, text);
|
||||
}
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,16 @@
|
||||
# The fine-tuned hand models ft-hands uses by default (hands/models.sh): a version folder of the
|
||||
# Hugging Face repo DeeJanuz/frametop-hands-models at a fixed commit, and every file's sha256.
|
||||
# Installs read this file from GitHub (main or experimental, by their channel) when hand tracking
|
||||
# starts, so a new pin pushed there reaches them without a release. "format" is what ft-hands
|
||||
# needs to run them: raise it when a model needs a newer ft-hands, and older installs keep theirs.
|
||||
# Made by frame-hands' model-repo/pin.py.
|
||||
format 1
|
||||
rev 41afd78fbd4527456fc4cbd840ffc9f39fdc939a
|
||||
model v13b-p1
|
||||
file NOTICE 0e6b4fa8ea37665cb699abc36fcf1ee0ffaa63a78dfee788be6613dea6607f84
|
||||
file hand.ncnn.bin d6a6f00d845fb9ee490276d84b1e978da884361c372614546ed2b1b4d2f5c449
|
||||
file hand.ncnn.param 54e08f284ff6f121ff00f4a9c65afc38c0dbb22a32f4af44fc4e0fde6c4a1bbd
|
||||
file palm.ncnn.bin 9865f5fa3c6bc885cb0b50f201c4ac1bd83b2c1ad64af4e122efa3fc56d61909
|
||||
file palm.ncnn.param c61df2dde5b221a55c35c9693e530845a95ce2732949a9b3d3c583e3b2bae92e
|
||||
file LICENSE 41003d4a74749c0220e33dd415042164b5a1093ed401f36277234f772d22d3d0
|
||||
file README.md 29c6c5460e47412a9a37b22d6a7c8a9ed266a872bbcce93d3867f9e56cfe0ca1
|
||||
+126
-44
@@ -1,55 +1,137 @@
|
||||
#!/usr/bin/env bash
|
||||
# Fetch the fine-tuned hand models into hands/models/student/ on the Frame (in the image build,
|
||||
# here: pack/Containerfile runs hands/build.sh, which runs this). They come from a fixed commit of
|
||||
# the Hugging Face repo below and are checked against their sha256. ft-hands uses them when they're
|
||||
# there (HANDS_MODELS overrides; hands/models/ncnn, the stock MediaPipe models, otherwise).
|
||||
# here: pack/Containerfile runs hands/build.sh, which runs this). hands/models.pin names them: a
|
||||
# fixed commit of the Hugging Face repo below, and every file's sha256. ft-hands uses them when
|
||||
# they're there (HANDS_MODELS overrides; hands/models/ncnn, the stock MediaPipe models, otherwise).
|
||||
# palm.ncnn.*, hand.ncnn.* MediaPipe's palm detector and hand landmark model, fine-tuned on the
|
||||
# Frametop hand dataset (DeeJanuz/frametop-hands)
|
||||
# LICENSE, README.md their licence (CC BY-NC 4.0, not Frametop's MIT) and model card
|
||||
# Usage: hands/models.sh [fetch] fetch what's missing or doesn't match (the default)
|
||||
# hands/models.sh check check every file's sha256 (exit 1 if any is missing or wrong)
|
||||
# Usage: hands/models.sh [fetch] fetch what's missing or doesn't match models.pin (the default)
|
||||
# hands/models.sh check check every file against the pin it was fetched by (exit 1 if not)
|
||||
# hands/models.sh update fetch the models the install's branch pins on GitHub now (main for
|
||||
# a stable release, experimental for an experimental one or a
|
||||
# clone on that branch), if they changed. Into a folder of its own,
|
||||
# swapped in once every file matches. Offline or nothing new: no-op.
|
||||
# hands/models.sh auto update, as frametop-hands.service runs it when hand tracking
|
||||
# starts: not with HANDS_MODELS_UPDATE=0 in ~/.config/frametop.conf,
|
||||
# and at most every 15 minutes.
|
||||
set -euo pipefail
|
||||
root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)
|
||||
. "$root/scripts/_env.sh"
|
||||
case ${1:-fetch} in fetch | check | update | auto) ;; *) sed -n '9,18p' "$0" >&2; exit 2 ;; esac
|
||||
|
||||
on_frame_script "${1:-fetch}" "$FRAME_REPO/hands/models/student" <<'SCRIPT'
|
||||
on_frame_script "${1:-fetch}" "$FRAME_REPO" <<'SCRIPT'
|
||||
set -euo pipefail
|
||||
mode=$1 dir=$2
|
||||
repo=DeeJanuz/frametop-hands-models
|
||||
rev=41afd78fbd4527456fc4cbd840ffc9f39fdc939a
|
||||
model=v13b-p1
|
||||
url=https://huggingface.co/$repo/resolve/$rev
|
||||
# file, sha256 (the repo's $model/ folder, but LICENSE and README.md at its top)
|
||||
files=(
|
||||
"NOTICE 0e6b4fa8ea37665cb699abc36fcf1ee0ffaa63a78dfee788be6613dea6607f84"
|
||||
"hand.ncnn.bin d6a6f00d845fb9ee490276d84b1e978da884361c372614546ed2b1b4d2f5c449"
|
||||
"hand.ncnn.param 54e08f284ff6f121ff00f4a9c65afc38c0dbb22a32f4af44fc4e0fde6c4a1bbd"
|
||||
"palm.ncnn.bin 9865f5fa3c6bc885cb0b50f201c4ac1bd83b2c1ad64af4e122efa3fc56d61909"
|
||||
"palm.ncnn.param c61df2dde5b221a55c35c9693e530845a95ce2732949a9b3d3c583e3b2bae92e"
|
||||
"LICENSE 41003d4a74749c0220e33dd415042164b5a1093ed401f36277234f772d22d3d0"
|
||||
"README.md 29c6c5460e47412a9a37b22d6a7c8a9ed266a872bbcce93d3867f9e56cfe0ca1"
|
||||
)
|
||||
[ "$mode" = check ] && [ ! -d "$dir" ] && { echo "hand models: $dir missing" >&2; exit 1; }
|
||||
mkdir -p "$dir"
|
||||
cd "$dir"
|
||||
bad=0
|
||||
for entry in "${files[@]}"; do
|
||||
read -r name sum <<<"$entry"
|
||||
if [ -f "$name" ] && echo "$sum $name" | sha256sum -c --quiet --status 2>/dev/null; then continue; fi
|
||||
if [ "$mode" = check ]; then echo "hands models: $name missing or changed" >&2; bad=1; continue; fi
|
||||
src=$url/$model/$name
|
||||
case $name in LICENSE | README.md) src=$url/$name ;; esac
|
||||
echo "fetching hand model $name"
|
||||
curl -fsSL --retry 3 -o "$name.part" "$src"
|
||||
if ! echo "$sum $name.part" | sha256sum -c --quiet --status; then
|
||||
echo "hands models: $name: the download's sha256 doesn't match; not kept" >&2
|
||||
rm -f "$name.part"
|
||||
exit 1
|
||||
fi
|
||||
mv "$name.part" "$name"
|
||||
done
|
||||
[ "$bad" = 0 ] || exit 1
|
||||
[ "$mode" = check ] && { echo "hand models: $model in $dir, all files match"; exit 0; }
|
||||
echo "$repo@${rev::7} $model" > VERSION
|
||||
echo "hand models: $model in $dir"
|
||||
mode=$1 tree=$2
|
||||
models=$tree/hands/models dir=$tree/hands/models/student
|
||||
hf=DeeJanuz/frametop-hands-models gh=Frametop/frametop
|
||||
names=" NOTICE hand.ncnn.bin hand.ncnn.param palm.ncnn.bin palm.ncnn.param LICENSE README.md "
|
||||
|
||||
# read_pin FILE: check every line of a pin, then set rev, model and files ("name sha256" each).
|
||||
# Anything unexpected fails it: a pin from GitHub only names files this list allows.
|
||||
read_pin() {
|
||||
local key a b n format=
|
||||
rev= model= files=()
|
||||
while read -r key a b; do
|
||||
case $key in '' | '#'*) continue ;; esac
|
||||
case $key in
|
||||
format) [ "$a" = 1 ] && [ -z "$b" ] || return 1; format=$a ;;
|
||||
rev) [[ $a =~ ^[0-9a-f]{40}$ ]] && [ -z "$b" ] || return 1; rev=$a ;;
|
||||
model) [[ $a =~ ^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$ ]] && [ -z "$b" ] || return 1; model=$a ;;
|
||||
file) [[ $names == *" $a "* && $b =~ ^[0-9a-f]{64}$ ]] || return 1; files+=("$a $b") ;;
|
||||
*) return 1 ;;
|
||||
esac
|
||||
done <"$1"
|
||||
[ -n "$format" ] && [ -n "$rev" ] && [ -n "$model" ] || return 1
|
||||
for n in hand.ncnn.bin hand.ncnn.param palm.ncnn.bin palm.ncnn.param; do
|
||||
[[ " ${files[*]} " == *" $n "* ]] || return 1
|
||||
done
|
||||
}
|
||||
|
||||
matches() { [ -f "$1" ] && [ "$(sha256sum <"$1" | cut -d' ' -f1)" = "$2" ]; }
|
||||
|
||||
# fetch_into DIR: fetch the pin's files that are missing there or don't match.
|
||||
fetch_into() {
|
||||
local d=$1 entry name sum src
|
||||
mkdir -p "$d"
|
||||
for entry in "${files[@]}"; do
|
||||
read -r name sum <<<"$entry"
|
||||
matches "$d/$name" "$sum" && continue
|
||||
src=https://huggingface.co/$hf/resolve/$rev/$model/$name
|
||||
case $name in LICENSE | README.md) src=https://huggingface.co/$hf/resolve/$rev/$name ;; esac
|
||||
echo "fetching hand model $name ($model)"
|
||||
if ! curl -fsSL --retry 3 --connect-timeout 10 --max-filesize 100000000 -o "$d/$name.part" "$src"; then
|
||||
rm -f "$d/$name.part"; return 1
|
||||
fi
|
||||
if ! matches "$d/$name.part" "$sum"; then
|
||||
echo "hand models: $name's sha256 doesn't match the pin; not kept" >&2
|
||||
rm -f "$d/$name.part"; return 1
|
||||
fi
|
||||
mv "$d/$name.part" "$d/$name"
|
||||
done
|
||||
}
|
||||
|
||||
stamp() { echo "$hf@${rev::7} $model" >"$1/VERSION"; }
|
||||
|
||||
case $mode in
|
||||
fetch)
|
||||
read_pin "$tree/hands/models.pin" || { echo "hand models: hands/models.pin isn't a valid pin" >&2; exit 1; }
|
||||
fetch_into "$dir"
|
||||
cp "$tree/hands/models.pin" "$dir/PIN"; stamp "$dir"
|
||||
echo "hand models: $model in $dir"
|
||||
;;
|
||||
check)
|
||||
# Against the pin they were fetched by: an update may have replaced the release's own.
|
||||
pin=$dir/PIN; [ -f "$pin" ] || pin=$tree/hands/models.pin
|
||||
[ -d "$dir" ] || { echo "hand models: $dir missing" >&2; exit 1; }
|
||||
read_pin "$pin" || { echo "hand models: $pin isn't a valid pin" >&2; exit 1; }
|
||||
bad=0
|
||||
for entry in "${files[@]}"; do
|
||||
read -r name sum <<<"$entry"
|
||||
matches "$dir/$name" "$sum" || { echo "hand models: $name missing or changed" >&2; bad=1; }
|
||||
done
|
||||
[ "$bad" = 0 ] || exit 1
|
||||
echo "hand models: $model in $dir, all files match"
|
||||
;;
|
||||
update | auto)
|
||||
if [ "$mode" = auto ]; then
|
||||
conf=$(sed -n 's/#.*//; s/^[[:space:]]*HANDS_MODELS_UPDATE[[:space:]]*=[[:space:]]*//p' ~/.config/frametop.conf 2>/dev/null | tail -1)
|
||||
conf=${FT_HANDS_MODELS_UPDATE:-$conf}
|
||||
[ "$(echo $conf | tr -d "\"'")" = 0 ] && { echo "hand models: not updating (HANDS_MODELS_UPDATE=0)"; exit 0; }
|
||||
[ -n "$(find "$models/.student-checked" -mmin -15 2>/dev/null)" ] && exit 0
|
||||
fi
|
||||
if [ -f "$tree/.frametop-release" ]; then
|
||||
case $(sed -n 's/^CHANNEL=//p' "$tree/.frametop-release" | tail -1) in
|
||||
experimental) branch=experimental ;; *) branch=main ;;
|
||||
esac
|
||||
else
|
||||
branch=$(git -C "$tree" symbolic-ref --short -q HEAD 2>/dev/null || true)
|
||||
fi
|
||||
case $branch in main | experimental) ;; *) echo "hand models: no updates for branch '$branch'"; exit 0 ;; esac
|
||||
mkdir -p "$models"
|
||||
exec 9>"$models/.student-lock"
|
||||
flock -n 9 || { echo "hand models: another update is running"; exit 0; }
|
||||
touch "$models/.student-checked"
|
||||
tmp=$(mktemp); trap 'rm -f "$tmp"' EXIT
|
||||
code=$(curl -sS --connect-timeout 4 --max-time 15 --max-filesize 16384 -o "$tmp" -w '%{http_code}' \
|
||||
"https://raw.githubusercontent.com/$gh/$branch/hands/models.pin") || code=
|
||||
case $code in
|
||||
200) ;;
|
||||
'') echo "hand models: couldn't reach GitHub; keeping these"; exit 0 ;;
|
||||
*) echo "hand models: $branch pins none ($code); keeping these"; exit 0 ;;
|
||||
esac
|
||||
cmp -s "$tmp" "$dir/PIN" 2>/dev/null && { echo "hand models: up to date ($(cat "$dir/VERSION" 2>/dev/null))"; exit 0; }
|
||||
read_pin "$tmp" || { echo "hand models: $branch's pin isn't valid (or needs a newer Frametop); keeping these" >&2; exit 1; }
|
||||
# Into .student-new, seeded with the files we have (a stopped update keeps what it fetched),
|
||||
# then swapped with student in one rename: ft-hands never sees half a set.
|
||||
new=$models/.student-new
|
||||
mkdir -p "$new"
|
||||
[ -d "$dir" ] && cp -a --update=none "$dir/." "$new/"
|
||||
rm -f "$new"/*.part
|
||||
fetch_into "$new" || { echo "hand models: update failed; keeping these" >&2; exit 1; }
|
||||
cp "$tmp" "$new/PIN"; stamp "$new"
|
||||
if [ -d "$dir" ]; then mv -T --exchange "$new" "$dir"; rm -rf "$new"; else mv -T "$new" "$dir"; fi
|
||||
echo "hand models: updated to $model from $branch's pin"
|
||||
;;
|
||||
esac
|
||||
SCRIPT
|
||||
@@ -1,14 +1,16 @@
|
||||
# Recording your hands for the Frametop hand dataset
|
||||
|
||||
Version: 2026-10-03
|
||||
Version: 2026-10-14
|
||||
|
||||
Frametop's hand tracking needs a small model that finds hands in the headset's camera images. To train it, we're collecting recordings of many people's hands into a public dataset. This page explains what the hand recorder records, where it goes, and what you agree to if you take part. Please read all of it.
|
||||
|
||||
It covers the **bounty rounds** (short rounds of what the hand tracker needs most, such as taps, pinches and grabs on windows that appear in front of you) and Frametop's **full sessions** and **quick rounds**. **Testing sessions**, where you use Frametop and mark what went wrong, have their own, separate text: they're never used for training and aren't part of this dataset.
|
||||
|
||||
## Who runs this
|
||||
|
||||
Frametop is an independent open-source project, maintained by DeeJanuz (<https://github.com/DeeJanuz>). It isn't affiliated with or endorsed by Valve or Hugging Face. Steam and Steam Frame are trademarks of Valve Corporation. This text was written without a lawyer, in good faith; if something in it seems wrong or unclear, please say so before you take part.
|
||||
|
||||
You can reach the maintainer through the Frametop repository's issues (<https://github.com/DeeJanuz/frametop/issues>) or the dataset's discussion page on Hugging Face. Both are public.
|
||||
You can reach the maintainer through the Frametop repository's issues (<https://github.com/Frametop/frametop/issues>) or the dataset's discussion page on Hugging Face. Both are public.
|
||||
|
||||
## Who can take part
|
||||
|
||||
@@ -20,9 +22,9 @@ You can reach the maintainer through the Frametop repository's issues (<https://
|
||||
|
||||
While a session runs, the hand recorder saves:
|
||||
|
||||
- **Camera images.** Infrared images from the headset's 4 tracking cameras, about 10 sets a second. They show your hands, your arms, your clothing and whatever is in front of you: your room, your desk and the things on it. They're grey, low-detail images, but people and things can be recognised in them. The size and shape of your hands can be measured from them: that's part of what they're for.
|
||||
- **Camera images.** Infrared images from the headset's 4 tracking cameras: about 10 sets a second in a full session, 30 in a bounty round. They show your hands, your arms, your clothing and whatever is in front of you: your room, your desk and the things on it. They're grey, low-detail images, but people and things can be recognised in them. The size and shape of your hands can be measured from them: that's part of what they're for.
|
||||
- **Motion.** The position and rotation of the headset, many times a second, and of the controllers when you use them in the recording.
|
||||
- **Prompts.** What you were asked to do and when, and what the live hand tracker saw at the time.
|
||||
- **Prompts.** What you were asked to do and when (in a bounty round, which window and which part of it), and what the live hand tracker saw at the time.
|
||||
- **Calibration.** Where the cameras sit on the headset and how their lenses bend the image. Serial numbers and other fields that identify your headset are removed first, and the export lists what was removed.
|
||||
- **Your answers.** Which objects you had, the lighting, whether you wore sleeves, rings or a watch, your handedness if you gave it, and any notes you typed in the checklist or in Review. Notes are uploaded with the recordings and read by the maintainer: don't put anything in them that identifies you.
|
||||
- **Times.** When each session was recorded, with your time zone.
|
||||
|
||||
+97
-5
@@ -12,6 +12,7 @@ The plan this belongs to is `~/Desktop/Projects/frame-hands/notes/hands-plan.md`
|
||||
| `hands/rec/session.py` (Python, no Qt) | The session runner. It reads `script.json`, starts and stops the recordings, and drives the panel. It reads the live hands file for feedback and writes each take's files. It also runs from the command line (`--dry-run`) for testing. |
|
||||
| `hands/rec/script.json` | The guided script: sections, prompts, timings. |
|
||||
| `hands/rec/script-v2.json` | The v2 script (`ft-handrec --script script-v2.json`, `session.py --script`): a demo clip per step (below). Being tried; it becomes `script.json` once settled. |
|
||||
| `hands/rec/bounties.json`, `hands/rec/bounties/` | The bounty board and each bounty's round script (below, "Bounties"). |
|
||||
| `hands/rec/poses/` | The pose pictures, `poses.json` and its PNGs (below). |
|
||||
| `hands/rec/demos/` | The demo clips, `<id>.gif`, made from `demo/parts.json` (below). |
|
||||
| `hands/rec/takes.py` (Python, no Qt) | Reads sessions and takes from disk: frame sets for review, deleted ranges, export (compress, strip, manifest, checksums). |
|
||||
@@ -20,7 +21,7 @@ The plan this belongs to is `~/Desktop/Projects/frame-hands/notes/hands-plan.md`
|
||||
| `hands/rec/build.sh` | Builds `ft-handpanel` into `hands/rec/build/`, like `gaze/build.sh`. |
|
||||
| `hands/rec/CONSENT.md`, `hands/rec/UPLOAD.md` | The texts the window shows. |
|
||||
| `hands/rec/install.sh` | Installs the recorder on a Frame with Frametop: the dev container's packages, hands/build.sh, the panel, ft-camd's capabilities, and the menu entry (`uninstall` removes the entry). |
|
||||
| ft-hands `--record-hz N` (done) | Records at most N frame sets a second. The recorder uses 10. |
|
||||
| ft-hands `--record-hz N` (done) | Records at most N frame sets a second. The recorder uses 10, or a script's `record_hz` (the bounties: 30). |
|
||||
| `hands/camcheck.py` (shared, standard library) | Are all four mono cameras running? The recorder runs it before a session and when a step sees no hands (below; `hands/README.md`, "Camera check"). |
|
||||
|
||||
Every part runs in the dev container, as ft-hands and Input Settings do. The host Python isn't used: it lacks PySide6 and zstd. `setup/dev-container.sh` gains `zstd`.
|
||||
@@ -36,6 +37,10 @@ Its tarball keeps this tree's layout (`hands/build/`, `hands/rec/`, `hands/model
|
||||
|
||||
The recordings go to the same `~/.local/share/frametop/hands/contrib`, so either recorder shows both's sessions.
|
||||
|
||||
Its window has a **Getting ready** page between the consent and Before you start (`Backend.getReady`, `session.get_ready`). It checks that SteamVR runs, then starts what isn't running yet: the cameras, hand tracking and the headset panel, in that order. With Frametop's hand tracking installed (`frametopHands`) that means Frametop's own `frametop-hands` (which brings `frametop-camd`); a testing session needs that one. Otherwise it starts the recorder's units (`frametop-handrec-camd`, `frametop-handrec-hands`). The first part that can't start stops it with a message and Try again. What it started stops when the window closes; the panel stays up hidden until then, and the session reuses it.
|
||||
|
||||
From Frametop's desktop, the window inherits that session's `XDG_RUNTIME_DIR` (`/run/user/UID/frametop`) and `XDG_CONFIG_HOME` (`~/.config/frametop`). With those, `systemctl --user` can't reach the user's systemd, and OpenVR finds no `openvrpaths.vrpath`. The launcher (`bin/frametop-hand-recorder`) puts the real ones back. ft-handpanel also gets `openvr_env()` wherever it starts. On a Plasma desktop the window runs on Wayland and 0.85 times Plasma's scale (`FT_HANDREC_SCALE` sets another factor).
|
||||
|
||||
## Processes during a session
|
||||
|
||||
- **ft-camd** publishes the camera ring. If it isn't running, the session starts it as the transient user unit `frametop-handrec-camd.service`, the way `hands/ft-cutouts` starts `frametop-cutouts-camd.service` (needs `hands/build/ft-camd` with capabilities: `hands/run.sh caps`). An ft-camd already running from ft-cutouts or ft-handsctl is used as it is.
|
||||
@@ -65,10 +70,13 @@ Test hooks:
|
||||
calibration.json /persist/xrservice.json with identifying fields removed (below)
|
||||
device.json the rig's pose in the CAD frame from /persist/device_config.json (below)
|
||||
takes/<NN>-<section>/
|
||||
sets.bin ft-hands' recording (FHSET01, hands/track/record.h), 10 sets/s: the first part
|
||||
sets.bin ft-hands' recording (FHSET01, hands/track/record.h), 10 sets/s (a bounty: 30): the first part
|
||||
sets-2.bin, sets-3.bin, ... the next parts (step mode: one per step; any mode: after a pause)
|
||||
prompts.jsonl what the person was asked to do, when (below)
|
||||
poses.jsonl head and controller poses from ft-handpanel (below)
|
||||
live.bin, live-2.bin, ... a testing session: the live tracker's output per part (below, "Testing sessions")
|
||||
voice.bin, voice-2.bin, ... a testing session with "Record my voice" ticked: the microphone per part (same)
|
||||
events.jsonl, events-2.jsonl, ... a testing session: what the panels and the pointer did with the hands per part (same)
|
||||
take.json {"section", "title", "started_ns", "ended_ns", "status": "complete"|"stopped"|"skipped",
|
||||
"deleted": [[from_ns, to_ns], ...], "notes": "",
|
||||
"parts": {"sets.bin": {"names_swapped": false}, "sets-2.bin": {...}},
|
||||
@@ -82,9 +90,15 @@ All `_ns` times are CLOCK_MONOTONIC nanoseconds, the clock of `dqbuf_ns` in sets
|
||||
|
||||
```json
|
||||
{"schema": 1, "contributor": "<random uuid4>", "consent": {"version": "2026-10-02", "accepted": "<ISO time>", "adult": true},
|
||||
"optional": {"handedness": "right|left|both|", "notes": ""}}
|
||||
"testing_consent": {"version": "2026-10-14", "accepted": "<ISO time>", "adult": true},
|
||||
"optional": {"handedness": "right|left|both|", "notes": ""},
|
||||
"bounties": {"<id>": {"rounds": 2, "last": "<session id>"}}}
|
||||
```
|
||||
|
||||
`consent` is the agreement to CONSENT.md (the public dataset), `testing_consent` to TESTING.md (the test phase, below); either can be there without the other, and writing one keeps the other.
|
||||
|
||||
`bounties`: how often each bounty was done (recorder/bounties.py `mark_done`, when a bounty's round ends `done`); a new consent keeps it.
|
||||
|
||||
No name, email, or account. The contributor id is random, so several sessions from one person can be held out together in evaluation. Withdrawal also goes by that id.
|
||||
|
||||
### session.json
|
||||
@@ -96,7 +110,7 @@ No name, email, or account. The contributor id is random, so several sessions fr
|
||||
"checklist": {"objects": ["pencil", "phone", "cup", "keyboard", "mouse", "gamepad", "small"], "own_objects": ["..."],
|
||||
"controllers": "straps|none", "sleeves": "short|long|", "rings": false, "watch": false, "notes": ""},
|
||||
"device": {"steamos": "<VERSION_ID from /etc/os-release>", "steamvr": "<version if known>", "cameras": [{"name", "width", "height"}]},
|
||||
"mode": "step|auto", "quick": false, "shuffle": {"seed": 123, "sweeps": {"pose-sweeps": [{"id", "hands", "cues"}]}},
|
||||
"mode": "step|auto", "quick": false, "record_hz": 10, "bounty": {"id": "pinches", "title": "Pinches", "board_updated": "2026-10-14"}, "shuffle": {"seed": 123, "sweeps": {"pose-sweeps": [{"id", "hands", "cues"}]}},
|
||||
"takes": ["01-hand-size", "..."],
|
||||
"camera": {"status": "ok|unknown|degraded", "reason": "..."}, "stop_reason": "...",
|
||||
"sides": {"swapped": true|false|null, "decided_by": "auto|config|option|manual", "state": "decided|confirmed|...",
|
||||
@@ -108,6 +122,10 @@ No name, email, or account. The contributor id is random, so several sessions fr
|
||||
|
||||
`camera` is the camera check's verdict at the start (no paths or log lines: those go to `session.log`). `stop_reason` is there when a session was stopped at the no-hands screen, with the check's result.
|
||||
|
||||
`consent` (`{"kind": "public"|"testing", "version"}`) is the consent the session was recorded under, and `debug` is there for a testing session (below, "Testing sessions"). Sessions before 2026-10-14 have neither: they're public.
|
||||
|
||||
`record_hz` is the sets a second recorded; `bounty` is there for a bounty's round (below, "Bounties"). Sessions before the bounties have neither: they were recorded at 10.
|
||||
|
||||
A session started before step mode existed has no `mode`: it ran as `auto`. `quick` and `shuffle` (below, "Sweeps" and "Quick round") are missing from sessions before the sweeps.
|
||||
|
||||
### calibration.json
|
||||
@@ -144,8 +162,15 @@ One JSON object per line:
|
||||
{"t": 123, "event": "redo", "id": "static-poses/fist/left/near", "from": 123, "to": 123}
|
||||
{"t": 123, "event": "nohands", "id": "hand-size/flat/both", "reads": 120, "published": 118}
|
||||
{"t": 123, "event": "end", "status": "complete|stopped|skipped"}
|
||||
{"t": 123, "event": "scene", "id": "pinches/drag-right", "windows": [{"id": "a", "at": [x, y, z], "size": [w, h], "turn": [yaw, pitch], "look": "plain", "room": [12 floats]}]}
|
||||
{"t": 123, "event": "steady", "win": "a", "lit": "bar", "gesture": "pinch", "hand": ""}
|
||||
{"t": 123, "event": "switch", "id": "hand-size/both", "to": "backs toward you", "k": 1}
|
||||
{"t": 123, "event": "prompt", "id": "pinches/drag-right/3-pinch", "text": "...", "pose": "pinch", "cue": true, "step": "pinches/drag-right",
|
||||
"act": {"win": "a", "lit": "bar", "gesture": "pinch", "hand": "", "label": "", "ghost": [dx, dy, dz, yaw, pitch, roll], "ghost_room": [12 floats]}}
|
||||
```
|
||||
|
||||
A scene step (below, "Scenes") logs `scene` when its windows go up, with each one's pose in the room (`room`, from the panel; null in a dry run), and `scene` with no windows when they go. Each act is a `prompt` like a sweep's cue, with `act`: the window, the lit part, the gesture asked for (`rest`: let go, nothing lit), the hand when the act names one, and the ghost (in the window's frame, and its pose in the room). `steady` is a light that stays on for the whole step. `switch` is a step's change of pose (below, "Two poses in a step"): the pose asked for from then until the step's end or the next `switch`.
|
||||
|
||||
`feedback` is written about twice a second while recording. It's the live tracker's view, kept for later checks; it's not a label.
|
||||
|
||||
A prompt holds from its `prompt` until the next `prompt`, `ready`, `wait` or `end`. A step in step mode reads:
|
||||
@@ -187,6 +212,7 @@ Commands (UTF-8; `|` starts a new line in text):
|
||||
| `text <text>` | The instruction, large, wrapped to the panel's width, centred. |
|
||||
| `note <text>` | An orange line under the instruction: a warning ("I can't see your left hand"). Empty clears it. |
|
||||
| `countdown <0..1>` / `countdown off` | A thin bar along the bottom: the share of this prompt's time left. |
|
||||
| `split <at> <label>;...` / `split off` | Ticks on that bar where the step changes pose, each at `at` (the share of time left then) with "Switch to <label>" over it, cyan once the bar comes down to it ("Two poses in a step"). |
|
||||
| `hands <left> <right>` | Two chips, "Left hand" and "Right hand", each `seen` (green), `lost` (orange) or `off` (hidden). |
|
||||
| `bar <target 0..1> <current 0..1 or -1> [near label] [far label]` / `bar off` | The near/far bar for the push out and back: a horizontal track with a target marker and the hand's current position. |
|
||||
| `paused on` / `paused off` | A "Paused" overlay over the picture. |
|
||||
@@ -199,12 +225,17 @@ Commands (UTF-8; `|` starts a new line in text):
|
||||
| `rec on` / `rec off` | The red "Rec" by the step line. |
|
||||
| `strip <cue> <path>\|<mode>\|<label>;...` / `strip off` | A sweep's row of pose pictures (path `-`: none; mode `-` or `mirror`), each with its label under it; the one at index `cue` (from 0, -1 for none) framed in cyan and bright, the others dim. It sits along the bottom of the space left, the where-to diagram at its right end if there is one, and the text above it; it takes the left column's place. Each picture is loaded once. A file that can't be read: `error can't read ...`, and that item shows an empty frame. |
|
||||
| `target <x> <y> <z> [show\|hold <0..1>\|done]` / `target off` | The touch target: a small sphere-like dot about 2 cm across, in its own overlay (`frametop.handpanel.target`). The point is in the head frame (metres, +x right, +y up, -z forward). The first `target` with a new point places it in the room with the current HMD pose, and it stays there. Later commands with the same point change only the state. `hold` draws a filling ring, `done` turns it green. Reply: `ok <room x> <room y> <room z>`. |
|
||||
| `layout full` / `layout top` | `top`: only a band along the panel's top (title, step, Rec, the big text, two lines of text, a note and the time-left bar), the rest clear, so a scene's windows can be seen below it. |
|
||||
| `win <id> place <x> <y> <z> <w> <h> [<yaw> <pitch>] [<look>]` | A scene's window (below, "Scenes"; at most 4), in its own overlay (`frametop.handpanel.win.<id>`): w x h metres, centred at the point in the head frame made level (the head's forward direction, flattened, +y up), facing the eye, turned by yaw and pitch. It stays where it was put. `look`: `plain` (faint text lines), `grid:CxR` (1-6 each way) or `list:N` (1-10 rows). Reply: `ok <12 floats>`, its pose in the room. |
|
||||
| `win <id> lit <part>[+<part>...] [<colour>] [<label>]` | Lights parts: `none`, `bar` (the title bar), `left`, `right`, `top`, `bottom`, `tl`, `tr`, `bl`, `br`, `button:N`, `item:N`. Colours: `cyan` (default), `purple`, `orange` (a button drawn hollow with a cross: don't touch), `green`. The label (`_` for a space) goes beside the first part. |
|
||||
| `win <id> ghost <dx> <dy> <dz> <yaw> <pitch> <roll> [<scale>]` / `win <id> ghost off` | A dashed outline of the window, moved, turned and scaled in the window's frame: where to take it. Reply: `ok <12 floats>`. |
|
||||
| `win <id> off` / `win clear` | Hides one window and its ghost, or all of them. |
|
||||
| `poses start <path>` / `poses stop` | Log poses to the path (appending, `poses.jsonl` format above) from a thread at 250 Hz, until stopped. |
|
||||
| `devices` | Reply: `ok hmd <r> left <r or -> right <r or ->`, the current `ETrackingResult` values (`-` for no device in that role). |
|
||||
| `head` | Reply: `ok <12 floats>`, the current HMD pose (standing universe). |
|
||||
| `ping` | `ok shown` or `ok hidden`. |
|
||||
|
||||
It quits on SteamVR's quit event, as ft-gazepanel does. `--no-vr --dump DIR` writes each picture to `DIR/panel.pam`, to check the layout without a headset.
|
||||
It quits on SteamVR's quit event, as ft-gazepanel does. `--no-vr --dump DIR` writes each picture to `DIR/panel.pam` (and the windows to `win-<id>.pam`, the ghosts to `ghost-<id>.pam`), to check the layout without a headset.
|
||||
|
||||
## The pose pictures (`poses/`)
|
||||
|
||||
@@ -276,6 +307,34 @@ The second in-headset session (sessions/20261002-202734) took about 16 min and w
|
||||
|
||||
For more lighting rounds: "Quick round (about 3 min)" on the checklist page, `session.py --quick`. It has the sections marked `quick` (hand size, the pose sweeps without the one-hand ones, touch the dot, no hands), about 2 min recorded in 6 steps. session.json gets `"quick": true`. The checklist page suggests it (and picks it) once a full session has gone to the end (`Backend.hasFullSession`: a session.json with status `done`, not quick, not a dry run).
|
||||
|
||||
### Scenes
|
||||
|
||||
A section with `"kind": "scene"` (the bounties, below) puts windows in front of the person and asks for real interactions on them: tap the lit button, pinch the lit corner and drag the window to a ghost, grab the edge like a playing card and throw it. The windows don't move or click. The point is the hand doing the whole motion as it would on a real window (the reach, the fingers shaping on the way, the contact, the carry, the let-go), with each try's target and time as weak labels, instead of held poses. The cameras don't see the windows, so the steps put them where the tracker struggles: low, close, at the view's edge, two hands, near misses.
|
||||
|
||||
A step:
|
||||
|
||||
```json
|
||||
{"id": "drag-right", "hands": "right", "text": "Pinch the lit bar and take it to the ghost.",
|
||||
"windows": [{"id": "a", "at": [0.05, -0.2, -0.45], "size": [0.34, 0.24], "turn": [0, 0], "look": "plain|grid:3x3|list:6"}],
|
||||
"steady": [{"win": "b", "lit": "bar", "gesture": "pinch", "label": "L"}],
|
||||
"acts": {"gen": "moves", "gesture": "pinch", "grab": "bar", "every_s": 3.5, "gap_s": 1.2, "count": 3,
|
||||
"moves": [[0, 0.15, 0], {"grab": "br", "ghost": [0.05, -0.04, 0, 0, 0, 0, 1.3], "gesture": "nogo"}]}}
|
||||
```
|
||||
|
||||
- `windows`: 1 to 4, placed when Next is pressed, where the head faces then (levelled), and left there for the step (the panel's `win ... place`). `at` is in that frame, metres.
|
||||
- `acts`: a list of acts, `{"s", "win", "lit", "gesture", "hand", "label", "say", "ghost"}`, or a generator (or a list mixing both):
|
||||
- `"gen": "buttons"`: a lit button or row every `every_s`, lit for `lit_s` with a rest after, never the same one twice running; `count` or `seconds`; `mix` weights the gestures (`{"tap": 2, "double": 1}`); `hands: ["left", "right"]` labels each button L or R; `says` gives a gesture its own line; `windows` limits it to some.
|
||||
- `"gen": "moves"`: each move lights a part to grab (`grab`, `+` for two: `tl+br`) and, with a ghost, shows where to take the window; `every_s` each, then a `gap_s` rest ("Let go"); shuffled unless `"shuffle": false`; `count` repeats or cuts the list. A move is a ghost `[dx, dy, dz, yaw, pitch, roll, scale]` or an object overriding the generator's keys.
|
||||
- `steady`: lights that stay on for the whole step, on windows no act uses (hold a pinch on one while the other hand taps).
|
||||
- Gestures (`session.GESTURES`), each with its colour, mark and default line: `tap`, `double` ("2"), `hold` ("hold"), `middle` (purple), `middle-hold`, `pinch`, `card` (three fingers, a playing card), `fist` (green), `touch` (open hand, orange), `nogo` (orange, stop about 1 cm short), `rest` (nothing lit).
|
||||
- A step whose acts switch between the hands (L and R buttons, or acts naming each hand) gives every act at least `CHOICE_MIN_S` (2 s) lit, so there's time to see which hand and use it (user, 2026-10-14): the buttons generator stretches its lit time to that (keeping the rest after), and a written act shorter than that fails the plan.
|
||||
- The step's `seconds` are its acts'. The acts are drawn with `random.Random("<seed>/<section>/<step>")`, so `build_plan` with the session's seed gives them again; session.json's `shuffle.sweeps` lists each scene step's acts as `[gesture, window, part, hand]`.
|
||||
- While a scene runs, the panel is in its `layout top` band, which shows the act's line; a scene's acts get no pose picture. The events are above (prompts.jsonl).
|
||||
|
||||
### Two poses in a step
|
||||
|
||||
A step that asks for one pose and then another (hand size: palms toward you, then the backs) says when to change: `"switch": "backs toward you"` (at half time), or `[{"at": 0.5, "to": "backs toward you", "text": "Both hands about 40 cm out, still|Backs toward you"}]`, each `at` a share of the step's time, after the one before. The time-left bar gets a tick there with "Switch to <to>" over it (the panel's `split`), lit cyan once the bar comes down to it; `text`, if given, replaces the instruction then (not in a scene, whose acts keep their own lines). The step's prompt event lists its `switch`, and a `switch` event (`id`, `to`, `k`, counting from 1) marks each one's time, so labels can tell the two poses apart. Plain steps and scene steps take it; a sweep has its own cues.
|
||||
|
||||
### Step mode (the default) and auto mode
|
||||
|
||||
The first in-headset session (2026-10-02) went too fast: each prompt advanced after 4-8 s, before there was time to read it and find the hand shape. So by default every step waits:
|
||||
@@ -336,6 +395,38 @@ On 2026-10-02 a whole session showed "I can't see your hands": after the headset
|
||||
- **R (redo).** During a step's countdown or hold: that step starts again from its ready screen. At a step's ready screen: the step before it (in this section) goes again. Either way a `redo` event marks the range of the try being redone, so its labels are skipped; the sets stay, to delete in review if wanted.
|
||||
- A pause stops the take's recording and starts it again on resume as the next part of the same take (`sets-2.bin`, and so on). takes.py reads all parts in order. A pause while a step waits only shows "Paused"; a Next pressed while paused doesn't count.
|
||||
|
||||
## Bounties
|
||||
|
||||
Short rounds (about 3 min recorded, 4 with reading) of what the hand tracker needs most, so people can help where it counts without a 15-minute session. As in the eye recorder (gaze/rec/DESIGN.md, "Bounty board"), using the shared `recorder/bounties.py`:
|
||||
|
||||
- `bounties.json` is the board: `{"schema": 1, "updated", "bounties": [{"id", "title", "blurb", "minutes", "priority", "script"}]}`. A newer copy fetched from the dataset (`hub.py board`, the window's "Get the newest board", into `BASE/board-cache.json`) can only describe, reorder, reprioritise or retire bounties: each one's steps come from `bounties/<script>.json`, shipped with the recorder. A bounty whose script this recorder lacks shows "Needs a newer recorder".
|
||||
- A round script is a normal script (above) with `"record_hz": 30`: hand size first (the no-hands check), then one scene section. At 30 sets a second a round is about 2.5 min recorded, about 2.1 MB a second exported.
|
||||
- The window opens on the Bounties page: the board, least done first, each with Record (Record again once done), "Get the newest board", and below them the full session and the quick round. Picking a round goes to the consent page if this contributor hasn't agreed to the current version, else straight to the checklist; the checklist's Round section shows the pick (or the full/quick choice) with "Pick another…" back to the board, and hides what a bounty doesn't use (objects within reach, controllers, guidance). Its camera check and Start then run the script (`session.py --bounty ID`; `--board` lists them). When the round is done, the Session page's "Export and upload" exports it and opens the Upload page for that export once the export finishes (`exportDone`). session.json gets `"bounty"`, the upload's title and description name it, and Review shows its title by the session. A round that ends `done` (not a dry run) counts in profile.json's `bounties`.
|
||||
|
||||
The first four (2026-10-14), each on the gestures Frametop uses:
|
||||
|
||||
| Bounty | What | Steps |
|
||||
|---|---|---|
|
||||
| `index-taps` | Thumb to index taps (click), the hand raised and pointing at the button as with Frametop's hand ray (user, 2026-10-14: not gaze-and-tap from the lap) | Each hand, palm away, the arm straight then bent (a switch); two windows left and right, the head ahead then turning; high and low; a big far window; both hands (L/R); double taps and holds, each hand; near misses (orange: stop short); own pace, then the other hand |
|
||||
| `middle-taps` | Thumb to middle taps (right click), pointing the same way | The index round's shape with middle taps and middle holds, index and middle mixed on each hand, near misses and a slow fist |
|
||||
| `pinches` | Thumb and index pinches | Pinch and hold the bar or a corner; drag to ghosts with each hand (across, nearer, farther); resize from a corner; tilt; two hands on two corners; hold a pinch on one window while tapping another; a far window pinched with the hand low; pinch vs stop short vs open touch; own pace |
|
||||
| `card-grabs` | Thumb, index and middle (a playing card) | Hold the right edge (R) and left (L); move and turn it while held; place it on ghosts; push it away; throw it to far ghosts; two hands on both edges; four windows low, high, close and to the side; card vs fist vs touch vs two-finger pinch vs stop short; own pace |
|
||||
|
||||
## Testing sessions and the test phase
|
||||
|
||||
**The test phase** (`takes.TEST_PHASE`, ended 2026-10-14). Since then bounty rounds are recorded under CONSENT.md and go to the public dataset, for training, like the full session and the quick round; only testing (debug) sessions are recorded under `TESTING.md` and go to the test dataset, never for training. While the phase was on, as the bounties were tried out with a few people, their rounds were recorded under a short testing-only consent, `TESTING.md`, instead of CONSENT.md, and went to a separate gated dataset, `DeeJanuz/frametop-hands-test` (`hub.HF_TEST_DATASET`; `FT_HANDREC_TEST_DATASET` overrides it). Nothing in it is used for training or published; moving a recording into the public dataset needs the person's agreement to CONSENT.md. The full session and the quick round stay public. `takes.round_consent_kind(bounty, debug)` says which kind a round uses; the window shows that kind's text on the Welcome page when a round is picked (`setConsentKind`), and gates and labels uploads per session (`uploadAllowedFor`, `datasetFor`). Each kind's text has its own draft gate: TESTING.md's first line says DRAFT until the maintainer approves it.
|
||||
|
||||
**Encryption.** A test-phase export is encrypted on the headset to the maintainer's keys in `hands/rec/test-recipients.txt` (age X25519, `recorder/age.py`, as the eye recordings), so people who can see the test dataset can't open each other's recordings. `sets.bin.zst.age` and `live.bin.zst.age` replace the plain files (zstd piped into age while compressing). The manifest says `"consent_kind": "testing"` and `"encryption": {"scheme": "age-v1-x25519", "recipients": [...], "files": [...]}`; what stays readable is the manifest: lengths, lighting, checklist, contributor id. Key 1 is on the maintainer's machine, key 2 offline; both are the only recipients `validate.py` accepts. The manifest's profile holds only the schema, the contributor, that kind's consent record and the optional answers.
|
||||
|
||||
**Testing sessions** (`debug.json`, `session.py --debug`, the board's first row "Testing session" when Frametop's hand tracking is installed: `frametop-hands.service` is known to systemd, or `FT_HANDREC_FRAMETOP_HANDS=1`). The person uses Frametop as usual for up to 3 minutes at 30 sets a second while the cameras record, so new versions of the tracker can be replayed on that person's own recording and compared with what the live one did. Frametop's hands and pointer keep running (nothing is paused), and the preflight wants `frametop-hands.service` active.
|
||||
|
||||
- One free step: the panel hides while it records. The headset button's press (the window's Mark, M) is a `mark` event, the panel flashing "Marked" for a second; its hold (Finish, F) ends the step early with an `end_early` event, and the session is `done`. Pause and Stop work as usual; Redo and Skip aren't offered.
|
||||
- **`live.bin`** (`livelog.py`): every new publish of the live tracker's hands and gestures files (include/fh_hands.h, fh_gestures.h), copied whole under their sequence lock, one file per part. A 16-byte header (`"FHLIVE01"`, u32 version 1, u32 0), then records: u8 source (0 hands, 1 gestures), 3 pad, u32 length, u64 read_ns (CLOCK_MONOTONIC), u64 seq, then the bytes. The files' own `capture_ns` and `publish_ns` are inside. take.json gets `"live": {"hands": n, "gestures": n}`. The tracker publishes only while the headset is in use.
|
||||
- **`voice.bin`** (`voicelog.py`), only when the checklist's "Record my voice" is ticked (`"voice": true`; never in a dry run): the default microphone through `pw-record --raw` at 16 kHz mono s16, one file per part, so the person can say what went wrong as it happens. A 16-byte header (`"FHVOIC01"`, u32 version 1, u32 sample rate), then records: u64 read_ns (CLOCK_MONOTONIC, when the chunk was read: its last sample is about then), u32 length, then s16le samples, about 100 ms each. take.json gets `"voice": {"voice.bin": {"seconds", "chunks", "peak", "error"}}`; a peak of 0 is a muted microphone (SteamVR mutes it with the headset off). A microphone that fails is logged and the session goes on. `voicelog.py VOICE.BIN OUT.WAV` makes a WAV with gaps filled and prints its first sample's time, for whisper.cpp on the maintainer's machines: its timestamps plus that time give the moments to flag.
|
||||
- session.json `debug`: `{"use": "testing", "max_s", "settings": {HANDS_* and POINTER_* from ~/.config/frametop.conf, paths cut to file names}, "frametop": "<git describe of the running ft-hands' checkout>", "models": {"<file>": "<sha256[:16]>"}, "models_dir"}`.
|
||||
- **`events.jsonl`** (`eventlog.py`): what Frametop did with the hands while the part recorded, from ft-screens and the pointer helper (`hands/include/fh_events.h`). While a part records the recorder keeps `events-want` in the run directory; the two append one JSON line per decision to `events.jsonl` there (`t` CLOCK_MONOTONIC ns, `src`, `kind` such as poke, pinch, grab, lazy-susan, click, drag, `outcome` done / refused / missed / passed / lost / end / info, `hand`, `panel`, `text` the log line), and at the part's end the recorder keeps the part's lines and removes the file. take.json gets `"events": {"events.jsonl": {"events", "by": {kind: {outcome: n}}}}`; `eventlog.py EVENTS.JSONL` prints those counts. Targets are overlay names (`frametop.screen.1`), never what a window shows.
|
||||
- Export joins the parts into `live.bin.zst.age`, leaving out records read in deleted ranges; the manifest's take entry gets `"live": {"records", "raw_bytes", "file"}`. `validate.py` allows a live file only in a testing session, and requires `debug.use` "testing" and the testing consent. Voice parts join into `voice.bin.zst.age` the same way (a chunk touching a deleted range left out; `"voice": {"chunks", "seconds", "raw_bytes", "file"}`), and only ever encrypted: an export without recipients leaves voice out. Events join into `events.jsonl.zst.age` the same way (`"events": {"events", "by", "raw_bytes", "file"}`), also only encrypted.
|
||||
|
||||
## Review and export (`takes.py`, the window)
|
||||
|
||||
- **Review.** Each session lists its takes: title, duration, status, a thumbnail (the first set's `slam_left`). A viewer shows one frame set (all cameras side by side, 8-bit grey) at a time with a slider. It can mark a range and delete it. Deleted ranges go into `take.json`, and export leaves them out; the files keep everything until export. A whole take or session can be deleted (files removed, after a confirmation).
|
||||
@@ -396,3 +487,4 @@ On 2026-10-02 a whole session showed "I can't see your hands": after the headset
|
||||
- The text explains what's recorded and that nothing uploads automatically, how review works, how to withdraw (by contributor id), and that a withdrawal is purged from the repo's history.
|
||||
- The texts need a legal review before the dataset launches. Until then they carry a "draft" banner, and the Upload page says contributions aren't open yet.
|
||||
- The dataset repo: `HF_DATASET` in `hub.py`, `DeeJanuz/frametop-hands` (private until launch).
|
||||
- Testing sessions have their own short text, `TESTING.md`: testing only, no training, nothing published, encrypted, deleted when no longer needed for testing or on request (above, "Testing sessions and the test phase").
|
||||
@@ -0,0 +1,52 @@
|
||||
# Testing sessions
|
||||
|
||||
Version: 2026-10-14d
|
||||
|
||||
A testing session records you using Frametop as you always do, while you press the headset button whenever the hand tracking gets something wrong. New versions of the hand tracking are then tested on your recording before they ship. This page says what you agree to by recording one. It's shorter than the hand dataset's consent because **nothing you send from a testing session is published or used for training**.
|
||||
|
||||
This page covers testing sessions only. Bounty rounds are different: they go to the public hand dataset and are used for training, under that dataset's own consent, which the Hand Recorder shows before your first bounty.
|
||||
|
||||
## Who runs this
|
||||
|
||||
Frametop is an independent open-source project, maintained by DeeJanuz (<https://github.com/DeeJanuz>). It isn't affiliated with or endorsed by Valve or Hugging Face. This text was written without a lawyer, in good faith. You can reach the maintainer through the Frametop repository's issues (<https://github.com/Frametop/frametop/issues>), which are public.
|
||||
|
||||
## Who can take part
|
||||
|
||||
- You must be **18 or older**.
|
||||
- For now, you can't take part if you live in **Illinois, Texas or Washington** (USA): their laws about biometric data (such as hand geometry) need more than this project can provide yet.
|
||||
|
||||
## What is recorded
|
||||
|
||||
- **Camera images** from the headset's 4 tracking cameras, 30 sets a second: your hands and arms, your clothing and whatever is in front of you. The size and shape of your hands can be measured from them.
|
||||
- **Motion** of the headset and, if you use them, the controllers; **what you were asked to do** and when.
|
||||
- **Calibration** of the cameras, with serial numbers and other fields that identify your headset removed.
|
||||
- What Frametop's hand tracking made of your hands, moment by moment, what Frametop did with them (each poke, pinch, grab or stroke it acted on or held back, and which of its windows, by number, not by what was in them), the moments you marked, and your hand tracking settings (file paths left out).
|
||||
- **Your voice**, only if you tick "Record my voice": the headset's microphone while you record, so you can say what went wrong. Anyone else who can be heard is recorded too. The maintainer turns it into text on their own computers (not an online service) and uses it to find those moments.
|
||||
- **Your answers** in the checklist, the **times** of the recordings, and a **contributor id**: a random number made on your headset, not from your name, Steam account or headset.
|
||||
|
||||
No other sound, no name, no email address, no Steam account. The cameras look outward, so your face isn't recorded unless a mirror shows it.
|
||||
|
||||
## Where it goes and who sees it
|
||||
|
||||
- Nothing leaves your headset until you upload it. Before that you can watch every recording in Review and delete any part of it.
|
||||
- Uploads go to a gated test dataset on Hugging Face (<https://huggingface.co/datasets/DeeJanuz/frametop-hands-test>). Its servers may be outside your country, for example in the USA.
|
||||
- **The recordings are encrypted on your headset before they're sent.** Only the maintainer holds the keys, so other people who can see the test dataset can't open them. What stays readable is the description of the upload: its length, lighting, your contributor id and checklist answers.
|
||||
- Your pull request shows your Hugging Face username next to your contributor id, to anyone who can see the test dataset.
|
||||
|
||||
## What it's used for
|
||||
|
||||
- **Testing** Frametop's hand tracking: new versions are run on your recordings to see whether they do better, especially at the moments you marked. Also checking that the recorder works.
|
||||
- **Never for training** models, and **not published**. Testing sessions stay out of the public hand dataset.
|
||||
|
||||
## Withdrawing
|
||||
|
||||
- Your uploads are deleted when they're no longer needed for testing, or earlier if you ask: close your pull request, or open an issue with your contributor id (shown in the Hand Recorder) and "delete my test uploads".
|
||||
- You can delete recordings on your headset at any time in Review.
|
||||
|
||||
## Keep other people and private things out of view
|
||||
|
||||
Face away from other people, mirrors and screens or papers you wouldn't want seen; don't record anyone else, and don't record children. Clear an arm's reach around you, and stop whenever anything is uncomfortable. You take part at your own risk.
|
||||
|
||||
## Agreeing
|
||||
|
||||
By ticking the three boxes and "Agree and continue", you confirm that you're 18 or older, that you don't live in Illinois, Texas or Washington, and that you agree to the above. You can still decide not to upload anything.
|
||||
@@ -0,0 +1,7 @@
|
||||
{"schema": 1, "updated": "2026-10-14",
|
||||
"bounties": [
|
||||
{"id": "index-taps", "title": "Index taps", "blurb": "Thumb to index taps with the hand raised and pointing at lit buttons, as with the hand ray: arm out and bent, two windows, high and low, far away, both hands, near misses and holds.", "minutes": 4, "priority": "high", "script": "index-taps"},
|
||||
{"id": "middle-taps", "title": "Middle taps", "blurb": "Thumb to middle finger taps (right clicks) with the hand raised and pointing, mixed with index taps so the two can be told apart.", "minutes": 4, "priority": "high", "script": "middle-taps"},
|
||||
{"id": "pinches", "title": "Pinches", "blurb": "Pinch a window's bar or corner and drag, resize, tilt and stretch it to a ghost outline.", "minutes": 4, "priority": "high", "script": "pinches"},
|
||||
{"id": "card-grabs", "title": "Three-finger grabs", "blurb": "Grab a window's edge like a playing card: hold it, place it, push it away, throw it.", "minutes": 4, "priority": "high", "script": "card-grabs"}
|
||||
]}
|
||||
@@ -0,0 +1,558 @@
|
||||
{
|
||||
"version": 1,
|
||||
"name": "card-grabs",
|
||||
"record_hz": 30,
|
||||
"intro_s": 4,
|
||||
"between_s": 3,
|
||||
"welcome": {
|
||||
"title": "Three-finger grabs",
|
||||
"seconds": 8,
|
||||
"text": "Windows will appear in front of you. Do what each one asks, as you would on a real one: they don't move or click, the cameras record your hands.|Press the button on the right side of the headset to start each step. Pressed while recording, it pauses."
|
||||
},
|
||||
"done": {
|
||||
"title": "All done",
|
||||
"seconds": 6,
|
||||
"text": "Thank you! You can take off the headset.|Review your recordings in the Hand recorder window."
|
||||
},
|
||||
"stopped": {
|
||||
"title": "Stopped",
|
||||
"seconds": 4,
|
||||
"text": "What was recorded so far is saved. Review it in the Hand recorder window."
|
||||
},
|
||||
"sections": [
|
||||
{
|
||||
"id": "hand-size",
|
||||
"title": "Hand size",
|
||||
"requires": [],
|
||||
"go": "Hold",
|
||||
"quick": true,
|
||||
"intro": "First, your hand size: both hands up about 40 cm out, still.",
|
||||
"prompts": [
|
||||
{
|
||||
"text": "Both hands about 40 cm out, still|Palms toward you",
|
||||
"seconds": 6,
|
||||
"hands": "both",
|
||||
"distance": "mid",
|
||||
"switch": [
|
||||
{
|
||||
"at": 0.5,
|
||||
"to": "backs toward you",
|
||||
"text": "Both hands about 40 cm out, still|Backs toward you"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "card-grabs",
|
||||
"title": "Three-finger grabs",
|
||||
"kind": "scene",
|
||||
"intro": "Grab the window's lit edge like a playing card: thumb on one side, index and middle on the other. It won't move: move your hand as if it did, then let go.",
|
||||
"prompts": [
|
||||
{
|
||||
"id": "hold",
|
||||
"hands": "both",
|
||||
"text": "Grab the lit edge like a card and hold until it goes out: the right edge with your right hand, the left edge with your left.",
|
||||
"windows": [
|
||||
{
|
||||
"id": "a",
|
||||
"at": [
|
||||
0,
|
||||
-0.2,
|
||||
-0.45
|
||||
],
|
||||
"size": [
|
||||
0.34,
|
||||
0.24
|
||||
],
|
||||
"look": "plain"
|
||||
}
|
||||
],
|
||||
"acts": {
|
||||
"gen": "moves",
|
||||
"gesture": "card",
|
||||
"every_s": 2.5,
|
||||
"gap_s": 1.0,
|
||||
"shuffle": false,
|
||||
"moves": [
|
||||
{
|
||||
"grab": "right",
|
||||
"hand": "right",
|
||||
"label": "R"
|
||||
},
|
||||
{
|
||||
"grab": "left",
|
||||
"hand": "left",
|
||||
"label": "L"
|
||||
},
|
||||
{
|
||||
"grab": "right",
|
||||
"hand": "right",
|
||||
"label": "R"
|
||||
},
|
||||
{
|
||||
"grab": "left",
|
||||
"hand": "left",
|
||||
"label": "L"
|
||||
}
|
||||
],
|
||||
"say": "Grab the lit edge like a card and hold"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "mime",
|
||||
"hands": "right",
|
||||
"text": "Right hand. Grab the right edge and move it up and down, then turn it around, as if it came with your hand.",
|
||||
"windows": [
|
||||
{
|
||||
"id": "a",
|
||||
"at": [
|
||||
0.05,
|
||||
-0.2,
|
||||
-0.45
|
||||
],
|
||||
"size": [
|
||||
0.34,
|
||||
0.24
|
||||
],
|
||||
"look": "plain"
|
||||
}
|
||||
],
|
||||
"acts": [
|
||||
{
|
||||
"s": 8,
|
||||
"lit": "right",
|
||||
"gesture": "card",
|
||||
"say": "Grab it: move it up and down, turn it around"
|
||||
},
|
||||
{
|
||||
"s": 1,
|
||||
"gesture": "rest",
|
||||
"say": "Let go"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "place",
|
||||
"hands": "right",
|
||||
"text": "Right hand. Grab the right edge and take the window to the ghost, then let go.",
|
||||
"windows": [
|
||||
{
|
||||
"id": "a",
|
||||
"at": [
|
||||
0.05,
|
||||
-0.2,
|
||||
-0.45
|
||||
],
|
||||
"size": [
|
||||
0.34,
|
||||
0.24
|
||||
],
|
||||
"look": "plain"
|
||||
}
|
||||
],
|
||||
"acts": {
|
||||
"gen": "moves",
|
||||
"gesture": "card",
|
||||
"grab": "right",
|
||||
"every_s": 3.5,
|
||||
"gap_s": 1.0,
|
||||
"count": 3,
|
||||
"moves": [
|
||||
[
|
||||
0.2,
|
||||
0.1,
|
||||
0
|
||||
],
|
||||
[
|
||||
-0.25,
|
||||
0,
|
||||
0.05
|
||||
],
|
||||
[
|
||||
0,
|
||||
-0.15,
|
||||
0.1
|
||||
],
|
||||
[
|
||||
0.1,
|
||||
0.05,
|
||||
-0.2
|
||||
]
|
||||
],
|
||||
"say": "Grab it, take it to the ghost"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "push",
|
||||
"hands": "both",
|
||||
"text": "Grab the lit edge, take the window to the ghost, then push it away from you and let go.",
|
||||
"windows": [
|
||||
{
|
||||
"id": "a",
|
||||
"at": [
|
||||
0,
|
||||
-0.2,
|
||||
-0.45
|
||||
],
|
||||
"size": [
|
||||
0.34,
|
||||
0.24
|
||||
],
|
||||
"look": "plain"
|
||||
}
|
||||
],
|
||||
"acts": [
|
||||
{
|
||||
"s": 3,
|
||||
"lit": "right",
|
||||
"gesture": "card",
|
||||
"ghost": [
|
||||
0.12,
|
||||
0.05,
|
||||
0
|
||||
],
|
||||
"say": "Right hand: grab it, take it to the ghost",
|
||||
"hand": "right",
|
||||
"label": "R"
|
||||
},
|
||||
{
|
||||
"s": 2.5,
|
||||
"lit": "right",
|
||||
"gesture": "card",
|
||||
"ghost": [
|
||||
0.15,
|
||||
0.1,
|
||||
-0.8
|
||||
],
|
||||
"say": "Now push it away and let go",
|
||||
"hand": "right",
|
||||
"label": "R"
|
||||
},
|
||||
{
|
||||
"s": 1,
|
||||
"gesture": "rest",
|
||||
"say": "Let go"
|
||||
},
|
||||
{
|
||||
"s": 3,
|
||||
"lit": "left",
|
||||
"gesture": "card",
|
||||
"ghost": [
|
||||
-0.12,
|
||||
0.05,
|
||||
0
|
||||
],
|
||||
"say": "Left hand: grab it, take it to the ghost",
|
||||
"hand": "left",
|
||||
"label": "L"
|
||||
},
|
||||
{
|
||||
"s": 2.5,
|
||||
"lit": "left",
|
||||
"gesture": "card",
|
||||
"ghost": [
|
||||
-0.15,
|
||||
0.1,
|
||||
-0.8
|
||||
],
|
||||
"say": "Now push it away and let go",
|
||||
"hand": "left",
|
||||
"label": "L"
|
||||
},
|
||||
{
|
||||
"s": 1,
|
||||
"gesture": "rest",
|
||||
"say": "Let go"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "flick",
|
||||
"hands": "right",
|
||||
"text": "Right hand. Grab the edge and flick the window away toward the ghost, like throwing it into space.",
|
||||
"windows": [
|
||||
{
|
||||
"id": "a",
|
||||
"at": [
|
||||
0.05,
|
||||
-0.2,
|
||||
-0.45
|
||||
],
|
||||
"size": [
|
||||
0.34,
|
||||
0.24
|
||||
],
|
||||
"look": "plain"
|
||||
}
|
||||
],
|
||||
"acts": {
|
||||
"gen": "moves",
|
||||
"gesture": "card",
|
||||
"grab": "right",
|
||||
"every_s": 2.5,
|
||||
"gap_s": 1.2,
|
||||
"count": 3,
|
||||
"moves": [
|
||||
[
|
||||
0.9,
|
||||
0.3,
|
||||
-1.5,
|
||||
30,
|
||||
0,
|
||||
0
|
||||
],
|
||||
[
|
||||
-0.9,
|
||||
0.2,
|
||||
-1.4,
|
||||
-30,
|
||||
0,
|
||||
0
|
||||
],
|
||||
[
|
||||
0,
|
||||
0.6,
|
||||
-2.0
|
||||
],
|
||||
[
|
||||
0.4,
|
||||
-0.3,
|
||||
-1.2,
|
||||
20,
|
||||
0,
|
||||
0
|
||||
]
|
||||
],
|
||||
"say": "Grab it, throw it toward the ghost"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "two-hands",
|
||||
"hands": "both",
|
||||
"text": "Both hands, one on each lit edge: turn the window like the ghost, or stretch it.",
|
||||
"windows": [
|
||||
{
|
||||
"id": "a",
|
||||
"at": [
|
||||
0,
|
||||
-0.2,
|
||||
-0.45
|
||||
],
|
||||
"size": [
|
||||
0.34,
|
||||
0.24
|
||||
],
|
||||
"look": "plain"
|
||||
}
|
||||
],
|
||||
"acts": {
|
||||
"gen": "moves",
|
||||
"gesture": "card",
|
||||
"grab": "left+right",
|
||||
"every_s": 3.5,
|
||||
"gap_s": 1.0,
|
||||
"moves": [
|
||||
[
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
35,
|
||||
0,
|
||||
0
|
||||
],
|
||||
[
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
20
|
||||
],
|
||||
[
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
1.4
|
||||
],
|
||||
[
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0.75
|
||||
]
|
||||
],
|
||||
"count": 3,
|
||||
"say": "Both edges: turn or stretch it like the ghost"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "places",
|
||||
"hands": "right",
|
||||
"text": "Four windows: low, high, close to your face and out to the right. Grab whichever edge lights up, with your right hand.",
|
||||
"windows": [
|
||||
{
|
||||
"id": "low",
|
||||
"at": [
|
||||
0,
|
||||
-0.45,
|
||||
-0.35
|
||||
],
|
||||
"size": [
|
||||
0.28,
|
||||
0.2
|
||||
],
|
||||
"look": "plain"
|
||||
},
|
||||
{
|
||||
"id": "high",
|
||||
"at": [
|
||||
0.05,
|
||||
0.2,
|
||||
-0.45
|
||||
],
|
||||
"size": [
|
||||
0.28,
|
||||
0.2
|
||||
],
|
||||
"look": "plain"
|
||||
},
|
||||
{
|
||||
"id": "face",
|
||||
"at": [
|
||||
0.1,
|
||||
-0.02,
|
||||
-0.25
|
||||
],
|
||||
"size": [
|
||||
0.18,
|
||||
0.13
|
||||
],
|
||||
"look": "plain"
|
||||
},
|
||||
{
|
||||
"id": "side",
|
||||
"at": [
|
||||
0.55,
|
||||
-0.1,
|
||||
-0.35
|
||||
],
|
||||
"size": [
|
||||
0.28,
|
||||
0.2
|
||||
],
|
||||
"look": "plain"
|
||||
}
|
||||
],
|
||||
"acts": {
|
||||
"gen": "moves",
|
||||
"gesture": "card",
|
||||
"every_s": 2.5,
|
||||
"gap_s": 0.8,
|
||||
"moves": [
|
||||
{
|
||||
"win": "low",
|
||||
"grab": "right"
|
||||
},
|
||||
{
|
||||
"win": "high",
|
||||
"grab": "right"
|
||||
},
|
||||
{
|
||||
"win": "face",
|
||||
"grab": "right"
|
||||
},
|
||||
{
|
||||
"win": "side",
|
||||
"grab": "left"
|
||||
}
|
||||
],
|
||||
"say": "Grab the lit edge like a card"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "look-alikes",
|
||||
"hands": "right",
|
||||
"text": "Right hand, on the lit edge. Each says how: a card grab, a fist, an open-hand touch, a two-finger pinch, or orange: stop short.",
|
||||
"windows": [
|
||||
{
|
||||
"id": "a",
|
||||
"at": [
|
||||
0.05,
|
||||
-0.2,
|
||||
-0.45
|
||||
],
|
||||
"size": [
|
||||
0.34,
|
||||
0.24
|
||||
],
|
||||
"look": "plain"
|
||||
}
|
||||
],
|
||||
"acts": {
|
||||
"gen": "moves",
|
||||
"every_s": 2.5,
|
||||
"gap_s": 0.8,
|
||||
"moves": [
|
||||
{
|
||||
"grab": "right",
|
||||
"gesture": "card"
|
||||
},
|
||||
{
|
||||
"grab": "right",
|
||||
"gesture": "fist"
|
||||
},
|
||||
{
|
||||
"grab": "right",
|
||||
"gesture": "touch"
|
||||
},
|
||||
{
|
||||
"grab": "right",
|
||||
"gesture": "pinch",
|
||||
"say": "Pinch it with two fingers: thumb and index only"
|
||||
},
|
||||
{
|
||||
"grab": "left",
|
||||
"gesture": "nogo"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "free",
|
||||
"hands": "any",
|
||||
"text": "Your own pace, either hand or both: grab it like a card, move it, turn it, throw it, any way you like.",
|
||||
"windows": [
|
||||
{
|
||||
"id": "a",
|
||||
"at": [
|
||||
0,
|
||||
-0.2,
|
||||
-0.45
|
||||
],
|
||||
"size": [
|
||||
0.34,
|
||||
0.24
|
||||
],
|
||||
"look": "plain"
|
||||
}
|
||||
],
|
||||
"acts": [
|
||||
{
|
||||
"s": 14,
|
||||
"lit": "left+right",
|
||||
"gesture": "card",
|
||||
"say": "Your own pace: grab it, move it, turn it, throw it"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,399 @@
|
||||
{
|
||||
"version": 1,
|
||||
"name": "index-taps",
|
||||
"record_hz": 30,
|
||||
"intro_s": 4,
|
||||
"between_s": 3,
|
||||
"welcome": {
|
||||
"title": "Index taps",
|
||||
"seconds": 8,
|
||||
"text": "Windows will appear in front of you. Do what each one asks, as you would on a real one: they don't move or click, the cameras record your hands.|Press the button on the right side of the headset to start each step. Pressed while recording, it pauses."
|
||||
},
|
||||
"done": {
|
||||
"title": "All done",
|
||||
"seconds": 6,
|
||||
"text": "Thank you! You can take off the headset.|Review your recordings in the Hand recorder window."
|
||||
},
|
||||
"stopped": {
|
||||
"title": "Stopped",
|
||||
"seconds": 4,
|
||||
"text": "What was recorded so far is saved. Review it in the Hand recorder window."
|
||||
},
|
||||
"sections": [
|
||||
{
|
||||
"id": "hand-size",
|
||||
"title": "Hand size",
|
||||
"requires": [],
|
||||
"go": "Hold",
|
||||
"quick": true,
|
||||
"intro": "First, your hand size: both hands up about 40 cm out, still.",
|
||||
"prompts": [
|
||||
{
|
||||
"text": "Both hands about 40 cm out, still|Palms toward you",
|
||||
"seconds": 6,
|
||||
"hands": "both",
|
||||
"distance": "mid",
|
||||
"switch": [
|
||||
{
|
||||
"at": 0.5,
|
||||
"to": "backs toward you",
|
||||
"text": "Both hands about 40 cm out, still|Backs toward you"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "index-taps",
|
||||
"title": "Index taps",
|
||||
"kind": "scene",
|
||||
"intro": "Hold your hand up in front of you, palm away, and point it at the lit button, as if a laser came out of your knuckles (Frametop's hand ray). Then tap: thumb to index fingertip, light and quick, the way you'd click. Nothing happens when you tap: the cameras record your hand.",
|
||||
"prompts": [
|
||||
{
|
||||
"id": "aim-right",
|
||||
"hands": "right",
|
||||
"text": "Right hand up in front of you, palm away, the left one down and out of view.|Point the hand at each button as it lights and tap. Start with the arm nearly straight.",
|
||||
"windows": [
|
||||
{
|
||||
"id": "a",
|
||||
"at": [
|
||||
0,
|
||||
-0.05,
|
||||
-1.1
|
||||
],
|
||||
"size": [
|
||||
0.7,
|
||||
0.45
|
||||
],
|
||||
"look": "grid:3x3"
|
||||
}
|
||||
],
|
||||
"acts": {
|
||||
"gen": "buttons",
|
||||
"every_s": 1.4,
|
||||
"seconds": 12,
|
||||
"mix": {
|
||||
"tap": 1
|
||||
}
|
||||
},
|
||||
"switch": [
|
||||
{
|
||||
"at": 0.5,
|
||||
"to": "elbow bent, hand near your chest"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "aim-left",
|
||||
"hands": "left",
|
||||
"text": "Left hand up in front of you, palm away, the right one down and out of view.|Point it at each lit button and tap. Start with the elbow bent, the hand near your chest.",
|
||||
"windows": [
|
||||
{
|
||||
"id": "a",
|
||||
"at": [
|
||||
0,
|
||||
-0.05,
|
||||
-1.1
|
||||
],
|
||||
"size": [
|
||||
0.7,
|
||||
0.45
|
||||
],
|
||||
"look": "grid:3x3"
|
||||
}
|
||||
],
|
||||
"acts": {
|
||||
"gen": "buttons",
|
||||
"every_s": 1.4,
|
||||
"seconds": 12,
|
||||
"mix": {
|
||||
"tap": 1
|
||||
}
|
||||
},
|
||||
"switch": [
|
||||
{
|
||||
"at": 0.5,
|
||||
"to": "arm nearly straight"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "two-windows",
|
||||
"hands": "right",
|
||||
"text": "Two windows, left and right: the lit button jumps between them.|Point your right hand at it and tap, keeping your head facing ahead.",
|
||||
"windows": [
|
||||
{
|
||||
"id": "a",
|
||||
"at": [
|
||||
-0.55,
|
||||
-0.05,
|
||||
-1.0
|
||||
],
|
||||
"size": [
|
||||
0.45,
|
||||
0.3
|
||||
],
|
||||
"look": "grid:2x2"
|
||||
},
|
||||
{
|
||||
"id": "b",
|
||||
"at": [
|
||||
0.55,
|
||||
-0.05,
|
||||
-1.0
|
||||
],
|
||||
"size": [
|
||||
0.45,
|
||||
0.3
|
||||
],
|
||||
"look": "grid:2x2"
|
||||
}
|
||||
],
|
||||
"acts": {
|
||||
"gen": "buttons",
|
||||
"every_s": 1.5,
|
||||
"seconds": 12,
|
||||
"mix": {
|
||||
"tap": 1
|
||||
}
|
||||
},
|
||||
"switch": [
|
||||
{
|
||||
"at": 0.5,
|
||||
"to": "turning your head to each one too"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "high-low",
|
||||
"hands": "left",
|
||||
"text": "One window high, one low. Point your left hand at the lit button and tap,|the hand going up and down with it.",
|
||||
"windows": [
|
||||
{
|
||||
"id": "a",
|
||||
"at": [
|
||||
0,
|
||||
0.3,
|
||||
-1.1
|
||||
],
|
||||
"size": [
|
||||
0.6,
|
||||
0.25
|
||||
],
|
||||
"look": "grid:3x1"
|
||||
},
|
||||
{
|
||||
"id": "b",
|
||||
"at": [
|
||||
0,
|
||||
-0.4,
|
||||
-0.9
|
||||
],
|
||||
"size": [
|
||||
0.6,
|
||||
0.25
|
||||
],
|
||||
"look": "grid:3x1"
|
||||
}
|
||||
],
|
||||
"acts": {
|
||||
"gen": "buttons",
|
||||
"every_s": 1.5,
|
||||
"seconds": 12,
|
||||
"mix": {
|
||||
"tap": 1
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "far",
|
||||
"hands": "right",
|
||||
"text": "A big window far away: small moves of the hand aim across it.|Right hand up, palm away: point and tap.",
|
||||
"windows": [
|
||||
{
|
||||
"id": "a",
|
||||
"at": [
|
||||
0.2,
|
||||
0.1,
|
||||
-3.0
|
||||
],
|
||||
"size": [
|
||||
1.6,
|
||||
1.0
|
||||
],
|
||||
"look": "grid:3x3"
|
||||
}
|
||||
],
|
||||
"acts": {
|
||||
"gen": "buttons",
|
||||
"every_s": 1.25,
|
||||
"seconds": 10,
|
||||
"mix": {
|
||||
"tap": 1
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "both-hands",
|
||||
"hands": "both",
|
||||
"text": "Both hands up, palms away. Each lit button says L or R:|point that hand at it and tap.",
|
||||
"windows": [
|
||||
{
|
||||
"id": "a",
|
||||
"at": [
|
||||
0,
|
||||
-0.05,
|
||||
-1.1
|
||||
],
|
||||
"size": [
|
||||
0.7,
|
||||
0.45
|
||||
],
|
||||
"look": "grid:3x3"
|
||||
}
|
||||
],
|
||||
"acts": {
|
||||
"gen": "buttons",
|
||||
"every_s": 2.4,
|
||||
"seconds": 16,
|
||||
"mix": {
|
||||
"tap": 1
|
||||
},
|
||||
"hands": [
|
||||
"left",
|
||||
"right"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "presses-right",
|
||||
"hands": "right",
|
||||
"text": "Right hand pointing. A button showing 2: a double tap. A button showing hold: press and keep it until it goes out, as if dragging.",
|
||||
"windows": [
|
||||
{
|
||||
"id": "a",
|
||||
"at": [
|
||||
0,
|
||||
-0.05,
|
||||
-1.1
|
||||
],
|
||||
"size": [
|
||||
0.7,
|
||||
0.45
|
||||
],
|
||||
"look": "grid:3x3"
|
||||
}
|
||||
],
|
||||
"acts": {
|
||||
"gen": "buttons",
|
||||
"every_s": 1.6,
|
||||
"seconds": 12,
|
||||
"mix": {
|
||||
"tap": 2,
|
||||
"double": 1,
|
||||
"hold": 1
|
||||
},
|
||||
"lit_s": 1.2
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "presses-left",
|
||||
"hands": "left",
|
||||
"text": "Left hand pointing. Quick taps, and holds: press until the button goes out.",
|
||||
"windows": [
|
||||
{
|
||||
"id": "a",
|
||||
"at": [
|
||||
-0.3,
|
||||
-0.05,
|
||||
-1.0
|
||||
],
|
||||
"size": [
|
||||
0.6,
|
||||
0.4
|
||||
],
|
||||
"look": "grid:3x3"
|
||||
}
|
||||
],
|
||||
"acts": {
|
||||
"gen": "buttons",
|
||||
"every_s": 1.8,
|
||||
"seconds": 12,
|
||||
"mix": {
|
||||
"tap": 1,
|
||||
"hold": 1
|
||||
},
|
||||
"lit_s": 1.3
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "look-alikes",
|
||||
"hands": "right",
|
||||
"text": "Right hand pointing. Cyan: tap. Orange with a cross: bring the fingers about 1 cm short, don't touch.|Now and then rub the thumb along the side of the index instead.",
|
||||
"windows": [
|
||||
{
|
||||
"id": "a",
|
||||
"at": [
|
||||
0,
|
||||
-0.05,
|
||||
-1.1
|
||||
],
|
||||
"size": [
|
||||
0.7,
|
||||
0.45
|
||||
],
|
||||
"look": "grid:3x3"
|
||||
}
|
||||
],
|
||||
"acts": {
|
||||
"gen": "buttons",
|
||||
"every_s": 1.5,
|
||||
"seconds": 12,
|
||||
"mix": {
|
||||
"tap": 1,
|
||||
"nogo": 1
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "free",
|
||||
"hands": "any",
|
||||
"text": "Your own pace: point at the lit button and tap as often as you like.|Move the arm while you do: straight out, bent, out to the side, higher and lower.",
|
||||
"windows": [
|
||||
{
|
||||
"id": "a",
|
||||
"at": [
|
||||
0,
|
||||
-0.05,
|
||||
-1.1
|
||||
],
|
||||
"size": [
|
||||
0.7,
|
||||
0.45
|
||||
],
|
||||
"look": "grid:3x3"
|
||||
}
|
||||
],
|
||||
"acts": {
|
||||
"gen": "buttons",
|
||||
"every_s": 4,
|
||||
"seconds": 16,
|
||||
"mix": {
|
||||
"tap": 1
|
||||
},
|
||||
"lit_s": 4,
|
||||
"says": {
|
||||
"tap": "Your own pace: tap it as often as you like"
|
||||
}
|
||||
},
|
||||
"switch": [
|
||||
{
|
||||
"at": 0.5,
|
||||
"to": "the other hand"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,426 @@
|
||||
{
|
||||
"version": 1,
|
||||
"name": "middle-taps",
|
||||
"record_hz": 30,
|
||||
"intro_s": 4,
|
||||
"between_s": 3,
|
||||
"welcome": {
|
||||
"title": "Middle taps",
|
||||
"seconds": 8,
|
||||
"text": "Windows will appear in front of you. Do what each one asks, as you would on a real one: they don't move or click, the cameras record your hands.|Press the button on the right side of the headset to start each step. Pressed while recording, it pauses."
|
||||
},
|
||||
"done": {
|
||||
"title": "All done",
|
||||
"seconds": 6,
|
||||
"text": "Thank you! You can take off the headset.|Review your recordings in the Hand recorder window."
|
||||
},
|
||||
"stopped": {
|
||||
"title": "Stopped",
|
||||
"seconds": 4,
|
||||
"text": "What was recorded so far is saved. Review it in the Hand recorder window."
|
||||
},
|
||||
"sections": [
|
||||
{
|
||||
"id": "hand-size",
|
||||
"title": "Hand size",
|
||||
"requires": [],
|
||||
"go": "Hold",
|
||||
"quick": true,
|
||||
"intro": "First, your hand size: both hands up about 40 cm out, still.",
|
||||
"prompts": [
|
||||
{
|
||||
"text": "Both hands about 40 cm out, still|Palms toward you",
|
||||
"seconds": 6,
|
||||
"hands": "both",
|
||||
"distance": "mid",
|
||||
"switch": [
|
||||
{
|
||||
"at": 0.5,
|
||||
"to": "backs toward you",
|
||||
"text": "Both hands about 40 cm out, still|Backs toward you"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "middle-taps",
|
||||
"title": "Middle taps",
|
||||
"kind": "scene",
|
||||
"intro": "Hold your hand up in front of you, palm away, and point it at the lit button, as if a laser came out of your knuckles (Frametop's hand ray). When it lights up purple, tap thumb to middle fingertip, the index out of the way: a right click. Nothing happens when you tap: the cameras record your hand.",
|
||||
"prompts": [
|
||||
{
|
||||
"id": "aim-right",
|
||||
"hands": "right",
|
||||
"text": "Right hand up in front of you, palm away, the left one down and out of view.|Point the hand at each button as it lights and middle tap. Start with the arm nearly straight.",
|
||||
"windows": [
|
||||
{
|
||||
"id": "a",
|
||||
"at": [
|
||||
0,
|
||||
-0.05,
|
||||
-1.1
|
||||
],
|
||||
"size": [
|
||||
0.7,
|
||||
0.45
|
||||
],
|
||||
"look": "grid:3x3"
|
||||
}
|
||||
],
|
||||
"acts": {
|
||||
"gen": "buttons",
|
||||
"every_s": 1.4,
|
||||
"seconds": 12,
|
||||
"mix": {
|
||||
"middle": 1
|
||||
}
|
||||
},
|
||||
"switch": [
|
||||
{
|
||||
"at": 0.5,
|
||||
"to": "elbow bent, hand near your chest"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "aim-left",
|
||||
"hands": "left",
|
||||
"text": "Left hand up in front of you, palm away, the right one down and out of view.|Point it at each lit button and middle tap. Start with the elbow bent, the hand near your chest.",
|
||||
"windows": [
|
||||
{
|
||||
"id": "a",
|
||||
"at": [
|
||||
0,
|
||||
-0.05,
|
||||
-1.1
|
||||
],
|
||||
"size": [
|
||||
0.7,
|
||||
0.45
|
||||
],
|
||||
"look": "grid:3x3"
|
||||
}
|
||||
],
|
||||
"acts": {
|
||||
"gen": "buttons",
|
||||
"every_s": 1.4,
|
||||
"seconds": 12,
|
||||
"mix": {
|
||||
"middle": 1
|
||||
}
|
||||
},
|
||||
"switch": [
|
||||
{
|
||||
"at": 0.5,
|
||||
"to": "arm nearly straight"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "two-windows",
|
||||
"hands": "right",
|
||||
"text": "Two windows, left and right: the lit button jumps between them.|Point your right hand at it and middle tap, keeping your head facing ahead.",
|
||||
"windows": [
|
||||
{
|
||||
"id": "a",
|
||||
"at": [
|
||||
-0.55,
|
||||
-0.05,
|
||||
-1.0
|
||||
],
|
||||
"size": [
|
||||
0.45,
|
||||
0.3
|
||||
],
|
||||
"look": "grid:2x2"
|
||||
},
|
||||
{
|
||||
"id": "b",
|
||||
"at": [
|
||||
0.55,
|
||||
-0.05,
|
||||
-1.0
|
||||
],
|
||||
"size": [
|
||||
0.45,
|
||||
0.3
|
||||
],
|
||||
"look": "grid:2x2"
|
||||
}
|
||||
],
|
||||
"acts": {
|
||||
"gen": "buttons",
|
||||
"every_s": 1.5,
|
||||
"seconds": 12,
|
||||
"mix": {
|
||||
"middle": 1
|
||||
}
|
||||
},
|
||||
"switch": [
|
||||
{
|
||||
"at": 0.5,
|
||||
"to": "turning your head to each one too"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "high-low",
|
||||
"hands": "left",
|
||||
"text": "One window high, one low. Point your left hand at the lit button and middle tap,|the hand going up and down with it.",
|
||||
"windows": [
|
||||
{
|
||||
"id": "a",
|
||||
"at": [
|
||||
0,
|
||||
0.3,
|
||||
-1.1
|
||||
],
|
||||
"size": [
|
||||
0.6,
|
||||
0.25
|
||||
],
|
||||
"look": "grid:3x1"
|
||||
},
|
||||
{
|
||||
"id": "b",
|
||||
"at": [
|
||||
0,
|
||||
-0.4,
|
||||
-0.9
|
||||
],
|
||||
"size": [
|
||||
0.6,
|
||||
0.25
|
||||
],
|
||||
"look": "grid:3x1"
|
||||
}
|
||||
],
|
||||
"acts": {
|
||||
"gen": "buttons",
|
||||
"every_s": 1.5,
|
||||
"seconds": 12,
|
||||
"mix": {
|
||||
"middle": 1
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "far",
|
||||
"hands": "right",
|
||||
"text": "A big window far away: small moves of the hand aim across it.|Right hand up, palm away: point and middle tap.",
|
||||
"windows": [
|
||||
{
|
||||
"id": "a",
|
||||
"at": [
|
||||
0.2,
|
||||
0.1,
|
||||
-3.0
|
||||
],
|
||||
"size": [
|
||||
1.6,
|
||||
1.0
|
||||
],
|
||||
"look": "grid:3x3"
|
||||
}
|
||||
],
|
||||
"acts": {
|
||||
"gen": "buttons",
|
||||
"every_s": 1.25,
|
||||
"seconds": 10,
|
||||
"mix": {
|
||||
"middle": 1
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "both-hands",
|
||||
"hands": "both",
|
||||
"text": "Both hands up, palms away. Each lit button says L or R:|point that hand at it and middle tap.",
|
||||
"windows": [
|
||||
{
|
||||
"id": "a",
|
||||
"at": [
|
||||
0,
|
||||
-0.05,
|
||||
-1.1
|
||||
],
|
||||
"size": [
|
||||
0.7,
|
||||
0.45
|
||||
],
|
||||
"look": "grid:3x3"
|
||||
}
|
||||
],
|
||||
"acts": {
|
||||
"gen": "buttons",
|
||||
"every_s": 2.4,
|
||||
"seconds": 16,
|
||||
"mix": {
|
||||
"middle": 1
|
||||
},
|
||||
"hands": [
|
||||
"left",
|
||||
"right"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "presses-right",
|
||||
"hands": "right",
|
||||
"text": "Right hand pointing. A button showing 2: a double middle tap. A button showing hold: press and keep it until it goes out, as if dragging.",
|
||||
"windows": [
|
||||
{
|
||||
"id": "a",
|
||||
"at": [
|
||||
0,
|
||||
-0.05,
|
||||
-1.1
|
||||
],
|
||||
"size": [
|
||||
0.7,
|
||||
0.45
|
||||
],
|
||||
"look": "grid:3x3"
|
||||
}
|
||||
],
|
||||
"acts": {
|
||||
"gen": "buttons",
|
||||
"every_s": 1.6,
|
||||
"seconds": 12,
|
||||
"mix": {
|
||||
"middle": 2,
|
||||
"middle-hold": 1
|
||||
},
|
||||
"lit_s": 1.2
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "mixed-right",
|
||||
"hands": "right",
|
||||
"text": "Right hand pointing. Cyan: tap with the index. Purple: tap with the middle finger.|Telling the two apart is the point: make each one clear.",
|
||||
"windows": [
|
||||
{
|
||||
"id": "a",
|
||||
"at": [
|
||||
0,
|
||||
-0.05,
|
||||
-1.1
|
||||
],
|
||||
"size": [
|
||||
0.7,
|
||||
0.45
|
||||
],
|
||||
"look": "grid:3x3"
|
||||
}
|
||||
],
|
||||
"acts": {
|
||||
"gen": "buttons",
|
||||
"every_s": 1.4,
|
||||
"seconds": 12,
|
||||
"mix": {
|
||||
"tap": 1,
|
||||
"middle": 1
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "mixed-left",
|
||||
"hands": "left",
|
||||
"text": "Left hand pointing. Cyan: index tap. Purple: middle tap.",
|
||||
"windows": [
|
||||
{
|
||||
"id": "a",
|
||||
"at": [
|
||||
-0.3,
|
||||
-0.05,
|
||||
-1.0
|
||||
],
|
||||
"size": [
|
||||
0.6,
|
||||
0.4
|
||||
],
|
||||
"look": "grid:3x3"
|
||||
}
|
||||
],
|
||||
"acts": {
|
||||
"gen": "buttons",
|
||||
"every_s": 1.4,
|
||||
"seconds": 12,
|
||||
"mix": {
|
||||
"tap": 1,
|
||||
"middle": 1
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "look-alikes",
|
||||
"hands": "right",
|
||||
"text": "Right hand pointing. Purple: middle tap. Orange with a cross: don't touch, stop about 1 cm short.|Now and then close the hand slowly into a fist instead.",
|
||||
"windows": [
|
||||
{
|
||||
"id": "a",
|
||||
"at": [
|
||||
0,
|
||||
-0.05,
|
||||
-1.1
|
||||
],
|
||||
"size": [
|
||||
0.7,
|
||||
0.45
|
||||
],
|
||||
"look": "grid:3x3"
|
||||
}
|
||||
],
|
||||
"acts": {
|
||||
"gen": "buttons",
|
||||
"every_s": 1.5,
|
||||
"seconds": 12,
|
||||
"mix": {
|
||||
"middle": 1,
|
||||
"nogo": 1
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "free",
|
||||
"hands": "any",
|
||||
"text": "Your own pace: point at the lit button and middle tap as often as you like.|Move the arm while you do: straight out, bent, out to the side, higher and lower.",
|
||||
"windows": [
|
||||
{
|
||||
"id": "a",
|
||||
"at": [
|
||||
0,
|
||||
-0.05,
|
||||
-1.1
|
||||
],
|
||||
"size": [
|
||||
0.7,
|
||||
0.45
|
||||
],
|
||||
"look": "grid:3x3"
|
||||
}
|
||||
],
|
||||
"acts": {
|
||||
"gen": "buttons",
|
||||
"every_s": 4,
|
||||
"seconds": 16,
|
||||
"mix": {
|
||||
"middle": 1
|
||||
},
|
||||
"lit_s": 4,
|
||||
"says": {
|
||||
"middle": "Your own pace: middle tap it as often as you like"
|
||||
}
|
||||
},
|
||||
"switch": [
|
||||
{
|
||||
"at": 0.5,
|
||||
"to": "the other hand"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,564 @@
|
||||
{
|
||||
"version": 1,
|
||||
"name": "pinches",
|
||||
"record_hz": 30,
|
||||
"intro_s": 4,
|
||||
"between_s": 3,
|
||||
"welcome": {
|
||||
"title": "Pinches",
|
||||
"seconds": 8,
|
||||
"text": "Windows will appear in front of you. Do what each one asks, as you would on a real one: they don't move or click, the cameras record your hands.|Press the button on the right side of the headset to start each step. Pressed while recording, it pauses."
|
||||
},
|
||||
"done": {
|
||||
"title": "All done",
|
||||
"seconds": 6,
|
||||
"text": "Thank you! You can take off the headset.|Review your recordings in the Hand recorder window."
|
||||
},
|
||||
"stopped": {
|
||||
"title": "Stopped",
|
||||
"seconds": 4,
|
||||
"text": "What was recorded so far is saved. Review it in the Hand recorder window."
|
||||
},
|
||||
"sections": [
|
||||
{
|
||||
"id": "hand-size",
|
||||
"title": "Hand size",
|
||||
"requires": [],
|
||||
"go": "Hold",
|
||||
"quick": true,
|
||||
"intro": "First, your hand size: both hands up about 40 cm out, still.",
|
||||
"prompts": [
|
||||
{
|
||||
"text": "Both hands about 40 cm out, still|Palms toward you",
|
||||
"seconds": 6,
|
||||
"hands": "both",
|
||||
"distance": "mid",
|
||||
"switch": [
|
||||
{
|
||||
"at": 0.5,
|
||||
"to": "backs toward you",
|
||||
"text": "Both hands about 40 cm out, still|Backs toward you"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "pinches",
|
||||
"title": "Pinches",
|
||||
"kind": "scene",
|
||||
"intro": "A window within reach. When a part lights up, pinch it there: thumb and index together, as if you were taking hold of it. It won't move: move your hand as if it did, then let go.",
|
||||
"prompts": [
|
||||
{
|
||||
"id": "hold-right",
|
||||
"hands": "right",
|
||||
"text": "Right hand. Pinch the lit part where it is, hold until it goes out, then let go.",
|
||||
"windows": [
|
||||
{
|
||||
"id": "a",
|
||||
"at": [
|
||||
0.05,
|
||||
-0.2,
|
||||
-0.45
|
||||
],
|
||||
"size": [
|
||||
0.34,
|
||||
0.24
|
||||
],
|
||||
"look": "plain"
|
||||
}
|
||||
],
|
||||
"acts": {
|
||||
"gen": "moves",
|
||||
"gesture": "pinch",
|
||||
"every_s": 2.5,
|
||||
"gap_s": 1.0,
|
||||
"moves": [
|
||||
{
|
||||
"grab": "bar"
|
||||
},
|
||||
{
|
||||
"grab": "br"
|
||||
},
|
||||
{
|
||||
"grab": "bar"
|
||||
},
|
||||
{
|
||||
"grab": "tr"
|
||||
}
|
||||
],
|
||||
"say": "Pinch the lit part and hold"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "drag-right",
|
||||
"hands": "right",
|
||||
"text": "Right hand. Pinch the lit bar and take it to the ghost, as if dragging the window, then let go.",
|
||||
"windows": [
|
||||
{
|
||||
"id": "a",
|
||||
"at": [
|
||||
0.05,
|
||||
-0.2,
|
||||
-0.45
|
||||
],
|
||||
"size": [
|
||||
0.34,
|
||||
0.24
|
||||
],
|
||||
"look": "plain"
|
||||
}
|
||||
],
|
||||
"acts": {
|
||||
"gen": "moves",
|
||||
"gesture": "pinch",
|
||||
"grab": "bar",
|
||||
"every_s": 3.5,
|
||||
"gap_s": 1.2,
|
||||
"count": 3,
|
||||
"moves": [
|
||||
[
|
||||
0,
|
||||
0.15,
|
||||
0
|
||||
],
|
||||
[
|
||||
0,
|
||||
-0.12,
|
||||
0
|
||||
],
|
||||
[
|
||||
-0.2,
|
||||
0,
|
||||
0
|
||||
],
|
||||
[
|
||||
0.2,
|
||||
0,
|
||||
0
|
||||
]
|
||||
],
|
||||
"say": "Pinch the bar, take it to the ghost"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "drag-left",
|
||||
"hands": "left",
|
||||
"text": "Left hand. Pinch the bar and take it to the ghost: nearer, farther, up and across.",
|
||||
"windows": [
|
||||
{
|
||||
"id": "a",
|
||||
"at": [
|
||||
-0.05,
|
||||
-0.2,
|
||||
-0.45
|
||||
],
|
||||
"size": [
|
||||
0.34,
|
||||
0.24
|
||||
],
|
||||
"look": "plain"
|
||||
}
|
||||
],
|
||||
"acts": {
|
||||
"gen": "moves",
|
||||
"gesture": "pinch",
|
||||
"grab": "bar",
|
||||
"every_s": 3.5,
|
||||
"gap_s": 1.2,
|
||||
"count": 3,
|
||||
"moves": [
|
||||
[
|
||||
0,
|
||||
0,
|
||||
0.15
|
||||
],
|
||||
[
|
||||
0,
|
||||
0,
|
||||
-0.25
|
||||
],
|
||||
[
|
||||
-0.15,
|
||||
0.1,
|
||||
0
|
||||
],
|
||||
[
|
||||
0.15,
|
||||
0.1,
|
||||
0
|
||||
]
|
||||
],
|
||||
"say": "Pinch the bar, take it to the ghost"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "resize",
|
||||
"hands": "right",
|
||||
"text": "Right hand. Pinch the lit corner and pull it until the window would be the ghost's size.",
|
||||
"windows": [
|
||||
{
|
||||
"id": "a",
|
||||
"at": [
|
||||
0.05,
|
||||
-0.2,
|
||||
-0.45
|
||||
],
|
||||
"size": [
|
||||
0.34,
|
||||
0.24
|
||||
],
|
||||
"look": "plain"
|
||||
}
|
||||
],
|
||||
"acts": {
|
||||
"gen": "moves",
|
||||
"gesture": "pinch",
|
||||
"every_s": 3.5,
|
||||
"gap_s": 1.0,
|
||||
"moves": [
|
||||
{
|
||||
"grab": "br",
|
||||
"ghost": [
|
||||
0.051000000000000004,
|
||||
-0.036,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
1.3
|
||||
]
|
||||
},
|
||||
{
|
||||
"grab": "tl",
|
||||
"ghost": [
|
||||
-0.051000000000000004,
|
||||
0.036,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
1.3
|
||||
]
|
||||
},
|
||||
{
|
||||
"grab": "br",
|
||||
"ghost": [
|
||||
-0.034,
|
||||
0.024,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0.8
|
||||
]
|
||||
}
|
||||
],
|
||||
"say": "Pinch the corner, pull it to the ghost's size"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "tilt",
|
||||
"hands": "left",
|
||||
"text": "Left hand. Pinch the bar and turn your wrist to tilt the window like the ghost.",
|
||||
"windows": [
|
||||
{
|
||||
"id": "a",
|
||||
"at": [
|
||||
-0.05,
|
||||
-0.2,
|
||||
-0.45
|
||||
],
|
||||
"size": [
|
||||
0.34,
|
||||
0.24
|
||||
],
|
||||
"look": "plain"
|
||||
}
|
||||
],
|
||||
"acts": {
|
||||
"gen": "moves",
|
||||
"gesture": "pinch",
|
||||
"grab": "bar",
|
||||
"every_s": 3.0,
|
||||
"gap_s": 1.0,
|
||||
"count": 3,
|
||||
"moves": [
|
||||
[
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
25
|
||||
],
|
||||
[
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
-25
|
||||
],
|
||||
[
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
30,
|
||||
0,
|
||||
0
|
||||
],
|
||||
[
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
-25,
|
||||
0
|
||||
]
|
||||
],
|
||||
"say": "Pinch the bar, tilt it like the ghost"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "stretch",
|
||||
"hands": "both",
|
||||
"text": "Both hands. Pinch the two lit corners at once and stretch or turn the window to the ghost.",
|
||||
"windows": [
|
||||
{
|
||||
"id": "a",
|
||||
"at": [
|
||||
0,
|
||||
-0.2,
|
||||
-0.45
|
||||
],
|
||||
"size": [
|
||||
0.34,
|
||||
0.24
|
||||
],
|
||||
"look": "plain"
|
||||
}
|
||||
],
|
||||
"acts": {
|
||||
"gen": "moves",
|
||||
"gesture": "pinch",
|
||||
"grab": "tl+br",
|
||||
"every_s": 3.5,
|
||||
"gap_s": 1.0,
|
||||
"moves": [
|
||||
[
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
1.35
|
||||
],
|
||||
[
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0.75
|
||||
],
|
||||
{
|
||||
"grab": "bl+tr",
|
||||
"ghost": [
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
30,
|
||||
0,
|
||||
0
|
||||
]
|
||||
}
|
||||
],
|
||||
"say": "Pinch both corners, stretch it to the ghost"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "hold-and-tap",
|
||||
"hands": "both",
|
||||
"text": "Left hand: pinch the lit bar on the left window and keep holding.|Right hand: tap the buttons on the right one as they light.",
|
||||
"windows": [
|
||||
{
|
||||
"id": "a",
|
||||
"at": [
|
||||
-0.22,
|
||||
-0.2,
|
||||
-0.45
|
||||
],
|
||||
"size": [
|
||||
0.28,
|
||||
0.2
|
||||
],
|
||||
"look": "plain"
|
||||
},
|
||||
{
|
||||
"id": "b",
|
||||
"at": [
|
||||
0.3,
|
||||
-0.15,
|
||||
-0.7
|
||||
],
|
||||
"size": [
|
||||
0.36,
|
||||
0.26
|
||||
],
|
||||
"look": "grid:2x2"
|
||||
}
|
||||
],
|
||||
"steady": [
|
||||
{
|
||||
"win": "a",
|
||||
"lit": "bar",
|
||||
"gesture": "pinch",
|
||||
"label": "L"
|
||||
}
|
||||
],
|
||||
"acts": {
|
||||
"gen": "buttons",
|
||||
"windows": [
|
||||
"b"
|
||||
],
|
||||
"every_s": 1.4,
|
||||
"seconds": 11,
|
||||
"mix": {
|
||||
"tap": 1
|
||||
},
|
||||
"says": {
|
||||
"tap": "Left: hold the pinch. Right: tap the lit button"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "far",
|
||||
"hands": "right",
|
||||
"text": "A window far away. Look at the lit row, pinch with your right hand low by your side, move the hand a few cm, then let go.",
|
||||
"windows": [
|
||||
{
|
||||
"id": "a",
|
||||
"at": [
|
||||
0,
|
||||
0,
|
||||
-2.0
|
||||
],
|
||||
"size": [
|
||||
1.0,
|
||||
0.7
|
||||
],
|
||||
"look": "list:6"
|
||||
}
|
||||
],
|
||||
"acts": {
|
||||
"gen": "buttons",
|
||||
"every_s": 2.6,
|
||||
"lit_s": 1.9,
|
||||
"seconds": 13,
|
||||
"mix": {
|
||||
"pinch": 1
|
||||
},
|
||||
"says": {
|
||||
"pinch": "Look at it, pinch with the hand low, nudge it, let go"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "look-alikes",
|
||||
"hands": "right",
|
||||
"text": "Right hand, reaching for the lit part. Cyan: pinch it. Orange: stop about 1 cm short, or touch it with an open hand: no pinch.",
|
||||
"windows": [
|
||||
{
|
||||
"id": "a",
|
||||
"at": [
|
||||
0.05,
|
||||
-0.2,
|
||||
-0.45
|
||||
],
|
||||
"size": [
|
||||
0.34,
|
||||
0.24
|
||||
],
|
||||
"look": "plain"
|
||||
}
|
||||
],
|
||||
"acts": {
|
||||
"gen": "moves",
|
||||
"every_s": 2.5,
|
||||
"gap_s": 0.8,
|
||||
"moves": [
|
||||
{
|
||||
"grab": "bar",
|
||||
"gesture": "pinch"
|
||||
},
|
||||
{
|
||||
"grab": "bar",
|
||||
"gesture": "nogo"
|
||||
},
|
||||
{
|
||||
"grab": "left",
|
||||
"gesture": "touch"
|
||||
},
|
||||
{
|
||||
"grab": "br",
|
||||
"gesture": "pinch"
|
||||
},
|
||||
{
|
||||
"grab": "tr",
|
||||
"gesture": "nogo"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "free",
|
||||
"hands": "any",
|
||||
"text": "Your own pace, either hand: pinch the bar and drag the window to each ghost in turn, any way you like.",
|
||||
"windows": [
|
||||
{
|
||||
"id": "a",
|
||||
"at": [
|
||||
0,
|
||||
-0.2,
|
||||
-0.5
|
||||
],
|
||||
"size": [
|
||||
0.34,
|
||||
0.24
|
||||
],
|
||||
"look": "plain"
|
||||
}
|
||||
],
|
||||
"acts": {
|
||||
"gen": "moves",
|
||||
"gesture": "pinch",
|
||||
"grab": "bar",
|
||||
"every_s": 5,
|
||||
"gap_s": 0,
|
||||
"moves": [
|
||||
[
|
||||
0.2,
|
||||
0.1,
|
||||
0
|
||||
],
|
||||
[
|
||||
-0.2,
|
||||
-0.05,
|
||||
0.1
|
||||
],
|
||||
[
|
||||
0,
|
||||
0.15,
|
||||
-0.2
|
||||
]
|
||||
],
|
||||
"say": "Your own pace: drag it to the ghost"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
+5
-5
@@ -1,18 +1,18 @@
|
||||
#!/usr/bin/env bash
|
||||
# Build the hand recorder's headset panel ft-handpanel in the dev container on the Frame
|
||||
# (hands/rec/build/; it also runs there). Like gaze/build.sh: the pinned public OpenVR header
|
||||
# (the DMA-BUF import is newer than the header shipped with SteamVR's samples), stb_truetype
|
||||
# (hands/rec/build/; it also runs there), or in the release image (pack/Containerfile). Like
|
||||
# gaze/build.sh: the pinned public OpenVR header (scripts/openvr.sh; the DMA-BUF import is newer
|
||||
# than the header shipped with SteamVR's samples), stb_truetype
|
||||
# for the text and stb_image (PNG and GIF) for the pose pictures and the demo clips, at the same stb commit.
|
||||
set -euo pipefail
|
||||
root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)
|
||||
"$root/scripts/sync.sh" >/dev/null
|
||||
exec "$root/scripts/frame.sh" -C hands/rec 'set -e; mkdir -p build/include
|
||||
openvr=v2.15.6
|
||||
[ -f build/include/openvr-$openvr ] || { curl -fsSL "https://raw.githubusercontent.com/ValveSoftware/openvr/$openvr/headers/openvr.h" -o build/include/openvr.h && touch build/include/openvr-$openvr; }
|
||||
. ../../scripts/openvr.sh
|
||||
stb=2c980bb59875b0d32144a71867fbdebb2f77cd20
|
||||
[ -f build/include/stb-$stb ] || { curl -fsSL "https://raw.githubusercontent.com/nothings/stb/$stb/stb_truetype.h" -o build/include/stb_truetype.h && touch build/include/stb-$stb; }
|
||||
[ -f build/include/stb_image-$stb ] || { curl -fsSL "https://raw.githubusercontent.com/nothings/stb/$stb/stb_image.h" -o build/include/stb_image.h && touch build/include/stb_image-$stb; }
|
||||
g++ -std=c++17 -O2 -Wall -Wno-unused-parameter -Wno-missing-field-initializers -Ibuild/include $(pkg-config --cflags gbm libdrm) \
|
||||
-o build/ft-handpanel panel/ft-handpanel.cpp -L/opt/steamvr/bin/linuxarm64 -lopenvr_api -Wl,-rpath,/opt/steamvr/bin/linuxarm64 \
|
||||
-o build/ft-handpanel panel/ft-handpanel.cpp $OPENVR_LIBS \
|
||||
$(pkg-config --libs gbm libdrm) -lpthread
|
||||
echo "built build/ft-handpanel"'
|
||||
@@ -0,0 +1,22 @@
|
||||
{
|
||||
"version": 1,
|
||||
"name": "debug",
|
||||
"debug": true,
|
||||
"record_hz": 30,
|
||||
"intro_s": 4,
|
||||
"between_s": 3,
|
||||
"welcome": {"title": "Testing session", "seconds": 6,
|
||||
"text": "Use your hands in Frametop as you always do, for up to 3 minutes. The cameras record them, and what the hand tracking made of them.|Something went wrong? Press the headset button right then: it marks the moment. Hold the button to finish."},
|
||||
"done": {"title": "All done", "seconds": 5,
|
||||
"text": "Thank you! Upload it from the Hand recorder window, so new versions of the hand tracking can be tested on it."},
|
||||
"stopped": {"title": "Stopped", "seconds": 4,
|
||||
"text": "What was recorded so far is saved. You can upload it from the Hand recorder window."},
|
||||
"sections": [
|
||||
{"id": "free", "title": "Testing session", "go": "Go",
|
||||
"intro": "Use Frametop as you normally would: point, tap, pinch, drag and scroll windows. Do what gives you trouble.",
|
||||
"prompts": [
|
||||
{"id": "use", "text": "Use Frametop as you normally would.|This panel goes away while you record. Press the headset button when something goes wrong; hold it to finish.",
|
||||
"seconds": 180, "hands": ""}
|
||||
]}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,109 @@
|
||||
#!/usr/bin/env python3
|
||||
"""What Frametop did with the hands during a testing session (hands/include/fh_events.h): the
|
||||
panels' (ft-screens) and the pointer helper's decisions, one JSON line each with its time, what
|
||||
was tried (kind) and how it went (outcome: done, refused, missed, ...). Beside the take's
|
||||
live.bin and the button's marks they say what a mark or a spoken note was about ("that opened
|
||||
the lazy susan" next to a lazy-susan "done"), and how many pokes, pinches and strokes worked.
|
||||
|
||||
While a part records, events-want sits in the run directory, so the writers append to
|
||||
events.jsonl there; at the end the lines from this part's time go into the take's events.jsonl
|
||||
(events-2.jsonl, ... for later parts) and the run directory's copy is removed.
|
||||
|
||||
Standard library only.
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) # recorder/, run on its own
|
||||
from recorder.common import mono_ns # noqa: E402
|
||||
|
||||
WANT = "events-want"
|
||||
FILE = "events.jsonl"
|
||||
OUTCOMES = ("done", "refused", "missed", "passed", "lost", "end", "info")
|
||||
|
||||
|
||||
def read_lines(path, start=0):
|
||||
"""The events in path from byte start: a list of dicts (lines that aren't one are skipped)."""
|
||||
out = []
|
||||
try:
|
||||
with open(path, "rb") as f:
|
||||
f.seek(start)
|
||||
data = f.read()
|
||||
except OSError:
|
||||
return out
|
||||
for line in data.split(b"\n"):
|
||||
try:
|
||||
e = json.loads(line)
|
||||
except ValueError:
|
||||
continue
|
||||
if isinstance(e, dict) and isinstance(e.get("t"), int) and isinstance(e.get("kind"), str):
|
||||
out.append(e)
|
||||
return out
|
||||
|
||||
|
||||
def summary(events):
|
||||
"""{kind: {outcome: n}} over events, outcomes in OUTCOMES' order."""
|
||||
by = {}
|
||||
for e in events:
|
||||
k = by.setdefault(str(e.get("kind")), {})
|
||||
o = str(e.get("outcome"))
|
||||
k[o] = k.get(o, 0) + 1
|
||||
return {k: {o: v[o] for o in sorted(v, key=lambda o: OUTCOMES.index(o) if o in OUTCOMES else 99)}
|
||||
for k, v in sorted(by.items())}
|
||||
|
||||
|
||||
def describe(by):
|
||||
""""poke 4 done, 2 refused; lazy-susan 1 done" for the log."""
|
||||
return "; ".join("%s %s" % (k, ", ".join("%d %s" % (n, o) for o, n in v.items())) for k, v in by.items()) or "none"
|
||||
|
||||
|
||||
class EventLog:
|
||||
"""Asks the writers for events while a part records and copies them into path at stop()."""
|
||||
|
||||
def __init__(self, run_dir, path):
|
||||
self.run_dir, self.path = run_dir, path
|
||||
self.start_ns = 0
|
||||
self._offset = 0
|
||||
|
||||
def start(self):
|
||||
os.makedirs(self.run_dir, mode=0o700, exist_ok=True)
|
||||
src = os.path.join(self.run_dir, FILE)
|
||||
try:
|
||||
os.remove(src) # a crashed session's: its lines aren't this part's
|
||||
except OSError:
|
||||
pass
|
||||
self._offset = 0
|
||||
self.start_ns = mono_ns()
|
||||
with open(os.path.join(self.run_dir, WANT), "w") as f:
|
||||
f.write("%d\n" % os.getpid())
|
||||
return self
|
||||
|
||||
def stop(self):
|
||||
"""Ends it: {"events": n, "by": summary}."""
|
||||
end_ns = mono_ns()
|
||||
stop_writers(self.run_dir)
|
||||
src = os.path.join(self.run_dir, FILE)
|
||||
events = [e for e in read_lines(src, self._offset) if self.start_ns <= e["t"] <= end_ns]
|
||||
with open(self.path, "w") as f:
|
||||
for e in events:
|
||||
f.write(json.dumps(e, separators=(", ", ": ")) + "\n")
|
||||
try:
|
||||
os.remove(src)
|
||||
except OSError:
|
||||
pass
|
||||
return {"events": len(events), "by": summary(events)}
|
||||
|
||||
|
||||
def stop_writers(run_dir):
|
||||
"""No more events (also for a session that crashed with them asked for)."""
|
||||
try:
|
||||
os.remove(os.path.join(run_dir, WANT))
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if len(sys.argv) != 2:
|
||||
sys.exit("eventlog.py EVENTS.JSONL: how many of each kind of interaction worked")
|
||||
print(json.dumps(summary(read_lines(sys.argv[1])), indent=1))
|
||||
@@ -1,6 +1,6 @@
|
||||
#!/bin/bash
|
||||
# Launch the Frametop Hand Recorder from a Plasma session on the Frame host.
|
||||
# The app runs in the dev container (PySide6, Kirigami and zstd come from Fedora there).
|
||||
# The app runs in Frametop's container (PySide6, Kirigami and zstd come from Fedora there).
|
||||
# podman needs the real XDG_RUNTIME_DIR and the real user bus (to reach systemd for
|
||||
# the container's cgroup; the Frametop session runs on a private bus from
|
||||
# dbus-run-session). The session's Wayland socket and bus go to the app itself.
|
||||
@@ -14,8 +14,7 @@ export DBUS_SESSION_BUS_ADDRESS=unix:path=$XDG_RUNTIME_DIR/bus
|
||||
# From the home folder: the app's host commands (distrobox-host-exec) run in its folder on the
|
||||
# host, and a folder only the container has, such as /run/host/tmp, makes them all fail.
|
||||
cd "$HOME" || exit 1
|
||||
"$here/../../scripts/container-up.sh"
|
||||
exec "$HOME/.local/bin/distrobox" enter dev -- env WAYLAND_DISPLAY="$wl" DISPLAY="${DISPLAY:-}" \
|
||||
exec "$here/../../scripts/in-box" env WAYLAND_DISPLAY="$wl" DISPLAY="${DISPLAY:-}" \
|
||||
XAUTHORITY="${XAUTHORITY:-}" DBUS_SESSION_BUS_ADDRESS="$session_bus" \
|
||||
QT_QPA_PLATFORM="wayland;xcb" \
|
||||
python3 "$here/ft_handrec.py" "$@"
|
||||
@@ -1,8 +1,8 @@
|
||||
[Desktop Entry]
|
||||
Type=Application
|
||||
Name=Frametop Hand Recorder
|
||||
GenericName=Record your hands for the hand dataset
|
||||
Comment=Record, review and export hand recordings for Frametop's open hand dataset
|
||||
GenericName=Hand recorder for Frametop's hand tracking
|
||||
Comment=Record your hands for Frametop's hand tracking: bounty rounds and full sessions for its hand dataset, testing sessions to show what goes wrong
|
||||
Exec=@REPO@/hands/rec/ft-handrec
|
||||
Icon=camera-video
|
||||
Categories=Utility;
|
||||
|
||||
+304
-637
File diff suppressed because it is too large.
Load diff
+124
-218
@@ -25,9 +25,17 @@ An upload:
|
||||
--dry-run does all of it except the network calls (4 to 6) and recording (5, 7), and says what
|
||||
it would upload.
|
||||
|
||||
The dataset is HF_DATASET; FT_HANDREC_DATASET overrides it (a test repo, for rehearsals).
|
||||
The dataset is HF_DATASET; FT_HANDREC_DATASET overrides it (a test repo, for rehearsals). In the
|
||||
bounties' test phase (DESIGN.md "Testing") a session recorded under TESTING.md's consent (a bounty
|
||||
round or a debug session: manifest "consent_kind": "testing", its recordings encrypted) goes to
|
||||
HF_TEST_DATASET instead (FT_HANDREC_TEST_DATASET overrides it), and its gate is TESTING.md's.
|
||||
|
||||
`board` fetches the bounty board (bounties.json) from the dataset into BASE/board-cache.json,
|
||||
with the saved login when there is one (the dataset is gated): the window's board then uses it
|
||||
when it's newer than the one shipped here (recorder/bounties.py checks what it may say).
|
||||
|
||||
usage: hub.py [--base DIR] whoami [--json]
|
||||
hub.py [--base DIR] board [--json]
|
||||
hub.py [--base DIR] login [--json]
|
||||
hub.py [--base DIR] upload SESSION [--dry-run] [--again] [--json]
|
||||
With --json, each line of output is one JSON object: {"phase", "text", "fraction"} as it goes,
|
||||
@@ -36,149 +44,124 @@ with {"phase": "opened", "pr_url"} once the pull request exists, then {"done": {
|
||||
"expires_in"} once it has the link, then {"done": {"name", "role"}}. Exit status 0: done.
|
||||
"""
|
||||
import argparse
|
||||
import datetime
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
sys.path.insert(0, HERE)
|
||||
import takes # noqa: E402 (next to this file)
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(HERE)))
|
||||
from recorder import bounties # noqa: E402
|
||||
from recorder import hub as rhub # noqa: E402
|
||||
from recorder.hub import HF_ENV, REPO_RE, TOKENS_URL, HubError, explain, export_sha, is_draft, now_iso # noqa: E402,F401
|
||||
|
||||
# The dataset contributions go to (DESIGN.md "Licensing and consent").
|
||||
# The datasets contributions go to (DESIGN.md "Licensing and consent", "Testing").
|
||||
HF_DATASET = "DeeJanuz/frametop-hands"
|
||||
HF_TEST_DATASET = "DeeJanuz/frametop-hands-test" # gated; everything in it encrypted to the maintainer
|
||||
DATASET_ENV = "FT_HANDREC_DATASET"
|
||||
TEST_DATASET_ENV = "FT_HANDREC_TEST_DATASET"
|
||||
ALLOW_ENV = "FT_HANDREC_ALLOW_UPLOAD"
|
||||
CONSENT_PATH = os.path.join(HERE, "CONSENT.md")
|
||||
TESTING_PATH = os.path.join(HERE, "TESTING.md")
|
||||
UPLOAD_PATH = os.path.join(HERE, "UPLOAD.md")
|
||||
REPO_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]*/[A-Za-z0-9][A-Za-z0-9._-]*$")
|
||||
TOKENS_URL = "https://huggingface.co/settings/tokens"
|
||||
# Quiet huggingface_hub: no progress bars on stderr, no telemetry or update hints.
|
||||
HF_ENV = {"HF_HUB_DISABLE_PROGRESS_BARS": "1", "HF_HUB_DISABLE_TELEMETRY": "1", "HF_HUB_DISABLE_UPDATE_CHECK": "1"}
|
||||
KINDS = ("public", "testing")
|
||||
|
||||
|
||||
class HubError(Exception):
|
||||
"""What went wrong, for people: kind (below), text, a link to open, validation errors.
|
||||
Kinds: missing (no huggingface_hub), login, read-token, terms, not-found, permission,
|
||||
network, hub, invalid, duplicate, closed, no-export."""
|
||||
|
||||
def __init__(self, kind, text, link="", errors=None, extra=None):
|
||||
super().__init__(text)
|
||||
self.kind, self.text, self.link = kind, text, link
|
||||
self.errors = errors or []
|
||||
self.extra = extra or {}
|
||||
|
||||
def as_dict(self):
|
||||
return dict(self.extra, kind=self.kind, text=self.text, link=self.link, errors=self.errors)
|
||||
|
||||
|
||||
def dataset_id():
|
||||
"""HF_DATASET, or FT_HANDREC_DATASET when set."""
|
||||
repo = os.environ.get(DATASET_ENV, "").strip() or HF_DATASET
|
||||
if not REPO_RE.match(repo):
|
||||
raise HubError("not-found", f"{DATASET_ENV}={repo!r} isn't a dataset id like owner/name")
|
||||
return repo
|
||||
def dataset_id(kind="public"):
|
||||
"""HF_DATASET (FT_HANDREC_DATASET when set), or for "testing" HF_TEST_DATASET
|
||||
(FT_HANDREC_TEST_DATASET when set)."""
|
||||
if kind == "testing":
|
||||
return rhub.dataset_id(TEST_DATASET_ENV, HF_TEST_DATASET)
|
||||
return rhub.dataset_id(DATASET_ENV, HF_DATASET)
|
||||
|
||||
|
||||
def dataset_url(repo=None):
|
||||
return "https://huggingface.co/datasets/" + (repo or dataset_id())
|
||||
return rhub.dataset_url(repo or dataset_id())
|
||||
|
||||
|
||||
def is_draft(path):
|
||||
try:
|
||||
with open(path, encoding="utf-8") as f:
|
||||
return "DRAFT" in f.readline()
|
||||
except OSError:
|
||||
return False
|
||||
def texts_draft(kind="public"):
|
||||
return is_draft(TESTING_PATH if kind == "testing" else CONSENT_PATH) or is_draft(UPLOAD_PATH)
|
||||
|
||||
|
||||
def texts_draft():
|
||||
return is_draft(CONSENT_PATH) or is_draft(UPLOAD_PATH)
|
||||
def upload_allowed(kind="public"):
|
||||
"""Real uploads: once the kind's texts aren't drafts, or for the maintainer's rehearsal
|
||||
against a test repo (FT_HANDREC_ALLOW_UPLOAD=1)."""
|
||||
return not texts_draft(kind) or os.environ.get(ALLOW_ENV) == "1"
|
||||
|
||||
|
||||
def upload_allowed():
|
||||
"""Real uploads: once the texts aren't drafts, or for the maintainer's rehearsal against a
|
||||
test repo (FT_HANDREC_ALLOW_UPLOAD=1)."""
|
||||
return not texts_draft() or os.environ.get(ALLOW_ENV) == "1"
|
||||
|
||||
|
||||
def now_iso():
|
||||
return datetime.datetime.now().astimezone().isoformat(timespec="seconds")
|
||||
def session_kind(store, session):
|
||||
"""The consent a session was recorded under (takes.consent_kind of its session.json)."""
|
||||
return takes.consent_kind(takes.read_json(meta_path(store, session)))
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- records in session.json
|
||||
|
||||
def export_sha(export_dir):
|
||||
"""The export's identity: the SHA256 of its SHA256SUMS ("" if there's none)."""
|
||||
try:
|
||||
with open(os.path.join(export_dir, "SHA256SUMS"), "rb") as f:
|
||||
return hashlib.sha256(f.read()).hexdigest()
|
||||
except OSError:
|
||||
return ""
|
||||
def meta_path(store, session):
|
||||
return os.path.join(store.session_dir(session), "session.json")
|
||||
|
||||
|
||||
def uploads(store, session):
|
||||
meta = takes.read_json(os.path.join(store.session_dir(session), "session.json"))
|
||||
return [u for u in meta.get("uploads") or [] if isinstance(u, dict)]
|
||||
return rhub.uploads(meta_path(store, session))
|
||||
|
||||
|
||||
def previous_upload(store, session, sha):
|
||||
"""The latest finished upload of this same export, or None. (Records from before
|
||||
2026-10-03 have no status: they were written only when an upload finished.)"""
|
||||
same = [u for u in uploads(store, session) if sha and u.get("export_sha") == sha
|
||||
and u.get("status", "done") == "done"]
|
||||
return same[-1] if same else None
|
||||
return rhub.previous_upload(meta_path(store, session), sha)
|
||||
|
||||
|
||||
def unfinished_upload(store, session, sha, repo):
|
||||
"""The latest upload of this same export to repo that opened its pull request and didn't
|
||||
finish, or None."""
|
||||
same = [u for u in uploads(store, session) if sha and u.get("export_sha") == sha and u.get("repo") == repo
|
||||
and u.get("status") == "started" and u.get("pr_num")]
|
||||
return same[-1] if same else None
|
||||
return rhub.unfinished_upload(meta_path(store, session), sha, repo)
|
||||
|
||||
|
||||
def record_upload(store, session, record):
|
||||
"""Add record to session.json's "uploads". The file keeps its time: an upload doesn't
|
||||
change the recordings, so the export mustn't count as out of date because of it
|
||||
(takes.Store.sessions compares times)."""
|
||||
path = os.path.join(store.session_dir(session), "session.json")
|
||||
try:
|
||||
st = os.stat(path)
|
||||
except OSError:
|
||||
st = None
|
||||
meta = takes.read_json(path)
|
||||
records = meta.setdefault("uploads", [])
|
||||
same = [i for i, u in enumerate(records) if isinstance(u, dict) and record.get("pr_num")
|
||||
and u.get("pr_num") == record["pr_num"] and u.get("repo") == record.get("repo")]
|
||||
if same:
|
||||
records[same[-1]] = record # the same pull request: started, then done
|
||||
else:
|
||||
records.append(record)
|
||||
takes.write_json(path, meta)
|
||||
if st:
|
||||
os.utime(path, ns=(st.st_atime_ns, st.st_mtime_ns))
|
||||
rhub.record_upload(meta_path(store, session), record, takes.write_json)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- the pull request's text
|
||||
|
||||
def consent_line(s):
|
||||
"""The consent the upload goes under, and the one it was recorded under when that's older."""
|
||||
now, then = s.get("consent_version") or "", s.get("recorded_consent_version") or ""
|
||||
return now + (f" (recorded under {then})" if then and then != now else "")
|
||||
|
||||
|
||||
def describe(summary, report, sha, nbytes):
|
||||
"""(commit message, commit description) from validate's summary of the manifest."""
|
||||
s = summary
|
||||
if s.get("testing"):
|
||||
what = "debug session" if s.get("debug") else f"{s.get('bounty') or 'test'} round"
|
||||
lines = [f"Test upload ({what}) for the Frametop hand tracking, from the hand recorder. Its recordings are "
|
||||
"encrypted to the maintainer's keys; for testing only.", "",
|
||||
f"- Session: {s.get('session')}",
|
||||
f"- Kind: {'debug session (Frametop in use, the live tracker recorded)' if s.get('debug') else 'bounty round'}",
|
||||
f"- Bounty: {s.get('bounty') or 'none'}",
|
||||
f"- Contributor: {s.get('contributor')}",
|
||||
f"- Takes: {s.get('takes')} ({s.get('minutes')} minutes, {s.get('sets')} frame sets"
|
||||
f"{', %g a second' % s['record_hz'] if s.get('record_hz') else ''})",
|
||||
f"- Lighting: {s.get('lighting') or 'not given'}",
|
||||
f"- Consent: testing, version {consent_line(s)}",
|
||||
f"- Tool: {s.get('tool')}",
|
||||
f"- Size: {takes.human_bytes(nbytes)}",
|
||||
f"- SHA256 of SHA256SUMS: {sha}", "",
|
||||
"Checked with hands/rec/validate.py: no errors" + (f", {len(report.warnings)} warnings:"
|
||||
if report.warnings else ".")]
|
||||
lines += [f"- {w}" for w in report.warnings]
|
||||
return f"Hands test: {what} {s.get('session')} from {s.get('contributor')}", "\n".join(lines) + "\n"
|
||||
objects = ", ".join(s.get("objects") or []) or "none"
|
||||
if s.get("own_objects"):
|
||||
objects += f" (+{s['own_objects']} of their own)"
|
||||
message = f"Hands: session {s.get('session')} from {s.get('contributor')}"
|
||||
bounty = f"{s['bounty']} " if s.get("bounty") else ""
|
||||
message = f"Hands: {bounty}session {s.get('session')} from {s.get('contributor')}"
|
||||
lines = ["Contribution to the Frametop hand dataset, uploaded from the hand recorder.", "",
|
||||
f"- Session: {s.get('session')}",
|
||||
f"- Bounty: {s.get('bounty') or 'none (a full session or a quick round)'}",
|
||||
f"- Contributor: {s.get('contributor')}",
|
||||
f"- Takes: {s.get('takes')} ({s.get('minutes')} minutes, {s.get('sets')} frame sets)",
|
||||
f"- Takes: {s.get('takes')} ({s.get('minutes')} minutes, {s.get('sets')} frame sets"
|
||||
f"{', %g a second' % s['record_hz'] if s.get('record_hz') else ''})",
|
||||
f"- Lighting: {s.get('lighting') or 'not given'}",
|
||||
f"- Objects: {objects}",
|
||||
f"- Controllers: {s.get('controllers') or 'not given'}",
|
||||
f"- Consent version: {s.get('consent_version')}",
|
||||
f"- Consent version: {consent_line(s)}",
|
||||
f"- Tool: {s.get('tool')}",
|
||||
f"- Size: {takes.human_bytes(nbytes)}",
|
||||
f"- SHA256 of SHA256SUMS: {sha}", "",
|
||||
@@ -188,92 +171,51 @@ def describe(summary, report, sha, nbytes):
|
||||
return message, "\n".join(lines) + "\n"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- the bounty board
|
||||
|
||||
def board_url(repo):
|
||||
return f"{rhub.dataset_url(repo)}/resolve/main/bounties.json"
|
||||
|
||||
|
||||
def fetch_board(base):
|
||||
"""The dataset's bounties.json into base/board-cache.json (session.load_board reads it).
|
||||
Returns {"updated", "bounties"}; raises HubError."""
|
||||
repo = dataset_id("public") # the board is the public dataset's
|
||||
token = None
|
||||
try:
|
||||
token = _hf().get_token()
|
||||
except (HubError, AttributeError):
|
||||
pass # no huggingface_hub, or an old one: try without a login
|
||||
try:
|
||||
board = bounties.fetch(board_url(repo), os.path.join(base, "board-cache.json"), token)
|
||||
except ValueError as e:
|
||||
raise HubError("hub", f"The dataset's bounty board isn't valid: {e}") from None
|
||||
except OSError as e:
|
||||
status = getattr(e, "code", None)
|
||||
if status in (401, 403):
|
||||
raise HubError("terms", "Log in and accept the dataset's terms to get its newest bounty board.",
|
||||
rhub.dataset_url(repo)) from None
|
||||
if status == 404:
|
||||
raise HubError("not-found", f"{repo} has no bounty board yet.") from None
|
||||
raise HubError("network", f"Couldn't fetch the bounty board ({type(e).__name__}: {e}).") from None
|
||||
return {"updated": board["updated"], "bounties": len(board["bounties"])}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- talking to the Hub
|
||||
|
||||
def _hf():
|
||||
os.environ.update({k: v for k, v in HF_ENV.items() if k not in os.environ})
|
||||
try:
|
||||
import huggingface_hub
|
||||
except ImportError:
|
||||
raise HubError("missing", "huggingface_hub isn't installed in the dev container: run setup/dev-container.sh "
|
||||
"(or: pip install --user huggingface_hub)") from None
|
||||
return huggingface_hub
|
||||
|
||||
|
||||
def explain(e, repo):
|
||||
"""A huggingface_hub (or network) exception as a HubError."""
|
||||
if isinstance(e, HubError):
|
||||
return e
|
||||
try:
|
||||
from huggingface_hub import errors as hf
|
||||
except ImportError:
|
||||
hf = None
|
||||
url = dataset_url(repo)
|
||||
if hf is not None:
|
||||
if isinstance(e, hf.LocalTokenNotFoundError):
|
||||
return HubError("login", "You're not logged in to Hugging Face. Press Log in.")
|
||||
if isinstance(e, hf.GatedRepoError):
|
||||
return HubError("terms", f"Accept the dataset's terms first: open {url}, read them and accept them, "
|
||||
"then try again.", url)
|
||||
if isinstance(e, hf.RepositoryNotFoundError):
|
||||
return HubError("not-found", f"The dataset {repo} wasn't found, or it's private and your account has "
|
||||
"no access to it.", url)
|
||||
if isinstance(e, hf.HfHubHTTPError):
|
||||
status = getattr(getattr(e, "response", None), "status_code", None)
|
||||
first = str(e).strip().splitlines()[0] if str(e).strip() else type(e).__name__
|
||||
if status == 401:
|
||||
return HubError("login", "Hugging Face didn't accept your login (it may have expired or been "
|
||||
"revoked). Log in again.")
|
||||
if status == 403:
|
||||
return HubError("permission", "Your login isn't allowed to open a pull request. Log in again, "
|
||||
"and allow everything the Hugging Face page asks for.")
|
||||
return HubError("hub", f"Hugging Face refused the upload ({status or 'no status'}): {first}")
|
||||
try:
|
||||
import httpx
|
||||
net = (httpx.TransportError, OSError)
|
||||
except ImportError:
|
||||
net = (OSError,)
|
||||
if isinstance(e, net):
|
||||
return HubError("network", f"Couldn't reach huggingface.co ({type(e).__name__}: {e}). Check the network "
|
||||
"and try again: files already sent usually aren't sent twice.")
|
||||
return HubError("hub", f"{type(e).__name__}: {e}")
|
||||
return rhub.hf_module("run setup/dev-container.sh (or: pip install --user huggingface_hub)")
|
||||
|
||||
|
||||
def whoami():
|
||||
"""{"name", "role"} for the saved login; HubError if there's none or it doesn't work.
|
||||
role: "write", "read", "fineGrained" or ""."""
|
||||
hf = _hf()
|
||||
try:
|
||||
info = hf.HfApi().whoami()
|
||||
except Exception as e:
|
||||
raise explain(e, dataset_id()) from None
|
||||
token = ((info.get("auth") or {}).get("accessToken") or {})
|
||||
return {"name": info.get("name", ""), "role": token.get("role", "")}
|
||||
"""{"name", "role"} for the saved login; HubError if there's none or it doesn't work."""
|
||||
return rhub.whoami(_hf(), dataset_id())
|
||||
|
||||
|
||||
def login(progress=None):
|
||||
"""The browser login: progress("code", text, url=, code=, expires_in=) once the link is ready,
|
||||
then wait (up to the code's expiry: 5 minutes on 2026-10-03) for the person to approve it, and save
|
||||
the token. Returns whoami(). Uses huggingface_hub's own device code helpers (1.x)."""
|
||||
tell = progress or (lambda phase, text, **extra: None)
|
||||
"""The browser login (recorder/hub.py's login). Returns whoami()."""
|
||||
_hf()
|
||||
try:
|
||||
from huggingface_hub._login import _save_oauth_token
|
||||
from huggingface_hub.errors import DeviceCodeError
|
||||
from huggingface_hub.utils._oauth_device import poll_device_token, request_device_code
|
||||
except ImportError:
|
||||
raise HubError("missing", "This huggingface_hub has no browser login: update the dev container "
|
||||
"(setup/dev-container.sh).") from None
|
||||
try:
|
||||
info = request_device_code()
|
||||
tell("code", "Approve the login in your browser", url=info["verification_uri_complete"],
|
||||
code=info["user_code"], expires_in=info["expires_in"])
|
||||
_save_oauth_token(poll_device_token(info))
|
||||
except DeviceCodeError as e:
|
||||
raise HubError("login", f"The login didn't go through: {e}. Press Log in to try again.") from None
|
||||
except Exception as e:
|
||||
raise explain(e, dataset_id()) from None
|
||||
return whoami()
|
||||
return rhub.login(dataset_id(), progress, lambda: whoami(), "update the dev container (setup/dev-container.sh).")
|
||||
|
||||
|
||||
def upload(store, session, dry_run=False, again=False, progress=None, log=None):
|
||||
@@ -282,7 +224,6 @@ def upload(store, session, dry_run=False, again=False, progress=None, log=None):
|
||||
tell = progress or (lambda phase, text, fraction=None, **extra: None)
|
||||
say = log or (lambda text: None)
|
||||
import validate
|
||||
repo = dataset_id()
|
||||
try:
|
||||
export = store.export_dir(session)
|
||||
except ValueError as e:
|
||||
@@ -296,13 +237,18 @@ def upload(store, session, dry_run=False, again=False, progress=None, log=None):
|
||||
raise HubError("invalid", f"The export has {len(report.errors)} problems, so it can't be uploaded. Export "
|
||||
"the session again; if that doesn't help, report it.", errors=report.errors)
|
||||
summary = report.summary
|
||||
kind = "testing" if summary.get("testing") else "public"
|
||||
repo = dataset_id(kind)
|
||||
contributor = summary.get("contributor", "")
|
||||
sha = export_sha(export)
|
||||
before = previous_upload(store, session, sha)
|
||||
if before and not again:
|
||||
raise HubError("duplicate", f"This export was uploaded already, on {before.get('uploaded', '?')}: "
|
||||
f"{before.get('pr_url', '')}", before.get("pr_url", ""), extra={"previous": before})
|
||||
if not dry_run and not upload_allowed():
|
||||
if not dry_run and not upload_allowed(kind):
|
||||
if kind == "testing":
|
||||
raise HubError("closed", "Test uploads aren't open yet: the testing text is a draft waiting for the "
|
||||
f"maintainer. (For a rehearsal against a test repo: {ALLOW_ENV}=1.)")
|
||||
raise HubError("closed", "Contributions aren't open yet: the texts are drafts waiting for a legal review. "
|
||||
f"(For a rehearsal against a test repo: {ALLOW_ENV}=1.)")
|
||||
path_in_repo = f"contributions/{contributor}/{session}"
|
||||
@@ -327,59 +273,10 @@ def upload(store, session, dry_run=False, again=False, progress=None, log=None):
|
||||
tell("done", "Dry run: nothing was uploaded", 1.0)
|
||||
return dict(plan, dry_run=True, pr_url="", record=record)
|
||||
|
||||
hf = _hf()
|
||||
api = hf.HfApi()
|
||||
tell("login", "Checking your Hugging Face login", None)
|
||||
try:
|
||||
who = api.whoami()
|
||||
except Exception as e:
|
||||
raise explain(e, repo) from None
|
||||
role = ((who.get("auth") or {}).get("accessToken") or {}).get("role", "")
|
||||
if role == "read":
|
||||
raise HubError("read-token", "Your saved login is a read-only token, so it can't open a pull request. "
|
||||
"Log in again.")
|
||||
tell("access", f"Checking access to {repo}", None)
|
||||
try:
|
||||
api.auth_check(repo, repo_type="dataset")
|
||||
except Exception as e:
|
||||
raise explain(e, repo) from None
|
||||
# The pull request first, so its link shows while the files go (and the person can plug the
|
||||
# headset in and leave it). A retry of this export goes on in its pull request if it's open.
|
||||
tell("open", "Opening your pull request", None)
|
||||
pr = None
|
||||
before = unfinished_upload(store, session, sha, repo)
|
||||
if before:
|
||||
try:
|
||||
d = api.get_discussion_details(repo, int(before["pr_num"]), repo_type="dataset")
|
||||
if d.is_pull_request and d.status in ("draft", "open"):
|
||||
pr = d
|
||||
except Exception:
|
||||
pr = None
|
||||
if pr is None:
|
||||
try:
|
||||
pr = api.create_pull_request(repo, message, description=description, repo_type="dataset")
|
||||
except Exception as e:
|
||||
raise explain(e, repo) from None
|
||||
pr_url = getattr(pr, "url", "") or f"{dataset_url(repo)}/discussions/{pr.num}"
|
||||
record = {"repo": repo, "pr_url": pr_url, "pr_num": pr.num, "started": now_iso(), "export_sha": sha,
|
||||
"status": "started"}
|
||||
record_upload(store, session, record)
|
||||
tell("opened", f"Pull request #{pr.num} is open. Uploading {len(files)} files ({takes.human_bytes(report.bytes)})",
|
||||
None, pr_url=pr_url)
|
||||
try:
|
||||
api.upload_folder(repo_id=repo, repo_type="dataset", folder_path=export, path_in_repo=path_in_repo,
|
||||
revision=f"refs/pr/{pr.num}", commit_message=message, commit_description=description)
|
||||
except Exception as e:
|
||||
raise explain(e, repo) from None
|
||||
try: # a pull request opened through the API stays a draft until it's marked open
|
||||
if api.get_discussion_details(repo, pr.num, repo_type="dataset").status == "draft":
|
||||
api.change_discussion_status(repo, pr.num, "open", repo_type="dataset")
|
||||
except Exception:
|
||||
pass # the maintainer can open it
|
||||
record = dict(record, uploaded=now_iso(), status="done")
|
||||
record_upload(store, session, record)
|
||||
pr, who, record = rhub.push(_hf(), repo, export, path_in_repo, message, description, meta_path(store, session), sha,
|
||||
len(files), report.bytes, tell, takes.write_json, takes.human_bytes)
|
||||
tell("done", "Uploaded", 1.0)
|
||||
return dict(plan, dry_run=False, pr_url=pr_url, pr_num=pr.num, user=who.get("name", ""), record=record)
|
||||
return dict(plan, dry_run=False, pr_url=record["pr_url"], pr_num=pr.num, user=who.get("name", ""), record=record)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- the command line
|
||||
@@ -392,6 +289,8 @@ def main():
|
||||
w.add_argument("--json", action="store_true")
|
||||
li = sub.add_parser("login", help="log in to Hugging Face in the browser")
|
||||
li.add_argument("--json", action="store_true")
|
||||
b = sub.add_parser("board", help="fetch the dataset's bounty board")
|
||||
b.add_argument("--json", action="store_true")
|
||||
u = sub.add_parser("upload", help="upload exports/SESSION as a pull request")
|
||||
u.add_argument("session")
|
||||
u.add_argument("--dry-run", action="store_true", help="no network: say what would be uploaded")
|
||||
@@ -410,6 +309,13 @@ def main():
|
||||
else:
|
||||
print(f"logged in as {who['name']} (token role: {who['role'] or 'unknown'})")
|
||||
return 0
|
||||
if a.cmd == "board":
|
||||
got = fetch_board(os.path.abspath(os.path.expanduser(a.base)))
|
||||
if a.json:
|
||||
emit({"done": got})
|
||||
else:
|
||||
print(f"bounty board of {got['updated']}: {got['bounties']} bounties")
|
||||
return 0
|
||||
if a.cmd == "login":
|
||||
def code(phase, text, **extra):
|
||||
if a.json:
|
||||
|
||||
+13
-7
@@ -6,6 +6,8 @@
|
||||
# "Frametop Hand Recorder" to the app menu (for Frametop's desktop: it isn't tested from SteamVR's
|
||||
# "Launch a program", which runs apps outside it; the standalone recorder is for that).
|
||||
# It doesn't turn on Frametop's live hand tracking (that's hands/run.sh install, which install.sh offers).
|
||||
# A release (FRAME_RELEASE) brings the three built in its image (pack/Containerfile), so there it only
|
||||
# sets the capabilities and adds the menu entry; its install.sh offers this.
|
||||
# Usage: hands/rec/install.sh install or update
|
||||
# hands/rec/install.sh uninstall remove the menu entry, and ft-camd's capabilities unless
|
||||
# hand tracking's services use them (hands/run.sh uncaps).
|
||||
@@ -17,13 +19,17 @@ entry='~/.local/share/applications/frametop-handrec.desktop'
|
||||
|
||||
case ${1:-install} in
|
||||
install)
|
||||
echo "== 1/4 dev container packages"
|
||||
"$root/setup/dev-container.sh"
|
||||
echo "== 2/4 hand tracking: ft-camd and ft-hands"
|
||||
"$root/hands/build.sh"
|
||||
echo "== 3/4 the headset panel: ft-handpanel"
|
||||
"$root/hands/rec/build.sh"
|
||||
echo "== 4/4 ft-camd's capabilities (asks for your password) and the menu entry"
|
||||
if [ "$FRAME_RELEASE" = 1 ]; then
|
||||
echo "== ft-camd's capabilities (asks for your password, unless it has them) and the menu entry"
|
||||
else
|
||||
echo "== 1/4 dev container packages"
|
||||
"$root/setup/dev-container.sh"
|
||||
echo "== 2/4 hand tracking: ft-camd and ft-hands"
|
||||
"$root/hands/build.sh"
|
||||
echo "== 3/4 the headset panel: ft-handpanel"
|
||||
"$root/hands/rec/build.sh"
|
||||
echo "== 4/4 ft-camd's capabilities (asks for your password) and the menu entry"
|
||||
fi
|
||||
"$root/hands/run.sh" caps
|
||||
"$root/scripts/conf-migrate.sh" # HANDS_SWAP_SIDES=0, the old default, becomes auto
|
||||
fill_template "$root/hands/rec/ft-handrec.desktop" |
|
||||
|
||||
@@ -0,0 +1,115 @@
|
||||
#!/usr/bin/env python3
|
||||
"""The live tracker's output during a debug session (DESIGN.md, "Debug sessions"): each new
|
||||
publish of Frametop's hands and gestures files (include/fh_hands.h, fh_gestures.h), copied whole
|
||||
under their sequence lock into the take's live.bin. Replaying the take's sets through a newer
|
||||
ft-hands and comparing with live.bin shows what changed for that person.
|
||||
|
||||
live.bin: a 16-byte header, then records.
|
||||
header: magic "FHLIVE01", u32 version (1), u32 0
|
||||
record: u8 source (0 hands, 1 gestures), 3 pad, u32 length, u64 read_ns, u64 seq, then the
|
||||
file's bytes (length). read_ns: CLOCK_MONOTONIC when copied; the files' own
|
||||
capture_ns and publish_ns are inside the copy.
|
||||
|
||||
Standard library only.
|
||||
"""
|
||||
import os
|
||||
import struct
|
||||
import threading
|
||||
|
||||
from recorder.common import mono_ns
|
||||
|
||||
MAGIC = b"FHLIVE01"
|
||||
VERSION = 1
|
||||
FILE_HDR = struct.Struct("<8sII") # 16 bytes
|
||||
REC_HDR = struct.Struct("<B3xIQQ") # 24 bytes
|
||||
SOURCES = (("hands", b"FHHANDS1"), ("gestures", b"FHGEST01"))
|
||||
SEQ_AT = 16 # u64 seq after magic, version and size in both files
|
||||
POLL_S = 0.004 # the tracker publishes about 30 times a second
|
||||
MAX_BYTES = 1 << 16 # a file claiming more than this is not one of ours
|
||||
|
||||
|
||||
def snapshot(fd, magic):
|
||||
"""(seq, bytes) of a consistent copy, or None (no file yet, being written, or not ours)."""
|
||||
for _ in range(4):
|
||||
head = os.pread(fd, 24, 0)
|
||||
if len(head) < 24 or head[:8] != magic:
|
||||
return None
|
||||
_, _, size, seq = struct.unpack_from("<8sIIQ", head, 0)
|
||||
if seq % 2:
|
||||
continue
|
||||
if not 24 <= size <= MAX_BYTES:
|
||||
return None
|
||||
data = os.pread(fd, size, 0)
|
||||
if len(data) == size and os.pread(fd, 8, SEQ_AT) == head[SEQ_AT:24]:
|
||||
return seq, data
|
||||
return None
|
||||
|
||||
|
||||
class LiveLog:
|
||||
"""A thread copying run_dir()'s hands and gestures files into path as they change. stop()
|
||||
ends it and returns {"hands": n, "gestures": n} records written."""
|
||||
|
||||
def __init__(self, run_dir, path, poll_s=POLL_S):
|
||||
self.run_dir, self.path, self.poll_s = run_dir, path, poll_s
|
||||
self.counts = {name: 0 for name, _ in SOURCES}
|
||||
self._stop = threading.Event()
|
||||
self._out = open(path, "wb")
|
||||
self._out.write(FILE_HDR.pack(MAGIC, VERSION, 0))
|
||||
self._thread = threading.Thread(target=self._run, name="handrec-livelog", daemon=True)
|
||||
|
||||
def start(self):
|
||||
self._thread.start()
|
||||
return self
|
||||
|
||||
def _run(self):
|
||||
fds = [None] * len(SOURCES)
|
||||
last = [None] * len(SOURCES)
|
||||
try:
|
||||
while not self._stop.is_set():
|
||||
for k, (name, magic) in enumerate(SOURCES):
|
||||
if fds[k] is None:
|
||||
try:
|
||||
fds[k] = os.open(os.path.join(self.run_dir, name), os.O_RDONLY)
|
||||
except OSError:
|
||||
continue
|
||||
try:
|
||||
snap = snapshot(fds[k], magic)
|
||||
except OSError:
|
||||
snap = None
|
||||
if snap is None or snap[0] == last[k]:
|
||||
continue
|
||||
last[k] = snap[0]
|
||||
self._out.write(REC_HDR.pack(k, len(snap[1]), mono_ns(), snap[0]))
|
||||
self._out.write(snap[1])
|
||||
self.counts[name] += 1
|
||||
self._stop.wait(self.poll_s)
|
||||
finally:
|
||||
for fd in fds:
|
||||
if fd is not None:
|
||||
os.close(fd)
|
||||
|
||||
def stop(self):
|
||||
self._stop.set()
|
||||
if self._thread.is_alive():
|
||||
self._thread.join(5)
|
||||
if not self._out.closed:
|
||||
self._out.close()
|
||||
return dict(self.counts)
|
||||
|
||||
|
||||
def read(path):
|
||||
"""The records of a live.bin: [(source name, read_ns, seq, bytes)]. Stops at a cut-off end."""
|
||||
out = []
|
||||
with open(path, "rb") as f:
|
||||
magic, version, _ = FILE_HDR.unpack(f.read(FILE_HDR.size))
|
||||
if magic != MAGIC or version != VERSION:
|
||||
raise ValueError("%s: not a live.bin" % path)
|
||||
while True:
|
||||
head = f.read(REC_HDR.size)
|
||||
if len(head) < REC_HDR.size:
|
||||
return out
|
||||
src, n, read_ns, seq = REC_HDR.unpack(head)
|
||||
data = f.read(n)
|
||||
if len(data) < n or src >= len(SOURCES):
|
||||
return out
|
||||
out.append((SOURCES[src][0], read_ns, seq, data))
|
||||
+362
-54
@@ -13,6 +13,13 @@ Kirigami.ApplicationWindow {
|
||||
|
||||
// The session chosen on the Export page, for the Upload page.
|
||||
property string chosenSession: ""
|
||||
// The round picked on the Bounties page: a bounty's id, or "" for a full session or (quick)
|
||||
// a quick round. roundPicked: something was picked (else Welcome's Continue goes to Bounties).
|
||||
property string chosenBounty: ""
|
||||
property bool chosenQuick: false
|
||||
property bool roundPicked: false
|
||||
// "Export and upload" on the Session page: once the export is done, the Upload page.
|
||||
property bool sendAfterExport: false
|
||||
|
||||
globalDrawer: Kirigami.GlobalDrawer {
|
||||
isMenu: false
|
||||
@@ -20,6 +27,7 @@ Kirigami.ApplicationWindow {
|
||||
collapsible: true
|
||||
collapsed: root.width < Kirigami.Units.gridUnit * 34
|
||||
actions: [
|
||||
Kirigami.Action { text: "Bounties"; icon.name: "view-list-details"; onTriggered: root.show(bountiesPage) },
|
||||
Kirigami.Action { text: "Welcome"; icon.name: "help-about"; onTriggered: root.show(welcomePage) },
|
||||
Kirigami.Action {
|
||||
text: "Before you start"; icon.name: "view-task"; enabled: !backend.needsConsent
|
||||
@@ -82,17 +90,37 @@ Kirigami.ApplicationWindow {
|
||||
pageStack.push(page)
|
||||
}
|
||||
|
||||
// --page welcome|checklist|session|review|export|upload opens the app on that page; the
|
||||
// Welcome page comes first until the consent is agreed to.
|
||||
pageStack.initialPage: backend.needsConsent && startPage !== "review" ? welcomePage
|
||||
: (({ welcome: welcomePage, checklist: checklistPage, session: sessionPage, review: reviewPage,
|
||||
export: exportPage, upload: uploadPage })[startPage] || checklistPage)
|
||||
// A round picked on the Bounties page: the consent first if it isn't agreed to yet, then
|
||||
// Before you start (the camera check, the privacy checks), then Start.
|
||||
// The consent asked for follows the round: the testing session's is the testing one (TESTING.md),
|
||||
// a bounty's or a full session's the dataset's (CONSENT.md; a bounty's was the testing one
|
||||
// during the bounties' test phase, takes.TEST_PHASE).
|
||||
function pickRound(bounty, quick) {
|
||||
chosenBounty = bounty
|
||||
chosenQuick = quick
|
||||
roundPicked = true
|
||||
backend.setConsentKind(backend.roundConsentKind(bounty))
|
||||
show(backend.needsConsent ? welcomePage : checklistPage)
|
||||
}
|
||||
|
||||
// --page bounties|welcome|checklist|session|review|export|upload opens the app on that page;
|
||||
// it opens on Bounties. Pages that record or send need the consent: they lead to Welcome.
|
||||
pageStack.initialPage: (({ welcome: welcomePage, checklist: backend.needsConsent ? welcomePage : checklistPage,
|
||||
session: sessionPage, review: reviewPage, export: exportPage,
|
||||
upload: uploadPage })[startPage] || bountiesPage)
|
||||
|
||||
Connections {
|
||||
target: backend
|
||||
function onMessage(text, isError) {
|
||||
root.showPassiveNotification(text, isError ? "long" : "short")
|
||||
}
|
||||
function onExportDone(session, ok) {
|
||||
if (root.sendAfterExport && session === root.chosenSession) {
|
||||
root.sendAfterExport = false
|
||||
if (ok)
|
||||
root.show(uploadPage)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Asks before something is deleted for good: ask(title, text, function).
|
||||
@@ -159,6 +187,149 @@ Kirigami.ApplicationWindow {
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- Bounties
|
||||
Component {
|
||||
id: bountiesPage
|
||||
Kirigami.ScrollablePage {
|
||||
id: board
|
||||
title: "Bounties"
|
||||
actions: [
|
||||
Kirigami.Action {
|
||||
text: "Get the newest board"
|
||||
icon.name: "view-refresh"
|
||||
enabled: backend.boardFetch.state !== "fetching" && !backend.hubDryRun
|
||||
onTriggered: backend.refreshBoard()
|
||||
}
|
||||
]
|
||||
|
||||
header: ColumnLayout {
|
||||
spacing: 0
|
||||
DraftBanner { Layout.fillWidth: true }
|
||||
Kirigami.InlineMessage {
|
||||
Layout.fillWidth: true
|
||||
visible: backend.runnerError !== ""
|
||||
position: Kirigami.InlineMessage.Position.Header
|
||||
type: Kirigami.MessageType.Error
|
||||
text: backend.runnerError
|
||||
}
|
||||
Kirigami.InlineMessage {
|
||||
Layout.fillWidth: true
|
||||
visible: backend.sessionActive
|
||||
position: Kirigami.InlineMessage.Position.Header
|
||||
type: Kirigami.MessageType.Information
|
||||
text: "A session is running."
|
||||
actions: [ Kirigami.Action { text: "Go to it"; onTriggered: root.show(sessionPage) } ]
|
||||
}
|
||||
Kirigami.InlineMessage {
|
||||
Layout.fillWidth: true
|
||||
visible: backend.boardFetch.state === "error" || backend.boardFetch.state === "ok"
|
||||
position: Kirigami.InlineMessage.Position.Header
|
||||
type: backend.boardFetch.state === "ok" ? Kirigami.MessageType.Positive : Kirigami.MessageType.Information
|
||||
text: backend.boardFetch.text || ""
|
||||
}
|
||||
}
|
||||
|
||||
ListView {
|
||||
model: backend.bounties
|
||||
spacing: Kirigami.Units.smallSpacing
|
||||
|
||||
header: Controls.Label {
|
||||
width: ListView.view.width
|
||||
padding: Kirigami.Units.largeSpacing
|
||||
wrapMode: Text.Wrap
|
||||
text: "Each bounty is a short round, about 4 minutes, of what the hand tracker needs most. "
|
||||
+ "Windows appear in front of you in the headset and you tap, pinch and grab them as you "
|
||||
+ "would real ones, while the cameras record your hands. The most wanted come first, and "
|
||||
+ "you can do one again. Afterwards you look over the recording and send it "
|
||||
+ (backend.roundConsentKind("bounty") === "testing"
|
||||
? "in: while the bounties are being tested, to a test dataset, for testing only."
|
||||
: "to the hand dataset.") + (backend.boardUpdated ? " (Board of " + backend.boardUpdated + ".)" : "")
|
||||
}
|
||||
|
||||
Kirigami.PlaceholderMessage {
|
||||
anchors.centerIn: parent
|
||||
width: parent.width - Kirigami.Units.gridUnit * 4
|
||||
visible: parent.count === 0
|
||||
text: "No bounties"
|
||||
explanation: "The board that came with the recorder is missing (hands/rec/bounties.json)."
|
||||
}
|
||||
|
||||
delegate: Controls.ItemDelegate {
|
||||
id: bountyRow
|
||||
required property var modelData
|
||||
width: ListView.view.width
|
||||
enabled: modelData.available === ""
|
||||
onClicked: root.pickRound(modelData.id, false)
|
||||
|
||||
contentItem: RowLayout {
|
||||
spacing: Kirigami.Units.largeSpacing
|
||||
ColumnLayout {
|
||||
Layout.fillWidth: true
|
||||
spacing: Kirigami.Units.smallSpacing
|
||||
Controls.Label {
|
||||
Layout.fillWidth: true
|
||||
wrapMode: Text.Wrap
|
||||
font.bold: true
|
||||
text: bountyRow.modelData.title
|
||||
+ (bountyRow.modelData.priority === "high" ? " · wanted most"
|
||||
: bountyRow.modelData.priority === "low" ? " · low priority" : "")
|
||||
+ (bountyRow.modelData.done === 1 ? " · done"
|
||||
: bountyRow.modelData.done > 1 ? " · done " + bountyRow.modelData.done + " times" : "")
|
||||
}
|
||||
Controls.Label {
|
||||
Layout.fillWidth: true
|
||||
wrapMode: Text.Wrap
|
||||
text: bountyRow.modelData.blurb
|
||||
}
|
||||
Controls.Label {
|
||||
Layout.fillWidth: true
|
||||
elide: Text.ElideRight
|
||||
opacity: 0.7
|
||||
font: Kirigami.Theme.smallFont
|
||||
text: [bountyRow.modelData.minutesText,
|
||||
bountyRow.modelData.lastText ? "last done " + bountyRow.modelData.lastText : "",
|
||||
bountyRow.modelData.available].filter(s => s !== "").join(" · ")
|
||||
}
|
||||
}
|
||||
Controls.Button {
|
||||
text: bountyRow.modelData.done > 0 ? "Record again" : "Record"
|
||||
icon.name: "media-record"
|
||||
enabled: bountyRow.modelData.available === "" && !backend.sessionActive
|
||||
onClicked: bountyRow.clicked()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
footer: ColumnLayout {
|
||||
width: ListView.view.width
|
||||
spacing: Kirigami.Units.smallSpacing
|
||||
Kirigami.Separator { Layout.fillWidth: true; Layout.topMargin: Kirigami.Units.largeSpacing }
|
||||
Controls.Label {
|
||||
Layout.fillWidth: true
|
||||
Layout.margins: Kirigami.Units.largeSpacing
|
||||
wrapMode: Text.Wrap
|
||||
text: "Or a longer round of held poses, gestures and the things on your desk: the full "
|
||||
+ "session (about 15 minutes), or a quick round (about 3) in a different light once "
|
||||
+ "you've done a full one."
|
||||
}
|
||||
RowLayout {
|
||||
Layout.leftMargin: Kirigami.Units.largeSpacing
|
||||
Controls.Button {
|
||||
text: "Full session"
|
||||
enabled: !backend.sessionActive
|
||||
onClicked: root.pickRound("", false)
|
||||
}
|
||||
Controls.Button {
|
||||
text: "Quick round"
|
||||
enabled: !backend.sessionActive
|
||||
onClicked: root.pickRound("", true)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- Welcome
|
||||
Component {
|
||||
id: welcomePage
|
||||
@@ -191,7 +362,7 @@ Kirigami.ApplicationWindow {
|
||||
}
|
||||
Kirigami.InlineMessage {
|
||||
Layout.fillWidth: true
|
||||
visible: backend.needsConsent && backend.contributor !== ""
|
||||
visible: backend.needsConsent && backend.consentAccepted !== ""
|
||||
type: Kirigami.MessageType.Information
|
||||
text: "This text changed since you last agreed to it. Please read it again; your contributor id stays the same."
|
||||
}
|
||||
@@ -230,14 +401,14 @@ Kirigami.ApplicationWindow {
|
||||
onClicked: {
|
||||
backend.acceptConsent(adult.checked, region.checked, agree.checked, handed.currentValue)
|
||||
if (!backend.needsConsent)
|
||||
root.show(checklistPage)
|
||||
root.show(root.roundPicked ? checklistPage : bountiesPage)
|
||||
}
|
||||
}
|
||||
Controls.Button {
|
||||
visible: !backend.needsConsent
|
||||
text: "Continue"
|
||||
icon.name: "go-next"
|
||||
onClicked: root.show(checklistPage)
|
||||
onClicked: root.show(root.roundPicked ? checklistPage : bountiesPage)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -250,25 +421,35 @@ Kirigami.ApplicationWindow {
|
||||
id: checklist
|
||||
title: "Before you start"
|
||||
readonly property bool privacyOk: privacy1.checked && privacy2.checked && privacy3.checked
|
||||
// The round: a bounty's id (picked on the Bounties page), or "" for a full session or a quick round.
|
||||
readonly property string bountyId: root.chosenBounty
|
||||
readonly property bool isBounty: bountyId !== ""
|
||||
readonly property var bounty: {
|
||||
for (const b of backend.bounties)
|
||||
if (b.id === bountyId)
|
||||
return b
|
||||
return null
|
||||
}
|
||||
readonly property bool ready: privacyOk && backend.diskOk
|
||||
&& backend.runnerError === "" && !backend.sessionActive
|
||||
|
||||
function answers() {
|
||||
const objects = []
|
||||
for (let i = 0; i < objectBoxes.count; ++i) {
|
||||
for (let i = 0; i < objectBoxes.count && !isBounty; ++i) {
|
||||
const box = objectBoxes.itemAt(i)
|
||||
if (box.checked)
|
||||
objects.push(box.value)
|
||||
}
|
||||
return {
|
||||
objects: objects,
|
||||
own_objects: ownObjects.text.split(",").map(s => s.trim()).filter(s => s !== ""),
|
||||
controllers: straps.checked ? "straps" : "none",
|
||||
own_objects: isBounty ? [] : ownObjects.text.split(",").map(s => s.trim()).filter(s => s !== ""),
|
||||
controllers: straps.checked && !isBounty ? "straps" : "none",
|
||||
sleeves: sleeves.currentValue,
|
||||
rings: rings.checked,
|
||||
watch: watch.checked,
|
||||
notes: notes.text,
|
||||
privacy: privacyOk
|
||||
privacy: privacyOk,
|
||||
voice: bountyId === "debug" && voice.checked
|
||||
}
|
||||
}
|
||||
|
||||
@@ -298,8 +479,60 @@ Kirigami.ApplicationWindow {
|
||||
Component.onCompleted: backend.checkCameras()
|
||||
|
||||
Kirigami.FormLayout {
|
||||
Kirigami.Separator { Kirigami.FormData.isSection: true; Kirigami.FormData.label: "Within reach" }
|
||||
Kirigami.Separator { Kirigami.FormData.isSection: true; Kirigami.FormData.label: "Round" }
|
||||
ColumnLayout {
|
||||
Kirigami.FormData.label: checklist.isBounty ? "Bounty:" : "Round:"
|
||||
Layout.maximumWidth: Kirigami.Units.gridUnit * 26
|
||||
Controls.Label {
|
||||
Layout.fillWidth: true
|
||||
visible: checklist.isBounty
|
||||
wrapMode: Text.Wrap
|
||||
font.bold: true
|
||||
text: checklist.bounty ? checklist.bounty.title : checklist.bountyId
|
||||
}
|
||||
Controls.Label {
|
||||
Layout.fillWidth: true
|
||||
visible: checklist.isBounty
|
||||
wrapMode: Text.Wrap
|
||||
opacity: 0.7
|
||||
text: checklist.bounty ? checklist.bounty.blurb : "This recorder doesn't have that bounty."
|
||||
}
|
||||
Controls.RadioButton {
|
||||
id: fullRound
|
||||
visible: !checklist.isBounty
|
||||
text: "Full session"
|
||||
checked: !(root.roundPicked ? root.chosenQuick : backend.hasFullSession)
|
||||
}
|
||||
Controls.RadioButton {
|
||||
id: quickRound
|
||||
visible: !checklist.isBounty
|
||||
text: "Quick round (about 3 min): hand size, poses, the dot, no hands"
|
||||
checked: root.roundPicked ? root.chosenQuick : backend.hasFullSession
|
||||
}
|
||||
Controls.Button {
|
||||
text: checklist.isBounty ? "Pick another…" : "Pick a bounty instead…"
|
||||
icon.name: "view-list-details"
|
||||
flat: true
|
||||
enabled: !backend.sessionActive
|
||||
onClicked: root.show(bountiesPage)
|
||||
}
|
||||
}
|
||||
Kirigami.InlineMessage {
|
||||
Layout.maximumWidth: Kirigami.Units.gridUnit * 26
|
||||
Layout.fillWidth: true
|
||||
visible: backend.hasFullSession && !checklist.isBounty
|
||||
Layout.preferredHeight: visible ? implicitHeight : 0
|
||||
type: Kirigami.MessageType.Information
|
||||
text: "You've recorded a full session already. A quick round in a different light (dim, "
|
||||
+ "daylight), or a bounty, adds the most now."
|
||||
}
|
||||
|
||||
Kirigami.Separator {
|
||||
visible: !checklist.isBounty
|
||||
Kirigami.FormData.isSection: true; Kirigami.FormData.label: "Within reach"
|
||||
}
|
||||
Controls.Label {
|
||||
visible: !checklist.isBounty
|
||||
Kirigami.FormData.label: ""
|
||||
Layout.maximumWidth: Kirigami.Units.gridUnit * 26
|
||||
wrapMode: Text.Wrap
|
||||
@@ -313,6 +546,7 @@ Kirigami.ApplicationWindow {
|
||||
required property var modelData
|
||||
required property int index
|
||||
readonly property string value: modelData.value
|
||||
visible: !checklist.isBounty
|
||||
Kirigami.FormData.label: index === 0 ? "Objects:" : ""
|
||||
text: modelData.text
|
||||
checked: true
|
||||
@@ -320,18 +554,24 @@ Kirigami.ApplicationWindow {
|
||||
}
|
||||
Controls.TextField {
|
||||
id: ownObjects
|
||||
visible: !checklist.isBounty
|
||||
Kirigami.FormData.label: "Your own:"
|
||||
Layout.preferredWidth: Kirigami.Units.gridUnit * 20
|
||||
placeholderText: "other things you use, separated by commas"
|
||||
}
|
||||
|
||||
Kirigami.Separator { Kirigami.FormData.isSection: true; Kirigami.FormData.label: "Controllers" }
|
||||
Kirigami.Separator {
|
||||
visible: !checklist.isBounty
|
||||
Kirigami.FormData.isSection: true; Kirigami.FormData.label: "Controllers"
|
||||
}
|
||||
ColumnLayout {
|
||||
visible: !checklist.isBounty
|
||||
Kirigami.FormData.label: "Ergonomic Kit straps:"
|
||||
Controls.RadioButton { id: straps; text: "Yes, I have the controllers with the straps" }
|
||||
Controls.RadioButton { text: "No"; checked: true }
|
||||
}
|
||||
Controls.Label {
|
||||
visible: !checklist.isBounty
|
||||
Layout.maximumWidth: Kirigami.Units.gridUnit * 26
|
||||
wrapMode: Text.Wrap
|
||||
opacity: 0.7
|
||||
@@ -421,7 +661,7 @@ Kirigami.ApplicationWindow {
|
||||
Kirigami.Separator { Kirigami.FormData.isSection: true; Kirigami.FormData.label: "Space" }
|
||||
Controls.Label {
|
||||
Kirigami.FormData.label: "Free space:"
|
||||
text: backend.freeText + " (a round takes about 10 GB)"
|
||||
text: backend.freeText + " (a round takes about 15 GB, and its export about 12 GB more)"
|
||||
}
|
||||
Kirigami.InlineMessage {
|
||||
Layout.maximumWidth: Kirigami.Units.gridUnit * 26
|
||||
@@ -475,11 +715,54 @@ Kirigami.ApplicationWindow {
|
||||
text: backend.cameraSummary + "\n" + backend.cameraEvidence
|
||||
}
|
||||
|
||||
Kirigami.Separator {
|
||||
visible: checklist.bountyId === "debug"
|
||||
Kirigami.FormData.isSection: true
|
||||
Kirigami.FormData.label: "Your voice"
|
||||
}
|
||||
Controls.CheckBox {
|
||||
id: voice
|
||||
visible: checklist.bountyId === "debug"
|
||||
text: "Record my voice (optional)"
|
||||
}
|
||||
Controls.Label {
|
||||
visible: checklist.bountyId === "debug"
|
||||
Layout.maximumWidth: Kirigami.Units.gridUnit * 26
|
||||
wrapMode: Text.Wrap
|
||||
opacity: 0.7
|
||||
text: "The headset's microphone records while you do: say what went wrong as it happens, e.g. "
|
||||
+ "\"that opened the lazy susan, I didn't mean to\". We turn what you say into notes at "
|
||||
+ "those moments. It's sent encrypted with the rest, for testing only. Leave it off if "
|
||||
+ "other people can be heard."
|
||||
}
|
||||
|
||||
Kirigami.Separator { Kirigami.FormData.isSection: true; Kirigami.FormData.label: "What will happen" }
|
||||
Controls.Label {
|
||||
Layout.maximumWidth: Kirigami.Units.gridUnit * 26
|
||||
wrapMode: Text.Wrap
|
||||
text: "Press Start, then put the headset on. A panel in the headset shows each step: a picture "
|
||||
text: checklist.bountyId === "debug"
|
||||
? "Press Start, then put the headset on. Press the headset button to start recording: "
|
||||
+ "after a 3-2-1 the panel goes away, and you use Frametop as you always do for up to 3 "
|
||||
+ "minutes: point, tap, pinch, drag, scroll. Do what gives you trouble. Frametop's hand "
|
||||
+ "tracking stays on, and what it made of your hands is recorded with the cameras.\n\n"
|
||||
+ "When the tracking gets something wrong, press the headset button at once: that marks "
|
||||
+ "the moment (the panel flashes \"Marked\"). Hold the button to finish early."
|
||||
+ (voice.checked ? " Say what went wrong too: the microphone is recording." : "") + "\n\n"
|
||||
+ "Nothing leaves the headset. Afterwards you watch it in Review, delete anything you "
|
||||
+ "don't want to share, and only then export. It's sent encrypted, for testing only."
|
||||
: checklist.bountyId !== ""
|
||||
? "Press Start, then put the headset on. After a few seconds of your hands held up, windows "
|
||||
+ "appear in front of you, and a band along the top says what to do. Do it as you would "
|
||||
+ "on a real window: tap the lit button, pinch the lit corner, grab the lit edge. The "
|
||||
+ "windows don't move or click; a ghost outline shows where to take one, and your hand "
|
||||
+ "goes there as if it came along. Some steps ask you to stop short or touch without "
|
||||
+ "grabbing: those count just as much.\n\n"
|
||||
+ "Each step waits until you're ready: press the button on the right side of the headset, "
|
||||
+ "or Space or Next in this window. The windows appear where you're facing when you press "
|
||||
+ "it. Press it twice to record a step again, or hold it to stop.\n\n"
|
||||
+ "Nothing leaves the headset. Afterwards you watch the takes in Review, delete anything "
|
||||
+ "you don't want to share, and only then export."
|
||||
: "Press Start, then put the headset on. A panel in the headset shows each step: a picture "
|
||||
+ "of the hand pose, where to hold your hands, and what to do. The sections: hand poses, "
|
||||
+ "gestures, typing and the mouse, your objects, touching a dot, and moves with the "
|
||||
+ "controllers on and off. Each section is recorded as one take. In the pose steps your "
|
||||
@@ -493,28 +776,6 @@ Kirigami.ApplicationWindow {
|
||||
+ "Nothing leaves the headset. Afterwards you watch the takes in Review, delete anything "
|
||||
+ "you don't want to share, and only then export."
|
||||
}
|
||||
ColumnLayout {
|
||||
Kirigami.FormData.label: "Round:"
|
||||
Controls.RadioButton {
|
||||
id: fullRound
|
||||
text: "Full session"
|
||||
checked: !backend.hasFullSession
|
||||
}
|
||||
Controls.RadioButton {
|
||||
id: quickRound
|
||||
text: "Quick round (about 3 min): hand size, poses, the dot, no hands"
|
||||
checked: backend.hasFullSession
|
||||
}
|
||||
}
|
||||
Kirigami.InlineMessage {
|
||||
Layout.maximumWidth: Kirigami.Units.gridUnit * 26
|
||||
Layout.fillWidth: true
|
||||
visible: backend.hasFullSession
|
||||
Layout.preferredHeight: visible ? implicitHeight : 0
|
||||
type: Kirigami.MessageType.Information
|
||||
text: "You've recorded a full session already. A quick round in a different light (dim, "
|
||||
+ "daylight) adds the most now."
|
||||
}
|
||||
Controls.CheckBox {
|
||||
id: autoAdvance
|
||||
Kirigami.FormData.label: "Pace:"
|
||||
@@ -527,11 +788,12 @@ Kirigami.ApplicationWindow {
|
||||
text: (autoAdvance.checked
|
||||
? "Each step shows for a few seconds and the next follows by itself. Quicker if you "
|
||||
+ "know the steps already. "
|
||||
: "") + "Length: " + backend.planText(checklist.answers(), autoAdvance.checked, quickRound.checked) + "."
|
||||
: "") + "Length: " + backend.planText(checklist.answers(), autoAdvance.checked, quickRound.checked,
|
||||
checklist.bountyId) + "."
|
||||
}
|
||||
ColumnLayout {
|
||||
Kirigami.FormData.label: "Guidance:"
|
||||
visible: backend.scriptHasDemos
|
||||
visible: backend.scriptHasDemos && checklist.bountyId === ""
|
||||
Repeater {
|
||||
model: backend.guidanceChoices
|
||||
Controls.RadioButton {
|
||||
@@ -550,7 +812,7 @@ Kirigami.ApplicationWindow {
|
||||
enabled: checklist.ready && !backend.camerasBlockStart && !backend.restartingSteamVR
|
||||
onClicked: {
|
||||
if (backend.startSession(checklist.answers(), lighting.currentValue, autoAdvance.checked,
|
||||
quickRound.checked))
|
||||
quickRound.checked, checklist.bountyId))
|
||||
root.show(sessionPage)
|
||||
}
|
||||
}
|
||||
@@ -606,6 +868,11 @@ Kirigami.ApplicationWindow {
|
||||
enabled: backend.sessionActive
|
||||
onActivated: backend.stopSession()
|
||||
}
|
||||
Shortcut {
|
||||
sequence: "M"
|
||||
enabled: backend.sessionActive && !!sessionView.st.debug
|
||||
onActivated: backend.markMoment()
|
||||
}
|
||||
|
||||
// A hand chip like the panel's: seen (green), lost (orange) or hidden.
|
||||
component HandChip: Controls.Label {
|
||||
@@ -802,10 +1069,10 @@ Kirigami.ApplicationWindow {
|
||||
visible: sessionView.state === ""
|
||||
icon.name: "media-record"
|
||||
text: "No session yet"
|
||||
explanation: "Go through Before you start, then press Start there."
|
||||
explanation: "Pick a bounty, go through Before you start, then press Start there."
|
||||
helpfulAction: Kirigami.Action {
|
||||
text: "Before you start"; icon.name: "view-task"
|
||||
onTriggered: root.show(checklistPage)
|
||||
text: "Bounties"; icon.name: "view-list-details"
|
||||
onTriggered: root.show(bountiesPage)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -992,7 +1259,13 @@ Kirigami.ApplicationWindow {
|
||||
Layout.fillWidth: true
|
||||
wrapMode: Text.Wrap
|
||||
opacity: 0.7
|
||||
text: "The instructions appear in the headset. "
|
||||
text: sessionView.st.debug
|
||||
? "The panel in the headset hides while you record. "
|
||||
+ (sessionView.st.button ? "The button on the right side of the headset: next; while "
|
||||
+ "recording, press it to mark a moment the tracking got wrong, hold it to finish. " : "")
|
||||
+ "While this window has focus: Space: next · M: mark · P: pause or resume · Esc: stop. "
|
||||
+ "Stopping keeps what's recorded so far."
|
||||
: "The instructions appear in the headset. "
|
||||
+ (sessionView.st.button ? "The button on the right side of the headset: "
|
||||
+ (sessionView.stepMode ? "next, " : "") + "pause or resume; twice: record "
|
||||
+ "the last step again; hold: stop. " : "")
|
||||
@@ -1006,9 +1279,9 @@ Kirigami.ApplicationWindow {
|
||||
Controls.Button {
|
||||
visible: !backend.sessionActive
|
||||
focusPolicy: Qt.NoFocus
|
||||
text: "New session…"
|
||||
text: "Another round…"
|
||||
icon.name: "media-record"
|
||||
onClicked: root.show(checklistPage)
|
||||
onClicked: root.show(bountiesPage)
|
||||
}
|
||||
Controls.Button {
|
||||
visible: backend.sessionActive
|
||||
@@ -1018,7 +1291,23 @@ Kirigami.ApplicationWindow {
|
||||
onClicked: backend.togglePause()
|
||||
}
|
||||
Controls.Button {
|
||||
visible: backend.sessionActive
|
||||
visible: backend.sessionActive && !!sessionView.st.debug
|
||||
enabled: sessionView.state === "running"
|
||||
focusPolicy: Qt.NoFocus
|
||||
text: "Mark (M)" + (sessionView.st.marks ? " · " + sessionView.st.marks : "")
|
||||
icon.name: "flag"
|
||||
onClicked: backend.markMoment()
|
||||
}
|
||||
Controls.Button {
|
||||
visible: backend.sessionActive && !!sessionView.st.debug
|
||||
enabled: sessionView.state === "running"
|
||||
focusPolicy: Qt.NoFocus
|
||||
text: "Finish"
|
||||
icon.name: "dialog-ok"
|
||||
onClicked: backend.finishSession()
|
||||
}
|
||||
Controls.Button {
|
||||
visible: backend.sessionActive && !sessionView.st.debug
|
||||
enabled: !!sessionView.st.can_redo
|
||||
focusPolicy: Qt.NoFocus
|
||||
text: "Redo step"
|
||||
@@ -1026,7 +1315,7 @@ Kirigami.ApplicationWindow {
|
||||
onClicked: backend.redo()
|
||||
}
|
||||
Controls.Button {
|
||||
visible: backend.sessionActive
|
||||
visible: backend.sessionActive && !sessionView.st.debug
|
||||
focusPolicy: Qt.NoFocus
|
||||
text: "Skip section"
|
||||
icon.name: "media-skip-forward"
|
||||
@@ -1039,6 +1328,18 @@ Kirigami.ApplicationWindow {
|
||||
icon.name: "media-playback-stop"
|
||||
onClicked: backend.stopSession()
|
||||
}
|
||||
Controls.Button {
|
||||
visible: !backend.sessionActive && backend.sessionId !== "" && !backend.exporting
|
||||
focusPolicy: Qt.NoFocus
|
||||
text: "Export and upload"
|
||||
icon.name: "cloud-upload"
|
||||
onClicked: {
|
||||
root.chosenSession = backend.sessionId
|
||||
root.sendAfterExport = true
|
||||
backend.exportSession(backend.sessionId, false)
|
||||
root.show(exportPage)
|
||||
}
|
||||
}
|
||||
Controls.Button {
|
||||
visible: !backend.sessionActive && backend.sessionId !== ""
|
||||
focusPolicy: Qt.NoFocus
|
||||
@@ -1432,6 +1733,13 @@ Kirigami.ApplicationWindow {
|
||||
header: ColumnLayout {
|
||||
spacing: 0
|
||||
DraftBanner { Layout.fillWidth: true }
|
||||
Kirigami.InlineMessage {
|
||||
Layout.fillWidth: true
|
||||
visible: root.sendAfterExport
|
||||
position: Kirigami.InlineMessage.Position.Header
|
||||
type: Kirigami.MessageType.Information
|
||||
text: "Exporting your session: the Upload page follows by itself once it's done."
|
||||
}
|
||||
Kirigami.InlineMessage {
|
||||
Layout.fillWidth: true
|
||||
visible: exportView.worn && backend.exporting
|
||||
@@ -1495,7 +1803,7 @@ Kirigami.ApplicationWindow {
|
||||
visible: backend.exporting
|
||||
text: "Cancel"
|
||||
icon.name: "dialog-cancel"
|
||||
onClicked: backend.cancelExport()
|
||||
onClicked: { root.sendAfterExport = false; backend.cancelExport() }
|
||||
}
|
||||
}
|
||||
// Drawn here: the desktop style's ProgressBar paints nothing without a QApplication.
|
||||
@@ -1574,7 +1882,7 @@ Kirigami.ApplicationWindow {
|
||||
if (backend.exporting) return "Wait for the export to finish"
|
||||
if (backend.uploading) return ""
|
||||
if (backend.hubDryRun) return ""
|
||||
if (!backend.uploadAllowed) return "Contributions aren't open yet"
|
||||
if (!backend.uploadAllowedFor(session)) return "Contributions aren't open yet"
|
||||
if (!loggedIn) return "Log in first (step 2)"
|
||||
return ""
|
||||
}
|
||||
@@ -1616,7 +1924,7 @@ Kirigami.ApplicationWindow {
|
||||
visible: backend.textsDraft && backend.allowUploadSet && !backend.hubDryRun
|
||||
position: Kirigami.InlineMessage.Position.Header
|
||||
type: Kirigami.MessageType.Warning
|
||||
text: "Rehearsal: FT_HANDREC_ALLOW_UPLOAD=1 allows uploads to " + backend.dataset
|
||||
text: "Rehearsal: FT_HANDREC_ALLOW_UPLOAD=1 allows uploads to " + backend.datasetFor(uploadView.session)
|
||||
+ " while the texts are drafts."
|
||||
}
|
||||
}
|
||||
@@ -1645,7 +1953,7 @@ Kirigami.ApplicationWindow {
|
||||
Layout.fillWidth: true
|
||||
wrapMode: Text.Wrap
|
||||
textFormat: Text.StyledText
|
||||
text: "Your export goes to <a href=\"" + backend.datasetUrl + "\">" + backend.dataset + "</a> as a "
|
||||
text: "Your export goes to <a href=\"" + backend.datasetUrlFor(uploadView.session) + "\">" + backend.datasetFor(uploadView.session) + "</a> as a "
|
||||
+ "pull request from your own Hugging Face account. Nothing is published until the "
|
||||
+ "maintainer has checked it."
|
||||
onLinkActivated: link => Qt.openUrlExternally(link)
|
||||
@@ -1774,7 +2082,7 @@ Kirigami.ApplicationWindow {
|
||||
Layout.fillWidth: true
|
||||
wrapMode: Text.Wrap
|
||||
textFormat: Text.StyledText
|
||||
text: "The first time, accept the dataset's terms on <a href=\"" + backend.datasetUrl
|
||||
text: "The first time, accept the dataset's terms on <a href=\"" + backend.datasetUrlFor(uploadView.session)
|
||||
+ "\">its page</a>. Upload checks the export, opens your pull request, then sends the files."
|
||||
onLinkActivated: link => Qt.openUrlExternally(link)
|
||||
}
|
||||
|
||||
@@ -17,6 +17,10 @@
|
||||
// text <text> the instruction: large, wrapped to the panel's width, centred
|
||||
// note <text> an orange warning line under the instruction (empty to clear)
|
||||
// countdown <0..1> | off a thin bar along the bottom: the share of the prompt's time left
|
||||
// split <at> <label>;... / split off
|
||||
// ticks on the countdown's bar where a step changes pose, each at
|
||||
// <at> (the share of time left then, 0..1) with "Switch to <label>"
|
||||
// above it; lit cyan once the bar has come down to it
|
||||
// hands <left> <right> the "Left hand" and "Right hand" chips: seen (green), lost
|
||||
// (orange) or off (hidden)
|
||||
// bar <target 0..1> <current 0..1|-1> [<near label>|<far label>] / bar off
|
||||
@@ -59,6 +63,28 @@
|
||||
// filling to the share given, done turns it green. "target off"
|
||||
// hides it and forgets the point. Reply: "ok <room x> <y> <z>"
|
||||
// (standing universe)
|
||||
// layout full|top full: the panel as above. top: only a band along its top (title,
|
||||
// step, Rec, big, two lines of text, a note, the time-left bar),
|
||||
// the rest clear, so a scene's windows below it can be seen
|
||||
// win <id> place <x> <y> <z> <w> <h> [<yaw> <pitch>] [<look>]
|
||||
// a scene's window (at most 4; id: a short name): w x h metres,
|
||||
// centred at x y z in the head frame made level (the head's forward
|
||||
// direction, flattened; +y up), facing the eye, turned by yaw and
|
||||
// pitch degrees. It stays where it was put in the room. look: plain
|
||||
// (faint text lines), grid:CxR (1-6 buttons each way) or list:N
|
||||
// (1-10 rows). Its own overlay, frametop.handpanel.win.<id>, made
|
||||
// at first use. Reply: "ok <12 floats>", its pose in the room
|
||||
// (standing universe, row-major 3x4)
|
||||
// win <id> lit <part>[+<part>...] [<colour>] [<label>]
|
||||
// light parts of it: none, bar (the title bar), left, right, top,
|
||||
// bottom (edges), tl, tr, bl, br (corners), button:N or item:N
|
||||
// (from 0). colour: cyan (default), purple, orange (a button drawn
|
||||
// hollow with a cross: don't touch) or green. label: a short mark
|
||||
// beside the first part ("_" for a space)
|
||||
// win <id> ghost <dx> <dy> <dz> <yaw> <pitch> <roll> [<scale>] / win <id> ghost off
|
||||
// a dashed outline of the window, moved and turned in the window's
|
||||
// own frame and scaled: where to take it. Reply: "ok <12 floats>"
|
||||
// win <id> off / win clear hide one window (and its ghost) / all of them
|
||||
// poses start <path> / poses stop
|
||||
// log poses, appending to path (poses.jsonl, DESIGN.md) at
|
||||
// 250 Hz until stopped. "poses stop" replies "ok <samples>"
|
||||
@@ -71,7 +97,7 @@
|
||||
// Each picture goes into the next of three shared buffers (linear DMA-BUFs SteamVR imported
|
||||
// once), and the overlay switches to it, as gaze/panel/ft-gazepanel.cpp does: SetOverlayRaw
|
||||
// flickered and lagged there, so it's only the fallback. The target has its own three small
|
||||
// buffers. Pictures are drawn only when a command changed something; while a demo clip plays,
|
||||
// buffers, and so does each window and ghost (made again only when its size changes). Pictures are drawn only when a command changed something; while a demo clip plays,
|
||||
// each of its frames redraws only the clip's rectangle, and copies only that to a buffer that
|
||||
// already has the rest.
|
||||
//
|
||||
@@ -82,8 +108,9 @@
|
||||
// Options: --watch-stdin (quit when stdin closes: the session runs it), --socket NAME,
|
||||
// --distance METRES, --up DEG and --width DEG (UP_DEG and WIDTH_DEG: raise it out of a
|
||||
// recorded headset view), --no-vr. --no-vr is for testing without a headset: no SteamVR at all; each
|
||||
// picture's text goes to stdout instead (and with --dump DIR, the pictures to DIR/panel.pam
|
||||
// and DIR/target.pam), "poses start" writes nothing, "devices" and "head" reply with an error,
|
||||
// picture's text goes to stdout instead (and with --dump DIR, the pictures to DIR/panel.pam,
|
||||
// DIR/target.pam, DIR/win-<id>.pam and DIR/ghost-<id>.pam), the windows are placed as if the
|
||||
// headset were at the room's origin, "poses start" writes nothing, "devices" and "head" reply with an error,
|
||||
// and "target" places the point as if the headset were at the room's origin.
|
||||
// Runs in the dev container (hands/rec/build.sh builds it into hands/rec/build).
|
||||
#include <openvr.h>
|
||||
@@ -555,11 +582,13 @@ struct Panel {
|
||||
Picture pic{kW, kH, {}};
|
||||
std::string title, step, text, note, action, big, keys;
|
||||
double countdown = -1; // the share of the prompt's time left, or -1: off
|
||||
std::vector<std::pair<double, std::string>> splits; // the countdown's ticks: share left, the pose after
|
||||
std::string hands[2] = {"off", "off"};
|
||||
bool barOn = false;
|
||||
double barTarget = 0, barCurrent = -1;
|
||||
std::string nearLabel = "Near", farLabel = "Far";
|
||||
bool paused = false, rec = false;
|
||||
bool compact = false; // layout top: only a band at the top, clear of a scene's windows
|
||||
Image image; // the pose picture, if image.path isn't empty
|
||||
std::string imageMode; // "", "mirror" or "both"
|
||||
std::string wherePos, whereDist; // the diagram's, "" for none
|
||||
@@ -572,6 +601,23 @@ struct Panel {
|
||||
double textDeg = 0;
|
||||
};
|
||||
|
||||
// The countdown's ticks (a step's pose changes): a line across the bar at each, "Switch to ..."
|
||||
// over it, kept inside the bar's span; cyan once the bar has come down to it. labelY is the
|
||||
// label's centre line.
|
||||
void DrawSplits(Picture &pic, const Panel &p, int x0, int width, int y0, int y1, int labelY, int size) {
|
||||
const double ppd = kPxPerDeg;
|
||||
for (const auto &[at, label] : p.splits) {
|
||||
const bool reached = p.countdown >= 0 && p.countdown <= at + 1e-6;
|
||||
const int x = x0 + int(std::lround(width * at)), lw = std::max(3, int(ppd * 0.12));
|
||||
Rect(pic, x - lw / 2, y0 - int(ppd * 0.3), x - lw / 2 + lw, y1 + int(ppd * 0.15), 1, 1, 1, reached ? 0.95 : 0.7);
|
||||
const std::string text = "Switch to " + label;
|
||||
const int w = TextWidth(text, size);
|
||||
const int cx = std::clamp(x, x0 + w / 2, std::max(x0 + w / 2, x0 + width - w / 2));
|
||||
if (reached) Text(pic, text, size, cx, labelY, kCyan[0], kCyan[1], kCyan[2]);
|
||||
else Text(pic, text, size, cx, labelY, 0.85);
|
||||
}
|
||||
}
|
||||
|
||||
void Draw(Panel &p) {
|
||||
// The dim rounded panel, see-through so it's clear of what's behind (the quick check's look,
|
||||
// with the same corner radius in degrees). It never changes, so it's drawn once.
|
||||
@@ -585,6 +631,47 @@ void Draw(Panel &p) {
|
||||
p.demoW = 0;
|
||||
const double ppd = kPxPerDeg, faint = 0.75;
|
||||
const int margin = int(ppd * 1.6), width = kW - 2 * margin;
|
||||
if (p.compact) {
|
||||
// A scene's step: a band along the top with the cue, the hold's word and the time
|
||||
// left; the rest is clear, so the windows below show through nothing.
|
||||
pic.px.assign(size_t(kW) * kH * 4, 0);
|
||||
const int splitSize = int(ppd * 0.8), splitH = p.splits.empty() || p.countdown < 0 ? 0 : int(splitSize * 1.5);
|
||||
const int bandH = int(ppd * 6.2) + splitH, m = int(ppd * 0.9), stepSize = int(ppd * 0.8);
|
||||
RoundRect(pic, 0, 0, kW, bandH, ppd * 1.2, 0.06, 0.06, 0.07, 0.85);
|
||||
const int cy0 = m + stepSize / 2;
|
||||
Text(pic, p.step, stepSize, kW - m, cy0, faint, kRight);
|
||||
if (p.rec) {
|
||||
const int rx = kW - m - TextWidth(p.step, stepSize) - (p.step.empty() ? 0 : int(ppd * 0.6));
|
||||
Text(pic, "Rec", stepSize, rx, cy0, 1.0, 0.45, 0.4, kRight);
|
||||
Disc(pic, rx - TextWidth("Rec", stepSize) - ppd * 0.4, cy0, ppd * 0.22, 0, 0.95, 0.2, 0.15, 1);
|
||||
}
|
||||
Text(pic, p.title, stepSize, m, cy0, faint, kLeft);
|
||||
const int bigW = p.big.empty() ? 0 : int(ppd * 6), size = int(ppd * 1.15), lh = int(size * 1.25);
|
||||
const int x0 = m + bigW, x1 = kW - m, noteSize = int(ppd * 0.95);
|
||||
p.lines = Wrap(p.text, size, x1 - x0);
|
||||
if (p.lines.size() > 2) p.lines.resize(2);
|
||||
p.noteLines = Wrap(p.note, noteSize, x1 - x0);
|
||||
if (p.noteLines.size() > 1) p.noteLines.resize(1);
|
||||
p.textDeg = 1.15;
|
||||
const int foot = bandH - m, block = int(p.lines.size()) * lh + int(p.noteLines.size()) * int(noteSize * 1.3);
|
||||
int y = cy0 + stepSize / 2 + (foot - int(ppd * 0.5) - splitH - cy0 - stepSize / 2 - block) / 2;
|
||||
if (!p.big.empty()) Text(pic, p.big, int(ppd * 2.2), m + bigW / 2, (cy0 + foot) / 2, kCyan[0], kCyan[1], kCyan[2]);
|
||||
for (const std::string &line : p.lines) Text(pic, line, size, (x0 + x1) / 2, y + lh / 2, 1.0), y += lh;
|
||||
for (const std::string &line : p.noteLines)
|
||||
Text(pic, line, noteSize, (x0 + x1) / 2, y + int(noteSize * 0.65), 1.0, 0.62, 0.3), y += int(noteSize * 1.3);
|
||||
if (p.countdown >= 0) {
|
||||
const int th = std::max(4, int(ppd * 0.18));
|
||||
Rect(pic, m, foot - th, kW - m, foot, 1, 1, 1, 0.15);
|
||||
Rect(pic, m, foot - th, m + int(std::lround((kW - 2 * m) * p.countdown)), foot, kCyan[0], kCyan[1], kCyan[2], 0.95);
|
||||
if (splitH) DrawSplits(pic, p, m, kW - 2 * m, foot - th, foot, foot - th - int(ppd * 0.3) - splitH / 2, splitSize);
|
||||
}
|
||||
if (p.paused) {
|
||||
const int size = int(ppd * 1.6);
|
||||
RoundRect(pic, 0, 0, kW, bandH, ppd * 1.2, 0.02, 0.02, 0.03, 0.8);
|
||||
Text(pic, "Paused", size, kW / 2, bandH / 2, 1.0);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
// The title and the step share the top line, a red "Rec" by the step while recording; a
|
||||
// faint rule under them.
|
||||
@@ -615,6 +702,11 @@ void Draw(Panel &p) {
|
||||
Rect(pic, margin, y, kW - margin, y + th, 1, 1, 1, 0.15);
|
||||
Rect(pic, margin, y, margin + int(std::lround(width * p.countdown)), y + th, kCyan[0], kCyan[1], kCyan[2], 0.95);
|
||||
bottom = std::min(bottom, y - int(ppd * 0.6));
|
||||
if (!p.splits.empty()) {
|
||||
const int size = int(ppd * 0.9), ly = y - int(ppd * 0.35) - size * 3 / 4;
|
||||
DrawSplits(pic, p, margin, width, y, y + th, ly, size);
|
||||
bottom = std::min(bottom, ly - size - int(ppd * 0.3));
|
||||
}
|
||||
}
|
||||
if (p.hands[0] != "off" || p.hands[1] != "off") {
|
||||
const int ch = int(ppd * 1.25), size = int(ppd * 0.85), cy = bottom - ch / 2, gap = int(ppd * 0.6);
|
||||
@@ -807,6 +899,176 @@ void DrawTarget(Target &t) {
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- the scene's windows
|
||||
|
||||
// A bounty's scene (DESIGN.md, "Scenes"): plain window shapes fixed in the room, with a part
|
||||
// lit to say where to tap or grab, and a ghost outline to say where to take it. They take no
|
||||
// input and never move by themselves: the person acts on them, the cameras record.
|
||||
constexpr int kWinMax = 4;
|
||||
constexpr int kWinPx = 768; // a window's picture: its longer side
|
||||
constexpr int kGhostPx = 384; // a ghost's
|
||||
constexpr double kWinEdgeM = 0.03; // a lit edge's band
|
||||
constexpr double kWinCornerM = 0.07; // a lit corner's arms
|
||||
|
||||
struct Win {
|
||||
std::string id;
|
||||
bool on = false;
|
||||
double w = 0.4, h = 0.3; // metres
|
||||
std::string look = "plain"; // plain, grid:CxR (buttons), list:N (rows)
|
||||
int cols = 0, rows = 0;
|
||||
std::string lit = "none", colour = "cyan", label;
|
||||
vr::HmdMatrix34_t pose{}; // in the room (standing universe)
|
||||
Picture pic{0, 0, {}};
|
||||
bool dirty = false, shown = false;
|
||||
bool ghostOn = false, ghostDirty = false, ghostShown = false;
|
||||
vr::HmdMatrix34_t ghostPose{};
|
||||
double ghostScale = 1;
|
||||
Picture ghostPic{0, 0, {}};
|
||||
vr::VROverlayHandle_t oh = vr::k_ulOverlayHandleInvalid, gh = vr::k_ulOverlayHandleInvalid;
|
||||
};
|
||||
|
||||
const double *Colour(const std::string &name) {
|
||||
static const double cyan[3] = {0.3, 0.85, 1.0}, purple[3] = {0.78, 0.48, 1.0}, orange[3] = {1.0, 0.56, 0.16},
|
||||
green[3] = {0.32, 0.9, 0.42};
|
||||
return name == "purple" ? purple : name == "orange" ? orange : name == "green" ? green : cyan;
|
||||
}
|
||||
|
||||
bool WinLook(Win &w, const std::string &look) {
|
||||
int a = 0, b = 0;
|
||||
if (look == "plain") w.cols = w.rows = 0;
|
||||
else if (std::sscanf(look.c_str(), "grid:%dx%d", &a, &b) == 2 && a >= 1 && a <= 6 && b >= 1 && b <= 6) w.cols = a, w.rows = b;
|
||||
else if (std::sscanf(look.c_str(), "list:%d", &a) == 1 && a >= 1 && a <= 10) w.cols = 1, w.rows = a;
|
||||
else return false;
|
||||
w.look = look;
|
||||
return true;
|
||||
}
|
||||
|
||||
bool WinPart1(const std::string &part) {
|
||||
static const char *parts[] = {"none", "bar", "left", "right", "top", "bottom", "tl", "tr", "bl", "br"};
|
||||
for (const char *p : parts)
|
||||
if (part == p) return true;
|
||||
int n = -1;
|
||||
return (std::sscanf(part.c_str(), "button:%d", &n) == 1 || std::sscanf(part.c_str(), "item:%d", &n) == 1) && n >= 0;
|
||||
}
|
||||
|
||||
// The parts of "a+b+...": a two-hand grab lights two corners or both edges at once.
|
||||
std::vector<std::string> WinParts(const std::string &lit) {
|
||||
std::vector<std::string> out;
|
||||
for (size_t at = 0; at <= lit.size();) {
|
||||
const size_t end = std::min(lit.find('+', at), lit.size());
|
||||
out.push_back(lit.substr(at, end - at));
|
||||
at = end + 1;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
bool WinPart(const std::string &lit) {
|
||||
for (const std::string &part : WinParts(lit))
|
||||
if (!WinPart1(part)) return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
// A window as it shows: a dark body, a title bar, its content (buttons, list rows or faint
|
||||
// lines of text), and the lit part in its colour, with its label.
|
||||
void DrawWin(Win &w) {
|
||||
const int pw = w.w >= w.h ? kWinPx : std::max(16, int(std::lround(kWinPx * w.w / w.h)));
|
||||
const int ph = w.w >= w.h ? std::max(16, int(std::lround(kWinPx * w.h / w.w))) : kWinPx;
|
||||
Picture &pic = w.pic;
|
||||
pic.w = pw, pic.h = ph;
|
||||
pic.px.assign(size_t(pw) * ph * 4, 0);
|
||||
const double s = pw / w.w; // pixels a metre
|
||||
const double *c = Colour(w.colour);
|
||||
const int rad = int(0.012 * s), bar = int(std::min(0.035, 0.16 * w.h) * s), m = int(0.012 * s);
|
||||
const std::vector<std::string> parts = WinParts(w.lit);
|
||||
auto isLit = [&parts](const std::string &part) { return std::find(parts.begin(), parts.end(), part) != parts.end(); };
|
||||
const bool litBar = isLit("bar");
|
||||
RoundRect(pic, 0, 0, pw, ph, rad, 0.11, 0.12, 0.15, 0.94);
|
||||
RoundRect(pic, 0, 0, pw, bar + rad, rad, litBar ? c[0] : 0.22, litBar ? c[1] : 0.23, litBar ? c[2] : 0.27, 1);
|
||||
Rect(pic, 0, bar, pw, bar + rad, 0.11, 0.12, 0.15, 1);
|
||||
for (int k = 0; k < 3; ++k) // the title bar's three dots, and a short title
|
||||
Disc(pic, pw - m - bar * 0.3 - k * bar * 0.55, bar / 2.0, bar * 0.14, 0, litBar ? 0.05 : 0.6, litBar ? 0.05 : 0.6,
|
||||
litBar ? 0.06 : 0.62, 0.9);
|
||||
Rect(pic, m, bar / 2 - std::max(1, bar / 12), m + int(0.08 * s), bar / 2 + std::max(1, bar / 12), litBar ? 0.05 : 0.6,
|
||||
litBar ? 0.05 : 0.6, litBar ? 0.06 : 0.62, 0.8);
|
||||
const int x0 = m, y0 = bar + m, x1 = pw - m, y1 = ph - m;
|
||||
int litCell = -1;
|
||||
for (const std::string &part : parts)
|
||||
if (std::sscanf(part.c_str(), "button:%d", &litCell) == 1 || std::sscanf(part.c_str(), "item:%d", &litCell) == 1) break;
|
||||
else litCell = -1;
|
||||
const int labelSize = std::max(10, int(0.035 * s));
|
||||
if (w.cols > 0) {
|
||||
const int gap = int(0.01 * s);
|
||||
const int cw = (x1 - x0 - (w.cols - 1) * gap) / w.cols, chh = (y1 - y0 - (w.rows - 1) * gap) / w.rows;
|
||||
for (int r = 0; r < w.rows; ++r)
|
||||
for (int k = 0; k < w.cols; ++k) {
|
||||
const int cx = x0 + k * (cw + gap), cy = y0 + r * (chh + gap), cell = r * w.cols + k;
|
||||
const bool lit = cell == litCell;
|
||||
const bool hollow = lit && w.colour == "orange"; // a no-go: outlined, with its cross
|
||||
if (lit && !hollow) RoundRect(pic, cx, cy, cx + cw, cy + chh, rad, c[0], c[1], c[2], 1);
|
||||
else RoundRect(pic, cx, cy, cx + cw, cy + chh, rad, 0.22, 0.23, 0.28, 1);
|
||||
if (hollow) {
|
||||
const int b = std::max(3, int(0.005 * s));
|
||||
RoundRect(pic, cx, cy, cx + cw, cy + chh, rad, c[0], c[1], c[2], 1);
|
||||
RoundRect(pic, cx + b, cy + b, cx + cw - b, cy + chh - b, std::max(1, rad - b), 0.16, 0.12, 0.1, 1);
|
||||
Text(pic, w.label.empty() ? "×" : w.label, std::min(chh, cw) * 2 / 3, cx + cw / 2, cy + chh / 2, c[0],
|
||||
c[1], c[2]);
|
||||
} else if (lit && !w.label.empty()) {
|
||||
Text(pic, w.label, std::min({chh * 2 / 3, cw / 2, labelSize * 3}), cx + cw / 2, cy + chh / 2, 0.05, 0.05, 0.07);
|
||||
} else if (w.rows > 1 && w.cols == 1) { // a list row: a faint line of text
|
||||
Rect(pic, cx + m, cy + chh / 2 - std::max(1, chh / 10), cx + cw / 2, cy + chh / 2 + std::max(1, chh / 10),
|
||||
0.45, 0.46, 0.5, 0.7);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
const int lh = int(0.022 * s); // faint lines of text: the window has something in it
|
||||
for (int y = y0 + lh; y + lh < y1; y += 2 * lh)
|
||||
Rect(pic, x0 + m, y, x0 + m + int((x1 - x0 - 2 * m) * (0.55 + 0.4 * ((y / lh) % 3) / 2.0)), y + lh / 2, 0.36,
|
||||
0.37, 0.42, 0.6);
|
||||
}
|
||||
// A lit edge or corner: a band in the window's colour along it, the label beside the first.
|
||||
const int band = int(kWinEdgeM * s), arm = int(kWinCornerM * s);
|
||||
int lx = -1, ly = -1;
|
||||
auto at = [&lx, &ly](int x, int y) {
|
||||
if (lx < 0) lx = x, ly = y;
|
||||
};
|
||||
for (const std::string &part : parts) {
|
||||
if (part == "left") RoundRect(pic, 0, 0, band, ph, rad, c[0], c[1], c[2], 0.95), at(band + labelSize, ph / 2);
|
||||
else if (part == "right") RoundRect(pic, pw - band, 0, pw, ph, rad, c[0], c[1], c[2], 0.95), at(pw - band - labelSize, ph / 2);
|
||||
else if (part == "top") RoundRect(pic, 0, 0, pw, band, rad, c[0], c[1], c[2], 0.95), at(pw / 2, band + labelSize);
|
||||
else if (part == "bottom") RoundRect(pic, 0, ph - band, pw, ph, rad, c[0], c[1], c[2], 0.95), at(pw / 2, ph - band - labelSize);
|
||||
else if (part.size() == 2 && (part[0] == 't' || part[0] == 'b') && (part[1] == 'l' || part[1] == 'r')) {
|
||||
const bool top = part[0] == 't', left = part[1] == 'l';
|
||||
const int ax = left ? 0 : pw - arm, ay = top ? 0 : ph - arm, bx = left ? 0 : pw - band, by = top ? 0 : ph - band;
|
||||
RoundRect(pic, ax, by, ax + arm, by + band, rad, c[0], c[1], c[2], 0.95);
|
||||
RoundRect(pic, bx, ay, bx + band, ay + arm, rad, c[0], c[1], c[2], 0.95);
|
||||
at(left ? arm + labelSize : pw - arm - labelSize, top ? arm / 2 + band : ph - arm / 2 - band);
|
||||
} else if (part == "bar") {
|
||||
at(pw / 2, bar + labelSize);
|
||||
}
|
||||
}
|
||||
if (lx >= 0 && !w.label.empty()) Text(pic, w.label, labelSize, lx, ly, c[0], c[1], c[2]);
|
||||
}
|
||||
|
||||
// The ghost: where to take the window, as a dashed outline with a faint fill, in its colour.
|
||||
void DrawGhost(Win &w) {
|
||||
const int pw = w.w >= w.h ? kGhostPx : std::max(8, int(std::lround(kGhostPx * w.w / w.h)));
|
||||
const int ph = w.w >= w.h ? std::max(8, int(std::lround(kGhostPx * w.h / w.w))) : kGhostPx;
|
||||
Picture &pic = w.ghostPic;
|
||||
pic.w = pw, pic.h = ph;
|
||||
pic.px.assign(size_t(pw) * ph * 4, 0);
|
||||
const double *c = Colour(w.colour);
|
||||
const int t = std::max(3, pw / 90), dash = std::max(8, pw / 18);
|
||||
RoundRect(pic, 0, 0, pw, ph, pw / 40.0, c[0], c[1], c[2], 0.1);
|
||||
for (int x = 0; x < pw; x += 2 * dash) {
|
||||
Rect(pic, x, 0, std::min(pw, x + dash), t, c[0], c[1], c[2], 0.95);
|
||||
Rect(pic, x, ph - t, std::min(pw, x + dash), ph, c[0], c[1], c[2], 0.95);
|
||||
}
|
||||
for (int y = 0; y < ph; y += 2 * dash) {
|
||||
Rect(pic, 0, y, t, std::min(ph, y + dash), c[0], c[1], c[2], 0.95);
|
||||
Rect(pic, pw - t, y, pw, std::min(ph, y + dash), c[0], c[1], c[2], 0.95);
|
||||
}
|
||||
}
|
||||
|
||||
// --no-vr: a picture as a PAM file (RGBA, straight alpha) for checking the layout by eye.
|
||||
void Dump(const Picture &pic, const std::string &path) {
|
||||
if (FILE *f = std::fopen(path.c_str(), "wb")) {
|
||||
@@ -826,6 +1088,8 @@ void Print(const Panel &p, const Target &t, bool visible, int number) {
|
||||
for (const std::string &line : p.noteLines) std::printf(" | %s\n", line.c_str());
|
||||
if (p.countdown >= 0) std::printf("countdown: %.2f\n", p.countdown);
|
||||
else std::printf("countdown: off\n");
|
||||
for (const auto &[at, label] : p.splits)
|
||||
std::printf("split: %.2f \"Switch to %s\"%s\n", at, label.c_str(), p.countdown >= 0 && p.countdown <= at + 1e-6 ? " (reached)" : "");
|
||||
std::printf("hands: left %s, right %s\n", p.hands[0].c_str(), p.hands[1].c_str());
|
||||
if (p.barOn)
|
||||
std::printf("bar: target %.2f, current %.2f, near \"%s\", far \"%s\"\n", p.barTarget, p.barCurrent,
|
||||
@@ -1083,6 +1347,55 @@ vr::HmdMatrix34_t Facing(const double at[3], const double eye[3]) {
|
||||
return m;
|
||||
}
|
||||
|
||||
// a then b: 3x4 transforms, each with an unwritten last row of 0 0 0 1.
|
||||
vr::HmdMatrix34_t Mul(const vr::HmdMatrix34_t &a, const vr::HmdMatrix34_t &b) {
|
||||
vr::HmdMatrix34_t m{};
|
||||
for (int r = 0; r < 3; ++r)
|
||||
for (int k = 0; k < 4; ++k)
|
||||
m.m[r][k] = a.m[r][0] * b.m[0][k] + a.m[r][1] * b.m[1][k] + a.m[r][2] * b.m[2][k] + (k == 3 ? a.m[r][3] : 0);
|
||||
return m;
|
||||
}
|
||||
|
||||
// Moved by (x, y, z), then turned by yaw (about +y), pitch (about +x) and roll (about +z), in
|
||||
// degrees, in that order: a window's or a ghost's pose in its parent's frame.
|
||||
vr::HmdMatrix34_t Local(double x, double y, double z, double yaw, double pitch, double roll) {
|
||||
const double a = yaw * M_PI / 180, b = pitch * M_PI / 180, c = roll * M_PI / 180;
|
||||
const double ry[3][3] = {{std::cos(a), 0, std::sin(a)}, {0, 1, 0}, {-std::sin(a), 0, std::cos(a)}};
|
||||
const double rx[3][3] = {{1, 0, 0}, {0, std::cos(b), -std::sin(b)}, {0, std::sin(b), std::cos(b)}};
|
||||
const double rz[3][3] = {{std::cos(c), -std::sin(c), 0}, {std::sin(c), std::cos(c), 0}, {0, 0, 1}};
|
||||
double t[3][3], r[3][3];
|
||||
for (int i = 0; i < 3; ++i)
|
||||
for (int j = 0; j < 3; ++j) t[i][j] = ry[i][0] * rx[0][j] + ry[i][1] * rx[1][j] + ry[i][2] * rx[2][j];
|
||||
for (int i = 0; i < 3; ++i)
|
||||
for (int j = 0; j < 3; ++j) r[i][j] = t[i][0] * rz[0][j] + t[i][1] * rz[1][j] + t[i][2] * rz[2][j];
|
||||
vr::HmdMatrix34_t m{};
|
||||
for (int i = 0; i < 3; ++i) m.m[i][0] = float(r[i][0]), m.m[i][1] = float(r[i][1]), m.m[i][2] = float(r[i][2]);
|
||||
m.m[0][3] = float(x), m.m[1][3] = float(y), m.m[2][3] = float(z);
|
||||
return m;
|
||||
}
|
||||
|
||||
std::string PoseReply(const vr::HmdMatrix34_t &m) {
|
||||
char out[256];
|
||||
int len = std::snprintf(out, sizeof out, "ok");
|
||||
for (int i = 0; i < 12; ++i) len += std::snprintf(out + len, sizeof out - len, " %.5f", m.m[i / 4][i % 4]);
|
||||
return out;
|
||||
}
|
||||
|
||||
// --no-vr: the windows as text.
|
||||
void PrintWins(const Win *wins) {
|
||||
for (int k = 0; k < kWinMax; ++k) {
|
||||
const Win &w = wins[k];
|
||||
if (!w.on) continue;
|
||||
std::printf("win %s: %.2f x %.2f m %s at %.3f %.3f %.3f, lit %s %s%s%s\n", w.id.c_str(), w.w, w.h, w.look.c_str(),
|
||||
w.pose.m[0][3], w.pose.m[1][3], w.pose.m[2][3], w.lit.c_str(), w.colour.c_str(),
|
||||
w.label.empty() ? "" : " ", w.label.c_str());
|
||||
if (w.ghostOn)
|
||||
std::printf("ghost %s: at %.3f %.3f %.3f, scale %.2f\n", w.id.c_str(), w.ghostPose.m[0][3], w.ghostPose.m[1][3],
|
||||
w.ghostPose.m[2][3], w.ghostScale);
|
||||
}
|
||||
std::fflush(stdout);
|
||||
}
|
||||
|
||||
// The command word: buf starts with word, then a space or the end; rest is what follows.
|
||||
bool Is(const char *buf, const char *word, const char **rest = nullptr) {
|
||||
const size_t n = std::strlen(word);
|
||||
@@ -1177,6 +1490,145 @@ int main(int argc, char **argv) {
|
||||
bool visible = false, dirty = false, shown = false; // shown: SteamVR shows it (after its first picture)
|
||||
bool targetDirty = false, targetShown = false;
|
||||
int pictures = 0;
|
||||
Win wins[kWinMax];
|
||||
Buffers winBuffers[kWinMax], ghostBuffers[kWinMax];
|
||||
bool winsChanged = false; // --no-vr: print them
|
||||
|
||||
// A window's slot by id, or (place) a free one; -1: none.
|
||||
auto winSlot = [&wins](const std::string &id, bool make) {
|
||||
for (int k = 0; k < kWinMax; ++k)
|
||||
if (wins[k].on && wins[k].id == id) return k;
|
||||
if (make)
|
||||
for (int k = 0; k < kWinMax; ++k)
|
||||
if (!wins[k].on) return k;
|
||||
return -1;
|
||||
};
|
||||
auto winOff = [&](Win &w) {
|
||||
if (ov) {
|
||||
std::lock_guard<std::mutex> lk(g_vr);
|
||||
if (w.shown) ov->HideOverlay(w.oh);
|
||||
if (w.ghostShown) ov->HideOverlay(w.gh);
|
||||
}
|
||||
const vr::VROverlayHandle_t oh = w.oh, gh = w.gh;
|
||||
w = Win{};
|
||||
w.oh = oh, w.gh = gh;
|
||||
winsChanged = true;
|
||||
};
|
||||
// win <id> place|lit|ghost|off ..., win clear: the reply.
|
||||
auto winCommand = [&](const char *rest) -> std::string {
|
||||
char id[32] = "", verb[16] = "";
|
||||
int used = 0;
|
||||
if (!std::strcmp(rest, "clear")) {
|
||||
for (Win &w : wins)
|
||||
if (w.on) winOff(w);
|
||||
return "ok";
|
||||
}
|
||||
if (std::sscanf(rest, "%31s %15s %n", id, verb, &used) < 2)
|
||||
return "error usage: win <id> place|lit|ghost|off ... | win clear";
|
||||
const char *args = rest + used;
|
||||
const std::string v = verb;
|
||||
const int k = winSlot(id, v == "place");
|
||||
if (k < 0) return v == "place" ? "error no free window (at most 4)" : std::string("error no window ") + id;
|
||||
Win &w = wins[k];
|
||||
if (v == "off") {
|
||||
winOff(w);
|
||||
return "ok";
|
||||
}
|
||||
if (v == "place") {
|
||||
// place <x> <y> <z> <w> <h> [<yaw> <pitch>] [<look>]: the centre in the head's frame now, levelled
|
||||
double x, y, z, ww, hh, yaw = 0, pitch = 0;
|
||||
char look[32] = "plain";
|
||||
int n = 0;
|
||||
if (std::sscanf(args, "%lf %lf %lf %lf %lf %n", &x, &y, &z, &ww, &hh, &n) < 5 || ww < 0.02 || hh < 0.02 || ww > 3 || hh > 3)
|
||||
return "error usage: win <id> place <x> <y> <z> <w> <h> [<yaw> <pitch>] [<look>]";
|
||||
const char *more = args + n;
|
||||
int n2 = 0;
|
||||
if (std::sscanf(more, "%lf %lf %n", &yaw, &pitch, &n2) == 2) more += n2;
|
||||
std::sscanf(more, "%31s", look);
|
||||
Win next = w;
|
||||
if (!WinLook(next, look)) return std::string("error look ") + look + ": plain, grid:CxR or list:N";
|
||||
vr::HmdMatrix34_t head{};
|
||||
head.m[0][0] = head.m[1][1] = head.m[2][2] = 1; // --no-vr: the head at the room's origin
|
||||
if (!noVr && !HeadPose(head)) return "error no head pose";
|
||||
// The head's frame made level: turned with the head, not tipped with it, so a window
|
||||
// "ahead" stays at the same height whether the person looks down at their lap or up.
|
||||
const double fx = head.m[0][2], fz = head.m[2][2], fn = std::hypot(fx, fz);
|
||||
if (fn > 0.05) {
|
||||
const double z[3] = {fx / fn, 0, fz / fn}, x[3] = {z[2], 0, -z[0]};
|
||||
for (int r = 0; r < 3; ++r) head.m[r][0] = float(x[r]), head.m[r][1] = r == 1, head.m[r][2] = float(z[r]);
|
||||
}
|
||||
const double local[3] = {x, y, z}, eye[3] = {head.m[0][3], head.m[1][3], head.m[2][3]};
|
||||
double at[3];
|
||||
for (int r = 0; r < 3; ++r) at[r] = head.m[r][0] * local[0] + head.m[r][1] * local[1] + head.m[r][2] * local[2] + head.m[r][3];
|
||||
next.id = id, next.on = true, next.w = ww, next.h = hh;
|
||||
next.pose = Mul(Facing(at, eye), Local(0, 0, 0, yaw, pitch, 0));
|
||||
next.lit = "none", next.label.clear(), next.colour = "cyan";
|
||||
next.ghostOn = false;
|
||||
if (w.ghostShown && ov) {
|
||||
std::lock_guard<std::mutex> lk(g_vr);
|
||||
ov->HideOverlay(w.gh);
|
||||
}
|
||||
next.ghostShown = false;
|
||||
next.dirty = true;
|
||||
w = next;
|
||||
if (ov) {
|
||||
if (w.oh == vr::k_ulOverlayHandleInvalid) {
|
||||
const std::string key = "frametop.handpanel.win." + std::to_string(k);
|
||||
std::lock_guard<std::mutex> lk(g_vr);
|
||||
if (ov->CreateOverlay(key.c_str(), "Frametop hand recorder window", &w.oh) != vr::VROverlayError_None)
|
||||
return "error can't create the window's overlay";
|
||||
ov->SetOverlaySortOrder(w.oh, 240); // under the panel
|
||||
}
|
||||
std::lock_guard<std::mutex> lk(g_vr);
|
||||
ov->SetOverlayTransformAbsolute(w.oh, vr::TrackingUniverseStanding, &w.pose);
|
||||
ov->SetOverlayWidthInMeters(w.oh, float(w.w));
|
||||
}
|
||||
winsChanged = true;
|
||||
return PoseReply(w.pose);
|
||||
}
|
||||
if (v == "lit") {
|
||||
// lit <part> [<colour>] [<label>]
|
||||
char part[64] = "", colour[16] = "cyan", label[32] = "";
|
||||
if (std::sscanf(args, "%63s %15s %31s", part, colour, label) < 1 || !WinPart(part))
|
||||
return "error usage: win <id> lit none|bar|left|right|top|bottom|tl|tr|bl|br|button:N|item:N[+...] [<colour>] [<label>]";
|
||||
w.lit = part, w.colour = colour, w.label = label;
|
||||
for (char &ch : w.label)
|
||||
if (ch == '_') ch = ' ';
|
||||
w.dirty = winsChanged = true;
|
||||
if (w.ghostOn) w.ghostDirty = true; // the ghost takes the lit colour
|
||||
return "ok";
|
||||
}
|
||||
if (v == "ghost") {
|
||||
// ghost <dx> <dy> <dz> <yaw> <pitch> <roll> [<scale>]: in the window's own frame
|
||||
if (!std::strcmp(args, "off")) {
|
||||
if (ov && w.ghostShown) {
|
||||
std::lock_guard<std::mutex> lk(g_vr);
|
||||
ov->HideOverlay(w.gh);
|
||||
}
|
||||
w.ghostOn = w.ghostShown = false, winsChanged = true;
|
||||
return "ok";
|
||||
}
|
||||
double dx, dy, dz, yaw, pitch, roll, scale = 1;
|
||||
if (std::sscanf(args, "%lf %lf %lf %lf %lf %lf %lf", &dx, &dy, &dz, &yaw, &pitch, &roll, &scale) < 6 || scale < 0.2 || scale > 5)
|
||||
return "error usage: win <id> ghost <dx> <dy> <dz> <yaw> <pitch> <roll> [<scale>] | win <id> ghost off";
|
||||
w.ghostPose = Mul(w.pose, Local(dx, dy, dz, yaw, pitch, roll));
|
||||
w.ghostScale = scale, w.ghostOn = w.ghostDirty = winsChanged = true;
|
||||
if (ov) {
|
||||
if (w.gh == vr::k_ulOverlayHandleInvalid) {
|
||||
const std::string key = "frametop.handpanel.ghost." + std::to_string(k);
|
||||
std::lock_guard<std::mutex> lk(g_vr);
|
||||
if (ov->CreateOverlay(key.c_str(), "Frametop hand recorder ghost", &w.gh) != vr::VROverlayError_None)
|
||||
return "error can't create the ghost's overlay";
|
||||
ov->SetOverlaySortOrder(w.gh, 245);
|
||||
}
|
||||
std::lock_guard<std::mutex> lk(g_vr);
|
||||
ov->SetOverlayTransformAbsolute(w.gh, vr::TrackingUniverseStanding, &w.ghostPose);
|
||||
ov->SetOverlayWidthInMeters(w.gh, float(w.w * scale));
|
||||
}
|
||||
return PoseReply(w.ghostPose);
|
||||
}
|
||||
return "error usage: win <id> place|lit|ghost|off ... | win clear";
|
||||
};
|
||||
std::fprintf(stderr, "ft-handpanel running: @%s, %.2f m%s\n", sockName.c_str(), distance, noVr ? ", no VR" : "");
|
||||
|
||||
while (!g_stop) {
|
||||
@@ -1217,6 +1669,27 @@ int main(int argc, char **argv) {
|
||||
if (!std::strcmp(rest, "off")) p.countdown = -1, dirty = true;
|
||||
else if (std::sscanf(rest, "%lf", &a) == 1) p.countdown = std::clamp(a, 0.0, 1.0), dirty = true;
|
||||
else reply = "error usage: countdown <0..1>|off";
|
||||
} else if (Is(buf, "split", &rest)) {
|
||||
// split <at> <label>;...
|
||||
std::vector<std::pair<double, std::string>> items;
|
||||
bool ok = std::strcmp(rest, "off") == 0;
|
||||
if (!ok) {
|
||||
std::string all = rest;
|
||||
ok = true;
|
||||
for (size_t at = 0; at <= all.size();) {
|
||||
const size_t end = std::min(all.find(';', at), all.size());
|
||||
const std::string item = all.substr(at, end - at);
|
||||
at = end + 1;
|
||||
if (item.empty()) continue;
|
||||
double share = 0;
|
||||
int n = 0;
|
||||
if (std::sscanf(item.c_str(), "%lf %n", &share, &n) < 1 || !n || !item[n]) { ok = false; break; }
|
||||
items.emplace_back(std::clamp(share, 0.0, 1.0), item.substr(n));
|
||||
}
|
||||
ok = ok && !items.empty();
|
||||
}
|
||||
if (ok) p.splits = items, dirty = true;
|
||||
else reply = "error usage: split <at 0..1> <label>;... | split off";
|
||||
} else if (Is(buf, "hands", &rest)) {
|
||||
char l[16] = "", r[16] = "";
|
||||
if (std::sscanf(rest, "%15s %15s", l, r) == 2 && HandState(l) && HandState(r))
|
||||
@@ -1425,6 +1898,11 @@ int main(int argc, char **argv) {
|
||||
for (int i = 0; i < 12; ++i) len += std::snprintf(out + len, sizeof out - len, " %.6f", m.m[i / 4][i % 4]);
|
||||
reply = out;
|
||||
}
|
||||
} else if (Is(buf, "win", &rest)) {
|
||||
reply = winCommand(rest);
|
||||
} else if (Is(buf, "layout", &rest)) {
|
||||
if (!std::strcmp(rest, "full") || !std::strcmp(rest, "top")) p.compact = rest[0] == 't', dirty = true;
|
||||
else reply = "error usage: layout full|top";
|
||||
} else if (Is(buf, "ping")) {
|
||||
reply = visible ? "ok shown" : "ok hidden";
|
||||
} else {
|
||||
@@ -1478,6 +1956,42 @@ int main(int argc, char **argv) {
|
||||
}
|
||||
targetDirty = false;
|
||||
}
|
||||
// The windows and their ghosts that changed. A picture of a new size gets buffers of its size.
|
||||
for (int k = 0; k < kWinMax; ++k) {
|
||||
Win &w = wins[k];
|
||||
if (w.on && w.dirty) {
|
||||
DrawWin(w);
|
||||
if (ov) {
|
||||
if (winBuffers[k].w != w.pic.w || winBuffers[k].h != w.pic.h) { // (a failed Make keeps the size: SetOverlayRaw then)
|
||||
std::lock_guard<std::mutex> lk(g_vr); // Make and Drop call OpenVR
|
||||
winBuffers[k].Drop();
|
||||
if (winBuffers[k].Make(w.pic.w, w.pic.h)) ov->SetOverlayFlag(w.oh, vr::VROverlayFlags_IsPremultiplied, true);
|
||||
}
|
||||
winBuffers[k].Present(ov, w.oh, w.pic);
|
||||
std::lock_guard<std::mutex> lk(g_vr);
|
||||
if (!w.shown) ov->ShowOverlay(w.oh), w.shown = true;
|
||||
}
|
||||
if (noVr && !dumpDir.empty()) Dump(w.pic, dumpDir + "/win-" + w.id + ".pam");
|
||||
w.dirty = false;
|
||||
}
|
||||
if (w.on && w.ghostOn && w.ghostDirty) {
|
||||
DrawGhost(w);
|
||||
if (ov) {
|
||||
if (ghostBuffers[k].w != w.ghostPic.w || ghostBuffers[k].h != w.ghostPic.h) { // (a failed Make keeps the size: SetOverlayRaw then)
|
||||
std::lock_guard<std::mutex> lk(g_vr); // Make and Drop call OpenVR
|
||||
ghostBuffers[k].Drop();
|
||||
if (ghostBuffers[k].Make(w.ghostPic.w, w.ghostPic.h)) ov->SetOverlayFlag(w.gh, vr::VROverlayFlags_IsPremultiplied, true);
|
||||
}
|
||||
ghostBuffers[k].Present(ov, w.gh, w.ghostPic);
|
||||
std::lock_guard<std::mutex> lk(g_vr);
|
||||
if (!w.ghostShown) ov->ShowOverlay(w.gh), w.ghostShown = true;
|
||||
}
|
||||
if (noVr && !dumpDir.empty()) Dump(w.ghostPic, dumpDir + "/ghost-" + w.id + ".pam");
|
||||
w.ghostDirty = false;
|
||||
}
|
||||
}
|
||||
if (noVr && winsChanged) std::printf("--- windows\n"), PrintWins(wins);
|
||||
winsChanged = false;
|
||||
if (noVr && (panelNow || targetNow)) {
|
||||
Print(p, t, visible, ++pictures);
|
||||
if (!dumpDir.empty() && panelNow) Dump(p.pic, dumpDir + "/panel.pam");
|
||||
@@ -1490,6 +2004,12 @@ int main(int argc, char **argv) {
|
||||
ov->DestroyOverlay(th);
|
||||
buffers.Drop();
|
||||
targetBuffers.Drop();
|
||||
for (int k = 0; k < kWinMax; ++k) {
|
||||
if (wins[k].oh != vr::k_ulOverlayHandleInvalid) ov->DestroyOverlay(wins[k].oh);
|
||||
if (wins[k].gh != vr::k_ulOverlayHandleInvalid) ov->DestroyOverlay(wins[k].gh);
|
||||
winBuffers[k].Drop();
|
||||
ghostBuffers[k].Drop();
|
||||
}
|
||||
vr::VR_Shutdown();
|
||||
}
|
||||
return 0;
|
||||
|
||||
+820
-412
File diff suppressed because it is too large.
Load diff
+189
-91
@@ -25,7 +25,6 @@ usage: takes.py [--base DIR] list | takes SESSION | export SESSION | sides SESSI
|
||||
"""
|
||||
import argparse
|
||||
import datetime
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
@@ -37,6 +36,11 @@ import threading
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
import sides # noqa: E402 (hands/rec/sides.py, next to this file)
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
|
||||
from recorder import age # noqa: E402 (a debug session's recordings are encrypted)
|
||||
import eventlog # noqa: E402 (hands/rec/eventlog.py: a debug session's panel and pointer decisions)
|
||||
from recorder.files import (Cancelled, free_bytes, human_bytes, in_ranges, merge_ranges, read_json, # noqa: E402,F401
|
||||
sha256_file, tree_bytes, tree_stat, write_sums)
|
||||
|
||||
DEFAULT_BASE = os.path.expanduser("~/.local/share/frametop/hands/contrib")
|
||||
REPO = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
@@ -45,25 +49,39 @@ HDR = struct.Struct("<8sII") # fh_set_hdr_t: magic, ncams, bytes (the whol
|
||||
CAM = struct.Struct("<16sIIQQ") # fh_set_cam_t: name, width, height, capture_ns, dqbuf_ns
|
||||
MAX_CAMS = 16 # SetReader's limit
|
||||
PART_RE = re.compile(r"^sets(?:-(\d+))?\.bin$")
|
||||
LIVE_RE = re.compile(r"^live(?:-(\d+))?\.bin$") # a debug session's copies of the live tracker's output
|
||||
LIVE_HDR = struct.Struct("<8sII") # livelog.py: magic, version, 0
|
||||
LIVE_REC = struct.Struct("<B3xIQQ") # source, length, read_ns, seq
|
||||
VOICE_RE = re.compile(r"^voice(?:-(\d+))?\.bin$") # a debug session's microphone, if ticked
|
||||
VOICE_HDR = struct.Struct("<8sII") # voicelog.py: magic, version, sample rate
|
||||
VOICE_REC = struct.Struct("<QI") # read_ns, bytes of s16le samples
|
||||
EVENTS_RE = re.compile(r"^events(?:-(\d+))?\.jsonl$") # a debug session's panel and pointer decisions
|
||||
TEST_RECIPIENTS = os.path.join(os.path.dirname(os.path.abspath(__file__)), "test-recipients.txt")
|
||||
# The bounties' test phase (DESIGN.md "Testing"): while it's on, bounty rounds are recorded under
|
||||
# TESTING.md's consent (profile.json "testing_consent") like debug sessions, and their exports are
|
||||
# encrypted for the test dataset. It ended 2026-10-14: bounty rounds go under CONSENT.md to the public
|
||||
# dataset, like full sessions and quick rounds; only debug (testing) sessions stay under TESTING.md.
|
||||
TEST_PHASE = False
|
||||
CONSENT_KEYS = {"public": "consent", "testing": "testing_consent"} # profile.json's record of each
|
||||
|
||||
|
||||
def round_consent_kind(bounty=False, debug=False):
|
||||
"""Which consent a new session is recorded under: "testing" for a debug session, and for a
|
||||
bounty's round in the test phase; else "public"."""
|
||||
return "testing" if debug or (bounty and TEST_PHASE) else "public"
|
||||
|
||||
|
||||
def consent_kind(meta):
|
||||
"""The consent a session was recorded under (session.json "consent", session.py): "testing"
|
||||
or "public" (sessions from before it was recorded, and full sessions)."""
|
||||
kind = (meta.get("consent") or {}).get("kind") if isinstance(meta.get("consent"), dict) else None
|
||||
return "testing" if kind == "testing" or meta.get("debug") else "public"
|
||||
SESSION_RE = re.compile(r"^\d{8}-\d{6}(?:-\d+)?$") # session.py adds -2, -3 to a second one in a second
|
||||
EXPORT_SCHEMA = 1
|
||||
# zstd as DESIGN.md has it: level 10, two threads, at the lowest CPU priority (CPU work while
|
||||
# someone is in VR makes the headset stutter).
|
||||
ZSTD_ARGS = ["-10", "-T2", "-q", "-c"]
|
||||
ZSTD_PATHS = ("/usr/bin/zstd", "/usr/local/bin/zstd", "/run/host/usr/bin/zstd")
|
||||
CHUNK = 1 << 20
|
||||
|
||||
|
||||
class Cancelled(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def read_json(path, default=None):
|
||||
try:
|
||||
with open(path) as f:
|
||||
return json.load(f)
|
||||
except (OSError, ValueError):
|
||||
return {} if default is None else default
|
||||
|
||||
|
||||
def write_json(path, obj):
|
||||
@@ -112,38 +130,6 @@ def tool_version():
|
||||
return "ft-handrec " + (described or "unknown")
|
||||
|
||||
|
||||
def free_bytes(path):
|
||||
"""Free space on the file system that holds path (or its nearest existing parent)."""
|
||||
while path and not os.path.exists(path):
|
||||
path = os.path.dirname(path)
|
||||
try:
|
||||
return shutil.disk_usage(path or "/").free
|
||||
except OSError:
|
||||
return 0
|
||||
|
||||
|
||||
def tree_stat(path):
|
||||
"""(bytes, newest mtime) of the files under path."""
|
||||
total, newest = 0, 0.0
|
||||
for root, _, files in os.walk(path):
|
||||
for name in files:
|
||||
try:
|
||||
st = os.lstat(os.path.join(root, name))
|
||||
except OSError:
|
||||
continue
|
||||
total += st.st_size
|
||||
newest = max(newest, st.st_mtime)
|
||||
return total, newest
|
||||
|
||||
|
||||
def tree_bytes(path):
|
||||
return tree_stat(path)[0]
|
||||
|
||||
|
||||
def in_ranges(t, ranges):
|
||||
return any(a <= t <= b for a, b in ranges)
|
||||
|
||||
|
||||
def export_jsonl(src, dst, ranges, keep_controllers):
|
||||
"""Copy a take's poses.jsonl or prompts.jsonl for export:
|
||||
- poses in the deleted ranges are left out, and the live tracker's feedback there (the
|
||||
@@ -178,15 +164,105 @@ def export_jsonl(src, dst, ranges, keep_controllers):
|
||||
out.write(json.dumps(obj, separators=(", ", ": ")) + "\n")
|
||||
|
||||
|
||||
def merge_ranges(ranges):
|
||||
"""Sorted, with overlapping or touching ranges joined."""
|
||||
out = []
|
||||
for a, b in sorted((min(a, b), max(a, b)) for a, b in ranges):
|
||||
if out and a <= out[-1][1] + 1:
|
||||
out[-1][1] = max(out[-1][1], b)
|
||||
else:
|
||||
out.append([a, b])
|
||||
return out
|
||||
def live_parts(take_dir, pattern=LIVE_RE):
|
||||
"""A debug take's live.bin, live-2.bin, ... in order (pattern=VOICE_RE: voice.bin, ...)."""
|
||||
try:
|
||||
names = os.listdir(take_dir)
|
||||
except OSError:
|
||||
return []
|
||||
found = [(int(m.group(1) or 1), name) for name in names for m in [pattern.match(name)] if m]
|
||||
return [os.path.join(take_dir, name) for _, name in sorted(found)]
|
||||
|
||||
|
||||
def export_live(take_dir, ranges, zstd, dest, recipients=None):
|
||||
"""A debug take's live parts as one live.bin (livelog.py's format), records in deleted ranges
|
||||
left out, through zstd into dest, encrypted to recipients if given. None without live
|
||||
parts; else {"records": {"hands": n, "gestures": n}, "raw_bytes": n}."""
|
||||
parts = live_parts(take_dir)
|
||||
if not parts:
|
||||
return None
|
||||
out = bytearray(LIVE_HDR.pack(b"FHLIVE01", 1, 0))
|
||||
counts = {"hands": 0, "gestures": 0}
|
||||
for path in parts:
|
||||
with open(path, "rb") as f:
|
||||
data = f.read()
|
||||
if len(data) < LIVE_HDR.size or data[:8] != b"FHLIVE01":
|
||||
continue
|
||||
off = LIVE_HDR.size
|
||||
while off + LIVE_REC.size <= len(data):
|
||||
src, n, read_ns, _ = LIVE_REC.unpack_from(data, off)
|
||||
end = off + LIVE_REC.size + n
|
||||
if end > len(data) or src > 1:
|
||||
break # cut off: the session ended mid-write
|
||||
if not in_ranges(read_ns, ranges):
|
||||
out += data[off:end]
|
||||
counts["hands" if src == 0 else "gestures"] += 1
|
||||
off = end
|
||||
r = subprocess.run(["nice", "-n", "19", zstd] + ZSTD_ARGS, input=bytes(out), capture_output=True)
|
||||
if r.returncode != 0:
|
||||
raise RuntimeError("zstd failed: %s" % (r.stderr.decode(errors="replace").strip() or r.returncode))
|
||||
payload = age.encrypt_bytes(r.stdout, recipients) if recipients else r.stdout
|
||||
with open(dest, "wb") as f:
|
||||
f.write(payload)
|
||||
return {"records": counts, "raw_bytes": len(out)}
|
||||
|
||||
|
||||
def export_voice(take_dir, ranges, zstd, dest, recipients):
|
||||
"""A debug take's voice parts as one voice.bin (voicelog.py's format), chunks touching a
|
||||
deleted range left out, through zstd into dest, always encrypted (no recipients: nothing
|
||||
written, None). None without voice parts; else {"chunks": n, "seconds": s, "raw_bytes": n}."""
|
||||
parts = live_parts(take_dir, VOICE_RE)
|
||||
if not parts or not recipients:
|
||||
return None
|
||||
out, rate, chunks, samples = bytearray(), None, 0, 0
|
||||
for path in parts:
|
||||
with open(path, "rb") as f:
|
||||
data = f.read()
|
||||
if len(data) < VOICE_HDR.size or data[:8] != b"FHVOIC01":
|
||||
continue
|
||||
_, _, part_rate = VOICE_HDR.unpack_from(data, 0)
|
||||
if rate is None:
|
||||
rate = part_rate
|
||||
out += VOICE_HDR.pack(b"FHVOIC01", 1, rate)
|
||||
elif part_rate != rate:
|
||||
continue
|
||||
off = VOICE_HDR.size
|
||||
while off + VOICE_REC.size <= len(data):
|
||||
read_ns, n = VOICE_REC.unpack_from(data, off)
|
||||
end = off + VOICE_REC.size + n
|
||||
if end > len(data):
|
||||
break # cut off: the session ended mid-write
|
||||
first_ns = read_ns - n // 2 * 1_000_000_000 // max(1, rate)
|
||||
if not any(a <= read_ns and first_ns <= b for a, b in ranges):
|
||||
out += data[off:end]
|
||||
chunks += 1
|
||||
samples += n // 2
|
||||
off = end
|
||||
if rate is None:
|
||||
return None
|
||||
r = subprocess.run(["nice", "-n", "19", zstd] + ZSTD_ARGS, input=bytes(out), capture_output=True)
|
||||
if r.returncode != 0:
|
||||
raise RuntimeError("zstd failed: %s" % (r.stderr.decode(errors="replace").strip() or r.returncode))
|
||||
with open(dest, "wb") as f:
|
||||
f.write(age.encrypt_bytes(r.stdout, recipients))
|
||||
return {"chunks": chunks, "seconds": round(samples / rate, 2), "raw_bytes": len(out)}
|
||||
|
||||
|
||||
def export_events(take_dir, ranges, zstd, dest, recipients):
|
||||
"""A debug take's events parts (eventlog.py) as one events.jsonl, events in deleted ranges left
|
||||
out, through zstd into dest, always encrypted (no recipients: nothing written, None). None
|
||||
without events parts; else {"events": n, "by": {kind: {outcome: n}}, "raw_bytes": n}."""
|
||||
parts = live_parts(take_dir, EVENTS_RE)
|
||||
if not parts or not recipients:
|
||||
return None
|
||||
events = [e for path in parts for e in eventlog.read_lines(path) if not in_ranges(e["t"], ranges)]
|
||||
out = "".join(json.dumps(e, separators=(", ", ": ")) + "\n" for e in events).encode()
|
||||
r = subprocess.run(["nice", "-n", "19", zstd] + ZSTD_ARGS, input=out, capture_output=True)
|
||||
if r.returncode != 0:
|
||||
raise RuntimeError("zstd failed: %s" % (r.stderr.decode(errors="replace").strip() or r.returncode))
|
||||
with open(dest, "wb") as f:
|
||||
f.write(age.encrypt_bytes(r.stdout, recipients))
|
||||
return {"events": len(events), "by": eventlog.summary(events), "raw_bytes": len(out)}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- one take's recording
|
||||
@@ -380,6 +456,8 @@ class Store:
|
||||
"bytes": size, "contributor": meta.get("contributor", ""),
|
||||
"lighting": (meta.get("lighting") or {}).get("chosen", ""),
|
||||
"status": meta.get("status", ""), "dry_run": bool(meta.get("dry_run")),
|
||||
"bounty": ((meta.get("bounty") or {}).get("title") or "") if isinstance(meta.get("bounty"), dict) else "",
|
||||
"debug": bool(meta.get("debug")), "testing": consent_kind(meta) == "testing",
|
||||
"exported": bool(exported), "export_stale": bool(exported) and changed > exported,
|
||||
"export_bytes": tree_bytes(self.export_dir(sid)) if exported else 0})
|
||||
return sorted(out, key=lambda s: s["id"], reverse=True)
|
||||
@@ -525,6 +603,10 @@ class Store:
|
||||
done = 0
|
||||
take_entries = []
|
||||
keep_controllers = (meta.get("checklist") or {}).get("controllers") == "straps"
|
||||
# A test-phase session's recordings go to the test dataset encrypted (DESIGN.md "Testing")
|
||||
kind = consent_kind(meta)
|
||||
recipients = age.read_recipients(TEST_RECIPIENTS) if kind == "testing" else None
|
||||
encrypted = []
|
||||
for t in takes:
|
||||
if cancelled():
|
||||
raise Cancelled()
|
||||
@@ -552,47 +634,73 @@ class Store:
|
||||
"duration_s": round(t["duration_s"], 2), "file": None,
|
||||
"sides": {"names_swapped": names_swapped, "renamed_sets": index.renamed_sets(keep)}}
|
||||
done += index.bytes() - sum(index.sets[i][2] for i in keep) # deleted sets count as done
|
||||
suffix = ".age" if recipients else ""
|
||||
if keep:
|
||||
entry["file"] = "takes/%s/sets.bin.zst" % t["id"]
|
||||
entry["file"] = "takes/%s/sets.bin.zst%s" % (t["id"], suffix)
|
||||
|
||||
def step(nbytes, title=t["title"]):
|
||||
nonlocal done
|
||||
done += nbytes
|
||||
report(min(done / total, 0.99), f"Compressing {title}")
|
||||
entry["raw_bytes"] = self._compress(index, keep, zstd, os.path.join(out, "sets.bin.zst"),
|
||||
step, cancelled)
|
||||
report(min(done / total, 0.99), f"{'Compressing and encrypting' if recipients else 'Compressing'} {title}")
|
||||
entry["raw_bytes"] = self._compress(index, keep, zstd, os.path.join(work, entry["file"]),
|
||||
step, cancelled, recipients)
|
||||
if recipients:
|
||||
encrypted.append(entry["file"])
|
||||
live = export_live(tdir, ranges, zstd, os.path.join(out, "live.bin.zst" + suffix), recipients)
|
||||
if live is not None:
|
||||
entry["live"] = dict(live, file="takes/%s/live.bin.zst%s" % (t["id"], suffix))
|
||||
if recipients:
|
||||
encrypted.append(entry["live"]["file"])
|
||||
voice = export_voice(tdir, ranges, zstd, os.path.join(out, "voice.bin.zst.age"), recipients)
|
||||
if voice is not None:
|
||||
entry["voice"] = dict(voice, file="takes/%s/voice.bin.zst.age" % t["id"])
|
||||
encrypted.append(entry["voice"]["file"])
|
||||
events = export_events(tdir, ranges, zstd, os.path.join(out, "events.jsonl.zst.age"), recipients)
|
||||
if events is not None:
|
||||
entry["events"] = dict(events, file="takes/%s/events.jsonl.zst.age" % t["id"])
|
||||
encrypted.append(entry["events"]["file"])
|
||||
take_entries.append(entry)
|
||||
if cancelled():
|
||||
raise Cancelled()
|
||||
for name in ("calibration.json", "device.json"):
|
||||
if os.path.isfile(os.path.join(src, name)):
|
||||
shutil.copyfile(os.path.join(src, name), os.path.join(work, name))
|
||||
shown_profile = json.loads(json.dumps(profile))
|
||||
# the profile as the dataset sees it: the consent this session was recorded under, not the
|
||||
# window's own keys (the bounties done) nor the other consent's record
|
||||
shown_profile = json.loads(json.dumps({k: profile[k] for k in ("schema", "contributor", CONSENT_KEYS[kind],
|
||||
"optional") if k in profile}))
|
||||
if not keep_notes and isinstance(shown_profile.get("optional"), dict):
|
||||
shown_profile["optional"].pop("notes", None)
|
||||
manifest = {"schema": EXPORT_SCHEMA, "tool": tool_version(),
|
||||
"exported": datetime.datetime.now().astimezone().isoformat(timespec="seconds"),
|
||||
"session_id": session, "contributor": profile.get("contributor", meta.get("contributor", "")),
|
||||
"consent_version": (profile.get("consent") or {}).get("version", ""),
|
||||
"consent_kind": kind,
|
||||
"consent_version": (profile.get(CONSENT_KEYS[kind]) or {}).get("version", ""),
|
||||
"profile": shown_profile, "session": meta, "takes": take_entries}
|
||||
if recipients:
|
||||
manifest["encryption"] = {"scheme": "age-v1-x25519", "recipients": recipients, "files": encrypted}
|
||||
write_json(os.path.join(work, "manifest.json"), manifest)
|
||||
report(0.995, "Writing checksums")
|
||||
sums = []
|
||||
for root, _, files in os.walk(work):
|
||||
for name in files:
|
||||
path = os.path.join(root, name)
|
||||
sums.append((os.path.relpath(path, work), sha256_file(path, cancelled)))
|
||||
with open(os.path.join(work, "SHA256SUMS"), "w") as f:
|
||||
for rel, digest in sorted(sums):
|
||||
f.write(f"{digest} {rel}\n")
|
||||
write_sums(work, cancelled)
|
||||
|
||||
@staticmethod
|
||||
def _compress(index, keep, zstd, dest, step, cancelled):
|
||||
"""Stream the kept sets through zstd into dest. Returns the uncompressed size."""
|
||||
def _compress(index, keep, zstd, dest, step, cancelled, recipients=None):
|
||||
"""Stream the kept sets through zstd into dest, encrypted to recipients if given (age,
|
||||
as zstd writes). Returns the uncompressed size."""
|
||||
raw = 0
|
||||
with open(dest, "wb") as out:
|
||||
proc = subprocess.Popen(["nice", "-n", "19", zstd] + ZSTD_ARGS, stdin=subprocess.PIPE, stdout=out,
|
||||
stderr=subprocess.PIPE)
|
||||
proc = subprocess.Popen(["nice", "-n", "19", zstd] + ZSTD_ARGS, stdin=subprocess.PIPE,
|
||||
stdout=subprocess.PIPE if recipients else out, stderr=subprocess.PIPE)
|
||||
sealer, sealed = None, {}
|
||||
if recipients:
|
||||
def seal():
|
||||
try:
|
||||
age.encrypt_stream(proc.stdout, out, recipients)
|
||||
except BaseException as e: # reported below, after zstd
|
||||
sealed["error"] = e
|
||||
proc.stdout.read() # keep zstd from blocking on a full pipe
|
||||
sealer = threading.Thread(target=seal, name="handrec-seal", daemon=True)
|
||||
sealer.start()
|
||||
try:
|
||||
for i in keep:
|
||||
if cancelled():
|
||||
@@ -605,33 +713,23 @@ class Store:
|
||||
err = proc.stderr.read().decode(errors="replace").strip()
|
||||
if proc.wait() != 0:
|
||||
raise RuntimeError(f"zstd failed: {err or proc.returncode}")
|
||||
if sealer:
|
||||
sealer.join()
|
||||
if "error" in sealed:
|
||||
raise RuntimeError(f"encrypting failed: {sealed['error']}")
|
||||
except BaseException:
|
||||
proc.kill()
|
||||
proc.wait()
|
||||
if sealer:
|
||||
sealer.join(10)
|
||||
raise
|
||||
finally:
|
||||
proc.stderr.close()
|
||||
if proc.stdout:
|
||||
proc.stdout.close()
|
||||
return raw
|
||||
|
||||
|
||||
def sha256_file(path, cancelled=lambda: False):
|
||||
h = hashlib.sha256()
|
||||
with open(path, "rb") as f:
|
||||
while chunk := f.read(CHUNK):
|
||||
if cancelled():
|
||||
raise Cancelled()
|
||||
h.update(chunk)
|
||||
return h.hexdigest()
|
||||
|
||||
|
||||
def human_bytes(n):
|
||||
for unit in ("B", "KB", "MB", "GB"):
|
||||
if n < 1000 or unit == "GB":
|
||||
return f"{n:.0f} {unit}" if unit == "B" else f"{n:.1f} {unit}"
|
||||
n /= 1000
|
||||
return f"{n:.1f} TB"
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description="List, check and export hand recorder sessions.")
|
||||
ap.add_argument("--base", default=DEFAULT_BASE)
|
||||
|
||||
@@ -0,0 +1,6 @@
|
||||
# The hand test dataset's recipients (DESIGN.md "Testing"): while the bounties are in testing, a
|
||||
# bounty round's or testing session's export encrypts its recordings (sets.bin.zst, live.bin.zst)
|
||||
# to all of them (recorder/age.py). Key 1 is kept on the maintainer's training machine; key 2 is
|
||||
# the eye dataset's offline key 2.
|
||||
age1hfqk4za5q9klarrm2906c43k6fvxflwrruqmr24jvaugavyg9sfsuzal5c
|
||||
age1r0zlq7kcyg4dz4y3vkl9j36jw3zc6ly6reddh9ag5kfy6aw0fy6s5hepat
|
||||
@@ -0,0 +1,282 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Tests for the hand bounties (DESIGN.md "Bounties", "Scenes"): the shipped board and round
|
||||
scripts, the scene steps' acts (buttons and moves), and a dry-run round: its act and scene
|
||||
events in prompts.jsonl, session.json's bounty and sets a second, and the done mark.
|
||||
|
||||
python3 hands/rec/tests/test_bounties.py
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import random
|
||||
import shutil
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
import unittest
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
sys.path.insert(0, os.path.dirname(HERE))
|
||||
import session # noqa: E402
|
||||
from recorder import bounties # noqa: E402
|
||||
|
||||
WIN = {"id": "a", "at": [0, -0.15, -1.0], "size": [0.6, 0.4], "look": "grid:3x3"}
|
||||
|
||||
SCRIPT = {
|
||||
"version": 1, "name": "test", "record_hz": 30, "intro_s": 1, "between_s": 1,
|
||||
"welcome": {"title": "Test", "seconds": 1, "text": "A test."},
|
||||
"done": {"title": "Done", "seconds": 0, "text": "Done."},
|
||||
"stopped": {"title": "Stopped", "seconds": 0, "text": "Stopped."},
|
||||
"sections": [{"id": "taps", "title": "Taps", "kind": "scene", "intro": "Tap.", "prompts": [
|
||||
{"id": "one", "hands": "right", "text": "Tap the lit button.", "windows": [WIN],
|
||||
"acts": {"gen": "buttons", "every_s": 1.0, "lit_s": 0.6, "count": 3}},
|
||||
{"id": "two", "hands": "both", "text": "Hold the bar, tap the buttons.",
|
||||
"windows": [dict(WIN, id="b", look="plain"), dict(WIN, at=[0.5, 0, -1])],
|
||||
"steady": [{"win": "b", "lit": "bar", "gesture": "pinch", "label": "L"}],
|
||||
"acts": [{"s": 0.8, "win": "a", "lit": "button:4", "gesture": "tap"},
|
||||
{"s": 0.5, "win": "a", "gesture": "rest"},
|
||||
{"s": 0.8, "win": "a", "lit": "tl+br", "gesture": "pinch", "ghost": [0.1, 0, 0, 0, 0, 0, 1.2]}]}]}]}
|
||||
|
||||
|
||||
def step(p, seed="x"):
|
||||
"""One scene step through _scene_step, as build_plan does."""
|
||||
p = json.loads(json.dumps(p))
|
||||
session._scene_step({"id": "t"}, p, random.Random(seed))
|
||||
return p
|
||||
|
||||
|
||||
class ShippedTest(unittest.TestCase):
|
||||
"""The board and round scripts that ship with the recorder."""
|
||||
|
||||
def test_board(self):
|
||||
board = bounties.load(session.BOARD_PATH)
|
||||
self.assertIsNotNone(board, "bounties.json doesn't validate")
|
||||
ids = [b["id"] for b in board["bounties"]]
|
||||
self.assertEqual(ids, ["index-taps", "middle-taps", "pinches", "card-grabs"])
|
||||
for b in board["bounties"]:
|
||||
self.assertTrue(session.have_bounty_script(b["script"]), b["script"])
|
||||
|
||||
def test_scripts(self):
|
||||
"""Each round: hand size first, then a scene section of 100 to 180 s of acts (plus the
|
||||
countdowns) at 30 sets a second,
|
||||
every act's gesture and lit part valid (build_plan raises otherwise)."""
|
||||
board = bounties.load(session.BOARD_PATH)
|
||||
for b in board["bounties"]:
|
||||
script = session.load_script(session.bounty_script(b["script"]))
|
||||
self.assertEqual(script.get("record_hz"), 30, b["id"])
|
||||
plan, skipped = session.build_plan(script, {}, seed=1)
|
||||
self.assertEqual(skipped, [])
|
||||
self.assertEqual([s["kind"] for s in plan], ["prompts", "scene"], b["id"])
|
||||
scene = plan[1]
|
||||
seconds = sum(p["seconds"] for p in scene["prompts"])
|
||||
self.assertTrue(100 <= seconds <= 180, "%s: %.0f s" % (b["id"], seconds))
|
||||
gestures = set()
|
||||
for p in scene["prompts"]:
|
||||
self.assertTrue(p["id"].startswith(scene["id"] + "/"), p["id"])
|
||||
for a in p["scene"]["acts"]:
|
||||
gestures.add(a["gesture"])
|
||||
if a["gesture"] != "rest":
|
||||
self.assertNotEqual(a["lit"], "none", p["id"])
|
||||
want = {"index-taps": "tap", "middle-taps": "middle", "pinches": "pinch", "card-grabs": "card"}[b["id"]]
|
||||
self.assertIn(want, gestures)
|
||||
self.assertIn("nogo", gestures, "%s: no near misses" % b["id"])
|
||||
|
||||
def test_same_plan_for_a_seed(self):
|
||||
script = session.load_script(session.bounty_script("index-taps"))
|
||||
a = session.build_plan(script, {}, seed=7)[0][1]["prompts"]
|
||||
b = session.build_plan(script, {}, seed=7)[0][1]["prompts"]
|
||||
c = session.build_plan(script, {}, seed=8)[0][1]["prompts"]
|
||||
self.assertEqual([p["scene"]["acts"] for p in a], [p["scene"]["acts"] for p in b])
|
||||
self.assertNotEqual([p["scene"]["acts"] for p in a], [p["scene"]["acts"] for p in c])
|
||||
|
||||
|
||||
class SceneTest(unittest.TestCase):
|
||||
def test_buttons(self):
|
||||
p = step({"text": "x", "windows": [WIN], "acts": {"gen": "buttons", "every_s": 1.5, "count": 12,
|
||||
"mix": {"tap": 1, "nogo": 1}, "hands": ["left", "right"]}})
|
||||
acts = p["scene"]["acts"]
|
||||
self.assertEqual(len(acts), 24) # each lit act, then a rest
|
||||
lit, rests = acts[0::2], acts[1::2]
|
||||
self.assertTrue(all(a["gesture"] == "rest" and a["lit"] == "none" for a in rests))
|
||||
for a, b in zip(lit, lit[1:]):
|
||||
self.assertNotEqual(a["lit"], b["lit"]) # never the same button twice in a row
|
||||
for a in lit:
|
||||
self.assertRegex(a["lit"], r"^button:[0-8]$")
|
||||
self.assertEqual(a["label"][0], "L" if a["hand"] == "left" else "R")
|
||||
self.assertEqual(a["colour"], {"tap": "cyan", "nogo": "orange"}[a["gesture"]])
|
||||
self.assertAlmostEqual(p["seconds"], 12 * (session.CHOICE_MIN_S + 1.5 * 0.3)) # L/R: 2 s lit, the rest kept
|
||||
self.assertEqual(p["pose"], lit[0]["gesture"])
|
||||
|
||||
def test_buttons_need_a_grid(self):
|
||||
with self.assertRaises(ValueError):
|
||||
step({"text": "x", "windows": [dict(WIN, look="plain")], "acts": {"gen": "buttons", "count": 2}})
|
||||
|
||||
def test_moves(self):
|
||||
p = step({"text": "x", "windows": [WIN], "acts": {
|
||||
"gen": "moves", "gesture": "card", "grab": "right", "every_s": 3, "gap_s": 1, "shuffle": False,
|
||||
"moves": [[0.1, 0, 0], {"grab": "left+right", "ghost": [0, 0, 0, 30, 0, 0, 1.4], "hand": "both"},
|
||||
{"grab": "tr", "gesture": "nogo"}]}})
|
||||
acts = p["scene"]["acts"]
|
||||
self.assertEqual([a["gesture"] for a in acts], ["card", "rest", "card", "rest", "nogo", "rest"])
|
||||
self.assertEqual(acts[0]["ghost"], [0.1, 0, 0, 0, 0, 0])
|
||||
self.assertEqual(acts[2]["ghost"], [0, 0, 0, 30, 0, 0, 1.4])
|
||||
self.assertEqual(acts[2]["lit"], "left+right")
|
||||
self.assertIsNone(acts[4]["ghost"])
|
||||
self.assertEqual(acts[1]["say"], "Let go")
|
||||
self.assertAlmostEqual(p["seconds"], 12.0)
|
||||
|
||||
def test_hand_choice(self):
|
||||
"""Acts that switch hands get CHOICE_MIN_S each: the generator stretches them, written ones fail."""
|
||||
p = step({"text": "x", "windows": [WIN], "acts": {"gen": "buttons", "every_s": 1.25, "count": 6,
|
||||
"hands": ["left", "right"]}})
|
||||
lit = [a for a in p["scene"]["acts"] if a["gesture"] != "rest"]
|
||||
self.assertTrue(all(a["s"] >= session.CHOICE_MIN_S for a in lit))
|
||||
one = step({"text": "x", "windows": [WIN], "acts": {"gen": "buttons", "every_s": 1.25, "count": 6, "hands": ["right"]}})
|
||||
self.assertLess(one["scene"]["acts"][0]["s"], session.CHOICE_MIN_S) # one hand: no choice, no stretch
|
||||
with self.assertRaises(ValueError):
|
||||
step({"text": "x", "windows": [WIN], "acts": [{"s": 1, "hand": "left"}, {"s": 1, "hand": "right"}]})
|
||||
step({"text": "x", "windows": [WIN], "acts": [{"s": 2, "hand": "left"}, {"s": 0.5, "gesture": "rest"},
|
||||
{"s": 2, "hand": "right"}]})
|
||||
|
||||
def test_bad_steps(self):
|
||||
bad = [{"windows": [], "acts": [{"s": 1}]}, # no window
|
||||
{"windows": [WIN] * 2, "acts": [{"s": 1}]}, # same id twice
|
||||
{"windows": [dict(WIN, look="grid:9x9")], "acts": [{"s": 1}]}, # too many buttons
|
||||
{"windows": [WIN], "acts": [{"s": 1, "gesture": "wave"}]}, # unknown gesture
|
||||
{"windows": [WIN], "acts": [{"s": 1, "lit": "middle"}]}, # unknown part
|
||||
{"windows": [WIN], "acts": [{"s": 1, "win": "z"}]}, # unknown window
|
||||
{"windows": [WIN], "acts": [{"s": 1, "ghost": [1, 2, 3, 4, 5, 6, 7, 8]}]},
|
||||
{"windows": [WIN], "acts": []},
|
||||
{"windows": [WIN], "acts": [{"s": 1}], "steady": [{"win": "a", "lit": "bar"}]}] # steady on an act's window
|
||||
for p in bad:
|
||||
with self.assertRaises(ValueError, msg=json.dumps(p)):
|
||||
step(dict(p, text="x"))
|
||||
|
||||
|
||||
class RoundTest(unittest.TestCase):
|
||||
"""A dry-run bounty round of SCRIPT."""
|
||||
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.mkdtemp(prefix="handrec-bounty-test-")
|
||||
self.base = os.path.join(self.tmp, "base")
|
||||
os.makedirs(self.base)
|
||||
self.script = os.path.join(self.tmp, "test.json")
|
||||
with open(self.script, "w") as f:
|
||||
json.dump(SCRIPT, f)
|
||||
with open(os.path.join(self.base, "profile.json"), "w") as f:
|
||||
json.dump({"schema": 1, "contributor": "c", "bounties": {"test": {"rounds": 1, "last": "x"}}}, f)
|
||||
self.panel = []
|
||||
|
||||
def tearDown(self):
|
||||
shutil.rmtree(self.tmp, ignore_errors=True)
|
||||
|
||||
def run_round(self):
|
||||
s = session.Session(self.base, {}, {}, "room", self.script, dry_run=True, speed=10, next_after=0.02,
|
||||
poses_dir=os.path.join(self.tmp, "no-poses"), button=False,
|
||||
bounty={"id": "test", "title": "Test taps", "updated": "2026-10-14"})
|
||||
s.print = self.panel.append
|
||||
s.start()
|
||||
end = time.monotonic() + 30
|
||||
while s._thread.is_alive() and time.monotonic() < end:
|
||||
s.join(0.1)
|
||||
self.assertEqual(s.state, "done")
|
||||
return s
|
||||
|
||||
def test_round(self):
|
||||
s = self.run_round()
|
||||
with open(os.path.join(s.session_dir, "session.json")) as f:
|
||||
meta = json.load(f)
|
||||
self.assertEqual(meta["bounty"], {"id": "test", "title": "Test taps", "board_updated": "2026-10-14"})
|
||||
self.assertEqual(meta["record_hz"], 30)
|
||||
take = os.path.join(s.session_dir, "takes", os.listdir(os.path.join(s.session_dir, "takes"))[0])
|
||||
with open(os.path.join(take, "prompts.jsonl")) as f:
|
||||
events = [json.loads(line) for line in f]
|
||||
acts = [e for e in events if e.get("event") == "prompt" and "act" in e]
|
||||
# step one: 3 taps, each with a rest; step two: 3 acts
|
||||
self.assertEqual([a["act"]["gesture"] for a in acts], ["tap", "rest"] * 3 + ["tap", "rest", "pinch"])
|
||||
self.assertEqual([a["id"].split("/")[-1] for a in acts[6:]], ["1-tap", "2-rest", "3-pinch"])
|
||||
self.assertEqual(acts[-1]["act"]["ghost"], [0.1, 0, 0, 0, 0, 0, 1.2])
|
||||
self.assertIn("ghost_room", acts[-1]["act"])
|
||||
scenes = [e for e in events if e.get("event") == "scene"]
|
||||
self.assertEqual([len(e["windows"]) for e in scenes], [1, 0, 2, 0])
|
||||
steady = [e for e in events if e.get("event") == "steady"]
|
||||
self.assertEqual([(e["win"], e["lit"], e["gesture"]) for e in steady], [("b", "bar", "pinch")])
|
||||
# the panel: windows up and down, the lights, the band layout and back
|
||||
cmds = [c for c in self.panel if c.startswith("panel: win") or c.startswith("panel: layout")]
|
||||
self.assertIn("panel: win b lit bar cyan L", cmds)
|
||||
self.assertIn("panel: win a lit tl+br cyan", cmds)
|
||||
self.assertIn("panel: win a ghost 0.1 0 0 0 0 0 1.2", cmds)
|
||||
self.assertEqual(cmds[-1], "panel: layout full")
|
||||
|
||||
def test_done_mark(self):
|
||||
"""Only a real round counts: a dry run leaves the profile alone, _mark_done adds one."""
|
||||
s = self.run_round()
|
||||
path = os.path.join(self.base, "profile.json")
|
||||
with open(path) as f:
|
||||
self.assertEqual(json.load(f)["bounties"]["test"]["rounds"], 1)
|
||||
s._mark_done()
|
||||
with open(path) as f:
|
||||
mark = json.load(f)["bounties"]["test"]
|
||||
self.assertEqual(mark, {"rounds": 2, "last": os.path.basename(s.session_dir)})
|
||||
|
||||
|
||||
SWITCH_SCRIPT = dict(SCRIPT, sections=[
|
||||
{"id": "size", "title": "Size", "intro": "", "prompts": [
|
||||
{"text": "Palms toward you", "seconds": 2, "hands": "both",
|
||||
"switch": [{"at": 0.5, "to": "backs toward you", "text": "Backs toward you"}]}]},
|
||||
{"id": "taps", "title": "Taps", "kind": "scene", "intro": "Tap.", "prompts": [
|
||||
{"id": "one", "hands": "right", "text": "Tap.", "windows": [WIN], "switch": "elbow bent",
|
||||
"acts": {"gen": "buttons", "every_s": 1.0, "lit_s": 0.6, "count": 2}}]}])
|
||||
|
||||
|
||||
class SwitchTest(unittest.TestCase):
|
||||
"""Two poses in a step (DESIGN.md): the bar's tick, "Switch to ...", a "switch" event."""
|
||||
|
||||
def test_list(self):
|
||||
sec = {"id": "s"}
|
||||
self.assertEqual(session._switch_list(sec, None), [])
|
||||
self.assertEqual(session._switch_list(sec, "backs"), [{"at": 0.5, "to": "backs", "text": ""}])
|
||||
two = session._switch_list(sec, [{"at": 0.3, "to": "a"}, {"at": 0.6, "to": "b", "text": "B"}])
|
||||
self.assertEqual([(w["at"], w["to"], w["text"]) for w in two], [(0.3, "a", ""), (0.6, "b", "B")])
|
||||
for bad in ([{"at": 0.6, "to": "a"}, {"at": 0.3, "to": "b"}], [{"at": 1, "to": "a"}], [{"at": 0.5}],
|
||||
[{"at": 0.5, "to": " "}], [{"at": 0.5, "to": "a", "text": 3}], 5):
|
||||
with self.assertRaises(ValueError, msg=repr(bad)):
|
||||
session._switch_list(sec, bad)
|
||||
|
||||
def test_round(self):
|
||||
tmp = tempfile.mkdtemp(prefix="handrec-switch-test-")
|
||||
self.addCleanup(shutil.rmtree, tmp, True)
|
||||
base, script = os.path.join(tmp, "base"), os.path.join(tmp, "test.json")
|
||||
os.makedirs(base)
|
||||
with open(script, "w") as f:
|
||||
json.dump(SWITCH_SCRIPT, f)
|
||||
panel = []
|
||||
s = session.Session(base, {}, {}, "room", script, dry_run=True, speed=10, next_after=0.02,
|
||||
poses_dir=os.path.join(tmp, "no-poses"), button=False)
|
||||
s.print = panel.append
|
||||
s.start()
|
||||
end = time.monotonic() + 30
|
||||
while s._thread.is_alive() and time.monotonic() < end:
|
||||
s.join(0.1)
|
||||
self.assertEqual(s.state, "done")
|
||||
events = []
|
||||
takes = os.path.join(s.session_dir, "takes")
|
||||
for t in sorted(os.listdir(takes)):
|
||||
with open(os.path.join(takes, t, "prompts.jsonl")) as f:
|
||||
events += [json.loads(line) for line in f]
|
||||
switches = [e for e in events if e.get("event") == "switch"]
|
||||
self.assertEqual([(e["id"], e["to"]) for e in switches], [("size/both", "backs toward you"), ("taps/one", "elbow bent")])
|
||||
size = [e for e in events if e.get("event") == "prompt" and e["id"] == "size/both"]
|
||||
self.assertEqual(size[0]["switch"][0]["to"], "backs toward you")
|
||||
# the switch comes between the step's prompt event and its end
|
||||
k = events.index(switches[0])
|
||||
self.assertLess(events.index(size[0]), k)
|
||||
self.assertEqual(events[k + 1]["event"], "wait")
|
||||
cmds = [c for c in panel if c.startswith("panel: split") or c.startswith("panel: text Backs")]
|
||||
self.assertIn("panel: split 0.500 backs toward you", cmds)
|
||||
self.assertIn("panel: split 0.500 elbow bent", cmds)
|
||||
self.assertIn("panel: text Backs toward you", cmds)
|
||||
self.assertEqual(cmds[-1], "panel: split off")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,657 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Debug sessions (DESIGN.md, "Debug sessions"): livelog.py's copies of the live tracker's
|
||||
files, and debug.json's free step in a dry run (marks, finishing early, session.json's record).
|
||||
|
||||
python3 hands/rec/tests/test_debug.py
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import struct
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
import unittest
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
REC = os.path.dirname(HERE)
|
||||
sys.path.insert(0, REC)
|
||||
sys.path.insert(0, HERE) # test_validate, also under python -I
|
||||
import session # noqa: E402
|
||||
import livelog # noqa: E402
|
||||
import voicelog # noqa: E402
|
||||
import eventlog # noqa: E402
|
||||
|
||||
|
||||
class FakeTracker:
|
||||
"""Publishes hands and gestures files in dir under the sequence lock, every period_s."""
|
||||
|
||||
def __init__(self, d, period_s=0.01):
|
||||
self.d, self.period_s, self.seq = d, period_s, 0
|
||||
self.files = {"hands": (b"FHHANDS1", 2, 4704), "gestures": (b"FHGEST01", 3, 448)}
|
||||
for name in self.files:
|
||||
self._write(name, 0)
|
||||
self._stop = threading.Event()
|
||||
self._thread = threading.Thread(target=self._run, daemon=True)
|
||||
|
||||
def _write(self, name, seq):
|
||||
magic, version, size = self.files[name]
|
||||
head = struct.pack("<8sIIQQQ", magic, version, size, seq, seq * 1000, session.mono_ns())
|
||||
with open(os.path.join(self.d, name), "r+b" if os.path.exists(os.path.join(self.d, name)) else "wb") as f:
|
||||
f.write(head + bytes([seq // 2 % 256]) * (size - len(head)))
|
||||
|
||||
def _run(self):
|
||||
while not self._stop.is_set():
|
||||
self.seq += 2
|
||||
for name in self.files:
|
||||
self._write(name, self.seq)
|
||||
time.sleep(self.period_s)
|
||||
|
||||
def __enter__(self):
|
||||
self._thread.start()
|
||||
return self
|
||||
|
||||
def __exit__(self, *_):
|
||||
self._stop.set()
|
||||
self._thread.join()
|
||||
|
||||
|
||||
class LiveLogTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.mkdtemp()
|
||||
self.addCleanup(shutil.rmtree, self.tmp)
|
||||
|
||||
def test_copies(self):
|
||||
path = os.path.join(self.tmp, "live.bin")
|
||||
with FakeTracker(self.tmp):
|
||||
log = livelog.LiveLog(self.tmp, path).start()
|
||||
time.sleep(0.3)
|
||||
counts = log.stop()
|
||||
recs = livelog.read(path)
|
||||
self.assertGreater(counts["hands"], 5)
|
||||
self.assertEqual(len(recs), counts["hands"] + counts["gestures"])
|
||||
for name in ("hands", "gestures"):
|
||||
mine = [r for r in recs if r[0] == name]
|
||||
seqs = [r[2] for r in mine]
|
||||
self.assertEqual(seqs, sorted(set(seqs))) # each publish once, in order
|
||||
self.assertTrue(all(s % 2 == 0 for s in seqs))
|
||||
self.assertEqual({len(r[3]) for r in mine}, {4704 if name == "hands" else 448})
|
||||
for _, _, seq, data in mine: # a whole copy of that publish, not half of the next
|
||||
self.assertEqual(struct.unpack_from("<Q", data, 16)[0], seq)
|
||||
self.assertEqual(data[-1], seq // 2 % 256)
|
||||
|
||||
def test_odd_and_missing(self):
|
||||
"""No file yet, or one being written (odd seq): nothing copied."""
|
||||
path = os.path.join(self.tmp, "live.bin")
|
||||
with open(os.path.join(self.tmp, "hands"), "wb") as f:
|
||||
f.write(struct.pack("<8sIIQ", b"FHHANDS1", 2, 4704, 7) + bytes(4704 - 24))
|
||||
counts = livelog.LiveLog(self.tmp, path).start()
|
||||
time.sleep(0.05)
|
||||
self.assertEqual(counts.stop(), {"hands": 0, "gestures": 0})
|
||||
self.assertEqual(livelog.read(path), [])
|
||||
|
||||
|
||||
FAKE_PW_RECORD = """#!/bin/sh
|
||||
# 0.5 s of a 1000-step ramp, then waits to be stopped
|
||||
python3 -c 'import struct, sys; sys.stdout.buffer.write(struct.pack("<8000h", *range(-4000, 4000)))'
|
||||
exec sleep 30
|
||||
"""
|
||||
|
||||
|
||||
class VoiceLogTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.mkdtemp()
|
||||
self.addCleanup(shutil.rmtree, self.tmp)
|
||||
self.path = os.environ.get("PATH", "")
|
||||
self.addCleanup(os.environ.__setitem__, "PATH", self.path)
|
||||
|
||||
def fake(self, script):
|
||||
bindir = os.path.join(self.tmp, "bin")
|
||||
os.makedirs(bindir, exist_ok=True)
|
||||
exe = os.path.join(bindir, "pw-record")
|
||||
with open(exe, "w") as f:
|
||||
f.write(script)
|
||||
os.chmod(exe, 0o755)
|
||||
os.environ["PATH"] = bindir + os.pathsep + self.path
|
||||
|
||||
def test_records(self):
|
||||
self.fake(FAKE_PW_RECORD)
|
||||
path = os.path.join(self.tmp, "voice.bin")
|
||||
log = voicelog.VoiceLog(path).start()
|
||||
end = time.monotonic() + 5
|
||||
while log.samples < 8000 and time.monotonic() < end:
|
||||
time.sleep(0.02)
|
||||
counts = log.stop()
|
||||
self.assertEqual(counts["error"], "")
|
||||
self.assertEqual(counts["seconds"], 0.5)
|
||||
self.assertEqual(counts["peak"], round(4000 / 32768, 3))
|
||||
rate, recs = voicelog.read(path)
|
||||
self.assertEqual(rate, 16000)
|
||||
self.assertEqual(len(recs), counts["chunks"])
|
||||
self.assertEqual(sum(len(d) for _, d in recs), 16000)
|
||||
self.assertEqual(recs, sorted(recs, key=lambda r: r[0]))
|
||||
wav = os.path.join(self.tmp, "voice.wav")
|
||||
voicelog.to_wav(path, wav)
|
||||
import wave
|
||||
with wave.open(wav) as w:
|
||||
self.assertEqual(w.getframerate(), 16000)
|
||||
self.assertGreaterEqual(w.getnframes(), 8000)
|
||||
|
||||
def test_no_microphone(self):
|
||||
"""No pw-record, or one that fails at once: an error to report, nothing raised."""
|
||||
os.environ["PATH"] = os.path.join(self.tmp, "empty")
|
||||
counts = voicelog.VoiceLog(os.path.join(self.tmp, "voice.bin")).start().stop()
|
||||
self.assertIn("pw-record", counts["error"])
|
||||
self.fake("#!/bin/sh\nexit 1\n")
|
||||
counts = voicelog.VoiceLog(os.path.join(self.tmp, "voice.bin")).start()
|
||||
time.sleep(0.2)
|
||||
counts = counts.stop()
|
||||
self.assertEqual(counts["seconds"], 0)
|
||||
self.assertIn("exited", counts["error"])
|
||||
self.assertEqual(voicelog.read(os.path.join(self.tmp, "voice.bin")), (16000, []))
|
||||
|
||||
|
||||
class VoiceFlagsTest(unittest.TestCase):
|
||||
"""hands/tools/voice_flags.py: a testing take's marks and spoken notes, with the events around them."""
|
||||
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.mkdtemp()
|
||||
self.addCleanup(shutil.rmtree, self.tmp)
|
||||
sys.path.insert(0, os.path.join(os.path.dirname(REC), "tools"))
|
||||
self.addCleanup(sys.path.remove, sys.path[0])
|
||||
import voice_flags
|
||||
self.vf = voice_flags
|
||||
|
||||
def test_loopback_only(self):
|
||||
for url in ("http://127.0.0.1:8178", "http://localhost:8178", "http://[::1]:8178"):
|
||||
self.assertTrue(self.vf.loopback(url), url)
|
||||
for url in ("http://192.168.1.5:8178", "https://127.0.0.1:8178", "http://example.com", "127.0.0.1:8178"):
|
||||
self.assertFalse(self.vf.loopback(url), url)
|
||||
|
||||
def test_bursts(self):
|
||||
"""Speech over the noise floor, short pauses kept in, silence left out."""
|
||||
import array
|
||||
rate = 16000
|
||||
noise, loud = [3, -3] * (rate // 2), [3000, -3000] * (rate // 2)
|
||||
samples = array.array("h", noise * 2 + loud + noise[:rate // 2] + loud + noise * 3 + loud)
|
||||
got = self.vf.bursts(samples, rate)
|
||||
pad = int(self.vf.PAD_S * rate)
|
||||
self.assertEqual(got, [(2 * rate - pad, 4.5 * rate + pad), (7.5 * rate - pad, 8.5 * rate)])
|
||||
|
||||
def test_flags(self):
|
||||
"""Marks and notes in time order, a note paired with a mark under PAIR_S away, events around each."""
|
||||
take = os.path.join(self.tmp, "01-free")
|
||||
os.makedirs(take)
|
||||
rate, t0 = voicelog.RATE, 1_000_000_000_000
|
||||
with open(os.path.join(take, "take.json"), "w") as f:
|
||||
json.dump({"started_ns": t0}, f)
|
||||
with open(os.path.join(take, "prompts.jsonl"), "w") as f:
|
||||
f.write(json.dumps({"t": t0 + 11 * 10**9, "event": "mark", "id": "use", "n": 1}) + "\n")
|
||||
f.write(json.dumps({"t": t0 + 40 * 10**9, "event": "mark", "id": "use", "n": 2}) + "\n")
|
||||
with open(os.path.join(take, "events.jsonl"), "w") as f:
|
||||
f.write(json.dumps({"t": t0 + 9 * 10**9, "kind": "poke", "outcome": "missed"}) + "\n")
|
||||
f.write(json.dumps({"t": t0 + 30 * 10**9, "kind": "grab", "outcome": "done"}) + "\n")
|
||||
with open(os.path.join(take, "voice.bin"), "wb") as f: # speech 10-11 s and 20-21 s
|
||||
f.write(voicelog.HDR.pack(voicelog.MAGIC, 1, rate))
|
||||
for k in range(1, 31):
|
||||
loud = k in (11, 21)
|
||||
f.write(voicelog.REC.pack(t0 + k * 10**9, rate * 2))
|
||||
f.write(struct.pack("<h", 3000 if loud else 0) * rate)
|
||||
said = iter(["that poke missed", "fine now"])
|
||||
orig = self.vf.transcribe
|
||||
self.vf.transcribe = lambda wav, server: [(0.0, 1.0, next(said))]
|
||||
self.addCleanup(setattr, self.vf, "transcribe", orig)
|
||||
out = self.vf.flags(take)
|
||||
self.assertEqual([(f["source"], round(f["s"])) for f in out],
|
||||
[("voice", 10), ("mark", 11), ("voice", 20), ("mark", 40)])
|
||||
self.assertEqual(out[0]["text"], "that poke missed")
|
||||
self.assertEqual(out[0]["mark"], 1)
|
||||
self.assertIsNone(out[2]["mark"])
|
||||
self.assertEqual([e["kind"] for e in out[0]["events"]], ["poke"])
|
||||
self.assertEqual(out[3]["events"], [])
|
||||
self.assertIn('voice (mark 1): "that poke missed" [poke 1 missed]', self.vf.line(out[0]))
|
||||
|
||||
|
||||
class SettingsTest(unittest.TestCase):
|
||||
def test_settings(self):
|
||||
tmp = tempfile.mkdtemp()
|
||||
self.addCleanup(shutil.rmtree, tmp)
|
||||
conf = os.path.join(tmp, "frametop.conf")
|
||||
with open(conf, "w") as f:
|
||||
f.write("HANDS_MODELS=/home/someone/models/student-v13b-p1 # a note\n"
|
||||
"POINTER_ROLE=right\n# HANDS_MODE=off\nGAZE_EYE=both\nHANDS_CPUS='5,6,7'\n")
|
||||
self.assertEqual(session.live_settings(conf), {"HANDS_MODELS": "student-v13b-p1", "POINTER_ROLE": "right",
|
||||
"HANDS_CPUS": "5,6,7"})
|
||||
|
||||
|
||||
class DebugSessionTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.mkdtemp()
|
||||
self.addCleanup(shutil.rmtree, self.tmp)
|
||||
self.live = os.path.join(self.tmp, "live")
|
||||
os.makedirs(self.live)
|
||||
self.panel = []
|
||||
|
||||
def run_debug(self, act):
|
||||
"""A dry debug session; act(s) runs once its free step is running."""
|
||||
s = session.Session(os.path.join(self.tmp, "base"), {}, {}, "room", session.DEBUG_SCRIPT, dry_run=True,
|
||||
speed=5, next_after=0.02, poses_dir=os.path.join(self.tmp, "no-poses"), button=False)
|
||||
self.assertTrue(s.debug)
|
||||
self.assertEqual(s.quiet_units, ()) # Frametop's hands and pointer stay on: they're what's tested
|
||||
s.live_dir = self.live
|
||||
s.print = self.panel.append
|
||||
with FakeTracker(self.live):
|
||||
s.start()
|
||||
end = time.monotonic() + 30
|
||||
while s.state != "running" and time.monotonic() < end:
|
||||
time.sleep(0.01)
|
||||
time.sleep(0.1)
|
||||
act(s)
|
||||
while s._thread.is_alive() and time.monotonic() < end:
|
||||
s.join(0.1)
|
||||
return s
|
||||
|
||||
def events(self, s):
|
||||
with open(os.path.join(s.session_dir, "takes", "01-free", "prompts.jsonl")) as f:
|
||||
return [json.loads(line) for line in f]
|
||||
|
||||
def test_marks_and_finish(self):
|
||||
def act(s):
|
||||
s.button_gesture("press")
|
||||
time.sleep(0.1)
|
||||
s.mark()
|
||||
time.sleep(0.1)
|
||||
s.button_gesture("hold") # finish, not stop
|
||||
s = self.run_debug(act)
|
||||
self.assertEqual(s.state, "done")
|
||||
ev = self.events(s)
|
||||
self.assertEqual([e["n"] for e in ev if e["event"] == "mark"], [1, 2])
|
||||
self.assertEqual([e["event"] for e in ev][-3:], ["end_early", "wait", "end"])
|
||||
self.assertIn("panel: hide", self.panel)
|
||||
self.assertIn("panel: big Marked", self.panel)
|
||||
with open(os.path.join(s.session_dir, "session.json")) as f:
|
||||
sj = json.load(f)
|
||||
self.assertEqual(sj["status"], "done")
|
||||
self.assertEqual(sj["debug"]["use"], "testing")
|
||||
self.assertEqual(sj["record_hz"], 30)
|
||||
with open(os.path.join(s.session_dir, "takes", "01-free", "take.json")) as f:
|
||||
tj = json.load(f)
|
||||
self.assertEqual(tj["status"], "complete")
|
||||
self.assertGreater(tj["live"]["live.bin"]["hands"], 3)
|
||||
recs = livelog.read(os.path.join(s.session_dir, "takes", "01-free", "live.bin"))
|
||||
self.assertEqual(len(recs), tj["live"]["live.bin"]["hands"] + tj["live"]["live.bin"]["gestures"])
|
||||
|
||||
def test_double_press_marks(self):
|
||||
"""Two quick presses mark (twice pressed, one moment): never a redo, which would throw the
|
||||
recording away (2026-10-14: a testing session lost its recording that way)."""
|
||||
def act(s):
|
||||
s.button_gesture("double")
|
||||
time.sleep(0.1)
|
||||
s.redo() # the window's R does nothing either
|
||||
time.sleep(0.1)
|
||||
s.button_gesture("hold")
|
||||
s = self.run_debug(act)
|
||||
self.assertEqual(s.state, "done")
|
||||
ev = self.events(s)
|
||||
self.assertEqual([e["n"] for e in ev if e["event"] == "mark"], [1])
|
||||
self.assertNotIn("redo", [e["event"] for e in ev])
|
||||
with open(os.path.join(s.session_dir, "takes", "01-free", "take.json")) as f:
|
||||
self.assertEqual(sorted(json.load(f)["live"]), ["live.bin"])
|
||||
|
||||
def test_pause_parts(self):
|
||||
"""A pause ends the recording part, and its live log with it: live-2.bin for the next."""
|
||||
def act(s):
|
||||
s.pause()
|
||||
time.sleep(0.2)
|
||||
s.resume()
|
||||
time.sleep(0.2)
|
||||
s.finish()
|
||||
s = self.run_debug(act)
|
||||
self.assertEqual(s.state, "done")
|
||||
with open(os.path.join(s.session_dir, "takes", "01-free", "take.json")) as f:
|
||||
self.assertEqual(sorted(json.load(f)["live"]), ["live-2.bin", "live.bin"])
|
||||
|
||||
def test_voice(self):
|
||||
"""Ticked "Record my voice": each recording part has its voice file beside the live log."""
|
||||
made = []
|
||||
|
||||
class FakeVoice:
|
||||
def __init__(self, path):
|
||||
self.path, self.error = path, ""
|
||||
made.append(os.path.basename(path))
|
||||
|
||||
def start(self):
|
||||
with open(self.path, "wb") as f:
|
||||
f.write(voicelog.HDR.pack(voicelog.MAGIC, 1, voicelog.RATE))
|
||||
return self
|
||||
|
||||
def stop(self):
|
||||
return {"seconds": 1.0, "chunks": 10, "peak": 0.2, "error": ""}
|
||||
|
||||
orig = session.voicelog.VoiceLog
|
||||
session.voicelog.VoiceLog = FakeVoice
|
||||
self.addCleanup(setattr, session.voicelog, "VoiceLog", orig)
|
||||
|
||||
def act(s):
|
||||
s.pause()
|
||||
time.sleep(0.2)
|
||||
s.resume()
|
||||
time.sleep(0.2)
|
||||
s.finish()
|
||||
orig_init = session.Session.__init__
|
||||
|
||||
def init(s, *a, **k):
|
||||
orig_init(s, *a, **k)
|
||||
self.assertFalse(s.voice) # a dry run records no voice, even ticked
|
||||
s.voice = True
|
||||
session.Session.__init__ = init
|
||||
self.addCleanup(setattr, session.Session, "__init__", orig_init)
|
||||
s = self.run_debug(act)
|
||||
self.assertEqual(s.state, "done")
|
||||
self.assertEqual(made, ["voice.bin", "voice-2.bin"])
|
||||
with open(os.path.join(s.session_dir, "takes", "01-free", "take.json")) as f:
|
||||
self.assertEqual(sorted(json.load(f)["voice"]), ["voice-2.bin", "voice.bin"])
|
||||
|
||||
def test_events(self):
|
||||
"""The panels' and pointer's events (fh_events.h) while each part records go into its
|
||||
events file: events-want asked for them, an older line is left out, and nothing is asked
|
||||
for after the session."""
|
||||
want = os.path.join(self.live, eventlog.WANT)
|
||||
src = os.path.join(self.live, eventlog.FILE)
|
||||
with open(src, "w") as f: # a crashed session's
|
||||
f.write('{"t": 1, "src": "screens", "kind": "poke", "outcome": "done", "hand": "right", "panel": 1, "text": "old"}\n')
|
||||
|
||||
def write(kind, outcome):
|
||||
self.assertTrue(os.path.exists(want))
|
||||
with open(src, "a") as f:
|
||||
f.write(json.dumps({"t": eventlog.mono_ns(), "src": "screens", "kind": kind, "outcome": outcome,
|
||||
"hand": "right", "panel": 2, "text": kind}) + "\n")
|
||||
f.write("not json\n")
|
||||
|
||||
def act(s):
|
||||
write("lazy-susan", "done")
|
||||
write("poke", "refused")
|
||||
s.pause()
|
||||
time.sleep(0.2)
|
||||
self.assertFalse(os.path.exists(want))
|
||||
s.resume()
|
||||
time.sleep(0.2)
|
||||
write("poke", "done")
|
||||
s.finish()
|
||||
s = self.run_debug(act)
|
||||
self.assertEqual(s.state, "done")
|
||||
self.assertFalse(os.path.exists(want))
|
||||
self.assertFalse(os.path.exists(src))
|
||||
tdir = os.path.join(s.session_dir, "takes", "01-free")
|
||||
self.assertEqual([e["kind"] for e in eventlog.read_lines(os.path.join(tdir, "events.jsonl"))], ["lazy-susan", "poke"])
|
||||
self.assertEqual([e["outcome"] for e in eventlog.read_lines(os.path.join(tdir, "events-2.jsonl"))], ["done"])
|
||||
with open(os.path.join(tdir, "take.json")) as f:
|
||||
tj = json.load(f)
|
||||
self.assertEqual(tj["events"]["events.jsonl"], {"events": 2, "by": {"lazy-susan": {"done": 1}, "poke": {"refused": 1}}})
|
||||
self.assertEqual(tj["events"]["events-2.jsonl"]["by"], {"poke": {"done": 1}})
|
||||
|
||||
def test_needs_live_hands(self):
|
||||
s = session.Session(os.path.join(self.tmp, "base"), {}, {}, "room", session.DEBUG_SCRIPT, button=False)
|
||||
s.dry_run = False # the check, without starting anything
|
||||
orig = session.unit_active
|
||||
session.unit_active = lambda unit: False
|
||||
try:
|
||||
self.assertTrue(s._preflight())
|
||||
finally:
|
||||
session.unit_active = orig
|
||||
self.assertEqual(s.state, "error")
|
||||
self.assertEqual(s._status["error"], session.DEBUG_NEEDS_HANDS)
|
||||
|
||||
|
||||
class DebugExportTest(unittest.TestCase):
|
||||
"""A debug session's export: its recordings encrypted to the debug keys, the live parts as one
|
||||
live.bin (deleted ranges left out), and validate.py's checks of it."""
|
||||
|
||||
def setUp(self):
|
||||
import takes
|
||||
import validate
|
||||
from recorder import age
|
||||
from test_validate import SESSION, make_session
|
||||
try:
|
||||
self.identity, recipient = age.keygen()
|
||||
except RuntimeError as e:
|
||||
self.skipTest(str(e))
|
||||
if not takes.find_zstd():
|
||||
self.skipTest("no zstd")
|
||||
self.takes, self.validate, self.age, self.sid = takes, validate, age, SESSION
|
||||
self.tmp = tempfile.mkdtemp()
|
||||
self.addCleanup(shutil.rmtree, self.tmp)
|
||||
keys = os.path.join(self.tmp, "recipients.txt")
|
||||
with open(keys, "w") as f:
|
||||
f.write("# test\n%s\n" % recipient)
|
||||
for mod in (takes, validate):
|
||||
orig = mod.TEST_RECIPIENTS
|
||||
mod.TEST_RECIPIENTS = keys
|
||||
self.addCleanup(setattr, mod, "TEST_RECIPIENTS", orig)
|
||||
self.base = os.path.join(self.tmp, "base")
|
||||
make_session(self.base)
|
||||
# recorded under the testing consent (TESTING.md), with the window's own profile keys
|
||||
profile = takes.read_json(os.path.join(self.base, "profile.json"))
|
||||
profile["testing_consent"] = dict(profile.pop("consent"), version="2026-10-14", region_ok=True)
|
||||
profile["bounties"] = {"index-taps": {"done": 1}}
|
||||
takes.write_json(os.path.join(self.base, "profile.json"), profile)
|
||||
sdir = os.path.join(self.base, "sessions", SESSION)
|
||||
self.session_json = os.path.join(sdir, "session.json")
|
||||
sj = takes.read_json(self.session_json)
|
||||
sj["debug"] = {"use": "testing", "max_s": 180, "settings": {"POINTER_ROLE": "right"}, "frametop": "", "models": {}}
|
||||
sj["consent"] = {"kind": "testing", "version": "2026-10-14"}
|
||||
takes.write_json(self.session_json, sj)
|
||||
self.tdir = os.path.join(sdir, "takes", "01-hand-size")
|
||||
t0 = 10 ** 12
|
||||
for name, times in (("live.bin", range(0, 10)), ("live-2.bin", range(10, 20))):
|
||||
with open(os.path.join(self.tdir, name), "wb") as f:
|
||||
f.write(livelog.FILE_HDR.pack(livelog.MAGIC, livelog.VERSION, 0))
|
||||
for i in times:
|
||||
for src, n in ((0, 4704), (1, 448)):
|
||||
f.write(livelog.REC_HDR.pack(src, n, t0 + i * 100_000_000, 2 * i))
|
||||
f.write(bytes([i]) * n)
|
||||
f.write(livelog.REC_HDR.pack(0, 4704, t0, 99)[:10]) # cut off mid-write
|
||||
self.store = takes.Store(self.base)
|
||||
index = takes.take_index(self.tdir)
|
||||
self.store.delete_range(SESSION, "01-hand-size", index.time_ns(3), index.time_ns(5))
|
||||
|
||||
def test_export(self):
|
||||
export = self.store.export(self.sid, low_priority=False)
|
||||
files = sorted(os.path.relpath(os.path.join(r, n), export) for r, _, ns in os.walk(export) for n in ns)
|
||||
self.assertNotIn("takes/01-hand-size/sets.bin.zst", files)
|
||||
self.assertIn("takes/01-hand-size/sets.bin.zst.age", files)
|
||||
self.assertIn("takes/01-hand-size/live.bin.zst.age", files)
|
||||
self.assertNotIn("takes/02-no-hands/live.bin.zst.age", files) # no live parts there
|
||||
with open(os.path.join(export, "manifest.json")) as f:
|
||||
m = json.load(f)
|
||||
self.assertEqual(sorted(m["encryption"]["files"]), ["takes/01-hand-size/live.bin.zst.age",
|
||||
"takes/01-hand-size/sets.bin.zst.age",
|
||||
"takes/02-no-hands/sets.bin.zst.age"])
|
||||
live = m["takes"][0]["live"]
|
||||
self.assertEqual(live["records"], {"hands": 17, "gestures": 17}) # 3 of 20 in the deleted range
|
||||
r = self.validate.validate(export)
|
||||
self.assertEqual(r.errors, [])
|
||||
self.assertTrue(r.summary["debug"])
|
||||
self.assertEqual(m["consent_kind"], "testing")
|
||||
self.assertEqual(m["consent_version"], "2026-10-14")
|
||||
self.assertEqual(sorted(m["profile"]), ["contributor", "optional", "schema", "testing_consent"])
|
||||
# the key holder gets the recordings back
|
||||
import subprocess
|
||||
with open(os.path.join(export, live["file"]), "rb") as f:
|
||||
raw = subprocess.run([self.takes.find_zstd(), "-dc"], input=self.age.decrypt_bytes(f.read(), self.identity),
|
||||
capture_output=True, check=True).stdout
|
||||
out = os.path.join(self.tmp, "live.bin")
|
||||
with open(out, "wb") as f:
|
||||
f.write(raw)
|
||||
recs = livelog.read(out)
|
||||
self.assertEqual(len(recs), 34)
|
||||
self.assertEqual(len(raw), live["raw_bytes"])
|
||||
with open(os.path.join(export, "takes/02-no-hands/sets.bin.zst.age"), "rb") as f:
|
||||
sets = subprocess.run([self.takes.find_zstd(), "-dc"], input=self.age.decrypt_bytes(f.read(), self.identity),
|
||||
capture_output=True, check=True).stdout
|
||||
self.assertEqual(len(sets), m["takes"][1]["raw_bytes"])
|
||||
|
||||
def write_voice(self):
|
||||
"""voice.bin and voice-2.bin around the deleted range: a chunk before it, one ending in it,
|
||||
one inside, one starting in it, one after; the second file also cut off mid-write."""
|
||||
a, b = self.takes.read_json(os.path.join(self.tdir, "take.json"))["deleted"][0]
|
||||
ms = 1_000_000
|
||||
chunk = bytes(range(64)) * 50 # 3200 bytes: 100 ms at 16 kHz
|
||||
ends = [(a - 500 * ms, a + 50 * ms, (a + b) // 2), (min(b, a + 50 * ms) + 120 * ms, b + 500 * ms)]
|
||||
for name, part in zip(("voice.bin", "voice-2.bin"), ends):
|
||||
with open(os.path.join(self.tdir, name), "wb") as f:
|
||||
f.write(voicelog.HDR.pack(voicelog.MAGIC, 1, 16000))
|
||||
for t in part:
|
||||
f.write(voicelog.REC.pack(t, len(chunk)) + chunk)
|
||||
if name == "voice-2.bin":
|
||||
f.write(voicelog.REC.pack(b + 900 * ms, len(chunk)) + chunk[:100])
|
||||
# chunks: kept, ends in range (gone), inside (gone), starts in range (gone if b - a > 70 ms), kept
|
||||
return 2 if b - a > 70 * ms else 3
|
||||
|
||||
def test_voice(self):
|
||||
kept = self.write_voice()
|
||||
export = self.store.export(self.sid, low_priority=False)
|
||||
with open(os.path.join(export, "manifest.json")) as f:
|
||||
m = json.load(f)
|
||||
voice = m["takes"][0]["voice"]
|
||||
self.assertEqual(voice["file"], "takes/01-hand-size/voice.bin.zst.age")
|
||||
self.assertIn(voice["file"], m["encryption"]["files"])
|
||||
self.assertNotIn("voice", m["takes"][1])
|
||||
self.assertEqual(voice["chunks"], kept)
|
||||
self.assertEqual(voice["seconds"], kept / 10)
|
||||
r = self.validate.validate(export)
|
||||
self.assertEqual(r.errors, [])
|
||||
import subprocess
|
||||
with open(os.path.join(export, voice["file"]), "rb") as f:
|
||||
raw = subprocess.run([self.takes.find_zstd(), "-dc"], input=self.age.decrypt_bytes(f.read(), self.identity),
|
||||
capture_output=True, check=True).stdout
|
||||
out = os.path.join(self.tmp, "voice.bin")
|
||||
with open(out, "wb") as f:
|
||||
f.write(raw)
|
||||
self.assertEqual(len(raw), voice["raw_bytes"])
|
||||
rate, recs = voicelog.read(out)
|
||||
self.assertEqual((rate, len(recs)), (16000, kept))
|
||||
# not ticked as a debug session's, or encrypted
|
||||
files = sorted(os.path.relpath(os.path.join(r, n), export) for r, _, ns in os.walk(export) for n in ns)
|
||||
self.assertFalse(any(f.endswith("voice.bin.zst") for f in files))
|
||||
m["takes"][0]["voice"]["file"] = "takes/01-hand-size/voice.bin.zst"
|
||||
m["session"].pop("debug")
|
||||
self.takes.write_json(os.path.join(export, "manifest.json"), m)
|
||||
from test_validate import write_sums
|
||||
write_sums(export)
|
||||
r = self.validate.validate(export)
|
||||
self.assertTrue(any("voice recording" in e for e in r.errors), r.errors)
|
||||
|
||||
def test_events(self):
|
||||
"""The events parts as one file, those in the deleted range left out, only ever encrypted."""
|
||||
a, b = self.takes.read_json(os.path.join(self.tdir, "take.json"))["deleted"][0]
|
||||
for name, times in (("events.jsonl", (a - 1, (a + b) // 2)), ("events-2.jsonl", (b + 1,))):
|
||||
with open(os.path.join(self.tdir, name), "w") as f:
|
||||
for t in times:
|
||||
f.write(json.dumps({"t": t, "src": "pointer", "kind": "click", "outcome": "done", "hand": "left",
|
||||
"panel": 0, "text": "pinch: clicked on x"}) + "\n")
|
||||
export = self.store.export(self.sid, low_priority=False)
|
||||
with open(os.path.join(export, "manifest.json")) as f:
|
||||
m = json.load(f)
|
||||
ev = m["takes"][0]["events"]
|
||||
self.assertEqual(ev["file"], "takes/01-hand-size/events.jsonl.zst.age")
|
||||
self.assertIn(ev["file"], m["encryption"]["files"])
|
||||
self.assertEqual((ev["events"], ev["by"]), (2, {"click": {"done": 2}}))
|
||||
self.assertEqual(self.validate.validate(export).errors, [])
|
||||
import subprocess
|
||||
with open(os.path.join(export, ev["file"]), "rb") as f:
|
||||
raw = subprocess.run([self.takes.find_zstd(), "-dc"], input=self.age.decrypt_bytes(f.read(), self.identity),
|
||||
capture_output=True, check=True).stdout
|
||||
self.assertEqual([json.loads(line)["t"] for line in raw.splitlines()], [a - 1, b + 1])
|
||||
# a public export has none
|
||||
sj = self.takes.read_json(self.session_json)
|
||||
del sj["debug"]
|
||||
sj["consent"] = {"kind": "public", "version": "2026-10-02"}
|
||||
self.takes.write_json(self.session_json, sj)
|
||||
profile = self.takes.read_json(os.path.join(self.base, "profile.json"))
|
||||
profile["consent"] = dict(profile["testing_consent"], version="2026-10-02")
|
||||
self.takes.write_json(os.path.join(self.base, "profile.json"), profile)
|
||||
export = self.store.export(self.sid, low_priority=False)
|
||||
self.assertFalse(any(n.startswith("events") for _, _, ns in os.walk(export) for n in ns))
|
||||
|
||||
def test_bounty_round(self):
|
||||
"""A bounty's round in the test phase (no debug): encrypted the same way, no live log."""
|
||||
sj = self.takes.read_json(self.session_json)
|
||||
del sj["debug"]
|
||||
sj["bounty"] = {"id": "index-taps", "title": "Index taps", "board_updated": "2026-10-14"}
|
||||
self.takes.write_json(self.session_json, sj)
|
||||
for name in ("live.bin", "live-2.bin"):
|
||||
os.remove(os.path.join(self.tdir, name))
|
||||
export = self.store.export(self.sid, low_priority=False)
|
||||
with open(os.path.join(export, "manifest.json")) as f:
|
||||
m = json.load(f)
|
||||
self.assertEqual(len(m["encryption"]["files"]), 2)
|
||||
r = self.validate.validate(export)
|
||||
self.assertEqual(r.errors, [])
|
||||
self.assertTrue(r.summary["testing"])
|
||||
self.assertFalse(r.summary["debug"])
|
||||
|
||||
def test_public(self):
|
||||
"""A full session (public consent): nothing encrypted, the window's profile keys left out."""
|
||||
sj = self.takes.read_json(self.session_json)
|
||||
del sj["debug"]
|
||||
sj["consent"] = {"kind": "public", "version": "2026-10-02"}
|
||||
self.takes.write_json(self.session_json, sj)
|
||||
profile = self.takes.read_json(os.path.join(self.base, "profile.json"))
|
||||
profile["consent"] = dict(profile["testing_consent"], version="2026-10-02")
|
||||
self.takes.write_json(os.path.join(self.base, "profile.json"), profile)
|
||||
export = self.store.export(self.sid, low_priority=False)
|
||||
with open(os.path.join(export, "manifest.json")) as f:
|
||||
m = json.load(f)
|
||||
self.assertNotIn("encryption", m)
|
||||
self.assertEqual(sorted(m["profile"]), ["consent", "contributor", "optional", "schema"])
|
||||
r = self.validate.validate(export)
|
||||
# the live log is a debug session's only
|
||||
self.assertTrue(any("live log" in e for e in r.errors), r.errors)
|
||||
|
||||
def test_public_no_voice(self):
|
||||
"""Voice is only ever exported encrypted: a public export leaves it out."""
|
||||
self.write_voice()
|
||||
sj = self.takes.read_json(self.session_json)
|
||||
del sj["debug"]
|
||||
sj["consent"] = {"kind": "public", "version": "2026-10-02"}
|
||||
self.takes.write_json(self.session_json, sj)
|
||||
profile = self.takes.read_json(os.path.join(self.base, "profile.json"))
|
||||
profile["consent"] = dict(profile["testing_consent"], version="2026-10-02")
|
||||
self.takes.write_json(os.path.join(self.base, "profile.json"), profile)
|
||||
export = self.store.export(self.sid, low_priority=False)
|
||||
files = [n for _, _, ns in os.walk(export) for n in ns]
|
||||
self.assertFalse(any(n.startswith("voice") for n in files), files)
|
||||
|
||||
def test_checks(self):
|
||||
export = self.store.export(self.sid, low_priority=False)
|
||||
from test_validate import write_sums
|
||||
# an unencrypted recording in a debug export
|
||||
shutil.copyfile(os.path.join(export, "takes/02-no-hands/sets.bin.zst.age"),
|
||||
os.path.join(export, "takes/02-no-hands/sets.bin.zst"))
|
||||
write_sums(export)
|
||||
r = self.validate.validate(export)
|
||||
self.assertTrue(any("unencrypted" in e for e in r.errors), r.errors)
|
||||
os.remove(os.path.join(export, "takes/02-no-hands/sets.bin.zst"))
|
||||
# not an age file (its insides can't be checked without a key: only the header and size)
|
||||
path = os.path.join(export, "takes/01-hand-size/live.bin.zst.age")
|
||||
with open(path, "r+b") as f:
|
||||
f.write(b"not-age!")
|
||||
write_sums(export)
|
||||
r = self.validate.validate(export)
|
||||
self.assertTrue(any("live.bin.zst.age" in e for e in r.errors), r.errors)
|
||||
# other keys
|
||||
with open(os.path.join(export, "manifest.json")) as f:
|
||||
m = json.load(f)
|
||||
m["encryption"]["recipients"] = [self.age.keygen()[1]]
|
||||
m["session"]["debug"]["use"] = "training"
|
||||
self.takes.write_json(os.path.join(export, "manifest.json"), m)
|
||||
write_sums(export)
|
||||
r = self.validate.validate(export)
|
||||
self.assertTrue(any("other keys" in e for e in r.errors), r.errors)
|
||||
self.assertTrue(any("testing only" in e for e in r.errors), r.errors)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -67,6 +67,23 @@ class QmlBackendTest(unittest.TestCase):
|
||||
self.assertTrue(read)
|
||||
self.assertEqual(sorted(read - self.props - self.slots), [], "read in main.qml but not on Backend")
|
||||
|
||||
def test_handlers_are_signals(self):
|
||||
"""Each function onName(...) in a Connections on backend has a signal name (a renamed
|
||||
signal leaves its handler silently unused: Export and upload would stop at Export)."""
|
||||
meta = self.meta
|
||||
signals = {bytes(meta.method(i).name()).decode() for i in range(meta.methodCount())
|
||||
if meta.method(i).methodType() == meta.method(i).MethodType.Signal}
|
||||
handlers = set()
|
||||
for m in re.finditer(r"Connections\s*\{\s*target:\s*backend\b", self.qml):
|
||||
depth, k = 1, m.end()
|
||||
while depth and k < len(self.qml):
|
||||
depth += {"{": 1, "}": -1}.get(self.qml[k], 0)
|
||||
k += 1
|
||||
block = self.qml[m.end():k]
|
||||
handlers |= {h[0].lower() + h[1:] for h in re.findall(r"function on([A-Z]\w*)\s*\(", block)}
|
||||
self.assertIn("exportDone", handlers)
|
||||
self.assertEqual(sorted(handlers - signals), [], "handled in main.qml but not a Backend signal")
|
||||
|
||||
|
||||
|
||||
class GuidanceTest(unittest.TestCase):
|
||||
@@ -99,5 +116,69 @@ class GuidanceTest(unittest.TestCase):
|
||||
self.assertEqual(backend.guidance, "skeleton")
|
||||
|
||||
|
||||
class TestingSessionTest(unittest.TestCase):
|
||||
"""The testing session: on the board only with Frametop's hand tracking installed, under the
|
||||
testing consent (TESTING.md) that bounty rounds use too in the test phase, and its uploads
|
||||
go to the test dataset."""
|
||||
|
||||
def backend(self, frametop_hands):
|
||||
import shutil
|
||||
import tempfile
|
||||
try:
|
||||
import ft_handrec
|
||||
import takes
|
||||
except ImportError as e:
|
||||
self.skipTest(f"no PySide6: {e}")
|
||||
tmp = tempfile.mkdtemp()
|
||||
self.addCleanup(shutil.rmtree, tmp)
|
||||
old = os.environ.get("FT_HANDREC_FRAMETOP_HANDS")
|
||||
os.environ["FT_HANDREC_FRAMETOP_HANDS"] = frametop_hands
|
||||
self.addCleanup(lambda: os.environ.pop("FT_HANDREC_FRAMETOP_HANDS") if old is None
|
||||
else os.environ.__setitem__("FT_HANDREC_FRAMETOP_HANDS", old))
|
||||
return ft_handrec, takes, ft_handrec.Backend(takes.Store(os.path.join(tmp, "base")),
|
||||
{"dry_run": True}, ignore_cameras=True)
|
||||
|
||||
def test_board(self):
|
||||
ft, _, backend = self.backend("1")
|
||||
rows = backend.bounties
|
||||
self.assertEqual(rows[0]["id"], ft.DEBUG_ID)
|
||||
self.assertEqual(rows[0]["done"], 0)
|
||||
self.assertIn("3 min", rows[0]["minutesText"])
|
||||
self.assertIn("min", backend.planText({}, True, False, ft.DEBUG_ID))
|
||||
_, _, backend = self.backend("0")
|
||||
self.assertNotIn(ft.DEBUG_ID, [b["id"] for b in backend.bounties])
|
||||
|
||||
def test_consent_kinds(self):
|
||||
ft, takes, backend = self.backend("1")
|
||||
bounty = next(b["id"] for b in backend.bounties if b["id"] != ft.DEBUG_ID)
|
||||
testing = "testing" if takes.TEST_PHASE else "public"
|
||||
self.assertEqual(backend.roundConsentKind(ft.DEBUG_ID), "testing")
|
||||
self.assertEqual(backend.roundConsentKind(bounty), testing)
|
||||
self.assertEqual(backend.roundConsentKind(""), "public")
|
||||
backend.setConsentKind("public")
|
||||
self.assertFalse(backend.startSession({}, "auto", True, False, ft.DEBUG_ID))
|
||||
self.assertEqual(backend.consentKind, "testing") # switched to the text to agree to
|
||||
self.assertTrue(backend.needsConsent)
|
||||
self.assertIn("# Testing sessions", backend.consentText)
|
||||
self.assertEqual(backend.dataset, ft.hub.HF_TEST_DATASET)
|
||||
backend.acceptConsent(True, True, True, "right")
|
||||
self.assertFalse(backend.needsConsent)
|
||||
self.assertTrue(backend.has_consent("testing"))
|
||||
self.assertFalse(backend.has_consent("public"))
|
||||
backend.setConsentKind("public")
|
||||
self.assertTrue(backend.needsConsent)
|
||||
self.assertEqual(backend.dataset, ft.hub.HF_DATASET)
|
||||
profile = backend.store.profile()
|
||||
self.assertIn("testing_consent", profile)
|
||||
self.assertNotIn("consent", profile)
|
||||
# after the test phase a bounty asks for the dataset's own consent and goes to it
|
||||
if not takes.TEST_PHASE:
|
||||
backend.setConsentKind(backend.roundConsentKind(bounty))
|
||||
self.assertEqual(backend.consentKind, "public")
|
||||
self.assertTrue(backend.needsConsent)
|
||||
self.assertIn("bounty rounds", backend.consentText)
|
||||
self.assertEqual(backend.dataset, ft.hub.HF_DATASET)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,167 @@
|
||||
#!/usr/bin/env python3
|
||||
"""session.get_ready (the window's Getting ready step) and openvr_env, with the processes,
|
||||
units and panel faked: the parts go in order, a missing one starts (Frametop's hand tracking
|
||||
when it's installed, else the recorder's own units), and the first that can't start stops it.
|
||||
|
||||
python3 hands/rec/tests/test_ready.py
|
||||
"""
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
sys.path.insert(0, os.path.dirname(HERE))
|
||||
|
||||
import session # noqa: E402
|
||||
|
||||
|
||||
class FakeHost:
|
||||
"""What get_ready looks at, as plain flags, and what it started."""
|
||||
|
||||
def __init__(self, steamvr=True, ring=True, tracker=True, panel=True, live=False):
|
||||
self.steamvr, self.ring, self.tracker, self.panel, self.live = steamvr, ring, tracker, panel, live
|
||||
self.started = []
|
||||
|
||||
def install(self, test):
|
||||
host = self
|
||||
saved = {}
|
||||
|
||||
def patch(name, value):
|
||||
saved[name] = getattr(session, name)
|
||||
setattr(session, name, value)
|
||||
|
||||
class Panel:
|
||||
def __init__(self, *a, **k):
|
||||
pass
|
||||
|
||||
def cmd(self, text, reply=False, timeout=0.5):
|
||||
return "ok hidden" if host.panel else None
|
||||
|
||||
def close(self):
|
||||
pass
|
||||
|
||||
class Done:
|
||||
returncode = 0
|
||||
stdout = stderr = ""
|
||||
|
||||
def run(argv, **k):
|
||||
if "start" in argv and session.LIVE_HANDS_UNIT in argv:
|
||||
host.started.append(session.LIVE_HANDS_UNIT)
|
||||
host.live = host.ring = host.tracker = True
|
||||
return Done()
|
||||
|
||||
def start_camd(path=None, log=None, stop=None, timeout=15.0):
|
||||
host.started.append(session.CAMD_UNIT)
|
||||
host.ring = True
|
||||
return True
|
||||
|
||||
def start_unit(unit, what, argv, log=None, **k):
|
||||
host.started.append(unit)
|
||||
host.tracker = True
|
||||
return True
|
||||
|
||||
def start_panel(panel_bin=None, stdout=None, timeout=15.0):
|
||||
host.started.append("ft-handpanel")
|
||||
host.panel = True
|
||||
return "panel process"
|
||||
|
||||
patch("steamvr_running", lambda: host.steamvr)
|
||||
patch("ring_alive", lambda path=None: host.ring)
|
||||
patch("tracker_running", lambda: host.tracker)
|
||||
patch("unit_active", lambda unit: host.live and unit == session.LIVE_HANDS_UNIT)
|
||||
patch("start_camd", start_camd)
|
||||
patch("start_unit", start_unit)
|
||||
patch("start_panel", start_panel)
|
||||
patch("Panel", Panel)
|
||||
patch("READY_WAIT_S", 0.3)
|
||||
saved_run = session.subprocess.run
|
||||
session.subprocess.run = run
|
||||
test.addCleanup(lambda: [setattr(session, k, v) for k, v in saved.items()])
|
||||
test.addCleanup(lambda: setattr(session.subprocess, "run", saved_run))
|
||||
return self
|
||||
|
||||
|
||||
def ready(host, **kw):
|
||||
parts = []
|
||||
r = session.get_ready(on_part=lambda key, state, text: parts.append((key, state)), **kw)
|
||||
return r, parts
|
||||
|
||||
|
||||
class GetReadyTest(unittest.TestCase):
|
||||
def test_all_running(self):
|
||||
host = FakeHost().install(self)
|
||||
r, parts = ready(host)
|
||||
self.assertTrue(r["ok"])
|
||||
self.assertEqual(r["units"], [])
|
||||
self.assertIsNone(r["panel"])
|
||||
self.assertEqual([k for k, s in parts if s == "ok"], [k for k, _ in session.READY_PARTS])
|
||||
self.assertEqual(host.started, [])
|
||||
|
||||
def test_no_steamvr(self):
|
||||
host = FakeHost(steamvr=False, ring=False).install(self)
|
||||
r, parts = ready(host)
|
||||
self.assertFalse(r["ok"])
|
||||
self.assertIn("SteamVR", r["error"])
|
||||
self.assertEqual(parts[-1], ("steamvr", "failed"))
|
||||
self.assertEqual(host.started, []) # nothing started without SteamVR
|
||||
|
||||
def test_standalone_starts_its_own(self):
|
||||
host = FakeHost(ring=False, tracker=False, panel=False).install(self)
|
||||
r, _ = ready(host)
|
||||
self.assertTrue(r["ok"], r["error"])
|
||||
self.assertEqual(host.started, [session.CAMD_UNIT, session.HANDS_UNIT, "ft-handpanel"])
|
||||
self.assertEqual(r["units"], [session.CAMD_UNIT, session.HANDS_UNIT])
|
||||
self.assertEqual(r["panel"], "panel process")
|
||||
|
||||
def test_frametop_starts_its_hand_tracking(self):
|
||||
host = FakeHost(ring=False, tracker=False).install(self)
|
||||
r, _ = ready(host, frametop_hands=True)
|
||||
self.assertTrue(r["ok"], r["error"])
|
||||
self.assertEqual(host.started, [session.LIVE_HANDS_UNIT]) # it brings its cameras
|
||||
self.assertEqual(r["units"], [session.LIVE_HANDS_UNIT])
|
||||
|
||||
def test_testing_session_needs_frametop(self):
|
||||
host = FakeHost().install(self)
|
||||
r, parts = ready(host, need_live=True)
|
||||
self.assertFalse(r["ok"])
|
||||
self.assertEqual(r["error"], session.DEBUG_NEEDS_HANDS)
|
||||
# Frametop's tracking off, the recorder's own running: Frametop's starts
|
||||
host = FakeHost(live=False).install(self)
|
||||
r, _ = ready(host, frametop_hands=True, need_live=True)
|
||||
self.assertTrue(r["ok"], r["error"])
|
||||
self.assertEqual(host.started, [session.LIVE_HANDS_UNIT])
|
||||
|
||||
def test_panel_fails(self):
|
||||
host = FakeHost(panel=False).install(self)
|
||||
|
||||
def broken(panel_bin=None, stdout=None, timeout=15.0):
|
||||
raise RuntimeError("ft-handpanel exited (is SteamVR running?)")
|
||||
session.start_panel = broken
|
||||
r, parts = ready(host)
|
||||
self.assertFalse(r["ok"])
|
||||
self.assertEqual(parts[-1], ("panel", "failed"))
|
||||
self.assertIn("ft-handpanel", r["error"])
|
||||
|
||||
|
||||
class OpenvrEnvTest(unittest.TestCase):
|
||||
def test_frametop_desktop(self):
|
||||
"""Frametop's desktop's XDG_CONFIG_HOME has no openvrpaths.vrpath: ~/.config instead."""
|
||||
home = tempfile.mkdtemp()
|
||||
self.addCleanup(lambda: __import__("shutil").rmtree(home))
|
||||
os.makedirs(os.path.join(home, ".config", "openvr"))
|
||||
open(os.path.join(home, ".config", "openvr", "openvrpaths.vrpath"), "w").close()
|
||||
os.makedirs(os.path.join(home, ".config", "frametop"))
|
||||
saved = dict(os.environ)
|
||||
self.addCleanup(lambda: (os.environ.clear(), os.environ.update(saved)))
|
||||
os.environ["HOME"] = home
|
||||
os.environ["XDG_CONFIG_HOME"] = os.path.join(home, ".config", "frametop")
|
||||
self.assertEqual(session.openvr_env()["XDG_CONFIG_HOME"], os.path.join(home, ".config"))
|
||||
os.environ["XDG_CONFIG_HOME"] = os.path.join(home, ".config")
|
||||
self.assertEqual(session.openvr_env()["XDG_CONFIG_HOME"], os.path.join(home, ".config"))
|
||||
del os.environ["XDG_CONFIG_HOME"]
|
||||
self.assertNotIn("XDG_CONFIG_HOME", session.openvr_env())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -20,6 +20,7 @@ sys.path.insert(0, os.path.dirname(HERE))
|
||||
import session # noqa: E402
|
||||
import takes # noqa: E402
|
||||
import validate # noqa: E402
|
||||
import recorder.units # noqa: E402 (pause_units is there now; session.py re-exports it)
|
||||
from test_validate import fhset, make_session, SESSION # noqa: E402
|
||||
|
||||
SCRIPT = {
|
||||
@@ -918,18 +919,23 @@ class StandaloneTest(unittest.TestCase):
|
||||
self.assertIn("--no-gestures", argv)
|
||||
run.assert_not_called() # no container to bring up
|
||||
|
||||
def test_ft_hands_in_the_dev_container(self):
|
||||
self.assertTrue(self.recorder_argv(None)[0].endswith("distrobox"))
|
||||
def test_ft_hands_in_frametops_container(self):
|
||||
# scripts/in-box: "dev" in a clone, the release's own container in a release
|
||||
self.assertEqual(self.recorder_argv(None)[:2], [session.IN_BOX, session.FT_HANDS])
|
||||
argv, _ = self.tracker_argv(None)
|
||||
self.assertEqual(argv[:4], [os.path.expanduser("~/.local/bin/distrobox"), "enter", "dev", "--"])
|
||||
self.assertEqual(argv[4], session.FT_HANDS)
|
||||
self.assertEqual(argv[:2], [session.IN_BOX, session.FT_HANDS])
|
||||
self.assertTrue(os.access(session.IN_BOX, os.X_OK))
|
||||
|
||||
def test_fix_hint(self):
|
||||
with mock.patch.object(session, "STANDALONE", self.INFO):
|
||||
self.assertEqual(session.fix_hint("hands/build.sh"), "run the install command again")
|
||||
text = session.camera_text({"status": "degraded", "reason": "2 of 4", "ring_missing": ["upper_left"]})
|
||||
with mock.patch.object(session, "STANDALONE", None):
|
||||
with mock.patch.object(session, "STANDALONE", None), mock.patch.object(session, "RELEASE", False):
|
||||
self.assertEqual(session.fix_hint("hands/build.sh"), "hands/build.sh")
|
||||
with mock.patch.object(session, "STANDALONE", None), mock.patch.object(session, "RELEASE", True):
|
||||
# a release never builds: its programs come in its image
|
||||
self.assertEqual(session.fix_hint("hands/rec/build.sh"), "install this Frametop release again")
|
||||
self.assertEqual(session.fix_hint("hands/run.sh caps (asks for sudo)"), "hands/run.sh caps (asks for sudo)")
|
||||
self.assertIn("run the install command again", text)
|
||||
self.assertNotIn("~/frametop", text)
|
||||
|
||||
@@ -1064,6 +1070,10 @@ class QuietHandsTest(SessionBase):
|
||||
super().setUp()
|
||||
self.path = os.path.join(self.tmp, "run", session.PAUSED_UNITS_FILE)
|
||||
self.calls = []
|
||||
# never the real units: a missed patch here once stopped the live hand pointer and hands
|
||||
guard = mock.patch.object(recorder.units, "stop_unit", lambda u: self.fail("real stop of %s" % u))
|
||||
guard.start()
|
||||
self.addCleanup(guard.stop)
|
||||
|
||||
def systemctl(self):
|
||||
calls = self.calls
|
||||
@@ -1076,8 +1086,8 @@ class QuietHandsTest(SessionBase):
|
||||
|
||||
def test_pauses_only_running_units_and_resumes_them(self):
|
||||
active = {"frametop-handview.service": True, "frametop-pointer.service": False}
|
||||
with mock.patch.object(session, "unit_active", lambda u: active[u]), \
|
||||
mock.patch.object(session, "stop_unit", lambda u: self.calls.append(["stop", u])):
|
||||
with mock.patch.object(recorder.units, "unit_active", lambda u: active[u]), \
|
||||
mock.patch.object(recorder.units, "stop_unit", lambda u: self.calls.append(["stop", u])):
|
||||
paused = session.pause_units(session.QUIET_UNITS, self.path)
|
||||
self.assertEqual(paused, ["frametop-handview.service"])
|
||||
self.assertEqual(self.calls, [["stop", "frametop-handview.service"]])
|
||||
@@ -1089,7 +1099,7 @@ class QuietHandsTest(SessionBase):
|
||||
self.assertFalse(os.path.exists(self.path))
|
||||
|
||||
def test_nothing_running_nothing_paused(self):
|
||||
with mock.patch.object(session, "unit_active", lambda u: False):
|
||||
with mock.patch.object(recorder.units, "unit_active", lambda u: False):
|
||||
self.assertEqual(session.pause_units(session.QUIET_UNITS, self.path), [])
|
||||
self.assertFalse(os.path.exists(self.path))
|
||||
self.assertEqual(session.resume_units(self.path), [])
|
||||
|
||||
@@ -432,6 +432,15 @@ class HubTest(unittest.TestCase):
|
||||
self.assertTrue(any("manifest.json" in line for line in logged))
|
||||
self.assertEqual(hub.uploads(self.store, SESSION), []) # a dry run records nothing
|
||||
|
||||
def test_consent_line(self):
|
||||
"""The description says which consent the upload goes under, and the older one it was
|
||||
recorded under when they differ."""
|
||||
self.assertEqual(hub.consent_line({"consent_version": "2026-10-14"}), "2026-10-14")
|
||||
self.assertEqual(hub.consent_line({"consent_version": "2026-10-14", "recorded_consent_version": "2026-10-14"}),
|
||||
"2026-10-14")
|
||||
self.assertEqual(hub.consent_line({"consent_version": "2026-10-14", "recorded_consent_version": "2026-10-03"}),
|
||||
"2026-10-14 (recorded under 2026-10-03)")
|
||||
|
||||
def test_dataset_env(self):
|
||||
self.assertEqual(hub.dataset_id(), hub.HF_DATASET)
|
||||
with mock.patch.dict(os.environ, {hub.DATASET_ENV: "not a repo"}):
|
||||
|
||||
+117
-124
@@ -15,6 +15,10 @@ checks:
|
||||
every set's header is checked (camera names, sizes, record length) and counted against the
|
||||
manifest; the pixels themselves aren't looked at;
|
||||
- every .jsonl line parses, and take.json does;
|
||||
- a test-phase export (a bounty round or debug session under TESTING.md, DESIGN.md "Testing"):
|
||||
its recordings (sets.bin.zst.age, live.bin.zst.age, voice.bin.zst.age, events.jsonl.zst.age)
|
||||
are whole age files to the test dataset's keys (test-recipients.txt), nothing of them
|
||||
unencrypted; their insides aren't checked (that needs a key);
|
||||
- the side cameras (sides.py): the session's "sides" decision, and each take's files named
|
||||
right by it (a warning when it wasn't decided: the maintainer's check tells then);
|
||||
- the total size.
|
||||
@@ -23,7 +27,6 @@ It runs on Linux and Windows, with Python 3.12 or later. It reports errors (don'
|
||||
usage: validate.py DIR [--json] exit status 0: no errors, 1: errors, 2: not an export
|
||||
"""
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
@@ -32,11 +35,14 @@ import shutil
|
||||
import struct
|
||||
import subprocess
|
||||
import sys
|
||||
import uuid
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
sys.path.insert(0, HERE)
|
||||
from session import _key_identifies, strip_calibration # noqa: E402 (standard library only, next to this file)
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(HERE)))
|
||||
from recorder import age # noqa: E402 (its shape check needs no cryptography library)
|
||||
from recorder.checks import (Cancelled, Report, check_sums, is_uuid4, listed, need, # noqa: E402,F401
|
||||
read_json_file)
|
||||
|
||||
# Decompression, the first that's there: Python 3.14's compression.zstd, the zstandard package
|
||||
# (pip install zstandard: the maintainer's Windows PC), the zstd program.
|
||||
@@ -52,10 +58,12 @@ except ImportError:
|
||||
EXPORT_SCHEMA = 1
|
||||
TOP_FILES = {"manifest.json", "calibration.json", "device.json", "SHA256SUMS"}
|
||||
REGION_CONSENT = "2026-10-03" # the consent version that added the residency confirmation
|
||||
TAKE_FILES = {"prompts.jsonl", "poses.jsonl", "take.json", "sets.bin.zst"}
|
||||
TAKE_FILES = {"prompts.jsonl", "poses.jsonl", "take.json", "sets.bin.zst", "sets.bin.zst.age", "live.bin.zst.age",
|
||||
"voice.bin.zst.age", "events.jsonl.zst.age"}
|
||||
TEST_RECIPIENTS = os.path.join(HERE, "test-recipients.txt")
|
||||
CONSENT_KEYS = {"public": "consent", "testing": "testing_consent"} # profile.json's record of each consent
|
||||
TAKE_RE = re.compile(r"^\d{2}-[a-z0-9][a-z0-9-]*$")
|
||||
SESSION_RE = re.compile(r"^\d{8}-\d{6}(?:-\d+)?$")
|
||||
SUM_RE = re.compile(r"^([0-9a-f]{64}) (\S.*)$")
|
||||
CAM_NAME_RE = re.compile(r"^[a-z][a-z0-9_]{0,15}$")
|
||||
TAKE_STATUSES = ("complete", "stopped", "skipped")
|
||||
HDR = struct.Struct("<8sII") # fh_set_hdr_t: magic, ncams, bytes (hands/track/record.h)
|
||||
@@ -70,60 +78,6 @@ OUT_CHUNK = 8 << 20
|
||||
# zstandard's decompressobj has no output limit: it gets the input this much at a time, so even a
|
||||
# stream made to inflate hugely gives out a few hundred MB at most per call.
|
||||
ZSTANDARD_PIECE = 8 << 10
|
||||
MAX_LISTED = 10 # removed calibration paths and the like: list this many, then "and N more"
|
||||
|
||||
|
||||
class Cancelled(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class Report:
|
||||
def __init__(self, path):
|
||||
self.path = path
|
||||
self.errors = []
|
||||
self.warnings = []
|
||||
self.summary = {}
|
||||
self.bytes = 0
|
||||
|
||||
def error(self, text):
|
||||
self.errors.append(text)
|
||||
|
||||
def warn(self, text):
|
||||
self.warnings.append(text)
|
||||
|
||||
@property
|
||||
def ok(self):
|
||||
return not self.errors
|
||||
|
||||
def as_dict(self):
|
||||
return {"path": self.path, "ok": self.ok, "errors": self.errors, "warnings": self.warnings,
|
||||
"summary": self.summary, "bytes": self.bytes}
|
||||
|
||||
|
||||
def listed(items):
|
||||
items = list(items)
|
||||
more = len(items) - MAX_LISTED
|
||||
return ", ".join(items[:MAX_LISTED]) + (f" and {more} more" if more > 0 else "")
|
||||
|
||||
|
||||
def is_uuid4(s):
|
||||
try:
|
||||
u = uuid.UUID(s)
|
||||
except (TypeError, ValueError, AttributeError):
|
||||
return False
|
||||
return u.version == 4 and str(u) == s
|
||||
|
||||
|
||||
def sha256_file(path, step=None, cancelled=lambda: False):
|
||||
h = hashlib.sha256()
|
||||
with open(path, "rb") as f:
|
||||
while chunk := f.read(CHUNK):
|
||||
if cancelled():
|
||||
raise Cancelled()
|
||||
h.update(chunk)
|
||||
if step:
|
||||
step(len(chunk))
|
||||
return h.hexdigest()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- the files
|
||||
@@ -156,52 +110,6 @@ def walk_files(root, report):
|
||||
return sorted(files)
|
||||
|
||||
|
||||
def check_sums(root, files, report, step, cancelled):
|
||||
path = os.path.join(root, "SHA256SUMS")
|
||||
if not os.path.isfile(path):
|
||||
report.error("SHA256SUMS is missing")
|
||||
return
|
||||
sums = {}
|
||||
with open(path, encoding="utf-8", errors="replace") as f:
|
||||
for n, line in enumerate(f, 1):
|
||||
line = line.rstrip("\n")
|
||||
m = SUM_RE.match(line)
|
||||
if not m:
|
||||
report.error(f"SHA256SUMS line {n}: not \"<sha256> <path>\"")
|
||||
continue
|
||||
digest, rel = m.groups()
|
||||
if rel.startswith("/") or ".." in rel.split("/") or rel == "SHA256SUMS":
|
||||
report.error(f"SHA256SUMS line {n}: a path that doesn't belong: {rel}")
|
||||
continue
|
||||
if rel in sums:
|
||||
report.error(f"SHA256SUMS lists {rel} twice")
|
||||
sums[rel] = digest
|
||||
present = set(files) - {"SHA256SUMS"}
|
||||
missing = sorted(set(sums) - present)
|
||||
if missing:
|
||||
report.error(f"in SHA256SUMS but missing: {listed(missing)}")
|
||||
unlisted = sorted(present - set(sums))
|
||||
if unlisted:
|
||||
report.error(f"not in SHA256SUMS: {listed(unlisted)}")
|
||||
bad = []
|
||||
for rel in sorted(set(sums) & present):
|
||||
if sha256_file(os.path.join(root, rel), step, cancelled) != sums[rel]:
|
||||
bad.append(rel)
|
||||
if bad:
|
||||
report.error(f"checksum mismatch: {listed(bad)}")
|
||||
|
||||
|
||||
def read_json_file(path, rel, report):
|
||||
try:
|
||||
with open(path, encoding="utf-8") as f:
|
||||
return json.load(f)
|
||||
except OSError as e:
|
||||
report.error(f"{rel}: can't read it: {e.strerror or e}")
|
||||
except ValueError as e:
|
||||
report.error(f"{rel}: not valid JSON: {e}")
|
||||
return None
|
||||
|
||||
|
||||
def check_jsonl(path, rel, report, cancelled):
|
||||
"""Every line a JSON object with a time "t"."""
|
||||
try:
|
||||
@@ -429,20 +337,6 @@ def check_sets(path, rel, entry, report, step, cancelled):
|
||||
|
||||
# ---------------------------------------------------------------- the manifest
|
||||
|
||||
def need(obj, key, types, where, report, required=True):
|
||||
"""obj[key] if it's of the given type(s); an error otherwise."""
|
||||
if not isinstance(obj, dict) or key not in obj:
|
||||
if required:
|
||||
report.error(f"manifest: {where}{key} is missing")
|
||||
return None
|
||||
v = obj[key]
|
||||
allowed = types if isinstance(types, tuple) else (types,)
|
||||
if not isinstance(v, allowed) or (isinstance(v, bool) and bool not in allowed):
|
||||
report.error(f"manifest: {where}{key} has the wrong type ({type(v).__name__})")
|
||||
return None
|
||||
return v
|
||||
|
||||
|
||||
def check_pose(pose, where, report, extra=()):
|
||||
"""A pose as device_config.json has it: plus_x, plus_z, position, each 3 finite numbers."""
|
||||
if not isinstance(pose, dict):
|
||||
@@ -492,11 +386,58 @@ def identifying_keys(o, path):
|
||||
return out
|
||||
|
||||
|
||||
def is_debug(m):
|
||||
"""A debug session's export: session.debug in the manifest."""
|
||||
return isinstance(m, dict) and isinstance(m.get("session"), dict) and isinstance(m["session"].get("debug"), dict)
|
||||
|
||||
|
||||
def is_testing(m):
|
||||
"""An export for the test dataset (DESIGN.md "Testing"): recorded under TESTING.md's consent,
|
||||
its recordings encrypted. A debug session always is."""
|
||||
return isinstance(m, dict) and (m.get("consent_kind") == "testing" or is_debug(m))
|
||||
|
||||
|
||||
def check_encryption(m, report):
|
||||
"""A test export's manifest "encryption": to the test dataset's keys. Returns the keys."""
|
||||
enc = m.get("encryption")
|
||||
if not isinstance(enc, dict):
|
||||
report.error("manifest: a test-phase export without encryption (export it again)")
|
||||
return []
|
||||
try:
|
||||
shipped = age.read_recipients(TEST_RECIPIENTS)
|
||||
except (OSError, ValueError) as e:
|
||||
report.error(f"the test dataset's keys ({TEST_RECIPIENTS}): {e}")
|
||||
shipped = None
|
||||
r = enc.get("recipients")
|
||||
if not isinstance(r, list) or not r or not all(isinstance(x, str) for x in r):
|
||||
report.error("manifest: encryption.recipients isn't a list of keys")
|
||||
return []
|
||||
if shipped is not None and sorted(r) != sorted(shipped):
|
||||
report.error("manifest: encrypted to other keys than the test dataset's (test-recipients.txt)")
|
||||
if not isinstance(enc.get("files"), list):
|
||||
report.error("manifest: encryption.files isn't a list")
|
||||
return r
|
||||
|
||||
|
||||
def check_manifest(m, report, root):
|
||||
"""The manifest's keys and types. Returns {take id: entry} for the file checks."""
|
||||
if not isinstance(m, dict):
|
||||
report.error("manifest: not a JSON object")
|
||||
return {}
|
||||
debug, testing = is_debug(m), is_testing(m)
|
||||
suffix = ".age" if testing else ""
|
||||
kind = "testing" if testing else "public"
|
||||
if m.get("consent_kind", "public") not in CONSENT_KEYS:
|
||||
report.error(f"manifest: consent_kind {m.get('consent_kind')!r}")
|
||||
if debug and m.get("consent_kind") != "testing":
|
||||
report.error("manifest: a debug session not recorded under the testing consent")
|
||||
if debug and m["session"]["debug"].get("use") != "testing":
|
||||
report.error("manifest: a debug session not marked for testing only")
|
||||
enc_files = None
|
||||
if testing:
|
||||
enc_files = (m.get("encryption") or {}).get("files") if isinstance(m.get("encryption"), dict) else None
|
||||
elif "encryption" in m:
|
||||
report.error("manifest: encryption in an export that isn't for the test dataset")
|
||||
if m.get("schema") != EXPORT_SCHEMA:
|
||||
report.error(f"manifest: schema {m.get('schema')!r}, expected {EXPORT_SCHEMA}")
|
||||
tool = need(m, "tool", str, "", report)
|
||||
@@ -522,7 +463,7 @@ def check_manifest(m, report, root):
|
||||
if profile is not None:
|
||||
if profile.get("contributor") != contributor:
|
||||
report.error("manifest: profile.contributor differs from contributor")
|
||||
pc = need(profile, "consent", dict, "profile.", report)
|
||||
pc = need(profile, CONSENT_KEYS[kind], dict, "profile.", report)
|
||||
if pc is not None:
|
||||
if pc.get("version") != consent:
|
||||
report.error("manifest: profile.consent.version differs from consent_version")
|
||||
@@ -535,7 +476,7 @@ def check_manifest(m, report, root):
|
||||
optional = profile.get("optional")
|
||||
if isinstance(optional, dict) and str(optional.get("notes") or "").strip():
|
||||
report.warn("the profile's notes are included: check they hold nothing private")
|
||||
extra = set(profile) - {"schema", "contributor", "consent", "optional"}
|
||||
extra = set(profile) - {"schema", "contributor", CONSENT_KEYS[kind], "optional"}
|
||||
if extra:
|
||||
report.error(f"manifest: the profile has unexpected keys: {listed(sorted(extra))}")
|
||||
|
||||
@@ -587,10 +528,34 @@ def check_manifest(m, report, root):
|
||||
if f is None:
|
||||
if t.get("sets"):
|
||||
report.error(f"manifest: take {tid} has {t.get('sets')} sets but no file")
|
||||
elif f != f"takes/{tid}/sets.bin.zst":
|
||||
elif f != f"takes/{tid}/sets.bin.zst{suffix}":
|
||||
report.error(f"manifest: take {tid}'s file is {f!r}")
|
||||
elif not isinstance(t.get("raw_bytes"), int):
|
||||
report.error(f"manifest: take {tid} has a file but no raw_bytes")
|
||||
live = t.get("live")
|
||||
if live is not None:
|
||||
if not debug:
|
||||
report.error(f"manifest: take {tid} has a live log, but this isn't a debug session")
|
||||
elif not isinstance(live, dict) or live.get("file") != f"takes/{tid}/live.bin.zst.age":
|
||||
report.error(f"manifest: take {tid}'s live log is {live!r}")
|
||||
voice = t.get("voice")
|
||||
if voice is not None:
|
||||
if not debug:
|
||||
report.error(f"manifest: take {tid} has a voice recording, but this isn't a debug session")
|
||||
elif not isinstance(voice, dict) or voice.get("file") != f"takes/{tid}/voice.bin.zst.age":
|
||||
report.error(f"manifest: take {tid}'s voice recording is {voice!r}")
|
||||
events = t.get("events")
|
||||
if events is not None:
|
||||
if not debug:
|
||||
report.error(f"manifest: take {tid} has the panels' and pointer's events, but this isn't a debug session")
|
||||
elif not isinstance(events, dict) or events.get("file") != f"takes/{tid}/events.jsonl.zst.age":
|
||||
report.error(f"manifest: take {tid}'s events are {events!r}")
|
||||
if testing and isinstance(enc_files, list):
|
||||
for name in (f, (live or {}).get("file") if isinstance(live, dict) else None,
|
||||
(voice or {}).get("file") if isinstance(voice, dict) else None,
|
||||
(events or {}).get("file") if isinstance(events, dict) else None):
|
||||
if name and name not in enc_files:
|
||||
report.error(f"manifest: {name} isn't in encryption.files")
|
||||
check_sides(m, entries, report)
|
||||
return entries
|
||||
|
||||
@@ -624,12 +589,14 @@ def check_sides(m, entries, report):
|
||||
|
||||
def summarize(m):
|
||||
"""What a pull request's description says about the export: takes, minutes, lighting,
|
||||
objects, controllers, the consent and tool versions."""
|
||||
objects, controllers, the bounty and sets a second, the consent and tool versions."""
|
||||
if not isinstance(m, dict):
|
||||
return {}
|
||||
session = m.get("session") if isinstance(m.get("session"), dict) else {}
|
||||
checklist = session.get("checklist") if isinstance(session.get("checklist"), dict) else {}
|
||||
lighting = session.get("lighting") if isinstance(session.get("lighting"), dict) else {}
|
||||
bounty = session.get("bounty") if isinstance(session.get("bounty"), dict) else {}
|
||||
recorded = session.get("consent") if isinstance(session.get("consent"), dict) else {}
|
||||
takes = [t for t in m.get("takes") or [] if isinstance(t, dict)]
|
||||
seconds = sum(t.get("duration_s") or 0 for t in takes if isinstance(t.get("duration_s"), (int, float)))
|
||||
return {"session": m.get("session_id", ""), "contributor": m.get("contributor", ""),
|
||||
@@ -638,7 +605,12 @@ def summarize(m):
|
||||
"minutes": round(seconds / 60, 1), "lighting": lighting.get("chosen", ""),
|
||||
"objects": list(checklist.get("objects") or []), "own_objects": len(checklist.get("own_objects") or []),
|
||||
"controllers": checklist.get("controllers", ""), "consent_version": m.get("consent_version", ""),
|
||||
"tool": m.get("tool", "")}
|
||||
# the version in force when it was recorded (session.json); consent_version is the one
|
||||
# agreed to now, which the upload goes under (CONSENT.md "Changes to this text")
|
||||
"recorded_consent_version": recorded.get("version", "") if isinstance(recorded.get("version"), str) else "",
|
||||
"tool": m.get("tool", ""), "bounty": bounty.get("id", "") if isinstance(bounty.get("id"), str) else "",
|
||||
"record_hz": session.get("record_hz") if isinstance(session.get("record_hz"), (int, float)) else None,
|
||||
"debug": is_debug(m), "testing": is_testing(m)}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- all of it
|
||||
@@ -674,6 +646,8 @@ def validate(root, progress=None, cancel=None):
|
||||
else:
|
||||
report.error("manifest.json is missing")
|
||||
entries = check_manifest(manifest, report, root) if manifest is not None else {}
|
||||
testing = is_testing(manifest)
|
||||
recipients = check_encryption(manifest, report) if testing else []
|
||||
report.summary = summarize(manifest)
|
||||
|
||||
if "calibration.json" in files:
|
||||
@@ -719,8 +693,27 @@ def validate(root, progress=None, cancel=None):
|
||||
report.error(f"{base}take.json: not a JSON object")
|
||||
else:
|
||||
report.warn(f"{base}take.json is missing")
|
||||
rel = base + "sets.bin.zst"
|
||||
entry = entries.get(tid)
|
||||
if testing:
|
||||
if base + "sets.bin.zst" in sizes:
|
||||
report.error(f"{base}sets.bin.zst: a test-phase recording unencrypted (export it again)")
|
||||
for name, listed_file in (("sets.bin.zst.age", (entry or {}).get("file")),
|
||||
("live.bin.zst.age", ((entry or {}).get("live") or {}).get("file")),
|
||||
("voice.bin.zst.age", ((entry or {}).get("voice") or {}).get("file")),
|
||||
("events.jsonl.zst.age", ((entry or {}).get("events") or {}).get("file"))):
|
||||
rel = base + name
|
||||
if rel in sizes:
|
||||
problem = age.shape(os.path.join(root, rel), recipients)
|
||||
if problem:
|
||||
report.error(f"{rel}: {problem}")
|
||||
step(sizes[rel], f"Checking {title}")
|
||||
elif listed_file:
|
||||
report.error(f"{rel} is missing")
|
||||
continue
|
||||
for name in ("sets.bin.zst.age", "live.bin.zst.age", "voice.bin.zst.age", "events.jsonl.zst.age"):
|
||||
if base + name in sizes:
|
||||
report.error(f"{base}{name}: encrypted, but this export isn't for the test dataset")
|
||||
rel = base + "sets.bin.zst"
|
||||
if rel in sizes:
|
||||
check_sets(os.path.join(root, rel), rel, entry, report,
|
||||
lambda n, title=title: step(n, f"Checking {title}"), cancelled)
|
||||
|
||||
@@ -0,0 +1,153 @@
|
||||
"""A testing session's voice, if the person ticked "Record my voice": the microphone during the
|
||||
recording, so they can say what went wrong as it happens ("that opened the lazy susan, I didn't
|
||||
mean to"). Transcribed later on the maintainer's machines (to_wav, then whisper.cpp), the speech
|
||||
becomes flags at its times, beside the button's marks.
|
||||
|
||||
pw-record captures the default microphone (16 kHz, mono, s16) in raw mode, at half volume (the
|
||||
Frame's microphone at full volume clipped loud speech); each chunk goes into the take's
|
||||
voice.bin with the time it was read (CLOCK_MONOTONIC, the session's clock).
|
||||
|
||||
voice.bin: a 16-byte header, then records.
|
||||
header: 8s magic "FHVOIC01", u32 version (1), u32 sample rate
|
||||
record: u64 read_ns (when the chunk was read: its last sample is about then), u32 bytes,
|
||||
then that many bytes of s16le samples
|
||||
"""
|
||||
import shutil
|
||||
import struct
|
||||
import subprocess
|
||||
import threading
|
||||
import time
|
||||
|
||||
MAGIC = b"FHVOIC01"
|
||||
HDR = struct.Struct("<8sII")
|
||||
REC = struct.Struct("<QI")
|
||||
RATE = 16000
|
||||
CHUNK = RATE // 10 * 2 # 100 ms of s16 mono
|
||||
VOLUME = "0.5" # the stream's, not the microphone's: other apps keep theirs
|
||||
|
||||
|
||||
def mono_ns():
|
||||
return time.clock_gettime_ns(time.CLOCK_MONOTONIC)
|
||||
|
||||
|
||||
class VoiceLog:
|
||||
"""Records the microphone into path until stop(). A missing pw-record or a microphone that
|
||||
fails doesn't stop the session: stop() reports it ("error")."""
|
||||
|
||||
def __init__(self, path, rate=RATE):
|
||||
self.path, self.rate = path, rate
|
||||
self.proc = None
|
||||
self._thread = None
|
||||
self._file = None
|
||||
self.samples = 0
|
||||
self.chunks = 0
|
||||
self.peak = 0
|
||||
self.error = ""
|
||||
|
||||
def start(self):
|
||||
exe = shutil.which("pw-record")
|
||||
if not exe:
|
||||
self.error = "pw-record isn't installed"
|
||||
return self
|
||||
self._file = open(self.path, "wb")
|
||||
self._file.write(HDR.pack(MAGIC, 1, self.rate))
|
||||
try:
|
||||
self.proc = subprocess.Popen([exe, "--raw", "--rate", str(self.rate), "--channels", "1",
|
||||
"--format", "s16", "--volume", VOLUME, "--latency", "20ms", "-"],
|
||||
stdin=subprocess.DEVNULL, stdout=subprocess.PIPE,
|
||||
stderr=subprocess.DEVNULL)
|
||||
except OSError as e:
|
||||
self.error = "pw-record didn't start: %s" % e
|
||||
return self
|
||||
self._thread = threading.Thread(target=self._run, name="handrec-voicelog", daemon=True)
|
||||
self._thread.start()
|
||||
return self
|
||||
|
||||
def _run(self):
|
||||
out = self.proc.stdout
|
||||
while True:
|
||||
data = out.read1(CHUNK) if hasattr(out, "read1") else out.read(CHUNK)
|
||||
if not data:
|
||||
break
|
||||
now = mono_ns()
|
||||
if len(data) % 2:
|
||||
data = data[:-1]
|
||||
self._file.write(REC.pack(now, len(data)) + data)
|
||||
self.samples += len(data) // 2
|
||||
self.chunks += 1
|
||||
peak = max(abs(v) for v in memoryview(data).cast("h")) if data else 0
|
||||
self.peak = max(self.peak, peak)
|
||||
|
||||
def stop(self):
|
||||
"""Ends the recording: {"seconds", "chunks", "peak" (0..1: 0 means a muted microphone),
|
||||
"error"}."""
|
||||
if self.proc is not None:
|
||||
self.proc.terminate()
|
||||
try:
|
||||
self.proc.wait(3)
|
||||
except subprocess.TimeoutExpired:
|
||||
self.proc.kill()
|
||||
self.proc.wait()
|
||||
if self.proc.returncode not in (0, -15, 143) and not self.chunks and not self.error:
|
||||
self.error = "pw-record exited (%s)" % self.proc.returncode
|
||||
if self._thread is not None:
|
||||
self._thread.join(3)
|
||||
if self.proc is not None:
|
||||
self.proc.stdout.close()
|
||||
if self._file is not None:
|
||||
self._file.close()
|
||||
return {"seconds": round(self.samples / self.rate, 2), "chunks": self.chunks,
|
||||
"peak": round(self.peak / 32768, 3), "error": self.error}
|
||||
|
||||
|
||||
def read(path):
|
||||
"""(rate, [(read_ns, bytes)]) of a voice.bin. Stops at a cut-off end."""
|
||||
with open(path, "rb") as f:
|
||||
data = f.read()
|
||||
if len(data) < HDR.size or data[:8] != MAGIC:
|
||||
raise ValueError("%s: not a voice.bin" % path)
|
||||
_, _, rate = HDR.unpack_from(data, 0)
|
||||
out, off = [], HDR.size
|
||||
while off + REC.size <= len(data):
|
||||
read_ns, n = REC.unpack_from(data, off)
|
||||
end = off + REC.size + n
|
||||
if end > len(data):
|
||||
break
|
||||
out.append((read_ns, data[off + REC.size:end]))
|
||||
off = end
|
||||
return rate, out
|
||||
|
||||
|
||||
def to_wav(path, dest):
|
||||
"""A voice.bin as a WAV file, gaps between chunks filled with silence. Returns the time
|
||||
(CLOCK_MONOTONIC ns) of the WAV's first sample."""
|
||||
import wave
|
||||
rate, records = read(path)
|
||||
if not records:
|
||||
raise ValueError("%s: no sound" % path)
|
||||
first_ns = records[0][0] - len(records[0][1]) // 2 * 1_000_000_000 // rate
|
||||
with wave.open(dest, "wb") as w:
|
||||
w.setnchannels(1)
|
||||
w.setsampwidth(2)
|
||||
w.setframerate(rate)
|
||||
written = 0
|
||||
for read_ns, data in records:
|
||||
start = (read_ns - first_ns) * rate // 1_000_000_000 - len(data) // 2
|
||||
if start > written:
|
||||
w.writeframes(b"\0\0" * (start - written))
|
||||
written = start
|
||||
w.writeframes(data)
|
||||
written += len(data) // 2
|
||||
return first_ns
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import sys
|
||||
if len(sys.argv) == 3 and sys.argv[1] == "--test":
|
||||
log = VoiceLog(sys.argv[2]).start()
|
||||
time.sleep(2)
|
||||
print(log.stop())
|
||||
elif len(sys.argv) == 3:
|
||||
print("first sample at %d ns" % to_wav(sys.argv[1], sys.argv[2]))
|
||||
else:
|
||||
sys.exit("voicelog.py VOICE.BIN OUT.WAV | --test OUT.BIN")
|
||||
@@ -75,12 +75,40 @@ struct Run {
|
||||
bool down() const { return pinch.side(1).flags & FH_PINCH_DOWN; }
|
||||
};
|
||||
|
||||
// The values these tests were written for (the defaults before 2026-10-14): they test the rules,
|
||||
// TestDefaults the defaults.
|
||||
PinchParams Params() {
|
||||
PinchParams p;
|
||||
p.other_finger = 12; // as ft-hands
|
||||
p.end_m = 0.035, p.drag_open_m = 0.05, p.drag_open_s = 0.12, p.grip_takeover_s = 0.3;
|
||||
return p;
|
||||
}
|
||||
|
||||
void TestDefaults() {
|
||||
// since 2026-10-14 a pinch lets go less easily: a tap still reading 4 cm hasn't ended
|
||||
PinchParams p;
|
||||
p.other_finger = 12;
|
||||
Run tap{Pinch(p), Grip()};
|
||||
for (int k = 0; k < 5; ++k) tap.Step(0.06);
|
||||
for (int k = 0; k < 3; ++k) tap.Step(0.01);
|
||||
tap.Step(0.04), tap.Step(0.04);
|
||||
CHECK(tap.down(), "defaults: a tap at 4 cm is still down");
|
||||
tap.Step(0.05), tap.Step(0.05);
|
||||
CHECK(!tap.down() && tap.lost == 0, "defaults: a tap ends on the second frame past 4.5 cm");
|
||||
// a drag loosened to 5.5 cm ends after drag_open_s (0.2 s: six or seven frames), opened to 7 cm at once
|
||||
Run drag{Pinch(p), Grip()};
|
||||
for (int k = 0; k < 5; ++k) drag.Step(0.06);
|
||||
for (int k = 0; k < 30; ++k) drag.Step(0.01);
|
||||
int frames = 0;
|
||||
while (drag.down() && frames < 20) drag.Step(0.055), ++frames;
|
||||
CHECK(!drag.down() && frames >= 6 && frames <= 8, "defaults: a drag at 5.5 cm ended after %d frames", frames);
|
||||
Run open{Pinch(p), Grip()};
|
||||
for (int k = 0; k < 5; ++k) open.Step(0.06);
|
||||
for (int k = 0; k < 30; ++k) open.Step(0.01);
|
||||
open.Step(0.07), open.Step(0.07);
|
||||
CHECK(!open.down() && open.lost == 0, "defaults: a drag opened to 7 cm ends on the second frame");
|
||||
}
|
||||
|
||||
void TestDragSpike() {
|
||||
// a drag (held 1 s): the tips read 3.8 cm for two frames (a hidden thumb), then close again
|
||||
Run r{Pinch(Params()), Grip()};
|
||||
@@ -267,6 +295,7 @@ int main() {
|
||||
TestGripTakeover();
|
||||
TestDip();
|
||||
TestFistGuard();
|
||||
TestDefaults();
|
||||
if (failures) {
|
||||
std::printf("%d failures\n", failures);
|
||||
return 1;
|
||||
|
||||
@@ -0,0 +1,51 @@
|
||||
#!/usr/bin/env python3
|
||||
"""A testing session take's live.bin (hands/rec/livelog.py) as a publish dump (FHDUMP02) for
|
||||
screens/tests/handinput_test --replay and --take, with the take's poses.jsonl beside it, so what
|
||||
the panels did with the hands can be played again.
|
||||
|
||||
live_dump.py TAKE_DIR OUT.dump (writes OUT.dump and OUT.poses.jsonl)
|
||||
|
||||
Each hands publish goes out with the gestures file as it stood then; dq is when the recorder
|
||||
copied the hands file. Standard library only.
|
||||
"""
|
||||
import shutil
|
||||
import struct
|
||||
import sys
|
||||
|
||||
FILE_HDR = struct.Struct("<8sII")
|
||||
REC_HDR = struct.Struct("<B3xIQQ")
|
||||
|
||||
|
||||
def main(take, out):
|
||||
with open(take + "/live.bin", "rb") as f:
|
||||
data = f.read()
|
||||
if data[:8] != b"FHLIVE01":
|
||||
sys.exit("%s/live.bin: not a live log" % take)
|
||||
off, gest, recs = FILE_HDR.size, None, []
|
||||
while off + REC_HDR.size <= len(data):
|
||||
src, n, read_ns, _ = REC_HDR.unpack_from(data, off)
|
||||
body = data[off + REC_HDR.size:off + REC_HDR.size + n]
|
||||
off += REC_HDR.size + n
|
||||
if len(body) < n:
|
||||
break
|
||||
if src == 1:
|
||||
gest = body
|
||||
elif src == 0 and gest is not None:
|
||||
recs.append((body, gest, read_ns))
|
||||
if not recs:
|
||||
sys.exit("%s/live.bin: no hands with gestures" % take)
|
||||
hs, gs = len(recs[0][0]), len(recs[0][1])
|
||||
with open(out, "wb") as f:
|
||||
f.write(b"FHDUMP02" + struct.pack("<II", hs, gs))
|
||||
for h, g, dq in recs:
|
||||
if len(h) == hs and len(g) == gs:
|
||||
f.write(h + g + struct.pack("<q", dq))
|
||||
base = out[:-5] if out.endswith(".dump") else out
|
||||
shutil.copyfile(take + "/poses.jsonl", base + ".poses.jsonl")
|
||||
print("%s: %d publishes" % (out, len(recs)))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if len(sys.argv) != 3:
|
||||
sys.exit("live_dump.py TAKE_DIR OUT.dump")
|
||||
main(sys.argv[1], sys.argv[2])
|
||||
Executable
+202
@@ -0,0 +1,202 @@
|
||||
#!/usr/bin/env python3
|
||||
"""A testing session take's flags: the headset button's marks (prompts.jsonl "mark") and what the
|
||||
person said (voice.bin, hands/rec/voicelog.py), each with what Frametop did with the hands around
|
||||
it (events.jsonl, hands/rec/eventlog.py), in time order.
|
||||
|
||||
voice_flags.py TAKE_DIR [--out FLAGS.jsonl] [--server URL] [--before S] [--after S]
|
||||
|
||||
The voice is cut into bursts of speech by its level (100 ms windows over the noise floor, gaps
|
||||
under GAP_S joined), and each burst is transcribed on its own by a whisper.cpp server on this
|
||||
machine (default frame-voice's, http://127.0.0.1:8178): its times stay exact, and whisper never
|
||||
hears the silence it makes words up for ("Thank you. Bye."). Only a loopback address is accepted,
|
||||
so the voice never leaves the machine. A note within PAIR_S of a mark says which mark it's about
|
||||
("mark": n).
|
||||
|
||||
Each flag, one JSON line in --out (and a line each on stdout):
|
||||
{"t": ns (CLOCK_MONOTONIC, the take's clock), "s": seconds into the take, "source": "mark" | "voice",
|
||||
"n": the mark's number | "text", "end_t", "mark": the nearest mark's number or null,
|
||||
"events": [events from `before` s ahead of it to `after` s after it (a note: after its end)]}
|
||||
|
||||
Standard library only.
|
||||
"""
|
||||
import argparse
|
||||
import array
|
||||
import ipaddress
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
import uuid
|
||||
import wave
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
sys.path.insert(0, os.path.join(os.path.dirname(HERE), "rec"))
|
||||
import eventlog # noqa: E402
|
||||
import voicelog # noqa: E402
|
||||
|
||||
SERVER = "http://127.0.0.1:8178"
|
||||
GAP_S = 1.0 # a pause shorter than this stays in the burst
|
||||
PAD_S = 0.3 # kept around each burst, so its first and last words aren't cut
|
||||
MIN_RMS = 150 # s16; a burst's windows are over this and over 4x the noise floor
|
||||
PAIR_S = 5.0
|
||||
# whisper's words for silence and noise, not speech
|
||||
NOT_SPEECH = ("[blank_audio]", "(silence)", "[silence]", "[music]", "(music)", "[noise]", "(noise)",
|
||||
"[inaudible]", "(inaudible)", "[ silence ]")
|
||||
|
||||
|
||||
def loopback(url):
|
||||
"""True for an http URL whose host is this machine."""
|
||||
u = urllib.parse.urlsplit(url)
|
||||
if u.scheme != "http" or not u.hostname:
|
||||
return False
|
||||
if u.hostname == "localhost":
|
||||
return True
|
||||
try:
|
||||
return ipaddress.ip_address(u.hostname).is_loopback
|
||||
except ValueError:
|
||||
return False
|
||||
|
||||
|
||||
def transcribe(wav, server):
|
||||
"""[(start_s, end_s, text)] of the WAV, from the whisper.cpp server's verbose_json."""
|
||||
boundary = uuid.uuid4().hex
|
||||
with open(wav, "rb") as f:
|
||||
audio = f.read()
|
||||
parts = []
|
||||
for name, value in (("response_format", "verbose_json"), ("temperature", "0")):
|
||||
parts.append(f'--{boundary}\r\nContent-Disposition: form-data; name="{name}"\r\n\r\n{value}\r\n'.encode())
|
||||
parts.append(f'--{boundary}\r\nContent-Disposition: form-data; name="file"; filename="voice.wav"\r\n'
|
||||
f'Content-Type: audio/wav\r\n\r\n'.encode() + audio + b"\r\n")
|
||||
parts.append(f"--{boundary}--\r\n".encode())
|
||||
req = urllib.request.Request(server.rstrip("/") + "/inference", data=b"".join(parts),
|
||||
headers={"Content-Type": f"multipart/form-data; boundary={boundary}"})
|
||||
with urllib.request.urlopen(req, timeout=600) as r:
|
||||
out = json.load(r)
|
||||
segs = []
|
||||
for s in out.get("segments", []):
|
||||
text = (s.get("text") or "").strip()
|
||||
if text and text.lower() not in NOT_SPEECH:
|
||||
segs.append((float(s["start"]), float(s["end"]), text))
|
||||
return segs
|
||||
|
||||
|
||||
def bursts(samples, rate, gap=GAP_S):
|
||||
"""[(start, end)] sample ranges of speech in samples (an array of s16)."""
|
||||
win = rate // 10
|
||||
rms = [math.sqrt(sum(v * v for v in samples[i:i + win]) / win) for i in range(0, len(samples) - win + 1, win)]
|
||||
quiet = sorted(rms)
|
||||
floor = quiet[len(quiet) // 10] if quiet else 0 # the quietest tenth: silence between notes
|
||||
loud = max(MIN_RMS, 4 * floor)
|
||||
out = []
|
||||
for k, r in enumerate(rms):
|
||||
if r < loud:
|
||||
continue
|
||||
if out and k - out[-1][1] <= gap * 10:
|
||||
out[-1][1] = k + 1
|
||||
else:
|
||||
out.append([k, k + 1])
|
||||
pad = int(PAD_S * rate)
|
||||
return [(max(0, a * win - pad), min(len(samples), b * win + pad)) for a, b in out]
|
||||
|
||||
|
||||
def notes(wav, server, tmp):
|
||||
"""[(start_s, end_s, text)] of the speech in wav, a burst at a time."""
|
||||
with wave.open(wav) as w:
|
||||
rate = w.getframerate()
|
||||
samples = array.array("h", w.readframes(w.getnframes()))
|
||||
out = []
|
||||
for a, b in bursts(samples, rate):
|
||||
part = os.path.join(tmp, "burst.wav")
|
||||
with wave.open(part, "wb") as w:
|
||||
w.setnchannels(1)
|
||||
w.setsampwidth(2)
|
||||
w.setframerate(rate)
|
||||
w.writeframes(samples[a:b].tobytes())
|
||||
text = " ".join(t for _, _, t in transcribe(part, server))
|
||||
if text:
|
||||
out.append((a / rate, b / rate, text))
|
||||
return out
|
||||
|
||||
|
||||
def read_jsonl(path):
|
||||
out = []
|
||||
try:
|
||||
with open(path) as f:
|
||||
for line in f:
|
||||
try:
|
||||
e = json.loads(line)
|
||||
except ValueError:
|
||||
continue
|
||||
if isinstance(e, dict) and isinstance(e.get("t"), int):
|
||||
out.append(e)
|
||||
except OSError:
|
||||
pass
|
||||
return out
|
||||
|
||||
|
||||
def flags(take, server=SERVER, before=5.0, after=2.0):
|
||||
"""The take's flags (see the top), in time order."""
|
||||
with open(os.path.join(take, "take.json")) as f:
|
||||
start = json.load(f).get("started_ns", 0)
|
||||
marks = [e for e in read_jsonl(os.path.join(take, "prompts.jsonl")) if e.get("event") == "mark"]
|
||||
events = sorted(read_jsonl(os.path.join(take, "events.jsonl")), key=lambda e: e["t"])
|
||||
out = [{"t": m["t"], "source": "mark", "n": m.get("n")} for m in marks]
|
||||
voice = os.path.join(take, "voice.bin")
|
||||
if os.path.exists(voice):
|
||||
with tempfile.TemporaryDirectory(prefix="voice-flags-") as tmp:
|
||||
wav = os.path.join(tmp, "voice.wav")
|
||||
try:
|
||||
first = voicelog.to_wav(voice, wav)
|
||||
except ValueError:
|
||||
first = None # no sound recorded
|
||||
if first is not None:
|
||||
for a, b, text in notes(wav, server, tmp):
|
||||
out.append({"t": first + int(a * 1e9), "end_t": first + int(b * 1e9),
|
||||
"source": "voice", "text": text})
|
||||
for f in out:
|
||||
f["s"] = round((f["t"] - start) / 1e9, 2) if start else None
|
||||
if f["source"] == "voice":
|
||||
near = [m for m in marks if abs(m["t"] - f["t"]) <= PAIR_S * 1e9]
|
||||
f["mark"] = min(near, key=lambda m: abs(m["t"] - f["t"])).get("n") if near else None
|
||||
lo, hi = f["t"] - int(before * 1e9), f.get("end_t", f["t"]) + int(after * 1e9)
|
||||
f["events"] = [e for e in events if lo <= e["t"] <= hi]
|
||||
out.sort(key=lambda f: f["t"])
|
||||
return out
|
||||
|
||||
|
||||
def line(f):
|
||||
"""' 34.1 s mark 1: poke missed, grab-that done' or ' 36.0 s voice (mark 1): "..."'."""
|
||||
what = eventlog.describe(eventlog.summary(f["events"]))
|
||||
s = "%7.1f s" % f["s"] if f["s"] is not None else " ? s"
|
||||
if f["source"] == "mark":
|
||||
return "%s mark %s: %s" % (s, f.get("n"), what)
|
||||
pair = " (mark %s)" % f["mark"] if f.get("mark") is not None else ""
|
||||
return '%s voice%s: "%s" [%s]' % (s, pair, f["text"], what)
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
|
||||
ap.add_argument("take", help="a testing session's take directory (voice.bin, prompts.jsonl, events.jsonl)")
|
||||
ap.add_argument("--out", help="write the flags here, one JSON line each")
|
||||
ap.add_argument("--server", default=SERVER, help="whisper.cpp server on this machine (default %(default)s)")
|
||||
ap.add_argument("--before", type=float, default=5.0, help="events this many seconds ahead of a flag (default 5)")
|
||||
ap.add_argument("--after", type=float, default=2.0, help="and this many after it (default 2)")
|
||||
a = ap.parse_args()
|
||||
if not loopback(a.server):
|
||||
sys.exit("--server must be on this machine (127.0.0.1, ::1 or localhost): the voice stays here")
|
||||
out = flags(a.take, a.server, a.before, a.after)
|
||||
for f in out:
|
||||
print(line(f))
|
||||
if a.out:
|
||||
with open(a.out, "w") as fh:
|
||||
for f in out:
|
||||
fh.write(json.dumps(f, separators=(", ", ": ")) + "\n")
|
||||
print("%d marks, %d spoken notes" % (sum(f["source"] == "mark" for f in out),
|
||||
sum(f["source"] == "voice" for f in out)), file=sys.stderr)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -371,7 +371,7 @@ int main(int argc, char **argv) {
|
||||
" [--flag-buffer 0|1] (1: keep the last seconds for `ft-handsctl flag`; HANDS_FLAG_BUFFER)\n"
|
||||
" [--cams auto|mono|color|all] (auto) [--bright all|color] (all) [--bright-on L] (40) [--bright-off L] (25)\n"
|
||||
" [--color-left color_video0|color_video3] [--color-crop subtract|none]\n"
|
||||
" [--pinch-begin M] (0.020) [--pinch-end M] (0.035) [--pinch-triangulated] [--pinch-palm-down MAX] (1: off)\n"
|
||||
" [--pinch-begin M] (0.020) [--pinch-end M] (0.045) [--pinch-triangulated] [--pinch-palm-down MAX] (1: off)\n"
|
||||
" [--pinch-dip 0|1] (0; 1: also on a sharp dip, see PinchParams::dip; HANDS_PINCH_DIP)\n"
|
||||
" [--pinch-fist-curl R] (1.2; 0: off: no pinch begins on a fist, see PinchParams::fist_curl;\n"
|
||||
" HANDS_PINCH_FIST_CURL)\n"
|
||||
|
||||
+7
-3
@@ -15,7 +15,10 @@ extern "C" {
|
||||
|
||||
struct PinchParams {
|
||||
double begin_m = 0.020; // thumb and index tips closer than this: the pinch begins
|
||||
double end_m = 0.035; // further apart than this: it ends (the gap keeps it from flickering)
|
||||
// further apart than this: it ends (the gap keeps it from flickering). 0.045 since 2026-10-14
|
||||
// (0.035 before): taps and pinches let go while still held (user, 2026-10-11, gaze + hands: ends
|
||||
// came at 4-6 cm); these four were the headset's HANDS_PINCH_* settings since, now everyone's.
|
||||
double end_m = 0.045;
|
||||
int end_frames = 2; // processed frames in a row past end_m before it ends, so one
|
||||
// noisy frame doesn't drop a drag
|
||||
double grace_s = 0.25; // a pinching hand lost this long ends its pinch (FH_PINCH_LOST)
|
||||
@@ -27,11 +30,12 @@ struct PinchParams {
|
||||
// replay (pinches held 0.3 s or more, typing among them) the ends came 33-134 ms later, at most
|
||||
// 200, where any dip under end_m starting the wait over kept one held 6 s longer. A tap ends as
|
||||
// before: its release is the click.
|
||||
double drag_s = 0.3, drag_open_m = 0.05, drag_open_s = 0.12, drag_reclose_m = 0.03;
|
||||
double drag_s = 0.3, drag_open_m = 0.065, drag_open_s = 0.2, drag_reclose_m = 0.03; // 0.05, 0.12 before 2026-10-14
|
||||
// A grip takes this pinch's hand (ends it, as lost) only within grip_takeover_s of its begin:
|
||||
// closing the hand into a fist passes through a pinch. Later the pinch is being held (a drag),
|
||||
// and the other fingers curling around it don't end it (3 of 28 pinches ended so, the same test).
|
||||
double grip_takeover_s = 0.3;
|
||||
// 0.1 since 2026-10-14 (0.3 before: grips took pinches 130 ms in).
|
||||
double grip_takeover_s = 0.1;
|
||||
// Where the distance comes from: MediaPipe's world landmarks (the model's own 3D hand
|
||||
// pose, averaged over the hand's views, at the user's hand size), or the tracker's
|
||||
// triangulated tips. The model's pose should hold up better when the fingers hide each
|
||||
|
||||
@@ -20,11 +20,11 @@ These are Vibepollo's:
|
||||
|
||||
```bash
|
||||
mkdir -p ~/frametop-host && cd ~/frametop-host
|
||||
release=https://github.com/Frametop/frametop-vibepollo/releases/download/frametop-2.0.0-2
|
||||
release=https://github.com/Frametop/frametop-vibepollo/releases/download/frametop-2.0.0-3
|
||||
curl -fLO "$release/vibepollo-2.0.0-1-x86_64.pkg.tar.zst"
|
||||
curl -fLO "$release/vibepollo-2.0.0-1-x86_64.pkg.tar.zst.sha256"
|
||||
sha256sum -c vibepollo-2.0.0-1-x86_64.pkg.tar.zst.sha256
|
||||
curl -fLO https://raw.githubusercontent.com/Frametop/frametop-vibepollo/frametop-2.0.0-2/scripts/linux_install.sh
|
||||
curl -fLO https://raw.githubusercontent.com/Frametop/frametop-vibepollo/frametop-2.0.0-3/scripts/linux_install.sh
|
||||
```
|
||||
|
||||
`sha256sum -c` should print `OK`.
|
||||
@@ -52,7 +52,7 @@ sudo ufw allow Vibepollo
|
||||
- The resolution switch and the HDR guard of Frametop's build are Windows-only for now. On Linux, a monitor keeps its own resolution and the stream scales it.
|
||||
- Vibepollo runs as system services. `sudo systemctl status vibepollo-session-controller.service vibepollo.service` shows them, and `sudo journalctl -u vibepollo-session-controller.service -u vibepollo.service --since '-10 minutes'` shows their log. Don't start `vibepollo.service` yourself: the session controller starts it when you log in.
|
||||
- Settings, the Web UI login and pairings are in `/var/lib/vibepollo`.
|
||||
- Vibepollo's [Linux install guide](https://github.com/Frametop/frametop-vibepollo/blob/frametop-2.0.0-2/docs/linux/install.md) has the details and troubleshooting.
|
||||
- Vibepollo's [Linux install guide](https://github.com/Frametop/frametop-vibepollo/blob/frametop-2.0.0-3/docs/linux/install.md) has the details and troubleshooting.
|
||||
|
||||
## Remove
|
||||
|
||||
|
||||
+33
-8
@@ -7,12 +7,15 @@
|
||||
# In a release (pack/install-release.sh), the programs come built from its image: this installs the
|
||||
# distrobox the release brings, makes the release's container from its image, and builds nothing.
|
||||
#
|
||||
# Usage: ./install.sh [--yes] [--no-voice] [--no-hands] [--no-eye-tracker] [--no-bluetooth | --bluetooth]
|
||||
# --yes don't ask; installs voice dictation and gaze mode, and hand tracking and our
|
||||
# eye tracker if sudo can run without a password prompt; skips the Bluetooth
|
||||
# fixes and the SteamVR restart
|
||||
# Usage: ./install.sh [--yes] [--no-voice] [--no-hands] [--no-hand-recorder] [--no-eye-tracker]
|
||||
# [--no-bluetooth | --bluetooth]
|
||||
# --yes don't ask; installs voice dictation and gaze mode, and hand tracking, the Hand
|
||||
# Recorder and our eye tracker if sudo can run without a password prompt; skips
|
||||
# the Bluetooth fixes and the SteamVR restart
|
||||
# --no-voice don't offer voice dictation (Frametop Voice: it downloads about 320 MB)
|
||||
# --no-hands don't offer hand tracking
|
||||
# --no-hands don't offer hand tracking or the Hand Recorder
|
||||
# --no-hand-recorder don't offer the Hand Recorder (a release's; a clone builds it with
|
||||
# hands/rec/install.sh)
|
||||
# --no-eye-tracker don't offer our own eye tracker
|
||||
# --no-bluetooth don't offer the Bluetooth fixes
|
||||
# --bluetooth install the Bluetooth fixes without asking (with --yes: if sudo can run
|
||||
@@ -22,7 +25,7 @@ set -euo pipefail
|
||||
root=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
. "$root/scripts/_env.sh"
|
||||
|
||||
assume_yes=0 bluetooth=1 eye_tracker=1 hands=1 voice=1
|
||||
assume_yes=0 bluetooth=1 eye_tracker=1 hands=1 hand_recorder=1 voice=1
|
||||
for arg in "$@"; do
|
||||
case $arg in
|
||||
--yes) assume_yes=1 ;;
|
||||
@@ -30,8 +33,9 @@ for arg in "$@"; do
|
||||
--bluetooth) bluetooth=2 ;;
|
||||
--no-eye-tracker) eye_tracker=0 ;;
|
||||
--no-voice) voice=0 ;;
|
||||
--no-hands) hands=0 ;;
|
||||
-h|--help) sed -n '2,19p' "$0"; exit 0 ;;
|
||||
--no-hands) hands=0 hand_recorder=0 ;;
|
||||
--no-hand-recorder) hand_recorder=0 ;;
|
||||
-h|--help) sed -n '2,22p' "$0"; exit 0 ;;
|
||||
*) echo "unknown option: $arg" >&2; exit 2 ;;
|
||||
esac
|
||||
done
|
||||
@@ -143,6 +147,7 @@ elif [ "$assume_yes" = 1 ] && ! sudo_quiet; then
|
||||
echo "skipped: it needs your password (sudo), and --yes doesn't ask. Install it later with: hands/run.sh install"
|
||||
elif ask "Install hand tracking? Poke, pinch and grab the screens with your hands. It runs while the Frametop desktop does (change that on the Hands page of Frametop Input Settings), and its camera broker needs your password (sudo)." y; then
|
||||
if "$root/hands/run.sh" install; then
|
||||
hands_caps=1
|
||||
mode=$(on_frame "sed -n 's/^HANDS_MODE=\([a-z]*\).*/\1/p' ~/.config/frametop.conf | tail -1") || mode=
|
||||
if [ "$mode" = always ]; then
|
||||
echo "hand tracking runs whenever SteamVR does (HANDS_MODE=always, kept from your settings)"
|
||||
@@ -159,6 +164,26 @@ elif ask "Install hand tracking? Poke, pinch and grab the screens with your hand
|
||||
else
|
||||
echo "skipped. Install later with: hands/run.sh install"
|
||||
fi
|
||||
# The Hand Recorder (hands/rec): a release brings it built, so it's only ft-camd's capabilities
|
||||
# (set already if hand tracking just installed) and the menu entry. An entry from the release
|
||||
# before points into that release's folder, which goes two releases on: it's renewed without asking.
|
||||
# A clone builds it with hands/rec/install.sh (a few minutes), so it isn't offered here.
|
||||
if [ "$FRAME_RELEASE" = 1 ]; then
|
||||
recorder="the Hand Recorder (records your hands for Frametop's open hand dataset)"
|
||||
if [ "$hand_recorder" = 0 ]; then
|
||||
echo "$recorder: skipped. Add it later with: hands/rec/install.sh"
|
||||
elif on_frame '[ -e ~/.local/share/applications/frametop-handrec.desktop ]'; then
|
||||
"$root/hands/rec/install.sh" install >/dev/null && echo "$recorder: updated" ||
|
||||
echo "$recorder: couldn't update it (above). Try again with: hands/rec/install.sh"
|
||||
elif [ "$assume_yes" = 1 ] && [ "${hands_caps:-0}" = 0 ] && ! sudo_quiet; then
|
||||
echo "$recorder: skipped: it needs your password (sudo), and --yes doesn't ask. Add it later with: hands/rec/install.sh"
|
||||
elif ask "Add the Hand Recorder to the app menu? It records your hands for Frametop's open hand dataset, which trains hand tracking; nothing leaves the headset until you press Upload. Its camera broker needs your password (sudo), as hand tracking does." y; then
|
||||
"$root/hands/rec/install.sh" install ||
|
||||
echo "$recorder: not installed (above). Try again later with: hands/rec/install.sh"
|
||||
else
|
||||
echo "$recorder: skipped. Add it later with: hands/rec/install.sh"
|
||||
fi
|
||||
fi
|
||||
|
||||
step "10/12 gaze mode (optional, experimental: the pointer goes where you look)"
|
||||
gaze=0
|
||||
|
||||
@@ -20,7 +20,7 @@ test-python:
|
||||
set -uo pipefail
|
||||
python3 -c 'import PySide6, numpy, cv2, sherpa_onnx' || { echo "python3 can't import PySide6, numpy, cv2, and sherpa_onnx"; exit 1; }
|
||||
status=0
|
||||
for t in input/test/*.py hands/tests/test_*.py hands/rec/tests/*.py gaze/test/*.py pack/test/*.py voice/test/*.py spotwatch/test/*.py; do
|
||||
for t in input/test/*.py hands/tests/test_*.py hands/rec/tests/*.py recorder/tests/*.py gaze/test/*.py gaze/rec/tests/*.py pack/test/*.py voice/test/*.py spotwatch/test/*.py; do
|
||||
[ -f "$t" ] || continue
|
||||
if python3 "$t" >/dev/null; then echo "$t: ok"; else echo "$t: FAILED"; status=1; fi
|
||||
done
|
||||
|
||||
@@ -98,6 +98,10 @@ RUN voice/build.sh \
|
||||
# file capabilities point, so they aren't copied to /opt/frametop.
|
||||
RUN hands/build.sh \
|
||||
&& test -x hands/build/ft-camd && test -x hands/build/ft-hands
|
||||
# The Hand Recorder's headset panel: it stays in hands/rec/build/, where the recorder
|
||||
# (hands/rec/session.py) runs it from.
|
||||
RUN hands/rec/build.sh \
|
||||
&& test -x hands/rec/build/ft-handpanel
|
||||
|
||||
# A release installs from this image (pack/install-release.sh): it copies
|
||||
# /src/frametop, built, to the Frame for its host side (services, the driver,
|
||||
|
||||
@@ -5,7 +5,8 @@
|
||||
# version, commit, the image file's sha256 and the image's ID, and the SteamOS table). Nothing
|
||||
# builds on the headset, and nothing else is downloaded, but voice dictation's speech models
|
||||
# (about 320 MB, fixed URLs checked by sha256: voice/models.sh) if you install it. Hand tracking
|
||||
# comes built too: its install only sets its camera broker's file capabilities (sudo).
|
||||
# and the Hand Recorder come built too: their installs only set their camera broker's file
|
||||
# capabilities (sudo), and the recorder's adds its menu entry.
|
||||
#
|
||||
# 1. It checks this SteamOS build against the SteamOS table: Frametop's newest from GitHub
|
||||
# (pack/steamos.json on main) when it can get it, else the one in the release. Tested: on.
|
||||
@@ -18,7 +19,8 @@
|
||||
# are removed, with their containers and images.
|
||||
#
|
||||
# Usage: install-release.sh [--yes] [--any-steamos] [--unpack-only] [--dir DIR]
|
||||
# [--no-voice] [--no-hands] [--no-eye-tracker] [--no-bluetooth | --bluetooth]
|
||||
# [--no-voice] [--no-hands] [--no-hand-recorder] [--no-eye-tracker]
|
||||
# [--no-bluetooth | --bluetooth]
|
||||
# --unpack-only stop after step 2 and the copy: don't run install.sh
|
||||
# --dir DIR where the releases go (default ~/.local/share/frametop/releases)
|
||||
# The rest go to install.sh. FRAMETOP_STEAMOS_TABLE names another table to fetch (a URL), or
|
||||
@@ -111,8 +113,8 @@ main() {
|
||||
--any-steamos) any=1 ;;
|
||||
--unpack-only) unpack_only=1 ;;
|
||||
--dir) base=${2:?--dir needs a folder}; shift ;;
|
||||
--no-voice|--no-hands|--no-eye-tracker|--no-bluetooth|--bluetooth) pass+=("$1") ;;
|
||||
-h|--help) sed -n '2,24p' "$0"; return 0 ;;
|
||||
--no-voice|--no-hands|--no-hand-recorder|--no-eye-tracker|--no-bluetooth|--bluetooth) pass+=("$1") ;;
|
||||
-h|--help) sed -n '2,27p' "$0"; return 0 ;;
|
||||
*) echo "unknown option: $1" >&2; return 2 ;;
|
||||
esac
|
||||
shift
|
||||
|
||||
@@ -6,7 +6,7 @@ On a Steam Frame (SteamOS, VR variant). Download **Frametop.zip** below (about 1
|
||||
- **On the headset:** unpack Frametop.zip (in the desktop's Dolphin or Konsole) and run `Frametop/frametop-install.sh`.
|
||||
- **In a terminal on the headset:** `curl -fsSL https://frametop.github.io/frametop/get.sh | bash -s -- --release`
|
||||
|
||||
A window asks what to install, and your SteamOS password for the three optional parts that need it (hand tracking, our own eye tracker, and the Bluetooth fixes); the password is only used for this install. Nothing is compiled on the headset. Restart SteamVR once afterwards.
|
||||
A window asks what to install, and your SteamOS password for the optional parts that need it (hand tracking, the Hand Recorder, our own eye tracker, and the Bluetooth fixes); the password is only used for this install. Nothing is compiled on the headset. Restart SteamVR once afterwards.
|
||||
|
||||
Before installing, the installer checks your SteamOS build against the builds Frametop was tested on, and stops if this release is known not to work there.
|
||||
|
||||
|
||||
@@ -506,6 +506,7 @@
|
||||
// POINTER_ROLE (right): gaze precision and keyboard clicks, above.
|
||||
#include <openvr.h>
|
||||
|
||||
#include "../../hands/include/fh_events.h"
|
||||
#include "../../hands/include/fh_ray.h"
|
||||
#include "calpanel.h"
|
||||
#include "gazehand.h"
|
||||
@@ -516,6 +517,22 @@
|
||||
#include "vrbuttons.h"
|
||||
#include "vrmath.h"
|
||||
|
||||
// A pinch's log line as a testing session's event (hands/include/fh_events.h): what it did,
|
||||
// from how the line starts (the lines are this file's own: pinchLog's callers, gazehand.h's Says).
|
||||
static void PinchEvent(int side, int kind, const std::string &what) {
|
||||
auto starts = [&](const char *p) { return what.rfind(p, 0) == 0; };
|
||||
const char *k = kind == 2 ? "right-click" : "click", *o = "info";
|
||||
if (starts("dragged")) k = "drag", o = "done";
|
||||
else if (starts("scrolled")) k = "scroll", o = "done";
|
||||
else if (starts("ignored") || starts("no click") || starts("it woke the pointer") || starts("no ray up: it woke")) o = "refused";
|
||||
else if (starts("left to ft-screens")) o = "passed";
|
||||
else if (starts("pressed on")) o = "lost"; // let go by losing the hand
|
||||
else if (starts("clicked") || starts("right click") || starts("became a press") || starts("nudged") ||
|
||||
starts("the right button") || starts("joined") || starts("the left button down"))
|
||||
o = "done";
|
||||
fh_event_text("pointer", k, o, side, 0, ((kind == 2 ? "middle pinch: " : "pinch: ") + what).c_str());
|
||||
}
|
||||
|
||||
#include <algorithm>
|
||||
#include <atomic>
|
||||
#include <chrono>
|
||||
@@ -1465,6 +1482,8 @@ int main() {
|
||||
: Clock::now() - lastMouse < std::chrono::seconds(1) ? "off: the mouse moved"
|
||||
: c.why;
|
||||
std::printf("hand ray: %s -> %s (%s)\n", rayown::SideName(c.from), rayown::SideName(c.to), why.c_str());
|
||||
fh_event("pointer", "ray", c.to >= 0 ? "done" : "end", c.to >= 0 ? c.to : c.from, 0, "hand ray: %s -> %s (%s)",
|
||||
rayown::SideName(c.from), rayown::SideName(c.to), why.c_str());
|
||||
if (!apply) continue;
|
||||
if (c.to >= 0) rayEnter(c.to);
|
||||
else rayLeave(c.off);
|
||||
@@ -2407,7 +2426,11 @@ int main() {
|
||||
seen.push_back(b);
|
||||
}
|
||||
arbiter.Update(secsOf(tnow), ctls, seen, live);
|
||||
for (const std::string &line : arbiter.log) std::printf("%s\n", line.c_str());
|
||||
for (const std::string &line : arbiter.log) {
|
||||
std::printf("%s\n", line.c_str());
|
||||
if (!line.rfind("handoff left:", 0) || !line.rfind("handoff right:", 0))
|
||||
fh_event_text("pointer", "handoff", "info", line[8] == 'r', 0, line.c_str());
|
||||
}
|
||||
if (!arbiter.log.empty()) std::fflush(stdout);
|
||||
arbiter.log.clear();
|
||||
// In a VR game, only with the dashboard open or Frametop toggled back on (not paused).
|
||||
@@ -2604,8 +2627,10 @@ int main() {
|
||||
if (!hold.gaze.Right()) tapArm = {hold.side, secsOf(tnow)}; // the right-drag's first tap (gazehand.h)
|
||||
} else {
|
||||
const std::string why = gazeEndWhy.empty() ? "the gaze calibration panel came up" : gazeEndWhy;
|
||||
const std::string says = hold.gaze.Says(secsOf(tnow), lost, why, "", hold.gaze.ByChord() ? lessonSaid : "");
|
||||
std::printf("%s %s: %s\n", hold.gaze.Right() && !hold.gaze.ByTap() ? "middle pinch" : "pinch", hold.side ? "right" : "left",
|
||||
hold.gaze.Says(secsOf(tnow), lost, why, "", hold.gaze.ByChord() ? lessonSaid : "").c_str());
|
||||
says.c_str());
|
||||
PinchEvent(hold.side, hold.gaze.Right() && !hold.gaze.ByTap() ? 2 : 0, says);
|
||||
std::fflush(stdout);
|
||||
}
|
||||
gazeEndWhy.clear();
|
||||
@@ -2686,6 +2711,7 @@ int main() {
|
||||
auto pinchLog = [&](int side, int kind, const std::string &what) {
|
||||
std::printf("%s %s: %s\n", kind == 2 ? "middle pinch" : "pinch", side ? "right" : "left", what.c_str());
|
||||
std::fflush(stdout);
|
||||
PinchEvent(side, kind, what);
|
||||
};
|
||||
// The chord's right button (see "Gaze pinch" at the top): to the driver as it changes, and the
|
||||
// pinch's log line once it's over.
|
||||
@@ -3891,8 +3917,9 @@ int main() {
|
||||
}
|
||||
// A gaze pinch's click went out (or was dropped with the pointer): its log line.
|
||||
if (gazeClick.On() && !clickPress) {
|
||||
std::printf("%s %s: %s\n", gazeClick.Right() ? "middle pinch" : "pinch", gazeClickSide ? "right" : "left",
|
||||
gazeClick.Says(secsOf(tnow), !clickRelease, "the pointer went", lastHit, lessonSaid).c_str());
|
||||
const std::string says = gazeClick.Says(secsOf(tnow), !clickRelease, "the pointer went", lastHit, lessonSaid);
|
||||
std::printf("%s %s: %s\n", gazeClick.Right() ? "middle pinch" : "pinch", gazeClickSide ? "right" : "left", says.c_str());
|
||||
PinchEvent(gazeClickSide, gazeClick.Right() ? 2 : 0, says);
|
||||
std::fflush(stdout);
|
||||
gazeClick = {};
|
||||
}
|
||||
|
||||
@@ -25,6 +25,8 @@ tracker = [
|
||||
"numpy>=2.2",
|
||||
"opencv-python-headless>=4.11",
|
||||
"huggingface_hub>=0.34",
|
||||
# the eye recorder's export encrypts with age (recorder/age.py), written on cryptography
|
||||
"cryptography>=42",
|
||||
]
|
||||
# Frametop Voice (voice/ft-voiced): Moonshine, the wake word spotter and Silero VAD run
|
||||
# through sherpa-onnx; sentencepiece spells out wake phrases (voice/requirements.txt has the
|
||||
|
||||
Loaded 100 of 124 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user