diff --git a/hands/rec/DESIGN.md b/hands/rec/DESIGN.md index a707f31..276ea0c 100644 --- a/hands/rec/DESIGN.md +++ b/hands/rec/DESIGN.md @@ -11,6 +11,7 @@ The plan this belongs to is `~/Desktop/Projects/frame-hands/notes/hands-plan.md` | `hands/rec/panel/ft-handpanel.cpp` (C++, dev container) | The headset panel: a SteamVR overlay fixed to the head that shows the prompts. It also places the "touch the dot" target in the room and logs head and controller poses. Driven over `@ft_handpanel`. | | `hands/rec/session.py` (Python, no Qt) | The session runner. It reads `script.json`, starts and stops the recordings, and drives the panel. It reads the live hands file for feedback and writes each take's files. It also runs from the command line (`--dry-run`) for testing. | | `hands/rec/script.json` | The guided script: sections, prompts, timings. | +| `hands/rec/poses/` | The pose pictures, `poses.json` and its PNGs (below). | | `hands/rec/takes.py` (Python, no Qt) | Reads sessions and takes from disk: frame sets for review, deleted ranges, export (compress, strip, manifest, checksums). | | `hands/rec/ft_handrec.py` + `main.qml` (PySide6 + Kirigami, dev container) | The desktop window: consent, the before-you-start checklist, the session controls, review, export and upload instructions. | | `hands/rec/ft-handrec` | The host launcher, like `input-settings/ft-input-settings`. | @@ -24,13 +25,16 @@ Every part runs in the dev container, as ft-hands and Input Settings do. The hos - **ft-camd** publishes the camera ring. If it isn't running, the session starts it as the transient user unit `frametop-handrec-camd.service`, the way `hands/ft-cutouts` starts `frametop-cutouts-camd.service` (needs `hands/build/ft-camd` with capabilities: `hands/run.sh caps`). An ft-camd already running from ft-cutouts or ft-handsctl is used as it is. - **A tracking ft-hands** gives feedback through the hands file: which hands are seen, the palm's distance, the index tip. If none is running, the session starts `ft-hands --no-gestures --status 0` (unit `frametop-handrec-hands.service`). An ft-hands already running is used as it is. -- **A recording ft-hands** runs once per take: `ft-hands --record-only --record TAKE_DIR --record-for SECONDS --record-hz 10 --status 0`. It runs as a plain child process of the session, ended with SIGTERM if the take stops early. SIGTERM ends ft-hands' loop, and `Recorder` writes out its queue when it's destroyed. +- **A recording ft-hands** runs once per recording part: `ft-hands --record-only --record DIR --record-for SECONDS --record-hz 10 --status 0`. It runs as a plain child process of the session, ended with SIGTERM when the part ends. SIGTERM ends ft-hands' loop, and `Recorder` writes out its queue when it's destroyed. In step mode (below) a part is one step's countdown and hold, so a take has one part per step (about 40 in the hand poses); in auto mode a take is one part, plus one more after each pause. `--record-for` is only a safety net. + - Why a process per part rather than one kept alive and paused: measured in the dev container with `ft-ringplay`'s ring (2026-10-02), `ft-hands --record-only` writes its first set 16-27 ms after it starts and ends 4-6 ms after SIGTERM, so a new part costs nothing the 3 s countdown doesn't cover. Every reader already takes parts in order (`takes.py`, `validate.py` through the export's single stream, the labeller's `fhl_io.py`, numbering `sets-10.bin` after `sets-9.bin`), ft-hands needs no new control, and nothing is written while a step waits. Before the hold starts the session checks that the part has written a set (`Recorder.has_data`, up to 3 s more), so the hold is recorded from its first frame. - **ft-handpanel** runs as a child process with `--watch-stdin`. It shows the panel and logs poses during each take. Test hooks: - `--ring PATH` goes to both ft-hands (`ft-ringplay` publishes a recording there, so the whole flow runs without the headset). - `--no-start` uses only what's already running. - `--dry-run` runs no processes and only prints the panel commands, with timing sped up by `--speed X`. +- `--next-after S` presses Next by itself after S seconds of waiting (real time), so a step-mode session runs unattended. Lines on stdin steer it too: `n` or an empty line is Next, `p` pause or resume, `r` redo, `s` skip the section, `q` stop. +- `--auto` runs the timed flow; `--poses DIR` takes the pose pictures from DIR; `--plan` prints the sections, their steps and length. ## Files @@ -38,11 +42,12 @@ Test hooks: ~/.local/share/frametop/hands/contrib/ profile.json consent and contributor id (below) sessions// (a second session started in the same second gets -2, and so on) - session.json the session: checklist answers, lighting, versions, takes + session.json the session: checklist answers, lighting, versions, mode, takes calibration.json /persist/xrservice.json with identifying fields removed (below) device.json the rig's pose in the CAD frame from /persist/device_config.json (below) takes/-
/ - sets.bin ft-hands' recording (FHSET01, hands/track/record.h), 10 sets/s + sets.bin ft-hands' recording (FHSET01, hands/track/record.h), 10 sets/s: the first part + sets-2.bin, sets-3.bin, ... the next parts (step mode: one per step; any mode: after a pause) prompts.jsonl what the person was asked to do, when (below) poses.jsonl head and controller poses from ft-handpanel (below) take.json {"section", "title", "started_ns", "ended_ns", "status": "complete"|"stopped"|"skipped", @@ -69,9 +74,11 @@ No name, email, or account. The contributor id is random, so several sessions fr "checklist": {"objects": ["pencil", "phone", "cup", "keyboard", "mouse", "gamepad", "small"], "own_objects": ["..."], "controllers": "straps|none", "sleeves": "short|long|", "rings": false, "watch": false, "notes": ""}, "device": {"steamos": "", "steamvr": "", "cameras": [{"name", "width", "height"}]}, - "takes": ["01-hand-size", "..."]} + "mode": "step|auto", "takes": ["01-hand-size", "..."]} ``` +A session started before step mode existed has no `mode`: it ran as `auto`. + ### calibration.json This is a copy of `/persist/xrservice.json` (`/run/host/persist/` in the container). Keep the cameras' intrinsics and extrinsics, and drop anything that identifies the unit: keys containing `serial`, `sn`, `uuid`, `mac` or `id`, or values that look like serial numbers. List what was removed in `session.json` (`"calibration_removed": [...]`), so a reviewer can check it. @@ -100,10 +107,24 @@ One JSON object per line: {"t": 123, "event": "feedback", "left": true, "right": false, "palm_m": [0.0, 0.0]} {"t": 123, "event": "pause"} {"t": 123, "event": "resume"} +{"t": 123, "event": "ready", "id": "static-poses/fist/left/near", "seconds": 3} +{"t": 123, "event": "wait"} +{"t": 123, "event": "redo", "id": "static-poses/fist/left/near", "from": 123, "to": 123} {"t": 123, "event": "end", "status": "complete|stopped|skipped"} ``` -`feedback` is written about twice a second. It's the live tracker's view, kept for later checks; it's not a label. +`feedback` is written about twice a second while recording. It's the live tracker's view, kept for later checks; it's not a label. + +A prompt holds from its `prompt` until the next `prompt`, `ready`, `wait` or `end`. A step in step mode reads: + +``` +ready Next pressed: recording part N starts, the 3-2-1 countdown runs (recorded, no label) +prompt the hold: its labels start here +(bar, target, feedback, pause/resume during the hold) +wait the hold is over: no labels from here; part N stops +``` + +The touch-the-dot targets after the first follow straight on: no `ready` or `wait` between them. `redo` marks a try done again (R): `from` is that step's `ready` (or its `prompt` if it had none), `to` its end. Its prompt is skipped; the sets stay. In auto mode there's no `ready` or `wait`, and the intro is a recorded prompt `
/intro`. Readers that knew only `prompt` and `end` keep working, but they'd give the countdown to the step before: `hub_review.py` (frame-hands `train/hub`) shows it as "(countdown)" and redone prompts as "(redone)"; `FORMAT.md` in the dataset repo has the rules. ### poses.jsonl @@ -118,7 +139,8 @@ ft-handpanel writes one line per sample, 250 a second, from `GetDeviceToAbsolute ## The panel (`ft-handpanel`) -- **Placement.** A SteamVR overlay fixed to the head, like `gaze/panel/ft-gazepanel.cpp`: key `frametop.handpanel`, sort order 250. It sits 1.2 m ahead, centred 12 degrees above straight ahead, so the hands stay clear below it. It's 36 degrees wide, 4:3, 1024x768 pixels, dim and see-through. It's drawn on the CPU with stb_truetype into three shared DMA-BUFs SteamVR imports once, as ft-gazepanel does, and is drawn again only when something changes. +- **Placement.** A SteamVR overlay fixed to the head, like `gaze/panel/ft-gazepanel.cpp`: key `frametop.handpanel`, sort order 250. It sits 1.2 m ahead, centred 12 degrees above straight ahead, so the hands stay clear below it. It's 36 degrees wide, 4:3, 1024x768 pixels, dim and see-through. It's drawn on the CPU with stb_truetype (and stb_image for the pose pictures, PNG only, the same pinned stb commit) into three shared DMA-BUFs SteamVR imports once, as ft-gazepanel does, and is drawn again only when something changes. +- **Layout.** The title and step on top (a red "Rec" by the step while recording), a rule. With a pose picture or a diagram, a 14-degree column on the left holds the picture (or the flipped copy and the picture side by side) and the where-to diagram under it; the text takes the right. The text column holds the instruction, the orange note, the big countdown ("3", "2", "1", then "Hold" or "Go") and the cyan action line ("Ready? Press Space or click Next"), centred together. At the bottom: the near/far bar, the hand chips, the time-left bar and the key hints. - **Socket.** Abstract unix datagram `@ft_handpanel` (`--socket NAME`). A sender with an address gets `ok ...` or `error ...`. - **Options:** `--watch-stdin` (quit when stdin closes), `--socket NAME`, `--distance M`, `--no-vr`. `--no-vr` makes no SteamVR connection and prints each picture's text to stdout: for testing without a headset. @@ -135,20 +157,39 @@ Commands (UTF-8; `|` starts a new line in text): | `hands ` | Two chips, "Left hand" and "Right hand", each `seen` (green), `lost` (orange) or `off` (hidden). | | `bar [near label] [far label]` / `bar off` | The near/far bar for the push out and back: a horizontal track with a target marker and the hand's current position. | | `paused on` / `paused off` | A "Paused" overlay over the picture. | +| `image [mirror\|both]` / `image off` | The pose picture, a PNG (below), in the left column. `mirror` flips it (a left hand); `both` draws a flipped copy on its left. A file that can't be read: `error ...` and no picture. | +| `where ` / `where off` | The where-to diagram under the picture: a 3x3 front view with the asked cell lit (`centre`, `left`, `right`, `up`, `down`, and the push sections' `chest`, `desk`, `eye`), and a side view of the head and three marks for `near` ("Close"), `mid` ("Halfway out") and `far` ("Arm out"). `-` leaves that half out. | +| `action ` | The cyan line under the instruction (empty clears it). | +| `big ` | Large cyan text under the instruction: the countdown (empty clears it). | +| `keys ` | The faint key hints along the bottom. | +| `rec on` / `rec off` | The red "Rec" by the step line. | | `target [show\|hold <0..1>\|done]` / `target off` | The touch target: a small sphere-like dot about 2 cm across, in its own overlay (`frametop.handpanel.target`). The point is in the head frame (metres, +x right, +y up, -z forward). The first `target` with a new point places it in the room with the current HMD pose, and it stays there. Later commands with the same point change only the state. `hold` draws a filling ring, `done` turns it green. Reply: `ok `. | | `poses start ` / `poses stop` | Log poses to the path (appending, `poses.jsonl` format above) from a thread at 250 Hz, until stopped. | | `devices` | Reply: `ok hmd left right `, the current `ETrackingResult` values (`-` for no device in that role). | | `head` | Reply: `ok <12 floats>`, the current HMD pose (standing universe). | | `ping` | `ok shown` or `ok hidden`. | -It quits on SteamVR's quit event, as ft-gazepanel does. +It quits on SteamVR's quit event, as ft-gazepanel does. `--no-vr --dump DIR` writes each picture to `DIR/panel.pam`, to check the layout without a headset. + +## The pose pictures (`poses/`) + +`poses/poses.json` maps the script's `pose` ids to pictures: `{"": {"file": ".png", "two_hands": false, "caption": "..."}}`. Each PNG is RGBA, square (512x512), drawn as the wearer sees it: a right hand, unless `two_hands` (then it shows both). How a prompt shows it (`session.pose_view`): + +| Prompt's `hands` | Picture | +|---|---| +| `right` (and `any`, `none`, empty) | as drawn | +| `left` | flipped left to right (`image ... mirror`) | +| `both`, not `two_hands` | a flipped copy on the left, the picture on the right (`image ... both`) | +| anything, `two_hands` | as drawn | + +A pose with no entry, or whose file is missing, shows no picture: the text alone. The session reads `poses.json` when it starts. The window shows the same picture (QML `Image.mirror`), its caption, and the same diagram. ## The script (`script.json`) ```json {"version": 1, "sections": [ - {"id": "hand-size", "title": "Hand size", "requires": [], "intro": "text shown for 4 s before the first prompt", + {"id": "hand-size", "title": "Hand size", "requires": [], "intro": "text shown before the first prompt: 4 s, or until Next", "go": "Hold", "prompts": [{"text": "...", "seconds": 8, "hands": "both", "pose": "flat", "distance": "near"}]}, {"id": "objects", "title": "Things you hold", "requires": ["objects"], "for_each": "object", "prompts": [{"text": "Pick up the {object} and use it the way you normally would.", "seconds": 15, "hands": "both", "object": "{object}"}]}, @@ -160,11 +201,26 @@ It quits on SteamVR's quit event, as ft-gazepanel does. ``` - `requires`: `objects` (at least one object ticked), `controllers` (straps ticked). A section whose requirements aren't met is skipped and logged. +- `go`: the word the countdown ends on, "Go" unless set ("Hold" for the still poses). - `kind`: - `prompts` (the default): each prompt shows for its `seconds` with a countdown. - `targets`: each target shows until the live index tip is within 3 cm of it for `hold_s`, or `timeout_s` passes. - `bar`: the target marker sweeps near to far and back, `reps` times per height, at `period_s` per sweep. The current marker follows the live palm distance. -- Each section is one take, one recording. Prompts within it are marked in `prompts.jsonl`. +- Each section is one take. Prompts within it are marked in `prompts.jsonl`. + +### Step mode (the default) and auto mode + +The first in-headset session (2026-10-02) went too fast: each prompt advanced after 4-8 s, before there was time to read it and find the hand shape. So by default every step waits: + +1. **Ready.** The panel shows the step: section title, "step N of M", the instruction, the pose picture, the where-to diagram, and "Ready? Press Space or click Next". The hand chips show which hands are seen, with no warnings yet. It waits as long as it takes, and nothing records. +2. **Countdown.** Next starts a new recording part and a "ready" event, and the panel counts 3, 2, 1 (big), recorded so the hold is captured from its first frame. In the push sections the bar sits at near meanwhile. +3. **Hold.** The `prompt` event, the section's word ("Hold" or "Go") and the time-left bar for the prompt's seconds (or the bar's sweeps, or the targets). Then a `wait` event and the part stops. + +Steps that wait: each prompt, each bar height, and the first touch-the-dot target (the others follow straight on, as each waits for the touch anyway). Before a section, one screen shows the section's intro (with its `before` text, such as putting on the controllers) and waits for Next too; the welcome screen as well. The take starts with the section's first countdown, so a section skipped at its intro leaves no take. A pause in a hold works as before (the part stops; resume starts the next one). + +Auto mode ("Advance by itself" on the checklist page, `session.py --auto`) is the old timed flow: the welcome, the between and before screens, the intro (recorded) and each prompt for its seconds, one recording per take. R still works there: it restarts the step running. + +Holds are 5 s for still poses (4 s counting fingers), 8-10 s for movements. The core session (no objects, no controllers) records about 11 min in 62 steps; with 5 s of reading a step that's about 16 min. Everything ticked: about 16.5 min recorded in 86 steps. The window and `session.py --plan` give these (`plan_summary`); in step mode they leave the reading time out and say so. The sections, in this order (see the plan): 1. hand size @@ -189,9 +245,9 @@ Before section 8: "Put on both controllers and tighten the straps". Before secti ### Controls -- The window has Start, Pause/Resume, Skip section and Stop. Space pauses and Esc stops while the window has focus. -- The panel's text says what to do: "Pause: Space in the Hand recorder window". -- A pause stops the take's recording and starts it again on resume as the next part of the same take (`sets.bin` is appended to as a second recording file `sets-2.bin`, and so on). takes.py reads all parts in order. +- The window has Start, a big Next (while a step waits), Pause/Resume, Redo step, Skip section and Stop. While it has focus: Space is Next, P pauses or resumes, R redoes, S skips the section, Esc stops. The panel's bottom line and the window list them. The window also shows the step's picture, diagram, countdown and "Hold". +- **R (redo).** During a step's countdown or hold: that step starts again from its ready screen. At a step's ready screen: the step before it (in this section) goes again. Either way a `redo` event marks the range of the try being redone, so its labels are skipped; the sets stay, to delete in review if wanted. +- A pause stops the take's recording and starts it again on resume as the next part of the same take (`sets-2.bin`, and so on). takes.py reads all parts in order. A pause while a step waits only shows "Paused"; a Next pressed while paused doesn't count. ## Review and export (`takes.py`, the window) @@ -230,7 +286,7 @@ Before section 8: "Put on both controllers and tighten the straps". Before secti Errors get a plain explanation: not logged in, a token Hugging Face rejects (401), a token that can't open a pull request (403), terms not accepted, dataset not found, network errors. `--dry-run` does everything except the network calls and the record, and lists what it would upload. - **The page** shows the login (`whoami`, with "Check again"). If nobody is logged in, it explains how to run `distrobox enter dev -- hf auth login` in Konsole with a write token: the token goes only into that terminal. The page then has Upload and Cancel, the phase with a progress bar (a share while the export is checked, a sweep while it's sent, as `huggingface_hub` reports no progress), and the pull request's link when it's done. If this export was uploaded before, the page says so, and uploading it again asks first. A stale export can't be uploaded. - **While the texts are drafts**, Upload stays off unless `FT_HANDREC_ALLOW_UPLOAD=1`, so the maintainer can rehearse against a private test repo. `FT_HANDREC_DATASET` overrides `HF_DATASET`. `ft-handrec --hub-dry-run` makes Upload a dry run: no network, so it isn't held back by the drafts. - - **Rehearsal: `hands/rec/rehearse.sh [--repo ID]`** runs it all without the headset, in the dev container, in one `frame-job --local` scope when frame-job is installed. `ft-ringplay` plays 30 s of a recording into a ring in `/run/user/UID`. A tracking ft-hands that's already running is used, or one is started on that ring. `ft-handpanel --no-vr` stands in for the panel. `session.py --no-start` records a two-section test script, about 11 s and about 200 MB once exported. Then `takes.py` exports, `validate.py` checks, and `hub.py` uploads: a dry run by default, or for real to `--repo ID` with `FT_HANDREC_ALLOW_UPLOAD=1`. It prints a summary, deletes its temporary folders (camera images of a room) and stops everything it started, Ctrl+C included. The `--no-vr` panel logs no poses, so `poses.jsonl` is missing there (a warning). + - **Rehearsal: `hands/rec/rehearse.sh [--repo ID]`** runs it all without the headset, in the dev container, in one `frame-job --local` scope when frame-job is installed. `ft-ringplay` plays 30 s of a recording into a ring in `/run/user/UID`. A tracking ft-hands that's already running is used, or one is started on that ring. `ft-handpanel --no-vr` stands in for the panel. `session.py --no-start --next-after 0.3` records a two-section test script in step mode, three parts of 6 s (countdown and hold), about 360 MB once exported. Then `takes.py` exports, `validate.py` checks, and `hub.py` uploads: a dry run by default, or for real to `--repo ID` with `FT_HANDREC_ALLOW_UPLOAD=1`. It prints a summary, deletes its temporary folders (camera images of a room) and stops everything it started, Ctrl+C included. The `--no-vr` panel logs no poses, so `poses.jsonl` is missing there (a warning). ## Licensing and consent (texts in `CONSENT.md`) diff --git a/hands/rec/build.sh b/hands/rec/build.sh index 8920367..21a730f 100755 --- a/hands/rec/build.sh +++ b/hands/rec/build.sh @@ -1,8 +1,8 @@ #!/usr/bin/env bash # Build the hand recorder's headset panel ft-handpanel in the dev container on the Frame # (hands/rec/build/; it also runs there). Like gaze/build.sh: the pinned public OpenVR header -# (the DMA-BUF import is newer than the header shipped with SteamVR's samples) and stb_truetype -# for the text. +# (the DMA-BUF import is newer than the header shipped with SteamVR's samples), stb_truetype +# for the text and stb_image (PNG only) for the pose pictures, at the same stb commit. set -euo pipefail root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd) "$root/scripts/sync.sh" >/dev/null @@ -11,6 +11,7 @@ openvr=v2.15.6 [ -f build/include/openvr-$openvr ] || { curl -fsSL "https://raw.githubusercontent.com/ValveSoftware/openvr/$openvr/headers/openvr.h" -o build/include/openvr.h && touch build/include/openvr-$openvr; } stb=2c980bb59875b0d32144a71867fbdebb2f77cd20 [ -f build/include/stb-$stb ] || { curl -fsSL "https://raw.githubusercontent.com/nothings/stb/$stb/stb_truetype.h" -o build/include/stb_truetype.h && touch build/include/stb-$stb; } +[ -f build/include/stb_image-$stb ] || { curl -fsSL "https://raw.githubusercontent.com/nothings/stb/$stb/stb_image.h" -o build/include/stb_image.h && touch build/include/stb_image-$stb; } g++ -std=c++17 -O2 -Wall -Wno-unused-parameter -Wno-missing-field-initializers -Ibuild/include $(pkg-config --cflags gbm libdrm) \ -o build/ft-handpanel panel/ft-handpanel.cpp -L/opt/steamvr/bin/linuxarm64 -lopenvr_api -Wl,-rpath,/opt/steamvr/bin/linuxarm64 \ $(pkg-config --libs gbm libdrm) -lpthread diff --git a/hands/rec/ft_handrec.py b/hands/rec/ft_handrec.py index 595c6aa..c707c8b 100755 --- a/hands/rec/ft_handrec.py +++ b/hands/rec/ft_handrec.py @@ -6,8 +6,10 @@ A Kirigami (QML) app with a Python backend. It runs in the dev container: changes. Agreeing writes profile.json with a random contributor id. - Before you start: the checklist (objects, controller straps, lighting, sleeves, privacy, free space) and what will happen. Start hands it to the session runner (session.py). - - Session: the runner's live status, Start, Pause/Resume, Skip section and Stop (Space - pauses and Esc stops while the window has focus). The prompts appear in the headset. + - Session: the runner's live status with the step's pose picture and where-to diagram, Next, + Pause/Resume, Redo, Skip section and Stop (Space: Next, P, R, S and Esc while the window has + focus). The prompts appear in the headset. Each step waits for Next unless "Advance by + itself" was ticked. - Review: sessions, their takes, and a viewer for one frame set at a time, where ranges, takes and sessions can be deleted (takes.py). - Export: compress what's kept into exports// at nice 19 (takes.py), with a warning @@ -66,7 +68,7 @@ LIGHTING = [("dim", "Dim: one lamp only"), ("room", "Normal room light"), ("dayl SLEEVES = [("short", "Short sleeves or bare arms"), ("long", "Long sleeves"), ("", "Rather not say")] HANDEDNESS = [("", "Rather not say"), ("right", "Right-handed"), ("left", "Left-handed"), ("both", "Both (ambidextrous)")] -ACTIVE_STATES = ("starting", "intro", "running", "paused", "between") +ACTIVE_STATES = ("starting", "intro", "ready", "countdown", "running", "paused", "between") # Shown side by side in the viewer at this height; thumbnails are smaller. SET_HEIGHT = 480 THUMB_HEIGHT = 96 @@ -210,7 +212,7 @@ class Backend(QObject): def __init__(self, store, session_options=None, hub_dry_run=False): super().__init__() self.store = store - self._session_options = session_options or {} # test hooks for Session: dry_run, speed + self._session_options = session_options or {} # test hooks for Session: dry_run, speed, poses_dir self._hub_dry_run = hub_dry_run self._login = {"state": "dry" if hub_dry_run else "unknown"} self._login_busy = False @@ -384,8 +386,21 @@ class Backend(QObject): def sessionId(self): return self._session_id - @Slot("QVariantMap", str, result=bool) - def startSession(self, checklist, lighting): + @Slot("QVariantMap", bool, result=str) + def planText(self, checklist, auto): + """How long a session with these answers takes (session.plan_summary).""" + mod = self._runner() + if not mod: + return "" + try: + script = mod.load_script(SCRIPT_PATH) + plan, _ = mod.build_plan(script, dict(checklist)) + return mod.plan_summary(script, plan, auto=auto) + except Exception as e: + return f"Couldn't read the script: {e}" + + @Slot("QVariantMap", str, bool, result=bool) + def startSession(self, checklist, lighting, auto): if self.sessionActive: return False mod = self._runner() @@ -400,7 +415,7 @@ class Backend(QObject): checklist["own_objects"] = [o.strip() for o in checklist.get("own_objects", []) if str(o).strip()] try: self._session = mod.Session(self.store.base, self.store.profile(), checklist, lighting, SCRIPT_PATH, - on_status=lambda s: self._statusArrived.emit(dict(s)), + on_status=lambda s: self._statusArrived.emit(dict(s)), auto=auto, **self._session_options) except Exception as e: self.message.emit(f"Couldn't set up the session: {e}", True) @@ -434,6 +449,16 @@ class Backend(QObject): except Exception as e: self.message.emit(f"{name}: {e}", True) + @Slot() + def nextStep(self): + if self.sessionActive and self._status.get("waiting"): + self._control("next_step") + + @Slot() + def redo(self): + if self.sessionActive and self._status.get("can_redo"): + self._control("redo") + @Slot() def togglePause(self): if self._status.get("state") == "paused": @@ -897,6 +922,7 @@ def main(): ap.add_argument("--dry-run", action="store_true", help="test: sessions start no processes and print the panel's commands") ap.add_argument("--speed", type=float, default=1.0, help="test, with --dry-run: run sessions this much faster") + ap.add_argument("--poses", help="test: the pose pictures' folder (default hands/rec/poses)") ap.add_argument("--hub-dry-run", action="store_true", help="test: Upload checks the export and says what it would send, with no network calls") a, qt_args = ap.parse_known_args() @@ -910,7 +936,10 @@ def main(): store = takes.Store(a.base) engine = QQmlApplicationEngine() engine.addImageProvider("frames", FrameProvider(store)) - backend = Backend(store, {"dry_run": True, "speed": a.speed} if a.dry_run else {}, hub_dry_run=a.hub_dry_run) + options = {"dry_run": True, "speed": a.speed} if a.dry_run else {} + if a.poses: + options["poses_dir"] = a.poses + backend = Backend(store, options, hub_dry_run=a.hub_dry_run) app.aboutToQuit.connect(backend.shutdown) engine.rootContext().setContextProperty("backend", backend) engine.rootContext().setContextProperty("startPage", a.page) diff --git a/hands/rec/main.qml b/hands/rec/main.qml index b4b4872..d0be800 100644 --- a/hands/rec/main.qml +++ b/hands/rec/main.qml @@ -383,19 +383,36 @@ Kirigami.ApplicationWindow { Controls.Label { Layout.maximumWidth: Kirigami.Units.gridUnit * 26 wrapMode: Text.Wrap - text: "Press Start, then put the headset on. A panel in the headset says what to do, one " - + "section at a time: hand poses, gestures, typing and the mouse, your objects, touching a " - + "dot, and moves with the controllers on and off. Each section is recorded as one take.\n\n" - + "Space in this window pauses and Esc stops. You can also skip a section.\n\n" + text: "Press Start, then put the headset on. A panel in the headset shows each step: a picture " + + "of the hand pose, where to hold your hands, and what to do. The sections: hand poses, " + + "gestures, typing and the mouse, your objects, touching a dot, and moves with the " + + "controllers on and off. Each section is recorded as one take.\n\n" + + "Each step waits until you're ready: press Space or click Next in this window. A 3-2-1 " + + "countdown follows, then hold the pose until the bar runs out. Nothing is recorded while " + + "a step waits. P pauses, R records the last step again, S skips a section and Esc stops.\n\n" + "Nothing leaves the headset. Afterwards you watch the takes in Review, delete anything " + "you don't want to share, and only then export." } + Controls.CheckBox { + id: autoAdvance + Kirigami.FormData.label: "Pace:" + text: "Advance by itself (no Next between steps)" + } + Controls.Label { + Layout.maximumWidth: Kirigami.Units.gridUnit * 26 + wrapMode: Text.Wrap + opacity: 0.7 + text: (autoAdvance.checked + ? "Each step shows for a few seconds and the next follows by itself. Quicker if you " + + "know the steps already. " + : "") + "Length: " + backend.planText(checklist.answers(), autoAdvance.checked) + "." + } Controls.Button { text: "Start" icon.name: "media-record" enabled: checklist.ready onClicked: { - if (backend.startSession(checklist.answers(), lighting.currentValue)) + if (backend.startSession(checklist.answers(), lighting.currentValue, autoAdvance.checked)) root.show(sessionPage) } } @@ -417,17 +434,35 @@ Kirigami.ApplicationWindow { title: "Session" readonly property var st: backend.status readonly property string state: st.state || "" + readonly property bool waiting: !!st.waiting + readonly property bool stepMode: st.mode !== "auto" readonly property var stateText: ({ - starting: "Starting…", intro: "Get ready", running: "Recording", paused: "Paused", - between: "Between sections", done: "Done", stopped: "Stopped", error: "Error" + starting: "Starting…", intro: "Get ready", ready: "Get ready", countdown: "Starting", + running: "Recording", paused: "Paused", between: "Between sections", done: "Done", + stopped: "Stopped", error: "Error" }) - // Space and Esc while the window has focus (the panel in the headset says so too). + // The keys while the window has focus (the panel in the headset lists them too). Shortcut { sequence: "Space" enabled: backend.sessionActive + onActivated: backend.nextStep() + } + Shortcut { + sequence: "P" + enabled: backend.sessionActive onActivated: backend.togglePause() } + Shortcut { + sequence: "R" + enabled: backend.sessionActive + onActivated: backend.redo() + } + Shortcut { + sequence: "S" + enabled: backend.sessionActive + onActivated: backend.skipSection() + } Shortcut { sequence: "Esc" enabled: backend.sessionActive @@ -449,6 +484,114 @@ Kirigami.ApplicationWindow { } } + // The pose picture as the panel shows it: a right hand as drawn, a left hand flipped, + // both hands as a flipped copy beside it (DESIGN.md "The pose pictures"). + component PosePicture: Row { + property string path + property string mode + property real side: Kirigami.Units.gridUnit * 9 + spacing: Kirigami.Units.smallSpacing + visible: path !== "" + Image { + visible: parent.mode === "both" + width: visible ? parent.side * 0.75 : 0 + height: width + source: parent.path ? "file://" + parent.path : "" + fillMode: Image.PreserveAspectFit + mirror: true + smooth: true + mipmap: true + } + Image { + width: parent.mode === "both" ? parent.side * 0.75 : parent.side + height: width + source: parent.path ? "file://" + parent.path : "" + fillMode: Image.PreserveAspectFit + mirror: parent.mode === "mirror" + smooth: true + mipmap: true + } + } + + // Where to hold the hands: a front view (left, centre, right, up, down; the push + // sections' chest, desk and eye) and how far out (near, mid, far), as on the panel. + component WhereDiagram: RowLayout { + id: where + property string position + property string distance + readonly property var cell: ({ centre: [1, 1], center: [1, 1], chest: [1, 1], left: [0, 1], + right: [2, 1], up: [1, 0], eye: [1, 0], down: [1, 2], + desk: [1, 2] })[position] || null + readonly property int step: ["near", "mid", "far"].indexOf(distance) + readonly property real unit: Kirigami.Units.gridUnit * 1.1 + readonly property color accent: "#4cd9ff" + spacing: Kirigami.Units.gridUnit + visible: cell !== null || step >= 0 + ColumnLayout { + visible: where.cell !== null + Grid { + Layout.alignment: Qt.AlignHCenter + columns: 3 + spacing: 2 + Repeater { + model: 9 + Rectangle { + required property int index + width: where.unit * 1.33 + height: where.unit + radius: 3 + readonly property bool on: where.cell !== null && index === where.cell[1] * 3 + where.cell[0] + color: on ? where.accent + : Qt.rgba(Kirigami.Theme.textColor.r, Kirigami.Theme.textColor.g, + Kirigami.Theme.textColor.b, 0.12) + } + } + } + Controls.Label { + Layout.alignment: Qt.AlignHCenter + font: Kirigami.Theme.smallFont + text: ({ left: "To your left", right: "To your right", up: "Up high", down: "Down low", + chest: "Chest height", desk: "Desk height", eye: "Eye level" })[where.position] + || "In front" + } + } + ColumnLayout { + visible: where.step >= 0 + Row { + Layout.alignment: Qt.AlignHCenter + spacing: where.unit * 0.6 + height: where.unit * 1.2 + // the head, seen from the side, then the arm's reach + Rectangle { + width: where.unit + height: width + radius: width / 2 + anchors.verticalCenter: parent.verticalCenter + color: Qt.rgba(Kirigami.Theme.textColor.r, Kirigami.Theme.textColor.g, + Kirigami.Theme.textColor.b, 0.6) + } + Repeater { + model: 3 + Rectangle { + required property int index + readonly property bool on: index === where.step + width: on ? where.unit * 0.8 : where.unit * 0.35 + height: width + radius: width / 2 + anchors.verticalCenter: parent.verticalCenter + color: on ? where.accent : Qt.rgba(Kirigami.Theme.textColor.r, Kirigami.Theme.textColor.g, + Kirigami.Theme.textColor.b, 0.35) + } + } + } + Controls.Label { + Layout.alignment: Qt.AlignHCenter + font: Kirigami.Theme.smallFont + text: ["Close: a hand's length", "Halfway out", "Arm stretched out"][Math.max(0, where.step)] + } + } + } + header: Kirigami.InlineMessage { visible: backend.runnerError !== "" || sessionView.state === "error" position: Kirigami.InlineMessage.Position.Header @@ -475,29 +618,105 @@ Kirigami.ApplicationWindow { visible: sessionView.state !== "" spacing: Kirigami.Units.largeSpacing - Kirigami.Heading { - level: 1 - text: sessionView.stateText[sessionView.state] || sessionView.state + RowLayout { + Layout.fillWidth: true + Kirigami.Heading { + level: 1 + text: sessionView.stateText[sessionView.state] || sessionView.state + } + Item { Layout.fillWidth: true } + Controls.Label { + visible: !!sessionView.st.take && sessionView.state !== "paused" + && (sessionView.state === "countdown" || sessionView.state === "running" + || !sessionView.stepMode) + text: "● Recording" + color: Kirigami.Theme.negativeTextColor + } } Controls.Label { visible: (sessionView.st.section_index || 0) > 0 text: "Section " + sessionView.st.section_index + " of " + sessionView.st.section_count + (sessionView.st.title ? ": " + sessionView.st.title : "") + + ((sessionView.st.step_index || 0) > 0 && sessionView.stepMode + ? " · step " + sessionView.st.step_index + " of " + sessionView.st.step_count : "") font.bold: true } - Controls.Label { + + RowLayout { Layout.fillWidth: true - visible: !!sessionView.st.prompt - wrapMode: Text.Wrap - font.pointSize: Kirigami.Theme.defaultFont.pointSize * 1.4 - text: sessionView.st.prompt || "" - } - Controls.Label { - visible: sessionView.st.seconds_left !== undefined && sessionView.st.seconds_left !== null - && sessionView.st.seconds_left >= 0 - text: Math.ceil(sessionView.st.seconds_left || 0) + " s left" - opacity: 0.7 + spacing: Kirigami.Units.gridUnit * 1.5 + + ColumnLayout { + visible: !!sessionView.st.image || !!sessionView.st.position || !!sessionView.st.distance + spacing: Kirigami.Units.largeSpacing + PosePicture { + Layout.alignment: Qt.AlignHCenter + path: sessionView.st.image || "" + mode: sessionView.st.image_mode || "" + } + Controls.Label { + Layout.alignment: Qt.AlignHCenter + Layout.maximumWidth: Kirigami.Units.gridUnit * 14 + visible: !!sessionView.st.image && !!sessionView.st.caption + wrapMode: Text.Wrap + horizontalAlignment: Text.AlignHCenter + opacity: 0.7 + font: Kirigami.Theme.smallFont + text: sessionView.st.caption || "" + } + WhereDiagram { + id: where + Layout.alignment: Qt.AlignHCenter + position: sessionView.st.position || "" + distance: sessionView.st.distance || "" + } + } + + ColumnLayout { + Layout.fillWidth: true + spacing: Kirigami.Units.largeSpacing + Controls.Label { + Layout.fillWidth: true + visible: !!sessionView.st.prompt + wrapMode: Text.Wrap + font.pointSize: Kirigami.Theme.defaultFont.pointSize * 1.4 + text: sessionView.st.prompt || "" + } + // The countdown's 3, 2, 1, then the hold's word, big + Controls.Label { + visible: !!sessionView.st.big && sessionView.state !== "paused" + text: sessionView.st.big || "" + font.pointSize: Kirigami.Theme.defaultFont.pointSize * 4 + font.bold: true + color: Kirigami.Theme.highlightColor + } + Controls.Label { + visible: !sessionView.waiting && sessionView.state !== "countdown" + && sessionView.state !== "ready" && sessionView.st.seconds_left !== undefined + && sessionView.st.seconds_left !== null && sessionView.st.seconds_left > 0 + text: Math.ceil(sessionView.st.seconds_left || 0) + " s left" + opacity: 0.7 + } + Controls.Button { + visible: sessionView.waiting && sessionView.state !== "paused" + focusPolicy: Qt.NoFocus + text: "Next" + icon.name: "go-next" + font.pointSize: Kirigami.Theme.defaultFont.pointSize * 1.5 + Layout.preferredWidth: Kirigami.Units.gridUnit * 10 + Layout.preferredHeight: Kirigami.Units.gridUnit * 3 + onClicked: backend.nextStep() + } + Controls.Label { + visible: sessionView.waiting && sessionView.state !== "paused" + opacity: 0.7 + text: "Ready? Press Space or click Next. A 3-2-1 countdown starts the recording." + wrapMode: Text.Wrap + Layout.fillWidth: true + } + } } + Kirigami.InlineMessage { Layout.fillWidth: true visible: !!sessionView.st.note @@ -513,7 +732,7 @@ Kirigami.ApplicationWindow { } Controls.Label { visible: !!sessionView.st.take - text: "Recording to take " + (sessionView.st.take || "") + text: "Take " + (sessionView.st.take || "") opacity: 0.7 font: Kirigami.Theme.smallFont } @@ -524,8 +743,10 @@ Kirigami.ApplicationWindow { Layout.fillWidth: true wrapMode: Text.Wrap opacity: 0.7 - text: "The instructions appear in the headset. While this window has focus, Space pauses " - + "and resumes, and Esc stops. Stopping keeps what's recorded so far." + text: "The instructions appear in the headset. While this window has focus: " + + (sessionView.stepMode ? "Space: next · " : "") + + "P: pause or resume · R: record the last step again · S: skip section · Esc: stop. " + + "Stopping keeps what's recorded so far." } RowLayout { // No keyboard focus on the buttons, so Space always reaches the shortcut. @@ -543,6 +764,14 @@ Kirigami.ApplicationWindow { icon.name: sessionView.state === "paused" ? "media-playback-start" : "media-playback-pause" onClicked: backend.togglePause() } + Controls.Button { + visible: backend.sessionActive + enabled: !!sessionView.st.can_redo + focusPolicy: Qt.NoFocus + text: "Redo step" + icon.name: "edit-undo" + onClicked: backend.redo() + } Controls.Button { visible: backend.sessionActive focusPolicy: Qt.NoFocus diff --git a/hands/rec/panel/ft-handpanel.cpp b/hands/rec/panel/ft-handpanel.cpp index 99f8582..539b12a 100644 --- a/hands/rec/panel/ft-handpanel.cpp +++ b/hands/rec/panel/ft-handpanel.cpp @@ -24,6 +24,18 @@ // at the hand's position (-1: not seen). The labels are split at // "|", or else at the first space (default "Near" and "Far") // paused on|off "Paused" over the picture +// image [mirror|both] / image off +// the pose picture (a PNG, square, RGBA, a right hand as the wearer +// sees it) in a column on the left, the text moving right: mirror +// flips it (a left hand), both shows a flipped copy on its left +// where / where off +// a small diagram under the picture: where to hold the hands, a +// front view (centre, left, right, up, down; or the bar's chest, +// desk, eye), and how far out (near, mid, far) +// action a cyan line under the instruction: "Ready? Press Space ..." +// big large, under the instruction: the countdown's 3, 2, 1, then "Hold" +// keys a small faint line along the bottom: the window's keys +// rec on|off a red "Rec" dot by the step line while recording // target [show|hold <0..1>|done] / target off // the touch target, about 2 cm across. The point is in the head // frame (metres, +x right, +y up, -z forward). The first command @@ -61,6 +73,14 @@ #define STB_TRUETYPE_IMPLEMENTATION #include "stb_truetype.h" +#define STB_IMAGE_IMPLEMENTATION +#define STBI_ONLY_PNG +#define STBI_NO_HDR +#define STBI_NO_LINEAR +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-function" // helpers only other formats use +#include "stb_image.h" +#pragma GCC diagnostic pop #include #include @@ -274,15 +294,146 @@ void Text(Picture &p, const std::string &s, int size, int x, int cy, double lum, Text(p, s, size, x, cy, lum, lum, lum, align); } +// A pose picture as loaded (straight alpha), and scaled copies of it as drawn. +struct Image { + std::string path; + Picture src{0, 0, {}}; + std::map, Picture> scaled; // (side, mirrored) -> picture +}; + +// A PNG into img; false (and img empty) if it can't be read. +bool LoadImage(Image &img, const std::string &path) { + img = Image{}; + int w = 0, h = 0, n = 0; + unsigned char *px = stbi_load(path.c_str(), &w, &h, &n, 4); + if (!px) return false; + img.path = path; + img.src = Picture{w, h, std::vector(px, px + size_t(w) * h * 4)}; + stbi_image_free(px); + return true; +} + +// The picture fitted into a side x side square (centred, aspect kept), averaging the source +// pixels each output pixel covers, weighted by alpha so transparent edges don't darken. +const Picture &Scaled(Image &img, int side, bool mirror) { + auto [it, fresh] = img.scaled.try_emplace({side, mirror}, Picture{side, side, {}}); + Picture &out = it->second; + if (!fresh) return out; + out.px.assign(size_t(side) * side * 4, 0); + const Picture &src = img.src; + const double scale = std::max(src.w, src.h) / double(side); // source pixels per output pixel + const double ox = (side - src.w / scale) / 2, oy = (side - src.h / scale) / 2; + for (int y = 0; y < side; ++y) + for (int x = 0; x < side; ++x) { + const double sx0 = (x - ox) * scale, sy0 = (y - oy) * scale; + const int x0 = std::max(0, int(std::floor(sx0))), y0 = std::max(0, int(std::floor(sy0))); + const int x1 = std::min(src.w, int(std::ceil(sx0 + scale))), y1 = std::min(src.h, int(std::ceil(sy0 + scale))); + double r = 0, g = 0, b = 0, a = 0; + int count = 0; + for (int sy = y0; sy < y1; ++sy) + for (int sx = x0; sx < x1; ++sx) { + const uint8_t *q = &src.px[(size_t(sy) * src.w + (mirror ? src.w - 1 - sx : sx)) * 4]; + const double qa = q[3] / 255.0; + r += q[0] * qa, g += q[1] * qa, b += q[2] * qa, a += qa, ++count; + } + if (!count || a <= 0) continue; + uint8_t *o = &out.px[(size_t(y) * side + x) * 4]; + o[0] = uint8_t(std::lround(r / a)), o[1] = uint8_t(std::lround(g / a)), o[2] = uint8_t(std::lround(b / a)); + o[3] = uint8_t(std::lround(255 * a / count)); + } + return out; +} + +void DrawPicture(Picture &dst, const Picture &src, int x0, int y0) { + for (int y = 0; y < src.h; ++y) + for (int x = 0; x < src.w; ++x) { + const uint8_t *q = &src.px[(size_t(y) * src.w + x) * 4]; + if (q[3]) Blend(dst, x0 + x, y0 + y, q[0] / 255.0, q[1] / 255.0, q[2] / 255.0, q[3] / 255.0); + } +} + +// The where-to diagram's cells: a front view, 3 x 3, (column, row) from the top left. +bool WhereCell(const std::string &pos, int &col, int &row) { + col = 1, row = 1; + if (pos == "left") col = 0; + else if (pos == "right") col = 2; + else if (pos == "up" || pos == "eye") row = 0; + else if (pos == "down" || pos == "desk") row = 2; + else if (pos != "centre" && pos != "center" && pos != "chest") return false; + return true; +} + +const char *WhereLabel(const std::string &pos) { + if (pos == "left") return "To your left"; + if (pos == "right") return "To your right"; + if (pos == "up") return "Up high"; + if (pos == "down") return "Down low"; + if (pos == "chest") return "Chest height"; + if (pos == "desk") return "Desk height"; + if (pos == "eye") return "Eye level"; + return "In front"; +} + +int DistanceStep(const std::string &dist) { return dist == "near" ? 0 : dist == "mid" ? 1 : dist == "far" ? 2 : -1; } + +const char *DistanceLabel(int k) { return k == 0 ? "Close" : k == 1 ? "Halfway out" : "Arm out"; } + +constexpr double kCyan[3] = {0.3, 0.85, 1.0}; + +// The diagram in x0..x0+w, y0..y0+h: the front view on the left, the distance on the right +// (or either alone, centred), each with its caption under it. +void DrawWhere(Picture &pic, int x0, int y0, int w, int h, const std::string &pos, const std::string &dist) { + const double ppd = kPxPerDeg; + int col, row; + const bool grid = WhereCell(pos, col, row); + const int step = DistanceStep(dist); + const int parts = int(grid) + int(step >= 0); + if (!parts) return; + const int capSize = int(ppd * 0.75), capH = int(capSize * 1.6), partW = w / parts; + const int boxH = h - capH; + int px = x0; + if (grid) { + const int gw = std::min(partW - int(ppd * 0.6), boxH * 4 / 3), gh = gw * 3 / 4; + const int gx = px + (partW - gw) / 2, gy = y0 + (boxH - gh) / 2; + RoundRect(pic, gx, gy, gx + gw, gy + gh, ppd * 0.3, 1, 1, 1, 0.08); + for (int k = 1; k < 3; ++k) { + Rect(pic, gx + gw * k / 3 - 1, gy + 3, gx + gw * k / 3 + 1, gy + gh - 3, 1, 1, 1, 0.18); + Rect(pic, gx + 3, gy + gh * k / 3 - 1, gx + gw - 3, gy + gh * k / 3 + 1, 1, 1, 1, 0.18); + } + const int pad = std::max(3, int(ppd * 0.12)); + RoundRect(pic, gx + gw * col / 3 + pad, gy + gh * row / 3 + pad, gx + gw * (col + 1) / 3 - pad, + gy + gh * (row + 1) / 3 - pad, ppd * 0.18, kCyan[0], kCyan[1], kCyan[2], 0.9); + Text(pic, WhereLabel(pos), capSize, px + partW / 2, y0 + boxH + capH / 2, 0.85); + px += partW; + } + if (step >= 0) { + // Seen from the side: the head on the left, the arm's reach to the right, three marks. + const int cy = y0 + boxH / 2, head = int(ppd * 0.55); + const int hx = px + int(ppd * 0.4) + head, ax0 = hx + head + int(ppd * 0.3), ax1 = px + partW - int(ppd * 0.5); + Disc(pic, hx, cy, head, 0, 1, 1, 1, 0.75); + RoundRect(pic, hx + head / 3, cy - head / 2, hx + head + int(ppd * 0.2), cy + head / 3, 3, 0.25, 0.25, 0.28, 1); + Rect(pic, ax0, cy - 1, ax1, cy + 1, 1, 1, 1, 0.3); + for (int k = 0; k < 3; ++k) { + const double x = ax0 + (ax1 - ax0) * (k + 1) / 3.0; + if (k == step) Disc(pic, x, cy, ppd * 0.38, 0, kCyan[0], kCyan[1], kCyan[2], 0.95); + else Disc(pic, x, cy, ppd * 0.16, 0, 1, 1, 1, 0.45); + } + Text(pic, DistanceLabel(step), capSize, px + partW / 2, y0 + boxH + capH / 2, 0.85); + } +} + struct Panel { Picture pic{kW, kH, {}}; - std::string title, step, text, note; + std::string title, step, text, note, action, big, keys; double countdown = -1; // the share of the prompt's time left, or -1: off std::string hands[2] = {"off", "off"}; bool barOn = false; double barTarget = 0, barCurrent = -1; std::string nearLabel = "Near", farLabel = "Far"; - bool paused = false; + bool paused = false, rec = false; + Image image; // the pose picture, if image.path isn't empty + std::string imageMode; // "", "mirror" or "both" + std::string wherePos, whereDist; // the diagram's, "" for none // What the last Draw laid out, for --no-vr's printout. std::vector lines, noteLines; double textDeg = 0; @@ -301,23 +452,34 @@ void Draw(Panel &p) { const double ppd = kPxPerDeg, faint = 0.75; const int margin = int(ppd * 1.6), width = kW - 2 * margin; - // The title and the step share the top line; a faint rule under them. + // The title and the step share the top line, a red "Rec" by the step while recording; a + // faint rule under them. int top = margin; - if (!p.title.empty() || !p.step.empty()) { + if (!p.title.empty() || !p.step.empty() || p.rec) { const int titleSize = int(ppd * 1.6), stepSize = int(ppd * 0.9), cy = margin + titleSize / 2; Text(pic, p.step, stepSize, kW - margin, cy, faint, kRight); + if (p.rec) { + const int rx = kW - margin - TextWidth(p.step, stepSize) - (p.step.empty() ? 0 : int(ppd * 0.8)); + Text(pic, "Rec", stepSize, rx, cy, 1.0, 0.45, 0.4, kRight); + Disc(pic, rx - TextWidth("Rec", stepSize) - ppd * 0.45, cy, ppd * 0.25, 0, 0.95, 0.2, 0.15, 1); + } Text(pic, p.title, titleSize, margin, cy, 1.0, kLeft); top = cy + int(titleSize * 0.85); Rect(pic, margin, top, kW - margin, top + 2, 1, 1, 1, 0.15); top += int(ppd * 0.8); } - // From the bottom up: the countdown, the chips, the near/far bar. The instruction gets what's left. - int bottom = kH - margin; + // From the bottom up: the keys, the countdown, the chips, the near/far bar. The rest gets what's left. + int bottom = kH - margin, foot = kH - int(ppd * 0.9); + if (!p.keys.empty()) { + Text(pic, p.keys, int(ppd * 0.7), kW / 2, foot, 0.55); + foot -= int(ppd * 1.0); + bottom = std::min(bottom, foot - int(ppd * 0.4)); + } if (p.countdown >= 0) { - const int y = kH - int(ppd * 0.9), th = std::max(4, int(ppd * 0.22)); + const int y = foot, th = std::max(4, int(ppd * 0.22)); Rect(pic, margin, y, kW - margin, y + th, 1, 1, 1, 0.15); - Rect(pic, margin, y, margin + int(std::lround(width * p.countdown)), y + th, 0.3, 0.85, 1.0, 0.95); + Rect(pic, margin, y, margin + int(std::lround(width * p.countdown)), y + th, kCyan[0], kCyan[1], kCyan[2], 0.95); bottom = std::min(bottom, y - int(ppd * 0.6)); } if (p.hands[0] != "off" || p.hands[1] != "off") { @@ -342,25 +504,72 @@ void Draw(Panel &p) { Text(pic, p.farLabel, labelSize, x1, ly, faint, kRight); const double ring = ppd * 0.65; Disc(pic, x0 + (x1 - x0) * p.barTarget, ty, ring, ring - ppd * 0.14, 1, 1, 1, 0.95); - if (p.barCurrent >= 0) Disc(pic, x0 + (x1 - x0) * p.barCurrent, ty, ppd * 0.4, 0, 0.3, 0.85, 1.0, 0.95); + if (p.barCurrent >= 0) Disc(pic, x0 + (x1 - x0) * p.barCurrent, ty, ppd * 0.4, 0, kCyan[0], kCyan[1], kCyan[2], 0.95); bottom = ty - int(ring) - int(ppd * 0.8); } - // The instruction, then the note, centred together in the space between. A long instruction - // gets smaller, down to 1 degree a line, before it runs over. - const int noteSize = int(ppd * 1.1), noteLh = int(noteSize * 1.3), noteGap = int(ppd * 0.6); - p.noteLines = Wrap(p.note, noteSize, width); + // The pose picture and the diagram in a column on the left, centred together; the text + // takes the rest. "both": the picture flipped (the left hand) beside it as it is. + int textX0 = margin, textX1 = kW - margin; + int col, row; + const bool imageOn = !p.image.path.empty(); + const bool whereOn = WhereCell(p.wherePos, col, row) || DistanceStep(p.whereDist) >= 0; + if (imageOn || whereOn) { + const int colW = int(ppd * 14), gap = int(ppd * 0.6); + const int whereH = whereOn ? int(ppd * 4.4) : 0, between = imageOn && whereOn ? gap : 0; + const bool both = p.imageMode == "both"; + const int side = imageOn ? std::max(0, std::min(both ? (colW - gap) / 2 : colW, bottom - top - whereH - between)) : 0; + int y = top + std::max(0, (bottom - top - side - between - whereH) / 2); + if (side > 0) { + const int pw = both ? 2 * side + gap : side, x = margin + (colW - pw) / 2; + RoundRect(pic, x - gap / 2, y - gap / 2, x + pw + gap / 2, y + side + gap / 2, ppd * 0.6, 1, 1, 1, 0.05); + if (both) { + DrawPicture(pic, Scaled(p.image, side, true), x, y); + DrawPicture(pic, Scaled(p.image, side, false), x + side + gap, y); + } else { + DrawPicture(pic, Scaled(p.image, side, p.imageMode == "mirror"), x, y); + } + y += side + between; + } + if (whereOn) DrawWhere(pic, margin, y, colW, whereH, p.wherePos, p.whereDist); + textX0 = margin + colW + int(ppd * 1.2); + } + const int textW = textX1 - textX0, textCx = (textX0 + textX1) / 2; + + // The instruction, the note, the big countdown and the action line, centred together in + // the space between. A long instruction gets smaller, down to 1 degree a line, before it runs over. + const int noteSize = int(ppd * 1.1), noteLh = int(noteSize * 1.3), gapY = int(ppd * 0.6); + const int bigSize = int(ppd * 3.0), bigLh = int(bigSize * 1.1), actionSize = int(ppd * 1.05), actionLh = int(actionSize * 1.4); + p.noteLines = Wrap(p.note, noteSize, textW); + const std::vector actionLines = Wrap(p.action, actionSize, textW); int size = 0, lh = 0, block = 0; for (p.textDeg = 1.4;; p.textDeg -= 0.1) { size = int(ppd * p.textDeg), lh = int(size * 1.3); - p.lines = Wrap(p.text, size, width); - block = int(p.lines.size()) * lh + (p.noteLines.empty() ? 0 : noteGap + int(p.noteLines.size()) * noteLh); + p.lines = Wrap(p.text, size, textW); + block = int(p.lines.size()) * lh + (p.noteLines.empty() ? 0 : gapY + int(p.noteLines.size()) * noteLh) + + (p.big.empty() ? 0 : gapY + bigLh) + (actionLines.empty() ? 0 : gapY + int(actionLines.size()) * actionLh); if (block <= bottom - top || p.textDeg < 1.05) break; } int y = top + std::max(0, (bottom - top - block) / 2); - for (const std::string &line : p.lines) Text(pic, line, size, kW / 2, y + lh / 2, 1.0), y += lh; - if (!p.noteLines.empty()) y += p.lines.empty() ? 0 : noteGap; - for (const std::string &line : p.noteLines) Text(pic, line, noteSize, kW / 2, y + noteLh / 2, 1.0, 0.62, 0.3), y += noteLh; + bool first = true; + auto gapBefore = [&] { if (!first) y += gapY; first = false; }; + if (!p.lines.empty()) { + first = false; + for (const std::string &line : p.lines) Text(pic, line, size, textCx, y + lh / 2, 1.0), y += lh; + } + if (!p.noteLines.empty()) { + gapBefore(); + for (const std::string &line : p.noteLines) Text(pic, line, noteSize, textCx, y + noteLh / 2, 1.0, 0.62, 0.3), y += noteLh; + } + if (!p.big.empty()) { + gapBefore(); + Text(pic, p.big, bigSize, textCx, y + bigLh / 2, kCyan[0], kCyan[1], kCyan[2]), y += bigLh; + } + if (!actionLines.empty()) { + gapBefore(); + for (const std::string &line : actionLines) + Text(pic, line, actionSize, textCx, y + actionLh / 2, kCyan[0], kCyan[1], kCyan[2]), y += actionLh; + } if (p.paused) { // The picture stays faintly behind, so it's clear what resumes; the word sits on a @@ -432,6 +641,15 @@ void Print(const Panel &p, const Target &t, bool visible, int number) { p.nearLabel.c_str(), p.farLabel.c_str()); else std::printf("bar: off\n"); std::printf("paused: %s\n", p.paused ? "on" : "off"); + std::printf("big: %s\naction: %s\nkeys: %s\nrec: %s\n", p.big.c_str(), p.action.c_str(), p.keys.c_str(), + p.rec ? "on" : "off"); + if (!p.image.path.empty()) + std::printf("image: %s %dx%d%s%s\n", p.image.path.c_str(), p.image.src.w, p.image.src.h, + p.imageMode.empty() ? "" : " ", p.imageMode.c_str()); + else std::printf("image: off\n"); + if (!p.wherePos.empty() || !p.whereDist.empty()) + std::printf("where: %s %s\n", p.wherePos.empty() ? "-" : p.wherePos.c_str(), p.whereDist.empty() ? "-" : p.whereDist.c_str()); + else std::printf("where: off\n"); if (t.on) std::printf("target: %s %.2f, head %.3f %.3f %.3f, room %.3f %.3f %.3f\n", t.state.c_str(), t.progress, t.head[0], t.head[1], t.head[2], t.room[0], t.room[1], t.room[2]); @@ -805,6 +1023,43 @@ int main(int argc, char **argv) { } else if (Is(buf, "paused", &rest)) { if (!std::strcmp(rest, "on") || !std::strcmp(rest, "off")) p.paused = rest[1] == 'n', dirty = true; else reply = "error usage: paused on|off"; + } else if (Is(buf, "rec", &rest)) { + if (!std::strcmp(rest, "on") || !std::strcmp(rest, "off")) p.rec = rest[1] == 'n', dirty = true; + else reply = "error usage: rec on|off"; + } else if (Is(buf, "action", &rest)) { + p.action = rest, dirty = true; + } else if (Is(buf, "big", &rest)) { + p.big = rest, dirty = true; + } else if (Is(buf, "keys", &rest)) { + p.keys = rest, dirty = true; + } else if (Is(buf, "image", &rest)) { + // image [mirror|both]: the path may hold spaces, the mode is the last word + std::string path = rest, mode; + for (const char *m : {" mirror", " both"}) { + const size_t n = std::strlen(m); + if (path.size() > n && !path.compare(path.size() - n, n, m)) mode = m + 1, path.resize(path.size() - n); + } + if (path.empty()) { + reply = "error usage: image [mirror|both] | image off"; + } else if (path == "off") { + p.image = Image{}, p.imageMode.clear(), dirty = true; + } else if (path != p.image.path && !LoadImage(p.image, path)) { + reply = "error can't read " + path + ": " + stbi_failure_reason(); + p.imageMode.clear(), dirty = true; // no picture rather than the last one + } else { + p.imageMode = mode, dirty = true; + } + } else if (Is(buf, "where", &rest)) { + char pos[16] = "", dist[16] = ""; + if (!std::strcmp(rest, "off")) { + p.wherePos.clear(), p.whereDist.clear(), dirty = true; + } else if (std::sscanf(rest, "%15s %15s", pos, dist) >= 1) { + p.wherePos = std::strcmp(pos, "-") ? pos : ""; + p.whereDist = *dist && std::strcmp(dist, "-") ? dist : ""; + dirty = true; + } else { + reply = "error usage: where | where off"; + } } else if (Is(buf, "target", &rest)) { char state[16] = "show"; double prog = 0; diff --git a/hands/rec/poses/README.md b/hands/rec/poses/README.md new file mode 100644 index 0000000..20f187f --- /dev/null +++ b/hands/rec/poses/README.md @@ -0,0 +1,50 @@ +# Pose images for the hand recorder + +Small pictures of each hand pose in `script.json`, shown on the headset panel next to the prompt text. + +- `.png`: one image per pose, 512x512 RGBA with a transparent background. Made for a dark panel and still readable at about 250 px. +- `poses.json`: maps each pose id to `{"file", "two_hands", "caption"}`. +- `contact-sheet.png`: every image in a grid with its id, for review. The panel doesn't use it. +- `make_poses.py`: the generator that makes all of the above. + +## How the images are drawn + +Every image shows a right hand as the wearer sees it from their own eyes. "Palm toward you" shows the palm, with its creases. "Back toward you" shows the back of the hand, with the nails and knuckles. Fingers point up unless the pose says otherwise. + +- **Left-hand prompts:** the panel mirrors the image horizontally. +- **"Both" prompts:** the panel shows two copies, one of them mirrored. +- **`two_hands: true`:** the image already shows the whole scene: both hands, or one hand with an object, as in `hold`. Show it once, without mirroring or copying. This applies to `cross`, `overlap`, `near-face`, `typing`, `lift`, `switch`, `hold`, `touch-stick` and `touch-stick-desk`. +- **`touch-stick`:** shows the left hand holding the controller while the right index touches its thumbstick. For prompts where the right hand holds the controller (`controllers: ["right"]`), mirror it. + +Motion poses show the start pose, a faint blue "ghost" of the end pose, and orange arrows. Objects (keyboard, mouse, bottle, bar, controller, screen, head and headset) are plain grey shapes. + +Besides the pose ids in `script.json`, these extra ids exist for prompts that need a different picture: + +| id | for | +| --- | --- | +| `push` | the bar sections: palms out, push out and back | +| `push-controller` | the same with controllers on | +| `touch-stick-desk` | touching the thumbstick of a controller lying on the desk | +| `no-hands` | the no-hands section: hands down, out of view | +| `open-close` | already used by the bar sections | + +`count-5` is the same picture as `spread`. + +## Regenerating + +You need Python 3 with numpy and Pillow. The script uses about one core-minute per image at the default quality, so don't run it on the headset. Run it on a build machine: + +```sh +python3 -m venv venv && venv/bin/pip install numpy pillow +venv/bin/python make_poses.py --jobs 6 # all images, poses.json, contact sheet +venv/bin/python make_poses.py --only fist,ok # just some (poses.json is left alone) +venv/bin/python make_poses.py --ss 1 --jobs 6 # rough and about 8x faster, for trying things +``` + +`--out DIR` writes somewhere other than this folder. + +Each pose is a short entry in the `POSES` table in `make_poses.py`: a caption, the `two_hands` flag, and a function that returns the scene. A scene is the camera, layers of hands and objects (a layer can be a faint ghost), and arrows. Hands are built from joint angles: flexion at each finger's three joints, sideways spread, and four thumb angles. A thumb can also be given a target point, such as "touch the index fingertip", and a small solver finds the angles. The presets near `FLAT`, `FIST` and `OK` are a good place to start a new pose. + +## License + +MIT, like the rest of the repository. The script draws everything itself. A hand made of a palm slab and tapered capsules is ray-marched as a signed distance field, then shaded and outlined. No photos, downloaded images, scanned or research hand models, or AI image generators are involved, so the images carry no other terms. diff --git a/hands/rec/poses/bend.png b/hands/rec/poses/bend.png new file mode 100644 index 0000000..6c12bef Binary files /dev/null and b/hands/rec/poses/bend.png differ diff --git a/hands/rec/poses/claw.png b/hands/rec/poses/claw.png new file mode 100644 index 0000000..2e8b157 Binary files /dev/null and b/hands/rec/poses/claw.png differ diff --git a/hands/rec/poses/contact-sheet.png b/hands/rec/poses/contact-sheet.png new file mode 100644 index 0000000..67cca45 Binary files /dev/null and b/hands/rec/poses/contact-sheet.png differ diff --git a/hands/rec/poses/count-1.png b/hands/rec/poses/count-1.png new file mode 100644 index 0000000..cd583fc Binary files /dev/null and b/hands/rec/poses/count-1.png differ diff --git a/hands/rec/poses/count-2.png b/hands/rec/poses/count-2.png new file mode 100644 index 0000000..cdc0a13 Binary files /dev/null and b/hands/rec/poses/count-2.png differ diff --git a/hands/rec/poses/count-3.png b/hands/rec/poses/count-3.png new file mode 100644 index 0000000..e28aa39 Binary files /dev/null and b/hands/rec/poses/count-3.png differ diff --git a/hands/rec/poses/count-4.png b/hands/rec/poses/count-4.png new file mode 100644 index 0000000..0b2f9f8 Binary files /dev/null and b/hands/rec/poses/count-4.png differ diff --git a/hands/rec/poses/count-5.png b/hands/rec/poses/count-5.png new file mode 100644 index 0000000..e56012a Binary files /dev/null and b/hands/rec/poses/count-5.png differ diff --git a/hands/rec/poses/cross.png b/hands/rec/poses/cross.png new file mode 100644 index 0000000..b426c3d Binary files /dev/null and b/hands/rec/poses/cross.png differ diff --git a/hands/rec/poses/fist.png b/hands/rec/poses/fist.png new file mode 100644 index 0000000..f2a85fc Binary files /dev/null and b/hands/rec/poses/fist.png differ diff --git a/hands/rec/poses/flat-back.png b/hands/rec/poses/flat-back.png new file mode 100644 index 0000000..9a08f9c Binary files /dev/null and b/hands/rec/poses/flat-back.png differ diff --git a/hands/rec/poses/flat.png b/hands/rec/poses/flat.png new file mode 100644 index 0000000..8a90a8a Binary files /dev/null and b/hands/rec/poses/flat.png differ diff --git a/hands/rec/poses/grab.png b/hands/rec/poses/grab.png new file mode 100644 index 0000000..5bb95f2 Binary files /dev/null and b/hands/rec/poses/grab.png differ diff --git a/hands/rec/poses/hold.png b/hands/rec/poses/hold.png new file mode 100644 index 0000000..67ee2d3 Binary files /dev/null and b/hands/rec/poses/hold.png differ diff --git a/hands/rec/poses/lift.png b/hands/rec/poses/lift.png new file mode 100644 index 0000000..b508059 Binary files /dev/null and b/hands/rec/poses/lift.png differ diff --git a/hands/rec/poses/make_poses.py b/hands/rec/poses/make_poses.py new file mode 100644 index 0000000..c0663d8 --- /dev/null +++ b/hands/rec/poses/make_poses.py @@ -0,0 +1,1313 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: MIT +# Copyright (c) 2026 DeeJanuz +"""Example pose images for the hand recorder's headset panel. + +Everything here is drawn by this script: a simple parametric hand (a palm +slab and tapered capsules for the finger bones), posed with joint angles, +ray-marched as a signed distance field with numpy, shaded and outlined. +No outside images, hand models or image generators. + + python3 make_poses.py # all poses into this folder + python3 make_poses.py --only fist,ok # some of them + python3 make_poses.py --size 256 --ss 1 # quick, rough preview + +Writes .png (RGBA, transparent), poses.json and contact-sheet.png. +Needs numpy and Pillow. It is CPU heavy (about a minute per image on one +core), so run it on a build machine, not on the headset. +""" + +import argparse +import json +import math +import os +import sys +from concurrent.futures import ProcessPoolExecutor + +for _v in ("OMP_NUM_THREADS", "OPENBLAS_NUM_THREADS", "MKL_NUM_THREADS"): + os.environ.setdefault(_v, "1") # one thread per process: --jobs sets the parallelism + +import numpy as np # noqa: E402 +from PIL import Image, ImageDraw, ImageFont # noqa: E402 + +HERE = os.path.dirname(os.path.abspath(__file__)) + +# ------------------------------------------------------------------ math + + +def Rx(a): + a = math.radians(a) + c, s = math.cos(a), math.sin(a) + return np.array([[1, 0, 0], [0, c, -s], [0, s, c]], float) + + +def Ry(a): + a = math.radians(a) + c, s = math.cos(a), math.sin(a) + return np.array([[c, 0, s], [0, 1, 0], [-s, 0, c]], float) + + +def Rz(a): + a = math.radians(a) + c, s = math.cos(a), math.sin(a) + return np.array([[c, -s, 0], [s, c, 0], [0, 0, 1]], float) + + +def unit(v): + v = np.asarray(v, float) + return v / np.linalg.norm(v) + + +def orient(f, p): + """Hand rotation: fingers along f, palm facing p (right hand: thumb = f x p).""" + f = unit(f) + p = np.asarray(p, float) + p = unit(p - f * (p @ f)) + return np.column_stack([np.cross(f, p), f, p]) + + +V = lambda *a: np.array(a, float) # noqa: E731 + + +def smin(a, b, k): + if k <= 0: + return np.minimum(a, b) + h = np.clip(0.5 + 0.5 * (b - a) / k, 0.0, 1.0) + return b + (a - b) * h - k * h * (1.0 - h) + + +# ------------------------------------------------------------------ SDF nodes +# Every node maps points P (N,3) to signed distances (N,). Units are cm. +# xf(M, t) returns a copy moved by p -> M p + t (M orthonormal, may mirror). + + +class Cone: + """Round cone (tapered capsule) from a (radius r1) to b (radius r2).""" + + def __init__(s, a, b, r1, r2): + s.a, s.b, s.r1, s.r2 = V(*a), V(*b), float(r1), float(r2) + + def xf(s, M, t): + return Cone(M @ s.a + t, M @ s.b + t, s.r1, s.r2) + + def bounds(s): + return [((s.a + s.b) / 2, np.linalg.norm(s.b - s.a) / 2 + max(s.r1, s.r2))] + + def d(s, P): + ba = s.b - s.a + l2 = ba @ ba + rr = s.r1 - s.r2 + a2 = l2 - rr * rr + il2 = 1.0 / l2 + pa = P - s.a + y = pa @ ba + z = y - l2 + q = pa * l2 - y[:, None] * ba + x2 = np.einsum("ij,ij->i", q, q) + y2 = y * y * l2 + z2 = z * z * l2 + k = math.copysign(1.0, rr) * rr * rr * x2 if rr != 0 else np.zeros_like(x2) + d1 = np.sqrt(x2 + z2) * il2 - s.r2 + d2 = np.sqrt(x2 + y2) * il2 - s.r1 + d3 = (np.sqrt(np.maximum(x2 * a2 * il2, 0.0)) + y * rr) * il2 - s.r1 + return np.where(np.sign(z) * a2 * z2 > k, d1, np.where(np.sign(y) * a2 * y2 < k, d2, d3)) + + +class Box: + """Rounded box. R's columns are the box axes. taper: x half-size factor at -y end.""" + + def __init__(s, c, R, h, r, taper=None, bulge=0.0): + s.c, s.R, s.h, s.r, s.taper, s.bulge = V(*c), np.asarray(R, float), V(*h), float(r), taper, bulge + + def xf(s, M, t): + return Box(M @ s.c + t, M @ s.R, s.h, s.r, s.taper, s.bulge) + + def bounds(s): + return [(s.c, np.linalg.norm(s.h) + s.r)] + + def d(s, P): + L = (P - s.c) @ s.R + q = np.abs(L) - s.h + if s.taper is not None: + tt = np.clip((L[:, 1] + s.h[1]) / (2 * s.h[1]), 0, 1) + q[:, 0] = np.abs(L[:, 0]) - s.h[0] * (s.taper + (1 - s.taper) * tt) + d = np.linalg.norm(np.maximum(q, 0), axis=1) + np.minimum(q.max(1), 0) - s.r + if s.bulge: # gently convex faces instead of flat ones + ex, ey = s.h[0] + s.r, s.h[1] + s.r + d = d - s.bulge * np.clip(1 - (L[:, 0] / ex) ** 2, 0, 1) * np.clip(1 - (L[:, 1] / ey) ** 2, 0, 1) + return d + + +class Ell: + """Ellipsoid (approximate distance).""" + + def __init__(s, c, R, rad): + s.c, s.R, s.rad = V(*c), np.asarray(R, float), V(*rad) + + def xf(s, M, t): + return Ell(M @ s.c + t, M @ s.R, s.rad) + + def bounds(s): + return [(s.c, s.rad.max())] + + def d(s, P): + L = (P - s.c) @ s.R + k0 = np.linalg.norm(L / s.rad, axis=1) + k1 = np.linalg.norm(L / (s.rad * s.rad), axis=1) + return k0 * (k0 - 1.0) / np.maximum(k1, 1e-6) + + +class Cyl: + """Rounded cylinder along local y: radius ra, half height hh, edge rounding rb.""" + + def __init__(s, c, R, ra, hh, rb=0.1): + s.c, s.R, s.ra, s.hh, s.rb = V(*c), np.asarray(R, float), ra, hh, rb + + def xf(s, M, t): + return Cyl(M @ s.c + t, M @ s.R, s.ra, s.hh, s.rb) + + def bounds(s): + return [(s.c, math.hypot(s.ra, s.hh))] + + def d(s, P): + L = (P - s.c) @ s.R + dx = np.hypot(L[:, 0], L[:, 2]) - s.ra + s.rb + dy = np.abs(L[:, 1]) - s.hh + s.rb + return np.minimum(np.maximum(dx, dy), 0) + np.hypot(np.maximum(dx, 0), np.maximum(dy, 0)) - s.rb + + +class Torus: + """Torus in the local xz plane (axis local y).""" + + def __init__(s, c, R, R1, r2): + s.c, s.R, s.R1, s.r2 = V(*c), np.asarray(R, float), R1, r2 + + def xf(s, M, t): + return Torus(M @ s.c + t, M @ s.R, s.R1, s.r2) + + def bounds(s): + return [(s.c, s.R1 + s.r2)] + + def d(s, P): + L = (P - s.c) @ s.R + return np.hypot(np.hypot(L[:, 0], L[:, 2]) - s.R1, L[:, 1]) - s.r2 + + +class Keys: + """A grid of nx * nz key caps on the local xz plane (local y up).""" + + def __init__(s, c, R, nx, nz, pitch, kh, r): + s.c, s.R, s.nx, s.nz, s.pitch, s.kh, s.r = V(*c), np.asarray(R, float), nx, nz, pitch, V(*kh), r + + def xf(s, M, t): + return Keys(M @ s.c + t, M @ s.R, s.nx, s.nz, s.pitch, s.kh, s.r) + + def bounds(s): + return [(s.c, math.hypot(s.nx * s.pitch, s.nz * s.pitch) / 2 + 1)] + + def d(s, P): + L = (P - s.c) @ s.R + cx, cz = (s.nx - 1) / 2, (s.nz - 1) / 2 + ix = np.clip(np.round(L[:, 0] / s.pitch + cx), 0, s.nx - 1) + iz = np.clip(np.round(L[:, 2] / s.pitch + cz), 0, s.nz - 1) + q = np.abs(np.stack([L[:, 0] - (ix - cx) * s.pitch, L[:, 1], L[:, 2] - (iz - cz) * s.pitch], 1)) - s.kh + return np.linalg.norm(np.maximum(q, 0), axis=1) + np.minimum(q.max(1), 0) - s.r + + +class U: + """Union of nodes, smooth when k > 0.""" + + def __init__(s, kids, k=0.0): + s.kids, s.k = list(kids), k + + def xf(s, M, t): + return U([c.xf(M, t) for c in s.kids], s.k) + + def bounds(s): + return [b for c in s.kids for b in c.bounds()] + + def d(s, P): + d = s.kids[0].d(P) + for c in s.kids[1:]: + d = smin(d, c.d(P), s.k) + return d + + +class Sub: + """a minus b.""" + + def __init__(s, a, b): + s.a, s.b = a, b + + def xf(s, M, t): + return Sub(s.a.xf(M, t), s.b.xf(M, t)) + + def bounds(s): + return s.a.bounds() + + def d(s, P): + return np.maximum(s.a.d(P), -s.b.d(P)) + + +class Clip: + """a cut by the plane through point o with outward normal n (keeps the -n side).""" + + def __init__(s, a, o, n): + s.a, s.o, s.n = a, V(*o), unit(n) + + def xf(s, M, t): + return Clip(s.a.xf(M, t), M @ s.o + t, M @ s.n) + + def bounds(s): + return s.a.bounds() + + def d(s, P): + return np.maximum(s.a.d(P), (P - s.o) @ s.n) + + +# ------------------------------------------------------------------ the hand +# Local frame of a right hand: wrist joint at the origin, fingers along +y, +# palm facing +z, thumb on the +x side. cm, adult proportions. + +FINGERS = [ + # name, MCP joint, bone lengths (proximal, middle, distal incl. tip), radii (MCP, PIP, DIP, tip) + ("index", (2.5, 9.45, 0.0), (4.2, 2.45, 2.1), (1.03, 0.95, 0.85, 0.77)), + ("middle", (0.62, 9.85, 0.0), (4.6, 2.85, 2.3), (1.06, 0.98, 0.87, 0.79)), + ("ring", (-1.25, 9.5, 0.0), (4.35, 2.7, 2.2), (1.0, 0.92, 0.83, 0.75)), + ("pinky", (-2.95, 8.6, 0.0), (3.5, 2.05, 2.0), (0.9, 0.82, 0.74, 0.67)), +] +THUMB_CMC = (2.1, 2.3, 0.6) +THUMB_LENS = (4.6, 3.3, 2.6) +THUMB_RADII = (1.45, 1.08, 0.98, 0.86) +PARTS = {"palm": 0, "thumb": 1, "index": 2, "middle": 3, "ring": 4, "pinky": 5, "arm": 7} + +SKIN = V(0.93, 0.885, 0.84) +NAIL = V(1.0, 0.84, 0.82) +CREASE = V(0.62, 0.55, 0.52) +# palm creases (hand local x, y on the palm side): they tell the palm from the back +CREASES = [ + [(-3.7, 7.3), (-2.0, 7.6), (-0.5, 8.0), (0.8, 8.5), (1.6, 9.1)], + [(3.5, 6.9), (2.0, 6.6), (0.5, 6.2), (-1.2, 5.8), (-2.8, 5.5)], + [(3.5, 6.9), (2.2, 6.2), (1.3, 5.0), (0.9, 3.5), (0.9, 2.2), (1.2, 0.9)], +] + + +def crease_dist(xy): + d = np.full(len(xy), 1e9) + for line in CREASES: + for a, b in zip(line[:-1], line[1:]): + a, b = V(*a), V(*b) + t = np.clip((xy - a) @ (b - a) / ((b - a) @ (b - a)), 0, 1) + d = np.minimum(d, np.linalg.norm(xy - (a + t[:, None] * (b - a)), axis=1)) + return d + + +def thumb_twist(palmar): + return -62.0 - 0.45 * palmar + + +def finger_fk(base, lens, radii, ang): + mcp, pip, dip, abd = ang + F1 = Rz(-abd) @ Rx(mcp) + F2 = F1 @ Rx(pip) + F3 = F2 @ Rx(dip) + p0 = V(*base) + p1 = p0 + F1 @ V(0, lens[0], 0) + p2 = p1 + F2 @ V(0, lens[1], 0) + p3 = p2 + F3 @ V(0, lens[2] - radii[3], 0) + return dict(pts=[p0, p1, p2, p3], frames=[F1, F2, F3], radii=radii) + + +def thumb_fk(q, twist=None): + spread, palmar, mcp, ip = q + tw = thumb_twist(palmar) if twist is None else twist + B = Rz(-spread) @ Rx(palmar) @ Ry(tw) + F2 = B @ Rx(mcp) + F3 = F2 @ Rx(ip) + p0 = V(*THUMB_CMC) + p1 = p0 + B @ V(0, THUMB_LENS[0], 0) + p2 = p1 + F2 @ V(0, THUMB_LENS[1], 0) + p3 = p2 + F3 @ V(0, THUMB_LENS[2] - THUMB_RADII[3], 0) + return dict(pts=[p0, p1, p2, p3], frames=[B, F2, F3], radii=THUMB_RADII) + + +THUMB_LO = V(-30, -15, -15, -25) +THUMB_HI = V(85, 85, 70, 85) + + +def thumb_ik(target, prior, twist=None, fixed=()): + """Thumb angles that put the thumb tip centre at target (hand local). + + A pattern search, first held close to the prior (which picks the natural + solution), then refined from there to hit the target.""" + target = V(*target) + prior = V(*prior) + + def search(q, w, step): + def cost(q): + tip = thumb_fk(q, twist)["pts"][3] + return float(np.sum((tip - target) ** 2) + w * np.sum((q - prior) ** 2)) + + c = cost(q) + while step > 0.05: + better = False + for i in range(4): + if i in fixed: + continue + for sgn in (1, -1): + q2 = q.copy() + q2[i] = np.clip(q2[i] + sgn * step, THUMB_LO[i], THUMB_HI[i]) + c2 = cost(q2) + if c2 < c: + q, c, better = q2, c2, True + if not better: + step *= 0.5 + return q + + q = search(prior.copy(), 0.003, 16.0) + q = search(q, 0.00002, 4.0) + err = float(np.linalg.norm(thumb_fk(q, twist)["pts"][3] - target)) + if err > 0.3: + print(f"warning: thumb misses its target by {err:.2f} cm", file=sys.stderr) + return q + + +def pad_point(f, gap=0.0, seg=2, at=1.0, thumb_r=THUMB_RADII[3]): + """A point just off the pad side of a finger segment (seg 0..2; at 0..1 along it).""" + p = f["pts"][seg] + (f["pts"][seg + 1] - f["pts"][seg]) * at + r = f["radii"][seg] + (f["radii"][seg + 1] - f["radii"][seg]) * at + return p + f["frames"][seg][:, 2] * (r + thumb_r + gap - 0.08) + + +def back_point(f, seg=1, at=0.5, gap=0.0, thumb_r=THUMB_RADII[3]): + """A point just off the back (nail side) of a finger segment.""" + p = f["pts"][seg] + (f["pts"][seg + 1] - f["pts"][seg]) * at + r = f["radii"][seg] + (f["radii"][seg + 1] - f["radii"][seg]) * at + return p - f["frames"][seg][:, 2] * (r + thumb_r + gap - 0.1) + + +def resolve(pose): + """Pose dict -> local joint data. pose['thumb'] is angles or ('to', fn(J) -> point, prior).""" + J = {} + for name, base, lens, radii in FINGERS: + J[name] = finger_fk(base, lens, radii, pose[name]) + th = pose["thumb"] + tw = pose.get("twist") + if isinstance(th, tuple) and th and th[0] == "to": + q = thumb_ik(th[1](J), th[2], tw, th[3] if len(th) > 3 else ()) + else: + q = V(*th) + J["thumb"] = thumb_fk(q, tw) + J["thumb_q"] = q + return J + + +def chain_nodes(f): + p, r = f["pts"], f["radii"] + cones = [Cone(p[i], p[i + 1], r[i], r[i + 1]) for i in range(3)] + # the nail: a flat ellipsoid on the back of the last bone + F = f["frames"][2] + j, tip = p[2], p[3] + L = np.linalg.norm(tip - j) + nail = Ell(j + (tip - j) * 0.8 - F[:, 2] * (r[3] * 0.64), F, (r[3] * 0.72, L * 0.4 + 0.24, 0.32)) + return cones, nail + + +class Hand: + """A posed hand plus forearm, as an SDF scene item.""" + + def __init__(s, pose, R=np.eye(3), t=(0, 0, 0), left=False, arm=6.5, wrist=(0, 0), _copy=None): + if _copy is not None: + return + J = resolve(pose) + s.J = J + wf, wd = pose.get("wrist", wrist) + R = np.asarray(R, float) + S = np.diag([-1.0, 1, 1]) if left else np.eye(3) + Mh = S @ R @ Rx(wf) @ Rz(-wd) + Ma = S @ R + t = S @ V(*t) + s.M, s.t = Mh, t + palm = Box((-0.05, 4.95, 0.0), np.eye(3), (2.75, 4.15, 0.35), 1.1, taper=0.74, bulge=0.45) + meta_t = Cone(J["thumb"]["pts"][0], J["thumb"]["pts"][1], THUMB_RADII[0], THUMB_RADII[1]) + heel = Cone((-1.9, 1.6, 0.45), (-2.4, 6.5, 0.35), 1.15, 0.95) + knuckles = [Ell(V(*base) + V(0, -0.35, -0.55), np.eye(3), (0.95, 0.9, 0.8)) for _, base, _, _ in FINGERS] + s.core = U([U([palm, meta_t, heel], 1.1)] + knuckles, 0.5).xf(Mh, t) + s.arm = Box((0, -arm / 2 + 0.6, 0), np.eye(3), (1.25, arm / 2, 0.3), 1.35, bulge=0.2).xf(Ma, t) + s.arm_o = Ma @ V(0, 0.6, 0) + t + s.arm_dir = Ma @ V(0, -1, 0) + s.arm_len = arm + s.fingers, s.nails, s.ids = [], [], [] + for name in ("thumb", "index", "middle", "ring", "pinky"): + f = J[name] + cones, nail = chain_nodes(f) + if name == "thumb": + cones = cones[1:] + s.fingers.append(U(cones + [nail]).xf(Mh, t)) + s.nails.append(nail.xf(Mh, t)) + s.ids.append(PARTS[name]) + s.k = 0.75 + s.hid = 0 + + def world(s, p): + return s.M @ V(*p) + s.t + + def joint(s, name, i): + return s.world(s.J[name]["pts"][i]) + + def xf(s, M, t): + h = Hand(None, _copy=True) + h.J = s.J + h.M, h.t = M @ s.M, M @ s.t + t + h.core, h.arm = s.core.xf(M, t), s.arm.xf(M, t) + h.arm_o, h.arm_dir, h.arm_len = M @ s.arm_o + t, M @ s.arm_dir, s.arm_len + h.fingers = [f.xf(M, t) for f in s.fingers] + h.nails = [n.xf(M, t) for n in s.nails] + h.ids, h.k, h.hid = s.ids, s.k, s.hid + return h + + def bounds(s): + out = s.core.bounds() + s.arm.bounds() + for f in s.fingers: + out += f.bounds() + return out + + def _palm(s, P): + return smin(s.core.d(P), s.arm.d(P), 1.0) + + def d(s, P): + dp = s._palm(P) + out = dp + for f in s.fingers: + out = np.minimum(out, smin(dp, f.d(P), s.k)) + return out + + def info(s, P): + ds = [s.core.d(P), s.arm.d(P)] + [f.d(P) for f in s.fingers] + ids = np.array([PARTS["palm"], PARTS["arm"]] + s.ids)[np.argmin(np.stack(ds), 0)] + col = np.tile(SKIN, (len(P), 1)) + nail = np.zeros(len(P), bool) + for pid, n in zip(s.ids, s.nails): + nail |= (ids == pid) & (n.d(P) < 0.03) + col[nail] = NAIL + palm = ids == PARTS["palm"] + if palm.any(): + Lc = (P[palm] - s.t) @ s.M + w = np.clip(1 - crease_dist(Lc[:, :2]) / 0.13, 0, 1) * np.clip((Lc[:, 2] - 0.9) / 0.4, 0, 1) + col[palm] = col[palm] * (1 - w[:, None]) + CREASE * w[:, None] + along = (P - s.arm_o) @ s.arm_dir + fade = np.clip((s.arm_len - 0.4 - along) / (s.arm_len * 0.55), 0, 1) + fade = fade * fade * (3 - 2 * fade) + return s.hid * 16 + ids, col, fade, nail + + +class Obj: + """A plain object: one SDF node, one colour.""" + + def __init__(s, node, color=(0.56, 0.6, 0.67), oid=0): + s.node, s.color, s.oid = node, V(*color), oid + + def xf(s, M, t): + return Obj(s.node.xf(M, t), s.color, s.oid) + + def bounds(s): + return s.node.bounds() + + def d(s, P): + return s.node.d(P) + + def info(s, P): + n = len(P) + return np.full(n, 100 + s.oid), np.tile(s.color, (n, 1)), np.ones(n), np.zeros(n, bool) + + +# ------------------------------------------------------------------ rendering + +LIGHT = unit((-0.45, 0.7, 0.6)) +INK = V(0.1, 0.11, 0.13) + + +def scene_d(items, P): + d = items[0].d(P) + for it in items[1:]: + d = np.minimum(d, it.d(P)) + return d + + +def to_cam(items, cam): + return [it.xf(cam["R"], V(0, 0, 0)) for it in items] + + +def march(items, cam, W, ss): + """Orthographic sphere tracing along -z in camera space. Returns per-pixel buffers.""" + s = cam["scale"] / ss + Wp = W * ss + cx, cy = cam["center"] + bs = [b for it in items for b in it.bounds()] + C = np.array([b[0] for b in bs]) + Rr = np.array([b[1] for b in bs]) + zmin = (C[:, 2] - Rr).min() - 0.1 + px0 = int(max(0, math.floor(((C[:, 0] - Rr).min() - cx) / s + Wp / 2))) + px1 = int(min(Wp, math.ceil(((C[:, 0] + Rr).max() - cx) / s + Wp / 2))) + py0 = int(max(0, math.floor((cy - (C[:, 1] + Rr).max()) / s + Wp / 2))) + py1 = int(min(Wp, math.ceil((cy - (C[:, 1] - Rr).min()) / s + Wp / 2))) + out = dict(hit=np.zeros((Wp, Wp), bool), z=np.full((Wp, Wp), -1e9), id=np.full((Wp, Wp), -1, int), + rgb=np.zeros((Wp, Wp, 3)), fade=np.zeros((Wp, Wp)), nail=np.zeros((Wp, Wp), bool)) + if px1 <= px0 or py1 <= py0: + return out + us = cx + (np.arange(px0, px1) + 0.5 - Wp / 2) * s + vs = cy - (np.arange(py0, py1) + 0.5 - Wp / 2) * s + UU, VV = np.meshgrid(us, vs) + n = UU.size + P = np.zeros((n, 3)) + P[:, 0], P[:, 1] = UU.ravel(), VV.ravel() + zst = np.full(n, -1e9) + for c, r in bs: # start each ray where it enters the first bounding sphere + dd = r * r - (P[:, 0] - c[0]) ** 2 - (P[:, 1] - c[1]) ** 2 + ok = dd > 0 + zst[ok] = np.maximum(zst[ok], c[2] + np.sqrt(dd[ok])) + act = np.nonzero(zst > -1e8)[0] + P[:, 2] = zst + 0.05 + hit = np.zeros(n, bool) + eps = 0.0025 + for _ in range(320): + if act.size == 0: + break + d = scene_d(items, P[act]) + h = d < eps + hit[act[h]] = True + z = P[act, 2] - d * 0.9 + P[act[~h], 2] = z[~h] + act = act[(~h) & (z > zmin)] + if act.size: + d = scene_d(items, P[act]) + hit[act[d < 0.03]] = True + hi = np.nonzero(hit)[0] + if hi.size == 0: + return out + Ph = P[hi] + # normals (tetrahedron) + e = 0.01 + K = np.array([[1, -1, -1], [-1, -1, 1], [-1, 1, -1], [1, 1, 1]], float) + N = np.zeros_like(Ph) + for k in K: + N += k * scene_d(items, Ph + k * e)[:, None] + N /= np.maximum(np.linalg.norm(N, axis=1, keepdims=True), 1e-9) + # ambient occlusion + occ = np.zeros(len(Ph)) + for i in range(5): + hh = 0.15 + 0.45 * i + occ += (hh - scene_d(items, Ph + N * hh)) * (0.8 ** i) + ao = np.clip(1.0 - 0.32 * occ, 0, 1) + # soft shadow toward the key light + sh = np.ones(len(Ph)) + t = np.full(len(Ph), 0.12) + a2 = np.arange(len(Ph)) + P0 = Ph + N * 0.03 + for _ in range(40): + if a2.size == 0: + break + hq = scene_d(items, P0[a2] + LIGHT * t[a2, None]) + sh[a2] = np.minimum(sh[a2], 7.0 * hq / t[a2]) + t[a2] += np.clip(hq, 0.06, 2.0) + a2 = a2[(sh[a2] > 0.03) & (t[a2] < 30)] + sh = np.clip(sh, 0, 1) + sh = sh * sh * (3 - 2 * sh) + # labels and base colours: nearest item wins + dist = np.stack([it.d(Ph) for it in items]) + which = np.argmin(dist, 0) + ids = np.zeros(len(Ph), int) + base = np.zeros((len(Ph), 3)) + fade = np.ones(len(Ph)) + nail = np.zeros(len(Ph), bool) + for k, it in enumerate(items): + m = which == k + if m.any(): + ids[m], base[m], fade[m], nail[m] = it.info(Ph[m]) + diff = np.clip(N @ LIGHT, 0, 1) * (0.25 + 0.75 * sh) + hemi = 0.5 + 0.5 * N[:, 1] + amb = (0.62 + 0.38 * hemi) * ao + rim = np.clip(1 - np.clip(N[:, 2], 0, 1), 0, 1) ** 3 * 0.12 + spec = np.clip(N @ unit(LIGHT + V(0, 0, 1)), 0, 1) ** 24 * 0.08 * sh + col = base * (0.5 * amb + 0.55 * diff)[:, None] + (rim + spec)[:, None] + col = np.clip(col, 0, 1) + yy, xx = np.divmod(np.arange(n)[hi], px1 - px0) + yy += py0 + xx += px0 + out["hit"][yy, xx] = True + out["z"][yy, xx] = Ph[:, 2] + out["id"][yy, xx] = ids + out["rgb"][yy, xx] = col + out["fade"][yy, xx] = fade + out["nail"][yy, xx] = nail + return out + + +def disk(r): + R = int(math.ceil(r + 1)) + return [(dx, dy, math.hypot(dx, dy)) for dy in range(-R, R + 1) for dx in range(-R, R + 1) + if math.hypot(dx, dy) <= r + 0.5] + + +def smooth_pair(a, b): + """No line where palm/forearm meets the same hand's fingers.""" + same = (a // 16 == b // 16) & (a < 100) & (b < 100) + pa, pb = a % 16, b % 16 + return same & ((pa == 0) | (pa == 7) | (pb == 0) | (pb == 7)) + + +def ink(buf, ss, out_w=2.6, in_w=2.3, zt=0.45, line=INK, nail_line=V(0.62, 0.58, 0.56)): + """Shaded buffers -> premultiplied RGBA with outline and inner lines.""" + hit, Z, ID, rgb, F, NL = buf["hit"], buf["z"], buf["id"], buf["rgb"], buf["fade"], buf["nail"] + ro, ri = out_w * ss / 2 + 0.5, in_w * ss / 2 + 0.25 + inner = np.zeros(hit.shape) + outer = np.zeros(hit.shape) + ofade = np.zeros(hit.shape) + nl = np.zeros(hit.shape) + Fh = np.where(hit, F, 0) + for dx, dy, r in disk(max(ro, ri) + 1): + Hq = np.roll(hit, (dy, dx), (0, 1)) + wo = np.clip(ro - r + 0.5, 0, 1) + wi = np.clip(ri - r + 0.5, 0, 1) + if wo > 0: + m = (~hit) & Hq + outer = np.maximum(outer, m * wo) + ofade = np.maximum(ofade, np.roll(Fh, (dy, dx), (0, 1)) * (m * wo > 0)) + if wi > 0: + Zq = np.roll(Z, (dy, dx), (0, 1)) + Iq = np.roll(ID, (dy, dx), (0, 1)) + e = hit & (((~Hq) & (r <= ro * 0.6)) | (Hq & (Zq - Z > zt)) | + (Hq & (Iq != ID) & ~smooth_pair(ID, Iq) & (Zq >= Z - 0.15))) + inner = np.maximum(inner, e * wi) + if r <= ss * 0.7 + 0.5: + Nq = np.roll(NL, (dy, dx), (0, 1)) + nl = np.maximum(nl, (hit & Hq & (Nq != NL)) * 1.0) + col = rgb.copy() + col = col * (1 - nl[..., None] * 0.7) + nail_line * nl[..., None] * 0.7 + col = col * (1 - inner[..., None]) + line * inner[..., None] + alpha = np.where(hit, F, outer * ofade) + col = np.where(hit[..., None], col, line) + return col * alpha[..., None], alpha + + +def over(dst_rgb, dst_a, src_rgb, src_a): + return src_rgb + dst_rgb * (1 - src_a[..., None]), src_a + dst_a * (1 - src_a) + + +# ------------------------------------------------------------------ arrows + +ACCENT = (255, 184, 64) +ACCENT_DARK = (24, 26, 32) + + +def arc(c, u, v, r, a0, a1, n=48): + c, u, v = V(*c), unit(u), unit(v) + a = np.radians(np.linspace(a0, a1, n)) + return [c + r * (math.cos(t) * u + math.sin(t) * v) for t in a] + + +def seg(p0, p1, n=12): + p0, p1 = V(*p0), V(*p1) + return [p0 + (p1 - p0) * t for t in np.linspace(0, 1, n)] + + +def project(cam, W, ss, p): + q = cam["R"] @ V(*p) + s = cam["scale"] / ss + return ((q[0] - cam["center"][0]) / s + W * ss / 2, (cam["center"][1] - q[1]) / s + W * ss / 2) + + +def draw_arrows(arrows, cam, W, ss): + img = Image.new("RGBA", (W * ss, W * ss), (0, 0, 0, 0)) + dr = ImageDraw.Draw(img) + sc = ss * W / 512 + lw, bw, hl, hw = 7 * sc, 3.2 * sc, 22 * sc, 15 * sc + for a in arrows: + pts = [project(cam, W, ss, p) for p in a["pts"]] + heads = a.get("heads", "end") + k = a.get("scale", 1.0) + shapes = [] + + def head(tip, prev): + dx, dy = tip[0] - prev[0], tip[1] - prev[1] + L = math.hypot(dx, dy) or 1 + dx, dy = dx / L, dy / L + base = (tip[0] - dx * hl * k, tip[1] - dy * hl * k) + return [(tip[0] + dx * 2 * sc, tip[1] + dy * 2 * sc), + (base[0] - dy * hw * k, base[1] + dx * hw * k), + (base[0] + dy * hw * k, base[1] - dx * hw * k)] + + def back_to(pts, dist): + acc = 0 + for i in range(len(pts) - 1, 0, -1): + acc += math.dist(pts[i], pts[i - 1]) + if acc >= dist: + return pts[i - 1] + return pts[0] + + body = list(pts) + if heads in ("end", "both"): + shapes.append(head(pts[-1], back_to(pts, hl * k))) + # stop the line under the arrow head + while len(body) > 2 and math.dist(body[-1], pts[-1]) < hl * k * 0.6: + body.pop() + if heads in ("start", "both"): + rp = pts[::-1] + shapes.append(head(rp[-1], back_to(rp, hl * k))) + while len(body) > 2 and math.dist(body[0], pts[0]) < hl * k * 0.6: + body.pop(0) + for width, colr, grow in ((lw * k + 2 * bw, ACCENT_DARK, bw), (lw * k, ACCENT, 0)): + dr.line(body, fill=colr, width=int(round(width)), joint="curve") + for e in (body[0], body[-1]): + r = width / 2 + dr.ellipse([e[0] - r, e[1] - r, e[0] + r, e[1] + r], fill=colr) + for sh in shapes: + if grow: + cxs = sum(p[0] for p in sh) / 3 + cys = sum(p[1] for p in sh) / 3 + big = [] + for p in sh: + dx, dy = p[0] - cxs, p[1] - cys + L = math.hypot(dx, dy) or 1 + big.append((p[0] + dx / L * grow * 1.8, p[1] + dy / L * grow * 1.8)) + dr.polygon(big, fill=colr) + else: + dr.polygon(sh, fill=colr) + return img + + +# ------------------------------------------------------------------ layout + + +def fit(spec, W): + """Pick scale and centre so that everything fits with a margin.""" + cam = spec["cam"] + pts = [] + lo = W // 4 + tmp = dict(cam, scale=1.0, center=(0.0, 0.0)) + for layer in spec["layers"]: + items = to_cam(layer["items"], cam) + bs = [b for it in items for b in it.bounds()] + C = np.array([b[0] for b in bs]) + Rr = np.array([b[1] for b in bs]) + ext = max(np.abs(C[:, :2]).max() + Rr.max(), 1.0) * 2.1 + tmp = dict(cam, scale=ext / lo, center=(0.0, 0.0)) + buf = march(items, tmp, lo, 1) + yy, xx = np.nonzero(buf["hit"] & (buf["fade"] > 0.35)) + s = tmp["scale"] + for x, y in ((xx.min(), yy.min()), (xx.max() + 1, yy.max() + 1)): + pts.append((((x - lo / 2) * s), (-(y - lo / 2) * s))) + for a in spec.get("arrows", []): + for p in a["pts"]: + q = cam["R"] @ V(*p) + pts.append((q[0], q[1])) + pts = np.array(pts) + x0, y0 = pts.min(0) + x1, y1 = pts.max(0) + margin = 26 if spec.get("arrows") else 14 + scale = max(spec.get("scale", 0.056), (x1 - x0) / (W - 2 * margin), (y1 - y0) / (W - 2 * margin)) + return dict(cam, scale=scale, center=((x0 + x1) / 2, (y0 + y1) / 2)) + + +def render_pose(spec, W=512, ss=2): + cam = fit(spec, W) + Wp = W * ss + rgb = np.zeros((Wp, Wp, 3)) + a = np.zeros((Wp, Wp)) + for layer in spec["layers"]: + items = to_cam(layer["items"], cam) + for i, it in enumerate(items): + if isinstance(it, Hand): + it.hid = i + buf = march(items, cam, W, ss) + ghost = layer.get("ghost", 0) + if ghost: + buf["rgb"] = buf["rgb"] * 0.55 + V(0.62, 0.78, 1.0) * 0.45 + lrgb, la = ink(buf, ss * W / 512, out_w=2.0, in_w=1.2, line=V(0.25, 0.3, 0.4)) + lrgb, la = lrgb * ghost, la * ghost + else: + lrgb, la = ink(buf, ss * W / 512) + rgb, a = over(rgb, a, lrgb, la) + if spec.get("arrows"): + arr = np.asarray(draw_arrows(spec["arrows"], cam, W, ss), float) / 255 + aa = arr[..., 3] + rgb, a = over(rgb, a, arr[..., :3] * aa[..., None], aa) + # downsample (premultiplied box filter) + rgb = rgb.reshape(W, ss, W, ss, 3).mean((1, 3)) + a = a.reshape(W, ss, W, ss).mean((1, 3)) + col = np.where(a[..., None] > 1e-4, rgb / np.maximum(a[..., None], 1e-4), 0) + img = np.dstack([np.clip(col, 0, 1) * 255, np.clip(a, 0, 1) * 255]).round().astype(np.uint8) + return Image.fromarray(img, "RGBA") + + +# ------------------------------------------------------------------ poses + +STRAIGHT = (0, 0, 0) + + +def P(index, middle, ring, pinky, thumb, **kw): + d = dict(index=index, middle=middle, ring=ring, pinky=pinky, thumb=thumb) + d.update(kw) + return d + + +def ab(f, a): + return tuple(f[:3]) + (a,) + + +CURL = {"index": (86, 100, 58, -2), "middle": (88, 102, 58, 1), "ring": (90, 102, 56, 4), "pinky": (92, 98, 52, 8)} +FLAT = P((0, 0, 0, 1), (0, 0, 0, 0), (0, 0, 0, -1), (0, 0, 0, -3), (16, 6, 6, 4)) +SPREAD = P((-2, 0, 0, 14), (-2, 0, 0, 2), (-2, 0, 0, -11), (-2, 0, 0, -25), (42, 14, -4, -8)) +RELAX = P((12, 16, 8, 6), (15, 20, 10, 1), (18, 24, 12, -5), (22, 28, 14, -12), (38, 24, 12, 14)) +FIST = P(CURL["index"], CURL["middle"], CURL["ring"], CURL["pinky"], + ("to", lambda J: back_point(J["middle"], 1, 0.35), (-5, 40, 40, 25))) +CLAW = P((-10, 80, 70, 14), (-10, 86, 72, 2), (-10, 86, 72, -11), (-10, 84, 68, -24), (45, 30, 35, 55)) +POINT = P((0, 0, 0, 0), CURL["middle"], CURL["ring"], CURL["pinky"], + ("to", lambda J: back_point(J["middle"], 1, 0.4), (-5, 40, 40, 25))) +OK = P((55, 50, 30, 3), (4, 6, 3, -3), (6, 8, 4, -13), (8, 10, 5, -26), + ("to", lambda J: pad_point(J["index"], 0.0), (12, 50, 5, 15))) +PINCH = P((40, 48, 24, 3), (70, 80, 44, 0), (76, 84, 46, -1), (80, 84, 46, -2), + ("to", lambda J: pad_point(J["index"], 0.0), (25, 40, 10, 10))) +PINCH_OPEN = P((34, 40, 18, 4), (46, 60, 30, 0), (54, 66, 34, -2), (60, 68, 36, -5), + ("to", lambda J: pad_point(J["index"], 2.8), (30, 35, 5, 5))) +THUMBS = P(CURL["index"], CURL["middle"], CURL["ring"], CURL["pinky"], (62, 12, -5, -8)) +GRIP = P((52, 70, 36, 0), (55, 72, 36, 1), (58, 72, 34, 3), (62, 70, 32, 6), (8, 48, 30, 25)) + + +def COUNT(n): + up = ["index", "middle", "ring"][:n] + f = {} + spread = {"index": 4, "middle": -1, "ring": -6} + for name in ("index", "middle", "ring", "pinky"): + f[name] = (0, 0, 0, spread[name]) if name in up else CURL[name] + first_down = ["index", "middle", "ring", "pinky"][n] + th = ("to", lambda J: back_point(J[first_down], 1, 0.4), (-5, 40, 40, 25)) + return P(f["index"], f["middle"], f["ring"], f["pinky"], th) + + +COUNT4 = P((0, 0, 0, 6), (0, 0, 0, 0), (0, 0, 0, -6), (0, 0, 0, -14), + ("to", lambda J: (-1.6, 7.6, 2.3), (-15, 40, 40, 20))) + +UP = np.eye(3) # palm toward you, fingers up +BACK = Ry(180) # back of the hand toward you +PALM_DOWN_FWD = orient((0, 0, -1), (0, -1, 0)) # on a desk: fingers away, palm down + + +def H(pose, R=UP, t=(0, 0, 0), **kw): + return Hand(pose, R, t, **kw) + + +def L(*items, ghost=0): + return dict(items=list(items), ghost=ghost) + + +def cam(pitch=0, yaw=0): + """World -> camera. pitch > 0 looks down, yaw > 0 looks from the right.""" + return dict(R=Rx(pitch) @ Ry(-yaw)) + + +CTRL_GRIP = P((50, 66, 36, 0), (54, 70, 36, 1), (58, 70, 34, 3), (62, 68, 32, 6), (30, 45, 10, 12)) + + +def ctrl_local(): + """A generic VR controller in the hand frame of CTRL_GRIP (hand local coordinates): + a handle across the curled fingers, a head above the thumb with a thumbstick, + and a strap over the back of the hand.""" + y, z, r = grip_hole(resolve(CTRL_GRIP)) + ax = unit((0.45, 1.0, 0.1)) + c = V(0.0, y - 0.6, z) + a, b = c - ax * 4.5, c + ax * 4.0 + grip = Cone(a, b, r * 0.92, r * 1.02) + head_c = b + ax * 1.6 + V(0.9, 0.0, 0.4) + head = Ell(head_c, orient(ax, (0.3, 0, 1)), (2.4, 2.0, 2.0)) + body = U([grip, head], 0.9) + up = unit((0.55, 0.55, 0.65)) + sb = head_c + up * 1.75 + stick = U([Cone(sb - up * 0.4, sb + up * 0.7, 0.36, 0.36), + Cyl(sb + up * 0.85, orient(up, (0, 0, 1)), 0.78, 0.2, 0.12)]) + strap = Box((0.3, y - 1.2, -2.3), orient(ax, (0, 0, 1)), (0.95, 4.4, 0.1), 0.18) + return dict(body=body, stick=stick, strap=strap, top=sb + up * 1.05) + + +CTRL = None + + +def ctrl_parts(M=np.eye(3), t=(0, 0, 0)): + global CTRL + if CTRL is None: + CTRL = ctrl_local() + t = V(*t) + return [Obj(CTRL["body"], (0.5, 0.54, 0.6), 1).xf(M, t), Obj(CTRL["stick"], (0.3, 0.32, 0.37), 2).xf(M, t), + Obj(CTRL["strap"], (0.36, 0.38, 0.43), 3).xf(M, t)] + + +def stick_top(hand): + if CTRL is None: + ctrl_parts() + return hand.world(CTRL["top"]) + + +def with_ctrl(hand): + """Controller objects placed in a hand's frame.""" + return ctrl_parts(hand.M, hand.t) + + +def keyboard(c=(0, 0, 0), nx=14, nz=5): + pitch = 1.9 + w, dpt = nx * pitch / 2 + 0.6, nz * pitch / 2 + 0.6 + base = Box((c[0], c[1] + 0.8, c[2]), np.eye(3), (w - 0.4, 0.4, dpt - 0.4), 0.4) + keys = Keys((c[0], c[1] + 1.75, c[2]), np.eye(3), nx, nz, pitch, (0.62, 0.18, 0.62), 0.12) + return [Obj(base, (0.42, 0.45, 0.5), 4), Obj(keys, (0.6, 0.64, 0.7), 5)] + + +def mouse(c=(0, 0, 0)): + body = Clip(Ell((c[0], c[1] + 0.2, c[2]), np.eye(3), (3.1, 3.6, 5.6)), (c[0], c[1], c[2]), (0, -1, 0)) + groove = Box((c[0], c[1] + 3.6, c[2] - 3.0), np.eye(3), (0.08, 1.2, 2.6), 0.02) + return [Obj(Sub(body, groove), (0.56, 0.6, 0.67), 6)] + + +def cup(c=(0, 0, 0)): + c = V(*c) + outer = Cyl(c + V(0, 4.75, 0), np.eye(3), 3.9, 4.75, 0.35) + inner = Cyl(c + V(0, 5.6, 0), np.eye(3), 3.45, 4.75, 0.3) + handle = Torus(c + V(-4.6, 5.2, 0), Rx(90), 2.0, 0.45) + return [Obj(U([Sub(outer, inner), Clip(handle, c + V(-4.2, 0, 0), (1, 0, 0))], 0.3), (0.56, 0.6, 0.67), 7)] + + +def place(hand_pose, R, anchor, target, joint=("middle", 3), **kw): + """Hand with a joint (default: middle fingertip) at a target point.""" + h = Hand(hand_pose, R, (0, 0, 0), **kw) + p = h.joint(*joint) + if kw.get("left"): + p = p # joint() already includes the mirror + off = V(*target) - p + V(*anchor) + return Hand(hand_pose, R, (np.diag([-1.0, 1, 1]) if kw.get("left") else np.eye(3)) @ off, **kw) + + +def typing_hands(lift=0.0): + pose = P((18, 42, 18, 3), (20, 46, 20, 0), (22, 48, 20, -3), (26, 46, 20, -7), (30, 28, 10, 10), + wrist=(-12, 0)) + R = PALM_DOWN_FWD + rh = place(pose, R, (0, lift, 0), (4.0, 2.55 + 0.74, -1.0)) + lh = place(pose, R, (0, lift, 0), (-4.0, 2.55 + 0.74, -1.0), left=True) + return rh, lh + + +def spec_typing(): + rh, lh = typing_hands() + return dict(cam=cam(48), layers=[L(*keyboard(), rh, lh)]) + + +def spec_lift(): + rh, lh = typing_hands() + g1, g2 = typing_hands(lift=7) + return dict(cam=cam(48), layers=[L(g1, g2, ghost=0.5), L(*keyboard(), rh, lh)], + arrows=[dict(pts=seg((13.5, 4, 3), (13.5, 11, 3)), heads="both")]) + + +MOUSE_POSE = P((12, 22, 10, 2), (12, 24, 12, -1), (30, 50, 26, -6), (40, 54, 28, -12), (52, 18, 10, 8), + wrist=(-8, 0)) + + +def mouse_hand(c=(0, 0, 0)): + return place(MOUSE_POSE, PALM_DOWN_FWD, (0, 0, 0), V(*c) + V(0.4, 3.6, -4.6)) + + +def spec_mouse(): + return dict(cam=cam(35, -40), layers=[L(*mouse(), mouse_hand())]) + + +def spec_switch(): + rh, lh = typing_hands() + mh = mouse_hand((26, 0, 0)) + return dict(cam=cam(48), layers=[L(mh, ghost=0.5), L(*keyboard(), *mouse((26, 0, 0)), rh, lh)], + arrows=[dict(pts=arc((17, 6, 2), (1, 0, 0), (0, 1, 0), 7, 160, 20), heads="both")]) + + +def grip_hole(J, name="middle", palm_z=1.9): + """Centre (y, z) and radius of the biggest bar that fits inside a curled finger (hand local).""" + f = J[name] + best = (0.0, 0.0, 0.0) + ys = [p[1] for p in f["pts"]] + zs = [p[2] for p in f["pts"]] + for y in np.linspace(ys[0], max(ys), 60): + for z in np.linspace(palm_z, max(zs), 60): + r = z - palm_z + c = V(y, z) + for i in range(3): + a, b = f["pts"][i][1:], f["pts"][i + 1][1:] + t = np.clip((c - a) @ (b - a) / ((b - a) @ (b - a)), 0, 1) + r = min(r, np.linalg.norm(c - (a + (b - a) * t)) - f["radii"][i + 1]) + if r > best[2]: + best = (y, z, r) + return best + + +GLASS = P((40, 70, 40, 0), (42, 72, 40, 1), (44, 72, 38, 3), (48, 70, 36, 6), + ("to", lambda J: (2.4, 9.0, 6.4), (10, 50, 20, 20))) +HOLD_VIEW = Rz(75) @ Ry(-15) + + +def spec_hold(): + h = H(GLASS, HOLD_VIEW) + y, z, r = grip_hole(h.J) + c = h.world((0.3, y, z)) + ax = unit(h.M @ V(1, 0, 0)) + R = orient(ax, (0, 0, 1)) # the bottle's axis runs along the hand's x axis + body = Cyl(c + ax * 1.0, R, r - 0.05, 6.5, 0.4) + neck = Cone(c + ax * 7.0, c + ax * 9.6, r * 0.75, r * 0.45) + cap = Cyl(c + ax * 10.0, R, r * 0.5, 0.6, 0.15) + bottle = Obj(U([body, neck, cap], 0.6), (0.56, 0.6, 0.67), 7) + top = c + ax * 9.5 + return dict(cam=cam(12), layers=[L(bottle, h)], + arrows=[dict(pts=seg(top + V(7, -6, 0), top + V(7, 1, 0)), heads="both", scale=0.85)]) + + +def touch_scene(lR, f, p, lt=(0, 0, 0), left=True): + """A hand holding a controller (left by default) and the other index on its thumbstick.""" + ch = Hand(CTRL_GRIP, lR, lt, left=left) + up = ch.M @ unit((0.55, 0.55, 0.65)) + tgt = stick_top(ch) + up * 0.7 + ph = place(POINT, orient(f, p), (0, 0, 0), tgt, joint=("index", 3), left=not left) + return [ch, *with_ctrl(ch), ph] + + +def spec_touch_stick(): + return dict(cam=cam(20), layers=[L(*touch_scene(Rz(-10) @ Rx(-50), (-1, -1, -0.6), (-0.3, -0.6, -1)))]) + + +def spec_touch_stick_desk(f=(-1, -0.8, 0), p=(0, -1, -0.3)): + hold = Hand(CTRL_GRIP, orient((0, 0, -1), (0, 1, 0))) + objs = with_ctrl(hold)[:2] # no strap: it hangs loose on the desk + up = hold.M @ unit((0.55, 0.55, 0.65)) + tgt = stick_top(hold) + up * 0.7 + rh = place(POINT, orient(f, p), (0, 0, 0), tgt, joint=("index", 3)) + return dict(cam=cam(25), layers=[L(*objs, rh)]) + + +def push_spec(controllers=False): + R = BACK + main = H(RELAX if not controllers else CTRL_GRIP, R, (0, 0, 0)) + ghost = H(RELAX if not controllers else CTRL_GRIP, R, (0, 0, -16)) + lm = [main] + (with_ctrl(main) if controllers else []) + lg = [ghost] + (with_ctrl(ghost) if controllers else []) + return dict(cam=cam(18, 50), layers=[L(*lg, ghost=0.45), L(*lm)], + arrows=[dict(pts=seg((-4.5, 21, -1), (-4.5, 21, -15)), heads="both")]) + + +def ghost_pair(main_pose, ghost_pose, R, cam_, arrows, ghost_alpha=0.45, **kw): + return dict(cam=cam_, layers=[L(H(ghost_pose, R, **kw), ghost=ghost_alpha), L(H(main_pose, R, **kw))], + arrows=arrows) + + +def spec_cross(): + R = Rz(30) + rh = H(FLAT, R, (-2.2, 0, 1.6), arm=12) + lh = H(FLAT, R, (-2.2, 0, -1.6), left=True, arm=12) + return dict(cam=cam(), layers=[L(rh, lh)], + arrows=[dict(pts=seg((-5, 22, 3), (-14, 22, 3)), scale=0.85), + dict(pts=seg((5, 22, 3), (14, 22, 3)), scale=0.85)]) + + +def spec_overlap(): + rh = H(FLAT, Rz(8), (-2.5, 2.5, 3)) + lh = H(FLAT, Rz(8), (-2.5, -2.5, -3), left=True) + return dict(cam=cam(), layers=[L(rh, lh)], + arrows=[dict(pts=arc((0, 9, 4), (1, 0, 0), (0, 1, 0), 13.5, 35, 145), heads="both")]) + + +def spec_near_face(): + head = Ell((0, 0, 0), np.eye(3), (7.6, 10.2, 8.8)) + visor = Box((0, 2.4, 6.5), np.eye(3), (8.2, 2.6, 2.6), 1.4) + band = Torus((0, 3.0, -0.5), Rx(-8), 8.0, 0.9) + objs = [Obj(U([head]), (0.5, 0.53, 0.58), 8), Obj(U([visor, band], 0.4), (0.36, 0.38, 0.43), 9)] + R = orient((0, 1, 0.15), (1, 0, 0.2)) + rh = H(RELAX, R, (-12.5, -9, 4)) + lh = H(RELAX, R, (-12.5, -9, 4), left=True) + return dict(cam=cam(5, 22), layers=[L(*objs, rh, lh)], + arrows=[dict(pts=arc((0, 2, 0), (1, 0, 0), (0, 1, 0), 15.5, 40, 140), heads="both")]) + + +def spec_screen_point(): + R = Rx(-35) @ BACK + h = H(POINT, R, (0, -12, 0)) + tip = h.joint("index", 3) + scr = Box((2, tip[1] + 9, -30), np.eye(3), (14, 8, 0.4), 0.5) + stand = Box((2, tip[1] - 1, -30.5), np.eye(3), (1.2, 2.5, 0.3), 0.3) + return dict(cam=cam(8), layers=[L(Obj(U([scr, stand]), (0.42, 0.45, 0.5), 10), h)], + arrows=[dict(pts=seg(tip + V(0, 1.2, 0), tip + V(0, 7.0, -20)), scale=0.75)]) + + +def spec_turn_in_out(): + return dict(cam=cam(16), layers=[L(H(RELAX, Ry(-25)))], + arrows=[dict(pts=arc((0, -1.5, 0), (1, 0, 0), (0, 0, 1), 6.5, -20, 200), heads="both")]) + + +def spec_turn_up_down(): + R = PALM_DOWN_FWD + Rg = orient((0, 0, -1), (0, 1, 0)) + return dict(cam=cam(40), layers=[L(H(FLAT, Rg, (0, 0.2, 0)), ghost=0.4), L(H(FLAT, R))], + arrows=[dict(pts=arc((0, 0, 0.5), (1, 0, 0), (0, 1, 0), 7.0, 15, 165), heads="both")]) + + +def spec_bend(): + R = PALM_DOWN_FWD + g1 = H(dict(FLAT, wrist=(0, 28)), R) + g2 = H(dict(FLAT, wrist=(0, -28)), R) + g3 = H(dict(FLAT, wrist=(-35, 0)), R) + return dict(cam=cam(38), layers=[L(g1, g2, ghost=0.35), L(g3, ghost=0.35), L(H(FLAT, R))], + arrows=[dict(pts=arc((0, 1, 0), (1, 0, 0), (0, 0, -1), 21, 60, 120), heads="both"), + dict(pts=arc((0, 1, 0), (0, 0, -1), (0, 1, 0), 21, 10, 50), heads="both", scale=0.8)]) + + +PINCH_VIEW = Ry(-90) + + +def spec_pinch_tap(): + h = H(PINCH_OPEN, PINCH_VIEW) + a, b = h.joint("index", 3), h.joint("thumb", 3) + side = V(-3.2, 0, 0) + return dict(cam=cam(20), layers=[L(h)], + arrows=[dict(pts=seg(a + side + V(0, 1.0, 0), b + side - V(0, 1.0, 0)), heads="both", scale=0.7)]) + + +def spec_pinch_drag(): + h = H(PINCH, PINCH_VIEW) + g = H(PINCH, PINCH_VIEW, (15, 0, 0)) + top = h.joint("middle", 1) + return dict(cam=cam(20), layers=[L(g, ghost=0.4), L(h)], + arrows=[dict(pts=seg(top + V(1, 3.5, 0), top + V(12, 3.5, 0)))]) + + +def spec_grab(): + R = Ry(-60) + h = H(GRIP, R) + y, z, r = grip_hole(h.J) + c = h.world((-0.4, y, z)) + bar = Cyl(c, orient(h.M @ V(1, 0, 0), (0, 0, 1)), r - 0.05, 8, 0.3) + return dict(cam=cam(10), layers=[L(H(RELAX, R), ghost=0.4), L(h, Obj(bar, (0.5, 0.54, 0.6), 11))], + arrows=[dict(pts=arc(c, (0, 1, 0), (1, 0, 0), 9.5, 20, 100), heads="both", scale=0.85)]) + + +def spec_open_close(): + R = UP + return dict(cam=cam(), layers=[L(H(SPREAD, R), ghost=0.45), L(H(FIST, R))], + arrows=[dict(pts=arc((0, 11, 3), (1, 0, 0), (0, 1, 0), 9.5, -20, 60), heads="both", scale=0.85)]) + + +def spec_no_hands(): + g = H(RELAX, BACK, (0, 6, 0)) + return dict(cam=cam(), layers=[L(g, ghost=0.4)], + arrows=[dict(pts=seg((7.5, 12, 4), (7.5, -2, 4)))]) + + +POSES = { + "flat": ("Palm toward you, fingers together", False, lambda: dict(cam=cam(), layers=[L(H(FLAT))])), + "flat-back": ("Back of the hand toward you", False, lambda: dict(cam=cam(), layers=[L(H(FLAT, BACK))])), + "spread": ("Fingers spread wide", False, lambda: dict(cam=cam(), layers=[L(H(SPREAD))])), + "open": ("Open and relaxed", False, lambda: dict(cam=cam(), layers=[L(H(RELAX, Ry(-25) @ Rx(-10)))])), + "fist": ("A fist", False, lambda: dict(cam=cam(), layers=[L(H(FIST, Ry(-20)))])), + "point": ("Point with your index finger", False, + lambda: dict(cam=cam(20, -30), layers=[L(H(POINT, orient((0.1, 0.5, -1), (-0.6, -1, 0))))])), + "pinch": ("Thumb and index tips touching", False, + lambda: dict(cam=cam(20), layers=[L(H(PINCH, Ry(-90)))])), + "ok": ("OK sign: a ring, three fingers up", False, + lambda: dict(cam=cam(10), layers=[L(H(OK, Ry(-70)))])), + "thumbs-up": ("Thumbs up", False, lambda: dict(cam=cam(), layers=[L(H(THUMBS, Rz(75) @ Ry(-15)))])), + "claw": ("Fingers curled like a claw", False, + lambda: dict(cam=cam(35), layers=[L(H(CLAW, Ry(-20)))])), + "count-1": ("One: index finger", False, lambda: dict(cam=cam(), layers=[L(H(COUNT(1)))])), + "count-2": ("Two: index and middle", False, lambda: dict(cam=cam(), layers=[L(H(COUNT(2)))])), + "count-3": ("Three: index, middle, ring", False, lambda: dict(cam=cam(), layers=[L(H(COUNT(3)))])), + "count-4": ("Four: thumb folded in", False, lambda: dict(cam=cam(), layers=[L(H(COUNT4))])), + "count-5": ("Five: all fingers spread", False, lambda: dict(cam=cam(), layers=[L(H(SPREAD))])), + "turn-in-out": ("Turn your wrist: palm in, palm out", False, spec_turn_in_out), + "turn-up-down": ("Palm down, turn it up, then back", False, spec_turn_up_down), + "bend": ("Bend your wrist up, down, side to side", False, spec_bend), + "pinch-tap": ("Tap thumb and index together", False, spec_pinch_tap), + "pinch-drag": ("Pinch, move to the side, let go", False, spec_pinch_drag), + "grab": ("Close your hand around a bar, open it", False, spec_grab), + "open-close": ("Open and close your hand", False, spec_open_close), + "cross": ("Cross your hands, then uncross", True, spec_cross), + "overlap": ("One hand over the other, slide around", True, spec_overlap), + "near-face": ("Hands near your face, not touching", True, spec_near_face), + "screen-point": ("Point at a screen at arm's length", False, spec_screen_point), + "typing": ("Type on the keyboard", True, spec_typing), + "lift": ("Lift your hands off the keyboard, put them back", True, spec_lift), + "mouse": ("Hand on the mouse, move and click", False, spec_mouse), + "switch": ("Switch between keyboard and mouse", True, spec_switch), + "hold": ("Pick it up, use it, put it down", True, spec_hold), + "push": ("Palms out, push out and back", False, lambda: push_spec(False)), + "push-controller": ("Controller on, push out and back", False, lambda: push_spec(True)), + "touch-stick": ("Touch the thumbstick with your other index", True, spec_touch_stick), + "touch-stick-desk": ("Touch the thumbstick of the controller on the desk", True, spec_touch_stick_desk), + "no-hands": ("Hands down, out of view", False, spec_no_hands), +} + + +# ------------------------------------------------------------------ main + + +def _job(args): + pid, out, W, ss = args + img = render_pose(POSES[pid][2](), W, ss) + img.save(os.path.join(out, pid + ".png"), optimize=True) + return pid + + +def contact_sheet(ids, out, thumb=240, cols=6): + rows = (len(ids) + cols - 1) // cols + pad, lab = 14, 26 + W = cols * (thumb + pad) + pad + Hh = rows * (thumb + lab + pad) + pad + sheet = Image.new("RGBA", (W, Hh), (22, 24, 29, 255)) + dr = ImageDraw.Draw(sheet) + try: + font = ImageFont.load_default(size=17) + except TypeError: + font = ImageFont.load_default() + for i, pid in enumerate(ids): + x = pad + (i % cols) * (thumb + pad) + y = pad + (i // cols) * (thumb + lab + pad) + dr.rounded_rectangle([x, y, x + thumb, y + thumb], 10, fill=(34, 37, 44, 255)) + im = Image.open(os.path.join(out, pid + ".png")).convert("RGBA").resize((thumb, thumb), Image.LANCZOS) + sheet.alpha_composite(im, (x, y)) + dr.text((x + thumb / 2, y + thumb + 4), pid, fill=(220, 224, 230, 255), font=font, anchor="mt") + sheet.convert("RGB").save(os.path.join(out, "contact-sheet.png"), optimize=True) + + +def main(): + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--out", default=HERE) + ap.add_argument("--only", default="", help="comma-separated pose ids") + ap.add_argument("--size", type=int, default=512) + ap.add_argument("--ss", type=int, default=3, help="supersampling per axis") + ap.add_argument("--jobs", type=int, default=1) + ap.add_argument("--no-sheet", action="store_true") + a = ap.parse_args() + ids = [p for p in a.only.split(",") if p] or list(POSES) + bad = [p for p in ids if p not in POSES] + if bad: + sys.exit("unknown pose: " + ", ".join(bad)) + os.makedirs(a.out, exist_ok=True) + work = [(p, a.out, a.size, a.ss) for p in ids] + if a.jobs > 1: + with ProcessPoolExecutor(a.jobs) as ex: + for pid in ex.map(_job, work): + print(pid, flush=True) + else: + for w in work: + print(_job(w), flush=True) + if not a.only: + meta = {pid: {"file": pid + ".png", "two_hands": two, "caption": cap} + for pid, (cap, two, _) in POSES.items()} + with open(os.path.join(a.out, "poses.json"), "w") as f: + json.dump(meta, f, indent=2) + f.write("\n") + if not a.no_sheet: + have = [p for p in POSES if os.path.exists(os.path.join(a.out, p + ".png"))] + contact_sheet(have, a.out) + + +if __name__ == "__main__": + main() diff --git a/hands/rec/poses/mouse.png b/hands/rec/poses/mouse.png new file mode 100644 index 0000000..9b8d92c Binary files /dev/null and b/hands/rec/poses/mouse.png differ diff --git a/hands/rec/poses/near-face.png b/hands/rec/poses/near-face.png new file mode 100644 index 0000000..4da0519 Binary files /dev/null and b/hands/rec/poses/near-face.png differ diff --git a/hands/rec/poses/no-hands.png b/hands/rec/poses/no-hands.png new file mode 100644 index 0000000..7b7eebe Binary files /dev/null and b/hands/rec/poses/no-hands.png differ diff --git a/hands/rec/poses/ok.png b/hands/rec/poses/ok.png new file mode 100644 index 0000000..d8b6c00 Binary files /dev/null and b/hands/rec/poses/ok.png differ diff --git a/hands/rec/poses/open-close.png b/hands/rec/poses/open-close.png new file mode 100644 index 0000000..051c372 Binary files /dev/null and b/hands/rec/poses/open-close.png differ diff --git a/hands/rec/poses/open.png b/hands/rec/poses/open.png new file mode 100644 index 0000000..7f9cf9b Binary files /dev/null and b/hands/rec/poses/open.png differ diff --git a/hands/rec/poses/overlap.png b/hands/rec/poses/overlap.png new file mode 100644 index 0000000..dc60427 Binary files /dev/null and b/hands/rec/poses/overlap.png differ diff --git a/hands/rec/poses/pinch-drag.png b/hands/rec/poses/pinch-drag.png new file mode 100644 index 0000000..eab4981 Binary files /dev/null and b/hands/rec/poses/pinch-drag.png differ diff --git a/hands/rec/poses/pinch-tap.png b/hands/rec/poses/pinch-tap.png new file mode 100644 index 0000000..4a7ca50 Binary files /dev/null and b/hands/rec/poses/pinch-tap.png differ diff --git a/hands/rec/poses/pinch.png b/hands/rec/poses/pinch.png new file mode 100644 index 0000000..d1906a9 Binary files /dev/null and b/hands/rec/poses/pinch.png differ diff --git a/hands/rec/poses/point.png b/hands/rec/poses/point.png new file mode 100644 index 0000000..e050565 Binary files /dev/null and b/hands/rec/poses/point.png differ diff --git a/hands/rec/poses/poses.json b/hands/rec/poses/poses.json new file mode 100644 index 0000000..7ecbcb6 --- /dev/null +++ b/hands/rec/poses/poses.json @@ -0,0 +1,182 @@ +{ + "flat": { + "file": "flat.png", + "two_hands": false, + "caption": "Palm toward you, fingers together" + }, + "flat-back": { + "file": "flat-back.png", + "two_hands": false, + "caption": "Back of the hand toward you" + }, + "spread": { + "file": "spread.png", + "two_hands": false, + "caption": "Fingers spread wide" + }, + "open": { + "file": "open.png", + "two_hands": false, + "caption": "Open and relaxed" + }, + "fist": { + "file": "fist.png", + "two_hands": false, + "caption": "A fist" + }, + "point": { + "file": "point.png", + "two_hands": false, + "caption": "Point with your index finger" + }, + "pinch": { + "file": "pinch.png", + "two_hands": false, + "caption": "Thumb and index tips touching" + }, + "ok": { + "file": "ok.png", + "two_hands": false, + "caption": "OK sign: a ring, three fingers up" + }, + "thumbs-up": { + "file": "thumbs-up.png", + "two_hands": false, + "caption": "Thumbs up" + }, + "claw": { + "file": "claw.png", + "two_hands": false, + "caption": "Fingers curled like a claw" + }, + "count-1": { + "file": "count-1.png", + "two_hands": false, + "caption": "One: index finger" + }, + "count-2": { + "file": "count-2.png", + "two_hands": false, + "caption": "Two: index and middle" + }, + "count-3": { + "file": "count-3.png", + "two_hands": false, + "caption": "Three: index, middle, ring" + }, + "count-4": { + "file": "count-4.png", + "two_hands": false, + "caption": "Four: thumb folded in" + }, + "count-5": { + "file": "count-5.png", + "two_hands": false, + "caption": "Five: all fingers spread" + }, + "turn-in-out": { + "file": "turn-in-out.png", + "two_hands": false, + "caption": "Turn your wrist: palm in, palm out" + }, + "turn-up-down": { + "file": "turn-up-down.png", + "two_hands": false, + "caption": "Palm down, turn it up, then back" + }, + "bend": { + "file": "bend.png", + "two_hands": false, + "caption": "Bend your wrist up, down, side to side" + }, + "pinch-tap": { + "file": "pinch-tap.png", + "two_hands": false, + "caption": "Tap thumb and index together" + }, + "pinch-drag": { + "file": "pinch-drag.png", + "two_hands": false, + "caption": "Pinch, move to the side, let go" + }, + "grab": { + "file": "grab.png", + "two_hands": false, + "caption": "Close your hand around a bar, open it" + }, + "open-close": { + "file": "open-close.png", + "two_hands": false, + "caption": "Open and close your hand" + }, + "cross": { + "file": "cross.png", + "two_hands": true, + "caption": "Cross your hands, then uncross" + }, + "overlap": { + "file": "overlap.png", + "two_hands": true, + "caption": "One hand over the other, slide around" + }, + "near-face": { + "file": "near-face.png", + "two_hands": true, + "caption": "Hands near your face, not touching" + }, + "screen-point": { + "file": "screen-point.png", + "two_hands": false, + "caption": "Point at a screen at arm's length" + }, + "typing": { + "file": "typing.png", + "two_hands": true, + "caption": "Type on the keyboard" + }, + "lift": { + "file": "lift.png", + "two_hands": true, + "caption": "Lift your hands off the keyboard, put them back" + }, + "mouse": { + "file": "mouse.png", + "two_hands": false, + "caption": "Hand on the mouse, move and click" + }, + "switch": { + "file": "switch.png", + "two_hands": true, + "caption": "Switch between keyboard and mouse" + }, + "hold": { + "file": "hold.png", + "two_hands": true, + "caption": "Pick it up, use it, put it down" + }, + "push": { + "file": "push.png", + "two_hands": false, + "caption": "Palms out, push out and back" + }, + "push-controller": { + "file": "push-controller.png", + "two_hands": false, + "caption": "Controller on, push out and back" + }, + "touch-stick": { + "file": "touch-stick.png", + "two_hands": true, + "caption": "Touch the thumbstick with your other index" + }, + "touch-stick-desk": { + "file": "touch-stick-desk.png", + "two_hands": true, + "caption": "Touch the thumbstick of the controller on the desk" + }, + "no-hands": { + "file": "no-hands.png", + "two_hands": false, + "caption": "Hands down, out of view" + } +} diff --git a/hands/rec/poses/push-controller.png b/hands/rec/poses/push-controller.png new file mode 100644 index 0000000..9ed1834 Binary files /dev/null and b/hands/rec/poses/push-controller.png differ diff --git a/hands/rec/poses/push.png b/hands/rec/poses/push.png new file mode 100644 index 0000000..d65d8b5 Binary files /dev/null and b/hands/rec/poses/push.png differ diff --git a/hands/rec/poses/screen-point.png b/hands/rec/poses/screen-point.png new file mode 100644 index 0000000..ca116f6 Binary files /dev/null and b/hands/rec/poses/screen-point.png differ diff --git a/hands/rec/poses/spread.png b/hands/rec/poses/spread.png new file mode 100644 index 0000000..e56012a Binary files /dev/null and b/hands/rec/poses/spread.png differ diff --git a/hands/rec/poses/switch.png b/hands/rec/poses/switch.png new file mode 100644 index 0000000..4583de5 Binary files /dev/null and b/hands/rec/poses/switch.png differ diff --git a/hands/rec/poses/thumbs-up.png b/hands/rec/poses/thumbs-up.png new file mode 100644 index 0000000..4d2cd90 Binary files /dev/null and b/hands/rec/poses/thumbs-up.png differ diff --git a/hands/rec/poses/touch-stick-desk.png b/hands/rec/poses/touch-stick-desk.png new file mode 100644 index 0000000..e57f8ce Binary files /dev/null and b/hands/rec/poses/touch-stick-desk.png differ diff --git a/hands/rec/poses/touch-stick.png b/hands/rec/poses/touch-stick.png new file mode 100644 index 0000000..c75f096 Binary files /dev/null and b/hands/rec/poses/touch-stick.png differ diff --git a/hands/rec/poses/turn-in-out.png b/hands/rec/poses/turn-in-out.png new file mode 100644 index 0000000..a6aaf0b Binary files /dev/null and b/hands/rec/poses/turn-in-out.png differ diff --git a/hands/rec/poses/turn-up-down.png b/hands/rec/poses/turn-up-down.png new file mode 100644 index 0000000..57e0759 Binary files /dev/null and b/hands/rec/poses/turn-up-down.png differ diff --git a/hands/rec/poses/typing.png b/hands/rec/poses/typing.png new file mode 100644 index 0000000..54e3220 Binary files /dev/null and b/hands/rec/poses/typing.png differ diff --git a/hands/rec/rehearse.sh b/hands/rec/rehearse.sh index 22c76b4..1233e2c 100755 --- a/hands/rec/rehearse.sh +++ b/hands/rec/rehearse.sh @@ -11,8 +11,10 @@ # drafts that also needs FT_HANDREC_ALLOW_UPLOAD=1 in the environment. # --capture DIR the recording to play (default the frame-hands capture # rec-20260930-103803-lit); --from S: start this far into it (60) -# --prompt-seconds N each test prompt's length (3): about 11 s are recorded in all, a few -# hundred MB before compression +# --prompt-seconds N each test prompt's length (3). The session runs in step mode, Next +# pressed by itself (--next-after): each prompt is a recording part of +# its own, with its 3 s countdown, so about 18 s are recorded in all, a +# few hundred MB before compression # --keep keep the temporary folders (camera images of a room: delete them after) # # It runs in the dev container (where ft-hands, ft-handpanel, zstd and huggingface_hub are), @@ -162,6 +164,7 @@ sys.exit(0 if session.Panel().cmd('ping', reply=True) else 1)" 2>/dev/null && br sleep 0.2 done run python3 "$here/session.py" --ring "$ring" --no-start --base "$base" --script "$work/script.json" --lighting room \ + --next-after 0.3 \ >(indent) sid=$(ls "$base/sessions" | tail -1) [ -n "$sid" ] || { echo "rehearse: the session left nothing" >&2; exit 1; } diff --git a/hands/rec/script.json b/hands/rec/script.json index 03d0fd4..bd1561b 100644 --- a/hands/rec/script.json +++ b/hands/rec/script.json @@ -3,7 +3,7 @@ "intro_s": 4, "between_s": 3, "welcome": {"title": "Hand recorder", "seconds": 7, - "text": "Follow the steps on this panel. Each one has a bar that shows its time left.|Pause: Space in the Hand recorder window."}, + "text": "Follow the steps on this panel. Each one shows what to do with your hands, and where. A bar shows the time left."}, "done": {"title": "All done", "seconds": 6, "text": "Thank you! You can take off the headset.|Review your recordings in the Hand recorder window."}, "stopped": {"title": "Stopped", "seconds": 4, @@ -11,16 +11,16 @@ "objects": {"pencil": "pencil or pen", "phone": "phone", "cup": "cup", "keyboard": "keyboard", "mouse": "mouse", "gamepad": "gamepad", "small": "small thing"}, "sections": [ - {"id": "hand-size", "title": "Hand size", "requires": [], + {"id": "hand-size", "title": "Hand size", "requires": [], "go": "Hold", "intro": "First, your hand size. Hold each pose still until the bar runs out.", "defaults": {"hands": "both", "distance": "mid", "position": "centre"}, "prompts": [ - {"text": "Both hands flat, palms toward you, about 40 cm away. Fingers together.", "seconds": 8, "pose": "flat"}, - {"text": "Turn them around: backs of your hands toward you.", "seconds": 6, "pose": "flat-back"}, - {"text": "Palms toward you again, fingers spread wide.", "seconds": 6, "pose": "spread"} + {"text": "Both hands flat, palms toward you, about 40 cm away. Fingers together.", "seconds": 6, "pose": "flat"}, + {"text": "Turn them around: backs of your hands toward you.", "seconds": 5, "pose": "flat-back"}, + {"text": "Palms toward you again, fingers spread wide.", "seconds": 5, "pose": "spread"} ]}, - {"id": "static-poses", "title": "Hand poses", "requires": [], + {"id": "static-poses", "title": "Hand poses", "requires": [], "go": "Hold", "intro": "Now some poses. Close means a hand's length away. Far means your arm stretched out.", "prompts": [ {"text": "Both hands: open and relaxed. Halfway out, in front of you.", "seconds": 5, "hands": "both", "pose": "open", "distance": "mid", "position": "centre"}, @@ -67,24 +67,24 @@ "intro": "Slow wrist turns. Keep your fingers relaxed.", "defaults": {"pose": "open", "distance": "mid"}, "prompts": [ - {"text": "Both hands open, palms toward you. Slowly turn them palms away, then back.", "seconds": 8, "hands": "both", "pose": "turn-in-out"}, - {"text": "Left hand, fingers up. Slowly turn your wrist: palm in, palm out.", "seconds": 7, "hands": "left", "pose": "turn-in-out"}, - {"text": "Right hand, fingers up. Slowly turn your wrist: palm in, palm out.", "seconds": 7, "hands": "right", "pose": "turn-in-out"}, - {"text": "Both hands flat, palms down. Slowly turn them palms up, then down again.", "seconds": 8, "hands": "both", "pose": "turn-up-down"}, - {"text": "Bend both wrists slowly up and down, then side to side.", "seconds": 8, "hands": "both", "pose": "bend"} + {"text": "Both hands open, palms toward you. Slowly turn them palms away, then back.", "seconds": 10, "hands": "both", "pose": "turn-in-out"}, + {"text": "Left hand, fingers up. Slowly turn your wrist: palm in, palm out.", "seconds": 9, "hands": "left", "pose": "turn-in-out"}, + {"text": "Right hand, fingers up. Slowly turn your wrist: palm in, palm out.", "seconds": 9, "hands": "right", "pose": "turn-in-out"}, + {"text": "Both hands flat, palms down. Slowly turn them palms up, then down again.", "seconds": 10, "hands": "both", "pose": "turn-up-down"}, + {"text": "Bend both wrists slowly up and down, then side to side.", "seconds": 10, "hands": "both", "pose": "bend"} ]}, {"id": "gestures", "title": "Gestures", "requires": [], "intro": "Now some movements. Go at a calm, normal pace.", "defaults": {"hands": "both"}, "prompts": [ - {"text": "Tap your thumb and index finger together a few times, both hands.", "seconds": 6, "pose": "pinch-tap"}, - {"text": "Pinch, move your hand slowly to the side, then let go. Do it again.", "seconds": 8, "pose": "pinch-drag"}, - {"text": "Close your hands as if grabbing a bar, then open them. A few times.", "seconds": 6, "pose": "grab"}, - {"text": "Cross your hands in front of you, then uncross them. Slowly.", "seconds": 8, "pose": "cross"}, - {"text": "Hold one hand over the other, palms toward you, and slide them around.", "seconds": 6, "pose": "overlap"}, - {"text": "Bring your hands close to your face and move them around it. Don't touch the headset.", "seconds": 6, "pose": "near-face", "distance": "near"}, - {"text": "Reach out to where a screen would be, about arm's length, and point at things on it.", "seconds": 6, "pose": "screen-point", "distance": "far"} + {"text": "Tap your thumb and index finger together a few times, both hands.", "seconds": 8, "pose": "pinch-tap"}, + {"text": "Pinch, move your hand slowly to the side, then let go. Do it again.", "seconds": 10, "pose": "pinch-drag"}, + {"text": "Close your hands as if grabbing a bar, then open them. A few times.", "seconds": 8, "pose": "grab"}, + {"text": "Cross your hands in front of you, then uncross them. Slowly.", "seconds": 10, "pose": "cross"}, + {"text": "Hold one hand over the other, palms toward you, and slide them around.", "seconds": 8, "pose": "overlap"}, + {"text": "Bring your hands close to your face and move them around it. Don't touch the headset.", "seconds": 8, "pose": "near-face", "distance": "near"}, + {"text": "Reach out to where a screen would be, about arm's length, and point at things on it.", "seconds": 8, "pose": "screen-point", "distance": "far"} ]}, {"id": "desk-work", "title": "Desk work", "requires": [], @@ -93,10 +93,10 @@ "prompts": [ {"when": "objects:keyboard", "text": "Type on your keyboard the way you normally do. Any text is fine.", "seconds": 10, "pose": "typing", "object": "keyboard"}, {"when": "!objects:keyboard", "text": "Rest your hands on the desk and pretend to type.", "seconds": 10, "pose": "typing"}, - {"when": "objects:keyboard", "text": "Lift your hands off the keyboard and put them back, a few times.", "seconds": 6, "pose": "lift", "object": "keyboard"}, + {"when": "objects:keyboard", "text": "Lift your hands off the keyboard and put them back, a few times.", "seconds": 8, "pose": "lift", "object": "keyboard"}, {"when": "objects:mouse", "text": "Use your mouse: move it around and click a few times.", "seconds": 10, "hands": "right", "pose": "mouse", "object": "mouse"}, - {"when": "!objects:mouse", "text": "Put one hand flat on the desk and slide it around, as if using a mouse.", "seconds": 8, "hands": "right", "pose": "mouse"}, - {"when": "objects:keyboard", "text": "Switch between typing and the mouse a few times.", "seconds": 8, "pose": "switch", "object": "keyboard"} + {"when": "!objects:mouse", "text": "Put one hand flat on the desk and slide it around, as if using a mouse.", "seconds": 10, "hands": "right", "pose": "mouse"}, + {"when": "objects:keyboard", "text": "Switch between typing and the mouse a few times.", "seconds": 10, "pose": "switch", "object": "keyboard"} ]}, {"id": "objects", "title": "Things you hold", "requires": ["objects"], "for_each": "object", @@ -117,9 +117,9 @@ [0.25, -0.05, -0.50], [0.05, -0.10, -0.55]]}, {"id": "controller-push", "title": "Depth with controllers", "kind": "bar", "requires": ["controllers"], - "before": {"text": "Put on both controllers and tighten the straps.|Take your time: Space pauses.", "seconds": 25}, + "before": {"text": "Put on both controllers and tighten the straps. Take your time.", "seconds": 25}, "intro": "Follow the marker on the bar with both hands: out as it moves right, back as it moves left.", - "defaults": {"hands": "both", "pose": "open", "controller": true, "controllers": ["left", "right"]}, + "defaults": {"hands": "both", "pose": "open", "picture": "push-controller", "controller": true, "controllers": ["left", "right"]}, "near_m": 0.2, "far_m": 0.6, "period_s": 6, "lead_s": 3, "reps": 5, "heights": [ {"id": "chest", "text": "Chest height: hands open, palms out like a stop sign. Push out and back with the marker."}, @@ -129,8 +129,8 @@ {"id": "right", "text": "Now off to your right side. Push out and back with the marker.", "reps": 2} ], "prompts": [ - {"text": "Slowly turn your wrists: palms out, palms toward you, and back.", "seconds": 8, "pose": "turn-in-out", "distance": "mid"}, - {"text": "Open and close your hands a few times.", "seconds": 6, "pose": "open-close", "distance": "mid"} + {"text": "Slowly turn your wrists: palms out, palms toward you, and back.", "seconds": 10, "pose": "turn-in-out", "distance": "mid"}, + {"text": "Open and close your hands a few times.", "seconds": 8, "pose": "open-close", "distance": "mid"} ]}, {"id": "bridge", "title": "Controller and fingertip", "requires": ["controllers"], @@ -142,20 +142,20 @@ {"text": "Keep touching it while you move your left hand halfway out.", "seconds": 6, "hands": "both", "pose": "touch-stick", "distance": "mid", "controllers": ["left"]}, {"text": "Keep touching it with your left arm stretched out.", "seconds": 6, "hands": "both", "pose": "touch-stick", "distance": "far", "controllers": ["left"]}, {"text": "Swap: put the right controller on, then take the left one off.", "seconds": 12, "hands": "both"}, - {"text": "Right hand close to you. Touch the right thumbstick with your left index fingertip.", "seconds": 6, "hands": "both", "pose": "touch-stick", "distance": "near", "controllers": ["right"]}, - {"text": "Keep touching it while you move your right hand halfway out.", "seconds": 6, "hands": "both", "pose": "touch-stick", "distance": "mid", "controllers": ["right"]}, - {"text": "Keep touching it with your right arm stretched out.", "seconds": 6, "hands": "both", "pose": "touch-stick", "distance": "far", "controllers": ["right"]}, + {"text": "Right hand close to you. Touch the right thumbstick with your left index fingertip.", "seconds": 6, "hands": "both", "pose": "touch-stick", "picture_mirror": true, "distance": "near", "controllers": ["right"]}, + {"text": "Keep touching it while you move your right hand halfway out.", "seconds": 6, "hands": "both", "pose": "touch-stick", "picture_mirror": true, "distance": "mid", "controllers": ["right"]}, + {"text": "Keep touching it with your right arm stretched out.", "seconds": 6, "hands": "both", "pose": "touch-stick", "picture_mirror": true, "distance": "far", "controllers": ["right"]}, {"text": "Take off the right controller and put it on the desk in front of you.", "seconds": 8, "hands": "both"}, - {"text": "Touch its thumbstick with your index fingertip, from the front.", "seconds": 5, "hands": "any", "pose": "touch-stick", "position": "down"}, - {"text": "Now touch the thumbstick reaching from the left.", "seconds": 5, "hands": "any", "pose": "touch-stick", "position": "left"}, - {"text": "Now from the right.", "seconds": 5, "hands": "any", "pose": "touch-stick", "position": "right"}, - {"text": "Now from straight above, with your other hand.", "seconds": 5, "hands": "any", "pose": "touch-stick", "position": "up"} + {"text": "Touch its thumbstick with your index fingertip, from the front.", "seconds": 5, "hands": "any", "pose": "touch-stick", "picture": "touch-stick-desk", "position": "down"}, + {"text": "Now touch the thumbstick reaching from the left.", "seconds": 5, "hands": "any", "pose": "touch-stick", "picture": "touch-stick-desk", "position": "left"}, + {"text": "Now from the right.", "seconds": 5, "hands": "any", "pose": "touch-stick", "picture": "touch-stick-desk", "position": "right"}, + {"text": "Now from straight above, with your other hand.", "seconds": 5, "hands": "any", "pose": "touch-stick", "picture": "touch-stick-desk", "position": "up"} ]}, {"id": "bare-push", "title": "Depth, bare hands", "kind": "bar", "requires": [], "before": {"when": "controllers", "text": "Take the controllers off and put them out of view.", "seconds": 15}, "intro": "Follow the marker on the bar with both hands: out as it moves right, back as it moves left.", - "defaults": {"hands": "both", "pose": "open"}, + "defaults": {"hands": "both", "pose": "open", "picture": "push"}, "near_m": 0.2, "far_m": 0.6, "period_s": 6, "lead_s": 3, "reps": 4, "heights": [ {"id": "chest", "text": "Chest height: hands open, palms out like a stop sign. Push out and back with the marker."}, @@ -165,14 +165,14 @@ {"id": "right", "text": "Now off to your right side. Push out and back with the marker.", "reps": 2} ], "prompts": [ - {"text": "Slowly turn your wrists: palms out, palms toward you, and back.", "seconds": 8, "pose": "turn-in-out", "distance": "mid"}, - {"text": "Open and close your hands a few times.", "seconds": 6, "pose": "open-close", "distance": "mid"} + {"text": "Slowly turn your wrists: palms out, palms toward you, and back.", "seconds": 10, "pose": "turn-in-out", "distance": "mid"}, + {"text": "Open and close your hands a few times.", "seconds": 8, "pose": "open-close", "distance": "mid"} ]}, {"id": "no-hands", "title": "No hands", "requires": [], "intro": "Last one: put your hands down, out of view.", "prompts": [ - {"text": "Keep your hands out of view and look slowly around the room.", "seconds": 10, "hands": "none"} + {"text": "Keep your hands out of view and look slowly around the room.", "seconds": 10, "hands": "none", "picture": "no-hands"} ]} ] } diff --git a/hands/rec/session.py b/hands/rec/session.py index 3ea4aae..d1e296c 100755 --- a/hands/rec/session.py +++ b/hands/rec/session.py @@ -11,8 +11,12 @@ gives feedback ("I can't see your left hand"). Plain Python, standard library only: ft_handrec.py imports it, and it runs from the command line for testing (in the dev container): - python3 hands/rec/session.py --dry-run --speed 20 # no processes: prints the panel commands - python3 hands/rec/session.py --ring /tmp/ring --base /tmp/hr # ft-ringplay's frames, no headset needed + python3 hands/rec/session.py --dry-run --speed 20 --next-after 0.2 # no processes: prints the panel commands + python3 hands/rec/session.py --ring /tmp/ring --base /tmp/hr # ft-ringplay's frames, no headset needed + +Step mode (the default) shows each step and waits for Next (Space in the window, n or Enter +here); a 3-2-1 countdown, recorded, then the hold. Only the countdowns and holds are recorded: +each is a part of the take's recording (sets-N.bin). --auto is the old timed flow. All _ns times are CLOCK_MONOTONIC nanoseconds. """ @@ -40,6 +44,7 @@ FT_HANDS = os.path.join(HANDS, "build", "ft-hands") FT_CAMD = os.path.join(HANDS, "build", "ft-camd") PANEL_BIN = os.path.join(HERE, "build", "ft-handpanel") SCRIPT_PATH = os.path.join(HERE, "script.json") +POSES_DIR = os.path.join(HERE, "poses") # the pose pictures: poses.json and its PNGs BASE_DIR = os.path.expanduser("~/.local/share/frametop/hands/contrib") PANEL_SOCKET = "ft_handpanel" CAMD_UNIT = "frametop-handrec-camd.service" @@ -52,7 +57,12 @@ LOST_S = 1.5 # an asked-for hand lost this long gets a note CONTROLLER_LOST_S = 1.0 # a controller off 200 (Running_OK) this long gets a note TOUCH_M = 0.03 # touch the dot: the index tip within this of the dot MIN_FREE = 1.5e9 # stop the session before the disk fills -RESUME_HINT = "Paused. Resume: Space in the Hand recorder window" +COUNTDOWN_S = 3 # step mode: the 3-2-1 before each step, recorded +FIRST_SET_S = 3.0 # step mode: how long the hold may wait for its recording's first set +RESUME_HINT = "Paused. Resume: P in the Hand recorder window" +READY_TEXT = "Ready? Press Space or click Next" +KEYS_STEP = "Hand recorder window: Space next \u00b7 P pause \u00b7 R redo \u00b7 S skip section \u00b7 Esc stop" +KEYS_AUTO = "Hand recorder window: P pause \u00b7 R redo \u00b7 S skip section \u00b7 Esc stop" def mono_ns(): @@ -470,6 +480,13 @@ class Recorder: def exited(self): return self.proc.poll() is not None + def has_data(self): + """It has written a set (ft-hands starts recording within a few tens of milliseconds).""" + try: + return os.path.getsize(os.path.join(self.dir, "sets.bin")) > 0 + except OSError: + return False + def stop(self): """End it (SIGTERM: ft-hands writes out its queue) and put the part in place.""" if self.proc.poll() is None: @@ -490,6 +507,40 @@ class Recorder: return self.proc.returncode +# ------------------------------------------------------------------------------------------ +# The pose pictures (poses/poses.json: {"": {"file": ".png", "two_hands", "caption"}}) + +def load_poses(poses_dir): + """{pose id: {"path", "two_hands", "caption"}} from poses_dir/poses.json, or {} without one.""" + try: + with open(os.path.join(poses_dir, "poses.json")) as f: + raw = json.load(f) + except (OSError, ValueError): + return {} + out = {} + for pose, e in (raw.items() if isinstance(raw, dict) else ()): + if isinstance(e, dict) and isinstance(e.get("file"), str) and e["file"]: + out[pose] = {"path": os.path.join(poses_dir, os.path.basename(e["file"])), + "two_hands": bool(e.get("two_hands")), "caption": clean_text(e.get("caption", ""))} + return out + + +def pose_view(poses, p): + """A prompt's picture: (path, mode, caption), mode "" (as drawn: a right hand, or both hands), + "mirror" (a left hand) or "both" (a mirrored copy on the left, the picture on the right). + The prompt's "picture" (a pose id) picks a picture other than its pose's, and + "picture_mirror" mirrors a two-hand picture. ("", "", "") when there's none.""" + e = poses.get(p.get("picture") or p.get("pose") or "") + if not e or not os.path.isfile(e["path"]): + return "", "", "" + hands = p.get("hands") + if e["two_hands"]: + mode = "mirror" if p.get("picture_mirror") else "" + else: + mode = "mirror" if hands == "left" else "both" if hands == "both" else "" + return e["path"], mode, e["caption"] + + # ------------------------------------------------------------------------------------------ # The script @@ -595,17 +646,30 @@ def build_plan(script, checklist): return plan, skipped -def section_seconds(s, worst=False): - """A section's length in script seconds (targets: about 4 s each, or the timeout if worst).""" - t = s["intro_s"] + sum(p["seconds"] for p in s["prompts"]) +def section_steps(s): + """The steps that wait for Next in step mode: each prompt and bar height, and the first target.""" + return (len(s["prompts"]) + (len(s.get("heights") or []) if s["kind"] == "bar" else 0) + + (1 if s["kind"] == "targets" and s.get("targets") else 0)) + + +def section_seconds(s, worst=False, auto=True): + """A section's length in script seconds (targets: about 4 s each, or the timeout if worst). + In step mode, what's recorded: a countdown before each step and the holds; the intro and + the waits for Next aren't.""" + t = (s["intro_s"] if auto else COUNTDOWN_S * section_steps(s)) + sum(p["seconds"] for p in s["prompts"]) if s["kind"] == "targets": t += len(s["targets"]) * (s.get("timeout_s", 8) if worst else min(4.0, s.get("timeout_s", 8))) if s["kind"] == "bar": - t += sum(s.get("lead_s", 3) + h["reps"] * s.get("period_s", 6) for h in s["heights"]) + lead = s.get("lead_s", 3) if auto else 0 + t += sum(lead + h["reps"] * s.get("period_s", 6) for h in s["heights"]) return t -def plan_seconds(script, plan, worst=False): +def plan_seconds(script, plan, worst=False, auto=True): + """The session's length: in auto mode all of it; in step mode only what's recorded, as the + time spent reading each step before pressing Next is up to the person (plan_steps).""" + if not auto: + return sum(section_seconds(s, worst, auto=False) for s in plan) t = (script.get("welcome") or {}).get("seconds", 0) + (script.get("done") or {}).get("seconds", 0) for i, s in enumerate(plan): t += section_seconds(s, worst) @@ -613,6 +677,20 @@ def plan_seconds(script, plan, worst=False): return t +def plan_steps(plan): + return sum(section_steps(s) for s in plan) + + +def plan_summary(script, plan, auto=False): + """The length in words, for the window and --plan.""" + minutes = max(1, round(plan_seconds(script, plan, auto=auto) / 60)) + if auto: + return "about %d min, each step advancing by itself" % minutes + n = plan_steps(plan) + return ("about %d min of recording in %d steps, plus the time you take to read each step before " + "pressing Next (at 5 s a step, about %d min more)" % (minutes, n, max(1, round(n * 5 / 60)))) + + # ------------------------------------------------------------------------------------------ # The session @@ -628,6 +706,10 @@ class _Fail(Exception): pass +class _Redo(Exception): + pass + + def _git_describe(): try: r = subprocess.run(["git", "-C", REPO, "describe", "--always", "--dirty", "--tags"], @@ -665,12 +747,18 @@ def _steamvr_version(): class Session: - """One recording session. start() runs it in its own thread; pause(), resume(), skip() - and stop() steer it from any thread. on_status(dict) is called from the session thread - whenever something changes (see STATUS_KEYS).""" + """One recording session. start() runs it in its own thread; next_step(), pause(), resume(), + redo(), skip() and stop() steer it from any thread. on_status(dict) is called from the + session thread whenever something changes (the keys of _status). + + auto=False is step mode: each step waits for next_step(), then a recorded countdown, then + the hold; nothing is recorded while it waits. auto=True: the timed flow, each prompt + advancing by itself, one recording per take. next_after (a test hook): press Next by itself + after that many seconds of waiting. poses_dir: the pose pictures (poses/).""" def __init__(self, base_dir, profile, checklist, lighting_choice, script_path, *, ring=None, - start_processes=True, dry_run=False, speed=1.0, on_status=None, hands_dir=None, panel_bin=None): + start_processes=True, dry_run=False, speed=1.0, on_status=None, hands_dir=None, panel_bin=None, + auto=False, next_after=None, poses_dir=None): self.base_dir = os.path.abspath(os.path.expanduser(base_dir or BASE_DIR)) self.profile = dict(profile or {}) self.checklist = dict(checklist or {}) @@ -683,17 +771,26 @@ class Session: self.on_status = on_status self.hands_dir = hands_dir or run_dir() self.panel_bin = panel_bin or PANEL_BIN + self.auto = bool(auto) + self.next_after = next_after self.print = print # where dry-run panel commands go (the CLI's stdout) self.script = load_script(self.script_path) self.plan, self.skipped = build_plan(self.script, self.checklist) + self._poses = load_poses(poses_dir or POSES_DIR) self.session_dir = "" self._thread = None self._lock = threading.Lock() - self._want = {"pause": False, "skip": False, "stop": False} + self._want = {"pause": False, "skip": False, "stop": False, "next": False, "redo": False} self._wake = threading.Event() - self._status = {"state": "starting", "section": "", "title": "", "section_index": 0, - "section_count": len(self.plan), "prompt": "", "seconds_left": 0.0, "note": "", - "hands": {"left": None, "right": None}, "take": None, "error": ""} + # state: starting, intro, ready (a step waits for Next), countdown, running (a hold), + # between, paused, done, stopped, error. waiting: Next is wanted. big: the countdown's + # number, then the hold's word ("Hold", "Go"), as the panel shows them. The picture and the + # diagram: image (a path or ""), image_mode ("", "mirror", "both"), position, distance. + self._status = {"state": "starting", "mode": "auto" if self.auto else "step", "section": "", "title": "", + "section_index": 0, "section_count": len(self.plan), "step_index": 0, "step_count": 0, + "prompt": "", "seconds_left": 0.0, "note": "", "hands": {"left": None, "right": None}, + "take": None, "error": "", "waiting": False, "countdown": 0, "big": "", "can_redo": False, + "image": "", "image_mode": "", "caption": "", "position": "", "distance": ""} self._last_emit = 0.0 self._log_file = None self._panel = None @@ -706,6 +803,11 @@ class Session: self._live = None # the hands file's last read self._hands_file = HandsFile(os.path.join(self.hands_dir, "hands")) self._paused = False + self._recording = False # a recording part is running + self._paused_recording = False + self._waiting = False # waiting for Next: no notes about lost hands + self._redo_ok = False # R does something now + self._step_t0 = None # the step's first event (ready or prompt), for R self._fb = {} # feedback timers # --- controls (any thread) @@ -720,6 +822,14 @@ class Session: self._want[key] = value self._wake.set() + def next_step(self): + """Step mode: start the step that's waiting (its countdown).""" + self._set("next", True) + + def redo(self): + """Record a step again: the one running, or at a step's ready screen the one before.""" + self._set("redo", True) + def pause(self): self._set("pause", True) @@ -763,12 +873,15 @@ class Session: self._log_file.flush() def _event(self, event, **fields): + """A line in the take's prompts.jsonl; returns its time.""" + t = mono_ns() if not self._take: - return - line = {"t": mono_ns(), "event": event} + return t + line = {"t": t, "event": event} line.update(fields) self._take["prompts"].write(json.dumps(line) + "\n") self._take["prompts"].flush() + return t # --- the run def _run(self): @@ -838,7 +951,7 @@ class Session: "calibration_removed": removed, "script": {"version": self.script.get("version"), "sections": [s["id"] for s in self.plan], "skipped": self.skipped}, - "takes": [], "status": "recording"} + "mode": "auto" if self.auto else "step", "takes": [], "status": "recording"} if self.dry_run: self._session_json["dry_run"] = True if self.speed != 1: @@ -850,7 +963,9 @@ class Session: self._panel.cmd("show") self._panel.cmd("paused off") for key, c in (("note", "note "), ("countdown", "countdown off"), ("hands", "hands off off"), - ("bar", "bar off"), ("target", "target off")): + ("bar", "bar off"), ("target", "target off"), ("image", "image off"), ("where", "where off"), + ("big", "big "), ("action", "action "), ("rec", "rec off"), + ("keys", "keys " + (KEYS_AUTO if self.auto else KEYS_STEP))): self._panel.set(key, c) def _write_calibration(self): @@ -1007,6 +1122,9 @@ class Session: "seconds": 4}}[state] try: self._panel.cmd("paused off") + for key, c in (("big", "big "), ("action", "action "), ("rec", "rec off"), ("keys", "keys "), + ("bar", "bar off"), ("target", "target off")): + self._panel.set(key, c) self._screen(state, screen, controls=False) except (_Stop, _Skip, _Fail): pass @@ -1016,18 +1134,23 @@ class Session: self._log_file = None screen = {"done": self.script.get("done"), "stopped": self.script.get("stopped")}.get(state) or {} self._emit(state=state, error=error, seconds_left=0.0, note="", take=None, section="", - prompt=error or screen.get("text", "").replace("|", "\n"), hands={"left": None, "right": None}) + prompt=error or screen.get("text", "").replace("|", "\n"), hands={"left": None, "right": None}, + waiting=False, countdown=0, big="", can_redo=False, image="", image_mode="", caption="", + position="", distance="") # --- the timing loop def _controls(self): - """Handle a pause (blocking until resumed), skip and stop. True if a pause happened.""" + """Handle a pause (blocking until resumed), skip, stop and redo (only where _redo_ok: + else it's dropped). True if a pause happened.""" with self._lock: want = dict(self._want) - self._want["skip"] = False + self._want["skip"] = self._want["redo"] = False if want["stop"]: raise _Stop() if want["skip"]: raise _Skip() + if want["redo"] and self._redo_ok: + raise _Redo() if not want["pause"]: return False self._pause() @@ -1037,13 +1160,16 @@ class Session: self._wake.clear() with self._lock: want = dict(self._want) - self._want["skip"] = False - if want["skip"]: + self._want["skip"] = self._want["redo"] = False + redo = want["redo"] and self._redo_ok + if want["skip"] or redo: self._want["pause"] = False if want["stop"]: outcome = _Stop elif want["skip"]: outcome = _Skip + elif redo: + outcome = _Redo elif not want["pause"]: outcome = True self._unpause(record=outcome is True) @@ -1054,7 +1180,8 @@ class Session: def _pause(self): self._paused = True self._state_before = self._status["state"] - if self._take: + self._paused_recording = self._recording + if self._recording: self._stop_recording() self._event("pause") self._panel.cmd("paused on") @@ -1067,7 +1194,9 @@ class Session: self._panel.cmd("paused off") self._panel.set("note", "note ") self._fb.clear() - if self._take and record: + with self._lock: + self._want["next"] = False # a Next pressed while paused doesn't count + if self._take and self._paused_recording and record: self._event("resume") self._start_recording() self._emit(state=self._state_before, note="") @@ -1115,6 +1244,7 @@ class Session: self._live = live p = self._prompt or {} asked = p.get("hands", "") + # While a step waits for Next the chips show what's seen, with no notes yet. seen = {s: (bool(live[s]) if live else None) for s in ("left", "right")} # the chips: the asked-for hands, seen or lost, while the tracker publishes chips = [] @@ -1124,7 +1254,7 @@ class Session: self._panel.set("hands", "hands %s %s" % tuple(chips)) # a note when an asked-for hand stays lost, or a hand shows when none is wanted notes = [] - if live is not None and asked: + if live is not None and asked and not self._waiting: if asked == "none": missing = [] if not (seen["left"] or seen["right"]) else ["shown"] elif asked == "any": @@ -1143,14 +1273,14 @@ class Session: notes.append("I can't see your %s hand: bring it into view" % missing[0]) else: fb.pop("lost_key", None) - notes = self._controller_feedback(now, p) + notes + notes = ([] if self._waiting else self._controller_feedback(now, p)) + notes note = notes[0] if notes else "" if not self._paused: self._panel.set("note", "note " + note) hands = {"left": seen["left"], "right": seen["right"]} if note != self._status["note"] or hands != self._status["hands"]: self._emit(note=note, hands=hands) - if self._take and live is not None and now - fb.get("logged", 0) >= 0.5: + if self._take and self._recording and live is not None and now - fb.get("logged", 0) >= 0.5: fb["logged"] = now self._event("feedback", left=seen["left"], right=seen["right"], palm_m=[round(live[s]["palm_m"], 4) if live[s] else None for s in ("left", "right")]) @@ -1199,19 +1329,58 @@ class Session: if text is not None: self._panel.set("text", "text " + clean_text(text)) + def _view(self, p=None): + """The prompt's picture and where-to diagram, on the panel and in the status (sent with + the next _emit); none without a prompt.""" + p = p or {} + path, mode, caption = pose_view(self._poses, p) + pos, dist = p.get("position") or "", p.get("distance") or "" + self._panel.set("image", "image %s%s" % (path, " " + mode if mode else "") if path else "image off") + self._panel.set("where", "where %s %s" % (clean_text(pos) or "-", clean_text(dist) or "-") + if pos or dist else "where off") + self._status.update(image=path, image_mode=mode, caption=caption, position=pos, distance=dist) + + def _await_next(self): + """Step mode: wait for Next (or the next_after test hook), handling the controls and the + hands chips. Nothing records meanwhile.""" + with self._lock: + self._want["next"] = False # one pressed during the hold doesn't skip this + self._waiting = True + self._panel.set("action", "action " + READY_TEXT) + self._emit(waiting=True, seconds_left=0.0) + t0 = time.monotonic() + try: + while True: + self._controls() + self._feedback() + with self._lock: + go, self._want["next"] = self._want["next"], False + if go or (self.next_after is not None and time.monotonic() - t0 >= self.next_after): + break + self._wake.wait(TICK_S) + self._wake.clear() + finally: + self._waiting = False + self._panel.set("action", "action ") + self._emit(waiting=False) + def _screen(self, state, screen, controls=True): - """A screen of its own (welcome, done): title, text, a few seconds. controls=False: the - session's end, which reports its state once all is done.""" + """A screen of its own (welcome, done): title, text, a few seconds, or in step mode + until Next. controls=False: the session's end, which reports its state once all is done.""" if not screen: return self._prompt = None self._show(screen.get("title", ""), "", screen.get("text", "")) + self._view() self._panel.set("countdown", "countdown off") if controls: self._emit(state=state, title=screen.get("title", ""), prompt=screen.get("text", "").replace("|", "\n"), section="", seconds_left=float(screen.get("seconds", 0))) try: - self._wait(screen.get("seconds", 0), countdown=False) + if self.auto: + self._wait(screen.get("seconds", 0), countdown=False) + else: + self._await_next() except _Skip: pass else: @@ -1221,29 +1390,36 @@ class Session: def _section(self, i, s): step = "Section %d of %d" % (i + 1, len(self.plan)) - status = {"section": s["id"], "title": s["title"], "section_index": i + 1, "section_count": len(self.plan)} + status = {"section": s["id"], "title": s["title"], "section_index": i + 1, "section_count": len(self.plan), + "step_index": 0, "step_count": section_steps(s) if not self.auto else 0, "can_redo": False} try: before = s.get("before") - if before or i > 0: - self._prompt = None - text = before["text"] if before else "Next: %s" % s["title"] - secs = before.get("seconds", 10) if before else self.script.get("between_s", 3) - self._show("Get ready" if before else s["title"], step, text) - self._emit(state="between", prompt=text.replace("|", "\n"), take=None, **status) - self._wait(secs) - self._start_take(i, s) - self._status.update(state="intro", take=self._take["id"], **status) # sent with the intro - self._run_prompt(s, {"id": s["id"] + "/intro", "text": s.get("intro", ""), "seconds": s["intro_s"], - "hands": "", "pose": "", "distance": "", "position": "", "object": "", - "controller": False, "controllers": []}, step, intro=True) + self._prompt = None + self._view() + if self.auto: + if before or i > 0: + text = before["text"] if before else "Next: %s" % s["title"] + secs = before.get("seconds", 10) if before else self.script.get("between_s", 3) + self._show("Get ready" if before else s["title"], step, text) + self._emit(state="between", prompt=text.replace("|", "\n"), take=None, **status) + self._wait(secs) + self._start_take(i, s) + self._status.update(state="intro", take=self._take["id"], **status) # sent with the intro + self._run_prompt(s, {"id": s["id"] + "/intro", "text": s.get("intro", ""), "seconds": s["intro_s"], + "hands": "", "pose": "", "distance": "", "position": "", "object": "", + "controller": False, "controllers": []}, step, intro=True) + else: + # One screen before the section: what to get ready, and the intro. The take + # starts with the first step's countdown, so a section skipped here leaves none. + text = "|".join(t for t in ((before or {}).get("text", ""), s.get("intro", "")) if t) + self._show(s["title"], step, text) + self._panel.set("countdown", "countdown off") + self._emit(state="intro", prompt=text.replace("|", "\n"), take=None, seconds_left=0.0, **status) + self._await_next() self._status["state"] = "running" # sent with the first prompt - if s["kind"] == "targets": - self._targets(s, step) - elif s["kind"] == "bar": - self._bar(s, step) - for p in s["prompts"]: - self._run_prompt(s, p, step) - self._end_take("complete") + self._steps(i, s, step) + if self._take: + self._end_take("complete") except _Skip: self._log("skipped %s" % s["id"]) if self._take: @@ -1251,10 +1427,12 @@ class Session: except _Stop: raise finally: - self._panel.set("bar", "bar off") - self._panel.set("target", "target off") + self._redo_ok = False + self._status["big"] = "" + for key, c in (("bar", "bar off"), ("target", "target off"), ("big", "big "), ("action", "action ")): + self._panel.set(key, c) - def _start_take(self, i, s): + def _start_take(self, i, s, record=True): n = len(self._session_json["takes"]) + 1 take_id = "%02d-%s" % (n, s["id"]) d = os.path.join(self.session_dir, "takes", take_id) @@ -1267,19 +1445,27 @@ class Session: self._session_json["takes"].append(take_id) self._save_session() self._event("take", section=s["id"], take=take_id) - self._start_recording() + if record: + self._start_recording() self._log("take %s" % take_id) + self._emit(force=False, take=take_id) def _start_recording(self): t = self._take t["part"] += 1 if not self.dry_run: # a safety net only: the session ends the recording itself - remaining = section_seconds(t["section"], worst=True) / self.speed + remaining = section_seconds(t["section"], worst=True, auto=self.auto) / self.speed self._recorder = Recorder(t["dir"], t["part"], remaining * 1.5 + 60, self.ring, self._log_file) + self._recording = True self._panel.cmd("poses start " + os.path.join(t["dir"], "poses.jsonl")) + self._panel.set("rec", "rec on") def _stop_recording(self): + if not self._recording: + return + self._recording = False + self._panel.set("rec", "rec off") self._panel.cmd("poses stop", reply=not self.dry_run, timeout=1.0) if self._recorder: rec, self._recorder = self._recorder, None @@ -1289,8 +1475,7 @@ class Session: def _end_take(self, status): t = self._take try: - if not self._paused: - self._stop_recording() + self._stop_recording() finally: self._event("end", status=status) self._take = None @@ -1301,67 +1486,216 @@ class Session: self._log("take %s %s" % (t["id"], status)) def _prompt_event(self, p): - self._event("prompt", id=p["id"], text=p["text"], hands=p["hands"], pose=p["pose"], distance=p["distance"], - position=p["position"], object=p["object"], controller=bool(p["controller"]), - **({"controllers": p["controllers"]} if p["controllers"] else {})) + return self._event("prompt", id=p["id"], text=p["text"], hands=p["hands"], pose=p["pose"], + distance=p["distance"], position=p["position"], object=p["object"], + controller=bool(p["controller"]), **({"controllers": p["controllers"]} if p["controllers"] else {})) - def _begin_prompt(self, s, p, step): + def _begin_prompt(self, s, p, step, state="running"): self._prompt = p self._fb.pop("lost_key", None) self._show(s["title"], step, p["text"]) - self._prompt_event(p) - self._emit(prompt=p["text"].replace("|", "\n"), seconds_left=float(p["seconds"])) + self._view(p) + t = self._prompt_event(p) + if self._step_t0 is None: + self._step_t0 = t + self._emit(state=state, prompt=p["text"].replace("|", "\n"), seconds_left=float(p["seconds"]), countdown=0) self._log(" %s" % p["id"]) def _run_prompt(self, s, p, step, intro=False): if intro and not p["text"]: return - self._begin_prompt(s, p, step) + self._begin_prompt(s, p, step, state="intro" if intro else "running") self._wait(p["seconds"]) - def _targets(self, s, step): - hold_s, timeout_s = float(s.get("hold_s", 1.0)), float(s.get("timeout_s", 8)) + # --- steps: a section's prompts, bar heights and targets + def _step_list(self, s): + """[{"kind": "prompt"|"bar"|"target", "p": prompt, "ready": waits for Next in step mode, ...}].""" + out = [] defaults = s.get("defaults") or {} - for k, pt in enumerate(s["targets"]): - p = {"id": "%s/%d" % (s["id"], k + 1), "text": s.get("text", ""), "seconds": timeout_s, - "hands": defaults.get("hands", "any"), "pose": defaults.get("pose", "point"), "distance": "", - "position": "", "object": "", "controller": False, "controllers": []} - self._begin_prompt(s, p, step) - xyz = "%.3f %.3f %.3f" % tuple(pt) - reply = self._panel.cmd("target %s show" % xyz, reply=True) or "" - self._panel.sent["target"] = "target %s show" % xyz - tok = reply.split() - room = [float(v) for v in tok[1:4]] if tok[:1] == ["ok"] and len(tok) >= 4 else None - target = {"id": p["id"], "head": list(pt), "room": room} - self._event("target", state="show", **target) - st = {"hold": 0.0, "sent": 0.0, "holding": False} + if s["kind"] == "targets": + for k, pt in enumerate(s["targets"]): + p = {"id": "%s/%d" % (s["id"], k + 1), "text": s.get("text", ""), "seconds": float(s.get("timeout_s", 8)), + "hands": defaults.get("hands", "any"), "pose": defaults.get("pose", "point"), "distance": "", + "position": "", "object": "", "controller": False, "controllers": []} + out.append({"kind": "target", "p": p, "pt": pt, "ready": k == 0}) + elif s["kind"] == "bar": + lead = float(s.get("lead_s", 3)) if self.auto else 0.0 # step mode: the countdown shows the bar at near + for h in s["heights"]: + p = {"id": "%s/%s" % (s["id"], h["id"]), "text": h["text"], "hands": defaults.get("hands", "both"), + "pose": defaults.get("pose", "open"), "distance": "", "position": h["id"], "object": "", + "picture": h.get("picture", defaults.get("picture", "")), + "controller": bool(defaults.get("controller", False)), + "controllers": list(defaults.get("controllers") or []), + "seconds": lead + h["reps"] * float(s.get("period_s", 6))} + out.append({"kind": "bar", "p": p, "h": h, "lead": lead, "ready": True}) + out += [{"kind": "prompt", "p": p, "ready": True} for p in s["prompts"]] + return out - def tick(dt, pt=pt, room=room, target=target, st=st, xyz=xyz): - d = self._tip_distance(pt, room) - if d is not None and d <= TOUCH_M: - if not st["holding"]: - st["holding"] = True - self._event("target", state="hold", **target) - st["hold"] += dt - elif st["holding"]: - st["holding"], st["hold"] = False, 0.0 - self._panel.set("target", "target %s show" % xyz) - if st["holding"]: - frac = min(1.0, st["hold"] / hold_s) - if time.monotonic() - st["sent"] >= 0.1 or frac >= 1: - st["sent"] = time.monotonic() - self._panel.set("target", "target %s hold %.2f" % (xyz, frac)) - return st["hold"] >= hold_s + def _steps(self, i, s, label): + """Run a section's steps. Step mode: each that's "ready" (and each to do again) shows + first and waits for Next; then the recorded countdown; the recording stops after a + step unless the next one follows straight on (the targets after the first). R: the + step running starts again; at a ready screen, the step before goes again. Either way + the range done before is marked with a "redo" event.""" + steps = self._step_list(s) + done = [] # the steps finished in this take: (index, id, from_ns, to_ns), for R + again = set() # steps to do again: they wait for Next too + k = 0 + while k < len(steps): + st, p = steps[k], steps[k]["p"] + where = label if self.auto else "%s · step %d of %d" % (label, k + 1, len(steps)) + self._status.update(step_index=k + 1, step_count=len(steps)) + started, self._step_t0 = None, None + try: + if not self.auto and (st["ready"] or k in again): + self._redo_ok = bool(done) + self._ready(s, st, where, can_redo=bool(done)) + self._await_next() + self._redo_ok = True + started = self._countdown(i, s, st) + else: + self._redo_ok = True + self._emit(force=False, can_redo=True) + self._run_step(s, st, where) + done.append((k, p["id"], started if started is not None else self._step_t0, mono_ns())) + if not self.auto and (k + 1 == len(steps) or steps[k + 1]["ready"] or k + 1 in again): + self._hold_end() + again.discard(k) + k += 1 + except _Redo: + self._redo_ok = False + from_ns = started if started is not None else self._step_t0 + self._panel.set("target", "target off") + self._panel.set("bar", "bar off") + if from_ns is not None: # R during the step: it starts again + self._event("redo", id=p["id"], **{"from": from_ns, "to": mono_ns()}) + self._log(" redo %s" % p["id"]) + if not self.auto: + self._hold_end() + elif self._take and not self._recording: # R ended a pause: record again + self._event("resume") + self._start_recording() + again.add(k) + elif done: # R at its ready screen: the step before goes again + k, pid, a, b = done.pop() + self._event("redo", id=pid, **{"from": a, "to": b}) + self._log(" redo %s" % pid) + again.add(k) - done = self._wait(timeout_s, tick) - state = "done" if done else "timeout" - if done: - self._panel.set("target", "target %s done" % xyz) - self._event("target", state=state, **target) - self._log(" %s %s" % (p["id"], state)) - if done: - self._wait(0.6, countdown=False) - self._panel.set("target", "target off") + def _ready(self, s, st, where, can_redo): + """Step mode: show the step (text, picture, diagram) with "Ready?".""" + p = st["p"] + self._prompt = p # the chips show which hands are seen while the person gets ready + self._fb.pop("lost_key", None) + self._show(s["title"], where, p["text"]) + self._view(p) + for key, c in (("big", "big "), ("countdown", "countdown off"), ("bar", "bar off"), ("target", "target off")): + self._panel.set(key, c) + self._emit(state="ready", prompt=p["text"].replace("|", "\n"), seconds_left=float(p["seconds"]), + can_redo=can_redo, countdown=0, big="") + + def _countdown(self, i, s, st): + """Step mode: start recording and count 3-2-1 (logged as a "ready" event, so labels + cover only the hold), then make sure the recording has its first set. Returns the + ready event's time.""" + p = st["p"] + if not self._take: + self._start_take(i, s, record=False) + t = self._event("ready", id=p["id"], seconds=COUNTDOWN_S) + self._step_t0 = t + self._start_recording() + tick = None + if st["kind"] == "bar": # the bar at near, where the sweep starts + labels = "%s %s" % (s.get("near_label", "Near"), s.get("far_label", "Far")) + near, far = float(s.get("near_m", 0.2)), float(s.get("far_m", 0.6)) + + def tick(dt): + self._panel.set("bar", "bar 0.000 %.3f %s" % (self._palm_share(near, far), labels)) + return False + for n in range(COUNTDOWN_S, 0, -1): + self._panel.set("big", "big %d" % n) + self._emit(state="countdown", countdown=n, big=str(n), can_redo=True) + self._wait(1.0, tick, countdown=False) + self._first_set() + self._panel.set("big", "big " + s.get("go", "Go")) + self._status["big"] = s.get("go", "Go") # sent with the prompt + return t + + def _first_set(self): + """The hold starts once its recording has a set. ft-hands --record-only writes its first + within about 30 ms of starting, so the countdown covers it; this is a safety net.""" + rec = self._recorder + if not rec: + return + end = time.monotonic() + FIRST_SET_S + while not rec.has_data(): + if rec.exited(): + raise _Fail("The recording stopped by itself: see session.log") + if self._want["stop"]: + raise _Stop() + if time.monotonic() > end: + self._log("recording part %d: no set yet after the countdown" % rec.part) + return + time.sleep(0.02) + + def _hold_end(self): + """Step mode: the step is over. A "wait" event ends its labels and the recording stops + until the next countdown.""" + self._event("wait") + self._stop_recording() + self._status["big"] = "" + for key, c in (("big", "big "), ("countdown", "countdown off"), ("bar", "bar off"), ("target", "target off")): + self._panel.set(key, c) + + def _run_step(self, s, st, where): + if st["kind"] != "bar": + self._panel.set("bar", "bar off") + if st["kind"] == "target": + self._target(s, st, where) + elif st["kind"] == "bar": + self._bar(s, st, where) + else: + self._run_prompt(s, st["p"], where) + + def _target(self, s, st, where): + hold_s, timeout_s = float(s.get("hold_s", 1.0)), float(s.get("timeout_s", 8)) + p, pt = st["p"], st["pt"] + self._begin_prompt(s, p, where) + xyz = "%.3f %.3f %.3f" % tuple(pt) + reply = self._panel.cmd("target %s show" % xyz, reply=True) or "" + self._panel.sent["target"] = "target %s show" % xyz + tok = reply.split() + room = [float(v) for v in tok[1:4]] if tok[:1] == ["ok"] and len(tok) >= 4 else None + target = {"id": p["id"], "head": list(pt), "room": room} + self._event("target", state="show", **target) + state = {"hold": 0.0, "sent": 0.0, "holding": False} + + def tick(dt): + d = self._tip_distance(pt, room) + if d is not None and d <= TOUCH_M: + if not state["holding"]: + state["holding"] = True + self._event("target", state="hold", **target) + state["hold"] += dt + elif state["holding"]: + state["holding"], state["hold"] = False, 0.0 + self._panel.set("target", "target %s show" % xyz) + if state["holding"]: + frac = min(1.0, state["hold"] / hold_s) + if time.monotonic() - state["sent"] >= 0.1 or frac >= 1: + state["sent"] = time.monotonic() + self._panel.set("target", "target %s hold %.2f" % (xyz, frac)) + return state["hold"] >= hold_s + + done = self._wait(timeout_s, tick) + result = "done" if done else "timeout" + if done: + self._panel.set("target", "target %s done" % xyz) + self._event("target", state=result, **target) + self._log(" %s %s" % (p["id"], result)) + if done: + self._wait(0.6, countdown=False) + self._panel.set("target", "target off") def _tip_distance(self, pt, room): """The nearest seen index tip's distance from the target: in the room (with the head's @@ -1391,35 +1725,28 @@ class Session: return -1.0 return max(0.0, min(1.0, (sum(ds) / len(ds) - near) / (far - near))) - def _bar(self, s, step): + def _bar(self, s, st, where): + """One height of the push out and back: the target sweeps near to far and back.""" near, far = float(s.get("near_m", 0.2)), float(s.get("far_m", 0.6)) - period, lead = float(s.get("period_s", 6)), float(s.get("lead_s", 3)) + period, lead = float(s.get("period_s", 6)), st["lead"] labels = "%s %s" % (s.get("near_label", "Near"), s.get("far_label", "Far")) - defaults = s.get("defaults") or {} - for h in s["heights"]: - p = {"id": "%s/%s" % (s["id"], h["id"]), "text": h["text"], "hands": defaults.get("hands", "both"), - "pose": defaults.get("pose", "open"), "distance": "", "position": h["id"], "object": "", - "controller": bool(defaults.get("controller", False)), - "controllers": list(defaults.get("controllers") or [])} - total = lead + h["reps"] * period - p["seconds"] = total - self._begin_prompt(s, p, step) - st = {"t": 0.0, "sent": 0.0} + p = st["p"] + self._begin_prompt(s, p, where) + state = {"t": 0.0, "sent": 0.0} - def tick(dt, st=st): - st["t"] += dt - sweep = st["t"] - lead - target = 0.0 if sweep <= 0 else 1 - abs(1 - 2 * ((sweep % period) / period)) - if time.monotonic() - st["sent"] >= 0.1: - st["sent"] = time.monotonic() - cur = self._palm_share(near, far) - self._panel.set("bar", "bar %.3f %.3f %s" % (target, cur, labels)) - if sweep > 0: - self._event("bar", target=round(target, 3), current=None if cur < 0 else round(cur, 3)) - return False + def tick(dt): + state["t"] += dt + sweep = state["t"] - lead + target = 0.0 if sweep <= 0 else 1 - abs(1 - 2 * ((sweep % period) / period)) + if time.monotonic() - state["sent"] >= 0.1: + state["sent"] = time.monotonic() + cur = self._palm_share(near, far) + self._panel.set("bar", "bar %.3f %.3f %s" % (target, cur, labels)) + if sweep > 0: + self._event("bar", target=round(target, 3), current=None if cur < 0 else round(cur, 3)) + return False - self._wait(total, tick) - self._panel.set("bar", "bar off") + self._wait(p["seconds"], tick) # ------------------------------------------------------------------------------------------ @@ -1439,6 +1766,11 @@ def main(): ap.add_argument("--panel", help="the panel program (default hands/rec/build/ft-handpanel)") ap.add_argument("--hands-dir", help="where the hands file is (default /run/user/UID/frametop-hands)") ap.add_argument("--plan", action="store_true", help="print the sections and their length, and exit") + ap.add_argument("--auto", action="store_true", + help="advance by itself: each prompt for its time, no waiting for Next (the old timed flow)") + ap.add_argument("--next-after", type=float, metavar="S", + help="test: press Next by itself after S seconds of waiting (real time)") + ap.add_argument("--poses", help="the pose pictures' folder, with poses.json (default hands/rec/poses)") a = ap.parse_args() known = ("pencil", "phone", "cup", "keyboard", "mouse", "gamepad", "small") @@ -1462,15 +1794,21 @@ def main(): last = {} def on_status(st): - key = (st["state"], st["section"], st["prompt"], st["note"], st["take"], + key = (st["state"], st["section"], st["prompt"], st["note"], st["take"], st["waiting"], st["countdown"], st["hands"]["left"], st["hands"]["right"], st["error"]) if key == last.get("key"): return last["key"] = key hands = "".join("%s%s" % (s[0].upper(), {True: "+", False: "-", None: "?"}[st["hands"][s]]) for s in ("left", "right")) - line = "[%6.1f] %-8s %d/%d %-16s %s %s" % (time.monotonic() - t0, st["state"], st["section_index"], - st["section_count"], st["section"] or "-", hands, st["prompt"]) + state = "%s %d" % (st["state"], st["countdown"]) if st["state"] == "countdown" else st["state"] + line = "[%6.1f] %-11s %d/%d %-16s %s %s" % (time.monotonic() - t0, state, st["section_index"], + st["section_count"], st["section"] or "-", hands, + st["prompt"].replace("\n", " | ")) + if st["image"]: + line += " [%s%s]" % (os.path.basename(st["image"]), " " + st["image_mode"] if st["image_mode"] else "") + if st["waiting"]: + line += " (waiting for Next)" if st["note"]: line += " (%s)" % st["note"] if st["error"]: @@ -1478,15 +1816,22 @@ def main(): print(line, flush=True) s = Session(base, profile, checklist, a.lighting, a.script, ring=a.ring, start_processes=not a.no_start, - dry_run=a.dry_run, speed=a.speed, on_status=on_status, hands_dir=a.hands_dir, panel_bin=a.panel) - est, worst = plan_seconds(s.script, s.plan), plan_seconds(s.script, s.plan, worst=True) - print("%d sections, about %.1f min (at most %.1f)%s" % (len(s.plan), est / 60, worst / 60, - ", %gx speed" % a.speed if a.speed != 1 else "")) + dry_run=a.dry_run, speed=a.speed, on_status=on_status, hands_dir=a.hands_dir, panel_bin=a.panel, + auto=a.auto, next_after=a.next_after, poses_dir=a.poses) + est = plan_seconds(s.script, s.plan, auto=a.auto) + worst = plan_seconds(s.script, s.plan, worst=True, auto=a.auto) + print("%d sections, %s mode: about %.1f min%s (at most %.1f)%s" % ( + len(s.plan), "auto" if a.auto else "step", est / 60, "" if a.auto else " recorded", worst / 60, + ", %gx speed" % a.speed if a.speed != 1 else "")) + if not a.auto: + print(" %d steps wait for Next: add your reading time (at 5 s a step, %.1f min)" + % (plan_steps(s.plan), plan_steps(s.plan) * 5 / 60)) for sk in s.skipped: print(" skipping %s: %s" % (sk["section"], sk["reason"])) if a.plan: for sec in s.plan: - print(" %-18s %-9s %3d prompts %5.0f s" % (sec["id"], sec["kind"], len(sec["prompts"]), section_seconds(sec))) + print(" %-18s %-9s %3d prompts %3d steps %5.0f s" % (sec["id"], sec["kind"], len(sec["prompts"]), + section_steps(sec), section_seconds(sec, auto=a.auto))) return 0 if not a.dry_run: light = ring_lighting(a.ring) @@ -1501,19 +1846,23 @@ def main(): signal.signal(signal.SIGINT, on_signal) signal.signal(signal.SIGTERM, on_signal) - if sys.stdin.isatty(): - def keys(): - print("keys: p pause, r resume, s skip, q stop (then Enter)", flush=True) - for line in sys.stdin: - c = line.strip()[:1] - if c == "p": - s.pause() - elif c == "r": - s.resume() - elif c == "s": - s.skip() - elif c == "q": - s.stop(wait=0) + def keys(): + # From a terminal, or lines piped in (a test); /dev/null ends at once. + if sys.stdin.isatty(): + print("keys (then Enter): n or just Enter next, p pause/resume, r redo, s skip section, q stop", flush=True) + for line in sys.stdin: + c = line.strip()[:1].lower() + if c in ("n", ""): + s.next_step() + elif c == "p": + s.resume() if s.state == "paused" else s.pause() + elif c == "r": + s.redo() + elif c == "s": + s.skip() + elif c == "q": + s.stop(wait=0) + if sys.stdin is not None: threading.Thread(target=keys, daemon=True).start() s.start() while s._thread.is_alive(): diff --git a/hands/rec/tests/test_session.py b/hands/rec/tests/test_session.py new file mode 100644 index 0000000..c96a412 --- /dev/null +++ b/hands/rec/tests/test_session.py @@ -0,0 +1,235 @@ +#!/usr/bin/env python3 +"""Tests for session.py's step mode (dry runs, no processes): the ready, countdown and hold +timeline in prompts.jsonl, R (redo), the timed flow (--auto), the pose pictures' display rule, +the length estimates, and a take recorded in many parts through review, export and validate. + + python3 hands/rec/tests/test_session.py +""" +import json +import os +import shutil +import sys +import tempfile +import time +import unittest + +HERE = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, os.path.dirname(HERE)) +import session # noqa: E402 +import takes # noqa: E402 +import validate # noqa: E402 +from test_validate import fhset, make_session, SESSION # noqa: E402 + +SCRIPT = { + "version": 1, "intro_s": 1, "between_s": 1, + "welcome": {"title": "Test", "seconds": 1, "text": "A test."}, + "done": {"title": "Done", "seconds": 0, "text": "Done."}, + "stopped": {"title": "Stopped", "seconds": 0, "text": "Stopped."}, + "sections": [ + {"id": "poses", "title": "Poses", "intro": "Some poses.", "go": "Hold", + "prompts": [{"text": "Fist.", "seconds": 4, "hands": "left", "pose": "fist", "distance": "near", + "position": "left"}, + {"text": "Open.", "seconds": 4, "hands": "both", "pose": "open"}]}]} + + +class SessionTest(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.mkdtemp(prefix="handrec-session-test-") + self.script = os.path.join(self.tmp, "script.json") + with open(self.script, "w") as f: + json.dump(SCRIPT, f) + self.panel = [] + + def tearDown(self): + shutil.rmtree(self.tmp, ignore_errors=True) + + def session(self, **kw): + kw.setdefault("speed", 10) + s = session.Session(os.path.join(self.tmp, "base"), {}, {}, "room", self.script, dry_run=True, + poses_dir=os.path.join(self.tmp, "no-poses"), **kw) + s.print = self.panel.append + return s + + def wait_for(self, s, pred, what, timeout=10): + end = time.monotonic() + timeout + while time.monotonic() < end: + st = dict(s._status) + if pred(st): + return st + time.sleep(0.005) + self.fail("timed out waiting for %s: %s" % (what, s._status)) + + def waiting(self, s, state, prompt=None): + return self.wait_for(s, lambda st: st["state"] == state and st["waiting"] + and (prompt is None or st["prompt"] == prompt), "%s %s" % (state, prompt or "")) + + def events(self, s): + with open(os.path.join(s.session_dir, "takes", "01-poses", "prompts.jsonl")) as f: + return [json.loads(line) for line in f] + + def test_step_flow(self): + s = self.session(next_after=0.0) + s.start() + s.join(20) + self.assertEqual(s.state, "done") + ev = self.events(s) + self.assertEqual([e["event"] for e in ev], + ["take", "ready", "prompt", "wait", "ready", "prompt", "wait", "end"]) + self.assertEqual(ev[1]["id"], ev[2]["id"]) + self.assertEqual(ev[1]["seconds"], session.COUNTDOWN_S) + self.assertTrue(ev[1]["t"] < ev[2]["t"] < ev[3]["t"] < ev[4]["t"]) + # the countdown is recorded (3 s at 10x speed), and the hold after it + self.assertGreater(ev[2]["t"] - ev[1]["t"], 0.25e9) + with open(os.path.join(s.session_dir, "session.json")) as f: + self.assertEqual(json.load(f)["mode"], "step") + # the panel: Ready?, then the countdown, then the section's word for the hold, the diagram + self.assertIn("panel: action " + session.READY_TEXT, self.panel) + self.assertLess(self.panel.index("panel: big 3"), self.panel.index("panel: big 1")) + self.assertIn("panel: big Hold", self.panel) + self.assertIn("panel: where left near", self.panel) + self.assertIn("panel: rec on", self.panel) + + def test_redo(self): + s = self.session() + s.start() + self.waiting(s, "starting") + s.next_step() + self.waiting(s, "intro") + s.redo() # nothing to do again yet: dropped + s.next_step() + st = self.waiting(s, "ready", "Fist.") + self.assertFalse(st["can_redo"]) + s.next_step() + self.wait_for(s, lambda st: st["state"] == "running", "the hold") + s.redo() # during the hold: it starts again + self.waiting(s, "ready", "Fist.") + s.next_step() + st = self.waiting(s, "ready", "Open.") + self.assertTrue(st["can_redo"]) + s.redo() # at the next step's ready screen: the one before goes again + self.waiting(s, "ready", "Fist.") + s.next_step() + self.waiting(s, "ready", "Open.") + s.next_step() + s.join(20) + self.assertEqual(s.state, "done") + ev = self.events(s) + names = [e["event"] for e in ev] + self.assertEqual(names, ["take", "ready", "prompt", "redo", "wait", "ready", "prompt", "wait", "redo", + "ready", "prompt", "wait", "ready", "prompt", "wait", "end"]) + first, second = ev[3], ev[8] + self.assertEqual(first["id"], "poses/fist/left/near/left") + self.assertEqual(first["from"], ev[1]["t"]) # from its countdown + self.assertTrue(ev[2]["t"] < first["to"] <= ev[4]["t"]) + self.assertEqual((second["from"], second["id"]), (ev[5]["t"], first["id"])) + self.assertTrue(ev[6]["t"] < second["to"] <= ev[7]["t"]) + # what labels keep: prompts outside every redo range + kept = [e["id"] for e in ev if e["event"] == "prompt" + and not any(r["from"] <= e["t"] <= r["to"] for r in ev if r["event"] == "redo")] + self.assertEqual(kept, ["poses/fist/left/near/left", "poses/open/both"]) + + def test_pause_while_waiting(self): + s = self.session() + s.start() + self.waiting(s, "starting") + s.next_step() + self.waiting(s, "intro") + s.next_step() + self.waiting(s, "ready", "Fist.") + s.pause() + self.wait_for(s, lambda st: st["state"] == "paused", "paused") + s.next_step() # ignored while paused + s.resume() + st = self.waiting(s, "ready", "Fist.") + s.stop(wait=10) + self.assertEqual(s.state, "stopped") + # nothing recorded: no take + self.assertEqual(os.listdir(os.path.join(s.session_dir, "takes")), []) + + def test_auto(self): + s = self.session(auto=True, speed=20) + s.start() + s.join(20) + self.assertEqual(s.state, "done") + ev = self.events(s) + self.assertEqual([e["event"] for e in ev], ["take", "prompt", "prompt", "prompt", "end"]) + self.assertEqual(ev[1]["id"], "poses/intro") + + def test_auto_redo_after_pause(self): + s = self.session(auto=True) + s.start() + self.wait_for(s, lambda st: st["state"] == "running" and st["prompt"] == "Fist.", "the first prompt") + s.pause() + self.wait_for(s, lambda st: st["state"] == "paused", "paused") + s.redo() # ends the pause; the prompt starts again, recording again + s.join(20) + self.assertEqual(s.state, "done") + names = [e["event"] for e in self.events(s)] + self.assertEqual(names, ["take", "prompt", "prompt", "pause", "redo", "resume", "prompt", "prompt", "end"]) + + def test_pose_view(self): + d = os.path.join(self.tmp, "poses") + os.makedirs(d) + for name in ("fist.png", "cross.png"): + open(os.path.join(d, name), "wb").close() + with open(os.path.join(d, "poses.json"), "w") as f: + json.dump({"fist": {"file": "fist.png", "two_hands": False, "caption": "A fist"}, + "cross": {"file": "../cross.png", "two_hands": True, "caption": ""}, + "ok": {"file": "ok.png", "two_hands": False}}, f) + poses = session.load_poses(d) + fist = os.path.join(d, "fist.png") + self.assertEqual(session.pose_view(poses, {"pose": "fist", "hands": "right"}), (fist, "", "A fist")) + self.assertEqual(session.pose_view(poses, {"pose": "fist", "hands": "left"})[1], "mirror") + self.assertEqual(session.pose_view(poses, {"pose": "fist", "hands": "both"})[1], "both") + self.assertEqual(session.pose_view(poses, {"pose": "fist", "hands": "any"})[1], "") + # two hands drawn already; the file stays in the folder + self.assertEqual(session.pose_view(poses, {"pose": "cross", "hands": "both"})[:2], + (os.path.join(d, "cross.png"), "")) + self.assertEqual(session.pose_view(poses, {"pose": "ok", "hands": "both"}), ("", "", "")) # no file + self.assertEqual(session.pose_view(poses, {"pose": "claw", "hands": "both"}), ("", "", "")) + self.assertEqual(session.load_poses(os.path.join(self.tmp, "none")), {}) + + def test_plan(self): + plan, _ = session.build_plan(SCRIPT, {}) + self.assertEqual(session.plan_steps(plan), 2) + self.assertEqual(session.plan_seconds(SCRIPT, plan, auto=False), 2 * session.COUNTDOWN_S + 8) + self.assertEqual(session.plan_seconds(SCRIPT, plan, auto=True), 1 + 1 + 8) + self.assertIn("2 steps", session.plan_summary(SCRIPT, plan)) + + +class ManyPartsTest(unittest.TestCase): + """A take recorded in 40 parts (step mode stops the recording between steps): review reads + them in order, export makes one stream, validate passes.""" + + def setUp(self): + if not takes.find_zstd(): + self.skipTest("no zstd") + self.tmp = tempfile.mkdtemp(prefix="handrec-parts-test-") + + def tearDown(self): + shutil.rmtree(self.tmp, ignore_errors=True) + + def test_parts(self): + make_session(self.tmp) + tdir = os.path.join(self.tmp, "sessions", SESSION, "takes", "01-hand-size") + t0 = 5 * 10 ** 12 + for part in range(1, 41): + name = "sets.bin" if part == 1 else "sets-%d.bin" % part + with open(os.path.join(tdir, name), "wb") as f: + for i in range(3): # 3 sets a part, 10 s apart between parts + f.write(fhset(t0 + part * 10 ** 10 + i * 10 ** 8, part)) + store = takes.Store(self.tmp) + index = takes.take_index(tdir) + self.assertEqual(len(index.parts), 40) + self.assertEqual(len(index), 120) + times = [index.time_ns(i) for i in range(len(index))] + self.assertEqual(times, sorted(times)) # sets-10.bin after sets-9.bin + self.assertAlmostEqual(index.duration_s(), 40 * 0.2) # the gaps between parts don't count + path = store.export(SESSION, low_priority=False) + r = validate.validate(path) + self.assertEqual(r.errors, []) + self.assertEqual(r.summary["sets"], 120 + 30) + + +if __name__ == "__main__": + unittest.main()