diff --git a/hands/rec/DESIGN.md b/hands/rec/DESIGN.md index e544d96..8d340c2 100644 --- a/hands/rec/DESIGN.md +++ b/hands/rec/DESIGN.md @@ -76,7 +76,8 @@ No name, email, or account. The contributor id is random, so several sessions fr ```json {"schema": 1, "tool": "ft-handrec ", "started": "", "contributor": "", - "lighting": {"chosen": "dim|room|daylight", "ring": {"": {"mean": 0.0, "dark_mean": 0.0}}}, + "lighting": {"chosen": "dim|room|daylight|indoor", "source": "measured|picked", "measured": "indoor|daylight|", + "ambient_ir": 0.0, "ring": {"": {"mean": 0.0, "dark_mean": 0.0}}}, "checklist": {"objects": ["pencil", "phone", "cup", "keyboard", "mouse", "gamepad", "small"], "own_objects": ["..."], "controllers": "straps|none", "sleeves": "short|long|", "rings": false, "watch": false, "notes": ""}, "device": {"steamos": "", "steamvr": "", "cameras": [{"name", "width", "height"}]}, @@ -279,7 +280,8 @@ Before section 7: "Put on both controllers and tighten the straps". Before secti - **Hands seen:** from the hands file. A hand counts as seen if its flags match the side and the file is fresh (publish within 0.3 s). Prompts with `hands` set show the `hands` chips. If an asked-for hand is lost for more than 1.5 s, the note says "I can't see your left hand: bring it into view". - **Controller tracking:** in sections 8 and 9, `devices` is polled once a second. A result other than 200 for more than 1 s says "The left controller lost tracking: turn your palm slightly toward you". Each such stretch goes into `prompts.jsonl` as `feedback` with `"controller": {...}`. -- **Lighting check at session start:** the mean of every mono camera's `mean` and `dark_mean` from the ring (`hands/tools/ring.py` layout; struct only, no numpy). It's compared with the person's earlier sessions. If the chosen lighting matches an earlier round's within 15%, the window says so before starting. +- **Lighting, measured:** the checklist page measures the light when it opens, starting ft-camd (`frametop-handrec-camd.service`) if nothing runs it; the window stops it again on quit if it started it. The mean of every mono camera's `mean` and `dark_mean` from the ring (`hands/tools/ring.py` layout; struct only, no numpy) is compared with the person's earlier sessions. If it matches an earlier round's within 15%, the window says so before starting. +- **Lighting label:** "Measured by the cameras" is the default. `ambient_ir`, the mono cameras' mean `dark_mean` (the room's infrared), labels the round `daylight` from 6.0 and `indoor` below (`session.classify_lighting`). Lamps and LEDs give off hardly any infrared, so a dim room and a lit one read about the same (1.8 by one lamp, 2.2 in a lamp-lit room) and the cameras can't tell them apart; the person can pick dim, room or daylight instead (`source: "picked"`). The 6.0 threshold is a guess until a daylight round is measured. ### Camera check diff --git a/hands/rec/ft_handrec.py b/hands/rec/ft_handrec.py index b7cde4c..96a51e8 100755 --- a/hands/rec/ft_handrec.py +++ b/hands/rec/ft_handrec.py @@ -68,7 +68,11 @@ ROUND_BYTES = 10 * 1000 ** 3 # about what one round of recording takes OBJECTS = [("pencil", "Pencil or pen"), ("phone", "Phone"), ("cup", "Cup or mug (empty)"), ("keyboard", "Keyboard"), ("mouse", "Mouse"), ("gamepad", "Gamepad"), ("small", "Something small (a coin, a key, a bottle cap)")] -LIGHTING = [("dim", "Dim: one lamp only"), ("room", "Normal room light"), ("daylight", "Daylight near a window")] +# "auto": the cameras' measurement (session.classify_lighting: indoor or daylight); the rest +# correct it, since the cameras can't tell a dim room from a bright one. +LIGHTING = [("auto", "Measured by the cameras"), ("dim", "Dim: one lamp only"), ("room", "Normal room light"), + ("daylight", "Daylight near a window")] +LIGHTING_TEXT = dict(LIGHTING[1:], indoor="Indoor light") SLEEVES = [("short", "Short sleeves or bare arms"), ("long", "Long sleeves"), ("", "Rather not say")] HANDEDNESS = [("", "Rather not say"), ("right", "Right-handed"), ("left", "Left-handed"), ("both", "Both (ambidextrous)")] @@ -207,7 +211,7 @@ class Backend(QObject): message = Signal(str, bool) # text, is error # From other threads (the session runner, export, the lighting check): queued to this one. _statusArrived = Signal(dict) - _lightingArrived = Signal(str) + _lightingArrived = Signal(str, str) _exportProgress = Signal(float, str) _exportFinished = Signal(str, str) # path, error ("" when it worked; "cancelled") loginChanged = Signal() @@ -236,6 +240,9 @@ class Backend(QObject): self._session_id = "" self._status = {} self._lighting_note = "" + self._lighting_measured = "" + self._lighting_busy = False + self._camd_started = False # ft-camd started for the light check: stopped on quit self._export_cancel = None self._export_thread = None self._export_fraction = 0.0 @@ -354,37 +361,56 @@ class Backend(QObject): @Property(str, notify=lightingChanged) def lightingNote(self): + """A warning: this light is like an earlier round's, or it couldn't be measured.""" return self._lighting_note + @Property(str, notify=lightingChanged) + def lightingMeasured(self): + """What the cameras see: "Indoor light" or "Daylight…", "Measuring…", or "".""" + return "Measuring…" if self._lighting_busy else self._lighting_measured + @Slot(str) def checkLighting(self, chosen): - """Compare the cameras' brightness now with earlier rounds (session.similar_lighting): - a round in light like an earlier one adds less to the dataset.""" + """Measure the light (starting ft-camd if nothing runs it) and compare it with earlier + rounds (session.similar_lighting): a round in light like an earlier one adds less.""" mod = self._runner() - if not mod: + if not mod or self._lighting_busy: return base = self.store.base - labels = dict(LIGHTING) + self._lighting_busy = True + self.lightingChanged.emit() def run(): + measured = "" try: ring = mod.ring_lighting() + if ring is None and not self.sessionActive: + if mod.start_camd(): + self._camd_started = True + time.sleep(1.0) # the first near-black frames + ring = mod.ring_lighting() if ring is None: - note = ("The cameras aren't running yet, so the light can't be compared with your " - "earlier rounds now. The session checks it when it starts.") + note = ("The cameras aren't running, so the light can't be measured now. The session " + "measures it when it starts.") else: + kind = mod.classify_lighting(ring) + ir = mod.ambient_ir(ring) + measured = (f"{LIGHTING_TEXT.get(kind, kind)} (infrared {ir:.1f})" if kind + else "The cameras couldn't measure the light yet.") match = mod.similar_lighting(base, {"chosen": chosen, "ring": ring}) note = "" if not match else ( f"The cameras see about the same light as in your round of {session_label(match[0])} " - f"({labels.get(match[1], match[1] or 'no choice')}). A different light helps the " + f"({LIGHTING_TEXT.get(match[1], match[1] or 'not given')}). A different light helps the " "dataset more: change the lighting if you can, or go ahead anyway.") except Exception as e: - note = f"Couldn't check the light: {e}" - self._lightingArrived.emit(note) + note = f"Couldn't measure the light: {e}" + self._lightingArrived.emit(note, measured) self._thread(run) - def _on_lighting(self, note): + def _on_lighting(self, note, measured): + self._lighting_busy = False self._lighting_note = note + self._lighting_measured = measured self.lightingChanged.emit() # --- the camera check (hands/camcheck.py through session.camera_check) @@ -618,6 +644,11 @@ class Backend(QObject): if self._upload_proc: self.cancelUpload() self._upload_thread.join(10) + if self._camd_started and self._session_mod: + try: + self._session_mod.stop_unit(self._session_mod.CAMD_UNIT) + except Exception: + pass # --- review @Property("QVariantList", notify=sessionsChanged) @@ -627,7 +658,7 @@ class Backend(QObject): s["label"] = session_label(s["id"]) s["sizeText"] = takes.human_bytes(s["bytes"]) s["exportText"] = takes.human_bytes(s["export_bytes"]) if s["exported"] else "" - s["lightingText"] = dict(LIGHTING).get(s["lighting"], "") + s["lightingText"] = LIGHTING_TEXT.get(s["lighting"], "") s["active"] = self.sessionActive and s["id"] == self._session_id s["statusText"] = {"recording": "" if s["active"] else "interrupted", "error": "ended with an error", "stopped": "stopped early"}.get(s["status"], "") diff --git a/hands/rec/main.qml b/hands/rec/main.qml index 942990d..f6281c3 100644 --- a/hands/rec/main.qml +++ b/hands/rec/main.qml @@ -245,7 +245,7 @@ Kirigami.ApplicationWindow { id: checklist title: "Before you start" readonly property bool privacyOk: privacy1.checked && privacy2.checked && privacy3.checked - readonly property bool ready: privacyOk && lighting.currentIndex >= 0 && backend.diskOk + readonly property bool ready: privacyOk && backend.diskOk && backend.runnerError === "" && !backend.sessionActive function answers() { @@ -336,33 +336,40 @@ Kirigami.ApplicationWindow { } Kirigami.Separator { Kirigami.FormData.isSection: true; Kirigami.FormData.label: "Light" } + Controls.Label { + Kirigami.FormData.label: "The cameras see:" + Layout.maximumWidth: Kirigami.Units.gridUnit * 26 + wrapMode: Text.Wrap + text: backend.lightingMeasured || "Not measured yet" + } Controls.ComboBox { id: lighting Kirigami.FormData.label: "Lighting this round:" model: backend.lightingChoices textRole: "text" valueRole: "value" - currentIndex: -1 - displayText: currentIndex < 0 ? "Choose…" : currentText + currentIndex: 0 onActivated: backend.checkLighting(currentValue) + Component.onCompleted: backend.checkLighting(currentValue) } Controls.Label { Layout.maximumWidth: Kirigami.Units.gridUnit * 26 wrapMode: Text.Wrap opacity: 0.7 - text: "Each round in a different light helps the most: dim, a normal room, daylight." + text: "Each round in a different light helps the most: dim, a normal room, daylight. The cameras " + + "tell daylight from indoor light themselves; to say dim or a normal room, pick it here." } Kirigami.InlineMessage { Layout.maximumWidth: Kirigami.Units.gridUnit * 26 Layout.fillWidth: true - visible: backend.lightingNote !== "" && lighting.currentIndex >= 0 + visible: backend.lightingNote !== "" Layout.preferredHeight: visible ? implicitHeight : 0 type: Kirigami.MessageType.Warning text: backend.lightingNote } Controls.Button { - visible: backend.lightingNote !== "" && lighting.currentIndex >= 0 - text: "Check the light again" + visible: backend.lightingNote !== "" || backend.lightingMeasured === "" + text: "Measure the light again" icon.name: "view-refresh" onClicked: backend.checkLighting(lighting.currentValue) } @@ -529,7 +536,6 @@ Kirigami.ApplicationWindow { visible: (!checklist.ready || backend.camerasBlockStart) && !backend.sessionActive opacity: 0.7 text: !checklist.privacyOk ? "Tick the three privacy checks to start." - : lighting.currentIndex < 0 ? "Choose the lighting to start." : backend.camerasBlockStart ? "The headset's cameras aren't all running: see the message at the top." : "" } } diff --git a/hands/rec/session.py b/hands/rec/session.py index 12be9c0..3f043d7 100755 --- a/hands/rec/session.py +++ b/hands/rec/session.py @@ -277,6 +277,98 @@ def similar_lighting(base_dir, lighting): return None +DAYLIGHT_IR = 6.0 # ambient IR (the mono cameras' mean dark_mean) from which it's daylight + + +def ambient_ir(ring): + """The mono cameras' mean dark_mean (the room's infrared light, as ft-hands logs it), or + None if none has one. ring is ring_lighting()'s dict.""" + vals = [float(v.get("dark_mean", 0)) for v in (ring or {}).values() if isinstance(v, dict)] + vals = [v for v in vals if v > 0] + return round(sum(vals) / len(vals), 2) if vals else None + + +def classify_lighting(ring): + """"daylight" or "indoor" from the room's infrared light, "" if it can't tell. Sunlight + carries a lot of infrared; lamps and LEDs hardly any, so a dim room and a bright one read + about the same (2026-10: 1.8 by one lamp, 2.2 in a lamp-lit room) and aren't told apart.""" + ir = ambient_ir(ring) + if ir is None: + return "" + return "daylight" if ir >= DAYLIGHT_IR else "indoor" + + +def lighting_record(choice, ring): + """session.json's "lighting": chosen is the person's pick, or with "auto" (or none) what + the cameras measured; measured and ambient_ir are always the cameras' reading.""" + measured = classify_lighting(ring) + picked = choice if choice in ("dim", "room", "daylight") else "" + return {"chosen": picked or measured, "source": "picked" if picked else "measured", + "measured": measured, "ambient_ir": ambient_ir(ring), "ring": ring or {}} + + +def unit_active(unit): + return subprocess.run(host_command("systemctl", "--user", "-q", "is-active", unit), + capture_output=True, timeout=30).returncode == 0 + + +def start_unit(unit, what, argv, log=lambda line: None): + """Start argv as a transient user unit that stops with SteamVR. True if it started it, + False if it was running already; raises RuntimeError if it couldn't.""" + if unit_active(unit): + log("%s is running already" % unit) + return False + cmd = host_command("systemd-run", "--user", "--quiet", "--collect", "--unit=" + unit, + "--description=Frametop hand recorder: " + what, + "-p", "PartOf=steamvr.service", "-p", "After=steamvr.service", + "-p", "Restart=on-failure", "-p", "RestartSec=3", "-p", "TimeoutStopSec=5", *argv) + for attempt in range(3): # distrobox-host-exec has failed once, silently, and worked again + r = subprocess.run(cmd, capture_output=True, text=True, timeout=30) + if r.returncode == 0 or unit_active(unit): + log("started %s" % unit) + return True + log("starting %s failed (exit %d): %s" % (unit, r.returncode, (r.stderr or r.stdout).strip())) + time.sleep(0.5) + raise RuntimeError("couldn't start %s (exit %d): %s" % (unit, r.returncode, (r.stderr or r.stdout).strip())) + + +def stop_unit(unit): + subprocess.run(host_command("systemctl", "--user", "stop", unit), capture_output=True, timeout=30) + + +def ring_alive(path=None): + try: + r = Ring(path or os.path.join(run_dir(), "cam-ring")) + except (OSError, ValueError): + return False + try: + return r.alive() + finally: + r.close() + + +def start_camd(path=None, log=lambda line: None, stop=lambda: False, timeout=15.0): + """Make sure ft-camd fills the camera ring: start it (CAMD_UNIT) if nothing does. True if + it started it, False if a ring was live already; raises RuntimeError if it can't.""" + path = path or os.path.join(run_dir(), "cam-ring") + if ring_alive(path): + return False + if not os.access(FT_CAMD, os.X_OK): + raise RuntimeError("ft-camd isn't built: hands/build.sh") + caps = subprocess.run(["getcap", FT_CAMD], capture_output=True, text=True) if shutil.which("getcap") else None + if caps is not None and "cap_sys_ptrace" not in caps.stdout: + raise RuntimeError("ft-camd needs its capabilities: hands/run.sh caps (asks for sudo)") + started = start_unit(CAMD_UNIT, "the camera broker", [FT_CAMD, "--status", "60"], log) + end = time.monotonic() + timeout + while time.monotonic() < end: + if ring_alive(path): + return started + if stop(): + return started + time.sleep(0.1) + raise RuntimeError("ft-camd didn't start (is SteamVR running?): journalctl --user -u " + CAMD_UNIT) + + # ------------------------------------------------------------------------------------------ # The factory calibration, without what identifies the unit @@ -1331,7 +1423,7 @@ class Session: self._session_json = { "schema": 1, "tool": "ft-handrec " + _git_describe(), "started": time.strftime("%Y-%m-%dT%H:%M:%S%z"), "contributor": self.profile.get("contributor", ""), - "lighting": {"chosen": self.lighting_choice, "ring": lighting or {}}, + "lighting": lighting_record(self.lighting_choice, lighting), "checklist": self.checklist, "device": {"steamos": _os_version(), "steamvr": _steamvr_version(), "cameras": cams}, "calibration_removed": removed, @@ -1444,35 +1536,19 @@ class Session: return (self._session_json or {}).get("sides", {}).get("swapped") def _ensure_ring(self, path): - def alive(): - try: - r = Ring(path) - except (OSError, ValueError): - return False - try: - return r.alive() - finally: - r.close() - - if alive(): + if ring_alive(path): return if self.ring: raise _Fail("No frames in %s (start ft-ringplay first)" % path) if not self.start_processes: raise _Fail("ft-camd isn't running (and --no-start)") - if not os.access(FT_CAMD, os.X_OK): - raise _Fail("ft-camd isn't built: hands/build.sh") - caps = subprocess.run(["getcap", FT_CAMD], capture_output=True, text=True) if shutil.which("getcap") else None - if caps is not None and "cap_sys_ptrace" not in caps.stdout: - raise _Fail("ft-camd needs its capabilities: hands/run.sh caps (asks for sudo)") - self._start_unit(CAMD_UNIT, "the camera broker", [FT_CAMD, "--status", "60"]) - for _ in range(150): - if alive(): - return - if self._want["stop"]: - raise _Stop() - time.sleep(0.1) - raise _Fail("ft-camd didn't start (is SteamVR running?): journalctl --user -u " + CAMD_UNIT) + try: + if start_camd(path, self._log, lambda: self._want["stop"]): + self._units.append(CAMD_UNIT) + except RuntimeError as e: + raise _Fail(str(e)) + if self._want["stop"]: + raise _Stop() def _start_tracker(self): up = os.path.join(REPO, "scripts", "container-up.sh") @@ -1485,30 +1561,15 @@ class Session: self._start_unit(HANDS_UNIT, "hand tracking for feedback", argv) def _start_unit(self, unit, what, argv): - def active(): - return subprocess.run(host_command("systemctl", "--user", "-q", "is-active", unit), - capture_output=True, timeout=30).returncode == 0 - - if active(): - self._log("%s is running already" % unit) - return - cmd = host_command("systemd-run", "--user", "--quiet", "--collect", "--unit=" + unit, - "--description=Frametop hand recorder: " + what, - "-p", "PartOf=steamvr.service", "-p", "After=steamvr.service", - "-p", "Restart=on-failure", "-p", "RestartSec=3", "-p", "TimeoutStopSec=5", *argv) - for attempt in range(3): # distrobox-host-exec has failed once, silently, and worked again - r = subprocess.run(cmd, capture_output=True, text=True, timeout=30) - if r.returncode == 0 or active(): + try: + if start_unit(unit, what, argv, self._log): self._units.append(unit) - self._log("started %s" % unit) - return - self._log("starting %s failed (exit %d): %s" % (unit, r.returncode, (r.stderr or r.stdout).strip())) - time.sleep(0.5) - raise _Fail("couldn't start %s (exit %d): %s" % (unit, r.returncode, (r.stderr or r.stdout).strip())) + except RuntimeError as e: + raise _Fail(str(e)) def _stop_units(self): for unit in reversed(self._units): - subprocess.run(host_command("systemctl", "--user", "stop", unit), capture_output=True, timeout=30) + stop_unit(unit) self._log("stopped %s" % unit) self._units = [] @@ -2369,7 +2430,8 @@ def main(): ap.add_argument("--base", help="where sessions go (default %s; a temporary folder with --dry-run)" % BASE_DIR) ap.add_argument("--objects", default="", help="ticked objects, comma-separated (unknown names are your own)") ap.add_argument("--controllers", action="store_true", help="controllers with the straps") - ap.add_argument("--lighting", default="room", choices=("dim", "room", "daylight")) + ap.add_argument("--lighting", default="auto", choices=("auto", "dim", "room", "daylight"), + help="this round's light (auto: indoor or daylight, from the cameras)") ap.add_argument("--script", default=SCRIPT_PATH) ap.add_argument("--panel", help="the panel program (default hands/rec/build/ft-handpanel)") ap.add_argument("--hands-dir", help="where the hands file is (default /run/user/UID/frametop-hands)") @@ -2468,6 +2530,8 @@ def main(): light = ring_lighting(a.ring) match = similar_lighting(base, {"chosen": a.lighting, "ring": light}) if light else None print("lighting: %s" % (json.dumps(light) if light else "no camera ring")) + if light: + print(" measured: %s (ambient IR %s)" % (classify_lighting(light) or "can't tell", ambient_ir(light))) if match: print(" about the same light as session %s (%s)" % match) diff --git a/hands/rec/tests/test_session.py b/hands/rec/tests/test_session.py index c2f3f60..82ffa72 100644 --- a/hands/rec/tests/test_session.py +++ b/hands/rec/tests/test_session.py @@ -566,6 +566,30 @@ class ButtonSessionTest(SessionBase): self.assertIn("panel: keys " + session.KEYS_STEP, self.panel) +class LightingTest(unittest.TestCase): + """The measured lighting label: the mono cameras' ambient infrared.""" + + def ring(self, slam, upper): + return {"slam_left": {"mean": 70.0, "dark_mean": slam}, "slam_right": {"mean": 80.0, "dark_mean": slam}, + "upper_left": {"mean": 50.0, "dark_mean": upper}, "upper_right": {"mean": 20.0, "dark_mean": upper}} + + def test_indoor_and_daylight(self): + self.assertEqual(session.classify_lighting(self.ring(3.0, 0.5)), "indoor") # one lamp, 2026-10-02 + self.assertEqual(session.classify_lighting(self.ring(3.7, 0.7)), "indoor") # a lamp-lit room + self.assertEqual(session.classify_lighting(self.ring(20.0, 8.0)), "daylight") + self.assertEqual(session.classify_lighting(self.ring(0.0, 0.0)), "") # no dark frames yet + self.assertEqual(session.classify_lighting(None), "") + self.assertEqual(session.ambient_ir(self.ring(3.0, 0.5)), 1.75) + + def test_record(self): + rec = session.lighting_record("auto", self.ring(3.0, 0.5)) + self.assertEqual((rec["chosen"], rec["source"], rec["measured"]), ("indoor", "measured", "indoor")) + rec = session.lighting_record("dim", self.ring(3.0, 0.5)) + self.assertEqual((rec["chosen"], rec["source"], rec["measured"]), ("dim", "picked", "indoor")) + rec = session.lighting_record("auto", None) + self.assertEqual((rec["chosen"], rec["source"], rec["ring"]), ("", "measured", {})) + + class ManyPartsTest(unittest.TestCase): """A take recorded in 40 parts (step mode stops the recording between steps): review reads them in order, export makes one stream, validate passes."""