diff --git a/docs/tracking.md b/docs/tracking.md index f3b9c88..8d4225d 100644 --- a/docs/tracking.md +++ b/docs/tracking.md @@ -201,9 +201,9 @@ How it works: while the headset is worn and tracking. - **Privacy.** Each image is reduced to a 16×16 grid of patch averages as soon as it is complete, then deleted. Three worker processes do this beside - the capture. If more than 900 images (about five seconds) ever wait, the - capture stops rather than letting eye images accumulate. The capture directory is removed on - exit, even after errors. No image is kept or leaves the Frame. The estimate + the capture. If more than 900 images (about five seconds) ever wait, we stop + reading them and delete them undecoded until the capture ends, and report + an error. The capture directory is removed on exit, even after errors. No image is kept or leaves the Frame. The estimate is printed only with `--show`, and sent or saved only with `--osc` or `--log`, as for the strap. - **Estimate.** Patch traces are averaged down to 15 Hz and turned into @@ -227,6 +227,16 @@ signal/noise (0.15-0.19), in two separate runs. That is a camera or illumination and the signal/noise gate kept it from being reported. A worn test should also record an unworn baseline, to rule out the same artifact. +**Do not interrupt the capture. Verified on the Frame, 2026-09-29:** sending +SIGTERM to `eyetracking --calib` left the DSP service's eye camera (OV6211) +stuck "streaming": its log had no "Stopping streaming" line, and every later +request failed with "Failed to start streaming". Head tracking kept working. +Clearing it needs the DSP service restarted or the Frame rebooted, so `pulse` +never signals the tool. After Ctrl-C or a failure it keeps deleting images +until the tool ends by itself (at most the `--seconds` plus a few seconds), +and only kills a tool that overruns by 30 s, with a warning that the eye +cameras may need a reboot. So an interrupted run can take a while to return. + **Verified on synthetic data** (unit tests): a 0.3% brightness pulse in a third of the patches, with noise, drift, blinks and eye movement, is recovered within 1.5 BPM at 58, 72 and 115 BPM; noise and blinks alone are diff --git a/frame/tracking/pulse.py b/frame/tracking/pulse.py index 6878bff..1e73444 100644 --- a/frame/tracking/pulse.py +++ b/frame/tracking/pulse.py @@ -71,7 +71,7 @@ class Capture: # Only SteamVR's own capture directories are read and removed. PREFIX = "/tmp/etcalib_" # Printed by the capture tool; its output is only flushed when it exits. - WRITING = re.compile(r"Writing capture to: (/tmp/etcalib_[\w-]+)") + WRITING = re.compile(r"Writing capture to: (\S+)") # About five seconds of frames (~65 MB in RAM-backed /tmp). If reduction # falls further behind than this, the capture stops rather than letting # eye images pile up. @@ -85,6 +85,8 @@ class Capture: self.submitted = set() self.futures = {} self.new = set() # every capture directory that appeared during our run + self.before = set() + self.log = None def candidates(self): parent, stem = os.path.split(self.PREFIX) @@ -96,9 +98,10 @@ class Capture: # failure messages. It is block-buffered, so the capture directory is # found by watching for a new one rather than waiting for its name. import tempfile - before = self.candidates() + self.before = before = self.candidates() process = None with tempfile.TemporaryFile("w+") as log: + self.log = log try: if self.workers: # SteamVR pins its eye tracker to cores 0-1; decoding runs beside it. @@ -136,29 +139,70 @@ class Capture: finally: # Each step runs even if an earlier one failed: eye images must # be removed whatever else went wrong. - failed = False - for step in (lambda: self.stop(process), - lambda: self.pool and self.pool.shutdown(wait=True, cancel_futures=True)): + # Ctrl-C and SIGTERM (raised as KeyboardInterrupt) are held + # until the images are gone, then re-raised. + interrupted, failed = None, None + for step in (lambda: self.finish(process), + lambda: self.pool and self.pool.shutdown(wait=True, cancel_futures=True), + self.adopt_reported): try: step() - except Exception: - failed = True - named = self.written(log) - if named: - self.new.add(named) # ours by the tool's own account + except BaseException as error: + if isinstance(error, Exception): + failed = failed or error + else: + interrupted = interrupted or error + self.log = None self.remove() + if interrupted: + raise interrupted if failed: - raise RuntimeError("the eye-camera capture did not stop cleanly") + raise RuntimeError(f"the eye-camera capture did not stop cleanly: {failed}") - @staticmethod - def stop(process): - if process and process.poll() is None: - process.terminate() + def finish(self, process): + """Let the capture tool end by itself, deleting its images meanwhile. + + Never signal it: stopping the tool early leaves the headset's eye + camera stuck streaming until the DSP service restarts (verified on the + Frame with SIGTERM). Its run is bounded by `seconds`, so waiting is + short. Only if it overruns by a wide margin is it killed.""" + if not process: + return + interrupted = None + give_up = time.monotonic() + self.seconds + 30 + while True: try: - process.wait(timeout=5) - except subprocess.TimeoutExpired: - process.kill() - process.wait() + if process.poll() is not None: + break + self.discard() + if time.monotonic() > give_up: + process.kill() + process.wait() + raise RuntimeError("the eye-camera capture had to be killed; " + "the eye cameras may need a reboot to stream again") + time.sleep(0.05) + except KeyboardInterrupt as error: # keep cleaning up until it ends + interrupted = interrupted or error + if interrupted: + raise interrupted + + def discard(self): + """Delete the eye images written so far, without reading them.""" + self.new |= self.candidates() - self.before + for directory in self.new | ({self.directory} if self.directory else set()): + if str(directory).startswith(self.PREFIX) and directory.is_dir() and not directory.is_symlink(): + for entry in os.scandir(directory): + if self.NAME.match(entry.name): + try: + os.unlink(entry.path) + except OSError: + pass + + def adopt_reported(self): + """The directory the tool reported is ours by its own account.""" + named = self.written(self.log) + if named and str(named).startswith(self.PREFIX): + self.new.add(named) def written(self, log): """The directory the capture tool reported, once its output is flushed.""" @@ -177,7 +221,7 @@ class Capture: if match: pending[match.group(1)].append((int(match.group(2)), entry.path)) if sum(len(files) for files in pending.values()) > self.MAX_BACKLOG: - raise RuntimeError("eye-image processing fell behind; capture stopped") + raise RuntimeError("eye-image processing fell behind; capture abandoned") for eye, files in pending.items(): files.sort() ready = files if final else files[:-1] diff --git a/scripts/heart-check.py b/scripts/heart-check.py index 1f0c3cd..048c429 100644 --- a/scripts/heart-check.py +++ b/scripts/heart-check.py @@ -118,11 +118,11 @@ def read_csv(path): continue try: when, bpm = parse_time(row[0]), int(float(row[1])) - except ValueError: + except (ValueError, OverflowError): continue # header or unparseable line try: flags = int(float(row[2])) if len(row) > 2 else None - except ValueError: + except (ValueError, OverflowError): flags = None # an unrecognised flag leaves the reading as it is if flags is not None and flags & 4 and not flags & 2: bpm = None # contact supported and not detected: no reading @@ -258,13 +258,13 @@ def main(argv=None): count = listen(args.port, args.address, args.out, args.seconds) print(f"Received {count} readings.") return 0 if count else 3 - ours = read_csv(args.ours) - if not ours: - parser.error("our recording has no readings") try: + ours = read_csv(args.ours) + if not ours: + raise ValueError("our recording has no readings") reference = read_any(args.reference, ours[0][0] - 60, ours[-1][0] + 60) result = compare(ours, reference, args.max_lag) - except (ValueError, OSError, ElementTree.ParseError, zipfile.BadZipFile) as error: + except (ValueError, OSError, csv.Error, ElementTree.ParseError, zipfile.BadZipFile) as error: print(f"Could not compare: {error}") return 1 return 0 if report(result, args.tolerance) else 1 diff --git a/tests/test_pulse.py b/tests/test_pulse.py index 10a8ad1..06c73b3 100644 --- a/tests/test_pulse.py +++ b/tests/test_pulse.py @@ -168,7 +168,9 @@ class FakeCaptureTool: return 0 def terminate(self): - self.returncode = -15 + raise AssertionError("the capture tool must never be signalled") + + kill = terminate def wait(self, timeout=None): return self.returncode @@ -194,7 +196,6 @@ class CaptureLifecycle(unittest.TestCase): class TestCapture(pulse.Capture): # A temporary directory stands in for /tmp/etcalib_*. PREFIX = str(self.root / "etcalib_") - WRITING = __import__("re").compile(r"Writing capture to: (\S+)") capture = TestCapture(20, runner=tool, loader=self.loader, workers=0) with patch.object(pulse.time, "sleep", lambda s: None): return capture, capture.run() @@ -215,9 +216,12 @@ class CaptureLifecycle(unittest.TestCase): self.capture(tool) self.assertIn("cameras unavailable", str(raised.exception)) - def test_backlog_stops_capture_and_removes_images(self): + def test_backlog_abandons_capture_and_removes_images(self): class Stalled(FakeCaptureTool): def poll(self): + if self.polls > 8: + self.returncode = 0 # the bounded run ends by itself + return 0 for i in range(20): for eye in ("left", "right"): (self.directory / f"{eye}_{self.polls * 20 + i}.png").write_bytes(b"png") @@ -226,7 +230,6 @@ class CaptureLifecycle(unittest.TestCase): tool = Stalled(self.directory) class TestCapture(pulse.Capture): PREFIX = str(self.root / "etcalib_") - WRITING = __import__("re").compile(r"Writing capture to: (\S+)") MAX_BACKLOG = 50 capture = TestCapture(20, runner=tool, loader=self.loader, workers=0) capture.reduce = lambda final=False, original=capture.reduce: ( @@ -234,7 +237,7 @@ class CaptureLifecycle(unittest.TestCase): with patch.object(pulse.time, "sleep", lambda s: None), self.assertRaises(RuntimeError) as raised: capture.run() self.assertIn("fell behind", str(raised.exception)) - self.assertEqual(tool.returncode, -15) # capture tool stopped + self.assertEqual(tool.returncode, 0) # left to end by itself, never signalled self.assertFalse(self.directory.exists()) # no image left behind def test_second_capture_directory_stops_and_removes_both(self): @@ -264,6 +267,37 @@ class CaptureLifecycle(unittest.TestCase): self.assertFalse(tool.announce.exists()) self.assertFalse(self.directory.exists()) + def test_interrupt_during_cleanup_still_removes_images(self): + class Stubborn(FakeCaptureTool): + interrupts = 0 + def poll(self): + if self.polls == 2 and self.interrupts < 2: + self.interrupts += 1 + raise KeyboardInterrupt # Ctrl-C mid-capture, and again while waiting + return super().poll() + tool = Stubborn(self.directory) + with self.assertRaises(KeyboardInterrupt): + self.capture(tool) + self.assertEqual(tool.returncode, 0) # still let finish, never signalled + self.assertFalse(self.directory.exists()) + + def test_overrunning_tool_is_killed_as_a_last_resort(self): + class Hung(FakeCaptureTool): + killed = False + def poll(self): + (self.directory / "left_0.png").write_bytes(b"png") + return None if not self.killed else -9 + def kill(self): + self.killed = True + tool = Hung(self.directory) + clock = iter(range(0, 10000, 20)) + with patch.object(pulse.time, "monotonic", lambda: next(clock)), \ + self.assertRaises(RuntimeError) as raised: + self.capture(tool) + self.assertIn("may need a reboot", str(raised.exception)) + self.assertTrue(tool.killed) + self.assertFalse(self.directory.exists()) + def test_cleanup_tries_every_directory(self): capture = pulse.Capture(20) capture.PREFIX = str(self.root / "etcalib_") @@ -371,6 +405,15 @@ class HeartCheck(unittest.TestCase): with self.assertRaises(ValueError): check.read_any(archive, 0, 1) + def test_compare_reports_bad_input_without_traceback(self): + good = self.write("ref.csv", "1000,70\n") + shown = io.StringIO() + with patch("sys.stdout", shown): + self.assertEqual(check.main(["compare", str(good.parent / "missing.csv"), str(good)]), 1) + self.assertEqual(check.main(["compare", str(self.write("ours.csv", "1000,inf\n1001,70\n")), + str(self.write("bad.xml", "