fix(tracking): never signal the eye-camera capture; finish cleanup exhaustively

Terminating eyetracking --calib left the DSP eye camera stuck streaming on the
Frame. Capture now lets the bounded run end by itself, deleting images
meanwhile, and only kills as a last resort. Also hardens Ctrl-C cleanup and
heart-check error handling.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>

docs/tracking.md

@@ -201,9 +201,9 @@ How it works:
   while the headset is worn and tracking.
 - **Privacy.** Each image is reduced to a 16×16 grid of patch averages as
   soon as it is complete, then deleted. Three worker processes do this beside
-  the capture. If more than 900 images (about five seconds) ever wait, the
-  capture stops rather than letting eye images accumulate. The capture directory is removed on
-  exit, even after errors. No image is kept or leaves the Frame. The estimate
+  the capture. If more than 900 images (about five seconds) ever wait, we stop
+  reading them and delete them undecoded until the capture ends, and report
+  an error. The capture directory is removed on exit, even after errors. No image is kept or leaves the Frame. The estimate
   is printed only with `--show`, and sent or saved only with `--osc` or
   `--log`, as for the strap.
 - **Estimate.** Patch traces are averaged down to 15 Hz and turned into
@@ -227,6 +227,16 @@ signal/noise (0.15-0.19), in two separate runs. That is a camera or illumination
 and the signal/noise gate kept it from being reported. A worn test should
 also record an unworn baseline, to rule out the same artifact.
 
+**Do not interrupt the capture. Verified on the Frame, 2026-09-29:** sending
+SIGTERM to `eyetracking --calib` left the DSP service's eye camera (OV6211)
+stuck "streaming": its log had no "Stopping streaming" line, and every later
+request failed with "Failed to start streaming". Head tracking kept working.
+Clearing it needs the DSP service restarted or the Frame rebooted, so `pulse`
+never signals the tool. After Ctrl-C or a failure it keeps deleting images
+until the tool ends by itself (at most the `--seconds` plus a few seconds),
+and only kills a tool that overruns by 30 s, with a warning that the eye
+cameras may need a reboot. So an interrupted run can take a while to return.
+
 **Verified on synthetic data** (unit tests): a 0.3% brightness pulse in a
 third of the patches, with noise, drift, blinks and eye movement, is
 recovered within 1.5 BPM at 58, 72 and 115 BPM; noise and blinks alone are

frame/tracking/pulse.py

@@ -71,7 +71,7 @@ class Capture:
     # Only SteamVR's own capture directories are read and removed.
     PREFIX = "/tmp/etcalib_"
     # Printed by the capture tool; its output is only flushed when it exits.
-    WRITING = re.compile(r"Writing capture to: (/tmp/etcalib_[\w-]+)")
+    WRITING = re.compile(r"Writing capture to: (\S+)")
     # About five seconds of frames (~65 MB in RAM-backed /tmp). If reduction
     # falls further behind than this, the capture stops rather than letting
     # eye images pile up.
@@ -85,6 +85,8 @@ class Capture:
         self.submitted = set()
         self.futures = {}
         self.new = set()  # every capture directory that appeared during our run
+        self.before = set()
+        self.log = None
 
     def candidates(self):
         parent, stem = os.path.split(self.PREFIX)
@@ -96,9 +98,10 @@ class Capture:
         # failure messages. It is block-buffered, so the capture directory is
         # found by watching for a new one rather than waiting for its name.
         import tempfile
-        before = self.candidates()
+        self.before = before = self.candidates()
         process = None
         with tempfile.TemporaryFile("w+") as log:
+            self.log = log
             try:
                 if self.workers:
                     # SteamVR pins its eye tracker to cores 0-1; decoding runs beside it.
@@ -136,29 +139,70 @@ class Capture:
             finally:
                 # Each step runs even if an earlier one failed: eye images must
                 # be removed whatever else went wrong.
-                failed = False
-                for step in (lambda: self.stop(process),
-                             lambda: self.pool and self.pool.shutdown(wait=True, cancel_futures=True)):
+                # Ctrl-C and SIGTERM (raised as KeyboardInterrupt) are held
+                # until the images are gone, then re-raised.
+                interrupted, failed = None, None
+                for step in (lambda: self.finish(process),
+                             lambda: self.pool and self.pool.shutdown(wait=True, cancel_futures=True),
+                             self.adopt_reported):
                     try:
                         step()
-                    except Exception:
-                        failed = True
-                named = self.written(log)
-                if named:
-                    self.new.add(named)  # ours by the tool's own account
+                    except BaseException as error:
+                        if isinstance(error, Exception):
+                            failed = failed or error
+                        else:
+                            interrupted = interrupted or error
+                self.log = None
                 self.remove()
+                if interrupted:
+                    raise interrupted
                 if failed:
-                    raise RuntimeError("the eye-camera capture did not stop cleanly")
+                    raise RuntimeError(f"the eye-camera capture did not stop cleanly: {failed}")
 
-    @staticmethod
-    def stop(process):
-        if process and process.poll() is None:
-            process.terminate()
+    def finish(self, process):
+        """Let the capture tool end by itself, deleting its images meanwhile.
+
+        Never signal it: stopping the tool early leaves the headset's eye
+        camera stuck streaming until the DSP service restarts (verified on the
+        Frame with SIGTERM). Its run is bounded by `seconds`, so waiting is
+        short. Only if it overruns by a wide margin is it killed."""
+        if not process:
+            return
+        interrupted = None
+        give_up = time.monotonic() + self.seconds + 30
+        while True:
             try:
-                process.wait(timeout=5)
-            except subprocess.TimeoutExpired:
-                process.kill()
-                process.wait()
+                if process.poll() is not None:
+                    break
+                self.discard()
+                if time.monotonic() > give_up:
+                    process.kill()
+                    process.wait()
+                    raise RuntimeError("the eye-camera capture had to be killed; "
+                                       "the eye cameras may need a reboot to stream again")
+                time.sleep(0.05)
+            except KeyboardInterrupt as error:  # keep cleaning up until it ends
+                interrupted = interrupted or error
+        if interrupted:
+            raise interrupted
+
+    def discard(self):
+        """Delete the eye images written so far, without reading them."""
+        self.new |= self.candidates() - self.before
+        for directory in self.new | ({self.directory} if self.directory else set()):
+            if str(directory).startswith(self.PREFIX) and directory.is_dir() and not directory.is_symlink():
+                for entry in os.scandir(directory):
+                    if self.NAME.match(entry.name):
+                        try:
+                            os.unlink(entry.path)
+                        except OSError:
+                            pass
+
+    def adopt_reported(self):
+        """The directory the tool reported is ours by its own account."""
+        named = self.written(self.log)
+        if named and str(named).startswith(self.PREFIX):
+            self.new.add(named)
 
     def written(self, log):
         """The directory the capture tool reported, once its output is flushed."""
@@ -177,7 +221,7 @@ class Capture:
             if match:
                 pending[match.group(1)].append((int(match.group(2)), entry.path))
         if sum(len(files) for files in pending.values()) > self.MAX_BACKLOG:
-            raise RuntimeError("eye-image processing fell behind; capture stopped")
+            raise RuntimeError("eye-image processing fell behind; capture abandoned")
         for eye, files in pending.items():
             files.sort()
             ready = files if final else files[:-1]

scripts/heart-check.py

@@ -118,11 +118,11 @@ def read_csv(path):
                 continue
             try:
                 when, bpm = parse_time(row[0]), int(float(row[1]))
-            except ValueError:
+            except (ValueError, OverflowError):
                 continue  # header or unparseable line
             try:
                 flags = int(float(row[2])) if len(row) > 2 else None
-            except ValueError:
+            except (ValueError, OverflowError):
                 flags = None  # an unrecognised flag leaves the reading as it is
             if flags is not None and flags & 4 and not flags & 2:
                 bpm = None  # contact supported and not detected: no reading
@@ -258,13 +258,13 @@ def main(argv=None):
         count = listen(args.port, args.address, args.out, args.seconds)
         print(f"Received {count} readings.")
         return 0 if count else 3
-    ours = read_csv(args.ours)
-    if not ours:
-        parser.error("our recording has no readings")
     try:
+        ours = read_csv(args.ours)
+        if not ours:
+            raise ValueError("our recording has no readings")
         reference = read_any(args.reference, ours[0][0] - 60, ours[-1][0] + 60)
         result = compare(ours, reference, args.max_lag)
-    except (ValueError, OSError, ElementTree.ParseError, zipfile.BadZipFile) as error:
+    except (ValueError, OSError, csv.Error, ElementTree.ParseError, zipfile.BadZipFile) as error:
         print(f"Could not compare: {error}")
         return 1
     return 0 if report(result, args.tolerance) else 1

tests/test_pulse.py

@@ -168,7 +168,9 @@ class FakeCaptureTool:
         return 0
 
     def terminate(self):
-        self.returncode = -15
+        raise AssertionError("the capture tool must never be signalled")
+
+    kill = terminate
 
     def wait(self, timeout=None):
         return self.returncode
@@ -194,7 +196,6 @@ class CaptureLifecycle(unittest.TestCase):
         class TestCapture(pulse.Capture):
             # A temporary directory stands in for /tmp/etcalib_*.
             PREFIX = str(self.root / "etcalib_")
-            WRITING = __import__("re").compile(r"Writing capture to: (\S+)")
         capture = TestCapture(20, runner=tool, loader=self.loader, workers=0)
         with patch.object(pulse.time, "sleep", lambda s: None):
             return capture, capture.run()
@@ -215,9 +216,12 @@ class CaptureLifecycle(unittest.TestCase):
             self.capture(tool)
         self.assertIn("cameras unavailable", str(raised.exception))
 
-    def test_backlog_stops_capture_and_removes_images(self):
+    def test_backlog_abandons_capture_and_removes_images(self):
         class Stalled(FakeCaptureTool):
             def poll(self):
+                if self.polls > 8:
+                    self.returncode = 0  # the bounded run ends by itself
+                    return 0
                 for i in range(20):
                     for eye in ("left", "right"):
                         (self.directory / f"{eye}_{self.polls * 20 + i}.png").write_bytes(b"png")
@@ -226,7 +230,6 @@ class CaptureLifecycle(unittest.TestCase):
         tool = Stalled(self.directory)
         class TestCapture(pulse.Capture):
             PREFIX = str(self.root / "etcalib_")
-            WRITING = __import__("re").compile(r"Writing capture to: (\S+)")
             MAX_BACKLOG = 50
         capture = TestCapture(20, runner=tool, loader=self.loader, workers=0)
         capture.reduce = lambda final=False, original=capture.reduce: (
@@ -234,7 +237,7 @@ class CaptureLifecycle(unittest.TestCase):
         with patch.object(pulse.time, "sleep", lambda s: None), self.assertRaises(RuntimeError) as raised:
             capture.run()
         self.assertIn("fell behind", str(raised.exception))
-        self.assertEqual(tool.returncode, -15)       # capture tool stopped
+        self.assertEqual(tool.returncode, 0)         # left to end by itself, never signalled
         self.assertFalse(self.directory.exists())    # no image left behind
 
     def test_second_capture_directory_stops_and_removes_both(self):
@@ -264,6 +267,37 @@ class CaptureLifecycle(unittest.TestCase):
         self.assertFalse(tool.announce.exists())
         self.assertFalse(self.directory.exists())
 
+    def test_interrupt_during_cleanup_still_removes_images(self):
+        class Stubborn(FakeCaptureTool):
+            interrupts = 0
+            def poll(self):
+                if self.polls == 2 and self.interrupts < 2:
+                    self.interrupts += 1
+                    raise KeyboardInterrupt  # Ctrl-C mid-capture, and again while waiting
+                return super().poll()
+        tool = Stubborn(self.directory)
+        with self.assertRaises(KeyboardInterrupt):
+            self.capture(tool)
+        self.assertEqual(tool.returncode, 0)         # still let finish, never signalled
+        self.assertFalse(self.directory.exists())
+
+    def test_overrunning_tool_is_killed_as_a_last_resort(self):
+        class Hung(FakeCaptureTool):
+            killed = False
+            def poll(self):
+                (self.directory / "left_0.png").write_bytes(b"png")
+                return None if not self.killed else -9
+            def kill(self):
+                self.killed = True
+        tool = Hung(self.directory)
+        clock = iter(range(0, 10000, 20))
+        with patch.object(pulse.time, "monotonic", lambda: next(clock)), \
+                self.assertRaises(RuntimeError) as raised:
+            self.capture(tool)
+        self.assertIn("may need a reboot", str(raised.exception))
+        self.assertTrue(tool.killed)
+        self.assertFalse(self.directory.exists())
+
     def test_cleanup_tries_every_directory(self):
         capture = pulse.Capture(20)
         capture.PREFIX = str(self.root / "etcalib_")
@@ -371,6 +405,15 @@ class HeartCheck(unittest.TestCase):
             with self.assertRaises(ValueError):
                 check.read_any(archive, 0, 1)
 
+    def test_compare_reports_bad_input_without_traceback(self):
+        good = self.write("ref.csv", "1000,70\n")
+        shown = io.StringIO()
+        with patch("sys.stdout", shown):
+            self.assertEqual(check.main(["compare", str(good.parent / "missing.csv"), str(good)]), 1)
+            self.assertEqual(check.main(["compare", str(self.write("ours.csv", "1000,inf\n1001,70\n")),
+                                         str(self.write("bad.xml", "<not closed"))]), 1)
+        self.assertEqual(shown.getvalue().count("Could not compare"), 2)
+
     def test_time_formats(self):
         self.assertEqual(check.parse_time("1700000000.5"), 1700000000.5)
         self.assertEqual(check.parse_time("2023-11-14T22:13:20Z"), 1700000000)