Merge pull request #49 from saphid/theatre-media-device-fixes

Media player: keep playing through headset standby (real-Frame test fixes)

docs/vr-video.md

@@ -75,6 +75,34 @@ All test media were generated by us. No paid content or DRM was involved.
 
 ![Our Gaussian-splat stereo preview on the Frame](img/media-splat-proof.png)
 
+**Verified end to end, 2026-09-29** (same build; headset unworn): uploads
+through the HTTP API, the web UI and `scripts/push-vr-video.sh`, all played by
+the owned player. Our generated test files:
+
+| Case | Result |
+|---|---|
+| H.264 half-SBS 1920×1080 with AAC, theatre | 240/240 frames in 8.09 s; audio stream "Frame Control Media" in PulseAudio |
+| H.265 half-OU 1920×1080 | 180/180 frames in 6.03 s; red left eye, cyan right |
+| H.264 full-SBS 3840×1080, `stereo_mode=left_right` only | Detected from metadata; 150/150 frames in 5.03 s |
+| H.264 1280×720, explicit 2D | 150/150 frames in 5.02 s |
+| SBS PNG, OU JPEG (theatre) | Correct eye in each capture |
+| 3,000-Gaussian `.splat` | Rendered in about 5 s, then held until Stop |
+| 2D file on Auto, `_SBS_OU` file, HEIC, VP9 | Refused with the documented message |
+| Second Play while one runs | Refused: "Stop the current media…" |
+
+Stop always left the unit inactive, and no player process remained.
+
+**Standby (verified):** an unworn Frame turns its displays off a few seconds
+after it wakes. `SetOverlayRaw` then returns `RequestFailed` (23). The
+first run's movie died there. The player now drops frames while the headset
+is in standby, keeps the audio and its clock going, and resumes the picture
+when the headset wakes. The 8 s movie above dropped 40 frames and finished.
+Stills and the theatre surround are re-sent after waking. Five minutes
+without an accepted frame is reported as an error. Headset-view captures
+taken during standby show a flat dark frame, not our screen.
+
+![Media panel in Frame Control while a photo plays on the Frame](img/media-ui-panel.png)
+
 **Verified failed route:** GStreamer 1.24.2's `playbin` selected
 `v4l2h264dec`, delivered the first RGBA sample and then segfaulted (exit 139)
 in the basic appsink probe and the OpenVR probe. We do not ship that route.

tests/test_media.py

@@ -1,6 +1,9 @@
 """Owned media planning, eye isolation, decoder choice and fake-Frame ownership."""
+import argparse
+import io
 import json
 import os
+import signal
 from pathlib import Path
 import struct
 import sys
@@ -16,6 +19,13 @@ import frame_splat as splat
 import server
 
 
+def stop_now():
+    """What systemd's SIGTERM does to the player, without signalling the test process."""
+    handler = signal.getsignal(signal.SIGTERM)
+    if callable(handler):
+        handler(signal.SIGTERM, None)
+
+
 class Media(unittest.TestCase):
     def test_layout_evidence_and_override(self):
         for name, layout in [('film_SBS.mp4', 'sbs'), ('film.OU.mkv', 'ou'),
@@ -56,6 +66,156 @@ class Media(unittest.TestCase):
         self.assertNotIn('-re', cmd)
         self.assertIn('-frames:v', cmd)
 
+    def play_with(self, name, busy=0, on_pixels=None, sleeps=None, info=None,
+                  fail=None, layout='auto'):
+        """Run the player against a fake OpenVR; returns (status, pixels calls).
+
+        Handle 0 is the theatre surround and 1 the screen. `fail(handle, n)` makes
+        the n-th upload busy; every status written is kept in self.writes."""
+        calls = []
+        self.writes = []
+        write_status = player.write_status
+
+        def record(path, **values):
+            self.writes.append(values)
+            write_status(path, **values)
+
+        class FakeOverlay:
+            def create(self, *a, **k):
+                return len(calls)
+
+            def call(self, *a):
+                pass
+
+            def pixels(self, handle, data, w, h):
+                calls.append((handle, w, h))
+                if on_pixels:
+                    on_pixels(len(calls))
+                if len(calls) <= busy or (fail and fail(handle, len(calls))):
+                    raise player.OverlayBusy('standby')
+
+            def close(self):
+                # A Stop landing during cleanup must be ignored, not become an error.
+                # Call the installed handler directly: a real SIGTERM kills Windows.
+                stop_now()
+
+        frame = bytes(4*2*4)
+        proc = unittest.mock.MagicMock()
+        proc.stdout = io.BytesIO(frame*4)
+        proc.wait.return_value = 0
+        proc.poll.return_value = 0
+        old = signal.getsignal(signal.SIGTERM), signal.getsignal(signal.SIGINT)
+        with tempfile.TemporaryDirectory() as d, \
+                patch.object(player, 'Overlay', FakeOverlay), \
+                patch.object(player, 'probe', return_value=(info or {'codec_name': 'h264', 'width': 4, 'height': 2}, False)), \
+                patch.object(player.subprocess, 'Popen', return_value=proc), \
+                patch.object(player, 'write_status', side_effect=record), \
+                patch.object(player.frame_splat, 'render', return_value=(bytes(4*4*2), 4, 2)), \
+                patch.object(player.time, 'sleep', side_effect=sleeps):
+            path = Path(d)/name
+            path.write_bytes(b'x')
+            status = Path(d)/'status.json'
+            try:
+                player.play(argparse.Namespace(file=str(path), layout=layout, theatre=True, status=str(status)))
+            finally:
+                signal.signal(signal.SIGTERM, old[0])
+                signal.signal(signal.SIGINT, old[1])
+            return json.loads(status.read_text()), calls
+
+    def test_video_survives_standby_and_stop_after_end_stays_ended(self):
+        # Verified 2026-09-29: an unworn Frame enters standby within seconds and
+        # SetOverlayRaw then returns RequestFailed (23) until it wakes.
+        result, calls = self.play_with('clip_SBS.mp4', busy=3)
+        self.assertEqual((result['state'], result['frames']), ('ended', 4))
+        # Two video frames were dropped; the surround (handle 0) waited and was re-sent.
+        self.assertEqual(result['dropped'], 2)
+        self.assertIn((0, 1, 1), calls[3:])
+
+    def test_stop_mid_video_reports_stopped(self):
+        result, _ = self.play_with('clip_SBS.mp4', on_pixels=lambda n: n == 3 and stop_now())
+        self.assertEqual(result['state'], 'stopped')
+
+    def test_video_errors_when_steamvr_never_takes_frames(self):
+        with patch.object(player, 'BUSY_LIMIT', -1), \
+                self.assertRaisesRegex(RuntimeError, 'stopped accepting frames'):
+            self.play_with('clip_SBS.mp4', busy=99)
+
+    def test_still_waits_out_standby_without_a_limit(self):
+        # Stills have no timeline: keep retrying (here past BUSY_LIMIT) until shown.
+        ticks = iter(range(10))
+        def sleep(_):
+            if next(ticks) == 8:
+                stop_now()
+        with patch.object(player, 'BUSY_LIMIT', -1):
+            result, calls = self.play_with('photo_SBS.png', busy=5, sleeps=sleep,
+                                           info={'codec_name': 'png', 'width': 4, 'height': 2})
+        self.assertEqual(result['state'], 'stopped')
+        screen = [c for c in calls if c[0] == 1]
+        self.assertGreater(len(screen), 1)  # retried through standby
+        self.assertEqual(calls[-1], (0, 1, 1))  # surround drained once the screen took a frame
+
+    def still_with_late_surround(self, name, **kw):
+        # The screen takes its first frame while the surround is still refused
+        # (its first upload, the drain right after the screen, and one retry).
+        ticks = iter(range(10))
+        def sleep(_):
+            if next(ticks) == 5:
+                stop_now()
+        surround_tries = []
+        def fail(handle, n):
+            if handle == 0:
+                surround_tries.append(n)
+                return len(surround_tries) <= 3
+            return False
+        result, calls = self.play_with(name, sleeps=sleep, fail=fail, **kw)
+        self.assertEqual(result['state'], 'stopped')
+        screen = [c for c in calls if c[0] == 1]
+        surround = [c for c in calls if c[0] == 0]
+        self.assertEqual(len(screen), 1)  # shown once, not re-sent every second
+        self.assertEqual(len(surround), 4)  # kept retrying after the screen, until it took
+        self.assertEqual(calls[-1][0], 0)
+        return calls
+
+    def test_photo_surround_recovers_after_screen_is_shown(self):
+        self.still_with_late_surround('photo_SBS.png',
+                                      info={'codec_name': 'png', 'width': 4, 'height': 2})
+
+    def test_splat_surround_recovers_after_screen_is_shown(self):
+        self.still_with_late_surround('scene.splat')
+
+    def test_video_standby_limit_is_five_minutes_without_an_accepted_frame(self):
+        self.assertEqual(player.BUSY_LIMIT, 300)
+        def run(times, busy):
+            # Upload n happens at times[n] seconds on a fake clock; 1 is the surround.
+            clock = [1000.0]
+            def on_pixels(n):
+                clock[0] = 1000.0 + times.get(n, times[max(times)])
+            with patch.object(player.time, 'monotonic', side_effect=lambda: clock[0]):
+                return self.play_with('clip_SBS.mp4', on_pixels=on_pixels,
+                                      fail=lambda h, n: h == 1 and n in busy)
+        # Busy for 299 s, then a frame lands: no error.
+        result, _ = run({1: 0, 2: 0, 3: 299, 4: 299, 5: 299}, busy={2, 3})
+        self.assertEqual((result['state'], result['dropped']), ('ended', 2))
+        # An accepted frame resets the timer: 600 s busy in total, never 300 s in a row.
+        result, _ = run({1: 0, 2: 0, 3: 200, 4: 250, 5: 450}, busy={2, 3, 5})
+        self.assertEqual((result['state'], result['dropped']), ('ended', 3))
+        # 301 s in a row without an accepted frame is an error.
+        with self.assertRaisesRegex(RuntimeError, 'stopped accepting frames for 300 s'):
+            run({1: 0, 2: 0, 3: 301}, busy={2, 3, 4, 5})
+
+    def test_status_reports_where_the_layout_came_from(self):
+        video = {'codec_name': 'h264', 'width': 4, 'height': 2}
+        for name, layout, tags, expect in [
+                ('clip_SBS.mp4', 'auto', None, ('sbs', 'filename')),
+                ('clip.mkv', 'auto', {'stereo_mode': 'left_right'}, ('full-sbs', 'metadata')),
+                ('clip_OU.mp4', 'sbs', None, ('sbs', 'explicit')),
+                ('clip.mkv', 'mono', {'stereo_mode': 'left_right'}, ('mono', 'explicit'))]:
+            with self.subTest(name=name, layout=layout):
+                self.play_with(name, layout=layout, info=dict(video, tags=tags) if tags else video)
+                playing = self.writes[0]
+                self.assertEqual(playing['state'], 'playing')
+                self.assertEqual((playing['layout'], playing['source']), expect)
+
     def test_fake_frame_library_and_traversal(self):
         with tempfile.TemporaryDirectory() as d, patch.object(remote, 'ROOT', Path(d)):
             identity = 'a'*32+'/space and quote\'.png'

ui/frame_media_player.py

@@ -33,6 +33,15 @@ SLOTS = {
 }
 
 
+class OverlayBusy(RuntimeError):
+    """SetOverlayRaw's RequestFailed (23): SteamVR isn't taking frames, e.g. the
+    unworn headset is in standby (verified 2026-09-29). Transient, not fatal."""
+
+
+# A screen that can't take a frame for this long is broken, not asleep.
+BUSY_LIMIT = 300
+
+
 class Overlay:
     def __init__(self):
         self.handles = []
@@ -53,6 +62,8 @@ class Overlay:
     def call(self, name, *values):
         slot, args = SLOTS[name]
         rc = C.CFUNCTYPE(C.c_int, *args)(self.table[slot])(*values)
+        if rc == 23 and name == 'SetOverlayRaw':
+            raise OverlayBusy('SteamVR is not accepting frames (standby?)')
         if rc:
             raise RuntimeError('OpenVR %s failed: %s' % (name, rc))
 
@@ -75,11 +86,14 @@ class Overlay:
         self.call('ShowOverlay', handle)
 
     def close(self):
-        try:
-            for h in reversed(self.handles):
+        # Best effort: SteamVR removes a disconnected client's overlays anyway,
+        # and a teardown error must not overwrite a finished playback's status.
+        for h in reversed(self.handles):
+            try:
                 self.call('DestroyOverlay', h)
-        finally:
-            self.vr.VR_ShutdownInternal()
+            except RuntimeError:
+                pass
+        self.vr.VR_ShutdownInternal()
 
 
 def probe(path):
@@ -114,6 +128,11 @@ def decoder_command(path, info, width, height, audio, photo=False):
     return cmd
 
 
+def ignore_signals():
+    signal.signal(signal.SIGTERM, signal.SIG_IGN)
+    signal.signal(signal.SIGINT, signal.SIG_IGN)
+
+
 def write_status(path, **values):
     tmp = path.with_suffix('.tmp')
     tmp.write_text(json.dumps(values))
@@ -135,6 +154,46 @@ def play(args):
         photo = plan['kind'] == 'photo'
         command = decoder_command(path, info, width, height, audio, photo)
     vr, proc, frames, started = None, None, 0, time.monotonic()
+    dropped, busy_since, pending = 0, None, []
+
+    def show(handle, data, w, h, video=False):
+        """Submit a frame. During standby return False; a video frame is dropped."""
+        nonlocal dropped, busy_since
+        try:
+            vr.pixels(handle, data, w, h)
+        except OverlayBusy:
+            if not video:
+                return False  # stills and the surround just wait; nothing is lost
+            dropped += 1
+            busy_since = busy_since or time.monotonic()
+            if time.monotonic() - busy_since > BUSY_LIMIT:
+                raise RuntimeError('SteamVR stopped accepting frames for %d s' % BUSY_LIMIT)
+            return False
+        if video:
+            busy_since = None
+        drain()
+        return True
+
+    def drain():
+        """Re-send anything that arrived during standby (e.g. the theatre surround)."""
+        while pending:
+            item = pending.pop(0)
+            try:
+                vr.pixels(*item)
+            except OverlayBusy:
+                pending.insert(0, item)
+                return
+
+    def hold(handle, data, w, h):
+        """Keep a still (photo or splat) up until Stop, retrying through standby."""
+        shown = False
+        while True:
+            if shown:
+                drain()
+            else:
+                shown = show(handle, data, w, h)
+            time.sleep(1)
+
     # systemd sends SIGTERM to the whole unit, including ffmpeg. Python unwinds
     # ownership; no unrelated Steam/SteamVR process or setting is touched.
     def stop(signum, frame):
@@ -146,14 +205,13 @@ def play(args):
         if args.theatre:
             surround = vr.create('framecontrol.media.surround', 40, 4, order=0)
             vr.call('SetOverlayAlpha', surround, .85)
-            vr.pixels(surround, b'\x00\x00\x00\xff', 1, 1)
+            if not show(surround, b'\x00\x00\x00\xff', 1, 1):
+                pending.append((surround, b'\x00\x00\x00\xff', 1, 1))
         screen = vr.create('framecontrol.media.screen', 3 if args.theatre else 1.6, 2,
                            plan['layout'] != 'mono', aspect)
         if splat:
-            vr.pixels(screen, data, width, height)
             write_status(status, state='playing', file=path.name, frames=1, **plan)
-            while True:
-                time.sleep(1)
+            hold(screen, data, width, height)
         proc = subprocess.Popen(command, stdout=subprocess.PIPE)
         video_start = time.monotonic()
         while True:
@@ -163,11 +221,14 @@ def play(args):
             data, outw, outh = frame_media.stereo_pixels(data, width, height, plan['layout'])
             if not photo:
                 time.sleep(max(0, video_start + frames/30 - time.monotonic()))
-            vr.pixels(screen, data, outw, outh)
+            if photo:
+                still = data, outw, outh
+            else:
+                show(screen, data, outw, outh, video=True)
             frames += 1
             if frames == 1 or frames % 30 == 0:
                 write_status(status, state='playing', file=path.name, frames=frames,
-                             seconds=time.monotonic()-started, **plan)
+                             dropped=dropped, seconds=time.monotonic()-started, **plan)
         if not photo:
             time.sleep(max(0, video_start + frames/30 - time.monotonic()))
         rc = proc.wait(timeout=10)
@@ -176,12 +237,17 @@ def play(args):
         if not frames:
             raise RuntimeError('Decoder produced no frames')
         if photo:
-            while True:
-                time.sleep(1)
-        write_status(status, state='ended', frames=frames, seconds=time.monotonic()-started)
+            hold(screen, *still)
+        # From here on a Stop can't change the outcome; don't let it turn
+        # 'ended' into an error while we write status and clean up.
+        ignore_signals()
+        write_status(status, state='ended', frames=frames, dropped=dropped,
+                     seconds=time.monotonic()-started)
     except InterruptedError:
-        write_status(status, state='stopped', frames=frames)
+        ignore_signals()
+        write_status(status, state='stopped', frames=frames, dropped=dropped)
     finally:
+        ignore_signals()
         if proc:
             if proc.poll() is None:
                 proc.terminate()

ui/frame_media_remote.py

@@ -88,7 +88,7 @@ def run(body):
                '--property=StandardOutput=append:'+str(RUNTIME/'player.log'),
                '--property=StandardError=append:'+str(RUNTIME/'player.log'),
                'python3', str(RUNTIME/'frame_media_player.py'), str(path),
-               '--layout', plan['layout'], '--status', str(STATUS)]
+               '--layout', body.get('layout', 'auto'), '--status', str(STATUS)]
     if body.get('theatre'):
         command.append('--theatre')
     started = subprocess.run(command, capture_output=True, text=True, timeout=15)