diff --git a/docs/img/media-ui-panel.png b/docs/img/media-ui-panel.png new file mode 100644 index 0000000..2e2a41b Binary files /dev/null and b/docs/img/media-ui-panel.png differ diff --git a/docs/vr-video.md b/docs/vr-video.md index 1e242a6..2232915 100644 --- a/docs/vr-video.md +++ b/docs/vr-video.md @@ -75,6 +75,34 @@ All test media were generated by us. No paid content or DRM was involved. ![Our Gaussian-splat stereo preview on the Frame](img/media-splat-proof.png) +**Verified end to end, 2026-09-29** (same build; headset unworn): uploads +through the HTTP API, the web UI and `scripts/push-vr-video.sh`, all played by +the owned player. Our generated test files: + +| Case | Result | +|---|---| +| H.264 half-SBS 1920×1080 with AAC, theatre | 240/240 frames in 8.09 s; audio stream "Frame Control Media" in PulseAudio | +| H.265 half-OU 1920×1080 | 180/180 frames in 6.03 s; red left eye, cyan right | +| H.264 full-SBS 3840×1080, `stereo_mode=left_right` only | Detected from metadata; 150/150 frames in 5.03 s | +| H.264 1280×720, explicit 2D | 150/150 frames in 5.02 s | +| SBS PNG, OU JPEG (theatre) | Correct eye in each capture | +| 3,000-Gaussian `.splat` | Rendered in about 5 s, then held until Stop | +| 2D file on Auto, `_SBS_OU` file, HEIC, VP9 | Refused with the documented message | +| Second Play while one runs | Refused: "Stop the current media…" | + +Stop always left the unit inactive, and no player process remained. + +**Standby (verified):** an unworn Frame turns its displays off a few seconds +after it wakes. `SetOverlayRaw` then returns `RequestFailed` (23). The +first run's movie died there. The player now drops frames while the headset +is in standby, keeps the audio and its clock going, and resumes the picture +when the headset wakes. The 8 s movie above dropped 40 frames and finished. +Stills and the theatre surround are re-sent after waking. Five minutes +without an accepted frame is reported as an error. Headset-view captures +taken during standby show a flat dark frame, not our screen. + +![Media panel in Frame Control while a photo plays on the Frame](img/media-ui-panel.png) + **Verified failed route:** GStreamer 1.24.2's `playbin` selected `v4l2h264dec`, delivered the first RGBA sample and then segfaulted (exit 139) in the basic appsink probe and the OpenVR probe. We do not ship that route. diff --git a/tests/test_media.py b/tests/test_media.py index 9665c27..f277341 100644 --- a/tests/test_media.py +++ b/tests/test_media.py @@ -1,6 +1,9 @@ """Owned media planning, eye isolation, decoder choice and fake-Frame ownership.""" +import argparse +import io import json import os +import signal from pathlib import Path import struct import sys @@ -16,6 +19,13 @@ import frame_splat as splat import server +def stop_now(): + """What systemd's SIGTERM does to the player, without signalling the test process.""" + handler = signal.getsignal(signal.SIGTERM) + if callable(handler): + handler(signal.SIGTERM, None) + + class Media(unittest.TestCase): def test_layout_evidence_and_override(self): for name, layout in [('film_SBS.mp4', 'sbs'), ('film.OU.mkv', 'ou'), @@ -56,6 +66,156 @@ class Media(unittest.TestCase): self.assertNotIn('-re', cmd) self.assertIn('-frames:v', cmd) + def play_with(self, name, busy=0, on_pixels=None, sleeps=None, info=None, + fail=None, layout='auto'): + """Run the player against a fake OpenVR; returns (status, pixels calls). + + Handle 0 is the theatre surround and 1 the screen. `fail(handle, n)` makes + the n-th upload busy; every status written is kept in self.writes.""" + calls = [] + self.writes = [] + write_status = player.write_status + + def record(path, **values): + self.writes.append(values) + write_status(path, **values) + + class FakeOverlay: + def create(self, *a, **k): + return len(calls) + + def call(self, *a): + pass + + def pixels(self, handle, data, w, h): + calls.append((handle, w, h)) + if on_pixels: + on_pixels(len(calls)) + if len(calls) <= busy or (fail and fail(handle, len(calls))): + raise player.OverlayBusy('standby') + + def close(self): + # A Stop landing during cleanup must be ignored, not become an error. + # Call the installed handler directly: a real SIGTERM kills Windows. + stop_now() + + frame = bytes(4*2*4) + proc = unittest.mock.MagicMock() + proc.stdout = io.BytesIO(frame*4) + proc.wait.return_value = 0 + proc.poll.return_value = 0 + old = signal.getsignal(signal.SIGTERM), signal.getsignal(signal.SIGINT) + with tempfile.TemporaryDirectory() as d, \ + patch.object(player, 'Overlay', FakeOverlay), \ + patch.object(player, 'probe', return_value=(info or {'codec_name': 'h264', 'width': 4, 'height': 2}, False)), \ + patch.object(player.subprocess, 'Popen', return_value=proc), \ + patch.object(player, 'write_status', side_effect=record), \ + patch.object(player.frame_splat, 'render', return_value=(bytes(4*4*2), 4, 2)), \ + patch.object(player.time, 'sleep', side_effect=sleeps): + path = Path(d)/name + path.write_bytes(b'x') + status = Path(d)/'status.json' + try: + player.play(argparse.Namespace(file=str(path), layout=layout, theatre=True, status=str(status))) + finally: + signal.signal(signal.SIGTERM, old[0]) + signal.signal(signal.SIGINT, old[1]) + return json.loads(status.read_text()), calls + + def test_video_survives_standby_and_stop_after_end_stays_ended(self): + # Verified 2026-09-29: an unworn Frame enters standby within seconds and + # SetOverlayRaw then returns RequestFailed (23) until it wakes. + result, calls = self.play_with('clip_SBS.mp4', busy=3) + self.assertEqual((result['state'], result['frames']), ('ended', 4)) + # Two video frames were dropped; the surround (handle 0) waited and was re-sent. + self.assertEqual(result['dropped'], 2) + self.assertIn((0, 1, 1), calls[3:]) + + def test_stop_mid_video_reports_stopped(self): + result, _ = self.play_with('clip_SBS.mp4', on_pixels=lambda n: n == 3 and stop_now()) + self.assertEqual(result['state'], 'stopped') + + def test_video_errors_when_steamvr_never_takes_frames(self): + with patch.object(player, 'BUSY_LIMIT', -1), \ + self.assertRaisesRegex(RuntimeError, 'stopped accepting frames'): + self.play_with('clip_SBS.mp4', busy=99) + + def test_still_waits_out_standby_without_a_limit(self): + # Stills have no timeline: keep retrying (here past BUSY_LIMIT) until shown. + ticks = iter(range(10)) + def sleep(_): + if next(ticks) == 8: + stop_now() + with patch.object(player, 'BUSY_LIMIT', -1): + result, calls = self.play_with('photo_SBS.png', busy=5, sleeps=sleep, + info={'codec_name': 'png', 'width': 4, 'height': 2}) + self.assertEqual(result['state'], 'stopped') + screen = [c for c in calls if c[0] == 1] + self.assertGreater(len(screen), 1) # retried through standby + self.assertEqual(calls[-1], (0, 1, 1)) # surround drained once the screen took a frame + + def still_with_late_surround(self, name, **kw): + # The screen takes its first frame while the surround is still refused + # (its first upload, the drain right after the screen, and one retry). + ticks = iter(range(10)) + def sleep(_): + if next(ticks) == 5: + stop_now() + surround_tries = [] + def fail(handle, n): + if handle == 0: + surround_tries.append(n) + return len(surround_tries) <= 3 + return False + result, calls = self.play_with(name, sleeps=sleep, fail=fail, **kw) + self.assertEqual(result['state'], 'stopped') + screen = [c for c in calls if c[0] == 1] + surround = [c for c in calls if c[0] == 0] + self.assertEqual(len(screen), 1) # shown once, not re-sent every second + self.assertEqual(len(surround), 4) # kept retrying after the screen, until it took + self.assertEqual(calls[-1][0], 0) + return calls + + def test_photo_surround_recovers_after_screen_is_shown(self): + self.still_with_late_surround('photo_SBS.png', + info={'codec_name': 'png', 'width': 4, 'height': 2}) + + def test_splat_surround_recovers_after_screen_is_shown(self): + self.still_with_late_surround('scene.splat') + + def test_video_standby_limit_is_five_minutes_without_an_accepted_frame(self): + self.assertEqual(player.BUSY_LIMIT, 300) + def run(times, busy): + # Upload n happens at times[n] seconds on a fake clock; 1 is the surround. + clock = [1000.0] + def on_pixels(n): + clock[0] = 1000.0 + times.get(n, times[max(times)]) + with patch.object(player.time, 'monotonic', side_effect=lambda: clock[0]): + return self.play_with('clip_SBS.mp4', on_pixels=on_pixels, + fail=lambda h, n: h == 1 and n in busy) + # Busy for 299 s, then a frame lands: no error. + result, _ = run({1: 0, 2: 0, 3: 299, 4: 299, 5: 299}, busy={2, 3}) + self.assertEqual((result['state'], result['dropped']), ('ended', 2)) + # An accepted frame resets the timer: 600 s busy in total, never 300 s in a row. + result, _ = run({1: 0, 2: 0, 3: 200, 4: 250, 5: 450}, busy={2, 3, 5}) + self.assertEqual((result['state'], result['dropped']), ('ended', 3)) + # 301 s in a row without an accepted frame is an error. + with self.assertRaisesRegex(RuntimeError, 'stopped accepting frames for 300 s'): + run({1: 0, 2: 0, 3: 301}, busy={2, 3, 4, 5}) + + def test_status_reports_where_the_layout_came_from(self): + video = {'codec_name': 'h264', 'width': 4, 'height': 2} + for name, layout, tags, expect in [ + ('clip_SBS.mp4', 'auto', None, ('sbs', 'filename')), + ('clip.mkv', 'auto', {'stereo_mode': 'left_right'}, ('full-sbs', 'metadata')), + ('clip_OU.mp4', 'sbs', None, ('sbs', 'explicit')), + ('clip.mkv', 'mono', {'stereo_mode': 'left_right'}, ('mono', 'explicit'))]: + with self.subTest(name=name, layout=layout): + self.play_with(name, layout=layout, info=dict(video, tags=tags) if tags else video) + playing = self.writes[0] + self.assertEqual(playing['state'], 'playing') + self.assertEqual((playing['layout'], playing['source']), expect) + def test_fake_frame_library_and_traversal(self): with tempfile.TemporaryDirectory() as d, patch.object(remote, 'ROOT', Path(d)): identity = 'a'*32+'/space and quote\'.png' diff --git a/ui/frame_media_player.py b/ui/frame_media_player.py index a6d2bdc..5227da2 100644 --- a/ui/frame_media_player.py +++ b/ui/frame_media_player.py @@ -33,6 +33,15 @@ SLOTS = { } +class OverlayBusy(RuntimeError): + """SetOverlayRaw's RequestFailed (23): SteamVR isn't taking frames, e.g. the + unworn headset is in standby (verified 2026-09-29). Transient, not fatal.""" + + +# A screen that can't take a frame for this long is broken, not asleep. +BUSY_LIMIT = 300 + + class Overlay: def __init__(self): self.handles = [] @@ -53,6 +62,8 @@ class Overlay: def call(self, name, *values): slot, args = SLOTS[name] rc = C.CFUNCTYPE(C.c_int, *args)(self.table[slot])(*values) + if rc == 23 and name == 'SetOverlayRaw': + raise OverlayBusy('SteamVR is not accepting frames (standby?)') if rc: raise RuntimeError('OpenVR %s failed: %s' % (name, rc)) @@ -75,11 +86,14 @@ class Overlay: self.call('ShowOverlay', handle) def close(self): - try: - for h in reversed(self.handles): + # Best effort: SteamVR removes a disconnected client's overlays anyway, + # and a teardown error must not overwrite a finished playback's status. + for h in reversed(self.handles): + try: self.call('DestroyOverlay', h) - finally: - self.vr.VR_ShutdownInternal() + except RuntimeError: + pass + self.vr.VR_ShutdownInternal() def probe(path): @@ -114,6 +128,11 @@ def decoder_command(path, info, width, height, audio, photo=False): return cmd +def ignore_signals(): + signal.signal(signal.SIGTERM, signal.SIG_IGN) + signal.signal(signal.SIGINT, signal.SIG_IGN) + + def write_status(path, **values): tmp = path.with_suffix('.tmp') tmp.write_text(json.dumps(values)) @@ -135,6 +154,46 @@ def play(args): photo = plan['kind'] == 'photo' command = decoder_command(path, info, width, height, audio, photo) vr, proc, frames, started = None, None, 0, time.monotonic() + dropped, busy_since, pending = 0, None, [] + + def show(handle, data, w, h, video=False): + """Submit a frame. During standby return False; a video frame is dropped.""" + nonlocal dropped, busy_since + try: + vr.pixels(handle, data, w, h) + except OverlayBusy: + if not video: + return False # stills and the surround just wait; nothing is lost + dropped += 1 + busy_since = busy_since or time.monotonic() + if time.monotonic() - busy_since > BUSY_LIMIT: + raise RuntimeError('SteamVR stopped accepting frames for %d s' % BUSY_LIMIT) + return False + if video: + busy_since = None + drain() + return True + + def drain(): + """Re-send anything that arrived during standby (e.g. the theatre surround).""" + while pending: + item = pending.pop(0) + try: + vr.pixels(*item) + except OverlayBusy: + pending.insert(0, item) + return + + def hold(handle, data, w, h): + """Keep a still (photo or splat) up until Stop, retrying through standby.""" + shown = False + while True: + if shown: + drain() + else: + shown = show(handle, data, w, h) + time.sleep(1) + # systemd sends SIGTERM to the whole unit, including ffmpeg. Python unwinds # ownership; no unrelated Steam/SteamVR process or setting is touched. def stop(signum, frame): @@ -146,14 +205,13 @@ def play(args): if args.theatre: surround = vr.create('framecontrol.media.surround', 40, 4, order=0) vr.call('SetOverlayAlpha', surround, .85) - vr.pixels(surround, b'\x00\x00\x00\xff', 1, 1) + if not show(surround, b'\x00\x00\x00\xff', 1, 1): + pending.append((surround, b'\x00\x00\x00\xff', 1, 1)) screen = vr.create('framecontrol.media.screen', 3 if args.theatre else 1.6, 2, plan['layout'] != 'mono', aspect) if splat: - vr.pixels(screen, data, width, height) write_status(status, state='playing', file=path.name, frames=1, **plan) - while True: - time.sleep(1) + hold(screen, data, width, height) proc = subprocess.Popen(command, stdout=subprocess.PIPE) video_start = time.monotonic() while True: @@ -163,11 +221,14 @@ def play(args): data, outw, outh = frame_media.stereo_pixels(data, width, height, plan['layout']) if not photo: time.sleep(max(0, video_start + frames/30 - time.monotonic())) - vr.pixels(screen, data, outw, outh) + if photo: + still = data, outw, outh + else: + show(screen, data, outw, outh, video=True) frames += 1 if frames == 1 or frames % 30 == 0: write_status(status, state='playing', file=path.name, frames=frames, - seconds=time.monotonic()-started, **plan) + dropped=dropped, seconds=time.monotonic()-started, **plan) if not photo: time.sleep(max(0, video_start + frames/30 - time.monotonic())) rc = proc.wait(timeout=10) @@ -176,12 +237,17 @@ def play(args): if not frames: raise RuntimeError('Decoder produced no frames') if photo: - while True: - time.sleep(1) - write_status(status, state='ended', frames=frames, seconds=time.monotonic()-started) + hold(screen, *still) + # From here on a Stop can't change the outcome; don't let it turn + # 'ended' into an error while we write status and clean up. + ignore_signals() + write_status(status, state='ended', frames=frames, dropped=dropped, + seconds=time.monotonic()-started) except InterruptedError: - write_status(status, state='stopped', frames=frames) + ignore_signals() + write_status(status, state='stopped', frames=frames, dropped=dropped) finally: + ignore_signals() if proc: if proc.poll() is None: proc.terminate() diff --git a/ui/frame_media_remote.py b/ui/frame_media_remote.py index 3b70bfe..fc0b2ae 100644 --- a/ui/frame_media_remote.py +++ b/ui/frame_media_remote.py @@ -88,7 +88,7 @@ def run(body): '--property=StandardOutput=append:'+str(RUNTIME/'player.log'), '--property=StandardError=append:'+str(RUNTIME/'player.log'), 'python3', str(RUNTIME/'frame_media_player.py'), str(path), - '--layout', plan['layout'], '--status', str(STATUS)] + '--layout', body.get('layout', 'auto'), '--status', str(STATUS)] if body.get('theatre'): command.append('--theatre') started = subprocess.run(command, capture_output=True, text=True, timeout=15)