Files
saphid--frame-control/tests/test_macview.py
T
saphidandClaude Opus 5.5 a3c6e5c984 Mac in the headset: stream Mac windows and screens into the Frame as panels
Frame Control can now show any Mac window, or a whole screen, as its own
SteamVR panel on the Steam Frame (Tools -> Mac in the headset, macOS only).
Place it with the SteamVR dashboard; the laser clicks and scrolls, and the
Mac's own keyboard types.

- mac/frame-mac-view (Swift, no dependencies): ScreenCaptureKit capture per
  window or display, VideoToolbox H.264 with low-latency rate control (JPEG
  fallback), a loopback HTTP/WebSocket server, CGEvent/AX input playback,
  and a display-awake assertion while anyone watches.
- ui/frame_macview.py: starts the agent, runs an ssh -R tunnel with a
  supervisor that reopens it on the same port, and launches a Chromium app
  window per stream on gamescope's :0, tagged with STEAM_GAME for its own panel.
- Frame Control's key never leaves the Mac: viewers get single-use,
  per-source tickets and reconnect keys that Stop revokes.
- ui/mac-view.html: WebCodecs decode, keyframe recovery, pointer/wheel/keys back.
- Bundled in the Mac app build; tests/test_macview.py builds and drives the
  agent on macOS.

Verified on the Frame (build 20260925.6191901) with the test pattern: panel
in about 1.5 s, about 60 fps, Mac-to-window about 11-17 ms, tunnel recovery
in 4 s. Laser input and real window capture still need a person in the headset.

Also commits the other thread's first-party rule (steam-frame skill) and
the first-party options table in docs/streaming.md.

Reviewed by GPT-6 Astra (xhigh, read-only) over six rounds; all findings fixed.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-09-28 13:33:46 +10:00

275 lines
11 KiB
Python

"""Mac in the headset (ui/frame_macview.py and the frame-mac-view agent).
The Python checks run anywhere. On macOS the agent is built and driven over
HTTP and WebSocket with its test pattern, which needs no Screen Recording
permission: status, the token, the viewer page, H.264 keyframes, pointer
input reaching the source, and closing.
Run: python3 -m unittest discover -s tests
"""
import base64
import http.client
import json
import os
import shutil
import socket
import struct
import subprocess
import sys
import tempfile
import time
import unittest
from pathlib import Path
ROOT = Path(__file__).resolve().parent.parent
sys.path.insert(0, str(ROOT / "ui"))
import frame_macview # noqa: E402
class Helpers(unittest.TestCase):
def test_panel_id_is_stable_and_in_range(self):
a = frame_macview.panel_id("window:123")
self.assertEqual(a, frame_macview.panel_id("window:123"))
self.assertNotEqual(a, frame_macview.panel_id("window:124"))
self.assertTrue(2_001_000_000 <= a < 2_002_000_000)
def test_fit_keeps_aspect_inside_the_panel(self):
self.assertEqual(frame_macview.fit(2560, 1440), (1920, 1080))
w, h = frame_macview.fit(800, 1600)
self.assertEqual(h, 1080)
self.assertAlmostEqual(w / h, 0.5, places=2)
@unittest.skipUnless(shutil.which("bash"), "needs bash")
def test_launch_script_parses(self):
r = subprocess.run(["bash", "-n"], input=frame_macview.LAUNCH, text=True, capture_output=True)
self.assertEqual(r.returncode, 0, r.stderr)
def test_show_checks_the_source_before_anything_else(self):
mv = frame_macview.MacView(["ssh"], lambda *a, **k: self.fail("no ssh"), "frame")
with self.assertRaises(frame_macview.MacViewError):
mv.show("rm -rf /")
def test_missing_browser_is_explained(self):
calls = []
def run(remote, stdin=None, timeout=30):
calls.append(remote)
e = RuntimeError("exit 3")
e.stdout = "NO_BROWSER\n"
raise e
mv = frame_macview.MacView(["ssh"], run, "frame")
mv.call = lambda path, **kw: {"screen": True, "ticket": "tk"}
mv.ensure_tunnel = lambda **kw: None
mv.remote_port = 47900
with self.assertRaises(frame_macview.MacViewError) as cm:
mv.show("window:5")
self.assertIn("Chromium", str(cm.exception))
self.assertTrue(calls[0].startswith("bash -s -- "))
def test_screen_permission_is_checked_before_the_headset(self):
mv = frame_macview.MacView(["ssh"], lambda *a, **k: self.fail("no ssh"), "frame")
mv.call = lambda path, **kw: {"screen": False}
with self.assertRaises(frame_macview.MacViewError) as cm:
mv.show("display:1")
self.assertIn("Screen Recording", str(cm.exception))
class WS:
"""A minimal WebSocket client (masked frames out, plain frames in)."""
def __init__(self, port, path):
self.s = socket.create_connection(("127.0.0.1", port), timeout=10)
key = base64.b64encode(os.urandom(16)).decode()
self.s.sendall(f"GET {path} HTTP/1.1\r\nHost: x\r\nUpgrade: websocket\r\nConnection: Upgrade\r\n"
f"Sec-WebSocket-Key: {key}\r\nSec-WebSocket-Version: 13\r\n\r\n".encode())
head = b""
while b"\r\n\r\n" not in head:
head += self.s.recv(1)
self.status = int(head.split()[1])
self.buf = b""
def _read(self, n):
while len(self.buf) < n:
chunk = self.s.recv(65536)
if not chunk:
raise EOFError
self.buf += chunk
out, self.buf = self.buf[:n], self.buf[n:]
return out
def recv(self):
b0, b1 = self._read(2)
n = b1 & 0x7F
if n == 126:
n = struct.unpack(">H", self._read(2))[0]
elif n == 127:
n = struct.unpack(">Q", self._read(8))[0]
return b0 & 0x0F, self._read(n)
def send_text(self, text):
data, mask = text.encode(), os.urandom(4)
head = bytes([0x81, 0x80 | len(data)]) if len(data) < 126 else bytes([0x81, 0xFE]) + struct.pack(">H", len(data))
self.s.sendall(head + mask + bytes(c ^ mask[i % 4] for i, c in enumerate(data)))
def close(self):
self.s.close()
@unittest.skipUnless(sys.platform == "darwin" and shutil.which("xcrun"), "the agent is macOS-only")
class Agent(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.tmp = tempfile.mkdtemp()
cls.bin = Path(cls.tmp) / "frame-mac-view"
r = subprocess.run(["/bin/sh", str(ROOT / "mac" / "frame-mac-view" / "build.sh"), str(cls.bin)],
capture_output=True, text=True, timeout=600)
if r.returncode: # a real failure on a Mac with Xcode: don't hide it as a skip
raise AssertionError("agent didn't build:\n" + (r.stderr or r.stdout)[-2000:])
cls.token = "t0ken-" + os.urandom(6).hex()
cls.proc = subprocess.Popen([str(cls.bin), "serve", "--port", "0", "--page", str(ROOT / "ui" / "mac-view.html"),
"--exit-on-eof"], env={**os.environ, "FRAME_MAC_VIEW_TOKEN": cls.token},
stdin=subprocess.PIPE, stdout=subprocess.PIPE, text=True)
line = cls.proc.stdout.readline()
cls.port = int(line.rsplit(":", 1)[1])
@classmethod
def tearDownClass(cls):
cls.proc.stdin.close() # --exit-on-eof
cls.proc.stdout.close()
try:
cls.proc.wait(5)
except subprocess.TimeoutExpired:
cls.proc.kill()
shutil.rmtree(cls.tmp, ignore_errors=True)
def get(self, path, method="GET"):
c = http.client.HTTPConnection("127.0.0.1", self.port, timeout=10)
c.request(method, path)
r = c.getresponse()
return r.status, r.read()
def test_token_is_required(self):
self.assertEqual(self.get("/status")[0], 403)
self.assertEqual(self.get("/status?k=wrong")[0], 403)
status, body = self.get(f"/status?k={self.token}")
self.assertEqual(status, 200)
self.assertIn("screen", json.loads(body))
def test_serves_the_viewer_page(self):
status, body = self.get(f"/view?k={self.token}&src=test")
self.assertEqual(status, 200)
self.assertIn(b"VideoDecoder", body)
def test_lists_displays(self):
status, body = self.get(f"/displays?k={self.token}")
self.assertEqual(status, 200)
self.assertIsInstance(json.loads(body)["displays"], list)
def ticket(self, src):
status, body = self.get(f"/ticket?k={self.token}&src={src}", method="POST")
self.assertEqual(status, 200)
return json.loads(body)["ticket"]
def test_ping_and_page_are_open_but_streams_are_not(self):
self.assertEqual(self.get("/ping"), (200, b"frame-mac-view"))
self.assertEqual(WS(self.port, "/stream?src=test").status, 403)
self.assertEqual(self.get("/ticket?src=test", method="POST")[0], 403)
def test_tickets_are_single_use_and_tied_to_one_source(self):
t = self.ticket("test")
self.assertEqual(WS(self.port, f"/stream?src=display:1&t={t}").status, 403) # wrong source
ws = WS(self.port, f"/stream?src=test&t={t}")
self.assertEqual(ws.status, 101)
hello = json.loads(ws.recv()[1])
self.assertEqual(hello["t"], "hello")
# Until the viewer acknowledges, a retry (the hello got lost) gets the
# same key, and replaces the first viewer rather than adding one.
retry = WS(self.port, f"/stream?src=test&t={t}")
self.assertEqual(retry.status, 101)
self.assertEqual(json.loads(retry.recv()[1])["r"], hello["r"])
time.sleep(0.3)
_, body = self.get(f"/status?k={self.token}")
self.assertEqual(len(json.loads(body)["streams"]), 1)
ws.close()
ws = retry
ws.send_text(json.dumps({"t": "ack"}))
time.sleep(0.3)
self.assertEqual(WS(self.port, f"/stream?src=test&t={t}").status, 403) # spent
again = WS(self.port, f"/stream?src=test&r={hello['r']}") # the viewer reconnecting
self.assertEqual(again.status, 101)
again.close()
ws.close()
# Stop revokes the reconnect key.
self.get(f"/close?k={self.token}&src=test", method="POST")
self.assertEqual(WS(self.port, f"/stream?src=test&r={hello['r']}").status, 403)
def test_stop_revokes_tickets_not_yet_used(self):
t = self.ticket("test")
self.get(f"/close?k={self.token}&src=test", method="POST")
self.assertEqual(WS(self.port, f"/stream?src=test&t={t}").status, 403)
def test_bad_frame_lengths_close_the_socket_not_the_agent(self):
ws = WS(self.port, f"/stream?src=test&t={self.ticket('test')}")
self.assertEqual(ws.status, 101)
# A masked frame claiming 2^63 bytes.
ws.s.sendall(bytes([0x81, 0xFF]) + struct.pack(">Q", 1 << 63) + os.urandom(4))
with self.assertRaises((EOFError, OSError)):
for _ in range(1000):
ws.recv()
ws.close()
self.assertEqual(self.get(f"/status?k={self.token}")[0], 200)
def test_stream_input_and_close(self):
ws = WS(self.port, f"/stream?k={self.token}&src=test&codec=h264&fps=30&max=640")
self.assertEqual(ws.status, 101)
info = None
key = None
for _ in range(200):
op, data = ws.recv()
if op == 1:
msg = json.loads(data)
if msg["t"] == "hello":
continue
if msg["t"] == "error":
self.skipTest("no H.264 encoder here: " + msg["message"])
if msg["t"] == "info":
info = msg
elif op == 2 and data[0] == 1:
key = data
if info and key:
break
self.assertEqual(info["src"], "test")
self.assertTrue(info["input"])
# A keyframe: flags, 8-byte timestamp, then Annex B with the SPS first.
self.assertEqual(key[9:13], b"\x00\x00\x00\x01")
self.assertEqual(key[13] & 0x1F, 7) # NAL type 7, SPS
ws.send_text(json.dumps({"t": "m", "e": "down", "b": 0, "x": 0.5, "y": 0.5}))
ws.send_text(json.dumps({"t": "key-frame"}))
got_key = False
for _ in range(200):
op, data = ws.recv()
if op == 2 and data[0] == 1:
got_key = True
break
self.assertTrue(got_key, "no keyframe after asking for one")
_, body = self.get(f"/status?k={self.token}")
self.assertEqual([s["src"] for s in json.loads(body)["streams"]], ["test"])
status, body = self.get(f"/close?k={self.token}", method="POST")
self.assertEqual(json.loads(body)["closed"], 1)
for _ in range(400):
op, data = ws.recv()
if op == 1:
break
self.assertEqual(json.loads(data)["t"], "close")
# Without the viewer doing anything, the agent ends the stream itself.
time.sleep(1)
_, body = self.get(f"/status?k={self.token}")
self.assertEqual(json.loads(body)["streams"], [])
ws.close()
if __name__ == "__main__":
unittest.main()