mirror of
https://github.com/saphid/frame-control.git
synced 2026-10-06 01:00:18 +02:00
Frame Control can now show any Mac window, or a whole screen, as its own SteamVR panel on the Steam Frame (Tools -> Mac in the headset, macOS only). Place it with the SteamVR dashboard; the laser clicks and scrolls, and the Mac's own keyboard types. - mac/frame-mac-view (Swift, no dependencies): ScreenCaptureKit capture per window or display, VideoToolbox H.264 with low-latency rate control (JPEG fallback), a loopback HTTP/WebSocket server, CGEvent/AX input playback, and a display-awake assertion while anyone watches. - ui/frame_macview.py: starts the agent, runs an ssh -R tunnel with a supervisor that reopens it on the same port, and launches a Chromium app window per stream on gamescope's :0, tagged with STEAM_GAME for its own panel. - Frame Control's key never leaves the Mac: viewers get single-use, per-source tickets and reconnect keys that Stop revokes. - ui/mac-view.html: WebCodecs decode, keyframe recovery, pointer/wheel/keys back. - Bundled in the Mac app build; tests/test_macview.py builds and drives the agent on macOS. Verified on the Frame (build 20260925.6191901) with the test pattern: panel in about 1.5 s, about 60 fps, Mac-to-window about 11-17 ms, tunnel recovery in 4 s. Laser input and real window capture still need a person in the headset. Also commits the other thread's first-party rule (steam-frame skill) and the first-party options table in docs/streaming.md. Reviewed by GPT-6 Astra (xhigh, read-only) over six rounds; all findings fixed. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
275 lines
11 KiB
Python
275 lines
11 KiB
Python
"""Mac in the headset (ui/frame_macview.py and the frame-mac-view agent).
|
|
|
|
The Python checks run anywhere. On macOS the agent is built and driven over
|
|
HTTP and WebSocket with its test pattern, which needs no Screen Recording
|
|
permission: status, the token, the viewer page, H.264 keyframes, pointer
|
|
input reaching the source, and closing.
|
|
|
|
Run: python3 -m unittest discover -s tests
|
|
"""
|
|
import base64
|
|
import http.client
|
|
import json
|
|
import os
|
|
import shutil
|
|
import socket
|
|
import struct
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import time
|
|
import unittest
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parent.parent
|
|
sys.path.insert(0, str(ROOT / "ui"))
|
|
|
|
import frame_macview # noqa: E402
|
|
|
|
|
|
class Helpers(unittest.TestCase):
|
|
def test_panel_id_is_stable_and_in_range(self):
|
|
a = frame_macview.panel_id("window:123")
|
|
self.assertEqual(a, frame_macview.panel_id("window:123"))
|
|
self.assertNotEqual(a, frame_macview.panel_id("window:124"))
|
|
self.assertTrue(2_001_000_000 <= a < 2_002_000_000)
|
|
|
|
def test_fit_keeps_aspect_inside_the_panel(self):
|
|
self.assertEqual(frame_macview.fit(2560, 1440), (1920, 1080))
|
|
w, h = frame_macview.fit(800, 1600)
|
|
self.assertEqual(h, 1080)
|
|
self.assertAlmostEqual(w / h, 0.5, places=2)
|
|
|
|
@unittest.skipUnless(shutil.which("bash"), "needs bash")
|
|
def test_launch_script_parses(self):
|
|
r = subprocess.run(["bash", "-n"], input=frame_macview.LAUNCH, text=True, capture_output=True)
|
|
self.assertEqual(r.returncode, 0, r.stderr)
|
|
|
|
def test_show_checks_the_source_before_anything_else(self):
|
|
mv = frame_macview.MacView(["ssh"], lambda *a, **k: self.fail("no ssh"), "frame")
|
|
with self.assertRaises(frame_macview.MacViewError):
|
|
mv.show("rm -rf /")
|
|
|
|
def test_missing_browser_is_explained(self):
|
|
calls = []
|
|
|
|
def run(remote, stdin=None, timeout=30):
|
|
calls.append(remote)
|
|
e = RuntimeError("exit 3")
|
|
e.stdout = "NO_BROWSER\n"
|
|
raise e
|
|
|
|
mv = frame_macview.MacView(["ssh"], run, "frame")
|
|
mv.call = lambda path, **kw: {"screen": True, "ticket": "tk"}
|
|
mv.ensure_tunnel = lambda **kw: None
|
|
mv.remote_port = 47900
|
|
with self.assertRaises(frame_macview.MacViewError) as cm:
|
|
mv.show("window:5")
|
|
self.assertIn("Chromium", str(cm.exception))
|
|
self.assertTrue(calls[0].startswith("bash -s -- "))
|
|
|
|
def test_screen_permission_is_checked_before_the_headset(self):
|
|
mv = frame_macview.MacView(["ssh"], lambda *a, **k: self.fail("no ssh"), "frame")
|
|
mv.call = lambda path, **kw: {"screen": False}
|
|
with self.assertRaises(frame_macview.MacViewError) as cm:
|
|
mv.show("display:1")
|
|
self.assertIn("Screen Recording", str(cm.exception))
|
|
|
|
|
|
class WS:
|
|
"""A minimal WebSocket client (masked frames out, plain frames in)."""
|
|
|
|
def __init__(self, port, path):
|
|
self.s = socket.create_connection(("127.0.0.1", port), timeout=10)
|
|
key = base64.b64encode(os.urandom(16)).decode()
|
|
self.s.sendall(f"GET {path} HTTP/1.1\r\nHost: x\r\nUpgrade: websocket\r\nConnection: Upgrade\r\n"
|
|
f"Sec-WebSocket-Key: {key}\r\nSec-WebSocket-Version: 13\r\n\r\n".encode())
|
|
head = b""
|
|
while b"\r\n\r\n" not in head:
|
|
head += self.s.recv(1)
|
|
self.status = int(head.split()[1])
|
|
self.buf = b""
|
|
|
|
def _read(self, n):
|
|
while len(self.buf) < n:
|
|
chunk = self.s.recv(65536)
|
|
if not chunk:
|
|
raise EOFError
|
|
self.buf += chunk
|
|
out, self.buf = self.buf[:n], self.buf[n:]
|
|
return out
|
|
|
|
def recv(self):
|
|
b0, b1 = self._read(2)
|
|
n = b1 & 0x7F
|
|
if n == 126:
|
|
n = struct.unpack(">H", self._read(2))[0]
|
|
elif n == 127:
|
|
n = struct.unpack(">Q", self._read(8))[0]
|
|
return b0 & 0x0F, self._read(n)
|
|
|
|
def send_text(self, text):
|
|
data, mask = text.encode(), os.urandom(4)
|
|
head = bytes([0x81, 0x80 | len(data)]) if len(data) < 126 else bytes([0x81, 0xFE]) + struct.pack(">H", len(data))
|
|
self.s.sendall(head + mask + bytes(c ^ mask[i % 4] for i, c in enumerate(data)))
|
|
|
|
def close(self):
|
|
self.s.close()
|
|
|
|
|
|
@unittest.skipUnless(sys.platform == "darwin" and shutil.which("xcrun"), "the agent is macOS-only")
|
|
class Agent(unittest.TestCase):
|
|
@classmethod
|
|
def setUpClass(cls):
|
|
cls.tmp = tempfile.mkdtemp()
|
|
cls.bin = Path(cls.tmp) / "frame-mac-view"
|
|
r = subprocess.run(["/bin/sh", str(ROOT / "mac" / "frame-mac-view" / "build.sh"), str(cls.bin)],
|
|
capture_output=True, text=True, timeout=600)
|
|
if r.returncode: # a real failure on a Mac with Xcode: don't hide it as a skip
|
|
raise AssertionError("agent didn't build:\n" + (r.stderr or r.stdout)[-2000:])
|
|
cls.token = "t0ken-" + os.urandom(6).hex()
|
|
cls.proc = subprocess.Popen([str(cls.bin), "serve", "--port", "0", "--page", str(ROOT / "ui" / "mac-view.html"),
|
|
"--exit-on-eof"], env={**os.environ, "FRAME_MAC_VIEW_TOKEN": cls.token},
|
|
stdin=subprocess.PIPE, stdout=subprocess.PIPE, text=True)
|
|
line = cls.proc.stdout.readline()
|
|
cls.port = int(line.rsplit(":", 1)[1])
|
|
|
|
@classmethod
|
|
def tearDownClass(cls):
|
|
cls.proc.stdin.close() # --exit-on-eof
|
|
cls.proc.stdout.close()
|
|
try:
|
|
cls.proc.wait(5)
|
|
except subprocess.TimeoutExpired:
|
|
cls.proc.kill()
|
|
shutil.rmtree(cls.tmp, ignore_errors=True)
|
|
|
|
def get(self, path, method="GET"):
|
|
c = http.client.HTTPConnection("127.0.0.1", self.port, timeout=10)
|
|
c.request(method, path)
|
|
r = c.getresponse()
|
|
return r.status, r.read()
|
|
|
|
def test_token_is_required(self):
|
|
self.assertEqual(self.get("/status")[0], 403)
|
|
self.assertEqual(self.get("/status?k=wrong")[0], 403)
|
|
status, body = self.get(f"/status?k={self.token}")
|
|
self.assertEqual(status, 200)
|
|
self.assertIn("screen", json.loads(body))
|
|
|
|
def test_serves_the_viewer_page(self):
|
|
status, body = self.get(f"/view?k={self.token}&src=test")
|
|
self.assertEqual(status, 200)
|
|
self.assertIn(b"VideoDecoder", body)
|
|
|
|
def test_lists_displays(self):
|
|
status, body = self.get(f"/displays?k={self.token}")
|
|
self.assertEqual(status, 200)
|
|
self.assertIsInstance(json.loads(body)["displays"], list)
|
|
|
|
def ticket(self, src):
|
|
status, body = self.get(f"/ticket?k={self.token}&src={src}", method="POST")
|
|
self.assertEqual(status, 200)
|
|
return json.loads(body)["ticket"]
|
|
|
|
def test_ping_and_page_are_open_but_streams_are_not(self):
|
|
self.assertEqual(self.get("/ping"), (200, b"frame-mac-view"))
|
|
self.assertEqual(WS(self.port, "/stream?src=test").status, 403)
|
|
self.assertEqual(self.get("/ticket?src=test", method="POST")[0], 403)
|
|
|
|
def test_tickets_are_single_use_and_tied_to_one_source(self):
|
|
t = self.ticket("test")
|
|
self.assertEqual(WS(self.port, f"/stream?src=display:1&t={t}").status, 403) # wrong source
|
|
ws = WS(self.port, f"/stream?src=test&t={t}")
|
|
self.assertEqual(ws.status, 101)
|
|
hello = json.loads(ws.recv()[1])
|
|
self.assertEqual(hello["t"], "hello")
|
|
# Until the viewer acknowledges, a retry (the hello got lost) gets the
|
|
# same key, and replaces the first viewer rather than adding one.
|
|
retry = WS(self.port, f"/stream?src=test&t={t}")
|
|
self.assertEqual(retry.status, 101)
|
|
self.assertEqual(json.loads(retry.recv()[1])["r"], hello["r"])
|
|
time.sleep(0.3)
|
|
_, body = self.get(f"/status?k={self.token}")
|
|
self.assertEqual(len(json.loads(body)["streams"]), 1)
|
|
ws.close()
|
|
ws = retry
|
|
ws.send_text(json.dumps({"t": "ack"}))
|
|
time.sleep(0.3)
|
|
self.assertEqual(WS(self.port, f"/stream?src=test&t={t}").status, 403) # spent
|
|
again = WS(self.port, f"/stream?src=test&r={hello['r']}") # the viewer reconnecting
|
|
self.assertEqual(again.status, 101)
|
|
again.close()
|
|
ws.close()
|
|
# Stop revokes the reconnect key.
|
|
self.get(f"/close?k={self.token}&src=test", method="POST")
|
|
self.assertEqual(WS(self.port, f"/stream?src=test&r={hello['r']}").status, 403)
|
|
|
|
def test_stop_revokes_tickets_not_yet_used(self):
|
|
t = self.ticket("test")
|
|
self.get(f"/close?k={self.token}&src=test", method="POST")
|
|
self.assertEqual(WS(self.port, f"/stream?src=test&t={t}").status, 403)
|
|
|
|
def test_bad_frame_lengths_close_the_socket_not_the_agent(self):
|
|
ws = WS(self.port, f"/stream?src=test&t={self.ticket('test')}")
|
|
self.assertEqual(ws.status, 101)
|
|
# A masked frame claiming 2^63 bytes.
|
|
ws.s.sendall(bytes([0x81, 0xFF]) + struct.pack(">Q", 1 << 63) + os.urandom(4))
|
|
with self.assertRaises((EOFError, OSError)):
|
|
for _ in range(1000):
|
|
ws.recv()
|
|
ws.close()
|
|
self.assertEqual(self.get(f"/status?k={self.token}")[0], 200)
|
|
|
|
def test_stream_input_and_close(self):
|
|
ws = WS(self.port, f"/stream?k={self.token}&src=test&codec=h264&fps=30&max=640")
|
|
self.assertEqual(ws.status, 101)
|
|
info = None
|
|
key = None
|
|
for _ in range(200):
|
|
op, data = ws.recv()
|
|
if op == 1:
|
|
msg = json.loads(data)
|
|
if msg["t"] == "hello":
|
|
continue
|
|
if msg["t"] == "error":
|
|
self.skipTest("no H.264 encoder here: " + msg["message"])
|
|
if msg["t"] == "info":
|
|
info = msg
|
|
elif op == 2 and data[0] == 1:
|
|
key = data
|
|
if info and key:
|
|
break
|
|
self.assertEqual(info["src"], "test")
|
|
self.assertTrue(info["input"])
|
|
# A keyframe: flags, 8-byte timestamp, then Annex B with the SPS first.
|
|
self.assertEqual(key[9:13], b"\x00\x00\x00\x01")
|
|
self.assertEqual(key[13] & 0x1F, 7) # NAL type 7, SPS
|
|
ws.send_text(json.dumps({"t": "m", "e": "down", "b": 0, "x": 0.5, "y": 0.5}))
|
|
ws.send_text(json.dumps({"t": "key-frame"}))
|
|
got_key = False
|
|
for _ in range(200):
|
|
op, data = ws.recv()
|
|
if op == 2 and data[0] == 1:
|
|
got_key = True
|
|
break
|
|
self.assertTrue(got_key, "no keyframe after asking for one")
|
|
_, body = self.get(f"/status?k={self.token}")
|
|
self.assertEqual([s["src"] for s in json.loads(body)["streams"]], ["test"])
|
|
status, body = self.get(f"/close?k={self.token}", method="POST")
|
|
self.assertEqual(json.loads(body)["closed"], 1)
|
|
for _ in range(400):
|
|
op, data = ws.recv()
|
|
if op == 1:
|
|
break
|
|
self.assertEqual(json.loads(data)["t"], "close")
|
|
# Without the viewer doing anything, the agent ends the stream itself.
|
|
time.sleep(1)
|
|
_, body = self.get(f"/status?k={self.token}")
|
|
self.assertEqual(json.loads(body)["streams"], [])
|
|
ws.close()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|