Mac in the headset: stream Mac windows and screens into the Frame as panels

Frame Control can now show any Mac window, or a whole screen, as its own
SteamVR panel on the Steam Frame (Tools -> Mac in the headset, macOS only).
Place it with the SteamVR dashboard; the laser clicks and scrolls, and the
Mac's own keyboard types.

- mac/frame-mac-view (Swift, no dependencies): ScreenCaptureKit capture per
  window or display, VideoToolbox H.264 with low-latency rate control (JPEG
  fallback), a loopback HTTP/WebSocket server, CGEvent/AX input playback,
  and a display-awake assertion while anyone watches.
- ui/frame_macview.py: starts the agent, runs an ssh -R tunnel with a
  supervisor that reopens it on the same port, and launches a Chromium app
  window per stream on gamescope's :0, tagged with STEAM_GAME for its own panel.
- Frame Control's key never leaves the Mac: viewers get single-use,
  per-source tickets and reconnect keys that Stop revokes.
- ui/mac-view.html: WebCodecs decode, keyframe recovery, pointer/wheel/keys back.
- Bundled in the Mac app build; tests/test_macview.py builds and drives the
  agent on macOS.

Verified on the Frame (build 20260925.6191901) with the test pattern: panel
in about 1.5 s, about 60 fps, Mac-to-window about 11-17 ms, tunnel recovery
in 4 s. Laser input and real window capture still need a person in the headset.

Also commits the other thread's first-party rule (steam-frame skill) and
the first-party options table in docs/streaming.md.

Reviewed by GPT-6 Astra (xhigh, read-only) over six rounds; all findings fixed.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

ui/mac-view.html

@@ -0,0 +1,242 @@
+<!doctype html>
+<!-- Frame Control: one Mac window (or display) inside the Steam Frame.
+     Served by the Mac's frame-mac-view agent through an SSH tunnel and opened
+     on the Frame as its own Chromium app window, which gamescope turns into a
+     SteamVR panel. Video arrives over a WebSocket (H.264 Annex B for
+     WebCodecs, or JPEG); pointer, wheel and keys go back the same way. -->
+<html lang="en">
+<head>
+<meta charset="utf-8">
+<meta name="viewport" content="width=device-width, initial-scale=1">
+<title>Mac</title>
+<style>
+  html, body { margin: 0; height: 100%; background: #000; overflow: hidden; cursor: default; }
+  canvas { position: fixed; inset: 0; width: 100%; height: 100%; object-fit: contain; image-rendering: auto; }
+  #note { position: fixed; left: 50%; top: 16px; transform: translateX(-50%); max-width: 80%;
+          font: 15px/1.4 system-ui, sans-serif; color: #eee; background: rgba(20,22,26,.88);
+          padding: 8px 14px; border-radius: 10px; pointer-events: none; transition: opacity .4s; }
+  #note[hidden] { display: block; opacity: 0; }
+</style>
+</head>
+<body>
+<canvas id="c" width="16" height="9"></canvas>
+<div id="note">Connecting to the Mac…</div>
+<script>
+"use strict";
+const q = new URLSearchParams(location.search);
+// t: a single-use ticket from Frame Control. After the first connection the
+// Mac sends a reconnect key, kept only in memory.
+const src = q.get("src") || "test", ticket = q.get("t") || "", tag = q.get("tag") || "";
+let reconnectKey = "";
+const c = document.getElementById("c"), ctx = c.getContext("2d", { alpha: false, desynchronized: true });
+const note = document.getElementById("note");
+// Counters for Frame Control's tests (read over DevTools).
+const stats = window.stats = { frames: 0, keyFrames: 0, bytes: 0, dropped: 0, codec: "", errors: [], info: null };
+// Frame Control finds this window on the Frame by the tag in its title.
+document.title = tag ? `Mac [${tag}]` : "Mac";
+let failedStarts = 0;
+let ws = null, dec = null, codecString = "", needKey = true, closing = false, retry = 0, noteTimer = 0, lastError = "";
+
+function show(text, ms) {
+  note.textContent = text;
+  note.hidden = false;
+  clearTimeout(noteTimer);
+  if (ms) noteTimer = setTimeout(() => { note.hidden = true; }, ms);
+}
+
+function send(obj) { if (ws && ws.readyState === 1) ws.send(JSON.stringify(obj)); }
+let lastKeyAsk = 0;
+function askKeyFrame() {
+  const now = performance.now();
+  if (now - lastKeyAsk < 500) return;
+  lastKeyAsk = now;
+  send({ t: "key-frame" });
+}
+
+let hideNoteOnFrame = true;  // "Connecting…"/"Reconnecting…" go once video flows again
+function drawFrame(img, w, h) {
+  if (c.width !== w || c.height !== h) { c.width = w; c.height = h; }
+  ctx.drawImage(img, 0, 0);
+  stats.frames++;
+  if (hideNoteOnFrame) { hideNoteOnFrame = false; note.hidden = true; }
+}
+
+// ---- H.264 ----
+function startCode(b, i) { return b[i] === 0 && b[i + 1] === 0 && (b[i + 2] === 1 || (b[i + 2] === 0 && b[i + 3] === 1)); }
+function spsCodec(au) {
+  for (let i = 0; i + 7 < au.length; i++) {
+    if (!startCode(au, i)) continue;
+    const n = au[i + 2] === 1 ? i + 3 : i + 4;
+    if ((au[n] & 0x1f) === 7) return "avc1." + [au[n + 1], au[n + 2], au[n + 3]].map(b => b.toString(16).padStart(2, "0")).join("");
+    i = n;
+  }
+  return "";
+}
+function makeDecoder() {
+  if (dec) try { dec.close(); } catch (e) {}
+  codecString = "";
+  dec = new VideoDecoder({
+    output: frame => { drawFrame(frame, frame.displayWidth, frame.displayHeight); frame.close(); },
+    error: e => {
+      stats.errors.push(String(e.message || e));
+      needKey = true;
+      makeDecoder();
+      askKeyFrame();
+    },
+  });
+}
+function onH264(isKey, ts, au) {
+  if (isKey) {
+    const cs = spsCodec(au);
+    if (cs && (cs !== codecString || dec.state !== "configured")) {
+      if (dec.state === "closed") makeDecoder();
+      dec.configure({ codec: cs, optimizeForLatency: true, hardwareAcceleration: "no-preference" });
+      codecString = stats.codec = cs;
+    }
+    stats.keyFrames++;
+  }
+  if (dec.state !== "configured" || (needKey && !isKey)) { stats.dropped++; askKeyFrame(); return; }
+  // Behind: skip to the next keyframe rather than show old pictures late.
+  if (!isKey && dec.decodeQueueSize > 2) { needKey = true; stats.dropped++; askKeyFrame(); return; }
+  needKey = false;
+  dec.decode(new EncodedVideoChunk({ type: isKey ? "key" : "delta", timestamp: ts, data: au }));
+}
+
+// ---- JPEG ----
+let jpegBusy = false;
+function onJPEG(data) {
+  if (jpegBusy) { stats.dropped++; return; }
+  jpegBusy = true;
+  createImageBitmap(new Blob([data], { type: "image/jpeg" })).then(bmp => {
+    drawFrame(bmp, bmp.width, bmp.height);
+    bmp.close();
+  }).catch(e => stats.errors.push(String(e))).finally(() => { jpegBusy = false; });
+}
+
+async function pickCodec() {
+  const want = q.get("codec");
+  if (want === "jpeg") return "jpeg";
+  if (!("VideoDecoder" in window)) return "jpeg";
+  try {
+    const r = await VideoDecoder.isConfigSupported({ codec: "avc1.640028", optimizeForLatency: true });
+    return r.supported ? "h264" : "jpeg";
+  } catch (e) { return "jpeg"; }
+}
+
+async function connect() {
+  const codec = await pickCodec();
+  stats.codec = codec;
+  if (codec === "h264") makeDecoder();
+  needKey = true;
+  const p = new URLSearchParams({ src, codec, max: q.get("max") || "1920", fps: q.get("fps") || "60" });
+  if (reconnectKey) p.set("r", reconnectKey); else p.set("t", ticket);
+  if (q.get("bpp")) p.set("bpp", q.get("bpp"));
+  ws = new WebSocket(`ws://${location.host}/stream?${p}`);
+  ws.binaryType = "arraybuffer";
+  ws.onopen = () => { retry = 0; lastError = ""; };
+  ws.onmessage = ev => {
+    if (typeof ev.data === "string") return control(JSON.parse(ev.data));
+    const b = new Uint8Array(ev.data);
+    stats.bytes += b.length;
+    if (b.length < 10) return;
+    const isKey = b[0] === 1;
+    const ts = Number(new DataView(ev.data).getBigUint64(1));
+    const payload = b.subarray(9);
+    if (codec === "h264") onH264(isKey, ts, payload); else onJPEG(payload);
+  };
+  ws.onclose = () => {
+    ws = null;
+    if (closing) return;
+    // Refused before ever getting a key: the ticket expired or was revoked,
+    // and retrying can't help.
+    if (!reconnectKey && ++failedStarts >= 3) {
+      show("This view has expired. Press Show in Frame Control on the Mac to open it again.");
+      return;
+    }
+    hideNoteOnFrame = true;
+    // The Mac may be asleep or the tunnel restarting: keep trying.
+    retry = Math.min(retry + 1, 6);
+    // Keep the Mac's reason (a missing permission, a closed window) on screen.
+    show(lastError ? `${lastError} Retrying…` : "Lost the Mac. Reconnecting…");
+    setTimeout(connect, 500 * 2 ** retry);
+  };
+}
+
+function control(m) {
+  if (m.t === "hello") {
+    reconnectKey = m.r;
+    send({ t: "ack" });  // the ticket is spent only now
+  } else if (m.t === "info") {
+    stats.info = m;
+    const name = m.title && m.app && m.title !== m.app ? `${m.title} — ${m.app}` : (m.title || m.app || "Mac");
+    document.title = tag ? `${name} [${tag}]` : name;
+    if (!m.input) {
+      hideNoteOnFrame = false;  // a warning, not a connection notice: let it stay its 8 s
+      show("Clicks and keys need Accessibility permission on the Mac (Frame Control asks for it).", 8000);
+    }
+  } else if (m.t === "error") {
+    stats.errors.push(m.message);
+    lastError = m.message.replace(/\.?$/, ".");
+    show(m.message);
+  } else if (m.t === "close" || m.t === "closed") {
+    closing = true;
+    if (ws) ws.close();
+    show(m.reason ? `Stopped: ${m.reason}.` : "Stopped.");
+    window.close();
+  }
+}
+
+// ---- input ----
+// Where the picture sits inside the window (object-fit: contain letterboxes it).
+function toPicture(e) {
+  const r = c.getBoundingClientRect(), s = Math.min(r.width / c.width, r.height / c.height);
+  const w = c.width * s, h = c.height * s, left = r.left + (r.width - w) / 2, top = r.top + (r.height - h) / 2;
+  const x = (e.clientX - left) / w, y = (e.clientY - top) / h;
+  return { x, y, inside: x >= 0 && x <= 1 && y >= 0 && y <= 1 };
+}
+let pendingMove = null, moveQueued = false;
+addEventListener("pointermove", e => {
+  const p = toPicture(e);
+  if (!p.inside && !e.buttons) return;
+  pendingMove = { t: "m", e: "move", x: p.x, y: p.y };
+  if (!moveQueued) {
+    moveQueued = true;
+    requestAnimationFrame(() => { moveQueued = false; if (pendingMove) send(pendingMove); pendingMove = null; });
+  }
+});
+addEventListener("pointerdown", e => {
+  const p = toPicture(e);
+  if (!p.inside) return;
+  if (e.pointerId !== undefined) try { c.setPointerCapture(e.pointerId); } catch (err) {}
+  pendingMove = null;
+  send({ t: "m", e: "down", b: e.button < 0 ? 0 : e.button, x: p.x, y: p.y });
+  e.preventDefault();
+});
+addEventListener("pointerup", e => {
+  const p = toPicture(e);
+  send({ t: "m", e: "up", b: e.button < 0 ? 0 : e.button, x: p.x, y: p.y });
+});
+// Let go of any held button where the pointer is on the Mac, not at a corner.
+addEventListener("pointercancel", () => send({ t: "release" }));
+addEventListener("contextmenu", e => e.preventDefault());
+addEventListener("wheel", e => {
+  const p = toPicture(e), unit = e.deltaMode === 1 ? 16 : e.deltaMode === 2 ? innerHeight : 1;
+  send({ t: "wheel", dx: e.deltaX * unit, dy: e.deltaY * unit, x: p.x, y: p.y });
+  e.preventDefault();
+}, { passive: false });
+function mods(e) {
+  return ["shift", "ctrl", "alt", "meta"].filter(m => e[m + "Key"]);
+}
+function onKey(e, down) {
+  send({ t: "k", e: down ? "down" : "up", code: e.code, key: e.key, mods: mods(e) });
+  e.preventDefault();
+}
+addEventListener("keydown", e => onKey(e, true));
+addEventListener("keyup", e => onKey(e, false));
+addEventListener("blur", () => send({ t: "release" }));
+
+show("Connecting to the Mac…");
+connect();
+</script>
+</body>
+</html>