steam-frame/saphid--frame-control
1
0 Fork 0
mirror of https://github.com/saphid/frame-control.git synced 2026-10-04 22:00:10 +02:00
Code Issues Packages Projects Releases 6 Wiki Activity

Mac in the headset: measure every frame, adapt to the network, separate windows

Browse Source 
- Per-frame timing on the Mac's clock (capture, encode, network, decode,
  draw), viewer clock sync and reports, input echo, /stats and a HUD.
- scripts/macview-bench.py: repeatable runs on the real Frame, a shaping
  relay (no sudo), interleaved A/B between agent settings; results in
  bench/results/.
- Adaptive controller: ack-based send gate with jitter-aware slack, AIMD
  bitrate that knows when a stream is app-limited, fps then size tiers.
  On a 50->3->50 Mbit/s step, scroll p95 went from 4.7 s to 72 ms; no cost
  on a clean link.
- Separate mode: real AppKit event loop (HiDPI and NSScreen now work),
  cropped capture for fixed-size windows, windows kept on their display,
  graceful quit restores windows; stop/start races fixed.
- Encoder timeline clamp (no oversized frame after a pause).
- Frame Control shows each live stream's fps, delay, bitrate and tier.

Reviewed by GPT-6 Astra xhigh (read-only), 7 rounds; findings fixed.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
saphidandClaude Opus 5.5 committed 2026-09-28 20:56:14 +10:00
1 parent a3c6e5c984
commit 9e4dcdd147
29 files changed
+21405 -104
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

bench/pages/present.html
+24
View File
@@ -0,0 +1,24 @@
<!doctype html>
<!-- Control experiment, run on the Frame itself: how often does Chromium on
gamescope put a canvas animation on screen, with no network or decoding
involved? Draws every animation frame for ?s= seconds and writes the
frame-gap histogram per second into the title, where xprop can read it. -->
<html><head><meta charset="utf-8"><title>fmv-present</title></head>
<body style="margin:0;background:#000"><canvas id="c" width="1280" height="720" style="width:100%;height:100%"></canvas>
<script>
const q = new URLSearchParams(location.search), secs = +q.get("s") || 15, tag = q.get("tag") || "";
const ctx = document.getElementById("c").getContext("2d", { alpha: false, desynchronized: true });
const perSec = []; let t0 = 0, last = 0, n = 0, gaps = [];
function frame(t) {
if (!t0) t0 = last = t;
ctx.fillStyle = "#123"; ctx.fillRect(0, 0, 1280, 720);
ctx.fillStyle = "#fff"; ctx.fillRect((n * 21) % 1280, 0, 20, 720); n++;
const s = Math.floor((t - t0) / 1000);
(perSec[s] = perSec[s] || []).push(t - last); last = t;
if (t - t0 < secs * 1000) return requestAnimationFrame(frame);
const out = perSec.map(g => g.length).join(",");
document.title = `[${tag}] done fps/s=${out}`;
}
document.title = `[${tag}] running`;
requestAnimationFrame(frame);
</script></body></html>
bench/pages/scroll.html
+32
View File
@@ -0,0 +1,32 @@
<!doctype html>
<!-- Benchmark content: a long page of text that scrolls itself at a steady
speed, so every frame changes the way reading a real page does. -->
<html lang="en"><head><meta charset="utf-8"><title>fmv-bench scroll</title>
<style>
body { font: 17px/1.55 -apple-system, "Helvetica Neue", sans-serif; margin: 0 auto; max-width: 900px; padding: 24px; color: #1d1d1f; background: #fff; }
h2 { font-size: 22px; margin: 28px 0 8px; } code { background: #f2f2f5; padding: 1px 4px; border-radius: 3px; }
</style></head><body><div id="doc"></div>
<script>
const words = "latency frame panel stream encoder decoder network display pixel window capture budget quality motion text sharp adaptive controller queue socket clock".split(" ");
let seed = 7; const rnd = () => (seed = (seed * 16807) % 2147483647) / 2147483647;
let html = "";
for (let s = 0; s < 120; s++) {
html += `<h2>Section ${s + 1}: ${words[s % words.length]} and ${words[(s * 7) % words.length]}</h2>`;
for (let p = 0; p < 4; p++) {
let para = [];
for (let w = 0; w < 70; w++) para.push(rnd() < 0.05 ? `<code>${words[Math.floor(rnd() * words.length)]}()</code>` : words[Math.floor(rnd() * words.length)]);
html += `<p>${para.join(" ")}.</p>`;
}
}
document.getElementById("doc").innerHTML = html;
// 240 points a second, whatever the display's refresh rate.
// The title carries the page's own frame rate, so the source's rate is known.
let last = performance.now(), frames = 0, since = last;
function step(now) {
const dy = (now - last) * 0.24; last = now;
if (++frames && now - since >= 1000) { document.title = `fmv-bench scroll ${frames} fps`; frames = 0; since = now; }
if (window.scrollY + innerHeight >= document.body.scrollHeight - 2) window.scrollTo(0, 0); else window.scrollBy(0, dy);
requestAnimationFrame(step);
}
requestAnimationFrame(step);
</script></body></html>
bench/pages/type.html
+9
View File
@@ -0,0 +1,9 @@
<!doctype html>
<!-- Benchmark content: a plain text editor, focused, for typing tests. -->
<html lang="en"><head><meta charset="utf-8"><title>fmv-bench type</title>
<style>
html, body { margin: 0; height: 100%; background: #fff; }
textarea { box-sizing: border-box; width: 100%; height: 100%; border: 0; padding: 20px; outline: none; resize: none;
font: 20px/1.5 ui-monospace, Menlo, monospace; color: #111; caret-color: transparent; } /* a blinking caret would pass for a reply to a key */
</style></head><body><textarea id="t" autofocus spellcheck="false"></textarea>
<script>document.getElementById("t").focus();</script></body></html>
bench/results/2026-09-28-a3c6e5c-dirty-ab-adapt-clean.json
+4211
View File
File diff suppressed because it is too large. Load diff
bench/results/2026-09-28-a3c6e5c-dirty-ab-adapt-clean2.json
+4150
View File
File diff suppressed because it is too large. Load diff
bench/results/2026-09-28-a3c6e5c-dirty-ab-adapt-step3.json
+3743
View File
File diff suppressed because it is too large. Load diff
bench/results/2026-09-28-a3c6e5c-dirty-ab-keepwarm.json
+2172
View File
File diff suppressed because it is too large. Load diff
bench/results/2026-09-28-a3c6e5c-dirty-adapt-clean.json
+984
View File
@@ -0,0 +1,984 @@
{
"label": "adapt-clean",
"date": "2026-09-28T17:13:59",
"commit": "a3c6e5c+dirty",
"config": {
"quality": "balanced",
"mode": "separate",
"net": "none",
"delay_ms": 0,
"buffer_ms": 250,
"host": "frame",
"ssh_opts": [],
"encoder": "",
"duration_s": 20.0,
"browser_flags": []
},
"frame_build": "20260925.6191901",
"mac": "26.5.2",
"headset": ":39.639829 [Info] - 0 - entering standby",
"scenarios": [
{
"scenario": "test",
"frames_sent": 653,
"frames_drawn": 653,
"frames_shown": 649,
"duration_s": 20.7,
"stages_ms": {
"capture": {
"p50": 0.0,
"p95": 0.0,
"p99": 0.0,
"n": 653
},
"queue": {
"p50": 11.2,
"p95": 16.1,
"p99": 17.1,
"n": 653
},
"encode": {
"p50": 3.5,
"p95": 4.2,
"p99": 4.4,
"n": 653
},
"socket": {
"p50": 0.1,
"p95": 0.2,
"p99": 0.2,
"n": 653
},
"network": {
"p50": 2.9,
"p95": 7.0,
"p99": 12.0,
"n": 653
},
"decode": {
"p50": 1.0,
"p95": 2.4,
"p99": 2.8,
"n": 653
},
"draw": {
"p50": 0.3,
"p95": 0.6,
"p99": 0.9,
"n": 653
},
"present": {
"p50": 0.5,
"p95": 0.9,
"p99": 5.9,
"n": 649
},
"content": {
"p50": 19.8,
"p95": 26.2,
"p99": 30.1,
"n": 653
},
"content_shown": {
"p50": 20.3,
"p95": 26.9,
"p99": 31.1,
"n": 649
}
},
"fps": 31.5,
"fps_shown": 31.3,
"late_pct": 98.01,
"stall_max": 144.0,
"stalls_over_100ms": 2,
"mbps": 0.13,
"keyframes": 2,
"size": "960x540",
"captured": 1336,
"viewer_never_drawn": 0,
"input_ms": {
"p50": 43.5,
"p95": 61.8,
"p99": 63.4,
"n": 40
},
"input_shown_ms": {
"p50": 44.3,
"p95": 62.3,
"p99": 63.9,
"n": 40
},
"input_parts_ms": {
"uplink": {
"p50": 11.1,
"p95": 29.2,
"p99": 36.1,
"n": 40
},
"mac": {
"p50": 10.4,
"p95": 28.2,
"p99": 32.4,
"n": 40
},
"back": {
"p50": 17.5,
"p95": 27.9,
"p99": 28.8,
"n": 40
}
},
"inputs": {
"sent": 40,
"seen": 40
},
"timeline": [
{
"t": 0,
"fps": 31,
"mbps": 0.13,
"content_p50": 21.5,
"content_p95": 27.1,
"bitrate": 300000,
"tier": 4,
"w": 960,
"net": null
},
{
"t": 1,
"fps": 31,
"mbps": 0.12,
"content_p50": 22.4,
"content_p95": 26.3,
"bitrate": 300000,
"tier": 4,
"w": 960,
"net": null
},
{
"t": 2,
"fps": 32,
"mbps": 0.13,
"content_p50": 18.9,
"content_p95": 24.8,
"bitrate": 300000,
"tier": 4,
"w": 960,
"net": null
},
{
"t": 3,
"fps": 31,
"mbps": 0.12,
"content_p50": 18.2,
"content_p95": 24.6,
"bitrate": 300000,
"tier": 4,
"w": 960,
"net": null
},
{
"t": 4,
"fps": 32,
"mbps": 0.13,
"content_p50": 21.0,
"content_p95": 25.6,
"bitrate": 300000,
"tier": 4,
"w": 960,
"net": null
},
{
"t": 5,
"fps": 32,
"mbps": 0.12,
"content_p50": 19.5,
"content_p95": 24.3,
"bitrate": 300000,
"tier": 4,
"w": 960,
"net": null
},
{
"t": 6,
"fps": 31,
"mbps": 0.12,
"content_p50": 22.2,
"content_p95": 25.6,
"bitrate": 300000,
"tier": 4,
"w": 960,
"net": null
},
{
"t": 7,
"fps": 32,
"mbps": 0.13,
"content_p50": 19.9,
"content_p95": 24.0,
"bitrate": 300000,
"tier": 4,
"w": 960,
"net": null
},
{
"t": 8,
"fps": 31,
"mbps": 0.21,
"content_p50": 22.7,
"content_p95": 26.9,
"bitrate": 300000,
"tier": 4,
"w": 960,
"net": null
},
{
"t": 9,
"fps": 31,
"mbps": 0.12,
"content_p50": 20.2,
"content_p95": 26.1,
"bitrate": 300000,
"tier": 4,
"w": 960,
"net": null
},
{
"t": 10,
"fps": 32,
"mbps": 0.12,
"content_p50": 18.2,
"content_p95": 23.2,
"bitrate": 300000,
"tier": 4,
"w": 960,
"net": null
},
{
"t": 11,
"fps": 31,
"mbps": 0.12,
"content_p50": 19.8,
"content_p95": 24.2,
"bitrate": 300000,
"tier": 4,
"w": 960,
"net": null
},
{
"t": 12,
"fps": 32,
"mbps": 0.13,
"content_p50": 19.2,
"content_p95": 24.8,
"bitrate": 300000,
"tier": 4,
"w": 960,
"net": null
},
{
"t": 13,
"fps": 32,
"mbps": 0.12,
"content_p50": 21.0,
"content_p95": 24.3,
"bitrate": 300000,
"tier": 4,
"w": 960,
"net": null
},
{
"t": 14,
"fps": 30,
"mbps": 0.13,
"content_p50": 20.4,
"content_p95": 96.7,
"bitrate": 300000,
"tier": 4,
"w": 960,
"net": null
},
{
"t": 15,
"fps": 32,
"mbps": 0.13,
"content_p50": 18.2,
"content_p95": 24.4,
"bitrate": 300000,
"tier": 4,
"w": 960,
"net": null
},
{
"t": 16,
"fps": 31,
"mbps": 0.12,
"content_p50": 18.4,
"content_p95": 25.1,
"bitrate": 300000,
"tier": 4,
"w": 960,
"net": null
},
{
"t": 17,
"fps": 32,
"mbps": 0.13,
"content_p50": 17.2,
"content_p95": 23.5,
"bitrate": 300000,
"tier": 4,
"w": 960,
"net": null
},
{
"t": 18,
"fps": 32,
"mbps": 0.19,
"content_p50": 19.4,
"content_p95": 25.2,
"bitrate": 300000,
"tier": 4,
"w": 960,
"net": null
},
{
"t": 19,
"fps": 31,
"mbps": 0.14,
"content_p50": 21.1,
"content_p95": 84.3,
"bitrate": 300000,
"tier": 4,
"w": 960,
"net": null
},
{
"t": 20,
"fps": 24,
"mbps": 0.08,
"content_p50": 20.8,
"content_p95": 25.1,
"bitrate": 300000,
"tier": 4,
"w": 960,
"net": null
}
],
"adapt": [],
"content_p50": 19.8,
"content_p95": 26.2,
"input_p50": 43.5,
"input_p95": 61.8,
"grades": {
"content_p50": "local",
"content_p95": "local",
"input_p50": "local",
"input_p95": "local",
"fps": "bad",
"late_pct": "bad",
"stall_max": "acceptable"
},
"source_fps": 64.5,
"cpu": {
"mac_agent_pct": 11.3,
"frame_viewer_pct": 35.9,
"frame_tailscaled_pct": 3.5,
"frame_sshd_pct": 0.6,
"frame_gamescope_pct": 7.5,
"frame_total_pct": 16.1
},
"viewer": "h264 software; ANGLE (Mesa, zink Vulkan 1.4(Turnip Adreno (TM) 750 (MESA_TURNIP)), OpenGL 4.6); raf 141 Hz",
"show_s": 4.68,
"panel": "valve.steam.desktopgame.2001639889",
"src": "test"
},
{
"scenario": "scroll",
"frames_sent": 937,
"frames_drawn": 937,
"frames_shown": 720,
"duration_s": 20.7,
"stages_ms": {
"capture": {
"p50": -3.9,
"p95": 0.9,
"p99": 8.2,
"n": 937
},
"queue": {
"p50": 0.0,
"p95": 16.3,
"p99": 20.1,
"n": 937
},
"encode": {
"p50": 6.4,
"p95": 7.2,
"p99": 9.3,
"n": 937
},
"socket": {
"p50": 0.1,
"p95": 0.2,
"p99": 0.2,
"n": 937
},
"network": {
"p50": 7.3,
"p95": 13.0,
"p99": 22.8,
"n": 937
},
"decode": {
"p50": 6.2,
"p95": 12.7,
"p99": 18.6,
"n": 937
},
"draw": {
"p50": 0.6,
"p95": 1.1,
"p99": 1.9,
"n": 937
},
"present": {
"p50": 2.9,
"p95": 18.5,
"p99": 24.5,
"n": 720
},
"content": {
"p50": 19.9,
"p95": 37.2,
"p99": 50.2,
"n": 937
},
"content_shown": {
"p50": 27.0,
"p95": 45.7,
"p99": 55.0,
"n": 720
}
},
"fps": 45.3,
"fps_shown": 34.8,
"late_pct": 28.63,
"stall_max": 99.3,
"stalls_over_100ms": 0,
"mbps": 7.59,
"keyframes": 2,
"size": "1920x1290",
"captured": 1215,
"viewer_never_drawn": 0,
"input_ms": {
"n": 0
},
"input_shown_ms": {
"n": 0
},
"input_parts_ms": {
"uplink": {
"n": 0
},
"mac": {
"n": 0
},
"back": {
"n": 0
}
},
"inputs": {
"sent": 0,
"seen": 0
},
"timeline": [
{
"t": 0,
"fps": 30,
"mbps": 4.01,
"content_p50": 31.9,
"content_p95": 52.3,
"bitrate": 4867140,
"tier": 2,
"w": 1920,
"net": null
},
{
"t": 1,
"fps": 30,
"mbps": 4.12,
"content_p50": 29.9,
"content_p95": 45.3,
"bitrate": 3807054,
"tier": 2,
"w": 1920,
"net": null
},
{
"t": 2,
"fps": 30,
"mbps": 3.26,
"content_p50": 31.8,
"content_p95": 52.5,
"bitrate": 3235995,
"tier": 2,
"w": 1920,
"net": null
},
{
"t": 3,
"fps": 30,
"mbps": 3.72,
"content_p50": 24.5,
"content_p95": 37.7,
"bitrate": 4238740,
"tier": 2,
"w": 1920,
"net": null
},
{
"t": 4,
"fps": 30,
"mbps": 5.09,
"content_p50": 25.2,
"content_p95": 39.6,
"bitrate": 5992063,
"tier": 2,
"w": 1920,
"net": null
},
{
"t": 5,
"fps": 31,
"mbps": 7.29,
"content_p50": 29.1,
"content_p95": 53.5,
"bitrate": 5526067,
"tier": 2,
"w": 1920,
"net": null
},
{
"t": 6,
"fps": 35,
"mbps": 4.58,
"content_p50": 25.3,
"content_p95": 32.4,
"bitrate": 6018152,
"tier": 1,
"w": 1920,
"net": null
},
{
"t": 7,
"fps": 46,
"mbps": 7.12,
"content_p50": 25.1,
"content_p95": 34.3,
"bitrate": 8412933,
"tier": 1,
"w": 1920,
"net": null
},
{
"t": 8,
"fps": 46,
"mbps": 8.25,
"content_p50": 27.1,
"content_p95": 40.3,
"bitrate": 10760191,
"tier": 1,
"w": 1920,
"net": null
},
{
"t": 9,
"fps": 47,
"mbps": 9.3,
"content_p50": 27.8,
"content_p95": 41.2,
"bitrate": 13717060,
"tier": 0,
"w": 1920,
"net": null
},
{
"t": 10,
"fps": 55,
"mbps": 8.98,
"content_p50": 16.5,
"content_p95": 21.7,
"bitrate": 14860800,
"tier": 0,
"w": 1920,
"net": null
},
{
"t": 11,
"fps": 51,
"mbps": 9.32,
"content_p50": 18.6,
"content_p95": 35.4,
"bitrate": 14860800,
"tier": 0,
"w": 1920,
"net": null
},
{
"t": 12,
"fps": 56,
"mbps": 9.53,
"content_p50": 15.4,
"content_p95": 26.2,
"bitrate": 14860800,
"tier": 0,
"w": 1920,
"net": null
},
{
"t": 13,
"fps": 55,
"mbps": 8.46,
"content_p50": 18.6,
"content_p95": 23.2,
"bitrate": 14860800,
"tier": 0,
"w": 1920,
"net": null
},
{
"t": 14,
"fps": 54,
"mbps": 9.82,
"content_p50": 15.9,
"content_p95": 28.4,
"bitrate": 14860800,
"tier": 0,
"w": 1920,
"net": null
},
{
"t": 15,
"fps": 54,
"mbps": 11.11,
"content_p50": 15.5,
"content_p95": 26.1,
"bitrate": 14860800,
"tier": 0,
"w": 1920,
"net": null
},
{
"t": 16,
"fps": 54,
"mbps": 9.79,
"content_p50": 17.4,
"content_p95": 25.3,
"bitrate": 14860800,
"tier": 0,
"w": 1920,
"net": null
},
{
"t": 17,
"fps": 57,
"mbps": 9.3,
"content_p50": 16.3,
"content_p95": 23.5,
"bitrate": 14860800,
"tier": 0,
"w": 1920,
"net": null
},
{
"t": 18,
"fps": 55,
"mbps": 8.3,
"content_p50": 19.5,
"content_p95": 31.0,
"bitrate": 7984791,
"tier": 0,
"w": 1920,
"net": null
},
{
"t": 19,
"fps": 51,
"mbps": 8.56,
"content_p50": 16.4,
"content_p95": 29.1,
"bitrate": 10220855,
"tier": 0,
"w": 1920,
"net": null
},
{
"t": 20,
"fps": 40,
"mbps": 7.13,
"content_p50": 18.3,
"content_p95": 23.9,
"bitrate": 12025604,
"tier": 0,
"w": 1920,
"net": null
}
],
"adapt": [],
"content_p50": 19.9,
"content_p95": 37.2,
"input_p50": null,
"input_p95": null,
"grades": {
"content_p50": "local",
"content_p95": "local",
"fps": "acceptable",
"late_pct": "bad",
"stall_max": "local"
},
"source_fps": 58.7,
"cpu": {
"mac_agent_pct": 6.6,
"frame_viewer_pct": 72.4,
"frame_tailscaled_pct": 8.1,
"frame_sshd_pct": 0.9,
"frame_gamescope_pct": 10.0,
"frame_total_pct": 24.2
},
"viewer": "h264 software; ANGLE (Mesa, zink Vulkan 1.4(Turnip Adreno (TM) 750 (MESA_TURNIP)), OpenGL 4.6); raf 141 Hz",
"show_s": 6.43,
"panel": "valve.steam.desktopgame.2001414593",
"src": "separate:9384"
},
{
"scenario": "type",
"frames_sent": 25,
"frames_drawn": 25,
"frames_shown": 25,
"duration_s": 14.7,
"stages_ms": {
"capture": {
"p50": -0.3,
"p95": 0.9,
"p99": 1.1,
"n": 25
},
"queue": {
"p50": 0.0,
"p95": 0.0,
"p99": 0.0,
"n": 25
},
"encode": {
"p50": 5.0,
"p95": 33.4,
"p99": 33.6,
"n": 25
},
"socket": {
"p50": 0.1,
"p95": 0.1,
"p99": 0.2,
"n": 25
},
"network": {
"p50": 6.1,
"p95": 14.9,
"p99": 24.4,
"n": 25
},
"decode": {
"p50": 3.0,
"p95": 10.5,
"p99": 11.4,
"n": 25
},
"draw": {
"p50": 0.6,
"p95": 1.0,
"p99": 1.1,
"n": 25
},
"present": {
"p50": 0.8,
"p95": 1.4,
"p99": 1.7,
"n": 25
},
"content": {
"p50": 12.6,
"p95": 41.0,
"p99": 46.3,
"n": 25
},
"content_shown": {
"p50": 13.9,
"p95": 41.8,
"p99": 46.6,
"n": 25
}
},
"fps": 1.7,
"fps_shown": 1.7,
"late_pct": 100.0,
"stall_max": 3099.2,
"stalls_over_100ms": 21,
"mbps": 0.04,
"keyframes": 4,
"size": "960x646",
"captured": 79,
"viewer_never_drawn": 0,
"input_ms": {
"p50": 41.3,
"p95": 60.3,
"p99": 68.6,
"n": 18
},
"input_shown_ms": {
"p50": 43.0,
"p95": 61.1,
"p99": 69.6,
"n": 18
},
"input_parts_ms": {
"uplink": {
"p50": 5.3,
"p95": 11.6,
"p99": 14.2,
"n": 18
},
"mac": {
"p50": 23.1,
"p95": 35.0,
"p99": 42.6,
"n": 18
},
"back": {
"p50": 11.8,
"p95": 19.2,
"p99": 22.7,
"n": 18
}
},
"inputs": {
"sent": 66,
"seen": 18
},
"timeline": [
{
"t": 3,
"fps": 1,
"mbps": 0.01,
"content_p50": 11.8,
"content_p95": 11.8,
"bitrate": 1224075,
"tier": 4,
"w": 960,
"net": null
},
{
"t": 4,
"fps": 1,
"mbps": 0.02,
"content_p50": 14.7,
"content_p95": 14.7,
"bitrate": 2091896,
"tier": 4,
"w": 960,
"net": null
},
{
"t": 5,
"fps": 4,
"mbps": 0.1,
"content_p50": 18.6,
"content_p95": 37.4,
"bitrate": 3071304,
"tier": 3,
"w": 1440,
"net": null
},
{
"t": 6,
"fps": 5,
"mbps": 0.13,
"content_p50": 4.2,
"content_p95": 19.2,
"bitrate": 4031277,
"tier": 3,
"w": 1440,
"net": null
},
{
"t": 7,
"fps": 6,
"mbps": 0.04,
"content_p50": 12.0,
"content_p95": 46.3,
"bitrate": 300000,
"tier": 4,
"w": 960,
"net": null
},
{
"t": 10,
"fps": 1,
"mbps": 0.01,
"content_p50": 12.2,
"content_p95": 12.2,
"bitrate": 300000,
"tier": 4,
"w": 960,
"net": null
},
{
"t": 12,
"fps": 1,
"mbps": 0.02,
"content_p50": -1.2,
"content_p95": -1.2,
"bitrate": 842857,
"tier": 4,
"w": 960,
"net": null
},
{
"t": 14,
"fps": 1,
"mbps": 0.02,
"content_p50": 14.4,
"content_p95": 14.4,
"bitrate": 2091896,
"tier": 4,
"w": 960,
"net": null
}
],
"adapt": [],
"content_p50": 12.6,
"content_p95": 41.0,
"input_p50": 41.3,
"input_p95": 60.3,
"grades": {
"content_p50": "local",
"content_p95": "acceptable",
"input_p50": "local",
"input_p95": "local"
},
"source_fps": 5.4,
"cpu": {
"mac_agent_pct": 1.2,
"frame_viewer_pct": 5.0,
"frame_tailscaled_pct": 1.5,
"frame_sshd_pct": 0.2,
"frame_gamescope_pct": 4.7,
"frame_total_pct": 8.5
},
"viewer": "h264 software; ANGLE (Mesa, zink Vulkan 1.4(Turnip Adreno (TM) 750 (MESA_TURNIP)), OpenGL 4.6); raf 140 Hz",
"show_s": 5.12,
"panel": "valve.steam.desktopgame.2001910179",
"src": "separate:9510"
}
]
}
bench/results/2026-09-28-a3c6e5c-dirty-adapt-clean2.json
+1116
View File
File diff suppressed because it is too large. Load diff
bench/results/2026-09-28-a3c6e5c-dirty-baseline-fixed.json
+1124
View File
File diff suppressed because it is too large. Load diff
bench/results/2026-09-28-a3c6e5c-dirty-baseline.json
+1124
View File
File diff suppressed because it is too large. Load diff
docs/how-the-frame-works.md
+1
View File
@@ -52,6 +52,7 @@ Lepton (Android 11, podman container "lepton-dev") ← its own panel, app 305600
| Power actions need `sudo`, which asks for the Developer Mode password over SSH. | Frame Control's power buttons |
| **SSH server:** OpenSSH 9.7p1. It offers `publickey,password` (keyboard-interactive is off, PAM on) and also asks `userdbctl ssh-authorized-keys` for keys. OpenSSH ≥ 8.8 rejects SHA-1 `ssh-rsa` signatures by default, so a client whose RSA support is SHA-1 only (the Swift library Citadel, for one) can't log in with the RSA key that devkit pairing installs; use ed25519 (**inferred** from OpenSSH defaults). **Verified 2026-09-27**, BUILD_ID 20260922.6101926. | [iphone.md](iphone.md), `ui/frame_connect.py` |
| **A Chromium app window on gamescope's `:0` with its own `STEAM_GAME` becomes a panel, even when started over SSH.** `chromium-xr/chrome --ozone-platform=x11 --app=URL --window-size=1280,720` got a 1920×1080 window, and tagging it produced `[Overlays] Created: valve.steam.desktopgame.<id>` in `~/.local/share/Steam/logs/vrwebhelper_systemui.txt`. Chromium XR decoded a 1080p H.264 WebCodecs stream from the Mac at about 60 fps. XTest events sent to `:0` (libXtst through Python ctypes) did not reach the page. The Frame has `libXtst`, `xprop`, `xwininfo`, `curl` and the `C.utf8` locale. **Verified 2026-09-28**, BUILD_ID 20260925.6191901. | [mac-in-headset.md](mac-in-headset.md) |
| **Streaming video into a panel: what the Frame adds.** Chromium XR decodes H.264 in **software** (1920×1290 at about 9 Mbit/s took 4–6 ms per frame); its GL is ANGLE → zink → Turnip on the Adreno 750, and it logs `GetVSyncParametersIfAvailable() failed`. An idle page's `requestAnimationFrame` runs at about 140 Hz; when the headset is worn or woken, SteamVR picks the 4320×2160 @ 144 Hz mode. **An unworn headset throttles panel apps** whatever they draw: a local canvas page ran at 58 fps for about 6 s, then 28, then about 15 once in standby (vrserver logs `entering standby`, with `power.pauseCompositorOnStandby 1`). `vrcmd --handlewakeup` gave 137–144 fps for about 2 s, then 36. So a panel's frame rate and compositor delay can only be measured while it's worn. `tailscaled` runs with `--tun=userspace-networking` and cost about 8% of a core at 9 Mbit/s (0.5% over the LAN address). Wi-Fi power saving is on (`iw … get power_save`). **Verified 2026-09-28**, BUILD_ID 20260925.6191901. | [mac-in-headset.md](mac-in-headset.md#measuring) |
| **Tools on the image:** Python 3.12.3, `ffmpeg`, `openssl`, `curl`, `rsync`, `zip`/`unzip`, `flatpak`, `wpctl`, `podman`. **No `adb`.** `steamos` is uid 1000, in `wheel`, and sudoers has `%wheel ALL=(ALL) ALL`, so `sudo -S` takes the Developer Mode password on stdin. **Verified 2026-09-27.** | Running Frame Control's server on the Frame (`FRAME_LOCAL=1`, [iphone.md](iphone.md)) |
| **Each Lepton instance is a podman container** named `lepton-steamlaunch-<instance id>`, labelled with its ADB port (`podman ps --format '{{.Names}} {{.Labels.adb_port}}'`). `podman exec <container> /system/bin/sh -c '…'` runs Android's shell inside it with no adb at all (used for `pidof` and `logcat` by the app tester). Running `wm size`/`wm density` that way is untested. **Verified 2026-09-27.** | `ui/frame_android.py`, the iPhone app's display settings |
| **Asleep means off the network.** In standby the Frame stops answering on its LAN address, `frame.local` and Tailscale alike (`Host is down`, `No route to host`, timeouts), and ping fails. It was unreachable for about 2.5 hours until woken. Nothing over SSH can wake it. **Verified 2026-09-27.** | Frame Control's offline banner and retries |
docs/mac-in-headset.md
+158 -3
View File
@@ -99,9 +99,10 @@ ScreenCaptureKit (one window or display)
keyframe instead of showing old frames late.
- It falls back to JPEG stills (**Compatible** quality) where H.264 isn't
available.
- **Flow control.** When the link backs up, the agent skips capture frames
*before* encoding, so no reference frame goes missing. Once the link drains,
it sends the newest picture.
- **Flow control.** The agent never lets frames queue up anywhere on the
way. It skips capture frames *before* encoding, so no reference frame goes
missing, and it lowers the bitrate, then the frame rate, then the size, to
fit the link (see [Adapting to the network](#adapting-to-the-network)).
- **Input.**
- A click on a window's panel brings that Mac window to the front
(Accessibility API), then clicks at the same point. Double clicks, right
@@ -133,6 +134,157 @@ may ask again after an update.
| Light | 1280 | 30 | H.264 | Weak Wi-Fi or Tailscale off the LAN |
| Compatible | 1280 | 20 | JPEG | A Frame browser without H.264 |
## Measuring
Every frame carries a sequence number, and the agent records its journey on
the Mac's clock (`Sources/Stats.swift`):
| Stage | From → to |
|---|---|
| capture | the Mac composited it (ScreenCaptureKit's display time) → the agent got it |
| queue, encode | → encoding started → the encoder finished |
| network | → the viewer received it |
| decode, draw | → WebCodecs decoded it → it was drawn on the page's canvas |
| present | → the page's next animation frame |
- **Clock sync.** The viewer syncs its clock to the Mac's the way NTP does:
it pings over the stream's own WebSocket and keeps the sample with the
shortest round trip. It then reports, in Mac time, when each frame arrived
(right away, so the agent can pace itself) and when it was decoded and
drawn (in batches every 250 ms).
- **Input.** The first frame captured after a click or key carries that
event's id. So input latency is the viewer's event → injected on the Mac →
the first frame after it → drawn in the headset.
- **Where to see it.**
- `GET /stats` (key required) returns every frame and input record.
- `/status` includes a two-second summary, which Frame Control's card
shows next to each live stream.
- In the headset, add `?stats=1` to the viewer or press
Ctrl+Alt+Shift+S for an overlay.
- **The benchmark.** `scripts/macview-bench.py` runs fixed scenarios on the
real Frame, from the Mac, with nobody wearing the headset:
- **test** is the moving test pattern.
- **scroll** is a Chrome page on its own display scrolling at 240 pt/s,
which gives about 9 Mbit/s of real 1920×1290 video.
- **type** types into a Chrome text box, first fast and then with pauses.
It writes `bench/results/<date>-<commit>-<label>.json`, compares two
results, and runs interleaved A/B tests between agent settings (`ab`).
Wi-Fi changes from minute to minute, so single runs at different times
aren't comparable. Throttled links come from a shaping relay on the Mac,
which needs no sudo (`--net 50@0,3@8,50@16` means 50 Mbit/s, then 3 from
8 s, then 50 from 16 s).
- **What's graded.** "Content" runs from when the Mac composited a frame
(or when ScreenCaptureKit delivered it, if that was earlier) to when it was
drawn in the viewer. The Frame's compositor adds its own delay after that.
That part is reported, but not graded: an unworn Frame throttles panels to
about 36 fps after a few seconds, and to 15 fps in standby, whatever they
draw. This was **verified** with a local canvas page that ran on the Frame
with no network involved (`bench/pages/present.html`). The compositor's
share needs a run with the headset worn.
Targets: content p50 ≤ 25 ms (p95 ≤ 40), click to photon p50 ≤ 50 ms
(p95 ≤ 70), 60 fps with ≤ 1% late frames, no stall over 100 ms, and adapting
to a new link rate within 1 s.
Baseline on 2026-09-28 (**verified**, home Wi-Fi, Tailscale, Balanced,
`bench/results/2026-09-28-a3c6e5c-dirty-baseline-fixed.json`; ms p50/p95):
| Scenario | Content | Input to drawn | fps drawn | Notes |
|---|---|---|---|---|
| test (1280×720) | 9.9 | 28.8 / 39.0 | 59 | encode 4.0, network 4.0, decode 1.3 |
| scroll (1920×1290) | 14.7 | – | 50.4 | encode 6.7, decode 6.4 (software), 9.4 Mbit/s, 16% late |
| type (1920×1290) | 16.7 | 51.2 / 73.2 | – | most of the input time is the Mac app reacting |
What was learned (all **verified**, unless marked):
- The biggest costs are encoding (4–7 ms), network (4–6 ms), and decoding
on the Frame. Chromium XR on the Frame decodes H.264 in **software**.
- Wi-Fi alone stalls for 240–580 ms now and then, over Tailscale and over
the LAN alike. Over the LAN (`--host 192.168.1.237`) latency was no
better, but the Frame used about 8% less CPU, because tailscaled runs in
userspace there.
- With no controller, a link that slows down queues without limit. In one
run, frames arrived 1.9 s late, and at worst 9.4 s late.
- Tried, and no help, so not kept: VideoToolbox options (require hardware,
no frame delay, prioritise speed, a hard data-rate cap), Chromium flags
(`--disable-gpu-vsync`, `--disable-frame-rate-limit`,
`--use-angle=vulkan`), and a 120 Hz virtual display.
- Inconclusive, so off by default: keeping the Frame's Wi-Fi awake during
typing (`FRAME_MAC_VIEW_WARM=40`, a tiny message every 40 ms for 5 s
after input). Over three interleaved runs each, input p50 went 44 → 47 ms
and p95 92 → 71 ms, and the ranges overlapped widely
(`…-ab-keepwarm.json`). In that run, and in the baseline's typing, the
harness typed spaces as "+" (a URL-encoding bug, since fixed), so they
went through the text path rather than as space keys.
- Pointer moves now go out on an 8 ms timer, not on the page's next
animation frame, which an unworn Frame slows to 15–36 Hz. This is
**inferred** to help dragging; the benchmark has no drag scenario yet.
- Kept: the encoder's timestamps never jump more than two frame intervals.
Before this, the first frame after a pause got a quarter of a second's
bit budget, and one P-frame reached 204 KB.
## Adapting to the network
`Sources/Controller.swift`, per stream, latency first:
- **The gate.** The viewer acknowledges every frame as it arrives. A new
frame is sent only while the oldest unacknowledged one is younger than
the path's usual round trip, plus one frame interval, plus room for this
link's normal jitter (1.5 times its recent spread, 25–80 ms). So frames
never queue in SSH, TCP or the Wi-Fi driver. While the link is stuck, the
newest picture waits and goes out as soon as it moves.
- **The bitrate.** The link counts as congested when, for two checks in a
row (100 ms apart), round trips grow by more than 40 ms while the stream
uses much of its budget, or the gate holds frames back, or a frame is
stuck for 100 ms. Then the bitrate drops to a bit under what actually got
through: at least a fifth off, and at most half. Once the link has been
clear for a second, it rises by 10% steps, never above the quality
setting's bitrate.
- **The tier.** When the bitrate stays low, and the stream is really
limited by the link rather than having little to send, it steps down:
60 → 45 → 30 fps, then 75%, then 50% of the pixels. It goes straight to
the tier the bitrate supports after half a second, and steps back up one
tier at a time after two seconds with room to spare.
- `FRAME_MAC_VIEW_ADAPT=0` turns it off, for comparison.
Measured on the real Frame, 2026-09-28 (**verified**; interleaved A/B, off
versus on, medians of the runs, ms):
| Link | Scenario | Content p95, off → on | fps drawn, off → on | Result file |
|---|---|---|---|---|
| Clean Wi-Fi (3 runs each) | scroll | 32.3 → 33.6 | 57 → 56.2 | `…-ab-adapt-clean2.json` |
| Clean Wi-Fi | test | 15 → 14.2 | 60 → 59.7 | same |
| 50 → 3 → 50 Mbit/s at 8 s and 16 s (2 runs each) | scroll | **4670 → 72** | 45 → 42 | `…-ab-adapt-step3.json` |
| 50 → 3 → 50 Mbit/s | test | 66 → 16 | 60 → 60 | same |
- **Clean link.** On a clean link it costs nothing measurable. An earlier
version with a fixed gate slack lost 9 fps to Wi-Fi jitter while
scrolling (47 → 38 fps), and that's why the slack now follows the link's
jitter.
- **Throttled link.** Without the controller, frames queued for up to 5.6 s
and never caught up while the link was slow (p95 1.8–5.6 s, second by
second). With it, in the two runs:
| | Run 1 | Run 2 |
|---|---|---|
| Worst second's p95 just after the drop | 219 ms | 428 ms |
| p95 back under 100 ms for 3 s in a row | after 1 s | after 4 s |
| Stepped down to 1440 px at 30 fps | 1.8 s after the drop | 3.5 s after |
| p95 per second after that, on the 3 Mbit/s link | 55–94 ms | 60–148 ms |
Sending one frame takes about 27 ms on that link by itself. After the
link recovered, the stream was back at full size and 60 fps in about
7.5 s. It steps up one tier every two seconds, on purpose, so it doesn't
bounce. The 1 s adaptation target was met in one run of two.
- **A hiccup on a small stream.** In one clean run, a Wi-Fi hiccup made an
earlier version halve the test pattern's bitrate five times and drop it
to half size. That cut couldn't help: the stream only sends
0.47 Mbit/s. Now the controller estimates what a stream wants (captures
per second × average frame size). While a stream wants about half its
budget or less, no cut takes it below twice what it wants, so it doesn't change
tier.
## Checked so far (2026-09-28)
- **Mac (checked by hand, macOS 26.5.2).**
@@ -191,6 +343,9 @@ may ask again after an update.
- Whether Flathub Chromium has H.264. Chromium XR is used when it's
installed, as it is on this Frame.
- Latency with real, busy windows at Sharp.
- A benchmark run while wearing the headset. Only then does the Frame show
panels at full rate, so only then can the compositor's share of the
latency, and the frame rate you actually see, be measured.
## Limits
docs/mac-window-separation.md
+200
View File
@@ -0,0 +1,200 @@
# Real separation of Mac windows in the headset
Research and experiments on 2026-09-28 (macOS 26.5.2, Apple M5 Pro), for
making each streamed Mac window truly independent. Builds on
[mac-in-headset.md](mac-in-headset.md). Labels: **verified** (tried here),
**documented**, **source** (read in someone's code) or **reported**.
## What "separate" lacks today
Today each window is captured on its own
(`SCContentFilter(desktopIndependentWindow:)`), but it still lives on the
Mac's one desktop:
- **Clicks.** A click has to raise the window first, so it reorders the
Mac's windows, steals focus and moves the real cursor.
- **Child windows.** A window's menus, popovers, sheets and tooltips are
separate windows, so they aren't in its panel. Apple documents that the
single-window filter leaves them out.
- **Size.** A panel's size is tied to the window's size on the Mac screen.
- **Hidden windows.** Minimised windows, and windows on other Spaces, can't
be shown live.
## How the Mac draws, and where pixels can be read
Apps draw with Core Animation into IOSurfaces. WindowServer's compositor
stacks every window onto each display, including virtual ones, and sends
the result to the screen. Pixels can be read at three points:
| Where | How | Gets | Doesn't get |
|---|---|---|---|
| One window's content | ScreenCaptureKit `desktopIndependentWindow` (public, what we use) | Live, zero-copy, even when covered by other windows (**documented**) | Menus, popovers, sheets. It pauses while the window is minimised (**reported**) |
| One window plus its children | `SCStreamConfiguration.includeChildWindows` (macOS 14.2+, **reported**) | Menus, popovers and sheets attached to the window | Anything the app puts outside the window's bounds |
| A whole display | ScreenCaptureKit display filter, with apps or windows included or excluded | Everything on that display, cursor included | Only what's on that display |
| A snapshot of any window | Private `CGSHWCaptureWindowList`, which AltTab uses for minimised windows and other Spaces (**source**: `alt-tab-macos` `PrivateApis.swift`) | Minimised windows and other Spaces | It's one still image, not a live stream |
| Another app's layer tree | Private `CALayerHost` with a context id | A live, zero-copy picture | It only works when the other app cooperates, so it's no good for arbitrary windows (**reported**) |
`CGWindowListCreateImage` and `CGDisplayStream` are obsolete in the macOS 15
SDK (**reported** by MacPorts and JUCE). Nothing reads another app's pixels
without the Screen Recording permission.
## The idea: each window gets its own virtual display
A virtual display (the private `CGVirtualDisplay`, as used by BetterDisplay,
DeskPad and quest-display) is a compositor target with no physical screen.
Put one streamed window alone on its own virtual display, sized to its
panel, and capture the whole display:
- **Nothing can overlap it.** A plain click at that point always lands on
that window, so input doesn't need raising or background-event tricks.
- **Menus, sheets, popovers, tooltips and context menus appear on the same
display, so they're in the panel.** With "Displays have separate Spaces"
on (it is on this Mac), each display also has its own menu bar, so the
app's menu bar can be part of the panel.
- **The panel's size is the display's size,** in HiDPI. Resizing the panel
means changing the display mode and resizing the window to fill it
(Accessibility API).
- **Minimised windows and other Spaces stop being a problem,** because
streamed windows live on their own displays.
- **The cursor goes where the laser points.** That display's panel is the
one being used, much as Vision Pro's Mac Virtual Display works. Keys from
the Mac's keyboard go to the window last clicked.
### Verified here (no permission needed)
- **Creating one.** It needs no permission or entitlement. 1920×1080 HiDPI
gives a 3840×2160-pixel, 60 Hz display, placed next to the built-in
screen, and `NSScreen.screensHaveSeparateSpaces` was true.
- **Many at once.** 16 were created at once with no error.
- **Placement.** `CGConfigureDisplayOrigin` far away failed with error
1014: macOS keeps displays edge to edge. Disabling one with the private
`CGSConfigureDisplayEnabled` from a command-line tool also failed with 1014.
- **Removal is deferred while the physical screen sleeps.** With the
built-in display asleep, virtual displays were not removed when released,
or when their process exited, even from their own `.app`. A second display
with the same vendor, product and serial couldn't be created. All of them
disappeared as soon as the screen woke (`caffeinate -u`). The helper must
therefore:
- keep a fixed pool of displays and reuse them rather than create new ones;
- keep the screen awake while they exist, which it already does while
streaming.
### Built: "Give each window its own display"
Frame Control's helper now does this (`mac/frame-mac-view/Sources/Separate.swift`),
and it's the default in the Tools card. Streams of this kind are named
`separate:<window id>`.
- For each window shown, it creates a HiDPI virtual display. The display is
sized to the window plus a menu bar, with a fresh serial each time, so a
display left over while the screen slept can't block a new one.
- The window is moved onto the display and resized to fill it (Accessibility
API), and the whole display is captured.
- On **Stop** the window goes back where it was, and the display is released.
- Plain per-window capture is still there: untick "Give each window its own
display". Separate mode needs Accessibility (to move the window), and
without it, Show says so rather than quietly falling back.
Verified 2026-09-28 (macOS 26.5.2, M5 Pro), using a TextEdit test document
and the benchmark's Chrome windows:
- **Separation works.** The window moved onto its own display and streamed,
with its first frames in 3.8 s. The display showed TextEdit's own menu bar.
- **Child windows come along.** A context menu and the Page Setup sheet
opened on the same display and were in the capture.
- **Input.** Typing through the stream reached the window. The benchmark's
typing scenario does this on every run.
- **Stop.** Stop put the window back at exactly its old size (656×422), and
the display was removed.
- **The helper must run a real Cocoa event loop.** Earlier, it ran only a
`RunLoop`. With that, AppKit never learned about new displays:
- `NSScreen.screens` never listed them, so placing the window always waited
3 s and then gave up;
- the display's HiDPI mode never applied, so windows were captured at 1×
(1280×860 instead of 1920×1290 pixels) and text was soft.
Running `NSApplication` (with no Dock icon) fixed both. The screen now
appears within 100 ms, and captures are at 2× (**verified** in
`bench/results/*-baseline.json` against `*-baseline-fixed.json`).
- **Frame rate.** Chrome drew at 60–61 fps on the virtual display, but
ScreenCaptureKit delivered only about 46–53 fps from it, whereas the
built-in ProMotion screen gave 121 fps (**verified**). Neither a 120 Hz
virtual display (`FRAME_MAC_VIEW_VD_HZ=120`) nor a looser
`minimumFrameInterval` changed that. Cause unknown.
- **Windows that keep their own size** (Calculator, 230×408). The window
moved and streamed, but stayed small in the display's corner, so the panel
was mostly wallpaper. Now only that corner is captured
(`SCStreamConfiguration.sourceRect`): the menu bar and the window, at least
480×360 points so menus fit. Clicks map to the cropped area. The first
frame arrived in 0.6–1.1 s, and on Stop the window went back and the
display was removed. A display's menu bar names the active app, so it
shows the window's own menus only once the panel has been clicked.
- **Several windows at once** (on the Mac, with a local stand-in viewer that
acknowledges frames). Three and four Chrome windows, each scrolling on its
own display at 1920×1290 pixels, streamed at 53–56 fps and about
9 Mbit/s each. The helper used 20% (three) or 24% (four) of one CPU core.
The hardware encoder is shared: its time per frame went from 6.7 ms for
one stream to 7–15 ms for three and 11–25 ms for four, so each extra
moving window adds latency to the others. Once in four runs, before a fix,
one window left its display when several displays appeared at the same
moment, and its stream stopped: nothing on that display was changing any
more. The helper now checks every second and puts the window back. Three
further runs with four windows were clean, and every display was gone
afterwards (checked with `CGGetOnlineDisplayList`).
- **Quitting.** On SIGTERM, or when Frame Control goes away, the helper ends
every stream first, so windows go back before their displays disappear.
Verified: a 900×600 window was back at 900×600 after SIGTERM. Closing a
viewer 0.05–1.5 s into startup left the window at its old size and no
extra display.
- **A consent prompt.** macOS 26 asks whether to let ScreenCaptureKit apps
"bypass the system private window picker". The prompt appeared *on the
virtual display*, so it would show up inside the headset panel. Allow it
once on the Mac.
### Still to check
1. That a context menu opens over the text being clicked, not just somewhere
on the display. This needs a retest after the consent prompt above is
allowed.
2. Stage Manager, which is reported to undo Accessibility resizes.
3. Where the Dock and the cursor go on a virtual display.
4. Closing the lid with virtual displays attached (clamshell).
5. Many moving windows at once in the headset: whether to lower the bitrate
or frame rate of panels you aren't using, so the one you are using keeps
the encoder to itself.
6. All of it in the headset, with the laser.
## Input without disturbing the Mac (a finer option)
For windows that stay on the Mac's own screen, input can go to a window
behind others without raising it:
- **Background click.** `CGEventPostToPid` with the CGEvent fields 91 and
92, which say which window a click is for (`kCGMouseEventWindowUnderMousePointer`
and `...ThatCanHandleThisEvent`), plus a window-relative location
(**reported**, reverse-engineered, needs testing).
- **Focus without raise.** yabai makes a window key without raising it by
posting event records with the private `SLPSPostEventRecordTo` (**source**:
`yabai/src/window_manager.c`).
- **Known failures.**
- Chromium and Electron ignore background clicks unless they're built
with the 2026 `acceptsFirstMouse` fix (electron/electron#54493).
- Games and canvas apps often need real activation.
- Password fields (Secure Event Input) drop synthetic keys.
- IME composition, as for Chinese or Japanese input, isn't reliable.
With a virtual display per window, most of this isn't needed. It's worth
having for hovering over one panel while typing in another.
## Encoding and transport
- **Codec.** Keep H.264 4:2:0. Apple's High Performance Screen Sharing uses
4:4:4 over two virtual displays (**documented**), but the Frame's Chromium
decodes H.264 in software, and no 4:4:4 decode path is confirmed there.
- **Sharper static text.** When a panel has been still for a moment, send a
high-quality refresh, either a keyframe at low quantisation or a lossless
WebP overlay, so static text is sharp. Moving content stays as video.
- **Transport.** Keep WebSocket over SSH for now, as it works everywhere.
WebRTC (UDP, congestion control) is the proven step up for Wi-Fi.
WebTransport over QUIC is promising, but its server side on macOS is
unverified.
mac/frame-mac-view/Sources/CGVirtualDisplay.h
+28
View File
@@ -0,0 +1,28 @@
// Private CoreGraphics classes for virtual displays (as used by DeskPad,
// BetterDisplay and quest-display). Not in any SDK; see
// docs/mac-window-separation.md.
#import <Foundation/Foundation.h>
#import <CoreGraphics/CoreGraphics.h>
@interface CGVirtualDisplayDescriptor : NSObject
@property (retain, nonatomic) dispatch_queue_t queue;
@property (retain, nonatomic) NSString *name;
@property (nonatomic) unsigned int maxPixelsHigh;
@property (nonatomic) unsigned int maxPixelsWide;
@property (nonatomic) CGSize sizeInMillimeters;
@property (nonatomic) unsigned int serialNum;
@property (nonatomic) unsigned int productID;
@property (nonatomic) unsigned int vendorID;
@property (copy, nonatomic) void (^terminationHandler)(id, id);
@end
@interface CGVirtualDisplayMode : NSObject
- (instancetype)initWithWidth:(unsigned int)width height:(unsigned int)height refreshRate:(double)refreshRate;
@end
@interface CGVirtualDisplaySettings : NSObject
@property (retain, nonatomic) NSArray *modes;
@property (nonatomic) unsigned int hiDPI;
@end
@interface CGVirtualDisplay : NSObject
@property (readonly, nonatomic) unsigned int displayID;
- (instancetype)initWithDescriptor:(CGVirtualDisplayDescriptor *)descriptor;
- (BOOL)applySettings:(CGVirtualDisplaySettings *)settings;
@end
mac/frame-mac-view/Sources/Capture.swift
+89 -14
View File
@@ -10,6 +10,7 @@ import ScreenCaptureKit
enum Source: Equatable {
case window(CGWindowID)
case display(CGDirectDisplayID)
case separate(CGWindowID) // the window on a display of its own
case test
init?(_ s: String) {
@@ -17,6 +18,7 @@ enum Source: Equatable {
switch (parts.first, parts.count > 1 ? UInt32(parts[1]) : nil) {
case ("window", let id?): self = .window(id)
case ("display", let id?): self = .display(id)
case ("separate", let id?): self = .separate(id)
case ("test", _): self = .test
default: return nil
}
@@ -26,6 +28,7 @@ enum Source: Equatable {
switch self {
case .window(let id): return "window:\(id)"
case .display(let id): return "display:\(id)"
case .separate(let id): return "separate:\(id)"
case .test: return "test"
}
}
@@ -76,14 +79,20 @@ struct WindowInfo {
}
protocol CaptureSource: AnyObject {
var onFrame: ((CVPixelBuffer, CMTime) -> Void)? { get set }
/// The picture, its presentation time, and when the Mac composited it (µs, host clock).
var onFrame: ((CVPixelBuffer, CMTime, Int64) -> Void)? { get set }
var onEnded: ((String) -> Void)? { get set }
var onChange: (() -> Void)? { get set } // title or size changed
/// True once the window or display is gone for good.
var gone: Bool { get }
/// Points on the Mac's global display space that the picture covers.
var frameRect: CGRect { get }
var title: String { get }
var app: String { get }
var pid: pid_t? { get }
func start(maxLong: Int, fps: Int, completion: @escaping (String?) -> Void)
/// A smaller or larger picture from now on (the long side, in pixels).
func setMaxLong(_ maxLong: Int)
func stop()
func pointer(x: Double, y: Double, text: String?) // for the test pattern
}
@@ -95,7 +104,7 @@ extension CaptureSource {
/// ScreenCaptureKit, for a window or a display.
final class SCKSource: NSObject, CaptureSource, SCStreamOutput, SCStreamDelegate {
let source: Source
var onFrame: ((CVPixelBuffer, CMTime) -> Void)?
var onFrame: ((CVPixelBuffer, CMTime, Int64) -> Void)?
var onEnded: ((String) -> Void)?
private(set) var frameRect = CGRect.zero
private(set) var title = ""
@@ -113,9 +122,19 @@ final class SCKSource: NSObject, CaptureSource, SCStreamOutput, SCStreamDelegate
/// The window or display no longer exists, so retrying can't help.
private(set) var gone = false
var onChange: (() -> Void)? // title or size changed
/// For a display: capture only this part of it (display points, top-left
/// origin). Set before start().
var crop: CGRect?
init(_ source: Source) { self.source = source }
/// What's captured, in global points: the display, or the cropped part of it.
private func displayRect(_ id: CGDirectDisplayID) -> CGRect {
let b = CGDisplayBounds(id)
guard let c = crop else { return b }
return c.offsetBy(dx: b.minX, dy: b.minY).intersection(b)
}
func start(maxLong: Int, fps: Int, completion: @escaping (String?) -> Void) {
self.maxLong = maxLong
SCShareableContent.getExcludingDesktopWindows(true, onScreenWindowsOnly: false) { [self] content, error in
@@ -148,11 +167,17 @@ final class SCKSource: NSObject, CaptureSource, SCStreamOutput, SCStreamDelegate
filter = SCContentFilter(display: d, excludingWindows: [])
title = displayName(id)
app = "Mac"
frameRect = CGDisplayBounds(id)
case .test:
frameRect = displayRect(id)
if crop != nil { config.sourceRect = frameRect.offsetBy(dx: -CGDisplayBounds(id).minX, dy: -CGDisplayBounds(id).minY) }
case .test, .separate:
return completion("not a ScreenCaptureKit source")
}
scale = Double(filter.pointPixelScale)
// A display's own mode knows best: ScreenCaptureKit can still say 1
// for a virtual display that has only just switched to HiDPI.
if case .display(let id) = source, let mode = CGDisplayCopyDisplayMode(id), mode.width > 0 {
scale = max(scale, Double(mode.pixelWidth) / Double(mode.width))
}
let (w, h) = fitSize(width: frameRect.width * scale, height: frameRect.height * scale, maxLong: maxLong)
config.width = w
config.height = h
@@ -195,8 +220,10 @@ final class SCKSource: NSObject, CaptureSource, SCStreamOutput, SCStreamDelegate
}
}
/// Windows move, resize, retitle and close; ScreenCaptureKit doesn't say.
/// Windows move, resize, retitle and close, and displays change mode;
/// ScreenCaptureKit doesn't say.
private func startPolling() {
if case .display(let id) = source { return pollDisplay(id) }
guard case .window(let id) = source else { return }
let t = DispatchSource.makeTimerSource(queue: queue)
t.schedule(deadline: .now() + 1, repeating: 1)
@@ -224,14 +251,48 @@ final class SCKSource: NSObject, CaptureSource, SCStreamOutput, SCStreamDelegate
poll = t
}
func setMaxLong(_ n: Int) {
queue.async {
self.maxLong = n
guard self.stream != nil else { return }
let (w, h) = fitSize(width: self.frameRect.width * self.scale, height: self.frameRect.height * self.scale, maxLong: n)
guard w != self.config.width || h != self.config.height else { return }
self.config.width = w
self.config.height = h
self.stream?.updateConfiguration(self.config) { _ in }
}
}
private func pollDisplay(_ id: CGDirectDisplayID) {
let t = DispatchSource.makeTimerSource(queue: queue)
t.schedule(deadline: .now() + 0.5, repeating: 1)
t.setEventHandler { [weak self] in
guard let self, let mode = CGDisplayCopyDisplayMode(id), mode.width > 0 else { return }
let bounds = self.displayRect(id), scale = Double(mode.pixelWidth) / Double(mode.width)
let (w, h) = fitSize(width: bounds.width * scale, height: bounds.height * scale, maxLong: self.maxLong)
self.frameRect = bounds
guard w != self.config.width || h != self.config.height else { return }
self.scale = scale
self.config.width = w
self.config.height = h
self.stream?.updateConfiguration(self.config) { _ in }
self.onChange?()
}
t.resume()
poll = t
}
func stream(_ stream: SCStream, didOutputSampleBuffer sample: CMSampleBuffer, of type: SCStreamOutputType) {
guard type == .screen, sample.isValid, let pb = CMSampleBufferGetImageBuffer(sample) else { return }
// Only complete frames carry new pixels; idle and blank ones don't.
if let atts = CMSampleBufferGetSampleAttachmentsArray(sample, createIfNecessary: false) as? [[SCStreamFrameInfo: Any]],
let raw = atts.first?[.status] as? Int, let status = SCFrameStatus(rawValue: raw), status != .complete {
let info = (CMSampleBufferGetSampleAttachmentsArray(sample, createIfNecessary: false) as? [[SCStreamFrameInfo: Any]])?.first
if let raw = info?[.status] as? Int, let status = SCFrameStatus(rawValue: raw), status != .complete {
return
}
onFrame?(pb, CMSampleBufferGetPresentationTimeStamp(sample))
let pts = CMSampleBufferGetPresentationTimeStamp(sample)
// When WindowServer composited it: the moment it showed on the Mac.
let shown = (info?[.displayTime] as? UInt64).map(hostUs) ?? hostUs(pts)
onFrame?(pb, pts, shown)
}
func stream(_ stream: SCStream, didStopWithError error: Error) {
@@ -251,8 +312,10 @@ func displayName(_ id: CGDirectDisplayID) -> String {
/// A moving test card: bars, a clock and a frame counter, plus a dot where the
/// viewer's pointer is and the last key it sent, so input can be checked too.
final class TestSource: CaptureSource {
var onFrame: ((CVPixelBuffer, CMTime) -> Void)?
var onFrame: ((CVPixelBuffer, CMTime, Int64) -> Void)?
var onEnded: ((String) -> Void)?
var onChange: (() -> Void)?
let gone = false
let frameRect = CGRect(x: 0, y: 0, width: 1280, height: 720)
let title = "Test pattern"
let app = "Frame Control"
@@ -267,10 +330,7 @@ final class TestSource: CaptureSource {
func start(maxLong: Int, fps: Int, completion: @escaping (String?) -> Void) {
size = fitSize(width: 1280, height: 720, maxLong: maxLong)
let attrs: [CFString: Any] = [kCVPixelBufferPixelFormatTypeKey: kCVPixelFormatType_32BGRA,
kCVPixelBufferWidthKey: size.0, kCVPixelBufferHeightKey: size.1,
kCVPixelBufferIOSurfacePropertiesKey: [:] as CFDictionary]
CVPixelBufferPoolCreate(nil, nil, attrs as CFDictionary, &pool)
makePool()
let t = DispatchSource.makeTimerSource(queue: queue)
t.schedule(deadline: .now(), repeating: 1.0 / Double(fps))
t.setEventHandler { [weak self] in self?.draw() }
@@ -284,6 +344,21 @@ final class TestSource: CaptureSource {
timer = nil
}
func setMaxLong(_ n: Int) {
queue.async {
self.size = fitSize(width: 1280, height: 720, maxLong: n)
self.makePool()
}
}
private func makePool() {
let attrs: [CFString: Any] = [kCVPixelBufferPixelFormatTypeKey: kCVPixelFormatType_32BGRA,
kCVPixelBufferWidthKey: size.0, kCVPixelBufferHeightKey: size.1,
kCVPixelBufferIOSurfacePropertiesKey: [:] as CFDictionary]
pool = nil
CVPixelBufferPoolCreate(nil, nil, attrs as CFDictionary, &pool)
}
func pointer(x: Double, y: Double, text: String?) {
queue.async {
if x >= 0 { self.dot = (x, y) }
@@ -334,6 +409,6 @@ final class TestSource: CaptureSource {
ctx.fillEllipse(in: CGRect(x: CGFloat(px) * CGFloat(w) - r, y: (1 - CGFloat(py)) * CGFloat(h) - r, width: 2 * r, height: 2 * r))
}
n += 1
onFrame?(pb, CMClockGetTime(CMClockGetHostTimeClock()))
onFrame?(pb, CMClockGetTime(CMClockGetHostTimeClock()), nowUs())
}
}
mac/frame-mac-view/Sources/Controller.swift
+234
View File
@@ -0,0 +1,234 @@
// Adapts each stream to its network, latency first. The viewer acknowledges
// every frame as it arrives ("rx"); from those acks the controller knows how
// long frames take to get through and how fast the link delivers them.
//
// - The gate: a new frame is sent only while the oldest unacknowledged one is
// younger than the path's usual round trip plus a little slack. So frames
// never queue up in SSH, TCP or the Wi-Fi driver; while the link is stuck
// the newest picture waits and goes out as soon as it moves again.
// - The bitrate: when frames start queueing (the round trip grows) or the
// gate has to hold frames back, it drops to a bit under what the link
// actually delivered; once things are clear it probes up again slowly, never
// above the quality setting's bitrate (the ceiling).
// - The tier: as the bitrate falls, fewer frames per second (60, 45, 30), then
// a smaller picture (75%, then 50% of the panel's pixels).
// See docs/mac-in-headset.md ("Adapting to the network"). Thread-safe.
import Foundation
final class RateController {
struct Tier: Equatable {
let fps: Int
let scale: Double // of the picture's long side
}
static let tiers = [Tier(fps: 60, scale: 1), Tier(fps: 45, scale: 1), Tier(fps: 30, scale: 1),
Tier(fps: 30, scale: 0.75), Tier(fps: 30, scale: 0.5)]
/// A tier is used while the target bitrate is at least this share of the ceiling.
static let floors = [0.45, 0.28, 0.16, 0.08, 0]
static let enabled = ProcessInfo.processInfo.environment["FRAME_MAC_VIEW_ADAPT"] != "0"
let maxFps: Int
private let lock = NSLock()
private var ceiling = 0 // bits/s at full size and frame rate
private(set) var target = 0
private(set) var tier = 0
private var unacked: [(seq: UInt32, sent: Int64, bytes: Int)] = []
/// Round trips (send -> ack arrives here), for the baseline: the lowest
/// in the last 10 s is the path without any queue.
private var rtts: [(t: Int64, v: Int64)] = []
private var acked: [(t: Int64, bytes: Int)] = [] // the last second
private var sentLog: [(t: Int64, bytes: Int)] = []
private var captures: [Int64] = [] // the last second
private var frameBytes = 0 // average recent frame, kept while the gate holds everything back
private var held = 0 // frames the gate held back since the last update
private var lastSignal = false
private var sawAck = false
private var lastDecrease: Int64 = 0
private var lastIncrease: Int64 = 0
private var belowSince: Int64 = 0, aboveSince: Int64 = 0
/// What changed, for the timeline: (time, event).
private(set) var events: [(Int64, String)] = []
init(maxFps: Int) { self.maxFps = maxFps }
private func locked<T>(_ f: () -> T) -> T { lock.lock(); defer { lock.unlock() }; return f() }
/// The quality setting's bitrate at full size; the first call also starts there.
func setCeiling(_ bps: Int) {
locked {
if target == 0 || target > bps { target = bps }
ceiling = bps
}
}
var fps: Int { locked { fpsLocked } }
private var fpsLocked: Int { min(maxFps, RateController.tiers[tier].fps) }
var scale: Double { locked { RateController.tiers[tier].scale } }
var baseRtt: Int64 { locked { baseline() } }
private func baseline() -> Int64 { rtts.map(\.v).min() ?? 0 }
/// How late a frame may be before the gate holds the next one: one frame
/// interval, plus room for the jitter this link normally has (1.5 times
/// its recent spread), so ordinary Wi-Fi jitter doesn't cost frames but a
/// real queue does. Updated in update().
private var slack: Int64 = 40_000
/// Whether a frame may be sent now without queueing behind earlier ones.
/// `counts`: a held frame is a sign of congestion (not when merely
/// re-checking whether a held frame can go yet).
func maySend(now: Int64, counts: Bool = true) -> Bool {
locked {
guard RateController.enabled, sawAck else { return true } // not heard from the viewer yet
// Unacknowledged for 2 s: gone with a reconnection, not in a queue.
unacked.removeAll { now - $0.sent > 2_000_000 }
guard let oldest = unacked.first else { return true }
// Age is what bounds latency. The count only stops a burst, and it
// allows a full round trip of frames, so a long but clear path
// (100 ms away) still gets every frame.
let interval = Int64(1_000_000 / max(1, fpsLocked))
let window = max(3, Int((baseline() + slack) / interval) + 1)
if unacked.count < window, now - oldest.sent <= baseline() + slack { return true }
if counts { held += 1 }
return false
}
}
/// A picture was captured (sent or not): with the frame sizes, what this
/// stream would send if the link allowed.
func captured(at t: Int64) {
locked {
captures.append(t)
if captures.count > 256 { captures.removeFirst(captures.count - 256) }
}
}
func sent(seq: UInt32, bytes: Int, at t: Int64) {
locked {
// Bounded even if the viewer never acknowledges (an old viewer, or
// the controller is off).
unacked.append((seq, t, bytes))
if unacked.count > 512 { unacked.removeFirst(unacked.count - 512) }
sentLog.append((t, bytes))
if sentLog.count > 1024 { sentLog.removeFirst(sentLog.count - 1024) }
}
}
/// The viewer has frame `seq`. Returns true if that may let a held frame go.
func acked(seq: UInt32, at now: Int64) -> Bool {
locked {
sawAck = true
guard let i = unacked.firstIndex(where: { $0.seq == seq }) else { return false }
let f = unacked[i]
unacked.removeSubrange(0...i) // TCP delivers in order: earlier ones arrived too
rtts.append((now, now - f.sent))
acked.append((now, f.bytes))
return true
}
}
/// Called every 100 ms. Returns the new bitrate target, or nil if the
/// controller is off.
func update(now: Int64) -> Int? {
locked {
rtts.removeAll { now - $0.t > 10_000_000 }
acked.removeAll { now - $0.t > 500_000 }
sentLog.removeAll { now - $0.t > 500_000 }
unacked.removeAll { now - $0.sent > 2_000_000 }
captures.removeAll { now - $0 > 1_000_000 }
guard RateController.enabled, ceiling > 0 else { return nil }
let base = baseline()
let spread = rtts.filter { now - $0.t < 2_000_000 }.map(\.v).sorted()
let jitter = spread.isEmpty ? 0 : spread[spread.count * 9 / 10] - base
let interval = Int64(1_000_000 / max(1, fpsLocked))
slack = interval + min(max(jitter * 3 / 2, 25_000), 80_000)
let recent = rtts.filter { now - $0.t < 300_000 }.map(\.v).sorted()
let queueing = recent.isEmpty ? 0 : recent[recent.count / 2] - base
let oldestAge = unacked.first.map { now - $0.sent } ?? 0
let stuck = oldestAge > base + 100_000
let delivered = acked.reduce(0) { $0 + $1.bytes } * 16 // bits/s over the last half second
let sending = sentLog.reduce(0) { $0 + $1.bytes } * 16
// Demand: captures per second (up to the tier's rate) times the
// average frame. A test card or a mostly still window wants far
// less than its budget; when the link hiccups, cutting its bitrate
// can't help, and it would only look link-limited afterwards.
if !sentLog.isEmpty { frameBytes = sentLog.reduce(0) { $0 + $1.bytes } / sentLog.count }
let demand = min(captures.count, fpsLocked) * frameBytes * 8
// Delay alone isn't our queue: Wi-Fi jitters by itself. It only
// counts while this stream uses a good part of its budget (so its
// own data could be what's queueing). Frames the gate had to hold,
// or one stuck in flight, show demand the link isn't carrying
// whatever was sent (the gate itself keeps what's sent low).
let busy = sending >= target / 2
// Twice in a row (200 ms), so one late ack doesn't count.
let signal = (busy && queueing > 40_000) || held >= 3 || stuck
let congested = signal && lastSignal
lastSignal = signal
let heldNow = held
held = 0
let floorBps = 300_000
if congested, now - lastDecrease > 300_000 {
// Down to a bit under what got through: at least a fifth off, at
// most half (a stall delivers nothing, but the link is still there).
let measured = Int(Double(delivered) * 0.9)
var next = max(floorBps, min(target * 4 / 5, max(measured, target / 2)))
// App-limited (it wants about half its budget or less): never
// below twice what it wants, however many cuts in a row. The
// extra quarter is hysteresis, so frame sizes wobbling at the
// floor don't switch the protection off.
if demand > 0, demand * 2 <= target * 5 / 4 { next = max(next, min(target, demand * 2)) }
target = next
lastDecrease = now
events.append((now, "down to \(target / 1000) kbit/s: queue \(queueing / 1000) ms, held \(heldNow), "
+ "oldest \(oldestAge / 1000) ms, base \(base / 1000) ms, sent \(sending / 1000) got \(delivered / 1000) "
+ "wants \(demand / 1000)"))
} else if !congested, now - lastDecrease > 1_000_000, now - lastIncrease > 250_000, target < ceiling,
sending > target * 6 / 10 || now - lastDecrease > 3_000_000 {
// Clear for a second and using what it has: probe up.
target = min(ceiling, Int(Double(target) * 1.1) + 50_000)
lastIncrease = now
}
// Fewer frames or pixels only help a stream that fills its budget;
// a small one (a still window, a test card) keeps its tier.
retier(now: now, linkLimited: sending >= target * 7 / 10)
if events.count > 200 { events.removeFirst(events.count - 200) }
return target
}
}
/// Steps down quickly, straight to the tier the bitrate supports, and back
/// up one tier at a time only when there's clearly room (hysteresis).
private func retier(now: Int64, linkLimited: Bool) {
let share = Double(target) / Double(max(ceiling, 1))
if tier < RateController.tiers.count - 1, share < RateController.floors[tier], linkLimited {
if belowSince == 0 { belowSince = now }
if now - belowSince > 500_000 {
tier = RateController.floors.firstIndex { share >= $0 } ?? RateController.tiers.count - 1
belowSince = 0
events.append((now, "tier \(tier)"))
}
} else {
belowSince = 0
}
if tier > 0, share > RateController.floors[tier - 1] * 1.25 {
if aboveSince == 0 { aboveSince = now }
if now - aboveSince > 2_000_000 {
tier -= 1
aboveSince = 0
events.append((now, "tier \(tier)"))
}
} else {
aboveSince = 0
}
}
func state() -> [String: Any] {
locked {
["target": target, "ceiling": ceiling, "tier": tier, "fps": min(maxFps, RateController.tiers[tier].fps),
"scale": RateController.tiers[tier].scale, "baseRtt": Double(baseline()) / 1000,
"inFlight": unacked.count, "slack": Double(slack) / 1000, "adapt": RateController.enabled]
}
}
func eventList() -> [[String: Any]] { locked { events.map { ["t": $0.0, "e": $0.1] } } }
}
mac/frame-mac-view/Sources/Encoder.swift
+59 -14
View File
@@ -18,11 +18,21 @@ final class Encoder {
private var session: VTCompressionSession?
private var forceKey = true
private var lastPts = CMTime.invalid
/// Called on VideoToolbox's thread with one access unit (or JPEG) per frame.
var onFrame: ((Data, Bool, CMTime) -> Void)?
private var lastGiven = CMTime.invalid
/// Bits per second to aim for; nil means from the size, frame rate and
/// bits per pixel. Changing it takes effect on the next frame.
private var target: Int?
private(set) var bitrate = 0
/// Called on VideoToolbox's thread with one access unit (or JPEG) per
/// frame, and the sequence number it was submitted with.
var onFrame: ((Data, Bool, CMTime, UInt32) -> Void)?
var onError: ((String) -> Void)?
/// Called once per frame handed to VideoToolbox, however it went.
var onDone: (() -> Void)?
var onDone: ((UInt32) -> Void)?
/// Experiment switches (FRAME_MAC_VIEW_ENCODER, comma-separated), so a
/// benchmark can compare them without a rebuild.
static let options = Set((ProcessInfo.processInfo.environment["FRAME_MAC_VIEW_ENCODER"] ?? "")
.split(separator: ",").map(String.init))
init(codec: Codec, fps: Int, bitsPerPixel: Double) {
self.codec = codec
@@ -30,6 +40,27 @@ final class Encoder {
self.bitsPerPixel = bitsPerPixel
}
/// The bitrate the size and quality setting would give.
func defaultBitrate(width w: Int, height h: Int) -> Int {
Int(min(max(Double(w * h * fps) * bitsPerPixel, 2_000_000), 60_000_000))
}
/// Live, without a new keyframe. Call on the encoding queue.
func setBitrate(_ bps: Int?) {
target = bps
guard codec == .h264, let s = session else { return }
let b = bps ?? defaultBitrate(width: width, height: height)
guard b != bitrate else { return }
bitrate = b
VTSessionSetProperty(s, key: kVTCompressionPropertyKey_AverageBitRate, value: b as CFTypeRef)
// A hard ceiling too: no more than 200 ms' worth of bits in any 200 ms,
// so a keyframe can't hold the link for long.
if Encoder.options.contains("cap") {
VTSessionSetProperty(s, key: kVTCompressionPropertyKey_DataRateLimits,
value: [b / 8 / 5, 0.2] as CFArray)
}
}
deinit { invalidate() }
func requestKeyFrame() { forceKey = true }
@@ -46,6 +77,7 @@ final class Encoder {
invalidate()
var spec: [CFString: Any] = [:]
if codec == .h264 { spec[kVTVideoEncoderSpecification_EnableLowLatencyRateControl] = true }
if Encoder.options.contains("hw") { spec[kVTVideoEncoderSpecification_RequireHardwareAcceleratedVideoEncoder] = true }
var s: VTCompressionSession?
let type = codec == .h264 ? kCMVideoCodecType_H264 : kCMVideoCodecType_JPEG
func create(_ spec: [CFString: Any]) -> OSStatus {
@@ -68,11 +100,11 @@ final class Encoder {
set(kVTCompressionPropertyKey_TransferFunction, kCVImageBufferTransferFunction_ITU_R_709_2)
set(kVTCompressionPropertyKey_YCbCrMatrix, kCVImageBufferYCbCrMatrix_ITU_R_709_2)
if codec == .h264 {
let bps = min(max(Double(w * h * fps) * bitsPerPixel, 2_000_000), 60_000_000)
set(kVTCompressionPropertyKey_ProfileLevel, kVTProfileLevel_H264_ConstrainedHigh_AutoLevel)
set(kVTCompressionPropertyKey_AllowFrameReordering, false)
set(kVTCompressionPropertyKey_AverageBitRate, Int(bps))
set(kVTCompressionPropertyKey_ExpectedFrameRate, fps)
if Encoder.options.contains("nodelay") { set(kVTCompressionPropertyKey_MaxFrameDelayCount, 0) }
if Encoder.options.contains("speed") { set(kVTCompressionPropertyKey_PrioritizeEncodingSpeedOverQuality, true) }
// A keyframe every 10 s at most, so a viewer that lost one recovers
// even if it never asks. Viewers ask for one when they start.
set(kVTCompressionPropertyKey_MaxKeyFrameIntervalDuration, 10)
@@ -82,24 +114,37 @@ final class Encoder {
VTCompressionSessionPrepareToEncodeFrames(s)
session = s
lastPts = .invalid
lastGiven = .invalid
width = w
height = h
forceKey = true
bitrate = 0
setBitrate(target)
return true
}
/// False if the frame never reached VideoToolbox (then onDone won't come).
@discardableResult
func encode(_ pb: CVPixelBuffer, pts given: CMTime) -> Bool {
// VideoToolbox needs strictly increasing timestamps; a resent picture
// stamped "now" can be followed by a capture stamped a moment earlier.
var pts = given
if lastPts.isValid, CMTimeCompare(pts, lastPts) <= 0 { pts = CMTimeAdd(lastPts, CMTime(value: 1, timescale: 1_000_000)) }
func encode(_ pb: CVPixelBuffer, pts given: CMTime, seq: UInt32) -> Bool {
// The encoder's own timeline: strictly increasing (a resent picture
// stamped "now" can be followed by a capture stamped a moment earlier),
// and never more than two frames on from the last one. Rate control
// budgets bits by elapsed time, so after a pause (the link held frames
// back, or nothing changed) one frame would otherwise get a quarter
// second's worth of bits: 200 KB that then hold a slow link for half a second.
let w = CVPixelBufferGetWidth(pb), h = CVPixelBufferGetHeight(pb)
if session == nil || w != width || h != height {
guard makeSession(width: w, height: h) else { return false }
guard makeSession(width: w, height: h) else { return false } // a new timeline too
}
guard let s = session else { return false }
var pts = given
if lastPts.isValid, lastGiven.isValid {
let gap = CMTimeSubtract(given, lastGiven)
let most = CMTime(value: 2, timescale: CMTimeScale(fps))
let step = CMTimeCompare(gap, most) > 0 ? most : gap
pts = CMTimeAdd(lastPts, CMTimeMaximum(step, CMTime(value: 1, timescale: 1_000_000)))
}
lastGiven = given
var props: CFDictionary?
if forceKey {
props = [kVTEncodeFrameOptionKey_ForceKeyFrame: true] as CFDictionary
@@ -110,12 +155,12 @@ final class Encoder {
let status = VTCompressionSessionEncodeFrame(s, imageBuffer: pb, presentationTimeStamp: pts, duration: .invalid,
frameProperties: props, infoFlagsOut: nil) { [weak self] status, _, sample in
guard let self else { return }
defer { self.onDone?() }
defer { self.onDone?(seq) }
guard status == noErr, let sample else { return }
if codec == .jpeg {
if let data = Self.bytes(sample) { self.onFrame?(data, true, pts) }
if let data = Self.bytes(sample) { self.onFrame?(data, true, pts, seq) }
} else if let (data, key) = Self.annexB(sample) {
self.onFrame?(data, key, pts)
self.onFrame?(data, key, pts, seq)
}
}
if status != noErr { forceKey = true }
mac/frame-mac-view/Sources/Separate.swift
+295
View File
@@ -0,0 +1,295 @@
// "Separate" mode: a streamed window gets a virtual display of its own. The
// window is moved onto it and sized to fill it, and the whole display is
// captured, so its menus, sheets, popovers and tooltips come along, nothing
// can cover it, and clicks land on it without raising anything. On stop the
// window goes back where it was. See docs/mac-window-separation.md.
import AppKit
import ApplicationServices
import CoreMedia
import Foundation
@_silgen_name("_AXUIElementGetWindow")
private func axWindowID(_ element: AXUIElement, _ id: UnsafeMutablePointer<CGWindowID>) -> AXError
/// One of our virtual displays, HiDPI (2 pixels per point). Main thread only.
final class VirtualDisplay {
static let vendor: UInt32 = 0xF0C0
/// How often macOS composites our displays (FRAME_MAC_VIEW_VD_HZ to experiment).
static let refreshRate = Double(ProcessInfo.processInfo.environment["FRAME_MAC_VIEW_VD_HZ"] ?? "") ?? 60
private var display: CGVirtualDisplay?
let id: CGDirectDisplayID
/// Removal is deferred while the Mac's screen sleeps, and an identity
/// can't be reused until it's gone, so each display gets a fresh serial.
private static var serial = (UInt32(truncatingIfNeeded: ProcessInfo.processInfo.processIdentifier) & 0xFFFF) << 12
init?(name: String, width: Int, height: Int) {
var made: CGVirtualDisplay?
for _ in 0..<8 where made == nil {
VirtualDisplay.serial &+= 1
let d = CGVirtualDisplayDescriptor()
d.queue = DispatchQueue.main
d.name = name
d.maxPixelsWide = UInt32(2 * width)
d.maxPixelsHigh = UInt32(2 * height)
d.sizeInMillimeters = CGSize(width: Double(width) * 0.2646, height: Double(height) * 0.2646)
d.vendorID = VirtualDisplay.vendor
d.productID = 0x5E9A
d.serialNum = VirtualDisplay.serial
d.terminationHandler = { _, _ in }
guard let vd = CGVirtualDisplay(descriptor: d) else { continue }
let s = CGVirtualDisplaySettings()
s.hiDPI = 1
let hz = VirtualDisplay.refreshRate
s.modes = [CGVirtualDisplayMode(width: UInt32(2 * width), height: UInt32(2 * height), refreshRate: hz)!,
CGVirtualDisplayMode(width: UInt32(width), height: UInt32(height), refreshRate: hz)!]
if vd.apply(s), vd.displayID != 0 { made = vd }
}
guard let made else { return nil }
display = made
id = made.displayID
// Pick the HiDPI mode: `width` points drawn with twice the pixels.
let modes = (CGDisplayCopyAllDisplayModes(id, [kCGDisplayShowDuplicateLowResolutionModes: true] as CFDictionary)
as? [CGDisplayMode]) ?? []
if let hidpi = modes.first(where: {
$0.width == width && $0.height == height && $0.pixelWidth == 2 * width && $0.refreshRate == VirtualDisplay.refreshRate
}) ?? modes.first(where: { $0.width == width && $0.height == height && $0.pixelWidth == 2 * width }) {
var cfg: CGDisplayConfigRef?
CGBeginDisplayConfiguration(&cfg)
CGConfigureDisplayWithDisplayMode(cfg, id, hidpi, nil)
CGCompleteDisplayConfiguration(cfg, .forSession)
}
}
var bounds: CGRect { CGDisplayBounds(id) }
var isHiDPI: Bool { CGDisplayCopyDisplayMode(id).map { $0.pixelWidth > $0.width } ?? false }
var screen: NSScreen? {
NSScreen.screens.first {
($0.deviceDescription[NSDeviceDescriptionKey("NSScreenNumber")] as? NSNumber)?.uint32Value == id
}
}
/// Where a window may go, in global (top-left origin) points: the display
/// minus its menu bar and anything else macOS reserves there.
var usableRect: CGRect {
let b = bounds
guard let s = screen else { return b }
let top = s.frame.maxY - s.visibleFrame.maxY, bottom = s.visibleFrame.minY - s.frame.minY
let left = s.visibleFrame.minX - s.frame.minX, right = s.frame.maxX - s.visibleFrame.maxX
return CGRect(x: b.minX + left, y: b.minY + top, width: b.width - left - right, height: b.height - top - bottom)
}
func release() { display = nil }
static func isOurs(_ id: CGDirectDisplayID) -> Bool { CGDisplayVendorNumber(id) == vendor }
}
/// A window through the Accessibility API.
struct AXWindow {
let element: AXUIElement
init?(windowID: CGWindowID, pid: pid_t) {
let app = AXUIElementCreateApplication(pid)
var value: CFTypeRef?
guard AXUIElementCopyAttributeValue(app, kAXWindowsAttribute as CFString, &value) == .success,
let windows = value as? [AXUIElement] else { return nil }
guard let match = windows.first(where: { w in
var id: CGWindowID = 0
return axWindowID(w, &id) == .success && id == windowID
}) else { return nil }
element = match
}
var frame: CGRect? {
var p: CFTypeRef?, s: CFTypeRef?
guard AXUIElementCopyAttributeValue(element, kAXPositionAttribute as CFString, &p) == .success,
AXUIElementCopyAttributeValue(element, kAXSizeAttribute as CFString, &s) == .success else { return nil }
var point = CGPoint.zero, size = CGSize.zero
AXValueGetValue(p as! AXValue, .cgPoint, &point)
AXValueGetValue(s as! AXValue, .cgSize, &size)
return CGRect(origin: point, size: size)
}
var isFullScreen: Bool {
var v: CFTypeRef?
return AXUIElementCopyAttributeValue(element, "AXFullScreen" as CFString, &v) == .success && (v as? Bool) == true
}
/// Moving to another display: position first (so the size fits there),
/// then size, then position again in case the size change moved it.
func setFrame(_ r: CGRect) {
var origin = r.origin, size = r.size
let pos = AXValueCreate(.cgPoint, &origin)!, sz = AXValueCreate(.cgSize, &size)!
AXUIElementSetAttributeValue(element, kAXPositionAttribute as CFString, pos)
AXUIElementSetAttributeValue(element, kAXSizeAttribute as CFString, sz)
AXUIElementSetAttributeValue(element, kAXPositionAttribute as CFString, pos)
}
}
/// A window on a display of its own, captured as that display.
final class SeparateSource: CaptureSource {
let windowID: CGWindowID
var onFrame: ((CVPixelBuffer, CMTime, Int64) -> Void)?
var onEnded: ((String) -> Void)?
var onChange: (() -> Void)?
private(set) var title = ""
private(set) var app = ""
private(set) var pid: pid_t?
private(set) var gone = false
private var display: VirtualDisplay?
private var inner: SCKSource?
private var window: AXWindow?
private var original: CGRect?
private var crop: CGRect? // display points, when only part of the display is captured
private var poll: DispatchSourceTimer?
private var stopped = false
private let queue = DispatchQueue(label: "frame-mac-view.separate")
init(windowID: CGWindowID) { self.windowID = windowID }
var frameRect: CGRect { inner?.frameRect ?? .zero }
var displayID: CGDirectDisplayID? { display?.id }
func start(maxLong: Int, fps: Int, completion: @escaping (String?) -> Void) {
guard AXIsProcessTrusted() else {
return completion("Separate windows need the Accessibility permission (to move the window)")
}
guard let info = WindowInfo.find(windowID) else {
gone = true
return completion("that window has closed")
}
title = info.title
app = info.app
pid = info.pid
DispatchQueue.main.async { [self] in
guard !stopped else { return }
guard let w = AXWindow(windowID: windowID, pid: info.pid), let frame = w.frame else {
return completion("couldn't reach that window through Accessibility")
}
if w.isFullScreen { return completion("take the window out of full screen first") }
// A display the window's size, plus room for the menu bar, in
// points; within what a panel can show sharply.
let width = min(max(Int(frame.width.rounded()), 640), 2560) / 2 * 2
let height = min(max(Int(frame.height.rounded()) + 40, 480), 1600) / 2 * 2
guard let vd = VirtualDisplay(name: app.isEmpty ? "Frame Control" : "\(app) (Frame)", width: width, height: height) else {
return completion("macOS wouldn't create a display for this window")
}
display = vd
window = w
original = frame
// The new screen shows up in NSScreen a moment later; its menu bar
// decides where the window can go.
placeWindow(attempt: 0, maxLong: maxLong, fps: fps, completion: completion)
}
}
private func placeWindow(attempt: Int, maxLong: Int, fps: Int, completion: @escaping (String?) -> Void) {
guard !stopped, let vd = display, let w = window else { return }
// Wait for the screen to appear, and for its HiDPI mode, so the
// first frames are captured sharp.
if vd.screen == nil || !vd.isHiDPI, attempt < 30 {
DispatchQueue.main.asyncAfter(deadline: .now() + 0.1) {
self.placeWindow(attempt: attempt + 1, maxLong: maxLong, fps: fps, completion: completion)
}
return
}
let target = vd.usableRect
w.setFrame(target)
guard let now = w.frame, vd.bounds.intersects(now) else {
restore()
return completion("macOS didn't let the window move to its own display (Stage Manager can prevent this)")
}
let sck = SCKSource(.display(vd.id))
// A window that keeps its own size (Calculator, some dialogs) sits in
// the display's top-left corner: send only that corner, menu bar
// included, with room around it for menus, not a panel of wallpaper.
let b = vd.bounds
if now.width < target.width - 8 || now.height < target.height - 8 {
let w = min(b.width, max(now.maxX - b.minX, 480)), h = min(b.height, max(now.maxY - b.minY, 360))
crop = CGRect(x: 0, y: 0, width: (w / 2).rounded(.up) * 2, height: (h / 2).rounded(.up) * 2)
sck.crop = crop
}
sck.onFrame = { [weak self] pb, pts, shown in self?.onFrame?(pb, pts, shown) }
sck.onEnded = { [weak self] reason in self?.onEnded?(reason) }
inner = sck
sck.start(maxLong: maxLong, fps: fps) { [weak self] error in
// Back on main, where stop() runs too, so a viewer that left during
// startup can't leave a capture running.
DispatchQueue.main.async {
guard let self, !self.stopped else { return sck.stop() }
if let error {
self.restore()
return completion(error)
}
self.startPolling()
completion(nil)
}
}
}
/// The window can close or be retitled; ScreenCaptureKit won't say.
private func startPolling() {
let t = DispatchSource.makeTimerSource(queue: queue)
t.schedule(deadline: .now() + 1, repeating: 1)
t.setEventHandler { [weak self] in
guard let self else { return }
guard let info = WindowInfo.find(self.windowID) else {
self.gone = true
self.onEnded?("the window closed")
return
}
if !info.title.isEmpty, info.title != self.title {
self.title = info.title
self.onChange?()
}
// Displays added at the same moment can rearrange each other, and
// someone may drag the window away: put it back on its display.
if !info.bounds.isEmpty {
DispatchQueue.main.async { self.keepOnDisplay(info.bounds) }
}
}
t.resume()
poll = t
}
private func keepOnDisplay(_ now: CGRect) {
guard !stopped, let vd = display, let w = window else { return }
let b = vd.bounds
if let c = crop {
// Only part of the display is sent: the window must stay inside it.
guard !c.offsetBy(dx: b.minX, dy: b.minY).insetBy(dx: -2, dy: -2).contains(now) else { return }
} else {
guard !b.contains(CGPoint(x: now.midX, y: now.minY + 10)) else { return }
}
let target = vd.usableRect
w.setFrame(CGRect(origin: target.origin, size: now.size.width < target.width - 8 ? now.size : target.size))
}
/// Lifecycle state (stopped, inner, poll, window, display) is only touched on main.
func stop() {
DispatchQueue.main.async { [self] in
stopped = true
poll?.cancel()
poll = nil
inner?.stop()
restore()
}
}
/// Puts the window back where it was, then removes the display. In that
/// order: removing a display first would let macOS pick where it goes.
private func restore() {
if let w = window, let r = original, WindowInfo.find(windowID) != nil { w.setFrame(r) }
window = nil
original = nil
let vd = display
display = nil
// Give WindowServer a moment to move the window before the display goes.
DispatchQueue.main.asyncAfter(deadline: .now() + 0.3) { vd?.release() }
}
func setMaxLong(_ n: Int) { DispatchQueue.main.async { self.inner?.setMaxLong(n) } }
func pointer(x: Double, y: Double, text: String?) {}
}
mac/frame-mac-view/Sources/Server.swift
+4 -2
View File
@@ -165,9 +165,10 @@ final class WebSocket {
func sendJSON(_ obj: Any) {
if let d = try? JSONSerialization.data(withJSONObject: obj), let s = String(data: d, encoding: .utf8) { sendText(s) }
}
func sendBinary(_ d: Data) { send(opcode: 2, d) }
/// `taken` runs once the network stack has taken the whole frame.
func sendBinary(_ d: Data, taken: (() -> Void)? = nil) { send(opcode: 2, d, taken: taken) }
func send(opcode: UInt8, _ payload: Data) {
func send(opcode: UInt8, _ payload: Data, taken: (() -> Void)? = nil) {
var frame = Data([0x80 | opcode])
let n = payload.count
if n < 126 {
@@ -185,6 +186,7 @@ final class WebSocket {
conn.send(content: frame, completion: .contentProcessed { [weak self] _ in
guard let self else { return }
self.lock.lock(); self._pending -= size; self.lock.unlock()
taken?()
self.onDrain?()
})
}
mac/frame-mac-view/Sources/Stats.swift
+180
View File
@@ -0,0 +1,180 @@
// Per-frame timing, so every change to the stream can be judged by numbers.
// All times are the agent's host clock in microseconds (the clock
// ScreenCaptureKit stamps frames with). The viewer syncs to it over the
// WebSocket and reports when each frame arrived, decoded and was drawn.
import CoreMedia
import Foundation
private let timebase: mach_timebase_info_data_t = {
var t = mach_timebase_info_data_t()
mach_timebase_info(&t)
return t
}()
/// Now on the host clock, in microseconds.
func nowUs() -> Int64 { hostUs(mach_absolute_time()) }
/// A mach_absolute_time value (as in SCStreamFrameInfo.displayTime) in microseconds.
func hostUs(_ ticks: UInt64) -> Int64 { Int64(ticks / 1000 * UInt64(timebase.numer) / UInt64(timebase.denom)) }
/// A host-clock CMTime (ScreenCaptureKit's presentation times) in microseconds.
func hostUs(_ t: CMTime) -> Int64 { t.isValid ? Int64(CMTimeGetSeconds(t) * 1_000_000) : nowUs() }
/// One frame's journey. Zero means "didn't happen" or "not reported yet".
struct FrameRecord {
var seq: UInt32 = 0
var key = false
var bytes = 0
var width = 0, height = 0
var capture: Int64 = 0 // the Mac composited it (display time)
var arrived: Int64 = 0 // ScreenCaptureKit handed it to us
var encodeStart: Int64 = 0
var encodeEnd: Int64 = 0
var sent: Int64 = 0 // handed to the WebSocket
var wire: Int64 = 0 // the socket took it
var received: Int64 = 0 // viewer, agent clock
var decoded: Int64 = 0
var drawn: Int64 = 0
var vsync: Int64 = 0 // the viewer's next animation frame after drawing it
var echo: UInt32 = 0 // the first frame after input `echo`
var tier = 0
var bitrate = 0
var json: [String: Any] {
["s": seq, "k": key ? 1 : 0, "b": bytes, "w": width, "h": height, "cap": capture, "arr": arrived,
"e0": encodeStart, "e1": encodeEnd, "snd": sent, "wire": wire, "rx": received, "dec": decoded,
"drw": drawn, "vs": vsync, "echo": echo, "tier": tier, "br": bitrate]
}
}
/// One input event from the viewer and what came of it.
struct InputRecord {
var id: UInt32
var kind: String
var viewer: Int64 // the viewer's event time, agent clock
var injected: Int64 = 0 // posted as a CGEvent
var frame: UInt32 = 0 // first frame captured after it (0: none yet)
var capture: Int64 = 0
var json: [String: Any] {
["id": id, "kind": kind, "tv": viewer, "inj": injected, "frame": frame, "cap": capture]
}
}
func percentile(_ sorted: [Double], _ p: Double) -> Double? {
guard !sorted.isEmpty else { return nil }
let i = min(sorted.count - 1, max(0, Int((p * Double(sorted.count - 1)).rounded())))
return sorted[i]
}
/// Frames and inputs of one stream, kept for the last few thousand frames.
/// Any thread.
final class StreamStats {
static let capacity = 4096
private let lock = NSLock()
private var frames = [FrameRecord?](repeating: nil, count: StreamStats.capacity)
private var inputs: [UInt32: InputRecord] = [:]
private var inputOrder: [UInt32] = []
private(set) var nextSeq: UInt32 = 1
var captured = 0 // frames ScreenCaptureKit delivered
var skipped = 0 // held back from encoding (link, encoder, pacing or gate busy); only the newest may go later
var viewerDropped = 0 // not shown by the viewer (behind, or waiting for a keyframe)
var rtt: Double = 0 // ms, the viewer's best recent ping
var clockSynced = false
var decoder = "" // what the viewer reports about its decoder
func withLock<T>(_ f: () -> T) -> T { lock.lock(); defer { lock.unlock() }; return f() }
func newFrame(_ r: FrameRecord) -> UInt32 {
withLock {
var r = r
r.seq = nextSeq
nextSeq &+= 1
frames[Int(r.seq) % StreamStats.capacity] = r
return r.seq
}
}
func update(_ seq: UInt32, _ f: (inout FrameRecord) -> Void) {
withLock {
let i = Int(seq) % StreamStats.capacity
guard var r = frames[i], r.seq == seq else { return }
f(&r)
frames[i] = r
}
}
func frame(_ seq: UInt32) -> FrameRecord? {
withLock {
let r = frames[Int(seq) % StreamStats.capacity]
return r?.seq == seq ? r : nil
}
}
func addInput(_ r: InputRecord) {
withLock {
inputs[r.id] = r
inputOrder.append(r.id)
if inputOrder.count > 512 { inputs[inputOrder.removeFirst()] = nil }
}
}
func updateInput(_ id: UInt32, _ f: (inout InputRecord) -> Void) {
withLock { if var r = inputs[id] { f(&r); inputs[id] = r } }
}
/// Frames with seq > `since` that were sent at least `settle` µs ago, so
/// the viewer has had time to report on them.
func settled(since: UInt32, settle: Int64 = 1_500_000) -> [FrameRecord] {
let cutoff = nowUs() - settle
return withLock {
frames.compactMap { $0 }.filter { $0.seq > since && $0.sent > 0 && $0.sent < cutoff }.sorted { $0.seq < $1.seq }
}
}
func inputList() -> [InputRecord] { withLock { inputOrder.compactMap { inputs[$0] } } }
/// Percentiles over the last `window` µs, for /status and the overlay.
func summary(window: Int64 = 2_000_000) -> [String: Any] {
let now = nowUs(), from = now - window
let recent = withLock { frames.compactMap { $0 }.filter { $0.sent > from } }
func ms(_ pick: (FrameRecord) -> (Int64, Int64)) -> [String: Double] {
let v = recent.compactMap { r -> Double? in
let (a, b) = pick(r)
return a > 0 && b > 0 ? Double(b - a) / 1000 : nil
}.sorted()
guard !v.isEmpty else { return [:] }
return ["p50": (percentile(v, 0.5)! * 10).rounded() / 10, "p95": (percentile(v, 0.95)! * 10).rounded() / 10]
}
let secs = Double(window) / 1_000_000
let shown = recent.filter { $0.vsync > 0 }.count
let bytes = recent.reduce(0) { $0 + $1.bytes }
var s: [String: Any] = [
"capture": ms { ($0.capture, $0.arrived) },
"queue": ms { ($0.arrived, $0.encodeStart) },
"encode": ms { ($0.encodeStart, $0.encodeEnd) },
"network": ms { ($0.encodeEnd, $0.received) },
"decode": ms { ($0.received, $0.decoded) },
"draw": ms { ($0.decoded, $0.vsync) },
"total": ms { ($0.capture, $0.vsync) },
"fps": (Double(shown) / secs * 10).rounded() / 10,
"sentFps": (Double(recent.count) / secs * 10).rounded() / 10,
"mbps": (Double(bytes * 8) / secs / 1e5).rounded() / 10,
"rtt": (rtt * 10).rounded() / 10,
"synced": clockSynced,
]
withLock {
s["captured"] = captured
s["skipped"] = skipped
s["dropped"] = viewerDropped
s["decoder"] = decoder
let done = inputOrder.suffix(20).compactMap { inputs[$0] }.compactMap { i -> Double? in
guard i.frame != 0, let f = frames[Int(i.frame) % StreamStats.capacity], f.seq == i.frame, f.vsync > 0
else { return nil }
return Double(f.vsync - i.viewer) / 1000
}.sorted()
if !done.isEmpty { s["input"] = ["p50": percentile(done, 0.5)!, "n": done.count] }
}
return s
}
}
mac/frame-mac-view/Sources/main.swift
+293 -28
View File
@@ -16,6 +16,8 @@
// POST /ticket?src=... ?k=: a ticket for one viewer of src
// POST /close[?src=...] ?k=: end those streams, close their windows
// POST /permissions ?k=: show macOS's permission prompts
// GET /stats[?id=&since=] ?k=: per-frame timing records (for benchmarks)
// POST /bench?src=&action=... ?k=: ask viewers of src to type, click or show their overlay
// src is window:<CGWindowID>, display:<CGDirectDisplayID> or test.
import AppKit
import ApplicationServices
@@ -43,7 +45,9 @@ func displaysJSON() -> [[String: Any]] {
var n: UInt32 = 0
// Online, not active: a display that's asleep is still one you can stream.
CGGetOnlineDisplayList(16, &ids, &n)
return ids.prefix(Int(n)).filter { CGDisplayMirrorsDisplay($0) == kCGNullDirectDisplay }.map { id in
return ids.prefix(Int(n)).filter {
CGDisplayMirrorsDisplay($0) == kCGNullDirectDisplay && !VirtualDisplay.isOurs($0) // not a separated window's
}.map { id in
let b = CGDisplayBounds(id)
return ["id": id, "src": "display:\(id)", "name": displayName(id), "w": Int(b.width), "h": Int(b.height),
"main": CGDisplayIsMain(id) != 0]
@@ -59,6 +63,9 @@ func printJSON(_ obj: Any) {
FileHandle.standardOutput.write(d + Data("\n".utf8))
}
/// A number from a JSON message, whatever its JSON type.
func num(_ v: Any?) -> Double? { (v as? NSNumber)?.doubleValue }
/// One viewer watching one source.
final class Session {
let id: Int
@@ -68,11 +75,28 @@ final class Session {
let ws: WebSocket
let codec: Codec
let reconnectKey: String
let stats = StreamStats()
let controller: RateController
private var appliedScale = 1.0
private var lastSubmit: Int64 = 0
private var paceScheduled = false
private let lock = NSLock()
private var last: (CVPixelBuffer, CMTime)?
/// A captured picture waiting to be encoded (or the last one encoded).
private struct Picture {
let pb: CVPixelBuffer
let pts: CMTime
let capture: Int64
let arrived: Int64
var echo: UInt32 // the input this picture is the first reply to
}
private var last: Picture?
private var skipped = false
private var stopped = false
private var inFlight = 0
/// Input posted to the Mac whose effect hasn't been captured yet: the
/// next picture composited after `after` is tagged with it.
private var pendingEcho: (id: UInt32, after: Int64)?
private var statsTimer: DispatchSourceTimer?
/// Frames already queued for the network; beyond this, or with two frames
/// already in the encoder, new frames are skipped (before encoding, so no
/// reference frame goes missing) and the newest picture is sent once
@@ -93,23 +117,63 @@ final class Session {
self.codec = codec
self.reconnectKey = reconnectKey
encoder = Encoder(codec: codec, fps: fps, bitsPerPixel: bitsPerPixel)
controller = RateController(maxFps: fps)
maxPending = codec == .jpeg ? 3 << 20 : 1 << 20
}
/// Frames come faster than the current tier's frame rate: this one waits,
/// and goes when its turn comes (unless a newer one replaces it).
/// Call with `lock` held.
private func paced(now: Int64) -> Bool {
let fps = controller.fps
guard fps < controller.maxFps, lastSubmit > 0 else { return false }
let interval = Int64(1_000_000 / fps), due = lastSubmit + interval * 9 / 10
guard now < due else { return false }
if !paceScheduled {
paceScheduled = true
encodeQueue.asyncAfter(deadline: .now() + .microseconds(Int(due - now))) { [weak self] in
guard let self else { return }
self.lock.lock(); self.paceScheduled = false; self.lock.unlock()
self.resendIfRoom()
}
}
return true
}
/// Encodes `pb` unless the link or the encoder is busy, in which case it
/// becomes the picture to send next.
private func offer(_ pb: CVPixelBuffer, pts: CMTime) {
private func offer(_ pb: CVPixelBuffer, pts: CMTime, capture shown: Int64) {
let arrived = nowUs()
stats.withLock { stats.captured += 1 }
controller.captured(at: arrived)
lock.lock()
last = (pb, pts)
// A reply to input stays with whichever picture ends up being sent.
var echo = skipped ? last?.echo ?? 0 : 0
if let p = pendingEcho, shown >= p.after {
echo = p.id
pendingEcho = nil
}
let picture = Picture(pb: pb, pts: pts, capture: shown, arrived: arrived, echo: echo)
last = picture
let busy = stopped || ws.pendingBytes > maxPending || inFlight >= Session.maxInFlight
if busy { skipped = true } else { inFlight += 1 }
|| paced(now: arrived) || !controller.maySend(now: arrived)
if busy { skipped = true } else { inFlight += 1; last?.echo = 0; lastSubmit = arrived }
lock.unlock()
if !busy { submit(pb, pts: pts) }
if busy { stats.withLock { stats.skipped += 1 } } else { submit(picture) }
}
private func submit(_ pb: CVPixelBuffer, pts: CMTime) {
private func submit(_ p: Picture) {
encodeQueue.async {
if !self.encoder.encode(pb, pts: pts) { self.finished() }
var r = FrameRecord()
r.capture = p.capture
r.arrived = p.arrived
r.echo = p.echo
r.bitrate = self.encoder.bitrate
r.tier = self.controller.tier
r.encodeStart = nowUs()
let seq = self.stats.newFrame(r)
if p.echo != 0 { self.stats.updateInput(p.echo) { $0.frame = seq; $0.capture = p.capture } }
if !self.encoder.encode(p.pb, pts: p.pts, seq: seq) { self.finished() }
}
}
@@ -124,28 +188,61 @@ final class Session {
private func resendIfRoom() {
lock.lock()
var next: (CVPixelBuffer, CMTime)?
if !stopped, skipped, ws.pendingBytes <= maxPending / 2, inFlight < Session.maxInFlight, let l = last {
var next: Picture?
let now = nowUs()
if !stopped, skipped, ws.pendingBytes <= maxPending / 2, inFlight < Session.maxInFlight, let l = last,
!paced(now: now), controller.maySend(now: now, counts: false) {
next = l
skipped = false
inFlight += 1
last?.echo = 0
lastSubmit = now
}
lock.unlock()
if let (pb, _) = next { submit(pb, pts: CMClockGetTime(CMClockGetHostTimeClock())) }
// Stamped now: VideoToolbox needs increasing times, and the capture
// time stays in the record, so the wait counts as latency.
if let p = next {
submit(Picture(pb: p.pb, pts: CMClockGetTime(CMClockGetHostTimeClock()), capture: p.capture,
arrived: p.arrived, echo: p.echo))
}
}
func start(maxLong: Int, fps: Int) {
encoder.onFrame = { [weak self] data, key, pts in
encoder.onFrame = { [weak self] data, key, pts, seq in
guard let self else { return }
var msg = Data([key ? 1 : 0])
let t = nowUs()
// The quality setting's bitrate, at full size, is the most it gets.
if self.appliedScale == 1 {
self.controller.setCeiling(self.encoder.defaultBitrate(width: self.encoder.width, height: self.encoder.height))
}
self.controller.sent(seq: seq, bytes: data.count + 17, at: t)
var echo: UInt32 = 0
let (w, h) = (self.encoder.width, self.encoder.height)
self.stats.update(seq) {
$0.encodeEnd = t
$0.sent = t
$0.bytes = data.count
$0.key = key
$0.width = w
$0.height = h
echo = $0.echo
}
// Header: flags (1 = keyframe), pts µs, sequence number, and the
// input this frame is the first reply to; all big-endian.
var msg = Data(capacity: data.count + 17)
msg.append(key ? 1 : 0)
var us = UInt64(max(0, CMTimeGetSeconds(pts)) * 1_000_000).bigEndian
msg.append(Data(bytes: &us, count: 8))
var s = seq.bigEndian, e = echo.bigEndian
msg.append(Data(bytes: &s, count: 4))
msg.append(Data(bytes: &e, count: 4))
msg.append(data)
self.ws.sendBinary(msg)
let stats = self.stats
self.ws.sendBinary(msg) { stats.update(seq) { $0.wire = nowUs() } }
}
encoder.onDone = { [weak self] in self?.finished() }
encoder.onDone = { [weak self] _ in self?.finished() }
encoder.onError = { [weak self] message in self?.ws.sendJSON(["t": "error", "message": message]) }
capture.onFrame = { [weak self] pb, pts in self?.offer(pb, pts: pts) }
capture.onFrame = { [weak self] pb, pts, shown in self?.offer(pb, pts: pts, capture: shown) }
capture.onEnded = { [weak self] reason in
guard let self else { return }
self.ws.sendJSON(["t": "closed", "reason": reason])
@@ -158,13 +255,30 @@ final class Session {
ws.start()
// Reconnect with this (kept in the page's memory, never on a command line).
ws.sendJSON(["t": "hello", "r": reconnectKey])
if let sck = capture as? SCKSource {
sck.onChange = { [weak self] in self?.sendInfo() }
capture.onChange = { [weak self] in self?.sendInfo() }
// The numbers, for the viewer's overlay.
// Adapt ten times a second; the numbers go to the viewer once a second.
let t = DispatchSource.makeTimerSource(queue: encodeQueue)
t.schedule(deadline: .now() + 0.1, repeating: 0.1)
var ticks = 0
t.setEventHandler { [weak self] in
guard let self, !self.isStopped else { return }
self.adapt(maxLong: maxLong)
ticks += 1
guard ticks % 10 == 0 else { return }
var s = self.stats.summary()
s.merge(self.controller.state()) { a, _ in a }
s["t"] = "stats"
s["bitrate"] = self.encoder.bitrate
s["size"] = "\(self.encoder.width)×\(self.encoder.height)"
self.ws.sendJSON(s)
}
t.resume()
statsTimer = t
capture.start(maxLong: maxLong, fps: fps) { [weak self] error in
guard let self else { return }
if let error {
if (self.capture as? SCKSource)?.gone == true {
if self.capture.gone {
// Final, like a window closing mid-stream: the viewer closes
// and its key is revoked, rather than retrying for ever.
self.ws.sendJSON(["t": "closed", "reason": error])
@@ -179,9 +293,26 @@ final class Session {
}
}
/// Applies the controller's bitrate and tier. On the encoding queue.
private func adapt(maxLong: Int) {
guard let bps = controller.update(now: nowUs()) else { return }
encoder.setBitrate(bps)
let scale = controller.scale
if scale != appliedScale {
appliedScale = scale
capture.setMaxLong(max(320, Int(Double(maxLong) * scale)))
}
resendIfRoom() // a lower tier may let a held frame go now
}
/// While someone is typing or clicking, the viewer sends a tiny message this
/// often (ms, 0 = off), so the Frame's Wi-Fi doesn't doze between the input
/// and the frame that answers it (power saving is on there).
static let warmMs = Int(ProcessInfo.processInfo.environment["FRAME_MAC_VIEW_WARM"] ?? "") ?? 0
func sendInfo() {
ws.sendJSON(["t": "info", "src": source.key, "title": capture.title, "app": capture.app, "codec": codec.rawValue,
"input": source == .test || Input.allowed,
"input": source == .test || Input.allowed, "warm": Session.warmMs,
"aspect": Double(capture.frameRect.width / max(capture.frameRect.height, 1))])
}
@@ -194,6 +325,7 @@ final class Session {
last = nil
lock.unlock()
guard !was else { return }
statsTimer?.cancel()
capture.stop()
encodeQueue.async { self.encoder.invalidate() }
let owner = id
@@ -208,9 +340,55 @@ final class Session {
return CGPoint(x: r.minX + min(max(x, 0), 1) * r.width, y: r.minY + min(max(y, 0), 1) * r.height)
}
/// Input with an id (from the viewer) has been posted: the next picture
/// the Mac composites is its first chance to show.
private func injected(_ m: [String: Any], kind: String) {
guard let iid = (m["i"] as? NSNumber)?.uint32Value, iid != 0 else { return }
let now = nowUs()
stats.addInput(InputRecord(id: iid, kind: kind, viewer: Int64(num(m["tv"]) ?? 0), injected: now))
lock.lock(); pendingEcho = (iid, now); lock.unlock()
}
/// Timing reports from the viewer, in the agent's clock.
private func report(_ t: String, _ m: [String: Any]) -> Bool {
switch t {
case "ping":
ws.sendJSON(["t": "pong", "c": m["c"] ?? 0, "a": nowUs()])
case "rx":
guard let s = (m["s"] as? NSNumber)?.uint32Value else { break }
if let r = num(m["r"]) { stats.update(s) { $0.received = Int64(r) } }
if controller.acked(seq: s, at: nowUs()) { resendIfRoom() }
case "fd":
for f in m["f"] as? [[NSNumber]] ?? [] where f.count >= 4 {
stats.update(f[0].uint32Value) {
$0.decoded = f[1].int64Value
$0.drawn = f[2].int64Value
$0.vsync = f[3].int64Value
}
}
let dropped = Int(num(m["drop"]) ?? 0)
stats.withLock { stats.viewerDropped += dropped }
case "w":
break // keep-warm filler, see sendInfo
case "clock":
stats.withLock {
stats.rtt = num(m["rtt"]) ?? 0
stats.clockSynced = true
if let d = m["dec"] as? String { stats.decoder = d }
}
default:
return false
}
return true
}
/// Asks the viewer to do something for a benchmark (type, click, show its overlay).
func bench(_ m: [String: Any]) { ws.sendJSON(m.merging(["t": "bench"]) { a, _ in a }) }
private func handle(_ text: String) {
guard !isStopped, let d = text.data(using: .utf8),
let m = try? JSONSerialization.jsonObject(with: d) as? [String: Any], let t = m["t"] as? String else { return }
if report(t, m) { return }
let x = m["x"] as? Double ?? -1, y = m["y"] as? Double ?? -1
if t == "ack" {
onAck?()
@@ -233,6 +411,7 @@ final class Session {
default: desc = nil
}
capture.pointer(x: x, y: y, text: desc)
if desc != nil { injected(m, kind: t) }
return
}
DispatchQueue.main.async { [self] in
@@ -245,13 +424,17 @@ final class Session {
Input.focus(window: wid, pid: pid)
}
Input.mouse(kind, button: b, at: p, owner: id)
if kind == "down" { injected(m, kind: "click") }
case "wheel":
Input.scroll(dx: m["dx"] as? Double ?? 0, dy: m["dy"] as? Double ?? 0, at: point(x, y), owner: id)
injected(m, kind: "wheel")
case "k":
let down = (m["e"] as? String) == "down"
Input.key(code: m["code"] as? String ?? "", key: m["key"] as? String ?? "",
down: (m["e"] as? String) == "down", mods: m["mods"] as? [String] ?? [], owner: id)
down: down, mods: m["mods"] as? [String] ?? [], owner: id)
if down { injected(m, kind: "key") }
case "text":
if let s = m["s"] as? String, s.count <= 4096 { Input.text(s) }
if let s = m["s"] as? String, s.count <= 4096 { Input.text(s); injected(m, kind: "text") }
case "release":
Input.releaseAll(owner: id)
case "focus":
@@ -292,6 +475,21 @@ final class Agent {
/// Sources that ended for good (the window closed), for Frame Control.
private var finished = Set<String>()
/// Ends every stream, then exits once separated windows are back on the
/// Mac's own screens (they're moved back, then their displays go 0.3 s later).
/// Idempotent: SIGTERM and the parent going away can both call it, and a
/// session already ended by /close may still be putting its window back.
private var quitting = false
func quit() {
lock.lock()
let all = Array(sessions.values), again = quitting
quitting = true
lock.unlock()
guard !again else { return }
for s in all { s.ws.sendJSON(["t": "close"]); s.end() }
DispatchQueue.main.asyncAfter(deadline: .now() + 0.8) { exit(0) }
}
/// While anyone watches, keep the Mac's display on: a sleeping display
/// stops being drawn, so there'd be nothing to capture (and it would lock).
private func keepDisplayAwake(_ on: Bool) {
@@ -355,7 +553,13 @@ final class Agent {
switch (req.method, req.path) {
case ("GET", "/status"):
lock.lock()
let list = sessions.values.map { ["id": $0.id, "src": $0.source.key, "title": $0.capture.title, "app": $0.capture.app] }
let list = sessions.values.map { s -> [String: Any] in
var e: [String: Any] = ["id": s.id, "src": s.source.key, "title": s.capture.title, "app": s.capture.app,
"stats": s.stats.summary(), "bitrate": s.encoder.bitrate,
"controller": s.controller.state()]
if let d = (s.capture as? SeparateSource)?.displayID { e["display"] = d }
return e
}
lock.unlock()
var s = permissionsJSON()
s["version"] = version
@@ -386,6 +590,33 @@ final class Agent {
for s in matching { s.ws.sendJSON(["t": "close"]) }
DispatchQueue.global().asyncAfter(deadline: .now() + 0.3) { for s in matching { s.end() } }
c.respond(json: ["closed": matching.count])
case ("GET", "/stats"):
// Frames the viewer has had time to report on, oldest first.
let since = UInt32(req.query["since"] ?? "") ?? 0
let settle = Int64(req.query["settle"] ?? "") ?? 1_500_000
lock.lock()
let list = sessions.values.filter { req.query["id"] == nil || "\($0.id)" == req.query["id"] }
lock.unlock()
c.respond(json: ["now": nowUs(), "streams": list.map { s -> [String: Any] in
["id": s.id, "src": s.source.key, "frames": s.stats.settled(since: since, settle: settle).map(\.json),
"inputs": s.stats.inputList().map(\.json), "summary": s.stats.summary(),
"captured": s.stats.withLock { s.stats.captured }, "controller": s.controller.state(),
"events": s.controller.eventList()]
}])
case ("POST", "/bench"):
lock.lock()
let list = sessions.values.filter { $0.source.key == req.query["src"] }
lock.unlock()
var m: [String: Any] = [:]
for (k, v) in req.query where k != "k" && k != "src" { m[k] = Double(v) ?? v as Any }
for s in list { s.bench(m) }
c.respond(json: ["sent": list.count])
case ("GET", "/snapshot"):
// One JPEG of a display (for checks and thumbnails): ?display=ID
guard let id = UInt32(req.query["display"] ?? "") else { return c.respond(400, text: "display=ID") }
snapshot(display: id) { data, error in
if let data { c.respond(200, body: data, type: "image/jpeg") } else { c.respond(500, text: error ?? "failed") }
}
case ("POST", "/permissions"):
DispatchQueue.main.async { requestPermissions() }
c.respond(json: permissionsJSON())
@@ -400,7 +631,12 @@ final class Agent {
let fps = min(max(Int(req.query["fps"] ?? "") ?? 60, 5), 120)
let bpp = min(max(Double(req.query["bpp"] ?? "") ?? 0.1, 0.02), 0.5)
guard let ws = c.upgrade(req) else { return }
let capture: CaptureSource = src == .test ? TestSource() : SCKSource(src)
let capture: CaptureSource
switch src {
case .test: capture = TestSource()
case .separate(let id): capture = SeparateSource(windowID: id)
default: capture = SCKSource(src)
}
lock.lock()
// One live viewer per key: a reconnection (or a second use of an
// unacknowledged ticket) replaces the one before.
@@ -428,6 +664,24 @@ final class Agent {
}
}
func snapshot(display id: CGDirectDisplayID, completion: @escaping (Data?, String?) -> Void) {
SCShareableContent.getExcludingDesktopWindows(false, onScreenWindowsOnly: true) { content, error in
guard let d = content?.displays.first(where: { $0.displayID == id }) else {
return completion(nil, error?.localizedDescription ?? "no such display")
}
let filter = SCContentFilter(display: d, excludingWindows: [])
let cfg = SCStreamConfiguration()
cfg.width = Int(Double(d.width) * Double(filter.pointPixelScale))
cfg.height = Int(Double(d.height) * Double(filter.pointPixelScale))
cfg.showsCursor = true
SCScreenshotManager.captureImage(contentFilter: filter, configuration: cfg) { image, error in
guard let image else { return completion(nil, error?.localizedDescription ?? "no image") }
let rep = NSBitmapImageRep(cgImage: image)
completion(rep.representation(using: .jpeg, properties: [.compressionFactor: 0.85]), nil)
}
}
}
func requestPermissions() {
if !CGPreflightScreenCaptureAccess() { CGRequestScreenCaptureAccess() }
if !AXIsProcessTrusted() {
@@ -460,13 +714,19 @@ case "serve":
}
let page = argument("--page", in: args).map { URL(fileURLWithPath: $0) }
let agent = Agent(token: token, page: page)
// Quit when the parent goes away (it holds our stdin open).
// Quit when the parent goes away (it holds our stdin open), or on SIGTERM,
// but first end every stream, so separated windows go back where they
// were before their displays disappear.
if args.contains("--exit-on-eof") {
DispatchQueue.global().async {
while FileHandle.standardInput.availableData.count > 0 {}
exit(0)
agent.quit()
}
}
signal(SIGTERM, SIG_IGN)
let sigterm = DispatchSource.makeSignalSource(signal: SIGTERM, queue: .main)
sigterm.setEventHandler { agent.quit() }
sigterm.resume()
let server: Server
do {
server = try Server(port: port) { req, c in agent.handle(req, c) }
@@ -482,8 +742,13 @@ case "serve":
print("frame-mac-view listening on 127.0.0.1:\(server.port ?? port)")
fflush(stdout)
}
_ = NSApplication.shared // AppKit for NSScreen names and app activation
withExtendedLifetime(server) { RunLoop.main.run() }
// A real (background, no Dock icon) AppKit event loop, not just a run
// loop: without it this process never hears that displays were added or
// changed mode, so NSScreen and CGDisplayCopyDisplayMode stay stale for
// the displays Separate mode creates.
let app = NSApplication.shared
app.setActivationPolicy(.prohibited)
withExtendedLifetime((server, sigterm)) { app.run() }
default:
FileHandle.standardError.write(Data("usage: frame-mac-view serve|windows|displays|permissions|request-permissions\n".utf8))
exit(2)
mac/frame-mac-view/build.sh
+1
View File
@@ -6,5 +6,6 @@ here=$(cd "$(dirname "$0")" && pwd)
out=${1:-$here/../bin/frame-mac-view}
mkdir -p "$(dirname "$out")"
xcrun swiftc -O -swift-version 5 -target arm64-apple-macos14.0 \
-import-objc-header "$here/Sources/CGVirtualDisplay.h" \
-o "$out" "$here"/Sources/*.swift
echo "built $out"
mac/frame-mac-view/lab.sh
Executable
+50
View File
@@ -0,0 +1,50 @@
#!/bin/sh
# Wraps the agent in "Frame Mac View Lab.app" for testing from a terminal or
# an agent session: macOS then asks for Screen Recording and Accessibility
# for this app rather than for the terminal. Signed with an Apple Development
# identity if there is one, so the permissions survive rebuilds.
# mac/frame-mac-view/lab.sh build the app into mac/bin/
# mac/frame-mac-view/lab.sh serve also start it (token, port and log in ~/Library/Caches/frame-mac-view-lab,
# readable only by you: the token opens every window and injects input)
# mac/frame-mac-view/lab.sh serve-only restart it without rebuilding
set -eu
here=$(cd "$(dirname "$0")" && pwd)
app="$here/../bin/Frame Mac View Lab.app"
if [ "${1:-}" != serve-only ]; then
mkdir -p "$app/Contents/MacOS"
sh "$here/build.sh" "$app/Contents/MacOS/frame-mac-view" >/dev/null
cat > "$app/Contents/Info.plist" <<PLIST
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0"><dict>
<key>CFBundleIdentifier</key><string>com.saphid.frame-mac-view-lab</string>
<key>CFBundleExecutable</key><string>frame-mac-view</string>
<key>CFBundleName</key><string>Frame Mac View Lab</string>
<key>CFBundlePackageType</key><string>APPL</string>
<key>CFBundleShortVersionString</key><string>1</string>
<key>LSUIElement</key><true/>
<key>NSScreenCaptureUsageDescription</key><string>Tests streaming Mac windows into the Steam Frame.</string>
</dict></plist>
PLIST
id=$(security find-identity -v -p codesigning 2>/dev/null | awk '/Apple Development/ {print $2; exit}')
codesign -f -s "${id:--}" "$app" >/dev/null 2>&1
echo "built $app (signed ${id:-ad hoc})"
fi
if [ "${1:-}" = serve ] || [ "${1:-}" = serve-only ]; then
pkill -f "Frame Mac View Lab.app/Contents/MacOS/frame-mac-view" 2>/dev/null || true
dir="$HOME/Library/Caches/frame-mac-view-lab"
mkdir -p "$dir" && chmod 700 "$dir"
umask 077
token=$(openssl rand -hex 16)
rm -f "$dir/token" "$dir/port" "$dir/log"
echo "$token" > "$dir/token"
: > "$dir/log"
# Experiment switches pass through: FRAME_MAC_VIEW_ENCODER (Encoder.swift),
# _VD_HZ (Separate.swift), _ADAPT (Controller.swift), _WARM (main.swift).
open -n "$app" --env "FRAME_MAC_VIEW_TOKEN=$token" --env "FRAME_MAC_VIEW_ENCODER=${FRAME_MAC_VIEW_ENCODER:-}" --env "FRAME_MAC_VIEW_VD_HZ=${FRAME_MAC_VIEW_VD_HZ:-}" \
--env "FRAME_MAC_VIEW_ADAPT=${FRAME_MAC_VIEW_ADAPT:-}" --env "FRAME_MAC_VIEW_WARM=${FRAME_MAC_VIEW_WARM:-}" --stdout "$dir/log" --stderr "$dir/log" \
--args serve --port 0 --page "$here/../../ui/mac-view.html"
for _ in 1 2 3 4 5 6 7 8 9 10; do grep -q listening "$dir/log" && break; sleep 0.3; done
sed -n 's/.*127\.0\.0\.1:\([0-9]*\).*/\1/p' "$dir/log" | head -n 1 > "$dir/port"
echo "lab on 127.0.0.1:$(cat "$dir/port")"
fi
scripts/macview-bench.py
Executable
+815
View File
@@ -0,0 +1,815 @@
#!/usr/bin/env python3
"""Benchmark "Mac in the headset" on the real Frame, so every change to the
stream is judged by numbers (docs/mac-in-headset.md, "Measuring").
scripts/macview-bench.py run [--scenario test,scroll,type] [--net 8] [--label x]
scripts/macview-bench.py compare bench/results/A.json bench/results/B.json
scripts/macview-bench.py ab --arm off:FRAME_MAC_VIEW_ADAPT=0 --arm on: --repeat 3 --scenario scroll
`run` uses the lab agent (mac/frame-mac-view/lab.sh serve), which has
Screen Recording and Accessibility. It opens a viewer on the Frame the way
Frame Control does, drives a fixed scenario, and reads the agent's per-frame
records: when the Mac composited each frame, when it was encoded and sent,
and when the viewer received, decoded and drew it (the viewer's clock is
synced to the Mac's). "content" is Mac composited -> drawn in the viewer;
what the Frame's compositor adds after that is reported ("present",
"fps_shown") but not graded, because an unworn Frame throttles panels to
36 fps (15 in standby) after a few seconds whatever they draw. It writes a summary to bench/results/ and exits 1 if a
target is missed ("bad"), or if --baseline is given and a key number got
more than 10% worse.
Wi-Fi changes from minute to minute, so single runs taken at different times
can differ by more than a change does. `ab` restarts the lab agent with each
arm's settings (environment variables read by the agent), runs the arms
interleaved, and prints each metric's median and range per arm.
Nobody needs to wear the headset. The scroll and type scenarios open a
throwaway Chrome window (its own profile in /tmp) on the Mac.
Network conditions come from a relay on the Mac between the agent and the
tunnel, so they need no sudo:
--net 8 8 Mbit/s the whole run
--net 50@0,8@10,50@20 50 Mbit/s, 8 from 10 s, back to 50 from 20 s
--delay 30 30 ms more one-way delay
The relay holds at most --buffer ms of data at the current rate (default
250 ms, like a bloated Wi-Fi queue), then pushes back on the agent.
Stdlib only.
"""
import argparse
import asyncio
import datetime
import json
import os
import platform
import re
import statistics
import subprocess
import sys
import threading
import time
import urllib.request
from pathlib import Path
from urllib.parse import quote, urlencode # %20, not +: the agent's URLComponents keeps +
ROOT = Path(__file__).resolve().parent.parent
sys.path.insert(0, str(ROOT / "ui"))
import frame_macview # noqa: E402
RESULTS = ROOT / "bench" / "results"
PAGES = ROOT / "bench" / "pages"
CHROME = "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome"
CHROME_PROFILE = "/tmp/fmv-bench-chrome"
LAB_DIR = Path.home() / "Library" / "Caches" / "frame-mac-view-lab" # lab.sh serve
# The acceptance bar (the handoff's targets): (feels local, acceptable) per
# metric; worse than acceptable is "bad". Latencies in ms.
TARGETS = {
"content_p50": (25, 60), "content_p95": (40, 100),
"input_p50": (50, 100), "input_p95": (70, 150),
"fps": (58, 45), # higher is better
"late_pct": (1, 5), "stall_max": (100, 250),
"adapt_s": (1, 3),
}
HIGHER_IS_BETTER = {"fps"}
# Content that changes every frame, so fps and stalls mean something.
MOVING = {"test", "scroll"}
def ms(a, b):
return (b - a) / 1000 if a and b else None
def pct(values, p):
v = sorted(x for x in values if x is not None)
if not v:
return None
return round(v[min(len(v) - 1, max(0, round(p * (len(v) - 1))))], 1)
def dist(values):
v = [x for x in values if x is not None]
return {"p50": pct(v, 0.5), "p95": pct(v, 0.95), "p99": pct(v, 0.99), "n": len(v)} if v else {"n": 0}
# ---- the network relay ----
class Relay:
"""Agent <-> tunnel, shaping the agent-to-Frame direction: a bottleneck
of `rate` Mbit/s with a queue of `buffer_ms`, plus `delay` ms each way."""
def __init__(self, target_port, schedule, delay_ms=0, buffer_ms=250):
self.target_port = target_port
self.schedule = schedule # [(seconds from start, Mbit/s or None)]
self.delay = delay_ms / 1000
self.buffer_ms = buffer_ms
self.rate = schedule[0][1] if schedule else None
self.next_free = 0.0
self.port = None
self.loop = asyncio.new_event_loop()
# Connections reset when a run tears the tunnel down; that's expected.
self.loop.set_exception_handler(lambda loop, ctx: None if isinstance(ctx.get("exception"), ConnectionError)
else loop.default_exception_handler(ctx))
self.started = None
self.queued_max = 0
def start(self):
ready = threading.Event()
def run():
asyncio.set_event_loop(self.loop)
server = self.loop.run_until_complete(asyncio.start_server(self._client, "127.0.0.1", 0))
self.port = server.sockets[0].getsockname()[1]
ready.set()
self.loop.run_forever()
threading.Thread(target=run, daemon=True).start()
ready.wait(5)
def begin(self):
"""Start the schedule's clock (when the measured part begins)."""
self.started = time.monotonic()
for at, rate in self.schedule:
self.loop.call_soon_threadsafe(self.loop.call_later, at, self._set_rate, rate)
def _set_rate(self, rate):
self.rate = rate
def rate_at(self, t):
r = None
for at, rate in self.schedule:
if t >= at:
r = rate
return r
async def _client(self, reader, writer):
try:
up_r, up_w = await asyncio.open_connection("127.0.0.1", self.target_port)
except OSError:
writer.close()
return
await asyncio.gather(self._shaped(up_r, writer), self._plain(reader, up_w), return_exceptions=True)
for w in (writer, up_w):
w.close()
async def _plain(self, reader, writer): # Frame -> agent: delay only
while data := await reader.read(65536):
if self.delay:
await asyncio.sleep(self.delay)
writer.write(data)
await writer.drain()
writer.close()
async def _shaped(self, reader, writer): # agent -> Frame
queue = asyncio.Queue()
state = {"bytes": 0}
room = asyncio.Event()
room.set()
async def send():
while True:
at, data = await queue.get()
if data is None:
return
wait = at - self.loop.time()
if wait > 0:
await asyncio.sleep(wait)
writer.write(data)
await writer.drain()
state["bytes"] -= len(data)
room.set()
sender = asyncio.ensure_future(send())
while data := await reader.read(16384):
now = self.loop.time()
depart = now
if self.rate:
depart = max(now, self.next_free) + len(data) * 8 / (self.rate * 1e6)
self.next_free = depart
state["bytes"] += len(data)
self.queued_max = max(self.queued_max, state["bytes"])
await queue.put((depart + self.delay, data))
# A full queue pushes back, as a real bottleneck's buffer does.
while self.rate and state["bytes"] > self.rate * 1e6 / 8 * self.buffer_ms / 1000:
room.clear()
await room.wait()
await queue.put((0, None))
await sender
# ---- the lab agent, through Frame Control's own code ----
class LabView(frame_macview.MacView):
"""MacView against the already-running lab agent, with the tunnel
pointing at `tunnel_port` (the relay, or the agent itself)."""
def __init__(self, tunnel_ssh, run, frame, agent_port, token, tunnel_port):
super().__init__(tunnel_ssh, run, frame)
self.agent_port = agent_port
self.token = token
self.port = tunnel_port
class Alive:
def poll(self):
return None
self.agent = Alive()
def ensure_agent(self):
pass
def call(self, path, method="GET", **query):
url = f"http://127.0.0.1:{self.agent_port}{path}?{urlencode({**query, 'k': self.token}, quote_via=quote)}"
req = urllib.request.Request(url, method=method, data=b"" if method == "POST" else None)
with urllib.request.urlopen(req, timeout=15) as r:
return json.load(r)
def ssh_runner(base):
def run(remote, stdin=None, timeout=30):
r = subprocess.run([*base, remote], input=stdin, capture_output=True, text=True, timeout=timeout)
if r.returncode:
e = RuntimeError((r.stderr or r.stdout).strip())
e.stdout = r.stdout
raise e
return r.stdout
return run
# ---- CPU on both ends ----
FRAME_CPU = r"""
hz=$(getconf CLK_TCK)
while :; do
t=$(date +%s.%N)
for g in viewer:frame-control/mac-view tailscaled:tailscaled sshd:'^sshd' gamescope:'^gamescope'; do
name=${g%%:*} pat=${g#*:} sum=0
for p in $(pgrep -f "$pat"); do
s=$(awk '{print $14+$15}' /proc/$p/stat 2>/dev/null) && sum=$((sum + s))
done
echo "$t $name $sum $hz"
done
echo "$t total $(awk '/^cpu /{print $2+$3+$4+$6+$7+$8}' /proc/stat) $hz"
sleep 2
done
"""
class CpuSampler:
def __init__(self, frame_ssh, agent_pid):
self.agent_pid = agent_pid
self.mac = [] # (t, cpu seconds)
self.frame = {} # name -> [(t, ticks, hz)]
self.proc = subprocess.Popen([*frame_ssh, "bash -s"], stdin=subprocess.PIPE, stdout=subprocess.PIPE,
stderr=subprocess.DEVNULL, text=True)
self.proc.stdin.write(FRAME_CPU)
self.proc.stdin.close()
threading.Thread(target=self._read, daemon=True).start()
self.stop_flag = False
threading.Thread(target=self._mac, daemon=True).start()
def _read(self):
for line in self.proc.stdout:
parts = line.split()
if len(parts) == 4:
self.frame.setdefault(parts[1], []).append((float(parts[0]), int(parts[2]), int(parts[3])))
def _mac(self):
while not self.stop_flag:
out = subprocess.run(["ps", "-o", "cputime=", "-p", str(self.agent_pid)], capture_output=True,
text=True).stdout.strip()
m = re.match(r"(?:(\d+):)?(\d+):(\d+(?:\.\d+)?)", out)
if m:
h, mnt, s = m.groups()
self.mac.append((time.monotonic(), int(h or 0) * 3600 + int(mnt) * 60 + float(s)))
time.sleep(2)
def stop(self):
self.stop_flag = True
self.proc.terminate()
def summary(self):
def rate(samples):
if len(samples) < 2:
return None
(t0, a0, *h0), (t1, a1, *_) = samples[0], samples[-1]
hz = h0[0] if h0 else 1
return round((a1 - a0) / hz / (t1 - t0) * 100, 1)
out = {"mac_agent_pct": rate(self.mac)}
for name, s in self.frame.items():
out[f"frame_{name}_pct"] = rate(s)
if "frame_total_pct" in out and out["frame_total_pct"] is not None:
out["frame_total_pct"] = round(out["frame_total_pct"] / 8, 1) # 8 cores -> share of the whole
return out
# ---- scenarios ----
def chrome_window(page, size=(1280, 820)):
"""Opens a page in a throwaway Chrome app window; returns (proc, title)."""
if not Path(CHROME).exists():
raise SystemExit("Google Chrome is needed for the scroll and type scenarios.")
proc = subprocess.Popen([CHROME, f"--user-data-dir={CHROME_PROFILE}", "--no-first-run",
"--no-default-browser-check", f"--window-size={size[0]},{size[1]}",
"--window-position=80,80", f"--app=file://{PAGES / page}"],
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
return proc
def find_window(mv, title, timeout=15):
end = time.time() + timeout
while time.time() < end:
for w in mv.call("/windows").get("windows", []):
if title in (w.get("title") or ""):
return w
time.sleep(0.5)
raise SystemExit(f"window {title!r} didn't appear")
def git(*a):
return subprocess.run(["git", *a], cwd=ROOT, capture_output=True, text=True).stdout.strip()
def parse_schedule(net):
if not net or net == "none":
return []
out = []
for part in net.split(","):
rate, _, at = part.partition("@")
out.append((float(at or 0), None if rate in ("", "none", "0") else float(rate)))
return sorted(out)
def run_scenario(args, scenario, agent_port, token, frame_ssh):
schedule = parse_schedule(args.net)
relay = None
tunnel_port = agent_port
if schedule or args.delay:
relay = Relay(agent_port, schedule, args.delay, args.buffer)
relay.start()
tunnel_port = relay.port
tunnel_ssh = list(frame_ssh[:-1]) + args.ssh_opt
mv = LabView(tunnel_ssh, ssh_runner(frame_ssh), frame_ssh[-1], agent_port, token, tunnel_port)
if args.browser_flag is not None:
mv.browser_flags = [f for f in args.browser_flag if f]
chrome = None
src = "test"
try:
if scenario in ("scroll", "type"):
chrome = chrome_window(f"{scenario}.html")
w = find_window(mv, f"fmv-bench {scenario}")
src = f"{args.mode}:{w['id']}"
t = time.time()
shown = mv.show(src, args.quality, 1280, 820)
show_s = round(time.time() - t, 2)
# Wait for frames, then measure from a clean start.
stream = None
end = time.time() + 20
while time.time() < end and not stream:
for s in mv.call("/status").get("streams", []):
if s["src"] == src and s.get("stats", {}).get("fps", 0) > 0:
stream = s
time.sleep(0.5)
if not stream:
raise SystemExit(f"{scenario}: no frames reached the viewer within 20 s")
time.sleep(args.warmup)
first = mv.call("/stats", id=stream["id"], settle=0)["streams"][0]
since = max([f["s"] for f in first["frames"]] or [0])
captured_before = first["captured"]
start_mac_us = mv.call("/stats", id=stream["id"])["now"]
agent_pid = int(subprocess.run(["pgrep", "-f", "Frame Mac View Lab.app/Contents/MacOS/frame-mac-view"],
capture_output=True, text=True).stdout.split()[0])
cpu = CpuSampler(frame_ssh, agent_pid)
if relay:
relay.begin()
expected = 0 # input events the harness asked the viewer for
if scenario == "type":
mv.call("/bench", method="POST", src=src, action="click", x=0.5, y=0.3)
time.sleep(0.5)
# Steady typing, then slow typing that lets the link go idle.
fast = "the quick brown fox jumps over the lazy dog "
n_fast = max(10, int((args.duration * 0.6) / 0.2))
mv.call("/bench", method="POST", src=src, action="type", text=(fast * (n_fast // len(fast) + 1))[:n_fast],
interval=200)
expected += n_fast
time.sleep(n_fast * 0.2 + 0.5)
slow_n = max(3, int(args.duration * 0.4 / 1.5))
mv.call("/bench", method="POST", src=src, action="type", text=("idle typing test " * (slow_n // 17 + 1))[:slow_n],
interval=1500)
expected += slow_n
time.sleep(slow_n * 1.5 + 0.5)
else:
# Clicks on the test pattern measure the input path; the scroll
# page just scrolls.
end = time.time() + args.duration
while time.time() < end:
if scenario == "test":
mv.call("/bench", method="POST", src=src, action="click", x=0.3, y=0.5)
expected += 1
time.sleep(0.5)
at_end = mv.call("/stats", id=stream["id"], since=2**32 - 1)
end_mac_us = at_end["now"]
captured_end = at_end["streams"][0]["captured"] if at_end["streams"] else None
time.sleep(2) # let the viewer report the last frames
streams = mv.call("/stats", id=stream["id"], since=since)["streams"]
if not streams:
raise SystemExit(f"{scenario}: the stream ended during the run (did the Frame go to sleep?)")
data = streams[0]
cpu.stop()
if args.raw:
Path(f"{args.raw}-{scenario}.json").write_text(json.dumps(data))
result = summarize(scenario, data, start_mac_us, end_mac_us, args, relay, schedule, expected)
result["controller"] = data.get("controller", {})
result["events"] = [{"t": round((e["t"] - start_mac_us) / 1e6, 2), "e": e["e"]}
for e in data.get("events", []) if e["t"] >= start_mac_us]
result["captured"] = (captured_end if captured_end is not None else data["captured"]) - captured_before
result["source_fps"] = round(result["captured"] / max(result["duration_s"], 1), 1)
result["cpu"] = cpu.summary()
result["viewer"] = data["summary"].get("decoder", "")
result["show_s"] = show_s
result["panel"] = shown.get("panel")
result["src"] = src
return result
finally:
try:
mv.stop(src)
except Exception: # noqa: BLE001 - best effort
pass
mv.closing = True # or its supervisor reopens the tunnel
if mv.tunnel and mv.tunnel.poll() is None:
mv.tunnel.terminate()
mv.tunnel.wait(5)
if relay:
relay.loop.call_soon_threadsafe(relay.loop.stop)
if chrome:
chrome.terminate()
time.sleep(1.5)
def composited(f):
"""When the Mac had the frame: its display time, or when ScreenCaptureKit
handed it over if that was earlier (display time can be a few ms ahead of
delivery, which would make latency look lower than it is)."""
return min(f["cap"], f["arr"]) if f["arr"] else f["cap"]
def summarize(scenario, data, start_us, end_us, args, relay, schedule, expected=0):
frames = [f for f in data["frames"] if start_us <= f["cap"] <= end_us]
inputs = [i for i in data["inputs"] if i["tv"] and i["tv"] >= start_us]
by_seq = {f["s"]: f for f in data["frames"]}
drawn = [f for f in frames if f["drw"]]
shown = [f for f in frames if f["vs"]]
# The whole measured interval, so a stall at either end still counts.
duration = (end_us - start_us) / 1e6
stages = {
"capture": [ms(f["cap"], f["arr"]) for f in frames],
"queue": [ms(f["arr"], f["e0"]) for f in frames],
"encode": [ms(f["e0"], f["e1"]) for f in frames],
"socket": [ms(f["snd"], f["wire"]) for f in frames],
"network": [ms(f["e1"], f["rx"]) for f in frames],
"decode": [ms(f["rx"], f["dec"]) for f in frames],
"draw": [ms(f["dec"], f["drw"]) for f in frames],
# Waiting for the browser's next frame: set by the Frame's compositor,
# which throttles panels nobody is looking at (see "Measuring").
"present": [ms(f["drw"], f["vs"]) for f in frames],
"content": [ms(composited(f), f["drw"]) for f in frames],
"content_shown": [ms(composited(f), f["vs"]) for f in frames],
}
out = {"scenario": scenario, "frames_sent": len(frames), "frames_drawn": len(drawn),
"frames_shown": len(shown), "duration_s": round(duration, 1)}
out["stages_ms"] = {k: dist(v) for k, v in stages.items()}
# Smoothness: gaps between frames reaching the viewer's canvas.
# Smoothness counts only what was drawn inside the interval; a frame
# captured just before the end but drawn after it still counts for latency.
dr = [start_us] + sorted(f["drw"] for f in drawn if f["drw"] <= end_us) + [end_us]
gaps = [(b - a) / 1000 for a, b in zip(dr, dr[1:]) if b > a]
target_fps = frame_macview.QUALITY[args.quality]["fps"]
out["fps"] = round(sum(f["drw"] <= end_us for f in drawn) / duration, 1) if duration else 0
out["fps_shown"] = round(sum(0 < f["vs"] <= end_us for f in shown) / duration, 1) if duration else 0
out["late_pct"] = round(100 * sum(g > 1.5 * 1000 / target_fps for g in gaps) / len(gaps), 2) if gaps else None
out["stall_max"] = round(max(gaps), 1) if gaps else None
out["stalls_over_100ms"] = sum(g > 100 for g in gaps)
out["mbps"] = round(sum(f["b"] for f in frames) * 8 / duration / 1e6, 2) if duration else 0
out["keyframes"] = sum(f["k"] for f in frames)
out["size"] = f"{frames[-1]['w']}x{frames[-1]['h']}" if frames else ""
out["captured"] = data.get("captured")
out["viewer_never_drawn"] = len(frames) - len(drawn)
# Input: the viewer's event -> the first frame after it drawn (and shown).
lat, lat_shown = [], []
parts = {"uplink": [], "mac": [], "back": []}
for i in inputs:
f = by_seq.get(i["frame"])
if f and f["drw"]:
lat.append(ms(i["tv"], f["drw"]))
lat_shown.append(ms(i["tv"], f["vs"]))
parts["uplink"].append(ms(i["tv"], i["inj"]))
parts["mac"].append(ms(i["inj"], composited(f)))
parts["back"].append(ms(composited(f), f["drw"]))
out["input_ms"] = dist(lat)
out["input_shown_ms"] = dist(lat_shown)
out["input_parts_ms"] = {k: dist(v) for k, v in parts.items()}
out["inputs"] = {"asked": expected, "sent": len(inputs), "seen": len(lat)}
# A per-second timeline, for adaptation.
timeline = []
t0 = start_us
for sec in range(int(duration) + 1):
a, b = t0 + sec * 1_000_000, t0 + (sec + 1) * 1_000_000
fs = [f for f in frames if a <= f["cap"] < b]
if not fs:
continue
timeline.append({
"t": sec, "fps": sum(1 for f in fs if f["drw"]), "mbps": round(sum(f["b"] for f in fs) * 8 / 1e6, 2),
"content_p50": pct([ms(composited(f), f["drw"]) for f in fs], 0.5),
"content_p95": pct([ms(composited(f), f["drw"]) for f in fs], 0.95),
"bitrate": fs[-1].get("br"), "tier": fs[-1].get("tier"), "w": fs[-1]["w"],
"net": relay.rate_at(sec) if relay else None,
})
out["timeline"] = timeline
out["adapt"] = adaptation(timeline, schedule, duration)
out["content_p50"] = out["stages_ms"]["content"].get("p50")
out["content_p95"] = out["stages_ms"]["content"].get("p95")
out["input_p50"] = out["input_ms"].get("p50")
out["input_p95"] = out["input_ms"].get("p95")
out["grades"] = grade(out, scenario, args.quality)
return out
def adaptation(timeline, schedule, duration=float("inf")):
"""After each drop in the link's rate: how long until latency was within
bounds again and stayed there for 3 s. The bound is the acceptable p95
(100 ms), or 1.5 times the p95 before the drop if that was higher: on a
slower link each frame takes longer to send, so latency can't return to
what it was on the fast one."""
steps = []
for n, ((at, rate), (_, before)) in enumerate(zip(schedule[1:], schedule)):
# Settling must happen while this rate lasts, not after the link recovers.
until = min(schedule[n + 2][0] if n + 2 < len(schedule) else float("inf"), duration) # or the run ends
if before is not None and (rate is None or rate >= before):
continue
pre = [s["content_p95"] for s in timeline if at - 4 <= s["t"] < at and s["content_p95"]]
base = max(float(TARGETS["content_p95"][1]), 1.5 * statistics.median(pre) if pre else 0)
after = [s for s in timeline if s["t"] >= at]
worst = max([s["content_p95"] or 0 for s in after[:10]] or [0])
settled = None
for i, s in enumerate(after):
window = after[i:i + 3]
# Three populated seconds in a row: a second with no frames at all
# is a stall, not a pass.
if (len(window) == 3 and [w["t"] for w in window] == [s["t"], s["t"] + 1, s["t"] + 2] and s["t"] + 3 <= until
and all((w["content_p95"] or 1e9) <= base for w in window)):
settled = s["t"] - at
break
steps.append({"at": at, "to_mbit": rate, "p95_bound": round(base, 1), "worst_p95": worst,
"settled_s": settled})
return steps
def grade(out, scenario, quality="balanced"):
g = {}
checks = ["content_p50", "content_p95"]
asked = max(out["inputs"].get("asked", 0), out["inputs"]["sent"])
if asked:
checks += ["input_p50", "input_p95"]
# Input that never reaches the Mac, or never shows up on screen, is a
# failure, not a gap in the data.
seen = out["inputs"]["seen"] / asked
g["input_replies"] = "local" if seen >= 0.95 else "acceptable" if seen >= 0.8 else "bad"
if scenario in MOVING:
checks += ["fps", "late_pct", "stall_max"]
for k in checks:
v = out.get(k)
if v is None:
g[k] = "missing"
continue
good, ok = TARGETS[k]
if k == "fps": # the targets are for 60 fps; Light and Compatible ask for fewer
want = frame_macview.QUALITY[quality]["fps"]
good, ok = good * want / 60, ok * want / 60
if k in HIGHER_IS_BETTER:
g[k] = "local" if v >= good else "acceptable" if v >= ok else "bad"
else:
g[k] = "local" if v <= good else "acceptable" if v <= ok else "bad"
for step in out["adapt"]:
s = step["settled_s"]
good, ok = TARGETS["adapt_s"]
g[f"adapt@{step['at']:g}s"] = "bad" if s is None or s > ok else "local" if s <= good else "acceptable"
return g
# ---- compare ----
KEYS = [("content_p50", "content p50 ms"), ("content_p95", "content p95 ms"), ("input_p50", "input p50 ms"),
("input_p95", "input p95 ms"), ("fps", "fps"), ("late_pct", "late %"), ("stall_max", "max gap ms"),
("fps_shown", "fps shown"), ("mbps", "Mbit/s")]
STAGE_KEYS = ["capture", "queue", "encode", "socket", "network", "decode", "draw", "present"]
def load(path):
return json.loads(Path(path).read_text())
def compare(a, b, threshold=0.10):
"""Prints before/after per scenario; returns the regressions."""
regressions = []
print(f"A: {a['label']} ({a['commit']}, {a['date']})\nB: {b['label']} ({b['commit']}, {b['date']})")
for sc in b["scenarios"]:
ra = next((x for x in a["scenarios"] if x["scenario"] == sc["scenario"]), None)
if not ra:
continue
print(f"\n{sc['scenario']:<10} {'A':>10} {'B':>10} {'change':>9}")
rows = [(k, n, ra.get(k), sc.get(k)) for k, n in KEYS]
rows += [(f"stage.{s}", f" {s} p50", ra["stages_ms"].get(s, {}).get("p50"), sc["stages_ms"].get(s, {}).get("p50"))
for s in STAGE_KEYS]
for k, name, va, vb in rows:
change = ""
if isinstance(va, (int, float)) and isinstance(vb, (int, float)) and va:
d = (vb - va) / abs(va)
change = f"{d:+.0%}"
worse = d < -threshold if k == "fps" else d > threshold
if worse and k in ("content_p50", "content_p95", "input_p50", "input_p95", "fps") and abs(vb - va) > 2:
regressions.append(f"{sc['scenario']} {name}: {va} -> {vb}")
change += " worse"
print(f"{name:<18} {fmt(va):>10} {fmt(vb):>10} {change:>9}")
return regressions
def fmt(v):
return "–" if v is None else f"{v:g}" if isinstance(v, (int, float)) else str(v)
# ---- main ----
def add_run_args(r):
r.add_argument("--scenario", default="test,scroll,type")
r.add_argument("--duration", type=float, default=20)
r.add_argument("--warmup", type=float, default=3)
r.add_argument("--quality", default="balanced", choices=list(frame_macview.QUALITY))
r.add_argument("--mode", default="separate", choices=["separate", "window"], help="how Mac windows are shown")
r.add_argument("--net", default="", help="Mbit/s, or a schedule like 50@0,8@10,50@20")
r.add_argument("--delay", type=float, default=0, help="extra one-way delay, ms")
r.add_argument("--buffer", type=float, default=250, help="bottleneck queue, ms at the current rate")
r.add_argument("--frame", default="frame", help="ssh host of the Frame")
r.add_argument("--host", default="", help="connect to this address instead (e.g. the Frame's LAN IP)")
r.add_argument("--ssh-opt", action="append", default=[], help="extra ssh option for the tunnel, e.g. -oIPQoS=ef")
r.add_argument("--browser-flag", action="append", default=None,
help="Chromium flag for the viewer (replaces Frame Control's defaults; '' for none)")
r.add_argument("--label", default="")
r.add_argument("--out", default="")
r.add_argument("--raw", default="", help="also write the agent's raw records here")
def lab_agent():
try:
agent_port = int((LAB_DIR / "port").read_text())
token = (LAB_DIR / "token").read_text().strip()
urllib.request.urlopen(f"http://127.0.0.1:{agent_port}/ping", timeout=3)
return agent_port, token
except (OSError, ValueError):
raise SystemExit("Start the lab agent first: mac/frame-mac-view/lab.sh serve")
def restart_lab(env):
"""The lab agent again, without rebuilding, with these experiment switches."""
subprocess.run(["sh", str(ROOT / "mac" / "frame-mac-view" / "lab.sh"), "serve-only"],
env={**os.environ, **env}, check=True, capture_output=True)
time.sleep(1)
def frame_ssh_for(args):
frame_ssh = ["ssh", "-o", "BatchMode=yes"]
if args.host:
# Another address for the same Frame: its host key must still match.
alias = next((line.split()[1] for line in subprocess.run(["ssh", "-G", args.frame], capture_output=True,
text=True).stdout.splitlines() if line.startswith("hostname ")), args.frame)
frame_ssh += ["-o", f"HostName={args.host}", "-o", f"HostKeyAlias={alias}"]
frame_ssh.append(args.frame)
return frame_ssh
def run_suite(args, frame_ssh, quiet=False):
agent_port, token = lab_agent()
results = []
for sc in args.scenario.split(","):
if not quiet:
print(f"-- {sc} ...", flush=True)
res = run_scenario(args, sc, agent_port, token, frame_ssh)
if not quiet:
print(f" content p50/p95 {res['content_p50']}/{res['content_p95']} ms, input p50/p95 "
f"{res['input_p50']}/{res['input_p95']} ms, {res['fps']} fps (shown {res['fps_shown']}), "
f"late {res['late_pct']}%, max gap {res['stall_max']} ms, {res['mbps']} Mbit/s, {res['size']}",
flush=True)
print(f" stages p50: " + ", ".join(f"{k} {v.get('p50')}" for k, v in res["stages_ms"].items()), flush=True)
print(f" grades: {res['grades']} cpu: {res['cpu']}\n viewer: {res['viewer']}", flush=True)
for e in res["events"][:12]:
print(f" {e['t']:6.2f}s {e['e']}", flush=True)
results.append(res)
return results
def document(args, frame_ssh, results, **extra):
build = subprocess.run([*frame_ssh, "grep -m1 BUILD_ID /etc/os-release"], capture_output=True, text=True).stdout
# Whether the headset is in standby (it throttles panels even harder then).
standby = subprocess.run([*frame_ssh, "grep -h standby ~/.local/share/Steam/logs/vrserver.txt | tail -n 1"],
capture_output=True, text=True).stdout.strip()
commit = git("rev-parse", "--short", "HEAD") + ("+dirty" if git("status", "--porcelain", "--", "mac", "ui") else "")
return {
"label": args.label or args.cmd, "date": datetime.datetime.now().isoformat(timespec="seconds"), "commit": commit,
"config": {"quality": args.quality, "mode": args.mode, "net": args.net or "none", "delay_ms": args.delay,
"buffer_ms": args.buffer, "host": args.host or args.frame, "ssh_opts": args.ssh_opt,
"encoder": os.environ.get("FRAME_MAC_VIEW_ENCODER", ""), "duration_s": args.duration,
"browser_flags": args.browser_flag if args.browser_flag is not None else frame_macview.BROWSER_FLAGS},
"frame_build": build.strip().partition("=")[2], "mac": platform.mac_ver()[0], "headset": standby[-40:],
"scenarios": results, **extra,
}
def write(doc, args):
RESULTS.mkdir(parents=True, exist_ok=True)
name = f"{doc['date'][:10]}-{doc['commit'].replace('+', '-')}-{re.sub(r'[^a-zA-Z0-9_.-]+', '-', doc['label'])}.json"
out = Path(args.out) if args.out else RESULTS / name
out.write_text(json.dumps(doc, indent=1))
print(f"wrote {out}")
return out
AB_KEYS = ["content_p50", "content_p95", "input_p50", "input_p95", "fps", "late_pct", "stall_max", "mbps"]
def ab(args, frame_ssh):
"""Arms interleaved (A B, B A, ...) so a Wi-Fi link that changes over
minutes affects both alike; medians and ranges per arm."""
arms = []
for a in args.arm:
name, _, envs = a.partition(":")
arms.append((name, dict(e.split("=", 1) for e in envs.split(",") if e)))
runs = {name: [] for name, _ in arms}
for rep in range(args.repeat):
for name, env in (arms if rep % 2 == 0 else arms[::-1]):
print(f"-- repeat {rep + 1}, arm {name} {env}", flush=True)
restart_lab(env)
runs[name].append(run_suite(args, frame_ssh, quiet=True))
table = {}
for sc in args.scenario.split(","):
print(f"\n{sc:<14}" + "".join(f"{name:>26}" for name, _ in arms))
for k in AB_KEYS:
row = []
for name, _ in arms:
vals = [r[k] for suite in runs[name] for r in suite if r["scenario"] == sc and r.get(k) is not None]
table.setdefault(sc, {}).setdefault(k, {})[name] = vals
row.append(f"{statistics.median(vals):g} ({min(vals):g}-{max(vals):g})" if vals else "–")
print(f"{k:<14}" + "".join(f"{c:>26}" for c in row))
restart_lab({}) # back to the defaults
return runs, table
def main():
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
sub = p.add_subparsers(dest="cmd", required=True)
r = sub.add_parser("run")
add_run_args(r)
r.add_argument("--baseline", default="", help="fail on >10%% regressions against this result")
a = sub.add_parser("ab", help="compare lab-agent settings, interleaved: --arm name:ENV=V,ENV2=V ...")
add_run_args(a)
a.add_argument("--arm", action="append", required=True)
a.add_argument("--repeat", type=int, default=3)
c = sub.add_parser("compare")
c.add_argument("a")
c.add_argument("b")
args = p.parse_args()
# The agent keeps the last 4096 frames (Stats.swift) and results are read
# at the end: at 60 fps that's 68 s, less warm-up and the reporting wait.
if args.cmd != "compare" and args.warmup + args.duration > 60:
p.error("--warmup plus --duration can be at most 60 s (the agent keeps 4096 frames)")
if args.cmd == "compare":
regs = compare(load(args.a), load(args.b))
if regs:
print("\nworse:\n " + "\n ".join(regs))
sys.exit(1 if regs else 0)
lab_agent()
frame_ssh = frame_ssh_for(args)
# Keep the Mac's screen awake: virtual displays aren't removed while it sleeps.
runs = 1 if args.cmd == "run" else args.repeat * len(args.arm)
awake = subprocess.Popen(["caffeinate", "-d", "-u", "-t", str(int(runs * (args.duration * 4 + 60) + 120))])
try:
if args.cmd == "ab":
arm_runs, table = ab(args, frame_ssh)
write(document(args, frame_ssh, [], arms=args.arm, runs=arm_runs, table=table), args)
return
results = run_suite(args, frame_ssh)
finally:
awake.terminate()
doc = document(args, frame_ssh, results)
write(doc, args)
bad = [f"{r['scenario']} {k}" for r in results for k, v in r["grades"].items() if v in ("bad", "missing")]
regs = compare(load(args.baseline), doc) if args.baseline else []
if bad:
print("missed targets: " + ", ".join(bad))
if regs:
print("worse than baseline:\n " + "\n ".join(regs))
sys.exit(1 if bad or regs else 0)
if __name__ == "__main__":
main()
tests/test_macview.py
+76 -4
View File
@@ -68,6 +68,13 @@ class Helpers(unittest.TestCase):
self.assertIn("Chromium", str(cm.exception))
self.assertTrue(calls[0].startswith("bash -s -- "))
def test_separate_windows_need_accessibility(self):
mv = frame_macview.MacView(["ssh"], lambda *a, **k: self.fail("no ssh"), "frame")
mv.call = lambda path, **kw: {"screen": True, "accessibility": False}
with self.assertRaises(frame_macview.MacViewError) as cm:
mv.show("separate:42")
self.assertIn("Accessibility", str(cm.exception))
def test_screen_permission_is_checked_before_the_headset(self):
mv = frame_macview.MacView(["ssh"], lambda *a, **k: self.fail("no ssh"), "frame")
mv.call = lambda path, **kw: {"screen": False}
@@ -162,6 +169,22 @@ class Agent(unittest.TestCase):
self.assertEqual(status, 200)
self.assertIn(b"VideoDecoder", body)
def test_snapshot_needs_the_key(self):
self.assertEqual(self.get("/snapshot?display=1")[0], 403)
def test_separate_without_accessibility_explains(self):
if json.loads(self.get(f"/status?k={self.token}")[1])["accessibility"]:
self.skipTest("this Mac allows Accessibility here")
ws = WS(self.port, f"/stream?k={self.token}&src=separate:1")
self.assertEqual(ws.status, 101)
for _ in range(5):
op, data = ws.recv()
msg = json.loads(data) if op == 1 else {}
if msg.get("t") in ("error", "closed"):
break
self.assertIn("Accessibility", msg.get("message", msg.get("reason", "")))
ws.close()
def test_lists_displays(self):
status, body = self.get(f"/displays?k={self.token}")
self.assertEqual(status, 200)
@@ -242,9 +265,10 @@ class Agent(unittest.TestCase):
break
self.assertEqual(info["src"], "test")
self.assertTrue(info["input"])
# A keyframe: flags, 8-byte timestamp, then Annex B with the SPS first.
self.assertEqual(key[9:13], b"\x00\x00\x00\x01")
self.assertEqual(key[13] & 0x1F, 7) # NAL type 7, SPS
# A keyframe: flags, 8-byte timestamp, sequence number, input echoed,
# then Annex B with the SPS first.
self.assertEqual(key[17:21], b"\x00\x00\x00\x01")
self.assertEqual(key[21] & 0x1F, 7) # NAL type 7, SPS
ws.send_text(json.dumps({"t": "m", "e": "down", "b": 0, "x": 0.5, "y": 0.5}))
ws.send_text(json.dumps({"t": "key-frame"}))
got_key = False
@@ -260,7 +284,7 @@ class Agent(unittest.TestCase):
self.assertEqual(json.loads(body)["closed"], 1)
for _ in range(400):
op, data = ws.recv()
if op == 1:
if op == 1 and json.loads(data)["t"] == "close":
break
self.assertEqual(json.loads(data)["t"], "close")
# Without the viewer doing anything, the agent ends the stream itself.
@@ -269,6 +293,54 @@ class Agent(unittest.TestCase):
self.assertEqual(json.loads(body)["streams"], [])
ws.close()
def test_timing_reports_and_input_echo(self):
# What a viewer does: sync clocks, report each frame, stamp input.
ws = WS(self.port, f"/stream?k={self.token}&src=test&codec=h264&fps=30&max=640")
self.assertEqual(ws.status, 101)
ws.send_text(json.dumps({"t": "w"})) # keep-warm filler: ignored
ws.send_text(json.dumps({"t": "ping", "c": 1.5}))
pong, echoed, seqs, info = None, None, [], None
clicked = False
deadline = time.time() + 10
while time.time() < deadline and not (pong and echoed):
op, data = ws.recv()
if op == 1:
msg = json.loads(data)
if msg["t"] == "error":
self.skipTest("no H.264 encoder here: " + msg["message"])
if msg["t"] == "pong":
pong = msg
if msg["t"] == "info":
info = msg
elif op == 2:
seq, echo = struct.unpack(">II", data[9:17])
seqs.append(seq)
now = pong["a"] if pong else 0
ws.send_text(json.dumps({"t": "rx", "s": seq, "r": now}))
ws.send_text(json.dumps({"t": "fd", "f": [[seq, now + 1, now + 2, now + 3]], "drop": 0}))
if echo == 7:
echoed = seq
if len(seqs) == 3 and not clicked:
ws.send_text(json.dumps({"t": "m", "e": "down", "b": 0, "x": 0.5, "y": 0.5, "i": 7, "tv": now}))
clicked = True
self.assertEqual(pong["c"], 1.5)
self.assertGreater(pong["a"], 0)
self.assertEqual(seqs[:3], sorted(seqs[:3]))
self.assertIsNotNone(echoed, "no frame was tagged as the first reply to the click")
time.sleep(1.7) # frames are reported once the viewer has had time
stream = json.loads(self.get(f"/stats?k={self.token}")[1])["streams"][0]
first = stream["frames"][0]
self.assertGreater(first["e1"], first["e0"])
self.assertGreaterEqual(first["e0"], first["cap"])
self.assertEqual(first["vs"] - first["rx"], 3)
self.assertEqual([i["frame"] for i in stream["inputs"] if i["id"] == 7], [echoed])
self.assertIn("total", stream["summary"])
self.assertEqual(info["warm"], 0) # off unless FRAME_MAC_VIEW_WARM is set
live = json.loads(self.get(f"/status?k={self.token}")[1])["streams"][0]
self.assertEqual(live["controller"]["tier"], 0)
self.assertIn("fps", live["stats"])
ws.close()
if __name__ == "__main__":
unittest.main()
ui/frame_macview.py
+19 -6
View File
@@ -30,7 +30,7 @@ import urllib.error
import urllib.request
import zlib
from pathlib import Path
from urllib.parse import urlencode
from urllib.parse import quote, urlencode # %20, not +: the agent's URLComponents keeps +
HERE = Path(__file__).resolve().parent
ROOT = HERE.parent
@@ -48,16 +48,19 @@ QUALITY = {
"light": {"max": 1280, "fps": 30, "bpp": 0.08, "codec": "h264"},
"compatible": {"max": 1280, "fps": 20, "bpp": 0.1, "codec": "jpeg"},
}
# Extra Chromium flags for viewers (see docs/mac-in-headset.md, "Measuring").
BROWSER_FLAGS = []
# Viewer windows are fitted inside a panel's size. gamescope made a 1280x720
# request 1920x1080 anyway (verified 2026-09-28, build 20260925.6191901).
PANEL_BOX = (1920, 1080)
# Opens one viewer on gamescope's X display and gives its window its own panel
# id. Args: appid url width height tag. The page puts "[tag]" in its title at
# id. Args: appid url width height tag [browser flags...]. The page puts "[tag]" in its title at
# once, which is how its X window is found (Chromium may hand the URL to an
# instance that's already running, so there's no process to follow).
LAUNCH = r"""set -u
appid=$1 url=$2 w=$3 h=$4 tag=$5
shift 5
export DISPLAY=:0 LC_ALL=C.UTF-8
unset WAYLAND_DISPLAY
if ! xprop -root GAMESCOPE_FOCUSABLE_WINDOWS >/dev/null 2>&1; then
@@ -67,7 +70,7 @@ fi
common=(--ozone-platform=x11 --force-device-scale-factor=1 --no-first-run --no-default-browser-check
--password-store=basic --disable-session-crashed-bubble --noerrdialogs --disable-infobars
--disable-features=Translate,MediaRouter --autoplay-policy=no-user-gesture-required
"--window-size=$w,$h" "--app=$url")
"--window-size=$w,$h" "$@" "--app=$url")
if [ -x "$HOME/chromium-xr/chrome" ]; then
cmd=("$HOME/chromium-xr/chrome" "--user-data-dir=$HOME/.local/share/frame-control/mac-view" "${common[@]}")
elif flatpak info org.chromium.Chromium >/dev/null 2>&1; then
@@ -126,6 +129,7 @@ class MacView:
self.supervisor = None
self.closing = False
self.shown = set() # sources with a viewer out there, connected or retrying
self.browser_flags = list(BROWSER_FLAGS)
# ---- the agent on this Mac ----
@@ -187,7 +191,7 @@ class MacView:
def call(self, path, method="GET", **query):
"""A request to the agent; starts it if needed."""
self.ensure_agent()
url = f"http://127.0.0.1:{self.port}{path}?{urlencode({**query, 'k': self.token})}"
url = f"http://127.0.0.1:{self.port}{path}?{urlencode({**query, 'k': self.token}, quote_via=quote)}"
req = urllib.request.Request(url, method=method, data=b"" if method == "POST" else None)
try:
with urllib.request.urlopen(req, timeout=10) as r:
@@ -275,12 +279,15 @@ class MacView:
# ---- viewers on the Frame ----
def show(self, src, quality="balanced", width=None, height=None):
if src != "test" and not (src.startswith("window:") or src.startswith("display:")):
if src != "test" and not src.startswith(("window:", "display:", "separate:")):
raise MacViewError("Pick a window or display to show.")
q = QUALITY.get(quality) or QUALITY["balanced"]
state = self.call("/status")
if src != "test" and not state.get("screen"):
raise MacViewError("Frame Control needs Screen Recording permission first (Allow… under Mac in the headset).")
if src.startswith("separate:") and not state.get("accessibility"):
raise MacViewError("A window on its own display needs the Accessibility permission too, to move it "
"(Allow… under Mac in the headset).")
self.shown -= set(state.get("finished", [])) # their Mac windows closed
if src in self.shown or any(st.get("src") == src for st in state.get("streams", [])):
# Showing it again replaces the old viewer, connected or not: this
@@ -300,7 +307,7 @@ class MacView:
"bpp": q["bpp"]}
url = f"http://127.0.0.1:{self.remote_port}/view?{urlencode(params)}"
w, h = fit(width or 1280, height or 720)
args = " ".join(_quote(str(a)) for a in (appid, url, w, h, tag))
args = " ".join(_quote(str(a)) for a in (appid, url, w, h, tag, *self.browser_flags))
try:
out = self.run("bash -s -- " + args, stdin=LAUNCH, timeout=60)
except Exception as e: # noqa: BLE001 - the server's Failure carries the Frame's words
@@ -355,6 +362,12 @@ class MacView:
for proc in (self.tunnel, self.agent):
if proc and proc.poll() is None:
proc.terminate()
# The helper puts separated windows back before it exits (about 1 s).
if self.agent:
try:
self.agent.wait(3)
except subprocess.TimeoutExpired:
self.agent.kill()
def _quote(s):
ui/index.html
+32 -7
View File
@@ -314,6 +314,8 @@
.mv-perm { background: rgba(255,190,60,.08); border: 1px solid rgba(255,190,60,.25); border-radius: 4px; padding: 10px 12px; margin-bottom: 12px; font-size: 13px; }
.mv-perm .row { margin-top: 8px; }
.mv-live { color: var(--green, #7fd67f); font-size: 12px; }
.mv-opt { display: block; font-size: 13px; margin: 0 0 12px; }
.mv-opt .sub { display: block; margin-left: 22px; font-size: 12px; }
.disp select { width: 100%; background: rgba(0,0,0,.28); color: var(--text); border: 1px solid transparent; border-radius: 3px;
padding: 8px 10px; font: inherit; font-size: 13px; }
.disp select:focus { outline: none; border-color: var(--blue); }
@@ -655,6 +657,8 @@
<option value="light">Light</option><option value="compatible">Compatible (JPEG)</option>
</select>
<button class="small" id="mvRefresh">Refresh</button></div>
<label class="mv-opt"><input type="checkbox" id="mvSeparate" checked> Give each window its own display
<span class="sub">its menus and sheets come along, nothing covers it, and it goes back when you stop</span></label>
<div class="mv-perm" id="mvPerm" hidden></div>
<div class="list" id="mvDisplays"></div>
<div class="shelf-head" style="margin-top:16px"><h2>Windows</h2><span class="count" id="mvWinCount"></span></div>
@@ -2132,11 +2136,23 @@ $("shotsSaveNew").onclick = e => saveShots(shots.list.filter(s => !s.saved), e.c
$("shotsFolder").onclick = e => act($("shotsFolder").textContent, () => api("/api/open", { what: "shots" }), e.currentTarget);
// ---- Mac in the headset: windows and displays as panels (ui/frame_macview.py) ----
let mvSeq = 0;
function mvRow(src, title, sub, w, h, streaming) {
let mvSeq = 0, mvTimer = 0;
// streaming: the src it's shown as (a window may be "separate:<id>"), or "".
// How a live stream is doing: frames shown, Mac-to-headset delay, bitrate, and
// whether the network made it step down (fewer frames, then fewer pixels).
function mvHealth(st) {
if (!st) return "";
const x = st.stats || {}, c = st.controller || {}, bits = [];
if (x.fps) bits.push(`${Math.round(x.fps)} fps`);
if (x.total && x.total.p50 != null) bits.push(`${Math.round(x.total.p50)} ms`);
if (x.mbps) bits.push(`${x.mbps} Mbit/s`);
if (c.tier > 0) bits.push(`weak link: ${c.fps} fps${c.scale < 1 ? `, ${Math.round(c.scale * 100)}% size` : ""}`);
return bits.length ? " · " + esc(bits.join(" · ")) : "";
}
function mvRow(src, title, sub, w, h, streaming, st) {
return `<div class="item"><div class="grow"><div class="t">${esc(title)}</div>
<div class="s">${esc(sub)}${streaming ? ' · <span class="mv-live">in the headset</span>' : ""}</div></div>
${streaming ? `<button class="small" data-mv="stop" data-src="${esc(src)}">Stop</button>`
<div class="s">${esc(sub)}${streaming ? ' · <span class="mv-live">in the headset</span>' + mvHealth(st) : ""}</div></div>
${streaming ? `<button class="small" data-mv="stop" data-src="${esc(streaming)}">Stop</button>`
: `<button class="small action" data-mv="show" data-src="${esc(src)}" data-title="${esc(title)}" data-w="${w}" data-h="${h}">Show</button>`}
</div>`;
}
@@ -2150,6 +2166,7 @@ async function loadMacView(restart) {
if (!s.available) { box.hidden = true; return; }
box.hidden = false;
const live = new Set((s.streams || []).map(x => x.src));
const byStream = src => (s.streams || []).find(x => x.src === src);
$("mvCount").textContent = live.size ? `${live.size} showing` : "";
const perm = [];
if (!s.screen) perm.push(`Frame Control needs <b>Screen Recording</b> permission to show your windows.`);
@@ -2157,21 +2174,29 @@ async function loadMacView(restart) {
$("mvPerm").hidden = !perm.length;
$("mvPerm").innerHTML = perm.join(" ") + `<div class="row"><button class="small action" data-mv="perm">Allow…</button>
<span class="sub">Then press Refresh. macOS may ask you to reopen Frame Control.</span></div>`;
const liveAs = src => live.has(src) ? src : src.startsWith("window:") && live.has("separate:" + src.slice(7)) ? "separate:" + src.slice(7) : "";
$("mvDisplays").innerHTML = (s.displays || []).map(d =>
mvRow(d.src, `Whole screen: ${d.name}`, `${d.w}×${d.h}${d.main ? " · main display" : ""}`, d.w, d.h, live.has(d.src))).join("");
mvRow(d.src, `Whole screen: ${d.name}`, `${d.w}×${d.h}${d.main ? " · main display" : ""}`, d.w, d.h, liveAs(d.src), byStream(liveAs(d.src)))).join("");
const wins = s.windows || [];
$("mvWinCount").textContent = wins.length ? `${wins.length}` : "";
$("mvWindows").innerHTML = !s.screen ? `<div class="sub">Windows appear here once Screen Recording is allowed.</div>`
: wins.length ? wins.map(w => mvRow(w.src, w.title || w.app, `${w.title ? w.app + " · " : ""}${w.w}×${w.h}`, w.w, w.h, live.has(w.src))).join("")
: wins.length ? wins.map(w => mvRow(w.src, w.title || w.app, `${w.title ? w.app + " · " : ""}${w.w}×${w.h}`, w.w, w.h, liveAs(w.src), byStream(liveAs(w.src)))).join("")
: `<div class="sub">No windows open on this Mac's current desktop.</div>`;
// Keep the numbers fresh while something is in the headset and the card is on screen.
clearTimeout(mvTimer);
const tick = () => { mvTimer = document.hidden || !box.offsetParent ? setTimeout(tick, 3000) : (loadMacView(), 0); };
if (live.size) mvTimer = setTimeout(tick, 3000);
}
$("mvRefresh").onclick = () => loadMacView(true);
$("mvSeparate").checked = localStorage.getItem("mvSeparate") !== "0";
$("mvSeparate").onchange = e => localStorage.setItem("mvSeparate", e.target.checked ? "1" : "0");
$("macview").onclick = async e => {
const b = e.target.closest("[data-mv]"); if (!b) return;
const src = b.dataset.src;
if (b.dataset.mv === "show") {
const as = $("mvSeparate").checked && src.startsWith("window:") ? "separate:" + src.slice(7) : src;
await act(`Show ${b.dataset.title} in the headset`, async () => {
const r = await api("/api/macview", { action: "show", src, quality: $("mvQuality").value,
const r = await api("/api/macview", { action: "show", src: as, quality: $("mvQuality").value,
w: +b.dataset.w || undefined, h: +b.dataset.h || undefined });
return { message: `${b.dataset.title} is a panel in the headset now. Float it with the SteamVR dashboard.`, ...r };
}, b);
ui/mac-view.html
+182 -26
View File
@@ -16,11 +16,14 @@
font: 15px/1.4 system-ui, sans-serif; color: #eee; background: rgba(20,22,26,.88);
padding: 8px 14px; border-radius: 10px; pointer-events: none; transition: opacity .4s; }
#note[hidden] { display: block; opacity: 0; }
#hud { position: fixed; left: 8px; top: 8px; font: 12px/1.35 ui-monospace, monospace; color: #dfe;
background: rgba(0,0,0,.72); padding: 6px 9px; border-radius: 6px; pointer-events: none; white-space: pre; }
</style>
</head>
<body>
<canvas id="c" width="16" height="9"></canvas>
<div id="note">Connecting to the Mac…</div>
<div id="hud" hidden></div>
<script>
"use strict";
const q = new URLSearchParams(location.search);
@@ -32,6 +35,85 @@ const c = document.getElementById("c"), ctx = c.getContext("2d", { alpha: false,
const note = document.getElementById("note");
// Counters for Frame Control's tests (read over DevTools).
const stats = window.stats = { frames: 0, keyFrames: 0, bytes: 0, dropped: 0, codec: "", errors: [], info: null };
// ---- timing: the Mac measures each frame's journey on its own clock ----
// Clock sync, NTP-style: the Mac's time minus ours, from the ping with the
// shortest round trip lately (the least queueing, so the least error).
const clock = { off: null, rtt: 0, samples: [], reported: 0 };
function toMac(ms) { return clock.off === null ? null : Math.round(ms * 1000 + clock.off); }
function onPong(m) {
const now = performance.now(), rtt = now - m.c;
clock.samples.push({ rtt, off: m.a - (m.c + now) / 2 * 1000 });
if (clock.samples.length > 16) clock.samples.shift();
const best = clock.samples.reduce((a, b) => (b.rtt < a.rtt ? b : a));
clock.off = best.off;
clock.rtt = best.rtt;
if (now - clock.reported > 1000) {
clock.reported = now;
send({ t: "clock", rtt: +best.rtt.toFixed(2), dec: `${decoderInfo}; ${glInfo}; raf ${rafHz} Hz` });
}
}
let pingTimer = 0;
function startPings() {
clearInterval(pingTimer);
clock.samples = [];
for (let i = 0; i < 10; i++) setTimeout(() => send({ t: "ping", c: performance.now() }), i * 40);
pingTimer = setInterval(() => send({ t: "ping", c: performance.now() }), 1000);
}
// Decoded, drawn and next-animation-frame times go back in batches.
let reports = [], dropsToReport = 0, shownPending = null, rafQueued = false;
setInterval(() => {
if (!reports.length && !dropsToReport) return;
send({ t: "fd", f: reports, drop: dropsToReport });
reports = [];
dropsToReport = 0;
}, 250);
function dropped() { stats.dropped++; dropsToReport++; }
// A drawn frame is on screen from the next animation frame, unless another
// replaces it first (then it was never seen).
function shown(seq, decodedAt, drawnAt) {
if (shownPending) reports.push([shownPending.seq, toMac(shownPending.dec), toMac(shownPending.drawn), 0]);
shownPending = { seq, dec: decodedAt, drawn: drawnAt };
if (rafQueued) return;
rafQueued = true;
requestAnimationFrame(() => {
rafQueued = false;
const p = shownPending, now = performance.now();
shownPending = null;
if (p && clock.off !== null) reports.push([p.seq, toMac(p.dec), toMac(p.drawn), toMac(now)]);
});
}
let decoderInfo = "";
// What this browser draws with, and how often it gives an animation frame
// when nothing else is going on (both only for the numbers).
let glInfo = "", rafHz = 0;
try {
const gl = document.createElement("canvas").getContext("webgl");
const ext = gl && gl.getExtension("WEBGL_debug_renderer_info");
glInfo = ext ? gl.getParameter(ext.UNMASKED_RENDERER_WEBGL) : gl ? "webgl" : "no webgl";
} catch (e) { glInfo = "?"; }
(function measureRaf() {
let n = 0, t0 = 0;
const tick = t => {
if (!t0) t0 = t;
if (t - t0 < 1000) { n++; return requestAnimationFrame(tick); }
rafHz = Math.round(n * 1000 / (t - t0));
};
requestAnimationFrame(tick);
})();
const hud = document.getElementById("hud");
function showHud(on) { hud.hidden = !on; }
function ms(o) { return o && o.p50 !== undefined ? `${o.p50}/${o.p95}` : "–"; }
function onStats(m) {
if (hud.hidden) return;
hud.textContent = `${m.size} ${m.fps} fps shown (${m.sentFps} sent) ${m.mbps} Mbit/s` +
(m.bitrate ? ` of ${(m.bitrate / 1e6).toFixed(1)}` : "") + (m.tier !== undefined ? ` tier ${m.tier}` : "") +
`\nlatency p50/p95 ms total ${ms(m.total)}\n capture ${ms(m.capture)} queue ${ms(m.queue)} encode ${ms(m.encode)}` +
`\n network ${ms(m.network)} decode ${ms(m.decode)} draw ${ms(m.draw)}` +
`\ninput→shown p50 ${m.input ? m.input.p50.toFixed(0) + " ms" : "–"} rtt ${m.rtt} ms` +
`\nskipped ${m.skipped}/${m.captured} dropped ${m.dropped} ${decoderInfo}`;
}
showHud(q.get("stats") === "1");
// Frame Control finds this window on the Frame by the tag in its title.
document.title = tag ? `Mac [${tag}]` : "Mac";
let failedStarts = 0;
@@ -54,9 +136,10 @@ function askKeyFrame() {
}
let hideNoteOnFrame = true; // "Connecting…"/"Reconnecting…" go once video flows again
function drawFrame(img, w, h) {
function drawFrame(img, w, h, seq, decodedAt) {
if (c.width !== w || c.height !== h) { c.width = w; c.height = h; }
ctx.drawImage(img, 0, 0);
shown(seq, decodedAt, performance.now());
stats.frames++;
if (hideNoteOnFrame) { hideNoteOnFrame = false; note.hidden = true; }
}
@@ -76,7 +159,8 @@ function makeDecoder() {
if (dec) try { dec.close(); } catch (e) {}
codecString = "";
dec = new VideoDecoder({
output: frame => { drawFrame(frame, frame.displayWidth, frame.displayHeight); frame.close(); },
// The chunk's timestamp is the Mac's sequence number, to match reports up.
output: frame => { drawFrame(frame, frame.displayWidth, frame.displayHeight, frame.timestamp, performance.now()); frame.close(); },
error: e => {
stats.errors.push(String(e.message || e));
needKey = true;
@@ -85,7 +169,7 @@ function makeDecoder() {
},
});
}
function onH264(isKey, ts, au) {
function onH264(isKey, seq, au) {
if (isKey) {
const cs = spsCodec(au);
if (cs && (cs !== codecString || dec.state !== "configured")) {
@@ -95,20 +179,22 @@ function onH264(isKey, ts, au) {
}
stats.keyFrames++;
}
if (dec.state !== "configured" || (needKey && !isKey)) { stats.dropped++; askKeyFrame(); return; }
// Behind: skip to the next keyframe rather than show old pictures late.
if (!isKey && dec.decodeQueueSize > 2) { needKey = true; stats.dropped++; askKeyFrame(); return; }
if (dec.state !== "configured" || (needKey && !isKey)) { dropped(); askKeyFrame(); return; }
// Far behind: skip to the next keyframe rather than show old pictures late.
// A few queued frames are fine: decoding catches up faster than a keyframe
// crosses a slow link, and only the newest decoded frame gets drawn.
if (!isKey && dec.decodeQueueSize > 10) { needKey = true; dropped(); askKeyFrame(); return; }
needKey = false;
dec.decode(new EncodedVideoChunk({ type: isKey ? "key" : "delta", timestamp: ts, data: au }));
dec.decode(new EncodedVideoChunk({ type: isKey ? "key" : "delta", timestamp: seq, data: au }));
}
// ---- JPEG ----
let jpegBusy = false;
function onJPEG(data) {
if (jpegBusy) { stats.dropped++; return; }
function onJPEG(seq, data) {
if (jpegBusy) { dropped(); return; }
jpegBusy = true;
createImageBitmap(new Blob([data], { type: "image/jpeg" })).then(bmp => {
drawFrame(bmp, bmp.width, bmp.height);
drawFrame(bmp, bmp.width, bmp.height, seq, performance.now());
bmp.close();
}).catch(e => stats.errors.push(String(e))).finally(() => { jpegBusy = false; });
}
@@ -119,7 +205,12 @@ async function pickCodec() {
if (!("VideoDecoder" in window)) return "jpeg";
try {
const r = await VideoDecoder.isConfigSupported({ codec: "avc1.640028", optimizeForLatency: true });
return r.supported ? "h264" : "jpeg";
if (!r.supported) return "jpeg";
// Whether this browser has a hardware H.264 decoder (for the numbers).
const hw = await VideoDecoder.isConfigSupported({ codec: "avc1.640028", hardwareAcceleration: "prefer-hardware" })
.catch(() => ({ supported: false }));
decoderInfo = hw.supported ? "h264 hardware" : "h264 software";
return "h264";
} catch (e) { return "jpeg"; }
}
@@ -133,19 +224,21 @@ async function connect() {
if (q.get("bpp")) p.set("bpp", q.get("bpp"));
ws = new WebSocket(`ws://${location.host}/stream?${p}`);
ws.binaryType = "arraybuffer";
ws.onopen = () => { retry = 0; lastError = ""; };
ws.onopen = () => { retry = 0; lastError = ""; startPings(); };
ws.onmessage = ev => {
if (typeof ev.data === "string") return control(JSON.parse(ev.data));
const b = new Uint8Array(ev.data);
const now = performance.now(), b = new Uint8Array(ev.data);
stats.bytes += b.length;
if (b.length < 10) return;
const isKey = b[0] === 1;
const ts = Number(new DataView(ev.data).getBigUint64(1));
const payload = b.subarray(9);
if (codec === "h264") onH264(isKey, ts, payload); else onJPEG(payload);
if (b.length < 18) return;
// flags (1 = keyframe), pts µs, sequence number, input echoed; big-endian.
const v = new DataView(ev.data), isKey = (b[0] & 1) === 1, seq = v.getUint32(9);
send({ t: "rx", s: seq, r: toMac(now) });
const payload = b.subarray(17);
if (codec === "h264") onH264(isKey, seq, payload); else onJPEG(seq, payload);
};
ws.onclose = () => {
ws = null;
clearInterval(pingTimer);
if (closing) return;
// Refused before ever getting a key: the ticket expired or was revoked,
// and retrying can't help.
@@ -163,11 +256,15 @@ async function connect() {
}
function control(m) {
if (m.t === "pong") return onPong(m);
if (m.t === "stats") return onStats(m);
if (m.t === "bench") return bench(m);
if (m.t === "hello") {
reconnectKey = m.r;
send({ t: "ack" }); // the ticket is spent only now
} else if (m.t === "info") {
stats.info = m;
warmMs = Math.max(0, +m.warm || 0);
const name = m.title && m.app && m.title !== m.app ? `${m.title} — ${m.app}` : (m.title || m.app || "Mac");
document.title = tag ? `${name} [${tag}]` : name;
if (!m.input) {
@@ -194,22 +291,51 @@ function toPicture(e) {
const x = (e.clientX - left) / w, y = (e.clientY - top) / h;
return { x, y, inside: x >= 0 && x <= 1 && y >= 0 && y <= 1 };
}
let pendingMove = null, moveQueued = false;
// Discrete input carries an id and when it happened (the Mac's clock), so
// the Mac can tag the first frame that could show its effect.
let inputId = 0;
// Keep the Frame's Wi-Fi awake for a few seconds after input, when the agent asks.
let warmMs = 0, warmUntil = 0, warmTimer = 0;
function keepWarm() {
if (!warmMs) return;
warmUntil = performance.now() + 5000;
if (warmTimer) return;
warmTimer = setInterval(() => {
if (performance.now() > warmUntil) { clearInterval(warmTimer); warmTimer = 0; return; }
send({ t: "w" });
}, warmMs);
}
function stamp(m, e) {
keepWarm();
m.i = ++inputId;
m.tv = toMac(e && e.timeStamp ? e.timeStamp : performance.now());
return m;
}
// Moves go at most every 8 ms, on a timer rather than the next animation frame:
// the Frame throttles a panel's animation frames to 15-36 Hz when it thinks
// nobody is looking, which would hold a move back by up to 60 ms.
let pendingMove = null, moveQueued = false, lastMove = 0;
function flushMove() {
moveQueued = false;
if (pendingMove) { send(pendingMove); lastMove = performance.now(); }
pendingMove = null;
}
addEventListener("pointermove", e => {
const p = toPicture(e);
if (!p.inside && !e.buttons) return;
pendingMove = { t: "m", e: "move", x: p.x, y: p.y };
if (!moveQueued) {
moveQueued = true;
requestAnimationFrame(() => { moveQueued = false; if (pendingMove) send(pendingMove); pendingMove = null; });
}
if (moveQueued) return;
const wait = lastMove + 8 - performance.now();
if (wait <= 0) return flushMove();
moveQueued = true;
setTimeout(flushMove, wait);
});
addEventListener("pointerdown", e => {
const p = toPicture(e);
if (!p.inside) return;
if (e.pointerId !== undefined) try { c.setPointerCapture(e.pointerId); } catch (err) {}
pendingMove = null;
send({ t: "m", e: "down", b: e.button < 0 ? 0 : e.button, x: p.x, y: p.y });
send(stamp({ t: "m", e: "down", b: e.button < 0 ? 0 : e.button, x: p.x, y: p.y }, e));
e.preventDefault();
});
addEventListener("pointerup", e => {
@@ -221,16 +347,46 @@ addEventListener("pointercancel", () => send({ t: "release" }));
addEventListener("contextmenu", e => e.preventDefault());
addEventListener("wheel", e => {
const p = toPicture(e), unit = e.deltaMode === 1 ? 16 : e.deltaMode === 2 ? innerHeight : 1;
send({ t: "wheel", dx: e.deltaX * unit, dy: e.deltaY * unit, x: p.x, y: p.y });
send(stamp({ t: "wheel", dx: e.deltaX * unit, dy: e.deltaY * unit, x: p.x, y: p.y }, e));
e.preventDefault();
}, { passive: false });
function mods(e) {
return ["shift", "ctrl", "alt", "meta"].filter(m => e[m + "Key"]);
}
function onKey(e, down) {
send({ t: "k", e: down ? "down" : "up", code: e.code, key: e.key, mods: mods(e) });
// Ctrl+Alt+Shift+S shows the numbers; it isn't sent to the Mac.
if (e.code === "KeyS" && e.ctrlKey && e.altKey && e.shiftKey) {
if (down) showHud(hud.hidden);
return e.preventDefault();
}
const m = { t: "k", e: down ? "down" : "up", code: e.code, key: e.key, mods: mods(e) };
send(down ? stamp(m, e) : m);
e.preventDefault();
}
// ---- benchmarks: the Mac asks the viewer to act as if someone did ----
function keyCode(ch) {
if (/[a-z]/i.test(ch)) return "Key" + ch.toUpperCase();
if (/[0-9]/.test(ch)) return "Digit" + ch;
return { " ": "Space", "\n": "Enter", ".": "Period", ",": "Comma" }[ch] || "";
}
function bench(m) {
if (m.action === "overlay") return showHud(m.on !== 0 && m.on !== "0");
if (m.action === "click") {
const at = { x: +m.x || 0.5, y: +m.y || 0.5 };
send(stamp({ t: "m", e: "down", b: 0, ...at }));
setTimeout(() => send({ t: "m", e: "up", b: 0, ...at }), 40);
} else if (m.action === "type") {
const text = String(m.text || ""), gap = +m.interval || 150;
[...text].forEach((ch, n) => setTimeout(() => {
const code = keyCode(ch);
if (!code) return send(stamp({ t: "text", s: ch }));
const mods = ch !== ch.toLowerCase() ? ["shift"] : [];
send(stamp({ t: "k", e: "down", code, key: ch, mods }));
setTimeout(() => send({ t: "k", e: "up", code, key: ch, mods }), 30);
}, n * gap));
}
}
addEventListener("keydown", e => onKey(e, true));
addEventListener("keyup", e => onKey(e, false));
addEventListener("blur", () => send({ t: "release" }));