steam-frame/saphid--frame-control
1
0 Fork 0
mirror of https://github.com/saphid/frame-control.git synced 2026-10-06 01:00:18 +02:00
Code Issues Packages Projects Releases 6 Wiki Activity

Merge remote-tracking branch 'origin/main' into theatre-media

Browse Source 
# Conflicts:
#	ui/server.py
This commit is contained in:
saphid committed 2026-09-29 10:21:27 +10:00
2 parents b00db2484d 363497fdd5
commit ae7f733b90
72 files changed
+47716 -18
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

.claude/skills/steam-frame/SKILL.md
+6
View File
@@ -48,3 +48,9 @@ running `--help`: `paste-to-frame.sh`, `serve-bootstrap.sh` and
- `sudo` on the Frame asks for the user's Developer Mode password. Hand those
steps to the user (open Terminal) and keep automation to non-sudo commands.
- The Mac uses BSD userland and zsh (no `timeout`, use `head -n`).
- **First-party first.** For any new capability, investigate the first-party
way before anything else: Valve (SteamOS, Steam, Steam Link), Apple (the Mac
and iPhone), and KDE (the Frame's desktop is Plasma). It's usually the best
answer. If it isn't, write down why not. If it is, find what Frame Control
can do to make it easier to set up (install over SSH, pre-seed settings,
pair automatically, tell the user the one setting to turn on).
.gitignore
+1
View File
@@ -5,3 +5,4 @@ apk-catalog/data/index-v2*.json*
compat-db/.env.lakebed.server
compat-db/.lakebed/
tests/smoke/results/
mac/bin/
README.md
+1
View File
@@ -207,6 +207,7 @@ Frame's software fits together, all checked against a real headset and labelled
| [Sideloading Linux and Windows games](docs/sideloading.md) | A .zip, folder or .exe as a Steam Devkit Game, runtime detection |
| [Install links for websites](docs/web-install.md) | `frame-control://install` links and manifests, the rules, a button to paste |
| [Steam games](docs/steam-games.md) · [VR video](docs/vr-video.md) · [WebXR in Chromium](docs/webxr-chromium.md) | Installing and buying, watching VR180/360, the Chromium build |
| [Mac in the headset](docs/mac-in-headset.md) | Mac windows and screens as panels in the Frame, with laser and keyboard input |
| [VR mods and custom songs](docs/mods.md) | Per-game feasibility, real-Frame results and blockers; no installer yet |
| [SSH](docs/ssh.md) · [Streaming](docs/streaming.md) · [Files](docs/file-transfer.md) · [Panels](docs/panels.md) · [Tailscale](docs/tailscale.md) | Topic notes |
| [Frame Control for iPhone](docs/iphone.md) | The iPhone and iPad app, how it runs the server on the Frame, pairing |
app/package.json
+21 -3
View File
@@ -9,8 +9,8 @@
"scripts": {
"start": "env -u ELECTRON_RUN_AS_NODE electron .",
"icon": "env -u ELECTRON_RUN_AS_NODE electron build/make-icon.js",
"dist": "node build/fetch-deps.js mac arm64 && electron-builder --mac --arm64 --publish never",
"dist:dir": "node build/fetch-deps.js mac arm64 && electron-builder --mac --arm64 --dir",
"dist": "sh ../mac/frame-mac-view/build.sh && node build/fetch-deps.js mac arm64 && electron-builder --mac --arm64 --publish never",
"dist:dir": "sh ../mac/frame-mac-view/build.sh && node build/fetch-deps.js mac arm64 && electron-builder --mac --arm64 --dir",
"dist:linux": "node build/fetch-deps.js linux x64 arm64 && electron-builder --linux --x64 --arm64 --publish never",
"dist:win": "node build/fetch-deps.js win x64 && electron-builder --win --x64 --publish never"
},
@@ -66,6 +66,14 @@
"*.py"
]
},
{
"from": "../frame/openxr-compat",
"to": "frame/openxr-compat",
"filter": [
"XrApiLayer_FRAME_compat.json",
"prebuilt/**/*.so"
]
},
{
"from": "../frame/devkit-utils",
"to": "frame/devkit-utils",
@@ -107,9 +115,19 @@
"dmg",
"zip"
],
"extraResources": [
{
"from": "../mac/bin",
"to": "mac/bin",
"filter": [
"frame-mac-view"
]
}
],
"extendInfo": {
"NSAppleEventsUsageDescription": "Frame Control opens Terminal for SSH sessions and for power actions that need the Developer Mode password.",
"NSLocalNetworkUsageDescription": "Frame Control connects to your Steam Frame over SSH on the local network."
"NSLocalNetworkUsageDescription": "Frame Control connects to your Steam Frame over SSH on the local network.",
"NSScreenCaptureUsageDescription": "Frame Control shows your Mac's windows and screens inside the Steam Frame when you ask it to."
},
"artifactName": "Frame-Control-mac-${arch}.${ext}"
},
bench/pages/present.html
+24
View File
@@ -0,0 +1,24 @@
<!doctype html>
<!-- Control experiment, run on the Frame itself: how often does Chromium on
gamescope put a canvas animation on screen, with no network or decoding
involved? Draws every animation frame for ?s= seconds and writes the
frame-gap histogram per second into the title, where xprop can read it. -->
<html><head><meta charset="utf-8"><title>fmv-present</title></head>
<body style="margin:0;background:#000"><canvas id="c" width="1280" height="720" style="width:100%;height:100%"></canvas>
<script>
const q = new URLSearchParams(location.search), secs = +q.get("s") || 15, tag = q.get("tag") || "";
const ctx = document.getElementById("c").getContext("2d", { alpha: false, desynchronized: true });
const perSec = []; let t0 = 0, last = 0, n = 0, gaps = [];
function frame(t) {
if (!t0) t0 = last = t;
ctx.fillStyle = "#123"; ctx.fillRect(0, 0, 1280, 720);
ctx.fillStyle = "#fff"; ctx.fillRect((n * 21) % 1280, 0, 20, 720); n++;
const s = Math.floor((t - t0) / 1000);
(perSec[s] = perSec[s] || []).push(t - last); last = t;
if (t - t0 < secs * 1000) return requestAnimationFrame(frame);
const out = perSec.map(g => g.length).join(",");
document.title = `[${tag}] done fps/s=${out}`;
}
document.title = `[${tag}] running`;
requestAnimationFrame(frame);
</script></body></html>
bench/pages/scroll.html
+32
View File
@@ -0,0 +1,32 @@
<!doctype html>
<!-- Benchmark content: a long page of text that scrolls itself at a steady
speed, so every frame changes the way reading a real page does. -->
<html lang="en"><head><meta charset="utf-8"><title>fmv-bench scroll</title>
<style>
body { font: 17px/1.55 -apple-system, "Helvetica Neue", sans-serif; margin: 0 auto; max-width: 900px; padding: 24px; color: #1d1d1f; background: #fff; }
h2 { font-size: 22px; margin: 28px 0 8px; } code { background: #f2f2f5; padding: 1px 4px; border-radius: 3px; }
</style></head><body><div id="doc"></div>
<script>
const words = "latency frame panel stream encoder decoder network display pixel window capture budget quality motion text sharp adaptive controller queue socket clock".split(" ");
let seed = 7; const rnd = () => (seed = (seed * 16807) % 2147483647) / 2147483647;
let html = "";
for (let s = 0; s < 120; s++) {
html += `<h2>Section ${s + 1}: ${words[s % words.length]} and ${words[(s * 7) % words.length]}</h2>`;
for (let p = 0; p < 4; p++) {
let para = [];
for (let w = 0; w < 70; w++) para.push(rnd() < 0.05 ? `<code>${words[Math.floor(rnd() * words.length)]}()</code>` : words[Math.floor(rnd() * words.length)]);
html += `<p>${para.join(" ")}.</p>`;
}
}
document.getElementById("doc").innerHTML = html;
// 240 points a second, whatever the display's refresh rate.
// The title carries the page's own frame rate, so the source's rate is known.
let last = performance.now(), frames = 0, since = last;
function step(now) {
const dy = (now - last) * 0.24; last = now;
if (++frames && now - since >= 1000) { document.title = `fmv-bench scroll ${frames} fps`; frames = 0; since = now; }
if (window.scrollY + innerHeight >= document.body.scrollHeight - 2) window.scrollTo(0, 0); else window.scrollBy(0, dy);
requestAnimationFrame(step);
}
requestAnimationFrame(step);
</script></body></html>
bench/pages/type.html
+9
View File
@@ -0,0 +1,9 @@
<!doctype html>
<!-- Benchmark content: a plain text editor, focused, for typing tests. -->
<html lang="en"><head><meta charset="utf-8"><title>fmv-bench type</title>
<style>
html, body { margin: 0; height: 100%; background: #fff; }
textarea { box-sizing: border-box; width: 100%; height: 100%; border: 0; padding: 20px; outline: none; resize: none;
font: 20px/1.5 ui-monospace, Menlo, monospace; color: #111; caret-color: transparent; } /* a blinking caret would pass for a reply to a key */
</style></head><body><textarea id="t" autofocus spellcheck="false"></textarea>
<script>document.getElementById("t").focus();</script></body></html>
bench/results/2026-09-28-1d05f57-usb1.json
+677
View File
@@ -0,0 +1,677 @@
{
"label": "usb1",
"date": "2026-09-28T21:16:03",
"commit": "1d05f57",
"config": {
"quality": "balanced",
"mode": "separate",
"net": "none",
"delay_ms": 0,
"buffer_ms": 250,
"host": "10.86.200.233",
"ssh_opts": [],
"encoder": "",
"duration_s": 15.0,
"browser_flags": []
},
"frame_build": "20260925.6191901",
"mac": "26.5.2",
"headset": "",
"scenarios": [
{
"scenario": "test",
"frames_sent": 911,
"frames_drawn": 911,
"frames_shown": 610,
"duration_s": 15.2,
"stages_ms": {
"capture": {
"p50": 0.0,
"p95": 0.0,
"p99": 0.0,
"n": 911
},
"queue": {
"p50": 0.0,
"p95": 0.1,
"p99": 0.1,
"n": 911
},
"encode": {
"p50": 4.4,
"p95": 5.0,
"p99": 5.2,
"n": 911
},
"socket": {
"p50": 0.1,
"p95": 0.2,
"p99": 0.2,
"n": 911
},
"network": {
"p50": 1.1,
"p95": 1.6,
"p99": 2.1,
"n": 911
},
"decode": {
"p50": 1.2,
"p95": 2.6,
"p99": 3.9,
"n": 911
},
"draw": {
"p50": 0.4,
"p95": 0.9,
"p99": 1.5,
"n": 911
},
"present": {
"p50": 5.8,
"p95": 13.6,
"p99": 16.7,
"n": 610
},
"content": {
"p50": 7.3,
"p95": 8.8,
"p99": 10.0,
"n": 911
},
"content_shown": {
"p50": 13.6,
"p95": 20.4,
"p99": 24.9,
"n": 610
}
},
"fps": 59.9,
"fps_shown": 40.1,
"late_pct": 0.22,
"stall_max": 32.3,
"stalls_over_100ms": 0,
"mbps": 0.48,
"keyframes": 1,
"size": "1280x720",
"captured": 912,
"viewer_never_drawn": 0,
"input_ms": {
"p50": 16.6,
"p95": 24.3,
"p99": 24.5,
"n": 30
},
"input_shown_ms": {
"p50": 20.8,
"p95": 31.8,
"p99": 33.8,
"n": 22
},
"input_parts_ms": {
"uplink": {
"p50": 0.7,
"p95": 1.0,
"p99": 1.0,
"n": 30
},
"mac": {
"p50": 9.1,
"p95": 15.0,
"p99": 15.2,
"n": 30
},
"back": {
"p50": 7.2,
"p95": 8.5,
"p99": 12.7,
"n": 30
}
},
"inputs": {
"asked": 30,
"sent": 30,
"seen": 30
},
"timeline": [
{
"t": 0,
"fps": 60,
"mbps": 0.5,
"content_p50": 7.3,
"content_p95": 8.6,
"bitrate": 5529600,
"tier": 0,
"w": 1280,
"net": null
},
{
"t": 1,
"fps": 60,
"mbps": 0.5,
"content_p50": 7.1,
"content_p95": 8.9,
"bitrate": 5529600,
"tier": 0,
"w": 1280,
"net": null
},
{
"t": 2,
"fps": 59,
"mbps": 0.48,
"content_p50": 7.2,
"content_p95": 8.6,
"bitrate": 5529600,
"tier": 0,
"w": 1280,
"net": null
},
{
"t": 3,
"fps": 60,
"mbps": 0.49,
"content_p50": 7.1,
"content_p95": 9.0,
"bitrate": 5529600,
"tier": 0,
"w": 1280,
"net": null
},
{
"t": 4,
"fps": 60,
"mbps": 0.49,
"content_p50": 6.9,
"content_p95": 8.2,
"bitrate": 5529600,
"tier": 0,
"w": 1280,
"net": null
},
{
"t": 5,
"fps": 60,
"mbps": 0.58,
"content_p50": 7.3,
"content_p95": 8.1,
"bitrate": 5529600,
"tier": 0,
"w": 1280,
"net": null
},
{
"t": 6,
"fps": 60,
"mbps": 0.47,
"content_p50": 7.4,
"content_p95": 8.6,
"bitrate": 5529600,
"tier": 0,
"w": 1280,
"net": null
},
{
"t": 7,
"fps": 60,
"mbps": 0.46,
"content_p50": 7.4,
"content_p95": 8.8,
"bitrate": 5529600,
"tier": 0,
"w": 1280,
"net": null
},
{
"t": 8,
"fps": 60,
"mbps": 0.47,
"content_p50": 7.3,
"content_p95": 8.9,
"bitrate": 5529600,
"tier": 0,
"w": 1280,
"net": null
},
{
"t": 9,
"fps": 60,
"mbps": 0.46,
"content_p50": 7.3,
"content_p95": 8.8,
"bitrate": 5529600,
"tier": 0,
"w": 1280,
"net": null
},
{
"t": 10,
"fps": 60,
"mbps": 0.45,
"content_p50": 7.3,
"content_p95": 8.7,
"bitrate": 5529600,
"tier": 0,
"w": 1280,
"net": null
},
{
"t": 11,
"fps": 60,
"mbps": 0.45,
"content_p50": 7.4,
"content_p95": 9.2,
"bitrate": 5529600,
"tier": 0,
"w": 1280,
"net": null
},
{
"t": 12,
"fps": 60,
"mbps": 0.46,
"content_p50": 7.4,
"content_p95": 8.4,
"bitrate": 5529600,
"tier": 0,
"w": 1280,
"net": null
},
{
"t": 13,
"fps": 60,
"mbps": 0.46,
"content_p50": 7.6,
"content_p95": 9.7,
"bitrate": 5529600,
"tier": 0,
"w": 1280,
"net": null
},
{
"t": 14,
"fps": 60,
"mbps": 0.46,
"content_p50": 7.6,
"content_p95": 8.5,
"bitrate": 5529600,
"tier": 0,
"w": 1280,
"net": null
},
{
"t": 15,
"fps": 12,
"mbps": 0.09,
"content_p50": 7.5,
"content_p95": 9.6,
"bitrate": 5529600,
"tier": 0,
"w": 1280,
"net": null
}
],
"adapt": [],
"content_p50": 7.3,
"content_p95": 8.8,
"input_p50": 16.6,
"input_p95": 24.3,
"grades": {
"input_replies": "local",
"content_p50": "local",
"content_p95": "local",
"input_p50": "local",
"input_p95": "local",
"fps": "local",
"late_pct": "local",
"stall_max": "local"
},
"controller": {
"adapt": true,
"baseRtt": 0.917,
"ceiling": 5529600,
"fps": 60,
"inFlight": 1,
"scale": 1,
"slack": 41.666,
"target": 5529600,
"tier": 0
},
"events": [],
"source_fps": 60.0,
"cpu": {
"mac_agent_pct": 16.7,
"frame_viewer_pct": 48.5,
"frame_tailscaled_pct": 0.5,
"frame_sshd_pct": 1.1,
"frame_gamescope_pct": 12.0,
"frame_total_pct": 39.3
},
"viewer": "h264 software; ANGLE (Mesa, zink Vulkan 1.4(Turnip Adreno (TM) 750 (MESA_TURNIP)), OpenGL 4.6); raf 137 Hz",
"show_s": 1.15,
"panel": "valve.steam.desktopgame.2001639889",
"src": "test"
},
{
"scenario": "scroll",
"frames_sent": 848,
"frames_drawn": 848,
"frames_shown": 617,
"duration_s": 15.1,
"stages_ms": {
"capture": {
"p50": -3.9,
"p95": 4.2,
"p99": 7.3,
"n": 848
},
"queue": {
"p50": 0.0,
"p95": 0.1,
"p99": 0.1,
"n": 848
},
"encode": {
"p50": 6.7,
"p95": 9.7,
"p99": 17.7,
"n": 848
},
"socket": {
"p50": 0.1,
"p95": 0.2,
"p99": 0.4,
"n": 848
},
"network": {
"p50": 1.7,
"p95": 2.7,
"p99": 5.7,
"n": 848
},
"decode": {
"p50": 5.9,
"p95": 10.3,
"p99": 13.1,
"n": 848
},
"draw": {
"p50": 0.7,
"p95": 1.6,
"p99": 2.2,
"n": 848
},
"present": {
"p50": 4.4,
"p95": 15.8,
"p99": 20.6,
"n": 617
},
"content": {
"p50": 15.7,
"p95": 23.0,
"p99": 39.3,
"n": 848
},
"content_shown": {
"p50": 21.3,
"p95": 34.1,
"p99": 45.6,
"n": 617
}
},
"fps": 56.1,
"fps_shown": 40.7,
"late_pct": 3.3,
"stall_max": 238.5,
"stalls_over_100ms": 1,
"mbps": 9.01,
"keyframes": 1,
"size": "1920x1290",
"captured": 857,
"viewer_never_drawn": 0,
"input_ms": {
"n": 0
},
"input_shown_ms": {
"n": 0
},
"input_parts_ms": {
"uplink": {
"n": 0
},
"mac": {
"n": 0
},
"back": {
"n": 0
}
},
"inputs": {
"asked": 0,
"sent": 0,
"seen": 0
},
"timeline": [
{
"t": 0,
"fps": 44,
"mbps": 8.13,
"content_p50": 15.7,
"content_p95": 35.7,
"bitrate": 14860800,
"tier": 0,
"w": 1920,
"net": null
},
{
"t": 1,
"fps": 56,
"mbps": 7.04,
"content_p50": 16.3,
"content_p95": 47.3,
"bitrate": 7430400,
"tier": 0,
"w": 1920,
"net": null
},
{
"t": 2,
"fps": 57,
"mbps": 8.58,
"content_p50": 16.0,
"content_p95": 22.6,
"bitrate": 10055362,
"tier": 0,
"w": 1920,
"net": null
},
{
"t": 3,
"fps": 56,
"mbps": 9.29,
"content_p50": 16.4,
"content_p95": 21.1,
"bitrate": 14860800,
"tier": 0,
"w": 1920,
"net": null
},
{
"t": 4,
"fps": 58,
"mbps": 9.01,
"content_p50": 15.6,
"content_p95": 21.7,
"bitrate": 14860800,
"tier": 0,
"w": 1920,
"net": null
},
{
"t": 5,
"fps": 57,
"mbps": 9.13,
"content_p50": 15.9,
"content_p95": 21.7,
"bitrate": 14860800,
"tier": 0,
"w": 1920,
"net": null
},
{
"t": 6,
"fps": 56,
"mbps": 11.63,
"content_p50": 16.1,
"content_p95": 22.1,
"bitrate": 14860800,
"tier": 0,
"w": 1920,
"net": null
},
{
"t": 7,
"fps": 57,
"mbps": 8.7,
"content_p50": 16.3,
"content_p95": 22.2,
"bitrate": 14860800,
"tier": 0,
"w": 1920,
"net": null
},
{
"t": 8,
"fps": 57,
"mbps": 9.24,
"content_p50": 15.7,
"content_p95": 21.2,
"bitrate": 14860800,
"tier": 0,
"w": 1920,
"net": null
},
{
"t": 9,
"fps": 57,
"mbps": 9.21,
"content_p50": 15.0,
"content_p95": 20.5,
"bitrate": 14860800,
"tier": 0,
"w": 1920,
"net": null
},
{
"t": 10,
"fps": 56,
"mbps": 8.55,
"content_p50": 15.7,
"content_p95": 20.6,
"bitrate": 14860800,
"tier": 0,
"w": 1920,
"net": null
},
{
"t": 11,
"fps": 58,
"mbps": 9.51,
"content_p50": 14.7,
"content_p95": 19.9,
"bitrate": 14860800,
"tier": 0,
"w": 1920,
"net": null
},
{
"t": 12,
"fps": 57,
"mbps": 8.75,
"content_p50": 16.0,
"content_p95": 20.4,
"bitrate": 14860800,
"tier": 0,
"w": 1920,
"net": null
},
{
"t": 13,
"fps": 58,
"mbps": 9.25,
"content_p50": 15.0,
"content_p95": 20.7,
"bitrate": 14860800,
"tier": 0,
"w": 1920,
"net": null
},
{
"t": 14,
"fps": 57,
"mbps": 9.13,
"content_p50": 15.3,
"content_p95": 21.0,
"bitrate": 14860800,
"tier": 0,
"w": 1920,
"net": null
},
{
"t": 15,
"fps": 7,
"mbps": 1.17,
"content_p50": 14.2,
"content_p95": 15.3,
"bitrate": 14860800,
"tier": 0,
"w": 1920,
"net": null
}
],
"adapt": [],
"content_p50": 15.7,
"content_p95": 23.0,
"input_p50": null,
"input_p95": null,
"grades": {
"content_p50": "local",
"content_p95": "local",
"fps": "acceptable",
"late_pct": "acceptable",
"stall_max": "acceptable"
},
"controller": {
"adapt": true,
"baseRtt": 1.25,
"ceiling": 14860800,
"fps": 60,
"inFlight": 0,
"scale": 1,
"slack": 41.666,
"target": 14860800,
"tier": 0
},
"events": [
{
"t": 1.06,
"e": "down to 7430 kbit/s: queue 35 ms, held 4, oldest 32 ms, base 1 ms, sent 6282 got 4914 wants 11892"
}
],
"source_fps": 56.8,
"cpu": {
"mac_agent_pct": 8.8,
"frame_viewer_pct": 74.8,
"frame_tailscaled_pct": 0.3,
"frame_sshd_pct": 1.2,
"frame_gamescope_pct": 10.5,
"frame_total_pct": 39.0
},
"viewer": "h264 software; ANGLE (Mesa, zink Vulkan 1.4(Turnip Adreno (TM) 750 (MESA_TURNIP)), OpenGL 4.6); raf 137 Hz",
"show_s": 1.16,
"panel": "valve.steam.desktopgame.2001968301",
"src": "separate:7914"
}
]
}
bench/results/2026-09-28-1d05f57-usb2.json
+690
View File
@@ -0,0 +1,690 @@
{
"label": "usb2",
"date": "2026-09-28T21:17:48",
"commit": "1d05f57",
"config": {
"quality": "balanced",
"mode": "separate",
"net": "none",
"delay_ms": 0,
"buffer_ms": 250,
"host": "10.86.200.233",
"ssh_opts": [],
"encoder": "",
"duration_s": 15.0,
"browser_flags": []
},
"frame_build": "20260925.6191901",
"mac": "26.5.2",
"headset": "",
"scenarios": [
{
"scenario": "test",
"frames_sent": 913,
"frames_drawn": 913,
"frames_shown": 856,
"duration_s": 15.2,
"stages_ms": {
"capture": {
"p50": 0.0,
"p95": 0.0,
"p99": 0.0,
"n": 913
},
"queue": {
"p50": 0.0,
"p95": 0.1,
"p99": 0.1,
"n": 913
},
"encode": {
"p50": 4.3,
"p95": 4.9,
"p99": 5.2,
"n": 913
},
"socket": {
"p50": 0.1,
"p95": 0.2,
"p99": 0.2,
"n": 913
},
"network": {
"p50": 0.9,
"p95": 1.2,
"p99": 1.7,
"n": 913
},
"decode": {
"p50": 1.1,
"p95": 2.5,
"p99": 3.8,
"n": 913
},
"draw": {
"p50": 0.3,
"p95": 0.8,
"p99": 1.2,
"n": 913
},
"present": {
"p50": 2.6,
"p95": 8.6,
"p99": 15.9,
"n": 856
},
"content": {
"p50": 6.9,
"p95": 8.4,
"p99": 9.6,
"n": 913
},
"content_shown": {
"p50": 9.7,
"p95": 15.8,
"p99": 23.1,
"n": 856
}
},
"fps": 60.0,
"fps_shown": 56.3,
"late_pct": 0.0,
"stall_max": 22.0,
"stalls_over_100ms": 0,
"mbps": 0.57,
"keyframes": 1,
"size": "1280x720",
"captured": 913,
"viewer_never_drawn": 0,
"input_ms": {
"p50": 16.9,
"p95": 24.1,
"p99": 26.1,
"n": 33
},
"input_shown_ms": {
"p50": 19.7,
"p95": 26.6,
"p99": 29.2,
"n": 31
},
"input_parts_ms": {
"uplink": {
"p50": 0.7,
"p95": 2.5,
"p99": 5.7,
"n": 33
},
"mac": {
"p50": 8.1,
"p95": 15.4,
"p99": 17.2,
"n": 33
},
"back": {
"p50": 7.3,
"p95": 8.3,
"p99": 8.9,
"n": 33
}
},
"inputs": {
"asked": 30,
"sent": 33,
"seen": 33
},
"timeline": [
{
"t": 0,
"fps": 60,
"mbps": 0.49,
"content_p50": 7.1,
"content_p95": 8.7,
"bitrate": 5529600,
"tier": 0,
"w": 1280,
"net": null
},
{
"t": 1,
"fps": 60,
"mbps": 0.47,
"content_p50": 7.0,
"content_p95": 8.3,
"bitrate": 5529600,
"tier": 0,
"w": 1280,
"net": null
},
{
"t": 2,
"fps": 60,
"mbps": 0.48,
"content_p50": 6.9,
"content_p95": 8.4,
"bitrate": 5529600,
"tier": 0,
"w": 1280,
"net": null
},
{
"t": 3,
"fps": 60,
"mbps": 0.48,
"content_p50": 7.0,
"content_p95": 9.1,
"bitrate": 5529600,
"tier": 0,
"w": 1280,
"net": null
},
{
"t": 4,
"fps": 60,
"mbps": 0.5,
"content_p50": 6.8,
"content_p95": 8.5,
"bitrate": 5529600,
"tier": 0,
"w": 1280,
"net": null
},
{
"t": 5,
"fps": 60,
"mbps": 0.67,
"content_p50": 7.0,
"content_p95": 8.2,
"bitrate": 5529600,
"tier": 0,
"w": 1280,
"net": null
},
{
"t": 6,
"fps": 60,
"mbps": 0.59,
"content_p50": 7.0,
"content_p95": 8.2,
"bitrate": 5529600,
"tier": 0,
"w": 1280,
"net": null
},
{
"t": 7,
"fps": 60,
"mbps": 0.63,
"content_p50": 6.7,
"content_p95": 7.5,
"bitrate": 5529600,
"tier": 0,
"w": 1280,
"net": null
},
{
"t": 8,
"fps": 60,
"mbps": 0.62,
"content_p50": 6.9,
"content_p95": 8.2,
"bitrate": 5529600,
"tier": 0,
"w": 1280,
"net": null
},
{
"t": 9,
"fps": 60,
"mbps": 0.64,
"content_p50": 6.8,
"content_p95": 8.0,
"bitrate": 5529600,
"tier": 0,
"w": 1280,
"net": null
},
{
"t": 10,
"fps": 60,
"mbps": 0.67,
"content_p50": 6.8,
"content_p95": 8.3,
"bitrate": 5529600,
"tier": 0,
"w": 1280,
"net": null
},
{
"t": 11,
"fps": 60,
"mbps": 0.62,
"content_p50": 6.7,
"content_p95": 7.9,
"bitrate": 5529600,
"tier": 0,
"w": 1280,
"net": null
},
{
"t": 12,
"fps": 60,
"mbps": 0.59,
"content_p50": 6.8,
"content_p95": 7.9,
"bitrate": 5529600,
"tier": 0,
"w": 1280,
"net": null
},
{
"t": 13,
"fps": 60,
"mbps": 0.57,
"content_p50": 6.7,
"content_p95": 8.3,
"bitrate": 5529600,
"tier": 0,
"w": 1280,
"net": null
},
{
"t": 14,
"fps": 60,
"mbps": 0.58,
"content_p50": 7.0,
"content_p95": 8.7,
"bitrate": 5529600,
"tier": 0,
"w": 1280,
"net": null
},
{
"t": 15,
"fps": 13,
"mbps": 0.13,
"content_p50": 6.4,
"content_p95": 7.3,
"bitrate": 5529600,
"tier": 0,
"w": 1280,
"net": null
}
],
"adapt": [],
"content_p50": 6.9,
"content_p95": 8.4,
"input_p50": 16.9,
"input_p95": 24.1,
"grades": {
"input_replies": "local",
"content_p50": "local",
"content_p95": "local",
"input_p50": "local",
"input_p95": "local",
"fps": "local",
"late_pct": "local",
"stall_max": "local"
},
"controller": {
"adapt": true,
"baseRtt": 0.709,
"ceiling": 5529600,
"fps": 60,
"inFlight": 0,
"scale": 1,
"slack": 41.666,
"target": 5529600,
"tier": 0
},
"events": [],
"source_fps": 60.1,
"cpu": {
"mac_agent_pct": 17.9,
"frame_viewer_pct": 51.9,
"frame_tailscaled_pct": 0.2,
"frame_sshd_pct": 1.1,
"frame_gamescope_pct": 9.6,
"frame_total_pct": 27.9
},
"viewer": "h264 software; ANGLE (Mesa, zink Vulkan 1.4(Turnip Adreno (TM) 750 (MESA_TURNIP)), OpenGL 4.6); raf 134 Hz",
"show_s": 1.15,
"panel": "valve.steam.desktopgame.2001639889",
"src": "test"
},
{
"scenario": "scroll",
"frames_sent": 868,
"frames_drawn": 868,
"frames_shown": 868,
"duration_s": 15.2,
"stages_ms": {
"capture": {
"p50": -4.2,
"p95": 0.5,
"p99": 6.7,
"n": 868
},
"queue": {
"p50": 0.0,
"p95": 0.1,
"p99": 0.2,
"n": 868
},
"encode": {
"p50": 6.8,
"p95": 10.2,
"p99": 17.9,
"n": 868
},
"socket": {
"p50": 0.1,
"p95": 0.2,
"p99": 0.4,
"n": 868
},
"network": {
"p50": 1.5,
"p95": 2.3,
"p99": 7.6,
"n": 868
},
"decode": {
"p50": 5.9,
"p95": 10.2,
"p99": 20.3,
"n": 868
},
"draw": {
"p50": 0.7,
"p95": 1.4,
"p99": 2.1,
"n": 868
},
"present": {
"p50": 0.9,
"p95": 6.1,
"p99": 6.8,
"n": 868
},
"content": {
"p50": 15.3,
"p95": 23.5,
"p99": 53.1,
"n": 868
},
"content_shown": {
"p50": 17.7,
"p95": 25.6,
"p99": 57.5,
"n": 868
}
},
"fps": 57.2,
"fps_shown": 57.2,
"late_pct": 3.23,
"stall_max": 108.0,
"stalls_over_100ms": 1,
"mbps": 9.84,
"keyframes": 1,
"size": "1920x1290",
"captured": 870,
"viewer_never_drawn": 0,
"input_ms": {
"p50": 24.2,
"p95": 36.3,
"p99": 36.3,
"n": 6
},
"input_shown_ms": {
"p50": 24.8,
"p95": 38.5,
"p99": 38.5,
"n": 6
},
"input_parts_ms": {
"uplink": {
"p50": 2.0,
"p95": 5.4,
"p99": 5.4,
"n": 6
},
"mac": {
"p50": 7.5,
"p95": 15.8,
"p99": 15.8,
"n": 6
},
"back": {
"p50": 15.3,
"p95": 16.1,
"p99": 16.1,
"n": 6
}
},
"inputs": {
"asked": 0,
"sent": 6,
"seen": 6
},
"timeline": [
{
"t": 0,
"fps": 53,
"mbps": 9.21,
"content_p50": 16.1,
"content_p95": 71.2,
"bitrate": 14860800,
"tier": 0,
"w": 1920,
"net": null
},
{
"t": 1,
"fps": 57,
"mbps": 9.64,
"content_p50": 15.9,
"content_p95": 46.8,
"bitrate": 14860800,
"tier": 0,
"w": 1920,
"net": null
},
{
"t": 2,
"fps": 58,
"mbps": 8.99,
"content_p50": 14.8,
"content_p95": 19.2,
"bitrate": 14860800,
"tier": 0,
"w": 1920,
"net": null
},
{
"t": 3,
"fps": 57,
"mbps": 9.72,
"content_p50": 15.2,
"content_p95": 22.3,
"bitrate": 14860800,
"tier": 0,
"w": 1920,
"net": null
},
{
"t": 4,
"fps": 58,
"mbps": 9.64,
"content_p50": 16.1,
"content_p95": 23.5,
"bitrate": 14860800,
"tier": 0,
"w": 1920,
"net": null
},
{
"t": 5,
"fps": 57,
"mbps": 11.27,
"content_p50": 15.6,
"content_p95": 20.9,
"bitrate": 14860800,
"tier": 0,
"w": 1920,
"net": null
},
{
"t": 6,
"fps": 58,
"mbps": 10.44,
"content_p50": 15.1,
"content_p95": 19.1,
"bitrate": 14860800,
"tier": 0,
"w": 1920,
"net": null
},
{
"t": 7,
"fps": 57,
"mbps": 9.31,
"content_p50": 15.0,
"content_p95": 19.9,
"bitrate": 14860800,
"tier": 0,
"w": 1920,
"net": null
},
{
"t": 8,
"fps": 58,
"mbps": 11.69,
"content_p50": 15.9,
"content_p95": 22.0,
"bitrate": 14860800,
"tier": 0,
"w": 1920,
"net": null
},
{
"t": 9,
"fps": 58,
"mbps": 10.31,
"content_p50": 14.5,
"content_p95": 20.7,
"bitrate": 14860800,
"tier": 0,
"w": 1920,
"net": null
},
{
"t": 10,
"fps": 57,
"mbps": 10.47,
"content_p50": 15.2,
"content_p95": 20.6,
"bitrate": 14860800,
"tier": 0,
"w": 1920,
"net": null
},
{
"t": 11,
"fps": 58,
"mbps": 9.44,
"content_p50": 15.3,
"content_p95": 20.5,
"bitrate": 14860800,
"tier": 0,
"w": 1920,
"net": null
},
{
"t": 12,
"fps": 57,
"mbps": 9.17,
"content_p50": 15.9,
"content_p95": 18.7,
"bitrate": 14860800,
"tier": 0,
"w": 1920,
"net": null
},
{
"t": 13,
"fps": 58,
"mbps": 8.77,
"content_p50": 15.1,
"content_p95": 18.6,
"bitrate": 14860800,
"tier": 0,
"w": 1920,
"net": null
},
{
"t": 14,
"fps": 58,
"mbps": 9.72,
"content_p50": 15.2,
"content_p95": 19.8,
"bitrate": 14860800,
"tier": 0,
"w": 1920,
"net": null
},
{
"t": 15,
"fps": 9,
"mbps": 1.33,
"content_p50": 14.8,
"content_p95": 17.7,
"bitrate": 14860800,
"tier": 0,
"w": 1920,
"net": null
}
],
"adapt": [],
"content_p50": 15.3,
"content_p95": 23.5,
"input_p50": 24.2,
"input_p95": 36.3,
"grades": {
"input_replies": "local",
"content_p50": "local",
"content_p95": "local",
"input_p50": "local",
"input_p95": "local",
"fps": "acceptable",
"late_pct": "acceptable",
"stall_max": "acceptable"
},
"controller": {
"adapt": true,
"baseRtt": 1.375,
"ceiling": 14860800,
"fps": 60,
"inFlight": 1,
"scale": 1,
"slack": 41.666,
"target": 14860800,
"tier": 0
},
"events": [],
"source_fps": 57.2,
"cpu": {
"mac_agent_pct": 11.4,
"frame_viewer_pct": 83.5,
"frame_tailscaled_pct": 0.3,
"frame_sshd_pct": 1.5,
"frame_gamescope_pct": 9.7,
"frame_total_pct": 32.5
},
"viewer": "h264 software; ANGLE (Mesa, zink Vulkan 1.4(Turnip Adreno (TM) 750 (MESA_TURNIP)), OpenGL 4.6); raf 139 Hz",
"show_s": 1.15,
"panel": "valve.steam.desktopgame.2001786096",
"src": "separate:8187"
}
]
}
bench/results/2026-09-28-1d05f57-wifi1.json
+682
View File
@@ -0,0 +1,682 @@
{
"label": "wifi1",
"date": "2026-09-28T21:16:56",
"commit": "1d05f57",
"config": {
"quality": "balanced",
"mode": "separate",
"net": "none",
"delay_ms": 0,
"buffer_ms": 250,
"host": "frame",
"ssh_opts": [],
"encoder": "",
"duration_s": 15.0,
"browser_flags": []
},
"frame_build": "20260925.6191901",
"mac": "26.5.2",
"headset": "",
"scenarios": [
{
"scenario": "test",
"frames_sent": 912,
"frames_drawn": 912,
"frames_shown": 611,
"duration_s": 15.2,
"stages_ms": {
"capture": {
"p50": 0.0,
"p95": 0.0,
"p99": 0.0,
"n": 912
},
"queue": {
"p50": 0.0,
"p95": 0.1,
"p99": 0.1,
"n": 912
},
"encode": {
"p50": 4.3,
"p95": 4.9,
"p99": 5.1,
"n": 912
},
"socket": {
"p50": 0.1,
"p95": 0.2,
"p99": 0.2,
"n": 912
},
"network": {
"p50": 4.0,
"p95": 5.3,
"p99": 7.8,
"n": 912
},
"decode": {
"p50": 1.1,
"p95": 2.8,
"p99": 4.0,
"n": 912
},
"draw": {
"p50": 0.4,
"p95": 0.8,
"p99": 1.2,
"n": 912
},
"present": {
"p50": 5.9,
"p95": 14.0,
"p99": 18.5,
"n": 611
},
"content": {
"p50": 10.1,
"p95": 12.3,
"p99": 13.9,
"n": 912
},
"content_shown": {
"p50": 16.7,
"p95": 23.6,
"p99": 28.3,
"n": 611
}
},
"fps": 60.0,
"fps_shown": 40.1,
"late_pct": 0.11,
"stall_max": 25.8,
"stalls_over_100ms": 0,
"mbps": 0.48,
"keyframes": 1,
"size": "1280x720",
"captured": 912,
"viewer_never_drawn": 0,
"input_ms": {
"p50": 26.4,
"p95": 34.2,
"p99": 53.5,
"n": 30
},
"input_shown_ms": {
"p50": 28.6,
"p95": 41.5,
"p99": 55.9,
"n": 18
},
"input_parts_ms": {
"uplink": {
"p50": 8.9,
"p95": 20.4,
"p99": 28.0,
"n": 30
},
"mac": {
"p50": 5.1,
"p95": 14.5,
"p99": 16.2,
"n": 30
},
"back": {
"p50": 10.1,
"p95": 13.7,
"p99": 13.8,
"n": 30
}
},
"inputs": {
"asked": 30,
"sent": 30,
"seen": 30
},
"timeline": [
{
"t": 0,
"fps": 60,
"mbps": 0.51,
"content_p50": 10.3,
"content_p95": 11.9,
"bitrate": 5529600,
"tier": 0,
"w": 1280,
"net": null
},
{
"t": 1,
"fps": 60,
"mbps": 0.47,
"content_p50": 10.5,
"content_p95": 12.1,
"bitrate": 5529600,
"tier": 0,
"w": 1280,
"net": null
},
{
"t": 2,
"fps": 60,
"mbps": 0.49,
"content_p50": 10.1,
"content_p95": 12.8,
"bitrate": 5529600,
"tier": 0,
"w": 1280,
"net": null
},
{
"t": 3,
"fps": 60,
"mbps": 0.48,
"content_p50": 9.9,
"content_p95": 11.6,
"bitrate": 5529600,
"tier": 0,
"w": 1280,
"net": null
},
{
"t": 4,
"fps": 60,
"mbps": 0.47,
"content_p50": 9.7,
"content_p95": 11.8,
"bitrate": 5529600,
"tier": 0,
"w": 1280,
"net": null
},
{
"t": 5,
"fps": 60,
"mbps": 0.59,
"content_p50": 10.2,
"content_p95": 11.7,
"bitrate": 5529600,
"tier": 0,
"w": 1280,
"net": null
},
{
"t": 6,
"fps": 60,
"mbps": 0.47,
"content_p50": 10.2,
"content_p95": 12.3,
"bitrate": 5529600,
"tier": 0,
"w": 1280,
"net": null
},
{
"t": 7,
"fps": 60,
"mbps": 0.47,
"content_p50": 9.7,
"content_p95": 12.1,
"bitrate": 5529600,
"tier": 0,
"w": 1280,
"net": null
},
{
"t": 8,
"fps": 60,
"mbps": 0.47,
"content_p50": 10.4,
"content_p95": 13.1,
"bitrate": 5529600,
"tier": 0,
"w": 1280,
"net": null
},
{
"t": 9,
"fps": 60,
"mbps": 0.47,
"content_p50": 9.9,
"content_p95": 12.7,
"bitrate": 5529600,
"tier": 0,
"w": 1280,
"net": null
},
{
"t": 10,
"fps": 60,
"mbps": 0.47,
"content_p50": 9.8,
"content_p95": 12.4,
"bitrate": 5529600,
"tier": 0,
"w": 1280,
"net": null
},
{
"t": 11,
"fps": 60,
"mbps": 0.46,
"content_p50": 10.3,
"content_p95": 12.7,
"bitrate": 5529600,
"tier": 0,
"w": 1280,
"net": null
},
{
"t": 12,
"fps": 60,
"mbps": 0.49,
"content_p50": 10.2,
"content_p95": 12.0,
"bitrate": 5529600,
"tier": 0,
"w": 1280,
"net": null
},
{
"t": 13,
"fps": 60,
"mbps": 0.48,
"content_p50": 10.0,
"content_p95": 11.3,
"bitrate": 5529600,
"tier": 0,
"w": 1280,
"net": null
},
{
"t": 14,
"fps": 60,
"mbps": 0.46,
"content_p50": 10.2,
"content_p95": 12.3,
"bitrate": 5529600,
"tier": 0,
"w": 1280,
"net": null
},
{
"t": 15,
"fps": 12,
"mbps": 0.1,
"content_p50": 10.7,
"content_p95": 11.4,
"bitrate": 5529600,
"tier": 0,
"w": 1280,
"net": null
}
],
"adapt": [],
"content_p50": 10.1,
"content_p95": 12.3,
"input_p50": 26.4,
"input_p95": 34.2,
"grades": {
"input_replies": "local",
"content_p50": "local",
"content_p95": "local",
"input_p50": "local",
"input_p95": "local",
"fps": "local",
"late_pct": "local",
"stall_max": "local"
},
"controller": {
"adapt": true,
"baseRtt": 4.958,
"ceiling": 5529600,
"fps": 60,
"inFlight": 1,
"scale": 1,
"slack": 47.417,
"target": 2764800,
"tier": 0
},
"events": [
{
"t": 16.58,
"e": "down to 2764 kbit/s: queue 19 ms, held 6, oldest 210 ms, base 4 ms, sent 322 got 265 wants 459"
}
],
"source_fps": 60.0,
"cpu": {
"mac_agent_pct": 15.7,
"frame_viewer_pct": 42.5,
"frame_tailscaled_pct": 5.5,
"frame_sshd_pct": 0.7,
"frame_gamescope_pct": 8.7,
"frame_total_pct": 26.1
},
"viewer": "h264 software; ANGLE (Mesa, zink Vulkan 1.4(Turnip Adreno (TM) 750 (MESA_TURNIP)), OpenGL 4.6); raf 136 Hz",
"show_s": 1.26,
"panel": "valve.steam.desktopgame.2001639889",
"src": "test"
},
{
"scenario": "scroll",
"frames_sent": 852,
"frames_drawn": 852,
"frames_shown": 614,
"duration_s": 15.2,
"stages_ms": {
"capture": {
"p50": -4.4,
"p95": 2.6,
"p99": 4.2,
"n": 852
},
"queue": {
"p50": 0.0,
"p95": 0.1,
"p99": 0.1,
"n": 852
},
"encode": {
"p50": 6.7,
"p95": 9.8,
"p99": 16.7,
"n": 852
},
"socket": {
"p50": 0.1,
"p95": 0.2,
"p99": 0.2,
"n": 852
},
"network": {
"p50": 5.3,
"p95": 13.1,
"p99": 28.2,
"n": 852
},
"decode": {
"p50": 6.0,
"p95": 10.9,
"p99": 19.0,
"n": 852
},
"draw": {
"p50": 0.7,
"p95": 1.4,
"p99": 1.8,
"n": 852
},
"present": {
"p50": 4.1,
"p95": 16.2,
"p99": 20.6,
"n": 614
},
"content": {
"p50": 19.5,
"p95": 31.4,
"p99": 64.5,
"n": 852
},
"content_shown": {
"p50": 25.4,
"p95": 38.8,
"p99": 76.7,
"n": 614
}
},
"fps": 56.2,
"fps_shown": 40.5,
"late_pct": 4.69,
"stall_max": 253.7,
"stalls_over_100ms": 2,
"mbps": 9.02,
"keyframes": 1,
"size": "1920x1290",
"captured": 866,
"viewer_never_drawn": 0,
"input_ms": {
"n": 0
},
"input_shown_ms": {
"n": 0
},
"input_parts_ms": {
"uplink": {
"n": 0
},
"mac": {
"n": 0
},
"back": {
"n": 0
}
},
"inputs": {
"asked": 0,
"sent": 0,
"seen": 0
},
"timeline": [
{
"t": 0,
"fps": 45,
"mbps": 7.77,
"content_p50": 19.0,
"content_p95": 163.1,
"bitrate": 14860800,
"tier": 0,
"w": 1920,
"net": null
},
{
"t": 1,
"fps": 51,
"mbps": 6.54,
"content_p50": 19.4,
"content_p95": 107.0,
"bitrate": 7430400,
"tier": 0,
"w": 1920,
"net": null
},
{
"t": 2,
"fps": 57,
"mbps": 8.64,
"content_p50": 19.9,
"content_p95": 31.8,
"bitrate": 10055362,
"tier": 0,
"w": 1920,
"net": null
},
{
"t": 3,
"fps": 58,
"mbps": 9.46,
"content_p50": 19.1,
"content_p95": 24.9,
"bitrate": 14860800,
"tier": 0,
"w": 1920,
"net": null
},
{
"t": 4,
"fps": 58,
"mbps": 9.11,
"content_p50": 19.4,
"content_p95": 22.9,
"bitrate": 14860800,
"tier": 0,
"w": 1920,
"net": null
},
{
"t": 5,
"fps": 57,
"mbps": 9.18,
"content_p50": 19.9,
"content_p95": 25.3,
"bitrate": 14860800,
"tier": 0,
"w": 1920,
"net": null
},
{
"t": 6,
"fps": 58,
"mbps": 11.64,
"content_p50": 19.8,
"content_p95": 49.6,
"bitrate": 14860800,
"tier": 0,
"w": 1920,
"net": null
},
{
"t": 7,
"fps": 57,
"mbps": 9.12,
"content_p50": 20.5,
"content_p95": 31.1,
"bitrate": 14860800,
"tier": 0,
"w": 1920,
"net": null
},
{
"t": 8,
"fps": 58,
"mbps": 9.29,
"content_p50": 19.5,
"content_p95": 25.7,
"bitrate": 14860800,
"tier": 0,
"w": 1920,
"net": null
},
{
"t": 9,
"fps": 57,
"mbps": 9.43,
"content_p50": 19.8,
"content_p95": 49.6,
"bitrate": 14860800,
"tier": 0,
"w": 1920,
"net": null
},
{
"t": 10,
"fps": 58,
"mbps": 8.59,
"content_p50": 19.2,
"content_p95": 24.8,
"bitrate": 14860800,
"tier": 0,
"w": 1920,
"net": null
},
{
"t": 11,
"fps": 57,
"mbps": 9.42,
"content_p50": 19.9,
"content_p95": 27.1,
"bitrate": 14860800,
"tier": 0,
"w": 1920,
"net": null
},
{
"t": 12,
"fps": 58,
"mbps": 9.0,
"content_p50": 20.6,
"content_p95": 31.2,
"bitrate": 14860800,
"tier": 0,
"w": 1920,
"net": null
},
{
"t": 13,
"fps": 57,
"mbps": 8.84,
"content_p50": 19.6,
"content_p95": 25.3,
"bitrate": 14860800,
"tier": 0,
"w": 1920,
"net": null
},
{
"t": 14,
"fps": 57,
"mbps": 9.17,
"content_p50": 19.3,
"content_p95": 27.8,
"bitrate": 14860800,
"tier": 0,
"w": 1920,
"net": null
},
{
"t": 15,
"fps": 9,
"mbps": 1.52,
"content_p50": 19.3,
"content_p95": 30.0,
"bitrate": 14860800,
"tier": 0,
"w": 1920,
"net": null
}
],
"adapt": [],
"content_p50": 19.5,
"content_p95": 31.4,
"input_p50": null,
"input_p95": null,
"grades": {
"content_p50": "local",
"content_p95": "local",
"fps": "acceptable",
"late_pct": "acceptable",
"stall_max": "bad"
},
"controller": {
"adapt": true,
"baseRtt": 6.959,
"ceiling": 14860800,
"fps": 60,
"inFlight": 2,
"scale": 1,
"slack": 43.54,
"target": 14860800,
"tier": 0
},
"events": [
{
"t": 1.09,
"e": "down to 7430 kbit/s: queue 131 ms, held 3, oldest 231 ms, base 6 ms, sent 4272 got 3724 wants 9321"
}
],
"source_fps": 57.0,
"cpu": {
"mac_agent_pct": 8.5,
"frame_viewer_pct": 71.7,
"frame_tailscaled_pct": 8.4,
"frame_sshd_pct": 0.9,
"frame_gamescope_pct": 8.4,
"frame_total_pct": 30.1
},
"viewer": "h264 software; ANGLE (Mesa, zink Vulkan 1.4(Turnip Adreno (TM) 750 (MESA_TURNIP)), OpenGL 4.6); raf 139 Hz",
"show_s": 1.25,
"panel": "valve.steam.desktopgame.2001195625",
"src": "separate:8050"
}
]
}
bench/results/2026-09-28-1d05f57-wifi2.json
+682
View File
@@ -0,0 +1,682 @@
{
"label": "wifi2",
"date": "2026-09-28T21:18:40",
"commit": "1d05f57",
"config": {
"quality": "balanced",
"mode": "separate",
"net": "none",
"delay_ms": 0,
"buffer_ms": 250,
"host": "frame",
"ssh_opts": [],
"encoder": "",
"duration_s": 15.0,
"browser_flags": []
},
"frame_build": "20260925.6191901",
"mac": "26.5.2",
"headset": "",
"scenarios": [
{
"scenario": "test",
"frames_sent": 902,
"frames_drawn": 902,
"frames_shown": 604,
"duration_s": 15.2,
"stages_ms": {
"capture": {
"p50": 0.0,
"p95": 0.0,
"p99": 0.0,
"n": 902
},
"queue": {
"p50": 0.0,
"p95": 0.1,
"p99": 0.1,
"n": 902
},
"encode": {
"p50": 4.3,
"p95": 5.0,
"p99": 5.3,
"n": 902
},
"socket": {
"p50": 0.1,
"p95": 0.2,
"p99": 0.3,
"n": 902
},
"network": {
"p50": 3.8,
"p95": 5.3,
"p99": 8.0,
"n": 902
},
"decode": {
"p50": 1.1,
"p95": 2.8,
"p99": 3.8,
"n": 902
},
"draw": {
"p50": 0.3,
"p95": 0.8,
"p99": 1.2,
"n": 902
},
"present": {
"p50": 5.9,
"p95": 17.0,
"p99": 18.6,
"n": 604
},
"content": {
"p50": 9.8,
"p95": 12.1,
"p99": 14.4,
"n": 902
},
"content_shown": {
"p50": 15.2,
"p95": 27.0,
"p99": 28.4,
"n": 604
}
},
"fps": 59.3,
"fps_shown": 39.7,
"late_pct": 0.55,
"stall_max": 187.5,
"stalls_over_100ms": 1,
"mbps": 0.47,
"keyframes": 1,
"size": "1280x720",
"captured": 913,
"viewer_never_drawn": 0,
"input_ms": {
"p50": 27.8,
"p95": 34.9,
"p99": 206.9,
"n": 30
},
"input_shown_ms": {
"p50": 35.1,
"p95": 52.6,
"p99": 207.2,
"n": 24
},
"input_parts_ms": {
"uplink": {
"p50": 9.8,
"p95": 22.5,
"p99": 184.8,
"n": 30
},
"mac": {
"p50": 5.3,
"p95": 14.4,
"p99": 19.6,
"n": 30
},
"back": {
"p50": 9.8,
"p95": 13.4,
"p99": 13.9,
"n": 30
}
},
"inputs": {
"asked": 30,
"sent": 30,
"seen": 30
},
"timeline": [
{
"t": 0,
"fps": 60,
"mbps": 0.48,
"content_p50": 9.3,
"content_p95": 11.2,
"bitrate": 5529600,
"tier": 0,
"w": 1280,
"net": null
},
{
"t": 1,
"fps": 50,
"mbps": 0.39,
"content_p50": 9.1,
"content_p95": 14.4,
"bitrate": 2764800,
"tier": 0,
"w": 1280,
"net": null
},
{
"t": 2,
"fps": 60,
"mbps": 0.48,
"content_p50": 9.2,
"content_p95": 11.8,
"bitrate": 2764800,
"tier": 0,
"w": 1280,
"net": null
},
{
"t": 3,
"fps": 60,
"mbps": 0.48,
"content_p50": 9.4,
"content_p95": 10.5,
"bitrate": 2764800,
"tier": 0,
"w": 1280,
"net": null
},
{
"t": 4,
"fps": 60,
"mbps": 0.48,
"content_p50": 9.5,
"content_p95": 12.5,
"bitrate": 3091280,
"tier": 0,
"w": 1280,
"net": null
},
{
"t": 5,
"fps": 60,
"mbps": 0.47,
"content_p50": 10.0,
"content_p95": 12.8,
"bitrate": 4757991,
"tier": 0,
"w": 1280,
"net": null
},
{
"t": 6,
"fps": 60,
"mbps": 0.57,
"content_p50": 9.5,
"content_p95": 11.7,
"bitrate": 5529600,
"tier": 0,
"w": 1280,
"net": null
},
{
"t": 7,
"fps": 60,
"mbps": 0.47,
"content_p50": 9.8,
"content_p95": 11.3,
"bitrate": 5529600,
"tier": 0,
"w": 1280,
"net": null
},
{
"t": 8,
"fps": 60,
"mbps": 0.47,
"content_p50": 10.2,
"content_p95": 12.2,
"bitrate": 5529600,
"tier": 0,
"w": 1280,
"net": null
},
{
"t": 9,
"fps": 60,
"mbps": 0.47,
"content_p50": 10.0,
"content_p95": 13.2,
"bitrate": 5529600,
"tier": 0,
"w": 1280,
"net": null
},
{
"t": 10,
"fps": 60,
"mbps": 0.47,
"content_p50": 10.2,
"content_p95": 12.5,
"bitrate": 5529600,
"tier": 0,
"w": 1280,
"net": null
},
{
"t": 11,
"fps": 60,
"mbps": 0.45,
"content_p50": 9.9,
"content_p95": 11.6,
"bitrate": 5529600,
"tier": 0,
"w": 1280,
"net": null
},
{
"t": 12,
"fps": 60,
"mbps": 0.46,
"content_p50": 10.0,
"content_p95": 11.6,
"bitrate": 5529600,
"tier": 0,
"w": 1280,
"net": null
},
{
"t": 13,
"fps": 60,
"mbps": 0.47,
"content_p50": 10.3,
"content_p95": 12.3,
"bitrate": 5529600,
"tier": 0,
"w": 1280,
"net": null
},
{
"t": 14,
"fps": 60,
"mbps": 0.48,
"content_p50": 9.8,
"content_p95": 11.6,
"bitrate": 5529600,
"tier": 0,
"w": 1280,
"net": null
},
{
"t": 15,
"fps": 12,
"mbps": 0.1,
"content_p50": 9.3,
"content_p95": 11.5,
"bitrate": 5529600,
"tier": 0,
"w": 1280,
"net": null
}
],
"adapt": [],
"content_p50": 9.8,
"content_p95": 12.1,
"input_p50": 27.8,
"input_p95": 34.9,
"grades": {
"input_replies": "local",
"content_p50": "local",
"content_p95": "local",
"input_p50": "local",
"input_p95": "local",
"fps": "local",
"late_pct": "local",
"stall_max": "acceptable"
},
"controller": {
"adapt": true,
"baseRtt": 5.084,
"ceiling": 5529600,
"fps": 60,
"inFlight": 1,
"scale": 1,
"slack": 49.603,
"target": 5529600,
"tier": 0
},
"events": [
{
"t": 1.77,
"e": "down to 2764 kbit/s: queue 23 ms, held 5, oldest 0 ms, base 5 ms, sent 315 got 315 wants 472"
}
],
"source_fps": 60.1,
"cpu": {
"mac_agent_pct": 16.1,
"frame_viewer_pct": 40.9,
"frame_tailscaled_pct": 5.2,
"frame_sshd_pct": 0.7,
"frame_gamescope_pct": 8.4,
"frame_total_pct": 25.2
},
"viewer": "h264 software; ANGLE (Mesa, zink Vulkan 1.4(Turnip Adreno (TM) 750 (MESA_TURNIP)), OpenGL 4.6); raf 136 Hz",
"show_s": 1.27,
"panel": "valve.steam.desktopgame.2001639889",
"src": "test"
},
{
"scenario": "scroll",
"frames_sent": 832,
"frames_drawn": 832,
"frames_shown": 602,
"duration_s": 15.1,
"stages_ms": {
"capture": {
"p50": -3.9,
"p95": 0.8,
"p99": 6.8,
"n": 832
},
"queue": {
"p50": 0.0,
"p95": 0.1,
"p99": 1.9,
"n": 832
},
"encode": {
"p50": 6.8,
"p95": 10.5,
"p99": 18.8,
"n": 832
},
"socket": {
"p50": 0.1,
"p95": 0.2,
"p99": 0.5,
"n": 832
},
"network": {
"p50": 5.8,
"p95": 18.7,
"p99": 56.7,
"n": 832
},
"decode": {
"p50": 5.9,
"p95": 11.7,
"p99": 26.0,
"n": 832
},
"draw": {
"p50": 0.7,
"p95": 1.4,
"p99": 1.7,
"n": 832
},
"present": {
"p50": 4.4,
"p95": 18.5,
"p99": 25.0,
"n": 602
},
"content": {
"p50": 20.2,
"p95": 36.9,
"p99": 90.7,
"n": 832
},
"content_shown": {
"p50": 26.4,
"p95": 45.4,
"p99": 101.0,
"n": 602
}
},
"fps": 54.9,
"fps_shown": 39.7,
"late_pct": 6.96,
"stall_max": 204.9,
"stalls_over_100ms": 2,
"mbps": 9.04,
"keyframes": 1,
"size": "1920x1290",
"captured": 852,
"viewer_never_drawn": 0,
"input_ms": {
"n": 0
},
"input_shown_ms": {
"n": 0
},
"input_parts_ms": {
"uplink": {
"n": 0
},
"mac": {
"n": 0
},
"back": {
"n": 0
}
},
"inputs": {
"asked": 0,
"sent": 0,
"seen": 0
},
"timeline": [
{
"t": 0,
"fps": 49,
"mbps": 8.68,
"content_p50": 19.8,
"content_p95": 119.1,
"bitrate": 14860800,
"tier": 0,
"w": 1920,
"net": null
},
{
"t": 1,
"fps": 47,
"mbps": 6.06,
"content_p50": 19.8,
"content_p95": 88.6,
"bitrate": 7430400,
"tier": 0,
"w": 1920,
"net": null
},
{
"t": 2,
"fps": 52,
"mbps": 8.07,
"content_p50": 19.4,
"content_p95": 25.5,
"bitrate": 10055362,
"tier": 0,
"w": 1920,
"net": null
},
{
"t": 3,
"fps": 56,
"mbps": 9.79,
"content_p50": 21.4,
"content_p95": 29.1,
"bitrate": 13549185,
"tier": 0,
"w": 1920,
"net": null
},
{
"t": 4,
"fps": 57,
"mbps": 8.93,
"content_p50": 21.5,
"content_p95": 35.1,
"bitrate": 14860800,
"tier": 0,
"w": 1920,
"net": null
},
{
"t": 5,
"fps": 57,
"mbps": 10.59,
"content_p50": 21.1,
"content_p95": 35.4,
"bitrate": 14860800,
"tier": 0,
"w": 1920,
"net": null
},
{
"t": 6,
"fps": 56,
"mbps": 11.55,
"content_p50": 19.7,
"content_p95": 66.9,
"bitrate": 14860800,
"tier": 0,
"w": 1920,
"net": null
},
{
"t": 7,
"fps": 57,
"mbps": 8.8,
"content_p50": 18.8,
"content_p95": 24.8,
"bitrate": 14860800,
"tier": 0,
"w": 1920,
"net": null
},
{
"t": 8,
"fps": 57,
"mbps": 9.17,
"content_p50": 19.5,
"content_p95": 24.4,
"bitrate": 14860800,
"tier": 0,
"w": 1920,
"net": null
},
{
"t": 9,
"fps": 57,
"mbps": 9.39,
"content_p50": 19.4,
"content_p95": 23.1,
"bitrate": 14860800,
"tier": 0,
"w": 1920,
"net": null
},
{
"t": 10,
"fps": 57,
"mbps": 8.53,
"content_p50": 19.5,
"content_p95": 38.1,
"bitrate": 14860800,
"tier": 0,
"w": 1920,
"net": null
},
{
"t": 11,
"fps": 56,
"mbps": 9.23,
"content_p50": 20.7,
"content_p95": 30.9,
"bitrate": 14860800,
"tier": 0,
"w": 1920,
"net": null
},
{
"t": 12,
"fps": 53,
"mbps": 8.87,
"content_p50": 20.6,
"content_p95": 35.4,
"bitrate": 14860800,
"tier": 0,
"w": 1920,
"net": null
},
{
"t": 13,
"fps": 57,
"mbps": 8.94,
"content_p50": 20.4,
"content_p95": 28.2,
"bitrate": 14860800,
"tier": 0,
"w": 1920,
"net": null
},
{
"t": 14,
"fps": 57,
"mbps": 9.14,
"content_p50": 21.0,
"content_p95": 35.5,
"bitrate": 14860800,
"tier": 0,
"w": 1920,
"net": null
},
{
"t": 15,
"fps": 7,
"mbps": 1.2,
"content_p50": 21.8,
"content_p95": 23.1,
"bitrate": 14860800,
"tier": 0,
"w": 1920,
"net": null
}
],
"adapt": [],
"content_p50": 20.2,
"content_p95": 36.9,
"input_p50": null,
"input_p95": null,
"grades": {
"content_p50": "local",
"content_p95": "local",
"fps": "acceptable",
"late_pct": "bad",
"stall_max": "acceptable"
},
"controller": {
"adapt": true,
"baseRtt": 6.833,
"ceiling": 14860800,
"fps": 60,
"inFlight": 2,
"scale": 1,
"slack": 44.791,
"target": 14860800,
"tier": 0
},
"events": [
{
"t": 1.22,
"e": "down to 7430 kbit/s: queue 74 ms, held 7, oldest 235 ms, base 6 ms, sent 5436 got 4487 wants 11082"
}
],
"source_fps": 56.4,
"cpu": {
"mac_agent_pct": 8.8,
"frame_viewer_pct": 69.5,
"frame_tailscaled_pct": 9.5,
"frame_sshd_pct": 0.8,
"frame_gamescope_pct": 8.4,
"frame_total_pct": 31.8
},
"viewer": "h264 software; ANGLE (Mesa, zink Vulkan 1.4(Turnip Adreno (TM) 750 (MESA_TURNIP)), OpenGL 4.6); raf 139 Hz",
"show_s": 1.25,
"panel": "valve.steam.desktopgame.2001116820",
"src": "separate:8327"
}
]
}
bench/results/2026-09-28-39afb23-worn.json
+1180
View File
File diff suppressed because it is too large. Load diff
bench/results/2026-09-28-9e4dcdd-final.json
+1173
View File
File diff suppressed because it is too large. Load diff
bench/results/2026-09-28-a3c6e5c-dirty-ab-adapt-clean.json
+4211
View File
File diff suppressed because it is too large. Load diff
bench/results/2026-09-28-a3c6e5c-dirty-ab-adapt-clean2.json
+4150
View File
File diff suppressed because it is too large. Load diff
bench/results/2026-09-28-a3c6e5c-dirty-ab-adapt-step3.json
+3743
View File
File diff suppressed because it is too large. Load diff
bench/results/2026-09-28-a3c6e5c-dirty-ab-keepwarm.json
+2172
View File
File diff suppressed because it is too large. Load diff
bench/results/2026-09-28-a3c6e5c-dirty-adapt-clean.json
+984
View File
@@ -0,0 +1,984 @@
{
"label": "adapt-clean",
"date": "2026-09-28T17:13:59",
"commit": "a3c6e5c+dirty",
"config": {
"quality": "balanced",
"mode": "separate",
"net": "none",
"delay_ms": 0,
"buffer_ms": 250,
"host": "frame",
"ssh_opts": [],
"encoder": "",
"duration_s": 20.0,
"browser_flags": []
},
"frame_build": "20260925.6191901",
"mac": "26.5.2",
"headset": ":39.639829 [Info] - 0 - entering standby",
"scenarios": [
{
"scenario": "test",
"frames_sent": 653,
"frames_drawn": 653,
"frames_shown": 649,
"duration_s": 20.7,
"stages_ms": {
"capture": {
"p50": 0.0,
"p95": 0.0,
"p99": 0.0,
"n": 653
},
"queue": {
"p50": 11.2,
"p95": 16.1,
"p99": 17.1,
"n": 653
},
"encode": {
"p50": 3.5,
"p95": 4.2,
"p99": 4.4,
"n": 653
},
"socket": {
"p50": 0.1,
"p95": 0.2,
"p99": 0.2,
"n": 653
},
"network": {
"p50": 2.9,
"p95": 7.0,
"p99": 12.0,
"n": 653
},
"decode": {
"p50": 1.0,
"p95": 2.4,
"p99": 2.8,
"n": 653
},
"draw": {
"p50": 0.3,
"p95": 0.6,
"p99": 0.9,
"n": 653
},
"present": {
"p50": 0.5,
"p95": 0.9,
"p99": 5.9,
"n": 649
},
"content": {
"p50": 19.8,
"p95": 26.2,
"p99": 30.1,
"n": 653
},
"content_shown": {
"p50": 20.3,
"p95": 26.9,
"p99": 31.1,
"n": 649
}
},
"fps": 31.5,
"fps_shown": 31.3,
"late_pct": 98.01,
"stall_max": 144.0,
"stalls_over_100ms": 2,
"mbps": 0.13,
"keyframes": 2,
"size": "960x540",
"captured": 1336,
"viewer_never_drawn": 0,
"input_ms": {
"p50": 43.5,
"p95": 61.8,
"p99": 63.4,
"n": 40
},
"input_shown_ms": {
"p50": 44.3,
"p95": 62.3,
"p99": 63.9,
"n": 40
},
"input_parts_ms": {
"uplink": {
"p50": 11.1,
"p95": 29.2,
"p99": 36.1,
"n": 40
},
"mac": {
"p50": 10.4,
"p95": 28.2,
"p99": 32.4,
"n": 40
},
"back": {
"p50": 17.5,
"p95": 27.9,
"p99": 28.8,
"n": 40
}
},
"inputs": {
"sent": 40,
"seen": 40
},
"timeline": [
{
"t": 0,
"fps": 31,
"mbps": 0.13,
"content_p50": 21.5,
"content_p95": 27.1,
"bitrate": 300000,
"tier": 4,
"w": 960,
"net": null
},
{
"t": 1,
"fps": 31,
"mbps": 0.12,
"content_p50": 22.4,
"content_p95": 26.3,
"bitrate": 300000,
"tier": 4,
"w": 960,
"net": null
},
{
"t": 2,
"fps": 32,
"mbps": 0.13,
"content_p50": 18.9,
"content_p95": 24.8,
"bitrate": 300000,
"tier": 4,
"w": 960,
"net": null
},
{
"t": 3,
"fps": 31,
"mbps": 0.12,
"content_p50": 18.2,
"content_p95": 24.6,
"bitrate": 300000,
"tier": 4,
"w": 960,
"net": null
},
{
"t": 4,
"fps": 32,
"mbps": 0.13,
"content_p50": 21.0,
"content_p95": 25.6,
"bitrate": 300000,
"tier": 4,
"w": 960,
"net": null
},
{
"t": 5,
"fps": 32,
"mbps": 0.12,
"content_p50": 19.5,
"content_p95": 24.3,
"bitrate": 300000,
"tier": 4,
"w": 960,
"net": null
},
{
"t": 6,
"fps": 31,
"mbps": 0.12,
"content_p50": 22.2,
"content_p95": 25.6,
"bitrate": 300000,
"tier": 4,
"w": 960,
"net": null
},
{
"t": 7,
"fps": 32,
"mbps": 0.13,
"content_p50": 19.9,
"content_p95": 24.0,
"bitrate": 300000,
"tier": 4,
"w": 960,
"net": null
},
{
"t": 8,
"fps": 31,
"mbps": 0.21,
"content_p50": 22.7,
"content_p95": 26.9,
"bitrate": 300000,
"tier": 4,
"w": 960,
"net": null
},
{
"t": 9,
"fps": 31,
"mbps": 0.12,
"content_p50": 20.2,
"content_p95": 26.1,
"bitrate": 300000,
"tier": 4,
"w": 960,
"net": null
},
{
"t": 10,
"fps": 32,
"mbps": 0.12,
"content_p50": 18.2,
"content_p95": 23.2,
"bitrate": 300000,
"tier": 4,
"w": 960,
"net": null
},
{
"t": 11,
"fps": 31,
"mbps": 0.12,
"content_p50": 19.8,
"content_p95": 24.2,
"bitrate": 300000,
"tier": 4,
"w": 960,
"net": null
},
{
"t": 12,
"fps": 32,
"mbps": 0.13,
"content_p50": 19.2,
"content_p95": 24.8,
"bitrate": 300000,
"tier": 4,
"w": 960,
"net": null
},
{
"t": 13,
"fps": 32,
"mbps": 0.12,
"content_p50": 21.0,
"content_p95": 24.3,
"bitrate": 300000,
"tier": 4,
"w": 960,
"net": null
},
{
"t": 14,
"fps": 30,
"mbps": 0.13,
"content_p50": 20.4,
"content_p95": 96.7,
"bitrate": 300000,
"tier": 4,
"w": 960,
"net": null
},
{
"t": 15,
"fps": 32,
"mbps": 0.13,
"content_p50": 18.2,
"content_p95": 24.4,
"bitrate": 300000,
"tier": 4,
"w": 960,
"net": null
},
{
"t": 16,
"fps": 31,
"mbps": 0.12,
"content_p50": 18.4,
"content_p95": 25.1,
"bitrate": 300000,
"tier": 4,
"w": 960,
"net": null
},
{
"t": 17,
"fps": 32,
"mbps": 0.13,
"content_p50": 17.2,
"content_p95": 23.5,
"bitrate": 300000,
"tier": 4,
"w": 960,
"net": null
},
{
"t": 18,
"fps": 32,
"mbps": 0.19,
"content_p50": 19.4,
"content_p95": 25.2,
"bitrate": 300000,
"tier": 4,
"w": 960,
"net": null
},
{
"t": 19,
"fps": 31,
"mbps": 0.14,
"content_p50": 21.1,
"content_p95": 84.3,
"bitrate": 300000,
"tier": 4,
"w": 960,
"net": null
},
{
"t": 20,
"fps": 24,
"mbps": 0.08,
"content_p50": 20.8,
"content_p95": 25.1,
"bitrate": 300000,
"tier": 4,
"w": 960,
"net": null
}
],
"adapt": [],
"content_p50": 19.8,
"content_p95": 26.2,
"input_p50": 43.5,
"input_p95": 61.8,
"grades": {
"content_p50": "local",
"content_p95": "local",
"input_p50": "local",
"input_p95": "local",
"fps": "bad",
"late_pct": "bad",
"stall_max": "acceptable"
},
"source_fps": 64.5,
"cpu": {
"mac_agent_pct": 11.3,
"frame_viewer_pct": 35.9,
"frame_tailscaled_pct": 3.5,
"frame_sshd_pct": 0.6,
"frame_gamescope_pct": 7.5,
"frame_total_pct": 16.1
},
"viewer": "h264 software; ANGLE (Mesa, zink Vulkan 1.4(Turnip Adreno (TM) 750 (MESA_TURNIP)), OpenGL 4.6); raf 141 Hz",
"show_s": 4.68,
"panel": "valve.steam.desktopgame.2001639889",
"src": "test"
},
{
"scenario": "scroll",
"frames_sent": 937,
"frames_drawn": 937,
"frames_shown": 720,
"duration_s": 20.7,
"stages_ms": {
"capture": {
"p50": -3.9,
"p95": 0.9,
"p99": 8.2,
"n": 937
},
"queue": {
"p50": 0.0,
"p95": 16.3,
"p99": 20.1,
"n": 937
},
"encode": {
"p50": 6.4,
"p95": 7.2,
"p99": 9.3,
"n": 937
},
"socket": {
"p50": 0.1,
"p95": 0.2,
"p99": 0.2,
"n": 937
},
"network": {
"p50": 7.3,
"p95": 13.0,
"p99": 22.8,
"n": 937
},
"decode": {
"p50": 6.2,
"p95": 12.7,
"p99": 18.6,
"n": 937
},
"draw": {
"p50": 0.6,
"p95": 1.1,
"p99": 1.9,
"n": 937
},
"present": {
"p50": 2.9,
"p95": 18.5,
"p99": 24.5,
"n": 720
},
"content": {
"p50": 19.9,
"p95": 37.2,
"p99": 50.2,
"n": 937
},
"content_shown": {
"p50": 27.0,
"p95": 45.7,
"p99": 55.0,
"n": 720
}
},
"fps": 45.3,
"fps_shown": 34.8,
"late_pct": 28.63,
"stall_max": 99.3,
"stalls_over_100ms": 0,
"mbps": 7.59,
"keyframes": 2,
"size": "1920x1290",
"captured": 1215,
"viewer_never_drawn": 0,
"input_ms": {
"n": 0
},
"input_shown_ms": {
"n": 0
},
"input_parts_ms": {
"uplink": {
"n": 0
},
"mac": {
"n": 0
},
"back": {
"n": 0
}
},
"inputs": {
"sent": 0,
"seen": 0
},
"timeline": [
{
"t": 0,
"fps": 30,
"mbps": 4.01,
"content_p50": 31.9,
"content_p95": 52.3,
"bitrate": 4867140,
"tier": 2,
"w": 1920,
"net": null
},
{
"t": 1,
"fps": 30,
"mbps": 4.12,
"content_p50": 29.9,
"content_p95": 45.3,
"bitrate": 3807054,
"tier": 2,
"w": 1920,
"net": null
},
{
"t": 2,
"fps": 30,
"mbps": 3.26,
"content_p50": 31.8,
"content_p95": 52.5,
"bitrate": 3235995,
"tier": 2,
"w": 1920,
"net": null
},
{
"t": 3,
"fps": 30,
"mbps": 3.72,
"content_p50": 24.5,
"content_p95": 37.7,
"bitrate": 4238740,
"tier": 2,
"w": 1920,
"net": null
},
{
"t": 4,
"fps": 30,
"mbps": 5.09,
"content_p50": 25.2,
"content_p95": 39.6,
"bitrate": 5992063,
"tier": 2,
"w": 1920,
"net": null
},
{
"t": 5,
"fps": 31,
"mbps": 7.29,
"content_p50": 29.1,
"content_p95": 53.5,
"bitrate": 5526067,
"tier": 2,
"w": 1920,
"net": null
},
{
"t": 6,
"fps": 35,
"mbps": 4.58,
"content_p50": 25.3,
"content_p95": 32.4,
"bitrate": 6018152,
"tier": 1,
"w": 1920,
"net": null
},
{
"t": 7,
"fps": 46,
"mbps": 7.12,
"content_p50": 25.1,
"content_p95": 34.3,
"bitrate": 8412933,
"tier": 1,
"w": 1920,
"net": null
},
{
"t": 8,
"fps": 46,
"mbps": 8.25,
"content_p50": 27.1,
"content_p95": 40.3,
"bitrate": 10760191,
"tier": 1,
"w": 1920,
"net": null
},
{
"t": 9,
"fps": 47,
"mbps": 9.3,
"content_p50": 27.8,
"content_p95": 41.2,
"bitrate": 13717060,
"tier": 0,
"w": 1920,
"net": null
},
{
"t": 10,
"fps": 55,
"mbps": 8.98,
"content_p50": 16.5,
"content_p95": 21.7,
"bitrate": 14860800,
"tier": 0,
"w": 1920,
"net": null
},
{
"t": 11,
"fps": 51,
"mbps": 9.32,
"content_p50": 18.6,
"content_p95": 35.4,
"bitrate": 14860800,
"tier": 0,
"w": 1920,
"net": null
},
{
"t": 12,
"fps": 56,
"mbps": 9.53,
"content_p50": 15.4,
"content_p95": 26.2,
"bitrate": 14860800,
"tier": 0,
"w": 1920,
"net": null
},
{
"t": 13,
"fps": 55,
"mbps": 8.46,
"content_p50": 18.6,
"content_p95": 23.2,
"bitrate": 14860800,
"tier": 0,
"w": 1920,
"net": null
},
{
"t": 14,
"fps": 54,
"mbps": 9.82,
"content_p50": 15.9,
"content_p95": 28.4,
"bitrate": 14860800,
"tier": 0,
"w": 1920,
"net": null
},
{
"t": 15,
"fps": 54,
"mbps": 11.11,
"content_p50": 15.5,
"content_p95": 26.1,
"bitrate": 14860800,
"tier": 0,
"w": 1920,
"net": null
},
{
"t": 16,
"fps": 54,
"mbps": 9.79,
"content_p50": 17.4,
"content_p95": 25.3,
"bitrate": 14860800,
"tier": 0,
"w": 1920,
"net": null
},
{
"t": 17,
"fps": 57,
"mbps": 9.3,
"content_p50": 16.3,
"content_p95": 23.5,
"bitrate": 14860800,
"tier": 0,
"w": 1920,
"net": null
},
{
"t": 18,
"fps": 55,
"mbps": 8.3,
"content_p50": 19.5,
"content_p95": 31.0,
"bitrate": 7984791,
"tier": 0,
"w": 1920,
"net": null
},
{
"t": 19,
"fps": 51,
"mbps": 8.56,
"content_p50": 16.4,
"content_p95": 29.1,
"bitrate": 10220855,
"tier": 0,
"w": 1920,
"net": null
},
{
"t": 20,
"fps": 40,
"mbps": 7.13,
"content_p50": 18.3,
"content_p95": 23.9,
"bitrate": 12025604,
"tier": 0,
"w": 1920,
"net": null
}
],
"adapt": [],
"content_p50": 19.9,
"content_p95": 37.2,
"input_p50": null,
"input_p95": null,
"grades": {
"content_p50": "local",
"content_p95": "local",
"fps": "acceptable",
"late_pct": "bad",
"stall_max": "local"
},
"source_fps": 58.7,
"cpu": {
"mac_agent_pct": 6.6,
"frame_viewer_pct": 72.4,
"frame_tailscaled_pct": 8.1,
"frame_sshd_pct": 0.9,
"frame_gamescope_pct": 10.0,
"frame_total_pct": 24.2
},
"viewer": "h264 software; ANGLE (Mesa, zink Vulkan 1.4(Turnip Adreno (TM) 750 (MESA_TURNIP)), OpenGL 4.6); raf 141 Hz",
"show_s": 6.43,
"panel": "valve.steam.desktopgame.2001414593",
"src": "separate:9384"
},
{
"scenario": "type",
"frames_sent": 25,
"frames_drawn": 25,
"frames_shown": 25,
"duration_s": 14.7,
"stages_ms": {
"capture": {
"p50": -0.3,
"p95": 0.9,
"p99": 1.1,
"n": 25
},
"queue": {
"p50": 0.0,
"p95": 0.0,
"p99": 0.0,
"n": 25
},
"encode": {
"p50": 5.0,
"p95": 33.4,
"p99": 33.6,
"n": 25
},
"socket": {
"p50": 0.1,
"p95": 0.1,
"p99": 0.2,
"n": 25
},
"network": {
"p50": 6.1,
"p95": 14.9,
"p99": 24.4,
"n": 25
},
"decode": {
"p50": 3.0,
"p95": 10.5,
"p99": 11.4,
"n": 25
},
"draw": {
"p50": 0.6,
"p95": 1.0,
"p99": 1.1,
"n": 25
},
"present": {
"p50": 0.8,
"p95": 1.4,
"p99": 1.7,
"n": 25
},
"content": {
"p50": 12.6,
"p95": 41.0,
"p99": 46.3,
"n": 25
},
"content_shown": {
"p50": 13.9,
"p95": 41.8,
"p99": 46.6,
"n": 25
}
},
"fps": 1.7,
"fps_shown": 1.7,
"late_pct": 100.0,
"stall_max": 3099.2,
"stalls_over_100ms": 21,
"mbps": 0.04,
"keyframes": 4,
"size": "960x646",
"captured": 79,
"viewer_never_drawn": 0,
"input_ms": {
"p50": 41.3,
"p95": 60.3,
"p99": 68.6,
"n": 18
},
"input_shown_ms": {
"p50": 43.0,
"p95": 61.1,
"p99": 69.6,
"n": 18
},
"input_parts_ms": {
"uplink": {
"p50": 5.3,
"p95": 11.6,
"p99": 14.2,
"n": 18
},
"mac": {
"p50": 23.1,
"p95": 35.0,
"p99": 42.6,
"n": 18
},
"back": {
"p50": 11.8,
"p95": 19.2,
"p99": 22.7,
"n": 18
}
},
"inputs": {
"sent": 66,
"seen": 18
},
"timeline": [
{
"t": 3,
"fps": 1,
"mbps": 0.01,
"content_p50": 11.8,
"content_p95": 11.8,
"bitrate": 1224075,
"tier": 4,
"w": 960,
"net": null
},
{
"t": 4,
"fps": 1,
"mbps": 0.02,
"content_p50": 14.7,
"content_p95": 14.7,
"bitrate": 2091896,
"tier": 4,
"w": 960,
"net": null
},
{
"t": 5,
"fps": 4,
"mbps": 0.1,
"content_p50": 18.6,
"content_p95": 37.4,
"bitrate": 3071304,
"tier": 3,
"w": 1440,
"net": null
},
{
"t": 6,
"fps": 5,
"mbps": 0.13,
"content_p50": 4.2,
"content_p95": 19.2,
"bitrate": 4031277,
"tier": 3,
"w": 1440,
"net": null
},
{
"t": 7,
"fps": 6,
"mbps": 0.04,
"content_p50": 12.0,
"content_p95": 46.3,
"bitrate": 300000,
"tier": 4,
"w": 960,
"net": null
},
{
"t": 10,
"fps": 1,
"mbps": 0.01,
"content_p50": 12.2,
"content_p95": 12.2,
"bitrate": 300000,
"tier": 4,
"w": 960,
"net": null
},
{
"t": 12,
"fps": 1,
"mbps": 0.02,
"content_p50": -1.2,
"content_p95": -1.2,
"bitrate": 842857,
"tier": 4,
"w": 960,
"net": null
},
{
"t": 14,
"fps": 1,
"mbps": 0.02,
"content_p50": 14.4,
"content_p95": 14.4,
"bitrate": 2091896,
"tier": 4,
"w": 960,
"net": null
}
],
"adapt": [],
"content_p50": 12.6,
"content_p95": 41.0,
"input_p50": 41.3,
"input_p95": 60.3,
"grades": {
"content_p50": "local",
"content_p95": "acceptable",
"input_p50": "local",
"input_p95": "local"
},
"source_fps": 5.4,
"cpu": {
"mac_agent_pct": 1.2,
"frame_viewer_pct": 5.0,
"frame_tailscaled_pct": 1.5,
"frame_sshd_pct": 0.2,
"frame_gamescope_pct": 4.7,
"frame_total_pct": 8.5
},
"viewer": "h264 software; ANGLE (Mesa, zink Vulkan 1.4(Turnip Adreno (TM) 750 (MESA_TURNIP)), OpenGL 4.6); raf 140 Hz",
"show_s": 5.12,
"panel": "valve.steam.desktopgame.2001910179",
"src": "separate:9510"
}
]
}
bench/results/2026-09-28-a3c6e5c-dirty-adapt-clean2.json
+1116
View File
File diff suppressed because it is too large. Load diff
bench/results/2026-09-28-a3c6e5c-dirty-baseline-fixed.json
+1124
View File
File diff suppressed because it is too large. Load diff
bench/results/2026-09-28-a3c6e5c-dirty-baseline.json
+1124
View File
File diff suppressed because it is too large. Load diff
docs/frame-control.md
+4
View File
@@ -72,6 +72,10 @@ counts them while they run.
Runtime picked from the program's header), listed under **Sideloaded titles**
with Launch and Remove; see [sideloading.md](sideloading.md). Send typed text, or your computer's clipboard, to the
Frame clipboard.
- **Mac in the headset** (macOS): show any Mac window, or a whole screen, as
its own panel in the headset. Place it with the SteamVR dashboard, click and
scroll with the laser, and type on the Mac. Streams hardware H.264 through
an SSH tunnel; see [mac-in-headset.md](mac-in-headset.md).
- **Flatpaks**: install and remove them (quick picks: Moonlight, Firefox, VLC,
Remmina).
- **One-click tools**: SSH or SFTP in a terminal window, Steam Link, and remote
docs/how-the-frame-works.md
+4 -1
View File
@@ -52,6 +52,9 @@ Lepton (Android 11, podman container "lepton-dev") ← its own panel, app 305600
| **T3 Code desktop runs natively.** The stock release `T3-Code-0.0.42-arm64.AppImage` in `~/Applications/T3CodeDesktop/` starts with no extra setup: glibc 2.39, `libfuse.so.2`, GTK 3, NSS and libsecret are on the image. `panel-on-frame.sh --name t3code-desktop -- '~/Applications/T3CodeDesktop/T3-Code.AppImage'` gives it its own panel (`valve.steam.desktopgame.2000281357`, `--ozone-platform=x11`). Its bundled server listens on `127.0.0.1:3773` and shows up in onboarding as the `frame` computer, with `passwordStore: gnome-libsecret`. The image has no agent CLI and no `node`. Agents run through the LAN CLIProxyAPI (`llm-proxy.lan:8317`, which resolves on the Frame). Claude Code 2.1.283 comes from `claude.ai/install.sh`, and Codex 0.157.1 from the `codex-aarch64-unknown-linux-musl` release tarball, both into `~/.local/bin`. `with-cliproxy` and a mode-600 `~/.config/cliproxyapi/secrets.env` are copied from the Mac. The wrappers `claude-cliproxy` and `codex-cliproxy` (a `-c model_provider=cliproxy`, `wire_api="responses"`, `env_key="CLIPROXY_API_KEY"`) are set as `providers.claudeAgent.binaryPath` and `providers.codex.binaryPath` in `~/.t3/userdata/settings.json`, and T3 picked that up without a restart. Through the wrappers, `claude auth status` reports `loggedIn: true` (`oauth_token`), and both CLIs answered a prompt with `kimi-k3`. `gamescopectl screenshot` captured another layer (the Lepton T3 app) rather than this panel. `DISPLAY=:0 xwd -id <win>` piped to `ffmpeg` captures the window itself (1920×1080). **Verified 2026-09-26**, BUILD_ID 20260922.6101926. | Running T3 Code as a host on the Frame |
| Power actions need `sudo`, which asks for the Developer Mode password over SSH. | Frame Control's power buttons |
| **SSH server:** OpenSSH 9.7p1. It offers `publickey,password` (keyboard-interactive is off, PAM on) and also asks `userdbctl ssh-authorized-keys` for keys. OpenSSH ≥ 8.8 rejects SHA-1 `ssh-rsa` signatures by default, so a client whose RSA support is SHA-1 only (the Swift library Citadel, for one) can't log in with the RSA key that devkit pairing installs; use ed25519 (**inferred** from OpenSSH defaults). **Verified 2026-09-27**, BUILD_ID 20260922.6101926. | [iphone.md](iphone.md), `ui/frame_connect.py` |
| **A Chromium app window on gamescope's `:0` with its own `STEAM_GAME` becomes a panel, even when started over SSH.** `chromium-xr/chrome --ozone-platform=x11 --app=URL --window-size=1280,720` got a 1920×1080 window, and tagging it produced `[Overlays] Created: valve.steam.desktopgame.<id>` in `~/.local/share/Steam/logs/vrwebhelper_systemui.txt`. Chromium XR decoded a 1080p H.264 WebCodecs stream from the Mac at about 60 fps. XTest events sent to `:0` (libXtst through Python ctypes) did not reach the page. The Frame has `libXtst`, `xprop`, `xwininfo`, `curl` and the `C.utf8` locale. **Verified 2026-09-28**, BUILD_ID 20260925.6191901. | [mac-in-headset.md](mac-in-headset.md) |
| **Streaming video into a panel: what the Frame adds.** Chromium XR decodes H.264 in **software** (1920×1290 at about 9 Mbit/s took 4–6 ms per frame); its GL is ANGLE → zink → Turnip on the Adreno 750, and it logs `GetVSyncParametersIfAvailable() failed`. An idle page's `requestAnimationFrame` runs at about 140 Hz; when the headset is worn or woken, SteamVR picks the 4320×2160 @ 144 Hz mode. **An unworn headset throttles panel apps** whatever they draw: a local canvas page ran at 58 fps for about 6 s, then 28, then about 15 once in standby (vrserver logs `entering standby`, with `power.pauseCompositorOnStandby 1`). `vrcmd --handlewakeup` gave 137–144 fps for about 2 s, then 36. So a panel's frame rate and compositor delay can only be measured while it's worn. `tailscaled` runs with `--tun=userspace-networking` and cost about 8% of a core at 9 Mbit/s (0.5% over the LAN address). Wi-Fi power saving is on (`iw … get power_save`). **Verified 2026-09-28**, BUILD_ID 20260925.6191901. | [mac-in-headset.md](mac-in-headset.md#measuring) |
| **USB-C networking.** Plugged into a Mac, the Frame is a USB network device: macOS names the port "Steam Frame" (here `en9`, 10.86.200.234/29), and the Frame's `usb0` is 10.86.200.233. Ping is about 0.9 ms, and SSH works with the usual host key (`-o HostName=10.86.200.233 -o HostKeyAlias=<tailscale name>`). Steam's Remote Play discovery also broadcasts over it. **Verified 2026-09-28**, BUILD_ID 20260925.6191901. | Frame Control's Mac stream uses it when present ([mac-in-headset.md](mac-in-headset.md)) |
| **Tools on the image:** Python 3.12.3, `ffmpeg`, `openssl`, `curl`, `rsync`, `zip`/`unzip`, `flatpak`, `wpctl`, `podman`. **No `adb`.** `steamos` is uid 1000, in `wheel`, and sudoers has `%wheel ALL=(ALL) ALL`, so `sudo -S` takes the Developer Mode password on stdin. **Verified 2026-09-27.** | Running Frame Control's server on the Frame (`FRAME_LOCAL=1`, [iphone.md](iphone.md)) |
| **Each Lepton instance is a podman container** named `lepton-steamlaunch-<instance id>`, labelled with its ADB port (`podman ps --format '{{.Names}} {{.Labels.adb_port}}'`). `podman exec <container> /system/bin/sh -c '…'` runs Android's shell inside it with no adb at all (used for `pidof` and `logcat` by the app tester). Running `wm size`/`wm density` that way is untested. **Verified 2026-09-27.** | `ui/frame_android.py`, the iPhone app's display settings |
| **Asleep means off the network.** In standby the Frame stops answering on its LAN address, `frame.local` and Tailscale alike (`Host is down`, `No route to host`, timeouts), and ping fails. It was unreachable for about 2.5 hours until woken. Nothing over SSH can wake it. **Verified 2026-09-27.** | Frame Control's offline banner and retries |
@@ -59,7 +62,7 @@ Lepton (Android 11, podman container "lepton-dev") ← its own panel, app 305600
| **Battery at full on a charger** can read `Discharging` at about 0 W (for example 99 %, 0.0 W, USB-C PD 18 W). Treat under 0.5 W on a charger as "not charging", not "draining". **Verified 2026-09-27.** | Frame Control's battery card |
| **The OS image is downloadable.** Valve's recovery images for the Frame are at `https://steamdeck-images.steamos.cloud/recovery/`. The root filesystem inside is btrfs, and it runs as an SSH test target on ARM64 Linux without the headset (`tests/frame-container/frame-image.sh`). **Verified 2026-09-27.** | [recovery-and-images.md](recovery-and-images.md) |
| **Boot / recovery menu.** Hold Power ~10 s until the LED goes off, then power on while holding the **AUX button on top of the Power button** (not the volume keys) until a text menu appears. Entries: `Current` (SteamOS-A/B + build), `Previous` (the other A/B slot), `Boot from USB`, `Repair Steam Installation`, `Erase User Data` (factory reset), `ADB mode`, `Battery Ship Mode`. It auto-boots `Current` after a ~15 s countdown. **Volume Up/Down (left side) move, AUX (right side) selects.** For a boot loop, Valve says pick `Previous` (keeps user data); then `Repair Steam Installation`; `Erase User Data` wipes `~` (SSH keys, Tailscale, Flatpaks, T3 setup). Last resort is a full re-image, two ways: (1) USB: write `steamframe-oobe-repair-<build>.img.bz2` to an 8 GB+ USB-C stick (Balena Etcher on the Mac), pick `Boot from USB`, then use "Wipe Device & Install SteamOS" / "Repair SteamOS" (keeps games and personal content) from the recovery desktop; (2) cable/EDL: `steamframe-oobe-repair-qdl-<build>.tar.gz`, run `flash.sh` (Linux) or `flash.cmd` (Windows), then with the Frame off for 10 s hold Power + Vol Up + Vol Down for 10 s and plug it in; it reflashes and reboots. Both images: `https://steamdeck-images.steamos.cloud/recovery/` (build 20260922.5153644, 0.3.0, 3.8 GiB each, no published checksums); local copies in `~/Downloads/steam-frame-recovery/`. File names, checksums and what's inside: [recovery-and-images.md](recovery-and-images.md). Source: Valve's [SteamOS Recovery FAQ](https://help.steampowered.com/en/faqs/view/1B71-EDF2-EB6D-2BB3) and [Installation and Repair FAQ](https://help.steampowered.com/en/faqs/view/65B4-2AA3-5F37-4227), plus a menu photo in [EloiStree/HelloSteamFrame#9](https://github.com/EloiStree/HelloSteamFrame/issues/9). **Inferred** (Valve docs, 2026-09-26); not yet tried on our Frame. | Recovering from a boot loop |
| **Boot loop cause: the SteamVR health check.** `steamvr.service` runs `/usr/share/deckard/steamvr-health-check`, which appends `frog:glasses:` to `$XDG_RUNTIME_DIR/steamvr-short-session-tracker` on every failed or <10 s SteamVR run. At 3 it runs `steam-health-check --repair-now`, which **deletes all of `~/.local/share/Steam` (games, login, Developer Mode) and `~/.steam`**, keeping only `registry.vdf`. At 4 it also tries `steamos-bootconf set-mode reboot-other` (fails as the user: `bootenv: Permission denied`). SteamVR normally fails 1–2 times per boot while it waits for the Steam client (`SteamAPI_InitEx failed … Steam is probably not running`, then `fatal stalled cross-thread pipe`). Once Steam has been wiped, it has to re-download a ~210 MB client on every boot, so SteamVR keeps failing, Steam keeps getting wiped and the Frame reboots, in a loop. Also, the Steam updater can deadlock at `Installing update...` (main process blocked writing to the `-child-update-ui` process, which is stuck in `drm_syncobj_array_wait_timeout`). Killing only the `-child-update-ui` process lets the install finish (`package/*.installed` appears). **Fix without sudo:** over USB-C ADB (`adb -s frame shell` works as `steamos` while the Frame is looping; SSH is refused once Developer Mode is lost), truncate both `/run/user/1000/steam{,vr}-short-session-tracker` files and `chmod 444` them (the health check then logs `Permission denied` and does nothing; this is tmpfs, so it resets on reboot). Unstick the updater if needed, let Steam finish installing, then hold Power 10 s and start the Frame normally. `systemctl reboot` over ADB needs interactive auth. After the fix, sign in to Steam and turn Developer Mode back on. **Verified 2026-09-26**, BUILD_ID 20260922.6101926, slot B (clean boot: 0 SteamVR failures, SSH and Tailscale back). | Diagnosing a boot loop |
| **Boot loop cause: the SteamVR health check.** `steamvr.service` runs `/usr/share/deckard/steamvr-health-check`, which appends `frog:glasses:` to `$XDG_RUNTIME_DIR/steamvr-short-session-tracker` on every failed or <10 s SteamVR run. At 3 it runs `steam-health-check --repair-now`, which **deletes all of `~/.local/share/Steam` (games, login, Developer Mode) and `~/.steam`**, keeping only `registry.vdf`. At 4 it also tries `steamos-bootconf set-mode reboot-other` (fails as the user: `bootenv: Permission denied`). SteamVR normally fails 1–2 times per boot while it waits for the Steam client (`SteamAPI_InitEx failed … Steam is probably not running`, then `fatal stalled cross-thread pipe`). Once Steam has been wiped, it has to re-download a ~210 MB client on every boot, so SteamVR keeps failing, Steam keeps getting wiped and the Frame reboots, in a loop. Also, the Steam updater can deadlock at `Installing update...` (main process blocked writing to the `-child-update-ui` process, which is stuck in `drm_syncobj_array_wait_timeout`). Killing only the `-child-update-ui` process lets the install finish (`package/*.installed` appears). **Fix without sudo:** over USB-C ADB (`adb -s frame shell` works as `steamos` while the Frame is looping; SSH is refused once Developer Mode is lost), truncate both `/run/user/1000/steam{,vr}-short-session-tracker` files and `chmod 444` them (the health check then logs `Permission denied` and does nothing; this is tmpfs, so it resets on reboot). Unstick the updater if needed, let Steam finish installing, then hold Power 10 s and start the Frame normally. `systemctl reboot` over ADB needs interactive auth. After the fix, sign in to Steam and turn Developer Mode back on. **Verified 2026-09-26**, BUILD_ID 20260922.6101926, slot B (clean boot: 0 SteamVR failures, SSH and Tailscale back). **Seen again 2026-09-28** on BUILD_ID 20260925.6191901, beta client 1790377368. The Frame rebooted by itself while off the network. At 21:13 the check tried `reboot-other` (`bootenv: Permission denied`) and reset the unpacked Steam install, and the tracker had 15 entries. The tracker fix above, applied over SSH, stopped the resets (the checks then log `Permission denied`), but Steam itself kept exiting about 17 s after each start (34 restarts). Cause not found yet. | Diagnosing a boot loop |
## Debug recipes
docs/mac-in-headset.md
+470
View File
@@ -0,0 +1,470 @@
# Mac in the headset
Frame Control can show any Mac window, or a whole Mac screen, as its own panel
in the Steam Frame. You place each panel anywhere in the room with the SteamVR
dashboard. The laser clicks and drags, the thumbstick scrolls, and you type on
the Mac's own keyboard. Find it under **Tools → Mac in the headset** (macOS
only).
The confidence labels are the same as in [ssh.md](ssh.md).
## Why this design
First-party options come first, as the repo's rule asks, with the reason
each one was or wasn't chosen. The full list for every device is in
[streaming.md](streaming.md#first-party-options-and-why-they-do-or-dont-fit).
Checked 2026-09-28.
| Goal | First-party option | Chosen? | Why |
|---|---|---|---|
| One Mac screen in the headset | **Apple Screen Sharing** (VNC) → Remmina (Remmina 1.4.43 is already installed on this Frame) | Kept as the fallback (`panel-on-frame.sh mac-screen`) | It's the closest to first-party and needs nothing new. But VNC sends compressed tiles rather than video, so moving content is slow: noticeable lag even on a good 5 GHz link (**verified** 2026-09-27, see [streaming.md](streaming.md)), and the Mac's pointer isn't in the picture without a helper. It shows only whole screens |
| One Mac screen | **Steam Remote Play**, Mac as host (Valve) | For Mac games only; see [Steam's own streaming](#steams-own-streaming) | The Mac's and the Frame's Steam clients already find each other (**verified**). But Remote Play streams a game (the whole desktop only while the game is out of focus, untested from a Mac), never single windows, and a Mac can't host the Frame's VR streaming |
| One Mac screen | **AirPlay** (Apple) | No | Apple licenses AirPlay receivers only to TV and speaker makers, and nothing official runs on Linux. UxPlay is an unofficial receiver, and it mirrors a whole screen, not single windows |
| One Mac screen | **Sidecar / Mac Virtual Display** (Apple) | No | These work only with an iPad or Apple Vision Pro |
| **Each Mac window as its own panel** | None | – | No first-party way does this: Apple's per-app streaming is only for Vision Pro, and Valve's desktop streaming needs a Windows SteamVR host. So Frame Control does it itself |
| Mac keyboard and trackpad driving the headset | **Bluetooth HID** | No | macOS can't act as a Bluetooth keyboard or mouse. A real Bluetooth keyboard paired with the Frame still works |
| Mac keyboard and trackpad | **KDE Connect** (KDE) | No | The Frame has no `kdeconnectd` and it isn't on Flathub. Its Mac app has no keyboard or mouse sharing (**inferred**), and on Wayland it can only reach the desktop panel |
| Mac keyboard and trackpad | **xrdp** (Valve, Developer Mode) | No | It runs a separate Linux session that you view on the Mac. It isn't the headset's view, and it doesn't carry input the other way |
What that leaves is our own stream: nothing to install on the Mac or the
Frame, and whole screens or single windows. Here the Mac's own keyboard and
trackpad need no forwarding, because the windows are still on the Mac. The
laser is the only input that has to be sent back.
Other routes that were compared:
| Option | One screen | Each window | Speed | Verdict |
|---|---|---|---|---|
| Sunshine → Moonlight | ✓ | – | Good | Sunshine's macOS support is still experimental ([discussion #777](https://github.com/orgs/LizardByte/discussions/777)), and it captures whole screens only |
| Virtual Desktop, Immersed | – | – | – | No Frame client as of September 2026 |
| **Frame Control's own stream** | ✓ | ✓ | Hardware H.264, sending only changed frames | **Built** |
To type into VR surfaces other than these panels (SteamVR's dashboard,
games), the Frame supports a uinput keyboard and mouse without sudo
(verified 2026-09-27: `steamos` is in `input`, and `/dev/uinput` is
`root:input 660`). That's a separate feature, not part of this one.
## Steam's own streaming
The Frame is built around Steam streaming, so this was checked first
(2026-09-28). It fits Mac games, not Mac windows.
- **VR streaming from the Mac: no.** The Frame streams VR from a PC running
SteamVR ("Steam Link" with foveated streaming). SteamVR dropped macOS in
2020, and Valve lists PCs, laptops, Steam Deck and Steam Machine as
hosts, never a Mac (**documented**:
[UploadVR](https://www.uploadvr.com/steamvr-drops-mac-support/),
[Road to VR](https://roadtovr.com/steam-frame-game-certification-specs/)).
- **Flat Remote Play from the Mac: probably, for Steam games.**
- Steam on this Mac has streaming on, and the two Steam clients already
see each other. The Frame's `remote_connections.txt` shows it
connecting directly to "Alexs-MacBook-Pro-7" at 192.168.1.211:27036,
and the Mac's shows the Frame connecting over Wi-Fi and over the USB-C
link (**verified** in both clients' logs).
- Whether a stream then starts, and how a flat game looks in the headset
(reviews describe a theater screen), is **not tested yet**. The Frame's
Steam was crash-looping during this session (below).
- Mac-hosted Remote Play has a long-standing report of the stream
closing as the game loads
([Steam forum](https://steamcommunity.com/groups/homestream/discussions/1/574921459914429988/),
**reported**).
- **The Mac desktop through Steam: untested; single windows: no.** Valve
says Remote Play shows the host's desktop when the game loses focus
([Steam Remote Play FAQ](https://help.steampowered.com/en/faqs/view/0689-74B8-92AC-10F2),
**documented**), so a whole Mac screen may be reachable by starting a
game, then switching away from it. Nobody has tried that from a Mac
host. It would still be one screen in one panel: Remote Play has
nothing like one panel per Mac window, so Frame Control's own stream
stays the way to see separate windows.
- **Steam has a "stream desktop" call, and it pairs with a Mac
(verified 2026-09-28).** In the Remote Play device list, the Frame's
Steam UI calls `SteamClient.RemotePlay.StartDesktopStream(<client id>)`
for a connected device. Called over CDP with the Mac's client ID, it made
the Mac's Steam show "Authorize Device" and ask for a 4-digit code shown
on the Frame. Once the code was entered, the Mac logged
`k_ERemoteDeviceAuthorizationSuccess`. No stream started in that attempt,
and a second attempt, now that the device is authorized, is the next
test. If it streams the Mac's desktop, that's a whole-screen option built
into Steam: one panel, Valve's encoder and transport. It still wouldn't
give each window its own panel.
- **What Steam's work did give us: the USB-C link.** Plugged into the Mac,
the Frame appears as a network port called "Steam Frame". Steam's Remote
Play discovery uses it, and so does Frame Control's stream now (see
"USB-C, when it's plugged in" below).
Next steps, once Steam on the Frame is healthy:
1. Call `StartDesktopStream` again now that the Frame is authorized, and
compare its latency and sharpness with Frame Control's stream of the same
screen.
2. Stream the one Mac game installed here (Fortune Mill) from the Frame's
library, over Wi-Fi and over USB-C.
3. Record whether it starts, how it's shown, and its latency. Steam's
streaming overlay shows this; our benchmark can't measure it.
4. While streaming, switch away from the game on the Mac, and see whether
the Mac's desktop appears in the headset, and whether its keyboard and
pointer work.
5. If it works, Frame Control's Games page could offer "Stream from the
Mac" for Mac-installed games.
## How it works
```
Mac Frame
ScreenCaptureKit (one window or display)
→ VideoToolbox H.264 (hardware, low-latency,
no B-frames)
→ frame-mac-view, 127.0.0.1 ──ssh -R──→ 127.0.0.1:479xx
→ Chromium app window per stream
(WebCodecs decode), on gamescope's
X display, tagged STEAM_GAME
→ its own SteamVR panel
← CGEvent (clicks, drags, wheel, keys) ←──── pointer, wheel and key events
```
- **The agent** is `mac/bin/frame-mac-view`, built from `mac/frame-mac-view`
(Swift, no dependencies; `build.sh`). Frame Control's server starts it on
first use and stops it on quit.
- It captures with ScreenCaptureKit, which sends frames only when something
changes, so idle windows cost nothing.
- It encodes in hardware with VideoToolbox's low-latency rate control (plain
real-time mode where that's unavailable).
- While anyone is watching, it keeps the Mac's display awake. A sleeping
display isn't drawn, so there would be nothing to capture.
- **The link** is an `ssh -R` tunnel on its own connection. It's encrypted and
works anywhere `ssh frame` works, Tailscale included, with no firewall
changes on the Mac. If the headset sleeps or the network drops, Frame
Control reopens the tunnel on the same port, and open viewers reconnect by
themselves.
- **USB-C, when it's plugged in.** Connected to the Mac by cable, the Frame
is also a USB network device: macOS lists a network port called "Steam
Frame", and the Frame's `usb0` answers in under 1 ms. Frame Control
checks for it each time it opens the tunnel and uses it when it's there,
with the Frame's usual SSH host key. Otherwise it uses the normal path.
`FRAME_MACVIEW_USB=0` turns this off. **Verified** 2026-09-28, in two
interleaved pairs of runs:
| | USB-C | Wi-Fi (Tailscale) |
|---|---|---|
| test: content p50 / p95 | 6.9–7.3 / 8.4–8.8 ms | 9.8–10.1 / 12.1–12.3 ms |
| test: click to drawn p50 | 16.6–16.9 ms | 26.4–27.8 ms |
| scroll: content p95 | 23.0–23.5 ms | 31.4–36.9 ms |
| scroll: late frames | 3.2–3.3% | 4.7–7.0% |
(`bench/results/2026-09-28-*-usb1.json`, `-usb2`, `-wifi1`, `-wifi2`.)
- **Access.**
- Frame Control's own key never leaves the Mac.
- Each viewer is opened with a **single-use ticket**. It's tied to one
window or display and expires after a minute. It's spent as soon as the
viewer confirms it has received its reconnect key. Until then, a retry
gets the same key, so a connection lost at that moment doesn't strand
the viewer. Stop revokes tickets that haven't been used yet.
- After that, the viewer holds a reconnect key for that one source, in
memory only. **Stop** revokes it.
- Remaining risk: a program running as `steamos` on the Frame could read a
ticket from Chromium's command line in the first second or so and use it
first. That gets it the one source being opened, not the Mac, and the
real viewer would then fail to connect. Android apps in Lepton run in
their own podman container, so they shouldn't see the Frame's process
list (inferred, not checked).
- **The viewer** is `ui/mac-view.html`, served by the agent. It opens on the
Frame as a Chromium app window, preferring Chromium XR (`~/chromium-xr`,
built with H.264) over Flathub Chromium.
- The page puts `[fcNNNNN]` in its title. The launcher finds the window by
that tag and sets `STEAM_GAME` to a stable id per source, which gives it
its own panel (see [panels.md](panels.md)). The same Mac window gets the
same panel id each time.
- It decodes with WebCodecs. If it falls behind, it skips to the next
keyframe instead of showing old frames late.
- It falls back to JPEG stills (**Compatible** quality) where H.264 isn't
available.
- **Flow control.** The agent never lets frames queue up anywhere on the
way. It skips capture frames *before* encoding, so no reference frame goes
missing, and it lowers the bitrate, then the frame rate, then the size, to
fit the link (see [Adapting to the network](#adapting-to-the-network)).
- **Input.**
- A click on a window's panel brings that Mac window to the front
(Accessibility API), then clicks at the same point. Double clicks, right
clicks, drags and the wheel work too.
- Keys typed into the panel are sent as Mac key codes. Any keys or buttons
still held down are released if the viewer loses focus or disconnects, or
when the stream stops. Characters the key
table doesn't know, such as those from other keyboard layouts, are typed
as text.
- The Mac's own keyboard and trackpad keep working as normal. Click a panel
with the laser, then type on the Mac.
## Permissions (Mac)
- **Screen Recording**, to see windows. Without it, the card asks for it.
- **Accessibility**, so input from the headset reaches the Mac. Without it the
stream still works, and the viewer says clicks won't go through.
Both are granted to Frame Control. After granting, press **Refresh**, which
restarts the helper so it picks them up. The app is ad-hoc signed, so macOS
may ask again after an update.
## Quality settings
| Setting | Long side | fps | Codec | Use |
|---|---|---|---|---|
| Sharp | 2560 | 60 | H.264, ~0.14 bits/pixel | Text-heavy windows on a strong link |
| Balanced (default) | 1920 | 60 | H.264, ~0.1 bits/pixel | Most things |
| Light | 1280 | 30 | H.264 | Weak Wi-Fi or Tailscale off the LAN |
| Compatible | 1280 | 20 | JPEG | A Frame browser without H.264 |
## Measuring
Every frame carries a sequence number, and the agent records its journey on
the Mac's clock (`Sources/Stats.swift`):
| Stage | From → to |
|---|---|
| capture | the Mac composited it (ScreenCaptureKit's display time) → the agent got it |
| queue, encode | → encoding started → the encoder finished |
| network | → the viewer received it |
| decode, draw | → WebCodecs decoded it → it was drawn on the page's canvas |
| present | → the page's next animation frame |
- **Clock sync.** The viewer syncs its clock to the Mac's the way NTP does:
it pings over the stream's own WebSocket and keeps the sample with the
shortest round trip. It then reports, in Mac time, when each frame arrived
(right away, so the agent can pace itself) and when it was decoded and
drawn (in batches every 250 ms).
- **Input.** The first frame captured after a click or key carries that
event's id. So input latency is the viewer's event → injected on the Mac →
the first frame after it → drawn in the headset.
- **Where to see it.**
- `GET /stats` (key required) returns every frame and input record.
- `/status` includes a two-second summary, which Frame Control's card
shows next to each live stream.
- In the headset, add `?stats=1` to the viewer or press
Ctrl+Alt+Shift+S for an overlay.
- **The benchmark.** `scripts/macview-bench.py` runs fixed scenarios on the
real Frame, from the Mac, with nobody wearing the headset:
- **test** is the moving test pattern.
- **scroll** is a Chrome page on its own display scrolling at 240 pt/s,
which gives about 9 Mbit/s of real 1920×1290 video.
- **type** types into a Chrome text box, first fast and then with pauses.
It writes `bench/results/<date>-<commit>-<label>.json`, compares two
results, and runs interleaved A/B tests between agent settings (`ab`).
Wi-Fi changes from minute to minute, so single runs at different times
aren't comparable. Throttled links come from a shaping relay on the Mac,
which needs no sudo (`--net 50@0,3@8,50@16` means 50 Mbit/s, then 3 from
8 s, then 50 from 16 s).
- **What's graded.** "Content" runs from when the Mac composited a frame
(or when ScreenCaptureKit delivered it, if that was earlier) to when it was
drawn in the viewer. The Frame's compositor adds its own delay after that.
That part is reported, but not graded: an unworn Frame throttles panels to
about 36 fps after a few seconds, and to 15 fps in standby, whatever they
draw. This was **verified** with a local canvas page that ran on the Frame
with no network involved (`bench/pages/present.html`). The compositor's
share needs a run with the headset worn.
Targets: content p50 ≤ 25 ms (p95 ≤ 40), click to photon p50 ≤ 50 ms
(p95 ≤ 70), 60 fps with ≤ 1% late frames, no stall over 100 ms, and adapting
to a new link rate within 1 s.
Baseline on 2026-09-28 (**verified**, home Wi-Fi, Tailscale, Balanced,
`bench/results/2026-09-28-a3c6e5c-dirty-baseline-fixed.json`; ms p50/p95):
| Scenario | Content | Input to drawn | fps drawn | Notes |
|---|---|---|---|---|
| test (1280×720) | 9.9 | 28.8 / 39.0 | 59 | encode 4.0, network 4.0, decode 1.3 |
| scroll (1920×1290) | 14.7 | – | 50.4 | encode 6.7, decode 6.4 (software), 9.4 Mbit/s, 16% late |
| type (1920×1290) | 16.7 | 51.2 / 73.2 | – | most of the input time is the Mac app reacting |
After this work, with the controller on (**verified**, same setup,
`bench/results/2026-09-28-9e4dcdd-final.json`; ms p50/p95). Content is
now measured from the earlier of display time and delivery, which adds
about 5 ms to scroll compared with the baseline's way of measuring:
| Scenario | Content | Input to drawn | fps drawn | Grades |
|---|---|---|---|---|
| test | 10.5 / 16.7 | 29.6 / 36.3 | 60 | all within target |
| scroll | 19.8 / 27.4 | – | 55.9 | fps, late frames (4.8%) and worst gap (222 ms) only "acceptable": Wi-Fi stalls (the Mac captured 57 fps in this run; earlier runs got 46–53 from virtual displays) |
| type | 15.0 / 21.4 | 43.0 / 60.5 | – | all within target |
Of the targets, click to photon is met without the Frame's compositor (the
headset has to be worn to measure its share), and so is content latency.
The frame-rate and no-stall targets aren't yet met while scrolling.
**Worn** (**verified** 2026-09-28 22:00, `vrcmd --stats` activity level 1,
home Wi-Fi over Tailscale, `bench/results/2026-09-28-39afb23-worn.json`;
ms p50/p95):
| Scenario | Content | Input to drawn | fps drawn | What happened |
|---|---|---|---|---|
| test | 10.8 / 15.3 | 25.7 / 34.4 | 58 | as unworn |
| scroll | 22.2 / 141 | – | 37.7 | Wi-Fi queued 56–364 ms and held frames for 220–270 ms; the controller went to 2.6 Mbit/s and 45 fps |
| type | 13.5 / 24.3 | 56.8 / 106 | – | slower replies than unworn (43 / 60) |
The Frame's CPU wasn't the limit: about 27% in total, and the viewer took
45% of one core. The link to a headset on someone's head is much rougher
than to one lying still. The adaptation keeps latency bounded there, but
the frame rate drops. The USB-C cable avoids Wi-Fi entirely. Frames
actually shown were still about 39 fps for the test pattern while worn, so
the unworn throttling isn't the whole story. The cause is **unknown**.
What was learned (all **verified**, unless marked):
- The biggest costs are encoding (4–7 ms), network (4–6 ms), and decoding
on the Frame. Chromium XR on the Frame decodes H.264 in **software**.
- Wi-Fi alone stalls for 240–580 ms now and then, over Tailscale and over
the LAN alike. Over the LAN (`--host 192.168.1.237`) latency was no
better, but the Frame used about 8% less CPU, because tailscaled runs in
userspace there.
- With no controller, a link that slows down queues without limit. In one
run, frames arrived 1.9 s late, and at worst 9.4 s late.
- Tried, and no help, so not kept: VideoToolbox options (require hardware,
no frame delay, prioritise speed, a hard data-rate cap), Chromium flags
(`--disable-gpu-vsync`, `--disable-frame-rate-limit`,
`--use-angle=vulkan`), and a 120 Hz virtual display.
- Inconclusive, so off by default: keeping the Frame's Wi-Fi awake during
typing (`FRAME_MAC_VIEW_WARM=40`, a tiny message every 40 ms for 5 s
after input). Over three interleaved runs each, input p50 went 44 → 47 ms
and p95 92 → 71 ms, and the ranges overlapped widely
(`…-ab-keepwarm.json`). In that run, and in the baseline's typing, the
harness typed spaces as "+" (a URL-encoding bug, since fixed), so they
went through the text path rather than as space keys.
- Pointer moves now go out on an 8 ms timer, not on the page's next
animation frame, which an unworn Frame slows to 15–36 Hz. This is
**inferred** to help dragging; the benchmark has no drag scenario yet.
- Kept: the encoder's timestamps never jump more than two frame intervals.
Before this, the first frame after a pause got a quarter of a second's
bit budget, and one P-frame reached 204 KB.
## Adapting to the network
`Sources/Controller.swift`, per stream, latency first:
- **The gate.** The viewer acknowledges every frame as it arrives. A new
frame is sent only while the oldest unacknowledged one is younger than
the path's usual round trip, plus one frame interval, plus room for this
link's normal jitter (1.5 times its recent spread, 25–80 ms). So frames
never queue in SSH, TCP or the Wi-Fi driver. While the link is stuck, the
newest picture waits and goes out as soon as it moves.
- **The bitrate.** The link counts as congested when, for two checks in a
row (100 ms apart), round trips grow by more than 40 ms while the stream
uses much of its budget, or the gate holds frames back, or a frame is
stuck for 100 ms. Then the bitrate drops to a bit under what actually got
through: at least a fifth off, and at most half. Once the link has been
clear for a second, it rises by 10% steps, never above the quality
setting's bitrate.
- **The tier.** When the bitrate stays low, and the stream is really
limited by the link rather than having little to send, it steps down:
60 → 45 → 30 fps, then 75%, then 50% of the pixels. It goes straight to
the tier the bitrate supports after half a second, and steps back up one
tier at a time after two seconds with room to spare.
- `FRAME_MAC_VIEW_ADAPT=0` turns it off, for comparison.
Measured on the real Frame, 2026-09-28 (**verified**; interleaved A/B, off
versus on, medians of the runs, ms):
| Link | Scenario | Content p95, off → on | fps drawn, off → on | Result file |
|---|---|---|---|---|
| Clean Wi-Fi (3 runs each) | scroll | 32.3 → 33.6 | 57 → 56.2 | `…-ab-adapt-clean2.json` |
| Clean Wi-Fi | test | 15 → 14.2 | 60 → 59.7 | same |
| 50 → 3 → 50 Mbit/s at 8 s and 16 s (2 runs each) | scroll | **4670 → 72** | 45 → 42 | `…-ab-adapt-step3.json` |
| 50 → 3 → 50 Mbit/s | test | 66 → 16 | 60 → 60 | same |
- **Clean link.** On a clean link it costs nothing measurable. An earlier
version with a fixed gate slack lost 9 fps to Wi-Fi jitter while
scrolling (47 → 38 fps), and that's why the slack now follows the link's
jitter.
- **Throttled link.** Without the controller, frames queued for up to 5.6 s
and never caught up while the link was slow (p95 1.8–5.6 s, second by
second). With it, in the two runs:
| | Run 1 | Run 2 |
|---|---|---|
| Worst second's p95 just after the drop | 219 ms | 428 ms |
| p95 back under 100 ms for 3 s in a row | after 1 s | after 4 s |
| Stepped down to 1440 px at 30 fps | 1.8 s after the drop | 3.5 s after |
| p95 per second after that, on the 3 Mbit/s link | 55–94 ms | 60–148 ms |
Sending one frame takes about 27 ms on that link by itself. After the
link recovered, the stream was back at full size and 60 fps in about
7.5 s. It steps up one tier every two seconds, on purpose, so it doesn't
bounce. The 1 s adaptation target was met in one run of two.
- **A hiccup on a small stream.** In one clean run, a Wi-Fi hiccup made an
earlier version halve the test pattern's bitrate five times and drop it
to half size. That cut couldn't help: the stream only sends
0.47 Mbit/s. Now the controller estimates what a stream wants (captures
per second × average frame size). While a stream wants about half its
budget or less, no cut takes it below twice what it wants, so it doesn't change
tier.
## Checked so far (2026-09-28)
- **Mac (checked by hand, macOS 26.5.2).**
- The agent builds, and lists windows and displays.
- In a browser, the test pattern decoded at about 60 fps (H.264) and 30 fps
(JPEG, about 22 Mbps).
- A click in the viewer arrived at the same point in the source.
- `pmset -g assertions` showed the display-awake assertion only while a
stream was being watched.
- **Automated** (`tests/test_macview.py`, on CI's macOS runner): the agent
builds (a build failure fails the job).
- `/ping` and the page are open; everything else needs the key.
- Tickets work once and only for their own source, and Stop revokes
reconnect keys.
- A WebSocket frame claiming 2^63 bytes closes that socket, and the agent
keeps running.
- A stream sends an SPS-led H.264 keyframe, and sends another when asked.
- Stop ends the stream on the Mac even if the viewer ignores it.
- The test doesn't decode video, time it, or check where clicks land.
- **Linux aarch64 (verified in a stand-in, not on the Frame).** In an Arch
Linux ARM container with sshd, Xvfb as `:0` and Chromium 153:
- The real `show` path worked in 1–2.3 s: tunnel, ticket, launcher,
window found and tagged `STEAM_GAME`.
- Frame Control's key didn't appear anywhere in the stand-in's process
list.
- After the tunnel was killed, it came back on the same port within 4 s,
and the viewer reconnected by itself.
- Chromium decoded the stream in software with the GPU off.
- Stop closed the window, and Chromium exited.
- This test found and fixed a bug: in a C locale, `xwininfo` can't print a
title with non-ASCII characters, so the launcher reads `_NET_WM_NAME`
with `xprop`.
- **On the Frame (verified 2026-09-28, SteamOS build 20260925.6191901,
from the Mac, nobody wearing the headset).** Frame Control's **Show** with
the test pattern:
- The panel was ready in 1.45 s. SteamVR logged `[Overlays] Created:
valve.steam.desktopgame.2001639889` (in `vrwebhelper_systemui.txt`).
- The window was tagged `STEAM_GAME`, and gamescope sized it to 1920×1080
although 1280×720 was asked for.
- Chromium XR decoded it live at about 60 fps: the frame counter advanced
62 in 1.04 s.
- Over home Wi-Fi, the frames in the viewer window had been drawn on the
Mac about 11–17 ms earlier, plus `xwd`'s own time. That's measured
against the two clocks, which were 37–39 ms apart (±4 ms, measured over
one SSH session). SteamVR's compositor and the display come on top.
- Clicks and keys injected with XTest into gamescope's Xwayland didn't
reach the page. That's inconclusive, not a failure: XTest on gamescope
isn't how real input arrives. The laser should arrive as a left mouse
button and the thumbstick as a wheel, because the window has an app id
(from gamescope's source; see [panels.md](panels.md)).
- **Not yet checked:**
- Clicking, dragging and scrolling with the laser while wearing the
headset.
- Real window capture and input on a Mac with both permissions granted.
- Keys from SteamVR's on-screen keyboard.
- Whether Flathub Chromium has H.264. Chromium XR is used when it's
installed, as it is on this Frame.
- Latency with real, busy windows at Sharp.
- A benchmark run while wearing the headset. Only then does the Frame show
panels at full rate, so only then can the compositor's share of the
latency, and the frame rate you actually see, be measured.
## Limits
- Only windows on the Mac's current desktop (Space) are listed, and
minimised windows can't be captured.
- A window's panel shows only that window. Its menus and sheets are separate
windows on the Mac, so open them from the Mac or use **Whole screen**.
- Keys go to whichever Mac window is in front. Clicking a panel brings its
window to the front first.
- Ctrl stays Ctrl. On the Mac, copy is ⌘C, so use Meta+C on a keyboard paired
with the Frame.
docs/mac-window-separation.md
+200
View File
@@ -0,0 +1,200 @@
# Real separation of Mac windows in the headset
Research and experiments on 2026-09-28 (macOS 26.5.2, Apple M5 Pro), for
making each streamed Mac window truly independent. Builds on
[mac-in-headset.md](mac-in-headset.md). Labels: **verified** (tried here),
**documented**, **source** (read in someone's code) or **reported**.
## What "separate" lacks today
Today each window is captured on its own
(`SCContentFilter(desktopIndependentWindow:)`), but it still lives on the
Mac's one desktop:
- **Clicks.** A click has to raise the window first, so it reorders the
Mac's windows, steals focus and moves the real cursor.
- **Child windows.** A window's menus, popovers, sheets and tooltips are
separate windows, so they aren't in its panel. Apple documents that the
single-window filter leaves them out.
- **Size.** A panel's size is tied to the window's size on the Mac screen.
- **Hidden windows.** Minimised windows, and windows on other Spaces, can't
be shown live.
## How the Mac draws, and where pixels can be read
Apps draw with Core Animation into IOSurfaces. WindowServer's compositor
stacks every window onto each display, including virtual ones, and sends
the result to the screen. Pixels can be read at three points:
| Where | How | Gets | Doesn't get |
|---|---|---|---|
| One window's content | ScreenCaptureKit `desktopIndependentWindow` (public, what we use) | Live, zero-copy, even when covered by other windows (**documented**) | Menus, popovers, sheets. It pauses while the window is minimised (**reported**) |
| One window plus its children | `SCStreamConfiguration.includeChildWindows` (macOS 14.2+, **reported**) | Menus, popovers and sheets attached to the window | Anything the app puts outside the window's bounds |
| A whole display | ScreenCaptureKit display filter, with apps or windows included or excluded | Everything on that display, cursor included | Only what's on that display |
| A snapshot of any window | Private `CGSHWCaptureWindowList`, which AltTab uses for minimised windows and other Spaces (**source**: `alt-tab-macos` `PrivateApis.swift`) | Minimised windows and other Spaces | It's one still image, not a live stream |
| Another app's layer tree | Private `CALayerHost` with a context id | A live, zero-copy picture | It only works when the other app cooperates, so it's no good for arbitrary windows (**reported**) |
`CGWindowListCreateImage` and `CGDisplayStream` are obsolete in the macOS 15
SDK (**reported** by MacPorts and JUCE). Nothing reads another app's pixels
without the Screen Recording permission.
## The idea: each window gets its own virtual display
A virtual display (the private `CGVirtualDisplay`, as used by BetterDisplay,
DeskPad and quest-display) is a compositor target with no physical screen.
Put one streamed window alone on its own virtual display, sized to its
panel, and capture the whole display:
- **Nothing can overlap it.** A plain click at that point always lands on
that window, so input doesn't need raising or background-event tricks.
- **Menus, sheets, popovers, tooltips and context menus appear on the same
display, so they're in the panel.** With "Displays have separate Spaces"
on (it is on this Mac), each display also has its own menu bar, so the
app's menu bar can be part of the panel.
- **The panel's size is the display's size,** in HiDPI. Resizing the panel
means changing the display mode and resizing the window to fill it
(Accessibility API).
- **Minimised windows and other Spaces stop being a problem,** because
streamed windows live on their own displays.
- **The cursor goes where the laser points.** That display's panel is the
one being used, much as Vision Pro's Mac Virtual Display works. Keys from
the Mac's keyboard go to the window last clicked.
### Verified here (no permission needed)
- **Creating one.** It needs no permission or entitlement. 1920×1080 HiDPI
gives a 3840×2160-pixel, 60 Hz display, placed next to the built-in
screen, and `NSScreen.screensHaveSeparateSpaces` was true.
- **Many at once.** 16 were created at once with no error.
- **Placement.** `CGConfigureDisplayOrigin` far away failed with error
1014: macOS keeps displays edge to edge. Disabling one with the private
`CGSConfigureDisplayEnabled` from a command-line tool also failed with 1014.
- **Removal is deferred while the physical screen sleeps.** With the
built-in display asleep, virtual displays were not removed when released,
or when their process exited, even from their own `.app`. A second display
with the same vendor, product and serial couldn't be created. All of them
disappeared as soon as the screen woke (`caffeinate -u`). The helper must
therefore:
- keep a fixed pool of displays and reuse them rather than create new ones;
- keep the screen awake while they exist, which it already does while
streaming.
### Built: "Give each window its own display"
Frame Control's helper now does this (`mac/frame-mac-view/Sources/Separate.swift`),
and it's the default in the Tools card. Streams of this kind are named
`separate:<window id>`.
- For each window shown, it creates a HiDPI virtual display. The display is
sized to the window plus a menu bar, with a fresh serial each time, so a
display left over while the screen slept can't block a new one.
- The window is moved onto the display and resized to fill it (Accessibility
API), and the whole display is captured.
- On **Stop** the window goes back where it was, and the display is released.
- Plain per-window capture is still there: untick "Give each window its own
display". Separate mode needs Accessibility (to move the window), and
without it, Show says so rather than quietly falling back.
Verified 2026-09-28 (macOS 26.5.2, M5 Pro), using a TextEdit test document
and the benchmark's Chrome windows:
- **Separation works.** The window moved onto its own display and streamed,
with its first frames in 3.8 s. The display showed TextEdit's own menu bar.
- **Child windows come along.** A context menu and the Page Setup sheet
opened on the same display and were in the capture.
- **Input.** Typing through the stream reached the window. The benchmark's
typing scenario does this on every run.
- **Stop.** Stop put the window back at exactly its old size (656×422), and
the display was removed.
- **The helper must run a real Cocoa event loop.** Earlier, it ran only a
`RunLoop`. With that, AppKit never learned about new displays:
- `NSScreen.screens` never listed them, so placing the window always waited
3 s and then gave up;
- the display's HiDPI mode never applied, so windows were captured at 1×
(1280×860 instead of 1920×1290 pixels) and text was soft.
Running `NSApplication` (with no Dock icon) fixed both. The screen now
appears within 100 ms, and captures are at 2× (**verified** in
`bench/results/*-baseline.json` against `*-baseline-fixed.json`).
- **Frame rate.** Chrome drew at 60–61 fps on the virtual display, but
ScreenCaptureKit delivered only about 46–53 fps from it, whereas the
built-in ProMotion screen gave 121 fps (**verified**). Neither a 120 Hz
virtual display (`FRAME_MAC_VIEW_VD_HZ=120`) nor a looser
`minimumFrameInterval` changed that. Cause unknown.
- **Windows that keep their own size** (Calculator, 230×408). The window
moved and streamed, but stayed small in the display's corner, so the panel
was mostly wallpaper. Now only that corner is captured
(`SCStreamConfiguration.sourceRect`): the menu bar and the window, at least
480×360 points so menus fit. Clicks map to the cropped area. The first
frame arrived in 0.6–1.1 s, and on Stop the window went back and the
display was removed. A display's menu bar names the active app, so it
shows the window's own menus only once the panel has been clicked.
- **Several windows at once** (on the Mac, with a local stand-in viewer that
acknowledges frames). Three and four Chrome windows, each scrolling on its
own display at 1920×1290 pixels, streamed at 53–56 fps and about
9 Mbit/s each. The helper used 20% (three) or 24% (four) of one CPU core.
The hardware encoder is shared: its time per frame went from 6.7 ms for
one stream to 7–15 ms for three and 11–25 ms for four, so each extra
moving window adds latency to the others. Once in four runs, before a fix,
one window left its display when several displays appeared at the same
moment, and its stream stopped: nothing on that display was changing any
more. The helper now checks every second and puts the window back. Three
further runs with four windows were clean, and every display was gone
afterwards (checked with `CGGetOnlineDisplayList`).
- **Quitting.** On SIGTERM, or when Frame Control goes away, the helper ends
every stream first, so windows go back before their displays disappear.
Verified: a 900×600 window was back at 900×600 after SIGTERM. Closing a
viewer 0.05–1.5 s into startup left the window at its old size and no
extra display.
- **A consent prompt.** macOS 26 asks whether to let ScreenCaptureKit apps
"bypass the system private window picker". The prompt appeared *on the
virtual display*, so it would show up inside the headset panel. Allow it
once on the Mac.
### Still to check
1. That a context menu opens over the text being clicked, not just somewhere
on the display. This needs a retest after the consent prompt above is
allowed.
2. Stage Manager, which is reported to undo Accessibility resizes.
3. Where the Dock and the cursor go on a virtual display.
4. Closing the lid with virtual displays attached (clamshell).
5. Many moving windows at once in the headset: whether to lower the bitrate
or frame rate of panels you aren't using, so the one you are using keeps
the encoder to itself.
6. All of it in the headset, with the laser.
## Input without disturbing the Mac (a finer option)
For windows that stay on the Mac's own screen, input can go to a window
behind others without raising it:
- **Background click.** `CGEventPostToPid` with the CGEvent fields 91 and
92, which say which window a click is for (`kCGMouseEventWindowUnderMousePointer`
and `...ThatCanHandleThisEvent`), plus a window-relative location
(**reported**, reverse-engineered, needs testing).
- **Focus without raise.** yabai makes a window key without raising it by
posting event records with the private `SLPSPostEventRecordTo` (**source**:
`yabai/src/window_manager.c`).
- **Known failures.**
- Chromium and Electron ignore background clicks unless they're built
with the 2026 `acceptsFirstMouse` fix (electron/electron#54493).
- Games and canvas apps often need real activation.
- Password fields (Secure Event Input) drop synthetic keys.
- IME composition, as for Chinese or Japanese input, isn't reliable.
With a virtual display per window, most of this isn't needed. It's worth
having for hovering over one panel while typing in another.
## Encoding and transport
- **Codec.** Keep H.264 4:2:0. Apple's High Performance Screen Sharing uses
4:4:4 over two virtual displays (**documented**), but the Frame's Chromium
decodes H.264 in software, and no 4:4:4 decode path is confirmed there.
- **Sharper static text.** When a panel has been still for a moment, send a
high-quality refresh, either a keyframe at low quantisation or a lossless
WebP overlay, so static text is sharp. Moving content stays as video.
- **Transport.** Keep WebSocket over SSH for now, as it works everywhere.
WebRTC (UDP, congestion control) is the proven step up for Wi-Fi.
WebTransport over QUIC is promising, but its server side on macOS is
unverified.
docs/open-questions.md
+13
View File
@@ -130,6 +130,19 @@ Still open: 4, 6, 7, 12–15, 16 (off-LAN and after a reboot), 17–21.
- The Mac EDL flashing script in `~/Downloads/steam-frame-recovery/` hasn't
been run against a Frame.
## Mac in the headset (2026-09-28)
The test pattern streams to the Frame as its own panel at about 60 fps
(verified, build 20260925.6191901; see
[mac-in-headset.md](mac-in-headset.md#checked-so-far-2026-09-28)). Still to
check in the headset:
- Laser clicks, drags and thumbstick scrolling in a viewer panel.
- Real window capture and input once Screen Recording and Accessibility are
granted to Frame Control.
- Keys from SteamVR's on-screen keyboard.
- Whether Flathub Chromium decodes H.264 (otherwise use Compatible).
## Unconfirmed claims made in these docs
- `/home` and `/etc` persist across Frame OS updates. This is inferred from
docs/streaming.md
+17 -1
View File
@@ -38,7 +38,8 @@ flat 2D desktop streaming into a window on the Frame's Linux desktop.
| Option | Setup | Confidence | Verdict |
|---|---|---|---|
| **macOS Screen Sharing (VNC) → Remmina on the Frame** | **Mac:** System Settings → General → Sharing → Screen Sharing on → (i) → enable "VNC viewers may control screen with password". **Frame:** `./scripts/install-apps.sh remmina` from the Mac, then open Remmina in the headset and connect to `vnc://<mac>.local` | **Verified 2026-09-27** (Frame BUILD_ID 20260925.6191901, macOS 27.0), in its own panel via `panel-on-frame.sh mac-screen`. Remmina is on Flathub for **aarch64** with VNC and RDP ([Flathub](https://flathub.org/apps/org.remmina.Remmina)). The Frame desktop runs Flatpaks ([UploadVR](https://www.uploadvr.com/flatpaks-open-source-steam-frame/)). macOS VNC is built in. | **Recommended.** Nothing to install on the Mac, and it's easy to set up. Noticeable lag, even at lower Remmina quality settings on a good 5 GHz link, where neither Wi-Fi nor the Frame's CPU was the bottleneck. Usable for reading and coding, but not for games. You'll type the Mac's hostname once in Remmina on the headset, then save the profile. To avoid even that, the script can pre-seed a Remmina profile over SSH (see below). |
| **Frame Control → Tools → Mac in the headset** | Nothing to install. Allow Screen Recording and Accessibility for Frame Control, then press **Show** next to any window or screen | **Verified 2026-09-28** on the Frame (panel in 1.5 s, measured with `scripts/macview-bench.py`: about 10–20 ms from the Mac drawing a frame to the viewer drawing it); laser input not yet tried while wearing it | **Recommended.** Hardware H.264 over an SSH tunnel, adapting to the link. Each window becomes its own panel you can place anywhere. Laser clicks and scrolls, and the Mac's keyboard types. See [mac-in-headset.md](mac-in-headset.md) |
| **macOS Screen Sharing (VNC) → Remmina on the Frame** | **Mac:** System Settings → General → Sharing → Screen Sharing on → (i) → enable "VNC viewers may control screen with password". **Frame:** `./scripts/install-apps.sh remmina` from the Mac, then open Remmina in the headset and connect to `vnc://<mac>.local` | **Verified 2026-09-27** (Frame BUILD_ID 20260925.6191901, macOS 27.0), in its own panel via `panel-on-frame.sh mac-screen`. Remmina is on Flathub for **aarch64** with VNC and RDP ([Flathub](https://flathub.org/apps/org.remmina.Remmina)). The Frame desktop runs Flatpaks ([UploadVR](https://www.uploadvr.com/flatpaks-open-source-steam-frame/)). macOS VNC is built in. | **Fallback** (whole screens only). Nothing to install on the Mac, and it's easy to set up. Noticeable lag, even at lower Remmina quality settings on a good 5 GHz link, where neither Wi-Fi nor the Frame's CPU was the bottleneck. Usable for reading and coding, but not for games. You'll type the Mac's hostname once in Remmina on the headset, then save the profile. To avoid even that, the script can pre-seed a Remmina profile over SSH (see below). |
| Sunshine (Mac) → Moonlight (Frame Flatpak) | `brew install` Sunshine on the Mac, then `./scripts/install-apps.sh moonlight` | Moonlight Flatpak supports **aarch64** ([Flathub](https://flathub.org/apps/com.moonlight_stream.Moonlight)). **Sunshine on macOS is poorly supported**: install problems on Apple Silicon/Sequoia, and no virtual gamepads ([LizardByte discussion #777](https://github.com/orgs/LizardByte/discussions/777)). | Try it if VNC is too laggy. Expect some friction. |
| Steam Remote Play with the Mac as host | Steam on the Mac, Steam Link/Remote Play on the Frame | macOS-hosted Remote Play is reported broken or flaky in 2024–2026 ([Steam discussion](https://steamcommunity.com/groups/homestream/discussions/1/574921459914429988/)) | Not recommended. It's only for games, if it works at all. |
| Immersed / Virtual Desktop | Vendor apps | Immersed has a Mac agent but no known Frame client. Virtual Desktop's developer said he'd "try" to port it ([NewsBreak](https://www.newsbreak.com/news/4892834783961-virtual-desktop-dev-says-he-ll-try-to-bring-the-app-to-steam-frame)). | Not available as of 2026-09-25. Check again later. |
@@ -86,6 +87,21 @@ its header. (Verified 2026-09-27.)
Going the other way, pointing a controller at the panel moves the Mac's mouse,
because Remmina forwards input (`viewonly=0`).
## First-party options, and why they do or don't fit
Checked 2026-09-28. The first-party way is usually the best one, so these are
listed first; the sections above and below explain the alternatives.
| Goal | First-party option | Fits? | Why, and what would make it easier |
|---|---|---|---|
| Type and point from the **iPhone** | **KDE Connect** (KDE; official [iOS app](https://apps.apple.com/app/kde-connect/id1580245991)) remote touchpad and keyboard, plus clipboard and files | **Best candidate, untested on the Frame** | The Frame doesn't have it (verified: no `kdeconnectd`), it isn't on Flathub, and the root is read-only, so it would have to run from `~` or a container. On Wayland it types through KWin, so it can only reach the desktop panel, not SteamVR or games (**inferred**). Steam Deck users report its remote input breaking after SteamOS updates ([SteamOS #1939](https://github.com/ValveSoftware/SteamOS/issues/1939)). If it works, Frame Control could install it and pair it for you. |
| Type and point from the **Mac** | KDE Connect for macOS (KDE builds) | Same as above | Its Mac app sends clipboard and files but has no keyboard/mouse sharing (**inferred**). |
| Either | **Bluetooth keyboard and mouse** paired in SteamOS (Valve) | Yes, with real hardware | Neither device can pretend to be one: iOS refuses the HID service ([Apple forums](https://developer.apple.com/forums/thread/733916)), and macOS has no built-in way. |
| Either | **xrdp** in Developer Mode (Valve) | No | Input goes into a *separate* desktop shown on the Mac, not into what you see in the headset. |
| **Mac screen** in the Frame | **Screen Sharing** (Apple's VNC server) + Remmina (already installed on this Frame, profile pre-seeded by `install-apps.sh`) | **Yes, closest to first-party** | Only the Mac side is first-party; Remmina is the client. Turn on System Settings → General → Sharing → Screen Sharing → (i) → "VNC viewers may control screen with password". Still to test in the headset (open question 11). |
| Mac screen | **Steam Remote Play** with the Mac as host (Valve) | Probably not | macOS isn't a SteamVR host, and Mac-hosted Remote Play is reported broken ([Steam forum](https://steamcommunity.com/groups/homestream/discussions/1/574921459914429988/)). One quick test is worth doing: Steam on the Mac, then Remote Play from the Frame's Steam. |
| Mac or **iPhone screen** | **AirPlay** (Apple) | Not officially | It's Apple's own mirroring for both, but Apple only licenses receivers to TV and speaker makers; nothing official runs on Linux. UxPlay (below) is the unofficial receiver. |
## C. Show the iPhone's screen inside the Frame
iOS only shares its screen two ways: **AirPlay** (Screen Mirroring in Control
docs/vr-apks.md
+148
View File
@@ -0,0 +1,148 @@
# VR APKs and Quest games in Lepton
What it takes to run an immersive (OpenXR) Android app, including Meta Quest
builds, on the Frame. Checked on SteamOS BUILD_ID 20260925.6191901, Lepton
v2.8.14 (rootfs v2.8.11), SteamVR 2.18.1, on 2026-09-28, unless marked
**inferred**.
## How a VR APK reaches SteamVR (verified)
- Lepton ships a standard Khronos system runtime manifest,
`/vendor/etc/openxr/1/active_runtime.json`, pointing at SteamVR's Android
client, `/data/steamvr/runtime/bin/androidarm64/vrclient.so`. That is the
host's `/opt/steamvr/bin/androidarm64/`, bind-mounted in.
- An APK's own Khronos-style `libopenxr_loader.so` tries the runtime brokers
(`org.khronos.openxr.runtime_broker`, `…system_runtime_broker`), finds
neither, then falls back to that manifest. Nothing in the APK has to change
for discovery.
- Install the APK without the flatscreen marker
(`python3 ui/frame_android.py install app.apk --vr`). The marker only
controls Lepton's 2D Android surface; the app itself has to start an
OpenXR session.
- Lepton also loads Valve's `XR_APILAYER_VALVE_fdm_injection` layer from
`/vendor/etc/openxr/1/api_layers/implicit.d/`. Only layers in Valve's own
directories are picked up (`liblepton/vulkan_layers.sh`), so a third-party
layer has to ship inside the APK.
**Open Brush 2.32.29, the Quest APK from its GitHub release (Unity OpenXR,
Vulkan), works unmodified.** Its manifest already has `LAUNCHER` next to
`com.oculus.intent.category.VR`. Unity asked for OpenXR 1.1, got
`XR_ERROR_API_VERSION_UNSUPPORTED`, retried with 1.0 and succeeded. SteamVR
took it as the scene app, created Touch, simple-controller and Frame-controller
bindings, and the session reached `XR_SESSION_STATE_FOCUSED`. A headset capture
(`ui/frame_vrshot.py`, after waking the compositor and closing the dashboard)
showed a dark sky over a mountain horizon; nobody wore the headset to confirm
it was Open Brush's scene or to try drawing.
**Khronos `hello_xr` (Vulkan, 1.1.63 release APK) works unmodified:**
`Instance RuntimeName=SteamVR/OpenXR RuntimeVersion=2.18.1`, 1728×1728
swapchains per eye, session `IDLE → READY → SYNCHRONIZED` (the headset was
not being worn, so it did not reach `FOCUSED`).
## What SteamVR's Android runtime supports (verified, from `vrclient.so`)
- **OpenXR 1.0 only.** An app requesting `XR_API_VERSION_1_0` works; one
requesting 1.1 (`XR_CURRENT_API_VERSION` in a 1.1 SDK) gets
`XR_ERROR_API_VERSION_UNSUPPORTED` from the runtime.
- Extensions include `XR_KHR_opengl_es_enable`, `XR_KHR_vulkan_enable{,2}`,
`XR_KHR_composition_layer_depth`, `XR_KHR_locate_spaces`,
`XR_EXT_local_floor`, `XR_EXT_uuid`, `XR_EXT_palm_pose`,
`XR_EXT_hand_tracking`, `XR_EXT_eye_gaze_interaction`, and these Meta ones:
`XR_FB_display_refresh_rate`, `XR_FB_foveation{,_configuration,_vulkan}`,
`XR_FB_space_warp`, `XR_FB_swapchain_update_state`,
`XR_META_foveation_eye_tracked`, `XR_META_recommended_layer_resolution`,
`XR_META_vulkan_swapchain_create_info`, `XR_META_performance_metrics`.
- Not present: `XR_FB_passthrough`, `XR_FB_hand_tracking_*`,
`XR_FB_spatial_entity*`, `XR_FB_color_space`,
`XR_KHR_android_thread_settings`, `XR_OCULUS_*`.
- Interaction profiles include `oculus/touch_controller`, `khr/simple_controller`,
`valve/frame_controller` and the usual PC controllers. Valve documents Touch
bindings as a working fallback on the Frame controllers.
## What stops a Quest APK (verified with Wolvic 1.9, `oculusvr` build)
1. **Lepton won't start it.** Lepton's `apk-info-extractor` only accepts an
activity whose intent filter has `android.intent.action.MAIN` and
`android.intent.category.LAUNCHER`. Quest apps use
`com.oculus.intent.category.VR` instead, so Lepton logs `APP_ACTIVITY is
empty` and exits. There is no override. **Fix:** add the `LAUNCHER`
category to that intent filter and re-sign. After that, Wolvic started.
2. **OpenXR 1.1.** Wolvic's Quest build then requested OpenXR 1.1 and aborted
on `XR_ERROR_API_VERSION_UNSUPPORTED`. Unity's OpenXR plugin retries with
1.0 (Open Brush, above), so this mostly bites native and non-Unity apps. **Fix (inferred):** an API layer
inside the APK that asks the runtime for 1.0 and maps the 1.1 core
functions to the extensions the runtime does have (`XR_KHR_locate_spaces`,
`XR_EXT_local_floor`, `XR_EXT_uuid`, `XR_EXT_palm_pose`).
3. **Lepton's missing clipboard service** still applies to VR apps. The Godot
XR Tools demo's Quest build (itch.io) dies in `Godot.<init>` casting the
null clipboard service to `ClipboardManager`, before any OpenXR call. See
the clipboard table in [apks.md](apks.md).
4. **Not yet reached:** required Meta-only extensions (each app differs),
swapchain formats (the Lynx Wolvic build needed `GL_SRGB8_ALPHA8`), and
Meta platform services.
The loader was never the problem: Wolvic's Quest `libopenxr_loader.so` is a
Khronos-style loader and found SteamVR through `/vendor`.
## Out of scope
- **Meta entitlement.** Apps that call the Oculus Platform SDK
(`libovrplatformloader.so`) to check the Quest store licence need Meta's
services. Frame Control won't work around that.
- **VrApi-era apps** (`libvrapi.so`, before OpenXR) need an API translator,
not a patch.
## Frame Control does this for you
APK uploads and `python3 ui/frame_android.py install app.apk` detect VR
manifest categories, Samsung's `vr_only` flag and the arm64 OpenXR loader.
VR apps default to immersive mode without the flatscreen marker. The upload
selector or CLI `--flat` / `--vr` overrides that choice. Compatibility notes
identify legacy VrApi, Meta platform SDK and OpenXR libraries.
Lepton only starts an `<activity>` whose MAIN intent filter has LAUNCHER; it
ignores `<activity-alias>`, which is where Godot 4 exports put LAUNCHER. When no
real activity qualifies, Frame Control adds LAUNCHER to the VR activity's MAIN
filter, or to the activity the launcher alias targets, then repacks and v2-signs
the APK locally before copying it; `meta.json` records
`"patched": ["launcher"]`. Unchanged ZIP members retain their compressed
bytes; stored libraries are aligned to 16 KiB. The RSA signing identity lives
in Frame Control's per-user app-data directory as `apk-signing-key.json`
(mode 0600). Keep this key to preserve the signer on subsequent patched
updates. A re-signed APK cannot update an installation signed by its original
publisher; Android also treats it as a different signer for signature checks.
VR apps with an arm64 OpenXR loader also get the OpenXR compatibility layer
([frame/openxr-compat](../frame/openxr-compat/README.md)): an implicit API
layer in the APK's `assets/openxr/1/api_layers/implicit.d/`, which the app's
own loader picks up next to Valve's layer. It asks SteamVR for OpenXR 1.0 when
the app wants 1.1 and enables the extensions that became 1.1 core; maps
`xrLocateSpaces` to `xrLocateSpacesKHR` and `grip_surface` to `palm_ext`; drops
1.1 controller profiles SteamVR doesn't know; stubs
`XR_KHR_android_thread_settings` and `XR_OCULUS_android_session_state_enable`;
and keeps the current refresh rate when SteamVR refuses a requested one.
`meta.json` records `"patched": ["openxr-compat"]`. Skip it with
`install … --no-xr-compat`. Its decisions go to logcat under `FrameXrCompat`.
Verified on the headset (2026-09-28):
- **Wolvic 1.9, Quest build**, installed as downloaded: Frame Control added
`LAUNCHER` and the layer. The layer turned OpenXR 1.1.48 into 1.0.63, the
instance and session were created, and a 144 Hz refresh request that SteamVR
refused was kept at the current rate. The session reached `SYNCHRONIZED`;
then Wolvic's Gecko engine crashed (null SIGSEGV on its Gecko thread, the
same crash its Lynx build has), which is Wolvic's, not OpenXR's.
- **Open Brush, Quest build**, with the layer: 1.1.54 → 1.0.63, the thread
settings stub in use, `bytedance/pico4_controller` bindings dropped, and the
session reached `FOCUSED`, the same as without the layer.
Inspect or prepare an APK without contacting the headset:
```sh
python3 ui/frame_android.py info app.apk
python3 ui/frame_android.py patch app.apk patched.apk
python3 ui/frame_android.py patch app.apk patched.apk --add assets/openxr/1/api_layers/implicit.d/X.json=X.json --add lib/arm64-v8a/libX.so=libX.so
```
The patch fixes Lepton's launch-category requirement. It does not supply an
OpenXR 1.1 translation layer, Meta services or a VrApi implementation.
frame/openxr-compat/.gitattributes
+2
View File
@@ -0,0 +1,2 @@
# Preserve upstream headers byte-for-byte, including their existing whitespace.
vendor/** -whitespace
frame/openxr-compat/.gitignore
+1
View File
@@ -0,0 +1 @@
/build-*/
frame/openxr-compat/CMakeLists.txt
+25
View File
@@ -0,0 +1,25 @@
cmake_minimum_required(VERSION 3.22)
project(FrameXrCompat LANGUAGES CXX)
set(CMAKE_CXX_STANDARD 17)
set(CMAKE_CXX_STANDARD_REQUIRED ON)
add_library(compat_logic INTERFACE)
target_include_directories(compat_logic INTERFACE . vendor)
if(ANDROID)
add_library(XrApiLayer_FRAME_compat SHARED layer.cpp)
target_link_libraries(XrApiLayer_FRAME_compat PRIVATE compat_logic log)
target_compile_definitions(XrApiLayer_FRAME_compat PRIVATE XR_USE_PLATFORM_ANDROID XR_NO_PROTOTYPES)
target_compile_options(XrApiLayer_FRAME_compat PRIVATE -O2 -fvisibility=hidden -Wall -Wextra -Werror)
target_link_options(XrApiLayer_FRAME_compat PRIVATE -Wl,-z,max-page-size=16384 -Wl,--no-undefined -Wl,--exclude-libs,ALL -Wl,-s)
else()
enable_testing()
add_executable(compat_tests tests/compat_tests.cpp)
target_link_libraries(compat_tests PRIVATE compat_logic)
target_compile_options(compat_tests PRIVATE -Wall -Wextra -Werror)
add_test(NAME compat_logic COMMAND compat_tests)
add_executable(dispatch_tests tests/dispatch_tests.cpp)
target_link_libraries(dispatch_tests PRIVATE compat_logic)
target_include_directories(dispatch_tests PRIVATE tests/shims)
target_compile_definitions(dispatch_tests PRIVATE XR_USE_PLATFORM_ANDROID XR_NO_PROTOTYPES)
target_compile_options(dispatch_tests PRIVATE -Wall -Wextra -Werror)
add_test(NAME layer_dispatch COMMAND dispatch_tests)
endif()
frame/openxr-compat/README.md
+254
View File
@@ -0,0 +1,254 @@
# Frame OpenXR compatibility API layer
`XR_APILAYER_FRAME_compat` is an APK-local implicit API layer for Steam Frame's
OpenXR 1.0 Android runtime. It translates selected 1.1 functionality; it is not
a conformant replacement for a complete 1.1 runtime. Device facts are in
[../../docs/vr-apks.md](../../docs/vr-apks.md). No headset was contacted during
implementation. Loading, Valve-layer coexistence, controller usability, and
Wolvic startup/rendering on Lepton remain **unverified on device**.
## Behavior
- The negotiated `createApiLayerInstance` hook handles `xrCreateInstance`.
API requests >= 1.1 become `XR_API_VERSION_1_0` (1.0.63 with these headers),
rather than carrying an arbitrary application's patch into the 1.0 request.
Requests below 1.1 are unchanged. The loader may reject future API versions
before any layer runs.
- Enumerate downstream extensions through the **next layer's** GIPA before
creation. Only advertised promoted extensions are added, with duplicates
removed. All other application extensions remain, except the two stubs below.
Log added, stubbed and missing extensions with tag `FrameXrCompat`.
- Missing `XR_KHR_android_thread_settings` is advertised and accepted;
`xrSetAndroidApplicationThreadKHR` logs and returns success without changing
thread scheduling. Missing `XR_OCULUS_android_session_state_enable` is
advertised and accepted, with no commands or additional session behavior.
If downstream supports either extension, preserve it and its real commands.
Stubbing these capabilities is intentionally limited to accepting the app's
request; it does not provide Meta services.
- The manifest's `instance_extensions` makes both stubs visible during the
loader's pre-instance enumeration and validation. The layer also exposes
an enumeration implementation with count/capacity handling. Before a next
dispatch is available it can enumerate only its own extensions; normal app
global enumeration is performed by the loader, merging runtime and manifest
entries. See [loader_core.cpp L106–145](https://github.com/KhronosGroup/OpenXR-SDK-Source/blob/release-1.1.63/src/loader/loader_core.cpp#L106)
and [manifest parsing L549–558](https://github.com/KhronosGroup/OpenXR-SDK-Source/blob/release-1.1.63/src/loader/manifest_file.cpp#L549).
- Unknown missing extensions are never silently removed. Khronos may reject
them **before** entering the layer and name them in `OpenXR-Loader` logs
([loader_instance.cpp L149–176](https://github.com/KhronosGroup/OpenXR-SDK-Source/blob/release-1.1.63/src/loader/loader_instance.cpp#L149)).
If they reach this layer, it logs their names and preserves them for the
downstream failure.
- `xrLocateSpaces` dispatches to `xrLocateSpacesKHR` per session/instance.
This is the only command added in 1.1. The three relevant structure types
(`SPACES_LOCATE_INFO`, `SPACE_LOCATIONS`, `SPACE_VELOCITIES`) and their KHR
versions are **numeric aliases**, and the structs are typedef aliases in
these headers. Explicit type conversion of local input/output copies is
therefore an identity conversion; the velocity chain passes through without
mutating its types or linkage. Output arrays remain shared and receive the
runtime's results, including on error; caller struct headers remain intact.
- With palm pose enabled, rewrite both hand paths ending in
`/input/grip_surface/pose` to `/input/palm_ext/pose` in `xrStringToPath` and
in binding suggestions (including paths obtained before this layer rewrote
them). Unknown paths are left alone.
- `xrGetInstanceProperties` and unrelated commands pass through. The real
runtime's identity is not spoofed. Dispatch tables and session ownership are
locked, and removed after successful destruction. Downstream calls occur
outside the lock.
## Spec basis and interaction profiles
The pinned [1.1 promotions appendix, versions.adoc L16–156](https://github.com/KhronosGroup/OpenXR-Docs/blob/release-1.1.63/specification/sources/chapters/versions.adoc#L16)
includes a generated promotion list. Its source is the
[1.1.63 XML registry](https://github.com/KhronosGroup/OpenXR-Docs/blob/release-1.1.63/specification/registry/xr.xml),
selecting `extension[@promotedto='XR_VERSION_1_1']`. The full list implemented:
| Promoted extension | Handling |
| --- | --- |
| `XR_KHR_locate_spaces` | Enable if advertised; core command alias |
| `XR_EXT_local_floor` | Enable if advertised; reference-space enum is an alias |
| `XR_EXT_uuid` | Enable if advertised; type alias, no commands |
| `XR_EXT_palm_pose` | Enable if advertised; rewrite grip-surface paths |
| `XR_VARJO_quad_views` | Enable if advertised; view-configuration enum alias, support remains optional |
| `XR_EXT_samsung_odyssey_controller` | Enable if advertised |
| `XR_EXT_hp_mixed_reality_controller` | Enable if advertised |
| `XR_HTC_vive_cosmos_controller_interaction` | Enable if advertised |
| `XR_HTC_vive_focus3_controller_interaction` | Enable if advertised |
| `XR_ML_ml2_controller_interaction` | Enable if advertised |
| `XR_FB_touch_controller_pro` | Enable if advertised; old profile still usable |
| `XR_META_touch_controller_plus` | Enable if advertised; old profile still usable |
| `XR_BD_controller_interaction` | Enable if advertised |
| `XR_KHR_maintenance1` | Enable if advertised (included in the pinned registry's promotion list) |
`XR_EXT_hand_interaction` is **not** promoted to 1.1. Preserve it if requested;
it is not auto-enabled merely because the runtime advertises it. The parent
provided its availability; the device notes list extensions non-exhaustively.
The registry's `XR_VERSION_1_1` feature introduces 13 profile paths. For
Samsung Odyssey, HP mixed reality, HTC Cosmos/Focus3, ML2 and the three
ByteDance Pico profiles, drop a whole suggestion call with success when the
corresponding extension is absent. Keep it when the extension is enabled.
Drop the five new Meta paths: `touch_pro_controller`, `touch_plus_controller`,
`touch_controller_rift_cv1`, `touch_controller_quest_1_rift_s`, and
`touch_controller_quest_2`. The verified runtime notes do not list these;
Pro/Plus promotion also renames several components, so the old extension's
presence alone does not prove support for the new profile. This layer does
not implement those component conversions. Existing old Pro/Plus profile
paths are passed through if the app uses them.
Preserve the documented `oculus/touch_controller`, `khr/simple_controller`,
`valve/frame_controller` and original PC profile paths. The notes' "usual PC
controllers" is not a complete enumerated list: availability of promoted PC
profiles is inferred from the runtime's extension advertisement, not invented
from that phrase. OpenXR has no general profile enumeration API. Dropping
suggestions prevents an unsupported 1.1 profile from aborting initialization;
it does not create bindings, so the app must also suggest a supported fallback.
## Headers and licensing
The required public headers (`openxr.h`, `openxr_platform.h`,
`openxr_platform_defines.h`) are unmodified from
[OpenXR-SDK release-1.1.63](https://github.com/KhronosGroup/OpenXR-SDK/tree/release-1.1.63/include/openxr),
commit [f2448a8](https://github.com/KhronosGroup/OpenXR-SDK/commit/f2448a8797c85814aa892efc1ab8707900fbcc78).
The requested filename `loader_interfaces.h` no longer exists in 1.1 SDK
releases: negotiation was ratified and moved to `openxr_loader_negotiation.h`
in 1.0.33 ([spec history L166–186](https://github.com/KhronosGroup/OpenXR-Docs/blob/release-1.1.63/specification/sources/chapters/versions.adoc#L166)).
To retain that requested interface filename, the layer uses the unmodified,
ABI-compatible [last legacy header from SDK-Source release-1.0.32](https://github.com/KhronosGroup/OpenXR-SDK-Source/blob/release-1.0.32/src/common/loader_interfaces.h).
Only these four headers are vendored. They offer Apache-2.0 OR MIT; this
vendoring uses Apache-2.0, reproduced in [vendor/LICENSE](vendor/LICENSE).
## Android discovery and coexistence
**Minimum asset-discovery loader release: 1.0.25 (2022-09-02).** Its
[release commit, 15c3d8e](https://github.com/KhronosGroup/OpenXR-SDK-Source/commit/15c3d8eb99994e5365d6b6f96eefaf4c51a65a9d)
explicitly announces APK-packaged API layers;
[manifest_file.cpp L665–723](https://github.com/KhronosGroup/OpenXR-SDK-Source/blob/release-1.0.25/src/loader/manifest_file.cpp#L665)
implements discovery, and [L934–940](https://github.com/KhronosGroup/OpenXR-SDK-Source/blob/release-1.0.25/src/loader/manifest_file.cpp#L934)
adds asset manifests after filesystem manifests. This is not a 1.1-only feature.
**For an app requesting API 1.1, use a 1.1 loader (first release 1.1.36) or
newer.** A 1.0 loader rejects the request before the layer can downgrade it;
see [loader_core.cpp L217–230](https://github.com/KhronosGroup/OpenXR-SDK-Source/blob/release-1.1.63/src/loader/loader_core.cpp#L217).
The runtime can remain 1.0.
Source inspection at release-1.1.63 confirms:
- [manifest_file.cpp L720–783](https://github.com/KhronosGroup/OpenXR-SDK-Source/blob/release-1.1.63/src/loader/manifest_file.cpp#L720)
uses the initialized Android asset manager and scans
`openxr/1/api_layers/implicit.d/` for JSON. The application must initialize
the loader with its Android context (`xrInitializeLoaderKHR`). In the APK,
the entry is `assets/openxr/1/api_layers/implicit.d/XrApiLayer_FRAME_compat.json`.
- A bare `library_path`, as used here, is **not explicitly concatenated** with
`nativeLibraryDir`. [L878–900](https://github.com/KhronosGroup/OpenXR-SDK-Source/blob/release-1.1.63/src/loader/manifest_file.cpp#L878)
leaves a bare name for normal dynamic-linker search in the application's
namespace (including its native library directory). Relative paths with a
slash use [LocateLibraryInAssets L943–952](https://github.com/KhronosGroup/OpenXR-SDK-Source/blob/release-1.1.63/src/loader/manifest_file.cpp#L943),
which resolves against `GetAndroidNativeLibraryDir()`.
[loader_init_data.cpp L87–96](https://github.com/KhronosGroup/OpenXR-SDK-Source/blob/release-1.1.63/src/loader/loader_init_data.cpp#L87)
obtains the asset manager and `ApplicationInfo.nativeLibraryDir` via JNI.
Wolvic's supplied manifest has `android:extractNativeLibs="true"`.
- [android_utilities.cpp L267–322](https://github.com/KhronosGroup/OpenXR-SDK-Source/blob/release-1.1.63/src/loader/android_utilities.cpp#L267)
handles runtime broker discovery, not APK layer discovery. Its
`native_lib_dir + so_filename` and `dlopen` refer to the runtime package.
They must not be confused with the app's layer library resolution.
- [manifest_file.cpp L1008–1023](https://github.com/KhronosGroup/OpenXR-SDK-Source/blob/release-1.1.63/src/loader/manifest_file.cpp#L1008)
retains filesystem manifests and then appends asset manifests. This leaves
`/vendor`'s `XR_APILAYER_VALVE_fdm_injection` discoverable. This layer advances
`nextInfo` exactly once, uses next-layer GIPA, preserves the rest of the
create-info chain and never opens `vrclient.so` directly. No environment
layer-list override or assumed ordering is needed.
`DISABLE_FRAME_XR_COMPAT` is the manifest's disable environment variable.
Leave it unset to activate the implicit layer. No `enable_environment` is
required.
## Rebuild and verify
NDK **r30 / 30.0.16248370**, installed at
`~/Library/Android/ndk/30.0.16248370`, was the latest stable/LTS package shown
by the [Android download page](https://developer.android.com/ndk/downloads)
on 2026-09-28. Downloaded
[android-ndk-r30-darwin.dmg](https://dl.google.com/android/repository/android-ndk-r30-darwin.dmg)
(1,072,970,961 bytes). The page publishes SHA-1, not SHA-256:
```
expected: 48591224b6657f46eebbc9d95d4af09bbed8d107
actual: 48591224b6657f46eebbc9d95d4af09bbed8d107 (PASS)
```
Mounted read-only with `hdiutil`; copied
`AndroidNDK16248370.app/Contents/NDK` into that version directory. Despite the
`darwin-x86_64` toolchain directory name, clang is universal arm64/x86_64 and
ran on this Apple Silicon Mac.
```sh
frame/openxr-compat/build.sh
# Or set ANDROID_NDK_HOME to an installed NDK.
cmake -S frame/openxr-compat -B frame/openxr-compat/build-host -G Ninja
cmake --build frame/openxr-compat/build-host
ctest --test-dir frame/openxr-compat/build-host --output-on-failure
```
The script produces arm64-v8a / android-24, C++17, `-O2`, static libc++, hidden
internal symbols, stripped output, and `-Wl,-z,max-page-size=16384`.
The committed prebuilt is `prebuilt/arm64-v8a/libXrApiLayer_FRAME_compat.so`.
Its SHA-256 is recorded below with the APK checks.
[Host test output](evidence/ctest.txt): two tests passed, covering pure logic
and the actual layer with a fake next layer and host-only log/JNI shims.
[Full llvm-readelf -d -s -l output](evidence/readelf.txt) records three LOAD
segments aligned `0x4000`, only `xrNegotiateLoaderApiLayerInterface` exported,
and only `libc.so`, `libm.so`, `libdl.so`, `liblog.so` as NEEDED libraries.
All 46 undefined imports are versioned `@LIBC` except `__android_log_print`.
There is no `libc++_shared.so` dependency and no unstripped `.symtab`.
No independent model review was run: the task explicitly prohibits delegation.
No `CODING_STANDARDS.md` exists in this worktree or the primary worktree.
## Wolvic injection artifact (Mac only)
Copied `/tmp/vrapk/wq-dec` to `/tmp/vrapk/wq-layer-dec`; retained its existing
LAUNCHER fix and native-library extraction setting. Added only:
```
assets/openxr/1/api_layers/implicit.d/XrApiLayer_FRAME_compat.json
lib/arm64-v8a/libXrApiLayer_FRAME_compat.so
```
Built using `/tmp/vrapk/jdk/Contents/Home/bin/java` and
`/tmp/vrapk/apktool_3.0.3.jar`, then signed in place using
`/tmp/vrapk/uber-apk-signer-1.3.0.jar --overwrite` (embedded debug certificate):
```sh
/tmp/vrapk/jdk/Contents/Home/bin/java -jar /tmp/vrapk/apktool_3.0.3.jar \
b /tmp/vrapk/wq-layer-dec -o /tmp/vrapk/wolvic-quest-compat.apk
/tmp/vrapk/jdk/Contents/Home/bin/java -jar /tmp/vrapk/uber-apk-signer-1.3.0.jar \
-a /tmp/vrapk/wolvic-quest-compat.apk --overwrite
/tmp/vrapk/jdk/Contents/Home/bin/java -jar /tmp/vrapk/uber-apk-signer-1.3.0.jar \
-a /tmp/vrapk/wolvic-quest-compat.apk --onlyVerify
```
Final APK: `/tmp/vrapk/wolvic-quest-compat.apk` (not committed).
The bundled `lib/arm64-v8a/libopenxr_loader.so` is byte-for-byte unchanged.
Its strings include both `openxr/1/api_layers/implicit.d/` and
`openxr/1/api_layers/explicit.d/`, `AddManifestFilesAndroid` error messages,
and `xrLocateSpaces`. Thus it contains APK asset discovery and evidence of
1.1 command support. Its exact upstream release number was not established
from the binary; runtime execution of those paths remains unverified.
Prebuilt library SHA-256:
```
479d31c374f137906e03f73209b581d65d7e6a41b8476e6425fa55a6aa40bd68
```
APK SHA-256:
```
355a681625e38b71563dcffecb031799eff27894d6ab910a659bace057e82c1f
```
Final `--onlyVerify` exited 0: zip alignment verified, v2/v3 signatures
verified, one APK processed and zero errors. ZIP readback confirmed that the
injected library and manifest match the committed inputs, the loader is
unchanged, and the binary Android manifest retains the LAUNCHER category.
See [APK verification evidence](evidence/apk-verification.txt).
frame/openxr-compat/XrApiLayer_FRAME_compat.json
+21
View File
@@ -0,0 +1,21 @@
{
"file_format_version": "1.0.0",
"api_layer": {
"name": "XR_APILAYER_FRAME_compat",
"library_path": "libXrApiLayer_FRAME_compat.so",
"api_version": "1.1",
"implementation_version": "1",
"description": "Steam Frame OpenXR 1.1 to 1.0 compatibility",
"disable_environment": "DISABLE_FRAME_XR_COMPAT",
"instance_extensions": [
{
"name": "XR_KHR_android_thread_settings",
"extension_version": "6"
},
{
"name": "XR_OCULUS_android_session_state_enable",
"extension_version": "1"
}
]
}
}
frame/openxr-compat/build.sh
Executable
+12
View File
@@ -0,0 +1,12 @@
#!/bin/sh
set -eu
here=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
ndk=${ANDROID_NDK_HOME:-"$HOME/Library/Android/ndk/30.0.16248370"}
cmake -S "$here" -B "$here/build-android" -G Ninja \
-DCMAKE_TOOLCHAIN_FILE="$ndk/build/cmake/android.toolchain.cmake" \
-DANDROID_ABI=arm64-v8a -DANDROID_PLATFORM=android-24 \
-DANDROID_STL=c++_static -DCMAKE_BUILD_TYPE=Release
cmake --build "$here/build-android"
mkdir -p "$here/prebuilt/arm64-v8a"
cp "$here/build-android/libXrApiLayer_FRAME_compat.so" "$here/prebuilt/arm64-v8a/"
shasum -a 256 "$here/prebuilt/arm64-v8a/libXrApiLayer_FRAME_compat.so"
frame/openxr-compat/compat_logic.h
+88
View File
@@ -0,0 +1,88 @@
#pragma once
#include <openxr/openxr.h>
#include <algorithm>
#include <string>
#include <vector>
namespace frame {
using Names = std::vector<std::string>;
inline constexpr const char* layerName = "XR_APILAYER_FRAME_compat";
inline constexpr const char* stubs[] = {"XR_KHR_android_thread_settings", "XR_OCULUS_android_session_state_enable"};
// OpenXR-Docs release-1.1.63 registry: promotedto="XR_VERSION_1_1".
inline constexpr const char* promotions[] = {
"XR_KHR_locate_spaces", "XR_EXT_local_floor", "XR_EXT_uuid", "XR_EXT_palm_pose",
"XR_VARJO_quad_views", "XR_EXT_samsung_odyssey_controller", "XR_EXT_hp_mixed_reality_controller",
"XR_HTC_vive_cosmos_controller_interaction", "XR_HTC_vive_focus3_controller_interaction",
"XR_ML_ml2_controller_interaction", "XR_FB_touch_controller_pro", "XR_META_touch_controller_plus",
"XR_BD_controller_interaction", "XR_KHR_maintenance1"};
inline bool has(const Names& names, const std::string& n) {
return std::find(names.begin(), names.end(), n) != names.end();
}
inline bool downgrade(XrVersion v) { return v >= XR_MAKE_VERSION(1, 1, 0); }
inline XrVersion runtimeVersion(XrVersion v) { return downgrade(v) ? XR_API_VERSION_1_0 : v; }
struct ExtensionPlan { Names downstream, stubbed, missing, added; };
inline ExtensionPlan extensions(const Names& requested, const Names& available, bool promote) {
ExtensionPlan p;
for (const auto& n : requested) {
if (!has(available, n) && (n == stubs[0] || n == stubs[1])) p.stubbed.push_back(n);
else {
if (!has(p.downstream, n)) p.downstream.push_back(n);
if (!has(available, n)) p.missing.push_back(n);
}
}
if (promote) for (auto n : promotions) if (has(available, n) && !has(p.downstream, n)) {
p.downstream.push_back(n); p.added.push_back(n);
}
return p;
}
// 1.1's grip_surface is XR_EXT_palm_pose's palm_ext renamed (same pose), not the grip pose.
inline std::string rewritePath(std::string p, bool palm) {
const std::string suffix = "/input/grip_surface/pose";
if (palm && (p == "/user/hand/left" + suffix || p == "/user/hand/right" + suffix))
p.replace(p.size() - suffix.size(), suffix.size(), "/input/palm_ext/pose");
return p;
}
// Profiles added to 1.1. The three legacy Meta splits have no 1.0 extension.
// The renamed Pro/Plus profiles have different component semantics; do not
// claim the old extension implements the new profile merely because it exists.
inline bool dropProfile(const std::string& p, const Names& enabled) {
for (auto name : {"touch_pro_controller", "touch_plus_controller", "touch_controller_rift_cv1",
"touch_controller_quest_1_rift_s", "touch_controller_quest_2"})
if (p == std::string("/interaction_profiles/meta/") + name) return true;
const char* profiles[][2] = {
{"samsung/odyssey_controller", "XR_EXT_samsung_odyssey_controller"},
{"hp/mixed_reality_controller", "XR_EXT_hp_mixed_reality_controller"},
{"htc/vive_cosmos_controller", "XR_HTC_vive_cosmos_controller_interaction"},
{"htc/vive_focus3_controller", "XR_HTC_vive_focus3_controller_interaction"},
{"ml/ml2_controller", "XR_ML_ml2_controller_interaction"},
{"bytedance/pico_neo3_controller", "XR_BD_controller_interaction"},
{"bytedance/pico4_controller", "XR_BD_controller_interaction"},
{"bytedance/pico_g3_controller", "XR_BD_controller_interaction"}};
for (auto& entry : profiles) if (p == std::string("/interaction_profiles/") + entry[0])
return !has(enabled, entry[1]);
return false;
}
inline XrStructureType extensionType(XrStructureType t) {
switch (t) {
case XR_TYPE_SPACES_LOCATE_INFO: return XR_TYPE_SPACES_LOCATE_INFO_KHR;
case XR_TYPE_SPACE_LOCATIONS: return XR_TYPE_SPACE_LOCATIONS_KHR;
case XR_TYPE_SPACE_VELOCITIES: return XR_TYPE_SPACE_VELOCITIES_KHR;
default: return t;
}
}
inline XrResult locate(PFN_xrLocateSpacesKHR next, XrSession session,
const XrSpacesLocateInfo* info, XrSpaceLocations* locations) {
if (!info || !locations) return XR_ERROR_VALIDATION_FAILURE;
static_assert(XR_TYPE_SPACE_VELOCITIES == XR_TYPE_SPACE_VELOCITIES_KHR);
static_assert(XR_TYPE_SPACE_LOCATIONS == XR_TYPE_SPACE_LOCATIONS_KHR);
static_assert(XR_TYPE_SPACES_LOCATE_INFO == XR_TYPE_SPACES_LOCATE_INFO_KHR);
auto in = *info;
auto out = *locations;
in.type = extensionType(in.type); out.type = extensionType(out.type);
// The enums and typedefs are exact aliases. All chained velocities,
// including unknown next structures, can be forwarded without mutation.
const auto result = next(session, &in, &out);
locations->locationCount = out.locationCount;
return result;
}
}
frame/openxr-compat/evidence/apk-verification.txt
+29
View File
@@ -0,0 +1,29 @@
2026-09-28, Mac only; no device connection.
APK: /tmp/vrapk/wolvic-quest-compat.apk
SHA-256: 355a681625e38b71563dcffecb031799eff27894d6ab910a659bace057e82c1f
Command:
/tmp/vrapk/jdk/Contents/Home/bin/java -jar /tmp/vrapk/uber-apk-signer-1.3.0.jar -a /tmp/vrapk/wolvic-quest-compat.apk --onlyVerify
Exit status: 0
Relevant output:
- zipalign verified
- signature verified [v2, v3]
Successfully processed 1 APKs and 0 errors in 0.63 seconds.
ZIP readback checks: PASS
- Injected JSON identical to frame/openxr-compat/XrApiLayer_FRAME_compat.json.
- Injected SO identical to prebuilt/arm64-v8a/libXrApiLayer_FRAME_compat.so.
- Existing libopenxr_loader.so identical to /tmp/vrapk/wq-dec version.
- Binary AndroidManifest.xml contains android.intent.category.LAUNCHER.
Loader strings present:
openxr/1/api_layers/implicit.d/
openxr/1/api_layers/explicit.d/
ApiLayerManifestFile::AddManifestFilesAndroid unable to open asset
xrLocateSpaces
NDK: 30.0.16248370 (r30)
DMG size: 1072970961 bytes
Download-page SHA-1: 48591224b6657f46eebbc9d95d4af09bbed8d107
Measured SHA-1: 48591224b6657f46eebbc9d95d4af09bbed8d107
SHA check: PASS
frame/openxr-compat/evidence/ctest.txt
+9
View File
@@ -0,0 +1,9 @@
Test project /Users/saphid/projects/steam-frame-xrlayer/frame/openxr-compat/build-host
Start 1: compat_logic
1/2 Test #1: compat_logic ..................... Passed 0.01 sec
Start 2: layer_dispatch
2/2 Test #2: layer_dispatch ................... Passed 0.42 sec
100% tests passed out of 2
Total Test time (real) = 0.43 sec
frame/openxr-compat/evidence/readelf.txt
+110
View File
@@ -0,0 +1,110 @@
Elf file type is DYN (Shared object file)
Entry point 0x0
There are 9 program headers, starting at offset 64
Program Headers:
Type Offset VirtAddr PhysAddr FileSiz MemSiz Flg Align
PHDR 0x000040 0x0000000000000040 0x0000000000000040 0x0001f8 0x0001f8 R 0x8
LOAD 0x000000 0x0000000000000000 0x0000000000000000 0x0169e0 0x0169e0 R E 0x4000
LOAD 0x0169e0 0x000000000001a9e0 0x000000000001a9e0 0x000a90 0x001620 RW 0x4000
LOAD 0x017470 0x000000000001f470 0x000000000001f470 0x000068 0x000bc0 RW 0x4000
DYNAMIC 0x017048 0x000000000001b048 0x000000000001b048 0x0001c0 0x0001c0 RW 0x8
GNU_RELRO 0x0169e0 0x000000000001a9e0 0x000000000001a9e0 0x000a90 0x001620 R 0x1
GNU_EH_FRAME 0x004fb4 0x0000000000004fb4 0x0000000000004fb4 0x00070c 0x00070c R 0x4
GNU_STACK 0x000000 0x0000000000000000 0x0000000000000000 0x000000 0x000000 RW 0x0
NOTE 0x000238 0x0000000000000238 0x0000000000000238 0x0000bc 0x0000bc R 0x4
Section to Segment mapping:
Segment Sections...
00
01 .note.android.ident .note.gnu.build-id .dynsym .gnu.version .gnu.version_r .gnu.hash .dynstr .rela.dyn .rela.plt .gcc_except_table .rodata .eh_frame_hdr .eh_frame .text __lcxx_override .plt
02 .data.rel.ro .fini_array .init_array .dynamic .got .got.plt .relro_padding
03 .data .bss
04 .dynamic
05 .data.rel.ro .fini_array .init_array .dynamic .got .got.plt .relro_padding
06 .eh_frame_hdr
07
08 .note.android.ident .note.gnu.build-id
None .comment .shstrtab
Dynamic section at offset 0x17048 contains 28 entries:
Tag Type Name/Value
0x0000000000000001 (NEEDED) Shared library: [liblog.so]
0x0000000000000001 (NEEDED) Shared library: [libm.so]
0x0000000000000001 (NEEDED) Shared library: [libdl.so]
0x0000000000000001 (NEEDED) Shared library: [libc.so]
0x000000000000000e (SONAME) Library soname: [libXrApiLayer_FRAME_compat.so]
0x000000000000001e (FLAGS) BIND_NOW
0x000000006ffffffb (FLAGS_1) NOW
0x0000000000000007 (RELA) 0xae8
0x0000000000000008 (RELASZ) 5424 (bytes)
0x0000000000000009 (RELAENT) 24 (bytes)
0x000000006ffffff9 (RELACOUNT) 225
0x0000000000000017 (JMPREL) 0x2018
0x0000000000000002 (PLTRELSZ) 1080 (bytes)
0x0000000000000003 (PLTGOT) 0x1b2f0
0x0000000000000014 (PLTREL) RELA
0x0000000000000006 (SYMTAB) 0x2f8
0x000000000000000b (SYMENT) 24 (bytes)
0x0000000000000005 (STRTAB) 0x838
0x000000000000000a (STRSZ) 681 (bytes)
0x000000006ffffef5 (GNU_HASH) 0x818
0x0000000000000019 (INIT_ARRAY) 0x1b030
0x000000000000001b (INIT_ARRAYSZ) 24 (bytes)
0x000000000000001a (FINI_ARRAY) 0x1b020
0x000000000000001c (FINI_ARRAYSZ) 16 (bytes)
0x000000006ffffff0 (VERSYM) 0x778
0x000000006ffffffe (VERNEED) 0x7d8
0x000000006fffffff (VERNEEDNUM) 2
0x0000000000000000 (NULL) 0x0
Symbol table '.dynsym' contains 48 entries:
Num: Value Size Type Bind Vis Ndx Name
0: 0000000000000000 0 NOTYPE LOCAL DEFAULT UND
1: 0000000000000000 0 FUNC GLOBAL DEFAULT UND __cxa_finalize@LIBC
2: 0000000000000000 0 FUNC GLOBAL DEFAULT UND __cxa_atexit@LIBC
3: 0000000000000000 0 FUNC GLOBAL DEFAULT UND strcmp@LIBC
4: 0000000000000000 0 FUNC GLOBAL DEFAULT UND __android_log_print
5: 0000000000000000 0 FUNC GLOBAL DEFAULT UND __stack_chk_fail@LIBC
6: 0000000000000000 0 FUNC GLOBAL DEFAULT UND memmove@LIBC
7: 0000000000000000 0 FUNC GLOBAL DEFAULT UND strlen@LIBC
8: 0000000000000000 0 FUNC GLOBAL DEFAULT UND memcpy@LIBC
9: 0000000000000000 0 FUNC GLOBAL DEFAULT UND __strcpy_chk@LIBC
10: 0000000000000000 0 FUNC GLOBAL DEFAULT UND memset@LIBC
11: 0000000000000000 0 FUNC GLOBAL DEFAULT UND memcmp@LIBC
12: 0000000000000000 0 FUNC GLOBAL DEFAULT UND pthread_mutex_lock@LIBC
13: 0000000000000000 0 FUNC GLOBAL DEFAULT UND pthread_mutex_unlock@LIBC
14: 0000000000000000 0 FUNC GLOBAL DEFAULT UND __errno@LIBC
15: 0000000000000000 0 FUNC GLOBAL DEFAULT UND snprintf@LIBC
16: 0000000000000000 0 FUNC GLOBAL DEFAULT UND pthread_mutex_destroy@LIBC
17: 0000000000000000 0 FUNC GLOBAL DEFAULT UND strerror_r@LIBC
18: 0000000000000000 0 FUNC GLOBAL DEFAULT UND abort@LIBC
19: 0000000000000000 0 FUNC GLOBAL DEFAULT UND pthread_getspecific@LIBC
20: 0000000000000000 0 FUNC GLOBAL DEFAULT UND pthread_setspecific@LIBC
21: 0000000000000000 0 FUNC GLOBAL DEFAULT UND pthread_key_create@LIBC
22: 0000000000000000 0 FUNC GLOBAL DEFAULT UND syscall@LIBC
23: 0000000000000000 0 OBJECT GLOBAL DEFAULT UND __sF@LIBC
24: 0000000000000000 0 FUNC GLOBAL DEFAULT UND fwrite@LIBC
25: 0000000000000000 0 FUNC GLOBAL DEFAULT UND vfprintf@LIBC
26: 0000000000000000 0 FUNC GLOBAL DEFAULT UND fputc@LIBC
27: 0000000000000000 0 FUNC GLOBAL DEFAULT UND vasprintf@LIBC
28: 0000000000000000 0 FUNC GLOBAL DEFAULT UND android_set_abort_message@LIBC
29: 0000000000000000 0 FUNC GLOBAL DEFAULT UND openlog@LIBC
30: 0000000000000000 0 FUNC GLOBAL DEFAULT UND syslog@LIBC
31: 0000000000000000 0 FUNC GLOBAL DEFAULT UND closelog@LIBC
32: 0000000000000000 0 FUNC GLOBAL DEFAULT UND malloc@LIBC
33: 0000000000000000 0 FUNC GLOBAL DEFAULT UND free@LIBC
34: 0000000000000000 0 FUNC GLOBAL DEFAULT UND posix_memalign@LIBC
35: 0000000000000000 0 FUNC GLOBAL DEFAULT UND realloc@LIBC
36: 0000000000000000 0 FUNC GLOBAL DEFAULT UND pthread_once@LIBC
37: 0000000000000000 0 FUNC GLOBAL DEFAULT UND pthread_key_delete@LIBC
38: 0000000000000000 0 FUNC GLOBAL DEFAULT UND getauxval@LIBC
39: 0000000000000000 0 FUNC GLOBAL DEFAULT UND __system_property_get@LIBC
40: 0000000000000000 0 FUNC GLOBAL DEFAULT UND strncmp@LIBC
41: 0000000000000000 0 FUNC GLOBAL DEFAULT UND fprintf@LIBC
42: 0000000000000000 0 FUNC GLOBAL DEFAULT UND fflush@LIBC
43: 0000000000000000 0 FUNC GLOBAL DEFAULT UND pthread_rwlock_wrlock@LIBC
44: 0000000000000000 0 FUNC GLOBAL DEFAULT UND pthread_rwlock_unlock@LIBC
45: 0000000000000000 0 FUNC GLOBAL DEFAULT UND dl_iterate_phdr@LIBC
46: 0000000000000000 0 FUNC GLOBAL DEFAULT UND pthread_rwlock_rdlock@LIBC
47: 00000000000081a0 284 FUNC GLOBAL DEFAULT 14 xrNegotiateLoaderApiLayerInterface
frame/openxr-compat/layer.cpp
+289
View File
@@ -0,0 +1,289 @@
#include "compat_logic.h"
#include <loader_interfaces.h>
#include <jni.h>
#include <openxr/openxr_platform.h>
#include <android/log.h>
#include <cstring>
#include <memory>
#include <mutex>
#include <unordered_map>
#define LOG(...) __android_log_print(ANDROID_LOG_INFO, "FrameXrCompat", __VA_ARGS__)
#define GUARD_END catch (const std::bad_alloc&) { return XR_ERROR_OUT_OF_MEMORY; } catch (...) { return XR_ERROR_RUNTIME_FAILURE; }
namespace {
struct Dispatch {
PFN_xrGetInstanceProcAddr gipa{};
PFN_xrDestroyInstance destroy{};
PFN_xrCreateSession createSession{};
PFN_xrDestroySession destroySession{};
PFN_xrLocateSpacesKHR locate{};
PFN_xrStringToPath stringToPath{};
PFN_xrPathToString pathToString{};
PFN_xrSuggestInteractionProfileBindings suggest{};
PFN_xrSetAndroidApplicationThreadKHR thread{};
PFN_xrRequestDisplayRefreshRateFB refresh{};
frame::Names enabled, stubbed;
XrInstance instance{};
bool promote{}, palm{};
};
std::mutex mutex;
std::unordered_map<XrInstance, std::shared_ptr<Dispatch>> instances;
std::unordered_map<XrSession, std::shared_ptr<Dispatch>> sessions;
// No runtime library is opened here: every call goes through the next layer.
PFN_xrEnumerateInstanceExtensionProperties enumerateNext{};
template<class T> T proc(PFN_xrGetInstanceProcAddr gipa, XrInstance i, const char* name) {
PFN_xrVoidFunction f{};
if (XR_FAILED(gipa(i, name, &f))) return nullptr;
return reinterpret_cast<T>(f);
}
std::shared_ptr<Dispatch> get(XrInstance i) {
std::lock_guard<std::mutex> lock(mutex);
auto it = instances.find(i); return it == instances.end() ? nullptr : it->second;
}
std::shared_ptr<Dispatch> get(XrSession s) {
std::lock_guard<std::mutex> lock(mutex);
auto it = sessions.find(s); return it == sessions.end() ? nullptr : it->second;
}
XrResult properties(PFN_xrEnumerateInstanceExtensionProperties fn,
std::vector<XrExtensionProperties>& out) {
if (!fn) return XR_ERROR_FUNCTION_UNSUPPORTED;
for (int attempt = 0; attempt < 4; ++attempt) {
uint32_t n{};
auto r = fn(nullptr, 0, &n, nullptr);
if (XR_FAILED(r)) return r;
out.assign(n, {XR_TYPE_EXTENSION_PROPERTIES, nullptr, {}, 0});
if (!n) return XR_SUCCESS;
r = fn(nullptr, n, &n, out.data());
if (r == XR_ERROR_SIZE_INSUFFICIENT) continue;
if (XR_FAILED(r)) return r;
out.resize(n); return XR_SUCCESS;
}
return XR_ERROR_RUNTIME_FAILURE;
}
XrResult XRAPI_CALL enumerate(const char* layer, uint32_t capacity, uint32_t* count,
XrExtensionProperties* output) try {
if (!count || (capacity && !output)) return XR_ERROR_VALIDATION_FAILURE;
const bool own = layer && std::strcmp(layer, frame::layerName) == 0;
PFN_xrEnumerateInstanceExtensionProperties next;
{ std::lock_guard<std::mutex> lock(mutex); next = enumerateNext; }
if (layer && *layer && !own) return next ? next(layer, capacity, count, output) : XR_ERROR_API_LAYER_NOT_PRESENT;
std::vector<XrExtensionProperties> all;
if (!own && next) {
auto r = properties(next, all); if (XR_FAILED(r)) return r;
}
for (unsigned i = 0; i < 2; ++i) {
bool found = false;
for (auto& e : all) if (!std::strcmp(e.extensionName, frame::stubs[i])) found = true;
if (!found) {
XrExtensionProperties p{XR_TYPE_EXTENSION_PROPERTIES, nullptr, {}, i == 0 ? XR_KHR_android_thread_settings_SPEC_VERSION : 1u};
std::strcpy(p.extensionName, frame::stubs[i]); all.push_back(p);
}
}
*count = static_cast<uint32_t>(all.size());
LOG("enumerate extensions: %u (includes stubs)", *count);
if (!capacity) return XR_SUCCESS;
if (capacity < all.size()) return XR_ERROR_SIZE_INSUFFICIENT;
for (size_t i = 0; i < all.size(); ++i) {
if (output[i].type != XR_TYPE_EXTENSION_PROPERTIES) return XR_ERROR_VALIDATION_FAILURE;
std::strcpy(output[i].extensionName, all[i].extensionName);
output[i].extensionVersion = all[i].extensionVersion;
}
return XR_SUCCESS;
} GUARD_END
XrResult XRAPI_CALL destroyInstance(XrInstance i) try {
auto d = get(i); if (!d) return XR_ERROR_HANDLE_INVALID;
auto r = d->destroy(i);
if (XR_SUCCEEDED(r)) {
std::lock_guard<std::mutex> lock(mutex);
for (auto it = sessions.begin(); it != sessions.end();) {
if (it->second == d) it = sessions.erase(it); else ++it;
}
instances.erase(i);
if (instances.empty()) enumerateNext = nullptr;
}
return r;
} GUARD_END
XrResult XRAPI_CALL createSession(XrInstance i, const XrSessionCreateInfo* info, XrSession* s) try {
auto d = get(i); if (!d) return XR_ERROR_HANDLE_INVALID;
auto r = d->createSession(i, info, s);
if (XR_SUCCEEDED(r)) {
try { std::lock_guard<std::mutex> lock(mutex); sessions.emplace(*s, d); }
catch (...) { d->destroySession(*s); *s = XR_NULL_HANDLE; throw; }
}
return r;
} GUARD_END
XrResult XRAPI_CALL destroySession(XrSession s) try {
auto d = get(s); if (!d) return XR_ERROR_HANDLE_INVALID;
auto r = d->destroySession(s);
if (XR_SUCCEEDED(r)) { std::lock_guard<std::mutex> lock(mutex); sessions.erase(s); }
return r;
} GUARD_END
XrResult XRAPI_CALL locateSpaces(XrSession s, const XrSpacesLocateInfo* in, XrSpaceLocations* out) try {
auto d = get(s); if (!d) return XR_ERROR_HANDLE_INVALID;
if (!d->locate) return XR_ERROR_FUNCTION_UNSUPPORTED;
static std::once_flag logged; // called every frame
std::call_once(logged, [] { LOG("xrLocateSpaces -> xrLocateSpacesKHR (core/KHR type aliases)"); });
return frame::locate(d->locate, s, in, out);
} GUARD_END
XrResult XRAPI_CALL threadSettings(XrSession s, XrAndroidThreadTypeKHR type, uint32_t tid) try {
auto d = get(s); if (!d) return XR_ERROR_HANDLE_INVALID;
if (frame::has(d->stubbed, frame::stubs[0])) {
LOG("stub xrSetAndroidApplicationThreadKHR type=%d tid=%u -> XR_SUCCESS (no scheduling change)", type, tid);
return XR_SUCCESS;
}
return d->thread ? d->thread(s, type, tid) : XR_ERROR_FUNCTION_UNSUPPORTED;
} GUARD_END
// SteamVR offers only the current refresh rate; Quest apps ask for 72/90/120 Hz
// and abort on the error, so keep the current rate and report success.
XrResult XRAPI_CALL requestRefreshRate(XrSession s, float hz) try {
auto d = get(s); if (!d) return XR_ERROR_HANDLE_INVALID;
auto r = d->refresh(s, hz);
if (r == XR_ERROR_DISPLAY_REFRESH_RATE_UNSUPPORTED_FB) {
LOG("xrRequestDisplayRefreshRateFB %.1f Hz unsupported; keeping the current rate", hz);
return XR_SUCCESS;
}
return r;
} GUARD_END
XrResult XRAPI_CALL stringToPath(XrInstance i, const char* path, XrPath* out) try {
auto d = get(i); if (!d) return XR_ERROR_HANDLE_INVALID;
if (!path) return XR_ERROR_VALIDATION_FAILURE;
auto rewritten = frame::rewritePath(path, d->palm);
if (rewritten != path) LOG("path %s -> %s", path, rewritten.c_str());
return d->stringToPath(i, rewritten.c_str(), out);
} GUARD_END
XrResult pathString(const Dispatch& d, XrPath path, std::string& out) {
uint32_t n{};
auto r = d.pathToString(d.instance, path, 0, &n, nullptr);
if (XR_FAILED(r)) return r;
std::vector<char> text(n);
r = d.pathToString(d.instance, path, n, &n, text.data());
if (XR_SUCCEEDED(r)) out.assign(text.data());
return r;
}
XrResult XRAPI_CALL suggest(XrInstance i, const XrInteractionProfileSuggestedBinding* info) try {
auto d = get(i); if (!d) return XR_ERROR_HANDLE_INVALID;
if (!info || (info->countSuggestedBindings && !info->suggestedBindings)) return XR_ERROR_VALIDATION_FAILURE;
std::string profile;
auto r = pathString(*d, info->interactionProfile, profile);
if (XR_FAILED(r)) return r;
if (d->promote && frame::dropProfile(profile, d->enabled)) {
LOG("drop unsupported 1.1 interaction profile %s (%u suggestions)", profile.c_str(), info->countSuggestedBindings);
return XR_SUCCESS;
}
auto copy = *info;
std::vector<XrActionSuggestedBinding> bindings;
for (uint32_t index = 0; index < info->countSuggestedBindings; ++index) {
auto b = info->suggestedBindings[index];
std::string path;
r = pathString(*d, b.binding, path); if (XR_FAILED(r)) return r;
auto rewritten = frame::rewritePath(path, d->palm);
if (rewritten != path) {
LOG("binding %s -> %s", path.c_str(), rewritten.c_str());
r = d->stringToPath(i, rewritten.c_str(), &b.binding); if (XR_FAILED(r)) return r;
}
bindings.push_back(b);
}
copy.suggestedBindings = bindings.data();
return d->suggest(i, &copy);
} GUARD_END
XrResult XRAPI_CALL gipa(XrInstance, const char*, PFN_xrVoidFunction*);
XrResult XRAPI_CALL createLayer(const XrInstanceCreateInfo* info, const XrApiLayerCreateInfo* layer,
XrInstance* instance) try {
if (!info || !instance || !layer || layer->structType != XR_LOADER_INTERFACE_STRUCT_API_LAYER_CREATE_INFO ||
layer->structVersion != XR_API_LAYER_CREATE_INFO_STRUCT_VERSION || layer->structSize != sizeof(*layer) ||
!layer->nextInfo || layer->nextInfo->structType != XR_LOADER_INTERFACE_STRUCT_API_LAYER_NEXT_INFO ||
layer->nextInfo->structVersion != XR_API_LAYER_NEXT_INFO_STRUCT_VERSION ||
layer->nextInfo->structSize != sizeof(XrApiLayerNextInfo) ||
!layer->nextInfo->nextGetInstanceProcAddr || !layer->nextInfo->nextCreateApiLayerInstance ||
(info->enabledExtensionCount && !info->enabledExtensionNames)) return XR_ERROR_INITIALIZATION_FAILED;
*instance = XR_NULL_HANDLE;
auto next = layer->nextInfo->nextGetInstanceProcAddr;
auto enumFn = proc<PFN_xrEnumerateInstanceExtensionProperties>(next, XR_NULL_HANDLE, "xrEnumerateInstanceExtensionProperties");
std::vector<XrExtensionProperties> available;
auto r = properties(enumFn, available); if (XR_FAILED(r)) { LOG("runtime extension enumeration failed: %d", r); return r; }
frame::Names requested, supported;
for (auto& p : available) supported.emplace_back(p.extensionName);
for (uint32_t n = 0; n < info->enabledExtensionCount; ++n) {
if (!info->enabledExtensionNames[n]) return XR_ERROR_VALIDATION_FAILURE;
requested.emplace_back(info->enabledExtensionNames[n]);
}
auto d = std::make_shared<Dispatch>();
d->promote = frame::downgrade(info->applicationInfo.apiVersion);
auto plan = frame::extensions(requested, supported, d->promote);
for (auto& n : plan.added) LOG("enable promoted extension %s", n.c_str());
for (auto& n : plan.stubbed) LOG("emulate missing extension %s; remove downstream", n.c_str());
for (auto& n : plan.missing) LOG("unsupported extension %s retained; runtime must reject", n.c_str());
auto copy = *info;
copy.applicationInfo.apiVersion = frame::runtimeVersion(info->applicationInfo.apiVersion);
if (d->promote) LOG("API %u.%u.%u -> 1.0.%u", unsigned(XR_VERSION_MAJOR(info->applicationInfo.apiVersion)),
unsigned(XR_VERSION_MINOR(info->applicationInfo.apiVersion)), unsigned(XR_VERSION_PATCH(info->applicationInfo.apiVersion)),
unsigned(XR_VERSION_PATCH(copy.applicationInfo.apiVersion)));
std::vector<const char*> names;
for (auto& n : plan.downstream) names.push_back(n.c_str());
copy.enabledExtensionCount = static_cast<uint32_t>(names.size()); copy.enabledExtensionNames = names.data();
auto chain = *layer; chain.nextInfo = layer->nextInfo->next;
r = layer->nextInfo->nextCreateApiLayerInstance(&copy, &chain, instance);
if (XR_FAILED(r)) { LOG("downstream xrCreateInstance -> %d", r); return r; }
d->gipa = next; d->instance = *instance;
d->destroy = proc<PFN_xrDestroyInstance>(next, *instance, "xrDestroyInstance");
// A successful runtime instance must expose xrDestroyInstance.
if (!d->destroy) { // nothing could ever destroy it, so don't hand it out
LOG("invalid downstream: missing xrDestroyInstance"); *instance = XR_NULL_HANDLE; return XR_ERROR_INITIALIZATION_FAILED;
}
try {
d->enabled = std::move(plan.downstream); d->stubbed = std::move(plan.stubbed);
d->palm = frame::has(d->enabled, "XR_EXT_palm_pose");
#define LOAD(field, name) d->field = proc<PFN_##name>(next, *instance, #name)
LOAD(createSession, xrCreateSession); LOAD(destroySession, xrDestroySession);
LOAD(stringToPath, xrStringToPath); LOAD(pathToString, xrPathToString);
LOAD(suggest, xrSuggestInteractionProfileBindings);
if (frame::has(d->enabled, "XR_KHR_locate_spaces")) LOAD(locate, xrLocateSpacesKHR);
if (frame::has(d->enabled, frame::stubs[0])) LOAD(thread, xrSetAndroidApplicationThreadKHR);
if (frame::has(d->enabled, "XR_FB_display_refresh_rate")) LOAD(refresh, xrRequestDisplayRefreshRateFB);
#undef LOAD
if (!d->createSession || !d->destroySession || !d->stringToPath || !d->pathToString || !d->suggest) {
d->destroy(*instance); *instance = XR_NULL_HANDLE; return XR_ERROR_INITIALIZATION_FAILED;
}
std::lock_guard<std::mutex> lock(mutex);
instances.emplace(*instance, d); enumerateNext = enumFn;
} catch (...) { d->destroy(*instance); *instance = XR_NULL_HANDLE; throw; }
LOG("created instance; next-layer dispatch retained (Valve layer coexists)");
return r;
} GUARD_END
XrResult XRAPI_CALL gipa(XrInstance i, const char* name, PFN_xrVoidFunction* out) try {
if (!name || !out) return XR_ERROR_VALIDATION_FAILURE;
*out = nullptr;
#define RETURN_PROC(n, fn) if (!std::strcmp(name, n)) { *out = reinterpret_cast<PFN_xrVoidFunction>(fn); return XR_SUCCESS; }
RETURN_PROC("xrGetInstanceProcAddr", gipa)
RETURN_PROC("xrCreateApiLayerInstance", createLayer)
RETURN_PROC("xrEnumerateInstanceExtensionProperties", enumerate)
auto d = get(i); if (!d) return XR_ERROR_HANDLE_INVALID;
RETURN_PROC("xrDestroyInstance", destroyInstance)
RETURN_PROC("xrCreateSession", createSession)
RETURN_PROC("xrDestroySession", destroySession)
RETURN_PROC("xrStringToPath", stringToPath)
RETURN_PROC("xrSuggestInteractionProfileBindings", suggest)
if (d->locate && d->promote) { RETURN_PROC("xrLocateSpaces", locateSpaces) }
if (frame::has(d->stubbed, frame::stubs[0])) { RETURN_PROC("xrSetAndroidApplicationThreadKHR", threadSettings) }
if (d->refresh) { RETURN_PROC("xrRequestDisplayRefreshRateFB", requestRefreshRate) }
#undef RETURN_PROC
// Includes xrGetInstanceProperties: report the actual runtime's identity.
return d->gipa(i, name, out);
} GUARD_END
} // namespace
extern "C" __attribute__((visibility("default"))) XrResult XRAPI_CALL xrNegotiateLoaderApiLayerInterface(
const XrNegotiateLoaderInfo* loader, const char* name, XrNegotiateApiLayerRequest* request) {
if (!loader || !name || !request || std::strcmp(name, frame::layerName) ||
loader->structType != XR_LOADER_INTERFACE_STRUCT_LOADER_INFO ||
loader->structVersion != XR_LOADER_INFO_STRUCT_VERSION || loader->structSize != sizeof(*loader) ||
request->structType != XR_LOADER_INTERFACE_STRUCT_API_LAYER_REQUEST ||
request->structVersion != XR_API_LAYER_INFO_STRUCT_VERSION || request->structSize != sizeof(*request) ||
loader->minInterfaceVersion > 1 || loader->maxInterfaceVersion < 1 ||
loader->minApiVersion > XR_CURRENT_API_VERSION || loader->maxApiVersion < XR_MAKE_VERSION(1, 1, 0))
return XR_ERROR_INITIALIZATION_FAILED;
request->layerInterfaceVersion = 1;
request->layerApiVersion = std::min<XrVersion>(XR_CURRENT_API_VERSION, loader->maxApiVersion);
request->getInstanceProcAddr = gipa; request->createApiLayerInstance = createLayer;
LOG("negotiated interface 1, API 1.1");
return XR_SUCCESS;
}
frame/openxr-compat/prebuilt/arm64-v8a/libXrApiLayer_FRAME_compat.so
Executable
BIN
View File
Binary file not shown.
frame/openxr-compat/tests/compat_tests.cpp
+70
View File
@@ -0,0 +1,70 @@
#include "compat_logic.h"
#include <cstdlib>
#include <iostream>
#include <type_traits>
#define CHECK(expr) do { if (!(expr)) { std::cerr << __LINE__ << ": " #expr "\n"; std::exit(1); } } while (false)
namespace {
void* sentinel = reinterpret_cast<void*>(0x1234);
XrResult XRAPI_CALL fakeLocate(XrSession, const XrSpacesLocateInfo* in, XrSpaceLocations* out) {
CHECK(in->type == XR_TYPE_SPACES_LOCATE_INFO_KHR);
CHECK(out->type == XR_TYPE_SPACE_LOCATIONS_KHR);
CHECK(in->next == sentinel);
auto* velocities = static_cast<XrSpaceVelocitiesKHR*>(out->next);
CHECK(velocities->type == XR_TYPE_SPACE_VELOCITIES_KHR);
CHECK(velocities->next == sentinel);
CHECK(in->spaceCount == 1 && in->time == 123);
out->locations[0].pose.position.x = 42;
velocities->velocities[0].linearVelocity.y = 7;
return XR_SUCCESS;
}
}
int main() {
using namespace frame;
CHECK(!downgrade(XR_MAKE_VERSION(1, 0, 99)));
CHECK(runtimeVersion(XR_MAKE_VERSION(1, 0, 42)) == XR_MAKE_VERSION(1, 0, 42));
CHECK(runtimeVersion(XR_MAKE_VERSION(1, 1, 0)) == XR_API_VERSION_1_0);
CHECK(runtimeVersion(XR_MAKE_VERSION(1, 1, 999)) == XR_API_VERSION_1_0);
CHECK(runtimeVersion(XR_MAKE_VERSION(2, 0, 0)) == XR_API_VERSION_1_0);
Names available{"XR_KHR_locate_spaces", "XR_EXT_palm_pose", "XR_EXT_hand_interaction", "XR_VARJO_quad_views"};
Names requested{stubs[0], stubs[1], "XR_MISSING_test", "XR_EXT_palm_pose"};
auto p = extensions(requested, available, true);
CHECK(p.stubbed.size() == 2 && p.missing == Names{"XR_MISSING_test"});
CHECK(has(p.downstream, "XR_MISSING_test"));
CHECK(has(p.downstream, "XR_KHR_locate_spaces") && has(p.downstream, "XR_VARJO_quad_views"));
CHECK(!has(p.downstream, "XR_EXT_hand_interaction"));
CHECK(!has(p.downstream, "XR_EXT_local_floor"));
CHECK(std::count(p.downstream.begin(), p.downstream.end(), "XR_EXT_palm_pose") == 1);
available.push_back(stubs[0]);
CHECK(extensions(requested, available, true).stubbed == Names{stubs[1]});
CHECK(extensions({}, available, false).downstream.empty());
CHECK(rewritePath("/user/hand/left/input/grip_surface/pose", true) == "/user/hand/left/input/palm_ext/pose");
CHECK(rewritePath("/user/hand/right/input/grip_surface/pose", true) == "/user/hand/right/input/palm_ext/pose");
CHECK(rewritePath("/user/hand/left/input/grip_surface/pose", false) == "/user/hand/left/input/grip_surface/pose");
CHECK(rewritePath("/user/hand/left/input/grip_surface/pose/extra", true) == "/user/hand/left/input/grip_surface/pose/extra");
CHECK(rewritePath("/user/hand/left/input/grip/pose", true) == "/user/hand/left/input/grip/pose");
CHECK(dropProfile("/interaction_profiles/meta/touch_pro_controller", {}));
CHECK(dropProfile("/interaction_profiles/meta/touch_controller_quest_2", {}));
CHECK(dropProfile("/interaction_profiles/hp/mixed_reality_controller", {}));
CHECK(!dropProfile("/interaction_profiles/hp/mixed_reality_controller", {"XR_EXT_hp_mixed_reality_controller"}));
CHECK(!dropProfile("/interaction_profiles/oculus/touch_controller", {}));
CHECK(!dropProfile("/interaction_profiles/khr/simple_controller", {}));
CHECK(!dropProfile("/interaction_profiles/valve/frame_controller", {}));
CHECK(!dropProfile("/interaction_profiles/ext/hand_interaction_ext", {}));
CHECK(extensionType(XR_TYPE_SPACES_LOCATE_INFO) == XR_TYPE_SPACES_LOCATE_INFO_KHR);
CHECK(extensionType(XR_TYPE_SPACE_LOCATIONS) == XR_TYPE_SPACE_LOCATIONS_KHR);
CHECK(extensionType(XR_TYPE_SPACE_VELOCITIES) == XR_TYPE_SPACE_VELOCITIES_KHR);
CHECK(extensionType(XR_TYPE_INSTANCE_CREATE_INFO) == XR_TYPE_INSTANCE_CREATE_INFO);
static_assert(std::is_same_v<XrSpaceVelocities, XrSpaceVelocitiesKHR>);
XrSpace space{};
XrSpaceLocationData data{};
XrSpaceVelocityData velocity{};
XrSpaceVelocities velocities{XR_TYPE_SPACE_VELOCITIES, sentinel, 1, &velocity};
XrSpaceLocations locations{XR_TYPE_SPACE_LOCATIONS, &velocities, 1, &data};
XrSpacesLocateInfo info{XR_TYPE_SPACES_LOCATE_INFO, sentinel, XR_NULL_HANDLE, 123, 1, &space};
CHECK(locate(fakeLocate, XR_NULL_HANDLE, &info, &locations) == XR_SUCCESS);
CHECK(data.pose.position.x == 42 && velocity.linearVelocity.y == 7);
CHECK(info.next == sentinel && locations.next == &velocities && velocities.next == sentinel);
CHECK(info.type == XR_TYPE_SPACES_LOCATE_INFO && locations.type == XR_TYPE_SPACE_LOCATIONS && velocities.type == XR_TYPE_SPACE_VELOCITIES);
CHECK(locate(fakeLocate, XR_NULL_HANDLE, nullptr, &locations) == XR_ERROR_VALIDATION_FAILURE);
std::cout << "All version, extension, path/profile and locate-chain checks passed\n";
}
frame/openxr-compat/tests/dispatch_tests.cpp
+134
View File
@@ -0,0 +1,134 @@
// Compile the actual layer with host-only JNI/logging shims, and a fake next
// layer. This exercises ABI routing without a runtime or a headset.
#include "../layer.cpp"
#include <cstdlib>
#include <iostream>
#define CHECK(expr) do { if (!(expr)) { std::cerr << __LINE__ << ": " #expr "\n"; std::exit(1); } } while (false)
namespace {
uintptr_t serial = 100;
frame::Names seenExtensions;
XrVersion seenVersion{};
XrApiLayerNextInfo* seenNext{};
int suggestionCalls{}, locateCalls{}, realThreadCalls{};
bool nativeThread{};
std::unordered_map<XrPath, std::string> paths;
XrResult XRAPI_CALL fakeEnumerate(const char*, uint32_t capacity, uint32_t* count, XrExtensionProperties* out) {
const char* extensions[] = {"XR_KHR_locate_spaces", "XR_EXT_palm_pose", frame::stubs[0]};
*count = nativeThread ? 3 : 2;
if (!capacity) return XR_SUCCESS;
if (capacity < *count) return XR_ERROR_SIZE_INSUFFICIENT;
for (uint32_t j = 0; j < *count; ++j) { std::strcpy(out[j].extensionName, extensions[j]); out[j].extensionVersion = 1; }
return XR_SUCCESS;
}
XrResult XRAPI_CALL fakeCreate(const XrInstanceCreateInfo* in, const XrApiLayerCreateInfo* layer, XrInstance* i) {
seenVersion = in->applicationInfo.apiVersion; seenNext = layer->nextInfo;
seenExtensions.clear();
for (uint32_t j = 0; j < in->enabledExtensionCount; ++j) seenExtensions.emplace_back(in->enabledExtensionNames[j]);
if (frame::has(seenExtensions, "XR_MISSING_test")) return XR_ERROR_EXTENSION_NOT_PRESENT;
*i = reinterpret_cast<XrInstance>(++serial); return XR_SUCCESS;
}
XrResult XRAPI_CALL fakeDestroy(XrInstance) { return XR_SUCCESS; }
XrResult XRAPI_CALL fakeCreateSession(XrInstance, const XrSessionCreateInfo*, XrSession* s) {
*s = reinterpret_cast<XrSession>(++serial); return XR_SUCCESS;
}
XrResult XRAPI_CALL fakeDestroySession(XrSession) { return XR_SUCCESS; }
XrResult XRAPI_CALL fakeString(XrInstance, const char* text, XrPath* p) {
*p = ++serial; paths[*p] = text; return XR_SUCCESS;
}
XrResult XRAPI_CALL fakePath(XrInstance, XrPath p, uint32_t cap, uint32_t* count, char* out) {
if (!paths.count(p)) return XR_ERROR_PATH_INVALID;
*count = static_cast<uint32_t>(paths[p].size() + 1);
if (!cap) return XR_SUCCESS;
if (cap < *count) return XR_ERROR_SIZE_INSUFFICIENT;
std::strcpy(out, paths[p].c_str()); return XR_SUCCESS;
}
XrResult XRAPI_CALL fakeSuggest(XrInstance, const XrInteractionProfileSuggestedBinding* info) {
++suggestionCalls;
if (info->countSuggestedBindings) CHECK(paths[info->suggestedBindings[0].binding] == "/user/hand/left/input/palm_ext/pose");
return XR_SUCCESS;
}
XrResult XRAPI_CALL fakeLocate(XrSession, const XrSpacesLocateInfo*, XrSpaceLocations*) { ++locateCalls; return XR_SUCCESS; }
XrResult XRAPI_CALL fakeThread(XrSession, XrAndroidThreadTypeKHR, uint32_t) { ++realThreadCalls; return XR_TIMEOUT_EXPIRED; }
XrResult XRAPI_CALL fakeProperties(XrInstance, XrInstanceProperties*) { return XR_SUCCESS; }
XrResult XRAPI_CALL fakeGipa(XrInstance, const char* n, PFN_xrVoidFunction* out) {
#define MAP(name, f) if (!std::strcmp(n, name)) { *out = reinterpret_cast<PFN_xrVoidFunction>(f); return XR_SUCCESS; }
MAP("xrEnumerateInstanceExtensionProperties", fakeEnumerate)
MAP("xrDestroyInstance", fakeDestroy)
MAP("xrCreateSession", fakeCreateSession) MAP("xrDestroySession", fakeDestroySession)
MAP("xrStringToPath", fakeString) MAP("xrPathToString", fakePath)
MAP("xrSuggestInteractionProfileBindings", fakeSuggest) MAP("xrLocateSpacesKHR", fakeLocate)
MAP("xrSetAndroidApplicationThreadKHR", fakeThread) MAP("xrGetInstanceProperties", fakeProperties)
#undef MAP
*out = nullptr; return XR_ERROR_FUNCTION_UNSUPPORTED;
}
}
int main() {
XrNegotiateLoaderInfo loader{XR_LOADER_INTERFACE_STRUCT_LOADER_INFO, 1, sizeof(XrNegotiateLoaderInfo), 1, 1,
XR_MAKE_VERSION(1,0,0), XR_MAKE_VERSION(1,1023,4095)};
XrNegotiateApiLayerRequest request{};
request.structType = XR_LOADER_INTERFACE_STRUCT_API_LAYER_REQUEST; request.structVersion = 1; request.structSize = sizeof(request);
CHECK(xrNegotiateLoaderApiLayerInterface(&loader, frame::layerName, &request) == XR_SUCCESS);
CHECK(request.getInstanceProcAddr && request.createApiLayerInstance);
loader.maxApiVersion = XR_MAKE_VERSION(1,1,0);
CHECK(xrNegotiateLoaderApiLayerInterface(&loader, frame::layerName, &request) == XR_SUCCESS);
CHECK(request.layerApiVersion == loader.maxApiVersion);
loader.maxApiVersion = XR_MAKE_VERSION(1,0,99);
CHECK(xrNegotiateLoaderApiLayerInterface(&loader, frame::layerName, &request) == XR_ERROR_INITIALIZATION_FAILED);
loader.maxApiVersion = XR_MAKE_VERSION(1,1023,4095);
CHECK(xrNegotiateLoaderApiLayerInterface(&loader, "wrong", &request) == XR_ERROR_INITIALIZATION_FAILED);
XrApiLayerNextInfo tail{};
XrApiLayerNextInfo next{XR_LOADER_INTERFACE_STRUCT_API_LAYER_NEXT_INFO, 1, sizeof(XrApiLayerNextInfo), {}, fakeGipa, fakeCreate, &tail};
XrApiLayerCreateInfo layer{XR_LOADER_INTERFACE_STRUCT_API_LAYER_CREATE_INFO, 1, sizeof(XrApiLayerCreateInfo), nullptr, {}, &next};
XrInstanceCreateInfo info{XR_TYPE_INSTANCE_CREATE_INFO, nullptr, 0, {}, 0, nullptr, 2, frame::stubs};
info.applicationInfo.apiVersion = XR_MAKE_VERSION(1,1,999);
XrInstance a{}, b{};
CHECK(createLayer(&info, &layer, &a) == XR_SUCCESS);
CHECK(seenVersion == XR_API_VERSION_1_0 && info.applicationInfo.apiVersion == XR_MAKE_VERSION(1,1,999));
CHECK(seenNext == &tail && layer.nextInfo == &next);
CHECK(!frame::has(seenExtensions, frame::stubs[0]) && frame::has(seenExtensions, "XR_KHR_locate_spaces"));
XrSession sa{}, sb{};
CHECK(createSession(a, nullptr, &sa) == XR_SUCCESS);
CHECK(threadSettings(sa, XR_ANDROID_THREAD_TYPE_APPLICATION_MAIN_KHR, 1) == XR_SUCCESS);
nativeThread = true;
CHECK(createLayer(&info, &layer, &b) == XR_SUCCESS);
CHECK(frame::has(seenExtensions, frame::stubs[0]));
CHECK(createSession(b, nullptr, &sb) == XR_SUCCESS);
PFN_xrVoidFunction fn{};
CHECK(gipa(b, "xrSetAndroidApplicationThreadKHR", &fn) == XR_SUCCESS);
CHECK(reinterpret_cast<PFN_xrSetAndroidApplicationThreadKHR>(fn)(sb, XR_ANDROID_THREAD_TYPE_APPLICATION_MAIN_KHR, 1) == XR_TIMEOUT_EXPIRED);
CHECK(realThreadCalls == 1);
CHECK(threadSettings(sa, XR_ANDROID_THREAD_TYPE_APPLICATION_MAIN_KHR, 1) == XR_SUCCESS);
CHECK(gipa(a, "xrGetInstanceProperties", &fn) == XR_SUCCESS && fn == reinterpret_cast<PFN_xrVoidFunction>(fakeProperties));
CHECK(gipa(a, "xrLocateSpaces", &fn) == XR_SUCCESS);
XrSpacesLocateInfo locateInfo{XR_TYPE_SPACES_LOCATE_INFO, nullptr, XR_NULL_HANDLE, 1, 0, nullptr};
XrSpaceLocations locations{XR_TYPE_SPACE_LOCATIONS, nullptr, 0, nullptr};
CHECK(reinterpret_cast<PFN_xrLocateSpaces>(fn)(sa, &locateInfo, &locations) == XR_SUCCESS && locateCalls == 1);
XrPath binding{}, profile{};
CHECK(stringToPath(a, "/user/hand/left/input/grip_surface/pose", &binding) == XR_SUCCESS);
CHECK(paths[binding] == "/user/hand/left/input/palm_ext/pose");
fakeString(a, "/user/hand/left/input/grip_surface/pose", &binding); // preexisting path: rewrite in suggest too
fakeString(a, "/interaction_profiles/oculus/touch_controller", &profile);
XrActionSuggestedBinding action{XR_NULL_HANDLE, binding};
XrInteractionProfileSuggestedBinding suggested{XR_TYPE_INTERACTION_PROFILE_SUGGESTED_BINDING, nullptr, profile, 1, &action};
CHECK(suggest(a, &suggested) == XR_SUCCESS && suggestionCalls == 1);
CHECK(paths[action.binding] == "/user/hand/left/input/grip_surface/pose");
fakeString(a, "/interaction_profiles/meta/touch_pro_controller", &suggested.interactionProfile);
CHECK(suggest(a, &suggested) == XR_SUCCESS && suggestionCalls == 1);
uint32_t count{};
CHECK(enumerate(frame::layerName, 0, &count, nullptr) == XR_SUCCESS && count == 2);
XrExtensionProperties props[2]{{XR_TYPE_EXTENSION_PROPERTIES, nullptr, {}, 0}, {XR_TYPE_EXTENSION_PROPERTIES, nullptr, {}, 0}};
CHECK(enumerate(frame::layerName, 1, &count, props) == XR_ERROR_SIZE_INSUFFICIENT);
CHECK(enumerate(frame::layerName, 2, &count, props) == XR_SUCCESS && count == 2);
CHECK(enumerate(nullptr, 0, &count, nullptr) == XR_SUCCESS && count == 4);
const char* missing = "XR_MISSING_test";
info.enabledExtensionCount = 1; info.enabledExtensionNames = &missing;
XrInstance failed{};
CHECK(createLayer(&info, &layer, &failed) == XR_ERROR_EXTENSION_NOT_PRESENT && failed == XR_NULL_HANDLE);
CHECK(destroySession(sa) == XR_SUCCESS);
CHECK(threadSettings(sa, XR_ANDROID_THREAD_TYPE_APPLICATION_MAIN_KHR, 1) == XR_ERROR_HANDLE_INVALID);
CHECK(destroyInstance(a) == XR_SUCCESS);
CHECK(get(sb) && !get(a));
CHECK(destroyInstance(b) == XR_SUCCESS && !get(sb));
CHECK(instances.empty() && sessions.empty() && enumerateNext == nullptr);
std::cout << "Negotiation, next-layer chaining, multi-instance dispatch, stubs, forwarding and cleanup passed\n";
}
frame/openxr-compat/tests/shims/android/log.h
+9
View File
@@ -0,0 +1,9 @@
#pragma once
// Host-only logcat replacement. Android builds use the NDK header and liblog.
#include <cstdarg>
#include <cstdio>
#define ANDROID_LOG_INFO 4
inline int __android_log_print(int, const char*, const char* fmt, ...) {
va_list args; va_start(args, fmt); auto r = std::vfprintf(stderr, fmt, args);
va_end(args); std::fputc('\n', stderr); return r;
}
frame/openxr-compat/tests/shims/jni.h
+3
View File
@@ -0,0 +1,3 @@
#pragma once
// Only the opaque jobject declaration is needed by openxr_platform.h.
typedef void* jobject;
frame/openxr-compat/vendor/LICENSE
Vendored
+208
View File
@@ -0,0 +1,208 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION,
AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction, and distribution
as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by the copyright
owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all other entities
that control, are controlled by, or are under common control with that entity.
For the purposes of this definition, "control" means (i) the power, direct
or indirect, to cause the direction or management of such entity, whether
by contract or otherwise, or (ii) ownership of fifty percent (50%) or more
of the outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity exercising permissions
granted by this License.
"Source" form shall mean the preferred form for making modifications, including
but not limited to software source code, documentation source, and configuration
files.
"Object" form shall mean any form resulting from mechanical transformation
or translation of a Source form, including but not limited to compiled object
code, generated documentation, and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or Object form,
made available under the License, as indicated by a copyright notice that
is included in or attached to the work (an example is provided in the Appendix
below).
"Derivative Works" shall mean any work, whether in Source or Object form,
that is based on (or derived from) the Work and for which the editorial revisions,
annotations, elaborations, or other modifications represent, as a whole, an
original work of authorship. For the purposes of this License, Derivative
Works shall not include works that remain separable from, or merely link (or
bind by name) to the interfaces of, the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including the original version
of the Work and any modifications or additions to that Work or Derivative
Works thereof, that is intentionally submitted to Licensor for inclusion in
the Work by the copyright owner or by an individual or Legal Entity authorized
to submit on behalf of the copyright owner. For the purposes of this definition,
"submitted" means any form of electronic, verbal, or written communication
sent to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems, and
issue tracking systems that are managed by, or on behalf of, the Licensor
for the purpose of discussing and improving the Work, but excluding communication
that is conspicuously marked or otherwise designated in writing by the copyright
owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity on behalf
of whom a Contribution has been received by Licensor and subsequently incorporated
within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of this
License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive,
no-charge, royalty-free, irrevocable copyright license to reproduce, prepare
Derivative Works of, publicly display, publicly perform, sublicense, and distribute
the Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of this License,
each Contributor hereby grants to You a perpetual, worldwide, non-exclusive,
no-charge, royalty-free, irrevocable (except as stated in this section) patent
license to make, have made, use, offer to sell, sell, import, and otherwise
transfer the Work, where such license applies only to those patent claims
licensable by such Contributor that are necessarily infringed by their Contribution(s)
alone or by combination of their Contribution(s) with the Work to which such
Contribution(s) was submitted. If You institute patent litigation against
any entity (including a cross-claim or counterclaim in a lawsuit) alleging
that the Work or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses granted to You
under this License for that Work shall terminate as of the date such litigation
is filed.
4. Redistribution. You may reproduce and distribute copies of the Work or
Derivative Works thereof in any medium, with or without modifications, and
in Source or Object form, provided that You meet the following conditions:
(a) You must give any other recipients of the Work or Derivative Works a copy
of this License; and
(b) You must cause any modified files to carry prominent notices stating that
You changed the files; and
(c) You must retain, in the Source form of any Derivative Works that You distribute,
all copyright, patent, trademark, and attribution notices from the Source
form of the Work, excluding those notices that do not pertain to any part
of the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its distribution,
then any Derivative Works that You distribute must include a readable copy
of the attribution notices contained within such NOTICE file, excluding those
notices that do not pertain to any part of the Derivative Works, in at least
one of the following places: within a NOTICE text file distributed as part
of the Derivative Works; within the Source form or documentation, if provided
along with the Derivative Works; or, within a display generated by the Derivative
Works, if and wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and do not modify the
License. You may add Your own attribution notices within Derivative Works
that You distribute, alongside or as an addendum to the NOTICE text from the
Work, provided that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and may provide
additional or different license terms and conditions for use, reproduction,
or distribution of Your modifications, or for any such Derivative Works as
a whole, provided Your use, reproduction, and distribution of the Work otherwise
complies with the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise, any
Contribution intentionally submitted for inclusion in the Work by You to the
Licensor shall be under the terms and conditions of this License, without
any additional terms or conditions. Notwithstanding the above, nothing herein
shall supersede or modify the terms of any separate license agreement you
may have executed with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade names,
trademarks, service marks, or product names of the Licensor, except as required
for reasonable and customary use in describing the origin of the Work and
reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or agreed to
in writing, Licensor provides the Work (and each Contributor provides its
Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied, including, without limitation, any warranties
or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR
A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness
of using or redistributing the Work and assume any risks associated with Your
exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory, whether
in tort (including negligence), contract, or otherwise, unless required by
applicable law (such as deliberate and grossly negligent acts) or agreed to
in writing, shall any Contributor be liable to You for damages, including
any direct, indirect, special, incidental, or consequential damages of any
character arising as a result of this License or out of the use or inability
to use the Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all other commercial
damages or losses), even if such Contributor has been advised of the possibility
of such damages.
9. Accepting Warranty or Additional Liability. While redistributing the Work
or Derivative Works thereof, You may choose to offer, and charge a fee for,
acceptance of support, warranty, indemnity, or other liability obligations
and/or rights consistent with this License. However, in accepting such obligations,
You may act only on Your own behalf and on Your sole responsibility, not on
behalf of any other Contributor, and only if You agree to indemnify, defend,
and hold each Contributor harmless for any liability incurred by, or claims
asserted against, such Contributor by reason of your accepting any such warranty
or additional liability. END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following boilerplate
notice, with the fields enclosed by brackets "[]" replaced with your own identifying
information. (Don't include the brackets!) The text should be enclosed in
the appropriate comment syntax for the file format. We also recommend that
a file or class name and description of purpose be included on the same "printed
page" as the copyright notice for easier identification within third-party
archives.
Copyright [yyyy] [name of copyright owner]
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
frame/openxr-compat/vendor/loader_interfaces.h
Vendored
+114
View File
@@ -0,0 +1,114 @@
// Copyright (c) 2017-2023, The Khronos Group Inc.
// Copyright (c) 2017 Valve Corporation
// Copyright (c) 2017 LunarG, Inc.
//
// SPDX-License-Identifier: Apache-2.0 OR MIT
//
// Initial Author: Mark Young <marky@lunarg.com>
//
#pragma once
#include <openxr/openxr.h>
#ifdef __cplusplus
extern "C" {
#endif
// Forward declare.
typedef struct XrApiLayerCreateInfo XrApiLayerCreateInfo;
// Function pointer prototype for the xrCreateApiLayerInstance function used in place of xrCreateInstance.
// This function allows us to pass special API layer information to each layer during the process of creating an Instance.
typedef XrResult(XRAPI_PTR *PFN_xrCreateApiLayerInstance)(const XrInstanceCreateInfo *info,
const XrApiLayerCreateInfo *apiLayerInfo, XrInstance *instance);
// Loader/API Layer Interface versions
// 1 - First version, introduces negotiation structure and functions
#define XR_CURRENT_LOADER_API_LAYER_VERSION 1
// Loader/Runtime Interface versions
// 1 - First version, introduces negotiation structure and functions
#define XR_CURRENT_LOADER_RUNTIME_VERSION 1
// Version negotiation values
typedef enum XrLoaderInterfaceStructs {
XR_LOADER_INTERFACE_STRUCT_UNINTIALIZED = 0,
XR_LOADER_INTERFACE_STRUCT_LOADER_INFO,
XR_LOADER_INTERFACE_STRUCT_API_LAYER_REQUEST,
XR_LOADER_INTERFACE_STRUCT_RUNTIME_REQUEST,
XR_LOADER_INTERFACE_STRUCT_API_LAYER_CREATE_INFO,
XR_LOADER_INTERFACE_STRUCT_API_LAYER_NEXT_INFO,
} XrLoaderInterfaceStructs;
#define XR_LOADER_INFO_STRUCT_VERSION 1
typedef struct XrNegotiateLoaderInfo {
XrLoaderInterfaceStructs structType; // XR_LOADER_INTERFACE_STRUCT_LOADER_INFO
uint32_t structVersion; // XR_LOADER_INFO_STRUCT_VERSION
size_t structSize; // sizeof(XrNegotiateLoaderInfo)
uint32_t minInterfaceVersion;
uint32_t maxInterfaceVersion;
XrVersion minApiVersion;
XrVersion maxApiVersion;
} XrNegotiateLoaderInfo;
#define XR_API_LAYER_INFO_STRUCT_VERSION 1
typedef struct XrNegotiateApiLayerRequest {
XrLoaderInterfaceStructs structType; // XR_LOADER_INTERFACE_STRUCT_API_LAYER_REQUEST
uint32_t structVersion; // XR_API_LAYER_INFO_STRUCT_VERSION
size_t structSize; // sizeof(XrNegotiateApiLayerRequest)
uint32_t layerInterfaceVersion; // CURRENT_LOADER_API_LAYER_VERSION
XrVersion layerApiVersion;
PFN_xrGetInstanceProcAddr getInstanceProcAddr;
PFN_xrCreateApiLayerInstance createApiLayerInstance;
} XrNegotiateApiLayerRequest;
#define XR_RUNTIME_INFO_STRUCT_VERSION 1
typedef struct XrNegotiateRuntimeRequest {
XrLoaderInterfaceStructs structType; // XR_LOADER_INTERFACE_STRUCT_RUNTIME_REQUEST
uint32_t structVersion; // XR_RUNTIME_INFO_STRUCT_VERSION
size_t structSize; // sizeof(XrNegotiateRuntimeRequest)
uint32_t runtimeInterfaceVersion; // CURRENT_LOADER_RUNTIME_VERSION
XrVersion runtimeApiVersion;
PFN_xrGetInstanceProcAddr getInstanceProcAddr;
} XrNegotiateRuntimeRequest;
// Function used to negotiate an interface betewen the loader and an API layer. Each library exposing one or
// more API layers needs to expose at least this function.
typedef XrResult(XRAPI_PTR *PFN_xrNegotiateLoaderApiLayerInterface)(const XrNegotiateLoaderInfo *loaderInfo,
const char *apiLayerName,
XrNegotiateApiLayerRequest *apiLayerRequest);
// Function used to negotiate an interface betewen the loader and a runtime. Each runtime should expose
// at least this function.
typedef XrResult(XRAPI_PTR *PFN_xrNegotiateLoaderRuntimeInterface)(const XrNegotiateLoaderInfo *loaderInfo,
XrNegotiateRuntimeRequest *runtimeRequest);
// Forward declare.
typedef struct XrApiLayerNextInfo XrApiLayerNextInfo;
#define XR_API_LAYER_NEXT_INFO_STRUCT_VERSION 1
struct XrApiLayerNextInfo {
XrLoaderInterfaceStructs structType; // XR_LOADER_INTERFACE_STRUCT_API_LAYER_NEXT_INFO
uint32_t structVersion; // XR_API_LAYER_NEXT_INFO_STRUCT_VERSION
size_t structSize; // sizeof(XrApiLayerNextInfo)
char layerName[XR_MAX_API_LAYER_NAME_SIZE]; // Name of API layer which should receive this info
PFN_xrGetInstanceProcAddr nextGetInstanceProcAddr; // Pointer to next API layer's xrGetInstanceProcAddr
PFN_xrCreateApiLayerInstance nextCreateApiLayerInstance; // Pointer to next API layer's xrCreateApiLayerInstance
XrApiLayerNextInfo *next; // Pointer to the next API layer info in the sequence
};
#define XR_API_LAYER_MAX_SETTINGS_PATH_SIZE 512
#define XR_API_LAYER_CREATE_INFO_STRUCT_VERSION 1
typedef struct XrApiLayerCreateInfo {
XrLoaderInterfaceStructs structType; // XR_LOADER_INTERFACE_STRUCT_API_LAYER_CREATE_INFO
uint32_t structVersion; // XR_API_LAYER_CREATE_INFO_STRUCT_VERSION
size_t structSize; // sizeof(XrApiLayerCreateInfo)
void *loaderInstance; // Pointer to the LoaderInstance class
char settings_file_location[XR_API_LAYER_MAX_SETTINGS_PATH_SIZE]; // Location to the found settings file (or empty '\0')
XrApiLayerNextInfo *nextInfo; // Pointer to the next API layer's Info
} XrApiLayerCreateInfo;
#ifdef __cplusplus
} // extern "C"
#endif
frame/openxr-compat/vendor/openxr/openxr.h
Vendored
+14911
View File
File diff suppressed because it is too large. Load diff
frame/openxr-compat/vendor/openxr/openxr_platform.h
Vendored
+819
View File
@@ -0,0 +1,819 @@
#ifndef OPENXR_PLATFORM_H_
#define OPENXR_PLATFORM_H_ 1
/*
** Copyright 2017-2026 The Khronos Group Inc.
**
** SPDX-License-Identifier: Apache-2.0 OR MIT
*/
/*
** This header is generated from the Khronos OpenXR XML API Registry.
**
*/
#include "openxr.h"
#ifdef __cplusplus
extern "C" {
#endif
#ifdef XR_USE_PLATFORM_ANDROID
// XR_KHR_android_thread_settings is a preprocessor guard. Do not pass it to API calls.
#define XR_KHR_android_thread_settings 1
#define XR_KHR_android_thread_settings_SPEC_VERSION 6
#define XR_KHR_ANDROID_THREAD_SETTINGS_EXTENSION_NAME "XR_KHR_android_thread_settings"
typedef enum XrAndroidThreadTypeKHR {
XR_ANDROID_THREAD_TYPE_APPLICATION_MAIN_KHR = 1,
XR_ANDROID_THREAD_TYPE_APPLICATION_WORKER_KHR = 2,
XR_ANDROID_THREAD_TYPE_RENDERER_MAIN_KHR = 3,
XR_ANDROID_THREAD_TYPE_RENDERER_WORKER_KHR = 4,
XR_ANDROID_THREAD_TYPE_MAX_ENUM_KHR = 0x7FFFFFFF
} XrAndroidThreadTypeKHR;
typedef XrResult (XRAPI_PTR *PFN_xrSetAndroidApplicationThreadKHR)(XrSession session, XrAndroidThreadTypeKHR threadType, uint32_t threadId);
#ifndef XR_NO_PROTOTYPES
#ifdef XR_EXTENSION_PROTOTYPES
XRAPI_ATTR XrResult XRAPI_CALL xrSetAndroidApplicationThreadKHR(
XrSession session,
XrAndroidThreadTypeKHR threadType,
uint32_t threadId);
#endif /* XR_EXTENSION_PROTOTYPES */
#endif /* !XR_NO_PROTOTYPES */
#endif /* XR_USE_PLATFORM_ANDROID */
#ifdef XR_USE_PLATFORM_ANDROID
// XR_KHR_android_surface_swapchain is a preprocessor guard. Do not pass it to API calls.
#define XR_KHR_android_surface_swapchain 1
#define XR_KHR_android_surface_swapchain_SPEC_VERSION 4
#define XR_KHR_ANDROID_SURFACE_SWAPCHAIN_EXTENSION_NAME "XR_KHR_android_surface_swapchain"
typedef XrResult (XRAPI_PTR *PFN_xrCreateSwapchainAndroidSurfaceKHR)(XrSession session, const XrSwapchainCreateInfo* info, XrSwapchain* swapchain, jobject* surface);
#ifndef XR_NO_PROTOTYPES
#ifdef XR_EXTENSION_PROTOTYPES
XRAPI_ATTR XrResult XRAPI_CALL xrCreateSwapchainAndroidSurfaceKHR(
XrSession session,
const XrSwapchainCreateInfo* info,
XrSwapchain* swapchain,
jobject* surface);
#endif /* XR_EXTENSION_PROTOTYPES */
#endif /* !XR_NO_PROTOTYPES */
#endif /* XR_USE_PLATFORM_ANDROID */
#ifdef XR_USE_PLATFORM_ANDROID
// XR_KHR_android_create_instance is a preprocessor guard. Do not pass it to API calls.
#define XR_KHR_android_create_instance 1
#define XR_KHR_android_create_instance_SPEC_VERSION 3
#define XR_KHR_ANDROID_CREATE_INSTANCE_EXTENSION_NAME "XR_KHR_android_create_instance"
// XrInstanceCreateInfoAndroidKHR extends XrInstanceCreateInfo
typedef struct XrInstanceCreateInfoAndroidKHR {
XrStructureType type;
const void* XR_MAY_ALIAS next;
void* XR_MAY_ALIAS applicationVM;
void* XR_MAY_ALIAS applicationActivity;
} XrInstanceCreateInfoAndroidKHR;
#endif /* XR_USE_PLATFORM_ANDROID */
#ifdef XR_USE_GRAPHICS_API_VULKAN
// XR_KHR_vulkan_swapchain_format_list is a preprocessor guard. Do not pass it to API calls.
#define XR_KHR_vulkan_swapchain_format_list 1
#define XR_KHR_vulkan_swapchain_format_list_SPEC_VERSION 5
#define XR_KHR_VULKAN_SWAPCHAIN_FORMAT_LIST_EXTENSION_NAME "XR_KHR_vulkan_swapchain_format_list"
// XrVulkanSwapchainFormatListCreateInfoKHR extends XrSwapchainCreateInfo
typedef struct XrVulkanSwapchainFormatListCreateInfoKHR {
XrStructureType type;
const void* XR_MAY_ALIAS next;
uint32_t viewFormatCount;
const VkFormat* viewFormats;
} XrVulkanSwapchainFormatListCreateInfoKHR;
#endif /* XR_USE_GRAPHICS_API_VULKAN */
#ifdef XR_USE_GRAPHICS_API_OPENGL
// XR_KHR_opengl_enable is a preprocessor guard. Do not pass it to API calls.
#define XR_KHR_opengl_enable 1
#define XR_KHR_opengl_enable_SPEC_VERSION 12
#define XR_KHR_OPENGL_ENABLE_EXTENSION_NAME "XR_KHR_opengl_enable"
#ifdef XR_USE_PLATFORM_WIN32
// XrGraphicsBindingOpenGLWin32KHR extends XrSessionCreateInfo
typedef struct XrGraphicsBindingOpenGLWin32KHR {
XrStructureType type;
const void* XR_MAY_ALIAS next;
HDC hDC;
HGLRC hGLRC;
} XrGraphicsBindingOpenGLWin32KHR;
#endif // XR_USE_PLATFORM_WIN32
#ifdef XR_USE_PLATFORM_XLIB
// XrGraphicsBindingOpenGLXlibKHR extends XrSessionCreateInfo
typedef struct XrGraphicsBindingOpenGLXlibKHR {
XrStructureType type;
const void* XR_MAY_ALIAS next;
Display* xDisplay;
uint32_t visualid;
GLXFBConfig glxFBConfig;
GLXDrawable glxDrawable;
GLXContext glxContext;
} XrGraphicsBindingOpenGLXlibKHR;
#endif // XR_USE_PLATFORM_XLIB
#ifdef XR_USE_PLATFORM_XCB
// XrGraphicsBindingOpenGLXcbKHR extends XrSessionCreateInfo
typedef struct XrGraphicsBindingOpenGLXcbKHR {
XrStructureType type;
const void* XR_MAY_ALIAS next;
xcb_connection_t* connection;
uint32_t screenNumber;
xcb_glx_fbconfig_t fbconfigid;
xcb_visualid_t visualid;
xcb_glx_drawable_t glxDrawable;
xcb_glx_context_t glxContext;
} XrGraphicsBindingOpenGLXcbKHR;
#endif // XR_USE_PLATFORM_XCB
#ifdef XR_USE_PLATFORM_WAYLAND
// XrGraphicsBindingOpenGLWaylandKHR extends XrSessionCreateInfo
typedef struct XrGraphicsBindingOpenGLWaylandKHR {
XrStructureType type;
const void* XR_MAY_ALIAS next;
struct wl_display* display;
} XrGraphicsBindingOpenGLWaylandKHR;
#endif // XR_USE_PLATFORM_WAYLAND
typedef struct XrSwapchainImageOpenGLKHR {
XrStructureType type;
void* XR_MAY_ALIAS next;
uint32_t image;
} XrSwapchainImageOpenGLKHR;
typedef struct XrGraphicsRequirementsOpenGLKHR {
XrStructureType type;
void* XR_MAY_ALIAS next;
XrVersion minApiVersionSupported;
XrVersion maxApiVersionSupported;
} XrGraphicsRequirementsOpenGLKHR;
typedef XrResult (XRAPI_PTR *PFN_xrGetOpenGLGraphicsRequirementsKHR)(XrInstance instance, XrSystemId systemId, XrGraphicsRequirementsOpenGLKHR* graphicsRequirements);
#ifndef XR_NO_PROTOTYPES
#ifdef XR_EXTENSION_PROTOTYPES
XRAPI_ATTR XrResult XRAPI_CALL xrGetOpenGLGraphicsRequirementsKHR(
XrInstance instance,
XrSystemId systemId,
XrGraphicsRequirementsOpenGLKHR* graphicsRequirements);
#endif /* XR_EXTENSION_PROTOTYPES */
#endif /* !XR_NO_PROTOTYPES */
#endif /* XR_USE_GRAPHICS_API_OPENGL */
#ifdef XR_USE_GRAPHICS_API_OPENGL_ES
// XR_KHR_opengl_es_enable is a preprocessor guard. Do not pass it to API calls.
#define XR_KHR_opengl_es_enable 1
#define XR_KHR_opengl_es_enable_SPEC_VERSION 10
#define XR_KHR_OPENGL_ES_ENABLE_EXTENSION_NAME "XR_KHR_opengl_es_enable"
#ifdef XR_USE_PLATFORM_ANDROID
// XrGraphicsBindingOpenGLESAndroidKHR extends XrSessionCreateInfo
typedef struct XrGraphicsBindingOpenGLESAndroidKHR {
XrStructureType type;
const void* XR_MAY_ALIAS next;
EGLDisplay display;
EGLConfig config;
EGLContext context;
} XrGraphicsBindingOpenGLESAndroidKHR;
#endif // XR_USE_PLATFORM_ANDROID
typedef struct XrSwapchainImageOpenGLESKHR {
XrStructureType type;
void* XR_MAY_ALIAS next;
uint32_t image;
} XrSwapchainImageOpenGLESKHR;
typedef struct XrGraphicsRequirementsOpenGLESKHR {
XrStructureType type;
void* XR_MAY_ALIAS next;
XrVersion minApiVersionSupported;
XrVersion maxApiVersionSupported;
} XrGraphicsRequirementsOpenGLESKHR;
typedef XrResult (XRAPI_PTR *PFN_xrGetOpenGLESGraphicsRequirementsKHR)(XrInstance instance, XrSystemId systemId, XrGraphicsRequirementsOpenGLESKHR* graphicsRequirements);
#ifndef XR_NO_PROTOTYPES
#ifdef XR_EXTENSION_PROTOTYPES
XRAPI_ATTR XrResult XRAPI_CALL xrGetOpenGLESGraphicsRequirementsKHR(
XrInstance instance,
XrSystemId systemId,
XrGraphicsRequirementsOpenGLESKHR* graphicsRequirements);
#endif /* XR_EXTENSION_PROTOTYPES */
#endif /* !XR_NO_PROTOTYPES */
#endif /* XR_USE_GRAPHICS_API_OPENGL_ES */
#ifdef XR_USE_GRAPHICS_API_VULKAN
// XR_KHR_vulkan_enable is a preprocessor guard. Do not pass it to API calls.
#define XR_KHR_vulkan_enable 1
#define XR_KHR_vulkan_enable_SPEC_VERSION 10
#define XR_KHR_VULKAN_ENABLE_EXTENSION_NAME "XR_KHR_vulkan_enable"
// XrGraphicsBindingVulkanKHR extends XrSessionCreateInfo
typedef struct XrGraphicsBindingVulkanKHR {
XrStructureType type;
const void* XR_MAY_ALIAS next;
VkInstance instance;
VkPhysicalDevice physicalDevice;
VkDevice device;
uint32_t queueFamilyIndex;
uint32_t queueIndex;
} XrGraphicsBindingVulkanKHR;
typedef struct XrSwapchainImageVulkanKHR {
XrStructureType type;
void* XR_MAY_ALIAS next;
VkImage image;
} XrSwapchainImageVulkanKHR;
typedef struct XrGraphicsRequirementsVulkanKHR {
XrStructureType type;
void* XR_MAY_ALIAS next;
XrVersion minApiVersionSupported;
XrVersion maxApiVersionSupported;
} XrGraphicsRequirementsVulkanKHR;
typedef XrResult (XRAPI_PTR *PFN_xrGetVulkanInstanceExtensionsKHR)(XrInstance instance, XrSystemId systemId, uint32_t bufferCapacityInput, uint32_t* bufferCountOutput, char* buffer);
typedef XrResult (XRAPI_PTR *PFN_xrGetVulkanDeviceExtensionsKHR)(XrInstance instance, XrSystemId systemId, uint32_t bufferCapacityInput, uint32_t* bufferCountOutput, char* buffer);
typedef XrResult (XRAPI_PTR *PFN_xrGetVulkanGraphicsDeviceKHR)(XrInstance instance, XrSystemId systemId, VkInstance vkInstance, VkPhysicalDevice* vkPhysicalDevice);
typedef XrResult (XRAPI_PTR *PFN_xrGetVulkanGraphicsRequirementsKHR)(XrInstance instance, XrSystemId systemId, XrGraphicsRequirementsVulkanKHR* graphicsRequirements);
#ifndef XR_NO_PROTOTYPES
#ifdef XR_EXTENSION_PROTOTYPES
XRAPI_ATTR XrResult XRAPI_CALL xrGetVulkanInstanceExtensionsKHR(
XrInstance instance,
XrSystemId systemId,
uint32_t bufferCapacityInput,
uint32_t* bufferCountOutput,
char* buffer);
XRAPI_ATTR XrResult XRAPI_CALL xrGetVulkanDeviceExtensionsKHR(
XrInstance instance,
XrSystemId systemId,
uint32_t bufferCapacityInput,
uint32_t* bufferCountOutput,
char* buffer);
XRAPI_ATTR XrResult XRAPI_CALL xrGetVulkanGraphicsDeviceKHR(
XrInstance instance,
XrSystemId systemId,
VkInstance vkInstance,
VkPhysicalDevice* vkPhysicalDevice);
XRAPI_ATTR XrResult XRAPI_CALL xrGetVulkanGraphicsRequirementsKHR(
XrInstance instance,
XrSystemId systemId,
XrGraphicsRequirementsVulkanKHR* graphicsRequirements);
#endif /* XR_EXTENSION_PROTOTYPES */
#endif /* !XR_NO_PROTOTYPES */
#endif /* XR_USE_GRAPHICS_API_VULKAN */
#ifdef XR_USE_GRAPHICS_API_D3D11
// XR_KHR_D3D11_enable is a preprocessor guard. Do not pass it to API calls.
#define XR_KHR_D3D11_enable 1
#define XR_KHR_D3D11_enable_SPEC_VERSION 11
#define XR_KHR_D3D11_ENABLE_EXTENSION_NAME "XR_KHR_D3D11_enable"
// XrGraphicsBindingD3D11KHR extends XrSessionCreateInfo
typedef struct XrGraphicsBindingD3D11KHR {
XrStructureType type;
const void* XR_MAY_ALIAS next;
ID3D11Device* device;
} XrGraphicsBindingD3D11KHR;
typedef struct XrSwapchainImageD3D11KHR {
XrStructureType type;
void* XR_MAY_ALIAS next;
ID3D11Texture2D* texture;
} XrSwapchainImageD3D11KHR;
typedef struct XrGraphicsRequirementsD3D11KHR {
XrStructureType type;
void* XR_MAY_ALIAS next;
LUID adapterLuid;
D3D_FEATURE_LEVEL minFeatureLevel;
} XrGraphicsRequirementsD3D11KHR;
typedef XrResult (XRAPI_PTR *PFN_xrGetD3D11GraphicsRequirementsKHR)(XrInstance instance, XrSystemId systemId, XrGraphicsRequirementsD3D11KHR* graphicsRequirements);
#ifndef XR_NO_PROTOTYPES
#ifdef XR_EXTENSION_PROTOTYPES
XRAPI_ATTR XrResult XRAPI_CALL xrGetD3D11GraphicsRequirementsKHR(
XrInstance instance,
XrSystemId systemId,
XrGraphicsRequirementsD3D11KHR* graphicsRequirements);
#endif /* XR_EXTENSION_PROTOTYPES */
#endif /* !XR_NO_PROTOTYPES */
#endif /* XR_USE_GRAPHICS_API_D3D11 */
#ifdef XR_USE_GRAPHICS_API_D3D12
// XR_KHR_D3D12_enable is a preprocessor guard. Do not pass it to API calls.
#define XR_KHR_D3D12_enable 1
#define XR_KHR_D3D12_enable_SPEC_VERSION 11
#define XR_KHR_D3D12_ENABLE_EXTENSION_NAME "XR_KHR_D3D12_enable"
// XrGraphicsBindingD3D12KHR extends XrSessionCreateInfo
typedef struct XrGraphicsBindingD3D12KHR {
XrStructureType type;
const void* XR_MAY_ALIAS next;
ID3D12Device* device;
ID3D12CommandQueue* queue;
} XrGraphicsBindingD3D12KHR;
typedef struct XrSwapchainImageD3D12KHR {
XrStructureType type;
void* XR_MAY_ALIAS next;
ID3D12Resource* texture;
} XrSwapchainImageD3D12KHR;
typedef struct XrGraphicsRequirementsD3D12KHR {
XrStructureType type;
void* XR_MAY_ALIAS next;
LUID adapterLuid;
D3D_FEATURE_LEVEL minFeatureLevel;
} XrGraphicsRequirementsD3D12KHR;
typedef XrResult (XRAPI_PTR *PFN_xrGetD3D12GraphicsRequirementsKHR)(XrInstance instance, XrSystemId systemId, XrGraphicsRequirementsD3D12KHR* graphicsRequirements);
#ifndef XR_NO_PROTOTYPES
#ifdef XR_EXTENSION_PROTOTYPES
XRAPI_ATTR XrResult XRAPI_CALL xrGetD3D12GraphicsRequirementsKHR(
XrInstance instance,
XrSystemId systemId,
XrGraphicsRequirementsD3D12KHR* graphicsRequirements);
#endif /* XR_EXTENSION_PROTOTYPES */
#endif /* !XR_NO_PROTOTYPES */
#endif /* XR_USE_GRAPHICS_API_D3D12 */
#ifdef XR_USE_GRAPHICS_API_METAL
// XR_KHR_metal_enable is a preprocessor guard. Do not pass it to API calls.
#define XR_KHR_metal_enable 1
#define XR_KHR_metal_enable_SPEC_VERSION 3
#define XR_KHR_METAL_ENABLE_EXTENSION_NAME "XR_KHR_metal_enable"
// XrGraphicsBindingMetalKHR extends XrSessionCreateInfo
typedef struct XrGraphicsBindingMetalKHR {
XrStructureType type;
const void* XR_MAY_ALIAS next;
void* XR_MAY_ALIAS commandQueue;
} XrGraphicsBindingMetalKHR;
typedef struct XrSwapchainImageMetalKHR {
XrStructureType type;
void* XR_MAY_ALIAS next;
void* XR_MAY_ALIAS texture;
} XrSwapchainImageMetalKHR;
typedef struct XrGraphicsRequirementsMetalKHR {
XrStructureType type;
void* XR_MAY_ALIAS next;
void* XR_MAY_ALIAS metalDevice;
} XrGraphicsRequirementsMetalKHR;
typedef XrResult (XRAPI_PTR *PFN_xrGetMetalGraphicsRequirementsKHR)(XrInstance instance, XrSystemId systemId, XrGraphicsRequirementsMetalKHR* graphicsRequirements);
#ifndef XR_NO_PROTOTYPES
#ifdef XR_EXTENSION_PROTOTYPES
XRAPI_ATTR XrResult XRAPI_CALL xrGetMetalGraphicsRequirementsKHR(
XrInstance instance,
XrSystemId systemId,
XrGraphicsRequirementsMetalKHR* graphicsRequirements);
#endif /* XR_EXTENSION_PROTOTYPES */
#endif /* !XR_NO_PROTOTYPES */
#endif /* XR_USE_GRAPHICS_API_METAL */
#ifdef XR_USE_PLATFORM_WIN32
// XR_KHR_win32_convert_performance_counter_time is a preprocessor guard. Do not pass it to API calls.
#define XR_KHR_win32_convert_performance_counter_time 1
#define XR_KHR_win32_convert_performance_counter_time_SPEC_VERSION 1
#define XR_KHR_WIN32_CONVERT_PERFORMANCE_COUNTER_TIME_EXTENSION_NAME "XR_KHR_win32_convert_performance_counter_time"
typedef XrResult (XRAPI_PTR *PFN_xrConvertWin32PerformanceCounterToTimeKHR)(XrInstance instance, const LARGE_INTEGER* performanceCounter, XrTime* time);
typedef XrResult (XRAPI_PTR *PFN_xrConvertTimeToWin32PerformanceCounterKHR)(XrInstance instance, XrTime time, LARGE_INTEGER* performanceCounter);
#ifndef XR_NO_PROTOTYPES
#ifdef XR_EXTENSION_PROTOTYPES
XRAPI_ATTR XrResult XRAPI_CALL xrConvertWin32PerformanceCounterToTimeKHR(
XrInstance instance,
const LARGE_INTEGER* performanceCounter,
XrTime* time);
XRAPI_ATTR XrResult XRAPI_CALL xrConvertTimeToWin32PerformanceCounterKHR(
XrInstance instance,
XrTime time,
LARGE_INTEGER* performanceCounter);
#endif /* XR_EXTENSION_PROTOTYPES */
#endif /* !XR_NO_PROTOTYPES */
#endif /* XR_USE_PLATFORM_WIN32 */
#ifdef XR_USE_TIMESPEC
// XR_KHR_convert_timespec_time is a preprocessor guard. Do not pass it to API calls.
#define XR_KHR_convert_timespec_time 1
#define XR_KHR_convert_timespec_time_SPEC_VERSION 1
#define XR_KHR_CONVERT_TIMESPEC_TIME_EXTENSION_NAME "XR_KHR_convert_timespec_time"
typedef XrResult (XRAPI_PTR *PFN_xrConvertTimespecTimeToTimeKHR)(XrInstance instance, const struct timespec* timespecTime, XrTime* time);
typedef XrResult (XRAPI_PTR *PFN_xrConvertTimeToTimespecTimeKHR)(XrInstance instance, XrTime time, struct timespec* timespecTime);
#ifndef XR_NO_PROTOTYPES
#ifdef XR_EXTENSION_PROTOTYPES
XRAPI_ATTR XrResult XRAPI_CALL xrConvertTimespecTimeToTimeKHR(
XrInstance instance,
const struct timespec* timespecTime,
XrTime* time);
XRAPI_ATTR XrResult XRAPI_CALL xrConvertTimeToTimespecTimeKHR(
XrInstance instance,
XrTime time,
struct timespec* timespecTime);
#endif /* XR_EXTENSION_PROTOTYPES */
#endif /* !XR_NO_PROTOTYPES */
#endif /* XR_USE_TIMESPEC */
#ifdef XR_USE_PLATFORM_ANDROID
// XR_KHR_loader_init_android is a preprocessor guard. Do not pass it to API calls.
#define XR_KHR_loader_init_android 1
#define XR_KHR_loader_init_android_SPEC_VERSION 1
#define XR_KHR_LOADER_INIT_ANDROID_EXTENSION_NAME "XR_KHR_loader_init_android"
typedef struct XrLoaderInitInfoAndroidKHR {
XrStructureType type;
const void* XR_MAY_ALIAS next;
void* XR_MAY_ALIAS applicationVM;
void* XR_MAY_ALIAS applicationContext;
} XrLoaderInitInfoAndroidKHR;
#endif /* XR_USE_PLATFORM_ANDROID */
#ifdef XR_USE_GRAPHICS_API_VULKAN
// XR_KHR_vulkan_enable2 is a preprocessor guard. Do not pass it to API calls.
#define XR_KHR_vulkan_enable2 1
#define XR_KHR_vulkan_enable2_SPEC_VERSION 4
#define XR_KHR_VULKAN_ENABLE2_EXTENSION_NAME "XR_KHR_vulkan_enable2"
typedef XrFlags64 XrVulkanInstanceCreateFlagsKHR;
// Flag bits for XrVulkanInstanceCreateFlagsKHR
typedef XrFlags64 XrVulkanDeviceCreateFlagsKHR;
// Flag bits for XrVulkanDeviceCreateFlagsKHR
typedef struct XrVulkanInstanceCreateInfoKHR {
XrStructureType type;
const void* XR_MAY_ALIAS next;
XrSystemId systemId;
XrVulkanInstanceCreateFlagsKHR createFlags;
PFN_vkGetInstanceProcAddr pfnGetInstanceProcAddr;
const VkInstanceCreateInfo* vulkanCreateInfo;
const VkAllocationCallbacks* vulkanAllocator;
} XrVulkanInstanceCreateInfoKHR;
typedef struct XrVulkanDeviceCreateInfoKHR {
XrStructureType type;
const void* XR_MAY_ALIAS next;
XrSystemId systemId;
XrVulkanDeviceCreateFlagsKHR createFlags;
PFN_vkGetInstanceProcAddr pfnGetInstanceProcAddr;
VkPhysicalDevice vulkanPhysicalDevice;
const VkDeviceCreateInfo* vulkanCreateInfo;
const VkAllocationCallbacks* vulkanAllocator;
} XrVulkanDeviceCreateInfoKHR;
typedef XrGraphicsBindingVulkanKHR XrGraphicsBindingVulkan2KHR;
typedef struct XrVulkanGraphicsDeviceGetInfoKHR {
XrStructureType type;
const void* XR_MAY_ALIAS next;
XrSystemId systemId;
VkInstance vulkanInstance;
} XrVulkanGraphicsDeviceGetInfoKHR;
typedef XrSwapchainImageVulkanKHR XrSwapchainImageVulkan2KHR;
typedef XrGraphicsRequirementsVulkanKHR XrGraphicsRequirementsVulkan2KHR;
typedef XrResult (XRAPI_PTR *PFN_xrCreateVulkanInstanceKHR)(XrInstance instance, const XrVulkanInstanceCreateInfoKHR* createInfo, VkInstance* vulkanInstance, VkResult* vulkanResult);
typedef XrResult (XRAPI_PTR *PFN_xrCreateVulkanDeviceKHR)(XrInstance instance, const XrVulkanDeviceCreateInfoKHR* createInfo, VkDevice* vulkanDevice, VkResult* vulkanResult);
typedef XrResult (XRAPI_PTR *PFN_xrGetVulkanGraphicsDevice2KHR)(XrInstance instance, const XrVulkanGraphicsDeviceGetInfoKHR* getInfo, VkPhysicalDevice* vulkanPhysicalDevice);
typedef XrResult (XRAPI_PTR *PFN_xrGetVulkanGraphicsRequirements2KHR)(XrInstance instance, XrSystemId systemId, XrGraphicsRequirementsVulkanKHR* graphicsRequirements);
#ifndef XR_NO_PROTOTYPES
#ifdef XR_EXTENSION_PROTOTYPES
XRAPI_ATTR XrResult XRAPI_CALL xrCreateVulkanInstanceKHR(
XrInstance instance,
const XrVulkanInstanceCreateInfoKHR* createInfo,
VkInstance* vulkanInstance,
VkResult* vulkanResult);
XRAPI_ATTR XrResult XRAPI_CALL xrCreateVulkanDeviceKHR(
XrInstance instance,
const XrVulkanDeviceCreateInfoKHR* createInfo,
VkDevice* vulkanDevice,
VkResult* vulkanResult);
XRAPI_ATTR XrResult XRAPI_CALL xrGetVulkanGraphicsDevice2KHR(
XrInstance instance,
const XrVulkanGraphicsDeviceGetInfoKHR* getInfo,
VkPhysicalDevice* vulkanPhysicalDevice);
XRAPI_ATTR XrResult XRAPI_CALL xrGetVulkanGraphicsRequirements2KHR(
XrInstance instance,
XrSystemId systemId,
XrGraphicsRequirementsVulkanKHR* graphicsRequirements);
#endif /* XR_EXTENSION_PROTOTYPES */
#endif /* !XR_NO_PROTOTYPES */
#endif /* XR_USE_GRAPHICS_API_VULKAN */
#ifdef XR_USE_PLATFORM_EGL
// XR_MNDX_egl_enable is a preprocessor guard. Do not pass it to API calls.
#define XR_MNDX_egl_enable 1
#define XR_MNDX_egl_enable_SPEC_VERSION 2
#define XR_MNDX_EGL_ENABLE_EXTENSION_NAME "XR_MNDX_egl_enable"
typedef PFN_xrVoidFunction (*PFN_xrEglGetProcAddressMNDX)(const char *name);
// XrGraphicsBindingEGLMNDX extends XrSessionCreateInfo
typedef struct XrGraphicsBindingEGLMNDX {
XrStructureType type;
const void* XR_MAY_ALIAS next;
PFN_xrEglGetProcAddressMNDX getProcAddress;
EGLDisplay display;
EGLConfig config;
EGLContext context;
} XrGraphicsBindingEGLMNDX;
#endif /* XR_USE_PLATFORM_EGL */
#ifdef XR_USE_PLATFORM_WIN32
// XR_MSFT_perception_anchor_interop is a preprocessor guard. Do not pass it to API calls.
#define XR_MSFT_perception_anchor_interop 1
#define XR_MSFT_perception_anchor_interop_SPEC_VERSION 1
#define XR_MSFT_PERCEPTION_ANCHOR_INTEROP_EXTENSION_NAME "XR_MSFT_perception_anchor_interop"
typedef XrResult (XRAPI_PTR *PFN_xrCreateSpatialAnchorFromPerceptionAnchorMSFT)(XrSession session, IUnknown* perceptionAnchor, XrSpatialAnchorMSFT* anchor);
typedef XrResult (XRAPI_PTR *PFN_xrTryGetPerceptionAnchorFromSpatialAnchorMSFT)(XrSession session, XrSpatialAnchorMSFT anchor, IUnknown** perceptionAnchor);
#ifndef XR_NO_PROTOTYPES
#ifdef XR_EXTENSION_PROTOTYPES
XRAPI_ATTR XrResult XRAPI_CALL xrCreateSpatialAnchorFromPerceptionAnchorMSFT(
XrSession session,
IUnknown* perceptionAnchor,
XrSpatialAnchorMSFT* anchor);
XRAPI_ATTR XrResult XRAPI_CALL xrTryGetPerceptionAnchorFromSpatialAnchorMSFT(
XrSession session,
XrSpatialAnchorMSFT anchor,
IUnknown** perceptionAnchor);
#endif /* XR_EXTENSION_PROTOTYPES */
#endif /* !XR_NO_PROTOTYPES */
#endif /* XR_USE_PLATFORM_WIN32 */
#ifdef XR_USE_PLATFORM_WIN32
// XR_MSFT_holographic_window_attachment is a preprocessor guard. Do not pass it to API calls.
#define XR_MSFT_holographic_window_attachment 1
#define XR_MSFT_holographic_window_attachment_SPEC_VERSION 1
#define XR_MSFT_HOLOGRAPHIC_WINDOW_ATTACHMENT_EXTENSION_NAME "XR_MSFT_holographic_window_attachment"
#ifdef XR_USE_PLATFORM_WIN32
// XrHolographicWindowAttachmentMSFT extends XrSessionCreateInfo
typedef struct XrHolographicWindowAttachmentMSFT {
XrStructureType type;
const void* XR_MAY_ALIAS next;
IUnknown* holographicSpace;
IUnknown* coreWindow;
} XrHolographicWindowAttachmentMSFT;
#endif // XR_USE_PLATFORM_WIN32
#endif /* XR_USE_PLATFORM_WIN32 */
#ifdef XR_USE_PLATFORM_ANDROID
// XR_FB_android_surface_swapchain_create is a preprocessor guard. Do not pass it to API calls.
#define XR_FB_android_surface_swapchain_create 1
#define XR_FB_android_surface_swapchain_create_SPEC_VERSION 1
#define XR_FB_ANDROID_SURFACE_SWAPCHAIN_CREATE_EXTENSION_NAME "XR_FB_android_surface_swapchain_create"
typedef XrFlags64 XrAndroidSurfaceSwapchainFlagsFB;
// Flag bits for XrAndroidSurfaceSwapchainFlagsFB
static const XrAndroidSurfaceSwapchainFlagsFB XR_ANDROID_SURFACE_SWAPCHAIN_SYNCHRONOUS_BIT_FB = 0x00000001;
static const XrAndroidSurfaceSwapchainFlagsFB XR_ANDROID_SURFACE_SWAPCHAIN_USE_TIMESTAMPS_BIT_FB = 0x00000002;
#ifdef XR_USE_PLATFORM_ANDROID
// XrAndroidSurfaceSwapchainCreateInfoFB extends XrSwapchainCreateInfo
typedef struct XrAndroidSurfaceSwapchainCreateInfoFB {
XrStructureType type;
const void* XR_MAY_ALIAS next;
XrAndroidSurfaceSwapchainFlagsFB createFlags;
} XrAndroidSurfaceSwapchainCreateInfoFB;
#endif // XR_USE_PLATFORM_ANDROID
#endif /* XR_USE_PLATFORM_ANDROID */
#ifdef XR_USE_PLATFORM_ML
// XR_ML_compat is a preprocessor guard. Do not pass it to API calls.
#define XR_ML_compat 1
#define XR_ML_compat_SPEC_VERSION 1
#define XR_ML_COMPAT_EXTENSION_NAME "XR_ML_compat"
typedef struct XrCoordinateSpaceCreateInfoML {
XrStructureType type;
const void* XR_MAY_ALIAS next;
MLCoordinateFrameUID cfuid;
XrPosef poseInCoordinateSpace;
} XrCoordinateSpaceCreateInfoML;
typedef XrResult (XRAPI_PTR *PFN_xrCreateSpaceFromCoordinateFrameUIDML)(XrSession session, const XrCoordinateSpaceCreateInfoML *createInfo, XrSpace* space);
#ifndef XR_NO_PROTOTYPES
#ifdef XR_EXTENSION_PROTOTYPES
XRAPI_ATTR XrResult XRAPI_CALL xrCreateSpaceFromCoordinateFrameUIDML(
XrSession session,
const XrCoordinateSpaceCreateInfoML * createInfo,
XrSpace* space);
#endif /* XR_EXTENSION_PROTOTYPES */
#endif /* !XR_NO_PROTOTYPES */
#endif /* XR_USE_PLATFORM_ML */
#ifdef XR_USE_PLATFORM_WIN32
// XR_OCULUS_audio_device_guid is a preprocessor guard. Do not pass it to API calls.
#define XR_OCULUS_audio_device_guid 1
#define XR_OCULUS_audio_device_guid_SPEC_VERSION 1
#define XR_OCULUS_AUDIO_DEVICE_GUID_EXTENSION_NAME "XR_OCULUS_audio_device_guid"
#define XR_MAX_AUDIO_DEVICE_STR_SIZE_OCULUS 128
typedef XrResult (XRAPI_PTR *PFN_xrGetAudioOutputDeviceGuidOculus)(XrInstance instance, wchar_t buffer[XR_MAX_AUDIO_DEVICE_STR_SIZE_OCULUS]);
typedef XrResult (XRAPI_PTR *PFN_xrGetAudioInputDeviceGuidOculus)(XrInstance instance, wchar_t buffer[XR_MAX_AUDIO_DEVICE_STR_SIZE_OCULUS]);
#ifndef XR_NO_PROTOTYPES
#ifdef XR_EXTENSION_PROTOTYPES
XRAPI_ATTR XrResult XRAPI_CALL xrGetAudioOutputDeviceGuidOculus(
XrInstance instance,
wchar_t buffer[XR_MAX_AUDIO_DEVICE_STR_SIZE_OCULUS]);
XRAPI_ATTR XrResult XRAPI_CALL xrGetAudioInputDeviceGuidOculus(
XrInstance instance,
wchar_t buffer[XR_MAX_AUDIO_DEVICE_STR_SIZE_OCULUS]);
#endif /* XR_EXTENSION_PROTOTYPES */
#endif /* !XR_NO_PROTOTYPES */
#endif /* XR_USE_PLATFORM_WIN32 */
#ifdef XR_USE_GRAPHICS_API_VULKAN
// XR_FB_foveation_vulkan is a preprocessor guard. Do not pass it to API calls.
#define XR_FB_foveation_vulkan 1
#define XR_FB_foveation_vulkan_SPEC_VERSION 1
#define XR_FB_FOVEATION_VULKAN_EXTENSION_NAME "XR_FB_foveation_vulkan"
// XrSwapchainImageFoveationVulkanFB extends XrSwapchainImageVulkanKHR
typedef struct XrSwapchainImageFoveationVulkanFB {
XrStructureType type;
void* XR_MAY_ALIAS next;
VkImage image;
uint32_t width;
uint32_t height;
} XrSwapchainImageFoveationVulkanFB;
#endif /* XR_USE_GRAPHICS_API_VULKAN */
#ifdef XR_USE_PLATFORM_ANDROID
// XR_FB_swapchain_update_state_android_surface is a preprocessor guard. Do not pass it to API calls.
#define XR_FB_swapchain_update_state_android_surface 1
#define XR_FB_swapchain_update_state_android_surface_SPEC_VERSION 1
#define XR_FB_SWAPCHAIN_UPDATE_STATE_ANDROID_SURFACE_EXTENSION_NAME "XR_FB_swapchain_update_state_android_surface"
#ifdef XR_USE_PLATFORM_ANDROID
typedef struct XrSwapchainStateAndroidSurfaceDimensionsFB {
XrStructureType type;
void* XR_MAY_ALIAS next;
uint32_t width;
uint32_t height;
} XrSwapchainStateAndroidSurfaceDimensionsFB;
#endif // XR_USE_PLATFORM_ANDROID
#endif /* XR_USE_PLATFORM_ANDROID */
#ifdef XR_USE_GRAPHICS_API_OPENGL_ES
// XR_FB_swapchain_update_state_opengl_es is a preprocessor guard. Do not pass it to API calls.
#define XR_FB_swapchain_update_state_opengl_es 1
#define XR_FB_swapchain_update_state_opengl_es_SPEC_VERSION 1
#define XR_FB_SWAPCHAIN_UPDATE_STATE_OPENGL_ES_EXTENSION_NAME "XR_FB_swapchain_update_state_opengl_es"
#ifdef XR_USE_GRAPHICS_API_OPENGL_ES
typedef struct XrSwapchainStateSamplerOpenGLESFB {
XrStructureType type;
void* XR_MAY_ALIAS next;
EGLenum minFilter;
EGLenum magFilter;
EGLenum wrapModeS;
EGLenum wrapModeT;
EGLenum swizzleRed;
EGLenum swizzleGreen;
EGLenum swizzleBlue;
EGLenum swizzleAlpha;
float maxAnisotropy;
XrColor4f borderColor;
} XrSwapchainStateSamplerOpenGLESFB;
#endif // XR_USE_GRAPHICS_API_OPENGL_ES
#endif /* XR_USE_GRAPHICS_API_OPENGL_ES */
#ifdef XR_USE_GRAPHICS_API_VULKAN
// XR_FB_swapchain_update_state_vulkan is a preprocessor guard. Do not pass it to API calls.
#define XR_FB_swapchain_update_state_vulkan 1
#define XR_FB_swapchain_update_state_vulkan_SPEC_VERSION 1
#define XR_FB_SWAPCHAIN_UPDATE_STATE_VULKAN_EXTENSION_NAME "XR_FB_swapchain_update_state_vulkan"
#ifdef XR_USE_GRAPHICS_API_VULKAN
typedef struct XrSwapchainStateSamplerVulkanFB {
XrStructureType type;
void* XR_MAY_ALIAS next;
VkFilter minFilter;
VkFilter magFilter;
VkSamplerMipmapMode mipmapMode;
VkSamplerAddressMode wrapModeS;
VkSamplerAddressMode wrapModeT;
VkComponentSwizzle swizzleRed;
VkComponentSwizzle swizzleGreen;
VkComponentSwizzle swizzleBlue;
VkComponentSwizzle swizzleAlpha;
float maxAnisotropy;
XrColor4f borderColor;
} XrSwapchainStateSamplerVulkanFB;
#endif // XR_USE_GRAPHICS_API_VULKAN
#endif /* XR_USE_GRAPHICS_API_VULKAN */
#ifdef XR_USE_GRAPHICS_API_VULKAN
// XR_META_vulkan_swapchain_create_info is a preprocessor guard. Do not pass it to API calls.
#define XR_META_vulkan_swapchain_create_info 1
#define XR_META_vulkan_swapchain_create_info_SPEC_VERSION 1
#define XR_META_VULKAN_SWAPCHAIN_CREATE_INFO_EXTENSION_NAME "XR_META_vulkan_swapchain_create_info"
// XrVulkanSwapchainCreateInfoMETA extends XrSwapchainCreateInfo
typedef struct XrVulkanSwapchainCreateInfoMETA {
XrStructureType type;
const void* XR_MAY_ALIAS next;
VkImageCreateFlags additionalCreateFlags;
VkImageUsageFlags additionalUsageFlags;
} XrVulkanSwapchainCreateInfoMETA;
#endif /* XR_USE_GRAPHICS_API_VULKAN */
#ifdef XR_USE_PLATFORM_ANDROID
// XR_ANDROID_anchor_sharing_export is a preprocessor guard. Do not pass it to API calls.
#define XR_ANDROID_anchor_sharing_export 1
#define XR_ANDROID_anchor_sharing_export_SPEC_VERSION 1
#define XR_ANDROID_ANCHOR_SHARING_EXPORT_EXTENSION_NAME "XR_ANDROID_anchor_sharing_export"
typedef struct XrAnchorSharingInfoANDROID {
XrStructureType type;
const void* XR_MAY_ALIAS next;
XrSpace anchor;
} XrAnchorSharingInfoANDROID;
typedef struct XrAnchorSharingTokenANDROID {
XrStructureType type;
void* XR_MAY_ALIAS next;
struct AIBinder* token;
} XrAnchorSharingTokenANDROID;
// XrSystemAnchorSharingExportPropertiesANDROID extends XrSystemProperties
typedef struct XrSystemAnchorSharingExportPropertiesANDROID {
XrStructureType type;
void* XR_MAY_ALIAS next;
XrBool32 supportsAnchorSharingExport;
} XrSystemAnchorSharingExportPropertiesANDROID;
typedef XrResult (XRAPI_PTR *PFN_xrShareAnchorANDROID)(XrSession session, const XrAnchorSharingInfoANDROID* sharingInfo, XrAnchorSharingTokenANDROID* anchorToken);
typedef XrResult (XRAPI_PTR *PFN_xrUnshareAnchorANDROID)(XrSession session, XrSpace anchor);
#ifndef XR_NO_PROTOTYPES
#ifdef XR_EXTENSION_PROTOTYPES
XRAPI_ATTR XrResult XRAPI_CALL xrShareAnchorANDROID(
XrSession session,
const XrAnchorSharingInfoANDROID* sharingInfo,
XrAnchorSharingTokenANDROID* anchorToken);
XRAPI_ATTR XrResult XRAPI_CALL xrUnshareAnchorANDROID(
XrSession session,
XrSpace anchor);
#endif /* XR_EXTENSION_PROTOTYPES */
#endif /* !XR_NO_PROTOTYPES */
#endif /* XR_USE_PLATFORM_ANDROID */
#ifdef __cplusplus
}
#endif
#endif
frame/openxr-compat/vendor/openxr/openxr_platform_defines.h
Vendored
+114
View File
@@ -0,0 +1,114 @@
/*
** Copyright (c) 2017-2026 The Khronos Group Inc.
**
** SPDX-License-Identifier: Apache-2.0 OR MIT
*/
#ifndef OPENXR_PLATFORM_DEFINES_H_
#define OPENXR_PLATFORM_DEFINES_H_ 1
#ifdef __cplusplus
extern "C" {
#endif
/* Platform-specific calling convention macros.
*
* Platforms should define these so that OpenXR clients call OpenXR functions
* with the same calling conventions that the OpenXR implementation expects.
*
* XRAPI_ATTR - Placed before the return type in function declarations.
* Useful for C++11 and GCC/Clang-style function attribute syntax.
* XRAPI_CALL - Placed after the return type in function declarations.
* Useful for MSVC-style calling convention syntax.
* XRAPI_PTR - Placed between the '(' and '*' in function pointer types.
*
* Function declaration: XRAPI_ATTR void XRAPI_CALL xrFunction(void);
* Function pointer type: typedef void (XRAPI_PTR *PFN_xrFunction)(void);
*/
#if defined(_WIN32)
#define XRAPI_ATTR
// On Windows, functions use the stdcall convention
#define XRAPI_CALL __stdcall
#define XRAPI_PTR XRAPI_CALL
#elif defined(__ANDROID__) && defined(__ARM_ARCH) && __ARM_ARCH < 7
#error "API not supported for the 'armeabi' NDK ABI"
#elif defined(__ANDROID__) && defined(__ARM_ARCH) && __ARM_ARCH >= 7 && defined(__ARM_32BIT_STATE)
// On Android 32-bit ARM targets, functions use the "hardfloat"
// calling convention, i.e. float parameters are passed in registers. This
// is true even if the rest of the application passes floats on the stack,
// as it does by default when compiling for the armeabi-v7a NDK ABI.
#define XRAPI_ATTR __attribute__((pcs("aapcs-vfp")))
#define XRAPI_CALL
#define XRAPI_PTR XRAPI_ATTR
#else
// On other platforms, use the default calling convention
#define XRAPI_ATTR
#define XRAPI_CALL
#define XRAPI_PTR
#endif
#include <stddef.h>
#if !defined(XR_NO_STDINT_H)
#if defined(_MSC_VER) && (_MSC_VER < 1600)
typedef signed __int8 int8_t;
typedef unsigned __int8 uint8_t;
typedef signed __int16 int16_t;
typedef unsigned __int16 uint16_t;
typedef signed __int32 int32_t;
typedef unsigned __int32 uint32_t;
typedef signed __int64 int64_t;
typedef unsigned __int64 uint64_t;
#else
#include <stdint.h>
#endif
#endif // !defined( XR_NO_STDINT_H )
// XR_PTR_SIZE (in bytes)
#if (defined(__LP64__) || defined(_WIN64) || (defined(__x86_64__) && !defined(__ILP32__) ) || defined(_M_X64) || defined(__ia64) || defined(_M_IA64) || defined(__aarch64__) || defined(__powerpc64__))
#define XR_PTR_SIZE 8
#else
#define XR_PTR_SIZE 4
#endif
// Needed so we can use clang __has_feature portably.
#if !defined(XR_COMPILER_HAS_FEATURE)
#if defined(__clang__)
#define XR_COMPILER_HAS_FEATURE(x) __has_feature(x)
#else
#define XR_COMPILER_HAS_FEATURE(x) 0
#endif
#endif
// Identifies if the current compiler has C++11 support enabled.
// Does not by itself identify if any given C++11 feature is present.
#if !defined(XR_CPP11_ENABLED) && defined(__cplusplus)
#if defined(__GNUC__) && defined(__GXX_EXPERIMENTAL_CXX0X__)
#define XR_CPP11_ENABLED 1
#elif defined(_MSC_VER) && (_MSC_VER >= 1600)
#define XR_CPP11_ENABLED 1
#elif (__cplusplus >= 201103L) // 201103 is the first C++11 version.
#define XR_CPP11_ENABLED 1
#endif
#endif
// Identifies if the current compiler supports C++11 nullptr.
#if !defined(XR_CPP_NULLPTR_SUPPORTED)
#if defined(XR_CPP11_ENABLED) && \
((defined(__clang__) && XR_COMPILER_HAS_FEATURE(cxx_nullptr)) || \
(defined(__GNUC__) && (((__GNUC__ * 1000) + __GNUC_MINOR__) >= 4006)) || \
(defined(_MSC_VER) && (_MSC_VER >= 1600)) || \
(defined(__EDG_VERSION__) && (__EDG_VERSION__ >= 403)))
#define XR_CPP_NULLPTR_SUPPORTED 1
#endif
#endif
#if !defined(XR_CPP_NULLPTR_SUPPORTED)
#define XR_CPP_NULLPTR_SUPPORTED 0
#endif // !defined(XR_CPP_NULLPTR_SUPPORTED)
#ifdef __cplusplus
}
#endif
#endif
mac/frame-mac-view/Sources/CGVirtualDisplay.h
+28
View File
@@ -0,0 +1,28 @@
// Private CoreGraphics classes for virtual displays (as used by DeskPad,
// BetterDisplay and quest-display). Not in any SDK; see
// docs/mac-window-separation.md.
#import <Foundation/Foundation.h>
#import <CoreGraphics/CoreGraphics.h>
@interface CGVirtualDisplayDescriptor : NSObject
@property (retain, nonatomic) dispatch_queue_t queue;
@property (retain, nonatomic) NSString *name;
@property (nonatomic) unsigned int maxPixelsHigh;
@property (nonatomic) unsigned int maxPixelsWide;
@property (nonatomic) CGSize sizeInMillimeters;
@property (nonatomic) unsigned int serialNum;
@property (nonatomic) unsigned int productID;
@property (nonatomic) unsigned int vendorID;
@property (copy, nonatomic) void (^terminationHandler)(id, id);
@end
@interface CGVirtualDisplayMode : NSObject
- (instancetype)initWithWidth:(unsigned int)width height:(unsigned int)height refreshRate:(double)refreshRate;
@end
@interface CGVirtualDisplaySettings : NSObject
@property (retain, nonatomic) NSArray *modes;
@property (nonatomic) unsigned int hiDPI;
@end
@interface CGVirtualDisplay : NSObject
@property (readonly, nonatomic) unsigned int displayID;
- (instancetype)initWithDescriptor:(CGVirtualDisplayDescriptor *)descriptor;
- (BOOL)applySettings:(CGVirtualDisplaySettings *)settings;
@end
mac/frame-mac-view/Sources/Capture.swift
+414
View File
@@ -0,0 +1,414 @@
// Where pictures come from: one Mac window or one display (ScreenCaptureKit),
// or a generated test pattern that needs no permissions and shows the input
// the viewer sends, for checking the whole path without touching the Mac.
import AppKit
import CoreMedia
import CoreVideo
import Foundation
import ScreenCaptureKit
enum Source: Equatable {
case window(CGWindowID)
case display(CGDirectDisplayID)
case separate(CGWindowID) // the window on a display of its own
case test
init?(_ s: String) {
let parts = s.split(separator: ":", maxSplits: 1).map(String.init)
switch (parts.first, parts.count > 1 ? UInt32(parts[1]) : nil) {
case ("window", let id?): self = .window(id)
case ("display", let id?): self = .display(id)
case ("separate", let id?): self = .separate(id)
case ("test", _): self = .test
default: return nil
}
}
var key: String {
switch self {
case .window(let id): return "window:\(id)"
case .display(let id): return "display:\(id)"
case .separate(let id): return "separate:\(id)"
case .test: return "test"
}
}
}
/// Size to capture at: the source's pixel size, shrunk to fit a box whose
/// long side is `maxLong` (default 1920), even numbers for the encoder.
func fitSize(width: Double, height: Double, maxLong: Int) -> (Int, Int) {
let scale = min(1, Double(maxLong) / max(width, height, 1))
let w = max(16, Int((width * scale / 2).rounded()) * 2), h = max(16, Int((height * scale / 2).rounded()) * 2)
return (w, h)
}
/// What CGWindowList says about a window right now (no permission needed for
/// bounds; titles need Screen Recording).
struct WindowInfo {
let id: CGWindowID
let pid: pid_t
let app: String
let title: String
let bounds: CGRect
let layer: Int
let onScreen: Bool
static func all(onScreenOnly: Bool = true) -> [WindowInfo] {
let opts: CGWindowListOption = onScreenOnly ? [.optionOnScreenOnly, .excludeDesktopElements] : [.excludeDesktopElements]
let list = CGWindowListCopyWindowInfo(opts, kCGNullWindowID) as? [[CFString: Any]] ?? []
return list.compactMap(WindowInfo.init)
}
static func find(_ id: CGWindowID) -> WindowInfo? {
let list = CGWindowListCopyWindowInfo(.optionIncludingWindow, id) as? [[CFString: Any]] ?? []
return list.compactMap(WindowInfo.init).first { $0.id == id }
}
init?(_ d: [CFString: Any]) {
guard let id = d[kCGWindowNumber] as? CGWindowID, let pid = d[kCGWindowOwnerPID] as? pid_t,
let b = d[kCGWindowBounds] as? [String: Any], let rect = CGRect(dictionaryRepresentation: b as CFDictionary)
else { return nil }
self.id = id
self.pid = pid
app = d[kCGWindowOwnerName] as? String ?? ""
title = d[kCGWindowName] as? String ?? ""
bounds = rect
layer = d[kCGWindowLayer] as? Int ?? 0
onScreen = d[kCGWindowIsOnscreen] as? Bool ?? false
}
}
protocol CaptureSource: AnyObject {
/// The picture, its presentation time, and when the Mac composited it (µs, host clock).
var onFrame: ((CVPixelBuffer, CMTime, Int64) -> Void)? { get set }
var onEnded: ((String) -> Void)? { get set }
var onChange: (() -> Void)? { get set } // title or size changed
/// True once the window or display is gone for good.
var gone: Bool { get }
/// Points on the Mac's global display space that the picture covers.
var frameRect: CGRect { get }
var title: String { get }
var app: String { get }
var pid: pid_t? { get }
func start(maxLong: Int, fps: Int, completion: @escaping (String?) -> Void)
/// A smaller or larger picture from now on (the long side, in pixels).
func setMaxLong(_ maxLong: Int)
func stop()
func pointer(x: Double, y: Double, text: String?) // for the test pattern
}
extension CaptureSource {
func pointer(x: Double, y: Double, text: String?) {}
}
/// ScreenCaptureKit, for a window or a display.
final class SCKSource: NSObject, CaptureSource, SCStreamOutput, SCStreamDelegate {
let source: Source
var onFrame: ((CVPixelBuffer, CMTime, Int64) -> Void)?
var onEnded: ((String) -> Void)?
private(set) var frameRect = CGRect.zero
private(set) var title = ""
private(set) var app = ""
private(set) var pid: pid_t?
private var stream: SCStream?
private var config = SCStreamConfiguration()
private var maxLong = 1920
private var scale = 2.0
private var poll: DispatchSourceTimer?
private let queue = DispatchQueue(label: "frame-mac-view.capture", qos: .userInteractive)
/// Set by stop(), on `queue`; a start still enumerating windows checks it
/// before it starts capturing, so a viewer that left early leaves nothing on.
private var cancelled = false
/// The window or display no longer exists, so retrying can't help.
private(set) var gone = false
var onChange: (() -> Void)? // title or size changed
/// For a display: capture only this part of it (display points, top-left
/// origin). Set before start().
var crop: CGRect?
init(_ source: Source) { self.source = source }
/// What's captured, in global points: the display, or the cropped part of it.
private func displayRect(_ id: CGDirectDisplayID) -> CGRect {
let b = CGDisplayBounds(id)
guard let c = crop else { return b }
return c.offsetBy(dx: b.minX, dy: b.minY).intersection(b)
}
func start(maxLong: Int, fps: Int, completion: @escaping (String?) -> Void) {
self.maxLong = maxLong
SCShareableContent.getExcludingDesktopWindows(true, onScreenWindowsOnly: false) { [self] content, error in
queue.async { self.begin(content, error, fps: fps, completion: completion) }
}
}
private func begin(_ content: SCShareableContent?, _ error: Error?, fps: Int, completion: @escaping (String?) -> Void) {
if cancelled { return }
guard let content else {
return completion("Screen Recording isn't allowed for Frame Control (\(error?.localizedDescription ?? "no content"))")
}
let filter: SCContentFilter
switch source {
case .window(let id):
guard let w = content.windows.first(where: { $0.windowID == id }) else {
gone = true
return completion("that window has closed")
}
filter = SCContentFilter(desktopIndependentWindow: w)
title = w.title ?? ""
app = w.owningApplication?.applicationName ?? ""
pid = w.owningApplication?.processID
frameRect = w.frame
case .display(let id):
guard let d = content.displays.first(where: { $0.displayID == id }) else {
gone = true
return completion("that display isn't connected")
}
filter = SCContentFilter(display: d, excludingWindows: [])
title = displayName(id)
app = "Mac"
frameRect = displayRect(id)
if crop != nil { config.sourceRect = frameRect.offsetBy(dx: -CGDisplayBounds(id).minX, dy: -CGDisplayBounds(id).minY) }
case .test, .separate:
return completion("not a ScreenCaptureKit source")
}
scale = Double(filter.pointPixelScale)
// A display's own mode knows best: ScreenCaptureKit can still say 1
// for a virtual display that has only just switched to HiDPI.
if case .display(let id) = source, let mode = CGDisplayCopyDisplayMode(id), mode.width > 0 {
scale = max(scale, Double(mode.pixelWidth) / Double(mode.width))
}
let (w, h) = fitSize(width: frameRect.width * scale, height: frameRect.height * scale, maxLong: maxLong)
config.width = w
config.height = h
config.minimumFrameInterval = CMTime(value: 1, timescale: CMTimeScale(fps))
config.pixelFormat = kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange
config.colorMatrix = CGDisplayStream.yCbCrMatrix_ITU_R_709_2
config.colorSpaceName = CGColorSpace.sRGB
config.showsCursor = true
config.queueDepth = 5
config.scalesToFit = true
config.preservesAspectRatio = true
config.capturesAudio = false
let stream = SCStream(filter: filter, configuration: config, delegate: self)
do {
try stream.addStreamOutput(self, type: .screen, sampleHandlerQueue: queue)
} catch {
return completion("couldn't capture: \(error.localizedDescription)")
}
self.stream = stream
stream.startCapture { error in
self.queue.async {
if self.cancelled {
stream.stopCapture { _ in }
return
}
if let error { return completion("couldn't capture: \(error.localizedDescription)") }
self.startPolling()
completion(nil)
}
}
}
func stop() {
queue.async {
self.cancelled = true
self.poll?.cancel()
self.poll = nil
self.stream?.stopCapture { _ in }
self.stream = nil
}
}
/// Windows move, resize, retitle and close, and displays change mode;
/// ScreenCaptureKit doesn't say.
private func startPolling() {
if case .display(let id) = source { return pollDisplay(id) }
guard case .window(let id) = source else { return }
let t = DispatchSource.makeTimerSource(queue: queue)
t.schedule(deadline: .now() + 1, repeating: 1)
t.setEventHandler { [weak self] in
guard let self else { return }
guard let info = WindowInfo.find(id) else {
self.stop()
self.onEnded?("the window closed")
return
}
var changed = false
if !info.title.isEmpty, info.title != self.title { self.title = info.title; changed = true }
let old = self.frameRect
self.frameRect = info.bounds
if abs(old.width - info.bounds.width) > 1 || abs(old.height - info.bounds.height) > 1 {
let (w, h) = fitSize(width: info.bounds.width * self.scale, height: info.bounds.height * self.scale, maxLong: self.maxLong)
self.config.width = w
self.config.height = h
self.stream?.updateConfiguration(self.config) { _ in }
changed = true
}
if changed { self.onChange?() }
}
t.resume()
poll = t
}
func setMaxLong(_ n: Int) {
queue.async {
self.maxLong = n
guard self.stream != nil else { return }
let (w, h) = fitSize(width: self.frameRect.width * self.scale, height: self.frameRect.height * self.scale, maxLong: n)
guard w != self.config.width || h != self.config.height else { return }
self.config.width = w
self.config.height = h
self.stream?.updateConfiguration(self.config) { _ in }
}
}
private func pollDisplay(_ id: CGDirectDisplayID) {
let t = DispatchSource.makeTimerSource(queue: queue)
t.schedule(deadline: .now() + 0.5, repeating: 1)
t.setEventHandler { [weak self] in
guard let self, let mode = CGDisplayCopyDisplayMode(id), mode.width > 0 else { return }
let bounds = self.displayRect(id), scale = Double(mode.pixelWidth) / Double(mode.width)
let (w, h) = fitSize(width: bounds.width * scale, height: bounds.height * scale, maxLong: self.maxLong)
self.frameRect = bounds
guard w != self.config.width || h != self.config.height else { return }
self.scale = scale
self.config.width = w
self.config.height = h
self.stream?.updateConfiguration(self.config) { _ in }
self.onChange?()
}
t.resume()
poll = t
}
func stream(_ stream: SCStream, didOutputSampleBuffer sample: CMSampleBuffer, of type: SCStreamOutputType) {
guard type == .screen, sample.isValid, let pb = CMSampleBufferGetImageBuffer(sample) else { return }
// Only complete frames carry new pixels; idle and blank ones don't.
let info = (CMSampleBufferGetSampleAttachmentsArray(sample, createIfNecessary: false) as? [[SCStreamFrameInfo: Any]])?.first
if let raw = info?[.status] as? Int, let status = SCFrameStatus(rawValue: raw), status != .complete {
return
}
let pts = CMSampleBufferGetPresentationTimeStamp(sample)
// When WindowServer composited it: the moment it showed on the Mac.
let shown = (info?[.displayTime] as? UInt64).map(hostUs) ?? hostUs(pts)
onFrame?(pb, pts, shown)
}
func stream(_ stream: SCStream, didStopWithError error: Error) {
onEnded?("capture stopped: \(error.localizedDescription)")
}
}
func displayName(_ id: CGDirectDisplayID) -> String {
for screen in NSScreen.screens {
if (screen.deviceDescription[NSDeviceDescriptionKey("NSScreenNumber")] as? NSNumber)?.uint32Value == id {
return screen.localizedName
}
}
return "Display \(id)"
}
/// A moving test card: bars, a clock and a frame counter, plus a dot where the
/// viewer's pointer is and the last key it sent, so input can be checked too.
final class TestSource: CaptureSource {
var onFrame: ((CVPixelBuffer, CMTime, Int64) -> Void)?
var onEnded: ((String) -> Void)?
var onChange: (() -> Void)?
let gone = false
let frameRect = CGRect(x: 0, y: 0, width: 1280, height: 720)
let title = "Test pattern"
let app = "Frame Control"
let pid: pid_t? = nil
private var timer: DispatchSourceTimer?
private var pool: CVPixelBufferPool?
private var n = 0
private var dot: (Double, Double)?
private var lastText = "Point or type in the headset"
private let queue = DispatchQueue(label: "frame-mac-view.test")
private var size = (1280, 720)
func start(maxLong: Int, fps: Int, completion: @escaping (String?) -> Void) {
size = fitSize(width: 1280, height: 720, maxLong: maxLong)
makePool()
let t = DispatchSource.makeTimerSource(queue: queue)
t.schedule(deadline: .now(), repeating: 1.0 / Double(fps))
t.setEventHandler { [weak self] in self?.draw() }
t.resume()
timer = t
completion(nil)
}
func stop() {
timer?.cancel()
timer = nil
}
func setMaxLong(_ n: Int) {
queue.async {
self.size = fitSize(width: 1280, height: 720, maxLong: n)
self.makePool()
}
}
private func makePool() {
let attrs: [CFString: Any] = [kCVPixelBufferPixelFormatTypeKey: kCVPixelFormatType_32BGRA,
kCVPixelBufferWidthKey: size.0, kCVPixelBufferHeightKey: size.1,
kCVPixelBufferIOSurfacePropertiesKey: [:] as CFDictionary]
pool = nil
CVPixelBufferPoolCreate(nil, nil, attrs as CFDictionary, &pool)
}
func pointer(x: Double, y: Double, text: String?) {
queue.async {
if x >= 0 { self.dot = (x, y) }
if let text { self.lastText = text }
}
}
private func draw() {
guard let pool else { return }
var out: CVPixelBuffer?
CVPixelBufferPoolCreatePixelBuffer(nil, pool, &out)
guard let pb = out else { return }
CVPixelBufferLockBaseAddress(pb, [])
defer { CVPixelBufferUnlockBaseAddress(pb, []) }
let (w, h) = size
guard let ctx = CGContext(data: CVPixelBufferGetBaseAddress(pb), width: w, height: h, bitsPerComponent: 8,
bytesPerRow: CVPixelBufferGetBytesPerRow(pb), space: CGColorSpace(name: CGColorSpace.sRGB)!,
bitmapInfo: CGImageAlphaInfo.premultipliedFirst.rawValue | CGBitmapInfo.byteOrder32Little.rawValue)
else { return }
let colors: [(CGFloat, CGFloat, CGFloat)] = [(0.75, 0.75, 0.75), (0.75, 0.75, 0), (0, 0.75, 0.75), (0, 0.75, 0),
(0.75, 0, 0.75), (0.75, 0, 0), (0, 0, 0.75)]
let bw = CGFloat(w) / CGFloat(colors.count)
for (i, c) in colors.enumerated() {
ctx.setFillColor(red: c.0, green: c.1, blue: c.2, alpha: 1)
ctx.fill(CGRect(x: CGFloat(i) * bw, y: CGFloat(h) * 0.35, width: bw + 1, height: CGFloat(h) * 0.65))
}
ctx.setFillColor(red: 0.08, green: 0.09, blue: 0.11, alpha: 1)
ctx.fill(CGRect(x: 0, y: 0, width: w, height: Int(Double(h) * 0.35)))
// A bar sweeping once a second shows motion and dropped frames.
let x = CGFloat(n % 60) / 60 * CGFloat(w)
ctx.setFillColor(red: 1, green: 1, blue: 1, alpha: 1)
ctx.fill(CGRect(x: x, y: CGFloat(h) * 0.35, width: max(4, CGFloat(w) / 120), height: CGFloat(h) * 0.65))
let f = DateFormatter()
f.dateFormat = "HH:mm:ss.SSS"
let label = "Frame Control test pattern \(f.string(from: Date())) frame \(n)\n\(lastText)"
let text = NSAttributedString(string: label, attributes: [
.font: NSFont.monospacedSystemFont(ofSize: CGFloat(h) / 24, weight: .medium),
.foregroundColor: NSColor.white,
])
let ns = NSGraphicsContext(cgContext: ctx, flipped: false)
NSGraphicsContext.saveGraphicsState()
NSGraphicsContext.current = ns
text.draw(at: CGPoint(x: CGFloat(w) * 0.03, y: CGFloat(h) * 0.08))
NSGraphicsContext.restoreGraphicsState()
if let (px, py) = dot {
let r = CGFloat(h) / 40
ctx.setFillColor(red: 1, green: 0.2, blue: 0.2, alpha: 1)
ctx.fillEllipse(in: CGRect(x: CGFloat(px) * CGFloat(w) - r, y: (1 - CGFloat(py)) * CGFloat(h) - r, width: 2 * r, height: 2 * r))
}
n += 1
onFrame?(pb, CMClockGetTime(CMClockGetHostTimeClock()), nowUs())
}
}
mac/frame-mac-view/Sources/Controller.swift
+234
View File
@@ -0,0 +1,234 @@
// Adapts each stream to its network, latency first. The viewer acknowledges
// every frame as it arrives ("rx"); from those acks the controller knows how
// long frames take to get through and how fast the link delivers them.
//
// - The gate: a new frame is sent only while the oldest unacknowledged one is
// younger than the path's usual round trip plus a little slack. So frames
// never queue up in SSH, TCP or the Wi-Fi driver; while the link is stuck
// the newest picture waits and goes out as soon as it moves again.
// - The bitrate: when frames start queueing (the round trip grows) or the
// gate has to hold frames back, it drops to a bit under what the link
// actually delivered; once things are clear it probes up again slowly, never
// above the quality setting's bitrate (the ceiling).
// - The tier: as the bitrate falls, fewer frames per second (60, 45, 30), then
// a smaller picture (75%, then 50% of the panel's pixels).
// See docs/mac-in-headset.md ("Adapting to the network"). Thread-safe.
import Foundation
final class RateController {
struct Tier: Equatable {
let fps: Int
let scale: Double // of the picture's long side
}
static let tiers = [Tier(fps: 60, scale: 1), Tier(fps: 45, scale: 1), Tier(fps: 30, scale: 1),
Tier(fps: 30, scale: 0.75), Tier(fps: 30, scale: 0.5)]
/// A tier is used while the target bitrate is at least this share of the ceiling.
static let floors = [0.45, 0.28, 0.16, 0.08, 0]
static let enabled = ProcessInfo.processInfo.environment["FRAME_MAC_VIEW_ADAPT"] != "0"
let maxFps: Int
private let lock = NSLock()
private var ceiling = 0 // bits/s at full size and frame rate
private(set) var target = 0
private(set) var tier = 0
private var unacked: [(seq: UInt32, sent: Int64, bytes: Int)] = []
/// Round trips (send -> ack arrives here), for the baseline: the lowest
/// in the last 10 s is the path without any queue.
private var rtts: [(t: Int64, v: Int64)] = []
private var acked: [(t: Int64, bytes: Int)] = [] // the last second
private var sentLog: [(t: Int64, bytes: Int)] = []
private var captures: [Int64] = [] // the last second
private var frameBytes = 0 // average recent frame, kept while the gate holds everything back
private var held = 0 // frames the gate held back since the last update
private var lastSignal = false
private var sawAck = false
private var lastDecrease: Int64 = 0
private var lastIncrease: Int64 = 0
private var belowSince: Int64 = 0, aboveSince: Int64 = 0
/// What changed, for the timeline: (time, event).
private(set) var events: [(Int64, String)] = []
init(maxFps: Int) { self.maxFps = maxFps }
private func locked<T>(_ f: () -> T) -> T { lock.lock(); defer { lock.unlock() }; return f() }
/// The quality setting's bitrate at full size; the first call also starts there.
func setCeiling(_ bps: Int) {
locked {
if target == 0 || target > bps { target = bps }
ceiling = bps
}
}
var fps: Int { locked { fpsLocked } }
private var fpsLocked: Int { min(maxFps, RateController.tiers[tier].fps) }
var scale: Double { locked { RateController.tiers[tier].scale } }
var baseRtt: Int64 { locked { baseline() } }
private func baseline() -> Int64 { rtts.map(\.v).min() ?? 0 }
/// How late a frame may be before the gate holds the next one: one frame
/// interval, plus room for the jitter this link normally has (1.5 times
/// its recent spread), so ordinary Wi-Fi jitter doesn't cost frames but a
/// real queue does. Updated in update().
private var slack: Int64 = 40_000
/// Whether a frame may be sent now without queueing behind earlier ones.
/// `counts`: a held frame is a sign of congestion (not when merely
/// re-checking whether a held frame can go yet).
func maySend(now: Int64, counts: Bool = true) -> Bool {
locked {
guard RateController.enabled, sawAck else { return true } // not heard from the viewer yet
// Unacknowledged for 2 s: gone with a reconnection, not in a queue.
unacked.removeAll { now - $0.sent > 2_000_000 }
guard let oldest = unacked.first else { return true }
// Age is what bounds latency. The count only stops a burst, and it
// allows a full round trip of frames, so a long but clear path
// (100 ms away) still gets every frame.
let interval = Int64(1_000_000 / max(1, fpsLocked))
let window = max(3, Int((baseline() + slack) / interval) + 1)
if unacked.count < window, now - oldest.sent <= baseline() + slack { return true }
if counts { held += 1 }
return false
}
}
/// A picture was captured (sent or not): with the frame sizes, what this
/// stream would send if the link allowed.
func captured(at t: Int64) {
locked {
captures.append(t)
if captures.count > 256 { captures.removeFirst(captures.count - 256) }
}
}
func sent(seq: UInt32, bytes: Int, at t: Int64) {
locked {
// Bounded even if the viewer never acknowledges (an old viewer, or
// the controller is off).
unacked.append((seq, t, bytes))
if unacked.count > 512 { unacked.removeFirst(unacked.count - 512) }
sentLog.append((t, bytes))
if sentLog.count > 1024 { sentLog.removeFirst(sentLog.count - 1024) }
}
}
/// The viewer has frame `seq`. Returns true if that may let a held frame go.
func acked(seq: UInt32, at now: Int64) -> Bool {
locked {
sawAck = true
guard let i = unacked.firstIndex(where: { $0.seq == seq }) else { return false }
let f = unacked[i]
unacked.removeSubrange(0...i) // TCP delivers in order: earlier ones arrived too
rtts.append((now, now - f.sent))
acked.append((now, f.bytes))
return true
}
}
/// Called every 100 ms. Returns the new bitrate target, or nil if the
/// controller is off.
func update(now: Int64) -> Int? {
locked {
rtts.removeAll { now - $0.t > 10_000_000 }
acked.removeAll { now - $0.t > 500_000 }
sentLog.removeAll { now - $0.t > 500_000 }
unacked.removeAll { now - $0.sent > 2_000_000 }
captures.removeAll { now - $0 > 1_000_000 }
guard RateController.enabled, ceiling > 0 else { return nil }
let base = baseline()
let spread = rtts.filter { now - $0.t < 2_000_000 }.map(\.v).sorted()
let jitter = spread.isEmpty ? 0 : spread[spread.count * 9 / 10] - base
let interval = Int64(1_000_000 / max(1, fpsLocked))
slack = interval + min(max(jitter * 3 / 2, 25_000), 80_000)
let recent = rtts.filter { now - $0.t < 300_000 }.map(\.v).sorted()
let queueing = recent.isEmpty ? 0 : recent[recent.count / 2] - base
let oldestAge = unacked.first.map { now - $0.sent } ?? 0
let stuck = oldestAge > base + 100_000
let delivered = acked.reduce(0) { $0 + $1.bytes } * 16 // bits/s over the last half second
let sending = sentLog.reduce(0) { $0 + $1.bytes } * 16
// Demand: captures per second (up to the tier's rate) times the
// average frame. A test card or a mostly still window wants far
// less than its budget; when the link hiccups, cutting its bitrate
// can't help, and it would only look link-limited afterwards.
if !sentLog.isEmpty { frameBytes = sentLog.reduce(0) { $0 + $1.bytes } / sentLog.count }
let demand = min(captures.count, fpsLocked) * frameBytes * 8
// Delay alone isn't our queue: Wi-Fi jitters by itself. It only
// counts while this stream uses a good part of its budget (so its
// own data could be what's queueing). Frames the gate had to hold,
// or one stuck in flight, show demand the link isn't carrying
// whatever was sent (the gate itself keeps what's sent low).
let busy = sending >= target / 2
// Twice in a row (200 ms), so one late ack doesn't count.
let signal = (busy && queueing > 40_000) || held >= 3 || stuck
let congested = signal && lastSignal
lastSignal = signal
let heldNow = held
held = 0
let floorBps = 300_000
if congested, now - lastDecrease > 300_000 {
// Down to a bit under what got through: at least a fifth off, at
// most half (a stall delivers nothing, but the link is still there).
let measured = Int(Double(delivered) * 0.9)
var next = max(floorBps, min(target * 4 / 5, max(measured, target / 2)))
// App-limited (it wants about half its budget or less): never
// below twice what it wants, however many cuts in a row. The
// extra quarter is hysteresis, so frame sizes wobbling at the
// floor don't switch the protection off.
if demand > 0, demand * 2 <= target * 5 / 4 { next = max(next, min(target, demand * 2)) }
target = next
lastDecrease = now
events.append((now, "down to \(target / 1000) kbit/s: queue \(queueing / 1000) ms, held \(heldNow), "
+ "oldest \(oldestAge / 1000) ms, base \(base / 1000) ms, sent \(sending / 1000) got \(delivered / 1000) "
+ "wants \(demand / 1000)"))
} else if !congested, now - lastDecrease > 1_000_000, now - lastIncrease > 250_000, target < ceiling,
sending > target * 6 / 10 || now - lastDecrease > 3_000_000 {
// Clear for a second and using what it has: probe up.
target = min(ceiling, Int(Double(target) * 1.1) + 50_000)
lastIncrease = now
}
// Fewer frames or pixels only help a stream that fills its budget;
// a small one (a still window, a test card) keeps its tier.
retier(now: now, linkLimited: sending >= target * 7 / 10)
if events.count > 200 { events.removeFirst(events.count - 200) }
return target
}
}
/// Steps down quickly, straight to the tier the bitrate supports, and back
/// up one tier at a time only when there's clearly room (hysteresis).
private func retier(now: Int64, linkLimited: Bool) {
let share = Double(target) / Double(max(ceiling, 1))
if tier < RateController.tiers.count - 1, share < RateController.floors[tier], linkLimited {
if belowSince == 0 { belowSince = now }
if now - belowSince > 500_000 {
tier = RateController.floors.firstIndex { share >= $0 } ?? RateController.tiers.count - 1
belowSince = 0
events.append((now, "tier \(tier)"))
}
} else {
belowSince = 0
}
if tier > 0, share > RateController.floors[tier - 1] * 1.25 {
if aboveSince == 0 { aboveSince = now }
if now - aboveSince > 2_000_000 {
tier -= 1
aboveSince = 0
events.append((now, "tier \(tier)"))
}
} else {
aboveSince = 0
}
}
func state() -> [String: Any] {
locked {
["target": target, "ceiling": ceiling, "tier": tier, "fps": min(maxFps, RateController.tiers[tier].fps),
"scale": RateController.tiers[tier].scale, "baseRtt": Double(baseline()) / 1000,
"inFlight": unacked.count, "slack": Double(slack) / 1000, "adapt": RateController.enabled]
}
}
func eventList() -> [[String: Any]] { locked { events.map { ["t": $0.0, "e": $0.1] } } }
}
mac/frame-mac-view/Sources/Encoder.swift
+217
View File
@@ -0,0 +1,217 @@
// VideoToolbox encoding: H.264 for the normal path (hardware, low-latency rate
// control, no B-frames, Annex B output that WebCodecs takes without a
// description), or JPEG stills for viewers that can't decode H.264.
import CoreMedia
import Foundation
import VideoToolbox
enum Codec: String {
case h264, jpeg
}
final class Encoder {
let codec: Codec
let fps: Int
let bitsPerPixel: Double
private(set) var width = 0
private(set) var height = 0
private var session: VTCompressionSession?
private var forceKey = true
private var lastPts = CMTime.invalid
private var lastGiven = CMTime.invalid
/// Bits per second to aim for; nil means from the size, frame rate and
/// bits per pixel. Changing it takes effect on the next frame.
private var target: Int?
private(set) var bitrate = 0
/// Called on VideoToolbox's thread with one access unit (or JPEG) per
/// frame, and the sequence number it was submitted with.
var onFrame: ((Data, Bool, CMTime, UInt32) -> Void)?
var onError: ((String) -> Void)?
/// Called once per frame handed to VideoToolbox, however it went.
var onDone: ((UInt32) -> Void)?
/// Experiment switches (FRAME_MAC_VIEW_ENCODER, comma-separated), so a
/// benchmark can compare them without a rebuild.
static let options = Set((ProcessInfo.processInfo.environment["FRAME_MAC_VIEW_ENCODER"] ?? "")
.split(separator: ",").map(String.init))
init(codec: Codec, fps: Int, bitsPerPixel: Double) {
self.codec = codec
self.fps = fps
self.bitsPerPixel = bitsPerPixel
}
/// The bitrate the size and quality setting would give.
func defaultBitrate(width w: Int, height h: Int) -> Int {
Int(min(max(Double(w * h * fps) * bitsPerPixel, 2_000_000), 60_000_000))
}
/// Live, without a new keyframe. Call on the encoding queue.
func setBitrate(_ bps: Int?) {
target = bps
guard codec == .h264, let s = session else { return }
let b = bps ?? defaultBitrate(width: width, height: height)
guard b != bitrate else { return }
bitrate = b
VTSessionSetProperty(s, key: kVTCompressionPropertyKey_AverageBitRate, value: b as CFTypeRef)
// A hard ceiling too: no more than 200 ms' worth of bits in any 200 ms,
// so a keyframe can't hold the link for long.
if Encoder.options.contains("cap") {
VTSessionSetProperty(s, key: kVTCompressionPropertyKey_DataRateLimits,
value: [b / 8 / 5, 0.2] as CFArray)
}
}
deinit { invalidate() }
func requestKeyFrame() { forceKey = true }
func invalidate() {
if let s = session {
VTCompressionSessionCompleteFrames(s, untilPresentationTimeStamp: .invalid)
VTCompressionSessionInvalidate(s)
}
session = nil
}
private func makeSession(width w: Int, height h: Int) -> Bool {
invalidate()
var spec: [CFString: Any] = [:]
if codec == .h264 { spec[kVTVideoEncoderSpecification_EnableLowLatencyRateControl] = true }
if Encoder.options.contains("hw") { spec[kVTVideoEncoderSpecification_RequireHardwareAcceleratedVideoEncoder] = true }
var s: VTCompressionSession?
let type = codec == .h264 ? kCMVideoCodecType_H264 : kCMVideoCodecType_JPEG
func create(_ spec: [CFString: Any]) -> OSStatus {
VTCompressionSessionCreate(allocator: nil, width: Int32(w), height: Int32(h), codecType: type,
encoderSpecification: spec as CFDictionary, imageBufferAttributes: nil,
compressedDataAllocator: nil, outputCallback: nil, refcon: nil,
compressionSessionOut: &s)
}
var err = create(spec)
// Low-latency rate control needs Apple's hardware encoder; without it
// (some VMs), plain real-time encoding still works.
if err != noErr, !spec.isEmpty { err = create([:]) }
guard err == noErr, let s else {
onError?("couldn't start the \(codec.rawValue) encoder (VideoToolbox \(err))")
return false
}
func set(_ key: CFString, _ value: Any) { VTSessionSetProperty(s, key: key, value: value as CFTypeRef) }
set(kVTCompressionPropertyKey_RealTime, true)
set(kVTCompressionPropertyKey_ColorPrimaries, kCVImageBufferColorPrimaries_ITU_R_709_2)
set(kVTCompressionPropertyKey_TransferFunction, kCVImageBufferTransferFunction_ITU_R_709_2)
set(kVTCompressionPropertyKey_YCbCrMatrix, kCVImageBufferYCbCrMatrix_ITU_R_709_2)
if codec == .h264 {
set(kVTCompressionPropertyKey_ProfileLevel, kVTProfileLevel_H264_ConstrainedHigh_AutoLevel)
set(kVTCompressionPropertyKey_AllowFrameReordering, false)
set(kVTCompressionPropertyKey_ExpectedFrameRate, fps)
if Encoder.options.contains("nodelay") { set(kVTCompressionPropertyKey_MaxFrameDelayCount, 0) }
if Encoder.options.contains("speed") { set(kVTCompressionPropertyKey_PrioritizeEncodingSpeedOverQuality, true) }
// A keyframe every 10 s at most, so a viewer that lost one recovers
// even if it never asks. Viewers ask for one when they start.
set(kVTCompressionPropertyKey_MaxKeyFrameIntervalDuration, 10)
} else {
set(kVTCompressionPropertyKey_Quality, 0.8)
}
VTCompressionSessionPrepareToEncodeFrames(s)
session = s
lastPts = .invalid
lastGiven = .invalid
width = w
height = h
forceKey = true
bitrate = 0
setBitrate(target)
return true
}
/// False if the frame never reached VideoToolbox (then onDone won't come).
@discardableResult
func encode(_ pb: CVPixelBuffer, pts given: CMTime, seq: UInt32) -> Bool {
// The encoder's own timeline: strictly increasing (a resent picture
// stamped "now" can be followed by a capture stamped a moment earlier),
// and never more than two frames on from the last one. Rate control
// budgets bits by elapsed time, so after a pause (the link held frames
// back, or nothing changed) one frame would otherwise get a quarter
// second's worth of bits: 200 KB that then hold a slow link for half a second.
let w = CVPixelBufferGetWidth(pb), h = CVPixelBufferGetHeight(pb)
if session == nil || w != width || h != height {
guard makeSession(width: w, height: h) else { return false } // a new timeline too
}
guard let s = session else { return false }
var pts = given
if lastPts.isValid, lastGiven.isValid {
let gap = CMTimeSubtract(given, lastGiven)
let most = CMTime(value: 2, timescale: CMTimeScale(fps))
let step = CMTimeCompare(gap, most) > 0 ? most : gap
pts = CMTimeAdd(lastPts, CMTimeMaximum(step, CMTime(value: 1, timescale: 1_000_000)))
}
lastGiven = given
var props: CFDictionary?
if forceKey {
props = [kVTEncodeFrameOptionKey_ForceKeyFrame: true] as CFDictionary
forceKey = false
}
let codec = self.codec
lastPts = pts
let status = VTCompressionSessionEncodeFrame(s, imageBuffer: pb, presentationTimeStamp: pts, duration: .invalid,
frameProperties: props, infoFlagsOut: nil) { [weak self] status, _, sample in
guard let self else { return }
defer { self.onDone?(seq) }
guard status == noErr, let sample else { return }
if codec == .jpeg {
if let data = Self.bytes(sample) { self.onFrame?(data, true, pts, seq) }
} else if let (data, key) = Self.annexB(sample) {
self.onFrame?(data, key, pts, seq)
}
}
if status != noErr { forceKey = true }
return status == noErr
}
private static func bytes(_ sample: CMSampleBuffer) -> Data? {
guard let block = CMSampleBufferGetDataBuffer(sample) else { return nil }
var length = 0
var ptr: UnsafeMutablePointer<CChar>?
guard CMBlockBufferGetDataPointer(block, atOffset: 0, lengthAtOffsetOut: nil, totalLengthOut: &length,
dataPointerOut: &ptr) == noErr, let ptr else { return nil }
return Data(bytes: ptr, count: length)
}
/// AVCC sample -> Annex B access unit, with SPS and PPS before keyframes.
static func annexB(_ sample: CMSampleBuffer) -> (Data, Bool)? {
guard let avcc = bytes(sample) else { return nil }
var key = true
if let atts = CMSampleBufferGetSampleAttachmentsArray(sample, createIfNecessary: false) as? [[CFString: Any]],
let first = atts.first, first[kCMSampleAttachmentKey_NotSync] as? Bool == true {
key = false
}
let start: [UInt8] = [0, 0, 0, 1]
var out = Data()
if key, let fmt = CMSampleBufferGetFormatDescription(sample) {
var count = 0
CMVideoFormatDescriptionGetH264ParameterSetAtIndex(fmt, parameterSetIndex: 0, parameterSetPointerOut: nil,
parameterSetSizeOut: nil, parameterSetCountOut: &count,
nalUnitHeaderLengthOut: nil)
for i in 0..<count {
var p: UnsafePointer<UInt8>?
var n = 0
if CMVideoFormatDescriptionGetH264ParameterSetAtIndex(fmt, parameterSetIndex: i, parameterSetPointerOut: &p,
parameterSetSizeOut: &n, parameterSetCountOut: nil,
nalUnitHeaderLengthOut: nil) == noErr, let p {
out.append(contentsOf: start)
out.append(p, count: n)
}
}
}
let bytes = [UInt8](avcc)
var i = 0
while i + 4 <= bytes.count {
let n = Int(bytes[i]) << 24 | Int(bytes[i + 1]) << 16 | Int(bytes[i + 2]) << 8 | Int(bytes[i + 3])
i += 4
guard n > 0, i + n <= bytes.count else { break }
out.append(contentsOf: start)
out.append(contentsOf: bytes[i..<(i + n)])
i += n
}
return (out, key)
}
}
mac/frame-mac-view/Sources/Input.swift
+168
View File
@@ -0,0 +1,168 @@
// Turns the viewer's pointer and key events into real Mac input. Needs the
// Accessibility permission ("control your computer"); without it macOS drops
// the events silently, so the agent reports the permission to the viewer.
import AppKit
import ApplicationServices
import Foundation
@_silgen_name("_AXUIElementGetWindow")
private func _AXUIElementGetWindow(_ element: AXUIElement, _ id: UnsafeMutablePointer<CGWindowID>) -> AXError
enum Input {
static let source = CGEventSource(stateID: .hidSystemState)
/// What each viewer (session id) is holding down, so one panel closing
/// lets go of its own keys and buttons and nobody else's. Main thread.
private static var buttonsHeld: [Int: Set<Int>] = [:]
private static var keysHeld: [Int: Set<CGKeyCode>] = [:]
private static var lastDown: (time: TimeInterval, point: CGPoint, button: Int, count: Int)?
static var allowed: Bool { AXIsProcessTrusted() }
/// Brings a window to the front so a click lands on it, not on whatever
/// covers it, and typing goes to it.
static func focus(window id: CGWindowID, pid: pid_t) {
if frontWindow() == id { return }
let app = AXUIElementCreateApplication(pid)
var value: CFTypeRef?
if AXUIElementCopyAttributeValue(app, kAXWindowsAttribute as CFString, &value) == .success,
let windows = value as? [AXUIElement] {
for w in windows {
var wid: CGWindowID = 0
if _AXUIElementGetWindow(w, &wid) == .success, wid == id {
AXUIElementPerformAction(w, kAXRaiseAction as CFString)
AXUIElementSetAttributeValue(w, kAXMainAttribute as CFString, kCFBooleanTrue)
break
}
}
}
AXUIElementSetAttributeValue(app, kAXFrontmostAttribute as CFString, kCFBooleanTrue)
NSRunningApplication(processIdentifier: pid)?.activate()
}
static func frontWindow() -> CGWindowID? {
WindowInfo.all().first { $0.layer == 0 }?.id
}
/// `button` uses the browser's numbering: 0 left, 1 middle, 2 right.
static func mouse(_ kind: String, button: Int, at p: CGPoint, owner: Int) {
let b: CGMouseButton = button == 2 ? .right : button == 1 ? .center : .left
let type: CGEventType
switch kind {
case "down":
type = b == .left ? .leftMouseDown : b == .right ? .rightMouseDown : .otherMouseDown
buttonsHeld[owner, default: []].insert(button)
case "up":
type = b == .left ? .leftMouseUp : b == .right ? .rightMouseUp : .otherMouseUp
buttonsHeld[owner]?.remove(button)
// Another viewer still holding it keeps it down.
if buttonsHeld.values.contains(where: { $0.contains(button) }) { return }
default: // a drag is whatever this viewer is holding
let mine = buttonsHeld[owner] ?? []
if mine.contains(0) { type = .leftMouseDragged }
else if mine.contains(2) { type = .rightMouseDragged }
else if mine.contains(1) { type = .otherMouseDragged }
else { type = .mouseMoved }
}
let mine = buttonsHeld[owner] ?? []
let held: CGMouseButton = mine.contains(0) ? .left : mine.contains(2) ? .right : mine.contains(1) ? .center : .left
guard let e = CGEvent(mouseEventSource: source, mouseType: type, mouseCursorPosition: p,
mouseButton: kind == "move" ? held : b) else { return }
if kind == "down" || kind == "up" {
let now = ProcessInfo.processInfo.systemUptime
var count = 1
if kind == "down" {
if let l = lastDown, l.button == button, now - l.time < NSEvent.doubleClickInterval,
abs(l.point.x - p.x) < 5, abs(l.point.y - p.y) < 5 { count = l.count + 1 }
lastDown = (now, p, button, count)
} else if let l = lastDown, l.button == button {
count = l.count
}
e.setIntegerValueField(.mouseEventClickState, value: Int64(count))
}
e.post(tap: .cghidEventTap)
}
static func scroll(dx: Double, dy: Double, at p: CGPoint, owner: Int) {
mouse("move", button: 0, at: p, owner: owner)
// Browsers report pixels with +y meaning "scroll down"; macOS's +y is up.
guard let e = CGEvent(scrollWheelEvent2Source: source, units: .pixel, wheelCount: 2,
wheel1: Int32(-dy.rounded()), wheel2: Int32(-dx.rounded()), wheel3: 0) else { return }
e.location = p
e.post(tap: .cghidEventTap)
}
/// Lets go of every button and key this viewer is holding down, so a
/// dropped connection can't leave, say, Shift or ⌘ stuck on.
static func releaseAll(owner: Int) {
let p = CGEvent(source: nil)?.location ?? .zero
let buttons = buttonsHeld.removeValue(forKey: owner) ?? []
let keys = keysHeld.removeValue(forKey: owner) ?? []
// Only what no other viewer is still holding.
for b in buttons where !buttonsHeld.values.contains(where: { $0.contains(b) }) {
let type: CGEventType = b == 2 ? .rightMouseUp : b == 1 ? .otherMouseUp : .leftMouseUp
CGEvent(mouseEventSource: source, mouseType: type, mouseCursorPosition: p,
mouseButton: b == 2 ? .right : b == 1 ? .center : .left)?.post(tap: .cghidEventTap)
}
for k in keys where !keysHeld.values.contains(where: { $0.contains(k) }) {
CGEvent(keyboardEventSource: source, virtualKey: k, keyDown: false)?.post(tap: .cghidEventTap)
}
}
static func key(code: String, key: String, down: Bool, mods: [String], owner: Int) {
var flags = CGEventFlags()
if mods.contains("shift") { flags.insert(.maskShift) }
if mods.contains("ctrl") { flags.insert(.maskControl) }
if mods.contains("alt") { flags.insert(.maskAlternate) }
if mods.contains("meta") { flags.insert(.maskCommand) }
if let vk = keyCodes[code] {
guard let e = CGEvent(keyboardEventSource: source, virtualKey: vk, keyDown: down) else { return }
if down {
keysHeld[owner, default: []].insert(vk)
} else {
keysHeld[owner]?.remove(vk)
if keysHeld.values.contains(where: { $0.contains(vk) }) { return } // still held elsewhere
}
e.flags = flags
e.post(tap: .cghidEventTap)
} else if down, !key.isEmpty, key.count <= 4, key.unicodeScalars.allSatisfy({ $0.value >= 0x20 }) {
// A key the table doesn't know (a non-US layout, the headset's
// on-screen keyboard): type its character instead.
text(key)
}
}
static func text(_ s: String) {
let units = Array(s.utf16)
for chunk in stride(from: 0, to: units.count, by: 16) {
let part = Array(units[chunk..<min(chunk + 16, units.count)])
for down in [true, false] {
guard let e = CGEvent(keyboardEventSource: source, virtualKey: 0, keyDown: down) else { continue }
e.keyboardSetUnicodeString(stringLength: part.count, unicodeString: part)
e.post(tap: .cghidEventTap)
}
}
}
/// DOM KeyboardEvent.code -> macOS virtual key code (ANSI positions).
static let keyCodes: [String: CGKeyCode] = [
"KeyA": 0x00, "KeyS": 0x01, "KeyD": 0x02, "KeyF": 0x03, "KeyH": 0x04, "KeyG": 0x05, "KeyZ": 0x06, "KeyX": 0x07,
"KeyC": 0x08, "KeyV": 0x09, "IntlBackslash": 0x0A, "KeyB": 0x0B, "KeyQ": 0x0C, "KeyW": 0x0D, "KeyE": 0x0E,
"KeyR": 0x0F, "KeyY": 0x10, "KeyT": 0x11, "Digit1": 0x12, "Digit2": 0x13, "Digit3": 0x14, "Digit4": 0x15,
"Digit6": 0x16, "Digit5": 0x17, "Equal": 0x18, "Digit9": 0x19, "Digit7": 0x1A, "Minus": 0x1B, "Digit8": 0x1C,
"Digit0": 0x1D, "BracketRight": 0x1E, "KeyO": 0x1F, "KeyU": 0x20, "BracketLeft": 0x21, "KeyI": 0x22,
"KeyP": 0x23, "Enter": 0x24, "KeyL": 0x25, "KeyJ": 0x26, "Quote": 0x27, "KeyK": 0x28, "Semicolon": 0x29,
"Backslash": 0x2A, "Comma": 0x2B, "Slash": 0x2C, "KeyN": 0x2D, "KeyM": 0x2E, "Period": 0x2F, "Tab": 0x30,
"Space": 0x31, "Backquote": 0x32, "Backspace": 0x33, "Escape": 0x35, "MetaRight": 0x36, "MetaLeft": 0x37,
"ShiftLeft": 0x38, "CapsLock": 0x39, "AltLeft": 0x3A, "ControlLeft": 0x3B, "ShiftRight": 0x3C,
"AltRight": 0x3D, "ControlRight": 0x3E, "F17": 0x40, "NumpadDecimal": 0x41, "NumpadMultiply": 0x43,
"NumpadAdd": 0x45, "NumLock": 0x47, "NumpadDivide": 0x4B, "NumpadEnter": 0x4C, "NumpadSubtract": 0x4E,
"F18": 0x4F, "F19": 0x50, "NumpadEqual": 0x51, "Numpad0": 0x52, "Numpad1": 0x53, "Numpad2": 0x54,
"Numpad3": 0x55, "Numpad4": 0x56, "Numpad5": 0x57, "Numpad6": 0x58, "Numpad7": 0x59, "F20": 0x5A,
"Numpad8": 0x5B, "Numpad9": 0x5C, "F5": 0x60, "F6": 0x61, "F7": 0x62, "F3": 0x63, "F8": 0x64, "F9": 0x65,
"F11": 0x67, "F13": 0x69, "F16": 0x6A, "F14": 0x6B, "F10": 0x6D, "ContextMenu": 0x6E, "F12": 0x6F,
"F15": 0x71, "Insert": 0x72, "Help": 0x72, "Home": 0x73, "PageUp": 0x74, "Delete": 0x75, "F4": 0x76,
"End": 0x77, "F2": 0x78, "PageDown": 0x79, "F1": 0x7A, "ArrowLeft": 0x7B, "ArrowRight": 0x7C,
"ArrowDown": 0x7D, "ArrowUp": 0x7E, "OSLeft": 0x37, "OSRight": 0x36,
]
}
mac/frame-mac-view/Sources/Separate.swift
+295
View File
@@ -0,0 +1,295 @@
// "Separate" mode: a streamed window gets a virtual display of its own. The
// window is moved onto it and sized to fill it, and the whole display is
// captured, so its menus, sheets, popovers and tooltips come along, nothing
// can cover it, and clicks land on it without raising anything. On stop the
// window goes back where it was. See docs/mac-window-separation.md.
import AppKit
import ApplicationServices
import CoreMedia
import Foundation
@_silgen_name("_AXUIElementGetWindow")
private func axWindowID(_ element: AXUIElement, _ id: UnsafeMutablePointer<CGWindowID>) -> AXError
/// One of our virtual displays, HiDPI (2 pixels per point). Main thread only.
final class VirtualDisplay {
static let vendor: UInt32 = 0xF0C0
/// How often macOS composites our displays (FRAME_MAC_VIEW_VD_HZ to experiment).
static let refreshRate = Double(ProcessInfo.processInfo.environment["FRAME_MAC_VIEW_VD_HZ"] ?? "") ?? 60
private var display: CGVirtualDisplay?
let id: CGDirectDisplayID
/// Removal is deferred while the Mac's screen sleeps, and an identity
/// can't be reused until it's gone, so each display gets a fresh serial.
private static var serial = (UInt32(truncatingIfNeeded: ProcessInfo.processInfo.processIdentifier) & 0xFFFF) << 12
init?(name: String, width: Int, height: Int) {
var made: CGVirtualDisplay?
for _ in 0..<8 where made == nil {
VirtualDisplay.serial &+= 1
let d = CGVirtualDisplayDescriptor()
d.queue = DispatchQueue.main
d.name = name
d.maxPixelsWide = UInt32(2 * width)
d.maxPixelsHigh = UInt32(2 * height)
d.sizeInMillimeters = CGSize(width: Double(width) * 0.2646, height: Double(height) * 0.2646)
d.vendorID = VirtualDisplay.vendor
d.productID = 0x5E9A
d.serialNum = VirtualDisplay.serial
d.terminationHandler = { _, _ in }
guard let vd = CGVirtualDisplay(descriptor: d) else { continue }
let s = CGVirtualDisplaySettings()
s.hiDPI = 1
let hz = VirtualDisplay.refreshRate
s.modes = [CGVirtualDisplayMode(width: UInt32(2 * width), height: UInt32(2 * height), refreshRate: hz)!,
CGVirtualDisplayMode(width: UInt32(width), height: UInt32(height), refreshRate: hz)!]
if vd.apply(s), vd.displayID != 0 { made = vd }
}
guard let made else { return nil }
display = made
id = made.displayID
// Pick the HiDPI mode: `width` points drawn with twice the pixels.
let modes = (CGDisplayCopyAllDisplayModes(id, [kCGDisplayShowDuplicateLowResolutionModes: true] as CFDictionary)
as? [CGDisplayMode]) ?? []
if let hidpi = modes.first(where: {
$0.width == width && $0.height == height && $0.pixelWidth == 2 * width && $0.refreshRate == VirtualDisplay.refreshRate
}) ?? modes.first(where: { $0.width == width && $0.height == height && $0.pixelWidth == 2 * width }) {
var cfg: CGDisplayConfigRef?
CGBeginDisplayConfiguration(&cfg)
CGConfigureDisplayWithDisplayMode(cfg, id, hidpi, nil)
CGCompleteDisplayConfiguration(cfg, .forSession)
}
}
var bounds: CGRect { CGDisplayBounds(id) }
var isHiDPI: Bool { CGDisplayCopyDisplayMode(id).map { $0.pixelWidth > $0.width } ?? false }
var screen: NSScreen? {
NSScreen.screens.first {
($0.deviceDescription[NSDeviceDescriptionKey("NSScreenNumber")] as? NSNumber)?.uint32Value == id
}
}
/// Where a window may go, in global (top-left origin) points: the display
/// minus its menu bar and anything else macOS reserves there.
var usableRect: CGRect {
let b = bounds
guard let s = screen else { return b }
let top = s.frame.maxY - s.visibleFrame.maxY, bottom = s.visibleFrame.minY - s.frame.minY
let left = s.visibleFrame.minX - s.frame.minX, right = s.frame.maxX - s.visibleFrame.maxX
return CGRect(x: b.minX + left, y: b.minY + top, width: b.width - left - right, height: b.height - top - bottom)
}
func release() { display = nil }
static func isOurs(_ id: CGDirectDisplayID) -> Bool { CGDisplayVendorNumber(id) == vendor }
}
/// A window through the Accessibility API.
struct AXWindow {
let element: AXUIElement
init?(windowID: CGWindowID, pid: pid_t) {
let app = AXUIElementCreateApplication(pid)
var value: CFTypeRef?
guard AXUIElementCopyAttributeValue(app, kAXWindowsAttribute as CFString, &value) == .success,
let windows = value as? [AXUIElement] else { return nil }
guard let match = windows.first(where: { w in
var id: CGWindowID = 0
return axWindowID(w, &id) == .success && id == windowID
}) else { return nil }
element = match
}
var frame: CGRect? {
var p: CFTypeRef?, s: CFTypeRef?
guard AXUIElementCopyAttributeValue(element, kAXPositionAttribute as CFString, &p) == .success,
AXUIElementCopyAttributeValue(element, kAXSizeAttribute as CFString, &s) == .success else { return nil }
var point = CGPoint.zero, size = CGSize.zero
AXValueGetValue(p as! AXValue, .cgPoint, &point)
AXValueGetValue(s as! AXValue, .cgSize, &size)
return CGRect(origin: point, size: size)
}
var isFullScreen: Bool {
var v: CFTypeRef?
return AXUIElementCopyAttributeValue(element, "AXFullScreen" as CFString, &v) == .success && (v as? Bool) == true
}
/// Moving to another display: position first (so the size fits there),
/// then size, then position again in case the size change moved it.
func setFrame(_ r: CGRect) {
var origin = r.origin, size = r.size
let pos = AXValueCreate(.cgPoint, &origin)!, sz = AXValueCreate(.cgSize, &size)!
AXUIElementSetAttributeValue(element, kAXPositionAttribute as CFString, pos)
AXUIElementSetAttributeValue(element, kAXSizeAttribute as CFString, sz)
AXUIElementSetAttributeValue(element, kAXPositionAttribute as CFString, pos)
}
}
/// A window on a display of its own, captured as that display.
final class SeparateSource: CaptureSource {
let windowID: CGWindowID
var onFrame: ((CVPixelBuffer, CMTime, Int64) -> Void)?
var onEnded: ((String) -> Void)?
var onChange: (() -> Void)?
private(set) var title = ""
private(set) var app = ""
private(set) var pid: pid_t?
private(set) var gone = false
private var display: VirtualDisplay?
private var inner: SCKSource?
private var window: AXWindow?
private var original: CGRect?
private var crop: CGRect? // display points, when only part of the display is captured
private var poll: DispatchSourceTimer?
private var stopped = false
private let queue = DispatchQueue(label: "frame-mac-view.separate")
init(windowID: CGWindowID) { self.windowID = windowID }
var frameRect: CGRect { inner?.frameRect ?? .zero }
var displayID: CGDirectDisplayID? { display?.id }
func start(maxLong: Int, fps: Int, completion: @escaping (String?) -> Void) {
guard AXIsProcessTrusted() else {
return completion("Separate windows need the Accessibility permission (to move the window)")
}
guard let info = WindowInfo.find(windowID) else {
gone = true
return completion("that window has closed")
}
title = info.title
app = info.app
pid = info.pid
DispatchQueue.main.async { [self] in
guard !stopped else { return }
guard let w = AXWindow(windowID: windowID, pid: info.pid), let frame = w.frame else {
return completion("couldn't reach that window through Accessibility")
}
if w.isFullScreen { return completion("take the window out of full screen first") }
// A display the window's size, plus room for the menu bar, in
// points; within what a panel can show sharply.
let width = min(max(Int(frame.width.rounded()), 640), 2560) / 2 * 2
let height = min(max(Int(frame.height.rounded()) + 40, 480), 1600) / 2 * 2
guard let vd = VirtualDisplay(name: app.isEmpty ? "Frame Control" : "\(app) (Frame)", width: width, height: height) else {
return completion("macOS wouldn't create a display for this window")
}
display = vd
window = w
original = frame
// The new screen shows up in NSScreen a moment later; its menu bar
// decides where the window can go.
placeWindow(attempt: 0, maxLong: maxLong, fps: fps, completion: completion)
}
}
private func placeWindow(attempt: Int, maxLong: Int, fps: Int, completion: @escaping (String?) -> Void) {
guard !stopped, let vd = display, let w = window else { return }
// Wait for the screen to appear, and for its HiDPI mode, so the
// first frames are captured sharp.
if vd.screen == nil || !vd.isHiDPI, attempt < 30 {
DispatchQueue.main.asyncAfter(deadline: .now() + 0.1) {
self.placeWindow(attempt: attempt + 1, maxLong: maxLong, fps: fps, completion: completion)
}
return
}
let target = vd.usableRect
w.setFrame(target)
guard let now = w.frame, vd.bounds.intersects(now) else {
restore()
return completion("macOS didn't let the window move to its own display (Stage Manager can prevent this)")
}
let sck = SCKSource(.display(vd.id))
// A window that keeps its own size (Calculator, some dialogs) sits in
// the display's top-left corner: send only that corner, menu bar
// included, with room around it for menus, not a panel of wallpaper.
let b = vd.bounds
if now.width < target.width - 8 || now.height < target.height - 8 {
let w = min(b.width, max(now.maxX - b.minX, 480)), h = min(b.height, max(now.maxY - b.minY, 360))
crop = CGRect(x: 0, y: 0, width: (w / 2).rounded(.up) * 2, height: (h / 2).rounded(.up) * 2)
sck.crop = crop
}
sck.onFrame = { [weak self] pb, pts, shown in self?.onFrame?(pb, pts, shown) }
sck.onEnded = { [weak self] reason in self?.onEnded?(reason) }
inner = sck
sck.start(maxLong: maxLong, fps: fps) { [weak self] error in
// Back on main, where stop() runs too, so a viewer that left during
// startup can't leave a capture running.
DispatchQueue.main.async {
guard let self, !self.stopped else { return sck.stop() }
if let error {
self.restore()
return completion(error)
}
self.startPolling()
completion(nil)
}
}
}
/// The window can close or be retitled; ScreenCaptureKit won't say.
private func startPolling() {
let t = DispatchSource.makeTimerSource(queue: queue)
t.schedule(deadline: .now() + 1, repeating: 1)
t.setEventHandler { [weak self] in
guard let self else { return }
guard let info = WindowInfo.find(self.windowID) else {
self.gone = true
self.onEnded?("the window closed")
return
}
if !info.title.isEmpty, info.title != self.title {
self.title = info.title
self.onChange?()
}
// Displays added at the same moment can rearrange each other, and
// someone may drag the window away: put it back on its display.
if !info.bounds.isEmpty {
DispatchQueue.main.async { self.keepOnDisplay(info.bounds) }
}
}
t.resume()
poll = t
}
private func keepOnDisplay(_ now: CGRect) {
guard !stopped, let vd = display, let w = window else { return }
let b = vd.bounds
if let c = crop {
// Only part of the display is sent: the window must stay inside it.
guard !c.offsetBy(dx: b.minX, dy: b.minY).insetBy(dx: -2, dy: -2).contains(now) else { return }
} else {
guard !b.contains(CGPoint(x: now.midX, y: now.minY + 10)) else { return }
}
let target = vd.usableRect
w.setFrame(CGRect(origin: target.origin, size: now.size.width < target.width - 8 ? now.size : target.size))
}
/// Lifecycle state (stopped, inner, poll, window, display) is only touched on main.
func stop() {
DispatchQueue.main.async { [self] in
stopped = true
poll?.cancel()
poll = nil
inner?.stop()
restore()
}
}
/// Puts the window back where it was, then removes the display. In that
/// order: removing a display first would let macOS pick where it goes.
private func restore() {
if let w = window, let r = original, WindowInfo.find(windowID) != nil { w.setFrame(r) }
window = nil
original = nil
let vd = display
display = nil
// Give WindowServer a moment to move the window before the display goes.
DispatchQueue.main.asyncAfter(deadline: .now() + 0.3) { vd?.release() }
}
func setMaxLong(_ n: Int) { DispatchQueue.main.async { self.inner?.setMaxLong(n) } }
func pointer(x: Double, y: Double, text: String?) {}
}
mac/frame-mac-view/Sources/Server.swift
+269
View File
@@ -0,0 +1,269 @@
// A small HTTP/1.1 + WebSocket server on Network.framework, loopback only.
// Enough for the agent's JSON endpoints, the viewer page and one WebSocket per
// stream; not a general web server.
import CryptoKit
import Foundation
import Network
struct Request {
let method: String
let path: String
let query: [String: String]
let headers: [String: String] // lower-cased names
}
final class Server {
let queue = DispatchQueue(label: "frame-mac-view.server")
private let listener: NWListener
private let handle: (Request, HTTPConnection) -> Void
init(port: UInt16, handle: @escaping (Request, HTTPConnection) -> Void) throws {
let tcp = NWProtocolTCP.Options()
tcp.noDelay = true
let params = NWParameters(tls: nil, tcp: tcp)
// Port 0 lets the system pick; `port` then says which.
params.requiredLocalEndpoint = .hostPort(host: "127.0.0.1", port: NWEndpoint.Port(rawValue: port) ?? .any)
params.allowLocalEndpointReuse = true
listener = try NWListener(using: params)
self.handle = handle
}
var port: UInt16? { listener.port?.rawValue }
func start(ready: @escaping (Error?) -> Void) {
listener.stateUpdateHandler = { state in
switch state {
case .ready: ready(nil)
case .failed(let e): ready(e)
default: break
}
}
listener.newConnectionHandler = { [weak self] conn in
guard let self else { return }
HTTPConnection(conn, queue: self.queue, handle: self.handle).start()
}
listener.start(queue: queue)
}
}
final class HTTPConnection {
let conn: NWConnection
let queue: DispatchQueue
private let handle: (Request, HTTPConnection) -> Void
private var buffer = Data()
private var retained: HTTPConnection? // alive until the response is sent
init(_ conn: NWConnection, queue: DispatchQueue, handle: @escaping (Request, HTTPConnection) -> Void) {
self.conn = conn
self.queue = queue
self.handle = handle
}
func start() {
retained = self
conn.start(queue: queue)
readHead()
}
private func readHead() {
conn.receive(minimumIncompleteLength: 1, maximumLength: 16384) { [self] data, _, done, error in
if let data { buffer.append(data) }
if let end = buffer.range(of: Data("\r\n\r\n".utf8)) {
guard let req = Self.parse(buffer[..<end.lowerBound]) else { return respond(400, text: "bad request") }
handle(req, self)
} else if error != nil || done || buffer.count > 16384 {
close()
} else {
readHead()
}
}
}
static func parse(_ head: Data) -> Request? {
guard let text = String(data: head, encoding: .utf8) else { return nil }
let lines = text.components(separatedBy: "\r\n")
let parts = lines[0].split(separator: " ")
guard parts.count >= 2, let url = URLComponents(string: String(parts[1])) else { return nil }
var headers: [String: String] = [:]
for line in lines.dropFirst() {
guard let colon = line.firstIndex(of: ":") else { continue }
headers[line[..<colon].lowercased()] = line[line.index(after: colon)...].trimmingCharacters(in: .whitespaces)
}
var query: [String: String] = [:]
for item in url.queryItems ?? [] { query[item.name] = item.value ?? "" }
return Request(method: String(parts[0]), path: url.path, query: query, headers: headers)
}
func respond(_ status: Int, body: Data, type: String) {
let reason = [200: "OK", 400: "Bad Request", 403: "Forbidden", 404: "Not Found", 409: "Conflict", 500: "Internal Server Error"][status] ?? "Error"
var head = "HTTP/1.1 \(status) \(reason)\r\nContent-Type: \(type)\r\nContent-Length: \(body.count)\r\n"
head += "Cache-Control: no-store\r\nConnection: close\r\n\r\n"
conn.send(content: Data(head.utf8) + body, isComplete: true, completion: .contentProcessed { [self] _ in close() })
}
func respond(_ status: Int, text: String) {
respond(status, body: Data(text.utf8), type: "text/plain; charset=utf-8")
}
func respond(_ status: Int = 200, json: Any) {
let body = (try? JSONSerialization.data(withJSONObject: json, options: [.sortedKeys])) ?? Data("{}".utf8)
respond(status, body: body, type: "application/json")
}
func close() {
conn.cancel()
retained = nil
}
/// Completes the WebSocket handshake and hands the connection over.
func upgrade(_ req: Request) -> WebSocket? {
guard req.headers["upgrade"]?.lowercased() == "websocket", let key = req.headers["sec-websocket-key"] else {
respond(400, text: "expected a WebSocket upgrade")
return nil
}
let accept = Data(Insecure.SHA1.hash(data: Data((key + "258EAFA5-E914-47DA-95CA-C5AB0DC85B11").utf8))).base64EncodedString()
let head = "HTTP/1.1 101 Switching Protocols\r\nUpgrade: websocket\r\nConnection: Upgrade\r\nSec-WebSocket-Accept: \(accept)\r\n\r\n"
conn.send(content: Data(head.utf8), completion: .contentProcessed { _ in })
let ws = WebSocket(conn, queue: queue)
retained = nil
return ws
}
}
/// Server side of RFC 6455. Sends are counted until the network stack has
/// taken them, so the stream can skip frames instead of queueing seconds of
/// video on a slow link.
final class WebSocket {
static let maxMessage: UInt64 = 1 << 20
let conn: NWConnection
let queue: DispatchQueue
var onText: ((String) -> Void)?
var onClose: (() -> Void)?
private var buffer = Data()
private var fragments = Data()
private var fragmentOpcode: UInt8 = 0
private var closed = false
private var retained: WebSocket?
private let lock = NSLock()
private var _pending = 0
var onDrain: (() -> Void)?
/// Bytes handed to the connection that it hasn't sent yet. Any thread.
var pendingBytes: Int { lock.lock(); defer { lock.unlock() }; return _pending }
init(_ conn: NWConnection, queue: DispatchQueue) {
self.conn = conn
self.queue = queue
}
func start() {
retained = self
read()
}
func sendText(_ s: String) { send(opcode: 1, Data(s.utf8)) }
func sendJSON(_ obj: Any) {
if let d = try? JSONSerialization.data(withJSONObject: obj), let s = String(data: d, encoding: .utf8) { sendText(s) }
}
/// `taken` runs once the network stack has taken the whole frame.
func sendBinary(_ d: Data, taken: (() -> Void)? = nil) { send(opcode: 2, d, taken: taken) }
func send(opcode: UInt8, _ payload: Data, taken: (() -> Void)? = nil) {
var frame = Data([0x80 | opcode])
let n = payload.count
if n < 126 {
frame.append(UInt8(n))
} else if n < 65536 {
frame.append(126)
frame.append(contentsOf: [UInt8(n >> 8), UInt8(n & 0xff)])
} else {
frame.append(127)
for shift in stride(from: 56, through: 0, by: -8) { frame.append(UInt8((UInt64(n) >> UInt64(shift)) & 0xff)) }
}
frame.append(payload)
let size = frame.count
lock.lock(); _pending += size; lock.unlock()
conn.send(content: frame, completion: .contentProcessed { [weak self] _ in
guard let self else { return }
self.lock.lock(); self._pending -= size; self.lock.unlock()
taken?()
self.onDrain?()
})
}
func close() {
guard !closed else { return }
closed = true
send(opcode: 8, Data([0x03, 0xe8])) // 1000, normal closure
conn.send(content: nil, isComplete: true, completion: .contentProcessed { [weak self] _ in self?.conn.cancel() })
finish()
}
private func finish() {
let cb = onClose
onClose = nil
onText = nil
onDrain = nil
cb?()
retained = nil
}
private func read() {
conn.receive(minimumIncompleteLength: 1, maximumLength: 65536) { [weak self] data, _, done, error in
guard let self, !self.closed else { return }
if let data { self.buffer.append(data) }
self.parse()
if error != nil || done {
self.closed = true
self.conn.cancel()
self.finish()
} else if !self.closed {
self.read()
}
}
}
private func parse() {
while buffer.count >= 2 {
let b = [UInt8](buffer.prefix(14))
let fin = b[0] & 0x80 != 0, opcode = b[0] & 0x0f, masked = b[1] & 0x80 != 0
// Lengths are unsigned and clients only send small control
// messages, so anything big (or a 64-bit length with the top bit
// set) is refused before it's turned into an Int.
var len64 = UInt64(b[1] & 0x7f), off = 2
if len64 == 126 {
guard b.count >= 4 else { return }
len64 = UInt64(b[2]) << 8 | UInt64(b[3]); off = 4
} else if len64 == 127 {
guard b.count >= 10 else { return }
len64 = 0
for i in 2..<10 { len64 = len64 << 8 | UInt64(b[i]) }
off = 10
}
guard len64 <= WebSocket.maxMessage else { return close() }
let len = Int(len64)
let maskOff = off
if masked { off += 4 }
guard buffer.count >= off + len else { return }
let start = buffer.startIndex
var payload = Data(buffer[(start + off)..<(start + off + len)])
if masked {
let mask = [UInt8](buffer[(start + maskOff)..<(start + maskOff + 4)])
payload.withUnsafeMutableBytes { p in
for i in 0..<len { p[i] ^= mask[i & 3] }
}
}
buffer.removeFirst(off + len)
switch opcode {
case 0, 1, 2:
if opcode != 0 { fragmentOpcode = opcode; fragments = Data() }
guard fragments.count + payload.count <= Int(WebSocket.maxMessage) else { return close() }
fragments.append(payload)
if fin, fragmentOpcode == 1, let s = String(data: fragments, encoding: .utf8) { onText?(s) }
case 8: close(); return
case 9: send(opcode: 10, payload)
default: break
}
}
}
}
mac/frame-mac-view/Sources/Stats.swift
+180
View File
@@ -0,0 +1,180 @@
// Per-frame timing, so every change to the stream can be judged by numbers.
// All times are the agent's host clock in microseconds (the clock
// ScreenCaptureKit stamps frames with). The viewer syncs to it over the
// WebSocket and reports when each frame arrived, decoded and was drawn.
import CoreMedia
import Foundation
private let timebase: mach_timebase_info_data_t = {
var t = mach_timebase_info_data_t()
mach_timebase_info(&t)
return t
}()
/// Now on the host clock, in microseconds.
func nowUs() -> Int64 { hostUs(mach_absolute_time()) }
/// A mach_absolute_time value (as in SCStreamFrameInfo.displayTime) in microseconds.
func hostUs(_ ticks: UInt64) -> Int64 { Int64(ticks / 1000 * UInt64(timebase.numer) / UInt64(timebase.denom)) }
/// A host-clock CMTime (ScreenCaptureKit's presentation times) in microseconds.
func hostUs(_ t: CMTime) -> Int64 { t.isValid ? Int64(CMTimeGetSeconds(t) * 1_000_000) : nowUs() }
/// One frame's journey. Zero means "didn't happen" or "not reported yet".
struct FrameRecord {
var seq: UInt32 = 0
var key = false
var bytes = 0
var width = 0, height = 0
var capture: Int64 = 0 // the Mac composited it (display time)
var arrived: Int64 = 0 // ScreenCaptureKit handed it to us
var encodeStart: Int64 = 0
var encodeEnd: Int64 = 0
var sent: Int64 = 0 // handed to the WebSocket
var wire: Int64 = 0 // the socket took it
var received: Int64 = 0 // viewer, agent clock
var decoded: Int64 = 0
var drawn: Int64 = 0
var vsync: Int64 = 0 // the viewer's next animation frame after drawing it
var echo: UInt32 = 0 // the first frame after input `echo`
var tier = 0
var bitrate = 0
var json: [String: Any] {
["s": seq, "k": key ? 1 : 0, "b": bytes, "w": width, "h": height, "cap": capture, "arr": arrived,
"e0": encodeStart, "e1": encodeEnd, "snd": sent, "wire": wire, "rx": received, "dec": decoded,
"drw": drawn, "vs": vsync, "echo": echo, "tier": tier, "br": bitrate]
}
}
/// One input event from the viewer and what came of it.
struct InputRecord {
var id: UInt32
var kind: String
var viewer: Int64 // the viewer's event time, agent clock
var injected: Int64 = 0 // posted as a CGEvent
var frame: UInt32 = 0 // first frame captured after it (0: none yet)
var capture: Int64 = 0
var json: [String: Any] {
["id": id, "kind": kind, "tv": viewer, "inj": injected, "frame": frame, "cap": capture]
}
}
func percentile(_ sorted: [Double], _ p: Double) -> Double? {
guard !sorted.isEmpty else { return nil }
let i = min(sorted.count - 1, max(0, Int((p * Double(sorted.count - 1)).rounded())))
return sorted[i]
}
/// Frames and inputs of one stream, kept for the last few thousand frames.
/// Any thread.
final class StreamStats {
static let capacity = 4096
private let lock = NSLock()
private var frames = [FrameRecord?](repeating: nil, count: StreamStats.capacity)
private var inputs: [UInt32: InputRecord] = [:]
private var inputOrder: [UInt32] = []
private(set) var nextSeq: UInt32 = 1
var captured = 0 // frames ScreenCaptureKit delivered
var skipped = 0 // held back from encoding (link, encoder, pacing or gate busy); only the newest may go later
var viewerDropped = 0 // not shown by the viewer (behind, or waiting for a keyframe)
var rtt: Double = 0 // ms, the viewer's best recent ping
var clockSynced = false
var decoder = "" // what the viewer reports about its decoder
func withLock<T>(_ f: () -> T) -> T { lock.lock(); defer { lock.unlock() }; return f() }
func newFrame(_ r: FrameRecord) -> UInt32 {
withLock {
var r = r
r.seq = nextSeq
nextSeq &+= 1
frames[Int(r.seq) % StreamStats.capacity] = r
return r.seq
}
}
func update(_ seq: UInt32, _ f: (inout FrameRecord) -> Void) {
withLock {
let i = Int(seq) % StreamStats.capacity
guard var r = frames[i], r.seq == seq else { return }
f(&r)
frames[i] = r
}
}
func frame(_ seq: UInt32) -> FrameRecord? {
withLock {
let r = frames[Int(seq) % StreamStats.capacity]
return r?.seq == seq ? r : nil
}
}
func addInput(_ r: InputRecord) {
withLock {
inputs[r.id] = r
inputOrder.append(r.id)
if inputOrder.count > 512 { inputs[inputOrder.removeFirst()] = nil }
}
}
func updateInput(_ id: UInt32, _ f: (inout InputRecord) -> Void) {
withLock { if var r = inputs[id] { f(&r); inputs[id] = r } }
}
/// Frames with seq > `since` that were sent at least `settle` µs ago, so
/// the viewer has had time to report on them.
func settled(since: UInt32, settle: Int64 = 1_500_000) -> [FrameRecord] {
let cutoff = nowUs() - settle
return withLock {
frames.compactMap { $0 }.filter { $0.seq > since && $0.sent > 0 && $0.sent < cutoff }.sorted { $0.seq < $1.seq }
}
}
func inputList() -> [InputRecord] { withLock { inputOrder.compactMap { inputs[$0] } } }
/// Percentiles over the last `window` µs, for /status and the overlay.
func summary(window: Int64 = 2_000_000) -> [String: Any] {
let now = nowUs(), from = now - window
let recent = withLock { frames.compactMap { $0 }.filter { $0.sent > from } }
func ms(_ pick: (FrameRecord) -> (Int64, Int64)) -> [String: Double] {
let v = recent.compactMap { r -> Double? in
let (a, b) = pick(r)
return a > 0 && b > 0 ? Double(b - a) / 1000 : nil
}.sorted()
guard !v.isEmpty else { return [:] }
return ["p50": (percentile(v, 0.5)! * 10).rounded() / 10, "p95": (percentile(v, 0.95)! * 10).rounded() / 10]
}
let secs = Double(window) / 1_000_000
let shown = recent.filter { $0.vsync > 0 }.count
let bytes = recent.reduce(0) { $0 + $1.bytes }
var s: [String: Any] = [
"capture": ms { ($0.capture, $0.arrived) },
"queue": ms { ($0.arrived, $0.encodeStart) },
"encode": ms { ($0.encodeStart, $0.encodeEnd) },
"network": ms { ($0.encodeEnd, $0.received) },
"decode": ms { ($0.received, $0.decoded) },
"draw": ms { ($0.decoded, $0.vsync) },
"total": ms { ($0.capture, $0.vsync) },
"fps": (Double(shown) / secs * 10).rounded() / 10,
"sentFps": (Double(recent.count) / secs * 10).rounded() / 10,
"mbps": (Double(bytes * 8) / secs / 1e5).rounded() / 10,
"rtt": (rtt * 10).rounded() / 10,
"synced": clockSynced,
]
withLock {
s["captured"] = captured
s["skipped"] = skipped
s["dropped"] = viewerDropped
s["decoder"] = decoder
let done = inputOrder.suffix(20).compactMap { inputs[$0] }.compactMap { i -> Double? in
guard i.frame != 0, let f = frames[Int(i.frame) % StreamStats.capacity], f.seq == i.frame, f.vsync > 0
else { return nil }
return Double(f.vsync - i.viewer) / 1000
}.sorted()
if !done.isEmpty { s["input"] = ["p50": percentile(done, 0.5)!, "n": done.count] }
}
return s
}
}
mac/frame-mac-view/Sources/main.swift
+755
View File
@@ -0,0 +1,755 @@
// frame-mac-view: streams Mac windows or displays to viewers on the Steam
// Frame and plays their pointer and key input back on the Mac.
//
// frame-mac-view serve --port 47811 --page ui/mac-view.html (token in FRAME_MAC_VIEW_TOKEN)
// frame-mac-view windows | displays | permissions (JSON on stdout)
// frame-mac-view request-permissions (shows macOS's prompts)
//
// It listens on 127.0.0.1 only. Frame Control reaches it from the Frame
// through an SSH reverse tunnel, and every request must carry the token.
//
// HTTP. Frame Control's key (?k=, from FRAME_MAC_VIEW_TOKEN) never leaves the
// Mac; the Frame gets a single-use ticket per viewer instead.
// GET /ping, /view open: tunnel check, viewer page
// GET /stream?src=...&t=TICKET|r=KEY WebSocket (&codec=h264|jpeg&max=&fps=&bpp=)
// GET /status, /windows, /displays ?k=: permissions, streams, sources
// POST /ticket?src=... ?k=: a ticket for one viewer of src
// POST /close[?src=...] ?k=: end those streams, close their windows
// POST /permissions ?k=: show macOS's permission prompts
// GET /stats[?id=&since=] ?k=: per-frame timing records (for benchmarks)
// POST /bench?src=&action=... ?k=: ask viewers of src to type, click or show their overlay
// src is window:<CGWindowID>, display:<CGDirectDisplayID> or test.
import AppKit
import ApplicationServices
import Foundation
import IOKit.pwr_mgt
import ScreenCaptureKit
import Security
let version = "1"
func windowsJSON() -> [[String: Any]] {
let me = ProcessInfo.processInfo.processIdentifier
let skip: Set<String> = ["Window Server", "Dock", "Control Centre", "Control Center", "Notification Centre",
"Notification Center", "Spotlight", "SystemUIServer", "Wallpaper", "WindowManager"]
return WindowInfo.all().filter {
$0.layer == 0 && $0.pid != me && !skip.contains($0.app) && $0.bounds.width >= 120 && $0.bounds.height >= 80
}.map {
["id": $0.id, "src": "window:\($0.id)", "pid": $0.pid, "app": $0.app, "title": $0.title,
"w": Int($0.bounds.width), "h": Int($0.bounds.height)]
}
}
func displaysJSON() -> [[String: Any]] {
var ids = [CGDirectDisplayID](repeating: 0, count: 16)
var n: UInt32 = 0
// Online, not active: a display that's asleep is still one you can stream.
CGGetOnlineDisplayList(16, &ids, &n)
return ids.prefix(Int(n)).filter {
CGDisplayMirrorsDisplay($0) == kCGNullDirectDisplay && !VirtualDisplay.isOurs($0) // not a separated window's
}.map { id in
let b = CGDisplayBounds(id)
return ["id": id, "src": "display:\(id)", "name": displayName(id), "w": Int(b.width), "h": Int(b.height),
"main": CGDisplayIsMain(id) != 0]
}
}
func permissionsJSON() -> [String: Any] {
["screen": CGPreflightScreenCaptureAccess(), "accessibility": AXIsProcessTrusted()]
}
func printJSON(_ obj: Any) {
let d = (try? JSONSerialization.data(withJSONObject: obj, options: [.sortedKeys, .prettyPrinted])) ?? Data()
FileHandle.standardOutput.write(d + Data("\n".utf8))
}
/// A number from a JSON message, whatever its JSON type.
func num(_ v: Any?) -> Double? { (v as? NSNumber)?.doubleValue }
/// One viewer watching one source.
final class Session {
let id: Int
let source: Source
let capture: CaptureSource
let encoder: Encoder
let ws: WebSocket
let codec: Codec
let reconnectKey: String
let stats = StreamStats()
let controller: RateController
private var appliedScale = 1.0
private var lastSubmit: Int64 = 0
private var paceScheduled = false
private let lock = NSLock()
/// A captured picture waiting to be encoded (or the last one encoded).
private struct Picture {
let pb: CVPixelBuffer
let pts: CMTime
let capture: Int64
let arrived: Int64
var echo: UInt32 // the input this picture is the first reply to
}
private var last: Picture?
private var skipped = false
private var stopped = false
private var inFlight = 0
/// Input posted to the Mac whose effect hasn't been captured yet: the
/// next picture composited after `after` is tagged with it.
private var pendingEcho: (id: UInt32, after: Int64)?
private var statsTimer: DispatchSourceTimer?
/// Frames already queued for the network; beyond this, or with two frames
/// already in the encoder, new frames are skipped (before encoding, so no
/// reference frame goes missing) and the newest picture is sent once
/// things catch up. Only that newest picture is kept meanwhile.
let maxPending: Int
static let maxInFlight = 2
var onEnd: ((Session) -> Void)?
var onAck: (() -> Void)?
var onFinished: ((String) -> Void)?
private let encodeQueue = DispatchQueue(label: "frame-mac-view.encode", qos: .userInteractive)
init(id: Int, source: Source, capture: CaptureSource, ws: WebSocket, codec: Codec, fps: Int, bitsPerPixel: Double,
reconnectKey: String) {
self.id = id
self.source = source
self.capture = capture
self.ws = ws
self.codec = codec
self.reconnectKey = reconnectKey
encoder = Encoder(codec: codec, fps: fps, bitsPerPixel: bitsPerPixel)
controller = RateController(maxFps: fps)
maxPending = codec == .jpeg ? 3 << 20 : 1 << 20
}
/// Frames come faster than the current tier's frame rate: this one waits,
/// and goes when its turn comes (unless a newer one replaces it).
/// Call with `lock` held.
private func paced(now: Int64) -> Bool {
let fps = controller.fps
guard fps < controller.maxFps, lastSubmit > 0 else { return false }
let interval = Int64(1_000_000 / fps), due = lastSubmit + interval * 9 / 10
guard now < due else { return false }
if !paceScheduled {
paceScheduled = true
encodeQueue.asyncAfter(deadline: .now() + .microseconds(Int(due - now))) { [weak self] in
guard let self else { return }
self.lock.lock(); self.paceScheduled = false; self.lock.unlock()
self.resendIfRoom()
}
}
return true
}
/// Encodes `pb` unless the link or the encoder is busy, in which case it
/// becomes the picture to send next.
private func offer(_ pb: CVPixelBuffer, pts: CMTime, capture shown: Int64) {
let arrived = nowUs()
stats.withLock { stats.captured += 1 }
controller.captured(at: arrived)
lock.lock()
// A reply to input stays with whichever picture ends up being sent.
var echo = skipped ? last?.echo ?? 0 : 0
if let p = pendingEcho, shown >= p.after {
echo = p.id
pendingEcho = nil
}
let picture = Picture(pb: pb, pts: pts, capture: shown, arrived: arrived, echo: echo)
last = picture
let busy = stopped || ws.pendingBytes > maxPending || inFlight >= Session.maxInFlight
|| paced(now: arrived) || !controller.maySend(now: arrived)
if busy { skipped = true } else { inFlight += 1; last?.echo = 0; lastSubmit = arrived }
lock.unlock()
if busy { stats.withLock { stats.skipped += 1 } } else { submit(picture) }
}
private func submit(_ p: Picture) {
encodeQueue.async {
var r = FrameRecord()
r.capture = p.capture
r.arrived = p.arrived
r.echo = p.echo
r.bitrate = self.encoder.bitrate
r.tier = self.controller.tier
r.encodeStart = nowUs()
let seq = self.stats.newFrame(r)
if p.echo != 0 { self.stats.updateInput(p.echo) { $0.frame = seq; $0.capture = p.capture } }
if !self.encoder.encode(p.pb, pts: p.pts, seq: seq) { self.finished() }
}
}
/// An encode finished (or failed): send the newest skipped picture if
/// there's room now.
private func finished() {
lock.lock()
inFlight = max(0, inFlight - 1)
lock.unlock()
resendIfRoom()
}
private func resendIfRoom() {
lock.lock()
var next: Picture?
let now = nowUs()
if !stopped, skipped, ws.pendingBytes <= maxPending / 2, inFlight < Session.maxInFlight, let l = last,
!paced(now: now), controller.maySend(now: now, counts: false) {
next = l
skipped = false
inFlight += 1
last?.echo = 0
lastSubmit = now
}
lock.unlock()
// Stamped now: VideoToolbox needs increasing times, and the capture
// time stays in the record, so the wait counts as latency.
if let p = next {
submit(Picture(pb: p.pb, pts: CMClockGetTime(CMClockGetHostTimeClock()), capture: p.capture,
arrived: p.arrived, echo: p.echo))
}
}
func start(maxLong: Int, fps: Int) {
encoder.onFrame = { [weak self] data, key, pts, seq in
guard let self else { return }
let t = nowUs()
// The quality setting's bitrate, at full size, is the most it gets.
if self.appliedScale == 1 {
self.controller.setCeiling(self.encoder.defaultBitrate(width: self.encoder.width, height: self.encoder.height))
}
self.controller.sent(seq: seq, bytes: data.count + 17, at: t)
var echo: UInt32 = 0
let (w, h) = (self.encoder.width, self.encoder.height)
self.stats.update(seq) {
$0.encodeEnd = t
$0.sent = t
$0.bytes = data.count
$0.key = key
$0.width = w
$0.height = h
echo = $0.echo
}
// Header: flags (1 = keyframe), pts µs, sequence number, and the
// input this frame is the first reply to; all big-endian.
var msg = Data(capacity: data.count + 17)
msg.append(key ? 1 : 0)
var us = UInt64(max(0, CMTimeGetSeconds(pts)) * 1_000_000).bigEndian
msg.append(Data(bytes: &us, count: 8))
var s = seq.bigEndian, e = echo.bigEndian
msg.append(Data(bytes: &s, count: 4))
msg.append(Data(bytes: &e, count: 4))
msg.append(data)
let stats = self.stats
self.ws.sendBinary(msg) { stats.update(seq) { $0.wire = nowUs() } }
}
encoder.onDone = { [weak self] _ in self?.finished() }
encoder.onError = { [weak self] message in self?.ws.sendJSON(["t": "error", "message": message]) }
capture.onFrame = { [weak self] pb, pts, shown in self?.offer(pb, pts: pts, capture: shown) }
capture.onEnded = { [weak self] reason in
guard let self else { return }
self.ws.sendJSON(["t": "closed", "reason": reason])
self.onFinished?(self.source.key)
self.end()
}
ws.onDrain = { [weak self] in self?.resendIfRoom() }
ws.onText = { [weak self] text in self?.handle(text) }
ws.onClose = { [weak self] in self?.end() }
ws.start()
// Reconnect with this (kept in the page's memory, never on a command line).
ws.sendJSON(["t": "hello", "r": reconnectKey])
capture.onChange = { [weak self] in self?.sendInfo() }
// The numbers, for the viewer's overlay.
// Adapt ten times a second; the numbers go to the viewer once a second.
let t = DispatchSource.makeTimerSource(queue: encodeQueue)
t.schedule(deadline: .now() + 0.1, repeating: 0.1)
var ticks = 0
t.setEventHandler { [weak self] in
guard let self, !self.isStopped else { return }
self.adapt(maxLong: maxLong)
ticks += 1
guard ticks % 10 == 0 else { return }
var s = self.stats.summary()
s.merge(self.controller.state()) { a, _ in a }
s["t"] = "stats"
s["bitrate"] = self.encoder.bitrate
s["size"] = "\(self.encoder.width)×\(self.encoder.height)"
self.ws.sendJSON(s)
}
t.resume()
statsTimer = t
capture.start(maxLong: maxLong, fps: fps) { [weak self] error in
guard let self else { return }
if let error {
if self.capture.gone {
// Final, like a window closing mid-stream: the viewer closes
// and its key is revoked, rather than retrying for ever.
self.ws.sendJSON(["t": "closed", "reason": error])
self.onFinished?(self.source.key)
} else {
self.ws.sendJSON(["t": "error", "message": error])
}
self.end()
return
}
self.sendInfo()
}
}
/// Applies the controller's bitrate and tier. On the encoding queue.
private func adapt(maxLong: Int) {
guard let bps = controller.update(now: nowUs()) else { return }
encoder.setBitrate(bps)
let scale = controller.scale
if scale != appliedScale {
appliedScale = scale
capture.setMaxLong(max(320, Int(Double(maxLong) * scale)))
}
resendIfRoom() // a lower tier may let a held frame go now
}
/// While someone is typing or clicking, the viewer sends a tiny message this
/// often (ms, 0 = off), so the Frame's Wi-Fi doesn't doze between the input
/// and the frame that answers it (power saving is on there).
static let warmMs = Int(ProcessInfo.processInfo.environment["FRAME_MAC_VIEW_WARM"] ?? "") ?? 0
func sendInfo() {
ws.sendJSON(["t": "info", "src": source.key, "title": capture.title, "app": capture.app, "codec": codec.rawValue,
"input": source == .test || Input.allowed, "warm": Session.warmMs,
"aspect": Double(capture.frameRect.width / max(capture.frameRect.height, 1))])
}
var isStopped: Bool { lock.lock(); defer { lock.unlock() }; return stopped }
func end() {
lock.lock()
let was = stopped
stopped = true
last = nil
lock.unlock()
guard !was else { return }
statsTimer?.cancel()
capture.stop()
encodeQueue.async { self.encoder.invalidate() }
let owner = id
DispatchQueue.main.async { Input.releaseAll(owner: owner) }
ws.close()
onEnd?(self)
}
/// A point in the picture (0...1 each way) -> Mac global coordinates.
private func point(_ x: Double, _ y: Double) -> CGPoint {
let r = capture.frameRect
return CGPoint(x: r.minX + min(max(x, 0), 1) * r.width, y: r.minY + min(max(y, 0), 1) * r.height)
}
/// Input with an id (from the viewer) has been posted: the next picture
/// the Mac composites is its first chance to show.
private func injected(_ m: [String: Any], kind: String) {
guard let iid = (m["i"] as? NSNumber)?.uint32Value, iid != 0 else { return }
let now = nowUs()
stats.addInput(InputRecord(id: iid, kind: kind, viewer: Int64(num(m["tv"]) ?? 0), injected: now))
lock.lock(); pendingEcho = (iid, now); lock.unlock()
}
/// Timing reports from the viewer, in the agent's clock.
private func report(_ t: String, _ m: [String: Any]) -> Bool {
switch t {
case "ping":
ws.sendJSON(["t": "pong", "c": m["c"] ?? 0, "a": nowUs()])
case "rx":
guard let s = (m["s"] as? NSNumber)?.uint32Value else { break }
if let r = num(m["r"]) { stats.update(s) { $0.received = Int64(r) } }
if controller.acked(seq: s, at: nowUs()) { resendIfRoom() }
case "fd":
for f in m["f"] as? [[NSNumber]] ?? [] where f.count >= 4 {
stats.update(f[0].uint32Value) {
$0.decoded = f[1].int64Value
$0.drawn = f[2].int64Value
$0.vsync = f[3].int64Value
}
}
let dropped = Int(num(m["drop"]) ?? 0)
stats.withLock { stats.viewerDropped += dropped }
case "w":
break // keep-warm filler, see sendInfo
case "clock":
stats.withLock {
stats.rtt = num(m["rtt"]) ?? 0
stats.clockSynced = true
if let d = m["dec"] as? String { stats.decoder = d }
}
default:
return false
}
return true
}
/// Asks the viewer to do something for a benchmark (type, click, show its overlay).
func bench(_ m: [String: Any]) { ws.sendJSON(m.merging(["t": "bench"]) { a, _ in a }) }
private func handle(_ text: String) {
guard !isStopped, let d = text.data(using: .utf8),
let m = try? JSONSerialization.jsonObject(with: d) as? [String: Any], let t = m["t"] as? String else { return }
if report(t, m) { return }
let x = m["x"] as? Double ?? -1, y = m["y"] as? Double ?? -1
if t == "ack" {
onAck?()
onAck = nil
return
}
if t == "key-frame" {
encodeQueue.async { self.encoder.requestKeyFrame() } // before the resend, same queue
lock.lock(); skipped = last != nil; lock.unlock()
resendIfRoom()
return
}
if source == .test {
let desc: String?
switch t {
case "m": desc = (m["e"] as? String) == "move" ? nil : "mouse \(m["e"] ?? "") button \(m["b"] ?? 0)"
case "wheel": desc = "wheel \(m["dx"] ?? 0), \(m["dy"] ?? 0)"
case "k": desc = (m["e"] as? String) == "down" ? "key \(m["code"] ?? "") \"\(m["key"] ?? "")\"" : nil
case "text": desc = "text \"\(m["s"] ?? "")\""
default: desc = nil
}
capture.pointer(x: x, y: y, text: desc)
if desc != nil { injected(m, kind: t) }
return
}
DispatchQueue.main.async { [self] in
guard !isStopped else { return } // Stop revokes input at once
switch t {
case "m":
let kind = m["e"] as? String ?? "move", b = m["b"] as? Int ?? 0
let p = point(x, y)
if kind == "down", case .window(let wid) = source, let pid = capture.pid {
Input.focus(window: wid, pid: pid)
}
Input.mouse(kind, button: b, at: p, owner: id)
if kind == "down" { injected(m, kind: "click") }
case "wheel":
Input.scroll(dx: m["dx"] as? Double ?? 0, dy: m["dy"] as? Double ?? 0, at: point(x, y), owner: id)
injected(m, kind: "wheel")
case "k":
let down = (m["e"] as? String) == "down"
Input.key(code: m["code"] as? String ?? "", key: m["key"] as? String ?? "",
down: down, mods: m["mods"] as? [String] ?? [], owner: id)
if down { injected(m, kind: "key") }
case "text":
if let s = m["s"] as? String, s.count <= 4096 { Input.text(s); injected(m, kind: "text") }
case "release":
Input.releaseAll(owner: id)
case "focus":
if case .window(let wid) = source, let pid = capture.pid { Input.focus(window: wid, pid: pid) }
default: break
}
}
}
}
func randomKey() -> String {
var bytes = [UInt8](repeating: 0, count: 24)
_ = SecRandomCopyBytes(kSecRandomDefault, bytes.count, &bytes)
return Data(bytes).base64EncodedString().replacingOccurrences(of: "+", with: "-")
.replacingOccurrences(of: "/", with: "_").replacingOccurrences(of: "=", with: "")
}
func sameSecret(_ a: String, _ b: String) -> Bool {
guard !a.isEmpty, a.utf8.count == b.utf8.count else { return false }
return zip(a.utf8, b.utf8).reduce(0, { $0 | ($1.0 ^ $1.1) }) == 0
}
final class Agent {
/// Frame Control's own key. It stays on the Mac: the Frame only ever sees
/// single-use tickets and per-stream reconnect keys, both tied to one source.
let token: String
let page: URL?
var sessions: [Int: Session] = [:] { didSet { keepDisplayAwake(!sessions.isEmpty) } }
var nextId = 1
let lock = NSLock()
private var assertion: IOPMAssertionID = 0
/// ticket -> (source, expiry, reconnect key once redeemed). A ticket opens
/// one viewer within a minute; it may be redeemed again, for the same key,
/// only until that viewer confirms it has the key ("ack").
private var tickets: [String: (src: String, expiry: Date, key: String?)] = [:]
/// reconnect key -> source, until Stop for that source.
private var reconnectKeys: [String: String] = [:]
/// Sources that ended for good (the window closed), for Frame Control.
private var finished = Set<String>()
/// Ends every stream, then exits once separated windows are back on the
/// Mac's own screens (they're moved back, then their displays go 0.3 s later).
/// Idempotent: SIGTERM and the parent going away can both call it, and a
/// session already ended by /close may still be putting its window back.
private var quitting = false
func quit() {
lock.lock()
let all = Array(sessions.values), again = quitting
quitting = true
lock.unlock()
guard !again else { return }
for s in all { s.ws.sendJSON(["t": "close"]); s.end() }
DispatchQueue.main.asyncAfter(deadline: .now() + 0.8) { exit(0) }
}
/// While anyone watches, keep the Mac's display on: a sleeping display
/// stops being drawn, so there'd be nothing to capture (and it would lock).
private func keepDisplayAwake(_ on: Bool) {
if on, assertion == 0 {
IOPMAssertionCreateWithName(kIOPMAssertionTypePreventUserIdleDisplaySleep as CFString,
IOPMAssertionLevel(kIOPMAssertionLevelOn),
"Frame Control is showing this Mac in a Steam Frame" as CFString, &assertion)
} else if !on, assertion != 0 {
IOPMAssertionRelease(assertion)
assertion = 0
}
}
init(token: String, page: URL?) {
self.token = token
self.page = page
}
/// Whether this request may open a stream of `src`: Frame Control's key,
/// an unused ticket for that source, or a live reconnect key for it.
private func mayStream(_ req: Request, src: String) -> String? {
lock.lock()
defer { lock.unlock() }
let now = Date()
tickets = tickets.filter { $0.value.expiry > now }
if sameSecret(req.query["k"] ?? "", token) { return randomKey() }
if let t = req.query["t"], let entry = tickets[t], entry.src == src {
// A retry before the viewer got its key gets the same key back.
let key = entry.key ?? randomKey()
tickets[t] = (src, entry.expiry, key)
reconnectKeys[key] = src
return key
}
if let r = req.query["r"], reconnectKeys[r] == src { return r }
return nil
}
/// The viewer has its reconnect key, so the ticket that led to it can't
/// be used again (also when the ack comes over a reconnection).
func acknowledged(key: String) {
lock.lock(); tickets = tickets.filter { $0.value.key != key }; lock.unlock()
}
func handle(_ req: Request, _ c: HTTPConnection) {
// Open to anything that reaches the port: a liveness check for the
// tunnel, and the viewer page, which holds no secrets.
switch (req.method, req.path) {
case ("GET", "/ping"): return c.respond(200, text: "frame-mac-view")
case ("GET", "/view"):
guard let page, let body = try? Data(contentsOf: page) else { return c.respond(404, text: "no viewer page") }
return c.respond(200, body: body, type: "text/html; charset=utf-8")
case ("GET", "/stream"):
guard let src = Source(req.query["src"] ?? "") else { return c.respond(400, text: "bad src") }
guard let key = mayStream(req, src: src.key) else { return c.respond(403, text: "forbidden") }
return stream(req, c, src: src, key: key)
default: break
}
guard sameSecret(req.query["k"] ?? req.headers["x-token"] ?? "", token) else {
return c.respond(403, text: "forbidden")
}
switch (req.method, req.path) {
case ("GET", "/status"):
lock.lock()
let list = sessions.values.map { s -> [String: Any] in
var e: [String: Any] = ["id": s.id, "src": s.source.key, "title": s.capture.title, "app": s.capture.app,
"stats": s.stats.summary(), "bitrate": s.encoder.bitrate,
"controller": s.controller.state()]
if let d = (s.capture as? SeparateSource)?.displayID { e["display"] = d }
return e
}
lock.unlock()
var s = permissionsJSON()
s["version"] = version
s["streams"] = list
lock.lock(); s["finished"] = Array(finished); lock.unlock()
c.respond(json: s)
case ("GET", "/windows"):
c.respond(json: ["windows": windowsJSON(), "screen": CGPreflightScreenCaptureAccess()])
case ("GET", "/displays"):
c.respond(json: ["displays": displaysJSON()])
case ("POST", "/ticket"):
guard let src = Source(req.query["src"] ?? "") else { return c.respond(400, text: "bad src") }
let t = randomKey()
lock.lock()
tickets[t] = (src.key, Date().addingTimeInterval(60), nil)
finished.remove(src.key)
lock.unlock()
c.respond(json: ["ticket": t])
case ("POST", "/close"):
// Tell viewers to close their windows, but don't rely on them:
// the sessions end here and can't reconnect.
let src = req.query["src"]
lock.lock()
let matching = sessions.values.filter { src == nil || $0.source.key == src }
reconnectKeys = reconnectKeys.filter { src != nil && $0.value != src }
tickets = tickets.filter { src != nil && $0.value.src != src } // not yet used ones too
lock.unlock()
for s in matching { s.ws.sendJSON(["t": "close"]) }
DispatchQueue.global().asyncAfter(deadline: .now() + 0.3) { for s in matching { s.end() } }
c.respond(json: ["closed": matching.count])
case ("GET", "/stats"):
// Frames the viewer has had time to report on, oldest first.
let since = UInt32(req.query["since"] ?? "") ?? 0
let settle = Int64(req.query["settle"] ?? "") ?? 1_500_000
lock.lock()
let list = sessions.values.filter { req.query["id"] == nil || "\($0.id)" == req.query["id"] }
lock.unlock()
c.respond(json: ["now": nowUs(), "streams": list.map { s -> [String: Any] in
["id": s.id, "src": s.source.key, "frames": s.stats.settled(since: since, settle: settle).map(\.json),
"inputs": s.stats.inputList().map(\.json), "summary": s.stats.summary(),
"captured": s.stats.withLock { s.stats.captured }, "controller": s.controller.state(),
"events": s.controller.eventList()]
}])
case ("POST", "/bench"):
lock.lock()
let list = sessions.values.filter { $0.source.key == req.query["src"] }
lock.unlock()
var m: [String: Any] = [:]
for (k, v) in req.query where k != "k" && k != "src" { m[k] = Double(v) ?? v as Any }
for s in list { s.bench(m) }
c.respond(json: ["sent": list.count])
case ("GET", "/snapshot"):
// One JPEG of a display (for checks and thumbnails): ?display=ID
guard let id = UInt32(req.query["display"] ?? "") else { return c.respond(400, text: "display=ID") }
snapshot(display: id) { data, error in
if let data { c.respond(200, body: data, type: "image/jpeg") } else { c.respond(500, text: error ?? "failed") }
}
case ("POST", "/permissions"):
DispatchQueue.main.async { requestPermissions() }
c.respond(json: permissionsJSON())
default:
c.respond(404, text: "not found")
}
}
private func stream(_ req: Request, _ c: HTTPConnection, src: Source, key: String) {
let codec = Codec(rawValue: req.query["codec"] ?? "h264") ?? .h264
let maxLong = min(max(Int(req.query["max"] ?? "") ?? 1920, 320), 3840)
let fps = min(max(Int(req.query["fps"] ?? "") ?? 60, 5), 120)
let bpp = min(max(Double(req.query["bpp"] ?? "") ?? 0.1, 0.02), 0.5)
guard let ws = c.upgrade(req) else { return }
let capture: CaptureSource
switch src {
case .test: capture = TestSource()
case .separate(let id): capture = SeparateSource(windowID: id)
default: capture = SCKSource(src)
}
lock.lock()
// One live viewer per key: a reconnection (or a second use of an
// unacknowledged ticket) replaces the one before.
let replaced = sessions.values.filter { $0.reconnectKey == key }
let session = Session(id: nextId, source: src, capture: capture, ws: ws, codec: codec, fps: fps,
bitsPerPixel: bpp, reconnectKey: key)
nextId += 1
sessions[session.id] = session
lock.unlock()
for old in replaced { old.end() }
session.onEnd = { [weak self] s in
guard let self else { return }
self.lock.lock(); self.sessions[s.id] = nil; self.lock.unlock()
}
// The source is gone (its window closed): its viewers can't come back.
session.onFinished = { [weak self] src in
guard let self else { return }
self.lock.lock()
self.finished.insert(src)
self.reconnectKeys = self.reconnectKeys.filter { $0.value != src }
self.lock.unlock()
}
session.onAck = { [weak self] in self?.acknowledged(key: key) }
session.start(maxLong: maxLong, fps: fps)
}
}
func snapshot(display id: CGDirectDisplayID, completion: @escaping (Data?, String?) -> Void) {
SCShareableContent.getExcludingDesktopWindows(false, onScreenWindowsOnly: true) { content, error in
guard let d = content?.displays.first(where: { $0.displayID == id }) else {
return completion(nil, error?.localizedDescription ?? "no such display")
}
let filter = SCContentFilter(display: d, excludingWindows: [])
let cfg = SCStreamConfiguration()
cfg.width = Int(Double(d.width) * Double(filter.pointPixelScale))
cfg.height = Int(Double(d.height) * Double(filter.pointPixelScale))
cfg.showsCursor = true
SCScreenshotManager.captureImage(contentFilter: filter, configuration: cfg) { image, error in
guard let image else { return completion(nil, error?.localizedDescription ?? "no image") }
let rep = NSBitmapImageRep(cgImage: image)
completion(rep.representation(using: .jpeg, properties: [.compressionFactor: 0.85]), nil)
}
}
}
func requestPermissions() {
if !CGPreflightScreenCaptureAccess() { CGRequestScreenCaptureAccess() }
if !AXIsProcessTrusted() {
AXIsProcessTrustedWithOptions([kAXTrustedCheckOptionPrompt.takeUnretainedValue() as String: true] as CFDictionary)
}
}
func argument(_ name: String, in args: [String]) -> String? {
guard let i = args.firstIndex(of: name), i + 1 < args.count else { return nil }
return args[i + 1]
}
let args = Array(CommandLine.arguments.dropFirst())
switch args.first {
case "windows":
printJSON(["windows": windowsJSON(), "screen": CGPreflightScreenCaptureAccess()])
case "displays":
printJSON(["displays": displaysJSON()])
case "permissions":
printJSON(permissionsJSON())
case "request-permissions":
requestPermissions()
printJSON(permissionsJSON())
case "serve":
let port = UInt16(argument("--port", in: args) ?? "") ?? 0
let token = ProcessInfo.processInfo.environment["FRAME_MAC_VIEW_TOKEN"] ?? ""
guard !token.isEmpty else {
FileHandle.standardError.write(Data("frame-mac-view: set FRAME_MAC_VIEW_TOKEN\n".utf8))
exit(2)
}
let page = argument("--page", in: args).map { URL(fileURLWithPath: $0) }
let agent = Agent(token: token, page: page)
// Quit when the parent goes away (it holds our stdin open), or on SIGTERM,
// but first end every stream, so separated windows go back where they
// were before their displays disappear.
if args.contains("--exit-on-eof") {
DispatchQueue.global().async {
while FileHandle.standardInput.availableData.count > 0 {}
agent.quit()
}
}
signal(SIGTERM, SIG_IGN)
let sigterm = DispatchSource.makeSignalSource(signal: SIGTERM, queue: .main)
sigterm.setEventHandler { agent.quit() }
sigterm.resume()
let server: Server
do {
server = try Server(port: port) { req, c in agent.handle(req, c) }
} catch {
FileHandle.standardError.write(Data("frame-mac-view: \(error)\n".utf8))
exit(1)
}
server.start { [server] error in
if let error {
FileHandle.standardError.write(Data("frame-mac-view: can't listen on 127.0.0.1:\(port): \(error)\n".utf8))
exit(1)
}
print("frame-mac-view listening on 127.0.0.1:\(server.port ?? port)")
fflush(stdout)
}
// A real (background, no Dock icon) AppKit event loop, not just a run
// loop: without it this process never hears that displays were added or
// changed mode, so NSScreen and CGDisplayCopyDisplayMode stay stale for
// the displays Separate mode creates.
let app = NSApplication.shared
app.setActivationPolicy(.prohibited)
withExtendedLifetime((server, sigterm)) { app.run() }
default:
FileHandle.standardError.write(Data("usage: frame-mac-view serve|windows|displays|permissions|request-permissions\n".utf8))
exit(2)
}
mac/frame-mac-view/build.sh
Executable
+11
View File
@@ -0,0 +1,11 @@
#!/bin/sh
# Builds the Mac streaming agent into mac/bin/frame-mac-view (arm64, macOS 14+).
# Frame Control runs it for "Mac in the headset"; the Electron app bundles it.
set -eu
here=$(cd "$(dirname "$0")" && pwd)
out=${1:-$here/../bin/frame-mac-view}
mkdir -p "$(dirname "$out")"
xcrun swiftc -O -swift-version 5 -target arm64-apple-macos14.0 \
-import-objc-header "$here/Sources/CGVirtualDisplay.h" \
-o "$out" "$here"/Sources/*.swift
echo "built $out"
mac/frame-mac-view/lab.sh
Executable
+50
View File
@@ -0,0 +1,50 @@
#!/bin/sh
# Wraps the agent in "Frame Mac View Lab.app" for testing from a terminal or
# an agent session: macOS then asks for Screen Recording and Accessibility
# for this app rather than for the terminal. Signed with an Apple Development
# identity if there is one, so the permissions survive rebuilds.
# mac/frame-mac-view/lab.sh build the app into mac/bin/
# mac/frame-mac-view/lab.sh serve also start it (token, port and log in ~/Library/Caches/frame-mac-view-lab,
# readable only by you: the token opens every window and injects input)
# mac/frame-mac-view/lab.sh serve-only restart it without rebuilding
set -eu
here=$(cd "$(dirname "$0")" && pwd)
app="$here/../bin/Frame Mac View Lab.app"
if [ "${1:-}" != serve-only ]; then
mkdir -p "$app/Contents/MacOS"
sh "$here/build.sh" "$app/Contents/MacOS/frame-mac-view" >/dev/null
cat > "$app/Contents/Info.plist" <<PLIST
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0"><dict>
<key>CFBundleIdentifier</key><string>com.saphid.frame-mac-view-lab</string>
<key>CFBundleExecutable</key><string>frame-mac-view</string>
<key>CFBundleName</key><string>Frame Mac View Lab</string>
<key>CFBundlePackageType</key><string>APPL</string>
<key>CFBundleShortVersionString</key><string>1</string>
<key>LSUIElement</key><true/>
<key>NSScreenCaptureUsageDescription</key><string>Tests streaming Mac windows into the Steam Frame.</string>
</dict></plist>
PLIST
id=$(security find-identity -v -p codesigning 2>/dev/null | awk '/Apple Development/ {print $2; exit}')
codesign -f -s "${id:--}" "$app" >/dev/null 2>&1
echo "built $app (signed ${id:-ad hoc})"
fi
if [ "${1:-}" = serve ] || [ "${1:-}" = serve-only ]; then
pkill -f "Frame Mac View Lab.app/Contents/MacOS/frame-mac-view" 2>/dev/null || true
dir="$HOME/Library/Caches/frame-mac-view-lab"
mkdir -p "$dir" && chmod 700 "$dir"
umask 077
token=$(openssl rand -hex 16)
rm -f "$dir/token" "$dir/port" "$dir/log"
echo "$token" > "$dir/token"
: > "$dir/log"
# Experiment switches pass through: FRAME_MAC_VIEW_ENCODER (Encoder.swift),
# _VD_HZ (Separate.swift), _ADAPT (Controller.swift), _WARM (main.swift).
open -n "$app" --env "FRAME_MAC_VIEW_TOKEN=$token" --env "FRAME_MAC_VIEW_ENCODER=${FRAME_MAC_VIEW_ENCODER:-}" --env "FRAME_MAC_VIEW_VD_HZ=${FRAME_MAC_VIEW_VD_HZ:-}" \
--env "FRAME_MAC_VIEW_ADAPT=${FRAME_MAC_VIEW_ADAPT:-}" --env "FRAME_MAC_VIEW_WARM=${FRAME_MAC_VIEW_WARM:-}" --stdout "$dir/log" --stderr "$dir/log" \
--args serve --port 0 --page "$here/../../ui/mac-view.html"
for _ in 1 2 3 4 5 6 7 8 9 10; do grep -q listening "$dir/log" && break; sleep 0.3; done
sed -n 's/.*127\.0\.0\.1:\([0-9]*\).*/\1/p' "$dir/log" | head -n 1 > "$dir/port"
echo "lab on 127.0.0.1:$(cat "$dir/port")"
fi
scripts/macview-bench.py
Executable
+847
View File
@@ -0,0 +1,847 @@
#!/usr/bin/env python3
"""Benchmark "Mac in the headset" on the real Frame, so every change to the
stream is judged by numbers (docs/mac-in-headset.md, "Measuring").
scripts/macview-bench.py run [--scenario test,scroll,type] [--net 8] [--label x]
scripts/macview-bench.py compare bench/results/A.json bench/results/B.json
scripts/macview-bench.py ab --arm off:FRAME_MAC_VIEW_ADAPT=0 --arm on: --repeat 3 --scenario scroll
`run` uses the lab agent (mac/frame-mac-view/lab.sh serve), which has
Screen Recording and Accessibility. It opens a viewer on the Frame the way
Frame Control does, drives a fixed scenario, and reads the agent's per-frame
records: when the Mac composited each frame, when it was encoded and sent,
and when the viewer received, decoded and drew it (the viewer's clock is
synced to the Mac's). "content" is Mac composited -> drawn in the viewer;
what the Frame's compositor adds after that is reported ("present",
"fps_shown") but not graded, because an unworn Frame throttles panels to
36 fps (15 in standby) after a few seconds whatever they draw. It writes a summary to bench/results/ and exits 1 if a
target is missed ("bad"), or if --baseline is given and a key number got
more than 10% worse.
Wi-Fi changes from minute to minute, so single runs taken at different times
can differ by more than a change does. `ab` restarts the lab agent with each
arm's settings (environment variables read by the agent), runs the arms
interleaved, and prints each metric's median and range per arm.
Nobody needs to wear the headset. The scroll and type scenarios open a
throwaway Chrome window (its own profile in /tmp) on the Mac.
Network conditions come from a relay on the Mac between the agent and the
tunnel, so they need no sudo:
--net 8 8 Mbit/s the whole run
--net 50@0,8@10,50@20 50 Mbit/s, 8 from 10 s, back to 50 from 20 s
--delay 30 30 ms more one-way delay
The relay holds at most --buffer ms of data at the current rate (default
250 ms, like a bloated Wi-Fi queue), then pushes back on the agent.
Stdlib only.
"""
import argparse
import asyncio
import datetime
import json
import os
import platform
import re
import statistics
import subprocess
import sys
import threading
import time
import urllib.request
from pathlib import Path
from urllib.parse import quote, urlencode # %20, not +: the agent's URLComponents keeps +
ROOT = Path(__file__).resolve().parent.parent
sys.path.insert(0, str(ROOT / "ui"))
import frame_macview # noqa: E402
RESULTS = ROOT / "bench" / "results"
PAGES = ROOT / "bench" / "pages"
CHROME = "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome"
CHROME_PROFILE = "/tmp/fmv-bench-chrome"
LAB_DIR = Path.home() / "Library" / "Caches" / "frame-mac-view-lab" # lab.sh serve
# The acceptance bar (the handoff's targets): (feels local, acceptable) per
# metric; worse than acceptable is "bad". Latencies in ms.
TARGETS = {
"content_p50": (25, 60), "content_p95": (40, 100),
"input_p50": (50, 100), "input_p95": (70, 150),
"fps": (58, 45), # higher is better
"late_pct": (1, 5), "stall_max": (100, 250),
"adapt_s": (1, 3),
}
HIGHER_IS_BETTER = {"fps"}
# Content that changes every frame, so fps and stalls mean something.
MOVING = {"test", "scroll"}
def ms(a, b):
return (b - a) / 1000 if a and b else None
def pct(values, p):
v = sorted(x for x in values if x is not None)
if not v:
return None
return round(v[min(len(v) - 1, max(0, round(p * (len(v) - 1))))], 1)
def dist(values):
v = [x for x in values if x is not None]
return {"p50": pct(v, 0.5), "p95": pct(v, 0.95), "p99": pct(v, 0.99), "n": len(v)} if v else {"n": 0}
# ---- the network relay ----
class Relay:
"""Agent <-> tunnel, shaping the agent-to-Frame direction: a bottleneck
of `rate` Mbit/s with a queue of `buffer_ms`, plus `delay` ms each way."""
def __init__(self, target_port, schedule, delay_ms=0, buffer_ms=250):
self.target_port = target_port
self.schedule = schedule # [(seconds from start, Mbit/s or None)]
self.delay = delay_ms / 1000
self.buffer_ms = buffer_ms
self.rate = schedule[0][1] if schedule else None
self.next_free = 0.0
self.port = None
self.loop = asyncio.new_event_loop()
# Connections reset when a run tears the tunnel down; that's expected.
self.loop.set_exception_handler(lambda loop, ctx: None if isinstance(ctx.get("exception"), ConnectionError)
else loop.default_exception_handler(ctx))
self.started = None
self.queued_max = 0
def start(self):
ready = threading.Event()
def run():
asyncio.set_event_loop(self.loop)
server = self.loop.run_until_complete(asyncio.start_server(self._client, "127.0.0.1", 0))
self.port = server.sockets[0].getsockname()[1]
ready.set()
self.loop.run_forever()
threading.Thread(target=run, daemon=True).start()
ready.wait(5)
def begin(self):
"""Start the schedule's clock (when the measured part begins)."""
self.started = time.monotonic()
for at, rate in self.schedule:
self.loop.call_soon_threadsafe(self.loop.call_later, at, self._set_rate, rate)
def _set_rate(self, rate):
self.rate = rate
def rate_at(self, t):
r = None
for at, rate in self.schedule:
if t >= at:
r = rate
return r
async def _client(self, reader, writer):
try:
up_r, up_w = await asyncio.open_connection("127.0.0.1", self.target_port)
except OSError:
writer.close()
return
await asyncio.gather(self._shaped(up_r, writer), self._plain(reader, up_w), return_exceptions=True)
for w in (writer, up_w):
w.close()
async def _plain(self, reader, writer): # Frame -> agent: delay only
if not self.delay:
while data := await reader.read(65536):
writer.write(data)
await writer.drain()
writer.close()
return
# Each chunk leaves `delay` after it arrived, while reading goes on:
# a delay line, not a pause (which would add up behind a busy stream).
queue = asyncio.Queue()
async def send():
while (item := await queue.get())[1] is not None:
wait = item[0] - self.loop.time()
if wait > 0:
await asyncio.sleep(wait)
writer.write(item[1])
await writer.drain()
writer.close()
async def pump():
try:
while data := await reader.read(65536):
await queue.put((self.loop.time() + self.delay, data))
finally: # EOF or a reset: either way the sender finishes and closes
queue.put_nowait((0, None))
sender, pumper = asyncio.ensure_future(send()), asyncio.ensure_future(pump())
done, _ = await asyncio.wait({sender, pumper}, return_when=asyncio.FIRST_COMPLETED)
if sender in done: # the agent's side went away: stop reading, so the connection closes
pumper.cancel()
writer.close()
return
await sender # the viewer's side ended: deliver what's queued first
async def _shaped(self, reader, writer): # agent -> Frame
queue = asyncio.Queue()
state = {"bytes": 0}
room = asyncio.Event()
room.set()
async def send():
while True:
at, data = await queue.get()
if data is None:
return
wait = at - self.loop.time()
if wait > 0:
await asyncio.sleep(wait)
writer.write(data)
await writer.drain()
state["bytes"] -= len(data)
room.set()
sender = asyncio.ensure_future(send())
while data := await reader.read(16384):
now = self.loop.time()
depart = now
if self.rate:
depart = max(now, self.next_free) + len(data) * 8 / (self.rate * 1e6)
self.next_free = depart
state["bytes"] += len(data)
self.queued_max = max(self.queued_max, state["bytes"])
await queue.put((depart + self.delay, data))
# A full queue pushes back, as a real bottleneck's buffer does.
while self.rate and state["bytes"] > self.rate * 1e6 / 8 * self.buffer_ms / 1000:
room.clear()
await room.wait()
await queue.put((0, None))
await sender
# ---- the lab agent, through Frame Control's own code ----
class LabView(frame_macview.MacView):
"""MacView against the already-running lab agent, with the tunnel
pointing at `tunnel_port` (the relay, or the agent itself)."""
def __init__(self, tunnel_ssh, run, frame, agent_port, token, tunnel_port):
super().__init__(tunnel_ssh, run, frame)
self.agent_port = agent_port
self.token = token
self.port = tunnel_port
class Alive:
def poll(self):
return None
self.agent = Alive()
def ensure_agent(self):
pass
def call(self, path, method="GET", **query):
url = f"http://127.0.0.1:{self.agent_port}{path}?{urlencode({**query, 'k': self.token}, quote_via=quote)}"
req = urllib.request.Request(url, method=method, data=b"" if method == "POST" else None)
with urllib.request.urlopen(req, timeout=15) as r:
return json.load(r)
def ssh_runner(base):
def run(remote, stdin=None, timeout=30):
r = subprocess.run([*base, remote], input=stdin, capture_output=True, text=True, timeout=timeout)
if r.returncode:
e = RuntimeError((r.stderr or r.stdout).strip())
e.stdout = r.stdout
raise e
return r.stdout
return run
# ---- CPU on both ends ----
FRAME_CPU = r"""
hz=$(getconf CLK_TCK)
while :; do
t=$(date +%s.%N)
for g in viewer:frame-control/mac-view tailscaled:tailscaled sshd:'^sshd' gamescope:'^gamescope'; do
name=${g%%:*} pat=${g#*:} sum=0
for p in $(pgrep -f "$pat"); do
s=$(awk '{print $14+$15}' /proc/$p/stat 2>/dev/null) && sum=$((sum + s))
done
echo "$t $name $sum $hz"
done
echo "$t total $(awk '/^cpu /{print $2+$3+$4+$6+$7+$8}' /proc/stat) $hz"
sleep 2
done
"""
class CpuSampler:
def __init__(self, frame_ssh, agent_pid):
self.agent_pid = agent_pid
self.mac = [] # (t, cpu seconds)
self.frame = {} # name -> [(t, ticks, hz)]
self.proc = subprocess.Popen([*frame_ssh, "bash -s"], stdin=subprocess.PIPE, stdout=subprocess.PIPE,
stderr=subprocess.DEVNULL, text=True)
self.proc.stdin.write(FRAME_CPU)
self.proc.stdin.close()
threading.Thread(target=self._read, daemon=True).start()
self.stop_flag = False
threading.Thread(target=self._mac, daemon=True).start()
def _read(self):
for line in self.proc.stdout:
parts = line.split()
if len(parts) == 4:
self.frame.setdefault(parts[1], []).append((float(parts[0]), int(parts[2]), int(parts[3])))
def _mac(self):
while not self.stop_flag:
out = subprocess.run(["ps", "-o", "cputime=", "-p", str(self.agent_pid)], capture_output=True,
text=True).stdout.strip()
m = re.match(r"(?:(\d+):)?(\d+):(\d+(?:\.\d+)?)", out)
if m:
h, mnt, s = m.groups()
self.mac.append((time.monotonic(), int(h or 0) * 3600 + int(mnt) * 60 + float(s)))
time.sleep(2)
def stop(self):
self.stop_flag = True
self.proc.terminate()
def summary(self):
def rate(samples):
if len(samples) < 2:
return None
(t0, a0, *h0), (t1, a1, *_) = samples[0], samples[-1]
hz = h0[0] if h0 else 1
return round((a1 - a0) / hz / (t1 - t0) * 100, 1)
out = {"mac_agent_pct": rate(self.mac)}
for name, s in self.frame.items():
out[f"frame_{name}_pct"] = rate(s)
if "frame_total_pct" in out and out["frame_total_pct"] is not None:
out["frame_total_pct"] = round(out["frame_total_pct"] / 8, 1) # 8 cores -> share of the whole
return out
# ---- scenarios ----
def chrome_window(page, size=(1280, 820)):
"""Opens a page in a throwaway Chrome app window; returns (proc, title)."""
if not Path(CHROME).exists():
raise SystemExit("Google Chrome is needed for the scroll and type scenarios.")
proc = subprocess.Popen([CHROME, f"--user-data-dir={CHROME_PROFILE}", "--no-first-run",
"--no-default-browser-check", f"--window-size={size[0]},{size[1]}",
"--window-position=80,80", f"--app=file://{PAGES / page}"],
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
return proc
def find_window(mv, title, timeout=15):
end = time.time() + timeout
while time.time() < end:
for w in mv.call("/windows").get("windows", []):
if title in (w.get("title") or ""):
return w
time.sleep(0.5)
raise SystemExit(f"window {title!r} didn't appear")
def git(*a):
return subprocess.run(["git", *a], cwd=ROOT, capture_output=True, text=True).stdout.strip()
def parse_schedule(net):
if not net or net == "none":
return []
out = []
for part in net.split(","):
rate, _, at = part.partition("@")
out.append((float(at or 0), None if rate in ("", "none", "0") else float(rate)))
return sorted(out)
def run_scenario(args, scenario, agent_port, token, frame_ssh):
schedule = parse_schedule(args.net)
relay = None
tunnel_port = agent_port
if schedule or args.delay:
relay = Relay(agent_port, schedule, args.delay, args.buffer)
relay.start()
tunnel_port = relay.port
tunnel_ssh = list(frame_ssh[:-1]) + args.ssh_opt
mv = LabView(tunnel_ssh, ssh_runner(frame_ssh), frame_ssh[-1], agent_port, token, tunnel_port)
mv.prefer_usb = args.usb # otherwise the path is --host's (or ssh's default)
if args.browser_flag is not None:
mv.browser_flags = [f for f in args.browser_flag if f]
chrome = None
src = "test"
try:
if scenario in ("scroll", "type"):
chrome = chrome_window(f"{scenario}.html")
w = find_window(mv, f"fmv-bench {scenario}")
src = f"{args.mode}:{w['id']}"
t = time.time()
shown = mv.show(src, args.quality, 1280, 820)
show_s = round(time.time() - t, 2)
# Wait for frames, then measure from a clean start.
stream = None
end = time.time() + 20
while time.time() < end and not stream:
for s in mv.call("/status").get("streams", []):
if s["src"] == src and s.get("stats", {}).get("fps", 0) > 0:
stream = s
time.sleep(0.5)
if not stream:
raise SystemExit(f"{scenario}: no frames reached the viewer within 20 s")
time.sleep(args.warmup)
first = mv.call("/stats", id=stream["id"], settle=0)["streams"][0]
since = max([f["s"] for f in first["frames"]] or [0])
captured_before = first["captured"]
start_mac_us = mv.call("/stats", id=stream["id"])["now"]
agent_pid = int(subprocess.run(["pgrep", "-f", "Frame Mac View Lab.app/Contents/MacOS/frame-mac-view"],
capture_output=True, text=True).stdout.split()[0])
cpu = CpuSampler(frame_ssh, agent_pid)
if relay:
relay.begin()
expected = 0 # input events the harness asked the viewer for
if scenario == "type":
mv.call("/bench", method="POST", src=src, action="click", x=0.5, y=0.3)
time.sleep(0.5)
# Steady typing, then slow typing that lets the link go idle.
fast = "the quick brown fox jumps over the lazy dog "
n_fast = max(10, int((args.duration * 0.6) / 0.2))
mv.call("/bench", method="POST", src=src, action="type", text=(fast * (n_fast // len(fast) + 1))[:n_fast],
interval=200)
expected += n_fast
time.sleep(n_fast * 0.2 + 0.5)
slow_n = max(3, int(args.duration * 0.4 / 1.5))
mv.call("/bench", method="POST", src=src, action="type", text=("idle typing test " * (slow_n // 17 + 1))[:slow_n],
interval=1500)
expected += slow_n
time.sleep(slow_n * 1.5 + 0.5)
else:
# Clicks on the test pattern measure the input path; the scroll
# page just scrolls.
end = time.time() + args.duration
while time.time() < end:
if scenario == "test":
mv.call("/bench", method="POST", src=src, action="click", x=0.3, y=0.5)
expected += 1
time.sleep(0.5)
at_end = mv.call("/stats", id=stream["id"], since=2**32 - 1)
end_mac_us = at_end["now"]
captured_end = at_end["streams"][0]["captured"] if at_end["streams"] else None
time.sleep(2) # let the viewer report the last frames
streams = mv.call("/stats", id=stream["id"], since=since)["streams"]
if not streams:
raise SystemExit(f"{scenario}: the stream ended during the run (did the Frame go to sleep?)")
data = streams[0]
cpu.stop()
if args.raw:
Path(f"{args.raw}-{scenario}.json").write_text(json.dumps(data))
result = summarize(scenario, data, start_mac_us, end_mac_us, args, relay, schedule, expected)
result["controller"] = data.get("controller", {})
result["events"] = [{"t": round((e["t"] - start_mac_us) / 1e6, 2), "e": e["e"]}
for e in data.get("events", []) if e["t"] >= start_mac_us]
result["captured"] = (captured_end if captured_end is not None else data["captured"]) - captured_before
result["source_fps"] = round(result["captured"] / max(result["duration_s"], 1), 1)
result["cpu"] = cpu.summary()
result["viewer"] = data["summary"].get("decoder", "")
result["show_s"] = show_s
result["panel"] = shown.get("panel")
result["src"] = src
result["route"] = mv.route
return result
finally:
try:
mv.stop(src)
except Exception: # noqa: BLE001 - best effort
pass
mv.closing = True # or its supervisor reopens the tunnel
if mv.tunnel and mv.tunnel.poll() is None:
mv.tunnel.terminate()
mv.tunnel.wait(5)
if relay:
relay.loop.call_soon_threadsafe(relay.loop.stop)
if chrome:
chrome.terminate()
time.sleep(3) # Stop ends the Frame's viewer browser 2 s later
def composited(f):
"""When the Mac had the frame: its display time, or when ScreenCaptureKit
handed it over if that was earlier (display time can be a few ms ahead of
delivery, which would make latency look lower than it is)."""
return min(f["cap"], f["arr"]) if f["arr"] else f["cap"]
def summarize(scenario, data, start_us, end_us, args, relay, schedule, expected=0):
frames = [f for f in data["frames"] if start_us <= f["cap"] <= end_us]
inputs = [i for i in data["inputs"] if i["tv"] and i["tv"] >= start_us]
by_seq = {f["s"]: f for f in data["frames"]}
drawn = [f for f in frames if f["drw"]]
shown = [f for f in frames if f["vs"]]
# The whole measured interval, so a stall at either end still counts.
duration = (end_us - start_us) / 1e6
stages = {
"capture": [ms(f["cap"], f["arr"]) for f in frames],
"queue": [ms(f["arr"], f["e0"]) for f in frames],
"encode": [ms(f["e0"], f["e1"]) for f in frames],
"socket": [ms(f["snd"], f["wire"]) for f in frames],
"network": [ms(f["e1"], f["rx"]) for f in frames],
"decode": [ms(f["rx"], f["dec"]) for f in frames],
"draw": [ms(f["dec"], f["drw"]) for f in frames],
# Waiting for the browser's next frame: set by the Frame's compositor,
# which throttles panels nobody is looking at (see "Measuring").
"present": [ms(f["drw"], f["vs"]) for f in frames],
"content": [ms(composited(f), f["drw"]) for f in frames],
"content_shown": [ms(composited(f), f["vs"]) for f in frames],
}
out = {"scenario": scenario, "frames_sent": len(frames), "frames_drawn": len(drawn),
"frames_shown": len(shown), "duration_s": round(duration, 1)}
out["stages_ms"] = {k: dist(v) for k, v in stages.items()}
# Smoothness: gaps between frames reaching the viewer's canvas.
# Smoothness counts only what was drawn inside the interval; a frame
# captured just before the end but drawn after it still counts for latency.
dr = [start_us] + sorted(f["drw"] for f in drawn if f["drw"] <= end_us) + [end_us]
gaps = [(b - a) / 1000 for a, b in zip(dr, dr[1:]) if b > a]
target_fps = frame_macview.QUALITY[args.quality]["fps"]
out["fps"] = round(sum(f["drw"] <= end_us for f in drawn) / duration, 1) if duration else 0
out["fps_shown"] = round(sum(0 < f["vs"] <= end_us for f in shown) / duration, 1) if duration else 0
out["late_pct"] = round(100 * sum(g > 1.5 * 1000 / target_fps for g in gaps) / len(gaps), 2) if gaps else None
out["stall_max"] = round(max(gaps), 1) if gaps else None
out["stalls_over_100ms"] = sum(g > 100 for g in gaps)
out["mbps"] = round(sum(f["b"] for f in frames) * 8 / duration / 1e6, 2) if duration else 0
out["keyframes"] = sum(f["k"] for f in frames)
out["size"] = f"{frames[-1]['w']}x{frames[-1]['h']}" if frames else ""
out["captured"] = data.get("captured")
out["viewer_never_drawn"] = len(frames) - len(drawn)
# Input: the viewer's event -> the first frame after it drawn (and shown).
lat, lat_shown = [], []
parts = {"uplink": [], "mac": [], "back": []}
for i in inputs:
f = by_seq.get(i["frame"])
if f and f["drw"]:
lat.append(ms(i["tv"], f["drw"]))
lat_shown.append(ms(i["tv"], f["vs"]))
parts["uplink"].append(ms(i["tv"], i["inj"]))
parts["mac"].append(ms(i["inj"], composited(f)))
parts["back"].append(ms(composited(f), f["drw"]))
out["input_ms"] = dist(lat)
out["input_shown_ms"] = dist(lat_shown)
out["input_parts_ms"] = {k: dist(v) for k, v in parts.items()}
out["inputs"] = {"asked": expected, "sent": len(inputs), "seen": len(lat)}
# A per-second timeline, for adaptation.
timeline = []
t0 = start_us
for sec in range(int(duration) + 1):
a, b = t0 + sec * 1_000_000, t0 + (sec + 1) * 1_000_000
fs = [f for f in frames if a <= f["cap"] < b]
if not fs:
continue
timeline.append({
"t": sec, "fps": sum(1 for f in fs if f["drw"]), "mbps": round(sum(f["b"] for f in fs) * 8 / 1e6, 2),
"content_p50": pct([ms(composited(f), f["drw"]) for f in fs], 0.5),
"content_p95": pct([ms(composited(f), f["drw"]) for f in fs], 0.95),
"bitrate": fs[-1].get("br"), "tier": fs[-1].get("tier"), "w": fs[-1]["w"],
"net": relay.rate_at(sec) if relay else None,
})
out["timeline"] = timeline
out["adapt"] = adaptation(timeline, schedule, duration)
out["content_p50"] = out["stages_ms"]["content"].get("p50")
out["content_p95"] = out["stages_ms"]["content"].get("p95")
out["input_p50"] = out["input_ms"].get("p50")
out["input_p95"] = out["input_ms"].get("p95")
out["grades"] = grade(out, scenario, args.quality)
return out
def adaptation(timeline, schedule, duration=float("inf")):
"""After each drop in the link's rate: how long until latency was within
bounds again and stayed there for 3 s. The bound is the acceptable p95
(100 ms), or 1.5 times the p95 before the drop if that was higher: on a
slower link each frame takes longer to send, so latency can't return to
what it was on the fast one."""
steps = []
for n, ((at, rate), (_, before)) in enumerate(zip(schedule[1:], schedule)):
# Settling must happen while this rate lasts, not after the link recovers.
until = min(schedule[n + 2][0] if n + 2 < len(schedule) else float("inf"), duration) # or the run ends
if before is not None and (rate is None or rate >= before):
continue
pre = [s["content_p95"] for s in timeline if at - 4 <= s["t"] < at and s["content_p95"]]
base = max(float(TARGETS["content_p95"][1]), 1.5 * statistics.median(pre) if pre else 0)
after = [s for s in timeline if s["t"] >= at]
worst = max([s["content_p95"] or 0 for s in after[:10]] or [0])
settled = None
for i, s in enumerate(after):
window = after[i:i + 3]
# Three populated seconds in a row: a second with no frames at all
# is a stall, not a pass.
if (len(window) == 3 and [w["t"] for w in window] == [s["t"], s["t"] + 1, s["t"] + 2] and s["t"] + 3 <= until
and all((w["content_p95"] or 1e9) <= base for w in window)):
settled = s["t"] - at
break
steps.append({"at": at, "to_mbit": rate, "p95_bound": round(base, 1), "worst_p95": worst,
"settled_s": settled})
return steps
def grade(out, scenario, quality="balanced"):
g = {}
checks = ["content_p50", "content_p95"]
asked = max(out["inputs"].get("asked", 0), out["inputs"]["sent"])
if asked:
checks += ["input_p50", "input_p95"]
# Input that never reaches the Mac, or never shows up on screen, is a
# failure, not a gap in the data.
seen = out["inputs"]["seen"] / asked
g["input_replies"] = "local" if seen >= 0.95 else "acceptable" if seen >= 0.8 else "bad"
if scenario in MOVING:
checks += ["fps", "late_pct", "stall_max"]
for k in checks:
v = out.get(k)
if v is None:
g[k] = "missing"
continue
good, ok = TARGETS[k]
if k == "fps": # the targets are for 60 fps; Light and Compatible ask for fewer
want = frame_macview.QUALITY[quality]["fps"]
good, ok = good * want / 60, ok * want / 60
if k in HIGHER_IS_BETTER:
g[k] = "local" if v >= good else "acceptable" if v >= ok else "bad"
else:
g[k] = "local" if v <= good else "acceptable" if v <= ok else "bad"
for step in out["adapt"]:
s = step["settled_s"]
good, ok = TARGETS["adapt_s"]
g[f"adapt@{step['at']:g}s"] = "bad" if s is None or s > ok else "local" if s <= good else "acceptable"
return g
# ---- compare ----
KEYS = [("content_p50", "content p50 ms"), ("content_p95", "content p95 ms"), ("input_p50", "input p50 ms"),
("input_p95", "input p95 ms"), ("fps", "fps"), ("late_pct", "late %"), ("stall_max", "max gap ms"),
("fps_shown", "fps shown"), ("mbps", "Mbit/s")]
STAGE_KEYS = ["capture", "queue", "encode", "socket", "network", "decode", "draw", "present"]
def load(path):
return json.loads(Path(path).read_text())
def compare(a, b, threshold=0.10):
"""Prints before/after per scenario; returns the regressions."""
regressions = []
print(f"A: {a['label']} ({a['commit']}, {a['date']})\nB: {b['label']} ({b['commit']}, {b['date']})")
for sc in b["scenarios"]:
ra = next((x for x in a["scenarios"] if x["scenario"] == sc["scenario"]), None)
if not ra:
continue
print(f"\n{sc['scenario']:<10} {'A':>10} {'B':>10} {'change':>9}")
rows = [(k, n, ra.get(k), sc.get(k)) for k, n in KEYS]
rows += [(f"stage.{s}", f" {s} p50", ra["stages_ms"].get(s, {}).get("p50"), sc["stages_ms"].get(s, {}).get("p50"))
for s in STAGE_KEYS]
for k, name, va, vb in rows:
change = ""
if isinstance(va, (int, float)) and isinstance(vb, (int, float)) and va:
d = (vb - va) / abs(va)
change = f"{d:+.0%}"
worse = d < -threshold if k == "fps" else d > threshold
if worse and k in ("content_p50", "content_p95", "input_p50", "input_p95", "fps") and abs(vb - va) > 2:
regressions.append(f"{sc['scenario']} {name}: {va} -> {vb}")
change += " worse"
print(f"{name:<18} {fmt(va):>10} {fmt(vb):>10} {change:>9}")
return regressions
def fmt(v):
return "–" if v is None else f"{v:g}" if isinstance(v, (int, float)) else str(v)
# ---- main ----
def add_run_args(r):
r.add_argument("--scenario", default="test,scroll,type")
r.add_argument("--duration", type=float, default=20)
r.add_argument("--warmup", type=float, default=3)
r.add_argument("--quality", default="balanced", choices=list(frame_macview.QUALITY))
r.add_argument("--mode", default="separate", choices=["separate", "window"], help="how Mac windows are shown")
r.add_argument("--net", default="", help="Mbit/s, or a schedule like 50@0,8@10,50@20")
r.add_argument("--delay", type=float, default=0, help="extra one-way delay, ms")
r.add_argument("--buffer", type=float, default=250, help="bottleneck queue, ms at the current rate")
r.add_argument("--frame", default="frame", help="ssh host of the Frame")
r.add_argument("--host", default="", help="connect to this address instead (e.g. the Frame's LAN IP)")
r.add_argument("--usb", action="store_true", help="use the Frame's USB-C network when plugged in, as Frame Control does")
r.add_argument("--ssh-opt", action="append", default=[], help="extra ssh option for the tunnel, e.g. -oIPQoS=ef")
r.add_argument("--browser-flag", action="append", default=None,
help="Chromium flag for the viewer (replaces Frame Control's defaults; '' for none)")
r.add_argument("--label", default="")
r.add_argument("--out", default="")
r.add_argument("--raw", default="", help="also write the agent's raw records here")
def lab_agent():
try:
agent_port = int((LAB_DIR / "port").read_text())
token = (LAB_DIR / "token").read_text().strip()
urllib.request.urlopen(f"http://127.0.0.1:{agent_port}/ping", timeout=3)
return agent_port, token
except (OSError, ValueError):
raise SystemExit("Start the lab agent first: mac/frame-mac-view/lab.sh serve")
def restart_lab(env):
"""The lab agent again, without rebuilding, with these experiment switches."""
subprocess.run(["sh", str(ROOT / "mac" / "frame-mac-view" / "lab.sh"), "serve-only"],
env={**os.environ, **env}, check=True, capture_output=True)
time.sleep(1)
def frame_ssh_for(args):
frame_ssh = ["ssh", "-o", "BatchMode=yes"]
if args.host:
# Another address for the same Frame: its host key must still match.
alias = next((line.split()[1] for line in subprocess.run(["ssh", "-G", args.frame], capture_output=True,
text=True).stdout.splitlines() if line.startswith("hostname ")), args.frame)
frame_ssh += ["-o", f"HostName={args.host}", "-o", f"HostKeyAlias={alias}"]
frame_ssh.append(args.frame)
return frame_ssh
def run_suite(args, frame_ssh, quiet=False):
agent_port, token = lab_agent()
results = []
for sc in args.scenario.split(","):
if not quiet:
print(f"-- {sc} ...", flush=True)
res = run_scenario(args, sc, agent_port, token, frame_ssh)
if not quiet:
print(f" content p50/p95 {res['content_p50']}/{res['content_p95']} ms, input p50/p95 "
f"{res['input_p50']}/{res['input_p95']} ms, {res['fps']} fps (shown {res['fps_shown']}), "
f"late {res['late_pct']}%, max gap {res['stall_max']} ms, {res['mbps']} Mbit/s, {res['size']}",
flush=True)
print(f" stages p50: " + ", ".join(f"{k} {v.get('p50')}" for k, v in res["stages_ms"].items()), flush=True)
print(f" grades: {res['grades']} cpu: {res['cpu']}\n viewer: {res['viewer']}", flush=True)
for e in res["events"][:12]:
print(f" {e['t']:6.2f}s {e['e']}", flush=True)
results.append(res)
return results
def document(args, frame_ssh, results, **extra):
build = subprocess.run([*frame_ssh, "grep -m1 BUILD_ID /etc/os-release"], capture_output=True, text=True).stdout
# Whether the headset is in standby (it throttles panels even harder then).
standby = subprocess.run([*frame_ssh, "grep -h standby ~/.local/share/Steam/logs/vrserver.txt | tail -n 1"],
capture_output=True, text=True).stdout.strip()
commit = git("rev-parse", "--short", "HEAD") + ("+dirty" if git("status", "--porcelain", "--", "mac", "ui") else "")
return {
"label": args.label or args.cmd, "date": datetime.datetime.now().isoformat(timespec="seconds"), "commit": commit,
"config": {"quality": args.quality, "mode": args.mode, "net": args.net or "none", "delay_ms": args.delay,
"buffer_ms": args.buffer, "host": args.host or args.frame, "usb": args.usb, "ssh_opts": args.ssh_opt,
"encoder": os.environ.get("FRAME_MAC_VIEW_ENCODER", ""), "duration_s": args.duration,
"browser_flags": args.browser_flag if args.browser_flag is not None else frame_macview.BROWSER_FLAGS},
"frame_build": build.strip().partition("=")[2], "mac": platform.mac_ver()[0], "headset": standby[-40:],
"scenarios": results, **extra,
}
def write(doc, args):
RESULTS.mkdir(parents=True, exist_ok=True)
name = f"{doc['date'][:10]}-{doc['commit'].replace('+', '-')}-{re.sub(r'[^a-zA-Z0-9_.-]+', '-', doc['label'])}.json"
out = Path(args.out) if args.out else RESULTS / name
out.write_text(json.dumps(doc, indent=1))
print(f"wrote {out}")
return out
AB_KEYS = ["content_p50", "content_p95", "input_p50", "input_p95", "fps", "late_pct", "stall_max", "mbps"]
def ab(args, frame_ssh):
"""Arms interleaved (A B, B A, ...) so a Wi-Fi link that changes over
minutes affects both alike; medians and ranges per arm."""
arms = []
for a in args.arm:
name, _, envs = a.partition(":")
arms.append((name, dict(e.split("=", 1) for e in envs.split(",") if e)))
runs = {name: [] for name, _ in arms}
for rep in range(args.repeat):
for name, env in (arms if rep % 2 == 0 else arms[::-1]):
print(f"-- repeat {rep + 1}, arm {name} {env}", flush=True)
restart_lab(env)
runs[name].append(run_suite(args, frame_ssh, quiet=True))
table = {}
for sc in args.scenario.split(","):
print(f"\n{sc:<14}" + "".join(f"{name:>26}" for name, _ in arms))
for k in AB_KEYS:
row = []
for name, _ in arms:
vals = [r[k] for suite in runs[name] for r in suite if r["scenario"] == sc and r.get(k) is not None]
table.setdefault(sc, {}).setdefault(k, {})[name] = vals
row.append(f"{statistics.median(vals):g} ({min(vals):g}-{max(vals):g})" if vals else "–")
print(f"{k:<14}" + "".join(f"{c:>26}" for c in row))
restart_lab({}) # back to the defaults
return runs, table
def main():
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
sub = p.add_subparsers(dest="cmd", required=True)
r = sub.add_parser("run")
add_run_args(r)
r.add_argument("--baseline", default="", help="fail on >10%% regressions against this result")
a = sub.add_parser("ab", help="compare lab-agent settings, interleaved: --arm name:ENV=V,ENV2=V ...")
add_run_args(a)
a.add_argument("--arm", action="append", required=True)
a.add_argument("--repeat", type=int, default=3)
c = sub.add_parser("compare")
c.add_argument("a")
c.add_argument("b")
args = p.parse_args()
# The agent keeps the last 4096 frames (Stats.swift) and results are read
# at the end: at 60 fps that's 68 s, less warm-up and the reporting wait.
if args.cmd != "compare" and args.host and args.usb:
p.error("--host and --usb pick the path two ways; use one")
if args.cmd != "compare" and args.warmup + args.duration > 60:
p.error("--warmup plus --duration can be at most 60 s (the agent keeps 4096 frames)")
if args.cmd == "compare":
regs = compare(load(args.a), load(args.b))
if regs:
print("\nworse:\n " + "\n ".join(regs))
sys.exit(1 if regs else 0)
lab_agent()
frame_ssh = frame_ssh_for(args)
# Keep the Mac's screen awake: virtual displays aren't removed while it sleeps.
runs = 1 if args.cmd == "run" else args.repeat * len(args.arm)
awake = subprocess.Popen(["caffeinate", "-d", "-u", "-t", str(int(runs * (args.duration * 4 + 60) + 120))])
try:
if args.cmd == "ab":
arm_runs, table = ab(args, frame_ssh)
write(document(args, frame_ssh, [], arms=args.arm, runs=arm_runs, table=table), args)
return
results = run_suite(args, frame_ssh)
finally:
awake.terminate()
doc = document(args, frame_ssh, results)
write(doc, args)
bad = [f"{r['scenario']} {k}" for r in results for k, v in r["grades"].items() if v in ("bad", "missing")]
regs = compare(load(args.baseline), doc) if args.baseline else []
if bad:
print("missed targets: " + ", ".join(bad))
if regs:
print("worse than baseline:\n " + "\n ".join(regs))
sys.exit(1 if bad or regs else 0)
if __name__ == "__main__":
main()
tests/test_frame_apk_vr.py
+297
View File
@@ -0,0 +1,297 @@
"""Local-only VR manifest, raw ZIP and independent signature checks."""
import io
import os
from pathlib import Path
import struct
import subprocess
import sys
import tempfile
import unittest
from unittest.mock import patch
import zipfile
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'ui'))
import frame_apk
import frame_apk_vr as vr
import frame_apk_sign as signing
import frame_android
from test_frame_apk import pool
DEFAULT = 'android.intent.category.DEFAULT'
def manifest(utf8=False, launcher=False, category=None, samsung=False, split=False, alias=False, splash=False):
strings = ['manifest', 'package', 'org.test.vr', 'application', 'activity', 'intent-filter',
'action', 'category', 'name', vr.MAIN, category or next(iter(sorted(vr.VR))),
vr.LAUNCHER, 'http://schemas.android.com/apk/res/android', 'meta-data', 'value',
'com.samsung.android.vr.application.mode', 'vr_only', 'activity-alias', DEFAULT, next(iter(sorted(vr.VR))), 'targetActivity', '.Splash', '.Game']
def start(tag, attrs=()):
body = struct.pack('<IIHHHHHH', 0xffffffff, strings.index(tag), 20, 20, len(attrs), 0, 0, 0)
for name, value in attrs:
ns = 0xffffffff if name == 'package' else 12
body += struct.pack('<IIIHBBI', ns, strings.index(name), strings.index(value), 8, 0, 3, strings.index(value))
return struct.pack('<HHIII', 0x102, 16, 16 + len(body), 1, 0xffffffff) + body
def end(tag):
return struct.pack('<HHIIIII', 0x103, 16, 24, 1, 0xffffffff, 0xffffffff, strings.index(tag))
def leaf(tag, attrs):
return start(tag, attrs) + end(tag)
b = pool(strings, utf8) + start('manifest', [('package', 'org.test.vr')]) + start('application')
if samsung:
b += leaf('meta-data', [('name', strings[15]), ('value', 'vr_only')])
if splash: # a helper activity with its own MAIN filter, ahead of the game's
b += start('activity', [('name', '.Splash')]) + start('intent-filter') + leaf('action', [('name', vr.MAIN)])
b += leaf('category', [('name', DEFAULT)]) + end('intent-filter') + end('activity')
b += start('activity', [('name', '.Game')] if splash else []) + start('intent-filter') + leaf('action', [('name', vr.MAIN)])
b += leaf('category', [('name', strings[10])])
if split:
b += end('intent-filter') + start('intent-filter')
if launcher:
b += leaf('category', [('name', vr.LAUNCHER)])
b += end('intent-filter') + end('activity')
if alias: # Godot 4: LAUNCHER only on an alias of the activity ('vr' or 'flat')
b += start('activity-alias', [('targetActivity', '.Game')] if splash else []) + start('intent-filter') + leaf('action', [('name', vr.MAIN)])
if alias == 'vr':
b += leaf('category', [('name', next(iter(sorted(vr.VR))))])
b += leaf('category', [('name', vr.LAUNCHER)])
b += end('intent-filter') + end('activity-alias')
b += end('application') + end('manifest')
return struct.pack('<HHI', 3, 8, len(b) + 8) + b
class VRTests(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.tmp = tempfile.TemporaryDirectory()
cls.keypath = Path(cls.tmp.name) / 'key.json'
cls.key = signing.signing_key(cls.keypath)
@classmethod
def tearDownClass(cls):
cls.tmp.cleanup()
def test_manifest_patch(self):
for utf8 in (False, True):
for category in vr.VR:
original = manifest(utf8, category=category)
result = vr.add_launcher_category(original)
info, filters = vr.inspect(result)
self.assertTrue(info['launchable'])
self.assertTrue(info['vr'])
self.assertIn(vr.LAUNCHER, filters[0]['categories'])
self.assertEqual(struct.unpack_from('<I', result, 4)[0], len(result))
self.assertEqual(vr.add_launcher_category(result), result)
self.assertEqual(vr.add_launcher_category(manifest(utf8, True)), manifest(utf8, True))
def test_filter_boundaries(self):
self.assertFalse(vr.inspect(manifest(launcher=True, split=True))[0]['launchable'])
self.assertTrue(vr.inspect(vr.add_launcher_category(manifest(launcher=True, split=True)))[0]['launchable'])
def test_alias_launcher_is_not_enough(self):
# (manifest, still VR after patching)
cases = [(manifest(alias='vr'), True), # Godot 4 VR export
(manifest(alias='vr', category=DEFAULT), True), # VR category only on the alias
(manifest(alias='flat', category=DEFAULT), False)] # Godot 4 flat export
for original, is_vr in cases:
info, filters = vr.inspect(original)
self.assertFalse(info['launchable'])
self.assertTrue(info['repairable'])
self.assertEqual(len(filters), 1)
self.assertFalse(filters[0]['alias'])
result = vr.add_launcher_category(original)
info, _ = vr.inspect(result)
self.assertTrue(info['launchable'])
self.assertFalse(info['repairable'])
self.assertEqual(info['vr'], is_vr)
def test_alias_target_activity_is_patched(self):
original = manifest(alias='flat', category=DEFAULT, splash=True)
info, filters = vr.inspect(original)
self.assertTrue(info['repairable'])
self.assertEqual(filters[0]['activity'], 'org.test.vr.Game')
owner, launchers = None, []
for tag, attrs in frame_apk.manifest_elements(vr.add_launcher_category(original)):
if tag in ('activity', 'activity-alias'):
owner = (tag, attrs.get('name', (0, 0, None))[2])
if tag == 'category' and attrs['name'][2] == vr.LAUNCHER:
launchers.append(owner)
self.assertEqual(launchers, [('activity', '.Game'), ('activity-alias', None)])
def test_alias_with_launchable_activity_is_left_alone(self):
original = manifest(launcher=True, alias='vr')
info, _ = vr.inspect(original)
self.assertTrue(info['launchable'])
self.assertFalse(info['repairable'])
self.assertEqual(vr.add_launcher_category(original), original)
def test_patch_alias_apk(self):
with tempfile.TemporaryDirectory() as d:
src, dst = Path(d) / 'in.apk', Path(d) / 'out.apk'
with zipfile.ZipFile(src, 'w') as z:
z.writestr('AndroidManifest.xml', manifest(alias='flat', category=DEFAULT))
with patch.object(signing, 'signing_key', return_value=self.key):
result = frame_android.patch(src, dst)
self.assertEqual(result['patched'], ['launcher'])
self.assertTrue(frame_apk.apk_info(dst)['launchable'])
self.assertTrue(signing.verify(dst))
def test_styled_pool(self):
for utf8 in (False, True):
p = bytearray(pool(['styled'], utf8))
old_start = struct.unpack_from('<I', p, 20)[0]
p[old_start:old_start] = struct.pack('<I', 0)
style_start = len(p)
p += struct.pack('<III', 0, 0, 2) + b'\xff' * 12
struct.pack_into('<I', p, 4, len(p))
struct.pack_into('<I', p, 16, (0x100 if utf8 else 0) | 1)
struct.pack_into('<I', p, 12, 1)
struct.pack_into('<II', p, 20, old_start + 4, style_start)
result, idx = vr._append_string(bytes(p), vr.LAUNCHER)
self.assertEqual(frame_apk._string_pool(result, 0), ['styled', vr.LAUNCHER])
new_style = struct.unpack_from('<I', result, 24)[0]
self.assertEqual(result[new_style:], p[style_start:])
self.assertEqual(len(result) % 4, 0)
def test_detection(self):
names = {'lib/arm64-v8a/' + n for n in ('libvrapi.so', 'libopenxr_loader.so', 'libovrplatformloader.so')}
info = vr.detection(manifest(category='android.intent.category.LAUNCHER'), names)
self.assertTrue(info['vr'])
self.assertTrue(info['launchable'])
self.assertEqual(len(info['vr_issues']), 3)
self.assertFalse(vr.detection(manifest(category=vr.LAUNCHER), set())['vr'])
self.assertTrue(vr.detection(manifest(category=vr.LAUNCHER, samsung=True), set())['vr'])
def test_key_cache(self):
with patch.object(signing, '_prime', side_effect=AssertionError('regenerated')):
self.assertEqual(signing.signing_key(self.keypath), self.key)
if os.name != 'nt':
self.assertEqual(self.keypath.stat().st_mode & 0o777, 0o600)
def test_repack_sign_tamper(self):
with tempfile.TemporaryDirectory() as d:
src, dst = Path(d) / 'in.apk', Path(d) / 'out.apk'
with zipfile.ZipFile(src, 'w') as z:
z.writestr('AndroidManifest.xml', manifest(), compress_type=8)
z.writestr('classes.dex', b'compress me' * 10000, compress_type=8)
z.writestr('resources.arsc', b'1234')
z.writestr('lib/arm64-v8a/libx.so', b'ELF' * 500000)
z.writestr('META-INF/OLD.RSA', b'old')
z.writestr('META-INF/MANIFEST.MF', b'old')
with patch.object(signing, 'signing_key', return_value=self.key):
result = frame_android.patch(src, dst, {'assets/layer.json': b'{}', 'lib/arm64-v8a/liblayer.so': b'layer'})
self.assertEqual(result['patched'], ['launcher'])
self.assertTrue(frame_apk.apk_info(dst)['launchable'])
self.assertTrue(signing.verify(dst))
def compressed(path, info):
data = path.read_bytes()
nl, el = struct.unpack_from('<HH', data, info.header_offset + 26)
start = info.header_offset + 30 + nl + el
return data[start:start + info.compress_size], start
with zipfile.ZipFile(src) as a, zipfile.ZipFile(dst) as b:
self.assertNotIn('META-INF/OLD.RSA', b.namelist())
self.assertNotIn('META-INF/MANIFEST.MF', b.namelist())
for i in b.infolist():
raw, start = compressed(dst, i)
if i.compress_type == 0:
self.assertEqual(start % (16384 if i.filename.endswith('.so') else 4), 0)
if i.filename in ('classes.dex', 'resources.arsc', 'lib/arm64-v8a/libx.so'):
self.assertEqual(raw, compressed(src, a.getinfo(i.filename))[0])
_, off = compressed(dst, b.getinfo('resources.arsc'))
original = dst.read_bytes()
data = bytearray(original)
eo, cd = signing._eocd(data)
size = struct.unpack_from('<Q', data, cd - 24)[0]
block_start = cd - size - 8
value = data[block_start + 20:cd - 24]
signer = signing._parts(signing._parts(value)[0])[0]
signed, signatures, pub = signing._parts(signer)
signature = signing._parts(signing._parts(signatures)[0][4:])[0]
sig_offset = data.index(signature, block_start)
data[sig_offset] ^= 1
dst.write_bytes(data)
with self.assertRaisesRegex(ValueError, 'RSA signature'):
signing.verify(dst)
data = bytearray(original)
data[off] ^= 1
dst.write_bytes(data)
with self.assertRaisesRegex(ValueError, 'digest'):
signing.verify(dst)
@unittest.skipUnless(Path('/usr/bin/openssl').exists(), 'openssl absent')
def test_openssl(self):
with tempfile.TemporaryDirectory() as d:
d = Path(d)
(d / 'cert.der').write_bytes(signing.certificate(self.key))
(d / 'message').write_bytes(b'independent signature check')
(d / 'signature').write_bytes(signing.rsa_sign(b'independent signature check', self.key))
def run(*args):
return subprocess.run(['/usr/bin/openssl', *args], check=True, capture_output=True).stdout
run('x509', '-inform', 'DER', '-in', str(d / 'cert.der'), '-out', str(d / 'cert.pem'))
run('verify', '-check_ss_sig', '-CAfile', str(d / 'cert.pem'), str(d / 'cert.pem'))
(d / 'pub.pem').write_bytes(run('x509', '-in', str(d / 'cert.pem'), '-pubkey', '-noout'))
output = run('dgst', '-sha256', '-verify', str(d / 'pub.pem'), '-signature', str(d / 'signature'), str(d / 'message'))
self.assertIn(b'Verified OK', output)
def test_install_auto_and_override(self):
base = {'package': 'org.test.vr', 'label': 'VR', 'abis': [], 'min_sdk': None,
'vr': True, 'vr_activity': True, 'launchable': False, 'repairable': True}
with patch.object(frame_android, 'apk_info', return_value=base), \
patch.object(frame_android, 'xr_compat_files', return_value={}), \
patch.object(frame_android, 'patch', return_value={'patched': ['launcher']}) as repair, \
patch.object(frame_android, '_install', return_value={}) as install:
frame_android.install('original.apk')
repair.assert_called_once()
self.assertFalse(install.call_args.args[3])
self.assertEqual(install.call_args.args[1]['patched'], ['launcher'])
frame_android.install('original.apk', flatscreen=True)
self.assertTrue(install.call_args.args[3])
def test_xr_compat_layer(self):
with tempfile.TemporaryDirectory() as d:
apk = Path(d) / 'a.apk'
with zipfile.ZipFile(apk, 'w') as z:
z.writestr('lib/arm64-v8a/libopenxr_loader.so', b'')
add = frame_android.xr_compat_files(str(apk))
self.assertEqual(set(add), set(frame_android.XR_COMPAT_FILES))
self.assertIn(b'XR_APILAYER_FRAME_compat', add['assets/openxr/1/api_layers/implicit.d/XrApiLayer_FRAME_compat.json'])
self.assertTrue(add['lib/arm64-v8a/libXrApiLayer_FRAME_compat.so'].startswith(b'\x7fELF'))
with zipfile.ZipFile(apk, 'a') as z: # already injected: nothing more to add
z.writestr('lib/arm64-v8a/libXrApiLayer_FRAME_compat.so', b'')
self.assertEqual(frame_android.xr_compat_files(str(apk)), {})
flat = Path(d) / 'flat.apk'
with zipfile.ZipFile(flat, 'w') as z:
z.writestr('classes.dex', b'')
self.assertEqual(frame_android.xr_compat_files(str(flat)), {})
def test_xr_compat_layer_missing(self):
with tempfile.TemporaryDirectory() as d:
apk = Path(d) / 'a.apk'
with zipfile.ZipFile(apk, 'w') as z:
z.writestr('lib/arm64-v8a/libopenxr_loader.so', b'')
with patch.object(frame_android, 'XR_COMPAT', d):
with self.assertRaisesRegex(frame_android.FrameError, 'build.sh'):
frame_android.xr_compat_files(str(apk))
def test_patch_rejects_corrupt_manifest_cleanly(self):
with patch.object(frame_android, 'apk_info', side_effect=struct.error('bad')):
with self.assertRaises(frame_android.FrameError):
frame_android.patch('x.apk', 'y.apk')
def test_install_adds_layer_to_vr_apps(self):
base = {'package': 'org.test.vr', 'label': 'VR', 'abis': [], 'min_sdk': None,
'vr': True, 'vr_activity': True, 'launchable': True, 'repairable': False}
layer = {'x': b''}
with patch.object(frame_android, 'apk_info', return_value=dict(base)), \
patch.object(frame_android, 'xr_compat_files', return_value=layer), \
patch.object(frame_android, 'patch', return_value={'patched': ['openxr-compat']}) as repair, \
patch.object(frame_android, '_install', return_value={}) as install:
frame_android.install('game.apk')
self.assertIs(repair.call_args.args[2], layer)
self.assertEqual(install.call_args.args[1]['patched'], ['openxr-compat'])
repair.reset_mock()
frame_android.install('game.apk', xr_compat=False) # launchable, no layer: install as is
repair.assert_not_called()
if __name__ == '__main__':
unittest.main()
tests/test_macview.py
+421
View File
@@ -0,0 +1,421 @@
"""Mac in the headset (ui/frame_macview.py and the frame-mac-view agent).
The Python checks run anywhere. On macOS the agent is built and driven over
HTTP and WebSocket with its test pattern, which needs no Screen Recording
permission: status, the token, the viewer page, H.264 keyframes, pointer
input reaching the source, and closing.
Run: python3 -m unittest discover -s tests
"""
import base64
import http.client
import json
import os
import shutil
import socket
import struct
import subprocess
import sys
import tempfile
import time
import threading
import unittest
from unittest import mock
from pathlib import Path
ROOT = Path(__file__).resolve().parent.parent
sys.path.insert(0, str(ROOT / "ui"))
import frame_macview # noqa: E402
class Helpers(unittest.TestCase):
def test_panel_id_is_stable_and_in_range(self):
a = frame_macview.panel_id("window:123")
self.assertEqual(a, frame_macview.panel_id("window:123"))
self.assertNotEqual(a, frame_macview.panel_id("window:124"))
self.assertTrue(2_001_000_000 <= a < 2_002_000_000)
def test_fit_keeps_aspect_inside_the_panel(self):
self.assertEqual(frame_macview.fit(2560, 1440), (1920, 1080))
w, h = frame_macview.fit(800, 1600)
self.assertEqual(h, 1080)
self.assertAlmostEqual(w / h, 0.5, places=2)
@unittest.skipUnless(shutil.which("bash"), "needs bash")
@unittest.skipIf(os.name == "nt", "Windows' bash.exe is WSL's launcher, and runners have no distribution")
def test_launch_script_parses(self):
r = subprocess.run(["bash", "-n"], input=frame_macview.LAUNCH, text=True, capture_output=True)
self.assertEqual(r.returncode, 0, r.stderr)
def test_show_checks_the_source_before_anything_else(self):
mv = frame_macview.MacView(["ssh"], lambda *a, **k: self.fail("no ssh"), "frame")
with self.assertRaises(frame_macview.MacViewError):
mv.show("rm -rf /")
def test_missing_browser_is_explained(self):
calls = []
def run(remote, stdin=None, timeout=30):
calls.append(remote)
e = RuntimeError("exit 3")
e.stdout = "NO_BROWSER\n"
raise e
mv = frame_macview.MacView(["ssh"], run, "frame")
mv.call = lambda path, **kw: {"screen": True, "ticket": "tk"}
mv.ensure_tunnel = lambda **kw: None
mv.remote_port = 47900
with self.assertRaises(frame_macview.MacViewError) as cm:
mv.show("window:5")
self.assertIn("Chromium", str(cm.exception))
self.assertTrue(calls[0].startswith("bash -s -- "))
def test_separate_windows_need_accessibility(self):
mv = frame_macview.MacView(["ssh"], lambda *a, **k: self.fail("no ssh"), "frame")
mv.call = lambda path, **kw: {"screen": True, "accessibility": False}
with self.assertRaises(frame_macview.MacViewError) as cm:
mv.show("separate:42")
self.assertIn("Accessibility", str(cm.exception))
def test_screen_permission_is_checked_before_the_headset(self):
mv = frame_macview.MacView(["ssh"], lambda *a, **k: self.fail("no ssh"), "frame")
mv.call = lambda path, **kw: {"screen": False}
with self.assertRaises(frame_macview.MacViewError) as cm:
mv.show("display:1")
self.assertIn("Screen Recording", str(cm.exception))
def test_stop_all_ends_the_viewer_browser_unless_shown_again(self):
calls = []
mv = frame_macview.MacView(["ssh"], lambda remote, **kw: calls.append(remote) or "", "frame")
mv.agent = mock.Mock(poll=lambda: None)
mv.call = lambda path, **kw: {"closed": 1}
mv.shown = {"window:5"}
with mock.patch.object(frame_macview.time, "sleep"):
gen = mv.shows
mv.shown.clear()
mv._end_viewer_browser(gen)
self.assertEqual(len(calls), 1)
self.assertIn("pkill -f '[f]rame-control/mac-view", calls[0])
mv.shows += 1 # Show pressed during the wait: leave the new viewer alone
mv._end_viewer_browser(gen)
self.assertEqual(len(calls), 1)
mv.launching = 1 # a Show replacing its own stream is still launching
mv._end_viewer_browser(mv.shows)
self.assertEqual(len(calls), 1)
# A Show waits while the cleanup checks and runs pkill.
mv.launching = 0
in_pkill, release, entered = threading.Event(), threading.Event(), threading.Event()
def blocking_pkill(remote, **kw):
in_pkill.set()
release.wait(5)
mv.run = blocking_pkill
mv._show = lambda *a: entered.set() or "shown"
with mock.patch.object(frame_macview.time, "sleep"):
cleanup = threading.Thread(target=mv._end_viewer_browser, args=(mv.shows,))
cleanup.start()
self.assertTrue(in_pkill.wait(2))
shower = threading.Thread(target=mv.show, args=("window:5",))
shower.start()
self.assertFalse(entered.wait(0.3), "Show started while the cleanup held the lock")
release.set()
self.assertTrue(entered.wait(2))
cleanup.join()
shower.join()
def test_tunnel_prefers_the_usb_c_network_when_plugged_in(self):
mv = frame_macview.MacView(["ssh"], lambda remote, **kw: "13: usb0 inet 10.86.200.233/29 scope global", "frame")
ssh_g = mock.Mock(stdout="user steamos\nhostname frame.example.ts.net\n")
with mock.patch.object(frame_macview.socket, "create_connection") as conn, \
mock.patch.object(frame_macview.subprocess, "run", return_value=ssh_g):
self.assertEqual(mv._usb_route(), ["-o", "HostName=10.86.200.233", "-o", "HostKeyAlias=frame.example.ts.net"])
conn.assert_called_once_with(("10.86.200.233", 22), timeout=1)
ssh_g.stdout = "hostname frame.example.ts.net\nhostkeyalias paired-frame\n" # a configured alias wins
conn.side_effect = None
self.assertIn("HostKeyAlias=paired-frame", mv._usb_route())
conn.side_effect = OSError("unplugged")
self.assertEqual(mv._usb_route(), [])
mv.run = lambda remote, **kw: "" # no usb0
self.assertEqual(mv._usb_route(), [])
mv.prefer_usb = False
mv.run = lambda remote, **kw: self.fail("no ssh when USB is off")
self.assertEqual(mv._usb_route(), [])
def test_a_failed_usb_tunnel_falls_back_to_the_network(self):
mv = frame_macview.MacView(["ssh"], lambda *a, **k: "", "frame")
mv._usb_route = lambda: ["-o", "HostName=10.86.200.233"]
tried = []
def attempt(via, ports):
tried.append(list(via))
mv._last_tunnel_error = "Connection refused"
return not via # USB fails, the normal path works
mv._open_tunnel = attempt
mv.tunnel_up = lambda: False
mv.ensure_tunnel()
self.assertEqual(tried, [["-o", "HostName=10.86.200.233"], []])
self.assertEqual(mv.route, "network")
class WS:
"""A minimal WebSocket client (masked frames out, plain frames in)."""
def __init__(self, port, path):
self.s = socket.create_connection(("127.0.0.1", port), timeout=10)
key = base64.b64encode(os.urandom(16)).decode()
self.s.sendall(f"GET {path} HTTP/1.1\r\nHost: x\r\nUpgrade: websocket\r\nConnection: Upgrade\r\n"
f"Sec-WebSocket-Key: {key}\r\nSec-WebSocket-Version: 13\r\n\r\n".encode())
head = b""
while b"\r\n\r\n" not in head:
head += self.s.recv(1)
self.status = int(head.split()[1])
self.buf = b""
def _read(self, n):
while len(self.buf) < n:
chunk = self.s.recv(65536)
if not chunk:
raise EOFError
self.buf += chunk
out, self.buf = self.buf[:n], self.buf[n:]
return out
def recv(self):
b0, b1 = self._read(2)
n = b1 & 0x7F
if n == 126:
n = struct.unpack(">H", self._read(2))[0]
elif n == 127:
n = struct.unpack(">Q", self._read(8))[0]
return b0 & 0x0F, self._read(n)
def send_text(self, text):
data, mask = text.encode(), os.urandom(4)
head = bytes([0x81, 0x80 | len(data)]) if len(data) < 126 else bytes([0x81, 0xFE]) + struct.pack(">H", len(data))
self.s.sendall(head + mask + bytes(c ^ mask[i % 4] for i, c in enumerate(data)))
def close(self):
self.s.close()
@unittest.skipUnless(sys.platform == "darwin" and shutil.which("xcrun"), "the agent is macOS-only")
class Agent(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.tmp = tempfile.mkdtemp()
cls.bin = Path(cls.tmp) / "frame-mac-view"
r = subprocess.run(["/bin/sh", str(ROOT / "mac" / "frame-mac-view" / "build.sh"), str(cls.bin)],
capture_output=True, text=True, timeout=600)
if r.returncode: # a real failure on a Mac with Xcode: don't hide it as a skip
raise AssertionError("agent didn't build:\n" + (r.stderr or r.stdout)[-2000:])
cls.token = "t0ken-" + os.urandom(6).hex()
cls.proc = subprocess.Popen([str(cls.bin), "serve", "--port", "0", "--page", str(ROOT / "ui" / "mac-view.html"),
"--exit-on-eof"], env={**os.environ, "FRAME_MAC_VIEW_TOKEN": cls.token},
stdin=subprocess.PIPE, stdout=subprocess.PIPE, text=True)
line = cls.proc.stdout.readline()
cls.port = int(line.rsplit(":", 1)[1])
@classmethod
def tearDownClass(cls):
cls.proc.stdin.close() # --exit-on-eof
cls.proc.stdout.close()
try:
cls.proc.wait(5)
except subprocess.TimeoutExpired:
cls.proc.kill()
shutil.rmtree(cls.tmp, ignore_errors=True)
def get(self, path, method="GET"):
c = http.client.HTTPConnection("127.0.0.1", self.port, timeout=10)
c.request(method, path)
r = c.getresponse()
return r.status, r.read()
def test_token_is_required(self):
self.assertEqual(self.get("/status")[0], 403)
self.assertEqual(self.get("/status?k=wrong")[0], 403)
status, body = self.get(f"/status?k={self.token}")
self.assertEqual(status, 200)
self.assertIn("screen", json.loads(body))
def test_serves_the_viewer_page(self):
status, body = self.get(f"/view?k={self.token}&src=test")
self.assertEqual(status, 200)
self.assertIn(b"VideoDecoder", body)
def test_snapshot_needs_the_key(self):
self.assertEqual(self.get("/snapshot?display=1")[0], 403)
def test_separate_without_accessibility_explains(self):
if json.loads(self.get(f"/status?k={self.token}")[1])["accessibility"]:
self.skipTest("this Mac allows Accessibility here")
ws = WS(self.port, f"/stream?k={self.token}&src=separate:1")
self.assertEqual(ws.status, 101)
for _ in range(5):
op, data = ws.recv()
msg = json.loads(data) if op == 1 else {}
if msg.get("t") in ("error", "closed"):
break
self.assertIn("Accessibility", msg.get("message", msg.get("reason", "")))
ws.close()
def test_lists_displays(self):
status, body = self.get(f"/displays?k={self.token}")
self.assertEqual(status, 200)
self.assertIsInstance(json.loads(body)["displays"], list)
def ticket(self, src):
status, body = self.get(f"/ticket?k={self.token}&src={src}", method="POST")
self.assertEqual(status, 200)
return json.loads(body)["ticket"]
def test_ping_and_page_are_open_but_streams_are_not(self):
self.assertEqual(self.get("/ping"), (200, b"frame-mac-view"))
self.assertEqual(WS(self.port, "/stream?src=test").status, 403)
self.assertEqual(self.get("/ticket?src=test", method="POST")[0], 403)
def test_tickets_are_single_use_and_tied_to_one_source(self):
t = self.ticket("test")
self.assertEqual(WS(self.port, f"/stream?src=display:1&t={t}").status, 403) # wrong source
ws = WS(self.port, f"/stream?src=test&t={t}")
self.assertEqual(ws.status, 101)
hello = json.loads(ws.recv()[1])
self.assertEqual(hello["t"], "hello")
# Until the viewer acknowledges, a retry (the hello got lost) gets the
# same key, and replaces the first viewer rather than adding one.
retry = WS(self.port, f"/stream?src=test&t={t}")
self.assertEqual(retry.status, 101)
self.assertEqual(json.loads(retry.recv()[1])["r"], hello["r"])
time.sleep(0.3)
_, body = self.get(f"/status?k={self.token}")
self.assertEqual(len(json.loads(body)["streams"]), 1)
ws.close()
ws = retry
ws.send_text(json.dumps({"t": "ack"}))
time.sleep(0.3)
self.assertEqual(WS(self.port, f"/stream?src=test&t={t}").status, 403) # spent
again = WS(self.port, f"/stream?src=test&r={hello['r']}") # the viewer reconnecting
self.assertEqual(again.status, 101)
again.close()
ws.close()
# Stop revokes the reconnect key.
self.get(f"/close?k={self.token}&src=test", method="POST")
self.assertEqual(WS(self.port, f"/stream?src=test&r={hello['r']}").status, 403)
def test_stop_revokes_tickets_not_yet_used(self):
t = self.ticket("test")
self.get(f"/close?k={self.token}&src=test", method="POST")
self.assertEqual(WS(self.port, f"/stream?src=test&t={t}").status, 403)
def test_bad_frame_lengths_close_the_socket_not_the_agent(self):
ws = WS(self.port, f"/stream?src=test&t={self.ticket('test')}")
self.assertEqual(ws.status, 101)
# A masked frame claiming 2^63 bytes.
ws.s.sendall(bytes([0x81, 0xFF]) + struct.pack(">Q", 1 << 63) + os.urandom(4))
with self.assertRaises((EOFError, OSError)):
for _ in range(1000):
ws.recv()
ws.close()
self.assertEqual(self.get(f"/status?k={self.token}")[0], 200)
def test_stream_input_and_close(self):
ws = WS(self.port, f"/stream?k={self.token}&src=test&codec=h264&fps=30&max=640")
self.assertEqual(ws.status, 101)
info = None
key = None
for _ in range(200):
op, data = ws.recv()
if op == 1:
msg = json.loads(data)
if msg["t"] == "hello":
continue
if msg["t"] == "error":
self.skipTest("no H.264 encoder here: " + msg["message"])
if msg["t"] == "info":
info = msg
elif op == 2 and data[0] == 1:
key = data
if info and key:
break
self.assertEqual(info["src"], "test")
self.assertTrue(info["input"])
# A keyframe: flags, 8-byte timestamp, sequence number, input echoed,
# then Annex B with the SPS first.
self.assertEqual(key[17:21], b"\x00\x00\x00\x01")
self.assertEqual(key[21] & 0x1F, 7) # NAL type 7, SPS
ws.send_text(json.dumps({"t": "m", "e": "down", "b": 0, "x": 0.5, "y": 0.5}))
ws.send_text(json.dumps({"t": "key-frame"}))
got_key = False
for _ in range(200):
op, data = ws.recv()
if op == 2 and data[0] == 1:
got_key = True
break
self.assertTrue(got_key, "no keyframe after asking for one")
_, body = self.get(f"/status?k={self.token}")
self.assertEqual([s["src"] for s in json.loads(body)["streams"]], ["test"])
status, body = self.get(f"/close?k={self.token}", method="POST")
self.assertEqual(json.loads(body)["closed"], 1)
for _ in range(400):
op, data = ws.recv()
if op == 1 and json.loads(data)["t"] == "close":
break
self.assertEqual(json.loads(data)["t"], "close")
# Without the viewer doing anything, the agent ends the stream itself.
time.sleep(1)
_, body = self.get(f"/status?k={self.token}")
self.assertEqual(json.loads(body)["streams"], [])
ws.close()
def test_timing_reports_and_input_echo(self):
# What a viewer does: sync clocks, report each frame, stamp input.
ws = WS(self.port, f"/stream?k={self.token}&src=test&codec=h264&fps=30&max=640")
self.assertEqual(ws.status, 101)
ws.send_text(json.dumps({"t": "w"})) # keep-warm filler: ignored
ws.send_text(json.dumps({"t": "ping", "c": 1.5}))
pong, echoed, seqs, info = None, None, [], None
clicked = False
deadline = time.time() + 10
while time.time() < deadline and not (pong and echoed):
op, data = ws.recv()
if op == 1:
msg = json.loads(data)
if msg["t"] == "error":
self.skipTest("no H.264 encoder here: " + msg["message"])
if msg["t"] == "pong":
pong = msg
if msg["t"] == "info":
info = msg
elif op == 2:
seq, echo = struct.unpack(">II", data[9:17])
seqs.append(seq)
now = pong["a"] if pong else 0
ws.send_text(json.dumps({"t": "rx", "s": seq, "r": now}))
ws.send_text(json.dumps({"t": "fd", "f": [[seq, now + 1, now + 2, now + 3]], "drop": 0}))
if echo == 7:
echoed = seq
if len(seqs) == 3 and not clicked:
ws.send_text(json.dumps({"t": "m", "e": "down", "b": 0, "x": 0.5, "y": 0.5, "i": 7, "tv": now}))
clicked = True
self.assertEqual(pong["c"], 1.5)
self.assertGreater(pong["a"], 0)
self.assertEqual(seqs[:3], sorted(seqs[:3]))
self.assertIsNotNone(echoed, "no frame was tagged as the first reply to the click")
time.sleep(1.7) # frames are reported once the viewer has had time
stream = json.loads(self.get(f"/stats?k={self.token}")[1])["streams"][0]
first = stream["frames"][0]
self.assertGreater(first["e1"], first["e0"])
self.assertGreaterEqual(first["e0"], first["cap"])
self.assertEqual(first["vs"] - first["rx"], 3)
self.assertEqual([i["frame"] for i in stream["inputs"] if i["id"] == 7], [echoed])
self.assertIn("total", stream["summary"])
self.assertEqual(info["warm"], 0) # off unless FRAME_MAC_VIEW_WARM is set
live = json.loads(self.get(f"/status?k={self.token}")[1])["streams"][0]
self.assertEqual(live["controller"]["tier"], 0)
self.assertIn("fps", live["stats"])
ws.close()
if __name__ == "__main__":
unittest.main()
ui/apk_sources/__init__.py
+45
View File
@@ -0,0 +1,45 @@
"""APK sources: every place Frame Control can find and download APKs.
One module per source kind in this package. Each module exposes the same small
interface so ``search.py`` can query them all and show where every result came
from. Python stdlib only; must run on Python 3.9.
Module interface
----------------
KIND = 'sidequest' # stable id of the source kind
def sources() -> list[dict] # configured sources of this kind (a kind can have
# several, e.g. one per F-Droid-format repo)
def search(source, query, limit=50) -> list[dict] # Entry dicts, best first
def details(source, entry_id) -> dict # Entry with 'versions'
def download(source, entry_id, version_code=None) -> dict
# {'apk': local path, 'obb': [paths], 'sha256': hex or None, 'verified': bool}
# Raise SourceError with a user-readable message on failure.
Source dict
-----------
{'id': 'sidequest', 'kind': KIND, 'name': 'SideQuest', 'url': 'https://...',
'builtin': True, 'enabled': True, 'trust': 'official' | 'community' | 'user'}
Entry dict (missing facts are None, never guessed)
----------
{'source': source id, 'id': source-local id, 'package': 'org.example.app' or None,
'name': str, 'summary': str, 'icon': url or None, 'page': url or None,
'version': '1.2', 'version_code': 12, 'min_sdk': 24, 'abis': ['arm64-v8a'],
'vr': True/False/None, 'size': bytes, 'free': True, 'license': 'GPL-3.0' or None,
'updated': 'YYYY-MM-DD', 'downloadable': bool, # False = open page only
'versions': [ {version, version_code, min_sdk, size, updated}, ... ]} # details() only
Rules
-----
- Only sources that distribute APKs with the developer's consent: free listings,
never paid apps re-hosted, no licence or entitlement workarounds.
- Honour each site's terms and robots rules; if automated download isn't allowed,
return entries with 'downloadable': False and a 'page' link instead.
- Cache indexes under frame_host.cache_dir('apk-sources'); send a clear
User-Agent ('FrameControl/<version>'); keep requests modest.
- Tests use recorded fixtures, never the network.
"""
class SourceError(Exception):
"""User-readable failure from a source (network, format, verification)."""
ui/frame_android.py
+90 -7
View File
@@ -6,12 +6,18 @@ apps, the lepton-show-flatscreen marker; plus a non-Steam shortcut, so it shows
in the Steam library and gets its own SteamVR panel. Nothing goes through
Lepton Development, which wipes its apps on exit. See docs/apks.md.
Python stdlib only. CLI: python3 ui/frame_android.py {info APK|versions APK-or-PKG|install APK|list|launch PKG|stop PKG|remove PKG|probe PKG}
Python stdlib only. CLI: python3 ui/frame_android.py
install APK [--vr|--flat] [--no-xr-compat] | info APK | versions APK-or-PKG
patch SRC DST [--add NAME=PATH ...] | list | launch PKG | stop PKG | remove PKG | probe PKG
"""
import json, os, re, shlex, shutil, subprocess, sys, threading, time, zlib
import json, os, re, shlex, shutil, struct, subprocess, sys, threading, time, zlib
import frame_apk
import frame_host
import tempfile
import zipfile
from frame_apk_vr import add_launcher_category
from frame_apk_sign import repack
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
FRAME = os.environ.get('FRAME_ALIAS', 'frame')
@@ -20,6 +26,13 @@ COMPAT = '.local/share/Steam/steamapps/compatdata'
SHADERS = '.local/share/Steam/steamapps/shadercache'
LAUNCHER = os.path.join(ROOT, 'frame', 'android', 'lepton-app.sh')
SHORTCUTS = os.path.join(ROOT, 'frame', 'android', 'steam_shortcuts.py')
# OpenXR API layer that lets OpenXR 1.1 apps run on SteamVR's 1.0-only Android
# runtime (frame/openxr-compat, docs/vr-apks.md). Injected into VR APKs.
XR_COMPAT = os.path.join(ROOT, 'frame', 'openxr-compat')
XR_COMPAT_FILES = {
'assets/openxr/1/api_layers/implicit.d/XrApiLayer_FRAME_compat.json': 'XrApiLayer_FRAME_compat.json',
'lib/arm64-v8a/libXrApiLayer_FRAME_compat.so': 'prebuilt/arm64-v8a/libXrApiLayer_FRAME_compat.so',
}
PKG_RE = re.compile(r'^[A-Za-z][\w]*(\.[A-Za-z_][\w]*)+$')
SSH_OPTS = ['-o', 'BatchMode=yes', '-o', 'ConnectTimeout=8']
@@ -65,6 +78,22 @@ def apk_info(path):
raise FrameError(f'{os.path.basename(path)}: {e}')
def xr_compat_files(apk_path):
"""The layer's files to add, or {} if the APK has no OpenXR loader or already has the layer."""
with zipfile.ZipFile(apk_path) as z:
names = set(z.namelist())
if 'lib/arm64-v8a/libopenxr_loader.so' not in names or names & set(XR_COMPAT_FILES):
return {}
add = {}
for entry, rel in XR_COMPAT_FILES.items():
try:
with open(os.path.join(XR_COMPAT, rel), 'rb') as f:
add[entry] = f.read()
except OSError:
raise FrameError("the OpenXR compatibility layer isn't built; run frame/openxr-compat/build.sh")
return add
def check_installable(info):
if info['min_sdk'] and info['min_sdk'] > 30:
raise FrameError(f"{info['label']} needs Android API {info['min_sdk']}; Lepton is Android 11 (API 30)")
@@ -111,7 +140,7 @@ def _write_meta(d, meta):
install_hooks = []
def install(apk_path, flatscreen=True, name=None, source=None, icon_png=None):
def install(apk_path, flatscreen=None, name=None, source=None, icon_png=None, xr_compat=None):
start, info = time.time(), None
try:
info = apk_info(apk_path)
@@ -121,8 +150,20 @@ def install(apk_path, flatscreen=True, name=None, source=None, icon_png=None):
pkg = info['package']
if not PKG_RE.match(pkg):
raise FrameError(f'unexpected package name {pkg!r}')
if flatscreen is None:
flatscreen = not info['vr']
# VR apps get the OpenXR compatibility layer unless told otherwise; it only
# changes calls SteamVR would otherwise reject.
add = xr_compat_files(apk_path) if (info['vr'] if xr_compat is None else xr_compat) else {}
with _install_lock:
meta = _install(apk_path, info, pkg, flatscreen, name, source)
if add or info['repairable']:
with tempfile.TemporaryDirectory(prefix='frame-vr-') as tmp:
patched = os.path.join(tmp, 'app.apk')
info['patched'] = patch(apk_path, patched, add)['patched']
info['launchable'] = True
meta = _install(patched, info, pkg, flatscreen, name, source or os.path.basename(apk_path))
else:
meta = _install(apk_path, info, pkg, flatscreen, name, source)
except FrameError as e:
_after_install(info, None, e, start)
raise
@@ -163,6 +204,8 @@ def _install(apk_path, info, pkg, flatscreen, name, source):
raise FrameError(f'Steam did not return a shortcut id (got {reply[:80]!r})')
meta = {'package': pkg, 'label': name or info['label'], 'version': info['version'],
'instance': iid, 'shortcut': shortcut, 'game_id': game_id(shortcut),
'vr': info.get('vr', False), 'vr_issues': info.get('vr_issues', []),
'launchable': info.get('launchable', False), 'patched': info.get('patched', []),
'flatscreen': flatscreen, 'installed': time.strftime('%Y-%m-%dT%H:%M:%S'),
'source': source or os.path.basename(apk_path)}
_write_meta(d, meta)
@@ -293,19 +336,59 @@ def probe(pkg, wait=20):
'container_up': ctr in running_instances()}
def patch(src, dst, add=None):
try:
info = apk_info(src)
with zipfile.ZipFile(src) as z:
original = frame_apk._read(z, 'AndroidManifest.xml', frame_apk.MAX_MANIFEST)
manifest = add_launcher_category(original) if info['repairable'] else original
if not info['launchable'] and not info['repairable']:
raise FrameError('APK has no MAIN/LAUNCHER activity that Frame Control can patch')
repack(src, dst, replace={'AndroidManifest.xml': manifest}, add=add)
result = apk_info(dst)
result.pop('icon_png', None)
result['patched'] = (['launcher'] if manifest != original else []) + \
(['openxr-compat'] if add and set(XR_COMPAT_FILES) <= set(add) else [])
return result
except (OSError, ValueError, IndexError, struct.error, zipfile.BadZipFile, frame_apk.ApkError) as e:
raise FrameError(str(e)) from e
def main():
cmd, *args = sys.argv[1:] or ['help']
try:
if cmd in ('info', 'versions'):
import frame_apk_versions
if cmd == 'info':
print(frame_apk_versions.describe(apk_info(args[0])))
info = apk_info(args[0])
print(frame_apk_versions.describe(info))
fix = '; Frame Control adds the LAUNCHER entry Lepton needs' if info.get('repairable') else ''
if info.get('vr') or fix:
print(('VR app' if info.get('vr') else 'Android app') + fix)
for note in info.get('vr_issues', []):
print(note)
return
info = apk_info(args[0]) if os.path.isfile(args[0]) or args[0].lower().endswith('.apk') else None
r = frame_apk_versions.alternatives(
info['package'] if info else args[0], info.get('version_code') if info else None)
elif cmd == 'install':
r = install(args[0], flatscreen='--vr' not in args)
r = install(args[0], flatscreen=False if '--vr' in args else True if '--flat' in args else None,
xr_compat=False if '--no-xr-compat' in args else None)
elif cmd == 'patch':
import argparse
parser = argparse.ArgumentParser(description='Patch and v2-sign an APK locally')
parser.add_argument('src')
parser.add_argument('dst')
parser.add_argument('--add', action='append', default=[], metavar='NAME=PATH')
opts = parser.parse_args(args)
additions = {}
for item in opts.add:
if '=' not in item:
raise FrameError('--add requires NAME=PATH')
entry, path = item.split('=', 1)
with open(path, 'rb') as f:
additions[entry] = f.read()
r = patch(opts.src, opts.dst, additions)
elif cmd == 'list':
r = list_apps()
elif cmd in ('launch', 'stop', 'probe'):
@@ -314,7 +397,7 @@ def main():
r = remove(args[0], keep_data='--keep-data' in args)
else:
sys.exit(__doc__)
except FrameError as e:
except (FrameError, OSError) as e:
sys.exit(f'error: {e}')
print(json.dumps(r, indent=1))
ui/frame_apk.py
+7 -2
View File
@@ -10,7 +10,8 @@ import zipfile
# android: attribute resource ids; names can be stripped by shrinkers, ids can't.
ATTR = {0x01010001: 'label', 0x01010002: 'icon', 0x01010003: 'name',
0x0101021b: 'versionCode', 0x0101021c: 'versionName', 0x0101020c: 'minSdkVersion'}
0x01010024: 'value', 0x0101021b: 'versionCode', 0x0101021c: 'versionName', 0x0101020c: 'minSdkVersion',
0x01010202: 'targetActivity'}
T_REF, T_STRING, T_INT_DEC, T_INT_HEX = 0x01, 0x03, 0x10, 0x11
# APKs can come from websites (install links), so nothing read from one may be
# unbounded. zipfile stops at a member's declared size, so checking it is enough.
@@ -215,8 +216,11 @@ def apk_info(path):
if 'AndroidManifest.xml' not in names:
raise ApkError('not an APK: no AndroidManifest.xml')
try:
elements = manifest_elements(_read(z, 'AndroidManifest.xml', MAX_MANIFEST))
manifest_data = _read(z, 'AndroidManifest.xml', MAX_MANIFEST)
elements = manifest_elements(manifest_data)
res = Resources(_read(z, 'resources.arsc', MAX_ARSC) if 'resources.arsc' in names else b'')
from frame_apk_vr import detection
vr_info = detection(manifest_data, names)
except (struct.error, IndexError, zipfile.BadZipFile) as e:
raise ApkError(f'could not read the APK manifest: {e}')
tags = {}
@@ -236,6 +240,7 @@ def apk_info(path):
'min_sdk': min_sdk[1] if min_sdk and min_sdk[0] in (T_INT_DEC, T_INT_HEX) else None,
'icon_png': None,
}
info.update(vr_info)
try:
info['icon_png'] = _icon_png(z, names, _icons(app.get('icon'), res))
except Exception: # noqa: BLE001 - any unreadable icon just means no icon
ui/frame_apk_sign.py
+361
View File
@@ -0,0 +1,361 @@
"""Lossless ZIP repacking and APK v2 RSA/SHA-256 signing, Python 3.9 stdlib.
Spec: https://source.android.com/docs/security/features/apksigning/v2
Sections: APK Signing Block; APK Signature Scheme v2 Block; Integrity-protected
contents; Verification. No verity algorithm is used, so no verity padding.
"""
import hashlib
import io
import json
import math
import os
from pathlib import Path
import re
import secrets
import struct
import tempfile
import zipfile
import zlib
import frame_host
MAGIC = b'APK Sig Block 42'
V2 = 0x7109871a
ALG = 0x0103
SHA256_DER = bytes.fromhex('3031300d060960864801650304020105000420')
def u32(n):
return struct.pack('<I', n)
def lp(b):
return u32(len(b)) + b
def der(tag, b):
n = len(b)
length = bytes([n]) if n < 128 else bytes([128 + (n.bit_length() + 7) // 8]) + n.to_bytes((n.bit_length() + 7) // 8, 'big')
return bytes([tag]) + length + b
def integer(n):
b = n.to_bytes((n.bit_length() + 7) // 8 or 1, 'big')
return der(2, (b'\0' if b[0] & 128 else b'') + b)
def sequence(*items):
return der(0x30, b''.join(items))
RSA_ALG = bytes.fromhex('300d06092a864886f70d0101010500')
CERT_ALG = bytes.fromhex('300d06092a864886f70d01010b0500')
def public_key(key):
return sequence(RSA_ALG, der(3, b'\0' + sequence(integer(key['n']), integer(key['e']))))
def encoded_hash(data, size):
digest = SHA256_DER + hashlib.sha256(data).digest()
return b'\0\1' + b'\xff' * (size - len(digest) - 3) + b'\0' + digest
def rsa_sign(data, key):
size = (key['n'].bit_length() + 7) // 8
return pow(int.from_bytes(encoded_hash(data, size), 'big'), key['d'], key['n']).to_bytes(size, 'big')
def rsa_verify(data, sig, n, e):
size = (n.bit_length() + 7) // 8
if len(sig) != size or int.from_bytes(sig, 'big') >= n:
raise ValueError('invalid RSA signature size/value')
actual = pow(int.from_bytes(sig, 'big'), e, n).to_bytes(size, 'big')
if actual != encoded_hash(data, size):
raise ValueError('RSA signature mismatch')
def certificate(key):
name = sequence(der(0x31, sequence(bytes.fromhex('0603550403'), der(12, b'Frame Control APK signer'))))
validity = sequence(der(0x17, b'200101000000Z'), der(0x18, b'21200101000000Z'))
tbs = sequence(der(0xa0, integer(2)), integer(1), CERT_ALG, name, validity, name, public_key(key))
return sequence(tbs, CERT_ALG, der(3, b'\0' + rsa_sign(tbs, key)))
def _prime(bits):
small = (3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47)
while True:
n = secrets.randbits(bits) | (3 << (bits - 2)) | 1
if any(n % p == 0 for p in small) or (n - 1) % 65537 == 0:
continue
d, s = n - 1, 0
while d % 2 == 0:
d //= 2
s += 1
for _ in range(40): # Miller-Rabin error bound <= 2^-80
x = pow(secrets.randbelow(n - 3) + 2, d, n)
if x in (1, n - 1):
continue
for _ in range(s - 1):
x = pow(x, 2, n)
if x == n - 1:
break
else:
break
else:
return n
def signing_key(path=None):
"""Persistent identity in app data, never an evictable cache. Atomic publication.
Hard-linking a fully written private temp file prevents concurrent first-use
callers from selecting different identities or reading a partial key.
"""
path = Path(path) if path is not None else frame_host.data_dir('apk-signing-key.json')
if not path.exists():
p, q = _prime(1024), _prime(1024)
while q == p:
q = _prime(1024)
key = {'n': p * q, 'e': 65537, 'd': pow(65537, -1, math.lcm(p - 1, q - 1))}
path.parent.mkdir(parents=True, exist_ok=True)
fd, tmp = tempfile.mkstemp(prefix='.apk-key-', dir=str(path.parent))
try:
with os.fdopen(fd, 'w') as f:
json.dump(key, f)
f.flush()
os.fsync(f.fileno())
try:
os.link(tmp, path) # never replaces a key another process wrote first
except FileExistsError:
pass
except OSError: # no hard links (FAT/exFAT): plain rename
if not path.exists():
os.replace(tmp, path)
finally:
if os.path.exists(tmp):
os.unlink(tmp)
os.chmod(path, 0o600) # Windows ignores this; the per-user app-data folder is the protection there
key = json.loads(path.read_text())
if key['n'].bit_length() != 2048 or key['e'] != 65537:
raise ValueError(f'invalid cached APK signing key; delete {path} to make a new one '
'(re-signed apps then need reinstalling)')
rsa_verify(b'key check', rsa_sign(b'key check', key), key['n'], key['e'])
return key
def _eocd(data):
# ZIP comments can contain the EOCD signature; accept only an exact EOF fit.
for at in range(len(data) - 22, max(-1, len(data) - 65558), -1):
if data[at:at + 4] == b'PK\5\6' and at + 22 + struct.unpack_from('<H', data, at + 20)[0] == len(data):
disk, cd_disk, count_disk, count, size, cd = struct.unpack_from('<HHHHII', data, at + 4)
if disk or cd_disk or count_disk != count or count == 65535 or cd + size != at:
raise ValueError('multi-disk/ZIP64 or invalid APK directory')
return at, cd
raise ValueError('missing ZIP end record')
def content_digest(sections):
chunks = []
for section in sections:
for off in range(0, len(section), 1024 * 1024):
part = section[off:off + 1024 * 1024]
chunks.append(hashlib.sha256(b'\xa5' + u32(len(part)) + part).digest())
return hashlib.sha256(b'\x5a' + u32(len(chunks)) + b''.join(chunks)).digest()
def sign_apk(data, key):
eo, cd = _eocd(data)
digest = content_digest((memoryview(data)[:cd], memoryview(data)[cd:eo], data[eo:]))
signed = lp(lp(u32(ALG) + lp(digest))) + lp(lp(certificate(key))) + lp(b'')
signer = lp(signed) + lp(lp(u32(ALG) + lp(rsa_sign(signed, key)))) + lp(public_key(key))
value = lp(lp(signer))
pair = struct.pack('<Q', 4 + len(value)) + u32(V2) + value
size = len(pair) + 24
block = struct.pack('<Q', size) + pair + struct.pack('<Q', size) + MAGIC
end = bytearray(data[eo:])
struct.pack_into('<I', end, 16, cd + len(block))
return data[:cd] + block + data[cd:eo] + end
def _parts(data):
result, off = [], 0
while off < len(data):
if off + 4 > len(data):
raise ValueError('truncated length prefix')
size = struct.unpack_from('<I', data, off)[0]
off += 4
if off + size > len(data):
raise ValueError('length prefix outside block')
result.append(data[off:off + size])
off += size
return result
def _der_parts(data):
result, off = [], 0
while off < len(data):
start = off
tag, size = data[off:off + 2]
off += 2
if size & 128:
count = size & 127
if not count or count > 4:
raise ValueError('invalid DER length')
size = int.from_bytes(data[off:off + count], 'big')
off += count
if off + size > len(data):
raise ValueError('truncated DER')
result.append((tag, data[off:off + size], data[start:off + size]))
off += size
return result
def _cert_key(cert):
outer = _der_parts(cert)
if len(outer) != 1 or outer[0][0] != 0x30:
raise ValueError('invalid certificate')
fields = _der_parts(outer[0][1])
tbs = _der_parts(fields[0][1])
spki = tbs[6 if tbs[0][0] == 0xa0 else 5][2]
pub = _der_parts(_der_parts(spki)[0][1])
if pub[0][2] != RSA_ALG or pub[1][1][:1] != b'\0':
raise ValueError('certificate is not RSA')
numbers = _der_parts(_der_parts(pub[1][1][1:])[0][1])
n, e = [int.from_bytes(item[1], 'big') for item in numbers]
return n, e, spki
def verify(path):
"""Verify this v2-only format; return True or raise ValueError on corruption.
A valid signature establishes integrity, not trust in the APK publisher.
"""
data = Path(path).read_bytes()
try:
eo, cd = _eocd(data)
if data[cd - 16:cd] != MAGIC:
raise ValueError('no APK signing block')
size = struct.unpack_from('<Q', data, cd - 24)[0]
start = cd - size - 8
if start < 0 or struct.unpack_from('<Q', data, start)[0] != size:
raise ValueError('invalid signing block size')
off, values = start + 8, []
while off < cd - 24:
length = struct.unpack_from('<Q', data, off)[0]
if length < 4 or off + 8 + length > cd - 24:
raise ValueError('invalid signing pair')
ident = struct.unpack_from('<I', data, off + 8)[0]
if ident == V2:
values.append(data[off + 12:off + 8 + length])
off += 8 + length
if off != cd - 24 or len(values) != 1:
raise ValueError('missing or duplicate v2 signer block')
end = bytearray(data[eo:])
struct.pack_into('<I', end, 16, start)
digest = content_digest((memoryview(data)[:start], memoryview(data)[cd:eo], end))
wrappers = _parts(values[0])
if len(wrappers) != 1:
raise ValueError('invalid signers sequence')
signers = _parts(wrappers[0])
if not signers:
raise ValueError('no signers')
for signer in signers:
signed, signatures, pub = _parts(signer)
digests, certs, attrs = _parts(signed)
cert = _parts(certs)[0]
n, e, cert_pub = _cert_key(cert)
if cert_pub != pub:
raise ValueError('public key differs from certificate')
sigs, digs = _parts(signatures), _parts(digests)
if len(sigs) != 1 or len(digs) != 1 or sigs[0][:4] != u32(ALG) or digs[0][:4] != u32(ALG):
raise ValueError('unsupported signature algorithm')
if _parts(digs[0][4:]) != [digest]:
raise ValueError('APK content digest mismatch')
sig = _parts(sigs[0][4:])
if len(sig) != 1:
raise ValueError('invalid signature sequence')
rsa_verify(signed, sig[0], n, e)
return True
except (IndexError, struct.error, OverflowError) as exc:
raise ValueError('malformed APK signature: ' + str(exc)) from exc
def repack(src, dst, replace=None, add=None, sign=True):
"""Copy raw compressed members; reconstruct ZIP headers without descriptors.
Reject ZIP64/encrypted archives. Output is atomically replaced after signing.
"""
replace, add = dict(replace or {}), dict(add or {})
central, output = [], io.BytesIO()
with open(src, 'rb') as raw, zipfile.ZipFile(raw) as archive:
names = archive.namelist()
if len(set(names)) != len(names):
raise ValueError('duplicate ZIP member names')
if set(replace) - set(names) or set(add) & set(names) or set(add) & set(replace):
raise ValueError('replace must exist and add must be new')
items = archive.infolist() + [zipfile.ZipInfo(name) for name in add]
for info in items:
name = info.filename
if re.match(r'^META-INF/(?:[^/]+\.(?:SF|RSA|EC|DSA)|MANIFEST\.MF)$', name, re.I):
continue
if info.flag_bits & 1 or info.compress_type not in (0, 8):
raise ValueError('unsupported ZIP encryption/compression')
method = info.compress_type
content = add.get(name) if name in add else replace.get(name)
if content is None:
raw.seek(info.header_offset)
header = raw.read(30)
if header[:4] != b'PK\3\4':
raise ValueError('invalid local ZIP header')
nl, el = struct.unpack_from('<HH', header, 26)
raw.seek(nl + el, 1)
compressed = raw.read(info.compress_size)
crc, usize = info.CRC, info.file_size
if len(compressed) != info.compress_size:
raise ValueError('truncated ZIP member')
else:
crc, usize = zlib.crc32(content), len(content)
if method == 8:
compressor = zlib.compressobj(6, zlib.DEFLATED, -15)
compressed = compressor.compress(content) + compressor.flush()
else:
compressed = content
encoded = name.encode('utf-8')
offset = output.tell()
align = 16384 if name.endswith('.so') else 4
needs_alignment = method == 0 or name.endswith('.so')
padding = (-(offset + 30 + len(encoded) + 6) % align) if needs_alignment else 0
# Android zipalign extra: alignment (uint16), then padding bytes.
extra = struct.pack('<HHH', 0xd935, padding + 2, align) + bytes(padding) if needs_alignment else b''
dt = info.date_time
dos_time = (dt[3] << 11) | (dt[4] << 5) | (dt[5] // 2)
dos_date = ((dt[0] - 1980) << 9) | (dt[1] << 5) | dt[2]
csize = len(compressed)
if max(offset, csize, usize) >= 0xffffffff:
raise ValueError('ZIP64 APKs are unsupported')
output.write(struct.pack('<IHHHHHIIIHH', 0x04034b50, 20, 0x800, method, dos_time, dos_date,
crc, csize, usize, len(encoded), len(extra)) + encoded + extra + compressed)
central.append(struct.pack('<IHHHHHHIIIHHHHHII', 0x02014b50, 0x314, 20, 0x800, method,
dos_time, dos_date, crc, csize, usize, len(encoded), 0, len(info.comment),
0, info.internal_attr, info.external_attr, offset) + encoded + info.comment)
cd = output.tell()
directory = b''.join(central)
if len(central) >= 65535 or cd + len(directory) >= 0xffffffff:
raise ValueError('ZIP64 APKs are unsupported')
output.write(directory)
output.write(struct.pack('<IHHHHIIH', 0x06054b50, 0, 0, len(central), len(central), len(directory), cd, len(archive.comment)) + archive.comment)
data = output.getvalue()
if sign:
data = sign_apk(data, signing_key())
dst = Path(dst)
fd, tmp = tempfile.mkstemp(prefix='.apk-', dir=str(dst.parent))
try:
with os.fdopen(fd, 'wb') as f:
f.write(data)
if sign:
verify(tmp)
os.replace(tmp, dst)
finally:
if os.path.exists(tmp):
os.unlink(tmp)
ui/frame_apk_vr.py
+128
View File
@@ -0,0 +1,128 @@
"""Binary-manifest VR inspection and minimal launcher repair (stdlib only)."""
import struct
import frame_apk
MAIN = 'android.intent.action.MAIN'
LAUNCHER = 'android.intent.category.LAUNCHER'
VR = {'com.oculus.intent.category.VR', 'org.khronos.openxr.intent.category.IMMERSIVE_HMD'}
def inspect(data):
elements = iter(frame_apk.manifest_elements(data))
stack, filters, samsung = [], [], False
current, owner, package = None, None, ''
for kind, hs, off, size in frame_apk._chunks(data, 8, len(data)):
if kind == 0x0102:
tag, attrs = next(elements)
value = attrs.get('name', (None, None, None))[2]
if tag == 'manifest':
package = attrs.get('package', (None, None, None))[2] or ''
if tag in ('activity', 'activity-alias'):
owner = attrs.get('targetActivity', (None, None, None))[2] if tag == 'activity-alias' else value
if owner and '.' not in owner: # PackageParser.buildClassName: bare names are relative too
owner = '.' + owner
owner = package + owner if owner and owner.startswith('.') else owner
if tag == 'intent-filter' and stack and stack[-1] in ('activity', 'activity-alias'):
current = {'actions': set(), 'categories': set(), 'templates': [], 'alias': stack[-1] == 'activity-alias', 'activity': owner}
if current is not None and stack and stack[-1] == 'intent-filter':
if tag == 'action':
current['actions'].add(value)
if tag == 'category':
current['categories'].add(value)
current['templates'].append((off, size, hs))
if tag == 'meta-data' and stack and stack[-1] == 'application':
samsung |= value == 'com.samsung.android.vr.application.mode' and attrs.get('value', (0, 0, None))[2] == 'vr_only'
stack.append(tag)
elif kind == 0x0103 and stack:
tag = stack.pop()
if tag == 'intent-filter' and current is not None:
current['end'] = off
filters.append(current)
current = None
mains = [f for f in filters if MAIN in f['actions']]
vr_filters = [f for f in mains if VR & f['categories']]
# Lepton's apk-info-extractor ignores <activity-alias>; Godot 4 puts LAUNCHER only there.
real = [f for f in mains if not f['alias']]
targets = [f for f in real if VR & f['categories']]
if not targets and any(LAUNCHER in f['categories'] or VR & f['categories'] for f in mains if f['alias']):
aimed = {f['activity'] for f in mains if f['alias'] and (LAUNCHER in f['categories'] or VR & f['categories'])}
targets = [f for f in real if f['templates']] # the patch copies an existing <category>
targets = [f for f in targets if f['activity'] in aimed] or targets
launchable = any(LAUNCHER in f['categories'] for f in real)
return {'launchable': launchable, 'repairable': not launchable and bool(targets),
'vr_activity': bool(vr_filters), 'vr': bool(vr_filters) or samsung}, targets
def _append_string(chunk, text):
_, hs, size, count, styles, flags, start, style_start = struct.unpack_from('<HHIIIIII', chunk)
strings_end = style_start or size
encoded = text.encode('utf-8' if flags & 0x100 else 'utf-16-le')
# LAUNCHER is short enough for both single-unit length encodings.
new = (bytes([len(text), len(encoded)]) + encoded + b'\0' if flags & 0x100
else struct.pack('<H', len(text)) + encoded + b'\0\0')
string_data = chunk[start:strings_end] + new
string_data += bytes(-len(string_data) % 4)
header = bytearray(chunk[:hs])
new_start = start + 4
new_styles = new_start + len(string_data) if style_start else 0
body = (chunk[hs:hs + count * 4] + struct.pack('<I', strings_end - start)
+ chunk[hs + count * 4:start] + string_data + (chunk[style_start:] if style_start else b''))
struct.pack_into('<IIIII', header, 8, count + 1, styles, flags & ~1, new_start, new_styles)
struct.pack_into('<I', header, 4, len(header) + len(body))
return bytes(header) + body, count
def add_launcher_category(axml_bytes):
info, filters = inspect(axml_bytes)
if info['launchable']:
return axml_bytes
if not filters:
raise frame_apk.ApkError('no activity with a MAIN intent filter that Frame Control can patch')
target = filters[0]
chunks = list(frame_apk._chunks(axml_bytes, 8, len(axml_bytes)))
pool = next(c for c in chunks if c[0] == 1)
_, _, po, ps = pool
new_pool, index = _append_string(axml_bytes[po:po + ps], LAUNCHER)
off, size, hs = target['templates'][0]
start = bytearray(axml_bytes[off:off + size])
attr_start, attr_size, count = struct.unpack_from('<HHH', start, hs + 8)
strings = frame_apk._string_pool(axml_bytes, po)
resmap = []
for kind, header_size, offset, chunk_size in chunks:
if kind == 0x180:
resmap = struct.unpack_from('<%dI' % ((chunk_size - header_size) // 4),
axml_bytes, offset + header_size)
for i in range(count):
a = hs + attr_start + i * attr_size
name = struct.unpack_from('<I', start, a + 4)[0]
if (name < len(resmap) and resmap[name] == 0x01010003) or strings[name] == 'name':
struct.pack_into('<I', start, a + 8, index)
struct.pack_into('<HBBI', start, a + 12, 8, 0, 3, index)
break
else:
raise frame_apk.ApkError('category has no name attribute')
end = struct.pack('<HHI', 0x0103, hs, hs + 8) + start[8:hs] + start[hs:hs + 8]
result = bytearray(axml_bytes[:8])
for _, _, off, size in chunks:
if off == target['end']:
result += start + end
result += new_pool if off == po else axml_bytes[off:off + size]
struct.pack_into('<I', result, 4, len(result))
result = bytes(result)
if not inspect(result)[0]['launchable']:
raise frame_apk.ApkError('launcher repair failed verification')
return result
def detection(data, names):
info, _ = inspect(data)
issues = []
if 'lib/arm64-v8a/libvrapi.so' in names:
issues.append("Uses Meta's legacy VrApi, which the Frame doesn't have; it won't run.")
if any(n.endswith('/libovrplatformloader.so') for n in names):
issues.append("Uses Meta's platform SDK; if it checks your Quest store licence it will quit.")
if 'lib/arm64-v8a/libopenxr_loader.so' in names:
info['vr'] = True
issues.append('Uses OpenXR (good).')
info['vr_issues'] = issues
return info
ui/frame_macview.py
+456
View File
@@ -0,0 +1,456 @@
"""Mac in the headset: stream Mac windows or displays into the Steam Frame as
panels you can place anywhere, with the laser, wheel and keys driving the Mac.
Runs on the Mac, inside Frame Control's server. The pieces:
- mac/bin/frame-mac-view (Swift, built from mac/frame-mac-view): captures with
ScreenCaptureKit, encodes with VideoToolbox, serves ui/mac-view.html and one
WebSocket per stream on 127.0.0.1, and plays input back with CGEvent.
- An `ssh -R` tunnel, so the Frame reaches the agent on its own 127.0.0.1.
Every request carries a random token, so other programs on the Frame
(Android apps included) can't watch or drive the Mac.
- A Chromium app window on the Frame per stream, on gamescope's X display
with its own STEAM_GAME id, which makes it its own SteamVR panel (see
docs/panels.md). Chromium XR (~/chromium-xr) is preferred because it's
built with H.264; Flathub Chromium is the fallback.
Stdlib only. MacView gets a plain ssh argv for the tunnel (its own
connection) and the server's `run(remote, stdin=, timeout=)` for commands.
"""
import json
import os
import re
import secrets
import shutil
import socket
import subprocess
import sys
import threading
import time
import urllib.error
import urllib.request
import zlib
from pathlib import Path
from urllib.parse import quote, urlencode # %20, not +: the agent's URLComponents keeps +
HERE = Path(__file__).resolve().parent
ROOT = HERE.parent
PAGE = HERE / "mac-view.html"
SOURCES = ROOT / "mac" / "frame-mac-view"
AGENT = Path(os.environ.get("FRAME_MAC_VIEW") or ROOT / "mac" / "bin" / "frame-mac-view")
REMOTE_PORTS = range(47900, 47920)
SUPPORTED = sys.platform == "darwin"
# Stream settings by name: long side in pixels, frames per second, H.264 bits
# per pixel per frame, codec. JPEG is for a Frame browser without H.264.
QUALITY = {
"sharp": {"max": 2560, "fps": 60, "bpp": 0.14, "codec": "h264"},
"balanced": {"max": 1920, "fps": 60, "bpp": 0.1, "codec": "h264"},
"light": {"max": 1280, "fps": 30, "bpp": 0.08, "codec": "h264"},
"compatible": {"max": 1280, "fps": 20, "bpp": 0.1, "codec": "jpeg"},
}
# Extra Chromium flags for viewers (see docs/mac-in-headset.md, "Measuring").
BROWSER_FLAGS = []
# Viewer windows are fitted inside a panel's size. gamescope made a 1280x720
# request 1920x1080 anyway (verified 2026-09-28, build 20260925.6191901).
PANEL_BOX = (1920, 1080)
# Opens one viewer on gamescope's X display and gives its window its own panel
# id. Args: appid url width height tag [browser flags...]. The page puts "[tag]" in its title at
# once, which is how its X window is found (Chromium may hand the URL to an
# instance that's already running, so there's no process to follow).
LAUNCH = r"""set -u
appid=$1 url=$2 w=$3 h=$4 tag=$5
shift 5
export DISPLAY=:0 LC_ALL=C.UTF-8
unset WAYLAND_DISPLAY
if ! xprop -root GAMESCOPE_FOCUSABLE_WINDOWS >/dev/null 2>&1; then
echo "The headset isn't showing anything (gamescope's display :0 isn't up). Wake it and try again." >&2
exit 2
fi
common=(--ozone-platform=x11 --force-device-scale-factor=1 --no-first-run --no-default-browser-check
--password-store=basic --disable-session-crashed-bubble --noerrdialogs --disable-infobars
--disable-features=Translate,MediaRouter --autoplay-policy=no-user-gesture-required
"--window-size=$w,$h" "$@" "--app=$url")
if [ -x "$HOME/chromium-xr/chrome" ]; then
cmd=("$HOME/chromium-xr/chrome" "--user-data-dir=$HOME/.local/share/frame-control/mac-view" "${common[@]}")
elif flatpak info org.chromium.Chromium >/dev/null 2>&1; then
cmd=(flatpak run org.chromium.Chromium
"--user-data-dir=$HOME/.var/app/org.chromium.Chromium/data/frame-mac-view" "${common[@]}")
else
echo "NO_BROWSER"
exit 3
fi
log=/tmp/frame-mac-view.log
setsid nohup "${cmd[@]}" >>"$log" 2>&1 </dev/null &
for _ in $(seq 1 60); do
sleep 0.5
# xprop, not xwininfo: in a C locale xwininfo can't print a non-ASCII title
# at all, while xprop escapes those bytes and leaves the ASCII tag readable.
for win in $(xwininfo -root -children 2>/dev/null | awk '/^ +0x/ {print $1}'); do
xprop -id "$win" _NET_WM_NAME WM_NAME 2>/dev/null | grep -qF "[$tag]" || continue
if xprop -id "$win" -f STEAM_GAME 32c -set STEAM_GAME "$appid" 2>/dev/null; then
echo "panel valve.steam.desktopgame.$appid window $win"
exit 0
fi
done
done
echo "The viewer started, but its window didn't appear within 30 s. Chromium's log:" >&2
tail -n 15 "$log" >&2
exit 1
"""
class MacViewError(Exception):
pass
def panel_id(src):
"""A stable panel id per source, in the range panel-on-frame.sh uses."""
return 2_001_000_000 + zlib.crc32(f"mac:{src}".encode()) % 1_000_000
def fit(w, h, box=PANEL_BOX):
s = min(box[0] / max(w, 1), box[1] / max(h, 1))
return max(320, round(w * s)), max(200, round(h * s))
class MacView:
def __init__(self, tunnel_ssh, run, frame, track=None):
self.tunnel_ssh = list(tunnel_ssh)
self.run = run
self.frame = frame
self.track = track or (lambda proc: None) # the server ends these on exit
self.lock = threading.Lock()
self.token = secrets.token_urlsafe(24)
self.agent = None
self.port = None
self.tunnel = None
self.remote_port = None
self.supervisor = None
self.closing = False
self.shows = 0 # counts Show presses, so a late cleanup can't close a new viewer
self.launching = 0 # Shows in progress (one may be replacing its own stream)
# Held by a Show while it counts itself in, and by the cleanup for its
# check and its pkill together, so a Show can't start in between.
self.viewer_lock = threading.Lock()
# Use the Frame's USB-C network when it's plugged into this Mac (see
# _usb_route); FRAME_MACVIEW_USB=0 turns that off.
self.prefer_usb = os.environ.get("FRAME_MACVIEW_USB") != "0"
self.route = "network"
self.shown = set() # sources with a viewer out there, connected or retrying
self.browser_flags = list(BROWSER_FLAGS)
# ---- the agent on this Mac ----
def unavailable(self):
"""Why this can't work here, or None."""
if not SUPPORTED:
return "Streaming your computer into the headset needs macOS."
if not AGENT.exists() and not (SOURCES.exists() and shutil.which("xcrun")):
return "The Mac streaming helper is missing from this copy of Frame Control."
return None
def build(self):
if AGENT.exists() and not self._stale():
return
if not (SOURCES / "build.sh").exists():
if AGENT.exists():
return
raise MacViewError("The Mac streaming helper is missing.")
r = subprocess.run(["/bin/sh", str(SOURCES / "build.sh"), str(AGENT)], capture_output=True, text=True,
stdin=subprocess.DEVNULL, timeout=600)
if r.returncode != 0:
raise MacViewError("Couldn't build the Mac streaming helper: " + (r.stderr or r.stdout).strip()[-400:])
def _stale(self):
try:
built = AGENT.stat().st_mtime
return any(p.stat().st_mtime > built for p in (SOURCES / "Sources").glob("*.swift"))
except OSError:
return False
def ensure_agent(self):
with self.lock:
if self.agent and self.agent.poll() is None:
return
reason = self.unavailable()
if reason:
raise MacViewError(reason)
self.build()
env = {**os.environ, "FRAME_MAC_VIEW_TOKEN": self.token}
# The same port as before when restarting, so a running tunnel still
# fits; otherwise (or if it's gone) whatever the system gives.
for port in dict.fromkeys([self.port or 0, 0]):
self.agent = subprocess.Popen([str(AGENT), "serve", "--port", str(port), "--page", str(PAGE),
"--exit-on-eof"], env=env, stdin=subprocess.PIPE,
stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)
line = self.agent.stdout.readline()
m = re.search(r"listening on 127\.0\.0\.1:(\d+)", line)
if m:
break
self.agent.kill()
else:
raise MacViewError(f"The Mac streaming helper didn't start: {line.strip() or 'no output'}")
if self.port != int(m.group(1)):
self._drop_tunnel()
self.port = int(m.group(1))
self.track(self.agent)
threading.Thread(target=self.agent.stdout.read, daemon=True).start() # drain
def call(self, path, method="GET", **query):
"""A request to the agent; starts it if needed."""
self.ensure_agent()
url = f"http://127.0.0.1:{self.port}{path}?{urlencode({**query, 'k': self.token}, quote_via=quote)}"
req = urllib.request.Request(url, method=method, data=b"" if method == "POST" else None)
try:
with urllib.request.urlopen(req, timeout=10) as r:
return json.load(r)
except (urllib.error.URLError, OSError, ValueError) as e:
raise MacViewError(f"The Mac streaming helper didn't answer: {e}")
# ---- the tunnel from the Frame ----
def _drop_tunnel(self):
if self.tunnel and self.tunnel.poll() is None:
self.tunnel.terminate()
self.tunnel = None
def tunnel_up(self):
return self.tunnel is not None and self.tunnel.poll() is None
def ensure_tunnel(self, allow_new_port=False):
"""Open the tunnel, on the port viewers already use if there is one.
A new port only when nobody is watching, since viewers can't move."""
with self.lock:
if self.tunnel_up():
return
last = ""
ports = [self.remote_port] if self.remote_port else list(REMOTE_PORTS)
if self.remote_port and allow_new_port:
ports += [p for p in REMOTE_PORTS if p != self.remote_port]
usb = self._usb_route()
# USB-C first when it's there; if that fails (unplugged just now,
# something else at that address), the normal path.
for via in ([usb, []] if usb else [[]]):
self.route = "usb" if via else "network"
if self._open_tunnel(via, ports):
return
last = self._last_tunnel_error
raise MacViewError(f"Couldn't open a tunnel from {self.frame} to this Mac: {last or 'no answer through it'}")
def _open_tunnel(self, via, ports):
"""Tries the ports on one route; True once the tunnel answers. With self.lock held."""
last = ""
for port in ports:
proc = subprocess.Popen([*self.tunnel_ssh, *via, "-o", "ExitOnForwardFailure=yes",
"-o", "ServerAliveInterval=5", "-o", "ServerAliveCountMax=3", "-N",
"-R", f"127.0.0.1:{port}:127.0.0.1:{self.port}", self.frame],
stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL,
stderr=subprocess.PIPE, text=True)
# A taken port makes ssh exit once it's connected; a working
# tunnel answers the agent's status from the Frame's side.
ok = False
for _ in range(15):
time.sleep(0.4)
if proc.poll() is not None:
break
if self._probe(port):
ok = True
break
if ok:
self.tunnel, self.remote_port = proc, port
self.track(proc)
self._supervise()
return True
if proc.poll() is None:
proc.terminate()
proc.wait()
last = (proc.stderr.read() or "").strip()
if "forward" not in last.lower():
break # not a port clash: the Frame is unreachable this way
self._last_tunnel_error = last
return False
def _usb_route(self):
"""ssh options to reach the Frame over its USB-C network, or [].
Plugged into a Mac, the Frame is a USB network device (macOS lists it
as "Steam Frame"): the Frame's usb0 answers at about 1 ms, with none
of Wi-Fi's stalls. Measured 2026-09-28: content latency 7 ms instead
of 10, click to screen 17 ms instead of 27 (docs/mac-in-headset.md).
The host key is the same, so it's checked against the usual name."""
if not self.prefer_usb:
return []
try:
out = self.run("ip -4 -o addr show usb0 2>/dev/null", timeout=8)
except Exception:
return []
m = re.search(r"inet (\d+\.\d+\.\d+\.\d+)/", out or "")
if not m:
return []
ip = m.group(1)
try:
socket.create_connection((ip, 22), timeout=1).close()
except OSError:
return [] # not plugged into this Mac
alias = self.frame
try:
cfg = subprocess.run(["ssh", "-G", self.frame], capture_output=True, text=True, timeout=5).stdout
opts = dict(line.split(None, 1) for line in cfg.splitlines() if " " in line)
alias = opts.get("hostkeyalias") or opts.get("hostname") or alias # a configured alias wins
except (OSError, subprocess.SubprocessError):
pass
return ["-o", f"HostName={ip}", "-o", f"HostKeyAlias={alias}"]
def _supervise(self):
"""Reopen the tunnel on the same port after the headset sleeps or the
network drops, so viewers that are retrying find the Mac again."""
if self.supervisor and self.supervisor.is_alive():
return
def loop():
while not self.closing:
time.sleep(5)
if self.closing or self.tunnel_up() or not (self.agent and self.agent.poll() is None):
continue
try:
self.ensure_tunnel()
except MacViewError:
pass # still unreachable; try again shortly
self.supervisor = threading.Thread(target=loop, daemon=True)
self.supervisor.start()
def _probe(self, port):
"""Whether the Frame reaches the agent through the tunnel on `port`."""
# /ping needs no key, so none appears on the Frame's command lines.
cmd = f"curl -s -m 2 'http://127.0.0.1:{port}/ping'"
try:
return self.run(cmd, timeout=8).strip() == "frame-mac-view"
except Exception: # noqa: BLE001 - the server's Failure, timeouts: all mean "not yet"
return False
# ---- viewers on the Frame ----
def show(self, src, quality="balanced", width=None, height=None):
with self.viewer_lock:
self.launching += 1
try:
return self._show(src, quality, width, height)
finally:
with self.viewer_lock:
self.launching -= 1
def _show(self, src, quality, width, height):
if src != "test" and not src.startswith(("window:", "display:", "separate:")):
raise MacViewError("Pick a window or display to show.")
self.shows += 1
q = QUALITY.get(quality) or QUALITY["balanced"]
state = self.call("/status")
if src != "test" and not state.get("screen"):
raise MacViewError("Frame Control needs Screen Recording permission first (Allow… under Mac in the headset).")
if src.startswith("separate:") and not state.get("accessibility"):
raise MacViewError("A window on its own display needs the Accessibility permission too, to move it "
"(Allow… under Mac in the headset).")
self.shown -= set(state.get("finished", [])) # their Mac windows closed
if src in self.shown or any(st.get("src") == src for st in state.get("streams", [])):
# Showing it again replaces the old viewer, connected or not: this
# revokes its keys so it can't come back alongside the new one.
self.stop(src)
# Viewers that lost the tunnel keep retrying its old port, even though
# the agent no longer counts them, so only move when none are out there.
others = self.shown - {src}
self.ensure_tunnel(allow_new_port=not others)
appid = panel_id(src)
# Unique per launch, so a new window is never confused with an old one.
tag = "fc" + secrets.token_hex(4)
# A single-use ticket for this source, not Frame Control's key: the URL
# is visible in the Frame's process list.
ticket = self.call("/ticket", method="POST", src=src)["ticket"]
params = {"src": src, "t": ticket, "tag": tag, "codec": q["codec"], "max": q["max"], "fps": q["fps"],
"bpp": q["bpp"]}
url = f"http://127.0.0.1:{self.remote_port}/view?{urlencode(params)}"
w, h = fit(width or 1280, height or 720)
args = " ".join(_quote(str(a)) for a in (appid, url, w, h, tag, *self.browser_flags))
try:
out = self.run("bash -s -- " + args, stdin=LAUNCH, timeout=60)
except Exception as e: # noqa: BLE001 - the server's Failure carries the Frame's words
if "NO_BROWSER" in (getattr(e, "stdout", "") or ""):
raise MacViewError("The Frame needs a browser for this: install Chromium (Tools → Linux apps, "
"org.chromium.Chromium) or Chromium XR.")
raise MacViewError(str(e))
self.shown.add(src)
return {"panel": f"valve.steam.desktopgame.{appid}", "src": src, "detail": out.strip()}
def stop(self, src=None):
if src:
self.shown.discard(src)
else:
self.shown.clear()
if not (self.agent and self.agent.poll() is None):
return {"closed": 0}
out = self.call("/close", method="POST", **({"src": src} if src else {}))
if not self.shown:
threading.Thread(target=self._end_viewer_browser, args=(self.shows,), daemon=True).start()
return out
def _end_viewer_browser(self, shows):
"""Chromium on the Frame outlives its last viewer window (verified
2026-09-28), so once nothing is shown, end it. It runs with a profile
of its own, so nothing else is touched."""
time.sleep(2) # the viewers close their windows first
with self.viewer_lock:
if self.shown or self.shows != shows or self.launching:
return
try:
self.run("pkill -f '[f]rame-control/mac-view|[d]ata/frame-mac-view' || true", timeout=10)
except Exception:
pass
def state(self):
reason = self.unavailable()
if reason:
return {"available": False, "reason": reason}
status = self.call("/status")
windows = self.call("/windows").get("windows", []) if status.get("screen") else []
displays = self.call("/displays").get("displays", [])
return {"available": True, "screen": status.get("screen", False),
"accessibility": status.get("accessibility", False), "streams": status.get("streams", []),
"windows": windows, "displays": displays, "tunnel": self.tunnel_up(),
"route": self.route}
def restart_agent(self):
"""Only if it's missing a permission: a running stream would stop."""
with self.lock:
if not (self.agent and self.agent.poll() is None):
return
status = self.call("/status")
if status.get("screen") and status.get("accessibility"):
return
with self.lock:
self.agent.terminate()
self.agent.wait(5)
def request_permissions(self):
return self.call("/permissions", method="POST")
def shutdown(self):
self.closing = True
try:
self.stop()
except MacViewError:
pass
for proc in (self.tunnel, self.agent):
if proc and proc.poll() is None:
proc.terminate()
# The helper puts separated windows back before it exits (about 1 s).
if self.agent:
try:
self.agent.wait(3)
except subprocess.TimeoutExpired:
self.agent.kill()
def _quote(s):
return "'" + s.replace("'", "'\\''") + "'"
ui/index.html
+112 -1
View File
@@ -329,6 +329,13 @@
.game-card .bar > i { display: block; height: 100%; background: var(--action); }
.f-0 { color: var(--muted); }
/* ---- Android display ---- */
#mvQuality { background: rgba(0,0,0,.28); color: var(--text); border: 1px solid transparent; border-radius: 3px; padding: 4px 6px; font: inherit; font-size: 12px; }
.mv-windows { max-height: 420px; overflow-y: auto; }
.mv-perm { background: rgba(255,190,60,.08); border: 1px solid rgba(255,190,60,.25); border-radius: 4px; padding: 10px 12px; margin-bottom: 12px; font-size: 13px; }
.mv-perm .row { margin-top: 8px; }
.mv-live { color: var(--green, #7fd67f); font-size: 12px; }
.mv-opt { display: block; font-size: 13px; margin: 0 0 12px; }
.mv-opt .sub { display: block; margin-left: 22px; font-size: 12px; }
.disp select { width: 100%; background: rgba(0,0,0,.28); color: var(--text); border: 1px solid transparent; border-radius: 3px;
padding: 8px 10px; font: inherit; font-size: 13px; }
.disp select:focus { outline: none; border-color: var(--blue); }
@@ -641,6 +648,12 @@
<span class="desk-only">You can also drop files anywhere in this window.</span>
<input type="file" id="fileInput" multiple hidden>
</div>
<label class="sub" for="apkDisplay">Android display</label>
<select id="apkDisplay" aria-label="Android app display mode">
<option value="auto">Automatic (detect VR)</option>
<option value="flat">Flat screen</option>
<option value="vr">VR app</option>
</select>
<div class="progress" id="prog"><i></i></div>
<div class="shelf-head" style="margin-top:18px"><h2>Clipboard</h2></div>
<textarea id="clipText" placeholder="Text to put on the Frame's clipboard…"></textarea>
@@ -708,6 +721,27 @@
</div>
</section>
<section class="panel" id="macview" hidden>
<div class="shelf-head"><h2>Mac in the headset</h2><span class="count" id="mvCount"></span><span class="spacer"></span>
<select id="mvQuality" title="Picture quality and bandwidth">
<option value="sharp">Sharp</option><option value="balanced" selected>Balanced</option>
<option value="light">Light</option><option value="compatible">Compatible (JPEG)</option>
</select>
<button class="small" id="mvRefresh">Refresh</button></div>
<label class="mv-opt"><input type="checkbox" id="mvSeparate" checked> Give each window its own display
<span class="sub">its menus and sheets come along, nothing covers it, and it goes back when you stop</span></label>
<div class="mv-perm" id="mvPerm" hidden></div>
<div class="list" id="mvDisplays"></div>
<div class="shelf-head" style="margin-top:16px"><h2>Windows</h2><span class="count" id="mvWinCount"></span></div>
<div class="list mv-windows" id="mvWindows"><div class="sub">Loading…</div></div>
<div class="row" style="margin-top:12px">
<button class="small" data-mv="show" data-src="test" data-title="Test pattern">Test pattern</button>
<button class="small danger" data-mv="stopall">Stop all</button>
</div>
<div class="hint">Each window or screen becomes its own panel in the headset. Place it with the SteamVR dashboard
(Float in World, Move, Size). Point and click with the laser, scroll with the thumbstick, and type on the Mac's keyboard.</div>
</section>
<section class="panel" id="privacy">
<div class="shelf-head"><h2>Privacy &amp; updates</h2><span class="spacer"></span><span class="sub" id="appVersion"></span></div>
<label class="popt"><input type="checkbox" id="tUsage"><b>Anonymous usage statistics</b>
@@ -1520,6 +1554,7 @@ function upload(file, mode) {
xhr.setRequestHeader("X-Frame-UI", UI_KEY);
xhr.setRequestHeader("X-Filename", encodeURIComponent(file.name));
xhr.setRequestHeader("X-Mode", mode);
if (mode === "apk") xhr.setRequestHeader("X-APK-Display", $("apkDisplay").value);
const bar = $("prog").firstElementChild;
$("prog").style.display = "block"; bar.style.width = "0";
xhr.upload.onprogress = e => { if (e.lengthComputable) bar.style.width = (100 * e.loaded / e.total) + "%"; };
@@ -2313,10 +2348,85 @@ $("shotsRefresh").onclick = loadShots;
$("shotsSaveNew").onclick = e => saveShots(shots.list.filter(s => !s.saved), e.currentTarget);
$("shotsFolder").onclick = e => act($("shotsFolder").textContent, () => api("/api/open", { what: "shots" }), e.currentTarget);
// ---- Mac in the headset: windows and displays as panels (ui/frame_macview.py) ----
let mvSeq = 0, mvTimer = 0;
// streaming: the src it's shown as (a window may be "separate:<id>"), or "".
// How a live stream is doing: frames shown, Mac-to-headset delay, bitrate, and
// whether the network made it step down (fewer frames, then fewer pixels).
function mvHealth(st) {
if (!st) return "";
const x = st.stats || {}, c = st.controller || {}, bits = [];
if (x.fps) bits.push(`${Math.round(x.fps)} fps`);
if (x.total && x.total.p50 != null) bits.push(`${Math.round(x.total.p50)} ms`);
if (x.mbps) bits.push(`${x.mbps} Mbit/s`);
if (c.tier > 0) bits.push(`weak link: ${c.fps} fps${c.scale < 1 ? `, ${Math.round(c.scale * 100)}% size` : ""}`);
return bits.length ? " · " + esc(bits.join(" · ")) : "";
}
function mvRow(src, title, sub, w, h, streaming, st) {
return `<div class="item"><div class="grow"><div class="t">${esc(title)}</div>
<div class="s">${esc(sub)}${streaming ? ' · <span class="mv-live">in the headset</span>' + mvHealth(st) : ""}</div></div>
${streaming ? `<button class="small" data-mv="stop" data-src="${esc(streaming)}">Stop</button>`
: `<button class="small action" data-mv="show" data-src="${esc(src)}" data-title="${esc(title)}" data-w="${w}" data-h="${h}">Show</button>`}
</div>`;
}
async function loadMacView(restart) {
const seq = ++mvSeq;
let s, err;
try { s = await api("/api/macview" + (restart ? "?restart=1" : "")); } catch (e) { err = e; }
if (seq !== mvSeq) return;
const box = $("macview");
if (err) { box.hidden = false; return failed($("mvWindows"), err); }
if (!s.available) { box.hidden = true; return; }
box.hidden = false;
const live = new Set((s.streams || []).map(x => x.src));
const byStream = src => (s.streams || []).find(x => x.src === src);
$("mvCount").textContent = live.size ? `${live.size} showing${s.route === "usb" ? " · over USB-C" : ""}` : "";
const perm = [];
if (!s.screen) perm.push(`Frame Control needs <b>Screen Recording</b> permission to show your windows.`);
if (!s.accessibility) perm.push(`Allow <b>Accessibility</b> too, so clicks and keys from the headset reach the Mac.`);
$("mvPerm").hidden = !perm.length;
$("mvPerm").innerHTML = perm.join(" ") + `<div class="row"><button class="small action" data-mv="perm">Allow…</button>
<span class="sub">Then press Refresh. macOS may ask you to reopen Frame Control.</span></div>`;
const liveAs = src => live.has(src) ? src : src.startsWith("window:") && live.has("separate:" + src.slice(7)) ? "separate:" + src.slice(7) : "";
$("mvDisplays").innerHTML = (s.displays || []).map(d =>
mvRow(d.src, `Whole screen: ${d.name}`, `${d.w}×${d.h}${d.main ? " · main display" : ""}`, d.w, d.h, liveAs(d.src), byStream(liveAs(d.src)))).join("");
const wins = s.windows || [];
$("mvWinCount").textContent = wins.length ? `${wins.length}` : "";
$("mvWindows").innerHTML = !s.screen ? `<div class="sub">Windows appear here once Screen Recording is allowed.</div>`
: wins.length ? wins.map(w => mvRow(w.src, w.title || w.app, `${w.title ? w.app + " · " : ""}${w.w}×${w.h}`, w.w, w.h, liveAs(w.src), byStream(liveAs(w.src)))).join("")
: `<div class="sub">No windows open on this Mac's current desktop.</div>`;
// Keep the numbers fresh while something is in the headset and the card is on screen.
clearTimeout(mvTimer);
const tick = () => { mvTimer = document.hidden || !box.offsetParent ? setTimeout(tick, 3000) : (loadMacView(), 0); };
if (live.size) mvTimer = setTimeout(tick, 3000);
}
$("mvRefresh").onclick = () => loadMacView(true);
$("mvSeparate").checked = localStorage.getItem("mvSeparate") !== "0";
$("mvSeparate").onchange = e => localStorage.setItem("mvSeparate", e.target.checked ? "1" : "0");
$("macview").onclick = async e => {
const b = e.target.closest("[data-mv]"); if (!b) return;
const src = b.dataset.src;
if (b.dataset.mv === "show") {
const as = $("mvSeparate").checked && src.startsWith("window:") ? "separate:" + src.slice(7) : src;
await act(`Show ${b.dataset.title} in the headset`, async () => {
const r = await api("/api/macview", { action: "show", src: as, quality: $("mvQuality").value,
w: +b.dataset.w || undefined, h: +b.dataset.h || undefined });
return { message: `${b.dataset.title} is a panel in the headset now. Float it with the SteamVR dashboard.`, ...r };
}, b);
} else if (b.dataset.mv === "stop") {
await act("Stop showing it", () => api("/api/macview", { action: "stop", src }), b);
} else if (b.dataset.mv === "stopall") {
await act("Stop everything in the headset", () => api("/api/macview", { action: "stop" }), b);
} else if (b.dataset.mv === "perm") {
await act("Ask macOS for permission", () => api("/api/macview", { action: "permissions" }), b);
}
setTimeout(loadMacView, 800);
};
// ---- pages: #home, #games, #android, #tools (older section links still work) ----
const PAGES = ["home", "games", "android", "tools"];
const SECTION_PAGE = { view: "home", device: "home", shots: "home", library: "games", sideloaded: "games", getgames: "games",
display: "android", transfer: "tools", apps: "tools", power: "tools", privacy: "tools" };
display: "android", transfer: "tools", apps: "tools", power: "tools", macview: "tools", privacy: "tools" };
let page = "home";
function showPage() {
const id = location.hash.slice(1);
@@ -2328,6 +2438,7 @@ function showPage() {
document.title = page === "home" ? "Frame Control" : `${page[0].toUpperCase() + page.slice(1)} · Frame Control`;
const section = !PAGES.includes(id) && id && $(id);
if (section) section.scrollIntoView(); else window.scrollTo(0, 0);
if (page === "tools") loadMacView();
}
window.addEventListener("hashchange", showPage);
// While a text field has focus, phones hide the bottom tab bar (see body.typing in the CSS).
ui/mac-view.html
+398
View File
@@ -0,0 +1,398 @@
<!doctype html>
<!-- Frame Control: one Mac window (or display) inside the Steam Frame.
Served by the Mac's frame-mac-view agent through an SSH tunnel and opened
on the Frame as its own Chromium app window, which gamescope turns into a
SteamVR panel. Video arrives over a WebSocket (H.264 Annex B for
WebCodecs, or JPEG); pointer, wheel and keys go back the same way. -->
<html lang="en">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>Mac</title>
<style>
html, body { margin: 0; height: 100%; background: #000; overflow: hidden; cursor: default; }
canvas { position: fixed; inset: 0; width: 100%; height: 100%; object-fit: contain; image-rendering: auto; }
#note { position: fixed; left: 50%; top: 16px; transform: translateX(-50%); max-width: 80%;
font: 15px/1.4 system-ui, sans-serif; color: #eee; background: rgba(20,22,26,.88);
padding: 8px 14px; border-radius: 10px; pointer-events: none; transition: opacity .4s; }
#note[hidden] { display: block; opacity: 0; }
#hud { position: fixed; left: 8px; top: 8px; font: 12px/1.35 ui-monospace, monospace; color: #dfe;
background: rgba(0,0,0,.72); padding: 6px 9px; border-radius: 6px; pointer-events: none; white-space: pre; }
</style>
</head>
<body>
<canvas id="c" width="16" height="9"></canvas>
<div id="note">Connecting to the Mac…</div>
<div id="hud" hidden></div>
<script>
"use strict";
const q = new URLSearchParams(location.search);
// t: a single-use ticket from Frame Control. After the first connection the
// Mac sends a reconnect key, kept only in memory.
const src = q.get("src") || "test", ticket = q.get("t") || "", tag = q.get("tag") || "";
let reconnectKey = "";
const c = document.getElementById("c"), ctx = c.getContext("2d", { alpha: false, desynchronized: true });
const note = document.getElementById("note");
// Counters for Frame Control's tests (read over DevTools).
const stats = window.stats = { frames: 0, keyFrames: 0, bytes: 0, dropped: 0, codec: "", errors: [], info: null };
// ---- timing: the Mac measures each frame's journey on its own clock ----
// Clock sync, NTP-style: the Mac's time minus ours, from the ping with the
// shortest round trip lately (the least queueing, so the least error).
const clock = { off: null, rtt: 0, samples: [], reported: 0 };
function toMac(ms) { return clock.off === null ? null : Math.round(ms * 1000 + clock.off); }
function onPong(m) {
const now = performance.now(), rtt = now - m.c;
clock.samples.push({ rtt, off: m.a - (m.c + now) / 2 * 1000 });
if (clock.samples.length > 16) clock.samples.shift();
const best = clock.samples.reduce((a, b) => (b.rtt < a.rtt ? b : a));
clock.off = best.off;
clock.rtt = best.rtt;
if (now - clock.reported > 1000) {
clock.reported = now;
send({ t: "clock", rtt: +best.rtt.toFixed(2), dec: `${decoderInfo}; ${glInfo}; raf ${rafHz} Hz` });
}
}
let pingTimer = 0;
function startPings() {
clearInterval(pingTimer);
clock.samples = [];
for (let i = 0; i < 10; i++) setTimeout(() => send({ t: "ping", c: performance.now() }), i * 40);
pingTimer = setInterval(() => send({ t: "ping", c: performance.now() }), 1000);
}
// Decoded, drawn and next-animation-frame times go back in batches.
let reports = [], dropsToReport = 0, shownPending = null, rafQueued = false;
setInterval(() => {
if (!reports.length && !dropsToReport) return;
send({ t: "fd", f: reports, drop: dropsToReport });
reports = [];
dropsToReport = 0;
}, 250);
function dropped() { stats.dropped++; dropsToReport++; }
// A drawn frame is on screen from the next animation frame, unless another
// replaces it first (then it was never seen).
function shown(seq, decodedAt, drawnAt) {
if (shownPending) reports.push([shownPending.seq, toMac(shownPending.dec), toMac(shownPending.drawn), 0]);
shownPending = { seq, dec: decodedAt, drawn: drawnAt };
if (rafQueued) return;
rafQueued = true;
requestAnimationFrame(() => {
rafQueued = false;
const p = shownPending, now = performance.now();
shownPending = null;
if (p && clock.off !== null) reports.push([p.seq, toMac(p.dec), toMac(p.drawn), toMac(now)]);
});
}
let decoderInfo = "";
// What this browser draws with, and how often it gives an animation frame
// when nothing else is going on (both only for the numbers).
let glInfo = "", rafHz = 0;
try {
const gl = document.createElement("canvas").getContext("webgl");
const ext = gl && gl.getExtension("WEBGL_debug_renderer_info");
glInfo = ext ? gl.getParameter(ext.UNMASKED_RENDERER_WEBGL) : gl ? "webgl" : "no webgl";
} catch (e) { glInfo = "?"; }
(function measureRaf() {
let n = 0, t0 = 0;
const tick = t => {
if (!t0) t0 = t;
if (t - t0 < 1000) { n++; return requestAnimationFrame(tick); }
rafHz = Math.round(n * 1000 / (t - t0));
};
requestAnimationFrame(tick);
})();
const hud = document.getElementById("hud");
function showHud(on) { hud.hidden = !on; }
function ms(o) { return o && o.p50 !== undefined ? `${o.p50}/${o.p95}` : "–"; }
function onStats(m) {
if (hud.hidden) return;
hud.textContent = `${m.size} ${m.fps} fps shown (${m.sentFps} sent) ${m.mbps} Mbit/s` +
(m.bitrate ? ` of ${(m.bitrate / 1e6).toFixed(1)}` : "") + (m.tier !== undefined ? ` tier ${m.tier}` : "") +
`\nlatency p50/p95 ms total ${ms(m.total)}\n capture ${ms(m.capture)} queue ${ms(m.queue)} encode ${ms(m.encode)}` +
`\n network ${ms(m.network)} decode ${ms(m.decode)} draw ${ms(m.draw)}` +
`\ninput→shown p50 ${m.input ? m.input.p50.toFixed(0) + " ms" : "–"} rtt ${m.rtt} ms` +
`\nskipped ${m.skipped}/${m.captured} dropped ${m.dropped} ${decoderInfo}`;
}
showHud(q.get("stats") === "1");
// Frame Control finds this window on the Frame by the tag in its title.
document.title = tag ? `Mac [${tag}]` : "Mac";
let failedStarts = 0;
let ws = null, dec = null, codecString = "", needKey = true, closing = false, retry = 0, noteTimer = 0, lastError = "";
function show(text, ms) {
note.textContent = text;
note.hidden = false;
clearTimeout(noteTimer);
if (ms) noteTimer = setTimeout(() => { note.hidden = true; }, ms);
}
function send(obj) { if (ws && ws.readyState === 1) ws.send(JSON.stringify(obj)); }
let lastKeyAsk = 0;
function askKeyFrame() {
const now = performance.now();
if (now - lastKeyAsk < 500) return;
lastKeyAsk = now;
send({ t: "key-frame" });
}
let hideNoteOnFrame = true; // "Connecting…"/"Reconnecting…" go once video flows again
function drawFrame(img, w, h, seq, decodedAt) {
if (c.width !== w || c.height !== h) { c.width = w; c.height = h; }
ctx.drawImage(img, 0, 0);
shown(seq, decodedAt, performance.now());
stats.frames++;
if (hideNoteOnFrame) { hideNoteOnFrame = false; note.hidden = true; }
}
// ---- H.264 ----
function startCode(b, i) { return b[i] === 0 && b[i + 1] === 0 && (b[i + 2] === 1 || (b[i + 2] === 0 && b[i + 3] === 1)); }
function spsCodec(au) {
for (let i = 0; i + 7 < au.length; i++) {
if (!startCode(au, i)) continue;
const n = au[i + 2] === 1 ? i + 3 : i + 4;
if ((au[n] & 0x1f) === 7) return "avc1." + [au[n + 1], au[n + 2], au[n + 3]].map(b => b.toString(16).padStart(2, "0")).join("");
i = n;
}
return "";
}
function makeDecoder() {
if (dec) try { dec.close(); } catch (e) {}
codecString = "";
dec = new VideoDecoder({
// The chunk's timestamp is the Mac's sequence number, to match reports up.
output: frame => { drawFrame(frame, frame.displayWidth, frame.displayHeight, frame.timestamp, performance.now()); frame.close(); },
error: e => {
stats.errors.push(String(e.message || e));
needKey = true;
makeDecoder();
askKeyFrame();
},
});
}
function onH264(isKey, seq, au) {
if (isKey) {
const cs = spsCodec(au);
if (cs && (cs !== codecString || dec.state !== "configured")) {
if (dec.state === "closed") makeDecoder();
dec.configure({ codec: cs, optimizeForLatency: true, hardwareAcceleration: "no-preference" });
codecString = stats.codec = cs;
}
stats.keyFrames++;
}
if (dec.state !== "configured" || (needKey && !isKey)) { dropped(); askKeyFrame(); return; }
// Far behind: skip to the next keyframe rather than show old pictures late.
// A few queued frames are fine: decoding catches up faster than a keyframe
// crosses a slow link, and only the newest decoded frame gets drawn.
if (!isKey && dec.decodeQueueSize > 10) { needKey = true; dropped(); askKeyFrame(); return; }
needKey = false;
dec.decode(new EncodedVideoChunk({ type: isKey ? "key" : "delta", timestamp: seq, data: au }));
}
// ---- JPEG ----
let jpegBusy = false;
function onJPEG(seq, data) {
if (jpegBusy) { dropped(); return; }
jpegBusy = true;
createImageBitmap(new Blob([data], { type: "image/jpeg" })).then(bmp => {
drawFrame(bmp, bmp.width, bmp.height, seq, performance.now());
bmp.close();
}).catch(e => stats.errors.push(String(e))).finally(() => { jpegBusy = false; });
}
async function pickCodec() {
const want = q.get("codec");
if (want === "jpeg") return "jpeg";
if (!("VideoDecoder" in window)) return "jpeg";
try {
const r = await VideoDecoder.isConfigSupported({ codec: "avc1.640028", optimizeForLatency: true });
if (!r.supported) return "jpeg";
// Whether this browser has a hardware H.264 decoder (for the numbers).
const hw = await VideoDecoder.isConfigSupported({ codec: "avc1.640028", hardwareAcceleration: "prefer-hardware" })
.catch(() => ({ supported: false }));
decoderInfo = hw.supported ? "h264 hardware" : "h264 software";
return "h264";
} catch (e) { return "jpeg"; }
}
async function connect() {
const codec = await pickCodec();
stats.codec = codec;
if (codec === "h264") makeDecoder();
needKey = true;
const p = new URLSearchParams({ src, codec, max: q.get("max") || "1920", fps: q.get("fps") || "60" });
if (reconnectKey) p.set("r", reconnectKey); else p.set("t", ticket);
if (q.get("bpp")) p.set("bpp", q.get("bpp"));
ws = new WebSocket(`ws://${location.host}/stream?${p}`);
ws.binaryType = "arraybuffer";
ws.onopen = () => { retry = 0; lastError = ""; startPings(); };
ws.onmessage = ev => {
if (typeof ev.data === "string") return control(JSON.parse(ev.data));
const now = performance.now(), b = new Uint8Array(ev.data);
stats.bytes += b.length;
if (b.length < 18) return;
// flags (1 = keyframe), pts µs, sequence number, input echoed; big-endian.
const v = new DataView(ev.data), isKey = (b[0] & 1) === 1, seq = v.getUint32(9);
send({ t: "rx", s: seq, r: toMac(now) });
const payload = b.subarray(17);
if (codec === "h264") onH264(isKey, seq, payload); else onJPEG(seq, payload);
};
ws.onclose = () => {
ws = null;
clearInterval(pingTimer);
if (closing) return;
// Refused before ever getting a key: the ticket expired or was revoked,
// and retrying can't help.
if (!reconnectKey && ++failedStarts >= 3) {
show("This view has expired. Press Show in Frame Control on the Mac to open it again.");
return;
}
hideNoteOnFrame = true;
// The Mac may be asleep or the tunnel restarting: keep trying.
retry = Math.min(retry + 1, 6);
// Keep the Mac's reason (a missing permission, a closed window) on screen.
show(lastError ? `${lastError} Retrying…` : "Lost the Mac. Reconnecting…");
setTimeout(connect, 500 * 2 ** retry);
};
}
function control(m) {
if (m.t === "pong") return onPong(m);
if (m.t === "stats") return onStats(m);
if (m.t === "bench") return bench(m);
if (m.t === "hello") {
reconnectKey = m.r;
send({ t: "ack" }); // the ticket is spent only now
} else if (m.t === "info") {
stats.info = m;
warmMs = Math.max(0, +m.warm || 0);
const name = m.title && m.app && m.title !== m.app ? `${m.title} — ${m.app}` : (m.title || m.app || "Mac");
document.title = tag ? `${name} [${tag}]` : name;
if (!m.input) {
hideNoteOnFrame = false; // a warning, not a connection notice: let it stay its 8 s
show("Clicks and keys need Accessibility permission on the Mac (Frame Control asks for it).", 8000);
}
} else if (m.t === "error") {
stats.errors.push(m.message);
lastError = m.message.replace(/\.?$/, ".");
show(m.message);
} else if (m.t === "close" || m.t === "closed") {
closing = true;
if (ws) ws.close();
show(m.reason ? `Stopped: ${m.reason}.` : "Stopped.");
window.close();
}
}
// ---- input ----
// Where the picture sits inside the window (object-fit: contain letterboxes it).
function toPicture(e) {
const r = c.getBoundingClientRect(), s = Math.min(r.width / c.width, r.height / c.height);
const w = c.width * s, h = c.height * s, left = r.left + (r.width - w) / 2, top = r.top + (r.height - h) / 2;
const x = (e.clientX - left) / w, y = (e.clientY - top) / h;
return { x, y, inside: x >= 0 && x <= 1 && y >= 0 && y <= 1 };
}
// Discrete input carries an id and when it happened (the Mac's clock), so
// the Mac can tag the first frame that could show its effect.
let inputId = 0;
// Keep the Frame's Wi-Fi awake for a few seconds after input, when the agent asks.
let warmMs = 0, warmUntil = 0, warmTimer = 0;
function keepWarm() {
if (!warmMs) return;
warmUntil = performance.now() + 5000;
if (warmTimer) return;
warmTimer = setInterval(() => {
if (performance.now() > warmUntil) { clearInterval(warmTimer); warmTimer = 0; return; }
send({ t: "w" });
}, warmMs);
}
function stamp(m, e) {
keepWarm();
m.i = ++inputId;
m.tv = toMac(e && e.timeStamp ? e.timeStamp : performance.now());
return m;
}
// Moves go at most every 8 ms, on a timer rather than the next animation frame:
// the Frame throttles a panel's animation frames to 15-36 Hz when it thinks
// nobody is looking, which would hold a move back by up to 60 ms.
let pendingMove = null, moveQueued = false, lastMove = 0;
function flushMove() {
moveQueued = false;
if (pendingMove) { send(pendingMove); lastMove = performance.now(); }
pendingMove = null;
}
addEventListener("pointermove", e => {
const p = toPicture(e);
if (!p.inside && !e.buttons) return;
pendingMove = { t: "m", e: "move", x: p.x, y: p.y };
if (moveQueued) return;
const wait = lastMove + 8 - performance.now();
if (wait <= 0) return flushMove();
moveQueued = true;
setTimeout(flushMove, wait);
});
addEventListener("pointerdown", e => {
const p = toPicture(e);
if (!p.inside) return;
if (e.pointerId !== undefined) try { c.setPointerCapture(e.pointerId); } catch (err) {}
pendingMove = null;
send(stamp({ t: "m", e: "down", b: e.button < 0 ? 0 : e.button, x: p.x, y: p.y }, e));
e.preventDefault();
});
addEventListener("pointerup", e => {
const p = toPicture(e);
send({ t: "m", e: "up", b: e.button < 0 ? 0 : e.button, x: p.x, y: p.y });
});
// Let go of any held button where the pointer is on the Mac, not at a corner.
addEventListener("pointercancel", () => send({ t: "release" }));
addEventListener("contextmenu", e => e.preventDefault());
addEventListener("wheel", e => {
const p = toPicture(e), unit = e.deltaMode === 1 ? 16 : e.deltaMode === 2 ? innerHeight : 1;
send(stamp({ t: "wheel", dx: e.deltaX * unit, dy: e.deltaY * unit, x: p.x, y: p.y }, e));
e.preventDefault();
}, { passive: false });
function mods(e) {
return ["shift", "ctrl", "alt", "meta"].filter(m => e[m + "Key"]);
}
function onKey(e, down) {
// Ctrl+Alt+Shift+S shows the numbers; it isn't sent to the Mac.
if (e.code === "KeyS" && e.ctrlKey && e.altKey && e.shiftKey) {
if (down) showHud(hud.hidden);
return e.preventDefault();
}
const m = { t: "k", e: down ? "down" : "up", code: e.code, key: e.key, mods: mods(e) };
send(down ? stamp(m, e) : m);
e.preventDefault();
}
// ---- benchmarks: the Mac asks the viewer to act as if someone did ----
function keyCode(ch) {
if (/[a-z]/i.test(ch)) return "Key" + ch.toUpperCase();
if (/[0-9]/.test(ch)) return "Digit" + ch;
return { " ": "Space", "\n": "Enter", ".": "Period", ",": "Comma" }[ch] || "";
}
function bench(m) {
if (m.action === "overlay") return showHud(m.on !== 0 && m.on !== "0");
if (m.action === "click") {
const at = { x: +m.x || 0.5, y: +m.y || 0.5 };
send(stamp({ t: "m", e: "down", b: 0, ...at }));
setTimeout(() => send({ t: "m", e: "up", b: 0, ...at }), 40);
} else if (m.action === "type") {
const text = String(m.text || ""), gap = +m.interval || 150;
[...text].forEach((ch, n) => setTimeout(() => {
const code = keyCode(ch);
if (!code) return send(stamp({ t: "text", s: ch }));
const mods = ch !== ch.toLowerCase() ? ["shift"] : [];
send(stamp({ t: "k", e: "down", code, key: ch, mods }));
setTimeout(() => send({ t: "k", e: "up", code, key: ch, mods }), 30);
}, n * gap));
}
}
addEventListener("keydown", e => onKey(e, true));
addEventListener("keyup", e => onKey(e, false));
addEventListener("blur", () => send({ t: "release" }));
show("Connecting to the Mac…");
connect();
</script>
</body>
</html>
ui/server.py
+53 -3
View File
@@ -43,6 +43,7 @@ import frame_android # noqa: E402
import frame_apk_versions # noqa: E402
import frame_catalog # noqa: E402
import frame_host # noqa: E402
import frame_macview # noqa: E402
import frame_media # noqa: E402
import frame_report # noqa: E402
import frame_store # noqa: E402
@@ -1358,6 +1359,46 @@ def push_media(path):
"id": token + "/" + name}
# ---- Mac in the headset (frame_macview.py) ----------------------------------
# The tunnel gets its own connection: the shared master's options would win
# over anything added after them.
macview = frame_macview.MacView(["ssh", "-o", "BatchMode=yes", "-o", "ConnectTimeout=8"],
lambda remote, stdin=None, timeout=30: ssh(remote, stdin=stdin, timeout=timeout),
FRAME, track=_live_tunnels.add)
def macview_state(query=None):
if LOCAL:
return {"available": False, "reason": "Runs on your computer, not the headset."}
try:
# Refresh restarts the helper, which picks up a permission just granted.
if (query or {}).get("restart") == ["1"]:
macview.restart_agent()
return macview.state()
except frame_macview.MacViewError as e:
raise Failure(str(e), 500)
def macview_action(body):
"""{action: show|stop|permissions, src, quality, w, h}."""
if LOCAL:
raise Failure("Runs on your computer, not the headset.", 400)
action = body.get("action")
try:
if action == "show":
w, h = body.get("w"), body.get("h")
return macview.show(str(body.get("src") or ""), str(body.get("quality") or "balanced"),
int(w) if w else None, int(h) if h else None)
if action == "stop":
return macview.stop(body.get("src") or None)
if action == "permissions":
return macview.request_permissions()
except frame_macview.MacViewError as e:
raise Failure(str(e), 502)
raise Failure("unknown action", 400)
# ---- Report a problem (frame_report.py) --------------------------------------
def report_preview(body):
@@ -1390,7 +1431,7 @@ POST = {"/api/media": media, "/api/agent/call": agent_call, "/api/agent/approval
"/api/webinstall/check": webinstall_check, "/api/webinstall/start": webinstall_start,
"/api/webinstall/cancel": webinstall_cancel,
"/api/telemetry": frame_telemetry.update_settings, "/api/telemetry/event": frame_telemetry.page_event,
"/api/report/preview": report_preview, "/api/report": report_send}
"/api/report/preview": report_preview, "/api/report": report_send, "/api/macview": macview_action}
# ---- HTTP ------------------------------------------------------------------
@@ -1526,6 +1567,8 @@ class Handler(BaseHTTPRequestHandler):
"shared": frame_catalog.compat_db.shared()})
elif path == "/api/android/catalog":
self.send_json({"apps": frame_catalog.catalog()})
elif path == "/api/macview":
self.send_json(macview_state(parse_qs(url.query)))
elif path == "/api/telemetry":
self.send_json(frame_telemetry.state())
elif path == "/api/computer/state":
@@ -1708,10 +1751,16 @@ class Handler(BaseHTTPRequestHandler):
raise Failure(str(e), 400, {"package": info["package"], "version_code": info.get("version_code"), "blocker": str(e)})
ensure_master()
try:
m = frame_android.install(str(dest), source=name)
display = self.headers.get("X-APK-Display", "auto")
if display not in ("auto", "flat", "vr"):
raise frame_android.FrameError("invalid APK display mode")
m = frame_android.install(str(dest), source=name,
flatscreen=None if display == "auto" else display == "flat")
except frame_android.FrameError as e:
raise Failure(str(e), 400)
return {"message": f"Installed {m['label']} as its own app in the Steam library", "app": m}
kind = "VR app" if not m['flatscreen'] else "app"
notes = " ".join(m.get("vr_issues", []))
return {"message": f"Installed {m['label']} as its own {kind} in the Steam library. {notes}".strip(), "app": m}
return {"message": push_file(dest)}
finally:
if not keep:
@@ -1756,6 +1805,7 @@ def main():
if not frame_host.WINDOWS:
signal.signal(signal.SIGTERM, signal.SIG_IGN)
webinstall_shutdown()
macview.shutdown() # close the headset's viewers before the agent goes
# The master was started with -N, so it stays up until told to exit.
if CONTROL:
subprocess.run([*MUX, "-O", "exit", FRAME], capture_output=True, stdin=subprocess.DEVNULL)