mirror of
https://github.com/saphid/frame-control.git
synced 2026-10-06 01:00:18 +02:00
Mac in the headset: stream Mac windows and screens into the Frame as panels
Frame Control can now show any Mac window, or a whole screen, as its own SteamVR panel on the Steam Frame (Tools -> Mac in the headset, macOS only). Place it with the SteamVR dashboard; the laser clicks and scrolls, and the Mac's own keyboard types. - mac/frame-mac-view (Swift, no dependencies): ScreenCaptureKit capture per window or display, VideoToolbox H.264 with low-latency rate control (JPEG fallback), a loopback HTTP/WebSocket server, CGEvent/AX input playback, and a display-awake assertion while anyone watches. - ui/frame_macview.py: starts the agent, runs an ssh -R tunnel with a supervisor that reopens it on the same port, and launches a Chromium app window per stream on gamescope's :0, tagged with STEAM_GAME for its own panel. - Frame Control's key never leaves the Mac: viewers get single-use, per-source tickets and reconnect keys that Stop revokes. - ui/mac-view.html: WebCodecs decode, keyframe recovery, pointer/wheel/keys back. - Bundled in the Mac app build; tests/test_macview.py builds and drives the agent on macOS. Verified on the Frame (build 20260925.6191901) with the test pattern: panel in about 1.5 s, about 60 fps, Mac-to-window about 11-17 ms, tunnel recovery in 4 s. Laser input and real window capture still need a person in the headset. Also commits the other thread's first-party rule (steam-frame skill) and the first-party options table in docs/streaming.md. Reviewed by GPT-6 Astra (xhigh, read-only) over six rounds; all findings fixed. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
saphidandClaude Opus 5.5 committed
1 parent
0016d9200c
commit
a3c6e5c984
20 files changed
+2708
-6
No files matched your search
@@ -48,3 +48,9 @@ running `--help`: `paste-to-frame.sh`, `serve-bootstrap.sh` and
|
||||
- `sudo` on the Frame asks for the user's Developer Mode password. Hand those
|
||||
steps to the user (open Terminal) and keep automation to non-sudo commands.
|
||||
- The Mac uses BSD userland and zsh (no `timeout`, use `head -n`).
|
||||
- **First-party first.** For any new capability, investigate the first-party
|
||||
way before anything else: Valve (SteamOS, Steam, Steam Link), Apple (the Mac
|
||||
and iPhone), and KDE (the Frame's desktop is Plasma). It's usually the best
|
||||
answer. If it isn't, write down why not. If it is, find what Frame Control
|
||||
can do to make it easier to set up (install over SSH, pre-seed settings,
|
||||
pair automatically, tell the user the one setting to turn on).
|
||||
@@ -5,3 +5,4 @@ apk-catalog/data/index-v2.json*
|
||||
compat-db/.env.lakebed.server
|
||||
compat-db/.lakebed/
|
||||
tests/smoke/results/
|
||||
mac/bin/
|
||||
@@ -192,6 +192,7 @@ Frame's software fits together, all checked against a real headset and labelled
|
||||
| [Sideloading Linux and Windows games](docs/sideloading.md) | A .zip, folder or .exe as a Steam Devkit Game, runtime detection |
|
||||
| [Install links for websites](docs/web-install.md) | `frame-control://install` links and manifests, the rules, a button to paste |
|
||||
| [Steam games](docs/steam-games.md) · [VR video](docs/vr-video.md) · [WebXR in Chromium](docs/webxr-chromium.md) | Installing and buying, watching VR180/360, the Chromium build |
|
||||
| [Mac in the headset](docs/mac-in-headset.md) | Mac windows and screens as panels in the Frame, with laser and keyboard input |
|
||||
| [SSH](docs/ssh.md) · [Streaming](docs/streaming.md) · [Files](docs/file-transfer.md) · [Panels](docs/panels.md) · [Tailscale](docs/tailscale.md) | Topic notes |
|
||||
| [Frame Control for iPhone](docs/iphone.md) | The iPhone and iPad app, how it runs the server on the Frame, pairing |
|
||||
| [Recovery and OS images](docs/recovery-and-images.md) | Where to download the Frame's OS, what's inside, testing without the headset |
|
||||
|
||||
+13
-3
@@ -9,8 +9,8 @@
|
||||
"scripts": {
|
||||
"start": "env -u ELECTRON_RUN_AS_NODE electron .",
|
||||
"icon": "env -u ELECTRON_RUN_AS_NODE electron build/make-icon.js",
|
||||
"dist": "node build/fetch-deps.js mac arm64 && electron-builder --mac --arm64 --publish never",
|
||||
"dist:dir": "node build/fetch-deps.js mac arm64 && electron-builder --mac --arm64 --dir",
|
||||
"dist": "sh ../mac/frame-mac-view/build.sh && node build/fetch-deps.js mac arm64 && electron-builder --mac --arm64 --publish never",
|
||||
"dist:dir": "sh ../mac/frame-mac-view/build.sh && node build/fetch-deps.js mac arm64 && electron-builder --mac --arm64 --dir",
|
||||
"dist:linux": "node build/fetch-deps.js linux x64 arm64 && electron-builder --linux --x64 --arm64 --publish never",
|
||||
"dist:win": "node build/fetch-deps.js win x64 && electron-builder --win --x64 --publish never"
|
||||
},
|
||||
@@ -105,9 +105,19 @@
|
||||
"dmg",
|
||||
"zip"
|
||||
],
|
||||
"extraResources": [
|
||||
{
|
||||
"from": "../mac/bin",
|
||||
"to": "mac/bin",
|
||||
"filter": [
|
||||
"frame-mac-view"
|
||||
]
|
||||
}
|
||||
],
|
||||
"extendInfo": {
|
||||
"NSAppleEventsUsageDescription": "Frame Control opens Terminal for SSH sessions and for power actions that need the Developer Mode password.",
|
||||
"NSLocalNetworkUsageDescription": "Frame Control connects to your Steam Frame over SSH on the local network."
|
||||
"NSLocalNetworkUsageDescription": "Frame Control connects to your Steam Frame over SSH on the local network.",
|
||||
"NSScreenCaptureUsageDescription": "Frame Control shows your Mac's windows and screens inside the Steam Frame when you ask it to."
|
||||
},
|
||||
"artifactName": "Frame-Control-mac-${arch}.${ext}"
|
||||
},
|
||||
|
||||
@@ -70,6 +70,10 @@ counts them while they run.
|
||||
Runtime picked from the program's header), listed under **Sideloaded titles**
|
||||
with Launch and Remove; see [sideloading.md](sideloading.md). Send typed text, or your computer's clipboard, to the
|
||||
Frame clipboard.
|
||||
- **Mac in the headset** (macOS): show any Mac window, or a whole screen, as
|
||||
its own panel in the headset. Place it with the SteamVR dashboard, click and
|
||||
scroll with the laser, and type on the Mac. Streams hardware H.264 through
|
||||
an SSH tunnel; see [mac-in-headset.md](mac-in-headset.md).
|
||||
- **Flatpaks**: install and remove them (quick picks: Moonlight, Firefox, VLC,
|
||||
Remmina).
|
||||
- **One-click tools**: SSH or SFTP in a terminal window, Steam Link, and remote
|
||||
|
||||
@@ -51,6 +51,7 @@ Lepton (Android 11, podman container "lepton-dev") ← its own panel, app 305600
|
||||
| **T3 Code desktop runs natively.** The stock release `T3-Code-0.0.42-arm64.AppImage` in `~/Applications/T3CodeDesktop/` starts with no extra setup: glibc 2.39, `libfuse.so.2`, GTK 3, NSS and libsecret are on the image. `panel-on-frame.sh --name t3code-desktop -- '~/Applications/T3CodeDesktop/T3-Code.AppImage'` gives it its own panel (`valve.steam.desktopgame.2000281357`, `--ozone-platform=x11`). Its bundled server listens on `127.0.0.1:3773` and shows up in onboarding as the `frame` computer, with `passwordStore: gnome-libsecret`. The image has no agent CLI and no `node`. Agents run through the LAN CLIProxyAPI (`llm-proxy.lan:8317`, which resolves on the Frame). Claude Code 2.1.283 comes from `claude.ai/install.sh`, and Codex 0.157.1 from the `codex-aarch64-unknown-linux-musl` release tarball, both into `~/.local/bin`. `with-cliproxy` and a mode-600 `~/.config/cliproxyapi/secrets.env` are copied from the Mac. The wrappers `claude-cliproxy` and `codex-cliproxy` (a `-c model_provider=cliproxy`, `wire_api="responses"`, `env_key="CLIPROXY_API_KEY"`) are set as `providers.claudeAgent.binaryPath` and `providers.codex.binaryPath` in `~/.t3/userdata/settings.json`, and T3 picked that up without a restart. Through the wrappers, `claude auth status` reports `loggedIn: true` (`oauth_token`), and both CLIs answered a prompt with `kimi-k3`. `gamescopectl screenshot` captured another layer (the Lepton T3 app) rather than this panel. `DISPLAY=:0 xwd -id <win>` piped to `ffmpeg` captures the window itself (1920×1080). **Verified 2026-09-26**, BUILD_ID 20260922.6101926. | Running T3 Code as a host on the Frame |
|
||||
| Power actions need `sudo`, which asks for the Developer Mode password over SSH. | Frame Control's power buttons |
|
||||
| **SSH server:** OpenSSH 9.7p1. It offers `publickey,password` (keyboard-interactive is off, PAM on) and also asks `userdbctl ssh-authorized-keys` for keys. OpenSSH ≥ 8.8 rejects SHA-1 `ssh-rsa` signatures by default, so a client whose RSA support is SHA-1 only (the Swift library Citadel, for one) can't log in with the RSA key that devkit pairing installs; use ed25519 (**inferred** from OpenSSH defaults). **Verified 2026-09-27**, BUILD_ID 20260922.6101926. | [iphone.md](iphone.md), `ui/frame_connect.py` |
|
||||
| **A Chromium app window on gamescope's `:0` with its own `STEAM_GAME` becomes a panel, even when started over SSH.** `chromium-xr/chrome --ozone-platform=x11 --app=URL --window-size=1280,720` got a 1920×1080 window, and tagging it produced `[Overlays] Created: valve.steam.desktopgame.<id>` in `~/.local/share/Steam/logs/vrwebhelper_systemui.txt`. Chromium XR decoded a 1080p H.264 WebCodecs stream from the Mac at about 60 fps. XTest events sent to `:0` (libXtst through Python ctypes) did not reach the page. The Frame has `libXtst`, `xprop`, `xwininfo`, `curl` and the `C.utf8` locale. **Verified 2026-09-28**, BUILD_ID 20260925.6191901. | [mac-in-headset.md](mac-in-headset.md) |
|
||||
| **Tools on the image:** Python 3.12.3, `ffmpeg`, `openssl`, `curl`, `rsync`, `zip`/`unzip`, `flatpak`, `wpctl`, `podman`. **No `adb`.** `steamos` is uid 1000, in `wheel`, and sudoers has `%wheel ALL=(ALL) ALL`, so `sudo -S` takes the Developer Mode password on stdin. **Verified 2026-09-27.** | Running Frame Control's server on the Frame (`FRAME_LOCAL=1`, [iphone.md](iphone.md)) |
|
||||
| **Each Lepton instance is a podman container** named `lepton-steamlaunch-<instance id>`, labelled with its ADB port (`podman ps --format '{{.Names}} {{.Labels.adb_port}}'`). `podman exec <container> /system/bin/sh -c '…'` runs Android's shell inside it with no adb at all (used for `pidof` and `logcat` by the app tester). Running `wm size`/`wm density` that way is untested. **Verified 2026-09-27.** | `ui/frame_android.py`, the iPhone app's display settings |
|
||||
| **Asleep means off the network.** In standby the Frame stops answering on its LAN address, `frame.local` and Tailscale alike (`Host is down`, `No route to host`, timeouts), and ping fails. It was unreachable for about 2.5 hours until woken. Nothing over SSH can wake it. **Verified 2026-09-27.** | Frame Control's offline banner and retries |
|
||||
|
||||
@@ -0,0 +1,204 @@
|
||||
# Mac in the headset
|
||||
|
||||
Frame Control can show any Mac window, or a whole Mac screen, as its own panel
|
||||
in the Steam Frame. You place each panel anywhere in the room with the SteamVR
|
||||
dashboard. The laser clicks and drags, the thumbstick scrolls, and you type on
|
||||
the Mac's own keyboard. Find it under **Tools → Mac in the headset** (macOS
|
||||
only).
|
||||
|
||||
The confidence labels are the same as in [ssh.md](ssh.md).
|
||||
|
||||
## Why this design
|
||||
|
||||
First-party options come first, as the repo's rule asks, with the reason
|
||||
each one was or wasn't chosen. The full list for every device is in
|
||||
[streaming.md](streaming.md#first-party-options-and-why-they-do-or-dont-fit).
|
||||
Checked 2026-09-28.
|
||||
|
||||
| Goal | First-party option | Chosen? | Why |
|
||||
|---|---|---|---|
|
||||
| One Mac screen in the headset | **Apple Screen Sharing** (VNC) → Remmina (Remmina 1.4.43 is already installed on this Frame) | Kept as the fallback (`panel-on-frame.sh mac-screen`) | It's the closest to first-party and needs nothing new. But VNC sends compressed tiles rather than video, so moving content is slow. It shows only whole screens |
|
||||
| One Mac screen | **Steam Remote Play**, Mac as host (Valve) | No | macOS isn't a SteamVR host, and Mac-hosted Remote Play is reported broken ([Steam forum](https://steamcommunity.com/groups/homestream/discussions/1/574921459914429988/)). It streams games, not the desktop. **Not tested here**; one real try is still worth doing |
|
||||
| One Mac screen | **AirPlay** (Apple) | No | Apple licenses AirPlay receivers only to TV and speaker makers, and nothing official runs on Linux. UxPlay is an unofficial receiver, and it mirrors a whole screen, not single windows |
|
||||
| One Mac screen | **Sidecar / Mac Virtual Display** (Apple) | No | These work only with an iPad or Apple Vision Pro |
|
||||
| **Each Mac window as its own panel** | None | – | No first-party way does this: Apple's per-app streaming is only for Vision Pro, and Valve's desktop streaming needs a Windows SteamVR host. So Frame Control does it itself |
|
||||
| Mac keyboard and trackpad driving the headset | **Bluetooth HID** | No | macOS can't act as a Bluetooth keyboard or mouse. A real Bluetooth keyboard paired with the Frame still works |
|
||||
| Mac keyboard and trackpad | **KDE Connect** (KDE) | No | The Frame has no `kdeconnectd` and it isn't on Flathub. Its Mac app has no keyboard or mouse sharing (**inferred**), and on Wayland it can only reach the desktop panel |
|
||||
| Mac keyboard and trackpad | **xrdp** (Valve, Developer Mode) | No | It runs a separate Linux session that you view on the Mac. It isn't the headset's view, and it doesn't carry input the other way |
|
||||
|
||||
What that leaves is our own stream: nothing to install on the Mac or the
|
||||
Frame, and whole screens or single windows. Here the Mac's own keyboard and
|
||||
trackpad need no forwarding, because the windows are still on the Mac. The
|
||||
laser is the only input that has to be sent back.
|
||||
|
||||
Other routes that were compared:
|
||||
|
||||
| Option | One screen | Each window | Speed | Verdict |
|
||||
|---|---|---|---|---|
|
||||
| Sunshine → Moonlight | ✓ | – | Good | Sunshine's macOS support is still experimental ([discussion #777](https://github.com/orgs/LizardByte/discussions/777)), and it captures whole screens only |
|
||||
| Virtual Desktop, Immersed | – | – | – | No Frame client as of September 2026 |
|
||||
| **Frame Control's own stream** | ✓ | ✓ | Hardware H.264, sending only changed frames | **Built** |
|
||||
|
||||
To type into VR surfaces other than these panels (SteamVR's dashboard,
|
||||
games), the Frame supports a uinput keyboard and mouse without sudo
|
||||
(verified 2026-09-27: `steamos` is in `input`, and `/dev/uinput` is
|
||||
`root:input 660`). That's a separate feature, not part of this one.
|
||||
|
||||
## How it works
|
||||
|
||||
```
|
||||
Mac Frame
|
||||
ScreenCaptureKit (one window or display)
|
||||
→ VideoToolbox H.264 (hardware, low-latency,
|
||||
no B-frames)
|
||||
→ frame-mac-view, 127.0.0.1 ──ssh -R──→ 127.0.0.1:479xx
|
||||
→ Chromium app window per stream
|
||||
(WebCodecs decode), on gamescope's
|
||||
X display, tagged STEAM_GAME
|
||||
→ its own SteamVR panel
|
||||
← CGEvent (clicks, drags, wheel, keys) ←──── pointer, wheel and key events
|
||||
```
|
||||
|
||||
- **The agent** is `mac/bin/frame-mac-view`, built from `mac/frame-mac-view`
|
||||
(Swift, no dependencies; `build.sh`). Frame Control's server starts it on
|
||||
first use and stops it on quit.
|
||||
- It captures with ScreenCaptureKit, which sends frames only when something
|
||||
changes, so idle windows cost nothing.
|
||||
- It encodes in hardware with VideoToolbox's low-latency rate control (plain
|
||||
real-time mode where that's unavailable).
|
||||
- While anyone is watching, it keeps the Mac's display awake. A sleeping
|
||||
display isn't drawn, so there would be nothing to capture.
|
||||
- **The link** is an `ssh -R` tunnel on its own connection. It's encrypted and
|
||||
works anywhere `ssh frame` works, Tailscale included, with no firewall
|
||||
changes on the Mac. If the headset sleeps or the network drops, Frame
|
||||
Control reopens the tunnel on the same port, and open viewers reconnect by
|
||||
themselves.
|
||||
- **Access.**
|
||||
- Frame Control's own key never leaves the Mac.
|
||||
- Each viewer is opened with a **single-use ticket**. It's tied to one
|
||||
window or display and expires after a minute. It's spent as soon as the
|
||||
viewer confirms it has received its reconnect key. Until then, a retry
|
||||
gets the same key, so a connection lost at that moment doesn't strand
|
||||
the viewer. Stop revokes tickets that haven't been used yet.
|
||||
- After that, the viewer holds a reconnect key for that one source, in
|
||||
memory only. **Stop** revokes it.
|
||||
- Remaining risk: a program running as `steamos` on the Frame could read a
|
||||
ticket from Chromium's command line in the first second or so and use it
|
||||
first. That gets it the one source being opened, not the Mac, and the
|
||||
real viewer would then fail to connect. Android apps in Lepton run in
|
||||
their own podman container, so they shouldn't see the Frame's process
|
||||
list (inferred, not checked).
|
||||
- **The viewer** is `ui/mac-view.html`, served by the agent. It opens on the
|
||||
Frame as a Chromium app window, preferring Chromium XR (`~/chromium-xr`,
|
||||
built with H.264) over Flathub Chromium.
|
||||
- The page puts `[fcNNNNN]` in its title. The launcher finds the window by
|
||||
that tag and sets `STEAM_GAME` to a stable id per source, which gives it
|
||||
its own panel (see [panels.md](panels.md)). The same Mac window gets the
|
||||
same panel id each time.
|
||||
- It decodes with WebCodecs. If it falls behind, it skips to the next
|
||||
keyframe instead of showing old frames late.
|
||||
- It falls back to JPEG stills (**Compatible** quality) where H.264 isn't
|
||||
available.
|
||||
- **Flow control.** When the link backs up, the agent skips capture frames
|
||||
*before* encoding, so no reference frame goes missing. Once the link drains,
|
||||
it sends the newest picture.
|
||||
- **Input.**
|
||||
- A click on a window's panel brings that Mac window to the front
|
||||
(Accessibility API), then clicks at the same point. Double clicks, right
|
||||
clicks, drags and the wheel work too.
|
||||
- Keys typed into the panel are sent as Mac key codes. Any keys or buttons
|
||||
still held down are released if the viewer loses focus or disconnects, or
|
||||
when the stream stops. Characters the key
|
||||
table doesn't know, such as those from other keyboard layouts, are typed
|
||||
as text.
|
||||
- The Mac's own keyboard and trackpad keep working as normal. Click a panel
|
||||
with the laser, then type on the Mac.
|
||||
|
||||
## Permissions (Mac)
|
||||
|
||||
- **Screen Recording**, to see windows. Without it, the card asks for it.
|
||||
- **Accessibility**, so input from the headset reaches the Mac. Without it the
|
||||
stream still works, and the viewer says clicks won't go through.
|
||||
|
||||
Both are granted to Frame Control. After granting, press **Refresh**, which
|
||||
restarts the helper so it picks them up. The app is ad-hoc signed, so macOS
|
||||
may ask again after an update.
|
||||
|
||||
## Quality settings
|
||||
|
||||
| Setting | Long side | fps | Codec | Use |
|
||||
|---|---|---|---|---|
|
||||
| Sharp | 2560 | 60 | H.264, ~0.14 bits/pixel | Text-heavy windows on a strong link |
|
||||
| Balanced (default) | 1920 | 60 | H.264, ~0.1 bits/pixel | Most things |
|
||||
| Light | 1280 | 30 | H.264 | Weak Wi-Fi or Tailscale off the LAN |
|
||||
| Compatible | 1280 | 20 | JPEG | A Frame browser without H.264 |
|
||||
|
||||
## Checked so far (2026-09-28)
|
||||
|
||||
- **Mac (checked by hand, macOS 26.5.2).**
|
||||
- The agent builds, and lists windows and displays.
|
||||
- In a browser, the test pattern decoded at about 60 fps (H.264) and 30 fps
|
||||
(JPEG, about 22 Mbps).
|
||||
- A click in the viewer arrived at the same point in the source.
|
||||
- `pmset -g assertions` showed the display-awake assertion only while a
|
||||
stream was being watched.
|
||||
- **Automated** (`tests/test_macview.py`, on CI's macOS runner): the agent
|
||||
builds (a build failure fails the job).
|
||||
- `/ping` and the page are open; everything else needs the key.
|
||||
- Tickets work once and only for their own source, and Stop revokes
|
||||
reconnect keys.
|
||||
- A WebSocket frame claiming 2^63 bytes closes that socket, and the agent
|
||||
keeps running.
|
||||
- A stream sends an SPS-led H.264 keyframe, and sends another when asked.
|
||||
- Stop ends the stream on the Mac even if the viewer ignores it.
|
||||
- The test doesn't decode video, time it, or check where clicks land.
|
||||
- **Linux aarch64 (verified in a stand-in, not on the Frame).** In an Arch
|
||||
Linux ARM container with sshd, Xvfb as `:0` and Chromium 153:
|
||||
- The real `show` path worked in 1–2.3 s: tunnel, ticket, launcher,
|
||||
window found and tagged `STEAM_GAME`.
|
||||
- Frame Control's key didn't appear anywhere in the stand-in's process
|
||||
list.
|
||||
- After the tunnel was killed, it came back on the same port within 4 s,
|
||||
and the viewer reconnected by itself.
|
||||
- Chromium decoded the stream in software with the GPU off.
|
||||
- Stop closed the window, and Chromium exited.
|
||||
- This test found and fixed a bug: in a C locale, `xwininfo` can't print a
|
||||
title with non-ASCII characters, so the launcher reads `_NET_WM_NAME`
|
||||
with `xprop`.
|
||||
- **On the Frame (verified 2026-09-28, SteamOS build 20260925.6191901,
|
||||
from the Mac, nobody wearing the headset).** Frame Control's **Show** with
|
||||
the test pattern:
|
||||
- The panel was ready in 1.45 s. SteamVR logged `[Overlays] Created:
|
||||
valve.steam.desktopgame.2001639889` (in `vrwebhelper_systemui.txt`).
|
||||
- The window was tagged `STEAM_GAME`, and gamescope sized it to 1920×1080
|
||||
although 1280×720 was asked for.
|
||||
- Chromium XR decoded it live at about 60 fps: the frame counter advanced
|
||||
62 in 1.04 s.
|
||||
- Over home Wi-Fi, the frames in the viewer window had been drawn on the
|
||||
Mac about 11–17 ms earlier, plus `xwd`'s own time. That's measured
|
||||
against the two clocks, which were 37–39 ms apart (±4 ms, measured over
|
||||
one SSH session). SteamVR's compositor and the display come on top.
|
||||
- Clicks and keys injected with XTest into gamescope's Xwayland didn't
|
||||
reach the page. That's inconclusive, not a failure: XTest on gamescope
|
||||
isn't how real input arrives. The laser should arrive as a left mouse
|
||||
button and the thumbstick as a wheel, because the window has an app id
|
||||
(from gamescope's source; see [panels.md](panels.md)).
|
||||
- **Not yet checked:**
|
||||
- Clicking, dragging and scrolling with the laser while wearing the
|
||||
headset.
|
||||
- Real window capture and input on a Mac with both permissions granted.
|
||||
- Keys from SteamVR's on-screen keyboard.
|
||||
- Whether Flathub Chromium has H.264. Chromium XR is used when it's
|
||||
installed, as it is on this Frame.
|
||||
- Latency with real, busy windows at Sharp.
|
||||
|
||||
## Limits
|
||||
|
||||
- Only windows on the Mac's current desktop (Space) are listed, and
|
||||
minimised windows can't be captured.
|
||||
- A window's panel shows only that window. Its menus and sheets are separate
|
||||
windows on the Mac, so open them from the Mac or use **Whole screen**.
|
||||
- Keys go to whichever Mac window is in front. Clicking a panel brings its
|
||||
window to the front first.
|
||||
- Ctrl stays Ctrl. On the Mac, copy is ⌘C, so use Meta+C on a keyboard paired
|
||||
with the Frame.
|
||||
@@ -125,6 +125,19 @@ Still open: 4, 6, 7, 11 (in-headset connect), 12–15, 16 (off-LAN and after a r
|
||||
- The Mac EDL flashing script in `~/Downloads/steam-frame-recovery/` hasn't
|
||||
been run against a Frame.
|
||||
|
||||
## Mac in the headset (2026-09-28)
|
||||
|
||||
The test pattern streams to the Frame as its own panel at about 60 fps
|
||||
(verified, build 20260925.6191901; see
|
||||
[mac-in-headset.md](mac-in-headset.md#checked-so-far-2026-09-28)). Still to
|
||||
check in the headset:
|
||||
|
||||
- Laser clicks, drags and thumbstick scrolling in a viewer panel.
|
||||
- Real window capture and input once Screen Recording and Accessibility are
|
||||
granted to Frame Control.
|
||||
- Keys from SteamVR's on-screen keyboard.
|
||||
- Whether Flathub Chromium decodes H.264 (otherwise use Compatible).
|
||||
|
||||
## Unconfirmed claims made in these docs
|
||||
|
||||
- `/home` and `/etc` persist across Frame OS updates. This is inferred from
|
||||
|
||||
+17
-1
@@ -34,7 +34,8 @@ flat 2D desktop streaming into a window on the Frame's Linux desktop.
|
||||
|
||||
| Option | Setup | Confidence | Verdict |
|
||||
|---|---|---|---|
|
||||
| **macOS Screen Sharing (VNC) → Remmina on the Frame** | **Mac:** System Settings → General → Sharing → Screen Sharing on → (i) → enable "VNC viewers may control screen with password". **Frame:** `./scripts/install-apps.sh remmina` from the Mac, then open Remmina in the headset and connect to `vnc://<mac>.local` | **Inferred.** Remmina is on Flathub for **aarch64** with VNC and RDP ([Flathub](https://flathub.org/apps/org.remmina.Remmina)). The Frame desktop runs Flatpaks ([UploadVR](https://www.uploadvr.com/flatpaks-open-source-steam-frame/)). macOS VNC is built in. | **Recommended.** Nothing to install on the Mac, and it's easy to set up. Latency is fine for productivity but not for games. You'll type the Mac's hostname once in Remmina on the headset, then save the profile. To avoid even that, the script can pre-seed a Remmina profile over SSH (see below). |
|
||||
| **Frame Control → Tools → Mac in the headset** | Nothing to install. Allow Screen Recording and Accessibility for Frame Control, then press **Show** next to any window or screen | **Verified** on the Mac and in an aarch64 Linux stand-in; **not yet tried in the headset** (2026-09-28) | **Recommended.** Hardware H.264 over an SSH tunnel. Each window becomes its own panel you can place anywhere. Laser clicks and scrolls, and the Mac's keyboard types. See [mac-in-headset.md](mac-in-headset.md) |
|
||||
| **macOS Screen Sharing (VNC) → Remmina on the Frame** | **Mac:** System Settings → General → Sharing → Screen Sharing on → (i) → enable "VNC viewers may control screen with password". **Frame:** `./scripts/install-apps.sh remmina` from the Mac, then open Remmina in the headset and connect to `vnc://<mac>.local` | **Inferred.** Remmina is on Flathub for **aarch64** with VNC and RDP ([Flathub](https://flathub.org/apps/org.remmina.Remmina)). The Frame desktop runs Flatpaks ([UploadVR](https://www.uploadvr.com/flatpaks-open-source-steam-frame/)). macOS VNC is built in. | **Fallback.** Nothing to install on the Mac, and it's easy to set up. Latency is fine for productivity but not for games. You'll type the Mac's hostname once in Remmina on the headset, then save the profile. To avoid even that, the script can pre-seed a Remmina profile over SSH (see below). |
|
||||
| Sunshine (Mac) → Moonlight (Frame Flatpak) | `brew install` Sunshine on the Mac, then `./scripts/install-apps.sh moonlight` | Moonlight Flatpak supports **aarch64** ([Flathub](https://flathub.org/apps/com.moonlight_stream.Moonlight)). **Sunshine on macOS is poorly supported**: install problems on Apple Silicon/Sequoia, and no virtual gamepads ([LizardByte discussion #777](https://github.com/orgs/LizardByte/discussions/777)). | Try it if VNC is too laggy. Expect some friction. |
|
||||
| Steam Remote Play with the Mac as host | Steam on the Mac, Steam Link/Remote Play on the Frame | macOS-hosted Remote Play is reported broken or flaky in 2024–2026 ([Steam discussion](https://steamcommunity.com/groups/homestream/discussions/1/574921459914429988/)) | Not recommended. It's only for games, if it works at all. |
|
||||
| Immersed / Virtual Desktop | Vendor apps | Immersed has a Mac agent but no known Frame client. Virtual Desktop's developer said he'd "try" to port it ([NewsBreak](https://www.newsbreak.com/news/4892834783961-virtual-desktop-dev-says-he-ll-try-to-bring-the-app-to-steam-frame)). | Not available as of 2026-09-25. Check again later. |
|
||||
@@ -53,6 +54,21 @@ choose to save it. Remmina stores passwords encrypted with a per-install key,
|
||||
so the script doesn't try to write the password. (The Remmina file format is
|
||||
standard; the Flatpak data path is inferred.)
|
||||
|
||||
## First-party options, and why they do or don't fit
|
||||
|
||||
Checked 2026-09-28. The first-party way is usually the best one, so these are
|
||||
listed first; the sections above and below explain the alternatives.
|
||||
|
||||
| Goal | First-party option | Fits? | Why, and what would make it easier |
|
||||
|---|---|---|---|
|
||||
| Type and point from the **iPhone** | **KDE Connect** (KDE; official [iOS app](https://apps.apple.com/app/kde-connect/id1580245991)) remote touchpad and keyboard, plus clipboard and files | **Best candidate, untested on the Frame** | The Frame doesn't have it (verified: no `kdeconnectd`), it isn't on Flathub, and the root is read-only, so it would have to run from `~` or a container. On Wayland it types through KWin, so it can only reach the desktop panel, not SteamVR or games (**inferred**). Steam Deck users report its remote input breaking after SteamOS updates ([SteamOS #1939](https://github.com/ValveSoftware/SteamOS/issues/1939)). If it works, Frame Control could install it and pair it for you. |
|
||||
| Type and point from the **Mac** | KDE Connect for macOS (KDE builds) | Same as above | Its Mac app sends clipboard and files but has no keyboard/mouse sharing (**inferred**). |
|
||||
| Either | **Bluetooth keyboard and mouse** paired in SteamOS (Valve) | Yes, with real hardware | Neither device can pretend to be one: iOS refuses the HID service ([Apple forums](https://developer.apple.com/forums/thread/733916)), and macOS has no built-in way. |
|
||||
| Either | **xrdp** in Developer Mode (Valve) | No | Input goes into a *separate* desktop shown on the Mac, not into what you see in the headset. |
|
||||
| **Mac screen** in the Frame | **Screen Sharing** (Apple's VNC server) + Remmina (already installed on this Frame, profile pre-seeded by `install-apps.sh`) | **Yes, closest to first-party** | Only the Mac side is first-party; Remmina is the client. Turn on System Settings → General → Sharing → Screen Sharing → (i) → "VNC viewers may control screen with password". Still to test in the headset (open question 11). |
|
||||
| Mac screen | **Steam Remote Play** with the Mac as host (Valve) | Probably not | macOS isn't a SteamVR host, and Mac-hosted Remote Play is reported broken ([Steam forum](https://steamcommunity.com/groups/homestream/discussions/1/574921459914429988/)). One quick test is worth doing: Steam on the Mac, then Remote Play from the Frame's Steam. |
|
||||
| Mac or **iPhone screen** | **AirPlay** (Apple) | Not officially | It's Apple's own mirroring for both, but Apple only licenses receivers to TV and speaker makers; nothing official runs on Linux. UxPlay (below) is the unofficial receiver. |
|
||||
|
||||
## C. Show the iPhone's screen inside the Frame
|
||||
|
||||
iOS only shares its screen two ways: **AirPlay** (Screen Mirroring in Control
|
||||
|
||||
@@ -0,0 +1,339 @@
|
||||
// Where pictures come from: one Mac window or one display (ScreenCaptureKit),
|
||||
// or a generated test pattern that needs no permissions and shows the input
|
||||
// the viewer sends, for checking the whole path without touching the Mac.
|
||||
import AppKit
|
||||
import CoreMedia
|
||||
import CoreVideo
|
||||
import Foundation
|
||||
import ScreenCaptureKit
|
||||
|
||||
enum Source: Equatable {
|
||||
case window(CGWindowID)
|
||||
case display(CGDirectDisplayID)
|
||||
case test
|
||||
|
||||
init?(_ s: String) {
|
||||
let parts = s.split(separator: ":", maxSplits: 1).map(String.init)
|
||||
switch (parts.first, parts.count > 1 ? UInt32(parts[1]) : nil) {
|
||||
case ("window", let id?): self = .window(id)
|
||||
case ("display", let id?): self = .display(id)
|
||||
case ("test", _): self = .test
|
||||
default: return nil
|
||||
}
|
||||
}
|
||||
|
||||
var key: String {
|
||||
switch self {
|
||||
case .window(let id): return "window:\(id)"
|
||||
case .display(let id): return "display:\(id)"
|
||||
case .test: return "test"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Size to capture at: the source's pixel size, shrunk to fit a box whose
|
||||
/// long side is `maxLong` (default 1920), even numbers for the encoder.
|
||||
func fitSize(width: Double, height: Double, maxLong: Int) -> (Int, Int) {
|
||||
let scale = min(1, Double(maxLong) / max(width, height, 1))
|
||||
let w = max(16, Int((width * scale / 2).rounded()) * 2), h = max(16, Int((height * scale / 2).rounded()) * 2)
|
||||
return (w, h)
|
||||
}
|
||||
|
||||
/// What CGWindowList says about a window right now (no permission needed for
|
||||
/// bounds; titles need Screen Recording).
|
||||
struct WindowInfo {
|
||||
let id: CGWindowID
|
||||
let pid: pid_t
|
||||
let app: String
|
||||
let title: String
|
||||
let bounds: CGRect
|
||||
let layer: Int
|
||||
let onScreen: Bool
|
||||
|
||||
static func all(onScreenOnly: Bool = true) -> [WindowInfo] {
|
||||
let opts: CGWindowListOption = onScreenOnly ? [.optionOnScreenOnly, .excludeDesktopElements] : [.excludeDesktopElements]
|
||||
let list = CGWindowListCopyWindowInfo(opts, kCGNullWindowID) as? [[CFString: Any]] ?? []
|
||||
return list.compactMap(WindowInfo.init)
|
||||
}
|
||||
|
||||
static func find(_ id: CGWindowID) -> WindowInfo? {
|
||||
let list = CGWindowListCopyWindowInfo(.optionIncludingWindow, id) as? [[CFString: Any]] ?? []
|
||||
return list.compactMap(WindowInfo.init).first { $0.id == id }
|
||||
}
|
||||
|
||||
init?(_ d: [CFString: Any]) {
|
||||
guard let id = d[kCGWindowNumber] as? CGWindowID, let pid = d[kCGWindowOwnerPID] as? pid_t,
|
||||
let b = d[kCGWindowBounds] as? [String: Any], let rect = CGRect(dictionaryRepresentation: b as CFDictionary)
|
||||
else { return nil }
|
||||
self.id = id
|
||||
self.pid = pid
|
||||
app = d[kCGWindowOwnerName] as? String ?? ""
|
||||
title = d[kCGWindowName] as? String ?? ""
|
||||
bounds = rect
|
||||
layer = d[kCGWindowLayer] as? Int ?? 0
|
||||
onScreen = d[kCGWindowIsOnscreen] as? Bool ?? false
|
||||
}
|
||||
}
|
||||
|
||||
protocol CaptureSource: AnyObject {
|
||||
var onFrame: ((CVPixelBuffer, CMTime) -> Void)? { get set }
|
||||
var onEnded: ((String) -> Void)? { get set }
|
||||
/// Points on the Mac's global display space that the picture covers.
|
||||
var frameRect: CGRect { get }
|
||||
var title: String { get }
|
||||
var app: String { get }
|
||||
var pid: pid_t? { get }
|
||||
func start(maxLong: Int, fps: Int, completion: @escaping (String?) -> Void)
|
||||
func stop()
|
||||
func pointer(x: Double, y: Double, text: String?) // for the test pattern
|
||||
}
|
||||
|
||||
extension CaptureSource {
|
||||
func pointer(x: Double, y: Double, text: String?) {}
|
||||
}
|
||||
|
||||
/// ScreenCaptureKit, for a window or a display.
|
||||
final class SCKSource: NSObject, CaptureSource, SCStreamOutput, SCStreamDelegate {
|
||||
let source: Source
|
||||
var onFrame: ((CVPixelBuffer, CMTime) -> Void)?
|
||||
var onEnded: ((String) -> Void)?
|
||||
private(set) var frameRect = CGRect.zero
|
||||
private(set) var title = ""
|
||||
private(set) var app = ""
|
||||
private(set) var pid: pid_t?
|
||||
private var stream: SCStream?
|
||||
private var config = SCStreamConfiguration()
|
||||
private var maxLong = 1920
|
||||
private var scale = 2.0
|
||||
private var poll: DispatchSourceTimer?
|
||||
private let queue = DispatchQueue(label: "frame-mac-view.capture", qos: .userInteractive)
|
||||
/// Set by stop(), on `queue`; a start still enumerating windows checks it
|
||||
/// before it starts capturing, so a viewer that left early leaves nothing on.
|
||||
private var cancelled = false
|
||||
/// The window or display no longer exists, so retrying can't help.
|
||||
private(set) var gone = false
|
||||
var onChange: (() -> Void)? // title or size changed
|
||||
|
||||
init(_ source: Source) { self.source = source }
|
||||
|
||||
func start(maxLong: Int, fps: Int, completion: @escaping (String?) -> Void) {
|
||||
self.maxLong = maxLong
|
||||
SCShareableContent.getExcludingDesktopWindows(true, onScreenWindowsOnly: false) { [self] content, error in
|
||||
queue.async { self.begin(content, error, fps: fps, completion: completion) }
|
||||
}
|
||||
}
|
||||
|
||||
private func begin(_ content: SCShareableContent?, _ error: Error?, fps: Int, completion: @escaping (String?) -> Void) {
|
||||
if cancelled { return }
|
||||
guard let content else {
|
||||
return completion("Screen Recording isn't allowed for Frame Control (\(error?.localizedDescription ?? "no content"))")
|
||||
}
|
||||
let filter: SCContentFilter
|
||||
switch source {
|
||||
case .window(let id):
|
||||
guard let w = content.windows.first(where: { $0.windowID == id }) else {
|
||||
gone = true
|
||||
return completion("that window has closed")
|
||||
}
|
||||
filter = SCContentFilter(desktopIndependentWindow: w)
|
||||
title = w.title ?? ""
|
||||
app = w.owningApplication?.applicationName ?? ""
|
||||
pid = w.owningApplication?.processID
|
||||
frameRect = w.frame
|
||||
case .display(let id):
|
||||
guard let d = content.displays.first(where: { $0.displayID == id }) else {
|
||||
gone = true
|
||||
return completion("that display isn't connected")
|
||||
}
|
||||
filter = SCContentFilter(display: d, excludingWindows: [])
|
||||
title = displayName(id)
|
||||
app = "Mac"
|
||||
frameRect = CGDisplayBounds(id)
|
||||
case .test:
|
||||
return completion("not a ScreenCaptureKit source")
|
||||
}
|
||||
scale = Double(filter.pointPixelScale)
|
||||
let (w, h) = fitSize(width: frameRect.width * scale, height: frameRect.height * scale, maxLong: maxLong)
|
||||
config.width = w
|
||||
config.height = h
|
||||
config.minimumFrameInterval = CMTime(value: 1, timescale: CMTimeScale(fps))
|
||||
config.pixelFormat = kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange
|
||||
config.colorMatrix = CGDisplayStream.yCbCrMatrix_ITU_R_709_2
|
||||
config.colorSpaceName = CGColorSpace.sRGB
|
||||
config.showsCursor = true
|
||||
config.queueDepth = 5
|
||||
config.scalesToFit = true
|
||||
config.preservesAspectRatio = true
|
||||
config.capturesAudio = false
|
||||
let stream = SCStream(filter: filter, configuration: config, delegate: self)
|
||||
do {
|
||||
try stream.addStreamOutput(self, type: .screen, sampleHandlerQueue: queue)
|
||||
} catch {
|
||||
return completion("couldn't capture: \(error.localizedDescription)")
|
||||
}
|
||||
self.stream = stream
|
||||
stream.startCapture { error in
|
||||
self.queue.async {
|
||||
if self.cancelled {
|
||||
stream.stopCapture { _ in }
|
||||
return
|
||||
}
|
||||
if let error { return completion("couldn't capture: \(error.localizedDescription)") }
|
||||
self.startPolling()
|
||||
completion(nil)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func stop() {
|
||||
queue.async {
|
||||
self.cancelled = true
|
||||
self.poll?.cancel()
|
||||
self.poll = nil
|
||||
self.stream?.stopCapture { _ in }
|
||||
self.stream = nil
|
||||
}
|
||||
}
|
||||
|
||||
/// Windows move, resize, retitle and close; ScreenCaptureKit doesn't say.
|
||||
private func startPolling() {
|
||||
guard case .window(let id) = source else { return }
|
||||
let t = DispatchSource.makeTimerSource(queue: queue)
|
||||
t.schedule(deadline: .now() + 1, repeating: 1)
|
||||
t.setEventHandler { [weak self] in
|
||||
guard let self else { return }
|
||||
guard let info = WindowInfo.find(id) else {
|
||||
self.stop()
|
||||
self.onEnded?("the window closed")
|
||||
return
|
||||
}
|
||||
var changed = false
|
||||
if !info.title.isEmpty, info.title != self.title { self.title = info.title; changed = true }
|
||||
let old = self.frameRect
|
||||
self.frameRect = info.bounds
|
||||
if abs(old.width - info.bounds.width) > 1 || abs(old.height - info.bounds.height) > 1 {
|
||||
let (w, h) = fitSize(width: info.bounds.width * self.scale, height: info.bounds.height * self.scale, maxLong: self.maxLong)
|
||||
self.config.width = w
|
||||
self.config.height = h
|
||||
self.stream?.updateConfiguration(self.config) { _ in }
|
||||
changed = true
|
||||
}
|
||||
if changed { self.onChange?() }
|
||||
}
|
||||
t.resume()
|
||||
poll = t
|
||||
}
|
||||
|
||||
func stream(_ stream: SCStream, didOutputSampleBuffer sample: CMSampleBuffer, of type: SCStreamOutputType) {
|
||||
guard type == .screen, sample.isValid, let pb = CMSampleBufferGetImageBuffer(sample) else { return }
|
||||
// Only complete frames carry new pixels; idle and blank ones don't.
|
||||
if let atts = CMSampleBufferGetSampleAttachmentsArray(sample, createIfNecessary: false) as? [[SCStreamFrameInfo: Any]],
|
||||
let raw = atts.first?[.status] as? Int, let status = SCFrameStatus(rawValue: raw), status != .complete {
|
||||
return
|
||||
}
|
||||
onFrame?(pb, CMSampleBufferGetPresentationTimeStamp(sample))
|
||||
}
|
||||
|
||||
func stream(_ stream: SCStream, didStopWithError error: Error) {
|
||||
onEnded?("capture stopped: \(error.localizedDescription)")
|
||||
}
|
||||
}
|
||||
|
||||
func displayName(_ id: CGDirectDisplayID) -> String {
|
||||
for screen in NSScreen.screens {
|
||||
if (screen.deviceDescription[NSDeviceDescriptionKey("NSScreenNumber")] as? NSNumber)?.uint32Value == id {
|
||||
return screen.localizedName
|
||||
}
|
||||
}
|
||||
return "Display \(id)"
|
||||
}
|
||||
|
||||
/// A moving test card: bars, a clock and a frame counter, plus a dot where the
|
||||
/// viewer's pointer is and the last key it sent, so input can be checked too.
|
||||
final class TestSource: CaptureSource {
|
||||
var onFrame: ((CVPixelBuffer, CMTime) -> Void)?
|
||||
var onEnded: ((String) -> Void)?
|
||||
let frameRect = CGRect(x: 0, y: 0, width: 1280, height: 720)
|
||||
let title = "Test pattern"
|
||||
let app = "Frame Control"
|
||||
let pid: pid_t? = nil
|
||||
private var timer: DispatchSourceTimer?
|
||||
private var pool: CVPixelBufferPool?
|
||||
private var n = 0
|
||||
private var dot: (Double, Double)?
|
||||
private var lastText = "Point or type in the headset"
|
||||
private let queue = DispatchQueue(label: "frame-mac-view.test")
|
||||
private var size = (1280, 720)
|
||||
|
||||
func start(maxLong: Int, fps: Int, completion: @escaping (String?) -> Void) {
|
||||
size = fitSize(width: 1280, height: 720, maxLong: maxLong)
|
||||
let attrs: [CFString: Any] = [kCVPixelBufferPixelFormatTypeKey: kCVPixelFormatType_32BGRA,
|
||||
kCVPixelBufferWidthKey: size.0, kCVPixelBufferHeightKey: size.1,
|
||||
kCVPixelBufferIOSurfacePropertiesKey: [:] as CFDictionary]
|
||||
CVPixelBufferPoolCreate(nil, nil, attrs as CFDictionary, &pool)
|
||||
let t = DispatchSource.makeTimerSource(queue: queue)
|
||||
t.schedule(deadline: .now(), repeating: 1.0 / Double(fps))
|
||||
t.setEventHandler { [weak self] in self?.draw() }
|
||||
t.resume()
|
||||
timer = t
|
||||
completion(nil)
|
||||
}
|
||||
|
||||
func stop() {
|
||||
timer?.cancel()
|
||||
timer = nil
|
||||
}
|
||||
|
||||
func pointer(x: Double, y: Double, text: String?) {
|
||||
queue.async {
|
||||
if x >= 0 { self.dot = (x, y) }
|
||||
if let text { self.lastText = text }
|
||||
}
|
||||
}
|
||||
|
||||
private func draw() {
|
||||
guard let pool else { return }
|
||||
var out: CVPixelBuffer?
|
||||
CVPixelBufferPoolCreatePixelBuffer(nil, pool, &out)
|
||||
guard let pb = out else { return }
|
||||
CVPixelBufferLockBaseAddress(pb, [])
|
||||
defer { CVPixelBufferUnlockBaseAddress(pb, []) }
|
||||
let (w, h) = size
|
||||
guard let ctx = CGContext(data: CVPixelBufferGetBaseAddress(pb), width: w, height: h, bitsPerComponent: 8,
|
||||
bytesPerRow: CVPixelBufferGetBytesPerRow(pb), space: CGColorSpace(name: CGColorSpace.sRGB)!,
|
||||
bitmapInfo: CGImageAlphaInfo.premultipliedFirst.rawValue | CGBitmapInfo.byteOrder32Little.rawValue)
|
||||
else { return }
|
||||
let colors: [(CGFloat, CGFloat, CGFloat)] = [(0.75, 0.75, 0.75), (0.75, 0.75, 0), (0, 0.75, 0.75), (0, 0.75, 0),
|
||||
(0.75, 0, 0.75), (0.75, 0, 0), (0, 0, 0.75)]
|
||||
let bw = CGFloat(w) / CGFloat(colors.count)
|
||||
for (i, c) in colors.enumerated() {
|
||||
ctx.setFillColor(red: c.0, green: c.1, blue: c.2, alpha: 1)
|
||||
ctx.fill(CGRect(x: CGFloat(i) * bw, y: CGFloat(h) * 0.35, width: bw + 1, height: CGFloat(h) * 0.65))
|
||||
}
|
||||
ctx.setFillColor(red: 0.08, green: 0.09, blue: 0.11, alpha: 1)
|
||||
ctx.fill(CGRect(x: 0, y: 0, width: w, height: Int(Double(h) * 0.35)))
|
||||
// A bar sweeping once a second shows motion and dropped frames.
|
||||
let x = CGFloat(n % 60) / 60 * CGFloat(w)
|
||||
ctx.setFillColor(red: 1, green: 1, blue: 1, alpha: 1)
|
||||
ctx.fill(CGRect(x: x, y: CGFloat(h) * 0.35, width: max(4, CGFloat(w) / 120), height: CGFloat(h) * 0.65))
|
||||
let f = DateFormatter()
|
||||
f.dateFormat = "HH:mm:ss.SSS"
|
||||
let label = "Frame Control test pattern \(f.string(from: Date())) frame \(n)\n\(lastText)"
|
||||
let text = NSAttributedString(string: label, attributes: [
|
||||
.font: NSFont.monospacedSystemFont(ofSize: CGFloat(h) / 24, weight: .medium),
|
||||
.foregroundColor: NSColor.white,
|
||||
])
|
||||
let ns = NSGraphicsContext(cgContext: ctx, flipped: false)
|
||||
NSGraphicsContext.saveGraphicsState()
|
||||
NSGraphicsContext.current = ns
|
||||
text.draw(at: CGPoint(x: CGFloat(w) * 0.03, y: CGFloat(h) * 0.08))
|
||||
NSGraphicsContext.restoreGraphicsState()
|
||||
if let (px, py) = dot {
|
||||
let r = CGFloat(h) / 40
|
||||
ctx.setFillColor(red: 1, green: 0.2, blue: 0.2, alpha: 1)
|
||||
ctx.fillEllipse(in: CGRect(x: CGFloat(px) * CGFloat(w) - r, y: (1 - CGFloat(py)) * CGFloat(h) - r, width: 2 * r, height: 2 * r))
|
||||
}
|
||||
n += 1
|
||||
onFrame?(pb, CMClockGetTime(CMClockGetHostTimeClock()))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,172 @@
|
||||
// VideoToolbox encoding: H.264 for the normal path (hardware, low-latency rate
|
||||
// control, no B-frames, Annex B output that WebCodecs takes without a
|
||||
// description), or JPEG stills for viewers that can't decode H.264.
|
||||
import CoreMedia
|
||||
import Foundation
|
||||
import VideoToolbox
|
||||
|
||||
enum Codec: String {
|
||||
case h264, jpeg
|
||||
}
|
||||
|
||||
final class Encoder {
|
||||
let codec: Codec
|
||||
let fps: Int
|
||||
let bitsPerPixel: Double
|
||||
private(set) var width = 0
|
||||
private(set) var height = 0
|
||||
private var session: VTCompressionSession?
|
||||
private var forceKey = true
|
||||
private var lastPts = CMTime.invalid
|
||||
/// Called on VideoToolbox's thread with one access unit (or JPEG) per frame.
|
||||
var onFrame: ((Data, Bool, CMTime) -> Void)?
|
||||
var onError: ((String) -> Void)?
|
||||
/// Called once per frame handed to VideoToolbox, however it went.
|
||||
var onDone: (() -> Void)?
|
||||
|
||||
init(codec: Codec, fps: Int, bitsPerPixel: Double) {
|
||||
self.codec = codec
|
||||
self.fps = fps
|
||||
self.bitsPerPixel = bitsPerPixel
|
||||
}
|
||||
|
||||
deinit { invalidate() }
|
||||
|
||||
func requestKeyFrame() { forceKey = true }
|
||||
|
||||
func invalidate() {
|
||||
if let s = session {
|
||||
VTCompressionSessionCompleteFrames(s, untilPresentationTimeStamp: .invalid)
|
||||
VTCompressionSessionInvalidate(s)
|
||||
}
|
||||
session = nil
|
||||
}
|
||||
|
||||
private func makeSession(width w: Int, height h: Int) -> Bool {
|
||||
invalidate()
|
||||
var spec: [CFString: Any] = [:]
|
||||
if codec == .h264 { spec[kVTVideoEncoderSpecification_EnableLowLatencyRateControl] = true }
|
||||
var s: VTCompressionSession?
|
||||
let type = codec == .h264 ? kCMVideoCodecType_H264 : kCMVideoCodecType_JPEG
|
||||
func create(_ spec: [CFString: Any]) -> OSStatus {
|
||||
VTCompressionSessionCreate(allocator: nil, width: Int32(w), height: Int32(h), codecType: type,
|
||||
encoderSpecification: spec as CFDictionary, imageBufferAttributes: nil,
|
||||
compressedDataAllocator: nil, outputCallback: nil, refcon: nil,
|
||||
compressionSessionOut: &s)
|
||||
}
|
||||
var err = create(spec)
|
||||
// Low-latency rate control needs Apple's hardware encoder; without it
|
||||
// (some VMs), plain real-time encoding still works.
|
||||
if err != noErr, !spec.isEmpty { err = create([:]) }
|
||||
guard err == noErr, let s else {
|
||||
onError?("couldn't start the \(codec.rawValue) encoder (VideoToolbox \(err))")
|
||||
return false
|
||||
}
|
||||
func set(_ key: CFString, _ value: Any) { VTSessionSetProperty(s, key: key, value: value as CFTypeRef) }
|
||||
set(kVTCompressionPropertyKey_RealTime, true)
|
||||
set(kVTCompressionPropertyKey_ColorPrimaries, kCVImageBufferColorPrimaries_ITU_R_709_2)
|
||||
set(kVTCompressionPropertyKey_TransferFunction, kCVImageBufferTransferFunction_ITU_R_709_2)
|
||||
set(kVTCompressionPropertyKey_YCbCrMatrix, kCVImageBufferYCbCrMatrix_ITU_R_709_2)
|
||||
if codec == .h264 {
|
||||
let bps = min(max(Double(w * h * fps) * bitsPerPixel, 2_000_000), 60_000_000)
|
||||
set(kVTCompressionPropertyKey_ProfileLevel, kVTProfileLevel_H264_ConstrainedHigh_AutoLevel)
|
||||
set(kVTCompressionPropertyKey_AllowFrameReordering, false)
|
||||
set(kVTCompressionPropertyKey_AverageBitRate, Int(bps))
|
||||
set(kVTCompressionPropertyKey_ExpectedFrameRate, fps)
|
||||
// A keyframe every 10 s at most, so a viewer that lost one recovers
|
||||
// even if it never asks. Viewers ask for one when they start.
|
||||
set(kVTCompressionPropertyKey_MaxKeyFrameIntervalDuration, 10)
|
||||
} else {
|
||||
set(kVTCompressionPropertyKey_Quality, 0.8)
|
||||
}
|
||||
VTCompressionSessionPrepareToEncodeFrames(s)
|
||||
session = s
|
||||
lastPts = .invalid
|
||||
width = w
|
||||
height = h
|
||||
forceKey = true
|
||||
return true
|
||||
}
|
||||
|
||||
/// False if the frame never reached VideoToolbox (then onDone won't come).
|
||||
@discardableResult
|
||||
func encode(_ pb: CVPixelBuffer, pts given: CMTime) -> Bool {
|
||||
// VideoToolbox needs strictly increasing timestamps; a resent picture
|
||||
// stamped "now" can be followed by a capture stamped a moment earlier.
|
||||
var pts = given
|
||||
if lastPts.isValid, CMTimeCompare(pts, lastPts) <= 0 { pts = CMTimeAdd(lastPts, CMTime(value: 1, timescale: 1_000_000)) }
|
||||
let w = CVPixelBufferGetWidth(pb), h = CVPixelBufferGetHeight(pb)
|
||||
if session == nil || w != width || h != height {
|
||||
guard makeSession(width: w, height: h) else { return false }
|
||||
}
|
||||
guard let s = session else { return false }
|
||||
var props: CFDictionary?
|
||||
if forceKey {
|
||||
props = [kVTEncodeFrameOptionKey_ForceKeyFrame: true] as CFDictionary
|
||||
forceKey = false
|
||||
}
|
||||
let codec = self.codec
|
||||
lastPts = pts
|
||||
let status = VTCompressionSessionEncodeFrame(s, imageBuffer: pb, presentationTimeStamp: pts, duration: .invalid,
|
||||
frameProperties: props, infoFlagsOut: nil) { [weak self] status, _, sample in
|
||||
guard let self else { return }
|
||||
defer { self.onDone?() }
|
||||
guard status == noErr, let sample else { return }
|
||||
if codec == .jpeg {
|
||||
if let data = Self.bytes(sample) { self.onFrame?(data, true, pts) }
|
||||
} else if let (data, key) = Self.annexB(sample) {
|
||||
self.onFrame?(data, key, pts)
|
||||
}
|
||||
}
|
||||
if status != noErr { forceKey = true }
|
||||
return status == noErr
|
||||
}
|
||||
|
||||
private static func bytes(_ sample: CMSampleBuffer) -> Data? {
|
||||
guard let block = CMSampleBufferGetDataBuffer(sample) else { return nil }
|
||||
var length = 0
|
||||
var ptr: UnsafeMutablePointer<CChar>?
|
||||
guard CMBlockBufferGetDataPointer(block, atOffset: 0, lengthAtOffsetOut: nil, totalLengthOut: &length,
|
||||
dataPointerOut: &ptr) == noErr, let ptr else { return nil }
|
||||
return Data(bytes: ptr, count: length)
|
||||
}
|
||||
|
||||
/// AVCC sample -> Annex B access unit, with SPS and PPS before keyframes.
|
||||
static func annexB(_ sample: CMSampleBuffer) -> (Data, Bool)? {
|
||||
guard let avcc = bytes(sample) else { return nil }
|
||||
var key = true
|
||||
if let atts = CMSampleBufferGetSampleAttachmentsArray(sample, createIfNecessary: false) as? [[CFString: Any]],
|
||||
let first = atts.first, first[kCMSampleAttachmentKey_NotSync] as? Bool == true {
|
||||
key = false
|
||||
}
|
||||
let start: [UInt8] = [0, 0, 0, 1]
|
||||
var out = Data()
|
||||
if key, let fmt = CMSampleBufferGetFormatDescription(sample) {
|
||||
var count = 0
|
||||
CMVideoFormatDescriptionGetH264ParameterSetAtIndex(fmt, parameterSetIndex: 0, parameterSetPointerOut: nil,
|
||||
parameterSetSizeOut: nil, parameterSetCountOut: &count,
|
||||
nalUnitHeaderLengthOut: nil)
|
||||
for i in 0..<count {
|
||||
var p: UnsafePointer<UInt8>?
|
||||
var n = 0
|
||||
if CMVideoFormatDescriptionGetH264ParameterSetAtIndex(fmt, parameterSetIndex: i, parameterSetPointerOut: &p,
|
||||
parameterSetSizeOut: &n, parameterSetCountOut: nil,
|
||||
nalUnitHeaderLengthOut: nil) == noErr, let p {
|
||||
out.append(contentsOf: start)
|
||||
out.append(p, count: n)
|
||||
}
|
||||
}
|
||||
}
|
||||
let bytes = [UInt8](avcc)
|
||||
var i = 0
|
||||
while i + 4 <= bytes.count {
|
||||
let n = Int(bytes[i]) << 24 | Int(bytes[i + 1]) << 16 | Int(bytes[i + 2]) << 8 | Int(bytes[i + 3])
|
||||
i += 4
|
||||
guard n > 0, i + n <= bytes.count else { break }
|
||||
out.append(contentsOf: start)
|
||||
out.append(contentsOf: bytes[i..<(i + n)])
|
||||
i += n
|
||||
}
|
||||
return (out, key)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,168 @@
|
||||
// Turns the viewer's pointer and key events into real Mac input. Needs the
|
||||
// Accessibility permission ("control your computer"); without it macOS drops
|
||||
// the events silently, so the agent reports the permission to the viewer.
|
||||
import AppKit
|
||||
import ApplicationServices
|
||||
import Foundation
|
||||
|
||||
@_silgen_name("_AXUIElementGetWindow")
|
||||
private func _AXUIElementGetWindow(_ element: AXUIElement, _ id: UnsafeMutablePointer<CGWindowID>) -> AXError
|
||||
|
||||
enum Input {
|
||||
static let source = CGEventSource(stateID: .hidSystemState)
|
||||
/// What each viewer (session id) is holding down, so one panel closing
|
||||
/// lets go of its own keys and buttons and nobody else's. Main thread.
|
||||
private static var buttonsHeld: [Int: Set<Int>] = [:]
|
||||
private static var keysHeld: [Int: Set<CGKeyCode>] = [:]
|
||||
|
||||
private static var lastDown: (time: TimeInterval, point: CGPoint, button: Int, count: Int)?
|
||||
|
||||
static var allowed: Bool { AXIsProcessTrusted() }
|
||||
|
||||
/// Brings a window to the front so a click lands on it, not on whatever
|
||||
/// covers it, and typing goes to it.
|
||||
static func focus(window id: CGWindowID, pid: pid_t) {
|
||||
if frontWindow() == id { return }
|
||||
let app = AXUIElementCreateApplication(pid)
|
||||
var value: CFTypeRef?
|
||||
if AXUIElementCopyAttributeValue(app, kAXWindowsAttribute as CFString, &value) == .success,
|
||||
let windows = value as? [AXUIElement] {
|
||||
for w in windows {
|
||||
var wid: CGWindowID = 0
|
||||
if _AXUIElementGetWindow(w, &wid) == .success, wid == id {
|
||||
AXUIElementPerformAction(w, kAXRaiseAction as CFString)
|
||||
AXUIElementSetAttributeValue(w, kAXMainAttribute as CFString, kCFBooleanTrue)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
AXUIElementSetAttributeValue(app, kAXFrontmostAttribute as CFString, kCFBooleanTrue)
|
||||
NSRunningApplication(processIdentifier: pid)?.activate()
|
||||
}
|
||||
|
||||
static func frontWindow() -> CGWindowID? {
|
||||
WindowInfo.all().first { $0.layer == 0 }?.id
|
||||
}
|
||||
|
||||
/// `button` uses the browser's numbering: 0 left, 1 middle, 2 right.
|
||||
static func mouse(_ kind: String, button: Int, at p: CGPoint, owner: Int) {
|
||||
let b: CGMouseButton = button == 2 ? .right : button == 1 ? .center : .left
|
||||
let type: CGEventType
|
||||
switch kind {
|
||||
case "down":
|
||||
type = b == .left ? .leftMouseDown : b == .right ? .rightMouseDown : .otherMouseDown
|
||||
buttonsHeld[owner, default: []].insert(button)
|
||||
case "up":
|
||||
type = b == .left ? .leftMouseUp : b == .right ? .rightMouseUp : .otherMouseUp
|
||||
buttonsHeld[owner]?.remove(button)
|
||||
// Another viewer still holding it keeps it down.
|
||||
if buttonsHeld.values.contains(where: { $0.contains(button) }) { return }
|
||||
default: // a drag is whatever this viewer is holding
|
||||
let mine = buttonsHeld[owner] ?? []
|
||||
if mine.contains(0) { type = .leftMouseDragged }
|
||||
else if mine.contains(2) { type = .rightMouseDragged }
|
||||
else if mine.contains(1) { type = .otherMouseDragged }
|
||||
else { type = .mouseMoved }
|
||||
}
|
||||
let mine = buttonsHeld[owner] ?? []
|
||||
let held: CGMouseButton = mine.contains(0) ? .left : mine.contains(2) ? .right : mine.contains(1) ? .center : .left
|
||||
guard let e = CGEvent(mouseEventSource: source, mouseType: type, mouseCursorPosition: p,
|
||||
mouseButton: kind == "move" ? held : b) else { return }
|
||||
if kind == "down" || kind == "up" {
|
||||
let now = ProcessInfo.processInfo.systemUptime
|
||||
var count = 1
|
||||
if kind == "down" {
|
||||
if let l = lastDown, l.button == button, now - l.time < NSEvent.doubleClickInterval,
|
||||
abs(l.point.x - p.x) < 5, abs(l.point.y - p.y) < 5 { count = l.count + 1 }
|
||||
lastDown = (now, p, button, count)
|
||||
} else if let l = lastDown, l.button == button {
|
||||
count = l.count
|
||||
}
|
||||
e.setIntegerValueField(.mouseEventClickState, value: Int64(count))
|
||||
}
|
||||
e.post(tap: .cghidEventTap)
|
||||
}
|
||||
|
||||
static func scroll(dx: Double, dy: Double, at p: CGPoint, owner: Int) {
|
||||
mouse("move", button: 0, at: p, owner: owner)
|
||||
// Browsers report pixels with +y meaning "scroll down"; macOS's +y is up.
|
||||
guard let e = CGEvent(scrollWheelEvent2Source: source, units: .pixel, wheelCount: 2,
|
||||
wheel1: Int32(-dy.rounded()), wheel2: Int32(-dx.rounded()), wheel3: 0) else { return }
|
||||
e.location = p
|
||||
e.post(tap: .cghidEventTap)
|
||||
}
|
||||
|
||||
/// Lets go of every button and key this viewer is holding down, so a
|
||||
/// dropped connection can't leave, say, Shift or ⌘ stuck on.
|
||||
static func releaseAll(owner: Int) {
|
||||
let p = CGEvent(source: nil)?.location ?? .zero
|
||||
let buttons = buttonsHeld.removeValue(forKey: owner) ?? []
|
||||
let keys = keysHeld.removeValue(forKey: owner) ?? []
|
||||
// Only what no other viewer is still holding.
|
||||
for b in buttons where !buttonsHeld.values.contains(where: { $0.contains(b) }) {
|
||||
let type: CGEventType = b == 2 ? .rightMouseUp : b == 1 ? .otherMouseUp : .leftMouseUp
|
||||
CGEvent(mouseEventSource: source, mouseType: type, mouseCursorPosition: p,
|
||||
mouseButton: b == 2 ? .right : b == 1 ? .center : .left)?.post(tap: .cghidEventTap)
|
||||
}
|
||||
for k in keys where !keysHeld.values.contains(where: { $0.contains(k) }) {
|
||||
CGEvent(keyboardEventSource: source, virtualKey: k, keyDown: false)?.post(tap: .cghidEventTap)
|
||||
}
|
||||
}
|
||||
|
||||
static func key(code: String, key: String, down: Bool, mods: [String], owner: Int) {
|
||||
var flags = CGEventFlags()
|
||||
if mods.contains("shift") { flags.insert(.maskShift) }
|
||||
if mods.contains("ctrl") { flags.insert(.maskControl) }
|
||||
if mods.contains("alt") { flags.insert(.maskAlternate) }
|
||||
if mods.contains("meta") { flags.insert(.maskCommand) }
|
||||
if let vk = keyCodes[code] {
|
||||
guard let e = CGEvent(keyboardEventSource: source, virtualKey: vk, keyDown: down) else { return }
|
||||
if down {
|
||||
keysHeld[owner, default: []].insert(vk)
|
||||
} else {
|
||||
keysHeld[owner]?.remove(vk)
|
||||
if keysHeld.values.contains(where: { $0.contains(vk) }) { return } // still held elsewhere
|
||||
}
|
||||
e.flags = flags
|
||||
e.post(tap: .cghidEventTap)
|
||||
} else if down, !key.isEmpty, key.count <= 4, key.unicodeScalars.allSatisfy({ $0.value >= 0x20 }) {
|
||||
// A key the table doesn't know (a non-US layout, the headset's
|
||||
// on-screen keyboard): type its character instead.
|
||||
text(key)
|
||||
}
|
||||
}
|
||||
|
||||
static func text(_ s: String) {
|
||||
let units = Array(s.utf16)
|
||||
for chunk in stride(from: 0, to: units.count, by: 16) {
|
||||
let part = Array(units[chunk..<min(chunk + 16, units.count)])
|
||||
for down in [true, false] {
|
||||
guard let e = CGEvent(keyboardEventSource: source, virtualKey: 0, keyDown: down) else { continue }
|
||||
e.keyboardSetUnicodeString(stringLength: part.count, unicodeString: part)
|
||||
e.post(tap: .cghidEventTap)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// DOM KeyboardEvent.code -> macOS virtual key code (ANSI positions).
|
||||
static let keyCodes: [String: CGKeyCode] = [
|
||||
"KeyA": 0x00, "KeyS": 0x01, "KeyD": 0x02, "KeyF": 0x03, "KeyH": 0x04, "KeyG": 0x05, "KeyZ": 0x06, "KeyX": 0x07,
|
||||
"KeyC": 0x08, "KeyV": 0x09, "IntlBackslash": 0x0A, "KeyB": 0x0B, "KeyQ": 0x0C, "KeyW": 0x0D, "KeyE": 0x0E,
|
||||
"KeyR": 0x0F, "KeyY": 0x10, "KeyT": 0x11, "Digit1": 0x12, "Digit2": 0x13, "Digit3": 0x14, "Digit4": 0x15,
|
||||
"Digit6": 0x16, "Digit5": 0x17, "Equal": 0x18, "Digit9": 0x19, "Digit7": 0x1A, "Minus": 0x1B, "Digit8": 0x1C,
|
||||
"Digit0": 0x1D, "BracketRight": 0x1E, "KeyO": 0x1F, "KeyU": 0x20, "BracketLeft": 0x21, "KeyI": 0x22,
|
||||
"KeyP": 0x23, "Enter": 0x24, "KeyL": 0x25, "KeyJ": 0x26, "Quote": 0x27, "KeyK": 0x28, "Semicolon": 0x29,
|
||||
"Backslash": 0x2A, "Comma": 0x2B, "Slash": 0x2C, "KeyN": 0x2D, "KeyM": 0x2E, "Period": 0x2F, "Tab": 0x30,
|
||||
"Space": 0x31, "Backquote": 0x32, "Backspace": 0x33, "Escape": 0x35, "MetaRight": 0x36, "MetaLeft": 0x37,
|
||||
"ShiftLeft": 0x38, "CapsLock": 0x39, "AltLeft": 0x3A, "ControlLeft": 0x3B, "ShiftRight": 0x3C,
|
||||
"AltRight": 0x3D, "ControlRight": 0x3E, "F17": 0x40, "NumpadDecimal": 0x41, "NumpadMultiply": 0x43,
|
||||
"NumpadAdd": 0x45, "NumLock": 0x47, "NumpadDivide": 0x4B, "NumpadEnter": 0x4C, "NumpadSubtract": 0x4E,
|
||||
"F18": 0x4F, "F19": 0x50, "NumpadEqual": 0x51, "Numpad0": 0x52, "Numpad1": 0x53, "Numpad2": 0x54,
|
||||
"Numpad3": 0x55, "Numpad4": 0x56, "Numpad5": 0x57, "Numpad6": 0x58, "Numpad7": 0x59, "F20": 0x5A,
|
||||
"Numpad8": 0x5B, "Numpad9": 0x5C, "F5": 0x60, "F6": 0x61, "F7": 0x62, "F3": 0x63, "F8": 0x64, "F9": 0x65,
|
||||
"F11": 0x67, "F13": 0x69, "F16": 0x6A, "F14": 0x6B, "F10": 0x6D, "ContextMenu": 0x6E, "F12": 0x6F,
|
||||
"F15": 0x71, "Insert": 0x72, "Help": 0x72, "Home": 0x73, "PageUp": 0x74, "Delete": 0x75, "F4": 0x76,
|
||||
"End": 0x77, "F2": 0x78, "PageDown": 0x79, "F1": 0x7A, "ArrowLeft": 0x7B, "ArrowRight": 0x7C,
|
||||
"ArrowDown": 0x7D, "ArrowUp": 0x7E, "OSLeft": 0x37, "OSRight": 0x36,
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,267 @@
|
||||
// A small HTTP/1.1 + WebSocket server on Network.framework, loopback only.
|
||||
// Enough for the agent's JSON endpoints, the viewer page and one WebSocket per
|
||||
// stream; not a general web server.
|
||||
import CryptoKit
|
||||
import Foundation
|
||||
import Network
|
||||
|
||||
struct Request {
|
||||
let method: String
|
||||
let path: String
|
||||
let query: [String: String]
|
||||
let headers: [String: String] // lower-cased names
|
||||
}
|
||||
|
||||
final class Server {
|
||||
let queue = DispatchQueue(label: "frame-mac-view.server")
|
||||
private let listener: NWListener
|
||||
private let handle: (Request, HTTPConnection) -> Void
|
||||
|
||||
init(port: UInt16, handle: @escaping (Request, HTTPConnection) -> Void) throws {
|
||||
let tcp = NWProtocolTCP.Options()
|
||||
tcp.noDelay = true
|
||||
let params = NWParameters(tls: nil, tcp: tcp)
|
||||
// Port 0 lets the system pick; `port` then says which.
|
||||
params.requiredLocalEndpoint = .hostPort(host: "127.0.0.1", port: NWEndpoint.Port(rawValue: port) ?? .any)
|
||||
params.allowLocalEndpointReuse = true
|
||||
listener = try NWListener(using: params)
|
||||
self.handle = handle
|
||||
}
|
||||
|
||||
var port: UInt16? { listener.port?.rawValue }
|
||||
|
||||
func start(ready: @escaping (Error?) -> Void) {
|
||||
listener.stateUpdateHandler = { state in
|
||||
switch state {
|
||||
case .ready: ready(nil)
|
||||
case .failed(let e): ready(e)
|
||||
default: break
|
||||
}
|
||||
}
|
||||
listener.newConnectionHandler = { [weak self] conn in
|
||||
guard let self else { return }
|
||||
HTTPConnection(conn, queue: self.queue, handle: self.handle).start()
|
||||
}
|
||||
listener.start(queue: queue)
|
||||
}
|
||||
}
|
||||
|
||||
final class HTTPConnection {
|
||||
let conn: NWConnection
|
||||
let queue: DispatchQueue
|
||||
private let handle: (Request, HTTPConnection) -> Void
|
||||
private var buffer = Data()
|
||||
private var retained: HTTPConnection? // alive until the response is sent
|
||||
|
||||
init(_ conn: NWConnection, queue: DispatchQueue, handle: @escaping (Request, HTTPConnection) -> Void) {
|
||||
self.conn = conn
|
||||
self.queue = queue
|
||||
self.handle = handle
|
||||
}
|
||||
|
||||
func start() {
|
||||
retained = self
|
||||
conn.start(queue: queue)
|
||||
readHead()
|
||||
}
|
||||
|
||||
private func readHead() {
|
||||
conn.receive(minimumIncompleteLength: 1, maximumLength: 16384) { [self] data, _, done, error in
|
||||
if let data { buffer.append(data) }
|
||||
if let end = buffer.range(of: Data("\r\n\r\n".utf8)) {
|
||||
guard let req = Self.parse(buffer[..<end.lowerBound]) else { return respond(400, text: "bad request") }
|
||||
handle(req, self)
|
||||
} else if error != nil || done || buffer.count > 16384 {
|
||||
close()
|
||||
} else {
|
||||
readHead()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static func parse(_ head: Data) -> Request? {
|
||||
guard let text = String(data: head, encoding: .utf8) else { return nil }
|
||||
let lines = text.components(separatedBy: "\r\n")
|
||||
let parts = lines[0].split(separator: " ")
|
||||
guard parts.count >= 2, let url = URLComponents(string: String(parts[1])) else { return nil }
|
||||
var headers: [String: String] = [:]
|
||||
for line in lines.dropFirst() {
|
||||
guard let colon = line.firstIndex(of: ":") else { continue }
|
||||
headers[line[..<colon].lowercased()] = line[line.index(after: colon)...].trimmingCharacters(in: .whitespaces)
|
||||
}
|
||||
var query: [String: String] = [:]
|
||||
for item in url.queryItems ?? [] { query[item.name] = item.value ?? "" }
|
||||
return Request(method: String(parts[0]), path: url.path, query: query, headers: headers)
|
||||
}
|
||||
|
||||
func respond(_ status: Int, body: Data, type: String) {
|
||||
let reason = [200: "OK", 400: "Bad Request", 403: "Forbidden", 404: "Not Found", 409: "Conflict", 500: "Internal Server Error"][status] ?? "Error"
|
||||
var head = "HTTP/1.1 \(status) \(reason)\r\nContent-Type: \(type)\r\nContent-Length: \(body.count)\r\n"
|
||||
head += "Cache-Control: no-store\r\nConnection: close\r\n\r\n"
|
||||
conn.send(content: Data(head.utf8) + body, isComplete: true, completion: .contentProcessed { [self] _ in close() })
|
||||
}
|
||||
|
||||
func respond(_ status: Int, text: String) {
|
||||
respond(status, body: Data(text.utf8), type: "text/plain; charset=utf-8")
|
||||
}
|
||||
|
||||
func respond(_ status: Int = 200, json: Any) {
|
||||
let body = (try? JSONSerialization.data(withJSONObject: json, options: [.sortedKeys])) ?? Data("{}".utf8)
|
||||
respond(status, body: body, type: "application/json")
|
||||
}
|
||||
|
||||
func close() {
|
||||
conn.cancel()
|
||||
retained = nil
|
||||
}
|
||||
|
||||
/// Completes the WebSocket handshake and hands the connection over.
|
||||
func upgrade(_ req: Request) -> WebSocket? {
|
||||
guard req.headers["upgrade"]?.lowercased() == "websocket", let key = req.headers["sec-websocket-key"] else {
|
||||
respond(400, text: "expected a WebSocket upgrade")
|
||||
return nil
|
||||
}
|
||||
let accept = Data(Insecure.SHA1.hash(data: Data((key + "258EAFA5-E914-47DA-95CA-C5AB0DC85B11").utf8))).base64EncodedString()
|
||||
let head = "HTTP/1.1 101 Switching Protocols\r\nUpgrade: websocket\r\nConnection: Upgrade\r\nSec-WebSocket-Accept: \(accept)\r\n\r\n"
|
||||
conn.send(content: Data(head.utf8), completion: .contentProcessed { _ in })
|
||||
let ws = WebSocket(conn, queue: queue)
|
||||
retained = nil
|
||||
return ws
|
||||
}
|
||||
}
|
||||
|
||||
/// Server side of RFC 6455. Sends are counted until the network stack has
|
||||
/// taken them, so the stream can skip frames instead of queueing seconds of
|
||||
/// video on a slow link.
|
||||
final class WebSocket {
|
||||
static let maxMessage: UInt64 = 1 << 20
|
||||
let conn: NWConnection
|
||||
let queue: DispatchQueue
|
||||
var onText: ((String) -> Void)?
|
||||
var onClose: (() -> Void)?
|
||||
private var buffer = Data()
|
||||
private var fragments = Data()
|
||||
private var fragmentOpcode: UInt8 = 0
|
||||
private var closed = false
|
||||
private var retained: WebSocket?
|
||||
private let lock = NSLock()
|
||||
private var _pending = 0
|
||||
var onDrain: (() -> Void)?
|
||||
|
||||
/// Bytes handed to the connection that it hasn't sent yet. Any thread.
|
||||
var pendingBytes: Int { lock.lock(); defer { lock.unlock() }; return _pending }
|
||||
|
||||
init(_ conn: NWConnection, queue: DispatchQueue) {
|
||||
self.conn = conn
|
||||
self.queue = queue
|
||||
}
|
||||
|
||||
func start() {
|
||||
retained = self
|
||||
read()
|
||||
}
|
||||
|
||||
func sendText(_ s: String) { send(opcode: 1, Data(s.utf8)) }
|
||||
func sendJSON(_ obj: Any) {
|
||||
if let d = try? JSONSerialization.data(withJSONObject: obj), let s = String(data: d, encoding: .utf8) { sendText(s) }
|
||||
}
|
||||
func sendBinary(_ d: Data) { send(opcode: 2, d) }
|
||||
|
||||
func send(opcode: UInt8, _ payload: Data) {
|
||||
var frame = Data([0x80 | opcode])
|
||||
let n = payload.count
|
||||
if n < 126 {
|
||||
frame.append(UInt8(n))
|
||||
} else if n < 65536 {
|
||||
frame.append(126)
|
||||
frame.append(contentsOf: [UInt8(n >> 8), UInt8(n & 0xff)])
|
||||
} else {
|
||||
frame.append(127)
|
||||
for shift in stride(from: 56, through: 0, by: -8) { frame.append(UInt8((UInt64(n) >> UInt64(shift)) & 0xff)) }
|
||||
}
|
||||
frame.append(payload)
|
||||
let size = frame.count
|
||||
lock.lock(); _pending += size; lock.unlock()
|
||||
conn.send(content: frame, completion: .contentProcessed { [weak self] _ in
|
||||
guard let self else { return }
|
||||
self.lock.lock(); self._pending -= size; self.lock.unlock()
|
||||
self.onDrain?()
|
||||
})
|
||||
}
|
||||
|
||||
func close() {
|
||||
guard !closed else { return }
|
||||
closed = true
|
||||
send(opcode: 8, Data([0x03, 0xe8])) // 1000, normal closure
|
||||
conn.send(content: nil, isComplete: true, completion: .contentProcessed { [weak self] _ in self?.conn.cancel() })
|
||||
finish()
|
||||
}
|
||||
|
||||
private func finish() {
|
||||
let cb = onClose
|
||||
onClose = nil
|
||||
onText = nil
|
||||
onDrain = nil
|
||||
cb?()
|
||||
retained = nil
|
||||
}
|
||||
|
||||
private func read() {
|
||||
conn.receive(minimumIncompleteLength: 1, maximumLength: 65536) { [weak self] data, _, done, error in
|
||||
guard let self, !self.closed else { return }
|
||||
if let data { self.buffer.append(data) }
|
||||
self.parse()
|
||||
if error != nil || done {
|
||||
self.closed = true
|
||||
self.conn.cancel()
|
||||
self.finish()
|
||||
} else if !self.closed {
|
||||
self.read()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private func parse() {
|
||||
while buffer.count >= 2 {
|
||||
let b = [UInt8](buffer.prefix(14))
|
||||
let fin = b[0] & 0x80 != 0, opcode = b[0] & 0x0f, masked = b[1] & 0x80 != 0
|
||||
// Lengths are unsigned and clients only send small control
|
||||
// messages, so anything big (or a 64-bit length with the top bit
|
||||
// set) is refused before it's turned into an Int.
|
||||
var len64 = UInt64(b[1] & 0x7f), off = 2
|
||||
if len64 == 126 {
|
||||
guard b.count >= 4 else { return }
|
||||
len64 = UInt64(b[2]) << 8 | UInt64(b[3]); off = 4
|
||||
} else if len64 == 127 {
|
||||
guard b.count >= 10 else { return }
|
||||
len64 = 0
|
||||
for i in 2..<10 { len64 = len64 << 8 | UInt64(b[i]) }
|
||||
off = 10
|
||||
}
|
||||
guard len64 <= WebSocket.maxMessage else { return close() }
|
||||
let len = Int(len64)
|
||||
let maskOff = off
|
||||
if masked { off += 4 }
|
||||
guard buffer.count >= off + len else { return }
|
||||
let start = buffer.startIndex
|
||||
var payload = Data(buffer[(start + off)..<(start + off + len)])
|
||||
if masked {
|
||||
let mask = [UInt8](buffer[(start + maskOff)..<(start + maskOff + 4)])
|
||||
payload.withUnsafeMutableBytes { p in
|
||||
for i in 0..<len { p[i] ^= mask[i & 3] }
|
||||
}
|
||||
}
|
||||
buffer.removeFirst(off + len)
|
||||
switch opcode {
|
||||
case 0, 1, 2:
|
||||
if opcode != 0 { fragmentOpcode = opcode; fragments = Data() }
|
||||
guard fragments.count + payload.count <= Int(WebSocket.maxMessage) else { return close() }
|
||||
fragments.append(payload)
|
||||
if fin, fragmentOpcode == 1, let s = String(data: fragments, encoding: .utf8) { onText?(s) }
|
||||
case 8: close(); return
|
||||
case 9: send(opcode: 10, payload)
|
||||
default: break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,490 @@
|
||||
// frame-mac-view: streams Mac windows or displays to viewers on the Steam
|
||||
// Frame and plays their pointer and key input back on the Mac.
|
||||
//
|
||||
// frame-mac-view serve --port 47811 --page ui/mac-view.html (token in FRAME_MAC_VIEW_TOKEN)
|
||||
// frame-mac-view windows | displays | permissions (JSON on stdout)
|
||||
// frame-mac-view request-permissions (shows macOS's prompts)
|
||||
//
|
||||
// It listens on 127.0.0.1 only. Frame Control reaches it from the Frame
|
||||
// through an SSH reverse tunnel, and every request must carry the token.
|
||||
//
|
||||
// HTTP. Frame Control's key (?k=, from FRAME_MAC_VIEW_TOKEN) never leaves the
|
||||
// Mac; the Frame gets a single-use ticket per viewer instead.
|
||||
// GET /ping, /view open: tunnel check, viewer page
|
||||
// GET /stream?src=...&t=TICKET|r=KEY WebSocket (&codec=h264|jpeg&max=&fps=&bpp=)
|
||||
// GET /status, /windows, /displays ?k=: permissions, streams, sources
|
||||
// POST /ticket?src=... ?k=: a ticket for one viewer of src
|
||||
// POST /close[?src=...] ?k=: end those streams, close their windows
|
||||
// POST /permissions ?k=: show macOS's permission prompts
|
||||
// src is window:<CGWindowID>, display:<CGDirectDisplayID> or test.
|
||||
import AppKit
|
||||
import ApplicationServices
|
||||
import Foundation
|
||||
import IOKit.pwr_mgt
|
||||
import ScreenCaptureKit
|
||||
import Security
|
||||
|
||||
let version = "1"
|
||||
|
||||
func windowsJSON() -> [[String: Any]] {
|
||||
let me = ProcessInfo.processInfo.processIdentifier
|
||||
let skip: Set<String> = ["Window Server", "Dock", "Control Centre", "Control Center", "Notification Centre",
|
||||
"Notification Center", "Spotlight", "SystemUIServer", "Wallpaper", "WindowManager"]
|
||||
return WindowInfo.all().filter {
|
||||
$0.layer == 0 && $0.pid != me && !skip.contains($0.app) && $0.bounds.width >= 120 && $0.bounds.height >= 80
|
||||
}.map {
|
||||
["id": $0.id, "src": "window:\($0.id)", "pid": $0.pid, "app": $0.app, "title": $0.title,
|
||||
"w": Int($0.bounds.width), "h": Int($0.bounds.height)]
|
||||
}
|
||||
}
|
||||
|
||||
func displaysJSON() -> [[String: Any]] {
|
||||
var ids = [CGDirectDisplayID](repeating: 0, count: 16)
|
||||
var n: UInt32 = 0
|
||||
// Online, not active: a display that's asleep is still one you can stream.
|
||||
CGGetOnlineDisplayList(16, &ids, &n)
|
||||
return ids.prefix(Int(n)).filter { CGDisplayMirrorsDisplay($0) == kCGNullDirectDisplay }.map { id in
|
||||
let b = CGDisplayBounds(id)
|
||||
return ["id": id, "src": "display:\(id)", "name": displayName(id), "w": Int(b.width), "h": Int(b.height),
|
||||
"main": CGDisplayIsMain(id) != 0]
|
||||
}
|
||||
}
|
||||
|
||||
func permissionsJSON() -> [String: Any] {
|
||||
["screen": CGPreflightScreenCaptureAccess(), "accessibility": AXIsProcessTrusted()]
|
||||
}
|
||||
|
||||
func printJSON(_ obj: Any) {
|
||||
let d = (try? JSONSerialization.data(withJSONObject: obj, options: [.sortedKeys, .prettyPrinted])) ?? Data()
|
||||
FileHandle.standardOutput.write(d + Data("\n".utf8))
|
||||
}
|
||||
|
||||
/// One viewer watching one source.
|
||||
final class Session {
|
||||
let id: Int
|
||||
let source: Source
|
||||
let capture: CaptureSource
|
||||
let encoder: Encoder
|
||||
let ws: WebSocket
|
||||
let codec: Codec
|
||||
let reconnectKey: String
|
||||
private let lock = NSLock()
|
||||
private var last: (CVPixelBuffer, CMTime)?
|
||||
private var skipped = false
|
||||
private var stopped = false
|
||||
private var inFlight = 0
|
||||
/// Frames already queued for the network; beyond this, or with two frames
|
||||
/// already in the encoder, new frames are skipped (before encoding, so no
|
||||
/// reference frame goes missing) and the newest picture is sent once
|
||||
/// things catch up. Only that newest picture is kept meanwhile.
|
||||
let maxPending: Int
|
||||
static let maxInFlight = 2
|
||||
var onEnd: ((Session) -> Void)?
|
||||
var onAck: (() -> Void)?
|
||||
var onFinished: ((String) -> Void)?
|
||||
private let encodeQueue = DispatchQueue(label: "frame-mac-view.encode", qos: .userInteractive)
|
||||
|
||||
init(id: Int, source: Source, capture: CaptureSource, ws: WebSocket, codec: Codec, fps: Int, bitsPerPixel: Double,
|
||||
reconnectKey: String) {
|
||||
self.id = id
|
||||
self.source = source
|
||||
self.capture = capture
|
||||
self.ws = ws
|
||||
self.codec = codec
|
||||
self.reconnectKey = reconnectKey
|
||||
encoder = Encoder(codec: codec, fps: fps, bitsPerPixel: bitsPerPixel)
|
||||
maxPending = codec == .jpeg ? 3 << 20 : 1 << 20
|
||||
}
|
||||
|
||||
/// Encodes `pb` unless the link or the encoder is busy, in which case it
|
||||
/// becomes the picture to send next.
|
||||
private func offer(_ pb: CVPixelBuffer, pts: CMTime) {
|
||||
lock.lock()
|
||||
last = (pb, pts)
|
||||
let busy = stopped || ws.pendingBytes > maxPending || inFlight >= Session.maxInFlight
|
||||
if busy { skipped = true } else { inFlight += 1 }
|
||||
lock.unlock()
|
||||
if !busy { submit(pb, pts: pts) }
|
||||
}
|
||||
|
||||
private func submit(_ pb: CVPixelBuffer, pts: CMTime) {
|
||||
encodeQueue.async {
|
||||
if !self.encoder.encode(pb, pts: pts) { self.finished() }
|
||||
}
|
||||
}
|
||||
|
||||
/// An encode finished (or failed): send the newest skipped picture if
|
||||
/// there's room now.
|
||||
private func finished() {
|
||||
lock.lock()
|
||||
inFlight = max(0, inFlight - 1)
|
||||
lock.unlock()
|
||||
resendIfRoom()
|
||||
}
|
||||
|
||||
private func resendIfRoom() {
|
||||
lock.lock()
|
||||
var next: (CVPixelBuffer, CMTime)?
|
||||
if !stopped, skipped, ws.pendingBytes <= maxPending / 2, inFlight < Session.maxInFlight, let l = last {
|
||||
next = l
|
||||
skipped = false
|
||||
inFlight += 1
|
||||
}
|
||||
lock.unlock()
|
||||
if let (pb, _) = next { submit(pb, pts: CMClockGetTime(CMClockGetHostTimeClock())) }
|
||||
}
|
||||
|
||||
func start(maxLong: Int, fps: Int) {
|
||||
encoder.onFrame = { [weak self] data, key, pts in
|
||||
guard let self else { return }
|
||||
var msg = Data([key ? 1 : 0])
|
||||
var us = UInt64(max(0, CMTimeGetSeconds(pts)) * 1_000_000).bigEndian
|
||||
msg.append(Data(bytes: &us, count: 8))
|
||||
msg.append(data)
|
||||
self.ws.sendBinary(msg)
|
||||
}
|
||||
encoder.onDone = { [weak self] in self?.finished() }
|
||||
encoder.onError = { [weak self] message in self?.ws.sendJSON(["t": "error", "message": message]) }
|
||||
capture.onFrame = { [weak self] pb, pts in self?.offer(pb, pts: pts) }
|
||||
capture.onEnded = { [weak self] reason in
|
||||
guard let self else { return }
|
||||
self.ws.sendJSON(["t": "closed", "reason": reason])
|
||||
self.onFinished?(self.source.key)
|
||||
self.end()
|
||||
}
|
||||
ws.onDrain = { [weak self] in self?.resendIfRoom() }
|
||||
ws.onText = { [weak self] text in self?.handle(text) }
|
||||
ws.onClose = { [weak self] in self?.end() }
|
||||
ws.start()
|
||||
// Reconnect with this (kept in the page's memory, never on a command line).
|
||||
ws.sendJSON(["t": "hello", "r": reconnectKey])
|
||||
if let sck = capture as? SCKSource {
|
||||
sck.onChange = { [weak self] in self?.sendInfo() }
|
||||
}
|
||||
capture.start(maxLong: maxLong, fps: fps) { [weak self] error in
|
||||
guard let self else { return }
|
||||
if let error {
|
||||
if (self.capture as? SCKSource)?.gone == true {
|
||||
// Final, like a window closing mid-stream: the viewer closes
|
||||
// and its key is revoked, rather than retrying for ever.
|
||||
self.ws.sendJSON(["t": "closed", "reason": error])
|
||||
self.onFinished?(self.source.key)
|
||||
} else {
|
||||
self.ws.sendJSON(["t": "error", "message": error])
|
||||
}
|
||||
self.end()
|
||||
return
|
||||
}
|
||||
self.sendInfo()
|
||||
}
|
||||
}
|
||||
|
||||
func sendInfo() {
|
||||
ws.sendJSON(["t": "info", "src": source.key, "title": capture.title, "app": capture.app, "codec": codec.rawValue,
|
||||
"input": source == .test || Input.allowed,
|
||||
"aspect": Double(capture.frameRect.width / max(capture.frameRect.height, 1))])
|
||||
}
|
||||
|
||||
var isStopped: Bool { lock.lock(); defer { lock.unlock() }; return stopped }
|
||||
|
||||
func end() {
|
||||
lock.lock()
|
||||
let was = stopped
|
||||
stopped = true
|
||||
last = nil
|
||||
lock.unlock()
|
||||
guard !was else { return }
|
||||
capture.stop()
|
||||
encodeQueue.async { self.encoder.invalidate() }
|
||||
let owner = id
|
||||
DispatchQueue.main.async { Input.releaseAll(owner: owner) }
|
||||
ws.close()
|
||||
onEnd?(self)
|
||||
}
|
||||
|
||||
/// A point in the picture (0...1 each way) -> Mac global coordinates.
|
||||
private func point(_ x: Double, _ y: Double) -> CGPoint {
|
||||
let r = capture.frameRect
|
||||
return CGPoint(x: r.minX + min(max(x, 0), 1) * r.width, y: r.minY + min(max(y, 0), 1) * r.height)
|
||||
}
|
||||
|
||||
private func handle(_ text: String) {
|
||||
guard !isStopped, let d = text.data(using: .utf8),
|
||||
let m = try? JSONSerialization.jsonObject(with: d) as? [String: Any], let t = m["t"] as? String else { return }
|
||||
let x = m["x"] as? Double ?? -1, y = m["y"] as? Double ?? -1
|
||||
if t == "ack" {
|
||||
onAck?()
|
||||
onAck = nil
|
||||
return
|
||||
}
|
||||
if t == "key-frame" {
|
||||
encodeQueue.async { self.encoder.requestKeyFrame() } // before the resend, same queue
|
||||
lock.lock(); skipped = last != nil; lock.unlock()
|
||||
resendIfRoom()
|
||||
return
|
||||
}
|
||||
if source == .test {
|
||||
let desc: String?
|
||||
switch t {
|
||||
case "m": desc = (m["e"] as? String) == "move" ? nil : "mouse \(m["e"] ?? "") button \(m["b"] ?? 0)"
|
||||
case "wheel": desc = "wheel \(m["dx"] ?? 0), \(m["dy"] ?? 0)"
|
||||
case "k": desc = (m["e"] as? String) == "down" ? "key \(m["code"] ?? "") \"\(m["key"] ?? "")\"" : nil
|
||||
case "text": desc = "text \"\(m["s"] ?? "")\""
|
||||
default: desc = nil
|
||||
}
|
||||
capture.pointer(x: x, y: y, text: desc)
|
||||
return
|
||||
}
|
||||
DispatchQueue.main.async { [self] in
|
||||
guard !isStopped else { return } // Stop revokes input at once
|
||||
switch t {
|
||||
case "m":
|
||||
let kind = m["e"] as? String ?? "move", b = m["b"] as? Int ?? 0
|
||||
let p = point(x, y)
|
||||
if kind == "down", case .window(let wid) = source, let pid = capture.pid {
|
||||
Input.focus(window: wid, pid: pid)
|
||||
}
|
||||
Input.mouse(kind, button: b, at: p, owner: id)
|
||||
case "wheel":
|
||||
Input.scroll(dx: m["dx"] as? Double ?? 0, dy: m["dy"] as? Double ?? 0, at: point(x, y), owner: id)
|
||||
case "k":
|
||||
Input.key(code: m["code"] as? String ?? "", key: m["key"] as? String ?? "",
|
||||
down: (m["e"] as? String) == "down", mods: m["mods"] as? [String] ?? [], owner: id)
|
||||
case "text":
|
||||
if let s = m["s"] as? String, s.count <= 4096 { Input.text(s) }
|
||||
case "release":
|
||||
Input.releaseAll(owner: id)
|
||||
case "focus":
|
||||
if case .window(let wid) = source, let pid = capture.pid { Input.focus(window: wid, pid: pid) }
|
||||
default: break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func randomKey() -> String {
|
||||
var bytes = [UInt8](repeating: 0, count: 24)
|
||||
_ = SecRandomCopyBytes(kSecRandomDefault, bytes.count, &bytes)
|
||||
return Data(bytes).base64EncodedString().replacingOccurrences(of: "+", with: "-")
|
||||
.replacingOccurrences(of: "/", with: "_").replacingOccurrences(of: "=", with: "")
|
||||
}
|
||||
|
||||
func sameSecret(_ a: String, _ b: String) -> Bool {
|
||||
guard !a.isEmpty, a.utf8.count == b.utf8.count else { return false }
|
||||
return zip(a.utf8, b.utf8).reduce(0, { $0 | ($1.0 ^ $1.1) }) == 0
|
||||
}
|
||||
|
||||
final class Agent {
|
||||
/// Frame Control's own key. It stays on the Mac: the Frame only ever sees
|
||||
/// single-use tickets and per-stream reconnect keys, both tied to one source.
|
||||
let token: String
|
||||
let page: URL?
|
||||
var sessions: [Int: Session] = [:] { didSet { keepDisplayAwake(!sessions.isEmpty) } }
|
||||
var nextId = 1
|
||||
let lock = NSLock()
|
||||
private var assertion: IOPMAssertionID = 0
|
||||
/// ticket -> (source, expiry, reconnect key once redeemed). A ticket opens
|
||||
/// one viewer within a minute; it may be redeemed again, for the same key,
|
||||
/// only until that viewer confirms it has the key ("ack").
|
||||
private var tickets: [String: (src: String, expiry: Date, key: String?)] = [:]
|
||||
/// reconnect key -> source, until Stop for that source.
|
||||
private var reconnectKeys: [String: String] = [:]
|
||||
/// Sources that ended for good (the window closed), for Frame Control.
|
||||
private var finished = Set<String>()
|
||||
|
||||
/// While anyone watches, keep the Mac's display on: a sleeping display
|
||||
/// stops being drawn, so there'd be nothing to capture (and it would lock).
|
||||
private func keepDisplayAwake(_ on: Bool) {
|
||||
if on, assertion == 0 {
|
||||
IOPMAssertionCreateWithName(kIOPMAssertionTypePreventUserIdleDisplaySleep as CFString,
|
||||
IOPMAssertionLevel(kIOPMAssertionLevelOn),
|
||||
"Frame Control is showing this Mac in a Steam Frame" as CFString, &assertion)
|
||||
} else if !on, assertion != 0 {
|
||||
IOPMAssertionRelease(assertion)
|
||||
assertion = 0
|
||||
}
|
||||
}
|
||||
|
||||
init(token: String, page: URL?) {
|
||||
self.token = token
|
||||
self.page = page
|
||||
}
|
||||
|
||||
/// Whether this request may open a stream of `src`: Frame Control's key,
|
||||
/// an unused ticket for that source, or a live reconnect key for it.
|
||||
private func mayStream(_ req: Request, src: String) -> String? {
|
||||
lock.lock()
|
||||
defer { lock.unlock() }
|
||||
let now = Date()
|
||||
tickets = tickets.filter { $0.value.expiry > now }
|
||||
if sameSecret(req.query["k"] ?? "", token) { return randomKey() }
|
||||
if let t = req.query["t"], let entry = tickets[t], entry.src == src {
|
||||
// A retry before the viewer got its key gets the same key back.
|
||||
let key = entry.key ?? randomKey()
|
||||
tickets[t] = (src, entry.expiry, key)
|
||||
reconnectKeys[key] = src
|
||||
return key
|
||||
}
|
||||
if let r = req.query["r"], reconnectKeys[r] == src { return r }
|
||||
return nil
|
||||
}
|
||||
|
||||
/// The viewer has its reconnect key, so the ticket that led to it can't
|
||||
/// be used again (also when the ack comes over a reconnection).
|
||||
func acknowledged(key: String) {
|
||||
lock.lock(); tickets = tickets.filter { $0.value.key != key }; lock.unlock()
|
||||
}
|
||||
|
||||
func handle(_ req: Request, _ c: HTTPConnection) {
|
||||
// Open to anything that reaches the port: a liveness check for the
|
||||
// tunnel, and the viewer page, which holds no secrets.
|
||||
switch (req.method, req.path) {
|
||||
case ("GET", "/ping"): return c.respond(200, text: "frame-mac-view")
|
||||
case ("GET", "/view"):
|
||||
guard let page, let body = try? Data(contentsOf: page) else { return c.respond(404, text: "no viewer page") }
|
||||
return c.respond(200, body: body, type: "text/html; charset=utf-8")
|
||||
case ("GET", "/stream"):
|
||||
guard let src = Source(req.query["src"] ?? "") else { return c.respond(400, text: "bad src") }
|
||||
guard let key = mayStream(req, src: src.key) else { return c.respond(403, text: "forbidden") }
|
||||
return stream(req, c, src: src, key: key)
|
||||
default: break
|
||||
}
|
||||
guard sameSecret(req.query["k"] ?? req.headers["x-token"] ?? "", token) else {
|
||||
return c.respond(403, text: "forbidden")
|
||||
}
|
||||
switch (req.method, req.path) {
|
||||
case ("GET", "/status"):
|
||||
lock.lock()
|
||||
let list = sessions.values.map { ["id": $0.id, "src": $0.source.key, "title": $0.capture.title, "app": $0.capture.app] }
|
||||
lock.unlock()
|
||||
var s = permissionsJSON()
|
||||
s["version"] = version
|
||||
s["streams"] = list
|
||||
lock.lock(); s["finished"] = Array(finished); lock.unlock()
|
||||
c.respond(json: s)
|
||||
case ("GET", "/windows"):
|
||||
c.respond(json: ["windows": windowsJSON(), "screen": CGPreflightScreenCaptureAccess()])
|
||||
case ("GET", "/displays"):
|
||||
c.respond(json: ["displays": displaysJSON()])
|
||||
case ("POST", "/ticket"):
|
||||
guard let src = Source(req.query["src"] ?? "") else { return c.respond(400, text: "bad src") }
|
||||
let t = randomKey()
|
||||
lock.lock()
|
||||
tickets[t] = (src.key, Date().addingTimeInterval(60), nil)
|
||||
finished.remove(src.key)
|
||||
lock.unlock()
|
||||
c.respond(json: ["ticket": t])
|
||||
case ("POST", "/close"):
|
||||
// Tell viewers to close their windows, but don't rely on them:
|
||||
// the sessions end here and can't reconnect.
|
||||
let src = req.query["src"]
|
||||
lock.lock()
|
||||
let matching = sessions.values.filter { src == nil || $0.source.key == src }
|
||||
reconnectKeys = reconnectKeys.filter { src != nil && $0.value != src }
|
||||
tickets = tickets.filter { src != nil && $0.value.src != src } // not yet used ones too
|
||||
lock.unlock()
|
||||
for s in matching { s.ws.sendJSON(["t": "close"]) }
|
||||
DispatchQueue.global().asyncAfter(deadline: .now() + 0.3) { for s in matching { s.end() } }
|
||||
c.respond(json: ["closed": matching.count])
|
||||
case ("POST", "/permissions"):
|
||||
DispatchQueue.main.async { requestPermissions() }
|
||||
c.respond(json: permissionsJSON())
|
||||
default:
|
||||
c.respond(404, text: "not found")
|
||||
}
|
||||
}
|
||||
|
||||
private func stream(_ req: Request, _ c: HTTPConnection, src: Source, key: String) {
|
||||
let codec = Codec(rawValue: req.query["codec"] ?? "h264") ?? .h264
|
||||
let maxLong = min(max(Int(req.query["max"] ?? "") ?? 1920, 320), 3840)
|
||||
let fps = min(max(Int(req.query["fps"] ?? "") ?? 60, 5), 120)
|
||||
let bpp = min(max(Double(req.query["bpp"] ?? "") ?? 0.1, 0.02), 0.5)
|
||||
guard let ws = c.upgrade(req) else { return }
|
||||
let capture: CaptureSource = src == .test ? TestSource() : SCKSource(src)
|
||||
lock.lock()
|
||||
// One live viewer per key: a reconnection (or a second use of an
|
||||
// unacknowledged ticket) replaces the one before.
|
||||
let replaced = sessions.values.filter { $0.reconnectKey == key }
|
||||
let session = Session(id: nextId, source: src, capture: capture, ws: ws, codec: codec, fps: fps,
|
||||
bitsPerPixel: bpp, reconnectKey: key)
|
||||
nextId += 1
|
||||
sessions[session.id] = session
|
||||
lock.unlock()
|
||||
for old in replaced { old.end() }
|
||||
session.onEnd = { [weak self] s in
|
||||
guard let self else { return }
|
||||
self.lock.lock(); self.sessions[s.id] = nil; self.lock.unlock()
|
||||
}
|
||||
// The source is gone (its window closed): its viewers can't come back.
|
||||
session.onFinished = { [weak self] src in
|
||||
guard let self else { return }
|
||||
self.lock.lock()
|
||||
self.finished.insert(src)
|
||||
self.reconnectKeys = self.reconnectKeys.filter { $0.value != src }
|
||||
self.lock.unlock()
|
||||
}
|
||||
session.onAck = { [weak self] in self?.acknowledged(key: key) }
|
||||
session.start(maxLong: maxLong, fps: fps)
|
||||
}
|
||||
}
|
||||
|
||||
func requestPermissions() {
|
||||
if !CGPreflightScreenCaptureAccess() { CGRequestScreenCaptureAccess() }
|
||||
if !AXIsProcessTrusted() {
|
||||
AXIsProcessTrustedWithOptions([kAXTrustedCheckOptionPrompt.takeUnretainedValue() as String: true] as CFDictionary)
|
||||
}
|
||||
}
|
||||
|
||||
func argument(_ name: String, in args: [String]) -> String? {
|
||||
guard let i = args.firstIndex(of: name), i + 1 < args.count else { return nil }
|
||||
return args[i + 1]
|
||||
}
|
||||
|
||||
let args = Array(CommandLine.arguments.dropFirst())
|
||||
switch args.first {
|
||||
case "windows":
|
||||
printJSON(["windows": windowsJSON(), "screen": CGPreflightScreenCaptureAccess()])
|
||||
case "displays":
|
||||
printJSON(["displays": displaysJSON()])
|
||||
case "permissions":
|
||||
printJSON(permissionsJSON())
|
||||
case "request-permissions":
|
||||
requestPermissions()
|
||||
printJSON(permissionsJSON())
|
||||
case "serve":
|
||||
let port = UInt16(argument("--port", in: args) ?? "") ?? 0
|
||||
let token = ProcessInfo.processInfo.environment["FRAME_MAC_VIEW_TOKEN"] ?? ""
|
||||
guard !token.isEmpty else {
|
||||
FileHandle.standardError.write(Data("frame-mac-view: set FRAME_MAC_VIEW_TOKEN\n".utf8))
|
||||
exit(2)
|
||||
}
|
||||
let page = argument("--page", in: args).map { URL(fileURLWithPath: $0) }
|
||||
let agent = Agent(token: token, page: page)
|
||||
// Quit when the parent goes away (it holds our stdin open).
|
||||
if args.contains("--exit-on-eof") {
|
||||
DispatchQueue.global().async {
|
||||
while FileHandle.standardInput.availableData.count > 0 {}
|
||||
exit(0)
|
||||
}
|
||||
}
|
||||
let server: Server
|
||||
do {
|
||||
server = try Server(port: port) { req, c in agent.handle(req, c) }
|
||||
} catch {
|
||||
FileHandle.standardError.write(Data("frame-mac-view: \(error)\n".utf8))
|
||||
exit(1)
|
||||
}
|
||||
server.start { [server] error in
|
||||
if let error {
|
||||
FileHandle.standardError.write(Data("frame-mac-view: can't listen on 127.0.0.1:\(port): \(error)\n".utf8))
|
||||
exit(1)
|
||||
}
|
||||
print("frame-mac-view listening on 127.0.0.1:\(server.port ?? port)")
|
||||
fflush(stdout)
|
||||
}
|
||||
_ = NSApplication.shared // AppKit for NSScreen names and app activation
|
||||
withExtendedLifetime(server) { RunLoop.main.run() }
|
||||
default:
|
||||
FileHandle.standardError.write(Data("usage: frame-mac-view serve|windows|displays|permissions|request-permissions\n".utf8))
|
||||
exit(2)
|
||||
}
|
||||
Executable
+10
@@ -0,0 +1,10 @@
|
||||
#!/bin/sh
|
||||
# Builds the Mac streaming agent into mac/bin/frame-mac-view (arm64, macOS 14+).
|
||||
# Frame Control runs it for "Mac in the headset"; the Electron app bundles it.
|
||||
set -eu
|
||||
here=$(cd "$(dirname "$0")" && pwd)
|
||||
out=${1:-$here/../bin/frame-mac-view}
|
||||
mkdir -p "$(dirname "$out")"
|
||||
xcrun swiftc -O -swift-version 5 -target arm64-apple-macos14.0 \
|
||||
-o "$out" "$here"/Sources/*.swift
|
||||
echo "built $out"
|
||||
@@ -0,0 +1,274 @@
|
||||
"""Mac in the headset (ui/frame_macview.py and the frame-mac-view agent).
|
||||
|
||||
The Python checks run anywhere. On macOS the agent is built and driven over
|
||||
HTTP and WebSocket with its test pattern, which needs no Screen Recording
|
||||
permission: status, the token, the viewer page, H.264 keyframes, pointer
|
||||
input reaching the source, and closing.
|
||||
|
||||
Run: python3 -m unittest discover -s tests
|
||||
"""
|
||||
import base64
|
||||
import http.client
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import socket
|
||||
import struct
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
sys.path.insert(0, str(ROOT / "ui"))
|
||||
|
||||
import frame_macview # noqa: E402
|
||||
|
||||
|
||||
class Helpers(unittest.TestCase):
|
||||
def test_panel_id_is_stable_and_in_range(self):
|
||||
a = frame_macview.panel_id("window:123")
|
||||
self.assertEqual(a, frame_macview.panel_id("window:123"))
|
||||
self.assertNotEqual(a, frame_macview.panel_id("window:124"))
|
||||
self.assertTrue(2_001_000_000 <= a < 2_002_000_000)
|
||||
|
||||
def test_fit_keeps_aspect_inside_the_panel(self):
|
||||
self.assertEqual(frame_macview.fit(2560, 1440), (1920, 1080))
|
||||
w, h = frame_macview.fit(800, 1600)
|
||||
self.assertEqual(h, 1080)
|
||||
self.assertAlmostEqual(w / h, 0.5, places=2)
|
||||
|
||||
@unittest.skipUnless(shutil.which("bash"), "needs bash")
|
||||
def test_launch_script_parses(self):
|
||||
r = subprocess.run(["bash", "-n"], input=frame_macview.LAUNCH, text=True, capture_output=True)
|
||||
self.assertEqual(r.returncode, 0, r.stderr)
|
||||
|
||||
def test_show_checks_the_source_before_anything_else(self):
|
||||
mv = frame_macview.MacView(["ssh"], lambda *a, **k: self.fail("no ssh"), "frame")
|
||||
with self.assertRaises(frame_macview.MacViewError):
|
||||
mv.show("rm -rf /")
|
||||
|
||||
def test_missing_browser_is_explained(self):
|
||||
calls = []
|
||||
|
||||
def run(remote, stdin=None, timeout=30):
|
||||
calls.append(remote)
|
||||
e = RuntimeError("exit 3")
|
||||
e.stdout = "NO_BROWSER\n"
|
||||
raise e
|
||||
|
||||
mv = frame_macview.MacView(["ssh"], run, "frame")
|
||||
mv.call = lambda path, **kw: {"screen": True, "ticket": "tk"}
|
||||
mv.ensure_tunnel = lambda **kw: None
|
||||
mv.remote_port = 47900
|
||||
with self.assertRaises(frame_macview.MacViewError) as cm:
|
||||
mv.show("window:5")
|
||||
self.assertIn("Chromium", str(cm.exception))
|
||||
self.assertTrue(calls[0].startswith("bash -s -- "))
|
||||
|
||||
def test_screen_permission_is_checked_before_the_headset(self):
|
||||
mv = frame_macview.MacView(["ssh"], lambda *a, **k: self.fail("no ssh"), "frame")
|
||||
mv.call = lambda path, **kw: {"screen": False}
|
||||
with self.assertRaises(frame_macview.MacViewError) as cm:
|
||||
mv.show("display:1")
|
||||
self.assertIn("Screen Recording", str(cm.exception))
|
||||
|
||||
|
||||
class WS:
|
||||
"""A minimal WebSocket client (masked frames out, plain frames in)."""
|
||||
|
||||
def __init__(self, port, path):
|
||||
self.s = socket.create_connection(("127.0.0.1", port), timeout=10)
|
||||
key = base64.b64encode(os.urandom(16)).decode()
|
||||
self.s.sendall(f"GET {path} HTTP/1.1\r\nHost: x\r\nUpgrade: websocket\r\nConnection: Upgrade\r\n"
|
||||
f"Sec-WebSocket-Key: {key}\r\nSec-WebSocket-Version: 13\r\n\r\n".encode())
|
||||
head = b""
|
||||
while b"\r\n\r\n" not in head:
|
||||
head += self.s.recv(1)
|
||||
self.status = int(head.split()[1])
|
||||
self.buf = b""
|
||||
|
||||
def _read(self, n):
|
||||
while len(self.buf) < n:
|
||||
chunk = self.s.recv(65536)
|
||||
if not chunk:
|
||||
raise EOFError
|
||||
self.buf += chunk
|
||||
out, self.buf = self.buf[:n], self.buf[n:]
|
||||
return out
|
||||
|
||||
def recv(self):
|
||||
b0, b1 = self._read(2)
|
||||
n = b1 & 0x7F
|
||||
if n == 126:
|
||||
n = struct.unpack(">H", self._read(2))[0]
|
||||
elif n == 127:
|
||||
n = struct.unpack(">Q", self._read(8))[0]
|
||||
return b0 & 0x0F, self._read(n)
|
||||
|
||||
def send_text(self, text):
|
||||
data, mask = text.encode(), os.urandom(4)
|
||||
head = bytes([0x81, 0x80 | len(data)]) if len(data) < 126 else bytes([0x81, 0xFE]) + struct.pack(">H", len(data))
|
||||
self.s.sendall(head + mask + bytes(c ^ mask[i % 4] for i, c in enumerate(data)))
|
||||
|
||||
def close(self):
|
||||
self.s.close()
|
||||
|
||||
|
||||
@unittest.skipUnless(sys.platform == "darwin" and shutil.which("xcrun"), "the agent is macOS-only")
|
||||
class Agent(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.tmp = tempfile.mkdtemp()
|
||||
cls.bin = Path(cls.tmp) / "frame-mac-view"
|
||||
r = subprocess.run(["/bin/sh", str(ROOT / "mac" / "frame-mac-view" / "build.sh"), str(cls.bin)],
|
||||
capture_output=True, text=True, timeout=600)
|
||||
if r.returncode: # a real failure on a Mac with Xcode: don't hide it as a skip
|
||||
raise AssertionError("agent didn't build:\n" + (r.stderr or r.stdout)[-2000:])
|
||||
cls.token = "t0ken-" + os.urandom(6).hex()
|
||||
cls.proc = subprocess.Popen([str(cls.bin), "serve", "--port", "0", "--page", str(ROOT / "ui" / "mac-view.html"),
|
||||
"--exit-on-eof"], env={**os.environ, "FRAME_MAC_VIEW_TOKEN": cls.token},
|
||||
stdin=subprocess.PIPE, stdout=subprocess.PIPE, text=True)
|
||||
line = cls.proc.stdout.readline()
|
||||
cls.port = int(line.rsplit(":", 1)[1])
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls):
|
||||
cls.proc.stdin.close() # --exit-on-eof
|
||||
cls.proc.stdout.close()
|
||||
try:
|
||||
cls.proc.wait(5)
|
||||
except subprocess.TimeoutExpired:
|
||||
cls.proc.kill()
|
||||
shutil.rmtree(cls.tmp, ignore_errors=True)
|
||||
|
||||
def get(self, path, method="GET"):
|
||||
c = http.client.HTTPConnection("127.0.0.1", self.port, timeout=10)
|
||||
c.request(method, path)
|
||||
r = c.getresponse()
|
||||
return r.status, r.read()
|
||||
|
||||
def test_token_is_required(self):
|
||||
self.assertEqual(self.get("/status")[0], 403)
|
||||
self.assertEqual(self.get("/status?k=wrong")[0], 403)
|
||||
status, body = self.get(f"/status?k={self.token}")
|
||||
self.assertEqual(status, 200)
|
||||
self.assertIn("screen", json.loads(body))
|
||||
|
||||
def test_serves_the_viewer_page(self):
|
||||
status, body = self.get(f"/view?k={self.token}&src=test")
|
||||
self.assertEqual(status, 200)
|
||||
self.assertIn(b"VideoDecoder", body)
|
||||
|
||||
def test_lists_displays(self):
|
||||
status, body = self.get(f"/displays?k={self.token}")
|
||||
self.assertEqual(status, 200)
|
||||
self.assertIsInstance(json.loads(body)["displays"], list)
|
||||
|
||||
def ticket(self, src):
|
||||
status, body = self.get(f"/ticket?k={self.token}&src={src}", method="POST")
|
||||
self.assertEqual(status, 200)
|
||||
return json.loads(body)["ticket"]
|
||||
|
||||
def test_ping_and_page_are_open_but_streams_are_not(self):
|
||||
self.assertEqual(self.get("/ping"), (200, b"frame-mac-view"))
|
||||
self.assertEqual(WS(self.port, "/stream?src=test").status, 403)
|
||||
self.assertEqual(self.get("/ticket?src=test", method="POST")[0], 403)
|
||||
|
||||
def test_tickets_are_single_use_and_tied_to_one_source(self):
|
||||
t = self.ticket("test")
|
||||
self.assertEqual(WS(self.port, f"/stream?src=display:1&t={t}").status, 403) # wrong source
|
||||
ws = WS(self.port, f"/stream?src=test&t={t}")
|
||||
self.assertEqual(ws.status, 101)
|
||||
hello = json.loads(ws.recv()[1])
|
||||
self.assertEqual(hello["t"], "hello")
|
||||
# Until the viewer acknowledges, a retry (the hello got lost) gets the
|
||||
# same key, and replaces the first viewer rather than adding one.
|
||||
retry = WS(self.port, f"/stream?src=test&t={t}")
|
||||
self.assertEqual(retry.status, 101)
|
||||
self.assertEqual(json.loads(retry.recv()[1])["r"], hello["r"])
|
||||
time.sleep(0.3)
|
||||
_, body = self.get(f"/status?k={self.token}")
|
||||
self.assertEqual(len(json.loads(body)["streams"]), 1)
|
||||
ws.close()
|
||||
ws = retry
|
||||
ws.send_text(json.dumps({"t": "ack"}))
|
||||
time.sleep(0.3)
|
||||
self.assertEqual(WS(self.port, f"/stream?src=test&t={t}").status, 403) # spent
|
||||
again = WS(self.port, f"/stream?src=test&r={hello['r']}") # the viewer reconnecting
|
||||
self.assertEqual(again.status, 101)
|
||||
again.close()
|
||||
ws.close()
|
||||
# Stop revokes the reconnect key.
|
||||
self.get(f"/close?k={self.token}&src=test", method="POST")
|
||||
self.assertEqual(WS(self.port, f"/stream?src=test&r={hello['r']}").status, 403)
|
||||
|
||||
def test_stop_revokes_tickets_not_yet_used(self):
|
||||
t = self.ticket("test")
|
||||
self.get(f"/close?k={self.token}&src=test", method="POST")
|
||||
self.assertEqual(WS(self.port, f"/stream?src=test&t={t}").status, 403)
|
||||
|
||||
def test_bad_frame_lengths_close_the_socket_not_the_agent(self):
|
||||
ws = WS(self.port, f"/stream?src=test&t={self.ticket('test')}")
|
||||
self.assertEqual(ws.status, 101)
|
||||
# A masked frame claiming 2^63 bytes.
|
||||
ws.s.sendall(bytes([0x81, 0xFF]) + struct.pack(">Q", 1 << 63) + os.urandom(4))
|
||||
with self.assertRaises((EOFError, OSError)):
|
||||
for _ in range(1000):
|
||||
ws.recv()
|
||||
ws.close()
|
||||
self.assertEqual(self.get(f"/status?k={self.token}")[0], 200)
|
||||
|
||||
def test_stream_input_and_close(self):
|
||||
ws = WS(self.port, f"/stream?k={self.token}&src=test&codec=h264&fps=30&max=640")
|
||||
self.assertEqual(ws.status, 101)
|
||||
info = None
|
||||
key = None
|
||||
for _ in range(200):
|
||||
op, data = ws.recv()
|
||||
if op == 1:
|
||||
msg = json.loads(data)
|
||||
if msg["t"] == "hello":
|
||||
continue
|
||||
if msg["t"] == "error":
|
||||
self.skipTest("no H.264 encoder here: " + msg["message"])
|
||||
if msg["t"] == "info":
|
||||
info = msg
|
||||
elif op == 2 and data[0] == 1:
|
||||
key = data
|
||||
if info and key:
|
||||
break
|
||||
self.assertEqual(info["src"], "test")
|
||||
self.assertTrue(info["input"])
|
||||
# A keyframe: flags, 8-byte timestamp, then Annex B with the SPS first.
|
||||
self.assertEqual(key[9:13], b"\x00\x00\x00\x01")
|
||||
self.assertEqual(key[13] & 0x1F, 7) # NAL type 7, SPS
|
||||
ws.send_text(json.dumps({"t": "m", "e": "down", "b": 0, "x": 0.5, "y": 0.5}))
|
||||
ws.send_text(json.dumps({"t": "key-frame"}))
|
||||
got_key = False
|
||||
for _ in range(200):
|
||||
op, data = ws.recv()
|
||||
if op == 2 and data[0] == 1:
|
||||
got_key = True
|
||||
break
|
||||
self.assertTrue(got_key, "no keyframe after asking for one")
|
||||
_, body = self.get(f"/status?k={self.token}")
|
||||
self.assertEqual([s["src"] for s in json.loads(body)["streams"]], ["test"])
|
||||
status, body = self.get(f"/close?k={self.token}", method="POST")
|
||||
self.assertEqual(json.loads(body)["closed"], 1)
|
||||
for _ in range(400):
|
||||
op, data = ws.recv()
|
||||
if op == 1:
|
||||
break
|
||||
self.assertEqual(json.loads(data)["t"], "close")
|
||||
# Without the viewer doing anything, the agent ends the stream itself.
|
||||
time.sleep(1)
|
||||
_, body = self.get(f"/status?k={self.token}")
|
||||
self.assertEqual(json.loads(body)["streams"], [])
|
||||
ws.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,361 @@
|
||||
"""Mac in the headset: stream Mac windows or displays into the Steam Frame as
|
||||
panels you can place anywhere, with the laser, wheel and keys driving the Mac.
|
||||
|
||||
Runs on the Mac, inside Frame Control's server. The pieces:
|
||||
|
||||
- mac/bin/frame-mac-view (Swift, built from mac/frame-mac-view): captures with
|
||||
ScreenCaptureKit, encodes with VideoToolbox, serves ui/mac-view.html and one
|
||||
WebSocket per stream on 127.0.0.1, and plays input back with CGEvent.
|
||||
- An `ssh -R` tunnel, so the Frame reaches the agent on its own 127.0.0.1.
|
||||
Every request carries a random token, so other programs on the Frame
|
||||
(Android apps included) can't watch or drive the Mac.
|
||||
- A Chromium app window on the Frame per stream, on gamescope's X display
|
||||
with its own STEAM_GAME id, which makes it its own SteamVR panel (see
|
||||
docs/panels.md). Chromium XR (~/chromium-xr) is preferred because it's
|
||||
built with H.264; Flathub Chromium is the fallback.
|
||||
|
||||
Stdlib only. MacView gets a plain ssh argv for the tunnel (its own
|
||||
connection) and the server's `run(remote, stdin=, timeout=)` for commands.
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import secrets
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
import zlib
|
||||
from pathlib import Path
|
||||
from urllib.parse import urlencode
|
||||
|
||||
HERE = Path(__file__).resolve().parent
|
||||
ROOT = HERE.parent
|
||||
PAGE = HERE / "mac-view.html"
|
||||
SOURCES = ROOT / "mac" / "frame-mac-view"
|
||||
AGENT = Path(os.environ.get("FRAME_MAC_VIEW") or ROOT / "mac" / "bin" / "frame-mac-view")
|
||||
REMOTE_PORTS = range(47900, 47920)
|
||||
SUPPORTED = sys.platform == "darwin"
|
||||
|
||||
# Stream settings by name: long side in pixels, frames per second, H.264 bits
|
||||
# per pixel per frame, codec. JPEG is for a Frame browser without H.264.
|
||||
QUALITY = {
|
||||
"sharp": {"max": 2560, "fps": 60, "bpp": 0.14, "codec": "h264"},
|
||||
"balanced": {"max": 1920, "fps": 60, "bpp": 0.1, "codec": "h264"},
|
||||
"light": {"max": 1280, "fps": 30, "bpp": 0.08, "codec": "h264"},
|
||||
"compatible": {"max": 1280, "fps": 20, "bpp": 0.1, "codec": "jpeg"},
|
||||
}
|
||||
# Viewer windows are fitted inside a panel's size. gamescope made a 1280x720
|
||||
# request 1920x1080 anyway (verified 2026-09-28, build 20260925.6191901).
|
||||
PANEL_BOX = (1920, 1080)
|
||||
|
||||
# Opens one viewer on gamescope's X display and gives its window its own panel
|
||||
# id. Args: appid url width height tag. The page puts "[tag]" in its title at
|
||||
# once, which is how its X window is found (Chromium may hand the URL to an
|
||||
# instance that's already running, so there's no process to follow).
|
||||
LAUNCH = r"""set -u
|
||||
appid=$1 url=$2 w=$3 h=$4 tag=$5
|
||||
export DISPLAY=:0 LC_ALL=C.UTF-8
|
||||
unset WAYLAND_DISPLAY
|
||||
if ! xprop -root GAMESCOPE_FOCUSABLE_WINDOWS >/dev/null 2>&1; then
|
||||
echo "The headset isn't showing anything (gamescope's display :0 isn't up). Wake it and try again." >&2
|
||||
exit 2
|
||||
fi
|
||||
common=(--ozone-platform=x11 --force-device-scale-factor=1 --no-first-run --no-default-browser-check
|
||||
--password-store=basic --disable-session-crashed-bubble --noerrdialogs --disable-infobars
|
||||
--disable-features=Translate,MediaRouter --autoplay-policy=no-user-gesture-required
|
||||
"--window-size=$w,$h" "--app=$url")
|
||||
if [ -x "$HOME/chromium-xr/chrome" ]; then
|
||||
cmd=("$HOME/chromium-xr/chrome" "--user-data-dir=$HOME/.local/share/frame-control/mac-view" "${common[@]}")
|
||||
elif flatpak info org.chromium.Chromium >/dev/null 2>&1; then
|
||||
cmd=(flatpak run org.chromium.Chromium
|
||||
"--user-data-dir=$HOME/.var/app/org.chromium.Chromium/data/frame-mac-view" "${common[@]}")
|
||||
else
|
||||
echo "NO_BROWSER"
|
||||
exit 3
|
||||
fi
|
||||
log=/tmp/frame-mac-view.log
|
||||
setsid nohup "${cmd[@]}" >>"$log" 2>&1 </dev/null &
|
||||
for _ in $(seq 1 60); do
|
||||
sleep 0.5
|
||||
# xprop, not xwininfo: in a C locale xwininfo can't print a non-ASCII title
|
||||
# at all, while xprop escapes those bytes and leaves the ASCII tag readable.
|
||||
for win in $(xwininfo -root -children 2>/dev/null | awk '/^ +0x/ {print $1}'); do
|
||||
xprop -id "$win" _NET_WM_NAME WM_NAME 2>/dev/null | grep -qF "[$tag]" || continue
|
||||
if xprop -id "$win" -f STEAM_GAME 32c -set STEAM_GAME "$appid" 2>/dev/null; then
|
||||
echo "panel valve.steam.desktopgame.$appid window $win"
|
||||
exit 0
|
||||
fi
|
||||
done
|
||||
done
|
||||
echo "The viewer started, but its window didn't appear within 30 s. Chromium's log:" >&2
|
||||
tail -n 15 "$log" >&2
|
||||
exit 1
|
||||
"""
|
||||
|
||||
|
||||
class MacViewError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def panel_id(src):
|
||||
"""A stable panel id per source, in the range panel-on-frame.sh uses."""
|
||||
return 2_001_000_000 + zlib.crc32(f"mac:{src}".encode()) % 1_000_000
|
||||
|
||||
|
||||
def fit(w, h, box=PANEL_BOX):
|
||||
s = min(box[0] / max(w, 1), box[1] / max(h, 1))
|
||||
return max(320, round(w * s)), max(200, round(h * s))
|
||||
|
||||
|
||||
class MacView:
|
||||
def __init__(self, tunnel_ssh, run, frame, track=None):
|
||||
self.tunnel_ssh = list(tunnel_ssh)
|
||||
self.run = run
|
||||
self.frame = frame
|
||||
self.track = track or (lambda proc: None) # the server ends these on exit
|
||||
self.lock = threading.Lock()
|
||||
self.token = secrets.token_urlsafe(24)
|
||||
self.agent = None
|
||||
self.port = None
|
||||
self.tunnel = None
|
||||
self.remote_port = None
|
||||
self.supervisor = None
|
||||
self.closing = False
|
||||
self.shown = set() # sources with a viewer out there, connected or retrying
|
||||
|
||||
# ---- the agent on this Mac ----
|
||||
|
||||
def unavailable(self):
|
||||
"""Why this can't work here, or None."""
|
||||
if not SUPPORTED:
|
||||
return "Streaming your computer into the headset needs macOS."
|
||||
if not AGENT.exists() and not (SOURCES.exists() and shutil.which("xcrun")):
|
||||
return "The Mac streaming helper is missing from this copy of Frame Control."
|
||||
return None
|
||||
|
||||
def build(self):
|
||||
if AGENT.exists() and not self._stale():
|
||||
return
|
||||
if not (SOURCES / "build.sh").exists():
|
||||
if AGENT.exists():
|
||||
return
|
||||
raise MacViewError("The Mac streaming helper is missing.")
|
||||
r = subprocess.run(["/bin/sh", str(SOURCES / "build.sh"), str(AGENT)], capture_output=True, text=True,
|
||||
stdin=subprocess.DEVNULL, timeout=600)
|
||||
if r.returncode != 0:
|
||||
raise MacViewError("Couldn't build the Mac streaming helper: " + (r.stderr or r.stdout).strip()[-400:])
|
||||
|
||||
def _stale(self):
|
||||
try:
|
||||
built = AGENT.stat().st_mtime
|
||||
return any(p.stat().st_mtime > built for p in (SOURCES / "Sources").glob("*.swift"))
|
||||
except OSError:
|
||||
return False
|
||||
|
||||
def ensure_agent(self):
|
||||
with self.lock:
|
||||
if self.agent and self.agent.poll() is None:
|
||||
return
|
||||
reason = self.unavailable()
|
||||
if reason:
|
||||
raise MacViewError(reason)
|
||||
self.build()
|
||||
env = {**os.environ, "FRAME_MAC_VIEW_TOKEN": self.token}
|
||||
# The same port as before when restarting, so a running tunnel still
|
||||
# fits; otherwise (or if it's gone) whatever the system gives.
|
||||
for port in dict.fromkeys([self.port or 0, 0]):
|
||||
self.agent = subprocess.Popen([str(AGENT), "serve", "--port", str(port), "--page", str(PAGE),
|
||||
"--exit-on-eof"], env=env, stdin=subprocess.PIPE,
|
||||
stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)
|
||||
line = self.agent.stdout.readline()
|
||||
m = re.search(r"listening on 127\.0\.0\.1:(\d+)", line)
|
||||
if m:
|
||||
break
|
||||
self.agent.kill()
|
||||
else:
|
||||
raise MacViewError(f"The Mac streaming helper didn't start: {line.strip() or 'no output'}")
|
||||
if self.port != int(m.group(1)):
|
||||
self._drop_tunnel()
|
||||
self.port = int(m.group(1))
|
||||
self.track(self.agent)
|
||||
threading.Thread(target=self.agent.stdout.read, daemon=True).start() # drain
|
||||
|
||||
def call(self, path, method="GET", **query):
|
||||
"""A request to the agent; starts it if needed."""
|
||||
self.ensure_agent()
|
||||
url = f"http://127.0.0.1:{self.port}{path}?{urlencode({**query, 'k': self.token})}"
|
||||
req = urllib.request.Request(url, method=method, data=b"" if method == "POST" else None)
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=10) as r:
|
||||
return json.load(r)
|
||||
except (urllib.error.URLError, OSError, ValueError) as e:
|
||||
raise MacViewError(f"The Mac streaming helper didn't answer: {e}")
|
||||
|
||||
# ---- the tunnel from the Frame ----
|
||||
|
||||
def _drop_tunnel(self):
|
||||
if self.tunnel and self.tunnel.poll() is None:
|
||||
self.tunnel.terminate()
|
||||
self.tunnel = None
|
||||
|
||||
def tunnel_up(self):
|
||||
return self.tunnel is not None and self.tunnel.poll() is None
|
||||
|
||||
def ensure_tunnel(self, allow_new_port=False):
|
||||
"""Open the tunnel, on the port viewers already use if there is one.
|
||||
A new port only when nobody is watching, since viewers can't move."""
|
||||
with self.lock:
|
||||
if self.tunnel_up():
|
||||
return
|
||||
last = ""
|
||||
ports = [self.remote_port] if self.remote_port else list(REMOTE_PORTS)
|
||||
if self.remote_port and allow_new_port:
|
||||
ports += [p for p in REMOTE_PORTS if p != self.remote_port]
|
||||
for port in ports:
|
||||
proc = subprocess.Popen([*self.tunnel_ssh, "-o", "ExitOnForwardFailure=yes",
|
||||
"-o", "ServerAliveInterval=5", "-o", "ServerAliveCountMax=3", "-N",
|
||||
"-R", f"127.0.0.1:{port}:127.0.0.1:{self.port}", self.frame],
|
||||
stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.PIPE, text=True)
|
||||
# A taken port makes ssh exit once it's connected; a working
|
||||
# tunnel answers the agent's status from the Frame's side.
|
||||
ok = False
|
||||
for _ in range(15):
|
||||
time.sleep(0.4)
|
||||
if proc.poll() is not None:
|
||||
break
|
||||
if self._probe(port):
|
||||
ok = True
|
||||
break
|
||||
if ok:
|
||||
self.tunnel, self.remote_port = proc, port
|
||||
self.track(proc)
|
||||
self._supervise()
|
||||
return
|
||||
if proc.poll() is None:
|
||||
proc.terminate()
|
||||
proc.wait()
|
||||
last = (proc.stderr.read() or "").strip()
|
||||
if "forward" not in last.lower():
|
||||
break # not a port clash: the Frame is unreachable
|
||||
raise MacViewError(f"Couldn't open a tunnel from {self.frame} to this Mac: {last or 'no answer through it'}")
|
||||
|
||||
def _supervise(self):
|
||||
"""Reopen the tunnel on the same port after the headset sleeps or the
|
||||
network drops, so viewers that are retrying find the Mac again."""
|
||||
if self.supervisor and self.supervisor.is_alive():
|
||||
return
|
||||
|
||||
def loop():
|
||||
while not self.closing:
|
||||
time.sleep(5)
|
||||
if self.closing or self.tunnel_up() or not (self.agent and self.agent.poll() is None):
|
||||
continue
|
||||
try:
|
||||
self.ensure_tunnel()
|
||||
except MacViewError:
|
||||
pass # still unreachable; try again shortly
|
||||
|
||||
self.supervisor = threading.Thread(target=loop, daemon=True)
|
||||
self.supervisor.start()
|
||||
|
||||
def _probe(self, port):
|
||||
"""Whether the Frame reaches the agent through the tunnel on `port`."""
|
||||
# /ping needs no key, so none appears on the Frame's command lines.
|
||||
cmd = f"curl -s -m 2 'http://127.0.0.1:{port}/ping'"
|
||||
try:
|
||||
return self.run(cmd, timeout=8).strip() == "frame-mac-view"
|
||||
except Exception: # noqa: BLE001 - the server's Failure, timeouts: all mean "not yet"
|
||||
return False
|
||||
|
||||
# ---- viewers on the Frame ----
|
||||
|
||||
def show(self, src, quality="balanced", width=None, height=None):
|
||||
if src != "test" and not (src.startswith("window:") or src.startswith("display:")):
|
||||
raise MacViewError("Pick a window or display to show.")
|
||||
q = QUALITY.get(quality) or QUALITY["balanced"]
|
||||
state = self.call("/status")
|
||||
if src != "test" and not state.get("screen"):
|
||||
raise MacViewError("Frame Control needs Screen Recording permission first (Allow… under Mac in the headset).")
|
||||
self.shown -= set(state.get("finished", [])) # their Mac windows closed
|
||||
if src in self.shown or any(st.get("src") == src for st in state.get("streams", [])):
|
||||
# Showing it again replaces the old viewer, connected or not: this
|
||||
# revokes its keys so it can't come back alongside the new one.
|
||||
self.stop(src)
|
||||
# Viewers that lost the tunnel keep retrying its old port, even though
|
||||
# the agent no longer counts them, so only move when none are out there.
|
||||
others = self.shown - {src}
|
||||
self.ensure_tunnel(allow_new_port=not others)
|
||||
appid = panel_id(src)
|
||||
# Unique per launch, so a new window is never confused with an old one.
|
||||
tag = "fc" + secrets.token_hex(4)
|
||||
# A single-use ticket for this source, not Frame Control's key: the URL
|
||||
# is visible in the Frame's process list.
|
||||
ticket = self.call("/ticket", method="POST", src=src)["ticket"]
|
||||
params = {"src": src, "t": ticket, "tag": tag, "codec": q["codec"], "max": q["max"], "fps": q["fps"],
|
||||
"bpp": q["bpp"]}
|
||||
url = f"http://127.0.0.1:{self.remote_port}/view?{urlencode(params)}"
|
||||
w, h = fit(width or 1280, height or 720)
|
||||
args = " ".join(_quote(str(a)) for a in (appid, url, w, h, tag))
|
||||
try:
|
||||
out = self.run("bash -s -- " + args, stdin=LAUNCH, timeout=60)
|
||||
except Exception as e: # noqa: BLE001 - the server's Failure carries the Frame's words
|
||||
if "NO_BROWSER" in (getattr(e, "stdout", "") or ""):
|
||||
raise MacViewError("The Frame needs a browser for this: install Chromium (Tools → Linux apps, "
|
||||
"org.chromium.Chromium) or Chromium XR.")
|
||||
raise MacViewError(str(e))
|
||||
self.shown.add(src)
|
||||
return {"panel": f"valve.steam.desktopgame.{appid}", "src": src, "detail": out.strip()}
|
||||
|
||||
def stop(self, src=None):
|
||||
if src:
|
||||
self.shown.discard(src)
|
||||
else:
|
||||
self.shown.clear()
|
||||
if not (self.agent and self.agent.poll() is None):
|
||||
return {"closed": 0}
|
||||
return self.call("/close", method="POST", **({"src": src} if src else {}))
|
||||
|
||||
def state(self):
|
||||
reason = self.unavailable()
|
||||
if reason:
|
||||
return {"available": False, "reason": reason}
|
||||
status = self.call("/status")
|
||||
windows = self.call("/windows").get("windows", []) if status.get("screen") else []
|
||||
displays = self.call("/displays").get("displays", [])
|
||||
return {"available": True, "screen": status.get("screen", False),
|
||||
"accessibility": status.get("accessibility", False), "streams": status.get("streams", []),
|
||||
"windows": windows, "displays": displays, "tunnel": self.tunnel_up()}
|
||||
|
||||
def restart_agent(self):
|
||||
"""Only if it's missing a permission: a running stream would stop."""
|
||||
with self.lock:
|
||||
if not (self.agent and self.agent.poll() is None):
|
||||
return
|
||||
status = self.call("/status")
|
||||
if status.get("screen") and status.get("accessibility"):
|
||||
return
|
||||
with self.lock:
|
||||
self.agent.terminate()
|
||||
self.agent.wait(5)
|
||||
|
||||
def request_permissions(self):
|
||||
return self.call("/permissions", method="POST")
|
||||
|
||||
def shutdown(self):
|
||||
self.closing = True
|
||||
try:
|
||||
self.stop()
|
||||
except MacViewError:
|
||||
pass
|
||||
for proc in (self.tunnel, self.agent):
|
||||
if proc and proc.poll() is None:
|
||||
proc.terminate()
|
||||
|
||||
|
||||
def _quote(s):
|
||||
return "'" + s.replace("'", "'\\''") + "'"
|
||||
+80
-1
@@ -309,6 +309,11 @@
|
||||
.game-card .bar > i { display: block; height: 100%; background: var(--action); }
|
||||
.f-0 { color: var(--muted); }
|
||||
/* ---- Android display ---- */
|
||||
#mvQuality { background: rgba(0,0,0,.28); color: var(--text); border: 1px solid transparent; border-radius: 3px; padding: 4px 6px; font: inherit; font-size: 12px; }
|
||||
.mv-windows { max-height: 420px; overflow-y: auto; }
|
||||
.mv-perm { background: rgba(255,190,60,.08); border: 1px solid rgba(255,190,60,.25); border-radius: 4px; padding: 10px 12px; margin-bottom: 12px; font-size: 13px; }
|
||||
.mv-perm .row { margin-top: 8px; }
|
||||
.mv-live { color: var(--green, #7fd67f); font-size: 12px; }
|
||||
.disp select { width: 100%; background: rgba(0,0,0,.28); color: var(--text); border: 1px solid transparent; border-radius: 3px;
|
||||
padding: 8px 10px; font: inherit; font-size: 13px; }
|
||||
.disp select:focus { outline: none; border-color: var(--blue); }
|
||||
@@ -642,6 +647,25 @@
|
||||
<div><a href="https://framedropvr.com" target="_blank">FrameDrop</a>: sideloader (Windows only for now)</div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section class="panel" id="macview" hidden>
|
||||
<div class="shelf-head"><h2>Mac in the headset</h2><span class="count" id="mvCount"></span><span class="spacer"></span>
|
||||
<select id="mvQuality" title="Picture quality and bandwidth">
|
||||
<option value="sharp">Sharp</option><option value="balanced" selected>Balanced</option>
|
||||
<option value="light">Light</option><option value="compatible">Compatible (JPEG)</option>
|
||||
</select>
|
||||
<button class="small" id="mvRefresh">Refresh</button></div>
|
||||
<div class="mv-perm" id="mvPerm" hidden></div>
|
||||
<div class="list" id="mvDisplays"></div>
|
||||
<div class="shelf-head" style="margin-top:16px"><h2>Windows</h2><span class="count" id="mvWinCount"></span></div>
|
||||
<div class="list mv-windows" id="mvWindows"><div class="sub">Loading…</div></div>
|
||||
<div class="row" style="margin-top:12px">
|
||||
<button class="small" data-mv="show" data-src="test" data-title="Test pattern">Test pattern</button>
|
||||
<button class="small danger" data-mv="stopall">Stop all</button>
|
||||
</div>
|
||||
<div class="hint">Each window or screen becomes its own panel in the headset. Place it with the SteamVR dashboard
|
||||
(Float in World, Move, Size). Point and click with the laser, scroll with the thumbstick, and type on the Mac's keyboard.</div>
|
||||
</section>
|
||||
</div>
|
||||
</div>
|
||||
</main>
|
||||
@@ -2107,10 +2131,64 @@ $("shotsRefresh").onclick = loadShots;
|
||||
$("shotsSaveNew").onclick = e => saveShots(shots.list.filter(s => !s.saved), e.currentTarget);
|
||||
$("shotsFolder").onclick = e => act($("shotsFolder").textContent, () => api("/api/open", { what: "shots" }), e.currentTarget);
|
||||
|
||||
// ---- Mac in the headset: windows and displays as panels (ui/frame_macview.py) ----
|
||||
let mvSeq = 0;
|
||||
function mvRow(src, title, sub, w, h, streaming) {
|
||||
return `<div class="item"><div class="grow"><div class="t">${esc(title)}</div>
|
||||
<div class="s">${esc(sub)}${streaming ? ' · <span class="mv-live">in the headset</span>' : ""}</div></div>
|
||||
${streaming ? `<button class="small" data-mv="stop" data-src="${esc(src)}">Stop</button>`
|
||||
: `<button class="small action" data-mv="show" data-src="${esc(src)}" data-title="${esc(title)}" data-w="${w}" data-h="${h}">Show</button>`}
|
||||
</div>`;
|
||||
}
|
||||
async function loadMacView(restart) {
|
||||
const seq = ++mvSeq;
|
||||
let s, err;
|
||||
try { s = await api("/api/macview" + (restart ? "?restart=1" : "")); } catch (e) { err = e; }
|
||||
if (seq !== mvSeq) return;
|
||||
const box = $("macview");
|
||||
if (err) { box.hidden = false; return failed($("mvWindows"), err); }
|
||||
if (!s.available) { box.hidden = true; return; }
|
||||
box.hidden = false;
|
||||
const live = new Set((s.streams || []).map(x => x.src));
|
||||
$("mvCount").textContent = live.size ? `${live.size} showing` : "";
|
||||
const perm = [];
|
||||
if (!s.screen) perm.push(`Frame Control needs <b>Screen Recording</b> permission to show your windows.`);
|
||||
if (!s.accessibility) perm.push(`Allow <b>Accessibility</b> too, so clicks and keys from the headset reach the Mac.`);
|
||||
$("mvPerm").hidden = !perm.length;
|
||||
$("mvPerm").innerHTML = perm.join(" ") + `<div class="row"><button class="small action" data-mv="perm">Allow…</button>
|
||||
<span class="sub">Then press Refresh. macOS may ask you to reopen Frame Control.</span></div>`;
|
||||
$("mvDisplays").innerHTML = (s.displays || []).map(d =>
|
||||
mvRow(d.src, `Whole screen: ${d.name}`, `${d.w}×${d.h}${d.main ? " · main display" : ""}`, d.w, d.h, live.has(d.src))).join("");
|
||||
const wins = s.windows || [];
|
||||
$("mvWinCount").textContent = wins.length ? `${wins.length}` : "";
|
||||
$("mvWindows").innerHTML = !s.screen ? `<div class="sub">Windows appear here once Screen Recording is allowed.</div>`
|
||||
: wins.length ? wins.map(w => mvRow(w.src, w.title || w.app, `${w.title ? w.app + " · " : ""}${w.w}×${w.h}`, w.w, w.h, live.has(w.src))).join("")
|
||||
: `<div class="sub">No windows open on this Mac's current desktop.</div>`;
|
||||
}
|
||||
$("mvRefresh").onclick = () => loadMacView(true);
|
||||
$("macview").onclick = async e => {
|
||||
const b = e.target.closest("[data-mv]"); if (!b) return;
|
||||
const src = b.dataset.src;
|
||||
if (b.dataset.mv === "show") {
|
||||
await act(`Show ${b.dataset.title} in the headset`, async () => {
|
||||
const r = await api("/api/macview", { action: "show", src, quality: $("mvQuality").value,
|
||||
w: +b.dataset.w || undefined, h: +b.dataset.h || undefined });
|
||||
return { message: `${b.dataset.title} is a panel in the headset now. Float it with the SteamVR dashboard.`, ...r };
|
||||
}, b);
|
||||
} else if (b.dataset.mv === "stop") {
|
||||
await act("Stop showing it", () => api("/api/macview", { action: "stop", src }), b);
|
||||
} else if (b.dataset.mv === "stopall") {
|
||||
await act("Stop everything in the headset", () => api("/api/macview", { action: "stop" }), b);
|
||||
} else if (b.dataset.mv === "perm") {
|
||||
await act("Ask macOS for permission", () => api("/api/macview", { action: "permissions" }), b);
|
||||
}
|
||||
setTimeout(loadMacView, 800);
|
||||
};
|
||||
|
||||
// ---- pages: #home, #games, #android, #tools (older section links still work) ----
|
||||
const PAGES = ["home", "games", "android", "tools"];
|
||||
const SECTION_PAGE = { view: "home", device: "home", shots: "home", library: "games", sideloaded: "games", getgames: "games",
|
||||
display: "android", transfer: "tools", apps: "tools", power: "tools" };
|
||||
display: "android", transfer: "tools", apps: "tools", power: "tools", macview: "tools" };
|
||||
let page = "home";
|
||||
function showPage() {
|
||||
const id = location.hash.slice(1);
|
||||
@@ -2122,6 +2200,7 @@ function showPage() {
|
||||
document.title = page === "home" ? "Frame Control" : `${page[0].toUpperCase() + page.slice(1)} · Frame Control`;
|
||||
const section = !PAGES.includes(id) && id && $(id);
|
||||
if (section) section.scrollIntoView(); else window.scrollTo(0, 0);
|
||||
if (page === "tools") loadMacView();
|
||||
}
|
||||
window.addEventListener("hashchange", showPage);
|
||||
// While a text field has focus, phones hide the bottom tab bar (see body.typing in the CSS).
|
||||
|
||||
@@ -0,0 +1,242 @@
|
||||
<!doctype html>
|
||||
<!-- Frame Control: one Mac window (or display) inside the Steam Frame.
|
||||
Served by the Mac's frame-mac-view agent through an SSH tunnel and opened
|
||||
on the Frame as its own Chromium app window, which gamescope turns into a
|
||||
SteamVR panel. Video arrives over a WebSocket (H.264 Annex B for
|
||||
WebCodecs, or JPEG); pointer, wheel and keys go back the same way. -->
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>Mac</title>
|
||||
<style>
|
||||
html, body { margin: 0; height: 100%; background: #000; overflow: hidden; cursor: default; }
|
||||
canvas { position: fixed; inset: 0; width: 100%; height: 100%; object-fit: contain; image-rendering: auto; }
|
||||
#note { position: fixed; left: 50%; top: 16px; transform: translateX(-50%); max-width: 80%;
|
||||
font: 15px/1.4 system-ui, sans-serif; color: #eee; background: rgba(20,22,26,.88);
|
||||
padding: 8px 14px; border-radius: 10px; pointer-events: none; transition: opacity .4s; }
|
||||
#note[hidden] { display: block; opacity: 0; }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<canvas id="c" width="16" height="9"></canvas>
|
||||
<div id="note">Connecting to the Mac…</div>
|
||||
<script>
|
||||
"use strict";
|
||||
const q = new URLSearchParams(location.search);
|
||||
// t: a single-use ticket from Frame Control. After the first connection the
|
||||
// Mac sends a reconnect key, kept only in memory.
|
||||
const src = q.get("src") || "test", ticket = q.get("t") || "", tag = q.get("tag") || "";
|
||||
let reconnectKey = "";
|
||||
const c = document.getElementById("c"), ctx = c.getContext("2d", { alpha: false, desynchronized: true });
|
||||
const note = document.getElementById("note");
|
||||
// Counters for Frame Control's tests (read over DevTools).
|
||||
const stats = window.stats = { frames: 0, keyFrames: 0, bytes: 0, dropped: 0, codec: "", errors: [], info: null };
|
||||
// Frame Control finds this window on the Frame by the tag in its title.
|
||||
document.title = tag ? `Mac [${tag}]` : "Mac";
|
||||
let failedStarts = 0;
|
||||
let ws = null, dec = null, codecString = "", needKey = true, closing = false, retry = 0, noteTimer = 0, lastError = "";
|
||||
|
||||
function show(text, ms) {
|
||||
note.textContent = text;
|
||||
note.hidden = false;
|
||||
clearTimeout(noteTimer);
|
||||
if (ms) noteTimer = setTimeout(() => { note.hidden = true; }, ms);
|
||||
}
|
||||
|
||||
function send(obj) { if (ws && ws.readyState === 1) ws.send(JSON.stringify(obj)); }
|
||||
let lastKeyAsk = 0;
|
||||
function askKeyFrame() {
|
||||
const now = performance.now();
|
||||
if (now - lastKeyAsk < 500) return;
|
||||
lastKeyAsk = now;
|
||||
send({ t: "key-frame" });
|
||||
}
|
||||
|
||||
let hideNoteOnFrame = true; // "Connecting…"/"Reconnecting…" go once video flows again
|
||||
function drawFrame(img, w, h) {
|
||||
if (c.width !== w || c.height !== h) { c.width = w; c.height = h; }
|
||||
ctx.drawImage(img, 0, 0);
|
||||
stats.frames++;
|
||||
if (hideNoteOnFrame) { hideNoteOnFrame = false; note.hidden = true; }
|
||||
}
|
||||
|
||||
// ---- H.264 ----
|
||||
function startCode(b, i) { return b[i] === 0 && b[i + 1] === 0 && (b[i + 2] === 1 || (b[i + 2] === 0 && b[i + 3] === 1)); }
|
||||
function spsCodec(au) {
|
||||
for (let i = 0; i + 7 < au.length; i++) {
|
||||
if (!startCode(au, i)) continue;
|
||||
const n = au[i + 2] === 1 ? i + 3 : i + 4;
|
||||
if ((au[n] & 0x1f) === 7) return "avc1." + [au[n + 1], au[n + 2], au[n + 3]].map(b => b.toString(16).padStart(2, "0")).join("");
|
||||
i = n;
|
||||
}
|
||||
return "";
|
||||
}
|
||||
function makeDecoder() {
|
||||
if (dec) try { dec.close(); } catch (e) {}
|
||||
codecString = "";
|
||||
dec = new VideoDecoder({
|
||||
output: frame => { drawFrame(frame, frame.displayWidth, frame.displayHeight); frame.close(); },
|
||||
error: e => {
|
||||
stats.errors.push(String(e.message || e));
|
||||
needKey = true;
|
||||
makeDecoder();
|
||||
askKeyFrame();
|
||||
},
|
||||
});
|
||||
}
|
||||
function onH264(isKey, ts, au) {
|
||||
if (isKey) {
|
||||
const cs = spsCodec(au);
|
||||
if (cs && (cs !== codecString || dec.state !== "configured")) {
|
||||
if (dec.state === "closed") makeDecoder();
|
||||
dec.configure({ codec: cs, optimizeForLatency: true, hardwareAcceleration: "no-preference" });
|
||||
codecString = stats.codec = cs;
|
||||
}
|
||||
stats.keyFrames++;
|
||||
}
|
||||
if (dec.state !== "configured" || (needKey && !isKey)) { stats.dropped++; askKeyFrame(); return; }
|
||||
// Behind: skip to the next keyframe rather than show old pictures late.
|
||||
if (!isKey && dec.decodeQueueSize > 2) { needKey = true; stats.dropped++; askKeyFrame(); return; }
|
||||
needKey = false;
|
||||
dec.decode(new EncodedVideoChunk({ type: isKey ? "key" : "delta", timestamp: ts, data: au }));
|
||||
}
|
||||
|
||||
// ---- JPEG ----
|
||||
let jpegBusy = false;
|
||||
function onJPEG(data) {
|
||||
if (jpegBusy) { stats.dropped++; return; }
|
||||
jpegBusy = true;
|
||||
createImageBitmap(new Blob([data], { type: "image/jpeg" })).then(bmp => {
|
||||
drawFrame(bmp, bmp.width, bmp.height);
|
||||
bmp.close();
|
||||
}).catch(e => stats.errors.push(String(e))).finally(() => { jpegBusy = false; });
|
||||
}
|
||||
|
||||
async function pickCodec() {
|
||||
const want = q.get("codec");
|
||||
if (want === "jpeg") return "jpeg";
|
||||
if (!("VideoDecoder" in window)) return "jpeg";
|
||||
try {
|
||||
const r = await VideoDecoder.isConfigSupported({ codec: "avc1.640028", optimizeForLatency: true });
|
||||
return r.supported ? "h264" : "jpeg";
|
||||
} catch (e) { return "jpeg"; }
|
||||
}
|
||||
|
||||
async function connect() {
|
||||
const codec = await pickCodec();
|
||||
stats.codec = codec;
|
||||
if (codec === "h264") makeDecoder();
|
||||
needKey = true;
|
||||
const p = new URLSearchParams({ src, codec, max: q.get("max") || "1920", fps: q.get("fps") || "60" });
|
||||
if (reconnectKey) p.set("r", reconnectKey); else p.set("t", ticket);
|
||||
if (q.get("bpp")) p.set("bpp", q.get("bpp"));
|
||||
ws = new WebSocket(`ws://${location.host}/stream?${p}`);
|
||||
ws.binaryType = "arraybuffer";
|
||||
ws.onopen = () => { retry = 0; lastError = ""; };
|
||||
ws.onmessage = ev => {
|
||||
if (typeof ev.data === "string") return control(JSON.parse(ev.data));
|
||||
const b = new Uint8Array(ev.data);
|
||||
stats.bytes += b.length;
|
||||
if (b.length < 10) return;
|
||||
const isKey = b[0] === 1;
|
||||
const ts = Number(new DataView(ev.data).getBigUint64(1));
|
||||
const payload = b.subarray(9);
|
||||
if (codec === "h264") onH264(isKey, ts, payload); else onJPEG(payload);
|
||||
};
|
||||
ws.onclose = () => {
|
||||
ws = null;
|
||||
if (closing) return;
|
||||
// Refused before ever getting a key: the ticket expired or was revoked,
|
||||
// and retrying can't help.
|
||||
if (!reconnectKey && ++failedStarts >= 3) {
|
||||
show("This view has expired. Press Show in Frame Control on the Mac to open it again.");
|
||||
return;
|
||||
}
|
||||
hideNoteOnFrame = true;
|
||||
// The Mac may be asleep or the tunnel restarting: keep trying.
|
||||
retry = Math.min(retry + 1, 6);
|
||||
// Keep the Mac's reason (a missing permission, a closed window) on screen.
|
||||
show(lastError ? `${lastError} Retrying…` : "Lost the Mac. Reconnecting…");
|
||||
setTimeout(connect, 500 * 2 ** retry);
|
||||
};
|
||||
}
|
||||
|
||||
function control(m) {
|
||||
if (m.t === "hello") {
|
||||
reconnectKey = m.r;
|
||||
send({ t: "ack" }); // the ticket is spent only now
|
||||
} else if (m.t === "info") {
|
||||
stats.info = m;
|
||||
const name = m.title && m.app && m.title !== m.app ? `${m.title} — ${m.app}` : (m.title || m.app || "Mac");
|
||||
document.title = tag ? `${name} [${tag}]` : name;
|
||||
if (!m.input) {
|
||||
hideNoteOnFrame = false; // a warning, not a connection notice: let it stay its 8 s
|
||||
show("Clicks and keys need Accessibility permission on the Mac (Frame Control asks for it).", 8000);
|
||||
}
|
||||
} else if (m.t === "error") {
|
||||
stats.errors.push(m.message);
|
||||
lastError = m.message.replace(/\.?$/, ".");
|
||||
show(m.message);
|
||||
} else if (m.t === "close" || m.t === "closed") {
|
||||
closing = true;
|
||||
if (ws) ws.close();
|
||||
show(m.reason ? `Stopped: ${m.reason}.` : "Stopped.");
|
||||
window.close();
|
||||
}
|
||||
}
|
||||
|
||||
// ---- input ----
|
||||
// Where the picture sits inside the window (object-fit: contain letterboxes it).
|
||||
function toPicture(e) {
|
||||
const r = c.getBoundingClientRect(), s = Math.min(r.width / c.width, r.height / c.height);
|
||||
const w = c.width * s, h = c.height * s, left = r.left + (r.width - w) / 2, top = r.top + (r.height - h) / 2;
|
||||
const x = (e.clientX - left) / w, y = (e.clientY - top) / h;
|
||||
return { x, y, inside: x >= 0 && x <= 1 && y >= 0 && y <= 1 };
|
||||
}
|
||||
let pendingMove = null, moveQueued = false;
|
||||
addEventListener("pointermove", e => {
|
||||
const p = toPicture(e);
|
||||
if (!p.inside && !e.buttons) return;
|
||||
pendingMove = { t: "m", e: "move", x: p.x, y: p.y };
|
||||
if (!moveQueued) {
|
||||
moveQueued = true;
|
||||
requestAnimationFrame(() => { moveQueued = false; if (pendingMove) send(pendingMove); pendingMove = null; });
|
||||
}
|
||||
});
|
||||
addEventListener("pointerdown", e => {
|
||||
const p = toPicture(e);
|
||||
if (!p.inside) return;
|
||||
if (e.pointerId !== undefined) try { c.setPointerCapture(e.pointerId); } catch (err) {}
|
||||
pendingMove = null;
|
||||
send({ t: "m", e: "down", b: e.button < 0 ? 0 : e.button, x: p.x, y: p.y });
|
||||
e.preventDefault();
|
||||
});
|
||||
addEventListener("pointerup", e => {
|
||||
const p = toPicture(e);
|
||||
send({ t: "m", e: "up", b: e.button < 0 ? 0 : e.button, x: p.x, y: p.y });
|
||||
});
|
||||
// Let go of any held button where the pointer is on the Mac, not at a corner.
|
||||
addEventListener("pointercancel", () => send({ t: "release" }));
|
||||
addEventListener("contextmenu", e => e.preventDefault());
|
||||
addEventListener("wheel", e => {
|
||||
const p = toPicture(e), unit = e.deltaMode === 1 ? 16 : e.deltaMode === 2 ? innerHeight : 1;
|
||||
send({ t: "wheel", dx: e.deltaX * unit, dy: e.deltaY * unit, x: p.x, y: p.y });
|
||||
e.preventDefault();
|
||||
}, { passive: false });
|
||||
function mods(e) {
|
||||
return ["shift", "ctrl", "alt", "meta"].filter(m => e[m + "Key"]);
|
||||
}
|
||||
function onKey(e, down) {
|
||||
send({ t: "k", e: down ? "down" : "up", code: e.code, key: e.key, mods: mods(e) });
|
||||
e.preventDefault();
|
||||
}
|
||||
addEventListener("keydown", e => onKey(e, true));
|
||||
addEventListener("keyup", e => onKey(e, false));
|
||||
addEventListener("blur", () => send({ t: "release" }));
|
||||
|
||||
show("Connecting to the Mac…");
|
||||
connect();
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
+45
-1
@@ -39,6 +39,7 @@ sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
import frame_android # noqa: E402
|
||||
import frame_catalog # noqa: E402
|
||||
import frame_host # noqa: E402
|
||||
import frame_macview # noqa: E402
|
||||
import frame_store # noqa: E402
|
||||
import frame_titles # noqa: E402
|
||||
import frame_webinstall # noqa: E402
|
||||
@@ -1213,10 +1214,50 @@ def _sweep_one(prefix, d):
|
||||
pass
|
||||
|
||||
|
||||
# ---- Mac in the headset (frame_macview.py) ----------------------------------
|
||||
|
||||
# The tunnel gets its own connection: the shared master's options would win
|
||||
# over anything added after them.
|
||||
macview = frame_macview.MacView(["ssh", "-o", "BatchMode=yes", "-o", "ConnectTimeout=8"],
|
||||
lambda remote, stdin=None, timeout=30: ssh(remote, stdin=stdin, timeout=timeout),
|
||||
FRAME, track=_live_tunnels.add)
|
||||
|
||||
|
||||
def macview_state(query=None):
|
||||
if LOCAL:
|
||||
return {"available": False, "reason": "Runs on your computer, not the headset."}
|
||||
try:
|
||||
# Refresh restarts the helper, which picks up a permission just granted.
|
||||
if (query or {}).get("restart") == ["1"]:
|
||||
macview.restart_agent()
|
||||
return macview.state()
|
||||
except frame_macview.MacViewError as e:
|
||||
raise Failure(str(e), 500)
|
||||
|
||||
|
||||
def macview_action(body):
|
||||
"""{action: show|stop|permissions, src, quality, w, h}."""
|
||||
if LOCAL:
|
||||
raise Failure("Runs on your computer, not the headset.", 400)
|
||||
action = body.get("action")
|
||||
try:
|
||||
if action == "show":
|
||||
w, h = body.get("w"), body.get("h")
|
||||
return macview.show(str(body.get("src") or ""), str(body.get("quality") or "balanced"),
|
||||
int(w) if w else None, int(h) if h else None)
|
||||
if action == "stop":
|
||||
return macview.stop(body.get("src") or None)
|
||||
if action == "permissions":
|
||||
return macview.request_permissions()
|
||||
except frame_macview.MacViewError as e:
|
||||
raise Failure(str(e), 502)
|
||||
raise Failure("unknown action", 400)
|
||||
|
||||
|
||||
POST = {"/api/android/display": android_display, "/api/android": android, "/api/titles": titles, "/api/launch": launch, "/api/steam": steam, "/api/volume": set_volume, "/api/clipboard": clipboard,
|
||||
"/api/flatpak": flatpak, "/api/open": open_thing, "/api/shots/save": save_shots,
|
||||
"/api/webinstall/check": webinstall_check, "/api/webinstall/start": webinstall_start,
|
||||
"/api/webinstall/cancel": webinstall_cancel}
|
||||
"/api/webinstall/cancel": webinstall_cancel, "/api/macview": macview_action}
|
||||
|
||||
|
||||
# ---- HTTP ------------------------------------------------------------------
|
||||
@@ -1336,6 +1377,8 @@ class Handler(BaseHTTPRequestHandler):
|
||||
"shared": frame_catalog.compat_db.shared()})
|
||||
elif path == "/api/android/catalog":
|
||||
self.send_json({"apps": frame_catalog.catalog()})
|
||||
elif path == "/api/macview":
|
||||
self.send_json(macview_state(parse_qs(url.query)))
|
||||
elif path == "/api/status":
|
||||
self.send_json(status({}))
|
||||
elif path == "/api/steam/owned":
|
||||
@@ -1529,6 +1572,7 @@ def main():
|
||||
if not frame_host.WINDOWS:
|
||||
signal.signal(signal.SIGTERM, signal.SIG_IGN)
|
||||
webinstall_shutdown()
|
||||
macview.shutdown() # close the headset's viewers before the agent goes
|
||||
# The master was started with -N, so it stays up until told to exit.
|
||||
if CONTROL:
|
||||
subprocess.run([*MUX, "-O", "exit", FRAME], capture_output=True, stdin=subprocess.DEVNULL)
|
||||
|
||||
Reference in new issue
Block a user