mirror of
https://github.com/saphid/frame-control.git
synced 2026-10-06 01:00:18 +02:00
Compare commits
14
Commits
v0.4.0
...
pc-in-headset
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
3cb3defaa7 | ||
|
|
a992f6d896 | ||
|
|
d9d538690e | ||
|
|
391ecbeac5 | ||
|
|
1dc2a99617 | ||
|
|
d6f3d7c380 | ||
|
|
838eaa4e6b | ||
|
|
6104c3f192 | ||
|
|
da5c954ce4 | ||
|
|
2466c5aa14 | ||
|
|
060cf9cd57 | ||
|
|
cd202433bb | ||
|
|
c5fc552c60 | ||
|
|
559caa2dd0 |
No files matched your search
@@ -0,0 +1,53 @@
|
||||
name: PC host libraries
|
||||
on:
|
||||
pull_request:
|
||||
paths: ["desktop/**", "ui/**", "tests/**", "mac/frame-mac-view/**", "scripts/macview-bench.py", ".github/workflows/pc-host.yml"]
|
||||
workflow_dispatch:
|
||||
jobs:
|
||||
linux:
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
os: [ubuntu-latest, ubuntu-24.04-arm]
|
||||
runs-on: ${{ matrix.os }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: Native libraries
|
||||
run: |
|
||||
sudo apt-get update -qq
|
||||
sudo apt-get install -y libgstreamer1.0-dev libgstreamer-plugins-base1.0-dev libglib2.0-dev gstreamer1.0-plugins-base gstreamer1.0-plugins-good gstreamer1.0-plugins-bad gstreamer1.0-plugins-ugly gstreamer1.0-pipewire libpipewire-0.3-modules libspa-0.2-modules
|
||||
python3 desktop/build.py
|
||||
- name: Tests (real x264, fake capture consent/input)
|
||||
env:
|
||||
FRAME_PC_REQUIRE_NATIVE: "1"
|
||||
run: python3 -m unittest discover -s tests -p 'test_pc*.py' -v
|
||||
- uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: pc-host-${{ matrix.os }}
|
||||
path: desktop/bundle
|
||||
windows:
|
||||
runs-on: windows-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: msys2/setup-msys2@v2
|
||||
with:
|
||||
msystem: UCRT64
|
||||
update: true
|
||||
install: >-
|
||||
mingw-w64-ucrt-x86_64-gcc
|
||||
mingw-w64-ucrt-x86_64-pkgconf
|
||||
mingw-w64-ucrt-x86_64-python
|
||||
mingw-w64-ucrt-x86_64-gstreamer
|
||||
mingw-w64-ucrt-x86_64-gst-plugins-base
|
||||
mingw-w64-ucrt-x86_64-gst-plugins-good
|
||||
mingw-w64-ucrt-x86_64-gst-plugins-bad
|
||||
mingw-w64-ucrt-x86_64-gst-plugins-ugly
|
||||
- name: Native build and tests (x264; WGC and MF need a desktop/GPU)
|
||||
shell: msys2 {0}
|
||||
run: |
|
||||
python desktop/build.py
|
||||
FRAME_PC_REQUIRE_NATIVE=1 python -m unittest discover -s tests -p 'test_pc*.py' -v
|
||||
- uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: pc-host-windows
|
||||
path: desktop/bundle
|
||||
@@ -7,7 +7,7 @@ on:
|
||||
push:
|
||||
tags: ["v*"]
|
||||
pull_request:
|
||||
paths: ["app/**", "ui/**", "scripts/**", "frame/**", "apk-catalog/**", ".github/workflows/release.yml"]
|
||||
paths: ["app/**", "ui/**", "desktop/**", "mac/**", "scripts/**", "frame/**", "apk-catalog/**", ".github/workflows/release.yml"]
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
@@ -28,18 +28,62 @@ jobs:
|
||||
- os: ubuntu-latest
|
||||
script: dist:linux
|
||||
files: app/dist/*.AppImage app/dist/*.deb
|
||||
- os: ubuntu-24.04-arm
|
||||
script: dist:linux:arm64
|
||||
files: app/dist/*.AppImage app/dist/*.deb
|
||||
runs-on: ${{ matrix.os }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: "24"
|
||||
- name: Linux streaming libraries
|
||||
if: runner.os == 'Linux'
|
||||
shell: bash
|
||||
run: |
|
||||
sudo apt-get update -qq
|
||||
sudo apt-get install -y libgstreamer1.0-dev libgstreamer-plugins-base1.0-dev libglib2.0-dev gstreamer1.0-plugins-base gstreamer1.0-plugins-good gstreamer1.0-plugins-bad gstreamer1.0-plugins-ugly gstreamer1.0-pipewire libpipewire-0.3-modules libspa-0.2-modules
|
||||
arch=x64; if [ "$(uname -m)" = aarch64 ]; then arch=arm64; fi
|
||||
python3 desktop/build.py --out "app/build/desktop/linux-$arch"
|
||||
- uses: msys2/setup-msys2@v2
|
||||
if: runner.os == 'Windows'
|
||||
with:
|
||||
msystem: UCRT64
|
||||
update: true
|
||||
install: >-
|
||||
mingw-w64-ucrt-x86_64-gcc
|
||||
mingw-w64-ucrt-x86_64-pkgconf
|
||||
mingw-w64-ucrt-x86_64-python
|
||||
mingw-w64-ucrt-x86_64-gstreamer
|
||||
mingw-w64-ucrt-x86_64-gst-plugins-base
|
||||
mingw-w64-ucrt-x86_64-gst-plugins-good
|
||||
mingw-w64-ucrt-x86_64-gst-plugins-bad
|
||||
mingw-w64-ucrt-x86_64-gst-plugins-ugly
|
||||
- name: Windows streaming libraries
|
||||
if: runner.os == 'Windows'
|
||||
shell: msys2 {0}
|
||||
run: python desktop/build.py --out app/build/desktop/win-x64
|
||||
- name: Build
|
||||
working-directory: app
|
||||
shell: bash
|
||||
run: npm ci && npm run ${{ matrix.script }}
|
||||
env:
|
||||
CSC_IDENTITY_AUTO_DISCOVERY: "false"
|
||||
- name: Test the bundled Python with the native PC libraries
|
||||
if: runner.os != 'macOS'
|
||||
shell: bash
|
||||
run: |
|
||||
if [ "$RUNNER_OS" = Windows ]; then
|
||||
platform=win; arch=x64
|
||||
python="$PWD/app/build/deps/win-x64/python/python.exe"
|
||||
native="$(cygpath -w "$PWD/app/build/desktop/win-x64")"
|
||||
else
|
||||
platform=linux; arch=x64
|
||||
if [ "$(uname -m)" = aarch64 ]; then arch=arm64; fi
|
||||
python="$PWD/app/build/deps/linux-$arch/python/bin/python3"
|
||||
native="$PWD/app/build/desktop/linux-$arch"
|
||||
fi
|
||||
FRAME_PC_NATIVE="$native" FRAME_PC_REQUIRE_NATIVE=1 "$python" -m unittest discover -s tests -p 'test_pc*.py' -v
|
||||
- name: Upload to the release
|
||||
if: startsWith(github.ref, 'refs/tags/')
|
||||
shell: bash
|
||||
|
||||
+10
@@ -7,5 +7,15 @@ compat-db/.lakebed/
|
||||
tests/smoke/results/
|
||||
mac/bin/
|
||||
|
||||
# Shared desktop streaming controller build products
|
||||
desktop/controller.dll
|
||||
desktop/controller.dylib
|
||||
desktop/controller.so
|
||||
desktop/*.obj
|
||||
desktop/*.lib
|
||||
desktop/*.exp
|
||||
desktop/bundle/
|
||||
app/build/desktop/
|
||||
|
||||
# Downloaded at build time (frame/kdeconnect/fetch.py, app/build/fetch-deps.js)
|
||||
frame/kdeconnect/packages/
|
||||
@@ -210,6 +210,7 @@ Frame's software fits together, all checked against a real headset and labelled
|
||||
| [Sideloading Linux and Windows games](docs/sideloading.md) | A .zip, folder or .exe as a Steam Devkit Game, runtime detection |
|
||||
| [Install links for websites](docs/web-install.md) | `frame-control://install` links and manifests, the rules, a button to paste |
|
||||
| [Steam games](docs/steam-games.md) · [VR video](docs/vr-video.md) · [WebXR in Chromium](docs/webxr-chromium.md) | Installing and buying, watching VR180/360, the Chromium build |
|
||||
| [PC in the headset](docs/pc-in-headset.md) | Windows and Linux host implementation and test coverage |
|
||||
| [Mac in the headset](docs/mac-in-headset.md) | Mac windows and screens as panels in the Frame, with laser and keyboard input |
|
||||
| [VR mods and custom songs](docs/mods.md) | Per-game feasibility, real-Frame results and blockers; no installer yet |
|
||||
| [SSH](docs/ssh.md) · [Streaming](docs/streaming.md) · [Files](docs/file-transfer.md) · [Panels](docs/panels.md) · [Tailscale](docs/tailscale.md) | Topic notes |
|
||||
|
||||
+23
-4
@@ -11,8 +11,9 @@
|
||||
"icon": "env -u ELECTRON_RUN_AS_NODE electron build/make-icon.js",
|
||||
"dist": "sh ../mac/frame-mac-view/build.sh && node build/fetch-deps.js mac arm64 && electron-builder --mac --arm64 --publish never",
|
||||
"dist:dir": "sh ../mac/frame-mac-view/build.sh && node build/fetch-deps.js mac arm64 && electron-builder --mac --arm64 --dir",
|
||||
"dist:linux": "node build/fetch-deps.js linux x64 arm64 && electron-builder --linux --x64 --arm64 --publish never",
|
||||
"dist:win": "node build/fetch-deps.js win x64 && electron-builder --win --x64 --publish never"
|
||||
"dist:linux": "node build/fetch-deps.js linux x64 && electron-builder --linux --x64 --publish never",
|
||||
"dist:win": "node build/fetch-deps.js win x64 && electron-builder --win --x64 --publish never",
|
||||
"dist:linux:arm64": "node build/fetch-deps.js linux arm64 && electron-builder --linux --arm64 --publish never"
|
||||
},
|
||||
"devDependencies": {
|
||||
"electron": "^44.4.5",
|
||||
@@ -171,7 +172,16 @@
|
||||
"entry": {
|
||||
"StartupWMClass": "frame-control"
|
||||
}
|
||||
}
|
||||
},
|
||||
"extraResources": [
|
||||
{
|
||||
"from": "build/desktop/${os}-${arch}",
|
||||
"to": "desktop/bundle",
|
||||
"filter": [
|
||||
"**/*"
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"deb": {
|
||||
"depends": [
|
||||
@@ -184,7 +194,16 @@
|
||||
"zip"
|
||||
],
|
||||
"icon": "build/icon.png",
|
||||
"artifactName": "Frame-Control-win-${arch}.${ext}"
|
||||
"artifactName": "Frame-Control-win-${arch}.${ext}",
|
||||
"extraResources": [
|
||||
{
|
||||
"from": "build/desktop/${os}-${arch}",
|
||||
"to": "desktop/bundle",
|
||||
"filter": [
|
||||
"**/*"
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"nsis": {
|
||||
"oneClick": false,
|
||||
|
||||
@@ -0,0 +1,311 @@
|
||||
{
|
||||
"label": "pc-agent-frame-arm64-hairpin",
|
||||
"date": "2026-09-28T22:36:19",
|
||||
"commit": "c5fc552+dirty",
|
||||
"config": {
|
||||
"quality": "balanced",
|
||||
"mode": "separate",
|
||||
"net": "none",
|
||||
"delay_ms": 0,
|
||||
"buffer_ms": 250,
|
||||
"host": "frame",
|
||||
"usb": false,
|
||||
"ssh_opts": [],
|
||||
"encoder": "",
|
||||
"duration_s": 10.0,
|
||||
"browser_flags": []
|
||||
},
|
||||
"frame_build": "20260925.6191901",
|
||||
"mac": "26.5.2",
|
||||
"host_platform": "Linux aarch64, SteamOS Frame, BUILD_ID 20260925.6191901 (synthetic host only)",
|
||||
"source": "",
|
||||
"pc_host": true,
|
||||
"headset": "0:03.056203 [Info] - 1 - leaving standby",
|
||||
"scenarios": [
|
||||
{
|
||||
"scenario": "test",
|
||||
"frames_sent": 351,
|
||||
"frames_drawn": 351,
|
||||
"frames_shown": 324,
|
||||
"duration_s": 10.1,
|
||||
"stages_ms": {
|
||||
"capture": {
|
||||
"p50": 0.1,
|
||||
"p95": 0.2,
|
||||
"p99": 0.5,
|
||||
"n": 351
|
||||
},
|
||||
"queue": {
|
||||
"p50": 0.1,
|
||||
"p95": 0.3,
|
||||
"p99": 0.4,
|
||||
"n": 351
|
||||
},
|
||||
"encode": {
|
||||
"p50": 2.0,
|
||||
"p95": 2.9,
|
||||
"p99": 3.5,
|
||||
"n": 351
|
||||
},
|
||||
"socket": {
|
||||
"p50": 0.1,
|
||||
"p95": 0.4,
|
||||
"p99": 0.6,
|
||||
"n": 351
|
||||
},
|
||||
"network": {
|
||||
"p50": 11.6,
|
||||
"p95": 75.7,
|
||||
"p99": 130.1,
|
||||
"n": 351
|
||||
},
|
||||
"decode": {
|
||||
"p50": 0.7,
|
||||
"p95": 2.1,
|
||||
"p99": 4.1,
|
||||
"n": 351
|
||||
},
|
||||
"draw": {
|
||||
"p50": 0.4,
|
||||
"p95": 0.8,
|
||||
"p99": 1.1,
|
||||
"n": 351
|
||||
},
|
||||
"present": {
|
||||
"p50": 0.8,
|
||||
"p95": 18.1,
|
||||
"p99": 24.9,
|
||||
"n": 324
|
||||
},
|
||||
"content": {
|
||||
"p50": 15.1,
|
||||
"p95": 80.3,
|
||||
"p99": 134.2,
|
||||
"n": 351
|
||||
},
|
||||
"content_shown": {
|
||||
"p50": 21.0,
|
||||
"p95": 71.8,
|
||||
"p99": 146.3,
|
||||
"n": 324
|
||||
}
|
||||
},
|
||||
"fps": 34.8,
|
||||
"fps_shown": 32.0,
|
||||
"late_pct": 53.41,
|
||||
"stall_max": 226.8,
|
||||
"stalls_over_100ms": 7,
|
||||
"mbps": 0.26,
|
||||
"keyframes": 6,
|
||||
"size": "1280x720",
|
||||
"captured": 607,
|
||||
"viewer_never_drawn": 0,
|
||||
"input_ms": {
|
||||
"p50": 43.3,
|
||||
"p95": 98.4,
|
||||
"p99": 110.1,
|
||||
"n": 18
|
||||
},
|
||||
"input_shown_ms": {
|
||||
"p50": 54.7,
|
||||
"p95": 98.9,
|
||||
"p99": 114.0,
|
||||
"n": 18
|
||||
},
|
||||
"input_parts_ms": {
|
||||
"uplink": {
|
||||
"p50": 17.1,
|
||||
"p95": 48.3,
|
||||
"p99": 53.1,
|
||||
"n": 18
|
||||
},
|
||||
"mac": {
|
||||
"p50": 12.5,
|
||||
"p95": 32.9,
|
||||
"p99": 35.7,
|
||||
"n": 18
|
||||
},
|
||||
"back": {
|
||||
"p50": 16.8,
|
||||
"p95": 50.0,
|
||||
"p99": 55.9,
|
||||
"n": 18
|
||||
}
|
||||
},
|
||||
"inputs": {
|
||||
"asked": 18,
|
||||
"sent": 18,
|
||||
"seen": 18
|
||||
},
|
||||
"timeline": [
|
||||
{
|
||||
"t": 0,
|
||||
"fps": 38,
|
||||
"mbps": 0.3,
|
||||
"content_p50": 14.8,
|
||||
"content_p95": 25.3,
|
||||
"bitrate": 2764800,
|
||||
"tier": 0,
|
||||
"w": 1280,
|
||||
"net": null
|
||||
},
|
||||
{
|
||||
"t": 1,
|
||||
"fps": 33,
|
||||
"mbps": 0.24,
|
||||
"content_p50": 15.6,
|
||||
"content_p95": 103.5,
|
||||
"bitrate": 3845448,
|
||||
"tier": 0,
|
||||
"w": 1280,
|
||||
"net": null
|
||||
},
|
||||
{
|
||||
"t": 2,
|
||||
"fps": 39,
|
||||
"mbps": 0.28,
|
||||
"content_p50": 15.6,
|
||||
"content_p95": 29.9,
|
||||
"bitrate": 5283790,
|
||||
"tier": 0,
|
||||
"w": 1280,
|
||||
"net": null
|
||||
},
|
||||
{
|
||||
"t": 3,
|
||||
"fps": 25,
|
||||
"mbps": 0.21,
|
||||
"content_p50": 17.8,
|
||||
"content_p95": 175.9,
|
||||
"bitrate": 2764800,
|
||||
"tier": 0,
|
||||
"w": 1280,
|
||||
"net": null
|
||||
},
|
||||
{
|
||||
"t": 4,
|
||||
"fps": 34,
|
||||
"mbps": 0.22,
|
||||
"content_p50": 15.4,
|
||||
"content_p95": 69.8,
|
||||
"bitrate": 2764800,
|
||||
"tier": 0,
|
||||
"w": 1280,
|
||||
"net": null
|
||||
},
|
||||
{
|
||||
"t": 5,
|
||||
"fps": 38,
|
||||
"mbps": 0.29,
|
||||
"content_p50": 17.7,
|
||||
"content_p95": 68.7,
|
||||
"bitrate": 2764800,
|
||||
"tier": 0,
|
||||
"w": 1280,
|
||||
"net": null
|
||||
},
|
||||
{
|
||||
"t": 6,
|
||||
"fps": 38,
|
||||
"mbps": 0.26,
|
||||
"content_p50": 15.0,
|
||||
"content_p95": 18.2,
|
||||
"bitrate": 3091280,
|
||||
"tier": 0,
|
||||
"w": 1280,
|
||||
"net": null
|
||||
},
|
||||
{
|
||||
"t": 7,
|
||||
"fps": 30,
|
||||
"mbps": 0.22,
|
||||
"content_p50": 19.2,
|
||||
"content_p95": 111.0,
|
||||
"bitrate": 1725204,
|
||||
"tier": 0,
|
||||
"w": 1280,
|
||||
"net": null
|
||||
},
|
||||
{
|
||||
"t": 8,
|
||||
"fps": 35,
|
||||
"mbps": 0.26,
|
||||
"content_p50": 13.4,
|
||||
"content_p95": 102.6,
|
||||
"bitrate": 1725204,
|
||||
"tier": 0,
|
||||
"w": 1280,
|
||||
"net": null
|
||||
},
|
||||
{
|
||||
"t": 9,
|
||||
"fps": 38,
|
||||
"mbps": 0.28,
|
||||
"content_p50": 12.3,
|
||||
"content_p95": 20.9,
|
||||
"bitrate": 1725204,
|
||||
"tier": 0,
|
||||
"w": 1280,
|
||||
"net": null
|
||||
},
|
||||
{
|
||||
"t": 10,
|
||||
"fps": 3,
|
||||
"mbps": 0.02,
|
||||
"content_p50": 20.9,
|
||||
"content_p95": 24.7,
|
||||
"bitrate": 1725204,
|
||||
"tier": 0,
|
||||
"w": 1280,
|
||||
"net": null
|
||||
}
|
||||
],
|
||||
"adapt": [],
|
||||
"content_p50": 15.1,
|
||||
"content_p95": 80.3,
|
||||
"input_p50": 43.3,
|
||||
"input_p95": 98.4,
|
||||
"grades": {
|
||||
"input_replies": "local",
|
||||
"content_p50": "local",
|
||||
"content_p95": "acceptable",
|
||||
"input_p50": "local",
|
||||
"input_p95": "acceptable",
|
||||
"fps": "bad",
|
||||
"late_pct": "bad",
|
||||
"stall_max": "acceptable"
|
||||
},
|
||||
"controller": {
|
||||
"target": 3442081,
|
||||
"ceiling": 5529600,
|
||||
"tier": 0,
|
||||
"fps": 60,
|
||||
"scale": 1.0,
|
||||
"baseRtt": 13.334,
|
||||
"inFlight": 1,
|
||||
"slack": 71.738,
|
||||
"adapt": true
|
||||
},
|
||||
"events": [
|
||||
{
|
||||
"t": 3.77,
|
||||
"e": "target 2764800 bit/s; tier 0"
|
||||
},
|
||||
{
|
||||
"t": 7.42,
|
||||
"e": "target 1725204 bit/s; tier 0"
|
||||
}
|
||||
],
|
||||
"source_fps": 60.1,
|
||||
"cpu": {
|
||||
"host": "not sampled"
|
||||
},
|
||||
"viewer": "h264 software; ANGLE (Mesa, zink Vulkan 1.4(Turnip Adreno (TM) 750 (MESA_TURNIP)), OpenGL 4.6); raf 71 Hz",
|
||||
"show_s": 1.88,
|
||||
"panel": "valve.steam.desktopgame.2001932457",
|
||||
"src": "test",
|
||||
"route": "network"
|
||||
}
|
||||
],
|
||||
"proof": "Synthetic x264 PC agent runs on the Frame itself, forwarded via Mac and back. Not Windows/Linux desktop capture or laser input."
|
||||
}
|
||||
@@ -0,0 +1,302 @@
|
||||
{
|
||||
"label": "pc-agent-frame-arm64-final",
|
||||
"date": "2026-09-28T22:55:27",
|
||||
"commit": "cd20243",
|
||||
"config": {
|
||||
"quality": "balanced",
|
||||
"mode": "native",
|
||||
"net": "none",
|
||||
"delay_ms": 0,
|
||||
"buffer_ms": 250,
|
||||
"host": "frame",
|
||||
"usb": false,
|
||||
"ssh_opts": [],
|
||||
"encoder": "",
|
||||
"duration_s": 10.0,
|
||||
"browser_flags": []
|
||||
},
|
||||
"frame_build": "20260925.6191901",
|
||||
"mac": "26.5.2",
|
||||
"host_platform": "Linux aarch64, SteamOS Frame, BUILD_ID 20260925.6191901 (synthetic host only)",
|
||||
"source": "",
|
||||
"pc_host": true,
|
||||
"headset": "0:03.056203 [Info] - 1 - leaving standby",
|
||||
"scenarios": [
|
||||
{
|
||||
"scenario": "test",
|
||||
"frames_sent": 400,
|
||||
"frames_drawn": 400,
|
||||
"frames_shown": 339,
|
||||
"duration_s": 10.2,
|
||||
"stages_ms": {
|
||||
"capture": {
|
||||
"p50": 0.1,
|
||||
"p95": 0.1,
|
||||
"p99": 0.2,
|
||||
"n": 400
|
||||
},
|
||||
"queue": {
|
||||
"p50": 0.1,
|
||||
"p95": 0.2,
|
||||
"p99": 0.4,
|
||||
"n": 400
|
||||
},
|
||||
"encode": {
|
||||
"p50": 2.0,
|
||||
"p95": 2.7,
|
||||
"p99": 3.1,
|
||||
"n": 400
|
||||
},
|
||||
"socket": {
|
||||
"p50": 0.1,
|
||||
"p95": 0.3,
|
||||
"p99": 0.4,
|
||||
"n": 400
|
||||
},
|
||||
"network": {
|
||||
"p50": 11.8,
|
||||
"p95": 25.2,
|
||||
"p99": 37.8,
|
||||
"n": 400
|
||||
},
|
||||
"decode": {
|
||||
"p50": 0.7,
|
||||
"p95": 1.5,
|
||||
"p99": 2.0,
|
||||
"n": 400
|
||||
},
|
||||
"draw": {
|
||||
"p50": 0.3,
|
||||
"p95": 0.7,
|
||||
"p99": 1.0,
|
||||
"n": 400
|
||||
},
|
||||
"present": {
|
||||
"p50": 0.8,
|
||||
"p95": 19.3,
|
||||
"p99": 26.8,
|
||||
"n": 339
|
||||
},
|
||||
"content": {
|
||||
"p50": 15.1,
|
||||
"p95": 28.6,
|
||||
"p99": 41.3,
|
||||
"n": 400
|
||||
},
|
||||
"content_shown": {
|
||||
"p50": 17.6,
|
||||
"p95": 39.0,
|
||||
"p99": 47.2,
|
||||
"n": 339
|
||||
}
|
||||
},
|
||||
"fps": 39.3,
|
||||
"fps_shown": 33.4,
|
||||
"late_pct": 50.25,
|
||||
"stall_max": 69.3,
|
||||
"stalls_over_100ms": 0,
|
||||
"mbps": 0.35,
|
||||
"keyframes": 6,
|
||||
"size": "1280x720",
|
||||
"captured": 612,
|
||||
"viewer_never_drawn": 0,
|
||||
"input_ms": {
|
||||
"p50": 76.6,
|
||||
"p95": 98.2,
|
||||
"p99": 127.8,
|
||||
"n": 19
|
||||
},
|
||||
"input_shown_ms": {
|
||||
"p50": 80.5,
|
||||
"p95": 107.8,
|
||||
"p99": 128.1,
|
||||
"n": 16
|
||||
},
|
||||
"input_parts_ms": {
|
||||
"uplink": {
|
||||
"p50": 36.5,
|
||||
"p95": 57.6,
|
||||
"p99": 83.2,
|
||||
"n": 19
|
||||
},
|
||||
"mac": {
|
||||
"p50": 14.6,
|
||||
"p95": 31.3,
|
||||
"p99": 31.7,
|
||||
"n": 19
|
||||
},
|
||||
"back": {
|
||||
"p50": 15.1,
|
||||
"p95": 32.8,
|
||||
"p99": 36.2,
|
||||
"n": 19
|
||||
}
|
||||
},
|
||||
"inputs": {
|
||||
"asked": 19,
|
||||
"sent": 19,
|
||||
"seen": 19
|
||||
},
|
||||
"timeline": [
|
||||
{
|
||||
"t": 0,
|
||||
"fps": 40,
|
||||
"mbps": 0.34,
|
||||
"content_p50": 11.7,
|
||||
"content_p95": 27.4,
|
||||
"bitrate": 5529600,
|
||||
"tier": 0,
|
||||
"w": 1280,
|
||||
"net": null
|
||||
},
|
||||
{
|
||||
"t": 1,
|
||||
"fps": 39,
|
||||
"mbps": 0.37,
|
||||
"content_p50": 11.9,
|
||||
"content_p95": 28.2,
|
||||
"bitrate": 5529600,
|
||||
"tier": 0,
|
||||
"w": 1280,
|
||||
"net": null
|
||||
},
|
||||
{
|
||||
"t": 2,
|
||||
"fps": 41,
|
||||
"mbps": 0.35,
|
||||
"content_p50": 11.4,
|
||||
"content_p95": 16.6,
|
||||
"bitrate": 5529600,
|
||||
"tier": 0,
|
||||
"w": 1280,
|
||||
"net": null
|
||||
},
|
||||
{
|
||||
"t": 3,
|
||||
"fps": 39,
|
||||
"mbps": 0.37,
|
||||
"content_p50": 12.7,
|
||||
"content_p95": 28.0,
|
||||
"bitrate": 5529600,
|
||||
"tier": 0,
|
||||
"w": 1280,
|
||||
"net": null
|
||||
},
|
||||
{
|
||||
"t": 4,
|
||||
"fps": 38,
|
||||
"mbps": 0.33,
|
||||
"content_p50": 15.3,
|
||||
"content_p95": 30.0,
|
||||
"bitrate": 5529600,
|
||||
"tier": 0,
|
||||
"w": 1280,
|
||||
"net": null
|
||||
},
|
||||
{
|
||||
"t": 5,
|
||||
"fps": 39,
|
||||
"mbps": 0.37,
|
||||
"content_p50": 16.7,
|
||||
"content_p95": 28.5,
|
||||
"bitrate": 5529600,
|
||||
"tier": 0,
|
||||
"w": 1280,
|
||||
"net": null
|
||||
},
|
||||
{
|
||||
"t": 6,
|
||||
"fps": 41,
|
||||
"mbps": 0.38,
|
||||
"content_p50": 15.9,
|
||||
"content_p95": 29.4,
|
||||
"bitrate": 5529600,
|
||||
"tier": 0,
|
||||
"w": 1280,
|
||||
"net": null
|
||||
},
|
||||
{
|
||||
"t": 7,
|
||||
"fps": 41,
|
||||
"mbps": 0.31,
|
||||
"content_p50": 15.6,
|
||||
"content_p95": 28.2,
|
||||
"bitrate": 5529600,
|
||||
"tier": 0,
|
||||
"w": 1280,
|
||||
"net": null
|
||||
},
|
||||
{
|
||||
"t": 8,
|
||||
"fps": 38,
|
||||
"mbps": 0.38,
|
||||
"content_p50": 15.5,
|
||||
"content_p95": 25.9,
|
||||
"bitrate": 5529600,
|
||||
"tier": 0,
|
||||
"w": 1280,
|
||||
"net": null
|
||||
},
|
||||
{
|
||||
"t": 9,
|
||||
"fps": 38,
|
||||
"mbps": 0.29,
|
||||
"content_p50": 16.7,
|
||||
"content_p95": 28.8,
|
||||
"bitrate": 5529600,
|
||||
"tier": 0,
|
||||
"w": 1280,
|
||||
"net": null
|
||||
},
|
||||
{
|
||||
"t": 10,
|
||||
"fps": 6,
|
||||
"mbps": 0.03,
|
||||
"content_p50": 17.6,
|
||||
"content_p95": 23.1,
|
||||
"bitrate": 5529600,
|
||||
"tier": 0,
|
||||
"w": 1280,
|
||||
"net": null
|
||||
}
|
||||
],
|
||||
"adapt": [],
|
||||
"content_p50": 15.1,
|
||||
"content_p95": 28.6,
|
||||
"input_p50": 76.6,
|
||||
"input_p95": 98.2,
|
||||
"grades": {
|
||||
"input_replies": "local",
|
||||
"content_p50": "local",
|
||||
"content_p95": "local",
|
||||
"input_p50": "acceptable",
|
||||
"input_p95": "acceptable",
|
||||
"fps": "bad",
|
||||
"late_pct": "bad",
|
||||
"stall_max": "local"
|
||||
},
|
||||
"controller": {
|
||||
"target": 5529600,
|
||||
"ceiling": 5529600,
|
||||
"tier": 0,
|
||||
"fps": 60,
|
||||
"scale": 1.0,
|
||||
"baseRtt": 12.332,
|
||||
"inFlight": 0,
|
||||
"slack": 74.12,
|
||||
"adapt": true
|
||||
},
|
||||
"events": [],
|
||||
"source_fps": 60.0,
|
||||
"cpu": {
|
||||
"host": "not sampled"
|
||||
},
|
||||
"viewer": "h264 software; ANGLE (Mesa, zink Vulkan 1.4(Turnip Adreno (TM) 750 (MESA_TURNIP)), OpenGL 4.6); raf 72 Hz",
|
||||
"show_s": 1.75,
|
||||
"panel": "valve.steam.desktopgame.2001932457",
|
||||
"src": "test",
|
||||
"route": "network"
|
||||
}
|
||||
],
|
||||
"proof": "Synthetic x264 PC agent runs on the Frame itself, forwarded via Mac and back. Not Windows/Linux desktop capture or laser input."
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Build the common controller for the Python PC host (a C11 compiler needed)."""
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
root = Path(__file__).resolve().parent
|
||||
suffix = '.dll' if sys.platform == 'win32' else '.dylib' if sys.platform == 'darwin' else '.so'
|
||||
out = root / ('controller' + suffix)
|
||||
if sys.platform == 'win32' and os.environ.get('CC', 'cl') == 'cl':
|
||||
cmd = ['cl', '/nologo', '/O2', '/LD', '/std:c11', str(root / 'controller.c'), '/link', '/OUT:' + str(out)]
|
||||
else:
|
||||
cmd = [os.environ.get('CC', 'cc'), '-O2', '-std=c11', '-shared', '-fPIC', str(root / 'controller.c'), '-o', str(out)]
|
||||
subprocess.run(cmd, cwd=root, check=True)
|
||||
print(out)
|
||||
@@ -0,0 +1,118 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Build our native PC adapter and bundle its ordinary shared libraries.
|
||||
|
||||
Run on the target architecture after installing GStreamer development packages.
|
||||
No GStreamer executable or desktop streaming application is shipped or invoked.
|
||||
Linux libc, display/GPU drivers and the portal service remain platform pieces.
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
ROOT = Path(__file__).resolve().parent
|
||||
|
||||
|
||||
def output(*args):
|
||||
return subprocess.check_output(args, text=True).strip()
|
||||
|
||||
|
||||
def build(destination):
|
||||
win = sys.platform == 'win32'
|
||||
destination.mkdir(parents=True, exist_ok=True)
|
||||
modules = ['gstreamer-1.0', 'gstreamer-app-1.0', 'gstreamer-video-1.0']
|
||||
if not win:
|
||||
modules += ['gio-unix-2.0']
|
||||
import shlex
|
||||
flags = shlex.split(output('pkg-config', '--cflags', '--libs', *modules))
|
||||
lib = destination / ('pc-host.dll' if win else 'pc-host.so')
|
||||
cmd = [os.environ.get('CC', 'gcc' if win else 'cc'), '-std=c11', '-O2', '-shared', '-Wall', '-Wextra']
|
||||
if not win:
|
||||
cmd += ['-fPIC', '-Wl,-rpath,$ORIGIN/lib']
|
||||
subprocess.run(cmd + [str(ROOT / f) for f in ('controller.c', 'capture.c', 'portal.c')] + flags + ['-o', str(lib)], check=True)
|
||||
prefix = Path(output('pkg-config', '--variable=prefix', 'gstreamer-1.0'))
|
||||
plugin_dir = Path(output('pkg-config', '--variable=pluginsdir', 'gstreamer-1.0'))
|
||||
plugins_out, libs_out = destination / 'lib' / 'gstreamer-1.0', destination / ('bin' if win else 'lib')
|
||||
plugins_out.mkdir(parents=True, exist_ok=True)
|
||||
libs_out.mkdir(parents=True, exist_ok=True)
|
||||
names = ['coreelements', 'videotestsrc', 'videoconvertscale', 'app', 'jpeg', 'videoparsersbad', 'x264']
|
||||
names += ['d3d11', 'mediafoundation'] if win else ['pipewire', 'va']
|
||||
pending = [lib]
|
||||
for name in names:
|
||||
matches = list(plugin_dir.glob('*gst' + name + ('.dll' if win else '.so')))
|
||||
if not matches:
|
||||
raise SystemExit('Missing GStreamer library: ' + name)
|
||||
for source in matches:
|
||||
target = plugins_out / source.name
|
||||
shutil.copy2(source, target)
|
||||
pending.append(source)
|
||||
if not win:
|
||||
# PipeWire dynamically loads support and protocol modules. ldd cannot
|
||||
# discover those; bundle them with a private client-only config instead
|
||||
# of accidentally loading a different distro's client modules/config.
|
||||
libdir = plugin_dir.parent
|
||||
for relative in ('pipewire-0.3/libpipewire-module-protocol-native.so',
|
||||
'pipewire-0.3/libpipewire-module-client-node.so',
|
||||
'pipewire-0.3/libpipewire-module-adapter.so',
|
||||
'spa-0.2/support/libspa-support.so',
|
||||
'spa-0.2/support/libspa-dbus.so',
|
||||
'spa-0.2/videoconvert/libspa-videoconvert.so'):
|
||||
source = libdir / relative
|
||||
if not source.is_file():
|
||||
raise SystemExit('Missing PipeWire runtime library: ' + str(source))
|
||||
target = destination / 'lib' / relative
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copy2(source, target)
|
||||
pending.append(source)
|
||||
config = destination / 'share' / 'pipewire'
|
||||
config.mkdir(parents=True, exist_ok=True)
|
||||
(config / 'client.conf').write_text(
|
||||
'context.spa-libs = { support.* = support/libspa-support video.* = videoconvert/libspa-videoconvert }\n'
|
||||
'context.modules = [\n'
|
||||
' { name = libpipewire-module-protocol-native }\n'
|
||||
' { name = libpipewire-module-client-node }\n'
|
||||
' { name = libpipewire-module-adapter }\n'
|
||||
']\n')
|
||||
copied = set()
|
||||
while pending:
|
||||
binary = pending.pop()
|
||||
if win:
|
||||
imported = re.findall(r'DLL Name:\s*(\S+)', output('objdump', '-p', str(binary)))
|
||||
dependencies = [prefix / 'bin' / name for name in imported]
|
||||
else:
|
||||
dependencies = [Path(p) for p in re.findall(r'=>\s+(/\S+)', output('ldd', str(binary)))]
|
||||
for dep in dependencies:
|
||||
if not dep.is_file() or dep.name in copied:
|
||||
continue
|
||||
if not win and re.match(r'lib(c|m|dl|rt|pthread|resolv)\.so', dep.name):
|
||||
continue
|
||||
shutil.copy2(dep, libs_out / dep.name)
|
||||
copied.add(dep.name)
|
||||
pending.append(dep)
|
||||
# License texts and package provenance accompany the dynamically linked
|
||||
# libraries. Distribution builders keep the upstream package/source URLs.
|
||||
licenses = destination / 'licenses'
|
||||
licenses.mkdir(exist_ok=True)
|
||||
if win:
|
||||
source = prefix / 'share' / 'licenses'
|
||||
if source.exists():
|
||||
shutil.copytree(source, licenses, dirs_exist_ok=True)
|
||||
packages = output('pacman', '-Q') if shutil.which('pacman') else 'See MSYS2 build log'
|
||||
else:
|
||||
packages = output('dpkg-query', '-W', '-f=${Package} ${Version}\n') if shutil.which('dpkg-query') else ''
|
||||
# Debian copyright files contain licenses and source homepage details.
|
||||
for path in Path('/usr/share/doc').glob('*/copyright'):
|
||||
shutil.copy2(path, licenses / (path.parent.name + '.copyright'))
|
||||
(destination / 'libraries.json').write_text(json.dumps(dict(gstreamer=output('pkg-config', '--modversion', 'gstreamer-1.0'),
|
||||
libraries=sorted(copied), packages=packages), indent=2))
|
||||
print(lib)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--out', type=Path, default=ROOT / 'bundle')
|
||||
build(parser.parse_args().out.resolve())
|
||||
@@ -0,0 +1,125 @@
|
||||
/* Frame Control's capture/encode adapter. GStreamer is a bundled library;
|
||||
* capture uses WGC on Windows and the consented PipeWire fd on Linux. */
|
||||
#include "controller.h"
|
||||
#include <gst/gst.h>
|
||||
#include <gst/app/gstappsink.h>
|
||||
#include <gst/video/video-event.h>
|
||||
#include <string.h>
|
||||
|
||||
typedef int (*Gate)(int stage, int64_t pts, int64_t capture, int64_t arrived);
|
||||
typedef struct {
|
||||
GstElement *pipeline, *encoder, *sink, *raw_queue;
|
||||
Gate gate;
|
||||
GstSample *sample;
|
||||
GstMapInfo map;
|
||||
int mapped;
|
||||
char error[512];
|
||||
} Capture;
|
||||
typedef struct { const unsigned char *data; int size, key, width, height; int64_t pts; } Encoded;
|
||||
FC_API int64_t fc_now(void) {return g_get_monotonic_time();}
|
||||
FC_API void fc_gst_init(void) {gst_init(NULL,NULL);}
|
||||
FC_API int fc_has_element(const char *name) {
|
||||
GstElementFactory *f=gst_element_factory_find(name);
|
||||
if(!f)return 0;gst_object_unref(f);return 1;
|
||||
}
|
||||
static GstPadProbeReturn probe(GstPad *pad,GstPadProbeInfo *info,gpointer data) {
|
||||
Capture *c=data; GstBuffer *b=GST_PAD_PROBE_INFO_BUFFER(info);
|
||||
if(!b)return GST_PAD_PROBE_OK;
|
||||
int64_t now=fc_now(), cap=now;
|
||||
GstClock *clock=gst_element_get_clock(c->pipeline);
|
||||
if(clock && GST_BUFFER_PTS_IS_VALID(b)) {
|
||||
GstClockTime current=gst_clock_get_time(clock),origin=gst_element_get_base_time(c->pipeline);
|
||||
if(current>=origin && current-origin>=GST_BUFFER_PTS(b))
|
||||
cap-= (int64_t)((current-origin-GST_BUFFER_PTS(b))/1000);
|
||||
}
|
||||
if(clock)gst_object_unref(clock);
|
||||
int stage=GPOINTER_TO_INT(g_object_get_data(G_OBJECT(pad),"stage"));
|
||||
if(stage==1) return c->gate(stage,(int64_t)GST_BUFFER_PTS(b),cap,now)>0 ? GST_PAD_PROBE_OK : GST_PAD_PROBE_DROP;
|
||||
int phase=0;
|
||||
for(;;) {
|
||||
/* While congested, prefer a newer raw picture queued upstream. If
|
||||
* nothing changed, retain this last picture until the gate opens;
|
||||
* an idle window must not stay stale after a dropped final update. */
|
||||
guint queued=0;
|
||||
if(phase && c->raw_queue)g_object_get(c->raw_queue,"current-level-buffers",&queued,NULL);
|
||||
if(queued)return GST_PAD_PROBE_DROP;
|
||||
int decision=c->gate(phase,(int64_t)GST_BUFFER_PTS(b),cap,now);
|
||||
if(decision)return decision>0 ? GST_PAD_PROBE_OK : GST_PAD_PROBE_DROP;
|
||||
phase=2;g_usleep(2000);
|
||||
}
|
||||
}
|
||||
FC_API Capture *fc_capture_open(const char *pipeline,Gate gate,char *error,int capacity) {
|
||||
GError *e=NULL;Capture *c=g_new0(Capture,1);c->gate=gate;
|
||||
c->pipeline=gst_parse_launch(pipeline,&e);
|
||||
if(e || !c->pipeline) {
|
||||
g_strlcpy(error,e?e->message:"No pipeline",capacity);
|
||||
if(e)g_error_free(e);if(c->pipeline)gst_object_unref(c->pipeline);g_free(c);return NULL;
|
||||
}
|
||||
c->encoder=gst_bin_get_by_name(GST_BIN(c->pipeline),"enc");
|
||||
c->sink=gst_bin_get_by_name(GST_BIN(c->pipeline),"out");
|
||||
GstElement *raw=gst_bin_get_by_name(GST_BIN(c->pipeline),"gate");
|
||||
if(!c->encoder || !c->sink || !raw) {
|
||||
g_strlcpy(error,"Pipeline is missing enc, out or gate",capacity);
|
||||
if(raw)gst_object_unref(raw);if(c->encoder)gst_object_unref(c->encoder);
|
||||
if(c->sink)gst_object_unref(c->sink);gst_object_unref(c->pipeline);g_free(c);return NULL;
|
||||
}
|
||||
c->raw_queue=gst_bin_get_by_name(GST_BIN(c->pipeline),"raw_queue");
|
||||
GstPad *p=gst_element_get_static_pad(raw,"src");
|
||||
g_object_set_data(G_OBJECT(p),"stage",GINT_TO_POINTER(0));
|
||||
gst_pad_add_probe(p,GST_PAD_PROBE_TYPE_BUFFER,probe,c,NULL);gst_object_unref(p);gst_object_unref(raw);
|
||||
p=gst_element_get_static_pad(c->encoder,"sink");
|
||||
g_object_set_data(G_OBJECT(p),"stage",GINT_TO_POINTER(1));
|
||||
gst_pad_add_probe(p,GST_PAD_PROBE_TYPE_BUFFER,probe,c,NULL);gst_object_unref(p);
|
||||
gst_element_set_state(c->pipeline,GST_STATE_PLAYING);
|
||||
return c;
|
||||
}
|
||||
FC_API int fc_capture_pull(Capture *c,Encoded *out) {
|
||||
if(c->mapped) {gst_buffer_unmap(gst_sample_get_buffer(c->sample),&c->map);c->mapped=0;}
|
||||
if(c->sample){gst_sample_unref(c->sample);c->sample=NULL;}
|
||||
GstBus *bus=gst_element_get_bus(c->pipeline);
|
||||
GstMessage *m=gst_bus_pop_filtered(bus,GST_MESSAGE_ERROR);gst_object_unref(bus);
|
||||
if(m) {
|
||||
if(GST_MESSAGE_TYPE(m)==GST_MESSAGE_ERROR) {
|
||||
GError *e=NULL;char *debug=NULL;gst_message_parse_error(m,&e,&debug);
|
||||
g_strlcpy(c->error,e->message,sizeof(c->error));g_error_free(e);g_free(debug);
|
||||
} else g_strlcpy(c->error,"The capture source closed",sizeof(c->error));
|
||||
gst_message_unref(m);return -1;
|
||||
}
|
||||
c->sample=gst_app_sink_try_pull_sample(GST_APP_SINK(c->sink),100*GST_MSECOND);
|
||||
if(!c->sample) {
|
||||
if(gst_app_sink_is_eos(GST_APP_SINK(c->sink))) {
|
||||
g_strlcpy(c->error,"The capture source closed",sizeof(c->error));return -1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
GstBuffer *b=gst_sample_get_buffer(c->sample);
|
||||
if(!gst_buffer_map(b,&c->map,GST_MAP_READ))return 0;
|
||||
c->mapped=1;out->data=c->map.data;out->size=(int)c->map.size;
|
||||
out->key=!GST_BUFFER_FLAG_IS_SET(b,GST_BUFFER_FLAG_DELTA_UNIT);out->pts=(int64_t)GST_BUFFER_PTS(b);
|
||||
const GstStructure *s=gst_caps_get_structure(gst_sample_get_caps(c->sample),0);
|
||||
gst_structure_get_int(s,"width",&out->width);gst_structure_get_int(s,"height",&out->height);return 1;
|
||||
}
|
||||
FC_API const char *fc_capture_error(Capture *c) {return c->error;}
|
||||
FC_API void fc_capture_bitrate(Capture *c,int bps) {
|
||||
g_object_set(c->encoder,"bitrate",(guint)MAX(1,bps/1000),NULL);
|
||||
}
|
||||
FC_API void fc_capture_test(Capture *c,uint32_t input) {
|
||||
GstElement *source=gst_bin_get_by_name(GST_BIN(c->pipeline),"source");
|
||||
if(source) {
|
||||
/* Each benchmark click visibly changes the ball, before injection is
|
||||
* timestamped. This is a response, not merely a protocol input echo. */
|
||||
guint color=0xff000000u | ((input*2654435761u)&0x00ffffffu);
|
||||
g_object_set(source,"foreground-color",color,NULL);gst_object_unref(source);
|
||||
}
|
||||
}
|
||||
FC_API void fc_capture_key(Capture *c) {
|
||||
GstPad *p=gst_element_get_static_pad(c->encoder,"src");
|
||||
gst_pad_send_event(p,gst_video_event_new_upstream_force_key_unit(GST_CLOCK_TIME_NONE,TRUE,0));gst_object_unref(p);
|
||||
}
|
||||
FC_API void fc_capture_close(Capture *c) {
|
||||
gst_element_set_state(c->pipeline,GST_STATE_NULL);
|
||||
if(c->mapped)gst_buffer_unmap(gst_sample_get_buffer(c->sample),&c->map);
|
||||
if(c->sample)gst_sample_unref(c->sample);
|
||||
if(c->raw_queue)gst_object_unref(c->raw_queue);
|
||||
gst_object_unref(c->sink);gst_object_unref(c->encoder);gst_object_unref(c->pipeline);g_free(c);
|
||||
}
|
||||
@@ -0,0 +1,127 @@
|
||||
/* Extracted from the Mac agent's RateController. Same gate, demand protection,
|
||||
* bitrate recovery and tier hysteresis for every host. Bounded sample buffers. */
|
||||
#include "controller.h"
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#define CAP 1024
|
||||
#define MIN(a,b) ((a)<(b)?(a):(b))
|
||||
#define MAX(a,b) ((a)>(b)?(a):(b))
|
||||
typedef struct { int64_t t, v; uint32_t seq; } Sample;
|
||||
typedef struct { Sample a[CAP]; int n; } Samples;
|
||||
struct FCController {
|
||||
int max_fps, enabled, ceiling, target, tier, frame_bytes, held, signal, saw_ack;
|
||||
int64_t decrease, increase, below, above, slack;
|
||||
Samples flight, rtts, acked, sent, captures;
|
||||
};
|
||||
static const int fps[] = {60,45,30,30,30}, scale[] = {100,100,100,75,50};
|
||||
static const double floors[] = {.45,.28,.16,.08,0};
|
||||
static void remove_first(Samples *s, int n) {
|
||||
s->n -= n; memmove(s->a, s->a+n, (size_t)s->n*sizeof(Sample));
|
||||
}
|
||||
static void add(Samples *s, Sample v, int cap) {
|
||||
if (s->n >= cap) remove_first(s, 1);
|
||||
s->a[s->n++] = v;
|
||||
}
|
||||
static void expire(Samples *s, int64_t oldest) {
|
||||
int n=0; while(n<s->n && s->a[n].t<oldest) n++;
|
||||
remove_first(s,n);
|
||||
}
|
||||
static int64_t base(FCController *c) {
|
||||
int64_t v=0;
|
||||
for(int i=0;i<c->rtts.n;i++) if(!i || c->rtts.a[i].v<v) v=c->rtts.a[i].v;
|
||||
return v;
|
||||
}
|
||||
static int compare(const void *a, const void *b) {
|
||||
int64_t x=*(const int64_t*)a, y=*(const int64_t*)b;
|
||||
return (x>y)-(x<y);
|
||||
}
|
||||
static int64_t quantile(Samples *s,int64_t since,int numerator,int denominator) {
|
||||
int64_t a[CAP]; int n=0;
|
||||
for(int i=0;i<s->n;i++) if(s->a[i].t>since) a[n++]=s->a[i].v;
|
||||
if(!n) return 0;
|
||||
qsort(a,(size_t)n,sizeof(int64_t),compare);
|
||||
return a[n*numerator/denominator];
|
||||
}
|
||||
static int rate(Samples *s) {
|
||||
int64_t total=0; for(int i=0;i<s->n;i++) total+=s->a[i].v;
|
||||
return (int)MIN(total*16,2147483647);
|
||||
}
|
||||
FCController *fc_new(int max_fps,int enabled) {
|
||||
FCController *c=calloc(1,sizeof(*c));
|
||||
if(c) {c->max_fps=MAX(1,max_fps);c->enabled=enabled;c->slack=40000;}
|
||||
return c;
|
||||
}
|
||||
void fc_free(FCController *c) {free(c);}
|
||||
void fc_ceiling(FCController *c,int bps) {
|
||||
if(!c->target || c->target>bps) c->target=bps;
|
||||
c->ceiling=bps;
|
||||
}
|
||||
int fc_gate(FCController *c,int64_t now,int counts) {
|
||||
if(!c->enabled || !c->saw_ack) return 1;
|
||||
expire(&c->flight,now-2000000);
|
||||
if(!c->flight.n) return 1;
|
||||
int64_t interval=1000000/MIN(c->max_fps,fps[c->tier]);
|
||||
int window=MAX(3,(int)((base(c)+c->slack)/interval)+1);
|
||||
if(c->flight.n<window && now-c->flight.a[0].t<=base(c)+c->slack) return 1;
|
||||
if(counts) c->held++;
|
||||
return 0;
|
||||
}
|
||||
void fc_capture(FCController *c,int64_t now) {add(&c->captures,(Sample){now,0,0},256);}
|
||||
void fc_sent(FCController *c,uint32_t seq,int bytes,int64_t now) {
|
||||
Sample s={now,bytes,seq};add(&c->flight,s,512);add(&c->sent,s,CAP);
|
||||
}
|
||||
int fc_ack(FCController *c,uint32_t seq,int64_t now) {
|
||||
c->saw_ack=1;
|
||||
for(int i=0;i<c->flight.n;i++) if(c->flight.a[i].seq==seq) {
|
||||
Sample s=c->flight.a[i];remove_first(&c->flight,i+1);
|
||||
add(&c->rtts,(Sample){now,now-s.t,0},CAP);
|
||||
add(&c->acked,(Sample){now,s.v,0},CAP);return 1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
int fc_update(FCController *c,int64_t now) {
|
||||
expire(&c->rtts,now-10000000);expire(&c->acked,now-500000);
|
||||
expire(&c->sent,now-500000);expire(&c->flight,now-2000000);
|
||||
expire(&c->captures,now-1000000);
|
||||
if(!c->enabled || c->ceiling<=0) return 0;
|
||||
int64_t baseline=base(c), spread=quantile(&c->rtts,now-2000000,9,10);
|
||||
int64_t jitter=spread ? spread-baseline : 0;
|
||||
c->slack=1000000/MIN(c->max_fps,fps[c->tier])+MIN(MAX(jitter*3/2,25000),80000);
|
||||
int64_t recent=quantile(&c->rtts,now-300000,1,2);
|
||||
int64_t queue=recent ? recent-baseline : 0;
|
||||
int64_t age=c->flight.n ? now-c->flight.a[0].t : 0;
|
||||
int delivered=rate(&c->acked),sending=rate(&c->sent);
|
||||
if(c->sent.n) c->frame_bytes=sending/16/c->sent.n;
|
||||
int demand=(int)MIN((int64_t)MIN(c->captures.n,MIN(c->max_fps,fps[c->tier]))*c->frame_bytes*8,2147483647);
|
||||
int signal=(sending>=c->target/2 && queue>40000)||c->held>=3||age>baseline+100000;
|
||||
int congested=signal && c->signal;c->signal=signal;c->held=0;
|
||||
if(congested && now-c->decrease>300000) {
|
||||
int next=MAX(300000,MIN((int64_t)c->target*4/5,MAX((int64_t)delivered*9/10,c->target/2)));
|
||||
if(demand>0 && (int64_t)demand*2<=(int64_t)c->target*5/4) next=MAX(next,MIN(c->target,(int64_t)demand*2));
|
||||
c->target=next;c->decrease=now;
|
||||
} else if(!congested && now-c->decrease>1000000 && now-c->increase>250000 && c->target<c->ceiling &&
|
||||
(sending>(int64_t)c->target*6/10 || now-c->decrease>3000000)) {
|
||||
c->target=MIN(c->ceiling,(int64_t)(c->target*1.1)+50000);c->increase=now;
|
||||
}
|
||||
double share=(double)c->target/MAX(c->ceiling,1);
|
||||
if(c->tier<4 && share<floors[c->tier] && sending>=(int64_t)c->target*7/10) {
|
||||
if(!c->below)c->below=now;
|
||||
if(now-c->below>500000) {
|
||||
for(c->tier=0;c->tier<4 && share<floors[c->tier];c->tier++) {}
|
||||
c->below=0;
|
||||
}
|
||||
} else c->below=0;
|
||||
if(c->tier>0 && share>floors[c->tier-1]*1.25) {
|
||||
if(!c->above)c->above=now;
|
||||
if(now-c->above>2000000) {c->tier--;c->above=0;}
|
||||
} else c->above=0;
|
||||
return c->target;
|
||||
}
|
||||
int64_t fc_value(FCController *c,int field) {
|
||||
switch(field) {
|
||||
case 0:return c->target;case 1:return c->ceiling;case 2:return c->tier;
|
||||
case 3:return MIN(c->max_fps,fps[c->tier]);case 4:return scale[c->tier];
|
||||
case 5:return base(c);case 6:return c->flight.n;case 7:return c->slack;
|
||||
default:return 0;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
/* Shared latency controller. Times are monotonic host microseconds.
|
||||
* Callers serialize access. Capture is gated BEFORE encoding: dropping an
|
||||
* encoded inter frame would break the decoder's reference chain. */
|
||||
#ifndef FRAME_CONTROLLER_H
|
||||
#define FRAME_CONTROLLER_H
|
||||
#include <stdint.h>
|
||||
#ifdef _WIN32
|
||||
#define FC_API __declspec(dllexport)
|
||||
#else
|
||||
#define FC_API
|
||||
#endif
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
typedef struct FCController FCController;
|
||||
FC_API FCController *fc_new(int fps, int enabled);
|
||||
FC_API void fc_free(FCController *c);
|
||||
FC_API void fc_ceiling(FCController *c, int bps);
|
||||
FC_API int fc_gate(FCController *c, int64_t now, int counts);
|
||||
FC_API void fc_capture(FCController *c, int64_t now);
|
||||
FC_API void fc_sent(FCController *c, uint32_t seq, int bytes, int64_t now);
|
||||
FC_API int fc_ack(FCController *c, uint32_t seq, int64_t now);
|
||||
FC_API int fc_update(FCController *c, int64_t now);
|
||||
/* target, ceiling, tier, fps, scale percent, baseline us, in flight, slack us */
|
||||
FC_API int64_t fc_value(FCController *c, int field);
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
#endif
|
||||
@@ -0,0 +1,147 @@
|
||||
/* xdg-desktop-portal RemoteDesktop + ScreenCast, one consented session per
|
||||
* panel. PipeWire receives the portal fd, never the unrestricted daemon fd.
|
||||
* Input uses only the devices granted in Start's response. */
|
||||
#ifndef _WIN32
|
||||
#include "controller.h"
|
||||
#include <gio/gio.h>
|
||||
#include <gio/gunixfdlist.h>
|
||||
#include <unistd.h>
|
||||
#include <string.h>
|
||||
#define BUS "org.freedesktop.portal.Desktop"
|
||||
#define PATH "/org/freedesktop/portal/desktop"
|
||||
#define RD "org.freedesktop.portal.RemoteDesktop"
|
||||
#define SC "org.freedesktop.portal.ScreenCast"
|
||||
typedef struct {
|
||||
GDBusConnection *bus;
|
||||
GMainContext *context;
|
||||
char *session;
|
||||
int fd;
|
||||
unsigned node, devices;
|
||||
int width,height;
|
||||
} Portal;
|
||||
typedef struct {GVariant *result;int done;unsigned code;} Response;
|
||||
static void response(GDBusConnection *c,const char *sender,const char *path,const char *iface,
|
||||
const char *signal,GVariant *params,gpointer data) {
|
||||
(void)c;(void)sender;(void)path;(void)iface;(void)signal;
|
||||
Response *r=data;g_variant_get(params,"(u@a{sv})",&r->code,&r->result);r->done=1;
|
||||
}
|
||||
static void option(GVariantBuilder *b,const char *key,GVariant *v) {g_variant_builder_add(b,"{sv}",key,v);}
|
||||
static GVariant *request(Portal *p,const char *iface,const char *method,GVariant *args,
|
||||
const char *token,char *error,int capacity) {
|
||||
char *sender=g_strdup(g_dbus_connection_get_unique_name(p->bus)+1);
|
||||
for(char *c=sender;*c;c++)if(*c=='.')*c='_';
|
||||
char *path=g_strdup_printf(PATH "/request/%s/%s",sender,token);g_free(sender);
|
||||
Response r={0};GError *e=NULL;
|
||||
guint sub=g_dbus_connection_signal_subscribe(p->bus,BUS,"org.freedesktop.portal.Request","Response",
|
||||
path,NULL,G_DBUS_SIGNAL_FLAGS_NONE,response,&r,NULL);
|
||||
GVariant *reply=g_dbus_connection_call_sync(p->bus,BUS,PATH,iface,method,args,G_VARIANT_TYPE("(o)"),
|
||||
G_DBUS_CALL_FLAGS_NONE,10000,NULL,&e);
|
||||
if(reply)g_variant_unref(reply);
|
||||
int64_t deadline=g_get_monotonic_time()+120000000;
|
||||
while(!e && !r.done && g_get_monotonic_time()<deadline) {
|
||||
while(g_main_context_iteration(p->context,FALSE)) {}
|
||||
g_usleep(10000);
|
||||
}
|
||||
if(!r.done) {
|
||||
GVariant *closed=g_dbus_connection_call_sync(p->bus,BUS,path,"org.freedesktop.portal.Request","Close",
|
||||
NULL,NULL,G_DBUS_CALL_FLAGS_NONE,2000,NULL,NULL);
|
||||
if(closed)g_variant_unref(closed);
|
||||
}
|
||||
g_dbus_connection_signal_unsubscribe(p->bus,sub);g_free(path);
|
||||
if(e || !r.done || r.code) {
|
||||
g_strlcpy(error,e?e->message:!r.done?"Screen sharing request timed out":"Screen sharing was cancelled or refused",capacity);
|
||||
if(e)g_error_free(e);if(r.result)g_variant_unref(r.result);return NULL;
|
||||
}
|
||||
return r.result;
|
||||
}
|
||||
FC_API void fc_portal_close(Portal *p) {
|
||||
if(!p)return;
|
||||
if(p->session && p->bus) {
|
||||
GVariant *r=g_dbus_connection_call_sync(p->bus,BUS,p->session,"org.freedesktop.portal.Session","Close",
|
||||
NULL,NULL,G_DBUS_CALL_FLAGS_NONE,2000,NULL,NULL);
|
||||
if(r)g_variant_unref(r);
|
||||
}
|
||||
if(p->fd>=0){close(p->fd);p->fd=-1;}
|
||||
g_free(p->session);if(p->bus)g_object_unref(p->bus);
|
||||
if(p->context)g_main_context_unref(p->context);g_free(p);
|
||||
}
|
||||
/* Each pipeline gets a fresh restricted PipeWire connection. A dup of a
|
||||
* previously consumed protocol socket is not a new connection.
|
||||
* Ownership: the Portal owns p->fd for its whole life and is its only closer.
|
||||
* pipewiresrc never takes the fd it is given: its core connects with
|
||||
* pw_context_connect_fd(ctx, fcntl(fd, F_DUPFD_CLOEXEC, 3), ...) and that
|
||||
* duplicate is what PipeWire closes on teardown (src/gst/gstpipewirecore.c,
|
||||
* unchanged from 0.3.19 through 1.x). So closing p->fd here is not a double
|
||||
* close; not closing it would leak one socket per pipeline reopen. The
|
||||
* caller closes the previous pipeline before asking for a new fd. */
|
||||
FC_API int fc_portal_refresh(Portal *p,char *error,int capacity) {
|
||||
GVariantBuilder b;g_variant_builder_init(&b,G_VARIANT_TYPE_VARDICT);
|
||||
GUnixFDList *fds=NULL;GError *e=NULL;
|
||||
GVariant *r=g_dbus_connection_call_with_unix_fd_list_sync(p->bus,BUS,PATH,SC,"OpenPipeWireRemote",
|
||||
g_variant_new("(oa{sv})",p->session,&b),G_VARIANT_TYPE("(h)"),G_DBUS_CALL_FLAGS_NONE,10000,NULL,&fds,NULL,&e);
|
||||
if(!r){g_strlcpy(error,e->message,capacity);g_error_free(e);return -1;}
|
||||
int handle;g_variant_get(r,"(h)",&handle);g_variant_unref(r);
|
||||
int fd=g_unix_fd_list_get(fds,handle,&e);g_object_unref(fds);
|
||||
if(fd<0){g_strlcpy(error,e->message,capacity);g_error_free(e);return -1;}
|
||||
if(p->fd>=0)close(p->fd);
|
||||
p->fd=fd;return fd;
|
||||
}
|
||||
FC_API Portal *fc_portal_select(char *error,int capacity) {
|
||||
Portal *p=g_new0(Portal,1);p->fd=-1;p->context=g_main_context_new();
|
||||
g_main_context_push_thread_default(p->context);
|
||||
GError *e=NULL;GVariant *r=NULL;
|
||||
p->bus=g_bus_get_sync(G_BUS_TYPE_SESSION,NULL,&e);
|
||||
if(!p->bus) {g_strlcpy(error,e->message,capacity);g_error_free(e);goto fail;}
|
||||
GVariantBuilder b;char token[64],session[64];
|
||||
g_snprintf(session,sizeof(session),"fc_%u",g_random_int());
|
||||
g_snprintf(token,sizeof(token),"fc_%u",g_random_int());
|
||||
g_variant_builder_init(&b,G_VARIANT_TYPE_VARDICT);
|
||||
option(&b,"handle_token",g_variant_new_string(token));option(&b,"session_handle_token",g_variant_new_string(session));
|
||||
r=request(p,RD,"CreateSession",g_variant_new("(a{sv})",&b),token,error,capacity);if(!r)goto fail;
|
||||
g_variant_lookup(r,"session_handle","s",&p->session);g_variant_unref(r);r=NULL;
|
||||
if(!p->session){g_strlcpy(error,"Portal did not return a session",capacity);goto fail;}
|
||||
g_snprintf(token,sizeof(token),"fc_%u",g_random_int());g_variant_builder_init(&b,G_VARIANT_TYPE_VARDICT);
|
||||
option(&b,"handle_token",g_variant_new_string(token));option(&b,"types",g_variant_new_uint32(3));
|
||||
r=request(p,RD,"SelectDevices",g_variant_new("(oa{sv})",p->session,&b),token,error,capacity);if(!r)goto fail;g_variant_unref(r);
|
||||
g_snprintf(token,sizeof(token),"fc_%u",g_random_int());g_variant_builder_init(&b,G_VARIANT_TYPE_VARDICT);
|
||||
option(&b,"handle_token",g_variant_new_string(token));option(&b,"types",g_variant_new_uint32(3));
|
||||
option(&b,"multiple",g_variant_new_boolean(FALSE));option(&b,"cursor_mode",g_variant_new_uint32(2));
|
||||
r=request(p,SC,"SelectSources",g_variant_new("(oa{sv})",p->session,&b),token,error,capacity);if(!r)goto fail;g_variant_unref(r);
|
||||
g_snprintf(token,sizeof(token),"fc_%u",g_random_int());g_variant_builder_init(&b,G_VARIANT_TYPE_VARDICT);
|
||||
option(&b,"handle_token",g_variant_new_string(token));
|
||||
r=request(p,RD,"Start",g_variant_new("(osa{sv})",p->session,"",&b),token,error,capacity);if(!r)goto fail;
|
||||
g_variant_lookup(r,"devices","u",&p->devices);
|
||||
GVariant *streams=g_variant_lookup_value(r,"streams",G_VARIANT_TYPE("a(ua{sv})"));
|
||||
if(streams && g_variant_n_children(streams)>0) {
|
||||
GVariant *entry=g_variant_get_child_value(streams,0),*props=NULL;
|
||||
g_variant_get(entry,"(u@a{sv})",&p->node,&props);
|
||||
g_variant_lookup(props,"size","(ii)",&p->width,&p->height);
|
||||
g_variant_unref(props);g_variant_unref(entry);
|
||||
}
|
||||
if(streams)g_variant_unref(streams);g_variant_unref(r);r=NULL;
|
||||
if(!p->node || p->width<=0 || p->height<=0) {g_strlcpy(error,"Portal returned no stream size",capacity);goto fail;}
|
||||
if(fc_portal_refresh(p,error,capacity)<0)goto fail;
|
||||
g_main_context_pop_thread_default(p->context);return p;
|
||||
fail:
|
||||
g_main_context_pop_thread_default(p->context);fc_portal_close(p);return NULL;
|
||||
}
|
||||
FC_API int fc_portal_value(Portal *p,int field) {
|
||||
switch(field){case 0:return p->fd;case 1:return (int)p->node;case 2:return p->width;
|
||||
case 3:return p->height;case 4:return (int)p->devices;default:return 0;}
|
||||
}
|
||||
FC_API int fc_portal_input(Portal *p,int type,double x,double y,int code,int down) {
|
||||
const char *method=NULL;GVariant *args=NULL;GVariantBuilder b;g_variant_builder_init(&b,G_VARIANT_TYPE_VARDICT);
|
||||
if(type==0 && (p->devices&2)) {
|
||||
method="NotifyPointerMotionAbsolute";args=g_variant_new("(oa{sv}udd)",p->session,&b,p->node,x,y);
|
||||
} else if(type==1 && (p->devices&2)) {
|
||||
method="NotifyPointerButton";args=g_variant_new("(oa{sv}iu)",p->session,&b,code,(guint)down);
|
||||
} else if(type==2 && (p->devices&2)) {
|
||||
method="NotifyPointerAxis";args=g_variant_new("(oa{sv}dd)",p->session,&b,x,y);
|
||||
} else if(type==3 && (p->devices&1)) {
|
||||
method="NotifyKeyboardKeysym";args=g_variant_new("(oa{sv}iu)",p->session,&b,code,(guint)down);
|
||||
} else return 0;
|
||||
GError *e=NULL;GVariant *r=g_dbus_connection_call_sync(p->bus,BUS,PATH,RD,method,args,NULL,
|
||||
G_DBUS_CALL_FLAGS_NONE,2000,NULL,&e);
|
||||
if(e)g_error_free(e);if(r)g_variant_unref(r);return r!=NULL;
|
||||
}
|
||||
#endif
|
||||
@@ -0,0 +1,199 @@
|
||||
# PC in the headset
|
||||
|
||||
Windows and Linux hosts use **Tools → PC in the headset**. The host shares a
|
||||
window or screen, and the existing Frame viewer makes it a SteamVR panel.
|
||||
Move it with the dashboard's Float in World, Move and Size controls.
|
||||
|
||||
**Inferred / not yet verified on a desktop host:** the Windows and Linux
|
||||
capture and input paths below. This work was developed on a Mac with no
|
||||
Windows or Linux desktop VM. A native build or test-pattern test in CI does
|
||||
not establish that desktop capture, a permission dialog, hardware encoding
|
||||
or laser input works. Keep this feature in the draft/testing stage until
|
||||
those paths have been tried on real hosts.
|
||||
|
||||
## Own implementation, platform APIs and bundled libraries
|
||||
|
||||
Frame Control owns the host agent, input routing, authentication, streaming
|
||||
protocol, panel launcher and adaptation. It does not launch or require
|
||||
Sunshine, OBS or another desktop-streaming app. GStreamer and its codec
|
||||
plugins are ordinary libraries bundled with the Windows and Linux app;
|
||||
users do not install a GStreamer application. The shared library build keeps
|
||||
license texts and package provenance alongside the libraries. Linux also
|
||||
bundles the PipeWire client’s dynamically loaded SPA/protocol modules and a
|
||||
private client configuration; it does not change the desktop’s configuration.
|
||||
|
||||
First-party alternatives considered (**documented**): Valve Remote Play
|
||||
streams a game/desktop, rather than providing this per-window panel protocol;
|
||||
Windows Remote Desktop opens a remote session; Linux's desktop portal is the
|
||||
consent mechanism for sharing the current desktop. The chosen paths are:
|
||||
|
||||
| Host | Capture | Encoding | Input |
|
||||
|---|---|---|---|
|
||||
| Windows | Windows.Graphics.Capture, through `d3d11screencapturesrc capture-api=wgc`; HWND or HMONITOR | Hardware Media Foundation (`mfh264enc`), low latency, no B-frames | `SendInput`, with source bounds and per-monitor DPI awareness |
|
||||
| Linux | RemoteDesktop + ScreenCast portal, then the returned PipeWire fd/node | VA-API (`vah264enc`) where registered; x264 otherwise | RemoteDesktop portal notifications, using only granted pointer/keyboard devices |
|
||||
|
||||
API choices are **documented**, not device verification:
|
||||
[Windows capture](https://learn.microsoft.com/en-us/windows/uwp/audio-video-camera/screen-capture),
|
||||
[GStreamer WGC](https://gstreamer.freedesktop.org/documentation/d3d11/d3d11screencapturesrc.html),
|
||||
[Media Foundation encoder](https://gstreamer.freedesktop.org/documentation/mediafoundation/mfh264enc.html),
|
||||
[ScreenCast portal](https://flatpak.github.io/xdg-desktop-portal/docs/doc-org.freedesktop.portal.ScreenCast.html),
|
||||
[RemoteDesktop portal](https://flatpak.github.io/xdg-desktop-portal/docs/doc-org.freedesktop.portal.RemoteDesktop.html).
|
||||
|
||||
Windows needs a WGC-capable Windows 10/11 desktop and an available hardware
|
||||
Media Foundation H.264 encoder. Elevated windows and the secure desktop
|
||||
cannot be driven by an ordinary Frame Control process. If Windows refuses
|
||||
to focus a selected window, input is refused rather than sent to the app
|
||||
covering it; bring the selected window forward, then Stop and Show again. Minimized/closed
|
||||
windows may stop producing frames. Protected content is not supported.
|
||||
|
||||
On Linux, press **Choose a window or screen…** and approve the desktop's
|
||||
sharing dialog. Choose another source to add another panel. Stop releases
|
||||
that source's portal session; sharing it again asks for consent again.
|
||||
A desktop must implement both ScreenCast and RemoteDesktop for this path;
|
||||
a ScreenCast-only compositor cannot provide laser input through this API.
|
||||
Cancelling or denying a dialog is reported on the card. No portal permission
|
||||
is bypassed, and Frame Control does not open `/dev/uinput` or the unrestricted
|
||||
PipeWire daemon on the host.
|
||||
|
||||
The host's own keyboard still works. Input from the viewer uses normalized
|
||||
picture coordinates, maps through the selected source's bounds, and releases
|
||||
held buttons/keys on blur, disconnect and Stop. Linux requires the pointer
|
||||
and keyboard grants. **Untested:** desktop-specific consent, mixed-DPI
|
||||
Windows input alignment, multi-monitor layouts, hardware encoder behavior,
|
||||
window resize/minimize, and non-US keyboard layouts.
|
||||
|
||||
## Shared pieces
|
||||
|
||||
- `ui/frame_macview.py` owns the SSH tunnel, reconnect supervision, quality
|
||||
presets and panel launch for all hosts. `ui/frame_pcview.py` selects the PC
|
||||
helper; `/api/macview` remains the compatible endpoint.
|
||||
- `ui/mac-view.html` is the one viewer. The 17-byte big-endian frame header,
|
||||
Annex-B H.264/JPEG payloads, `hello`/`ack` reconnect handshake, clock sync,
|
||||
`rx`/`fd` timing reports and input messages are unchanged.
|
||||
- `desktop/controller.c` is the rate controller shared by the Mac Swift
|
||||
binding and the PC Python binding. Capture is gated **before** encoding;
|
||||
encoded reference frames are never discarded. A bounded raw-frame queue
|
||||
keeps the newest picture, including the last update of an idle window,
|
||||
until the gate opens. A native one-frame-source test covers that case.
|
||||
It keeps the Mac's bitrate demand protection and tier hysteresis.
|
||||
- PC records use the existing `Stats.swift` JSON schema, with bounded
|
||||
4096-frame/512-input storage in `ui/frame_stream_stats.py`. The benchmark's
|
||||
analysis, targets and network shaping are shared, not reimplemented.
|
||||
Capture timestamps describe the native pipeline's source time; they do
|
||||
not prove the time at which the host compositor displayed the pixels.
|
||||
- Mac virtual-display separation remains Mac-only. Windows WGC and the
|
||||
Linux portal share the selected window directly.
|
||||
|
||||
PC capture follows the shared frame-rate and resolution tiers. x264 updates
|
||||
bitrate while running. Hardware encoders are drained and reopened when the
|
||||
budget changes materially, at most once a second, because their live property
|
||||
support varies. Reopening starts a new keyframe and retains the consented
|
||||
portal session. **Untested:** hardware reconfiguration latency and whether a
|
||||
particular desktop permits reconnecting its PipeWire stream this way.
|
||||
|
||||
## Build and measure
|
||||
|
||||
Packaged Windows/Linux builds include `desktop/bundle/pc-host` and its shared
|
||||
libraries. Source checkouts build them with `python3 desktop/build.py` after
|
||||
installing GStreamer development packages (see the `PC host libraries` CI
|
||||
workflow). The feature reports a missing bundle; it does not download or
|
||||
install a streaming app on first use.
|
||||
|
||||
The existing benchmark now accepts a PC host:
|
||||
|
||||
```sh
|
||||
python3 scripts/macview-bench.py run --pc --scenario test --label pc-test
|
||||
# Linux: select a real source in the desktop's sharing dialog
|
||||
python3 scripts/macview-bench.py run --pc --scenario capture --source choose --label linux-window
|
||||
# Windows: use the HWND/monitor source ID shown by the host's /windows or /displays
|
||||
python3 scripts/macview-bench.py run --pc --scenario capture --source window:12345 --label windows-window
|
||||
```
|
||||
|
||||
The synthetic PC pattern uses bundled x264 so headless CI can verify the
|
||||
wire protocol without claiming that a GPU was exercised. The `capture`
|
||||
scenario measures the selected real source without injecting input or
|
||||
assuming that it animates at 60 fps. Mac-only Chrome/virtual-display typing
|
||||
and scrolling automation is not run on PC hosts. Results retain the same
|
||||
latency stages and record `host_platform`, `pc_host` and `source`. CPU sampling
|
||||
on PC hosts is explicitly unavailable. `--net` and `--delay` still use the
|
||||
same bounded shaping relay, without administrator privileges.
|
||||
|
||||
## Evidence
|
||||
|
||||
- **Verified, Mac, 2026-09-28:** the final full unit/integration suite ran
|
||||
183 tests successfully, including the real Mac helper's H.264, ticket, timing
|
||||
and input-echo tests. The native PC test class was skipped locally because
|
||||
its libraries were absent; the PC adapter and shared-controller tests passed.
|
||||
- **Verified, real Frame, 2026-09-28, BUILD_ID 20260925.6191901:** the base
|
||||
helper's synthetic source created panel `valve.steam.desktopgame.2001639889`,
|
||||
and the shared Chromium viewer decoded H.264. It recorded 286 frames over
|
||||
the short probe, with a two-second summary of 19.5 fps shown and total
|
||||
latency p50/p95 83/156.5 ms. This establishes the existing viewer/transport
|
||||
route, not Windows/Linux capture, input or a latency target. The probe's
|
||||
helper, tunnel and viewer were stopped afterward.
|
||||
- **Verified in CI:** native library builds and real x264 protocol tests passed
|
||||
on Windows, Ubuntu x64 and Ubuntu ARM64 in
|
||||
[run 36422214445](https://github.com/saphid/frame-control/actions/runs/36422214445).
|
||||
All four installer builds passed in
|
||||
[run 36422214425](https://github.com/saphid/frame-control/actions/runs/36422214425).
|
||||
These are build/synthetic tests, not desktop-host verification. No VM was used.
|
||||
- **Verified, real Frame, same build/date:** the ARM64 PC agent and its
|
||||
bundled libraries ran from a temporary user directory, using x264's moving
|
||||
test pattern. Traffic travelled Frame → Mac SSH relay → Frame viewer.
|
||||
The shared bench recorded 351 drawn frames, content p50/p95 15.1/80.3 ms,
|
||||
and 34.8 fps. The frame-rate and late-frame targets failed. This checks the
|
||||
new agent and real viewer together; it is not a representative PC link.
|
||||
The helper exited 0, its viewer/tunnels stopped, and its directory was removed.
|
||||
[Raw benchmark result](../bench/results/2026-09-28-c5fc552-dirty-pc-agent-frame-arm64-hairpin.json).
|
||||
This first probe's input echo measured message receipt to the next capture,
|
||||
not a visible pattern response; later builds make test clicks change its color.
|
||||
- **Verified, Mac:** all 600 states in a 60-second congestion/recovery trace
|
||||
matched the original Swift controller. `tests/test_pc_controller.py` retains
|
||||
the original trace digest as a regression check.
|
||||
- **Verified, real Frame, agent at `cd20243`:** the repeated synthetic
|
||||
probe drew 400 frames at 39.3 fps, content p50/p95 15.1/28.6 ms, and synthetic
|
||||
input-to-drawn p50 76.6 ms. Test clicks now change the pattern color before
|
||||
injection is timestamped. The frame-rate/late-frame targets still failed;
|
||||
this remains a Frame-hosted x264 test through a Mac relay, not a desktop or
|
||||
physical-laser measurement. Helper exit 0 and cleanup succeeded.
|
||||
[Latest device probe result](../bench/results/2026-09-28-cd20243-pc-agent-frame-arm64-final.json).
|
||||
- **Untested:** real Windows WGC → Media Foundation → Frame; real Linux
|
||||
portal → PipeWire → VA-API/x264 → Frame; physical laser input on either.
|
||||
No benchmark numbers for those desktop paths are claimed.
|
||||
|
||||
## Independent review availability
|
||||
|
||||
A direct read-only review was attempted with
|
||||
`devin -p --model swe-2-max --permission-mode auto --prompt-file …`. The first
|
||||
attempt exited 0 after rejecting a tool that needed interactive permission;
|
||||
it did not inspect the diff. A full inline-diff attempt returned no output
|
||||
for 15 minutes and was terminated (shell exit 143). A smaller inline native
|
||||
code review returned no output within 300 seconds (process exit -15).
|
||||
SWE-2 Max was requested; no completed review or findings were received, so
|
||||
independent review is **unverified**, not a passed check. The PR remains draft.
|
||||
|
||||
## Device follow-up, 2026-09-29
|
||||
|
||||
**Verified, read-only, SteamOS BUILD_ID 20260925.6191901:** the Frame was
|
||||
reachable, charging (44% initially, 63% at the end), and SteamVR reported
|
||||
activity level 3 (standby). No Plasma desktop was running. Introspection of
|
||||
the active `org.freedesktop.portal.Desktop` service exposed ScreenCast with
|
||||
source types 3, but no RemoteDesktop interface. **Inferred:** this gamescope
|
||||
session cannot provide this feature's required consented input path; a
|
||||
ScreenCast-only session is insufficient.
|
||||
|
||||
**Blocked, not a device execution result:** the shared `/tmp/frame-test.lock`
|
||||
was held by another thread. Ten lock-acquisition attempts over five minutes
|
||||
all failed, after earlier preparation-time attempts also found it occupied.
|
||||
The current `a992f6d` ARM64 CI bundle was downloaded to the Mac, but nothing
|
||||
was installed, launched or stopped on the Frame. No new screenshots or
|
||||
current-revision device timings were obtained. The lock was not removed and
|
||||
no global settings were changed. The earlier synthetic results above remain
|
||||
valid only for their named revisions; standby visibility, physical laser
|
||||
input and worn-headset performance remain unverified.
|
||||
|
||||
**Verified, Mac:** a fresh full suite ran 309 tests successfully, with one
|
||||
native-PC class skipped. Current-revision CI passed
|
||||
[native hosts](https://github.com/saphid/frame-control/actions/runs/36504546909),
|
||||
[installers](https://github.com/saphid/frame-control/actions/runs/36504546800)
|
||||
and [general checks](https://github.com/saphid/frame-control/actions/runs/36504546887).
|
||||
@@ -149,6 +149,17 @@ refuses ids with a hyphen (`missing/invalid arguments`), which the fake had
|
||||
accepted. The fake now refuses them the same way, and Frame Control makes ids
|
||||
Steam accepts.
|
||||
|
||||
## PC panel host tests
|
||||
|
||||
`tests/test_pcview.py` checks source-bound tickets, reconnect/Stop revocation,
|
||||
input release, source-coordinate mapping and use of the existing panel
|
||||
launcher. The `PC host libraries` workflow builds the bundled native adapter
|
||||
on Windows, Ubuntu x64 and Ubuntu ARM64, then runs a real x264 synthetic
|
||||
stream through the HTTP/WebSocket agent. `FRAME_PC_REQUIRE_NATIVE=1` makes a
|
||||
missing native bundle fail CI instead of skipping. These tests do not grant a
|
||||
portal dialog, capture a desktop, exercise a hardware encoder or validate
|
||||
laser alignment. See [PC host evidence](pc-in-headset.md#evidence).
|
||||
|
||||
## Owned media player
|
||||
|
||||
`tests/test_media.py` covers layout evidence and overrides, OU eye ordering,
|
||||
|
||||
@@ -26,3 +26,5 @@
|
||||
- (instancetype)initWithDescriptor:(CGVirtualDisplayDescriptor *)descriptor;
|
||||
- (BOOL)applySettings:(CGVirtualDisplaySettings *)settings;
|
||||
@end
|
||||
|
||||
#include "../../../desktop/controller.h"
|
||||
@@ -1,234 +1,45 @@
|
||||
// Adapts each stream to its network, latency first. The viewer acknowledges
|
||||
// every frame as it arrives ("rx"); from those acks the controller knows how
|
||||
// long frames take to get through and how fast the link delivers them.
|
||||
//
|
||||
// - The gate: a new frame is sent only while the oldest unacknowledged one is
|
||||
// younger than the path's usual round trip plus a little slack. So frames
|
||||
// never queue up in SSH, TCP or the Wi-Fi driver; while the link is stuck
|
||||
// the newest picture waits and goes out as soon as it moves again.
|
||||
// - The bitrate: when frames start queueing (the round trip grows) or the
|
||||
// gate has to hold frames back, it drops to a bit under what the link
|
||||
// actually delivered; once things are clear it probes up again slowly, never
|
||||
// above the quality setting's bitrate (the ceiling).
|
||||
// - The tier: as the bitrate falls, fewer frames per second (60, 45, 30), then
|
||||
// a smaller picture (75%, then 50% of the panel's pixels).
|
||||
// See docs/mac-in-headset.md ("Adapting to the network"). Thread-safe.
|
||||
// Thin, serialized binding to the same controller used by the PC host.
|
||||
// Algorithm and bounded state live in desktop/controller.c.
|
||||
import Foundation
|
||||
|
||||
final class RateController {
|
||||
struct Tier: Equatable {
|
||||
let fps: Int
|
||||
let scale: Double // of the picture's long side
|
||||
}
|
||||
|
||||
static let tiers = [Tier(fps: 60, scale: 1), Tier(fps: 45, scale: 1), Tier(fps: 30, scale: 1),
|
||||
Tier(fps: 30, scale: 0.75), Tier(fps: 30, scale: 0.5)]
|
||||
/// A tier is used while the target bitrate is at least this share of the ceiling.
|
||||
static let floors = [0.45, 0.28, 0.16, 0.08, 0]
|
||||
static let enabled = ProcessInfo.processInfo.environment["FRAME_MAC_VIEW_ADAPT"] != "0"
|
||||
|
||||
let maxFps: Int
|
||||
private let lock = NSLock()
|
||||
private var ceiling = 0 // bits/s at full size and frame rate
|
||||
private(set) var target = 0
|
||||
private(set) var tier = 0
|
||||
private var unacked: [(seq: UInt32, sent: Int64, bytes: Int)] = []
|
||||
/// Round trips (send -> ack arrives here), for the baseline: the lowest
|
||||
/// in the last 10 s is the path without any queue.
|
||||
private var rtts: [(t: Int64, v: Int64)] = []
|
||||
private var acked: [(t: Int64, bytes: Int)] = [] // the last second
|
||||
private var sentLog: [(t: Int64, bytes: Int)] = []
|
||||
private var captures: [Int64] = [] // the last second
|
||||
private var frameBytes = 0 // average recent frame, kept while the gate holds everything back
|
||||
private var held = 0 // frames the gate held back since the last update
|
||||
private var lastSignal = false
|
||||
private var sawAck = false
|
||||
private var lastDecrease: Int64 = 0
|
||||
private var lastIncrease: Int64 = 0
|
||||
private var belowSince: Int64 = 0, aboveSince: Int64 = 0
|
||||
/// What changed, for the timeline: (time, event).
|
||||
private(set) var events: [(Int64, String)] = []
|
||||
|
||||
init(maxFps: Int) { self.maxFps = maxFps }
|
||||
|
||||
private let core: OpaquePointer
|
||||
private var events: [[String: Any]] = []
|
||||
init(maxFps: Int) {
|
||||
self.maxFps = maxFps
|
||||
core = fc_new(Int32(maxFps), Self.enabled ? 1 : 0)!
|
||||
}
|
||||
deinit { fc_free(core) }
|
||||
private func locked<T>(_ f: () -> T) -> T { lock.lock(); defer { lock.unlock() }; return f() }
|
||||
|
||||
/// The quality setting's bitrate at full size; the first call also starts there.
|
||||
func setCeiling(_ bps: Int) {
|
||||
locked {
|
||||
if target == 0 || target > bps { target = bps }
|
||||
ceiling = bps
|
||||
}
|
||||
}
|
||||
|
||||
var fps: Int { locked { fpsLocked } }
|
||||
private var fpsLocked: Int { min(maxFps, RateController.tiers[tier].fps) }
|
||||
var scale: Double { locked { RateController.tiers[tier].scale } }
|
||||
var baseRtt: Int64 { locked { baseline() } }
|
||||
|
||||
private func baseline() -> Int64 { rtts.map(\.v).min() ?? 0 }
|
||||
|
||||
/// How late a frame may be before the gate holds the next one: one frame
|
||||
/// interval, plus room for the jitter this link normally has (1.5 times
|
||||
/// its recent spread), so ordinary Wi-Fi jitter doesn't cost frames but a
|
||||
/// real queue does. Updated in update().
|
||||
private var slack: Int64 = 40_000
|
||||
|
||||
/// Whether a frame may be sent now without queueing behind earlier ones.
|
||||
/// `counts`: a held frame is a sign of congestion (not when merely
|
||||
/// re-checking whether a held frame can go yet).
|
||||
func maySend(now: Int64, counts: Bool = true) -> Bool {
|
||||
locked {
|
||||
guard RateController.enabled, sawAck else { return true } // not heard from the viewer yet
|
||||
// Unacknowledged for 2 s: gone with a reconnection, not in a queue.
|
||||
unacked.removeAll { now - $0.sent > 2_000_000 }
|
||||
guard let oldest = unacked.first else { return true }
|
||||
// Age is what bounds latency. The count only stops a burst, and it
|
||||
// allows a full round trip of frames, so a long but clear path
|
||||
// (100 ms away) still gets every frame.
|
||||
let interval = Int64(1_000_000 / max(1, fpsLocked))
|
||||
let window = max(3, Int((baseline() + slack) / interval) + 1)
|
||||
if unacked.count < window, now - oldest.sent <= baseline() + slack { return true }
|
||||
if counts { held += 1 }
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
/// A picture was captured (sent or not): with the frame sizes, what this
|
||||
/// stream would send if the link allowed.
|
||||
func captured(at t: Int64) {
|
||||
locked {
|
||||
captures.append(t)
|
||||
if captures.count > 256 { captures.removeFirst(captures.count - 256) }
|
||||
}
|
||||
}
|
||||
|
||||
func sent(seq: UInt32, bytes: Int, at t: Int64) {
|
||||
locked {
|
||||
// Bounded even if the viewer never acknowledges (an old viewer, or
|
||||
// the controller is off).
|
||||
unacked.append((seq, t, bytes))
|
||||
if unacked.count > 512 { unacked.removeFirst(unacked.count - 512) }
|
||||
sentLog.append((t, bytes))
|
||||
if sentLog.count > 1024 { sentLog.removeFirst(sentLog.count - 1024) }
|
||||
}
|
||||
}
|
||||
|
||||
/// The viewer has frame `seq`. Returns true if that may let a held frame go.
|
||||
func acked(seq: UInt32, at now: Int64) -> Bool {
|
||||
locked {
|
||||
sawAck = true
|
||||
guard let i = unacked.firstIndex(where: { $0.seq == seq }) else { return false }
|
||||
let f = unacked[i]
|
||||
unacked.removeSubrange(0...i) // TCP delivers in order: earlier ones arrived too
|
||||
rtts.append((now, now - f.sent))
|
||||
acked.append((now, f.bytes))
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
/// Called every 100 ms. Returns the new bitrate target, or nil if the
|
||||
/// controller is off.
|
||||
private func value(_ field: Int32) -> Int { Int(fc_value(core, field)) }
|
||||
var target: Int { locked { value(0) } }
|
||||
var tier: Int { locked { value(2) } }
|
||||
var fps: Int { locked { value(3) } }
|
||||
var scale: Double { locked { Double(value(4)) / 100 } }
|
||||
var baseRtt: Int64 { locked { fc_value(core, 5) } }
|
||||
func setCeiling(_ bps: Int) { locked { fc_ceiling(core, Int32(bps)) } }
|
||||
func maySend(now: Int64, counts: Bool = true) -> Bool { locked { fc_gate(core, now, counts ? 1 : 0) != 0 } }
|
||||
func captured(at t: Int64) { locked { fc_capture(core, t) } }
|
||||
func sent(seq: UInt32, bytes: Int, at t: Int64) { locked { fc_sent(core, seq, Int32(bytes), t) } }
|
||||
func acked(seq: UInt32, at t: Int64) -> Bool { locked { fc_ack(core, seq, t) != 0 } }
|
||||
func update(now: Int64) -> Int? {
|
||||
locked {
|
||||
rtts.removeAll { now - $0.t > 10_000_000 }
|
||||
acked.removeAll { now - $0.t > 500_000 }
|
||||
sentLog.removeAll { now - $0.t > 500_000 }
|
||||
unacked.removeAll { now - $0.sent > 2_000_000 }
|
||||
captures.removeAll { now - $0 > 1_000_000 }
|
||||
guard RateController.enabled, ceiling > 0 else { return nil }
|
||||
let base = baseline()
|
||||
let spread = rtts.filter { now - $0.t < 2_000_000 }.map(\.v).sorted()
|
||||
let jitter = spread.isEmpty ? 0 : spread[spread.count * 9 / 10] - base
|
||||
let interval = Int64(1_000_000 / max(1, fpsLocked))
|
||||
slack = interval + min(max(jitter * 3 / 2, 25_000), 80_000)
|
||||
let recent = rtts.filter { now - $0.t < 300_000 }.map(\.v).sorted()
|
||||
let queueing = recent.isEmpty ? 0 : recent[recent.count / 2] - base
|
||||
let oldestAge = unacked.first.map { now - $0.sent } ?? 0
|
||||
let stuck = oldestAge > base + 100_000
|
||||
let delivered = acked.reduce(0) { $0 + $1.bytes } * 16 // bits/s over the last half second
|
||||
let sending = sentLog.reduce(0) { $0 + $1.bytes } * 16
|
||||
// Demand: captures per second (up to the tier's rate) times the
|
||||
// average frame. A test card or a mostly still window wants far
|
||||
// less than its budget; when the link hiccups, cutting its bitrate
|
||||
// can't help, and it would only look link-limited afterwards.
|
||||
if !sentLog.isEmpty { frameBytes = sentLog.reduce(0) { $0 + $1.bytes } / sentLog.count }
|
||||
let demand = min(captures.count, fpsLocked) * frameBytes * 8
|
||||
// Delay alone isn't our queue: Wi-Fi jitters by itself. It only
|
||||
// counts while this stream uses a good part of its budget (so its
|
||||
// own data could be what's queueing). Frames the gate had to hold,
|
||||
// or one stuck in flight, show demand the link isn't carrying
|
||||
// whatever was sent (the gate itself keeps what's sent low).
|
||||
let busy = sending >= target / 2
|
||||
// Twice in a row (200 ms), so one late ack doesn't count.
|
||||
let signal = (busy && queueing > 40_000) || held >= 3 || stuck
|
||||
let congested = signal && lastSignal
|
||||
lastSignal = signal
|
||||
let heldNow = held
|
||||
held = 0
|
||||
let floorBps = 300_000
|
||||
if congested, now - lastDecrease > 300_000 {
|
||||
// Down to a bit under what got through: at least a fifth off, at
|
||||
// most half (a stall delivers nothing, but the link is still there).
|
||||
let measured = Int(Double(delivered) * 0.9)
|
||||
var next = max(floorBps, min(target * 4 / 5, max(measured, target / 2)))
|
||||
// App-limited (it wants about half its budget or less): never
|
||||
// below twice what it wants, however many cuts in a row. The
|
||||
// extra quarter is hysteresis, so frame sizes wobbling at the
|
||||
// floor don't switch the protection off.
|
||||
if demand > 0, demand * 2 <= target * 5 / 4 { next = max(next, min(target, demand * 2)) }
|
||||
target = next
|
||||
lastDecrease = now
|
||||
events.append((now, "down to \(target / 1000) kbit/s: queue \(queueing / 1000) ms, held \(heldNow), "
|
||||
+ "oldest \(oldestAge / 1000) ms, base \(base / 1000) ms, sent \(sending / 1000) got \(delivered / 1000) "
|
||||
+ "wants \(demand / 1000)"))
|
||||
} else if !congested, now - lastDecrease > 1_000_000, now - lastIncrease > 250_000, target < ceiling,
|
||||
sending > target * 6 / 10 || now - lastDecrease > 3_000_000 {
|
||||
// Clear for a second and using what it has: probe up.
|
||||
target = min(ceiling, Int(Double(target) * 1.1) + 50_000)
|
||||
lastIncrease = now
|
||||
let old = value(0), tier = value(2)
|
||||
let result = Int(fc_update(core, now))
|
||||
if value(0) < old || value(2) != tier {
|
||||
events.append(["t": now, "e": "target \(value(0)) bit/s; tier \(value(2))"])
|
||||
if events.count > 200 { events.removeFirst() }
|
||||
}
|
||||
// Fewer frames or pixels only help a stream that fills its budget;
|
||||
// a small one (a still window, a test card) keeps its tier.
|
||||
retier(now: now, linkLimited: sending >= target * 7 / 10)
|
||||
if events.count > 200 { events.removeFirst(events.count - 200) }
|
||||
return target
|
||||
return result == 0 ? nil : result
|
||||
}
|
||||
}
|
||||
|
||||
/// Steps down quickly, straight to the tier the bitrate supports, and back
|
||||
/// up one tier at a time only when there's clearly room (hysteresis).
|
||||
private func retier(now: Int64, linkLimited: Bool) {
|
||||
let share = Double(target) / Double(max(ceiling, 1))
|
||||
if tier < RateController.tiers.count - 1, share < RateController.floors[tier], linkLimited {
|
||||
if belowSince == 0 { belowSince = now }
|
||||
if now - belowSince > 500_000 {
|
||||
tier = RateController.floors.firstIndex { share >= $0 } ?? RateController.tiers.count - 1
|
||||
belowSince = 0
|
||||
events.append((now, "tier \(tier)"))
|
||||
}
|
||||
} else {
|
||||
belowSince = 0
|
||||
}
|
||||
if tier > 0, share > RateController.floors[tier - 1] * 1.25 {
|
||||
if aboveSince == 0 { aboveSince = now }
|
||||
if now - aboveSince > 2_000_000 {
|
||||
tier -= 1
|
||||
aboveSince = 0
|
||||
events.append((now, "tier \(tier)"))
|
||||
}
|
||||
} else {
|
||||
aboveSince = 0
|
||||
}
|
||||
}
|
||||
|
||||
func state() -> [String: Any] {
|
||||
locked {
|
||||
["target": target, "ceiling": ceiling, "tier": tier, "fps": min(maxFps, RateController.tiers[tier].fps),
|
||||
"scale": RateController.tiers[tier].scale, "baseRtt": Double(baseline()) / 1000,
|
||||
"inFlight": unacked.count, "slack": Double(slack) / 1000, "adapt": RateController.enabled]
|
||||
}
|
||||
locked { ["target": value(0), "ceiling": value(1), "tier": value(2), "fps": value(3),
|
||||
"scale": Double(value(4)) / 100, "baseRtt": Double(value(5)) / 1000,
|
||||
"inFlight": value(6), "slack": Double(value(7)) / 1000, "adapt": Self.enabled] }
|
||||
}
|
||||
|
||||
func eventList() -> [[String: Any]] { locked { events.map { ["t": $0.0, "e": $0.1] } } }
|
||||
func eventList() -> [[String: Any]] { locked { events } }
|
||||
}
|
||||
@@ -5,7 +5,10 @@ set -eu
|
||||
here=$(cd "$(dirname "$0")" && pwd)
|
||||
out=${1:-$here/../bin/frame-mac-view}
|
||||
mkdir -p "$(dirname "$out")"
|
||||
obj=$(mktemp /tmp/frame-controller.XXXXXX)
|
||||
trap 'rm -f "$obj"' EXIT
|
||||
xcrun clang -O2 -target arm64-apple-macos14.0 -c "$here/../../desktop/controller.c" -o "$obj"
|
||||
xcrun swiftc -O -swift-version 5 -target arm64-apple-macos14.0 \
|
||||
-import-objc-header "$here/Sources/CGVirtualDisplay.h" \
|
||||
-o "$out" "$here"/Sources/*.swift
|
||||
-o "$out" "$here"/Sources/*.swift "$obj"
|
||||
echo "built $out"
|
||||
+55
-13
@@ -55,6 +55,7 @@ from urllib.parse import quote, urlencode # %20, not +: the agent's URLComponen
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
sys.path.insert(0, str(ROOT / "ui"))
|
||||
import frame_macview # noqa: E402
|
||||
import frame_pcview # noqa: E402
|
||||
|
||||
RESULTS = ROOT / "bench" / "results"
|
||||
PAGES = ROOT / "bench" / "pages"
|
||||
@@ -254,7 +255,10 @@ class LabView(frame_macview.MacView):
|
||||
|
||||
def ssh_runner(base):
|
||||
def run(remote, stdin=None, timeout=30):
|
||||
r = subprocess.run([*base, remote], input=stdin, capture_output=True, text=True, timeout=timeout)
|
||||
r = subprocess.run([*base, remote], input=stdin.encode("utf-8") if stdin is not None else None,
|
||||
capture_output=True, timeout=timeout)
|
||||
r.stdout = r.stdout.decode("utf-8", errors="replace")
|
||||
r.stderr = r.stderr.decode("utf-8", errors="replace")
|
||||
if r.returncode:
|
||||
e = RuntimeError((r.stderr or r.stdout).strip())
|
||||
e.stdout = r.stdout
|
||||
@@ -381,7 +385,11 @@ def run_scenario(args, scenario, agent_port, token, frame_ssh):
|
||||
if args.browser_flag is not None:
|
||||
mv.browser_flags = [f for f in args.browser_flag if f]
|
||||
chrome = None
|
||||
src = "test"
|
||||
src = args.source or "test"
|
||||
cpu = None
|
||||
if args.pc:
|
||||
mv.host = "windows" if sys.platform == "win32" else "linux"
|
||||
mv.viewer_profile = "pc-view"
|
||||
try:
|
||||
if scenario in ("scroll", "type"):
|
||||
chrome = chrome_window(f"{scenario}.html")
|
||||
@@ -405,9 +413,10 @@ def run_scenario(args, scenario, agent_port, token, frame_ssh):
|
||||
since = max([f["s"] for f in first["frames"]] or [0])
|
||||
captured_before = first["captured"]
|
||||
start_mac_us = mv.call("/stats", id=stream["id"])["now"]
|
||||
agent_pid = int(subprocess.run(["pgrep", "-f", "Frame Mac View Lab.app/Contents/MacOS/frame-mac-view"],
|
||||
capture_output=True, text=True).stdout.split()[0])
|
||||
cpu = CpuSampler(frame_ssh, agent_pid)
|
||||
if not args.pc:
|
||||
agent_pid = int(subprocess.run(["pgrep", "-f", "Frame Mac View Lab.app/Contents/MacOS/frame-mac-view"],
|
||||
capture_output=True, text=True).stdout.split()[0])
|
||||
cpu = CpuSampler(frame_ssh, agent_pid)
|
||||
if relay:
|
||||
relay.begin()
|
||||
expected = 0 # input events the harness asked the viewer for
|
||||
@@ -443,7 +452,8 @@ def run_scenario(args, scenario, agent_port, token, frame_ssh):
|
||||
if not streams:
|
||||
raise SystemExit(f"{scenario}: the stream ended during the run (did the Frame go to sleep?)")
|
||||
data = streams[0]
|
||||
cpu.stop()
|
||||
if cpu:
|
||||
cpu.stop()
|
||||
if args.raw:
|
||||
Path(f"{args.raw}-{scenario}.json").write_text(json.dumps(data))
|
||||
result = summarize(scenario, data, start_mac_us, end_mac_us, args, relay, schedule, expected)
|
||||
@@ -452,7 +462,7 @@ def run_scenario(args, scenario, agent_port, token, frame_ssh):
|
||||
for e in data.get("events", []) if e["t"] >= start_mac_us]
|
||||
result["captured"] = (captured_end if captured_end is not None else data["captured"]) - captured_before
|
||||
result["source_fps"] = round(result["captured"] / max(result["duration_s"], 1), 1)
|
||||
result["cpu"] = cpu.summary()
|
||||
result["cpu"] = cpu.summary() if cpu else {"host": "not sampled"}
|
||||
result["viewer"] = data["summary"].get("decoder", "")
|
||||
result["show_s"] = show_s
|
||||
result["panel"] = shown.get("panel")
|
||||
@@ -460,6 +470,8 @@ def run_scenario(args, scenario, agent_port, token, frame_ssh):
|
||||
result["route"] = mv.route
|
||||
return result
|
||||
finally:
|
||||
if cpu:
|
||||
cpu.stop()
|
||||
try:
|
||||
mv.stop(src)
|
||||
except Exception: # noqa: BLE001 - best effort
|
||||
@@ -670,6 +682,8 @@ def fmt(v):
|
||||
|
||||
def add_run_args(r):
|
||||
r.add_argument("--scenario", default="test,scroll,type")
|
||||
r.add_argument("--pc", action="store_true", help="run the bundled PC host; use --scenario test or capture")
|
||||
r.add_argument("--source", default="", help="PC source: window:ID, display:ID, or choose (Linux portal)")
|
||||
r.add_argument("--duration", type=float, default=20)
|
||||
r.add_argument("--warmup", type=float, default=3)
|
||||
r.add_argument("--quality", default="balanced", choices=list(frame_macview.QUALITY))
|
||||
@@ -717,7 +731,7 @@ def frame_ssh_for(args):
|
||||
|
||||
|
||||
def run_suite(args, frame_ssh, quiet=False):
|
||||
agent_port, token = lab_agent()
|
||||
agent_port, token = (args.pc_view.port, args.pc_view.token) if args.pc else lab_agent()
|
||||
results = []
|
||||
for sc in args.scenario.split(","):
|
||||
if not quiet:
|
||||
@@ -744,11 +758,12 @@ def document(args, frame_ssh, results, **extra):
|
||||
commit = git("rev-parse", "--short", "HEAD") + ("+dirty" if git("status", "--porcelain", "--", "mac", "ui") else "")
|
||||
return {
|
||||
"label": args.label or args.cmd, "date": datetime.datetime.now().isoformat(timespec="seconds"), "commit": commit,
|
||||
"config": {"quality": args.quality, "mode": args.mode, "net": args.net or "none", "delay_ms": args.delay,
|
||||
"config": {"quality": args.quality, "mode": "native" if args.pc else args.mode, "net": args.net or "none", "delay_ms": args.delay,
|
||||
"buffer_ms": args.buffer, "host": args.host or args.frame, "usb": args.usb, "ssh_opts": args.ssh_opt,
|
||||
"encoder": os.environ.get("FRAME_MAC_VIEW_ENCODER", ""), "duration_s": args.duration,
|
||||
"browser_flags": args.browser_flag if args.browser_flag is not None else frame_macview.BROWSER_FLAGS},
|
||||
"frame_build": build.strip().partition("=")[2], "mac": platform.mac_ver()[0], "headset": standby[-40:],
|
||||
"frame_build": build.strip().partition("=")[2], "mac": platform.mac_ver()[0],
|
||||
"host_platform": platform.platform(), "source": args.source, "pc_host": args.pc, "headset": standby[-40:],
|
||||
"scenarios": results, **extra,
|
||||
}
|
||||
|
||||
@@ -819,19 +834,46 @@ def main():
|
||||
print("\nworse:\n " + "\n ".join(regs))
|
||||
sys.exit(1 if regs else 0)
|
||||
|
||||
lab_agent()
|
||||
if args.pc and (args.cmd == "ab" or args.scenario not in ("test", "capture")):
|
||||
p.error("PC runs use --scenario test or --scenario capture; Mac automation is not portable")
|
||||
if args.pc and args.scenario == "capture" and not args.source:
|
||||
p.error("capture needs --source window:ID, display:ID, or choose")
|
||||
if not args.pc:
|
||||
lab_agent()
|
||||
frame_ssh = frame_ssh_for(args)
|
||||
# Keep the Mac's screen awake: virtual displays aren't removed while it sleeps.
|
||||
runs = 1 if args.cmd == "run" else args.repeat * len(args.arm)
|
||||
awake = subprocess.Popen(["caffeinate", "-d", "-u", "-t", str(int(runs * (args.duration * 4 + 60) + 120))])
|
||||
awake = subprocess.Popen(["caffeinate", "-d", "-u", "-t", str(int(runs * (args.duration * 4 + 60) + 120))]) if sys.platform == "darwin" else None
|
||||
args.pc_view = None
|
||||
try:
|
||||
if args.pc:
|
||||
args.pc_view = frame_pcview.PCView(frame_ssh[:-1], ssh_runner(frame_ssh), frame_ssh[-1])
|
||||
args.pc_view.ensure_agent()
|
||||
if args.source == "choose":
|
||||
args.pc_view.call("/permissions", method="POST")
|
||||
print("Choose a window or screen in your desktop's sharing dialog.", flush=True)
|
||||
deadline = time.monotonic()+125
|
||||
while time.monotonic() < deadline:
|
||||
state = args.pc_view.call("/status")
|
||||
if not state.get("selecting"):
|
||||
sources = args.pc_view.call("/windows")["windows"]
|
||||
if not sources:
|
||||
raise SystemExit(state.get("selectionError") or "Nothing was shared")
|
||||
args.source = sources[-1]["src"]
|
||||
break
|
||||
time.sleep(.5)
|
||||
else:
|
||||
raise SystemExit("Sharing dialog timed out")
|
||||
if args.cmd == "ab":
|
||||
arm_runs, table = ab(args, frame_ssh)
|
||||
write(document(args, frame_ssh, [], arms=args.arm, runs=arm_runs, table=table), args)
|
||||
return
|
||||
results = run_suite(args, frame_ssh)
|
||||
finally:
|
||||
awake.terminate()
|
||||
if args.pc_view:
|
||||
args.pc_view.shutdown()
|
||||
if awake:
|
||||
awake.terminate()
|
||||
doc = document(args, frame_ssh, results)
|
||||
write(doc, args)
|
||||
bad = [f"{r['scenario']} {k}" for r in results for k, v in r["grades"].items() if v in ("bad", "missing")]
|
||||
|
||||
@@ -45,7 +45,10 @@ class Helpers(unittest.TestCase):
|
||||
@unittest.skipUnless(shutil.which("bash"), "needs bash")
|
||||
@unittest.skipIf(os.name == "nt", "Windows' bash.exe is WSL's launcher, and runners have no distribution")
|
||||
def test_launch_script_parses(self):
|
||||
r = subprocess.run(["bash", "-n"], input=frame_macview.LAUNCH, text=True, capture_output=True)
|
||||
bash = str(Path(os.environ.get("ProgramFiles", "C:/Program Files")) / "Git/bin/bash.exe") if sys.platform == "win32" else "bash"
|
||||
if sys.platform == "win32" and not Path(bash).exists():
|
||||
self.skipTest("Git Bash is not installed (WSL bash is not a local shell)")
|
||||
r = subprocess.run([bash, "-n"], input=frame_macview.LAUNCH.encode("utf-8"), capture_output=True)
|
||||
self.assertEqual(r.returncode, 0, r.stderr)
|
||||
|
||||
def test_show_checks_the_source_before_anything_else(self):
|
||||
|
||||
@@ -0,0 +1,64 @@
|
||||
"""Frozen regression trace from the original mac-in-headset Swift controller.
|
||||
The 600-state reference was produced at 1b90c64, before the C extraction:
|
||||
60 seconds with a slow/congested middle, then recovery. No generated behavior
|
||||
from the new implementation is used to calculate the expected digest.
|
||||
"""
|
||||
import ctypes as C
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
sys.path.insert(0, str(ROOT / 'ui'))
|
||||
from frame_pc_capture import Controller
|
||||
|
||||
|
||||
class ControllerTrace(unittest.TestCase):
|
||||
def test_original_mac_congestion_and_recovery(self):
|
||||
cc = shutil.which('cc') or shutil.which('gcc')
|
||||
if not cc:
|
||||
self.skipTest('C compiler not installed; native build jobs cover this trace')
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
path = Path(tmp) / ('trace.dll' if sys.platform == 'win32' else 'trace.so')
|
||||
subprocess.run([cc, '-O2', '-shared', '-fPIC', str(ROOT/'desktop/controller.c'), '-o', str(path)], check=True, capture_output=True)
|
||||
lib = C.CDLL(str(path))
|
||||
sig = {'new': (C.c_void_p, [C.c_int, C.c_int]), 'free': (None, [C.c_void_p]),
|
||||
'ceiling': (None, [C.c_void_p, C.c_int]), 'value': (C.c_int64, [C.c_void_p, C.c_int]),
|
||||
'capture': (None, [C.c_void_p, C.c_int64]), 'gate': (C.c_int, [C.c_void_p, C.c_int64, C.c_int]),
|
||||
'sent': (None, [C.c_void_p, C.c_uint32, C.c_int, C.c_int64]),
|
||||
'ack': (C.c_int, [C.c_void_p, C.c_uint32, C.c_int64]), 'update': (C.c_int, [C.c_void_p, C.c_int64])}
|
||||
for name, (ret, args) in sig.items():
|
||||
fn = getattr(lib, 'fc_'+name)
|
||||
fn.restype, fn.argtypes = ret, args
|
||||
c = Controller(lib, 60, 8000000)
|
||||
due, states, seq = [], [], 0
|
||||
try:
|
||||
for tick in range(6000):
|
||||
now = 10000000+tick*10000
|
||||
for t, s in due:
|
||||
if t <= now:
|
||||
c.call('ack', s, now)
|
||||
due = [(t, s) for t, s in due if t > now]
|
||||
if tick % 2 == 0:
|
||||
c.call('capture', now)
|
||||
if c.call('gate', now, 1):
|
||||
seq += 1
|
||||
c.call('sent', seq, 8000 if tick < 1000 or tick > 4000 else 22000, now)
|
||||
due.append((now+(240000 if 1500 < tick < 3500 else 20000), seq))
|
||||
if tick % 10 == 0:
|
||||
c.update(now)
|
||||
x = c.state()
|
||||
states.append([x[k] for k in ('target', 'ceiling', 'tier', 'fps', 'inFlight')] +
|
||||
[round(x['scale']*100), round(x['baseRtt']*1000), round(x['slack']*1000)])
|
||||
digest = hashlib.sha256(json.dumps(states, separators=(',', ':')).encode()).hexdigest()
|
||||
self.assertEqual(digest, '1932f16aafce5c9157bcd2aba5d90719a366f9999422920aaeae78a6cdec08f6')
|
||||
finally:
|
||||
c.close()
|
||||
if sys.platform == 'win32':
|
||||
import _ctypes
|
||||
_ctypes.FreeLibrary(lib._handle)
|
||||
@@ -0,0 +1,388 @@
|
||||
"""PC adapters: grants, input cleanup, launcher reuse and native codec contract.
|
||||
Native tests require the bundled libraries; CI sets REQUIRE_NATIVE so a missing
|
||||
bundle is a failure, never a silent skip. No desktop consent or GPU is faked as
|
||||
successful real-host capture.
|
||||
"""
|
||||
import ctypes
|
||||
import http.client
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import struct
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import unittest
|
||||
from unittest import mock
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
sys.path.insert(0, str(ROOT / 'ui'))
|
||||
import frame_pc_agent as agent
|
||||
import frame_pc_capture as capture
|
||||
import frame_pcview
|
||||
import frame_macview
|
||||
from frame_stream_stats import Stats
|
||||
from test_macview import WS
|
||||
|
||||
|
||||
class GrantsTests(unittest.TestCase):
|
||||
def test_source_bound_retry_ack_and_stop(self):
|
||||
g = agent.Grants('master')
|
||||
ticket = g.ticket('window:1')
|
||||
self.assertIsNone(g.redeem('window:2', {'t': ticket}))
|
||||
key = g.redeem('window:1', {'t': ticket})
|
||||
self.assertEqual(key, g.redeem('window:1', {'t': ticket}))
|
||||
g.ack(key)
|
||||
self.assertIsNone(g.redeem('window:1', {'t': ticket}))
|
||||
self.assertEqual(key, g.redeem('window:1', {'r': key}))
|
||||
self.assertIsNone(g.redeem('window:2', {'r': key}))
|
||||
g.revoke('window:1')
|
||||
self.assertIsNone(g.redeem('window:1', {'r': key}))
|
||||
|
||||
def test_expiry_and_stop_before_redeem(self):
|
||||
g = agent.Grants('master')
|
||||
with mock.patch.object(agent.time, 'monotonic', return_value=1):
|
||||
ticket = g.ticket('test')
|
||||
with mock.patch.object(agent.time, 'monotonic', return_value=62):
|
||||
self.assertIsNone(g.redeem('test', {'t': ticket}))
|
||||
ticket = g.ticket('test')
|
||||
g.revoke()
|
||||
self.assertIsNone(g.redeem('test', {'t': ticket}))
|
||||
|
||||
|
||||
class AdapterTests(unittest.TestCase):
|
||||
def test_windows_uses_wgc_and_media_foundation(self):
|
||||
for kind in ('window', 'display'):
|
||||
pipeline = capture.pipeline(dict(src=kind+':42'), 'win32', 'mfh264enc', 640, 360, 30, 1000000)
|
||||
self.assertIn('capture-api=wgc', pipeline)
|
||||
self.assertIn('window-handle=42' if kind == 'window' else 'monitor-handle=42', pipeline)
|
||||
self.assertIn('mfh264enc name=enc low-latency=true bframes=0', pipeline)
|
||||
self.assertNotIn('drop=true', pipeline)
|
||||
with self.assertRaises(ValueError):
|
||||
capture.pipeline(dict(src='window:42 ! fakesink'), 'win32', 'mfh264enc', 640, 360, 30, 1000000)
|
||||
|
||||
def test_linux_uses_only_portal_fd_and_node(self):
|
||||
p = capture.pipeline(dict(src='window:consented', fd=19, node=47), 'linux', 'x264enc', 640, 360, 30, 1000000)
|
||||
self.assertIn('pipewiresrc fd=19 path=47', p)
|
||||
self.assertIn('tune=zerolatency', p)
|
||||
self.assertEqual(capture.dimensions(1920, 1080, 1280), (1280, 720))
|
||||
|
||||
def test_portal_input_clamps_and_releases_held_state(self):
|
||||
native = mock.Mock()
|
||||
native.lib.fc_portal_input.return_value = 1
|
||||
inp = capture.PortalInput(native, dict(portal=123, w=100, h=50))
|
||||
inp.handle(dict(t='m', e='down', b=0, x=2, y=-1))
|
||||
inp.handle(dict(t='k', e='down', key='Control'))
|
||||
inp.release()
|
||||
calls = native.lib.fc_portal_input.call_args_list
|
||||
self.assertEqual(calls[0].args, (123, 0, 99, 0, 0, 0))
|
||||
self.assertIn(mock.call(123, 1, 0, 0, 0x110, 0), calls)
|
||||
self.assertIn(mock.call(123, 3, 0, 0, 0xffe3, 0), calls)
|
||||
self.assertFalse(inp.buttons or inp.keys)
|
||||
|
||||
def test_windows_release_and_negative_monitor_origin(self):
|
||||
host = mock.Mock()
|
||||
host.user.GetSystemMetrics.side_effect = [-1920, 0, 3840, 1080]
|
||||
inp = capture.WindowsInput(host, dict(src='display:4', x=-1920, y=0, w=1920, h=1080))
|
||||
inp.handle(dict(t='m', e='down', b=2, x=0, y=0))
|
||||
self.assertEqual(host.send.call_args_list[0], mock.call(mouse=(0, 0, 0, 0xc001)))
|
||||
inp.handle(dict(t='k', e='down', code='ControlLeft', key='Control'))
|
||||
inp.release()
|
||||
self.assertIn(mock.call(mouse=(0, 0, 0, 0x10)), host.send.call_args_list)
|
||||
self.assertIn(mock.call(key=(162, 0, 2)), host.send.call_args_list)
|
||||
|
||||
def test_refused_windows_focus_never_clicks_the_covering_app(self):
|
||||
host = mock.Mock()
|
||||
host.rect.return_value = (0, 0, 800, 600)
|
||||
host.user.GetForegroundWindow.return_value = 99
|
||||
host.user.SetForegroundWindow.return_value = False
|
||||
inp = capture.WindowsInput(host, dict(src='window:12', w=800, h=600))
|
||||
with self.assertRaisesRegex(RuntimeError, 'could not focus'):
|
||||
inp.handle(dict(t='m', e='down', b=0, x=.5, y=.5))
|
||||
host.send.assert_not_called()
|
||||
|
||||
def test_pc_reuses_launcher_and_namespaces_panel_ids(self):
|
||||
view = frame_pcview.PCView(['ssh'], mock.Mock(return_value='panel created'), 'frame')
|
||||
view.call = mock.Mock(side_effect=lambda path, **kw: {'screen': True, 'ticket': 'one-use', 'streams': []})
|
||||
view.ensure_tunnel = mock.Mock()
|
||||
view.remote_port = 47900
|
||||
view.show('window:12')
|
||||
self.assertEqual(view.run.call_args.kwargs['stdin'], frame_macview.LAUNCH)
|
||||
self.assertNotEqual(frame_macview.panel_id('window:12'), frame_macview.panel_id('window:12', 'windows'))
|
||||
self.assertFalse(view.prefer_usb)
|
||||
with self.assertRaises(frame_macview.MacViewError):
|
||||
view.show('separate:12')
|
||||
|
||||
def test_session_end_releases_native_input_only_once(self):
|
||||
session = object.__new__(agent.Session)
|
||||
session.lock, session.input_lock = threading.RLock(), threading.RLock()
|
||||
session.stop_event = threading.Event()
|
||||
session.ws, session.input = mock.Mock(), mock.Mock()
|
||||
session.released = False
|
||||
session.end()
|
||||
session.end()
|
||||
session.input.release.assert_called_once()
|
||||
self.assertTrue(session.stop_event.is_set())
|
||||
|
||||
def test_idle_source_is_not_a_stall(self):
|
||||
# PipeWire/WGC deliver frames only on damage: a static desktop sends
|
||||
# its first frame and then nothing. That must not end the session.
|
||||
session = object.__new__(agent.Session)
|
||||
session.lock, session.pending = threading.RLock(), {}
|
||||
session.stats = Stats(lambda: 0)
|
||||
session.stats.captured = 1
|
||||
self.assertIsNone(session.stalled(60000000, 0, 0))
|
||||
# A frame that entered the encoder and never came out is a fault.
|
||||
session.pending[5] = dict(cap=0, arr=0, e0=1000000, tier=0, br=0)
|
||||
self.assertIn('encoder', session.stalled(12000000, 0, 0))
|
||||
self.assertIsNone(session.stalled(10000000, 0, 0))
|
||||
# No initial frame at all after the pipeline opened is a fault.
|
||||
session.pending.clear()
|
||||
self.assertIn('capture', session.stalled(12000000, 0, 1))
|
||||
self.assertIsNone(session.stalled(9000000, 0, 1))
|
||||
|
||||
def test_controller_is_inert_after_close(self):
|
||||
lib = mock.Mock()
|
||||
lib.fc_new.return_value = 1234
|
||||
lib.fc_value.return_value = 100
|
||||
controller = capture.Controller(lib, 60, 5000000)
|
||||
controller.close()
|
||||
lib.fc_free.assert_called_once_with(1234)
|
||||
lib.reset_mock()
|
||||
state = controller.state()
|
||||
self.assertEqual((state['fps'], state['target'], state['scale']), (60, 0, 1))
|
||||
self.assertEqual(controller.call('gate', 1, 1), 0)
|
||||
self.assertEqual(controller.update(1), 0)
|
||||
controller.close()
|
||||
self.assertEqual(lib.method_calls, []) # nothing touched the freed pointer
|
||||
|
||||
def test_library_path_prepend_adds_no_empty_entry(self):
|
||||
env = {}
|
||||
frame_pcview.prepend(env, 'LD_LIBRARY_PATH', '/opt/fc/lib')
|
||||
self.assertEqual(env['LD_LIBRARY_PATH'], '/opt/fc/lib')
|
||||
frame_pcview.prepend(env, 'LD_LIBRARY_PATH', '/x')
|
||||
self.assertEqual(env['LD_LIBRARY_PATH'], '/x' + os.pathsep + '/opt/fc/lib')
|
||||
with mock.patch.dict(os.environ, clear=True):
|
||||
env = frame_pcview.PCView([], lambda *a: None, 'frame').agent_environment()
|
||||
for name in ('PATH', 'LD_LIBRARY_PATH'):
|
||||
parts = env.get(name, 'x').split(os.pathsep)
|
||||
self.assertNotIn('', parts, name)
|
||||
|
||||
def test_malformed_viewer_input_is_dropped_not_fatal(self):
|
||||
session = object.__new__(agent.Session)
|
||||
session.lock, session.input_lock = threading.RLock(), threading.RLock()
|
||||
session.stop_event, session.key_event = threading.Event(), threading.Event()
|
||||
session.src, session.codec, session.key, session.w, session.h = 'display:1', 'h264', 'r', 100, 100
|
||||
session.source = dict(src='display:1', w=100, h=100, portal=1)
|
||||
session.input_enabled, session.released = True, False
|
||||
session.native = mock.Mock()
|
||||
session.native.now.return_value = 0
|
||||
session.controller, session.agent = mock.Mock(), mock.Mock()
|
||||
session.stats = Stats(lambda: 0)
|
||||
session.produce = lambda: None
|
||||
lib = mock.Mock()
|
||||
lib.fc_portal_input.return_value = 1
|
||||
session.input = capture.PortalInput(mock.Mock(lib=lib), session.source)
|
||||
session.ws = mock.Mock()
|
||||
session.ws.receive.side_effect = [
|
||||
dict(t='m', x='left', y=0), dict(t='wheel', dx=[1]), dict(t='k', e='down', key=None, i='x'),
|
||||
dict(t='fd', drop='many'), dict(t='rx', s=[1]), dict(t='m', x=.5, y=.5), ConnectionError('closed')]
|
||||
with self.assertRaises(ConnectionError):
|
||||
session.start()
|
||||
# Every malformed message was skipped; the good move after them still ran.
|
||||
moves = [c.args for c in lib.fc_portal_input.call_args_list if c.args[1] == 0]
|
||||
self.assertAlmostEqual(moves[-1][2], .5*99)
|
||||
self.assertEqual(session.ws.receive.call_count, 7)
|
||||
|
||||
def test_stats_keep_capture_time_and_bound_records(self):
|
||||
stats = Stats(lambda: 10000000)
|
||||
stats.input(dict(t='m', i=1, tv=9999990))
|
||||
f = stats.add(cap=10000001, arr=10000002, e0=10000003, e1=10000004)
|
||||
self.assertEqual(f['echo'], 1)
|
||||
stats.report(dict(t='rx', s=f['s'], r=10000010))
|
||||
stats.report(dict(t='fd', f=[[f['s'], 10000011, 10000012, 10000013]]))
|
||||
self.assertEqual(stats.snapshot(settle=0)['inputs'][0]['frame'], f['s'])
|
||||
for _ in range(4100):
|
||||
stats.add(cap=10000001)
|
||||
self.assertEqual(len(stats.frames), 4096)
|
||||
|
||||
|
||||
class NativeTests(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
if not capture.LIBRARY.exists():
|
||||
if os.environ.get('FRAME_PC_REQUIRE_NATIVE') == '1':
|
||||
raise AssertionError('CI required native PC libraries but they were not built')
|
||||
raise unittest.SkipTest('native PC libraries not built on this host')
|
||||
cls.token = 'test-' + os.urandom(8).hex()
|
||||
env = frame_pcview.PCView([], lambda *a: None, 'frame').agent_environment()
|
||||
env['FRAME_MAC_VIEW_TOKEN'] = cls.token
|
||||
cls.proc = subprocess.Popen([sys.executable, str(ROOT/'ui/frame_pc_agent.py'), 'serve', '--port', '0',
|
||||
'--page', str(ROOT/'ui/mac-view.html'), '--exit-on-eof'], stdin=subprocess.PIPE, stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE, text=True, env=env)
|
||||
line = cls.proc.stdout.readline()
|
||||
if 'listening' not in line:
|
||||
raise AssertionError('Native agent failed: ' + cls.proc.stderr.read())
|
||||
cls.port = int(line.rsplit(':', 1)[1])
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls):
|
||||
cls.proc.stdin.close()
|
||||
try:
|
||||
rc = cls.proc.wait(10)
|
||||
except subprocess.TimeoutExpired:
|
||||
cls.proc.kill()
|
||||
raise AssertionError('Native agent did not stop after EOF')
|
||||
error = cls.proc.stderr.read()
|
||||
cls.proc.stdout.close()
|
||||
cls.proc.stderr.close()
|
||||
if rc:
|
||||
raise AssertionError('Native agent exited %s: %s' % (rc, error))
|
||||
|
||||
def request(self, path, method='GET'):
|
||||
conn = http.client.HTTPConnection('127.0.0.1', self.port, timeout=10)
|
||||
try:
|
||||
conn.request(method, path)
|
||||
response = conn.getresponse()
|
||||
return response.status, response.read()
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
def tearDown(self):
|
||||
self.request('/close?k='+self.token, 'POST')
|
||||
|
||||
@unittest.skipUnless(sys.platform.startswith('linux'), 'Linux PipeWire client libraries')
|
||||
def test_bundled_pipewire_context_needs_no_system_modules(self):
|
||||
# No portal request, remote connection or desktop capture: create and
|
||||
# destroy a private client context to test the dynamically loaded SPA
|
||||
# and protocol modules that ldd cannot discover.
|
||||
code = """
|
||||
import ctypes as C, os
|
||||
lib = C.CDLL(os.path.join(os.environ['FRAME_PC_NATIVE'], 'lib', 'libpipewire-0.3.so.0'))
|
||||
for name, ret, args in [
|
||||
('pw_init', None, [C.c_void_p, C.c_void_p]),
|
||||
('pw_main_loop_new', C.c_void_p, [C.c_void_p]),
|
||||
('pw_main_loop_get_loop', C.c_void_p, [C.c_void_p]),
|
||||
('pw_context_new', C.c_void_p, [C.c_void_p, C.c_void_p, C.c_size_t]),
|
||||
('pw_context_find_factory', C.c_void_p, [C.c_void_p, C.c_char_p]),
|
||||
('pw_context_find_spa_lib', C.c_char_p, [C.c_void_p, C.c_char_p]),
|
||||
('pw_context_destroy', None, [C.c_void_p]),
|
||||
('pw_main_loop_destroy', None, [C.c_void_p])]:
|
||||
fn = getattr(lib, name)
|
||||
fn.restype, fn.argtypes = ret, args
|
||||
lib.pw_init(None, None)
|
||||
loop = lib.pw_main_loop_new(None)
|
||||
assert loop, 'bundled SPA loop support did not load'
|
||||
context = lib.pw_context_new(lib.pw_main_loop_get_loop(loop), None, 0)
|
||||
assert context, 'bundled PipeWire client modules did not load'
|
||||
assert lib.pw_context_find_factory(context, b'adapter'), 'video adapter factory is missing'
|
||||
# The exported factory is video.adapt, which needs a live follower node to
|
||||
# instantiate. Check its configured library and exported factory without
|
||||
# pretending a headless context is a consented video stream.
|
||||
plugin_name = lib.pw_context_find_spa_lib(context, b'video.adapt')
|
||||
assert plugin_name, 'video.adapt has no bundled library mapping'
|
||||
plugin = C.CDLL(os.path.join(os.environ['SPA_PLUGIN_DIR'], plugin_name.decode() + '.so'))
|
||||
class Factory(C.Structure):
|
||||
_fields_ = [('version', C.c_uint32), ('name', C.c_char_p)]
|
||||
plugin.spa_handle_factory_enum.argtypes = [C.POINTER(C.POINTER(Factory)), C.POINTER(C.c_uint32)]
|
||||
plugin.spa_handle_factory_enum.restype = C.c_int
|
||||
factory, index, names = C.POINTER(Factory)(), C.c_uint32(), []
|
||||
while plugin.spa_handle_factory_enum(C.byref(factory), C.byref(index)) > 0:
|
||||
names.append(factory.contents.name)
|
||||
assert b'video.adapt' in names, 'bundled SPA video adapter factory is missing'
|
||||
lib.pw_context_destroy(context)
|
||||
lib.pw_main_loop_destroy(loop)
|
||||
"""
|
||||
env = frame_pcview.PCView([], lambda *a: None, 'frame').agent_environment()
|
||||
env['FRAME_PC_NATIVE'] = str(capture.NATIVE)
|
||||
result = subprocess.run([sys.executable, '-c', code], env=env, capture_output=True, text=True, timeout=15)
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
|
||||
def test_last_raw_picture_survives_a_closed_gate(self):
|
||||
# A source sends ONE changed frame, then becomes idle. It must reach
|
||||
# the encoder after congestion clears, without needing another update.
|
||||
code = """
|
||||
import ctypes as C, sys, time
|
||||
from frame_pc_capture import Native, GATE, Encoded, pipeline
|
||||
native = Native()
|
||||
start = None
|
||||
closing = False
|
||||
phases = []
|
||||
def gate(stage, pts, capture, arrived):
|
||||
global start
|
||||
phases.append(stage)
|
||||
if closing: return -1
|
||||
if start is None: start = time.monotonic()
|
||||
return int(stage == 1 or time.monotonic()-start >= .15)
|
||||
callback = GATE(gate)
|
||||
p = pipeline(dict(src='test'), sys.platform, 'x264enc', 320, 180, 30, 300000)
|
||||
p = p.replace('is-live=true', 'is-live=true num-buffers=1')
|
||||
error = C.create_string_buffer(1024)
|
||||
handle = native.lib.fc_capture_open(p.encode(), callback, error, len(error))
|
||||
assert handle, error.value
|
||||
try:
|
||||
frame = Encoded()
|
||||
result = 0
|
||||
deadline = time.monotonic()+5
|
||||
while not result and time.monotonic() < deadline:
|
||||
result = native.lib.fc_capture_pull(handle, C.byref(frame))
|
||||
assert result == 1, native.lib.fc_capture_error(handle)
|
||||
assert frame.key and frame.size > 17
|
||||
assert phases.count(0) == 1 and 2 in phases, phases
|
||||
finally:
|
||||
closing = True
|
||||
native.lib.fc_capture_close(handle)
|
||||
"""
|
||||
env = frame_pcview.PCView([], lambda *a: None, 'frame').agent_environment()
|
||||
env['PYTHONPATH'] = str(ROOT / 'ui')
|
||||
result = subprocess.run([sys.executable, '-c', code], env=env, capture_output=True, text=True, timeout=10)
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
|
||||
def test_auth_and_real_h264_stats(self):
|
||||
self.assertEqual(self.request('/status')[0], 403)
|
||||
status, body = self.request('/ticket?src=test&k='+self.token, 'POST')
|
||||
self.assertEqual(status, 200)
|
||||
ticket = json.loads(body)['ticket']
|
||||
ws = WS(self.port, '/stream?src=test&fps=30&max=640&t='+ticket)
|
||||
try:
|
||||
self.assertEqual(ws.status, 101)
|
||||
key, got, echo = None, 0, False
|
||||
ws.send_text(json.dumps(dict(t='m', e='down', i=7, tv=1, x=.2, y=.3)))
|
||||
deadline = time.monotonic()+10
|
||||
while time.monotonic() < deadline and (got < 5 or not echo):
|
||||
op, data = ws.recv()
|
||||
if op == 1:
|
||||
m = json.loads(data)
|
||||
self.assertNotEqual(m.get('t'), 'error', m)
|
||||
if m.get('t') == 'hello':
|
||||
key = m['r']
|
||||
ws.send_text(json.dumps(dict(t='ack')))
|
||||
elif op == 2:
|
||||
got += 1
|
||||
seq, echoed = struct.unpack('!II', data[9:17])
|
||||
now = struct.unpack('!Q', data[1:9])[0]
|
||||
echo |= echoed == 7
|
||||
self.assertIn(b'\x00\x00\x00\x01', data[17:])
|
||||
ws.send_text(json.dumps(dict(t='rx', s=seq, r=now+100)))
|
||||
ws.send_text(json.dumps(dict(t='fd', f=[[seq, now+200, now+300, now+400]])))
|
||||
self.assertGreaterEqual(got, 5)
|
||||
self.assertTrue(echo)
|
||||
time.sleep(.1)
|
||||
stats = json.loads(self.request('/stats?settle=0&k='+self.token)[1])['streams'][0]
|
||||
self.assertGreater(stats['frames'][0]['e1'], stats['frames'][0]['e0'])
|
||||
self.assertIn('controller', stats)
|
||||
used = WS(self.port, '/stream?src=test&t='+ticket)
|
||||
self.assertEqual(used.status, 403)
|
||||
used.close()
|
||||
self.request('/close?src=test&k='+self.token, 'POST')
|
||||
revoked = WS(self.port, '/stream?src=test&r='+key)
|
||||
self.assertEqual(revoked.status, 403)
|
||||
revoked.close()
|
||||
finally:
|
||||
ws.close()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
+29
-17
@@ -56,12 +56,12 @@ BROWSER_FLAGS = []
|
||||
PANEL_BOX = (1920, 1080)
|
||||
|
||||
# Opens one viewer on gamescope's X display and gives its window its own panel
|
||||
# id. Args: appid url width height tag [browser flags...]. The page puts "[tag]" in its title at
|
||||
# id. Args: appid url width height tag profile [browser flags...]. The page puts "[tag]" in its title at
|
||||
# once, which is how its X window is found (Chromium may hand the URL to an
|
||||
# instance that's already running, so there's no process to follow).
|
||||
LAUNCH = r"""set -u
|
||||
appid=$1 url=$2 w=$3 h=$4 tag=$5
|
||||
shift 5
|
||||
appid=$1 url=$2 w=$3 h=$4 tag=$5 profile=$6
|
||||
shift 6
|
||||
export DISPLAY=:0 LC_ALL=C.UTF-8
|
||||
unset WAYLAND_DISPLAY
|
||||
if ! xprop -root GAMESCOPE_FOCUSABLE_WINDOWS >/dev/null 2>&1; then
|
||||
@@ -73,15 +73,15 @@ common=(--ozone-platform=x11 --force-device-scale-factor=1 --no-first-run --no-d
|
||||
--disable-features=Translate,MediaRouter --autoplay-policy=no-user-gesture-required
|
||||
"--window-size=$w,$h" "$@" "--app=$url")
|
||||
if [ -x "$HOME/chromium-xr/chrome" ]; then
|
||||
cmd=("$HOME/chromium-xr/chrome" "--user-data-dir=$HOME/.local/share/frame-control/mac-view" "${common[@]}")
|
||||
cmd=("$HOME/chromium-xr/chrome" "--user-data-dir=$HOME/.local/share/frame-control/$profile" "${common[@]}")
|
||||
elif flatpak info org.chromium.Chromium >/dev/null 2>&1; then
|
||||
cmd=(flatpak run org.chromium.Chromium
|
||||
"--user-data-dir=$HOME/.var/app/org.chromium.Chromium/data/frame-mac-view" "${common[@]}")
|
||||
"--user-data-dir=$HOME/.var/app/org.chromium.Chromium/data/frame-$profile" "${common[@]}")
|
||||
else
|
||||
echo "NO_BROWSER"
|
||||
exit 3
|
||||
fi
|
||||
log=/tmp/frame-mac-view.log
|
||||
log=/tmp/frame-$profile.log
|
||||
setsid nohup "${cmd[@]}" >>"$log" 2>&1 </dev/null &
|
||||
for _ in $(seq 1 60); do
|
||||
sleep 0.5
|
||||
@@ -105,9 +105,9 @@ class MacViewError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def panel_id(src):
|
||||
def panel_id(src, host="mac"):
|
||||
"""A stable panel id per source, in the range panel-on-frame.sh uses."""
|
||||
return 2_001_000_000 + zlib.crc32(f"mac:{src}".encode()) % 1_000_000
|
||||
return 2_001_000_000 + zlib.crc32(f"{host}:{src}".encode()) % 1_000_000
|
||||
|
||||
|
||||
def fit(w, h, box=PANEL_BOX):
|
||||
@@ -116,6 +116,9 @@ def fit(w, h, box=PANEL_BOX):
|
||||
|
||||
|
||||
class MacView:
|
||||
host = "mac"
|
||||
viewer_profile = "mac-view"
|
||||
|
||||
def __init__(self, tunnel_ssh, run, frame, track=None):
|
||||
self.tunnel_ssh = list(tunnel_ssh)
|
||||
self.run = run
|
||||
@@ -166,10 +169,17 @@ class MacView:
|
||||
def _stale(self):
|
||||
try:
|
||||
built = AGENT.stat().st_mtime
|
||||
return any(p.stat().st_mtime > built for p in (SOURCES / "Sources").glob("*.swift"))
|
||||
return any(p.stat().st_mtime > built for p in [*(SOURCES / "Sources").glob("*.swift"),
|
||||
ROOT / "desktop" / "controller.c", ROOT / "desktop" / "controller.h"])
|
||||
except OSError:
|
||||
return False
|
||||
|
||||
def agent_command(self):
|
||||
return [str(AGENT)]
|
||||
|
||||
def agent_environment(self):
|
||||
return {**os.environ, "FRAME_MAC_VIEW_TOKEN": self.token}
|
||||
|
||||
def ensure_agent(self):
|
||||
with self.lock:
|
||||
if self.agent and self.agent.poll() is None:
|
||||
@@ -178,11 +188,11 @@ class MacView:
|
||||
if reason:
|
||||
raise MacViewError(reason)
|
||||
self.build()
|
||||
env = {**os.environ, "FRAME_MAC_VIEW_TOKEN": self.token}
|
||||
env = self.agent_environment()
|
||||
# The same port as before when restarting, so a running tunnel still
|
||||
# fits; otherwise (or if it's gone) whatever the system gives.
|
||||
for port in dict.fromkeys([self.port or 0, 0]):
|
||||
self.agent = subprocess.Popen([str(AGENT), "serve", "--port", str(port), "--page", str(PAGE),
|
||||
self.agent = subprocess.Popen([*self.agent_command(), "serve", "--port", str(port), "--page", str(PAGE),
|
||||
"--exit-on-eof"], env=env, stdin=subprocess.PIPE,
|
||||
stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)
|
||||
line = self.agent.stdout.readline()
|
||||
@@ -191,7 +201,7 @@ class MacView:
|
||||
break
|
||||
self.agent.kill()
|
||||
else:
|
||||
raise MacViewError(f"The Mac streaming helper didn't start: {line.strip() or 'no output'}")
|
||||
raise MacViewError(f"The streaming helper didn't start: {line.strip() or 'no output'}")
|
||||
if self.port != int(m.group(1)):
|
||||
self._drop_tunnel()
|
||||
self.port = int(m.group(1))
|
||||
@@ -207,7 +217,7 @@ class MacView:
|
||||
with urllib.request.urlopen(req, timeout=10) as r:
|
||||
return json.load(r)
|
||||
except (urllib.error.URLError, OSError, ValueError) as e:
|
||||
raise MacViewError(f"The Mac streaming helper didn't answer: {e}")
|
||||
raise MacViewError(f"The streaming helper didn't answer: {e}")
|
||||
|
||||
# ---- the tunnel from the Frame ----
|
||||
|
||||
@@ -362,7 +372,7 @@ class MacView:
|
||||
# the agent no longer counts them, so only move when none are out there.
|
||||
others = self.shown - {src}
|
||||
self.ensure_tunnel(allow_new_port=not others)
|
||||
appid = panel_id(src)
|
||||
appid = panel_id(src, self.host)
|
||||
# Unique per launch, so a new window is never confused with an old one.
|
||||
tag = "fc" + secrets.token_hex(4)
|
||||
# A single-use ticket for this source, not Frame Control's key: the URL
|
||||
@@ -372,7 +382,7 @@ class MacView:
|
||||
"bpp": q["bpp"]}
|
||||
url = f"http://127.0.0.1:{self.remote_port}/view?{urlencode(params)}"
|
||||
w, h = fit(width or 1280, height or 720)
|
||||
args = " ".join(_quote(str(a)) for a in (appid, url, w, h, tag, *self.browser_flags))
|
||||
args = " ".join(_quote(str(a)) for a in (appid, url, w, h, tag, self.viewer_profile, *self.browser_flags))
|
||||
try:
|
||||
out = self.run("bash -s -- " + args, stdin=LAUNCH, timeout=60)
|
||||
except Exception as e: # noqa: BLE001 - the server's Failure carries the Frame's words
|
||||
@@ -404,7 +414,8 @@ class MacView:
|
||||
if self.shown or self.shows != shows or self.launching:
|
||||
return
|
||||
try:
|
||||
self.run("pkill -f '[f]rame-control/mac-view|[d]ata/frame-mac-view' || true", timeout=10)
|
||||
pattern = "[f]rame-control/" + self.viewer_profile + "|[d]ata/frame-" + self.viewer_profile
|
||||
self.run("pkill -f " + _quote(pattern) + " || true", timeout=10)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
@@ -415,7 +426,8 @@ class MacView:
|
||||
status = self.call("/status")
|
||||
windows = self.call("/windows").get("windows", []) if status.get("screen") else []
|
||||
displays = self.call("/displays").get("displays", [])
|
||||
return {"available": True, "screen": status.get("screen", False),
|
||||
return {"available": True, "host": self.host, "selecting": status.get("selecting", False),
|
||||
"selectionError": status.get("selectionError", ""), "screen": status.get("screen", False),
|
||||
"accessibility": status.get("accessibility", False), "streams": status.get("streams", []),
|
||||
"windows": windows, "displays": displays, "tunnel": self.tunnel_up(),
|
||||
"route": self.route}
|
||||
|
||||
@@ -0,0 +1,624 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Frame Control's Windows/Linux host. Same ticket, H.264/JPEG, timing and
|
||||
input protocol as frame-mac-view; serves the same ui/mac-view.html.
|
||||
|
||||
Only loopback is bound. Frame Control owns the master token, the Frame gets
|
||||
one-source tickets and reconnect keys. Native libraries are bundled.
|
||||
"""
|
||||
import argparse
|
||||
import base64
|
||||
import ctypes as C
|
||||
import hashlib
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import queue
|
||||
from pathlib import Path
|
||||
import secrets
|
||||
import socket
|
||||
import struct
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from urllib.parse import parse_qs, urlsplit
|
||||
|
||||
from frame_pc_capture import Native, Controller, Windows, WindowsInput, PortalInput, Encoded, GATE, dimensions, pipeline
|
||||
from frame_stream_stats import Stats
|
||||
|
||||
|
||||
# Viewer fields are untrusted JSON: float('x'), int(None), m['t'] missing.
|
||||
MALFORMED = (ValueError, TypeError, KeyError, OverflowError)
|
||||
|
||||
|
||||
class Grants:
|
||||
"""Caller holds the agent lock, including replacement and Stop."""
|
||||
def __init__(self, token):
|
||||
self.token, self.tickets, self.keys = token, {}, {}
|
||||
|
||||
def master(self, key):
|
||||
return isinstance(key, str) and secrets.compare_digest(key, self.token)
|
||||
|
||||
def ticket(self, src):
|
||||
self.tickets = {k: v for k, v in self.tickets.items() if v[1] > time.monotonic()}
|
||||
if len(self.tickets) >= 128:
|
||||
raise ValueError('Too many pending viewers')
|
||||
ticket = secrets.token_urlsafe(24)
|
||||
self.tickets[ticket] = (src, time.monotonic()+60, None)
|
||||
return ticket
|
||||
|
||||
def redeem(self, src, q):
|
||||
if self.master(q.get('k')):
|
||||
key = secrets.token_urlsafe(24)
|
||||
self.keys[key] = src
|
||||
return key
|
||||
entry = self.tickets.get(q.get('t'))
|
||||
if entry and entry[0] == src and entry[1] > time.monotonic():
|
||||
key = entry[2] or secrets.token_urlsafe(24)
|
||||
self.tickets[q['t']] = (src, entry[1], key)
|
||||
self.keys[key] = src
|
||||
return key
|
||||
key = q.get('r')
|
||||
return key if key and self.keys.get(key) == src else None
|
||||
|
||||
def ack(self, key):
|
||||
self.tickets = {k: v for k, v in self.tickets.items() if v[2] != key}
|
||||
|
||||
def revoke(self, src=None):
|
||||
self.tickets = {k: v for k, v in self.tickets.items() if src is not None and v[0] != src}
|
||||
self.keys = {k: v for k, v in self.keys.items() if src is not None and v != src}
|
||||
|
||||
|
||||
class WebSocket:
|
||||
def __init__(self, handler):
|
||||
self.sock, self.reader = handler.connection, handler.rfile
|
||||
self.lock = threading.Lock()
|
||||
self.closed = False
|
||||
self.sock.settimeout(5)
|
||||
self.sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
|
||||
|
||||
def send(self, data, opcode=1):
|
||||
if not isinstance(data, bytes):
|
||||
data = json.dumps(data, separators=(',', ':')).encode()
|
||||
n = len(data)
|
||||
head = bytes([0x80 | opcode, n]) if n < 126 else bytes([0x80 | opcode, 126]) + struct.pack('!H', n) if n <= 65535 else bytes([0x80 | opcode, 127]) + struct.pack('!Q', n)
|
||||
with self.lock:
|
||||
if self.closed:
|
||||
raise ConnectionError('Viewer disconnected')
|
||||
self.sock.sendall(head + data)
|
||||
|
||||
def exact(self, n):
|
||||
data = self.reader.read(n)
|
||||
if len(data) != n:
|
||||
raise ConnectionError('Viewer disconnected')
|
||||
return data
|
||||
|
||||
def receive(self):
|
||||
a, b = self.exact(2)
|
||||
opcode, size = a & 15, b & 127
|
||||
if a & 0x70 or not a & 0x80 or not b & 0x80 or opcode not in (1, 8, 9, 10):
|
||||
raise ValueError('Unsupported WebSocket frame')
|
||||
if size == 126:
|
||||
size = struct.unpack('!H', self.exact(2))[0]
|
||||
elif size == 127:
|
||||
size = struct.unpack('!Q', self.exact(8))[0]
|
||||
if size > 65536 or (opcode >= 8 and size > 125):
|
||||
raise ValueError('WebSocket message too large')
|
||||
mask = self.exact(4)
|
||||
data = bytes(v ^ mask[i % 4] for i, v in enumerate(self.exact(size)))
|
||||
if opcode == 8:
|
||||
raise ConnectionError('Viewer closed')
|
||||
if opcode == 9:
|
||||
self.send(data, 10)
|
||||
if opcode != 1:
|
||||
return {}
|
||||
message = json.loads(data)
|
||||
if not isinstance(message, dict):
|
||||
raise ValueError('Expected an input object')
|
||||
return message
|
||||
|
||||
def close(self):
|
||||
self.closed = True
|
||||
try:
|
||||
self.sock.shutdown(socket.SHUT_RDWR)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
class Session:
|
||||
def __init__(self, agent, ws, key, source, query):
|
||||
self.agent, self.native, self.ws, self.key, self.source = agent, agent.native, ws, key, source
|
||||
self.src = source['src']
|
||||
self.codec = query.get('codec', 'h264')
|
||||
if self.codec not in ('h264', 'jpeg'):
|
||||
raise ValueError('Unsupported codec')
|
||||
self.fps = min(120, max(5, int(query.get('fps', 60))))
|
||||
bpp = float(query.get('bpp', .1))
|
||||
if not math.isfinite(bpp):
|
||||
raise ValueError('Invalid bitrate')
|
||||
self.w, self.h = dimensions(source['w'], source['h'], min(3840, max(320, int(query.get('max', 1920)))))
|
||||
self.bitrate = max(300000, int(self.w*self.h*self.fps*min(.5, max(.02, bpp))))
|
||||
self.controller = Controller(self.native.lib, self.fps, self.bitrate)
|
||||
self.stats = Stats(self.native.now)
|
||||
self.stop_event, self.key_event = threading.Event(), threading.Event()
|
||||
self.lock, self.input_lock = threading.RLock(), threading.RLock()
|
||||
self.pending, self.last_submit = {}, 0
|
||||
self.reconfiguring = False
|
||||
self.test_inputs = queue.Queue(maxsize=128)
|
||||
self.input = None if self.src == 'test' else WindowsInput(agent.windows, source) if agent.windows else PortalInput(self.native, source)
|
||||
self.writer = None
|
||||
self.released = False
|
||||
self.input_enabled = self.src == 'test' or self.source.get('devices', 3) == 3
|
||||
|
||||
def gate(self, stage, pts, capture, arrived):
|
||||
# Exceptions cannot cross a ctypes callback boundary.
|
||||
try:
|
||||
with self.lock:
|
||||
if self.stop_event.is_set() or self.reconfiguring:
|
||||
return -1
|
||||
if stage == 1:
|
||||
if pts in self.pending:
|
||||
self.pending[pts]['e0'] = self.native.now()
|
||||
return 1
|
||||
if stage == 0:
|
||||
self.stats.captured += 1
|
||||
self.controller.call('capture', arrived)
|
||||
now = self.native.now()
|
||||
state = self.controller.state()
|
||||
if len(self.pending) >= 3 or now-self.last_submit < 1000000/state['fps'] or not self.controller.call('gate', now, int(stage == 0)):
|
||||
if stage == 0:
|
||||
self.stats.skipped += 1
|
||||
return 0
|
||||
self.pending[pts] = dict(cap=capture, arr=arrived, e0=now, tier=state['tier'], br=state['target'])
|
||||
self.last_submit = now
|
||||
return 1
|
||||
except Exception:
|
||||
self.stop_event.set()
|
||||
return -1
|
||||
|
||||
STALL = 10000000
|
||||
|
||||
def stalled(self, now, opened, captured_at_open):
|
||||
"""PipeWire and WGC deliver frames only on screen damage, so an idle
|
||||
desktop legitimately produces no output. Fail only on evidence of a
|
||||
fault: a frame entered the encoder and never came out, or the capture
|
||||
never produced its initial frame after the pipeline opened."""
|
||||
with self.lock:
|
||||
oldest = min((r['e0'] for r in self.pending.values()), default=None)
|
||||
captured = self.stats.captured
|
||||
if oldest is not None and now-oldest > self.STALL:
|
||||
return 'The encoder stopped producing frames; check the video encoder'
|
||||
if captured == captured_at_open and now-opened > self.STALL:
|
||||
return 'No frames captured for 10 seconds; check capture permissions'
|
||||
return None
|
||||
|
||||
def fresh_pipewire(self):
|
||||
if 'portal' not in self.source:
|
||||
return
|
||||
error = C.create_string_buffer(1024)
|
||||
fd = self.native.lib.fc_portal_refresh(self.source['portal'], error, len(error))
|
||||
if fd < 0:
|
||||
raise RuntimeError(error.value.decode(errors='replace'))
|
||||
self.source['fd'] = fd
|
||||
|
||||
def produce(self):
|
||||
native, capture = self.native.lib, None
|
||||
try:
|
||||
encoder = "x264enc" if self.src == "test" and self.native.has("x264enc") else self.agent.encoder
|
||||
self.fresh_pipewire()
|
||||
description = pipeline(self.source, sys.platform, encoder, self.w, self.h, self.fps, self.bitrate, self.codec)
|
||||
self.callback = GATE(self.gate)
|
||||
error = C.create_string_buffer(1024)
|
||||
capture = native.fc_capture_open(description.encode(), self.callback, error, len(error))
|
||||
if not capture:
|
||||
raise RuntimeError(error.value.decode(errors='replace'))
|
||||
last_update = last_stats = self.native.now()
|
||||
last_config = opened = last_update
|
||||
captured_at_open = self.stats.captured
|
||||
applied = (self.w, self.h, self.bitrate)
|
||||
wanted = applied
|
||||
while not self.stop_event.is_set():
|
||||
while not self.test_inputs.empty():
|
||||
event = self.test_inputs.get_nowait()
|
||||
try:
|
||||
native.fc_capture_test(capture, int(event.get('i', 0)) & 0xffffffff)
|
||||
self.stats.input(event)
|
||||
except MALFORMED:
|
||||
pass # a bad probe field drops the event, not the stream
|
||||
output = Encoded()
|
||||
result = native.fc_capture_pull(capture, C.byref(output))
|
||||
now = self.native.now()
|
||||
if result < 0:
|
||||
raise RuntimeError(native.fc_capture_error(capture).decode(errors='replace'))
|
||||
if result:
|
||||
with self.lock:
|
||||
# At most three raw frames are in flight; no B-frames.
|
||||
# x264 offsets PTS, so match the FIFO encode order while
|
||||
# retaining the actual pre-encode capture timestamp.
|
||||
raw_pts = next(iter(self.pending), None)
|
||||
record = self.pending.get(raw_pts)
|
||||
if record is None:
|
||||
raise RuntimeError('Encoder changed frame timestamps; timing cannot be matched')
|
||||
data = C.string_at(output.data, output.size)
|
||||
f = self.stats.add(**record, e1=now, snd=now, b=len(data), k=output.key,
|
||||
w=output.width or self.w, h=output.height or self.h)
|
||||
self.controller.call('sent', f['s'], len(data)+17, now)
|
||||
self.ws.send(struct.pack('!BQII', output.key, max(0, f['cap']), f['s'], f['echo'])+data, 2)
|
||||
with self.lock:
|
||||
f['wire'] = self.native.now()
|
||||
self.pending.pop(raw_pts, None)
|
||||
stall = self.stalled(now, opened, captured_at_open)
|
||||
if stall:
|
||||
raise RuntimeError(stall)
|
||||
if self.key_event.is_set():
|
||||
self.key_event.clear()
|
||||
if self.codec == 'h264':
|
||||
native.fc_capture_key(capture)
|
||||
if now-last_update >= 100000:
|
||||
target = self.controller.update(now)
|
||||
state = self.controller.state()
|
||||
w, h = dimensions(self.w, self.h, max(320, int(max(self.w, self.h)*state['scale'])))
|
||||
if target and self.codec == 'h264' and encoder == 'x264enc':
|
||||
native.fc_capture_bitrate(capture, target)
|
||||
self.bitrate = target
|
||||
# Hardware properties are not uniformly mutable in PLAYING.
|
||||
# Drain then reopen the pipeline at a keyframe when its
|
||||
# budget changes materially. The portal fd/session stays
|
||||
# alive, so this does not bypass or repeat user consent.
|
||||
desired_bitrate = target or applied[2]
|
||||
bitrate_change = self.codec == 'h264' and encoder != 'x264enc' and abs(desired_bitrate-applied[2]) > applied[2]*.2
|
||||
if not self.reconfiguring and now-last_config >= 1000000 and ((w, h) != applied[:2] or bitrate_change):
|
||||
wanted = (w, h, desired_bitrate)
|
||||
with self.lock:
|
||||
self.reconfiguring = True
|
||||
last_update = now
|
||||
if self.reconfiguring:
|
||||
with self.lock:
|
||||
drained = not self.pending
|
||||
if drained:
|
||||
native.fc_capture_close(capture)
|
||||
capture = None
|
||||
self.fresh_pipewire()
|
||||
description = pipeline(self.source, sys.platform, encoder, wanted[0], wanted[1], self.fps, wanted[2], self.codec)
|
||||
capture = native.fc_capture_open(description.encode(), self.callback, error, len(error))
|
||||
if not capture:
|
||||
raise RuntimeError(error.value.decode(errors='replace'))
|
||||
applied, self.bitrate, last_config = wanted, wanted[2], now
|
||||
opened, captured_at_open = now, self.stats.captured
|
||||
with self.lock:
|
||||
self.reconfiguring = False
|
||||
if now-last_stats >= 1000000:
|
||||
self.ws.send(dict(self.stats.summary(), t='stats', bitrate=self.bitrate,
|
||||
size='%dx%d' % (self.w, self.h), tier=self.controller.state()['tier']))
|
||||
last_stats = now
|
||||
except Exception as e:
|
||||
try:
|
||||
self.ws.send({'t': 'error', 'message': str(e)})
|
||||
except OSError:
|
||||
pass
|
||||
finally:
|
||||
self.stop_event.set()
|
||||
if capture:
|
||||
native.fc_capture_close(capture)
|
||||
self.ws.close()
|
||||
|
||||
def start(self):
|
||||
if self.stop_event.is_set():
|
||||
self.controller.close()
|
||||
return
|
||||
self.ws.send(dict(t='hello', r=self.key))
|
||||
self.ws.send(dict(t='info', src=self.src, title=self.source.get('title', self.source.get('name', 'Test pattern')),
|
||||
app='PC', codec=self.codec, input=self.input_enabled,
|
||||
inputMessage='Allow pointer and keyboard control in the host sharing dialog.',
|
||||
aspect=self.w/self.h, warm=0))
|
||||
with self.lock:
|
||||
if self.stop_event.is_set():
|
||||
self.controller.close()
|
||||
return
|
||||
self.writer = threading.Thread(target=self.produce, daemon=True)
|
||||
self.writer.start()
|
||||
try:
|
||||
while not self.stop_event.is_set():
|
||||
m = self.ws.receive()
|
||||
t = m.get('t')
|
||||
if t == 'ping':
|
||||
self.ws.send(dict(t='pong', c=m.get('c', 0), a=self.native.now()))
|
||||
elif t == 'ack':
|
||||
with self.agent.lock:
|
||||
self.agent.grants.ack(self.key)
|
||||
elif t == 'key-frame':
|
||||
self.key_event.set()
|
||||
elif t in ('rx', 'fd', 'clock'):
|
||||
try:
|
||||
self.stats.report(m)
|
||||
except MALFORMED:
|
||||
pass
|
||||
if t == 'rx' and isinstance(m.get('s'), int):
|
||||
self.controller.call('ack', m['s'] & 0xffffffff, self.native.now())
|
||||
elif t in ('m', 'wheel', 'k', 'text', 'release'):
|
||||
with self.input_lock:
|
||||
if self.stop_event.is_set():
|
||||
break
|
||||
if self.input:
|
||||
if not self.input_enabled and t != 'release':
|
||||
continue
|
||||
try:
|
||||
self.input.handle(m)
|
||||
self.stats.input(m)
|
||||
except MALFORMED:
|
||||
pass # drop a malformed viewer event; keep the session
|
||||
except RuntimeError as e:
|
||||
self.input_enabled = False
|
||||
try:
|
||||
self.input.release()
|
||||
except RuntimeError:
|
||||
pass
|
||||
self.ws.send(dict(t='error', message=str(e)))
|
||||
elif m.get('i'):
|
||||
try:
|
||||
self.test_inputs.put_nowait(m)
|
||||
except queue.Full:
|
||||
pass
|
||||
finally:
|
||||
self.end()
|
||||
self.writer.join(6)
|
||||
if not self.writer.is_alive():
|
||||
self.controller.close()
|
||||
|
||||
def end(self):
|
||||
with self.lock:
|
||||
self.stop_event.set()
|
||||
self.ws.close()
|
||||
with self.input_lock:
|
||||
if self.input and not self.released:
|
||||
self.released = True
|
||||
try:
|
||||
self.input.release()
|
||||
except (RuntimeError, OSError):
|
||||
pass
|
||||
|
||||
|
||||
class Agent:
|
||||
def __init__(self, native, token, page):
|
||||
self.native, self.page = native, page
|
||||
self.lock = threading.RLock()
|
||||
self.grants = Grants(token)
|
||||
self.sessions, self.sources = {}, {}
|
||||
self.next_id, self.selecting, self.selection_error = 1, False, ''
|
||||
self.windows = Windows() if sys.platform == 'win32' else None
|
||||
self.encoder = 'mfh264enc' if self.windows else 'vah264enc' if native.has('vah264enc') else 'x264enc'
|
||||
self.shutting_down = False
|
||||
self.selection_generation = 0
|
||||
|
||||
def lists(self):
|
||||
if self.windows:
|
||||
windows, displays = self.windows.sources()
|
||||
self.sources = {s['src']: s for s in windows + displays}
|
||||
return windows, displays
|
||||
return [{k: v for k, v in s.items() if k not in ('portal', 'fd', 'node')} for s in self.sources.values()], []
|
||||
|
||||
def source(self, src):
|
||||
if src == 'test':
|
||||
return dict(src='test', title='Test pattern', w=1280, h=720)
|
||||
self.lists()
|
||||
if src not in self.sources:
|
||||
raise ValueError('Choose a window or screen on this computer first')
|
||||
return dict(self.sources[src])
|
||||
|
||||
def select(self):
|
||||
if self.windows or self.selecting or self.shutting_down:
|
||||
return
|
||||
if len(self.sources) >= 8:
|
||||
raise ValueError('Stop a panel before sharing another source')
|
||||
self.selecting, self.selection_error = True, ''
|
||||
generation = self.selection_generation
|
||||
def choose():
|
||||
error = C.create_string_buffer(1024)
|
||||
portal = self.native.lib.fc_portal_select(error, len(error))
|
||||
with self.lock:
|
||||
self.selecting = False
|
||||
if not portal:
|
||||
self.selection_error = error.value.decode(errors='replace')
|
||||
elif self.shutting_down or generation != self.selection_generation:
|
||||
self.native.lib.fc_portal_close(portal)
|
||||
else:
|
||||
fd, node, w, h, devices = [self.native.lib.fc_portal_value(portal, i) for i in range(5)]
|
||||
src = 'window:' + secrets.token_hex(8)
|
||||
self.sources[src] = dict(src=src, title='Shared window or screen', app='Linux portal',
|
||||
w=w, h=h, fd=fd, node=node, portal=portal, devices=devices)
|
||||
threading.Thread(target=choose, daemon=True).start()
|
||||
|
||||
def stop(self, src=None):
|
||||
with self.lock:
|
||||
if src is None:
|
||||
self.selection_generation += 1
|
||||
self.grants.revoke(src)
|
||||
sessions = [s for s in self.sessions.values() if src is None or s.src == src]
|
||||
for session in sessions:
|
||||
try:
|
||||
session.ws.send(dict(t='close'))
|
||||
except OSError:
|
||||
pass
|
||||
session.end()
|
||||
for session in sessions:
|
||||
if session.writer and session.writer is not threading.current_thread():
|
||||
session.writer.join(6)
|
||||
# The capture is stopped before releasing its PipeWire fd/session.
|
||||
with self.lock:
|
||||
if not self.windows:
|
||||
for key, source in list(self.sources.items()):
|
||||
if src is None or key == src:
|
||||
if any(s.src == key and s.writer and s.writer.is_alive() for s in sessions):
|
||||
continue
|
||||
self.native.lib.fc_portal_close(source['portal'])
|
||||
self.sources.pop(key, None)
|
||||
return {'closed': len(sessions)}
|
||||
|
||||
|
||||
class Handler(BaseHTTPRequestHandler):
|
||||
protocol_version = 'HTTP/1.1'
|
||||
|
||||
def log_message(self, *args):
|
||||
pass # URLs contain credentials
|
||||
|
||||
def reply(self, data, status=200, content='application/json'):
|
||||
if not isinstance(data, bytes):
|
||||
data = json.dumps(data).encode()
|
||||
self.send_response(status)
|
||||
self.send_header('Content-Type', content)
|
||||
self.send_header('Content-Length', str(len(data)))
|
||||
self.send_header('Cache-Control', 'no-store')
|
||||
self.send_header('Connection', 'close')
|
||||
self.end_headers()
|
||||
self.wfile.write(data)
|
||||
self.close_connection = True
|
||||
|
||||
def do_GET(self):
|
||||
self.dispatch('GET')
|
||||
|
||||
def do_POST(self):
|
||||
self.dispatch('POST')
|
||||
|
||||
def dispatch(self, method):
|
||||
agent = self.server.agent
|
||||
url = urlsplit(self.path)
|
||||
q = {k: v[-1] for k, v in parse_qs(url.query).items()}
|
||||
try:
|
||||
if method == 'GET' and url.path == '/ping':
|
||||
return self.reply(b'frame-mac-view', content='text/plain')
|
||||
if method == 'GET' and url.path == '/view':
|
||||
return self.reply(agent.page.read_bytes(), content='text/html; charset=utf-8')
|
||||
if method == 'GET' and url.path == '/stream':
|
||||
return self.stream(q)
|
||||
if not agent.grants.master(q.get('k', self.headers.get('X-Token'))):
|
||||
return self.reply({'error': 'forbidden'}, 403)
|
||||
if method == 'POST' and url.path == '/close':
|
||||
return self.reply(agent.stop(q.get('src')))
|
||||
with agent.lock:
|
||||
windows, displays = agent.lists()
|
||||
if method == 'GET' and url.path == '/status':
|
||||
data = dict(version=1, host='windows' if agent.windows else 'linux', screen=True,
|
||||
accessibility=True, selecting=agent.selecting, selectionError=agent.selection_error,
|
||||
encoder=agent.encoder, finished=[], streams=[dict(id=i, src=s.src,
|
||||
title=s.source.get('title', ''), stats=s.stats.summary(), controller=s.controller.state())
|
||||
for i, s in agent.sessions.items() if not s.stop_event.is_set()])
|
||||
elif method == 'GET' and url.path in ('/windows', '/displays'):
|
||||
data = dict(windows=windows, displays=displays, screen=True)
|
||||
elif method == 'POST' and url.path == '/ticket':
|
||||
agent.source(q.get('src'))
|
||||
data = {'ticket': agent.grants.ticket(q['src'])}
|
||||
elif method == 'POST' and url.path == '/permissions':
|
||||
agent.select()
|
||||
data = {'selecting': agent.selecting}
|
||||
elif method == 'GET' and url.path == '/stats':
|
||||
data = dict(now=agent.native.now(), streams=[dict(id=i, src=s.src, controller=s.controller.state(),
|
||||
events=list(s.controller.events), **s.stats.snapshot(max(0, int(q.get('since', 0))), max(0, int(q.get('settle', 1500000)))))
|
||||
for i, s in agent.sessions.items() if q.get('id', str(i)) == str(i) and not s.stop_event.is_set()])
|
||||
elif method == 'POST' and url.path == '/bench':
|
||||
data = {'sent': 0}
|
||||
for s in agent.sessions.values():
|
||||
if s.src == q.get('src'):
|
||||
m = {k: v for k, v in q.items() if k not in ('k', 'src')}
|
||||
for k in ('x', 'y', 'interval'):
|
||||
if k in m:
|
||||
m[k] = float(m[k])
|
||||
s.ws.send(dict(m, t='bench'))
|
||||
data['sent'] += 1
|
||||
else:
|
||||
return self.reply({'error': 'not found'}, 404)
|
||||
self.reply(data)
|
||||
except (ValueError, RuntimeError) as e:
|
||||
self.reply({'error': str(e)}, 400)
|
||||
except (OSError, ConnectionError):
|
||||
self.close_connection = True
|
||||
|
||||
def stream(self, query):
|
||||
agent = self.server.agent
|
||||
with agent.lock:
|
||||
key = agent.grants.redeem(query.get('src'), query)
|
||||
if not key:
|
||||
return self.reply({'error': 'forbidden'}, 403)
|
||||
source = agent.source(query.get('src'))
|
||||
if self.headers.get('Upgrade', '').lower() != 'websocket' or self.headers.get('Sec-WebSocket-Version') != '13':
|
||||
return self.reply({'error': 'expected WebSocket'}, 400)
|
||||
wskey = self.headers.get('Sec-WebSocket-Key', '')
|
||||
if len(base64.b64decode(wskey, validate=True)) != 16:
|
||||
raise ValueError('Bad WebSocket key')
|
||||
if len(agent.sessions) >= 8:
|
||||
raise ValueError('At most eight panels may be open')
|
||||
# Stop can revoke and close only a registered session. Register
|
||||
# under the same lock as redemption, before any capture starts.
|
||||
session = Session(agent, WebSocket(self), key, source, query)
|
||||
for old in list(agent.sessions.values()):
|
||||
if old.key == key or old.src == source['src']:
|
||||
if old.key != key:
|
||||
agent.grants.keys.pop(old.key, None)
|
||||
try:
|
||||
old.ws.send(dict(t='close'))
|
||||
except OSError:
|
||||
pass
|
||||
old.end()
|
||||
if old.writer:
|
||||
old.writer.join(6)
|
||||
if old.writer.is_alive():
|
||||
session.controller.close()
|
||||
raise ValueError('The previous capture is still stopping; retry shortly')
|
||||
ident = agent.next_id
|
||||
agent.next_id += 1
|
||||
agent.sessions[ident] = session
|
||||
accept = base64.b64encode(hashlib.sha1((wskey+'258EAFA5-E914-47DA-95CA-C5AB0DC85B11').encode()).digest()).decode()
|
||||
self.send_response(101)
|
||||
self.send_header('Upgrade', 'websocket')
|
||||
self.send_header('Connection', 'Upgrade')
|
||||
self.send_header('Sec-WebSocket-Accept', accept)
|
||||
self.end_headers()
|
||||
try:
|
||||
session.start()
|
||||
except (OSError, ValueError, RuntimeError):
|
||||
session.end()
|
||||
finally:
|
||||
session.end()
|
||||
if session.writer:
|
||||
session.writer.join(6)
|
||||
if not session.writer or not session.writer.is_alive():
|
||||
session.controller.close()
|
||||
with agent.lock:
|
||||
agent.sessions.pop(ident, None)
|
||||
self.close_connection = True
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('command', choices=['serve'])
|
||||
parser.add_argument('--port', type=int, default=0)
|
||||
parser.add_argument('--page', type=Path, required=True)
|
||||
parser.add_argument('--exit-on-eof', action='store_true')
|
||||
args = parser.parse_args()
|
||||
token = os.environ.get('FRAME_MAC_VIEW_TOKEN')
|
||||
if not token:
|
||||
raise SystemExit('Frame Control must provide a private token')
|
||||
native = Native()
|
||||
server = ThreadingHTTPServer(('127.0.0.1', args.port), Handler)
|
||||
agent = server.agent = Agent(native, token, args.page)
|
||||
if args.exit_on_eof:
|
||||
def eof():
|
||||
sys.stdin.buffer.read()
|
||||
with agent.lock:
|
||||
agent.shutting_down = True
|
||||
agent.stop()
|
||||
server.shutdown()
|
||||
threading.Thread(target=eof, daemon=True).start()
|
||||
print('listening on 127.0.0.1:%d' % server.server_port, flush=True)
|
||||
try:
|
||||
server.serve_forever()
|
||||
finally:
|
||||
agent.shutting_down = True
|
||||
agent.stop()
|
||||
server.server_close()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
try:
|
||||
main()
|
||||
except (OSError, RuntimeError, AttributeError) as e:
|
||||
raise SystemExit('PC streaming helper: ' + str(e))
|
||||
@@ -0,0 +1,383 @@
|
||||
"""Native PC capture and input bindings. No desktop-streaming app required.
|
||||
|
||||
The packaged native library contains our adapter and the shared rate controller;
|
||||
GStreamer supplies WGC/Media Foundation and PipeWire/VA-API/x264 libraries.
|
||||
"""
|
||||
import ctypes as C
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import sys
|
||||
import threading
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
NATIVE = Path(os.environ.get('FRAME_PC_NATIVE', ROOT / 'desktop' / 'bundle'))
|
||||
LIBRARY = NATIVE / ('pc-host.dll' if sys.platform == 'win32' else 'pc-host.so')
|
||||
|
||||
|
||||
class Encoded(C.Structure):
|
||||
_fields_ = [('data', C.c_void_p), ('size', C.c_int), ('key', C.c_int),
|
||||
('width', C.c_int), ('height', C.c_int), ('pts', C.c_int64)]
|
||||
|
||||
|
||||
GATE = C.CFUNCTYPE(C.c_int, C.c_int, C.c_int64, C.c_int64, C.c_int64)
|
||||
|
||||
|
||||
class Native:
|
||||
def __init__(self, path=LIBRARY):
|
||||
self.dll_dirs = []
|
||||
if sys.platform == 'win32':
|
||||
# Set process coordinates before loading GStreamer: a plugin may
|
||||
# create a helper HWND during discovery, after which it is too late.
|
||||
user = C.WinDLL('user32')
|
||||
user.SetProcessDpiAwarenessContext.argtypes = [C.c_void_p]
|
||||
user.SetProcessDpiAwarenessContext(C.c_void_p(-4))
|
||||
for folder in (path.parent, path.parent / 'bin'):
|
||||
self.dll_dirs.append(os.add_dll_directory(str(folder)))
|
||||
self.lib = C.CDLL(str(path))
|
||||
signatures = {
|
||||
'fc_now': (C.c_int64, []), 'fc_gst_init': (None, []),
|
||||
'fc_has_element': (C.c_int, [C.c_char_p]),
|
||||
'fc_new': (C.c_void_p, [C.c_int, C.c_int]), 'fc_free': (None, [C.c_void_p]),
|
||||
'fc_ceiling': (None, [C.c_void_p, C.c_int]),
|
||||
'fc_gate': (C.c_int, [C.c_void_p, C.c_int64, C.c_int]),
|
||||
'fc_capture': (None, [C.c_void_p, C.c_int64]),
|
||||
'fc_sent': (None, [C.c_void_p, C.c_uint32, C.c_int, C.c_int64]),
|
||||
'fc_ack': (C.c_int, [C.c_void_p, C.c_uint32, C.c_int64]),
|
||||
'fc_update': (C.c_int, [C.c_void_p, C.c_int64]),
|
||||
'fc_value': (C.c_int64, [C.c_void_p, C.c_int]),
|
||||
'fc_capture_open': (C.c_void_p, [C.c_char_p, GATE, C.c_char_p, C.c_int]),
|
||||
'fc_capture_pull': (C.c_int, [C.c_void_p, C.POINTER(Encoded)]),
|
||||
'fc_capture_error': (C.c_char_p, [C.c_void_p]),
|
||||
'fc_capture_bitrate': (None, [C.c_void_p, C.c_int]),
|
||||
'fc_capture_key': (None, [C.c_void_p]),
|
||||
'fc_capture_test': (None, [C.c_void_p, C.c_uint32]), 'fc_capture_close': (None, [C.c_void_p]),
|
||||
}
|
||||
if sys.platform.startswith('linux'):
|
||||
signatures.update({
|
||||
'fc_portal_select': (C.c_void_p, [C.c_char_p, C.c_int]),
|
||||
'fc_portal_close': (None, [C.c_void_p]),
|
||||
'fc_portal_refresh': (C.c_int, [C.c_void_p, C.c_char_p, C.c_int]),
|
||||
'fc_portal_value': (C.c_int, [C.c_void_p, C.c_int]),
|
||||
'fc_portal_input': (C.c_int, [C.c_void_p, C.c_int, C.c_double, C.c_double, C.c_int, C.c_int]),
|
||||
})
|
||||
for name, (result, args) in signatures.items():
|
||||
fn = getattr(self.lib, name)
|
||||
fn.restype, fn.argtypes = result, args
|
||||
self.lib.fc_gst_init()
|
||||
|
||||
def has(self, name):
|
||||
return bool(self.lib.fc_has_element(name.encode()))
|
||||
|
||||
def now(self):
|
||||
return self.lib.fc_now()
|
||||
|
||||
|
||||
class Controller:
|
||||
def __init__(self, lib, fps, ceiling):
|
||||
# close() may run on the stream thread while /status reads state();
|
||||
# after close every entry point is a no-op, never a NULL dereference.
|
||||
self.lib, self.lock, self.fps = lib, threading.RLock(), fps
|
||||
self.enabled = os.environ.get('FRAME_MAC_VIEW_ADAPT') != '0'
|
||||
self.ptr = lib.fc_new(fps, self.enabled)
|
||||
if not self.ptr:
|
||||
raise MemoryError('Unable to allocate the streaming controller')
|
||||
lib.fc_ceiling(self.ptr, ceiling)
|
||||
self.events = []
|
||||
|
||||
def state(self):
|
||||
with self.lock:
|
||||
names = ['target', 'ceiling', 'tier', 'fps', 'scale', 'baseRtt', 'inFlight', 'slack']
|
||||
if not self.ptr:
|
||||
out = dict.fromkeys(names, 0)
|
||||
out.update(fps=self.fps, scale=100)
|
||||
else:
|
||||
out = {k: self.lib.fc_value(self.ptr, i) for i, k in enumerate(names)}
|
||||
out.update(scale=out['scale'] / 100, baseRtt=out['baseRtt'] / 1000,
|
||||
slack=out['slack'] / 1000, adapt=self.enabled)
|
||||
return out
|
||||
|
||||
def call(self, name, *args):
|
||||
with self.lock:
|
||||
return getattr(self.lib, 'fc_' + name)(self.ptr, *args) if self.ptr else 0
|
||||
|
||||
def update(self, now):
|
||||
with self.lock:
|
||||
if not self.ptr:
|
||||
return 0
|
||||
old = self.state()
|
||||
result = self.lib.fc_update(self.ptr, now)
|
||||
state = self.state()
|
||||
if state['target'] < old['target'] or state['tier'] != old['tier']:
|
||||
self.events.append({'t': now, 'e': 'target %s bit/s; tier %s' % (state['target'], state['tier'])})
|
||||
self.events = self.events[-200:]
|
||||
return result
|
||||
|
||||
def close(self):
|
||||
with self.lock:
|
||||
if self.ptr:
|
||||
self.lib.fc_free(self.ptr)
|
||||
self.ptr = None
|
||||
|
||||
|
||||
def dimensions(w, h, maximum):
|
||||
scale = min(1, maximum / max(w, h))
|
||||
return max(2, int(w * scale) // 2 * 2), max(2, int(h * scale) // 2 * 2)
|
||||
|
||||
|
||||
def pipeline(source, platform, encoder, w, h, fps, bitrate, codec='h264'):
|
||||
"""Only locally constructed numeric source IDs enter the pipeline parser."""
|
||||
if source['src'] == 'test':
|
||||
capture = 'videotestsrc name=source is-live=true pattern=ball'
|
||||
elif platform == 'win32':
|
||||
kind, ident = source['src'].split(':')
|
||||
if kind not in ('window', 'display') or not re.fullmatch(r'[0-9]+', ident):
|
||||
raise ValueError('Invalid Windows capture source')
|
||||
prop = 'window-handle' if kind == 'window' else 'monitor-handle'
|
||||
capture = 'd3d11screencapturesrc capture-api=wgc show-cursor=true show-border=true %s=%d' % (prop, int(ident))
|
||||
else:
|
||||
capture = 'pipewiresrc fd=%d path=%d do-timestamp=true' % (source['fd'], source['node'])
|
||||
# The source gate runs before conversion/encoding. There is no leaky queue
|
||||
# of H.264 frames; every encoded reference frame reaches the socket.
|
||||
raw = '%s ! video/x-raw,framerate=%d/1 ! queue name=raw_queue leaky=downstream max-size-buffers=1 max-size-bytes=0 max-size-time=0 ! identity name=gate ! videoconvert ! videoscale add-borders=false ! video/x-raw,width=%d,height=%d' % (capture, fps, w, h)
|
||||
if codec == 'jpeg':
|
||||
enc = 'jpegenc name=enc quality=80'
|
||||
parse = ''
|
||||
else:
|
||||
choices = {
|
||||
'mfh264enc': 'mfh264enc name=enc low-latency=true bframes=0 gop-size=60',
|
||||
'vah264enc': 'vah264enc name=enc b-frames=0 key-int-max=60',
|
||||
'x264enc': 'x264enc name=enc tune=zerolatency speed-preset=ultrafast bframes=0 key-int-max=60',
|
||||
}
|
||||
enc = choices[encoder] + ' bitrate=%d' % max(1, bitrate // 1000)
|
||||
raw += ',format=NV12' if encoder != 'x264enc' else ',format=I420'
|
||||
parse = ' ! h264parse config-interval=-1 ! video/x-h264,stream-format=byte-stream,alignment=au'
|
||||
return raw + ' ! ' + enc + parse + ' ! appsink name=out sync=false max-buffers=2 drop=false'
|
||||
|
||||
|
||||
# X11 keysyms work on Wayland through the RemoteDesktop portal too. The host's
|
||||
# keyboard layout handles physical keys; Unicode text is sent as Unicode keysyms.
|
||||
KEYSYMS = {'Enter': 0xff0d, 'Tab': 0xff09, 'Backspace': 0xff08, 'Escape': 0xff1b,
|
||||
'Delete': 0xffff, 'ArrowLeft': 0xff51, 'ArrowUp': 0xff52, 'ArrowRight': 0xff53,
|
||||
'ArrowDown': 0xff54, 'Home': 0xff50, 'End': 0xff57, 'PageUp': 0xff55,
|
||||
'PageDown': 0xff56, 'Shift': 0xffe1, 'Control': 0xffe3, 'Alt': 0xffe9, 'Meta': 0xffeb}
|
||||
|
||||
|
||||
class PortalInput:
|
||||
def __init__(self, native, source):
|
||||
self.lib, self.source = native.lib, source
|
||||
self.buttons, self.keys = set(), set()
|
||||
|
||||
def emit(self, kind, x=0, y=0, code=0, down=0):
|
||||
if not self.lib.fc_portal_input(self.source['portal'], kind, x, y, code, down):
|
||||
raise RuntimeError('The desktop portal refused input; check the sharing permission')
|
||||
|
||||
def release(self):
|
||||
error = None
|
||||
for values, kind in ((self.buttons, 1), (self.keys, 3)):
|
||||
for code in list(values):
|
||||
try:
|
||||
self.emit(kind, code=code, down=0)
|
||||
values.discard(code)
|
||||
except RuntimeError as e:
|
||||
error = e
|
||||
if error:
|
||||
raise error
|
||||
|
||||
def handle(self, m):
|
||||
t = m['t']
|
||||
if t == 'release':
|
||||
return self.release()
|
||||
if t in ('m', 'wheel'):
|
||||
x, y = (max(0, min(1, float(m.get(k, 0)))) for k in ('x', 'y'))
|
||||
self.emit(0, x * (self.source['w'] - 1), y * (self.source['h'] - 1))
|
||||
if t == 'm' and m.get('e') in ('up', 'down'):
|
||||
b = {0: 0x110, 1: 0x112, 2: 0x111}.get(m.get('b', 0))
|
||||
if b is not None:
|
||||
down = m['e'] == 'down'
|
||||
self.emit(1, code=b, down=down)
|
||||
(self.buttons.add if down else self.buttons.discard)(b)
|
||||
elif t == 'wheel':
|
||||
self.emit(2, max(-4096, min(4096, float(m.get('dx', 0)))), max(-4096, min(4096, float(m.get('dy', 0)))))
|
||||
elif t == 'k':
|
||||
key = str(m.get('key', ''))
|
||||
k = KEYSYMS.get(key) or (ord(key) if len(key) == 1 and ord(key) < 256 else
|
||||
0x1000000 + ord(key) if len(key) == 1 else 0)
|
||||
if k:
|
||||
down = m.get('e') == 'down'
|
||||
self.emit(3, code=k, down=down)
|
||||
(self.keys.add if down else self.keys.discard)(k)
|
||||
elif t == 'text':
|
||||
for ch in str(m.get('s', ''))[:4096]:
|
||||
k = ord(ch) if ord(ch) < 256 else 0x1000000 + ord(ch)
|
||||
self.emit(3, code=k, down=1)
|
||||
self.emit(3, code=k, down=0)
|
||||
|
||||
|
||||
class Windows:
|
||||
"""WGC source handles and SendInput. No elevation or global input hook."""
|
||||
def __init__(self):
|
||||
from ctypes import wintypes as W
|
||||
self.W = W
|
||||
self.user = C.WinDLL('user32', use_last_error=True)
|
||||
self.dwm = C.WinDLL('dwmapi')
|
||||
self.user.IsWindow.argtypes = [W.HWND]
|
||||
self.user.IsWindowVisible.argtypes = [W.HWND]
|
||||
self.user.IsIconic.argtypes = [W.HWND]
|
||||
self.user.GetWindowTextLengthW.argtypes = [W.HWND]
|
||||
self.user.GetWindowTextW.argtypes = [W.HWND, W.LPWSTR, C.c_int]
|
||||
self.user.GetWindowRect.argtypes = [W.HWND, C.POINTER(W.RECT)]
|
||||
self.user.SetForegroundWindow.argtypes = [W.HWND]
|
||||
self.user.GetForegroundWindow.argtypes = []
|
||||
self.user.GetForegroundWindow.restype = W.HWND
|
||||
self.dwm.DwmGetWindowAttribute.argtypes = [W.HWND, W.DWORD, C.c_void_p, W.DWORD]
|
||||
self.callback = C.WINFUNCTYPE(W.BOOL, W.HWND, W.LPARAM)
|
||||
self.monitor_callback = C.WINFUNCTYPE(W.BOOL, W.HMONITOR, W.HDC, C.POINTER(W.RECT), W.LPARAM)
|
||||
self.user.EnumWindows.argtypes = [self.callback, W.LPARAM]
|
||||
self.user.EnumDisplayMonitors.argtypes = [W.HDC, C.c_void_p, self.monitor_callback, W.LPARAM]
|
||||
class Mouse(C.Structure):
|
||||
_fields_ = [('dx', W.LONG), ('dy', W.LONG), ('data', W.DWORD), ('flags', W.DWORD),
|
||||
('time', W.DWORD), ('extra', C.c_size_t)]
|
||||
class Key(C.Structure):
|
||||
_fields_ = [('vk', W.WORD), ('scan', W.WORD), ('flags', W.DWORD), ('time', W.DWORD), ('extra', C.c_size_t)]
|
||||
class Union(C.Union):
|
||||
_fields_ = [('mouse', Mouse), ('key', Key)]
|
||||
class Input(C.Structure):
|
||||
_fields_ = [('type', W.DWORD), ('data', Union)]
|
||||
self.Mouse, self.Key, self.Input = Mouse, Key, Input
|
||||
self.user.SendInput.argtypes = [W.UINT, C.POINTER(Input), C.c_int]
|
||||
self.user.SendInput.restype = W.UINT
|
||||
|
||||
def rect(self, hwnd):
|
||||
r = self.W.RECT()
|
||||
if not self.user.IsWindow(hwnd) or self.user.IsIconic(hwnd):
|
||||
raise RuntimeError('The captured window closed or was minimized')
|
||||
# WGC captures the visible extended frame, excluding invisible resize borders.
|
||||
if self.dwm.DwmGetWindowAttribute(hwnd, 9, C.byref(r), C.sizeof(r)):
|
||||
if not self.user.GetWindowRect(hwnd, C.byref(r)):
|
||||
raise RuntimeError('Cannot locate the captured window')
|
||||
return r.left, r.top, r.right - r.left, r.bottom - r.top
|
||||
|
||||
def sources(self):
|
||||
windows, displays = [], []
|
||||
def window(hwnd, _):
|
||||
if not self.user.IsWindowVisible(hwnd) or self.user.IsIconic(hwnd):
|
||||
return True
|
||||
n = self.user.GetWindowTextLengthW(hwnd)
|
||||
if n:
|
||||
title = C.create_unicode_buffer(n + 1)
|
||||
self.user.GetWindowTextW(hwnd, title, n + 1)
|
||||
try:
|
||||
x, y, w, h = self.rect(hwnd)
|
||||
if w > 0 and h > 0:
|
||||
windows.append(dict(src='window:%d' % hwnd, id=hwnd, title=title.value,
|
||||
app='Windows', w=w, h=h, x=x, y=y))
|
||||
except RuntimeError:
|
||||
pass
|
||||
return True
|
||||
def monitor(handle, dc, rect, data):
|
||||
r = rect.contents
|
||||
displays.append(dict(src='display:%d' % handle, name='Display %d' % (len(displays) + 1),
|
||||
x=r.left, y=r.top, w=r.right-r.left, h=r.bottom-r.top))
|
||||
return True
|
||||
self.user.EnumWindows(self.callback(window), 0)
|
||||
self.user.EnumDisplayMonitors(None, None, self.monitor_callback(monitor), 0)
|
||||
return windows, displays
|
||||
|
||||
def send(self, mouse=None, key=None):
|
||||
inp = self.Input()
|
||||
inp.type = 0 if mouse else 1
|
||||
if mouse:
|
||||
inp.data.mouse = self.Mouse(*mouse, 0, 0)
|
||||
else:
|
||||
inp.data.key = self.Key(*key, 0, 0)
|
||||
if self.user.SendInput(1, C.byref(inp), C.sizeof(inp)) != 1:
|
||||
raise RuntimeError('Windows refused input (elevated apps and the secure desktop cannot be controlled)')
|
||||
|
||||
|
||||
class WindowsInput:
|
||||
def __init__(self, host, source):
|
||||
self.host, self.source = host, source
|
||||
self.buttons, self.keys = set(), set()
|
||||
|
||||
def release(self):
|
||||
error = None
|
||||
for button in list(self.buttons):
|
||||
try:
|
||||
self.host.send(mouse=(0, 0, 0, {0: 4, 1: 0x40, 2: 0x10}[button]))
|
||||
self.buttons.discard(button)
|
||||
except RuntimeError as e:
|
||||
error = e
|
||||
for vk in list(self.keys):
|
||||
try:
|
||||
extended = 1 if vk in (33, 34, 35, 36, 37, 38, 39, 40, 45, 46, 91, 92, 163, 165) else 0
|
||||
self.host.send(key=(vk, 0, 2 | extended))
|
||||
self.keys.discard(vk)
|
||||
except RuntimeError as e:
|
||||
error = e
|
||||
if error:
|
||||
raise error
|
||||
|
||||
def focus(self):
|
||||
if not self.source['src'].startswith('window:'):
|
||||
return
|
||||
hwnd = int(self.source['src'].split(':')[1])
|
||||
user = self.host.user
|
||||
if user.GetForegroundWindow() != hwnd and not user.SetForegroundWindow(hwnd):
|
||||
raise RuntimeError('Windows could not focus this window. Bring it to the front on your PC, then press Stop and Show again.')
|
||||
if user.GetForegroundWindow() != hwnd:
|
||||
raise RuntimeError('The selected window is not in front. Reopen the view after bringing it to the front on your PC.')
|
||||
|
||||
def handle(self, m):
|
||||
t, u = m['t'], self.host.user
|
||||
if t == 'release':
|
||||
return self.release()
|
||||
if t in ('m', 'wheel'):
|
||||
source = self.source
|
||||
if source['src'].startswith('window:'):
|
||||
hwnd = int(source['src'].split(':')[1])
|
||||
x, y, w, h = self.host.rect(hwnd)
|
||||
if m.get('e') == 'down' or t == 'wheel':
|
||||
self.focus()
|
||||
else:
|
||||
x, y, w, h = (source[k] for k in ('x', 'y', 'w', 'h'))
|
||||
x += max(0, min(1, float(m.get('x', 0)))) * (w - 1)
|
||||
y += max(0, min(1, float(m.get('y', 0)))) * (h - 1)
|
||||
left, top, width, height = (u.GetSystemMetrics(i) for i in (76, 77, 78, 79))
|
||||
self.host.send(mouse=(round((x-left)*65535/max(1,width-1)), round((y-top)*65535/max(1,height-1)), 0, 0xc001))
|
||||
if t == 'm' and m.get('e') in ('up', 'down'):
|
||||
b = m.get('b', 0)
|
||||
if b in (0, 1, 2):
|
||||
down = m['e'] == 'down'
|
||||
self.host.send(mouse=(0, 0, 0, {0: 2, 1: 0x20, 2: 8}[b] * (1 if down else 2)))
|
||||
(self.buttons.add if down else self.buttons.discard)(b)
|
||||
elif t == 'wheel':
|
||||
for name, flag, direction in (('dy', 0x800, -1), ('dx', 0x1000, 1)):
|
||||
delta = int(max(-4096, min(4096, float(m.get(name, 0))))) * direction
|
||||
if delta:
|
||||
self.host.send(mouse=(0, 0, delta & 0xffffffff, flag))
|
||||
elif t == 'k':
|
||||
code, down = str(m.get('code', '')), m.get('e') == 'down'
|
||||
if down:
|
||||
self.focus()
|
||||
special = {'Enter': 13, 'Escape': 27, 'Tab': 9, 'Backspace': 8, 'Space': 32,
|
||||
'ArrowLeft': 37, 'ArrowUp': 38, 'ArrowRight': 39, 'ArrowDown': 40,
|
||||
'Delete': 46, 'Home': 36, 'End': 35, 'PageUp': 33, 'PageDown': 34,
|
||||
'ShiftLeft': 160, 'ShiftRight': 161, 'ControlLeft': 162, 'ControlRight': 163,
|
||||
'AltLeft': 164, 'AltRight': 165, 'MetaLeft': 91, 'MetaRight': 92}
|
||||
vk = special.get(code, 0)
|
||||
if re.fullmatch(r'Key[A-Z]', code) or re.fullmatch(r'Digit[0-9]', code):
|
||||
vk = ord(code[-1])
|
||||
if vk:
|
||||
extended = 1 if vk in (33, 34, 35, 36, 37, 38, 39, 40, 45, 46, 91, 92, 163, 165) else 0
|
||||
self.host.send(key=(vk, 0, extended | (0 if down else 2)))
|
||||
(self.keys.add if down else self.keys.discard)(vk)
|
||||
elif down and len(str(m.get('key', ''))) == 1:
|
||||
self.handle({'t': 'text', 's': m['key']})
|
||||
elif t == 'text':
|
||||
self.focus()
|
||||
raw = str(m.get('s', ''))[:4096].encode('utf-16-le')
|
||||
for i in range(0, len(raw), 2):
|
||||
unit = int.from_bytes(raw[i:i+2], 'little')
|
||||
self.host.send(key=(0, unit, 4))
|
||||
self.host.send(key=(0, unit, 6))
|
||||
@@ -0,0 +1,90 @@
|
||||
"""PC host selection behind the existing MacView tunnel/panel controller.
|
||||
|
||||
The public /api/macview name and viewer URL remain compatible with the Mac
|
||||
base branch. Only helper launch, platform availability and source validation
|
||||
are different. No second viewer, SSH supervisor or benchmark launcher.
|
||||
"""
|
||||
import os
|
||||
import sys
|
||||
import subprocess
|
||||
from frame_macview import MacView, MacViewError, ROOT
|
||||
from frame_pc_capture import LIBRARY, NATIVE
|
||||
|
||||
|
||||
def prepend(env, name, path):
|
||||
"""An empty entry means the current directory to the loader; never add one."""
|
||||
rest = env.get(name, '')
|
||||
env[name] = str(path) + (os.pathsep + rest if rest else '')
|
||||
|
||||
|
||||
class PCView(MacView):
|
||||
viewer_profile = 'pc-view'
|
||||
host = 'windows' if sys.platform == 'win32' else 'linux'
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
self.prefer_usb = False # Mac networksetup probe is platform-specific
|
||||
|
||||
def unavailable(self):
|
||||
if sys.platform not in ('win32', 'linux'):
|
||||
return 'PC streaming needs Windows or a Linux desktop.'
|
||||
if not LIBRARY.is_file():
|
||||
return 'The PC streaming libraries are missing from this build. See docs/pc-in-headset.md.'
|
||||
return None
|
||||
|
||||
def state(self):
|
||||
result = super().state()
|
||||
result["host"] = self.host
|
||||
return result
|
||||
|
||||
def build(self):
|
||||
pass # native libraries are built and bundled with the app
|
||||
|
||||
def agent_command(self):
|
||||
return [sys.executable, str(ROOT / 'ui' / 'frame_pc_agent.py')]
|
||||
|
||||
def agent_environment(self):
|
||||
env = super().agent_environment()
|
||||
env['GST_PLUGIN_PATH_1_0'] = str(NATIVE / 'lib' / 'gstreamer-1.0')
|
||||
env['GST_PLUGIN_SYSTEM_PATH_1_0'] = ''
|
||||
cache = os.path.join(os.path.expanduser('~'), '.cache', 'frame-control')
|
||||
os.makedirs(cache, exist_ok=True)
|
||||
env['GST_REGISTRY_1_0'] = os.path.join(cache, 'gstreamer-registry.bin')
|
||||
env['GST_REGISTRY_FORK'] = 'no'
|
||||
if sys.platform == 'win32':
|
||||
prepend(env, 'PATH', NATIVE / 'bin')
|
||||
else:
|
||||
prepend(env, 'LD_LIBRARY_PATH', NATIVE / 'lib')
|
||||
env['PIPEWIRE_MODULE_DIR'] = str(NATIVE / 'lib' / 'pipewire-0.3')
|
||||
env['SPA_PLUGIN_DIR'] = str(NATIVE / 'lib' / 'spa-0.2')
|
||||
env['PIPEWIRE_CONFIG_DIR'] = str(NATIVE / 'share' / 'pipewire')
|
||||
env['PIPEWIRE_CONFIG_NAME'] = 'client.conf'
|
||||
return env
|
||||
|
||||
def shutdown(self):
|
||||
self.closing = True
|
||||
try:
|
||||
self.stop()
|
||||
except MacViewError:
|
||||
pass
|
||||
if self.tunnel and self.tunnel.poll() is None:
|
||||
self.tunnel.terminate()
|
||||
if self.agent and self.agent.poll() is None:
|
||||
# EOF gives the host a chance to release held input and portal
|
||||
# sessions, including on Windows where terminate is uncatchable.
|
||||
if self.agent.stdin:
|
||||
self.agent.stdin.close()
|
||||
try:
|
||||
self.agent.wait(10)
|
||||
except subprocess.TimeoutExpired:
|
||||
self.agent.kill()
|
||||
self.agent.wait(5)
|
||||
|
||||
def _show(self, src, quality, width, height):
|
||||
if src.startswith('separate:'):
|
||||
raise MacViewError('Separate virtual displays are a Mac-only feature. Choose a window or screen.')
|
||||
return super()._show(src, quality, width, height)
|
||||
|
||||
|
||||
def host_view(*args, **kwargs):
|
||||
return (MacView if sys.platform == 'darwin' else PCView)(*args, **kwargs)
|
||||
@@ -0,0 +1,88 @@
|
||||
"""The desktop stream's timing records, in the Mac viewer/bench schema.
|
||||
|
||||
Times are host monotonic microseconds. Zero means unknown, never a fabricated
|
||||
capture/display timestamp. Storage is bounded to the same 4096/512 records as
|
||||
Stats.swift. The existing macview-bench.py grades these records unchanged.
|
||||
"""
|
||||
from collections import OrderedDict
|
||||
import threading
|
||||
|
||||
|
||||
def distribution(values):
|
||||
values = sorted(v for v in values if v is not None)
|
||||
if not values:
|
||||
return {}
|
||||
return {key: round(values[min(len(values)-1, int((len(values)-1)*p + .5))], 1)
|
||||
for key, p in (('p50', .5), ('p95', .95))}
|
||||
|
||||
|
||||
class Stats:
|
||||
def __init__(self, now):
|
||||
self.now, self.lock = now, threading.RLock()
|
||||
self.frames, self.inputs = OrderedDict(), OrderedDict()
|
||||
self.captured = self.skipped = self.dropped = 0
|
||||
self.rtt, self.decoder, self.synced = 0, '', False
|
||||
self.sequence = 0
|
||||
|
||||
def add(self, **values):
|
||||
with self.lock:
|
||||
self.sequence = (self.sequence + 1) & 0xffffffff
|
||||
record = dict.fromkeys(('s', 'k', 'b', 'w', 'h', 'cap', 'arr', 'e0', 'e1', 'snd',
|
||||
'wire', 'rx', 'dec', 'drw', 'vs', 'echo', 'tier', 'br'), 0)
|
||||
record.update(values, s=self.sequence)
|
||||
self.frames[self.sequence] = record
|
||||
while len(self.frames) > 4096:
|
||||
self.frames.popitem(last=False)
|
||||
for event in self.inputs.values():
|
||||
if not event['frame'] and event['inj'] <= record['cap']:
|
||||
event.update(frame=self.sequence, cap=record['cap'])
|
||||
record['echo'] = event['id']
|
||||
return record
|
||||
|
||||
def report(self, message):
|
||||
with self.lock:
|
||||
if message['t'] == 'rx':
|
||||
frame = self.frames.get(message.get('s'))
|
||||
if frame and isinstance(message.get('r'), (int, float)):
|
||||
frame['rx'] = int(message['r'])
|
||||
elif message['t'] == 'fd':
|
||||
for item in message.get('f', [])[:4096]:
|
||||
if isinstance(item, list) and len(item) >= 4:
|
||||
frame = self.frames.get(item[0])
|
||||
if frame:
|
||||
for name, val in zip(('dec', 'drw', 'vs'), item[1:4]):
|
||||
if isinstance(val, (int, float)):
|
||||
frame[name] = int(val)
|
||||
self.dropped += max(0, int(message.get('drop', 0)))
|
||||
elif message['t'] == 'clock':
|
||||
self.rtt = float(message.get('rtt', 0))
|
||||
self.decoder = str(message.get('dec', ''))[:1024]
|
||||
self.synced = True
|
||||
|
||||
def input(self, m):
|
||||
if not isinstance(m.get('i'), int) or not m['i']:
|
||||
return
|
||||
with self.lock:
|
||||
self.inputs[m['i']] = dict(id=m['i'], kind=m['t'], tv=m.get('tv') or 0,
|
||||
inj=self.now(), frame=0, cap=0)
|
||||
while len(self.inputs) > 512:
|
||||
self.inputs.popitem(last=False)
|
||||
|
||||
def summary(self):
|
||||
with self.lock:
|
||||
now = self.now()
|
||||
frames = [f for f in self.frames.values() if f['cap'] >= now-2000000]
|
||||
out = dict(captured=self.captured, skipped=self.skipped, dropped=self.dropped,
|
||||
rtt=self.rtt, decoder=self.decoder, synced=self.synced,
|
||||
fps=sum(f['vs'] > 0 for f in frames)/2, sentFps=len(frames)/2,
|
||||
mbps=round(sum(f['b'] for f in frames)*4/1e6, 2))
|
||||
for key, start, end in (('capture', 'cap', 'arr'), ('queue', 'arr', 'e0'),
|
||||
('encode', 'e0', 'e1'), ('network', 'e1', 'rx'),
|
||||
('decode', 'rx', 'dec'), ('draw', 'dec', 'drw'), ('total', 'cap', 'drw')):
|
||||
out[key] = distribution([(f[end]-f[start])/1000 for f in frames if f[start] and f[end]])
|
||||
return out
|
||||
|
||||
def snapshot(self, since=0, settle=1500000):
|
||||
with self.lock:
|
||||
return dict(frames=[dict(f) for f in self.frames.values() if f['s'] > since and f['cap'] <= self.now()-settle],
|
||||
inputs=[dict(i) for i in self.inputs.values()], captured=self.captured, summary=self.summary())
|
||||
+21
-7
@@ -774,7 +774,7 @@
|
||||
</section>
|
||||
|
||||
<section class="panel" id="macview" hidden>
|
||||
<div class="shelf-head"><h2>Mac in the headset</h2><span class="count" id="mvCount"></span><span class="spacer"></span>
|
||||
<div class="shelf-head"><h2 id="mvTitle">Mac in the headset</h2><span class="count" id="mvCount"></span><span class="spacer"></span>
|
||||
<select id="mvQuality" title="Picture quality and bandwidth">
|
||||
<option value="sharp">Sharp</option><option value="balanced" selected>Balanced</option>
|
||||
<option value="light">Light</option><option value="compatible">Compatible (JPEG)</option>
|
||||
@@ -791,7 +791,7 @@
|
||||
<button class="small danger" data-mv="stopall">Stop all</button>
|
||||
</div>
|
||||
<div class="hint">Each window or screen becomes its own panel in the headset. Place it with the SteamVR dashboard
|
||||
(Float in World, Move, Size). Point and click with the laser, scroll with the thumbstick, and type on the Mac's keyboard.</div>
|
||||
(Float in World, Move, Size). Point and click with the laser, scroll with the thumbstick, and type on your computer’s keyboard.</div>
|
||||
</section>
|
||||
|
||||
<section class="panel" id="privacy">
|
||||
@@ -2451,8 +2451,16 @@ async function loadMacView(restart) {
|
||||
if (seq !== mvSeq) return;
|
||||
const box = $("macview");
|
||||
if (err) { box.hidden = false; return failed($("mvWindows"), err); }
|
||||
if (!s.available) { box.hidden = true; return; }
|
||||
if (!s.available) {
|
||||
box.hidden = !s.host;
|
||||
if (s.host) { $("mvTitle").textContent = "PC in the headset"; $("mvWindows").textContent = s.reason; }
|
||||
return;
|
||||
}
|
||||
box.hidden = false;
|
||||
const pc = s.host && s.host !== "mac";
|
||||
$("mvTitle").textContent = pc ? "PC in the headset" : "Mac in the headset";
|
||||
$("mvSeparate").closest("label").hidden = pc;
|
||||
$("mvSeparate").disabled = pc;
|
||||
const live = new Set((s.streams || []).map(x => x.src));
|
||||
const byStream = src => (s.streams || []).find(x => x.src === src);
|
||||
$("mvCount").textContent = live.size ? `${live.size} showing${s.route === "usb" ? " · over USB-C" : ""}` : "";
|
||||
@@ -2462,6 +2470,12 @@ async function loadMacView(restart) {
|
||||
$("mvPerm").hidden = !perm.length;
|
||||
$("mvPerm").innerHTML = perm.join(" ") + `<div class="row"><button class="small action" data-mv="perm">Allow…</button>
|
||||
<span class="sub">Then press Refresh. macOS may ask you to reopen Frame Control.</span></div>`;
|
||||
if (pc) {
|
||||
$("mvPerm").hidden = s.host !== "linux";
|
||||
$("mvPerm").innerHTML = `<button class="small action" data-mv="perm" ${s.selecting ? "disabled" : ""}>${s.selecting ? "Choose on your computer…" : "Choose a window or screen…"}</button>
|
||||
<span class="sub">Your desktop asks what to share and whether to allow pointer and keyboard control.</span>` +
|
||||
(s.selectionError ? `<div class="sub">${esc(s.selectionError)}</div>` : "");
|
||||
}
|
||||
const liveAs = src => live.has(src) ? src : src.startsWith("window:") && live.has("separate:" + src.slice(7)) ? "separate:" + src.slice(7) : "";
|
||||
$("mvDisplays").innerHTML = (s.displays || []).map(d =>
|
||||
mvRow(d.src, `Whole screen: ${d.name}`, `${d.w}×${d.h}${d.main ? " · main display" : ""}`, d.w, d.h, liveAs(d.src), byStream(liveAs(d.src)))).join("");
|
||||
@@ -2469,11 +2483,11 @@ async function loadMacView(restart) {
|
||||
$("mvWinCount").textContent = wins.length ? `${wins.length}` : "";
|
||||
$("mvWindows").innerHTML = !s.screen ? `<div class="sub">Windows appear here once Screen Recording is allowed.</div>`
|
||||
: wins.length ? wins.map(w => mvRow(w.src, w.title || w.app, `${w.title ? w.app + " · " : ""}${w.w}×${w.h}`, w.w, w.h, liveAs(w.src), byStream(liveAs(w.src)))).join("")
|
||||
: `<div class="sub">No windows open on this Mac's current desktop.</div>`;
|
||||
: `<div class="sub">No shared windows yet.</div>`;
|
||||
// Keep the numbers fresh while something is in the headset and the card is on screen.
|
||||
clearTimeout(mvTimer);
|
||||
const tick = () => { mvTimer = document.hidden || !box.offsetParent ? setTimeout(tick, 3000) : (loadMacView(), 0); };
|
||||
if (live.size) mvTimer = setTimeout(tick, 3000);
|
||||
if (live.size || s.selecting) mvTimer = setTimeout(tick, 3000);
|
||||
}
|
||||
$("mvRefresh").onclick = () => loadMacView(true);
|
||||
$("mvSeparate").checked = localStorage.getItem("mvSeparate") !== "0";
|
||||
@@ -2482,7 +2496,7 @@ $("macview").onclick = async e => {
|
||||
const b = e.target.closest("[data-mv]"); if (!b) return;
|
||||
const src = b.dataset.src;
|
||||
if (b.dataset.mv === "show") {
|
||||
const as = $("mvSeparate").checked && src.startsWith("window:") ? "separate:" + src.slice(7) : src;
|
||||
const as = !$("mvSeparate").disabled && $("mvSeparate").checked && src.startsWith("window:") ? "separate:" + src.slice(7) : src;
|
||||
await act(`Show ${b.dataset.title} in the headset`, async () => {
|
||||
const r = await api("/api/macview", { action: "show", src: as, quality: $("mvQuality").value,
|
||||
w: +b.dataset.w || undefined, h: +b.dataset.h || undefined });
|
||||
@@ -2493,7 +2507,7 @@ $("macview").onclick = async e => {
|
||||
} else if (b.dataset.mv === "stopall") {
|
||||
await act("Stop everything in the headset", () => api("/api/macview", { action: "stop" }), b);
|
||||
} else if (b.dataset.mv === "perm") {
|
||||
await act("Ask macOS for permission", () => api("/api/macview", { action: "permissions" }), b);
|
||||
await act("Choose what to share", () => api("/api/macview", { action: "permissions" }), b);
|
||||
}
|
||||
setTimeout(loadMacView, 800);
|
||||
};
|
||||
|
||||
+5
-5
@@ -22,7 +22,7 @@
|
||||
</head>
|
||||
<body>
|
||||
<canvas id="c" width="16" height="9"></canvas>
|
||||
<div id="note">Connecting to the Mac…</div>
|
||||
<div id="note">Connecting to your computer…</div>
|
||||
<div id="hud" hidden></div>
|
||||
<script>
|
||||
"use strict";
|
||||
@@ -243,14 +243,14 @@ async function connect() {
|
||||
// Refused before ever getting a key: the ticket expired or was revoked,
|
||||
// and retrying can't help.
|
||||
if (!reconnectKey && ++failedStarts >= 3) {
|
||||
show("This view has expired. Press Show in Frame Control on the Mac to open it again.");
|
||||
show("This view has expired. Press Show in Frame Control on your computer to open it again.");
|
||||
return;
|
||||
}
|
||||
hideNoteOnFrame = true;
|
||||
// The Mac may be asleep or the tunnel restarting: keep trying.
|
||||
retry = Math.min(retry + 1, 6);
|
||||
// Keep the Mac's reason (a missing permission, a closed window) on screen.
|
||||
show(lastError ? `${lastError} Retrying…` : "Lost the Mac. Reconnecting…");
|
||||
show(lastError ? `${lastError} Retrying…` : "Lost the computer. Reconnecting…");
|
||||
setTimeout(connect, 500 * 2 ** retry);
|
||||
};
|
||||
}
|
||||
@@ -269,7 +269,7 @@ function control(m) {
|
||||
document.title = tag ? `${name} [${tag}]` : name;
|
||||
if (!m.input) {
|
||||
hideNoteOnFrame = false; // a warning, not a connection notice: let it stay its 8 s
|
||||
show("Clicks and keys need Accessibility permission on the Mac (Frame Control asks for it).", 8000);
|
||||
show(m.inputMessage || "Clicks and keys need Accessibility permission on the Mac (Frame Control asks for it).", 8000);
|
||||
}
|
||||
} else if (m.t === "error") {
|
||||
stats.errors.push(m.message);
|
||||
@@ -391,7 +391,7 @@ addEventListener("keydown", e => onKey(e, true));
|
||||
addEventListener("keyup", e => onKey(e, false));
|
||||
addEventListener("blur", () => send({ t: "release" }));
|
||||
|
||||
show("Connecting to the Mac…");
|
||||
show("Connecting to your computer…");
|
||||
connect();
|
||||
</script>
|
||||
</body>
|
||||
|
||||
+4
-2
@@ -46,6 +46,7 @@ import frame_apk_versions # noqa: E402
|
||||
import frame_catalog # noqa: E402
|
||||
import frame_host # noqa: E402
|
||||
import frame_macview # noqa: E402
|
||||
import frame_pcview # noqa: E402
|
||||
import frame_media # noqa: E402
|
||||
import frame_report # noqa: E402
|
||||
import frame_store # noqa: E402
|
||||
@@ -1683,8 +1684,9 @@ def push_media(path):
|
||||
|
||||
# The tunnel gets its own connection: the shared master's options would win
|
||||
# over anything added after them.
|
||||
macview = frame_macview.MacView(["ssh", "-o", "BatchMode=yes", "-o", "ConnectTimeout=8"],
|
||||
lambda remote, stdin=None, timeout=30: ssh(remote, stdin=stdin, timeout=timeout),
|
||||
macview = frame_pcview.host_view(["ssh", "-o", "BatchMode=yes", "-o", "ConnectTimeout=8"],
|
||||
lambda remote, stdin=None, timeout=30: ssh(remote, stdin=stdin.encode("utf-8") if stdin is not None else None,
|
||||
timeout=timeout, text=False).decode("utf-8", errors="replace"),
|
||||
FRAME, track=_live_tunnels.add)
|
||||
|
||||
|
||||
|
||||
Reference in new issue
Block a user