From cd202433bb94e4ea15da01abd56b10a825b511d6 Mon Sep 17 00:00:00 2001 From: saphid <4596216+saphid@users.noreply.github.com> Date: Mon, 28 Sep 2026 22:50:58 +1000 Subject: [PATCH] Verify shared adaptation and harden PC stream lifecycle --- .github/workflows/release.yml | 15 + ...52-dirty-pc-agent-frame-arm64-hairpin.json | 311 ++++++++++++++++++ desktop/capture.c | 9 + desktop/controller.c | 12 +- desktop/portal.c | 22 +- docs/pc-in-headset.md | 43 ++- scripts/macview-bench.py | 8 +- tests/test_macview.py | 5 +- tests/test_pc_controller.py | 64 ++++ tests/test_pcview.py | 11 + ui/frame_macview.py | 22 +- ui/frame_pc_agent.py | 109 +++++- ui/frame_pc_capture.py | 45 ++- ui/frame_pcview.py | 25 +- ui/server.py | 3 +- 15 files changed, 641 insertions(+), 63 deletions(-) create mode 100644 bench/results/2026-09-28-c5fc552-dirty-pc-agent-frame-arm64-hairpin.json create mode 100644 tests/test_pc_controller.py diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d8b23ed..4279eb1 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -69,6 +69,21 @@ jobs: run: npm ci && npm run ${{ matrix.script }} env: CSC_IDENTITY_AUTO_DISCOVERY: "false" + - name: Test the bundled Python with the native PC libraries + if: runner.os != 'macOS' + shell: bash + run: | + if [ "$RUNNER_OS" = Windows ]; then + platform=win; arch=x64 + python="$PWD/app/build/deps/win-x64/python/python.exe" + native="$(cygpath -w "$PWD/app/build/desktop/win-x64")" + else + platform=linux; arch=x64 + if [ "$(uname -m)" = aarch64 ]; then arch=arm64; fi + python="$PWD/app/build/deps/linux-$arch/python/bin/python3" + native="$PWD/app/build/desktop/linux-$arch" + fi + FRAME_PC_NATIVE="$native" FRAME_PC_REQUIRE_NATIVE=1 "$python" -m unittest discover -s tests -p 'test_pc*.py' -v - name: Upload to the release if: startsWith(github.ref, 'refs/tags/') shell: bash diff --git a/bench/results/2026-09-28-c5fc552-dirty-pc-agent-frame-arm64-hairpin.json b/bench/results/2026-09-28-c5fc552-dirty-pc-agent-frame-arm64-hairpin.json new file mode 100644 index 0000000..abccae8 --- /dev/null +++ b/bench/results/2026-09-28-c5fc552-dirty-pc-agent-frame-arm64-hairpin.json @@ -0,0 +1,311 @@ +{ + "label": "pc-agent-frame-arm64-hairpin", + "date": "2026-09-28T22:36:19", + "commit": "c5fc552+dirty", + "config": { + "quality": "balanced", + "mode": "separate", + "net": "none", + "delay_ms": 0, + "buffer_ms": 250, + "host": "frame", + "usb": false, + "ssh_opts": [], + "encoder": "", + "duration_s": 10.0, + "browser_flags": [] + }, + "frame_build": "20260925.6191901", + "mac": "26.5.2", + "host_platform": "Linux aarch64, SteamOS Frame, BUILD_ID 20260925.6191901 (synthetic host only)", + "source": "", + "pc_host": true, + "headset": "0:03.056203 [Info] - 1 - leaving standby", + "scenarios": [ + { + "scenario": "test", + "frames_sent": 351, + "frames_drawn": 351, + "frames_shown": 324, + "duration_s": 10.1, + "stages_ms": { + "capture": { + "p50": 0.1, + "p95": 0.2, + "p99": 0.5, + "n": 351 + }, + "queue": { + "p50": 0.1, + "p95": 0.3, + "p99": 0.4, + "n": 351 + }, + "encode": { + "p50": 2.0, + "p95": 2.9, + "p99": 3.5, + "n": 351 + }, + "socket": { + "p50": 0.1, + "p95": 0.4, + "p99": 0.6, + "n": 351 + }, + "network": { + "p50": 11.6, + "p95": 75.7, + "p99": 130.1, + "n": 351 + }, + "decode": { + "p50": 0.7, + "p95": 2.1, + "p99": 4.1, + "n": 351 + }, + "draw": { + "p50": 0.4, + "p95": 0.8, + "p99": 1.1, + "n": 351 + }, + "present": { + "p50": 0.8, + "p95": 18.1, + "p99": 24.9, + "n": 324 + }, + "content": { + "p50": 15.1, + "p95": 80.3, + "p99": 134.2, + "n": 351 + }, + "content_shown": { + "p50": 21.0, + "p95": 71.8, + "p99": 146.3, + "n": 324 + } + }, + "fps": 34.8, + "fps_shown": 32.0, + "late_pct": 53.41, + "stall_max": 226.8, + "stalls_over_100ms": 7, + "mbps": 0.26, + "keyframes": 6, + "size": "1280x720", + "captured": 607, + "viewer_never_drawn": 0, + "input_ms": { + "p50": 43.3, + "p95": 98.4, + "p99": 110.1, + "n": 18 + }, + "input_shown_ms": { + "p50": 54.7, + "p95": 98.9, + "p99": 114.0, + "n": 18 + }, + "input_parts_ms": { + "uplink": { + "p50": 17.1, + "p95": 48.3, + "p99": 53.1, + "n": 18 + }, + "mac": { + "p50": 12.5, + "p95": 32.9, + "p99": 35.7, + "n": 18 + }, + "back": { + "p50": 16.8, + "p95": 50.0, + "p99": 55.9, + "n": 18 + } + }, + "inputs": { + "asked": 18, + "sent": 18, + "seen": 18 + }, + "timeline": [ + { + "t": 0, + "fps": 38, + "mbps": 0.3, + "content_p50": 14.8, + "content_p95": 25.3, + "bitrate": 2764800, + "tier": 0, + "w": 1280, + "net": null + }, + { + "t": 1, + "fps": 33, + "mbps": 0.24, + "content_p50": 15.6, + "content_p95": 103.5, + "bitrate": 3845448, + "tier": 0, + "w": 1280, + "net": null + }, + { + "t": 2, + "fps": 39, + "mbps": 0.28, + "content_p50": 15.6, + "content_p95": 29.9, + "bitrate": 5283790, + "tier": 0, + "w": 1280, + "net": null + }, + { + "t": 3, + "fps": 25, + "mbps": 0.21, + "content_p50": 17.8, + "content_p95": 175.9, + "bitrate": 2764800, + "tier": 0, + "w": 1280, + "net": null + }, + { + "t": 4, + "fps": 34, + "mbps": 0.22, + "content_p50": 15.4, + "content_p95": 69.8, + "bitrate": 2764800, + "tier": 0, + "w": 1280, + "net": null + }, + { + "t": 5, + "fps": 38, + "mbps": 0.29, + "content_p50": 17.7, + "content_p95": 68.7, + "bitrate": 2764800, + "tier": 0, + "w": 1280, + "net": null + }, + { + "t": 6, + "fps": 38, + "mbps": 0.26, + "content_p50": 15.0, + "content_p95": 18.2, + "bitrate": 3091280, + "tier": 0, + "w": 1280, + "net": null + }, + { + "t": 7, + "fps": 30, + "mbps": 0.22, + "content_p50": 19.2, + "content_p95": 111.0, + "bitrate": 1725204, + "tier": 0, + "w": 1280, + "net": null + }, + { + "t": 8, + "fps": 35, + "mbps": 0.26, + "content_p50": 13.4, + "content_p95": 102.6, + "bitrate": 1725204, + "tier": 0, + "w": 1280, + "net": null + }, + { + "t": 9, + "fps": 38, + "mbps": 0.28, + "content_p50": 12.3, + "content_p95": 20.9, + "bitrate": 1725204, + "tier": 0, + "w": 1280, + "net": null + }, + { + "t": 10, + "fps": 3, + "mbps": 0.02, + "content_p50": 20.9, + "content_p95": 24.7, + "bitrate": 1725204, + "tier": 0, + "w": 1280, + "net": null + } + ], + "adapt": [], + "content_p50": 15.1, + "content_p95": 80.3, + "input_p50": 43.3, + "input_p95": 98.4, + "grades": { + "input_replies": "local", + "content_p50": "local", + "content_p95": "acceptable", + "input_p50": "local", + "input_p95": "acceptable", + "fps": "bad", + "late_pct": "bad", + "stall_max": "acceptable" + }, + "controller": { + "target": 3442081, + "ceiling": 5529600, + "tier": 0, + "fps": 60, + "scale": 1.0, + "baseRtt": 13.334, + "inFlight": 1, + "slack": 71.738, + "adapt": true + }, + "events": [ + { + "t": 3.77, + "e": "target 2764800 bit/s; tier 0" + }, + { + "t": 7.42, + "e": "target 1725204 bit/s; tier 0" + } + ], + "source_fps": 60.1, + "cpu": { + "host": "not sampled" + }, + "viewer": "h264 software; ANGLE (Mesa, zink Vulkan 1.4(Turnip Adreno (TM) 750 (MESA_TURNIP)), OpenGL 4.6); raf 71 Hz", + "show_s": 1.88, + "panel": "valve.steam.desktopgame.2001932457", + "src": "test", + "route": "network" + } + ], + "proof": "Synthetic x264 PC agent runs on the Frame itself, forwarded via Mac and back. Not Windows/Linux desktop capture or laser input." +} \ No newline at end of file diff --git a/desktop/capture.c b/desktop/capture.c index 07c9379..06e146d 100644 --- a/desktop/capture.c +++ b/desktop/capture.c @@ -85,6 +85,15 @@ FC_API const char *fc_capture_error(Capture *c) {return c->error;} FC_API void fc_capture_bitrate(Capture *c,int bps) { g_object_set(c->encoder,"bitrate",(guint)MAX(1,bps/1000),NULL); } +FC_API void fc_capture_test(Capture *c,uint32_t input) { + GstElement *source=gst_bin_get_by_name(GST_BIN(c->pipeline),"source"); + if(source) { + /* Each benchmark click visibly changes the ball, before injection is + * timestamped. This is a response, not merely a protocol input echo. */ + guint color=0xff000000u | ((input*2654435761u)&0x00ffffffu); + g_object_set(source,"foreground-color",color,NULL);gst_object_unref(source); + } +} FC_API void fc_capture_key(Capture *c) { GstPad *p=gst_element_get_static_pad(c->encoder,"src"); gst_pad_send_event(p,gst_video_event_new_upstream_force_key_unit(GST_CLOCK_TIME_NONE,TRUE,0));gst_object_unref(p); diff --git a/desktop/controller.c b/desktop/controller.c index 2ab7aad..12dd8fc 100644 --- a/desktop/controller.c +++ b/desktop/controller.c @@ -92,19 +92,19 @@ int fc_update(FCController *c,int64_t now) { int64_t age=c->flight.n ? now-c->flight.a[0].t : 0; int delivered=rate(&c->acked),sending=rate(&c->sent); if(c->sent.n) c->frame_bytes=sending/16/c->sent.n; - int demand=MIN(c->captures.n,MIN(c->max_fps,fps[c->tier]))*c->frame_bytes*8; + int demand=(int)MIN((int64_t)MIN(c->captures.n,MIN(c->max_fps,fps[c->tier]))*c->frame_bytes*8,2147483647); int signal=(sending>=c->target/2 && queue>40000)||c->held>=3||age>baseline+100000; int congested=signal && c->signal;c->signal=signal;c->held=0; if(congested && now-c->decrease>300000) { - int next=MAX(300000,MIN(c->target*4/5,MAX(delivered*9/10,c->target/2))); - if(demand>0 && demand*2<=c->target*5/4) next=MAX(next,MIN(c->target,demand*2)); + int next=MAX(300000,MIN((int64_t)c->target*4/5,MAX((int64_t)delivered*9/10,c->target/2))); + if(demand>0 && (int64_t)demand*2<=(int64_t)c->target*5/4) next=MAX(next,MIN(c->target,(int64_t)demand*2)); c->target=next;c->decrease=now; } else if(!congested && now-c->decrease>1000000 && now-c->increase>250000 && c->targetceiling && - (sending>c->target*6/10 || now-c->decrease>3000000)) { - c->target=MIN(c->ceiling,(int)(c->target*1.1)+50000);c->increase=now; + (sending>(int64_t)c->target*6/10 || now-c->decrease>3000000)) { + c->target=MIN(c->ceiling,(int64_t)(c->target*1.1)+50000);c->increase=now; } double share=(double)c->target/MAX(c->ceiling,1); - if(c->tier<4 && sharetier] && sending>=c->target*7/10) { + if(c->tier<4 && sharetier] && sending>=(int64_t)c->target*7/10) { if(!c->below)c->below=now; if(now-c->below>500000) { for(c->tier=0;c->tier<4 && sharetier];c->tier++) {} diff --git a/desktop/portal.c b/desktop/portal.c index 0254ba3..cdf7cc7 100644 --- a/desktop/portal.c +++ b/desktop/portal.c @@ -65,6 +65,20 @@ FC_API void fc_portal_close(Portal *p) { g_free(p->session);if(p->bus)g_object_unref(p->bus); if(p->context)g_main_context_unref(p->context);g_free(p); } +/* Each pipeline gets a fresh restricted PipeWire connection. A dup of a + * previously consumed protocol socket is not a new connection. */ +FC_API int fc_portal_refresh(Portal *p,char *error,int capacity) { + GVariantBuilder b;g_variant_builder_init(&b,G_VARIANT_TYPE_VARDICT); + GUnixFDList *fds=NULL;GError *e=NULL; + GVariant *r=g_dbus_connection_call_with_unix_fd_list_sync(p->bus,BUS,PATH,SC,"OpenPipeWireRemote", + g_variant_new("(oa{sv})",p->session,&b),G_VARIANT_TYPE("(h)"),G_DBUS_CALL_FLAGS_NONE,10000,NULL,&fds,NULL,&e); + if(!r){g_strlcpy(error,e->message,capacity);g_error_free(e);return -1;} + int handle;g_variant_get(r,"(h)",&handle);g_variant_unref(r); + int fd=g_unix_fd_list_get(fds,handle,&e);g_object_unref(fds); + if(fd<0){g_strlcpy(error,e->message,capacity);g_error_free(e);return -1;} + if(p->fd>=0)close(p->fd); + p->fd=fd;return fd; +} FC_API Portal *fc_portal_select(char *error,int capacity) { Portal *p=g_new0(Portal,1);p->fd=-1;p->context=g_main_context_new(); g_main_context_push_thread_default(p->context); @@ -99,13 +113,7 @@ FC_API Portal *fc_portal_select(char *error,int capacity) { } if(streams)g_variant_unref(streams);g_variant_unref(r);r=NULL; if(!p->node || p->width<=0 || p->height<=0) {g_strlcpy(error,"Portal returned no stream size",capacity);goto fail;} - g_variant_builder_init(&b,G_VARIANT_TYPE_VARDICT);GUnixFDList *fds=NULL; - r=g_dbus_connection_call_with_unix_fd_list_sync(p->bus,BUS,PATH,SC,"OpenPipeWireRemote", - g_variant_new("(oa{sv})",p->session,&b),G_VARIANT_TYPE("(h)"),G_DBUS_CALL_FLAGS_NONE,10000,NULL,&fds,NULL,&e); - if(!r){g_strlcpy(error,e->message,capacity);g_error_free(e);goto fail;} - int handle;g_variant_get(r,"(h)",&handle);g_variant_unref(r); - p->fd=g_unix_fd_list_get(fds,handle,&e);g_object_unref(fds); - if(p->fd<0){g_strlcpy(error,e->message,capacity);g_error_free(e);goto fail;} + if(fc_portal_refresh(p,error,capacity)<0)goto fail; g_main_context_pop_thread_default(p->context);return p; fail: g_main_context_pop_thread_default(p->context);fc_portal_close(p);return NULL; diff --git a/docs/pc-in-headset.md b/docs/pc-in-headset.md index 59c3784..0841095 100644 --- a/docs/pc-in-headset.md +++ b/docs/pc-in-headset.md @@ -78,11 +78,12 @@ window resize/minimize, and non-US keyboard layouts. - Mac virtual-display separation remains Mac-only. Windows WGC and the Linux portal share the selected window directly. -Current adaptation limitation: PC gating follows the shared frame-rate tier. -Live bitrate updates are applied to x264. Hardware encoders retain their -initial bitrate, and PC resolution does not yet follow the controller's -scale tier. This is an explicit remaining gap, not a measured performance -claim. +PC capture follows the shared frame-rate and resolution tiers. x264 updates +bitrate while running. Hardware encoders are drained and reopened when the +budget changes materially, at most once a second, because their live property +support varies. Reopening starts a new keyframe and retains the consented +portal session. **Untested:** hardware reconfiguration latency and whether a +particular desktop permits reconnecting its PipeWire stream this way. ## Build and measure @@ -124,8 +125,36 @@ same bounded shaping relay, without administrator privileges. latency p50/p95 83/156.5 ms. This establishes the existing viewer/transport route, not Windows/Linux capture, input or a latency target. The probe's helper, tunnel and viewer were stopped afterward. -- **CI, pending:** Ubuntu x64/ARM64 and Windows native library builds and - x264 test-pattern protocol tests. No Windows or Linux desktop VM was used. +- **Verified in CI:** native library builds and real x264 protocol tests passed + on Windows, Ubuntu x64 and Ubuntu ARM64 in + [run 36422214445](https://github.com/saphid/frame-control/actions/runs/36422214445). + All four installer builds passed in + [run 36422214425](https://github.com/saphid/frame-control/actions/runs/36422214425). + These are build/synthetic tests, not desktop-host verification. No VM was used. +- **Verified, real Frame, same build/date:** the ARM64 PC agent and its + bundled libraries ran from a temporary user directory, using x264's moving + test pattern. Traffic travelled Frame → Mac SSH relay → Frame viewer. + The shared bench recorded 351 drawn frames, content p50/p95 15.1/80.3 ms, + and 34.8 fps. The frame-rate and late-frame targets failed. This checks the + new agent and real viewer together; it is not a representative PC link. + The helper exited 0, its viewer/tunnels stopped, and its directory was removed. + [Raw benchmark result](../bench/results/2026-09-28-c5fc552-dirty-pc-agent-frame-arm64-hairpin.json). + This first probe's input echo measured message receipt to the next capture, + not a visible pattern response; later builds make test clicks change its color. +- **Verified, Mac:** all 600 states in a 60-second congestion/recovery trace + matched the original Swift controller. `tests/test_pc_controller.py` retains + the original trace digest as a regression check. - **Untested:** real Windows WGC → Media Foundation → Frame; real Linux portal → PipeWire → VA-API/x264 → Frame; physical laser input on either. No benchmark numbers for those desktop paths are claimed. + +## Independent review availability + +A direct read-only review was attempted with +`devin -p --model swe-2-max --permission-mode auto --prompt-file …`. The first +attempt exited 0 after rejecting a tool that needed interactive permission; +it did not inspect the diff. A full inline-diff attempt returned no output +for 15 minutes and was terminated (shell exit 143). A smaller inline native +code review returned no output within 300 seconds (process exit -15). +SWE-2 Max was requested; no completed review or findings were received, so +independent review is **unverified**, not a passed check. The PR remains draft. diff --git a/scripts/macview-bench.py b/scripts/macview-bench.py index 4019d9d..ac4a332 100755 --- a/scripts/macview-bench.py +++ b/scripts/macview-bench.py @@ -255,7 +255,10 @@ class LabView(frame_macview.MacView): def ssh_runner(base): def run(remote, stdin=None, timeout=30): - r = subprocess.run([*base, remote], input=stdin, capture_output=True, text=True, timeout=timeout) + r = subprocess.run([*base, remote], input=stdin.encode("utf-8") if stdin is not None else None, + capture_output=True, timeout=timeout) + r.stdout = r.stdout.decode("utf-8", errors="replace") + r.stderr = r.stderr.decode("utf-8", errors="replace") if r.returncode: e = RuntimeError((r.stderr or r.stdout).strip()) e.stdout = r.stdout @@ -386,6 +389,7 @@ def run_scenario(args, scenario, agent_port, token, frame_ssh): cpu = None if args.pc: mv.host = "windows" if sys.platform == "win32" else "linux" + mv.viewer_profile = "pc-view" try: if scenario in ("scroll", "type"): chrome = chrome_window(f"{scenario}.html") @@ -754,7 +758,7 @@ def document(args, frame_ssh, results, **extra): commit = git("rev-parse", "--short", "HEAD") + ("+dirty" if git("status", "--porcelain", "--", "mac", "ui") else "") return { "label": args.label or args.cmd, "date": datetime.datetime.now().isoformat(timespec="seconds"), "commit": commit, - "config": {"quality": args.quality, "mode": args.mode, "net": args.net or "none", "delay_ms": args.delay, + "config": {"quality": args.quality, "mode": "native" if args.pc else args.mode, "net": args.net or "none", "delay_ms": args.delay, "buffer_ms": args.buffer, "host": args.host or args.frame, "usb": args.usb, "ssh_opts": args.ssh_opt, "encoder": os.environ.get("FRAME_MAC_VIEW_ENCODER", ""), "duration_s": args.duration, "browser_flags": args.browser_flag if args.browser_flag is not None else frame_macview.BROWSER_FLAGS}, diff --git a/tests/test_macview.py b/tests/test_macview.py index 6d72a7c..213860b 100644 --- a/tests/test_macview.py +++ b/tests/test_macview.py @@ -44,7 +44,10 @@ class Helpers(unittest.TestCase): @unittest.skipUnless(shutil.which("bash"), "needs bash") def test_launch_script_parses(self): - r = subprocess.run(["bash", "-n"], input=frame_macview.LAUNCH, text=True, capture_output=True) + bash = str(Path(os.environ.get("ProgramFiles", "C:/Program Files")) / "Git/bin/bash.exe") if sys.platform == "win32" else "bash" + if sys.platform == "win32" and not Path(bash).exists(): + self.skipTest("Git Bash is not installed (WSL bash is not a local shell)") + r = subprocess.run([bash, "-n"], input=frame_macview.LAUNCH.encode("utf-8"), capture_output=True) self.assertEqual(r.returncode, 0, r.stderr) def test_show_checks_the_source_before_anything_else(self): diff --git a/tests/test_pc_controller.py b/tests/test_pc_controller.py new file mode 100644 index 0000000..07fbb34 --- /dev/null +++ b/tests/test_pc_controller.py @@ -0,0 +1,64 @@ +"""Frozen regression trace from the original mac-in-headset Swift controller. +The 600-state reference was produced at 1b90c64, before the C extraction: +60 seconds with a slow/congested middle, then recovery. No generated behavior +from the new implementation is used to calculate the expected digest. +""" +import ctypes as C +import hashlib +import json +from pathlib import Path +import shutil +import subprocess +import sys +import tempfile +import unittest + +ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(ROOT / 'ui')) +from frame_pc_capture import Controller + + +class ControllerTrace(unittest.TestCase): + def test_original_mac_congestion_and_recovery(self): + cc = shutil.which('cc') or shutil.which('gcc') + if not cc: + self.skipTest('C compiler not installed; native build jobs cover this trace') + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / ('trace.dll' if sys.platform == 'win32' else 'trace.so') + subprocess.run([cc, '-O2', '-shared', '-fPIC', str(ROOT/'desktop/controller.c'), '-o', str(path)], check=True, capture_output=True) + lib = C.CDLL(str(path)) + sig = {'new': (C.c_void_p, [C.c_int, C.c_int]), 'free': (None, [C.c_void_p]), + 'ceiling': (None, [C.c_void_p, C.c_int]), 'value': (C.c_int64, [C.c_void_p, C.c_int]), + 'capture': (None, [C.c_void_p, C.c_int64]), 'gate': (C.c_int, [C.c_void_p, C.c_int64, C.c_int]), + 'sent': (None, [C.c_void_p, C.c_uint32, C.c_int, C.c_int64]), + 'ack': (C.c_int, [C.c_void_p, C.c_uint32, C.c_int64]), 'update': (C.c_int, [C.c_void_p, C.c_int64])} + for name, (ret, args) in sig.items(): + fn = getattr(lib, 'fc_'+name) + fn.restype, fn.argtypes = ret, args + c = Controller(lib, 60, 8000000) + due, states, seq = [], [], 0 + try: + for tick in range(6000): + now = 10000000+tick*10000 + for t, s in due: + if t <= now: + c.call('ack', s, now) + due = [(t, s) for t, s in due if t > now] + if tick % 2 == 0: + c.call('capture', now) + if c.call('gate', now, 1): + seq += 1 + c.call('sent', seq, 8000 if tick < 1000 or tick > 4000 else 22000, now) + due.append((now+(240000 if 1500 < tick < 3500 else 20000), seq)) + if tick % 10 == 0: + c.update(now) + x = c.state() + states.append([x[k] for k in ('target', 'ceiling', 'tier', 'fps', 'inFlight')] + + [round(x['scale']*100), round(x['baseRtt']*1000), round(x['slack']*1000)]) + digest = hashlib.sha256(json.dumps(states, separators=(',', ':')).encode()).hexdigest() + self.assertEqual(digest, '1932f16aafce5c9157bcd2aba5d90719a366f9999422920aaeae78a6cdec08f6') + finally: + c.close() + if sys.platform == 'win32': + import _ctypes + _ctypes.FreeLibrary(lib._handle) diff --git a/tests/test_pcview.py b/tests/test_pcview.py index 6af5734..ba119b1 100644 --- a/tests/test_pcview.py +++ b/tests/test_pcview.py @@ -105,6 +105,17 @@ class AdapterTests(unittest.TestCase): with self.assertRaises(frame_macview.MacViewError): view.show('separate:12') + def test_session_end_releases_native_input_only_once(self): + session = object.__new__(agent.Session) + session.lock, session.input_lock = threading.RLock(), threading.RLock() + session.stop_event = threading.Event() + session.ws, session.input = mock.Mock(), mock.Mock() + session.released = False + session.end() + session.end() + session.input.release.assert_called_once() + self.assertTrue(session.stop_event.is_set()) + def test_stats_keep_capture_time_and_bound_records(self): stats = Stats(lambda: 10000000) stats.input(dict(t='m', i=1, tv=9999990)) diff --git a/ui/frame_macview.py b/ui/frame_macview.py index ee95686..fcd56f6 100644 --- a/ui/frame_macview.py +++ b/ui/frame_macview.py @@ -56,12 +56,12 @@ BROWSER_FLAGS = [] PANEL_BOX = (1920, 1080) # Opens one viewer on gamescope's X display and gives its window its own panel -# id. Args: appid url width height tag [browser flags...]. The page puts "[tag]" in its title at +# id. Args: appid url width height tag profile [browser flags...]. The page puts "[tag]" in its title at # once, which is how its X window is found (Chromium may hand the URL to an # instance that's already running, so there's no process to follow). LAUNCH = r"""set -u -appid=$1 url=$2 w=$3 h=$4 tag=$5 -shift 5 +appid=$1 url=$2 w=$3 h=$4 tag=$5 profile=$6 +shift 6 export DISPLAY=:0 LC_ALL=C.UTF-8 unset WAYLAND_DISPLAY if ! xprop -root GAMESCOPE_FOCUSABLE_WINDOWS >/dev/null 2>&1; then @@ -73,15 +73,15 @@ common=(--ozone-platform=x11 --force-device-scale-factor=1 --no-first-run --no-d --disable-features=Translate,MediaRouter --autoplay-policy=no-user-gesture-required "--window-size=$w,$h" "$@" "--app=$url") if [ -x "$HOME/chromium-xr/chrome" ]; then - cmd=("$HOME/chromium-xr/chrome" "--user-data-dir=$HOME/.local/share/frame-control/mac-view" "${common[@]}") + cmd=("$HOME/chromium-xr/chrome" "--user-data-dir=$HOME/.local/share/frame-control/$profile" "${common[@]}") elif flatpak info org.chromium.Chromium >/dev/null 2>&1; then cmd=(flatpak run org.chromium.Chromium - "--user-data-dir=$HOME/.var/app/org.chromium.Chromium/data/frame-mac-view" "${common[@]}") + "--user-data-dir=$HOME/.var/app/org.chromium.Chromium/data/frame-$profile" "${common[@]}") else echo "NO_BROWSER" exit 3 fi -log=/tmp/frame-mac-view.log +log=/tmp/frame-$profile.log setsid nohup "${cmd[@]}" >>"$log" 2>&1 = 3 or arrived-self.last_submit < 1000000/state['fps'] or not self.controller.call('gate', arrived, 1): self.stats.skipped += 1 return 0 self.pending[pts] = dict(cap=capture, arr=arrived, e0=arrived, tier=state['tier'], br=state['target']) @@ -164,10 +169,20 @@ class Session: self.stop_event.set() return 0 + def fresh_pipewire(self): + if 'portal' not in self.source: + return + error = C.create_string_buffer(1024) + fd = self.native.lib.fc_portal_refresh(self.source['portal'], error, len(error)) + if fd < 0: + raise RuntimeError(error.value.decode(errors='replace')) + self.source['fd'] = fd + def produce(self): native, capture = self.native.lib, None try: encoder = "x264enc" if self.src == "test" and self.native.has("x264enc") else self.agent.encoder + self.fresh_pipewire() description = pipeline(self.source, sys.platform, encoder, self.w, self.h, self.fps, self.bitrate, self.codec) self.callback = GATE(self.gate) error = C.create_string_buffer(1024) @@ -175,8 +190,14 @@ class Session: if not capture: raise RuntimeError(error.value.decode(errors='replace')) last_update = last_stats = self.native.now() - last_output = last_update + last_output = last_config = last_update + applied = (self.w, self.h, self.bitrate) + wanted = applied while not self.stop_event.is_set(): + while not self.test_inputs.empty(): + event = self.test_inputs.get_nowait() + native.fc_capture_test(capture, int(event.get('i', 0)) & 0xffffffff) + self.stats.input(event) output = Encoded() result = native.fc_capture_pull(capture, C.byref(output)) now = self.native.now() @@ -185,9 +206,9 @@ class Session: if result: last_output = now with self.lock: - # Exactly one raw frame is in flight and B-frames are disabled. - # x264 may offset PTS; associate by that single frame, - # keeping the actual pre-encode capture timestamp. + # At most three raw frames are in flight; no B-frames. + # x264 offsets PTS, so match the FIFO encode order while + # retaining the actual pre-encode capture timestamp. raw_pts = next(iter(self.pending), None) record = self.pending.get(raw_pts) if record is None: @@ -208,12 +229,36 @@ class Session: native.fc_capture_key(capture) if now-last_update >= 100000: target = self.controller.update(now) - # x264 supports live bitrate changes. Hardware elements - # advertise their mutability; the initial bitrate always - # applies. Frame gating remains active for every encoder. + state = self.controller.state() + w, h = dimensions(self.w, self.h, max(320, int(max(self.w, self.h)*state['scale']))) if target and self.codec == 'h264' and encoder == 'x264enc': native.fc_capture_bitrate(capture, target) + self.bitrate = target + # Hardware properties are not uniformly mutable in PLAYING. + # Drain then reopen the pipeline at a keyframe when its + # budget changes materially. The portal fd/session stays + # alive, so this does not bypass or repeat user consent. + desired_bitrate = target or applied[2] + bitrate_change = self.codec == 'h264' and encoder != 'x264enc' and abs(desired_bitrate-applied[2]) > applied[2]*.2 + if not self.reconfiguring and now-last_config >= 1000000 and ((w, h) != applied[:2] or bitrate_change): + wanted = (w, h, desired_bitrate) + with self.lock: + self.reconfiguring = True last_update = now + if self.reconfiguring: + with self.lock: + drained = not self.pending + if drained: + native.fc_capture_close(capture) + capture = None + self.fresh_pipewire() + description = pipeline(self.source, sys.platform, encoder, wanted[0], wanted[1], self.fps, wanted[2], self.codec) + capture = native.fc_capture_open(description.encode(), self.callback, error, len(error)) + if not capture: + raise RuntimeError(error.value.decode(errors='replace')) + applied, self.bitrate, last_config = wanted, wanted[2], now + with self.lock: + self.reconfiguring = False if now-last_stats >= 1000000: self.ws.send(dict(self.stats.summary(), t='stats', bitrate=self.bitrate, size='%dx%d' % (self.w, self.h), tier=self.controller.state()['tier'])) @@ -235,7 +280,7 @@ class Session: return self.ws.send(dict(t='hello', r=self.key)) self.ws.send(dict(t='info', src=self.src, title=self.source.get('title', self.source.get('name', 'Test pattern')), - app='PC', codec=self.codec, input=self.src == 'test' or self.source.get('devices', 3) == 3, + app='PC', codec=self.codec, input=self.input_enabled, inputMessage='Allow pointer and keyboard control in the host sharing dialog.', aspect=self.w/self.h, warm=0)) with self.lock: @@ -264,8 +309,23 @@ class Session: if self.stop_event.is_set(): break if self.input: - self.input.handle(m) - self.stats.input(m) + if not self.input_enabled and t != 'release': + continue + try: + self.input.handle(m) + self.stats.input(m) + except RuntimeError as e: + self.input_enabled = False + try: + self.input.release() + except RuntimeError: + pass + self.ws.send(dict(t='error', message=str(e))) + elif m.get('i'): + try: + self.test_inputs.put_nowait(m) + except queue.Full: + pass finally: self.end() self.writer.join(6) @@ -277,7 +337,8 @@ class Session: self.stop_event.set() self.ws.close() with self.input_lock: - if self.input: + if self.input and not self.released: + self.released = True try: self.input.release() except (RuntimeError, OSError): @@ -294,6 +355,7 @@ class Agent: self.windows = Windows() if sys.platform == 'win32' else None self.encoder = 'mfh264enc' if self.windows else 'vah264enc' if native.has('vah264enc') else 'x264enc' self.shutting_down = False + self.selection_generation = 0 def lists(self): if self.windows: @@ -316,6 +378,7 @@ class Agent: if len(self.sources) >= 8: raise ValueError('Stop a panel before sharing another source') self.selecting, self.selection_error = True, '' + generation = self.selection_generation def choose(): error = C.create_string_buffer(1024) portal = self.native.lib.fc_portal_select(error, len(error)) @@ -323,7 +386,7 @@ class Agent: self.selecting = False if not portal: self.selection_error = error.value.decode(errors='replace') - elif self.shutting_down: + elif self.shutting_down or generation != self.selection_generation: self.native.lib.fc_portal_close(portal) else: fd, node, w, h, devices = [self.native.lib.fc_portal_value(portal, i) for i in range(5)] @@ -334,6 +397,8 @@ class Agent: def stop(self, src=None): with self.lock: + if src is None: + self.selection_generation += 1 self.grants.revoke(src) sessions = [s for s in self.sessions.values() if src is None or s.src == src] for session in sessions: @@ -452,8 +517,19 @@ class Handler(BaseHTTPRequestHandler): # under the same lock as redemption, before any capture starts. session = Session(agent, WebSocket(self), key, source, query) for old in list(agent.sessions.values()): - if old.key == key: + if old.key == key or old.src == source['src']: + if old.key != key: + agent.grants.keys.pop(old.key, None) + try: + old.ws.send(dict(t='close')) + except OSError: + pass old.end() + if old.writer: + old.writer.join(6) + if old.writer.is_alive(): + session.controller.close() + raise ValueError('The previous capture is still stopping; retry shortly') ident = agent.next_id agent.next_id += 1 agent.sessions[ident] = session @@ -468,6 +544,11 @@ class Handler(BaseHTTPRequestHandler): except (OSError, ValueError, RuntimeError): session.end() finally: + session.end() + if session.writer: + session.writer.join(6) + if not session.writer or not session.writer.is_alive(): + session.controller.close() with agent.lock: agent.sessions.pop(ident, None) self.close_connection = True diff --git a/ui/frame_pc_capture.py b/ui/frame_pc_capture.py index a4a0bb3..da002d5 100644 --- a/ui/frame_pc_capture.py +++ b/ui/frame_pc_capture.py @@ -45,12 +45,14 @@ class Native: 'fc_capture_pull': (C.c_int, [C.c_void_p, C.POINTER(Encoded)]), 'fc_capture_error': (C.c_char_p, [C.c_void_p]), 'fc_capture_bitrate': (None, [C.c_void_p, C.c_int]), - 'fc_capture_key': (None, [C.c_void_p]), 'fc_capture_close': (None, [C.c_void_p]), + 'fc_capture_key': (None, [C.c_void_p]), + 'fc_capture_test': (None, [C.c_void_p, C.c_uint32]), 'fc_capture_close': (None, [C.c_void_p]), } if sys.platform.startswith('linux'): signatures.update({ 'fc_portal_select': (C.c_void_p, [C.c_char_p, C.c_int]), 'fc_portal_close': (None, [C.c_void_p]), + 'fc_portal_refresh': (C.c_int, [C.c_void_p, C.c_char_p, C.c_int]), 'fc_portal_value': (C.c_int, [C.c_void_p, C.c_int]), 'fc_portal_input': (C.c_int, [C.c_void_p, C.c_int, C.c_double, C.c_double, C.c_int, C.c_int]), }) @@ -113,7 +115,7 @@ def dimensions(w, h, maximum): def pipeline(source, platform, encoder, w, h, fps, bitrate, codec='h264'): """Only locally constructed numeric source IDs enter the pipeline parser.""" if source['src'] == 'test': - capture = 'videotestsrc is-live=true pattern=ball' + capture = 'videotestsrc name=source is-live=true pattern=ball' elif platform == 'win32': kind, ident = source['src'].split(':') if kind not in ('window', 'display') or not re.fullmatch(r'[0-9]+', ident): @@ -124,7 +126,7 @@ def pipeline(source, platform, encoder, w, h, fps, bitrate, codec='h264'): capture = 'pipewiresrc fd=%d path=%d do-timestamp=true' % (source['fd'], source['node']) # The source gate runs before conversion/encoding. There is no leaky queue # of H.264 frames; every encoded reference frame reaches the socket. - raw = '%s ! video/x-raw,framerate=%d/1 ! identity name=gate ! videoconvert ! videoscale ! video/x-raw,width=%d,height=%d' % (capture, fps, w, h) + raw = '%s ! video/x-raw,framerate=%d/1 ! identity name=gate ! videoconvert ! videoscale add-borders=false ! video/x-raw,width=%d,height=%d' % (capture, fps, w, h) if codec == 'jpeg': enc = 'jpegenc name=enc quality=80' parse = '' @@ -158,12 +160,16 @@ class PortalInput: raise RuntimeError('The desktop portal refused input; check the sharing permission') def release(self): - for b in list(self.buttons): - self.emit(1, code=b, down=0) - self.buttons.discard(b) - for k in list(self.keys): - self.emit(3, code=k, down=0) - self.keys.discard(k) + error = None + for values, kind in ((self.buttons, 1), (self.keys, 3)): + for code in list(values): + try: + self.emit(kind, code=code, down=0) + values.discard(code) + except RuntimeError as e: + error = e + if error: + raise error def handle(self, m): t = m['t'] @@ -282,12 +288,22 @@ class WindowsInput: self.buttons, self.keys = set(), set() def release(self): + error = None for button in list(self.buttons): - self.host.send(mouse=(0, 0, 0, {0: 4, 1: 0x40, 2: 0x10}[button])) - self.buttons.discard(button) + try: + self.host.send(mouse=(0, 0, 0, {0: 4, 1: 0x40, 2: 0x10}[button])) + self.buttons.discard(button) + except RuntimeError as e: + error = e for vk in list(self.keys): - self.host.send(key=(vk, 0, 2)) - self.keys.discard(vk) + try: + extended = 1 if vk in (33, 34, 35, 36, 37, 38, 39, 40, 45, 46, 91, 92, 163, 165) else 0 + self.host.send(key=(vk, 0, 2 | extended)) + self.keys.discard(vk) + except RuntimeError as e: + error = e + if error: + raise error def handle(self, m): t, u = m['t'], self.host.user @@ -328,7 +344,8 @@ class WindowsInput: if re.fullmatch(r'Key[A-Z]', code) or re.fullmatch(r'Digit[0-9]', code): vk = ord(code[-1]) if vk: - self.host.send(key=(vk, 0, 0 if down else 2)) + extended = 1 if vk in (33, 34, 35, 36, 37, 38, 39, 40, 45, 46, 91, 92, 163, 165) else 0 + self.host.send(key=(vk, 0, extended | (0 if down else 2))) (self.keys.add if down else self.keys.discard)(vk) elif down and len(str(m.get('key', ''))) == 1: self.handle({'t': 'text', 's': m['key']}) diff --git a/ui/frame_pcview.py b/ui/frame_pcview.py index 317b61f..65c00bf 100644 --- a/ui/frame_pcview.py +++ b/ui/frame_pcview.py @@ -6,11 +6,13 @@ are different. No second viewer, SSH supervisor or benchmark launcher. """ import os import sys +import subprocess from frame_macview import MacView, MacViewError, ROOT from frame_pc_capture import LIBRARY, NATIVE class PCView(MacView): + viewer_profile = 'pc-view' host = 'windows' if sys.platform == 'win32' else 'linux' def __init__(self, *args, **kwargs): @@ -39,7 +41,9 @@ class PCView(MacView): env = super().agent_environment() env['GST_PLUGIN_PATH_1_0'] = str(NATIVE / 'lib' / 'gstreamer-1.0') env['GST_PLUGIN_SYSTEM_PATH_1_0'] = '' - env['GST_REGISTRY_1_0'] = str(NATIVE.parent / 'registry.bin') if os.access(NATIVE.parent, os.W_OK) else os.path.join(os.path.expanduser('~'), '.cache', 'frame-control-gst.bin') + cache = os.path.join(os.path.expanduser('~'), '.cache', 'frame-control') + os.makedirs(cache, exist_ok=True) + env['GST_REGISTRY_1_0'] = os.path.join(cache, 'gstreamer-registry.bin') env['GST_REGISTRY_FORK'] = 'no' if sys.platform == 'win32': env['PATH'] = str(NATIVE / 'bin') + os.pathsep + env.get('PATH', '') @@ -47,6 +51,25 @@ class PCView(MacView): env['LD_LIBRARY_PATH'] = str(NATIVE / 'lib') + os.pathsep + env.get('LD_LIBRARY_PATH', '') return env + def shutdown(self): + self.closing = True + try: + self.stop() + except MacViewError: + pass + if self.tunnel and self.tunnel.poll() is None: + self.tunnel.terminate() + if self.agent and self.agent.poll() is None: + # EOF gives the host a chance to release held input and portal + # sessions, including on Windows where terminate is uncatchable. + if self.agent.stdin: + self.agent.stdin.close() + try: + self.agent.wait(10) + except subprocess.TimeoutExpired: + self.agent.kill() + self.agent.wait(5) + def _show(self, src, quality, width, height): if src.startswith('separate:'): raise MacViewError('Separate virtual displays are a Mac-only feature. Choose a window or screen.') diff --git a/ui/server.py b/ui/server.py index fa53cac..9cb9633 100755 --- a/ui/server.py +++ b/ui/server.py @@ -1234,7 +1234,8 @@ def _sweep_one(prefix, d): # The tunnel gets its own connection: the shared master's options would win # over anything added after them. macview = frame_pcview.host_view(["ssh", "-o", "BatchMode=yes", "-o", "ConnectTimeout=8"], - lambda remote, stdin=None, timeout=30: ssh(remote, stdin=stdin, timeout=timeout), + lambda remote, stdin=None, timeout=30: ssh(remote, stdin=stdin.encode("utf-8") if stdin is not None else None, + timeout=timeout, text=False).decode("utf-8", errors="replace"), FRAME, track=_live_tunnels.add)