From 6a8e3fadbfea500f3d51a4bfcf6cd75366aaf149 Mon Sep 17 00:00:00 2001 From: saphid <4596216+saphid@users.noreply.github.com> Date: Mon, 28 Sep 2026 22:21:22 +1000 Subject: [PATCH] Open assistant on Frame and document verified agent workflows --- README.md | 1 + docs/agents.md | 153 ++++++++++++++++++++++++++++++++++ docs/frame-control.md | 10 +++ docs/testing.md | 8 ++ scripts/assistant-on-frame.py | 92 ++++++++++++++++++++ tests/e2e/test_agents.py | 46 ++++++++++ 6 files changed, 310 insertions(+) create mode 100644 docs/agents.md create mode 100644 scripts/assistant-on-frame.py create mode 100644 tests/e2e/test_agents.py diff --git a/README.md b/README.md index 6dd32be..d900cb7 100644 --- a/README.md +++ b/README.md @@ -204,6 +204,7 @@ Frame's software fits together, all checked against a real headset and labelled | [SSH](docs/ssh.md) · [Streaming](docs/streaming.md) · [Files](docs/file-transfer.md) · [Panels](docs/panels.md) · [Tailscale](docs/tailscale.md) | Topic notes | | [Frame Control for iPhone](docs/iphone.md) | The iPhone and iPad app, how it runs the server on the Frame, pairing | | [Recovery and OS images](docs/recovery-and-images.md) | Where to download the Frame's OS, what's inside, testing without the headset | +| [AI agents and assistant](docs/agents.md) | Key-free MCP tools, human approvals, and an opt-in assistant panel | | [Testing](docs/testing.md) | Unit tests, end-to-end tests against a fake Frame in Docker, and the headset smoke test | | [Open questions](docs/open-questions.md) | What's still unchecked | diff --git a/docs/agents.md b/docs/agents.md new file mode 100644 index 0000000..f01d315 --- /dev/null +++ b/docs/agents.md @@ -0,0 +1,153 @@ +# Frame Control for AI agents + +**Documented interface:** Frame Control's own stdlib Python MCP adapter wraps +its loopback HTTP API. No API key, hosted service, model SDK or third-party +helper app is needed. The assistant is our HTML/Python implementation hosted +in the platform Chromium browser. Its optional LLM endpoint is user configuration. +Installing other apps is an optional management action, never a prerequisite. + +## Connect an MCP client + +Start the HTTP server from this checkout: + +```sh +python3 ui/server.py --port 47810 +``` + +Add a stdio server to your MCP client (use absolute paths): + +```json +{ + "mcpServers": { + "frame-control": { + "command": "python3", + "args": ["/absolute/path/frame-control/ui/frame_mcp.py", "--url", "http://127.0.0.1:47810"] + } + } +} +``` + +The desktop app uses a random port; use that port with `--url`, or run the +checkout server above. If the HTTP server uses `FRAME_UI_KEY`, pass the same +value in the MCP process environment. This is local access control, not an LLM +API key. The adapter only accepts loopback HTTP servers, refuses redirects and +ignores environment proxies. Stdout contains newline-delimited JSON-RPC only. +It supports MCP initialization, ping, tool listing and tool calls; no sampling, +resources, prompts or streaming transport. + +| Tool | Arguments | Effect | +|---|---|---| +| `status` | none | Battery, services, installed games and Flatpaks | +| `screenshot` | `view`: `headset` (default) or `desktop` | Returns PNG image content to the MCP client | +| `job` | `id` | Background install status; poll until `done`, inspect `error` | +| `launch` | `appid` | Launch an installed Steam app | +| `install` / `uninstall` | `id` | Install from Flathub / remove a user Flatpak | +| `send_text` | `text` | Frame desktop clipboard; desktop must be open | +| `send_file` | `path` | File on the HTTP server computer, up to 16 MiB, copied to Frame `~/Downloads` | +| `panel` | `id` | Launch an installed Flatpak as a panel using the existing launcher | +| `power` | `action`: `suspend`, `reboot`, `poweroff` | Open a terminal for the user to enter the sudo password | +| `keep_awake` | `action`: `on`, `off`, `status` | Optional keep-awake script interface | + +Only install free software with its developer's consent. There is no purchase, +entitlement bypass or arbitrary shell tool. `install` returns a background job +ID; it does not claim the installation has finished. APK and sideloaded title +installs remain in the main UI for now. + +### Approval is a separate human action + +Every mutation first returns an `approvalUrl`, exact action and `confirmation` +token. Ask the user to open that URL and choose **Approve this action** or +**Reject**. Then repeat the same tool and arguments with the token in +`confirmation`. The server refuses execution before approval, changed arguments, +expired tokens and reuse. A file approval binds the content hash as well as the +path. Approvals last five minutes and disappear when the HTTP server restarts. +A failed execution also consumes the approval; review a fresh request to retry. +The panel does not execute an action merely because it was approved. + +MCP has no approval tool. This is protection against accidental model tool +calls, not a sandbox against a client with independent shell/HTTP access to your +computer. Grant the MCP client only the access you intend. Status and captures +are returned directly to that client, which may forward them to its configured +model. The assistant's separate opt-in does not govern an external MCP client. + +Power still requires the existing password prompt in a local terminal. MCP +never receives passwords. Power via `FRAME_LOCAL=1` is unsupported: use the main +UI. The panel launcher and keep-awake adapter require zsh on the computer. + +[PR #16](https://github.com/saphid/frame-control/pull/16) owns +`scripts/keep-awake.sh on|off|status`. This branch does not copy or change it. +Until that script is present, the tool reports it unavailable. Keep-awake is +never automatic: `on` changes the shared idle timers; explicitly approve `off` +to restore them after work. It is not a per-agent lease; coordinate with other +users. No changes are made to the analytics/update interfaces in +[PR #17](https://github.com/saphid/frame-control/pull/17). Prompts, keys, model +replies, screenshots and approval payloads are not sent to analytics. + +## Assistant panel + +Open **Tools → Open assistant**, or `http://127.0.0.1:47810/assistant`. +To put the same page in the headset, with the HTTP server still running: + +```sh +python3 scripts/assistant-on-frame.py --port 47810 +``` + +This starts an SSH reverse forward bound to Frame loopback (port 47812 by +default), then a dedicated Chromium profile tagged as a SteamVR panel. Keep the +command running. Ctrl-C closes this browser profile and the tunnel; it leaves +other Chromium windows and the existing HTTP server alone. A failed cleanup +prints the temporary profile path so it can be removed when the Frame returns. +Use `--frame-port` if the default is busy. Chromium must already be available as +`org.chromium.Chromium`; the launcher never installs anything automatically. +Place the panel with SteamVR's normal docking controls. + +Enter your full **chat-completions endpoint**, model name and optional key. +An OpenAI-compatible local server works without a key; no OpenAI account is +required. HTTP is allowed only on loopback; other endpoints require HTTPS. +Loopback refers to the computer running the HTTP server, even in the headset. +Endpoints with embedded credentials, query strings or redirects are refused. + +Check the message consent box and press **Send message**. Screenshot context is +a separate unchecked box and sends one fresh capture with that request. Both +boxes reset after sending, and changing endpoint/model revokes consent. Nothing +is sent when opening the page or entering configuration. There is no model +list fetch, saved history, automatic screenshot capture or assistant telemetry. +Each send is independent: previous messages and replies are not included. + +Configuration, credentials and chat remain in page memory; close/reload the page +or choose **Clear everything** to clear them. A request already sent cannot be +recalled. Only the chosen endpoint gets the request; proxy environment variables +and redirects are disabled. Its privacy and retention policy still applies. +Replies are plain text and cannot call tools or operate the Frame. A model must +support image inputs to accept screenshot context. + +## Evidence and limits + +**Verified 2026-09-28, SteamOS 0.4.1, BUILD_ID 20260925.6191901:** loopback HTTP +status through an SSH reverse tunnel; platform Chromium created a separate +SteamVR panel (confirmed in `GAMESCOPE_FOCUSABLE_APPS`); headset capture returned +a PNG. These checks preceded the UI implementation. No power or global settings +were changed. + +**Inferred:** visual comfort and controller keyboard usability while wearing +the headset; panel creation in gamescope alone does not establish these. +Windows/Linux launcher support, live third-party model endpoints, installs, +uninstalls, power and keep-awake changes are not covered by that feasibility +check. See the PR for the final unit and end-to-end results. + +**Verified end to end on the same Frame/build (2026-09-28):** a stdio MCP client +initialized, read status, retrieved a headset PNG, and transferred a test file +only after approval through the Chromium page. Remote file bytes matched; +reusing the confirmation was rejected. The actual headset Chromium page sent +text and then separately opted-in image context to a local test endpoint and +displayed its replies. Without consent there were zero endpoint requests. +The test endpoint returned canned replies: model inference and a live external +provider remain **unverified**. The launcher’s Ctrl-C cleanup was checked; +profiles, SSH tunnels and the test file were removed. No installs, removals, +launches of user games, power operations or keep-awake changes were performed. + +**Verified locally:** unit coverage includes the stdio subprocess, approval +binding/expiry/replay/concurrency, file-change rejection, and a real local HTTP +endpoint for opt-in, text/image payloads and redirect refusal. Fake-Frame +regressions are in `tests/e2e/test_agents.py`; local Docker execution was blocked +because the Docker daemon was unavailable. CI runs those regressions. diff --git a/docs/frame-control.md b/docs/frame-control.md index 299c57c..6e4c872 100644 --- a/docs/frame-control.md +++ b/docs/frame-control.md @@ -150,3 +150,13 @@ npm run dist:linux # Linux: AppImage and .deb, x64 and arm64 Pushing a `v*` tag builds all three in GitHub Actions and attaches them to the release (`.github/workflows/release.yml`). + +## AI agents and assistant + +**Documented:** [the MCP adapter and assistant panel](agents.md) are Frame +Control implementations. MCP wraps this HTTP API without API keys. Changes +require a separate user approval; power also retains its password prompt. The +assistant uses a user-chosen endpoint and sends nothing until the user opts in +for a message. Screenshot context is separately opt-in. Model replies cannot +operate the headset. Tools → Open assistant opens the page; the linked guide +covers putting it in a Chromium panel on the Frame. diff --git a/docs/testing.md b/docs/testing.md index b06577d..469440e 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -148,3 +148,11 @@ For example, on 2026-09-27 the smoke test found that Steam's `create-shortcut` refuses ids with a hyphen (`missing/invalid arguments`), which the fake had accepted. The fake now refuses them the same way, and Frame Control makes ids Steam accepts. + +## Agent interfaces + +`tests/test_agent.py` exercises MCP stdio, exact-action human approvals and the +assistant against an in-process HTTP endpoint with canned responses (no keys or +external calls). `tests/e2e/test_agents.py` runs the MCP/HTTP/SSH path against the +fake Frame for approved installs, clipboard and file transfer. Headset Chromium +rendering and real screenshots still need a device; see [agent evidence](agents.md#evidence-and-limits). diff --git a/scripts/assistant-on-frame.py b/scripts/assistant-on-frame.py new file mode 100644 index 0000000..7c8b59f --- /dev/null +++ b/scripts/assistant-on-frame.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python3 +"""Open Frame Control's assistant as a Chromium panel. Ctrl-C closes it and its SSH tunnel. + +Start ui/server.py first. Requires the platform Chromium Flatpak and zsh on the +computer (the existing panel launcher). No model endpoint or key is configured. +""" +import argparse +import os +from pathlib import Path +import re +import shlex +import shutil +import subprocess +import sys +import time +import uuid + +ROOT = Path(__file__).resolve().parent.parent + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--port', type=int, default=47810, help='local Frame Control port') + parser.add_argument('--frame-port', type=int, default=47812, help='Frame loopback tunnel port') + args = parser.parse_args() + alias = os.environ.get('FRAME_ALIAS', 'frame') + if not re.fullmatch(r'[A-Za-z0-9][A-Za-z0-9._-]*', alias) or any(not 1 <= p <= 65535 for p in (args.port, args.frame_port)): + parser.error('Invalid alias or port') + if not shutil.which('zsh'): + parser.error('The panel launcher requires zsh on this computer') + sys.path.insert(0, str(ROOT / 'ui')) + from frame_mcp import Client + Client('http://127.0.0.1:' + str(args.port), os.environ.get('FRAME_UI_KEY', '1')).request('/api/host') + profile = '/tmp/frame-control-assistant-' + uuid.uuid4().hex + tunnel = subprocess.Popen(['ssh', '-N', '-o', 'BatchMode=yes', '-o', 'ConnectTimeout=8', + '-o', 'ExitOnForwardFailure=yes', '-o', 'ServerAliveInterval=15', + '-o', 'ServerAliveCountMax=2', '-R', + f'127.0.0.1:{args.frame_port}:127.0.0.1:{args.port}', alias]) + try: + # Check the forwarded page before starting a browser; no arbitrary sleeps. + probe = subprocess.run(['ssh', '-o', 'BatchMode=yes', '-o', 'ConnectTimeout=8', alias, + 'curl --retry 5 --retry-connrefused --retry-delay 1 --max-time 10 -fsS ' + + shlex.quote(f'http://127.0.0.1:{args.frame_port}/assistant')], + stdout=subprocess.DEVNULL, timeout=30) + if probe.returncode or tunnel.poll() is not None: + raise RuntimeError('Could not forward Frame Control to the Frame') + subprocess.run(['zsh', str(ROOT / 'scripts/panel-on-frame.sh'), '--name', 'Frame Control Assistant', + 'org.chromium.Chromium', '--user-data-dir=' + profile, '--no-first-run', + '--disable-background-networking', '--disable-sync', + f'--app=http://127.0.0.1:{args.frame_port}/assistant'], check=True, timeout=45) + print('Assistant panel open. Ctrl-C closes this panel and its tunnel.', flush=True) + tunnel.wait() + raise RuntimeError('SSH tunnel ended') + except KeyboardInterrupt: + return 0 + finally: + tunnel.terminate() + try: + tunnel.wait(timeout=10) + except subprocess.TimeoutExpired: + tunnel.kill() + tunnel.wait() + # Only this unique browser profile, never a shared Chromium instance. + cleanup = '''import os, pathlib, signal, shutil, sys, time +profile = sys.argv[1] +needle = ('--user-data-dir=' + profile).encode() +owned = [] +for p in pathlib.Path('/proc').iterdir(): + try: + if p.name.isdigit() and p.stat().st_uid == os.getuid() and needle in (p / 'cmdline').read_bytes().split(b'\\0'): + owned.append(int(p.name)) + except OSError: + pass +for sig in (signal.SIGTERM, signal.SIGKILL): + for pid in owned: + try: os.kill(pid, sig) + except ProcessLookupError: pass + time.sleep(.3) +shutil.rmtree(profile, ignore_errors=True) +''' + result = subprocess.run(['ssh', '-o', 'BatchMode=yes', '-o', 'ConnectTimeout=8', alias, + 'python3 - ' + shlex.quote(profile)], input=cleanup, text=True, timeout=20) + if result.returncode: + print('Cleanup failed; close the assistant panel and remove ' + profile + ' on the Frame.', file=sys.stderr) + + +if __name__ == '__main__': + try: + sys.exit(main()) + except (OSError, RuntimeError, subprocess.SubprocessError) as exc: + print(str(exc), file=sys.stderr) + sys.exit(1) diff --git a/tests/e2e/test_agents.py b/tests/e2e/test_agents.py new file mode 100644 index 0000000..f02f338 --- /dev/null +++ b/tests/e2e/test_agents.py @@ -0,0 +1,46 @@ +"""Real HTTP/MCP adapter against fake-Frame SSH; no model service needed.""" +import json +from pathlib import Path +import sys + +import harness +from harness import api, ok, finished, ssh + +sys.path.insert(0, str(harness.ROOT / 'ui')) +import frame_mcp + + +class Agents(harness.FrameTestCase): + def client(self): + return frame_mcp.Client('http://127.0.0.1:%d' % harness.Server.port) + + def call(self, name, args): + return json.loads(frame_mcp.call(self.client(), name, args)['content'][0]['text']) + + def approve(self, proposal): + ok('POST', '/api/agent/approval', {'confirmation': proposal['confirmation'], 'accept': True}) + return proposal['confirmation'] + + def test_status_and_approved_install_job(self): + self.assertIn('battery', self.call('status', {})) + proposal = self.call('install', {'id': 'org.example.AgentTest'}) + before = api('POST', '/api/agent/call', {'name': 'install', 'arguments': {'id': 'org.example.AgentTest'}, 'confirmation': proposal['confirmation']}) + self.assertEqual(before[0], 400) + token = self.approve(proposal) + job = self.call('install', {'id': 'org.example.AgentTest', 'confirmation': token}) + self.assertFalse(finished(job).get('error')) + self.assertIn('org.example.AgentTest', ssh('flatpak list --app --columns=application')) + denied = api('POST', '/api/agent/call', {'name': 'install', 'arguments': {'id': 'org.example.AgentTest'}, 'confirmation': token}) + self.assertEqual(denied[0], 400) + + def test_approved_file_and_text(self): + path = Path(self.path('agent-note.txt')) + path.write_text('MCP file content\n') + args = {'path': str(path)} + token = self.approve(self.call('send_file', args)) + self.call('send_file', {**args, 'confirmation': token}) + self.assertEqual(ssh('cat ~/Downloads/agent-note.txt'), path.read_text()) + args = {'text': 'MCP clipboard text'} + token = self.approve(self.call('send_text', args)) + self.call('send_text', {**args, 'confirmation': token}) + self.assertEqual(harness.state()['clipboard'], ['MCP clipboard text'])