From b9e31178f3c7d81058a8da674f3bfc3fdf15ea3d Mon Sep 17 00:00:00 2001 From: saphid <4596216+saphid@users.noreply.github.com> Date: Mon, 28 Sep 2026 17:42:19 +1000 Subject: [PATCH] Keyboard and trackpad for the Frame, through its own KDE Connect MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Home → Keyboard and trackpad, in every version of Frame Control (Mac, Windows, Linux, iPhone, iPad) with nothing to install on the device in your hand. On a phone: a trackpad (drag, tap, two-finger scroll and right-click) and a field that types on the Frame. On a computer: click the pad to pass the mouse and keyboard through; Esc to stop. First-party route: ui/frame_input_agent.py runs on the Frame and talks KDE Connect's LAN protocol (v7) to kdeconnectd as if it were a phone. KDE Connect isn't on the image, but Valve's package repository has it; the agent fetches it and four libraries into ~ (no root, survives updates), starts it, pairs by itself (accepting over D-Bus), and stops it again when the last device disconnects. Each device has its own identity; a stuck KDE Connect is restarted once. Verified against the real Frame (SteamOS 0.4.1): first-time install, pairing, pointer moves from the Mac's server and the iPhone app (Simulator), Mac and iPhone at once, two installs at once, a frozen daemon replaced, and the daemon stopping when the app quits. Reviewed by GPT-6 Astra (xhigh) over seven rounds; all findings fixed except per-event delivery acknowledgement (documented known limit). Co-Authored-By: Claude Opus 5.5 (1M context) --- README.md | 3 + app/main.js | 1 + app/preload.js | 2 + docs/how-the-frame-works.md | 3 + docs/streaming.md | 67 +++- ios/FrameControl/App/AppModel.swift | 8 +- ios/FrameControl/SSH/HeadsetServer.swift | 4 +- tests/page/pad_queue.mjs | 48 +++ tests/test_input.py | 196 ++++++++++++ ui/frame_input_agent.py | 376 +++++++++++++++++++++++ ui/index.html | 260 ++++++++++++++++ ui/server.py | 161 ++++++++++ 12 files changed, 1118 insertions(+), 11 deletions(-) create mode 100644 tests/page/pad_queue.mjs create mode 100644 tests/test_input.py create mode 100644 ui/frame_input_agent.py diff --git a/README.md b/README.md index 5f25235..1fc7fc6 100644 --- a/README.md +++ b/README.md @@ -67,6 +67,9 @@ Drag files onto the window to send them. Drop a game's .zip, folder or .exe to a **📸 Screenshots**
Browse the shots you take in the headset and save them to your Pictures folder. +**⌨️ Keyboard and trackpad**
+Type and point in the Frame's apps from your computer or phone, through KDE Connect on the Frame. Nothing to install on the device in your hand. + diff --git a/app/main.js b/app/main.js index bf6ff66..06c10f1 100644 --- a/app/main.js +++ b/app/main.js @@ -244,6 +244,7 @@ function fromUi(e) { ipcMain.handle("clipboard:read", (e) => fromUi(e) ? clipboard.readText() : ""); ipcMain.handle("connection:setup", (e) => { if (fromUi(e)) setUpConnection(); }); +ipcMain.on("keys:capture", (e, on) => { if (fromUi(e)) win.webContents.setIgnoreMenuShortcuts(on === true); }); // frame-control://install links from websites (docs/web-install.md). They can // arrive before the window or server exists (macOS open-url on a cold launch), diff --git a/app/preload.js b/app/preload.js index 43bc34d..24065a4 100644 --- a/app/preload.js +++ b/app/preload.js @@ -10,6 +10,8 @@ const { contextBridge, ipcRenderer, webUtils } = require("electron"); contextBridge.exposeInMainWorld("frameApp", { readClipboard: () => ipcRenderer.invoke("clipboard:read"), setUpConnection: () => ipcRenderer.invoke("connection:setup"), + // While the keyboard-and-trackpad panel holds the keyboard, ⌘W, ⌘R and the rest go to the Frame. + captureKeys: (on) => ipcRenderer.send("keys:capture", !!on), pathForFile: (file) => { try { return webUtils.getPathForFile(file) || ""; } catch { return ""; } }, onInstallLink: (cb) => { ipcRenderer.removeAllListeners("install-link"); diff --git a/docs/how-the-frame-works.md b/docs/how-the-frame-works.md index 74f81f1..696854e 100644 --- a/docs/how-the-frame-works.md +++ b/docs/how-the-frame-works.md @@ -37,6 +37,9 @@ Lepton (Android 11, podman container "lepton-dev") ← its own panel, app 305600 | SteamVR settings live in `~/.config/openvr/config/steamvr.vrsettings`, not under `~/.local/share/Steam/config/`. `dashboard.lastAccessedExternalOverlayKey` names the last panel you used. | Settings tweaks | | The Steam client's journal (`journalctl --user`) carries SteamVR system UI lines such as `[Overlays] Created: …` and `vroverlay_uid`. It's the quickest way to see panels come and go. | Debugging | | Present: `rsync`, `flatpak`, `python3`, `git`, `qdbus6`, `xrdp`, `xprop`, `xwininfo`, `xterm`, `konsole`, `dolphin`, `gamescopectl`. Missing: `wl-copy`, `xclip`, `xsel`, `kdeconnect-cli`, `tailscale` (installable in `~`, see below), `krfb`, `wayvnc`. | Script design | +| **SteamOS updates arrive on their own.** The Frame went from 0.3.0 (build 20260922.6101926) to **0.4.1, build 20260925.6191901**, between 2026-09-27 and 2026-09-28 with no action from us; `~` (keys, user Flatpaks, `~/.local/share`) survived. **Verified 2026-09-28.** | Keep changes in `~` | +| **Valve's package repository has more than the image.** `pacman -Si` / `pacman -Sp` work as `steamos` without root and list Valve's own builds, such as `kdeconnect` 24.02.2 and `python-evdev` 1.7.0 in `extra`. Downloading those packages and unpacking them into `~` runs them without touching the read-only root. The repository URLs say not to share them, so fetch them on the Frame with `pacman -Sp` rather than writing them down. **Verified 2026-09-28**, SteamOS 0.4.1. | [streaming.md](streaming.md#input-type-and-point-in-the-frame-from-the-mac-or-iphone) | +| **gamescope runs two Xwayland displays.** `:0` holds Steam's VR bar and menus (`valve.steam.gamepadui.*`) and ignores XTest pointer motion; `:1` holds apps such as Chromium and takes it. There's also a libei socket, `/run/user/1000/gamescope-0-ei`. **Verified 2026-09-28**, SteamOS 0.4.1. | Keyboard and trackpad | | Flathub is a **system** remote. `--user` installs over SSH work and show up in the desktop menu. | `install-apps.sh` | | `/` is 10 GB and read-only. `/home` is 929 GB. | Where to put things | | Clipboard: Klipper over the nested D-Bus bus (`qdbus6 org.kde.klipper …`). | `paste-to-frame.sh` | diff --git a/docs/streaming.md b/docs/streaming.md index 8a1b448..a9a4f5f 100644 --- a/docs/streaming.md +++ b/docs/streaming.md @@ -66,18 +66,67 @@ capture it. ## Input: type and point in the Frame from the Mac or iPhone -**Verified 2026-09-27** on the headset: `steamos` is in the `input` group and -`/dev/uinput` is `crw-rw-r-- root input`, so **our own code can create a -virtual keyboard and mouse without sudo**. The Frame has no `python-evdev`, -`ydotool`, `wtype` or KDE Connect; `kwin_wayland` and `plasmashell` run only -while the desktop panel is open in the headset. +**Built: Home → Keyboard and trackpad**, in every version of Frame Control +(Mac, Windows, Linux, iPhone and iPad), with nothing to install on the device +you're holding. On a phone the panel is a trackpad (drag to move, tap to click, +two fingers to scroll, two-finger tap to right-click) plus a text field that +types on the Frame. On a computer, clicking the pad passes your mouse and +keyboard through to the Frame until you press Esc (⌘ is sent as Ctrl on a Mac). -| Option | Mac | iPhone | Notes | +It goes through **KDE Connect**, the first-party route (KDE makes the Frame's +desktop): Frame Control's server runs [`ui/frame_input_agent.py`](../ui/frame_input_agent.py) +on the Frame, which talks KDE Connect's own LAN protocol to the Frame's +`kdeconnectd` as if it were a phone. KDE Connect does the typing and clicking. + +**Verified 2026-09-28** (SteamOS 0.4.1, build 20260925.6191901): + +- KDE Connect isn't installed, but **Valve's package repository for the Frame + has it** (`kdeconnect` 24.02.2 in `extra`). The Frame lacks only `kpeople`, + `libfakekey`, `modemmanager-qt` and `pulseaudio-qt`. The agent fetches them + with `pacman -Sp` + `curl` into `~/.local/share/frame-control/kdeconnect` + (8 MB download, 82 MB unpacked, about 11 s), with no root and nothing on the + read-only system, so SteamOS updates leave it alone. +- It pairs by itself: the agent asks to pair and accepts on KDE Connect's side + over D-Bus (`qdbus6 … acceptPairing`). It keeps its identity in + `…/kdeconnect/bridge`, so later connections are already paired. (A pair + request to a device that's already paired makes KDE Connect unpair it, so + the agent only asks when it isn't paired.) +- Protocol version 7: whoever opens the TCP connection sends its identity line + in plain text, then acts as the **TLS server** (KDE Connect's + `lanlinkprovider.cpp`). Remote input is `kdeconnect.mousepad.request` with + `dx`/`dy`, `singleclick`, `rightclick`, `singlehold`/`singlerelease`, + `scroll`, `key` (any text) or `specialKey` (1 Backspace … 14 Escape, + 21–32 F1–F12) and modifier flags. +- **KDE Connect runs only while something uses the keyboard and trackpad.** + Each device gets its own KDE Connect identity (KDE Connect keeps one + connection per device, so a shared one would make a phone and a computer + knock each other off). The last one to disconnect stops KDE Connect, so it + isn't left running, or discoverable on your network, afterwards. +- KDE Connect 24.02 **hangs or crashes when asked to unpair a device that's + offline** (seen twice: once spinning at 100% CPU with D-Bus unresponsive, + once exiting). Frame Control never unpairs. If its copy stops answering, the + agent restarts it once (tested by freezing it with `kill -STOP`). +- Moves from the iPhone app (Simulator) and the Mac's server moved the Frame's + X pointer by exactly the amount sent. +- gamescope runs **two Xwayland displays**. `:0` holds Steam's VR bar and menus + and ignores injected pointer motion; `:1` holds apps such as Chromium and + takes it. KDE Connect runs on `:1`, so it reaches apps, not Steam's own menus. + There's also a `gamescope-0-ei` (libei) socket. +- **Not yet tested:** typing and clicking as seen in the headset, and whether + it reaches the KDE desktop panel (Plasma is its own session). +- **Known limit:** keys and clicks typed while the link is reconnecting wait + and are sent once it's back, but anything sent in the moment the Wi-Fi + drops, before SSH notices, can be lost. Confirming every event would add a + round trip to each pointer move. + +Our own `uinput` keyboard and mouse would also work (`steamos` is in the +`input` group and `/dev/uinput` is group-writable, verified 2026-09-27), and +remains the fallback if KDE Connect ever can't be fetched. + +| Other option | Mac | iPhone | Why not | |---|---|---|---| -| **A uinput keyboard and mouse in Frame Control's server** | ✓ | ✓ | **Recommended.** The server opens `/dev/uinput` with `ctypes` (standard library only) and the page sends key and pointer events through the tunnel it already has. On the phone: a trackpad area (drag to move, tap to click, two fingers to scroll) and the iOS keyboard for typing. On the Mac: a "control the Frame" mode that captures the keyboard and pointer (Esc to release). Uinput devices look like real hardware to the kernel, so libinput, KWin and gamescope should take them; [frame-voice](https://github.com/DeeJanuz/frame-voice) already types into a Frame through a uinput keyboard. **Untested**: which surfaces in VR (desktop panel, SteamVR dashboard, games, Android apps in Lepton) accept the pointer. About a day or two of work | -| **Bluetooth keyboard and mouse** | – | – | Real hardware paired in SteamOS settings. The iPhone can't pretend to be a Bluetooth keyboard: iOS won't advertise the HID service ([Apple forums](https://developer.apple.com/forums/thread/733916)) | +| **Bluetooth keyboard and mouse** | – | – | Needs real hardware, paired in SteamOS settings. The iPhone can't pretend to be a Bluetooth keyboard: iOS won't advertise the HID service ([Apple forums](https://developer.apple.com/forums/thread/733916)) | | **Deskflow** (formerly Input Leap / Barrier) | ✓ | – | Moves the Mac's own mouse and keyboard onto the Frame's screen edge. Flathub has an aarch64 build ([Flathub](https://flathub.org/apps/org.deskflow.deskflow)); on Wayland it needs the InputCapture/libei portal, and only works while Plasma is running. No iPhone client | -| **KDE Connect** | ~ | ✓ | Its iOS app has a remote touchpad and keyboard, but the Frame would need KDE Connect installed (not on Flathub; `pacman` on a read-only root). More moving parts than the uinput route | | **Remmina / Steam Link / RDP** | ✓ | – | Input only reaches the streamed session, not the headset's own apps | Other ways to get text in: diff --git a/ios/FrameControl/App/AppModel.swift b/ios/FrameControl/App/AppModel.swift index 94e25fa..c56f144 100644 --- a/ios/FrameControl/App/AppModel.swift +++ b/ios/FrameControl/App/AppModel.swift @@ -1,4 +1,5 @@ import Citadel +import CryptoKit import Foundation import SwiftUI import UIKit @@ -33,6 +34,11 @@ final class AppModel: ObservableObject { } var deviceName: String { UIDevice.current.userInterfaceIdiom == .pad ? "iPad" : "iPhone" } + /// Stable per install: a hash of this device's SSH key, which is made once and kept in the Keychain. + var clientID: String { + let digest = SHA256.hash(data: Data(authorizedKeysLine.utf8)) + return deviceName.lowercased() + "-" + digest.prefix(6).map { String(format: "%02x", $0) }.joined() + } private var hostKey: String? { UserDefaults.standard.string(forKey: Self.hostKeyKey) } // MARK: pairing @@ -167,7 +173,7 @@ final class AppModel: ObservableObject { guard current() else { throw CancellationError() } step("Starting Frame Control on the headset") let key = Self.randomKey() - let server = try await HeadsetServer.start(in: dir, over: l, key: key, device: deviceName) + let server = try await HeadsetServer.start(in: dir, over: l, key: key, device: deviceName, client: clientID) guard current() else { throw CancellationError() } let f = try await PortForwarder.start(over: l, to: server.port) forwarder = f diff --git a/ios/FrameControl/SSH/HeadsetServer.swift b/ios/FrameControl/SSH/HeadsetServer.swift index 443e99d..d59388d 100644 --- a/ios/FrameControl/SSH/HeadsetServer.swift +++ b/ios/FrameControl/SSH/HeadsetServer.swift @@ -80,8 +80,10 @@ final class HeadsetServer: @unchecked Sendable { } /// Starts the server in dir and waits for it to say which port it took. - static func start(in dir: String, over link: FrameLink, key: String, device: String) async throws -> HeadsetServer { + /// client is a stable id for this install (keyboard-and-trackpad pairing is kept per client). + static func start(in dir: String, over link: FrameLink, key: String, device: String, client: String) async throws -> HeadsetServer { let command = "cd \(dir) && FRAME_LOCAL=1 FRAME_UI_KEY=\(key) FRAME_DEVICE=\(shellQuote(device)) " + + "FRAME_CLIENT=\(shellQuote(client)) " + "exec python3 -I -u -B \"$PWD/ui/server.py\" --port 0 --exit-on-eof 2>&1" let stream = try await link.client.executeCommandStream(command) let box = PortWaiter() diff --git a/tests/page/pad_queue.mjs b/tests/page/pad_queue.mjs new file mode 100644 index 0000000..f8e748c --- /dev/null +++ b/tests/page/pad_queue.mjs @@ -0,0 +1,48 @@ +// The page's input queue, run in node against the real functions from ui/index.html. +import { readFileSync } from "fs"; +const src = readFileSync(new URL("../../ui/index.html", import.meta.url), "utf8"); +const grab = name => src.match(new RegExp(`(?:const ${name} = [^\\n]*\\n)|((?:async )?function ${name}\\([\\s\\S]*?\\n}\\n)`))[0]; +const code = ["isMove", "padSend", "padKeep", "padFlush"].map(grab).join(""); +const fail = msg => { console.log("FAIL " + msg); process.exit(1); }; + +function page(api) { + const pad = { state: "ready", queue: [], sending: false }; + const shown = []; + const fns = new Function("pad", "api", "padShow", code + "; return { padSend, padFlush };")(pad, api, s => { shown.push(s); pad.state = s.state; }); + return { pad, shown, ...fns }; +} + +// While a request is in flight, events queue by these rules. +{ + const { pad, padSend } = page(() => new Promise(() => {})); // a request that never returns + const send = (state, e) => { pad.state = state; padSend(e); }; + send("ready", { singlehold: true }); // goes out, and stays in flight + send("error", { dx: 3 }); // dropped: a stale move while broken + send("error", { singlerelease: true }); // kept + send("off", { singlerelease: true }); // off: nothing was held + send("pairing", { key: "a" }); // kept while reconnecting + send("pairing", { dx: 1 }); // dropped + for (let i = 0; i < 250; i++) send("pairing", { key: "x" }); + send("pairing", { singlerelease: true }); // kept past the 200 cap + const q = pad.queue; + if (!(q[0].singlerelease && q[1].key === "a" && q.at(-1).singlerelease && !q.some(e => "dx" in e) + && q.filter(e => e.singlerelease).length === 2)) fail("queueing " + JSON.stringify(q.slice(0, 3)) + " len " + q.length); +} + +// A request that fails, or reaches a link that had dropped, keeps its keys and releases. +for (const [label, api] of [["request failed", async () => { throw new Error("offline"); }], + ["not sent", async () => ({ state: "starting", sent: false })]]) { + const { pad, padSend, shown } = page(api); + padSend({ dx: 5 }); // sent and lost: a move, fine to drop + await new Promise(r => setTimeout(r, 0)); + pad.state = "ready"; + padSend({ singlerelease: true }); + padSend({ dx: 7 }); // queued while that request is in flight: stale once it fails + padSend({ key: "b" }); + await new Promise(r => setTimeout(r, 0)); + const q = pad.queue; + if (!(q.some(e => e.singlerelease) && q.some(e => e.key === "b") && !q.some(e => "dx" in e))) + fail(label + ": " + JSON.stringify(q)); + if (!shown.length || shown.at(-1).state === "ready") fail(label + ": state not shown"); +} +console.log("queue rules ok"); diff --git a/tests/test_input.py b/tests/test_input.py new file mode 100644 index 0000000..bc2afbc --- /dev/null +++ b/tests/test_input.py @@ -0,0 +1,196 @@ +"""Keyboard and pointer: event checks, and the agent's KDE Connect protocol against a fake kdeconnectd. + +The fake follows what KDE Connect 24.02 does (read from its source, +core/backends/lan/lanlinkprovider.cpp): it accepts the TCP connection, reads the +identity line, then starts TLS as the *client*. + +Run: python3 -m unittest discover -s tests +""" +import json +import os +import shutil +import socket +import ssl +import subprocess +import sys +import tempfile +import threading +import time +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(ROOT / "ui")) + + +class InputEvents(unittest.TestCase): + @classmethod + def setUpClass(cls): + import server + cls.server = server + + def check(self, event): + return self.server.input_event(event) + + def test_keeps_known_fields(self): + self.assertEqual(self.check({"dx": 3, "dy": -1.234}), {"dx": 3.0, "dy": -1.23}) + self.assertEqual(self.check({"singleclick": True, "other": 1}), {"singleclick": True}) + self.assertEqual(self.check({"key": "héllo", "shift": True}), {"key": "héllo", "shift": True}) + self.assertEqual(self.check({"specialKey": 12, "ctrl": True}), {"specialKey": 12, "ctrl": True}) + self.assertEqual(self.check({"scroll": True, "dy": 1}), {"scroll": True, "dy": 1.0}) + + def test_clamps_movement(self): + self.assertEqual(self.check({"dx": 1e9})["dx"], self.server.INPUT_MOVE_LIMIT) + self.assertEqual(self.check({"dy": -1e9})["dy"], -self.server.INPUT_MOVE_LIMIT) + + def test_rejects_bad_events(self): + bad = [None, [], "a", {}, {"shift": True}, {"dx": "1"}, {"dx": True}, {"dx": float("nan")}, + {"key": ""}, {"key": 5}, {"key": "x" * 501}, {"specialKey": 0}, {"specialKey": 33}, + {"specialKey": True}, {"specialKey": 1.5}, {"singleclick": "yes"}] + for event in bad: + with self.assertRaises(self.server.Failure, msg=repr(event)): + self.check(event) + + def test_send_while_link_is_down_says_not_sent(self): + # The page keeps unsent keys and clicks and sends them once the agent is ready again. + from types import SimpleNamespace + + class Probe(self.server.InputAgent): + def start(self): + self.proc, self.status = SimpleNamespace(poll=lambda: None, stdin=None), {"state": "starting"} + + agent = Probe() + agent.status, agent.proc = {"state": "ready"}, SimpleNamespace(poll=lambda: 255) # ssh has exited + self.assertEqual(agent.send([{"key": "x"}]), {"state": "starting", "sent": False}) + + def test_command_passes_client_quoted(self): + os.environ["FRAME_CLIENT"] = "test-client-1" + try: + cmd = self.server.InputAgent().command() + finally: + del os.environ["FRAME_CLIENT"] + self.assertTrue(cmd.startswith("python3 -u -c '")) + self.assertIn(" test-client-1 ", cmd) + + def test_batch_limits(self): + with self.assertRaises(self.server.Failure): + self.server.remote_input({"events": "dx"}) + with self.assertRaises(self.server.Failure): + self.server.remote_input({"events": [{"dx": 1}] * (self.server.INPUT_BATCH_LIMIT + 1)}) + + +class FakeKdeConnect: + """Just enough of kdeconnectd: pairs when asked and records remote-input packets.""" + + def __init__(self, paired=False): + self.listener = socket.socket() + self.listener.bind(("127.0.0.1", 0)) + self.listener.listen(1) + self.port = self.listener.getsockname()[1] + self.paired, self.identity, self.received, self.pair_requests = paired, None, [], 0 + threading.Thread(target=self.serve, daemon=True).start() + + def serve(self): + conn, _ = self.listener.accept() + line = b"" + while not line.endswith(b"\n"): + line += conn.recv(1) + self.identity = json.loads(line) + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + ctx.check_hostname, ctx.verify_mode = False, ssl.CERT_NONE + tls = ctx.wrap_socket(conn) # "Starting client ssl (but I'm the server TCP socket)" + tls.sendall(b'{"id": 1, "type": "kdeconnect.mousepad.keyboardstate", "body": {"state": true}}\n') + buf = b"" + while True: + try: + chunk = tls.recv(65536) + except OSError: + return + if not chunk: + return + buf += chunk + while b"\n" in buf: + raw, buf = buf.split(b"\n", 1) + p = json.loads(raw) + if p["type"] == "kdeconnect.pair": + self.pair_requests += 1 + elif p["type"] == "kdeconnect.mousepad.request" and self.paired: + self.received.append(p["body"]) + + +@unittest.skipIf(sys.platform == "win32", "the agent runs on the Frame (Linux)") +@unittest.skipUnless(shutil.which("openssl"), "needs openssl to make a certificate") +class AgentProtocol(unittest.TestCase): + @classmethod + def setUpClass(cls): + import frame_input_agent + cls.agent = frame_input_agent + cls.dir = tempfile.TemporaryDirectory() + cls.cert, cls.key = Path(cls.dir.name, "cert.pem"), Path(cls.dir.name, "key.pem") + subprocess.run(["openssl", "req", "-x509", "-newkey", "rsa:2048", "-nodes", "-days", "1", + "-subj", "/CN=framecontrol_test", "-keyout", str(cls.key), "-out", str(cls.cert)], + check=True, capture_output=True) + + @classmethod + def tearDownClass(cls): + cls.dir.cleanup() + + def wait_for(self, check): + for _ in range(100): + if check(): + return + time.sleep(0.02) + self.fail("timed out") + + def test_pairs_then_forwards_events(self): + fake = FakeKdeConnect() + link = self.agent.Link("framecontrol_0123456789abcdef01234567", self.cert, self.key, port=fake.port) + link.pair(lambda: fake.paired, lambda: setattr(fake, "paired", fake.pair_requests > 0), timeout=5) + self.assertEqual(fake.pair_requests, 1) + ident = fake.identity["body"] + self.assertEqual(fake.identity["type"], "kdeconnect.identity") + self.assertEqual(ident["protocolVersion"], 7) + self.assertIn("kdeconnect.mousepad.request", ident["outgoingCapabilities"]) + link.read(0.5) + self.assertTrue(link.keyboard) + for body in self.agent.events(b'[{"dx": 4, "dy": -2}, {"key": "hi"}]'): + link.send(body) + self.wait_for(lambda: len(fake.received) == 2) + self.assertEqual(fake.received, [{"dx": 4, "dy": -2}, {"key": "hi"}]) + + def test_already_paired_sends_no_pair_request(self): + # A pair request to a device that's already paired makes KDE Connect unpair it. + fake = FakeKdeConnect(paired=True) + link = self.agent.Link("framecontrol_0123456789abcdef01234567", self.cert, self.key, port=fake.port) + link.pair(lambda: True, lambda: self.fail("accepted a pairing that wasn't needed")) + link.send({"singleclick": True}) + self.wait_for(lambda: fake.received == [{"singleclick": True}]) + self.assertEqual(fake.pair_requests, 0) + + def test_client_args_are_folder_safe(self): + old = sys.argv + try: + sys.argv = ["-c", "../../etc x", "Alex's Mac"] + self.assertEqual(self.agent.client_args(), ("etcx", "Frame Control (Alex's Mac)")) + sys.argv = ["-c"] + self.assertEqual(self.agent.client_args(), ("default", "Frame Control")) + finally: + sys.argv = old + + def test_events_parsing(self): + self.assertEqual(self.agent.events(b'{"dx": 1}'), [{"dx": 1}]) + self.assertEqual(self.agent.events(b'[{"dx": 1}, 5, {}]'), [{"dx": 1}]) + self.assertEqual(self.agent.events(b"not json"), []) + + +@unittest.skipUnless(shutil.which("node"), "needs node") +class PageQueue(unittest.TestCase): + """The page's input queue (tests/page/pad_queue.mjs runs the real functions from index.html).""" + + def test_queue_rules(self): + r = subprocess.run(["node", str(ROOT / "tests" / "page" / "pad_queue.mjs")], capture_output=True, text=True) + self.assertEqual(r.returncode, 0, r.stdout + r.stderr) + + +if __name__ == "__main__": + unittest.main() diff --git a/ui/frame_input_agent.py b/ui/frame_input_agent.py new file mode 100644 index 0000000..6a38861 --- /dev/null +++ b/ui/frame_input_agent.py @@ -0,0 +1,376 @@ +"""Keyboard and pointer for the Steam Frame. Frame Control's server runs this ON the Frame. + +It speaks KDE Connect's LAN protocol (version 7, as in KDE Connect 24.02) to the +Frame's own kdeconnectd, as a phone would, and forwards remote-input events read +from stdin: one JSON object (or list of them) per line, each a KDE Connect +"mousepad" request body such as {"dx": 4, "dy": -2} or {"key": "hello"}. +KDE Connect does the typing and clicking. + +KDE Connect isn't installed on the Frame, but Valve's package repository for it +has a build. The first run fetches that and the few libraries the Frame lacks +into ~/.local/share/frame-control/kdeconnect: no root, and SteamOS updates +leave it alone. + +Status goes to stdout, one JSON object per line: +{"state": "installing" | "starting" | "pairing" | "ready" | "error", ...}. + +Standard library only: this runs on the Frame's own Python. +""" +import fcntl +import json +import os +import selectors +import shutil +import signal +import socket +import ssl +import subprocess +import sys +import time +from pathlib import Path + +BASE = Path.home() / ".local/share/frame-control/kdeconnect" +ROOT = BASE / "root" +BRIDGE = BASE / "bridge" +# kdeconnect plus the dependencies the Frame's image doesn't have (checked 2026-09-28, +# SteamOS 0.4.1); `pacman -Sp` adds any others still missing. +PACKAGES = ["kdeconnect", "kpeople", "libfakekey", "modemmanager-qt", "pulseaudio-qt"] +PORT = int(os.environ.get("FRAME_INPUT_PORT", "1716")) +UID = os.getuid() +MOUSEPAD = "kdeconnect.mousepad.request" + + +def say(state, **more): + print(json.dumps({"state": state, **more}), flush=True) + + +def packet(kind, body): + return (json.dumps({"id": int(time.time() * 1000), "type": kind, "body": body}) + "\n").encode() + + +# ---- KDE Connect on the Frame ------------------------------------------------ + +def daemon_path(): + for path in (Path("/usr/lib/kdeconnectd"), ROOT / "usr/lib/kdeconnectd"): + if path.exists(): + return path + return None + + +def install(): + say("installing", message="Fetching KDE Connect from the Frame's package repository") + found = subprocess.run(["pacman", "-Sp", *PACKAGES], capture_output=True, text=True, timeout=120) + urls = [u for u in found.stdout.split() if u.startswith("https://")] + if found.returncode or not urls: + raise RuntimeError("Couldn't find KDE Connect in the Frame's package repository: " + + (found.stderr.strip() or "no packages listed")) + download, stage = BASE / "download", BASE / "root.new" + for d in (download, stage): + shutil.rmtree(d, ignore_errors=True) + d.mkdir(parents=True) + for url in urls: + name = download / url.rsplit("/", 1)[1] + subprocess.run(["curl", "-fsSL", "--retry", "2", "-o", str(name), url], check=True, timeout=600) + if subprocess.run(["tar", "--zstd", "-xf", str(name), "-C", str(stage)], capture_output=True).returncode: + subprocess.run(["bsdtar", "-xf", str(name), "-C", str(stage)], check=True, capture_output=True) + shutil.rmtree(ROOT, ignore_errors=True) + stage.rename(ROOT) + shutil.rmtree(download, ignore_errors=True) + + +def app_display(): + """The X display that apps (not Steam's own VR menus) are on. + + gamescope runs two Xwayland servers: on 2026-09-28 :0 held Steam's VR bar and + menus and ignored XTest pointer motion, while :1 held apps such as Chromium and + took it. Inferred to hold in general. + """ + return ":1" if Path("/tmp/.X11-unix/X1").exists() else ":0" + + +def daemon_env(daemon): + env = dict(os.environ, DBUS_SESSION_BUS_ADDRESS=f"unix:path=/run/user/{UID}/bus", + XDG_RUNTIME_DIR=f"/run/user/{UID}", DISPLAY=app_display(), QT_QPA_PLATFORM="xcb") + if str(daemon).startswith(str(ROOT)): + env.update(LD_LIBRARY_PATH=str(ROOT / "usr/lib"), QT_PLUGIN_PATH=str(ROOT / "usr/lib/qt6/plugins"), + QML_IMPORT_PATH=str(ROOT / "usr/lib/qt6/qml"), + XDG_DATA_DIRS=f"{ROOT / 'usr/share'}:/usr/share") + return env + + +def listening(): + try: + socket.create_connection(("127.0.0.1", PORT), 1).close() + return True + except OSError: + return False + + +def our_daemons(): + """Process ids of the kdeconnectd that Frame Control installed (never a system one).""" + pids = [] + for proc in Path("/proc").iterdir(): + if proc.name.isdigit(): + try: + if os.readlink(proc / "exe").startswith(str(ROOT) + "/"): + pids.append(int(proc.name)) + except OSError: + pass + return pids + + +def stop_daemon(): + """Stop our kdeconnectd and wait until it's gone (so its port is closed too).""" + for sig, wait in ((signal.SIGTERM, 30), (signal.SIGKILL, 30)): # tenths of a second + for pid in our_daemons(): + try: + os.kill(pid, sig) + except ProcessLookupError: + pass + for _ in range(wait): + if not our_daemons() and not listening(): + return + time.sleep(0.1) + + +def ensure_daemon(): + if listening(): + return + BASE.mkdir(parents=True, exist_ok=True) + daemon = daemon_path() + if not daemon: + install() + daemon = daemon_path() + say("starting", message="Starting KDE Connect on the Frame") + log = open(BASE / "kdeconnectd.log", "ab") + # Its own session, so it outlives this connection and serves the next one. + subprocess.Popen([str(daemon)], env=daemon_env(daemon), cwd=str(Path.home()), stdin=subprocess.DEVNULL, + stdout=log, stderr=log, start_new_session=True) + for _ in range(40): + if listening(): + return + time.sleep(0.25) + raise RuntimeError(f"KDE Connect didn't start; see {BASE / 'kdeconnectd.log'} on the Frame") + + +def qdbus(device, method): + """Call a method on KDE Connect's D-Bus object for our device; its output, or None.""" + env = dict(os.environ, DBUS_SESSION_BUS_ADDRESS=f"unix:path=/run/user/{UID}/bus") + try: + r = subprocess.run(["qdbus6", "org.kde.kdeconnect", f"/modules/kdeconnect/devices/{device}", + f"org.kde.kdeconnect.device.{method}"], capture_output=True, text=True, env=env, timeout=5) + except (OSError, subprocess.TimeoutExpired): + return None + return r.stdout.strip() if r.returncode == 0 else None + + +# ---- our identity -------------------------------------------------------------- + +def identity(client): + """A device id and certificate for this client, made once and kept (pairing is tied to them). + + Each computer or phone gets its own: KDE Connect keeps one connection per device, + so a shared identity would make them knock each other off. + """ + folder = BRIDGE / client + folder.mkdir(parents=True, exist_ok=True) + id_file, cert, key = folder / "id", folder / "cert.pem", folder / "key.pem" + if not (id_file.exists() and cert.exists() and key.exists()): + device = "framecontrol_" + os.urandom(12).hex() # KDE Connect wants 32-38 of [A-Za-z0-9_] + subprocess.run(["openssl", "req", "-x509", "-newkey", "ec", "-pkeyopt", "ec_paramgen_curve:prime256v1", + "-nodes", "-days", "3650", "-subj", f"/O=KDE/OU=Kde connect/CN={device}", + "-keyout", str(key), "-out", str(cert)], check=True, capture_output=True) + os.chmod(key, 0o600) + id_file.write_text(device) + return id_file.read_text().strip(), cert, key + + +# ---- the link ------------------------------------------------------------------ + +class Link: + """One TLS connection to kdeconnectd, as a paired device that sends remote input.""" + + def __init__(self, device, cert, key, port=PORT, name="Frame Control"): + self.device, self.buf, self.keyboard = device, b"", None + raw = socket.create_connection(("127.0.0.1", port), 5) + raw.sendall(packet("kdeconnect.identity", { + "deviceId": device, "deviceName": name, "deviceType": "phone", "protocolVersion": 7, + "incomingCapabilities": [], "outgoingCapabilities": [MOUSEPAD], "tcpPort": port})) + # KDE Connect's rule: whoever opened the TCP connection is the TLS server. + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + ctx.load_cert_chain(str(cert), str(key)) + ctx.verify_mode = ssl.CERT_NONE # both sides are on this machine + self.sock = ctx.wrap_socket(raw, server_side=True) + self.sock.setblocking(False) + + def send(self, body): + # Bounded: if KDE Connect stops reading, fail (and be restarted) rather than hang. + self.sock.settimeout(5) + try: + self.sock.sendall(packet(MOUSEPAD, body)) + finally: + self.sock.setblocking(False) + + def pair(self, paired, accept, timeout=15): + """Ask to pair and accept it on KDE Connect's side (we control both ends). + + Only asks when not already paired: a pair request to a device that is + already paired makes KDE Connect unpair it. + """ + if paired(): + return + self.sock.settimeout(5) + self.sock.sendall(packet("kdeconnect.pair", {"pair": True})) + self.sock.setblocking(False) + end = time.time() + timeout + while time.time() < end: + accept() + self.read(0.5) + if paired(): + return + raise RuntimeError("KDE Connect didn't accept the pairing") + + def read(self, wait=0.0): + """Packets waiting from kdeconnectd; None once it has closed the connection.""" + if wait: + sel = selectors.DefaultSelector() + sel.register(self.sock, selectors.EVENT_READ) + sel.select(wait) + sel.close() + try: + while True: + chunk = self.sock.recv(65536) + if not chunk: + return None + self.buf += chunk + except (ssl.SSLWantReadError, BlockingIOError): + pass + out = [] + while b"\n" in self.buf: + line, self.buf = self.buf.split(b"\n", 1) + if line.strip(): + p = json.loads(line) + if p.get("type") == "kdeconnect.mousepad.keyboardstate": + self.keyboard = bool(p.get("body", {}).get("state")) + out.append(p) + return out + + +def events(line): + """The event bodies in one stdin line (an object or a list of objects).""" + try: + value = json.loads(line) + except ValueError: + return [] + return [e for e in (value if isinstance(value, list) else [value]) if isinstance(e, dict) and e] + + +def connect(device, cert, key, name): + link = Link(device, cert, key, name=name) + link.pair(lambda: qdbus(device, "isPaired") == "true", lambda: qdbus(device, "acceptPairing")) + link.read(0.5) # its hello, including whether it can type + return link + + +def client_args(): + """argv: a folder-safe id for the computer or phone, and the name KDE Connect shows for it.""" + client = sys.argv[1] if len(sys.argv) > 1 else "default" + client = "".join(c for c in client if c.isalnum() or c in "-_")[:64] or "default" + name = (sys.argv[2] if len(sys.argv) > 2 else "")[:60].strip() + return client, f"Frame Control ({name})" if name else "Frame Control" + + +def main(): + client, name = client_args() + # A dropped ssh (the Frame slept, the app quit) hangs up on us: exit through the + # clean-up below rather than dying on the spot. + for sig in (signal.SIGHUP, signal.SIGTERM): + signal.signal(sig, lambda *_: sys.exit(0)) + BASE.mkdir(parents=True, exist_ok=True) + # Every agent holds this lock shared while it runs. The last one out gets it + # exclusively and stops KDE Connect, so it runs, and shows up on the network, + # only while something is using the keyboard and trackpad. + clients = open(BASE / "clients.lock", "w") + fcntl.flock(clients, fcntl.LOCK_SH) + try: + return run(client, name) + finally: + # Finish the clean-up even if a second hang-up or TERM arrives meanwhile. + for sig in (signal.SIGHUP, signal.SIGTERM): + signal.signal(sig, signal.SIG_IGN) + fcntl.flock(clients, fcntl.LOCK_UN) + try: + fcntl.flock(clients, fcntl.LOCK_EX | fcntl.LOCK_NB) + except OSError: + pass # another device is still using it + else: + with daemon_lock(): + stop_daemon() + + +class daemon_lock: + """Installing, starting and restarting KDE Connect happen one agent at a time.""" + + def __enter__(self): + self.file = open(BASE / "daemon.lock", "w") + fcntl.flock(self.file, fcntl.LOCK_EX) + + def __exit__(self, *_): + self.file.close() + + +def run(client, name): + try: + with daemon_lock(): + ensure_daemon() + device, cert, key = identity(client) + say("pairing") + seen = our_daemons() + try: + link = connect(device, cert, key, name) + except (OSError, RuntimeError): + if not seen: + raise + # Ours, but not answering (KDE Connect 24.02 can hang, for one after + # unpairing a device that's offline): start it afresh, once. If another + # agent already replaced it, just use the new one. + say("starting", message="Restarting KDE Connect on the Frame") + with daemon_lock(): + if set(our_daemons()) & set(seen): + stop_daemon() + ensure_daemon() + link = connect(device, cert, key, name) + except (OSError, RuntimeError, subprocess.SubprocessError) as e: + say("error", message=str(e)) + return 1 + say("ready", keyboard=link.keyboard is not False) + stdin, pending = sys.stdin.fileno(), b"" + sel = selectors.DefaultSelector() + sel.register(stdin, selectors.EVENT_READ) + sel.register(link.sock, selectors.EVENT_READ) + while True: + for key_, _ in sel.select(30): + if key_.fileobj == stdin: + chunk = os.read(stdin, 65536) # raw reads: a buffered readline could strand lines select can't see + if not chunk: # the server went away + return 0 + *lines, pending = (pending + chunk).split(b"\n") + try: + for line in lines: + for body in events(line): + link.send(body) + except OSError as e: + say("error", message=f"Lost KDE Connect: {e}") + return 1 + else: + packets = link.read() + if packets is None: + say("error", message="KDE Connect closed the connection") + return 1 + if any(p.get("type") == "kdeconnect.pair" and not p.get("body", {}).get("pair") for p in packets): + say("error", message="KDE Connect unpaired Frame Control") + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/ui/index.html b/ui/index.html index 3845ef7..7cf4b42 100644 --- a/ui/index.html +++ b/ui/index.html @@ -322,6 +322,19 @@ border-radius: 3px; height: 32px; padding: 0 8px; font: inherit; font-size: 13px; } /* Touch screens can't hover: keep the library's name and Play button showing. */ @media (hover: none) { .capsule .over { opacity: 1; } .capsule:hover { transform: none; } } + /* ---- keyboard and trackpad ---- */ + .pad-area { position: relative; height: 190px; margin: 12px 0; border-radius: 4px; background: rgba(0,0,0,.28); + border: 1px dashed rgba(255,255,255,.14); display: grid; place-items: center; cursor: pointer; + touch-action: none; user-select: none; -webkit-user-select: none; -webkit-touch-callout: none; outline: none; } + .pad-area:focus-visible, .pad-area.captured { border: 1px solid var(--blue); box-shadow: 0 0 0 1px var(--blue) inset; } + .pad-area.off { cursor: default; opacity: .55; } + .pad-hint { max-width: 420px; padding: 0 16px; text-align: center; color: var(--muted); font-size: 13px; pointer-events: none; } + .pad-keys { display: flex; flex-wrap: wrap; gap: 8px; align-items: center; } + .pad-keys input { flex: 1 1 220px; min-width: 0; height: 32px; padding: 0 10px; border-radius: 3px; border: 1px solid transparent; + background: rgba(0,0,0,.28); color: var(--text); font: inherit; } + .pad-keys input:focus { outline: none; border-color: var(--blue); } + #pad .hint { margin-top: 10px; } + /* ---- phones, upright or on their side: tabs move to a bottom bar, as in iOS apps. Every edge keeps clear of the safe area (notch or Dynamic Island, rounded corners, home indicator); env() is zero on desktops. ---- */ @media (max-width: 640px), (max-height: 500px) and (hover: none) { @@ -358,6 +371,10 @@ button.small { height: 32px; } .actions button { height: 46px; } .cat-tools select { max-width: none; flex: 1 1 100%; } + .pad-area { height: 240px; } + .pad-keys .seg { flex: 1 1 100%; display: grid; grid-template-columns: repeat(4, 1fr); } + .pad-keys .pad-clicks { grid-template-columns: repeat(2, 1fr); } + .pad-keys .seg button { padding: 0 4px; justify-content: center; text-align: center; } /* The on-screen keyboard covers the bottom of the screen; the tab bar would ride on top of it. */ body.typing nav { display: none; } } @@ -471,6 +488,36 @@ +
+

Keyboard and trackpad

+ +
+
+
+
+
+ + + + + + + + + + + + + + + +
+
Types and points in apps on the Frame, such as Chromium or the desktop's Linux apps; Steam's own VR menus + don't take it. It works through KDE Connect on the Frame: the first time, Frame Control fetches Valve's build of it + (an 8 MB download, 82 MB unpacked) into your home folder there and pairs with it. Nothing to install on this device.
+
+

Screenshots

@@ -2128,6 +2175,219 @@ window.addEventListener("hashchange", showPage); const typesText = el => el?.matches?.('textarea, input:not([type=range], [type=checkbox], [type=radio], [type=file], [type=button])'); document.addEventListener("focusin", e => document.body.classList.toggle("typing", !!typesText(e.target))); document.addEventListener("focusout", () => document.body.classList.remove("typing")); + +// ---- keyboard and trackpad: events go to KDE Connect on the Frame (see ui/frame_input_agent.py) ---- +const pad = { state: "off", queue: [], sending: false, poll: null, captured: false }; +const PAD_STATES = { + off: "", starting: "Connecting…", installing: "Setting up KDE Connect on the Frame (first time only)…", + pairing: "Pairing with KDE Connect…", ready: "On", +}; +// KDE Connect's specialKey numbers (plugins/mousepad in its source). +const PAD_SPECIAL = { Backspace: 1, Tab: 2, ArrowLeft: 4, ArrowUp: 5, ArrowRight: 6, ArrowDown: 7, PageUp: 8, + PageDown: 9, Home: 10, End: 11, Enter: 12, Delete: 13, Escape: 14, + ...Object.fromEntries(Array.from({ length: 12 }, (_, i) => [`F${i + 1}`, 21 + i])) }; +const IS_MAC = /^Mac/.test(navigator.platform); +const TOUCH = matchMedia("(hover: none)").matches; + +function padShow(status) { + pad.state = status.state || "off"; + const ready = pad.state === "ready"; + const busy = !ready && pad.state !== "off" && pad.state !== "error"; + $("padState").textContent = pad.state === "error" ? "" : PAD_STATES[pad.state] || ""; + $("padOn").hidden = ready || busy; + $("padOn").textContent = pad.state === "error" ? "Try again" : "Turn on"; + $("padArea").classList.toggle("off", !ready); + $("padHint").textContent = pad.state === "error" ? status.message || "Couldn't reach KDE Connect on the Frame." + : !ready ? (busy ? PAD_STATES[pad.state] : "Turn on to use this as a trackpad for the Frame.") + : TOUCH ? "Drag to move the pointer · tap to click · two fingers to scroll · two-finger tap to right-click" + : pad.captured ? "Your mouse and keyboard now control the Frame. Press Esc to stop." + : "Click here to use your mouse and keyboard on the Frame (Esc to stop), or drag like a trackpad."; + clearTimeout(pad.poll); + if (ready && pad.queue.length) padFlush(); + if (busy) pad.poll = setTimeout(async () => padShow(await api("/api/input").catch(e => ({ state: "error", message: e.message }))), 1000); +} +async function padStart() { + localStorage.padUsed = "1"; // from now on it turns on with the page + padShow({ state: "starting" }); + padShow(await api("/api/input?start=1").catch(e => ({ state: "error", message: e.message }))); +} +$("padOn").onclick = padStart; + +// Queue events and send them one request at a time, merging pointer moves made meanwhile. +const isMove = e => Object.keys(e).every(k => k === "dx" || k === "dy"); +function padSend(event) { + // While it reconnects, keys and clicks wait in the queue; pointer moves would be stale. + // A button release is always kept, so a drag never leaves the button held on the Frame. + const room = pad.queue.length < 200 || event.singlerelease; + const reconnecting = ["starting", "pairing", "installing"].includes(pad.state) && room; + // Even after an error: a button pressed on the Frame must be let go once it's back. + const release = event.singlerelease && pad.state !== "off"; + if (pad.state !== "ready" && !(reconnecting && !isMove(event)) && !release) return; + const last = pad.queue[pad.queue.length - 1]; + if (last && isMove(last) && isMove(event)) { last.dx = (last.dx || 0) + (event.dx || 0); last.dy = (last.dy || 0) + (event.dy || 0); } + else pad.queue.push(event); + padFlush(); +} +// Events that didn't reach the Frame (the link had dropped, and is reconnecting): keep +// the keys and clicks for when it's back, but not pointer moves, which would be stale. +// Button releases are kept whatever the queue's length. +function padKeep(events) { + pad.queue = [...events, ...pad.queue].filter(e => !isMove(e)).filter((e, i) => i < 200 || e.singlerelease); +} +async function padFlush() { + if (pad.sending || !pad.queue.length || pad.state !== "ready") return; + pad.sending = true; + const events = pad.queue.splice(0, 200); + try { + const status = await api("/api/input", { events }); + if (!status.sent) padKeep(events); + if (status.state !== "ready") padShow(status); + } catch (e) { + padKeep(events); + padShow({ state: "error", message: e.message }); + } finally { + pad.sending = false; + if (pad.queue.length) padFlush(); + } +} +function padKey(e) { + const mods = { ...(e.ctrlKey || (IS_MAC && e.metaKey) ? { ctrl: true } : {}), ...(e.altKey ? { alt: true } : {}), + ...(!IS_MAC && e.metaKey ? { super: true } : {}) }; + if (PAD_SPECIAL[e.key]) return { specialKey: PAD_SPECIAL[e.key], ...mods, ...(e.shiftKey ? { shift: true } : {}) }; + if (e.key.length === 1) return { key: e.key, ...mods }; // already shifted, so no shift flag + return null; +} + +// Touch: one finger moves, a quick tap clicks; two fingers scroll, or right-click when tapped. +const PAD_SPEED = 1.6, PAD_SCROLL_STEP = 18; +const touches = new Map(); +let gesture = null; +$("padArea").addEventListener("pointerdown", e => { + if (pad.state === "off" || pad.state === "error") { padStart(); return; } + if (pad.state !== "ready") return; + if (e.pointerType === "mouse") { + if (pad.captured) return; // mousedown and mouseup below send the clicks + // Lock the pointer to the pad and pass the whole mouse and keyboard through; where + // that isn't allowed, the mouse drags like a finger on a trackpad instead. + if (e.button === 0 && !pad.noLock && $("padArea").requestPointerLock) { $("padArea").requestPointerLock(); return; } + // Only the left button drags and taps like a finger; the others are clicks of their own. + if (e.button === 1 || e.button === 2) { padSend(e.button === 2 ? { rightclick: true } : { middleclick: true }); return; } + if (e.button !== 0) return; + } + try { $("padArea").setPointerCapture(e.pointerId); } catch {} // keeps the drag if the finger leaves the pad + touches.set(e.pointerId, { x: e.clientX, y: e.clientY }); + if (touches.size === 1) gesture = { start: performance.now(), moved: 0, fingers: 1, scroll: 0 }; + else if (gesture) gesture.fingers = Math.max(gesture.fingers, touches.size); + e.preventDefault(); +}); +$("padArea").addEventListener("pointermove", e => { + if (pad.captured) { padSend({ dx: e.movementX, dy: e.movementY }); return; } + const t = touches.get(e.pointerId); + if (!t || !gesture) return; + const dx = e.clientX - t.x, dy = e.clientY - t.y; + t.x = e.clientX; t.y = e.clientY; + gesture.moved += Math.abs(dx) + Math.abs(dy); + if (touches.size >= 2) { + // Scroll with the lead finger only, a notch per step, like a laptop trackpad (content follows the fingers). + if (e.pointerId !== touches.keys().next().value) return; + gesture.scroll += dy; + while (Math.abs(gesture.scroll) >= PAD_SCROLL_STEP) { + const up = gesture.scroll > 0; + padSend({ scroll: true, dy: up ? 1 : -1 }); + gesture.scroll -= up ? PAD_SCROLL_STEP : -PAD_SCROLL_STEP; + } + } else if (gesture.fingers === 1) { + padSend({ dx: dx * PAD_SPEED, dy: dy * PAD_SPEED }); + } +}); +function padLift(e) { + if (!touches.delete(e.pointerId) || touches.size || !gesture) return; + const quick = performance.now() - gesture.start < 250 && gesture.moved < 10; + if (quick) padSend(gesture.fingers >= 2 ? { rightclick: true } : { singleclick: true }); + gesture = null; +} +$("padArea").addEventListener("pointerup", padLift); +$("padArea").addEventListener("pointercancel", e => { touches.delete(e.pointerId); if (!touches.size) gesture = null; }); + +// A computer's mouse and keyboard: captured while the pointer is locked to the pad. +document.addEventListener("pointerlockerror", () => { pad.noLock = true; }); +document.addEventListener("pointerlockchange", () => { + pad.captured = document.pointerLockElement === $("padArea"); + window.frameApp?.captureKeys?.(pad.captured); // the desktop app's menu shortcuts go to the Frame too + // Esc or switching windows mid-drag: let go of the button on the Frame as well. + if (!pad.captured && pad.held) { padSend({ singlerelease: true }); pad.held = false; } + $("padArea").classList.toggle("captured", pad.captured); + padShow({ state: pad.state }); +}); +$("padArea").addEventListener("mousedown", e => { + if (!pad.captured) return; + if (e.button === 0) { padSend({ singlehold: true }); pad.held = true; } + else if (e.button === 1) padSend({ middleclick: true }); + else if (e.button === 2) padSend({ rightclick: true }); +}); +$("padArea").addEventListener("mouseup", e => { if (pad.held && e.button === 0) { padSend({ singlerelease: true }); pad.held = false; } }); +$("padArea").addEventListener("contextmenu", e => e.preventDefault()); +$("padArea").addEventListener("wheel", e => { + if (pad.state !== "ready") return; + e.preventDefault(); + if (e.deltaY) padSend({ scroll: true, dy: e.deltaY > 0 ? -1 : 1 }); +}, { passive: false }); +// Capture phase, so the page's own shortcuts (tabs 1-4, R, zoom) don't also fire. +window.addEventListener("keydown", e => { + if (!pad.captured) return; + e.preventDefault(); e.stopImmediatePropagation(); + const event = padKey(e); + if (event) padSend(event); +}, true); + +// The text field: each keystroke goes to the Frame. While focused it holds one space, +// so Backspace always has something to delete; empty otherwise, to show its prompt. +const padType = $("padType"); +padType.addEventListener("beforeinput", e => { + if (e.inputType === "insertCompositionText") return; // IME composing; sent on compositionend + e.preventDefault(); + if (pad.state === "off" || pad.state === "error") padStart(); // what's typed meanwhile waits in the queue + const text = e.data ?? e.dataTransfer?.getData("text/plain") ?? ""; + if (/^insert(Text|ReplacementText|FromPaste|FromDrop)$/.test(e.inputType) && text) padText(text); + else if (e.inputType === "deleteContentBackward") padSend({ specialKey: 1 }); + else if (e.inputType === "deleteWordBackward") padSend({ specialKey: 1, ctrl: true }); // Ctrl+Backspace: a word, on Linux + else if (e.inputType === "deleteContentForward") padSend({ specialKey: 13 }); + else if (e.inputType === "deleteWordForward") padSend({ specialKey: 13, ctrl: true }); + else if (e.inputType === "insertLineBreak" || e.inputType === "insertParagraph") padSend({ specialKey: 12 }); +}); +padType.addEventListener("compositionend", e => { if (e.data) padText(e.data); padType.value = " "; padCaret(); }); +// Text in pieces the server takes (500 characters), with Enter for each line break. +function padText(text) { + text.replace(/\r\n?/g, "\n").split("\n").forEach((line, i) => { + if (i) padSend({ specialKey: 12 }); + const chars = Array.from(line); // whole characters, so an emoji isn't cut in two + for (let at = 0; at < chars.length; at += 500) padSend({ key: chars.slice(at, at + 500).join("") }); + }); +} +padType.addEventListener("keydown", e => { + const special = PAD_SPECIAL[e.key]; + // Text and deletions arrive as beforeinput; this takes the keys that don't edit the field. + if (!special || e.key === "Backspace" || e.key === "Delete" || e.isComposing) return; + e.preventDefault(); + padSend(padKey(e)); +}); +// The caret stays after the space, so Backspace always has something to delete. +const padCaret = () => requestAnimationFrame(() => padType.setSelectionRange(1, 1)); +padType.addEventListener("focus", () => { padType.value = " "; padCaret(); if (pad.state === "off") padStart(); }); +padType.addEventListener("blur", () => { padType.value = ""; }); +padType.addEventListener("input", e => { + if (e.isComposing) return; // an input method is still composing; compositionend sends it + if (padType.value !== " ") padType.value = " "; + padCaret(); +}); +padType.addEventListener("click", padCaret); +document.querySelector("#pad .pad-keys").addEventListener("click", e => { + const b = e.target.closest("button"); + if (!b) return; + if (b.dataset.padClick) padSend({ [b.dataset.padClick]: true }); + else if (b.dataset.padKey) padSend({ specialKey: +b.dataset.padKey }); +}); +api("/api/input").then(s => s.state === "off" && localStorage.padUsed ? padStart() : padShow(s), () => padShow({ state: "off" })); showPage(); document.addEventListener("keydown", e => { if (e.metaKey || e.ctrlKey || e.altKey || /INPUT|TEXTAREA|SELECT/.test(document.activeElement.tagName) || document.querySelector("dialog[open]")) return; diff --git a/ui/server.py b/ui/server.py index ef22af6..df1c63f 100755 --- a/ui/server.py +++ b/ui/server.py @@ -10,6 +10,7 @@ Env: FRAME_ALIAS (default frame) FRAME_LOCAL=1 run on the Frame itself (the iPhone app starts it there over SSH) FRAME_UI_KEY required X-Frame-UI value (the iPhone app passes a fresh one) FRAME_DEVICE what to call the device the page runs on (e.g. iPhone) + FRAME_CLIENT a stable id for that device (keyboard-and-trackpad pairing is kept per id) """ import argparse import base64 @@ -54,6 +55,8 @@ if LOCAL: os.environ["PATH"] = f"{HERE / 'local-bin'}{os.pathsep}{os.environ.get('PATH', '')}" UI_KEY = os.environ.get("FRAME_UI_KEY") or "1" DEVICE = os.environ.get("FRAME_DEVICE") or "phone" +# What the Frame's KDE Connect calls this device (keyboard and trackpad). +INPUT_NAME = DEVICE if LOCAL else socket.gethostname().split(".")[0] FRAME = os.environ.get("FRAME_ALIAS", "frame") if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._-]*", FRAME): sys.exit(f"FRAME_ALIAS must be a plain host alias, not {FRAME!r}") @@ -497,6 +500,161 @@ def clipboard(body): return {"message": ssh(PASTE_CMD, stdin=text, timeout=30).strip()} +# ---- keyboard and pointer (KDE Connect on the Frame, see frame_input_agent.py) ---- + +INPUT_FLAGS = ("singleclick", "doubleclick", "middleclick", "rightclick", "singlehold", "singlerelease", + "scroll", "ctrl", "alt", "shift", "super") +INPUT_MOVE_LIMIT = 2000 # pixels per event +INPUT_TEXT_LIMIT = 500 # characters per event +INPUT_BATCH_LIMIT = 200 # events per request + + +def input_event(event): + """A KDE Connect remote-input body with only the fields it knows, in range.""" + if not isinstance(event, dict): + raise Failure("each input event must be an object", 400) + out = {} + for name in ("dx", "dy"): + value = event.get(name) + if value is None: + continue + if isinstance(value, bool) or not isinstance(value, (int, float)) or value != value: + raise Failure(f"{name} must be a number", 400) + out[name] = max(-INPUT_MOVE_LIMIT, min(INPUT_MOVE_LIMIT, round(float(value), 2))) + for name in INPUT_FLAGS: + if event.get(name) is True: + out[name] = True + key = event.get("key") + if key is not None: + if not isinstance(key, str) or not 0 < len(key) <= INPUT_TEXT_LIMIT: + raise Failure(f"key must be text of 1 to {INPUT_TEXT_LIMIT} characters", 400) + out["key"] = key + special = event.get("specialKey") + if special is not None: + # KDE Connect's numbering: 1 Backspace … 14 Escape, 21-32 F1-F12. + if isinstance(special, bool) or not isinstance(special, int) or not 1 <= special <= 32: + raise Failure("specialKey must be a whole number from 1 to 32", 400) + out["specialKey"] = special + if not set(out) - {"ctrl", "alt", "shift", "super"}: + raise Failure("input event has nothing to do", 400) + return out + + +def input_client(): + """A stable id for this device, so KDE Connect on the Frame keeps its pairing apart. + + The iPhone app passes one (FRAME_CLIENT). A computer makes one the first time + and keeps it: host names alone can clash (desk.home and desk.office). + """ + if os.environ.get("FRAME_CLIENT"): + return os.environ["FRAME_CLIENT"] + if LOCAL: + return DEVICE + path = frame_host.data_dir("input-client-id") + try: + saved = path.read_text().strip() + if re.fullmatch(r"[A-Za-z0-9_-]{4,64}", saved): + return saved + except OSError: + pass + made = (re.sub(r"[^A-Za-z0-9-]", "", INPUT_NAME)[:24] or "computer") + "-" + secrets.token_hex(4) + try: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(made) + except OSError: + pass # still works this time; it pairs again next time + return made + + +class InputAgent: + """frame_input_agent.py running on the Frame, fed events over one long-lived ssh. + + It sets up KDE Connect there if needed, pairs, and reports its state + ({"state": "off" | "installing" | "starting" | "pairing" | "ready" | "error"}). + """ + + def __init__(self, source=HERE / "frame_input_agent.py"): + self.source, self.proc, self.lock = source, None, threading.Lock() + self.status = {"state": "off"} + + def command(self): + code = base64.b64encode(self.source.read_bytes()).decode() + client = input_client() + return ("python3 -u -c " + shlex.quote( + f"import base64;exec(compile(base64.b64decode('{code}'),'frame_input_agent','exec'))") + + f" {shlex.quote(client)} {shlex.quote(INPUT_NAME)}") + + def start(self): + with self.lock: + if self.proc and self.proc.poll() is None: + return + ensure_master() + self.status = {"state": "starting"} + errors = tempfile.TemporaryFile() + self.proc = proc = subprocess.Popen([*SSH, FRAME, self.command()], stdin=subprocess.PIPE, + stdout=subprocess.PIPE, stderr=errors) + _live_tunnels.add(proc) + threading.Thread(target=self._watch, args=(proc, errors), daemon=True).start() + + def _watch(self, proc, errors): + for line in proc.stdout: + try: + status = json.loads(line) + except ValueError: + continue + if isinstance(status, dict) and isinstance(status.get("state"), str): + with self.lock: + if self.proc is proc: + self.status = status + proc.wait() + _live_tunnels.discard(proc) + errors.seek(0) + detail = strip_ansi(errors.read().decode(errors="replace")).strip() + with self.lock: + if self.proc is proc and self.status.get("state") != "error": + message = detail.splitlines()[-1] if detail else "The connection to the Frame ended" + friendly = unreachable(message) + self.status = {"state": "error", "message": friendly or message, **({"offline": True} if friendly else {})} + + def send(self, events): + """Forward events if the agent is ready; start it if it isn't running. + + Returns the state, with "sent" saying whether the events went; if not, + the page keeps them and sends them again once the state is "ready". + """ + with self.lock: + proc, ready = self.proc, self.status.get("state") == "ready" + sent = False + if not (proc and proc.poll() is None): + self.start() + elif ready and events: + try: + proc.stdin.write((json.dumps(events) + "\n").encode()) + proc.stdin.flush() + sent = True + except (BrokenPipeError, OSError, ValueError): + pass # _watch reports how it ended + with self.lock: + return {**self.status, "sent": sent} + + def stop(self): + with self.lock: + proc, self.proc, self.status = self.proc, None, {"state": "off"} + if proc and proc.poll() is None: + proc.terminate() + + +_input = InputAgent() + + +def remote_input(body): + """{"events": [...]} sends keyboard and pointer events; {} (or none yet) just starts the agent.""" + events = body.get("events", []) + if not isinstance(events, list) or len(events) > INPUT_BATCH_LIMIT: + raise Failure(f"events must be a list of at most {INPUT_BATCH_LIMIT}", 400) + return _input.send([input_event(e) for e in events]) + + def flatpak(body): app, action = str(body.get("id", "")), body.get("action") if not FLATPAK_ID.match(app): @@ -1214,6 +1372,7 @@ def _sweep_one(prefix, d): POST = {"/api/android/display": android_display, "/api/android": android, "/api/titles": titles, "/api/launch": launch, "/api/steam": steam, "/api/volume": set_volume, "/api/clipboard": clipboard, + "/api/input": remote_input, "/api/flatpak": flatpak, "/api/open": open_thing, "/api/shots/save": save_shots, "/api/webinstall/check": webinstall_check, "/api/webinstall/start": webinstall_start, "/api/webinstall/cancel": webinstall_cancel} @@ -1327,6 +1486,8 @@ class Handler(BaseHTTPRequestHandler): self.send_json({"titles": frame_titles.list_titles()}) elif path == "/api/titles/job": self.send_json(title_job(url.query)) + elif path == "/api/input": + self.send_json(_input.send([]) if parse_qs(url.query).get("start") == ["1"] else dict(_input.status)) elif path == "/api/job": self.send_json(job_status(url.query)) elif path == "/api/android/displays":