Keyboard and trackpad for the Frame, through its own KDE Connect

Home → Keyboard and trackpad, in every version of Frame Control (Mac,
Windows, Linux, iPhone, iPad) with nothing to install on the device in
your hand. On a phone: a trackpad (drag, tap, two-finger scroll and
right-click) and a field that types on the Frame. On a computer: click
the pad to pass the mouse and keyboard through; Esc to stop.

First-party route: ui/frame_input_agent.py runs on the Frame and talks
KDE Connect's LAN protocol (v7) to kdeconnectd as if it were a phone.
KDE Connect isn't on the image, but Valve's package repository has it;
the agent fetches it and four libraries into ~ (no root, survives
updates), starts it, pairs by itself (accepting over D-Bus), and stops
it again when the last device disconnects. Each device has its own
identity; a stuck KDE Connect is restarted once.

Verified against the real Frame (SteamOS 0.4.1): first-time install,
pairing, pointer moves from the Mac's server and the iPhone app
(Simulator), Mac and iPhone at once, two installs at once, a frozen
daemon replaced, and the daemon stopping when the app quits.

Reviewed by GPT-6 Astra (xhigh) over seven rounds; all findings fixed
except per-event delivery acknowledgement (documented known limit).

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

ui/frame_input_agent.py

@@ -0,0 +1,376 @@
+"""Keyboard and pointer for the Steam Frame. Frame Control's server runs this ON the Frame.
+
+It speaks KDE Connect's LAN protocol (version 7, as in KDE Connect 24.02) to the
+Frame's own kdeconnectd, as a phone would, and forwards remote-input events read
+from stdin: one JSON object (or list of them) per line, each a KDE Connect
+"mousepad" request body such as {"dx": 4, "dy": -2} or {"key": "hello"}.
+KDE Connect does the typing and clicking.
+
+KDE Connect isn't installed on the Frame, but Valve's package repository for it
+has a build. The first run fetches that and the few libraries the Frame lacks
+into ~/.local/share/frame-control/kdeconnect: no root, and SteamOS updates
+leave it alone.
+
+Status goes to stdout, one JSON object per line:
+{"state": "installing" | "starting" | "pairing" | "ready" | "error", ...}.
+
+Standard library only: this runs on the Frame's own Python.
+"""
+import fcntl
+import json
+import os
+import selectors
+import shutil
+import signal
+import socket
+import ssl
+import subprocess
+import sys
+import time
+from pathlib import Path
+
+BASE = Path.home() / ".local/share/frame-control/kdeconnect"
+ROOT = BASE / "root"
+BRIDGE = BASE / "bridge"
+# kdeconnect plus the dependencies the Frame's image doesn't have (checked 2026-09-28,
+# SteamOS 0.4.1); `pacman -Sp` adds any others still missing.
+PACKAGES = ["kdeconnect", "kpeople", "libfakekey", "modemmanager-qt", "pulseaudio-qt"]
+PORT = int(os.environ.get("FRAME_INPUT_PORT", "1716"))
+UID = os.getuid()
+MOUSEPAD = "kdeconnect.mousepad.request"
+
+
+def say(state, **more):
+    print(json.dumps({"state": state, **more}), flush=True)
+
+
+def packet(kind, body):
+    return (json.dumps({"id": int(time.time() * 1000), "type": kind, "body": body}) + "\n").encode()
+
+
+# ---- KDE Connect on the Frame ------------------------------------------------
+
+def daemon_path():
+    for path in (Path("/usr/lib/kdeconnectd"), ROOT / "usr/lib/kdeconnectd"):
+        if path.exists():
+            return path
+    return None
+
+
+def install():
+    say("installing", message="Fetching KDE Connect from the Frame's package repository")
+    found = subprocess.run(["pacman", "-Sp", *PACKAGES], capture_output=True, text=True, timeout=120)
+    urls = [u for u in found.stdout.split() if u.startswith("https://")]
+    if found.returncode or not urls:
+        raise RuntimeError("Couldn't find KDE Connect in the Frame's package repository: "
+                           + (found.stderr.strip() or "no packages listed"))
+    download, stage = BASE / "download", BASE / "root.new"
+    for d in (download, stage):
+        shutil.rmtree(d, ignore_errors=True)
+        d.mkdir(parents=True)
+    for url in urls:
+        name = download / url.rsplit("/", 1)[1]
+        subprocess.run(["curl", "-fsSL", "--retry", "2", "-o", str(name), url], check=True, timeout=600)
+        if subprocess.run(["tar", "--zstd", "-xf", str(name), "-C", str(stage)], capture_output=True).returncode:
+            subprocess.run(["bsdtar", "-xf", str(name), "-C", str(stage)], check=True, capture_output=True)
+    shutil.rmtree(ROOT, ignore_errors=True)
+    stage.rename(ROOT)
+    shutil.rmtree(download, ignore_errors=True)
+
+
+def app_display():
+    """The X display that apps (not Steam's own VR menus) are on.
+
+    gamescope runs two Xwayland servers: on 2026-09-28 :0 held Steam's VR bar and
+    menus and ignored XTest pointer motion, while :1 held apps such as Chromium and
+    took it. Inferred to hold in general.
+    """
+    return ":1" if Path("/tmp/.X11-unix/X1").exists() else ":0"
+
+
+def daemon_env(daemon):
+    env = dict(os.environ, DBUS_SESSION_BUS_ADDRESS=f"unix:path=/run/user/{UID}/bus",
+               XDG_RUNTIME_DIR=f"/run/user/{UID}", DISPLAY=app_display(), QT_QPA_PLATFORM="xcb")
+    if str(daemon).startswith(str(ROOT)):
+        env.update(LD_LIBRARY_PATH=str(ROOT / "usr/lib"), QT_PLUGIN_PATH=str(ROOT / "usr/lib/qt6/plugins"),
+                   QML_IMPORT_PATH=str(ROOT / "usr/lib/qt6/qml"),
+                   XDG_DATA_DIRS=f"{ROOT / 'usr/share'}:/usr/share")
+    return env
+
+
+def listening():
+    try:
+        socket.create_connection(("127.0.0.1", PORT), 1).close()
+        return True
+    except OSError:
+        return False
+
+
+def our_daemons():
+    """Process ids of the kdeconnectd that Frame Control installed (never a system one)."""
+    pids = []
+    for proc in Path("/proc").iterdir():
+        if proc.name.isdigit():
+            try:
+                if os.readlink(proc / "exe").startswith(str(ROOT) + "/"):
+                    pids.append(int(proc.name))
+            except OSError:
+                pass
+    return pids
+
+
+def stop_daemon():
+    """Stop our kdeconnectd and wait until it's gone (so its port is closed too)."""
+    for sig, wait in ((signal.SIGTERM, 30), (signal.SIGKILL, 30)):  # tenths of a second
+        for pid in our_daemons():
+            try:
+                os.kill(pid, sig)
+            except ProcessLookupError:
+                pass
+        for _ in range(wait):
+            if not our_daemons() and not listening():
+                return
+            time.sleep(0.1)
+
+
+def ensure_daemon():
+    if listening():
+        return
+    BASE.mkdir(parents=True, exist_ok=True)
+    daemon = daemon_path()
+    if not daemon:
+        install()
+        daemon = daemon_path()
+    say("starting", message="Starting KDE Connect on the Frame")
+    log = open(BASE / "kdeconnectd.log", "ab")
+    # Its own session, so it outlives this connection and serves the next one.
+    subprocess.Popen([str(daemon)], env=daemon_env(daemon), cwd=str(Path.home()), stdin=subprocess.DEVNULL,
+                     stdout=log, stderr=log, start_new_session=True)
+    for _ in range(40):
+        if listening():
+            return
+        time.sleep(0.25)
+    raise RuntimeError(f"KDE Connect didn't start; see {BASE / 'kdeconnectd.log'} on the Frame")
+
+
+def qdbus(device, method):
+    """Call a method on KDE Connect's D-Bus object for our device; its output, or None."""
+    env = dict(os.environ, DBUS_SESSION_BUS_ADDRESS=f"unix:path=/run/user/{UID}/bus")
+    try:
+        r = subprocess.run(["qdbus6", "org.kde.kdeconnect", f"/modules/kdeconnect/devices/{device}",
+                            f"org.kde.kdeconnect.device.{method}"], capture_output=True, text=True, env=env, timeout=5)
+    except (OSError, subprocess.TimeoutExpired):
+        return None
+    return r.stdout.strip() if r.returncode == 0 else None
+
+
+# ---- our identity --------------------------------------------------------------
+
+def identity(client):
+    """A device id and certificate for this client, made once and kept (pairing is tied to them).
+
+    Each computer or phone gets its own: KDE Connect keeps one connection per device,
+    so a shared identity would make them knock each other off.
+    """
+    folder = BRIDGE / client
+    folder.mkdir(parents=True, exist_ok=True)
+    id_file, cert, key = folder / "id", folder / "cert.pem", folder / "key.pem"
+    if not (id_file.exists() and cert.exists() and key.exists()):
+        device = "framecontrol_" + os.urandom(12).hex()  # KDE Connect wants 32-38 of [A-Za-z0-9_]
+        subprocess.run(["openssl", "req", "-x509", "-newkey", "ec", "-pkeyopt", "ec_paramgen_curve:prime256v1",
+                        "-nodes", "-days", "3650", "-subj", f"/O=KDE/OU=Kde connect/CN={device}",
+                        "-keyout", str(key), "-out", str(cert)], check=True, capture_output=True)
+        os.chmod(key, 0o600)
+        id_file.write_text(device)
+    return id_file.read_text().strip(), cert, key
+
+
+# ---- the link ------------------------------------------------------------------
+
+class Link:
+    """One TLS connection to kdeconnectd, as a paired device that sends remote input."""
+
+    def __init__(self, device, cert, key, port=PORT, name="Frame Control"):
+        self.device, self.buf, self.keyboard = device, b"", None
+        raw = socket.create_connection(("127.0.0.1", port), 5)
+        raw.sendall(packet("kdeconnect.identity", {
+            "deviceId": device, "deviceName": name, "deviceType": "phone", "protocolVersion": 7,
+            "incomingCapabilities": [], "outgoingCapabilities": [MOUSEPAD], "tcpPort": port}))
+        # KDE Connect's rule: whoever opened the TCP connection is the TLS server.
+        ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
+        ctx.load_cert_chain(str(cert), str(key))
+        ctx.verify_mode = ssl.CERT_NONE  # both sides are on this machine
+        self.sock = ctx.wrap_socket(raw, server_side=True)
+        self.sock.setblocking(False)
+
+    def send(self, body):
+        # Bounded: if KDE Connect stops reading, fail (and be restarted) rather than hang.
+        self.sock.settimeout(5)
+        try:
+            self.sock.sendall(packet(MOUSEPAD, body))
+        finally:
+            self.sock.setblocking(False)
+
+    def pair(self, paired, accept, timeout=15):
+        """Ask to pair and accept it on KDE Connect's side (we control both ends).
+
+        Only asks when not already paired: a pair request to a device that is
+        already paired makes KDE Connect unpair it.
+        """
+        if paired():
+            return
+        self.sock.settimeout(5)
+        self.sock.sendall(packet("kdeconnect.pair", {"pair": True}))
+        self.sock.setblocking(False)
+        end = time.time() + timeout
+        while time.time() < end:
+            accept()
+            self.read(0.5)
+            if paired():
+                return
+        raise RuntimeError("KDE Connect didn't accept the pairing")
+
+    def read(self, wait=0.0):
+        """Packets waiting from kdeconnectd; None once it has closed the connection."""
+        if wait:
+            sel = selectors.DefaultSelector()
+            sel.register(self.sock, selectors.EVENT_READ)
+            sel.select(wait)
+            sel.close()
+        try:
+            while True:
+                chunk = self.sock.recv(65536)
+                if not chunk:
+                    return None
+                self.buf += chunk
+        except (ssl.SSLWantReadError, BlockingIOError):
+            pass
+        out = []
+        while b"\n" in self.buf:
+            line, self.buf = self.buf.split(b"\n", 1)
+            if line.strip():
+                p = json.loads(line)
+                if p.get("type") == "kdeconnect.mousepad.keyboardstate":
+                    self.keyboard = bool(p.get("body", {}).get("state"))
+                out.append(p)
+        return out
+
+
+def events(line):
+    """The event bodies in one stdin line (an object or a list of objects)."""
+    try:
+        value = json.loads(line)
+    except ValueError:
+        return []
+    return [e for e in (value if isinstance(value, list) else [value]) if isinstance(e, dict) and e]
+
+
+def connect(device, cert, key, name):
+    link = Link(device, cert, key, name=name)
+    link.pair(lambda: qdbus(device, "isPaired") == "true", lambda: qdbus(device, "acceptPairing"))
+    link.read(0.5)  # its hello, including whether it can type
+    return link
+
+
+def client_args():
+    """argv: a folder-safe id for the computer or phone, and the name KDE Connect shows for it."""
+    client = sys.argv[1] if len(sys.argv) > 1 else "default"
+    client = "".join(c for c in client if c.isalnum() or c in "-_")[:64] or "default"
+    name = (sys.argv[2] if len(sys.argv) > 2 else "")[:60].strip()
+    return client, f"Frame Control ({name})" if name else "Frame Control"
+
+
+def main():
+    client, name = client_args()
+    # A dropped ssh (the Frame slept, the app quit) hangs up on us: exit through the
+    # clean-up below rather than dying on the spot.
+    for sig in (signal.SIGHUP, signal.SIGTERM):
+        signal.signal(sig, lambda *_: sys.exit(0))
+    BASE.mkdir(parents=True, exist_ok=True)
+    # Every agent holds this lock shared while it runs. The last one out gets it
+    # exclusively and stops KDE Connect, so it runs, and shows up on the network,
+    # only while something is using the keyboard and trackpad.
+    clients = open(BASE / "clients.lock", "w")
+    fcntl.flock(clients, fcntl.LOCK_SH)
+    try:
+        return run(client, name)
+    finally:
+        # Finish the clean-up even if a second hang-up or TERM arrives meanwhile.
+        for sig in (signal.SIGHUP, signal.SIGTERM):
+            signal.signal(sig, signal.SIG_IGN)
+        fcntl.flock(clients, fcntl.LOCK_UN)
+        try:
+            fcntl.flock(clients, fcntl.LOCK_EX | fcntl.LOCK_NB)
+        except OSError:
+            pass  # another device is still using it
+        else:
+            with daemon_lock():
+                stop_daemon()
+
+
+class daemon_lock:
+    """Installing, starting and restarting KDE Connect happen one agent at a time."""
+
+    def __enter__(self):
+        self.file = open(BASE / "daemon.lock", "w")
+        fcntl.flock(self.file, fcntl.LOCK_EX)
+
+    def __exit__(self, *_):
+        self.file.close()
+
+
+def run(client, name):
+    try:
+        with daemon_lock():
+            ensure_daemon()
+        device, cert, key = identity(client)
+        say("pairing")
+        seen = our_daemons()
+        try:
+            link = connect(device, cert, key, name)
+        except (OSError, RuntimeError):
+            if not seen:
+                raise
+            # Ours, but not answering (KDE Connect 24.02 can hang, for one after
+            # unpairing a device that's offline): start it afresh, once. If another
+            # agent already replaced it, just use the new one.
+            say("starting", message="Restarting KDE Connect on the Frame")
+            with daemon_lock():
+                if set(our_daemons()) & set(seen):
+                    stop_daemon()
+                ensure_daemon()
+            link = connect(device, cert, key, name)
+    except (OSError, RuntimeError, subprocess.SubprocessError) as e:
+        say("error", message=str(e))
+        return 1
+    say("ready", keyboard=link.keyboard is not False)
+    stdin, pending = sys.stdin.fileno(), b""
+    sel = selectors.DefaultSelector()
+    sel.register(stdin, selectors.EVENT_READ)
+    sel.register(link.sock, selectors.EVENT_READ)
+    while True:
+        for key_, _ in sel.select(30):
+            if key_.fileobj == stdin:
+                chunk = os.read(stdin, 65536)  # raw reads: a buffered readline could strand lines select can't see
+                if not chunk:  # the server went away
+                    return 0
+                *lines, pending = (pending + chunk).split(b"\n")
+                try:
+                    for line in lines:
+                        for body in events(line):
+                            link.send(body)
+                except OSError as e:
+                    say("error", message=f"Lost KDE Connect: {e}")
+                    return 1
+            else:
+                packets = link.read()
+                if packets is None:
+                    say("error", message="KDE Connect closed the connection")
+                    return 1
+                if any(p.get("type") == "kdeconnect.pair" and not p.get("body", {}).get("pair") for p in packets):
+                    say("error", message="KDE Connect unpaired Frame Control")
+                    return 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())

ui/index.html

@@ -322,6 +322,19 @@
                              border-radius: 3px; height: 32px; padding: 0 8px; font: inherit; font-size: 13px; }
   /* Touch screens can't hover: keep the library's name and Play button showing. */
   @media (hover: none) { .capsule .over { opacity: 1; } .capsule:hover { transform: none; } }
+  /* ---- keyboard and trackpad ---- */
+  .pad-area { position: relative; height: 190px; margin: 12px 0; border-radius: 4px; background: rgba(0,0,0,.28);
+              border: 1px dashed rgba(255,255,255,.14); display: grid; place-items: center; cursor: pointer;
+              touch-action: none; user-select: none; -webkit-user-select: none; -webkit-touch-callout: none; outline: none; }
+  .pad-area:focus-visible, .pad-area.captured { border: 1px solid var(--blue); box-shadow: 0 0 0 1px var(--blue) inset; }
+  .pad-area.off { cursor: default; opacity: .55; }
+  .pad-hint { max-width: 420px; padding: 0 16px; text-align: center; color: var(--muted); font-size: 13px; pointer-events: none; }
+  .pad-keys { display: flex; flex-wrap: wrap; gap: 8px; align-items: center; }
+  .pad-keys input { flex: 1 1 220px; min-width: 0; height: 32px; padding: 0 10px; border-radius: 3px; border: 1px solid transparent;
+                    background: rgba(0,0,0,.28); color: var(--text); font: inherit; }
+  .pad-keys input:focus { outline: none; border-color: var(--blue); }
+  #pad .hint { margin-top: 10px; }
+
   /* ---- phones, upright or on their side: tabs move to a bottom bar, as in iOS apps. Every edge keeps clear of
      the safe area (notch or Dynamic Island, rounded corners, home indicator); env() is zero on desktops. ---- */
   @media (max-width: 640px), (max-height: 500px) and (hover: none) {
@@ -358,6 +371,10 @@
     button.small { height: 32px; }
     .actions button { height: 46px; }
     .cat-tools select { max-width: none; flex: 1 1 100%; }
+    .pad-area { height: 240px; }
+    .pad-keys .seg { flex: 1 1 100%; display: grid; grid-template-columns: repeat(4, 1fr); }
+    .pad-keys .pad-clicks { grid-template-columns: repeat(2, 1fr); }
+    .pad-keys .seg button { padding: 0 4px; justify-content: center; text-align: center; }
     /* The on-screen keyboard covers the bottom of the screen; the tab bar would ride on top of it. */
     body.typing nav { display: none; }
   }
@@ -471,6 +488,36 @@
     </section>
   </div>
 
+  <section class="panel" id="pad">
+    <div class="shelf-head"><h2>Keyboard and trackpad</h2><span class="sub" id="padState"></span><span class="spacer"></span>
+      <button class="small" id="padOn">Turn on</button>
+    </div>
+    <div class="pad-area" id="padArea" tabindex="0" role="application" aria-label="Trackpad for the Frame">
+      <div class="pad-hint" id="padHint"></div>
+    </div>
+    <div class="pad-keys" aria-label="Keys and clicks">
+      <input id="padType" type="text" placeholder="Type here to type on the Frame" autocomplete="off" autocapitalize="off"
+             autocorrect="off" spellcheck="false" enterkeyhint="send">
+      <span class="seg pad-clicks">
+        <button data-pad-click="singleclick" title="Left click">Click</button>
+        <button data-pad-click="rightclick" title="Right click">Right-click</button>
+      </span>
+      <span class="seg pad-special">
+        <button data-pad-key="14" title="Escape">Esc</button>
+        <button data-pad-key="2" title="Tab">Tab</button>
+        <button data-pad-key="4" title="Left arrow" aria-label="Left arrow">←</button>
+        <button data-pad-key="5" title="Up arrow" aria-label="Up arrow">↑</button>
+        <button data-pad-key="7" title="Down arrow" aria-label="Down arrow">↓</button>
+        <button data-pad-key="6" title="Right arrow" aria-label="Right arrow">→</button>
+        <button data-pad-key="1" title="Backspace" aria-label="Backspace">⌫</button>
+        <button data-pad-key="12" title="Enter">Enter</button>
+      </span>
+    </div>
+    <div class="hint">Types and points in apps on the Frame, such as Chromium or the desktop's Linux apps; Steam's own VR menus
+      don't take it. It works through KDE Connect on the Frame: the first time, Frame Control fetches Valve's build of it
+      (an 8 MB download, 82 MB unpacked) into your home folder there and pairs with it. Nothing to install on this device.</div>
+  </section>
+
   <section id="shots">
     <div class="shelf-head"><h2>Screenshots</h2><span class="count" id="shotCount"></span><span class="spacer"></span>
       <button class="small" id="shotsRefresh">Refresh</button>
@@ -2128,6 +2175,219 @@ window.addEventListener("hashchange", showPage);
 const typesText = el => el?.matches?.('textarea, input:not([type=range], [type=checkbox], [type=radio], [type=file], [type=button])');
 document.addEventListener("focusin", e => document.body.classList.toggle("typing", !!typesText(e.target)));
 document.addEventListener("focusout", () => document.body.classList.remove("typing"));
+
+// ---- keyboard and trackpad: events go to KDE Connect on the Frame (see ui/frame_input_agent.py) ----
+const pad = { state: "off", queue: [], sending: false, poll: null, captured: false };
+const PAD_STATES = {
+  off: "", starting: "Connecting…", installing: "Setting up KDE Connect on the Frame (first time only)…",
+  pairing: "Pairing with KDE Connect…", ready: "On",
+};
+// KDE Connect's specialKey numbers (plugins/mousepad in its source).
+const PAD_SPECIAL = { Backspace: 1, Tab: 2, ArrowLeft: 4, ArrowUp: 5, ArrowRight: 6, ArrowDown: 7, PageUp: 8,
+  PageDown: 9, Home: 10, End: 11, Enter: 12, Delete: 13, Escape: 14,
+  ...Object.fromEntries(Array.from({ length: 12 }, (_, i) => [`F${i + 1}`, 21 + i])) };
+const IS_MAC = /^Mac/.test(navigator.platform);
+const TOUCH = matchMedia("(hover: none)").matches;
+
+function padShow(status) {
+  pad.state = status.state || "off";
+  const ready = pad.state === "ready";
+  const busy = !ready && pad.state !== "off" && pad.state !== "error";
+  $("padState").textContent = pad.state === "error" ? "" : PAD_STATES[pad.state] || "";
+  $("padOn").hidden = ready || busy;
+  $("padOn").textContent = pad.state === "error" ? "Try again" : "Turn on";
+  $("padArea").classList.toggle("off", !ready);
+  $("padHint").textContent = pad.state === "error" ? status.message || "Couldn't reach KDE Connect on the Frame."
+    : !ready ? (busy ? PAD_STATES[pad.state] : "Turn on to use this as a trackpad for the Frame.")
+    : TOUCH ? "Drag to move the pointer · tap to click · two fingers to scroll · two-finger tap to right-click"
+    : pad.captured ? "Your mouse and keyboard now control the Frame. Press Esc to stop."
+    : "Click here to use your mouse and keyboard on the Frame (Esc to stop), or drag like a trackpad.";
+  clearTimeout(pad.poll);
+  if (ready && pad.queue.length) padFlush();
+  if (busy) pad.poll = setTimeout(async () => padShow(await api("/api/input").catch(e => ({ state: "error", message: e.message }))), 1000);
+}
+async function padStart() {
+  localStorage.padUsed = "1";  // from now on it turns on with the page
+  padShow({ state: "starting" });
+  padShow(await api("/api/input?start=1").catch(e => ({ state: "error", message: e.message })));
+}
+$("padOn").onclick = padStart;
+
+// Queue events and send them one request at a time, merging pointer moves made meanwhile.
+const isMove = e => Object.keys(e).every(k => k === "dx" || k === "dy");
+function padSend(event) {
+  // While it reconnects, keys and clicks wait in the queue; pointer moves would be stale.
+  // A button release is always kept, so a drag never leaves the button held on the Frame.
+  const room = pad.queue.length < 200 || event.singlerelease;
+  const reconnecting = ["starting", "pairing", "installing"].includes(pad.state) && room;
+  // Even after an error: a button pressed on the Frame must be let go once it's back.
+  const release = event.singlerelease && pad.state !== "off";
+  if (pad.state !== "ready" && !(reconnecting && !isMove(event)) && !release) return;
+  const last = pad.queue[pad.queue.length - 1];
+  if (last && isMove(last) && isMove(event)) { last.dx = (last.dx || 0) + (event.dx || 0); last.dy = (last.dy || 0) + (event.dy || 0); }
+  else pad.queue.push(event);
+  padFlush();
+}
+// Events that didn't reach the Frame (the link had dropped, and is reconnecting): keep
+// the keys and clicks for when it's back, but not pointer moves, which would be stale.
+// Button releases are kept whatever the queue's length.
+function padKeep(events) {
+  pad.queue = [...events, ...pad.queue].filter(e => !isMove(e)).filter((e, i) => i < 200 || e.singlerelease);
+}
+async function padFlush() {
+  if (pad.sending || !pad.queue.length || pad.state !== "ready") return;
+  pad.sending = true;
+  const events = pad.queue.splice(0, 200);
+  try {
+    const status = await api("/api/input", { events });
+    if (!status.sent) padKeep(events);
+    if (status.state !== "ready") padShow(status);
+  } catch (e) {
+    padKeep(events);
+    padShow({ state: "error", message: e.message });
+  } finally {
+    pad.sending = false;
+    if (pad.queue.length) padFlush();
+  }
+}
+function padKey(e) {
+  const mods = { ...(e.ctrlKey || (IS_MAC && e.metaKey) ? { ctrl: true } : {}), ...(e.altKey ? { alt: true } : {}),
+                 ...(!IS_MAC && e.metaKey ? { super: true } : {}) };
+  if (PAD_SPECIAL[e.key]) return { specialKey: PAD_SPECIAL[e.key], ...mods, ...(e.shiftKey ? { shift: true } : {}) };
+  if (e.key.length === 1) return { key: e.key, ...mods };  // already shifted, so no shift flag
+  return null;
+}
+
+// Touch: one finger moves, a quick tap clicks; two fingers scroll, or right-click when tapped.
+const PAD_SPEED = 1.6, PAD_SCROLL_STEP = 18;
+const touches = new Map();
+let gesture = null;
+$("padArea").addEventListener("pointerdown", e => {
+  if (pad.state === "off" || pad.state === "error") { padStart(); return; }
+  if (pad.state !== "ready") return;
+  if (e.pointerType === "mouse") {
+    if (pad.captured) return;  // mousedown and mouseup below send the clicks
+    // Lock the pointer to the pad and pass the whole mouse and keyboard through; where
+    // that isn't allowed, the mouse drags like a finger on a trackpad instead.
+    if (e.button === 0 && !pad.noLock && $("padArea").requestPointerLock) { $("padArea").requestPointerLock(); return; }
+    // Only the left button drags and taps like a finger; the others are clicks of their own.
+    if (e.button === 1 || e.button === 2) { padSend(e.button === 2 ? { rightclick: true } : { middleclick: true }); return; }
+    if (e.button !== 0) return;
+  }
+  try { $("padArea").setPointerCapture(e.pointerId); } catch {}  // keeps the drag if the finger leaves the pad
+  touches.set(e.pointerId, { x: e.clientX, y: e.clientY });
+  if (touches.size === 1) gesture = { start: performance.now(), moved: 0, fingers: 1, scroll: 0 };
+  else if (gesture) gesture.fingers = Math.max(gesture.fingers, touches.size);
+  e.preventDefault();
+});
+$("padArea").addEventListener("pointermove", e => {
+  if (pad.captured) { padSend({ dx: e.movementX, dy: e.movementY }); return; }
+  const t = touches.get(e.pointerId);
+  if (!t || !gesture) return;
+  const dx = e.clientX - t.x, dy = e.clientY - t.y;
+  t.x = e.clientX; t.y = e.clientY;
+  gesture.moved += Math.abs(dx) + Math.abs(dy);
+  if (touches.size >= 2) {
+    // Scroll with the lead finger only, a notch per step, like a laptop trackpad (content follows the fingers).
+    if (e.pointerId !== touches.keys().next().value) return;
+    gesture.scroll += dy;
+    while (Math.abs(gesture.scroll) >= PAD_SCROLL_STEP) {
+      const up = gesture.scroll > 0;
+      padSend({ scroll: true, dy: up ? 1 : -1 });
+      gesture.scroll -= up ? PAD_SCROLL_STEP : -PAD_SCROLL_STEP;
+    }
+  } else if (gesture.fingers === 1) {
+    padSend({ dx: dx * PAD_SPEED, dy: dy * PAD_SPEED });
+  }
+});
+function padLift(e) {
+  if (!touches.delete(e.pointerId) || touches.size || !gesture) return;
+  const quick = performance.now() - gesture.start < 250 && gesture.moved < 10;
+  if (quick) padSend(gesture.fingers >= 2 ? { rightclick: true } : { singleclick: true });
+  gesture = null;
+}
+$("padArea").addEventListener("pointerup", padLift);
+$("padArea").addEventListener("pointercancel", e => { touches.delete(e.pointerId); if (!touches.size) gesture = null; });
+
+// A computer's mouse and keyboard: captured while the pointer is locked to the pad.
+document.addEventListener("pointerlockerror", () => { pad.noLock = true; });
+document.addEventListener("pointerlockchange", () => {
+  pad.captured = document.pointerLockElement === $("padArea");
+  window.frameApp?.captureKeys?.(pad.captured);  // the desktop app's menu shortcuts go to the Frame too
+  // Esc or switching windows mid-drag: let go of the button on the Frame as well.
+  if (!pad.captured && pad.held) { padSend({ singlerelease: true }); pad.held = false; }
+  $("padArea").classList.toggle("captured", pad.captured);
+  padShow({ state: pad.state });
+});
+$("padArea").addEventListener("mousedown", e => {
+  if (!pad.captured) return;
+  if (e.button === 0) { padSend({ singlehold: true }); pad.held = true; }
+  else if (e.button === 1) padSend({ middleclick: true });
+  else if (e.button === 2) padSend({ rightclick: true });
+});
+$("padArea").addEventListener("mouseup", e => { if (pad.held && e.button === 0) { padSend({ singlerelease: true }); pad.held = false; } });
+$("padArea").addEventListener("contextmenu", e => e.preventDefault());
+$("padArea").addEventListener("wheel", e => {
+  if (pad.state !== "ready") return;
+  e.preventDefault();
+  if (e.deltaY) padSend({ scroll: true, dy: e.deltaY > 0 ? -1 : 1 });
+}, { passive: false });
+// Capture phase, so the page's own shortcuts (tabs 1-4, R, zoom) don't also fire.
+window.addEventListener("keydown", e => {
+  if (!pad.captured) return;
+  e.preventDefault(); e.stopImmediatePropagation();
+  const event = padKey(e);
+  if (event) padSend(event);
+}, true);
+
+// The text field: each keystroke goes to the Frame. While focused it holds one space,
+// so Backspace always has something to delete; empty otherwise, to show its prompt.
+const padType = $("padType");
+padType.addEventListener("beforeinput", e => {
+  if (e.inputType === "insertCompositionText") return;  // IME composing; sent on compositionend
+  e.preventDefault();
+  if (pad.state === "off" || pad.state === "error") padStart();  // what's typed meanwhile waits in the queue
+  const text = e.data ?? e.dataTransfer?.getData("text/plain") ?? "";
+  if (/^insert(Text|ReplacementText|FromPaste|FromDrop)$/.test(e.inputType) && text) padText(text);
+  else if (e.inputType === "deleteContentBackward") padSend({ specialKey: 1 });
+  else if (e.inputType === "deleteWordBackward") padSend({ specialKey: 1, ctrl: true });  // Ctrl+Backspace: a word, on Linux
+  else if (e.inputType === "deleteContentForward") padSend({ specialKey: 13 });
+  else if (e.inputType === "deleteWordForward") padSend({ specialKey: 13, ctrl: true });
+  else if (e.inputType === "insertLineBreak" || e.inputType === "insertParagraph") padSend({ specialKey: 12 });
+});
+padType.addEventListener("compositionend", e => { if (e.data) padText(e.data); padType.value = " "; padCaret(); });
+// Text in pieces the server takes (500 characters), with Enter for each line break.
+function padText(text) {
+  text.replace(/\r\n?/g, "\n").split("\n").forEach((line, i) => {
+    if (i) padSend({ specialKey: 12 });
+    const chars = Array.from(line);  // whole characters, so an emoji isn't cut in two
+    for (let at = 0; at < chars.length; at += 500) padSend({ key: chars.slice(at, at + 500).join("") });
+  });
+}
+padType.addEventListener("keydown", e => {
+  const special = PAD_SPECIAL[e.key];
+  // Text and deletions arrive as beforeinput; this takes the keys that don't edit the field.
+  if (!special || e.key === "Backspace" || e.key === "Delete" || e.isComposing) return;
+  e.preventDefault();
+  padSend(padKey(e));
+});
+// The caret stays after the space, so Backspace always has something to delete.
+const padCaret = () => requestAnimationFrame(() => padType.setSelectionRange(1, 1));
+padType.addEventListener("focus", () => { padType.value = " "; padCaret(); if (pad.state === "off") padStart(); });
+padType.addEventListener("blur", () => { padType.value = ""; });
+padType.addEventListener("input", e => {
+  if (e.isComposing) return;  // an input method is still composing; compositionend sends it
+  if (padType.value !== " ") padType.value = " ";
+  padCaret();
+});
+padType.addEventListener("click", padCaret);
+document.querySelector("#pad .pad-keys").addEventListener("click", e => {
+  const b = e.target.closest("button");
+  if (!b) return;
+  if (b.dataset.padClick) padSend({ [b.dataset.padClick]: true });
+  else if (b.dataset.padKey) padSend({ specialKey: +b.dataset.padKey });
+});
+api("/api/input").then(s => s.state === "off" && localStorage.padUsed ? padStart() : padShow(s), () => padShow({ state: "off" }));
 showPage();
 document.addEventListener("keydown", e => {
   if (e.metaKey || e.ctrlKey || e.altKey || /INPUT|TEXTAREA|SELECT/.test(document.activeElement.tagName) || document.querySelector("dialog[open]")) return;

ui/server.py

@@ -10,6 +10,7 @@ Env:   FRAME_ALIAS (default frame)
        FRAME_LOCAL=1   run on the Frame itself (the iPhone app starts it there over SSH)
        FRAME_UI_KEY    required X-Frame-UI value (the iPhone app passes a fresh one)
        FRAME_DEVICE    what to call the device the page runs on (e.g. iPhone)
+       FRAME_CLIENT    a stable id for that device (keyboard-and-trackpad pairing is kept per id)
 """
 import argparse
 import base64
@@ -54,6 +55,8 @@ if LOCAL:
     os.environ["PATH"] = f"{HERE / 'local-bin'}{os.pathsep}{os.environ.get('PATH', '')}"
 UI_KEY = os.environ.get("FRAME_UI_KEY") or "1"
 DEVICE = os.environ.get("FRAME_DEVICE") or "phone"
+# What the Frame's KDE Connect calls this device (keyboard and trackpad).
+INPUT_NAME = DEVICE if LOCAL else socket.gethostname().split(".")[0]
 FRAME = os.environ.get("FRAME_ALIAS", "frame")
 if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._-]*", FRAME):
     sys.exit(f"FRAME_ALIAS must be a plain host alias, not {FRAME!r}")
@@ -497,6 +500,161 @@ def clipboard(body):
     return {"message": ssh(PASTE_CMD, stdin=text, timeout=30).strip()}
 
 
+# ---- keyboard and pointer (KDE Connect on the Frame, see frame_input_agent.py) ----
+
+INPUT_FLAGS = ("singleclick", "doubleclick", "middleclick", "rightclick", "singlehold", "singlerelease",
+               "scroll", "ctrl", "alt", "shift", "super")
+INPUT_MOVE_LIMIT = 2000  # pixels per event
+INPUT_TEXT_LIMIT = 500  # characters per event
+INPUT_BATCH_LIMIT = 200  # events per request
+
+
+def input_event(event):
+    """A KDE Connect remote-input body with only the fields it knows, in range."""
+    if not isinstance(event, dict):
+        raise Failure("each input event must be an object", 400)
+    out = {}
+    for name in ("dx", "dy"):
+        value = event.get(name)
+        if value is None:
+            continue
+        if isinstance(value, bool) or not isinstance(value, (int, float)) or value != value:
+            raise Failure(f"{name} must be a number", 400)
+        out[name] = max(-INPUT_MOVE_LIMIT, min(INPUT_MOVE_LIMIT, round(float(value), 2)))
+    for name in INPUT_FLAGS:
+        if event.get(name) is True:
+            out[name] = True
+    key = event.get("key")
+    if key is not None:
+        if not isinstance(key, str) or not 0 < len(key) <= INPUT_TEXT_LIMIT:
+            raise Failure(f"key must be text of 1 to {INPUT_TEXT_LIMIT} characters", 400)
+        out["key"] = key
+    special = event.get("specialKey")
+    if special is not None:
+        # KDE Connect's numbering: 1 Backspace … 14 Escape, 21-32 F1-F12.
+        if isinstance(special, bool) or not isinstance(special, int) or not 1 <= special <= 32:
+            raise Failure("specialKey must be a whole number from 1 to 32", 400)
+        out["specialKey"] = special
+    if not set(out) - {"ctrl", "alt", "shift", "super"}:
+        raise Failure("input event has nothing to do", 400)
+    return out
+
+
+def input_client():
+    """A stable id for this device, so KDE Connect on the Frame keeps its pairing apart.
+
+    The iPhone app passes one (FRAME_CLIENT). A computer makes one the first time
+    and keeps it: host names alone can clash (desk.home and desk.office).
+    """
+    if os.environ.get("FRAME_CLIENT"):
+        return os.environ["FRAME_CLIENT"]
+    if LOCAL:
+        return DEVICE
+    path = frame_host.data_dir("input-client-id")
+    try:
+        saved = path.read_text().strip()
+        if re.fullmatch(r"[A-Za-z0-9_-]{4,64}", saved):
+            return saved
+    except OSError:
+        pass
+    made = (re.sub(r"[^A-Za-z0-9-]", "", INPUT_NAME)[:24] or "computer") + "-" + secrets.token_hex(4)
+    try:
+        path.parent.mkdir(parents=True, exist_ok=True)
+        path.write_text(made)
+    except OSError:
+        pass  # still works this time; it pairs again next time
+    return made
+
+
+class InputAgent:
+    """frame_input_agent.py running on the Frame, fed events over one long-lived ssh.
+
+    It sets up KDE Connect there if needed, pairs, and reports its state
+    ({"state": "off" | "installing" | "starting" | "pairing" | "ready" | "error"}).
+    """
+
+    def __init__(self, source=HERE / "frame_input_agent.py"):
+        self.source, self.proc, self.lock = source, None, threading.Lock()
+        self.status = {"state": "off"}
+
+    def command(self):
+        code = base64.b64encode(self.source.read_bytes()).decode()
+        client = input_client()
+        return ("python3 -u -c " + shlex.quote(
+            f"import base64;exec(compile(base64.b64decode('{code}'),'frame_input_agent','exec'))")
+            + f" {shlex.quote(client)} {shlex.quote(INPUT_NAME)}")
+
+    def start(self):
+        with self.lock:
+            if self.proc and self.proc.poll() is None:
+                return
+            ensure_master()
+            self.status = {"state": "starting"}
+            errors = tempfile.TemporaryFile()
+            self.proc = proc = subprocess.Popen([*SSH, FRAME, self.command()], stdin=subprocess.PIPE,
+                                                stdout=subprocess.PIPE, stderr=errors)
+            _live_tunnels.add(proc)
+        threading.Thread(target=self._watch, args=(proc, errors), daemon=True).start()
+
+    def _watch(self, proc, errors):
+        for line in proc.stdout:
+            try:
+                status = json.loads(line)
+            except ValueError:
+                continue
+            if isinstance(status, dict) and isinstance(status.get("state"), str):
+                with self.lock:
+                    if self.proc is proc:
+                        self.status = status
+        proc.wait()
+        _live_tunnels.discard(proc)
+        errors.seek(0)
+        detail = strip_ansi(errors.read().decode(errors="replace")).strip()
+        with self.lock:
+            if self.proc is proc and self.status.get("state") != "error":
+                message = detail.splitlines()[-1] if detail else "The connection to the Frame ended"
+                friendly = unreachable(message)
+                self.status = {"state": "error", "message": friendly or message, **({"offline": True} if friendly else {})}
+
+    def send(self, events):
+        """Forward events if the agent is ready; start it if it isn't running.
+
+        Returns the state, with "sent" saying whether the events went; if not,
+        the page keeps them and sends them again once the state is "ready".
+        """
+        with self.lock:
+            proc, ready = self.proc, self.status.get("state") == "ready"
+        sent = False
+        if not (proc and proc.poll() is None):
+            self.start()
+        elif ready and events:
+            try:
+                proc.stdin.write((json.dumps(events) + "\n").encode())
+                proc.stdin.flush()
+                sent = True
+            except (BrokenPipeError, OSError, ValueError):
+                pass  # _watch reports how it ended
+        with self.lock:
+            return {**self.status, "sent": sent}
+
+    def stop(self):
+        with self.lock:
+            proc, self.proc, self.status = self.proc, None, {"state": "off"}
+        if proc and proc.poll() is None:
+            proc.terminate()
+
+
+_input = InputAgent()
+
+
+def remote_input(body):
+    """{"events": [...]} sends keyboard and pointer events; {} (or none yet) just starts the agent."""
+    events = body.get("events", [])
+    if not isinstance(events, list) or len(events) > INPUT_BATCH_LIMIT:
+        raise Failure(f"events must be a list of at most {INPUT_BATCH_LIMIT}", 400)
+    return _input.send([input_event(e) for e in events])
+
+
 def flatpak(body):
     app, action = str(body.get("id", "")), body.get("action")
     if not FLATPAK_ID.match(app):
@@ -1214,6 +1372,7 @@ def _sweep_one(prefix, d):
 
 
 POST = {"/api/android/display": android_display, "/api/android": android, "/api/titles": titles, "/api/launch": launch, "/api/steam": steam, "/api/volume": set_volume, "/api/clipboard": clipboard,
+        "/api/input": remote_input,
         "/api/flatpak": flatpak, "/api/open": open_thing, "/api/shots/save": save_shots,
         "/api/webinstall/check": webinstall_check, "/api/webinstall/start": webinstall_start,
         "/api/webinstall/cancel": webinstall_cancel}
@@ -1327,6 +1486,8 @@ class Handler(BaseHTTPRequestHandler):
                 self.send_json({"titles": frame_titles.list_titles()})
             elif path == "/api/titles/job":
                 self.send_json(title_job(url.query))
+            elif path == "/api/input":
+                self.send_json(_input.send([]) if parse_qs(url.query).get("start") == ["1"] else dict(_input.status))
             elif path == "/api/job":
                 self.send_json(job_status(url.query))
             elif path == "/api/android/displays":