Keyboard and trackpad for the Frame, through its own KDE Connect

Home → Keyboard and trackpad, in every version of Frame Control (Mac,
Windows, Linux, iPhone, iPad) with nothing to install on the device in
your hand. On a phone: a trackpad (drag, tap, two-finger scroll and
right-click) and a field that types on the Frame. On a computer: click
the pad to pass the mouse and keyboard through; Esc to stop.

First-party route: ui/frame_input_agent.py runs on the Frame and talks
KDE Connect's LAN protocol (v7) to kdeconnectd as if it were a phone.
KDE Connect isn't on the image, but Valve's package repository has it;
the agent fetches it and four libraries into ~ (no root, survives
updates), starts it, pairs by itself (accepting over D-Bus), and stops
it again when the last device disconnects. Each device has its own
identity; a stuck KDE Connect is restarted once.

Verified against the real Frame (SteamOS 0.4.1): first-time install,
pairing, pointer moves from the Mac's server and the iPhone app
(Simulator), Mac and iPhone at once, two installs at once, a frozen
daemon replaced, and the daemon stopping when the app quits.

Reviewed by GPT-6 Astra (xhigh) over seven rounds; all findings fixed
except per-event delivery acknowledgement (documented known limit).

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

ui/index.html

@@ -322,6 +322,19 @@
                              border-radius: 3px; height: 32px; padding: 0 8px; font: inherit; font-size: 13px; }
   /* Touch screens can't hover: keep the library's name and Play button showing. */
   @media (hover: none) { .capsule .over { opacity: 1; } .capsule:hover { transform: none; } }
+  /* ---- keyboard and trackpad ---- */
+  .pad-area { position: relative; height: 190px; margin: 12px 0; border-radius: 4px; background: rgba(0,0,0,.28);
+              border: 1px dashed rgba(255,255,255,.14); display: grid; place-items: center; cursor: pointer;
+              touch-action: none; user-select: none; -webkit-user-select: none; -webkit-touch-callout: none; outline: none; }
+  .pad-area:focus-visible, .pad-area.captured { border: 1px solid var(--blue); box-shadow: 0 0 0 1px var(--blue) inset; }
+  .pad-area.off { cursor: default; opacity: .55; }
+  .pad-hint { max-width: 420px; padding: 0 16px; text-align: center; color: var(--muted); font-size: 13px; pointer-events: none; }
+  .pad-keys { display: flex; flex-wrap: wrap; gap: 8px; align-items: center; }
+  .pad-keys input { flex: 1 1 220px; min-width: 0; height: 32px; padding: 0 10px; border-radius: 3px; border: 1px solid transparent;
+                    background: rgba(0,0,0,.28); color: var(--text); font: inherit; }
+  .pad-keys input:focus { outline: none; border-color: var(--blue); }
+  #pad .hint { margin-top: 10px; }
+
   /* ---- phones, upright or on their side: tabs move to a bottom bar, as in iOS apps. Every edge keeps clear of
      the safe area (notch or Dynamic Island, rounded corners, home indicator); env() is zero on desktops. ---- */
   @media (max-width: 640px), (max-height: 500px) and (hover: none) {
@@ -358,6 +371,10 @@
     button.small { height: 32px; }
     .actions button { height: 46px; }
     .cat-tools select { max-width: none; flex: 1 1 100%; }
+    .pad-area { height: 240px; }
+    .pad-keys .seg { flex: 1 1 100%; display: grid; grid-template-columns: repeat(4, 1fr); }
+    .pad-keys .pad-clicks { grid-template-columns: repeat(2, 1fr); }
+    .pad-keys .seg button { padding: 0 4px; justify-content: center; text-align: center; }
     /* The on-screen keyboard covers the bottom of the screen; the tab bar would ride on top of it. */
     body.typing nav { display: none; }
   }
@@ -471,6 +488,36 @@
     </section>
   </div>
 
+  <section class="panel" id="pad">
+    <div class="shelf-head"><h2>Keyboard and trackpad</h2><span class="sub" id="padState"></span><span class="spacer"></span>
+      <button class="small" id="padOn">Turn on</button>
+    </div>
+    <div class="pad-area" id="padArea" tabindex="0" role="application" aria-label="Trackpad for the Frame">
+      <div class="pad-hint" id="padHint"></div>
+    </div>
+    <div class="pad-keys" aria-label="Keys and clicks">
+      <input id="padType" type="text" placeholder="Type here to type on the Frame" autocomplete="off" autocapitalize="off"
+             autocorrect="off" spellcheck="false" enterkeyhint="send">
+      <span class="seg pad-clicks">
+        <button data-pad-click="singleclick" title="Left click">Click</button>
+        <button data-pad-click="rightclick" title="Right click">Right-click</button>
+      </span>
+      <span class="seg pad-special">
+        <button data-pad-key="14" title="Escape">Esc</button>
+        <button data-pad-key="2" title="Tab">Tab</button>
+        <button data-pad-key="4" title="Left arrow" aria-label="Left arrow">←</button>
+        <button data-pad-key="5" title="Up arrow" aria-label="Up arrow">↑</button>
+        <button data-pad-key="7" title="Down arrow" aria-label="Down arrow">↓</button>
+        <button data-pad-key="6" title="Right arrow" aria-label="Right arrow">→</button>
+        <button data-pad-key="1" title="Backspace" aria-label="Backspace">⌫</button>
+        <button data-pad-key="12" title="Enter">Enter</button>
+      </span>
+    </div>
+    <div class="hint">Types and points in apps on the Frame, such as Chromium or the desktop's Linux apps; Steam's own VR menus
+      don't take it. It works through KDE Connect on the Frame: the first time, Frame Control fetches Valve's build of it
+      (an 8 MB download, 82 MB unpacked) into your home folder there and pairs with it. Nothing to install on this device.</div>
+  </section>
+
   <section id="shots">
     <div class="shelf-head"><h2>Screenshots</h2><span class="count" id="shotCount"></span><span class="spacer"></span>
       <button class="small" id="shotsRefresh">Refresh</button>
@@ -2128,6 +2175,219 @@ window.addEventListener("hashchange", showPage);
 const typesText = el => el?.matches?.('textarea, input:not([type=range], [type=checkbox], [type=radio], [type=file], [type=button])');
 document.addEventListener("focusin", e => document.body.classList.toggle("typing", !!typesText(e.target)));
 document.addEventListener("focusout", () => document.body.classList.remove("typing"));
+
+// ---- keyboard and trackpad: events go to KDE Connect on the Frame (see ui/frame_input_agent.py) ----
+const pad = { state: "off", queue: [], sending: false, poll: null, captured: false };
+const PAD_STATES = {
+  off: "", starting: "Connecting…", installing: "Setting up KDE Connect on the Frame (first time only)…",
+  pairing: "Pairing with KDE Connect…", ready: "On",
+};
+// KDE Connect's specialKey numbers (plugins/mousepad in its source).
+const PAD_SPECIAL = { Backspace: 1, Tab: 2, ArrowLeft: 4, ArrowUp: 5, ArrowRight: 6, ArrowDown: 7, PageUp: 8,
+  PageDown: 9, Home: 10, End: 11, Enter: 12, Delete: 13, Escape: 14,
+  ...Object.fromEntries(Array.from({ length: 12 }, (_, i) => [`F${i + 1}`, 21 + i])) };
+const IS_MAC = /^Mac/.test(navigator.platform);
+const TOUCH = matchMedia("(hover: none)").matches;
+
+function padShow(status) {
+  pad.state = status.state || "off";
+  const ready = pad.state === "ready";
+  const busy = !ready && pad.state !== "off" && pad.state !== "error";
+  $("padState").textContent = pad.state === "error" ? "" : PAD_STATES[pad.state] || "";
+  $("padOn").hidden = ready || busy;
+  $("padOn").textContent = pad.state === "error" ? "Try again" : "Turn on";
+  $("padArea").classList.toggle("off", !ready);
+  $("padHint").textContent = pad.state === "error" ? status.message || "Couldn't reach KDE Connect on the Frame."
+    : !ready ? (busy ? PAD_STATES[pad.state] : "Turn on to use this as a trackpad for the Frame.")
+    : TOUCH ? "Drag to move the pointer · tap to click · two fingers to scroll · two-finger tap to right-click"
+    : pad.captured ? "Your mouse and keyboard now control the Frame. Press Esc to stop."
+    : "Click here to use your mouse and keyboard on the Frame (Esc to stop), or drag like a trackpad.";
+  clearTimeout(pad.poll);
+  if (ready && pad.queue.length) padFlush();
+  if (busy) pad.poll = setTimeout(async () => padShow(await api("/api/input").catch(e => ({ state: "error", message: e.message }))), 1000);
+}
+async function padStart() {
+  localStorage.padUsed = "1";  // from now on it turns on with the page
+  padShow({ state: "starting" });
+  padShow(await api("/api/input?start=1").catch(e => ({ state: "error", message: e.message })));
+}
+$("padOn").onclick = padStart;
+
+// Queue events and send them one request at a time, merging pointer moves made meanwhile.
+const isMove = e => Object.keys(e).every(k => k === "dx" || k === "dy");
+function padSend(event) {
+  // While it reconnects, keys and clicks wait in the queue; pointer moves would be stale.
+  // A button release is always kept, so a drag never leaves the button held on the Frame.
+  const room = pad.queue.length < 200 || event.singlerelease;
+  const reconnecting = ["starting", "pairing", "installing"].includes(pad.state) && room;
+  // Even after an error: a button pressed on the Frame must be let go once it's back.
+  const release = event.singlerelease && pad.state !== "off";
+  if (pad.state !== "ready" && !(reconnecting && !isMove(event)) && !release) return;
+  const last = pad.queue[pad.queue.length - 1];
+  if (last && isMove(last) && isMove(event)) { last.dx = (last.dx || 0) + (event.dx || 0); last.dy = (last.dy || 0) + (event.dy || 0); }
+  else pad.queue.push(event);
+  padFlush();
+}
+// Events that didn't reach the Frame (the link had dropped, and is reconnecting): keep
+// the keys and clicks for when it's back, but not pointer moves, which would be stale.
+// Button releases are kept whatever the queue's length.
+function padKeep(events) {
+  pad.queue = [...events, ...pad.queue].filter(e => !isMove(e)).filter((e, i) => i < 200 || e.singlerelease);
+}
+async function padFlush() {
+  if (pad.sending || !pad.queue.length || pad.state !== "ready") return;
+  pad.sending = true;
+  const events = pad.queue.splice(0, 200);
+  try {
+    const status = await api("/api/input", { events });
+    if (!status.sent) padKeep(events);
+    if (status.state !== "ready") padShow(status);
+  } catch (e) {
+    padKeep(events);
+    padShow({ state: "error", message: e.message });
+  } finally {
+    pad.sending = false;
+    if (pad.queue.length) padFlush();
+  }
+}
+function padKey(e) {
+  const mods = { ...(e.ctrlKey || (IS_MAC && e.metaKey) ? { ctrl: true } : {}), ...(e.altKey ? { alt: true } : {}),
+                 ...(!IS_MAC && e.metaKey ? { super: true } : {}) };
+  if (PAD_SPECIAL[e.key]) return { specialKey: PAD_SPECIAL[e.key], ...mods, ...(e.shiftKey ? { shift: true } : {}) };
+  if (e.key.length === 1) return { key: e.key, ...mods };  // already shifted, so no shift flag
+  return null;
+}
+
+// Touch: one finger moves, a quick tap clicks; two fingers scroll, or right-click when tapped.
+const PAD_SPEED = 1.6, PAD_SCROLL_STEP = 18;
+const touches = new Map();
+let gesture = null;
+$("padArea").addEventListener("pointerdown", e => {
+  if (pad.state === "off" || pad.state === "error") { padStart(); return; }
+  if (pad.state !== "ready") return;
+  if (e.pointerType === "mouse") {
+    if (pad.captured) return;  // mousedown and mouseup below send the clicks
+    // Lock the pointer to the pad and pass the whole mouse and keyboard through; where
+    // that isn't allowed, the mouse drags like a finger on a trackpad instead.
+    if (e.button === 0 && !pad.noLock && $("padArea").requestPointerLock) { $("padArea").requestPointerLock(); return; }
+    // Only the left button drags and taps like a finger; the others are clicks of their own.
+    if (e.button === 1 || e.button === 2) { padSend(e.button === 2 ? { rightclick: true } : { middleclick: true }); return; }
+    if (e.button !== 0) return;
+  }
+  try { $("padArea").setPointerCapture(e.pointerId); } catch {}  // keeps the drag if the finger leaves the pad
+  touches.set(e.pointerId, { x: e.clientX, y: e.clientY });
+  if (touches.size === 1) gesture = { start: performance.now(), moved: 0, fingers: 1, scroll: 0 };
+  else if (gesture) gesture.fingers = Math.max(gesture.fingers, touches.size);
+  e.preventDefault();
+});
+$("padArea").addEventListener("pointermove", e => {
+  if (pad.captured) { padSend({ dx: e.movementX, dy: e.movementY }); return; }
+  const t = touches.get(e.pointerId);
+  if (!t || !gesture) return;
+  const dx = e.clientX - t.x, dy = e.clientY - t.y;
+  t.x = e.clientX; t.y = e.clientY;
+  gesture.moved += Math.abs(dx) + Math.abs(dy);
+  if (touches.size >= 2) {
+    // Scroll with the lead finger only, a notch per step, like a laptop trackpad (content follows the fingers).
+    if (e.pointerId !== touches.keys().next().value) return;
+    gesture.scroll += dy;
+    while (Math.abs(gesture.scroll) >= PAD_SCROLL_STEP) {
+      const up = gesture.scroll > 0;
+      padSend({ scroll: true, dy: up ? 1 : -1 });
+      gesture.scroll -= up ? PAD_SCROLL_STEP : -PAD_SCROLL_STEP;
+    }
+  } else if (gesture.fingers === 1) {
+    padSend({ dx: dx * PAD_SPEED, dy: dy * PAD_SPEED });
+  }
+});
+function padLift(e) {
+  if (!touches.delete(e.pointerId) || touches.size || !gesture) return;
+  const quick = performance.now() - gesture.start < 250 && gesture.moved < 10;
+  if (quick) padSend(gesture.fingers >= 2 ? { rightclick: true } : { singleclick: true });
+  gesture = null;
+}
+$("padArea").addEventListener("pointerup", padLift);
+$("padArea").addEventListener("pointercancel", e => { touches.delete(e.pointerId); if (!touches.size) gesture = null; });
+
+// A computer's mouse and keyboard: captured while the pointer is locked to the pad.
+document.addEventListener("pointerlockerror", () => { pad.noLock = true; });
+document.addEventListener("pointerlockchange", () => {
+  pad.captured = document.pointerLockElement === $("padArea");
+  window.frameApp?.captureKeys?.(pad.captured);  // the desktop app's menu shortcuts go to the Frame too
+  // Esc or switching windows mid-drag: let go of the button on the Frame as well.
+  if (!pad.captured && pad.held) { padSend({ singlerelease: true }); pad.held = false; }
+  $("padArea").classList.toggle("captured", pad.captured);
+  padShow({ state: pad.state });
+});
+$("padArea").addEventListener("mousedown", e => {
+  if (!pad.captured) return;
+  if (e.button === 0) { padSend({ singlehold: true }); pad.held = true; }
+  else if (e.button === 1) padSend({ middleclick: true });
+  else if (e.button === 2) padSend({ rightclick: true });
+});
+$("padArea").addEventListener("mouseup", e => { if (pad.held && e.button === 0) { padSend({ singlerelease: true }); pad.held = false; } });
+$("padArea").addEventListener("contextmenu", e => e.preventDefault());
+$("padArea").addEventListener("wheel", e => {
+  if (pad.state !== "ready") return;
+  e.preventDefault();
+  if (e.deltaY) padSend({ scroll: true, dy: e.deltaY > 0 ? -1 : 1 });
+}, { passive: false });
+// Capture phase, so the page's own shortcuts (tabs 1-4, R, zoom) don't also fire.
+window.addEventListener("keydown", e => {
+  if (!pad.captured) return;
+  e.preventDefault(); e.stopImmediatePropagation();
+  const event = padKey(e);
+  if (event) padSend(event);
+}, true);
+
+// The text field: each keystroke goes to the Frame. While focused it holds one space,
+// so Backspace always has something to delete; empty otherwise, to show its prompt.
+const padType = $("padType");
+padType.addEventListener("beforeinput", e => {
+  if (e.inputType === "insertCompositionText") return;  // IME composing; sent on compositionend
+  e.preventDefault();
+  if (pad.state === "off" || pad.state === "error") padStart();  // what's typed meanwhile waits in the queue
+  const text = e.data ?? e.dataTransfer?.getData("text/plain") ?? "";
+  if (/^insert(Text|ReplacementText|FromPaste|FromDrop)$/.test(e.inputType) && text) padText(text);
+  else if (e.inputType === "deleteContentBackward") padSend({ specialKey: 1 });
+  else if (e.inputType === "deleteWordBackward") padSend({ specialKey: 1, ctrl: true });  // Ctrl+Backspace: a word, on Linux
+  else if (e.inputType === "deleteContentForward") padSend({ specialKey: 13 });
+  else if (e.inputType === "deleteWordForward") padSend({ specialKey: 13, ctrl: true });
+  else if (e.inputType === "insertLineBreak" || e.inputType === "insertParagraph") padSend({ specialKey: 12 });
+});
+padType.addEventListener("compositionend", e => { if (e.data) padText(e.data); padType.value = " "; padCaret(); });
+// Text in pieces the server takes (500 characters), with Enter for each line break.
+function padText(text) {
+  text.replace(/\r\n?/g, "\n").split("\n").forEach((line, i) => {
+    if (i) padSend({ specialKey: 12 });
+    const chars = Array.from(line);  // whole characters, so an emoji isn't cut in two
+    for (let at = 0; at < chars.length; at += 500) padSend({ key: chars.slice(at, at + 500).join("") });
+  });
+}
+padType.addEventListener("keydown", e => {
+  const special = PAD_SPECIAL[e.key];
+  // Text and deletions arrive as beforeinput; this takes the keys that don't edit the field.
+  if (!special || e.key === "Backspace" || e.key === "Delete" || e.isComposing) return;
+  e.preventDefault();
+  padSend(padKey(e));
+});
+// The caret stays after the space, so Backspace always has something to delete.
+const padCaret = () => requestAnimationFrame(() => padType.setSelectionRange(1, 1));
+padType.addEventListener("focus", () => { padType.value = " "; padCaret(); if (pad.state === "off") padStart(); });
+padType.addEventListener("blur", () => { padType.value = ""; });
+padType.addEventListener("input", e => {
+  if (e.isComposing) return;  // an input method is still composing; compositionend sends it
+  if (padType.value !== " ") padType.value = " ";
+  padCaret();
+});
+padType.addEventListener("click", padCaret);
+document.querySelector("#pad .pad-keys").addEventListener("click", e => {
+  const b = e.target.closest("button");
+  if (!b) return;
+  if (b.dataset.padClick) padSend({ [b.dataset.padClick]: true });
+  else if (b.dataset.padKey) padSend({ specialKey: +b.dataset.padKey });
+});
+api("/api/input").then(s => s.state === "off" && localStorage.padUsed ? padStart() : padShow(s), () => padShow({ state: "off" }));
 showPage();
 document.addEventListener("keydown", e => {
   if (e.metaKey || e.ctrlKey || e.altKey || /INPUT|TEXTAREA|SELECT/.test(document.activeElement.tagName) || document.querySelector("dialog[open]")) return;