Trackpad and key row go through gamescope's input; KDE Connect only for accents and emoji

The pad reaches whichever panel has focus (Steam's menus too), needs nothing installed, and no longer clamps to 1280x720. Characters that aren't on a US keyboard start KDE Connect the first time they're typed.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>

README.md

@@ -73,7 +73,7 @@ Drag files onto the window to send them. Drop a game's .zip, folder or .exe to a
 Browse the shots you take in the headset and save them to your Pictures folder.
 
 **⌨️ Keyboard and trackpad**<br>
-Type and point in the Frame's apps from your computer or phone, through KDE Connect, which Frame Control brings along and sets up on the Frame. Nothing else to install, anywhere.
+Type and point in the headset from your computer or phone, through Valve's own input path, with nothing to install. Accents and emoji use KDE Connect, which Frame Control brings along and sets up the first time you type one.
 
 </td>
 </tr>

docs/streaming.md

@@ -123,10 +123,13 @@ two fingers to scroll, two-finger tap to right-click) plus a text field that
 types on the Frame. On a computer, clicking the pad passes your mouse and
 keyboard through to the Frame until you press Esc (⌘ is sent as Ctrl on a Mac).
 
-It goes through **KDE Connect**, the first-party route (KDE makes the Frame's
-desktop): Frame Control's server runs [`ui/frame_input_agent.py`](../ui/frame_input_agent.py)
-on the Frame, which talks KDE Connect's own LAN protocol to the Frame's
-`kdeconnectd` as if it were a phone. KDE Connect does the typing and clicking.
+Since Control (below), the pad goes through **gamescope's own input socket**
+([`ui/frame_touch.py`](../ui/frame_touch.py)), so it reaches whichever panel has
+focus, Steam's own menus included, and needs nothing installed on the Frame.
+Only characters that aren't on a US keyboard (accents, emoji) go through
+**KDE Connect** as below; it starts the first time you type one. (Before
+Control, everything went through KDE Connect, which is why the notes below
+describe it in detail.)
 
 **Verified 2026-09-28** (SteamOS 0.4.1, build 20260925.6191901):
 

tests/test_input.py

@@ -630,3 +630,38 @@ class PageQueue(unittest.TestCase):
 
 if __name__ == "__main__":
     unittest.main()
+
+
+class PadEvents(unittest.TestCase):
+    """The trackpad and key row go through gamescope; accents and emoji go the KDE Connect way."""
+
+    @classmethod
+    def setUpClass(cls):
+        import server
+        cls.pad = staticmethod(server.pad_events)
+
+    def test_moves_clicks_and_scroll(self):
+        out, extra = self.pad([{"dx": 3, "dy": -2}, {"singleclick": True}, {"singlehold": True},
+                               {"singlerelease": True}, {"scroll": True, "dy": 1}, {"rightclick": True}])
+        self.assertEqual(out[0], {"dx": 3, "dy": -2})
+        self.assertEqual(out[1:3], [{"button": "left", "down": True}, {"button": "left", "down": False}])
+        self.assertEqual(out[3:5], [{"button": "left", "down": True}, {"button": "left", "down": False}])
+        self.assertEqual(out[5], {"scroll": [0, -15]})  # up = content follows the finger
+        self.assertEqual(out[6:], [{"button": "right", "down": True}, {"button": "right", "down": False}])
+        self.assertEqual(extra, "")
+
+    def test_keys_and_modifiers(self):
+        out, _ = self.pad([{"specialKey": 12}, {"specialKey": 1, "ctrl": True}, {"key": "c", "ctrl": True}])
+        self.assertEqual(out[:2], [{"key": 28, "down": True}, {"key": 28, "down": False}])
+        self.assertEqual([e["key"] for e in out[2:6]], [29, 14, 14, 29])  # Ctrl+Backspace
+        self.assertEqual([(e["key"], e["down"]) for e in out[6:]],
+                         [(29, True), (46, True), (46, False), (29, False)])  # Ctrl+C
+
+    def test_text_and_accents(self):
+        out, extra = self.pad([{"key": "Hi "}, {"key": "café 😀"}])
+        self.assertEqual(out, [{"text": "Hi caf "}])
+        self.assertEqual(extra, "é😀")
+
+    def test_shifted_key_with_modifier(self):
+        out, _ = self.pad([{"key": "A", "ctrl": True}])
+        self.assertEqual([e["key"] for e in out], [29, 42, 30, 30, 42, 29])

ui/index.html

@@ -600,10 +600,9 @@
         <button data-pad-key="12" title="Enter">Enter</button>
       </span>
     </div>
-    <div class="hint">Types and points in apps on the Frame, such as Chromium or the desktop's Linux apps; Steam's own VR menus
-      don't take it. It works through KDE Connect, which comes with Frame Control: the first time, it copies Valve's build
-      for the Frame (3.6 MB, 18 MB unpacked) into your home folder there and pairs with it. No internet needed, and nothing
-      else to install. <a href="#" data-about>Licences</a></div>
+    <div class="hint">Types and points in whichever panel has focus in the headset, through Valve's own input path: nothing to
+      install and no internet needed. Accents and emoji go through KDE Connect, which comes with Frame Control and starts
+      the first time you type one (it copies Valve's build for the Frame, 3.6 MB, into your home folder there). <a href="#" data-about>Licences</a></div>
   </section>
 
   <section id="shots">
@@ -3006,10 +3005,10 @@ const typesText = el => el?.matches?.('textarea, input:not([type=range], [type=c
 document.addEventListener("focusin", e => document.body.classList.toggle("typing", !!typesText(e.target)));
 document.addEventListener("focusout", () => document.body.classList.remove("typing"));
 
-// ---- keyboard and trackpad: events go to KDE Connect on the Frame (see ui/frame_input_agent.py) ----
+// ---- keyboard and trackpad: events go to the Frame through gamescope (ui/frame_touch.py); accents and emoji through KDE Connect ----
 const pad = { state: "off", queue: [], sending: false, poll: null, captured: false };
 const PAD_STATES = {
-  off: "", starting: "Connecting…", installing: "Setting up KDE Connect on the Frame (first time only)…",
+  off: "", starting: "Connecting…", installing: "Setting up (first time only)…",
   pairing: "Pairing with KDE Connect…", ready: "On",
 };
 // KDE Connect's specialKey numbers (plugins/mousepad in its source).
@@ -3027,7 +3026,7 @@ function padShow(status) {
   $("padOn").hidden = ready || busy;
   $("padOn").textContent = pad.state === "error" ? "Try again" : "Turn on";
   $("padArea").classList.toggle("off", !ready);
-  $("padHint").textContent = pad.state === "error" ? status.message || "Couldn't reach KDE Connect on the Frame."
+  $("padHint").textContent = pad.state === "error" ? status.message || "Couldn't reach the Frame."
     : !ready ? (busy ? PAD_STATES[pad.state] : "Turn on to use this as a trackpad for the Frame.")
     : TOUCH ? "Drag to move the pointer · tap to click · two fingers to scroll · two-finger tap to right-click"
     : pad.captured ? "Your mouse and keyboard now control the Frame. Press Esc to stop."

ui/server.py

@@ -51,6 +51,7 @@ import frame_report  # noqa: E402
 import frame_store  # noqa: E402
 import frame_telemetry  # noqa: E402
 import frame_titles  # noqa: E402
+import frame_touch  # noqa: E402
 import frame_webinstall  # noqa: E402
 
 frame_host.trust_bundled_cas()
@@ -847,12 +848,87 @@ def licenses():
     return notices
 
 
+# KDE Connect's specialKey numbers -> Linux key codes (KEY_BACKSPACE, KEY_TAB, ...).
+PAD_KEYS = {1: 14, 2: 15, 4: 105, 5: 103, 6: 106, 7: 108, 8: 104, 9: 109, 10: 102, 11: 107, 12: 28, 13: 111, 14: 1,
+            **{21 + i: code for i, code in enumerate([59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 87, 88])}}
+PAD_MODS = (("ctrl", 29), ("alt", 56), ("shift", 42), ("super", 125))
+PAD_CLICKS = {"singleclick": "left", "rightclick": "right", "middleclick": "middle"}
+PAD_NOTCH = 15  # scroll units for one wheel notch
+
+
+def pad_events(events):
+    """The trackpad's and key row's events as (gamescope events, text for KDE Connect).
+
+    Gamescope's own input reaches every panel and needs nothing installed, but types only
+    US-keyboard characters; accents and emoji go the KDE Connect way.
+    """
+    ascii_keys = frame_touch.ASCII
+    out, extra = [], []
+
+    def tap(code, mods):
+        down = [{"key": c, "down": True} for c in mods]
+        out.extend(down + [{"key": code, "down": True}, {"key": code, "down": False}]
+                   + [{"key": c, "down": False} for c in reversed(mods)])
+    for e in events:
+        mods = [code for name, code in PAD_MODS if e.get(name)]
+        if "dx" in e or "dy" in e:
+            if not e.get("scroll"):
+                out.append({k: e[k] for k in ("dx", "dy") if k in e})
+        if e.get("scroll"):
+            out.append({"scroll": [0, -(e.get("dy") or 0) * PAD_NOTCH]})
+        for flag, button in PAD_CLICKS.items():
+            if e.get(flag):
+                out += [{"button": button, "down": True}, {"button": button, "down": False}]
+        if e.get("doubleclick"):
+            out += [{"button": "left", "down": d} for d in (True, False, True, False)]
+        if e.get("singlehold"):
+            out.append({"button": "left", "down": True})
+        if e.get("singlerelease"):
+            out.append({"button": "left", "down": False})
+        if "specialKey" in e and e["specialKey"] in PAD_KEYS:
+            tap(PAD_KEYS[e["specialKey"]], mods)
+        if "key" in e:
+            for ch in e["key"]:
+                if ch not in ascii_keys:
+                    extra.append(ch)
+                elif mods:
+                    code, shifted = ascii_keys[ch]
+                    tap(code, mods + ([42] if shifted and 42 not in mods else []))
+                else:
+                    if out and "text" in out[-1] and len(out[-1]["text"]) < 100:
+                        out[-1]["text"] += ch
+                    else:
+                        out.append({"text": ch})
+    return out, "".join(extra)
+
+
 def remote_input(body):
     """{"events": [...]} sends keyboard and pointer events; {} (or none yet) just starts the agent."""
     events = body.get("events", [])
     if not isinstance(events, list) or len(events) > INPUT_BATCH_LIMIT:
         raise Failure(f"events must be a list of at most {INPUT_BATCH_LIMIT}", 400)
-    return _input.send([input_event(e) for e in events])
+    events = [input_event(e) for e in events]
+    touch, accents = pad_events(events)
+    status = _touch.send(touch)
+    if accents:
+        _accents.append(accents)
+    _send_accents(status.get("sent") or not touch)
+    return status
+
+
+_accents = []  # accents and emoji waiting for KDE Connect, which starts the first time they're typed
+
+
+def _send_accents(wanted):
+    """Type what gamescope can't through KDE Connect, starting it if needed; kept until it's up."""
+    if not _accents or not wanted:
+        return
+    status = _input.send([{"key": "".join(_accents)}])
+    if status.get("sent"):
+        _accents.clear()
+    elif status.get("state") == "error":
+        _accents.clear()  # not coming; don't hold them forever
+    del _accents[:-50]
 
 
 # ---- touch: the headset's panels, through gamescope's own input (frame_touch.py) ----
@@ -2008,7 +2084,7 @@ class Handler(BaseHTTPRequestHandler):
             elif path == "/api/touch":
                 self.send_json(_touch.send([]) if parse_qs(url.query).get("start") == ["1"] else dict(_touch.status))
             elif path == "/api/input":
-                self.send_json(_input.send([]) if parse_qs(url.query).get("start") == ["1"] else dict(_input.status))
+                self.send_json(_touch.send([]) if parse_qs(url.query).get("start") == ["1"] else dict(_touch.status))
             elif path == "/api/job":
                 self.send_json(job_status(url.query))
             elif path == "/api/android/displays":