Control: close the targeting gaps from the second review

- The focused panel's display comes from GAMESCOPE_FOCUS_DISPLAY (gamescope
  packs ":1" into the first value), so a window id repeated across :0 and :1
  can't be mistaken; the pid is only the fallback.
- Presses, keys, text and scrolls read focus afresh on the Frame; only moves
  use a reading up to a second old.
- A gesture remembers the panel it started on and does nothing more if that
  stops being the one in use; a press with no panel to aim at isn't sent.
- Opening a screenshot clears the panel Control would act on; switching to
  another app releases held keys and buttons.
- Trimming keeps a click with its position; the error backoff holds for new
  input too.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

tests/page/ctrl_gestures.mjs

@@ -9,7 +9,7 @@ const grab = name => {
   if (!m) throw new Error("not found: " + name);
   return m[0];
 };
-const NAMES = ["isMoveEvent", "isRelease", "ctrlKeepable", "TAP_MOVE", "ctrlAim", "ctrlKeyEvent", "ctrlTouchCancel", "ctrlSend", "ctrlFlush", "ctrlMoveTo", "ctrlMoveBy", "ctrlSchedule",
+const NAMES = ["panelKey", "ctrlAimedAt", "ctrlSameTarget", "isMoveEvent", "isRelease", "ctrlKeepable", "TAP_MOVE", "ctrlAim", "ctrlKeyEvent", "ctrlTouchCancel", "ctrlSend", "ctrlFlush", "ctrlMoveTo", "ctrlMoveBy", "ctrlSchedule",
   "ctrlFlushMoves", "ctrlButton", "ctrlClick", "ctrlRelease", "ctrlFraction", "ctrlTouchDown", "centroid",
   "ctrlTouchMove", "ctrlTouchUp", "ctrlTap", "ctrlText"];
 const code = NAMES.map(grab).join("");
@@ -17,13 +17,14 @@ const fail = msg => { console.log("FAIL " + msg); process.exit(1); };
 const tick = (ms = 0) => new Promise(r => setTimeout(r, ms));
 
 function page({ mode = "abs", rect = { left: 0, top: 0, width: 640, height: 360 }, api } = {}) {
+  const target = { panel: { window: 42, display: ":1" } };
   const ctrl = { on: true, queue: [], sending: false, state: "ready", message: "", move: null, rel: [0, 0], raf: 0,
                  held: new Set(), keys: new Set(), pointers: new Map(), g: null, retry: null };
   const sent = [];
   const canvas = { width: 1280, height: 720, getBoundingClientRect: () => rect };
   const env = {
     ctrl, $: () => canvas, ctrlMode: () => mode, ctrlShow: () => {}, toast: () => {},
-    ctrlTarget: () => (mode === "abs" ? { ok: true, panel: { window: 42, display: ":1" } } : { ok: true }),
+    ctrlTarget: () => (mode !== "abs" ? { ok: true } : target.panel ? { ok: true, panel: target.panel } : { why: "gone" }),
     api: api || (async (path, body) => { sent.push(...body.events); return { state: "ready", sent: true }; }),
     requestAnimationFrame: cb => { setTimeout(cb, 0); return 1; },
     navigator: {},
@@ -31,7 +32,7 @@ function page({ mode = "abs", rect = { left: 0, top: 0, width: 640, height: 360
   const fns = new Function(...Object.keys(env), `let ctrlWarned = false;\n${code}
     return { ctrlTouchDown, ctrlTouchMove, ctrlTouchUp, ctrlTouchCancel, ctrlSend, ctrlText, ctrlButton, ctrlKeyEvent, ctrlRelease, ctrlFlushMoves };`)(...Object.values(env));
   const at = (id, x, y) => ({ pointerId: id, clientX: x, clientY: y });
-  return { ctrl, sent, ...fns, at };
+  return { ctrl, sent, target, ...fns, at };
 }
 
 // A tap lands where it was tapped: pointer there first, then the click.
@@ -115,7 +116,7 @@ function page({ mode = "abs", rect = { left: 0, top: 0, width: 640, height: 360
   const q = p.ctrl.queue;
   if (!(q.length === 3 && "fx" in q[0] && q[1].button === "left" && q[2].key === 30)) fail("kept while starting " + JSON.stringify(q));
   clearTimeout(p.ctrl.retry);
-  p.ctrl.queue = [];
+  p.ctrl.queue = []; p.ctrl.retryAt = 0;
   p.ctrlSend([{ button: "left", down: false }, { key: 31, down: true }, { dx: 3, dy: 1 }]);
   await tick(5);
   if (!(p.ctrl.queue.length === 1 && p.ctrl.queue[0].button === "left" && p.ctrl.queue[0].down === false))
@@ -198,4 +199,36 @@ function page({ mode = "abs", rect = { left: 0, top: 0, width: 640, height: 360
   if (check({ ...base, pick: ":0/5", shown: ":0/5" }).ok) fail("target: a watched panel that isn't in use");
   if (check({ ...base, focus: null }).ok) fail("target: nothing in use");
 }
+// Focus moves to another panel mid-gesture: the tap or hold does nothing.
+{
+  const p = page();
+  p.ctrlTouchDown(p.at(1, 100, 100));
+  p.target.panel = { window: 43, display: ":1" };
+  p.ctrlTouchUp(p.at(1, 100, 100)); await tick(10);
+  p.target.panel = { window: 42, display: ":1" };
+  p.ctrlTouchDown(p.at(1, 100, 100));
+  p.target.panel = null;
+  await tick(620); p.ctrlTouchUp(p.at(1, 100, 100)); await tick(10);
+  if (p.sent.some(e => "button" in e)) fail("gesture outlived its panel " + JSON.stringify(p.sent));
+}
+// Trimming keeps a click together with its position.
+{
+  const p = page({ api: () => new Promise(() => {}) });
+  p.ctrlSend([{ dx: 1, dy: 0 }]);  // in flight forever
+  for (let i = 0; i < 100; i++) p.ctrlSend([{ scroll: [0, 1] }]);
+  p.ctrlSend([{ fx: 0.5, fy: 0.5, window: 42, display: ":1" }, { button: "left", down: true }, { button: "left", down: false }]);
+  for (let i = 0; i < 198; i++) p.ctrlSend([{ scroll: [0, 1] }]);
+  const q = p.ctrl.queue, i = q.findIndex(e => e.button === "left" && e.down);
+  if (i < 1 || !("fx" in q[i - 1])) fail("trim split a click from its position");
+}
+// Backing off holds for new input too.
+{
+  let calls = 0;
+  const p = page({ api: async () => { calls++; return { state: "error", sent: false }; } });
+  p.ctrlSend([{ button: "left", down: false }]);
+  await tick(5);
+  for (let i = 0; i < 10; i++) { p.ctrlSend([{ key: 30, down: false }]); await tick(2); }
+  if (calls !== 1) fail("new input bypassed the backoff: " + calls + " requests");
+  clearTimeout(p.ctrl.retry);
+}
 console.log("control gestures ok");

tests/test_touch.py

@@ -50,6 +50,14 @@ class Mapping(unittest.TestCase):
         self.assertEqual(t.locate(5, 111)["display"], ":0")
         self.assertEqual(t.locate(5, None)["display"], ":0")
 
+    def test_focus_display_is_decoded(self):
+        t = self.t
+        saved = t.xprop_root
+        self.addCleanup(lambda: setattr(t, "xprop_root", saved))
+        for values, want in (([12602, 0, 58], ":1"), ([12346], ":0"), ([], None), ([0x41], None)):
+            t.xprop_root = lambda d, n, v=values: v
+            self.assertEqual(t.focus_display(), want)
+
     def test_ascii_table_covers_printable_characters(self):
         for code in range(0x20, 0x7F):
             self.assertIn(chr(code), self.t.ASCII, chr(code))
@@ -104,6 +112,19 @@ class Apply(unittest.TestCase):
         self.assertEqual(gs.calls, [("button", "left", False)])
         self.assertEqual(self.said[0], ("ready", {"focus": 7, "display": ":1", "stale": True}))
 
+    def test_presses_read_focus_afresh(self):
+        # A move may use a recent reading; a press checks again, so a panel that just
+        # took focus doesn't get a click meant for another.
+        gs, calls = FakeGamescope(), []
+        self.t.focus = lambda: calls.append(1) or dict(self.focused)
+        panel = self.t.apply(gs, {"fx": 0.5, "fy": 0.5, "window": 7, "display": ":1"}, None)
+        panel = self.t.apply(gs, {"fx": 0.6, "fy": 0.5, "window": 7, "display": ":1"}, panel)
+        self.assertEqual(len(calls), 1)
+        self.focused["window"] = 8
+        self.t.apply(gs, {"button": "left", "down": True, "window": 7, "display": ":1"}, panel)
+        self.assertEqual(len(calls), 2)
+        self.assertEqual([c[0] for c in gs.calls], ["move_to", "move_to"])
+
     def test_same_window_id_on_the_other_display_is_another_panel(self):
         gs = FakeGamescope()
         self.t.apply(gs, {"fx": 0.5, "fy": 0.5, "window": 7, "display": ":0"}, None)

ui/frame_touch.py

@@ -126,14 +126,35 @@ def focusable():
     return [tuple(t[i:i + 3]) for i in range(0, len(t) - 2, 3)]
 
 
+def focus_display():
+    """The display of the focused window, from GAMESCOPE_FOCUS_DISPLAY on :0's root.
+
+    gamescope writes the name (":1") as 32-bit items, so its first four bytes land,
+    little-endian, in the first value: 12602 is 0x313A, ":1" (steamcompmgr.cpp;
+    seen 2026-09-29).
+    """
+    values = xprop_root(":0", "GAMESCOPE_FOCUS_DISPLAY")
+    if not values:
+        return None
+    name = (values[0] & 0xFFFFFFFF).to_bytes(4, "little").split(b"\0", 1)[0].decode("ascii", "replace")
+    return name if name[:1] == ":" and name[1:].isdigit() else None
+
+
 def focus():
     """The panel that has focus in the headset: window, display, name and sizes (gamescope
-    publishes the window on :0's root as GAMESCOPE_FOCUSED_WINDOW)."""
+    publishes the window and its display on :0's root)."""
     window = (xprop_root(":0", "GAMESCOPE_FOCUSED_WINDOW") or [0])[0]
     if not window:
         return {"window": None}
     app, pid = next(((a, p) for w, a, p in focusable() if w == window), (None, None))
-    panel = locate(window, pid)
+    display = focus_display()
+    info = window_info(display, window) if display else None
+    if info:
+        root = window_info(display, "root") or {}
+        panel = {"window": window, "display": display, **info,
+                 "root": [root.get("width", info["width"]), root.get("height", info["height"])]}
+    else:
+        panel = locate(window, pid)  # no display published: tell them apart by pid
     return {**panel, "app": app} if panel else {"window": None}
 
 
@@ -317,7 +338,10 @@ def apply(gs, event, panel):
     another panel since, they go nowhere, so a tap can't land on the wrong one;
     releases always go, so nothing stays held.
     """
-    if "window" in event and (not panel or time.time() - panel.get("_at", 0) > 1 or aimed_elsewhere(event, panel)):
+    # Moves may use a focus reading up to a second old; anything that acts (a press, key,
+    # text or scroll) reads it afresh, so it can't land on a panel that took focus since.
+    acts = any(k in event for k in ("button", "key", "text", "scroll")) and event.get("down") is not False
+    if "window" in event and (acts or not panel or time.time() - panel.get("_at", 0) > 1 or aimed_elsewhere(event, panel)):
         panel = {**focus(), "_at": time.time()}
     stale = aimed_elsewhere(event, panel) if "window" in event else False
     if stale and not (event.get("down") is False and ("button" in event or "key" in event)):

ui/index.html

@@ -1515,6 +1515,7 @@ $("viewer").tabIndex = 0;
 $("viewer").addEventListener("focus", () => { if (ctrl.on && window.frameApp?.captureKeys) window.frameApp.captureKeys(true); });
 $("viewer").addEventListener("blur", () => { ctrlRelease(); if (window.frameApp?.captureKeys) window.frameApp.captureKeys(false); });
 document.addEventListener("visibilitychange", () => { if (document.hidden) ctrlRelease(); });
+window.addEventListener("blur", () => ctrlRelease());  // another app took the keyboard
 
 const isMoveEvent = e => ("fx" in e || "dx" in e) && !("button" in e) && !("key" in e);
 const isRelease = e => ("button" in e || "key" in e) && e.down === false;
@@ -1533,17 +1534,27 @@ function ctrlKeepable(events) {
   return out;
 }
 function ctrlSend(events) {
-  ctrl.queue.push(...events.map(e => isMoveEvent(e) ? e : ctrlAim(e)));
+  ctrl.queue.push(...events.map(e => isMoveEvent(e) ? e : ctrlAim(e)).filter(Boolean));
   // Keep order; if the Frame is slow to answer, drop pointer moves first, then the oldest
   // presses and scrolls, never a release.
   if (ctrl.queue.length > 300) {
-    const q = ctrlKeepable(ctrl.queue);
-    ctrl.queue = q.filter((e, i) => isRelease(e) || i >= q.length - 200);
+    const q = ctrlKeepable(ctrl.queue), keep = new Set();
+    q.forEach((e, i) => { if (isRelease(e) || i >= q.length - 200) keep.add(i); });
+    // A kept press keeps the position before it, so it lands where it was made.
+    for (const i of [...keep]) if (i > 0 && "button" in q[i] && q[i].down !== false && "fx" in q[i - 1]) keep.add(i - 1);
+    ctrl.queue = q.filter((e, i) => keep.has(i));
   }
   ctrlFlush();
 }
 async function ctrlFlush() {
   if (ctrl.sending || !ctrl.queue.length) return;
+  // Backing off after a failure: new input waits for the same retry.
+  const wait = (ctrl.retryAt || 0) - Date.now();
+  if (wait > 0) {
+    clearTimeout(ctrl.retry);
+    ctrl.retry = setTimeout(ctrlFlush, wait);
+    return;
+  }
   ctrl.sending = true;
   const batch = ctrl.queue.splice(0, 200);
   let again = 0;
@@ -1566,6 +1577,7 @@ async function ctrlFlush() {
     ctrl.sending = false;
     ctrlShow();
     clearTimeout(ctrl.retry);
+    ctrl.retryAt = again ? Date.now() + again : 0;
     if (again) { if (ctrl.queue.length) ctrl.retry = setTimeout(ctrlFlush, again); }
     else if (ctrl.queue.length) ctrlFlush();
   }
@@ -1588,9 +1600,11 @@ function ctrlFlushMoves() {
 }
 // On the Desktop view, presses name the panel they're for, so the Frame drops them if
 // focus has moved on (releases always go).
+// A press with no panel to aim at (the Desktop view lost its target) isn't sent at all.
 function ctrlAim(e) {
-  const p = ctrlMode() === "abs" && e.down !== false ? ctrlTarget().panel : null;
-  return p ? { ...e, window: p.window, display: p.display } : e;
+  if (ctrlMode() !== "abs" || e.down === false) return e;
+  const p = ctrlTarget().panel;
+  return p ? { ...e, window: p.window, display: p.display } : null;
 }
 function ctrlButton(name, down) {
   ctrlFlushMoves();  // the click lands where the pointer was sent
@@ -1633,9 +1647,9 @@ const TAP_MOVE = 8, HOLD_MS = 550, TOUCH_SPEED = 1.6, SCROLL_SPEED = 2;
 function ctrlTouchDown(e) {
   ctrl.pointers.set(e.pointerId, { x: e.clientX, y: e.clientY });
   if (ctrl.pointers.size === 1) {
-    const g = ctrl.g = { kind: "maybe", x: e.clientX, y: e.clientY, e };
+    const g = ctrl.g = { kind: "maybe", x: e.clientX, y: e.clientY, e, target: ctrlAimedAt() };
     g.timer = setTimeout(() => {
-      if (ctrl.g !== g || g.kind !== "maybe") return;
+      if (ctrl.g !== g || g.kind !== "maybe" || !ctrlSameTarget(g)) return;
       g.kind = "held";
       if (ctrlMode() === "abs") {
         const f = ctrlFraction(g.e);
@@ -1651,6 +1665,10 @@ function ctrlTouchDown(e) {
     ctrl.g = { kind: "scroll", ...centroid() };
   }
 }
+// The panel a gesture started on; if that stops being the one in use, the gesture
+// does nothing more (except let go of what it holds).
+function ctrlAimedAt() { return ctrlMode() === "abs" ? panelKey(ctrlTarget().panel) : "headset"; }
+function ctrlSameTarget(g) { return g.target && g.target === ctrlAimedAt(); }
 function centroid() {
   const ps = [...ctrl.pointers.values()];
   return { x: ps.reduce((a, p) => a + p.x, 0) / ps.length, y: ps.reduce((a, p) => a + p.y, 0) / ps.length };
@@ -1669,6 +1687,7 @@ function ctrlTouchMove(e) {
   }
   if (g.kind === "maybe" && Math.hypot(e.clientX - g.x, e.clientY - g.y) > TAP_MOVE) {
     clearTimeout(g.timer);
+    if (!ctrlSameTarget(g)) { g.kind = "done"; return; }
     if (ctrlMode() === "abs") {
       const f = ctrlFraction(g.e);
       if (f) { ctrlMoveTo(...f); ctrlButton("left", true); }
@@ -1683,6 +1702,7 @@ function ctrlTouchUp(e) {
   ctrl.pointers.delete(e.pointerId);
   if (!g) return;
   if (g.kind === "scroll") g.kind = "done";  // the finger left behind doesn't jump the scroll or drag
+  if (g.kind === "maybe" && !ctrlSameTarget(g)) g.kind = "done";
   if (g.kind === "maybe" && ctrl.pointers.size === 0) {
     clearTimeout(g.timer);
     if (ctrlMode() === "abs") { const f = ctrlFraction(g.e); if (!f) { ctrl.g = null; return; } ctrlMoveTo(...f); }
@@ -2654,6 +2674,7 @@ async function openShot(s) {
     await new Promise((ok, bad) => { img.onload = ok; img.onerror = () => bad(new Error("not an image")); img.src = url; });
     if (gen !== viewGen) return;
     lastImg = img; lastSource = "shot"; lastShot = { blob, file: s.file };
+    desk.shown = null;  // a screenshot, not a panel: Control has nothing to act on here
     draw();
     $("srcBadge").textContent = `Screenshot · ${shotApp(s.appid)}`;
     $("stamp").hidden = false; $("stamp").textContent = new Date(s.time * 1000).toLocaleString();