Control: fixes from review

- Keys held on the Frame are released with buttons when Control stops or
  the view loses focus; keys for the Frame no longer trigger Frame Control's
  own shortcuts.
- Taps only act when the picture on screen is the panel in use; positions,
  presses, keys, text and scrolls name their panel (display and window: ids
  repeat across :0 and :1, told apart by pid), and the Frame drops them if
  focus has moved on. Releases always go.
- While connecting, a tap keeps its position; on an error only releases wait
  and retries back off; trimming a long queue never drops a release.
- Lifting one of two scrolling fingers ends the scroll; a cancelled touch
  isn't a tap; clicks and holds on the bars around the picture do nothing.
- A capture loop from before a Live restart can't stop the new video.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

tests/page/ctrl_gestures.mjs

@@ -9,7 +9,7 @@ const grab = name => {
   if (!m) throw new Error("not found: " + name);
   return m[0];
 };
-const NAMES = ["isMoveEvent", "TAP_MOVE", "ctrlSend", "ctrlFlush", "ctrlMoveTo", "ctrlMoveBy", "ctrlSchedule",
+const NAMES = ["isMoveEvent", "isRelease", "ctrlKeepable", "TAP_MOVE", "ctrlAim", "ctrlKeyEvent", "ctrlTouchCancel", "ctrlSend", "ctrlFlush", "ctrlMoveTo", "ctrlMoveBy", "ctrlSchedule",
   "ctrlFlushMoves", "ctrlButton", "ctrlClick", "ctrlRelease", "ctrlFraction", "ctrlTouchDown", "centroid",
   "ctrlTouchMove", "ctrlTouchUp", "ctrlTap", "ctrlText"];
 const code = NAMES.map(grab).join("");
@@ -18,18 +18,18 @@ const tick = (ms = 0) => new Promise(r => setTimeout(r, ms));
 
 function page({ mode = "abs", rect = { left: 0, top: 0, width: 640, height: 360 }, api } = {}) {
   const ctrl = { on: true, queue: [], sending: false, state: "ready", message: "", move: null, rel: [0, 0], raf: 0,
-                 held: new Set(), pointers: new Map(), g: null, retry: null };
+                 held: new Set(), keys: new Set(), pointers: new Map(), g: null, retry: null };
   const sent = [];
   const canvas = { width: 1280, height: 720, getBoundingClientRect: () => rect };
   const env = {
     ctrl, $: () => canvas, ctrlMode: () => mode, ctrlShow: () => {}, toast: () => {},
-    ctrlTarget: () => (mode === "abs" ? { ok: true, panel: { window: 42 } } : { ok: true }),
+    ctrlTarget: () => (mode === "abs" ? { ok: true, panel: { window: 42, display: ":1" } } : { ok: true }),
     api: api || (async (path, body) => { sent.push(...body.events); return { state: "ready", sent: true }; }),
     requestAnimationFrame: cb => { setTimeout(cb, 0); return 1; },
     navigator: {},
   };
   const fns = new Function(...Object.keys(env), `let ctrlWarned = false;\n${code}
-    return { ctrlTouchDown, ctrlTouchMove, ctrlTouchUp, ctrlSend, ctrlText, ctrlButton, ctrlRelease, ctrlFlushMoves };`)(...Object.values(env));
+    return { ctrlTouchDown, ctrlTouchMove, ctrlTouchUp, ctrlTouchCancel, ctrlSend, ctrlText, ctrlButton, ctrlKeyEvent, ctrlRelease, ctrlFlushMoves };`)(...Object.values(env));
   const at = (id, x, y) => ({ pointerId: id, clientX: x, clientY: y });
   return { ctrl, sent, ...fns, at };
 }
@@ -40,7 +40,8 @@ function page({ mode = "abs", rect = { left: 0, top: 0, width: 640, height: 360
   p.ctrlTouchDown(p.at(1, 320, 90)); p.ctrlTouchUp(p.at(1, 320, 90));
   await tick(10);
   const [move, down, up] = p.sent;
-  if (!(move.fx === 0.5 && Math.abs(move.fy - 0.25) < 1e-9 && move.window === 42)) fail("tap position " + JSON.stringify(move));
+  if (!(move.fx === 0.5 && Math.abs(move.fy - 0.25) < 1e-9 && move.window === 42 && move.display === ":1")) fail("tap position " + JSON.stringify(move));
+  if (!(down.window === 42 && down.display === ":1")) fail("a press names its panel " + JSON.stringify(down));
   if (!(down.button === "left" && down.down && up.button === "left" && up.down === false && p.sent.length === 3))
     fail("tap click " + JSON.stringify(p.sent));
 }
@@ -105,14 +106,14 @@ function page({ mode = "abs", rect = { left: 0, top: 0, width: 640, height: 360
   const text = p.sent.map(e => e.text).join("");
   if (!(text === "hllo " + "x".repeat(600) && p.sent.every(e => e.text.length <= 500))) fail("text " + text.length);
 }
-// Not connected yet, or the request fails: clicks and keys wait, stale moves don't; a release is never lost.
+// Not connected yet: clicks and keys wait with their position. The request fails: only releases wait.
 {
   let calls = 0;
   const p = page({ api: async () => { calls++; if (calls === 1) return { state: "starting", sent: false }; throw new Error("offline"); } });
   p.ctrlSend([{ fx: 0.1, fy: 0.1, window: 42 }, { button: "left", down: true }, { key: 30, down: true }]);
   await tick(5);
   const q = p.ctrl.queue;
-  if (!(q.length === 2 && q[0].button === "left" && q[1].key === 30)) fail("kept while starting " + JSON.stringify(q));
+  if (!(q.length === 3 && "fx" in q[0] && q[1].button === "left" && q[2].key === 30)) fail("kept while starting " + JSON.stringify(q));
   clearTimeout(p.ctrl.retry);
   p.ctrl.queue = [];
   p.ctrlSend([{ button: "left", down: false }, { key: 31, down: true }, { dx: 3, dy: 1 }]);
@@ -129,4 +130,72 @@ function page({ mode = "abs", rect = { left: 0, top: 0, width: 640, height: 360
   p.ctrlRelease(); await tick(10);
   if (!(p.sent.at(-1).button === "left" && p.sent.at(-1).down === false && !p.ctrl.held.size)) fail("release " + JSON.stringify(p.sent));
 }
+// Keys held on the Frame are let go too.
+{
+  const p = page();
+  p.ctrlKeyEvent(42, true);
+  p.ctrlRelease(); await tick(10);
+  if (!(p.sent.at(-1).key === 42 && p.sent.at(-1).down === false && !p.ctrl.keys.size)) fail("key release " + JSON.stringify(p.sent));
+}
+// A long queue sheds moves and old scrolls, never a release.
+{
+  const p = page({ api: () => new Promise(() => {}) });  // stuck request
+  p.ctrlSend([{ dx: 1, dy: 1 }]);
+  p.ctrlSend([{ button: "left", down: false }]);
+  for (let i = 0; i < 320; i++) p.ctrlSend([{ scroll: [0, 1] }]);
+  if (!p.ctrl.queue.some(e => e.button === "left" && e.down === false)) fail("trim dropped a release");
+  if (p.ctrl.queue.length > 300) fail("trim kept " + p.ctrl.queue.length);
+}
+// Broken on the Frame side: no hammering, and only releases wait.
+{
+  let calls = 0;
+  const p = page({ api: async () => { calls++; return { state: "error", sent: false }; } });
+  p.ctrlSend([{ button: "left", down: true }, { button: "left", down: false }]);
+  await tick(50);
+  if (calls !== 1) fail("error response reposted " + calls + " times");
+  if (!(p.ctrl.queue.length === 1 && p.ctrl.queue[0].down === false)) fail("error kept " + JSON.stringify(p.ctrl.queue));
+  clearTimeout(p.ctrl.retry);
+}
+// Connecting: a tap keeps its position, so it lands where it was made.
+{
+  const p = page({ api: async () => ({ state: "starting", sent: false }) });
+  p.ctrlTouchDown(p.at(1, 320, 90)); p.ctrlTouchUp(p.at(1, 320, 90));
+  await tick(10);
+  const q = p.ctrl.queue;
+  if (!("fx" in q[0] && q[1].button === "left")) fail("connecting tap " + JSON.stringify(q));
+  clearTimeout(p.ctrl.retry);
+}
+// Lifting one of two scrolling fingers doesn't jump the scroll or start a drag.
+{
+  const p = page();
+  p.ctrlTouchDown(p.at(1, 100, 200)); p.ctrlTouchDown(p.at(2, 300, 200));
+  p.ctrlTouchUp(p.at(1, 100, 200));
+  p.ctrlTouchMove(p.at(2, 300, 199));
+  p.ctrlTouchUp(p.at(2, 300, 199)); await tick(10);
+  if (p.sent.length) fail("one finger left after scrolling " + JSON.stringify(p.sent));
+}
+// A cancelled touch isn't a tap.
+{
+  const p = page();
+  p.ctrlTouchDown(p.at(1, 100, 100)); p.ctrlTouchCancel(p.at(1, 100, 100)); await tick(10);
+  if (p.sent.length) fail("cancel clicked " + JSON.stringify(p.sent));
+}
+// Press and hold on the bars around the picture does nothing.
+{
+  const p = page({ rect: { left: 0, top: 0, width: 640, height: 480 } });
+  p.ctrlTouchDown(p.at(1, 320, 10)); await tick(620); p.ctrlTouchUp(p.at(1, 320, 10)); await tick(10);
+  if (p.sent.length) fail("hold on the bars " + JSON.stringify(p.sent));
+}
+// Taps only reach the panel in use, and only once its picture is the one on screen.
+{
+  const fns = ["panelKey", "deskPanel", "ctrlTarget"].map(grab).join("");
+  const check = (desk, live = true) => new Function("desk", "live", "ctrlMode", fns + "; return ctrlTarget();")(desk, live, () => "abs");
+  const a = { display: ":1", window: 5 }, b = { display: ":0", window: 5 };
+  const base = { panels: [a, b], loaded: true, pick: "", focus: ":1/5", shown: ":1/5" };
+  if (!check(base).ok) fail("target: shown and in use");
+  if (check({ ...base, shown: ":0/5" }).ok) fail("target: the picture is another panel with the same id");
+  if (!/Capture or Live/.test(check({ ...base, shown: null }, false).why)) fail("target: stale picture message");
+  if (check({ ...base, pick: ":0/5", shown: ":0/5" }).ok) fail("target: a watched panel that isn't in use");
+  if (check({ ...base, focus: null }).ok) fail("target: nothing in use");
+}
 console.log("control gestures ok");

tests/test_touch.py

@@ -38,6 +38,18 @@ class Mapping(unittest.TestCase):
         self.assertAlmostEqual(x, 280, delta=0.5)
         self.assertAlmostEqual(y, 359.5, delta=0.5)
 
+    def test_same_id_on_both_displays_is_told_apart_by_pid(self):
+        t = self.t
+        saved = t.displays, t.window_info, t.window_pid
+        self.addCleanup(lambda: (setattr(t, "displays", saved[0]), setattr(t, "window_info", saved[1]),
+                                 setattr(t, "window_pid", saved[2])))
+        t.displays = lambda: [":0", ":1"]
+        t.window_info = lambda d, w: {"name": f"on {d}", "width": 1280 if w != "root" else 1920, "height": 720}
+        t.window_pid = lambda d, w: {":0": 111, ":1": 222}[d]
+        self.assertEqual(t.locate(5, 222)["display"], ":1")
+        self.assertEqual(t.locate(5, 111)["display"], ":0")
+        self.assertEqual(t.locate(5, None)["display"], ":0")
+
     def test_ascii_table_covers_printable_characters(self):
         for code in range(0x20, 0x7F):
             self.assertIn(chr(code), self.t.ASCII, chr(code))
@@ -77,16 +89,25 @@ class Apply(unittest.TestCase):
 
     def test_tap_moves_then_clicks_in_order(self):
         gs, panel = FakeGamescope(), None
-        for e in ({"fx": 0.5, "fy": 0.5, "window": 7}, {"button": "left", "down": True}, {"button": "left", "down": False}):
+        for e in ({"fx": 0.5, "fy": 0.5, "window": 7, "display": ":1"}, {"button": "left", "down": True, "window": 7, "display": ":1"},
+                  {"button": "left", "down": False}):
             panel = self.t.apply(gs, e, panel)
         self.assertEqual([c[0] for c in gs.calls], ["move_to", "button", "button"])
         self.assertEqual(gs.calls[1][1:], ("left", True))
 
     def test_a_tap_meant_for_another_panel_goes_nowhere(self):
+        # Focus moved on: the position and the press are dropped; the release still goes.
+        gs, panel = FakeGamescope(), None
+        for e in ({"fx": 0.5, "fy": 0.5, "window": 99, "display": ":1"}, {"button": "left", "down": True, "window": 99, "display": ":1"},
+                  {"key": 30, "down": True, "window": 7, "display": ":0"}, {"button": "left", "down": False, "window": 99, "display": ":1"}):
+            panel = self.t.apply(gs, e, panel)
+        self.assertEqual(gs.calls, [("button", "left", False)])
+        self.assertEqual(self.said[0], ("ready", {"focus": 7, "display": ":1", "stale": True}))
+
+    def test_same_window_id_on_the_other_display_is_another_panel(self):
         gs = FakeGamescope()
-        self.t.apply(gs, {"fx": 0.5, "fy": 0.5, "window": 99}, None)
+        self.t.apply(gs, {"fx": 0.5, "fy": 0.5, "window": 7, "display": ":0"}, None)
         self.assertEqual(gs.calls, [])
-        self.assertEqual(self.said, [("ready", {"focus": 7, "stale": True})])
 
     def test_relative_scroll_keys_text(self):
         gs = FakeGamescope()
@@ -115,16 +136,19 @@ class ServerChecks(unittest.TestCase):
 
     def test_touch_event_keeps_known_fields(self):
         ev = self.s.touch_event
-        self.assertEqual(ev({"fx": 0.5, "fy": 2, "window": 5}), {"fx": 0.5, "fy": 1.0, "window": 5})
+        self.assertEqual(ev({"fx": 0.5, "fy": 2, "window": 5, "display": ":1"}), {"fx": 0.5, "fy": 1.0, "window": 5, "display": ":1"})
+        self.assertEqual(ev({"button": "left", "window": 5, "display": ":0"}), {"button": "left", "down": True, "window": 5, "display": ":0"})
         self.assertEqual(ev({"button": "right"}), {"button": "right", "down": True})
         self.assertEqual(ev({"key": 30, "down": False}), {"key": 30, "down": False})
         self.assertEqual(ev({"scroll": [0, 1e9]}), {"scroll": [0.0, 5000.0]})
         self.assertEqual(ev({"dx": 3, "other": 1}), {"dx": 3.0})
 
     def test_touch_event_rejects_bad_ones(self):
-        for bad in (None, {}, {"fx": 0.5, "fy": 0.5}, {"fx": "1", "fy": 0, "window": 1}, {"button": "side"},
+        for bad in (None, {}, {"fx": 0.5, "fy": 0.5}, {"fx": "1", "fy": 0, "window": 1, "display": ":1"}, {"button": "side"},
+                    {"fx": 0.5, "fy": 0.5, "window": 1}, {"fx": 0.5, "fy": 0.5, "window": 1, "display": ":1;x"},
+                    {"window": 1, "display": ":1"},
                     {"key": 0}, {"key": 999}, {"key": True}, {"scroll": [1]}, {"text": ""}, {"text": "x" * 501},
-                    {"fx": 0.1, "fy": 0.1, "window": True}):
+                    {"fx": 0.1, "fy": 0.1, "window": True, "display": ":1"}):
             with self.assertRaises(self.s.Failure, msg=repr(bad)):
                 self.s.touch_event(bad)
 

ui/frame_touch.py

@@ -10,8 +10,9 @@ of gamescope's X displays (see docs/streaming.md).
   python3 frame_touch.py focus   print the focused panel as JSON
   python3 frame_touch.py panels  print every app panel as JSON, and which has focus
   python3 frame_touch.py         read events on stdin, one JSON object (or list) per line:
-    {"fx": 0.5, "fy": 0.2, "window": 123}  pointer to that fraction of the focused panel
+    {"fx": 0.5, "fy": 0.2, "window": 123, "display": ":1"}
-                                           (ignored if another panel has focus since)
+                                           pointer to that fraction of that panel; any event can
+                                           name its panel, and goes nowhere if another has focus
     {"dx": 4, "dy": -2}                    pointer by that much
     {"button": "left", "down": true}       left, right or middle; "down" false releases
     {"scroll": [0, 120]}                   by pixels; positive y scrolls down
@@ -88,44 +89,66 @@ def displays():
     return sorted(f":{n[1:]}" for n in os.listdir("/tmp/.X11-unix") if n[1:].isdigit())
 
 
-def focus():
+def window_pid(display, window):
-    """The panel that has focus in the headset: its window, display, name and sizes.
+    try:
+        out = subprocess.run(["xprop", "-id", str(window), "_NET_WM_PID"], env=dict(os.environ, DISPLAY=display),
+                             capture_output=True, text=True, timeout=5).stdout
+    except (OSError, subprocess.SubprocessError):
+        return None
+    value = out.rsplit("=", 1)[-1].strip() if "=" in out else ""
+    return int(value) if value.isdigit() else None
 
-    gamescope publishes it on :0's root (GAMESCOPE_FOCUSED_WINDOW). The window can be
+
-    on any of its Xwayland displays; the pid in GAMESCOPE_FOCUSABLE_WINDOWS (window,
+def locate(window, pid):
-    app id, pid triples) tells which one when ids repeat across displays.
+    """The display a focusable window is on, with its name and geometry.
+
+    Window ids are per X server, so :0 and :1 can both have one; the pid gamescope
+    lists with it (GAMESCOPE_FOCUSABLE_WINDOWS) tells them apart.
     """
-    window = (xprop_root(":0", "GAMESCOPE_FOCUSED_WINDOW") or [0])[0]
+    found = []
-    if not window:
-        return {"window": None}
-    triples = xprop_root(":0", "GAMESCOPE_FOCUSABLE_WINDOWS")
-    app = next((triples[i + 1] for i in range(0, len(triples) - 2, 3) if triples[i] == window), None)
     for display in displays():
         info = window_info(display, window)
         if info:
-            root = window_info(display, "root") or {}
+            found.append((display, info))
-            return {"window": window, "display": display, "app": app, **info,
+    if len(found) > 1 and pid:
-                    "root": [root.get("width", info["width"]), root.get("height", info["height"])]}
+        found = [f for f in found if window_pid(f[0], window) == pid] or found
-    return {"window": None}
+    if not found:
+        return None
+    display, info = found[0]
+    root = window_info(display, "root") or {}
+    return {"window": window, "display": display, **info,
+            "root": [root.get("width", info["width"]), root.get("height", info["height"])]}
+
+
+def focusable():
+    """gamescope's focusable windows as (window, app id, pid)."""
+    t = xprop_root(":0", "GAMESCOPE_FOCUSABLE_WINDOWS")
+    return [tuple(t[i:i + 3]) for i in range(0, len(t) - 2, 3)]
+
+
+def focus():
+    """The panel that has focus in the headset: window, display, name and sizes (gamescope
+    publishes the window on :0's root as GAMESCOPE_FOCUSED_WINDOW)."""
+    window = (xprop_root(":0", "GAMESCOPE_FOCUSED_WINDOW") or [0])[0]
+    if not window:
+        return {"window": None}
+    app, pid = next(((a, p) for w, a, p in focusable() if w == window), (None, None))
+    panel = locate(window, pid)
+    return {**panel, "app": app} if panel else {"window": None}
 
 
 def panels():
     """Every app panel (gamescope's focusable windows), for watching one that hasn't focus."""
-    focused = (xprop_root(":0", "GAMESCOPE_FOCUSED_WINDOW") or [0])[0]
+    now = focus()
-    triples = xprop_root(":0", "GAMESCOPE_FOCUSABLE_WINDOWS")
+    found = []
-    found, seen = [], set()
+    for window, app, pid in focusable():
-    for i in range(0, len(triples) - 2, 3):
+        panel = locate(window, pid)
-        window, app = triples[i], triples[i + 1]
+        if panel and panel["width"] > 1 and panel["height"] > 1 and \
-        if window in seen:
+                not any(f["window"] == window and f["display"] == panel["display"] for f in found):
-            continue
+            panel.pop("root", None)
-        seen.add(window)
+            found.append({**panel, "app": app, "focused": (window, panel["display"]) ==
-        for display in displays():
+                          (now.get("window"), now.get("display"))})
-            info = window_info(display, window)
+    return {"focus": now.get("window"), "focus_display": now.get("display"), "panels": found}
-            if info and info["width"] > 1 and info["height"] > 1:
-                found.append({"window": window, "display": display, "app": app, **info,
-                              "focused": window == focused})
-                break
-    return {"focus": focused or None, "panels": found}
 
 
 def to_root(panel, fx, fy):
@@ -280,15 +303,27 @@ def number(value, limit=100000.0):
     return max(-limit, min(limit, float(value)))
 
 
+def aimed_elsewhere(event, panel):
+    """Whether an event names a panel that isn't the one with focus now."""
+    if "window" not in event:
+        return False
+    return (panel.get("window"), panel.get("display")) != (event.get("window"), event.get("display"))
+
+
 def apply(gs, event, panel):
-    """Send one event; returns the panel it was aimed at (refreshed if focus moved)."""
+    """Send one event; returns the focused panel it checked against (looked up at most once a second).
-    if "fx" in event:
+
-        window = event.get("window")
+    Positions and presses name the panel they were meant for. If focus has moved to
-        if not panel or panel.get("window") != window or time.time() - panel.get("_at", 0) > 1:
+    another panel since, they go nowhere, so a tap can't land on the wrong one;
-            panel = {**focus(), "_at": time.time()}
+    releases always go, so nothing stays held.
-        if panel.get("window") is None or panel["window"] != window:
+    """
-            say("ready", focus=panel.get("window"), stale=True)  # the page re-syncs its view
+    if "window" in event and (not panel or time.time() - panel.get("_at", 0) > 1 or aimed_elsewhere(event, panel)):
-            return panel
+        panel = {**focus(), "_at": time.time()}
+    stale = aimed_elsewhere(event, panel) if "window" in event else False
+    if stale and not (event.get("down") is False and ("button" in event or "key" in event)):
+        say("ready", focus=panel.get("window"), display=panel.get("display"), stale=True)  # the page re-syncs
+        return panel
+    if "fx" in event and panel and panel.get("window"):
         gs.move_to(*to_root(panel, number(event["fx"], 1), number(event["fy"], 1)))
     if "dx" in event or "dy" in event:
         gs.move_by(number(event.get("dx", 0), 2000), number(event.get("dy", 0), 2000))

ui/index.html

@@ -1211,10 +1211,11 @@ async function capture() {
   let url = null;
   try {
     let url_ = "/api/screenshot?view=headset";
+    var shotPanel = null;
     if (view === "flat") {
-      const p = deskPanel() || (await loadPanels(), deskPanel());
+      shotPanel = deskPanel() || (await loadPanels(), deskPanel());
-      if (!p) throw new Error("no app panel is open in the headset");
+      if (!shotPanel) throw new Error("no app panel is open in the headset");
-      url_ = `/api/screenshot?view=panel&window=${p.window}&display=${encodeURIComponent(p.display)}`;
+      url_ = `/api/screenshot?view=panel&window=${shotPanel.window}&display=${encodeURIComponent(shotPanel.display)}`;
     }
     const r = await fetch(url_, { headers: {"X-Frame-UI": UI_KEY} });
     if (!r.ok) throw new Error((await r.json().catch(() => ({}))).error || `HTTP ${r.status}`);
@@ -1225,7 +1226,9 @@ async function capture() {
     if (gen !== viewGen) return true;
     lastImg = img; lastSource = source; lastShot = null;
     draw();
-    if (source === "panel") $("srcBadge").textContent = `Desktop · ${deskPanel()?.name || "panel"}`;
+    desk.shown = source === "panel" ? panelKey(shotPanel) : null;
+    if (source === "panel") $("srcBadge").textContent = `Desktop · ${shotPanel?.name || "panel"}`;
+    ctrlShow();
     $("stamp").hidden = false; $("stamp").textContent = new Date().toLocaleTimeString();
     $("saveBtn").disabled = false;
     return true;
@@ -1238,19 +1241,22 @@ async function capture() {
     $("viewer").classList.remove("busy");
   }
 }
-let liveFailures = 0;
+let liveFailures = 0, liveGen = 0;
-async function liveLoop() {
+async function liveLoop(gen = liveGen) {
-  if (!live) return;
+  if (!live || gen !== liveGen) return;
-  liveFailures = (await capture()) ? 0 : liveFailures + 1;
+  const ok = await capture();
+  if (gen !== liveGen) return;  // Live was restarted (say, as video) while this capture ran
+  liveFailures = ok ? 0 : liveFailures + 1;
   if (liveFailures >= 5) {
     toggleLive(false);
     toast(view === "flat" ? "Live stopped: couldn't capture the panel." : "Live stopped after 5 failed captures. Is the headset on?", true);
     return;
   }
-  if (live) liveTimer = setTimeout(liveLoop, view === "headset" ? 300 : 700);
+  if (live) liveTimer = setTimeout(() => liveLoop(gen), view === "headset" ? 300 : 700);
 }
 function toggleLive(on) {
   live = on;
+  liveGen++;
   liveFailures = 0;
   $("liveBtn").classList.toggle("on", live);
   $("liveBadge").hidden = !live;
@@ -1302,6 +1308,7 @@ async function startVideo() {
       if (!shown++) {
         viewGen++;  // a capture still in flight mustn't replace the video
         lastImg = null; lastSource = "video"; lastShot = null;
+        desk.shown = panelKey(panel); ctrlShow();
         $("viewer").classList.remove("busy");
         c.hidden = false; $("viewerEmpty").hidden = true; $("zoombar").hidden = false; $("asleep").hidden = true;
         $("srcBadge").hidden = false; $("srcBadge").textContent = `${video.label} · video`;
@@ -1317,6 +1324,7 @@ async function startVideo() {
     error: e => log("Video decoder: " + e.message, "e"),
   });
   const panel = view === "flat" ? deskPanel() : null;
+  desk.shown = null;  // until this stream's first picture
   video.label = panel ? `Desktop · ${panel.name || "panel"}` : "Headset view";
   const query = panel ? `src=panel&window=${panel.window}&display=${encodeURIComponent(panel.display)}&${STREAM_QUERY}` : STREAM_QUERY;
   const r = await fetch(`/api/stream?${query}`, { headers: {"X-Frame-UI": UI_KEY}, signal: ctl.signal });
@@ -1387,20 +1395,24 @@ document.querySelectorAll("[data-eye]").forEach(b => b.onclick = () => setEye(b.
 // ---- Desktop view: the app panel in use in the headset (or one picked to watch) ----
 // gamescope says which panel has focus: the one the wearer last used. That's the one
 // Control reaches, so the view follows it unless you pick another to watch.
-const desk = { panels: [], focus: null, pick: "", timer: null, loaded: false };
+// Panels are "display/window": window ids are per X display, and the Frame has two.
+// desk.shown is the panel whose picture is on screen now; Control only acts on that one.
+const desk = { panels: [], focus: null, pick: "", timer: null, loaded: false, shown: null };
+const panelKey = p => p ? `${p.display}/${p.window}` : null;
 function deskPanel() {
-  const want = desk.pick ? +desk.pick : desk.focus;
+  const want = desk.pick || desk.focus;
   // Nothing in use and nothing picked: show the first panel rather than nothing.
-  return desk.panels.find(p => p.window === want) || (!desk.pick && !desk.focus && desk.panels[0]) || null;
+  return desk.panels.find(p => panelKey(p) === want) || (!desk.pick && !desk.focus && desk.panels[0]) || null;
 }
 async function loadPanels() {
   try {
-    const before = deskPanel()?.window;
+    const before = panelKey(deskPanel());
     const r = await api("/api/panels");
-    desk.panels = r.panels || []; desk.focus = r.focus || null; desk.loaded = true;
+    desk.panels = r.panels || []; desk.loaded = true;
-    if (desk.pick && !desk.panels.some(p => p.window === +desk.pick)) desk.pick = "";
+    desk.focus = r.focus ? `${r.focus_display}/${r.focus}` : null;
+    if (desk.pick && !desk.panels.some(p => panelKey(p) === desk.pick)) desk.pick = "";
     renderPanelPick();
-    const now = deskPanel()?.window;
+    const now = panelKey(deskPanel());
     if (view === "flat" && live && now !== before) { toggleLive(false); toggleLive(true); }
     ctrlShow();
   } catch (e) {
@@ -1415,7 +1427,7 @@ function watchPanels() {
 function renderPanelPick() {
   const sel = $("panelPick");
   sel.innerHTML = `<option value="">${desk.focus ? "Panel in use" : "Panel in use (none yet)"}</option>` +
-    desk.panels.map(p => `<option value="${p.window}">${esc(p.name || "Untitled")}${p.focused ? " (in use)" : ""}</option>`).join("");
+    desk.panels.map(p => `<option value="${esc(panelKey(p))}">${esc(p.name || "Untitled")}${p.focused ? " (in use)" : ""}</option>`).join("");
   sel.value = desk.pick;
 }
 $("panelPick").onchange = () => {
@@ -1429,7 +1441,7 @@ $("panelPick").onchange = () => {
 // exactly where you tapped; on the headset view, which moves with the wearer's head,
 // the view is a trackpad for the pointer instead.
 const ctrl = { on: false, queue: [], sending: false, state: "off", message: "", move: null, rel: [0, 0], raf: 0,
-               held: new Set(), pointers: new Map(), g: null, retry: null };
+               held: new Set(), keys: new Set(), pointers: new Map(), g: null, retry: null };
 const EVDEV = { Escape: 1, Minus: 12, Equal: 13, Backspace: 14, Tab: 15, BracketLeft: 26, BracketRight: 27, Enter: 28,
   ControlLeft: 29, Semicolon: 39, Quote: 40, Backquote: 41, ShiftLeft: 42, Backslash: 43, Comma: 51, Period: 52,
   Slash: 53, ShiftRight: 54, NumpadMultiply: 55, AltLeft: 56, Space: 57, CapsLock: 58, F11: 87, F12: 88,
@@ -1452,7 +1464,9 @@ function ctrlTarget() {
   if (!desk.loaded) return { why: "Finding the headset's panels…" };
   if (!p) return { why: "No app panel is open in the headset." };
   if (!desk.focus) return { why: "Nobody's using a panel in the headset yet. Taps work once one is in use." };
-  if (p.window !== desk.focus) return { why: "Only the panel in use takes taps. Pick “Panel in use” to control it." };
+  if (panelKey(p) !== desk.focus) return { why: "Only the panel in use takes taps. Pick “Panel in use” to control it." };
+  if (desk.shown !== desk.focus) return { why: live ? "Waiting for the picture of the panel in use…"
+                                                    : "The panel in use has changed. Press Capture or Live to see it first." };
   return { ok: true, panel: p };
 }
 function ctrlShow() {
@@ -1503,10 +1517,29 @@ $("viewer").addEventListener("blur", () => { ctrlRelease(); if (window.frameApp?
 document.addEventListener("visibilitychange", () => { if (document.hidden) ctrlRelease(); });
 
 const isMoveEvent = e => ("fx" in e || "dx" in e) && !("button" in e) && !("key" in e);
+const isRelease = e => ("button" in e || "key" in e) && e.down === false;
+// While it connects, keep clicks and keys, each with the last position before it, so a
+// tap still lands where it was made; lone moves are stale by then.
+function ctrlKeepable(events) {
+  const out = [];
+  let move = null;
+  for (const e of events) {
+    if (isMoveEvent(e)) { if ("fx" in e) move = e; continue; }
+    if (move && "button" in e && e.down !== false) out.push(move);
+    move = null;
+    out.push(e);
+  }
+  if (move) out.push(move);  // the last position: a click queued after it needs it
+  return out;
+}
 function ctrlSend(events) {
-  ctrl.queue.push(...events);
+  ctrl.queue.push(...events.map(e => isMoveEvent(e) ? e : ctrlAim(e)));
-  // Keep order; if the Frame is slow to answer, drop pointer moves first, never a release.
+  // Keep order; if the Frame is slow to answer, drop pointer moves first, then the oldest
-  if (ctrl.queue.length > 300) ctrl.queue = ctrl.queue.filter(e => !isMoveEvent(e)).slice(-200);
+  // presses and scrolls, never a release.
+  if (ctrl.queue.length > 300) {
+    const q = ctrlKeepable(ctrl.queue);
+    ctrl.queue = q.filter((e, i) => isRelease(e) || i >= q.length - 200);
+  }
   ctrlFlush();
 }
 async function ctrlFlush() {
@@ -1517,20 +1550,23 @@ async function ctrlFlush() {
   try {
     const r = await api("/api/touch", { events: batch });
     ctrl.state = r.state; ctrl.message = r.message || "";
-    if (!r.sent) {
+    if (!r.sent && r.state === "error") {
-      // Not connected yet: keep clicks and keys for when it is (moves are stale by then).
+      // Broken: presses would be stale by the time it's back; releases still matter.
-      ctrl.queue.unshift(...batch.filter(e => !isMoveEvent(e)));
+      ctrl.queue.unshift(...batch.filter(isRelease));
-      if (r.state !== "error") again = 300;
+      again = 2000;
+    } else if (!r.sent) {
+      ctrl.queue.unshift(...ctrlKeepable(batch));  // connecting: send them once it's ready
+      again = 300;
     }
   } catch (e) {
     ctrl.state = "error"; ctrl.message = e.message;
-    ctrl.queue.unshift(...batch.filter(e => "button" in e && e.down === false || "key" in e && e.down === false));
+    ctrl.queue.unshift(...batch.filter(isRelease));
     again = 1000;  // try the releases again in a moment, not in a tight loop
   } finally {
     ctrl.sending = false;
     ctrlShow();
     clearTimeout(ctrl.retry);
-    if (again) ctrl.retry = setTimeout(ctrlFlush, again);
+    if (again) { if (ctrl.queue.length) ctrl.retry = setTimeout(ctrlFlush, again); }
     else if (ctrl.queue.length) ctrlFlush();
   }
 }
@@ -1538,7 +1574,7 @@ async function ctrlFlush() {
 function ctrlMoveTo(fx, fy) {
   const p = ctrlTarget().panel;
   if (!p) return;
-  ctrl.move = { fx, fy, window: p.window };
+  ctrl.move = { fx, fy, window: p.window, display: p.display };
   ctrlSchedule();
 }
 function ctrlMoveBy(dx, dy) { ctrl.rel[0] += dx; ctrl.rel[1] += dy; ctrlSchedule(); }
@@ -1550,14 +1586,26 @@ function ctrlFlushMoves() {
   if (ctrl.rel[0] || ctrl.rel[1]) { out.push({ dx: Math.round(ctrl.rel[0]), dy: Math.round(ctrl.rel[1]) }); ctrl.rel = [0, 0]; }
   if (out.length) ctrlSend(out);
 }
+// On the Desktop view, presses name the panel they're for, so the Frame drops them if
+// focus has moved on (releases always go).
+function ctrlAim(e) {
+  const p = ctrlMode() === "abs" && e.down !== false ? ctrlTarget().panel : null;
+  return p ? { ...e, window: p.window, display: p.display } : e;
+}
 function ctrlButton(name, down) {
   ctrlFlushMoves();  // the click lands where the pointer was sent
   if (down) ctrl.held.add(name); else ctrl.held.delete(name);
-  ctrlSend([{ button: name, down }]);
+  ctrlSend([ctrlAim({ button: name, down })]);
+}
+function ctrlKeyEvent(code, down) {
+  ctrlFlushMoves();
+  if (down) ctrl.keys.add(code); else ctrl.keys.delete(code);
+  ctrlSend([ctrlAim({ key: code, down })]);
 }
 function ctrlClick(name = "left") { ctrlButton(name, true); ctrlButton(name, false); }
 function ctrlRelease() {
   for (const b of [...ctrl.held]) ctrlButton(b, false);
+  for (const k of [...ctrl.keys]) ctrlKeyEvent(k, false);
   if (ctrl.g && ctrl.g.timer) clearTimeout(ctrl.g.timer);
   ctrl.pointers.clear(); ctrl.g = null;
 }
@@ -1589,7 +1637,11 @@ function ctrlTouchDown(e) {
     g.timer = setTimeout(() => {
       if (ctrl.g !== g || g.kind !== "maybe") return;
       g.kind = "held";
-      if (ctrlMode() === "abs") { const f = ctrlFraction(g.e); if (f) ctrlMoveTo(...f); }
+      if (ctrlMode() === "abs") {
+        const f = ctrlFraction(g.e);
+        if (!f) return;  // on the bars around the picture: nothing to press
+        ctrlMoveTo(...f);
+      }
       ctrlClick("right");
       navigator.vibrate?.(10);
     }, HOLD_MS);
@@ -1630,6 +1682,7 @@ function ctrlTouchUp(e) {
   const g = ctrl.g;
   ctrl.pointers.delete(e.pointerId);
   if (!g) return;
+  if (g.kind === "scroll") g.kind = "done";  // the finger left behind doesn't jump the scroll or drag
   if (g.kind === "maybe" && ctrl.pointers.size === 0) {
     clearTimeout(g.timer);
     if (ctrlMode() === "abs") { const f = ctrlFraction(g.e); if (!f) { ctrl.g = null; return; } ctrlMoveTo(...f); }
@@ -1637,6 +1690,14 @@ function ctrlTouchUp(e) {
   } else if (g.kind === "drag") ctrlButton("left", false);
   if (ctrl.pointers.size === 0) ctrl.g = null;
 }
+// The system took the touch (a swipe home, a call): no tap, and let go of anything held.
+function ctrlTouchCancel(e) {
+  ctrl.pointers.delete(e.pointerId);
+  if (ctrl.g?.timer) clearTimeout(ctrl.g.timer);
+  if (ctrl.held.has("left")) ctrlButton("left", false);
+  if (ctrl.g) ctrl.g.kind = "done";
+  if (ctrl.pointers.size === 0) ctrl.g = null;
+}
 const canvasEl = $("canvas");
 canvasEl.addEventListener("pointerdown", e => {
   if (!ctrl.on) return;
@@ -1645,6 +1706,7 @@ canvasEl.addEventListener("pointerdown", e => {
   try { canvasEl.setPointerCapture(e.pointerId); } catch { /* a pointer that's already gone */ }
   if (!ctrlTarget().ok) return ctrlShow();
   if (e.pointerType === "touch") return ctrlTouchDown(e);
+  if (ctrlMode() === "abs" && !ctrlFraction(e)) return;  // on the bars around the picture
   ctrlMouse(e);
   ctrlButton(CTRL_BUTTONS[e.button] || "left", true);
 });
@@ -1655,7 +1717,7 @@ canvasEl.addEventListener("pointermove", e => {
 });
 for (const type of ["pointerup", "pointercancel"]) canvasEl.addEventListener(type, e => {
   if (!ctrl.on) return;
-  if (e.pointerType === "touch") return ctrlTouchUp(e);
+  if (e.pointerType === "touch") return type === "pointercancel" ? ctrlTouchCancel(e) : ctrlTouchUp(e);
   const name = CTRL_BUTTONS[e.button] || "left";
   if (ctrl.held.has(name)) ctrlButton(name, false);
 });
@@ -1668,19 +1730,21 @@ $("viewer").addEventListener("wheel", e => {
 }, { passive: false });
 // Keys while the view has focus go to the Frame (⌘ as Ctrl on a Mac); click elsewhere to stop.
 function ctrlKey(e, down) {
-  if (!ctrl.on || document.activeElement !== $("viewer") || !ctrlTarget().ok) return;
+  if (!ctrl.on || document.activeElement !== $("viewer") || (down && !ctrlTarget().ok)) return;
   let code = EVDEV[e.code];
   if (IS_MAC && (e.code === "MetaLeft" || e.code === "MetaRight")) code = 29;
   if (!code) return;
   e.preventDefault();
-  if (down && e.repeat && code >= 29 && [29, 42, 54, 56, 97, 100, 125, 126].includes(code)) return;
+  e.stopPropagation();  // it's for the Frame, not Frame Control's own shortcuts
-  ctrlSend([{ key: code, down }]);
+  if (down && e.repeat && [29, 42, 54, 56, 97, 100, 125, 126].includes(code)) return;
+  if (!down && !ctrl.keys.has(code)) return;
+  ctrlKeyEvent(code, down);
 }
 $("viewer").addEventListener("keydown", e => ctrlKey(e, true));
 $("viewer").addEventListener("keyup", e => ctrlKey(e, false));
 
 // Phone keyboard: the field types plain text; the buttons send single keys and clicks.
-function ctrlTap(code) { ctrlSend([{ key: code, down: true }, { key: code, down: false }]); }
+function ctrlTap(code) { ctrlKeyEvent(code, true); ctrlKeyEvent(code, false); }
 let ctrlWarned = false;
 function ctrlText(text) {
   const plain = text.replace(/[^\x20-\x7e\n\t]/g, "");

ui/server.py

@@ -868,11 +868,17 @@ def touch_event(event):
         return max(-limit, min(limit, round(float(value), 4)))
     out = {}
     if "fx" in event or "fy" in event:
+        if "window" not in event:
+            raise Failure("a position needs the panel's window and display", 400)
         out.update(fx=num("fx", 1), fy=num("fy", 1))
-        window = event.get("window")
+    # Any event can name the panel it's meant for; the Frame drops it if another has focus.
+    if "window" in event:
+        window, display = event.get("window"), event.get("display")
         if isinstance(window, bool) or not isinstance(window, int) or window <= 0:
             raise Failure("window must be the panel's window id", 400)
-        out["window"] = window
+        if not isinstance(display, str) or not PANEL_DISPLAY.match(display):
+            raise Failure("display must be an X display such as :1", 400)
+        out.update(window=window, display=display)
     for name in ("dx", "dy"):
         if name in event:
             out[name] = num(name, INPUT_MOVE_LIMIT)
@@ -895,7 +901,7 @@ def touch_event(event):
         if not isinstance(text, str) or not 0 < len(text) <= INPUT_TEXT_LIMIT:
             raise Failure(f"text must be 1 to {INPUT_TEXT_LIMIT} characters", 400)
         out["text"] = text
-    if not out:
+    if not set(out) - {"window", "display"}:
         raise Failure("touch event has nothing to do", 400)
     return out