From ac07efa8f114a61660e971bcd3a3a7cd1ff2e339 Mon Sep 17 00:00:00 2001 From: saphid <4596216+saphid@users.noreply.github.com> Date: Tue, 29 Sep 2026 08:52:00 +1000 Subject: [PATCH] Control: close the targeting gaps from the second review - The focused panel's display comes from GAMESCOPE_FOCUS_DISPLAY (gamescope packs ":1" into the first value), so a window id repeated across :0 and :1 can't be mistaken; the pid is only the fallback. - Presses, keys, text and scrolls read focus afresh on the Frame; only moves use a reading up to a second old. - A gesture remembers the panel it started on and does nothing more if that stops being the one in use; a press with no panel to aim at isn't sent. - Opening a screenshot clears the panel Control would act on; switching to another app releases held keys and buttons. - Trimming keeps a click with its position; the error backoff holds for new input too. Co-Authored-By: Claude Opus 5.5 (1M context) --- tests/page/ctrl_gestures.mjs | 41 ++++++++++++++++++++++++++++++++---- tests/test_touch.py | 21 ++++++++++++++++++ ui/frame_touch.py | 30 +++++++++++++++++++++++--- ui/index.html | 35 ++++++++++++++++++++++++------ 4 files changed, 113 insertions(+), 14 deletions(-) diff --git a/tests/page/ctrl_gestures.mjs b/tests/page/ctrl_gestures.mjs index 078a019..43bb9b2 100644 --- a/tests/page/ctrl_gestures.mjs +++ b/tests/page/ctrl_gestures.mjs @@ -9,7 +9,7 @@ const grab = name => { if (!m) throw new Error("not found: " + name); return m[0]; }; -const NAMES = ["isMoveEvent", "isRelease", "ctrlKeepable", "TAP_MOVE", "ctrlAim", "ctrlKeyEvent", "ctrlTouchCancel", "ctrlSend", "ctrlFlush", "ctrlMoveTo", "ctrlMoveBy", "ctrlSchedule", +const NAMES = ["panelKey", "ctrlAimedAt", "ctrlSameTarget", "isMoveEvent", "isRelease", "ctrlKeepable", "TAP_MOVE", "ctrlAim", "ctrlKeyEvent", "ctrlTouchCancel", "ctrlSend", "ctrlFlush", "ctrlMoveTo", "ctrlMoveBy", "ctrlSchedule", "ctrlFlushMoves", "ctrlButton", "ctrlClick", "ctrlRelease", "ctrlFraction", "ctrlTouchDown", "centroid", "ctrlTouchMove", "ctrlTouchUp", "ctrlTap", "ctrlText"]; const code = NAMES.map(grab).join(""); @@ -17,13 +17,14 @@ const fail = msg => { console.log("FAIL " + msg); process.exit(1); }; const tick = (ms = 0) => new Promise(r => setTimeout(r, ms)); function page({ mode = "abs", rect = { left: 0, top: 0, width: 640, height: 360 }, api } = {}) { + const target = { panel: { window: 42, display: ":1" } }; const ctrl = { on: true, queue: [], sending: false, state: "ready", message: "", move: null, rel: [0, 0], raf: 0, held: new Set(), keys: new Set(), pointers: new Map(), g: null, retry: null }; const sent = []; const canvas = { width: 1280, height: 720, getBoundingClientRect: () => rect }; const env = { ctrl, $: () => canvas, ctrlMode: () => mode, ctrlShow: () => {}, toast: () => {}, - ctrlTarget: () => (mode === "abs" ? { ok: true, panel: { window: 42, display: ":1" } } : { ok: true }), + ctrlTarget: () => (mode !== "abs" ? { ok: true } : target.panel ? { ok: true, panel: target.panel } : { why: "gone" }), api: api || (async (path, body) => { sent.push(...body.events); return { state: "ready", sent: true }; }), requestAnimationFrame: cb => { setTimeout(cb, 0); return 1; }, navigator: {}, @@ -31,7 +32,7 @@ function page({ mode = "abs", rect = { left: 0, top: 0, width: 640, height: 360 const fns = new Function(...Object.keys(env), `let ctrlWarned = false;\n${code} return { ctrlTouchDown, ctrlTouchMove, ctrlTouchUp, ctrlTouchCancel, ctrlSend, ctrlText, ctrlButton, ctrlKeyEvent, ctrlRelease, ctrlFlushMoves };`)(...Object.values(env)); const at = (id, x, y) => ({ pointerId: id, clientX: x, clientY: y }); - return { ctrl, sent, ...fns, at }; + return { ctrl, sent, target, ...fns, at }; } // A tap lands where it was tapped: pointer there first, then the click. @@ -115,7 +116,7 @@ function page({ mode = "abs", rect = { left: 0, top: 0, width: 640, height: 360 const q = p.ctrl.queue; if (!(q.length === 3 && "fx" in q[0] && q[1].button === "left" && q[2].key === 30)) fail("kept while starting " + JSON.stringify(q)); clearTimeout(p.ctrl.retry); - p.ctrl.queue = []; + p.ctrl.queue = []; p.ctrl.retryAt = 0; p.ctrlSend([{ button: "left", down: false }, { key: 31, down: true }, { dx: 3, dy: 1 }]); await tick(5); if (!(p.ctrl.queue.length === 1 && p.ctrl.queue[0].button === "left" && p.ctrl.queue[0].down === false)) @@ -198,4 +199,36 @@ function page({ mode = "abs", rect = { left: 0, top: 0, width: 640, height: 360 if (check({ ...base, pick: ":0/5", shown: ":0/5" }).ok) fail("target: a watched panel that isn't in use"); if (check({ ...base, focus: null }).ok) fail("target: nothing in use"); } +// Focus moves to another panel mid-gesture: the tap or hold does nothing. +{ + const p = page(); + p.ctrlTouchDown(p.at(1, 100, 100)); + p.target.panel = { window: 43, display: ":1" }; + p.ctrlTouchUp(p.at(1, 100, 100)); await tick(10); + p.target.panel = { window: 42, display: ":1" }; + p.ctrlTouchDown(p.at(1, 100, 100)); + p.target.panel = null; + await tick(620); p.ctrlTouchUp(p.at(1, 100, 100)); await tick(10); + if (p.sent.some(e => "button" in e)) fail("gesture outlived its panel " + JSON.stringify(p.sent)); +} +// Trimming keeps a click together with its position. +{ + const p = page({ api: () => new Promise(() => {}) }); + p.ctrlSend([{ dx: 1, dy: 0 }]); // in flight forever + for (let i = 0; i < 100; i++) p.ctrlSend([{ scroll: [0, 1] }]); + p.ctrlSend([{ fx: 0.5, fy: 0.5, window: 42, display: ":1" }, { button: "left", down: true }, { button: "left", down: false }]); + for (let i = 0; i < 198; i++) p.ctrlSend([{ scroll: [0, 1] }]); + const q = p.ctrl.queue, i = q.findIndex(e => e.button === "left" && e.down); + if (i < 1 || !("fx" in q[i - 1])) fail("trim split a click from its position"); +} +// Backing off holds for new input too. +{ + let calls = 0; + const p = page({ api: async () => { calls++; return { state: "error", sent: false }; } }); + p.ctrlSend([{ button: "left", down: false }]); + await tick(5); + for (let i = 0; i < 10; i++) { p.ctrlSend([{ key: 30, down: false }]); await tick(2); } + if (calls !== 1) fail("new input bypassed the backoff: " + calls + " requests"); + clearTimeout(p.ctrl.retry); +} console.log("control gestures ok"); diff --git a/tests/test_touch.py b/tests/test_touch.py index 686cbf7..1848561 100644 --- a/tests/test_touch.py +++ b/tests/test_touch.py @@ -50,6 +50,14 @@ class Mapping(unittest.TestCase): self.assertEqual(t.locate(5, 111)["display"], ":0") self.assertEqual(t.locate(5, None)["display"], ":0") + def test_focus_display_is_decoded(self): + t = self.t + saved = t.xprop_root + self.addCleanup(lambda: setattr(t, "xprop_root", saved)) + for values, want in (([12602, 0, 58], ":1"), ([12346], ":0"), ([], None), ([0x41], None)): + t.xprop_root = lambda d, n, v=values: v + self.assertEqual(t.focus_display(), want) + def test_ascii_table_covers_printable_characters(self): for code in range(0x20, 0x7F): self.assertIn(chr(code), self.t.ASCII, chr(code)) @@ -104,6 +112,19 @@ class Apply(unittest.TestCase): self.assertEqual(gs.calls, [("button", "left", False)]) self.assertEqual(self.said[0], ("ready", {"focus": 7, "display": ":1", "stale": True})) + def test_presses_read_focus_afresh(self): + # A move may use a recent reading; a press checks again, so a panel that just + # took focus doesn't get a click meant for another. + gs, calls = FakeGamescope(), [] + self.t.focus = lambda: calls.append(1) or dict(self.focused) + panel = self.t.apply(gs, {"fx": 0.5, "fy": 0.5, "window": 7, "display": ":1"}, None) + panel = self.t.apply(gs, {"fx": 0.6, "fy": 0.5, "window": 7, "display": ":1"}, panel) + self.assertEqual(len(calls), 1) + self.focused["window"] = 8 + self.t.apply(gs, {"button": "left", "down": True, "window": 7, "display": ":1"}, panel) + self.assertEqual(len(calls), 2) + self.assertEqual([c[0] for c in gs.calls], ["move_to", "move_to"]) + def test_same_window_id_on_the_other_display_is_another_panel(self): gs = FakeGamescope() self.t.apply(gs, {"fx": 0.5, "fy": 0.5, "window": 7, "display": ":0"}, None) diff --git a/ui/frame_touch.py b/ui/frame_touch.py index 2dc3a77..41de832 100644 --- a/ui/frame_touch.py +++ b/ui/frame_touch.py @@ -126,14 +126,35 @@ def focusable(): return [tuple(t[i:i + 3]) for i in range(0, len(t) - 2, 3)] +def focus_display(): + """The display of the focused window, from GAMESCOPE_FOCUS_DISPLAY on :0's root. + + gamescope writes the name (":1") as 32-bit items, so its first four bytes land, + little-endian, in the first value: 12602 is 0x313A, ":1" (steamcompmgr.cpp; + seen 2026-09-29). + """ + values = xprop_root(":0", "GAMESCOPE_FOCUS_DISPLAY") + if not values: + return None + name = (values[0] & 0xFFFFFFFF).to_bytes(4, "little").split(b"\0", 1)[0].decode("ascii", "replace") + return name if name[:1] == ":" and name[1:].isdigit() else None + + def focus(): """The panel that has focus in the headset: window, display, name and sizes (gamescope - publishes the window on :0's root as GAMESCOPE_FOCUSED_WINDOW).""" + publishes the window and its display on :0's root).""" window = (xprop_root(":0", "GAMESCOPE_FOCUSED_WINDOW") or [0])[0] if not window: return {"window": None} app, pid = next(((a, p) for w, a, p in focusable() if w == window), (None, None)) - panel = locate(window, pid) + display = focus_display() + info = window_info(display, window) if display else None + if info: + root = window_info(display, "root") or {} + panel = {"window": window, "display": display, **info, + "root": [root.get("width", info["width"]), root.get("height", info["height"])]} + else: + panel = locate(window, pid) # no display published: tell them apart by pid return {**panel, "app": app} if panel else {"window": None} @@ -317,7 +338,10 @@ def apply(gs, event, panel): another panel since, they go nowhere, so a tap can't land on the wrong one; releases always go, so nothing stays held. """ - if "window" in event and (not panel or time.time() - panel.get("_at", 0) > 1 or aimed_elsewhere(event, panel)): + # Moves may use a focus reading up to a second old; anything that acts (a press, key, + # text or scroll) reads it afresh, so it can't land on a panel that took focus since. + acts = any(k in event for k in ("button", "key", "text", "scroll")) and event.get("down") is not False + if "window" in event and (acts or not panel or time.time() - panel.get("_at", 0) > 1 or aimed_elsewhere(event, panel)): panel = {**focus(), "_at": time.time()} stale = aimed_elsewhere(event, panel) if "window" in event else False if stale and not (event.get("down") is False and ("button" in event or "key" in event)): diff --git a/ui/index.html b/ui/index.html index 6948ef5..bdabd63 100644 --- a/ui/index.html +++ b/ui/index.html @@ -1515,6 +1515,7 @@ $("viewer").tabIndex = 0; $("viewer").addEventListener("focus", () => { if (ctrl.on && window.frameApp?.captureKeys) window.frameApp.captureKeys(true); }); $("viewer").addEventListener("blur", () => { ctrlRelease(); if (window.frameApp?.captureKeys) window.frameApp.captureKeys(false); }); document.addEventListener("visibilitychange", () => { if (document.hidden) ctrlRelease(); }); +window.addEventListener("blur", () => ctrlRelease()); // another app took the keyboard const isMoveEvent = e => ("fx" in e || "dx" in e) && !("button" in e) && !("key" in e); const isRelease = e => ("button" in e || "key" in e) && e.down === false; @@ -1533,17 +1534,27 @@ function ctrlKeepable(events) { return out; } function ctrlSend(events) { - ctrl.queue.push(...events.map(e => isMoveEvent(e) ? e : ctrlAim(e))); + ctrl.queue.push(...events.map(e => isMoveEvent(e) ? e : ctrlAim(e)).filter(Boolean)); // Keep order; if the Frame is slow to answer, drop pointer moves first, then the oldest // presses and scrolls, never a release. if (ctrl.queue.length > 300) { - const q = ctrlKeepable(ctrl.queue); - ctrl.queue = q.filter((e, i) => isRelease(e) || i >= q.length - 200); + const q = ctrlKeepable(ctrl.queue), keep = new Set(); + q.forEach((e, i) => { if (isRelease(e) || i >= q.length - 200) keep.add(i); }); + // A kept press keeps the position before it, so it lands where it was made. + for (const i of [...keep]) if (i > 0 && "button" in q[i] && q[i].down !== false && "fx" in q[i - 1]) keep.add(i - 1); + ctrl.queue = q.filter((e, i) => keep.has(i)); } ctrlFlush(); } async function ctrlFlush() { if (ctrl.sending || !ctrl.queue.length) return; + // Backing off after a failure: new input waits for the same retry. + const wait = (ctrl.retryAt || 0) - Date.now(); + if (wait > 0) { + clearTimeout(ctrl.retry); + ctrl.retry = setTimeout(ctrlFlush, wait); + return; + } ctrl.sending = true; const batch = ctrl.queue.splice(0, 200); let again = 0; @@ -1566,6 +1577,7 @@ async function ctrlFlush() { ctrl.sending = false; ctrlShow(); clearTimeout(ctrl.retry); + ctrl.retryAt = again ? Date.now() + again : 0; if (again) { if (ctrl.queue.length) ctrl.retry = setTimeout(ctrlFlush, again); } else if (ctrl.queue.length) ctrlFlush(); } @@ -1588,9 +1600,11 @@ function ctrlFlushMoves() { } // On the Desktop view, presses name the panel they're for, so the Frame drops them if // focus has moved on (releases always go). +// A press with no panel to aim at (the Desktop view lost its target) isn't sent at all. function ctrlAim(e) { - const p = ctrlMode() === "abs" && e.down !== false ? ctrlTarget().panel : null; - return p ? { ...e, window: p.window, display: p.display } : e; + if (ctrlMode() !== "abs" || e.down === false) return e; + const p = ctrlTarget().panel; + return p ? { ...e, window: p.window, display: p.display } : null; } function ctrlButton(name, down) { ctrlFlushMoves(); // the click lands where the pointer was sent @@ -1633,9 +1647,9 @@ const TAP_MOVE = 8, HOLD_MS = 550, TOUCH_SPEED = 1.6, SCROLL_SPEED = 2; function ctrlTouchDown(e) { ctrl.pointers.set(e.pointerId, { x: e.clientX, y: e.clientY }); if (ctrl.pointers.size === 1) { - const g = ctrl.g = { kind: "maybe", x: e.clientX, y: e.clientY, e }; + const g = ctrl.g = { kind: "maybe", x: e.clientX, y: e.clientY, e, target: ctrlAimedAt() }; g.timer = setTimeout(() => { - if (ctrl.g !== g || g.kind !== "maybe") return; + if (ctrl.g !== g || g.kind !== "maybe" || !ctrlSameTarget(g)) return; g.kind = "held"; if (ctrlMode() === "abs") { const f = ctrlFraction(g.e); @@ -1651,6 +1665,10 @@ function ctrlTouchDown(e) { ctrl.g = { kind: "scroll", ...centroid() }; } } +// The panel a gesture started on; if that stops being the one in use, the gesture +// does nothing more (except let go of what it holds). +function ctrlAimedAt() { return ctrlMode() === "abs" ? panelKey(ctrlTarget().panel) : "headset"; } +function ctrlSameTarget(g) { return g.target && g.target === ctrlAimedAt(); } function centroid() { const ps = [...ctrl.pointers.values()]; return { x: ps.reduce((a, p) => a + p.x, 0) / ps.length, y: ps.reduce((a, p) => a + p.y, 0) / ps.length }; @@ -1669,6 +1687,7 @@ function ctrlTouchMove(e) { } if (g.kind === "maybe" && Math.hypot(e.clientX - g.x, e.clientY - g.y) > TAP_MOVE) { clearTimeout(g.timer); + if (!ctrlSameTarget(g)) { g.kind = "done"; return; } if (ctrlMode() === "abs") { const f = ctrlFraction(g.e); if (f) { ctrlMoveTo(...f); ctrlButton("left", true); } @@ -1683,6 +1702,7 @@ function ctrlTouchUp(e) { ctrl.pointers.delete(e.pointerId); if (!g) return; if (g.kind === "scroll") g.kind = "done"; // the finger left behind doesn't jump the scroll or drag + if (g.kind === "maybe" && !ctrlSameTarget(g)) g.kind = "done"; if (g.kind === "maybe" && ctrl.pointers.size === 0) { clearTimeout(g.timer); if (ctrlMode() === "abs") { const f = ctrlFraction(g.e); if (!f) { ctrl.g = null; return; } ctrlMoveTo(...f); } @@ -2654,6 +2674,7 @@ async function openShot(s) { await new Promise((ok, bad) => { img.onload = ok; img.onerror = () => bad(new Error("not an image")); img.src = url; }); if (gen !== viewGen) return; lastImg = img; lastSource = "shot"; lastShot = { blob, file: s.file }; + desk.shown = null; // a screenshot, not a panel: Control has nothing to act on here draw(); $("srcBadge").textContent = `Screenshot ยท ${shotApp(s.appid)}`; $("stamp").hidden = false; $("stamp").textContent = new Date(s.time * 1000).toLocaleString();