From 0f4f0b863f9109ebe7e453e54cc52c58368eb5dd Mon Sep 17 00:00:00 2001 From: saphid <4596216+saphid@users.noreply.github.com> Date: Tue, 29 Sep 2026 08:40:46 +1000 Subject: [PATCH] Control: fixes from review - Keys held on the Frame are released with buttons when Control stops or the view loses focus; keys for the Frame no longer trigger Frame Control's own shortcuts. - Taps only act when the picture on screen is the panel in use; positions, presses, keys, text and scrolls name their panel (display and window: ids repeat across :0 and :1, told apart by pid), and the Frame drops them if focus has moved on. Releases always go. - While connecting, a tap keeps its position; on an error only releases wait and retries back off; trimming a long queue never drops a release. - Lifting one of two scrolling fingers ends the scroll; a cancelled touch isn't a tap; clicks and holds on the bars around the picture do nothing. - A capture loop from before a Live restart can't stop the new video. Co-Authored-By: Claude Opus 5.5 (1M context) --- tests/page/ctrl_gestures.mjs | 83 +++++++++++++++++++-- tests/test_touch.py | 36 ++++++++-- ui/frame_touch.py | 113 +++++++++++++++++++---------- ui/index.html | 136 +++++++++++++++++++++++++---------- ui/server.py | 12 +++- 5 files changed, 289 insertions(+), 91 deletions(-) diff --git a/tests/page/ctrl_gestures.mjs b/tests/page/ctrl_gestures.mjs index 512eeb8..078a019 100644 --- a/tests/page/ctrl_gestures.mjs +++ b/tests/page/ctrl_gestures.mjs @@ -9,7 +9,7 @@ const grab = name => { if (!m) throw new Error("not found: " + name); return m[0]; }; -const NAMES = ["isMoveEvent", "TAP_MOVE", "ctrlSend", "ctrlFlush", "ctrlMoveTo", "ctrlMoveBy", "ctrlSchedule", +const NAMES = ["isMoveEvent", "isRelease", "ctrlKeepable", "TAP_MOVE", "ctrlAim", "ctrlKeyEvent", "ctrlTouchCancel", "ctrlSend", "ctrlFlush", "ctrlMoveTo", "ctrlMoveBy", "ctrlSchedule", "ctrlFlushMoves", "ctrlButton", "ctrlClick", "ctrlRelease", "ctrlFraction", "ctrlTouchDown", "centroid", "ctrlTouchMove", "ctrlTouchUp", "ctrlTap", "ctrlText"]; const code = NAMES.map(grab).join(""); @@ -18,18 +18,18 @@ const tick = (ms = 0) => new Promise(r => setTimeout(r, ms)); function page({ mode = "abs", rect = { left: 0, top: 0, width: 640, height: 360 }, api } = {}) { const ctrl = { on: true, queue: [], sending: false, state: "ready", message: "", move: null, rel: [0, 0], raf: 0, - held: new Set(), pointers: new Map(), g: null, retry: null }; + held: new Set(), keys: new Set(), pointers: new Map(), g: null, retry: null }; const sent = []; const canvas = { width: 1280, height: 720, getBoundingClientRect: () => rect }; const env = { ctrl, $: () => canvas, ctrlMode: () => mode, ctrlShow: () => {}, toast: () => {}, - ctrlTarget: () => (mode === "abs" ? { ok: true, panel: { window: 42 } } : { ok: true }), + ctrlTarget: () => (mode === "abs" ? { ok: true, panel: { window: 42, display: ":1" } } : { ok: true }), api: api || (async (path, body) => { sent.push(...body.events); return { state: "ready", sent: true }; }), requestAnimationFrame: cb => { setTimeout(cb, 0); return 1; }, navigator: {}, }; const fns = new Function(...Object.keys(env), `let ctrlWarned = false;\n${code} - return { ctrlTouchDown, ctrlTouchMove, ctrlTouchUp, ctrlSend, ctrlText, ctrlButton, ctrlRelease, ctrlFlushMoves };`)(...Object.values(env)); + return { ctrlTouchDown, ctrlTouchMove, ctrlTouchUp, ctrlTouchCancel, ctrlSend, ctrlText, ctrlButton, ctrlKeyEvent, ctrlRelease, ctrlFlushMoves };`)(...Object.values(env)); const at = (id, x, y) => ({ pointerId: id, clientX: x, clientY: y }); return { ctrl, sent, ...fns, at }; } @@ -40,7 +40,8 @@ function page({ mode = "abs", rect = { left: 0, top: 0, width: 640, height: 360 p.ctrlTouchDown(p.at(1, 320, 90)); p.ctrlTouchUp(p.at(1, 320, 90)); await tick(10); const [move, down, up] = p.sent; - if (!(move.fx === 0.5 && Math.abs(move.fy - 0.25) < 1e-9 && move.window === 42)) fail("tap position " + JSON.stringify(move)); + if (!(move.fx === 0.5 && Math.abs(move.fy - 0.25) < 1e-9 && move.window === 42 && move.display === ":1")) fail("tap position " + JSON.stringify(move)); + if (!(down.window === 42 && down.display === ":1")) fail("a press names its panel " + JSON.stringify(down)); if (!(down.button === "left" && down.down && up.button === "left" && up.down === false && p.sent.length === 3)) fail("tap click " + JSON.stringify(p.sent)); } @@ -105,14 +106,14 @@ function page({ mode = "abs", rect = { left: 0, top: 0, width: 640, height: 360 const text = p.sent.map(e => e.text).join(""); if (!(text === "hllo " + "x".repeat(600) && p.sent.every(e => e.text.length <= 500))) fail("text " + text.length); } -// Not connected yet, or the request fails: clicks and keys wait, stale moves don't; a release is never lost. +// Not connected yet: clicks and keys wait with their position. The request fails: only releases wait. { let calls = 0; const p = page({ api: async () => { calls++; if (calls === 1) return { state: "starting", sent: false }; throw new Error("offline"); } }); p.ctrlSend([{ fx: 0.1, fy: 0.1, window: 42 }, { button: "left", down: true }, { key: 30, down: true }]); await tick(5); const q = p.ctrl.queue; - if (!(q.length === 2 && q[0].button === "left" && q[1].key === 30)) fail("kept while starting " + JSON.stringify(q)); + if (!(q.length === 3 && "fx" in q[0] && q[1].button === "left" && q[2].key === 30)) fail("kept while starting " + JSON.stringify(q)); clearTimeout(p.ctrl.retry); p.ctrl.queue = []; p.ctrlSend([{ button: "left", down: false }, { key: 31, down: true }, { dx: 3, dy: 1 }]); @@ -129,4 +130,72 @@ function page({ mode = "abs", rect = { left: 0, top: 0, width: 640, height: 360 p.ctrlRelease(); await tick(10); if (!(p.sent.at(-1).button === "left" && p.sent.at(-1).down === false && !p.ctrl.held.size)) fail("release " + JSON.stringify(p.sent)); } +// Keys held on the Frame are let go too. +{ + const p = page(); + p.ctrlKeyEvent(42, true); + p.ctrlRelease(); await tick(10); + if (!(p.sent.at(-1).key === 42 && p.sent.at(-1).down === false && !p.ctrl.keys.size)) fail("key release " + JSON.stringify(p.sent)); +} +// A long queue sheds moves and old scrolls, never a release. +{ + const p = page({ api: () => new Promise(() => {}) }); // stuck request + p.ctrlSend([{ dx: 1, dy: 1 }]); + p.ctrlSend([{ button: "left", down: false }]); + for (let i = 0; i < 320; i++) p.ctrlSend([{ scroll: [0, 1] }]); + if (!p.ctrl.queue.some(e => e.button === "left" && e.down === false)) fail("trim dropped a release"); + if (p.ctrl.queue.length > 300) fail("trim kept " + p.ctrl.queue.length); +} +// Broken on the Frame side: no hammering, and only releases wait. +{ + let calls = 0; + const p = page({ api: async () => { calls++; return { state: "error", sent: false }; } }); + p.ctrlSend([{ button: "left", down: true }, { button: "left", down: false }]); + await tick(50); + if (calls !== 1) fail("error response reposted " + calls + " times"); + if (!(p.ctrl.queue.length === 1 && p.ctrl.queue[0].down === false)) fail("error kept " + JSON.stringify(p.ctrl.queue)); + clearTimeout(p.ctrl.retry); +} +// Connecting: a tap keeps its position, so it lands where it was made. +{ + const p = page({ api: async () => ({ state: "starting", sent: false }) }); + p.ctrlTouchDown(p.at(1, 320, 90)); p.ctrlTouchUp(p.at(1, 320, 90)); + await tick(10); + const q = p.ctrl.queue; + if (!("fx" in q[0] && q[1].button === "left")) fail("connecting tap " + JSON.stringify(q)); + clearTimeout(p.ctrl.retry); +} +// Lifting one of two scrolling fingers doesn't jump the scroll or start a drag. +{ + const p = page(); + p.ctrlTouchDown(p.at(1, 100, 200)); p.ctrlTouchDown(p.at(2, 300, 200)); + p.ctrlTouchUp(p.at(1, 100, 200)); + p.ctrlTouchMove(p.at(2, 300, 199)); + p.ctrlTouchUp(p.at(2, 300, 199)); await tick(10); + if (p.sent.length) fail("one finger left after scrolling " + JSON.stringify(p.sent)); +} +// A cancelled touch isn't a tap. +{ + const p = page(); + p.ctrlTouchDown(p.at(1, 100, 100)); p.ctrlTouchCancel(p.at(1, 100, 100)); await tick(10); + if (p.sent.length) fail("cancel clicked " + JSON.stringify(p.sent)); +} +// Press and hold on the bars around the picture does nothing. +{ + const p = page({ rect: { left: 0, top: 0, width: 640, height: 480 } }); + p.ctrlTouchDown(p.at(1, 320, 10)); await tick(620); p.ctrlTouchUp(p.at(1, 320, 10)); await tick(10); + if (p.sent.length) fail("hold on the bars " + JSON.stringify(p.sent)); +} +// Taps only reach the panel in use, and only once its picture is the one on screen. +{ + const fns = ["panelKey", "deskPanel", "ctrlTarget"].map(grab).join(""); + const check = (desk, live = true) => new Function("desk", "live", "ctrlMode", fns + "; return ctrlTarget();")(desk, live, () => "abs"); + const a = { display: ":1", window: 5 }, b = { display: ":0", window: 5 }; + const base = { panels: [a, b], loaded: true, pick: "", focus: ":1/5", shown: ":1/5" }; + if (!check(base).ok) fail("target: shown and in use"); + if (check({ ...base, shown: ":0/5" }).ok) fail("target: the picture is another panel with the same id"); + if (!/Capture or Live/.test(check({ ...base, shown: null }, false).why)) fail("target: stale picture message"); + if (check({ ...base, pick: ":0/5", shown: ":0/5" }).ok) fail("target: a watched panel that isn't in use"); + if (check({ ...base, focus: null }).ok) fail("target: nothing in use"); +} console.log("control gestures ok"); diff --git a/tests/test_touch.py b/tests/test_touch.py index 507cdd6..686cbf7 100644 --- a/tests/test_touch.py +++ b/tests/test_touch.py @@ -38,6 +38,18 @@ class Mapping(unittest.TestCase): self.assertAlmostEqual(x, 280, delta=0.5) self.assertAlmostEqual(y, 359.5, delta=0.5) + def test_same_id_on_both_displays_is_told_apart_by_pid(self): + t = self.t + saved = t.displays, t.window_info, t.window_pid + self.addCleanup(lambda: (setattr(t, "displays", saved[0]), setattr(t, "window_info", saved[1]), + setattr(t, "window_pid", saved[2]))) + t.displays = lambda: [":0", ":1"] + t.window_info = lambda d, w: {"name": f"on {d}", "width": 1280 if w != "root" else 1920, "height": 720} + t.window_pid = lambda d, w: {":0": 111, ":1": 222}[d] + self.assertEqual(t.locate(5, 222)["display"], ":1") + self.assertEqual(t.locate(5, 111)["display"], ":0") + self.assertEqual(t.locate(5, None)["display"], ":0") + def test_ascii_table_covers_printable_characters(self): for code in range(0x20, 0x7F): self.assertIn(chr(code), self.t.ASCII, chr(code)) @@ -77,16 +89,25 @@ class Apply(unittest.TestCase): def test_tap_moves_then_clicks_in_order(self): gs, panel = FakeGamescope(), None - for e in ({"fx": 0.5, "fy": 0.5, "window": 7}, {"button": "left", "down": True}, {"button": "left", "down": False}): + for e in ({"fx": 0.5, "fy": 0.5, "window": 7, "display": ":1"}, {"button": "left", "down": True, "window": 7, "display": ":1"}, + {"button": "left", "down": False}): panel = self.t.apply(gs, e, panel) self.assertEqual([c[0] for c in gs.calls], ["move_to", "button", "button"]) self.assertEqual(gs.calls[1][1:], ("left", True)) def test_a_tap_meant_for_another_panel_goes_nowhere(self): + # Focus moved on: the position and the press are dropped; the release still goes. + gs, panel = FakeGamescope(), None + for e in ({"fx": 0.5, "fy": 0.5, "window": 99, "display": ":1"}, {"button": "left", "down": True, "window": 99, "display": ":1"}, + {"key": 30, "down": True, "window": 7, "display": ":0"}, {"button": "left", "down": False, "window": 99, "display": ":1"}): + panel = self.t.apply(gs, e, panel) + self.assertEqual(gs.calls, [("button", "left", False)]) + self.assertEqual(self.said[0], ("ready", {"focus": 7, "display": ":1", "stale": True})) + + def test_same_window_id_on_the_other_display_is_another_panel(self): gs = FakeGamescope() - self.t.apply(gs, {"fx": 0.5, "fy": 0.5, "window": 99}, None) + self.t.apply(gs, {"fx": 0.5, "fy": 0.5, "window": 7, "display": ":0"}, None) self.assertEqual(gs.calls, []) - self.assertEqual(self.said, [("ready", {"focus": 7, "stale": True})]) def test_relative_scroll_keys_text(self): gs = FakeGamescope() @@ -115,16 +136,19 @@ class ServerChecks(unittest.TestCase): def test_touch_event_keeps_known_fields(self): ev = self.s.touch_event - self.assertEqual(ev({"fx": 0.5, "fy": 2, "window": 5}), {"fx": 0.5, "fy": 1.0, "window": 5}) + self.assertEqual(ev({"fx": 0.5, "fy": 2, "window": 5, "display": ":1"}), {"fx": 0.5, "fy": 1.0, "window": 5, "display": ":1"}) + self.assertEqual(ev({"button": "left", "window": 5, "display": ":0"}), {"button": "left", "down": True, "window": 5, "display": ":0"}) self.assertEqual(ev({"button": "right"}), {"button": "right", "down": True}) self.assertEqual(ev({"key": 30, "down": False}), {"key": 30, "down": False}) self.assertEqual(ev({"scroll": [0, 1e9]}), {"scroll": [0.0, 5000.0]}) self.assertEqual(ev({"dx": 3, "other": 1}), {"dx": 3.0}) def test_touch_event_rejects_bad_ones(self): - for bad in (None, {}, {"fx": 0.5, "fy": 0.5}, {"fx": "1", "fy": 0, "window": 1}, {"button": "side"}, + for bad in (None, {}, {"fx": 0.5, "fy": 0.5}, {"fx": "1", "fy": 0, "window": 1, "display": ":1"}, {"button": "side"}, + {"fx": 0.5, "fy": 0.5, "window": 1}, {"fx": 0.5, "fy": 0.5, "window": 1, "display": ":1;x"}, + {"window": 1, "display": ":1"}, {"key": 0}, {"key": 999}, {"key": True}, {"scroll": [1]}, {"text": ""}, {"text": "x" * 501}, - {"fx": 0.1, "fy": 0.1, "window": True}): + {"fx": 0.1, "fy": 0.1, "window": True, "display": ":1"}): with self.assertRaises(self.s.Failure, msg=repr(bad)): self.s.touch_event(bad) diff --git a/ui/frame_touch.py b/ui/frame_touch.py index 36c2f43..2dc3a77 100644 --- a/ui/frame_touch.py +++ b/ui/frame_touch.py @@ -10,8 +10,9 @@ of gamescope's X displays (see docs/streaming.md). python3 frame_touch.py focus print the focused panel as JSON python3 frame_touch.py panels print every app panel as JSON, and which has focus python3 frame_touch.py read events on stdin, one JSON object (or list) per line: - {"fx": 0.5, "fy": 0.2, "window": 123} pointer to that fraction of the focused panel - (ignored if another panel has focus since) + {"fx": 0.5, "fy": 0.2, "window": 123, "display": ":1"} + pointer to that fraction of that panel; any event can + name its panel, and goes nowhere if another has focus {"dx": 4, "dy": -2} pointer by that much {"button": "left", "down": true} left, right or middle; "down" false releases {"scroll": [0, 120]} by pixels; positive y scrolls down @@ -88,44 +89,66 @@ def displays(): return sorted(f":{n[1:]}" for n in os.listdir("/tmp/.X11-unix") if n[1:].isdigit()) -def focus(): - """The panel that has focus in the headset: its window, display, name and sizes. +def window_pid(display, window): + try: + out = subprocess.run(["xprop", "-id", str(window), "_NET_WM_PID"], env=dict(os.environ, DISPLAY=display), + capture_output=True, text=True, timeout=5).stdout + except (OSError, subprocess.SubprocessError): + return None + value = out.rsplit("=", 1)[-1].strip() if "=" in out else "" + return int(value) if value.isdigit() else None - gamescope publishes it on :0's root (GAMESCOPE_FOCUSED_WINDOW). The window can be - on any of its Xwayland displays; the pid in GAMESCOPE_FOCUSABLE_WINDOWS (window, - app id, pid triples) tells which one when ids repeat across displays. + +def locate(window, pid): + """The display a focusable window is on, with its name and geometry. + + Window ids are per X server, so :0 and :1 can both have one; the pid gamescope + lists with it (GAMESCOPE_FOCUSABLE_WINDOWS) tells them apart. """ - window = (xprop_root(":0", "GAMESCOPE_FOCUSED_WINDOW") or [0])[0] - if not window: - return {"window": None} - triples = xprop_root(":0", "GAMESCOPE_FOCUSABLE_WINDOWS") - app = next((triples[i + 1] for i in range(0, len(triples) - 2, 3) if triples[i] == window), None) + found = [] for display in displays(): info = window_info(display, window) if info: - root = window_info(display, "root") or {} - return {"window": window, "display": display, "app": app, **info, - "root": [root.get("width", info["width"]), root.get("height", info["height"])]} - return {"window": None} + found.append((display, info)) + if len(found) > 1 and pid: + found = [f for f in found if window_pid(f[0], window) == pid] or found + if not found: + return None + display, info = found[0] + root = window_info(display, "root") or {} + return {"window": window, "display": display, **info, + "root": [root.get("width", info["width"]), root.get("height", info["height"])]} + + +def focusable(): + """gamescope's focusable windows as (window, app id, pid).""" + t = xprop_root(":0", "GAMESCOPE_FOCUSABLE_WINDOWS") + return [tuple(t[i:i + 3]) for i in range(0, len(t) - 2, 3)] + + +def focus(): + """The panel that has focus in the headset: window, display, name and sizes (gamescope + publishes the window on :0's root as GAMESCOPE_FOCUSED_WINDOW).""" + window = (xprop_root(":0", "GAMESCOPE_FOCUSED_WINDOW") or [0])[0] + if not window: + return {"window": None} + app, pid = next(((a, p) for w, a, p in focusable() if w == window), (None, None)) + panel = locate(window, pid) + return {**panel, "app": app} if panel else {"window": None} def panels(): """Every app panel (gamescope's focusable windows), for watching one that hasn't focus.""" - focused = (xprop_root(":0", "GAMESCOPE_FOCUSED_WINDOW") or [0])[0] - triples = xprop_root(":0", "GAMESCOPE_FOCUSABLE_WINDOWS") - found, seen = [], set() - for i in range(0, len(triples) - 2, 3): - window, app = triples[i], triples[i + 1] - if window in seen: - continue - seen.add(window) - for display in displays(): - info = window_info(display, window) - if info and info["width"] > 1 and info["height"] > 1: - found.append({"window": window, "display": display, "app": app, **info, - "focused": window == focused}) - break - return {"focus": focused or None, "panels": found} + now = focus() + found = [] + for window, app, pid in focusable(): + panel = locate(window, pid) + if panel and panel["width"] > 1 and panel["height"] > 1 and \ + not any(f["window"] == window and f["display"] == panel["display"] for f in found): + panel.pop("root", None) + found.append({**panel, "app": app, "focused": (window, panel["display"]) == + (now.get("window"), now.get("display"))}) + return {"focus": now.get("window"), "focus_display": now.get("display"), "panels": found} def to_root(panel, fx, fy): @@ -280,15 +303,27 @@ def number(value, limit=100000.0): return max(-limit, min(limit, float(value))) +def aimed_elsewhere(event, panel): + """Whether an event names a panel that isn't the one with focus now.""" + if "window" not in event: + return False + return (panel.get("window"), panel.get("display")) != (event.get("window"), event.get("display")) + + def apply(gs, event, panel): - """Send one event; returns the panel it was aimed at (refreshed if focus moved).""" - if "fx" in event: - window = event.get("window") - if not panel or panel.get("window") != window or time.time() - panel.get("_at", 0) > 1: - panel = {**focus(), "_at": time.time()} - if panel.get("window") is None or panel["window"] != window: - say("ready", focus=panel.get("window"), stale=True) # the page re-syncs its view - return panel + """Send one event; returns the focused panel it checked against (looked up at most once a second). + + Positions and presses name the panel they were meant for. If focus has moved to + another panel since, they go nowhere, so a tap can't land on the wrong one; + releases always go, so nothing stays held. + """ + if "window" in event and (not panel or time.time() - panel.get("_at", 0) > 1 or aimed_elsewhere(event, panel)): + panel = {**focus(), "_at": time.time()} + stale = aimed_elsewhere(event, panel) if "window" in event else False + if stale and not (event.get("down") is False and ("button" in event or "key" in event)): + say("ready", focus=panel.get("window"), display=panel.get("display"), stale=True) # the page re-syncs + return panel + if "fx" in event and panel and panel.get("window"): gs.move_to(*to_root(panel, number(event["fx"], 1), number(event["fy"], 1))) if "dx" in event or "dy" in event: gs.move_by(number(event.get("dx", 0), 2000), number(event.get("dy", 0), 2000)) diff --git a/ui/index.html b/ui/index.html index 6d1ba5e..6948ef5 100644 --- a/ui/index.html +++ b/ui/index.html @@ -1211,10 +1211,11 @@ async function capture() { let url = null; try { let url_ = "/api/screenshot?view=headset"; + var shotPanel = null; if (view === "flat") { - const p = deskPanel() || (await loadPanels(), deskPanel()); - if (!p) throw new Error("no app panel is open in the headset"); - url_ = `/api/screenshot?view=panel&window=${p.window}&display=${encodeURIComponent(p.display)}`; + shotPanel = deskPanel() || (await loadPanels(), deskPanel()); + if (!shotPanel) throw new Error("no app panel is open in the headset"); + url_ = `/api/screenshot?view=panel&window=${shotPanel.window}&display=${encodeURIComponent(shotPanel.display)}`; } const r = await fetch(url_, { headers: {"X-Frame-UI": UI_KEY} }); if (!r.ok) throw new Error((await r.json().catch(() => ({}))).error || `HTTP ${r.status}`); @@ -1225,7 +1226,9 @@ async function capture() { if (gen !== viewGen) return true; lastImg = img; lastSource = source; lastShot = null; draw(); - if (source === "panel") $("srcBadge").textContent = `Desktop · ${deskPanel()?.name || "panel"}`; + desk.shown = source === "panel" ? panelKey(shotPanel) : null; + if (source === "panel") $("srcBadge").textContent = `Desktop · ${shotPanel?.name || "panel"}`; + ctrlShow(); $("stamp").hidden = false; $("stamp").textContent = new Date().toLocaleTimeString(); $("saveBtn").disabled = false; return true; @@ -1238,19 +1241,22 @@ async function capture() { $("viewer").classList.remove("busy"); } } -let liveFailures = 0; -async function liveLoop() { - if (!live) return; - liveFailures = (await capture()) ? 0 : liveFailures + 1; +let liveFailures = 0, liveGen = 0; +async function liveLoop(gen = liveGen) { + if (!live || gen !== liveGen) return; + const ok = await capture(); + if (gen !== liveGen) return; // Live was restarted (say, as video) while this capture ran + liveFailures = ok ? 0 : liveFailures + 1; if (liveFailures >= 5) { toggleLive(false); toast(view === "flat" ? "Live stopped: couldn't capture the panel." : "Live stopped after 5 failed captures. Is the headset on?", true); return; } - if (live) liveTimer = setTimeout(liveLoop, view === "headset" ? 300 : 700); + if (live) liveTimer = setTimeout(() => liveLoop(gen), view === "headset" ? 300 : 700); } function toggleLive(on) { live = on; + liveGen++; liveFailures = 0; $("liveBtn").classList.toggle("on", live); $("liveBadge").hidden = !live; @@ -1302,6 +1308,7 @@ async function startVideo() { if (!shown++) { viewGen++; // a capture still in flight mustn't replace the video lastImg = null; lastSource = "video"; lastShot = null; + desk.shown = panelKey(panel); ctrlShow(); $("viewer").classList.remove("busy"); c.hidden = false; $("viewerEmpty").hidden = true; $("zoombar").hidden = false; $("asleep").hidden = true; $("srcBadge").hidden = false; $("srcBadge").textContent = `${video.label} · video`; @@ -1317,6 +1324,7 @@ async function startVideo() { error: e => log("Video decoder: " + e.message, "e"), }); const panel = view === "flat" ? deskPanel() : null; + desk.shown = null; // until this stream's first picture video.label = panel ? `Desktop · ${panel.name || "panel"}` : "Headset view"; const query = panel ? `src=panel&window=${panel.window}&display=${encodeURIComponent(panel.display)}&${STREAM_QUERY}` : STREAM_QUERY; const r = await fetch(`/api/stream?${query}`, { headers: {"X-Frame-UI": UI_KEY}, signal: ctl.signal }); @@ -1387,20 +1395,24 @@ document.querySelectorAll("[data-eye]").forEach(b => b.onclick = () => setEye(b. // ---- Desktop view: the app panel in use in the headset (or one picked to watch) ---- // gamescope says which panel has focus: the one the wearer last used. That's the one // Control reaches, so the view follows it unless you pick another to watch. -const desk = { panels: [], focus: null, pick: "", timer: null, loaded: false }; +// Panels are "display/window": window ids are per X display, and the Frame has two. +// desk.shown is the panel whose picture is on screen now; Control only acts on that one. +const desk = { panels: [], focus: null, pick: "", timer: null, loaded: false, shown: null }; +const panelKey = p => p ? `${p.display}/${p.window}` : null; function deskPanel() { - const want = desk.pick ? +desk.pick : desk.focus; + const want = desk.pick || desk.focus; // Nothing in use and nothing picked: show the first panel rather than nothing. - return desk.panels.find(p => p.window === want) || (!desk.pick && !desk.focus && desk.panels[0]) || null; + return desk.panels.find(p => panelKey(p) === want) || (!desk.pick && !desk.focus && desk.panels[0]) || null; } async function loadPanels() { try { - const before = deskPanel()?.window; + const before = panelKey(deskPanel()); const r = await api("/api/panels"); - desk.panels = r.panels || []; desk.focus = r.focus || null; desk.loaded = true; - if (desk.pick && !desk.panels.some(p => p.window === +desk.pick)) desk.pick = ""; + desk.panels = r.panels || []; desk.loaded = true; + desk.focus = r.focus ? `${r.focus_display}/${r.focus}` : null; + if (desk.pick && !desk.panels.some(p => panelKey(p) === desk.pick)) desk.pick = ""; renderPanelPick(); - const now = deskPanel()?.window; + const now = panelKey(deskPanel()); if (view === "flat" && live && now !== before) { toggleLive(false); toggleLive(true); } ctrlShow(); } catch (e) { @@ -1415,7 +1427,7 @@ function watchPanels() { function renderPanelPick() { const sel = $("panelPick"); sel.innerHTML = `` + - desk.panels.map(p => ``).join(""); + desk.panels.map(p => ``).join(""); sel.value = desk.pick; } $("panelPick").onchange = () => { @@ -1429,7 +1441,7 @@ $("panelPick").onchange = () => { // exactly where you tapped; on the headset view, which moves with the wearer's head, // the view is a trackpad for the pointer instead. const ctrl = { on: false, queue: [], sending: false, state: "off", message: "", move: null, rel: [0, 0], raf: 0, - held: new Set(), pointers: new Map(), g: null, retry: null }; + held: new Set(), keys: new Set(), pointers: new Map(), g: null, retry: null }; const EVDEV = { Escape: 1, Minus: 12, Equal: 13, Backspace: 14, Tab: 15, BracketLeft: 26, BracketRight: 27, Enter: 28, ControlLeft: 29, Semicolon: 39, Quote: 40, Backquote: 41, ShiftLeft: 42, Backslash: 43, Comma: 51, Period: 52, Slash: 53, ShiftRight: 54, NumpadMultiply: 55, AltLeft: 56, Space: 57, CapsLock: 58, F11: 87, F12: 88, @@ -1452,7 +1464,9 @@ function ctrlTarget() { if (!desk.loaded) return { why: "Finding the headset's panels…" }; if (!p) return { why: "No app panel is open in the headset." }; if (!desk.focus) return { why: "Nobody's using a panel in the headset yet. Taps work once one is in use." }; - if (p.window !== desk.focus) return { why: "Only the panel in use takes taps. Pick “Panel in use” to control it." }; + if (panelKey(p) !== desk.focus) return { why: "Only the panel in use takes taps. Pick “Panel in use” to control it." }; + if (desk.shown !== desk.focus) return { why: live ? "Waiting for the picture of the panel in use…" + : "The panel in use has changed. Press Capture or Live to see it first." }; return { ok: true, panel: p }; } function ctrlShow() { @@ -1503,10 +1517,29 @@ $("viewer").addEventListener("blur", () => { ctrlRelease(); if (window.frameApp? document.addEventListener("visibilitychange", () => { if (document.hidden) ctrlRelease(); }); const isMoveEvent = e => ("fx" in e || "dx" in e) && !("button" in e) && !("key" in e); +const isRelease = e => ("button" in e || "key" in e) && e.down === false; +// While it connects, keep clicks and keys, each with the last position before it, so a +// tap still lands where it was made; lone moves are stale by then. +function ctrlKeepable(events) { + const out = []; + let move = null; + for (const e of events) { + if (isMoveEvent(e)) { if ("fx" in e) move = e; continue; } + if (move && "button" in e && e.down !== false) out.push(move); + move = null; + out.push(e); + } + if (move) out.push(move); // the last position: a click queued after it needs it + return out; +} function ctrlSend(events) { - ctrl.queue.push(...events); - // Keep order; if the Frame is slow to answer, drop pointer moves first, never a release. - if (ctrl.queue.length > 300) ctrl.queue = ctrl.queue.filter(e => !isMoveEvent(e)).slice(-200); + ctrl.queue.push(...events.map(e => isMoveEvent(e) ? e : ctrlAim(e))); + // Keep order; if the Frame is slow to answer, drop pointer moves first, then the oldest + // presses and scrolls, never a release. + if (ctrl.queue.length > 300) { + const q = ctrlKeepable(ctrl.queue); + ctrl.queue = q.filter((e, i) => isRelease(e) || i >= q.length - 200); + } ctrlFlush(); } async function ctrlFlush() { @@ -1517,20 +1550,23 @@ async function ctrlFlush() { try { const r = await api("/api/touch", { events: batch }); ctrl.state = r.state; ctrl.message = r.message || ""; - if (!r.sent) { - // Not connected yet: keep clicks and keys for when it is (moves are stale by then). - ctrl.queue.unshift(...batch.filter(e => !isMoveEvent(e))); - if (r.state !== "error") again = 300; + if (!r.sent && r.state === "error") { + // Broken: presses would be stale by the time it's back; releases still matter. + ctrl.queue.unshift(...batch.filter(isRelease)); + again = 2000; + } else if (!r.sent) { + ctrl.queue.unshift(...ctrlKeepable(batch)); // connecting: send them once it's ready + again = 300; } } catch (e) { ctrl.state = "error"; ctrl.message = e.message; - ctrl.queue.unshift(...batch.filter(e => "button" in e && e.down === false || "key" in e && e.down === false)); + ctrl.queue.unshift(...batch.filter(isRelease)); again = 1000; // try the releases again in a moment, not in a tight loop } finally { ctrl.sending = false; ctrlShow(); clearTimeout(ctrl.retry); - if (again) ctrl.retry = setTimeout(ctrlFlush, again); + if (again) { if (ctrl.queue.length) ctrl.retry = setTimeout(ctrlFlush, again); } else if (ctrl.queue.length) ctrlFlush(); } } @@ -1538,7 +1574,7 @@ async function ctrlFlush() { function ctrlMoveTo(fx, fy) { const p = ctrlTarget().panel; if (!p) return; - ctrl.move = { fx, fy, window: p.window }; + ctrl.move = { fx, fy, window: p.window, display: p.display }; ctrlSchedule(); } function ctrlMoveBy(dx, dy) { ctrl.rel[0] += dx; ctrl.rel[1] += dy; ctrlSchedule(); } @@ -1550,14 +1586,26 @@ function ctrlFlushMoves() { if (ctrl.rel[0] || ctrl.rel[1]) { out.push({ dx: Math.round(ctrl.rel[0]), dy: Math.round(ctrl.rel[1]) }); ctrl.rel = [0, 0]; } if (out.length) ctrlSend(out); } +// On the Desktop view, presses name the panel they're for, so the Frame drops them if +// focus has moved on (releases always go). +function ctrlAim(e) { + const p = ctrlMode() === "abs" && e.down !== false ? ctrlTarget().panel : null; + return p ? { ...e, window: p.window, display: p.display } : e; +} function ctrlButton(name, down) { ctrlFlushMoves(); // the click lands where the pointer was sent if (down) ctrl.held.add(name); else ctrl.held.delete(name); - ctrlSend([{ button: name, down }]); + ctrlSend([ctrlAim({ button: name, down })]); +} +function ctrlKeyEvent(code, down) { + ctrlFlushMoves(); + if (down) ctrl.keys.add(code); else ctrl.keys.delete(code); + ctrlSend([ctrlAim({ key: code, down })]); } function ctrlClick(name = "left") { ctrlButton(name, true); ctrlButton(name, false); } function ctrlRelease() { for (const b of [...ctrl.held]) ctrlButton(b, false); + for (const k of [...ctrl.keys]) ctrlKeyEvent(k, false); if (ctrl.g && ctrl.g.timer) clearTimeout(ctrl.g.timer); ctrl.pointers.clear(); ctrl.g = null; } @@ -1589,7 +1637,11 @@ function ctrlTouchDown(e) { g.timer = setTimeout(() => { if (ctrl.g !== g || g.kind !== "maybe") return; g.kind = "held"; - if (ctrlMode() === "abs") { const f = ctrlFraction(g.e); if (f) ctrlMoveTo(...f); } + if (ctrlMode() === "abs") { + const f = ctrlFraction(g.e); + if (!f) return; // on the bars around the picture: nothing to press + ctrlMoveTo(...f); + } ctrlClick("right"); navigator.vibrate?.(10); }, HOLD_MS); @@ -1630,6 +1682,7 @@ function ctrlTouchUp(e) { const g = ctrl.g; ctrl.pointers.delete(e.pointerId); if (!g) return; + if (g.kind === "scroll") g.kind = "done"; // the finger left behind doesn't jump the scroll or drag if (g.kind === "maybe" && ctrl.pointers.size === 0) { clearTimeout(g.timer); if (ctrlMode() === "abs") { const f = ctrlFraction(g.e); if (!f) { ctrl.g = null; return; } ctrlMoveTo(...f); } @@ -1637,6 +1690,14 @@ function ctrlTouchUp(e) { } else if (g.kind === "drag") ctrlButton("left", false); if (ctrl.pointers.size === 0) ctrl.g = null; } +// The system took the touch (a swipe home, a call): no tap, and let go of anything held. +function ctrlTouchCancel(e) { + ctrl.pointers.delete(e.pointerId); + if (ctrl.g?.timer) clearTimeout(ctrl.g.timer); + if (ctrl.held.has("left")) ctrlButton("left", false); + if (ctrl.g) ctrl.g.kind = "done"; + if (ctrl.pointers.size === 0) ctrl.g = null; +} const canvasEl = $("canvas"); canvasEl.addEventListener("pointerdown", e => { if (!ctrl.on) return; @@ -1645,6 +1706,7 @@ canvasEl.addEventListener("pointerdown", e => { try { canvasEl.setPointerCapture(e.pointerId); } catch { /* a pointer that's already gone */ } if (!ctrlTarget().ok) return ctrlShow(); if (e.pointerType === "touch") return ctrlTouchDown(e); + if (ctrlMode() === "abs" && !ctrlFraction(e)) return; // on the bars around the picture ctrlMouse(e); ctrlButton(CTRL_BUTTONS[e.button] || "left", true); }); @@ -1655,7 +1717,7 @@ canvasEl.addEventListener("pointermove", e => { }); for (const type of ["pointerup", "pointercancel"]) canvasEl.addEventListener(type, e => { if (!ctrl.on) return; - if (e.pointerType === "touch") return ctrlTouchUp(e); + if (e.pointerType === "touch") return type === "pointercancel" ? ctrlTouchCancel(e) : ctrlTouchUp(e); const name = CTRL_BUTTONS[e.button] || "left"; if (ctrl.held.has(name)) ctrlButton(name, false); }); @@ -1668,19 +1730,21 @@ $("viewer").addEventListener("wheel", e => { }, { passive: false }); // Keys while the view has focus go to the Frame (⌘ as Ctrl on a Mac); click elsewhere to stop. function ctrlKey(e, down) { - if (!ctrl.on || document.activeElement !== $("viewer") || !ctrlTarget().ok) return; + if (!ctrl.on || document.activeElement !== $("viewer") || (down && !ctrlTarget().ok)) return; let code = EVDEV[e.code]; if (IS_MAC && (e.code === "MetaLeft" || e.code === "MetaRight")) code = 29; if (!code) return; e.preventDefault(); - if (down && e.repeat && code >= 29 && [29, 42, 54, 56, 97, 100, 125, 126].includes(code)) return; - ctrlSend([{ key: code, down }]); + e.stopPropagation(); // it's for the Frame, not Frame Control's own shortcuts + if (down && e.repeat && [29, 42, 54, 56, 97, 100, 125, 126].includes(code)) return; + if (!down && !ctrl.keys.has(code)) return; + ctrlKeyEvent(code, down); } $("viewer").addEventListener("keydown", e => ctrlKey(e, true)); $("viewer").addEventListener("keyup", e => ctrlKey(e, false)); // Phone keyboard: the field types plain text; the buttons send single keys and clicks. -function ctrlTap(code) { ctrlSend([{ key: code, down: true }, { key: code, down: false }]); } +function ctrlTap(code) { ctrlKeyEvent(code, true); ctrlKeyEvent(code, false); } let ctrlWarned = false; function ctrlText(text) { const plain = text.replace(/[^\x20-\x7e\n\t]/g, ""); diff --git a/ui/server.py b/ui/server.py index 9d332cf..7eebcd1 100755 --- a/ui/server.py +++ b/ui/server.py @@ -868,11 +868,17 @@ def touch_event(event): return max(-limit, min(limit, round(float(value), 4))) out = {} if "fx" in event or "fy" in event: + if "window" not in event: + raise Failure("a position needs the panel's window and display", 400) out.update(fx=num("fx", 1), fy=num("fy", 1)) - window = event.get("window") + # Any event can name the panel it's meant for; the Frame drops it if another has focus. + if "window" in event: + window, display = event.get("window"), event.get("display") if isinstance(window, bool) or not isinstance(window, int) or window <= 0: raise Failure("window must be the panel's window id", 400) - out["window"] = window + if not isinstance(display, str) or not PANEL_DISPLAY.match(display): + raise Failure("display must be an X display such as :1", 400) + out.update(window=window, display=display) for name in ("dx", "dy"): if name in event: out[name] = num(name, INPUT_MOVE_LIMIT) @@ -895,7 +901,7 @@ def touch_event(event): if not isinstance(text, str) or not 0 < len(text) <= INPUT_TEXT_LIMIT: raise Failure(f"text must be 1 to {INPUT_TEXT_LIMIT} characters", 400) out["text"] = text - if not out: + if not set(out) - {"window", "display"}: raise Failure("touch event has nothing to do", 400) return out