diff --git a/tests/page/ctrl_gestures.mjs b/tests/page/ctrl_gestures.mjs
index 512eeb8..078a019 100644
--- a/tests/page/ctrl_gestures.mjs
+++ b/tests/page/ctrl_gestures.mjs
@@ -9,7 +9,7 @@ const grab = name => {
if (!m) throw new Error("not found: " + name);
return m[0];
};
-const NAMES = ["isMoveEvent", "TAP_MOVE", "ctrlSend", "ctrlFlush", "ctrlMoveTo", "ctrlMoveBy", "ctrlSchedule",
+const NAMES = ["isMoveEvent", "isRelease", "ctrlKeepable", "TAP_MOVE", "ctrlAim", "ctrlKeyEvent", "ctrlTouchCancel", "ctrlSend", "ctrlFlush", "ctrlMoveTo", "ctrlMoveBy", "ctrlSchedule",
"ctrlFlushMoves", "ctrlButton", "ctrlClick", "ctrlRelease", "ctrlFraction", "ctrlTouchDown", "centroid",
"ctrlTouchMove", "ctrlTouchUp", "ctrlTap", "ctrlText"];
const code = NAMES.map(grab).join("");
@@ -18,18 +18,18 @@ const tick = (ms = 0) => new Promise(r => setTimeout(r, ms));
function page({ mode = "abs", rect = { left: 0, top: 0, width: 640, height: 360 }, api } = {}) {
const ctrl = { on: true, queue: [], sending: false, state: "ready", message: "", move: null, rel: [0, 0], raf: 0,
- held: new Set(), pointers: new Map(), g: null, retry: null };
+ held: new Set(), keys: new Set(), pointers: new Map(), g: null, retry: null };
const sent = [];
const canvas = { width: 1280, height: 720, getBoundingClientRect: () => rect };
const env = {
ctrl, $: () => canvas, ctrlMode: () => mode, ctrlShow: () => {}, toast: () => {},
- ctrlTarget: () => (mode === "abs" ? { ok: true, panel: { window: 42 } } : { ok: true }),
+ ctrlTarget: () => (mode === "abs" ? { ok: true, panel: { window: 42, display: ":1" } } : { ok: true }),
api: api || (async (path, body) => { sent.push(...body.events); return { state: "ready", sent: true }; }),
requestAnimationFrame: cb => { setTimeout(cb, 0); return 1; },
navigator: {},
};
const fns = new Function(...Object.keys(env), `let ctrlWarned = false;\n${code}
- return { ctrlTouchDown, ctrlTouchMove, ctrlTouchUp, ctrlSend, ctrlText, ctrlButton, ctrlRelease, ctrlFlushMoves };`)(...Object.values(env));
+ return { ctrlTouchDown, ctrlTouchMove, ctrlTouchUp, ctrlTouchCancel, ctrlSend, ctrlText, ctrlButton, ctrlKeyEvent, ctrlRelease, ctrlFlushMoves };`)(...Object.values(env));
const at = (id, x, y) => ({ pointerId: id, clientX: x, clientY: y });
return { ctrl, sent, ...fns, at };
}
@@ -40,7 +40,8 @@ function page({ mode = "abs", rect = { left: 0, top: 0, width: 640, height: 360
p.ctrlTouchDown(p.at(1, 320, 90)); p.ctrlTouchUp(p.at(1, 320, 90));
await tick(10);
const [move, down, up] = p.sent;
- if (!(move.fx === 0.5 && Math.abs(move.fy - 0.25) < 1e-9 && move.window === 42)) fail("tap position " + JSON.stringify(move));
+ if (!(move.fx === 0.5 && Math.abs(move.fy - 0.25) < 1e-9 && move.window === 42 && move.display === ":1")) fail("tap position " + JSON.stringify(move));
+ if (!(down.window === 42 && down.display === ":1")) fail("a press names its panel " + JSON.stringify(down));
if (!(down.button === "left" && down.down && up.button === "left" && up.down === false && p.sent.length === 3))
fail("tap click " + JSON.stringify(p.sent));
}
@@ -105,14 +106,14 @@ function page({ mode = "abs", rect = { left: 0, top: 0, width: 640, height: 360
const text = p.sent.map(e => e.text).join("");
if (!(text === "hllo " + "x".repeat(600) && p.sent.every(e => e.text.length <= 500))) fail("text " + text.length);
}
-// Not connected yet, or the request fails: clicks and keys wait, stale moves don't; a release is never lost.
+// Not connected yet: clicks and keys wait with their position. The request fails: only releases wait.
{
let calls = 0;
const p = page({ api: async () => { calls++; if (calls === 1) return { state: "starting", sent: false }; throw new Error("offline"); } });
p.ctrlSend([{ fx: 0.1, fy: 0.1, window: 42 }, { button: "left", down: true }, { key: 30, down: true }]);
await tick(5);
const q = p.ctrl.queue;
- if (!(q.length === 2 && q[0].button === "left" && q[1].key === 30)) fail("kept while starting " + JSON.stringify(q));
+ if (!(q.length === 3 && "fx" in q[0] && q[1].button === "left" && q[2].key === 30)) fail("kept while starting " + JSON.stringify(q));
clearTimeout(p.ctrl.retry);
p.ctrl.queue = [];
p.ctrlSend([{ button: "left", down: false }, { key: 31, down: true }, { dx: 3, dy: 1 }]);
@@ -129,4 +130,72 @@ function page({ mode = "abs", rect = { left: 0, top: 0, width: 640, height: 360
p.ctrlRelease(); await tick(10);
if (!(p.sent.at(-1).button === "left" && p.sent.at(-1).down === false && !p.ctrl.held.size)) fail("release " + JSON.stringify(p.sent));
}
+// Keys held on the Frame are let go too.
+{
+ const p = page();
+ p.ctrlKeyEvent(42, true);
+ p.ctrlRelease(); await tick(10);
+ if (!(p.sent.at(-1).key === 42 && p.sent.at(-1).down === false && !p.ctrl.keys.size)) fail("key release " + JSON.stringify(p.sent));
+}
+// A long queue sheds moves and old scrolls, never a release.
+{
+ const p = page({ api: () => new Promise(() => {}) }); // stuck request
+ p.ctrlSend([{ dx: 1, dy: 1 }]);
+ p.ctrlSend([{ button: "left", down: false }]);
+ for (let i = 0; i < 320; i++) p.ctrlSend([{ scroll: [0, 1] }]);
+ if (!p.ctrl.queue.some(e => e.button === "left" && e.down === false)) fail("trim dropped a release");
+ if (p.ctrl.queue.length > 300) fail("trim kept " + p.ctrl.queue.length);
+}
+// Broken on the Frame side: no hammering, and only releases wait.
+{
+ let calls = 0;
+ const p = page({ api: async () => { calls++; return { state: "error", sent: false }; } });
+ p.ctrlSend([{ button: "left", down: true }, { button: "left", down: false }]);
+ await tick(50);
+ if (calls !== 1) fail("error response reposted " + calls + " times");
+ if (!(p.ctrl.queue.length === 1 && p.ctrl.queue[0].down === false)) fail("error kept " + JSON.stringify(p.ctrl.queue));
+ clearTimeout(p.ctrl.retry);
+}
+// Connecting: a tap keeps its position, so it lands where it was made.
+{
+ const p = page({ api: async () => ({ state: "starting", sent: false }) });
+ p.ctrlTouchDown(p.at(1, 320, 90)); p.ctrlTouchUp(p.at(1, 320, 90));
+ await tick(10);
+ const q = p.ctrl.queue;
+ if (!("fx" in q[0] && q[1].button === "left")) fail("connecting tap " + JSON.stringify(q));
+ clearTimeout(p.ctrl.retry);
+}
+// Lifting one of two scrolling fingers doesn't jump the scroll or start a drag.
+{
+ const p = page();
+ p.ctrlTouchDown(p.at(1, 100, 200)); p.ctrlTouchDown(p.at(2, 300, 200));
+ p.ctrlTouchUp(p.at(1, 100, 200));
+ p.ctrlTouchMove(p.at(2, 300, 199));
+ p.ctrlTouchUp(p.at(2, 300, 199)); await tick(10);
+ if (p.sent.length) fail("one finger left after scrolling " + JSON.stringify(p.sent));
+}
+// A cancelled touch isn't a tap.
+{
+ const p = page();
+ p.ctrlTouchDown(p.at(1, 100, 100)); p.ctrlTouchCancel(p.at(1, 100, 100)); await tick(10);
+ if (p.sent.length) fail("cancel clicked " + JSON.stringify(p.sent));
+}
+// Press and hold on the bars around the picture does nothing.
+{
+ const p = page({ rect: { left: 0, top: 0, width: 640, height: 480 } });
+ p.ctrlTouchDown(p.at(1, 320, 10)); await tick(620); p.ctrlTouchUp(p.at(1, 320, 10)); await tick(10);
+ if (p.sent.length) fail("hold on the bars " + JSON.stringify(p.sent));
+}
+// Taps only reach the panel in use, and only once its picture is the one on screen.
+{
+ const fns = ["panelKey", "deskPanel", "ctrlTarget"].map(grab).join("");
+ const check = (desk, live = true) => new Function("desk", "live", "ctrlMode", fns + "; return ctrlTarget();")(desk, live, () => "abs");
+ const a = { display: ":1", window: 5 }, b = { display: ":0", window: 5 };
+ const base = { panels: [a, b], loaded: true, pick: "", focus: ":1/5", shown: ":1/5" };
+ if (!check(base).ok) fail("target: shown and in use");
+ if (check({ ...base, shown: ":0/5" }).ok) fail("target: the picture is another panel with the same id");
+ if (!/Capture or Live/.test(check({ ...base, shown: null }, false).why)) fail("target: stale picture message");
+ if (check({ ...base, pick: ":0/5", shown: ":0/5" }).ok) fail("target: a watched panel that isn't in use");
+ if (check({ ...base, focus: null }).ok) fail("target: nothing in use");
+}
console.log("control gestures ok");
diff --git a/tests/test_touch.py b/tests/test_touch.py
index 507cdd6..686cbf7 100644
--- a/tests/test_touch.py
+++ b/tests/test_touch.py
@@ -38,6 +38,18 @@ class Mapping(unittest.TestCase):
self.assertAlmostEqual(x, 280, delta=0.5)
self.assertAlmostEqual(y, 359.5, delta=0.5)
+ def test_same_id_on_both_displays_is_told_apart_by_pid(self):
+ t = self.t
+ saved = t.displays, t.window_info, t.window_pid
+ self.addCleanup(lambda: (setattr(t, "displays", saved[0]), setattr(t, "window_info", saved[1]),
+ setattr(t, "window_pid", saved[2])))
+ t.displays = lambda: [":0", ":1"]
+ t.window_info = lambda d, w: {"name": f"on {d}", "width": 1280 if w != "root" else 1920, "height": 720}
+ t.window_pid = lambda d, w: {":0": 111, ":1": 222}[d]
+ self.assertEqual(t.locate(5, 222)["display"], ":1")
+ self.assertEqual(t.locate(5, 111)["display"], ":0")
+ self.assertEqual(t.locate(5, None)["display"], ":0")
+
def test_ascii_table_covers_printable_characters(self):
for code in range(0x20, 0x7F):
self.assertIn(chr(code), self.t.ASCII, chr(code))
@@ -77,16 +89,25 @@ class Apply(unittest.TestCase):
def test_tap_moves_then_clicks_in_order(self):
gs, panel = FakeGamescope(), None
- for e in ({"fx": 0.5, "fy": 0.5, "window": 7}, {"button": "left", "down": True}, {"button": "left", "down": False}):
+ for e in ({"fx": 0.5, "fy": 0.5, "window": 7, "display": ":1"}, {"button": "left", "down": True, "window": 7, "display": ":1"},
+ {"button": "left", "down": False}):
panel = self.t.apply(gs, e, panel)
self.assertEqual([c[0] for c in gs.calls], ["move_to", "button", "button"])
self.assertEqual(gs.calls[1][1:], ("left", True))
def test_a_tap_meant_for_another_panel_goes_nowhere(self):
+ # Focus moved on: the position and the press are dropped; the release still goes.
+ gs, panel = FakeGamescope(), None
+ for e in ({"fx": 0.5, "fy": 0.5, "window": 99, "display": ":1"}, {"button": "left", "down": True, "window": 99, "display": ":1"},
+ {"key": 30, "down": True, "window": 7, "display": ":0"}, {"button": "left", "down": False, "window": 99, "display": ":1"}):
+ panel = self.t.apply(gs, e, panel)
+ self.assertEqual(gs.calls, [("button", "left", False)])
+ self.assertEqual(self.said[0], ("ready", {"focus": 7, "display": ":1", "stale": True}))
+
+ def test_same_window_id_on_the_other_display_is_another_panel(self):
gs = FakeGamescope()
- self.t.apply(gs, {"fx": 0.5, "fy": 0.5, "window": 99}, None)
+ self.t.apply(gs, {"fx": 0.5, "fy": 0.5, "window": 7, "display": ":0"}, None)
self.assertEqual(gs.calls, [])
- self.assertEqual(self.said, [("ready", {"focus": 7, "stale": True})])
def test_relative_scroll_keys_text(self):
gs = FakeGamescope()
@@ -115,16 +136,19 @@ class ServerChecks(unittest.TestCase):
def test_touch_event_keeps_known_fields(self):
ev = self.s.touch_event
- self.assertEqual(ev({"fx": 0.5, "fy": 2, "window": 5}), {"fx": 0.5, "fy": 1.0, "window": 5})
+ self.assertEqual(ev({"fx": 0.5, "fy": 2, "window": 5, "display": ":1"}), {"fx": 0.5, "fy": 1.0, "window": 5, "display": ":1"})
+ self.assertEqual(ev({"button": "left", "window": 5, "display": ":0"}), {"button": "left", "down": True, "window": 5, "display": ":0"})
self.assertEqual(ev({"button": "right"}), {"button": "right", "down": True})
self.assertEqual(ev({"key": 30, "down": False}), {"key": 30, "down": False})
self.assertEqual(ev({"scroll": [0, 1e9]}), {"scroll": [0.0, 5000.0]})
self.assertEqual(ev({"dx": 3, "other": 1}), {"dx": 3.0})
def test_touch_event_rejects_bad_ones(self):
- for bad in (None, {}, {"fx": 0.5, "fy": 0.5}, {"fx": "1", "fy": 0, "window": 1}, {"button": "side"},
+ for bad in (None, {}, {"fx": 0.5, "fy": 0.5}, {"fx": "1", "fy": 0, "window": 1, "display": ":1"}, {"button": "side"},
+ {"fx": 0.5, "fy": 0.5, "window": 1}, {"fx": 0.5, "fy": 0.5, "window": 1, "display": ":1;x"},
+ {"window": 1, "display": ":1"},
{"key": 0}, {"key": 999}, {"key": True}, {"scroll": [1]}, {"text": ""}, {"text": "x" * 501},
- {"fx": 0.1, "fy": 0.1, "window": True}):
+ {"fx": 0.1, "fy": 0.1, "window": True, "display": ":1"}):
with self.assertRaises(self.s.Failure, msg=repr(bad)):
self.s.touch_event(bad)
diff --git a/ui/frame_touch.py b/ui/frame_touch.py
index 36c2f43..2dc3a77 100644
--- a/ui/frame_touch.py
+++ b/ui/frame_touch.py
@@ -10,8 +10,9 @@ of gamescope's X displays (see docs/streaming.md).
python3 frame_touch.py focus print the focused panel as JSON
python3 frame_touch.py panels print every app panel as JSON, and which has focus
python3 frame_touch.py read events on stdin, one JSON object (or list) per line:
- {"fx": 0.5, "fy": 0.2, "window": 123} pointer to that fraction of the focused panel
- (ignored if another panel has focus since)
+ {"fx": 0.5, "fy": 0.2, "window": 123, "display": ":1"}
+ pointer to that fraction of that panel; any event can
+ name its panel, and goes nowhere if another has focus
{"dx": 4, "dy": -2} pointer by that much
{"button": "left", "down": true} left, right or middle; "down" false releases
{"scroll": [0, 120]} by pixels; positive y scrolls down
@@ -88,44 +89,66 @@ def displays():
return sorted(f":{n[1:]}" for n in os.listdir("/tmp/.X11-unix") if n[1:].isdigit())
-def focus():
- """The panel that has focus in the headset: its window, display, name and sizes.
+def window_pid(display, window):
+ try:
+ out = subprocess.run(["xprop", "-id", str(window), "_NET_WM_PID"], env=dict(os.environ, DISPLAY=display),
+ capture_output=True, text=True, timeout=5).stdout
+ except (OSError, subprocess.SubprocessError):
+ return None
+ value = out.rsplit("=", 1)[-1].strip() if "=" in out else ""
+ return int(value) if value.isdigit() else None
- gamescope publishes it on :0's root (GAMESCOPE_FOCUSED_WINDOW). The window can be
- on any of its Xwayland displays; the pid in GAMESCOPE_FOCUSABLE_WINDOWS (window,
- app id, pid triples) tells which one when ids repeat across displays.
+
+def locate(window, pid):
+ """The display a focusable window is on, with its name and geometry.
+
+ Window ids are per X server, so :0 and :1 can both have one; the pid gamescope
+ lists with it (GAMESCOPE_FOCUSABLE_WINDOWS) tells them apart.
"""
- window = (xprop_root(":0", "GAMESCOPE_FOCUSED_WINDOW") or [0])[0]
- if not window:
- return {"window": None}
- triples = xprop_root(":0", "GAMESCOPE_FOCUSABLE_WINDOWS")
- app = next((triples[i + 1] for i in range(0, len(triples) - 2, 3) if triples[i] == window), None)
+ found = []
for display in displays():
info = window_info(display, window)
if info:
- root = window_info(display, "root") or {}
- return {"window": window, "display": display, "app": app, **info,
- "root": [root.get("width", info["width"]), root.get("height", info["height"])]}
- return {"window": None}
+ found.append((display, info))
+ if len(found) > 1 and pid:
+ found = [f for f in found if window_pid(f[0], window) == pid] or found
+ if not found:
+ return None
+ display, info = found[0]
+ root = window_info(display, "root") or {}
+ return {"window": window, "display": display, **info,
+ "root": [root.get("width", info["width"]), root.get("height", info["height"])]}
+
+
+def focusable():
+ """gamescope's focusable windows as (window, app id, pid)."""
+ t = xprop_root(":0", "GAMESCOPE_FOCUSABLE_WINDOWS")
+ return [tuple(t[i:i + 3]) for i in range(0, len(t) - 2, 3)]
+
+
+def focus():
+ """The panel that has focus in the headset: window, display, name and sizes (gamescope
+ publishes the window on :0's root as GAMESCOPE_FOCUSED_WINDOW)."""
+ window = (xprop_root(":0", "GAMESCOPE_FOCUSED_WINDOW") or [0])[0]
+ if not window:
+ return {"window": None}
+ app, pid = next(((a, p) for w, a, p in focusable() if w == window), (None, None))
+ panel = locate(window, pid)
+ return {**panel, "app": app} if panel else {"window": None}
def panels():
"""Every app panel (gamescope's focusable windows), for watching one that hasn't focus."""
- focused = (xprop_root(":0", "GAMESCOPE_FOCUSED_WINDOW") or [0])[0]
- triples = xprop_root(":0", "GAMESCOPE_FOCUSABLE_WINDOWS")
- found, seen = [], set()
- for i in range(0, len(triples) - 2, 3):
- window, app = triples[i], triples[i + 1]
- if window in seen:
- continue
- seen.add(window)
- for display in displays():
- info = window_info(display, window)
- if info and info["width"] > 1 and info["height"] > 1:
- found.append({"window": window, "display": display, "app": app, **info,
- "focused": window == focused})
- break
- return {"focus": focused or None, "panels": found}
+ now = focus()
+ found = []
+ for window, app, pid in focusable():
+ panel = locate(window, pid)
+ if panel and panel["width"] > 1 and panel["height"] > 1 and \
+ not any(f["window"] == window and f["display"] == panel["display"] for f in found):
+ panel.pop("root", None)
+ found.append({**panel, "app": app, "focused": (window, panel["display"]) ==
+ (now.get("window"), now.get("display"))})
+ return {"focus": now.get("window"), "focus_display": now.get("display"), "panels": found}
def to_root(panel, fx, fy):
@@ -280,15 +303,27 @@ def number(value, limit=100000.0):
return max(-limit, min(limit, float(value)))
+def aimed_elsewhere(event, panel):
+ """Whether an event names a panel that isn't the one with focus now."""
+ if "window" not in event:
+ return False
+ return (panel.get("window"), panel.get("display")) != (event.get("window"), event.get("display"))
+
+
def apply(gs, event, panel):
- """Send one event; returns the panel it was aimed at (refreshed if focus moved)."""
- if "fx" in event:
- window = event.get("window")
- if not panel or panel.get("window") != window or time.time() - panel.get("_at", 0) > 1:
- panel = {**focus(), "_at": time.time()}
- if panel.get("window") is None or panel["window"] != window:
- say("ready", focus=panel.get("window"), stale=True) # the page re-syncs its view
- return panel
+ """Send one event; returns the focused panel it checked against (looked up at most once a second).
+
+ Positions and presses name the panel they were meant for. If focus has moved to
+ another panel since, they go nowhere, so a tap can't land on the wrong one;
+ releases always go, so nothing stays held.
+ """
+ if "window" in event and (not panel or time.time() - panel.get("_at", 0) > 1 or aimed_elsewhere(event, panel)):
+ panel = {**focus(), "_at": time.time()}
+ stale = aimed_elsewhere(event, panel) if "window" in event else False
+ if stale and not (event.get("down") is False and ("button" in event or "key" in event)):
+ say("ready", focus=panel.get("window"), display=panel.get("display"), stale=True) # the page re-syncs
+ return panel
+ if "fx" in event and panel and panel.get("window"):
gs.move_to(*to_root(panel, number(event["fx"], 1), number(event["fy"], 1)))
if "dx" in event or "dy" in event:
gs.move_by(number(event.get("dx", 0), 2000), number(event.get("dy", 0), 2000))
diff --git a/ui/index.html b/ui/index.html
index 6d1ba5e..6948ef5 100644
--- a/ui/index.html
+++ b/ui/index.html
@@ -1211,10 +1211,11 @@ async function capture() {
let url = null;
try {
let url_ = "/api/screenshot?view=headset";
+ var shotPanel = null;
if (view === "flat") {
- const p = deskPanel() || (await loadPanels(), deskPanel());
- if (!p) throw new Error("no app panel is open in the headset");
- url_ = `/api/screenshot?view=panel&window=${p.window}&display=${encodeURIComponent(p.display)}`;
+ shotPanel = deskPanel() || (await loadPanels(), deskPanel());
+ if (!shotPanel) throw new Error("no app panel is open in the headset");
+ url_ = `/api/screenshot?view=panel&window=${shotPanel.window}&display=${encodeURIComponent(shotPanel.display)}`;
}
const r = await fetch(url_, { headers: {"X-Frame-UI": UI_KEY} });
if (!r.ok) throw new Error((await r.json().catch(() => ({}))).error || `HTTP ${r.status}`);
@@ -1225,7 +1226,9 @@ async function capture() {
if (gen !== viewGen) return true;
lastImg = img; lastSource = source; lastShot = null;
draw();
- if (source === "panel") $("srcBadge").textContent = `Desktop · ${deskPanel()?.name || "panel"}`;
+ desk.shown = source === "panel" ? panelKey(shotPanel) : null;
+ if (source === "panel") $("srcBadge").textContent = `Desktop · ${shotPanel?.name || "panel"}`;
+ ctrlShow();
$("stamp").hidden = false; $("stamp").textContent = new Date().toLocaleTimeString();
$("saveBtn").disabled = false;
return true;
@@ -1238,19 +1241,22 @@ async function capture() {
$("viewer").classList.remove("busy");
}
}
-let liveFailures = 0;
-async function liveLoop() {
- if (!live) return;
- liveFailures = (await capture()) ? 0 : liveFailures + 1;
+let liveFailures = 0, liveGen = 0;
+async function liveLoop(gen = liveGen) {
+ if (!live || gen !== liveGen) return;
+ const ok = await capture();
+ if (gen !== liveGen) return; // Live was restarted (say, as video) while this capture ran
+ liveFailures = ok ? 0 : liveFailures + 1;
if (liveFailures >= 5) {
toggleLive(false);
toast(view === "flat" ? "Live stopped: couldn't capture the panel." : "Live stopped after 5 failed captures. Is the headset on?", true);
return;
}
- if (live) liveTimer = setTimeout(liveLoop, view === "headset" ? 300 : 700);
+ if (live) liveTimer = setTimeout(() => liveLoop(gen), view === "headset" ? 300 : 700);
}
function toggleLive(on) {
live = on;
+ liveGen++;
liveFailures = 0;
$("liveBtn").classList.toggle("on", live);
$("liveBadge").hidden = !live;
@@ -1302,6 +1308,7 @@ async function startVideo() {
if (!shown++) {
viewGen++; // a capture still in flight mustn't replace the video
lastImg = null; lastSource = "video"; lastShot = null;
+ desk.shown = panelKey(panel); ctrlShow();
$("viewer").classList.remove("busy");
c.hidden = false; $("viewerEmpty").hidden = true; $("zoombar").hidden = false; $("asleep").hidden = true;
$("srcBadge").hidden = false; $("srcBadge").textContent = `${video.label} · video`;
@@ -1317,6 +1324,7 @@ async function startVideo() {
error: e => log("Video decoder: " + e.message, "e"),
});
const panel = view === "flat" ? deskPanel() : null;
+ desk.shown = null; // until this stream's first picture
video.label = panel ? `Desktop · ${panel.name || "panel"}` : "Headset view";
const query = panel ? `src=panel&window=${panel.window}&display=${encodeURIComponent(panel.display)}&${STREAM_QUERY}` : STREAM_QUERY;
const r = await fetch(`/api/stream?${query}`, { headers: {"X-Frame-UI": UI_KEY}, signal: ctl.signal });
@@ -1387,20 +1395,24 @@ document.querySelectorAll("[data-eye]").forEach(b => b.onclick = () => setEye(b.
// ---- Desktop view: the app panel in use in the headset (or one picked to watch) ----
// gamescope says which panel has focus: the one the wearer last used. That's the one
// Control reaches, so the view follows it unless you pick another to watch.
-const desk = { panels: [], focus: null, pick: "", timer: null, loaded: false };
+// Panels are "display/window": window ids are per X display, and the Frame has two.
+// desk.shown is the panel whose picture is on screen now; Control only acts on that one.
+const desk = { panels: [], focus: null, pick: "", timer: null, loaded: false, shown: null };
+const panelKey = p => p ? `${p.display}/${p.window}` : null;
function deskPanel() {
- const want = desk.pick ? +desk.pick : desk.focus;
+ const want = desk.pick || desk.focus;
// Nothing in use and nothing picked: show the first panel rather than nothing.
- return desk.panels.find(p => p.window === want) || (!desk.pick && !desk.focus && desk.panels[0]) || null;
+ return desk.panels.find(p => panelKey(p) === want) || (!desk.pick && !desk.focus && desk.panels[0]) || null;
}
async function loadPanels() {
try {
- const before = deskPanel()?.window;
+ const before = panelKey(deskPanel());
const r = await api("/api/panels");
- desk.panels = r.panels || []; desk.focus = r.focus || null; desk.loaded = true;
- if (desk.pick && !desk.panels.some(p => p.window === +desk.pick)) desk.pick = "";
+ desk.panels = r.panels || []; desk.loaded = true;
+ desk.focus = r.focus ? `${r.focus_display}/${r.focus}` : null;
+ if (desk.pick && !desk.panels.some(p => panelKey(p) === desk.pick)) desk.pick = "";
renderPanelPick();
- const now = deskPanel()?.window;
+ const now = panelKey(deskPanel());
if (view === "flat" && live && now !== before) { toggleLive(false); toggleLive(true); }
ctrlShow();
} catch (e) {
@@ -1415,7 +1427,7 @@ function watchPanels() {
function renderPanelPick() {
const sel = $("panelPick");
sel.innerHTML = `` +
- desk.panels.map(p => ``).join("");
+ desk.panels.map(p => ``).join("");
sel.value = desk.pick;
}
$("panelPick").onchange = () => {
@@ -1429,7 +1441,7 @@ $("panelPick").onchange = () => {
// exactly where you tapped; on the headset view, which moves with the wearer's head,
// the view is a trackpad for the pointer instead.
const ctrl = { on: false, queue: [], sending: false, state: "off", message: "", move: null, rel: [0, 0], raf: 0,
- held: new Set(), pointers: new Map(), g: null, retry: null };
+ held: new Set(), keys: new Set(), pointers: new Map(), g: null, retry: null };
const EVDEV = { Escape: 1, Minus: 12, Equal: 13, Backspace: 14, Tab: 15, BracketLeft: 26, BracketRight: 27, Enter: 28,
ControlLeft: 29, Semicolon: 39, Quote: 40, Backquote: 41, ShiftLeft: 42, Backslash: 43, Comma: 51, Period: 52,
Slash: 53, ShiftRight: 54, NumpadMultiply: 55, AltLeft: 56, Space: 57, CapsLock: 58, F11: 87, F12: 88,
@@ -1452,7 +1464,9 @@ function ctrlTarget() {
if (!desk.loaded) return { why: "Finding the headset's panels…" };
if (!p) return { why: "No app panel is open in the headset." };
if (!desk.focus) return { why: "Nobody's using a panel in the headset yet. Taps work once one is in use." };
- if (p.window !== desk.focus) return { why: "Only the panel in use takes taps. Pick “Panel in use” to control it." };
+ if (panelKey(p) !== desk.focus) return { why: "Only the panel in use takes taps. Pick “Panel in use” to control it." };
+ if (desk.shown !== desk.focus) return { why: live ? "Waiting for the picture of the panel in use…"
+ : "The panel in use has changed. Press Capture or Live to see it first." };
return { ok: true, panel: p };
}
function ctrlShow() {
@@ -1503,10 +1517,29 @@ $("viewer").addEventListener("blur", () => { ctrlRelease(); if (window.frameApp?
document.addEventListener("visibilitychange", () => { if (document.hidden) ctrlRelease(); });
const isMoveEvent = e => ("fx" in e || "dx" in e) && !("button" in e) && !("key" in e);
+const isRelease = e => ("button" in e || "key" in e) && e.down === false;
+// While it connects, keep clicks and keys, each with the last position before it, so a
+// tap still lands where it was made; lone moves are stale by then.
+function ctrlKeepable(events) {
+ const out = [];
+ let move = null;
+ for (const e of events) {
+ if (isMoveEvent(e)) { if ("fx" in e) move = e; continue; }
+ if (move && "button" in e && e.down !== false) out.push(move);
+ move = null;
+ out.push(e);
+ }
+ if (move) out.push(move); // the last position: a click queued after it needs it
+ return out;
+}
function ctrlSend(events) {
- ctrl.queue.push(...events);
- // Keep order; if the Frame is slow to answer, drop pointer moves first, never a release.
- if (ctrl.queue.length > 300) ctrl.queue = ctrl.queue.filter(e => !isMoveEvent(e)).slice(-200);
+ ctrl.queue.push(...events.map(e => isMoveEvent(e) ? e : ctrlAim(e)));
+ // Keep order; if the Frame is slow to answer, drop pointer moves first, then the oldest
+ // presses and scrolls, never a release.
+ if (ctrl.queue.length > 300) {
+ const q = ctrlKeepable(ctrl.queue);
+ ctrl.queue = q.filter((e, i) => isRelease(e) || i >= q.length - 200);
+ }
ctrlFlush();
}
async function ctrlFlush() {
@@ -1517,20 +1550,23 @@ async function ctrlFlush() {
try {
const r = await api("/api/touch", { events: batch });
ctrl.state = r.state; ctrl.message = r.message || "";
- if (!r.sent) {
- // Not connected yet: keep clicks and keys for when it is (moves are stale by then).
- ctrl.queue.unshift(...batch.filter(e => !isMoveEvent(e)));
- if (r.state !== "error") again = 300;
+ if (!r.sent && r.state === "error") {
+ // Broken: presses would be stale by the time it's back; releases still matter.
+ ctrl.queue.unshift(...batch.filter(isRelease));
+ again = 2000;
+ } else if (!r.sent) {
+ ctrl.queue.unshift(...ctrlKeepable(batch)); // connecting: send them once it's ready
+ again = 300;
}
} catch (e) {
ctrl.state = "error"; ctrl.message = e.message;
- ctrl.queue.unshift(...batch.filter(e => "button" in e && e.down === false || "key" in e && e.down === false));
+ ctrl.queue.unshift(...batch.filter(isRelease));
again = 1000; // try the releases again in a moment, not in a tight loop
} finally {
ctrl.sending = false;
ctrlShow();
clearTimeout(ctrl.retry);
- if (again) ctrl.retry = setTimeout(ctrlFlush, again);
+ if (again) { if (ctrl.queue.length) ctrl.retry = setTimeout(ctrlFlush, again); }
else if (ctrl.queue.length) ctrlFlush();
}
}
@@ -1538,7 +1574,7 @@ async function ctrlFlush() {
function ctrlMoveTo(fx, fy) {
const p = ctrlTarget().panel;
if (!p) return;
- ctrl.move = { fx, fy, window: p.window };
+ ctrl.move = { fx, fy, window: p.window, display: p.display };
ctrlSchedule();
}
function ctrlMoveBy(dx, dy) { ctrl.rel[0] += dx; ctrl.rel[1] += dy; ctrlSchedule(); }
@@ -1550,14 +1586,26 @@ function ctrlFlushMoves() {
if (ctrl.rel[0] || ctrl.rel[1]) { out.push({ dx: Math.round(ctrl.rel[0]), dy: Math.round(ctrl.rel[1]) }); ctrl.rel = [0, 0]; }
if (out.length) ctrlSend(out);
}
+// On the Desktop view, presses name the panel they're for, so the Frame drops them if
+// focus has moved on (releases always go).
+function ctrlAim(e) {
+ const p = ctrlMode() === "abs" && e.down !== false ? ctrlTarget().panel : null;
+ return p ? { ...e, window: p.window, display: p.display } : e;
+}
function ctrlButton(name, down) {
ctrlFlushMoves(); // the click lands where the pointer was sent
if (down) ctrl.held.add(name); else ctrl.held.delete(name);
- ctrlSend([{ button: name, down }]);
+ ctrlSend([ctrlAim({ button: name, down })]);
+}
+function ctrlKeyEvent(code, down) {
+ ctrlFlushMoves();
+ if (down) ctrl.keys.add(code); else ctrl.keys.delete(code);
+ ctrlSend([ctrlAim({ key: code, down })]);
}
function ctrlClick(name = "left") { ctrlButton(name, true); ctrlButton(name, false); }
function ctrlRelease() {
for (const b of [...ctrl.held]) ctrlButton(b, false);
+ for (const k of [...ctrl.keys]) ctrlKeyEvent(k, false);
if (ctrl.g && ctrl.g.timer) clearTimeout(ctrl.g.timer);
ctrl.pointers.clear(); ctrl.g = null;
}
@@ -1589,7 +1637,11 @@ function ctrlTouchDown(e) {
g.timer = setTimeout(() => {
if (ctrl.g !== g || g.kind !== "maybe") return;
g.kind = "held";
- if (ctrlMode() === "abs") { const f = ctrlFraction(g.e); if (f) ctrlMoveTo(...f); }
+ if (ctrlMode() === "abs") {
+ const f = ctrlFraction(g.e);
+ if (!f) return; // on the bars around the picture: nothing to press
+ ctrlMoveTo(...f);
+ }
ctrlClick("right");
navigator.vibrate?.(10);
}, HOLD_MS);
@@ -1630,6 +1682,7 @@ function ctrlTouchUp(e) {
const g = ctrl.g;
ctrl.pointers.delete(e.pointerId);
if (!g) return;
+ if (g.kind === "scroll") g.kind = "done"; // the finger left behind doesn't jump the scroll or drag
if (g.kind === "maybe" && ctrl.pointers.size === 0) {
clearTimeout(g.timer);
if (ctrlMode() === "abs") { const f = ctrlFraction(g.e); if (!f) { ctrl.g = null; return; } ctrlMoveTo(...f); }
@@ -1637,6 +1690,14 @@ function ctrlTouchUp(e) {
} else if (g.kind === "drag") ctrlButton("left", false);
if (ctrl.pointers.size === 0) ctrl.g = null;
}
+// The system took the touch (a swipe home, a call): no tap, and let go of anything held.
+function ctrlTouchCancel(e) {
+ ctrl.pointers.delete(e.pointerId);
+ if (ctrl.g?.timer) clearTimeout(ctrl.g.timer);
+ if (ctrl.held.has("left")) ctrlButton("left", false);
+ if (ctrl.g) ctrl.g.kind = "done";
+ if (ctrl.pointers.size === 0) ctrl.g = null;
+}
const canvasEl = $("canvas");
canvasEl.addEventListener("pointerdown", e => {
if (!ctrl.on) return;
@@ -1645,6 +1706,7 @@ canvasEl.addEventListener("pointerdown", e => {
try { canvasEl.setPointerCapture(e.pointerId); } catch { /* a pointer that's already gone */ }
if (!ctrlTarget().ok) return ctrlShow();
if (e.pointerType === "touch") return ctrlTouchDown(e);
+ if (ctrlMode() === "abs" && !ctrlFraction(e)) return; // on the bars around the picture
ctrlMouse(e);
ctrlButton(CTRL_BUTTONS[e.button] || "left", true);
});
@@ -1655,7 +1717,7 @@ canvasEl.addEventListener("pointermove", e => {
});
for (const type of ["pointerup", "pointercancel"]) canvasEl.addEventListener(type, e => {
if (!ctrl.on) return;
- if (e.pointerType === "touch") return ctrlTouchUp(e);
+ if (e.pointerType === "touch") return type === "pointercancel" ? ctrlTouchCancel(e) : ctrlTouchUp(e);
const name = CTRL_BUTTONS[e.button] || "left";
if (ctrl.held.has(name)) ctrlButton(name, false);
});
@@ -1668,19 +1730,21 @@ $("viewer").addEventListener("wheel", e => {
}, { passive: false });
// Keys while the view has focus go to the Frame (⌘ as Ctrl on a Mac); click elsewhere to stop.
function ctrlKey(e, down) {
- if (!ctrl.on || document.activeElement !== $("viewer") || !ctrlTarget().ok) return;
+ if (!ctrl.on || document.activeElement !== $("viewer") || (down && !ctrlTarget().ok)) return;
let code = EVDEV[e.code];
if (IS_MAC && (e.code === "MetaLeft" || e.code === "MetaRight")) code = 29;
if (!code) return;
e.preventDefault();
- if (down && e.repeat && code >= 29 && [29, 42, 54, 56, 97, 100, 125, 126].includes(code)) return;
- ctrlSend([{ key: code, down }]);
+ e.stopPropagation(); // it's for the Frame, not Frame Control's own shortcuts
+ if (down && e.repeat && [29, 42, 54, 56, 97, 100, 125, 126].includes(code)) return;
+ if (!down && !ctrl.keys.has(code)) return;
+ ctrlKeyEvent(code, down);
}
$("viewer").addEventListener("keydown", e => ctrlKey(e, true));
$("viewer").addEventListener("keyup", e => ctrlKey(e, false));
// Phone keyboard: the field types plain text; the buttons send single keys and clicks.
-function ctrlTap(code) { ctrlSend([{ key: code, down: true }, { key: code, down: false }]); }
+function ctrlTap(code) { ctrlKeyEvent(code, true); ctrlKeyEvent(code, false); }
let ctrlWarned = false;
function ctrlText(text) {
const plain = text.replace(/[^\x20-\x7e\n\t]/g, "");
diff --git a/ui/server.py b/ui/server.py
index 9d332cf..7eebcd1 100755
--- a/ui/server.py
+++ b/ui/server.py
@@ -868,11 +868,17 @@ def touch_event(event):
return max(-limit, min(limit, round(float(value), 4)))
out = {}
if "fx" in event or "fy" in event:
+ if "window" not in event:
+ raise Failure("a position needs the panel's window and display", 400)
out.update(fx=num("fx", 1), fy=num("fy", 1))
- window = event.get("window")
+ # Any event can name the panel it's meant for; the Frame drops it if another has focus.
+ if "window" in event:
+ window, display = event.get("window"), event.get("display")
if isinstance(window, bool) or not isinstance(window, int) or window <= 0:
raise Failure("window must be the panel's window id", 400)
- out["window"] = window
+ if not isinstance(display, str) or not PANEL_DISPLAY.match(display):
+ raise Failure("display must be an X display such as :1", 400)
+ out.update(window=window, display=display)
for name in ("dx", "dy"):
if name in event:
out[name] = num(name, INPUT_MOVE_LIMIT)
@@ -895,7 +901,7 @@ def touch_event(event):
if not isinstance(text, str) or not 0 < len(text) <= INPUT_TEXT_LIMIT:
raise Failure(f"text must be 1 to {INPUT_TEXT_LIMIT} characters", 400)
out["text"] = text
- if not out:
+ if not set(out) - {"window", "display"}:
raise Failure("touch event has nothing to do", 400)
return out