VR keyboard swipe with two controllers: the path from the controller bridge

With both lasers on the keyboard SteamVR forwards per poll only one laser's
movement, so the pressing one may get no touchmoves and its swipe became a
tap. keyboard.vr.swipe.twoHanded (default on; turns on the controller
bridge): a press is attributed to the hand whose laser hit is nearest to it
(within 30 px, the trigger as a tiebreak; without recent frames the first
frame after the press decides, within 60 px). The path comes from runs of
one source: the touch's own moves while they flow, that hand's laser hit
per bridge frame once they pause for 60 ms, touchmoves again if frames
pause. A touchend disagreeing with the laser during a bridge run ends at
the laser. Touchmoves on the other hand's laser move the attribution.
Without fresh frames: touch events only, as before. A second press during
a swipe stays Steam's tap (one text field, one word at a time).

The bridge sends full rate only on demand now: hub.js demand(ms) goes
back through the relay (CDP binding __sfuiCtlIn) to systemui's
__sfuiCtl.demand; the swipe patch asks from the press to the release.

Tests: gesture-input with the two-controller recording and synthetic bridge
frames (a pressing hand without touchmoves, attribution with both lasers on
the keyboard, a stale bridge, frames stopping mid-swipe, re-attribution,
interleaved hover and presses); hub demand.

modules/vr-keyboard-controllers.nix

@@ -1,15 +1,16 @@
 # The controller bridge for VR keyboard features (internal option
 # steamFrame.keyboard.vr.controllers.enable, set by the features using it,
-# e.g. keyboard.vr.touchTyping): both controllers relative to Steam's VR
-# keyboard, from SteamVR's dashboard to Steam's UI.
+# keyboard.vr.touchTyping and keyboard.vr.swipe.twoHanded): both controllers
+# relative to Steam's VR keyboard, from SteamVR's dashboard to Steam's UI.
 # - vr-keyboard-controllers/bridge-patch.js (SteamVR systemui, 8087; patch
 #   "vr-keyboard-controllers"): the keyboard's pose, per hand the tip (where
 #   the laser starts), the laser's hit and the trigger (geometry.js), ~90 Hz
-#   while relevant; CDP binding __sfuiCtlOut.
+#   while relevant or asked for (demand); CDP binding __sfuiCtlOut.
 # - vr-keyboard-controllers/relay.mjs (user service
-#   vr-keyboard-controllers-relay): to Steam's SharedJSContext.
+#   vr-keyboard-controllers-relay): to Steam's SharedJSContext; demand back
+#   (CDP binding __sfuiCtlIn there).
 # - vr-keyboard-controllers/hub.js: window.__sfuiControllers there (page px,
-#   subscribe); consumer patches take it as an extraArg.
+#   subscribe, demand); consumer patches take it as an extraArg.
 # Tests: vr-keyboard-controllers/check.nix (flake check
 # `vr-keyboard-controllers`).
 { config, pkgs, lib, ... }:

modules/vr-keyboard-controllers/bridge-patch.js

@@ -17,8 +17,9 @@
 //   the laser's hit on the keyboard (geometry.js), the trigger from the
 //   render model's animated "trigger" component (null without one).
 // Frames go out through the CDP binding __sfuiCtlOut(json) at TICK_MS while
-// a hand is relevant (tip within NEAR m of the keyboard or trigger pulled),
-// else every IDLE_MS; one { keyboard: null } frame when the keyboard goes.
+// a hand is relevant (tip within NEAR m of the keyboard or trigger pulled) or
+// Steam asked for it (S.demand(ms), hub.js demand: a laser press), else every
+// IDLE_MS; one { keyboard: null } frame when the keyboard goes.
 // Nothing at all while the keyboard is hidden (one DOM lookup per POLL_MS). Frame (hub.js adds page px):
 //   { seq, t, moving, keyboard: { width } | null,
 //     hands: { left, right: null | { tip: { u, v, d }, ray: { u, v, dist } | null,
@@ -26,7 +27,7 @@
 // Never touches SteamVR's input or the lasers. Debugging: __sfuiCtl.log,
 // __sfuiCtl.last (the last frame).
 ((find, sigs, opts, hooks, GEO) => {
-  const VERSION = 3;
+  const VERSION = 4;
   const G = window;
   const TICK_MS = 11, IDLE_MS = 1000, POLL_MS = 250, MOVE_QUIET_MS = 300, TIP_CACHE_MS = 5000;
   const NEAR = 0.1;                                  // m
@@ -139,7 +140,8 @@
     h.trigger >= 0.5);
 
   // ---- loops ---------------------------------------------------------------------
-  let active = false;
+  let active = false, fastUntil = -Infinity;
+  S.demand = (ms) => { fastUntil = performance.now() + Math.max(0, Math.min(5000, +ms || 0)); };
   const slow = setInterval(() => {
     const parent = findScaled();
     if (!parent) {
@@ -159,7 +161,7 @@
     for (const h of HANDS) {
       try { hands[h] = handFrame(h, now); } catch (e) { hands[h] = null; log('pose', String(e)); }
     }
-    const rel = HANDS.some((h) => relevant(hands[h]));
+    const rel = now < fastUntil || HANDS.some((h) => relevant(hands[h]));
     if (rel || wasRelevant || now - sentAt >= IDLE_MS) {
       send({ seq: ++seq, t: now, moving: now - movedAt < MOVE_QUIET_MS, keyboard: { width: S.kb.width }, hands });
     }

modules/vr-keyboard-controllers/hub.js

@@ -6,13 +6,16 @@
 // frames and lost() when the SteamVR side goes away.
 //   subscribe(fn) -> unsubscribe   fn(frame) per frame, fn(null) when lost
 //   last                           the last frame (null: none / lost)
+//   demand(ms)                     full rate for the next ms (0: end), e.g.
+//                                  during a laser press; via the relay's
+//                                  CDP binding __sfuiCtlIn
 // Each frame's hand points (tip, ray) get the keyboard page's px: x, y
 // (CSS px of the keyboard popup) and onKeyboard (inside the page), if the
 // keyboard popup exists. Frames come at up to ~90 Hz while a hand's tip is
 // within 10 cm of the keyboard or its trigger is pulled, else every 1 s, and
 // only while SteamVR shows the keyboard (then one with keyboard null).
 (() => {
-  const VERSION = 1;
+  const VERSION = 2;
   const G = globalThis;
   const have = G.__sfuiControllers;
   if (have && have.version >= VERSION) return have;
@@ -52,6 +55,9 @@
   const hub = {
     version: VERSION, subs, errors, last: null,
     frame, lost: () => deliver(null),
+    demand(ms) {
+      try { G.__sfuiCtlIn?.(JSON.stringify({ fast: Math.max(0, Math.min(5000, +ms || 0)) })); } catch (e) { errors.count++; errors.last = String(e); }
+    },
     subscribe(fn) { subs.add(fn); return () => subs.delete(fn); },
   };
   G.__sfuiControllers = hub;

modules/vr-keyboard-controllers/relay.mjs

@@ -2,10 +2,12 @@
 // controller frames from SteamVR's systemui page (bridge-patch.js, 8087; CDP
 // binding __sfuiCtlOut) to Steam's SharedJSContext (hub.js, 8080:
 // __sfuiControllers.frame(frame)). Frames are rebuilt from their numbers
-// (nothing else passes). If the systemui page goes away: lost().
+// (nothing else passes). If the systemui page goes away: lost(). Back from
+// Steam (hub.js demand(ms), CDP binding __sfuiCtlIn): { fast: ms } ->
+// __sfuiCtl.demand(ms) in systemui.
 const SIDES = {
-  steam: { url: 'http://127.0.0.1:8080/json/list', title: 'SharedJSContext' },
-  vr: { url: 'http://127.0.0.1:8087/json/list', title: 'systemui', binding: '__sfuiCtlOut' },
+  steam: { url: 'http://127.0.0.1:8080/json/list', title: 'SharedJSContext', binding: '__sfuiCtlIn', forward: demand },
+  vr: { url: 'http://127.0.0.1:8087/json/list', title: 'systemui', binding: '__sfuiCtlOut', forward },
 };
 const live = {};                                  // side -> evaluate(expression)
 
@@ -32,6 +34,12 @@ function forward(payload) {
   if (m) live.steam?.(`window.__sfuiControllers?.frame(${JSON.stringify(m)})`);
 }
 
+function demand(payload) {
+  let ms;
+  try { ms = num(JSON.parse(payload)?.fast); } catch { return; }
+  if (ms !== null) live.vr?.(`window.__sfuiCtl?.demand?.(${Math.max(0, Math.min(5000, ms))})`);
+}
+
 async function connect(side) {
   const cfg = SIDES[side];
   const list = await (await fetch(cfg.url, { signal: AbortSignal.timeout(3000) })).json();
@@ -43,7 +51,7 @@ async function connect(side) {
   const send = (method, params = {}) => { if (ws.readyState === WebSocket.OPEN) ws.send(JSON.stringify({ id: ++id, method, params })); };
   ws.onmessage = (e) => {
     const m = JSON.parse(e.data);
-    if (m.method === 'Runtime.bindingCalled' && m.params.name === cfg.binding) forward(m.params.payload);
+    if (m.method === 'Runtime.bindingCalled' && m.params.name === cfg.binding) cfg.forward(m.params.payload);
   };
   const closed = new Promise((res) => { ws.onclose = res; });
   // No Runtime.enable: bindings work without it, and it would stream the

modules/vr-keyboard-controllers/tests/geometry.test.mjs

@@ -1,5 +1,5 @@
 // geometry.js (tip and laser relative to the keyboard) and hub.js (page px,
-// subscribers). usage: node geometry.test.mjs <geometry.js> <hub.js>
+// subscribers, demand). usage: node geometry.test.mjs <geometry.js> <hub.js>
 import { readFileSync } from 'node:fs';
 import assert from 'node:assert/strict';
 
@@ -76,5 +76,14 @@ test('hub: page px and onKeyboard, subscribers, lost', () => {
   assert.equal(got.length, 2);
   assert.equal((0, eval)(readFileSync(process.argv[3], 'utf8')), HUB, 'same version: kept');
 });
+test('hub: demand(ms) through the relay binding, clamped; none without it', () => {
+  const sent = [];
+  globalThis.__sfuiCtlIn = (s) => sent.push(JSON.parse(s));
+  HUB.demand(800); HUB.demand(0); HUB.demand(1e9);
+  assert.deepEqual(sent, [{ fast: 800 }, { fast: 0 }, { fast: 5000 }]);
+  delete globalThis.__sfuiCtlIn;
+  HUB.demand(800);                                 // no relay: nothing, no error
+  assert.equal(HUB.errors.count, 0);
+});
 
 console.log(`${passed} tests passed`);

modules/vr-keyboard.nix

@@ -1,7 +1,9 @@
 # Swipe typing, suggestions and a Backspace drag for Steam's VR keyboard
 # (steamFrame.keyboard.vr; the extra keys are vr-keyboard-extra-keys.nix).
 # - vr-keyboard/patch.js (Steam UI, 8080): swipe gestures (swipe-decoder.js;
-#   gesture-input.js: the events of the laser that pressed),
+#   gesture-input.js: the events of the laser that pressed; with
+#   swipe.twoHanded its path from the controller bridge,
+#   vr-keyboard-controllers.nix, turned on here),
 #   a model of what the keyboard typed (textmodel.js), suggestions (swipe
 #   alternatives, corrections and completions, corrector.js; never changing
 #   text by themselves), Backspace drag (left: delete with a detent at word
@@ -23,6 +25,8 @@ let
 
   on = cfg.enable && (cfg.swipe.enable || cfg.autocorrect.enable || cfg.completions.enable || cfg.backspaceDrag.enable);
   panel = on && cfg.suggestions.position != "inside";
+  twoHanded = on && cfg.swipe.enable && cfg.swipe.twoHanded;
+  noHub = pkgs.writeText "no-controllers.js" "null";
 
   # Default dictionary: the language of keyboard.layout (if listed here) plus
   # English, else English only.
@@ -56,7 +60,8 @@ let
         inherit (cfg.backspaceDrag) wordDetentPixels;
         inherit (cfg) haptics;
       };
-      extraArgs = [ ./vr-keyboard/swipe-decoder.js ./vr-keyboard/textmodel.js ./vr-keyboard/corrector.js dictionaryJs ./vr-keyboard/gesture-input.js ];
+      extraArgs = [ ./vr-keyboard/swipe-decoder.js ./vr-keyboard/textmodel.js ./vr-keyboard/corrector.js dictionaryJs ./vr-keyboard/gesture-input.js
+        (if twoHanded then config.steamFrame.keyboard.vr.controllers.files.hub else noHub) ];
     }} $out
     node --check $out
   '';
@@ -89,7 +94,7 @@ let
     };
   };
 in {
-  imports = [ ./session.nix ./cleanup.nix ./steam-ui-patches.nix ./keyboard-layout.nix ];
+  imports = [ ./session.nix ./cleanup.nix ./steam-ui-patches.nix ./keyboard-layout.nix ./vr-keyboard-controllers.nix ];
 
   options.steamFrame.keyboard.vr = {
     enable = mkEnableOption ''
@@ -98,6 +103,17 @@ in {
       sub-features below are on by default)'';
 
     swipe.enable = mkEnableOption "swipe typing" // { default = true; };
+    swipe.twoHanded = mkOption {
+      type = types.bool;
+      default = true;
+      description = ''
+        Swipes also with both lasers on the keyboard: the path comes from the
+        pressing controller's pose when SteamVR forwards no laser movement
+        for it (the controller bridge, sampling during a press while the
+        keyboard is shown). Off: laser events only; such a swipe can become a
+        tap.
+      '';
+    };
 
     dictionary = {
       languages = mkOption {
@@ -210,6 +226,7 @@ in {
         unpatch = ./vr-keyboard/suggestions-panel/unpatch.js;
       };
       steamFrame.session.services.${if panel then "restart" else "stop"} = [ "vr-keyboard-relay.service" ];
+      steamFrame.keyboard.vr.controllers.enable = lib.mkIf twoHanded true;
     }
     (lib.mkIf panel {
       systemd.user.services.vr-keyboard-relay = {

modules/vr-keyboard/check.nix

@@ -1,6 +1,7 @@
 # Tests of the VR keyboard (flake check `vr-keyboard`, also built before the
 # patch by vr-keyboard.nix with the configured dictionary): text model,
-# gesture input (two controllers), corrector and swipe-decoder accuracy
+# gesture input (two controllers, the controller bridge's path), corrector
+# and swipe-decoder accuracy
 # (tests/*.test.mjs). The accuracy
 # thresholds (German keyboard geometry, German and English words) fail the
 # build only for German + English dictionaries, the default here.

modules/vr-keyboard/gesture-input.js

@@ -1,5 +1,6 @@
 // gesture-input.js: which input events belong to the gesture in progress
-// (patch.js). Evaluates to { VERSION, create }.
+// (patch.js), and its path from the controller bridge. Evaluates to
+// { VERSION, create }.
 //
 // SteamVR's lasers reach the keyboard page as touches (a trigger press: one
 // touch with a new identifier per press) and as mouse hover (pointerId 1,
@@ -11,11 +12,34 @@
 // they never enter the swipe path or end it. A second press while a gesture
 // runs is ignored (Steam handles it as stock); one whose owner is gone
 // (missed release) replaces it.
+//
+// Controller bridge (frame(f): vr-keyboard-controllers' hub.js frames): with
+// both lasers on the keyboard SteamVR forwards per poll only one laser's
+// movement, so the pressing one may get no touchmoves at all. A press is
+// attributed to the hand whose laser hit (ray x/y) was within TOL px of it in
+// the last STALE ms (nearest; trigger as a tiebreak). Without such frames
+// (the bridge sends full rate only on demand, patch.js asks at the press)
+// the first usable frame within WAIT ms decides: the hand nearest to the
+// press or the touch's last move, within LATE px. The path then comes
+// from runs of one source: the owner's touchmoves while they flow (or
+// Chromium's pointer moves of the touch, which precede them), the hand's
+// ray per frame once they pause for GAP ms, and touchmoves again once frames
+// pause. The release is the touch's; its point is the hand's last ray point
+// if the touchend's disagrees (by more than AGREE px) during a bridge run.
+// Touchmoves far from the hand's ray but on the other hand's move the
+// attribution. No fresh frames at the press: touch events only, as without
+// the bridge.
 (() => {
-  const VERSION = 1;
+  const VERSION = 2;
+  const STALE = 150, GAP = 60, TOL = 30, AGREE = 40, FAR = 60, WAIT = 250, LATE = 60;
 
-  function create() {
+  function create({ now = () => performance.now() } = {}) {
     let owner = null, sawPointer = false;
+    // Bridge: recent frames [{ at, f }]; the gesture's hand and source.
+    let frames = [];
+    let hand = null, mode = 'touch', touchAt = 0, bridgeAt = -Infinity, bridgePt = null;
+    let waitUntil = -Infinity, refs = [];          // a press not attributed yet: its points
+    const stats = { touch: 0, bridge: 0, switches: 0, late: false };
 
     // The contacts of an event: [{ id, x, y, target }].
     function contacts(e) {
@@ -38,37 +62,115 @@
       touchcancel: 'cancel', pointercancel: 'cancel',
     };
 
+    const usable = (f) => !!(f?.keyboard && !f.moving);
+    const ray = (f, h) => { const r = f?.hands?.[h]?.ray; return r?.onKeyboard ? r : null; };
+    const dist = (r, x, y) => Math.hypot(r.x - x, r.y - y);
+    const recent = (t) => frames.filter((e) => t - e.at <= STALE && usable(e.f));
+    // The hand whose ray in frames fs was nearest to one of the points pts, within tol.
+    function attribute(fs, pts, tol) {
+      if (!fs.length) return null;
+      const near = ['left', 'right'].map((h) => ({
+        h, d: Math.min(...fs.flatMap((e) => { const r = ray(e.f, h); return r ? pts.map(([x, y]) => dist(r, x, y)) : [Infinity]; })),
+        trig: fs[fs.length - 1].f.hands?.[h]?.trigger ?? null,
+      })).filter((c) => c.d <= tol).sort((a, b) => a.d - b.d);
+      if (near.length === 2 && near[1].d - near[0].d < 10) {
+        const pulled = near.filter((c) => c.trig !== null && c.trig >= 0.5);
+        if (pulled.length === 1) return pulled[0].h;
+      }
+      return near[0]?.h ?? null;
+    }
+    function lastRay(h, t) {
+      const e = frames[frames.length - 1];
+      return e && t - e.at <= STALE && usable(e.f) ? ray(e.f, h) : null;
+    }
+    function reset() { owner = null; hand = null; mode = 'touch'; bridgePt = null; bridgeAt = -Infinity; waitUntil = -Infinity; refs = []; }
+
     return {
       VERSION,
       get owner() { return owner; },
+      get hand() { return hand; },
+      get mode() { return mode; },
+      stats,
       // { phase, id, x, y, target } if the event concerns the gesture (or may
       // start one: phase 'down'), else null. An 'up' or 'cancel' ends the
-      // ownership.
+      // ownership. A move during a bridge run: null (the frames carry the path).
       read(e) {
         const phase = PHASE[e.type];
         if (!phase) return null;
         const cs = contacts(e);
+        // Chromium's pointer moves of a touch (before its pointercancel): the touch is moving.
+        if (e.type === 'pointermove' && e.pointerType === 'touch' && owner?.[0] === 't' && mode === 'touch') touchAt = now();
         if (phase === 'down') {
           // One touch at a time starts a gesture (a multi-touch press is Steam's).
           if (e.type === 'touchstart' && (e.touches?.length ?? 0) > 1) {
-            if (owner?.[0] === 't' && ![...e.touches].some((t) => `t${t.identifier}` === owner)) owner = null;
+            if (owner?.[0] === 't' && ![...e.touches].some((t) => `t${t.identifier}` === owner)) reset();
             return null;
           }
           const c = cs[0];
           if (!c) return null;
           // Busy: ignored, unless the owner is gone (a lone touch, or its own id again).
           if (owner && !(e.type === 'touchstart' || c.id === owner)) return null;
-          owner = null;
+          reset();
           return { phase, ...c };
         }
         if (!owner) return null;
         const c = cs.find((x) => x.id === owner);
         if (!c) return null;
-        if (phase !== 'move') owner = null;
-        return { phase, ...c };
+        const t = now();
+        if (phase === 'move') {
+          if (t < waitUntil) refs[1] = [c.x, c.y];
+          if (hand) {
+            // A touchmove far from the hand's laser but on the other's: the other hand pressed.
+            const other = hand === 'left' ? 'right' : 'left', r = lastRay(hand, t), o = lastRay(other, t);
+            if (r && o && dist(r, c.x, c.y) > FAR && dist(o, c.x, c.y) <= TOL) { hand = other; stats.switches++; }
+          }
+          if (mode === 'bridge') {
+            if (t - bridgeAt <= GAP) return null;   // frames flowing: they carry the path
+            mode = 'touch';
+          }
+          touchAt = t;
+          stats.touch++;
+          return { phase, ...c };
+        }
+        let r = { phase, ...c };
+        if (phase === 'up' && mode === 'bridge' && bridgePt && t - bridgeAt <= STALE && Math.hypot(bridgePt[0] - c.x, bridgePt[1] - c.y) > AGREE) {
+          r = { ...r, x: bridgePt[0], y: bridgePt[1] };
+        }
+        reset();
+        return r;
+      },
+      // The 'down' contact starts a gesture; returns the hand it is attributed to (or null).
+      claim(c) {
+        owner = c.id;
+        touchAt = now();
+        Object.assign(stats, { touch: 0, bridge: 0, switches: 0, late: false });
+        hand = attribute(recent(touchAt), [[c.x, c.y]], TOL);
+        if (!hand) { waitUntil = touchAt + WAIT; refs = [[c.x, c.y]]; }
+        return hand;
+      },
+      release() { reset(); },
+      // A bridge frame (null: bridge lost). Returns [x, y] to add to the path, or null.
+      frame(f) {
+        const t = now();
+        if (!f) { frames = []; return null; }
+        frames.push({ at: t, f });
+        while (frames.length && t - frames[0].at > STALE) frames.shift();
+        if (!owner || !usable(f)) return null;
+        if (!hand && t < waitUntil) {
+          hand = attribute([{ at: t, f }], refs, LATE);
+          if (hand) { waitUntil = -Infinity; stats.late = true; }
+        }
+        if (!hand) return null;
+        const r = f.hands?.[hand]?.ray;
+        if (!r) return null;
+        bridgeAt = t;
+        if (!r.onKeyboard) return null;
+        bridgePt = [r.x, r.y];
+        if (mode === 'touch' && t - touchAt > GAP) mode = 'bridge';
+        if (mode !== 'bridge') return null;
+        stats.bridge++;
+        return bridgePt;
       },
-      claim(c) { owner = c.id; },                 // the 'down' contact starts a gesture
-      release() { owner = null; },
     };
   }
 

modules/vr-keyboard/patch.js

@@ -1,12 +1,16 @@
 // Gestures and suggestions for Steam's VR keyboard (vr-keyboard.nix),
 // injected into Steam's SharedJSContext (8080). The keyboard is a popup of
 // that context ("SteamVR - Keyboard"); everything works on its document.
-// mkPatch convention plus five more arguments: swipe-decoder.js,
-// textmodel.js, corrector.js, the dictionary text ("word<TAB>zipf*10\n...")
-// and gesture-input.js.
+// mkPatch convention plus six more arguments: swipe-decoder.js,
+// textmodel.js, corrector.js, the dictionary text ("word<TAB>zipf*10\n..."),
+// gesture-input.js and vr-keyboard-controllers' hub.js (null without
+// swipe.twoHanded).
 //
 // - Swipe: the laser arrives as touch events; Steam types the key a touch
-//   started on at release. Once a press leaves its first key we drop that
+//   started on at release. With both lasers on the keyboard the pressing one
+//   may get no touchmoves; its path then comes from the controller bridge
+//   (gesture-input.js: the hand whose laser hit the press point; full rate
+//   asked for from the press to the release). Once a press leaves its first key we drop that
 //   key from Steam's pending touches (m_mapTouched) and cancel its long press
 //   (Steam's own touch-end cleanup, minus the typing), then decode the path.
 // - Output: one character or "Backspace" per HandleVirtualKeyDown, the path
@@ -28,8 +32,8 @@
 // missing the keyboard stays stock. Only passive listeners; never blocks
 // Steam's events. Debugging: __sfuiSwipeLog, __sfuiSwipePaths
 // (scripts/vr-keyboard-replay.mjs).
-((find, sigs, opts, hooks, D, T, C, DICT, P) => {
-  const VERSION = 25;
+((find, sigs, opts, hooks, D, T, C, DICT, P, HUB) => {
+  const VERSION = 26;
   const G = window;
   const O = opts;
 
@@ -63,7 +67,7 @@
   const popup = [...(g_PopupManager.GetPopups?.() || [])].find((p) => p.window?.document.querySelector('[data-key]'));
   if (!popup) return 'no keyboard popup yet';
   const doc = popup.window.document;
-  const stamp = `${VERSION} ${JSON.stringify(O)}`;   // new code or options: re-attach
+  const stamp = `${VERSION}:${P.VERSION}:${HUB?.version} ${JSON.stringify(O)}`;   // new code or options: re-attach
   if (doc.__sfuiSwipe === stamp && S.docs.has(doc)) return 'unchanged';
   S.docs.get(doc)?.();
   S.docs.delete(doc);
@@ -382,7 +386,7 @@
     // mouse from pointer events, or mouse events if there are none). The
     // other controller's hover, a second press: ignored (Steam's, as stock).
     let down = null;
-    const input = P.create();
+    const input = P.create({ now: () => G.performance.now() });
     const keyOf = (t) => t?.closest?.('[data-key]');
     const stripIndexOf = (t) => {
       const el = t?.closest?.('#sfui-swipe-strip > div');
@@ -404,7 +408,9 @@
       if (!c) return;
       if (down?.active) clearTrail();            // its release was missed
       down = start(c);
-      if (down) input.claim(c);
+      if (!down) return;
+      const hand = input.claim(c);
+      if (bridge) { demand(); log('press', c.id, hand ?? '-'); }
     }
     function start({ x, y, target }) {
       const si = stripIndexOf(target);
@@ -442,8 +448,10 @@
     }
     function onMove(e) {
       const c = input.read(e);
-      if (!c || !down || down.kind === 'strip') return;
-      const { x, y } = c;
+      if (c) move(c.x, c.y);
+    }
+    function move(x, y) {
+      if (!down || down.kind === 'strip') return;
       if (down.kind === 'backspace') { backspaceMove(down, x); return; }
       const q = down.pts[down.pts.length - 1];
       if (Math.hypot(x - q[0], y - q[1]) < 2) return;
@@ -457,10 +465,12 @@
       drawTrail(down.pts, 'rgba(26, 159, 255, 0.75)');
     }
     function onUp(e) {
+      const st = { ...input.stats, hand: input.hand };
       const c = input.read(e);
       if (!c) return;
       const { x, y } = c, d = down;
       down = null;
+      if (bridge) { endDemand(); if (d) d.src = st; }
       if (d?.kind === 'strip') {                  // released anywhere on the strip: the button pressed
         if (strip.contains(at(x, y))) pick(d.index);
         return;
@@ -481,7 +491,8 @@
       try { results = D.decode(d.lay.dec, d.pts, { max: O.count, unit: d.lay.unit }); } catch (err) { log('decode-error', String(err)); }
       log('decode', results.map((r) => r.word).join(' '));
       const paths = (G.__sfuiSwipePaths ??= []);
-      paths.push({ unit: d.lay.unit, keys: d.lay.keys, pts: d.pts.map(([x, y]) => [Math.round(x), Math.round(y)]), top: results.map((r) => r.word) });
+      paths.push({ unit: d.lay.unit, keys: d.lay.keys, pts: d.pts.map(([x, y]) => [Math.round(x), Math.round(y)]), top: results.map((r) => r.word), src: d.src });
+      if (d.src) log('path', d.src.hand ?? '-', `touch ${d.src.touch}`, `bridge ${d.src.bridge}`, d.src.late ? 'late' : '', d.src.switches ? `switches ${d.src.switches}` : '');
       if (paths.length > 20) paths.shift();
       if (!results.length) { drawTrail(d.pts, 'rgba(255, 80, 80, 0.75)', 400); return; }
       fadeTimer = setTimeout(clearTrail, 150);
@@ -491,6 +502,24 @@
       if (!input.read(e)) return;                // another contact; Chromium's touch pointercancel
       if (down?.active) clearTrail();
       down = null;
+      if (bridge) endDemand();
+    }
+
+    // Controller bridge (HUB, swipe.twoHanded): the pressing hand's laser per
+    // frame while its touchmoves pause (gesture-input.js). The bridge sends
+    // full rate only while asked to (a lease, renewed during the gesture).
+    const bridge = HUB && typeof HUB.subscribe === 'function' ? HUB : null;
+    let demandAt = -Infinity;
+    const DEMAND_MS = 800;
+    function demand() { demandAt = G.performance.now(); try { bridge.demand?.(DEMAND_MS); } catch { /* old hub */ } }
+    function endDemand() { if (demandAt > -Infinity) { demandAt = -Infinity; try { bridge.demand?.(0); } catch { /* old hub */ } } }
+    if (bridge) {
+      cleanup.push(bridge.subscribe((f) => {
+        const p = input.frame(f);
+        if (down && input.owner && G.performance.now() - demandAt > DEMAND_MS / 3) demand();
+        if (p) move(p[0], p[1]);
+      }));
+      cleanup.push(endDemand);
     }
     function onHover(e) {
       const si = stripIndexOf(e.target);

modules/vr-keyboard/tests/gesture-input.test.mjs

@@ -3,6 +3,8 @@
 // it, the release ends it. Fixture two-controllers.json: a swipe recorded in
 // VR (Steam's keyboard page) while the other controller pointed at the
 // keyboard; its hover (485,106) once broke into the path ("tust" for "test").
+// The controller bridge: that recording plus synthetic hub.js frames (the
+// pressing hand's laser along the recorded path, the other at its hover).
 // usage: node gesture-input.test.mjs <gesture-input.js> <two-controllers.json>
 import { readFileSync } from 'node:fs';
 import assert from 'node:assert/strict';
@@ -131,4 +133,179 @@ test('mouse: pointer events by pointerId, mouse events only without them', () =>
   ]), [{ id: 'p1', pts: [[1, 1], [2, 1], [3, 1]], ended: 'up' }]);
 });
 
+// ---- controller bridge ------------------------------------------------------------
+// A frame as hub.js delivers it; a hand: [x, y] (ray on the keyboard), 'off'
+// (ray beside it) or null (no pose).
+const hand = (p, trigger = null) => (p === null ? null
+  : { tip: { u: 0, v: 0, d: 0.3 }, trigger, ray: p === 'off' ? { x: -50, y: 10, onKeyboard: false } : { x: p[0], y: p[1], onKeyboard: true } });
+const frame = (left, right, { trigger = {}, moving = false } = {}) =>
+  ({ keyboard: { width: 0.74 }, moving, hands: { left: hand(left, trigger.left), right: hand(right, trigger.right) } });
+
+// Timed steps [ms, event | { frame }], fed like patch.js: frame points extend the path.
+function runT(steps) {
+  let clock = 0;
+  const input = P.create({ now: () => clock }), paths = [];
+  let cur = null;
+  for (const [t, x] of steps) {
+    clock = t;
+    if ('frame' in x) {
+      const p = input.frame(x.frame);
+      if (p && cur) cur.pts.push([...p]);
+      continue;
+    }
+    const handNow = input.hand;                    // the hand at the release (read() ends it)
+    const c = input.read(x);
+    if (!c) continue;
+    if (c.phase === 'down') {
+      cur = { id: c.id, pts: [[c.x, c.y]] };
+      cur.hand = input.claim(c);
+    } else if (c.phase === 'move') {
+      cur.pts.push([c.x, c.y]);
+    } else {
+      cur.pts.push([c.x, c.y]);
+      Object.assign(cur, { ended: c.phase, hand: handNow, stats: { ...input.stats } });
+      paths.push(cur);
+      cur = null;
+    }
+  }
+  if (cur) paths.push(cur);
+  return paths;
+}
+const T = (type, touches, changed) => ev.touch(type, touches, changed);
+
+// The recording: its touch path, the other controller's hover.
+const rec = JSON.parse(readFileSync(process.argv[3], 'utf8'));
+const recPath = rec.filter((r) => /^touch(start|move|end)$/.test(r[0])).map((r) => r[2][0].slice(1));
+const HOVER = [485, 106];
+
+test('recorded swipe, the pressing hand gets no touchmoves: the path comes from the bridge', () => {
+  // Every 20 ms an event of the recording, without the touchmoves; the touchend
+  // at the press point (SteamVR forwarded nothing); a frame every 11 ms with
+  // the right laser along the recorded path, the left at the hover.
+  const events = rec.filter((r) => r[0] !== 'touchmove').map(fromRecord);
+  const end = events.findIndex((e) => e.type === 'touchend');
+  events[end] = T('touchend', [], [[814, ...recPath[0]]]);
+  const startAt = 20 * events.findIndex((e) => e.type === 'touchstart'), endAt = 20 * end;
+  const steps = events.map((e, i) => [20 * i, e]);
+  for (let t = startAt - 33, k = 0; t < endAt; t += 11, k++) {
+    const i = Math.min(recPath.length - 1, Math.max(0, Math.round((t - startAt) / (endAt - startAt) * (recPath.length - 1))));
+    steps.push([t + 0.5, { frame: frame(HOVER, recPath[i]) }]);
+  }
+  steps.sort((a, b) => a[0] - b[0]);
+  const [p, ...more] = runT(steps);
+  assert.equal(more.length, 0);
+  assert.equal(p.hand, 'right');
+  assert.equal(p.ended, 'up');
+  assert.deepEqual(p.pts[0], recPath[0]);
+  assert.ok(p.stats.bridge > 20 && p.stats.touch === 0, `bridge points ${p.stats.bridge}`);
+  // "test" ends near its start: the stale touchend agrees (a far one: the next tests).
+  assert.deepEqual(p.pts.at(-2), recPath.at(-1), 'the laser up to the release');
+  assert.ok(Math.hypot(p.pts.at(-1)[0] - recPath.at(-1)[0], p.pts.at(-1)[1] - recPath.at(-1)[1]) <= 40);
+  assert.ok(!p.pts.some(([x, y]) => x === HOVER[0] && y === HOVER[1]), 'no hover of the other controller');
+  for (const q of p.pts) assert.ok(recPath.some((r) => r[0] === q[0] && r[1] === q[1]), `${q} on the recorded path`);
+});
+
+test('recorded swipe with its touchmoves and frames interleaved: the touchmoves, as without the bridge', () => {
+  const steps = rec.map((r, i) => [5 * i, fromRecord(r)]);
+  for (let t = 0; t < 5 * rec.length; t += 11) steps.push([t + 0.5, { frame: frame(HOVER, recPath[0]) }]);
+  steps.sort((a, b) => a[0] - b[0]);
+  const [p] = runT(steps);
+  assert.equal(p.hand, 'right');
+  assert.equal(p.stats.bridge, 0, "Chromium's pointer moves of the touch, then its touchmoves: flowing");
+  assert.deepEqual(p.pts, recPath);
+});
+
+test('attribution with both lasers on the keyboard: nearest within 30 px, trigger as a tiebreak', () => {
+  const press = (f, x, y) => runT([[0, { frame: f }], [10, T('touchstart', [[1, x, y]], [[1, x, y]])]])[0].hand;
+  const apart = frame([100, 50], [300, 80]);
+  assert.equal(press(apart, 302, 79), 'right');
+  assert.equal(press(apart, 98, 52), 'left');
+  assert.equal(press(apart, 200, 200), null, 'neither laser there');
+  assert.equal(press(frame([100, 50], 'off'), 300, 80), null, 'the near laser beside the keyboard');
+  assert.equal(press(frame([300, 70], [305, 82]), 301, 74), 'left', 'a near tie, no trigger: nearest');
+  assert.equal(press(frame([300, 70], [305, 82], { trigger: { left: 0, right: 1 } }), 301, 74), 'right');
+  assert.equal(press(frame([300, 70], [305, 82], { trigger: { left: 1, right: 0.1 } }), 301, 74), 'left');
+  assert.equal(press(frame([300, 70], [303, 120], { trigger: { left: 0, right: 1 } }), 301, 74), 'left', 'no tie: the trigger is not asked');
+  assert.equal(press(frame([100, 50], [300, 80], { moving: true }), 302, 79), null, 'keyboard moving');
+});
+
+test('no recent frames at the press: the first frame after it attributes (the bridge woke up)', () => {
+  const [p] = runT([
+    [0, { frame: frame([100, 50], [300, 80]) }],             // an idle frame, long ago
+    [1000, T('touchstart', [[1, 300, 80]], [[1, 300, 80]])],
+    [1035, { frame: frame([100, 50], [340, 82]) }],           // moved 40 px since
+    [1046, { frame: frame([100, 50], [360, 83]) }],
+    [1070, { frame: frame([100, 50], [380, 84]) }],
+    [1100, { frame: frame([100, 50], [420, 86]) }],
+    [1110, T('touchend', [], [[1, 300, 80]])],
+  ]);
+  assert.equal(p.hand, 'right');
+  assert.equal(p.stats.late, true);
+  assert.deepEqual(p.pts, [[300, 80], [380, 84], [420, 86], [420, 86]], 'frames once touchmoves paused 60 ms; the release at the laser, not at the stale touchend');
+});
+
+test('stale bridge: touch events only (as without it)', () => {
+  const [p] = runT([
+    [0, { frame: frame([100, 50], [300, 80]) }],
+    [200, T('touchstart', [[1, 300, 80]], [[1, 300, 80]])],
+    [300, T('touchmove', [[1, 320, 80]], [[1, 320, 80]])],
+    [500, { frame: frame([100, 50], [360, 80]) }],           // after the wait: not attributed
+    [600, T('touchmove', [[1, 340, 80]], [[1, 340, 80]])],
+    [700, T('touchend', [], [[1, 350, 80]])],
+  ]);
+  assert.equal(p.hand, null);
+  assert.deepEqual(p.pts, [[300, 80], [320, 80], [340, 80], [350, 80]]);
+});
+
+test('frames stopping mid-swipe: touchmoves again; frames back: after a touch pause', () => {
+  const steps = [[0, { frame: frame(null, [300, 80]) }], [5, T('touchstart', [[1, 300, 80]], [[1, 300, 80]])]];
+  for (let t = 11; t <= 110; t += 11) steps.push([t, { frame: frame(null, [300 + t, 80]) }]);   // to 410
+  steps.push([140, T('touchmove', [[1, 405, 80]], [[1, 405, 80]])]);                             // frames still fresh: dropped
+  steps.push([200, T('touchmove', [[1, 420, 80]], [[1, 420, 80]])]);                             // 90 ms without frames
+  steps.push([216, T('touchmove', [[1, 440, 80]], [[1, 440, 80]])]);
+  steps.push([230, { frame: frame(null, [445, 80]) }]);                                           // touch flowing: ignored
+  steps.push([300, { frame: frame(null, [470, 80]) }]);                                           // touch paused 84 ms
+  steps.push([310, T('touchend', [], [[1, 472, 80]])]);
+  const [p] = runT(steps);
+  assert.equal(p.hand, 'right');
+  assert.deepEqual(p.pts.map(([x]) => x), [300, 366, 377, 388, 399, 410, 420, 440, 470, 472]);
+});
+
+test("touchmoves on the other hand's laser move the attribution", () => {
+  const [p] = runT([
+    [0, { frame: frame([300, 80], [310, 90]) }],             // both near the press: left (nearest)
+    [5, T('touchstart', [[1, 302, 82]], [[1, 302, 82]])],
+    [11, { frame: frame([300, 80], [330, 90]) }],
+    [16, T('touchmove', [[1, 330, 91]], [[1, 330, 91]])],    // still near both
+    [22, { frame: frame([300, 80], [380, 92]) }],
+    [27, T('touchmove', [[1, 381, 92]], [[1, 381, 92]])],    // 81 px from left, on right
+    [33, { frame: frame([300, 80], [420, 94]) }],
+    [100, { frame: frame([300, 80], [470, 96]) }],
+    [110, T('touchend', [], [[1, 302, 82]])],
+  ]);
+  assert.equal(p.stats.switches, 1);
+  assert.deepEqual(p.pts, [[302, 82], [330, 91], [381, 92], [470, 96], [470, 96]]);
+});
+
+test('bridge swipe: a second press and the other hover are ignored, the next press is attributed anew', () => {
+  const paths = runT([
+    [0, { frame: frame([100, 50], [300, 80]) }],
+    [5, T('touchstart', [[1, 300, 80]], [[1, 300, 80]])],                 // right presses
+    [8, ev.pointer('pointermove', 'mouse', 1, 102, 50)],                  // left's hover
+    [80, { frame: frame([102, 50], [340, 80]) }],
+    [85, T('touchstart', [[1, 300, 80], [2, 102, 50]], [[2, 102, 50]])], // left presses: Steam's
+    [90, T('touchend', [[1, 300, 80]], [[2, 102, 50]])],
+    [91, { frame: frame([102, 50], [380, 80]) }],
+    [100, T('touchend', [], [[1, 300, 80]])],
+    [200, { frame: frame([102, 50], [380, 80]) }],
+    [205, T('touchstart', [[3, 101, 51]], [[3, 101, 51]])],               // left swipes alone
+    [290, { frame: frame([150, 52], [380, 80]) }],
+    [300, T('touchend', [], [[3, 101, 51]])],
+  ]);
+  assert.deepEqual(paths.map((p) => [p.id, p.hand, p.pts]), [
+    ['t1', 'right', [[300, 80], [340, 80], [380, 80], [380, 80]]],
+    ['t3', 'left', [[101, 51], [150, 52], [150, 52]]],
+  ]);
+});
+
 console.log(`gesture-input: ${passed} tests passed`);