VR keyboard touch typing: the touched key highlights (and long-presses)

Steam's UpdateTouchState counts the event's `touches` per key: that count
is the pressed highlight (rgLayoutTouchCount -> Touched) and where the long
press (Backspace repeat, accents) starts. The touch-start event left the
new touch out of `touches`, like a touchend; now it is in, as in a DOM
touchstart (tracker.js touchEvent, with a test against a model of Steam's
handlers).

docs/keyboard.md

@@ -231,7 +231,7 @@ to the laser only. Depends on Steam and SteamVR UI internals (see
 
 **Tests:** `nix flake check` (checks `vr-keyboard-controllers`: tip and
 laser relative to the keyboard; `vr-keyboard-touch`: contact, hysteresis,
-two hands).
+two hands, the touch list Steam's highlight and long press read).
 
 **Debugging:** `window.__sfuiTouchTypeLog` (`SharedJSContext`: presses,
 misses), `__sfuiControllers.last` (the last frame), and in `systemui`

modules/vr-keyboard-touch/check.nix

@@ -1,6 +1,6 @@
 # Tests of touch typing (flake check `vr-keyboard-touch`): controller tip ->
-# keyboard surface -> key, press/release hysteresis, two hands
-# (tests/tracker.test.mjs).
+# keyboard surface -> key, press/release hysteresis, two hands, the touch
+# event Steam's handlers read (tests/tracker.test.mjs).
 { pkgs }:
 pkgs.runCommand "vr-keyboard-touch-checks" { nativeBuildInputs = [ pkgs.nodejs ]; } ''
   node ${./tests/tracker.test.mjs} ${./tracker.js} ${../vr-keyboard-controllers/geometry.js} | tee $out

modules/vr-keyboard-touch/keyboard-patch.js

@@ -17,7 +17,7 @@
 // "cancel" (no pose), frames stopping for STALE_MS, the keyboard moving or
 // closing end a touch without typing. Debugging: __sfuiTouchTypeLog.
 ((find, sigs, opts, hooks, TR, HUB) => {
-  const VERSION = 2;
+  const VERSION = 3;
   const G = window;
   const STALE_MS = 1000;                           // Steam's UI thread can stall frames for ~0.3 s
   const SNAP = 3;                                   // SteamVR overlay haptic effect (1 ButtonEnter, 3 Snap)
@@ -47,13 +47,9 @@
     const p = [...(G.g_PopupManager?.GetPopups?.() || [])].find((x) => x.window?.document.querySelector('[data-key]'));
     return p?.window ?? null;
   }
-  // Steam's handlers read target, changedTouches, touches and the touches'
-  // target / clientX / clientY.
-  const event = (t, inst) => ({
-    type: 'touch', target: t.target, changedTouches: [t],
-    touches: [...touches.values()].filter((e) => e.inst === inst && e.t !== t).map((e) => e.t),
-    preventDefault() {}, stopPropagation() {},
-  });
+  // Steam's handlers read target, changedTouches, touches (tracker.js
+  // touchEvent) and the touches' target / clientX / clientY.
+  const others = (t, inst) => [...touches.values()].filter((e) => e.inst === inst && e.t !== t).map((e) => e.t);
 
   function end(hand, { cancel = false, u, v } = {}) {
     const e = touches.get(hand);
@@ -64,7 +60,7 @@
     const win = e.doc.defaultView, x = u * win?.innerWidth, y = v * win?.innerWidth;
     if (!cancel && x >= 0 && y >= 0 && x < win.innerWidth && y < win.innerHeight) Object.assign(e.t, { clientX: x, clientY: y, pageX: x, pageY: y });
     if (cancel) e.inst.m_mapTouched.delete(e.t.target);         // Steam types only keys still in m_mapTouched
-    try { e.inst.HandleTouchEnd(event(e.t, e.inst)); } catch (err) { log('end-error', String(err)); }
+    try { e.inst.HandleTouchEnd(TR.touchEvent(e.t, others(e.t, e.inst), false)); } catch (err) { log('end-error', String(err)); }
     log(hand, cancel ? 'cancel' : 'up', e.t.target.getAttribute('data-key'));
   }
 
@@ -79,7 +75,7 @@
     if (!inst) { log(hand, 'miss', Math.round(x), Math.round(y)); return; }
     const t = { identifier: ID[hand], target: keyEl, clientX: x, clientY: y, pageX: x, pageY: y };
     touches.set(hand, { t, inst, doc });
-    try { inst.HandleTouchStart(event(t, inst)); } catch (err) { log('start-error', String(err)); end(hand, { cancel: true }); return; }
+    try { inst.HandleTouchStart(TR.touchEvent(t, others(t, inst), true)); } catch (err) { log('start-error', String(err)); end(hand, { cancel: true }); return; }
     if (opts.haptics) win.SteamClient?.OpenVR?.TriggerOverlayHapticEffect?.(SNAP, 0);
     log(hand, 'down', keyEl.getAttribute('data-key'), Math.round(x), Math.round(y));
   }

modules/vr-keyboard-touch/tests/tracker.test.mjs

@@ -139,4 +139,37 @@ test('two hands: independent, overlapping presses', () => {
   assert.deepEqual(evs, ['L:down:f', 'R:down:h', 'R:up', 'R:down:g', 'R:up', 'L:up']);
 });
 
+// Steam's keyboard as far as touches go (chunk~2dcc5aaf7.js): HandleTouchStart
+// adds the changed touches' targets to m_mapTouched, then UpdateTouchState
+// counts `touches` per key (rgLayoutTouchCount: the pressed highlight) and
+// starts the long press for a touched key; HandleTouchEnd types a key still in
+// m_mapTouched and recounts.
+function steamKeyboard() {
+  const kb = { m_mapTouched: new Set(), count: {}, longPress: null, typed: [] };
+  const update = (touches) => {
+    kb.count = {};
+    for (const t of touches) {
+      kb.count[t.target.key] = (kb.count[t.target.key] ?? 0) + 1;
+      if (kb.m_mapTouched.has(t.target)) kb.longPress = t.target.key;
+    }
+  };
+  kb.HandleTouchStart = (e) => { for (const t of e.changedTouches) kb.m_mapTouched.add(t.target); update(e.touches); };
+  kb.HandleTouchEnd = (e) => {
+    for (const t of e.changedTouches) { if (kb.m_mapTouched.has(t.target)) kb.typed.push(t.target.key); kb.m_mapTouched.delete(t.target); }
+    kb.longPress = null; update(e.touches);
+  };
+  return kb;
+}
+test('touchEvent: the touched key is counted (highlight, long press) until its release', () => {
+  const kb = steamKeyboard(), g = { target: { key: 'g' } }, sh = { target: { key: 'Shift' } };
+  kb.HandleTouchStart(TR.touchEvent(sh, [], true));
+  assert.deepEqual(kb.count, { Shift: 1 });
+  kb.HandleTouchStart(TR.touchEvent(g, [sh], true));               // second hand
+  assert.deepEqual(kb.count, { Shift: 1, g: 1 }); assert.equal(kb.longPress, 'g');
+  kb.HandleTouchEnd(TR.touchEvent(g, [sh], false));
+  assert.deepEqual(kb.count, { Shift: 1 }); assert.deepEqual(kb.typed, ['g']);
+  kb.HandleTouchEnd(TR.touchEvent(sh, [], false));
+  assert.deepEqual(kb.count, {}); assert.deepEqual(kb.typed, ['g', 'Shift']);
+});
+
 console.log(`${passed} tests passed`);

modules/vr-keyboard-touch/tracker.js

@@ -1,6 +1,6 @@
 // tracker.js: when a controller's tip touches a key (vr-keyboard-touch.nix;
 // argument of keyboard-patch.js, tested by tests/tracker.test.mjs).
-// Evaluates to { VERSION, DEFAULTS, createTracker }.
+// Evaluates to { VERSION, DEFAULTS, createTracker, touchEvent }.
 //
 // createTracker(opts): one hand. update(sample, tMs) with the tip relative
 // to the keyboard ({ u, v, d } of vr-keyboard-controllers' geometry.js:
@@ -16,8 +16,14 @@
 // A press needs the tip in front by arm (re-armed) since the last one, so
 // tracking jitter at the surface can't repeat it, and the tip coming from
 // behind or around the keyboard never presses.
+//
+// touchEvent(t, others, starting): what Steam's HandleTouchStart /
+// HandleTouchEnd read, shaped like a DOM TouchEvent: changedTouches [t];
+// touches: the touches down, with t at its start, without it at its end.
+// Steam's UpdateTouchState counts `touches` per key for the pressed
+// highlight and starts the long press (Backspace repeat, accents) there.
 (() => {
-  const VERSION = 1;
+  const VERSION = 2;
   const DEFAULTS = {
     depth: 0,             // m behind the drawn surface where a touch registers
     arm: 0.005,           // m in front of it to re-arm
@@ -62,5 +68,11 @@
     return { update, get pressed() { return pressed; } };
   }
 
-  return { VERSION, DEFAULTS, createTracker };
+  const touchEvent = (t, others, starting) => ({
+    type: starting ? 'touchstart' : 'touchend', target: t.target, changedTouches: [t],
+    touches: starting ? [...others, t] : [...others],
+    preventDefault() {}, stopPropagation() {},
+  });
+
+  return { VERSION, DEFAULTS, createTracker, touchEvent };
 })()