From 0699d0f5327c4e9f44b248e7a49cb4896a0f4e54 Mon Sep 17 00:00:00 2001 From: Pierre Kisters <1524059+lhns@users.noreply.github.com> Date: Thu, 1 Oct 2026 03:36:13 +0200 Subject: [PATCH] VR keyboard touch typing: the touched key highlights (and long-presses) Steam's UpdateTouchState counts the event's `touches` per key: that count is the pressed highlight (rgLayoutTouchCount -> Touched) and where the long press (Backspace repeat, accents) starts. The touch-start event left the new touch out of `touches`, like a touchend; now it is in, as in a DOM touchstart (tracker.js touchEvent, with a test against a model of Steam's handlers). --- docs/keyboard.md | 2 +- modules/vr-keyboard-touch/check.nix | 4 +-- modules/vr-keyboard-touch/keyboard-patch.js | 16 ++++----- .../vr-keyboard-touch/tests/tracker.test.mjs | 33 +++++++++++++++++++ modules/vr-keyboard-touch/tracker.js | 18 ++++++++-- 5 files changed, 57 insertions(+), 16 deletions(-) diff --git a/docs/keyboard.md b/docs/keyboard.md index e6695c7..f279cad 100644 --- a/docs/keyboard.md +++ b/docs/keyboard.md @@ -231,7 +231,7 @@ to the laser only. Depends on Steam and SteamVR UI internals (see **Tests:** `nix flake check` (checks `vr-keyboard-controllers`: tip and laser relative to the keyboard; `vr-keyboard-touch`: contact, hysteresis, -two hands). +two hands, the touch list Steam's highlight and long press read). **Debugging:** `window.__sfuiTouchTypeLog` (`SharedJSContext`: presses, misses), `__sfuiControllers.last` (the last frame), and in `systemui` diff --git a/modules/vr-keyboard-touch/check.nix b/modules/vr-keyboard-touch/check.nix index 717fab3..719edd0 100644 --- a/modules/vr-keyboard-touch/check.nix +++ b/modules/vr-keyboard-touch/check.nix @@ -1,6 +1,6 @@ # Tests of touch typing (flake check `vr-keyboard-touch`): controller tip -> -# keyboard surface -> key, press/release hysteresis, two hands -# (tests/tracker.test.mjs). +# keyboard surface -> key, press/release hysteresis, two hands, the touch +# event Steam's handlers read (tests/tracker.test.mjs). { pkgs }: pkgs.runCommand "vr-keyboard-touch-checks" { nativeBuildInputs = [ pkgs.nodejs ]; } '' node ${./tests/tracker.test.mjs} ${./tracker.js} ${../vr-keyboard-controllers/geometry.js} | tee $out diff --git a/modules/vr-keyboard-touch/keyboard-patch.js b/modules/vr-keyboard-touch/keyboard-patch.js index 5d9487d..75d5db7 100644 --- a/modules/vr-keyboard-touch/keyboard-patch.js +++ b/modules/vr-keyboard-touch/keyboard-patch.js @@ -17,7 +17,7 @@ // "cancel" (no pose), frames stopping for STALE_MS, the keyboard moving or // closing end a touch without typing. Debugging: __sfuiTouchTypeLog. ((find, sigs, opts, hooks, TR, HUB) => { - const VERSION = 2; + const VERSION = 3; const G = window; const STALE_MS = 1000; // Steam's UI thread can stall frames for ~0.3 s const SNAP = 3; // SteamVR overlay haptic effect (1 ButtonEnter, 3 Snap) @@ -47,13 +47,9 @@ const p = [...(G.g_PopupManager?.GetPopups?.() || [])].find((x) => x.window?.document.querySelector('[data-key]')); return p?.window ?? null; } - // Steam's handlers read target, changedTouches, touches and the touches' - // target / clientX / clientY. - const event = (t, inst) => ({ - type: 'touch', target: t.target, changedTouches: [t], - touches: [...touches.values()].filter((e) => e.inst === inst && e.t !== t).map((e) => e.t), - preventDefault() {}, stopPropagation() {}, - }); + // Steam's handlers read target, changedTouches, touches (tracker.js + // touchEvent) and the touches' target / clientX / clientY. + const others = (t, inst) => [...touches.values()].filter((e) => e.inst === inst && e.t !== t).map((e) => e.t); function end(hand, { cancel = false, u, v } = {}) { const e = touches.get(hand); @@ -64,7 +60,7 @@ const win = e.doc.defaultView, x = u * win?.innerWidth, y = v * win?.innerWidth; if (!cancel && x >= 0 && y >= 0 && x < win.innerWidth && y < win.innerHeight) Object.assign(e.t, { clientX: x, clientY: y, pageX: x, pageY: y }); if (cancel) e.inst.m_mapTouched.delete(e.t.target); // Steam types only keys still in m_mapTouched - try { e.inst.HandleTouchEnd(event(e.t, e.inst)); } catch (err) { log('end-error', String(err)); } + try { e.inst.HandleTouchEnd(TR.touchEvent(e.t, others(e.t, e.inst), false)); } catch (err) { log('end-error', String(err)); } log(hand, cancel ? 'cancel' : 'up', e.t.target.getAttribute('data-key')); } @@ -79,7 +75,7 @@ if (!inst) { log(hand, 'miss', Math.round(x), Math.round(y)); return; } const t = { identifier: ID[hand], target: keyEl, clientX: x, clientY: y, pageX: x, pageY: y }; touches.set(hand, { t, inst, doc }); - try { inst.HandleTouchStart(event(t, inst)); } catch (err) { log('start-error', String(err)); end(hand, { cancel: true }); return; } + try { inst.HandleTouchStart(TR.touchEvent(t, others(t, inst), true)); } catch (err) { log('start-error', String(err)); end(hand, { cancel: true }); return; } if (opts.haptics) win.SteamClient?.OpenVR?.TriggerOverlayHapticEffect?.(SNAP, 0); log(hand, 'down', keyEl.getAttribute('data-key'), Math.round(x), Math.round(y)); } diff --git a/modules/vr-keyboard-touch/tests/tracker.test.mjs b/modules/vr-keyboard-touch/tests/tracker.test.mjs index 0884633..c2f7830 100644 --- a/modules/vr-keyboard-touch/tests/tracker.test.mjs +++ b/modules/vr-keyboard-touch/tests/tracker.test.mjs @@ -139,4 +139,37 @@ test('two hands: independent, overlapping presses', () => { assert.deepEqual(evs, ['L:down:f', 'R:down:h', 'R:up', 'R:down:g', 'R:up', 'L:up']); }); +// Steam's keyboard as far as touches go (chunk~2dcc5aaf7.js): HandleTouchStart +// adds the changed touches' targets to m_mapTouched, then UpdateTouchState +// counts `touches` per key (rgLayoutTouchCount: the pressed highlight) and +// starts the long press for a touched key; HandleTouchEnd types a key still in +// m_mapTouched and recounts. +function steamKeyboard() { + const kb = { m_mapTouched: new Set(), count: {}, longPress: null, typed: [] }; + const update = (touches) => { + kb.count = {}; + for (const t of touches) { + kb.count[t.target.key] = (kb.count[t.target.key] ?? 0) + 1; + if (kb.m_mapTouched.has(t.target)) kb.longPress = t.target.key; + } + }; + kb.HandleTouchStart = (e) => { for (const t of e.changedTouches) kb.m_mapTouched.add(t.target); update(e.touches); }; + kb.HandleTouchEnd = (e) => { + for (const t of e.changedTouches) { if (kb.m_mapTouched.has(t.target)) kb.typed.push(t.target.key); kb.m_mapTouched.delete(t.target); } + kb.longPress = null; update(e.touches); + }; + return kb; +} +test('touchEvent: the touched key is counted (highlight, long press) until its release', () => { + const kb = steamKeyboard(), g = { target: { key: 'g' } }, sh = { target: { key: 'Shift' } }; + kb.HandleTouchStart(TR.touchEvent(sh, [], true)); + assert.deepEqual(kb.count, { Shift: 1 }); + kb.HandleTouchStart(TR.touchEvent(g, [sh], true)); // second hand + assert.deepEqual(kb.count, { Shift: 1, g: 1 }); assert.equal(kb.longPress, 'g'); + kb.HandleTouchEnd(TR.touchEvent(g, [sh], false)); + assert.deepEqual(kb.count, { Shift: 1 }); assert.deepEqual(kb.typed, ['g']); + kb.HandleTouchEnd(TR.touchEvent(sh, [], false)); + assert.deepEqual(kb.count, {}); assert.deepEqual(kb.typed, ['g', 'Shift']); +}); + console.log(`${passed} tests passed`); diff --git a/modules/vr-keyboard-touch/tracker.js b/modules/vr-keyboard-touch/tracker.js index d3a7dc5..3e2556b 100644 --- a/modules/vr-keyboard-touch/tracker.js +++ b/modules/vr-keyboard-touch/tracker.js @@ -1,6 +1,6 @@ // tracker.js: when a controller's tip touches a key (vr-keyboard-touch.nix; // argument of keyboard-patch.js, tested by tests/tracker.test.mjs). -// Evaluates to { VERSION, DEFAULTS, createTracker }. +// Evaluates to { VERSION, DEFAULTS, createTracker, touchEvent }. // // createTracker(opts): one hand. update(sample, tMs) with the tip relative // to the keyboard ({ u, v, d } of vr-keyboard-controllers' geometry.js: @@ -16,8 +16,14 @@ // A press needs the tip in front by arm (re-armed) since the last one, so // tracking jitter at the surface can't repeat it, and the tip coming from // behind or around the keyboard never presses. +// +// touchEvent(t, others, starting): what Steam's HandleTouchStart / +// HandleTouchEnd read, shaped like a DOM TouchEvent: changedTouches [t]; +// touches: the touches down, with t at its start, without it at its end. +// Steam's UpdateTouchState counts `touches` per key for the pressed +// highlight and starts the long press (Backspace repeat, accents) there. (() => { - const VERSION = 1; + const VERSION = 2; const DEFAULTS = { depth: 0, // m behind the drawn surface where a touch registers arm: 0.005, // m in front of it to re-arm @@ -62,5 +68,11 @@ return { update, get pressed() { return pressed; } }; } - return { VERSION, DEFAULTS, createTracker }; + const touchEvent = (t, others, starting) => ({ + type: starting ? 'touchstart' : 'touchend', target: t.target, changedTouches: [t], + touches: starting ? [...others, t] : [...others], + preventDefault() {}, stopPropagation() {}, + }); + + return { VERSION, DEFAULTS, createTracker, touchEvent }; })()