mirror of
https://github.com/lhns/steam-frame-nix.git
synced 2026-10-06 01:00:13 +02:00
VR keyboard touch typing, on a controller bridge from the SteamVR dashboard
keyboard.vr.touchTyping (off by default): a key is pressed when a controller's tip, where SteamVR's laser starts (/pose/tip: the device pose times the render model's "tip" component), passes through it from the front; released when pulled back 1 cm, re-armed 5 mm in front. Both hands, also at once. Keys go through the keyboard's own HandleTouchStart / HandleTouchEnd, like a laser press (Backspace repeats while held). Options depth (cm) and haptics. The geometry lives in a reusable controller bridge (vr-keyboard-controllers, internal option): a systemui patch reads the keyboard's pose with SteamVR's SGQueryService (an empty transform of ours in the keyboard's mount) and both controllers' poses, and streams per hand the tip, the laser's hit on the keyboard and the trigger (~90 Hz while relevant) through the vr-keyboard-controllers-relay service to __sfuiControllers in Steam's SharedJSContext. Tests: flake checks vr-keyboard-controllers and vr-keyboard-touch.
This commit is contained in:
Pierre Kisters committed
1 parent
7d050b8c44
commit
224715d926
19 files changed
+1042
-8
No files matched your search
@@ -35,6 +35,7 @@ configuration, limitations and how it works.
|
||||
|
||||
- [Extra keys](docs/keyboard.md#extra-keys) (`keyboard.vr.extraKeys`): Esc/Ctrl/Alt, arrows, Delete, Shift+Tab, real chords and AltGr/non-ASCII characters.
|
||||
- [Swipe and suggestions](docs/keyboard.md#swipe-and-suggestions) (`keyboard.vr`): swipe typing, corrections, completions, Backspace drag.
|
||||
- [Touch typing](docs/keyboard.md#touch-typing) (`keyboard.vr.touchTyping`): press keys by touching them with the controller's tip, both hands.
|
||||
|
||||
**VR "+" menu** ([docs/launcher-menu.md](docs/launcher-menu.md)):
|
||||
|
||||
@@ -260,6 +261,9 @@ which: [Repository layout](docs/development.md).
|
||||
| `steamFrame.keyboard.vr.backspaceDrag.wordDetentPixels` | int | `90` | Extra travel across a word border (`0`: none). |
|
||||
| `steamFrame.keyboard.vr.haptics` | bool | `true` | Haptic ticks for drag steps, word detents and picks. |
|
||||
| `steamFrame.keyboard.vr.checks` | package, read-only | | The tests, built with the configured dictionary. |
|
||||
| `steamFrame.keyboard.vr.touchTyping.enable` | bool | `false` | Touch typing: a key is pressed when a controller's tip touches it, both hands; the lasers work as before, see [Touch typing](docs/keyboard.md#touch-typing). |
|
||||
| `steamFrame.keyboard.vr.touchTyping.depth` | number | `0` | How far behind the keyboard's surface a touch registers (cm, `-2` to `5`; negative: in front). |
|
||||
| `steamFrame.keyboard.vr.touchTyping.haptics` | bool | `true` | A haptic tick when a touch presses a key. |
|
||||
| `steamFrame.uiPatches.patches` | list of submodules | `[ ]` | Runtime patches of Steam's web UIs, see [UI patches](docs/ui-patches.md#defining-a-patch). Fields below. |
|
||||
| `steamFrame.uiPatches.patches.*.name` | str | required | Unique name (log). |
|
||||
| `steamFrame.uiPatches.patches.*.endpoint` | str | `"http://127.0.0.1:8080"` | DevTools base URL, `/json/list` polled every 5 s: Steam 8080, SteamVR 8087. |
|
||||
|
||||
+15
-1
@@ -111,6 +111,18 @@ modules/
|
||||
dictionary.nix, gen-dict.py build: dictionary from wordfreq + Hunspell
|
||||
check.nix, tests/ test (also built before the patch): text model,
|
||||
corrector, swipe-decoder accuracy, gesture input
|
||||
vr-keyboard-controllers.nix controller bridge for keyboard features (internal option)
|
||||
vr-keyboard-controllers/
|
||||
bridge-patch.js, unpatch.js SteamVR: keyboard pose, tips, laser hits, triggers
|
||||
geometry.js SteamVR (argument of bridge-patch.js): tip/laser -> keyboard
|
||||
hub.js Steam (argument of consumer patches): __sfuiControllers
|
||||
relay.mjs service vr-keyboard-controllers-relay: frames SteamVR -> Steam
|
||||
check.nix, tests/ test: geometry, hub
|
||||
vr-keyboard-touch.nix touch typing (keyboard.vr.touchTyping)
|
||||
vr-keyboard-touch/
|
||||
keyboard-patch.js, keyboard-unpatch.js Steam: presses touched keys
|
||||
tracker.js Steam (argument of keyboard-patch.js): contact, hysteresis
|
||||
check.nix, tests/ test (also built before the patch): touch paths
|
||||
dashboard-windows.nix, dashboard-windows/ SteamVR: window scale/distance limits
|
||||
steam-close-button.nix, steam-close-button/ SteamVR: X on the Steam window
|
||||
window-curvature.nix, window-curvature/ SteamVR: curvature wheel
|
||||
@@ -128,6 +140,8 @@ they are kept even where a file name says more:
|
||||
| `launcher-menu` | `launcherMenu` | `launcher-menu-{order,pinned-desktop,launch,grid,show-all}` (Steam) | `steam-ui-patches` |
|
||||
| `vr-keyboard` | `keyboard.vr` | `vr-keyboard` (Steam), `vr-keyboard-panel` = `suggestions-panel/` (SteamVR) | `steam-ui-patches`, `vr-keyboard-relay` |
|
||||
| `vr-keyboard-extra-keys` | `keyboard.vr.extraKeys` | `steam-keyboard-patch` (Steam) | `steam-keyboard-patch` |
|
||||
| `vr-keyboard-touch` | `keyboard.vr.touchTyping` | `vr-keyboard-touch` (Steam) | `steam-ui-patches` |
|
||||
| `vr-keyboard-controllers` | (internal, `keyboard.vr.controllers`) | `vr-keyboard-controllers` (SteamVR) | `steam-ui-patches`, `vr-keyboard-controllers-relay` |
|
||||
| `dashboard-windows` | `dashboard.windows` | `dashboard-windows` (SteamVR) | `steam-ui-patches` |
|
||||
| `steam-close-button` | `dashboard.steamCloseButton` | `steam-close-button` (SteamVR, state) | `steam-ui-patches` |
|
||||
| `window-curvature` | `dashboard.windowCurvature` | `window-curvature` (SteamVR) | `steam-ui-patches` |
|
||||
@@ -137,7 +151,7 @@ they are kept even where a file name says more:
|
||||
| `screenshots` | `screenshots` | | `steam-frame-nix-screenshots` (`.service`, `.path`) |
|
||||
|
||||
Log of all patches:
|
||||
`journalctl --user -u steam-ui-patches -u steam-keyboard-patch -u vr-keyboard-relay`.
|
||||
`journalctl --user -u steam-ui-patches -u steam-keyboard-patch -u vr-keyboard-relay -u vr-keyboard-controllers-relay`.
|
||||
|
||||
## Checks
|
||||
|
||||
|
||||
+66
-3
@@ -1,8 +1,8 @@
|
||||
# VR keyboard
|
||||
|
||||
Two patches of Steam's VR keyboard: [extra keys](#extra-keys) and
|
||||
[swipe and suggestions](#swipe-and-suggestions). They work together or
|
||||
alone. Options: [README, Options](../README.md#options) (`keyboard.vr.*`).
|
||||
Three patches of Steam's VR keyboard: [extra keys](#extra-keys),
|
||||
[swipe and suggestions](#swipe-and-suggestions) and
|
||||
[touch typing](#touch-typing). They work together or alone. Options: [README, Options](../README.md#options) (`keyboard.vr.*`).
|
||||
The keyboard layout of the Steam session itself is a separate fix
|
||||
([keyboard layout](session.md#keyboard-layout)).
|
||||
|
||||
@@ -173,3 +173,66 @@ configured dictionary.
|
||||
|
||||
**Debugging:** `window.__sfuiSwipeLog` and `__sfuiSwipePaths` in Steam's
|
||||
SharedJSContext (replay swipes with `scripts/vr-keyboard-replay.mjs`).
|
||||
|
||||
## Touch typing
|
||||
|
||||
`keyboard.vr.touchTyping.enable`, module `vr-keyboard-touch`.
|
||||
|
||||
**Problem:** Steam's VR keyboard can only be typed on with the laser and
|
||||
the trigger.
|
||||
|
||||
**What you get:**
|
||||
|
||||
- Touch a key with the controller's tip, the point the laser starts from,
|
||||
to press it; both hands, also at once (e.g. Shift held by one hand).
|
||||
The key lights up on contact and is typed when the tip comes back out,
|
||||
like a laser press: holding Backspace repeats, holding a letter opens its
|
||||
accents.
|
||||
- A press needs the tip to pass through the keyboard's surface from the
|
||||
front (`depth`: that far behind it; default `0`). It ends when the tip is
|
||||
pulled back 1 cm from its deepest point. The next press needs the tip
|
||||
5 mm in front of the surface again, so resting on a key doesn't repeat it.
|
||||
Pointing, the tip behind the keyboard, or coming in from the side
|
||||
presses nothing.
|
||||
- The lasers work as before, also while touching.
|
||||
- A haptic tick on contact (`haptics`), routed by SteamVR like the
|
||||
keyboard's other ticks.
|
||||
|
||||
**Caveats:** moving the keyboard pauses touches for 0.3 s. The extra keys' Delete repeat and the swipe patch's gestures react
|
||||
to the laser only. Depends on Steam and SteamVR UI internals (see
|
||||
[after a Steam update](ui-patches.md#after-a-steam-update)).
|
||||
|
||||
**Remove when** Steam's VR keyboard gets touch input.
|
||||
|
||||
### How it works
|
||||
|
||||
- **Controller bridge** (module `vr-keyboard-controllers`, turned on by
|
||||
touch typing; meant for other keyboard features too): a patch of SteamVR's
|
||||
`systemui` page (port 8087, [SteamVR debugger](steamvr-debugger.md),
|
||||
turned on automatically). It reads the keyboard's pose with SteamVR's own
|
||||
scene graph query (`SGQueryService.requestSGTransform` on an empty
|
||||
transform of ours inside the keyboard's mount; every 250 ms) and both
|
||||
controllers' poses (`VRHTML.GetPose`). The tip is the device pose times
|
||||
the render model's `tip` component; the laser mouse uses the same
|
||||
`/pose/tip`. For each hand it computes the tip and the laser's hit
|
||||
relative to the keyboard, plus the trigger from the render model's
|
||||
animated `trigger` component (not yet verified). The
|
||||
`vr-keyboard-controllers-relay` user service carries these frames to
|
||||
Steam's `SharedJSContext`: up to ~90 Hz (45 Hz and more under load)
|
||||
while a hand is within 10 cm of the keyboard, its laser is on it or its
|
||||
trigger is pulled, else every 250 ms, and only while SteamVR shows the
|
||||
keyboard. The laser hit matches SteamVR's laser to a few px.
|
||||
- **Touch typing** (patch `vr-keyboard-touch`, `SharedJSContext`): per hand
|
||||
`tracker.js` turns the tip's path into press and release, then calls the
|
||||
keyboard's own `HandleTouchStart` / `HandleTouchEnd` with the key under
|
||||
the contact point. No input is synthesized.
|
||||
- Found by signature (entries `vr-keyboard-touch`,
|
||||
`vr-keyboard-controllers`).
|
||||
|
||||
**Tests:** `nix flake check` (checks `vr-keyboard-controllers`: tip and
|
||||
laser relative to the keyboard; `vr-keyboard-touch`: contact, hysteresis,
|
||||
two hands).
|
||||
|
||||
**Debugging:** `window.__sfuiTouchTypeLog` (`SharedJSContext`: presses,
|
||||
misses), `__sfuiControllers.last` (the last frame), and in `systemui`
|
||||
`__sfuiCtl.log` and `__sfuiCtl.last`.
|
||||
@@ -18,6 +18,8 @@
|
||||
keyboard-layout = ./modules/keyboard-layout.nix;
|
||||
vr-keyboard-extra-keys = ./modules/vr-keyboard-extra-keys.nix;
|
||||
vr-keyboard = ./modules/vr-keyboard.nix;
|
||||
vr-keyboard-controllers = ./modules/vr-keyboard-controllers.nix;
|
||||
vr-keyboard-touch = ./modules/vr-keyboard-touch.nix;
|
||||
hidden-apps = ./modules/hidden-apps.nix;
|
||||
steam-ui-patches = ./modules/steam-ui-patches.nix;
|
||||
launcher-menu = ./modules/launcher-menu.nix;
|
||||
@@ -66,10 +68,11 @@
|
||||
}) self.packages;
|
||||
|
||||
# Tests of the VR keyboard (text model, corrector, swipe decoder on the
|
||||
# default German + English dictionary), the extra keys' xdotool
|
||||
# allowlist, the Jellyfin mpv shim, the Firefox wrapper, the launchers,
|
||||
# install.sh cleanup, the applications.menu link, the portal config and
|
||||
# the screenshots account detection: nix flake check
|
||||
# default German + English dictionary), the controller geometry and touch
|
||||
# typing, the extra keys' xdotool allowlist, the Jellyfin mpv shim, the
|
||||
# Firefox wrapper, the launchers, install.sh cleanup, the
|
||||
# applications.menu link, the portal config and the screenshots account
|
||||
# detection: nix flake check
|
||||
checks = forSystems (pkgs: {
|
||||
applications-menu = import ./modules/applications-menu/check.nix { inherit pkgs; };
|
||||
cleanup = import ./modules/cleanup/check.nix { inherit pkgs; };
|
||||
@@ -80,6 +83,8 @@
|
||||
screenshots = import ./modules/screenshots/check.nix { inherit pkgs; };
|
||||
vr-keyboard = import ./modules/vr-keyboard/check.nix { inherit pkgs; };
|
||||
vr-keyboard-extra-keys = import ./modules/vr-keyboard-extra-keys/check.nix { inherit pkgs; };
|
||||
vr-keyboard-controllers = import ./modules/vr-keyboard-controllers/check.nix { inherit pkgs; };
|
||||
vr-keyboard-touch = import ./modules/vr-keyboard-touch/check.nix { inherit pkgs; };
|
||||
});
|
||||
|
||||
# nix flake init -t github:lhns/steam-frame-nix
|
||||
|
||||
@@ -486,6 +486,36 @@
|
||||
"module": { "includes": ["\"valve.steam.gamepadui.keyboard\""] }
|
||||
}
|
||||
}
|
||||
},
|
||||
"vr-keyboard-touch": {
|
||||
"bundle": "steamui",
|
||||
"modules": {
|
||||
"vrStatus": {
|
||||
"module": { "includes": ["get VRKeyboardStatus()"] },
|
||||
"exports": { "holder": { "type": "object", "getters": ["VRKeyboardStatus"] } },
|
||||
"expects": ["bIsOpen:"]
|
||||
},
|
||||
"keyboardComponent": {
|
||||
"checkOnly": true,
|
||||
"module": { "includes": ["TypeKeyInternal(", "m_mapTouched", "HandleTouchStart(", "HandleTouchEnd(", "ElementFromTouch("] },
|
||||
"expects": ["\"data-key\"", "MouseHoverBlockerHack", "StartLongPressTimer(", "LongPressTimerExpired("]
|
||||
}
|
||||
}
|
||||
},
|
||||
"vr-keyboard-controllers": {
|
||||
"bundle": "vrwebui-systemui",
|
||||
"modules": {
|
||||
"sgQuery": {
|
||||
"checkOnly": true,
|
||||
"module": { "includes": ["window.SGQueryService=", "transform_request"] },
|
||||
"expects": ["requestSGTransform("]
|
||||
},
|
||||
"keyboardMount": {
|
||||
"checkOnly": true,
|
||||
"module": { "includes": ["m_sKeyboardOverlayKey", "GrabHandleTint", "k_nKeyboardGrabTransformOffset"] },
|
||||
"expects": ["mountedId"]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
# The controller bridge for VR keyboard features (internal option
|
||||
# steamFrame.keyboard.vr.controllers.enable, set by the features using it,
|
||||
# e.g. keyboard.vr.touchTyping): both controllers relative to Steam's VR
|
||||
# keyboard, from SteamVR's dashboard to Steam's UI.
|
||||
# - vr-keyboard-controllers/bridge-patch.js (SteamVR systemui, 8087; patch
|
||||
# "vr-keyboard-controllers"): the keyboard's pose, per hand the tip (where
|
||||
# the laser starts), the laser's hit and the trigger (geometry.js), ~90 Hz
|
||||
# while relevant; CDP binding __sfuiCtlOut.
|
||||
# - vr-keyboard-controllers/relay.mjs (user service
|
||||
# vr-keyboard-controllers-relay): to Steam's SharedJSContext.
|
||||
# - vr-keyboard-controllers/hub.js: window.__sfuiControllers there (page px,
|
||||
# subscribe); consumer patches take it as an extraArg.
|
||||
# Tests: vr-keyboard-controllers/check.nix (flake check
|
||||
# `vr-keyboard-controllers`).
|
||||
{ config, pkgs, lib, ... }:
|
||||
let
|
||||
cfg = config.steamFrame.keyboard.vr.controllers;
|
||||
uiLib = import ./steam-ui-patches/lib { inherit pkgs; };
|
||||
checks = import ./vr-keyboard-controllers/check.nix { inherit pkgs; };
|
||||
in {
|
||||
imports = [ ./session.nix ./steam-ui-patches.nix ];
|
||||
|
||||
options.steamFrame.keyboard.vr.controllers = {
|
||||
enable = lib.mkOption {
|
||||
type = lib.types.bool;
|
||||
default = false;
|
||||
internal = true;
|
||||
description = "The controller bridge (set by the VR keyboard features that use it).";
|
||||
};
|
||||
files = lib.mkOption {
|
||||
type = lib.types.attrsOf lib.types.path;
|
||||
readOnly = true;
|
||||
internal = true;
|
||||
default = { geometry = ./vr-keyboard-controllers/geometry.js; hub = ./vr-keyboard-controllers/hub.js; };
|
||||
description = "geometry.js and hub.js, for consumer patches' extraArgs.";
|
||||
};
|
||||
};
|
||||
|
||||
config = lib.mkMerge [
|
||||
{
|
||||
steamFrame.uiPatches.patches = lib.optional cfg.enable {
|
||||
name = "vr-keyboard-controllers";
|
||||
endpoint = "http://127.0.0.1:8087";
|
||||
target.title = "systemui";
|
||||
patch = pkgs.runCommand "vr-keyboard-controllers.js" { nativeBuildInputs = [ pkgs.nodejs ]; } ''
|
||||
: ${checks}
|
||||
cp ${uiLib.mkPatch {
|
||||
name = "vr-keyboard-controllers";
|
||||
src = ./vr-keyboard-controllers/bridge-patch.js;
|
||||
extraArgs = [ ./vr-keyboard-controllers/geometry.js ];
|
||||
}} $out
|
||||
node --check $out
|
||||
'';
|
||||
unpatch = ./vr-keyboard-controllers/unpatch.js;
|
||||
};
|
||||
steamFrame.session.services.${if cfg.enable then "restart" else "stop"} = [ "vr-keyboard-controllers-relay.service" ];
|
||||
}
|
||||
(lib.mkIf cfg.enable {
|
||||
systemd.user.services.vr-keyboard-controllers-relay = {
|
||||
Unit.Description = "Relay of the VR keyboard's controller frames (SteamVR dashboard -> Steam UI)";
|
||||
Service = { ExecStart = "${pkgs.nodejs}/bin/node ${./vr-keyboard-controllers/relay.mjs}"; Restart = "always"; RestartSec = 5; };
|
||||
Install.WantedBy = [ "default.target" ];
|
||||
};
|
||||
})
|
||||
];
|
||||
}
|
||||
@@ -0,0 +1,175 @@
|
||||
// The controller bridge (vr-keyboard-controllers.nix; patch name
|
||||
// "vr-keyboard-controllers"), injected into SteamVR's systemui page (8087):
|
||||
// both controllers relative to Steam's VR keyboard, streamed to Steam's
|
||||
// SharedJSContext (relay.mjs -> hub.js). mkPatch convention plus one more
|
||||
// argument: geometry.js.
|
||||
//
|
||||
// While SteamVR shows the keyboard (systemui mounts its overlay, mountedId
|
||||
// "...gamepadui.keyboard", only then, also without the dashboard, in a
|
||||
// transform scaled to its size; the overlay hangs from its top edge, 1
|
||||
// overlay width = 1 unit there):
|
||||
// - its pose: an empty vsg-transform of ours (id PROBE) in that transform,
|
||||
// read with SteamVR's own SGQueryService.requestSGTransform every POLL_MS;
|
||||
// for MOVE_QUIET_MS after it moved (drag, dashboard re-latch) frames say
|
||||
// moving: true;
|
||||
// - every TICK_MS both controllers: VRHTML.GetPose(hand, Standing) times the
|
||||
// render model's "tip" component (SteamVR's laser origin) -> the tip and
|
||||
// the laser's hit on the keyboard (geometry.js), the trigger from the
|
||||
// render model's animated "trigger" component (null without one).
|
||||
// Frames go out through the CDP binding __sfuiCtlOut(json) at TICK_MS while
|
||||
// a hand is relevant (tip within NEAR m of the keyboard, laser on it, or
|
||||
// trigger pulled), else every IDLE_MS; one { keyboard: null } frame when the
|
||||
// keyboard goes. Frame (hub.js adds page px):
|
||||
// { seq, t, moving, keyboard: { width } | null,
|
||||
// hands: { left, right: null | { tip: { u, v, d }, ray: { u, v, dist } | null,
|
||||
// trigger: 0..1 | null } } }
|
||||
// Never touches SteamVR's input or the lasers. Debugging: __sfuiCtl.log,
|
||||
// __sfuiCtl.last (the last frame).
|
||||
((find, sigs, opts, hooks, GEO) => {
|
||||
const VERSION = 2;
|
||||
const G = window;
|
||||
const TICK_MS = 11, IDLE_MS = 250, POLL_MS = 250, MOVE_QUIET_MS = 300, TIP_CACHE_MS = 5000;
|
||||
const NEAR = 0.1; // m
|
||||
const TRIGGER_DEG = 12.5; // frame controller: trigger component's full travel
|
||||
const PROBE = 'sfui-ctl-keyboard';
|
||||
const HANDS = ['left', 'right'];
|
||||
const STANDING = 1; // ETrackingUniverseOrigin, as SteamVR's dashboard reads poses
|
||||
const stamp = `${VERSION}:${GEO.VERSION} ${JSON.stringify(opts)}`;
|
||||
if (G.__sfuiCtl?.stamp === stamp) return 'unchanged';
|
||||
const V = G.VRHTML;
|
||||
const missing = [
|
||||
typeof V?.GetPose !== 'function' && 'VRHTML.GetPose',
|
||||
typeof V?.VRRenderModels?.GetComponentStateForDevicePath !== 'function' && 'VRHTML.VRRenderModels.GetComponentStateForDevicePath',
|
||||
typeof V?.VRProperties?.GetStringProperty !== 'function' && 'VRHTML.VRProperties.GetStringProperty',
|
||||
typeof V?.NextSGID !== 'function' && 'VRHTML.NextSGID',
|
||||
typeof V?.VROverlay?.ThisOverlayKey !== 'function' && 'VRHTML.VROverlay.ThisOverlayKey',
|
||||
typeof G.SGQueryService?.requestSGTransform !== 'function' && 'SGQueryService.requestSGTransform',
|
||||
typeof G.forceLayoutUpdate !== 'function' && 'forceLayoutUpdate',
|
||||
].filter(Boolean);
|
||||
if (missing.length) return `SteamVR internals changed, no controller bridge: missing ${missing.join(', ')}`;
|
||||
try { G.__sfuiCtl?.dispose?.(); } catch { /* gone */ }
|
||||
|
||||
const S = G.__sfuiCtl = { stamp, log: [], kb: null, last: null };
|
||||
const log = (...a) => { S.log.push([Math.round(performance.now()), ...a]); if (S.log.length > 100) S.log.shift(); };
|
||||
let seq = 0, sentAt = -Infinity, wasRelevant = false;
|
||||
const send = (f) => {
|
||||
S.last = f;
|
||||
sentAt = f.t;
|
||||
try { G.__sfuiCtlOut?.(JSON.stringify(f)); } catch (e) { log('send', String(e)); }
|
||||
};
|
||||
|
||||
// ---- the keyboard -----------------------------------------------------------
|
||||
const fiberProps = (e) => {
|
||||
const k = Object.keys(e).find((x) => x.startsWith('__reactFiber$'));
|
||||
let f = k && e[k];
|
||||
for (let i = 0; f && i < 6; f = f.return, i++) if (f.memoizedProps?.mountedId) return f.memoizedProps;
|
||||
return null;
|
||||
};
|
||||
let scaled = null, probe = null;
|
||||
function findScaled() {
|
||||
if (scaled?.isConnected) return scaled;
|
||||
scaled = null;
|
||||
for (const e of document.querySelectorAll('[vsg-type="mountedscenegraph"]')) {
|
||||
if (String(fiberProps(e)?.mountedId ?? '').endsWith('gamepadui.keyboard') && e.parentElement?.tagName === 'VSG-TRANSFORM') {
|
||||
scaled = e.parentElement;
|
||||
break;
|
||||
}
|
||||
}
|
||||
return scaled;
|
||||
}
|
||||
function ensureProbe(parent) {
|
||||
if (probe?.parentElement === parent) return;
|
||||
probe?.remove();
|
||||
probe = document.createElement('vsg-transform');
|
||||
for (const [k, v] of Object.entries({ translation: '0 0 0', rotation: '1 0 0 0', scale: '1 1 1', sgid: V.NextSGID() }))
|
||||
probe.setAttribute(k, String(v));
|
||||
probe.id = PROBE;
|
||||
parent.appendChild(probe);
|
||||
G.forceLayoutUpdate();
|
||||
}
|
||||
const removeProbe = () => { if (probe) { probe.remove(); probe = null; G.forceLayoutUpdate(); } };
|
||||
|
||||
let pending = false, movedAt = -Infinity;
|
||||
async function pollKeyboard() {
|
||||
pending = true;
|
||||
try {
|
||||
const xf = await Promise.race([
|
||||
G.SGQueryService.requestSGTransform(`${V.VROverlay.ThisOverlayKey()}::${PROBE}`),
|
||||
new Promise((_, rej) => setTimeout(() => rej(new Error('timeout')), 1000)),
|
||||
]);
|
||||
if (!probe) return;
|
||||
let ow = 1;
|
||||
try { ow = V.VROverlay.GetWidthInMeters(V.VROverlay.FindOverlay('valve.steam.gamepadui.keyboard')) || 1; } catch { /* 1 */ }
|
||||
const kb = { translation: xf.translation, rotation: xf.rotation, width: xf.scale.x * ow };
|
||||
const o = S.kb;
|
||||
if (!o || Math.hypot(kb.translation.x - o.translation.x, kb.translation.y - o.translation.y, kb.translation.z - o.translation.z) > 0.002 ||
|
||||
Math.abs(kb.rotation.w * o.rotation.w + kb.rotation.x * o.rotation.x + kb.rotation.y * o.rotation.y + kb.rotation.z * o.rotation.z) < 0.99998 ||
|
||||
Math.abs(kb.width - o.width) > 0.002) movedAt = performance.now();
|
||||
S.kb = kb;
|
||||
} catch (e) { log('keyboard pose', String(e?.message ?? e)); } finally { pending = false; }
|
||||
}
|
||||
|
||||
// ---- the controllers ------------------------------------------------------------
|
||||
const models = {}; // hand -> { at, rm, tip }
|
||||
function model(hand, now) {
|
||||
const c = models[hand];
|
||||
if (c && now - c.at < TIP_CACHE_MS) return c;
|
||||
const path = `/user/hand/${hand}`;
|
||||
let rm = null, tip = null;
|
||||
try {
|
||||
rm = V.VRProperties.GetStringProperty(path, 1003) || null; // Prop_RenderModelName_String
|
||||
tip = rm ? V.VRRenderModels.GetComponentStateForDevicePath(rm, 'tip', path)?.xfTrackingToComponentLocal ?? null : null;
|
||||
} catch { /* the device pose */ }
|
||||
return (models[hand] = { at: now, rm, tip });
|
||||
}
|
||||
function trigger(hand, rm) {
|
||||
try {
|
||||
const q = rm && V.VRRenderModels.GetComponentStateForDevicePath(rm, 'trigger', `/user/hand/${hand}`)?.xfTrackingToComponentRenderModel?.rotation;
|
||||
return q ? Math.min(1, GEO.rotationDegrees(q) / TRIGGER_DEG) : null;
|
||||
} catch { return null; }
|
||||
}
|
||||
function handFrame(hand, now) {
|
||||
const [p] = V.GetPose(`/user/hand/${hand}`, STANDING) || [];
|
||||
if (!p?.bPoseIsValid) return null;
|
||||
const m = model(hand, now);
|
||||
const pose = GEO.tipPose(p.xfDeviceToAbsoluteTracking, m.tip);
|
||||
return { tip: GEO.toKeyboard(pose.translation, S.kb), ray: GEO.rayHit(pose, S.kb), trigger: trigger(hand, m.rm) };
|
||||
}
|
||||
const relevant = (h) => !!h && ((Math.abs(h.tip.d) < NEAR && h.tip.u > -0.2 && h.tip.u < 1.2 && h.tip.v > -0.2 && h.tip.v < 0.7) ||
|
||||
(h.ray && h.ray.u >= 0 && h.ray.u <= 1 && h.ray.v >= 0 && h.ray.v <= 0.6) || h.trigger >= 0.5);
|
||||
|
||||
// ---- loops ---------------------------------------------------------------------
|
||||
let active = false;
|
||||
const slow = setInterval(() => {
|
||||
const parent = findScaled();
|
||||
if (!parent) {
|
||||
if (active) { send({ seq: ++seq, t: performance.now(), moving: false, keyboard: null, hands: { left: null, right: null } }); log('inactive'); }
|
||||
active = false; S.kb = null;
|
||||
removeProbe();
|
||||
return;
|
||||
}
|
||||
ensureProbe(parent);
|
||||
if (!active) log('active');
|
||||
active = true;
|
||||
if (!pending) pollKeyboard();
|
||||
}, POLL_MS);
|
||||
const fast = setInterval(() => {
|
||||
if (!active || !S.kb) return;
|
||||
const now = performance.now(), hands = {};
|
||||
for (const h of HANDS) {
|
||||
try { hands[h] = handFrame(h, now); } catch (e) { hands[h] = null; log('pose', String(e)); }
|
||||
}
|
||||
const rel = HANDS.some((h) => relevant(hands[h]));
|
||||
if (rel || wasRelevant || now - sentAt >= IDLE_MS) {
|
||||
send({ seq: ++seq, t: now, moving: now - movedAt < MOVE_QUIET_MS, keyboard: { width: S.kb.width }, hands });
|
||||
}
|
||||
wasRelevant = rel;
|
||||
}, TICK_MS);
|
||||
|
||||
S.dispose = () => {
|
||||
clearInterval(slow); clearInterval(fast);
|
||||
if (active) send({ seq: ++seq, t: performance.now(), moving: false, keyboard: null, hands: { left: null, right: null } });
|
||||
removeProbe();
|
||||
};
|
||||
return 'patched';
|
||||
})
|
||||
@@ -0,0 +1,7 @@
|
||||
# Tests of the controller bridge (flake check `vr-keyboard-controllers`):
|
||||
# tip and laser relative to the keyboard, the hub's page px
|
||||
# (tests/geometry.test.mjs).
|
||||
{ pkgs }:
|
||||
pkgs.runCommand "vr-keyboard-controllers-checks" { nativeBuildInputs = [ pkgs.nodejs ]; } ''
|
||||
node ${./tests/geometry.test.mjs} ${./geometry.js} ${./hub.js} | tee $out
|
||||
''
|
||||
@@ -0,0 +1,59 @@
|
||||
// geometry.js: controllers relative to Steam's VR keyboard
|
||||
// (vr-keyboard-controllers.nix). Pure functions, used by bridge-patch.js
|
||||
// (SteamVR systemui) and tests; consumers may take it as an extraArg too.
|
||||
// Evaluates to { VERSION, rotate, multiply, tipPose, toKeyboard, rayHit,
|
||||
// rotationDegrees }.
|
||||
//
|
||||
// Poses are SteamVR transforms { translation: {x,y,z}, rotation: {w,x,y,z} }.
|
||||
// - tipPose(device, tip): the controller's tip, where SteamVR's laser starts
|
||||
// (the laser mouse's pose is /pose/tip): the device pose
|
||||
// (xfDeviceToAbsoluteTracking) times the render model's "tip" component
|
||||
// (xfTrackingToComponentLocal), or the device pose without one.
|
||||
// - Keyboard kb = { translation, rotation, width }: its top edge centre in
|
||||
// tracking space, +z toward the viewer, width in m (bridge-patch.js).
|
||||
// Keyboard coordinates: u 0..1 left to right, v down from the top edge in
|
||||
// keyboard widths (page px = u * page width, v * page width).
|
||||
// - toKeyboard(p, kb): a point -> { u, v, d } (d: m behind the surface).
|
||||
// - rayHit(pose, kb): the laser (the pose's -z axis) on the keyboard's
|
||||
// plane, from the front -> { u, v, dist (m) } or null.
|
||||
(() => {
|
||||
const VERSION = 1;
|
||||
const sub = (a, b) => ({ x: a.x - b.x, y: a.y - b.y, z: a.z - b.z });
|
||||
const add = (a, b) => ({ x: a.x + b.x, y: a.y + b.y, z: a.z + b.z });
|
||||
// v rotated by the unit quaternion q.
|
||||
function rotate(q, v) {
|
||||
const tx = 2 * (q.y * v.z - q.z * v.y), ty = 2 * (q.z * v.x - q.x * v.z), tz = 2 * (q.x * v.y - q.y * v.x);
|
||||
return {
|
||||
x: v.x + q.w * tx + q.y * tz - q.z * ty,
|
||||
y: v.y + q.w * ty + q.z * tx - q.x * tz,
|
||||
z: v.z + q.w * tz + q.x * ty - q.y * tx,
|
||||
};
|
||||
}
|
||||
const qmul = (a, b) => ({
|
||||
w: a.w * b.w - a.x * b.x - a.y * b.y - a.z * b.z,
|
||||
x: a.w * b.x + a.x * b.w + a.y * b.z - a.z * b.y,
|
||||
y: a.w * b.y - a.x * b.z + a.y * b.w + a.z * b.x,
|
||||
z: a.w * b.z + a.x * b.y - a.y * b.x + a.z * b.w,
|
||||
});
|
||||
const conj = (q) => ({ w: q.w, x: -q.x, y: -q.y, z: -q.z });
|
||||
// a * b (b in a's frame).
|
||||
const multiply = (a, b) => ({ translation: add(a.translation, rotate(a.rotation, b.translation)), rotation: qmul(a.rotation, b.rotation) });
|
||||
const tipPose = (device, tip) => (tip ? multiply(device, tip) : { translation: { ...device.translation }, rotation: { ...device.rotation } });
|
||||
const local = (p, kb) => rotate(conj(kb.rotation), sub(p, kb.translation));
|
||||
|
||||
function toKeyboard(p, kb) {
|
||||
const l = local(p, kb);
|
||||
return { u: l.x / kb.width + 0.5, v: -l.y / kb.width, d: -l.z };
|
||||
}
|
||||
function rayHit(pose, kb) {
|
||||
const o = local(pose.translation, kb), dir = rotate(conj(kb.rotation), rotate(pose.rotation, { x: 0, y: 0, z: -1 }));
|
||||
if (o.z <= 0 || dir.z >= -1e-6) return null; // behind the plane or not pointing at its front
|
||||
const t = -o.z / dir.z;
|
||||
return { u: (o.x + t * dir.x) / kb.width + 0.5, v: -(o.y + t * dir.y) / kb.width, dist: t };
|
||||
}
|
||||
// Rotation angle of a quaternion (degrees), e.g. of an animated render
|
||||
// model component (the trigger).
|
||||
const rotationDegrees = (q) => (2 * Math.acos(Math.min(1, Math.abs(q.w))) * 180) / Math.PI;
|
||||
|
||||
return { VERSION, rotate, multiply, tipPose, toKeyboard, rayHit, rotationDegrees };
|
||||
})()
|
||||
@@ -0,0 +1,59 @@
|
||||
// hub.js: the controller frames in Steam's SharedJSContext
|
||||
// (vr-keyboard-controllers.nix). A single expression for consumer patches'
|
||||
// extraArgs (like lib/hooks.js): installs window.__sfuiControllers unless an
|
||||
// equal or newer VERSION is there (a newer one takes over the subscribers)
|
||||
// and evaluates to it. relay.mjs calls frame(f) with bridge-patch.js's
|
||||
// frames and lost() when the SteamVR side goes away.
|
||||
// subscribe(fn) -> unsubscribe fn(frame) per frame, fn(null) when lost
|
||||
// last the last frame (null: none / lost)
|
||||
// Each frame's hand points (tip, ray) get the keyboard page's px: x, y
|
||||
// (CSS px of the keyboard popup) and onKeyboard (inside the page), if the
|
||||
// keyboard popup exists. Frames come every ~11 ms while a hand is near the
|
||||
// keyboard, its laser on it or its trigger pulled, else every 250 ms, and
|
||||
// only while SteamVR shows the keyboard (then one with keyboard null).
|
||||
(() => {
|
||||
const VERSION = 1;
|
||||
const G = globalThis;
|
||||
const have = G.__sfuiControllers;
|
||||
if (have && have.version >= VERSION) return have;
|
||||
const subs = have?.subs ?? new Set();
|
||||
const errors = have?.errors ?? { count: 0, last: null };
|
||||
|
||||
let win = null, winAt = -Infinity;
|
||||
function keyboardWindow(now) {
|
||||
if (now - winAt < 1000 && win && !win.closed) return win;
|
||||
winAt = now;
|
||||
win = [...(G.g_PopupManager?.GetPopups?.() || [])].find((p) => p.window?.document.querySelector('[data-key]'))?.window ?? null;
|
||||
return win;
|
||||
}
|
||||
const px = (p, w, h) => {
|
||||
if (!p) return p;
|
||||
const x = p.u * w, y = p.v * w;
|
||||
return { ...p, x, y, onKeyboard: x >= 0 && y >= 0 && x < w && y < h };
|
||||
};
|
||||
function deliver(f) {
|
||||
hub.last = f;
|
||||
for (const fn of [...subs]) {
|
||||
try { fn(f); } catch (e) { errors.count++; errors.last = String(e?.stack ?? e); }
|
||||
}
|
||||
}
|
||||
function frame(f) {
|
||||
if (!f || typeof f !== 'object') return;
|
||||
const now = performance.now(), w = f.keyboard ? keyboardWindow(now) : null;
|
||||
if (w?.innerWidth) {
|
||||
for (const h of ['left', 'right']) {
|
||||
const x = f.hands?.[h];
|
||||
if (x) f.hands[h] = { ...x, tip: px(x.tip, w.innerWidth, w.innerHeight), ray: px(x.ray, w.innerWidth, w.innerHeight) };
|
||||
}
|
||||
}
|
||||
f.receivedAt = now;
|
||||
deliver(f);
|
||||
}
|
||||
const hub = {
|
||||
version: VERSION, subs, errors, last: null,
|
||||
frame, lost: () => deliver(null),
|
||||
subscribe(fn) { subs.add(fn); return () => subs.delete(fn); },
|
||||
};
|
||||
G.__sfuiControllers = hub;
|
||||
return hub;
|
||||
})()
|
||||
@@ -0,0 +1,63 @@
|
||||
// relay.mjs (user service vr-keyboard-controllers-relay): carries the
|
||||
// controller frames from SteamVR's systemui page (bridge-patch.js, 8087; CDP
|
||||
// binding __sfuiCtlOut) to Steam's SharedJSContext (hub.js, 8080:
|
||||
// __sfuiControllers.frame(frame)). Frames are rebuilt from their numbers
|
||||
// (nothing else passes). If the systemui page goes away: lost().
|
||||
const SIDES = {
|
||||
steam: { url: 'http://127.0.0.1:8080/json/list', title: 'SharedJSContext' },
|
||||
vr: { url: 'http://127.0.0.1:8087/json/list', title: 'systemui', binding: '__sfuiCtlOut' },
|
||||
};
|
||||
const live = {}; // side -> evaluate(expression)
|
||||
|
||||
const num = (x) => (Number.isFinite(x) ? +x : null);
|
||||
const point = (p, keys) => {
|
||||
if (!p || typeof p !== 'object') return null;
|
||||
const o = {};
|
||||
for (const k of keys) { o[k] = num(p[k]); if (o[k] === null) return null; }
|
||||
return o;
|
||||
};
|
||||
const hand = (h) => (h && typeof h === 'object'
|
||||
? { tip: point(h.tip, ['u', 'v', 'd']), ray: point(h.ray, ['u', 'v', 'dist']), trigger: num(h.trigger) }
|
||||
: null);
|
||||
function clean(m) {
|
||||
if (!m || typeof m !== 'object' || !Number.isFinite(m.seq) || !Number.isFinite(m.t)) return null;
|
||||
const f = { seq: m.seq, t: m.t, moving: !!m.moving, keyboard: point(m.keyboard, ['width']), hands: { left: hand(m.hands?.left), right: hand(m.hands?.right) } };
|
||||
for (const h of ['left', 'right']) if (f.hands[h] && !f.hands[h].tip) f.hands[h] = null;
|
||||
return f;
|
||||
}
|
||||
|
||||
function forward(payload) {
|
||||
let m;
|
||||
try { m = clean(JSON.parse(payload)); } catch { return; }
|
||||
if (m) live.steam?.(`window.__sfuiControllers?.frame(${JSON.stringify(m)})`);
|
||||
}
|
||||
|
||||
async function connect(side) {
|
||||
const cfg = SIDES[side];
|
||||
const list = await (await fetch(cfg.url, { signal: AbortSignal.timeout(3000) })).json();
|
||||
const t = list.find((x) => x.title === cfg.title && x.webSocketDebuggerUrl);
|
||||
if (!t) throw new Error(`${cfg.title} not found`);
|
||||
const ws = new WebSocket(t.webSocketDebuggerUrl);
|
||||
await new Promise((res, rej) => { ws.onopen = res; ws.onerror = rej; });
|
||||
let id = 0;
|
||||
const send = (method, params = {}) => { if (ws.readyState === WebSocket.OPEN) ws.send(JSON.stringify({ id: ++id, method, params })); };
|
||||
ws.onmessage = (e) => {
|
||||
const m = JSON.parse(e.data);
|
||||
if (m.method === 'Runtime.bindingCalled' && m.params.name === cfg.binding) forward(m.params.payload);
|
||||
};
|
||||
const closed = new Promise((res) => { ws.onclose = res; });
|
||||
if (cfg.binding) { send('Runtime.enable'); send('Runtime.addBinding', { name: cfg.binding }); }
|
||||
live[side] = (expression) => send('Runtime.evaluate', { expression });
|
||||
console.log(`${side}: connected`);
|
||||
await closed;
|
||||
delete live[side];
|
||||
if (side === 'vr') live.steam?.('window.__sfuiControllers?.lost()');
|
||||
console.log(`${side}: disconnected`);
|
||||
}
|
||||
for (const side of Object.keys(SIDES)) {
|
||||
(async () => {
|
||||
for (let failed = false; ; await new Promise((r) => setTimeout(r, 3000))) {
|
||||
try { await connect(side); failed = false; } catch (e) { if (!failed) console.error(`${side}: ${e.message}`); failed = true; }
|
||||
}
|
||||
})();
|
||||
}
|
||||
@@ -0,0 +1,80 @@
|
||||
// geometry.js (tip and laser relative to the keyboard) and hub.js (page px,
|
||||
// subscribers). usage: node geometry.test.mjs <geometry.js> <hub.js>
|
||||
import { readFileSync } from 'node:fs';
|
||||
import assert from 'node:assert/strict';
|
||||
|
||||
const GEO = (0, eval)(readFileSync(process.argv[2], 'utf8'));
|
||||
let passed = 0;
|
||||
const test = (name, f) => { f(); passed++; console.log(`ok ${name}`); };
|
||||
const near = (a, b, eps = 1e-6) => assert.ok(Math.abs(a - b) <= eps, `${a} != ${b}`);
|
||||
const norm = (q) => { const n = Math.hypot(q.w, q.x, q.y, q.z); return { w: q.w / n, x: q.x / n, y: q.y / n, z: q.z / n }; };
|
||||
const axis = (deg, x, y, z) => { const h = (deg * Math.PI) / 360; return { w: Math.cos(h), x: x * Math.sin(h), y: y * Math.sin(h), z: z * Math.sin(h) }; };
|
||||
const ID = { w: 1, x: 0, y: 0, z: 0 };
|
||||
|
||||
// A keyboard as measured live (SGQueryService world transform of the
|
||||
// overlay's scaled frame), 0.738 m wide, page 854 x 280 px.
|
||||
const KB = { translation: { x: -0.8552, y: 0.4129, z: -3.8837 }, rotation: norm({ w: 0.88649, x: -0.11484, y: -0.44456, z: -0.05758 }), width: 0.738 };
|
||||
const PAGE = 854;
|
||||
function world(x, y, front) { // page px, m in front -> tracking space
|
||||
const r = GEO.rotate(KB.rotation, { x: (x / PAGE - 0.5) * KB.width, y: -(y / PAGE) * KB.width, z: front });
|
||||
return { x: r.x + KB.translation.x, y: r.y + KB.translation.y, z: r.z + KB.translation.z };
|
||||
}
|
||||
// The frame controller's render model "tip" component.
|
||||
const TIP = { translation: { x: -0.012694, y: -0.02522, z: 0.020687 }, rotation: axis(-40, 1, 0, 0) };
|
||||
|
||||
test('toKeyboard: identity keyboard', () => {
|
||||
const s = GEO.toKeyboard({ x: 0.25, y: -0.1, z: 0.02 }, { translation: { x: 0, y: 0, z: 0 }, rotation: ID, width: 1 });
|
||||
near(s.u, 0.75); near(s.v, 0.1); near(s.d, -0.02);
|
||||
});
|
||||
test('toKeyboard inverts the keyboard transform', () => {
|
||||
const s = GEO.toKeyboard(world(356, 120, 0.03), KB);
|
||||
near(s.u * PAGE, 356, 1e-4); near(s.v * PAGE, 120, 1e-4); near(s.d, -0.03);
|
||||
});
|
||||
test('tipPose: device pose times the tip component', () => {
|
||||
const dev = { translation: { x: 1, y: 2, z: 3 }, rotation: axis(90, 0, 1, 0) };
|
||||
const p = GEO.tipPose(dev, TIP);
|
||||
// 90 degrees about y: (x, y, z) -> (z, y, -x)
|
||||
near(p.translation.x, 1 + TIP.translation.z); near(p.translation.y, 2 + TIP.translation.y); near(p.translation.z, 3 - TIP.translation.x);
|
||||
assert.deepEqual(GEO.tipPose(dev, null).translation, dev.translation);
|
||||
});
|
||||
test('rayHit: a laser pointing at a key from 30 cm', () => {
|
||||
// the tip 30 cm in front of (356, 120), pointing straight at the keyboard:
|
||||
// tip -z = keyboard -z, i.e. the tip's rotation = the keyboard's
|
||||
const hit = GEO.rayHit({ translation: world(356, 120, 0.3), rotation: KB.rotation }, KB);
|
||||
near(hit.u * PAGE, 356, 1e-4); near(hit.v * PAGE, 120, 1e-4); near(hit.dist, 0.3, 1e-9);
|
||||
// tilted 20 degrees about the keyboard's x axis (pointing down): hits lower
|
||||
const down = GEO.rayHit({ translation: world(356, 120, 0.3), rotation: GEO.multiply({ translation: { x: 0, y: 0, z: 0 }, rotation: KB.rotation }, { translation: { x: 0, y: 0, z: 0 }, rotation: axis(-20, 1, 0, 0) }).rotation }, KB);
|
||||
near(down.v * KB.width, 120 / PAGE * KB.width + 0.3 * Math.tan(Math.PI / 9), 1e-6);
|
||||
near(down.u * PAGE, 356, 1e-4);
|
||||
});
|
||||
test('rayHit: none from behind or pointing away', () => {
|
||||
assert.equal(GEO.rayHit({ translation: world(356, 120, -0.1), rotation: KB.rotation }, KB), null);
|
||||
const away = GEO.multiply({ translation: { x: 0, y: 0, z: 0 }, rotation: KB.rotation }, { translation: { x: 0, y: 0, z: 0 }, rotation: axis(180, 0, 1, 0) }).rotation;
|
||||
assert.equal(GEO.rayHit({ translation: world(356, 120, 0.3), rotation: away }, KB), null);
|
||||
});
|
||||
test('rotationDegrees: the trigger component angle', () => {
|
||||
near(GEO.rotationDegrees(axis(12.5, 0.98, 0.076, -0.178)), 12.5, 1e-6);
|
||||
near(GEO.rotationDegrees(ID), 0);
|
||||
});
|
||||
|
||||
// hub.js with a fake keyboard popup (854 x 280).
|
||||
globalThis.performance ??= { now: () => Date.now() };
|
||||
globalThis.g_PopupManager = { GetPopups: () => [{ window: { innerWidth: 854, innerHeight: 280, closed: false, document: { querySelector: () => ({}) } } }] };
|
||||
const HUB = (0, eval)(readFileSync(process.argv[3], 'utf8'));
|
||||
test('hub: page px and onKeyboard, subscribers, lost', () => {
|
||||
const got = [];
|
||||
const off = HUB.subscribe((f) => got.push(f));
|
||||
HUB.frame({ seq: 1, t: 5, moving: false, keyboard: { width: 0.738 }, hands: {
|
||||
left: { tip: { u: 0.5, v: 0.1, d: -0.01 }, ray: { u: 1.2, v: 0.1, dist: 0.2 }, trigger: 0 }, right: null } });
|
||||
const l = got[0].hands.left;
|
||||
near(l.tip.x, 427); near(l.tip.y, 85.4, 1e-9); assert.equal(l.tip.onKeyboard, true); assert.equal(l.ray.onKeyboard, false);
|
||||
assert.equal(HUB.last, got[0]);
|
||||
HUB.lost();
|
||||
assert.equal(got[1], null); assert.equal(HUB.last, null);
|
||||
off();
|
||||
HUB.frame({ seq: 2, t: 6, keyboard: null, hands: { left: null, right: null } });
|
||||
assert.equal(got.length, 2);
|
||||
assert.equal((0, eval)(readFileSync(process.argv[3], 'utf8')), HUB, 'same version: kept');
|
||||
});
|
||||
|
||||
console.log(`${passed} tests passed`);
|
||||
@@ -0,0 +1,10 @@
|
||||
// Reverts bridge-patch.js in SteamVR's systemui page: stops the loops (a last
|
||||
// { keyboard: null } frame) and removes the keyboard probe transform. Safe
|
||||
// when nothing is patched.
|
||||
(() => {
|
||||
const S = window.__sfuiCtl;
|
||||
if (!S) return 'not patched';
|
||||
try { S.dispose?.(); } catch { /* gone */ }
|
||||
delete window.__sfuiCtl;
|
||||
return 'unpatched';
|
||||
})()
|
||||
@@ -0,0 +1,55 @@
|
||||
# Touch typing on Steam's VR keyboard (steamFrame.keyboard.vr.touchTyping): a
|
||||
# key is pressed when a controller's tip (where SteamVR's laser starts)
|
||||
# passes through it. Independent of keyboard.vr.enable / extraKeys.
|
||||
# - The tips relative to the keyboard come from the controller bridge
|
||||
# (vr-keyboard-controllers.nix, turned on here).
|
||||
# - vr-keyboard-touch/keyboard-patch.js (Steam UI, 8080; patch
|
||||
# "vr-keyboard-touch"): tracker.js per hand (contact, hysteresis); presses
|
||||
# the key with the keyboard's own touch handlers, like a laser press.
|
||||
# Tests: vr-keyboard-touch/check.nix (built before the patch; also flake
|
||||
# check `vr-keyboard-touch`).
|
||||
{ config, pkgs, lib, ... }:
|
||||
let
|
||||
cfg = config.steamFrame.keyboard.vr.touchTyping;
|
||||
uiLib = import ./steam-ui-patches/lib { inherit pkgs; };
|
||||
checks = import ./vr-keyboard-touch/check.nix { inherit pkgs; };
|
||||
ctl = config.steamFrame.keyboard.vr.controllers.files;
|
||||
in {
|
||||
imports = [ ./steam-ui-patches.nix ./vr-keyboard-controllers.nix ];
|
||||
|
||||
options.steamFrame.keyboard.vr.touchTyping = {
|
||||
enable = lib.mkEnableOption ''
|
||||
touch typing on Steam's VR keyboard: a key is pressed when a
|
||||
controller's tip (where the laser starts) touches it, with both hands;
|
||||
the lasers work as before'';
|
||||
depth = lib.mkOption {
|
||||
type = lib.types.numbers.between (-2) 5;
|
||||
default = 0;
|
||||
description = "How far behind the keyboard's surface a touch registers (cm; negative: in front).";
|
||||
};
|
||||
haptics = lib.mkOption {
|
||||
type = lib.types.bool;
|
||||
default = true;
|
||||
description = "A haptic tick when a touch presses a key.";
|
||||
};
|
||||
};
|
||||
|
||||
config = lib.mkIf cfg.enable {
|
||||
steamFrame.keyboard.vr.controllers.enable = true;
|
||||
steamFrame.uiPatches.patches = [ {
|
||||
name = "vr-keyboard-touch";
|
||||
target.title = "SharedJSContext";
|
||||
patch = pkgs.runCommand "vr-keyboard-touch.js" { nativeBuildInputs = [ pkgs.nodejs ]; } ''
|
||||
: ${checks}
|
||||
cp ${uiLib.mkPatch {
|
||||
name = "vr-keyboard-touch";
|
||||
src = ./vr-keyboard-touch/keyboard-patch.js;
|
||||
opts = { inherit (cfg) haptics; depth = cfg.depth / 100.0; };
|
||||
extraArgs = [ ./vr-keyboard-touch/tracker.js ctl.hub ];
|
||||
}} $out
|
||||
node --check $out
|
||||
'';
|
||||
unpatch = ./vr-keyboard-touch/keyboard-unpatch.js;
|
||||
} ];
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
# Tests of touch typing (flake check `vr-keyboard-touch`): controller tip ->
|
||||
# keyboard surface -> key, press/release hysteresis, two hands
|
||||
# (tests/tracker.test.mjs).
|
||||
{ pkgs }:
|
||||
pkgs.runCommand "vr-keyboard-touch-checks" { nativeBuildInputs = [ pkgs.nodejs ]; } ''
|
||||
node ${./tests/tracker.test.mjs} ${./tracker.js} ${../vr-keyboard-controllers/geometry.js} | tee $out
|
||||
''
|
||||
@@ -0,0 +1,119 @@
|
||||
// Touch typing (vr-keyboard-touch.nix; patch name "vr-keyboard-touch"),
|
||||
// injected into Steam's SharedJSContext (8080). mkPatch convention plus two
|
||||
// more arguments: tracker.js and vr-keyboard-controllers' hub.js.
|
||||
//
|
||||
// The controller frames (hub.js; the tips relative to the keyboard) go
|
||||
// through one tracker.js per hand; its events press keys with the keyboard's
|
||||
// own touch handlers, like a laser press (the laser arrives as touch
|
||||
// events): HandleTouchStart at "down" (key highlighted, sound, Steam's long
|
||||
// press: repeating Backspace, the accent popup), HandleTouchEnd at "up"
|
||||
// (Steam types the key, unless its long press already did). The key is the
|
||||
// [data-key] element at the point (skipping MouseHoverBlockerHack, like
|
||||
// Steam's hit test); no key there: nothing. Each hand is its own touch, so
|
||||
// both hands can overlap like two fingers (Steam's multi-touch: Shift held
|
||||
// by one hand). No DOM events: the swipe patch's and the extra keys'
|
||||
// listeners don't see these touches; the swipe patch's text model does (it
|
||||
// hooks the methods the keys go through).
|
||||
// "cancel" (no pose), frames stopping for STALE_MS, the keyboard moving or
|
||||
// closing end a touch without typing. Debugging: __sfuiTouchTypeLog.
|
||||
((find, sigs, opts, hooks, TR, HUB) => {
|
||||
const VERSION = 2;
|
||||
const G = window;
|
||||
const STALE_MS = 1000; // Steam's UI thread can stall frames for ~0.3 s
|
||||
const SNAP = 3; // SteamVR overlay haptic effect (1 ButtonEnter, 3 Snap)
|
||||
const HANDS = ['left', 'right'];
|
||||
const ID = { left: 0x5f10, right: 0x5f11 }; // touch identifiers
|
||||
let status;
|
||||
try {
|
||||
const mods = find.resolveAll(find.getWebpackRequire('webpackChunksteamui'), sigs);
|
||||
status = mods.vrStatus.exports.holder;
|
||||
} catch (e) {
|
||||
return `Steam internals changed, no touch typing: ${e.message}`;
|
||||
}
|
||||
const stamp = `${VERSION}:${TR.VERSION}:${HUB.version} ${JSON.stringify(opts)}`;
|
||||
if (G.__sfuiTouchType?.stamp === stamp) return 'unchanged';
|
||||
try { G.__sfuiTouchType?.dispose?.(); } catch { /* gone */ }
|
||||
|
||||
const log = (...a) => {
|
||||
const l = (G.__sfuiTouchTypeLog ??= []);
|
||||
l.push([Math.round(performance.now()), ...a]);
|
||||
if (l.length > 200) l.splice(0, l.length - 200);
|
||||
};
|
||||
const kbInst = (el) => find.findFiberUp(el, (f) => typeof f.stateNode?.HandleTouchStart === 'function' &&
|
||||
typeof f.stateNode?.HandleTouchEnd === 'function' && f.stateNode.m_mapTouched instanceof Set, 200)?.stateNode;
|
||||
const touches = new Map(); // hand -> { t, inst, doc }
|
||||
|
||||
function keyboardWindow() {
|
||||
const p = [...(G.g_PopupManager?.GetPopups?.() || [])].find((x) => x.window?.document.querySelector('[data-key]'));
|
||||
return p?.window ?? null;
|
||||
}
|
||||
// Steam's handlers read target, changedTouches, touches and the touches'
|
||||
// target / clientX / clientY.
|
||||
const event = (t, inst) => ({
|
||||
type: 'touch', target: t.target, changedTouches: [t],
|
||||
touches: [...touches.values()].filter((e) => e.inst === inst && e.t !== t).map((e) => e.t),
|
||||
preventDefault() {}, stopPropagation() {},
|
||||
});
|
||||
|
||||
function end(hand, { cancel = false, u, v } = {}) {
|
||||
const e = touches.get(hand);
|
||||
if (!e) return;
|
||||
touches.delete(hand);
|
||||
// Where it left, if on the page (an accent picked in Steam's popup);
|
||||
// Steam types nothing for a release off the page.
|
||||
const win = e.doc.defaultView, x = u * win?.innerWidth, y = v * win?.innerWidth;
|
||||
if (!cancel && x >= 0 && y >= 0 && x < win.innerWidth && y < win.innerHeight) Object.assign(e.t, { clientX: x, clientY: y, pageX: x, pageY: y });
|
||||
if (cancel) e.inst.m_mapTouched.delete(e.t.target); // Steam types only keys still in m_mapTouched
|
||||
try { e.inst.HandleTouchEnd(event(e.t, e.inst)); } catch (err) { log('end-error', String(err)); }
|
||||
log(hand, cancel ? 'cancel' : 'up', e.t.target.getAttribute('data-key'));
|
||||
}
|
||||
|
||||
function down(hand, u, v) {
|
||||
end(hand, { cancel: true });
|
||||
const win = keyboardWindow();
|
||||
if (!win) return;
|
||||
const doc = win.document, x = u * win.innerWidth, y = v * win.innerWidth;
|
||||
const el = doc.elementsFromPoint(x, y).find((e) => e.id !== 'MouseHoverBlockerHack');
|
||||
const keyEl = el?.closest?.('[data-key]');
|
||||
const inst = keyEl && kbInst(keyEl);
|
||||
if (!inst) { log(hand, 'miss', Math.round(x), Math.round(y)); return; }
|
||||
const t = { identifier: ID[hand], target: keyEl, clientX: x, clientY: y, pageX: x, pageY: y };
|
||||
touches.set(hand, { t, inst, doc });
|
||||
try { inst.HandleTouchStart(event(t, inst)); } catch (err) { log('start-error', String(err)); end(hand, { cancel: true }); return; }
|
||||
if (opts.haptics) win.SteamClient?.OpenVR?.TriggerOverlayHapticEffect?.(SNAP, 0);
|
||||
log(hand, 'down', keyEl.getAttribute('data-key'), Math.round(x), Math.round(y));
|
||||
}
|
||||
|
||||
const trackers = Object.fromEntries(HANDS.map((h) => [h, TR.createTracker({ depth: opts.depth })]));
|
||||
let lastT = 0, lastAt = -Infinity;
|
||||
function step(hand, sample, t) {
|
||||
const ev = trackers[hand].update(sample, t);
|
||||
if (ev?.phase === 'down') down(hand, ev.u, ev.v);
|
||||
else if (ev?.phase === 'up') end(hand, { u: ev.u, v: ev.v });
|
||||
else if (ev?.phase === 'cancel') end(hand, { cancel: true });
|
||||
}
|
||||
const unsubscribe = HUB.subscribe((f) => {
|
||||
const now = performance.now();
|
||||
const t = f?.t ?? lastT + (now - lastAt);
|
||||
lastT = t; lastAt = now;
|
||||
const usable = !!(f?.keyboard && !f.moving && status.VRKeyboardStatus?.bIsOpen);
|
||||
for (const h of HANDS) step(h, usable ? f.hands?.[h]?.tip ?? null : null, t);
|
||||
});
|
||||
// No frames (relay or SteamVR gone) or the keyboard closed: end the touches.
|
||||
const watchdog = setInterval(() => {
|
||||
const now = performance.now();
|
||||
if (now - lastAt > STALE_MS || !status.VRKeyboardStatus?.bIsOpen) {
|
||||
for (const h of HANDS) if (trackers[h].pressed || touches.has(h)) { step(h, null, lastT + (now - lastAt)); end(h, { cancel: true }); }
|
||||
}
|
||||
}, 250);
|
||||
|
||||
G.__sfuiTouchType = {
|
||||
stamp,
|
||||
dispose() {
|
||||
unsubscribe(); clearInterval(watchdog);
|
||||
for (const h of HANDS) end(h, { cancel: true });
|
||||
},
|
||||
};
|
||||
log('attached', VERSION);
|
||||
return 'patched';
|
||||
})
|
||||
@@ -0,0 +1,10 @@
|
||||
// Reverts keyboard-patch.js in Steam's SharedJSContext: unsubscribes from the
|
||||
// controller frames and ends its touches without typing. Safe when nothing is
|
||||
// patched. (window.__sfuiControllers, shared with other consumers, stays.)
|
||||
(() => {
|
||||
const S = window.__sfuiTouchType;
|
||||
if (!S) return 'not patched';
|
||||
try { S.dispose?.(); } catch { /* gone */ }
|
||||
delete window.__sfuiTouchType;
|
||||
return 'unpatched';
|
||||
})()
|
||||
@@ -0,0 +1,142 @@
|
||||
// tracker.js: controller tip -> keyboard surface -> key (through
|
||||
// vr-keyboard-controllers' geometry.js, as the bridge computes it),
|
||||
// press/release hysteresis, two hands.
|
||||
// usage: node tracker.test.mjs <tracker.js> <geometry.js>
|
||||
import { readFileSync } from 'node:fs';
|
||||
import assert from 'node:assert/strict';
|
||||
|
||||
const TR = (0, eval)(readFileSync(process.argv[2], 'utf8'));
|
||||
const GEO = (0, eval)(readFileSync(process.argv[3], 'utf8'));
|
||||
let passed = 0;
|
||||
const test = (name, f) => { f(); passed++; console.log(`ok ${name}`); };
|
||||
const near = (a, b, eps = 1e-6, msg) => assert.ok(Math.abs(a - b) <= eps, `${msg ?? ''} ${a} != ${b}`);
|
||||
|
||||
// A keyboard as measured live: world transform of its scaled frame
|
||||
// (SGQueryService), 0.738 m wide; page 854 x 280 px.
|
||||
const KB = {
|
||||
translation: { x: -0.8552, y: 0.4129, z: -3.8837 },
|
||||
rotation: { w: 0.88649, x: -0.11484, y: -0.44456, z: -0.05758 },
|
||||
width: 0.738,
|
||||
};
|
||||
const PAGE = { w: 854, h: 280 };
|
||||
// Keys of a row (page px), like Steam's German layout.
|
||||
const KEYS = { f: [265, 97, 60, 47], g: [325.6, 97, 60.4, 47], h: [386, 97, 60.4, 47], Backspace: [760, 3, 90, 47] };
|
||||
const keyAt = (u, v) => {
|
||||
const x = u * PAGE.w, y = v * PAGE.w;
|
||||
return Object.entries(KEYS).find(([, [kx, ky, w, h]]) => x >= kx && x < kx + w && y >= ky && y < ky + h)?.[0] ?? null;
|
||||
};
|
||||
// World point of keyboard-local (page px x, y; m in front of the surface).
|
||||
const norm = (q) => { const n = Math.hypot(q.w, q.x, q.y, q.z); return { w: q.w / n, x: q.x / n, y: q.y / n, z: q.z / n }; };
|
||||
KB.rotation = norm(KB.rotation);
|
||||
function world(x, y, front) {
|
||||
const l = { x: (x / PAGE.w - 0.5) * KB.width, y: -(y / PAGE.w) * KB.width, z: front };
|
||||
const r = GEO.rotate(KB.rotation, l);
|
||||
return { x: r.x + KB.translation.x, y: r.y + KB.translation.y, z: r.z + KB.translation.z };
|
||||
}
|
||||
// The frame controller's tip component (render model "tip").
|
||||
const TIP = { translation: { x: -0.012694, y: -0.02522, z: 0.020687 }, rotation: { w: 0.93969, x: -0.34202, y: 0, z: 0 } };
|
||||
const HAND_ROT = norm({ w: 0.8294, x: 0.2383, y: -0.3948, z: 0.3154 });
|
||||
// The device pose that puts the tip at p.
|
||||
function deviceFor(p) {
|
||||
const o = GEO.rotate(HAND_ROT, TIP.translation);
|
||||
return { translation: { x: p.x - o.x, y: p.y - o.y, z: p.z - o.z }, rotation: HAND_ROT };
|
||||
}
|
||||
const sampleAt = (x, y, front) => GEO.toKeyboard(GEO.tipPose(deviceFor(world(x, y, front)), TIP).translation, KB);
|
||||
|
||||
// Feeds a path [x, y, front(m)] at 90 Hz from t0; returns the events.
|
||||
function run(tr, path, t0 = 0) {
|
||||
const out = [];
|
||||
path.forEach((p, i) => {
|
||||
const t = t0 + i * 11;
|
||||
const ev = tr.update(p && sampleAt(...p), t);
|
||||
if (ev) out.push({ ...ev, t, key: ev.u !== undefined ? keyAt(ev.u, ev.v) : undefined });
|
||||
});
|
||||
return out;
|
||||
}
|
||||
const poke = (x, y, from, to, back, steps = 6) => {
|
||||
const p = [];
|
||||
for (let i = 0; i <= steps; i++) p.push([x, y, from + (to - from) * i / steps]);
|
||||
for (let i = 1; i <= steps; i++) p.push([x, y, to + (back - to) * i / steps]);
|
||||
return p;
|
||||
};
|
||||
const phases = (evs) => evs.map((e) => (e.phase === 'down' ? `down:${e.key}` : e.phase));
|
||||
|
||||
test('a poke presses the key under the tip once, released on the way back', () => {
|
||||
const evs = run(TR.createTracker(), poke(356, 120, 0.05, -0.015, 0.05));
|
||||
assert.deepEqual(phases(evs), ['down:g', 'up']);
|
||||
});
|
||||
test('pointing (tip in front) never presses', () => {
|
||||
const p = [];
|
||||
for (let i = 0; i < 200; i++) p.push([100 + i * 3, 50 + (i % 40), 0.02 + 0.01 * Math.sin(i / 7)]);
|
||||
assert.deepEqual(run(TR.createTracker(), p), []);
|
||||
});
|
||||
test('fast poke between two samples: pressed at the crossing point', () => {
|
||||
const evs = run(TR.createTracker(), [[356, 120, 0.03], [356, 120, 0.01], [356, 120, -0.03], [356, 120, -0.035], [356, 120, 0.02]]);
|
||||
assert.deepEqual(phases(evs), ['down:g', 'up']);
|
||||
// moving sideways: the point where the path meets the surface (f | g | h)
|
||||
const side = run(TR.createTracker(), [[300, 120, 0.03], [300, 120, 0.01], [400, 120, -0.01], [400, 120, 0.02]]);
|
||||
assert.deepEqual(phases(side), ['down:g', 'up']);
|
||||
near(side[0].u * PAGE.w, 350, 1e-3);
|
||||
});
|
||||
test('jitter at the surface while held: one press; release needs 1 cm back', () => {
|
||||
const p = [[356, 120, 0.02], [356, 120, 0.004], [356, 120, -0.002]];
|
||||
for (let i = 0; i < 60; i++) p.push([356, 120, -0.002 - 0.004 * (i % 2)]);
|
||||
assert.deepEqual(phases(run(TR.createTracker(), p)), ['down:g']);
|
||||
});
|
||||
test('re-armed only in front by 5 mm: jitter after a release does not repeat', () => {
|
||||
const p = [...poke(356, 120, 0.02, -0.02, -0.005)]; // released 1.5 cm back, still behind
|
||||
for (let i = 0; i < 30; i++) p.push([356, 120, -0.004 + 0.006 * (i % 2)]); // jitter -4..+2 mm
|
||||
assert.deepEqual(phases(run(TR.createTracker(), p)), ['down:g', 'up']);
|
||||
// ...but out to the front and in again is a second press.
|
||||
const q = [...poke(356, 120, 0.02, -0.02, 0.02), ...poke(356, 120, 0.02, -0.02, 0.02)];
|
||||
assert.deepEqual(phases(run(TR.createTracker(), q)), ['down:g', 'up', 'down:g', 'up']);
|
||||
});
|
||||
test('debounce: no new press within minIntervalMs of a release', () => {
|
||||
const tr = TR.createTracker({ minIntervalMs: 100 });
|
||||
const F = [356, 120, 0.02], B = [356, 120, -0.005];
|
||||
// released at 22 ms; pokes at 44 and 66 ms come too soon, the one at 143 ms counts
|
||||
const p = [F, B, F, F, B, F, B, F, F, F, F, F, F, B];
|
||||
assert.deepEqual(phases(run(tr, p)), ['down:g', 'up', 'down:g']);
|
||||
});
|
||||
test('from behind or around the edge: no press', () => {
|
||||
// starts behind the keyboard, inside
|
||||
assert.deepEqual(run(TR.createTracker(), [[356, 120, -0.05], [356, 120, -0.04], [356, 120, -0.03]]), []);
|
||||
// in front below the keyboard, behind it there, then up into the keys behind
|
||||
const p = [[356, 500, 0.03], [356, 500, 0.01], [356, 500, -0.02], [356, 300, -0.02], [356, 120, -0.02], [356, 120, -0.01]];
|
||||
assert.deepEqual(run(TR.createTracker(), p), []);
|
||||
});
|
||||
test('crossing outside the keyboard: no press', () => {
|
||||
assert.deepEqual(run(TR.createTracker(), poke(-100, 120, 0.03, -0.02, 0.03)), []);
|
||||
assert.deepEqual(run(TR.createTracker(), poke(356, 600, 0.03, -0.02, 0.03)), []);
|
||||
});
|
||||
test('a tracking jump through the surface is no press', () => {
|
||||
assert.deepEqual(run(TR.createTracker(), [[356, 120, 0.06], [356, 120, -0.06], [356, 120, 0.06]]), []);
|
||||
});
|
||||
test('lost tracking while pressed: cancel', () => {
|
||||
const evs = run(TR.createTracker(), [[356, 120, 0.02], [356, 120, -0.005], null, [356, 120, -0.005]]);
|
||||
assert.deepEqual(phases(evs), ['down:g', 'cancel']);
|
||||
});
|
||||
test('depth: the touch registers that far behind the surface', () => {
|
||||
const tr = TR.createTracker({ depth: 0.01 });
|
||||
assert.deepEqual(run(tr, poke(356, 120, 0.03, 0.002 - 0.01, 0.03)), []); // 8 mm in: nothing
|
||||
assert.deepEqual(phases(run(TR.createTracker({ depth: 0.01 }), poke(356, 120, 0.03, -0.015, 0.03))), ['down:g', 'up']);
|
||||
});
|
||||
test('two hands: independent, overlapping presses', () => {
|
||||
const L = TR.createTracker(), R = TR.createTracker();
|
||||
const left = [[300, 120, 0.02], [300, 120, -0.005]]; // f, held
|
||||
for (let i = 0; i < 40; i++) left.push([300, 120, -0.006]);
|
||||
const right = [[356, 300, 0.02], ...poke(400, 120, 0.02, -0.01, 0.02), ...poke(356, 120, 0.02, -0.01, 0.02)];
|
||||
while (right.length < left.length) right.push([400, 120, 0.03]);
|
||||
left.push([300, 120, 0.02]);
|
||||
right.push([400, 120, 0.03]);
|
||||
const evs = [];
|
||||
left.forEach((p, i) => {
|
||||
for (const [hand, tr, q] of [['L', L, p], ['R', R, right[i]]]) {
|
||||
const ev = tr.update(sampleAt(...q), i * 11);
|
||||
if (ev) evs.push(`${hand}:${ev.phase}${ev.phase === 'down' ? `:${keyAt(ev.u, ev.v)}` : ''}`);
|
||||
}
|
||||
});
|
||||
assert.deepEqual(evs, ['L:down:f', 'R:down:h', 'R:up', 'R:down:g', 'R:up', 'L:up']);
|
||||
});
|
||||
|
||||
console.log(`${passed} tests passed`);
|
||||
@@ -0,0 +1,66 @@
|
||||
// tracker.js: when a controller's tip touches a key (vr-keyboard-touch.nix;
|
||||
// argument of keyboard-patch.js, tested by tests/tracker.test.mjs).
|
||||
// Evaluates to { VERSION, DEFAULTS, createTracker }.
|
||||
//
|
||||
// createTracker(opts): one hand. update(sample, tMs) with the tip relative
|
||||
// to the keyboard ({ u, v, d } of vr-keyboard-controllers' geometry.js:
|
||||
// keyboard widths from the top left corner, m behind the surface; null: no
|
||||
// pose / keyboard unusable) returns null or an event { phase, u, v }:
|
||||
// down the tip passed through the surface (at `depth` behind it) from
|
||||
// the front between two samples, at u, v (interpolated) inside the
|
||||
// keyboard, not faster than maxStep per sample, minIntervalMs
|
||||
// after the last release;
|
||||
// up it came back out: release m back from its deepest point, or in
|
||||
// front of the surface by arm; u, v where it is now;
|
||||
// cancel the sample was null while down.
|
||||
// A press needs the tip in front by arm (re-armed) since the last one, so
|
||||
// tracking jitter at the surface can't repeat it, and the tip coming from
|
||||
// behind or around the keyboard never presses.
|
||||
(() => {
|
||||
const VERSION = 1;
|
||||
const DEFAULTS = {
|
||||
depth: 0, // m behind the drawn surface where a touch registers
|
||||
arm: 0.005, // m in front of it to re-arm
|
||||
release: 0.01, // m back from the deepest point to release
|
||||
maxV: 0.5, // keyboard height bound (widths; the page is ~0.33)
|
||||
maxStep: 0.08, // m per sample; more is a tracking jump, not a touch
|
||||
minIntervalMs: 60, // release -> next press
|
||||
};
|
||||
|
||||
function createTracker(opts = {}) {
|
||||
const O = { ...DEFAULTS, ...opts };
|
||||
let prev = null, armed = false, pressed = false, deepest = 0, releasedAt = -Infinity;
|
||||
const inside = (u, v) => u >= 0 && u <= 1 && v >= 0 && v <= O.maxV;
|
||||
function update(s, t) {
|
||||
if (!s) {
|
||||
prev = null; armed = false;
|
||||
if (!pressed) return null;
|
||||
pressed = false; releasedAt = t;
|
||||
return { phase: 'cancel' };
|
||||
}
|
||||
const d = s.d - O.depth;
|
||||
let ev = null;
|
||||
if (pressed) {
|
||||
deepest = Math.max(deepest, d);
|
||||
if (d < deepest - O.release || d < -O.arm) {
|
||||
pressed = false; armed = false; releasedAt = t;
|
||||
ev = { phase: 'up', u: s.u, v: s.v };
|
||||
}
|
||||
} else if (armed && prev && prev.d < 0 && d >= 0) {
|
||||
armed = false;
|
||||
const f = prev.d / (prev.d - d); // where the segment meets the surface
|
||||
const u = prev.u + f * (s.u - prev.u), v = prev.v + f * (s.v - prev.v);
|
||||
if (inside(u, v) && d - prev.d <= O.maxStep && t - releasedAt >= O.minIntervalMs) {
|
||||
pressed = true; deepest = d;
|
||||
ev = { phase: 'down', u, v };
|
||||
}
|
||||
} else if (d < -O.arm) armed = true;
|
||||
else if (d >= 0) armed = false;
|
||||
prev = { u: s.u, v: s.v, d };
|
||||
return ev;
|
||||
}
|
||||
return { update, get pressed() { return pressed; } };
|
||||
}
|
||||
|
||||
return { VERSION, DEFAULTS, createTracker };
|
||||
})()
|
||||
Reference in new issue
Block a user