diff --git a/flake.nix b/flake.nix index ee75c60..b049ceb 100644 --- a/flake.nix +++ b/flake.nix @@ -16,6 +16,7 @@ portal = ./modules/portal.nix; keyboard-layout = ./modules/keyboard-layout.nix; steam-keyboard-patch = ./modules/steam-keyboard-patch.nix; + vr-keyboard = ./modules/vr-keyboard.nix; hidden-apps = ./modules/hidden-apps.nix; steam-ui-patches = ./modules/steam-ui-patches.nix; launcher-menu = ./modules/launcher-menu.nix; @@ -37,6 +38,21 @@ default = { imports = builtins.attrValues modules; }; }; + # Tests of the VR keyboard (text model, corrector, swipe decoder on the + # default German + English dictionary): nix flake check + checks = nixpkgs.lib.genAttrs [ "aarch64-linux" "x86_64-linux" ] (system: { + vr-keyboard = (import ./modules/vr-keyboard/build.nix { + pkgs = nixpkgs.legacyPackages.${system}; + dictionary = { + languages = map (l: l // { keepFrequentAbove = 4.0; }) [ + { language = "de"; hunspell = "de_DE"; words = 60000; frequencyOffset = 0.0; } + { language = "en"; hunspell = "en_US"; words = 40000; frequencyOffset = -0.3; } + ]; + contractions = true; extraWords = [ ]; extraWordsFrequency = 5.0; extraWordFiles = [ ]; excludeWords = [ ]; + }; + }).checks; + }); + # nix flake init -t github:lhns/steam-frame-nix templates.default = { path = ./template; diff --git a/modules/lib/signatures.json b/modules/lib/signatures.json index 2d5af58..adf858b 100644 --- a/modules/lib/signatures.json +++ b/modules/lib/signatures.json @@ -446,6 +446,46 @@ "expects": ["SetDockLocation(e){", "SetJustFloatedFromDashboard("] } } + }, + "vr-keyboard": { + "bundle": "steamui", + "modules": { + "vrStatus": { + "module": { "includes": ["get VRKeyboardStatus()"] }, + "exports": { "holder": { "type": "object", "getters": ["VRKeyboardStatus"] } }, + "expects": ["bIsOpen:", "sOverlayKey", "unAppID"] + }, + "keyboardManager": { + "module": { "includes": ["SendClientPasteCommand(){", "HandleVirtualKeyDown(", "GetEnterKeyLabel("] }, + "exports": { "VirtualKeyboardManager": { "type": "class", "protoMethods": ["HandleVirtualKeyDown"] } }, + "expects": ["m_ActiveElementProps", "m_strDeadKeyPending"] + }, + "keyboardComponent": { + "checkOnly": true, + "module": { "includes": ["TypeKeyInternal(", "m_mapTouched", "CancelLongPressTimer(){", "DismissLongPress(){", "ClearHoldTarget(){", + "toggleStates"] }, + "expects": ["\"data-key\"", "\"data-key-row\"", "OnCommitText(", "OnForwardKeyEvent(", "HandleTouchEnd(", "ElementFromTouch("] + } + } + }, + "vr-keyboard-panel": { + "bundle": "vrwebui-systemui", + "modules": { + "sgApp": { + "checkOnly": true, + "module": { "includes": ["window.SGApp=this", "addEmbeddedPanelUVs("] }, + "expects": ["removeEmbeddedPanelUVs(", "t.forceLayoutUpdate=this.forceLayoutUpdate", "getEmbeddedIndex()"] + }, + "keyboardMount": { + "checkOnly": true, + "module": { "includes": ["m_sKeyboardOverlayKey", "GrabHandleTint", "k_nKeyboardGrabTransformOffset"] }, + "expects": ["mountedId", "m_nKeyboardFlags"] + }, + "overlayKeys": { + "checkOnly": true, + "module": { "includes": ["\"valve.steam.gamepadui.keyboard\""] } + } + } } } } diff --git a/modules/vr-keyboard.nix b/modules/vr-keyboard.nix new file mode 100644 index 0000000..2eb732d --- /dev/null +++ b/modules/vr-keyboard.nix @@ -0,0 +1,218 @@ +# Swipe typing, suggestions and a Backspace drag for Steam's VR keyboard. +# - vr-keyboard/patch.js (Steam UI, 8080): swipe gestures (decoder.js), a +# model of what the keyboard typed (textmodel.js), suggestions (swipe +# alternatives, corrections and completions, corrector.js; never changing +# text by themselves), Backspace drag (left: delete with a detent at word +# borders; right: retype), haptic ticks. +# - The suggestion strip: over the number row ("inside"), or a SteamVR +# dashboard panel above/below the keyboard (panel.js in systemui, 8087), +# fed by relay.mjs (user service vr-keyboard-relay). +# - Dictionary: built by gen-dict.py from wordfreq frequency lists, filtered +# and cased by Hunspell (nixpkgs' hunspellDicts). +# Works alongside keyboard.vr.extraKeys (both hook the same keyboard; this +# one resets its text model on the extra keys it can't follow). Tests: the +# checks below (also flake check `vr-keyboard`). +{ config, pkgs, lib, ... }: +let + cfg = config.steamFrame.keyboard.vr; + inherit (lib) mkOption mkEnableOption types; + uiLib = import ./lib { inherit pkgs; }; + + on = cfg.enable && (cfg.swipe.enable || cfg.autocorrect.enable || cfg.completions.enable || cfg.backspaceDrag.enable); + panel = on && cfg.suggestions.position != "inside"; + + # Default dictionary: the language of keyboard.layout (if listed here) plus + # English, else English only. + layoutLanguages = { + de = "de_DE"; fr = "fr-moderne"; es = "es_ES"; it = "it_IT"; nl = "nl_NL"; pt = "pt_PT"; sv = "sv_SE"; + }; + layoutLanguage = let l = config.steamFrame.keyboard.layout; in + if l != null && layoutLanguages ? ${builtins.head (lib.splitString "," l)} then builtins.head (lib.splitString "," l) else null; + defaultLanguages = + lib.optional (layoutLanguage != null) + { language = layoutLanguage; hunspell = layoutLanguages.${layoutLanguage}; words = 60000; frequencyOffset = 0.0; } + ++ [ { language = "en"; hunspell = "en_US"; words = if layoutLanguage != null then 40000 else 60000; + frequencyOffset = if layoutLanguage != null then -0.3 else 0.0; } ]; + + built = import ./vr-keyboard/build.nix { inherit pkgs; inherit (cfg) dictionary; }; + patch = pkgs.runCommand "vr-keyboard.js" { nativeBuildInputs = [ pkgs.nodejs ]; } '' + : ${built.checks} + cp ${uiLib.mkPatch { + name = "vr-keyboard"; + src = ./vr-keyboard/patch.js; + opts = { + swipe = cfg.swipe.enable; + inherit (cfg.text) autoSpace bufferChars resetAfterIdleSeconds; + inherit (cfg.suggestions) position count; + autocorrect = cfg.autocorrect.enable; + inherit (cfg.autocorrect) maxEditDistance; + completions = cfg.completions.enable; + inherit (cfg.completions) minPrefix; + pixelsPerChar = if cfg.backspaceDrag.enable then cfg.backspaceDrag.pixelsPerChar else 0; + inherit (cfg.backspaceDrag) wordDetentPixels; + inherit (cfg) haptics; + }; + extraArgs = [ ./vr-keyboard/decoder.js ./vr-keyboard/textmodel.js ./vr-keyboard/corrector.js built.dictionary ]; + }} $out + node --check $out + ''; + + languageType = types.submodule { + options = { + language = mkOption { type = types.str; example = "fr"; description = "wordfreq language (de, en, fr, es, it, nl, pt, sv, ...)."; }; + hunspell = mkOption { + type = types.nullOr types.str; + example = "fr-moderne"; + description = '' + pkgs.hunspellDicts attribute that filters the words and sets their + casing; null: all words of the frequency list, lowercase. + ''; + }; + words = mkOption { type = types.ints.positive; description = "Most frequent words taken."; }; + frequencyOffset = mkOption { + type = types.number; + default = 0.0; + description = "Added to the words' zipf frequency, e.g. -0.3 for a second language."; + }; + keepFrequentAbove = mkOption { + type = types.nullOr types.number; + default = 4.0; + description = '' + Words Hunspell rejects are kept from this zipf frequency on ("ok", + "lol"; 3+ letters unless zipf >= 5); null: Hunspell only. + ''; + }; + }; + }; +in { + imports = [ ./session.nix ./steam-ui-patches.nix ./keyboard-layout.nix ]; + + options.steamFrame.keyboard.vr = { + enable = mkEnableOption '' + swipe typing, suggestions (swipe alternatives, corrections, completions) + and a Backspace drag on Steam's VR keyboard (a Steam UI patch; the + sub-features below are on by default)''; + + swipe.enable = mkEnableOption "swipe typing" // { default = true; }; + + dictionary = { + languages = mkOption { + type = types.listOf languageType; + default = defaultLanguages; + defaultText = lib.literalMD '' + the language of `steamFrame.keyboard.layout` (de, fr, es, it, nl, pt, + sv; 60000 words) plus English (40000, frequency -0.3), else English + only (60000)''; + description = "Dictionary languages."; + }; + contractions = mkOption { + type = types.bool; + default = true; + description = ''Words with apostrophes ("couldn't", "geht's"), swiped by their letters.''; + }; + extraWords = mkOption { + type = types.listOf types.str; + default = [ ]; + example = [ "SteamVR" "E-Mail" ]; + description = "Words always included (casing as given; ' and - are typed, not swiped)."; + }; + extraWordsFrequency = mkOption { type = types.number; default = 5.0; description = "Zipf frequency of extra words."; }; + extraWordFiles = mkOption { + type = types.listOf types.path; + default = [ ]; + description = ''Word lists: one "word" or "wordzipf" per line.''; + }; + excludeWords = mkOption { type = types.listOf types.str; default = [ ]; description = "Words never suggested (any casing)."; }; + }; + + text = { + bufferChars = mkOption { type = types.ints.between 8 1024; default = 64; description = "Characters of typed text remembered."; }; + resetAfterIdleSeconds = mkOption { + type = types.ints.unsigned; + default = 30; + description = "Forget the remembered text after this long without typing (0: never)."; + }; + autoSpace = mkOption { + type = types.bool; + default = true; + description = "Space before a swiped word after a known non-space, non-opening character."; + }; + }; + + suggestions = { + position = mkOption { + type = types.enum [ "below" "inside" "above" ]; + default = "below"; + description = '' + Suggestion strip: a SteamVR dashboard panel "below" or "above" the + keyboard, moving with it, or "inside" the keyboard over the number + row. + ''; + }; + count = mkOption { type = types.ints.between 1 8; default = 5; description = "Suggestions shown."; }; + }; + + autocorrect = { + enable = mkEnableOption "correction suggestions for finished tapped words not in the dictionary" // { default = true; }; + maxEditDistance = mkOption { type = types.ints.between 1 3; default = 2; description = "Largest edit distance (neighbouring keys, swaps: 0.5)."; }; + }; + completions = { + enable = mkEnableOption "completions of the tapped word" // { default = true; }; + minPrefix = mkOption { type = types.ints.between 1 10; default = 2; description = "Letters typed before completions show."; }; + }; + + backspaceDrag = { + enable = mkEnableOption "Backspace drag: drag left to delete, back right to retype" // { default = true; }; + pixelsPerChar = mkOption { type = types.ints.positive; default = 25; description = "Travel per character (keyboard px; a key is ~60)."; }; + wordDetentPixels = mkOption { + type = types.ints.unsigned; + default = 90; + description = "Detent at a word border: extra travel to delete across it (0: none)."; + }; + }; + + haptics = mkOption { + type = types.bool; + default = true; + description = "Haptic ticks: Backspace-drag steps, a stronger one at word detents, suggestion picks."; + }; + + checks = mkOption { + type = types.package; + readOnly = true; + default = built.checks; + defaultText = lib.literalMD "the tests, built with the configured dictionary"; + description = "The tests (also built with the patch)."; + }; + }; + + config = lib.mkMerge [ + { + assertions = map (l: { + assertion = l.hunspell == null || pkgs.hunspellDicts ? ${l.hunspell}; + message = "steamFrame.keyboard.vr.dictionary.languages: no pkgs.hunspellDicts.${toString l.hunspell}."; + }) cfg.dictionary.languages; + + steamFrame.uiPatches.patches = lib.optional on { + name = "vr-keyboard"; + target.title = "SharedJSContext"; + inherit patch; + unpatch = ./vr-keyboard/unpatch.js; + } ++ lib.optional panel { + name = "vr-keyboard-panel"; + endpoint = "http://127.0.0.1:8087"; + target.title = "systemui"; + patch = pkgs.writeText "vr-keyboard-panel.js" "(${builtins.readFile ./vr-keyboard/panel.js})()"; + unpatch = ./vr-keyboard/unpatch-panel.js; + }; + steamFrame.session.services.${if panel then "restart" else "stop"} = [ "vr-keyboard-relay.service" ]; + } + (lib.mkIf panel { + systemd.user.services.vr-keyboard-relay = { + Unit.Description = "Relay of the VR keyboard's suggestion strip (Steam UI <-> SteamVR dashboard)"; + Service = { ExecStart = "${pkgs.nodejs}/bin/node ${./vr-keyboard/relay.mjs}"; Restart = "always"; RestartSec = 5; }; + Install.WantedBy = [ "default.target" ]; + }; + }) + ]; +} diff --git a/modules/vr-keyboard/build.nix b/modules/vr-keyboard/build.nix new file mode 100644 index 0000000..f7a8b7a --- /dev/null +++ b/modules/vr-keyboard/build.nix @@ -0,0 +1,39 @@ +# Build-time parts of the VR keyboard: the dictionary and the tests. Used by +# vr-keyboard.nix and by the flake's checks..vr-keyboard. +{ pkgs, dictionary }: +let + inherit (pkgs) lib; + d = dictionary; + words = pkgs.runCommand "vr-keyboard-dictionary.js" { } '' + ${pkgs.python3.withPackages (ps: [ ps.wordfreq ])}/bin/python3 ${./gen-dict.py} $out ${pkgs.hunspell}/bin/hunspell \ + ${pkgs.writeText "vr-keyboard-dictionary.json" (builtins.toJSON { + languages = map (l: { + inherit (l) words keepFrequentAbove; + lang = l.language; + offset = l.frequencyOffset; + dict = if l.hunspell == null then null else "${pkgs.hunspellDicts.${l.hunspell}}/share/hunspell/${l.hunspell}"; + }) d.languages; + extra = map (w: [ w d.extraWordsFrequency ]) d.extraWords; + extraFiles = map toString d.extraWordFiles; + extraZipf = d.extraWordsFrequency; + exclude = d.excludeWords; + inherit (d) contractions; + })} + ''; + # The accuracy thresholds (German keyboard geometry, German and English + # words) fail the build only for German + English dictionaries. + strict = lib.sort lib.lessThan (map (l: l.language) d.languages) == [ "de" "en" ]; +in { + dictionary = words; + checks = pkgs.runCommand "vr-keyboard-checks" { nativeBuildInputs = [ pkgs.nodejs ]; } '' + set -o pipefail + node ${./tests/textmodel.test.mjs} ${./textmodel.js} + node ${./tests/corrector.test.mjs} ${./corrector.js} ${./decoder.js} ${words} ${lib.optionalString (!strict) "|| echo warning: corrector test failed"} + node ${./tests/decoder.test.mjs} ${./decoder.js} ${words} 100 1 | tee $out ${lib.optionalString (!strict) "|| true"} + node -e ' + const bad = require("fs").readFileSync(process.argv[1], "utf8").split("\n") + .filter((l) => /sigma 0.25/.test(l) && +l.match(/top-3 ([\d.]+)%/)[1] < 85); + if (bad.length) { console.error("decoder below 85 % top-3:", bad); process.exit(process.argv[2] === "strict" ? 1 : 0); }' \ + $out ${if strict then "strict" else "warn"} + ''; +} diff --git a/modules/vr-keyboard/corrector.js b/modules/vr-keyboard/corrector.js new file mode 100644 index 0000000..fc227fd --- /dev/null +++ b/modules/vr-keyboard/corrector.js @@ -0,0 +1,121 @@ +// corrector.js: suggestions for tap-typed words -- corrections of a finished +// word that isn't in the dictionary, completions of the word being typed. +// Evaluates to { VERSION, create }. +// +// Words are compared by their folded form (lowercase, no ' or -, ä ö ü ß as +// ae oe ue ss), so umlaut spellings and missing apostrophes are free. +// Corrections: weighted Damerau-Levenshtein <= maxDistance (neighbouring keys +// and swapped letters 0.5, other edits 1) against all words of similar +// length, prefiltered by letter counts (an edit changes them by at most 2); +// ranked by distance - 0.1 * zipf. A valid word still gets suggestions if a +// same-folded variant is >= 1 zipf more frequent ("cant" -> "can't"). +// Memory: one folded string + 26 bytes per word. +(() => { + const VERSION = 2; + const fold = (w) => w.toLowerCase().replace(/['’-]/g, '') + .replace(/ä/g, 'ae').replace(/ö/g, 'oe').replace(/ü/g, 'ue').replace(/ß/g, 'ss'); + const VARIANT_GAP = 1.0; + + function create(dict) { + const n = dict.words.length; + const folded = dict.words.map(fold); + const lower = new Map(), byFold = new Map(), byLen = []; + const bags = new Uint8Array(n * 26); + for (let i = 0; i < n; i++) { + const f = folded[i], l = dict.words[i].toLowerCase(); + if (!lower.has(l) || dict.freq[i] > dict.freq[lower.get(l)]) lower.set(l, i); + (byFold.get(f) ?? byFold.set(f, []).get(f)).push(i); + (byLen[f.length] ??= []).push(i); + for (let j = 0; j < f.length; j++) { const c = f.charCodeAt(j) - 97; if (c >= 0 && c < 26) bags[i * 26 + c]++; } + } + const sorted = Int32Array.from({ length: n }, (_, i) => i).sort((a, b) => (folded[a] < folded[b] ? -1 : folded[a] > folded[b] ? 1 : 0)); + + // Substitution costs: 0.5 between keys closer than 1.3 key widths. + const sub = new Float64Array(128 * 128).fill(1); + function setLayout(centres) { + sub.fill(1); + const ks = Object.keys(centres || {}).filter((k) => k.charCodeAt(0) < 128); + for (const a of ks) for (const b of ks) { + if (a !== b && Math.hypot(centres[a][0] - centres[b][0], centres[a][1] - centres[b][1]) < 1.3) sub[a.charCodeAt(0) * 128 + b.charCodeAt(0)] = 0.5; + } + } + + // Weighted optimal-string-alignment distance; Infinity once a row exceeds max. + const R0 = new Float64Array(64), R1 = new Float64Array(64), R2 = new Float64Array(64); + function distance(s, t, max) { + const m = s.length, k = t.length; + if (Math.abs(m - k) > max || m > 62 || k > 62) return Infinity; + let prev2 = R2, prev = R0, cur = R1; + for (let j = 0; j <= k; j++) prev[j] = j; + for (let i = 1; i <= m; i++) { + cur[0] = i; + let rowMin = i; + for (let j = 1; j <= k; j++) { + const a = s.charCodeAt(i - 1), b = t.charCodeAt(j - 1); + let v = Math.min(prev[j] + 1, cur[j - 1] + 1, prev[j - 1] + (a === b ? 0 : sub[(a & 127) * 128 + (b & 127)])); + if (i > 1 && j > 1 && a === t.charCodeAt(j - 2) && s.charCodeAt(i - 2) === b) v = Math.min(v, prev2[j - 2] + 0.5); + cur[j] = v; + if (v < rowMin) rowMin = v; + } + if (rowMin > max) return Infinity; + [prev2, prev, cur] = [prev, cur, prev2]; + } + return prev[k]; + } + + // Casing as typed: ALL CAPS, Capitalised, else the dictionary's ("haus" -> "Haus"). + function shapeLike(typed, w) { + if (typed.length > 1 && typed === typed.toUpperCase() && typed !== typed.toLowerCase()) return w.toUpperCase(); + if (typed[0] !== typed[0].toLowerCase()) return w[0].toUpperCase() + w.slice(1); + return w; + } + // Up to max distinct words (as shaped), in the given order, without typed. + function pickWords(typed, indices, max) { + const out = [], seen = new Set([typed.toLowerCase()]); + for (const i of indices) { + if (out.length >= max) break; + const w = shapeLike(typed, dict.words[i]); + if (!seen.has(w.toLowerCase())) { seen.add(w.toLowerCase()); out.push(w); } + } + return out; + } + + function corrections(typed, { maxDistance = 2, max = 4 } = {}) { + const f = fold(typed); + if (max <= 0 || f.length < 2 || !/^[a-z]+$/.test(f)) return []; + const own = lower.get(typed.toLowerCase()); + const cost = new Map(); // word index -> distance + for (const i of byFold.get(f) || []) if (own === undefined || dict.freq[i] >= dict.freq[own] + VARIANT_GAP) cost.set(i, 0.1); + if (own === undefined) { + const bag = new Int16Array(26); + for (let j = 0; j < f.length; j++) bag[f.charCodeAt(j) - 97]++; + for (let len = Math.max(1, f.length - maxDistance); len <= f.length + maxDistance; len++) { + for (const i of byLen[len] || []) { + if (cost.has(i)) continue; + let diff = 0; + for (let c = 0, o = i * 26; c < 26 && diff <= 2 * maxDistance; c++) diff += Math.abs(bags[o + c] - bag[c]); + if (diff > 2 * maxDistance) continue; + const d = distance(f, folded[i], maxDistance); + if (d <= maxDistance) cost.set(i, d); + } + } + } + const score = (i) => cost.get(i) - 0.1 * dict.freq[i]; + return pickWords(typed, [...cost.keys()].sort((a, b) => score(a) - score(b)), max); + } + + function completions(prefix, { max = 5 } = {}) { + const f = fold(prefix); + if (max <= 0 || !/^[a-z]+$/.test(f)) return []; + let lo = 0, hi = n; + while (lo < hi) { const mid = (lo + hi) >> 1; if (folded[sorted[mid]] < f) lo = mid + 1; else hi = mid; } + const hits = []; + for (let j = lo; j < n && folded[sorted[j]].startsWith(f); j++) hits.push(sorted[j]); + return pickWords(prefix, hits.sort((a, b) => dict.freq[b] - dict.freq[a]), max); + } + + return { corrections, completions, setLayout, distance }; + } + + return { VERSION, create, fold }; +})() diff --git a/modules/vr-keyboard/decoder.js b/modules/vr-keyboard/decoder.js new file mode 100644 index 0000000..727620e --- /dev/null +++ b/modules/vr-keyboard/decoder.js @@ -0,0 +1,229 @@ +// decoder.js: swipe path -> words. Evaluates to { VERSION, parseDict, layout, +// decode, ... }. SHARK2-style template matching (Kristensson & Zhai 2004): +// each word's ideal path runs through its key centres; candidates starting +// and ending near the path's ends, with every key near the path, are scored +// by point distances of the resampled paths (proportional and DTW), how +// closely the path passes their keys in order, path length and frequency. +// Distances are in key widths. ' and - are typed, not swiped. +(() => { + const VERSION = 3; + const N = 32; // resample points + const SKIP = new Set(["'", '-', '\u2019']); + // Cost weights and pruning limits (tuned with tests/decoder.test.mjs; distances in key widths). + const W = { + loc: 4.4, // mean point distance, proportional alignment + dtw: 2.0, // mean point distance, dynamic time warping + letters: 0.5, // mean letter-to-path distance, in order + extra: 1.0, // mean path-to-template distance + length: 0.5, // |log(path length / template length)| + freq: 0.3, // zipf frequency (0..8) + startSlack: 0.9, endSlack: 1.0, // start/end keys: beyond the nearest key + letterMax: 1.1, // max distance of any key from the path + smooth: 2, // path smoothing: +- samples of 0.1 key widths + shortlist: 150, // candidates that get the costly terms (dtw, extra) + }; + + // "word\tzipf*10\n..." -> { words, freq } + function parseDict(text) { + const words = [], freq = []; + for (const line of text.split('\n')) { + const i = line.indexOf('\t'); + if (i <= 0) continue; + words.push(line.slice(0, i)); + freq.push(+line.slice(i + 1) / 10); + } + return { words, freq }; + } + + const dist = (a, b) => Math.hypot(a[0] - b[0], a[1] - b[1]); + function pathLength(pts) { + let l = 0; + for (let i = 1; i < pts.length; i++) l += dist(pts[i - 1], pts[i]); + return l; + } + // n points equally spaced along the polyline pts (flat [x0,y0,x1,y1,...]). + function resample(pts, n) { + const out = new Float64Array(n * 2); + if (pts.length === 1) { for (let i = 0; i < n; i++) { out[2 * i] = pts[0][0]; out[2 * i + 1] = pts[0][1]; } return out; } + const total = pathLength(pts); + if (total === 0) return resample([pts[0]], n); + const step = total / (n - 1); + let seg = 1, segStart = 0, segLen = dist(pts[0], pts[1]); + for (let i = 0; i < n; i++) { + const target = Math.min(i * step, total); + while (segStart + segLen < target && seg < pts.length - 1) { segStart += segLen; seg++; segLen = dist(pts[seg - 1], pts[seg]); } + const t = segLen > 0 ? (target - segStart) / segLen : 0; + out[2 * i] = pts[seg - 1][0] + (pts[seg][0] - pts[seg - 1][0]) * t; + out[2 * i + 1] = pts[seg - 1][1] + (pts[seg][1] - pts[seg - 1][1]) * t; + } + return out; + } + + // Keyboard layout: keys = { char: [centreX, centreY] } in any unit, unit = + // key width in that unit. A word's path uses its letters only (apostrophes + // and hyphens are typed but not swiped: "couldn't" = c-o-u-l-d-n-t, next to + // "couldnt" if the dictionary had it; both are separate candidates). Words + // with another character the layout lacks are left out. Returns an object for decode() (templates are built lazily). + function layout(dict, keys, unit) { + const centre = {}; + for (const [c, p] of Object.entries(keys)) centre[c] = [p[0] / unit, p[1] / unit]; + const chars = Object.keys(centre); + const buckets = new Map(); // first key + last key -> word indices + const seqs = new Array(dict.words.length); // collapsed key sequence per word + for (let i = 0; i < dict.words.length; i++) { + const w = dict.words[i].toLowerCase(); + let seq = '', ok = true; + for (const c of w) { + if (SKIP.has(c)) continue; // "couldn't" is swiped as "couldnt" + if (!centre[c]) { ok = false; break; } + if (seq[seq.length - 1] !== c) seq += c; + } + if (!ok || [...seq].length < 2) continue; + seqs[i] = seq; + const cs = [...seq], b = cs[0] + cs[cs.length - 1]; + let list = buckets.get(b); + if (!list) buckets.set(b, list = []); + list.push(i); + } + return { dict, centre, chars, buckets, seqs, templates: new Map() }; + } + + function template(lay, seq) { + let t = lay.templates.get(seq); + if (!t) { + const pts = [...seq].map((c) => lay.centre[c]); + t = { pts: resample(pts, N), length: pathLength(pts), keys: pts }; + lay.templates.set(seq, t); + } + return t; + } + + // Distance of (x, y) to the polyline pts ([[x, y], ...]). + function segDist(x, y, pts) { + let m = Infinity; + for (let i = 1; i < pts.length; i++) { + const [ax, ay] = pts[i - 1], [bx, by] = pts[i]; + const dx = bx - ax, dy = by - ay, l2 = dx * dx + dy * dy; + const t = l2 ? Math.max(0, Math.min(1, ((x - ax) * dx + (y - ay) * dy) / l2)) : 0; + m = Math.min(m, Math.hypot(x - ax - t * dx, y - ay - t * dy)); + } + return m; + } + // Mean point distance of the resampled paths a and b under dynamic time + // warping (band of N/4), normalised by the warping path length. + const dtwRow = new Float64Array((N + 1) * (N + 1)), dtwLen = new Float64Array((N + 1) * (N + 1)); + function dtw(a, b) { + const band = N >> 2, S = N + 1; + dtwRow.fill(Infinity); dtwRow[0] = 0; dtwLen[0] = 0; + for (let i = 1; i <= N; i++) { + for (let j = Math.max(1, i - band); j <= Math.min(N, i + band); j++) { + const d = Math.hypot(a[2 * i - 2] - b[2 * j - 2], a[2 * i - 1] - b[2 * j - 1]); + let k = (i - 1) * S + j - 1; // diagonal + if (dtwRow[(i - 1) * S + j] < dtwRow[k]) k = (i - 1) * S + j; + if (dtwRow[i * S + j - 1] < dtwRow[k]) k = i * S + j - 1; + dtwRow[i * S + j] = dtwRow[k] + d; dtwLen[i * S + j] = dtwLen[k] + 1; + } + } + return dtwRow[N * S + N] / dtwLen[N * S + N]; + } + + // Keys within `slack` key widths of the nearest key to p. + function nearKeys(lay, p, slack) { + const d = lay.chars.map((c) => [c, dist(lay.centre[c], p)]); + const min = Math.min(...d.map((x) => x[1])); + return d.filter((x) => x[1] <= min + slack).map((x) => x[0]); + } + + const cost = (f) => W.loc * f.loc + W.dtw * f.dtw + W.letters * f.letters + W.extra * f.extra + + W.length * f.length - W.freq * f.freq; + // Laser shake adds length and wiggles: resample the path every 0.1 key + // widths and average over +-W.smooth samples (ends kept). + function smooth(path) { + const k = W.smooth; + if (!k || path.length < 3) return path; + const n = Math.max(2, Math.ceil(pathLength(path) / 0.1) + 1); + const r = resample(path, n), out = []; + for (let i = 0; i < n; i++) { + let x = 0, y = 0, c = 0; + for (let j = Math.max(0, i - k); j <= Math.min(n - 1, i + k); j++) { x += r[2 * j]; y += r[2 * j + 1]; c++; } + out.push(i === 0 || i === n - 1 ? [r[2 * i], r[2 * i + 1]] : [x / c, y / c]); + } + return out; + } + + // rawPath in the layout's unit before division by `unit`; returns + // [{ word, cost }] best first (opts.features: every candidate's cost terms). + function decode(lay, rawPath, opts = {}) { + const { max = 5, unit = 1 } = opts; + const path = smooth(rawPath.map((p) => [p[0] / unit, p[1] / unit])); + if (path.length < 2) return []; + const plen = pathLength(path); + const P = resample(path, N); + const dense = resample(path, Math.max(N, Math.ceil(plen * 8))); // ~8 points per key width + const denseN = dense.length / 2; + // Min distance from every key to the path (for pruning). + const keyDist = {}; + for (const c of lay.chars) { + const [x, y] = lay.centre[c]; + let m = Infinity; + for (let i = 0; i < denseN; i++) m = Math.min(m, Math.hypot(dense[2 * i] - x, dense[2 * i + 1] - y)); + keyDist[c] = m; + } + const starts = nearKeys(lay, path[0], W.startSlack); + const ends = nearKeys(lay, path[path.length - 1], W.endSlack); + const out = []; + for (const s of starts) for (const e of ends) { + for (const i of lay.buckets.get(s + e) || []) { + const seq = lay.seqs[i]; + let ok = true; + for (const c of seq) if (keyDist[c] > W.letterMax) { ok = false; break; } + if (!ok) continue; + const t = template(lay, seq); + let loc = 0; + for (let k = 0; k < N; k++) loc += Math.hypot(P[2 * k] - t.pts[2 * k], P[2 * k + 1] - t.pts[2 * k + 1]); + loc /= N; + // Letters in order: each key's nearest dense point at or after the + // previous letter's (greedy, monotone). + let from = 0, lettersCost = 0; + for (const [x, y] of t.keys) { + let best = Infinity, bi = from; + for (let j = from; j < denseN; j++) { + const d = Math.hypot(dense[2 * j] - x, dense[2 * j + 1] - y); + if (d < best) { best = d; bi = j; } + } + lettersCost += best; from = bi; + } + lettersCost /= t.keys.length; + out.push({ i, t, f: { + loc, letters: lettersCost, freq: lay.dict.freq[i], + length: Math.abs(Math.log((plen + 0.5) / (t.length + 0.5))), dtw: 0, extra: 0, + } }); + } + } + // The costly terms only for the best candidates by the cheap ones. + let list = out; + if (!opts.features && out.length > W.shortlist) { + for (const c of out) c.cost = cost(c.f); + list = out.sort((a, b) => a.cost - b.cost).slice(0, W.shortlist); + } + for (const c of list) { + c.f.dtw = dtw(P, c.t.pts); + for (let k = 0; k < N; k++) c.f.extra += segDist(P[2 * k], P[2 * k + 1], c.t.keys); + c.f.extra /= N; + c.cost = cost(c.f); + } + if (opts.features) return list.map((c) => ({ word: lay.dict.words[c.i], f: c.f })); + list.sort((a, b) => a.cost - b.cost); + const res = [], seen = new Set(); + for (const { i, cost } of list) { + const w = lay.dict.words[i]; + if (seen.has(w)) continue; + seen.add(w); + res.push({ word: w, cost }); + if (res.length >= max) break; + } + return res; + } + + return { VERSION, N, W, cost, parseDict, layout, decode, resample }; +})() diff --git a/modules/vr-keyboard/gen-dict.py b/modules/vr-keyboard/gen-dict.py new file mode 100644 index 0000000..7d7d718 --- /dev/null +++ b/modules/vr-keyboard/gen-dict.py @@ -0,0 +1,180 @@ +# Builds the dictionary at build time: the most frequent words per language +# (wordfreq), kept if Hunspell accepts them, in Hunspell's spelling and casing +# (wordfreq lowercases and folds ß: "strasse" is tried as "straße"; lowercase +# only if accepted without compounding, else Capitalised; "essen"/"Essen" +# both, the second slightly less frequent). Contractions Hunspell rejects +# ("geht's") are kept by frequency, other rejected words if their zipf is +# >= keepFrequentAbove (3+ letters, or zipf >= 5: "ok"). +# usage: python3 gen-dict.py +# config: { languages: [{ lang, words, offset (zipf), dict (path or null), +# keepFrequentAbove (zipf or null) }], +# extra: [[word, zipf]], extraFiles: [path], extraZipf, exclude: [word], +# contractions: bool } +# Output: a JS string literal "wordzipf*10\n..." (most frequent first). +import itertools, json, re, shutil, subprocess, sys +import wordfreq + +out, hunspell, config = sys.argv[1:] +cfg = json.load(open(config)) +# Word letters: German/English a-z ä ö ü ß; other languages any lowercase +# letter (accents). Contractions ("couldn't", "geht's"): letters around +# apostrophes (wordfreq splits words at hyphens: no hyphenated words). +def word_patterns(lang): + L = "a-zäöüß" if lang in ("de", "en") else r"^\W\d_A-Z" + return re.compile(rf"^[{L}]{{2,24}}$"), re.compile(rf"^[{L}]+(?:'[{L}]+)+$") +SECONDARY = 5 # frequency penalty (zipf/10) of the other casing +cap = lambda w: w[0].upper() + w[1:] + + +def spellings(w): + """w, then the variants with some "ss" written as "ß" (wordfreq folds ß).""" + idx = [m.start() for m in re.finditer("ss", w)][:3] + out = [w] + for r in range(1, len(idx) + 1): + for combo in itertools.combinations(idx, r): + s = w + for i in sorted(combo, reverse=True): + s = s[:i] + "ß" + s[i + 2:] + out.append(s) + return out + + +def check(dic, words): + """The words Hunspell accepts (-G) with dictionary dic.""" + res = subprocess.run([hunspell, "-i", "utf-8", "-d", dic, "-G"], + input="\n".join(words) + "\n", capture_output=True, text=True, check=True) + return set(res.stdout.split("\n")) + + +def strict_dict(dic, tmp): + """dic without compounding: German .dic files also list nouns in lowercase + as compound parts ("wetter" for "Regenwetter"), which Hunspell then accepts + on their own; the strict copy decides the casing. Also returns the + capitalised stems (nouns, names).""" + aff = open(dic + ".aff", encoding="latin-1").read() + aff = "\n".join(l for l in aff.split("\n") if not l.startswith("COMPOUND")) + open(tmp + ".aff", "w", encoding="latin-1").write(aff) # latin-1: bytes unchanged + shutil.copyfile(dic + ".dic", tmp + ".dic") + enc = re.search(r"^SET\s+(\S+)", aff, re.M) + text = open(dic + ".dic", "rb").read().decode(enc.group(1) if enc else "latin-1", "replace") + caps = {l.split("/")[0].strip() for l in text.split("\n")[1:] if l[:1].isupper()} + return tmp, caps + + +best = {} +frequent_all = set() # kept by frequency only (keepFrequentAbove) +verified = set() # accepted by some language's Hunspell +unverified = set() # contractions Hunspell rejects ("geht's"), kept by frequency +def add(form, z): + if z > best.get(form, -1): + best[form] = z + +contractions = cfg.get("contractions", True) +def top_words(lang, n): + """The n most frequent plain words, plus the contractions among them.""" + LETTERS, CONTRACTION = word_patterns(lang) + out, plain = [], 0 + for w in wordfreq.iter_wordlist(lang): + w = w.replace("\u2019", "'") + if LETTERS.match(w): + out.append(w); plain += 1 + if plain >= n: + break + elif contractions and CONTRACTION.match(w) and len(w) <= 24: + out.append(w) + return out + +available = set(wordfreq.available_languages()) +for spec in cfg["languages"]: + lang, n, dic = spec["lang"], int(spec["words"]), spec.get("dict") + keep_above = spec.get("keepFrequentAbove") + frequent = [] + bias = round(float(spec.get("offset", 0)) * 10) + if lang not in available: + sys.exit(f"wordfreq has no language {lang!r}; available: {' '.join(sorted(available))}") + words = top_words(lang, n) + if not dic: + for w in words: + add(w, round(wordfreq.zipf_frequency(w, lang) * 10) + bias) + print(f"{lang}: {len(words)} words (no Hunspell filter)", file=sys.stderr) + continue + variants = {w: spellings(w) for w in words} + query = [f for vs in variants.values() for v in vs for f in (v, cap(v))] + ok = check(dic, query) + sdic, caps = strict_dict(dic, f"strict-{lang}") + strict = check(sdic, [f for f in query if f in ok]) + kept = 0 + for w in words: + z = round(wordfreq.zipf_frequency(w, lang) * 10) + bias + if not any(v in ok or cap(v) in ok for v in variants[w]): + zipf = wordfreq.zipf_frequency(w, lang) + if "'" in w: # Hunspell splits "geht's": keep by frequency + add(w, z) + unverified.add(w) + kept += 1 + elif keep_above is not None and zipf >= keep_above and (len(w) >= 3 or zipf >= 5): + add(w, z) # frequent, but not in Hunspell ("ok", "lol", "colour") + frequent.append(w) + frequent_all.add(w) + continue + for v in variants[w]: + lo, up = v in ok, cap(v) in ok + if not (lo or up): + continue + kept += 1 + verified.update(f for f in (v, cap(v)) if f in ok) + slo, sup = v in strict, cap(v) in strict + if slo: + add(v, z) + if cap(v) in caps: # "essen" / "Essen" + add(cap(v), z - SECONDARY) + elif sup or not lo: # "Wetter", "Haus", "Weihnachtsmarkt" + add(cap(v), z) + else: + add(v, z) + break + print(f"{lang}: {kept} of {len(words)} words kept, {len(frequent)} more by frequency: {' '.join(frequent[:30])}", file=sys.stderr) + +# An unverified contraction that another language has in a checked casing +# ("i'm" from the German list, "I'm" from English): keep only the latter. +checked = {} +for w in best: + if "'" in w and w not in unverified: + checked.setdefault(w.lower(), w) +for w in [w for w in unverified if w in best and w.lower() in checked and checked[w.lower()] != w]: + k = checked[w.lower()] + best[k] = max(best[k], best.pop(w)) +print(f"contractions: {sum(1 for w in best if chr(39) in w)} ({len(unverified)} not in Hunspell)", file=sys.stderr) + +# A frequency-only word that is a contraction without its apostrophe ("dont", +# "thats") would only compete with the real one: dropped. +# Only words no Hunspell accepts ("is" stays although "i's" exists), and only +# if the contraction is more frequent. +contraction_freq = {} +for w, z in best.items(): + if "'" in w: + k = w.replace("'", "").lower() + contraction_freq[k] = max(contraction_freq.get(k, -1), z) +dropped = [w for w in frequent_all if w in best and w not in verified and best[w] < contraction_freq.get(w, -1)] +for w in dropped: + del best[w] +print(f"frequency-only words dropped as contractions without apostrophe: {len(dropped)}", file=sys.stderr) + +extra = [(w, z) for w, z in cfg.get("extra", [])] +for path in cfg.get("extraFiles", []): + for line in open(path, encoding="utf-8"): + parts = line.strip().split("\t") + if parts[0]: + extra.append((parts[0], float(parts[1]) if len(parts) > 1 else None)) +for w, z in extra: + z = cfg.get("extraZipf", 5.0) if z is None else z + best[w] = max(best.get(w, -1), round(float(z) * 10)) +exclude = {w.lower() for w in cfg.get("exclude", [])} +for w in [w for w in best if w.lower() in exclude]: + del best[w] +print(f"extra: {len(extra)}, excluded: {len(exclude)}", file=sys.stderr) + +items = sorted(best.items(), key=lambda kv: (-kv[1], kv[0])) +with open(out, "w", encoding="utf-8") as f: + f.write(json.dumps("\n".join(f"{w}\t{z}" for w, z in items), ensure_ascii=False)) +print(f"total: {len(items)} entries", file=sys.stderr) diff --git a/modules/vr-keyboard/panel.js b/modules/vr-keyboard/panel.js new file mode 100644 index 0000000..b600728 --- /dev/null +++ b/modules/vr-keyboard/panel.js @@ -0,0 +1,200 @@ +// The suggestion strip as a SteamVR dashboard panel above or below Steam's +// VR keyboard (suggestions.position "above" / "below"), injected into +// SteamVR's systemui page (8087). State comes from the keyboard page through relay.mjs: +// __sfuiKbdStrip.show({ seq, items, current, visible, style }); a click calls +// the binding __sfuiStripPick('{"seq":n,"index":i}'). +// +// systemui mounts the keyboard overlay (mountedId "...gamepadui.keyboard") +// under an anchor transform, scaled by its sibling transform; the overlay +// hangs from its top edge (SteamVR's grab handle sits in the page's +// transparent bottom band). Our panel is a child of that anchor, as wide as +// the keyboard, so it follows it: "above" at y = +GAP (origin bottom +// centre); "below" at the keyboard's lower edge (origin top centre), in +// front of the grab handle. Built like +// frame-controls' own popups: a vsg-node panel whose buildNode publishes a +// region of this page (SGApp embedded-UV slot) at a free spot of the page. +(() => { + const VERSION = 12; + const G = window; + const GAP = 0.012, DEPTH = 0.005; // m above the keyboard's top edge / in front of it + // "below": the strip's top edge sits this much above the lower of the + // keyboard's and the grab handle's lower edges (user-tuned), overlapping + // the handle's hit area a little -- so the strip lies in front of the + // handle (+z toward the viewer; the handle is at z 0.01) and gets the laser. + const BELOW_OFFSET = 0.013, BELOW_DEPTH = 0.015; + const O = { heightPx: 110, widthPx: 1700 }; // max page region for the strip texture (px) + const old = G.__sfuiKbdStrip; + if (old?.version === VERSION) return 'unchanged'; + const missing = [ + typeof G.SGApp?.addEmbeddedPanelUVs !== 'function' && 'SGApp.addEmbeddedPanelUVs', + typeof G.SGApp?.removeEmbeddedPanelUVs !== 'function' && 'SGApp.removeEmbeddedPanelUVs', + typeof G.VRHTML?.NextSGID !== 'function' && 'VRHTML.NextSGID', + typeof G.VRHTML?.VROverlay?.TriggerOverlayHapticEffect !== 'function' && 'VRHTML.VROverlay.TriggerOverlayHapticEffect', + typeof G.forceLayoutUpdate !== 'function' && 'forceLayoutUpdate', + ].filter(Boolean); + if (missing.length) return `SteamVR internals changed, no strip panel: missing ${missing.join(', ')}`; + const saved = old?.state ?? null; + try { old?.dispose?.(); } catch { /* gone */ } + + const S = G.__sfuiKbdStrip = { version: VERSION, state: saved, own: null, log: [] }; + const log = (...a) => { S.log.push([Math.round(performance.now()), ...a]); if (S.log.length > 100) S.log.shift(); }; + + const style = document.createElement('style'); + style.id = 'sfui-kbd-strip-style'; + style.textContent = ` + .sfui-kbd-strip { position: fixed; display: flex; box-sizing: border-box; width: ${O.widthPx}px; + background: rgb(35, 38, 46); z-index: 1; pointer-events: auto; } + .sfui-kbd-strip > div { flex: 1 1 0; min-width: 0; display: flex; align-items: center; justify-content: center; + white-space: nowrap; overflow: hidden; text-overflow: ellipsis; box-sizing: border-box; } + .sfui-kbd-strip > div:hover { filter: brightness(1.6); }`; + document.head.appendChild(style); + + const fiberProps = (e) => { + const k = Object.keys(e).find((x) => x.startsWith('__reactFiber$')); + let f = k && e[k]; + for (let i = 0; f && i < 6; f = f.return, i++) if (f.memoizedProps?.mountedId) return f.memoizedProps; + return null; + }; + // The keyboard's anchor transform (see header) and its overlay scale, or null. + function keyboardAnchor() { + for (const e of document.querySelectorAll('[vsg-type="mountedscenegraph"]')) { + if (!String(fiberProps(e)?.mountedId ?? '').endsWith('gamepadui.keyboard')) continue; + const scaled = e.parentElement, anchor = scaled?.parentElement; + if (!anchor || anchor.tagName !== 'VSG-TRANSFORM') return null; + const scale = parseFloat((scaled.getAttribute('scale') || '1').split(' ')[0]) || 1; + let width = 1; + try { width = VRHTML.VROverlay.GetWidthInMeters(VRHTML.VROverlay.FindOverlay('valve.steam.gamepadui.keyboard')) || 1; } catch { /* 1 m */ } + // SteamVR's grab handle: a sibling transform (origin bottom centre) whose + // y is where its panel ends; 0.66 m if not found. + const handle = [...anchor.children].find((c) => c !== scaled && !c.querySelector?.('.sfui-kbd-own') && c.querySelector?.('[vsg-type="panel"]')); + const handleBottom = -parseFloat((handle?.getAttribute('translation') || '0 -0.66').split(' ')[1]) || 0.66; + return { anchor, width: width * scale, handleBottom }; + } + return null; + } + // A free spot of the page for the texture region (below every panel). + function freeSpot() { + const rs = [...document.querySelectorAll('[vsg-type="panel"]')] + .filter((p) => !p.classList.contains('sfui-kbd-own')).map((p) => p.getBoundingClientRect()).filter((r) => r.width && r.height); + const W = Math.min(O.widthPx, innerWidth - 4), H = O.heightPx; + for (let y = innerHeight - H - 4; y >= 0; y -= 16) { + if (!rs.some((r) => r.top < y + H + 2 && r.bottom > y - 2 && r.left < W + 4)) return { x: 2, y, W }; + } + return null; + } + + function mount(kb) { + const app = G.SGApp, spot = freeSpot(); + if (!spot) { log('no free spot'); return null; } + const outer = document.createElement('vsg-transform'); + for (const [k, v] of Object.entries({ translation: '0 0 0', rotation: '1 0 0 0', scale: '1 1 1', sgid: VRHTML.NextSGID() })) + outer.setAttribute(k, String(v)); + const node = document.createElement('vsg-node'); + node.id = 'sfui-kbd-strip-panel'; + node.className = 'sfui-kbd-own'; + node.setAttribute('vsg-type', 'panel'); + const sgid = VRHTML.NextSGID(); + node.setAttribute('sgid', String(sgid)); + const content = document.createElement('div'); + content.className = 'sfui-kbd-strip'; + content.style.left = `${spot.x}px`; content.style.top = `${spot.y}px`; content.style.width = `${spot.W}px`; + content.style.height = `${O.heightPx}px`; + for (const t of ['mousedown', 'mouseup', 'click', 'dblclick', 'pointerdown', 'pointerup', 'contextmenu']) + content.addEventListener(t, (e) => e.stopPropagation()); // nothing reaches the dashboard's handlers + content.addEventListener('click', (e) => { + const el = e.target.closest?.('[data-index]'); + if (!el || !S.state) return; + log('pick', S.state.seq, +el.dataset.index); + if (+el.dataset.index !== S.state.current && S.state.haptic) { // tick on this (the laser's) overlay + try { VRHTML.VROverlay.TriggerOverlayHapticEffect(VRHTML.VROverlay.ThisOverlayHandle(), S.state.haptic); } catch (err) { log('haptic', String(err)); } + } + G.__sfuiStripPick?.(JSON.stringify({ seq: S.state.seq, index: +el.dataset.index })); + }); + node.appendChild(content); + outer.appendChild(node); + const fp = { // what SGApp's embedded-UV table reads of a panel + props: { debug_name: node.id }, isExternal: false, m_Rect: { x: 0, y: 0, width: 0, height: 0 }, idx: undefined, + getSGID: () => sgid, getEmbeddedIndex: () => fp.idx, getCurrentRootElement: () => node, + updateLayoutValues() { const r = content.getBoundingClientRect(); fp.m_Rect = { x: r.x, y: r.y, width: r.width, height: r.height }; }, + }; + fp.idx = app.addEmbeddedPanelUVs(fp); + if (fp.idx == null) { log('no embedded UV slot'); return null; } + node.buildNode = (ctx) => { + if (!kb.anchor.isConnected) { queueMicrotask(() => unmount('keyboard gone')); return [ctx, null]; } + const r = content.getBoundingClientRect(), W = innerWidth, H = innerHeight; + return [{ ...ctx, currentPanel: fp, bInsideReparentedPanel: false }, { type: 'panel', properties: { + id: `system.systemui::${node.id}`, sgid, key: 'system.systemui', debug_name: node.id, + width: kb.width, origin: S.state?.position === 'below' ? [0, 1] : [0, -1], interactive: true, scrollable: false, visibility: 0, + 'only-visible-with-laser': false, 'lasermouse-filtering': 0, 'can-take-keyboard-focus': false, + 'sort-depth-bias': -1, 'scale-index': 0, + 'embedded-uv-index': fp.idx, uv_min: [r.x / W, r.y / H], uv_max: [(r.x + r.width) / W, (r.y + r.height) / H], + } }]; + }; + kb.anchor.appendChild(outer); + G.forceLayoutUpdate(); + log('mounted', spot.y, kb.width); + return { outer, node, content, fp, anchor: kb.anchor }; + } + function unmount(why) { + const o = S.own; + if (!o) return; + S.own = null; + o.outer.remove(); + try { G.SGApp.removeEmbeddedPanelUVs(o.fp); } catch (e) { log('remove UVs', String(e)); } + G.forceLayoutUpdate(); + log('unmounted', why); + } + // The keyboard's key look (patch.js keyStyle), scaled from its page + // (pageWidth px across the keyboard) to ours (content width across it). + const FALLBACK = { pageWidth: 854, pageHeight: 280, keyHeight: 44, pad: [1, 1, 2, 2], background: 'rgb(14, 20, 27)', color: 'rgb(255, 255, 255)', + fontFamily: '"Motiva Sans", Arial, Helvetica, sans-serif', fontSize: 16, fontWeight: '400', radius: 0, + border: '0px none rgb(255, 255, 255)', boxShadow: 'none', board: 'rgb(35, 38, 46)', pressed: 'rgb(26, 159, 255)' }; + function applyStyle(content, st) { + const k = { ...FALLBACK, ...(st || {}) }; + const f = content.getBoundingClientRect().width / (k.pageWidth || 854) || 2; + const [pt, pr, pb, pl] = (Array.isArray(k.pad) && k.pad.length === 4 ? k.pad : FALLBACK.pad).map((v) => v * f); + const h = Math.min(O.heightPx, Math.round((k.keyHeight + (k.pad?.[0] ?? 1) + (k.pad?.[2] ?? 2)) * f)); + Object.assign(content.style, { height: `${h}px`, background: k.board, padding: `${pt}px ${pr}px ${pb}px ${pl}px`, gap: `${pl + pr}px` }); + S.applied = { f, h, key: k }; + return { k, f }; + } + function render() { + const st = S.state; + const kb = st?.visible && st.items?.length ? keyboardAnchor() : null; + if (!kb) { unmount(st?.visible ? 'no keyboard' : 'hidden'); return; } + if (S.own && (S.own.anchor !== kb.anchor || !S.own.outer.isConnected)) unmount('anchor changed'); + if (!S.own) S.own = mount(kb); + if (!S.own) return; + const { k, f } = applyStyle(S.own.content, st.style); + // Placement (m, in the keyboard anchor's frame; the keyboard hangs from y = 0). + const kbHeight = kb.width * (k.pageHeight || 280) / (k.pageWidth || 854); + const below = st.position === 'below'; + const y = below ? -(Math.max(kbHeight, kb.handleBottom) - BELOW_OFFSET) : GAP; + S.own.outer.setAttribute('translation', `0 ${y.toFixed(4)} ${below ? BELOW_DEPTH : DEPTH}`); + S.placement = { position: st.position, y, kbHeight, handleBottom: kb.handleBottom }; + S.own.content.replaceChildren(...st.items.map((text, i) => { + const el = document.createElement('div'); + el.textContent = text; + el.dataset.index = i; + Object.assign(el.style, { + background: k.background, color: k.color, fontFamily: k.fontFamily, fontSize: `${k.fontSize * f}px`, + fontWeight: k.fontWeight, borderRadius: `${k.radius * f}px`, border: k.border, boxShadow: k.boxShadow, + }); + // The word as it stands in the text: a thin bar in the keyboard's pressed colour. + if (i === st.current) el.style.boxShadow = `inset 0 ${-Math.max(2, Math.round(3 * f))}px 0 ${k.pressed}`; + el.addEventListener('mousedown', () => { el.style.background = k.pressed; }); + el.addEventListener('mouseleave', () => { el.style.background = k.background; }); + return el; + })); + G.forceLayoutUpdate(); + } + S.show = (state) => { S.state = state; render(); return 'ok'; }; + // The keyboard can appear/disappear without a new state (dashboard toggled). + S.timer = setInterval(() => { + const want = !!(S.state?.visible && S.state.items?.length); + if (want !== !!S.own || (S.own && !S.own.anchor.isConnected)) render(); + }, 500); + S.dispose = () => { clearInterval(S.timer); unmount('dispose'); style.remove(); }; + render(); + return 'patched'; +}) diff --git a/modules/vr-keyboard/patch.js b/modules/vr-keyboard/patch.js new file mode 100644 index 0000000..63271d9 --- /dev/null +++ b/modules/vr-keyboard/patch.js @@ -0,0 +1,522 @@ +// Gestures and suggestions for Steam's VR keyboard (vr-keyboard.nix), +// injected into Steam's SharedJSContext (8080). The keyboard is a popup of +// that context ("SteamVR - Keyboard"); everything works on its document. +// mkPatch convention plus four more arguments: decoder.js, textmodel.js, +// corrector.js and the dictionary text ("wordzipf*10\n..."). +// +// - Swipe: the laser arrives as touch events; Steam types the key a touch +// started on at release. Once a press leaves its first key we drop that +// key from Steam's pending touches (m_mapTouched) and cancel its long press +// (Steam's own touch-end cleanup, minus the typing), then decode the path. +// - Output: one character or "Backspace" per HandleVirtualKeyDown, the path +// of Steam's own keys (incl. extraKeys' xdotool fallback for non-ASCII, +// which is async: we pause after such characters). +// - Text model: an own-property hook on the manager's HandleVirtualKeyDown +// feeds everything the keyboard emits to textmodel.js; one on +// TypeKeyInternal resets it for keys the model can't follow (extraKeys' +// xdotool keys, dead keys) and freezes suggestions at once for taps (their +// character arrives later). Keyboard +// closed/retargeted, another Steam text field, idle: reset. Suggestions +// replace text only while the model proves the characters they replace +// intact. +// - Strip: in the page over the number row ("inside") or as a SteamVR panel +// above/below the keyboard ("above"/"below", panel.js): state out via the +// CDP binding __sfuiStripOut, picks back via +// __sfuiSwipe.remote.pick(seq, index) (relay.mjs). +// All Steam internals are checked first (sigs, instance members); if one is +// missing the keyboard stays stock. Only passive listeners; never blocks +// Steam's events. Debugging: __sfuiSwipeLog, __sfuiSwipePaths +// (scripts/vr-keyboard-replay.mjs). +((find, sigs, opts, hooks, D, T, C, DICT) => { + const VERSION = 23; + const G = window; + const O = opts; + + const log = (...a) => { + const l = (G.__sfuiSwipeLog ??= []); + l.push([Math.round(performance.now()), ...a]); + if (l.length > 300) l.splice(0, l.length - 300); + }; + + // Shared across injections: the parsed dictionary (and decoder layouts, + // corrector index), one detach function per patched keyboard document. + const dictId = `${DICT.length}:${DICT.slice(0, 80)}:${D.VERSION}:${C.VERSION}`; + let S = G.__sfuiSwipe; + if (S?.dictId !== dictId) { + for (const detach of S?.docs?.values() || []) detach(); + S = G.__sfuiSwipe = { dictId, dict: D.parseDict(DICT), layouts: new Map(), corrector: null, docs: new Map() }; + } + for (const [doc, detach] of S.docs) if (!doc.defaultView) { detach(); S.docs.delete(doc); } + + let status, Manager; + try { + const req = find.getWebpackRequire('webpackChunksteamui'); + const mods = find.resolveAll(req, sigs); + status = mods.vrStatus.exports.holder; + Manager = mods.keyboardManager.exports.VirtualKeyboardManager; + find.findModule(req, sigs.keyboardComponent.module, 'keyboardComponent'); + } catch (e) { + return `Steam internals changed, keyboard left stock: ${e.message}`; + } + + const popup = [...(g_PopupManager.GetPopups?.() || [])].find((p) => p.window?.document.querySelector('[data-key]')); + if (!popup) return 'no keyboard popup yet'; + const doc = popup.window.document; + const stamp = `${VERSION} ${JSON.stringify(O)}`; // new code or options: re-attach + if (doc.__sfuiSwipe === stamp && S.docs.has(doc)) return 'unchanged'; + S.docs.get(doc)?.(); + S.docs.delete(doc); + const kbInst = (el) => find.findFiberUp(el, (f) => typeof f.stateNode?.TypeKeyInternal === 'function', 200)?.stateNode; + const inst0 = kbInst(doc.querySelector('[data-key]')); + const missing = [ + !inst0 && 'keyboard component', + !(inst0?.props?.VirtualKeyboardManager instanceof Manager) && 'VirtualKeyboardManager instance', + !(inst0?.m_mapTouched instanceof Set) && 'm_mapTouched', + ...['CancelLongPressTimer', 'DismissLongPress', 'ClearHoldTarget'].filter((m) => typeof inst0?.[m] !== 'function'), + typeof inst0?.state?.toggleStates !== 'object' && 'toggleStates', + ].filter(Boolean); + if (missing.length) return `Steam internals changed, keyboard left stock: missing ${missing.join(', ')}`; + S.docs.set(doc, attach(popup.window, doc)); + doc.__sfuiSwipe = stamp; + return `patched (${S.dict.words.length} words)`; + + function attach(win, doc) { + const cleanup = []; + const isLetter = (k) => typeof k === 'string' && [...k].length === 1 && /\p{L}/u.test(k); + const on = (v) => (v & 7) !== 0; // toggle state: 1 one-shot, 2 locked + const own = (o, k) => Object.prototype.hasOwnProperty.call(o, k); + + // ---- trail and in-page strip ------------------------------------------------- + const inPage = O.position === 'inside'; // else a SteamVR panel (panel.js) + const style = doc.createElement('style'); + style.textContent = ` + #sfui-swipe-trail { position: fixed; left: 0; top: 0; width: 100vw; height: 100vh; pointer-events: none; z-index: 10000; } + #sfui-swipe-strip { position: fixed; left: 2px; right: 2px; z-index: 10001; + background: rgb(35, 38, 46); display: none; gap: 4px; box-sizing: border-box; } + #sfui-swipe-strip.shown { display: flex; } + #sfui-swipe-strip > div { flex: 1 1 0; min-width: 0; display: flex; align-items: center; justify-content: center; + background: rgb(35, 38, 46); color: rgb(220, 222, 226); border-radius: 4px; + font-family: "Motiva Sans", Arial, Helvetica, sans-serif; white-space: nowrap; overflow: hidden; text-overflow: ellipsis; } + #sfui-swipe-strip > div.current { background: rgb(26, 105, 170); color: white; } + #sfui-swipe-strip > div.hover { outline: 2px solid rgba(255, 255, 255, 0.6); outline-offset: -2px; }`; + const canvas = doc.createElement('canvas'); + canvas.id = 'sfui-swipe-trail'; + const strip = doc.createElement('div'); + strip.id = 'sfui-swipe-strip'; + doc.head.appendChild(style); + doc.body.append(canvas, strip); + cleanup.push(() => { style.remove(); canvas.remove(); strip.remove(); }); + const ctx = canvas.getContext('2d'); + let fadeTimer = 0; + cleanup.push(() => clearTimeout(fadeTimer)); + const clearTrail = () => ctx.clearRect(0, 0, canvas.width, canvas.height); + function drawTrail(pts, color, fadeMs = 0) { + const dpr = win.devicePixelRatio || 1, w = Math.round(win.innerWidth * dpr), h = Math.round(win.innerHeight * dpr); + if (canvas.width !== w || canvas.height !== h) { canvas.width = w; canvas.height = h; } + clearTimeout(fadeTimer); + clearTrail(); + ctx.save(); + ctx.scale(dpr, dpr); + Object.assign(ctx, { lineWidth: 6, lineCap: 'round', lineJoin: 'round', strokeStyle: color }); + ctx.beginPath(); + pts.forEach(([x, y], i) => (i ? ctx.lineTo(x, y) : ctx.moveTo(x, y))); + ctx.stroke(); + ctx.restore(); + if (fadeMs) fadeTimer = setTimeout(clearTrail, fadeMs); + } + + // ---- key geometry -> decoder layout (cached per key positions) ------------- + function currentLayout() { + const keys = {}, widths = []; + for (const el of doc.querySelectorAll('[data-key]')) { + const k = el.getAttribute('data-key'), r = el.getBoundingClientRect(); + if (!isLetter(k) || !r.width) continue; + keys[k.toLowerCase()] = [r.x + r.width / 2, r.y + r.height / 2]; + widths.push(r.width); + } + if (widths.length < 10) return null; + const sig = Object.entries(keys).map(([k, [x, y]]) => `${k}${Math.round(x)},${Math.round(y)}`).join(' '); + let lay = S.layouts.get(sig); + if (!lay) { + if (S.layouts.size >= 4) S.layouts.clear(); + const unit = widths.sort((a, b) => a - b)[widths.length >> 1]; + lay = { unit, keys, dec: D.layout(S.dict, keys, unit) }; + S.layouts.set(sig, lay); + } + return lay; + } + + // ---- text model, hooks, resets -------------------------------------------------- + const model = T.create({ size: O.bufferChars }); + S.model = model; + let lastActivity = Date.now(); + const activity = () => { lastActivity = Date.now(); }; + let refreshQueued = false; + const refreshLater = () => { + if (refreshQueued) return; + refreshQueued = true; + queueMicrotask(() => { refreshQueued = false; showStrip(); }); + }; + const reset = (why, o) => { model.reset(why, o); log('reset', why); refreshLater(); }; + + // Own-property hooks calling the (possibly keyboard-patch-wrapped) + // prototype method; re-installed if the instance changes. + function hook(obj, name, make) { + if (!obj || (own(obj, name) && obj[name].__sfuiSwipe === hook)) return; + // The prototype's current method (the keyboard patch may re-wrap it). + const f = make((self, args) => Object.getPrototypeOf(obj)[name].apply(self, args)); + f.__sfuiSwipe = hook; + obj[name] = f; + cleanup.push(() => { if (own(obj, name) && obj[name] === f) delete obj[name]; }); + } + const TOGGLES = new Set(['Shift', 'CapsLock', 'Control', 'Alt', 'AltGr']); + function hookInst(inst) { + if (!inst) return; + const mgr = inst.props?.VirtualKeyboardManager; + hook(mgr, 'HandleVirtualKeyDown', (call) => function (key, ...rest) { + if (this === mgr) { + if (!emitting) log('k', key); // what the keyboard typed (not our own output) + if (this.m_strDeadKeyPending) reset('dead key'); + else if (!model.observe(key)) log('reset', `key ${key}`); + activity(); refreshLater(); + } + return call(this, [key, ...rest]); + }); + hook(inst, 'TypeKeyInternal', (call) => function (st) { + const key = st?.strKey, ts = this.state?.toggleStates || {}; + if (typeof key === 'string' && !TOGGLES.has(key) && !/^(SwitchKeys_|IME_)/.test(key)) { + // Extra keys typed by keyboard.vr.extraKeys with xdotool (chords, Esc, + // arrows, AltGr Delete: text after the cursor) and dead keys: the + // model can't follow them. + if (on(ts.Control) || on(ts.Alt) || key.startsWith('VKX_') || st.strDeadKeyNext || (on(ts.Shift) && key.startsWith('Arrow'))) { + reset(`key ${key}`); + } else { model.freeze(); refreshLater(); } + } + activity(); + return call(this, [st]); + }); + } + hookInst(inst0); + + const ids = new WeakMap(); + let nextId = 1; + const idOf = (o) => (o && typeof o === 'object' ? (ids.get(o) ?? (ids.set(o, nextId), nextId++)) : 0); + let lastTarget = null, idle = false; + const poll = setInterval(() => { + const s = status.VRKeyboardStatus; + const target = [s?.bIsOpen, s?.sOverlayKey, s?.unAppID, idOf(inst0.props?.VirtualKeyboardManager?.m_ActiveElementProps)].join('|'); + if (lastTarget !== null && target !== lastTarget) reset(`target ${target}`, { boundary: true }); + lastTarget = target; + const quiet = O.resetAfterIdleSeconds > 0 && Date.now() - lastActivity > O.resetAfterIdleSeconds * 1000; + // Idle: the cursor may have moved; the next word starts a new known text. + // (A boundary only matters for suggestions, whose replacements stay exact.) + if (quiet && !idle) reset('idle', { boundary: true }); + idle = quiet; + }, 500); + cleanup.push(() => clearInterval(poll)); + + // ---- output: serialised operations ---------------------------------------------- + let queue = Promise.resolve(), busy = 0; + const sleep = (ms) => new Promise((r) => setTimeout(r, ms)); + function run(name, fn) { + busy++; + queue = queue.then(fn).catch((e) => log(`${name}-error`, String(e))).finally(() => { busy--; refreshLater(); }); + } + let emitting = false; + const emitter = (mgr) => (key) => { + emitting = true; + try { mgr.HandleVirtualKeyDown(key); } finally { emitting = false; } + if (key.codePointAt(0) > 127 && G.__vrkbdKey) return sleep(120); + }; + // SteamVR overlay haptic effects: 1 ButtonEnter (light), 3 Snap. + const HAPTIC = { step: 1, border: 3, pick: 1 }; + const haptic = (effect) => O.haptics && win.SteamClient?.OpenVR?.TriggerOverlayHapticEffect?.(effect, 0); + + // ---- suggestions: one strip, last event wins -------------------------------- + // cur: { kind, items, index (what the text shows), anchor } -- swipe (the + // swiped word's alternatives), correct (a finished tapped word not in the + // dictionary: itself, then corrections; anchor incl. the terminator), + // complete (the typed prefix, then its completions). A pick + // replaces the anchored text and keeps the strip, so picks can be + // switched; the strip stays while its anchor is intact. + let cur = null, computedAt = -1; + const corrector = () => (S.corrector ??= C.create(S.dict)); + const valid = (c) => model.anchorIntact(c.anchor); + function suggest() { + if (cur && valid(cur)) return; + cur = null; + if (computedAt === model.version) return; + computedAt = model.version; + const mgr = inst0.props?.VirtualKeyboardManager; + const w = O.autocorrect && model.endedWord(); + if (O.autocorrect && !w && /[\p{L}\p{N}][ .,!?;:]$/u.test(model.text)) log('word-end', 'start unknown'); + if (w) { + const lay = currentLayout(); // neighbouring keys are cheap substitutions + if (lay && S.correctorLayout !== lay) { corrector().setLayout(lay.dec.centre); S.correctorLayout = lay; } + const c = corrector().corrections(w.word, { maxDistance: O.maxEditDistance, max: O.count - 1 }); + log('word-end', w.word, c.length ? c.join(' ') : 'ok'); + if (c.length) cur = { kind: 'correct', items: [w.word, ...c], index: 0, term: w.term, anchor: model.anchor(w.n), mgr }; + } + const p = !cur && O.completions && model.currentWord(); + if (p && [...p.text].length >= O.minPrefix) { + const c = corrector().completions(p.text, { max: O.count - 1 }); + if (c.length) cur = { kind: 'complete', items: [p.text, ...c], index: 0, anchor: model.anchor(p.n), mgr }; + } + if (cur) log('suggest', cur.kind, cur.items.join(' ')); + } + function showStrip() { + if (busy) return; // shown again when the operation is done + const dragging = down?.kind === 'backspace' && down.active; // no suggestions during a Backspace drag + if (!dragging) suggest(); + const shown = !dragging && !!(cur && valid(cur)); + if (!inPage) { publish(shown); return; } + strip.replaceChildren(...(shown ? cur.items : []).map((text, i) => { + const el = doc.createElement('div'); + el.textContent = text; + el.dataset.index = i; + if (i === cur.index) el.className = 'current'; + return el; + })); + strip.classList.toggle('shown', shown); + if (shown) placeStrip(); + } + // The in-page strip covers exactly the number row (up to Backspace). + function placeStrip() { + const row0 = [...doc.querySelectorAll('[data-key-row="0"]')].map((el) => el.getBoundingClientRect()); + if (!row0.length) return; + const top = Math.min(...row0.map((r) => r.top)), height = Math.max(...row0.map((r) => r.bottom)) - top; + const bs = doc.querySelector('[data-key="Backspace"]')?.getBoundingClientRect(); + strip.style.right = bs?.width && bs.top < top + height ? `${Math.max(2, win.innerWidth - bs.x + 2)}px` : '2px'; + Object.assign(strip.style, { top: `${top}px`, height: `${height}px`, fontSize: `${Math.max(10, Math.min(18, height - 10))}px` }); + } + + function commit(inst, results) { + const ts = inst.state.toggleStates; + const upper = on(ts.CapsLock) || (ts.Shift & 2) !== 0, first = !upper && on(ts.Shift); + const shape = (w) => (upper ? w.toUpperCase() : first ? w[0].toUpperCase() + w.slice(1) : w); + if (first && (ts.Shift & 3) === 1) { // release a one-shot Shift, like a typed key + inst.setState((s) => ({ ...s, toggleStates: { ...s.toggleStates, Shift: s.toggleStates.Shift & 4 } })); + } + const mgr = inst.props.VirtualKeyboardManager; + const entry = { kind: 'swipe', items: results.map((r) => shape(r.word)), index: 0, mgr, anchor: null }; + cur = null; + run('commit', async () => { + if (!O.autoSpace) model.reset('no auto-space'); + const r = await model.commitWord(emitter(mgr), entry.items[0]); + entry.anchor = r.anchor; + cur = entry; + log('commit', entry.items[0], r.space ? 'space' : ''); + }); + } + function pick(i) { + const c = cur; + if (!c || i === c.index || c.items[i] === undefined) return; + run('pick', async () => { + if (cur !== c || !valid(c)) return; + const before = model.text.slice(-24); + // A correction keeps the typed terminator; nothing is appended otherwise. + const a = await model.replaceAnchored(c.anchor, emitter(c.mgr), c.items[i] + (c.kind === 'correct' ? c.term : '')); + log('pick', c.kind, c.items[i], !!a, JSON.stringify(before), JSON.stringify(model.text.slice(-24))); + if (a) { + Object.assign(c, { anchor: a, index: i }); // every kind can be switched again + if (inPage) haptic(HAPTIC.pick); // a panel ticks on its own overlay + } else if (cur === c) cur = null; + }); + } + + // ---- strip as a SteamVR panel ("above" / "below") ------------------------------- + // State { seq, items, current, visible, style } out via __sfuiStripOut; + // style: a letter key's look, so the panel's buttons match the keys. + let outSeq = 0, outLast = '', published = null, keyStyleCache = null; + function keyStyle() { + const key = doc.querySelector('[data-key="g"]') || doc.querySelector('[data-key="a"]'); + const face = key?.firstElementChild; + if (!face) return null; + const board = key.closest('[class*="Layout_"]'); + const sig = `${board?.className}|${face.className}|${win.innerWidth}`; + if (keyStyleCache?.sig === sig) return keyStyleCache.style; + const cs = win.getComputedStyle(face), ks = win.getComputedStyle(key); + // Pressed colour: the theme's rule for the key face plus one class. + const faceCls = String(face.className).split(/\s+/)[0], boardCls = String(board?.className || '').split(/\s+/).filter(Boolean); + let pressed = null; + for (const sh of doc.styleSheets) { + let rules; try { rules = sh.cssRules; } catch { continue; } + pressed = [...rules].find((r) => r.selectorText?.includes(`.${faceCls}.`) && !r.selectorText.includes('::') && + !r.selectorText.includes('KeyTheme_') && r.style?.backgroundColor && boardCls.some((c) => r.selectorText.includes(`.${c} `)))?.style.backgroundColor; + if (pressed) break; + } + const style = { + pageWidth: win.innerWidth, pageHeight: win.innerHeight, keyHeight: face.getBoundingClientRect().height, + pad: [ks.paddingTop, ks.paddingRight, ks.paddingBottom, ks.paddingLeft].map((v) => parseFloat(v) || 0), + background: cs.backgroundColor, color: cs.color, fontFamily: cs.fontFamily, fontSize: parseFloat(cs.fontSize) || 16, + fontWeight: cs.fontWeight, radius: parseFloat(cs.borderRadius) || 0, border: cs.border, boxShadow: cs.boxShadow, + board: board ? win.getComputedStyle(board).backgroundColor : undefined, pressed: pressed || undefined, + }; + keyStyleCache = { sig, style }; + return style; + } + function publish(shown, force = false) { + let st = null; + if (shown) try { st = keyStyle(); } catch (e) { log('key-style-error', String(e)); } + const body = JSON.stringify(shown ? { items: cur.items, current: cur.index, visible: true, style: st, haptic: O.haptics ? HAPTIC.pick : 0, position: O.position } : { items: [], current: -1, visible: false }); + if (body === outLast && !force) return; + outLast = body; + published = shown ? cur : null; + try { G.__sfuiStripOut?.(JSON.stringify({ seq: ++outSeq, ...JSON.parse(body) })); } catch (e) { log('publish-error', String(e)); } + } + const remote = { + pick(seq, i) { if (seq === outSeq && published && published === cur) pick(i); else log('stale-pick', seq, outSeq); }, + sync() { outLast = ''; showStrip(); return 'ok'; }, + }; + S.remote = remote; + cleanup.push(() => { + if (!inPage) publish(false, true); + if (S.remote === remote) delete S.remote; + }); + + // ---- gestures ------------------------------------------------------------------ + // down: { kind: 'strip', index } | { kind: 'backspace', ... } | { kind: 'key', ... }. + // Touch is read from touch events (they go on after Chromium's + // pointercancel); a mouse from pointer events, or mouse events if there + // are no pointer events. + let down = null, sawPointer = false; + const keyOf = (t) => t?.closest?.('[data-key]'); + const stripIndexOf = (t) => { + const el = t?.closest?.('#sfui-swipe-strip > div'); + return el && strip.contains(el) ? +el.dataset.index : -1; + }; + const at = (x, y) => doc.elementFromPoint(x, y); + function point(e) { + if (e.type.startsWith('touch')) { + const t = e.type === 'touchend' || e.type === 'touchcancel' ? e.changedTouches[0] : e.touches[0]; + return t && !(e.type === 'touchstart' && e.touches.length > 1) ? [t.clientX, t.clientY, t.target] : null; + } + if (e.type.startsWith('pointer')) { sawPointer = true; if (e.pointerType === 'touch') return null; } + else if (sawPointer) return null; + return [e.clientX, e.clientY, e.target]; + } + // Keep Steam from typing the pressed key at release and from its long press. + function takeOver(d) { + const inst = d.inst; + try { + for (const el of [...inst.m_mapTouched]) if (d.keyEl.contains(el)) inst.m_mapTouched.delete(el); + inst.CancelLongPressTimer(); + inst.DismissLongPress(); + inst.ClearHoldTarget(); + } catch (e) { log('takeover-error', String(e)); } + } + function onDown(e) { + const p = point(e); + if (!p) return; + const [x, y, target] = p; + const si = stripIndexOf(target); + if (si !== -1) { down = { kind: 'strip', index: si }; return; } + const keyEl = keyOf(target), key = keyEl?.getAttribute('data-key'), inst = keyEl && kbInst(keyEl); + down = null; + if (!inst) return; + hookInst(inst); + if (key === 'Backspace' && O.pixelsPerChar > 0) { + down = { kind: 'backspace', keyEl, inst, x0: x, active: false, planned: 0, + g: model.dragStart({ px: O.pixelsPerChar, detentPx: O.wordDetentPixels }) }; + } else if (O.swipe && isLetter(key)) { + const lay = currentLayout(); + if (lay) down = { kind: 'key', key, keyEl, inst, lay, pts: [[x, y]], active: false }; + } + } + // Backspace drag: a character per px of leftward travel (the first at px; + // releasing before is Steam's tap), a detent of detentPx at a word border; + // back right retypes what this drag deleted while the model is sure. + function backspaceMove(d, x) { + const travel = d.x0 - x; + if (!d.active) { + if (travel < O.pixelsPerChar) return; + d.active = true; + takeOver(d); + refreshLater(); // hides the strip + } + const target = d.g.target(travel), emit = emitter(d.inst.props.VirtualKeyboardManager); + const tick = (r) => haptic(r.border ? HAPTIC.border : HAPTIC.step); + for (; d.planned < target; d.planned++) run('backspace', async () => tick(await model.dragDelete(d.g, emit))); + for (; d.planned > target && d.g.restorable; d.planned--) { + run('restore', async () => { + const r = await model.dragRestore(d.g, emit); + if (r) tick(r); else d.planned = d.g.applied; + }); + } + } + function onMove(e) { + const p = point(e); + if (!p || !down || down.kind === 'strip') return; + const [x, y] = p; + if (down.kind === 'backspace') { backspaceMove(down, x); return; } + const q = down.pts[down.pts.length - 1]; + if (Math.hypot(x - q[0], y - q[1]) < 2) return; + down.pts.push([x, y]); + if (!down.active) { + const s = down.pts[0]; + if (Math.hypot(x - s[0], y - s[1]) <= 0.6 * down.lay.unit || keyOf(at(x, y))?.getAttribute('data-key') === down.key) return; + down.active = true; + takeOver(down); + } + drawTrail(down.pts, 'rgba(26, 159, 255, 0.75)'); + } + function onUp(e) { + const p = point(e); + if (!p) return; + const [x, y] = p, d = down; + down = null; + if (d?.kind === 'strip') { // released anywhere on the strip: the button pressed + if (strip.contains(at(x, y))) pick(d.index); + return; + } + if (d?.kind === 'backspace' && d.active) { + run('backspace-log', () => { // after the drag's queued steps + const st = d.g.stats; + log('backspace-drag', `known ${d.g.snap.length}`, `deleted ${st.deleted}`, `restored ${st.restored}`, + `borders ${st.borders}`, `unknown ${st.unknown}`, st.stop || 'ok'); + }); + } + if (d?.kind !== 'key' || !d.active) return; // taps are Steam's + d.pts.push([x, y]); + setTimeout(() => decode(d), 0); // after Steam has handled the release + } + function decode(d) { + let results = []; + try { results = D.decode(d.lay.dec, d.pts, { max: O.count, unit: d.lay.unit }); } catch (err) { log('decode-error', String(err)); } + log('decode', results.map((r) => r.word).join(' ')); + const paths = (G.__sfuiSwipePaths ??= []); + paths.push({ unit: d.lay.unit, keys: d.lay.keys, pts: d.pts.map(([x, y]) => [Math.round(x), Math.round(y)]), top: results.map((r) => r.word) }); + if (paths.length > 20) paths.shift(); + if (!results.length) { drawTrail(d.pts, 'rgba(255, 80, 80, 0.75)', 400); return; } + fadeTimer = setTimeout(clearTrail, 150); + commit(d.inst, results); + } + function onCancel(e) { + if (e.type === 'pointercancel' && e.pointerType === 'touch') return; // touch events go on + if (down?.active) clearTrail(); + down = null; + } + function onHover(e) { + const si = stripIndexOf(e.target); + for (const el of strip.children) el.classList.toggle('hover', +el.dataset.index === si); + } + + const L = { capture: true, passive: true }; + // No 'blur': the VR keyboard window blurs on every touch. + for (const [type, f] of [ + ['touchstart', onDown], ['pointerdown', onDown], ['mousedown', onDown], + ['touchmove', onMove], ['pointermove', onMove], ['mousemove', onMove], + ['touchend', onUp], ['pointerup', onUp], ['mouseup', onUp], + ['touchcancel', onCancel], ['pointercancel', onCancel], + ['pointerover', onHover], ['mouseover', onHover], + ]) { + win.addEventListener(type, f, L); + cleanup.push(() => win.removeEventListener(type, f, L)); + } + log('attached', VERSION); + return () => { + for (const f of cleanup.reverse()) { try { f(); } catch { /* page gone */ } } + delete doc.__sfuiSwipe; + }; + } +}) diff --git a/modules/vr-keyboard/relay.mjs b/modules/vr-keyboard/relay.mjs new file mode 100644 index 0000000..e3c97d2 --- /dev/null +++ b/modules/vr-keyboard/relay.mjs @@ -0,0 +1,76 @@ +// relay.mjs: carries the suggestion strip between Steam's keyboard page +// (SharedJSContext, 8080) and SteamVR's systemui page (8087) for +// suggestions.position "above" / "below". CDP bindings: the keyboard page calls +// __sfuiStripOut(state json) -> systemui __sfuiKbdStrip.show(state); the +// panel calls __sfuiStripPick({ seq, index } json) -> keyboard page +// __sfuiSwipe.remote.pick(seq, index). Only these two messages, validated. +// On (re)connects and page reloads the keyboard page re-sends its state; if +// the keyboard page goes away, the panel is hidden. +const SIDES = { + steam: { url: 'http://127.0.0.1:8080/json/list', title: 'SharedJSContext', binding: '__sfuiStripOut' }, + vr: { url: 'http://127.0.0.1:8087/json/list', title: 'systemui', binding: '__sfuiStripPick' }, +}; +const HIDDEN = { seq: 0, visible: false, current: -1, items: [] }; +const live = {}; // side -> evaluate(expression) +let lastState = HIDDEN; + +// Key style values: numbers, short CSS values that can't end a declaration. +function cleanStyle(st) { + if (!st || typeof st !== 'object') return null; + const out = {}; + for (const [k, v] of Object.entries(st)) { + if (!/^[a-zA-Z]{1,20}$/.test(k)) continue; + if (typeof v === 'number' && Number.isFinite(v)) out[k] = v; + else if (typeof v === 'string' && v.length < 200 && !/[;{}<>\\]/.test(v)) out[k] = v; + else if (Array.isArray(v) && v.length <= 4 && v.every((x) => typeof x === 'number' && Number.isFinite(x))) out[k] = v; + } + return out; +} +const show = (state) => { lastState = state; live.vr?.(`window.__sfuiKbdStrip?.show(${JSON.stringify(state)})`); }; +const resync = () => live.steam?.('window.__sfuiSwipe?.remote?.sync()'); + +function forward(side, payload) { + let msg; + try { msg = JSON.parse(payload); } catch { return; } + if (side === 'steam' && Array.isArray(msg?.items) && typeof msg.seq === 'number') { + show({ seq: msg.seq, visible: !!msg.visible, current: Number(msg.current) | 0, + items: msg.items.slice(0, 8).map((s) => String(s).slice(0, 64)), style: cleanStyle(msg.style), + haptic: Math.max(0, Math.min(5, Number(msg.haptic) | 0)), position: msg.position === 'below' ? 'below' : 'above' }); + } else if (side === 'vr' && typeof msg?.seq === 'number' && typeof msg.index === 'number') { + live.steam?.(`window.__sfuiSwipe?.remote?.pick(${msg.seq | 0}, ${msg.index | 0})`); + } +} + +async function connect(side) { + const cfg = SIDES[side]; + const list = await (await fetch(cfg.url, { signal: AbortSignal.timeout(3000) })).json(); + const t = list.find((x) => x.title === cfg.title && x.webSocketDebuggerUrl); + if (!t) throw new Error(`${cfg.title} not found`); + const ws = new WebSocket(t.webSocketDebuggerUrl); + await new Promise((res, rej) => { ws.onopen = res; ws.onerror = rej; }); + let id = 0; + const send = (method, params = {}) => { if (ws.readyState === WebSocket.OPEN) ws.send(JSON.stringify({ id: ++id, method, params })); }; + ws.onmessage = (e) => { + const m = JSON.parse(e.data); + if (m.method === 'Runtime.bindingCalled' && m.params.name === cfg.binding) forward(side, m.params.payload); + else if (m.method === 'Runtime.executionContextCreated') setTimeout(resync, 3500); // the injector re-patches after ~3 s + }; + const closed = new Promise((res) => { ws.onclose = res; }); + send('Runtime.enable'); + send('Runtime.addBinding', { name: cfg.binding }); + live[side] = (expression) => send('Runtime.evaluate', { expression }); + console.log(`${side}: connected`); + if (side === 'vr') show(lastState); + resync(); + await closed; + delete live[side]; + if (side === 'steam') show(HIDDEN); // no keyboard page: no strip + console.log(`${side}: disconnected`); +} +for (const side of Object.keys(SIDES)) { + (async () => { + for (let failed = false; ; await new Promise((r) => setTimeout(r, 3000))) { + try { await connect(side); failed = false; } catch (e) { if (!failed) console.error(`${side}: ${e.message}`); failed = true; } + } + })(); +} diff --git a/modules/vr-keyboard/tests/corrector.test.mjs b/modules/vr-keyboard/tests/corrector.test.mjs new file mode 100644 index 0000000..36d95eb --- /dev/null +++ b/modules/vr-keyboard/tests/corrector.test.mjs @@ -0,0 +1,66 @@ +// Offline check of corrector.js on the built dictionary. +// usage: node corrector.test.mjs +import { readFileSync } from 'node:fs'; +import assert from 'node:assert/strict'; + +const C = (0, eval)(readFileSync(process.argv[2], 'utf8')); +const D = (0, eval)(readFileSync(process.argv[3], 'utf8')); +const dict = D.parseDict(JSON.parse(readFileSync(process.argv[4], 'utf8'))); +let t = performance.now(); +const c = C.create(dict); +console.log(`index: ${dict.words.length} words, ${(performance.now() - t).toFixed(0)} ms`); +// German QWERTZ key centres in key widths (as patch.js passes them). +const KEYS = { q: [93.8, 73.5], w: [154.4, 73.5], e: [214.9, 73.5], r: [275.5, 73.5], t: [336.1, 73.5], z: [396.7, 73.5], u: [457.2, 73.5], i: [517.8, 73.5], o: [578.4, 73.5], p: [638.9, 73.5], a: [114.1, 120.5], s: [174.5, 120.5], d: [235.0, 120.5], f: [295.4, 120.5], g: [355.8, 120.5], h: [416.2, 120.5], j: [476.7, 120.5], k: [537.1, 120.5], l: [597.5, 120.5], y: [205.6, 167.5], x: [260.7, 167.5], c: [315.8, 167.5], v: [370.9, 167.5], b: [426.0, 167.5], n: [481.1, 167.5], m: [536.2, 167.5] }; +c.setLayout(Object.fromEntries(Object.entries(KEYS).map(([k, [x, y]]) => [k, [x / 60.5, y / 60.5]]))); +let passed = 0; +const ok = (name, fn) => { fn(); passed++; console.log('ok', name); }; +const top = (w, n = 3) => c.corrections(w).slice(0, n); +const timed = []; +const corr = (w) => { const s = performance.now(); const r = c.corrections(w); timed.push(performance.now() - s); return r; }; + +ok('corrections in the top 3', () => { + for (const [typed, want] of [['hsllo', 'hallo'], ['teh', 'the'], ['schoen', 'schön'], ['strasse', 'Straße'], ['cant', "can't"], + ['dont', "don't"], ['vielleciht', 'vielleicht'], ['keyboad', 'keyboard'], ['Tastatrr', 'Tastatur'], ['gehst', null]]) { + const r = corr(typed); + if (want === null) { assert.deepEqual(r, [], typed); continue; } + assert.ok(r.slice(0, 3).includes(want), `${typed} -> ${r.join(', ')}`); + } +}); +ok('no suggestions for valid words', () => { + for (const w of ['hallo', 'Haus', 'haus', 'schön', 'Straße', "can't", 'the', 'keyboard', 'Wetter', 'über', 'gehen']) assert.deepEqual(corr(w), [], w); +}); +ok('capitalisation kept', () => { + assert.ok(top('Hsllo').includes('Hallo'), top('Hsllo').join()); + assert.ok(top('HSLLO').includes('HALLO'), top('HSLLO').join()); + assert.ok(top('Schoen').includes('Schön')); +}); +ok('completions', () => { + const s = performance.now(); + const r = c.completions('Hal'); + timed.push(performance.now() - s); + assert.ok(r.includes('Hallo') || r.includes('Halle') || r.includes('Hälfte'), r.join()); + assert.ok(r.every((w) => w[0] === 'H'), r.join()); + assert.ok(c.completions('schö').includes('schön'), c.completions('schö').join()); + assert.ok(c.completions('could').includes("couldn't"), c.completions('could').join()); + assert.ok(c.completions('the').length > 0 && !c.completions('the').includes('the')); + assert.deepEqual(c.completions('xqzv'), []); +}); +ok('fast enough', () => { + const words = ['hsllo', 'teh', 'vielleciht', 'Donaudampfschifffahrt', 'abcdefghijklmn', 'strasse', 'ich', 'Wettr', 'schreibn', 'kannst']; + for (const w of words) corr(w); + timed.sort((a, b) => a - b); + const p = timed[Math.floor(timed.length * 0.9)], maxT = timed[timed.length - 1]; + console.log(` ${timed.length} lookups: median ${timed[timed.length >> 1].toFixed(1)} ms, p90 ${p.toFixed(1)} ms, max ${maxT.toFixed(1)} ms`); + assert.ok(p < 60, `p90 ${p} ms`); +}); +ok('max is respected (candidates = 1 leaves no room for corrections)', () => { + assert.deepEqual(c.corrections('hsllo', { max: 0 }), []); + assert.equal(c.corrections('hsllo', { max: 1 }).length, 1); + assert.deepEqual(c.completions('Hal', { max: 0 }), []); +}); +ok('single letters and non-letters: nothing', () => { + assert.deepEqual(c.corrections('x'), []); + assert.deepEqual(c.corrections('123'), []); + assert.deepEqual(c.completions(''), []); +}); +console.log(`${passed} corrector tests passed`); diff --git a/modules/vr-keyboard/tests/decoder.test.mjs b/modules/vr-keyboard/tests/decoder.test.mjs new file mode 100644 index 0000000..54c2988 --- /dev/null +++ b/modules/vr-keyboard/tests/decoder.test.mjs @@ -0,0 +1,106 @@ +// Offline check of decoder.js with synthetic swipes on Steam's German VR +// keyboard (key centres measured in the keyboard popup, 854x280 CSS px). +// usage: node decoder.test.mjs [words-per-sample] [seed] +// Each word's path goes through its key centres with random offsets (up to +// about half a key), cuts corners (spline) and jitters. Prints top-1/top-3 +// accuracy for frequent German and English words and a list of fixed words. +import { readFileSync } from 'node:fs'; + +const [, , decoderPath, dictPath, perLang = '300', seedArg = '1'] = process.argv; +const D = (0, eval)(readFileSync(decoderPath, 'utf8')); +const dict = D.parseDict(JSON.parse(readFileSync(dictPath, 'utf8'))); + +const KEYS = { ß: [659.0, 26.5], q: [93.8, 73.5], w: [154.4, 73.5], e: [214.9, 73.5], r: [275.5, 73.5], t: [336.1, 73.5], z: [396.7, 73.5], u: [457.2, 73.5], i: [517.8, 73.5], o: [578.4, 73.5], p: [638.9, 73.5], ü: [699.5, 73.5], a: [114.1, 120.5], s: [174.5, 120.5], d: [235.0, 120.5], f: [295.4, 120.5], g: [355.8, 120.5], h: [416.2, 120.5], j: [476.7, 120.5], k: [537.1, 120.5], l: [597.5, 120.5], ö: [658.0, 120.5], ä: [718.4, 120.5], y: [205.6, 167.5], x: [260.7, 167.5], c: [315.8, 167.5], v: [370.9, 167.5], b: [426.0, 167.5], n: [481.1, 167.5], m: [536.2, 167.5] }; +const UNIT = 60.5; // key width (px) +const t0 = Date.now(); +const lay = D.layout(dict, KEYS, UNIT); +console.log(`dictionary: ${dict.words.length} entries, layout ${Date.now() - t0} ms`); + +let seed = +seedArg; +const rnd = () => { seed = (seed * 1664525 + 1013904223) % 4294967296; return seed / 4294967296; }; +const gauss = () => Math.sqrt(-2 * Math.log(rnd() + 1e-12)) * Math.cos(2 * Math.PI * rnd()); +const clamp = (v, m) => Math.max(-m, Math.min(m, v)); + +let JITTER = 1.5; // px per sample (laser shake) +function swipe(word, sigma) { + const seq = []; + for (const c of word.toLowerCase()) if (KEYS[c] && seq[seq.length - 1] !== c) seq.push(c); + const way = seq.map((c) => [KEYS[c][0] + clamp(gauss() * sigma, 0.45) * UNIT, KEYS[c][1] + clamp(gauss() * sigma, 0.4) * 47]); + // Catmull-Rom through the waypoints, ~3 px steps, plus jitter. + const pts = []; + const P = (i) => way[Math.max(0, Math.min(way.length - 1, i))]; + for (let i = 0; i < way.length - 1; i++) { + const [p0, p1, p2, p3] = [P(i - 1), P(i), P(i + 1), P(i + 2)]; + const steps = Math.max(2, Math.ceil(Math.hypot(p2[0] - p1[0], p2[1] - p1[1]) / 3)); + for (let s = 0; s < steps; s++) { + const t = s / steps, t2 = t * t, t3 = t2 * t; + const f = (a, b, c, d) => 0.5 * (2 * b + (-a + c) * t + (2 * a - 5 * b + 4 * c - d) * t2 + (-a + 3 * b - 3 * c + d) * t3); + pts.push([f(p0[0], p1[0], p2[0], p3[0]) + gauss() * JITTER, f(p0[1], p1[1], p2[1], p3[1]) + gauss() * JITTER]); + } + } + pts.push(way[way.length - 1]); + return pts; +} + +function evaluate(words, sigma, label, show = false) { + let top1 = 0, top3 = 0, ms = 0; + const misses = []; + for (const w of words) { + const t = performance.now(); + const res = D.decode(lay, swipe(w, sigma), { max: 5, unit: UNIT }); + ms += performance.now() - t; + const rank = res.findIndex((r) => r.word === w); + if (rank === 0) top1++; + if (rank >= 0 && rank < 3) top3++; + if (show || rank < 0 || rank >= 3) misses.push(`${w} -> ${res.slice(0, 3).map((r) => r.word).join(', ')}`); + } + const pc = (n) => `${((100 * n) / words.length).toFixed(1)}%`; + console.log(`${label} (n=${words.length}, sigma ${sigma}): top-1 ${pc(top1)}, top-3 ${pc(top3)}, ${(ms / words.length).toFixed(1)} ms/word`); + return misses; +} + +// Frequent words per language: German ones contain an umlaut/ß or are in the +// de list's typical forms; simplest: take dictionary entries by rank and tell +// the language by a list check is not possible offline, so sample fixed ranges. +const pick = (from, to, n) => { + const pool = dict.words.slice(from, to).filter((w) => [...w].length >= 2 && [...w].every((c) => KEYS[c.toLowerCase()] || "'-".includes(c))); + const out = []; + for (let i = 0; i < n && pool.length; i++) out.push(pool.splice(Math.floor(rnd() * pool.length), 1)[0]); + return out; +}; +const n = +perLang; +const frequent = pick(0, 5000, n), mid = pick(5000, 30000, n); +const FIXED_DE = ['hallo', 'danke', 'bitte', 'Haus', 'schön', 'für', 'über', 'Straße', 'heute', 'morgen', 'wir', 'nicht', 'vielleicht', 'Tastatur', 'Wetter', 'gehen', 'können', 'möchte', 'Brille', 'spielen']; +const FIXED_EN = ["couldn't", "don't", "can't", "geht's", 'hello', 'thanks', 'the', 'keyboard', 'swipe', 'typing', 'world', 'would', 'should', 'because', 'people', 'something', 'window', 'games', 'steam']; +for (const sigma of [0.15, 0.25, 0.35]) { + evaluate(frequent, sigma, 'rank <5000'); + evaluate(mid, sigma, 'rank 5000-30000'); +} +const fixedMisses = evaluate([...FIXED_DE, ...FIXED_EN], 0.25, 'fixed de+en', true); +console.log(fixedMisses.join('\n')); +// Short words and contractions, several noisy paths each. +const SHORT = ["let's", "it's", "I'm", "that's", "don't", "can't", "he's", "we're", "geht's", "gibt's", 'lets', 'its', 'the', 'und', 'ist']; +const rep = []; for (let r = 0; r < 5; r++) rep.push(...SHORT); +console.log(evaluate(rep, 0.25, 'short words x5').join('\n')); +// Extra words from the configuration swipe like any other word. +for (const w of ['config', 'GitHub', 'SteamVR']) { + if (!dict.words.includes(w)) continue; + const top = D.decode(lay, swipe(w, 0.15), { max: 3, unit: UNIT }).map((r) => r.word); + console.log(`${w}: ${top.join(', ')}`); + if (top[0] !== w) { console.error(`${w} is not top-1 (sigma 0.15)`); process.exitCode = 1; } +} +// Words that share their letters with a contraction stay (is / i's, well / we'll). +{ + const must = (cond, msg) => { if (!cond) { console.error(msg); process.exitCode = 1; } }; + must(dict.words.includes('is'), '"is" missing from the dictionary'); + const is = D.decode(lay, swipe('is', 0), { max: 3, unit: UNIT }).map((r) => r.word); + console.log(`is: ${is.join(', ')}`); + must(is[0] === 'is', '"is" is not top-1 for an i-s swipe'); + const well = D.decode(lay, swipe('well', 0), { max: 5, unit: UNIT }).map((r) => r.word); + console.log(`well: ${well.join(', ')}`); + must(well.includes('well') && well.includes("we'll"), 'well and we\'ll not both offered'); +} +// Shaky laser: 5 px jitter per sample. +JITTER = 5; +console.log(evaluate(rep, 0.25, 'short words x5, shaky').join('\n')); +evaluate(frequent, 0.25, 'rank <5000, shaky'); diff --git a/modules/vr-keyboard/tests/textmodel.test.mjs b/modules/vr-keyboard/tests/textmodel.test.mjs new file mode 100644 index 0000000..a83600a --- /dev/null +++ b/modules/vr-keyboard/tests/textmodel.test.mjs @@ -0,0 +1,354 @@ +// textmodel.js against a simulated text field. Like patch.js: every emitted +// key reaches the model; a tapped key also freezes anchors first (the +// TypeKeyInternal hook). usage: node textmodel.test.mjs +import { readFileSync } from 'node:fs'; +import assert from 'node:assert/strict'; + +const T = (0, eval)(readFileSync(process.argv[2], 'utf8')); +let passed = 0; +function setup(size = 64) { + const m = T.create({ size }); + const f = { text: '' }; + const raw = (key) => { + if (key === 'Backspace') f.text = [...f.text].slice(0, -1).join(''); + else if ([...key].length === 1) f.text += key; + else if (key === 'Enter') f.text += '\n'; + m.observe(key); + }; + const out = (key) => { m.freeze(); raw(key); }; // a tapped key + // Our output, with a pause after non-ASCII characters (as in the patch). + const emit = async (k) => { raw(k); if (k.codePointAt(0) > 127) await new Promise((r) => setTimeout(r, 1)); }; + const tap = (s) => { for (const c of s) out(c); }; + // The last swipe, as patch.js tracks it. + let last = null; + const swipe = async (w) => { const r = await m.commitWord(emit, w); last = r.anchor; return r; }; + const replace = async (w) => { const a = await m.replaceAnchored(last, emit, w); if (a) last = a; return !!a; }; + return { m, f, out, emit, tap, swipe, replace }; +} +async function test(name, fn) { await fn(); passed++; console.log('ok', name); } + +await test('switch 3 -> 7 -> 2 letters, no space at start', async () => { + const { m, f, emit, swipe, replace } = setup(); + await swipe('die'); + assert.equal(f.text, 'die'); + assert.ok(await replace('dienten')); + assert.equal(f.text, 'dienten'); + assert.ok(await replace('du')); + assert.equal(f.text, 'du'); +}); +await test('auto-space after a swipe, replacement keeps it', async () => { + const { m, f, emit, swipe, replace } = setup(); + await swipe('Hallo'); + await swipe('die'); + assert.equal(f.text, 'Hallo die'); + await replace('dienten'); await replace('du'); + assert.equal(f.text, 'Hallo du'); +}); +await test('umlauts and ß count as one character each', async () => { + const { m, f, emit, tap, swipe, replace } = setup(); + tap('Ja'); + await swipe('schön'); + assert.equal(f.text, 'Ja schön'); + await replace('Straße'); assert.equal(f.text, 'Ja Straße'); + await replace('für'); assert.equal(f.text, 'Ja für'); + await replace('übergrößen'); assert.equal(f.text, 'Ja übergrößen'); +}); +await test('auto-space rules', async () => { + for (const [before, space] of [['Hallo', true], ['Hallo ', false], ['Hallo.', true], ['Hallo,', true], ['wirklich?', true], + ['(', false], ['„', false], ['"', false], ['a-', false], ['', false]]) { + const { m, f, emit, tap, swipe, replace } = setup(); + tap(before); + await swipe('du'); + assert.equal(f.text, before + (space ? ' ' : '') + 'du', `after ${JSON.stringify(before)}`); + } +}); +await test('manual Backspace inside the word freezes it; buffer stays right', async () => { + const { m, f, emit, out, swipe, replace } = setup(); + await swipe('Hallo'); await swipe('Welt'); + out('Backspace'); + assert.equal(f.text, 'Hallo Wel'); + assert.equal(await replace('du'), false); + assert.equal(f.text, 'Hallo Wel'); + // Delete the rest of the word: the auto-space stays, the next swipe adds none. + for (let i = 0; i < 3; i++) out('Backspace'); + assert.equal(f.text, 'Hallo '); + await swipe('du'); + assert.equal(f.text, 'Hallo du'); + // And after deleting the space too, one space again. + for (let i = 0; i < 3; i++) out('Backspace'); + await swipe('du'); + assert.equal(f.text, 'Hallo du'); + assert.ok(await replace('die')); assert.equal(f.text, 'Hallo die'); +}); +await test('a tap after the word freezes it; next swipe spaced', async () => { + const { m, f, emit, tap, swipe, replace } = setup(); + await swipe('Hallo'); + tap('!'); + assert.equal(await replace('Halle'), false); + assert.equal(f.text, 'Hallo!'); + await swipe('wie'); + assert.equal(f.text, 'Hallo! wie'); + tap(' '); + await swipe('geht'); + assert.equal(f.text, 'Hallo! wie geht'); +}); +await test('tap, then Backspace of that tap: word still frozen (never mangles)', async () => { + const { m, f, emit, tap, out, swipe, replace } = setup(); + await swipe('Hallo'); + tap('x'); out('Backspace'); + assert.equal(await replace('Halle'), false); + assert.equal(f.text, 'Hallo'); +}); +await test('Enter resets: no space, no replacement', async () => { + const { m, f, emit, out, swipe, replace } = setup(); + await swipe('Hallo'); + out('Enter'); + assert.equal(await replace('Halle'), false); + await swipe('du'); + assert.equal(f.text, 'Hallo\ndu'); +}); +await test('reset (idle, keyboard closed, target change): no space, frozen', async () => { + const { m, f, emit, swipe, replace } = setup(); + await swipe('Hallo'); + m.reset('idle'); + await swipe('du'); + assert.equal(f.text, 'Hallodu'); + assert.ok(await replace('die')); assert.equal(f.text, 'Hallodie'); +}); +await test('other keys (arrows, Tab) reset', async () => { + const { m, f, emit, out, swipe, replace } = setup(); + await swipe('Hallo'); + out('ArrowLeft'); + assert.equal(await replace('Halle'), false); + await swipe('du'); + assert.equal(f.text, 'Hallodu'); +}); +await test('buffer overflow: bounded, long word beyond the buffer frozen', async () => { + const { m, f, emit, tap, swipe, replace } = setup(8); + tap('abcdefghijklmnop'); + assert.equal(m.text, 'ijklmnop'); + await swipe('du'); + assert.equal(f.text, 'abcdefghijklmnop du'); + assert.equal(m.text.length, 8); + assert.ok(await replace('die')); + await swipe('Donaudampfschiff'); // 1 + 16 > 8: can't prove it intact + assert.equal(await replace('x'), false); + assert.equal(f.text, 'abcdefghijklmnop die Donaudampfschiff'); +}); +await test('deleting past the known text: unknown, no auto-space', async () => { + const { m, f, emit, tap, out, swipe, replace } = setup(); + tap('ab'); out('Backspace'); out('Backspace'); out('Backspace'); + await swipe('du'); + assert.equal(f.text, 'du'); +}); +await test("contractions: apostrophes count as characters", async () => { + const { m, f, emit, tap, swipe, replace } = setup(); + tap('I'); + await swipe("couldn't"); + assert.equal(f.text, "I couldn't"); + assert.ok(await replace('could')); assert.equal(f.text, 'I could'); + assert.ok(await replace("can't")); assert.equal(f.text, "I can't"); + assert.ok(await replace('E-Mail')); assert.equal(f.text, 'I E-Mail'); + tap("'"); + await swipe('geht'); + assert.equal(f.text, "I E-Mail'geht", "' may open a quote: no space"); +}); +// A Backspace drag as patch.js does it: characters deleted after `travel` +// px, with px per character and a detent of detentPx before a word border. +function dragDeletes(borders, travel, px = 25, detentPx = 90) { + let need = 0, k = 0; + for (;;) { + need += px + (borders[k] ? detentPx : 0); + if (need > travel) return k; + k++; + } +} +await test('Backspace drag: word borders', async () => { + for (const [typed, expect] of [['Hallo wie', 'Hallo '], ['Hallo wie ', 'Hallo '], ['Hallo wie. ', 'Hallo '], ['Hallo wie?', 'Hallo '], + ["I couldn't", 'I '], ['per E-Mail', 'per '], ['Hallo', ''], ['Hallo, ', ''], ['a b ', 'a '], ['x(y', 'x(']]) { + const { m, f, tap, out, swipe, replace } = setup(); + tap(typed); + const b = m.wordBorders(); + const stop = b.indexOf(true); + // Far past the word (up to 100 px more), but not the 25 + 90 px of the border. + const n = dragDeletes(b, 25 * (stop === -1 ? b.length : stop) + 100); + if (stop !== -1) assert.equal(n, stop, JSON.stringify(typed)); + for (let i = 0; i < Math.min(n, b.length); i++) out('Backspace'); + assert.equal(f.text, expect, JSON.stringify(typed)); + } +}); +await test('Backspace drag: crossing a border with enough travel, consecutive drags', async () => { + const { m, f, tap, out, swipe, replace } = setup(); + tap('Hallo wie geht'); + const drag = (travel) => { const n = dragDeletes(m.wordBorders(), travel); for (let i = 0; i < n; i++) out('Backspace'); }; + drag(200); // "geht" = 4 x 25 = 100 px; the space needs 25 + 90 more + assert.equal(f.text, 'Hallo wie '); + drag(25 + 90 + 3 * 25 + 10); // across the border: the space and "wie", stops before the next + assert.equal(f.text, 'Hallo '); + drag(25 + 25); // only what's before the cursor's space goes first: " " then "o" + assert.equal(f.text, 'Hall'); + drag(1000); assert.equal(f.text, ''); + const b = m.wordBorders(); + assert.deepEqual(b, [], 'nothing known: no borders (plain per character)'); + f.text = 'unbekannt'; + drag(5 * 25); assert.equal(f.text, 'unbe', 'unknown text: plain per character'); +}); +await test('Backspace drag: swiped word and its auto-space', async () => { + const { m, f, emit, out, swipe, replace } = setup(); + await swipe('Hallo'); await swipe('wie'); + const n = dragDeletes(m.wordBorders(), 150); + for (let i = 0; i < n; i++) out('Backspace'); + assert.equal(f.text, 'Hallo ', 'the swiped word; its auto-space stays (border)'); +}); +// A drag gesture as patch.js runs it: move to `travel`, deletes/restores. +async function dragTo(m, g, emit, travel) { + const t = g.target(travel); + while (g.applied < t) await m.dragDelete(g, emit); + while (g.applied > t) if (!(await m.dragRestore(g, emit))) break; +} +await test('drag back right restores: 5 -> 3 -> 5 -> 0', async () => { + const { m, f, tap, emit, swipe, replace } = setup(); + tap('Hallo Welt'); + const g = m.dragStart({ px: 25, detentPx: 90 }); + await dragTo(m, g, emit, 5 * 25 + 89); assert.equal(f.text, 'Hallo ', 'Welt, stuck before the space'); + await dragTo(m, g, emit, 25 + 5); assert.equal(f.text, 'Hallo Wel', 'restore to 1 deleted'); + await dragTo(m, g, emit, 3 * 25); assert.equal(f.text, 'Hallo W'); + await dragTo(m, g, emit, 5 * 25 + 90); assert.equal(f.text, 'Hallo', 'across the border'); + await dragTo(m, g, emit, -50); assert.equal(f.text, 'Hallo Welt', 'all restored, not more'); + assert.equal(m.text, 'Hallo Welt'); +}); +await test('drag restore across a word border (symmetric detent)', async () => { + const { m, f, tap, emit, swipe, replace } = setup(); + tap('ab cd'); + const g = m.dragStart({ px: 25, detentPx: 90 }); + await dragTo(m, g, emit, 2 * 25 + 90 + 24); assert.equal(f.text, 'ab ', 'at the border: 25 + 90 px more needed'); + await dragTo(m, g, emit, 2 * 25 + 90 + 25); assert.equal(f.text, 'ab', 'over the border'); + await dragTo(m, g, emit, 2 * 25 + 90 + 24); assert.equal(f.text, 'ab ', 'back over the border (same distance)'); + await dragTo(m, g, emit, 0); assert.equal(f.text, 'ab cd'); +}); +await test('drag: after an unknown delete nothing is retyped; umlauts are', async () => { + const { m, f, tap, emit, swipe, replace } = setup(); + f.text = 'alt '; // not typed by the keyboard + tap('für'); + const g = m.dragStart({ px: 25, detentPx: 90 }); + await dragTo(m, g, emit, 3 * 25); assert.equal(f.text, 'alt '); + await dragTo(m, g, emit, 3 * 25 + 114); assert.equal(f.text, 'alt ', 'stuck at the start of the known text'); + await dragTo(m, g, emit, 3 * 25 + 115 + 25); assert.equal(f.text, 'al', 'blind deletes past the known text'); + await dragTo(m, g, emit, 0); assert.equal(f.text, 'al', 'strict: unknown deleted, nothing retyped'); + const { m: m2, f: f2, tap: tap2, emit: emit2 } = setup(); + tap2('Grüße'); + const g2 = m2.dragStart({ px: 25, detentPx: 90 }); + await dragTo(m2, g2, emit2, 4 * 25); assert.equal(f2.text, 'G'); + await dragTo(m2, g2, emit2, 0); assert.equal(f2.text, 'Grüße'); +}); +await test('drag: something else typed in between stops restoring', async () => { + const { m, f, tap, emit, swipe, replace } = setup(); + tap('Hallo'); + const g = m.dragStart({ px: 25, detentPx: 90 }); + await dragTo(m, g, emit, 2 * 25); assert.equal(f.text, 'Hal'); + tap('x'); + await dragTo(m, g, emit, 0); assert.equal(f.text, 'Halx'); +}); +await test('drag over a swiped word: restored or not, it stays frozen', async () => { + const { m, f, emit, swipe, replace } = setup(); + await swipe('Hallo'); await swipe('wie'); + const g = m.dragStart({ px: 25, detentPx: 90 }); + await dragTo(m, g, emit, 50); await dragTo(m, g, emit, 0); + assert.equal(f.text, 'Hallo wie'); + assert.equal(await replace('die'), false, 'touched: frozen'); +}); +await test('tap-typed words: whole only when the start is known', async () => { + const { m, tap, out, swipe, replace } = setup(); + tap('hsllo '); + assert.equal(m.endedWord(), null, 'after an idle/unknown start the word may be longer'); + out('Enter'); // boundary + tap('hsllo '); + assert.deepEqual(m.endedWord(), { word: 'hsllo', term: ' ', n: 6 }); + tap('Wie'); + assert.deepEqual(m.currentWord(), { text: 'Wie', n: 3 }); + tap('?'); + assert.deepEqual(m.endedWord(), { word: 'Wie', term: '?', n: 4 }); + tap(' '); + assert.equal(m.endedWord(), null, 'two terminators: the word is not "just finished"'); + m.reset('opened', { boundary: true }); tap('ab'); + assert.deepEqual(m.currentWord(), { text: 'ab', n: 2 }); +}); +await test('correction replace keeps the terminator; edits freeze the anchor', async () => { + const { m: m2, f: f2, emit: e2, tap: t2, out: o2, swipe, replace } = setup(); + o2('Enter'); f2.text = ''; + t2('Ich sage hsllo,'); + const w = m2.endedWord(); + assert.deepEqual([w.word, w.term, w.n], ['hsllo', ',', 6]); + let a = m2.anchor(w.n); + a = await m2.replaceAnchored(a, e2, 'hallo' + w.term); assert.equal(f2.text, 'Ich sage hallo,'); + a = await m2.replaceAnchored(a, e2, 'Hallen' + w.term); assert.equal(f2.text, 'Ich sage Hallen,'); + a = await m2.replaceAnchored(a, e2, 'hsllo' + w.term); assert.equal(f2.text, 'Ich sage hsllo,', 'back to the original'); + t2(' '); + assert.equal(await m2.replaceAnchored(a, e2, 'hallo,'), null, 'typed after it: frozen'); + o2('Backspace'); + assert.equal(await m2.replaceAnchored(a, e2, 'hallo,'), null, 'a tap froze it, even once deleted again'); + assert.equal(f2.text, 'Ich sage hsllo,'); + assert.equal(await m2.replaceAnchored(m2.anchor(0), e2, 'x'), null, 'empty anchor'); + const b = m2.anchor(6); m2.observe('Backspace'); m2.observe(','); // untagged re-typing, no freeze + assert.equal(await m2.replaceAnchored(b, e2, 'x'), null, 'retyped terminator: new characters, not provably the same'); +}); +await test('completion: prefix -> word (no space), switchable incl. back to the prefix, next swipe auto-spaced', async () => { + const { m, f, emit, tap, out, swipe } = setup(); + out('Enter'); f.text = ''; + tap('Das ist schö'); + const cw = m.currentWord(); + assert.deepEqual(cw, { text: 'schö', n: 4 }); + let a = await m.replaceAnchored(m.anchor(cw.n), emit, 'schön'); + assert.equal(f.text, 'Das ist schön'); + a = await m.replaceAnchored(a, emit, 'schöne'); + assert.equal(f.text, 'Das ist schöne', 'another completion replaces the picked one exactly'); + a = await m.replaceAnchored(a, emit, 'schö'); + assert.equal(f.text, 'Das ist schö', 'back to the typed prefix (first strip entry)'); + a = await m.replaceAnchored(a, emit, 'schöne'); + await swipe('Tag'); + assert.equal(f.text, 'Das ist schöne Tag', 'the next swipe gets its auto-space'); +}); +await test('idle reset is a word boundary (suggestions for the next word)', async () => { + const { m, tap } = setup(); + tap('abc'); + m.reset('idle', { boundary: true }); + tap('hsllo '); + assert.deepEqual(m.endedWord(), { word: 'hsllo', term: ' ', n: 6 }); +}); +await test('drag: "hello world" stops at "hello ", then at the known start', async () => { + const { m, f, tap, emit } = setup(); + m.reset('opened', { boundary: true }); f.text = 'alt '; // text before the keyboard's + tap('hello world'); + const g = m.dragStart({ px: 25, detentPx: 90 }); + await dragTo(m, g, emit, 5 * 25 + 114); assert.equal(f.text, 'alt hello ', 'world gone, stuck before the space'); + await dragTo(m, g, emit, 5 * 25 + 115 + 5 * 25 + 114); assert.equal(f.text, 'alt ', 'space + hello gone, stuck at the known start'); + await dragTo(m, g, emit, 5 * 25 + 115 + 5 * 25 + 115); assert.equal(f.text, 'alt', 'across the start: unknown text'); + assert.equal(g.stats.unknown, 1); + await dragTo(m, g, emit, 0); assert.equal(f.text, 'alt', 'strict: after an unknown delete nothing is retyped'); + assert.deepEqual([g.stats.restored, g.stats.stop], [0, 'unknown deleted']); + const { m: m2, f: f2, tap: tap2, emit: emit2 } = setup(); + m2.reset('opened', { boundary: true }); f2.text = 'alt '; + tap2('hello world'); + const g2 = m2.dragStart({ px: 25, detentPx: 90 }); + await dragTo(m2, g2, emit2, 5 * 25 + 115 + 5 * 25 + 114); assert.equal(f2.text, 'alt ', 'at the known start'); + await dragTo(m2, g2, emit2, 0); assert.equal(f2.text, 'alt hello world', 'no unknown delete: full restore'); + assert.deepEqual([g2.stats.restored, g2.stats.stop], [11, '']); +}); +await test('drag: start of a line (after Enter) is a border too', async () => { + const { m, f, out, emit } = setup(); + out('Enter'); + const g = m.dragStart({ px: 25, detentPx: 90 }); + await dragTo(m, g, emit, 110); assert.equal(f.text, '\n', 'not yet: 25 + 90 px'); + await dragTo(m, g, emit, 115); assert.equal(f.text, ''); +}); +await test('drag: suggestion lookups during the drag do not stop restoring', async () => { + const { m, f, tap, emit } = setup(); + m.reset('opened', { boundary: true }); + tap('hello wor'); + const g = m.dragStart({ px: 25, detentPx: 90 }); + await dragTo(m, g, emit, 50); + m.currentWord(); m.endedWord(); m.anchor(2); // what suggest() reads + await dragTo(m, g, emit, 0); assert.equal(f.text, 'hello wor'); + assert.equal(g.stats.stop, ''); +}); +console.log(`${passed} text model tests passed`); diff --git a/modules/vr-keyboard/textmodel.js b/modules/vr-keyboard/textmodel.js new file mode 100644 index 0000000..1f8ba40 --- /dev/null +++ b/modules/vr-keyboard/textmodel.js @@ -0,0 +1,165 @@ +// textmodel.js: what the VR keyboard itself typed recently (it can't read the +// text field). Fed with every key the keyboard emits, in order (observe): +// single characters, "Backspace"; anything else resets it. Evaluates to +// { VERSION, create }. +// +// Buffer: the last `size` code points as emitted; empty = unknown. +// Anchors: a slice of buffer entries; intact while exactly those entries (by +// identity) end the buffer and no freeze() / reset() happened since. Every +// replacement (swipe candidates, corrections, completions) goes through an +// intact anchor, so it deletes exactly what it typed or nothing. +(() => { + const VERSION = 5; + const TERMINATORS = new Set([...' .,!?;:']); + const OPENING = new Set([...'([{<"\'„“‚‘«‹¿¡/@#-_']); // no auto-space after these + const isWordChar = (c) => /[\p{L}\p{N}'’-]/u.test(c); + + function create({ size = 64 } = {}) { + let buf = []; // [{ c }] + let epoch = 0; // bumped by reset/freeze: older anchors are void + const chars = (from = 0, to = buf.length) => buf.slice(from, to).map((e) => e.c).join(''); + const m = { + version: 0, // bumped on every change (for caching suggestions) + startKnown: false, // the buffer starts at a word boundary (Enter, field opened) + get text() { return chars(); }, + reset(why, { boundary = false } = {}) { + buf = []; + m.startKnown = boundary; + m.version++; + epoch++; + m.lastReset = why; + }, + freeze() { epoch++; }, + // Returns false if the key reset the model. + observe(key) { + m.version++; + if (key === 'Backspace') { + if (!buf.length) m.startKnown = false; // deleting text the model never saw + buf.pop(); + } else if (typeof key === 'string' && [...key].length === 1) { + buf.push({ c: key }); + if (buf.length > size) { buf.splice(0, buf.length - size); m.startKnown = false; } + } else { + m.reset(`key ${key}`, { boundary: key === 'Enter' }); + return false; + } + return true; + }, + + anchor(n) { return n > 0 && n <= buf.length ? { entries: buf.slice(-n), epoch } : null; }, + anchorIntact(a) { + if (!a || a.epoch !== epoch || a.entries.length > buf.length) return false; + const o = buf.length - a.entries.length; + return a.entries.every((e, i) => buf[o + i] === e); + }, + // Replace the anchored characters by text; returns the new anchor + // (over text), or null if the anchor isn't intact. emit(key) sends one + // key to the keyboard's output, which calls observe() synchronously; it + // may return a promise (a pause after asynchronously typed characters). + async replaceAnchored(a, emit, text) { + if (!m.anchorIntact(a)) return null; + for (let i = 0; i < a.entries.length; i++) await emit('Backspace'); + for (const c of text) await emit(c); + return m.anchor([...text].length); + }, + + // ---- swipe -------------------------------------------------------------- + // Auto-space only after a known character that isn't whitespace or + // opening punctuation. Returns { anchor (the word, not its space), space }. + async commitWord(emit, word) { + const e = buf[buf.length - 1]; + const space = !!e && !/\s/u.test(e.c) && !OPENING.has(e.c); + if (space) await emit(' '); + for (const c of word) await emit(c); + return { anchor: m.anchor([...word].length), space }; + }, + + // ---- tap-typed words ---------------------------------------------------- + // The word at the end, if known whole (a known non-word character or a + // boundary before it): { text, n }. + currentWord() { + let i = buf.length; + while (i > 0 && isWordChar(buf[i - 1].c)) i--; + if (i === buf.length || (i === 0 && !m.startKnown)) return null; + return { text: chars(i), n: buf.length - i }; + }, + // A word just finished by one terminator: { word, term, n (incl. term) }. + endedWord() { + const e = buf[buf.length - 1]; + if (!e || !TERMINATORS.has(e.c)) return null; + let i = buf.length - 1; + while (i > 0 && isWordChar(buf[i - 1].c)) i--; + if (i === buf.length - 1 || (i === 0 && !m.startKnown)) return null; + return { word: chars(i, -1), term: e.c, n: buf.length - i }; + }, + + // ---- Backspace drag ----------------------------------------------------- + // borders[k-1]: deleting the k-th character from the end crosses a word + // border (the word at the cursor is gone, its space/punctuation would + // be next): "Hallo wie geht" sticks at "Hallo wie ". Whitespace and + // punctuation at the cursor go first, without a border. Known text only. + wordBorders() { + const out = []; + let state = 'tail'; + for (let i = buf.length - 1; i >= 0; i--) { + const w = isWordChar(buf[i].c); + out.push(state === 'word' && !w); + if (state === 'tail' ? w : state === 'word' ? !w : w) state = state === 'word' ? 'gap' : 'word'; + } + return out; + }, + // A drag gesture: snapshot at the press. borders also marks the start + // of the known text (or of the line after Enter): deleting past it + // needs the extra travel too. target(travel): characters to be deleted + // at that leftward travel (px each, a detent of detentPx before a border). + // Restoring retypes this gesture's deletes in reverse, only while the + // buffer is exactly the snapshot minus what is still deleted and the + // gesture deleted nothing beyond the known text; else it stops for good. + dragStart({ px, detentPx }) { + const snap = buf.map((e) => e.c), borders = m.wordBorders(); + borders[snap.length] = snap.length > 0 || m.startKnown; + return { + snap, borders, applied: 0, restorable: true, + stats: { deleted: 0, restored: 0, borders: 0, unknown: 0, stop: '' }, + target(travel) { + let k = 0; + for (let need = px + (borders[0] ? detentPx : 0); need <= travel; need += px + (borders[k] ? detentPx : 0)) k++; + return k; + }, + }; + }, + async dragDelete(g, emit) { + const border = !!g.borders[g.applied]; + await emit('Backspace'); + g.applied++; + g.stats.deleted++; + if (border) g.stats.borders++; + if (g.applied > g.snap.length) g.stats.unknown++; + return { border }; + }, + // { border } or null (restoring stopped). + async dragRestore(g, emit) { + const L = g.snap.length, n = g.applied; + if (!g.restorable || n <= 0) return null; + const border = !!g.borders[n - 1]; + if (g.stats.unknown) { // strict: never a partial, wrong-looking retype + g.restorable = false; + g.stats.stop = 'unknown deleted'; + return null; + } + if (buf.length !== L - n || buf.some((e, i) => e.c !== g.snap[i])) { + g.restorable = false; + g.stats.stop = 'text changed'; + return null; + } + await emit(g.snap[L - n]); + g.applied--; + g.stats.restored++; + return { border }; + }, + }; + return m; + } + + return { VERSION, create }; +})() diff --git a/modules/vr-keyboard/unpatch-panel.js b/modules/vr-keyboard/unpatch-panel.js new file mode 100644 index 0000000..a375f2c --- /dev/null +++ b/modules/vr-keyboard/unpatch-panel.js @@ -0,0 +1,9 @@ +// Reverts panel.js in SteamVR's systemui page: removes the strip panel, its +// embedded-UV slot and stylesheet. Safe when nothing is patched. +(() => { + const S = window.__sfuiKbdStrip; + if (!S) return 'not patched'; + try { S.dispose?.(); } catch { /* gone */ } + delete window.__sfuiKbdStrip; + return 'unpatched'; +})() diff --git a/modules/vr-keyboard/unpatch.js b/modules/vr-keyboard/unpatch.js new file mode 100644 index 0000000..207f8e6 --- /dev/null +++ b/modules/vr-keyboard/unpatch.js @@ -0,0 +1,13 @@ +// Reverts patch.js in Steam's SharedJSContext (evaluated by the +// steam-ui-patches service when it stops): removes the listeners, the trail +// canvas and the candidate strip from every keyboard popup it was attached +// to, and the globals. Safe when nothing is patched. +(() => { + const S = window.__sfuiSwipe; + if (!S) return 'not patched'; + for (const detach of S.docs?.values() || []) { try { detach(); } catch { /* popup gone */ } } + delete window.__sfuiSwipe; + // __sfuiSwipeLog / __sfuiSwipePaths (debugging data) are kept, so a switch + // doesn't lose the evidence of a misrecognition. + return 'unpatched'; +})() diff --git a/scripts/vr-keyboard-replay.mjs b/scripts/vr-keyboard-replay.mjs new file mode 100644 index 0000000..3cd93bf --- /dev/null +++ b/scripts/vr-keyboard-replay.mjs @@ -0,0 +1,17 @@ +// Replays recorded swipes (window.__sfuiSwipePaths of Steam's SharedJSContext, +// saved as JSON) through the decoder: top candidates and the cost terms of +// the expected word. The dictionary is the built vr-keyboard-dictionary.js. +// usage: node scripts/vr-keyboard-replay.mjs modules/vr-keyboard/decoder.js [word] +import { readFileSync } from 'node:fs'; +const [, , decoderPath, dictPath, pathsPath, expected] = process.argv; +const D = (0, eval)(readFileSync(decoderPath, 'utf8')); +const dict = D.parseDict(JSON.parse(readFileSync(dictPath, 'utf8'))); +for (const p of JSON.parse(readFileSync(pathsPath, 'utf8'))) { + const lay = D.layout(dict, p.keys, p.unit); + const res = D.decode(lay, p.pts, { unit: p.unit, max: 8 }); + console.log(`${p.t}: recorded ${p.top.join(' ')}\n now ${res.map((r) => `${r.word} ${r.cost.toFixed(2)}`).join(', ')}`); + if (expected) { + const f = D.decode(lay, p.pts, { unit: p.unit, features: true }).find((x) => x.word === expected); + console.log(` ${expected}: ${f ? JSON.stringify(f.f) : 'pruned (start/end key or a letter too far from the path)'}`); + } +}