commit f79db54ca9b853f866083924eaaf62f2edf1b394 Author: baketnk Date: Thu Sep 24 10:43:24 2026 -0400 Initialize standalone Frame Dictation scaffold and install preflight diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..c77e722 --- /dev/null +++ b/.gitignore @@ -0,0 +1,13 @@ +/build*/ +/.venv/ +/.cache/ +/models/ +/recordings/ +/transcripts/ +/logs/ +__pycache__/ +*.py[cod] +.env +.env.* +!.env.example +.DS_Store diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..d3bdd71 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,25 @@ +# Frame Dictation development + +This is an independent project, not a plugin for another application. +Read README.md and docs/design.md before implementation. Dated evidence records +past observations; it never proves current device availability or grants a live run. + +- Keep dependencies explicit and small. Do not add a dependency/submodule/symlink + to an unrelated application's build tree, assets or Python environment. +- External source reuse needs a deliberate, license-reviewed standalone extraction, + not hidden coupling. +- Preserve the one-command GitHub installation goal: no Steam store AppID, sudo + or end-user compiler requirement. See docs/install-design.md. Do not publish a + placeholder installer as functional or conflate an OpenVR app key with a store ID. +- Default builds and tests are offline and hardware-free. No implicit package/model + downloads, microphone recording, input injection or OpenVR initialization. +- Hardware tests must be opt-in and distinguish API discovery from delivered input, + transcription quality, performance and human headset acceptance. +- Preserve existing SSH and user sessions; never terminate SSH/session processes or + use broad cleanup/restart commands. Stop only processes this project owns. +- Keep audio, transcripts, model weights, credentials and private logs out of Git. + No automatic Enter/submit, speech commands or cloud/desktop ASR fallback. +- Build/check: `cmake -S . -B build && cmake --build build`, then + `ctest --test-dir build --output-on-failure` and `git diff --check`. +- Keep docs honest about implemented versus proposed behavior. Review exact diffs + and create small verified commits. diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 0000000..24ca460 --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,19 @@ +cmake_minimum_required(VERSION 3.20) +project(frame_dictation VERSION 0.1.0 LANGUAGES CXX) + +add_executable(frame-dictation src/main.cpp) +target_compile_features(frame-dictation PRIVATE cxx_std_20) +set_target_properties(frame-dictation PROPERTIES CXX_EXTENSIONS OFF) +target_compile_definitions(frame-dictation PRIVATE FRAME_DICTATION_VERSION="${PROJECT_VERSION}") + +include(CTest) +if(BUILD_TESTING AND NOT CMAKE_CROSSCOMPILING) + add_test(NAME frame_dictation.cli + COMMAND "${CMAKE_COMMAND}" + "-DAPP=$" + "-DEXPECTED_VERSION=${PROJECT_VERSION}" + -P "${CMAKE_CURRENT_SOURCE_DIR}/tests/cli.cmake") + add_test(NAME frame_dictation.install_preflight + COMMAND sh "${CMAKE_CURRENT_SOURCE_DIR}/tests/install-preflight.sh" + "${CMAKE_CURRENT_SOURCE_DIR}/scripts/install-preflight.sh") +endif() diff --git a/README.md b/README.md new file mode 100644 index 0000000..178fa27 --- /dev/null +++ b/README.md @@ -0,0 +1,53 @@ +# Frame Dictation + +Standalone, on-device voice typing for Steam Frame. + +**Status: project scaffold and design only.** The executable prints help/version; +it does not yet render an overlay, open a microphone, load a model or type text. + +Planned path: **OpenVR overlay → local Parakeet Redux CPU inference → Gamescope +Unicode input**. No external application checkout, library, submodule, desktop +inference server or running scene host is required. + +**Distribution goal:** one-command installation from GitHub releases, entirely +user-local, with no Steam store AppID. See the [installer design](docs/install-design.md). +There is no working installer or published release yet. + +For a read-only check of proposed release prerequisites on a Frame, run +`sh scripts/install-preflight.sh`. It checks Linux AArch64/glibc and bootstrap +tools, reports system Python (3.12+ for a possible source worker), Git and uv. +Git and uv are not required for the planned bundled release. This check downloads +and installs nothing; passing it does **not** mean an install or dictation works. + +## Build the scaffold + +Requirements: CMake 3.20+ and a C++20 compiler. No third-party packages or downloads. + +```sh +cmake -S . -B build +cmake --build build +ctest --test-dir build --output-on-failure +./build/frame-dictation --help +``` + +This is a host-native scaffold build, not an ARM64 deployment or headset test. +Future OpenVR, audio and inference integrations must be explicitly configured; +configuration/build/tests must never install packages or launch SteamVR implicitly. + +## Project map + +- [Design](docs/design.md): UI, local inference, input backend and acceptance gates. +- [Frame API evidence](docs/evidence/frame-dictation-apis-2026-09-24.md): successful + mixed-script Unicode test and overlay-client initialization; known limits. +- [Installer design](docs/install-design.md): GitHub install, standalone OpenVR + identity, packaging, upgrades and uninstall. +- [Provenance](docs/provenance.md): origin of the investigation, model/runtime pins. +- `src/main.cpp`: inert CLI entry point. +- `tests/cli.cmake`: hardware-free scaffold smoke test. +- `scripts/install-preflight.sh`: read-only proposed packaging prerequisite check. +- [Agent guidance](AGENTS.md): project boundaries and safe validation. + +First implementation gate: an isolated ARM64 CPU trial of the pinned Redux model. +Then a minimal overlay and explicit insertion into a disposable target. Neither +inference nor an overlay is implemented here yet. No model weights, recordings, +credentials, engine assets or runtime binaries are included. diff --git a/docs/design.md b/docs/design.md new file mode 100644 index 0000000..41e2e2d --- /dev/null +++ b/docs/design.md @@ -0,0 +1,292 @@ +# Native Frame dictation overlay — proposal + +Standalone project design; see +[provenance](provenance.md) and [historical Frame probes](evidence/frame-dictation-apis-2026-09-24.md). +Only the inert build scaffold is implemented. This document describes proposed runtime behavior. + +## Recommendation + +Build a small native ARM64 **OpenVR overlay application**, independent of +other applications, with **local Parakeet Redux CPU inference**. Use Gamescope's +input-method protocol for literal Unicode text; keep Linux key-injection APIs +as explicit fallbacks. No desktop ASR server, network hop, LLM cleanup, scene +renderer, avatar, desktop capture or root service is needed in the primary path. + +A first-class product goal is a **one-command GitHub install without a Steam store +AppID**. Package a prebuilt native executable and isolated CPU runtime; use a normal +OpenVR application key for registration, not Steamworks. Installation must remain +user-local with opt-in autolaunch. See [installation design](install-design.md). + +```text +controller PTT / overlay mic button + ↓ +SDL3 → PipeWire/PulseAudio mic → bounded 16 kHz mono clip + ↓ +one persistent local Parakeet Redux CPU worker + ↓ +correlated literal transcript → focus/delivery policy + ↓ +Gamescope IME set_string + commit → focused Frame application + ↘ overlay preview / error / explicit Insert when delivery is unsafe +``` + +The CPU worker is local process isolation, not remote inference or a service +framework. Implement a small transport owned by this repository. Keep the model +loaded between utterances; do not spawn Python/load 178 MB for every release. + +## Minimal interaction + +A small hand-attached status chip while armed; a compact review panel only when +needed. Allow head-relative placement for users who prefer it. No permanent large +dashboard obscuring the application. + +```text +[ mic ] Ready · On-device [ settings ] + Target: selected application + +Recording… 00:04 [ Cancel ] + +"The recognized text appears here." +[ Insert ] [ Discard ] [ Enter — separate action ] +``` + +- States: disabled, warming, ready, recording, transcribing, review, inserted, + unavailable/error. Recording uses visible icon + text, not color alone. +- Hold to speak, release to finish. A click-to-start/stop overlay button provides + a binding-independent alternative. Bound recording to 20 seconds; discard + accidental taps (initial threshold: 200 ms). +- **Quick typing:** insert on completion only when the explicitly armed target + is still valid. **Review mode:** always wait for Insert. Bring up review instead + of silently losing a transcript or typing into a new target. +- Enter is always a separate press after insertion. Never interpret "submit", + "delete" or other speech as commands in this utility. Do not auto-submit. +- No generic "undo last dictation" initially: another application's edits/cursor + cannot be reliably rolled back by a guessed number of backspaces. +- Stop/disable releases owned keys and microphone, invalidates pending delivery, + and terminates only the owned worker. No Steam/session/SSH cleanup commands. + +## Host and rendering boundary + +Implement the standalone `frame-dictation` executable, initialized +with `VRApplication_Overlay`. Frame accepted that application type and +`IVROverlay_028` in the probe. Use `CreateOverlay`, tracked-device-relative +transform, `ShowOverlay`/`HideOverlay`, and `PollNextOverlayEvent` for the panel. +Use SteamVR's overlay interaction rather than inventing scene controller rays. + +Do not link an external scene host or launch another application behind the +panel. A small overlay-specific RAII owner in this repository should manage OpenVR, overlay +handles, input manifest and shutdown. No dependency on external application +libraries, assets, build trees or Python environments. + +Start with one small RGBA panel updated only on UI changes and a bounded recording +indicator cadence. `SetOverlayRaw` is the simplest proof route; measure upload +cost before selecting a persistent Vulkan `SetOverlayTexture` path. No stereo +eye targets or per-eye scene rendering. Keep tracking in compositor transforms, +not an application-rendered hand-pose animation loop. Do not promise a particular +GPU cost until measured. + +A scene renderer's canvas/MSDF resources are not +an OpenVR overlay backend and are not imported here. Use a small independently +licensed font/icon set and minimal panel renderer. Any later source/asset reuse +requires an explicit license-reviewed extraction, never a runtime path into the +engine checkout. + +### Controller bindings + +Expose PTT, cancel and explicit insert/Enter as named SteamVR actions; let the +user bind them. Do not assume a scene app's left-bumper mapping works globally or +silently takes a game's button away. Check action activity and neutral rearm. + +OpenVR 2.15.6 documents experimental overlay action-set priorities +`0x01000000..0x01FFFFFF`, gated by SteamVR's **Experimental overlay input overrides** +setting. This can selectively override scene input, but is not guaranteed enabled +or usable on this Frame. Do not toggle it automatically. Begin with overlay mic +controls; validate a global PTT binding separately with a scene active, dashboard +open/closed, lost tracking, and reconnection. Overlay interactivity/input ownership +is distinct from OS keyboard focus. + +## Text delivery: use the proven path first + +### 1. Gamescope IME — primary Frame backend + +Connect to the discovered Gamescope socket; bind +`gamescope_input_method_manager` and `wl_seat`, handle unavailable/done events, +then issue `set_string(valid_utf8)` and `commit(last_serial)`. A v2 binding is +sufficient for text and actions even though the current server advertises v3. +Use generated bindings from the pinned XML, not a production handwritten wire +protocol. Own/destroy the IME object deliberately and release it while disabled. + +This delivered an exact mixed-script string into the disposable Xwayland +receiver. It does not need clipboard ownership, sudo, `uinput`, application +plugins, or a new input daemon. However it is a **private Gamescope extension**; +isolate it behind a version-gated backend and test after SteamOS updates. + +The server implements text through synthetic key events and a temporary keymap, +not a guaranteed rich-text/IME edit operation in every app. Verify actual target +toolkits and games. Respect singleton/unavailable handling and Steam-keyboard +coexistence. Never use the installed `gamescope-type` CLI as a transcript pipe: +its inspected sample loop is byte-oriented and interprets newline as Submit. + +### 2. Explicit fallbacks, not a framework built up front + +- **X11 clipboard + XTEST paste:** useful for apps that reject direct Unicode + key events but accept paste. Own the appropriate X selection (CLIPBOARD or + PRIMARY), serving the `UTF8_STRING` target on the destination X server; + select the app's paste chord explicitly. Do not assume Ctrl+V in terminals. + Do not overwrite/restore arbitrary clipboard history silently; expose Copy as + a deliberate fallback. This backend has not yet been exercised on Frame. +- **libei:** current server accepts a sender and advertises KEYBOARD but **not + TEXT**. Suitable for evdev-style key chords, not automatic Unicode insertion. + Bind/resume/device lifecycle must still be tested. Direct EIS socket access is + Gamescope-specific here; no portal RemoteDesktop route was exposed. +- **uinput:** current account can open it without sudo. Reserve for raw virtual + keyboard needs; creating/retiring a device is unnecessary for primary dictation. + Unicode is not an evdev keycode, so this alone does not solve text delivery. + +Do not assume generic labwc virtual-keyboard/data-control protocols are present: +those globals are absent. Fail visibly if the primary backend is unavailable; +no automatic privilege escalation or OS package/configuration edits. + +### Target/focus policy + +The IME serial is **not** an established target-generation guard. Input goes to +the seat's current focus. No general Linux input API makes insertion into an +arbitrary app atomic with a focus check. + +For the first supported route, use an explicitly armed Xwayland destination. +Track display identity, active top-level, actual X keyboard-focus window and a +monotonic focus generation; invalidate on focus loss/regain, window destruction, +disconnect or target change. Observe changes throughout capture/inference and +recheck immediately before delivery. Do not restore another app's focus behind +the user's back. Refuse quick insertion when modifier keys are held or target +identity is ambiguous. A matching final window ID alone is insufficient. + +Gamescope exposes focus-display/window root properties, but their encoding and +relationship to seat focus need implementation-specific validation. Do not infer +that X display `:0` is always the destination, or that an X focus observation +identifies a native Wayland text field. For unobservable native Wayland focus, +require explicit review/Insert; do not advertise safe auto-targeting. + +If focus changes, keep the result in review. A fresh Insert explicitly approves +the current destination and creates a new delivery authorization. Recheck again +at insertion. This minimizes stale delivery but does **not** eliminate a race +between the final check and global input processing; do not claim otherwise. +A Wayland roundtrip means compositor processing, not application consumption. +Report `input queued`, never `message sent`. + +Validate UTF-8 and enforce the 4096-byte bound; flatten line breaks/tabs, reject +NUL, escape and other control characters. No shell evaluation, commands, Enter, +terminal escapes or action inference from recognized text. One correlated request +may deliver at most once; cancellation invalidates it before any later reply. + +## On-device Redux + +Use the **same** `moondream/parakeet-redux` revision from the benchmark: +`fad622f25f303105c20d70e201bcc477c88b620c` (177,774,490-byte weight file), initially +moondream 2.4.0 / kestrel 0.8.0. Do not substitute dense Ultra or silently fall +back to desktop/cloud inference. + +The earlier benchmark used this local API (its source belongs to the originating +repository, not this project): + +```python +model = md.photon("moondream/parakeet-redux", model_path=local_model_directory, + device="cpu", cpu_threads=thread_budget) +text = model.transcribe(audio=mono_float32, sample_rate=16000)["text"] +``` + +Linux ARM64 Python 3.12 native wheels and a packaged CPU payload exist. That is +sufficient reason to **try the native CPU path first**, not proof of Frame +performance. Its 61/137 ms desktop median/p95 must not be reused as an estimate +for the headset. Its historical dictation quality was worse than Small.en on +that tiny dataset; keep transcript visibility and a cheap retry. + +Implement a small independent audio/worker adapter with explicit capture, +single-request bounds, owner-only runtime files, correlated replies and cancellation. +The worker should be implemented independently; do not link, vendor or import +another application's speech code. Avoid a generic provider framework: one +explicit Redux worker is enough for the first version. + +Suggested ownership, introduced only as implementation needs it: + +- `src/overlay.*`: OpenVR lifetime, panel presentation and controller actions. +- `src/audio.*`: explicit SDL capture and bounded mono PCM. +- `src/worker.*`: one local child process, bounded requests/replies, timeout/reaping. +- `src/text_input.*`: Gamescope protocol, focus observation and delivery policy. +- `python/frame_dictation/`: persistent CPU Redux worker, no external application imports. + +The worker reads a fixed private clip and returns an ID-correlated literal string; +one request at a time, 64 KiB framed messages, 4096-byte transcript, and a bounded +processing timeout. No shell commands in IPC and no input authority in the worker. + +- One persistent worker and one loaded model; no queue of utterances. Initially + batch at PTT release, not speculative streaming or endpoint/VAD delay. +- Start with **two CPU threads**, compare against four on Frame; bound Torch and + native kernel pools. Choose measured latency versus compositor contention, + not the desktop's thread count by habit. No real-time scheduling or permanent + CPU pinning initially; inspect runtime affinity behaviour during measurement. +- Explicit local model path and offline loading. Missing runtime/weights produces + an actionable error, not an unsolicited download/network fallback. +- Optional explicit enable/warm-up before first PTT; warming must not record audio. + Expose that lifecycle explicitly rather than warming on import or construction. + Otherwise display first-use loading honestly. Keep + model reuse after normal completion; cancellation may restart the owned worker. +- Use a private owner-only directory under `$XDG_RUNTIME_DIR` for bounded + tmpfs-backed clips; remove them on completion, error, cancellation and shutdown. + Avoid persistent audio/transcripts by default. Local IPC is not a network hop; + do not add shared-memory complexity before measuring it. +- Use CPU-only Torch where supported; test native kernel import/model load with + no CUDA device/runtime assumption. Do not copy the desktop's x86 venv. +- Weights are ~178 MB; Torch, kernels, temporary conversion and activations mean + install size/RSS will be larger. Measure cold load, peak RSS and package size. + Review runtime redistribution licensing separately from model attribution. + +Microphone access does not mute VRChat or any other social-voice app. Shared +PipeWire capture may let both hear the same utterance; the overlay must not claim +private dictation unless the other app's transmission is separately muted. No +automatic global microphone mute/reroute in this scope. + +## Delivery milestones and acceptance gates + +These are proposed implementation gates, **not completed acceptance**: + +1. **Offline ARM64 CPU spike:** isolated user-directory environment, pinned local + model, permitted runtime distribution. Run known nonprivate clips, then the + consented benchmark clips only if explicitly made available within their data + scope. Record cold load, first/warm p50/p95, errors, peak RSS, two/four threads and dependencies. + Confirm no required NVIDIA/desktop connection. Stop and report if the packed + CPU runtime is incompatible; do not silently expand to dense weights. +2. **Minimal overlay:** render status/review panel while another scene stays active; + confirm dashboard/hand placement, input events, close/reopen and no scene-focus + takeover. Validate global PTT separately rather than blocking the clickable + prototype on experimental override support. +3. **Real dictation path:** microphone → local Redux → preview → explicit insert + into a disposable target; then enable quick typing after target tracking tests. + Test Unicode, punctuation, long bounded clips, silence, cancellation, duplicate + replies, lost mic, worker crash and missing model without persisting speech. +4. **Target matrix:** Xwayland terminal/browser and selected native/Proton game + text fields; Steam keyboard coexistence; native Wayland targets separately. + Test focus changes during capture/inference, rapid loss/regain, held modifiers, + explicit Insert retargeting, no hidden Enter, and no second delivery. +5. **In-headset acceptance:** readable feedback and comfortable PTT; measured + release-to-insert latency and compositor timing while an actual scene runs; + no noticeable sustained thermal/battery regression. Set numeric budgets after + the ARM64 spike, not from an x86 result. Physical success remains human-led. + +Native Frame app typing is the first scope. Local keystrokes may or may not be +forwarded by the PC-streaming client; that route needs its own test. If necessary, +a future optional **text-only** host bridge could send the completed transcript, +without moving recognition off Frame. It is not part of this initial design. + +## Standalone dependency and test policy + +The scaffold currently needs only CMake and a C++20 compiler. Later integrations +will explicitly select OpenVR, SDL3, Wayland client/protocol bindings and a Python +Redux environment. Pin revisions and review licenses when introduced. No automatic +fetch/install in configure or normal tests; no external checkout discovery. + +Hardware-free tests should cover state transitions, bounded PCM/transcripts, +worker framing/timeout/cancellation, duplicate/stale replies and focus generations +with fakes. Separate opt-in Frame checks cover protocol availability, owned-window +insertion, microphones, inference and overlays. No fixture establishes physical +headset acceptance. See [provenance](provenance.md) for historical source anchors. diff --git a/docs/evidence/frame-dictation-apis-2026-09-24.md b/docs/evidence/frame-dictation-apis-2026-09-24.md new file mode 100644 index 0000000..63eaa44 --- /dev/null +++ b/docs/evidence/frame-dictation-apis-2026-09-24.md @@ -0,0 +1,44 @@ +# Historical Steam Frame API probes — 2026-09-24 + +These observations were made on one device and SteamOS build, not rerun as part +of this repository's scaffold. They do not prove current availability or headset +acceptance. No probe code, user audio, credentials or runtime binaries are shipped. +See the [design](../design.md) for proposed implementation and safety gates. + +## Environment + +SteamOS 0.4.0 (VR variant), AArch64, glibc 2.39, Python 3.12.3; +Gamescope 3.16.28-2. Linux ARM64 package availability is not evidence of +inference speed or compatibility with this device. + +## Observations and limits + +| Surface | Historical observation | Limit | +| --- | --- | --- | +| OpenVR | Native `VRApplication_Overlay` initialization succeeded; `IVROverlay_028`, `IVRInput_011`, `IVRSystem_026` and `IVRApplications_008` were accepted. | No rendered overlay, binding or autolaunch test. | +| Gamescope IME | `gamescope_input_method_manager` v3 advertised; v2 binding accepted and returned `done(serial=1)`. | Discovery is not delivered input. | +| Unicode delivery | A separate disposable X11/XIM receiver on the device accepted an exact mixed-script Unicode fixture via `set_string` and `commit` after checking focus on its owned window. | One Xwayland receiver, not general games, native Wayland or PC-streamed targets. No Enter was sent. | +| libei | Sender connected; seat advertised KEYBOARD, not TEXT. | No bound device or delivered key test. | +| XTEST / uinput | XTEST advertised; `/dev/uinput` was openable by the test account. | No injected XTEST key or virtual device. | +| Audio | PipeWire input device enumerated. | No microphone recording or model inference in this probe. | +| Portal | Existing portal introspection exposed no RemoteDesktop, InputCapture, Clipboard, ScreenCast or GlobalShortcuts interface. | Not a guarantee for future OS releases. | + +The Xwayland receiver was destroyed after the test. No packages, services or +Steam settings were changed. No microphone audio was recorded. This is a +historical API probe, not a live availability check or install smoke test. + +## Implementation cautions + +Gamescope's [input-method protocol](https://github.com/ValveSoftware/gamescope/blob/3.16.28/protocol/gamescope-input-method.xml) +is private and version-sensitive; see its [implementation](https://github.com/ValveSoftware/gamescope/blob/3.16.28/src/ime.cpp). +The serial is not a verified focus-generation guard; a roundtrip is not a text +consumption acknowledgement. The inspected [`gamescope-type` example](https://github.com/ValveSoftware/gamescope/blob/3.16.28/src/Apps/gamescope_type.c) +interprets newline as Submit: do not use it as a transcript pipe. +Generic virtual-keyboard and data-control Wayland globals were not advertised +in this probe. Focus tracking and Steam keyboard coexistence need separate tests. + +The proposed Redux model is pinned to revision +`fad622f25f303105c20d70e201bcc477c88b620c`; the weight file was +177,774,490 bytes in an earlier local inspection. ARM64 Python 3.12 wheels +appeared available, but no native model load or inference was performed on the +Frame. Runtime redistribution terms require separate review. diff --git a/docs/install-design.md b/docs/install-design.md new file mode 100644 index 0000000..d549747 --- /dev/null +++ b/docs/install-design.md @@ -0,0 +1,105 @@ +# Installation and distribution goal + +**User goal:** install from GitHub with a `curl … | bash`-style command, without a +Steam store AppID. This is a requirement for the future release, not a working +installer. No release artifacts or functional installer are published yet. + +The read-only `scripts/install-preflight.sh` checks whether a host appears suitable +for the **proposed** Linux ARM64 glibc package format and has the expected basic +bootstrap utilities. It reports system Python, Git and uv, but none is required +for the intended bundled release. A read-only check on one Frame observed +Python 3.12.3 and Git, but not uv; availability may change. This script +is not an installer or a model/runtime compatibility test; it has no downloads, +registration, SteamVR initialization or persistent changes. No glibc minimum can +be certified until release artifacts are chosen and tested. + +## Non-Steam overlay identity + +OpenVR overlay applications do not require a Steam store AppID or Steamworks. +The native executable initializes as `VRApplication_Overlay`. For discoverability +and optional autolaunch, register an OpenVR application manifest with a stable, +project-owned **string application key** (proposed: `local.frame-dictation.overlay`). +That key is not a numeric Steam AppID. No purchase/store listing or non-Steam Steam +library shortcut should be necessary for the normal route. + +The manifest identifies the installed executable; the installer/registration +helper should use `IVRApplications::AddApplicationManifest` and the corresponding +remove operation, not hand-edit Steam's internal JSON. Autolaunch uses the OpenVR +application setting only when explicitly requested. Validate the exact manifest, +launch behaviour, registration persistence and uninstall on native Frame before +claiming this route works end to end. Existing probes established overlay client +initialization, not manifest installation. + +SteamVR/OpenVR must already be installed and usable. If registration needs a +running runtime, defer it to the first explicit launch rather than starting or +restarting SteamVR behind the user's back. + +## Intended user experience + +1. Run one documented command from the eventual GitHub repository/release. +2. Installer identifies native Linux ARM64 Frame, resolves a pinned release and + explains/downloads the application, compatible CPU runtime and pinned model. +3. User-local installation provides a simple `frame-dictation` launcher, desktop + entry where supported, and an OpenVR manifest. No compiler, engine checkout, + Python dependency troubleshooting or separate ASR server for ordinary users. +4. The user explicitly launches/enables dictation. No installation-time microphone + recording, input injection, inference benchmark or overlay takeover. + +Illustrative command shape only; `OWNER`, `REPO` and `VERSION` are placeholders: + +```sh +curl --fail --silent --show-error --location \ + https://raw.githubusercontent.com/OWNER/REPO/VERSION/install.sh | bash +``` + +Also document a download-inspect-run path for users who do not want to pipe remote +code into a shell. Pin a release/tag instead of executing a moving branch by default. +Offer explicit version selection and noninteractive flags; do not read interactive +confirmation from stdin while the installer itself is arriving through that pipe. + +## Packaging boundary + +- Prebuilt ARM64 executable plus a known-compatible, isolated CPU inference runtime; + no external application libraries/assets and no system Python modification. +- Model fetched during explicit installation/setup, with pinned revision/hash and + attribution. A documented `--without-model` option can defer the large download. + No surprise first-utterance downloads. Licensing may require obtaining particular + runtime components from their vendor instead of redistributing them in our tarball. +- Install under `$XDG_DATA_HOME/frame-dictation` (default `~/.local/share/...`), + configuration under `$XDG_CONFIG_HOME/frame-dictation`, optional launcher in + `~/.local/bin`; transient audio stays in a private `$XDG_RUNTIME_DIR` directory. +- No sudo, OS read-only-root changes, package-manager installs, udev changes, + `/dev/uinput` permission changes or modifications to unrelated launchers. +- No Steam store AppID, Steamworks SDK, root service or network ASR dependency. +- Distribution license/third-party notices must be resolved before publication; + the small model file alone is not the full runtime or redistribution permission. + +## Installer lifecycle and safety + +- Detect architecture, libc and prerequisites first. Unsupported hosts fail with a + clear explanation; never install an x86 payload silently on ARM64. +- Download to a staging directory, check versioned SHA-256 manifests and archive + paths, then atomically select the completed version. HTTPS/checksums alone do not + authenticate a compromised publisher; use signed release metadata if provided. +- Keep configuration across upgrades; retain the previous version for rollback. + Refuse or defer replacement while this application's process is running rather + than killing arbitrary processes. Never touch SSH or unrelated sessions. +- Autostart is opt-in (`--autostart` or explicit settings); do not enable a systemd + service or SteamVR autolaunch by default. Do not enable overlay input overrides. +- Uninstall removes only owned launcher, manifest registration and install files; + model/config deletion is separately explicit. Preserve other SteamVR apps. +- No update daemon initially. A deliberate rerun/update command is sufficient. + +## Acceptance before advertising one-command installation + +- Clean supported Frame: install without sudo/compiler/engine checkout/store AppID; + launch overlay, load local Redux and type into an owned disposable target. +- Normal use after installation needs no network connection or desktop ASR host. +- Failed download/hash, unsupported architecture, low disk space and interrupted + upgrades leave a usable previous install or a cleanly reported failure. +- Reinstall, rollback and uninstall preserve unrelated data and SteamVR entries. +- Noninteractive piped invocation never hangs on stdin; inspection-first path works. +- Autolaunch remains off unless chosen; uninstall removes only our registration. +- Hardware-free installer tests use temporary homes, mocked runtime registration + and local fixture artifacts. Never exercise a real user's Steam configuration + in ordinary CI/CTest. diff --git a/docs/provenance.md b/docs/provenance.md new file mode 100644 index 0000000..e97a1e9 --- /dev/null +++ b/docs/provenance.md @@ -0,0 +1,29 @@ +# Technical provenance and limits + +Frame Dictation is a standalone scaffold and proposal. No external application +source, build tree, assets, Python environment, model weights or binaries are +included. The dated [device probe](evidence/frame-dictation-apis-2026-09-24.md) +records limited historical observations; its fixture code and private logs are +not part of this repository. No current device availability or release install +can be inferred from those observations. + +## Model and preliminary measurements + +Proposed model: [moondream/parakeet-redux](https://huggingface.co/moondream/parakeet-redux/tree/fad622f25f303105c20d70e201bcc477c88b620c), +revision `fad622f25f303105c20d70e201bcc477c88b620c`. An earlier local +inspection measured a 177,774,490-byte weight file. Its model card identifies +CC-BY-4.0; separate runtime/kernel redistribution terms must be reviewed before +packaging. No license for those artifacts is granted by this repository. + +A preliminary, unpublished desktop benchmark used Python 3.12.13, moondream +2.4.0, kestrel 0.8.0 and four CPU threads. On an i7-13700K, warm decode +median/p95 were 61/137 ms and raw WER 11.11% on just 25 reviewed clips / 189 +words. This is not a general accuracy estimate or Frame latency prediction. The +CPU-selected desktop process also occupied GPU memory; GPU-free operation was +not established. No Frame inference benchmark exists yet. + +## Public platform references + +- [OpenVR 2.15.6](https://github.com/ValveSoftware/openvr/releases/tag/v2.15.6) +- [Gamescope 3.16.28 input-method protocol](https://github.com/ValveSoftware/gamescope/blob/3.16.28/protocol/gamescope-input-method.xml) +- [Gamescope IME implementation](https://github.com/ValveSoftware/gamescope/blob/3.16.28/src/ime.cpp) diff --git a/scripts/install-preflight.sh b/scripts/install-preflight.sh new file mode 100644 index 0000000..f55f740 --- /dev/null +++ b/scripts/install-preflight.sh @@ -0,0 +1,79 @@ +#!/bin/sh +# Read-only host check for the proposed release installer. Does not install anything. +set -u + +if [ "$#" -ne 0 ]; then + if [ "$#" -eq 1 ] && [ "$1" = --help ]; then + printf '%s\n' 'Usage: sh scripts/install-preflight.sh' \ + 'Checks the proposed Linux ARM64 installation prerequisites; makes no changes.' + exit 0 + fi + printf '%s\n' 'No arguments are supported (except --help).' >&2 + exit 2 +fi + +errors=0 +os=$(uname -s 2>/dev/null) || os=unknown +arch=$(uname -m 2>/dev/null) || arch=unknown +printf 'Host: %s %s\n' "$os" "$arch" +if [ "$os" != Linux ] || [ "$arch" != aarch64 ]; then + printf '%s\n' 'Unsupported host: planned release payload is Linux AArch64 only.' >&2 + errors=1 +fi + +libc=unknown +if command -v getconf >/dev/null 2>&1; then + libc=$(getconf GNU_LIBC_VERSION 2>/dev/null) || libc=unknown +fi +printf 'C library: %s\n' "$libc" +case "$libc" in + 'glibc '*) ;; + *) printf '%s\n' 'Unsupported/unknown C library: planned ARM64 wheels require glibc.' >&2 + errors=1 ;; +esac + +# Expected for a future curl/tar release bootstrap, not for building from source. +for dep in curl tar sha256sum mktemp mkdir mv; do + if command -v "$dep" >/dev/null 2>&1; then + printf 'Required tool: %s found\n' "$dep" + else + printf 'Required tool: %s MISSING\n' "$dep" >&2 + errors=1 + fi +done + +if command -v git >/dev/null 2>&1; then + printf '%s\n' 'Git: found (not required for a release install)' +else + printf '%s\n' 'Git: missing (not required for a release install)' +fi +if command -v uv >/dev/null 2>&1; then + printf '%s\n' 'uv: found (not required for a bundled runtime)' +else + printf '%s\n' 'uv: missing (not required for a bundled runtime)' +fi + +if command -v python3 >/dev/null 2>&1; then + version=$(python3 --version 2>&1) || version=unknown + printf 'System Python: %s (not required for a bundled runtime)\n' "$version" + case "$version" in + 'Python 3.'*) + minor=${version#Python 3.} + minor=${minor%%.*} + case "$minor" in + ''|*[!0-9]*) printf '%s\n' 'System Python version could not be parsed.' ;; + *) if [ "$minor" -lt 12 ]; then + printf '%s\n' 'System Python is older than 3.12; unsuitable for the proposed optional source worker.' + fi ;; + esac ;; + *) printf '%s\n' 'System Python version could not be parsed.' ;; + esac +else + printf '%s\n' 'System Python: missing (not required for a bundled runtime)' +fi + +if [ "$errors" -ne 0 ]; then + printf '%s\n' 'Preflight failed. No changes were made.' >&2 + exit 1 +fi +printf '%s\n' 'Preflight passed for the proposed package format. No installer or release payload exists yet; nothing was installed.' diff --git a/src/main.cpp b/src/main.cpp new file mode 100644 index 0000000..7db597e --- /dev/null +++ b/src/main.cpp @@ -0,0 +1,24 @@ +#include +#include + +namespace { +void help() { + std::cout << "Frame Dictation — standalone Steam Frame voice typing\n" + "Scaffold only: overlay, microphone, ASR and input are not implemented.\n\n" + "Usage: frame-dictation [--help | --version]\n" + "No device access or background processes are started.\n"; +} +} + +int main(int argc, char** argv) { + if (argc == 1 || (argc == 2 && std::string_view(argv[1]) == "--help")) { + help(); + return 0; + } + if (argc == 2 && std::string_view(argv[1]) == "--version") { + std::cout << "frame-dictation " << FRAME_DICTATION_VERSION << " (scaffold)\n"; + return 0; + } + std::cerr << "Unsupported arguments. Use --help; runtime features are not implemented.\n"; + return 2; +} diff --git a/tests/cli.cmake b/tests/cli.cmake new file mode 100644 index 0000000..d8ae469 --- /dev/null +++ b/tests/cli.cmake @@ -0,0 +1,35 @@ +if(NOT DEFINED APP OR NOT DEFINED EXPECTED_VERSION) + message(FATAL_ERROR "APP and EXPECTED_VERSION are required") +endif() + +foreach(mode IN ITEMS default help version invalid extra) + set(args) + set(expected_exit 0) + if(mode STREQUAL "help") + set(args --help) + elseif(mode STREQUAL "version") + set(args --version) + elseif(mode STREQUAL "invalid") + set(args --record) + set(expected_exit 2) + elseif(mode STREQUAL "extra") + set(args --help --record) + set(expected_exit 2) + endif() + execute_process(COMMAND "${APP}" ${args} + RESULT_VARIABLE result OUTPUT_VARIABLE output ERROR_VARIABLE error TIMEOUT 5) + if(NOT "${result}" STREQUAL "${expected_exit}") + message(FATAL_ERROR "${mode}: exit ${result}, expected ${expected_exit}: ${error}") + endif() + if(mode STREQUAL "version") + if(NOT output STREQUAL "frame-dictation ${EXPECTED_VERSION} (scaffold)\n") + message(FATAL_ERROR "Unexpected version: ${output}") + endif() + elseif(expected_exit EQUAL 2) + if(NOT error MATCHES "Unsupported arguments") + message(FATAL_ERROR "Missing rejection diagnostic") + endif() + elseif(NOT output MATCHES "Scaffold only") + message(FATAL_ERROR "Missing scaffold status") + endif() +endforeach() diff --git a/tests/install-preflight.sh b/tests/install-preflight.sh new file mode 100644 index 0000000..143142d --- /dev/null +++ b/tests/install-preflight.sh @@ -0,0 +1,40 @@ +#!/bin/sh +# Offline, hardware-free contract test using a restricted PATH of fake host tools. +set -eu +script=$1 +tmp=$(/bin/mktemp -d) +trap '/bin/rm -rf "$tmp"' EXIT HUP INT TERM +/bin/mkdir "$tmp/bin" +for dep in curl tar sha256sum mktemp mkdir mv; do + printf '#!/bin/sh\nexit 0\n' > "$tmp/bin/$dep" + /bin/chmod +x "$tmp/bin/$dep" +done +printf '#!/bin/sh\ncase "$1" in -s) printf "%%s\\n" "${MOCK_OS:-Linux}" ;; -m) printf "%%s\\n" "${MOCK_ARCH:-aarch64}" ;; esac\n' > "$tmp/bin/uname" +printf '#!/bin/sh\nprintf "%%s\\n" "${MOCK_LIBC:-glibc 2.39}"\n' > "$tmp/bin/getconf" +printf '#!/bin/sh\nprintf "%%s\\n" "${MOCK_PYTHON:-Python 3.12.3}"\n' > "$tmp/bin/python3" +/bin/chmod +x "$tmp/bin/uname" "$tmp/bin/getconf" "$tmp/bin/python3" + +check() { + expected=$1 + needle=$2 + shift 2 + status=0 + output=$(PATH="$tmp/bin" "$@" /bin/sh "$script" 2>&1) || status=$? + if [ "$status" -ne "$expected" ]; then + printf 'Expected exit %s, got %s: %s\n' "$expected" "$status" "$output" >&2 + exit 1 + fi + case "$output" in + *"$needle"*) ;; + *) printf 'Missing expected text %s: %s\n' "$needle" "$output" >&2; exit 1 ;; + esac +} + +check 0 'Git: missing (not required' /usr/bin/env +check 0 'uv: missing (not required' /usr/bin/env +check 0 'Preflight passed' /usr/bin/env +check 0 'older than 3.12' /usr/bin/env MOCK_PYTHON='Python 3.11.9' +check 1 'Linux AArch64 only' /usr/bin/env MOCK_ARCH=x86_64 +check 1 'require glibc' /usr/bin/env MOCK_LIBC=musl +/bin/rm "$tmp/bin/curl" +check 1 'curl MISSING' /usr/bin/env