mirror of
https://github.com/baketnk/frame-yap.git
synced 2026-10-06 01:00:04 +02:00
Initialize standalone Frame Dictation scaffold and install preflight
This commit is contained in:
commit
f79db54ca9
12 files changed
+758
No files matched your search
+13
@@ -0,0 +1,13 @@
|
||||
/build*/
|
||||
/.venv/
|
||||
/.cache/
|
||||
/models/
|
||||
/recordings/
|
||||
/transcripts/
|
||||
/logs/
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
.env
|
||||
.env.*
|
||||
!.env.example
|
||||
.DS_Store
|
||||
@@ -0,0 +1,25 @@
|
||||
# Frame Dictation development
|
||||
|
||||
This is an independent project, not a plugin for another application.
|
||||
Read README.md and docs/design.md before implementation. Dated evidence records
|
||||
past observations; it never proves current device availability or grants a live run.
|
||||
|
||||
- Keep dependencies explicit and small. Do not add a dependency/submodule/symlink
|
||||
to an unrelated application's build tree, assets or Python environment.
|
||||
- External source reuse needs a deliberate, license-reviewed standalone extraction,
|
||||
not hidden coupling.
|
||||
- Preserve the one-command GitHub installation goal: no Steam store AppID, sudo
|
||||
or end-user compiler requirement. See docs/install-design.md. Do not publish a
|
||||
placeholder installer as functional or conflate an OpenVR app key with a store ID.
|
||||
- Default builds and tests are offline and hardware-free. No implicit package/model
|
||||
downloads, microphone recording, input injection or OpenVR initialization.
|
||||
- Hardware tests must be opt-in and distinguish API discovery from delivered input,
|
||||
transcription quality, performance and human headset acceptance.
|
||||
- Preserve existing SSH and user sessions; never terminate SSH/session processes or
|
||||
use broad cleanup/restart commands. Stop only processes this project owns.
|
||||
- Keep audio, transcripts, model weights, credentials and private logs out of Git.
|
||||
No automatic Enter/submit, speech commands or cloud/desktop ASR fallback.
|
||||
- Build/check: `cmake -S . -B build && cmake --build build`, then
|
||||
`ctest --test-dir build --output-on-failure` and `git diff --check`.
|
||||
- Keep docs honest about implemented versus proposed behavior. Review exact diffs
|
||||
and create small verified commits.
|
||||
@@ -0,0 +1,19 @@
|
||||
cmake_minimum_required(VERSION 3.20)
|
||||
project(frame_dictation VERSION 0.1.0 LANGUAGES CXX)
|
||||
|
||||
add_executable(frame-dictation src/main.cpp)
|
||||
target_compile_features(frame-dictation PRIVATE cxx_std_20)
|
||||
set_target_properties(frame-dictation PROPERTIES CXX_EXTENSIONS OFF)
|
||||
target_compile_definitions(frame-dictation PRIVATE FRAME_DICTATION_VERSION="${PROJECT_VERSION}")
|
||||
|
||||
include(CTest)
|
||||
if(BUILD_TESTING AND NOT CMAKE_CROSSCOMPILING)
|
||||
add_test(NAME frame_dictation.cli
|
||||
COMMAND "${CMAKE_COMMAND}"
|
||||
"-DAPP=$<TARGET_FILE:frame-dictation>"
|
||||
"-DEXPECTED_VERSION=${PROJECT_VERSION}"
|
||||
-P "${CMAKE_CURRENT_SOURCE_DIR}/tests/cli.cmake")
|
||||
add_test(NAME frame_dictation.install_preflight
|
||||
COMMAND sh "${CMAKE_CURRENT_SOURCE_DIR}/tests/install-preflight.sh"
|
||||
"${CMAKE_CURRENT_SOURCE_DIR}/scripts/install-preflight.sh")
|
||||
endif()
|
||||
@@ -0,0 +1,53 @@
|
||||
# Frame Dictation
|
||||
|
||||
Standalone, on-device voice typing for Steam Frame.
|
||||
|
||||
**Status: project scaffold and design only.** The executable prints help/version;
|
||||
it does not yet render an overlay, open a microphone, load a model or type text.
|
||||
|
||||
Planned path: **OpenVR overlay → local Parakeet Redux CPU inference → Gamescope
|
||||
Unicode input**. No external application checkout, library, submodule, desktop
|
||||
inference server or running scene host is required.
|
||||
|
||||
**Distribution goal:** one-command installation from GitHub releases, entirely
|
||||
user-local, with no Steam store AppID. See the [installer design](docs/install-design.md).
|
||||
There is no working installer or published release yet.
|
||||
|
||||
For a read-only check of proposed release prerequisites on a Frame, run
|
||||
`sh scripts/install-preflight.sh`. It checks Linux AArch64/glibc and bootstrap
|
||||
tools, reports system Python (3.12+ for a possible source worker), Git and uv.
|
||||
Git and uv are not required for the planned bundled release. This check downloads
|
||||
and installs nothing; passing it does **not** mean an install or dictation works.
|
||||
|
||||
## Build the scaffold
|
||||
|
||||
Requirements: CMake 3.20+ and a C++20 compiler. No third-party packages or downloads.
|
||||
|
||||
```sh
|
||||
cmake -S . -B build
|
||||
cmake --build build
|
||||
ctest --test-dir build --output-on-failure
|
||||
./build/frame-dictation --help
|
||||
```
|
||||
|
||||
This is a host-native scaffold build, not an ARM64 deployment or headset test.
|
||||
Future OpenVR, audio and inference integrations must be explicitly configured;
|
||||
configuration/build/tests must never install packages or launch SteamVR implicitly.
|
||||
|
||||
## Project map
|
||||
|
||||
- [Design](docs/design.md): UI, local inference, input backend and acceptance gates.
|
||||
- [Frame API evidence](docs/evidence/frame-dictation-apis-2026-09-24.md): successful
|
||||
mixed-script Unicode test and overlay-client initialization; known limits.
|
||||
- [Installer design](docs/install-design.md): GitHub install, standalone OpenVR
|
||||
identity, packaging, upgrades and uninstall.
|
||||
- [Provenance](docs/provenance.md): origin of the investigation, model/runtime pins.
|
||||
- `src/main.cpp`: inert CLI entry point.
|
||||
- `tests/cli.cmake`: hardware-free scaffold smoke test.
|
||||
- `scripts/install-preflight.sh`: read-only proposed packaging prerequisite check.
|
||||
- [Agent guidance](AGENTS.md): project boundaries and safe validation.
|
||||
|
||||
First implementation gate: an isolated ARM64 CPU trial of the pinned Redux model.
|
||||
Then a minimal overlay and explicit insertion into a disposable target. Neither
|
||||
inference nor an overlay is implemented here yet. No model weights, recordings,
|
||||
credentials, engine assets or runtime binaries are included.
|
||||
+292
@@ -0,0 +1,292 @@
|
||||
# Native Frame dictation overlay — proposal
|
||||
|
||||
Standalone project design; see
|
||||
[provenance](provenance.md) and [historical Frame probes](evidence/frame-dictation-apis-2026-09-24.md).
|
||||
Only the inert build scaffold is implemented. This document describes proposed runtime behavior.
|
||||
|
||||
## Recommendation
|
||||
|
||||
Build a small native ARM64 **OpenVR overlay application**, independent of
|
||||
other applications, with **local Parakeet Redux CPU inference**. Use Gamescope's
|
||||
input-method protocol for literal Unicode text; keep Linux key-injection APIs
|
||||
as explicit fallbacks. No desktop ASR server, network hop, LLM cleanup, scene
|
||||
renderer, avatar, desktop capture or root service is needed in the primary path.
|
||||
|
||||
A first-class product goal is a **one-command GitHub install without a Steam store
|
||||
AppID**. Package a prebuilt native executable and isolated CPU runtime; use a normal
|
||||
OpenVR application key for registration, not Steamworks. Installation must remain
|
||||
user-local with opt-in autolaunch. See [installation design](install-design.md).
|
||||
|
||||
```text
|
||||
controller PTT / overlay mic button
|
||||
↓
|
||||
SDL3 → PipeWire/PulseAudio mic → bounded 16 kHz mono clip
|
||||
↓
|
||||
one persistent local Parakeet Redux CPU worker
|
||||
↓
|
||||
correlated literal transcript → focus/delivery policy
|
||||
↓
|
||||
Gamescope IME set_string + commit → focused Frame application
|
||||
↘ overlay preview / error / explicit Insert when delivery is unsafe
|
||||
```
|
||||
|
||||
The CPU worker is local process isolation, not remote inference or a service
|
||||
framework. Implement a small transport owned by this repository. Keep the model
|
||||
loaded between utterances; do not spawn Python/load 178 MB for every release.
|
||||
|
||||
## Minimal interaction
|
||||
|
||||
A small hand-attached status chip while armed; a compact review panel only when
|
||||
needed. Allow head-relative placement for users who prefer it. No permanent large
|
||||
dashboard obscuring the application.
|
||||
|
||||
```text
|
||||
[ mic ] Ready · On-device [ settings ]
|
||||
Target: selected application
|
||||
|
||||
Recording… 00:04 [ Cancel ]
|
||||
|
||||
"The recognized text appears here."
|
||||
[ Insert ] [ Discard ] [ Enter — separate action ]
|
||||
```
|
||||
|
||||
- States: disabled, warming, ready, recording, transcribing, review, inserted,
|
||||
unavailable/error. Recording uses visible icon + text, not color alone.
|
||||
- Hold to speak, release to finish. A click-to-start/stop overlay button provides
|
||||
a binding-independent alternative. Bound recording to 20 seconds; discard
|
||||
accidental taps (initial threshold: 200 ms).
|
||||
- **Quick typing:** insert on completion only when the explicitly armed target
|
||||
is still valid. **Review mode:** always wait for Insert. Bring up review instead
|
||||
of silently losing a transcript or typing into a new target.
|
||||
- Enter is always a separate press after insertion. Never interpret "submit",
|
||||
"delete" or other speech as commands in this utility. Do not auto-submit.
|
||||
- No generic "undo last dictation" initially: another application's edits/cursor
|
||||
cannot be reliably rolled back by a guessed number of backspaces.
|
||||
- Stop/disable releases owned keys and microphone, invalidates pending delivery,
|
||||
and terminates only the owned worker. No Steam/session/SSH cleanup commands.
|
||||
|
||||
## Host and rendering boundary
|
||||
|
||||
Implement the standalone `frame-dictation` executable, initialized
|
||||
with `VRApplication_Overlay`. Frame accepted that application type and
|
||||
`IVROverlay_028` in the probe. Use `CreateOverlay`, tracked-device-relative
|
||||
transform, `ShowOverlay`/`HideOverlay`, and `PollNextOverlayEvent` for the panel.
|
||||
Use SteamVR's overlay interaction rather than inventing scene controller rays.
|
||||
|
||||
Do not link an external scene host or launch another application behind the
|
||||
panel. A small overlay-specific RAII owner in this repository should manage OpenVR, overlay
|
||||
handles, input manifest and shutdown. No dependency on external application
|
||||
libraries, assets, build trees or Python environments.
|
||||
|
||||
Start with one small RGBA panel updated only on UI changes and a bounded recording
|
||||
indicator cadence. `SetOverlayRaw` is the simplest proof route; measure upload
|
||||
cost before selecting a persistent Vulkan `SetOverlayTexture` path. No stereo
|
||||
eye targets or per-eye scene rendering. Keep tracking in compositor transforms,
|
||||
not an application-rendered hand-pose animation loop. Do not promise a particular
|
||||
GPU cost until measured.
|
||||
|
||||
A scene renderer's canvas/MSDF resources are not
|
||||
an OpenVR overlay backend and are not imported here. Use a small independently
|
||||
licensed font/icon set and minimal panel renderer. Any later source/asset reuse
|
||||
requires an explicit license-reviewed extraction, never a runtime path into the
|
||||
engine checkout.
|
||||
|
||||
### Controller bindings
|
||||
|
||||
Expose PTT, cancel and explicit insert/Enter as named SteamVR actions; let the
|
||||
user bind them. Do not assume a scene app's left-bumper mapping works globally or
|
||||
silently takes a game's button away. Check action activity and neutral rearm.
|
||||
|
||||
OpenVR 2.15.6 documents experimental overlay action-set priorities
|
||||
`0x01000000..0x01FFFFFF`, gated by SteamVR's **Experimental overlay input overrides**
|
||||
setting. This can selectively override scene input, but is not guaranteed enabled
|
||||
or usable on this Frame. Do not toggle it automatically. Begin with overlay mic
|
||||
controls; validate a global PTT binding separately with a scene active, dashboard
|
||||
open/closed, lost tracking, and reconnection. Overlay interactivity/input ownership
|
||||
is distinct from OS keyboard focus.
|
||||
|
||||
## Text delivery: use the proven path first
|
||||
|
||||
### 1. Gamescope IME — primary Frame backend
|
||||
|
||||
Connect to the discovered Gamescope socket; bind
|
||||
`gamescope_input_method_manager` and `wl_seat`, handle unavailable/done events,
|
||||
then issue `set_string(valid_utf8)` and `commit(last_serial)`. A v2 binding is
|
||||
sufficient for text and actions even though the current server advertises v3.
|
||||
Use generated bindings from the pinned XML, not a production handwritten wire
|
||||
protocol. Own/destroy the IME object deliberately and release it while disabled.
|
||||
|
||||
This delivered an exact mixed-script string into the disposable Xwayland
|
||||
receiver. It does not need clipboard ownership, sudo, `uinput`, application
|
||||
plugins, or a new input daemon. However it is a **private Gamescope extension**;
|
||||
isolate it behind a version-gated backend and test after SteamOS updates.
|
||||
|
||||
The server implements text through synthetic key events and a temporary keymap,
|
||||
not a guaranteed rich-text/IME edit operation in every app. Verify actual target
|
||||
toolkits and games. Respect singleton/unavailable handling and Steam-keyboard
|
||||
coexistence. Never use the installed `gamescope-type` CLI as a transcript pipe:
|
||||
its inspected sample loop is byte-oriented and interprets newline as Submit.
|
||||
|
||||
### 2. Explicit fallbacks, not a framework built up front
|
||||
|
||||
- **X11 clipboard + XTEST paste:** useful for apps that reject direct Unicode
|
||||
key events but accept paste. Own the appropriate X selection (CLIPBOARD or
|
||||
PRIMARY), serving the `UTF8_STRING` target on the destination X server;
|
||||
select the app's paste chord explicitly. Do not assume Ctrl+V in terminals.
|
||||
Do not overwrite/restore arbitrary clipboard history silently; expose Copy as
|
||||
a deliberate fallback. This backend has not yet been exercised on Frame.
|
||||
- **libei:** current server accepts a sender and advertises KEYBOARD but **not
|
||||
TEXT**. Suitable for evdev-style key chords, not automatic Unicode insertion.
|
||||
Bind/resume/device lifecycle must still be tested. Direct EIS socket access is
|
||||
Gamescope-specific here; no portal RemoteDesktop route was exposed.
|
||||
- **uinput:** current account can open it without sudo. Reserve for raw virtual
|
||||
keyboard needs; creating/retiring a device is unnecessary for primary dictation.
|
||||
Unicode is not an evdev keycode, so this alone does not solve text delivery.
|
||||
|
||||
Do not assume generic labwc virtual-keyboard/data-control protocols are present:
|
||||
those globals are absent. Fail visibly if the primary backend is unavailable;
|
||||
no automatic privilege escalation or OS package/configuration edits.
|
||||
|
||||
### Target/focus policy
|
||||
|
||||
The IME serial is **not** an established target-generation guard. Input goes to
|
||||
the seat's current focus. No general Linux input API makes insertion into an
|
||||
arbitrary app atomic with a focus check.
|
||||
|
||||
For the first supported route, use an explicitly armed Xwayland destination.
|
||||
Track display identity, active top-level, actual X keyboard-focus window and a
|
||||
monotonic focus generation; invalidate on focus loss/regain, window destruction,
|
||||
disconnect or target change. Observe changes throughout capture/inference and
|
||||
recheck immediately before delivery. Do not restore another app's focus behind
|
||||
the user's back. Refuse quick insertion when modifier keys are held or target
|
||||
identity is ambiguous. A matching final window ID alone is insufficient.
|
||||
|
||||
Gamescope exposes focus-display/window root properties, but their encoding and
|
||||
relationship to seat focus need implementation-specific validation. Do not infer
|
||||
that X display `:0` is always the destination, or that an X focus observation
|
||||
identifies a native Wayland text field. For unobservable native Wayland focus,
|
||||
require explicit review/Insert; do not advertise safe auto-targeting.
|
||||
|
||||
If focus changes, keep the result in review. A fresh Insert explicitly approves
|
||||
the current destination and creates a new delivery authorization. Recheck again
|
||||
at insertion. This minimizes stale delivery but does **not** eliminate a race
|
||||
between the final check and global input processing; do not claim otherwise.
|
||||
A Wayland roundtrip means compositor processing, not application consumption.
|
||||
Report `input queued`, never `message sent`.
|
||||
|
||||
Validate UTF-8 and enforce the 4096-byte bound; flatten line breaks/tabs, reject
|
||||
NUL, escape and other control characters. No shell evaluation, commands, Enter,
|
||||
terminal escapes or action inference from recognized text. One correlated request
|
||||
may deliver at most once; cancellation invalidates it before any later reply.
|
||||
|
||||
## On-device Redux
|
||||
|
||||
Use the **same** `moondream/parakeet-redux` revision from the benchmark:
|
||||
`fad622f25f303105c20d70e201bcc477c88b620c` (177,774,490-byte weight file), initially
|
||||
moondream 2.4.0 / kestrel 0.8.0. Do not substitute dense Ultra or silently fall
|
||||
back to desktop/cloud inference.
|
||||
|
||||
The earlier benchmark used this local API (its source belongs to the originating
|
||||
repository, not this project):
|
||||
|
||||
```python
|
||||
model = md.photon("moondream/parakeet-redux", model_path=local_model_directory,
|
||||
device="cpu", cpu_threads=thread_budget)
|
||||
text = model.transcribe(audio=mono_float32, sample_rate=16000)["text"]
|
||||
```
|
||||
|
||||
Linux ARM64 Python 3.12 native wheels and a packaged CPU payload exist. That is
|
||||
sufficient reason to **try the native CPU path first**, not proof of Frame
|
||||
performance. Its 61/137 ms desktop median/p95 must not be reused as an estimate
|
||||
for the headset. Its historical dictation quality was worse than Small.en on
|
||||
that tiny dataset; keep transcript visibility and a cheap retry.
|
||||
|
||||
Implement a small independent audio/worker adapter with explicit capture,
|
||||
single-request bounds, owner-only runtime files, correlated replies and cancellation.
|
||||
The worker should be implemented independently; do not link, vendor or import
|
||||
another application's speech code. Avoid a generic provider framework: one
|
||||
explicit Redux worker is enough for the first version.
|
||||
|
||||
Suggested ownership, introduced only as implementation needs it:
|
||||
|
||||
- `src/overlay.*`: OpenVR lifetime, panel presentation and controller actions.
|
||||
- `src/audio.*`: explicit SDL capture and bounded mono PCM.
|
||||
- `src/worker.*`: one local child process, bounded requests/replies, timeout/reaping.
|
||||
- `src/text_input.*`: Gamescope protocol, focus observation and delivery policy.
|
||||
- `python/frame_dictation/`: persistent CPU Redux worker, no external application imports.
|
||||
|
||||
The worker reads a fixed private clip and returns an ID-correlated literal string;
|
||||
one request at a time, 64 KiB framed messages, 4096-byte transcript, and a bounded
|
||||
processing timeout. No shell commands in IPC and no input authority in the worker.
|
||||
|
||||
- One persistent worker and one loaded model; no queue of utterances. Initially
|
||||
batch at PTT release, not speculative streaming or endpoint/VAD delay.
|
||||
- Start with **two CPU threads**, compare against four on Frame; bound Torch and
|
||||
native kernel pools. Choose measured latency versus compositor contention,
|
||||
not the desktop's thread count by habit. No real-time scheduling or permanent
|
||||
CPU pinning initially; inspect runtime affinity behaviour during measurement.
|
||||
- Explicit local model path and offline loading. Missing runtime/weights produces
|
||||
an actionable error, not an unsolicited download/network fallback.
|
||||
- Optional explicit enable/warm-up before first PTT; warming must not record audio.
|
||||
Expose that lifecycle explicitly rather than warming on import or construction.
|
||||
Otherwise display first-use loading honestly. Keep
|
||||
model reuse after normal completion; cancellation may restart the owned worker.
|
||||
- Use a private owner-only directory under `$XDG_RUNTIME_DIR` for bounded
|
||||
tmpfs-backed clips; remove them on completion, error, cancellation and shutdown.
|
||||
Avoid persistent audio/transcripts by default. Local IPC is not a network hop;
|
||||
do not add shared-memory complexity before measuring it.
|
||||
- Use CPU-only Torch where supported; test native kernel import/model load with
|
||||
no CUDA device/runtime assumption. Do not copy the desktop's x86 venv.
|
||||
- Weights are ~178 MB; Torch, kernels, temporary conversion and activations mean
|
||||
install size/RSS will be larger. Measure cold load, peak RSS and package size.
|
||||
Review runtime redistribution licensing separately from model attribution.
|
||||
|
||||
Microphone access does not mute VRChat or any other social-voice app. Shared
|
||||
PipeWire capture may let both hear the same utterance; the overlay must not claim
|
||||
private dictation unless the other app's transmission is separately muted. No
|
||||
automatic global microphone mute/reroute in this scope.
|
||||
|
||||
## Delivery milestones and acceptance gates
|
||||
|
||||
These are proposed implementation gates, **not completed acceptance**:
|
||||
|
||||
1. **Offline ARM64 CPU spike:** isolated user-directory environment, pinned local
|
||||
model, permitted runtime distribution. Run known nonprivate clips, then the
|
||||
consented benchmark clips only if explicitly made available within their data
|
||||
scope. Record cold load, first/warm p50/p95, errors, peak RSS, two/four threads and dependencies.
|
||||
Confirm no required NVIDIA/desktop connection. Stop and report if the packed
|
||||
CPU runtime is incompatible; do not silently expand to dense weights.
|
||||
2. **Minimal overlay:** render status/review panel while another scene stays active;
|
||||
confirm dashboard/hand placement, input events, close/reopen and no scene-focus
|
||||
takeover. Validate global PTT separately rather than blocking the clickable
|
||||
prototype on experimental override support.
|
||||
3. **Real dictation path:** microphone → local Redux → preview → explicit insert
|
||||
into a disposable target; then enable quick typing after target tracking tests.
|
||||
Test Unicode, punctuation, long bounded clips, silence, cancellation, duplicate
|
||||
replies, lost mic, worker crash and missing model without persisting speech.
|
||||
4. **Target matrix:** Xwayland terminal/browser and selected native/Proton game
|
||||
text fields; Steam keyboard coexistence; native Wayland targets separately.
|
||||
Test focus changes during capture/inference, rapid loss/regain, held modifiers,
|
||||
explicit Insert retargeting, no hidden Enter, and no second delivery.
|
||||
5. **In-headset acceptance:** readable feedback and comfortable PTT; measured
|
||||
release-to-insert latency and compositor timing while an actual scene runs;
|
||||
no noticeable sustained thermal/battery regression. Set numeric budgets after
|
||||
the ARM64 spike, not from an x86 result. Physical success remains human-led.
|
||||
|
||||
Native Frame app typing is the first scope. Local keystrokes may or may not be
|
||||
forwarded by the PC-streaming client; that route needs its own test. If necessary,
|
||||
a future optional **text-only** host bridge could send the completed transcript,
|
||||
without moving recognition off Frame. It is not part of this initial design.
|
||||
|
||||
## Standalone dependency and test policy
|
||||
|
||||
The scaffold currently needs only CMake and a C++20 compiler. Later integrations
|
||||
will explicitly select OpenVR, SDL3, Wayland client/protocol bindings and a Python
|
||||
Redux environment. Pin revisions and review licenses when introduced. No automatic
|
||||
fetch/install in configure or normal tests; no external checkout discovery.
|
||||
|
||||
Hardware-free tests should cover state transitions, bounded PCM/transcripts,
|
||||
worker framing/timeout/cancellation, duplicate/stale replies and focus generations
|
||||
with fakes. Separate opt-in Frame checks cover protocol availability, owned-window
|
||||
insertion, microphones, inference and overlays. No fixture establishes physical
|
||||
headset acceptance. See [provenance](provenance.md) for historical source anchors.
|
||||
@@ -0,0 +1,44 @@
|
||||
# Historical Steam Frame API probes — 2026-09-24
|
||||
|
||||
These observations were made on one device and SteamOS build, not rerun as part
|
||||
of this repository's scaffold. They do not prove current availability or headset
|
||||
acceptance. No probe code, user audio, credentials or runtime binaries are shipped.
|
||||
See the [design](../design.md) for proposed implementation and safety gates.
|
||||
|
||||
## Environment
|
||||
|
||||
SteamOS 0.4.0 (VR variant), AArch64, glibc 2.39, Python 3.12.3;
|
||||
Gamescope 3.16.28-2. Linux ARM64 package availability is not evidence of
|
||||
inference speed or compatibility with this device.
|
||||
|
||||
## Observations and limits
|
||||
|
||||
| Surface | Historical observation | Limit |
|
||||
| --- | --- | --- |
|
||||
| OpenVR | Native `VRApplication_Overlay` initialization succeeded; `IVROverlay_028`, `IVRInput_011`, `IVRSystem_026` and `IVRApplications_008` were accepted. | No rendered overlay, binding or autolaunch test. |
|
||||
| Gamescope IME | `gamescope_input_method_manager` v3 advertised; v2 binding accepted and returned `done(serial=1)`. | Discovery is not delivered input. |
|
||||
| Unicode delivery | A separate disposable X11/XIM receiver on the device accepted an exact mixed-script Unicode fixture via `set_string` and `commit` after checking focus on its owned window. | One Xwayland receiver, not general games, native Wayland or PC-streamed targets. No Enter was sent. |
|
||||
| libei | Sender connected; seat advertised KEYBOARD, not TEXT. | No bound device or delivered key test. |
|
||||
| XTEST / uinput | XTEST advertised; `/dev/uinput` was openable by the test account. | No injected XTEST key or virtual device. |
|
||||
| Audio | PipeWire input device enumerated. | No microphone recording or model inference in this probe. |
|
||||
| Portal | Existing portal introspection exposed no RemoteDesktop, InputCapture, Clipboard, ScreenCast or GlobalShortcuts interface. | Not a guarantee for future OS releases. |
|
||||
|
||||
The Xwayland receiver was destroyed after the test. No packages, services or
|
||||
Steam settings were changed. No microphone audio was recorded. This is a
|
||||
historical API probe, not a live availability check or install smoke test.
|
||||
|
||||
## Implementation cautions
|
||||
|
||||
Gamescope's [input-method protocol](https://github.com/ValveSoftware/gamescope/blob/3.16.28/protocol/gamescope-input-method.xml)
|
||||
is private and version-sensitive; see its [implementation](https://github.com/ValveSoftware/gamescope/blob/3.16.28/src/ime.cpp).
|
||||
The serial is not a verified focus-generation guard; a roundtrip is not a text
|
||||
consumption acknowledgement. The inspected [`gamescope-type` example](https://github.com/ValveSoftware/gamescope/blob/3.16.28/src/Apps/gamescope_type.c)
|
||||
interprets newline as Submit: do not use it as a transcript pipe.
|
||||
Generic virtual-keyboard and data-control Wayland globals were not advertised
|
||||
in this probe. Focus tracking and Steam keyboard coexistence need separate tests.
|
||||
|
||||
The proposed Redux model is pinned to revision
|
||||
`fad622f25f303105c20d70e201bcc477c88b620c`; the weight file was
|
||||
177,774,490 bytes in an earlier local inspection. ARM64 Python 3.12 wheels
|
||||
appeared available, but no native model load or inference was performed on the
|
||||
Frame. Runtime redistribution terms require separate review.
|
||||
@@ -0,0 +1,105 @@
|
||||
# Installation and distribution goal
|
||||
|
||||
**User goal:** install from GitHub with a `curl … | bash`-style command, without a
|
||||
Steam store AppID. This is a requirement for the future release, not a working
|
||||
installer. No release artifacts or functional installer are published yet.
|
||||
|
||||
The read-only `scripts/install-preflight.sh` checks whether a host appears suitable
|
||||
for the **proposed** Linux ARM64 glibc package format and has the expected basic
|
||||
bootstrap utilities. It reports system Python, Git and uv, but none is required
|
||||
for the intended bundled release. A read-only check on one Frame observed
|
||||
Python 3.12.3 and Git, but not uv; availability may change. This script
|
||||
is not an installer or a model/runtime compatibility test; it has no downloads,
|
||||
registration, SteamVR initialization or persistent changes. No glibc minimum can
|
||||
be certified until release artifacts are chosen and tested.
|
||||
|
||||
## Non-Steam overlay identity
|
||||
|
||||
OpenVR overlay applications do not require a Steam store AppID or Steamworks.
|
||||
The native executable initializes as `VRApplication_Overlay`. For discoverability
|
||||
and optional autolaunch, register an OpenVR application manifest with a stable,
|
||||
project-owned **string application key** (proposed: `local.frame-dictation.overlay`).
|
||||
That key is not a numeric Steam AppID. No purchase/store listing or non-Steam Steam
|
||||
library shortcut should be necessary for the normal route.
|
||||
|
||||
The manifest identifies the installed executable; the installer/registration
|
||||
helper should use `IVRApplications::AddApplicationManifest` and the corresponding
|
||||
remove operation, not hand-edit Steam's internal JSON. Autolaunch uses the OpenVR
|
||||
application setting only when explicitly requested. Validate the exact manifest,
|
||||
launch behaviour, registration persistence and uninstall on native Frame before
|
||||
claiming this route works end to end. Existing probes established overlay client
|
||||
initialization, not manifest installation.
|
||||
|
||||
SteamVR/OpenVR must already be installed and usable. If registration needs a
|
||||
running runtime, defer it to the first explicit launch rather than starting or
|
||||
restarting SteamVR behind the user's back.
|
||||
|
||||
## Intended user experience
|
||||
|
||||
1. Run one documented command from the eventual GitHub repository/release.
|
||||
2. Installer identifies native Linux ARM64 Frame, resolves a pinned release and
|
||||
explains/downloads the application, compatible CPU runtime and pinned model.
|
||||
3. User-local installation provides a simple `frame-dictation` launcher, desktop
|
||||
entry where supported, and an OpenVR manifest. No compiler, engine checkout,
|
||||
Python dependency troubleshooting or separate ASR server for ordinary users.
|
||||
4. The user explicitly launches/enables dictation. No installation-time microphone
|
||||
recording, input injection, inference benchmark or overlay takeover.
|
||||
|
||||
Illustrative command shape only; `OWNER`, `REPO` and `VERSION` are placeholders:
|
||||
|
||||
```sh
|
||||
curl --fail --silent --show-error --location \
|
||||
https://raw.githubusercontent.com/OWNER/REPO/VERSION/install.sh | bash
|
||||
```
|
||||
|
||||
Also document a download-inspect-run path for users who do not want to pipe remote
|
||||
code into a shell. Pin a release/tag instead of executing a moving branch by default.
|
||||
Offer explicit version selection and noninteractive flags; do not read interactive
|
||||
confirmation from stdin while the installer itself is arriving through that pipe.
|
||||
|
||||
## Packaging boundary
|
||||
|
||||
- Prebuilt ARM64 executable plus a known-compatible, isolated CPU inference runtime;
|
||||
no external application libraries/assets and no system Python modification.
|
||||
- Model fetched during explicit installation/setup, with pinned revision/hash and
|
||||
attribution. A documented `--without-model` option can defer the large download.
|
||||
No surprise first-utterance downloads. Licensing may require obtaining particular
|
||||
runtime components from their vendor instead of redistributing them in our tarball.
|
||||
- Install under `$XDG_DATA_HOME/frame-dictation` (default `~/.local/share/...`),
|
||||
configuration under `$XDG_CONFIG_HOME/frame-dictation`, optional launcher in
|
||||
`~/.local/bin`; transient audio stays in a private `$XDG_RUNTIME_DIR` directory.
|
||||
- No sudo, OS read-only-root changes, package-manager installs, udev changes,
|
||||
`/dev/uinput` permission changes or modifications to unrelated launchers.
|
||||
- No Steam store AppID, Steamworks SDK, root service or network ASR dependency.
|
||||
- Distribution license/third-party notices must be resolved before publication;
|
||||
the small model file alone is not the full runtime or redistribution permission.
|
||||
|
||||
## Installer lifecycle and safety
|
||||
|
||||
- Detect architecture, libc and prerequisites first. Unsupported hosts fail with a
|
||||
clear explanation; never install an x86 payload silently on ARM64.
|
||||
- Download to a staging directory, check versioned SHA-256 manifests and archive
|
||||
paths, then atomically select the completed version. HTTPS/checksums alone do not
|
||||
authenticate a compromised publisher; use signed release metadata if provided.
|
||||
- Keep configuration across upgrades; retain the previous version for rollback.
|
||||
Refuse or defer replacement while this application's process is running rather
|
||||
than killing arbitrary processes. Never touch SSH or unrelated sessions.
|
||||
- Autostart is opt-in (`--autostart` or explicit settings); do not enable a systemd
|
||||
service or SteamVR autolaunch by default. Do not enable overlay input overrides.
|
||||
- Uninstall removes only owned launcher, manifest registration and install files;
|
||||
model/config deletion is separately explicit. Preserve other SteamVR apps.
|
||||
- No update daemon initially. A deliberate rerun/update command is sufficient.
|
||||
|
||||
## Acceptance before advertising one-command installation
|
||||
|
||||
- Clean supported Frame: install without sudo/compiler/engine checkout/store AppID;
|
||||
launch overlay, load local Redux and type into an owned disposable target.
|
||||
- Normal use after installation needs no network connection or desktop ASR host.
|
||||
- Failed download/hash, unsupported architecture, low disk space and interrupted
|
||||
upgrades leave a usable previous install or a cleanly reported failure.
|
||||
- Reinstall, rollback and uninstall preserve unrelated data and SteamVR entries.
|
||||
- Noninteractive piped invocation never hangs on stdin; inspection-first path works.
|
||||
- Autolaunch remains off unless chosen; uninstall removes only our registration.
|
||||
- Hardware-free installer tests use temporary homes, mocked runtime registration
|
||||
and local fixture artifacts. Never exercise a real user's Steam configuration
|
||||
in ordinary CI/CTest.
|
||||
@@ -0,0 +1,29 @@
|
||||
# Technical provenance and limits
|
||||
|
||||
Frame Dictation is a standalone scaffold and proposal. No external application
|
||||
source, build tree, assets, Python environment, model weights or binaries are
|
||||
included. The dated [device probe](evidence/frame-dictation-apis-2026-09-24.md)
|
||||
records limited historical observations; its fixture code and private logs are
|
||||
not part of this repository. No current device availability or release install
|
||||
can be inferred from those observations.
|
||||
|
||||
## Model and preliminary measurements
|
||||
|
||||
Proposed model: [moondream/parakeet-redux](https://huggingface.co/moondream/parakeet-redux/tree/fad622f25f303105c20d70e201bcc477c88b620c),
|
||||
revision `fad622f25f303105c20d70e201bcc477c88b620c`. An earlier local
|
||||
inspection measured a 177,774,490-byte weight file. Its model card identifies
|
||||
CC-BY-4.0; separate runtime/kernel redistribution terms must be reviewed before
|
||||
packaging. No license for those artifacts is granted by this repository.
|
||||
|
||||
A preliminary, unpublished desktop benchmark used Python 3.12.13, moondream
|
||||
2.4.0, kestrel 0.8.0 and four CPU threads. On an i7-13700K, warm decode
|
||||
median/p95 were 61/137 ms and raw WER 11.11% on just 25 reviewed clips / 189
|
||||
words. This is not a general accuracy estimate or Frame latency prediction. The
|
||||
CPU-selected desktop process also occupied GPU memory; GPU-free operation was
|
||||
not established. No Frame inference benchmark exists yet.
|
||||
|
||||
## Public platform references
|
||||
|
||||
- [OpenVR 2.15.6](https://github.com/ValveSoftware/openvr/releases/tag/v2.15.6)
|
||||
- [Gamescope 3.16.28 input-method protocol](https://github.com/ValveSoftware/gamescope/blob/3.16.28/protocol/gamescope-input-method.xml)
|
||||
- [Gamescope IME implementation](https://github.com/ValveSoftware/gamescope/blob/3.16.28/src/ime.cpp)
|
||||
@@ -0,0 +1,79 @@
|
||||
#!/bin/sh
|
||||
# Read-only host check for the proposed release installer. Does not install anything.
|
||||
set -u
|
||||
|
||||
if [ "$#" -ne 0 ]; then
|
||||
if [ "$#" -eq 1 ] && [ "$1" = --help ]; then
|
||||
printf '%s\n' 'Usage: sh scripts/install-preflight.sh' \
|
||||
'Checks the proposed Linux ARM64 installation prerequisites; makes no changes.'
|
||||
exit 0
|
||||
fi
|
||||
printf '%s\n' 'No arguments are supported (except --help).' >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
errors=0
|
||||
os=$(uname -s 2>/dev/null) || os=unknown
|
||||
arch=$(uname -m 2>/dev/null) || arch=unknown
|
||||
printf 'Host: %s %s\n' "$os" "$arch"
|
||||
if [ "$os" != Linux ] || [ "$arch" != aarch64 ]; then
|
||||
printf '%s\n' 'Unsupported host: planned release payload is Linux AArch64 only.' >&2
|
||||
errors=1
|
||||
fi
|
||||
|
||||
libc=unknown
|
||||
if command -v getconf >/dev/null 2>&1; then
|
||||
libc=$(getconf GNU_LIBC_VERSION 2>/dev/null) || libc=unknown
|
||||
fi
|
||||
printf 'C library: %s\n' "$libc"
|
||||
case "$libc" in
|
||||
'glibc '*) ;;
|
||||
*) printf '%s\n' 'Unsupported/unknown C library: planned ARM64 wheels require glibc.' >&2
|
||||
errors=1 ;;
|
||||
esac
|
||||
|
||||
# Expected for a future curl/tar release bootstrap, not for building from source.
|
||||
for dep in curl tar sha256sum mktemp mkdir mv; do
|
||||
if command -v "$dep" >/dev/null 2>&1; then
|
||||
printf 'Required tool: %s found\n' "$dep"
|
||||
else
|
||||
printf 'Required tool: %s MISSING\n' "$dep" >&2
|
||||
errors=1
|
||||
fi
|
||||
done
|
||||
|
||||
if command -v git >/dev/null 2>&1; then
|
||||
printf '%s\n' 'Git: found (not required for a release install)'
|
||||
else
|
||||
printf '%s\n' 'Git: missing (not required for a release install)'
|
||||
fi
|
||||
if command -v uv >/dev/null 2>&1; then
|
||||
printf '%s\n' 'uv: found (not required for a bundled runtime)'
|
||||
else
|
||||
printf '%s\n' 'uv: missing (not required for a bundled runtime)'
|
||||
fi
|
||||
|
||||
if command -v python3 >/dev/null 2>&1; then
|
||||
version=$(python3 --version 2>&1) || version=unknown
|
||||
printf 'System Python: %s (not required for a bundled runtime)\n' "$version"
|
||||
case "$version" in
|
||||
'Python 3.'*)
|
||||
minor=${version#Python 3.}
|
||||
minor=${minor%%.*}
|
||||
case "$minor" in
|
||||
''|*[!0-9]*) printf '%s\n' 'System Python version could not be parsed.' ;;
|
||||
*) if [ "$minor" -lt 12 ]; then
|
||||
printf '%s\n' 'System Python is older than 3.12; unsuitable for the proposed optional source worker.'
|
||||
fi ;;
|
||||
esac ;;
|
||||
*) printf '%s\n' 'System Python version could not be parsed.' ;;
|
||||
esac
|
||||
else
|
||||
printf '%s\n' 'System Python: missing (not required for a bundled runtime)'
|
||||
fi
|
||||
|
||||
if [ "$errors" -ne 0 ]; then
|
||||
printf '%s\n' 'Preflight failed. No changes were made.' >&2
|
||||
exit 1
|
||||
fi
|
||||
printf '%s\n' 'Preflight passed for the proposed package format. No installer or release payload exists yet; nothing was installed.'
|
||||
@@ -0,0 +1,24 @@
|
||||
#include <iostream>
|
||||
#include <string_view>
|
||||
|
||||
namespace {
|
||||
void help() {
|
||||
std::cout << "Frame Dictation — standalone Steam Frame voice typing\n"
|
||||
"Scaffold only: overlay, microphone, ASR and input are not implemented.\n\n"
|
||||
"Usage: frame-dictation [--help | --version]\n"
|
||||
"No device access or background processes are started.\n";
|
||||
}
|
||||
}
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
if (argc == 1 || (argc == 2 && std::string_view(argv[1]) == "--help")) {
|
||||
help();
|
||||
return 0;
|
||||
}
|
||||
if (argc == 2 && std::string_view(argv[1]) == "--version") {
|
||||
std::cout << "frame-dictation " << FRAME_DICTATION_VERSION << " (scaffold)\n";
|
||||
return 0;
|
||||
}
|
||||
std::cerr << "Unsupported arguments. Use --help; runtime features are not implemented.\n";
|
||||
return 2;
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
if(NOT DEFINED APP OR NOT DEFINED EXPECTED_VERSION)
|
||||
message(FATAL_ERROR "APP and EXPECTED_VERSION are required")
|
||||
endif()
|
||||
|
||||
foreach(mode IN ITEMS default help version invalid extra)
|
||||
set(args)
|
||||
set(expected_exit 0)
|
||||
if(mode STREQUAL "help")
|
||||
set(args --help)
|
||||
elseif(mode STREQUAL "version")
|
||||
set(args --version)
|
||||
elseif(mode STREQUAL "invalid")
|
||||
set(args --record)
|
||||
set(expected_exit 2)
|
||||
elseif(mode STREQUAL "extra")
|
||||
set(args --help --record)
|
||||
set(expected_exit 2)
|
||||
endif()
|
||||
execute_process(COMMAND "${APP}" ${args}
|
||||
RESULT_VARIABLE result OUTPUT_VARIABLE output ERROR_VARIABLE error TIMEOUT 5)
|
||||
if(NOT "${result}" STREQUAL "${expected_exit}")
|
||||
message(FATAL_ERROR "${mode}: exit ${result}, expected ${expected_exit}: ${error}")
|
||||
endif()
|
||||
if(mode STREQUAL "version")
|
||||
if(NOT output STREQUAL "frame-dictation ${EXPECTED_VERSION} (scaffold)\n")
|
||||
message(FATAL_ERROR "Unexpected version: ${output}")
|
||||
endif()
|
||||
elseif(expected_exit EQUAL 2)
|
||||
if(NOT error MATCHES "Unsupported arguments")
|
||||
message(FATAL_ERROR "Missing rejection diagnostic")
|
||||
endif()
|
||||
elseif(NOT output MATCHES "Scaffold only")
|
||||
message(FATAL_ERROR "Missing scaffold status")
|
||||
endif()
|
||||
endforeach()
|
||||
@@ -0,0 +1,40 @@
|
||||
#!/bin/sh
|
||||
# Offline, hardware-free contract test using a restricted PATH of fake host tools.
|
||||
set -eu
|
||||
script=$1
|
||||
tmp=$(/bin/mktemp -d)
|
||||
trap '/bin/rm -rf "$tmp"' EXIT HUP INT TERM
|
||||
/bin/mkdir "$tmp/bin"
|
||||
for dep in curl tar sha256sum mktemp mkdir mv; do
|
||||
printf '#!/bin/sh\nexit 0\n' > "$tmp/bin/$dep"
|
||||
/bin/chmod +x "$tmp/bin/$dep"
|
||||
done
|
||||
printf '#!/bin/sh\ncase "$1" in -s) printf "%%s\\n" "${MOCK_OS:-Linux}" ;; -m) printf "%%s\\n" "${MOCK_ARCH:-aarch64}" ;; esac\n' > "$tmp/bin/uname"
|
||||
printf '#!/bin/sh\nprintf "%%s\\n" "${MOCK_LIBC:-glibc 2.39}"\n' > "$tmp/bin/getconf"
|
||||
printf '#!/bin/sh\nprintf "%%s\\n" "${MOCK_PYTHON:-Python 3.12.3}"\n' > "$tmp/bin/python3"
|
||||
/bin/chmod +x "$tmp/bin/uname" "$tmp/bin/getconf" "$tmp/bin/python3"
|
||||
|
||||
check() {
|
||||
expected=$1
|
||||
needle=$2
|
||||
shift 2
|
||||
status=0
|
||||
output=$(PATH="$tmp/bin" "$@" /bin/sh "$script" 2>&1) || status=$?
|
||||
if [ "$status" -ne "$expected" ]; then
|
||||
printf 'Expected exit %s, got %s: %s\n' "$expected" "$status" "$output" >&2
|
||||
exit 1
|
||||
fi
|
||||
case "$output" in
|
||||
*"$needle"*) ;;
|
||||
*) printf 'Missing expected text %s: %s\n' "$needle" "$output" >&2; exit 1 ;;
|
||||
esac
|
||||
}
|
||||
|
||||
check 0 'Git: missing (not required' /usr/bin/env
|
||||
check 0 'uv: missing (not required' /usr/bin/env
|
||||
check 0 'Preflight passed' /usr/bin/env
|
||||
check 0 'older than 3.12' /usr/bin/env MOCK_PYTHON='Python 3.11.9'
|
||||
check 1 'Linux AArch64 only' /usr/bin/env MOCK_ARCH=x86_64
|
||||
check 1 'require glibc' /usr/bin/env MOCK_LIBC=musl
|
||||
/bin/rm "$tmp/bin/curl"
|
||||
check 1 'curl MISSING' /usr/bin/env
|
||||
Reference in new issue
Block a user