22 Commits
Author SHA1 Message Date
baketnk 1467d57f30 Remove the idle microphone footer hint 2026-10-04 22:11:09 -04:00
baketnk f43848731b Use controller-tagged compositor scroll for grab depth 2026-10-04 22:03:33 -04:00
baketnk fe8726864e Keep installed model-status checks from modifying release inventories 2026-10-04 21:53:48 -04:00
baketnk 0681d4941a Explain missing X display when the focus guard refuses Type 2026-10-04 21:48:06 -04:00
baketnk b3a9d39482 Pin release installer and accept clean tagged version output 2026-10-04 21:44:18 -04:00
baketnk 2c0e4ca4dc Update packaging documentation for published native releases 2026-10-04 21:43:55 -04:00
baketnk bcd330d42c Finish companion controls and retain panel textures during manipulation 2026-10-04 21:43:55 -04:00
baketnk b0767d1b36 Merge remote-tracking branch 'origin/main' 2026-10-04 21:39:15 -04:00
baketnk fb79dc18c6 Merge pull request #2 from bod09/fix/redux-model-revision
Repin Parakeet Redux to the current upstream revision (fixes install 404)
2026-10-04 21:17:22 -04:00
baketnk c0176b8595 Preserve verified Redux model cards across revision upgrades 2026-10-04 21:16:22 -04:00
bod09andClaude Opus 5.5 bbbd31fa6d Repin Parakeet Redux to the current upstream revision
moondream/parakeet-redux was rewritten upstream on 2026-09-23 to a single
commit (2bf128600aac), so the pinned revision fad622f no longer resolves.
model.safetensors still downloads from the old revision, but config.json,
ternary.json, tokenizer.json and README.md return HTTP 404, which makes
`install.sh --install-model` fail.

The weights, config, ternary map and tokenizer at 2bf1286 have the same
size and SHA-256 as the old pins, so the model itself is unchanged. Only
the model card README.md differs (8533 -> 8832 bytes), so its pin is
updated along with the revision.

Fixes #1

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-02 19:05:06 +01:00
baketnk eb054199e1 Add opt-in Settings release check and attended update handoff 2026-09-28 18:50:38 -04:00
baketnk dd164aacee Scroll review text and launch installed companion overlays 2026-09-28 18:30:14 -04:00
baketnk 4b0b29700d Persist per-mount wrist and head placement after drag 2026-09-26 23:41:13 -04:00
baketnk 8f4c1877f7 Make wrist fallback optional and scroll Settings with laser 2026-09-26 23:17:13 -04:00
baketnkandClaude Opus 5.5 0f36a45358 Add Settings -> About screen with version, license and credits
Static page: build version (with git commit when untagged/dirty), source URL,
MIT license, selected model's manifest license, font license and a pointer to
docs/third-party.md. Recording controls stay live; Settings returns.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-09-26 23:02:22 -04:00
baketnkandClaude Opus 5.5 9ec5dd03ba Fix paced delivery stopping after the first 24-codepoint chunk
Gamescope rewrites GAMESCOPE_FOCUSED_WINDOW (same window) after an IME
commit. The focus guard treated any PropertyNotify on the root focus
properties as a focus change, so every paced delivery stopped after its
first chunk. Re-read the property and fail only when it names a
different window; real X focus moves still fail via FocusOut, and every
check still re-validates both properties and X input focus.

Diagnosed from delivery-debug.log on Frame:
  sent offset=0 bytes=24
  focus invalid: GAMESCOPE_FOCUSED_WINDOW changed

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-09-26 22:49:52 -04:00
baketnkandClaude Opus 5.5 045cd4f45f Add opt-in delivery-debug.log for paced text delivery
Advanced debugging now also records paced-delivery metadata (byte
offsets, lease/send errors, finish results and the exact focus-guard
check that failed) to an owner-private delivery-debug.log. No transcript
text is logged. Diagnoses later chunks being dropped after the first.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-09-26 22:41:44 -04:00
baketnkandClaude Opus 5.5 279455134b README: link the YouTube tutorial
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-09-26 11:15:59 -04:00
baketnk 448ed8c2cd Show the recorded FrameYap UI in the README 2026-09-25 12:21:42 -04:00
baketnkandClaude Opus 5.5 b8eeb50b39 docs: user-facing README; move technical detail to docs/development.md
README now covers install, launch, controls, troubleshooting, privacy and
update/uninstall for users. Validation status, version format, CLI and full
control semantics move to docs/development.md. TODO records the
v0.1.202609251524 release and the owner's fresh curl | sh install.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-09-25 11:41:01 -04:00
baketnkandClaude Opus 5.5 3f451087ca installer: clarify post-install runtime and model messages
The native-only notice read as an error in the attended flow, where the
runtime step follows; the model message still asked for manual paths
although the app finds the XDG model store itself.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-09-25 11:36:42 -04:00
66 changed files with 2723 additions and 303 deletions

No files matched your search

+34 -3
View File
@@ -58,7 +58,11 @@ target_compile_definitions(frameyap PRIVATE FRAMEYAP_VERSION="${FRAMEYAP_VERSION
FRAMEYAP_BUILD_ROOT="${CMAKE_CURRENT_BINARY_DIR}"
FRAMEYAP_SOURCE_ROOT="${CMAKE_CURRENT_SOURCE_DIR}")
if(CMAKE_SYSTEM_NAME STREQUAL "Linux")
add_library(frameyap_mount src/mount.cpp src/laser_setting.cpp)
add_library(frameyap_update src/update_check.cpp)
target_include_directories(frameyap_update PUBLIC src)
add_library(frameyap_companions src/companion_apps.cpp)
target_include_directories(frameyap_companions PUBLIC src)
add_library(frameyap_mount src/mount.cpp src/laser_setting.cpp src/placement.cpp)
target_include_directories(frameyap_mount PUBLIC src)
add_library(frameyap_worker src/worker.cpp)
target_include_directories(frameyap_worker PUBLIC src)
@@ -102,7 +106,7 @@ if(FRAMEYAP_NATIVE)
target_sources(frameyap PRIVATE src/runtime.cpp src/controller.cpp src/backend_manager.cpp src/overlay.cpp src/overlay_texture.cpp src/audio.cpp src/text_input.cpp src/focus_guard.cpp
"${CMAKE_CURRENT_BINARY_DIR}/gamescope-input-method.c" "${CMAKE_CURRENT_BINARY_DIR}/gamescope-input-method-client.h")
target_include_directories(frameyap PRIVATE "${OPENVR_INCLUDE_DIR}" "${CMAKE_CURRENT_BINARY_DIR}")
target_link_libraries(frameyap PRIVATE frameyap_paced frameyap_worker frameyap_mount frameyap_battery frameyap_panel PkgConfig::SDL3 PkgConfig::WAYLAND PkgConfig::XCB Vulkan::Vulkan "${OPENVR_LIBRARY}")
target_link_libraries(frameyap PRIVATE frameyap_paced frameyap_worker frameyap_mount frameyap_battery frameyap_panel frameyap_companions frameyap_update PkgConfig::SDL3 PkgConfig::WAYLAND PkgConfig::XCB Vulkan::Vulkan "${OPENVR_LIBRARY}")
target_compile_definitions(frameyap PRIVATE FRAMEYAP_NATIVE=1)
set_target_properties(frameyap PROPERTIES INSTALL_RPATH "$ORIGIN/../lib")
endif()
@@ -113,7 +117,7 @@ install(TARGETS frameyap RUNTIME DESTINATION bin)
install(PROGRAMS install.sh DESTINATION bin)
install(DIRECTORY assets/ DESTINATION assets)
install(DIRECTORY python/frameyap DESTINATION python FILES_MATCHING PATTERN "*.py" PATTERN "__pycache__" EXCLUDE)
install(FILES scripts/model-status.py scripts/fetch-model.py scripts/backend-service.py DESTINATION scripts)
install(FILES scripts/model-status.py scripts/fetch-model.py scripts/backend-service.py scripts/check-update.py scripts/install-update.py DESTINATION scripts)
include(CTest)
if(BUILD_TESTING AND NOT CMAKE_CROSSCOMPILING)
@@ -149,6 +153,16 @@ if(BUILD_TESTING AND NOT CMAKE_CROSSCOMPILING)
add_test(NAME frameyap.panel COMMAND frameyap_panel_test "${CMAKE_CURRENT_SOURCE_DIR}/assets/fonts/Inconsolata-Regular.ttf")
endif()
if(CMAKE_SYSTEM_NAME STREQUAL "Linux")
if(Python3_FOUND)
add_executable(frameyap_update_test tests/update_check_test.cpp)
target_link_libraries(frameyap_update_test PRIVATE frameyap_update)
target_compile_options(frameyap_update_test PRIVATE -UNDEBUG)
add_test(NAME frameyap.update_check COMMAND frameyap_update_test "${Python3_EXECUTABLE}")
endif()
add_executable(frameyap_companions_test tests/companion_apps_test.cpp)
target_link_libraries(frameyap_companions_test PRIVATE frameyap_companions)
target_compile_options(frameyap_companions_test PRIVATE -UNDEBUG)
add_test(NAME frameyap.companions COMMAND frameyap_companions_test)
add_executable(frameyap_config_test tests/config_test.cpp)
target_link_libraries(frameyap_config_test PRIVATE frameyap_config)
target_compile_options(frameyap_config_test PRIVATE -UNDEBUG)
@@ -167,6 +181,9 @@ if(BUILD_TESTING AND NOT CMAKE_CROSSCOMPILING)
add_executable(frameyap_mount_test tests/mount_test.cpp)
target_link_libraries(frameyap_mount_test PRIVATE frameyap_mount)
add_test(NAME frameyap.mount COMMAND frameyap_mount_test)
add_executable(frameyap_placement_test tests/placement_test.cpp)
target_link_libraries(frameyap_placement_test PRIVATE frameyap_mount)
add_test(NAME frameyap.placement COMMAND frameyap_placement_test)
add_executable(frameyap_angle_fade_test tests/angle_fade_test.cpp)
target_include_directories(frameyap_angle_fade_test PRIVATE src)
target_compile_options(frameyap_angle_fade_test PRIVATE -UNDEBUG)
@@ -183,6 +200,10 @@ if(BUILD_TESTING AND NOT CMAKE_CROSSCOMPILING)
target_compile_options(frameyap_battery_test PRIVATE -UNDEBUG)
add_test(NAME frameyap.battery COMMAND frameyap_battery_test)
endif()
add_executable(frameyap_texture_refresh_test tests/texture_refresh_test.cpp)
target_include_directories(frameyap_texture_refresh_test PRIVATE src)
target_compile_options(frameyap_texture_refresh_test PRIVATE -UNDEBUG)
add_test(NAME frameyap.texture_refresh COMMAND frameyap_texture_refresh_test)
add_executable(frameyap_panel_drag_test tests/panel_drag_test.cpp)
target_include_directories(frameyap_panel_drag_test PRIVATE src)
target_compile_options(frameyap_panel_drag_test PRIVATE -UNDEBUG)
@@ -230,6 +251,8 @@ if(BUILD_TESTING AND NOT CMAKE_CROSSCOMPILING)
endif()
endif()
if(Python3_FOUND)
add_test(NAME frameyap.update_helpers COMMAND "${Python3_EXECUTABLE}" -m unittest discover -s tests -p test_update.py)
set_tests_properties(frameyap.update_helpers PROPERTIES WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}")
foreach(test IN ITEMS test_backends.py test_model_files.py test_backend_dispatch.py cli_git_tag.py)
string(REPLACE ".py" "" test_name "${test}")
add_test(NAME "frameyap.${test_name}" COMMAND "${Python3_EXECUTABLE}" -m unittest discover -s tests -p "${test}")
@@ -249,3 +272,11 @@ if(BUILD_TESTING AND NOT CMAKE_CROSSCOMPILING)
endif()
endif()
endif()
# Binding isolation is checked without initializing OpenVR.
if(BUILD_TESTING AND NOT CMAKE_CROSSCOMPILING)
find_package(Python3 3.10 COMPONENTS Interpreter QUIET)
if(Python3_FOUND)
add_test(NAME grab_bindings COMMAND "${Python3_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/tests/test_grab_bindings.py")
endif()
endif()
+52 -29
View File
@@ -1,48 +1,71 @@
# FrameYap
Voice typing on Steam Frame, with recognition on the headset rather than a desktop or cloud server.
Hold a controller button to record, review the transcript, then deliberately type it into the focused app.
Standalone OpenVR overlay; no Steam store AppID or sudo. First release: v0.1.202609251524.
On-device voice typing for Steam Frame. Hold a button, speak, check the text, and type it into whatever app has focus. Speech recognition runs on the headset itself; nothing is sent to the cloud.
## Requirements
[![FrameYap’s review panel on Steam Frame, with recording controls, controller and headset battery levels, and Settings and Bindings tabs.](docs/images/frameyap-ui.png)](https://youtu.be/9dg2KfojYTE)
- Steam Frame with usable SteamVR/OpenVR and Gamescope for the **native** overlay and text delivery; Linux ARM64/glibc for the current installer payload format.
- For voice recognition, a **CPU Python runtime** (moondream 2.4.0 / Kestrel 0.8.0, CPU Torch 2.8.0) and the pinned local Parakeet Redux model. Neither is bundled; each has its own explicit install step: `sh install.sh --install-model --backend redux --yes` and `sh install.sh --install-runtime --yes` (needs Python 3.10–3.13 with venv; about 200 MB download). Use `--print-plan` first to see exactly what each fetches. There is no fallback ASR service.
- A local source build needs CMake 3.20+, C++20 and explicit native libraries/SDK; the default hardware-free build needs only CMake and C++20. See [build requirements](docs/build.md).
*In-headset UI, captured from the [video tutorial on YouTube](https://youtu.be/9dg2KfojYTE).*
## Install
On the Frame, open a terminal (desktop mode's Konsole, or SSH) and run:
1. On the Frame, switch to **desktop mode** and open **Konsole** (or connect over SSH).
2. Paste this and press Enter:
```sh
curl -fsSL https://github.com/baketnk/frame-yap/releases/latest/download/install.sh | sh
```
```sh
curl -fsSL https://github.com/baketnk/frame-yap/releases/latest/download/install.sh | sh
```
You can download and read `install.sh` first; it is one self-contained file. The installer pins its own release (currently `v0.1.202609251524`) and verifies the archive's SHA-256. It asks three y/n questions: install FrameYap, download the Parakeet Redux speech model (about 180 MB, CC-BY-4.0), and pip-install the CPU Python runtime (moondream/Kestrel with CPU Torch; about 200 MB download, roughly 1–1.5 GB on disk). Everything goes under your home directory: no sudo, compiler or Steam store AppID. Afterwards, launch **FrameYap** from the desktop application menu.
3. Answer **y** to the three questions:
- install FrameYap,
- download the speech model (about 180 MB),
- install the speech runtime (about 200 MB download, roughly 1–1.5 GB on disk).
The release archive contains FrameYap, SDL3 and the OpenVR client library only; Kestrel, Torch, moondream and the model weights are fetched on your machine from PyPI and Hugging Face. For automation, the same steps are `sh install.sh --yes`, `sh install.sh --install-model --backend redux --yes` and `sh install.sh --install-runtime --yes`; add `--print-plan --json` to preview any step. Maintainers build release archives on Linux ARM64 with `sh scripts/build-release.sh WORKDIR VERSION`. See [packaging](docs/packaging.md) and [install design](docs/install-design.md).
The downloads can take a few minutes with little output; that is normal.
## Controls (default Frame binding)
Everything installs into your home folder. No sudo, no compiler, no Steam store page. Prefer to read the script first? Download [`install.sh`](https://github.com/baketnk/frame-yap/releases/latest/download/install.sh) and open it; it is a single file.
| Button / control | Action |
## Launch
Open **FrameYap** from the desktop application menu, or in the headset from your Steam library's **Non-Steam** section. The panel appears in front of you; drag the bar under it to move it, or the corner handle to resize.
## Use it
| Button | What it does |
| --- | --- |
| Right X, hold / release | Record while held; release to transcribe. |
| Right B | Cancel/discard, or close the Quick phrases picker. |
| Right A | **Type:** queue reviewed text, normally with a trailing space. With nothing to review, a press queues Enter alone (so a quick double press types then submits). |
| Right Y | Open **Quick phrases**; press again to cycle the selection. |
| Left grip, double-tap | **Type + Enter:** queue the selected phrase verbatim + Enter, pending review (normally + space) then Enter, or Enter alone if neither exists. |
| Overlay Record / Stop | Click-to-start/stop alternative to the PTT binding. |
| Overlay Type / Type + Enter | The same deliberate text / Enter actions. |
| Overlay Hold Quit | Hold for 0.9 seconds, then release to quit (prevents accidental exit). |
| **X** (right), hold | Record while held; let go to transcribe. |
| **A** (right) | Type the text into the focused app. With nothing to type, A presses **Enter** (so A, A types and submits). |
| **B** (right) | Cancel / discard. |
| **Y** (right) | Quick phrases; press again to pick the next one. |
| Left grip, double-tap | Type + Enter. |
Controls are remappable in SteamVR. Grip gestures may be unavailable with the dashboard open; pointer controls are an alternative. At the 4096-byte transcript limit, Type preserves the full text without appending a space if none fits. No speech commands, automatic Enter or automatic submit. Review is the default; Settings → Auto insert is opt-in, normally text + space only under continuously observed Xwayland focus. Type and Type + Enter now require a verified Xwayland target and use paced direct typing; Cancel stops remaining batches, never undoing prior input. The clipboard stays untouched. Check the destination before typing. Edit literal Quick phrases (`quick_inputs`) in `$XDG_CONFIG_HOME/frameyap/config.json`, then restart. See [overlay and settings](docs/overlay.md).
You can also click the panel's buttons with the laser pointer. The Review tab shows **Open Plan**, **Open Keyboard**, and **Open Draw** for companion apps installed on the Frame. Tap **Open Keyboard** to show it; hold for 0.8 seconds to recenter it in front of you. Nothing is ever typed or submitted without you pressing a button. Buttons can be remapped with **Bindings** on the panel.
## Status / not yet validated
The header shows battery levels for your controllers and headset. The **Buttons paused** badge means the Steam menu is open: controller buttons go to Steam, so use the pointer instead (or close the menu).
Native ARM64 build, CPU inference, overlay visibility, Gamescope API discovery and native-only installation have been exercised on Frame. The owner uses live microphone → reviewed transcript → paced delivery into real apps day to day, and the earlier front-prefix loss on repeated submissions (P1) is no longer observed. General app compatibility (for example browser fields), Auto insert with speech, physical resize, battery/thermal cost and a clean-account install are not yet validated. Local code/build status does not mean the device was updated. A completed Gamescope IME call means *input queued*, not that an app consumed or submitted it.
## Troubleshooting
The native app normally keeps the mic device open while Ready and discards idle audio; it never mutes other applications' microphones. Settings → **Close mic when idle** (default OFF) closes it between clips, but reopening on PTT can cause an audio spike, delay or first-syllable clipping. **Lasers anytime** (default OFF) requests system-wide lasers while the panel is visible, potentially affecting games; it is not SteamVR's experimental input override. Review, settings, placement and diagnostic details: [overlay](docs/overlay.md), [worker](docs/worker.md), [design](docs/design.md).
- **Panel says Unavailable, or shows an error while loading:** the speech model or runtime is missing (for example, you answered **n** during install). Run these, then restart FrameYap:
Version output is numeric `frameyap MAJOR.MINOR.YYYYMMDDHHMM` (currently `0.1`); a development build may print `git HASH` and optionally `(uncommitted changes)` on a **separate** line. The UTC timestamp is set at configuration time (`SOURCE_DATE_EPOCH` can supply it); release archives must be built from a clean `v0.1.<timestamp>` tag. Offline model inventory and pinned SHA-256 checks in a source-tree build: `./build/frameyap --list-models`, `./build/frameyap --check-model redux --model-dir /absolute/model`, or `python3 scripts/model-status.py` with the same options. Packaging retains the verifier script for installed CLI use, which still needs a clean-account artifact check. Native `--run` accepts `--backend ID`, `--model-store /absolute/dir`, `--manifest-dir /absolute/dir` overrides; the installed launcher passes explicit flags through to the binary. These locally wired paths do not imply an installed release was tested or a model/runtime was supplied. No model or runtime is downloaded by status checks or on normal launch. [Dependency licenses and outstanding release audit](docs/third-party.md); [TODO](TODO.md).
```sh
sh ~/.local/share/frameyap/current/bin/install.sh --install-model --backend redux --yes
sh ~/.local/share/frameyap/current/bin/install.sh --install-runtime --yes
```
- **Text went to the wrong place:** FrameYap types into the currently focused window. Click the target text field first.
- **Controller buttons do nothing:** check for the **Buttons paused** badge (see above).
No recordings, transcripts, private logs, weights or CPU runtime binaries are committed. No default build/test downloads or initializes SteamVR, microphone or input injection.
## Privacy
Recognition runs locally with the Parakeet Redux model. The installer downloads FrameYap from GitHub, the model from Hugging Face and the Python packages (moondream, Kestrel, CPU Torch) from PyPI and PyTorch; after that, FrameYap works offline. By default the microphone stays open while FrameYap is running and idle audio is thrown away; turn on **Settings → Close mic when idle** to close it between recordings (it may clip the first syllable). Recordings and transcripts are not kept, unless you turn on **Advanced debug**, whose logs may contain speech.
## Update and uninstall
- **Update:** run the install command again and answer **y** to FrameYap. Answer **n** to the model and runtime questions unless the release notes say they changed.
- **Check for updates:** Settings → scroll to **Check for updates**. FrameYap contacts GitHub only when you press it. If a newer published release is available, **Install update...** opens a desktop terminal; quit FrameYap and press Enter there to install. Nothing is downloaded by the check itself.
- **Settings:** Animated background is off by default; Settings → **Animated background** turns it on and saves your choice. `~/.config/frameyap/config.json` holds Quick phrases, theme and placement; restart FrameYap after manual edits.
- **Uninstall:** `sh ~/.local/share/frameyap/current/bin/install.sh --uninstall --unregistered`. Your settings and downloaded model are kept. Delete `~/.local/share/frameyap` and `~/.config/frameyap` to remove everything.
## More
- [Development notes](docs/development.md): build from source, validation status, full control details.
- [Third-party components and licenses](docs/third-party.md). FrameYap itself is [MIT licensed](LICENSE).
- The release archive contains FrameYap, SDL3 and the OpenVR client library only; the speech runtime and model are downloaded on your machine, not redistributed.
+12 -8
View File
@@ -50,11 +50,14 @@ accepted; local code/tests cannot establish a fixed delivery regression.
## A. First release (v0.1) blockers
Remaining release gates (2026-09-25): D4 is implemented; next a
fresh clean install on Frame (install → `--install-model` → `--install-runtime`) that also checks the missing-runtime panel message
(H), then tag `v0.1.<timestamp>` and update the README install section.
**v0.1.202609251524 released (2026-09-25).** Built on the Frame with
`scripts/build-release.sh`, published on GitHub, and installed fresh on the
Frame by the owner via `curl | sh` (install → model → pip runtime), followed
by successful live voice typing. The README is now user-facing; technical
detail lives in `docs/development.md`. Still unchecked: the exact
missing-runtime panel message and the battery readouts on Frame.
- [x] **A8. UI polish batch (local; deploy pending).** Removed the fixed
- [x] **A8. UI polish batch (shipped in v0.1.202609251524).** Removed the fixed
"Type adds a space" hint, grew the review card to six lines to close the empty
band, added L/HMD/R battery readouts and a Buttons ready/paused dashboard chip,
and made Type with nothing to review send Enter. Which batteries Frame reports
@@ -70,7 +73,8 @@ fresh clean install on Frame (install → `--install-model` → `--install-runti
- [x] **A3. Commit the pending `AGENTS.md` rename** ("Frame Dictation" →
"FrameYap"). (S) Done in baseline checkpoint `07c03ea`.
- [~] **A4. Runtime dependency pointers.** Repos and licenses are listed in
`docs/third-party.md`. No release audit is needed while v0.1 is source-only.
`docs/third-party.md`. Audit the staged native libraries and notices for each published archive;
speech runtime packages and model files remain separate explicit downloads.
- [x] **A5. Drop "POC" from the shipped surface.** (S) Public help, README,
CMake and installer use the release name. `scripts/stage-native-poc.py` remains
a deprecated compatibility wrapper for `scripts/stage-native.py`, not the
@@ -213,6 +217,6 @@ this app is future design and out of scope for v0.1.
## Open questions
The clean-account Frame install (including D4's ARM64 pip run and the
missing-runtime message) is the remaining unresolved gate; checked source tasks
do not imply it.
The fresh Frame install passed (2026-09-25). Open follow-ups: the
missing-runtime panel message, which batteries Frame reports, browser text
fields (G1), and whether the installer's y/n defaults suit first-time users.
+68 -21
View File
@@ -1,29 +1,76 @@
{
"default_bindings": [
{ "controller_type": "frame_controller", "binding_url": "bindings_frame_controller.json" },
{ "controller_type": "knuckles", "binding_url": "bindings_knuckles.json" }
{
"controller_type": "frame_controller",
"binding_url": "bindings_frame_controller.json"
},
{
"controller_type": "knuckles",
"binding_url": "bindings_knuckles.json"
}
],
"action_sets": [
{ "name": "/actions/frameyap", "usage": "single" }
{
"name": "/actions/frameyap",
"usage": "single"
},
{
"name": "/actions/grab",
"usage": "single"
}
],
"actions": [
{ "name": "/actions/frameyap/in/left_grip", "type": "boolean" },
{ "name": "/actions/frameyap/in/right_grip", "type": "boolean" },
{ "name": "/actions/frameyap/in/ptt", "type": "boolean" },
{ "name": "/actions/frameyap/in/cancel", "type": "boolean" },
{ "name": "/actions/frameyap/in/insert", "type": "boolean" },
{ "name": "/actions/frameyap/in/enter", "type": "boolean" },
{ "name": "/actions/frameyap/in/quick_chat", "type": "boolean" }
{
"name": "/actions/frameyap/in/left_grip",
"type": "boolean"
},
{
"name": "/actions/frameyap/in/right_grip",
"type": "boolean"
},
{
"name": "/actions/frameyap/in/ptt",
"type": "boolean"
},
{
"name": "/actions/frameyap/in/cancel",
"type": "boolean"
},
{
"name": "/actions/frameyap/in/insert",
"type": "boolean"
},
{
"name": "/actions/frameyap/in/enter",
"type": "boolean"
},
{
"name": "/actions/frameyap/in/quick_chat",
"type": "boolean"
},
{
"name": "/actions/grab/in/left_depth",
"type": "vector2"
},
{
"name": "/actions/grab/in/right_depth",
"type": "vector2"
}
],
"localization": [{
"language_tag": "en_US",
"/actions/frameyap": "FrameYap controls",
"/actions/frameyap/in/left_grip": "Double tap left grip: Type + Enter (enabled only)",
"/actions/frameyap/in/right_grip": "Tap right grip briefly, then squeeze and hold to record; release to finish",
"/actions/frameyap/in/ptt": "Hold right X to record (default; remappable)",
"/actions/frameyap/in/cancel": "Cancel recording or discard",
"/actions/frameyap/in/insert": "Type reviewed text + trailing space",
"/actions/frameyap/in/enter": "Type + Enter: pending text + Enter, or Enter alone",
"/actions/frameyap/in/quick_chat": "Open quick phrases / cycle selection"
}]
"localization": [
{
"language_tag": "en_US",
"/actions/frameyap": "FrameYap controls",
"/actions/frameyap/in/left_grip": "Double tap left grip: Type + Enter (enabled only)",
"/actions/frameyap/in/right_grip": "Tap right grip briefly, then squeeze and hold to record; release to finish",
"/actions/frameyap/in/ptt": "Hold right X to record (default; remappable)",
"/actions/frameyap/in/cancel": "Cancel recording or discard",
"/actions/frameyap/in/insert": "Type reviewed text + trailing space",
"/actions/frameyap/in/enter": "Type + Enter: pending text + Enter, or Enter alone",
"/actions/frameyap/in/quick_chat": "Open quick phrases / cycle selection",
"/actions/grab": "Held panel depth",
"/actions/grab/in/left_depth": "Left stick: move held panel forward/back",
"/actions/grab/in/right_depth": "Right stick: move held panel forward/back"
}
]
}
+2 -2
View File
@@ -10,13 +10,13 @@
},
"model": {
"source": "https://huggingface.co/moondream/parakeet-redux",
"revision": "fad622f25f303105c20d70e201bcc477c88b620c",
"revision": "2bf128600aac4b16946f7ed8372e56117fe5e23b",
"files": [
{"path": "model.safetensors", "size": 177774490, "sha256": "78ec25733ee0d0c1586d1346fc86db9d0c2e436e3a8ab1d32a82d1bb8f848d21"},
{"path": "config.json", "size": 12988, "sha256": "503c653b2e3bb788adbcb04f5abdee532d958686564081baeed133ff10143f6e"},
{"path": "ternary.json", "size": 57970, "sha256": "1221c6d3ce901ffe09c089da758a8db8b76189f80cff41c5afc244fc61e2051d"},
{"path": "tokenizer.json", "size": 1159960, "sha256": "bd321b096832a3f270bd3b2a88823957920f1a5c5ada71114a26ea729d0cbe91"},
{"path": "README.md", "size": 8533, "sha256": "a8b327f983a8b8ff262ff7bead3a791fbed9350632002af8db85ab5cd84cdaa5"}
{"path": "README.md", "size": 8832, "sha256": "930055e6bac7d75114e719850ffaffc8d327e73b4295518c38c81399acd63e46", "compatible": [{"size": 8533, "sha256": "a8b327f983a8b8ff262ff7bead3a791fbed9350632002af8db85ab5cd84cdaa5"}]}
]
},
"attribution": "Moondream/M87 Labs, Parakeet Redux, derived from NVIDIA Parakeet TDT 0.6B v3; original model card README.md retained with the files.",
+52 -6
View File
@@ -8,32 +8,78 @@
{
"path": "/user/hand/left/input/grip",
"mode": "button",
"inputs": { "click": { "output": "/actions/frameyap/in/left_grip" } }
"inputs": {
"click": {
"output": "/actions/frameyap/in/left_grip"
}
}
},
{
"path": "/user/hand/right/input/grip",
"mode": "button",
"inputs": { "click": { "output": "/actions/frameyap/in/right_grip" } }
"inputs": {
"click": {
"output": "/actions/frameyap/in/right_grip"
}
}
},
{
"path": "/user/hand/right/input/x",
"mode": "button",
"inputs": { "click": { "output": "/actions/frameyap/in/ptt" } }
"inputs": {
"click": {
"output": "/actions/frameyap/in/ptt"
}
}
},
{
"path": "/user/hand/right/input/b",
"mode": "button",
"inputs": { "click": { "output": "/actions/frameyap/in/cancel" } }
"inputs": {
"click": {
"output": "/actions/frameyap/in/cancel"
}
}
},
{
"path": "/user/hand/right/input/a",
"mode": "button",
"inputs": { "click": { "output": "/actions/frameyap/in/insert" } }
"inputs": {
"click": {
"output": "/actions/frameyap/in/insert"
}
}
},
{
"path": "/user/hand/right/input/y",
"mode": "button",
"inputs": { "click": { "output": "/actions/frameyap/in/quick_chat" } }
"inputs": {
"click": {
"output": "/actions/frameyap/in/quick_chat"
}
}
}
]
},
"/actions/grab": {
"sources": [
{
"path": "/user/hand/left/input/thumbstick",
"mode": "joystick",
"inputs": {
"position": {
"output": "/actions/grab/in/left_depth"
}
}
},
{
"path": "/user/hand/right/input/thumbstick",
"mode": "joystick",
"inputs": {
"position": {
"output": "/actions/grab/in/right_depth"
}
}
}
]
}
+32 -2
View File
@@ -8,12 +8,42 @@
{
"path": "/user/hand/left/input/grip",
"mode": "button",
"inputs": { "click": { "output": "/actions/frameyap/in/left_grip" } }
"inputs": {
"click": {
"output": "/actions/frameyap/in/left_grip"
}
}
},
{
"path": "/user/hand/right/input/grip",
"mode": "button",
"inputs": { "click": { "output": "/actions/frameyap/in/right_grip" } }
"inputs": {
"click": {
"output": "/actions/frameyap/in/right_grip"
}
}
}
]
},
"/actions/grab": {
"sources": [
{
"path": "/user/hand/left/input/thumbstick",
"mode": "joystick",
"inputs": {
"position": {
"output": "/actions/grab/in/left_depth"
}
}
},
{
"path": "/user/hand/right/input/thumbstick",
"mode": "joystick",
"inputs": {
"position": {
"output": "/actions/grab/in/right_depth"
}
}
}
]
}
+2 -1
View File
@@ -6,6 +6,7 @@
"close_mic_when_idle": false,
"backend": "redux",
"lock_layout": false,
"wrist_world_fallback": true,
"clock_24h": false,
"date_format": "mdy",
"quick_inputs": ["/new", "/questions", "/help"],
@@ -20,7 +21,7 @@
"frame_start": "#1fff91",
"frame_end": "#1f70ff"
},
"gradient": {"enabled": true, "period_seconds": 30, "strength": 0.12},
"gradient": {"enabled": false, "period_seconds": 30, "strength": 0.12},
"buttons": {
"left_grip": "/user/hand/left/input/grip",
"right_grip": "/user/hand/right/input/grip",
+31
View File
@@ -0,0 +1,31 @@
# Animation opt-in and keyboard hold — 2026-09-29
Animated background now defaults off in native config, the example config and
installer defaults. Settings → Animated background writes the app's nested
`gradient.enabled` field without replacing other settings. Static theme colors
and the frame remain; existing explicit opt-ins are retained by ordinary upgrades.
For this user's requested rollout, the existing Frame config was backed up and
explicitly set off before restart.
Review → Open Keyboard sends `tnkboard --show` on short press. An 800 ms hold
sends `--recenter` once, suppressing the short action on release. Leaving the
button, resetting pointers, hiding, focus/tracking loss or entering panel drag
cancels the hold. The launcher uses fixed exec arguments, not a shell command.
The keyboard's matching command cancels placement/releases active input, shows
it and uses the current headset pose; saved keyboard size is retained.
Verification:
- Full native x86-64 suite: 34/34 passed.
- Frame ARM64 suite: 34/34 passed.
- Inspected synthetic Settings and keyboard-hold previews.
- Managed local artifact `0.1.202609290204` installed and launched on the Frame;
prior version retained for rollback. Existing authorized speech runtime/model
retained; the restarted process spawned its Redux backend worker using
`~/.local/share/frameyap/models/redux`.
- Installed native executable SHA-256:
`62587bed7f7edfc0c4f2d0ef7f2624eac915f2e9a9839b3de058200e6148ce65`.
- Final config check: FrameYap, tnkboard and tnkdraw animation disabled.
- No audio recording, key injection or human acceptance result was manufactured.
Long-press compositor delivery and subjective headset behavior still need the
wearer's check. This is a local development rollout, not a published release.
+1 -1
View File
@@ -93,7 +93,7 @@ and target application compatibility require further headset work.
## Inference runtime
Redux weights at `fad622f25f303105c20d70e201bcc477c88b620c` are CC-BY-4.0. Pinned
Redux weights at `2bf128600aac4b16946f7ed8372e56117fe5e23b` are CC-BY-4.0. Pinned
file sizes/hashes and attribution live in `assets/backends/redux.json`; offline
`--list-models`/`--check-model redux --model-dir /absolute/model` (or
`scripts/model-status.py`) verify without inference or downloads. Manifest
+17 -7
View File
@@ -22,7 +22,8 @@ as explicit fallbacks. No desktop ASR server, network hop, LLM cleanup, scene
renderer, avatar, desktop capture or root service is needed in the primary path.
A first-class product goal is a **one-command GitHub install without a Steam store
AppID**. v0.1 is source-only; a prebuilt archive is deferred. Use a normal
AppID**. Published v0.1 archives supply the native app, while the model and
CPU runtime have separate explicit download steps. Use a normal
OpenVR application key for registration, not Steamworks. Installation must remain
user-local with opt-in autolaunch. See [installation design](install-design.md).
@@ -109,12 +110,13 @@ one animation redraw per 100 ms on a monotonic clock. Hidden panels do not
repaint for animation. The locally implemented gradient uses a smooth periodic
cosine field: one start/end/start cycle across the canvas width, with a shared
time phase and global canvas coordinates on the perimeter and grab/scale handles.
`gradient.enabled` defaults to true; `period_seconds` defaults to 30 (finite
number 5–300), and `strength` to 0.12 (finite number 0–0.3). Its colors derive
`gradient.enabled` defaults to false; Settings → Animated background can opt in
and persist the choice. `period_seconds` defaults to 30 (finite number 5–300),
and `strength` to 0.12 (finite number 0–0.3). Its colors derive
only from `theme.frame_start` and `theme.frame_end`; strength blends those colors
into `theme.background`. Disabled mode restores the static solid background
and existing linear edge/handle gradients. Config is read on restart, with no
in-panel gradient switch. This is local implementation, **not** device-validated
and existing linear edge/handle gradients. Manual config changes are read on restart; the in-panel switch takes effect
immediately. This is local implementation, **not** device-validated
appearance, frame pacing, performance or headset acceptance. See
[config and rendering](overlay.md).
The native implementation uploads the CPU-rasterized panel to
@@ -226,9 +228,17 @@ may deliver at most once; cancellation invalidates it before any later reply.
## On-device Redux
Use the **same** `moondream/parakeet-redux` revision from the benchmark:
`fad622f25f303105c20d70e201bcc477c88b620c` (177,774,490-byte weight file), initially
`2bf128600aac4b16946f7ed8372e56117fe5e23b` (177,774,490-byte weight file), initially
moondream 2.4.0 / kestrel 0.8.0. Do not substitute dense Ultra or silently fall
back to desktop/cloud inference.
back to desktop/cloud inference. The benchmark ran on revision
`fad622f25f303105c20d70e201bcc477c88b620c`, which no longer resolves after the
upstream history was rewritten; the weights, config, ternary map and tokenizer at
`2bf1286` are byte-identical to it (same SHA-256), only the model card README differs.
The manifest explicitly accepts the previous card's pinned size and SHA-256 for
existing installations, so updating FrameYap requires no model download. New
downloads must verify against the current revision's card; unknown or modified
cards are still rejected. The optional per-file `compatible` list contains exact
`size`/`sha256` pairs for previously installed artifacts, never download fallbacks.
The earlier benchmark used this local API (its source belongs to the originating
repository, not this project):
+52
View File
@@ -0,0 +1,52 @@
# Development notes
Technical details moved out of the user-facing README. See also
[build](build.md), [overlay](overlay.md), [worker](worker.md),
[packaging](packaging.md), [install design](install-design.md),
[design](design.md) and [third-party notes](third-party.md).
## Requirements (detail)
- Steam Frame with usable SteamVR/OpenVR and Gamescope for the **native** overlay and text delivery; Linux ARM64/glibc for the current installer payload format.
- For voice recognition, a **CPU Python runtime** (moondream 2.4.0 / Kestrel 0.8.0, CPU Torch 2.8.0) and the pinned local Parakeet Redux model. Neither is bundled; each has its own explicit install step: `sh install.sh --install-model --backend redux --yes` and `sh install.sh --install-runtime --yes` (needs Python 3.10–3.13 with venv; about 200 MB download). Use `--print-plan` first to see exactly what each fetches. There is no fallback ASR service.
- A local source build needs CMake 3.20+, C++20 and explicit native libraries/SDK; the default hardware-free build needs only CMake and C++20. See [build requirements](build.md).
## Controls (full semantics)
| Button / control | Action |
| --- | --- |
| Right X, hold / release | Record while held; release to transcribe. |
| Right B | Cancel/discard, or close the Quick phrases picker. |
| Right A | **Type:** queue reviewed text, normally with a trailing space. With nothing to review, a press queues Enter alone (so a quick double press types then submits). |
| Right Y | Open **Quick phrases**; press again to cycle the selection. |
| Left grip, double-tap | **Type + Enter:** queue the selected phrase verbatim + Enter, pending review (normally + space) then Enter, or Enter alone if neither exists. |
| Overlay Record / Stop | Click-to-start/stop alternative to the PTT binding. |
| Overlay Type / Type + Enter | The same deliberate text / Enter actions. |
| Overlay Hold Quit | Hold for 0.9 seconds, then release to quit (prevents accidental exit). |
Controls are remappable in SteamVR. Grip gestures may be unavailable with the dashboard open; pointer controls are an alternative. At the 4096-byte transcript limit, Type preserves the full text without appending a space if none fits. No speech commands, automatic Enter or automatic submit. Review is the default; Settings → Auto insert is opt-in, normally text + space only under continuously observed Xwayland focus. Type and Type + Enter now require a verified Xwayland target and use paced direct typing; Cancel stops remaining batches, never undoing prior input. The clipboard stays untouched. Check the destination before typing. Edit literal Quick phrases (`quick_inputs`) in `$XDG_CONFIG_HOME/frameyap/config.json`, then restart. See [overlay and settings](overlay.md).
## Validation status
Native ARM64 build, CPU inference, overlay visibility, Gamescope API discovery and native-only installation have been exercised on Frame. The owner uses live microphone → reviewed transcript → paced delivery into real apps day to day, and the earlier front-prefix loss on repeated submissions (P1) is no longer observed. General app compatibility (for example browser fields), Auto insert with speech, physical resize, and battery/thermal cost are not yet validated. On 2026-09-25 the owner removed the previous install, ran the published v0.1.202609251524 `curl | sh` installer on the Frame (FrameYap, model and pip runtime) and voice-typed successfully; the ARM64 runtime install is therefore exercised, but the missing-runtime panel message and battery readouts on Frame were not separately checked. Local code/build status does not mean the device was updated. A completed Gamescope IME call means *input queued*, not that an app consumed or submitted it.
The native app normally keeps the mic device open while Ready and discards idle audio; it never mutes other applications' microphones. Settings → **Close mic when idle** (default OFF) closes it between clips, but reopening on PTT can cause an audio spike, delay or first-syllable clipping. **Lasers anytime** (default OFF) requests system-wide lasers while the panel is visible, potentially affecting games; it is not SteamVR's experimental input override. Review, settings, placement and diagnostic details: [overlay](overlay.md), [worker](worker.md), [design](design.md).
Version output is numeric `frameyap MAJOR.MINOR.YYYYMMDDHHMM` (currently `0.1`); a development build may print `git HASH` and optionally `(uncommitted changes)` on a **separate** line. The UTC timestamp is set at configuration time (`SOURCE_DATE_EPOCH` can supply it); release archives must be built from a clean `v0.1.<timestamp>` tag. Offline model inventory and pinned SHA-256 checks in a source-tree build: `./build/frameyap --list-models`, `./build/frameyap --check-model redux --model-dir /absolute/model`, or `python3 scripts/model-status.py` with the same options. Packaging retains the verifier script for installed CLI use, which still needs a clean-account artifact check. Native `--run` accepts `--backend ID`, `--model-store /absolute/dir`, `--manifest-dir /absolute/dir` overrides; the installed launcher passes explicit flags through to the binary. These locally wired paths do not imply an installed release was tested or a model/runtime was supplied. No model or runtime is downloaded by status checks or on normal launch. [Dependency licenses and outstanding release audit](third-party.md); [TODO](../TODO.md).
No recordings, transcripts, private logs, weights or CPU runtime binaries are committed. No default build/test downloads or initializes SteamVR, microphone or input injection.
## Release builds
Release archives are built on Linux ARM64 from a clean `v0.1.<timestamp>` tag
with `sh scripts/build-release.sh WORKDIR VERSION`, which fetches SHA-256-pinned
SDL3 and OpenVR SDK sources, builds, tests, stages and packages. The release
commit sets `RELEASE_VERSION` in `scripts/install_payload.py` (then
`python3 scripts/sync-installer.py`); upload the archive, its `.sha256` and
`install.sh` to the GitHub release.
When starting over SSH, supply the verified Gamescope Xwayland `DISPLAY` as well
as `XDG_RUNTIME_DIR` from the user's session. Do not assume `:0` is always the
text destination. Without `DISPLAY`, the focus guard refuses Type and preserves
the review; the status line reports `Input unavailable: DISPLAY missing`.
Launch from Steam/the desktop menu to inherit the normal session environment.
Binary file not shown.

After

Width:  |  Height:  |  Size: 220 KiB

+106 -27
View File
@@ -26,14 +26,14 @@ persistent Vulkan RGBA8 image and submits it with `SetOverlayTexture`. The image
staging allocation and command buffer are reused; tabs do not create extra
overlays or render targets. The rounded mint-to-blue perimeter,
shallow curved accent, and dark cards form the panel's visual language.
With `gradient.enabled` (the default), a low-strength animated field blends
With `gradient.enabled` (opt-in), a low-strength animated field blends
`theme.frame_start` and `theme.frame_end` into `theme.background`; the perimeter
and grab/scale handles share its phase and canvas coordinates. Rounded
preview, status and control surfaces use independently rasterized antialiased edges
and restrained baked neon halos rather than GPU bloom. The recording indicator and
selected controls remain distinguishable by their labels, not color alone. Rounded
control hit areas exclude their clipped corners.
Rendering/uploads occur for changed content, page or settings, and, when the
Rendering/uploads occur for changed content, scroll position or settings, and, when the
animated gradient is enabled and visible, at most every 100 ms (10 fps) on a
monotonic clock. There is no animation redraw while hidden. Laser hover and
button down/up are hit-tested without an upload; static frames are reused when
@@ -74,10 +74,43 @@ release) and Lasers anytime (system-wide lasers may affect games). The header
updates when the displayed minute, date, battery or dashboard state changes,
not every frame. Settings toggles 12/24-hour time and cycles date Off →
MM/DD/YYYY → DD/MM/YYYY → YYYY-MM-DD → Off. These only affect display;
mount choices remain in Settings.
mount choices remain in Settings. The Settings controls form one vertically
scrollable list: hover the laser over the list and use the right stick to scroll.
The tabs, status and action footer remain fixed. Scrolling cancels a pending
pointer press rather than activating a different control. There are no Settings
pages. OpenVR's discrete/smooth laser scroll events are requested; right-stick
behavior with SteamVR's keyboard open still needs a live Frame check.
The complete transcript preview is paginated by glyph width and six-line
height; Previous and Next navigate it without changing the source transcript.
Settings → **About** replaces the settings area with static information: the
build version (plus git commit when built from an untagged or modified tree),
source URL, MIT license, the selected model's name and license from its manifest,
the bundled font's license, and a pointer to `docs/third-party.md`. It performs
no checks and no network access; Settings returns. Recording controls stay live.
At the bottom of Settings, **Check for updates** performs an explicit, asynchronous
request to GitHub's latest published release metadata. It never runs at launch,
while idle or as part of offline tests. The bounded, timeout-limited helper
compares the release's numeric tag with this build's version; repository commits
and prereleases are not update candidates. No archive or installer is downloaded
by a check. If a newer release is found, **Install update...** opens Konsole
with the installed handoff script. The terminal waits for the wearer to close
FrameYap and press Enter before the existing installer downloads a pinned
version and checksum and installs it. Until that confirmation, the terminal
does not fetch anything. An unavailable check or terminal reports an error in
Settings; no fallback auto-install is attempted.
The transcript wraps by glyph width and scrolls in its review viewport with
the right-stick laser wheel; a new transcript resets the scroll position.
The former paging row holds **Open Plan**, **Open Keyboard** and **Open Draw**
when the installed `tnkplan`, `tnkboard` or `tnkdraw` wrapper is found at startup
in `~/.local/bin` or an absolute PATH directory. Missing apps have no button;
available buttons share the row. Open Keyboard short press runs the installed
wrapper with fixed `--show`; holding it for 800 ms runs fixed `--recenter`
once and suppresses the short action, placing the keyboard in front of the
wearer. Leaving the button, losing focus/tracking or hiding the overlay cancels
a pending hold. Other companion buttons launch their installed wrappers without
a shell (their normal second-launch behavior may toggle the existing panel).
Restart FrameYap after installing a companion so its button is discovered.
Status fits on the single status line; the old bottom detail label is gone.
The footer remains available on all tabs: Record (labelled Stop while recording),
Cancel, Type (labelled Enter when nothing is pending review), Type + Enter, Hold Quit. Hold Quit needs a 900 ms press and release on
@@ -136,6 +169,7 @@ installer creates one with defaults on first install. Copy the shipped
"close_mic_when_idle": false,
"backend": "redux",
"lock_layout": false,
"wrist_world_fallback": true,
"clock_24h": false,
"date_format": "mdy",
"quick_inputs": ["/new", "/questions", "/help"],
@@ -145,7 +179,7 @@ installer creates one with defaults on first install. Copy the shipped
"muted": "#97adc1", "accent": "#1ff0a4", "warning": "#ff6e87",
"frame_start": "#1fff91", "frame_end": "#1f70ff"
},
"gradient": {"enabled": true, "period_seconds": 30, "strength": 0.12},
"gradient": {"enabled": false, "period_seconds": 30, "strength": 0.12},
"buttons": {
"ptt": "/user/hand/right/input/x",
"cancel": "/user/hand/right/input/b",
@@ -165,7 +199,7 @@ A failed save warns and leaves the selection active for this run. Time uses
the device's local timezone; these controls do not change system time.
Each theme color is `#RRGGBB`; omitted colors keep the default. The optional
`gradient` object defaults to `{"enabled": true, "period_seconds": 30, "strength": 0.12}`.
`gradient` object defaults to `{"enabled": false, "period_seconds": 30, "strength": 0.12}`.
`enabled` must be a boolean; `period_seconds` must be a
finite number from 5 to 300 (seconds per full cycle), and `strength` a finite
number from 0 to 0.3. The animation uses a smooth periodic cosine field with
@@ -173,9 +207,11 @@ one start/end/start cycle across the canvas width. Its edge and handle colors
use the same global canvas coordinates and time phase; it does not add a
separate handle animation. Colors always come from `theme.frame_start` and
`theme.frame_end`, while `strength` controls how much of those colors blends
into `theme.background`. Set `enabled` to `false` for a static solid background
and the existing linear perimeter/handle gradients. Edit the JSON and restart;
there is no in-panel gradient switch or hot reload. This is locally implemented,
into `theme.background`. The default static mode keeps the solid background
and existing linear perimeter/handle gradients. Settings → **Animated background**
turns animation on/off immediately and saves only `gradient.enabled` in the JSON
config (failed saves leave a session-only choice and a warning). Manual JSON edits
take effect on restart; existing enabled configs stay enabled on upgrade. This is locally implemented,
not device validated; headset appearance and rendering cost remain unverified.
`font` is a TTF/OTF file path (not a family name); a missing file uses the
bundled font.
@@ -346,9 +382,31 @@ Hold the laser's primary click on the bar to **freely position and rotate** the
panel with the controller, including depth, pitch, yaw and roll. Grab captures
`inverse(controller_down) * panel_down` and applies that unchanged relative pose
to each controller pose, so grabbing does not snap or reset the panel orientation.
During a grab, the grabbing hand's thumbstick Y axis also moves the panel along
the captured panel's local -Z normal: forward pushes away, backward pulls closer.
The axis is velocity with a 0.2 dead zone, 0.6 m/s at full deflection and a
±1.5 m offset limit from the pose-only grab. Returning to center holds the
current depth; the other hand's stick and scale drags do not change it. A lost
axis stops adding depth without resetting placement. Frame controller bindings
provide left/right vector2 actions in a separate `/actions/grab` set, including
in generated custom button manifests. Only a grab activates this set, restricted
to the grabbing hand at overlay priority; that app-action route requires SteamVR Experimental
overlay input overrides. If the app axis is unavailable, controller-tagged
compositor smooth-scroll events provide depth at 0.04 m per scroll unit, bounded
to 3 cm per event and the same total travel limit. Only the grabbing controller
is accepted. Smooth events are requested only during grab, integrated once
without multiplying by elapsed time, and discarded whenever the app axis is
available to avoid applying both streams. It does not promote recording or text actions, and does
not depend on FrameYap's ordinary `input_priority` preference. Axis delivery during laser drag and dashboard/game focus needs a
separate headset check. Grab scroll events affect only depth; scale scroll
events are ignored, and ordinary UI scrolling resumes after release.
Release leaves the last pose in the chosen mount frame; head/wrist mounts continue
following that anchor afterward. Mount changes/recenter reset the pose, while
retaining the size factor. Restart restores configured position/size defaults.
following that anchor afterward. A completed grab or scale on Head, Left wrist or
Right wrist saves the full device-relative canvas position, rotation and scale
separately for that mount. Mount changes restore each saved relative placement;
restarting does too. A cancelled drag does not update the saved placement.
World-space placement is resampled on startup/recenter rather than restored
against a possibly different tracking origin.
Drag the corner bracket to scale between half and twice the configured width.
Scaling freezes its initial plane and calibrates a ray from the source controller
@@ -384,24 +442,35 @@ fade from 60° to 75°, then hidden (including laser interaction). Pitch, yaw an
roll contribute together; turning the wrist away or moving the head around it
changes the angle. OpenVR's overlay alpha changes without rerendering the panel.
World and head mounts do not fade. Missing headset tracking hides a wrist panel;
a lost wrist still uses the existing world-space fallback. Headset readability,
by default a lost wrist uses a world-space fallback. Headset readability,
fade feel and interaction at the threshold still need live acceptance.
To tune the selected wrist, set `wrist` in `config.json` as in the example above:
`x`, `y`, `z` are controller-local meters (each -0.3 to 0.3), `width` is panel
width in meters (0.15 to 0.6), and `roll_degrees` rotates about controller -Z
(-180 to 180) before applying the offset. These settings apply to both wrists,
are read at startup, and do not alter world/head placement. Invalid values
fail direct native startup; the installer backs up and repairs invalid entries.
(-180 to 180) before applying the offset. These settings are initial defaults
for both wrists; a saved per-wrist pose takes precedence. They are read at
startup and do not alter world/head placement. Invalid values fail direct native
startup; the installer backs up and repairs invalid entries.
A missing/untracked selected wrist temporarily falls back to world space,
with a visible explanation in Settings, then reattaches when tracking returns.
The saved preference is not replaced by the fallback. These offsets and sizes are
Use Settings → **Wrist world fallback** to switch between the default
front-of-you fallback (ON) and hiding the panel when wrist tracking is lost
(OFF). It reappears on the selected wrist when tracking returns. The toggle is
saved to `"wrist_world_fallback"` in `config.json`; manual edits take effect on
restart. This option affects wrist mounts only, not World or Head. While hidden,
pointer controls cannot be used.
The saved mount preference is not replaced by the fallback. These offsets and sizes are
initial choices, **not headset-comfort acceptance**.
A selection saves only the mount token to `$XDG_CONFIG_HOME/frameyap/mount`
(or `$HOME/.config/frameyap/mount`), using an atomic replacement. No pose, audio
or transcript is saved. Missing/invalid settings default to world; write failure
keeps the selection for the session and displays a warning. `--mount
A selection saves the mount token to `$XDG_CONFIG_HOME/frameyap/mount`
(or `$HOME/.config/frameyap/mount`). Completed relative adjustments save to
`placement-left-wrist`, `placement-right-wrist`, or `placement-head` beside that
file. Remove one placement file while FrameYap is closed to restore that mount's
default. These are bounded, versioned device-relative transforms with owner-only
atomic writes; invalid files are ignored. World-space poses, audio and transcripts
are not saved. Missing/invalid mount settings default to World; write failure
keeps the placement for the session and displays a warning. `--mount
world|left-wrist|right-wrist|head` overrides the saved choice for one launch without
writing it; `--head` remains an alias for `--mount head`.
@@ -426,8 +495,9 @@ coexistence remains unverified.
### Hardware-free UI checks
The default build tests mount parsing, laser preference persistence, pose geometry
and controller-relative grab/captured-ray scale math without any native dependencies.
The default build tests mount parsing, laser preference persistence, pose geometry,
controller-relative grab/captured-ray scale math and stick-depth integration without
any native dependencies.
Drag tests cover XYZ translation, pitch/yaw/roll and lever-arm rotation, re-grabbing
a moved panel, stationary stability, relative mounts, out-of-bounds scale hits,
invalid poses/rays and no feedback from prior updates. UI/config/installer tests
@@ -468,8 +538,8 @@ cmake --build build-native --target frameyap_texture_check
It does not initialize OpenVR or establish compositor/headset acceptance.
`assets/actions.json` names seven actions: left/right grip, PTT, cancel,
Type, Type + Enter and Quick phrases. `insert`, `enter`, and `quick_chat` remain
`assets/actions.json` names nine actions: left/right grip, PTT, cancel,
Type, Type + Enter, Quick phrases and the two hand-specific depth axes. `insert`, `enter`, and `quick_chat` remain
internal binding keys; visible controls read Type, Type + Enter, Quick phrases.
`bindings_frame_controller.json` maps right X click to hold-to-talk PTT;
the grip bindings remain for optional remapping/diagnosis. In one dashboard
@@ -477,7 +547,8 @@ probe grips were inactive; a later controls-only probe delivered repeated right
X PTT BeginRecord/EndRecord callbacks. The wearer reports controller actions
are usable with Steam's dashboard closed, not with the dashboard itself open.
Neither probe used a microphone or established game-scene pass-through.
`bindings_knuckles.json` is an additional **Index/knuckles example only**.
`bindings_knuckles.json` is an additional **Index/knuckles example only**;
it includes the two thumbstick depth axes alongside grip taps.
Collisions with scene actions require
separate on-device validation. Left grip double tap
(releases <=250 ms, second press within 350 ms) requests explicit Enter only
@@ -515,8 +586,16 @@ Registration, repeated registration and removal were tested against the running
Frame runtime, with autolaunch verified off. The overlay explicitly identifies
its process with the registered app key before setting its action manifest.
Cold-runtime behavior and actual SteamVR-menu launch still need validation.
See [packaging](packaging.md); no published release is claimed here.
See [packaging](packaging.md) for the published native archive and release checks.
A live headset check must be opt-in and distinguish overlay API discovery from
controller delivery, actual transcription, insertion into a disposable target,
and human comfort/acceptance.
### Opt-in rendering measurements
`FRAMEYAP_PROFILE=1` prints aggregate CPU raster time, upload time, and grab/scale
tracking/transform time on exit. Counts and timings contain no speech, typed text
or pointer coordinates. The motion measurements exclude sleep and other runtime
work; they do not measure motion-to-photon latency. During an initialized grab or
scale, neither the renderer nor uploader runs; content updates resume on release.
+12 -10
View File
@@ -1,7 +1,7 @@
# Release packaging and idempotent user-local installer
**No GitHub release is published.** Native-only local artifacts have been installed
and reinstalled on Frame. The installer never pretends an ASR runtime is included when it is not;
Native-only GitHub release archives are published with the standalone installer.
The ARM64 artifacts are built, tested, installed and checked on Frame before publication. The installer never pretends an ASR runtime is included when it is not;
see [third-party notes](third-party.md).
## Producer
@@ -25,9 +25,9 @@ runtime/bin/python3 # ONLY for an authorized bundled-runtime artif
The stage copies the self-contained installer into `bin/` and CMake installs
`scripts/model-status.py` alongside the backend manifests. The package allowlist
permits `scripts/`; installed `frameyap --list-models` expects the verifier at
`../scripts/model-status.py`. **No actual staged archive has been audited/tested
from a clean account for publication**; exercise the entire producer pipeline
and the installed CLI before treating the payload layout as release-ready.
`../scripts/model-status.py`. Exercise the entire producer pipeline and installed CLI for each release.
Upgrade validation on the existing Frame account does not establish a clean-account
installation or general platform compatibility.
For the current **external-runtime** package, `scripts/stage-native.py --help`
documents explicit inputs. The old `scripts/stage-native-poc.py` is retained as
@@ -45,7 +45,7 @@ prefix. ARM64/glibc packaging is not a claim of compatibility with arbitrary Lin
```sh
python3 scripts/package-release.py --stage /path/to/stage --output /existing/output \
--version 0.1.202609241627 --arch linux-aarch64 \
--model-revision fad622f25f303105c20d70e201bcc477c88b620c --external-runtime
--model-revision 2bf128600aac4b16946f7ed8372e56117fe5e23b --external-runtime
```
Use the actual binary's numeric `MAJOR.MINOR.YYYYMMDDHHMM` UTC version
@@ -76,18 +76,19 @@ bootstrap, **not** the inference runtime), curl for network release downloads,
sha256sum and tar for archive handling. **No compiler, sudo, Steam store AppID
or engine checkout in binary mode.** No minimum glibc floor has been certified;
preflight alone cannot guarantee compatibility. Download/inspect a pinned installer
before running it. Current local artifact route:
before running it. Local artifact route:
```sh
sh install.sh --archive /path/to/frameyap-VERSION-linux-aarch64.tar.gz \
--sha256 64_HEX_DIGIT_HASH --version VERSION
```
After an actual vetted release exists, use a real numeric version (for example,
For a published release, use its numeric version (for example,
`sh install.sh --mode binary --version 0.1.202609241627 --yes`); the installer
will retrieve tag `v0.1.202609241627` and its versioned checksum; no `latest` or moving-branch
lookup. A pipe invocation is supported, never reads answers from the pipe, and must also
pin a real published tag. **There is no functional public download command yet.**
pin a real published tag. The README's `releases/latest/download/install.sh` bootstrap
selects the fixed release version embedded in that published installer.
`--without-model` omits any model files in the selected archive, never deletes
an existing current model on rerun, and records the choice. Same
@@ -116,7 +117,8 @@ only the new venv and leaves `paths.conf` unchanged. `--print-plan --json` shows
the packages, index and size (about 200 MB of wheels, roughly 1–1.5 GB on disk)
without touching anything. The app must be closed (install lock). Transitive
dependencies are not pinned. Validated end to end on x86-64 Python 3.12
(2026-09-25); a clean Frame install is still pending.
(2026-09-25); the owner also reported a successful fresh Frame model/runtime
installation from v0.1.202609251524. Later upgrade tests preserve that runtime.
In the source tree, `python3 scripts/model-status.py --list-models` or
`--check-model redux --model-dir /absolute/model` hash-checks local files;
the native `frameyap --list-models` / `--check-model` entry points use the
+4 -3
View File
@@ -2,8 +2,9 @@
FrameYap's own code is under [LICENSE](../LICENSE). Everything below is fetched
from, or built against, its upstream project; each keeps its own license. This is
a pointer list, not a legal audit. No prebuilt archive is published: v0.1 is
source-only, and Python packages are fetched from PyPI on the user's machine.
a pointer list, not a legal audit. Published native archives include SDL3 and the OpenVR client library, plus
the font and required notices. The speech runtime and model are downloaded
separately on the user's machine; they are not redistributed in the archive.
## Included in this repository
@@ -27,7 +28,7 @@ source-only, and Python packages are fetched from PyPI on the user's machine.
| Component | Where | License |
| --- | --- | --- |
| Parakeet Redux weights, revision `fad622f25f303105c20d70e201bcc477c88b620c`, derived from NVIDIA Parakeet TDT 0.6B v3 by Moondream/M87 Labs; pinned sizes and hashes in `assets/backends/redux.json` | <https://huggingface.co/moondream/parakeet-redux> | CC-BY-4.0 |
| Parakeet Redux weights, revision `2bf128600aac4b16946f7ed8372e56117fe5e23b`, derived from NVIDIA Parakeet TDT 0.6B v3 by Moondream/M87 Labs; pinned sizes and hashes in `assets/backends/redux.json` | <https://huggingface.co/moondream/parakeet-redux> | CC-BY-4.0 |
| Kestrel, `kestrel-kernels` | <https://github.com/m87-labs/kestrel> | Local inference is free and needs no API key, per the Kestrel README. Finetuned-model inference needs an API key and is not used here. |
| `moondream` | PyPI | See the package |
| PyTorch (CPU build) | <https://github.com/pytorch/pytorch> | BSD-3-Clause |
+7 -1
View File
@@ -57,7 +57,7 @@ device="cpu", cpu_threads=threads)` and persistent
Install an **isolated** Python runtime with the
moondream 2.4.0, kestrel 0.8.0 and compatible CPU dependencies; provide
preinstalled local weights from revision
`fad622f25f303105c20d70e201bcc477c88b620c` and pass its directory
`2bf128600aac4b16946f7ed8372e56117fe5e23b` and pass its directory
explicitly. The code verifies exact sizes and SHA-256 of weights/config/tokenizer
against `assets/backends/redux.json` via the shared, offline
`python/frameyap/model_files.py` schema/verifier before importing model libraries.
@@ -112,6 +112,12 @@ is required to avoid competing rotation by multiple workers. Debug output never
shares the framed protocol stdout. Direct worker CLI use with `--advanced-debug`
writes to its caller's stderr; the native adapter supplies the private bounded sink.
The same opt-in also writes `delivery-debug.log` (rotated to
`delivery-debug.previous.log` when debugging is enabled, capped near 1 MiB) in
that directory: paced-delivery metadata only — start/sent byte counts and
offsets, lease/send errors, finish results and which focus-guard check failed.
It never contains transcript text.
Hardware-free tests run through CTest, including fake-child cancellation, short
injected warmup/request deadlines, duplicate/stale replies, malformed frames,
missing/hash-mismatched model files and symlink refusal. Default production
+8 -4
View File
@@ -45,7 +45,7 @@ import urllib.request
KEY = "local.frameyap.overlay"
# The pinned release this installer belongs to (set in the release commit that
# is tagged vRELEASE_VERSION); empty in a development tree.
RELEASE_VERSION = "0.1.202609251524"
RELEASE_VERSION = "0.1.202610050210"
MARKER = "# FrameYap managed launcher v1\n"
ARCHIVE_LIMIT = 12 * 1024**3
MEMBER_LIMIT = 50000
@@ -59,6 +59,7 @@ CONFIG_DEFAULTS = {
"close_mic_when_idle": False,
"backend": "redux",
"lock_layout": False,
"wrist_world_fallback": True,
"clock_24h": False,
"date_format": "mdy",
"quick_inputs": ["/new", "/questions", "/help"],
@@ -66,7 +67,7 @@ CONFIG_DEFAULTS = {
"theme": {"background": "#0c101b", "card": "#141c2b", "ink": "#e6f0f9",
"muted": "#97adc1", "accent": "#1ff0a4", "warning": "#ff6e87",
"frame_start": "#1fff91", "frame_end": "#1f70ff"},
"gradient": {"enabled": True, "period_seconds": 30, "strength": 0.12},
"gradient": {"enabled": False, "period_seconds": 30, "strength": 0.12},
"buttons": {"left_grip": "/user/hand/left/input/grip",
"right_grip": "/user/hand/right/input/grip",
"ptt": "/user/hand/right/input/x", "cancel": "/user/hand/right/input/b",
@@ -167,6 +168,8 @@ def normalized_config(data):
fixed["backend"] = backend if isinstance(backend, str) and BACKEND_RE.fullmatch(backend) else "redux"
layout = data.get("lock_layout", False)
fixed["lock_layout"] = layout if type(layout) is bool else False
fallback = data.get("wrist_world_fallback", True)
fixed["wrist_world_fallback"] = fallback if type(fallback) is bool else True
clock = data.get("clock_24h", False)
fixed["clock_24h"] = clock if type(clock) is bool else False
date = data.get("date_format", "mdy")
@@ -686,7 +689,7 @@ def install_model(args, root):
if state["state"] != "installed_verified":
fail(f"model did not verify: {state['reason']}")
if not args.json:
print(f"Backend {backend.id} model verified at {dest}; license {backend.license_id}. Configure runtime/model paths explicitly for launch.")
print(f"Backend {backend.id} model verified at {dest}; license {backend.license_id}.")
def runtime_step(args, name, command):
@@ -881,7 +884,8 @@ def do_install(args, root, launcher):
select(root, "current", f"versions/{version}")
print(f"FrameYap {version} installed. OpenVR registration is NOT automatic; see docs/packaging.md.")
if json.loads((target / "release.json").read_text())["runtime"] == "external-authorized-python":
print("ASR runtime is NOT included. Run install.sh --install-runtime --yes to pip-install it, or supply one with FRAMEYAP_PYTHON/--python or paths.conf.")
print("The speech runtime (Python/moondream) is installed separately: the installer offers it next, "
"or run install.sh --install-runtime --yes later.")
def uninstall(root, launcher):
+26 -5
View File
@@ -61,6 +61,7 @@ class ModelFile:
path: str
size: int
sha256: str
compatible: tuple[tuple[int, str], ...] = ()
@dataclass(frozen=True)
@@ -127,13 +128,28 @@ def _parse(value):
parsed = []
names = set()
for item in files:
_fields(item, ("path", "size", "sha256"), "model file")
if not isinstance(item, dict):
raise ManifestError("invalid model file fields")
_fields(item, ("path", "size", "sha256", "compatible") if "compatible" in item
else ("path", "size", "sha256"), "model file")
name = _relative(item["path"], "model file path")
size = item["size"]
if name in names or type(size) is not int or not 0 < size <= _MAX_FILE or not isinstance(item["sha256"], str) or not _HASH.fullmatch(item["sha256"]):
raise ManifestError("duplicate or invalid pinned model file")
names.add(name)
parsed.append(ModelFile(name, size, item["sha256"]))
compatible = item.get("compatible", [])
if not isinstance(compatible, list) or len(compatible) > 8:
raise ManifestError("invalid compatible model files")
pins = []
for previous in compatible:
_fields(previous, ("size", "sha256"), "compatible model file")
old_size, old_hash = previous["size"], previous["sha256"]
if (type(old_size) is not int or not 0 < old_size <= _MAX_FILE or
not isinstance(old_hash, str) or not _HASH.fullmatch(old_hash) or
(old_size, old_hash) in [(size, item["sha256"]), *pins]):
raise ManifestError("duplicate or invalid compatible model file")
pins.append((old_size, old_hash))
parsed.append(ModelFile(name, size, item["sha256"], tuple(pins)))
license_info = value["license"]
_fields(license_info, ("id", "text"), "license")
requirements = value["requirements"]
@@ -177,7 +193,11 @@ def load_backends(manifest_dir=DEFAULT_MANIFEST_DIR):
def check_file(directory, item):
"""Return (reason, filename); reason None means the pinned file verifies."""
"""Verify the current pin or an explicitly compatible existing artifact.
Downloaders construct a temporary ModelFile with only the current pin so
newly fetched bytes must match the revision advertised to the user.
"""
root = Path(directory)
if not root.is_absolute():
return "model_dir_not_absolute", item.path
@@ -201,13 +221,14 @@ def check_file(directory, item):
before = os.fstat(file_fd)
if not stat.S_ISREG(before.st_mode):
return "unsafe_file", item.path
if before.st_size != item.size:
pins = ((item.size, item.sha256), *item.compatible)
if before.st_size not in {size for size, _ in pins}:
return "size_mismatch", item.path
digest = hashlib.sha256()
while chunk := os.read(file_fd, 1024 * 1024):
digest.update(chunk)
after = os.fstat(file_fd)
if (before.st_size, before.st_mtime_ns, before.st_ctime_ns) != (after.st_size, after.st_mtime_ns, after.st_ctime_ns) or digest.hexdigest() != item.sha256:
if (before.st_size, before.st_mtime_ns, before.st_ctime_ns) != (after.st_size, after.st_mtime_ns, after.st_ctime_ns) or (before.st_size, digest.hexdigest()) not in pins:
return "hash_mismatch", item.path
return None, None
finally:
+63
View File
@@ -0,0 +1,63 @@
#!/usr/bin/env python3
"""Explicit, bounded GitHub release check. No network access unless --check is passed."""
import argparse
from datetime import datetime
import json
import re
import sys
from urllib import request
RELEASE = re.compile(r"v?((?:0|[1-9][0-9]*)\.(?:0|[1-9][0-9]*)\.([1-9][0-9]{11}))\Z")
URL = "https://api.github.com/repos/baketnk/frame-yap/releases/latest"
def version(value):
match = RELEASE.fullmatch(value)
if not match:
raise ValueError("invalid release version")
datetime.strptime(match[2], "%Y%m%d%H%M")
return tuple(map(int, match[1].split(".")))
def outcome(payload, current):
data = json.loads(payload)
tag = data["tag_name"]
if not isinstance(tag, str) or not tag.startswith("v"):
raise ValueError("invalid release tag")
candidate = version(tag)
return "AVAILABLE " + tag[1:] if candidate > version(current) else "CURRENT"
class NoRedirect(request.HTTPRedirectHandler):
def redirect_request(self, req, fp, code, msg, headers, newurl):
return None
def check(current):
version(current) # Reject malformed local metadata before making any request.
opener = request.build_opener(request.ProxyHandler({}), NoRedirect())
req = request.Request(URL, headers={"Accept": "application/vnd.github+json", "User-Agent": "FrameYap-update-check"})
with opener.open(req, timeout=6) as response:
payload = response.read(65537)
if len(payload) > 65536:
raise ValueError("oversized response")
return outcome(payload, current)
def main(argv=None):
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--check", action="store_true", help="explicit network check")
parser.add_argument("--current", required=True)
args = parser.parse_args(argv)
if not args.check:
parser.error("--check required for network access")
try:
print(check(args.current), flush=True)
except (ValueError, KeyError, TypeError, OSError):
print("ERROR", flush=True)
return 1
return 0
if __name__ == "__main__":
sys.exit(main())
+47
View File
@@ -0,0 +1,47 @@
#!/usr/bin/env python3
"""Attended terminal handoff for a user-requested FrameYap update."""
import argparse
from pathlib import Path
import subprocess
import sys
# The same validation as check-update.py, without importing the network helper.
import re
VERSION = re.compile(r"(?:0|[1-9][0-9]*)\.(?:0|[1-9][0-9]*)\.[1-9][0-9]{11}\Z")
def installer_command(script, version):
if not VERSION.fullmatch(version):
raise ValueError("invalid version")
return ["sh", str(script), "--mode", "binary", "--version", version, "--yes"]
def main(argv=None):
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--version", required=True)
args = parser.parse_args(argv)
script = Path(__file__).resolve().parent.parent / "bin/install.sh"
try:
command = installer_command(script, args.version)
except ValueError:
parser.error("invalid release version")
if not script.is_file():
print("Installed FrameYap installer not found.")
return 1
print(f"FrameYap update: {args.version}")
print("Close FrameYap, then press Enter here to download and install this version.")
if not sys.stdin.isatty():
print("A terminal is required; no download started.")
return 1
try:
input()
except (KeyboardInterrupt, EOFError):
print("Update cancelled.")
return 1
result = subprocess.run(command, check=False)
print("Update installed." if result.returncode == 0 else "Update failed; check the installer message above.")
return result.returncode
if __name__ == "__main__":
sys.exit(main())
+8 -4
View File
@@ -25,7 +25,7 @@ import urllib.request
KEY = "local.frameyap.overlay"
# The pinned release this installer belongs to (set in the release commit that
# is tagged vRELEASE_VERSION); empty in a development tree.
RELEASE_VERSION = "0.1.202609251524"
RELEASE_VERSION = "0.1.202610050210"
MARKER = "# FrameYap managed launcher v1\n"
ARCHIVE_LIMIT = 12 * 1024**3
MEMBER_LIMIT = 50000
@@ -39,6 +39,7 @@ CONFIG_DEFAULTS = {
"close_mic_when_idle": False,
"backend": "redux",
"lock_layout": False,
"wrist_world_fallback": True,
"clock_24h": False,
"date_format": "mdy",
"quick_inputs": ["/new", "/questions", "/help"],
@@ -46,7 +47,7 @@ CONFIG_DEFAULTS = {
"theme": {"background": "#0c101b", "card": "#141c2b", "ink": "#e6f0f9",
"muted": "#97adc1", "accent": "#1ff0a4", "warning": "#ff6e87",
"frame_start": "#1fff91", "frame_end": "#1f70ff"},
"gradient": {"enabled": True, "period_seconds": 30, "strength": 0.12},
"gradient": {"enabled": False, "period_seconds": 30, "strength": 0.12},
"buttons": {"left_grip": "/user/hand/left/input/grip",
"right_grip": "/user/hand/right/input/grip",
"ptt": "/user/hand/right/input/x", "cancel": "/user/hand/right/input/b",
@@ -147,6 +148,8 @@ def normalized_config(data):
fixed["backend"] = backend if isinstance(backend, str) and BACKEND_RE.fullmatch(backend) else "redux"
layout = data.get("lock_layout", False)
fixed["lock_layout"] = layout if type(layout) is bool else False
fallback = data.get("wrist_world_fallback", True)
fixed["wrist_world_fallback"] = fallback if type(fallback) is bool else True
clock = data.get("clock_24h", False)
fixed["clock_24h"] = clock if type(clock) is bool else False
date = data.get("date_format", "mdy")
@@ -666,7 +669,7 @@ def install_model(args, root):
if state["state"] != "installed_verified":
fail(f"model did not verify: {state['reason']}")
if not args.json:
print(f"Backend {backend.id} model verified at {dest}; license {backend.license_id}. Configure runtime/model paths explicitly for launch.")
print(f"Backend {backend.id} model verified at {dest}; license {backend.license_id}.")
def runtime_step(args, name, command):
@@ -861,7 +864,8 @@ def do_install(args, root, launcher):
select(root, "current", f"versions/{version}")
print(f"FrameYap {version} installed. OpenVR registration is NOT automatic; see docs/packaging.md.")
if json.loads((target / "release.json").read_text())["runtime"] == "external-authorized-python":
print("ASR runtime is NOT included. Run install.sh --install-runtime --yes to pip-install it, or supply one with FRAMEYAP_PYTHON/--python or paths.conf.")
print("The speech runtime (Python/moondream) is installed separately: the installer offers it next, "
"or run install.sh --install-runtime --yes later.")
def uninstall(root, launcher):
+2
View File
@@ -5,6 +5,8 @@ import json
from pathlib import Path
import sys
# Installed release inventories must remain unchanged by status queries.
sys.dont_write_bytecode = True
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "python"))
from frameyap.model_files import DEFAULT_MANIFEST_DIR, ManifestError, check_model, load_backends
+72
View File
@@ -0,0 +1,72 @@
#include "companion_apps.hpp"
#include <cerrno>
#include <cstdlib>
#include <string>
#include <sys/stat.h>
#include <sys/wait.h>
#include <unistd.h>
#include <fcntl.h>
namespace frameyap {
namespace {
bool executable(const std::filesystem::path& path) {
struct stat info{};
return path.is_absolute() && ::stat(path.c_str(), &info) == 0 && S_ISREG(info.st_mode) &&
::access(path.c_str(), X_OK) == 0;
}
}
std::optional<std::filesystem::path> find_companion(std::string_view name) {
if (name != "tnkplan" && name != "tnkboard" && name != "tnkdraw") return {};
if (const char* home = std::getenv("HOME")) {
const auto launcher = std::filesystem::path(home) / ".local/bin" / name;
if (executable(launcher)) return launcher;
}
if (const char* path = std::getenv("PATH")) {
const std::string dirs(path);
for (size_t start = 0; start <= dirs.size();) {
const size_t end = dirs.find(':', start);
const auto launcher = std::filesystem::path(dirs.substr(start, end - start)) / name;
// Ignore empty/relative PATH entries: never execute from an untrusted cwd.
if (executable(launcher)) return launcher;
if (end == std::string::npos) break;
start = end + 1;
}
}
return {};
}
bool launch_companion(const std::filesystem::path& launcher, CompanionCommand command) {
if (!executable(launcher)) return false;
const auto path = launcher.string();
const char* option = command == CompanionCommand::Show ? "--show" :
command == CompanionCommand::Recenter ? "--recenter" : nullptr;
char* const args[] = {const_cast<char*>(path.c_str()), const_cast<char*>(option), nullptr};
// Double fork so the long-lived companion is adopted rather than becoming
// a zombie owned by FrameYap. An O_CLOEXEC pipe reports exec failure only.
int pipefd[2];
if (::pipe2(pipefd, O_CLOEXEC) != 0) return false;
const pid_t intermediate = ::fork();
if (intermediate < 0) { ::close(pipefd[0]); ::close(pipefd[1]); return false; }
if (intermediate == 0) {
::close(pipefd[0]);
const pid_t child = ::fork();
if (child < 0) {
const int error = errno;
(void)::write(pipefd[1], &error, sizeof(error));
} else if (child == 0) {
::execv(path.c_str(), args);
const int error = errno;
(void)::write(pipefd[1], &error, sizeof(error));
}
::_exit(child < 0 ? 1 : 0);
}
::close(pipefd[1]);
int error = 0;
ssize_t count;
do { count = ::read(pipefd[0], &error, sizeof(error)); } while (count < 0 && errno == EINTR);
::close(pipefd[0]);
int status = 0;
pid_t reaped;
do { reaped = ::waitpid(intermediate, &status, 0); } while (reaped < 0 && errno == EINTR);
return count == 0 && reaped == intermediate && WIFEXITED(status) && WEXITSTATUS(status) == 0;
}
}
+13
View File
@@ -0,0 +1,13 @@
#pragma once
#include <filesystem>
#include <optional>
#include <string_view>
namespace frameyap {
// Resolve an installed launcher once at boot; never search the source checkout.
std::optional<std::filesystem::path> find_companion(std::string_view name);
// Fixed control arguments only; never pass pointer/UI text to a shell.
enum class CompanionCommand { Default, Show, Recenter };
// Returns false if fork or exec fails; does not wait for the overlay to exit.
bool launch_companion(const std::filesystem::path& launcher, CompanionCommand command = CompanionCommand::Default);
}
+49 -9
View File
@@ -215,6 +215,9 @@ Config load_config(const std::filesystem::path& path) {
} else if (key == "lock_layout") {
if (!value.is_bool) throw std::runtime_error("Config lock_layout must be a boolean");
config.lock_layout = value.value == "true";
} else if (key == "wrist_world_fallback") {
if (!value.is_bool) throw std::runtime_error("Config wrist_world_fallback must be a boolean");
config.wrist_world_fallback = value.value == "true";
} else if (key == "clock_24h") {
if (!value.is_bool) throw std::runtime_error("Config clock_24h must be a boolean");
config.clock_24h = value.value == "true";
@@ -304,7 +307,7 @@ Config load_config(const std::filesystem::path& path) {
}
namespace {
bool save_option(const std::filesystem::path& path, std::string_view key,
std::string_view value, bool string_value) noexcept {
std::string_view value, bool string_value, std::string_view parent_key = {}) noexcept {
try {
if (path.empty() || !path.is_absolute() || std::filesystem::is_symlink(path)) return false;
const bool existing = std::filesystem::exists(path);
@@ -315,14 +318,28 @@ bool save_option(const std::filesystem::path& path, std::string_view key,
auto root = parser.parse(); parser.ws();
if (!root.is_object || parser.pos != bytes.size()) return false;
const std::string encoded = string_value ? "\"" + std::string(value) + "\"" : std::string(value);
auto it = root.object.find(std::string(key));
if (it != root.object.end()) {
if (it->second.is_string != string_value || (!string_value && !it->second.is_bool)) return false;
if (it->second.value == value) return true;
bytes.replace(it->second.start, it->second.end - it->second.start, encoded);
} else {
bytes.insert(root.end - 1, std::string(root.object.empty() ? "" : ",") +
"\"" + std::string(key) + "\":" + encoded);
const Json* object = &root;
if (!parent_key.empty()) {
const auto parent = root.object.find(std::string(parent_key));
if (parent == root.object.end()) {
bytes.insert(root.end - 1, std::string(root.object.empty() ? "" : ",") +
"\"" + std::string(parent_key) + "\":{\"" + std::string(key) + "\":" + encoded + "}");
object = nullptr;
} else {
if (!parent->second.is_object) return false;
object = &parent->second;
}
}
if (object) {
auto it = object->object.find(std::string(key));
if (it != object->object.end()) {
if (it->second.is_string != string_value || (!string_value && !it->second.is_bool)) return false;
if (it->second.value == value) return true;
bytes.replace(it->second.start, it->second.end - it->second.start, encoded);
} else {
bytes.insert(object->end - 1, std::string(object->object.empty() ? "" : ",") +
"\"" + std::string(key) + "\":" + encoded);
}
}
// The native reader rejects files >=4097 bytes, even if the JSON is valid.
if (bytes.size() > 4096) return false;
@@ -345,12 +362,18 @@ bool save_advanced_debug(const std::filesystem::path& path, bool enabled) noexce
bool save_auto_insert(const std::filesystem::path& path, bool enabled) noexcept {
return save_bool_option(path, "auto_insert", enabled);
}
bool save_gradient_enabled(const std::filesystem::path& path, bool enabled) noexcept {
return save_option(path, "enabled", enabled ? "true" : "false", false, "gradient");
}
bool save_close_mic_when_idle(const std::filesystem::path& path, bool enabled) noexcept {
return save_bool_option(path, "close_mic_when_idle", enabled);
}
bool save_lock_layout(const std::filesystem::path& path, bool enabled) noexcept {
return save_bool_option(path, "lock_layout", enabled);
}
bool save_wrist_world_fallback(const std::filesystem::path& path, bool enabled) noexcept {
return save_bool_option(path, "wrist_world_fallback", enabled);
}
bool save_clock_24h(const std::filesystem::path& path, bool enabled) noexcept {
return save_bool_option(path, "clock_24h", enabled);
}
@@ -399,9 +422,26 @@ std::filesystem::path action_manifest(const std::string& assets, const Config& c
for (const auto& [action, path] : buttons) {
if (path.empty()) continue;
if (!used.emplace(path, action).second) throw std::runtime_error("Two actions share Frame button: " + path);
// A remapped thumbstick click must share the joystick source rather
// than create a second source for the same physical input path.
if (path == "/user/hand/left/input/thumbstick" || path == "/user/hand/right/input/thumbstick") continue;
if (binding.back() != '[') binding += ',';
binding += "{\"path\":\"" + path + "\",\"mode\":\"button\",\"inputs\":{\"click\":{\"output\":\"/actions/frameyap/in/" + action + "\"}}}";
}
// Keep configured stick clicks in the ordinary set. The dedicated grab
// set contains only position, so its priority cannot promote a text action.
for (const auto& hand : {"left", "right"}) {
const std::string path = std::string("/user/hand/") + hand + "/input/thumbstick";
if (auto it = used.find(path); it != used.end()) {
if (binding.back() != '[') binding += ',';
binding += "{\"path\":\"" + path + "\",\"mode\":\"joystick\",\"inputs\":{\"click\":{\"output\":\"/actions/frameyap/in/" + it->second + "\"}}}";
}
}
binding += "]},\"/actions/grab\":{\"sources\":[";
for (const auto& hand : {"left", "right"}) {
if (binding.back() != '[') binding += ',';
binding += "{\"path\":\"/user/hand/" + std::string(hand) + "/input/thumbstick\",\"mode\":\"joystick\",\"inputs\":{\"position\":{\"output\":\"/actions/grab/in/" + hand + "_depth\"}}}";
}
binding += "]}}}";
auto manifest = read_file(source);
// The OpenVR SDK resolves relative binding URLs beside its manifest.
+4 -1
View File
@@ -15,7 +15,7 @@ struct Theme {
Rgba frame_start{31, 255, 145, 255}, frame_end{31, 112, 255, 255};
};
struct GradientConfig {
bool enabled = true;
bool enabled = false; // animated background is opt-in; static frame styling remains
float period_seconds = 30.f;
float strength = .12f; // muted edge colors blended into theme.background
};
@@ -31,6 +31,7 @@ struct Config {
bool auto_insert = false; // opt-in; runtime also requires uninterrupted verified Xwayland focus
bool close_mic_when_idle = false; // default keeps the device open, discarding idle audio
bool lock_layout = false; // hide the grab and scale handles when enabled
bool wrist_world_fallback = true; // show a world-space panel if a selected wrist loses tracking
bool clock_24h = false;
DateFormat date_format = DateFormat::MonthDayYear;
WristPlacement wrist;
@@ -44,8 +45,10 @@ Config load_config(const std::filesystem::path& path);
// Other user customizations and formatting are retained; creates a minimal config if absent.
bool save_advanced_debug(const std::filesystem::path& path, bool enabled) noexcept;
bool save_auto_insert(const std::filesystem::path& path, bool enabled) noexcept;
bool save_gradient_enabled(const std::filesystem::path& path, bool enabled) noexcept;
bool save_close_mic_when_idle(const std::filesystem::path& path, bool enabled) noexcept;
bool save_lock_layout(const std::filesystem::path& path, bool enabled) noexcept;
bool save_wrist_world_fallback(const std::filesystem::path& path, bool enabled) noexcept;
bool save_clock_24h(const std::filesystem::path& path, bool enabled) noexcept;
bool save_date_format(const std::filesystem::path& path, DateFormat format) noexcept;
bool save_backend(const std::filesystem::path& path, const std::string& id) noexcept;
+5 -2
View File
@@ -89,7 +89,9 @@ void Controller::delivery_detail(DeliveryResult result, bool submit) {
}
std::shared_ptr<ControllerFocus> Controller::manual_focus() {
auto focus = focus_factory_();
if (!focus || !focus->arm()) throw std::runtime_error("Current target not verified; review kept");
if (!focus || !focus->arm())
throw std::runtime_error(std::string("Current target not verified: ") +
(focus ? focus->failure() : "focus unavailable") + "; review kept");
return std::shared_ptr<ControllerFocus>(std::move(focus));
}
void Controller::queue_paced(std::string text, bool enter, bool review,
@@ -370,7 +372,8 @@ void Controller::action(UiAction action) {
} catch (const std::exception& e) {
if (session_.state() == State::Recording || session_.state() == State::Transcribing) {
audio_.close(); session_.fail(); armed_focus_.reset(); status_note_.clear();
} else status_note_ = "Input unavailable - check destination";
} else status_note_ = std::string(e.what()).find("DISPLAY missing") != std::string::npos ?
"Input unavailable: DISPLAY missing" : "Input unavailable - check destination";
detail_ = e.what();
}
}
+1
View File
@@ -36,6 +36,7 @@ public:
virtual ~ControllerFocus() = default;
virtual bool arm() = 0;
virtual bool valid() = 0;
virtual const char* failure() const { return "focus unavailable"; }
};
using FocusFactory = std::function<std::unique_ptr<ControllerFocus>()>;
+34 -12
View File
@@ -21,6 +21,7 @@ struct FocusGuard::Impl {
bool attempted = false;
bool armed = false;
bool dead = false;
const char* reason = "none";
~Impl() { if (connection) xcb_disconnect(connection); }
@@ -80,10 +81,12 @@ struct FocusGuard::Impl {
bool snapshot(xcb_window_t& current) {
xcb_window_t active = XCB_WINDOW_NONE, gamescope = XCB_WINDOW_NONE, actual = XCB_WINDOW_NONE;
if (!property_window(active_atom, XCB_ATOM_WINDOW, active) ||
!property_window(gamescope_atom, XCB_ATOM_CARDINAL, gamescope) ||
!focus(actual) || active != gamescope || active != actual || !keys_up())
return false;
if (!property_window(active_atom, XCB_ATOM_WINDOW, active)) { reason = "no _NET_ACTIVE_WINDOW"; return false; }
if (!property_window(gamescope_atom, XCB_ATOM_CARDINAL, gamescope)) { reason = "no GAMESCOPE_FOCUSED_WINDOW"; return false; }
if (!focus(actual)) { reason = "no X input focus"; return false; }
if (active != gamescope) { reason = "active != gamescope focus"; return false; }
if (active != actual) { reason = "active != X input focus"; return false; }
if (!keys_up()) { reason = "key held"; return false; }
current = active;
return true;
}
@@ -92,15 +95,27 @@ struct FocusGuard::Impl {
while (xcb_generic_event_t* event = xcb_poll_for_event(connection)) {
const uint8_t type = event->response_type & 0x7f;
bool bad = type == 0; // asynchronous X error
if (bad) reason = "X error";
if (type == XCB_PROPERTY_NOTIFY) {
const auto* e = reinterpret_cast<xcb_property_notify_event_t*>(event);
if (e->window == root && (e->atom == active_atom || e->atom == gamescope_atom)) bad = true;
// Gamescope rewrites its focus properties with the same window
// after IME commits; only a different value is a focus change.
// A real X focus move still reports FocusOut on the target.
xcb_window_t value = XCB_WINDOW_NONE;
if (e->window == root && e->atom == active_atom &&
(!property_window(active_atom, XCB_ATOM_WINDOW, value) || value != target)) {
bad = true; reason = "_NET_ACTIVE_WINDOW changed";
}
if (e->window == root && e->atom == gamescope_atom &&
(!property_window(gamescope_atom, XCB_ATOM_CARDINAL, value) || value != target)) {
bad = true; reason = "GAMESCOPE_FOCUSED_WINDOW changed";
}
} else if (type == XCB_FOCUS_OUT) {
const auto* e = reinterpret_cast<xcb_focus_out_event_t*>(event);
if (e->event == target) bad = true;
if (e->event == target) { bad = true; reason = "FocusOut on target"; }
} else if (type == XCB_DESTROY_NOTIFY) {
const auto* e = reinterpret_cast<xcb_destroy_notify_event_t*>(event);
if (e->window == target) bad = true;
if (e->window == target) { bad = true; reason = "target destroyed"; }
} else if (type == XCB_CREATE_NOTIFY || type == XCB_UNMAP_NOTIFY ||
type == XCB_MAP_NOTIFY || type == XCB_MAP_REQUEST ||
type == XCB_REPARENT_NOTIFY || type == XCB_CONFIGURE_NOTIFY ||
@@ -122,18 +137,22 @@ struct FocusGuard::Impl {
case XCB_CIRCULATE_NOTIFY: window = reinterpret_cast<xcb_circulate_notify_event_t*>(event)->window; break;
case XCB_CIRCULATE_REQUEST: window = reinterpret_cast<xcb_circulate_request_event_t*>(event)->window; break;
}
if (window == target) bad = true;
if (window == target) { bad = true; reason = "structure event on target"; }
}
std::free(event);
if (bad) return false;
}
return !failed();
if (failed()) { reason = "X connection failed"; return false; }
return true;
}
bool check() {
if (!armed || failed() || !drain_events()) { dead = true; return false; }
if (!armed || dead) { reason = armed ? reason : "not armed"; dead = true; return false; }
if (failed() || !drain_events()) { dead = true; return false; }
xcb_window_t current = XCB_WINDOW_NONE;
if (!snapshot(current) || current != target || !drain_events()) {
if (!snapshot(current)) { dead = true; return false; }
if (current != target) { reason = "focus moved to another window"; dead = true; return false; }
if (!drain_events()) {
dead = true;
return false;
}
@@ -148,9 +167,11 @@ bool FocusGuard::arm() {
auto& p = *impl_;
if (p.attempted || p.dead) return false;
p.attempted = true;
const char* display = std::getenv("DISPLAY");
if (!display || !*display) { p.reason = "DISPLAY missing"; p.dead = true; return false; }
int screen_number = 0;
p.connection = xcb_connect(nullptr, &screen_number);
if (!p.connection || xcb_connection_has_error(p.connection)) { p.dead = true; return false; }
if (!p.connection || xcb_connection_has_error(p.connection)) { p.reason = "X display unavailable"; p.dead = true; return false; }
const xcb_setup_t* setup = xcb_get_setup(p.connection);
auto iter = xcb_setup_roots_iterator(setup);
for (int i = 0; i < screen_number && iter.rem; ++i) xcb_screen_next(&iter);
@@ -178,6 +199,7 @@ bool FocusGuard::arm() {
}
bool FocusGuard::valid() { return impl_->check(); }
const char* FocusGuard::failure() const { return impl_->reason; }
void FocusGuard::invalidate() { impl_->dead = true; impl_->armed = false; }
} // namespace frameyap
+2
View File
@@ -18,6 +18,8 @@ public:
bool arm();
bool valid();
void invalidate();
// Static label of the most recent failed check, for opt-in diagnostics.
const char* failure() const;
private:
struct Impl;
+1 -1
View File
@@ -29,7 +29,7 @@ int model_status(const CliOptions& options, const char* executable) {
script = fs::path(FRAMEYAP_SOURCE_ROOT) / "scripts/model-status.py";
if (!fs::is_regular_file(script))
throw std::runtime_error("model-status.py missing beside application layout: " + script.string());
std::vector<std::string> args{"python3", script.string()};
std::vector<std::string> args{"python3", "-B", script.string()};
if (options.mode == CliMode::ListModels) args.emplace_back("--list-models");
else { args.emplace_back("--check-model"); args.push_back(options.model_id); }
if (!options.model_dir.empty()) { args.emplace_back("--model-dir"); args.push_back(options.model_dir); }
+7
View File
@@ -38,6 +38,13 @@ std::optional<Mount> parse_mount(std::string_view name) {
return std::nullopt;
}
std::optional<Mount> effective_mount(Mount selected, bool wrist_tracked, bool wrist_world_fallback) {
if (selected != Mount::LeftWrist && selected != Mount::RightWrist) return selected;
if (wrist_tracked) return selected;
if (wrist_world_fallback) return Mount::World;
return std::nullopt;
}
std::optional<Matrix34> world_mount_pose(const Matrix34& hmd) {
for (const auto& row : hmd)
for (float value : row)
+2
View File
@@ -11,6 +11,8 @@ enum class Mount { World, LeftWrist, RightWrist, Head };
std::string_view mount_name(Mount mount);
std::optional<Mount> parse_mount(std::string_view name);
// No mount means hide the panel until the selected wrist is tracked again.
std::optional<Mount> effective_mount(Mount selected, bool wrist_tracked, bool wrist_world_fallback);
// Row-major 3x4 affine transform (OpenVR axes: +Y up, -Z forward).
using Matrix34 = std::array<std::array<float, 4>, 3>;
+277 -35
View File
@@ -1,9 +1,13 @@
#include "overlay.hpp"
#include "overlay_texture.hpp"
#include "texture_refresh.hpp"
#include "angle_fade.hpp"
#include "gestures.hpp"
#include "laser_setting.hpp"
#include "placement.hpp"
#include "panel_surface.hpp"
#include "companion_apps.hpp"
#include "update_check.hpp"
#include <openvr.h>
@@ -19,6 +23,7 @@
#include <stdexcept>
#include <string>
#include <string_view>
#include <utility>
#include <vector>
#include <unistd.h>
@@ -79,17 +84,25 @@ struct Overlay::Impl {
vr::VROverlayHandle_t handle = vr::k_ulOverlayHandleInvalid;
std::unique_ptr<OverlayTexture> gpu_texture;
vr::VRActionSetHandle_t action_set = vr::k_ulInvalidActionSetHandle;
vr::VRActionSetHandle_t grab_action_set = vr::k_ulInvalidActionSetHandle;
std::array<vr::VRInputValueHandle_t, 2> hands{};
std::array<vr::VRActionHandle_t, 7> actions{};
std::array<vr::VRActionHandle_t, 2> depth_actions{};
struct Persistence {
std::filesystem::path settings_path, laser_settings_path;
bool save_failed = false, debug_save_failed = false, auto_save_failed = false;
bool layout_save_failed = false, mic_save_failed = false, laser_change_failed = false;
bool layout_save_failed = false, mic_save_failed = false, gradient_save_failed = false, laser_change_failed = false;
bool placement_save_failed = false;
bool persist_mount = true;
} persistence;
Mount mount;
bool lasers_anytime = false;
Config config;
PanelSurface surface;
std::optional<std::filesystem::path> plan_launcher, keyboard_launcher, draw_launcher;
UpdateCheck updates;
std::filesystem::path install_update_script;
std::string update_version;
Panel panel;
std::vector<ModelAction> model_actions;
StatusIndicators indicators;
@@ -99,6 +112,7 @@ struct Overlay::Impl {
float size_scale = 1.f;
bool placement_dirty = false;
Matrix34 canvas_pose{};
std::array<std::optional<RelativePlacement>, 4> saved_placements{};
struct DragState {
Matrix34 canvas{};
PanelDrag panel;
@@ -107,7 +121,7 @@ struct Overlay::Impl {
vr::TrackedDeviceIndex_t device = vr::k_unTrackedDeviceIndexInvalid;
bool trigger_observed = false;
float scale = 1.f;
std::chrono::steady_clock::time_point started{};
std::chrono::steady_clock::time_point started{}, last_depth_sample{};
} dragging;
vr::HmdMatrix34_t world_transform{};
std::optional<Mount> applied_mount;
@@ -118,6 +132,7 @@ struct Overlay::Impl {
std::array<vr::TrackedDevicePose_t, vr::k_unMaxTrackedDeviceCount> poses{};
bool grip_capture = false, ptt_capture = false;
bool focus = true;
std::array<std::optional<std::pair<float, float>>, 2> cursor_positions{};
struct Diagnostics {
vr::EVRInputError action_update_error = vr::VRInputError_None;
unsigned pointer_downs = 0, pointer_ups = 0, pointer_actions = 0, pointer_resets = 0;
@@ -126,14 +141,45 @@ struct Overlay::Impl {
unsigned overlay_focus_events = 0, global_focus_events = 0, input_focus_captured_events = 0;
std::string last_pointer_event = "none";
} diagnostics;
struct Performance {
bool enabled = [] { const char* flag = std::getenv("FRAMEYAP_PROFILE");
return flag && std::string_view(flag) == "1"; }();
uint64_t renders = 0, uploads = 0, motion_ticks = 0, retained_frames = 0, motion_redraws = 0, motion_uploads = 0;
uint64_t depth_samples = 0, depth_axis_active = 0, depth_scroll_events = 0, depth_scroll_accepted = 0;
double max_depth = 0;
double render_ms = 0, upload_ms = 0, motion_ms = 0, motion_max_ms = 0;
static double elapsed(std::chrono::steady_clock::time_point start) {
return std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - start).count();
}
void report() const {
if (!enabled) return;
std::cout << "FrameYap profile: renders=" << renders << " render_ms=" << render_ms
<< " uploads=" << uploads << " upload_ms=" << upload_ms
<< " motion_ticks=" << motion_ticks << " motion_ms=" << motion_ms
<< " motion_max_ms=" << motion_max_ms << " retained_frames=" << retained_frames
<< " motion_redraws=" << motion_redraws << " motion_uploads=" << motion_uploads
<< " depth_samples=" << depth_samples << " depth_axis_active=" << depth_axis_active
<< " depth_scroll_events=" << depth_scroll_events << " depth_scroll_accepted=" << depth_scroll_accepted
<< " max_depth=" << max_depth << std::endl;
}
} performance;
Impl(const std::string& assets, const std::string& font, std::optional<Mount> requested, bool persist)
: persistence{default_mount_settings_path(), default_laser_settings_path()},
mount(requested ? *requested : load_mount(persistence.settings_path)),
lasers_anytime(load_lasers_anytime(persistence.laser_settings_path)),
config(load_config(default_config_path())),
surface(resolve_font(assets, font.empty() ? config.font : font), mount, config.theme, config.gradient) {
surface(resolve_font(assets, font.empty() ? config.font : font), mount, config.theme, config.gradient),
updates((std::filesystem::absolute(assets) / "../scripts/check-update.py").lexically_normal(), FRAMEYAP_VERSION),
install_update_script((std::filesystem::absolute(assets) / "../scripts/install-update.py").lexically_normal()) {
persistence.persist_mount = persist;
plan_launcher = find_companion("tnkplan");
keyboard_launcher = find_companion("tnkboard");
draw_launcher = find_companion("tnkdraw");
surface.set_companions(plan_launcher.has_value(), keyboard_launcher.has_value(), draw_launcher.has_value());
for (auto selected : {Mount::LeftWrist, Mount::RightWrist, Mount::Head})
saved_placements[static_cast<size_t>(selected)] =
load_relative_placement(default_placement_path(selected), selected);
const auto action_path = absolute_file(action_manifest(assets, config));
absolute_file(std::filesystem::path(assets) / "bindings_knuckles.json");
try {
@@ -170,9 +216,18 @@ struct Overlay::Impl {
throw std::runtime_error("Could not set OpenVR action manifest path");
if (input->GetActionSetHandle("/actions/frameyap", &action_set) != vr::VRInputError_None)
throw std::runtime_error("Could not find FrameYap action set");
if (input->GetActionSetHandle("/actions/grab", &grab_action_set) != vr::VRInputError_None)
throw std::runtime_error("Could not find grab action set");
input->GetInputSourceHandle("/user/hand/left", &hands[0]);
input->GetInputSourceHandle("/user/hand/right", &hands[1]);
for (size_t i = 0; i < action_names.size(); ++i)
if (input->GetActionHandle((std::string("/actions/frameyap/in/") + action_names[i]).c_str(), &actions[i]) != vr::VRInputError_None)
throw std::runtime_error(std::string("Could not find action ") + action_names[i]);
for (size_t i = 0; i < depth_actions.size(); ++i) {
const char* name = i == 0 ? "left_depth" : "right_depth";
if (input->GetActionHandle((std::string("/actions/grab/in/") + name).c_str(), &depth_actions[i]) != vr::VRInputError_None)
throw std::runtime_error(std::string("Could not find action ") + name);
}
overlay_check(overlay->CreateOverlay("local.frameyap.overlay.panel", "FrameYap", &handle), overlay, "CreateOverlay");
overlay_check(overlay->SetOverlayWidthInMeters(handle, 0.85f), overlay, "SetOverlayWidthInMeters");
overlay_check(overlay->SetOverlayInputMethod(handle, vr::VROverlayInputMethod_Mouse), overlay, "SetOverlayInputMethod");
@@ -182,13 +237,21 @@ struct Overlay::Impl {
lasers_anytime = false;
persistence.laser_change_failed = true;
}
overlay_check(overlay->SetOverlayFlag(handle, vr::VROverlayFlags_SendVRDiscreteScrollEvents, true), overlay, "Laser scroll");
// Normal UI scrolling uses discrete events. Request the compositor
// smooth stream only while grabbing for the depth fallback.
overlay->SetOverlayFlag(handle, vr::VROverlayFlags_SendVRSmoothScrollEvents, false);
surface.set_lasers_anytime(lasers_anytime);
surface.set_wrist_world_fallback(config.wrist_world_fallback);
surface.set_advanced_debug(config.advanced_debug);
surface.set_auto_insert(config.auto_insert);
surface.set_gradient_enabled(config.gradient.enabled);
surface.set_close_mic_when_idle(config.close_mic_when_idle);
surface.set_layout_locked(config.lock_layout);
surface.set_clock_24h(config.clock_24h);
surface.set_date_format(config.date_format);
surface.set_version(std::string(FRAMEYAP_VERSION) +
(std::string(FRAMEYAP_GIT_INFO).empty() ? "" : " " FRAMEYAP_GIT_INFO));
overlay_check(overlay->SetOverlayFlag(handle, vr::VROverlayFlags_VisibleInDashboard, true), overlay, "VisibleInDashboard");
vr::HmdVector2_t mouse_scale{{float(W), float(H)}};
overlay_check(overlay->SetOverlayMouseScale(handle, &mouse_scale), overlay, "SetOverlayMouseScale");
@@ -199,7 +262,7 @@ struct Overlay::Impl {
std::cout << input_mode_status() << std::endl;
} catch (...) { cleanup(); throw; }
}
~Impl() { cleanup(); }
~Impl() { performance.report(); cleanup(); }
void update_intersection_mask() {
const auto regions = surface.input_regions();
std::vector<vr::VROverlayIntersectionMaskPrimitive_t> mask(regions.size());
@@ -246,17 +309,27 @@ struct Overlay::Impl {
shown = wanted;
}
void place() {
Mount effective = mount;
vr::TrackedDeviceIndex_t target = vr::k_unTrackedDeviceIndex_Hmd;
if (mount == Mount::LeftWrist || mount == Mount::RightWrist) {
target = system->GetTrackedDeviceIndexForControllerRole(mount == Mount::LeftWrist
? vr::TrackedControllerRole_LeftHand : vr::TrackedControllerRole_RightHand);
if (target >= poses.size() || !poses[target].bPoseIsValid || !system->IsTrackedDeviceConnected(target))
effective = Mount::World;
}
const bool wrist_tracked = target < poses.size() && poses[target].bPoseIsValid &&
system->IsTrackedDeviceConnected(target);
const auto selected = effective_mount(mount, wrist_tracked, config.wrist_world_fallback);
if (!selected) {
if (placed) { surface.reset_pointers(); dragging.panel.reset(); }
placed = false;
applied_mount.reset();
visibility();
return;
}
const Mount effective = *selected;
if (effective == Mount::World && applied_mount && *applied_mount != Mount::World)
world_ready = false; // a fresh world fallback near the wearer, not an old room location
std::string note = persistence.mic_save_failed ? "Mic preference not saved; using it only for this session." :
std::string note = persistence.placement_save_failed ? "Placement not saved; using it only for this session." :
persistence.mic_save_failed ? "Mic preference not saved; using it only for this session." :
persistence.gradient_save_failed ? "Could not save background setting." :
persistence.layout_save_failed ? "Layout lock not saved; using it only for this session." :
persistence.auto_save_failed ? "Auto insert preference not saved; using it only for this session." :
persistence.debug_save_failed ? "Debug preference not saved; using it only for this session." :
@@ -285,14 +358,22 @@ struct Overlay::Impl {
const float base_width = mount_width(effective, config.wrist);
if (relocated) {
surface.reset_pointers(); dragging.panel.reset();
auto pose = effective == Mount::World ? matrix(world_transform) : relative_mount_pose(effective, config.wrist);
pose = resized_mount_pose(pose, base_width, size_scale, float(CH) / CW);
// Preserve main-panel dimensions when adding transparent margins.
const float meters_per_pixel = base_width * size_scale / CW;
const float dx = (W - CW) * .5f * meters_per_pixel;
const float dy = -(H - CH) * .5f * meters_per_pixel;
for (int r = 0; r < 3; ++r) pose[r][3] += pose[r][0] * dx + pose[r][1] * dy;
canvas_pose = pose;
const auto& remembered = saved_placements[static_cast<size_t>(effective)];
if (effective != Mount::World && remembered) {
// Saved canvas coordinates already include transparent margins.
size_scale = remembered->scale;
canvas_pose = remembered->canvas_pose;
} else {
if (effective != Mount::World) size_scale = 1.f;
auto pose = effective == Mount::World ? matrix(world_transform) : relative_mount_pose(effective, config.wrist);
pose = resized_mount_pose(pose, base_width, size_scale, float(CH) / CW);
// Preserve main-panel dimensions when adding transparent margins.
const float meters_per_pixel = base_width * size_scale / CW;
const float dx = (W - CW) * .5f * meters_per_pixel;
const float dy = -(H - CH) * .5f * meters_per_pixel;
for (int r = 0; r < 3; ++r) pose[r][3] += pose[r][0] * dx + pose[r][1] * dy;
canvas_pose = pose;
}
}
// After a grab, keep the full released pose. Never rebuild it from
// a planar offset or configured orientation on the next poll.
@@ -345,13 +426,46 @@ struct Overlay::Impl {
return;
}
dragging.scale = size_scale; dragging.canvas = canvas_pose;
dragging.started = std::chrono::steady_clock::now();
dragging.started = dragging.last_depth_sample = std::chrono::steady_clock::now();
if (kind == PanelDragKind::Grab)
overlay->SetOverlayFlag(handle, vr::VROverlayFlags_SendVRSmoothScrollEvents, true);
vr::VRControllerState_t state{};
dragging.trigger_observed = system->GetControllerState(dragging.device, &state, sizeof(state)) &&
(state.ulButtonPressed & vr::ButtonMaskFromId(vr::k_EButton_SteamVR_Trigger));
diagnostics.last_pointer_event = std::string(kind == PanelDragKind::Grab ? "grab" : "scale") +
" device=" + std::to_string(dragging.device) + " trigger-watch=" + (dragging.trigger_observed ? "Y" : "N");
}
void finish_drag(bool released) {
overlay->SetOverlayFlag(handle, vr::VROverlayFlags_SendVRSmoothScrollEvents, false);
if (!released && applied_mount && *applied_mount == mount) {
canvas_pose = dragging.canvas;
size_scale = dragging.scale;
placement_dirty = true;
}
if (released && applied_mount && *applied_mount == mount && mount != Mount::World &&
(size_scale != dragging.scale || canvas_pose != dragging.canvas)) {
const RelativePlacement current{canvas_pose, size_scale};
if (valid_relative_placement(current)) {
saved_placements[static_cast<size_t>(mount)] = current;
persistence.placement_save_failed = persistence.persist_mount &&
!save_relative_placement(default_placement_path(mount), mount, current);
} else persistence.placement_save_failed = true;
}
surface.reset_pointers(); dragging.panel.reset();
}
std::optional<float> depth_axis() const {
const auto left = system->GetTrackedDeviceIndexForControllerRole(vr::TrackedControllerRole_LeftHand);
const auto right = system->GetTrackedDeviceIndexForControllerRole(vr::TrackedControllerRole_RightHand);
const size_t hand = dragging.device == left ? 0 : dragging.device == right ? 1 : depth_actions.size();
if (hand == depth_actions.size()) return {};
vr::InputAnalogActionData_t data{};
if (input->GetAnalogActionData(depth_actions[hand], &data, sizeof(data), hands[hand]) != vr::VRInputError_None ||
!data.bActive || data.activeOrigin == vr::k_ulInvalidInputValueHandle || !std::isfinite(data.y)) return {};
vr::InputOriginInfo_t origin{};
if (input->GetOriginTrackedDeviceInfo(data.activeOrigin, &origin, sizeof(origin)) != vr::VRInputError_None ||
origin.trackedDeviceIndex != dragging.device) return {};
return data.y;
}
void update_drag() {
if (!dragging.panel.active()) return;
const auto source = drag_source();
@@ -362,12 +476,27 @@ struct Overlay::Impl {
// Without a readable release watchdog, never keep manipulating after
// the pointer leaves our hit region: an outside MouseUp is not assured.
const bool lost_unwatched_pointer = !dragging.trigger_observed && !overlay->IsHoverTargetOverlay(handle);
if (!surface.dragging(dragging.cursor) || !shown || !focus || !source || trigger_released || lost_unwatched_pointer ||
std::chrono::steady_clock::now() - dragging.started > std::chrono::seconds(15)) {
surface.reset_pointers(); dragging.panel.reset(); return;
const bool timed_out = std::chrono::steady_clock::now() - dragging.started > std::chrono::seconds(15);
if (!surface.dragging(dragging.cursor) || !shown || !focus || !source || trigger_released || lost_unwatched_pointer || timed_out) {
finish_drag((!surface.dragging(dragging.cursor) || trigger_released) &&
shown && focus && bool(source) && !lost_unwatched_pointer && !timed_out);
return;
}
const auto now = std::chrono::steady_clock::now();
const double dt = std::chrono::duration<double>(now - dragging.last_depth_sample).count();
dragging.last_depth_sample = now;
if (dragging.kind == PanelDragKind::Grab) {
const auto axis = diagnostics.action_update_error == vr::VRInputError_None ? depth_axis() : std::nullopt;
if (performance.enabled) {
++performance.depth_samples;
performance.depth_axis_active += bool(axis);
}
dragging.panel.update_depth(axis, dt);
if (performance.enabled)
performance.max_depth = std::max(performance.max_depth, std::abs(dragging.panel.depth()));
}
const auto change = dragging.panel.update(*source);
if (!change) { surface.reset_pointers(); dragging.panel.reset(); return; }
if (!change) { finish_drag(false); return; }
const float scale = dragging.kind == PanelDragKind::Scale ? std::clamp(dragging.scale * change->factor, .5f, 2.f) : size_scale;
const auto pose = dragging.kind == PanelDragKind::Grab ? change->pose :
resized_mount_pose(dragging.canvas, mount_width(*applied_mount, config.wrist) * dragging.scale * W / CW,
@@ -404,13 +533,26 @@ struct Overlay::Impl {
panel = p;
surface.set_clock_time(std::time(nullptr));
update_indicators();
if (surface.render(p, PanelSurface::Clock::now(), shown)) {
// Placement updates reuse the initialized image. Defer cosmetic
// rendering/uploads (including gradient ticks) until after the drag;
// keep panel and indicator state live for the next render.
const bool initialized_motion = dragging.panel.active() && has_texture;
if (performance.enabled && initialized_motion) ++performance.retained_frames;
refresh_texture(dragging.panel.active(), has_texture,
[&] {
const auto start = performance.enabled ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{};
const bool rendered = surface.render(p, PanelSurface::Clock::now(), shown);
if (performance.enabled && rendered) { if (initialized_motion) ++performance.motion_redraws; ++performance.renders; performance.render_ms += Performance::elapsed(start); }
return rendered;
}, [&] {
const auto start = performance.enabled ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{};
gpu_texture->upload(surface.pixels());
auto texture = gpu_texture->texture();
overlay_check(overlay->SetOverlayTexture(handle, &texture), overlay, "SetOverlayTexture (Vulkan)");
++diagnostics.texture_uploads;
if (performance.enabled) { if (initialized_motion) ++performance.motion_uploads; ++performance.uploads; performance.upload_ms += Performance::elapsed(start); }
has_texture = true;
}
});
visibility();
}
void reset_input(std::vector<UiAction>& result) {
@@ -467,6 +609,21 @@ struct Overlay::Impl {
}
std::vector<UiAction> poll() {
std::vector<UiAction> result;
const auto open_companion = [&](SurfaceEvent::Companion selected) {
const auto& launcher = selected == SurfaceEvent::Companion::Plan ? plan_launcher :
selected == SurfaceEvent::Companion::Draw ? draw_launcher : keyboard_launcher;
const auto command = selected == SurfaceEvent::Companion::KeyboardRecenter ? CompanionCommand::Recenter :
selected == SurfaceEvent::Companion::Keyboard ? CompanionCommand::Show : CompanionCommand::Default;
if (launcher && !launch_companion(*launcher, command))
surface.set_binding_note("Could not open companion app.");
else surface.set_binding_note("");
};
if (auto update = updates.poll()) {
update_version = std::move(update->version);
surface.set_update_status(update->state == UpdateResult::State::Available ? UpdateStatus::Available :
update->state == UpdateResult::State::Current ? UpdateStatus::Current : UpdateStatus::Failed,
update_version);
}
system->GetDeviceToAbsoluteTrackingPose(vr::TrackingUniverseStanding, 0, poses.data(), uint32_t(poses.size()));
place();
vr::VREvent_t event{};
@@ -486,7 +643,7 @@ struct Overlay::Impl {
result.push_back(UiAction::Quit); break;
case vr::VREvent_OverlayHidden:
++diagnostics.overlay_hidden_events;
focus = false; reset_input(result);
focus = false; cursor_positions.fill(std::nullopt); reset_input(result);
if (panel.recording) result.push_back(UiAction::Cancel);
break;
case vr::VREvent_OverlayShown:
@@ -495,6 +652,7 @@ struct Overlay::Impl {
case vr::VREvent_ImageLoaded: ++diagnostics.image_loaded_events; break;
case vr::VREvent_ImageFailed: ++diagnostics.image_failed_events; break;
case vr::VREvent_OverlayGamepadFocusLost:
cursor_positions.fill(std::nullopt);
surface.reset_pointers(); ++diagnostics.pointer_resets;
diagnostics.last_pointer_event = "gamepad focus lost"; break;
case vr::VREvent_OverlayFocusChanged:
@@ -504,10 +662,34 @@ struct Overlay::Impl {
// a release must still hit the same enabled control.
diagnostics.last_pointer_event = "overlay focus changed"; break;
case vr::VREvent_MouseMove:
// Manipulation uses the captured controller ray, not coordinates
// fed back from a changing overlay or a batch of stale mouse hits.
// Hover drives scrolling and hold cancellation. Manipulation uses
// the captured controller ray, not changing overlay coordinates.
if (event.data.mouse.cursorIndex < cursor_positions.size()) {
cursor_positions[event.data.mouse.cursorIndex] = {event.data.mouse.x, H - event.data.mouse.y};
surface.pointer_move(event.data.mouse.cursorIndex, event.data.mouse.x, H - event.data.mouse.y);
}
break;
case vr::VREvent_ScrollDiscrete: case vr::VREvent_ScrollSmooth:
if (dragging.panel.active()) {
if (event.eventType == vr::VREvent_ScrollSmooth && dragging.kind == PanelDragKind::Grab) {
const bool accepted = dragging.panel.queue_scroll_depth(event.trackedDeviceIndex,
dragging.device, event.data.scroll.ydelta);
if (performance.enabled) {
++performance.depth_scroll_events;
performance.depth_scroll_accepted += accepted;
}
}
break; // depth consumes the grab input; never scroll the UI
}
if (!dragging.panel.active() && event.data.scroll.cursorIndex < cursor_positions.size() &&
cursor_positions[event.data.scroll.cursorIndex]) {
const auto [x, y] = *cursor_positions[event.data.scroll.cursorIndex];
surface.scroll(x, y, event.data.scroll.ydelta);
}
break;
case vr::VREvent_MouseButtonDown:
if (event.data.mouse.cursorIndex < cursor_positions.size())
cursor_positions[event.data.mouse.cursorIndex] = {event.data.mouse.x, H - event.data.mouse.y};
++diagnostics.pointer_downs;
diagnostics.last_pointer_event = "down button=" + std::to_string(event.data.mouse.button);
if (event.data.mouse.button == vr::VRMouseButton_Left)
@@ -515,12 +697,22 @@ struct Overlay::Impl {
begin_drag(*kind, event);
break;
case vr::VREvent_MouseButtonUp:
if (event.data.mouse.cursorIndex < cursor_positions.size())
cursor_positions[event.data.mouse.cursorIndex] = {event.data.mouse.x, H - event.data.mouse.y};
++diagnostics.pointer_ups;
diagnostics.last_pointer_event = "up button=" + std::to_string(event.data.mouse.button);
if (event.data.mouse.button == vr::VRMouseButton_Left) {
auto event_result = surface.pointer_up(event.data.mouse.cursorIndex, event.data.mouse.x, H - event.data.mouse.y);
if (event_result.action || event_result.mount || event_result.recenter || event_result.lasers_anytime || event_result.open_bindings || event_result.advanced_debug || event_result.auto_insert || event_result.close_mic_when_idle || event_result.lock_layout || event_result.clock_24h || event_result.date_format || event_result.model_action) ++diagnostics.pointer_actions;
if (event_result.action || event_result.mount || event_result.recenter || event_result.lasers_anytime || event_result.open_bindings || event_result.launch_companion || event_result.check_updates || event_result.install_update || event_result.advanced_debug || event_result.auto_insert || event_result.gradient_enabled || event_result.close_mic_when_idle || event_result.lock_layout || event_result.wrist_world_fallback || event_result.clock_24h || event_result.date_format || event_result.model_action) ++diagnostics.pointer_actions;
if (event_result.action) result.push_back(*event_result.action);
if (event_result.check_updates) {
update_version.clear();
surface.set_update_status(updates.start() ? UpdateStatus::Checking : UpdateStatus::Failed);
}
if (event_result.install_update && !update_version.empty() &&
!open_update_terminal(install_update_script, update_version))
surface.set_update_status(UpdateStatus::TerminalFailed);
if (event_result.launch_companion) open_companion(*event_result.launch_companion);
if (event_result.model_action) {
model_actions.push_back(*event_result.model_action);
reset_input(result); // revoke held PTT, delivery and stale pointer approval
@@ -548,6 +740,15 @@ struct Overlay::Impl {
reset_input(result); dragging.panel.reset();
return result;
}
if (event_result.wrist_world_fallback) {
config.wrist_world_fallback = *event_result.wrist_world_fallback;
persistence.save_failed = persistence.persist_mount &&
!save_wrist_world_fallback(default_config_path(), config.wrist_world_fallback);
surface.set_wrist_world_fallback(config.wrist_world_fallback);
reset_input(result);
place(); // hide or show immediately if the wrist is currently untracked
return result;
}
if (event_result.close_mic_when_idle) {
config.close_mic_when_idle = *event_result.close_mic_when_idle;
persistence.mic_save_failed = persistence.persist_mount &&
@@ -556,6 +757,14 @@ struct Overlay::Impl {
reset_input(result); // a setting change invalidates held actions
return result;
}
if (event_result.gradient_enabled) {
config.gradient.enabled = *event_result.gradient_enabled;
persistence.gradient_save_failed = persistence.persist_mount &&
!save_gradient_enabled(default_config_path(), config.gradient.enabled);
surface.set_gradient_enabled(config.gradient.enabled);
reset_input(result);
return result;
}
if (event_result.auto_insert) {
config.auto_insert = *event_result.auto_insert;
persistence.auto_save_failed = persistence.persist_mount && !save_auto_insert(default_config_path(), config.auto_insert);
@@ -605,8 +814,49 @@ struct Overlay::Impl {
default: break;
}
}
vr::VRActiveActionSet_t set{};
set.ulActionSet = action_set;
set.ulRestrictedToDevice = vr::k_ulInvalidInputValueHandle;
// This is a request, not proof of delivery through the dashboard/game.
set.nPriority = action_priority();
std::array<vr::VRActiveActionSet_t, 2> sets{set, {}};
uint32_t count = 1;
if (dragging.panel.active() && dragging.kind == PanelDragKind::Grab && tracked(dragging.device)) {
const auto role = system->GetControllerRoleForTrackedDeviceIndex(dragging.device);
const auto hand = role == vr::TrackedControllerRole_LeftHand ? hands[0] :
role == vr::TrackedControllerRole_RightHand ? hands[1] : vr::k_ulInvalidInputValueHandle;
if (hand != vr::k_ulInvalidInputValueHandle) {
// Depth priority must never promote recording/typing bindings.
sets[1].ulActionSet = grab_action_set;
sets[1].ulRestrictedToDevice = hand;
sets[1].nPriority = vr::k_nActionSetOverlayGlobalPriorityMin;
count = 2;
}
}
diagnostics.action_update_error = input->UpdateActionState(sets.data(), sizeof(sets[0]), count);
const bool measuring_motion = performance.enabled && dragging.panel.active();
const auto motion_start = measuring_motion ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{};
// Event handlers can launch a companion or update a setting. Refresh
// tracking immediately before motion submission after that work.
if (dragging.panel.active())
system->GetDeviceToAbsoluteTrackingPose(vr::TrackingUniverseStanding, 0, poses.data(), uint32_t(poses.size()));
update_drag();
place();
if (measuring_motion) {
++performance.motion_ticks;
const auto elapsed = Performance::elapsed(motion_start);
performance.motion_ms += elapsed;
performance.motion_max_ms = std::max(performance.motion_max_ms, elapsed);
}
const auto& hmd = poses[vr::k_unTrackedDeviceIndex_Hmd];
if (!shown || !focus || !placed || !hmd.bPoseIsValid || !hmd.bDeviceIsConnected ||
!overlay->IsHoverTargetOverlay(handle)) {
// Tracking/visibility/hover loss revokes the hold even if OpenVR never sends MouseUp.
surface.reset_pointers();
} else if (auto command = surface.poll_keyboard_hold()) {
++diagnostics.pointer_actions;
open_companion(*command);
}
if (dragging.panel.active()) {
// A controller used to manipulate the panel must not also authorize
// Record/Insert/Enter. Require neutral rearm after the drag ends.
@@ -616,14 +866,6 @@ struct Overlay::Impl {
for (auto& edge : edges) edge.reset();
return result;
}
vr::VRActiveActionSet_t set{};
set.ulActionSet = action_set;
set.ulRestrictedToDevice = vr::k_ulInvalidInputValueHandle;
// Explicit opt-in affects only sources bound to our existing action set.
// Keep requesting it across dashboard/laser states: those are what the
// experiment compares. SteamVR's separate permission gate is never changed here.
set.nPriority = action_priority();
diagnostics.action_update_error = input->UpdateActionState(&set, sizeof(set), 1);
if (diagnostics.action_update_error != vr::VRInputError_None) {
reset_input(result); return result;
}
+35 -4
View File
@@ -9,6 +9,9 @@ PacedDelivery::PacedDelivery(DeliveryFactory acquire, Clock clock,
: acquire_(std::move(acquire)), clock_(std::move(clock)),
release_idle_(std::move(release_idle)), permitted_(std::move(permitted)) {}
void PacedDelivery::trace(const std::string& line) const {
if (trace_) { try { trace_(line); } catch (...) {} }
}
bool PacedDelivery::valid_guard() const {
try { return (!permitted_ || permitted_()) && task_->guard && task_->guard(); }
catch (...) { return false; }
@@ -32,15 +35,25 @@ void PacedDelivery::start(std::string text, bool enter, Guard guard,
candidate->first_lease = acquire_();
if (!candidate->first_lease || (permitted_ && !permitted_()) || !candidate->guard())
throw std::runtime_error("Input focus changed during acquisition");
} catch (const std::exception& e) {
trace(std::string("start rejected: ") + e.what());
std::fill(candidate->text.begin(), candidate->text.end(), '\0');
throw;
} catch (...) {
trace("start rejected: unknown");
std::fill(candidate->text.begin(), candidate->text.end(), '\0');
throw;
}
trace("start bytes=" + std::to_string(candidate->text.size()) +
" enter=" + (candidate->enter ? "1" : "0"));
retained_ = true;
task_ = std::move(candidate);
}
PacedDelivery::Outcome PacedDelivery::finish(DeliveryResult result, bool focus_lost) {
Outcome out{result, task_->began, focus_lost};
trace("finish result=" + std::to_string(static_cast<int>(result)) + " began=" +
(task_->began ? "1" : "0") + " focus_lost=" + (focus_lost ? "1" : "0") +
" offset=" + std::to_string(task_->offset) + "/" + std::to_string(task_->text.size()));
std::fill(task_->text.begin(), task_->text.end(), '\0');
task_.reset();
return out;
@@ -59,15 +72,25 @@ std::optional<PacedDelivery::Outcome> PacedDelivery::tick() {
auto& task = *task_;
// A focus loss before ANY send leaves the review unconsumed. After the
// first possible send, the remainder is discarded, not retargeted/retried.
if (!valid_guard()) return finish(task.began ? DeliveryResult::TextUncertain : DeliveryResult::Ignored, true);
if (!valid_guard()) {
trace("guard failed before acquire");
return finish(task.began ? DeliveryResult::TextUncertain : DeliveryResult::Ignored, true);
}
std::unique_ptr<DeliveryLease> lease = std::move(task.first_lease);
if (!lease) {
try { lease = acquire_(); }
catch (...) { return finish(task.began ? (task.offset == task.text.size() ?
catch (const std::exception& e) {
trace(std::string("acquire failed: ") + e.what());
return finish(task.began ? (task.offset == task.text.size() ?
DeliveryResult::TextQueuedEnterUnavailable : DeliveryResult::TextUncertain) : DeliveryResult::Ignored); }
catch (...) { trace("acquire failed: unknown"); return finish(task.began ? (task.offset == task.text.size() ?
DeliveryResult::TextQueuedEnterUnavailable : DeliveryResult::TextUncertain) : DeliveryResult::Ignored); }
if (!lease) return finish(task.began ? DeliveryResult::TextUncertain : DeliveryResult::Ignored);
}
if (!valid_guard()) return finish(task.began ? DeliveryResult::TextUncertain : DeliveryResult::Ignored, true);
if (!valid_guard()) {
trace("guard failed after acquire");
return finish(task.began ? DeliveryResult::TextUncertain : DeliveryResult::Ignored, true);
}
if (!task.began) {
// A callback consumes review immediately before a possibly ambiguous send.
try { if (task.on_first_send) task.on_first_send(); }
@@ -87,17 +110,25 @@ std::optional<PacedDelivery::Outcome> PacedDelivery::tick() {
try {
chunk = task.text.substr(task.offset, end - task.offset);
lease->text(chunk);
} catch (const std::exception& e) {
trace("text send failed at offset=" + std::to_string(task.offset) + ": " + e.what());
last_commit_ = clock_();
std::fill(chunk.begin(), chunk.end(), '\0');
return finish(DeliveryResult::TextUncertain);
} catch (...) {
trace("text send failed at offset=" + std::to_string(task.offset) + ": unknown");
last_commit_ = clock_();
std::fill(chunk.begin(), chunk.end(), '\0');
return finish(DeliveryResult::TextUncertain);
}
last_commit_ = clock_();
trace("sent offset=" + std::to_string(task.offset) + " bytes=" + std::to_string(end - task.offset));
std::fill(chunk.begin(), chunk.end(), '\0');
task.offset = end;
if (end == task.text.size() && !task.enter) return finish(DeliveryResult::TextQueued);
} else {
try { lease->enter(); }
try { lease->enter(); trace("sent enter"); }
catch (const std::exception& e) { trace(std::string("enter failed: ") + e.what()); last_commit_ = clock_(); return finish(DeliveryResult::EnterUncertain); }
catch (...) { last_commit_ = clock_(); return finish(DeliveryResult::EnterUncertain); }
last_commit_ = clock_();
return finish(DeliveryResult::EnterQueued);
+5
View File
@@ -13,6 +13,8 @@ class PacedDelivery {
public:
using Clock = std::function<std::chrono::steady_clock::time_point()>;
using Guard = std::function<bool()>;
// Opt-in diagnostics: metadata only (sizes, offsets, reasons), never text.
using Trace = std::function<void(const std::string&)>;
struct Outcome {
DeliveryResult result;
bool began; // first send may have reached the compositor
@@ -32,6 +34,7 @@ public:
bool active() const { return bool(task_); }
bool began() const { return task_ && task_->began; }
void cancel(); // drops and scrubs the remaining literal; never sends Enter
void set_trace(Trace trace) { trace_ = std::move(trace); }
private:
struct Task {
std::string text;
@@ -44,10 +47,12 @@ private:
bool valid_guard() const;
Outcome finish(DeliveryResult result, bool focus_lost = false);
void release_if_idle();
void trace(const std::string& line) const;
DeliveryFactory acquire_;
Clock clock_;
std::function<void()> release_idle_;
Guard permitted_;
Trace trace_;
std::unique_ptr<Task> task_;
std::optional<std::chrono::steady_clock::time_point> last_commit_;
bool retained_ = false;
+40 -2
View File
@@ -2,7 +2,11 @@
#include "mount.hpp"
#include <algorithm>
#include <cmath>
#include <cstdint>
#include <limits>
#include <utility>
#include <optional>
namespace frameyap {
@@ -128,11 +132,44 @@ public:
return true;
}
// The stick is velocity, not absolute placement. Positive Y moves along
// the captured panel's -Z normal in controller coordinates, even for an angled ray.
void step_depth(float axis_y, double seconds) {
if (!active_ || kind_ != PanelDragKind::Grab || !std::isfinite(axis_y) ||
!std::isfinite(seconds) || seconds <= 0) return;
const double magnitude = std::abs(std::clamp(double(axis_y), -1., 1.));
if (magnitude <= .2) return;
const double velocity = std::copysign((magnitude - .2) / .8 * .6, axis_y);
depth_ = std::clamp(depth_ + velocity * std::min(seconds, .05), -1.5, 1.5);
}
// Compositor smooth scroll is already integrated over time. Accept only
// the captured controller, cap each event at 3 cm, and consume it once.
bool queue_scroll_depth(uint32_t source, uint32_t grabbing_device, float delta) {
if (!active_ || kind_ != PanelDragKind::Grab ||
source == std::numeric_limits<uint32_t>::max() || source != grabbing_device ||
!std::isfinite(delta)) return false;
pending_scroll_ = std::clamp(pending_scroll_ + std::clamp(double(delta) * .04, -.03, .03), -1.5, 1.5);
return true;
}
void update_depth(std::optional<float> axis, double seconds) {
const double scroll = std::exchange(pending_scroll_, 0.0);
if (!active_ || kind_ != PanelDragKind::Grab) return;
if (axis && std::isfinite(*axis)) step_depth(*axis, seconds);
else depth_ = std::clamp(depth_ + scroll, -1.5, 1.5);
}
double depth() const { return depth_; }
std::optional<PanelDragUpdate> update(const Matrix34& controller_pose) const {
using namespace panel_drag_detail;
if (!active_ || !rigid(controller_pose)) return std::nullopt;
if (kind_ == PanelDragKind::Grab) {
const Matrix34 pose = compose_pose(controller_pose, relative_panel_);
auto relative = relative_panel_;
for (int row = 0; row < 3; ++row)
relative[row][3] -= relative_panel_[row][2] * static_cast<float>(depth_);
const Matrix34 pose = compose_pose(controller_pose, relative);
if (!rigid(pose)) return std::nullopt;
return PanelDragUpdate{pose, 1.f};
}
@@ -150,7 +187,7 @@ public:
return PanelDragUpdate{panel_pose_, static_cast<float>(factor)};
}
void reset() { active_ = false; }
void reset() { active_ = false; depth_ = pending_scroll_ = 0; }
bool active() const { return active_; }
private:
@@ -160,6 +197,7 @@ private:
Matrix34 panel_pose_{}, relative_panel_{};
Vec3 center_{}, normal_{}, anchor_{}, down_hit_{}, local_ray_{};
double baseline_sq_ = 0;
double depth_ = 0, pending_scroll_ = 0;
};
} // namespace frameyap
+259 -51
View File
@@ -28,19 +28,20 @@ struct Rect {
rounded_distance(px, py, x, y, w, h, std::min(16, h / 3)) <= 0.f;
}
};
enum class Control { Review, Settings, Bindings, Prev, Next, Record, Cancel, Insert, Enter, Quit,
enum class Control { Review, Settings, Bindings, Plan, Keyboard, Record, Cancel, Insert, Enter, Quit,
World, Left, Right, Head, Recenter, LasersAnytime, AdvancedDebug, AutoInsert,
Clock24h, Date, LockLayout, CloseMicWhenIdle, Models, ModelRow,
ModelInstall, ModelPrev, ModelNext };
enum class Tab { Review, Settings, Models };
ModelInstall, ModelPrev, ModelNext, About, WristWorldFallback, UpdateCheck, UpdateInstall, Draw,
AnimatedBackground };
enum class Tab { Review, Settings, Models, About };
struct Button { Rect r; Control id; const char* label; };
constexpr std::array<Button, 32> buttons{{
constexpr std::array<Button, 38> buttons{{
{{32, 138, 180, 46}, Control::Review, "Review"},
{{226, 138, 180, 46}, Control::Settings, "Settings"},
{{420, 138, 180, 46}, Control::Bindings, "Bindings"},
{{620, 138, 348, 46}, Control::LockLayout, "Lock grab/scale"},
{{32, 496, 154, 44}, Control::Prev, "Previous"},
{{838, 496, 130, 44}, Control::Next, "Next"},
{{32, 496, 454, 44}, Control::Plan, "Open Plan"},
{{514, 496, 454, 44}, Control::Keyboard, "Open Keyboard"},
{{32, 574, 176, 68}, Control::Record, "Record"},
{{222, 574, 176, 68}, Control::Cancel, "Cancel"},
{{412, 574, 176, 68}, Control::Insert, "Type"},
@@ -67,7 +68,18 @@ constexpr std::array<Button, 32> buttons{{
{{514, 530, 454, 36}, Control::ModelInstall, "Install"},
{{32, 530, 214, 36}, Control::ModelPrev, "Previous"},
{{260, 530, 214, 36}, Control::ModelNext, "Next"},
{{346, 378, 154, 44}, Control::About, "About"},
{{32, 626, 936, 50}, Control::WristWorldFallback, "Wrist world fallback"},
{{32, 682, 454, 42}, Control::UpdateCheck, "Check for updates"},
{{514, 682, 454, 42}, Control::UpdateInstall, "Install update..."},
{{32, 496, 454, 44}, Control::Draw, "Open Draw"},
{{32, 574, 936, 36}, Control::AnimatedBackground, "Animated background"},
}};
constexpr Rect settings_view{32, 232, 936, 316};
constexpr Rect review_view{48, 212, 904, 260};
constexpr int review_line_height = 42;
// Leave a status row after the optional update controls, within the viewport.
constexpr int settings_max_scroll = 764 - (settings_view.y + settings_view.h);
std::optional<UiAction> action(Control c) {
switch (c) {
case Control::Record: return UiAction::Record;
@@ -87,6 +99,13 @@ std::optional<Mount> mounting(Control c) {
default: return {};
}
}
bool settings_control(Control c) {
return c == Control::Models || c == Control::About || mounting(c) ||
c == Control::Recenter || c == Control::LasersAnytime || c == Control::AdvancedDebug ||
c == Control::AutoInsert || c == Control::Clock24h || c == Control::Date ||
c == Control::CloseMicWhenIdle || c == Control::AnimatedBackground || c == Control::WristWorldFallback ||
c == Control::UpdateCheck || c == Control::UpdateInstall;
}
// Invalid bytes become visible replacement glyphs, never control commands.
uint32_t next_codepoint(std::string_view s, size_t& i) {
const auto a = static_cast<unsigned char>(s[i++]);
@@ -121,26 +140,31 @@ struct PanelSurface::Impl {
Color background, card, ink, muted, cyan, pink;
Tab tab = Tab::Review;
bool dirty = true, lasers_anytime = false, advanced_debug = false, auto_insert = false;
bool close_mic_when_idle = false;
bool close_mic_when_idle = false, wrist_world_fallback = true;
bool clock_24h = false, layout_locked = false;
float settings_scroll = 0.f;
std::optional<Rect> clip_view;
bool plan_present = false, keyboard_present = false, draw_present = false;
UpdateStatus update_status = UpdateStatus::Idle;
std::string update_version;
float review_scroll = 0.f;
DateFormat date_format = DateFormat::MonthDayYear;
std::time_t clock_time = std::time(nullptr);
ClockLabel displayed_clock;
std::string placement_note, binding_note;
std::string placement_note, binding_note, version;
StatusIndicators indicators;
std::array<int, 2> pressed{{-1, -1}};
std::array<PanelSurface::Clock::time_point, 2> press_time{};
int hold_progress = 0;
std::array<bool, 2> keyboard_fired{};
int hold_progress = 0, keyboard_progress = 0;
int drag_cursor = -1;
std::vector<std::string> lines;
size_t page = 0;
size_t model_page = 0;
size_t consent_page = 0;
size_t consent_rendered_page = size_t(-1);
bool install_confirm = false;
std::optional<ModelOption> consent_snapshot;
std::vector<std::string> consent_lines;
static constexpr size_t lines_per_page = 6;
static constexpr size_t consent_lines_per_page = 6;
size_t consent_pages() const {
return std::max(size_t(1), (consent_lines.size() + consent_lines_per_page - 1) / consent_lines_per_page);
@@ -191,11 +215,15 @@ struct PanelSurface::Impl {
return result;
}
void rect(Rect r, Color c) {
for (int y = std::max(r.y, 0); y < std::min(H, r.y + r.h); ++y)
for (int x = std::max(r.x, 0); x < std::min(W, r.x + r.w); ++x)
for (int y = std::max(r.y, clip_view ? clip_view->y : 0);
y < std::min(clip_view ? clip_view->y + clip_view->h : H, r.y + r.h); ++y)
for (int x = std::max(r.x, clip_view ? clip_view->x : 0);
x < std::min(clip_view ? clip_view->x + clip_view->w : W, r.x + r.w); ++x)
std::copy(c.begin(), c.end(), pixels.begin() + (size_t(y) * W + x) * 4);
}
void blend(int x, int y, Color color, float amount) {
if (clip_view && (x < clip_view->x || x >= clip_view->x + clip_view->w ||
y < clip_view->y || y >= clip_view->y + clip_view->h)) return;
auto* dst = pixels.data() + (size_t(y) * W + x) * 4;
// Straight-alpha source-over, including strokes in the transparent margin.
const float alpha = amount * color[3] / 255.f;
@@ -333,7 +361,9 @@ struct PanelSurface::Impl {
for (unsigned row = 0; row < b.rows; ++row)
for (unsigned col = 0; col < b.width; ++col) {
int xx = x + glyph->bitmap_left + int(col), yy = baseline - glyph->bitmap_top + int(row);
if (xx < 0 || xx >= right || xx >= W || yy < 0 || yy >= H) continue;
if (xx < 0 || xx >= right || xx >= W || yy < 0 || yy >= H ||
(clip_view && (xx < clip_view->x || xx >= clip_view->x + clip_view->w ||
yy < clip_view->y || yy >= clip_view->y + clip_view->h))) continue;
unsigned char alpha = b.buffer[int(row) * b.pitch + int(col)];
auto* dst = pixels.data() + (size_t(yy) * W + xx) * 4;
for (int k = 0; k < 3; ++k)
@@ -342,7 +372,9 @@ struct PanelSurface::Impl {
x += a;
}
}
size_t page_count() const { return std::max(size_t(1), (lines.size() + lines_per_page - 1) / lines_per_page); }
float max_review_scroll() const {
return float(lines.size() > 6 ? (lines.size() - 6) * review_line_height : 0);
}
bool available(UiAction a) const {
if (a == UiAction::Record) return tab != Tab::Models && !panel.quick_open && (panel.recording || panel.record_available);
// With nothing to review, Type is an explicit Enter.
@@ -354,21 +386,24 @@ struct PanelSurface::Impl {
}
bool visible(Control c) const {
if (panel.quick_open && c != Control::Cancel && c != Control::Enter && c != Control::Quit) return false;
if (c == Control::Models) return tab == Tab::Settings;
if (c == Control::Models || c == Control::About || c == Control::WristWorldFallback ||
c == Control::UpdateCheck) return tab == Tab::Settings;
if (c == Control::UpdateInstall) return tab == Tab::Settings && update_status == UpdateStatus::Available;
if (c == Control::ModelRow) return tab == Tab::Models && !install_confirm;
if (c == Control::ModelPrev || c == Control::ModelNext) return tab == Tab::Models;
if (c == Control::ModelInstall) return tab == Tab::Models;
if (c == Control::Prev || c == Control::Next) return tab == Tab::Review && !panel.quick_open;
if (c == Control::Plan) return tab == Tab::Review && plan_present;
if (c == Control::Keyboard) return tab == Tab::Review && keyboard_present;
if (c == Control::Draw) return tab == Tab::Review && draw_present;
if (mounting(c) || c == Control::Recenter || c == Control::LasersAnytime ||
c == Control::AdvancedDebug || c == Control::AutoInsert ||
c == Control::Clock24h || c == Control::Date || c == Control::LockLayout ||
c == Control::CloseMicWhenIdle) return tab == Tab::Settings;
c == Control::CloseMicWhenIdle || c == Control::AnimatedBackground) return tab == Tab::Settings;
return true;
}
bool enabled(Control c) const {
if (auto a = action(c)) return available(*a);
if (c == Control::Prev) return page > 0;
if (c == Control::Next) return page + 1 < page_count();
if (c == Control::UpdateCheck) return update_status != UpdateStatus::Checking;
if (c == Control::Recenter) return mount == Mount::World;
if (c == Control::ModelRow) return !panel.model_busy;
if (c == Control::ModelInstall) {
@@ -384,17 +419,58 @@ struct PanelSurface::Impl {
(model_page + 1) * 6 < panel.models.size();
return true;
}
Rect button_rect(const Button& b) const {
auto r = b.r;
if (b.id == Control::Plan || b.id == Control::Keyboard || b.id == Control::Draw) {
// Only installed companions occupy the review row. Keep the full
// hit region and label together as the row grows from one to three.
const int count = int(plan_present) + int(keyboard_present) + int(draw_present);
const int index = (b.id == Control::Keyboard ? int(plan_present) : 0) +
(b.id == Control::Draw ? int(plan_present) + int(keyboard_present) : 0);
if (count) {
constexpr int gap = 12, available_width = 936;
const int width = (available_width - gap * (count - 1)) / count;
r.x = 32 + index * (width + gap);
r.w = index == count - 1 ? 968 - r.x : width;
}
}
if (settings_control(b.id)) r.y -= int(std::lround(settings_scroll));
return r;
}
int hit(float x, float y) const {
for (size_t i = 0; i < buttons.size(); ++i) {
if (buttons[i].id == Control::ModelRow && model_page * 6 + (i - 23) >= panel.models.size()) continue;
if (visible(buttons[i].id) && enabled(buttons[i].id) && buttons[i].r.contains(x, y)) return int(i);
if (settings_control(buttons[i].id) && !settings_view.contains(x, y)) continue;
if (visible(buttons[i].id) && enabled(buttons[i].id) && button_rect(buttons[i]).contains(x, y)) return int(i);
}
return -1;
}
void reset() {
pressed.fill(-1);
keyboard_fired.fill(false);
drag_cursor = -1;
}
// Static facts only: nothing here is a live check of the model or device.
void about() {
rounded({32, 204, 936, 334}, 16, mix(card, muted, .08f), mix(card, cyan, .25f), .18f);
text("FrameYap", 48, 244, 30, ink, 952);
text("Version " + (version.empty() ? std::string("unknown") : version), 48, 276, 22, cyan, 952);
text("On-device voice typing for Steam Frame. Nothing is sent to the cloud.", 48, 306, 19, muted, 952);
const auto it = std::find_if(panel.models.begin(), panel.models.end(), [&](const auto& m) { return m.id == panel.selected_backend; });
const std::array<std::pair<const char*, std::string>, 5> rows{{
{"Source", "github.com/baketnk/frame-yap"},
{"License", "MIT (FrameYap contributors)"},
{"Model", it == panel.models.end() ? "none selected" :
it->license.empty() ? it->name : it->name + " (" + it->license + ")"},
{"Font", "Inconsolata (SIL OFL 1.1)"},
{"Notices", "docs/third-party.md in the source"}}};
for (size_t i = 0; i < rows.size(); ++i) {
const int y = 352 + int(i) * 36;
text(rows[i].first, 48, y, 22, cyan, 190);
text(rows[i].second, 200, y, 22, ink, 952);
}
text("Press Settings to go back.", 48, 524, 17, muted, 952);
}
bool render(const Panel& p, Clock::time_point now, bool animate) {
if (!animation_start) animation_start = now;
if (gradient.enabled && animate && now - last_animation_frame >= std::chrono::milliseconds(100))
@@ -406,7 +482,7 @@ struct PanelSurface::Impl {
}
if (p.transcript != panel.transcript || lines.empty()) {
lines = wrap(p.transcript.empty() ? "Your words will appear here.\nReview them, then choose Type." : p.transcript, 32, 904);
page = 0; dirty = true;
review_scroll = 0.f; dirty = true;
}
if (p.status != panel.status || p.enabled != panel.enabled || p.recording != panel.recording ||
p.quick_open != panel.quick_open || p.quick_selected != panel.quick_selected || p.quick_inputs != panel.quick_inputs ||
@@ -419,12 +495,17 @@ struct PanelSurface::Impl {
}
panel = p;
if (panel.quick_open && tab != Tab::Review) { tab = Tab::Review; dirty = true; }
int progress = 0;
for (size_t cursor = 0; cursor < pressed.size(); ++cursor)
int progress = 0, keyboard = 0;
for (size_t cursor = 0; cursor < pressed.size(); ++cursor) {
const auto elapsed = std::chrono::duration_cast<std::chrono::milliseconds>(now - press_time[cursor]).count();
if (pressed[cursor] == 10)
progress = std::max(progress, std::clamp(int(std::chrono::duration_cast<std::chrono::milliseconds>(
PanelSurface::Clock::now() - press_time[cursor]).count() / 100) + 1, 1, 10));
if (progress != hold_progress) { hold_progress = progress; dirty = true; }
progress = std::max(progress, std::clamp(int(elapsed / 100) + 1, 1, 10));
if (pressed[cursor] >= 0 && buttons[pressed[cursor]].id == Control::Keyboard && !keyboard_fired[cursor])
keyboard = std::max(keyboard, std::clamp(int(elapsed / 100) + 1, 1, 8));
}
if (progress != hold_progress || keyboard != keyboard_progress) {
hold_progress = progress; keyboard_progress = keyboard; dirty = true;
}
auto label = panel_clock(clock_time, clock_24h, date_format);
if (label.time != displayed_clock.time || label.date != displayed_clock.date) dirty = true;
if (!dirty) return false;
@@ -478,12 +559,19 @@ struct PanelSurface::Impl {
}
} else if (tab == Tab::Review) {
rounded({32, 204, 936, 278}, 16, mix(card, muted, .08f), mix(card, cyan, .25f), .18f);
for (size_t i = 0; i < lines_per_page && page * lines_per_page + i < lines.size(); ++i)
text(lines[page * lines_per_page + i], 48, 240 + int(i) * 42, 32,
clip_view = review_view;
const size_t first = size_t(review_scroll / review_line_height);
for (size_t i = first; i < lines.size() && i <= first + 7; ++i)
text(lines[i], 48, 240 + int(i * review_line_height - review_scroll), 32,
panel.transcript.empty() ? muted : ink, 952);
// A Bindings failure is the only note here; it replaces the page count.
if (!binding_note.empty()) text(binding_note, 200, 526, 18, pink, 824);
else text("PAGE " + std::to_string(page + 1) + " / " + std::to_string(page_count()), 416, 526, 23, muted, 790);
clip_view.reset();
if (max_review_scroll() > 0) {
rect({956, review_view.y, 3, review_view.h}, mix(card, muted, .24f));
const int thumb = std::max(24, int(review_view.h * review_view.h /
(review_view.h + max_review_scroll())));
rect({955, review_view.y + int(review_scroll * (review_view.h - thumb) / max_review_scroll()),
5, thumb}, cyan);
}
} else if (tab == Tab::Models) {
text("Models: select to restart. Install always needs explicit confirmation.", 32, 202, 19, muted, 968);
const auto it = std::find_if(panel.models.begin(), panel.models.end(), [&](const auto& m) { return m.id == panel.selected_backend; });
@@ -502,6 +590,8 @@ struct PanelSurface::Impl {
text("Select Install to review full source, size and license before download.", 32, 225, 18, cyan, 968);
}
} else text("No model selected.", 32, 225, 18, pink, 968);
} else if (tab == Tab::About) {
about();
} else {
text("Hold Quit: hold 0.9s then release. Lasers anytime: system-wide; may affect games.",
32, 204, 18, pink, 968);
@@ -509,17 +599,25 @@ struct PanelSurface::Impl {
32, 225, 17, muted, 968);
}
rect({32, 550, 936, 1}, mix(card, cyan, .17f));
if (tab == Tab::Review && !panel.quick_open) {
if (keyboard_present) text("Hold Open Keyboard 0.8s: recenter",
max_review_scroll() > 0 ? 450 : 32, 568, 19, muted, 968);
if (max_review_scroll() > 0) text("Right stick: scroll review", 32, 568, 16, muted, 400);
}
for (size_t i = 0; i < buttons.size(); ++i) {
const auto& b = buttons[i];
if (!visible(b.id)) continue;
bool on = enabled(b.id);
if (b.id == Control::ModelRow && model_page * 6 + (i - 23) >= panel.models.size()) continue;
const Rect r = button_rect(b);
if (settings_control(b.id)) clip_view = settings_view;
bool on = enabled(b.id);
bool selected = (b.id == Control::Review && tab == Tab::Review) ||
(b.id == Control::Settings && tab == Tab::Settings) ||
(mounting(b.id) && *mounting(b.id) == mount) ||
(b.id == Control::LasersAnytime && lasers_anytime) ||
(b.id == Control::AdvancedDebug && advanced_debug) ||
(b.id == Control::AutoInsert && auto_insert) ||
(b.id == Control::AnimatedBackground && gradient.enabled) ||
(b.id == Control::CloseMicWhenIdle && close_mic_when_idle) ||
(b.id == Control::LockLayout && layout_locked);
const Color fill = !on ? mix(background, card, .40f) :
@@ -527,14 +625,17 @@ struct PanelSurface::Impl {
const Color accent = b.id == Control::Record && panel.recording ? pink : cyan;
const bool highlighted = on && (selected || b.id == Control::Record ||
(b.id == Control::Insert && panel.transcript.size()));
rounded(b.r, std::min(16, b.r.h / 3), fill,
rounded(r, std::min(16, r.h / 3), fill,
!on ? mix(card, muted, .13f) : highlighted ? accent : mix(card, muted, .38f),
highlighted ? .23f : 0.f, highlighted ? 2 : 1);
if (b.id == Control::Quit && (pressed[0] == int(i) || pressed[1] == int(i))) {
if ((b.id == Control::Quit || b.id == Control::Keyboard) &&
(pressed[0] == int(i) || pressed[1] == int(i))) {
const int cursor = pressed[0] == int(i) ? 0 : 1;
const auto duration = b.id == Control::Quit ? PanelSurface::quit_hold : PanelSurface::keyboard_hold;
const float fraction = std::clamp(float(std::chrono::duration_cast<std::chrono::milliseconds>(
PanelSurface::Clock::now() - press_time[cursor]).count()) / PanelSurface::quit_hold.count(), 0.f, 1.f);
rect({b.r.x + 7, b.r.y + b.r.h - 9, int((b.r.w - 14) * fraction), 3}, pink);
now - press_time[cursor]).count()) / duration.count(), 0.f, 1.f);
rect({r.x + 7, r.y + r.h - 9, int((r.w - 14) * fraction), 3},
b.id == Control::Quit ? pink : cyan);
}
const std::string label = b.id == Control::ModelRow ? [&]() {
const auto& m = panel.models[model_page * 6 + i - 23];
@@ -547,23 +648,46 @@ struct PanelSurface::Impl {
b.id == Control::Date ? (date_format == DateFormat::Off ? "Date: Off" :
date_format == DateFormat::MonthDayYear ? "Date: MM/DD/YYYY" :
date_format == DateFormat::DayMonthYear ? "Date: DD/MM/YYYY" : "Date: YYYY-MM-DD") : b.label;
text(label, b.r.x + 16, b.r.y + b.r.h / 2 + 9,
text(label, r.x + 16, r.y + r.h / 2 + 9,
b.id == Control::Date ? 23 : b.id == Control::CloseMicWhenIdle ? 22 :
b.id == Control::Enter ? 20 : 27,
on ? ink : mix(background, muted, .48f), b.r.x + b.r.w - 8);
if (mounting(b.id) && selected) text("ON", b.r.x + b.r.w - 56, b.r.y + 38, 23, cyan, b.r.x + b.r.w - 12);
on ? ink : mix(background, muted, .48f), r.x + r.w - 8);
if (mounting(b.id) && selected) text("ON", r.x + r.w - 56, r.y + 38, 23, cyan, r.x + r.w - 12);
if (b.id == Control::LasersAnytime || b.id == Control::AdvancedDebug || b.id == Control::AutoInsert ||
b.id == Control::LockLayout || b.id == Control::CloseMicWhenIdle) {
b.id == Control::LockLayout || b.id == Control::CloseMicWhenIdle || b.id == Control::AnimatedBackground ||
b.id == Control::WristWorldFallback) {
bool active = b.id == Control::LasersAnytime ? lasers_anytime :
b.id == Control::AutoInsert ? auto_insert :
b.id == Control::LockLayout ? layout_locked :
b.id == Control::CloseMicWhenIdle ? close_mic_when_idle : advanced_debug;
text(active ? "ON" : "OFF", b.r.x + b.r.w - 66, b.r.y + b.r.h / 2 + 9, 22,
active ? cyan : muted, b.r.x + b.r.w - 12);
b.id == Control::AnimatedBackground ? gradient.enabled :
b.id == Control::CloseMicWhenIdle ? close_mic_when_idle :
b.id == Control::WristWorldFallback ? wrist_world_fallback : advanced_debug;
text(active ? "ON" : "OFF", r.x + r.w - 66, r.y + r.h / 2 + 9, 22,
active ? cyan : muted, r.x + r.w - 12);
}
clip_view.reset();
}
if (tab == Tab::Settings) {
rect({976, settings_view.y, 3, settings_view.h}, mix(card, muted, .24f));
const int thumb = settings_view.h * settings_view.h / (settings_view.h + settings_max_scroll);
rect({975, settings_view.y + int(settings_scroll * (settings_view.h - thumb) / settings_max_scroll),
5, thumb}, cyan);
text("Right stick: scroll", 32, 565, 16, muted, 320);
const std::string update_note = update_status == UpdateStatus::Idle ? "" :
update_status == UpdateStatus::Checking ? "Checking for updates..." :
update_status == UpdateStatus::Current ? "FrameYap is up to date." :
update_status == UpdateStatus::Available ? "Update available: " + update_version :
update_status == UpdateStatus::TerminalFailed ? "Could not open update terminal." :
"Could not check for updates.";
if (!update_note.empty()) {
clip_view = settings_view;
text(update_note, 32, 758 - int(std::lround(settings_scroll)), 18,
(update_status == UpdateStatus::Failed || update_status == UpdateStatus::TerminalFailed) ? pink : cyan, 968);
clip_view.reset();
}
}
if (tab == Tab::Settings)
text("OFF: discard idle audio; ON: spike / start latency.", 390, 565, 16, muted, 887);
if (tab == Tab::Review && !binding_note.empty())
text(binding_note, 32, 665, 18, pink, 968);
if (tab == Tab::Models && !panel.model_note.empty()) {
// The install/navigation controls occupy y=530..566; the status
// belongs BELOW the shared footer (574..642), not under buttons.
@@ -607,7 +731,7 @@ bool PanelSurface::dragging(unsigned cursor) const {
return cursor < impl_->pressed.size() && int(cursor) == impl_->drag_cursor;
}
std::optional<PanelDragKind> PanelSurface::pointer_down(unsigned cursor, float x, float y, Clock::time_point now) {
if (cursor >= impl_->pressed.size() || impl_->drag_cursor >= 0) return {};
if (cursor >= impl_->pressed.size() || impl_->drag_cursor >= 0 || impl_->pressed[cursor] >= 0) return {};
const auto contains = [&](Bounds b) { return Rect{b.x, b.y, b.w, b.h}.contains(x, y); };
if (!impl_->layout_locked && (contains(grab) || contains(scale))) {
impl_->reset(); // other cursor's prior approval cannot survive relocation
@@ -616,7 +740,30 @@ std::optional<PanelDragKind> PanelSurface::pointer_down(unsigned cursor, float x
}
impl_->pressed[cursor] = impl_->hit(x, y);
impl_->press_time[cursor] = now;
if (impl_->pressed[cursor] == 10) impl_->dirty = true;
impl_->keyboard_fired[cursor] = false;
if (impl_->pressed[cursor] == 10 ||
(impl_->pressed[cursor] >= 0 && buttons[impl_->pressed[cursor]].id == Control::Keyboard)) impl_->dirty = true;
return {};
}
void PanelSurface::pointer_move(unsigned cursor, float x, float y) {
if (cursor >= impl_->pressed.size() || impl_->pressed[cursor] < 0) return;
const int index = impl_->pressed[cursor];
if (buttons[index].id == Control::Keyboard && impl_->hit(x, y) != index) {
impl_->pressed[cursor] = -1; // leaving cancels; returning cannot rearm the same press
impl_->keyboard_fired[cursor] = false;
impl_->dirty = true;
}
}
std::optional<SurfaceEvent::Companion> PanelSurface::poll_keyboard_hold(Clock::time_point now) {
for (size_t cursor = 0; cursor < impl_->pressed.size(); ++cursor) {
const int index = impl_->pressed[cursor];
if (index >= 0 && buttons[index].id == Control::Keyboard && !impl_->keyboard_fired[cursor] &&
now - impl_->press_time[cursor] >= keyboard_hold) {
impl_->keyboard_fired[cursor] = true;
impl_->dirty = true;
return SurfaceEvent::Companion::KeyboardRecenter;
}
}
return {};
}
SurfaceEvent PanelSurface::pointer_up(unsigned cursor, float x, float y, Clock::time_point now) {
@@ -627,9 +774,15 @@ SurfaceEvent PanelSurface::pointer_up(unsigned cursor, float x, float y, Clock::
return result;
}
int index = std::exchange(impl_->pressed[cursor], -1);
if (index == 10) impl_->dirty = true;
const bool fired = std::exchange(impl_->keyboard_fired[cursor], false);
if (index == 10 || (index >= 0 && buttons[index].id == Control::Keyboard)) impl_->dirty = true;
if (index < 0 || impl_->hit(x, y) != index) return result;
auto c = buttons[index].id;
if (c == Control::Keyboard) {
if (!fired) result.launch_companion = now - impl_->press_time[cursor] >= keyboard_hold ?
SurfaceEvent::Companion::KeyboardRecenter : SurfaceEvent::Companion::Keyboard;
return result;
}
if (c == Control::Quit && now - impl_->press_time[cursor] < quit_hold) return result;
if (c == Control::Record) result.action = impl_->panel.recording ? UiAction::EndRecord : UiAction::BeginRecord;
else if (auto a = action(c)) result.action = a;
@@ -638,15 +791,20 @@ SurfaceEvent PanelSurface::pointer_up(unsigned cursor, float x, float y, Clock::
else if (c == Control::LasersAnytime) result.lasers_anytime = !impl_->lasers_anytime;
else if (c == Control::AdvancedDebug) result.advanced_debug = !impl_->advanced_debug;
else if (c == Control::AutoInsert) result.auto_insert = !impl_->auto_insert;
else if (c == Control::AnimatedBackground) result.gradient_enabled = !impl_->gradient.enabled;
else if (c == Control::CloseMicWhenIdle) result.close_mic_when_idle = !impl_->close_mic_when_idle;
else if (c == Control::LockLayout) result.lock_layout = !impl_->layout_locked;
else if (c == Control::WristWorldFallback) result.wrist_world_fallback = !impl_->wrist_world_fallback;
else if (c == Control::Clock24h) result.clock_24h = !impl_->clock_24h;
else if (c == Control::Date) result.date_format = static_cast<DateFormat>((static_cast<int>(impl_->date_format) + 1) % 4);
else if (c == Control::Bindings) { result.open_bindings = true; impl_->reset(); }
else if (c == Control::UpdateCheck) { result.check_updates = true; impl_->reset(); }
else if (c == Control::UpdateInstall) { result.install_update = true; impl_->reset(); }
else if (c == Control::Models) {
impl_->tab = Tab::Models; impl_->model_page = 0;
impl_->install_confirm = false; impl_->consent_snapshot.reset(); impl_->reset(); impl_->dirty = true;
}
else if (c == Control::About) { impl_->tab = Tab::About; impl_->reset(); impl_->dirty = true; }
else if (c == Control::ModelRow) {
const auto& model = impl_->panel.models[impl_->model_page * 6 + index - 23];
result.model_action = ModelAction{model.id, false, {}};
@@ -679,10 +837,44 @@ SurfaceEvent PanelSurface::pointer_up(unsigned cursor, float x, float y, Clock::
impl_->tab = c == Control::Review ? Tab::Review : Tab::Settings;
impl_->install_confirm = false; impl_->consent_snapshot.reset(); impl_->reset(); impl_->dirty = true;
}
else if (c == Control::Prev) { --impl_->page; impl_->dirty = true; }
else if (c == Control::Next) { ++impl_->page; impl_->dirty = true; }
else if (c == Control::Plan || c == Control::Keyboard || c == Control::Draw) {
result.launch_companion = c == Control::Plan ? SurfaceEvent::Companion::Plan :
c == Control::Keyboard ? SurfaceEvent::Companion::Keyboard : SurfaceEvent::Companion::Draw;
impl_->reset();
}
return result;
}
bool PanelSurface::scroll(float x, float y, float vertical_delta) {
if (impl_->drag_cursor >= 0 || !std::isfinite(vertical_delta)) return false;
if (impl_->tab == Tab::Review && !impl_->panel.quick_open && review_view.contains(x, y)) {
const float next = std::clamp(impl_->review_scroll - std::clamp(vertical_delta, -4.f, 4.f) * 48.f,
0.f, impl_->max_review_scroll());
if (next == impl_->review_scroll) return false;
impl_->review_scroll = next;
impl_->reset(); impl_->dirty = true;
return true;
}
if (impl_->tab != Tab::Settings || !settings_view.contains(x, y)) return false;
const float next = std::clamp(impl_->settings_scroll - std::clamp(vertical_delta, -4.f, 4.f) * 48.f,
0.f, float(settings_max_scroll));
if (next == impl_->settings_scroll) return false;
impl_->settings_scroll = next;
impl_->reset(); // a press must not turn into a click on a different setting
impl_->dirty = true;
return true;
}
void PanelSurface::set_companions(bool plan, bool keyboard, bool draw) {
if (impl_->plan_present != plan || impl_->keyboard_present != keyboard || impl_->draw_present != draw) {
impl_->plan_present = plan; impl_->keyboard_present = keyboard; impl_->draw_present = draw;
impl_->reset(); impl_->dirty = true;
}
}
void PanelSurface::set_update_status(UpdateStatus status, std::string version) {
if (impl_->update_status != status || impl_->update_version != version) {
impl_->update_status = status; impl_->update_version = std::move(version);
impl_->reset(); impl_->dirty = true;
}
}
void PanelSurface::set_indicators(const StatusIndicators& indicators) {
if (impl_->indicators != indicators) { impl_->indicators = indicators; impl_->dirty = true; }
}
@@ -695,11 +887,19 @@ std::vector<PanelSurface::Bounds> PanelSurface::input_regions() const {
if (!impl_->layout_locked) { result.push_back(grab); result.push_back(scale); }
return result;
}
void PanelSurface::set_wrist_world_fallback(bool enabled) {
if (impl_->wrist_world_fallback != enabled) {
impl_->wrist_world_fallback = enabled; impl_->reset(); impl_->dirty = true;
}
}
void PanelSurface::set_layout_locked(bool locked) {
if (impl_->layout_locked != locked) {
impl_->layout_locked = locked; impl_->reset(); impl_->dirty = true;
}
}
void PanelSurface::set_version(std::string version) {
if (impl_->version != version) { impl_->version = std::move(version); impl_->dirty = true; }
}
void PanelSurface::set_placement_note(std::string note) {
if (impl_->placement_note != note) { impl_->placement_note = std::move(note); impl_->dirty = true; }
}
@@ -724,6 +924,14 @@ void PanelSurface::set_auto_insert(bool enabled) {
impl_->dirty = true;
}
}
void PanelSurface::set_gradient_enabled(bool enabled) {
if (impl_->gradient.enabled != enabled) {
impl_->gradient.enabled = enabled;
impl_->animation_start.reset(); // a new opt-in cycle starts at the next render
impl_->reset();
impl_->dirty = true;
}
}
void PanelSurface::set_close_mic_when_idle(bool enabled) {
if (impl_->close_mic_when_idle != enabled) {
impl_->close_mic_when_idle = enabled;
+19
View File
@@ -12,17 +12,23 @@
namespace frameyap {
struct SurfaceEvent {
enum class Companion { Plan, Keyboard, KeyboardRecenter, Draw };
std::optional<Companion> launch_companion;
std::optional<UiAction> action;
std::optional<Mount> mount;
std::optional<bool> lasers_anytime;
std::optional<bool> advanced_debug;
std::optional<bool> auto_insert;
std::optional<bool> gradient_enabled;
std::optional<bool> close_mic_when_idle;
std::optional<bool> lock_layout;
std::optional<bool> wrist_world_fallback;
std::optional<bool> clock_24h;
std::optional<DateFormat> date_format;
bool recenter = false;
bool open_bindings = false;
bool check_updates = false;
bool install_update = false;
std::optional<ModelAction> model_action;
};
// Header indicators. Absent values are hidden, never shown as zero or as a guess.
@@ -31,6 +37,7 @@ struct StatusIndicators {
std::optional<BatteryLevel> left, head, right;
bool operator==(const StatusIndicators&) const = default;
};
enum class UpdateStatus { Idle, Checking, Current, Available, Failed, TerminalFailed };
// One CPU RGBA canvas, independent of OpenVR. Settings replace the review area;
// status and safety controls remain on the same surface.
class PanelSurface {
@@ -51,17 +58,29 @@ public:
const std::vector<unsigned char>& pixels() const;
// Handles capture one cursor and never authorize a UI action on release.
static constexpr auto quit_hold = std::chrono::milliseconds(900);
static constexpr auto keyboard_hold = std::chrono::milliseconds(800);
std::optional<PanelDragKind> pointer_down(unsigned cursor, float x, float y, Clock::time_point now = Clock::now());
void pointer_move(unsigned cursor, float x, float y);
// Called while visible and tracked. Fires once at the threshold without waiting for release.
std::optional<SurfaceEvent::Companion> poll_keyboard_hold(Clock::time_point now = Clock::now());
SurfaceEvent pointer_up(unsigned cursor, float x, float y, Clock::time_point now = Clock::now());
// OpenVR laser wheel input at the last hovered panel coordinate.
bool scroll(float x, float y, float vertical_delta);
void set_companions(bool plan, bool keyboard, bool draw);
void set_update_status(UpdateStatus status, std::string version = {});
bool dragging(unsigned cursor) const;
// OpenVR intersection masks use top-left coordinates, unlike mouse events.
std::vector<Bounds> input_regions() const;
void set_layout_locked(bool locked);
void set_wrist_world_fallback(bool enabled);
void reset_pointers();
void set_placement_note(std::string note);
// Shown on Settings -> About; empty shows "version unknown".
void set_version(std::string version);
void set_lasers_anytime(bool enabled);
void set_advanced_debug(bool enabled);
void set_auto_insert(bool enabled);
void set_gradient_enabled(bool enabled);
void set_close_mic_when_idle(bool enabled);
void set_clock_24h(bool enabled);
void set_date_format(DateFormat format);
+115
View File
@@ -0,0 +1,115 @@
#include "placement.hpp"
#include "panel_drag.hpp"
#include <atomic>
#include <cerrno>
#include <cmath>
#include <fstream>
#include <iomanip>
#include <limits>
#include <locale>
#include <sstream>
#include <string>
#include <fcntl.h>
#include <unistd.h>
namespace frameyap {
namespace {
constexpr std::size_t max_placement_bytes = 512;
std::atomic<unsigned long> next_temp{0};
bool relative_mount(Mount mount) {
return mount == Mount::LeftWrist || mount == Mount::RightWrist || mount == Mount::Head;
}
} // namespace
std::filesystem::path default_placement_path(Mount mount) {
if (!relative_mount(mount)) return {};
const auto preference = default_mount_settings_path();
if (preference.empty()) return {};
return preference.parent_path() / ("placement-" + std::string(mount_name(mount)));
}
bool valid_relative_placement(const RelativePlacement& placement) {
if (!panel_drag_detail::rigid(placement.canvas_pose) ||
!std::isfinite(placement.scale) || placement.scale < .5f || placement.scale > 2.f) return false;
for (int row = 0; row < 3; ++row)
if (std::abs(placement.canvas_pose[row][3]) > 5.f) return false;
return true;
}
std::optional<RelativePlacement> load_relative_placement(const std::filesystem::path& path, Mount mount) {
try {
if (!relative_mount(mount) || path.empty() || std::filesystem::is_symlink(path) ||
!std::filesystem::is_regular_file(path)) return {};
std::ifstream file(path, std::ios::binary);
if (!file) return {};
char bytes[max_placement_bytes + 1];
file.read(bytes, sizeof(bytes));
const auto count = file.gcount();
if (count <= 0 || count > static_cast<std::streamsize>(max_placement_bytes)) return {};
std::istringstream input(std::string(bytes, static_cast<std::size_t>(count)));
input.imbue(std::locale::classic());
std::string magic, name;
RelativePlacement placement{};
if (!(input >> magic >> name >> placement.scale) || magic != "frameyap-placement-v1" ||
name != mount_name(mount)) return {};
for (auto& row : placement.canvas_pose)
for (auto& value : row)
if (!(input >> value)) return {};
input >> std::ws;
if (!input.eof() || !valid_relative_placement(placement)) return {};
return placement;
} catch (...) { return {}; }
}
bool save_relative_placement(const std::filesystem::path& path, Mount mount,
const RelativePlacement& placement) {
if (!relative_mount(mount) || path.empty() || !path.is_absolute() ||
!valid_relative_placement(placement)) return false;
std::filesystem::path temporary;
int fd = -1;
try {
const auto dir = path.parent_path();
if (std::filesystem::is_symlink(dir) || std::filesystem::is_symlink(path) ||
(std::filesystem::exists(path) && !std::filesystem::is_regular_file(path))) return false;
std::ostringstream out;
out.imbue(std::locale::classic());
out << "frameyap-placement-v1 " << mount_name(mount) << ' ' <<
std::setprecision(std::numeric_limits<float>::max_digits10) << placement.scale;
for (const auto& row : placement.canvas_pose)
for (float value : row) out << ' ' << value;
out << '\n';
const auto content = out.str();
if (content.size() > max_placement_bytes) return false;
std::filesystem::create_directories(dir);
if (std::filesystem::is_symlink(dir)) return false;
for (int attempt = 0; attempt < 16; ++attempt) {
temporary = dir / (path.filename().string() + ".tmp." + std::to_string(::getpid()) +
"." + std::to_string(next_temp.fetch_add(1)));
fd = ::open(temporary.c_str(), O_WRONLY | O_CREAT | O_EXCL | O_CLOEXEC, 0600);
if (fd >= 0) break;
if (errno != EEXIST) return false;
}
if (fd < 0) return false;
std::size_t written = 0;
while (written < content.size()) {
const auto n = ::write(fd, content.data() + written, content.size() - written);
if (n < 0 && errno == EINTR) continue;
if (n <= 0) { ::close(fd); fd = -1; std::filesystem::remove(temporary); return false; }
written += static_cast<std::size_t>(n);
}
const bool synced = ::fsync(fd) == 0;
const bool closed = ::close(fd) == 0;
fd = -1;
if (!synced || !closed) { std::filesystem::remove(temporary); return false; }
std::filesystem::rename(temporary, path);
return true;
} catch (...) {
if (fd >= 0) ::close(fd);
std::error_code ignored;
if (!temporary.empty()) std::filesystem::remove(temporary, ignored);
return false;
}
}
} // namespace frameyap
+22
View File
@@ -0,0 +1,22 @@
#pragma once
#include "mount.hpp"
#include <filesystem>
#include <optional>
namespace frameyap {
// The exact OpenVR canvas transform in the selected tracked device's local
// coordinate frame, including the transparent grab/scale margins.
struct RelativePlacement {
Matrix34 canvas_pose;
float scale = 1.f;
};
// World-space placement is deliberately not persisted across tracking origins.
std::filesystem::path default_placement_path(Mount mount);
bool valid_relative_placement(const RelativePlacement& placement);
std::optional<RelativePlacement> load_relative_placement(const std::filesystem::path& path, Mount mount);
// Atomic owner-only write; does not follow a symlink at path or its parent.
bool save_relative_placement(const std::filesystem::path& path, Mount mount,
const RelativePlacement& placement);
} // namespace frameyap
+47 -3
View File
@@ -17,6 +17,7 @@
#include <memory>
#include <stdexcept>
#include <thread>
#include <unistd.h>
namespace frameyap {
namespace {
@@ -64,12 +65,51 @@ private:
Worker worker_;
std::string backend_, model_, manifest_, root_;
};
// Opt-in (Advanced debug) native delivery diagnostics. Metadata only: never
// transcript text. Owner-private file next to worker-debug.log.
class DeliveryTrace {
public:
~DeliveryTrace() { set_enabled(false); }
void set_enabled(bool enabled) {
if (enabled == enabled_) return;
enabled_ = enabled;
if (fd_ >= 0) { ::close(fd_); fd_ = -1; }
if (enabled_) {
try { fd_ = open_private_debug_log("delivery-debug.log", "delivery-debug.previous.log"); }
catch (...) { fd_ = -1; }
written_ = 0;
}
}
void operator()(const std::string& line) {
if (fd_ < 0 || written_ > 1024 * 1024) return;
const auto ms = std::chrono::duration_cast<std::chrono::milliseconds>(
std::chrono::steady_clock::now() - start_).count();
const auto out = std::to_string(ms) + "ms " + line + "\n";
if (::write(fd_, out.data(), out.size()) > 0) written_ += out.size();
}
private:
bool enabled_ = false;
int fd_ = -1;
size_t written_ = 0;
std::chrono::steady_clock::time_point start_ = std::chrono::steady_clock::now();
};
class NativeFocus final : public ControllerFocus {
public:
bool arm() override { return guard_.arm(); }
bool valid() override { return guard_.valid(); }
explicit NativeFocus(DeliveryTrace& trace) : trace_(trace) {}
bool arm() override {
if (guard_.arm()) { trace_("focus armed"); return true; }
trace_(std::string("focus arm failed: ") + guard_.failure());
return false;
}
bool valid() override {
if (guard_.valid()) return true;
trace_(std::string("focus invalid: ") + guard_.failure());
return false;
}
const char* failure() const override { return guard_.failure(); }
private:
FocusGuard guard_;
DeliveryTrace& trace_;
};
} // namespace
int run(const Options& options) {
@@ -85,6 +125,8 @@ int run(const Options& options) {
auto old_term = std::signal(SIGTERM, signal_stop);
struct Restore { decltype(old_int) a, b; ~Restore() { std::signal(SIGINT, a); std::signal(SIGTERM, b); } } restore{old_int, old_term};
Overlay overlay(options.assets, options.font, options.mount);
DeliveryTrace trace;
trace.set_enabled(overlay.advanced_debug());
NativeWorker worker(options);
NativeAudio audio;
const DeliveryFactory acquire = [&]() -> std::unique_ptr<DeliveryLease> {
@@ -94,7 +136,8 @@ int run(const Options& options) {
};
PacedDelivery paced(acquire, std::chrono::steady_clock::now,
[] { TextInput::release_idle(); }, [] { return !interrupted; });
Controller controller(audio, worker, acquire, [] { return std::make_unique<NativeFocus>(); },
paced.set_trace([&trace](const std::string& line) { trace(line); });
Controller controller(audio, worker, acquire, [&trace] { return std::make_unique<NativeFocus>(trace); },
overlay.quick_inputs(), overlay.auto_insert(), overlay.advanced_debug(),
overlay.close_mic_when_idle(), &paced);
namespace fs = std::filesystem;
@@ -190,6 +233,7 @@ int run(const Options& options) {
"Selected model unavailable or being checked. Recording disabled until offline verification.");
}
bool debug_changed = overlay.advanced_debug() != previous_debug;
trace.set_enabled(overlay.advanced_debug());
controller.settings(overlay.auto_insert(), overlay.advanced_debug(), overlay.close_mic_when_idle());
// Debug consent and model actions cancel work before any stale input.
if (debug_changed || !model_actions.empty()) {
+14
View File
@@ -0,0 +1,14 @@
#pragma once
#include <utility>
namespace frameyap {
// Follow docs/rendering-performance.md: retain initialized images throughout
// manipulation, leaving content invalidation pending until the next refresh.
template<class Render, class Upload>
bool refresh_texture(bool manipulating, bool initialized, Render&& render, Upload&& upload) {
if (manipulating && initialized) return false;
if (!std::forward<Render>(render)()) return false;
std::forward<Upload>(upload)();
return true;
}
}
+158
View File
@@ -0,0 +1,158 @@
#include "update_check.hpp"
#include <cerrno>
#include <csignal>
#include <cstdlib>
#include <fcntl.h>
#include <spawn.h>
#include <sys/stat.h>
#include <sys/wait.h>
#include <unistd.h>
#include <regex>
extern char** environ;
namespace frameyap {
namespace {
bool release_version(const std::string& version) {
static const std::regex pattern(R"((0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.[1-9][0-9]{11})");
return std::regex_match(version, pattern);
}
std::optional<std::filesystem::path> terminal_executable() {
const char* path = std::getenv("PATH");
if (!path) return {};
const std::string dirs(path);
for (size_t start = 0; start <= dirs.size();) {
const size_t end = dirs.find(':', start);
const auto candidate = std::filesystem::path(dirs.substr(start, end - start)) / "konsole";
struct stat info{};
if (candidate.is_absolute() && ::stat(candidate.c_str(), &info) == 0 &&
S_ISREG(info.st_mode) && ::access(candidate.c_str(), X_OK) == 0) return candidate;
if (end == std::string::npos) break;
start = end + 1;
}
return {};
}
}
UpdateCheck::UpdateCheck(std::filesystem::path helper, std::string current, std::string python)
: helper_(std::move(helper)), current_(std::move(current)), python_(std::move(python)) {}
UpdateCheck::~UpdateCheck() { stop(); }
void UpdateCheck::stop() noexcept {
if (pid_ > 0) {
::kill(pid_, SIGTERM); // exact owned checker, never SteamVR or the terminal
int status = 0;
bool reaped = false;
for (int i = 0; i < 20; ++i) {
auto result = ::waitpid(pid_, &status, WNOHANG);
if (result == pid_ || (result < 0 && errno == ECHILD)) { reaped = true; break; }
if (result < 0 && errno != EINTR) break;
::usleep(5000);
}
if (!reaped) {
::kill(pid_, SIGKILL);
while (::waitpid(pid_, &status, 0) < 0 && errno == EINTR) {}
}
pid_ = -1;
}
if (fd_ >= 0) { ::close(fd_); fd_ = -1; }
output_.clear(); eof_ = false;
}
bool UpdateCheck::start() {
if (busy() || !helper_.is_absolute() || !std::filesystem::is_regular_file(helper_) ||
!release_version(current_)) return false;
int pipe[2];
if (::pipe2(pipe, O_CLOEXEC) < 0) return false;
if (::fcntl(pipe[0], F_SETFL, O_NONBLOCK) < 0) {
::close(pipe[0]); ::close(pipe[1]); return false;
}
posix_spawn_file_actions_t actions;
int error = posix_spawn_file_actions_init(&actions);
if (error) { ::close(pipe[0]); ::close(pipe[1]); return false; }
error = posix_spawn_file_actions_adddup2(&actions, pipe[1], STDOUT_FILENO);
if (!error) error = posix_spawn_file_actions_addopen(&actions, STDIN_FILENO, "/dev/null", O_RDONLY, 0);
if (!error) error = posix_spawn_file_actions_addopen(&actions, STDERR_FILENO, "/dev/null", O_WRONLY, 0);
if (!error) error = posix_spawn_file_actions_addclose(&actions, pipe[0]);
if (!error) error = posix_spawn_file_actions_addclose(&actions, pipe[1]);
const auto helper = helper_.string();
std::vector<std::string> args{python_, "-I", helper, "--check", "--current", current_};
std::vector<char*> argv;
for (auto& arg : args) argv.push_back(arg.data());
argv.push_back(nullptr);
pid_t child = -1;
if (!error) error = ::posix_spawnp(&child, python_.c_str(), &actions, nullptr, argv.data(), environ);
posix_spawn_file_actions_destroy(&actions);
::close(pipe[1]);
if (error) { ::close(pipe[0]); return false; }
pid_ = child; fd_ = pipe[0]; eof_ = false; output_.clear();
deadline_ = std::chrono::steady_clock::now() + std::chrono::seconds(10);
return true;
}
std::optional<UpdateResult> UpdateCheck::poll() {
if (!busy()) return {};
if (std::chrono::steady_clock::now() > deadline_) { stop(); return UpdateResult{UpdateResult::State::Failed, {}}; }
if (!eof_) {
char bytes[128];
for (;;) {
const auto n = ::read(fd_, bytes, sizeof bytes);
if (n > 0) {
output_.append(bytes, size_t(n));
if (output_.size() > 128) { stop(); return UpdateResult{UpdateResult::State::Failed, {}}; }
} else if (n == 0) { ::close(fd_); fd_ = -1; eof_ = true; break; }
else if (errno == EINTR) continue;
else if (errno == EAGAIN || errno == EWOULDBLOCK) break;
else { stop(); return UpdateResult{UpdateResult::State::Failed, {}}; }
}
}
if (!eof_) return {};
int status = 0;
const auto reaped = ::waitpid(pid_, &status, WNOHANG);
if (reaped == 0) return {};
pid_ = -1;
if (reaped < 0 || !WIFEXITED(status) || WEXITSTATUS(status))
return UpdateResult{UpdateResult::State::Failed, {}};
if (output_ == "CURRENT\n") return UpdateResult{UpdateResult::State::Current, {}};
constexpr std::string_view prefix = "AVAILABLE ";
if (output_.starts_with(prefix) && output_.ends_with('\n')) {
auto version = output_.substr(prefix.size(), output_.size() - prefix.size() - 1);
if (release_version(version) && version != current_)
return UpdateResult{UpdateResult::State::Available, std::move(version)};
}
return UpdateResult{UpdateResult::State::Failed, {}};
}
bool open_update_terminal(const std::filesystem::path& helper, const std::string& version) {
if (!helper.is_absolute() || !std::filesystem::is_regular_file(helper) || !release_version(version)) return false;
const auto terminal = terminal_executable();
if (!terminal) return false;
const auto script = helper.string();
std::vector<std::string> args{terminal->string(), "--hold", "-e", "python3", "-I", script, "--version", version};
std::vector<char*> argv;
for (auto& arg : args) argv.push_back(arg.data());
argv.push_back(nullptr);
// Konsole is an independent desktop window and must survive FrameYap quitting.
// Double fork avoids a zombie; the close-on-exec pipe detects launch errors.
int pipe[2];
if (::pipe2(pipe, O_CLOEXEC)) return false;
const pid_t intermediate = ::fork();
if (intermediate < 0) { ::close(pipe[0]); ::close(pipe[1]); return false; }
if (intermediate == 0) {
::close(pipe[0]);
const pid_t child = ::fork();
if (child == 0) {
::execv(args[0].c_str(), argv.data());
const int error = errno;
(void)::write(pipe[1], &error, sizeof(error));
} else if (child < 0) {
const int error = errno;
(void)::write(pipe[1], &error, sizeof(error));
}
::_exit(child < 0 ? 1 : child == 0 ? 127 : 0);
}
::close(pipe[1]);
int error = 0;
ssize_t count;
do { count = ::read(pipe[0], &error, sizeof(error)); } while (count < 0 && errno == EINTR);
::close(pipe[0]);
int status = 0;
pid_t reaped;
do { reaped = ::waitpid(intermediate, &status, 0); } while (reaped < 0 && errno == EINTR);
return count == 0 && reaped == intermediate && WIFEXITED(status) && WEXITSTATUS(status) == 0;
}
} // namespace frameyap
+37
View File
@@ -0,0 +1,37 @@
#pragma once
#include <chrono>
#include <filesystem>
#include <optional>
#include <string>
#include <vector>
#include <sys/types.h>
namespace frameyap {
struct UpdateResult {
enum class State { Current, Available, Failed } state;
std::string version; // only populated for a newer, validated release
};
// The only network path is an explicit start() call from Settings. The helper
// emits a tiny protocol; polling never blocks the OpenVR interaction loop.
class UpdateCheck {
public:
UpdateCheck(std::filesystem::path helper, std::string current, std::string python = "python3");
~UpdateCheck();
UpdateCheck(const UpdateCheck&) = delete;
UpdateCheck& operator=(const UpdateCheck&) = delete;
bool start();
std::optional<UpdateResult> poll();
bool busy() const { return pid_ > 0; }
private:
void stop() noexcept;
std::filesystem::path helper_;
std::string current_, python_, output_;
pid_t pid_ = -1;
int fd_ = -1;
bool eof_ = false;
std::chrono::steady_clock::time_point deadline_{};
};
// A separate explicit click opens a desktop terminal. The terminal waits for
// the user to quit FrameYap and press Enter; it survives FrameYap's own exit.
bool open_update_terminal(const std::filesystem::path& helper, const std::string& version);
}
+17 -16
View File
@@ -138,21 +138,6 @@ void check_log_target(int dir, const char* name) {
::close(fd);
if (!valid) throw std::runtime_error("unsafe existing worker debug log");
}
int create_debug_log() {
int dir = state_directory();
try {
check_log_target(dir, "worker-debug.log");
check_log_target(dir, "worker-debug.previous.log");
if (::unlinkat(dir, "worker-debug.previous.log", 0) && errno != ENOENT)
throw std::runtime_error("cannot rotate worker debug log");
if (::renameat(dir, "worker-debug.log", dir, "worker-debug.previous.log") && errno != ENOENT)
throw std::runtime_error("cannot rotate worker debug log");
int fd = ::openat(dir, "worker-debug.log", O_WRONLY | O_CREAT | O_EXCL | O_NOFOLLOW | O_CLOEXEC, 0600);
if (fd < 0) throw std::runtime_error("cannot create private worker debug log");
::close(dir);
return fd;
} catch (...) { ::close(dir); throw; }
}
void log_bytes(int fd, const unsigned char* data, size_t size) {
while (size) {
ssize_t n = ::write(fd, data, size);
@@ -194,6 +179,22 @@ void write_all(int fd, const unsigned char* data, size_t size) {
}
} // namespace
int open_private_debug_log(const char* name, const char* previous) {
int dir = state_directory();
try {
check_log_target(dir, name);
check_log_target(dir, previous);
if (::unlinkat(dir, previous, 0) && errno != ENOENT)
throw std::runtime_error("cannot rotate debug log");
if (::renameat(dir, name, dir, previous) && errno != ENOENT)
throw std::runtime_error("cannot rotate debug log");
int fd = ::openat(dir, name, O_WRONLY | O_CREAT | O_EXCL | O_NOFOLLOW | O_CLOEXEC, 0600);
if (fd < 0) throw std::runtime_error("cannot create private debug log");
::close(dir);
return fd;
} catch (...) { ::close(dir); throw; }
}
struct Worker::State {
pid_t pid = -1;
int to_child = -1, from_child = -1, debug_pipe = -1, debug_file = -1;
@@ -293,7 +294,7 @@ void Worker::start(const std::string& python, const std::string& script,
if (!::mkdtemp(tmp.data())) throw std::runtime_error("cannot create private clip directory");
state_->dir = tmp.data();
::chmod(state_->dir.c_str(), 0700);
if (advanced_debug) state_->debug_file = create_debug_log();
if (advanced_debug) state_->debug_file = open_private_debug_log("worker-debug.log", "worker-debug.previous.log");
int in[2]{-1,-1}, out[2]{-1,-1}, debug[2]{-1,-1};
if (::pipe2(in, O_CLOEXEC) || ::pipe2(out, O_CLOEXEC) ||
(advanced_debug && ::pipe2(debug, O_CLOEXEC))) {
+4
View File
@@ -15,6 +15,10 @@ struct WorkerReply {
std::string error;
};
// Opens (rotating `previous`) an owner-private 0600 file in the FrameYap state
// directory, or throws. Used only for opt-in advanced debugging.
int open_private_debug_log(const char* name, const char* previous);
class Worker {
public:
explicit Worker(std::chrono::milliseconds warmup_timeout = std::chrono::seconds(120),
+3 -1
View File
@@ -24,7 +24,9 @@ foreach(mode IN ITEMS default help version invalid extra)
message(FATAL_ERROR "${mode}: exit ${result}, expected ${expected_exit}: ${error}")
endif()
if(mode STREQUAL "version")
string(REPLACE "\n" ";" lines "${output}")
# Remove only the final terminator; a clean release has no git line.
string(REGEX REPLACE "\n$" "" version_lines "${output}")
string(REPLACE "\n" ";" lines "${version_lines}")
list(GET lines 0 first_line)
if(NOT first_line STREQUAL "frameyap ${EXPECTED_VERSION}")
message(FATAL_ERROR "Unexpected version: ${output}")
+64
View File
@@ -0,0 +1,64 @@
#include "companion_apps.hpp"
#include <cassert>
#include <chrono>
#include <cstdlib>
#include <filesystem>
#include <fstream>
#include <string>
#include <thread>
#include <sys/stat.h>
#include <unistd.h>
namespace fs = std::filesystem;
int main() {
char name[] = "/tmp/frameyap-companions-XXXXXX";
const auto root = fs::path(::mkdtemp(name));
const auto home = root / "home", local = home / ".local/bin", path_dir = root / "path";
fs::create_directories(local); fs::create_directories(path_dir);
::setenv("HOME", home.c_str(), 1);
::setenv("PATH", path_dir.c_str(), 1);
assert(!frameyap::find_companion("tnkplan"));
assert(!frameyap::find_companion("tnkboard"));
assert(!frameyap::find_companion("tnkdraw"));
assert(!frameyap::find_companion("other"));
const auto plan = local / "tnkplan", keyboard = path_dir / "tnkboard", draw = local / "tnkdraw";
std::ofstream(plan) << "#!/bin/sh\nexit 0\n";
std::ofstream(keyboard) << "#!/bin/sh\nexit 0\n";
std::ofstream(draw) << "#!/bin/sh\nexit 0\n";
assert(!frameyap::find_companion("tnkplan")); // a file is not yet executable
assert(!frameyap::find_companion("tnkdraw"));
::chmod(plan.c_str(), 0700);
::chmod(keyboard.c_str(), 0700);
::chmod(draw.c_str(), 0700);
assert(frameyap::find_companion("tnkplan") == plan);
assert(frameyap::find_companion("tnkboard") == keyboard);
assert(frameyap::find_companion("tnkdraw") == draw);
::setenv("PATH", (":" + path_dir.string()).c_str(), 1);
assert(frameyap::find_companion("tnkboard") == keyboard);
const auto marker = root / "launched";
std::ofstream(plan) << "#!/bin/sh\nprintf launched > '" << marker.string() << "'\n";
assert(frameyap::launch_companion(plan));
for (int i = 0; i < 100 && !fs::exists(marker); ++i)
std::this_thread::sleep_for(std::chrono::milliseconds(10));
assert(fs::exists(marker));
const auto argv_marker = root / "argv";
std::ofstream(keyboard) << "#!/bin/sh\nprintf '%s\\n' \"$#\" \"$1\" > '" << argv_marker.string() << "'\n";
const auto expect_argv = [&](frameyap::CompanionCommand command, const std::string& expected) {
fs::remove(argv_marker);
assert(frameyap::launch_companion(keyboard, command));
std::string count, argument;
for (int i = 0; i < 100; ++i) {
std::ifstream in(argv_marker);
if (std::getline(in, count) && std::getline(in, argument)) break;
std::this_thread::sleep_for(std::chrono::milliseconds(10));
}
assert(count == (expected.empty() ? "0" : "1"));
assert(argument == expected);
};
expect_argv(frameyap::CompanionCommand::Default, "");
expect_argv(frameyap::CompanionCommand::Show, "--show");
expect_argv(frameyap::CompanionCommand::Recenter, "--recenter");
fs::remove(plan);
assert(!frameyap::launch_companion(plan));
fs::remove_all(root);
}
+42 -3
View File
@@ -46,10 +46,11 @@ int main(int argc, char** argv) {
assert(!load_config(path).auto_insert);
assert(!load_config(path).close_mic_when_idle);
assert(!load_config(path).lock_layout);
assert(load_config(path).wrist_world_fallback);
assert(!load_config(path).clock_24h);
assert(load_config(path).date_format == DateFormat::MonthDayYear);
assert(load_config(path).wrist.width == .30f);
assert(load_config(path).gradient.enabled);
assert(!load_config(path).gradient.enabled);
assert(load_config(path).gradient.period_seconds == 30.f);
assert(load_config(path).gradient.strength == .12f);
auto example = load_config(std::filesystem::path(argv[1]) / "config.example.json");
@@ -63,9 +64,10 @@ int main(int argc, char** argv) {
assert(!example.auto_insert);
assert(!example.close_mic_when_idle);
assert(!example.lock_layout);
assert(example.wrist_world_fallback);
assert(!example.clock_24h && example.date_format == DateFormat::MonthDayYear);
assert(example.wrist.y == .18f && example.wrist.z == .089f);
assert(example.gradient.enabled && example.gradient.period_seconds == 30.f && example.gradient.strength == .12f);
assert(!example.gradient.enabled && example.gradient.period_seconds == 30.f && example.gradient.strength == .12f);
std::filesystem::create_directories(path.parent_path());
assert(save_backend(path, "fixture_cpu"));
assert(load_config(path).backend == "fixture_cpu");
@@ -79,6 +81,11 @@ int main(int argc, char** argv) {
assert(load_config(path).auto_insert);
assert(save_auto_insert(path, false));
assert(!load_config(path).auto_insert);
assert(save_gradient_enabled(path, true)); // missing gradient creates only the nested setting
assert(load_config(path).gradient.enabled);
assert(get(path).find("\"gradient\":{\"enabled\":true}") != std::string::npos);
assert(save_gradient_enabled(path, false));
assert(!load_config(path).gradient.enabled);
assert(save_close_mic_when_idle(path, true));
assert(load_config(path).close_mic_when_idle);
assert(save_close_mic_when_idle(path, false));
@@ -190,6 +197,20 @@ int main(int argc, char** argv) {
assert(save_lock_layout(path, true));
assert(get(path) == R"({"font":"escaped \u0061","theme":{"ink":"#123ABC"},"auto_insert":true,"lock_layout":true})");
assert(load_config(path).lock_layout && load_config(path).auto_insert);
put(path, R"({"wrist_world_fallback":false})");
assert(!load_config(path).wrist_world_fallback);
assert(save_wrist_world_fallback(path, true));
assert(load_config(path).wrist_world_fallback);
assert(save_wrist_world_fallback(path, false));
assert(!load_config(path).wrist_world_fallback);
put(path, R"({"wrist_world_fallback":true})");
assert(load_config(path).wrist_world_fallback);
for (const auto* invalid : {R"({"wrist_world_fallback":0})", R"({"wrist_world_fallback":"false"})",
R"({"wrist_world_fallback":null})"}) {
put(path, invalid); fails([&] { load_config(path); });
assert(!save_wrist_world_fallback(path, false));
assert(get(path) == invalid);
}
put(path, R"({"input_priority":"normal"})");
assert(!load_config(path).experimental_input_priority);
for (const auto* invalid : {R"({"input_priority":true})", R"({"input_priority":16777216})",
@@ -213,12 +234,24 @@ int main(int argc, char** argv) {
assert(save_auto_insert(path, true)); // unrelated settings retain the effect
assert(load_config(path).gradient.period_seconds == 45.5f);
put(path, R"({"gradient":{"strength":0}})");
assert(load_config(path).gradient.enabled && load_config(path).gradient.period_seconds == 30.f);
assert(!load_config(path).gradient.enabled && load_config(path).gradient.period_seconds == 30.f);
assert(load_config(path).gradient.strength == 0.f);
put(path, R"({"gradient":{"period_seconds":5,"strength":0.3}})");
assert(load_config(path).gradient.period_seconds == 5.f && load_config(path).gradient.strength == .3f);
put(path, R"({"gradient":{"period_seconds":300}})");
assert(load_config(path).gradient.period_seconds == 300.f);
put(path, R"({"font":"escaped \u0061","gradient":{"period_seconds":45.5,"strength":0.2},"auto_insert":true})");
assert(save_gradient_enabled(path, true));
assert(load_config(path).gradient.enabled);
assert(load_config(path).gradient.period_seconds == 45.5f && load_config(path).gradient.strength == .2f);
assert(get(path).find(R"("font":"escaped \u0061")") != std::string::npos);
assert(get(path).find(R"("strength":0.2,"enabled":true},"auto_insert":true)") != std::string::npos);
assert(save_gradient_enabled(path, false) && !load_config(path).gradient.enabled);
assert(save_gradient_enabled(path, true) && load_config(path).gradient.enabled);
put(path, R"({"gradient":{"enabled":false,"strength":0.21},"theme":{"ink":"#123ABC"}})");
assert(save_gradient_enabled(path, true));
assert(get(path) == R"({"gradient":{"enabled":true,"strength":0.21},"theme":{"ink":"#123ABC"}})");
assert(load_config(path).gradient.enabled);
for (auto invalid : {R"({"gradient":true})", R"({"gradient":{"enabled":1}})",
R"({"gradient":{"enabled":"false"}})", R"({"gradient":{"period_seconds":0}})",
R"({"gradient":{"period_seconds":301}})", R"({"gradient":{"period_seconds":1e99}})",
@@ -226,6 +259,7 @@ int main(int argc, char** argv) {
R"({"gradient":{"strength":0.31}})", R"({"gradient":{"strength":1e99}})",
R"({"gradient":{"strength":"0.1"}})", R"({"gradient":{"unknown":1}})"}) {
put(path, invalid); fails([&] { load_config(path); });
assert(!save_gradient_enabled(path, true) && get(path) == invalid);
}
put(path, R"({"theme":{"background":"#123ABC","accent":"#abcdef","frame_end":"#010203"},"font":"/nonexistent/face.ttf","buttons":{"ptt":"/user/hand/left/input/y","cancel":"/user/hand/right/input/b","right_grip":""}})");
auto config = load_config(path);
@@ -243,6 +277,8 @@ int main(int argc, char** argv) {
assert(generated.find("/user/hand/right/input/a") != std::string::npos);
assert(generated.find("/user/hand/right/input/y") != std::string::npos);
assert(generated.find("/actions/frameyap/in/quick_chat") != std::string::npos);
assert(generated.find("/actions/grab/in/left_depth") != std::string::npos);
assert(generated.find("/actions/grab/in/right_depth") != std::string::npos);
config.buttons["cancel"] = ""; config.buttons["insert"] = ""; config.buttons["enter"] = "";
auto disabled = action_manifest(argv[1], config);
generated = get(disabled.parent_path() / "bindings_frame_controller.json");
@@ -250,6 +286,9 @@ int main(int argc, char** argv) {
assert(generated.find("/actions/frameyap/in/insert") == std::string::npos);
assert(generated.find("/actions/frameyap/in/enter") == std::string::npos);
assert(action_manifest(argv[1], {}) == std::filesystem::absolute(std::filesystem::path(argv[1]) / "actions.json"));
put(path, R"({"buttons":{"quick_chat":"/user/hand/right/input/thumbstick"}})");
generated = get(action_manifest(argv[1], load_config(path)).parent_path() / "bindings_frame_controller.json");
assert(generated.find("\"mode\":\"joystick\",\"inputs\":{\"click\":{\"output\":\"/actions/frameyap/in/quick_chat\"}") != std::string::npos);
put(path, R"({"buttons":{"enter":"/user/hand/right/input/y"}})");
generated = get(action_manifest(argv[1], load_config(path)).parent_path() / "bindings_frame_controller.json");
assert(generated.find("/actions/frameyap/in/quick_chat") != std::string::npos);
+6
View File
@@ -33,6 +33,12 @@ private:
int main() {
try {
DisplaySetting isolated_display;
unsetenv("DISPLAY");
FocusGuard missing_display;
CHECK(!missing_display.arm());
CHECK(std::string(missing_display.failure()) == "DISPLAY missing");
CHECK(!missing_display.arm());
setenv("DISPLAY", "frameyap-no-such-display:9876", 1);
FocusGuard guard; // Construction must not connect to X.
CHECK(!guard.valid());
CHECK(!guard.arm()); // No server: fail closed.
+6
View File
@@ -50,6 +50,12 @@ int main() {
assert(!stable.arm());
FocusGuard new_capture;
assert(new_capture.arm() && new_capture.valid());
// Gamescope rewrites both properties with the same window after IME commits.
xcb_change_property(c, XCB_PROP_MODE_REPLACE, root, active, XCB_ATOM_WINDOW, 32, 1, &a);
xcb_change_property(c, XCB_PROP_MODE_REPLACE, root, gamescope, XCB_ATOM_CARDINAL, 32, 1, &a);
auto sync = xcb_get_input_focus(c);
std::free(xcb_get_input_focus_reply(c, sync, nullptr));
assert(new_capture.valid()); // same-value rewrite is not a focus change
xcb_change_property(c, XCB_PROP_MODE_REPLACE, root, active, XCB_ATOM_WINDOW, 32, 1, &b);
auto cookie = xcb_get_input_focus(c);
Reply<xcb_get_input_focus_reply_t> reply(xcb_get_input_focus_reply(c, cookie, nullptr));
+13
View File
@@ -12,6 +12,8 @@ check("${APP}" 0 "\"schema\": 1.*\"models\"|\"models\".*\"schema\": 1" --list-mo
check("${APP}" 1 "\"state\": \"not_installed\"" --check-model redux --model-dir "${STAGE}/absent" --json)
check("${APP}" 2 "^$" --check-model "redux;echo" --model-dir "${STAGE}/absent" --json)
check("${APP}" 2 "^$" --check-model "../redux" --model-dir "${STAGE}/absent" --json)
# Remove the previous fixture, including caches left by older test runs.
file(REMOVE_RECURSE "${STAGE}")
# Minimal installed payload, independent of the source tree's script/module/asset paths.
file(MAKE_DIRECTORY "${STAGE}/bin" "${STAGE}/scripts" "${STAGE}/python/frameyap" "${STAGE}/assets/backends")
file(COPY "${APP}" DESTINATION "${STAGE}/bin")
@@ -41,3 +43,14 @@ check("${STAGE}/bin/${name}" 0 "\"state\": \"installed_verified\""
file(WRITE "${STAGE}/models/toy/weights.bin" "HELLO")
check("${STAGE}/bin/${name}" 1 "\"reason\": \"hash_mismatch\""
--check-model toy --manifest-dir "${STAGE}/fixtures" --model-dir "${STAGE}/models/toy" --json)
# The standalone script must also preserve the inventory without the launcher.
execute_process(COMMAND python3 "${STAGE}/scripts/model-status.py" --list-models
RESULT_VARIABLE standalone_status OUTPUT_QUIET ERROR_VARIABLE standalone_error)
if(NOT standalone_status EQUAL 0)
message(FATAL_ERROR "Standalone model status failed: ${standalone_error}")
endif()
file(GLOB_RECURSE generated_cache "${STAGE}/python/*.pyc")
if(generated_cache OR EXISTS "${STAGE}/python/frameyap/__pycache__")
message(FATAL_ERROR "Offline model status modified the installed release: ${generated_cache}")
endif()
+9
View File
@@ -58,6 +58,15 @@ int main() {
}
CHECK(!parse_mount("World")); CHECK(!parse_mount("head\n")); CHECK(!parse_mount("left_wrist"));
CHECK(!parse_mount("")); CHECK(!parse_mount("world extra"));
for (auto wrist : {Mount::LeftWrist, Mount::RightWrist}) {
CHECK(effective_mount(wrist, true, true) == wrist);
CHECK(effective_mount(wrist, true, false) == wrist);
CHECK(effective_mount(wrist, false, true) == Mount::World);
CHECK(!effective_mount(wrist, false, false));
CHECK(effective_mount(wrist, true, false) == wrist); // restored tracking
}
CHECK(effective_mount(Mount::World, false, false) == Mount::World);
CHECK(effective_mount(Mount::Head, false, false) == Mount::Head);
CHECK(near(mount_width(Mount::World), .85f)); CHECK(near(mount_width(Mount::Head), .85f));
CHECK(near(mount_width(Mount::LeftWrist), .30f)); CHECK(near(mount_width(Mount::RightWrist), .30f));
+78
View File
@@ -98,6 +98,84 @@ int main() {
assert(moved); near_pose(moved->pose, translated(rotated_pose, -.1f, .2f, -.3f));
drag.reset(); assert(!drag.update(second_down));
// Stick depth is independent of controller translation, preserves pose
// rotation, and follows the captured panel normal as the controller turns.
assert(drag.begin(PanelDragKind::Grab, panel, 1, 1, .5f, .5f, down));
drag.step_depth(.1f, 1.); near(float(drag.depth()), 0);
drag.step_depth(1.f, .05); near(float(drag.depth()), .03f);
auto depth_pose = drag.update(down);
assert(depth_pose); near_pose(depth_pose->pose, translated(panel, 0, 0, -.03f));
depth_pose = drag.update(translated(down, .1f, .2f, 0));
assert(depth_pose); near_pose(depth_pose->pose, translated(panel, .1f, .2f, -.03f));
drag.step_depth(0.f, .05); near(float(drag.depth()), .03f);
drag.step_depth(-1.f, .05); near(float(drag.depth()), 0);
drag.step_depth(1.f, 10.); near(float(drag.depth()), .03f); // bounded time step
drag.step_depth(std::numeric_limits<float>::quiet_NaN(), .05);
drag.step_depth(1.f, std::numeric_limits<double>::infinity());
near(float(drag.depth()), .03f);
const auto turned_controller = compose_pose(down, yaw());
depth_pose = drag.update(turned_controller);
assert(depth_pose);
near_pose(depth_pose->pose, translated(compose_pose(turned_controller, relative_pose(down, panel)), -.03f, 0, 0));
for (int i = 0; i < 100; ++i) drag.step_depth(1.f, .05);
near(float(drag.depth()), 1.5f);
for (int i = 0; i < 100; ++i) drag.step_depth(-1.f, .05);
near(float(drag.depth()), -1.5f);
drag.reset(); near(float(drag.depth()), 0);
assert(drag.begin(PanelDragKind::Scale, identity, 1, 1, .5f, .5f, down));
drag.step_depth(1.f, .05); near(float(drag.depth()), 0);
drag.reset();
// A controller pointing obliquely at a rotated panel must move depth
// perpendicular to the panel rather than along its own pointing axis.
const auto angled_panel = translated(yaw(), .25f, .4f, -.3f);
assert(drag.begin(PanelDragKind::Grab, angled_panel, 1, 1, .5f, .5f, down));
drag.step_depth(1.f, .05);
depth_pose = drag.update(down);
assert(depth_pose); near_pose(depth_pose->pose, translated(angled_panel, -.03f, 0, 0));
const auto angled_moved = translated(compose_pose(down, pitch()), .1f, .2f, -.1f);
depth_pose = drag.update(angled_moved);
auto expected_relative = relative_pose(down, angled_panel);
expected_relative[0][3] -= .03f;
assert(depth_pose); near_pose(depth_pose->pose, compose_pose(angled_moved, expected_relative));
// The same elapsed movement has the same result at 50 and 100 Hz.
for (const double dt : {.01, .02}) {
assert(drag.begin(PanelDragKind::Grab, angled_panel, 1, 1, .5f, .5f, down));
for (int i = 0; i < int(1. / dt); ++i) drag.step_depth(1.f, dt);
near(float(drag.depth()), .6f);
}
drag.reset();
// Compositor events are source-tagged and already time-integrated.
assert(drag.begin(PanelDragKind::Grab, angled_panel, 1, 1, .5f, .5f, down));
assert(!drag.queue_scroll_depth(2, 1, 1.f)); // the other hand
assert(!drag.queue_scroll_depth(std::numeric_limits<uint32_t>::max(), 1, 1.f));
assert(!drag.queue_scroll_depth(1, 1, std::numeric_limits<float>::quiet_NaN()));
assert(drag.queue_scroll_depth(1, 1, .5f));
drag.update_depth({}, .001); near(float(drag.depth()), .02f);
drag.update_depth({}, .05); near(float(drag.depth()), .02f); // no replay
depth_pose = drag.update(down);
assert(depth_pose); near_pose(depth_pose->pose, translated(angled_panel, -.02f, 0, 0));
assert(drag.queue_scroll_depth(1, 1, 100.f));
drag.update_depth({}, 100.); near(float(drag.depth()), .05f); // capped, no dt multiplication
assert(drag.queue_scroll_depth(1, 1, -.5f));
drag.update_depth({}, 0); near(float(drag.depth()), .03f); // opposite direction
assert(drag.queue_scroll_depth(1, 1, 1.f));
drag.update_depth(0.f, .05); near(float(drag.depth()), .03f); // active centered axis wins
assert(drag.queue_scroll_depth(1, 1, 1.f));
drag.update_depth(1.f, .05); near(float(drag.depth()), .06f); // no double application
for (int i = 0; i < 100; ++i) assert(drag.queue_scroll_depth(1, 1, 1.f));
drag.update_depth({}, .01); near(float(drag.depth()), 1.5f);
assert(drag.queue_scroll_depth(1, 1, -.5f));
drag.reset(); // queued events cannot leak into a later grab
assert(drag.begin(PanelDragKind::Grab, angled_panel, 1, 1, .5f, .5f, down));
drag.update_depth({}, .05); near(float(drag.depth()), 0);
assert(drag.begin(PanelDragKind::Scale, identity, 1, 1, .5f, .5f, down));
assert(!drag.queue_scroll_depth(1, 1, 1.f));
drag.update_depth({}, .05); near(float(drag.depth()), 0);
drag.reset();
assert(!drag.queue_scroll_depth(1, 1, 1.f));
grab_rotation_case(pitch(), Matrix34{{{{1.f, 0.f, 0.f, .25f}},
{{0.f, 0.f, -1.f, 1.3f}},
{{0.f, 1.f, 0.f, 1.4f}}}});
+212 -24
View File
@@ -1,10 +1,12 @@
#include "panel_surface.hpp"
#include <cassert>
#include <ctime>
#include <filesystem>
#include <fstream>
#include <iostream>
#include <limits>
#include <string>
#include <unistd.h>
using namespace frameyap;
namespace {
@@ -14,7 +16,7 @@ SurfaceEvent click(PanelSurface& surface, float x, float y, unsigned cursor = 0)
}
void no_action(const SurfaceEvent& event) {
assert(!event.action && !event.mount && !event.lasers_anytime && !event.advanced_debug && !event.auto_insert &&
!event.close_mic_when_idle && !event.lock_layout && !event.clock_24h && !event.date_format && !event.recenter && !event.open_bindings && !event.model_action);
!event.close_mic_when_idle && !event.gradient_enabled && !event.lock_layout && !event.wrist_world_fallback && !event.clock_24h && !event.date_format && !event.recenter && !event.open_bindings && !event.model_action && !event.launch_companion && !event.check_updates && !event.install_update);
}
void snapshot(PanelSurface& surface, const std::string& path) {
std::ofstream out(path, std::ios::binary);
@@ -27,13 +29,14 @@ void snapshot(PanelSurface& surface, const std::string& path) {
}
int main(int argc, char** argv) {
assert(argc >= 2);
// Interaction tests use the opt-out path; animation has a deterministic
// injected-clock suite below rather than wall-time-sensitive assertions.
PanelSurface surface(argv[1], Mount::World, {}, {.enabled = false});
// Ordinary UI uses the default static path; animation has an explicitly
// opted-in injected-clock suite below.
PanelSurface surface(argv[1], Mount::World);
Panel p{"Ready to record", "", "Focus your destination before Type. Enter is always separate.", false, false};
assert(surface.render(p));
assert(surface.pixels().size() == size_t(PanelSurface::width * PanelSurface::height * 4));
assert(!surface.render(p));
assert(!surface.render(p, PanelSurface::Clock::now() + std::chrono::seconds(60))); // default: no timed redraw
surface.set_clock_time(std::time_t{1704211440}); // local clock changes only when rendered minute changes
assert(surface.render(p));
assert(!surface.render(p));
@@ -138,7 +141,7 @@ int main(int argc, char** argv) {
// civil clock, and full cycles return byte-identical pixels (including alpha).
using namespace std::chrono_literals;
const auto epoch = PanelSurface::Clock::time_point{};
PanelSurface animated(argv[1], Mount::World, gradient_theme);
PanelSurface animated(argv[1], Mount::World, gradient_theme, {.enabled = true});
animated.set_clock_time(std::time_t{1704211440});
assert(animated.render(p, epoch));
const auto first = animated.pixels();
@@ -185,14 +188,14 @@ int main(int argc, char** argv) {
assert(animated.render(p, epoch + 41s));
assert(animated.pointer_up(0, 100, 610, epoch + 41s).action == UiAction::BeginRecord);
assert(!gradient_surface.render(p, epoch + 300s)); // disabled has no timed redraws
PanelSurface faster(argv[1], Mount::World, gradient_theme, {.period_seconds = 10.f, .strength = 0.f});
PanelSurface faster(argv[1], Mount::World, gradient_theme, {.enabled = true, .period_seconds = 10.f, .strength = 0.f});
assert(faster.render(p, epoch));
const auto fast_first = faster.pixels();
assert(rgb(fast_first, 500, 190) == gradient_theme.background);
assert(faster.render(p, epoch + 2500ms) && faster.pixels() != fast_first);
assert(faster.render(p, epoch + 10s) && faster.pixels() == fast_first);
if (argc >= 3) {
PanelSurface preview(argv[1], Mount::World);
PanelSurface preview(argv[1], Mount::World, {}, {.enabled = true});
preview.render(p, epoch);
snapshot(preview, std::string(argv[2]) + "-gradient.ppm");
}
@@ -327,6 +330,77 @@ int main(int argc, char** argv) {
surface.set_close_mic_when_idle(true); assert(surface.render(p));
assert(click(surface, 200, 532).close_mic_when_idle == false);
surface.set_close_mic_when_idle(false); assert(surface.render(p));
// The Settings viewport scrolls continuously under the laser; it never pages.
assert(!surface.scroll(290, 160, -1.f)); // tab/header, not settings content
assert(!surface.scroll(200, 400, std::numeric_limits<float>::quiet_NaN()));
surface.pointer_down(0, 200, 532); // pending mic press must not activate after movement
assert(surface.scroll(200, 400, -.5f));
no_action(surface.pointer_up(0, 200, 532));
assert(surface.render(p));
assert(!surface.scroll(200, 610, -1.f)); // fixed footer
assert(surface.scroll(200, 400, -2.f));
assert(surface.render(p));
no_action(click(surface, 180, 231)); // scrolled settings cannot be clicked above the viewport
using namespace std::chrono_literals;
const auto tick = PanelSurface::Clock::time_point{};
const auto static_background = surface.pixels()[pixel];
const auto preference = std::filesystem::temp_directory_path() /
("frameyap-gradient-ui-" + std::to_string(::getpid()) + ".json");
auto animated_background = click(surface, 200, 470); // y=574 minus 120 scroll
assert(animated_background.gradient_enabled == true && !animated_background.action);
assert(!surface.render(p)); // caller must apply the change
assert(save_gradient_enabled(preference, *animated_background.gradient_enabled));
assert(load_config(preference).gradient.enabled);
PanelSurface restored(argv[1], Mount::World, {}, load_config(preference).gradient);
restored.set_clock_time(std::time_t{1704211500});
assert(restored.render(p, tick) && restored.pixels()[pixel] != static_background);
surface.set_gradient_enabled(true);
assert(surface.render(p, tick));
assert(surface.pixels()[pixel] != static_background);
if (argc >= 3) snapshot(surface, std::string(argv[2]) + "-settings-animated.ppm");
assert(!surface.render(p, tick + 99ms));
assert(surface.render(p, tick + 100ms));
surface.pointer_down(1, 200, 470);
surface.set_gradient_enabled(false);
assert(surface.render(p, tick + 200ms));
no_action(surface.pointer_up(1, 200, 470)); // old press cannot re-enable it
assert(surface.pixels()[pixel] == static_background);
assert(!surface.render(p, tick + 300s)); // off: no timed redraws
animated_background = click(surface, 200, 470);
assert(animated_background.gradient_enabled == true);
assert(save_gradient_enabled(preference, false));
assert(!load_config(preference).gradient.enabled);
std::filesystem::remove(preference);
assert(surface.scroll(200, 400, -.5f)); // reveal wrist fallback below the new switch
assert(surface.render(p));
auto fallback = click(surface, 200, 500);
assert(fallback.wrist_world_fallback == false && !fallback.action);
assert(!surface.render(p));
surface.set_wrist_world_fallback(false); assert(surface.render(p));
fallback = click(surface, 200, 500);
assert(fallback.wrist_world_fallback == true);
surface.set_wrist_world_fallback(true); assert(surface.render(p));
assert(surface.scroll(200, 400, -2.f)); // reveal bottom update row and status
assert(surface.render(p));
assert(!surface.scroll(200, 400, -1.f)); // bottom clamp
no_action(click(surface, 700, 486)); // no install button until a newer release is checked
assert(click(surface, 180, 486).check_updates);
surface.set_update_status(UpdateStatus::Checking); assert(surface.render(p));
no_action(click(surface, 180, 486)); no_action(click(surface, 700, 486));
surface.set_update_status(UpdateStatus::Available, "0.1.202609292232"); assert(surface.render(p));
if (argc >= 3) snapshot(surface, std::string(argv[2]) + "-updates.ppm");
assert(click(surface, 700, 486).install_update);
surface.set_update_status(UpdateStatus::Current); assert(surface.render(p));
no_action(click(surface, 700, 486));
assert(click(surface, 180, 486).check_updates);
surface.set_update_status(UpdateStatus::Failed); assert(surface.render(p));
no_action(click(surface, 700, 486));
assert(click(surface, 280, 610).action == UiAction::Cancel); // footer stays fixed
assert(surface.scroll(200, 400, 4.f));
assert(surface.scroll(200, 400, 1.f));
assert(surface.render(p));
assert(!surface.scroll(200, 400, 1.f)); // top clamp
assert(click(surface, 180, 260).mount == Mount::World);
assert(click(surface, 280, 610).action == UiAction::Cancel);
surface.set_placement_note("Wrist not tracked - using world space until it returns.");
assert(surface.render(p)); assert(!surface.render(p));
@@ -344,6 +418,8 @@ int main(int argc, char** argv) {
no_action(click(surface, 200, 492)); // clock/date controls only exist on Settings
no_action(click(surface, 680, 492));
no_action(click(surface, 200, 532)); // mic preference only exists on Settings
no_action(click(surface, 200, 470)); // animation toggle only exists on Settings
assert(!surface.scroll(200, 400, -1.f)); // short Review needs no scrolling
// Bindings opens SteamVR directly, from either tab, without replacing review.
surface.pointer_down(1, 480, 610);
@@ -366,29 +442,122 @@ int main(int argc, char** argv) {
assert(click(surface, 680, 260).mount == Mount::Head); // still in Settings
no_action(click(surface, 100, 160));
// Long UTF-8, newlines and malformed bytes are bounded, paginated and navigable.
// Only installed companions occupy the review row, in a stable order.
no_action(click(surface, 90, 518)); no_action(click(surface, 900, 518));
surface.set_companions(true, false, false); assert(surface.render(p));
assert(click(surface, 90, 518).launch_companion == SurfaceEvent::Companion::Plan);
surface.set_companions(false, true, false); assert(surface.render(p));
assert(click(surface, 90, 518).launch_companion == SurfaceEvent::Companion::Keyboard);
surface.set_companions(false, false, true); assert(surface.render(p));
assert(click(surface, 90, 518).launch_companion == SurfaceEvent::Companion::Draw);
surface.set_companions(true, true, false); assert(surface.render(p));
assert(click(surface, 90, 518).launch_companion == SurfaceEvent::Companion::Plan);
assert(click(surface, 900, 518).launch_companion == SurfaceEvent::Companion::Keyboard);
surface.set_companions(true, true, true); assert(surface.render(p));
assert(click(surface, 90, 518).launch_companion == SurfaceEvent::Companion::Plan);
assert(click(surface, 480, 518).launch_companion == SurfaceEvent::Companion::Keyboard);
assert(click(surface, 900, 518).launch_companion == SurfaceEvent::Companion::Draw);
{
using namespace std::chrono_literals;
const auto start = PanelSurface::Clock::now();
// A short release explicitly shows; the boundary and either laser cursor
// launch exactly one recenter, never a subsequent show on release.
for (unsigned cursor : {0u, 1u}) {
surface.pointer_down(cursor, 480, 518, start);
assert(surface.render(p, start)); // visible hold cue
assert(!surface.render(p, start + 99ms)); // no idle/full-speed redraw
assert(surface.render(p, start + 100ms));
assert(!surface.poll_keyboard_hold(start + 799ms));
assert(surface.pointer_up(cursor, 480, 518, start + 799ms).launch_companion ==
SurfaceEvent::Companion::Keyboard);
surface.pointer_down(cursor, 480, 518, start);
assert(!surface.poll_keyboard_hold(start + 799ms));
assert(surface.poll_keyboard_hold(start + 800ms) == SurfaceEvent::Companion::KeyboardRecenter);
assert(!surface.poll_keyboard_hold(start + 800ms));
assert(!surface.poll_keyboard_hold(start + 5s));
no_action(surface.pointer_up(cursor, 480, 518, start + 5s));
no_action(surface.pointer_up(cursor, 480, 518, start + 5s));
surface.pointer_down(cursor, 480, 518, start);
assert(surface.pointer_up(cursor, 480, 518, start + 800ms).launch_companion ==
SurfaceEvent::Companion::KeyboardRecenter); // release on threshold before polling
}
surface.pointer_down(0, 480, 518, start);
surface.pointer_down(0, 480, 518, start + 500ms); // duplicate down must not restart the clock
assert(surface.poll_keyboard_hold(start + 800ms) == SurfaceEvent::Companion::KeyboardRecenter);
no_action(surface.pointer_up(0, 480, 518, start + 800ms));
surface.pointer_down(0, 480, 518, start);
surface.pointer_down(1, 480, 518, start);
surface.pointer_move(0, 300, 518); // one cursor leaving cannot cancel the other
assert(surface.poll_keyboard_hold(start + 800ms) == SurfaceEvent::Companion::KeyboardRecenter);
assert(!surface.poll_keyboard_hold(start + 800ms));
no_action(surface.pointer_up(0, 480, 518, start + 800ms));
no_action(surface.pointer_up(1, 480, 518, start + 800ms));
surface.pointer_down(0, 480, 518, start);
surface.pointer_move(0, 300, 518); // leaving cancels even if pointer comes back
surface.pointer_move(0, 480, 518);
assert(!surface.poll_keyboard_hold(start + 5s));
no_action(surface.pointer_up(0, 480, 518, start + 5s));
surface.pointer_down(1, 480, 518, start);
no_action(surface.pointer_up(1, 900, 518, start + 5s)); // release outside
assert(!surface.poll_keyboard_hold(start + 5s));
surface.pointer_down(0, 480, 518, start);
surface.reset_pointers(); // tracking/focus loss, hiding, relocation
assert(!surface.poll_keyboard_hold(start + 5s));
no_action(surface.pointer_up(0, 480, 518, start + 5s));
surface.pointer_down(1, 480, 518, start);
no_action(click(surface, 290, 160)); // changing tab revokes other cursor
assert(!surface.poll_keyboard_hold(start + 5s));
no_action(surface.pointer_up(1, 480, 518, start + 5s));
no_action(click(surface, 90, 160)); // back to Review
surface.pointer_down(0, 480, 518, start);
surface.set_companions(true, false, true);
assert(!surface.poll_keyboard_hold(start + 5s));
no_action(surface.pointer_up(0, 480, 518, start + 5s));
surface.set_companions(true, true, true);
no_action(surface.pointer_up(1, 480, 518, start + 5s)); // no matching left down (e.g. right mouse)
assert(!surface.poll_keyboard_hold(start + 5s));
surface.pointer_down(0, 90, 518, start); // other companions never turn into recenter
assert(!surface.poll_keyboard_hold(start + 5s));
assert(surface.pointer_up(0, 90, 518, start + 5s).launch_companion == SurfaceEvent::Companion::Plan);
surface.pointer_down(1, 900, 518, start);
assert(!surface.poll_keyboard_hold(start + 5s));
assert(surface.pointer_up(1, 900, 518, start + 5s).launch_companion == SurfaceEvent::Companion::Draw);
assert(surface.render(p, start + 5s));
assert(!surface.render(p, start + 5s)); // no perpetual hold repaint
if (argc >= 3) {
surface.pointer_down(0, 480, 518, start + 6s);
assert(surface.render(p, start + 6s + 400ms));
snapshot(surface, std::string(argv[2]) + "-keyboard-hold.ppm");
surface.reset_pointers();
}
}
p.quick_open = true; assert(surface.render(p));
no_action(click(surface, 90, 518)); no_action(click(surface, 480, 518)); no_action(click(surface, 900, 518));
assert(!surface.scroll(200, 400, -1.f));
p.quick_open = false; assert(surface.render(p));
surface.set_companions(false, false, false); assert(surface.render(p));
// Long UTF-8, newlines and malformed bytes scroll within the preview.
p.transcript.clear();
for (int i = 0; i < 60; ++i) p.transcript += "Line " + std::to_string(i) + ": café / 日本語 / naïve / Ω\n";
p.transcript += std::string("invalid: \xff\xc0\xe0\x80", 13);
surface.render(p);
surface.reset_pointers(); surface.render(p);
const auto first = surface.pixels();
no_action(click(surface, 900, 518));
surface.reset_pointers(); assert(surface.render(p));
assert(surface.pixels() != first);
no_action(click(surface, 90, 518));
surface.reset_pointers(); surface.render(p);
assert(surface.pixels() == first);
for (int i = 0; i < 200; ++i) { no_action(click(surface, 900, 518)); surface.render(p); }
for (int i = 0; i < 200; ++i) { no_action(click(surface, 90, 518)); surface.render(p); }
surface.reset_pointers(); surface.render(p);
assert(surface.pixels() == first);
assert(!surface.scroll(200, 610, -1.f)); // fixed footer cannot scroll
assert(!surface.scroll(200, 400, std::numeric_limits<float>::quiet_NaN()));
surface.pointer_down(0, 100, 160);
assert(surface.scroll(200, 400, -.5f)); // partial line and stale press
no_action(surface.pointer_up(0, 100, 160));
assert(surface.render(p) && surface.pixels() != first);
assert(surface.scroll(200, 400, .5f));
assert(surface.render(p) && surface.pixels() == first);
assert(!surface.scroll(200, 400, 1.f)); // top clamp
for (int i = 0; i < 200; ++i) { surface.scroll(200, 400, -4.f); surface.render(p); }
assert(!surface.scroll(200, 400, -1.f)); // bottom clamp
assert(surface.scroll(200, 400, 4.f));
p.transcript = "New result";
assert(surface.render(p));
surface.reset_pointers(); surface.render(p);
auto replaced = surface.pixels();
no_action(click(surface, 900, 518));
surface.reset_pointers(); surface.render(p);
assert(surface.render(p)); // new text resets scroll
const auto replaced = surface.pixels();
assert(!surface.scroll(200, 400, -1.f));
assert(surface.pixels() == replaced);
p.status = std::string(4096, 's'); p.detail = std::string(4096, 'd');
assert(surface.render(p)); assert(!surface.render(p));
@@ -518,5 +687,24 @@ int main(int argc, char** argv) {
model_panel.model_busy = true;
chooser.render(model_panel);
no_action(chooser.pointer_up(0, 700, 546)); // change invalidates approval
// About: reached only from Settings; display-only, recording stays usable.
PanelSurface about(argv[1], Mount::World, {}, {.enabled = false});
Panel about_panel{"Ready", "", "", true, false};
about.render(about_panel);
no_action(click(about, 420, 400)); // hidden on Review
assert(!about.render(about_panel));
no_action(click(about, 290, 160)); about.render(about_panel); // Settings
const auto settings_pixels = about.pixels();
no_action(click(about, 420, 400)); // About
assert(about.render(about_panel));
const auto about_pixels = about.pixels();
assert(about_pixels != settings_pixels);
no_action(click(about, 100, 250)); // Settings' World-space button is not live under About
no_action(click(about, 700, 400)); // nor Lasers anytime
about.set_version("0.1.202609261200");
assert(about.render(about_panel) && about.pixels() != about_pixels);
assert(click(about, 100, 610).action == UiAction::BeginRecord);
no_action(click(about, 290, 160)); about.render(about_panel); // Settings returns
assert(about.pixels() == settings_pixels);
std::cout << "panel checks passed (no OpenVR, microphone or input injection)\n";
}
+100
View File
@@ -0,0 +1,100 @@
#include "placement.hpp"
#include "panel_drag.hpp"
#include <cmath>
#include <cstdlib>
#include <filesystem>
#include <fstream>
#include <iostream>
#include <iterator>
#include <limits>
#include <string>
#include <unistd.h>
using namespace frameyap;
namespace fs = std::filesystem;
#define CHECK(x) do { if (!(x)) { std::cerr << "line " << __LINE__ << ": " #x "\n"; std::exit(1); } } while (0)
namespace {
constexpr Matrix34 identity{{{{1.f, 0.f, 0.f, 0.f}},
{{0.f, 1.f, 0.f, 0.f}},
{{0.f, 0.f, 1.f, 0.f}}}};
void put(const fs::path& path, const std::string& bytes) {
std::ofstream file(path, std::ios::binary | std::ios::trunc);
CHECK(bool(file)); file << bytes; CHECK(bool(file));
}
std::string get(const fs::path& path) {
std::ifstream file(path, std::ios::binary);
return {std::istreambuf_iterator<char>(file), {}};
}
bool near(float a, float b) { return std::abs(a - b) < 1e-6f; }
} // namespace
int main() {
std::string pattern = (fs::temp_directory_path() / "frameyap-placement-test-XXXXXX").string();
CHECK(::mkdtemp(pattern.data()) != nullptr);
const fs::path dir = pattern;
::setenv("XDG_CONFIG_HOME", dir.c_str(), 1);
const auto left = default_placement_path(Mount::LeftWrist);
const auto right = default_placement_path(Mount::RightWrist);
const auto head = default_placement_path(Mount::Head);
CHECK(left == dir / "frameyap/placement-left-wrist");
CHECK(right == dir / "frameyap/placement-right-wrist");
CHECK(head == dir / "frameyap/placement-head");
CHECK(default_placement_path(Mount::World).empty());
CHECK(!load_relative_placement(left, Mount::LeftWrist));
RelativePlacement placed{relative_mount_pose(Mount::LeftWrist), 1.37f};
// A freely rotated canvas with both depth and lateral offsets, rather than
// only the editable roll angle in config.json.
const auto turn = relative_mount_pose(Mount::RightWrist);
placed.canvas_pose = compose_pose(placed.canvas_pose, turn);
placed.canvas_pose[0][3] += .31f;
placed.canvas_pose[1][3] -= .14f;
placed.canvas_pose[2][3] += .22f;
CHECK(valid_relative_placement(placed));
CHECK(save_relative_placement(left, Mount::LeftWrist, placed));
CHECK((fs::status(left).permissions() & fs::perms::group_read) == fs::perms::none);
auto restored = load_relative_placement(left, Mount::LeftWrist);
CHECK(restored && near(restored->scale, placed.scale));
for (int r = 0; r < 3; ++r) for (int c = 0; c < 4; ++c)
CHECK(near(restored->canvas_pose[r][c], placed.canvas_pose[r][c]));
CHECK(!load_relative_placement(left, Mount::RightWrist));
CHECK(!load_relative_placement(right, Mount::RightWrist));
CHECK(save_relative_placement(right, Mount::RightWrist, RelativePlacement{relative_mount_pose(Mount::RightWrist), .7f}));
CHECK(load_relative_placement(right, Mount::RightWrist)->scale == .7f);
CHECK(load_relative_placement(left, Mount::LeftWrist)->scale == placed.scale);
CHECK(save_relative_placement(head, Mount::Head, RelativePlacement{relative_mount_pose(Mount::Head), 2.f}));
CHECK(load_relative_placement(head, Mount::Head)->scale == 2.f);
CHECK(!save_relative_placement(left, Mount::World, placed));
const auto good = get(left);
for (const auto& bad : {std::string("frameyap-placement-v2 left-wrist 1\n"),
good + "junk", std::string(513, 'x')}) {
put(left, bad);
CHECK(!load_relative_placement(left, Mount::LeftWrist));
}
put(left, good);
auto invalid = placed;
invalid.scale = 0.f;
CHECK(!save_relative_placement(left, Mount::LeftWrist, invalid));
invalid = placed; invalid.scale = std::numeric_limits<float>::quiet_NaN();
CHECK(!save_relative_placement(left, Mount::LeftWrist, invalid));
invalid = placed; invalid.canvas_pose[0][0] *= 1.1f;
CHECK(!save_relative_placement(left, Mount::LeftWrist, invalid));
invalid = placed; invalid.canvas_pose[2][3] = 6.f;
CHECK(!save_relative_placement(left, Mount::LeftWrist, invalid));
CHECK(get(left) == good); // invalid writes cannot corrupt a prior placement
const auto linked = dir / "linked-placement";
fs::create_symlink(left, linked);
CHECK(!load_relative_placement(linked, Mount::LeftWrist));
CHECK(!save_relative_placement(linked, Mount::LeftWrist, placed));
CHECK(get(left) == good);
fs::create_directory(dir / "directory-placement");
CHECK(!save_relative_placement(dir / "directory-placement", Mount::Head, placed));
fs::create_directory_symlink(left.parent_path(), dir / "linked-directory");
CHECK(!save_relative_placement(dir / "linked-directory/placement-left-wrist", Mount::LeftWrist, placed));
fs::remove_all(dir);
std::cout << "relative placement persistence checks passed\n";
}
+14 -2
View File
@@ -19,9 +19,9 @@ class ActionBindingTests(unittest.TestCase):
path = f"/user/hand/right/input/{button}"
self.assertIn({"name": name, "type": "boolean"}, manifest["actions"])
self.assertEqual(config["buttons"][action], path)
self.assertEqual([s["path"] for s in sources if s["inputs"]["click"]["output"] == name], [path])
self.assertEqual([s["path"] for s in sources if s["inputs"].get("click", {}).get("output") == name], [path])
self.assertEqual(config["buttons"]["enter"], "")
self.assertFalse(any(s["inputs"]["click"]["output"] == "/actions/frameyap/in/enter" for s in sources))
self.assertFalse(any(s["inputs"].get("click", {}).get("output") == "/actions/frameyap/in/enter" for s in sources))
self.assertEqual(config["quick_inputs"], ["/new", "/questions", "/help"])
def test_right_x_is_hold_to_talk(self):
@@ -49,6 +49,18 @@ class ActionBindingTests(unittest.TestCase):
), 1)
self.assertFalse(any(source["path"].endswith("/input/trigger") for source in sources))
def test_grab_depth_axes_are_hand_specific(self):
manifest = json.loads((ROOT / "assets/actions.json").read_text())
for binding_file in ("bindings_frame_controller.json", "bindings_knuckles.json"):
sources = json.loads((ROOT / "assets" / binding_file).read_text())["bindings"]["/actions/grab"]["sources"]
for hand in ("left", "right"):
name = f"/actions/grab/in/{hand}_depth"
path = f"/user/hand/{hand}/input/thumbstick"
self.assertIn({"name": name, "type": "vector2"}, manifest["actions"])
self.assertEqual([s for s in sources if s["path"] == path], [{
"path": path, "mode": "joystick", "inputs": {"position": {"output": name}},
}])
if __name__ == "__main__":
unittest.main()
+9 -2
View File
@@ -13,14 +13,17 @@ from frameyap.model_files import DEFAULT_MANIFEST_DIR, ManifestError, load_backe
class BackendTests(unittest.TestCase):
def test_redux_manifest_preserves_pinned_artifacts_and_launcher_contract(self):
redux = load_backends()["redux"]
self.assertEqual(redux.revision, "fad622f25f303105c20d70e201bcc477c88b620c")
self.assertEqual(redux.revision, "2bf128600aac4b16946f7ed8372e56117fe5e23b")
self.assertEqual({f.path: (f.size, f.sha256) for f in redux.files}, {
"model.safetensors": (177774490, "78ec25733ee0d0c1586d1346fc86db9d0c2e436e3a8ab1d32a82d1bb8f848d21"),
"config.json": (12988, "503c653b2e3bb788adbcb04f5abdee532d958686564081baeed133ff10143f6e"),
"ternary.json": (57970, "1221c6d3ce901ffe09c089da758a8db8b76189f80cff41c5afc244fc61e2051d"),
"tokenizer.json": (1159960, "bd321b096832a3f270bd3b2a88823957920f1a5c5ada71114a26ea729d0cbe91"),
"README.md": (8533, "a8b327f983a8b8ff262ff7bead3a791fbed9350632002af8db85ab5cd84cdaa5"),
"README.md": (8832, "930055e6bac7d75114e719850ffaffc8d327e73b4295518c38c81399acd63e46"),
})
self.assertEqual(redux.files[-1].compatible,
((8533, "a8b327f983a8b8ff262ff7bead3a791fbed9350632002af8db85ab5cd84cdaa5"),))
self.assertTrue(all(not file.compatible for file in redux.files[:-1]))
self.assertEqual(redux.launcher["protocol"], "frameyap-worker-v1")
self.assertEqual(redux.launcher["arguments"], ["--model", "{model_dir}", "--threads", "{threads}", "--clip-dir", "{clip_dir}"])
self.assertIn("CC-BY-4.0", redux.license_id)
@@ -47,6 +50,10 @@ class BackendTests(unittest.TestCase):
lambda f: f["model"]["files"][0].update(path="../secret"),
lambda f: f["model"]["files"][0].update(size=True),
lambda f: f["model"]["files"][0].update(sha256="A" * 64),
lambda f: f["model"]["files"][0].update(compatible="bad"),
lambda f: f["model"]["files"][0].update(compatible=[{"size": True, "sha256": "a" * 64}]),
lambda f: f["model"]["files"][0].update(compatible=[{"size": 3, "sha256": "a" * 64}]),
lambda f: f["model"]["files"][0].update(compatible=[{"size": 4, "sha256": "a" * 64, "path": "other"}]),
lambda f: f["launcher"]["arguments"].append("{unknown}"),
lambda f: f["launcher"].update(path="/bin/sh"),
lambda f: f["launcher"].update(protocol="not-the-wire-protocol"),
+35
View File
@@ -0,0 +1,35 @@
"""Offline contract for isolated, stick-only grab bindings."""
import json
from pathlib import Path
import unittest
ROOT = Path(__file__).resolve().parents[1]
class GrabBindingTests(unittest.TestCase):
def test_grab_set_claims_only_each_hands_stick(self):
manifest = json.loads((ROOT / 'assets/actions.json').read_text())
self.assertIn({'name': '/actions/grab', 'usage': 'single'}, manifest['action_sets'])
actions = {a['name']: a['type'] for a in manifest['actions']}
for filename in ('bindings_frame_controller.json', 'bindings_knuckles.json'):
bindings = json.loads((ROOT / 'assets' / filename).read_text())['bindings']
grab = bindings['/actions/grab']
self.assertEqual(set(grab), {'sources'})
self.assertEqual({s['path'] for s in grab['sources']}, {
'/user/hand/left/input/thumbstick', '/user/hand/right/input/thumbstick'})
self.assertEqual(len(grab['sources']), 2)
for source in grab['sources']:
self.assertEqual(source['mode'], 'joystick')
self.assertEqual(set(source['inputs']), {'position'})
output = source['inputs']['position']['output']
self.assertTrue(output.startswith('/actions/grab/in/'))
self.assertEqual(actions[output], 'vector2')
for name, binding in bindings.items():
if name == '/actions/grab':
continue
for source in binding.get('sources', []):
self.assertNotIn('position', source['inputs'])
if __name__ == '__main__':
unittest.main()
+69 -1
View File
@@ -87,8 +87,9 @@ class InstallTests(unittest.TestCase):
self.assertIs(json.loads(config.read_text())["close_mic_when_idle"], False)
self.assertEqual(json.loads(config.read_text())["backend"], "redux")
self.assertEqual(json.loads(config.read_text())["gradient"],
{"enabled": True, "period_seconds": 30, "strength": 0.12})
{"enabled": False, "period_seconds": 30, "strength": 0.12})
self.assertIs(json.loads(config.read_text())["lock_layout"], False)
self.assertIs(json.loads(config.read_text())["wrist_world_fallback"], True)
self.assertEqual(list(config.parent.glob("config.json.backup-*")), [])
self.assertIn("PYTHONDONTWRITEBYTECODE=1", launcher.read_text())
self.assertTrue(os.access(root / "versions/0.1.202609241530/runtime/bin/helper", os.X_OK))
@@ -186,6 +187,7 @@ class InstallTests(unittest.TestCase):
fixed["close_mic_when_idle"] = True
fixed["backend"] = "custom_v2-1"
fixed["lock_layout"] = True
fixed["wrist_world_fallback"] = False
fixed["clock_24h"] = True
fixed["date_format"] = "iso"
fixed["buttons"]["enter"] = "" # intentional disabling survives upgrades
@@ -200,6 +202,7 @@ class InstallTests(unittest.TestCase):
self.assertIs(json.loads(config.read_text())["close_mic_when_idle"], True)
self.assertEqual(json.loads(config.read_text())["backend"], "custom_v2-1")
self.assertIs(json.loads(config.read_text())["lock_layout"], True)
self.assertIs(json.loads(config.read_text())["wrist_world_fallback"], False)
self.assertIs(json.loads(config.read_text())["clock_24h"], True)
self.assertEqual(json.loads(config.read_text())["date_format"], "iso")
self.assertEqual(len(list(config.parent.glob("config.json.backup-*"))), 1)
@@ -221,6 +224,7 @@ class InstallTests(unittest.TestCase):
self.assertIs(fixed["advanced_debug"], False)
self.assertIs(fixed["auto_insert"], False)
self.assertIs(fixed["lock_layout"], False)
self.assertIs(fixed["wrist_world_fallback"], True)
self.assertEqual(fixed["wrist"]["x"], 0.04)
self.assertEqual(fixed["wrist"]["y"], 0.18)
self.assertEqual(fixed["wrist"]["width"], 0.30)
@@ -251,6 +255,7 @@ class InstallTests(unittest.TestCase):
config = self.home / ".config/frameyap/config.json"
config.parent.mkdir(parents=True)
defaults = installer.CONFIG_DEFAULTS["gradient"]
self.assertIs(defaults["enabled"], False)
# Older configs receive defaults; a non-object section is replaced.
for value in (None, [], "animated", 1):
@@ -273,6 +278,12 @@ class InstallTests(unittest.TestCase):
self.assertEqual(json.loads(config.read_text())["gradient"],
{"enabled": False, "period_seconds": 30, "strength": 0.12})
self.assertIn(original, [p.read_bytes() for p in config.parent.glob("config.json.backup-*")])
original = b'{"gradient":{"enabled":true},"font":"/custom.ttf"}'
config.write_bytes(original)
self.install("0.1.202609241530", archive, digest)
self.assertEqual(json.loads(config.read_text())["gradient"],
{"enabled": True, "period_seconds": 30, "strength": 0.12})
self.assertIn(original, [p.read_bytes() for p in config.parent.glob("config.json.backup-*")])
# A complete valid section, including numeric boundaries and explicit
# disabled state, retains exact user bytes and gets no new backup.
@@ -350,6 +361,12 @@ class InstallTests(unittest.TestCase):
self.assertEqual(fixed["font"], "/custom/font.ttf")
self.assertIn(original, [p.read_bytes() for p in config.parent.glob("config.json.backup-*")])
def test_wrist_world_fallback_normalization(self):
self.assertIs(installer.normalized_config({})["wrist_world_fallback"], True)
self.assertIs(installer.normalized_config({"wrist_world_fallback": False})["wrist_world_fallback"], False)
for invalid in ("false", 0, None, [], {}):
self.assertIs(installer.normalized_config({"wrist_world_fallback": invalid})["wrist_world_fallback"], True)
def test_new_config_fields_repair_only_invalid_values(self):
archive, digest = self.package("0.1.202609241530")
config = self.home / ".config/frameyap/config.json"
@@ -1206,6 +1223,57 @@ with patch.object(module, "check_host"), patch.object(module.urllib.request, "ur
self.assertIn("mismatched", json.loads(output.getvalue())["message"])
fetch.assert_not_called()
def test_model_card_upgrade_retains_verified_legacy_without_network(self):
shutil.copyfile(REPO / "python/frameyap/model_files.py",
self.stage / "python/frameyap/model_files.py")
manifest = json.loads((REPO / "assets/backends/redux.json").read_text())
old, new = b"previous attributed card", b"updated attributed model card"
def pin(content):
return {"size": len(content), "sha256": hashlib.sha256(content).hexdigest()}
manifest["model"]["files"] = [{"path": "README.md", **pin(old)}]
manifests = self.stage / "assets/backends"
manifests.mkdir()
card_manifest = manifests / "redux.json"
card_manifest.write_text(json.dumps(manifest))
archive, digest = self.package("0.1.202609241530")
self.install("0.1.202609241530", archive, digest, "--without-model")
destination = self.base / "models"
destination.mkdir()
card = destination / "README.md"
card.write_bytes(old)
manifest["model"]["files"] = [{"path": "README.md", **pin(new),
"compatible": [pin(old)]}]
card_manifest.write_text(json.dumps(manifest))
archive, digest = self.package("0.1.202609241531")
with patch.object(installer.urllib.request, "urlopen") as fetch:
self.install("0.1.202609241531", archive, digest, "--without-model")
module, backend, _ = installer.installed_backend(self.data / "frameyap", "redux")
self.assertEqual(module.check_model(backend, destination)["state"], "installed_verified")
argv = ["--install-model", "--model-dir", str(destination), "--yes", "--json"]
with contextlib.redirect_stdout(io.StringIO()):
self.assertEqual(installer.cli(argv), 0)
fetch.assert_not_called()
self.assertEqual(card.read_bytes(), old)
card.write_bytes(b"x" * len(old))
with contextlib.redirect_stdout(io.StringIO()):
self.assertEqual(installer.cli(argv), 1)
fetch.assert_not_called()
card.unlink()
class Response(io.BytesIO):
def geturl(self):
return "https://huggingface.co/fixture"
# Fresh downloads require the current card even though the old one is
# permitted for already installed, independently hashed model files.
with patch.object(installer.urllib.request, "urlopen", return_value=Response(old)):
with contextlib.redirect_stdout(io.StringIO()):
self.assertEqual(installer.cli(argv), 1)
self.assertFalse(card.exists())
with patch.object(installer.urllib.request, "urlopen", return_value=Response(new)) as fetch:
with contextlib.redirect_stdout(io.StringIO()):
self.assertEqual(installer.cli(argv), 0)
fetch.assert_called_once()
self.assertEqual(card.read_bytes(), new)
def test_runtime_install_is_explicit_cpu_pinned_and_updates_only_python_path(self):
# subprocess is mocked: no venv, pip or network is touched.
root = self.data / "frameyap"
+23
View File
@@ -74,6 +74,29 @@ class ModelFileTests(unittest.TestCase):
(model / "nested").symlink_to(root)
self.assertEqual(check_model(backend, model)["reason"], "unsafe_file")
def test_compatible_card_still_requires_exact_hash_and_safe_file(self):
with tempfile.TemporaryDirectory() as path:
root = Path(path)
_, backend = self.fixture(root)
current, previous = b"current card", b"old card"
item = ModelFile("README.md", len(current), hashlib.sha256(current).hexdigest(),
((len(previous), hashlib.sha256(previous).hexdigest()),))
backend = replace(backend, files=(item,))
card = root / "README.md"
for content in (current, previous):
card.write_bytes(content)
self.assertEqual(check_model(backend, root)["state"], "installed_verified")
# A download temp pin must never accept an older artifact.
self.assertEqual(check_file(root, ModelFile(item.path, item.size, item.sha256))[0],
"size_mismatch")
card.write_bytes(b"bad card")
self.assertEqual(check_model(backend, root)["reason"], "hash_mismatch")
card.unlink()
outside = root / "outside"
outside.write_bytes(previous)
card.symlink_to(outside)
self.assertEqual(check_model(backend, root)["reason"], "unsafe_file")
def test_second_backend_status_without_runtime_changes(self):
with tempfile.TemporaryDirectory() as path:
root = Path(path)
+79
View File
@@ -0,0 +1,79 @@
"""Offline checks for the manual release lookup and attended installer handoff."""
import importlib.util
import io
from pathlib import Path
import tempfile
import unittest
from unittest.mock import patch
ROOT = Path(__file__).resolve().parents[1]
def load(name, filename):
spec = importlib.util.spec_from_file_location(name, ROOT / 'scripts' / filename)
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
class UpdateTests(unittest.TestCase):
def test_tags(self):
check = load('frameyap_update', 'check-update.py')
current = '0.1.202609282232'
self.assertEqual(check.outcome('{"tag_name":"v0.1.202609292232"}', current),
'AVAILABLE 0.1.202609292232')
for tag in ('v0.1.202609282232', 'v0.1.202609272232'):
self.assertEqual(check.outcome('{"tag_name":"' + tag + '"}', current), 'CURRENT')
for tag in ('v0.1.202613292232', '0.1.202609292232', 'v0.1.202609292232;sh',
'v0.1.20260929', '../v0.1.202609292232'):
with self.assertRaises(ValueError, msg=tag):
check.outcome('{"tag_name":"' + tag + '"}', current)
with self.assertRaises(ValueError):
check.outcome('{"tag_name":"v0.1.202609292232"}', '0.1.invalid')
def test_network_only_on_explicit_check(self):
check = load('frameyap_update_2', 'check-update.py')
with patch.object(check.request, 'build_opener') as build:
with self.assertRaises(SystemExit):
check.main(['--current', '0.1.202609282232'])
build.assert_not_called()
class Response:
def __enter__(self): return self
def __exit__(self, *args): return False
def read(self, limit):
self.limit = limit
return b'{"tag_name":"v0.1.202609292232"}'
class Opener:
def open(self, req, timeout):
self.url, self.timeout = req.full_url, timeout
return Response()
opener = Opener()
with patch.object(check.request, 'build_opener', return_value=opener):
self.assertEqual(check.check('0.1.202609282232'), 'AVAILABLE 0.1.202609292232')
self.assertEqual(opener.url, 'https://api.github.com/repos/baketnk/frame-yap/releases/latest')
self.assertEqual(opener.timeout, 6)
def test_terminal_command_is_fixed_and_version_checked(self):
helper = load('frameyap_installer_handoff', 'install-update.py')
cmd = helper.installer_command(Path('/owned/bin/install.sh'), '0.1.202609292232')
self.assertEqual(cmd, ['sh', '/owned/bin/install.sh', '--mode', 'binary',
'--version', '0.1.202609292232', '--yes'])
for bad in ('0.1.202609292232;rm', 'v0.1.202609292232', 'latest'):
with self.assertRaises(ValueError):
helper.installer_command(Path('/owned/bin/install.sh'), bad)
def test_terminal_handoff_requires_tty_before_install(self):
helper = load('frameyap_installer_handoff_2', 'install-update.py')
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
(root / 'scripts').mkdir()
(root / 'bin').mkdir()
(root / 'bin/install.sh').write_text('#!/bin/sh\nexit 0\n')
helper.__file__ = str(root / 'scripts/install-update.py')
with patch.object(helper.sys, 'stdin', io.StringIO('\n')), patch.object(helper.subprocess, 'run') as run:
self.assertEqual(helper.main(['--version', '0.1.202609292232']), 1)
run.assert_not_called()
if __name__ == '__main__':
unittest.main()
+26
View File
@@ -0,0 +1,26 @@
#include "texture_refresh.hpp"
#include <cassert>
int main() {
bool initialized = false, dirty = true;
int renders = 0, uploads = 0, model = 0, visible = -1;
const auto refresh = [&](bool moving) {
return frameyap::refresh_texture(moving, initialized, [&] {
++renders;
if (!dirty) return false;
visible = model; dirty = false;
return true;
}, [&] { ++uploads; initialized = true; });
};
assert(refresh(true)); // first image is allowed during manipulation
assert(renders == 1 && uploads == 1 && visible == 0);
for (int i = 1; i <= 200; ++i) {
model = i; dirty = true; // model and cosmetic changes stay live
assert(!refresh(true));
}
assert(renders == 1 && uploads == 1 && visible == 0 && dirty);
assert(refresh(false)); // display the latest state once, after release
assert(renders == 2 && uploads == 2 && visible == 200 && !dirty);
assert(!refresh(false));
assert(renders == 3 && uploads == 2);
}
+58
View File
@@ -0,0 +1,58 @@
#include "update_check.hpp"
#include <cassert>
#include <chrono>
#include <cstdlib>
#include <filesystem>
#include <fstream>
#include <string>
#include <thread>
#include <unistd.h>
namespace fs = std::filesystem;
using namespace frameyap;
namespace {
UpdateResult finish(UpdateCheck& checker) {
for (int i = 0; i < 300; ++i) {
if (auto result = checker.poll()) return *result;
std::this_thread::sleep_for(std::chrono::milliseconds(10));
}
assert(false && "update helper did not complete");
return {UpdateResult::State::Failed, {}};
}
}
int main(int argc, char** argv) {
assert(argc == 2);
char pattern[] = "/tmp/frameyap-update-XXXXXX";
const auto root = fs::path(::mkdtemp(pattern));
const auto helper = root / "fixture.py";
UpdateCheck check(helper, "0.1.202609282232", argv[1]);
assert(!check.busy() && !check.poll());
assert(!check.start()); // missing file; no child, no network
std::ofstream(helper) << "import time\ntime.sleep(.2)\nprint('AVAILABLE 0.1.202609292232')\n";
assert(check.start() && check.busy());
assert(!check.start()); // one explicit check at a time
assert(!check.poll()); // interaction loop does not wait for network/child
auto result = finish(check);
assert(result.state == UpdateResult::State::Available && result.version == "0.1.202609292232");
assert(!check.busy());
std::ofstream(helper) << "print('CURRENT')\n";
assert(check.start());
assert(finish(check).state == UpdateResult::State::Current);
std::ofstream(helper) << "print('AVAILABLE 0.1.20260929;echo surprise')\n";
assert(check.start());
assert(finish(check).state == UpdateResult::State::Failed);
std::ofstream(helper) << "print('x'*400)\n";
assert(check.start());
assert(finish(check).state == UpdateResult::State::Failed);
std::ofstream(helper) << "raise SystemExit(1)\n";
assert(check.start());
assert(finish(check).state == UpdateResult::State::Failed);
assert(!open_update_terminal(root / "missing.py", "0.1.202609292232"));
assert(!open_update_terminal(helper, "0.1.20260929;echo surprise"));
const char* path = ::getenv("PATH");
const std::string old_path = path ? path : "";
::setenv("PATH", root.c_str(), 1);
assert(!open_update_terminal(helper, "0.1.202609292232")); // no Konsole
::setenv("PATH", old_path.c_str(), 1);
fs::remove_all(root);
}