Consolidate VR UI with Inconsolata neon styling and mount settings

This commit is contained in:
baketnk committed 2026-09-24 12:32:43 -04:00
1 parent 417f81cb09
commit 1ba0f26e0e
19 files changed
+1179 -245

No files matched your search

+19 -2
View File
@@ -4,6 +4,13 @@ set(CMAKE_CXX_STANDARD 20)
set(CMAKE_CXX_STANDARD_REQUIRED ON)
set(CMAKE_CXX_EXTENSIONS OFF)
option(FRAMEYAP_NATIVE "Build opt-in OpenVR/SDL3/Gamescope runtime (no automatic downloads)" OFF)
option(FRAMEYAP_UI_TESTS "Build hardware-free panel renderer tests (requires FreeType)" OFF)
if(FRAMEYAP_NATIVE OR FRAMEYAP_UI_TESTS)
find_package(Freetype REQUIRED)
add_library(frameyap_panel src/panel_surface.cpp)
target_include_directories(frameyap_panel PUBLIC src)
target_link_libraries(frameyap_panel PRIVATE Freetype::Freetype)
endif()
add_library(frameyap_core src/core.cpp)
target_include_directories(frameyap_core PUBLIC src)
@@ -11,6 +18,8 @@ add_executable(frameyap src/main.cpp)
target_link_libraries(frameyap PRIVATE frameyap_core)
target_compile_definitions(frameyap PRIVATE FRAMEYAP_VERSION="${PROJECT_VERSION}")
if(CMAKE_SYSTEM_NAME STREQUAL "Linux")
add_library(frameyap_mount src/mount.cpp)
target_include_directories(frameyap_mount PUBLIC src)
add_library(frameyap_worker src/worker.cpp)
target_include_directories(frameyap_worker PUBLIC src)
endif()
@@ -28,7 +37,6 @@ if(FRAMEYAP_NATIVE)
set_property(TARGET PkgConfig::SDL3 PROPERTY INTERFACE_LINK_OPTIONS "${SDL3_LINK_OPTIONS}")
endif()
pkg_check_modules(WAYLAND REQUIRED IMPORTED_TARGET wayland-client)
pkg_check_modules(FREETYPE REQUIRED IMPORTED_TARGET freetype2)
find_program(WAYLAND_SCANNER wayland-scanner REQUIRED)
set(OPENVR_ROOT "" CACHE PATH "Explicit standalone OpenVR v2.15.6 SDK root")
find_path(OPENVR_INCLUDE_DIR openvr.h HINTS "${OPENVR_ROOT}/headers" REQUIRED)
@@ -50,7 +58,7 @@ if(FRAMEYAP_NATIVE)
target_sources(frameyap PRIVATE src/runtime.cpp src/overlay.cpp src/audio.cpp src/text_input.cpp
"${CMAKE_CURRENT_BINARY_DIR}/gamescope-input-method.c" "${CMAKE_CURRENT_BINARY_DIR}/gamescope-input-method-client.h")
target_include_directories(frameyap PRIVATE "${OPENVR_INCLUDE_DIR}" "${CMAKE_CURRENT_BINARY_DIR}")
target_link_libraries(frameyap PRIVATE frameyap_worker PkgConfig::SDL3 PkgConfig::WAYLAND PkgConfig::FREETYPE "${OPENVR_LIBRARY}")
target_link_libraries(frameyap PRIVATE frameyap_worker frameyap_mount frameyap_panel PkgConfig::SDL3 PkgConfig::WAYLAND "${OPENVR_LIBRARY}")
target_compile_definitions(frameyap PRIVATE FRAMEYAP_NATIVE=1)
set_target_properties(frameyap PROPERTIES INSTALL_RPATH "$ORIGIN/../lib")
endif()
@@ -67,7 +75,16 @@ if(BUILD_TESTING AND NOT CMAKE_CROSSCOMPILING)
add_executable(frameyap_core_test tests/core_test.cpp)
target_link_libraries(frameyap_core_test PRIVATE frameyap_core)
add_test(NAME frameyap.core COMMAND frameyap_core_test)
if(TARGET frameyap_panel)
add_executable(frameyap_panel_test tests/panel_test.cpp)
target_link_libraries(frameyap_panel_test PRIVATE frameyap_panel)
target_compile_options(frameyap_panel_test PRIVATE -UNDEBUG)
add_test(NAME frameyap.panel COMMAND frameyap_panel_test "${CMAKE_CURRENT_SOURCE_DIR}/assets/fonts/Inconsolata-Regular.ttf")
endif()
if(CMAKE_SYSTEM_NAME STREQUAL "Linux")
add_executable(frameyap_mount_test tests/mount_test.cpp)
target_link_libraries(frameyap_mount_test PRIVATE frameyap_mount)
add_test(NAME frameyap.mount COMMAND frameyap_mount_test)
add_executable(frameyap_lock_test tests/lock_test.cpp)
target_include_directories(frameyap_lock_test PRIVATE src)
add_test(NAME frameyap.instance_lock COMMAND frameyap_lock_test)
+2
View File
@@ -23,6 +23,8 @@ See [third-party notes](docs/third-party.md). No GitHub release is published yet
transcribe. Remappable through SteamVR bindings; separate hold-to-talk action too.
- **Left grip:** double-tap to explicitly send Enter. Never inferred from speech.
- **Overlay:** Record/Stop, Cancel, paginated preview, Insert, Enter and Quit.
Review and settings share one Inconsolata/neon-framed surface. World-space by
default; settings offer left wrist, right wrist and head mounting, plus recenter.
Dashboard lasers provide clickable controls without forcing global laser mode.
- **Review-first:** focus your destination, then press Insert. No automatic insertion
or submission. Maximum clip 20 seconds; accidental taps under 200 ms are discarded.
Binary file not shown.
+93
View File
@@ -0,0 +1,93 @@
Copyright 2006 The Inconsolata Project Authors
This Font Software is licensed under the SIL Open Font License, Version 1.1.
This license is copied below, and is also available with a FAQ at:
https://openfontlicense.org
-----------------------------------------------------------
SIL OPEN FONT LICENSE Version 1.1 - 26 February 2007
-----------------------------------------------------------
PREAMBLE
The goals of the Open Font License (OFL) are to stimulate worldwide
development of collaborative font projects, to support the font creation
efforts of academic and linguistic communities, and to provide a free and
open framework in which fonts may be shared and improved in partnership
with others.
The OFL allows the licensed fonts to be used, studied, modified and
redistributed freely as long as they are not sold by themselves. The
fonts, including any derivative works, can be bundled, embedded,
redistributed and/or sold with any software provided that any reserved
names are not used by derivative works. The fonts and derivatives,
however, cannot be released under any other type of license. The
requirement for fonts to remain under this license does not apply
to any document created using the fonts or their derivatives.
DEFINITIONS
"Font Software" refers to the set of files released by the Copyright
Holder(s) under this license and clearly marked as such. This may
include source files, build scripts and documentation.
"Reserved Font Name" refers to any names specified as such after the
copyright statement(s).
"Original Version" refers to the collection of Font Software components as
distributed by the Copyright Holder(s).
"Modified Version" refers to any derivative made by adding to, deleting,
or substituting -- in part or in whole -- any of the components of the
Original Version, by changing formats or by porting the Font Software to a
new environment.
"Author" refers to any designer, engineer, programmer, technical
writer or other person who contributed to the Font Software.
PERMISSION & CONDITIONS
Permission is hereby granted, free of charge, to any person obtaining
a copy of the Font Software, to use, study, copy, merge, embed, modify,
redistribute, and sell modified and unmodified copies of the Font
Software, subject to the following conditions:
1) Neither the Font Software nor any of its individual components,
in Original or Modified Versions, may be sold by itself.
2) Original or Modified Versions of the Font Software may be bundled,
redistributed and/or sold with any software, provided that each copy
contains the above copyright notice and this license. These can be
included either as stand-alone text files, human-readable headers or
in the appropriate machine-readable metadata fields within text or
binary files as long as those fields can be easily viewed by the user.
3) No Modified Version of the Font Software may use the Reserved Font
Name(s) unless explicit written permission is granted by the corresponding
Copyright Holder. This restriction only applies to the primary font name as
presented to the users.
4) The name(s) of the Copyright Holder(s) or the Author(s) of the Font
Software shall not be used to promote, endorse or advertise any
Modified Version, except to acknowledge the contribution(s) of the
Copyright Holder(s) and the Author(s) or with their explicit written
permission.
5) The Font Software, modified or unmodified, in part or in whole,
must be distributed entirely under this license, and must not be
distributed under any other license. The requirement for fonts to
remain under this license does not apply to any document created
using the Font Software.
TERMINATION
This license becomes null and void if any of the above conditions are
not met.
DISCLAIMER
THE FONT SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTIES OF
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT
OF COPYRIGHT, PATENT, TRADEMARK, OR OTHER RIGHT. IN NO EVENT SHALL THE
COPYRIGHT HOLDER BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
INCLUDING ANY GENERAL, SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL
DAMAGES, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
FROM, OUT OF THE USE OR INABILITY TO USE THE FONT SOFTWARE OR FROM
OTHER DEALINGS IN THE FONT SOFTWARE.
+14 -10
View File
@@ -41,9 +41,11 @@ loaded between utterances; do not spawn Python/load 178 MB for every release.
## Minimal interaction
A small hand-attached status chip while armed; a compact review panel only when
needed. Allow head-relative placement for users who prefer it. No permanent large
dashboard obscuring the application.
The implemented menu consolidates review, status, controls and settings onto one
rendering surface, with world-space placement by default. Settings offer left
wrist, right wrist and head mounting. A smaller status-chip presentation while
armed remains a possible refinement, not a second implemented overlay. See
[the current UI and placement behavior](overlay.md).
```text
[ mic ] Ready · On-device [ settings ]
@@ -77,8 +79,9 @@ Recording… 00:04 [ Cancel ]
Implement the standalone `frameyap` executable, initialized
with `VRApplication_Overlay`. Frame accepted that application type and
`IVROverlay_028` in the probe. Use `CreateOverlay`, tracked-device-relative
transform, `ShowOverlay`/`HideOverlay`, and `PollNextOverlayEvent` for the panel.
`IVROverlay_028` in the probe. Use one `CreateOverlay` handle, an absolute world
transform or tracked-device-relative mount, `ShowOverlay`/`HideOverlay`, and
`PollNextOverlayEvent` for the panel.
Use SteamVR's overlay interaction rather than inventing scene controller rays.
Do not link an external scene host or launch another application behind the
@@ -93,11 +96,12 @@ eye targets or per-eye scene rendering. Keep tracking in compositor transforms,
not an application-rendered hand-pose animation loop. Do not promise a particular
GPU cost until measured.
A scene renderer's canvas/MSDF resources are not
an OpenVR overlay backend and are not imported here. Use a small independently
licensed font/icon set and minimal panel renderer. Any later source/asset reuse
requires an explicit license-reviewed extraction, never a runtime path into the
engine checkout.
A scene renderer's canvas/MSDF resources are not an OpenVR overlay backend and
are not imported here. The Inconsolata TTF used by kouseki is independently
bundled under its retained OFL; the panel renderer is original FrameYap code.
The neon HUD frame is a visual reference, not an engine dependency. Further
source/asset reuse requires an explicit license-reviewed extraction, never a
runtime path into the engine checkout.
### Controller bindings
+72 -17
View File
@@ -10,19 +10,73 @@ normal build or test.
## Rendering and controls
Explicit development dependencies: Valve OpenVR SDK v2.15.6 and FreeType 2.
Configure/build must not fetch them. An installed font is passed explicitly;
Unicode coverage depends on that font. The 900×500 RGBA, 0.85 m panel uses
`SetOverlayRaw` only when its status, detail, transcript, recording timer, or
preview page changes. The caller may call `draw(Panel)` at 10 ms intervals.
The complete transcript preview is paginated by glyph width and three-line
height; Prev and Next controls navigate it. Long status/detail messages display
a prefix and a visible truncation marker rather than disappearing silently.
`hand=true` attaches to the validly tracked left controller and falls back to
head-relative placement otherwise. Overlay pointer controls are Record
(click-to-start/stop), Cancel, Insert, Enter, Quit and Prev, plus Next in the
transcript area. Record is available to retry during startup/error; Cancel can
stop worker startup. Enter is *always* a separate deliberate action, not
inferred from text. Recording never automatically submits Enter.
Configure/build must not fetch them. The default font is the bundled Inconsolata
Regular, also used by kouseki; its OFL and extraction provenance are included in
[third-party notes](third-party.md). `--font FILE` is an explicit override. Glyph
coverage depends on the selected face (no automatic system-font fallback or full
CJK coverage is claimed).
`src/panel_surface.*` renders **one 1000×680 RGBA canvas** for review, settings,
status and controls. One OpenVR handle receives it with `SetOverlayRaw`; tabs do
not create extra overlays or render targets. The rounded mint-to-blue perimeter,
shallow curved accent, and dark cards borrow kouseki's VR visual language, with
an independently implemented CPU renderer and a baked halo rather than GPU bloom.
Rendering/uploads occur only for changed content, page, settings, or pointer
feedback; static frames are reused. The caller may call `draw(Panel)` at 10 ms
intervals. Tracking transforms do not require repainting the canvas.
The complete transcript preview is paginated by glyph width and four-line
height; Previous and Next navigate it without changing the source transcript.
Long status/detail messages show a prefix with a visible truncation marker.
The footer remains available on both tabs: Record (labelled Stop while recording),
Cancel, Insert, Enter, Quit. Record can retry after an error; it is disabled while
warming/transcribing and until an existing review is inserted or discarded.
Cancel can stop worker startup. Insert and Enter are disabled during recording
and transcription. A pointer action requires
a press/release on the same enabled control from the same cursor; focus loss,
tab changes, action-state changes and relocation clear pending presses. Enter is *always* a separate
deliberate action, not inferred from text. Recording never automatically submits Enter.
### Placement settings
First launch defaults to **World space**: a standing-universe absolute transform,
1.05 m ahead of the first valid headset pose, 0.16 m below eye height, upright
(yaw only). It stays there as the wearer moves. Startup waits for valid tracking
rather than placing a menu at the world origin. A tracking-origin reset requests
a fresh placement. Settings → Recenter in front deliberately resamples the pose.
Settings offers World space, Left wrist, Right wrist and Head on that same canvas.
World/head width is 0.85 m; wrist width is 0.42 m with mirrored controller-relative
offsets. A missing/untracked selected wrist temporarily falls back to world space,
with a visible explanation in Settings, then reattaches when tracking returns.
The saved preference is not replaced by the fallback. These offsets and sizes are
initial choices, **not headset-comfort acceptance**.
A selection saves only the mount token to `$XDG_CONFIG_HOME/frameyap/mount`
(or `$HOME/.config/frameyap/mount`), using an atomic replacement. No pose, audio
or transcript is saved. Missing/invalid settings default to world; write failure
keeps the selection for the session and displays a warning. `--mount
world|left-wrist|right-wrist|head` overrides the saved choice for one launch without
writing it; `--head` remains an alias for `--mount head`.
### Hardware-free UI checks
The default build tests mount parsing, persistence and pose geometry without any
native dependencies. A FreeType-only opt-in build exercises the actual renderer,
pointer gating, tab switches, pagination, recording state and redraw invalidation:
```sh
cmake -S . -B build-ui -DFRAMEYAP_UI_TESTS=ON
cmake --build build-ui
ctest --test-dir build-ui --output-on-failure
# Optional synthetic fixture previews; no OpenVR initialization:
./build-ui/frameyap_panel_test assets/fonts/Inconsolata-Regular.ttf /tmp/frameyap-ui
```
The last command writes `-review.ppm`, `-settings.ppm`, `-recording.ppm` to the
supplied prefix. The native build includes these tests too; tests never initialize
OpenVR or touch the real mounting preference. Physical pointing, tracking loss,
recentring and readability still require a separately authorized headset check.
`assets/actions.json` names six actions: left/right grip, PTT, cancel, insert,
Enter. `bindings_frame_controller.json` uses the observed Frame profile's grip
@@ -37,8 +91,8 @@ second squeeze **down** within 350 ms starts capture; hold as long as needed
action explicitly begins on down and ends on up. On tracking-pose invalidity,
action inactivity or overlay focus loss, a held capture emits Cancel, and
reconnection requires a neutral observation before any new press. PTT and left
Enter require an enabled panel; the clickable Record/Cancel controls remain
available while disabled. The action set has normal priority: global input
Enter require an enabled panel; clickable Record remains available for retry
after an error and Cancel is always available. The action set has normal priority: global input
while a scene is active is not guaranteed, and experimental overlay overrides
are not switched on automatically.
@@ -46,8 +100,9 @@ The UI cannot itself guarantee a capture started when a BeginRecord action
arrives: the owning runtime checks worker readiness. `Cancel` invalidates
capture/worker work; a failed startup can be retried with Record. The
`UiAction::Toggle` enumerator remains for caller ABI compatibility but the
panel no longer emits it: Prev is navigation only. A pointer Record click
emits `Record` to toggle capture, unlike controller PTT edges.
panel no longer emits it: Previous is navigation only. Pointer Record/Stop emits
explicit `BeginRecord`/`EndRecord`, just like PTT edges, so a stale Stop cannot
become a new recording when another input has already ended capture.
## Registration
+26 -3
View File
@@ -16,7 +16,29 @@ with SteamOS; compatibility must be rechecked.
Frame controller bindings were authored here using the observed public input
profile names (`frame_controller`, `/input/grip`, `click`); no SteamVR driver code,
images, protected kernels or another application's assets were extracted.
images, protected kernels or another application's assets were extracted for these bindings.
## Bundled font and UI reference
`assets/fonts/Inconsolata-Regular.ttf` is an unmodified copy of the typeface used
by kouseki's editor and VR canvas, extracted from its `assets/fonts` directory at
checkout revision `738569f4c41ff4c8fc9edd5bfff9c861957ea39e`.
SHA-256: `e0267abf9d734e2b9f766f8cb7a496b552c57cdfeacfa0efdc5bfd21940ae145`.
Copyright 2006 The Inconsolata Project Authors; **SIL Open Font License 1.1**,
retained in `assets/fonts/OFL-Inconsolata.txt` (line endings and trailing whitespace
normalized; license text unchanged). The reviewed OFL permits
bundling and redistribution with its copyright/license notice; the font remains
OFL, not MIT, and is not sold by itself. Upstream: <https://github.com/googlefonts/Inconsolata>.
The TTF is unchanged. No MSDF atlas, icons, engine code or renderer dependencies
were copied. Unicode coverage is finite; missing glyphs use the face's notdef glyph.
Visual references: kouseki's `apps/vr_workspace/hud.cpp` (rounded mint-to-blue
perimeter and curved accent) and `menu_tablet.hpp` (dark cards, highlighted
selection). FrameYap implements those design ideas independently on a single
CPU RGBA surface. Building, installing and running require no kouseki checkout.
CMake installs the font and OFL with assets; native staging defaults to that
font, places the launcher copy at `fonts/font.ttf`, and includes its license in
`THIRD_PARTY_NOTICES.txt`. Custom staging fonts still require an explicit license.
## Explicit native build inputs (not vendored)
@@ -25,8 +47,9 @@ images, protected kernels or another application's assets were extracted.
- SDL3: zlib license; device trial used SDL 3.2.16 built in a private user prefix.
- Wayland client and scanner: retain upstream MIT-style notices.
- FreeType: choose and comply with its applicable FTL/GPL licensing option.
- Font: explicit user-supplied path; observed device check used system Hack Regular.
A release must include the selected font's own license and assess glyph coverage.
- Optional font override: the earlier device check used system Hack Regular.
A custom release font must include its own license and assessed glyph coverage;
new builds default to the bundled Inconsolata described above.
- Compiler runtime, libc minimum and transitive shared libraries require a release
dependency audit. Passing a developer build is not a portable-runtime guarantee.
+8 -2
View File
@@ -14,10 +14,16 @@ def main():
p = argparse.ArgumentParser(description=__doc__)
p.add_argument("--build", type=Path, required=True)
p.add_argument("--destination", type=Path, required=True, help="new staging directory")
for name in ("openvr-library", "openvr-license", "sdl-library", "sdl-license", "font", "font-license"):
for name in ("openvr-library", "openvr-license", "sdl-library", "sdl-license"):
p.add_argument("--" + name, type=Path, required=True)
p.add_argument("--font", type=Path, help="override bundled Inconsolata (requires --font-license)")
p.add_argument("--font-license", type=Path)
args = p.parse_args()
root = Path(__file__).resolve().parents[1]
if bool(args.font) != bool(args.font_license):
p.error("a font override requires both --font and --font-license")
args.font = args.font or root / "assets/fonts/Inconsolata-Regular.ttf"
args.font_license = args.font_license or root / "assets/fonts/OFL-Inconsolata.txt"
dest = args.destination.absolute()
if dest.exists() or dest.is_symlink():
p.error("destination already exists; use a new staging directory")
@@ -36,7 +42,7 @@ def main():
shutil.copyfile(args.font, dest / "fonts/font.ttf")
notices = ["FrameYap native-only POC. No ASR runtime or model is included.\n",
"Original FrameYap code: MIT. System Wayland/FreeType/libstdc++/glibc are not bundled.\n",
"Bundled libraries: Valve OpenVR and unmodified SDL3; font supplied explicitly below.\n",
"Bundled libraries: Valve OpenVR and unmodified SDL3; font license included below.\n",
"This package does not grant any rights to kestrel-kernels or provide a functioning ASR environment.\n"]
for label, file in (("FrameYap", root / "LICENSE"), ("OpenVR", args.openvr_license),
("SDL3", args.sdl_license), ("Font", args.font_license)):
+23 -8
View File
@@ -15,12 +15,15 @@ namespace {
void help() {
std::cout << "FrameYap — standalone Steam Frame voice typing POC\n"
"Usage: frameyap [--help | --version]\n"
" --run --assets DIR --font FILE --worker FILE --model DIR\n"
" [--python FILE] [--threads 2|4] [--socket gamescope-0] [--head]\n"
" --run --assets DIR [--font FILE] --worker FILE --model DIR\n"
" [--python FILE] [--threads 2|4] [--socket gamescope-0] [--mount MODE]\n"
" --register MANIFEST [--autostart] | --unregister MANIFEST\n"
" --check-input [--socket gamescope-0] (discovery only, no typing)\n"
" --check-overlay --assets DIR --font FILE [--head] (5s, no mic/input)\n"
" --check-controls --assets DIR --font FILE [--head] (30s, gestures only)\n\n"
" --check-overlay --assets DIR [--font FILE] [--mount MODE] (5s, no mic/input)\n"
" --check-controls --assets DIR [--font FILE] [--mount MODE] (30s, gestures only)\n\n"
"Mount: world (first-launch default), left-wrist, right-wrist, head.\n"
"Settings save the mount; --mount overrides it for this launch. --head is an alias.\n"
"Font defaults to bundled Inconsolata in the assets directory.\n"
"Right grip: double-tap, hold second squeeze to speak, release to review.\n"
"Left grip: double-tap for explicit Enter. Bindings are remappable.\n"
"Review first: Insert approves current focus; never automatic Enter.\n"
@@ -49,12 +52,16 @@ int main(int argc, char** argv) {
options.socket = socket && *socket ? socket : "gamescope-0";
for (int i = 2; i < argc; ++i) {
std::string_view arg = argv[i];
if (arg == "--head" && mode != "--check-input") { options.hand = false; continue; }
if (arg == "--head" && mode != "--check-input") { options.mount = frameyap::Mount::Head; continue; }
if (i + 1 == argc) throw std::runtime_error("Missing option value");
std::string value = argv[++i];
if (arg == "--socket" && mode != "--check-overlay") options.socket = value;
else if (arg == "--assets" && mode != "--check-input") options.assets = value;
else if (arg == "--font" && mode != "--check-input") options.font = value;
else if (arg == "--mount" && mode != "--check-input") {
options.mount = frameyap::parse_mount(value);
if (!options.mount) throw std::runtime_error("Mount must be world, left-wrist, right-wrist or head");
}
else if (arg == "--worker" && mode == "--run") options.worker = value;
else if (arg == "--python" && mode == "--run") options.python = value;
else if (arg == "--model" && mode == "--run") options.model = value;
@@ -66,10 +73,10 @@ int main(int argc, char** argv) {
frameyap::TextInput input(options.socket);
std::cout << "Gamescope IME v2 ready; no input delivered.\n"; return 0;
}
if (options.assets.empty() || options.font.empty()) throw std::runtime_error("--assets and --font required");
if (options.assets.empty()) throw std::runtime_error("--assets required");
if (mode == "--check-overlay" || mode == "--check-controls") {
frameyap::InstanceLock lock;
frameyap::Overlay overlay(options.assets, options.font, options.hand);
frameyap::Overlay overlay(options.assets, options.font, options.mount);
overlay.draw({"FrameYap five-second visual check", "No audio captured. No text or Enter delivered.", "Controls inactive during this check.", false, false});
for (vr::TrackedDeviceIndex_t i = 0; i < vr::k_unMaxTrackedDeviceCount; ++i) {
if (vr::VRSystem()->GetTrackedDeviceClass(i) != vr::TrackedDeviceClass_Controller) continue;
@@ -95,7 +102,15 @@ int main(int argc, char** argv) {
if (next != status) { status = next; std::cout << status << std::endl; }
std::this_thread::sleep_for(std::chrono::milliseconds(10));
}
} else std::this_thread::sleep_for(std::chrono::seconds(5));
} else {
const auto end = std::chrono::steady_clock::now() + std::chrono::seconds(5);
while (std::chrono::steady_clock::now() < end) {
for (auto action : overlay.poll()) if (action == frameyap::UiAction::Quit) return 0;
overlay.draw({"FrameYap five-second visual check", "No audio captured. No text or Enter delivered.",
"Actions are diagnostic only during this check.", false, false});
std::this_thread::sleep_for(std::chrono::milliseconds(10));
}
}
std::cout << "Overlay APIs accepted panel; physical visibility requires human confirmation.\n"; return 0;
}
if (options.worker.empty() || options.model.empty()) throw std::runtime_error("--worker and --model required");
+148
View File
@@ -0,0 +1,148 @@
#include "mount.hpp"
#include <atomic>
#include <cerrno>
#include <cmath>
#include <cstdlib>
#include <filesystem>
#include <fstream>
#include <string>
#include <fcntl.h>
#include <unistd.h>
namespace frameyap {
namespace {
constexpr Matrix34 identity{{{{1.f, 0.f, 0.f, 0.f}},
{{0.f, 1.f, 0.f, 0.f}},
{{0.f, 0.f, 1.f, 0.f}}}};
constexpr std::size_t max_preference_bytes = 32;
std::atomic<unsigned long> next_temp{0};
} // namespace
std::string_view mount_name(Mount mount) {
switch (mount) {
case Mount::World: return "world";
case Mount::LeftWrist: return "left-wrist";
case Mount::RightWrist: return "right-wrist";
case Mount::Head: return "head";
}
return {};
}
std::optional<Mount> parse_mount(std::string_view name) {
if (name == "world") return Mount::World;
if (name == "left-wrist") return Mount::LeftWrist;
if (name == "right-wrist") return Mount::RightWrist;
if (name == "head") return Mount::Head;
return std::nullopt;
}
std::optional<Matrix34> world_mount_pose(const Matrix34& hmd) {
for (const auto& row : hmd)
for (float value : row)
if (!std::isfinite(value)) return std::nullopt;
// The HMD's -Z is its forward direction. Discard pitch and roll, so that
// the panel stays level even when the wearer tilts their head.
const double fx = -static_cast<double>(hmd[0][2]);
const double fz = -static_cast<double>(hmd[2][2]);
const double horizontal = std::hypot(fx, fz);
if (!std::isfinite(horizontal) || horizontal < 1e-4) return std::nullopt;
const double x = fx / horizontal, z = fz / horizontal;
const double px = static_cast<double>(hmd[0][3]) + 1.05 * x;
const double py = static_cast<double>(hmd[1][3]) - 0.16;
const double pz = static_cast<double>(hmd[2][3]) + 1.05 * z;
const Matrix34 pose{{{{static_cast<float>(-z), 0.f, static_cast<float>(-x), static_cast<float>(px)}},
{{0.f, 1.f, 0.f, static_cast<float>(py)}},
{{static_cast<float>(x), 0.f, static_cast<float>(-z), static_cast<float>(pz)}}}};
for (const auto& row : pose)
for (float value : row)
if (!std::isfinite(value)) return std::nullopt;
return pose;
}
Matrix34 relative_mount_pose(Mount mount) {
auto pose = identity;
switch (mount) {
case Mount::Head: pose[1][3] = -0.16f; pose[2][3] = -1.05f; break;
case Mount::LeftWrist: pose[0][3] = 0.13f; pose[1][3] = 0.12f; pose[2][3] = -0.18f; break;
case Mount::RightWrist: pose[0][3] = -0.13f; pose[1][3] = 0.12f; pose[2][3] = -0.18f; break;
case Mount::World: break;
}
return pose;
}
float mount_width(Mount mount) {
return mount == Mount::LeftWrist || mount == Mount::RightWrist ? 0.42f : 0.85f;
}
std::filesystem::path default_mount_settings_path() {
try {
const char* xdg = std::getenv("XDG_CONFIG_HOME");
if (xdg && *xdg && std::filesystem::path(xdg).is_absolute())
return std::filesystem::path(xdg) / "frameyap/mount";
const char* home = std::getenv("HOME");
if (home && *home && std::filesystem::path(home).is_absolute())
return std::filesystem::path(home) / ".config/frameyap/mount";
} catch (...) { return {}; }
return {};
}
Mount load_mount(const std::filesystem::path& path) {
try {
if (path.empty() || !std::filesystem::is_regular_file(path)) return Mount::World;
std::ifstream input(path, std::ios::binary);
if (!input) return Mount::World;
char bytes[max_preference_bytes + 1];
input.read(bytes, sizeof(bytes));
const auto size = input.gcount();
if (size <= 0 || size > static_cast<std::streamsize>(max_preference_bytes)) return Mount::World;
std::string_view token(bytes, static_cast<std::size_t>(size));
if (token.back() == '\n') token.remove_suffix(1);
return parse_mount(token).value_or(Mount::World);
} catch (...) { return Mount::World; }
}
bool save_mount(const std::filesystem::path& path, Mount mount) {
std::filesystem::path temporary;
bool created = false;
int fd = -1;
try {
const auto name = mount_name(mount);
if (path.empty() || path.filename().empty() || name.empty()) return false;
const auto dir = path.has_parent_path() ? path.parent_path() : std::filesystem::path(".");
std::error_code ec;
std::filesystem::create_directories(dir, ec);
if (ec) return false;
for (int attempt = 0; attempt < 16; ++attempt) {
temporary = dir / (path.filename().string() + ".tmp." + std::to_string(::getpid()) +
"." + std::to_string(next_temp.fetch_add(1)));
fd = ::open(temporary.c_str(), O_WRONLY | O_CREAT | O_EXCL | O_CLOEXEC, 0600);
if (fd >= 0) { created = true; break; }
if (errno != EEXIST) return false;
}
if (!created) return false;
const std::string content = std::string(name) + '\n';
std::size_t written = 0;
while (written < content.size()) {
const auto n = ::write(fd, content.data() + written, content.size() - written);
if (n < 0 && errno == EINTR) continue;
if (n <= 0) { ::close(fd); fd = -1; std::filesystem::remove(temporary, ec); return false; }
written += static_cast<std::size_t>(n);
}
const bool synced = ::fsync(fd) == 0;
const bool closed = ::close(fd) == 0;
fd = -1;
if (!synced || !closed) { std::filesystem::remove(temporary, ec); return false; }
std::filesystem::rename(temporary, path, ec);
if (ec) { std::filesystem::remove(temporary, ec); return false; }
return true;
} catch (...) {
if (fd >= 0) ::close(fd);
if (created) { std::error_code ec; std::filesystem::remove(temporary, ec); }
return false;
}
}
} // namespace frameyap
+33
View File
@@ -0,0 +1,33 @@
#pragma once
#include <array>
#include <filesystem>
#include <optional>
#include <string_view>
namespace frameyap {
enum class Mount { World, LeftWrist, RightWrist, Head };
std::string_view mount_name(Mount mount);
std::optional<Mount> parse_mount(std::string_view name);
// Row-major 3x4 affine transform (OpenVR axes: +Y up, -Z forward).
using Matrix34 = std::array<std::array<float, 4>, 3>;
// World-space, yaw-only pose in front of the HMD, upright and facing the viewer.
// Invalid/nonfinite HMD transforms or a vanishing horizontal forward return nullopt.
std::optional<Matrix34> world_mount_pose(const Matrix34& hmd);
// Device-relative transform; World has no relative anchor and returns identity.
Matrix34 relative_mount_pose(Mount mount);
float mount_width(Mount mount);
// Empty when neither XDG_CONFIG_HOME nor HOME specifies an absolute base.
std::filesystem::path default_mount_settings_path();
// A missing, unreadable, overlong or malformed preference defaults to World.
Mount load_mount(const std::filesystem::path& path);
// Create local parent directories and atomically replace a single mount preference.
// Never stores any audio or transcript; failures return false without throwing.
bool save_mount(const std::filesystem::path& path, Mount mount);
} // namespace frameyap
+93 -198
View File
@@ -1,9 +1,8 @@
#include "overlay.hpp"
#include "gestures.hpp"
#include "panel_surface.hpp"
#include <openvr.h>
#include <ft2build.h>
#include FT_FREETYPE_H
#include <algorithm>
#include <array>
@@ -31,19 +30,7 @@ void configure_registry() {
if (root.is_absolute() && std::filesystem::is_regular_file(registry))
::setenv("VR_PATHREG_OVERRIDE", registry.c_str(), 0);
}
constexpr int W = 900, H = 500;
struct Rect { int x, y, w, h; bool contains(int px, int py) const {
return px >= x && px < x + w && py >= y && py < y + h;
} };
struct Button { Rect bounds; const char* label; UiAction action; };
constexpr std::array<Button, 6> buttons{{
{{24, 422, 130, 58}, "Prev", UiAction::Toggle}, // handled as page navigation, not a toggle
{{168, 422, 130, 58}, "Record", UiAction::Record},
{{312, 422, 130, 58}, "Cancel", UiAction::Cancel},
{{456, 422, 130, 58}, "Insert", UiAction::Insert},
{{600, 422, 130, 58}, "Enter", UiAction::Enter},
{{744, 422, 130, 58}, "Quit", UiAction::Quit},
}};
constexpr int W = PanelSurface::width, H = PanelSurface::height;
void overlay_check(vr::EVROverlayError err, vr::IVROverlay* api, const char* op) {
if (err != vr::VROverlayError_None)
throw std::runtime_error(std::string(op) + ": " + api->GetOverlayErrorNameFromEnum(err));
@@ -53,24 +40,6 @@ std::filesystem::path absolute_file(const std::filesystem::path& p) {
if (!std::filesystem::is_regular_file(result)) throw std::runtime_error("Missing file: " + result.string());
return result;
}
// A strict, bounded UTF-8 decoder: malformed input is shown as replacement glyphs.
uint32_t next_codepoint(std::string_view s, size_t& i) {
const unsigned char a = static_cast<unsigned char>(s[i++]);
if (a < 0x80) return a;
int count = a >= 0xc2 && a <= 0xdf ? 1 : a >= 0xe0 && a <= 0xef ? 2 : a >= 0xf0 && a <= 0xf4 ? 3 : 0;
if (!count || i + count > s.size()) return 0xfffd;
uint32_t value = a & (count == 1 ? 0x1f : count == 2 ? 0x0f : 0x07);
for (int k = 0; k < count; ++k) {
unsigned char b = static_cast<unsigned char>(s[i + k]);
if ((b & 0xc0) != 0x80) return 0xfffd;
value = (value << 6) | (b & 0x3f);
}
if ((count == 1 && value < 0x80) || (count == 2 && value < 0x800) ||
(count == 3 && value < 0x10000) || (value >= 0xd800 && value <= 0xdfff) || value > 0x10ffff)
return 0xfffd;
i += count;
return value;
}
} // namespace
struct Overlay::Impl {
@@ -80,32 +49,29 @@ struct Overlay::Impl {
vr::VROverlayHandle_t handle = vr::k_ulOverlayHandleInvalid;
vr::VRActionSetHandle_t action_set = vr::k_ulInvalidActionSetHandle;
std::array<vr::VRActionHandle_t, 6> actions{};
FT_Library library = nullptr;
FT_Face face = nullptr;
std::vector<unsigned char> pixels = std::vector<unsigned char>(W * H * 4);
std::filesystem::path settings_path;
Mount mount;
PanelSurface surface;
Panel panel;
bool hand = false;
bool save_failed = false;
bool world_ready = false, placed = false, has_texture = false, shown = false;
vr::HmdMatrix34_t world_transform{};
std::optional<Mount> applied_mount;
vr::TrackedDeviceIndex_t anchor = vr::k_unTrackedDeviceIndexInvalid;
DoubleTap left;
GripRecord right;
std::array<NeutralEdge, 4> edges{};
std::array<int, 2> pressed_button{{-1, -1}};
std::array<vr::TrackedDevicePose_t, vr::k_unMaxTrackedDeviceCount> poses{};
std::vector<std::string> pages;
size_t page = 0;
bool drawn = false;
bool grip_capture = false, ptt_capture = false;
bool focus = true;
static constexpr Rect next_page{754, 292, 112, 30};
Impl(const std::string& assets, const std::string& font, bool attach) : hand(attach) {
Impl(const std::string& assets, const std::string& font, std::optional<Mount> requested)
: settings_path(default_mount_settings_path()),
mount(requested ? *requested : load_mount(settings_path)),
surface(font.empty() ? (std::filesystem::path(assets) / "fonts/Inconsolata-Regular.ttf").string() : font, mount) {
const auto action_path = absolute_file(std::filesystem::path(assets) / "actions.json");
absolute_file(std::filesystem::path(assets) / "bindings_knuckles.json");
const auto font_path = absolute_file(font);
if (FT_Init_FreeType(&library)) throw std::runtime_error("FreeType initialization failed");
try {
if (FT_New_Face(library, font_path.c_str(), 0, &face) || FT_Set_Pixel_Sizes(face, 0, 31))
throw std::runtime_error("Could not load specified font");
configure_registry();
vr::EVRInitError err = vr::VRInitError_None;
system = vr::VR_Init(&err, vr::VRApplication_Overlay);
@@ -136,6 +102,7 @@ struct Overlay::Impl {
overlay_check(overlay->SetOverlayFlag(handle, vr::VROverlayFlags_VisibleInDashboard, true), overlay, "VisibleInDashboard");
vr::HmdVector2_t mouse_scale{{float(W), float(H)}};
overlay_check(overlay->SetOverlayMouseScale(handle, &mouse_scale), overlay, "SetOverlayMouseScale");
system->GetDeviceToAbsoluteTrackingPose(vr::TrackingUniverseStanding, 0, poses.data(), uint32_t(poses.size()));
place();
draw(Panel{"Disabled", "", "Record to start local worker", false, false});
} catch (...) { cleanup(); throw; }
@@ -148,140 +115,71 @@ struct Overlay::Impl {
handle = vr::k_ulOverlayHandleInvalid;
}
if (system) { vr::VR_Shutdown(); system = nullptr; overlay = nullptr; input = nullptr; }
if (face) { FT_Done_Face(face); face = nullptr; }
if (library) { FT_Done_FreeType(library); library = nullptr; }
}
void visibility() {
const bool wanted = placed && has_texture;
if (wanted == shown) return;
overlay_check(wanted ? overlay->ShowOverlay(handle) : overlay->HideOverlay(handle), overlay, "Overlay visibility");
shown = wanted;
}
void place() {
Mount effective = mount;
vr::TrackedDeviceIndex_t target = vr::k_unTrackedDeviceIndex_Hmd;
if (hand) {
const auto left_index = system->GetTrackedDeviceIndexForControllerRole(vr::TrackedControllerRole_LeftHand);
if (left_index != vr::k_unTrackedDeviceIndexInvalid && left_index < poses.size() &&
poses[left_index].bPoseIsValid && system->IsTrackedDeviceConnected(left_index))
target = left_index;
if (mount == Mount::LeftWrist || mount == Mount::RightWrist) {
target = system->GetTrackedDeviceIndexForControllerRole(mount == Mount::LeftWrist
? vr::TrackedControllerRole_LeftHand : vr::TrackedControllerRole_RightHand);
if (target >= poses.size() || !poses[target].bPoseIsValid || !system->IsTrackedDeviceConnected(target))
effective = Mount::World;
}
if (anchor == target) return;
vr::HmdMatrix34_t transform{{{1.f, 0.f, 0.f, target == vr::k_unTrackedDeviceIndex_Hmd ? 0.f : 0.13f},
{0.f, 1.f, 0.f, target == vr::k_unTrackedDeviceIndex_Hmd ? -0.16f : 0.12f},
{0.f, 0.f, 1.f, target == vr::k_unTrackedDeviceIndex_Hmd ? -1.05f : -0.18f}}};
overlay_check(overlay->SetOverlayTransformTrackedDeviceRelative(handle, target, &transform), overlay, "SetOverlayTransformTrackedDeviceRelative");
anchor = target;
}
void rect(Rect r, std::array<unsigned char, 4> c) {
for (int y = std::max(r.y, 0); y < std::min(H, r.y + r.h); ++y)
for (int x = std::max(r.x, 0); x < std::min(W, r.x + r.w); ++x) {
const size_t p = (size_t(y) * W + x) * 4;
std::copy(c.begin(), c.end(), pixels.begin() + p);
if (effective == Mount::World && applied_mount && *applied_mount != Mount::World)
world_ready = false; // a fresh world fallback near the wearer, not an old room location
std::string note = save_failed ? "Preference could not be saved; using it for this session." : "";
if (effective != mount) note = save_failed ? "Wrist untracked; world fallback. Preference not saved." :
"Wrist not tracked - using world space until it returns.";
if (effective == Mount::World && !world_ready) {
const auto& hmd = poses[vr::k_unTrackedDeviceIndex_Hmd];
Matrix34 pose{};
for (int r = 0; r < 3; ++r) for (int c = 0; c < 4; ++c) pose[r][c] = hmd.mDeviceToAbsoluteTracking.m[r][c];
const auto world = hmd.bPoseIsValid && hmd.bDeviceIsConnected ? world_mount_pose(pose) : std::nullopt;
if (!world) {
placed = false;
surface.set_placement_note("Waiting for headset tracking to place the menu.");
visibility();
return;
}
}
void text(std::string_view s, int x, int baseline, int max_x, int max_y, int max_bytes) {
size_t i = 0;
int used = 0;
while (i < s.size() && used < max_bytes && baseline < max_y) {
const size_t before = i;
const auto cp = next_codepoint(s, i);
used += int(i - before);
if (cp == '\n') { x = 28; baseline += 39; continue; }
if (cp == '\r' || cp == '\t') continue;
if (FT_Load_Char(face, cp, FT_LOAD_RENDER)) continue;
const auto glyph = face->glyph;
if (x + (glyph->advance.x >> 6) >= max_x) { x = 28; baseline += 39; }
if (baseline >= max_y) break;
const auto& b = glyph->bitmap;
for (unsigned row = 0; row < b.rows; ++row)
for (unsigned col = 0; col < b.width; ++col) {
int px = x + glyph->bitmap_left + int(col);
int py = baseline - glyph->bitmap_top + int(row);
if (px < 0 || px >= W || py < 0 || py >= max_y) continue;
const unsigned char alpha = b.buffer[int(row) * b.pitch + int(col)];
auto* dst = pixels.data() + (size_t(py) * W + px) * 4;
for (int c = 0; c < 3; ++c)
dst[c] = static_cast<unsigned char>((unsigned(dst[c]) * (255 - alpha) + 240u * alpha) / 255);
}
x += glyph->advance.x >> 6;
for (int r = 0; r < 3; ++r) for (int c = 0; c < 4; ++c) world_transform.m[r][c] = (*world)[r][c];
world_ready = true;
applied_mount.reset();
}
}
bool available(UiAction a) const {
switch (a) {
case UiAction::Toggle: return page > 0;
case UiAction::Quit: case UiAction::Record: case UiAction::Cancel: return true;
case UiAction::BeginRecord: case UiAction::EndRecord: return true;
case UiAction::Enter: return panel.enabled;
case UiAction::Insert: return panel.enabled && !panel.transcript.empty();
}
return false;
}
// Split by rendered glyph advances, never by byte count or arbitrary characters.
// UTF-8 slices remain intact; all bytes (including invalid UTF-8) stay visible.
std::vector<std::string> paginate(std::string_view s, int lines, int right) {
std::vector<std::string> out;
size_t start = 0, i = 0;
int x = 28, line = 0;
while (i < s.size()) {
size_t before = i;
auto cp = next_codepoint(s, i);
if (cp == '\n') {
if (++line == lines) {
out.emplace_back(s.substr(start, i - start));
start = i; line = 0;
}
x = 28;
continue;
surface.set_placement_note(note);
if (applied_mount != effective || (effective != Mount::World && anchor != target)) {
surface.reset_pointers(); // a release on a relocated surface cannot activate an old press
if (effective == Mount::World) {
overlay_check(overlay->SetOverlayTransformAbsolute(handle, vr::TrackingUniverseStanding, &world_transform), overlay,
"SetOverlayTransformAbsolute");
} else {
const auto pose = relative_mount_pose(effective);
vr::HmdMatrix34_t transform{};
for (int r = 0; r < 3; ++r) for (int c = 0; c < 4; ++c) transform.m[r][c] = pose[r][c];
overlay_check(overlay->SetOverlayTransformTrackedDeviceRelative(handle, target, &transform), overlay,
"SetOverlayTransformTrackedDeviceRelative");
}
int advance = 0;
if (cp != '\r' && cp != '\t' && !FT_Load_Char(face, cp, FT_LOAD_DEFAULT))
advance = face->glyph->advance.x >> 6;
if (x + advance >= right && x != 28) {
if (++line == lines) {
out.emplace_back(s.substr(start, before - start));
start = before; line = 0;
} else {
// Add a line break in the displayed page while keeping source intact.
// The renderer's own width wrapping uses the same advance rule.
}
x = 28;
}
x += advance;
overlay_check(overlay->SetOverlayWidthInMeters(handle, mount_width(effective)), overlay, "SetOverlayWidthInMeters");
applied_mount = effective;
anchor = target;
}
if (start < s.size() || out.empty()) out.emplace_back(s.substr(start));
return out;
placed = true;
visibility();
}
bool available(UiAction action) const { return surface.available(action); }
void draw(const Panel& p) {
if (p.transcript != panel.transcript || pages.empty()) {
pages = paginate(p.transcript.empty() ? "Transcript preview" : p.transcript, 3, W - 28);
page = 0;
drawn = false;
}
if (drawn && p.status == panel.status && p.transcript == panel.transcript &&
p.detail == panel.detail && p.enabled == panel.enabled && p.recording == panel.recording) return;
panel = p;
rect({0, 0, W, H}, {20, 26, 39, 255});
rect({0, 0, W, 12}, {static_cast<unsigned char>(p.recording ? 220 : 55),
static_cast<unsigned char>(p.recording ? 82 : 135), 72, 255});
text(p.enabled ? "FrameYap | Enabled" : "FrameYap | Disabled", 28, 52, W - 20, 72, 128);
const auto status_lines = paginate(p.status, 2, 620);
text(status_lines[0], 28, 96, 620, 150, int(status_lines[0].size()));
if (status_lines.size() > 1) text("[status truncated]", 635, 140, W - 28, 150, 32);
rect({20, 153, W - 40, 171}, {37, 46, 63, 255});
text(pages[page], 28, 190, W - 28, 290, int(pages[page].size()));
const std::string page_label = "Page " + std::to_string(page + 1) + " / " + std::to_string(pages.size());
text(page_label, 383, 315, 720, 324, 64);
rect(next_page, page + 1 < pages.size() ? std::array<unsigned char, 4>{56, 94, 129, 255}
: std::array<unsigned char, 4>{49, 52, 60, 255});
text("Next", 767, 316, 860, 324, 8);
const auto detail_lines = paginate(p.detail, 2, 620);
text(detail_lines[0], 28, 355, 620, 412, int(detail_lines[0].size()));
if (detail_lines.size() > 1)
text("[detail truncated]", 635, 405, W - 28, 412, 32);
for (const auto& button : buttons) {
const bool enabled = available(button.action);
rect(button.bounds, enabled ? std::array<unsigned char, 4>{56, 94, 129, 255}
: std::array<unsigned char, 4>{49, 52, 60, 255});
text(button.label, button.bounds.x + 10, button.bounds.y + 38,
button.bounds.x + button.bounds.w - 5, H, 64);
if (surface.render(p)) {
// OpenVR's API takes void*, but does not modify the submitted RGBA bytes.
overlay_check(overlay->SetOverlayRaw(handle, const_cast<unsigned char*>(surface.pixels().data()), W, H, 4), overlay, "SetOverlayRaw");
has_texture = true;
}
overlay_check(overlay->SetOverlayRaw(handle, pixels.data(), W, H, 4), overlay, "SetOverlayRaw");
overlay_check(overlay->ShowOverlay(handle), overlay, "ShowOverlay");
drawn = true;
visibility();
}
void reset_input(std::vector<UiAction>& result) {
left.reset();
@@ -291,7 +189,7 @@ struct Overlay::Impl {
ptt_capture = false;
if (lost_grip || lost_ptt) result.push_back(UiAction::Cancel);
for (size_t i = 1; i < edges.size(); ++i) edges[i].reset();
pressed_button.fill(-1);
surface.reset_pointers();
}
// Bound actions are accepted only with a connected tracked source. A held input
// following loss of activity must return to neutral before generating an edge.
@@ -318,6 +216,9 @@ struct Overlay::Impl {
vr::VREvent_t event{};
while (system->PollNextEvent(&event, sizeof(event))) {
if (event.eventType == vr::VREvent_Quit) result.push_back(UiAction::Quit);
if (event.eventType == vr::VREvent_SeatedZeroPoseReset || event.eventType == vr::VREvent_ChaperoneUniverseHasChanged) {
world_ready = false; applied_mount.reset(); surface.reset_pointers();
}
if (event.eventType == vr::VREvent_TrackedDeviceDeactivated ||
event.eventType == vr::VREvent_InputFocusCaptured) reset_input(result);
}
@@ -337,36 +238,30 @@ struct Overlay::Impl {
// activity. Cancel the old gesture, then require neutral rearm;
// do not permanently latch global grip actions off.
reset_input(result); break;
case vr::VREvent_MouseButtonDown: {
const auto& m = event.data.mouse;
if (m.button != vr::VRMouseButton_Left || m.cursorIndex >= pressed_button.size()) break;
const int x = int(m.x), y = H - int(m.y);
pressed_button[m.cursorIndex] = -1;
for (size_t i = 0; i < buttons.size(); ++i)
if (buttons[i].bounds.contains(x, y) && available(buttons[i].action)) {
pressed_button[m.cursorIndex] = int(i); break;
case vr::VREvent_MouseMove:
surface.pointer_move(event.data.mouse.cursorIndex, event.data.mouse.x, H - event.data.mouse.y);
break;
case vr::VREvent_MouseButtonDown:
if (event.data.mouse.button == vr::VRMouseButton_Left)
surface.pointer_down(event.data.mouse.cursorIndex, event.data.mouse.x, H - event.data.mouse.y);
break;
case vr::VREvent_MouseButtonUp:
if (event.data.mouse.button == vr::VRMouseButton_Left) {
auto event_result = surface.pointer_up(event.data.mouse.cursorIndex, event.data.mouse.x, H - event.data.mouse.y);
if (event_result.action) result.push_back(*event_result.action);
if (event_result.mount) {
mount = *event_result.mount;
save_failed = !save_mount(settings_path, mount);
world_ready = false;
applied_mount.reset();
}
if (next_page.contains(x, y) && page + 1 < pages.size())
pressed_button[m.cursorIndex] = int(buttons.size());
if (event_result.recenter) { world_ready = false; applied_mount.reset(); }
}
break;
}
case vr::VREvent_MouseButtonUp: {
const auto& m = event.data.mouse;
if (m.button != vr::VRMouseButton_Left || m.cursorIndex >= pressed_button.size()) break;
const int index = pressed_button[m.cursorIndex];
pressed_button[m.cursorIndex] = -1;
if (index == int(buttons.size()) && next_page.contains(int(m.x), H - int(m.y)) &&
page + 1 < pages.size()) { ++page; drawn = false; }
else if (index == 0 && buttons[0].bounds.contains(int(m.x), H - int(m.y)) && page > 0) {
--page; drawn = false;
} else if (index > 0 && index < int(buttons.size()) &&
buttons[index].bounds.contains(int(m.x), H - int(m.y)) &&
available(buttons[index].action)) result.push_back(buttons[index].action);
break;
}
default: break;
}
}
place();
vr::VRActiveActionSet_t set{};
set.ulActionSet = action_set;
set.ulRestrictedToDevice = vr::k_ulInvalidInputValueHandle;
@@ -377,7 +272,7 @@ struct Overlay::Impl {
auto [la, ld] = digital(0);
auto [ra, rd] = digital(1);
const auto now = DoubleTap::Clock::now();
if (left.update(la && focus, ld, panel.enabled, now)) result.push_back(UiAction::Enter);
if (left.update(la && focus, ld, available(UiAction::Enter), now)) result.push_back(UiAction::Enter);
switch (right.update(ra && focus, rd, now)) {
case GripRecord::Change::Begin: grip_capture = true; result.push_back(UiAction::BeginRecord); break;
case GripRecord::Change::End: grip_capture = false; result.push_back(UiAction::EndRecord); break;
@@ -405,8 +300,8 @@ struct Overlay::Impl {
}
};
Overlay::Overlay(const std::string& assets, const std::string& font, bool hand)
: impl_(std::make_unique<Impl>(assets, font, hand)) {}
Overlay::Overlay(const std::string& assets, const std::string& font, std::optional<Mount> mount)
: impl_(std::make_unique<Impl>(assets, font, mount)) {}
Overlay::~Overlay() = default;
std::vector<UiAction> Overlay::poll() { return impl_->poll(); }
void Overlay::draw(const Panel& panel) { impl_->draw(panel); }
+4 -1
View File
@@ -1,4 +1,6 @@
#pragma once
#include "mount.hpp"
#include <optional>
#include <memory>
#include <string>
#include <vector>
@@ -11,10 +13,11 @@ struct Panel {
std::string detail;
bool enabled = false;
bool recording = false;
bool record_available = true;
};
class Overlay {
public:
Overlay(const std::string& assets, const std::string& font, bool hand);
Overlay(const std::string& assets, const std::string& font = {}, std::optional<Mount> mount = {});
~Overlay();
Overlay(const Overlay&) = delete;
Overlay& operator=(const Overlay&) = delete;
+338
View File
@@ -0,0 +1,338 @@
#include "panel_surface.hpp"
#include <ft2build.h>
#include FT_FREETYPE_H
#include <algorithm>
#include <cmath>
#include <cstdint>
#include <stdexcept>
#include <string_view>
#include <utility>
namespace frameyap {
namespace {
constexpr int W = PanelSurface::width, H = PanelSurface::height;
using Color = std::array<unsigned char, 4>;
constexpr Color background{12, 16, 27, 255}, card{20, 28, 43, 255};
constexpr Color ink{230, 240, 249, 255}, muted{151, 173, 193, 255};
constexpr Color cyan{31, 240, 164, 255}, pink{255, 110, 135, 255};
struct Rect {
int x, y, w, h;
bool contains(float px, float py) const {
return std::isfinite(px) && std::isfinite(py) && px >= x && px < x + w && py >= y && py < y + h;
}
};
enum class Control { Review, Settings, Prev, Next, Record, Cancel, Insert, Enter, Quit,
World, Left, Right, Head, Recenter };
struct Button { Rect r; Control id; const char* label; };
constexpr std::array<Button, 14> buttons{{
{{32, 138, 180, 46}, Control::Review, "Review"},
{{226, 138, 180, 46}, Control::Settings, "Settings"},
{{32, 406, 154, 44}, Control::Prev, "Previous"},
{{838, 406, 130, 44}, Control::Next, "Next"},
{{32, 574, 176, 68}, Control::Record, "Record"},
{{222, 574, 176, 68}, Control::Cancel, "Cancel"},
{{412, 574, 176, 68}, Control::Insert, "Insert"},
{{602, 574, 176, 68}, Control::Enter, "Enter"},
{{792, 574, 176, 68}, Control::Quit, "Quit"},
{{32, 234, 454, 58}, Control::World, "World space"},
{{514, 234, 454, 58}, Control::Head, "Head"},
{{32, 308, 454, 58}, Control::Left, "Left wrist"},
{{514, 308, 454, 58}, Control::Right, "Right wrist"},
{{32, 394, 300, 50}, Control::Recenter, "Recenter in front"},
}};
std::optional<UiAction> action(Control c) {
switch (c) {
case Control::Record: return UiAction::Record;
case Control::Cancel: return UiAction::Cancel;
case Control::Insert: return UiAction::Insert;
case Control::Enter: return UiAction::Enter;
case Control::Quit: return UiAction::Quit;
default: return {};
}
}
std::optional<Mount> mounting(Control c) {
switch (c) {
case Control::World: return Mount::World;
case Control::Left: return Mount::LeftWrist;
case Control::Right: return Mount::RightWrist;
case Control::Head: return Mount::Head;
default: return {};
}
}
std::string_view mount_label(Mount m) {
switch (m) {
case Mount::World: return "World space";
case Mount::LeftWrist: return "Left wrist";
case Mount::RightWrist: return "Right wrist";
case Mount::Head: return "Head";
}
return "World space";
}
// Invalid bytes become visible replacement glyphs, never control commands.
uint32_t next_codepoint(std::string_view s, size_t& i) {
const auto a = static_cast<unsigned char>(s[i++]);
if (a < 0x80) return a;
int count = a >= 0xc2 && a <= 0xdf ? 1 : a >= 0xe0 && a <= 0xef ? 2 : a >= 0xf0 && a <= 0xf4 ? 3 : 0;
if (!count || i + count > s.size()) return 0xfffd;
uint32_t value = a & (count == 1 ? 0x1f : count == 2 ? 0x0f : 0x07);
for (int k = 0; k < count; ++k) {
auto b = static_cast<unsigned char>(s[i + k]);
if ((b & 0xc0) != 0x80) return 0xfffd;
value = (value << 6) | (b & 0x3f);
}
if ((count == 1 && value < 0x80) || (count == 2 && value < 0x800) ||
(count == 3 && value < 0x10000) || (value >= 0xd800 && value <= 0xdfff) || value > 0x10ffff)
return 0xfffd;
i += count;
return value;
}
} // namespace
struct PanelSurface::Impl {
FT_Library library = nullptr;
FT_Face face = nullptr;
std::vector<unsigned char> pixels = std::vector<unsigned char>(W * H * 4);
Panel panel;
Mount mount;
bool settings = false, dirty = true;
std::string placement_note;
std::array<int, 2> pressed{{-1, -1}}, hovered{{-1, -1}};
std::vector<std::string> lines;
size_t page = 0;
static constexpr size_t lines_per_page = 4;
Impl(const std::string& font, Mount m) : mount(m) {
if (FT_Init_FreeType(&library)) throw std::runtime_error("FreeType initialization failed");
if (FT_New_Face(library, font.c_str(), 0, &face)) {
FT_Done_FreeType(library);
throw std::runtime_error("Could not load specified font: " + font);
}
}
~Impl() { FT_Done_Face(face); FT_Done_FreeType(library); }
void size(unsigned px) {
if (FT_Set_Pixel_Sizes(face, 0, px)) throw std::runtime_error("Could not size panel font");
}
void rect(Rect r, Color c) {
for (int y = std::max(r.y, 0); y < std::min(H, r.y + r.h); ++y)
for (int x = std::max(r.x, 0); x < std::min(W, r.x + r.w); ++x)
std::copy(c.begin(), c.end(), pixels.begin() + (size_t(y) * W + x) * 4);
}
void border(Rect r, Color c, int thickness = 1) {
rect({r.x, r.y, r.w, thickness}, c); rect({r.x, r.y + r.h - thickness, r.w, thickness}, c);
rect({r.x, r.y, thickness, r.h}, c); rect({r.x + r.w - thickness, r.y, thickness, r.h}, c);
}
void neon(Rect r, Color c) {
// Baked halo + bright core: no extra overlay, mesh, bloom pass or animation.
for (int spread = 5; spread >= 1; --spread) {
Color halo = background;
for (int k = 0; k < 3; ++k) halo[k] += (c[k] - halo[k]) / (spread * 3 + 2);
border({r.x - spread, r.y - spread, r.w + spread * 2, r.h + spread * 2}, halo);
}
border(r, c, 2);
}
void frame() {
// Independently rasterized version of kouseki's HUD visual language:
// rounded mint-to-blue perimeter and a second shallow curved accent.
for (int y = 0; y < H; ++y) for (int x = 0; x < W; ++x) {
if (x > 34 && x < W - 34 && y > 34 && y < H - 34) continue;
const float qx = std::abs(x - W / 2.f) - (W / 2.f - 28.f);
const float qy = std::abs(y - H / 2.f) - (H / 2.f - 28.f);
const float distance = std::hypot(std::max(qx, 0.f), std::max(qy, 0.f)) +
std::min(std::max(qx, qy), 0.f) - 18.f;
const float edge = std::abs(distance);
const float strength = edge <= 1.f ? 1.f : .30f * std::max(0.f, 1.f - (edge - 1.f) / 6.f);
const float t = float(x) / W;
const Color gradient{31, static_cast<unsigned char>(255 - 143 * t),
static_cast<unsigned char>(145 + 110 * t), 255};
auto* dst = pixels.data() + (size_t(y) * W + x) * 4;
for (int k = 0; k < 3; ++k) dst[k] = static_cast<unsigned char>(background[k] + (gradient[k] - background[k]) * strength);
dst[3] = distance <= 1.f ? 255 : static_cast<unsigned char>(255 * std::max(0.f, 1.f - (distance - 1.f) / 6.f));
}
for (int x = 32; x < W - 32; ++x) {
const float t = float(x - 32) / (W - 64);
const int y = H - 21 - int(5 * std::sin(t * 3.14159265f));
rect({x, y, 1, 1}, {31, static_cast<unsigned char>(175 - 75 * t),
static_cast<unsigned char>(115 + 80 * t), 255});
}
}
int advance(uint32_t cp) {
if (cp == '\r' || cp == '\t') return 0;
if (FT_Load_Char(face, cp, FT_LOAD_DEFAULT)) return 0;
return int(face->glyph->advance.x >> 6);
}
std::vector<std::string> wrap(std::string_view s, unsigned px, int width) {
size(px);
std::vector<std::string> result;
size_t start = 0, i = 0;
int x = 0;
while (i < s.size()) {
size_t before = i;
auto cp = next_codepoint(s, i);
if (cp == '\n') {
result.emplace_back(s.substr(start, before - start)); start = i; x = 0; continue;
}
int a = advance(cp);
if (x + a > width && x > 0) {
result.emplace_back(s.substr(start, before - start)); start = before; x = 0;
}
x += a;
}
result.emplace_back(s.substr(start));
return result;
}
void text(std::string_view s, int x, int baseline, unsigned px, Color color, int right) {
size(px);
size_t i = 0;
while (i < s.size()) {
auto cp = next_codepoint(s, i);
if (cp == '\n' || cp == '\r' || cp == '\t') continue;
if (FT_Load_Char(face, cp, FT_LOAD_RENDER)) continue;
auto glyph = face->glyph;
int a = int(glyph->advance.x >> 6);
if (x + a > right) break;
const auto& b = glyph->bitmap;
for (unsigned row = 0; row < b.rows; ++row)
for (unsigned col = 0; col < b.width; ++col) {
int xx = x + glyph->bitmap_left + int(col), yy = baseline - glyph->bitmap_top + int(row);
if (xx < 0 || xx >= right || xx >= W || yy < 0 || yy >= H) continue;
unsigned char alpha = b.buffer[int(row) * b.pitch + int(col)];
auto* dst = pixels.data() + (size_t(yy) * W + xx) * 4;
for (int k = 0; k < 3; ++k)
dst[k] = static_cast<unsigned char>((unsigned(dst[k]) * (255 - alpha) + unsigned(color[k]) * alpha) / 255);
}
x += a;
}
}
size_t page_count() const { return std::max(size_t(1), (lines.size() + lines_per_page - 1) / lines_per_page); }
bool available(UiAction a) const {
if (a == UiAction::Record) return panel.recording || panel.record_available;
if (a == UiAction::Insert) return panel.enabled && !panel.recording && !panel.transcript.empty();
if (a == UiAction::Enter) return panel.enabled && !panel.recording;
if (a == UiAction::Toggle) return false;
return true;
}
bool visible(Control c) const {
if (c == Control::Prev || c == Control::Next) return !settings;
if (mounting(c) || c == Control::Recenter) return settings;
return true;
}
bool enabled(Control c) const {
if (auto a = action(c)) return available(*a);
if (c == Control::Prev) return page > 0;
if (c == Control::Next) return page + 1 < page_count();
if (c == Control::Recenter) return mount == Mount::World;
return true;
}
int hit(float x, float y) const {
for (size_t i = 0; i < buttons.size(); ++i)
if (visible(buttons[i].id) && enabled(buttons[i].id) && buttons[i].r.contains(x, y)) return int(i);
return -1;
}
void reset() {
if (std::any_of(pressed.begin(), pressed.end(), [](int i) { return i >= 0; }) ||
std::any_of(hovered.begin(), hovered.end(), [](int i) { return i >= 0; })) dirty = true;
pressed.fill(-1); hovered.fill(-1);
}
bool render(const Panel& p) {
if (p.recording != panel.recording || p.enabled != panel.enabled ||
p.record_available != panel.record_available || p.transcript != panel.transcript) {
reset(); // an old press cannot authorize a changed action or replacement transcript
dirty = true;
}
if (p.transcript != panel.transcript || lines.empty()) {
lines = wrap(p.transcript.empty() ? "Your words will appear here.\nReview them, then choose Insert." : p.transcript, 32, 904);
page = 0; dirty = true;
}
if (p.status != panel.status || p.detail != panel.detail || p.enabled != panel.enabled || p.recording != panel.recording)
dirty = true;
panel = p;
if (!dirty) return false;
rect({0, 0, W, H}, background);
frame();
text("FrameYap", 32, 61, 40, ink, 300);
text("ON-DEVICE / REVIEW FIRST", 280, 58, 22, muted, 720);
text(mount_label(mount), 756, 58, 24, cyan, 968);
rect({32, 84, 7, 34}, panel.recording ? pink : cyan);
auto status = wrap(panel.status, 27, 790);
text(status.front(), 54, 109, 27, ink, 844);
if (status.size() > 1) text("[...]", 864, 109, 23, muted, 968);
rect({32, 198, 936, 1}, {44, 61, 79, 255});
if (!settings) {
rect({32, 214, 936, 176}, card);
rect({32, 214, 3, 176}, {50, 102, 119, 255});
for (size_t i = 0; i < lines_per_page && page * lines_per_page + i < lines.size(); ++i)
text(lines[page * lines_per_page + i], 48, 246 + int(i) * 40, 32,
panel.transcript.empty() ? muted : ink, 952);
text("PAGE " + std::to_string(page + 1) + " / " + std::to_string(page_count()), 416, 436, 23, muted, 790);
auto detail = wrap(panel.detail, 24, 904);
for (size_t i = 0; i < std::min(size_t(2), detail.size()); ++i)
text(detail[i], 32, 486 + int(i) * 30, 24, muted, 936);
if (detail.size() > 2) text("[detail truncated]", 730, 546, 22, pink, 968);
} else {
text("MOUNT THE MENU", 32, 224, 22, muted, 968);
text("World stays put. Recenter places it in front of you.", 354, 426, 22, muted, 968);
text(placement_note.empty() ? "Choose a mount. Preference is saved on this device." : placement_note,
32, 486, 23, muted, 968);
text("Tracking lost? Wrist placement falls back to world space.", 32, 518, 23, muted, 968);
}
rect({32, 556, 936, 1}, {44, 61, 79, 255});
for (size_t i = 0; i < buttons.size(); ++i) {
const auto& b = buttons[i];
if (!visible(b.id)) continue;
bool on = enabled(b.id);
bool selected = (b.id == Control::Review && !settings) || (b.id == Control::Settings && settings) ||
(mounting(b.id) && *mounting(b.id) == mount);
bool hover = std::find(hovered.begin(), hovered.end(), int(i)) != hovered.end();
bool down = std::find(pressed.begin(), pressed.end(), int(i)) != pressed.end();
rect(b.r, !on ? Color{18, 23, 34, 255} : down ? Color{37, 74, 89, 255} :
hover || selected ? Color{27, 53, 68, 255} : card);
Color accent = b.id == Control::Record && panel.recording ? pink : cyan;
if (on && (hover || selected || b.id == Control::Record)) neon(b.r, accent);
else border(b.r, on ? Color{58, 79, 101, 255} : Color{30, 40, 53, 255});
if (selected) rect({b.r.x + 1, b.r.y + 1, 4, b.r.h - 2}, cyan);
const auto label = b.id == Control::Record && panel.recording ? "Stop" : b.label;
text(label, b.r.x + 16, b.r.y + b.r.h / 2 + 9, 27, on ? ink : Color{81, 96, 113, 255}, b.r.x + b.r.w - 8);
if (mounting(b.id) && selected) text("ON", b.r.x + b.r.w - 56, b.r.y + 38, 23, cyan, b.r.x + b.r.w - 12);
}
dirty = false;
return true;
}
};
PanelSurface::PanelSurface(const std::string& font, Mount mount) : impl_(std::make_unique<Impl>(font, mount)) {}
PanelSurface::~PanelSurface() = default;
bool PanelSurface::render(const Panel& p) { return impl_->render(p); }
const std::vector<unsigned char>& PanelSurface::pixels() const { return impl_->pixels; }
bool PanelSurface::available(UiAction a) const { return impl_->available(a); }
void PanelSurface::pointer_move(unsigned cursor, float x, float y) {
if (cursor >= impl_->hovered.size()) return;
const int next = impl_->hit(x, y);
if (next != impl_->hovered[cursor]) { impl_->hovered[cursor] = next; impl_->dirty = true; }
}
void PanelSurface::pointer_down(unsigned cursor, float x, float y) {
if (cursor >= impl_->pressed.size()) return;
impl_->pressed[cursor] = impl_->hit(x, y);
pointer_move(cursor, x, y);
impl_->dirty = true;
}
SurfaceEvent PanelSurface::pointer_up(unsigned cursor, float x, float y) {
SurfaceEvent result;
if (cursor >= impl_->pressed.size()) return result;
int index = std::exchange(impl_->pressed[cursor], -1);
pointer_move(cursor, x, y);
impl_->dirty = true;
if (index < 0 || impl_->hit(x, y) != index) return result;
auto c = buttons[index].id;
if (c == Control::Record) result.action = impl_->panel.recording ? UiAction::EndRecord : UiAction::BeginRecord;
else if (auto a = action(c)) result.action = a;
else if (auto m = mounting(c)) { impl_->mount = *m; result.mount = *m; impl_->reset(); }
else if (c == Control::Recenter) { result.recenter = true; impl_->reset(); }
else if (c == Control::Review || c == Control::Settings) { impl_->settings = c == Control::Settings; impl_->reset(); }
else if (c == Control::Prev) --impl_->page;
else if (c == Control::Next) ++impl_->page;
return result;
}
void PanelSurface::reset_pointers() { impl_->reset(); }
void PanelSurface::set_placement_note(std::string note) {
if (impl_->placement_note != note) { impl_->placement_note = std::move(note); impl_->dirty = true; }
}
} // namespace frameyap
+36
View File
@@ -0,0 +1,36 @@
#pragma once
#include "overlay.hpp"
#include <array>
#include <memory>
#include <optional>
#include <string>
#include <vector>
namespace frameyap {
struct SurfaceEvent {
std::optional<UiAction> action;
std::optional<Mount> mount;
bool recenter = false;
};
// One CPU RGBA canvas, independent of OpenVR. Settings replace the review area;
// status and safety controls remain on the same surface.
class PanelSurface {
public:
static constexpr int width = 1000, height = 680;
PanelSurface(const std::string& font, Mount mount);
~PanelSurface();
PanelSurface(const PanelSurface&) = delete;
PanelSurface& operator=(const PanelSurface&) = delete;
bool render(const Panel& panel);
const std::vector<unsigned char>& pixels() const;
void pointer_down(unsigned cursor, float x, float y);
SurfaceEvent pointer_up(unsigned cursor, float x, float y);
void pointer_move(unsigned cursor, float x, float y);
void reset_pointers();
void set_placement_note(std::string note);
bool available(UiAction action) const;
private:
struct Impl;
std::unique_ptr<Impl> impl_;
};
} // namespace frameyap
+4 -3
View File
@@ -40,7 +40,7 @@ int run(const Options& options) {
auto old_int = std::signal(SIGINT, signal_stop);
auto old_term = std::signal(SIGTERM, signal_stop);
struct Restore { decltype(old_int) a, b; ~Restore() { std::signal(SIGINT, a); std::signal(SIGTERM, b); } } restore{old_int, old_term};
Overlay overlay(options.assets, options.font, options.hand);
Overlay overlay(options.assets, options.font, options.mount);
Worker worker;
Audio audio;
Session session;
@@ -88,8 +88,9 @@ int run(const Options& options) {
}
// Drawing precedes input polling: Enter is disabled until Ready is visible.
Panel panel{state_label(session.state()), session.text(), detail,
session.state() != State::Error && session.state() != State::Warming,
session.state() == State::Recording};
session.state() != State::Error && session.state() != State::Warming && session.state() != State::Transcribing,
session.state() == State::Recording,
session.state() != State::Warming && session.state() != State::Transcribing && session.state() != State::Review};
if (panel.recording) panel.status += " - " + std::to_string(audio.seconds()) + " / 20s";
overlay.draw(panel);
for (auto action : overlay.poll()) {
+3 -1
View File
@@ -1,10 +1,12 @@
#pragma once
#include "mount.hpp"
#include <optional>
#include <string>
namespace frameyap {
struct Options {
std::string assets, font, python = "python3", worker, model, socket;
int threads = 2;
bool hand = true;
std::optional<Mount> mount; // saved preference, or world on first launch
};
int run(const Options& options);
}
+127
View File
@@ -0,0 +1,127 @@
#include "mount.hpp"
#include <array>
#include <cmath>
#include <cstdlib>
#include <filesystem>
#include <fstream>
#include <iostream>
#include <iterator>
#include <limits>
#include <string>
#include <string_view>
#include <unistd.h>
using namespace frameyap;
namespace fs = std::filesystem;
#define CHECK(x) do { if (!(x)) { std::cerr << "line " << __LINE__ << ": " #x "\n"; std::exit(1); } } while (0)
namespace {
constexpr Matrix34 identity{{{{1.f, 0.f, 0.f, 0.f}},
{{0.f, 1.f, 0.f, 0.f}},
{{0.f, 0.f, 1.f, 0.f}}}};
bool near(float a, float b) { return std::abs(a - b) < 1e-4f; }
void check_basis(const Matrix34& pose, float right_x, float right_z, float back_x, float back_z) {
CHECK(near(pose[0][0], right_x)); CHECK(near(pose[1][0], 0)); CHECK(near(pose[2][0], right_z));
CHECK(near(pose[0][1], 0)); CHECK(near(pose[1][1], 1)); CHECK(near(pose[2][1], 0));
CHECK(near(pose[0][2], back_x)); CHECK(near(pose[1][2], 0)); CHECK(near(pose[2][2], back_z));
}
struct TempDir {
fs::path path;
TempDir() {
std::string pattern = (fs::temp_directory_path() / "frameyap-mount-test-XXXXXX").string();
auto* dir = ::mkdtemp(pattern.data());
CHECK(dir != nullptr);
path = dir;
}
~TempDir() { std::error_code ec; fs::remove_all(path, ec); }
};
void put(const fs::path& path, const std::string& content) {
std::ofstream out(path, std::ios::binary | std::ios::trunc);
CHECK(bool(out)); out << content; CHECK(bool(out));
}
struct EnvRestore {
std::string key;
bool had;
std::string original;
explicit EnvRestore(const char* k) : key(k), had(std::getenv(k) != nullptr), original(had ? std::getenv(k) : "") {}
~EnvRestore() { if (had) ::setenv(key.c_str(), original.c_str(), 1); else ::unsetenv(key.c_str()); }
};
} // namespace
int main() {
const std::array<Mount, 4> mounts{Mount::World, Mount::LeftWrist, Mount::RightWrist, Mount::Head};
const std::array<std::string_view, 4> names{"world", "left-wrist", "right-wrist", "head"};
for (std::size_t i = 0; i < mounts.size(); ++i) {
CHECK(mount_name(mounts[i]) == names[i]);
CHECK(parse_mount(names[i]) == mounts[i]);
}
CHECK(!parse_mount("World")); CHECK(!parse_mount("head\n")); CHECK(!parse_mount("left_wrist"));
CHECK(!parse_mount("")); CHECK(!parse_mount("world extra"));
CHECK(near(mount_width(Mount::World), .85f)); CHECK(near(mount_width(Mount::Head), .85f));
CHECK(near(mount_width(Mount::LeftWrist), .42f)); CHECK(near(mount_width(Mount::RightWrist), .42f));
CHECK(relative_mount_pose(Mount::World) == identity);
const auto head = relative_mount_pose(Mount::Head);
check_basis(head, 1, 0, 0, 1);
CHECK(near(head[0][3], 0)); CHECK(near(head[1][3], -.16f)); CHECK(near(head[2][3], -1.05f));
const auto left = relative_mount_pose(Mount::LeftWrist), right = relative_mount_pose(Mount::RightWrist);
check_basis(left, 1, 0, 0, 1); check_basis(right, 1, 0, 0, 1);
CHECK(near(left[0][3], .13f)); CHECK(near(right[0][3], -.13f));
CHECK(near(left[1][3], .12f)); CHECK(near(right[1][3], .12f));
CHECK(near(left[2][3], -.18f)); CHECK(near(right[2][3], -.18f));
auto hmd = identity;
hmd[0][3] = 2; hmd[1][3] = 1.7f; hmd[2][3] = 3;
auto pose = world_mount_pose(hmd);
CHECK(pose); check_basis(*pose, 1, 0, 0, 1);
CHECK(near((*pose)[0][3], 2)); CHECK(near((*pose)[1][3], 1.54f)); CHECK(near((*pose)[2][3], 1.95f));
// 90 degree yaw: HMD looks +X. The panel's +Z points back toward -X.
hmd[0][0] = 0; hmd[0][2] = -1; hmd[2][0] = 1; hmd[2][2] = 0;
pose = world_mount_pose(hmd);
CHECK(pose); check_basis(*pose, 0, 1, -1, 0);
CHECK(near((*pose)[0][3], 3.05f)); CHECK(near((*pose)[2][3], 3));
// Pitch must not tilt the panel or change its upright orientation.
hmd = identity;
hmd[1][1] = .8f; hmd[1][2] = -.6f;
hmd[2][1] = .6f; hmd[2][2] = .8f;
pose = world_mount_pose(hmd);
CHECK(pose); check_basis(*pose, 1, 0, 0, 1);
CHECK(near((*pose)[1][3], -.16f)); CHECK(near((*pose)[2][3], -1.05f));
hmd[0][2] = 0; hmd[2][2] = 0; // forward is vertical, yaw undefined
CHECK(!world_mount_pose(hmd));
hmd[2][2] = 1e-6f; CHECK(!world_mount_pose(hmd));
hmd = identity; hmd[0][3] = std::numeric_limits<float>::infinity(); CHECK(!world_mount_pose(hmd));
hmd = identity; hmd[0][0] = std::numeric_limits<float>::quiet_NaN(); CHECK(!world_mount_pose(hmd));
TempDir tmp;
const auto file = tmp.path / "nested" / "mount";
CHECK(load_mount(file) == Mount::World); CHECK(!save_mount({}, Mount::Head));
for (auto mount : mounts) {
CHECK(save_mount(file, mount));
CHECK(load_mount(file) == mount);
std::ifstream in(file, std::ios::binary);
CHECK(bool(in));
const std::string data{std::istreambuf_iterator<char>(in), std::istreambuf_iterator<char>()};
CHECK(data == std::string(mount_name(mount)) + "\n");
}
put(file, "other\n"); CHECK(load_mount(file) == Mount::World);
put(file, "head\nextra\n"); CHECK(load_mount(file) == Mount::World);
put(file, std::string(128, 'a')); CHECK(load_mount(file) == Mount::World);
put(file, std::string("head\0\n", 6)); CHECK(load_mount(file) == Mount::World);
CHECK(!save_mount(tmp.path, Mount::Head)); // cannot rename over an existing directory
CHECK(!save_mount(file, static_cast<Mount>(-1)));
CHECK(load_mount(tmp.path) == Mount::World);
EnvRestore restore_xdg("XDG_CONFIG_HOME"), restore_home("HOME");
::setenv("XDG_CONFIG_HOME", tmp.path.c_str(), 1);
::setenv("HOME", "/other-absolute-home", 1);
CHECK(default_mount_settings_path() == tmp.path / "frameyap/mount");
::setenv("XDG_CONFIG_HOME", "relative", 1);
CHECK(default_mount_settings_path() == fs::path("/other-absolute-home/.config/frameyap/mount"));
::setenv("HOME", "relative", 1);
CHECK(default_mount_settings_path().empty());
::unsetenv("HOME"); ::unsetenv("XDG_CONFIG_HOME");
CHECK(default_mount_settings_path().empty());
std::cout << "mount checks passed\n";
}
+136
View File
@@ -0,0 +1,136 @@
#include "panel_surface.hpp"
#include <cassert>
#include <fstream>
#include <iostream>
#include <limits>
#include <string>
using namespace frameyap;
namespace {
SurfaceEvent click(PanelSurface& surface, float x, float y, unsigned cursor = 0) {
surface.pointer_down(cursor, x, y);
return surface.pointer_up(cursor, x, y);
}
void no_action(const SurfaceEvent& event) { assert(!event.action && !event.mount && !event.recenter); }
void snapshot(PanelSurface& surface, const std::string& path) {
std::ofstream out(path, std::ios::binary);
assert(out);
out << "P6\n" << PanelSurface::width << ' ' << PanelSurface::height << "\n255\n";
const auto& pixels = surface.pixels();
for (size_t i = 0; i < pixels.size(); i += 4) out.write(reinterpret_cast<const char*>(pixels.data() + i), 3);
assert(out);
}
}
int main(int argc, char** argv) {
assert(argc >= 2);
PanelSurface surface(argv[1], Mount::World);
Panel p{"Ready to record", "", "Focus your destination before Insert. Enter is always separate.", false, false};
assert(surface.render(p));
assert(surface.pixels().size() == size_t(PanelSurface::width * PanelSurface::height * 4));
assert(!surface.render(p));
surface.reset_pointers();
assert(!surface.render(p)); // repeated inactive-input resets do not force uploads
assert(surface.pixels()[3] == 0); // rounded outer corner is transparent
assert(surface.pixels()[(40 * PanelSurface::width + 40) * 4 + 3] == 255);
// Disabled controls never emit insertion/submission. Retry/Cancel remain available.
no_action(click(surface, 480, 610)); no_action(click(surface, 680, 610));
assert(click(surface, 100, 610).action == UiAction::BeginRecord);
assert(click(surface, 280, 610).action == UiAction::Cancel);
no_action(surface.pointer_up(0, 100, 610));
surface.pointer_down(0, 100, 610);
no_action(surface.pointer_up(0, 280, 610));
surface.pointer_down(0, 100, 610);
surface.reset_pointers();
no_action(surface.pointer_up(0, 100, 610));
no_action(click(surface, 100, 610, 2));
no_action(click(surface, std::numeric_limits<float>::quiet_NaN(), 610));
no_action(click(surface, -1, 610));
no_action(click(surface, 1000, 680));
p.enabled = true;
p.record_available = false;
p.status = "Review your words";
p.transcript = "A quieter way to type in VR.\nKeep the menu where it feels comfortable.\nNothing is sent until you choose Insert.";
assert(surface.render(p));
surface.reset_pointers(); surface.render(p);
if (argc >= 3) snapshot(surface, std::string(argv[2]) + "-review.ppm");
assert(click(surface, 480, 610).action == UiAction::Insert);
assert(click(surface, 680, 610).action == UiAction::Enter);
// Down/up must use the same controller, and release rechecks availability.
surface.pointer_down(0, 480, 610);
no_action(surface.pointer_up(1, 480, 610));
p.recording = true; p.status = "Recording - 00:04";
surface.render(p);
no_action(surface.pointer_up(0, 480, 610));
no_action(click(surface, 680, 610));
assert(click(surface, 100, 610).action == UiAction::EndRecord); // explicit Stop, never a stale toggle
surface.reset_pointers(); surface.render(p);
if (argc >= 3) snapshot(surface, std::string(argv[2]) + "-recording.ppm");
surface.pointer_down(0, 100, 610); // an old Stop release must not begin recording again
p.recording = false; p.status = "Review your words";
surface.render(p);
no_action(surface.pointer_up(0, 100, 610));
p.record_available = false; p.enabled = false; // warming/transcribing
surface.render(p);
no_action(click(surface, 100, 610)); no_action(click(surface, 680, 610));
assert(click(surface, 280, 610).action == UiAction::Cancel);
p.enabled = true; // review: Insert/Enter available, Record waits for discard/insertion
surface.render(p);
no_action(click(surface, 100, 610));
assert(click(surface, 480, 610).action == UiAction::Insert);
surface.render(p);
// Settings replace preview in the same RGBA surface; footer remains available.
no_action(click(surface, 290, 160));
surface.render(p);
surface.reset_pointers(); surface.render(p);
if (argc >= 3) snapshot(surface, std::string(argv[2]) + "-settings.ppm");
assert(click(surface, 200, 420).recenter);
auto event = click(surface, 180, 332);
assert(event.mount == Mount::LeftWrist && !event.action);
no_action(click(surface, 200, 420)); // recenter is world-only
assert(click(surface, 680, 332).mount == Mount::RightWrist);
assert(click(surface, 680, 260).mount == Mount::Head);
assert(click(surface, 180, 260).mount == Mount::World);
assert(click(surface, 280, 610).action == UiAction::Cancel);
surface.set_placement_note("Wrist not tracked - using world space until it returns.");
assert(surface.render(p)); assert(!surface.render(p));
surface.set_placement_note("Wrist not tracked - using world space until it returns.");
assert(!surface.render(p));
// Switching tabs invalidates the other pointer's old press.
surface.pointer_down(1, 680, 260);
no_action(click(surface, 100, 160));
no_action(surface.pointer_up(1, 680, 260));
no_action(click(surface, 680, 260)); // mount controls not active on Review
// Long UTF-8, newlines and malformed bytes are bounded, paginated and navigable.
p.transcript.clear();
for (int i = 0; i < 60; ++i) p.transcript += "Line " + std::to_string(i) + ": café / 日本語 / naïve / Ω\n";
p.transcript += std::string("invalid: \xff\xc0\xe0\x80", 13);
surface.render(p);
surface.reset_pointers(); surface.render(p);
const auto first = surface.pixels();
no_action(click(surface, 900, 430));
surface.reset_pointers(); assert(surface.render(p));
assert(surface.pixels() != first);
no_action(click(surface, 90, 430));
surface.reset_pointers(); surface.render(p);
assert(surface.pixels() == first);
for (int i = 0; i < 200; ++i) { no_action(click(surface, 900, 430)); surface.render(p); }
for (int i = 0; i < 200; ++i) { no_action(click(surface, 90, 430)); surface.render(p); }
surface.reset_pointers(); surface.render(p);
assert(surface.pixels() == first);
p.transcript = "New result";
assert(surface.render(p));
surface.reset_pointers(); surface.render(p);
auto replaced = surface.pixels();
no_action(click(surface, 900, 430));
surface.reset_pointers(); surface.render(p);
assert(surface.pixels() == replaced);
p.status = std::string(4096, 's'); p.detail = std::string(4096, 'd');
assert(surface.render(p)); assert(!surface.render(p));
surface.pointer_move(0, 900, 610); assert(surface.render(p));
surface.pointer_move(0, 900, 610); assert(!surface.render(p));
assert(click(surface, 900, 610).action == UiAction::Quit);
std::cout << "panel checks passed (no OpenVR, microphone or input injection)\n";
}