diff --git a/README.md b/README.md index c798b6c..8d391f7 100644 --- a/README.md +++ b/README.md @@ -23,9 +23,14 @@ See [third-party notes](docs/third-party.md). No GitHub release is published yet transcribe. Repeated presses reached the controls-only diagnostic on Frame; live mic capture through this shortcut still needs guided acceptance. This PTT action is remappable through SteamVR bindings. -- **Left grip (when active):** double-tap to explicitly send Enter. Never inferred from speech. +- **Right B:** cancel/discard. **Right A:** insert reviewed text with a trailing space. + **Right Y:** insert pending text, then send Enter; with no pending text, Enter only. + **Left grip (when active):** double-tap for the same explicit Enter action. + Nothing inserts or submits automatically after transcription. - **Overlay:** Record/Stop, Cancel, paginated preview, Insert, Enter and Quit. - Review and settings share one Inconsolata/neon-framed surface. World-space by + Review, settings and bindings share one Inconsolata/neon-framed surface. + **Bindings → Edit in SteamVR** opens the runtime binding editor; the tab shows + runtime-provided control names. Existing SteamVR overrides may supersede defaults. World-space by default; settings offer left wrist, right wrist and head mounting, plus recenter. Dashboard lasers provide clickable controls. Settings → Lasers anytime is an opt-in, default-off system-wide laser mode while the panel is visible; it may @@ -39,8 +44,8 @@ See [third-party notes](docs/third-party.md). No GitHub release is published yet overlays**. FrameYap then requests priority for its bound controller sources. This may consume controls used by games or the dashboard; coexistence on Frame is under test. The default is `"normal"`; restart FrameYap after changing it. -- **Review-first:** focus your destination, then press Insert. No automatic insertion - or submission. Maximum clip 20 seconds; accidental taps under 200 ms are discarded. +- **Review-first:** focus your destination, then press Insert (text + space) or + explicitly Enter (text + space, then Enter). No automatic insertion or submission. Maximum clip 20 seconds; accidental taps under 200 ms are discarded. - While the native app is Ready, it keeps the mic device open and discards idle audio instead of opening/closing on every PTT. Quit releases the device. Other apps may still hear/transmit your voice; FrameYap does not mute them. diff --git a/assets/actions.json b/assets/actions.json index 80188f7..253f2cb 100644 --- a/assets/actions.json +++ b/assets/actions.json @@ -17,11 +17,11 @@ "localization": [{ "language_tag": "en_US", "/actions/frameyap": "FrameYap controls", - "/actions/frameyap/in/left_grip": "Double tap left grip: Enter (enabled only)", + "/actions/frameyap/in/left_grip": "Double tap left grip: insert pending text + Enter (enabled only)", "/actions/frameyap/in/right_grip": "Tap right grip briefly, then squeeze and hold to record; release to finish", "/actions/frameyap/in/ptt": "Hold right X to record (default; remappable)", "/actions/frameyap/in/cancel": "Cancel recording or discard", - "/actions/frameyap/in/insert": "Insert reviewed text", - "/actions/frameyap/in/enter": "Explicit Enter" + "/actions/frameyap/in/insert": "Insert reviewed text + trailing space", + "/actions/frameyap/in/enter": "Insert pending text + Enter (Enter only if no text)" }] } diff --git a/assets/bindings_frame_controller.json b/assets/bindings_frame_controller.json index b563bbe..0c59c62 100644 --- a/assets/bindings_frame_controller.json +++ b/assets/bindings_frame_controller.json @@ -1,6 +1,6 @@ { "controller_type": "frame_controller", - "description": "Hold right X to dictate; release to review. Grip gestures remain remappable in SteamVR.", + "description": "Right X: hold to dictate. B: cancel. A: insert + space. Y: insert + Enter. Remap in SteamVR.", "name": "FrameYap Steam Frame controls", "bindings": { "/actions/frameyap": { @@ -19,6 +19,21 @@ "path": "/user/hand/right/input/x", "mode": "button", "inputs": { "click": { "output": "/actions/frameyap/in/ptt" } } + }, + { + "path": "/user/hand/right/input/b", + "mode": "button", + "inputs": { "click": { "output": "/actions/frameyap/in/cancel" } } + }, + { + "path": "/user/hand/right/input/a", + "mode": "button", + "inputs": { "click": { "output": "/actions/frameyap/in/insert" } } + }, + { + "path": "/user/hand/right/input/y", + "mode": "button", + "inputs": { "click": { "output": "/actions/frameyap/in/enter" } } } ] } diff --git a/assets/config.example.json b/assets/config.example.json index 894826d..d4ae151 100644 --- a/assets/config.example.json +++ b/assets/config.example.json @@ -16,8 +16,8 @@ "left_grip": "/user/hand/left/input/grip", "right_grip": "/user/hand/right/input/grip", "ptt": "/user/hand/right/input/x", - "cancel": "", - "insert": "", - "enter": "" + "cancel": "/user/hand/right/input/b", + "insert": "/user/hand/right/input/a", + "enter": "/user/hand/right/input/y" } } diff --git a/docs/design.md b/docs/design.md index d39bd65..814c70b 100644 --- a/docs/design.md +++ b/docs/design.md @@ -54,22 +54,25 @@ armed remains a possible refinement, not a second implemented overlay. See Recording… 00:04 [ Cancel ] "The recognized text appears here." -[ Insert ] [ Discard ] [ Enter — separate action ] +[ Insert + space ] [ Discard ] [ Insert + Enter — explicit ] ``` - States: disabled, warming, ready, recording, transcribing, review, inserted, unavailable/error. Recording uses visible icon + text, not color alone. -- Default POC binding: tap right grip briefly, then hold the second squeeze to - speak; release to finish. Double-tap left grip is a separate explicit Enter. - Both are remappable, with a separate named hold-to-talk action available. +- Default Frame bindings: hold right X to speak, release to finish; B cancels, + A inserts with a trailing space, Y inserts pending text then sends Enter. + The alternate right-grip tap-then-hold gesture and left-grip double-tap Enter + remain available. All are remappable through the Bindings tab's SteamVR editor. A click-to-start/stop overlay button provides a binding-independent alternative. Bound recording to 20 seconds; discard accidental taps (initial threshold: 200 ms). - **Quick typing:** insert on completion only when the explicitly armed target is still valid. **Review mode:** always wait for Insert. Bring up review instead of silently losing a transcript or typing into a new target. -- Enter is always a separate press after insertion. Never interpret "submit", - "delete" or other speech as commands in this utility. Do not auto-submit. +- Enter requires its own explicit control activation: insert pending review with + a trailing space, then queue Enter only if the text step succeeds. With no + review, it queues only Enter. Never interpret "submit", "delete" or other speech + as commands. Transcription completion never inserts or auto-submits. - No generic "undo last dictation" initially: another application's edits/cursor cannot be reliably rolled back by a guessed number of backspaces. - Stop/disable releases owned keys and microphone, invalidates pending delivery, diff --git a/docs/overlay.md b/docs/overlay.md index 297883b..2f7a3f7 100644 --- a/docs/overlay.md +++ b/docs/overlay.md @@ -19,7 +19,7 @@ Sans face if present. Glyph coverage depends on the selected face; full CJK coverage is not claimed. `src/panel_surface.*` renders **one 1000×680 RGBA canvas** for review, settings, -status and controls. `src/overlay_texture.*` uploads this CPU canvas into one +bindings, status and controls. `src/overlay_texture.*` uploads this CPU canvas into one persistent Vulkan RGBA8 image and submits it with `SetOverlayTexture`. The image, staging allocation and command buffer are reused; tabs do not create extra overlays or render targets. The rounded mint-to-blue perimeter, @@ -53,14 +53,38 @@ native installation and offscreen GPU checks separately from headset acceptance. The complete transcript preview is paginated by glyph width and four-line height; Previous and Next navigate it without changing the source transcript. Long status/detail messages show a prefix with a visible truncation marker. -The footer remains available on both tabs: Record (labelled Stop while recording), +The footer remains available on all tabs: Record (labelled Stop while recording), Cancel, Insert, Enter, Quit. Record can retry after an error; it is disabled while warming/transcribing and until an existing review is inserted or discarded. Cancel can stop worker startup. Insert and Enter are disabled during recording and transcription. A pointer action requires a press/release on the same enabled control from the same cursor; focus loss, tab changes, action-state changes and relocation clear pending presses. Enter is *always* a separate -deliberate action, not inferred from text. Recording never automatically submits Enter. +deliberate action, not inferred from text. Insert appends a trailing space (without +doubling an existing trailing space). Enter inserts any pending review and then +queues Enter; with no pending text it queues Enter only. A failed text step never +proceeds to Enter. Recording never automatically inserts or submits. + +### Bindings menu + +The **Bindings** tab shows the runtime-provided hand/control names for PTT, +Cancel, Insert, Enter and both grip gestures. **Edit in SteamVR** requests the +in-headset binding editor for the current process/action set. SteamVR owns +remapping and persistence; the tab refreshes its origin labels once per second +while visible. Unknown/unavailable origins are reported instead of showing +bundled defaults as if they were live bindings. Long labels show a truncation +marker; the editor is the full binding view. Opening it clears pending pointer +presses and rearms controller gestures from neutral. + +OpenVR 2.15.6 provides `GetActionOrigins`, `GetOriginLocalizedName` and +`OpenBindingUI`, not a generic 2D button-glyph API. Frame's installed controller +profile references left/right SVG diagrams for SteamVR's own editor. FrameYap +uses runtime text labels; it does not copy runtime artwork into its package. +Editor availability and artwork rendering still require headset acceptance. + +Normal priority with Lasers anytime off is the practical baseline: the wearer +reports controller actions with the dashboard closed and clickable UI with it +open. The menu does not enable global overrides or promise simultaneous access. ### User theme and controller configuration @@ -82,6 +106,9 @@ installer creates one with defaults on first install. Copy the shipped }, "buttons": { "ptt": "/user/hand/right/input/x", + "cancel": "/user/hand/right/input/b", + "insert": "/user/hand/right/input/a", + "enter": "/user/hand/right/input/y", "left_grip": "/user/hand/left/input/grip", "right_grip": "/user/hand/right/input/grip" } @@ -93,7 +120,10 @@ TTF/OTF file path (not a family name); a missing file uses the bundled font. `buttons` maps named OpenVR actions (`left_grip`, `right_grip`, `ptt`, `cancel`, `insert`, `enter`) to Frame physical `/user/hand/{left|right}/input/NAME` button paths. Omitted actions retain their bundled defaults; an empty string -disables a mapping. Paths must be distinct. Only the Frame binding is customized; +disables a mapping, including after an upgrade. The Frame defaults are right +X = hold-to-talk, B = Cancel, A = Insert + space, Y = Insert + Enter. Existing +configs with empty actions retain those disabled mappings; change them explicitly +or use SteamVR's binding editor. Paths must be distinct. Only the Frame binding is customized; SteamVR user overrides may still supersede it. On customized launches a generated action manifest and adjacent bindings are placed in `$XDG_CACHE_HOME/frameyap/bindings` (or `~/.cache/frameyap/bindings`); the bundled manifest remains unchanged. The @@ -202,7 +232,7 @@ ctest --test-dir build-ui --output-on-failure ./build-ui/frameyap_panel_test assets/fonts/Inconsolata-Regular.ttf /tmp/frameyap-ui ``` -The last command writes `-review.ppm`, `-settings.ppm`, `-recording.ppm` to the +The last command writes `-review.ppm`, `-settings.ppm`, `-bindings.ppm` and `-recording.ppm` to the supplied prefix. The native build includes these tests too; tests never initialize OpenVR or touch the real mounting preference. Physical pointing, tracking loss, recentring and readability still require a separately authorized headset check. diff --git a/docs/poc.md b/docs/poc.md index 4aa4242..dab8645 100644 --- a/docs/poc.md +++ b/docs/poc.md @@ -11,7 +11,9 @@ initialize OpenVR, open a microphone, run ASR, download files or inject input. mode while FrameYap is visible; it may affect games, and is not an input override. - Remappable SteamVR actions. The default Steam Frame binding maps right X (hold to record, release to transcribe) to the existing PTT action using the - observed `frame_controller` profile. Grip bindings remain, but both grip + observed `frame_controller` profile. Right B cancels, A inserts + space, and Y + inserts pending text + Enter (or Enter only with no preview). A Bindings tab + shows runtime origin labels and opens SteamVR's remapping editor. Grip bindings remain, but both grip actions were inactive in the observed dashboard check; do not rely on them. If left grip becomes active, two short taps request explicit Enter. For grip gestures, first squeeze <=250 ms; second squeeze begins <=350 ms @@ -32,7 +34,12 @@ initialize OpenVR, open a microphone, run ASR, download files or inject input. in-flight request or broken worker protocol may require reloading. No cloud/desktop fallback. - Gamescope IME v2 generated bindings, per-action short-lived lease, unavailable - handling, UTF-8/control validation and explicit separate Submit action for Enter. + handling, UTF-8/control validation and explicit Submit action for Enter. + Insert ensures a trailing space without doubling an existing one. Enter + first inserts pending review, releases the text lease, then acquires a fresh + lease for Submit. Failed/uncertain text never proceeds to Submit; failed + Submit acquisition never replays text. A full 4096-byte transcript without + room for a space is preserved with an error, never silently truncated. - Idempotent user-local release-archive installer: SHA-256, safe extraction, atomic current-version selection, retained rollback, runtime/install lock, foreign-file refusal and explicit unregister-before-uninstall acknowledgement. diff --git a/install.sh b/install.sh index 13742dd..7c678fb 100755 --- a/install.sh +++ b/install.sh @@ -40,7 +40,8 @@ CONFIG_DEFAULTS = { "frame_start": "#1fff91", "frame_end": "#1f70ff"}, "buttons": {"left_grip": "/user/hand/left/input/grip", "right_grip": "/user/hand/right/input/grip", - "ptt": "/user/hand/right/input/x", "cancel": "", "insert": "", "enter": ""}, + "ptt": "/user/hand/right/input/x", "cancel": "/user/hand/right/input/b", + "insert": "/user/hand/right/input/a", "enter": "/user/hand/right/input/y"}, } COLOR_RE = re.compile(r"#[0-9a-fA-F]{6}\Z") BUTTON_RE = re.compile(r"/user/hand/(left|right)/input/[A-Za-z0-9_]+\Z") diff --git a/scripts/install_payload.py b/scripts/install_payload.py index 4643f78..c7216be 100644 --- a/scripts/install_payload.py +++ b/scripts/install_payload.py @@ -29,7 +29,8 @@ CONFIG_DEFAULTS = { "frame_start": "#1fff91", "frame_end": "#1f70ff"}, "buttons": {"left_grip": "/user/hand/left/input/grip", "right_grip": "/user/hand/right/input/grip", - "ptt": "/user/hand/right/input/x", "cancel": "", "insert": "", "enter": ""}, + "ptt": "/user/hand/right/input/x", "cancel": "/user/hand/right/input/b", + "insert": "/user/hand/right/input/a", "enter": "/user/hand/right/input/y"}, } COLOR_RE = re.compile(r"#[0-9a-fA-F]{6}\Z") BUTTON_RE = re.compile(r"/user/hand/(left|right)/input/[A-Za-z0-9_]+\Z") diff --git a/src/config.cpp b/src/config.cpp index 536345a..31fa818 100644 --- a/src/config.cpp +++ b/src/config.cpp @@ -254,7 +254,9 @@ std::filesystem::path action_manifest(const std::string& assets, const Config& c auto dir = root / "frameyap/bindings"; std::filesystem::create_directories(dir); std::map buttons{{"left_grip", "/user/hand/left/input/grip"}, - {"right_grip", "/user/hand/right/input/grip"}, {"ptt", "/user/hand/right/input/x"}}; + {"right_grip", "/user/hand/right/input/grip"}, {"ptt", "/user/hand/right/input/x"}, + {"cancel", "/user/hand/right/input/b"}, {"insert", "/user/hand/right/input/a"}, + {"enter", "/user/hand/right/input/y"}}; for (const auto& [key, path] : config.buttons) buttons[key] = path; std::string binding = "{\"controller_type\":\"frame_controller\",\"name\":\"FrameYap configured controls\",\"bindings\":{\"/actions/frameyap\":{\"sources\":["; std::map used; diff --git a/src/core.cpp b/src/core.cpp index 5d362e3..32dacc1 100644 --- a/src/core.cpp +++ b/src/core.cpp @@ -57,4 +57,47 @@ std::optional Session::take_insert() { } void Session::cancel() { ++id_; text_.clear(); state_ = State::Ready; } void Session::fail() { ++id_; text_.clear(); state_ = State::Error; } +namespace { +std::unique_ptr lease(const DeliveryFactory& acquire) { + auto result = acquire(); + if (!result) throw std::runtime_error("Input lease unavailable"); + return result; +} +DeliveryResult send_review(Session& session, const DeliveryFactory& acquire, bool submit) { + // literal_text already validated the transcript at reply time. Do not + // truncate at the boundary: the suffix is also subject to the 4096-byte + // Gamescope text limit, including any existing trailing whitespace. + const bool needs_space = session.text().back() != ' '; + if (needs_space && session.text().size() >= 4096) + throw std::runtime_error("Cannot append space: transcript fills the 4096-byte input limit; discard and dictate a shorter clip"); + auto spaced = session.text(); + if (needs_space) spaced += ' '; + auto input = lease(acquire); // Failed acquisition keeps the review available. + session.take_insert(); // Consume before any potentially ambiguous send. + try { input->text(spaced); } + catch (...) { return DeliveryResult::TextUncertain; } + if (!submit) return DeliveryResult::TextQueued; + // The Gamescope IME is exclusive and each lease is single-use. Release + // the text lease BEFORE acquiring another one for Enter. Never send Enter + // unless text returned successfully; do not replay text on lease failure. + input.reset(); + try { input = lease(acquire); } + catch (...) { return DeliveryResult::TextQueuedEnterUnavailable; } + try { input->enter(); } + catch (...) { return DeliveryResult::EnterUncertain; } + return DeliveryResult::EnterQueued; +} +} +DeliveryResult deliver_insert(Session& session, const DeliveryFactory& acquire) { + if (session.state() != State::Review) return DeliveryResult::Ignored; + return send_review(session, acquire, false); +} +DeliveryResult deliver_enter(Session& session, const DeliveryFactory& acquire) { + if (session.state() == State::Review) return send_review(session, acquire, true); + if (session.state() != State::Ready && session.state() != State::Queued) return DeliveryResult::Ignored; + auto input = lease(acquire); + try { input->enter(); } + catch (...) { return DeliveryResult::EnterUncertain; } + return DeliveryResult::EnterQueued; +} } diff --git a/src/core.hpp b/src/core.hpp index 307d6f3..e640e39 100644 --- a/src/core.hpp +++ b/src/core.hpp @@ -1,5 +1,7 @@ #pragma once #include +#include +#include #include #include #include @@ -27,4 +29,23 @@ private: uint64_t id_ = 0; std::string text_; }; +// Each lease is single-use. Constructing it must only check availability, not +// deliver input. After a possibly ambiguous text failure the review is consumed. +class DeliveryLease { +public: + virtual ~DeliveryLease() = default; + virtual void text(const std::string& literal) = 0; + virtual void enter() = 0; +}; +using DeliveryFactory = std::function()>; +enum class DeliveryResult { + Ignored, TextQueued, EnterQueued, TextUncertain, EnterUncertain, TextQueuedEnterUnavailable +}; +// Lease acquisition before consumption can throw while leaving review intact. +// The suffix must fit inside the same 4096-byte validated text bound; overflow +// throws before consumption rather than silently dropping transcript bytes. +DeliveryResult deliver_insert(Session& session, const DeliveryFactory& acquire); +// Review: insert text plus a trailing space, then explicitly send Enter only +// after successful text delivery. Ready/Queued: Enter alone. +DeliveryResult deliver_enter(Session& session, const DeliveryFactory& acquire); } diff --git a/src/main.cpp b/src/main.cpp index 9c1aa16..dae3fa7 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -31,7 +31,8 @@ void help() { "SteamVR Developer setting Enable global input from overlays.\n" "Right X: hold to speak, release to review (default Frame binding).\n" "Grip gestures are remappable but may be unavailable in the dashboard.\n" - "Left grip: double-tap for explicit Enter when active.\n" + "Right B: cancel; A: insert + space; Y: insert pending text + Enter.\n" + "Left grip: double-tap for the same explicit Enter action when active.\n" "Review first: Insert approves current focus; never automatic Enter.\n" "No device access unless an explicit runtime/check/registration mode is used.\n"; #ifndef FRAMEYAP_NATIVE diff --git a/src/overlay.cpp b/src/overlay.cpp index ab74fac..3ba23c2 100644 --- a/src/overlay.cpp +++ b/src/overlay.cpp @@ -92,6 +92,7 @@ struct Overlay::Impl { unsigned overlay_shown_events = 0, overlay_hidden_events = 0, image_loaded_events = 0, image_failed_events = 0; unsigned overlay_focus_events = 0, global_focus_events = 0, input_focus_captured_events = 0; std::string last_pointer_event = "none"; + std::chrono::steady_clock::time_point next_binding_refresh{}; Impl(const std::string& assets, const std::string& font, std::optional requested, bool persist) : settings_path(default_mount_settings_path()), @@ -290,6 +291,34 @@ struct Overlay::Impl { } return {true, data.bState}; } + void refresh_bindings() { + if (!surface.bindings_visible()) { next_binding_refresh = {}; return; } + const auto now = std::chrono::steady_clock::now(); + if (now < next_binding_refresh) return; + next_binding_refresh = now + std::chrono::seconds(1); + std::array labels; + constexpr std::array order{{2, 3, 4, 5, 0, 1}}; + for (size_t row = 0; row < order.size(); ++row) { + std::array origins{}; + const auto error = input->GetActionOrigins(action_set, actions[order[row]], origins.data(), uint32_t(origins.size())); + if (error != vr::VRInputError_None) { labels[row] = "Binding information unavailable"; continue; } + for (auto origin : origins) { + if (origin == vr::k_ulInvalidInputValueHandle) continue; + std::array name{}; + if (input->GetOriginLocalizedName(origin, name.data(), uint32_t(name.size()), + vr::VRInputString_Hand | vr::VRInputString_InputSource) != vr::VRInputError_None || !name[0]) { + if (!labels[row].empty()) labels[row] += " / "; + labels[row] += "Bound (name unavailable)"; + } else { + name.back() = '\0'; + if (!labels[row].empty()) labels[row] += " / "; + labels[row] += name.data(); + } + } + if (labels[row].empty()) labels[row] = "Unbound / controller unavailable"; + } + surface.set_bindings(std::move(labels)); + } std::vector poll() { std::vector result; system->GetDeviceToAbsoluteTrackingPose(vr::TrackingUniverseStanding, 0, poses.data(), uint32_t(poses.size())); @@ -342,8 +371,18 @@ struct Overlay::Impl { last_pointer_event = "up button=" + std::to_string(event.data.mouse.button); if (event.data.mouse.button == vr::VRMouseButton_Left) { auto event_result = surface.pointer_up(event.data.mouse.cursorIndex, event.data.mouse.x, H - event.data.mouse.y); - if (event_result.action || event_result.mount || event_result.recenter || event_result.lasers_anytime) ++pointer_actions; + if (event_result.action || event_result.mount || event_result.recenter || event_result.lasers_anytime || event_result.open_bindings) ++pointer_actions; if (event_result.action) result.push_back(*event_result.action); + if (event_result.open_bindings) { + // The editor changes input ownership. Invalidate held gestures + // and pointer presses; never turn the returning release into input. + reset_input(result); + const auto error = input->OpenBindingUI(nullptr, action_set, vr::k_ulInvalidInputValueHandle, false); + surface.set_binding_note(error == vr::VRInputError_None + ? "SteamVR editor requested. Changes appear here when available." + : "SteamVR could not open bindings. Try its controller settings."); + next_binding_refresh = {}; + } if (event_result.lasers_anytime) { const bool enabled = *event_result.lasers_anytime; if (overlay->SetOverlayFlag(handle, vr::VROverlayFlags_MakeOverlaysInteractiveIfVisible, @@ -377,6 +416,7 @@ struct Overlay::Impl { // experiment compares. SteamVR's separate permission gate is never changed here. set.nPriority = action_priority(); action_update_error = input->UpdateActionState(&set, sizeof(set), 1); + refresh_bindings(); if (action_update_error != vr::VRInputError_None) { reset_input(result); return result; } diff --git a/src/panel_surface.cpp b/src/panel_surface.cpp index e009b43..e984c06 100644 --- a/src/panel_surface.cpp +++ b/src/panel_surface.cpp @@ -25,12 +25,15 @@ struct Rect { rounded_distance(px, py, x, y, w, h, std::min(16, h / 3)) <= 0.f; } }; -enum class Control { Review, Settings, Prev, Next, Record, Cancel, Insert, Enter, Quit, +enum class Control { Review, Settings, Bindings, OpenBindings, Prev, Next, Record, Cancel, Insert, Enter, Quit, World, Left, Right, Head, Recenter, LasersAnytime }; +enum class Tab { Review, Settings, Bindings }; struct Button { Rect r; Control id; const char* label; }; -constexpr std::array buttons{{ +constexpr std::array buttons{{ {{32, 138, 180, 46}, Control::Review, "Review"}, {{226, 138, 180, 46}, Control::Settings, "Settings"}, + {{420, 138, 180, 46}, Control::Bindings, "Bindings"}, + {{32, 410, 420, 48}, Control::OpenBindings, "Edit in SteamVR"}, {{32, 406, 154, 44}, Control::Prev, "Previous"}, {{838, 406, 130, 44}, Control::Next, "Next"}, {{32, 574, 176, 68}, Control::Record, "Record"}, @@ -101,8 +104,10 @@ struct PanelSurface::Impl { Mount mount; Theme theme; Color background, card, ink, muted, cyan, pink; - bool settings = false, dirty = true, lasers_anytime = false; - std::string placement_note; + Tab tab = Tab::Review; + bool dirty = true, lasers_anytime = false; + std::string placement_note, binding_note; + std::array bindings{}; std::array pressed{{-1, -1}}; std::vector lines; size_t page = 0; @@ -238,8 +243,9 @@ struct PanelSurface::Impl { return true; } bool visible(Control c) const { - if (c == Control::Prev || c == Control::Next) return !settings; - if (mounting(c) || c == Control::Recenter || c == Control::LasersAnytime) return settings; + if (c == Control::Prev || c == Control::Next) return tab == Tab::Review; + if (mounting(c) || c == Control::Recenter || c == Control::LasersAnytime) return tab == Tab::Settings; + if (c == Control::OpenBindings) return tab == Tab::Bindings; return true; } bool enabled(Control c) const { @@ -284,7 +290,7 @@ struct PanelSurface::Impl { auto status = wrap(panel.status, 27, 790); text(status.front(), 70, 109, 27, ink, 844); if (status.size() > 1) text("[...]", 864, 109, 23, muted, 968); - if (!settings) { + if (tab == Tab::Review) { rounded({32, 212, 936, 178}, 16, mix(card, muted, .08f), mix(card, cyan, .25f), .18f); for (size_t i = 0; i < lines_per_page && page * lines_per_page + i < lines.size(); ++i) text(lines[page * lines_per_page + i], 48, 246 + int(i) * 40, 32, @@ -294,6 +300,20 @@ struct PanelSurface::Impl { for (size_t i = 0; i < std::min(size_t(2), detail.size()); ++i) text(detail[i], 32, 486 + int(i) * 30, 24, muted, 936); if (detail.size() > 2) text("[detail truncated]", 730, 546, 22, pink, 968); + } else if (tab == Tab::Bindings) { + constexpr std::array names{{"Hold to record", "Cancel / discard", "Insert + space", + "Insert + Enter", "Double tap: Enter", "Tap then hold: record"}}; + for (size_t i = 0; i < names.size(); ++i) { + const int y = 229 + int(i) * 30; + text(names[i], 32, y, 23, muted, 335); + auto label = wrap(bindings[i].empty() ? "Waiting for SteamVR..." : bindings[i], 23, 582); + text(label.front(), 346, y, 23, ink, 928); + if (label.size() > 1) text("...", 930, y, 23, muted, 968); + } + text("Enter inserts pending text, then presses Enter; no text = Enter only.", 32, 484, 22, muted, 968); + text("Normal mode: bindings with dashboard closed; UI with it open.", 32, 512, 22, muted, 968); + text(binding_note.empty() ? "SteamVR owns remapping. Multiple controls may appear per action." : binding_note, + 32, 539, 21, muted, 968); } else { text("MOUNT AND INTERACTION", 32, 224, 22, muted, 968); text("Lasers anytime enables system-wide laser mode while this panel is visible.", 32, 472, 20, muted, 968); @@ -306,7 +326,9 @@ struct PanelSurface::Impl { const auto& b = buttons[i]; if (!visible(b.id)) continue; bool on = enabled(b.id); - bool selected = (b.id == Control::Review && !settings) || (b.id == Control::Settings && settings) || + bool selected = (b.id == Control::Review && tab == Tab::Review) || + (b.id == Control::Settings && tab == Tab::Settings) || + (b.id == Control::Bindings && tab == Tab::Bindings) || (mounting(b.id) && *mounting(b.id) == mount) || (b.id == Control::LasersAnytime && lasers_anytime); const Color fill = !on ? mix(background, card, .40f) : @@ -353,11 +375,22 @@ SurfaceEvent PanelSurface::pointer_up(unsigned cursor, float x, float y) { else if (auto m = mounting(c)) { impl_->mount = *m; result.mount = *m; impl_->reset(); impl_->dirty = true; } else if (c == Control::Recenter) { result.recenter = true; impl_->reset(); } else if (c == Control::LasersAnytime) result.lasers_anytime = !impl_->lasers_anytime; - else if (c == Control::Review || c == Control::Settings) { impl_->settings = c == Control::Settings; impl_->reset(); impl_->dirty = true; } + else if (c == Control::OpenBindings) { result.open_bindings = true; impl_->reset(); } + else if (c == Control::Review || c == Control::Settings || c == Control::Bindings) { + impl_->tab = c == Control::Review ? Tab::Review : c == Control::Settings ? Tab::Settings : Tab::Bindings; + impl_->reset(); impl_->dirty = true; + } else if (c == Control::Prev) { --impl_->page; impl_->dirty = true; } else if (c == Control::Next) { ++impl_->page; impl_->dirty = true; } return result; } +bool PanelSurface::bindings_visible() const { return impl_->tab == Tab::Bindings; } +void PanelSurface::set_bindings(std::array labels) { + if (impl_->bindings != labels) { impl_->bindings = std::move(labels); impl_->dirty = true; } +} +void PanelSurface::set_binding_note(std::string note) { + if (impl_->binding_note != note) { impl_->binding_note = std::move(note); impl_->dirty = true; } +} void PanelSurface::reset_pointers() { impl_->reset(); } void PanelSurface::set_placement_note(std::string note) { if (impl_->placement_note != note) { impl_->placement_note = std::move(note); impl_->dirty = true; } diff --git a/src/panel_surface.hpp b/src/panel_surface.hpp index 8f8c8f0..c313efc 100644 --- a/src/panel_surface.hpp +++ b/src/panel_surface.hpp @@ -13,6 +13,7 @@ struct SurfaceEvent { std::optional mount; std::optional lasers_anytime; bool recenter = false; + bool open_bindings = false; }; // One CPU RGBA canvas, independent of OpenVR. Settings replace the review area; // status and safety controls remain on the same surface. @@ -31,6 +32,10 @@ public: void reset_pointers(); void set_placement_note(std::string note); void set_lasers_anytime(bool enabled); + // PTT, Cancel, Insert, Enter, left-grip gesture, right-grip gesture. + void set_bindings(std::array labels); + void set_binding_note(std::string note); + bool bindings_visible() const; bool available(UiAction action) const; private: struct Impl; diff --git a/src/runtime.cpp b/src/runtime.cpp index 1331d54..848365b 100644 --- a/src/runtime.cpp +++ b/src/runtime.cpp @@ -21,6 +21,14 @@ namespace frameyap { namespace { volatile sig_atomic_t interrupted = 0; void signal_stop(int) { interrupted = 1; } +class NativeDeliveryLease final : public DeliveryLease { +public: + explicit NativeDeliveryLease(const std::string& socket) : input_(socket) {} + void text(const std::string& literal) override { input_.text(literal); } + void enter() override { input_.enter(); } +private: + TextInput input_; +}; std::string state_label(State state) { switch (state) { case State::Warming: return "Warming - on-device Redux CPU"; @@ -44,8 +52,29 @@ int run(const Options& options) { Worker worker; Audio audio; Session session; - std::string detail = "Review mode. Other apps may also hear your mic. Enter is separate."; + std::string detail = "Review mode. Other apps may also hear your mic. Enter is explicit."; bool quit = false; + const DeliveryFactory acquire = [&]() -> std::unique_ptr { + auto input = std::make_unique(options.socket); + if (interrupted) throw std::runtime_error("Input cancelled before delivery"); + return input; + }; + auto delivery_detail = [&](DeliveryResult result, bool submit) { + switch (result) { + case DeliveryResult::Ignored: break; + case DeliveryResult::TextQueued: + detail = "Text and trailing space queued to current focus. Enter remains explicit."; break; + case DeliveryResult::EnterQueued: + detail = submit ? "Explicit Enter queued to current focus; not a delivery receipt." + : "Input queued to current focus; not a delivery receipt."; break; + case DeliveryResult::TextUncertain: + detail = "Text delivery uncertain; Enter not sent. Not retried; check destination."; break; + case DeliveryResult::EnterUncertain: + detail = "Enter delivery uncertain; not retried. Check destination."; break; + case DeliveryResult::TextQueuedEnterUnavailable: + detail = "Text and trailing space queued; Enter unavailable and not sent. Check destination."; break; + } + }; auto warm = [&] { audio.close(); worker.stop(); session = Session{}; worker.start(options.python, options.worker, options.model, options.threads); @@ -132,22 +161,10 @@ int run(const Options& options) { break; } case UiAction::Insert: - if (session.state() == State::Review) { - // Acquire IME before consuming: unavailable means the preview remains. - TextInput input(options.socket); - if (interrupted) break; - auto literal = session.take_insert(); - try { input.text(*literal); detail = "Input queued to current focus. Enter remains a separate action."; } - catch (...) { detail = "Delivery uncertain; not retried. Check destination before dictating again."; } - } + delivery_detail(deliver_insert(session, acquire), false); break; case UiAction::Enter: - if (session.state() == State::Ready || session.state() == State::Queued || session.state() == State::Review) { - TextInput input(options.socket); - if (interrupted) break; - try { input.enter(); detail = "Explicit Enter queued to current focus."; } - catch (...) { detail = "Enter delivery uncertain; not retried."; } - } + delivery_detail(deliver_enter(session, acquire), true); break; } } catch (const std::exception& e) { diff --git a/tests/config_test.cpp b/tests/config_test.cpp index 8604668..2c78163 100644 --- a/tests/config_test.cpp +++ b/tests/config_test.cpp @@ -70,6 +70,14 @@ int main(int argc, char** argv) { assert(generated.find("/user/hand/left/input/y") != std::string::npos); assert(generated.find("/actions/frameyap/in/cancel") != std::string::npos); assert(generated.find("/actions/frameyap/in/right_grip") == std::string::npos); + assert(generated.find("/user/hand/right/input/a") != std::string::npos); + assert(generated.find("/user/hand/right/input/y") != std::string::npos); + config.buttons["cancel"] = ""; config.buttons["insert"] = ""; config.buttons["enter"] = ""; + auto disabled = action_manifest(argv[1], config); + generated = get(disabled.parent_path() / "bindings_frame_controller.json"); + assert(generated.find("/actions/frameyap/in/cancel") == std::string::npos); + assert(generated.find("/actions/frameyap/in/insert") == std::string::npos); + assert(generated.find("/actions/frameyap/in/enter") == std::string::npos); assert(action_manifest(argv[1], {}) == std::filesystem::absolute(std::filesystem::path(argv[1]) / "actions.json")); put(path, R"({"buttons":{"ptt":"/user/hand/left/input/grip"}})"); fails([&] { action_manifest(argv[1], load_config(path)); }); // overlapping bindings are never silently chosen diff --git a/tests/core_test.cpp b/tests/core_test.cpp index 03246e6..d3c4028 100644 --- a/tests/core_test.cpp +++ b/tests/core_test.cpp @@ -2,9 +2,128 @@ #include #include #include +#include using namespace frameyap; #define CHECK(x) do { if (!(x)) { std::cerr << "line " << __LINE__ << ": " #x "\n"; std::exit(1); } } while (0) void rejects(std::string text) { try { literal_text(text); } catch (const std::runtime_error&) { return; } CHECK(false); } +struct FakeDelivery { + std::vector events; + int acquisitions = 0; + int active_leases = 0; + int fail_acquisition = 0; + bool fail_text = false, fail_enter = false; + struct Lease : DeliveryLease { + FakeDelivery& fake; + explicit Lease(FakeDelivery& f) : fake(f) { ++fake.active_leases; } + ~Lease() override { --fake.active_leases; } + void text(const std::string& literal) override { + fake.events.push_back("text:" + literal); + if (fake.fail_text) throw std::runtime_error("ambiguous text failure"); + } + void enter() override { + fake.events.push_back("enter"); + if (fake.fail_enter) throw std::runtime_error("ambiguous Enter failure"); + } + }; + DeliveryFactory factory() { + return [this]() -> std::unique_ptr { + ++acquisitions; + if (active_leases || acquisitions == fail_acquisition) + throw std::runtime_error("exclusive lease unavailable"); + return std::make_unique(*this); + }; + } +}; +Session review(std::string_view transcript) { + Session session; + session.ready(); CHECK(session.record()); CHECK(session.finish(16000)); + CHECK(session.reply(session.id(), transcript)); CHECK(session.state() == State::Review); + return session; +} +void delivery_checks() { + { + auto s = review("Hello 世界"); FakeDelivery fake; + CHECK(deliver_insert(s, fake.factory()) == DeliveryResult::TextQueued); + CHECK((fake.events == std::vector{"text:Hello 世界 "})); + CHECK(fake.acquisitions == 1 && s.state() == State::Queued); + CHECK(deliver_insert(s, fake.factory()) == DeliveryResult::Ignored); + CHECK(deliver_enter(s, fake.factory()) == DeliveryResult::EnterQueued); + CHECK((fake.events == std::vector{"text:Hello 世界 ", "enter"})); + } + { + auto s = review("one\n two"); FakeDelivery fake; + CHECK(deliver_enter(s, fake.factory()) == DeliveryResult::EnterQueued); + CHECK((fake.events == std::vector{"text:one two ", "enter"})); + CHECK(fake.acquisitions == 2 && fake.active_leases == 0 && s.state() == State::Queued); + CHECK(deliver_insert(s, fake.factory()) == DeliveryResult::Ignored); + } + { + Session s; FakeDelivery fake; + CHECK(deliver_enter(s, fake.factory()) == DeliveryResult::Ignored); + s.ready(); CHECK(deliver_enter(s, fake.factory()) == DeliveryResult::EnterQueued); + CHECK((fake.events == std::vector{"enter"})); + CHECK(s.record()); CHECK(deliver_enter(s, fake.factory()) == DeliveryResult::Ignored); + CHECK(s.finish(16000)); CHECK(deliver_enter(s, fake.factory()) == DeliveryResult::Ignored); + s.fail(); CHECK(deliver_enter(s, fake.factory()) == DeliveryResult::Ignored); + CHECK(fake.acquisitions == 1); + } + { + auto s = review("preserve"); FakeDelivery fake; fake.fail_acquisition = 1; + try { (void)deliver_enter(s, fake.factory()); CHECK(false); } + catch (const std::runtime_error&) {} + CHECK(s.state() == State::Review && s.text() == "preserve" && fake.events.empty()); + CHECK(deliver_enter(s, fake.factory()) == DeliveryResult::EnterQueued); + CHECK((fake.events == std::vector{"text:preserve ", "enter"})); + } + { + auto s = review("uncertain"); FakeDelivery fake; fake.fail_text = true; + CHECK(deliver_enter(s, fake.factory()) == DeliveryResult::TextUncertain); + CHECK(fake.acquisitions == 1 && fake.events.size() == 1); + CHECK(s.state() == State::Queued && s.text().empty()); + CHECK(deliver_insert(s, fake.factory()) == DeliveryResult::Ignored); + } + { + auto s = review("uncertain insert"); FakeDelivery fake; fake.fail_text = true; + CHECK(deliver_insert(s, fake.factory()) == DeliveryResult::TextUncertain); + CHECK(fake.acquisitions == 1 && fake.events.size() == 1); + CHECK(s.state() == State::Queued && s.text().empty()); + } + { + auto s = review("first"); FakeDelivery fake; fake.fail_acquisition = 2; + CHECK(deliver_enter(s, fake.factory()) == DeliveryResult::TextQueuedEnterUnavailable); + CHECK((fake.events == std::vector{"text:first "})); + CHECK(s.state() == State::Queued && fake.acquisitions == 2); + CHECK(deliver_enter(s, fake.factory()) == DeliveryResult::EnterQueued); // a NEW explicit press + CHECK((fake.events == std::vector{"text:first ", "enter"})); + } + { + auto s = review("first"); FakeDelivery fake; fake.fail_enter = true; + CHECK(deliver_enter(s, fake.factory()) == DeliveryResult::EnterUncertain); + CHECK((fake.events == std::vector{"text:first ", "enter"})); + CHECK(s.state() == State::Queued && fake.acquisitions == 2); + } + { + auto s = review("already spaced "); FakeDelivery fake; + CHECK(deliver_enter(s, fake.factory()) == DeliveryResult::EnterQueued); + CHECK((fake.events == std::vector{"text:already spaced ", "enter"})); + } + { + auto s = review(std::string(4095, 'a')); FakeDelivery fake; + CHECK(deliver_insert(s, fake.factory()) == DeliveryResult::TextQueued); + CHECK(fake.events[0].size() == 5 + 4096 && fake.events[0].back() == ' '); + } + { + auto s = review(std::string(4096, 'a')); FakeDelivery fake; + try { (void)deliver_enter(s, fake.factory()); CHECK(false); } + catch (const std::runtime_error&) {} + CHECK(s.state() == State::Review && s.text().size() == 4096); + CHECK(fake.acquisitions == 0 && fake.events.empty()); + auto with_existing_space = review(std::string(4095, 'a') + " "); + CHECK(deliver_insert(with_existing_space, fake.factory()) == DeliveryResult::TextQueued); + CHECK(fake.events[0] == "text:" + std::string(4095, 'a') + " "); + CHECK(with_existing_space.state() == State::Queued && fake.acquisitions == 1); + } +} int main() { CHECK(literal_text("Hello 世界 😀") == "Hello 世界 😀"); CHECK(literal_text("one\r\ntwo\tthree\xe2\x80\xa8") == "one two three "); @@ -33,5 +152,6 @@ int main() { CHECK(s.record()); CHECK(s.finish(16000)); CHECK(s.reply(s.id(), "again")); CHECK(s.take_insert() == "again"); CHECK(s.record()); s.fail(); s.cancel(); CHECK(s.record()); + delivery_checks(); std::cout << "core checks passed\n"; } diff --git a/tests/panel_test.cpp b/tests/panel_test.cpp index aaa8db1..92578e9 100644 --- a/tests/panel_test.cpp +++ b/tests/panel_test.cpp @@ -12,7 +12,7 @@ SurfaceEvent click(PanelSurface& surface, float x, float y, unsigned cursor = 0) return surface.pointer_up(cursor, x, y); } void no_action(const SurfaceEvent& event) { - assert(!event.action && !event.mount && !event.lasers_anytime && !event.recenter); + assert(!event.action && !event.mount && !event.lasers_anytime && !event.recenter && !event.open_bindings); } void snapshot(PanelSurface& surface, const std::string& path) { std::ofstream out(path, std::ios::binary); @@ -124,6 +124,31 @@ int main(int argc, char** argv) { no_action(click(surface, 680, 260)); // mount controls not active on Review no_action(click(surface, 680, 420)); // laser toggle only exists on Settings + // Binding navigation is not a delivery action; settings and paging are hidden. + assert(!surface.bindings_visible()); + surface.pointer_down(1, 480, 610); + no_action(click(surface, 500, 160)); + assert(surface.bindings_visible()); + no_action(surface.pointer_up(1, 480, 610)); // switching tabs invalidates old approval + surface.set_bindings({"Right X", "Unbound / controller unavailable", "Left Y", "Right Y", "Left grip", "Right grip"}); + assert(surface.render(p)); assert(!surface.render(p)); + surface.set_bindings({"Right X", "Unbound / controller unavailable", "Left Y", "Right Y", "Left grip", "Right grip"}); + assert(!surface.render(p)); + if (argc >= 3) snapshot(surface, std::string(argv[2]) + "-bindings.ppm"); + no_action(click(surface, 680, 260)); + no_action(click(surface, 900, 430)); + no_action(click(surface, 680, 420)); + const auto editor = click(surface, 200, 434); + assert(editor.open_bindings && !editor.action && !editor.mount && !editor.recenter && !editor.lasers_anytime); + no_action(surface.pointer_up(0, 200, 434)); // one launch per deliberate click + surface.set_binding_note("SteamVR could not open bindings. Try its controller settings."); + assert(surface.render(p)); assert(!surface.render(p)); + assert(click(surface, 480, 610).action == UiAction::Insert); + assert(click(surface, 680, 610).action == UiAction::Enter); + assert(click(surface, 280, 610).action == UiAction::Cancel); + no_action(click(surface, 100, 160)); + assert(!surface.bindings_visible()); + // Long UTF-8, newlines and malformed bytes are bounded, paginated and navigable. p.transcript.clear(); for (int i = 0; i < 60; ++i) p.transcript += "Line " + std::to_string(i) + ": café / 日本語 / naïve / Ω\n"; diff --git a/tests/test_action_bindings.py b/tests/test_action_bindings.py index b01eae3..8d741ff 100644 --- a/tests/test_action_bindings.py +++ b/tests/test_action_bindings.py @@ -8,6 +8,19 @@ ROOT = Path(__file__).resolve().parents[1] class ActionBindingTests(unittest.TestCase): + def test_review_controls_match_config_defaults(self): + manifest = json.loads((ROOT / "assets/actions.json").read_text()) + binding = json.loads((ROOT / "assets/bindings_frame_controller.json").read_text()) + config = json.loads((ROOT / "assets/config.example.json").read_text()) + sources = binding["bindings"]["/actions/frameyap"]["sources"] + self.assertEqual(len({s["path"] for s in sources}), len(sources)) + for action, button in {"ptt": "x", "cancel": "b", "insert": "a", "enter": "y"}.items(): + name = f"/actions/frameyap/in/{action}" + path = f"/user/hand/right/input/{button}" + self.assertIn({"name": name, "type": "boolean"}, manifest["actions"]) + self.assertEqual(config["buttons"][action], path) + self.assertEqual([s["path"] for s in sources if s["inputs"]["click"]["output"] == name], [path]) + def test_right_x_is_hold_to_talk(self): manifest = json.loads((ROOT / "assets/actions.json").read_text()) binding = json.loads((ROOT / "assets/bindings_frame_controller.json").read_text()) diff --git a/tests/test_installer.py b/tests/test_installer.py index f488c74..2d7a524 100644 --- a/tests/test_installer.py +++ b/tests/test_installer.py @@ -123,13 +123,14 @@ class InstallTests(unittest.TestCase): self.assertEqual(fixed["theme"]["ink"], "#F1f2F3") self.assertEqual(fixed["buttons"]["ptt"], "/user/hand/left/input/y") self.assertEqual(fixed["theme"]["card"], installer.CONFIG_DEFAULTS["theme"]["card"]) - self.assertEqual(fixed["buttons"]["cancel"], "") + self.assertEqual(fixed["buttons"]["cancel"], "/user/hand/right/input/b") self.assertEqual(fixed["input_priority"], "normal") self.assertEqual(fixed["wrist"], installer.CONFIG_DEFAULTS["wrist"]) backups = list(config.parent.glob("config.json.backup-*")) self.assertEqual(len(backups), 1) self.assertEqual(backups[0].read_bytes(), original) fixed["input_priority"] = "experimental" + fixed["buttons"]["enter"] = "" # intentional disabling survives upgrades fixed["wrist"]["y"] = 0.2 compact = json.dumps(fixed, separators=(",", ":")).encode() config.write_bytes(compact)