From 6b1d20858a864926221e205670dd8b74b505daaa Mon Sep 17 00:00:00 2001 From: baketnk Date: Thu, 24 Sep 2026 13:40:27 -0400 Subject: [PATCH] Bind Frame right X to normal-priority hold-to-talk --- CMakeLists.txt | 4 ++- README.md | 9 ++++--- assets/actions.json | 2 +- assets/bindings_frame_controller.json | 9 +++++-- docs/overlay.md | 8 +++--- docs/poc.md | 20 ++++++++------ src/main.cpp | 5 ++-- src/overlay.cpp | 9 +++++++ src/runtime.cpp | 4 +-- tests/test_action_bindings.py | 38 +++++++++++++++++++++++++++ 10 files changed, 85 insertions(+), 23 deletions(-) create mode 100644 tests/test_action_bindings.py diff --git a/CMakeLists.txt b/CMakeLists.txt index 5d41b82..44427c3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -138,9 +138,11 @@ if(BUILD_TESTING AND NOT CMAKE_CROSSCOMPILING) endif() endif() if(Python3_FOUND) + add_test(NAME frameyap.action_bindings COMMAND "${Python3_EXECUTABLE}" -m unittest discover -s tests -p test_action_bindings.py) add_test(NAME frameyap.python_worker COMMAND "${Python3_EXECUTABLE}" -m unittest discover -s tests -p test_worker.py) add_test(NAME frameyap.installer COMMAND "${Python3_EXECUTABLE}" -m unittest discover -s tests -p test_installer.py) - set_tests_properties(frameyap.python_worker frameyap.installer PROPERTIES WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" TIMEOUT 60) + set_tests_properties(frameyap.action_bindings frameyap.python_worker frameyap.installer + PROPERTIES WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" TIMEOUT 60) if(TARGET frameyap_worker) add_executable(frameyap_worker_test tests/worker_test.cpp) target_link_libraries(frameyap_worker_test PRIVATE frameyap_worker) diff --git a/README.md b/README.md index e276281..ac43d61 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ Standalone, on-device voice typing POC for Steam Frame. **MIT licensed.** -Implemented: native OpenVR overlay, remappable grip controls, bounded SDL3 capture, +Implemented: native OpenVR overlay, remappable controller actions, bounded SDL3 capture, persistent local Parakeet Redux worker, preview/explicit insertion through Gamescope, and an idempotent user-local installer. No Steam store AppID, sudo, desktop ASR server, cloud fallback or unrelated application dependency. @@ -19,9 +19,10 @@ See [third-party notes](docs/third-party.md). No GitHub release is published yet ## Controls -- **Right grip:** short tap, then hold the second squeeze to record; release to - transcribe. Remappable through SteamVR bindings; separate hold-to-talk action too. -- **Left grip:** double-tap to explicitly send Enter. Never inferred from speech. +- **Right X (default Frame binding):** hold to record; release to + transcribe. This PTT action is remappable through SteamVR bindings; delivery + in the dashboard still requires an on-device test. +- **Left grip (when active):** double-tap to explicitly send Enter. Never inferred from speech. - **Overlay:** Record/Stop, Cancel, paginated preview, Insert, Enter and Quit. Review and settings share one Inconsolata/neon-framed surface. World-space by default; settings offer left wrist, right wrist and head mounting, plus recenter. diff --git a/assets/actions.json b/assets/actions.json index 63d2039..80188f7 100644 --- a/assets/actions.json +++ b/assets/actions.json @@ -19,7 +19,7 @@ "/actions/frameyap": "FrameYap controls", "/actions/frameyap/in/left_grip": "Double tap left grip: Enter (enabled only)", "/actions/frameyap/in/right_grip": "Tap right grip briefly, then squeeze and hold to record; release to finish", - "/actions/frameyap/in/ptt": "Hold to record", + "/actions/frameyap/in/ptt": "Hold right X to record (default; remappable)", "/actions/frameyap/in/cancel": "Cancel recording or discard", "/actions/frameyap/in/insert": "Insert reviewed text", "/actions/frameyap/in/enter": "Explicit Enter" diff --git a/assets/bindings_frame_controller.json b/assets/bindings_frame_controller.json index 88d25b5..b563bbe 100644 --- a/assets/bindings_frame_controller.json +++ b/assets/bindings_frame_controller.json @@ -1,7 +1,7 @@ { "controller_type": "frame_controller", - "description": "Right grip: tap then hold to dictate. Left grip: double tap Enter. Remap other actions in SteamVR.", - "name": "FrameYap Steam Frame grips", + "description": "Hold right X to dictate; release to review. Grip gestures remain remappable in SteamVR.", + "name": "FrameYap Steam Frame controls", "bindings": { "/actions/frameyap": { "sources": [ @@ -14,6 +14,11 @@ "path": "/user/hand/right/input/grip", "mode": "button", "inputs": { "click": { "output": "/actions/frameyap/in/right_grip" } } + }, + { + "path": "/user/hand/right/input/x", + "mode": "button", + "inputs": { "click": { "output": "/actions/frameyap/in/ptt" } } } ] } diff --git a/docs/overlay.md b/docs/overlay.md index 9148610..ae7cf9d 100644 --- a/docs/overlay.md +++ b/docs/overlay.md @@ -92,9 +92,11 @@ texture replacements; it does not establish that ordinary state-changing UI updates are flicker-free. `assets/actions.json` names six actions: left/right grip, PTT, cancel, insert, -Enter. `bindings_frame_controller.json` uses the observed Frame profile's grip -click paths; both bound actions were reported tracked/active in the device check. -This does not prove physical gesture delivery. `bindings_knuckles.json` is an +Enter. `bindings_frame_controller.json` maps right X click to hold-to-talk PTT; +the grip bindings remain for optional remapping/diagnosis. Both grip actions +were **inactive** during the dashboard controls check despite a loaded binding +file; this does not prove whether right X will deliver until an on-device test. +`bindings_knuckles.json` is an additional **Index/knuckles example only**. Collisions with scene actions require separate on-device validation. Left grip double tap (releases <=250 ms, second press within 350 ms) requests explicit Enter only diff --git a/docs/poc.md b/docs/poc.md index c7b314d..ebc9e7b 100644 --- a/docs/poc.md +++ b/docs/poc.md @@ -7,10 +7,13 @@ initialize OpenVR, open a microphone, run ASR, download files or inject input. - Opt-in OpenVR RGBA overlay with world-space default and selectable wrist/head mounts, status, recording timer, paginated UTF-8 preview and explicit controls. -- Remappable SteamVR actions. Default Steam Frame grip bindings use the observed - `frame_controller` profile. Right: short tap, then hold the second squeeze to - record; release to transcribe. Left: two short taps request explicit Enter. - First squeeze <=250 ms; second squeeze begins <=350 ms after first release. +- Remappable SteamVR actions. The default Steam Frame binding maps right X + (hold to record, release to transcribe) to the existing PTT action using the + observed `frame_controller` profile. Grip bindings remain, but both grip + actions were inactive in the observed dashboard check; do not rely on them. + If left grip becomes active, two short taps request explicit Enter. + For grip gestures, first squeeze <=250 ms; second squeeze begins <=350 ms + after first release. Activity/tracking loss cancels a held recording and requires neutral rearm. - SDL3 default recording device, mono float32 conversion at 16 kHz, 200 ms minimum, 20 second maximum. Microphone is closed outside actual capture. Other apps may @@ -90,10 +93,11 @@ That is a producer workflow, not an end-user compiler requirement. The first acquires/releases an IME without text/actions. The second displays a five-second inert panel. The third displays a 30-second diagnostic panel and -reports pointer actions, SteamVR action activity, tracking and read-only legacy -grip state, **without microphone or input injection**. Mount clicks in check -modes do not save a preference. A SteamVR error code or inactive grip means the -gesture cannot be accepted; raw grip reads do not authorize a fallback binding. +reports pointer actions, SteamVR grip and PTT action activity, tracking and +read-only legacy grip state, **without microphone or input injection**. Mount +clicks in check modes do not save a preference. A SteamVR error code or inactive +action means the gesture cannot be accepted; raw grip reads do not authorize a +fallback binding. Action handles alone are not proof that gestures were delivered. Checks must be explicitly launched while the user expects the panel. Normal CLI/help/version remain inert. diff --git a/src/main.cpp b/src/main.cpp index ddffea8..deacb09 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -24,8 +24,9 @@ void help() { "Mount: world (first-launch default), left-wrist, right-wrist, head.\n" "Settings save the mount; --mount overrides it for this launch. --head is an alias.\n" "Font defaults to bundled Inconsolata in the assets directory.\n" - "Right grip: double-tap, hold second squeeze to speak, release to review.\n" - "Left grip: double-tap for explicit Enter. Bindings are remappable.\n" + "Right X: hold to speak, release to review (default Frame binding).\n" + "Grip gestures are remappable but may be unavailable in the dashboard.\n" + "Left grip: double-tap for explicit Enter when active.\n" "Review first: Insert approves current focus; never automatic Enter.\n" "No device access unless an explicit runtime/check/registration mode is used.\n"; #ifndef FRAMEYAP_NATIVE diff --git a/src/overlay.cpp b/src/overlay.cpp index 8ad75ab..41c4ec7 100644 --- a/src/overlay.cpp +++ b/src/overlay.cpp @@ -341,6 +341,15 @@ std::string Overlay::controls_status() { " origin=" + (origin_ok ? "Y" : "N") + " raw=" + (raw_ok ? ((raw.ulButtonPressed & vr::ButtonMaskFromId(vr::k_EButton_Grip)) ? "down" : "up") : "n/a"); } + // The default Frame binding maps right X to the named hold-to-talk action. + // Report the action's own activity separately from the stricter pose gate; + // remapped bindings may intentionally have a different origin. + vr::InputDigitalActionData_t ptt{}; + const auto ptt_error = impl_->input->GetDigitalActionData(impl_->actions[2], &ptt, sizeof(ptt), vr::k_ulInvalidInputValueHandle); + const auto [accepted, down] = impl_->digital(2); + result += "\nPTT (default right X): err=" + std::to_string(int(ptt_error)) + + " active=" + (ptt.bActive ? "Y" : "N") + " down=" + (ptt.bState ? "Y" : "N") + + " accepted=" + (accepted ? "Y" : "N") + " gated-down=" + (down ? "Y" : "N"); return result; } std::string Overlay::pointer_status() const { diff --git a/src/runtime.cpp b/src/runtime.cpp index f136b81..a68337a 100644 --- a/src/runtime.cpp +++ b/src/runtime.cpp @@ -24,7 +24,7 @@ void signal_stop(int) { interrupted = 1; } std::string state_label(State state) { switch (state) { case State::Warming: return "Warming - on-device Redux CPU"; - case State::Ready: return "Ready - double-tap + hold right grip"; + case State::Ready: return "Ready - hold X or click Record"; case State::Recording: return "RECORDING"; case State::Transcribing: return "Transcribing - on device"; case State::Review: return "Review - Insert approves CURRENT focus"; @@ -66,7 +66,7 @@ int run(const Options& options) { if (session.state() == State::Error) session.cancel(); if (!session.record()) return; audio.start(); - detail = "Release grip to finish. Cancel discards. Maximum 20 seconds."; + detail = "Release to finish. Cancel discards. Maximum 20 seconds."; }; try { warm(); } catch (const std::exception& e) { session.fail(); detail = e.what(); } diff --git a/tests/test_action_bindings.py b/tests/test_action_bindings.py new file mode 100644 index 0000000..b01eae3 --- /dev/null +++ b/tests/test_action_bindings.py @@ -0,0 +1,38 @@ +"""Offline contract for the default Frame controller shortcut (no OpenVR).""" +import json +from pathlib import Path +import unittest + + +ROOT = Path(__file__).resolve().parents[1] + + +class ActionBindingTests(unittest.TestCase): + def test_right_x_is_hold_to_talk(self): + manifest = json.loads((ROOT / "assets/actions.json").read_text()) + binding = json.loads((ROOT / "assets/bindings_frame_controller.json").read_text()) + self.assertIn( + {"controller_type": "frame_controller", "binding_url": "bindings_frame_controller.json"}, + manifest["default_bindings"], + ) + self.assertIn( + {"name": "/actions/frameyap/in/ptt", "type": "boolean"}, + manifest["actions"], + ) + sources = binding["bindings"]["/actions/frameyap"]["sources"] + right_x = [source for source in sources if source["path"] == "/user/hand/right/input/x"] + self.assertEqual(right_x, [{ + "path": "/user/hand/right/input/x", + "mode": "button", + "inputs": {"click": {"output": "/actions/frameyap/in/ptt"}}, + }]) + # Do not map a second PTT source or replace the dashboard's pointer trigger. + self.assertEqual(sum( + source["inputs"].get("click", {}).get("output") == "/actions/frameyap/in/ptt" + for source in sources + ), 1) + self.assertFalse(any(source["path"].endswith("/input/trigger") for source in sources)) + + +if __name__ == "__main__": + unittest.main()