From 9b4bf9f047cf3b930a00ee9e6784d9fcca7d1061 Mon Sep 17 00:00:00 2001 From: baketnk Date: Thu, 24 Sep 2026 12:50:25 -0400 Subject: [PATCH] Keep Redux loaded across capture failures and diagnose grip actions --- docs/poc.md | 20 +++++++++++------ src/audio.cpp | 28 ++++++++++++++---------- src/main.cpp | 23 +++++++++++++++---- src/overlay.cpp | 51 ++++++++++++++++++++++++++++++++----------- src/overlay.hpp | 3 ++- src/runtime.cpp | 33 +++++++++++++++++++++------- tests/core_test.cpp | 6 +++++ tests/test_worker.py | 5 +++++ tests/worker_test.cpp | 9 ++++++++ 9 files changed, 134 insertions(+), 44 deletions(-) diff --git a/docs/poc.md b/docs/poc.md index 916bb43..c7b314d 100644 --- a/docs/poc.md +++ b/docs/poc.md @@ -5,8 +5,8 @@ initialize OpenVR, open a microphone, run ASR, download files or inject input. ## Implemented -- Opt-in OpenVR RGBA overlay with head/left-hand placement, status, recording timer, - paginated UTF-8 preview, Record/Cancel/Insert/Enter/Quit controls. +- Opt-in OpenVR RGBA overlay with world-space default and selectable wrist/head + mounts, status, recording timer, paginated UTF-8 preview and explicit controls. - Remappable SteamVR actions. Default Steam Frame grip bindings use the observed `frame_controller` profile. Right: short tap, then hold the second squeeze to record; release to transcribe. Left: two short taps request explicit Enter. @@ -17,7 +17,10 @@ initialize OpenVR, open a microphone, run ASR, download files or inject input. still transmit your voice: this app does **not** mute VRChat or any other app. - Persistent local Redux worker, correlated bounded pipes, private tmpfs clips, cancellation/reaping and deadlines; exact pinned model SHA-256 verification. - Model imports are lazy and loading is offline. No cloud/desktop fallback. + Model imports are lazy and loading is offline. Normal repeats, request-local + transcription failures and microphone failures retain the loaded model; + microphone device streams close outside capture. A cancelled in-flight request + or broken worker protocol may require reloading. No cloud/desktop fallback. - Gamescope IME v2 generated bindings, per-action short-lived lease, unavailable handling, UTF-8/control validation and explicit separate Submit action for Enter. - Idempotent user-local release-archive installer: SHA-256, safe extraction, @@ -81,14 +84,17 @@ That is a producer workflow, not an end-user compiler requirement. ```sh ./build-native/frameyap --check-input --socket gamescope-0 -./build-native/frameyap --check-overlay --assets "$PWD/assets" --font /path/font.ttf --head -./build-native/frameyap --check-controls --assets "$PWD/assets" --font /path/font.ttf --head +./build-native/frameyap --check-overlay --assets "$PWD/assets" --mount world +./build-native/frameyap --check-controls --assets "$PWD/assets" --mount world ``` The first acquires/releases an IME without text/actions. The second displays a five-second inert panel. The third displays a 30-second diagnostic panel and -reports gestures, **without microphone or input injection**. Active action handles -are not proof that gestures were delivered. Checks must be explicitly launched +reports pointer actions, SteamVR action activity, tracking and read-only legacy +grip state, **without microphone or input injection**. Mount clicks in check +modes do not save a preference. A SteamVR error code or inactive grip means the +gesture cannot be accepted; raw grip reads do not authorize a fallback binding. +Action handles alone are not proof that gestures were delivered. Checks must be explicitly launched while the user expects the panel. Normal CLI/help/version remain inert. The ARM64 OpenVR loader's default `/data/work/openvrpaths.vrpath` failed on the diff --git a/src/audio.cpp b/src/audio.cpp index ae1b8b2..4d9e9a8 100644 --- a/src/audio.cpp +++ b/src/audio.cpp @@ -8,13 +8,19 @@ namespace frameyap { namespace { void check(bool result) { if (!result) throw std::runtime_error(std::string("Microphone: ") + SDL_GetError()); } } -Audio::~Audio() { cancel(); } +Audio::~Audio() { + cancel(); + if (initialized_) SDL_QuitSubSystem(SDL_INIT_AUDIO); +} void Audio::start() { if (stream_) throw std::runtime_error("Already recording"); - check(SDL_InitSubSystem(SDL_INIT_AUDIO)); initialized_ = true; + // Keep SDL's audio backend initialized across utterances, but own the + // recording device only while actually capturing. Reinitializing PipeWire + // after every clip can fail even though the local model remains healthy. + if (!initialized_) { check(SDL_InitSubSystem(SDL_INIT_AUDIO)); initialized_ = true; } SDL_AudioSpec spec{SDL_AUDIO_F32, 1, 16000}; stream_ = SDL_OpenAudioDeviceStream(SDL_AUDIO_DEVICE_DEFAULT_RECORDING, &spec, nullptr, nullptr); - if (!stream_) { cancel(); check(false); } + if (!stream_) throw std::runtime_error(std::string("Microphone: ") + SDL_GetError()); pcm_.clear(); pcm_.reserve(320000); started_ = last_data_ = std::chrono::steady_clock::now(); try { check(SDL_ResumeAudioStreamDevice(stream_)); } @@ -51,20 +57,20 @@ bool Audio::poll() { } std::vector Audio::finish() { if (!stream_) return {}; - check(SDL_PauseAudioStreamDevice(stream_)); - check(SDL_FlushAudioStream(stream_)); - drain(); - auto result = std::move(pcm_); - cancel(); - return result; + try { + check(SDL_PauseAudioStreamDevice(stream_)); + check(SDL_FlushAudioStream(stream_)); + drain(); + auto result = std::move(pcm_); + cancel(); + return result; + } catch (...) { cancel(); throw; } } void Audio::cancel() { if (stream_) SDL_DestroyAudioStream(stream_); stream_ = nullptr; // Best effort clearing of the owned buffer; no persistent recording/logging. std::fill(pcm_.begin(), pcm_.end(), 0.0f); pcm_.clear(); - if (initialized_) SDL_QuitSubSystem(SDL_INIT_AUDIO); - initialized_ = false; } int Audio::seconds() const { return stream_ ? int(std::chrono::duration_cast(std::chrono::steady_clock::now() - started_).count()) : 0; diff --git a/src/main.cpp b/src/main.cpp index 41f4c1a..aa65438 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -76,7 +76,7 @@ int main(int argc, char** argv) { if (options.assets.empty()) throw std::runtime_error("--assets required"); if (mode == "--check-overlay" || mode == "--check-controls") { frameyap::InstanceLock lock; - frameyap::Overlay overlay(options.assets, options.font, options.mount); + frameyap::Overlay overlay(options.assets, options.font, options.mount, false); overlay.draw({"FrameYap five-second visual check", "No audio captured. No text or Enter delivered.", "Controls inactive during this check.", false, false}); for (vr::TrackedDeviceIndex_t i = 0; i < vr::k_unMaxTrackedDeviceCount; ++i) { if (vr::VRSystem()->GetTrackedDeviceClass(i) != vr::TrackedDeviceClass_Controller) continue; @@ -89,17 +89,32 @@ int main(int argc, char** argv) { const auto end = std::chrono::steady_clock::now() + std::chrono::seconds(30); std::string status; bool held = false; + std::string last_action = "Press panel controls or tap then hold right grip. No mic or typing."; + auto action_name = [](frameyap::UiAction action) { + switch (action) { + case frameyap::UiAction::BeginRecord: return "BeginRecord"; + case frameyap::UiAction::EndRecord: return "EndRecord"; + case frameyap::UiAction::Record: return "Record"; + case frameyap::UiAction::Cancel: return "Cancel"; + case frameyap::UiAction::Insert: return "Insert"; + case frameyap::UiAction::Enter: return "Enter"; + case frameyap::UiAction::Quit: return "Quit"; + case frameyap::UiAction::Toggle: return "Toggle"; + } + return "Unknown"; + }; while (std::chrono::steady_clock::now() < end) { - overlay.draw({"Controls check ONLY - no mic or typing", "Tap then hold right grip; double-tap left grip.", - held ? "HOLD gesture recognized. Release to finish." : "Try grips or click buttons. Auto-closes after 30 seconds.", true, held}); for (auto action : overlay.poll()) { if (action == frameyap::UiAction::Quit) return 0; if (action == frameyap::UiAction::BeginRecord) held = true; if (action == frameyap::UiAction::EndRecord || action == frameyap::UiAction::Cancel) held = false; - std::cout << "Control action=" << static_cast(action) << " (diagnostic only)" << std::endl; + last_action = std::string(action_name(action)) + " received; diagnostic only."; + std::cout << last_action << std::endl; } auto next = overlay.controls_status(); if (next != status) { status = next; std::cout << status << std::endl; } + overlay.draw({"Controls ONLY - " + std::string(held ? "HOLD recognized" : "no microphone or typing"), + status, last_action, true, held}); std::this_thread::sleep_for(std::chrono::milliseconds(10)); } } else { diff --git a/src/overlay.cpp b/src/overlay.cpp index 8f0cc4c..69f67b7 100644 --- a/src/overlay.cpp +++ b/src/overlay.cpp @@ -64,11 +64,14 @@ struct Overlay::Impl { std::array poses{}; bool grip_capture = false, ptt_capture = false; bool focus = true; + bool persist_mount = true; + vr::EVRInputError action_update_error = vr::VRInputError_None; - Impl(const std::string& assets, const std::string& font, std::optional requested) + Impl(const std::string& assets, const std::string& font, std::optional requested, bool persist) : settings_path(default_mount_settings_path()), mount(requested ? *requested : load_mount(settings_path)), - surface(font.empty() ? (std::filesystem::path(assets) / "fonts/Inconsolata-Regular.ttf").string() : font, mount) { + surface(font.empty() ? (std::filesystem::path(assets) / "fonts/Inconsolata-Regular.ttf").string() : font, mount), + persist_mount(persist) { const auto action_path = absolute_file(std::filesystem::path(assets) / "actions.json"); absolute_file(std::filesystem::path(assets) / "bindings_knuckles.json"); try { @@ -234,10 +237,11 @@ struct Overlay::Impl { focus = true; reset_input(result); break; case vr::VREvent_OverlayGamepadFocusLost: case vr::VREvent_OverlayFocusChanged: - // Pointer/gamepad overlay focus is not OS keyboard focus or action - // activity. Cancel the old gesture, then require neutral rearm; - // do not permanently latch global grip actions off. - reset_input(result); break; + // Dashboard laser/gamepad focus can change between the two grip + // squeezes. It is not action activity or controller tracking loss: + // invalidate pointer presses, but leave a physical grip gesture + // armed. Hidden overlays and true OpenVR input capture still reset. + surface.reset_pointers(); break; case vr::VREvent_MouseMove: surface.pointer_move(event.data.mouse.cursorIndex, event.data.mouse.x, H - event.data.mouse.y); break; @@ -251,7 +255,7 @@ struct Overlay::Impl { if (event_result.action) result.push_back(*event_result.action); if (event_result.mount) { mount = *event_result.mount; - save_failed = !save_mount(settings_path, mount); + save_failed = persist_mount && !save_mount(settings_path, mount); world_ready = false; applied_mount.reset(); } @@ -266,7 +270,8 @@ struct Overlay::Impl { set.ulActionSet = action_set; set.ulRestrictedToDevice = vr::k_ulInvalidInputValueHandle; set.nPriority = 0; // normal priority; no experimental scene-input overrides - if (input->UpdateActionState(&set, sizeof(set), 1) != vr::VRInputError_None) { + action_update_error = input->UpdateActionState(&set, sizeof(set), 1); + if (action_update_error != vr::VRInputError_None) { reset_input(result); return result; } auto [la, ld] = digital(0); @@ -300,15 +305,35 @@ struct Overlay::Impl { } }; -Overlay::Overlay(const std::string& assets, const std::string& font, std::optional mount) - : impl_(std::make_unique(assets, font, mount)) {} +Overlay::Overlay(const std::string& assets, const std::string& font, std::optional mount, bool persist_mount) + : impl_(std::make_unique(assets, font, mount, persist_mount)) {} Overlay::~Overlay() = default; std::vector Overlay::poll() { return impl_->poll(); } void Overlay::draw(const Panel& panel) { impl_->draw(panel); } std::string Overlay::controls_status() { - auto left = impl_->digital(0), right = impl_->digital(1); - return std::string("Grip actions tracked/active: left=") + (left.first ? "yes" : "no") + - " right=" + (right.first ? "yes" : "no"); + // Diagnostic only: distinguish SteamVR binding/activity from our stricter + // pose/role gate. Raw legacy state is read-only and may be unavailable. + std::string result = "SteamVR update=" + std::to_string(int(impl_->action_update_error)); + for (size_t i = 0; i < 2; ++i) { + const auto role = i == 0 ? vr::TrackedControllerRole_LeftHand : vr::TrackedControllerRole_RightHand; + const auto device = impl_->system->GetTrackedDeviceIndexForControllerRole(role); + const bool tracked = device < impl_->poses.size() && impl_->poses[device].bPoseIsValid && + impl_->system->IsTrackedDeviceConnected(device); + vr::InputDigitalActionData_t data{}; + const auto error = impl_->input->GetDigitalActionData(impl_->actions[i], &data, sizeof(data), vr::k_ulInvalidInputValueHandle); + vr::InputOriginInfo_t origin{}; + const bool origin_ok = error == vr::VRInputError_None && data.activeOrigin != vr::k_ulInvalidInputValueHandle && + impl_->input->GetOriginTrackedDeviceInfo(data.activeOrigin, &origin, sizeof(origin)) == vr::VRInputError_None && + origin.trackedDeviceIndex == device; + vr::VRControllerState_t raw{}; + const bool raw_ok = device < impl_->poses.size() && impl_->system->GetControllerState(device, &raw, sizeof(raw)); + result += std::string(i == 0 ? "\nLeft:" : "\nRight:") + + " err=" + std::to_string(int(error)) + " bound=" + (data.bActive ? "Y" : "N") + + " down=" + (data.bState ? "Y" : "N") + " pose=" + (tracked ? "Y" : "N") + + " origin=" + (origin_ok ? "Y" : "N") + + " raw=" + (raw_ok ? ((raw.ulButtonPressed & vr::ButtonMaskFromId(vr::k_EButton_Grip)) ? "down" : "up") : "n/a"); + } + return result; } int registration(const std::string& manifest, bool remove, bool autostart) { diff --git a/src/overlay.hpp b/src/overlay.hpp index a8bed0f..4c0fad1 100644 --- a/src/overlay.hpp +++ b/src/overlay.hpp @@ -17,7 +17,8 @@ struct Panel { }; class Overlay { public: - Overlay(const std::string& assets, const std::string& font = {}, std::optional mount = {}); + Overlay(const std::string& assets, const std::string& font = {}, std::optional mount = {}, + bool persist_mount = true); ~Overlay(); Overlay(const Overlay&) = delete; Overlay& operator=(const Overlay&) = delete; diff --git a/src/runtime.cpp b/src/runtime.cpp index b93e781..f136b81 100644 --- a/src/runtime.cpp +++ b/src/runtime.cpp @@ -74,20 +74,33 @@ int run(const Options& options) { try { if (auto reply = worker.poll()) { if (reply->id == session.id() && session.state() == State::Transcribing) { - if (!reply->error.empty()) throw std::runtime_error(reply->error); - session.reply(reply->id, reply->text); - detail = session.text().empty() ? "No speech recognized; try again." : "Focus your destination, then Insert. Cancel discards."; + if (!reply->error.empty()) { + // E is a request-local error. The child still owns its + // loaded model and can accept the next utterance. + session.fail(); detail = reply->error + "; model ready. Record to retry."; + } else { + session.reply(reply->id, reply->text); + detail = session.text().empty() ? "No speech recognized; try again." : "Focus your destination, then Insert. Cancel discards."; + } } } if (worker.ready()) session.ready(); - if (audio.recording() && audio.poll()) stop_record(); } catch (const std::exception& e) { audio.cancel(); worker.stop(); if (session.state() != State::Review && session.state() != State::Queued) session.fail(); detail = e.what(); // Preserve an already-correlated preview if the worker dies. } + try { + if (audio.recording() && audio.poll()) stop_record(); + } catch (const std::exception& e) { + // A microphone failure is not a model failure. Release the device, + // retain the loaded worker, and allow Record to retry capture. + audio.cancel(); session.fail(); detail = e.what(); + } // Drawing precedes input polling: Enter is disabled until Ready is visible. - Panel panel{state_label(session.state()), session.text(), detail, + const auto status = session.state() == State::Error && worker.ready() + ? "Retry available - local model still loaded" : state_label(session.state()); + Panel panel{status, session.text(), detail, session.state() != State::Error && session.state() != State::Warming && session.state() != State::Transcribing, session.state() == State::Recording, session.state() != State::Warming && session.state() != State::Transcribing && session.state() != State::Review}; @@ -106,8 +119,10 @@ int run(const Options& options) { case UiAction::Cancel: { auto state = session.state(); audio.cancel(); session.cancel(); detail = "Discarded. Microphone closed."; - if (state == State::Warming || state == State::Transcribing || state == State::Error) { + if (state == State::Warming || state == State::Transcribing) { worker.stop(); session.fail(); detail = "Cancelled. Record to reload local worker."; + } else if (state == State::Error && !worker.ready()) { + session.fail(); detail = "Worker unavailable. Record to reload local worker."; } break; } @@ -131,9 +146,11 @@ int run(const Options& options) { break; } } catch (const std::exception& e) { - // Input lease failures preserve preview; capture failures release mic. + // Input lease failures preserve preview. Capture failures close + // only the microphone; submit failures stop their own child if + // the IPC stream was partially written. if (session.state() == State::Recording || session.state() == State::Transcribing) { - audio.cancel(); worker.stop(); session.fail(); + audio.cancel(); session.fail(); } detail = e.what(); } diff --git a/tests/core_test.cpp b/tests/core_test.cpp index 3085eda..03246e6 100644 --- a/tests/core_test.cpp +++ b/tests/core_test.cpp @@ -27,5 +27,11 @@ int main() { CHECK(s.record()); CHECK(s.finish(16000)); CHECK(s.reply(s.id(), "")); CHECK(s.state() == State::Ready); CHECK(!s.take_insert()); CHECK(s.record()); CHECK(s.finish(16000)); s.fail(); CHECK(!s.reply(s.id(), "bad")); + // Request-local model or microphone errors can leave the worker loaded: + // discard the failed session and record again without a new warmup state. + s.cancel(); CHECK(s.state() == State::Ready); + CHECK(s.record()); CHECK(s.finish(16000)); CHECK(s.reply(s.id(), "again")); + CHECK(s.take_insert() == "again"); CHECK(s.record()); + s.fail(); s.cancel(); CHECK(s.record()); std::cout << "core checks passed\n"; } diff --git a/tests/test_worker.py b/tests/test_worker.py index ac1c3a4..a4b455a 100644 --- a/tests/test_worker.py +++ b/tests/test_worker.py @@ -43,6 +43,7 @@ def fake_child(): time.sleep(130) return worker.send_frame(1, b"Y") + requests = 0 while True: msg = worker.read_frame(0) if msg is None: @@ -52,6 +53,7 @@ def fake_child(): time.sleep(70) continue assert len(msg) == 9 and msg[:1] == b"T" + requests += 1 clip = Path(args.clip_dir) / "clip.raw" assert clip.stat().st_size == 3200 * 4 ident = msg[1:9] @@ -60,6 +62,9 @@ def fake_child(): if args.model == "oversized-frame": os.write(1, struct.pack("id == 200 && reply->error == "transcription failed" && worker.ready()); + worker.submit(201, clip); // A request-local failure must not unload Redux. + until([&] { reply = worker.poll(); return reply.has_value(); }); + assert(reply->id == 201 && reply->text == "héllo 世界" && worker.ready()); + worker.stop(); worker.start(argv[1], argv[2], "fail", 2); bool failed = false; until([&] {