mirror of
https://github.com/baketnk/frame-yap.git
synced 2026-10-06 07:00:04 +02:00
Wire native OpenVR voice-typing POC with safe review and grip controls
This commit is contained in:
1 parent
73dc4fda06
commit
d60d8d2f5d
21 files changed
+1929
-29
No files matched your search
+146
@@ -0,0 +1,146 @@
|
||||
#include "runtime.hpp"
|
||||
#include "audio.hpp"
|
||||
#include "core.hpp"
|
||||
#include "overlay.hpp"
|
||||
#include "text_input.hpp"
|
||||
#include "worker.hpp"
|
||||
#include "instance_lock.hpp"
|
||||
#include <algorithm>
|
||||
#include <chrono>
|
||||
#include <csignal>
|
||||
#include <cstdlib>
|
||||
#include <filesystem>
|
||||
#include <fcntl.h>
|
||||
#include <stdexcept>
|
||||
#include <sys/file.h>
|
||||
#include <sys/stat.h>
|
||||
#include <thread>
|
||||
#include <unistd.h>
|
||||
|
||||
namespace frameyap {
|
||||
namespace {
|
||||
volatile sig_atomic_t interrupted = 0;
|
||||
void signal_stop(int) { interrupted = 1; }
|
||||
std::string state_label(State state) {
|
||||
switch (state) {
|
||||
case State::Warming: return "Warming - on-device Redux CPU";
|
||||
case State::Ready: return "Ready - double-tap + hold right grip";
|
||||
case State::Recording: return "RECORDING";
|
||||
case State::Transcribing: return "Transcribing - on device";
|
||||
case State::Review: return "Review - Insert approves CURRENT focus";
|
||||
case State::Queued: return "Input queued - not a delivery receipt";
|
||||
case State::Error: return "Unavailable - Record to retry";
|
||||
}
|
||||
return {};
|
||||
}
|
||||
}
|
||||
int run(const Options& options) {
|
||||
InstanceLock lock;
|
||||
interrupted = 0;
|
||||
auto old_int = std::signal(SIGINT, signal_stop);
|
||||
auto old_term = std::signal(SIGTERM, signal_stop);
|
||||
struct Restore { decltype(old_int) a, b; ~Restore() { std::signal(SIGINT, a); std::signal(SIGTERM, b); } } restore{old_int, old_term};
|
||||
Overlay overlay(options.assets, options.font, options.hand);
|
||||
Worker worker;
|
||||
Audio audio;
|
||||
Session session;
|
||||
std::string detail = "Review mode. Other apps may also hear your mic. Enter is separate.";
|
||||
bool quit = false;
|
||||
auto warm = [&] {
|
||||
worker.stop(); session = Session{};
|
||||
worker.start(options.python, options.worker, options.model, options.threads);
|
||||
detail = "Loading local model; microphone closed. Record again when Ready.";
|
||||
};
|
||||
auto stop_record = [&] {
|
||||
if (session.state() != State::Recording) return;
|
||||
auto pcm = audio.finish();
|
||||
if (session.finish(pcm.size())) {
|
||||
worker.submit(session.id(), pcm);
|
||||
detail = "Release complete. No text is inserted automatically.";
|
||||
} else detail = "Short tap discarded (minimum 200ms).";
|
||||
std::fill(pcm.begin(), pcm.end(), 0.0f);
|
||||
};
|
||||
auto start_record = [&] {
|
||||
if (session.state() == State::Review || session.state() == State::Transcribing || session.state() == State::Warming) return;
|
||||
if (!worker.ready()) { warm(); return; }
|
||||
if (session.state() == State::Error) session.cancel();
|
||||
if (!session.record()) return;
|
||||
audio.start();
|
||||
detail = "Release grip to finish. Cancel discards. Maximum 20 seconds.";
|
||||
};
|
||||
try { warm(); }
|
||||
catch (const std::exception& e) { session.fail(); detail = e.what(); }
|
||||
while (!quit && !interrupted) {
|
||||
try {
|
||||
if (auto reply = worker.poll()) {
|
||||
if (reply->id == session.id() && session.state() == State::Transcribing) {
|
||||
if (!reply->error.empty()) throw std::runtime_error(reply->error);
|
||||
session.reply(reply->id, reply->text);
|
||||
detail = session.text().empty() ? "No speech recognized; try again." : "Focus your destination, then Insert. Cancel discards.";
|
||||
}
|
||||
}
|
||||
if (worker.ready()) session.ready();
|
||||
if (audio.recording() && audio.poll()) stop_record();
|
||||
} catch (const std::exception& e) {
|
||||
audio.cancel(); worker.stop();
|
||||
if (session.state() != State::Review && session.state() != State::Queued) session.fail();
|
||||
detail = e.what(); // Preserve an already-correlated preview if the worker dies.
|
||||
}
|
||||
// Drawing precedes input polling: Enter is disabled until Ready is visible.
|
||||
Panel panel{state_label(session.state()), session.text(), detail,
|
||||
session.state() != State::Error && session.state() != State::Warming,
|
||||
session.state() == State::Recording};
|
||||
if (panel.recording) panel.status += " - " + std::to_string(audio.seconds()) + " / 20s";
|
||||
overlay.draw(panel);
|
||||
for (auto action : overlay.poll()) {
|
||||
try {
|
||||
switch (action) {
|
||||
case UiAction::Quit: quit = true; break;
|
||||
case UiAction::Toggle:
|
||||
case UiAction::Record:
|
||||
if (session.state() == State::Recording) stop_record(); else start_record();
|
||||
break;
|
||||
case UiAction::BeginRecord: start_record(); break;
|
||||
case UiAction::EndRecord: stop_record(); break;
|
||||
case UiAction::Cancel: {
|
||||
auto state = session.state();
|
||||
audio.cancel(); session.cancel(); detail = "Discarded. Microphone closed.";
|
||||
if (state == State::Warming || state == State::Transcribing || state == State::Error) {
|
||||
worker.stop(); session.fail(); detail = "Cancelled. Record to reload local worker.";
|
||||
}
|
||||
break;
|
||||
}
|
||||
case UiAction::Insert:
|
||||
if (session.state() == State::Review) {
|
||||
// Acquire IME before consuming: unavailable means the preview remains.
|
||||
TextInput input(options.socket);
|
||||
if (interrupted) break;
|
||||
auto literal = session.take_insert();
|
||||
try { input.text(*literal); detail = "Input queued to current focus. Enter remains a separate action."; }
|
||||
catch (...) { detail = "Delivery uncertain; not retried. Check destination before dictating again."; }
|
||||
}
|
||||
break;
|
||||
case UiAction::Enter:
|
||||
if (session.state() == State::Ready || session.state() == State::Queued || session.state() == State::Review) {
|
||||
TextInput input(options.socket);
|
||||
if (interrupted) break;
|
||||
try { input.enter(); detail = "Explicit Enter queued to current focus."; }
|
||||
catch (...) { detail = "Enter delivery uncertain; not retried."; }
|
||||
}
|
||||
break;
|
||||
}
|
||||
} catch (const std::exception& e) {
|
||||
// Input lease failures preserve preview; capture failures release mic.
|
||||
if (session.state() == State::Recording || session.state() == State::Transcribing) {
|
||||
audio.cancel(); worker.stop(); session.fail();
|
||||
}
|
||||
detail = e.what();
|
||||
}
|
||||
if (quit || interrupted) break;
|
||||
}
|
||||
std::this_thread::sleep_for(std::chrono::milliseconds(10));
|
||||
}
|
||||
audio.cancel(); session.cancel(); worker.stop();
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user