From 4498ef5d04f49776b488800a4db0513de8b827f1 Mon Sep 17 00:00:00 2001 From: baketnk Date: Thu, 24 Sep 2026 23:03:09 -0400 Subject: [PATCH] feat(ui): add consent-bound model chooser and microphone controls --- assets/actions.json | 8 +- scripts/backend-service.py | 16 +- src/angle_fade.hpp | 4 +- src/backend_manager.cpp | 310 +++++++++++++++++++++++++++++++++ src/backend_manager.hpp | 52 ++++++ src/overlay.cpp | 244 ++++++++++++++------------ src/overlay.hpp | 14 ++ src/panel_surface.cpp | 214 +++++++++++++++++++---- src/panel_surface.hpp | 6 + tests/backend_manager_test.cpp | 153 ++++++++++++++++ tests/panel_test.cpp | 164 ++++++++++++++--- tests/test_backend_dispatch.py | 8 + 12 files changed, 1025 insertions(+), 168 deletions(-) create mode 100644 src/backend_manager.cpp create mode 100644 src/backend_manager.hpp create mode 100644 tests/backend_manager_test.cpp diff --git a/assets/actions.json b/assets/actions.json index 530435c..ed1e68b 100644 --- a/assets/actions.json +++ b/assets/actions.json @@ -18,12 +18,12 @@ "localization": [{ "language_tag": "en_US", "/actions/frameyap": "FrameYap controls", - "/actions/frameyap/in/left_grip": "Double tap left grip: insert pending text + Enter (enabled only)", + "/actions/frameyap/in/left_grip": "Double tap left grip: Type + Enter (enabled only)", "/actions/frameyap/in/right_grip": "Tap right grip briefly, then squeeze and hold to record; release to finish", "/actions/frameyap/in/ptt": "Hold right X to record (default; remappable)", "/actions/frameyap/in/cancel": "Cancel recording or discard", - "/actions/frameyap/in/insert": "Insert reviewed text + trailing space", - "/actions/frameyap/in/enter": "Submit: pending text + Enter, or Enter alone", - "/actions/frameyap/in/quick_chat": "Open quick chat / cycle selection" + "/actions/frameyap/in/insert": "Type reviewed text + trailing space", + "/actions/frameyap/in/enter": "Type + Enter: pending text + Enter, or Enter alone", + "/actions/frameyap/in/quick_chat": "Open quick phrases / cycle selection" }] } diff --git a/scripts/backend-service.py b/scripts/backend-service.py index 8f15ea7..713cb77 100644 --- a/scripts/backend-service.py +++ b/scripts/backend-service.py @@ -6,6 +6,7 @@ second manifest/JSON/hash implementation. Installer execution is opt-in ONLY. """ import argparse import hashlib +import json import os import re import stat @@ -40,6 +41,13 @@ def manifest_sha(manifest_dir, ident): return hashlib.sha256(raw).hexdigest() +def install_error(code, message): + # stdout is the manager's bounded event channel; stderr is intentionally + # closed there. Never claim success or start an installer on these paths. + print(json.dumps({"ok": False, "code": code, "message": message}), flush=True) + return 2 + + def main(argv=None): parser = argparse.ArgumentParser(description=__doc__) operation = parser.add_mutually_exclusive_group(required=True) @@ -87,15 +95,15 @@ def main(argv=None): # consent. The exec makes the installer the owned child (no hidden grandchild). if (not args.backend or not args.installer or not args.expected_manifest_sha256 or not _DIGEST.fullmatch(args.expected_manifest_sha256)): - parser.error("backend, installer and 64-character consent fingerprint required") + return install_error("usage", "backend, installer and consent fingerprint required") try: backends = load_backends(args.manifest_dir) if args.backend not in backends or manifest_sha(args.manifest_dir, args.backend) != args.expected_manifest_sha256: - parser.error("selected backend manifest changed since consent") + return install_error("manifest_mismatch", "selected backend manifest changed since consent") except (OSError, ValueError) as error: - parser.error(f"selected backend manifest unavailable: {type(error).__name__}") + return install_error("operation_failed", f"selected backend manifest unavailable: {type(error).__name__}") if not args.installer.is_file() or args.installer.is_symlink(): - parser.error("installer missing or unsafe") + return install_error("operation_failed", "installer missing or unsafe") dest = args.model_store / args.backend os.execv("/bin/sh", ["sh", str(args.installer), "--install-model", "--backend", args.backend, "--model-dir", str(dest), "--expected-manifest-sha256", diff --git a/src/angle_fade.hpp b/src/angle_fade.hpp index 03b7a7a..558526f 100644 --- a/src/angle_fade.hpp +++ b/src/angle_fade.hpp @@ -24,8 +24,8 @@ inline bool normalize(Vec3& v) { } // namespace angle_fade_detail // Wrist-only visibility: compare the entire panel orientation (including roll) -// to an upright panel pointing at the viewer. Kouseki's watch fades linearly -// between 60 and 75 degrees; the matrix math here has no engine dependency. +// to an upright panel pointing at the viewer. Fade linearly between 60 and +// 75 degrees; the matrix math here has no engine dependency. // Both poses must be in the same tracking space, with rigid orthonormal axes. // Invalid geometry hides the panel rather than making it clickable at full alpha. inline float wrist_view_opacity(const Matrix34& panel, const Matrix34& head) { diff --git a/src/backend_manager.cpp b/src/backend_manager.cpp new file mode 100644 index 0000000..613acfc --- /dev/null +++ b/src/backend_manager.cpp @@ -0,0 +1,310 @@ +#include "backend_manager.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +extern char** environ; + +namespace frameyap { +namespace { +std::string decode(const std::string& value) { + std::string output; + for (size_t i = 0; i < value.size(); ++i) { + if (value[i] == '%') { + if (i + 2 >= value.size()) throw std::runtime_error("bad status encoding"); + unsigned n = 0; + auto [end, error] = std::from_chars(value.data() + i + 1, value.data() + i + 3, n, 16); + if (error != std::errc{} || end != value.data() + i + 3) throw std::runtime_error("bad status encoding"); + output += char(n); i += 2; + } else output += value[i]; + } + if (output.size() > 1100 || std::any_of(output.begin(), output.end(), [](unsigned char c) { + return c < 32 || c == 127; + })) throw std::runtime_error("unsafe status text"); + return output; +} +// The installed shell script emits one flat JSON object per line. Read only +// known fields; malformed/unknown records have no UI or success authority. +// Pipe byte and line caps are enforced by poll() before this parser runs. +struct JsonValue { std::string text; bool quoted = false; }; +using JsonObject = std::map; +bool json_string(const std::string& s, size_t& pos, std::string& out) { + if (pos >= s.size() || s[pos++] != '"') return false; + while (pos < s.size()) { + unsigned char c = static_cast(s[pos++]); + if (c == '"') return true; + if (c < 32) return false; + if (c != '\\') { out += char(c); continue; } + if (pos == s.size()) return false; + c = static_cast(s[pos++]); + switch (c) { + case '"': case '\\': case '/': out += char(c); break; + case 'b': case 'f': case 'n': case 'r': case 't': out += ' '; break; + case 'u': { + if (s.size() - pos < 4) return false; + unsigned n = 0; + auto [end, err] = std::from_chars(s.data() + pos, s.data() + pos + 4, n, 16); + if (err != std::errc{} || end != s.data() + pos + 4) return false; + pos += 4; + out += n >= 32 && n < 127 ? char(n) : '?'; + break; + } + default: return false; + } + } + return false; +} +std::optional json_event(const std::string& s) { + size_t pos = 0; + auto space = [&] { while (pos < s.size() && (s[pos] == ' ' || s[pos] == '\t')) ++pos; }; + space(); + if (pos == s.size() || s[pos++] != '{') return {}; + JsonObject result; + bool closed = false; + space(); + if (pos < s.size() && s[pos] == '}') { ++pos; closed = true; } + else while (pos < s.size() && result.size() < 16) { + std::string key, value; + if (!json_string(s, pos, key) || key.size() > 40) return {}; + space(); + if (pos == s.size() || s[pos++] != ':') return {}; + space(); + bool quoted = pos < s.size() && s[pos] == '"'; + if (quoted) { if (!json_string(s, pos, value)) return {}; } + else { + size_t start = pos; + while (pos < s.size() && s[pos] != ',' && s[pos] != '}' && s[pos] != ' ' && s[pos] != '\t') ++pos; + value = s.substr(start, pos - start); + if (value != "true" && value != "false" && value != "null" && + (value.empty() || !std::all_of(value.begin(), value.end(), [](unsigned char c) { return c >= '0' && c <= '9'; }))) return {}; + } + if (!result.emplace(std::move(key), JsonValue{std::move(value), quoted}).second) return {}; + space(); + if (pos < s.size() && s[pos] == '}') { ++pos; closed = true; break; } + if (pos == s.size() || s[pos++] != ',') return {}; + space(); + } + space(); + if (!closed || pos != s.size()) return {}; + return result; +} +std::string safe_label(const std::string& s, size_t max) { + std::string result; + for (unsigned char c : s) { + if (result.size() == max) break; + result += c >= 32 && c < 127 ? char(c) : '?'; + } + return result; +} +std::vector fields(const std::string& line) { + std::vector result; + size_t start = 0; + while (start <= line.size()) { + size_t end = line.find('\t', start); + if (end == std::string::npos) end = line.size(); + result.push_back(decode(line.substr(start, end - start))); + start = end + 1; + } + return result; +} +} +BackendManager::BackendManager(std::string python, std::string service, std::string manifest_dir, + std::string model_store, std::string installer, std::string legacy_model) + : python_(std::move(python)), service_(std::move(service)), manifest_(std::move(manifest_dir)), + store_(std::move(model_store)), installer_(std::move(installer)), legacy_model_(std::move(legacy_model)) {} +std::string BackendManager::model_path(const std::string& id) const { + auto it = std::find_if(entries_.begin(), entries_.end(), [&](const auto& entry) { return entry.id == id; }); + return it == entries_.end() ? store_ + "/" + id : it->model_path; +} +BackendManager::~BackendManager() { cancel(); } +void BackendManager::cancel() noexcept { + if (pid_ > 0) { + ::kill(pid_, SIGTERM); // exact owned child only; service execs installer + int status = 0; + bool reaped = false; + // Give the owned installer a short chance to remove its own download + // temp in a SIGTERM handler; never wait indefinitely on the overlay. + for (int n = 0; n < 20; ++n) { + pid_t result = ::waitpid(pid_, &status, WNOHANG); + if (result == pid_ || (result < 0 && errno == ECHILD)) { reaped = true; break; } + if (result < 0 && errno != EINTR) break; + ::usleep(5000); + } + if (!reaped) { + ::kill(pid_, SIGKILL); + while (::waitpid(pid_, &status, 0) < 0 && errno == EINTR) {} + } + pid_ = -1; + } + if (output_ >= 0) { ::close(output_); output_ = -1; } + buffer_.clear(); installing_ = false; eof_ = false; output_bytes_ = 0; +} +void BackendManager::launch(std::vector args, bool installing) { + if (busy()) throw std::logic_error("backend operation already running"); + int pipe[2]; + if (::pipe2(pipe, O_CLOEXEC)) throw std::runtime_error("backend status pipe unavailable"); + // Pipe ends have independent flags: only the parent's read must be + // nonblocking. A nonblocking child stdout can lose a burst to EAGAIN. + if (::fcntl(pipe[0], F_SETFL, O_NONBLOCK) < 0) { + ::close(pipe[0]); ::close(pipe[1]); + throw std::runtime_error("backend status pipe unavailable"); + } + posix_spawn_file_actions_t actions; + int error = posix_spawn_file_actions_init(&actions); + bool actions_initialized = error == 0; + if (!error) error = posix_spawn_file_actions_adddup2(&actions, pipe[1], STDOUT_FILENO); + if (!error) error = posix_spawn_file_actions_addopen(&actions, STDERR_FILENO, "/dev/null", O_WRONLY, 0); + if (!error) error = posix_spawn_file_actions_addclose(&actions, pipe[0]); + if (!error) error = posix_spawn_file_actions_addclose(&actions, pipe[1]); + std::vector argv; + for (auto& arg : args) argv.push_back(arg.data()); + argv.push_back(nullptr); + pid_t child = -1; + if (!error) error = posix_spawnp(&child, args[0].c_str(), &actions, nullptr, argv.data(), environ); + if (actions_initialized) posix_spawn_file_actions_destroy(&actions); + ::close(pipe[1]); + if (error) { ::close(pipe[0]); throw std::runtime_error("could not start local backend helper"); } + pid_ = child; output_ = pipe[0]; installing_ = installing; + done_ = failed_ = eof_ = false; buffer_.clear(); output_bytes_ = 0; + if (installing) { install_failed_ = false; install_error_.clear(); } + deadline_ = std::chrono::steady_clock::now() + (installing ? std::chrono::minutes(30) : std::chrono::minutes(5)); + note_ = installing ? "Downloading pinned model after explicit consent..." : + install_failed_ ? "Install failed: " + + (install_error_.empty() ? std::string("installer exited unsuccessfully") : install_error_) + + "; rechecking local model files..." : "Checking local model files (offline)..."; +} +void BackendManager::refresh() { + if (busy()) return; + entries_.clear(); checked_ = false; + std::vector args{python_, service_, "--status", "--manifest-dir", manifest_, "--model-store", store_}; + if (!legacy_model_.empty()) args.insert(args.end(), {"--legacy-model", legacy_model_}); + launch(std::move(args), false); +} +void BackendManager::install(const std::string& id, const std::string& consent_sha256) { + if (busy() || !checked_ || installer_.empty()) throw std::runtime_error("Installer unavailable or model status unchecked"); + auto found = std::find_if(entries_.begin(), entries_.end(), [&](const auto& e) { return e.id == id; }); + if (found == entries_.end() || found->state == "installed_verified" || + found->manifest_sha256 != consent_sha256 || consent_sha256.size() != 64) + throw std::runtime_error("Selected backend metadata changed; review consent again"); + launch({python_, service_, "--install", "--backend", id, "--installer", installer_, + "--manifest-dir", manifest_, "--model-store", store_, + "--expected-manifest-sha256", consent_sha256}, true); +} +void BackendManager::line(const std::string& text) { + if (installing_) { + // These are advisory, bounded display events only. Exit zero followed + // by a fresh offline check is the sole route to installed status. + auto object = json_event(text); + if (!object) return; + auto field = [&](const char* key, bool quoted) -> std::string { + auto it = object->find(key); + return it != object->end() && it->second.quoted == quoted ? it->second.text : ""; + }; + if (field("ok", false) == "false") { + const auto code = field("code", true); + const auto message = field("message", true); + if (!message.empty()) { + install_error_ = safe_label(message, 130); + if (code == "manifest_mismatch" || code == "usage") + install_error_ = code + ": " + install_error_; + note_ = "Install error: " + install_error_; + ++revision_; + } + } else if (field("ok", false) == "true" && field("event", true) == "model_file") { + auto state = field("state", true); + auto file = field("file", true); + if ((state == "downloading" || state == "verified") && !file.empty()) { + note_ = (state == "downloading" ? "Downloading: " : "Verified file: ") + safe_label(file, 95); + ++revision_; + } + } + return; + } + if (done_ || failed_) throw std::runtime_error("backend status after terminal record"); + const auto f = fields(text); + if (f.size() == 1 && f[0] == "DONE") { done_ = true; return; } + if (f.size() == 2 && f[0] == "ERROR") { failed_ = true; return; } + if (f.size() != 12 || f[0] != "ST" || f[1].empty() || f[1].size() > 48 || + (f[3] != "not_installed" && f[3] != "installed_verified" && f[3] != "invalid") || + entries_.size() >= 32) throw std::runtime_error("invalid backend status record"); + uint64_t bytes = 0; + auto [end, error] = std::from_chars(f[5].data(), f[5].data() + f[5].size(), bytes); + if (error != std::errc{} || end != f[5].data() + f[5].size() || + std::any_of(entries_.begin(), entries_.end(), [&](const auto& e) { return e.id == f[1]; })) + throw std::runtime_error("invalid backend status metadata"); + static const std::regex id_pattern("[a-z][a-z0-9_-]{0,47}"); + static const std::regex digest_pattern("[0-9a-f]{64}"); + if (f[10].empty() || f[10][0] != '/' || !std::regex_match(f[1], id_pattern) || + !std::regex_match(f[11], digest_pattern)) throw std::runtime_error("invalid model location or fingerprint"); + entries_.push_back({f[1], f[2], f[3], f[4], f[6], f[7], f[8], f[9], f[10], bytes, f[11]}); +} +void BackendManager::poll() { + if (!busy()) return; + try { + if (std::chrono::steady_clock::now() > deadline_) throw std::runtime_error("backend operation timed out"); + size_t budget = 64 * 1024; + char chunk[4096]; + while (!eof_ && budget) { + ssize_t count = ::read(output_, chunk, std::min(budget, sizeof chunk)); + if (count > 0) { + budget -= size_t(count); output_bytes_ += size_t(count); + if (output_bytes_ > 256 * 1024) throw std::runtime_error("excessive backend output"); + buffer_.append(chunk, size_t(count)); + for (auto end = buffer_.find('\n'); end != std::string::npos; end = buffer_.find('\n')) { + if (end > 8192) throw std::runtime_error("oversized backend status line"); + line(buffer_.substr(0, end)); buffer_.erase(0, end + 1); + } + if (buffer_.size() > 8192) throw std::runtime_error("oversized backend status line"); + } else if (count == 0) { eof_ = true; ::close(output_); output_ = -1; } + else if (errno == EINTR) continue; + else if (errno == EAGAIN || errno == EWOULDBLOCK) break; + else throw std::runtime_error("backend status pipe failed"); + } + if (!eof_) return; // a successful child may still have unread pipe data + int status = 0; + pid_t result = ::waitpid(pid_, &status, WNOHANG); + if (result < 0) throw std::runtime_error("backend helper wait failed"); + if (!result) return; + pid_ = -1; + if (!WIFEXITED(status) || WEXITSTATUS(status) != 0 || failed_ || !buffer_.empty() || + (!installing_ && !done_)) { + bool was_installing = installing_; + installing_ = false; + if (was_installing) { + // Source metadata may have changed since consent. Recheck before + // another attempt; never keep stale manifest consent reusable. + install_failed_ = true; + checked_ = false; + refresh(); + } else note_ = install_failed_ ? "Install failed: " + + (install_error_.empty() ? std::string("installer exited unsuccessfully") : install_error_) + + "; local model recheck failed." : "Local model check failed."; + ++revision_; + return; + } + if (installing_) { installing_ = false; note_ = "Install finished; verifying local model files..."; refresh(); } + else { checked_ = true; note_ = install_failed_ ? "Install failed: " + + (install_error_.empty() ? std::string("installer exited unsuccessfully") : install_error_) + + "; local model status checked offline." : "Local model status checked offline."; ++revision_; } + } catch (const std::exception& error) { + bool was_installing = installing_; + if (was_installing && install_error_.empty()) install_error_ = safe_label(error.what(), 130); + cancel(); + if (was_installing) checked_ = false; + if (was_installing) install_failed_ = true; + note_ = was_installing ? "Install failed: " + install_error_ + ". Restart to recheck models." + : install_failed_ ? "Install failed: " + install_error_ + "; local model recheck failed." + : "Local model check failed."; + ++revision_; + } +} +} // namespace frameyap diff --git a/src/backend_manager.hpp b/src/backend_manager.hpp new file mode 100644 index 0000000..409cb2f --- /dev/null +++ b/src/backend_manager.hpp @@ -0,0 +1,52 @@ +#pragma once +#include +#include +#include +#include +#include + +namespace frameyap { +struct BackendEntry { + std::string id, name, state, reason, source, license, license_text, attribution, model_path; + uint64_t bytes = 0; + std::string manifest_sha256; +}; +// All hashing/metadata comes from model_files.py via backend-service.py. +// This class never opens the network or blocks waiting for verification/install. +class BackendManager { +public: + BackendManager(std::string python, std::string service, std::string manifest_dir, + std::string model_store, std::string installer, std::string legacy_model = {}); + ~BackendManager(); + BackendManager(const BackendManager&) = delete; + BackendManager& operator=(const BackendManager&) = delete; + void refresh(); + void install(const std::string& id, const std::string& consent_sha256); // immutable digest captured from displayed confirmation + void poll(); + void cancel() noexcept; + const std::vector& entries() const { return entries_; } + const std::string& note() const { return note_; } + bool busy() const { return pid_ > 0; } + bool installing() const { return installing_; } + size_t revision() const { return revision_; } + bool checked() const { return checked_; } + std::string model_path(const std::string& id) const; + const std::string& manifest_dir() const { return manifest_; } +private: + void launch(std::vector arguments, bool installing); + void line(const std::string& text); + std::string python_, service_, manifest_, store_, installer_, legacy_model_; + std::vector entries_; + std::string note_ = "Checking local models (offline)..."; + std::string buffer_; + pid_t pid_ = -1; + int output_ = -1; + size_t revision_ = 0; + bool installing_ = false, done_ = false, failed_ = false, checked_ = false; + bool install_failed_ = false; // retain failed-install feedback through the offline recheck + std::string install_error_; // sanitized, bounded installer error; not success authority + bool eof_ = false; + size_t output_bytes_ = 0; + std::chrono::steady_clock::time_point deadline_{}; +}; +} diff --git a/src/overlay.cpp b/src/overlay.cpp index 0241e1e..b270ed3 100644 --- a/src/overlay.cpp +++ b/src/overlay.cpp @@ -76,26 +76,33 @@ struct Overlay::Impl { std::unique_ptr gpu_texture; vr::VRActionSetHandle_t action_set = vr::k_ulInvalidActionSetHandle; std::array actions{}; - std::filesystem::path settings_path; - std::filesystem::path laser_settings_path; + struct Persistence { + std::filesystem::path settings_path, laser_settings_path; + bool save_failed = false, debug_save_failed = false, auto_save_failed = false; + bool layout_save_failed = false, mic_save_failed = false, laser_change_failed = false; + bool persist_mount = true; + } persistence; Mount mount; - bool lasers_anytime = false, laser_change_failed = false; + bool lasers_anytime = false; Config config; PanelSurface surface; Panel panel; - bool save_failed = false, debug_save_failed = false, auto_save_failed = false, layout_save_failed = false; + std::vector model_actions; bool world_ready = false, placed = false, has_texture = false, shown = false; float published_alpha = -1.f; float size_scale = 1.f; bool placement_dirty = false; - Matrix34 canvas_pose{}, drag_canvas{}; - PanelDrag drag; - PanelDragKind drag_kind = PanelDragKind::Grab; - unsigned drag_cursor = 0; - vr::TrackedDeviceIndex_t drag_device = vr::k_unTrackedDeviceIndexInvalid; - bool drag_trigger_observed = false; - float drag_scale = 1.f; - std::chrono::steady_clock::time_point drag_started{}; + Matrix34 canvas_pose{}; + struct DragState { + Matrix34 canvas{}; + PanelDrag panel; + PanelDragKind kind = PanelDragKind::Grab; + unsigned cursor = 0; + vr::TrackedDeviceIndex_t device = vr::k_unTrackedDeviceIndexInvalid; + bool trigger_observed = false; + float scale = 1.f; + std::chrono::steady_clock::time_point started{}; + } dragging; vr::HmdMatrix34_t world_transform{}; std::optional applied_mount; vr::TrackedDeviceIndex_t anchor = vr::k_unTrackedDeviceIndexInvalid; @@ -105,22 +112,22 @@ struct Overlay::Impl { std::array poses{}; bool grip_capture = false, ptt_capture = false; bool focus = true; - bool persist_mount = true; - vr::EVRInputError action_update_error = vr::VRInputError_None; - unsigned pointer_downs = 0, pointer_ups = 0, pointer_actions = 0, pointer_resets = 0; - unsigned texture_uploads = 0, show_calls = 0, hide_calls = 0; - unsigned overlay_shown_events = 0, overlay_hidden_events = 0, image_loaded_events = 0, image_failed_events = 0; - unsigned overlay_focus_events = 0, global_focus_events = 0, input_focus_captured_events = 0; - std::string last_pointer_event = "none"; + struct Diagnostics { + vr::EVRInputError action_update_error = vr::VRInputError_None; + unsigned pointer_downs = 0, pointer_ups = 0, pointer_actions = 0, pointer_resets = 0; + unsigned texture_uploads = 0, show_calls = 0, hide_calls = 0; + unsigned overlay_shown_events = 0, overlay_hidden_events = 0, image_loaded_events = 0, image_failed_events = 0; + unsigned overlay_focus_events = 0, global_focus_events = 0, input_focus_captured_events = 0; + std::string last_pointer_event = "none"; + } diagnostics; Impl(const std::string& assets, const std::string& font, std::optional requested, bool persist) - : settings_path(default_mount_settings_path()), - laser_settings_path(default_laser_settings_path()), - mount(requested ? *requested : load_mount(settings_path)), - lasers_anytime(load_lasers_anytime(laser_settings_path)), + : persistence{default_mount_settings_path(), default_laser_settings_path()}, + mount(requested ? *requested : load_mount(persistence.settings_path)), + lasers_anytime(load_lasers_anytime(persistence.laser_settings_path)), config(load_config(default_config_path())), - surface(resolve_font(assets, font.empty() ? config.font : font), mount, config.theme), - persist_mount(persist) { + surface(resolve_font(assets, font.empty() ? config.font : font), mount, config.theme) { + persistence.persist_mount = persist; const auto action_path = absolute_file(action_manifest(assets, config)); absolute_file(std::filesystem::path(assets) / "bindings_knuckles.json"); try { @@ -167,11 +174,12 @@ struct Overlay::Impl { // panel is visible; it may affect interaction with a running game. if (lasers_anytime && overlay->SetOverlayFlag(handle, vr::VROverlayFlags_MakeOverlaysInteractiveIfVisible, true) != vr::VROverlayError_None) { lasers_anytime = false; - laser_change_failed = true; + persistence.laser_change_failed = true; } surface.set_lasers_anytime(lasers_anytime); surface.set_advanced_debug(config.advanced_debug); surface.set_auto_insert(config.auto_insert); + surface.set_close_mic_when_idle(config.close_mic_when_idle); surface.set_layout_locked(config.lock_layout); surface.set_clock_24h(config.clock_24h); surface.set_date_format(config.date_format); @@ -228,7 +236,7 @@ struct Overlay::Impl { const bool wanted = placed && has_texture && alpha > 0.f; if (wanted == shown) return; overlay_check(wanted ? overlay->ShowOverlay(handle) : overlay->HideOverlay(handle), overlay, "Overlay visibility"); - if (wanted) ++show_calls; else ++hide_calls; + if (wanted) ++diagnostics.show_calls; else ++diagnostics.hide_calls; shown = wanted; } void place() { @@ -242,12 +250,13 @@ struct Overlay::Impl { } if (effective == Mount::World && applied_mount && *applied_mount != Mount::World) world_ready = false; // a fresh world fallback near the wearer, not an old room location - std::string note = layout_save_failed ? "Layout lock not saved; using it only for this session." : - auto_save_failed ? "Auto insert preference not saved; using it only for this session." : - debug_save_failed ? "Debug preference not saved; using it only for this session." : - laser_change_failed ? "SteamVR declined the laser mode change." : - save_failed ? "Preference could not be saved; using it for this session." : ""; - if (effective != mount) note = save_failed ? "Wrist untracked; world fallback. Preference not saved." : + std::string note = persistence.mic_save_failed ? "Mic preference not saved; using it only for this session." : + persistence.layout_save_failed ? "Layout lock not saved; using it only for this session." : + persistence.auto_save_failed ? "Auto insert preference not saved; using it only for this session." : + persistence.debug_save_failed ? "Debug preference not saved; using it only for this session." : + persistence.laser_change_failed ? "SteamVR declined the laser mode change." : + persistence.save_failed ? "Preference could not be saved; using it for this session." : ""; + if (effective != mount) note = persistence.save_failed ? "Wrist untracked; world fallback. Preference not saved." : "Wrist not tracked - using world space until it returns."; if (effective == Mount::World && !world_ready) { const auto& hmd = poses[vr::k_unTrackedDeviceIndex_Hmd]; @@ -269,7 +278,7 @@ struct Overlay::Impl { if (relocated || placement_dirty) { const float base_width = mount_width(effective, config.wrist); if (relocated) { - surface.reset_pointers(); drag.reset(); + surface.reset_pointers(); dragging.panel.reset(); auto pose = effective == Mount::World ? matrix(world_transform) : relative_mount_pose(effective, config.wrist); pose = resized_mount_pose(pose, base_width, size_scale, float(CH) / CW); // Preserve main-panel dimensions when adding transparent margins. @@ -301,8 +310,8 @@ struct Overlay::Impl { return device < poses.size() && poses[device].bPoseIsValid && poses[device].bDeviceIsConnected; } std::optional drag_source() const { - if (!tracked(drag_device) || !applied_mount) return {}; - auto pose = matrix(poses[drag_device].mDeviceToAbsoluteTracking); + if (!tracked(dragging.device) || !applied_mount) return {}; + auto pose = matrix(poses[dragging.device].mDeviceToAbsoluteTracking); if (*applied_mount != Mount::World) { if (!tracked(anchor)) return {}; pose = relative_pose(matrix(poses[anchor].mDeviceToAbsoluteTracking), pose); @@ -310,53 +319,53 @@ struct Overlay::Impl { return pose; } void begin_drag(PanelDragKind kind, const vr::VREvent_t& event) { - if (config.lock_layout) { surface.reset_pointers(); drag.reset(); return; } - drag_device = event.trackedDeviceIndex; - drag_cursor = event.data.mouse.cursorIndex; + if (config.lock_layout) { surface.reset_pointers(); dragging.panel.reset(); return; } + dragging.device = event.trackedDeviceIndex; + dragging.cursor = event.data.mouse.cursorIndex; // Single-cursor overlay: some runtime mouse events omit the source. // The dashboard's primary device is the only supported fallback, never // a guessed left/right hand or a source chosen by proximity. - if (drag_cursor == 0 && (drag_device == vr::k_unTrackedDeviceIndexInvalid || - drag_device == vr::k_unTrackedDeviceIndex_Hmd)) - drag_device = overlay->GetPrimaryDashboardDevice(); - drag_kind = kind; + if (dragging.cursor == 0 && (dragging.device == vr::k_unTrackedDeviceIndexInvalid || + dragging.device == vr::k_unTrackedDeviceIndex_Hmd)) + dragging.device = overlay->GetPrimaryDashboardDevice(); + dragging.kind = kind; const auto source = drag_source(); const float w = applied_mount ? mount_width(*applied_mount, config.wrist) * size_scale * W / CW : 0.f; - if (!source || system->GetTrackedDeviceClass(drag_device) != vr::TrackedDeviceClass_Controller || - !drag.begin(kind, canvas_pose, w, w * H / W, event.data.mouse.x / W, - 1.f - event.data.mouse.y / H, *source)) { - surface.reset_pointers(); drag.reset(); - last_pointer_event = "drag source unavailable"; + if (!source || system->GetTrackedDeviceClass(dragging.device) != vr::TrackedDeviceClass_Controller || + !dragging.panel.begin(kind, canvas_pose, w, w * H / W, event.data.mouse.x / W, + 1.f - event.data.mouse.y / H, *source)) { + surface.reset_pointers(); dragging.panel.reset(); + diagnostics.last_pointer_event = "drag source unavailable"; return; } - drag_scale = size_scale; drag_canvas = canvas_pose; - drag_started = std::chrono::steady_clock::now(); + dragging.scale = size_scale; dragging.canvas = canvas_pose; + dragging.started = std::chrono::steady_clock::now(); vr::VRControllerState_t state{}; - drag_trigger_observed = system->GetControllerState(drag_device, &state, sizeof(state)) && + dragging.trigger_observed = system->GetControllerState(dragging.device, &state, sizeof(state)) && (state.ulButtonPressed & vr::ButtonMaskFromId(vr::k_EButton_SteamVR_Trigger)); - last_pointer_event = std::string(kind == PanelDragKind::Grab ? "grab" : "scale") + - " device=" + std::to_string(drag_device) + " trigger-watch=" + (drag_trigger_observed ? "Y" : "N"); + diagnostics.last_pointer_event = std::string(kind == PanelDragKind::Grab ? "grab" : "scale") + + " device=" + std::to_string(dragging.device) + " trigger-watch=" + (dragging.trigger_observed ? "Y" : "N"); } void update_drag() { - if (!drag.active()) return; + if (!dragging.panel.active()) return; const auto source = drag_source(); vr::VRControllerState_t state{}; - const bool trigger_released = drag_trigger_observed && - (!system->GetControllerState(drag_device, &state, sizeof(state)) || + const bool trigger_released = dragging.trigger_observed && + (!system->GetControllerState(dragging.device, &state, sizeof(state)) || !(state.ulButtonPressed & vr::ButtonMaskFromId(vr::k_EButton_SteamVR_Trigger))); // Without a readable release watchdog, never keep manipulating after // the pointer leaves our hit region: an outside MouseUp is not assured. - const bool lost_unwatched_pointer = !drag_trigger_observed && !overlay->IsHoverTargetOverlay(handle); - if (!surface.dragging(drag_cursor) || !shown || !focus || !source || trigger_released || lost_unwatched_pointer || - std::chrono::steady_clock::now() - drag_started > std::chrono::seconds(15)) { - surface.reset_pointers(); drag.reset(); return; + const bool lost_unwatched_pointer = !dragging.trigger_observed && !overlay->IsHoverTargetOverlay(handle); + if (!surface.dragging(dragging.cursor) || !shown || !focus || !source || trigger_released || lost_unwatched_pointer || + std::chrono::steady_clock::now() - dragging.started > std::chrono::seconds(15)) { + surface.reset_pointers(); dragging.panel.reset(); return; } - const auto change = drag.update(*source); - if (!change) { surface.reset_pointers(); drag.reset(); return; } - const float scale = drag_kind == PanelDragKind::Scale ? std::clamp(drag_scale * change->factor, .5f, 2.f) : size_scale; - const auto pose = drag_kind == PanelDragKind::Grab ? change->pose : - resized_mount_pose(drag_canvas, mount_width(*applied_mount, config.wrist) * drag_scale * W / CW, - scale / drag_scale, float(H) / W); + const auto change = dragging.panel.update(*source); + if (!change) { surface.reset_pointers(); dragging.panel.reset(); return; } + const float scale = dragging.kind == PanelDragKind::Scale ? std::clamp(dragging.scale * change->factor, .5f, 2.f) : size_scale; + const auto pose = dragging.kind == PanelDragKind::Grab ? change->pose : + resized_mount_pose(dragging.canvas, mount_width(*applied_mount, config.wrist) * dragging.scale * W / CW, + scale / dragging.scale, float(H) / W); if (scale != size_scale || pose != canvas_pose) { size_scale = scale; canvas_pose = pose; placement_dirty = true; } @@ -369,7 +378,7 @@ struct Overlay::Impl { gpu_texture->upload(surface.pixels()); auto texture = gpu_texture->texture(); overlay_check(overlay->SetOverlayTexture(handle, &texture), overlay, "SetOverlayTexture (Vulkan)"); - ++texture_uploads; + ++diagnostics.texture_uploads; has_texture = true; } visibility(); @@ -431,8 +440,8 @@ struct Overlay::Impl { vr::VREvent_t event{}; while (system->PollNextEvent(&event, sizeof(event))) { if (event.eventType == vr::VREvent_Quit) result.push_back(UiAction::Quit); - if (event.eventType == vr::VREvent_OverlayFocusChanged) ++global_focus_events; - if (event.eventType == vr::VREvent_InputFocusCaptured) ++input_focus_captured_events; + if (event.eventType == vr::VREvent_OverlayFocusChanged) ++diagnostics.global_focus_events; + if (event.eventType == vr::VREvent_InputFocusCaptured) ++diagnostics.input_focus_captured_events; if (event.eventType == vr::VREvent_SeatedZeroPoseReset || event.eventType == vr::VREvent_ChaperoneUniverseHasChanged) { world_ready = false; applied_mount.reset(); surface.reset_pointers(); } @@ -444,52 +453,57 @@ struct Overlay::Impl { case vr::VREvent_Quit: case vr::VREvent_OverlayClosed: result.push_back(UiAction::Quit); break; case vr::VREvent_OverlayHidden: - ++overlay_hidden_events; + ++diagnostics.overlay_hidden_events; focus = false; reset_input(result); if (panel.recording) result.push_back(UiAction::Cancel); break; case vr::VREvent_OverlayShown: - ++overlay_shown_events; + ++diagnostics.overlay_shown_events; focus = true; reset_input(result); break; - case vr::VREvent_ImageLoaded: ++image_loaded_events; break; - case vr::VREvent_ImageFailed: ++image_failed_events; break; + case vr::VREvent_ImageLoaded: ++diagnostics.image_loaded_events; break; + case vr::VREvent_ImageFailed: ++diagnostics.image_failed_events; break; case vr::VREvent_OverlayGamepadFocusLost: - surface.reset_pointers(); ++pointer_resets; - last_pointer_event = "gamepad focus lost"; break; + surface.reset_pointers(); ++diagnostics.pointer_resets; + diagnostics.last_pointer_event = "gamepad focus lost"; break; case vr::VREvent_OverlayFocusChanged: - ++overlay_focus_events; + ++diagnostics.overlay_focus_events; // This global focus notification also fires when the dashboard // laser enters our overlay. Do not erase a press between down/up; // a release must still hit the same enabled control. - last_pointer_event = "overlay focus changed"; break; + diagnostics.last_pointer_event = "overlay focus changed"; break; case vr::VREvent_MouseMove: // Manipulation uses the captured controller ray, not coordinates // fed back from a changing overlay or a batch of stale mouse hits. break; case vr::VREvent_MouseButtonDown: - ++pointer_downs; - last_pointer_event = "down button=" + std::to_string(event.data.mouse.button); + ++diagnostics.pointer_downs; + diagnostics.last_pointer_event = "down button=" + std::to_string(event.data.mouse.button); if (event.data.mouse.button == vr::VRMouseButton_Left) if (auto kind = surface.pointer_down(event.data.mouse.cursorIndex, event.data.mouse.x, H - event.data.mouse.y)) begin_drag(*kind, event); break; case vr::VREvent_MouseButtonUp: - ++pointer_ups; - last_pointer_event = "up button=" + std::to_string(event.data.mouse.button); + ++diagnostics.pointer_ups; + diagnostics.last_pointer_event = "up button=" + std::to_string(event.data.mouse.button); if (event.data.mouse.button == vr::VRMouseButton_Left) { auto event_result = surface.pointer_up(event.data.mouse.cursorIndex, event.data.mouse.x, H - event.data.mouse.y); - if (event_result.action || event_result.mount || event_result.recenter || event_result.lasers_anytime || event_result.open_bindings || event_result.advanced_debug || event_result.auto_insert || event_result.lock_layout || event_result.clock_24h || event_result.date_format) ++pointer_actions; + if (event_result.action || event_result.mount || event_result.recenter || event_result.lasers_anytime || event_result.open_bindings || event_result.advanced_debug || event_result.auto_insert || event_result.close_mic_when_idle || event_result.lock_layout || event_result.clock_24h || event_result.date_format || event_result.model_action) ++diagnostics.pointer_actions; if (event_result.action) result.push_back(*event_result.action); + if (event_result.model_action) { + model_actions.push_back(*event_result.model_action); + reset_input(result); // revoke held PTT, delivery and stale pointer approval + return result; + } if (event_result.clock_24h) { config.clock_24h = *event_result.clock_24h; - save_failed = persist_mount && !save_clock_24h(default_config_path(), config.clock_24h); + persistence.save_failed = persistence.persist_mount && !save_clock_24h(default_config_path(), config.clock_24h); surface.set_clock_24h(config.clock_24h); reset_input(result); return result; } if (event_result.date_format) { config.date_format = *event_result.date_format; - save_failed = persist_mount && !save_date_format(default_config_path(), config.date_format); + persistence.save_failed = persistence.persist_mount && !save_date_format(default_config_path(), config.date_format); surface.set_date_format(config.date_format); reset_input(result); return result; @@ -498,20 +512,28 @@ struct Overlay::Impl { config.lock_layout = *event_result.lock_layout; surface.set_layout_locked(config.lock_layout); update_intersection_mask(); - layout_save_failed = persist_mount && !save_lock_layout(default_config_path(), config.lock_layout); - reset_input(result); drag.reset(); + persistence.layout_save_failed = persistence.persist_mount && !save_lock_layout(default_config_path(), config.lock_layout); + reset_input(result); dragging.panel.reset(); + return result; + } + if (event_result.close_mic_when_idle) { + config.close_mic_when_idle = *event_result.close_mic_when_idle; + persistence.mic_save_failed = persistence.persist_mount && + !save_close_mic_when_idle(default_config_path(), config.close_mic_when_idle); + surface.set_close_mic_when_idle(config.close_mic_when_idle); + reset_input(result); // a setting change invalidates held actions return result; } if (event_result.auto_insert) { config.auto_insert = *event_result.auto_insert; - auto_save_failed = persist_mount && !save_auto_insert(default_config_path(), config.auto_insert); + persistence.auto_save_failed = persistence.persist_mount && !save_auto_insert(default_config_path(), config.auto_insert); surface.set_auto_insert(config.auto_insert); reset_input(result); // setting change invalidates held actions return result; } if (event_result.advanced_debug) { config.advanced_debug = *event_result.advanced_debug; - debug_save_failed = persist_mount && !save_advanced_debug(default_config_path(), config.advanced_debug); + persistence.debug_save_failed = persistence.persist_mount && !save_advanced_debug(default_config_path(), config.advanced_debug); surface.set_advanced_debug(config.advanced_debug); reset_input(result); // Runtime observes the change before handling this batch, @@ -532,16 +554,16 @@ struct Overlay::Impl { if (overlay->SetOverlayFlag(handle, vr::VROverlayFlags_MakeOverlaysInteractiveIfVisible, enabled) == vr::VROverlayError_None) { lasers_anytime = enabled; - laser_change_failed = false; + persistence.laser_change_failed = false; surface.set_lasers_anytime(enabled); - save_failed = persist_mount && !save_lasers_anytime(laser_settings_path, enabled); + persistence.save_failed = persistence.persist_mount && !save_lasers_anytime(persistence.laser_settings_path, enabled); } else { - laser_change_failed = true; + persistence.laser_change_failed = true; } } if (event_result.mount) { mount = *event_result.mount; - save_failed = persist_mount && !save_mount(settings_path, mount); + persistence.save_failed = persistence.persist_mount && !save_mount(persistence.settings_path, mount); world_ready = false; applied_mount.reset(); } @@ -553,7 +575,7 @@ struct Overlay::Impl { } update_drag(); place(); - if (drag.active()) { + if (dragging.panel.active()) { // A controller used to manipulate the panel must not also authorize // Record/Insert/Enter. Require neutral rearm after the drag ends. left.reset(); right.reset(); @@ -569,8 +591,8 @@ struct Overlay::Impl { // Keep requesting it across dashboard/laser states: those are what the // experiment compares. SteamVR's separate permission gate is never changed here. set.nPriority = action_priority(); - action_update_error = input->UpdateActionState(&set, sizeof(set), 1); - if (action_update_error != vr::VRInputError_None) { + diagnostics.action_update_error = input->UpdateActionState(&set, sizeof(set), 1); + if (diagnostics.action_update_error != vr::VRInputError_None) { reset_input(result); return result; } auto [la, ld] = digital(0); @@ -608,15 +630,21 @@ Overlay::Overlay(const std::string& assets, const std::string& font, std::option : impl_(std::make_unique(assets, font, mount, persist_mount)) {} Overlay::~Overlay() = default; std::vector Overlay::poll() { return impl_->poll(); } +std::vector Overlay::take_model_actions() { + auto result = std::move(impl_->model_actions); + impl_->model_actions.clear(); + return result; +} void Overlay::draw(const Panel& panel) { impl_->draw(panel); } bool Overlay::advanced_debug() const { return impl_->config.advanced_debug; } bool Overlay::auto_insert() const { return impl_->config.auto_insert; } +bool Overlay::close_mic_when_idle() const { return impl_->config.close_mic_when_idle; } const std::vector& Overlay::quick_inputs() const { return impl_->config.quick_inputs; } std::string Overlay::controls_status() { // Compare the same actions across modes, before and after our pose/role gate. // IsInputAvailable and a successful UpdateActionState are not delivery proof. std::string result = impl_->input_mode_status() + - " update=" + std::to_string(int(impl_->action_update_error)); + " update=" + std::to_string(int(impl_->diagnostics.action_update_error)); for (size_t i = 0; i < action_names.size(); ++i) { vr::InputDigitalActionData_t data{}; const auto error = impl_->input->GetDigitalActionData(impl_->actions[i], &data, sizeof(data), vr::k_ulInvalidInputValueHandle); @@ -629,18 +657,18 @@ std::string Overlay::controls_status() { return result; } std::string Overlay::pointer_status() const { - return "Pointer down=" + std::to_string(impl_->pointer_downs) + " up=" + std::to_string(impl_->pointer_ups) + - " hits=" + std::to_string(impl_->pointer_actions) + " resets=" + std::to_string(impl_->pointer_resets) + - " last=" + impl_->last_pointer_event + - "\nOverlay renderer=Vulkan textureUploads=" + std::to_string(impl_->texture_uploads) + - " showCalls=" + std::to_string(impl_->show_calls) + " hideCalls=" + std::to_string(impl_->hide_calls) + - " shownEvents=" + std::to_string(impl_->overlay_shown_events) + - " hiddenEvents=" + std::to_string(impl_->overlay_hidden_events) + - " imageLoaded=" + std::to_string(impl_->image_loaded_events) + - " imageFailed=" + std::to_string(impl_->image_failed_events) + - " overlayFocus=" + std::to_string(impl_->overlay_focus_events) + - " globalFocus=" + std::to_string(impl_->global_focus_events) + - " inputCaptured=" + std::to_string(impl_->input_focus_captured_events); + return "Pointer down=" + std::to_string(impl_->diagnostics.pointer_downs) + " up=" + std::to_string(impl_->diagnostics.pointer_ups) + + " hits=" + std::to_string(impl_->diagnostics.pointer_actions) + " resets=" + std::to_string(impl_->diagnostics.pointer_resets) + + " last=" + impl_->diagnostics.last_pointer_event + + "\nOverlay renderer=Vulkan textureUploads=" + std::to_string(impl_->diagnostics.texture_uploads) + + " showCalls=" + std::to_string(impl_->diagnostics.show_calls) + " hideCalls=" + std::to_string(impl_->diagnostics.hide_calls) + + " shownEvents=" + std::to_string(impl_->diagnostics.overlay_shown_events) + + " hiddenEvents=" + std::to_string(impl_->diagnostics.overlay_hidden_events) + + " imageLoaded=" + std::to_string(impl_->diagnostics.image_loaded_events) + + " imageFailed=" + std::to_string(impl_->diagnostics.image_failed_events) + + " overlayFocus=" + std::to_string(impl_->diagnostics.overlay_focus_events) + + " globalFocus=" + std::to_string(impl_->diagnostics.global_focus_events) + + " inputCaptured=" + std::to_string(impl_->diagnostics.input_focus_captured_events); } int registration(const std::string& manifest, bool remove, bool autostart) { diff --git a/src/overlay.hpp b/src/overlay.hpp index b017e11..e95d289 100644 --- a/src/overlay.hpp +++ b/src/overlay.hpp @@ -1,6 +1,7 @@ #pragma once #include "mount.hpp" #include +#include #include #include #include @@ -8,6 +9,14 @@ namespace frameyap { enum class UiAction { Toggle, Record, BeginRecord, EndRecord, Cancel, Insert, Enter, QuickChat, Quit }; +struct ModelOption { + std::string id, name, state, source, license, license_text, attribution; + uint64_t bytes = 0; + bool verified = false; + std::string manifest_sha256; + bool operator==(const ModelOption&) const = default; +}; +struct ModelAction { std::string id; bool install = false; std::string manifest_sha256; }; struct Panel { Panel() = default; Panel(std::string status, std::string transcript, std::string detail, @@ -23,6 +32,9 @@ struct Panel { bool quick_open = false; size_t quick_selected = 0; std::vector quick_inputs; + std::vector models; + std::string selected_backend, model_note; + bool model_busy = false; }; class Overlay { public: @@ -32,9 +44,11 @@ public: Overlay(const Overlay&) = delete; Overlay& operator=(const Overlay&) = delete; std::vector poll(); + std::vector take_model_actions(); void draw(const Panel& panel); bool advanced_debug() const; bool auto_insert() const; + bool close_mic_when_idle() const; const std::vector& quick_inputs() const; std::string controls_status(); // diagnostic only, no input delivery std::string pointer_status() const; // diagnostic counters, no input delivery diff --git a/src/panel_surface.cpp b/src/panel_surface.cpp index 7719def..611519e 100644 --- a/src/panel_surface.cpp +++ b/src/panel_surface.cpp @@ -30,10 +30,11 @@ struct Rect { }; enum class Control { Review, Settings, Bindings, Prev, Next, Record, Cancel, Insert, Enter, Quit, World, Left, Right, Head, Recenter, LasersAnytime, AdvancedDebug, AutoInsert, - Clock24h, Date, LockLayout }; -enum class Tab { Review, Settings }; + Clock24h, Date, LockLayout, CloseMicWhenIdle, Models, ModelRow, + ModelInstall, ModelPrev, ModelNext }; +enum class Tab { Review, Settings, Models }; struct Button { Rect r; Control id; const char* label; }; -constexpr std::array buttons{{ +constexpr std::array buttons{{ {{32, 138, 180, 46}, Control::Review, "Review"}, {{226, 138, 180, 46}, Control::Settings, "Settings"}, {{420, 138, 180, 46}, Control::Bindings, "Bindings"}, @@ -42,19 +43,30 @@ constexpr std::array buttons{{ {{838, 406, 130, 44}, Control::Next, "Next"}, {{32, 574, 176, 68}, Control::Record, "Record"}, {{222, 574, 176, 68}, Control::Cancel, "Cancel"}, - {{412, 574, 176, 68}, Control::Insert, "Insert"}, - {{602, 574, 176, 68}, Control::Enter, "Submit"}, + {{412, 574, 176, 68}, Control::Insert, "Type"}, + {{602, 574, 176, 68}, Control::Enter, "Type + Enter"}, {{792, 574, 176, 68}, Control::Quit, "Hold Quit"}, {{32, 234, 454, 58}, Control::World, "World space"}, {{514, 234, 454, 58}, Control::Head, "Head"}, {{32, 308, 454, 58}, Control::Left, "Left wrist"}, {{514, 308, 454, 58}, Control::Right, "Right wrist"}, - {{32, 394, 300, 50}, Control::Recenter, "Recenter in front"}, - {{514, 394, 454, 50}, Control::LasersAnytime, "Lasers anytime"}, - {{32, 452, 454, 48}, Control::AutoInsert, "Auto insert"}, - {{514, 452, 454, 48}, Control::AdvancedDebug, "Advanced debug"}, - {{32, 506, 454, 42}, Control::Clock24h, "Clock"}, - {{514, 506, 454, 42}, Control::Date, "Date"}, + {{32, 378, 300, 44}, Control::Recenter, "Recenter in front"}, + {{514, 378, 454, 44}, Control::LasersAnytime, "Lasers anytime"}, + {{32, 428, 454, 40}, Control::AutoInsert, "Auto insert"}, + {{514, 428, 454, 40}, Control::AdvancedDebug, "Advanced debug"}, + {{32, 474, 454, 36}, Control::Clock24h, "Clock"}, + {{514, 474, 454, 36}, Control::Date, "Date"}, + {{32, 516, 454, 32}, Control::CloseMicWhenIdle, "Close mic when idle"}, + {{514, 516, 454, 32}, Control::Models, "Models / backends"}, + {{32, 242, 936, 42}, Control::ModelRow, ""}, + {{32, 290, 936, 42}, Control::ModelRow, ""}, + {{32, 338, 936, 42}, Control::ModelRow, ""}, + {{32, 386, 936, 42}, Control::ModelRow, ""}, + {{32, 434, 936, 42}, Control::ModelRow, ""}, + {{32, 482, 936, 42}, Control::ModelRow, ""}, + {{514, 530, 454, 36}, Control::ModelInstall, "Install"}, + {{32, 530, 214, 36}, Control::ModelPrev, "Previous"}, + {{260, 530, 214, 36}, Control::ModelNext, "Next"}, }}; std::optional action(Control c) { switch (c) { @@ -105,6 +117,7 @@ struct PanelSurface::Impl { Color background, card, ink, muted, cyan, pink; Tab tab = Tab::Review; bool dirty = true, lasers_anytime = false, advanced_debug = false, auto_insert = false; + bool close_mic_when_idle = false; bool clock_24h = false, layout_locked = false; DateFormat date_format = DateFormat::MonthDayYear; std::time_t clock_time = std::time(nullptr); @@ -116,7 +129,41 @@ struct PanelSurface::Impl { int drag_cursor = -1; std::vector lines; size_t page = 0; + size_t model_page = 0; + size_t consent_page = 0; + size_t consent_rendered_page = size_t(-1); + bool install_confirm = false; + std::optional consent_snapshot; + std::vector consent_lines; static constexpr size_t lines_per_page = 4; + static constexpr size_t consent_lines_per_page = 6; + size_t consent_pages() const { + return std::max(size_t(1), (consent_lines.size() + consent_lines_per_page - 1) / consent_lines_per_page); + } + std::vector visible_consent_lines() const { + if (!install_confirm || !consent_snapshot || consent_page != consent_rendered_page) return {}; + auto first = consent_page * consent_lines_per_page; + auto last = std::min(consent_lines.size(), first + consent_lines_per_page); + return {consent_lines.begin() + first, consent_lines.begin() + last}; + } + void prepare_consent(const ModelOption& m) { + consent_snapshot = m; + consent_lines.clear(); + // Each field is independently wrapped; no hidden tail of the source, + // license or attribution may be mistaken for reviewed consent. + for (const auto& field : {"Backend: " + m.name + " (" + m.id + ")", + "Download bytes: " + std::to_string(m.bytes), + "Source: " + m.source, "License: " + m.license, + "License text: " + m.license_text, + "Attribution: " + m.attribution, + "Manifest SHA-256: " + m.manifest_sha256}) { + auto rows = wrap(field, 19, 890); + consent_lines.insert(consent_lines.end(), rows.begin(), rows.end()); + } + consent_page = 0; + consent_rendered_page = size_t(-1); + install_confirm = true; + } Impl(const std::string& font, Mount m, Theme t) : mount(m), theme(t), background(t.background), card(t.card), ink(t.ink), muted(t.muted), @@ -247,19 +294,24 @@ struct PanelSurface::Impl { } size_t page_count() const { return std::max(size_t(1), (lines.size() + lines_per_page - 1) / lines_per_page); } bool available(UiAction a) const { - if (a == UiAction::Record) return !panel.quick_open && (panel.recording || panel.record_available); - if (a == UiAction::Insert) return !panel.quick_open && panel.enabled && !panel.recording && !panel.transcript.empty(); - if (a == UiAction::Enter) return panel.enabled && !panel.recording; - if (a == UiAction::QuickChat) return panel.enabled && !panel.recording && !panel.quick_inputs.empty(); + if (a == UiAction::Record) return tab != Tab::Models && !panel.quick_open && (panel.recording || panel.record_available); + if (a == UiAction::Insert) return tab != Tab::Models && !panel.quick_open && panel.enabled && !panel.recording && !panel.transcript.empty(); + if (a == UiAction::Enter) return tab != Tab::Models && panel.enabled && !panel.recording; + if (a == UiAction::QuickChat) return tab != Tab::Models && panel.enabled && !panel.recording && !panel.quick_inputs.empty(); if (a == UiAction::Toggle) return false; return true; } bool visible(Control c) const { if (panel.quick_open && c != Control::Cancel && c != Control::Enter && c != Control::Quit) return false; + if (c == Control::Models) return tab == Tab::Settings; + if (c == Control::ModelRow) return tab == Tab::Models && !install_confirm; + if (c == Control::ModelPrev || c == Control::ModelNext) return tab == Tab::Models; + if (c == Control::ModelInstall) return tab == Tab::Models; if (c == Control::Prev || c == Control::Next) return tab == Tab::Review && !panel.quick_open; if (mounting(c) || c == Control::Recenter || c == Control::LasersAnytime || c == Control::AdvancedDebug || c == Control::AutoInsert || - c == Control::Clock24h || c == Control::Date || c == Control::LockLayout) return tab == Tab::Settings; + c == Control::Clock24h || c == Control::Date || c == Control::LockLayout || + c == Control::CloseMicWhenIdle) return tab == Tab::Settings; return true; } bool enabled(Control c) const { @@ -267,11 +319,25 @@ struct PanelSurface::Impl { if (c == Control::Prev) return page > 0; if (c == Control::Next) return page + 1 < page_count(); if (c == Control::Recenter) return mount == Mount::World; + if (c == Control::ModelRow) return !panel.model_busy; + if (c == Control::ModelInstall) { + if (panel.model_busy) return false; + auto it = std::find_if(panel.models.begin(), panel.models.end(), [&](const auto& m) { return m.id == panel.selected_backend; }); + return it != panel.models.end() && !it->verified && + (!install_confirm || (consent_snapshot == *it && consent_page == consent_rendered_page && + consent_page + 1 == consent_pages())); + } + if (c == Control::ModelPrev) return install_confirm ? consent_page > 0 : model_page > 0; + if (c == Control::ModelNext) return install_confirm ? + consent_page == consent_rendered_page && consent_page + 1 < consent_pages() : + (model_page + 1) * 6 < panel.models.size(); return true; } int hit(float x, float y) const { - for (size_t i = 0; i < buttons.size(); ++i) + for (size_t i = 0; i < buttons.size(); ++i) { + if (buttons[i].id == Control::ModelRow && model_page * 6 + (i - 23) >= panel.models.size()) continue; if (visible(buttons[i].id) && enabled(buttons[i].id) && buttons[i].r.contains(x, y)) return int(i); + } return -1; } void reset() { @@ -285,13 +351,18 @@ struct PanelSurface::Impl { dirty = true; } if (p.transcript != panel.transcript || lines.empty()) { - lines = wrap(p.transcript.empty() ? "Your words will appear here.\nReview them, then choose Insert." : p.transcript, 32, 904); + lines = wrap(p.transcript.empty() ? "Your words will appear here.\nReview them, then choose Type." : p.transcript, 32, 904); page = 0; dirty = true; } if (p.status != panel.status || p.enabled != panel.enabled || p.recording != panel.recording || - p.quick_open != panel.quick_open || p.quick_selected != panel.quick_selected || p.quick_inputs != panel.quick_inputs) + p.quick_open != panel.quick_open || p.quick_selected != panel.quick_selected || p.quick_inputs != panel.quick_inputs || + p.models != panel.models || p.selected_backend != panel.selected_backend || + p.model_note != panel.model_note || p.model_busy != panel.model_busy) dirty = true; - if (p.quick_open != panel.quick_open) reset(); + if (p.quick_open != panel.quick_open || p.models != panel.models || + p.selected_backend != panel.selected_backend || p.model_busy != panel.model_busy) { + reset(); install_confirm = false; consent_snapshot.reset(); + } panel = p; if (panel.quick_open && tab != Tab::Review) { tab = Tab::Review; dirty = true; } int progress = 0; @@ -320,7 +391,7 @@ struct PanelSurface::Impl { if (status.size() > 1) text("[...]", 864, 109, 23, muted, 968); if (tab == Tab::Review && panel.quick_open) { rounded({32, 212, 936, 324}, 16, mix(card, muted, .08f), mix(card, cyan, .25f), .18f); - text("QUICK CHAT Y: NEXT SUBMIT: SEND CANCEL: CLOSE", 48, 243, 22, cyan, 952); + text("QUICK PHRASES Y: NEXT TYPE + ENTER: CHOICE + ENTER CANCEL: CLOSE", 48, 243, 21, cyan, 952); for (size_t i = 0; i < panel.quick_inputs.size() && i < 6; ++i) { const Rect row{48, 254 + int(i) * 45, 904, 40}; const bool selected = i == panel.quick_selected; @@ -334,23 +405,45 @@ struct PanelSurface::Impl { text(lines[page * lines_per_page + i], 48, 246 + int(i) * 40, 32, panel.transcript.empty() ? muted : ink, 952); text("PAGE " + std::to_string(page + 1) + " / " + std::to_string(page_count()), 416, 436, 23, muted, 790); - if (!binding_note.empty()) text(binding_note, 32, 203, 20, muted, 968); + text(binding_note.empty() ? "Type adds a space; Type + Enter explicitly submits." : binding_note, + 32, 203, 20, muted, 968); + } else if (tab == Tab::Models) { + text("Models: select to restart. Install always needs explicit confirmation.", 32, 202, 19, muted, 968); + const auto it = std::find_if(panel.models.begin(), panel.models.end(), [&](const auto& m) { return m.id == panel.selected_backend; }); + if (it != panel.models.end()) { + if (install_confirm && consent_snapshot == *it) { + text("Review every page before Confirm Install. Settings cancels.", 32, 225, 19, pink, 968); + // Only a painted page can advance review or authorize the + // install: multiple pointer events between draws cannot skip it. + consent_rendered_page = consent_page; + const auto rows = visible_consent_lines(); + for (size_t i = 0; i < rows.size(); ++i) + text(rows[i], 48, 271 + int(i) * 34, 19, ink, 950); + text("METADATA PAGE " + std::to_string(consent_page + 1) + " / " + std::to_string(consent_pages()), + 32, 505, 19, cyan, 968); + } else { + text("Select Install to review full source, size and license before download.", 32, 225, 18, cyan, 968); + } + } else text("No model selected.", 32, 225, 18, pink, 968); } else { - text("Auto insert needs stable X focus; debug logs may contain speech.", 32, 204, 18, pink, 968); - text(placement_note.empty() ? "No automatic Enter; changing debug restarts the worker." : placement_note, - 32, 225, 18, muted, 968); + text("Hold Quit: hold 0.9s then release. Lasers anytime: system-wide; may affect games.", + 32, 204, 18, pink, 968); + text(placement_note.empty() ? "Auto insert needs stable X focus; debug logs may contain speech." : placement_note, + 32, 225, 17, muted, 968); } rect({32, 550, 936, 1}, mix(card, cyan, .17f)); for (size_t i = 0; i < buttons.size(); ++i) { const auto& b = buttons[i]; if (!visible(b.id)) continue; bool on = enabled(b.id); + if (b.id == Control::ModelRow && model_page * 6 + (i - 23) >= panel.models.size()) continue; bool selected = (b.id == Control::Review && tab == Tab::Review) || (b.id == Control::Settings && tab == Tab::Settings) || (mounting(b.id) && *mounting(b.id) == mount) || (b.id == Control::LasersAnytime && lasers_anytime) || (b.id == Control::AdvancedDebug && advanced_debug) || (b.id == Control::AutoInsert && auto_insert) || + (b.id == Control::CloseMicWhenIdle && close_mic_when_idle) || (b.id == Control::LockLayout && layout_locked); const Color fill = !on ? mix(background, card, .40f) : selected ? mix(card, cyan, .14f) : card; @@ -366,23 +459,41 @@ struct PanelSurface::Impl { PanelSurface::Clock::now() - press_time[cursor]).count()) / PanelSurface::quit_hold.count(), 0.f, 1.f); rect({b.r.x + 7, b.r.y + b.r.h - 9, int((b.r.w - 14) * fraction), 3}, pink); } - const std::string label = b.id == Control::Record && panel.recording ? "Stop" : + const std::string label = b.id == Control::ModelRow ? [&]() { + const auto& m = panel.models[model_page * 6 + i - 23]; + return (m.id == panel.selected_backend ? "[ACTIVE] " : "") + m.name + " - " + + (m.id == panel.selected_backend && panel.model_busy ? "checking" : m.state); + }() : b.id == Control::ModelInstall && install_confirm ? "Confirm Install" : + b.id == Control::Record && panel.recording ? "Stop" : b.id == Control::Clock24h ? (clock_24h ? "Clock: 24 hour" : "Clock: 12 hour") : b.id == Control::Date ? (date_format == DateFormat::Off ? "Date: Off" : date_format == DateFormat::MonthDayYear ? "Date: MM/DD/YYYY" : date_format == DateFormat::DayMonthYear ? "Date: DD/MM/YYYY" : "Date: YYYY-MM-DD") : b.label; - text(label, b.r.x + 16, b.r.y + b.r.h / 2 + 9, b.id == Control::Date ? 23 : 27, + text(label, b.r.x + 16, b.r.y + b.r.h / 2 + 9, + b.id == Control::Date ? 23 : b.id == Control::CloseMicWhenIdle ? 22 : + b.id == Control::Enter ? 20 : 27, on ? ink : mix(background, muted, .48f), b.r.x + b.r.w - 8); if (mounting(b.id) && selected) text("ON", b.r.x + b.r.w - 56, b.r.y + 38, 23, cyan, b.r.x + b.r.w - 12); if (b.id == Control::LasersAnytime || b.id == Control::AdvancedDebug || b.id == Control::AutoInsert || - b.id == Control::LockLayout) { + b.id == Control::LockLayout || b.id == Control::CloseMicWhenIdle) { bool active = b.id == Control::LasersAnytime ? lasers_anytime : b.id == Control::AutoInsert ? auto_insert : - b.id == Control::LockLayout ? layout_locked : advanced_debug; + b.id == Control::LockLayout ? layout_locked : + b.id == Control::CloseMicWhenIdle ? close_mic_when_idle : advanced_debug; text(active ? "ON" : "OFF", b.r.x + b.r.w - 66, b.r.y + b.r.h / 2 + 9, 22, active ? cyan : muted, b.r.x + b.r.w - 12); } } + if (tab == Tab::Settings) + text("OFF: discard idle audio; ON: spike / start latency.", 390, 565, 16, muted, 887); + if (tab == Tab::Models && !panel.model_note.empty()) { + // The install/navigation controls occupy y=530..566; the status + // belongs BELOW the shared footer (574..642), not under buttons. + auto rows = wrap(panel.model_note, 15, 904); + for (size_t i = 0; i < rows.size() && i < 2; ++i) + text(rows[i], 32, 658 + int(i) * 18, 15, pink, 968); + if (rows.size() > 2) text("[note shortened]", 805, 677, 13, pink, 968); + } // Screenshot-inspired grab underline and outside corner bracket. No // opaque toolbar backing; broad hit targets surround the slender strokes. if (!layout_locked) { @@ -411,6 +522,9 @@ PanelSurface::~PanelSurface() = default; bool PanelSurface::render(const Panel& p) { return impl_->render(p); } const std::vector& PanelSurface::pixels() const { return impl_->pixels; } bool PanelSurface::available(UiAction a) const { return impl_->available(a); } +std::vector PanelSurface::visible_model_review_lines() const { + return impl_->visible_consent_lines(); +} bool PanelSurface::dragging(unsigned cursor) const { return cursor < impl_->pressed.size() && int(cursor) == impl_->drag_cursor; } @@ -446,13 +560,46 @@ SurfaceEvent PanelSurface::pointer_up(unsigned cursor, float x, float y, Clock:: else if (c == Control::LasersAnytime) result.lasers_anytime = !impl_->lasers_anytime; else if (c == Control::AdvancedDebug) result.advanced_debug = !impl_->advanced_debug; else if (c == Control::AutoInsert) result.auto_insert = !impl_->auto_insert; + else if (c == Control::CloseMicWhenIdle) result.close_mic_when_idle = !impl_->close_mic_when_idle; else if (c == Control::LockLayout) result.lock_layout = !impl_->layout_locked; else if (c == Control::Clock24h) result.clock_24h = !impl_->clock_24h; else if (c == Control::Date) result.date_format = static_cast((static_cast(impl_->date_format) + 1) % 4); else if (c == Control::Bindings) { result.open_bindings = true; impl_->reset(); } + else if (c == Control::Models) { + impl_->tab = Tab::Models; impl_->model_page = 0; + impl_->install_confirm = false; impl_->consent_snapshot.reset(); impl_->reset(); impl_->dirty = true; + } + else if (c == Control::ModelRow) { + const auto& model = impl_->panel.models[impl_->model_page * 6 + index - 23]; + result.model_action = ModelAction{model.id, false, {}}; + impl_->install_confirm = false; impl_->consent_snapshot.reset(); impl_->reset(); impl_->dirty = true; + } + else if (c == Control::ModelInstall) { + auto it = std::find_if(impl_->panel.models.begin(), impl_->panel.models.end(), [&](const auto& m) { + return m.id == impl_->panel.selected_backend; + }); + if (it != impl_->panel.models.end() && !it->verified && !impl_->panel.model_busy && + it->manifest_sha256.size() == 64) { + if (impl_->install_confirm && impl_->consent_snapshot == *it) + result.model_action = ModelAction{it->id, true, impl_->consent_snapshot->manifest_sha256}; + else if (!impl_->install_confirm) impl_->prepare_consent(*it); + } + if (result.model_action) { impl_->install_confirm = false; impl_->consent_snapshot.reset(); } + impl_->reset(); impl_->dirty = true; + } + else if (c == Control::ModelPrev) { + if (impl_->install_confirm) --impl_->consent_page; + else --impl_->model_page; + impl_->reset(); impl_->dirty = true; + } + else if (c == Control::ModelNext) { + if (impl_->install_confirm) ++impl_->consent_page; + else ++impl_->model_page; + impl_->reset(); impl_->dirty = true; + } else if (c == Control::Review || c == Control::Settings) { impl_->tab = c == Control::Review ? Tab::Review : Tab::Settings; - impl_->reset(); impl_->dirty = true; + impl_->install_confirm = false; impl_->consent_snapshot.reset(); impl_->reset(); impl_->dirty = true; } else if (c == Control::Prev) { --impl_->page; impl_->dirty = true; } else if (c == Control::Next) { ++impl_->page; impl_->dirty = true; } @@ -496,6 +643,13 @@ void PanelSurface::set_auto_insert(bool enabled) { impl_->dirty = true; } } +void PanelSurface::set_close_mic_when_idle(bool enabled) { + if (impl_->close_mic_when_idle != enabled) { + impl_->close_mic_when_idle = enabled; + impl_->reset(); + impl_->dirty = true; + } +} void PanelSurface::set_clock_24h(bool enabled) { if (impl_->clock_24h != enabled) { impl_->clock_24h = enabled; impl_->reset(); impl_->dirty = true; diff --git a/src/panel_surface.hpp b/src/panel_surface.hpp index 5ce1c6b..df4178a 100644 --- a/src/panel_surface.hpp +++ b/src/panel_surface.hpp @@ -16,11 +16,13 @@ struct SurfaceEvent { std::optional lasers_anytime; std::optional advanced_debug; std::optional auto_insert; + std::optional close_mic_when_idle; std::optional lock_layout; std::optional clock_24h; std::optional date_format; bool recenter = false; bool open_bindings = false; + std::optional model_action; }; // One CPU RGBA canvas, independent of OpenVR. Settings replace the review area; // status and safety controls remain on the same surface. @@ -51,11 +53,15 @@ public: void set_lasers_anytime(bool enabled); void set_advanced_debug(bool enabled); void set_auto_insert(bool enabled); + void set_close_mic_when_idle(bool enabled); void set_clock_24h(bool enabled); void set_date_format(DateFormat format); void set_clock_time(std::time_t now); void set_binding_note(std::string note); bool available(UiAction action) const; + // Exact text rows currently shown in the consent viewport (empty outside it). + // OpenVR has no screen-reader accessibility channel for this canvas. + std::vector visible_model_review_lines() const; private: struct Impl; std::unique_ptr impl_; diff --git a/tests/backend_manager_test.cpp b/tests/backend_manager_test.cpp new file mode 100644 index 0000000..d9d364a --- /dev/null +++ b/tests/backend_manager_test.cpp @@ -0,0 +1,153 @@ +#include "backend_manager.hpp" +#include +#include +#include +#include +#include +#include +#include + +using namespace frameyap; +namespace { +const std::string digest(64, 'a'); +void put(const std::filesystem::path& file, const std::string& contents) { + std::ofstream out(file); out << contents; +} +void until_idle(BackendManager& manager) { + for (int n = 0; n < 500 && manager.busy(); ++n) { + manager.poll(); std::this_thread::sleep_for(std::chrono::milliseconds(2)); + } + assert(!manager.busy()); +} +std::string status() { + return "print('ST\\tfake\\tFake%20backend\\tnot_installed\\tmissing_files\\t7\\thttps%3A%2F%2Fexample.org\\tMIT\\tTest%20license\\tFixture\\t/tmp/fake\\t" + digest + "', flush=True)\n" + " print('DONE', flush=True)\n"; +} +} +int main() { + auto path = std::filesystem::temp_directory_path() / ("frameyap-backend-test-" + std::to_string(getpid())); + std::filesystem::create_directory(path); + auto service = path / "service.py"; + put(service, "import sys\nif '--status' in sys.argv:\n " + status() + + "else:\n assert sys.argv[sys.argv.index('--expected-manifest-sha256') + 1] == '" + digest + "'\n" + " print('{\"ok\": true, \"event\": \"complete\"}', flush=True)\n"); + { + BackendManager manager("python3", service.string(), path.string(), path.string(), service.string()); + assert(!manager.busy()); + manager.refresh(); assert(manager.busy()); + until_idle(manager); + assert(manager.entries().size() == 1 && manager.entries()[0].id == "fake"); + assert(manager.entries()[0].bytes == 7 && manager.entries()[0].name == "Fake backend"); + assert(manager.entries()[0].state == "not_installed" && manager.checked()); + assert(manager.entries()[0].manifest_sha256 == digest); + assert(manager.revision() == 1 && manager.model_path("fake") == "/tmp/fake"); + try { manager.install("unknown", digest); assert(false); } catch (const std::runtime_error&) {} + try { manager.install("fake", std::string(64, 'b')); assert(false); } catch (const std::runtime_error&) {} + assert(!manager.busy()); // mismatched consent cannot launch a helper + manager.install("fake", digest); + assert(manager.installing()); + until_idle(manager); // success must trigger a fresh status check + assert(manager.revision() == 2 && manager.checked()); + manager.refresh(); manager.cancel(); assert(!manager.busy()); + } + // Installer output larger than one poll budget may remain after helper exit. + // Success still requires EOF followed by a fresh offline status check. + put(service, "import os, sys\nif '--status' in sys.argv:\n " + status() + + "else:\n assert os.get_blocking(1), 'child stdout must be blocking'\n" + " for _ in range(3000): print('{\"event\":\"complete\",\"ok\":true}')\n"); + { + BackendManager manager("python3", service.string(), path.string(), path.string(), service.string()); + manager.refresh(); until_idle(manager); assert(manager.checked()); + manager.install("fake", digest); until_idle(manager); + assert(manager.checked() && manager.revision() == 2); + } + // A fast-exiting helper can leave output behind. Read through EOF, then + // reject any trailing record after DONE, never accept a success substring. + put(service, "import sys\nif True:\n " + status() + "print('trailing', flush=True)\n"); + { + BackendManager manager("python3", service.string(), path.string(), path.string(), service.string()); + manager.refresh(); until_idle(manager); + assert(!manager.checked() && manager.note() == "Local model check failed."); + } + put(service, "import sys\nif '--status' in sys.argv:\n " + status() + + "else:\n print('{\"event\":\"complete\",\"ok\":true}')\n raise SystemExit(1)\n"); + { + BackendManager manager("python3", service.string(), path.string(), path.string(), service.string()); + manager.refresh(); until_idle(manager); assert(manager.checked()); + manager.install("fake", digest); until_idle(manager); + assert(manager.checked() && manager.revision() == 3); // failure plus offline recheck + assert(manager.note() == "Install failed: installer exited unsuccessfully; local model status checked offline."); + manager.refresh(); until_idle(manager); + assert(manager.note() == "Install failed: installer exited unsuccessfully; local model status checked offline."); + } + // JSON is display-only. Per-file progress appears before process exit and + // a structured error survives rechecks without echoing terminal controls. + put(service, "import sys, time\nif '--status' in sys.argv:\n " + status() + + "else:\n print('{\"ok\":true,\"event\":\"model_file\",\"file\":\"weights.dat\",\"state\":\"downloading\",\"bytes\":7}', flush=True)\n" + " time.sleep(.2)\n" + " print('{\"ok\":true,\"event\":\"model_file\",\"file\":\"weights.dat\",\"state\":\"verified\"}', flush=True)\n" + " time.sleep(.2)\n" + " print('{\"ok\":false,\"code\":\"manifest_mismatch\",\"message\":\"metadata changed \\\\u001b[31m\"}', flush=True)\n" + " raise SystemExit(1)\n"); + { + BackendManager manager("python3", service.string(), path.string(), path.string(), service.string()); + manager.refresh(); until_idle(manager); manager.install("fake", digest); + bool downloading = false, verified = false; + for (int n = 0; n < 500 && manager.busy(); ++n) { + manager.poll(); + downloading |= manager.note() == "Downloading: weights.dat"; + verified |= manager.note() == "Verified file: weights.dat"; + std::this_thread::sleep_for(std::chrono::milliseconds(2)); + } + assert(downloading && verified && manager.checked()); + assert(manager.note().find("manifest_mismatch: metadata changed ?[31m") != std::string::npos); + manager.refresh(); + assert(manager.note().find("metadata changed") != std::string::npos); // during recheck + until_idle(manager); + assert(manager.note().find("metadata changed") != std::string::npos); + } + // A lying success event, unknown event, and malformed object never certify + // installation: exit status and the fresh offline check still decide. + put(service, "import sys\nif '--status' in sys.argv:\n " + status() + + "else:\n print('{\"ok\":true,\"event\":\"complete\"}', flush=True)\n" + " print('{\"ok\":true,\"event\":\"model_file\",\"state\":\"unknown\",\"file\":\"fake\"}', flush=True)\n" + " print('{\"ok\":true,\"event\":\"model_file\",\"file\":{}}', flush=True)\n" + " raise SystemExit(1)\n"); + { + BackendManager manager("python3", service.string(), path.string(), path.string(), service.string()); + manager.refresh(); until_idle(manager); manager.install("fake", digest); until_idle(manager); + assert(manager.checked() && manager.note().find("Install failed:") == 0); + } + // A truncated object cannot inject even advisory error text. Complete lines + // are bounded too, not only a partial line waiting for its newline. + for (bool oversized : {false, true}) { + put(service, "import sys\nif '--status' in sys.argv:\n " + status() + + "else:\n " + (oversized ? std::string("print('x' * 9000, flush=True)\n") : + std::string("print('{\"ok\":false,\"message\":\"not-real\",', flush=True)\n")) + + " raise SystemExit(1)\n"); + BackendManager manager("python3", service.string(), path.string(), path.string(), service.string()); + manager.refresh(); until_idle(manager); manager.install("fake", digest); until_idle(manager); + assert(manager.note().find("not-real") == std::string::npos); + if (oversized) assert(!manager.checked() && manager.note().find("oversized") != std::string::npos); + else assert(manager.checked() && manager.note().find("installer exited unsuccessfully") != std::string::npos); + } + // Cancellation gives the exact owned child a bounded SIGTERM cleanup window. + auto started = path / "started", terminated = path / "terminated"; + put(service, "import pathlib, signal, sys, time\nif '--status' in sys.argv:\n " + status() + + "else:\n" + " def stop(*_):\n pathlib.Path('" + terminated.string() + "').write_text('done')\n raise SystemExit(1)\n" + " signal.signal(signal.SIGTERM, stop)\n" + " pathlib.Path('" + started.string() + "').write_text('ready')\n" + " time.sleep(60)\n"); + { + BackendManager manager("python3", service.string(), path.string(), path.string(), service.string()); + manager.refresh(); until_idle(manager); assert(manager.checked()); + manager.install("fake", digest); + for (int n = 0; n < 500 && !std::filesystem::exists(started); ++n) + std::this_thread::sleep_for(std::chrono::milliseconds(2)); + assert(std::filesystem::exists(started)); + manager.cancel(); + assert(!manager.busy() && std::filesystem::exists(terminated)); + } + std::filesystem::remove_all(path); +} diff --git a/tests/panel_test.cpp b/tests/panel_test.cpp index 64b8b42..65e8852 100644 --- a/tests/panel_test.cpp +++ b/tests/panel_test.cpp @@ -14,7 +14,7 @@ SurfaceEvent click(PanelSurface& surface, float x, float y, unsigned cursor = 0) } void no_action(const SurfaceEvent& event) { assert(!event.action && !event.mount && !event.lasers_anytime && !event.advanced_debug && !event.auto_insert && - !event.lock_layout && !event.clock_24h && !event.date_format && !event.recenter && !event.open_bindings); + !event.close_mic_when_idle && !event.lock_layout && !event.clock_24h && !event.date_format && !event.recenter && !event.open_bindings && !event.model_action); } void snapshot(PanelSurface& surface, const std::string& path) { std::ofstream out(path, std::ios::binary); @@ -28,7 +28,7 @@ void snapshot(PanelSurface& surface, const std::string& path) { int main(int argc, char** argv) { assert(argc >= 2); PanelSurface surface(argv[1], Mount::World); - Panel p{"Ready to record", "", "Focus your destination before Insert. Enter is always separate.", false, false}; + Panel p{"Ready to record", "", "Focus your destination before Type. Enter is always separate.", false, false}; assert(surface.render(p)); assert(surface.pixels().size() == size_t(PanelSurface::width * PanelSurface::height * 4)); assert(!surface.render(p)); @@ -128,7 +128,7 @@ int main(int argc, char** argv) { p.enabled = true; p.record_available = false; p.status = "Review your words"; - p.transcript = "A quieter way to type in VR.\nKeep the menu where it feels comfortable.\nNothing is sent until you choose Insert."; + p.transcript = "A quieter way to type in VR.\nKeep the menu where it feels comfortable.\nNothing is sent until you choose Type."; assert(surface.render(p)); surface.reset_pointers(); surface.render(p); if (argc >= 3) snapshot(surface, std::string(argv[2]) + "-review.ppm"); @@ -189,44 +189,56 @@ int main(int argc, char** argv) { assert(click(surface, 680, 332).mount == Mount::RightWrist); assert(click(surface, 680, 260).mount == Mount::Head); assert(click(surface, 180, 260).mount == Mount::World); - auto laser = click(surface, 680, 420); + auto laser = click(surface, 680, 400); assert(laser.lasers_anytime == true && !laser.action && !laser.mount); surface.set_lasers_anytime(true); assert(surface.render(p)); - laser = click(surface, 680, 420); + laser = click(surface, 680, 400); assert(laser.lasers_anytime == false && !laser.action && !laser.mount); surface.set_lasers_anytime(false); assert(surface.render(p)); - auto automatic = click(surface, 200, 474); + auto automatic = click(surface, 200, 446); assert(automatic.auto_insert == true && !automatic.action); assert(!surface.render(p)); // request alone has no effect surface.set_auto_insert(true); assert(surface.render(p)); - automatic = click(surface, 200, 474); + automatic = click(surface, 200, 446); assert(automatic.auto_insert == false); surface.set_auto_insert(false); assert(surface.render(p)); - auto debug = click(surface, 680, 474); + auto debug = click(surface, 680, 446); assert(debug.advanced_debug == true && !debug.action && !debug.mount && !debug.lasers_anytime); assert(!surface.render(p)); // an event is only a request; caller sets the accepted value surface.set_advanced_debug(true); assert(surface.render(p)); assert(!surface.render(p)); - surface.pointer_down(1, 680, 474); + surface.pointer_down(1, 680, 446); surface.set_advanced_debug(false); assert(surface.render(p)); - no_action(surface.pointer_up(1, 680, 474)); // stale press cannot toggle after state change - debug = click(surface, 680, 474); + no_action(surface.pointer_up(1, 680, 446)); // stale press cannot toggle after state change + debug = click(surface, 680, 446); assert(debug.advanced_debug == true); surface.set_advanced_debug(true); assert(surface.render(p)); - debug = click(surface, 680, 474); + debug = click(surface, 680, 446); assert(debug.advanced_debug == false); surface.set_advanced_debug(false); assert(surface.render(p)); - auto clock = click(surface, 200, 528); + auto clock = click(surface, 200, 492); assert(clock.clock_24h == true && !clock.action); assert(!surface.render(p)); // setting request waits for caller's accepted value surface.set_clock_24h(true); assert(surface.render(p)); - assert(click(surface, 200, 528).clock_24h == false); + assert(click(surface, 200, 492).clock_24h == false); surface.set_clock_24h(false); assert(surface.render(p)); for (DateFormat format : {DateFormat::DayMonthYear, DateFormat::Iso, DateFormat::Off, DateFormat::MonthDayYear}) { - auto date = click(surface, 680, 528); + auto date = click(surface, 680, 492); assert(date.date_format == format && !date.action); surface.set_date_format(format); assert(surface.render(p)); } + auto mic = click(surface, 200, 532); + assert(mic.close_mic_when_idle == true && !mic.action); + assert(!surface.render(p)); // only the caller can accept a setting event + surface.set_close_mic_when_idle(true); assert(surface.render(p)); + surface.pointer_down(1, 200, 532); + surface.set_close_mic_when_idle(false); assert(surface.render(p)); + no_action(surface.pointer_up(1, 200, 532)); // stale press cannot change mic policy + mic = click(surface, 200, 532); + assert(mic.close_mic_when_idle == true); + surface.set_close_mic_when_idle(true); assert(surface.render(p)); + assert(click(surface, 200, 532).close_mic_when_idle == false); + surface.set_close_mic_when_idle(false); assert(surface.render(p)); assert(click(surface, 280, 610).action == UiAction::Cancel); surface.set_placement_note("Wrist not tracked - using world space until it returns."); assert(surface.render(p)); assert(!surface.render(p)); @@ -237,12 +249,13 @@ int main(int argc, char** argv) { no_action(click(surface, 100, 160)); no_action(surface.pointer_up(1, 680, 260)); no_action(click(surface, 680, 260)); // mount controls not active on Review - no_action(click(surface, 680, 420)); // laser toggle only exists on Settings - no_action(click(surface, 680, 474)); // debug toggle only exists on Settings - no_action(click(surface, 200, 474)); // auto insert only exists on Settings + no_action(click(surface, 680, 400)); // laser toggle only exists on Settings + no_action(click(surface, 680, 446)); // debug toggle only exists on Settings + no_action(click(surface, 200, 446)); // auto insert only exists on Settings no_action(click(surface, 790, 160)); // layout lock only exists on Settings - no_action(click(surface, 200, 528)); // clock/date controls only exist on Settings - no_action(click(surface, 680, 528)); + no_action(click(surface, 200, 492)); // clock/date controls only exist on Settings + no_action(click(surface, 680, 492)); + no_action(click(surface, 200, 532)); // mic preference only exists on Settings // Bindings opens SteamVR directly, from either tab, without replacing review. surface.pointer_down(1, 480, 610); @@ -306,5 +319,116 @@ int main(int argc, char** argv) { assert(surface.pointer_up(0, 900, 610, at + PanelSurface::quit_hold).action == UiAction::Quit); assert(surface.render(p)); assert(!surface.render(p)); + // Model page: selection is separate from installation; the first Install + // click reveals the source/size/license and cannot launch a child. + PanelSurface chooser(argv[1], Mount::World); + Panel model_panel{"Ready", "", "", true, false}; + model_panel.selected_backend = "redux"; + model_panel.models = {{"redux", "Parakeet Redux", "not_installed", "https://example.org/model", + "CC-BY-4.0", "Pinned license text", "Attribution", 177774490, false, std::string(64, 'a')}, + {"fake", "Fixture", "installed_verified", "https://example.org/fake", + "MIT", "Fixture license", "Fixture", 7, true, std::string(64, 'b')}}; + chooser.render(model_panel); + no_action(click(chooser, 290, 160)); // Settings + chooser.render(model_panel); + no_action(click(chooser, 680, 530)); // Models + chooser.render(model_panel); + auto select = click(chooser, 100, 310); + assert(select.model_action && select.model_action->id == "fake" && !select.model_action->install); + model_panel.selected_backend = "fake"; + chooser.render(model_panel); + no_action(click(chooser, 700, 546)); // verified model cannot be installed + select = click(chooser, 100, 260); + assert(select.model_action && select.model_action->id == "redux" && !select.model_action->install); + model_panel.selected_backend = "redux"; + chooser.render(model_panel); + no_action(click(chooser, 700, 546)); // consent preview only + chooser.render(model_panel); + no_action(click(chooser, 700, 546)); // first page cannot authorize installation + while (true) { + auto rows = chooser.visible_model_review_lines(); + assert(!rows.empty()); + auto before = chooser.pixels(); + no_action(click(chooser, 370, 546)); + if (!chooser.render(model_panel)) break; // final page has disabled Next + assert(chooser.pixels() != before); + } + auto install = click(chooser, 700, 546); + assert(install.model_action && install.model_action->id == "redux" && install.model_action->install && + install.model_action->manifest_sha256 == std::string(64, 'a')); + // Same ID with a new source or fingerprint invalidates displayed approval. + no_action(click(chooser, 700, 546)); + chooser.render(model_panel); + model_panel.models[0].source = "https://example.org/changed"; + model_panel.models[0].manifest_sha256 = std::string(64, 'c'); + chooser.render(model_panel); + no_action(click(chooser, 700, 546)); // now only a new preview, no install + chooser.render(model_panel); + while (true) { + auto before = chooser.pixels(); + no_action(click(chooser, 370, 546)); + if (!chooser.render(model_panel)) break; + assert(chooser.pixels() != before); + } + install = click(chooser, 700, 546); + assert(install.model_action && install.model_action->manifest_sha256 == std::string(64, 'c')); + // Max-length metadata must have every byte represented on navigable pages, + // including the source tail, long license text, attribution and exact hash. + PanelSurface long_review(argv[1], Mount::World); + Panel long_panel{"Ready", "", "", true, false}; + long_panel.selected_backend = "redux"; + auto long_model = model_panel.models[0]; + long_model.source = "https://" + std::string(1016, 's'); + long_model.license = std::string(1024, 'L'); + long_model.license_text = std::string(1024, 'T'); + long_model.attribution = std::string(1024, 'A'); + long_panel.models = {long_model}; + long_review.render(long_panel); + no_action(click(long_review, 290, 160)); long_review.render(long_panel); + no_action(click(long_review, 680, 530)); long_review.render(long_panel); + no_action(click(long_review, 700, 546)); + assert(long_review.visible_model_review_lines().empty()); // preview not yet drawn + no_action(click(long_review, 370, 546)); // cannot skip unpainted consent page + long_review.render(long_panel); + std::string reviewed; + int pages = 0; + for (;;) { + auto rows = long_review.visible_model_review_lines(); + assert(!rows.empty()); + for (const auto& row : rows) reviewed += row; + ++pages; + if (pages == 1) no_action(click(long_review, 700, 546)); + auto before = long_review.pixels(); + no_action(click(long_review, 370, 546)); + if (long_review.visible_model_review_lines().empty()) { + no_action(click(long_review, 700, 546)); // next page not painted yet + no_action(click(long_review, 370, 546)); // cannot skip next unpainted page + } + if (!long_review.render(long_panel)) break; + assert(long_review.pixels() != before); + } + assert(pages > 8); + for (const auto& value : {long_model.source, long_model.license, long_model.license_text, + long_model.attribution, long_model.manifest_sha256}) + assert(reviewed.find(value) != std::string::npos); + auto long_install = click(long_review, 700, 546); + assert(long_install.model_action && long_install.model_action->install && + long_install.model_action->manifest_sha256 == long_model.manifest_sha256); + long_panel.model_note = "Downloading: weights.dat"; + long_review.render(long_panel); + auto without_error = long_review.pixels(); + long_panel.model_note = "Install failed: manifest_mismatch: pinned metadata changed; local model status checked offline."; + assert(long_review.render(long_panel)); + bool footer_note_changed = false; + for (int y = 644; y < 680; ++y) for (int x = 32; x < 968; ++x) { + auto pixel = (size_t(y) * PanelSurface::width + x) * 4; + footer_note_changed |= long_review.pixels()[pixel] != without_error[pixel]; + } + assert(footer_note_changed); // feedback below buttons, never underneath them + chooser.render(model_panel); + chooser.pointer_down(0, 700, 546); + model_panel.model_busy = true; + chooser.render(model_panel); + no_action(chooser.pointer_up(0, 700, 546)); // change invalidates approval std::cout << "panel checks passed (no OpenVR, microphone or input injection)\n"; } diff --git a/tests/test_backend_dispatch.py b/tests/test_backend_dispatch.py index 32e6813..2c41576 100644 --- a/tests/test_backend_dispatch.py +++ b/tests/test_backend_dispatch.py @@ -142,6 +142,14 @@ class BackendDispatchTest(unittest.TestCase): self.assertNotEqual(wrong.returncode, 0) self.assertFalse(marker.exists(), wrong.stdout) self.assertNotIn('DONE', wrong.stdout) + self.assertEqual(json.loads(wrong.stdout), { + 'ok': False, 'code': 'manifest_mismatch', + 'message': 'selected backend manifest changed since consent'}) + missing = subprocess.run(install[:-1] + [hashlib.sha256(source.read_bytes()).hexdigest(), + '--installer', str(root / 'absent.sh')], capture_output=True, text=True) + self.assertNotEqual(missing.returncode, 0) + self.assertEqual(json.loads(missing.stdout)['message'], 'installer missing or unsafe') + self.assertFalse(marker.exists()) # Reusing a backend ID cannot silently re-authorize new sources. new = hashlib.sha256(source.read_bytes()).hexdigest() self.assertNotEqual(new, old)