From 0cb2991f0220b75b4b30b22a7ef4cd47990197d8 Mon Sep 17 00:00:00 2001 From: baketnk Date: Thu, 24 Sep 2026 11:44:24 -0400 Subject: [PATCH] Add bounded dictation state and persistent offline Redux worker --- LICENSE | 21 +++ python/frameyap/__init__.py | 1 + python/frameyap/model_files.py | 9 + python/frameyap/worker.py | 191 +++++++++++++++++++++ scripts/benchmark-worker.py | 102 ++++++++++++ scripts/fetch-model.py | 56 +++++++ src/core.cpp | 60 +++++++ src/core.hpp | 30 ++++ src/worker.cpp | 296 +++++++++++++++++++++++++++++++++ src/worker.hpp | 37 +++++ tests/core_test.cpp | 31 ++++ tests/test_worker.py | 167 +++++++++++++++++++ tests/worker_test.cpp | 131 +++++++++++++++ 13 files changed, 1132 insertions(+) create mode 100644 LICENSE create mode 100644 python/frameyap/__init__.py create mode 100644 python/frameyap/model_files.py create mode 100644 python/frameyap/worker.py create mode 100644 scripts/benchmark-worker.py create mode 100644 scripts/fetch-model.py create mode 100644 src/core.cpp create mode 100644 src/core.hpp create mode 100644 src/worker.cpp create mode 100644 src/worker.hpp create mode 100644 tests/core_test.cpp create mode 100644 tests/test_worker.py create mode 100644 tests/worker_test.cpp diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..55ee47f --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 FrameYap contributors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/python/frameyap/__init__.py b/python/frameyap/__init__.py new file mode 100644 index 0000000..395fbc7 --- /dev/null +++ b/python/frameyap/__init__.py @@ -0,0 +1 @@ +"""Standalone FrameYap offline CPU worker (no model imports on package import).""" diff --git a/python/frameyap/model_files.py b/python/frameyap/model_files.py new file mode 100644 index 0000000..d8cf966 --- /dev/null +++ b/python/frameyap/model_files.py @@ -0,0 +1,9 @@ +"""Pinned public moondream/parakeet-redux artifacts; weights are never in Git.""" +REVISION = "fad622f25f303105c20d70e201bcc477c88b620c" +FILES = { + "model.safetensors": (177774490, "78ec25733ee0d0c1586d1346fc86db9d0c2e436e3a8ab1d32a82d1bb8f848d21"), + "config.json": (12988, "503c653b2e3bb788adbcb04f5abdee532d958686564081baeed133ff10143f6e"), + "ternary.json": (57970, "1221c6d3ce901ffe09c089da758a8db8b76189f80cff41c5afc244fc61e2051d"), + "tokenizer.json": (1159960, "bd321b096832a3f270bd3b2a88823957920f1a5c5ada71114a26ea729d0cbe91"), + "README.md": (8533, "a8b327f983a8b8ff262ff7bead3a791fbed9350632002af8db85ab5cd84cdaa5"), +} diff --git a/python/frameyap/worker.py b/python/frameyap/worker.py new file mode 100644 index 0000000..a06cb41 --- /dev/null +++ b/python/frameyap/worker.py @@ -0,0 +1,191 @@ +"""Persistent local Redux worker. No imports of model dependencies until explicit start. + +Wire protocol: unsigned LE32 payload length (max 65536), type byte, unsigned +LE64 request id for T/R/E; Y (ready) and F (load failure) have no id. T refers +to the fixed clip.raw in the private directory. R/E carry <=4096 UTF-8 bytes. +Only one T may be outstanding. No stdout other than frames; stderr is suppressed +by the native launcher. This module never captures audio or delivers input. +""" + +import argparse +import hashlib +import os +from pathlib import Path +import stat +import struct +import sys + +MAX_FRAME = 65536 +MAX_TEXT = 4096 +MIN_SAMPLES = 3200 +MAX_SAMPLES = 320000 +try: + from .model_files import REVISION, FILES +except ImportError: # direct executable script + from model_files import REVISION, FILES + + +def read_exact(fd, count): + parts = bytearray() + while len(parts) < count: + part = os.read(fd, count - len(parts)) + if not part: + if not parts: + return None + raise ValueError("truncated worker frame") + parts.extend(part) + return bytes(parts) + + +def read_frame(fd): + header = read_exact(fd, 4) + if header is None: + return None + size, = struct.unpack(" MAX_FRAME: + raise ValueError("oversize response") + frame = struct.pack(" MAX_TEXT: + raise ValueError("transcript exceeds 4096 bytes") + send_frame(output_fd, b"R", request_id + encoded) + except Exception: + # Model exceptions may include audio or transcripts. Do not log or + # forward them to the overlay; request-local failure only. + send_frame(output_fd, b"E", request_id + b"transcription failed") + + +def main(argv=None): + parser = argparse.ArgumentParser(description="FrameYap local offline Redux worker") + parser.add_argument("--model", required=True) + parser.add_argument("--threads", type=int, default=2) + parser.add_argument("--clip-dir", required=True) + args = parser.parse_args(argv) + if not 1 <= args.threads <= 64: + parser.error("threads must be 1..64") + # Native libraries sometimes print directly to fd 1. Keep those bytes out of + # the framed channel, not merely Python's sys.stdout wrapper. + protocol_fd = os.dup(1) + os.set_inheritable(protocol_fd, False) + with open(os.devnull, "wb") as null: + os.dup2(null.fileno(), 1) + os.dup2(null.fileno(), 2) + try: + try: + private_dir(args.clip_dir) + # The full hash validation occurs exactly once in load_model. + if not Path(args.model).is_dir(): + raise ValueError("missing model") + except Exception: + send_frame(protocol_fd, b"F", b"M") + return 1 + try: + model = load_model(args.model, args.threads) + except Exception: + send_frame(protocol_fd, b"F", b"D") + return 1 + run(model, args.clip_dir, output_fd=protocol_fd) + return 0 + finally: + os.close(protocol_fd) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/benchmark-worker.py b/scripts/benchmark-worker.py new file mode 100644 index 0000000..b0e80ea --- /dev/null +++ b/scripts/benchmark-worker.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python3 +"""Opt-in local CPU/IPC benchmark of a supplied nonprivate WAV. Never captures or types.""" +import argparse +import array +import os +from pathlib import Path +import select +import statistics +import struct +import subprocess +import tempfile +import time +import wave + + +def receive(fd, timeout): + end = time.monotonic() + timeout + def exact(count): + data = bytearray() + while len(data) < count: + remaining = end - time.monotonic() + if remaining <= 0 or not select.select([fd], [], [], remaining)[0]: + raise TimeoutError("worker deadline") + chunk = os.read(fd, count - len(data)) + if not chunk: + raise RuntimeError("worker exited") + data.extend(chunk) + return bytes(data) + size, = struct.unpack(" size: + raise ValueError("download exceeded pinned size") + output.write(chunk) + if not matches(Path(temp), size, digest): + raise ValueError(f"pinned SHA-256/size mismatch: {name}") + os.replace(temp, target) + finally: + Path(temp).unlink(missing_ok=True) + print(f"Pinned Redux {REVISION} verified in {dest}") + print("Model attribution: moondream/parakeet-redux, CC-BY-4.0; see downloaded README.md.") + + +if __name__ == "__main__": + main() diff --git a/src/core.cpp b/src/core.cpp new file mode 100644 index 0000000..5d362e3 --- /dev/null +++ b/src/core.cpp @@ -0,0 +1,60 @@ +#include "core.hpp" +#include + +namespace frameyap { +std::string literal_text(std::string_view input) { + if (input.size() > 4096) throw std::runtime_error("Transcript exceeds 4096 bytes"); + std::string out; + for (std::size_t i = 0; i < input.size();) { + const auto start = i; + const auto first = static_cast(input[i++]); + uint32_t cp = first; + int continuation = 0; + uint32_t minimum = 0; + if (first >= 0xc2 && first <= 0xdf) { cp = first & 31; continuation = 1; minimum = 0x80; } + else if (first >= 0xe0 && first <= 0xef) { cp = first & 15; continuation = 2; minimum = 0x800; } + else if (first >= 0xf0 && first <= 0xf4) { cp = first & 7; continuation = 3; minimum = 0x10000; } + else if (first >= 0x80) throw std::runtime_error("Invalid UTF-8 transcript"); + while (continuation--) { + if (i == input.size()) throw std::runtime_error("Truncated UTF-8 transcript"); + auto byte = static_cast(input[i++]); + if ((byte & 0xc0) != 0x80) throw std::runtime_error("Invalid UTF-8 transcript"); + cp = (cp << 6) | (byte & 63); + } + if (cp < minimum || cp > 0x10ffff || (cp >= 0xd800 && cp <= 0xdfff)) + throw std::runtime_error("Invalid UTF-8 codepoint"); + if (cp == '\n' || cp == '\r' || cp == '\t' || cp == 0x2028 || cp == 0x2029) { + out += ' '; + } else { + if (cp < 32 || (cp >= 0x7f && cp <= 0x9f)) + throw std::runtime_error("Control characters rejected"); + out.append(input.substr(start, i - start)); + } + } + if (out.find_first_not_of(' ') == std::string::npos) out.clear(); + return out; +} +void Session::ready() { if (state_ == State::Warming) state_ = State::Ready; } +bool Session::record() { + if (state_ != State::Ready && state_ != State::Queued) return false; + ++id_; text_.clear(); state_ = State::Recording; return true; +} +bool Session::finish(std::size_t samples) { + if (state_ != State::Recording) return false; + if (samples < 3200 || samples > 320000) { cancel(); return false; } + state_ = State::Transcribing; return true; +} +bool Session::reply(uint64_t id, std::string_view text) { + if (id != id_ || state_ != State::Transcribing) return false; + text_ = literal_text(text); + state_ = text_.empty() ? State::Ready : State::Review; + return true; +} +std::optional Session::take_insert() { + if (state_ != State::Review) return std::nullopt; + auto text = std::move(text_); text_.clear(); state_ = State::Queued; + return text; +} +void Session::cancel() { ++id_; text_.clear(); state_ = State::Ready; } +void Session::fail() { ++id_; text_.clear(); state_ = State::Error; } +} diff --git a/src/core.hpp b/src/core.hpp new file mode 100644 index 0000000..307d6f3 --- /dev/null +++ b/src/core.hpp @@ -0,0 +1,30 @@ +#pragma once +#include +#include +#include +#include + +namespace frameyap { +// Throws on malformed UTF-8, controls, or >4096 bytes. Never interprets commands. +std::string literal_text(std::string_view input); +enum class State { Warming, Ready, Recording, Transcribing, Review, Queued, Error }; +// Hardware-free delivery ledger. A request can be consumed only once, including +// ambiguous transport failures. Explicit Insert authorizes CURRENT seat focus. +class Session { +public: + State state() const { return state_; } + const std::string& text() const { return text_; } + uint64_t id() const { return id_; } + void ready(); + bool record(); + bool finish(std::size_t samples); + bool reply(uint64_t id, std::string_view text); + std::optional take_insert(); + void cancel(); + void fail(); +private: + State state_ = State::Warming; + uint64_t id_ = 0; + std::string text_; +}; +} diff --git a/src/worker.cpp b/src/worker.cpp new file mode 100644 index 0000000..1ef59ac --- /dev/null +++ b/src/worker.cpp @@ -0,0 +1,296 @@ +#include "worker.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +extern char** environ; + +namespace frameyap { +namespace { +using Clock = std::chrono::steady_clock; +constexpr size_t max_frame = 65536; +constexpr size_t max_text = 4096; + +void close_fd(int& fd) { if (fd >= 0) { ::close(fd); fd = -1; } } +void put32(unsigned char* p, uint32_t n) { + for (int i = 0; i < 4; ++i) p[i] = static_cast(n >> (8 * i)); +} +void put64(unsigned char* p, uint64_t n) { + for (int i = 0; i < 8; ++i) p[i] = static_cast(n >> (8 * i)); +} +uint32_t get32(const unsigned char* p) { + uint32_t n = 0; + for (int i = 0; i < 4; ++i) n |= uint32_t(p[i]) << (8 * i); + return n; +} +uint64_t get64(const unsigned char* p) { + uint64_t n = 0; + for (int i = 0; i < 8; ++i) n |= uint64_t(p[i]) << (8 * i); + return n; +} +bool valid_utf8(std::string_view text) { + for (size_t i = 0; i < text.size();) { + unsigned char first = static_cast(text[i]); + if (first == 0) return false; + if (first < 0x80) { ++i; continue; } + unsigned count = first >= 0xc2 && first <= 0xdf ? 2 : + first >= 0xe0 && first <= 0xef ? 3 : + first >= 0xf0 && first <= 0xf4 ? 4 : 0; + if (!count || i + count > text.size()) return false; + for (unsigned j = 1; j < count; ++j) + if ((static_cast(text[i + j]) & 0xc0) != 0x80) return false; + unsigned char second = static_cast(text[i + 1]); + if ((first == 0xe0 && second < 0xa0) || (first == 0xed && second >= 0xa0) || + (first == 0xf0 && second < 0x90) || (first == 0xf4 && second >= 0x90)) return false; + i += count; + } + return true; +} +void check_runtime(const char* path) { + if (!path || path[0] != '/') throw std::runtime_error("XDG_RUNTIME_DIR must be an absolute private directory"); + struct stat st{}; + if (::lstat(path, &st) || !S_ISDIR(st.st_mode) || st.st_uid != ::geteuid() || (st.st_mode & 0077)) + throw std::runtime_error("XDG_RUNTIME_DIR must be owned by this user and inaccessible to others (no symlinks)"); +} +// Blocking SIGPIPE only on this thread avoids changing the host application's signal disposition. +ssize_t safe_write(int fd, const void* data, size_t size) { + sigset_t mask{}, prior{}, pending{}; + sigemptyset(&mask); sigaddset(&mask, SIGPIPE); + if (pthread_sigmask(SIG_BLOCK, &mask, &prior)) throw std::runtime_error("cannot block SIGPIPE"); + sigpending(&pending); + bool was_pending = sigismember(&pending, SIGPIPE); + ssize_t result = ::write(fd, data, size); + int saved = errno; + if (result < 0 && saved == EPIPE && !was_pending) { + timespec zero{}; + ::sigtimedwait(&mask, nullptr, &zero); + } + pthread_sigmask(SIG_SETMASK, &prior, nullptr); + errno = saved; + return result; +} +void write_all(int fd, const unsigned char* data, size_t size) { + while (size) { + ssize_t n = safe_write(fd, data, size); + if (n < 0 && errno == EINTR) continue; + if (n <= 0) throw std::runtime_error("worker pipe write failed"); + data += n; size -= static_cast(n); + } +} +} // namespace + +struct Worker::State { + pid_t pid = -1; + int to_child = -1, from_child = -1; + std::string dir; + bool loaded = false; + std::optional pending; + Clock::time_point deadline{}; + std::vector input; + std::chrono::milliseconds warmup_timeout{120000}, request_timeout{60000}; +}; + +Worker::Worker(std::chrono::milliseconds warmup, std::chrono::milliseconds request) + : state_(std::make_unique()) { + if (warmup.count() <= 0 || request.count() <= 0) throw std::invalid_argument("positive worker deadlines required"); + state_->warmup_timeout = warmup; + state_->request_timeout = request; +} +Worker::~Worker() { stop(); } + +void Worker::stop() { + auto& s = *state_; + close_fd(s.to_child); + close_fd(s.from_child); + if (s.pid > 0) { + ::kill(s.pid, SIGTERM); // Only our direct child, never a process group. + auto until = Clock::now() + std::chrono::milliseconds(500); + int status = 0; + while (::waitpid(s.pid, &status, WNOHANG) == 0 && Clock::now() < until) + ::usleep(10000); + if (::waitpid(s.pid, &status, WNOHANG) == 0) { + ::kill(s.pid, SIGKILL); + while (::waitpid(s.pid, &status, 0) < 0 && errno == EINTR) {} + } + s.pid = -1; + } + if (!s.dir.empty()) { + ::unlink((s.dir + "/clip.raw").c_str()); + ::rmdir(s.dir.c_str()); + s.dir.clear(); + } + s.input.clear(); s.pending.reset(); s.loaded = false; +} + +void Worker::start(const std::string& python, const std::string& script, + const std::string& model, int threads) { + if (state_->pid > 0) throw std::logic_error("worker already started"); + if (python.empty() || script.empty() || model.empty() || threads < 1 || threads > 64) + throw std::invalid_argument("python, script, local model and 1..64 threads required"); + check_runtime(::getenv("XDG_RUNTIME_DIR")); + try { + // The model's internal files are checked by the child before loading, never fetched. + std::string pattern = std::string(::getenv("XDG_RUNTIME_DIR")) + "/frameyap-XXXXXX"; + std::vector tmp(pattern.begin(), pattern.end()); tmp.push_back('\0'); + if (!::mkdtemp(tmp.data())) throw std::runtime_error("cannot create private clip directory"); + state_->dir = tmp.data(); + ::chmod(state_->dir.c_str(), 0700); + int in[2]{-1,-1}, out[2]{-1,-1}; + if (::pipe2(in, O_CLOEXEC) || ::pipe2(out, O_CLOEXEC)) { + close_fd(in[0]); close_fd(in[1]); close_fd(out[0]); close_fd(out[1]); + throw std::runtime_error("cannot create worker pipes"); + } + std::string thread_arg = std::to_string(threads); + // OpenVR/SDL already have threads. posix_spawn avoids allocations and + // setenv in a forked child of a multithreaded process. + std::vector environment; + for (char** e = environ; *e; ++e) { + std::string_view entry(*e); + if (entry.starts_with("PYTHONDONTWRITEBYTECODE=")) continue; + environment.emplace_back(*e); + } + environment.emplace_back("PYTHONDONTWRITEBYTECODE=1"); + std::vector envp; + for (auto& value : environment) envp.push_back(value.data()); + envp.push_back(nullptr); + const char* args[] = {python.c_str(), script.c_str(), "--model", model.c_str(), + "--threads", thread_arg.c_str(), "--clip-dir", state_->dir.c_str(), nullptr}; + posix_spawn_file_actions_t actions; + int error = posix_spawn_file_actions_init(&actions); + if (error) { + close_fd(in[0]); close_fd(in[1]); close_fd(out[0]); close_fd(out[1]); + throw std::runtime_error("cannot initialize worker spawn"); + } + error = posix_spawn_file_actions_adddup2(&actions, in[0], STDIN_FILENO); + if (!error) error = posix_spawn_file_actions_adddup2(&actions, out[1], STDOUT_FILENO); + if (!error) error = posix_spawn_file_actions_addopen(&actions, STDERR_FILENO, "/dev/null", O_WRONLY, 0); + for (int fd : std::array{in[0], in[1], out[0], out[1]}) + if (!error && fd > STDERR_FILENO) error = posix_spawn_file_actions_addclose(&actions, fd); + pid_t pid = -1; + if (!error) error = posix_spawnp(&pid, python.c_str(), &actions, nullptr, + const_cast(args), envp.data()); + posix_spawn_file_actions_destroy(&actions); + if (error) { + close_fd(in[0]); close_fd(in[1]); close_fd(out[0]); close_fd(out[1]); + throw std::runtime_error("cannot launch configured Python worker"); + } + close_fd(in[0]); close_fd(out[1]); + state_->pid = pid; state_->to_child = in[1]; state_->from_child = out[0]; + int flags = ::fcntl(state_->from_child, F_GETFL); + if (flags < 0 || ::fcntl(state_->from_child, F_SETFL, flags | O_NONBLOCK)) + throw std::runtime_error("cannot set nonblocking worker pipe"); + state_->deadline = Clock::now() + state_->warmup_timeout; + } catch (...) { stop(); throw; } +} + +bool Worker::ready() const { return state_->pid > 0 && state_->loaded; } + +void Worker::submit(uint64_t id, const std::vector& pcm) { + auto& s = *state_; + if (!ready() || s.pending) throw std::logic_error("worker not ready or request already pending"); + if (pcm.size() < 3200 || pcm.size() > 320000) throw std::invalid_argument("clip must be 0.2..20 seconds at 16 kHz"); + for (float value : pcm) if (!std::isfinite(value)) throw std::invalid_argument("nonfinite PCM sample"); + const std::string path = s.dir + "/clip.raw"; + int fd = ::open(path.c_str(), O_WRONLY | O_CREAT | O_EXCL | O_NOFOLLOW | O_CLOEXEC, 0600); + if (fd < 0) throw std::runtime_error("cannot create exclusive private clip"); + try { + // IEEE-754 binary32 encoded explicitly little-endian, independent of host byte order. + std::array bytes{}; + size_t index = 0; + for (float value : pcm) { + uint32_t bits; static_assert(sizeof bits == sizeof value); + std::memcpy(&bits, &value, sizeof bits); + put32(bytes.data() + index, bits); index += 4; + if (index == bytes.size()) { write_all(fd, bytes.data(), index); index = 0; } + } + if (index) write_all(fd, bytes.data(), index); + if (::close(fd)) { fd = -1; throw std::runtime_error("clip close failed"); } + fd = -1; + unsigned char request[13]{}; + put32(request, 9); request[4] = 'T'; put64(request + 5, id); + // Pipe is empty (single outstanding request); 13 bytes <= PIPE_BUF. + write_all(s.to_child, request, sizeof request); + s.pending = id; + s.deadline = Clock::now() + s.request_timeout; + } catch (...) { + if (fd >= 0) ::close(fd); + ::unlink(path.c_str()); + stop(); // Any partial pipe write means protocol synchronization is unknown. + throw; + } +} + +std::optional Worker::poll() { + auto& s = *state_; + if (s.pid <= 0) return std::nullopt; + try { + if ((!s.loaded || s.pending) && Clock::now() > s.deadline) + throw std::runtime_error(s.loaded ? "worker transcription timed out" : "worker warmup timed out"); + unsigned char block[4096]; + bool eof = false; + for (;;) { + ssize_t n = ::read(s.from_child, block, sizeof block); + if (n > 0) { + s.input.insert(s.input.end(), block, block + n); + if (s.input.size() > max_frame + 4) throw std::runtime_error("worker frame exceeded limit"); + continue; + } + if (n == 0) { eof = true; break; } + if (errno == EINTR) continue; + if (errno != EAGAIN && errno != EWOULDBLOCK) throw std::runtime_error("worker pipe read failed"); + break; + } + if (s.input.size() >= 4) { + auto size = get32(s.input.data()); + if (size > max_frame || size < 1) throw std::runtime_error("invalid worker frame length"); + if (s.input.size() >= 4 + size) { + if (s.input.size() != 4 + size) throw std::runtime_error("unexpected extra worker frame"); + char type = static_cast(s.input[4]); + if (type == 'Y' && size == 1 && !s.loaded && !s.pending) { + if (eof) throw std::runtime_error("worker exited after warmup"); + s.loaded = true; s.input.clear(); return std::nullopt; + } + if (type == 'F' && !s.loaded) { + if (size == 2 && s.input[5] == 'M') + throw std::runtime_error("worker missing local model weights or private clip directory"); + throw std::runtime_error("worker failed to load local CPU model/runtime"); + } + if ((type != 'R' && type != 'E') || size < 9 || !s.loaded || !s.pending || + get64(s.input.data() + 5) != *s.pending || size - 9 > max_text) + throw std::runtime_error("invalid or stale worker reply"); + WorkerReply reply{*s.pending, {}, {}}; + std::string text(reinterpret_cast(s.input.data() + 13), size - 9); + if (!valid_utf8(text)) throw std::runtime_error("invalid worker UTF-8 reply"); + if (type == 'R') reply.text = std::move(text); + else reply.error = std::move(text); + s.pending.reset(); s.input.clear(); + ::unlink((s.dir + "/clip.raw").c_str()); + return reply; + } + } + if (eof) throw std::runtime_error("worker exited or closed its pipe"); + int status{}; + if (::waitpid(s.pid, &status, WNOHANG) == s.pid) { + s.pid = -1; + throw std::runtime_error("worker exited unexpectedly"); + } + return std::nullopt; + } catch (...) { stop(); throw; } +} +} // namespace frameyap diff --git a/src/worker.hpp b/src/worker.hpp new file mode 100644 index 0000000..8bcd8d1 --- /dev/null +++ b/src/worker.hpp @@ -0,0 +1,37 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +namespace frameyap { + +struct WorkerReply { + uint64_t id; + std::string text; + std::string error; +}; + +class Worker { +public: + explicit Worker(std::chrono::milliseconds warmup_timeout = std::chrono::seconds(120), + std::chrono::milliseconds request_timeout = std::chrono::seconds(60)); + ~Worker(); + Worker(const Worker&) = delete; + Worker& operator=(const Worker&) = delete; + void start(const std::string& python, const std::string& script, + const std::string& model, int threads = 2); + bool ready() const; + void submit(uint64_t id, const std::vector& pcm); + std::optional poll(); + void stop(); + +private: + struct State; + std::unique_ptr state_; +}; + +} // namespace frameyap diff --git a/tests/core_test.cpp b/tests/core_test.cpp new file mode 100644 index 0000000..3085eda --- /dev/null +++ b/tests/core_test.cpp @@ -0,0 +1,31 @@ +#include "core.hpp" +#include +#include +#include +using namespace frameyap; +#define CHECK(x) do { if (!(x)) { std::cerr << "line " << __LINE__ << ": " #x "\n"; std::exit(1); } } while (0) +void rejects(std::string text) { try { literal_text(text); } catch (const std::runtime_error&) { return; } CHECK(false); } +int main() { + CHECK(literal_text("Hello 世界 😀") == "Hello 世界 😀"); + CHECK(literal_text("one\r\ntwo\tthree\xe2\x80\xa8") == "one two three "); + CHECK(literal_text("submit; $(echo x)") == "submit; $(echo x)"); + CHECK(literal_text(" \t\n").empty()); + CHECK(literal_text(std::string(4096, 'a')).size() == 4096); + rejects(std::string(4097, 'a')); rejects(std::string("a\0b", 3)); + rejects("\x1b[0m"); rejects("\xc0\xaf"); rejects("\xed\xa0\x80"); + rejects("\xf4\x90\x80\x80"); rejects("\xe2\x82"); rejects("\xc2\x85"); + Session s; + CHECK(!s.record()); s.ready(); CHECK(s.record()); + CHECK(!s.finish(3199)); CHECK(s.state() == State::Ready); + CHECK(s.record()); auto cancelled = s.id(); CHECK(s.finish(3200)); + CHECK(!s.record()); s.cancel(); CHECK(!s.reply(cancelled, "stale")); + CHECK(s.record()); auto current = s.id(); CHECK(s.finish(320000)); + CHECK(!s.reply(current - 1, "stale")); CHECK(s.reply(current, "text\n")); + CHECK(!s.reply(current, "duplicate")); CHECK(!s.record()); + CHECK(s.take_insert() == "text "); CHECK(!s.take_insert()); + CHECK(!s.reply(current, "again")); CHECK(s.record()); CHECK(!s.finish(320001)); + CHECK(s.record()); CHECK(s.finish(16000)); CHECK(s.reply(s.id(), "")); + CHECK(s.state() == State::Ready); CHECK(!s.take_insert()); + CHECK(s.record()); CHECK(s.finish(16000)); s.fail(); CHECK(!s.reply(s.id(), "bad")); + std::cout << "core checks passed\n"; +} diff --git a/tests/test_worker.py b/tests/test_worker.py new file mode 100644 index 0000000..6fdd275 --- /dev/null +++ b/tests/test_worker.py @@ -0,0 +1,167 @@ +"""Hardware-free tests, and a deliberately fake child for native IPC tests. + +Run: python3 -m unittest discover -s tests -p test_worker.py +Native fixture: python3 tests/test_worker.py --model ok --threads 2 --clip-dir DIR +""" +import os +from pathlib import Path +import struct +import sys +import tempfile +import unittest +from unittest.mock import patch + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "python")) +from frameyap import worker + + +class FakeModel: + def __init__(self): + self.calls = [] + + def transcribe(self, audio, sample_rate): + self.calls.append((audio, sample_rate)) + return {"text": "héllo 世界"} + + +def fake_child(): + # This branch intentionally never imports numpy, torch, moondream or a mic. + import argparse + parser = argparse.ArgumentParser() + parser.add_argument("--model", required=True) + parser.add_argument("--threads", required=True) + parser.add_argument("--clip-dir", required=True) + args = parser.parse_args() + if args.model == "fail": + worker.send_frame(1, b"F") + return + if args.model == "crash": + return + if args.model == "hang-warm": + import time + time.sleep(130) + return + worker.send_frame(1, b"Y") + while True: + msg = worker.read_frame(0) + if msg is None: + return + if args.model == "hang": + import time + time.sleep(70) + continue + assert len(msg) == 9 and msg[:1] == b"T" + clip = Path(args.clip_dir) / "clip.raw" + assert clip.stat().st_size == 3200 * 4 + ident = msg[1:9] + if args.model == "stale": + ident = struct.pack("= 0: + os.close(in_w) + os.close(out_r) + os.close(out_w) + + def test_missing_weights_no_dependency_import(self): + with tempfile.TemporaryDirectory() as path: + with self.assertRaisesRegex(ValueError, "weights missing"): + worker.local_model(path) + with patch.dict(sys.modules, {"moondream": None}): + with self.assertRaises(ValueError): + worker.load_model(path, 2) + self.assertEqual(os.environ["HF_HUB_OFFLINE"], "1") + + def test_pinned_model_hashes_and_symlinks(self): + import hashlib + with tempfile.TemporaryDirectory() as path: + root = Path(path) + (root / "model.safetensors").write_bytes(b"fixture") + digest = hashlib.sha256(b"fixture").hexdigest() + with patch.object(worker, "FILES", {"model.safetensors": (7, digest)}): + self.assertEqual(worker.local_model(path), path) + (root / "model.safetensors").write_bytes(b"changed") + with self.assertRaisesRegex(ValueError, "pinned revision"): + worker.local_model(path) + (root / "model.safetensors").unlink() + (root / "other").write_bytes(b"fixture") + (root / "model.safetensors").symlink_to(root / "other") + with self.assertRaises(ValueError): + worker.local_model(path) + + def test_unsafe_clip_and_oversize_text(self): + with tempfile.TemporaryDirectory() as path: + os.chmod(path, 0o700) + outside = Path(path) / "outside" + outside.write_bytes(b"0" * 12800) + os.symlink(outside, Path(path) / "clip.raw") + with self.assertRaises(OSError): + worker.read_clip(path) + (Path(path) / "clip.raw").unlink() + class Large: + def transcribe(self, **kwargs): + return {"text": "a" * 4097} + with patch.object(worker, "read_clip", return_value=[]): + in_r, in_w = os.pipe() + out_r, out_w = os.pipe() + try: + worker.send_frame(in_w, b"T", struct.pack("= 0: + os.close(in_w) + os.close(out_r) + os.close(out_w) + + +if __name__ == "__main__": + if "--clip-dir" in sys.argv: + fake_child() + else: + unittest.main() diff --git a/tests/worker_test.cpp b/tests/worker_test.cpp new file mode 100644 index 0000000..838c01a --- /dev/null +++ b/tests/worker_test.cpp @@ -0,0 +1,131 @@ +#include "worker.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace frameyap; +using namespace std::chrono_literals; + +static void until(const std::function& fn) { + auto deadline = std::chrono::steady_clock::now() + 4s; + while (!fn()) { + if (std::chrono::steady_clock::now() >= deadline) throw std::runtime_error("fixture timed out"); + std::this_thread::sleep_for(5ms); + } +} + +int main(int argc, char** argv) { + // CMake should pass its Python interpreter and the absolute test_worker.py path. + if (argc != 3) return 2; + char runtime[] = "/tmp/frameyap-test-XXXXXX"; + if (!::mkdtemp(runtime)) return 3; + ::setenv("XDG_RUNTIME_DIR", runtime, 1); + try { + std::vector clip(3200, 0.25f); + Worker worker; + worker.start(argv[1], argv[2], "ok", 2); + assert(!worker.ready()); + until([&] { worker.poll(); return worker.ready(); }); + bool rejected = false; + try { worker.submit(1, std::vector(3199)); } + catch (const std::invalid_argument&) { rejected = true; } + assert(rejected); + worker.submit(100, clip); + rejected = false; + try { worker.submit(101, clip); } + catch (const std::logic_error&) { rejected = true; } + assert(rejected); + std::optional reply; + until([&] { reply = worker.poll(); return reply.has_value(); }); + assert(reply->id == 100 && reply->text == "héllo 世界" && reply->error.empty()); + worker.submit(101, clip); + until([&] { reply = worker.poll(); return reply.has_value(); }); + assert(reply->id == 101); + worker.stop(); + assert(!worker.ready()); + for (const auto& entry : std::filesystem::directory_iterator(runtime)) { + (void)entry; + throw std::runtime_error("private directory leaked"); + } + worker.start(argv[1], argv[2], "fail", 2); + bool failed = false; + until([&] { + try { worker.poll(); } + catch (const std::runtime_error&) { failed = true; } + return failed; + }); + assert(!worker.ready()); + worker.start(argv[1], argv[2], "stale", 2); + until([&] { worker.poll(); return worker.ready(); }); + worker.submit(4, clip); + failed = false; + until([&] { + try { worker.poll(); } + catch (const std::runtime_error&) { failed = true; } + return failed; + }); + for (const char* mode : {"long", "oversized-frame", "duplicate"}) { + worker.start(argv[1], argv[2], mode, 2); + until([&] { worker.poll(); return worker.ready(); }); + worker.submit(5, clip); + failed = false; + until([&] { + try { worker.poll(); } + catch (const std::runtime_error&) { failed = true; } + return failed; + }); + assert(!worker.ready()); + } + // Cancellation must not wait for inference or leak the private clip. + worker.start(argv[1], argv[2], "hang", 2); + until([&] { worker.poll(); return worker.ready(); }); + worker.submit(6, clip); + auto cancel_start = std::chrono::steady_clock::now(); + worker.stop(); + assert(std::chrono::steady_clock::now() - cancel_start < 2s); + assert(std::filesystem::is_empty(runtime)); + worker.start(argv[1], argv[2], "hang-warm", 2); + worker.stop(); + assert(std::filesystem::is_empty(runtime)); + worker.start(argv[1], argv[2], "crash", 2); + failed = false; + until([&] { + try { worker.poll(); } + catch (const std::runtime_error&) { failed = true; } + return failed; + }); + worker.stop(); + Worker deadlines(2s, 50ms); + deadlines.start(argv[1], argv[2], "hang", 2); + until([&] { deadlines.poll(); return deadlines.ready(); }); + deadlines.submit(9, clip); + failed = false; + until([&] { + try { deadlines.poll(); } + catch (const std::runtime_error& e) { failed = std::string(e.what()).find("timed out") != std::string::npos; } + return failed; + }); + assert(std::filesystem::is_empty(runtime)); + Worker warm_deadline(50ms, 2s); + warm_deadline.start(argv[1], argv[2], "hang-warm", 2); + failed = false; + until([&] { + try { warm_deadline.poll(); } + catch (const std::runtime_error& e) { failed = std::string(e.what()).find("timed out") != std::string::npos; } + return failed; + }); + assert(std::filesystem::is_empty(runtime)); + std::filesystem::remove(runtime); + } catch (...) { + std::filesystem::remove(runtime); + throw; + } +}