From d60d8d2f5d8cf1d4c4e2a6b0369f58e3f956107d Mon Sep 17 00:00:00 2001 From: baketnk Date: Thu, 24 Sep 2026 11:45:05 -0400 Subject: [PATCH] Wire native OpenVR voice-typing POC with safe review and grip controls --- CMakeLists.txt | 116 ++++++- assets/actions.json | 27 ++ assets/application.vrmanifest.in | 19 ++ assets/bindings_frame_controller.json | 21 ++ assets/bindings_knuckles.json | 21 ++ protocol/gamescope-input-method.xml | 214 ++++++++++++ src/audio.cpp | 72 ++++ src/audio.hpp | 23 ++ src/gestures.hpp | 107 ++++++ src/instance_lock.hpp | 40 +++ src/main.cpp | 110 ++++++- src/overlay.cpp | 457 ++++++++++++++++++++++++++ src/overlay.hpp | 30 ++ src/runtime.cpp | 146 ++++++++ src/runtime.hpp | 10 + src/text_input.cpp | 126 +++++++ src/text_input.hpp | 19 ++ tests/cli.cmake | 6 +- tests/gestures_test.cpp | 73 ++++ tests/lock_test.cpp | 30 ++ tests/text_input_test.cpp | 291 ++++++++++++++++ 21 files changed, 1929 insertions(+), 29 deletions(-) create mode 100644 assets/actions.json create mode 100644 assets/application.vrmanifest.in create mode 100644 assets/bindings_frame_controller.json create mode 100644 assets/bindings_knuckles.json create mode 100644 protocol/gamescope-input-method.xml create mode 100644 src/audio.cpp create mode 100644 src/audio.hpp create mode 100644 src/gestures.hpp create mode 100644 src/instance_lock.hpp create mode 100644 src/overlay.cpp create mode 100644 src/overlay.hpp create mode 100644 src/runtime.cpp create mode 100644 src/runtime.hpp create mode 100644 src/text_input.cpp create mode 100644 src/text_input.hpp create mode 100644 tests/gestures_test.cpp create mode 100644 tests/lock_test.cpp create mode 100644 tests/text_input_test.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 24ca460..05e8239 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,19 +1,109 @@ cmake_minimum_required(VERSION 3.20) -project(frame_dictation VERSION 0.1.0 LANGUAGES CXX) +project(frameyap VERSION 0.1.0 LANGUAGES C CXX) +set(CMAKE_CXX_STANDARD 20) +set(CMAKE_CXX_STANDARD_REQUIRED ON) +set(CMAKE_CXX_EXTENSIONS OFF) +option(FRAMEYAP_NATIVE "Build opt-in OpenVR/SDL3/Gamescope runtime (no automatic downloads)" OFF) -add_executable(frame-dictation src/main.cpp) -target_compile_features(frame-dictation PRIVATE cxx_std_20) -set_target_properties(frame-dictation PROPERTIES CXX_EXTENSIONS OFF) -target_compile_definitions(frame-dictation PRIVATE FRAME_DICTATION_VERSION="${PROJECT_VERSION}") +add_library(frameyap_core src/core.cpp) +target_include_directories(frameyap_core PUBLIC src) +add_executable(frameyap src/main.cpp) +target_link_libraries(frameyap PRIVATE frameyap_core) +target_compile_definitions(frameyap PRIVATE FRAMEYAP_VERSION="${PROJECT_VERSION}") +if(CMAKE_SYSTEM_NAME STREQUAL "Linux") + add_library(frameyap_worker src/worker.cpp) + target_include_directories(frameyap_worker PUBLIC src) +endif() +if(FRAMEYAP_NATIVE) + if(NOT CMAKE_SYSTEM_NAME STREQUAL "Linux") + message(FATAL_ERROR "Native POC supports Linux only") + endif() + find_package(PkgConfig REQUIRED) + pkg_check_modules(SDL3 REQUIRED IMPORTED_TARGET sdl3>=3.2) + # SDL's .pc can inject its producer-prefix RPATH. Let CMake manage build + # RPATH and install only our relative path, never a developer environment. + get_target_property(SDL3_LINK_OPTIONS PkgConfig::SDL3 INTERFACE_LINK_OPTIONS) + if(SDL3_LINK_OPTIONS) + list(FILTER SDL3_LINK_OPTIONS EXCLUDE REGEX "^-Wl,-rpath,") + set_property(TARGET PkgConfig::SDL3 PROPERTY INTERFACE_LINK_OPTIONS "${SDL3_LINK_OPTIONS}") + endif() + pkg_check_modules(WAYLAND REQUIRED IMPORTED_TARGET wayland-client) + pkg_check_modules(FREETYPE REQUIRED IMPORTED_TARGET freetype2) + find_program(WAYLAND_SCANNER wayland-scanner REQUIRED) + set(OPENVR_ROOT "" CACHE PATH "Explicit standalone OpenVR v2.15.6 SDK root") + find_path(OPENVR_INCLUDE_DIR openvr.h HINTS "${OPENVR_ROOT}/headers" REQUIRED) + if(CMAKE_SYSTEM_PROCESSOR MATCHES "^(aarch64|arm64)$") + set(OPENVR_PLATFORM linuxarm64) + elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "^(x86_64|AMD64|amd64)$") + set(OPENVR_PLATFORM linux64) + else() + message(FATAL_ERROR "Native POC requires Linux ARM64 or x86-64") + endif() + find_library(OPENVR_LIBRARY openvr_api HINTS "${OPENVR_ROOT}/lib/${OPENVR_PLATFORM}" "${OPENVR_ROOT}/bin/${OPENVR_PLATFORM}" REQUIRED) + set(PROTOCOL "${CMAKE_CURRENT_SOURCE_DIR}/protocol/gamescope-input-method.xml") + add_custom_command(OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/gamescope-input-method-client.h" + COMMAND "${WAYLAND_SCANNER}" client-header "${PROTOCOL}" "${CMAKE_CURRENT_BINARY_DIR}/gamescope-input-method-client.h" + DEPENDS "${PROTOCOL}" VERBATIM) + add_custom_command(OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/gamescope-input-method.c" + COMMAND "${WAYLAND_SCANNER}" private-code "${PROTOCOL}" "${CMAKE_CURRENT_BINARY_DIR}/gamescope-input-method.c" + DEPENDS "${PROTOCOL}" VERBATIM) + target_sources(frameyap PRIVATE src/runtime.cpp src/overlay.cpp src/audio.cpp src/text_input.cpp + "${CMAKE_CURRENT_BINARY_DIR}/gamescope-input-method.c" "${CMAKE_CURRENT_BINARY_DIR}/gamescope-input-method-client.h") + target_include_directories(frameyap PRIVATE "${OPENVR_INCLUDE_DIR}" "${CMAKE_CURRENT_BINARY_DIR}") + target_link_libraries(frameyap PRIVATE frameyap_worker PkgConfig::SDL3 PkgConfig::WAYLAND PkgConfig::FREETYPE "${OPENVR_LIBRARY}") + target_compile_definitions(frameyap PRIVATE FRAMEYAP_NATIVE=1) + set_target_properties(frameyap PROPERTIES INSTALL_RPATH "$ORIGIN/../lib") +endif() +install(TARGETS frameyap RUNTIME DESTINATION bin) +install(DIRECTORY assets/ DESTINATION assets) +install(DIRECTORY python/frameyap DESTINATION python FILES_MATCHING PATTERN "*.py" PATTERN "__pycache__" EXCLUDE) include(CTest) if(BUILD_TESTING AND NOT CMAKE_CROSSCOMPILING) - add_test(NAME frame_dictation.cli - COMMAND "${CMAKE_COMMAND}" - "-DAPP=$" - "-DEXPECTED_VERSION=${PROJECT_VERSION}" - -P "${CMAKE_CURRENT_SOURCE_DIR}/tests/cli.cmake") - add_test(NAME frame_dictation.install_preflight - COMMAND sh "${CMAKE_CURRENT_SOURCE_DIR}/tests/install-preflight.sh" - "${CMAKE_CURRENT_SOURCE_DIR}/scripts/install-preflight.sh") + add_test(NAME frameyap.cli COMMAND "${CMAKE_COMMAND}" "-DAPP=$" + "-DEXPECTED_VERSION=${PROJECT_VERSION}" -P "${CMAKE_CURRENT_SOURCE_DIR}/tests/cli.cmake") + add_test(NAME frameyap.install_preflight COMMAND sh "${CMAKE_CURRENT_SOURCE_DIR}/tests/install-preflight.sh" + "${CMAKE_CURRENT_SOURCE_DIR}/scripts/install-preflight.sh") + add_executable(frameyap_core_test tests/core_test.cpp) + target_link_libraries(frameyap_core_test PRIVATE frameyap_core) + add_test(NAME frameyap.core COMMAND frameyap_core_test) + if(CMAKE_SYSTEM_NAME STREQUAL "Linux") + add_executable(frameyap_lock_test tests/lock_test.cpp) + target_include_directories(frameyap_lock_test PRIVATE src) + add_test(NAME frameyap.instance_lock COMMAND frameyap_lock_test) + endif() + add_executable(frameyap_gestures_test tests/gestures_test.cpp) + target_include_directories(frameyap_gestures_test PRIVATE src) + target_compile_options(frameyap_gestures_test PRIVATE -UNDEBUG) + add_test(NAME frameyap.gestures COMMAND frameyap_gestures_test) + find_package(Python3 3.10 COMPONENTS Interpreter) + if(FRAMEYAP_NATIVE) + pkg_check_modules(WAYLAND_SERVER IMPORTED_TARGET wayland-server) + if(WAYLAND_SERVER_FOUND) + find_package(Threads REQUIRED) + add_custom_command(OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/gamescope-input-method-server.h" + COMMAND "${WAYLAND_SCANNER}" server-header "${PROTOCOL}" "${CMAKE_CURRENT_BINARY_DIR}/gamescope-input-method-server.h" + DEPENDS "${PROTOCOL}" VERBATIM) + add_executable(frameyap_text_input_test tests/text_input_test.cpp src/text_input.cpp + "${CMAKE_CURRENT_BINARY_DIR}/gamescope-input-method.c" + "${CMAKE_CURRENT_BINARY_DIR}/gamescope-input-method-client.h" + "${CMAKE_CURRENT_BINARY_DIR}/gamescope-input-method-server.h") + target_include_directories(frameyap_text_input_test PRIVATE "${CMAKE_CURRENT_BINARY_DIR}") + target_link_libraries(frameyap_text_input_test PRIVATE frameyap_core PkgConfig::WAYLAND PkgConfig::WAYLAND_SERVER Threads::Threads) + add_test(NAME frameyap.text_input_protocol COMMAND frameyap_text_input_test) + set_tests_properties(frameyap.text_input_protocol PROPERTIES TIMEOUT 15) + endif() + endif() + if(Python3_FOUND) + add_test(NAME frameyap.python_worker COMMAND "${Python3_EXECUTABLE}" -m unittest discover -s tests -p test_worker.py) + add_test(NAME frameyap.installer COMMAND "${Python3_EXECUTABLE}" -m unittest discover -s tests -p test_installer.py) + set_tests_properties(frameyap.python_worker frameyap.installer PROPERTIES WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" TIMEOUT 60) + if(TARGET frameyap_worker) + add_executable(frameyap_worker_test tests/worker_test.cpp) + target_link_libraries(frameyap_worker_test PRIVATE frameyap_worker) + target_compile_options(frameyap_worker_test PRIVATE -UNDEBUG) + add_test(NAME frameyap.worker_ipc COMMAND frameyap_worker_test "${Python3_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/tests/test_worker.py") + set_tests_properties(frameyap.worker_ipc PROPERTIES TIMEOUT 30) + endif() + endif() endif() diff --git a/assets/actions.json b/assets/actions.json new file mode 100644 index 0000000..63d2039 --- /dev/null +++ b/assets/actions.json @@ -0,0 +1,27 @@ +{ + "default_bindings": [ + { "controller_type": "frame_controller", "binding_url": "bindings_frame_controller.json" }, + { "controller_type": "knuckles", "binding_url": "bindings_knuckles.json" } + ], + "action_sets": [ + { "name": "/actions/frameyap", "usage": "single" } + ], + "actions": [ + { "name": "/actions/frameyap/in/left_grip", "type": "boolean" }, + { "name": "/actions/frameyap/in/right_grip", "type": "boolean" }, + { "name": "/actions/frameyap/in/ptt", "type": "boolean" }, + { "name": "/actions/frameyap/in/cancel", "type": "boolean" }, + { "name": "/actions/frameyap/in/insert", "type": "boolean" }, + { "name": "/actions/frameyap/in/enter", "type": "boolean" } + ], + "localization": [{ + "language_tag": "en_US", + "/actions/frameyap": "FrameYap controls", + "/actions/frameyap/in/left_grip": "Double tap left grip: Enter (enabled only)", + "/actions/frameyap/in/right_grip": "Tap right grip briefly, then squeeze and hold to record; release to finish", + "/actions/frameyap/in/ptt": "Hold to record", + "/actions/frameyap/in/cancel": "Cancel recording or discard", + "/actions/frameyap/in/insert": "Insert reviewed text", + "/actions/frameyap/in/enter": "Explicit Enter" + }] +} diff --git a/assets/application.vrmanifest.in b/assets/application.vrmanifest.in new file mode 100644 index 0000000..362ea20 --- /dev/null +++ b/assets/application.vrmanifest.in @@ -0,0 +1,19 @@ +{ + "source": "builtin", + "applications": [ + { + "app_key": "local.frameyap.overlay", + "launch_type": "binary", + "binary_path_linux": "@ABSOLUTE_INSTALLED_FRAMEYAP_EXECUTABLE@", + "binary_path_linux_arm": "@ABSOLUTE_INSTALLED_FRAMEYAP_EXECUTABLE@", + "action_manifest_path": "@ABSOLUTE_INSTALLED_ACTIONS_JSON@", + "is_dashboard_overlay": true, + "strings": { + "en_us": { + "name": "FrameYap", + "description": "Local voice typing overlay" + } + } + } + ] +} diff --git a/assets/bindings_frame_controller.json b/assets/bindings_frame_controller.json new file mode 100644 index 0000000..88d25b5 --- /dev/null +++ b/assets/bindings_frame_controller.json @@ -0,0 +1,21 @@ +{ + "controller_type": "frame_controller", + "description": "Right grip: tap then hold to dictate. Left grip: double tap Enter. Remap other actions in SteamVR.", + "name": "FrameYap Steam Frame grips", + "bindings": { + "/actions/frameyap": { + "sources": [ + { + "path": "/user/hand/left/input/grip", + "mode": "button", + "inputs": { "click": { "output": "/actions/frameyap/in/left_grip" } } + }, + { + "path": "/user/hand/right/input/grip", + "mode": "button", + "inputs": { "click": { "output": "/actions/frameyap/in/right_grip" } } + } + ] + } + } +} diff --git a/assets/bindings_knuckles.json b/assets/bindings_knuckles.json new file mode 100644 index 0000000..b370f8b --- /dev/null +++ b/assets/bindings_knuckles.json @@ -0,0 +1,21 @@ +{ + "controller_type": "knuckles", + "description": "Example Index controller grip taps; other actions require user rebinding", + "name": "FrameYap sample grip bindings", + "bindings": { + "/actions/frameyap": { + "sources": [ + { + "path": "/user/hand/left/input/grip", + "mode": "button", + "inputs": { "click": { "output": "/actions/frameyap/in/left_grip" } } + }, + { + "path": "/user/hand/right/input/grip", + "mode": "button", + "inputs": { "click": { "output": "/actions/frameyap/in/right_grip" } } + } + ] + } + } +} diff --git a/protocol/gamescope-input-method.xml b/protocol/gamescope-input-method.xml new file mode 100644 index 0000000..10a4190 --- /dev/null +++ b/protocol/gamescope-input-method.xml @@ -0,0 +1,214 @@ + + + + + Copyright © 2008-2011 Kristian Høgsberg + Copyright © 2010-2011 Intel Corporation + Copyright © 2012-2013 Collabora, Ltd. + Copyright © 2012, 2013 Intel Corporation + Copyright © 2015, 2016 Jan Arne Petersen + Copyright © 2017, 2018 Red Hat, Inc. + Copyright © 2018 Purism SPC + Copyright © 2021 Valve Corporation + + Permission is hereby granted, free of charge, to any person obtaining a + copy of this software and associated documentation files (the "Software"), + to deal in the Software without restriction, including without limitation + the rights to use, copy, modify, merge, publish, distribute, sublicense, + and/or sell copies of the Software, and to permit persons to whom the + Software is furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice (including the next + paragraph) shall be included in all copies or substantial portions of the + Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + DEALINGS IN THE SOFTWARE. + + + + This protocol allows applications to act as input methods for compositors. + + An input method context is used to manage the state of the input method. + + Text strings are UTF-8 encoded, their indices and lengths are in bytes. + + This iteration of the protocol is a subset of this work-in-progress proposal: + https://gitlab.freedesktop.org/wayland/wayland-protocols/-/merge_requests/112 + + This is a private Gamescope protocol. Regular Wayland clients must not use + it. + + + + + The input method manager allows the client to become the input method on + a chosen seat. + + No more than one input method must be associated with any seat at any + given time. + + + + + Destroys the gamescope_input_method_manager object. + + The gamescope_input_method objects originating from it remain valid. + + + + + + Create a new gamescope_input_method object associated with a given + seat. + + + + + + + + + An input method object allows for clients to compose text. + + The objects connects the client to a text input in an application, and + lets the client to serve as an input method for a seat. + + There must be no more than one input method object per seat. + + + + + The input method ceased to be available. + + The compositor must issue this event as the only event on the object if + there was another input_method object associated with the same seat at + the time of its creation. + + The compositor must issue this request when the object is no longer + useable, e.g. due to seat removal. + + The input method context becomes inert and should be destroyed. Any + further requests and events except for the destroy request must be + ignored. + + + + + + Notify the client that the compositor state has changed. + + The serial is used to synchronize compositor state changes and client + state changes. The serial allows the compositor to ignore requests + meant for a previous text input. + + In the future, will be used to atomically apply compositor state + changes sent to the client (but currently there's no such state). + + + + + + + Destroys the gamescope_text_input object. + + + + + + Apply state changes from set_string and set_action requests. + + The state relating to these events is double-buffered, and each one + modifies the pending state. This request replaces the current state + with the pending state. + + The client must set the serial to the last received serial in the done + event. + + + + + + + Send the string text for insertion to the application. + + Inserts a string at current cursor position (see commit event + sequence). The string to commit could be either just a single character + after a key press or the result of some composing. + + Values set with this event are double-buffered. They must be applied + and reset to initial on the next gamescope_text_input.commit request. + + The initial value of text is an empty string. + + + + + + + A possible action to perform on a text input. + + The backspace and delete actions should be handled in a similar manner + to backpace and delete keys being pressed on a keyboard. + + + + + + + + + + + + + + Set the action to be performed on commit. + + Values set with this event are double-buffered. They must be applied + and reset to initial on the next gamescope_text_input.commit request. + + + + + + + + + Describes the physical state of a button that produced the button + event. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/audio.cpp b/src/audio.cpp new file mode 100644 index 0000000..ae1b8b2 --- /dev/null +++ b/src/audio.cpp @@ -0,0 +1,72 @@ +#include "audio.hpp" +#include +#include +#include +#include +#include +#include + +namespace frameyap { +namespace { void check(bool result) { if (!result) throw std::runtime_error(std::string("Microphone: ") + SDL_GetError()); } } +Audio::~Audio() { cancel(); } +void Audio::start() { + if (stream_) throw std::runtime_error("Already recording"); + check(SDL_InitSubSystem(SDL_INIT_AUDIO)); initialized_ = true; + SDL_AudioSpec spec{SDL_AUDIO_F32, 1, 16000}; + stream_ = SDL_OpenAudioDeviceStream(SDL_AUDIO_DEVICE_DEFAULT_RECORDING, &spec, nullptr, nullptr); + if (!stream_) { cancel(); check(false); } + pcm_.clear(); pcm_.reserve(320000); + started_ = last_data_ = std::chrono::steady_clock::now(); + try { check(SDL_ResumeAudioStreamDevice(stream_)); } + catch (...) { cancel(); throw; } +} +void Audio::drain() { + std::array chunk{}; + while (pcm_.size() < 320000) { + int available = SDL_GetAudioStreamAvailable(stream_); + check(available >= 0); + if (available < int(sizeof(float))) break; + const int count = static_cast(std::min({chunk.size(), std::size_t(available) / sizeof(float), 320000 - pcm_.size()})); + int bytes = SDL_GetAudioStreamData(stream_, chunk.data(), count * sizeof(float)); + check(bytes >= 0); + if (bytes == 0) break; + if (bytes % sizeof(float)) throw std::runtime_error("Misaligned microphone samples"); + for (int i = 0; i < bytes / int(sizeof(float)); ++i) + if (!std::isfinite(chunk[i])) throw std::runtime_error("Invalid microphone samples"); + pcm_.insert(pcm_.end(), chunk.begin(), chunk.begin() + bytes / sizeof(float)); + last_data_ = std::chrono::steady_clock::now(); + } +} +bool Audio::poll() { + if (!stream_) return false; + SDL_Event event; + while (SDL_PollEvent(&event)) { + if (event.type == SDL_EVENT_AUDIO_DEVICE_REMOVED && event.adevice.recording) + throw std::runtime_error("Recording device removed; clip discarded"); + } + drain(); + if (std::chrono::steady_clock::now() - last_data_ > std::chrono::seconds(2)) + throw std::runtime_error("Microphone stalled; clip discarded"); + return pcm_.size() == 320000 || seconds() >= 20; +} +std::vector Audio::finish() { + if (!stream_) return {}; + check(SDL_PauseAudioStreamDevice(stream_)); + check(SDL_FlushAudioStream(stream_)); + drain(); + auto result = std::move(pcm_); + cancel(); + return result; +} +void Audio::cancel() { + if (stream_) SDL_DestroyAudioStream(stream_); + stream_ = nullptr; + // Best effort clearing of the owned buffer; no persistent recording/logging. + std::fill(pcm_.begin(), pcm_.end(), 0.0f); pcm_.clear(); + if (initialized_) SDL_QuitSubSystem(SDL_INIT_AUDIO); + initialized_ = false; +} +int Audio::seconds() const { + return stream_ ? int(std::chrono::duration_cast(std::chrono::steady_clock::now() - started_).count()) : 0; +} +} diff --git a/src/audio.hpp b/src/audio.hpp new file mode 100644 index 0000000..c80453b --- /dev/null +++ b/src/audio.hpp @@ -0,0 +1,23 @@ +#pragma once +#include +#include +struct SDL_AudioStream; +namespace frameyap { +class Audio { +public: + Audio() = default; + ~Audio(); + void start(); + bool poll(); // true at 20s limit; throws if capture was lost/stalled + std::vector finish(); + void cancel(); + bool recording() const { return stream_ != nullptr; } + int seconds() const; +private: + void drain(); + SDL_AudioStream* stream_ = nullptr; + bool initialized_ = false; + std::vector pcm_; + std::chrono::steady_clock::time_point started_, last_data_; +}; +} diff --git a/src/gestures.hpp b/src/gestures.hpp new file mode 100644 index 0000000..d5f1d69 --- /dev/null +++ b/src/gestures.hpp @@ -0,0 +1,107 @@ +#pragma once + +#include + +namespace frameyap { +// Feed monotonic samples. Inactive, tracking/focus loss and reconnect require an +// observed neutral sample before any new press can be interpreted. +class DoubleTap { +public: + using Clock = std::chrono::steady_clock; + bool update(bool active, bool down, bool enabled, Clock::time_point now) { + if (!active || !enabled) { reset(); return false; } + if (!armed_) { + if (!down) { armed_ = true; previous_ = false; } + return false; + } + if (down == previous_) return false; + previous_ = down; + if (down) { + pressed_ = now; + if (waiting_ && now >= released_ && now - released_ <= std::chrono::milliseconds(350)) { + waiting_ = false; + // The second tap must itself be short; report on its release. + second_ = true; + } else { waiting_ = false; second_ = false; } + return false; + } + if (now < pressed_ || now - pressed_ > std::chrono::milliseconds(250)) { + waiting_ = second_ = false; + return false; + } + if (second_) { second_ = false; waiting_ = false; return true; } + waiting_ = true; + released_ = now; + return false; + } + void reset() { armed_ = previous_ = waiting_ = second_ = false; } +private: + bool armed_ = false, previous_ = false, waiting_ = false, second_ = false; + Clock::time_point pressed_{}, released_{}; +}; + +// First short right-grip tap (<=250ms), then second down within 350ms +// begins capture immediately; second release ends it, regardless of its length. +// Any loss during capture cancels, and reconnect requires an observed neutral. +class GripRecord { +public: + using Clock = std::chrono::steady_clock; + enum class Change { None, Begin, End, Cancel }; + Change update(bool active, bool down, Clock::time_point now) { + if (!active) return reset(); + if (!armed_) { + if (!down) { armed_ = true; previous_ = false; } + return Change::None; + } + if (down == previous_) return Change::None; + previous_ = down; + if (down) { + if (waiting_ && now >= released_ && now - released_ <= std::chrono::milliseconds(350)) { + waiting_ = false; + holding_ = true; + return Change::Begin; + } + waiting_ = false; + pressed_ = now; + return Change::None; + } + if (holding_) { holding_ = false; return Change::End; } + if (now >= pressed_ && now - pressed_ <= std::chrono::milliseconds(250)) { + waiting_ = true; + released_ = now; + } else waiting_ = false; + return Change::None; + } + Change reset() { + bool was_holding = holding_; + armed_ = previous_ = waiting_ = holding_ = false; + return was_holding ? Change::Cancel : Change::None; + } +private: + bool armed_ = false, previous_ = false, waiting_ = false, holding_ = false; + Clock::time_point pressed_{}, released_{}; +}; + +// Edge detector for hold-to-talk: first observed active state never starts recording. +class NeutralEdge { +public: + enum class Change { None, Down, Up }; + Change update(bool active, bool down) { + if (!active) { reset(); return Change::None; } + if (!armed_) { + if (!down) armed_ = true; + return Change::None; + } + if (down == previous_) return Change::None; + previous_ = down; + return down ? Change::Down : Change::Up; + } + bool reset() { + const bool was_down = armed_ && previous_; + armed_ = previous_ = false; + return was_down; + } +private: + bool armed_ = false, previous_ = false; +}; +} // namespace frameyap diff --git a/src/instance_lock.hpp b/src/instance_lock.hpp new file mode 100644 index 0000000..4c626e8 --- /dev/null +++ b/src/instance_lock.hpp @@ -0,0 +1,40 @@ +#pragma once +#include +#include +#include +#include +#include +#include +#include + +namespace frameyap { +// Shared contract with install.sh. Held for the entire runtime/check/registration +// operation, never acquired by default/help/version or hardware-free tests. +class InstanceLock { +public: + InstanceLock() { + const char* installed = std::getenv("FRAMEYAP_INSTALL_ROOT"); + const char* data = std::getenv("XDG_DATA_HOME"); + const char* home = std::getenv("HOME"); + if ((!installed || !*installed) && (!data || !*data) && (!home || !*home)) + throw std::runtime_error("HOME or XDG_DATA_HOME required"); + auto root = installed && *installed ? std::filesystem::path(installed) : + std::filesystem::path(data && *data ? data : std::string(home) + "/.local/share") / "frameyap"; + if (!root.is_absolute() || std::filesystem::is_symlink(root)) + throw std::runtime_error("Data directory must be absolute and not a symlink"); + std::filesystem::create_directories(root); + fd_ = open((root / ".lock").c_str(), O_CREAT | O_RDWR | O_CLOEXEC | O_NOFOLLOW, 0600); + struct stat st{}; + if (fd_ < 0 || fstat(fd_, &st) || !S_ISREG(st.st_mode) || st.st_uid != getuid() || flock(fd_, LOCK_EX | LOCK_NB)) { + if (fd_ >= 0) close(fd_); + fd_ = -1; + throw std::runtime_error("FrameYap already running/installing, or unsafe install lock"); + } + } + ~InstanceLock() { if (fd_ >= 0) close(fd_); } + InstanceLock(const InstanceLock&) = delete; + InstanceLock& operator=(const InstanceLock&) = delete; +private: + int fd_ = -1; +}; +} diff --git a/src/main.cpp b/src/main.cpp index 7db597e..762076d 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -1,24 +1,108 @@ #include #include - +#include +#include +#ifdef FRAMEYAP_NATIVE +#include "runtime.hpp" +#include "overlay.hpp" +#include "text_input.hpp" +#include "instance_lock.hpp" +#include +#include +#include +#endif namespace { void help() { - std::cout << "Frame Dictation — standalone Steam Frame voice typing\n" - "Scaffold only: overlay, microphone, ASR and input are not implemented.\n\n" - "Usage: frame-dictation [--help | --version]\n" - "No device access or background processes are started.\n"; + std::cout << "FrameYap — standalone Steam Frame voice typing POC\n" + "Usage: frameyap [--help | --version]\n" + " --run --assets DIR --font FILE --worker FILE --model DIR\n" + " [--python FILE] [--threads 2|4] [--socket gamescope-0] [--head]\n" + " --register MANIFEST [--autostart] | --unregister MANIFEST\n" + " --check-input [--socket gamescope-0] (discovery only, no typing)\n" + " --check-overlay --assets DIR --font FILE [--head] (5s, no mic/input)\n" + " --check-controls --assets DIR --font FILE [--head] (30s, gestures only)\n\n" + "Right grip: double-tap, hold second squeeze to speak, release to review.\n" + "Left grip: double-tap for explicit Enter. Bindings are remappable.\n" + "Review first: Insert approves current focus; never automatic Enter.\n" + "No device access unless an explicit runtime/check/registration mode is used.\n"; +#ifndef FRAMEYAP_NATIVE + std::cout << "This offline build has no hardware backends; enable FRAMEYAP_NATIVE to run.\n"; +#endif } } - int main(int argc, char** argv) { - if (argc == 1 || (argc == 2 && std::string_view(argv[1]) == "--help")) { - help(); - return 0; - } + if (argc == 1 || (argc == 2 && std::string_view(argv[1]) == "--help")) { help(); return 0; } if (argc == 2 && std::string_view(argv[1]) == "--version") { - std::cout << "frame-dictation " << FRAME_DICTATION_VERSION << " (scaffold)\n"; - return 0; + std::cout << "frameyap " << FRAMEYAP_VERSION << "\n"; return 0; } - std::cerr << "Unsupported arguments. Use --help; runtime features are not implemented.\n"; +#ifdef FRAMEYAP_NATIVE + try { + const std::string mode = argv[1]; + if (mode == "--register" || mode == "--unregister") { + if (argc == 3 || (mode == "--register" && argc == 4 && std::string_view(argv[3]) == "--autostart")) { + frameyap::InstanceLock lock; + return frameyap::registration(argv[2], mode == "--unregister", argc == 4); + } + } else if (mode == "--run" || mode == "--check-input" || mode == "--check-overlay" || mode == "--check-controls") { + frameyap::Options options; + const char* socket = std::getenv("GAMESCOPE_WAYLAND_DISPLAY"); + options.socket = socket && *socket ? socket : "gamescope-0"; + for (int i = 2; i < argc; ++i) { + std::string_view arg = argv[i]; + if (arg == "--head" && mode != "--check-input") { options.hand = false; continue; } + if (i + 1 == argc) throw std::runtime_error("Missing option value"); + std::string value = argv[++i]; + if (arg == "--socket" && mode != "--check-overlay") options.socket = value; + else if (arg == "--assets" && mode != "--check-input") options.assets = value; + else if (arg == "--font" && mode != "--check-input") options.font = value; + else if (arg == "--worker" && mode == "--run") options.worker = value; + else if (arg == "--python" && mode == "--run") options.python = value; + else if (arg == "--model" && mode == "--run") options.model = value; + else if (arg == "--threads" && mode == "--run" && (value == "2" || value == "4")) options.threads = value == "2" ? 2 : 4; + else throw std::runtime_error("Unsupported option/value"); + } + if (mode == "--check-input") { + frameyap::InstanceLock lock; + frameyap::TextInput input(options.socket); + std::cout << "Gamescope IME v2 ready; no input delivered.\n"; return 0; + } + if (options.assets.empty() || options.font.empty()) throw std::runtime_error("--assets and --font required"); + if (mode == "--check-overlay" || mode == "--check-controls") { + frameyap::InstanceLock lock; + frameyap::Overlay overlay(options.assets, options.font, options.hand); + overlay.draw({"FrameYap five-second visual check", "No audio captured. No text or Enter delivered.", "Controls inactive during this check.", false, false}); + for (vr::TrackedDeviceIndex_t i = 0; i < vr::k_unMaxTrackedDeviceCount; ++i) { + if (vr::VRSystem()->GetTrackedDeviceClass(i) != vr::TrackedDeviceClass_Controller) continue; + char type[256]{}, profile[512]{}; + vr::VRSystem()->GetStringTrackedDeviceProperty(i, vr::Prop_ControllerType_String, type, sizeof(type)); + vr::VRSystem()->GetStringTrackedDeviceProperty(i, vr::Prop_InputProfilePath_String, profile, sizeof(profile)); + std::cout << "Controller type=" << type << " profile=" << profile << '\n'; + } + if (mode == "--check-controls") { + const auto end = std::chrono::steady_clock::now() + std::chrono::seconds(30); + std::string status; + bool held = false; + while (std::chrono::steady_clock::now() < end) { + overlay.draw({"Controls check ONLY - no mic or typing", "Tap then hold right grip; double-tap left grip.", + held ? "HOLD gesture recognized. Release to finish." : "Try grips or click buttons. Auto-closes after 30 seconds.", true, held}); + for (auto action : overlay.poll()) { + if (action == frameyap::UiAction::Quit) return 0; + if (action == frameyap::UiAction::BeginRecord) held = true; + if (action == frameyap::UiAction::EndRecord || action == frameyap::UiAction::Cancel) held = false; + std::cout << "Control action=" << static_cast(action) << " (diagnostic only)" << std::endl; + } + auto next = overlay.controls_status(); + if (next != status) { status = next; std::cout << status << std::endl; } + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + } + } else std::this_thread::sleep_for(std::chrono::seconds(5)); + std::cout << "Overlay APIs accepted panel; physical visibility requires human confirmation.\n"; return 0; + } + if (options.worker.empty() || options.model.empty()) throw std::runtime_error("--worker and --model required"); + return frameyap::run(options); + } + } catch (const std::exception& e) { std::cerr << "FrameYap: " << e.what() << '\n'; return 1; } +#endif + std::cerr << "Unsupported arguments or runtime disabled in this build. Use --help.\n"; return 2; } diff --git a/src/overlay.cpp b/src/overlay.cpp new file mode 100644 index 0000000..fc75ad0 --- /dev/null +++ b/src/overlay.cpp @@ -0,0 +1,457 @@ +#include "overlay.hpp" +#include "gestures.hpp" + +#include +#include +#include FT_FREETYPE_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace frameyap { +namespace { +// The ARM64 SDK defaults to /data/work on Frame. Respect explicit overrides, +// otherwise use the user's standard OpenVR registry when it exists. +void configure_registry() { + if (std::getenv("VR_PATHREG_OVERRIDE")) return; + const char* config = std::getenv("XDG_CONFIG_HOME"); + const char* home = std::getenv("HOME"); + if ((!config || !*config) && !home) return; + auto root = std::filesystem::path(config && *config ? config : std::string(home) + "/.config"); + auto registry = root / "openvr/openvrpaths.vrpath"; + if (root.is_absolute() && std::filesystem::is_regular_file(registry)) + ::setenv("VR_PATHREG_OVERRIDE", registry.c_str(), 0); +} +constexpr int W = 900, H = 500; +struct Rect { int x, y, w, h; bool contains(int px, int py) const { + return px >= x && px < x + w && py >= y && py < y + h; +} }; +struct Button { Rect bounds; const char* label; UiAction action; }; +constexpr std::array buttons{{ + {{24, 422, 130, 58}, "Prev", UiAction::Toggle}, // handled as page navigation, not a toggle + {{168, 422, 130, 58}, "Record", UiAction::Record}, + {{312, 422, 130, 58}, "Cancel", UiAction::Cancel}, + {{456, 422, 130, 58}, "Insert", UiAction::Insert}, + {{600, 422, 130, 58}, "Enter", UiAction::Enter}, + {{744, 422, 130, 58}, "Quit", UiAction::Quit}, +}}; +void overlay_check(vr::EVROverlayError err, vr::IVROverlay* api, const char* op) { + if (err != vr::VROverlayError_None) + throw std::runtime_error(std::string(op) + ": " + api->GetOverlayErrorNameFromEnum(err)); +} +std::filesystem::path absolute_file(const std::filesystem::path& p) { + auto result = std::filesystem::absolute(p); + if (!std::filesystem::is_regular_file(result)) throw std::runtime_error("Missing file: " + result.string()); + return result; +} +// A strict, bounded UTF-8 decoder: malformed input is shown as replacement glyphs. +uint32_t next_codepoint(std::string_view s, size_t& i) { + const unsigned char a = static_cast(s[i++]); + if (a < 0x80) return a; + int count = a >= 0xc2 && a <= 0xdf ? 1 : a >= 0xe0 && a <= 0xef ? 2 : a >= 0xf0 && a <= 0xf4 ? 3 : 0; + if (!count || i + count > s.size()) return 0xfffd; + uint32_t value = a & (count == 1 ? 0x1f : count == 2 ? 0x0f : 0x07); + for (int k = 0; k < count; ++k) { + unsigned char b = static_cast(s[i + k]); + if ((b & 0xc0) != 0x80) return 0xfffd; + value = (value << 6) | (b & 0x3f); + } + if ((count == 1 && value < 0x80) || (count == 2 && value < 0x800) || + (count == 3 && value < 0x10000) || (value >= 0xd800 && value <= 0xdfff) || value > 0x10ffff) + return 0xfffd; + i += count; + return value; +} +} // namespace + +struct Overlay::Impl { + vr::IVRSystem* system = nullptr; + vr::IVROverlay* overlay = nullptr; + vr::IVRInput* input = nullptr; + vr::VROverlayHandle_t handle = vr::k_ulOverlayHandleInvalid; + vr::VRActionSetHandle_t action_set = vr::k_ulInvalidActionSetHandle; + std::array actions{}; + FT_Library library = nullptr; + FT_Face face = nullptr; + std::vector pixels = std::vector(W * H * 4); + Panel panel; + bool hand = false; + vr::TrackedDeviceIndex_t anchor = vr::k_unTrackedDeviceIndexInvalid; + DoubleTap left; + GripRecord right; + std::array edges{}; + std::array pressed_button{{-1, -1}}; + std::array poses{}; + std::vector pages; + size_t page = 0; + bool drawn = false; + bool grip_capture = false, ptt_capture = false; + bool focus = true; + static constexpr Rect next_page{754, 292, 112, 30}; + + Impl(const std::string& assets, const std::string& font, bool attach) : hand(attach) { + const auto action_path = absolute_file(std::filesystem::path(assets) / "actions.json"); + absolute_file(std::filesystem::path(assets) / "bindings_knuckles.json"); + const auto font_path = absolute_file(font); + if (FT_Init_FreeType(&library)) throw std::runtime_error("FreeType initialization failed"); + try { + if (FT_New_Face(library, font_path.c_str(), 0, &face) || FT_Set_Pixel_Sizes(face, 0, 31)) + throw std::runtime_error("Could not load specified font"); + configure_registry(); + vr::EVRInitError err = vr::VRInitError_None; + system = vr::VR_Init(&err, vr::VRApplication_Overlay); + if (err != vr::VRInitError_None || !system) + throw std::runtime_error(std::string("OpenVR overlay init: ") + vr::VR_GetVRInitErrorAsEnglishDescription(err)); + overlay = vr::VROverlay(); + input = vr::VRInput(); + if (!overlay || !input) throw std::runtime_error("OpenVR overlay/input interface unavailable"); + // The manifest launches a shell launcher, then execs this binary. + // Associate manually launched instances with our registered app key too. + auto* applications = vr::VRApplications(); + if (applications && applications->IsApplicationInstalled("local.frameyap.overlay") && + applications->IdentifyApplication(static_cast(::getpid()), "local.frameyap.overlay") != vr::VRApplicationError_None) + throw std::runtime_error("Could not identify the registered FrameYap application"); + if (input->SetActionManifestPath(action_path.c_str()) != vr::VRInputError_None) + throw std::runtime_error("Could not set OpenVR action manifest path"); + if (input->GetActionSetHandle("/actions/frameyap", &action_set) != vr::VRInputError_None) + throw std::runtime_error("Could not find FrameYap action set"); + constexpr std::array names{{"left_grip", "right_grip", "ptt", "cancel", "insert", "enter"}}; + for (size_t i = 0; i < names.size(); ++i) + if (input->GetActionHandle((std::string("/actions/frameyap/in/") + names[i]).c_str(), &actions[i]) != vr::VRInputError_None) + throw std::runtime_error(std::string("Could not find action ") + names[i]); + overlay_check(overlay->CreateOverlay("local.frameyap.overlay.panel", "FrameYap", &handle), overlay, "CreateOverlay"); + overlay_check(overlay->SetOverlayWidthInMeters(handle, 0.85f), overlay, "SetOverlayWidthInMeters"); + overlay_check(overlay->SetOverlayInputMethod(handle, vr::VROverlayInputMethod_Mouse), overlay, "SetOverlayInputMethod"); + // Let dashboard lasers reach our controls without forcing global + // laser-mouse mode over a running scene. + overlay_check(overlay->SetOverlayFlag(handle, vr::VROverlayFlags_VisibleInDashboard, true), overlay, "VisibleInDashboard"); + vr::HmdVector2_t mouse_scale{{float(W), float(H)}}; + overlay_check(overlay->SetOverlayMouseScale(handle, &mouse_scale), overlay, "SetOverlayMouseScale"); + place(); + draw(Panel{"Disabled", "", "Record to start local worker", false, false}); + } catch (...) { cleanup(); throw; } + } + ~Impl() { cleanup(); } + void cleanup() { + if (overlay && handle != vr::k_ulOverlayHandleInvalid) { + overlay->HideOverlay(handle); + overlay->DestroyOverlay(handle); + handle = vr::k_ulOverlayHandleInvalid; + } + if (system) { vr::VR_Shutdown(); system = nullptr; overlay = nullptr; input = nullptr; } + if (face) { FT_Done_Face(face); face = nullptr; } + if (library) { FT_Done_FreeType(library); library = nullptr; } + } + void place() { + vr::TrackedDeviceIndex_t target = vr::k_unTrackedDeviceIndex_Hmd; + if (hand) { + const auto left_index = system->GetTrackedDeviceIndexForControllerRole(vr::TrackedControllerRole_LeftHand); + if (left_index != vr::k_unTrackedDeviceIndexInvalid && left_index < poses.size() && + poses[left_index].bPoseIsValid && system->IsTrackedDeviceConnected(left_index)) + target = left_index; + } + if (anchor == target) return; + vr::HmdMatrix34_t transform{{{1.f, 0.f, 0.f, target == vr::k_unTrackedDeviceIndex_Hmd ? 0.f : 0.13f}, + {0.f, 1.f, 0.f, target == vr::k_unTrackedDeviceIndex_Hmd ? -0.16f : 0.12f}, + {0.f, 0.f, 1.f, target == vr::k_unTrackedDeviceIndex_Hmd ? -1.05f : -0.18f}}}; + overlay_check(overlay->SetOverlayTransformTrackedDeviceRelative(handle, target, &transform), overlay, "SetOverlayTransformTrackedDeviceRelative"); + anchor = target; + } + void rect(Rect r, std::array c) { + for (int y = std::max(r.y, 0); y < std::min(H, r.y + r.h); ++y) + for (int x = std::max(r.x, 0); x < std::min(W, r.x + r.w); ++x) { + const size_t p = (size_t(y) * W + x) * 4; + std::copy(c.begin(), c.end(), pixels.begin() + p); + } + } + void text(std::string_view s, int x, int baseline, int max_x, int max_y, int max_bytes) { + size_t i = 0; + int used = 0; + while (i < s.size() && used < max_bytes && baseline < max_y) { + const size_t before = i; + const auto cp = next_codepoint(s, i); + used += int(i - before); + if (cp == '\n') { x = 28; baseline += 39; continue; } + if (cp == '\r' || cp == '\t') continue; + if (FT_Load_Char(face, cp, FT_LOAD_RENDER)) continue; + const auto glyph = face->glyph; + if (x + (glyph->advance.x >> 6) >= max_x) { x = 28; baseline += 39; } + if (baseline >= max_y) break; + const auto& b = glyph->bitmap; + for (unsigned row = 0; row < b.rows; ++row) + for (unsigned col = 0; col < b.width; ++col) { + int px = x + glyph->bitmap_left + int(col); + int py = baseline - glyph->bitmap_top + int(row); + if (px < 0 || px >= W || py < 0 || py >= max_y) continue; + const unsigned char alpha = b.buffer[int(row) * b.pitch + int(col)]; + auto* dst = pixels.data() + (size_t(py) * W + px) * 4; + for (int c = 0; c < 3; ++c) + dst[c] = static_cast((unsigned(dst[c]) * (255 - alpha) + 240u * alpha) / 255); + } + x += glyph->advance.x >> 6; + } + } + bool available(UiAction a) const { + switch (a) { + case UiAction::Toggle: return page > 0; + case UiAction::Quit: case UiAction::Record: case UiAction::Cancel: return true; + case UiAction::BeginRecord: case UiAction::EndRecord: return true; + case UiAction::Enter: return panel.enabled; + case UiAction::Insert: return panel.enabled && !panel.transcript.empty(); + } + return false; + } + // Split by rendered glyph advances, never by byte count or arbitrary characters. + // UTF-8 slices remain intact; all bytes (including invalid UTF-8) stay visible. + std::vector paginate(std::string_view s, int lines, int right) { + std::vector out; + size_t start = 0, i = 0; + int x = 28, line = 0; + while (i < s.size()) { + size_t before = i; + auto cp = next_codepoint(s, i); + if (cp == '\n') { + if (++line == lines) { + out.emplace_back(s.substr(start, i - start)); + start = i; line = 0; + } + x = 28; + continue; + } + int advance = 0; + if (cp != '\r' && cp != '\t' && !FT_Load_Char(face, cp, FT_LOAD_DEFAULT)) + advance = face->glyph->advance.x >> 6; + if (x + advance >= right && x != 28) { + if (++line == lines) { + out.emplace_back(s.substr(start, before - start)); + start = before; line = 0; + } else { + // Add a line break in the displayed page while keeping source intact. + // The renderer's own width wrapping uses the same advance rule. + } + x = 28; + } + x += advance; + } + if (start < s.size() || out.empty()) out.emplace_back(s.substr(start)); + return out; + } + void draw(const Panel& p) { + if (p.transcript != panel.transcript || pages.empty()) { + pages = paginate(p.transcript.empty() ? "Transcript preview" : p.transcript, 3, W - 28); + page = 0; + drawn = false; + } + if (drawn && p.status == panel.status && p.transcript == panel.transcript && + p.detail == panel.detail && p.enabled == panel.enabled && p.recording == panel.recording) return; + panel = p; + rect({0, 0, W, H}, {20, 26, 39, 255}); + rect({0, 0, W, 12}, {static_cast(p.recording ? 220 : 55), + static_cast(p.recording ? 82 : 135), 72, 255}); + text(p.enabled ? "FrameYap | Enabled" : "FrameYap | Disabled", 28, 52, W - 20, 72, 128); + const auto status_lines = paginate(p.status, 2, 620); + text(status_lines[0], 28, 96, 620, 150, int(status_lines[0].size())); + if (status_lines.size() > 1) text("[status truncated]", 635, 140, W - 28, 150, 32); + rect({20, 153, W - 40, 171}, {37, 46, 63, 255}); + text(pages[page], 28, 190, W - 28, 290, int(pages[page].size())); + const std::string page_label = "Page " + std::to_string(page + 1) + " / " + std::to_string(pages.size()); + text(page_label, 383, 315, 720, 324, 64); + rect(next_page, page + 1 < pages.size() ? std::array{56, 94, 129, 255} + : std::array{49, 52, 60, 255}); + text("Next", 767, 316, 860, 324, 8); + const auto detail_lines = paginate(p.detail, 2, 620); + text(detail_lines[0], 28, 355, 620, 412, int(detail_lines[0].size())); + if (detail_lines.size() > 1) + text("[detail truncated]", 635, 405, W - 28, 412, 32); + for (const auto& button : buttons) { + const bool enabled = available(button.action); + rect(button.bounds, enabled ? std::array{56, 94, 129, 255} + : std::array{49, 52, 60, 255}); + text(button.label, button.bounds.x + 10, button.bounds.y + 38, + button.bounds.x + button.bounds.w - 5, H, 64); + } + overlay_check(overlay->SetOverlayRaw(handle, pixels.data(), W, H, 4), overlay, "SetOverlayRaw"); + overlay_check(overlay->ShowOverlay(handle), overlay, "ShowOverlay"); + drawn = true; + } + void reset_input(std::vector& result) { + left.reset(); + const bool lost_grip = right.reset() == GripRecord::Change::Cancel || grip_capture; + grip_capture = false; + const bool lost_ptt = edges[0].reset() || ptt_capture; + ptt_capture = false; + if (lost_grip || lost_ptt) result.push_back(UiAction::Cancel); + for (size_t i = 1; i < edges.size(); ++i) edges[i].reset(); + pressed_button.fill(-1); + } + // Bound actions are accepted only with a connected tracked source. A held input + // following loss of activity must return to neutral before generating an edge. + std::pair digital(size_t index) { + vr::InputDigitalActionData_t data{}; + if (input->GetDigitalActionData(actions[index], &data, sizeof(data), vr::k_ulInvalidInputValueHandle) != vr::VRInputError_None || + !data.bActive || data.activeOrigin == vr::k_ulInvalidInputValueHandle) return {false, false}; + vr::InputOriginInfo_t origin{}; + if (input->GetOriginTrackedDeviceInfo(data.activeOrigin, &origin, sizeof(origin)) != vr::VRInputError_None || + origin.trackedDeviceIndex == vr::k_unTrackedDeviceIndexInvalid || + origin.trackedDeviceIndex >= poses.size() || + !system->IsTrackedDeviceConnected(origin.trackedDeviceIndex) || + !poses[origin.trackedDeviceIndex].bPoseIsValid) return {false, false}; + if (index < 2) { + const auto role = index == 0 ? vr::TrackedControllerRole_LeftHand : vr::TrackedControllerRole_RightHand; + if (system->GetTrackedDeviceIndexForControllerRole(role) != origin.trackedDeviceIndex) return {false, false}; + } + return {true, data.bState}; + } + std::vector poll() { + std::vector result; + system->GetDeviceToAbsoluteTrackingPose(vr::TrackingUniverseStanding, 0, poses.data(), uint32_t(poses.size())); + place(); + vr::VREvent_t event{}; + while (system->PollNextEvent(&event, sizeof(event))) { + if (event.eventType == vr::VREvent_Quit) result.push_back(UiAction::Quit); + if (event.eventType == vr::VREvent_TrackedDeviceDeactivated || + event.eventType == vr::VREvent_InputFocusCaptured) reset_input(result); + } + while (overlay->PollNextOverlayEvent(handle, &event, sizeof(event))) { + switch (event.eventType) { + case vr::VREvent_Quit: case vr::VREvent_OverlayClosed: + result.push_back(UiAction::Quit); break; + case vr::VREvent_OverlayHidden: + focus = false; reset_input(result); + if (panel.recording) result.push_back(UiAction::Cancel); + break; + case vr::VREvent_OverlayShown: + focus = true; reset_input(result); break; + case vr::VREvent_OverlayGamepadFocusLost: + case vr::VREvent_OverlayFocusChanged: + // Pointer/gamepad overlay focus is not OS keyboard focus or action + // activity. Cancel the old gesture, then require neutral rearm; + // do not permanently latch global grip actions off. + reset_input(result); break; + case vr::VREvent_MouseButtonDown: { + const auto& m = event.data.mouse; + if (m.button != vr::VRMouseButton_Left || m.cursorIndex >= pressed_button.size()) break; + const int x = int(m.x), y = H - int(m.y); + pressed_button[m.cursorIndex] = -1; + for (size_t i = 0; i < buttons.size(); ++i) + if (buttons[i].bounds.contains(x, y) && available(buttons[i].action)) { + pressed_button[m.cursorIndex] = int(i); break; + } + if (next_page.contains(x, y) && page + 1 < pages.size()) + pressed_button[m.cursorIndex] = int(buttons.size()); + break; + } + case vr::VREvent_MouseButtonUp: { + const auto& m = event.data.mouse; + if (m.button != vr::VRMouseButton_Left || m.cursorIndex >= pressed_button.size()) break; + const int index = pressed_button[m.cursorIndex]; + pressed_button[m.cursorIndex] = -1; + if (index == int(buttons.size()) && next_page.contains(int(m.x), H - int(m.y)) && + page + 1 < pages.size()) { ++page; drawn = false; } + else if (index == 0 && buttons[0].bounds.contains(int(m.x), H - int(m.y)) && page > 0) { + --page; drawn = false; + } else if (index > 0 && index < int(buttons.size()) && + buttons[index].bounds.contains(int(m.x), H - int(m.y)) && + available(buttons[index].action)) result.push_back(buttons[index].action); + break; + } + default: break; + } + } + vr::VRActiveActionSet_t set{}; + set.ulActionSet = action_set; + set.ulRestrictedToDevice = vr::k_ulInvalidInputValueHandle; + set.nPriority = 0; // normal priority; no experimental scene-input overrides + if (input->UpdateActionState(&set, sizeof(set), 1) != vr::VRInputError_None) { + reset_input(result); return result; + } + auto [la, ld] = digital(0); + auto [ra, rd] = digital(1); + const auto now = DoubleTap::Clock::now(); + if (left.update(la && focus, ld, panel.enabled, now)) result.push_back(UiAction::Enter); + switch (right.update(ra && focus, rd, now)) { + case GripRecord::Change::Begin: grip_capture = true; result.push_back(UiAction::BeginRecord); break; + case GripRecord::Change::End: grip_capture = false; result.push_back(UiAction::EndRecord); break; + case GripRecord::Change::Cancel: grip_capture = false; result.push_back(UiAction::Cancel); break; + case GripRecord::Change::None: break; + } + constexpr std::array mapped{{UiAction::Record, UiAction::Cancel, UiAction::Insert, UiAction::Enter}}; + for (size_t i = 0; i < edges.size(); ++i) { + auto [active, down] = digital(i + 2); + if (i == 0 && (!active || !focus || !panel.enabled) && edges[i].reset()) { + ptt_capture = false; result.push_back(UiAction::Cancel); + } + auto change = edges[i].update(active && focus && (panel.enabled || i == 1), down); + if (i == 0) { + if (change == NeutralEdge::Change::Down) { + ptt_capture = true; result.push_back(UiAction::BeginRecord); + } else if (change == NeutralEdge::Change::Up) { + ptt_capture = false; result.push_back(UiAction::EndRecord); + } + } else if (change == NeutralEdge::Change::Down && available(mapped[i])) { + result.push_back(mapped[i]); + } + } + return result; + } +}; + +Overlay::Overlay(const std::string& assets, const std::string& font, bool hand) + : impl_(std::make_unique(assets, font, hand)) {} +Overlay::~Overlay() = default; +std::vector Overlay::poll() { return impl_->poll(); } +void Overlay::draw(const Panel& panel) { impl_->draw(panel); } +std::string Overlay::controls_status() { + auto left = impl_->digital(0), right = impl_->digital(1); + return std::string("Grip actions tracked/active: left=") + (left.first ? "yes" : "no") + + " right=" + (right.first ? "yes" : "no"); +} + +int registration(const std::string& manifest, bool remove, bool autostart) { + try { + const auto path = absolute_file(manifest); + // Utility initialization does not load hardware drivers. Call only for an + // explicitly requested registration, never at static initialization. + configure_registry(); + if (!vr::VR_IsRuntimeInstalled()) throw std::runtime_error("OpenVR runtime not installed"); + vr::EVRInitError err = vr::VRInitError_None; + auto* system = vr::VR_Init(&err, vr::VRApplication_Utility); + if (err != vr::VRInitError_None || !system) + throw std::runtime_error(std::string("OpenVR utility init: ") + vr::VR_GetVRInitErrorAsEnglishDescription(err)); + struct Shutdown { ~Shutdown() { vr::VR_Shutdown(); } } shutdown; + auto* apps = vr::VRApplications(); + if (!apps) throw std::runtime_error("OpenVR applications interface unavailable"); + constexpr const char* key = "local.frameyap.overlay"; + if (remove) { + // Only remove the supplied manifest; never touch other applications. + auto e = apps->RemoveApplicationManifest(path.c_str()); + if (e != vr::VRApplicationError_None) throw std::runtime_error(apps->GetApplicationsErrorNameFromEnum(e)); + } else { + auto e = apps->AddApplicationManifest(path.c_str(), false); + if (e != vr::VRApplicationError_None) throw std::runtime_error(apps->GetApplicationsErrorNameFromEnum(e)); + if (autostart) { + e = apps->SetApplicationAutoLaunch(key, true); + if (e != vr::VRApplicationError_None) throw std::runtime_error(apps->GetApplicationsErrorNameFromEnum(e)); + } + } + if (apps->IsApplicationInstalled(key) == remove) + throw std::runtime_error("OpenVR registration state did not match the requested operation"); + std::cout << (remove ? "Unregistered " : "Registered ") << key; + if (!remove) std::cout << " (autolaunch=" << (apps->GetApplicationAutoLaunch(key) ? "on" : "off") << ")"; + std::cout << '\n'; + return 0; + } catch (const std::exception& e) { + std::cerr << "FrameYap registration: " << e.what() << '\n'; + return 1; + } +} +} // namespace frameyap diff --git a/src/overlay.hpp b/src/overlay.hpp new file mode 100644 index 0000000..8ab7e96 --- /dev/null +++ b/src/overlay.hpp @@ -0,0 +1,30 @@ +#pragma once +#include +#include +#include + +namespace frameyap { +enum class UiAction { Toggle, Record, BeginRecord, EndRecord, Cancel, Insert, Enter, Quit }; +struct Panel { + std::string status; + std::string transcript; + std::string detail; + bool enabled = false; + bool recording = false; +}; +class Overlay { +public: + Overlay(const std::string& assets, const std::string& font, bool hand); + ~Overlay(); + Overlay(const Overlay&) = delete; + Overlay& operator=(const Overlay&) = delete; + std::vector poll(); + void draw(const Panel& panel); + std::string controls_status(); // diagnostic only, no input delivery +private: + struct Impl; + std::unique_ptr impl_; +}; +// Explicit registration only; zero on success, nonzero on failure. +int registration(const std::string& manifest, bool remove, bool autostart); +} // namespace frameyap diff --git a/src/runtime.cpp b/src/runtime.cpp new file mode 100644 index 0000000..abd83e4 --- /dev/null +++ b/src/runtime.cpp @@ -0,0 +1,146 @@ +#include "runtime.hpp" +#include "audio.hpp" +#include "core.hpp" +#include "overlay.hpp" +#include "text_input.hpp" +#include "worker.hpp" +#include "instance_lock.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace frameyap { +namespace { +volatile sig_atomic_t interrupted = 0; +void signal_stop(int) { interrupted = 1; } +std::string state_label(State state) { + switch (state) { + case State::Warming: return "Warming - on-device Redux CPU"; + case State::Ready: return "Ready - double-tap + hold right grip"; + case State::Recording: return "RECORDING"; + case State::Transcribing: return "Transcribing - on device"; + case State::Review: return "Review - Insert approves CURRENT focus"; + case State::Queued: return "Input queued - not a delivery receipt"; + case State::Error: return "Unavailable - Record to retry"; + } + return {}; +} +} +int run(const Options& options) { + InstanceLock lock; + interrupted = 0; + auto old_int = std::signal(SIGINT, signal_stop); + auto old_term = std::signal(SIGTERM, signal_stop); + struct Restore { decltype(old_int) a, b; ~Restore() { std::signal(SIGINT, a); std::signal(SIGTERM, b); } } restore{old_int, old_term}; + Overlay overlay(options.assets, options.font, options.hand); + Worker worker; + Audio audio; + Session session; + std::string detail = "Review mode. Other apps may also hear your mic. Enter is separate."; + bool quit = false; + auto warm = [&] { + worker.stop(); session = Session{}; + worker.start(options.python, options.worker, options.model, options.threads); + detail = "Loading local model; microphone closed. Record again when Ready."; + }; + auto stop_record = [&] { + if (session.state() != State::Recording) return; + auto pcm = audio.finish(); + if (session.finish(pcm.size())) { + worker.submit(session.id(), pcm); + detail = "Release complete. No text is inserted automatically."; + } else detail = "Short tap discarded (minimum 200ms)."; + std::fill(pcm.begin(), pcm.end(), 0.0f); + }; + auto start_record = [&] { + if (session.state() == State::Review || session.state() == State::Transcribing || session.state() == State::Warming) return; + if (!worker.ready()) { warm(); return; } + if (session.state() == State::Error) session.cancel(); + if (!session.record()) return; + audio.start(); + detail = "Release grip to finish. Cancel discards. Maximum 20 seconds."; + }; + try { warm(); } + catch (const std::exception& e) { session.fail(); detail = e.what(); } + while (!quit && !interrupted) { + try { + if (auto reply = worker.poll()) { + if (reply->id == session.id() && session.state() == State::Transcribing) { + if (!reply->error.empty()) throw std::runtime_error(reply->error); + session.reply(reply->id, reply->text); + detail = session.text().empty() ? "No speech recognized; try again." : "Focus your destination, then Insert. Cancel discards."; + } + } + if (worker.ready()) session.ready(); + if (audio.recording() && audio.poll()) stop_record(); + } catch (const std::exception& e) { + audio.cancel(); worker.stop(); + if (session.state() != State::Review && session.state() != State::Queued) session.fail(); + detail = e.what(); // Preserve an already-correlated preview if the worker dies. + } + // Drawing precedes input polling: Enter is disabled until Ready is visible. + Panel panel{state_label(session.state()), session.text(), detail, + session.state() != State::Error && session.state() != State::Warming, + session.state() == State::Recording}; + if (panel.recording) panel.status += " - " + std::to_string(audio.seconds()) + " / 20s"; + overlay.draw(panel); + for (auto action : overlay.poll()) { + try { + switch (action) { + case UiAction::Quit: quit = true; break; + case UiAction::Toggle: + case UiAction::Record: + if (session.state() == State::Recording) stop_record(); else start_record(); + break; + case UiAction::BeginRecord: start_record(); break; + case UiAction::EndRecord: stop_record(); break; + case UiAction::Cancel: { + auto state = session.state(); + audio.cancel(); session.cancel(); detail = "Discarded. Microphone closed."; + if (state == State::Warming || state == State::Transcribing || state == State::Error) { + worker.stop(); session.fail(); detail = "Cancelled. Record to reload local worker."; + } + break; + } + case UiAction::Insert: + if (session.state() == State::Review) { + // Acquire IME before consuming: unavailable means the preview remains. + TextInput input(options.socket); + if (interrupted) break; + auto literal = session.take_insert(); + try { input.text(*literal); detail = "Input queued to current focus. Enter remains a separate action."; } + catch (...) { detail = "Delivery uncertain; not retried. Check destination before dictating again."; } + } + break; + case UiAction::Enter: + if (session.state() == State::Ready || session.state() == State::Queued || session.state() == State::Review) { + TextInput input(options.socket); + if (interrupted) break; + try { input.enter(); detail = "Explicit Enter queued to current focus."; } + catch (...) { detail = "Enter delivery uncertain; not retried."; } + } + break; + } + } catch (const std::exception& e) { + // Input lease failures preserve preview; capture failures release mic. + if (session.state() == State::Recording || session.state() == State::Transcribing) { + audio.cancel(); worker.stop(); session.fail(); + } + detail = e.what(); + } + if (quit || interrupted) break; + } + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + } + audio.cancel(); session.cancel(); worker.stop(); + return 0; +} +} diff --git a/src/runtime.hpp b/src/runtime.hpp new file mode 100644 index 0000000..9fa1746 --- /dev/null +++ b/src/runtime.hpp @@ -0,0 +1,10 @@ +#pragma once +#include +namespace frameyap { +struct Options { + std::string assets, font, python = "python3", worker, model, socket; + int threads = 2; + bool hand = true; +}; +int run(const Options& options); +} diff --git a/src/text_input.cpp b/src/text_input.cpp new file mode 100644 index 0000000..cb41ff5 --- /dev/null +++ b/src/text_input.cpp @@ -0,0 +1,126 @@ +#include "text_input.hpp" +#include "core.hpp" +#include "gamescope-input-method-client.h" +#include +#include +#include +#include +#include +#include + +namespace frameyap { +struct TextInput::Impl { + wl_display* display = nullptr; + wl_registry* registry = nullptr; + wl_seat* seat = nullptr; + gamescope_input_method_manager* manager = nullptr; + gamescope_input_method* ime = nullptr; + uint32_t serial = 0; + uint32_t seat_name = 0, manager_name = 0; + bool done = false, unavailable = false, used = false, ambiguous = false; + ~Impl() { + // Local proxy destruction then socket close avoids a blocking flush on teardown. + if (ime) wl_proxy_destroy(reinterpret_cast(ime)); + if (manager) wl_proxy_destroy(reinterpret_cast(manager)); + if (seat) wl_seat_destroy(seat); + if (registry) wl_registry_destroy(registry); + if (display) wl_display_disconnect(display); + } + static void global(void* data, wl_registry* registry, uint32_t name, const char* interface, uint32_t version) { + auto& self = *static_cast(data); + if (std::strcmp(interface, "wl_seat") == 0) { + if (self.seat) { self.ambiguous = true; return; } + self.seat_name = name; + self.seat = static_cast(wl_registry_bind(registry, name, &wl_seat_interface, 1)); + } + if (std::strcmp(interface, "gamescope_input_method_manager") == 0 && version >= 2) { + if (self.manager) { self.ambiguous = true; return; } + self.manager_name = name; + self.manager = static_cast(wl_registry_bind(registry, name, &gamescope_input_method_manager_interface, 2)); + } + } + static void removed(void* data, wl_registry*, uint32_t name) { + auto& self = *static_cast(data); + if (name == self.seat_name || name == self.manager_name) self.unavailable = true; + } + static void unavailable_event(void* data, gamescope_input_method*) { static_cast(data)->unavailable = true; } + static void done_event(void* data, gamescope_input_method*, uint32_t serial) { + auto& self = *static_cast(data); self.serial = serial; self.done = true; + } + static void synced(void* data, wl_callback*, uint32_t) { *static_cast(data) = true; } + void roundtrip() { + bool complete = false; + wl_callback* callback = wl_display_sync(display); + if (!callback) throw std::runtime_error("Gamescope sync allocation failed"); + const wl_callback_listener listener{synced}; + wl_callback_add_listener(callback, &listener, &complete); + auto end = std::chrono::steady_clock::now() + std::chrono::milliseconds(1000); + try { + while (!complete) { + if (wl_display_dispatch_pending(display) < 0) throw std::runtime_error("Gamescope disconnected"); + if (complete) break; + int flushed = wl_display_flush(display); + if (flushed < 0 && errno != EAGAIN) throw std::runtime_error("Gamescope flush failed"); + while (wl_display_prepare_read(display) != 0) { + if (wl_display_dispatch_pending(display) < 0) throw std::runtime_error("Gamescope disconnected"); + } + auto ms = std::chrono::duration_cast(end - std::chrono::steady_clock::now()).count(); + pollfd fd{wl_display_get_fd(display), short(POLLIN | (flushed < 0 ? POLLOUT : 0)), 0}; + int result = ms > 0 ? ::poll(&fd, 1, static_cast(ms)) : 0; + if (result <= 0 || (fd.revents & (POLLERR | POLLHUP | POLLNVAL))) { + wl_display_cancel_read(display); + if (result < 0 && errno == EINTR) continue; + throw std::runtime_error("Gamescope unavailable or timed out"); + } + if (fd.revents & POLLIN) { + if (wl_display_read_events(display) < 0) throw std::runtime_error("Gamescope read failed"); + } else wl_display_cancel_read(display); + } + } catch (...) { wl_callback_destroy(callback); throw; } + wl_callback_destroy(callback); + if (unavailable) throw std::runtime_error("Gamescope IME unavailable (Steam keyboard may own it)"); + } + void check_authorization() const { + if (used || unavailable || !done) throw std::runtime_error("Input authorization is no longer valid"); + } + void commit() { + check_authorization(); + used = true; + gamescope_input_method_commit(ime, serial); + // A roundtrip only acknowledges compositor processing, not target consumption. + roundtrip(); + } +}; +TextInput::TextInput(const std::string& socket) : impl_(std::make_unique()) { + if (socket.empty()) throw std::runtime_error("Specify the Gamescope Wayland socket with --socket"); + auto& p = *impl_; + p.display = wl_display_connect(socket.c_str()); + if (!p.display) throw std::runtime_error("Cannot connect to Gamescope socket; check --socket"); + p.registry = wl_display_get_registry(p.display); + if (!p.registry) throw std::runtime_error("Cannot get Gamescope registry"); + static const wl_registry_listener registry_listener{Impl::global, Impl::removed}; + wl_registry_add_listener(p.registry, ®istry_listener, &p); + p.roundtrip(); + if (!p.seat || !p.manager || p.ambiguous) + throw std::runtime_error("Gamescope IME v2/unique seat unavailable on this socket"); + p.ime = gamescope_input_method_manager_create_input_method(p.manager, p.seat); + if (!p.ime) throw std::runtime_error("Cannot create Gamescope IME"); + static const gamescope_input_method_listener ime_listener{Impl::unavailable_event, Impl::done_event}; + gamescope_input_method_add_listener(p.ime, &ime_listener, &p); + p.roundtrip(); + if (!p.done) throw std::runtime_error("Gamescope IME not ready"); +} +TextInput::~TextInput() = default; +void TextInput::text(const std::string& literal) { + auto validated = literal_text(literal); + if (validated.empty()) throw std::runtime_error("No text to insert"); + impl_->check_authorization(); + gamescope_input_method_set_string(impl_->ime, validated.c_str()); + impl_->commit(); +} +void TextInput::enter() { + impl_->check_authorization(); + gamescope_input_method_set_action(impl_->ime, GAMESCOPE_INPUT_METHOD_ACTION_SUBMIT); + impl_->commit(); +} +} diff --git a/src/text_input.hpp b/src/text_input.hpp new file mode 100644 index 0000000..ec2cd07 --- /dev/null +++ b/src/text_input.hpp @@ -0,0 +1,19 @@ +#pragma once +#include +#include +namespace frameyap { +// Short-lived, exclusive IME lease acquired only for an explicit user action. +// Construction checks protocol readiness without delivering input. +class TextInput { +public: + explicit TextInput(const std::string& socket); + ~TextInput(); + TextInput(const TextInput&) = delete; + TextInput& operator=(const TextInput&) = delete; + void text(const std::string& literal); + void enter(); +private: + struct Impl; + std::unique_ptr impl_; +}; +} diff --git a/tests/cli.cmake b/tests/cli.cmake index d8ae469..4fa6894 100644 --- a/tests/cli.cmake +++ b/tests/cli.cmake @@ -22,14 +22,14 @@ foreach(mode IN ITEMS default help version invalid extra) message(FATAL_ERROR "${mode}: exit ${result}, expected ${expected_exit}: ${error}") endif() if(mode STREQUAL "version") - if(NOT output STREQUAL "frame-dictation ${EXPECTED_VERSION} (scaffold)\n") + if(NOT output STREQUAL "frameyap ${EXPECTED_VERSION}\n") message(FATAL_ERROR "Unexpected version: ${output}") endif() elseif(expected_exit EQUAL 2) if(NOT error MATCHES "Unsupported arguments") message(FATAL_ERROR "Missing rejection diagnostic") endif() - elseif(NOT output MATCHES "Scaffold only") - message(FATAL_ERROR "Missing scaffold status") + elseif(NOT output MATCHES "No device access unless") + message(FATAL_ERROR "Missing explicit runtime opt-in status") endif() endforeach() diff --git a/tests/gestures_test.cpp b/tests/gestures_test.cpp new file mode 100644 index 0000000..8d5bb15 --- /dev/null +++ b/tests/gestures_test.cpp @@ -0,0 +1,73 @@ +#include "gestures.hpp" +#include +#include +#include +using namespace frameyap; +using namespace std::chrono_literals; +#define CHECK(x) do { if (!(x)) { std::cerr << __FILE__ << ':' << __LINE__ << " CHECK failed: " #x << '\n'; std::exit(1); } } while (false) +int main() { + using Clock = DoubleTap::Clock; + const auto t = Clock::time_point{}; + DoubleTap left; + auto l = [&](bool active, bool down, bool enabled, int ms) { + return left.update(active, down, enabled, t + std::chrono::milliseconds(ms)); + }; + CHECK(!l(true, true, true, 0)); // held on reconnect must rearm + CHECK(!l(true, false, true, 10)); + CHECK(!l(true, true, true, 20)); + CHECK(!l(true, false, true, 270)); // inclusive 250ms + CHECK(!l(true, true, true, 620)); // inclusive 350ms + CHECK(l(true, false, true, 870)); + CHECK(!l(true, true, true, 880)); + CHECK(!l(true, false, true, 890)); + CHECK(!l(true, true, true, 1241)); // late second press + CHECK(!l(true, false, true, 1250)); + CHECK(!l(true, true, true, 1260)); + CHECK(!l(true, false, true, 1511)); // long second press: no Enter + CHECK(!l(false, false, true, 1520)); + CHECK(!l(true, true, true, 1530)); + CHECK(!l(true, false, true, 1540)); + CHECK(!l(true, true, false, 1550)); + CHECK(!l(true, false, true, 1560)); + + using R = GripRecord::Change; + GripRecord right; + auto r = [&](bool active, bool down, int ms) { + return right.update(active, down, t + std::chrono::milliseconds(ms)); + }; + CHECK(r(true, true, 0) == R::None); + CHECK(r(true, false, 10) == R::None); + CHECK(r(true, true, 20) == R::None); + CHECK(r(true, false, 270) == R::None); + CHECK(r(true, true, 620) == R::Begin); + CHECK(r(true, true, 99999) == R::None); // no accidental Enter or stop on long hold + CHECK(r(true, false, 100000) == R::End); + CHECK(r(true, true, 100010) == R::None); + CHECK(r(true, false, 100011) == R::None); + CHECK(r(true, true, 100362) == R::None); + CHECK(r(true, false, 100370) == R::None); + CHECK(r(true, true, 100380) == R::Begin); + CHECK(r(false, true, 100381) == R::Cancel); // lost pose/activity while held + CHECK(r(true, true, 100382) == R::None); // reconnect held cannot finish/start + CHECK(r(true, false, 100383) == R::None); + CHECK(r(true, true, 100390) == R::None); + CHECK(r(true, false, 100641) == R::None); // first squeeze too long + CHECK(r(true, true, 100650) == R::None); + CHECK(r(true, false, 100660) == R::None); + CHECK(r(true, true, 101010) == R::Begin); + CHECK(right.reset() == R::Cancel); // overlay focus lost + CHECK(right.reset() == R::None); + CHECK(r(true, true, 101020) == R::None); + CHECK(r(true, false, 101030) == R::None); + + NeutralEdge e; + CHECK(e.update(true, true) == NeutralEdge::Change::None); + CHECK(e.update(true, false) == NeutralEdge::Change::None); + CHECK(e.update(true, true) == NeutralEdge::Change::Down); + CHECK(e.reset()); // PTT activity/tracking/focus loss cancels capture + CHECK(e.update(true, true) == NeutralEdge::Change::None); + CHECK(e.update(true, false) == NeutralEdge::Change::None); + CHECK(e.update(true, true) == NeutralEdge::Change::Down); + CHECK(e.update(true, false) == NeutralEdge::Change::Up); + CHECK(!e.reset()); +} diff --git a/tests/lock_test.cpp b/tests/lock_test.cpp new file mode 100644 index 0000000..625e903 --- /dev/null +++ b/tests/lock_test.cpp @@ -0,0 +1,30 @@ +#include "instance_lock.hpp" +#include +#include +int main() { + char pattern[] = "/tmp/frameyap-lock-test-XXXXXX"; + char* dir = mkdtemp(pattern); + if (!dir) return 1; + setenv("FRAMEYAP_INSTALL_ROOT", dir, 1); + try { + auto first = std::make_unique(); + bool rejected = false; + try { frameyap::InstanceLock second; } catch (const std::runtime_error&) { rejected = true; } + if (!rejected) throw std::runtime_error("second instance acquired lock"); + first.reset(); + { frameyap::InstanceLock next; } + auto lock = std::filesystem::path(dir) / ".lock"; + std::filesystem::remove(lock); + std::filesystem::create_symlink("foreign", lock); + rejected = false; + try { frameyap::InstanceLock unsafe; } catch (const std::runtime_error&) { rejected = true; } + if (!rejected) throw std::runtime_error("followed unsafe lock symlink"); + std::filesystem::remove(lock); + std::filesystem::remove(dir); + std::cout << "instance lock checks passed\n"; + } catch (const std::exception& e) { + std::cerr << e.what() << '\n'; + std::filesystem::remove_all(dir); + return 1; + } +} diff --git a/tests/text_input_test.cpp b/tests/text_input_test.cpp new file mode 100644 index 0000000..3483d8c --- /dev/null +++ b/tests/text_input_test.cpp @@ -0,0 +1,291 @@ +#include "core.hpp" +#include "text_input.hpp" +#include "gamescope-input-method-server.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace frameyap; +using namespace std::chrono_literals; +#define CHECK(condition) do { if (!(condition)) throw std::runtime_error( \ + std::string("line ") + std::to_string(__LINE__) + ": " #condition); } while (false) + +template void throws(F&& f) { + try { f(); } catch (const std::runtime_error&) { return; } + CHECK(false); +} + +// This fixture never connects to the ambient compositor: the client receives only +// this private socket name, scoped to a freshly-created 0700 runtime directory. +class RuntimeDir { +public: + RuntimeDir() { + if (const char* old = std::getenv("XDG_RUNTIME_DIR")) { prior_ = old; had_prior_ = true; } + char pattern[] = "/tmp/frameyap-wayland-test-XXXXXX"; + char* created = mkdtemp(pattern); + if (!created) throw std::runtime_error("mkdtemp failed"); + path_ = created; + struct stat st{}; + if (stat(path_.c_str(), &st) != 0 || (st.st_mode & 0777) != 0700) { + rmdir(path_.c_str()); + throw std::runtime_error("runtime directory is not private"); + } + if (setenv("XDG_RUNTIME_DIR", path_.c_str(), 1)) { + rmdir(path_.c_str()); + throw std::runtime_error("setenv failed"); + } + } + ~RuntimeDir() { + if (had_prior_) setenv("XDG_RUNTIME_DIR", prior_.c_str(), 1); + else unsetenv("XDG_RUNTIME_DIR"); + unlink((path_ + "/frameyap-fake").c_str()); + unlink((path_ + "/frameyap-fake.lock").c_str()); + rmdir(path_.c_str()); + } + RuntimeDir(const RuntimeDir&) = delete; + RuntimeDir& operator=(const RuntimeDir&) = delete; +private: + std::string path_, prior_; + bool had_prior_ = false; +}; + +struct Event { + std::string kind, text; + uint32_t number = 0; +}; +struct Snapshot { + int creates = 0, destroyed = 0; + bool bad_seat = false; + std::vector events; +}; +class FakeServer { +public: + enum class Mode { Ready, Unavailable, Stalled }; + explicit FakeServer(Mode mode = Mode::Ready) : mode_(mode) { + display_.reset(wl_display_create()); + CHECK(display_ != nullptr); + CHECK(wl_display_add_socket(display_.get(), "frameyap-fake") == 0); + CHECK(wl_global_create(display_.get(), &wl_seat_interface, 1, this, bind_seat)); + CHECK(wl_global_create(display_.get(), &gamescope_input_method_manager_interface, + 2, this, bind_manager)); + thread_ = std::thread([this] { + if (mode_ == Mode::Stalled) { + while (!stop_) std::this_thread::sleep_for(5ms); + return; + } + while (!stop_) { + wl_event_loop_dispatch(wl_display_get_event_loop(display_.get()), 10); + wl_display_flush_clients(display_.get()); + } + }); + } + ~FakeServer() { + stop_ = true; + if (thread_.joinable()) thread_.join(); + // Destroy client resources while callback state/mutex are still alive, + // including on a test assertion failure. + wl_display_destroy_clients(display_.get()); + display_.reset(); + } + FakeServer(const FakeServer&) = delete; + FakeServer& operator=(const FakeServer&) = delete; + Snapshot snapshot() const { + std::lock_guard lock(mutex_); + return state_; + } + bool wait_destroyed(int count) { + std::unique_lock lock(mutex_); + return changed_.wait_for(lock, 1s, [&] { return state_.destroyed >= count; }); + } + static constexpr const char* socket = "frameyap-fake"; +private: + struct DisplayDeleter { void operator()(wl_display* p) const { if (p) wl_display_destroy(p); } }; + RuntimeDir runtime_; + std::unique_ptr display_; + Mode mode_; + std::atomic stop_{false}; + std::thread thread_; + mutable std::mutex mutex_; + std::condition_variable changed_; + Snapshot state_; + + static FakeServer& owner(wl_resource* r) { + return *static_cast(wl_resource_get_user_data(r)); + } + static void bind_seat(wl_client* client, void* data, uint32_t version, uint32_t id) { + auto* r = wl_resource_create(client, &wl_seat_interface, version, id); + if (!r) { wl_client_post_no_memory(client); return; } + wl_resource_set_implementation(r, nullptr, data, nullptr); + } + static void bind_manager(wl_client* client, void* data, uint32_t version, uint32_t id) { + auto* r = wl_resource_create(client, &gamescope_input_method_manager_interface, version, id); + if (!r) { wl_client_post_no_memory(client); return; } + static const struct gamescope_input_method_manager_interface methods{destroy, create}; + wl_resource_set_implementation(r, &methods, data, nullptr); + } + static void destroy(wl_client*, wl_resource* r) { wl_resource_destroy(r); } + static void create(wl_client* client, wl_resource* manager, wl_resource* seat, uint32_t id) { + auto& server = owner(manager); + auto* r = wl_resource_create(client, &gamescope_input_method_interface, + wl_resource_get_version(manager), id); + if (!r) { wl_client_post_no_memory(client); return; } + static const struct gamescope_input_method_interface methods{ + destroy, commit, set_string, set_action, nullptr, nullptr, nullptr, nullptr}; + wl_resource_set_implementation(r, &methods, &server, ime_destroyed); + { + std::lock_guard lock(server.mutex_); + ++server.state_.creates; + if (!seat || wl_resource_get_client(seat) != client || + std::strcmp(wl_resource_get_class(seat), "wl_seat") != 0 || + wl_resource_get_version(manager) != 2) server.state_.bad_seat = true; + } + if (server.mode_ == Mode::Ready) gamescope_input_method_send_done(r, 42); + else gamescope_input_method_send_unavailable(r); + } + static void ime_destroyed(wl_resource* r) { + auto& server = owner(r); + { + std::lock_guard lock(server.mutex_); + ++server.state_.destroyed; + } + server.changed_.notify_all(); + } + static void commit(wl_client*, wl_resource* r, uint32_t serial) { + auto& server = owner(r); + std::lock_guard lock(server.mutex_); + server.state_.events.push_back({"commit", "", serial}); + } + static void set_string(wl_client*, wl_resource* r, const char* text) { + auto& server = owner(r); + std::lock_guard lock(server.mutex_); + server.state_.events.push_back({"string", text, 0}); + } + static void set_action(wl_client*, wl_resource* r, uint32_t action) { + auto& server = owner(r); + std::lock_guard lock(server.mutex_); + server.state_.events.push_back({"action", "", action}); + } +}; + +void test_text() { + FakeServer server; + { + TextInput input(server.socket); + auto initial = server.snapshot(); + CHECK(initial.creates == 1 && !initial.bad_seat && initial.events.empty()); + input.text("Hello 世界 😀\r\nnext\tline\xe2\x80\xa8 end"); + auto state = server.snapshot(); + CHECK(state.events.size() == 2); + CHECK(state.events[0].kind == "string"); + CHECK(state.events[0].text == "Hello 世界 😀 next line end"); + CHECK(state.events[1].kind == "commit" && state.events[1].number == 42); + // Authorization is consumed even when a duplicate delivery is attempted. + throws([&] { input.enter(); }); + CHECK(server.snapshot().events.size() == 2); + } + CHECK(server.wait_destroyed(1)); + CHECK(server.snapshot().events.size() == 2); +} + +void test_enter() { + FakeServer server; + { + TextInput input(server.socket); + CHECK(server.snapshot().events.empty()); + input.enter(); + auto state = server.snapshot(); + CHECK(state.events.size() == 2); + CHECK(state.events[0].kind == "action" && + state.events[0].number == GAMESCOPE_INPUT_METHOD_ACTION_SUBMIT); + CHECK(state.events[1].kind == "commit" && state.events[1].number == 42); + throws([&] { input.text("duplicate"); }); + CHECK(server.snapshot().events.size() == 2); + } + CHECK(server.wait_destroyed(1)); + CHECK(server.snapshot().events.size() == 2); +} + +void test_invalid() { + FakeServer server; + { + TextInput input(server.socket); + throws([&] { input.text(std::string("x\0y", 3)); }); + throws([&] { input.text("\x1b[31m"); }); + throws([&] { input.text("\xc0\xaf"); }); + throws([&] { input.text("\xed\xa0\x80"); }); + throws([&] { input.text("\xf4\x90\x80\x80"); }); + throws([&] { input.text("\xe2\x82"); }); + throws([&] { input.text("\xc2\x85"); }); + throws([&] { input.text(std::string(4097, 'a')); }); + throws([&] { input.text("\t\n"); }); + CHECK(server.snapshot().events.empty()); + input.text("submit; $(echo x)"); + auto events = server.snapshot().events; + CHECK(events.size() == 2 && events[0].text == "submit; $(echo x)"); + CHECK(events[1].kind == "commit"); + } + CHECK(server.wait_destroyed(1)); +} + +void test_review_insert() { + Session session; + session.ready(); + CHECK(session.record() && session.finish(16000)); + auto id = session.id(); + CHECK(session.reply(id, "send\n世界")); + CHECK(session.state() == State::Review); + FakeServer server; + { + TextInput input(server.socket); + CHECK(server.snapshot().events.empty()); // Discovery is not insertion. + auto approved = session.take_insert(); + CHECK(approved == "send 世界" && !session.take_insert()); + CHECK(!session.reply(id, "duplicate")); + input.text(*approved); + auto events = server.snapshot().events; + CHECK(events.size() == 2 && events[0].kind == "string"); + CHECK(events[0].text == "send 世界"); + CHECK(events[1].kind == "commit" && events[1].number == 42); + } + CHECK(server.wait_destroyed(1)); +} + +void test_unavailable() { + FakeServer server(FakeServer::Mode::Unavailable); + throws([&] { TextInput input(server.socket); }); + CHECK(server.wait_destroyed(1)); + auto state = server.snapshot(); + CHECK(state.creates == 1 && state.events.empty()); +} + +void test_stalled() { + FakeServer server(FakeServer::Mode::Stalled); + auto start = std::chrono::steady_clock::now(); + throws([&] { TextInput input(server.socket); }); + auto elapsed = std::chrono::steady_clock::now() - start; + CHECK(elapsed >= 700ms && elapsed < 4s); + CHECK(server.snapshot().creates == 0); +} + +int main() { + try { + test_text(); test_enter(); test_invalid(); test_review_insert(); + test_unavailable(); test_stalled(); + std::cout << "native fake Wayland input checks passed\n"; + } catch (const std::exception& e) { + std::cerr << "text_input_test: " << e.what() << '\n'; + return 1; + } +}