From 7fbcd177d302dfa056ca05946c180f126de854e3 Mon Sep 17 00:00:00 2001 From: DeeJanuz <45082401+DeeJanuz@users.noreply.github.com> Date: Fri, 2 Oct 2026 21:58:09 -0600 Subject: [PATCH 1/3] Input relay test: let the fake devices past the udev permission check Since the relay leaves a node it can't read yet for the next scan (12f2e84, PR #12), it checks os.access first, and the test's fake /dev/input paths don't exist, so the relay never opened them and every key check failed. The fake os now says they're readable. Co-Authored-By: Claude Opus 5.5 --- input/test/keys-test.py | 1 + 1 file changed, 1 insertion(+) diff --git a/input/test/keys-test.py b/input/test/keys-test.py index 98b493a..12a3cd3 100755 --- a/input/test/keys-test.py +++ b/input/test/keys-test.py @@ -80,6 +80,7 @@ fake_os = type(os)("os") fake_os.__dict__.update(os.__dict__) fake_os.listdir = lambda path: ["event900", "event901"] if path == "/dev/input" else os.listdir(path) fake_os.stat = lambda path, *a, **k: Inode() if path in FAKE else os.stat(path, *a, **k) +fake_os.access = lambda path, mode, *a, **k: path in FAKE or os.access(path, mode, *a, **k) relay.os = fake_os From 8aaf7db05a9353b6518a0281605b327b74a94b5b Mon Sep 17 00:00:00 2001 From: DeeJanuz <45082401+DeeJanuz@users.noreply.github.com> Date: Fri, 2 Oct 2026 21:58:09 -0600 Subject: [PATCH 2/3] Pause Frametop for VR games Frametop kept using the headset during games: with gaze mode off, our eye tracker still took about 60% of a core, remote desktop about 2 cores while on, and KWin kept drawing hidden screens because ft-screens sent their frame callbacks at 90 Hz. Pausing gives that back, and resuming brings back only what pausing stopped. It's also a way to keep the gaze service and our eye tracker off during games, which PR #13 asked for. Paused (input/game_pause.py, run by the input relay): - frametop-gaze stops (ft-eyegrab then idles by itself), and hand tracking and remote desktop stop if they run - the desktop hides and slows down: ft-screens "pause on" hides every panel and sends KWin a frame callback once a second; or, with pause_desktop "close", the desktop closes and starts again on resume - the relay lets go of the 3D mouse, typing goes to Steam, and mapped buttons and key combinations do only pause_toggle, steam_menu and commands Toggled by both thumbsticks clicked together twice (configurable), read passively from vrserver's web socket (input/vrws.py) so it works in games and takes nothing from them; by the new pause_toggle action; by input/ft-pause; and, with pause_auto (default on), by a VR game starting and ending, which the pointer helper now reports ("vrgame 1|0"). Frametop Input Settings has a Games page for it. update-check.py checks the web socket, and doesn't count a paused gaze service as failed. Co-Authored-By: Claude Opus 5.5 --- README.md | 18 +- display-settings/main.qml | 2 +- docs/design.md | 9 + docs/reference.md | 36 ++- input-settings/ft_input_settings.py | 129 +++++++- input-settings/main.qml | 139 +++++++- input/ft-pause | 45 +++ input/game_pause.py | 483 ++++++++++++++++++++++++++++ input/input-relay.py | 112 ++++++- input/test/keys-test.py | 69 +++- input/test/pause-test.py | 214 ++++++++++++ input/vrws.py | 198 ++++++++++++ pointer/helper/ft-pointer.cpp | 11 +- screens/compositor.c | 10 +- screens/vr.cpp | 20 +- screens/vr.h | 2 + scripts/report.sh | 1 + scripts/update-check.py | 77 ++++- 18 files changed, 1534 insertions(+), 41 deletions(-) create mode 100755 input/ft-pause create mode 100644 input/game_pause.py create mode 100755 input/test/pause-test.py create mode 100755 input/vrws.py diff --git a/README.md b/README.md index 3a74731..d93b707 100644 --- a/README.md +++ b/README.md @@ -60,7 +60,7 @@ If you work in the desktop for long stretches, or leave the headset on a stand, | Meta+Shift+H in the desktop | Hides or shows all screens (also in the menu as Hide/Show Screens, and mappable). The Visibility & pins tab of Frametop Display Settings can instead show them only with the dashboard open, or while you look at your wrist | | Tap Meta, on any keyboard | Opens the Steam menu in the SteamVR dashboard, or closes the dashboard, wherever you are. The desktop's launcher is still on the taskbar and Alt+F1. Change it in Frametop Input Settings (Keyboard page), where any key combination or modifier tap can do a Frametop or Steam action, open a profile, or run a command of your own | | Switch a screen to Hidden (Frametop Display Settings, Visibility & pins → Screens shown) | Hides just that screen until you switch it back, whatever the other visibility settings say; new windows that would open on it float instead | -| Play a VR game | The screens hide and your controllers stay in the game. Open the SteamVR dashboard, or press Meta+Shift+H, to see and use them. To keep them visible over games, change During VR games on the Visibility & pins tab; the controllers still stay in the game, and you use the screens with the mouse or the dashboard | +| Play a VR game | Frametop pauses so the game gets the headset to itself (see [Pause for VR games](#pause-for-vr-games)): the screens hide and your controllers stay in the game. Click both thumbsticks together twice to bring Frametop back. With the automatic pause off, the screens still hide, and the SteamVR dashboard or Meta+Shift+H shows them; to keep them visible over games, change During VR games on the Visibility & pins tab | Restarting the desktop (Restart desktop in Frametop Display Settings) closes its windows, but background work you started in it, such as servers, tmux sessions, or builds, keeps running. @@ -95,6 +95,18 @@ A profile is a named setup: where the screens are, with their sizes and pins, wh | Pick a profile under Arrangement and press Open profile | Switches to it: the screens move, open windows of its apps go to their places, and the apps that aren't open start. Nothing closes | | Pick a profile under Start in profile, or run its entry (Frametop: NAME) from SteamVR's Launch a program list | The desktop starts in that profile, or switches to it if it's running. A profile can also go on a key combination, mouse button, or controller button in Frametop Input Settings | +### Pause for VR games + +Frametop pauses while a VR game runs, so the game gets the headset's CPU and GPU, and comes back a few seconds after the game ends. Paused, the screens hide and the desktop nearly stops drawing, but its windows stay open. Gaze mode's eye tracking stops, and so do remote desktop and hand tracking if they run. The mouse works as a plain mouse in SteamVR. + +| Do this | To get this | +| --- | --- | +| Click both thumbsticks together, twice | Pauses Frametop, or brings it back, in a game or not. You hear a short sound. The game sees the clicks too | +| Start a VR game | Frametop pauses, and comes back 5 seconds after the game ends. Bring it back during the game, and it stays on until that game ends | +| Map Pause/resume Frametop to a mouse button, key combination, or controller button | The same, from that button (Frametop Input Settings) | + +The Games page of Frametop Input Settings turns the automatic pause off, changes the gesture, closes the desktop instead of hiding it (more for the game, but its windows close), and turns the sound off. From a terminal: `input/ft-pause on`, `off`, or `status`. + ### Gaze mode (experimental) In gaze mode the pointer goes where you look, and the mouse or the keyboard does the last bit. The installer offers it (or run `gaze/run.sh install` later), and then our own eye tracker for it, which is more accurate than SteamVR's (or run `gaze/tracker/install.sh` later; it needs `sudo`). Gaze mode uses ours once it's installed, and SteamVR's until then. Turn it on and calibrate it on the Gaze page of Frametop Input Settings. @@ -124,8 +136,8 @@ This is an early release, tested on one Steam Frame (SteamOS 0.3.0 build 2026092 - A SteamOS or SteamVR update can break parts of it until Frametop catches up. After an update, run `cd ~/frametop && scripts/doctor.sh` in a terminal. It checks what Frametop needs from SteamOS, and says what changed since the versions you last marked as working and what to try. Once everything works, `scripts/doctor.sh --mark-good` records the versions. If something stops working, please report it. - The first install downloads 1–2 GB for the build container and compiles everything on the headset, which takes several minutes. -- During a VR game you can't show the screens with a controller button, because the game owns the buttons. Open the SteamVR dashboard, press Meta+Shift+H, or use a mapped mouse button instead. -- Flatscreen games aren't detected as games. If your controllers end up working the screens instead of the game, set Controllers on the screens to "Only with the SteamVR dashboard open" (Frametop Display Settings, Visibility & pins tab). +- During a VR game, mapped controller buttons belong to the game, so they can't bring the screens up. The pause gesture still works: Frametop reads it without taking the thumbsticks from the game. With the automatic pause off, open the SteamVR dashboard, press Meta+Shift+H, or use a mapped mouse button instead. +- Flatscreen games aren't detected as games, so they don't pause Frametop by themselves: click both thumbsticks twice to pause it. If your controllers end up working the screens instead of the game, set Controllers on the screens to "Only with the SteamVR dashboard open" (Frametop Display Settings, Visibility & pins tab). - Typing follows your last click. A controller click on a panel other than the screens (the dashboard, a Steam app) doesn't move typing there; click it with the mouse, or click a screen to bring typing back. - The screens don't draw a mouse cursor of their own. The 3D mouse's dot or SteamVR's laser shows where you're pointing. - Profiles reopen apps, not what the apps had open. Tabs, files, and folders are left to each app's own restore. diff --git a/display-settings/main.qml b/display-settings/main.qml index 1aa820d..589902b 100644 --- a/display-settings/main.qml +++ b/display-settings/main.qml @@ -802,7 +802,7 @@ Kirigami.ApplicationWindow { Repeater { model: [ { value: "hide", text: "Hide them unless the SteamVR dashboard is open", help: "The game has the view to itself; open the dashboard (or press Meta+Shift+H) to see the screens." }, - { value: "visible", text: "Keep them visible over the game", help: "They float over the game as they are outside it." } + { value: "visible", text: "Keep them visible over the game", help: "They float over the game as they are outside it. Turn off Pause while a VR game runs in Frametop Input Settings (Games), or Frametop pauses and hides them anyway." } ] delegate: ColumnLayout { required property var modelData diff --git a/docs/design.md b/docs/design.md index fa02653..9937620 100644 --- a/docs/design.md +++ b/docs/design.md @@ -189,6 +189,15 @@ Movement is judged within 10-second windows. On the mount, the head pose jittere Staying awake while charging uses Steam's own setting rather than a logind sleep inhibitor. Steam suspends with `dbus-send ... login1.Manager.Suspend boolean:true`, and a block inhibitor does stop that (`CanSuspend` answers "challenge" while one is held), but it stops the power button too. `system_idle_suspend_ac_sec` is field 24004 of Steam's CMsgClientSettings. In Steam's SharedJSContext, reachable over CDP on port 8080 because Steam runs with `-cef-enable-debugging`, `SteamClient.Settings.SetSetting` takes a change as a base64 protobuf, the way Steam's Power page sends it (0 is never), and `settingsStore.clientSettings` has the current values. +## Pausing for VR games + +Hiding the screens during a game kept them out of view, but Frametop kept using the headset. Measured on 2026-10-02 with gaze mode off and no game running, in shares of one core: our eye tracker (ft-eyes) about 60%, ft-eyegrab, ft-gaze and ft-gazed about 3 to 4% each; remote desktop (krdpserver, FreeRDP, Xvnc) about 2 cores while it ran; KWin about 13%, ft-screens about 4%. The gaze service runs at full rate whether gaze mode is on or not. Reading SteamVR's eye tracking 90 times a second also made it restart every 10 to 13 seconds during Beat Saber, and each restart took input focus from the game, which paused it (PR #13; since then ft-gaze skips SteamVR's gaze action during games, but our own tracker kept running). So pausing stops what costs the most and leaves windows where they are. + +- A hidden screen still cost as much as a visible one. ft-screens sent every committed screen its frame callback at 90 Hz whether its overlay showed or not, so KWin kept drawing, and its apps with it. Paused, ft-screens sends the callbacks once a second. A Wayland client draws again only after its last frame's callback, so KWin's output stalls, KWin's own clients stop getting theirs, and the whole desktop idles, without anything losing its connection. A second's pace, rather than none, keeps any client that waits on a callback from waiting forever. Stopping KWin or the apps with SIGSTOP would free the same, but a Wayland peer that stops reading overflows the other side's 4 KB socket buffer, which ends the connection: that's how the live desktop died once when its KWin stalled (`Data too big for buffer`). They also sit in different cgroups (KWin under steam.service when the VR launcher starts it, ft-screens in the dev container's), so no single freeze stops them together. +- The relay does the pausing because it's the one part that always runs, and the pointer helper keeps running because stopping it leaves its virtual controller connected with its last pose (the driver has no staleness timeout), maybe holding a hand role, with the 3D mouse dead. Releasing it does the job. The helper already checks for a scene app twice a second, so it's what tells the relay a game started. +- The gesture has to work during a game, but SteamVR input reaches only the app with input focus, and an overlay with global input (`steamvr/globalActionSetPriority`) takes the buttons it binds from the game. vrserver's web socket on 127.0.0.1:27062, which its controller binding page uses for the live view, reports every controller component whatever has focus, and reading it takes nothing. The game sees the clicks too, so the default is a gesture games hardly use: both thumbsticks, together, twice. "Together" means within 0.3 seconds of each other, so a stick held down to sprint while the other clicks doesn't count. The stream is about 160 messages a second, nearly all capacitive sensing, so the reader parses only the few that mention a gesture's button. A controller's root path changes while the 3D mouse holds its hand role (`/devices/cv/` instead of `/user/hand/right`), so the reader looks the controllers up again every 3 seconds. +- Resuming starts remote desktop through `systemd-run --scope`: started straight from the relay, it would join the relay's cgroup and end with the next relay restart. + ## SteamOS updates On the Frame, SteamVR is part of the OS image (`/opt/steamvr`, the `deckard-steamvr-rel` package), next to KWin, gamescope, and the kernel, so every SteamOS update can bring a new SteamVR too. Frametop survives updates: it lives in the home folder and the `dev` container, the Bluetooth fixes are in `/etc`, which SteamOS keeps across updates, and nothing goes into `/usr`. What an update can break is what Frametop uses from the image. The public OpenVR API is versioned and stays put. The rest is less certain: `IVRIPCResourceManagerClient`, which is newer than the header SteamVR ships; the text `vrcmd --overlays` prints; the eye tracker's shared memory layout; XRService's camera buffers; KWin's nested backend; and behavior Frametop works around, such as the SteamVR Settings page that `ComputeOverlayIntersection` can't find or the scale KWin's nested backend doesn't undo. diff --git a/docs/reference.md b/docs/reference.md index 03281f5..ecc9f06 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -66,12 +66,12 @@ place N x y z yaw pitch roll width N metres curve N radius|on|off pin N|all left|right|head [matrix] unpin N|all size N w h get N screens head state key code value scale N s vrkeyboard show|hide|toggle|close visibility always|dashboard|gesture|toggle wrist degrees gesture left|right degrees -hide | show | toggle controllers always|outside_games|dashboard ingames hide|visible +hide | show | toggle controllers always|outside_games|dashboard ingames hide|visible pause on|off|state conceal N|all reveal N|all concealed cutouts on|off|state cutouts predict on|off cutouts lead ms float N mpp x y w h title unfloat N pose N matrix sub N k x y w h | sub N k off minimized N 0|1 carry N ``` -`conceal` and `reveal` hide and show one screen on its own (`ft-layout hide` and `show` send them), and `concealed` lists those screens. `cutouts` turns the hand cutouts on and off (`ft-handsctl cutouts`). The last line is ft-floatd's, for floating windows: N is a floating window's panel, numbered on from the screens, one per spare output. `float` gives the window's rectangle in its output, metres per pixel, and the title bar's height, and shows the panel; `unfloat` hides it. `pose` places it (a 3x4 matrix, standing universe), `sub` shows popup or dialog k over it, `minimized` hides it while its window is minimized, and `carry` moves it with the laser that pressed the window's own title bar. +`conceal` and `reveal` hide and show one screen on its own (`ft-layout hide` and `show` send them), and `concealed` lists those screens. `pause on` (from the input relay, when Frametop pauses for a VR game) hides every screen and floating window whatever else says, and slows the desktop down; `pause off` undoes it. `cutouts` turns the hand cutouts on and off (`ft-handsctl cutouts`). The last line is ft-floatd's, for floating windows: N is a floating window's panel, numbered on from the screens, one per spare output. `float` gives the window's rectangle in its output, metres per pixel, and the title bar's height, and shows the panel; `unfloat` hides it. `pose` places it (a 3x4 matrix, standing universe), `sub` shows popup or dialog k over it, `minimized` hides it while its window is minimized, and `carry` moves it with the laser that pressed the window's own title bar. ## Input relay @@ -114,11 +114,12 @@ The pointer settings are in `~/.config/frametop.conf`: `POINTER_SENSITIVITY`, `P ## Frametop Input Settings -A Kirigami app with a Python backend, in the Plasma menu under Settings. It runs in the `dev` container and talks to the relay over its control socket, `@frametop_relay`. It has eight pages: +A Kirigami app with a Python backend, in the Plasma menu under Settings. It runs in the `dev` container and talks to the relay over its control socket, `@frametop_relay`. It has nine pages: - Devices lists every USB and Bluetooth mouse and keyboard, with a light that flashes when the device is used. Each device gets a role: 3D pointer (grabbed, drives the pointer; the default for anything with a mouse), Pass through (grabbed only while typing goes to the desktop; the default for keyboards, whose key combinations work everywhere), or Ignore. A device is identified by its Bluetooth address, or its USB ids and name, so all of its input nodes share one role. Forget drops everything saved for a device. -- Buttons maps a pointer device's buttons. Choose Capture a button, press the button or key, then pick an action: a click, back, scroll, toggle dashboard, recenter, pointer on or off, head follow on or off, gaze pointer on or off, gaze precision, gaze drag, gaze quick check, faster or slower, reset the screen layout, hide or show the screens, open or close the keyboard, float a window in VR or put it back, put all floating windows back, Open profile NAME (one per profile, [profiles.md](profiles.md)), pass the key through, or nothing. Devices with saved mappings are listed even while they're asleep. +- Buttons maps a pointer device's buttons. Choose Capture a button, press the button or key, then pick an action: a click, back, scroll, toggle dashboard, recenter, pointer on or off, head follow on or off, gaze pointer on or off, gaze precision, gaze drag, gaze quick check, faster or slower, reset the screen layout, hide or show the screens, open or close the keyboard, float a window in VR or put it back, put all floating windows back, pause or resume Frametop ([Pausing for VR games](#pausing-for-vr-games)), Open profile NAME (one per profile, [profiles.md](profiles.md)), pass the key through, or nothing. Devices with saved mappings are listed even while they're asleep. - Controllers maps the Frame controllers' buttons (every button but the system button) to the same actions, except passing a key through and the gaze actions: gaze mode is a mouse and keyboard feature ([gaze-controllers.md](gaze-controllers.md)). Capture a button and press it on a controller, or pick it from the list. The controllers aren't input devices on the host; only SteamVR sees them. So the pointer helper reads them with SteamVR input (`pointer/helper/vrbuttons.h`, `pointer/helper/actions/`) and sends presses to the relay (`vrbtn right/a 1`), which does the mapped action. The helper only takes the buttons that are mapped (the relay tells it with `vrbind`), at an overlay-global priority, and only while no game (scene application) runs, so games keep every button; with In games on (`controller_in_games`), a mapped button is taken from games too. That needs SteamVR's "Enable global input from overlays (Experimental)" setting (`steamvr/globalActionSetPriority`), which the page's Global input switch turns on and off. Mappings are saved as `controller_buttons` in `~/.config/frametop-input.json`. +- Games has the pause for VR games: its state with Pause now or Resume, whether VR games pause Frametop by themselves, the controller gesture (one or two buttons, pressed once or twice; one button always takes two presses), what happens to the desktop, and the sound. They're saved as `pause_auto`, `pause_gesture`, `pause_desktop`, and `pause_sound` in `~/.config/frametop-input.json`. See [Pausing for VR games](#pausing-for-vr-games). - Keyboard sets when Frametop's keyboard opens: whenever a text field is selected; only while no pass-through keyboard is connected (the default; keyboards other programs make through uinput, like frame-voice's, don't count); only with a mouse or controller button mapped to Open/close keyboard; or never, which turns the button off too. Keep it open (on by default, `vr_keyboard_persist`) leaves it open after the text field loses focus. The mode is saved as `vr_keyboard` in `~/.config/frametop-input.json`, and the page lists the keyboards that count as connected. Its Key combinations section maps modifiers plus a key, or one modifier tapped on its own, on any keyboard, to any action but passing a key through or nothing, or to Run a command…: a command line the input relay runs with `sh -c` when you press the keys (`command:CMD`). The command runs as the relay's user service, outside the desktop's session, with `layout/`, `float/` and `steam/` on its `PATH` (so `ft-layout use Work` or `ft-float launch org.kde.dolphin` work as they are), and its output goes to the relay's journal. The gaze clicks (Gaze left click and Gaze right click) only go on key combinations. The defaults are a Meta tap (open the Steam menu, or close the dashboard), Meta+J (gaze left click), Meta+K (gaze right click), and Meta+Shift+F (float window in VR or put it back); remove them or add others there. A tap is a press and release with no other key, mouse button, or scroll in between; a bound one sends the desktop F24 before the release, so Plasma's launcher doesn't open on it. The combination's last key isn't typed, and the modifiers still reach the app; while typing goes to Steam rather than the desktop, keyboards aren't grabbed, so Steam or the game sees the keys too. They're saved as `key_bindings` in the same file; a file with its own list, even an empty one, gets no defaults. - Pointer has a Head follow switch and sliders for the pointer settings, which apply immediately, and a Recenter button. - Ignored panels lists the SteamVR overlays that are showing, grouped by app (the first two parts of the overlay key, such as `sasaken.frame-perf-overlay`), from the pointer helper (`overlays`). Tick a panel, or Ignore the whole app, and the pointer passes through it to what's behind. It's for panels you only look at, like a performance overlay that follows your view. The list is saved as `POINTER_IGNORE` in `~/.config/frametop.conf`: comma-separated overlay keys, where a shell pattern like `vendor.app*` covers a whole app, including panels it opens later. The helper reloads at once. Frametop's own screens aren't listed, and entries for apps that aren't open are listed below, to remove. @@ -199,6 +200,33 @@ power/run.sh off | on # the displays off now, or back on power/run.sh log ``` +## Pausing for VR games + +Paused, Frametop leaves the headset's CPU and GPU to a VR game. The input relay does it (`input/game_pause.py`), since it's the one part that always runs: + +- The gaze service stops (`frametop-gaze`: ft-gazed, ft-gaze, our own eye tracker, the gaze panel), so nothing reads SteamVR's eye tracking. Our frame grabber, the root service `ft-eyegrab`, goes idle by itself 3 seconds after our eye tracker stops asking it for frames. +- Hand tracking stops if it runs (`frametop-camd`, `frametop-hands`). +- The desktop, as the Games page of Frametop Input Settings says (`pause_desktop`): hidden (the default) or closed. Hidden, ft-screens hides every screen and floating window whatever the visibility mode, the hotkey, or the dashboard says, and gives KWin a frame callback once a second instead of 90 times. KWin draws a screen only after its frame callback, and its apps wait for theirs, so the desktop hardly draws, but its windows stay open. Remote desktop stops if it runs (`session/remote-ctl.sh`). Closed, `desktops.sh stop` closes the desktop and its windows, and resuming starts it again (about 12 seconds), in its start profile if it has one. +- The relay lets go of the 3D mouse and feeds pointer devices to its virtual mouse and keyboard, as with `POINTER=0`. Typing goes to Steam. Mapped buttons and key combinations do nothing but pausing, the Steam menu, and commands; a key combination that does nothing is typed as usual. + +Resuming starts again only what pausing stopped, and plays a second sound. The pointer helper and ft-powerd keep running: they cost little, the helper is what says a game started, and stopping it would leave its virtual controller connected with its last pose. + +Ways to pause and resume: + +- The controller gesture, by default both thumbsticks clicked together twice: both go down within 0.3 seconds of each other, and the second time within 0.7 seconds of the first. The relay reads it from vrserver's web socket (`input/vrws.py`), which works whatever has input focus and takes nothing from the game, so the game sees the clicks too. The Games page changes it: one or two of the buttons the Controllers page lists, pressed once or twice (one button always takes two), or none. +- The Pause/resume Frametop action, on a mouse button, a key combination, or a controller button (outside games, like every mapped controller button). +- VR games, with Pause while a VR game runs on (`pause_auto`, the default). The pointer helper tells the relay when a scene app starts and ends (`vrgame 1|0`, repeated every 5 seconds). A game starting pauses Frametop. A pause that starts while a game runs ends 5 seconds after the game does, unless another game starts first. Resumed during a game, Frametop stays on until that game ends. A pause that starts outside a game lasts until you resume. Flatscreen games aren't scene apps, so they don't pause it. +- From a terminal or a script: + +``` +input/ft-pause on | off | toggle # pause or resume +input/ft-pause status # the state as JSON (the relay's "pause ?") +input/vrws.py 10 # the controllers' buttons from vrserver's web socket, for 10 s +input/test/pause-test.py # the gesture and the automatic pause, offline +``` + +The state outlives a relay restart, in `/run/user/UID/frametop-pause.json`. A SteamVR restart while paused starts the gaze service with it, and the relay stops it again when the pointer helper comes back. + ## Gaze pointer (experimental) In gaze mode the 3D mouse's pointer goes where you look, and the mouse or the keyboard does the last bit. It needs the gaze service, which `install.sh` offers (yes by default) and `gaze/run.sh install` installs on its own: it builds it and runs `gaze/ft-gazed` as `frametop-gaze.service`, which starts with SteamVR. Turn gaze mode on with the Gaze page of Frametop Input Settings, `gaze/ft-gazectl on`, `POINTER_GAZE=1`, or a button or key combination mapped to Gaze pointer on/off. diff --git a/input-settings/ft_input_settings.py b/input-settings/ft_input_settings.py index f9d0660..536b7be 100644 --- a/input-settings/ft_input_settings.py +++ b/input-settings/ft_input_settings.py @@ -15,6 +15,9 @@ to the input relay over its control socket (@frametop_relay): - Pointer: speed, dot size, distance and the rest, applied live. - Ignored panels: SteamVR overlays the pointer passes through (POINTER_IGNORE), by app or one by one. The helper lists them (@ft_pointer_helper "overlays"). + - Games: pausing Frametop while a VR game runs, so it leaves the CPU and GPU to the game + (input/game_pause.py, in the relay: "pause on|off|?"): pause now, pause by itself during VR + games, the controller gesture that pauses and resumes, what happens to the desktop, a sound. - Gaze: the pointer's gaze mode (@ft_pointer_helper "gaze") and the gaze service (gaze/ft-gazed, @ft_gazed: status, forget, reload), with its eye tracker (SteamVR's or our own) and eye bias (GAZE_TRACKER, GAZE_EYE in frametop.conf). @@ -73,6 +76,7 @@ ACTION_LABELS = { "sens_down": "Slower pointer", "layout_reset": "Reset desktop screen layout", "screens_toggle": "Hide/show desktop screens", "keyboard_toggle": "Open/close keyboard", "float_toggle": "Float window in VR / put it back", "dock_all": "Put all floating windows back", + "pause_toggle": "Pause/resume Frametop (for VR games)", "key": "Pass through as key", "none": "Do nothing", } @@ -95,6 +99,13 @@ CONTROLLER_BUTTONS = { "right/bumper": "Right bumper", "right/trigger": "Right trigger", "right/grip": "Right grip", "right/thumbstick": "Right stick click", } +# Pausing for VR games (input/game_pause.py): the gesture a rules file without "pause_gesture" gets, +# both thumbsticks clicked together twice, and what can happen to the desktop. +DEFAULT_PAUSE_GESTURE = {"buttons": ["left/thumbstick", "right/thumbstick"], "presses": 2} +PAUSE_DESKTOP_MODES = {"hide": "Hide it and slow it down (windows stay open)", + "close": "Close it (windows close; it starts again when you resume)"} +PAUSE_REASONS = {"game": "a VR game started", "gesture": "controller gesture", "command": "ft-pause", + "settings": "from here"} # Gaze mode is a mouse and keyboard feature (docs/gaze-controllers.md; the relay's GAZE_ACTIONS). GAZE_ACTIONS = ("gaze_toggle", "gaze_precision", "gaze_drag", "gaze_left", "gaze_right", "gaze_quickcal") CONTROLLER_ACTIONS = [a for a in ACTION_LABELS if a not in ("key", "none") + GAZE_ACTIONS] @@ -293,6 +304,7 @@ class Backend(QObject): gazeChanged = Signal() driverChanged = Signal() panelsChanged = Signal() + pauseChanged = Signal() activity = Signal(str) # device id captured = Signal(int, str) # code, name capturedController = Signal(str, str) # button, label @@ -320,6 +332,8 @@ class Backend(QObject): self._check_asked = 0.0 # when quickcal or calibrate went to the gaze service (its errors) self._driver_block = "" # set by _check_driver self._panels = None # SteamVR's overlays, from the helper; None until it answers + self._pause = {} # the relay's pause state ("pause ?"), {} when it doesn't answer + self._pause_at = 0.0 self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) self.sock.bind("") # autobind an abstract address the relay can reply to self.sock.setblocking(False) @@ -352,6 +366,7 @@ class Backend(QObject): def _refresh(self): self._send("devices") + self._send("pause ?") self._send("vrstatus", HELPER) self._send("gaze ?", HELPER) if not self._send("status", GAZED) and self._gaze: @@ -365,6 +380,9 @@ class Backend(QObject): if self._gaze_mode is not None and now - self._gaze_at > 5: self._gaze_mode = None self.gazeChanged.emit() + if self._pause and now - self._pause_at > 5: + self._pause = {} + self.pauseChanged.emit() def _check_driver(self): block = driver_block() @@ -403,13 +421,20 @@ class Backend(QObject): if t == "devices": self._nodes = msg.get("nodes", []) self._pointer_mode = bool(msg.get("pointer_mode")) - self._relay_ok = True + was_ok, self._relay_ok = self._relay_ok, True + if not was_ok: + self.pauseChanged.emit() # (pauseStatus' "old") self.devicesChanged.emit() elif t == "overlays": panels = [o for o in msg.get("list", []) if isinstance(o, dict) and isinstance(o.get("key"), str)] if panels != self._panels: self._panels = panels self.panelsChanged.emit() + elif t == "pause": + self._pause_at = time.monotonic() + if msg != self._pause: + self._pause = msg + self.pauseChanged.emit() elif t == "vrstatus": self._vr = msg self._vr_at = time.monotonic() @@ -817,6 +842,108 @@ class Backend(QObject): else: self.message.emit("The pointer helper isn't running (frametop-pointer.service)", True) + # --- pausing for VR games --- + @Property("QVariantMap", notify=pauseChanged) + def pauseStatus(self): + """The relay's pause state, and a line saying it ("relay": False when it doesn't answer).""" + st = self._pause + if not st: + # "old": the relay answers, but not about pausing (it predates it). + return {"relay": False, "old": self._relay_ok, "paused": False, + "summary": "Unknown: the input relay " + ("can't pause" if self._relay_ok else "isn't answering")} + if st.get("paused"): + reason = PAUSE_REASONS.get(st.get("reason"), st.get("reason") or "") + summary = f"Paused since {time.strftime('%H:%M', time.localtime(st.get('since') or time.time()))}" + summary += f" ({reason})" if reason else "" + if st.get("ends_with_game"): + summary += ", until the game ends" + else: + summary = "Running" + (", with a VR game" if st.get("game") else "") + if st.get("busy"): + summary += "; working on it…" + return {"relay": True, "paused": bool(st.get("paused")), "summary": summary, + "gestureReader": bool(st.get("gesture_reader"))} + + @Slot(bool) + def setPaused(self, on): + if self._send(f"pause {'on' if on else 'off'} settings"): + QTimer.singleShot(300, self._refresh) + else: + self.message.emit("The input relay isn't running (frametop-input-relay.service)", True) + + def _set_rule(self, key, value, text): + rules = read_json(RULES_PATH) + rules[key] = value + self._save_rules(rules) + self.message.emit(text, False) + + @Property(bool, notify=mappingsChanged) + def pauseAuto(self): + return read_json(RULES_PATH).get("pause_auto", True) is not False + + @Slot(bool) + def setPauseAuto(self, on): + self._set_rule("pause_auto", bool(on), "VR games pause Frametop" if on else "VR games no longer pause Frametop") + + @Property("QVariantList", constant=True) + def pauseDesktopModes(self): + return [{"value": k, "text": v} for k, v in PAUSE_DESKTOP_MODES.items()] + + @Property(str, notify=mappingsChanged) + def pauseDesktop(self): + mode = read_json(RULES_PATH).get("pause_desktop") + return mode if mode in PAUSE_DESKTOP_MODES else "hide" + + @Slot(str) + def setPauseDesktop(self, mode): + if mode in PAUSE_DESKTOP_MODES: + self._set_rule("pause_desktop", mode, "Paused, the desktop: " + PAUSE_DESKTOP_MODES[mode].split(" (")[0].lower()) + + @Property(bool, notify=mappingsChanged) + def pauseSound(self): + return read_json(RULES_PATH).get("pause_sound", True) is not False + + @Slot(bool) + def setPauseSound(self, on): + self._set_rule("pause_sound", bool(on), "A sound on pause and resume" if on else "No sound on pause and resume") + + @Property("QVariantList", constant=True) + def gestureButtons(self): + return [{"value": "", "text": "None"}] + [{"value": b, "text": label} for b, label in CONTROLLER_BUTTONS.items()] + + @Property("QVariantMap", notify=mappingsChanged) + def pauseGesture(self): + """{"first", "second" ("" for none), "presses"} (the relay's game_pause.gesture_of).""" + spec = read_json(RULES_PATH).get("pause_gesture", DEFAULT_PAUSE_GESTURE) + buttons = [b for b in (spec or {}).get("buttons", []) if b in CONTROLLER_BUTTONS][:2] \ + if isinstance(spec, dict) and isinstance(spec.get("buttons"), list) else [] + presses = 1 if len(buttons) == 2 and spec.get("presses") == 1 else 2 + return {"first": buttons[0] if buttons else "", "second": buttons[1] if len(buttons) > 1 else "", + "presses": presses} + + @Slot(str, str, int) + def setPauseGesture(self, first, second, presses): + """No first button: no gesture. One button always takes two presses.""" + buttons = list(dict.fromkeys(b for b in (first, second) if b in CONTROLLER_BUTTONS)) + if not buttons: + self._set_rule("pause_gesture", None, "No controller gesture pauses Frametop") + return + presses = 1 if len(buttons) == 2 and presses == 1 else 2 + self._set_rule("pause_gesture", {"buttons": buttons, "presses": presses}, + "Pause gesture: " + self.gestureText(buttons, presses)) + + @staticmethod + def gestureText(buttons, presses): + names = " and ".join(CONTROLLER_BUTTONS[b] for b in buttons) + together = " together" if len(buttons) == 2 else "" + return f"{names}{together}, {'twice' if presses == 2 else 'once'}" + + @Property(str, notify=mappingsChanged) + def pauseGestureText(self): + g = self.pauseGesture + buttons = [b for b in (g["first"], g["second"]) if b] + return self.gestureText(buttons, g["presses"]) if buttons else "None" + # --- gaze --- def _gaze_status(self, status): now = time.monotonic() diff --git a/input-settings/main.qml b/input-settings/main.qml index fa8146c..3d6a83e 100644 --- a/input-settings/main.qml +++ b/input-settings/main.qml @@ -19,6 +19,7 @@ Kirigami.ApplicationWindow { Kirigami.Action { text: "Devices"; icon.name: "input-mouse"; onTriggered: root.show(devicesPage) }, Kirigami.Action { text: "Buttons"; icon.name: "input-keyboard"; onTriggered: root.show(buttonsPage) }, Kirigami.Action { text: "Controllers"; icon.name: "input-gamepad"; onTriggered: root.show(controllersPage) }, + Kirigami.Action { text: "Games"; icon.name: "applications-games"; onTriggered: root.show(gamesPage) }, Kirigami.Action { text: "Keyboard"; icon.name: "input-keyboard-virtual"; onTriggered: root.show(keyboardPage) }, Kirigami.Action { text: "Pointer"; icon.name: "transform-move"; onTriggered: root.show(pointerPage) }, Kirigami.Action { text: "Ignored panels"; icon.name: "view-hidden"; onTriggered: root.show(ignorePage) }, @@ -56,8 +57,8 @@ Kirigami.ApplicationWindow { pageStack.push(page) } - // FT_INPUT_PAGE=buttons|controllers|keyboard|pointer|ignore|gaze|bluetooth opens the app on that page. - pageStack.initialPage: ({ buttons: buttonsPage, controllers: controllersPage, keyboard: keyboardPage, + // FT_INPUT_PAGE=buttons|controllers|games|keyboard|pointer|ignore|gaze|bluetooth opens the app on that page. + pageStack.initialPage: ({ buttons: buttonsPage, controllers: controllersPage, games: gamesPage, keyboard: keyboardPage, pointer: pointerPage, ignore: ignorePage, gaze: gazePage, bluetooth: bluetoothPage })[startPage] || devicesPage @@ -826,6 +827,140 @@ Kirigami.ApplicationWindow { } } + // ---------------------------------------------------------------- Games + Component { + id: gamesPage + Kirigami.ScrollablePage { + id: gmpage + title: "Games" + property var st: backend.pauseStatus + + // The gesture's boxes follow the saved gesture (which may tidy what was picked). + function syncGesture() { + const g = backend.pauseGesture + firstBox.currentIndex = firstBox.indexOfValue(g.first) + secondBox.currentIndex = secondBox.indexOfValue(g.second) + pressBox.currentIndex = pressBox.indexOfValue(g.presses) + } + function saveGesture() { + backend.setPauseGesture(firstBox.currentValue, secondBox.currentValue, pressBox.currentValue) + } + Component.onCompleted: syncGesture() + Connections { + target: backend + function onMappingsChanged() { gmpage.syncGesture() } + } + + ColumnLayout { + spacing: Kirigami.Units.largeSpacing + + Kirigami.InlineMessage { + Layout.fillWidth: true + visible: !gmpage.st.relay + type: Kirigami.MessageType.Error + text: gmpage.st.old + ? "The input relay that runs is older and can't pause. Restart it: " + + "systemctl --user restart frametop-input-relay" + : "The input relay isn't answering (frametop-input-relay.service). It does the pausing." + } + + Kirigami.FormLayout { + Layout.fillWidth: true + + RowLayout { + Kirigami.FormData.label: "Frametop:" + Controls.Label { text: gmpage.st.summary } + Controls.Button { + text: gmpage.st.paused ? "Resume" : "Pause now" + icon.name: gmpage.st.paused ? "media-playback-start" : "media-playback-pause" + enabled: gmpage.st.relay + onClicked: backend.setPaused(!gmpage.st.paused) + } + } + Controls.Switch { + Kirigami.FormData.label: "VR games:" + text: "Pause while a VR game runs, and resume when it ends" + checked: backend.pauseAuto + onToggled: backend.setPauseAuto(checked) + } + + Kirigami.Separator { Kirigami.FormData.isSection: true; Kirigami.FormData.label: "Controller gesture" } + Controls.ComboBox { + id: firstBox + Kirigami.FormData.label: "Button:" + model: backend.gestureButtons + textRole: "text" + valueRole: "value" + Layout.preferredWidth: Kirigami.Units.gridUnit * 10 + onActivated: gmpage.saveGesture() + } + Controls.ComboBox { + id: secondBox + Kirigami.FormData.label: "Together with:" + model: backend.gestureButtons + textRole: "text" + valueRole: "value" + enabled: firstBox.currentValue !== "" + Layout.preferredWidth: Kirigami.Units.gridUnit * 10 + onActivated: gmpage.saveGesture() + } + Controls.ComboBox { + id: pressBox + Kirigami.FormData.label: "Pressed:" + model: [{ value: 2, text: "Twice" }, { value: 1, text: "Once" }] + textRole: "text" + valueRole: "value" + // One button pressed once would go off far too easily in games. + enabled: firstBox.currentValue !== "" && secondBox.currentValue !== "" + Layout.preferredWidth: Kirigami.Units.gridUnit * 10 + onActivated: gmpage.saveGesture() + } + Controls.Label { + Layout.fillWidth: true + wrapMode: Text.Wrap + opacity: 0.7 + text: (backend.pauseGesture.first === "" ? "No gesture: a mapped button, a key combination, or ft-pause still work." + : "Pauses and resumes Frametop, in games too. The game still gets the presses, so pick buttons it " + + "doesn't use this way. Two buttons count as together when both go down within a third of a " + + "second, and twice means within 0.7 s.") + + (gmpage.st.relay && backend.pauseGesture.first !== "" && !gmpage.st.gestureReader + ? " Not reading the controllers right now (SteamVR isn't running?)." : "") + } + + Kirigami.Separator { Kirigami.FormData.isSection: true; Kirigami.FormData.label: "While paused" } + Controls.ComboBox { + Kirigami.FormData.label: "The desktop:" + model: backend.pauseDesktopModes + textRole: "text" + valueRole: "value" + Layout.preferredWidth: Kirigami.Units.gridUnit * 20 + Component.onCompleted: currentIndex = indexOfValue(backend.pauseDesktop) + onActivated: backend.setPauseDesktop(currentValue) + } + Controls.Switch { + Kirigami.FormData.label: "Sound:" + text: "Play a sound on pause and resume" + checked: backend.pauseSound + onToggled: backend.setPauseSound(checked) + } + } + + Controls.Label { + Layout.fillWidth: true + wrapMode: Text.Wrap + opacity: 0.7 + text: "Paused, Frametop leaves the headset's CPU and GPU to the game. The gaze service and our eye " + + "tracker stop, and nothing reads SteamVR's eye tracking. Hand tracking and remote desktop stop " + + "if they run. The desktop hides and slows down, or closes, as chosen above. The 3D mouse lets go, " + + "so the mouse is a plain one for SteamVR. Mapped buttons and key combinations do nothing but " + + "pausing, the Steam menu, and your own commands. Resuming brings back what pausing stopped. " + + "Map \"Pause/resume Frametop\" to a button or key combination on the other pages, or run " + + "input/ft-pause." + } + } + } + } + // ---------------------------------------------------------------- Gaze Component { id: gazePage diff --git a/input/ft-pause b/input/ft-pause new file mode 100755 index 0000000..4a4d756 --- /dev/null +++ b/input/ft-pause @@ -0,0 +1,45 @@ +#!/usr/bin/python3 +"""ft-pause: pause Frametop for a VR game, so it leaves the CPU and GPU to the game, or resume it. + + ft-pause on|off|toggle pause or resume (input/game_pause.py has what pausing stops) + ft-pause status the pause state, as JSON (the default) + +The input relay does it (frametop-input-relay.service). So do the pause_toggle action, the +controller gesture (by default both thumbsticks clicked together twice), and, with auto pause on, +a VR game starting; Frametop Input Settings has the settings, on its Games page. +""" + +import json +import os +import socket +import sys + + +def ask(msg, timeout=2.0): + s = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM | socket.SOCK_CLOEXEC) + s.bind(f"\0ft_pause.{os.getpid()}") + s.settimeout(timeout) + try: + s.sendto(msg.encode(), "\0frametop_relay") + return json.loads(s.recv(65536)) + except ConnectionRefusedError: + sys.exit("the input relay isn't running (frametop-input-relay.service)") + except (socket.timeout, ValueError): + sys.exit("no answer from the input relay (an older one, without pausing?)") + finally: + s.close() + + +def main(): + cmd = sys.argv[1] if len(sys.argv) > 1 else "status" + if cmd in ("on", "off", "toggle"): + r = ask(f"pause {cmd} command") + print("Frametop paused" if r.get("paused") else "Frametop running") + elif cmd == "status": + print(json.dumps(ask("pause ?"), indent=1)) + else: + sys.exit(__doc__) + + +if __name__ == "__main__": + main() diff --git a/input/game_pause.py b/input/game_pause.py new file mode 100644 index 0000000..1d9e267 --- /dev/null +++ b/input/game_pause.py @@ -0,0 +1,483 @@ +"""Pause Frametop for VR games, so it leaves the headset's CPU and GPU to the game. The input +relay (input-relay.py) runs this: it's the one part of Frametop that always runs. + +Paused: + - The gaze service stops (frametop-gaze: ft-gazed, ft-gaze, our own eye tracker ft-eyes, the + gaze panel), and with it every read of SteamVR's eye tracking. Our frame grabber, the root + service ft-eyegrab, goes idle by itself 3 seconds after ft-eyes stops asking for frames. + - Hand tracking stops if it runs (frametop-camd, frametop-hands). + - The desktop, by "pause_desktop": "hide" (the default) hides every screen and floating window + and slows the desktop down ("pause on" to ft-screens: KWin gets one frame callback a second + instead of 90, so it and the apps on it hardly draw), and remote desktop stops if it runs + (session/remote-ctl.sh: krdp, Xvnc, FreeRDP). Windows stay open. "close" closes the desktop + (desktops.sh stop): its windows close, and resuming starts it again (about 12 seconds), in + its start profile if it has one. + - In the relay: the 3D mouse lets go and the mouse is a plain mouse for SteamVR, typing goes to + Steam, and mapped buttons and key combinations do nothing but PAUSED_ACTIONS and commands. +Resuming starts again only what pausing stopped. The pointer helper and ft-powerd keep running: +they cost little, and the helper is what says a VR game started. + +Toggled by: + - The pause_toggle action: a mouse button, a key combination, or a controller button outside + games (the pointer helper only reads those outside games). + - A controller gesture, by default both thumbsticks clicked together twice ("pause_gesture"). + It's read from vrserver's web socket (vrws.py), which works in games and takes nothing from + them, so the game sees the clicks too. + - input/ft-pause on|off|toggle|status, which uses the relay's control socket + ("pause on|off|toggle|?"). + - VR games, with "pause_auto" (on by default): the pointer helper says "vrgame 1|0" as a SteamVR + scene app starts and ends (and every 5 seconds). A game starting pauses; a pause that starts + while a game runs ends RESUME_DELAY after the game does, unless another game starts first. + Toggled back on during a game, Frametop stays on until that game ends. A pause that starts + outside a game lasts until it's toggled off. + +Settings in ~/.config/frametop-input.json (Frametop Input Settings, Games page): "pause_auto" +(bool), "pause_gesture" ({"buttons": one or two controller buttons, "presses": 1 or 2}, or null +for none; one button always takes two presses), "pause_desktop" ("hide" or "close"), and +"pause_sound" (a sound on pause and resume; bool). The state outlives a relay restart in +STATE_PATH. +""" +import json +import os +import queue +import socket +import subprocess +import threading +import time + +import vrws + +REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +DESKTOPS = os.path.join(REPO, "desktops.sh") +REMOTE_CTL = os.path.join(REPO, "session", "remote-ctl.sh") +RELAY = "\0frametop_relay" +SCREENS = "\0ft_screens" +STATE_PATH = f"/run/user/{os.getuid()}/frametop-pause.json" +# User services that pausing stops, if they run. +UNITS = ("frametop-gaze.service", "frametop-hands.service", "frametop-camd.service") +# What still works while paused, besides commands ("command:CMD"): the toggle itself, and Steam's menu. +PAUSED_ACTIONS = ("pause_toggle", "steam_menu") +DEFAULT_GESTURE = {"buttons": ["left/thumbstick", "right/thumbstick"], "presses": 2} +DESKTOP_MODES = ("hide", "close") +SOUNDS = {True: "/usr/share/sounds/ocean/stereo/device-removed.oga", + False: "/usr/share/sounds/ocean/stereo/device-added.oga"} +RESUME_DELAY = 5.0 # after a VR game ends: loading the next one can end one scene app and start another +GAME_STALE = 12.0 # the helper repeats "vrgame" every 5 s; silence this long means no game (or no SteamVR) +CHORD_WINDOW = 0.3 # a two-button gesture's buttons go down within this of each other +DOUBLE_WINDOW = 0.7 # a double press's second press comes within this of the first + + +def gesture_of(rules, buttons): + """The rules' pause gesture as (buttons, presses), or None for none. `buttons` are the + controller buttons there are (the relay's VR_BUTTONS).""" + spec = rules.get("pause_gesture", DEFAULT_GESTURE) + if not isinstance(spec, dict) or not isinstance(spec.get("buttons"), list): + return None + chosen = tuple(dict.fromkeys(b for b in spec["buttons"] if b in buttons))[:2] + if not chosen: + return None + return chosen, 1 if len(chosen) == 2 and spec.get("presses") == 1 else 2 + + +def settings_of(rules, buttons): + return {"auto": rules.get("pause_auto", True) is not False, + "gesture": gesture_of(rules, buttons), + "desktop": rules.get("pause_desktop") if rules.get("pause_desktop") in DESKTOP_MODES else "hide", + "sound": rules.get("pause_sound", True) is not False} + + +class Gesture: + """The pause gesture, from button presses and releases. A press of a two-button gesture is + both buttons going down within CHORD_WINDOW of each other, so a button held for a while (to + sprint, say) and the other clicked doesn't count. Two presses within DOUBLE_WINDOW make a + double press.""" + + def __init__(self, buttons, presses): + self.buttons, self.presses = tuple(buttons), presses + self.down = {} # button -> when it went down + self.first = None # when the first press of a double press was + self.fired = None + + def reset(self): + self.down.clear() + self.first = None + + def feed(self, button, pressed, now): + """A button's state; True when it completes the gesture.""" + if button not in self.buttons: + return False + if not pressed: + self.down.pop(button, None) + return False + if button in self.down: + return False # still down + self.down[button] = now + if len(self.down) < len(self.buttons) or now - min(self.down.values()) > CHORD_WINDOW: + return False + if self.presses == 2 and (self.first is None or now - self.first > DOUBLE_WINDOW): + self.first = now + return False + self.first = None + if self.fired is not None and now - self.fired < 1.0: + return False # a triple press is still one + self.fired = now + return True + + +class ControllerWatch(threading.Thread): + """Follows the Frame controllers on vrserver's web socket for the pause gesture, and sends + the relay "pause toggle gesture" when it's made. Reconnects every 5 seconds while SteamVR + is away. With no gesture set, it doesn't connect.""" + + def __init__(self, log): + super().__init__(name="pause-gesture", daemon=True) + self.log = log + self.lock = threading.Lock() + self.gesture = None + self.changed = threading.Event() + self.connected = False + self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) + + def set_gesture(self, spec): + """(buttons, presses), or None for no gesture.""" + with self.lock: + g = self.gesture + if spec is None: + self.gesture = None + elif g is None or (g.buttons, g.presses) != spec: + self.gesture = Gesture(*spec) + self.changed.set() + + def current(self): + with self.lock: + return self.gesture + + def run(self): + while True: + if self.current() is None: + if self.connected: + self.log("pause gesture: none set, stopped reading the controllers") + self.connected = False + self.changed.wait() + self.changed.clear() + continue + try: + self.session() + except (OSError, ValueError) as e: # urllib's errors are OSErrors + if self.connected: + self.log(f"pause gesture: SteamVR's web socket went away ({e}); retrying") + self.connected = False + self.changed.wait(5) + self.changed.clear() + + def session(self): + ws = vrws.VrSocket() + try: + ws.open(f"frametop_pause_{os.getpid()}") + sides = {} # root path -> side + next_poll = 0.0 + while True: + g = self.current() + if g is None: + return + now = time.monotonic() + if now >= next_poll: + # A controller connects later, or changes its root path when the 3D mouse's + # virtual controller takes or gives back its hand role. + next_poll = now + 3.0 + found = vrws.controllers() + if found != sides: + for path in sides.keys() - found.keys(): + ws.subscribe(path, on=False) + for path in found.keys() - sides.keys(): + ws.subscribe(path) + sides = found + g.reset() + if not self.connected: + self.connected = True + self.log(f"pause gesture: following {' and '.join(sorted(sides.values())) or 'no'} " + f"controller{'s' if len(sides) != 1 else ''} on SteamVR's web socket") + wanted = {b: (b.split("/", 1)[0], f"/input/{b.split('/', 1)[1]}/click") for b in g.buttons} + for text in ws.messages(timeout=1.0): + # About 160 messages a second, nearly all capacitive sensing: parse only the + # few that carry one of the gesture's buttons. + if not any(c in text for _, c in wanted.values()): + continue + try: + msg = json.loads(text) + except ValueError: + continue + side = sides.get(msg.get("device")) + components = msg.get("components") or {} + for button, (s, component) in wanted.items(): + if s == side and component in components: + if g.feed(button, bool(components[component]), time.monotonic()): + self.log("pause gesture") + try: + self.sock.sendto(b"pause toggle gesture", RELAY) + except OSError: + pass + finally: + ws.close() + + +def run(cmd, timeout=30): + """A command's exit status (None if it couldn't run), its output going nowhere.""" + try: + return subprocess.run(cmd, stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, timeout=timeout).returncode + except (OSError, subprocess.TimeoutExpired): + return None + + +def output(cmd, timeout=10): + try: + return subprocess.run(cmd, stdin=subprocess.DEVNULL, capture_output=True, text=True, + timeout=timeout).stdout + except (OSError, subprocess.TimeoutExpired): + return "" + + +def active(unit): + return run(["systemctl", "--user", "-q", "is-active", unit], timeout=10) == 0 + + +def desktop_running(): + return run(["pgrep", "-x", "ft-screens"], timeout=5) == 0 + + +class Worker(threading.Thread): + """Does the slow part of pausing and resuming, in order, one job at a time: stopping and + starting services, remote desktop, and closing or starting the desktop. `stopped` is what + the last pause stopped, for the resume to start again.""" + + def __init__(self, log, save): + super().__init__(name="pause-worker", daemon=True) + self.log, self.save = log, save + self.jobs = queue.Queue() + self.lock = threading.Lock() + self.stopped = {"units": [], "remote": False, "desktop": None} + self.busy = False + + def submit(self, job, *args): + self.busy = True + self.jobs.put((job, args)) + + def run(self): + while True: + job, args = self.jobs.get() + try: + job(*args) + except Exception as e: # one failure mustn't stop the next pause or resume + self.log(f"pause: {job.__name__} failed: {e!r}") + self.busy = not self.jobs.empty() + self.save() + + def snapshot(self): + with self.lock: + return {"units": list(self.stopped["units"]), "remote": self.stopped["remote"], + "desktop": self.stopped["desktop"]} + + def restore(self, stopped): + with self.lock: + self.stopped = {"units": [u for u in stopped.get("units", []) if u in UNITS], + "remote": bool(stopped.get("remote")), + "desktop": stopped.get("desktop") if stopped.get("desktop") in ("hidden", "closed") else None} + + def stop_units(self): + """Stop the UNITS that run, adding them to what the resume starts again.""" + units = [u for u in UNITS if active(u)] + if units: + run(["systemctl", "--user", "stop", *units], timeout=30) + with self.lock: + self.stopped["units"] += [u for u in units if u not in self.stopped["units"]] + self.log(f"pause: stopped {', '.join(units)}") + + def pause(self, desktop): + if desktop == "close": + if desktop_running(): + run([DESKTOPS, "stop"], timeout=40) # remote desktop goes with it + with self.lock: + self.stopped["desktop"] = "closed" + self.log("pause: desktop closed") + else: + if "running=1" in output([REMOTE_CTL, "status"]): + run([REMOTE_CTL, "stop"]) + with self.lock: + self.stopped["remote"] = True + self.log("pause: remote desktop stopped") + with self.lock: + self.stopped["desktop"] = "hidden" # ft-screens was told already (or isn't running) + self.stop_units() + + def enforce(self): + """While paused: SteamVR restarted, and started the gaze service with it.""" + self.stop_units() + + def resume(self): + stopped = self.snapshot() + units = stopped["units"] + if units: + # They need SteamVR (Requisite=steamvr.service): without it they start with it next time. + if run(["systemctl", "--user", "start", *units], timeout=30) == 0: + self.log(f"resume: started {', '.join(units)}") + else: + self.log(f"resume: couldn't start {', '.join(units)} (SteamVR not running?)") + if stopped["remote"]: + # In a scope of its own, not this service's, so a relay restart doesn't end it. + run(["systemd-run", "--user", "--scope", "--collect", "--quiet", REMOTE_CTL, "start"]) + self.log("resume: remote desktop started") + if stopped["desktop"] == "closed": + if run([DESKTOPS, "start"], timeout=60) == 0: + self.log("resume: desktop started") + else: + self.log("resume: the desktop didn't start (see /tmp/frametop-session.log)") + with self.lock: + self.stopped = {"units": [], "remote": False, "desktop": None} + + +class GamePause: + """The pause state, its automatic side, and the relay's way in. `on_change(paused)` is + called (from the relay's thread) when the state flips, for the relay's own part.""" + + def __init__(self, log, on_change, buttons): + self.log, self.on_change, self.buttons = log, on_change, buttons + self.lock = threading.Lock() # the state file, written from both threads + self.paused = False + self.reason = "" # what paused it: gesture, button, key, controller, command, game + self.since = 0.0 # wall clock + self.ends_with_game = False # resume when the VR game ends + self.game = False + self.game_seen = 0.0 # last "vrgame" from the helper + self.resume_at = None + self.settings = settings_of({}, buttons) + self.screens = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM | socket.SOCK_NONBLOCK) + self.worker = Worker(log, self.save) + self.watch = ControllerWatch(log) + self.worker.start() + self.watch.start() + self.load() + + # --- state file --- + def save(self): + with self.lock: + state = {"paused": self.paused, "reason": self.reason, "since": self.since, + "ends_with_game": self.ends_with_game, "stopped": self.worker.snapshot()} + try: + tmp = STATE_PATH + ".tmp" + with open(tmp, "w") as f: + json.dump(state, f) + os.replace(tmp, STATE_PATH) + except OSError as e: + self.log(f"pause: can't save {STATE_PATH}: {e}") + + def load(self): + """A relay restarted while paused stays paused (the relay's part is applied by its first + configure).""" + try: + with open(STATE_PATH) as f: + state = json.load(f) + except (OSError, ValueError): + return + if not isinstance(state, dict) or not state.get("paused"): + return + self.paused = True + self.reason = str(state.get("reason", "")) + self.since = float(state.get("since", 0) or 0) + self.ends_with_game = bool(state.get("ends_with_game")) + self.worker.restore(state.get("stopped") or {}) + self.log(f"paused since before this relay started ({self.reason})") + self.worker.submit(self.worker.enforce) + + # --- settings --- + def configure(self, rules): + self.settings = settings_of(rules, self.buttons) + self.watch.set_gesture(self.settings["gesture"]) + if not self.settings["auto"]: + self.ends_with_game, self.resume_at = False, None + + # --- pausing and resuming --- + def sound(self, paused): + if self.settings["sound"] and os.path.exists(SOUNDS[paused]): + try: + subprocess.Popen(["pw-play", SOUNDS[paused]], stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, start_new_session=True) + except OSError: + pass + + def tell_screens(self, paused): + try: + self.screens.sendto(b"pause on" if paused else b"pause off", SCREENS) + except OSError: + pass # the desktop isn't running + + def pause(self, reason, now): + if self.paused: + return + self.paused, self.reason, self.since = True, reason, time.time() + self.ends_with_game = self.game and self.settings["auto"] + self.resume_at = None + self.log(f"Frametop paused ({reason}{', until the game ends' if self.ends_with_game else ''})") + self.sound(True) + self.tell_screens(True) # at once, even when the desktop is closing: that takes a while + self.on_change(True) + self.worker.submit(self.worker.pause, self.settings["desktop"]) + self.save() + + def resume(self, reason, now): + if not self.paused: + return + self.paused, self.resume_at, self.ends_with_game = False, None, False + self.log(f"Frametop resumed ({reason})") + self.sound(False) + self.tell_screens(False) + self.on_change(False) + self.worker.submit(self.worker.resume) + self.save() + + def toggle(self, reason, now): + """Pausing only happens by itself as a game starts, so a game resumed during stays on.""" + (self.resume if self.paused else self.pause)(reason, now) + + def set(self, on, reason, now): + if on != self.paused: + self.toggle(reason, now) + + # --- VR games --- + def game_state(self, running, now): + """From the pointer helper: a VR game runs or not (on a change and every 5 s).""" + self.game_seen = now + if running == self.game: + return + self.game = running + self.log(f"a VR game {'started' if running else 'ended'}") + if running: + self.resume_at = None # the next game came before the resume + if self.settings["auto"]: + self.pause("game", now) + elif self.paused and self.ends_with_game: + self.resume_at = now + RESUME_DELAY + + def tick(self, now): + if self.game and now - self.game_seen > GAME_STALE: + self.game_state(False, now) # the helper went quiet: SteamVR (or the helper) stopped + if self.resume_at is not None and now >= self.resume_at: + self.resume("the game ended", now) + + def timeout(self, now): + """How long the relay may wait before tick() has something to do.""" + if self.resume_at is not None: + return max(0.05, self.resume_at - now) + return 1.0 if self.game else 3600.0 + + def helper_started(self): + """The pointer helper (re)started, so SteamVR did too, maybe with the gaze service.""" + if self.paused: + self.worker.submit(self.worker.enforce) + + def status(self): + return {"t": "pause", "paused": self.paused, "reason": self.reason, "since": self.since, + "ends_with_game": self.ends_with_game, "game": self.game, + "busy": self.worker.busy, "stopped": self.worker.snapshot(), + "gesture_reader": self.watch.connected, "auto": self.settings["auto"], + "desktop": self.settings["desktop"]} diff --git a/input/input-relay.py b/input/input-relay.py index 8f6c861..5d935b7 100755 --- a/input/input-relay.py +++ b/input/input-relay.py @@ -32,7 +32,8 @@ keyboard_toggle = open or close Frametop's keyboard, float_toggle = float the de pointer (else the active one) in VR, or put it back if it floats, dock_all = put every floating window back (both to ft-floatd, @frametop_float), profile:NAME = switch to that profile (ft-layout use NAME: its screens and apps; docs/profiles.md), steam_menu = open the SteamVR dashboard on -Steam's menu, or close the dashboard (steam/ft-steam menu, through Steam's UI), command:CMD = run +Steam's menu, or close the dashboard (steam/ft-steam menu, through Steam's UI), pause_toggle = +pause Frametop for a VR game, or resume it (game_pause.py), command:CMD = run CMD with sh -c (on the host, as this service: its environment, output to its log, and COMMAND_PATH, so ft-layout, ft-float and ft-steam need no path), key = pass through as a key, none). @@ -45,8 +46,8 @@ A modifier on its own ("125": Meta) is a tap: pressed and released with no other button, or scroll in between; the desktop gets an F24 press before its release, so Plasma's launcher doesn't open on a Meta tap that's bound. A rules file without "key_bindings" gets DEFAULT_KEY_BINDINGS (Meta tap: steam_menu, Meta+J: gaze_left, Meta+K: gaze_right, Meta+Shift+F: -float_toggle); one with its own, even an empty one, doesn't. The float, profile, Steam menu and -command actions work without pointer mode too. A combination with Meta also sends the desktop an +float_toggle); one with its own, even an empty one, doesn't. The float, profile, Steam menu, +pause and command actions work without pointer mode too. A combination with Meta also sends the desktop an F24 press and Meta's release right away: so letting go of Meta doesn't open Plasma's launcher, and a gaze click isn't Meta+click (KWin's window move and resize). Another key while Meta is still held gives the desktop Meta back. While typing goes to Steam, keyboards aren't grabbed, so Steam @@ -92,6 +93,12 @@ keyboards) get their volume entries remapped to unused stand-in codes, so their other keys keep working for SteamVR; a device without a keymap that has only volume keys (the headset's pmic_resin) is grabbed. The keymaps go back when the relay exits. +Frametop can pause for VR games (game_pause.py: by hand, with a controller gesture, or by itself +while a game runs). Paused, it stops the services that cost the game CPU and GPU, and the relay +plays a plain one: the pointer devices feed the virtual mouse and keyboard as without pointer +mode, typing goes to Steam, and mapped buttons and key combinations do only pause_toggle, +steam_menu and command:CMD (game_pause.PAUSED_ACTIONS). + Pointer mode (POINTER=1 in ~/.config/frametop.conf) sends pointer devices to the ft-pointer helper (pointer/helper), which drives the ft_pointer SteamVR driver. With POINTER=0, pointer devices go to the virtual mouse and @@ -104,7 +111,10 @@ Control socket (abstract datagram @frametop_relay, JSON replies to the sender): vrcapture take every controller button for s seconds (0: stop), so the settings app can capture one; watchers see them as events with id frame_controller vrbtn, vrhello, gazeawake from the pointer helper (above) + vrgame 1|0 from the pointer helper: a VR game runs (on a change and every 5 s), for pausing textfield 1|0 from the desktop's input method (above) + pause on|off|toggle [reason] pause Frametop or resume it (input/ft-pause; the gesture reader) + pause ? the pause state, as {"t": "pause", ...} Runs on the Frame host as a user service (frametop-input-relay.service). The virtual devices are parked in systemd's file descriptor store, so a relay @@ -130,6 +140,8 @@ import subprocess import sys import time +import game_pause + # Linux input constants (include/uapi/linux/input-event-codes.h, input.h, uinput.h). EV_SYN, EV_KEY, EV_REL, EV_MSC = 0x00, 0x01, 0x02, 0x04 SYN_REPORT = 0 @@ -196,7 +208,7 @@ RULES_PATH = os.path.expanduser("~/.config/frametop-input.json") ACTIONS = ("left", "right", "middle", "back", "scroll_up", "scroll_down", "dashboard", "recenter", "pointer_toggle", "follow_toggle", "gaze_toggle", "gaze_precision", "gaze_drag", "gaze_left", "gaze_right", "gaze_quickcal", "sens_up", "sens_down", "layout_reset", "screens_toggle", "keyboard_toggle", "float_toggle", - "dock_all", "steam_menu", "key", "none") + "dock_all", "steam_menu", "pause_toggle", "key", "none") # Gaze mode is a mouse feature: these never come from a controller button (docs/gaze-controllers.md). GAZE_ACTIONS = ("gaze_toggle", "gaze_precision", "gaze_drag", "gaze_left", "gaze_right", "gaze_quickcal") # Key combinations a rules file without "key_bindings" gets: a Meta tap opens Steam's menu, Meta+J @@ -229,7 +241,17 @@ def known_action(a): def needs_pointer(a): - return a not in FLOAT_ACTIONS and a != "steam_menu" and not a.startswith((PROFILE, COMMAND)) + return a not in FLOAT_ACTIONS and a not in ("steam_menu", "pause_toggle") and not a.startswith((PROFILE, COMMAND)) + + +def works_paused(a): + """An action that still does something while Frametop is paused (game_pause.py).""" + return a in game_pause.PAUSED_ACTIONS or a.startswith(COMMAND) + + +# What pressed pause_toggle, for the log (do_action's source). +PAUSE_SOURCES = {"mouse": "mouse button", "keyboard": "key combination", "left": "controller button", + "right": "controller button"} KEYS = "\0frametop_keys" # keys of keyboards grabbed for the desktop, for other readers REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) FT_LAYOUT = os.path.join(REPO, "layout", "ft-layout") @@ -605,6 +627,22 @@ class Pointer: pending = (self.scroll_until, self.claim_at, self.claim_release, self.system_at, self.system_release) return 0.02 if any(t is not None for t in pending) else 0.5 + def stand_down(self): + """Frametop is pausing: a pulse under way ends now, and the pointer lets go.""" + if self.system_release is not None: + self.send("btn system 0") + if self.claim_release is not None: + self.send("btn a 0") + if self.scroll_until is not None: + self.send("scroll 0 0") + self.system_at = self.system_release = self.claim_at = self.claim_release = self.scroll_until = None + self.dx = self.dy = self.pending = 0 + self.gaze_awake_until = 0.0 + if self.active: + self.send("hide") + self.active = False + log("pointer off (paused)") + class Node: """One input event node: a candidate device (mouse or keyboard, USB or Bluetooth), @@ -688,23 +726,42 @@ def main(): # desktop_until: typing goes to the Frametop desktop until then (ft-screens says so # every second); typing_applied: the grabs match that as of the last apply_roles(). # vr_capture_until: every controller button is taken until then (the settings app capturing one). - state = {"pointer": None, "rules": {}, "share_keys": False, + # pointer: the 3D mouse while it's in use, pointer_conf: the one the config asks for (they + # differ while Frametop is paused). + state = {"pointer": None, "pointer_conf": None, "rules": {}, "share_keys": False, "desktop_until": 0.0, "typing_applied": None, "vr_capture_until": 0.0} + def pause_changed(paused): + """Frametop paused or resumed (game_pause.py): the relay's own part.""" + p = state["pointer_conf"] + if paused: + if p: + p.stand_down() + state["pointer"] = None + state["desktop_until"] = 0.0 # typing goes to Steam + else: + state["pointer"] = p + apply_roles() + vr_bind(time.monotonic()) + + pause = game_pause.GamePause(log, pause_changed, VR_BUTTONS) + def load_config(): conf = read_config() state["rules"] = read_rules() state["share_keys"] = conf.get("SHARE_KEYS", "0") == "1" if conf.get("POINTER", "0") == "1": - p = state["pointer"] or Pointer(0.03, 30) + p = state["pointer_conf"] or Pointer(0.03, 30) p.sensitivity = float(conf.get("POINTER_SENSITIVITY", "0.03")) p.idle = float(conf.get("POINTER_IDLE", "30")) p.wake_counts = int(conf.get("POINTER_WAKE_COUNTS", "40")) - state["pointer"] = p + state["pointer_conf"] = p log(f"pointer mode: {p.sensitivity} deg/count, idle {p.idle} s, wake {p.wake_counts} counts") else: - state["pointer"] = None + state["pointer_conf"] = None log("pointer mode off: pointer devices feed the virtual mouse and keyboard") + state["pointer"] = None if pause.paused else state["pointer_conf"] + pause.configure(state["rules"]) load_config() tap = None # the modifier (folded, MODIFIERS) pressed alone, with nothing since: its release is a tap @@ -719,7 +776,7 @@ def main(): state["vr_capture_until"] = 0.0 buttons = " ".join(b for b, a in state["rules"]["controller_buttons"].items() if b in VR_BUTTONS and known_action(a) and a not in ("key", "none") - and a not in GAZE_ACTIONS) or "-" + and a not in GAZE_ACTIONS and (not pause.paused or works_paused(a))) or "-" if state["rules"].get("controller_in_games"): buttons = "+games " + buttons try: @@ -764,6 +821,12 @@ def main(): def do_action(action, value, now, source="mouse"): """A mapped mouse or controller button, or key combination (pointer mode only, but for the actions needs_pointer() says don't).""" + if action == "pause_toggle": + if value == 1: + pause.toggle(PAUSE_SOURCES.get(source, source), now) + return + if pause.paused and not works_paused(action): + return if action == "keyboard_toggle": if value == 1 and vr_keyboard_mode() != "never": vr_keyboard("toggle") @@ -832,7 +895,7 @@ def main(): return value == 2 and code in combos_down combo = "+".join(str(c) for c in sorted(held_modifiers) + [code]) action = state["rules"]["key_bindings"].get(combo) - if not known_action(action) or action in ("key", "none"): + if not known_action(action) or action in ("key", "none") or (pause.paused and not works_paused(action)): return False combos_down[code] = action if held_meta - meta_hidden: @@ -853,7 +916,7 @@ def main(): def modifier_tap(mod, now): """A modifier pressed and released alone: its binding, if it has one ("125": Meta tap).""" action = state["rules"]["key_bindings"].get(str(mod)) - if not known_action(action) or action in ("key", "none"): + if not known_action(action) or action in ("key", "none") or (pause.paused and not works_paused(action)): return # The desktop gets a key in between before the release goes there, so the tap isn't one # there too: a Meta tap would open Plasma's launcher. @@ -1054,7 +1117,7 @@ def main(): try: if cmd == "keyboard": # From ft-screens (unbound, no reply): where typing goes, repeated every second. - desktop = len(words) > 1 and words[1] == "desktop" + desktop = len(words) > 1 and words[1] == "desktop" and not pause.paused state["desktop_until"] = now + 3.0 if desktop else 0.0 continue if cmd == "vrbtn" and len(words) == 3 and words[1] in VR_BUTTONS and words[2] in ("0", "1"): @@ -1062,6 +1125,20 @@ def main(): continue if cmd == "vrhello": vr_bind(now) + pause.helper_started() + continue + if cmd == "vrgame" and len(words) == 2: + pause.game_state(words[1] == "1", now) + continue + if cmd == "pause" and len(words) >= 2 and words[1] in ("on", "off", "toggle", "?"): + # From input/ft-pause, the settings app, or the gesture reader (unbound, no reply). + reason = words[2] if len(words) > 2 else "command" + if words[1] == "toggle": + pause.toggle(reason, now) + elif words[1] != "?": + pause.set(words[1] == "on", reason, now) + if addr: + reply(addr, pause.status()) continue if cmd == "textfield" and len(words) == 2: text_field(words[1] == "1") @@ -1073,7 +1150,8 @@ def main(): if not addr: continue # unbound sender, nowhere to reply if cmd == "devices": - reply(addr, {"t": "devices", "pointer_mode": state["pointer"] is not None, + reply(addr, {"t": "devices", "pointer_mode": state["pointer_conf"] is not None, + "paused": pause.paused, "actions": ACTIONS, "nodes": [n.describe() for n in nodes.values() if n.candidate]}) elif cmd == "watch": @@ -1156,16 +1234,18 @@ def main(): apply_roles() ready, _, _ = select.select(list(nodes) + [control], [], [], - volume.timeout(now, pointer.timeout() if pointer else 0.5)) + min(volume.timeout(now, pointer.timeout() if pointer else 0.5), pause.timeout(now))) now = time.monotonic() if pointer: pointer.tick(now) volume.tick(now) + pause.tick(now) if state["vr_capture_until"] and now >= state["vr_capture_until"]: vr_bind(now) # capture over: back to the mapped buttons if (now < state["desktop_until"]) != state["typing_applied"] or waiting: waiting = apply_roles() for fd in ready: + pointer = state["pointer"] # (a pause or resume in this batch changes it) if fd is control: handle_control(now) continue @@ -1220,7 +1300,7 @@ def main(): continue if etype == EV_KEY: action = buttons.get(str(code), DEFAULT_BUTTONS.get(code, "key")) - if pointer and action not in ("key", "none"): + if (pointer or action == "pause_toggle") and action not in ("key", "none"): do_action(action, value, now) continue if action == "none": diff --git a/input/test/keys-test.py b/input/test/keys-test.py index 12a3cd3..005af84 100755 --- a/input/test/keys-test.py +++ b/input/test/keys-test.py @@ -3,8 +3,9 @@ Runs the relay's main() against a fake pass-through keyboard and a fake mouse (pipes), with every socket it sends to renamed, no uinput devices, no grabs, and ft-steam swapped for a -logger. Nothing reaches the live desktop, SteamVR, or the running relay, so it's safe next -to them. Rules come from the test, not ~/.config/frametop-input.json. +logger. Pausing (game_pause.py) is a stub: no threads, state file, or services. Nothing reaches +the live desktop, SteamVR, or the running relay, so it's safe next to them. Rules come from the +test, not ~/.config/frametop-input.json. input/test/keys-test.py [RELAY] (default: input/input-relay.py next to this folder) """ @@ -26,8 +27,41 @@ with open(relay_path) as f: src = f.read().replace('"\\0frametop_relay"', f'"\\0{tag}_relay"') relay = importlib.util.module_from_spec(importlib.util.spec_from_loader("relay", loader=None)) relay.__file__ = os.path.abspath(relay_path) +sys.path.insert(0, os.path.dirname(os.path.abspath(relay_path))) # its game_pause and vrws exec(compile(src, relay_path, "exec"), relay.__dict__) + +class StubPause: + """game_pause.GamePause without its threads, state file, or services.""" + paused = False + + def __init__(self, log, on_change, buttons): + self.on_change = on_change + stub["pause"] = self + + def toggle(self, reason, now): + self.paused = not self.paused + self.on_change(self.paused) + + def set(self, on, reason, now): + if on != self.paused: + self.toggle(reason, now) + + def status(self): + return {"t": "pause", "paused": self.paused} + + def timeout(self, now): + return 3600.0 + + def configure(self, *args): + pass + + game_state = tick = helper_started = configure + + +stub = {} +relay.game_pause.GamePause = StubPause + for name in ("SCREENS", "HELPER", "FLOAT", "GAZED", "KEYS"): setattr(relay, name, f"\0{tag}_{name.lower()}") OUT = tempfile.mkdtemp(prefix="ft-keys-test-") @@ -144,6 +178,15 @@ def use(key_bindings): typed() +def pause(cmd): + c = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) + c.bind("") + c.settimeout(2) + c.sendto(f"pause {cmd} test".encode(), f"\0{tag}_relay") + c.recv(4096) + time.sleep(0.05) + + def lines(path, wait=0.3): time.sleep(wait) try: @@ -163,7 +206,7 @@ def check(label, got, want): failures.append(label) -META, RMETA, SHIFT, CTRL, RCTRL, A, F, J = 125, 126, 42, 29, 97, 30, 33, 36 +META, RMETA, SHIFT, CTRL, RCTRL, A, F, J, P = 125, 126, 42, 29, 97, 30, 33, 36, 25 F24 = ["key 194 1", "key 194 0"] @@ -207,9 +250,25 @@ def tests(): check("right Ctrl tap runs its command, which finds Frametop's tools", lines(cmd_log), ["combo", "3"]) check("Ctrl tap: F24 before Ctrl's release", typed(), ["key 97 1"] + F24 + ["key 97 0"]) + use({"125+36": "gaze_left", "29": f"command:echo paused >> {cmd_log}", "42+125+25": "pause_toggle"}) + pause("on") + check("pause on: paused", stub["pause"].paused, True) + key(META, 1); key(J, 1); key(J, 0); key(META, 0) + check("paused: Meta+J (gaze click) does nothing and is typed as is", typed(), + ["key 125 1", "key 36 1", "key 36 0", "key 125 0"]) + key(RCTRL, 1); key(RCTRL, 0) + check("paused: a command still runs", lines(cmd_log), ["combo", "3", "paused"]) + typed() + key(SHIFT, 1); key(META, 1); key(P, 1); key(P, 0); key(META, 0); key(SHIFT, 0) + check("paused: Meta+Shift+P (pause_toggle) resumes", stub["pause"].paused, False) + check("Meta+Shift+P isn't typed", [k for k in typed() if k.startswith("key 25 ")], []) + key(META, 1); key(J, 1); key(J, 0); key(META, 0) + check("resumed: Meta+J is a combination again", typed(), ["key 125 1"] + F24 + ["key 125 0"]) + check("an empty command isn't an action", relay.known_action("command: "), False) - check("steam_menu and commands work without pointer mode", - (relay.needs_pointer("steam_menu"), relay.needs_pointer("command:ls")), (False, False)) + check("steam_menu, pause_toggle and commands work without pointer mode", + (relay.needs_pointer("steam_menu"), relay.needs_pointer("pause_toggle"), relay.needs_pointer("command:ls")), + (False, False, False)) print("FAILED: " + ", ".join(failures) if failures else "all passed", flush=True) shutil.rmtree(OUT, ignore_errors=True) os._exit(1 if failures else 0) diff --git a/input/test/pause-test.py b/input/test/pause-test.py new file mode 100755 index 0000000..ac110bf --- /dev/null +++ b/input/test/pause-test.py @@ -0,0 +1,214 @@ +#!/usr/bin/env python3 +"""Offline test of pausing for VR games (input/game_pause.py): the controller gesture, and when +Frametop pauses and resumes by itself. The worker (services, remote desktop, the desktop) and +the controller reader are stubs, the state file is a temporary one, and ft-screens' socket is +renamed, so nothing reaches the live Frametop, SteamVR, or the running relay. + + input/test/pause-test.py +""" +import os +import sys +import tempfile + +sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), "..")) +import game_pause as gp # noqa: E402 + +failures = [] + + +def check(label, got, want): + ok = got == want + print(("ok " if ok else "FAIL ") + label + ("" if ok else f": got {got!r}, want {want!r}"), flush=True) + if not ok: + failures.append(label) + + +# ---------------------------------------------------------------- the gesture +L, R = "left/thumbstick", "right/thumbstick" + + +def presses(g, events): + """events: (time, button, down); how many times the gesture went off.""" + return sum(g.feed(b, d, t) for t, b, d in events) + + +def both(t, hold=0.1, gap=0.02): + """Both sticks clicked at t (the right one gap later), held for hold.""" + return [(t, L, True), (t + gap, R, True), (t + hold, L, False), (t + hold + gap, R, False)] + + +g = gp.Gesture((L, R), 2) +check("both sticks twice: goes off", presses(g, both(0) + both(0.3)), 1) +g = gp.Gesture((L, R), 2) +check("both sticks once: nothing", presses(g, both(0)), 0) +g = gp.Gesture((L, R), 2) +check("both sticks twice, 0.9 s apart: nothing", presses(g, both(0) + both(0.9)), 0) +g = gp.Gesture((L, R), 2) +check("left held (sprint), right clicked twice: nothing", + presses(g, [(0, L, True), (1.0, R, True), (1.1, R, False), (1.3, R, True), (1.4, R, False)]), 0) +g = gp.Gesture((L, R), 2) +check("sticks 0.5 s apart, twice: nothing", presses(g, both(0, gap=0.5, hold=0.6) + both(1.2, gap=0.5, hold=0.6)), 0) +g = gp.Gesture((L, R), 2) +check("three double presses in a row: once", presses(g, both(0) + both(0.3) + both(0.6)), 1) +g = gp.Gesture((L, R), 2) +check("again after a second: twice in all", presses(g, both(0) + both(0.3) + both(2.0) + both(2.3)), 2) +g = gp.Gesture((L, R), 2) +check("repeated down states count once", + presses(g, [(0, L, True), (0.01, L, True), (0.02, R, True), (0.03, R, True), (0.1, L, False), (0.1, R, False)] + + both(0.3)), 1) +g = gp.Gesture((L, R), 1) +check("both sticks once (presses 1): goes off", presses(g, both(0)), 1) +g = gp.Gesture(("right/b",), 2) +check("one button double-clicked: goes off", + presses(g, [(0, "right/b", True), (0.1, "right/b", False), (0.3, "right/b", True)]), 1) +g = gp.Gesture(("right/b",), 2) +check("other buttons don't count", presses(g, [(0, "right/a", True), (0.2, "right/a", True)]), 0) + +VR = ("left/thumbstick", "right/thumbstick", "right/b", "right/a") +check("no pause_gesture: both sticks twice", gp.gesture_of({}, VR), ((L, R), 2)) +check("pause_gesture null: none", gp.gesture_of({"pause_gesture": None}, VR), None) +check("one button, presses 1: still twice", gp.gesture_of({"pause_gesture": {"buttons": ["right/b"], "presses": 1}}, VR), + (("right/b",), 2)) +check("unknown buttons dropped", gp.gesture_of({"pause_gesture": {"buttons": ["right/zz", "right/a"]}}, VR), + (("right/a",), 2)) +check("settings: defaults", gp.settings_of({}, VR)["auto"], True) +check("settings: unknown desktop mode is hide", gp.settings_of({"pause_desktop": "x"}, VR)["desktop"], "hide") + + +# ---------------------------------------------------------------- pausing by itself +class StubWorker: + def __init__(self, log, save): + self.jobs = [] + self.busy = False + + def start(self): + pass + + def submit(self, job, *args): + self.jobs.append(job.__name__) + + def snapshot(self): + return {"units": [], "remote": False, "desktop": None} + + def restore(self, stopped): + self.restored = stopped + + def pause(self, desktop): + pass + + def resume(self): + pass + + def enforce(self): + pass + + +class StubWatch: + connected = False + + def __init__(self, log): + pass + + def start(self): + pass + + def set_gesture(self, spec): + self.spec = spec + + +tmp = tempfile.mkdtemp(prefix="ft-pause-test-") +gp.STATE_PATH = os.path.join(tmp, "pause.json") +gp.SCREENS = f"\0ft_pause_test_{os.getpid()}_screens" +gp.Worker, gp.ControllerWatch = StubWorker, StubWatch +changes = [] + + +def new(rules=None): + """A relay starting afresh: not paused before.""" + if os.path.exists(gp.STATE_PATH): + os.remove(gp.STATE_PATH) + changes.clear() + p = gp.GamePause(lambda *a: None, changes.append, VR) + p.configure(dict({"pause_sound": False}, **(rules or {}))) + return p + + +p = new() +p.game_state(True, 0) +check("a game starts: paused", (p.paused, p.reason, p.ends_with_game), (True, "game", True)) +check("the relay heard", changes, [True]) +p.game_state(True, 5) +p.tick(5) +check("the helper repeats it: still paused, nothing new", (p.paused, changes), (True, [True])) +p.game_state(False, 10) +p.tick(12) +check("the game ends: not yet resumed", p.paused, True) +p.tick(15.1) +check("5 s later: resumed", (p.paused, changes), (False, [True, False])) +check("the worker paused and resumed", p.worker.jobs, ["pause", "resume"]) + +p = new() +p.game_state(True, 0) +p.game_state(False, 10) +p.game_state(True, 12) +p.tick(16) +check("the next game starts within 5 s: stays paused", p.paused, True) +p.game_state(False, 20) +p.tick(25.1) +check("and resumes 5 s after that one ends", p.paused, False) + +p = new() +p.game_state(True, 0) +p.toggle("gesture", 1) +check("resumed during a game: running", p.paused, False) +p.game_state(True, 6) +p.tick(30) +check("and stays running while that game runs", p.paused, False) +p.game_state(False, 40) +p.game_state(True, 50) +check("the next game pauses again", p.paused, True) + +p = new() +p.toggle("command", 0) +check("paused outside a game: not tied to one", (p.paused, p.ends_with_game), (True, False)) +p.game_state(True, 1) +p.game_state(False, 10) +p.tick(20) +check("a game comes and goes: still paused", p.paused, True) +p.toggle("command", 21) +check("toggled: running", p.paused, False) + +p = new({"pause_auto": False}) +p.game_state(True, 0) +check("auto off: a game doesn't pause", p.paused, False) +p.toggle("gesture", 1) +p.game_state(False, 10) +p.tick(20) +check("auto off: a pause during a game outlasts it", p.paused, True) + +p = new() +p.game_state(True, 0) +p.tick(0 + gp.GAME_STALE + 0.1) +check("the helper goes quiet: the game counts as ended", p.game, False) +p.tick(0 + gp.GAME_STALE + gp.RESUME_DELAY + 0.2) +check("and the pause ends", p.paused, False) + +p = new() +p.toggle("gesture", 0) +q = gp.GamePause(lambda *a: None, changes.append, VR) +check("a relay restarted while paused stays paused", (q.paused, q.reason), (True, "gesture")) +check("and stops the services again", q.worker.jobs, ["enforce"]) +q.toggle("gesture", 1) +r = gp.GamePause(lambda *a: None, changes.append, VR) +check("resumed: the next relay starts running", r.paused, False) + +p = new({"pause_gesture": None}) +check("pause_gesture null: the reader gets none", p.watch.spec, None) +check("status", sorted(p.status()), sorted(["t", "paused", "reason", "since", "ends_with_game", "game", "busy", + "stopped", "gesture_reader", "auto", "desktop"])) + +print("FAILED: " + ", ".join(failures) if failures else "all passed", flush=True) +for f in os.listdir(tmp): + os.remove(os.path.join(tmp, f)) +os.rmdir(tmp) +sys.exit(1 if failures else 0) diff --git a/input/vrws.py b/input/vrws.py new file mode 100755 index 0000000..ae41784 --- /dev/null +++ b/input/vrws.py @@ -0,0 +1,198 @@ +#!/usr/bin/python3 +"""vrws: vrserver's local web socket, with nothing but the standard library. + +vrserver (SteamVR) serves its controller binding page on 127.0.0.1:27062, and that page +follows the controllers' raw input through a web socket there (mailbox input_server, +request_input_state_updates). Reading it takes nothing from anyone: whatever the buttons are +bound to still happens, and it works whatever app has input focus, VR games included. SteamVR +input only reaches the focused app, or an overlay with global input, which then takes the button +from the game (pointer/helper/vrbuttons.h). frame-voice reads its push-to-talk button the same +way. It's undocumented, so a SteamVR update could change it (scripts/update-check.py checks it). +The host's Python has no `websockets` module, so this is a small RFC 6455 client of its own. + + controllers() the Frame controllers SteamVR has now: {root path: "left" | "right"}. A + controller's root path is /user/hand/, or /devices/cv/ while the + 3D mouse's virtual controller holds that hand role. + VrSocket() a connection: open(mailbox), subscribe(path), messages(timeout) -> raw texts, + each a JSON object like {"type": "update_component_states", "device": path, + "components": {"/input/thumbstick/click": true, ...}} with the components + that changed (mostly capacitive sensing, about 160 a second) + +Run as a program, it prints the controllers' button changes for a while (default 10 s): + + input/vrws.py [seconds] +""" +import base64 +import json +import os +import select +import socket +import struct +import sys +import time +import urllib.request + +HOST, PORT = "127.0.0.1", 27062 +ORIGIN = f"http://{HOST}:{PORT}" +# vrserver answers the binding page's own requests; these headers make ours look like them. +HEADERS = {"Referer": f"{ORIGIN}/dashboard/controllerbinding.html"} + + +def getstate(timeout=3): + """vrserver's /input/getstate.json: the devices and their input components.""" + req = urllib.request.Request(f"{ORIGIN}/input/getstate.json", headers=HEADERS) + with urllib.request.urlopen(req, timeout=timeout) as resp: + return json.load(resp) + + +def controllers(timeout=3): + """{root path: side} for the Frame controllers SteamVR has now.""" + return {d["root_path"]: d["side"] for d in getstate(timeout).get("devices", []) + if d.get("controller_type") == "frame_controller" and d.get("root_path") + and d.get("side") in ("left", "right")} + + +class VrSocket: + def __init__(self, timeout=3): + self.sock = socket.create_connection((HOST, PORT), timeout=timeout) + try: + key = base64.b64encode(os.urandom(16)).decode() + self.sock.sendall((f"GET / HTTP/1.1\r\nHost: {HOST}:{PORT}\r\nUpgrade: websocket\r\n" + f"Connection: Upgrade\r\nSec-WebSocket-Key: {key}\r\nSec-WebSocket-Version: 13\r\n" + f"Origin: {ORIGIN}\r\nReferer: {HEADERS['Referer']}\r\n\r\n").encode()) + head = b"" + while b"\r\n\r\n" not in head: + chunk = self.sock.recv(4096) + if not chunk: + raise OSError("vrserver closed the connection during the handshake") + head += chunk + head, rest = head.split(b"\r\n\r\n", 1) + status = head.split(b"\r\n", 1)[0] + if b" 101 " not in status + b" ": + raise OSError(f"vrserver refused the web socket: {status.decode(errors='replace')}") + except BaseException: + self.sock.close() + raise + self.sock.settimeout(None) + self.buf = bytearray(rest) + self.parts = bytearray() # a fragmented message so far + self.mailbox = "" + + def fileno(self): + return self.sock.fileno() + + def close(self): + try: + self._frame(0x8, b"") + except OSError: + pass + self.sock.close() + + def _frame(self, opcode, payload): + """One frame to vrserver: final, masked (as a client's must be).""" + n = len(payload) + head = bytes([0x80 | opcode]) + if n < 126: + head += bytes([0x80 | n]) + elif n < 65536: + head += bytes([0x80 | 126]) + struct.pack(">H", n) + else: + head += bytes([0x80 | 127]) + struct.pack(">Q", n) + mask = os.urandom(4) + self.sock.sendall(head + mask + bytes(b ^ mask[i % 4] for i, b in enumerate(payload))) + + def send(self, text): + self._frame(0x1, text.encode()) + + def open(self, mailbox): + self.send(f"mailbox_open {mailbox}") + self.mailbox = mailbox + + def subscribe(self, path, on=True): + kind = "request_input_state_updates" if on else "cancel_input_state_updates" + self.send("mailbox_send input_server " + + json.dumps({"type": kind, "device_path": path, "returnAddress": self.mailbox})) + + def messages(self, timeout=None): + """The text messages that came in, as strings: waits up to `timeout` seconds (None: + forever) for some, [] at the timeout. Raises OSError when the connection ends.""" + out = self._parse() + if out: + return out + if not select.select([self.sock], [], [], timeout)[0]: + return [] + chunk = self.sock.recv(65536) + if not chunk: + raise OSError("vrserver closed the web socket") + self.buf += chunk + return self._parse() + + def _parse(self): + """Every whole frame in the buffer; a partial one waits for the rest.""" + out, buf, pos = [], self.buf, 0 + while len(buf) - pos >= 2: + b0, b1 = buf[pos], buf[pos + 1] + n, head = b1 & 0x7F, 2 + if n == 126: + if len(buf) - pos < 4: + break + n, head = struct.unpack_from(">H", buf, pos + 2)[0], 4 + elif n == 127: + if len(buf) - pos < 10: + break + n, head = struct.unpack_from(">Q", buf, pos + 2)[0], 10 + masked = b1 & 0x80 + if masked: + head += 4 + if len(buf) - pos < head + n: + break + data = bytes(buf[pos + head:pos + head + n]) + if masked: # servers don't mask, but nothing says they can't + mask = buf[pos + head - 4:pos + head] + data = bytes(b ^ mask[i % 4] for i, b in enumerate(data)) + pos += head + n + opcode = b0 & 0x0F + if opcode == 0x8: + raise OSError("vrserver closed the web socket") + if opcode == 0x9: + self._frame(0xA, data) # ping: pong + elif opcode in (0x0, 0x1, 0x2): + self.parts += data + if b0 & 0x80: + out.append(self.parts.decode(errors="replace")) + self.parts = bytearray() + del buf[:pos] + return out + + +def main(): + seconds = float(sys.argv[1]) if len(sys.argv) > 1 else 10.0 + ws = VrSocket() + ws.open(f"frametop_vrws_{os.getpid()}") + sides = controllers() + for path, side in sides.items(): + print(f"{side} controller: {path}", flush=True) + ws.subscribe(path) + if not sides: + print("no Frame controllers", flush=True) + start = time.monotonic() + count = 0 + try: + while time.monotonic() - start < seconds: + for text in ws.messages(timeout=0.5): + count += 1 + if "/click" not in text: + continue + msg = json.loads(text) + side = sides.get(msg.get("device"), msg.get("device")) + for name, value in sorted(msg.get("components", {}).items()): + if name.endswith("/click"): + print(f"{time.monotonic() - start:7.3f} {side} {name[len('/input/'):-len('/click')]} " + f"{'down' if value else 'up'}", flush=True) + finally: + ws.close() + print(f"{count} messages in {seconds:.0f} s") + + +if __name__ == "__main__": + main() diff --git a/pointer/helper/ft-pointer.cpp b/pointer/helper/ft-pointer.cpp index 98d5e82..0b04582 100644 --- a/pointer/helper/ft-pointer.cpp +++ b/pointer/helper/ft-pointer.cpp @@ -931,8 +931,8 @@ int main() { Clock::time_point handUsed{}; // a gesture began then (keeps the pointer, like gaze mode) Clock::time_point lastTyping{}; // the relay's last "typing": a key on a keyboard // Gaze mode outside games, and its dot (see the top): lastMove/lastHeld/pulseAt. - bool inGame = false, gazeAwake = false; - Clock::time_point inGameAt{}, gazeAwakeAt{}; + bool inGame = false, gazeAwake = false, gameSent = false; + Clock::time_point inGameAt{}, gazeAwakeAt{}, gameSentAt{}; Clock::time_point lastMove{}, lastHeld{}, pulseAt{}; // The left button, as sent to the driver; pressRight: the next press is the right button // instead (gaze_right), heldButton: the one pressed. @@ -1283,6 +1283,13 @@ int main() { gazeAwakeAt = t; SendTo(out, "frametop_relay", awake ? "gazeawake 1" : "gazeawake 0"); } + // The relay pauses Frametop for VR games (input/game_pause.py). Repeated, so a relay + // that restarts learns it, and silence (SteamVR gone) ends the game there. + if (inGame != gameSent || t - gameSentAt > std::chrono::seconds(5)) { + gameSent = inGame; + gameSentAt = t; + SendTo(out, "frametop_relay", inGame ? "vrgame 1" : "vrgame 0"); + } } // Commands from the relay. diff --git a/screens/compositor.c b/screens/compositor.c index 95a44d6..c635d33 100644 --- a/screens/compositor.c +++ b/screens/compositor.c @@ -381,7 +381,10 @@ static void keys_update(struct server *s) { offsetof(struct sockaddr_un, sun_path) + 1 + sizeof name - 1); } -// Every ~11 ms (90 Hz): SteamVR events, and frame callbacks for screens that committed. +// Every ~11 ms (90 Hz): SteamVR events, and frame callbacks for screens that committed. While +// Frametop is paused for a VR game (everything hidden), every 100 ms, and the frame callbacks +// once a second: KWin draws a screen only after its callback, and its apps wait for theirs, so +// the desktop hardly draws until it's resumed. static int tick(void *data) { struct server *s = data; ft_vr_poll(handle_vr_event, s); @@ -395,14 +398,15 @@ static int tick(void *data) { } struct timespec now; clock_gettime(CLOCK_MONOTONIC, &now); - for (int i = 0; i < MAX_SCREENS; ++i) { + const bool paused = ft_vr_paused(); + for (int i = 0; i < MAX_SCREENS && (!paused || s->ticks % 10 == 0); ++i) { struct screen *sc = s->screens[i]; if (sc && sc->frame_pending) { sc->frame_pending = false; wlr_surface_send_frame_done(sc->toplevel->base->surface, &now); } } - wl_event_source_timer_update(s->tick, 11); + wl_event_source_timer_update(s->tick, paused ? 100 : 11); return 0; } diff --git a/screens/vr.cpp b/screens/vr.cpp index 065a869..bdda9b2 100644 --- a/screens/vr.cpp +++ b/screens/vr.cpp @@ -43,6 +43,10 @@ // own; also for flatscreen games, which aren't scene apps). // - during a VR game the screens hide unless the dashboard is open (g_inGames, default), // or stay visible over it; the hotkey still shows them. +// - paused ("pause on", from the input relay when Frametop pauses for a VR game, +// input/game_pause.py): every screen and floating window hides whatever the mode, the +// hotkey, or the dashboard says, and compositor.c gives KWin a frame callback once a +// second instead of 90 times, so KWin and its apps hardly draw. "pause off" undoes it. // - hand cutouts (handcut.cpp): where ft-hands (hands/) tracks a hand between an eye and a // screen, that eye sees through the screen (to Room View). Only then is the screen // drawn by us, into a side-by-side buffer (one half per eye); otherwise its client @@ -320,6 +324,7 @@ double g_gestureAngle = 20; // gesture: look within this of the controller std::string g_gestureHand = "left"; Lasers g_lasers = Lasers::OutsideGames; // when controllers' lasers work the screens (see the top) bool g_gameRunning = false; // a scene app (VR game) is running +bool g_paused = false; // Frametop paused for a VR game: everything hidden (see the top) InGames g_inGames = InGames::Hide; // during a VR game, the always mode acts like the dashboard mode // ---------------------------------------------------------------- chrome (bar, button, handle) @@ -734,6 +739,7 @@ void UpdateGame() { } bool ModeVisible() { + if (g_paused) return false; switch (EffectiveMode()) { case Mode::Always: return !g_manual; case Mode::Toggle: return g_manual; @@ -792,7 +798,7 @@ void UpdateVisibility() { Mat head; const bool haveHead = DevicePose(vr::k_unTrackedDeviceIndex_Hmd, &head); for (auto &[i, s] : g_screens) { - bool visible = s.shown && (shared || s.drag != Drag::None) && !s.alone; + bool visible = !g_paused && s.shown && (shared || s.drag != Drag::None) && !s.alone; // A floating window's panel: while a window floats on it, its output is on, and the // window isn't minimized (and once it has a crop). if (s.floating) visible = visible && s.floatOn && s.outputOn && !s.minimized && s.cropW > 0; @@ -1569,6 +1575,7 @@ int ft_vr_modifiers(uint32_t format, uint64_t *out, int max) { } bool ft_vr_screens_shown(void) { return g_vr && ModeVisible(); } +bool ft_vr_paused(void) { return g_paused; } } // extern "C" @@ -2104,6 +2111,17 @@ void ft_vr_command(const char *cmd, char *reply, int size) { g_manual = always ? !want : want; UpdateVisibility(); std::snprintf(reply, size, "ok %s", want ? "shown" : "hidden"); + } else if (std::sscanf(cmd, "pause %15s", word) == 1) { + if (!std::strcmp(word, "on") || !std::strcmp(word, "off")) { + const bool on = !std::strcmp(word, "on"); + if (on != g_paused) + std::printf("%s\n", on ? "paused for a VR game: everything hidden, KWin slowed down" : "resumed"); + g_paused = on; + UpdateVisibility(); + } else if (std::strcmp(word, "state") != 0) { + return (void)std::snprintf(reply, size, "error pause on|off|state"); + } + std::snprintf(reply, size, "ok %s", g_paused ? "paused" : "running"); } else if (std::sscanf(cmd, "ingames %15s", word) == 1) { if (!std::strcmp(word, "hide")) g_inGames = InGames::Hide; else if (!std::strcmp(word, "visible")) g_inGames = InGames::Visible; diff --git a/screens/vr.h b/screens/vr.h index 6189327..4ebd99a 100644 --- a/screens/vr.h +++ b/screens/vr.h @@ -36,6 +36,8 @@ void ft_vr_shutdown(void); int ft_vr_modifiers(uint32_t format, uint64_t *out, int max); // The screens are showing (by the visibility mode; not counting a wrist-pinned screen). bool ft_vr_screens_shown(void); +// Frametop is paused for a VR game ("pause on"): everything is hidden, and KWin slows down. +bool ft_vr_paused(void); // A panel for screen `index`, width in metres, placed in a row in front of the head. void ft_vr_screen_create(int index, double metres, int count); void ft_vr_screen_destroy(int index); diff --git a/scripts/report.sh b/scripts/report.sh index 3e3c027..8164eb0 100755 --- a/scripts/report.sh +++ b/scripts/report.sh @@ -25,6 +25,7 @@ for u in frametop-input-relay frametop-pointer frametop-power; do echo "$u: $(systemctl --user is-enabled $u 2>/dev/null) / $(systemctl --user is-active $u 2>/dev/null)" done echo "desktop: $(pgrep -x ft-screens >/dev/null && echo running || echo 'not running'), plasmashell: $(pgrep -c plasmashell || true)" +echo "paused for VR games: $(cat /run/user/$(id -u)/frametop-pause.json 2>/dev/null || echo 'no (never paused since boot)')" LD_LIBRARY_PATH=/opt/steamvr/bin/linuxarm64 /opt/steamvr/bin/linuxarm64/vrpathreg show 2>/dev/null | sed -n '/xternal/,$p' section "What Frametop needs from SteamOS (scripts/update-check.py)" diff --git a/scripts/update-check.py b/scripts/update-check.py index e16a32b..c90fe71 100755 --- a/scripts/update-check.py +++ b/scripts/update-check.py @@ -12,15 +12,18 @@ This checks each one, and compares the versions with the last ones recorded as w From a PC: ssh frame python3 - [--mark-good] < scripts/update-check.py It only reads. It connects to SteamVR as a background app (which never starts SteamVR), -runs `vrcmd --overlays` as the pointer helper does, and maps the eye tracker's shared -memory read-only. It never starts, stops, or restarts anything. +runs `vrcmd --overlays` as the pointer helper does, maps the eye tracker's shared memory +read-only, and follows a controller on vrserver's web socket for a moment. It never starts, +stops, or restarts anything. """ +import base64 import json import math import mmap import os import platform import re +import socket import struct import subprocess import sys @@ -34,6 +37,8 @@ LAUNCHER = os.path.join(HOME, ".local/share/applications/deckard-nested-desktop. VRPATHS = os.path.join(HOME, ".config/openvr/openvrpaths.vrpath") BACKLIGHT = "/sys/class/backlight/ae94000.dsi.0/brightness" # as in power/ft-powerd.cpp EYE_MMAP = "/dev/shm/eye-server.mmap" +VRSERVER = "http://127.0.0.1:27062" # its web socket: input/vrws.py +PAUSE_STATE = f"/run/user/{os.getuid()}/frametop-pause.json" # input/game_pause.py HOST_GLIBC = (2, 39) # the newest the pointer driver may need (pointer/driver/build.sh) # Packages in the OS image that Frametop depends on, and what to try by hand when one changes. @@ -297,11 +302,19 @@ def check_overlays(desktop_up): def check_services(sockets, desktop_up): + try: + with open(PAUSE_STATE) as f: + pause = json.load(f) + paused = pause["stopped"]["units"] if pause.get("paused") else [] + except (OSError, ValueError, KeyError, TypeError): + paused = [] for unit, socket in UNITS.items(): if systemctl("is-enabled", unit) != "enabled": continue state = systemctl("is-active", unit) - if state != "active": + if state != "active" and unit + ".service" in paused: + report("skip", unit, "stopped while Frametop is paused for a VR game (input/ft-pause off resumes it)") + elif state != "active": report("FAIL", unit, f"{state}; see journalctl --user -u {unit}") elif socket and socket not in sockets: report("FAIL", unit, f"runs, but hasn't opened {socket}") @@ -345,6 +358,63 @@ def check_steam_ui(): report("warn", "Steam menu shortcut", f"{detail}; a Meta tap won't open the Steam menu (steam/ft-steam)") +def check_vr_socket(): + """vrserver's web socket, which the pause gesture reads the controllers from (input/vrws.py, + input/game_pause.py). It's undocumented: SteamVR's controller binding page uses it.""" + label, headers = "vrserver web socket", {"Referer": f"{VRSERVER}/dashboard/controllerbinding.html"} + try: + import urllib.request + with urllib.request.urlopen(urllib.request.Request(f"{VRSERVER}/input/getstate.json", headers=headers), + timeout=3) as r: + devices = json.load(r).get("devices", []) + except (OSError, ValueError) as e: + report("FAIL", label, f"/input/getstate.json didn't answer ({e}); the pause gesture can't read the controllers") + return + found = [d for d in devices if d.get("controller_type") == "frame_controller" and d.get("root_path") + and d.get("side") in ("left", "right")] + if not found: + report("skip", label, "no Frame controller is on, so it wasn't checked") + return + if not all(any(c.get("path") == "/input/thumbstick/click" for c in d.get("components", [])) for d in found): + report("warn", label, "a controller lists no /input/thumbstick/click; the default pause gesture needs it") + data = b"" + try: + with socket.create_connection(("127.0.0.1", 27062), timeout=3) as s: + key = base64.b64encode(os.urandom(16)).decode() + s.sendall((f"GET / HTTP/1.1\r\nHost: 127.0.0.1:27062\r\nUpgrade: websocket\r\nConnection: Upgrade\r\n" + f"Sec-WebSocket-Key: {key}\r\nSec-WebSocket-Version: 13\r\nOrigin: {VRSERVER}\r\n" + f"Referer: {headers['Referer']}\r\n\r\n").encode()) + while b"\r\n\r\n" not in data: + chunk = s.recv(4096) + if not chunk: + break + data += chunk + if b" 101 " not in data.split(b"\r\n", 1)[0] + b" ": + report("FAIL", label, "vrserver refused the web socket; the pause gesture can't read the controllers") + return + mailbox = f"frametop_check_{os.getpid()}" + for text in (f"mailbox_open {mailbox}", "mailbox_send input_server " + json.dumps( + {"type": "request_input_state_updates", "device_path": found[0]["root_path"], + "returnAddress": mailbox})): + payload, mask = text.encode(), os.urandom(4) + size = bytes([0x80 | len(payload)]) if len(payload) < 126 else bytes([0x80 | 126]) + struct.pack(">H", len(payload)) + s.sendall(b"\x81" + size + mask + bytes(b ^ mask[i % 4] for i, b in enumerate(payload))) + deadline = time.monotonic() + 3 + while b"update_component_states" not in data and time.monotonic() < deadline: + chunk = s.recv(65536) + if not chunk: + break + data += chunk + except OSError as e: + report("FAIL", label, f"{e}; the pause gesture can't read the controllers") + return + if b"update_component_states" in data and b"/click" in data: + report("ok", label, "it still streams the controllers' buttons, which the pause gesture reads") + else: + report("FAIL", label, "no button states came, so the pause gesture can't read the controllers " + "(input/vrws.py: the message format changed?)") + + def check_eye_tracker(): gaze = systemctl("is-enabled", "frametop-gaze") == "enabled" try: @@ -394,6 +464,7 @@ def main(): check_overlays(desktop_up) check_services(sockets, desktop_up) check_steam_ui() + check_vr_socket() check_eye_tracker() if args == ["--mark-good"]: From 5b08e43a5d56236879dabaa6341f319bc3296a22 Mon Sep 17 00:00:00 2001 From: DeeJanuz <45082401+DeeJanuz@users.noreply.github.com> Date: Sat, 3 Oct 2026 08:32:49 -0600 Subject: [PATCH 3/3] Gaze: idle while the gaze isn't used The gaze service ran ft-gaze and our own eye tracker all the time: with gaze mode off, ft-eyes still took about 60% of a core, and ft-eyegrab, ft-gaze and ft-gazed 3 to 4% each. Now ft-gaze and our tracker run only while gaze mode is on and someone wears the headset, while a check or the calibration is open or asked for, or under a "wake" lease, which the Gaze page of Frametop Input Settings renews while it's open. 30 s after the last use they stop, and the frame grabber idles with our tracker. - The pointer helper answers "gaze ? headset" with worn|away (SteamVR's activity level for the headset); an older helper answers it as before, and the service then goes by gaze mode alone. - A quick check, calibration, or fit check asked for while idle wakes the tracker and opens once it sends; the automatic calibration waits quietly while it starts. - Status has "awake" and "idle" (why), and the Gaze page shows it. - gaze/test/idle-test.py runs the service with a fake helper and ft-gaze, offline. Co-Authored-By: Claude Opus 5.5 --- docs/design.md | 2 +- docs/reference.md | 2 +- gaze/README.md | 1 + gaze/ft-gazed | 78 +++++++++++++- gaze/gazecheck.py | 60 +++++++++-- gaze/test/idle-test.py | 158 ++++++++++++++++++++++++++++ input-settings/ft_input_settings.py | 6 ++ input-settings/main.qml | 11 +- pointer/helper/ft-pointer.cpp | 10 +- 9 files changed, 307 insertions(+), 21 deletions(-) create mode 100755 gaze/test/idle-test.py diff --git a/docs/design.md b/docs/design.md index 9937620..4725296 100644 --- a/docs/design.md +++ b/docs/design.md @@ -191,7 +191,7 @@ Staying awake while charging uses Steam's own setting rather than a logind sleep ## Pausing for VR games -Hiding the screens during a game kept them out of view, but Frametop kept using the headset. Measured on 2026-10-02 with gaze mode off and no game running, in shares of one core: our eye tracker (ft-eyes) about 60%, ft-eyegrab, ft-gaze and ft-gazed about 3 to 4% each; remote desktop (krdpserver, FreeRDP, Xvnc) about 2 cores while it ran; KWin about 13%, ft-screens about 4%. The gaze service runs at full rate whether gaze mode is on or not. Reading SteamVR's eye tracking 90 times a second also made it restart every 10 to 13 seconds during Beat Saber, and each restart took input focus from the game, which paused it (PR #13; since then ft-gaze skips SteamVR's gaze action during games, but our own tracker kept running). So pausing stops what costs the most and leaves windows where they are. +Hiding the screens during a game kept them out of view, but Frametop kept using the headset. Measured on 2026-10-02 with gaze mode off and no game running, in shares of one core: our eye tracker (ft-eyes) about 60%, ft-eyegrab, ft-gaze and ft-gazed about 3 to 4% each; remote desktop (krdpserver, FreeRDP, Xvnc) about 2 cores while it ran; KWin about 13%, ft-screens about 4%. The gaze service ran at full rate whether gaze mode was on or not; now it idles while the gaze isn't used (gaze/README.md), and pausing stops it outright. Reading SteamVR's eye tracking 90 times a second also made it restart every 10 to 13 seconds during Beat Saber, and each restart took input focus from the game, which paused it (PR #13; since then ft-gaze skips SteamVR's gaze action during games, but our own tracker kept running). So pausing stops what costs the most and leaves windows where they are. - A hidden screen still cost as much as a visible one. ft-screens sent every committed screen its frame callback at 90 Hz whether its overlay showed or not, so KWin kept drawing, and its apps with it. Paused, ft-screens sends the callbacks once a second. A Wayland client draws again only after its last frame's callback, so KWin's output stalls, KWin's own clients stop getting theirs, and the whole desktop idles, without anything losing its connection. A second's pace, rather than none, keeps any client that waits on a callback from waiting forever. Stopping KWin or the apps with SIGSTOP would free the same, but a Wayland peer that stops reading overflows the other side's 4 KB socket buffer, which ends the connection: that's how the live desktop died once when its KWin stalled (`Data too big for buffer`). They also sit in different cgroups (KWin under steam.service when the VR launcher starts it, ft-screens in the dev container's), so no single freeze stops them together. - The relay does the pausing because it's the one part that always runs, and the pointer helper keeps running because stopping it leaves its virtual controller connected with its last pose (the driver has no staleness timeout), maybe holding a hand role, with the 3D mouse dead. Releasing it does the job. The helper already checks for a scene app twice a second, so it's what tells the relay a game started. diff --git a/docs/reference.md b/docs/reference.md index ecc9f06..84b00d2 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -229,7 +229,7 @@ The state outlives a relay restart, in `/run/user/UID/frametop-pause.json`. A St ## Gaze pointer (experimental) -In gaze mode the 3D mouse's pointer goes where you look, and the mouse or the keyboard does the last bit. It needs the gaze service, which `install.sh` offers (yes by default) and `gaze/run.sh install` installs on its own: it builds it and runs `gaze/ft-gazed` as `frametop-gaze.service`, which starts with SteamVR. Turn gaze mode on with the Gaze page of Frametop Input Settings, `gaze/ft-gazectl on`, `POINTER_GAZE=1`, or a button or key combination mapped to Gaze pointer on/off. +In gaze mode the 3D mouse's pointer goes where you look, and the mouse or the keyboard does the last bit. It needs the gaze service, which `install.sh` offers (yes by default) and `gaze/run.sh install` installs on its own: it builds it and runs `gaze/ft-gazed` as `frametop-gaze.service`, which starts with SteamVR. The service idles while the gaze isn't used: its eye tracker reader and our own eye tracker run only while gaze mode is on and someone wears the headset, while a check or the calibration runs, or while the Gaze page of Frametop Input Settings is open, and stop 30 seconds after ([gaze/README.md](../gaze/README.md)). Turn gaze mode on with the Gaze page of Frametop Input Settings, `gaze/ft-gazectl on`, `POINTER_GAZE=1`, or a button or key combination mapped to Gaze pointer on/off. - Meta+J left-clicks and Meta+K right-clicks where you look. A quick tap clicks where the dot was at the press. Hold instead, and the dot stays put in your view: turn your head until it's on what you meant, and let go to click there. Held still for `POINTER_GAZE_HOLD` (0.5 s), the press becomes a real one, and your head drags. Meta+K with Meta+J held presses where the dot is now, to drag from there, and a second Meta+K during that drag (a double Meta+K) pans and tilts what you're dragging while it's held. - The mouse's buttons work the same way, with the mouse steering instead of your head (`POINTER_GAZE_MOUSE=precision`, the default): the right button with the left held starts a drag, and a double right click pans and tilts what you're dragging. With `POINTER_GAZE_MOUSE_MOVE=held`, the default, the mouse only corrects: while the gaze has the pointer, moving it does nothing unless a button is held. `free` lets the mouse take the pointer any time. With the gaze stale for a second, in a game, or with the headset off, the mouse works as usual. diff --git a/gaze/README.md b/gaze/README.md index b981870..8143f51 100644 --- a/gaze/README.md +++ b/gaze/README.md @@ -35,6 +35,7 @@ Gaze as an input method for the whole desktop, without replacing anything of Ste - **Learning from nudges:** if the mouse took the pointer from the gaze and moved it (0.2 degrees or more, and the correction within `POINTER_GAZE_NUDGE_MAX`: 55 degrees by default, half of the 109 the headset shows across, and 1 to 110; the same limit for mouse, keyboard, and pinch clicks) before you clicked, or you dragged a held press that far, you were nudging it onto what you looked at. The helper sends that as a lesson, from the raw gaze when the mouse took over to where you clicked, and ft-gazed learns it. So using it is what calibrates it. The raw gaze is one ft-gazed sent, so it also finds when that look was, and what each eye read then. With SteamVR, each eye learns its own error. With our tracker, the look goes to it as a click, like the probe's, and it relearns how the headset sits on your face. After the headset was off, your first nudge and click there resets that (the quick check's dot does the same). A correction past `POINTER_GAZE_NUDGE_MAX` isn't learned: the helper asks ft-gazed for the quick check instead ("recheck", after its 2-minute cooldown). Tested on our tracker's 409 clicks since its Sep 29 calibration: a one-dot check set from any one of them put the next 2 minutes' clicks within 15 degrees (99% within 4.2) and the next 10 minutes' within 25 (the far ones after the headset moved), so the check gets back well under it. The limit used to be 8 degrees, and live on 2026-10-01 our tracker was 12 off after the headset went on, so every correction was dropped. One lesson moves the whole correction by only a third of what it measured (more near where it was taken), since in the first live test one 6 degree lesson moved everything and put the next target 7 degrees off. `ft-gazectl status` shows the lessons, and `ft-gazectl forget` drops them. - **Checks and calibration in the headset** (`gaze/gazecheck.py`, shown by `gaze/panel/ft-gazepanel`, a panel fixed to the headset that ft-gazed runs): a one-dot quick check opens when you put the headset on (SteamVR's tracker sees your eyes for 3 s after none for 3 s; its "HMD on" log line can't say, since it repeats every minute or so and can stay on for hours with nobody in the headset), when our tracker asks for a click (its "reseat", when the headset may sit differently), at most once every 2 minutes, and from Quick check on the Gaze page. Look at the dot: it takes your gaze once it has held still for 0.6 s (the steadiness counts, not where the tracker puts it, so it works however far off it is), or at once with a left click or Meta+J; a right click or Meta+K closes it, and ignoring it changes nothing. It also runs when a click's correction was past `POINTER_GAZE_NUDGE_MAX`. The dot is still and the ring fills in quarters, so the panel is drawn again only a few times per dot. If the first 3 lessons after it are still over 2 degrees off, five dots follow. The full calibration (Calibrate on the Gaze page, or by itself whenever gaze mode is on without one and your eyes are seen) is the probe's: three rounds, dark, medium and bright, of the middle and a ring around it, in a panel 64 degrees wide, with Frametop's screens hidden. Its dots (and the five-dot check's) wait for a click: look at the dot and left click or press Meta+J, and the gaze held still up to then is taken. A dot that isn't taken says why, on an orange line over the instructions: with SteamVR's tracker, what dropped most of that look's samples (an eye lost, a blink, the two eyes disagreeing); with ours, its reply (an eye seen in too few frames, or moving). A dot gets two tries, then it's skipped. A calibration left with under two thirds of its dots fails and names the most common reason, as the Gaze page does after it. A click that has taken nothing after 1.5 s says what it waits for: the gaze to hold still, or an eye tracker that isn't sending. Capturing whenever the gaze held still sometimes took a look that wasn't on the dot. The panel draws into three shared buffers SteamVR imported once, as Frametop's keyboard does: uploading each picture anew (SetOverlayRaw) flickered, and in one live test left the headset showing an old picture. Quitting it while there's still no calibration turns gaze mode off; turning it on again reopens it. One that closes otherwise unfinished (ignored for 2 minutes, too few dots) opens again after the headset comes off and on. Why gaze mode, on, can't work yet goes in the service's status as `checks.problem`, which the Gaze page shows under the Gaze pointer switch. For our tracker a check is a click and the calibration is its own (calib-point per dot); for SteamVR's, a check is a lesson for each eye and the calibration replaces calibration.json, and the lessons start over. Checks go to `checks.jsonl`. - Nothing writes to SteamVR, its eye tracker, or its files: ft-gaze maps the eye tracker's shared memory read-only. With no fresh gaze (a blink, the service stopped, the headset off), the pointer stays where it is, and the mouse works as always. +- **Idle while the gaze isn't used:** ft-gaze and our own tracker run only while gaze mode is on and someone wears the headset (the pointer helper says both: SteamVR drops the headset's activity level as soon as it comes off), while a check or the calibration is open or asked for, or while the Gaze page of Frametop Input Settings is open (it renews a `wake` lease). 30 seconds after the last use they stop, and our frame grabber goes idle with our tracker. With our tracker, that saves over half a core: on 2026-10-02, with gaze mode off, ft-eyes took about 60% of a core, and ft-eyegrab, ft-gaze and ft-gazed 3 to 4% each. A check asked for while it idles starts the tracker and opens once it sends. When the gaze is used again, it takes a few seconds to come back, and our tracker's first click re-seats it, as after the headset was off: so the quick check opens when gaze mode comes on after the service idled, as it does when you put the headset on. `ft-gazectl status` says `"awake"`, and `"idle"` says why it isn't. A stand that covers the proximity sensor makes the headset seem worn, so with gaze mode on it doesn't idle there. Lessons are logged to `pointer-lessons.jsonl`: the raw gaze, the true direction, the correction at the time, and how far off it was. diff --git a/gaze/ft-gazed b/gaze/ft-gazed index 8b55175..f751ba6 100755 --- a/gaze/ft-gazed +++ b/gaze/ft-gazed @@ -63,9 +63,18 @@ and the error moves): lessons from before count less then, so the first few afte relearn the offset. Our own tracker (gaze/tracker/ft-eyes) runs here too, in the dev container, while -GAZE_TRACKER=own or the gaze probe asks for it ("eyes SECONDS", a lease the probe renews). It -reads the eye-camera frames the root service frametop-eyegrab copies (gaze/tracker/install.sh), -which copies them only while ft-eyes runs. +GAZE_TRACKER=own and the gaze is in use (below), or the gaze probe asks for it ("eyes SECONDS", +a lease the probe renews). It reads the eye-camera frames the root service frametop-eyegrab +copies (gaze/tracker/install.sh), which copies them only while ft-eyes runs. + +Idle: ft-gaze and our own tracker run only while the gaze is in use: gaze mode on with someone +wearing the headset (the pointer helper says both: "gaze ? headset" -> "ok on|off worn|away"), a +check or calibration open or asked for, or a "wake" lease (the Gaze page of Frametop Input +Settings renews one while it's open). IDLE_AFTER after the last use they stop, and the frame +grabber goes idle with our tracker; with our tracker, that was over half a core with gaze mode +off. A check asked for while idle waits for the tracker to start (at most WAKE_SETTLE). Our +tracker's next click after it starts again re-seats it, as after the headset was off, so turning +gaze mode on after a while opens the quick check. Checks and calibration (gaze/gazecheck.py): a one-dot quick check when the headset goes on or our tracker asks for a click, and the full calibration when gaze mode comes on without one, @@ -83,6 +92,8 @@ Control socket: abstract unix datagram "@ft_gazed": reload read calibration.json and the settings again eyes keep our own tracker running that much longer (at most 120), whatever GAZE_TRACKER says: the probe's lease. Reply: "ok" + wake keep the gaze running (not idle) that much longer (at most 120), + whatever gaze mode says: Input Settings' Gaze page. Reply: "ok" quickcal the one-dot check now (the calibration if there's none) calibrate the full calibration in the panel fitcheck the headset fit check in the panel @@ -140,6 +151,9 @@ RETRY = 3.0 # seconds before starting ft-gaze again EYES_RETRY = 10.0 # seconds before starting ft-eyes again after it stopped on its own EYES_LEASE_MAX = 120.0 SETTLE = 0.3 # seconds after an eye is found again before the fallback learns from it +IDLE_AFTER = 30.0 # seconds after the gaze was last in use before ft-gaze and our tracker stop +WAKE_SETTLE = 15.0 # seconds after waking that the tracker may take to start sending +WAKE_MAX = 120.0 KEYBOARD_PITCH = -20.0 # degrees: gaze under this, on no screen, is a look at the keyboard @@ -230,6 +244,10 @@ class Service: self.eyes_proc = None # ft-eyes, while it runs self.eyes_until = 0.0 # the probe's lease (monotonic time) self.eyes_restart_at = 0.0 + self.awake = False # ft-gaze (and our tracker) run: the gaze is in use (see the top) + self.idle_at = 0.0 # idle from then, unless it's in use again first + self.woke_at = 0.0 + self.wake_until = 0.0 # a "wake" lease self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM | socket.SOCK_CLOEXEC | socket.SOCK_NONBLOCK) self.sock.bind(ME) @@ -399,6 +417,46 @@ class Service: log(f"lesson log: {e}") return rec + # --- Idle (see the top) --- + + def use_reason(self): + """Why the gaze is in use now, or None.""" + c = self.checks + if c.gaze_on and c.headset is not False: + return "gaze mode is on" + if c.active or c.pending: + return "a check" + if time.monotonic() < self.wake_until: + return "asked to stay awake" + return None + + def idle_reason(self): + c = self.checks + if c.gaze_on is None: + return "the pointer helper isn't answering (SteamVR not running?)" + if c.gaze_on and c.headset is False: + return "nobody is wearing the headset" + return "gaze mode is off" + + def waking(self): + """Idle, or awake too briefly for the tracker to be sending yet.""" + return not self.awake or time.monotonic() - self.woke_at < WAKE_SETTLE + + def update_awake(self): + now = time.monotonic() + why = self.use_reason() + if why: + self.idle_at = now + IDLE_AFTER + if why and not self.awake: + self.awake, self.woke_at, self.restart_at = True, now, 0.0 + log(f"awake: {why}") + elif not why and self.awake and now >= self.idle_at: + self.awake = False + log(f"idle: {self.idle_reason()}; ft-gaze and our tracker stop until the gaze is used again") + self.stop_helper() + if self.eyes_proc and not self.eyes_wanted(): + self.stop_eyes() + # --- ft-gaze --- def start_helper(self): @@ -441,7 +499,7 @@ class Service: self.proc = None def eyes_wanted(self): - return (self.tracker == "own" and not self.override) or time.monotonic() < self.eyes_until + return (self.tracker == "own" and not self.override and self.awake) or time.monotonic() < self.eyes_until def start_eyes(self): """Our own tracker, in the dev container, with build/venv's numpy and OpenCV. Like @@ -700,6 +758,14 @@ class Service: reply = "ok" except ValueError: reply = "error bad seconds" + elif words[:1] == ["wake"] and len(words) == 2: + try: + secs = min(max(float(words[1]), 0.0), WAKE_MAX) + self.wake_until = max(self.wake_until, time.monotonic() + secs) + self.update_awake() + reply = "ok" + except ValueError: + reply = "error bad seconds" elif words[:1] == ["reload"]: self.load_settings() self.load_calibration() @@ -750,6 +816,7 @@ class Service: st.update(calibration_samples=cal.get("dots", 0), calibration_made=cal.get("made"), lessons=max(len(m) for m in w.misses), own_running=bool(own), own_reseat=any(e.get("reseat") for e in own.get("eyes", {}).values())) + st.update(awake=self.awake, idle=None if self.awake else self.idle_reason()) st.update(eyes_process=self.eyes_proc is not None, eyegrab=EYES_CAMS.exists(), tracker_setting=self.tracker_setting, own_installed=own_installed()) st.update({"ft_gaze": self.proc is not None, "sample_age_s": round(time.monotonic() - self.last_sample, 2) @@ -770,6 +837,7 @@ class Service: if mtime(CONF) != self.conf_mtime or (self.tracker_setting == "auto" and own_installed() != (self.tracker == "own")): self.load_settings() + self.update_awake() want = self.eyes_wanted() if want and not self.eyes_proc and time.monotonic() >= self.eyes_restart_at: self.start_eyes() @@ -790,7 +858,7 @@ class Service: next_verbose = time.monotonic() + 5 while self.running: now = time.monotonic() - if not self.proc and now >= self.restart_at: + if self.awake and not self.proc and now >= self.restart_at: self.start_helper() for key, _ in self.sel.select(timeout=0.05 if self.checks.active else 0.5): if key.data == "control": diff --git a/gaze/gazecheck.py b/gaze/gazecheck.py index ee4c07a..a912f2e 100644 --- a/gaze/gazecheck.py +++ b/gaze/gazecheck.py @@ -10,7 +10,9 @@ directions: look at each one. once every QUICK_COOLDOWN, and on "quickcal" (Frametop Input Settings, or a mouse button or key combination mapped to Gaze quick check), and when a click's correction was past POINTER_GAZE_NUDGE_MAX (55 degrees; the helper's "recheck"): the tracker is - far off. Ignored, it closes after QUICK_TIMEOUT and changes nothing. + far off. Ignored, it closes after QUICK_TIMEOUT and changes nothing. The headset going + on is seen only while the gaze service is awake (gaze mode on, someone wearing it: see + ft-gazed), so it also opens when gaze mode comes on after the service idled. five the middle and four around it, when the first FIVE_COUNT lessons after a quick check were all over FIVE_LIMIT degrees off: the quick check didn't fix it. full the calibration, as the gaze probe's: three rounds, dark, medium and bright (pupil @@ -30,6 +32,9 @@ directions: look at each one. adjust the headset. A left click or Meta+J runs its guided check (dots, then looks down, up, left and right); a right click or Meta+K closes it, as does FIT_TIMEOUT. +A check asked for while the gaze service idles (quickcal, calibrate, fitcheck) wakes it and +waits until the tracker sends, at most ft-gazed's WAKE_SETTLE; then it opens, or logs why not. + The quick check's dot captures itself: from CHECK_SETTLE after it shows (the eyes getting there), once the gaze has held within CHECK_SPREAD for CHECK_WINDOW (the probe's max spread and capture time). It's the gaze holding still that counts, not where the tracker puts it, so it @@ -209,7 +214,9 @@ class Checks: sel.register(self.out, selectors.EVENT_READ, "checks") self.check = None self.gaze_on = None + self.headset = None # someone wears it (the helper's "worn"), False "away", None unknown self.gaze_heard = 0.0 + self.pending = None # (command words, when): asked for while the gaze service idled self.full_armed = True # gaze mode on without a calibration opens the full one (need_full) self.full_blocked = None # why it can't open now self.full_retry_at = 0.0 @@ -294,17 +301,22 @@ class Checks: pass def on_readable(self): - """Replies on our socket: the helper's "ok on|off" to "gaze ?"; the panel's are dropped.""" + """Replies on our socket: the helper's "ok on|off [worn|away]" to "gaze ? headset" (an + older helper leaves the headset out); the panel's are dropped.""" while True: try: - data = self.out.recv(4096).decode("utf-8", "replace") + words = self.out.recv(4096).decode("utf-8", "replace").split() except (BlockingIOError, OSError): return - if data in ("ok on", "ok off"): - on = data == "ok on" - was, self.gaze_on, self.gaze_heard = self.gaze_on, on, time.monotonic() - if on and was is False: + if words[:1] == ["ok"] and len(words) in (2, 3) and words[1] in ("on", "off"): + on = words[1] == "on" + headset = None if len(words) == 2 else words[2] == "worn" + was = (self.gaze_on, self.headset) + self.gaze_on, self.headset, self.gaze_heard = on, headset, time.monotonic() + if on and was[0] is False: self.on_gaze_on() + if was != (on, headset): + self.svc.update_awake() # --- State --- @@ -341,6 +353,8 @@ class Checks: self.full_blocked = None return now = time.monotonic() + if not self.can_run() and self.svc.waking(): + return # the tracker is still starting (the service idled) if not self.can_run(): why = ("the eye tracker isn't sending" if now - self.svc.last_sample >= 2 else "no eyes seen (is the headset on?)") @@ -490,6 +504,8 @@ class Checks: def on_sample(self, s): self.sample_at = time.monotonic() + if self.pending: + self.run_pending() unc = (s["src"].get("mmap1") or {}).get("unc") if unc and min(unc) <= EYE_LOST: self.seen_at = time.monotonic() @@ -742,9 +758,28 @@ class Checks: # --- From the service --- - def command(self, words): + def run_pending(self): + """A check asked for while the service idled: once the tracker sends (and our tracker has + said whether it's calibrated), or WAKE_SETTLE after waking, when its error is the real one.""" + svc = self.svc + words, _ = self.pending + ready = (time.monotonic() - svc.last_sample < 2 if words[0] == "fitcheck" else self.can_run()) \ + and (svc.kind != "own" or self.calibrated() is not None) + if not ready and svc.waking(): + return + self.pending = None + reply = self.command(words, queue=False) + if reply != "ok": + log(f"{words[0]}, asked for while idle: {reply.removeprefix('error ')}") + + def command(self, words, queue=True): """quickcal, calibrate, calaccept, calquit -> a reply.""" cmd = words[0] + if cmd in ("quickcal", "calibrate", "fitcheck") and queue and self.svc.waking() and not self.check: + # The tracker isn't running (or only just started): wake it, and do this once it sends. + self.pending = (words, time.monotonic()) + self.svc.update_awake() + return "ok waking the eye tracker first" if cmd == "quickcal": return self.start("full" if self.calibrated() is False else "quick", "asked for") if cmd == "calibrate": @@ -820,9 +855,11 @@ class Checks: now = time.monotonic() if not self.panel_proc and now >= self.panel_restart_at: self.start_panel() - self.to_helper("gaze ?") + self.to_helper("gaze ? headset") if now - self.gaze_heard > 5: - self.gaze_on = None # the helper isn't answering + self.gaze_on = self.headset = None # the helper isn't answering + if self.pending: + self.run_pending() if self.check: self.to_helper("calpanel 1") if not self.eyes_seen(AWAY_MIN): @@ -844,7 +881,8 @@ class Checks: def status(self): c = self.check - st = {"check": None, "gaze_mode": self.gaze_on, "calibrated": self.calibrated(), "eyes": self.eyes_seen(), + st = {"check": None, "gaze_mode": self.gaze_on, "headset_worn": self.headset, "pending": self.pending[0][0] + if self.pending else None, "calibrated": self.calibrated(), "eyes": self.eyes_seen(), "problem": self.problem(), "panel": self.panel_proc is not None, "last_quick_s": round(time.monotonic() - self.last_quick) if self.last_quick else None} diff --git a/gaze/test/idle-test.py b/gaze/test/idle-test.py new file mode 100755 index 0000000..a0cd240 --- /dev/null +++ b/gaze/test/idle-test.py @@ -0,0 +1,158 @@ +#!/usr/bin/env python3 +"""Offline test of the gaze service idling (gaze/ft-gazed, gaze/gazecheck.py): ft-gaze runs only +while the gaze is in use, and a check asked for while it idles waits for the tracker. + +Runs ft-gazed's Service with its sockets renamed, a fake pointer helper (answers "gaze ? +headset" as the test says), and a fake ft-gaze (prints samples, quits when its stdin closes). +SteamVR's tracker is the one in use, so our own isn't started; the panel isn't either. Nothing +reaches the live gaze service, the pointer helper, or SteamVR, so it's safe next to them. + + gaze/test/idle-test.py +""" +import importlib.machinery +import importlib.util +import os +import socket +import subprocess +import sys +import tempfile +import threading +import time + +HERE = os.path.dirname(os.path.abspath(__file__)) +GAZE = os.path.join(HERE, "..") +sys.path.insert(0, GAZE) +loader = importlib.machinery.SourceFileLoader("ftgazed", os.path.join(GAZE, "ft-gazed")) +gazed = importlib.util.module_from_spec(importlib.util.spec_from_loader("ftgazed", loader)) +loader.exec_module(gazed) +import gazecheck # noqa: E402 (the module ft-gazed imported) + +tag = f"ft_gaze_idle_test_{os.getpid()}" +gazed.ME = f"\0{tag}_gazed" +gazed.POINTER = gazecheck.POINTER = f"\0{tag}_helper" +gazecheck.SCREENS = f"\0{tag}_screens" +gazecheck.PANEL_PROG = gazed.REPO / "nonexistent-panel" # "isn't built": no panel +gazed.IDLE_AFTER, gazed.WAKE_SETTLE = 1.0, 3.0 +gazed.read_settings = lambda: ("steam", "steam", "auto", 55.0) +logs = [] +gazed.log = gazecheck.log = lambda msg: logs.append(msg) + +# The fake ft-gaze: 90 samples a second, both eyes seen, until its stdin closes. +tmp = tempfile.mkdtemp(prefix="ft-gaze-idle-test-") +FAKE = os.path.join(tmp, "ft-gaze") +with open(FAKE, "w") as f: + f.write('''import json, select, sys, time +while True: + if select.select([sys.stdin], [], [], 1 / 90)[0] and not sys.stdin.read(1): + break + eye = {"hy": 1.0, "hp": 2.0} + print(json.dumps({"t": time.monotonic(), "src": {"mmap1": {"hy": 1.0, "hp": 2.0, "unc": [0.001, 0.001], + "open": [0.8, 0.8]}, "left": eye, "right": eye}}), flush=True) +''') +started = [] + + +def start_helper(self): + """ft-gaze, straight from here instead of the dev container.""" + import selectors + self.proc = subprocess.Popen([sys.executable, FAKE], stdin=subprocess.PIPE, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + os.set_blocking(self.proc.stdout.fileno(), False) + os.set_blocking(self.proc.stderr.fileno(), False) + self.sel.register(self.proc.stdout, selectors.EVENT_READ, "stdout") + self.sel.register(self.proc.stderr, selectors.EVENT_READ, "stderr") + self.buf = b"" + started.append(time.monotonic()) + + +gazed.Service.start_helper = start_helper + +# The fake pointer helper. +helper_state = {"reply": "ok off worn"} +helper = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) +helper.bind(gazed.POINTER) +helper.settimeout(0.2) + + +def answer(): + while True: + try: + data, addr = helper.recvfrom(512) + except socket.timeout: + continue + except OSError: + return + if data.startswith(b"gaze ?") and addr: + helper.sendto(helper_state["reply"].encode(), addr) + + +threading.Thread(target=answer, daemon=True).start() +svc = gazed.Service(None, False, gazed.POINTER) +threading.Thread(target=svc.run, daemon=True).start() +ctl = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) +ctl.bind("") +ctl.settimeout(2) + + +def ask(cmd): + ctl.sendto(cmd.encode(), gazed.ME) + return ctl.recv(4096).decode() + + +failures = [] + + +def check(label, got, want): + ok = got == want + print(("ok " if ok else "FAIL ") + label + ("" if ok else f": got {got!r}, want {want!r}"), flush=True) + if not ok: + failures.append(label) + + +def wait(cond, seconds): + end = time.monotonic() + seconds + while time.monotonic() < end: + if cond(): + return True + time.sleep(0.05) + return cond() + + +time.sleep(2.5) +check("gaze mode off: idle, no ft-gaze", (svc.awake, svc.proc is None), (False, True)) +check("status says why", ask("status").count('"idle": "gaze mode is off"'), 1) + +helper_state["reply"] = "ok on worn" +check("gaze mode on, headset worn: awake within 2 s", wait(lambda: svc.awake and svc.proc is not None, 2.5), True) +check("samples come in", wait(lambda: svc.counts["samples"] > 20, 2), True) + +helper_state["reply"] = "ok on away" +check("headset off: still awake for IDLE_AFTER", wait(lambda: not svc.awake, 0.5), False) +check("then idle, ft-gaze stopped", wait(lambda: not svc.awake and svc.proc is None, 3.5), True) +check("why: nobody wears it", ask("status").count('"idle": "nobody is wearing the headset"'), 1) + +helper_state["reply"] = "ok on" +check("an older helper (no headset word): awake with gaze mode on", wait(lambda: svc.awake, 2.5), True) +helper_state["reply"] = "ok off worn" +check("gaze mode off again: idle", wait(lambda: not svc.awake and svc.proc is None, 4.5), True) + +check("wake lease", ask("wake 2"), "ok") +check("wake: awake at once", (svc.awake, wait(lambda: svc.proc is not None, 1)), (True, True)) +check("lease over (2 s + IDLE_AFTER): idle", wait(lambda: not svc.awake, 4.5), True) + +time.sleep(0.5) +logs.clear() +count = len(started) +check("quick check while idle: queued, waking", ask("quickcal"), "ok waking the eye tracker first") +check("it woke", wait(lambda: svc.awake and len(started) > count, 1.5), True) +check("once the tracker sends, it ran (and says why it couldn't open)", + wait(lambda: any("quickcal, asked for while idle: the panel isn't running" in m for m in logs), 3), True) +check("nothing left pending", svc.checks.pending, None) +check("idle again after", wait(lambda: not svc.awake, 3), True) + +print("FAILED: " + ", ".join(failures) if failures else "all passed", flush=True) +svc.running = False +time.sleep(0.7) +os.remove(FAKE) +os.rmdir(tmp) +os._exit(1 if failures else 0) diff --git a/input-settings/ft_input_settings.py b/input-settings/ft_input_settings.py index 536b7be..7c952b4 100644 --- a/input-settings/ft_input_settings.py +++ b/input-settings/ft_input_settings.py @@ -1163,6 +1163,12 @@ class Backend(QObject): else: self.message.emit("The gaze service isn't running (frametop-gaze.service)", True) + @Slot() + def keepGazeAwake(self): + """The Gaze page is open: the gaze service doesn't idle meanwhile (its status stays live, + and a check opens without waiting for the tracker). The page renews this every 10 s.""" + self._send("wake 30", GAZED) + @Slot() def gazeQuickCheck(self): """The gaze service's one-dot check, in the panel fixed to the headset.""" diff --git a/input-settings/main.qml b/input-settings/main.qml index 3d6a83e..2f08bf9 100644 --- a/input-settings/main.qml +++ b/input-settings/main.qml @@ -969,6 +969,14 @@ Kirigami.ApplicationWindow { title: "Gaze" header: DriverWarning {} property var status: backend.gazeStatus + // The gaze service idles while the gaze isn't used; open, this page keeps it going. + Timer { + running: true + repeat: true + interval: 10000 + triggeredOnStart: true + onTriggered: backend.keepGazeAwake() + } actions: [ Kirigami.Action { text: "Quick check" @@ -1202,7 +1210,8 @@ Kirigami.ApplicationWindow { Controls.Label { Kirigami.FormData.label: "Service:" text: backend.gazeServiceRunning - ? (gpage.status.ft_gaze ? "running" : "running, eye tracker reader restarting") + ? (gpage.status.awake === false ? "idle: " + gpage.status.idle + " (it starts when the gaze is used)" + : gpage.status.ft_gaze ? "running" : "running, eye tracker reader starting") : backend.gazeServiceInstalled ? "not running (frametop-gaze.service, starts with SteamVR)" : "not installed: run gaze/run.sh install in the Frametop folder, in a terminal" color: backend.gazeServiceRunning ? Kirigami.Theme.textColor : Kirigami.Theme.negativeTextColor diff --git a/pointer/helper/ft-pointer.cpp b/pointer/helper/ft-pointer.cpp index 0b04582..a06a5fc 100644 --- a/pointer/helper/ft-pointer.cpp +++ b/pointer/helper/ft-pointer.cpp @@ -267,7 +267,8 @@ // // Commands (datagrams on @ft_pointer_helper): show, hide, recenter, move , // follow on|off|toggle (head follow, until the next restart or a change to POINTER_FOLLOW), -// gaze on|off|toggle|? (gaze mode, likewise with POINTER_GAZE; ? only asks), gz ... (the gaze, from ft-gazed), +// gaze on|off|toggle|? (gaze mode, likewise with POINTER_GAZE; ? only asks, and "? headset" adds +// worn|away to the reply), gz ... (the gaze, from ft-gazed), // reload (re-read the settings below), debug (toggle a twice-a-second state log), // overlays (replies with the overlay list as JSON, see OverlayList), // vrbind/vrglobal/vrstatus (Frame controller buttons, see vrbuttons.h), @@ -1387,7 +1388,12 @@ int main() { std::printf("gaze mode %s\n", gazeOn ? "on" : "off"); std::fflush(stdout); } - reply(gazeOn ? "ok on" : "ok off"); + // "gaze ? headset" (the gaze service, which idles while nobody wears the headset) + // also says whether someone does. + if (std::strstr(arg, "headset")) + reply(gazeOn ? (headsetOff ? "ok on away" : "ok on worn") : (headsetOff ? "ok off away" : "ok off worn")); + else + reply(gazeOn ? "ok on" : "ok off"); continue; } const bool mouseInput = std::strncmp(buf, "move", 4) == 0 || std::strncmp(buf, "btn", 3) == 0 ||