From 8a02e41b21d9b8c91339bff3ee57194982790e97 Mon Sep 17 00:00:00 2001 From: DeeJanuz <45082401+DeeJanuz@users.noreply.github.com> Date: Sat, 3 Oct 2026 09:08:16 -0600 Subject: [PATCH 01/24] Eye tracker: one thread for OpenCV and numpy Nothing called cv2.setNumThreads, so OpenCV kept a pool of one worker per core for pupil windows of 140 to 240 px. Live, its idle workers spun and yielded about 14,000 times a second each, about a quarter of a core, next to SteamVR's compositor. numpy's OpenBLAS also started 8 threads that never had work. eyes_pupil.py now sets OpenCV to one thread, and ft-eyes sets OPENBLAS_NUM_THREADS and OMP_NUM_THREADS to 1 before numpy loads (a value already in the environment wins). Replaying fit1 into a scratch share (ft-eyes-replay, 14 s measured, capped at one core with the replay): threads 13 -> 1, involuntary context switches 3,812/s -> 430/s, system time 6.9% -> 1.6% of a core. Under that cap the frames it kept up with went from 21-36 to 57-69 a second per eye. Co-Authored-By: Claude Opus 5.5 --- gaze/README.md | 2 +- gaze/tracker/eyes_pupil.py | 7 ++++++- gaze/tracker/ft-eyes | 7 ++++++- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/gaze/README.md b/gaze/README.md index 8143f51..3346530 100644 --- a/gaze/README.md +++ b/gaze/README.md @@ -63,7 +63,7 @@ Ground rules, for anyone changing it: - **Clean room.** Nothing of Valve's goes in: we don't decompile, disassemble, or patch the `eyetracking` binary or its network weights, and we don't copy their code or weights. Its public output (eye-server.mmap, read-only) is fair game as a baseline and as labels, and so are published papers and openly licensed pupil detectors (check each one's license: PuRe, PuReST, ElSe, and ExCuSe are non-commercial only). - **Root only reads.** ft-eyegrab never writes to, stops, or signals the `eyetracking` process, vrserver, or vrcompositor, never opens `/dev/adsp`, `/dev/cdsp`, or `/dev/spidev0.1`, and never writes to `/dev/shm/eye-server.mmap` (it also carries calibration clicks into SteamVR's tracker), `/opt`, or `/persist`. - **Eye images are biometric data.** Recordings live outside the repo, in `~/.local/share/frametop/eyes/captures` (0700), and `.gitignore` catches stray frame dumps. They go nowhere but the machine that runs your offline jobs. -- **Mind the headset's budget.** Finding a pupil takes about 0.4 ms a frame while ft-eyes follows it, and 1.4-2.1 ms when it searches the whole frame. Replays, scoring, and training go to a PC. +- **Mind the headset's budget.** Finding a pupil takes about 0.4 ms a frame while ft-eyes follows it, and 1.4-2.1 ms when it searches the whole frame. ft-eyes keeps OpenCV and numpy to one thread: their pools of one per core spun idle workers at about a quarter of a core, for frames this small. Replays, scoring, and training go to a PC. ## Headset fit diff --git a/gaze/tracker/eyes_pupil.py b/gaze/tracker/eyes_pupil.py index 8884363..cf3672d 100644 --- a/gaze/tracker/eyes_pupil.py +++ b/gaze/tracker/eyes_pupil.py @@ -9,7 +9,12 @@ the search runs again with a smaller closing. import cv2 import numpy as np -DARK = 30 # pupil pixels are below this (the face around it is 40-180) +# One thread: OpenCV's pool of one per core costs more than it saves on a frame this small +# (a 140-240 px window while it follows the pupil). Its idle workers spun and yielded about +# 14,000 times a second each, a quarter of a core, beside SteamVR's compositor. +cv2.setNumThreads(1) + +DARK = 30 # pupil pixels are below this (the face around it is 40-180) MIN_AREA = 150 # pupil area range in pixels MAX_AREA = 20000 MIN_FILL = 0.75 # blob area / fitted-ellipse area diff --git a/gaze/tracker/ft-eyes b/gaze/tracker/ft-eyes index 5e25799..66bbff7 100755 --- a/gaze/tracker/ft-eyes +++ b/gaze/tracker/ft-eyes @@ -58,7 +58,12 @@ import time from collections import deque from pathlib import Path -import numpy as np +# One thread for numpy's BLAS and OpenMP, set before numpy loads: it would start one per core +# (8 here) for small arrays that never need them. eyes_pupil keeps OpenCV to one as well. +for _var in ("OPENBLAS_NUM_THREADS", "OMP_NUM_THREADS"): + os.environ.setdefault(_var, "1") + +import numpy as np # noqa: E402 sys.path.insert(0, str(Path(__file__).resolve().parent)) import eyes_model # noqa: E402 From 19032f8963f3fad63167067ea5fe0a58714026c5 Mon Sep 17 00:00:00 2001 From: DeeJanuz <45082401+DeeJanuz@users.noreply.github.com> Date: Sat, 3 Oct 2026 09:10:30 -0600 Subject: [PATCH 02/24] Pointer: read the overlay list every 20 s, not every second The helper ran `vrcmd --overlays` once a second while the pointer was awake, and in gaze mode the pointer never sleeps. Each run is a shell plus vrcmd, a new SteamVR client, about 26 to 30 ms of CPU, so about 3% of a core all the time. The list is now read every 20 seconds, and at once (at most once a second) when it may have changed: the pointer waking, the dashboard opening or closing or creating an overlay, the scene app changing, an "overlays" request, and a left click that hit nothing, which may be on a panel that came up since. The thread waits on a condition variable instead of waking every 100 ms, so it sleeps while paused. The main loop looks the keys up again as soon as a new list is in, rather than at its next 1 s tick. Overlays already on the list still show and hide within 50 ms, from the IsOverlayVisible poll. Co-Authored-By: Claude Opus 5.5 --- docs/design.md | 2 +- pointer/helper/ft-pointer.cpp | 77 +++++++++++++++++++++++++++-------- 2 files changed, 62 insertions(+), 17 deletions(-) diff --git a/docs/design.md b/docs/design.md index 4725296..fb29cbf 100644 --- a/docs/design.md +++ b/docs/design.md @@ -118,7 +118,7 @@ The driver starts disconnected, because holding the right-hand role while SteamV Mouse motion turns into yaw and pitch around an anchor, the head position at the last recenter. A ray from the anchor is tested against every visible overlay with `ComputeOverlayIntersection`. On a hit, the cursor sits on that surface; otherwise it floats at `POINTER_DISTANCE`. Since the anchor isn't your current eye position, a second test runs along your line of sight to the cursor point, and anything nearer wins, so the cursor always lands on what you see under it. Overlays in `POINTER_IGNORE` are left out of both tests. A display-only panel, like a performance overlay locked to your view, has no input method, so SteamVR's laser passes through it, but `ComputeOverlayIntersection` still hits it, and the cursor stuck to it. The laser starts just before the cursor point, so an ignored panel nearer to you doesn't catch it either. -OpenVR has no call to list other programs' overlays, so the helper runs `vrcmd --overlays` in the background. It includes hidden overlays, because a floating window's controls only appear while something hovers the window, and the cursor has to find them immediately. +OpenVR has no call to list other programs' overlays, so the helper runs `vrcmd --overlays` in the background. It includes hidden overlays, because a floating window's controls only appear while something hovers the window, and the cursor has to find them immediately. Each run is a shell and a new SteamVR client, about 30 ms of CPU, and it ran every second while the pointer was awake, which in gaze mode is all the time. Now it runs every 20 seconds, and at once when the pointer wakes, when the dashboard opens or closes, when a game starts or ends, and when a left click hits nothing (a panel that came up since). An overlay already on the list showing or hiding needs no new list: the helper checks the visibility of the ones it knows every 50 ms. The laser starts partway along your line of sight to the cursor rather than at your eye. SteamVR sizes its hit dot by distance from the laser's origin, and a laser from the eye still shows a beam in each eye. Starting it close to the target makes the beam and the dot tiny, while `POINTER_ORIGIN_MARGIN` keeps the origin in front of the small window controls, which float a few centimetres in front of their panels. The helper's own white dot is the visible cursor. In empty space it's an interactive overlay that the laser lands on, so SteamVR never draws a laser into nothing. diff --git a/pointer/helper/ft-pointer.cpp b/pointer/helper/ft-pointer.cpp index a06a5fc..4820c2c 100644 --- a/pointer/helper/ft-pointer.cpp +++ b/pointer/helper/ft-pointer.cpp @@ -311,6 +311,7 @@ extern "C" { #include #include #include +#include #include #include #include @@ -500,29 +501,62 @@ std::string JsonQuote(const std::string &s) { // public call to enumerate other apps' overlays). Hidden ones are listed too: the // window controls under a floating panel only appear while something hovers the // panel, and the cursor has to find them the moment they do, not a second later. +// Each vrcmd run is a shell and a new SteamVR client, about 30 ms of CPU, so the list is read +// every 20 seconds (kEvery) while the pointer is awake, and at once when something says it may +// have changed (Kick: the pointer waking, the dashboard opening or closing, a click that hit +// nothing), at most once a second. Showing and hiding the overlays it knows doesn't need a new +// list: the main loop polls their visibility (IsOverlayVisible) every 50 ms. // Paused while the pointer is off: each vrcmd run connects to SteamVR as a new app, and a new // app every second kept SteamVR (and the headset's displays) from going to standby. // "overlays" requests (Frametop Input Settings' Ignored panels page) refresh the list even // while paused, and are answered from this thread once it's fresh. class OverlayList { public: + static constexpr auto kEvery = std::chrono::seconds(20), kGap = std::chrono::seconds(1); void Start() { out_ = socket(AF_UNIX, SOCK_DGRAM | SOCK_CLOEXEC, 0); thread_ = std::thread([this] { + std::unique_lock lk(lock_); while (running_) { - if (!paused_ || requested_) Refresh(); - // Wait a second, or less when the pointer wakes (refresh right away then). - for (int i = 0; i < 10 && running_; ++i) { - const bool wasPaused = paused_; - std::this_thread::sleep_for(std::chrono::milliseconds(100)); - if ((wasPaused && !paused_) || requested_) break; + // Sleeps while paused (until a request or the wake); otherwise until it's time, or + // a kick once kGap has passed since the last read. + const auto due = std::max(last_ + kGap, kicked_ ? last_ : last_ + kEvery); + if (!waiting_.empty()) { + } else if (paused_) { + cv_.wait(lk); + continue; + } else if (std::chrono::steady_clock::now() < due) { + cv_.wait_until(lk, due); + continue; } + kicked_ = false; + lk.unlock(); + Refresh(); + lk.lock(); + last_ = std::chrono::steady_clock::now(); } }); } - void SetPaused(bool paused) { paused_ = paused; } + // Unpausing (the pointer woke) reads the list again at once. + void SetPaused(bool paused) { + std::lock_guard guard(lock_); + if (paused == paused_) return; + if (!paused) kicked_ = true, last_ = {}; + paused_ = paused; + cv_.notify_one(); + } + // The list may have changed: read it again soon (not while paused). + void Kick() { + std::lock_guard guard(lock_); + kicked_ = true; + cv_.notify_one(); + } void Stop() { - running_ = false; + { + std::lock_guard guard(lock_); + running_ = false; + cv_.notify_one(); + } if (thread_.joinable()) thread_.join(); } std::vector Keys() { @@ -531,12 +565,14 @@ public: for (const auto &e : entries_) keys.push_back(e.key); return keys; } + // Goes up by one each time the list is read, so the main loop knows to look the keys up again. + unsigned Generation() const { return generation_; } // Answer `to` with {"t":"overlays","list":[{"key","name","visible"}...]} after the next refresh. void Request(const sockaddr_un &to, socklen_t len) { if (len <= offsetof(sockaddr_un, sun_path)) return; std::lock_guard guard(lock_); if (waiting_.size() < 8) waiting_.push_back({to, len}); - requested_ = true; + cv_.notify_one(); } private: @@ -545,7 +581,6 @@ private: bool visible; }; void Refresh() { - requested_ = false; FILE *p = popen("LD_LIBRARY_PATH=/opt/steamvr/bin/linuxarm64 /opt/steamvr/bin/linuxarm64/vrcmd --overlays 2>/dev/null", "r"); if (!p) return; std::vector entries; @@ -577,6 +612,7 @@ private: entries_ = std::move(entries); waiting.swap(waiting_); } + ++generation_; if (waiting.empty()) return; std::string msg = "{\"t\":\"overlays\",\"list\":["; for (size_t i = 0; i < entries_.size(); ++i) @@ -589,10 +625,11 @@ private: std::thread thread_; int out_ = -1; - std::atomic paused_{false}; - std::atomic running_{true}; - std::atomic requested_{false}; - std::mutex lock_; + std::mutex lock_; // guards everything below + std::condition_variable cv_; + bool paused_ = false, running_ = true, kicked_ = false; + std::chrono::steady_clock::time_point last_{}; // the last read + std::atomic generation_{0}; std::vector entries_; // written only by the thread; the lock guards readers std::vector> waiting_; }; @@ -965,6 +1002,8 @@ int main() { if (debug) std::fflush(stdout); } nudging = confirmLesson = false; + // A click on nothing: maybe on an overlay that came up since the list was read. + if (lastHit.empty()) overlays.Kick(); leftHeld = true; dragDistance = lastDistance; pressKey.clear(); @@ -1089,6 +1128,7 @@ int main() { !aimHeld && !leftHeld && !tilting && hold.src == Src::None && !clickPress && !clickRelease; }; auto lastSlow = std::chrono::steady_clock::now() - std::chrono::seconds(10); + unsigned listGeneration = 0; // the overlay list the handles were looked up from // --- Panel placement (see "Placement" at the top of the file) --- // Device pose, given in the standing universe, sent to the driver in raw space. @@ -1928,10 +1968,11 @@ int main() { } followAt = tnow; - // Slow work, once a second: overlay handles, our device index, laser width. + // Slow work, once a second and when the overlay list is new: overlay handles, our device index. const auto now = std::chrono::steady_clock::now(); - if (now - lastSlow > std::chrono::seconds(1)) { + if (now - lastSlow > std::chrono::seconds(1) || overlays.Generation() != listGeneration) { lastSlow = now; + listGeneration = overlays.Generation(); handles.clear(); for (const auto &key : overlays.Keys()) { if (Ignored(ignore, key)) continue; @@ -2275,6 +2316,10 @@ int main() { vr::VREvent_t ev; while (sys->PollNextEvent(&ev, sizeof ev)) { + // The dashboard's overlays come and go with it, and a game brings its own. + if (ev.eventType == vr::VREvent_DashboardActivated || ev.eventType == vr::VREvent_DashboardDeactivated || + ev.eventType == vr::VREvent_DashboardOverlayCreated || ev.eventType == vr::VREvent_SceneApplicationChanged) + overlays.Kick(); if (ev.eventType == vr::VREvent_Quit) { sys->AcknowledgeQuit_Exiting(); From a11cef49bad187e1e036a228985eed6997fe4f23 Mon Sep 17 00:00:00 2001 From: DeeJanuz <45082401+DeeJanuz@users.noreply.github.com> Date: Sat, 3 Oct 2026 09:11:15 -0600 Subject: [PATCH 03/24] Gaze: ft-eyes and ft-gaze below SteamVR's priority ft-eyes and ft-gaze run in the dev container through distrobox, so they live in podman's libpod scope: frametop-gaze.service's limits never reach them, and they ran at nice 0 next to vrcompositor and vrserver, also at nice 0. - ft-eyes sets itself to nice 10 and SCHED_BATCH at start, before its threads. Batch turns off wakeup preemption, so a frame ft-eyes wakes up for can wait out a running compositor's turn; a few ms late costs the gaze little. - ft-gaze sets nice 5 before its threads start, but stays SCHED_OTHER: each sample goes on to the pointer, and batch would add the same wait to every one of them. - Both only ever lower their priority (a higher nice already set wins), and a failure is logged and ignored. Checked in the dev container: nice 0 -> 10, policy 0 -> 3 (SCHED_BATCH) without any capability. Co-Authored-By: Claude Opus 5.5 --- gaze/README.md | 2 +- gaze/ft-gaze.cpp | 10 ++++++++++ gaze/tracker/ft-eyes | 17 +++++++++++++++++ 3 files changed, 28 insertions(+), 1 deletion(-) diff --git a/gaze/README.md b/gaze/README.md index 3346530..2f47612 100644 --- a/gaze/README.md +++ b/gaze/README.md @@ -63,7 +63,7 @@ Ground rules, for anyone changing it: - **Clean room.** Nothing of Valve's goes in: we don't decompile, disassemble, or patch the `eyetracking` binary or its network weights, and we don't copy their code or weights. Its public output (eye-server.mmap, read-only) is fair game as a baseline and as labels, and so are published papers and openly licensed pupil detectors (check each one's license: PuRe, PuReST, ElSe, and ExCuSe are non-commercial only). - **Root only reads.** ft-eyegrab never writes to, stops, or signals the `eyetracking` process, vrserver, or vrcompositor, never opens `/dev/adsp`, `/dev/cdsp`, or `/dev/spidev0.1`, and never writes to `/dev/shm/eye-server.mmap` (it also carries calibration clicks into SteamVR's tracker), `/opt`, or `/persist`. - **Eye images are biometric data.** Recordings live outside the repo, in `~/.local/share/frametop/eyes/captures` (0700), and `.gitignore` catches stray frame dumps. They go nowhere but the machine that runs your offline jobs. -- **Mind the headset's budget.** Finding a pupil takes about 0.4 ms a frame while ft-eyes follows it, and 1.4-2.1 ms when it searches the whole frame. ft-eyes keeps OpenCV and numpy to one thread: their pools of one per core spun idle workers at about a quarter of a core, for frames this small. Replays, scoring, and training go to a PC. +- **Mind the headset's budget.** Finding a pupil takes about 0.4 ms a frame while ft-eyes follows it, and 1.4-2.1 ms when it searches the whole frame. ft-eyes keeps OpenCV and numpy to one thread: their pools of one per core spun idle workers at about a quarter of a core, for frames this small. It also runs at nice 10 with SCHED_BATCH, and ft-gaze at nice 5 (not batch, since each sample goes on to the pointer): both run in the dev container's podman scope, out of reach of the gaze service's unit, on the cores vrcompositor and vrserver use at nice 0. Replays, scoring, and training go to a PC. ## Headset fit diff --git a/gaze/ft-gaze.cpp b/gaze/ft-gaze.cpp index 2d4a107..f11dd4d 100644 --- a/gaze/ft-gaze.cpp +++ b/gaze/ft-gaze.cpp @@ -58,6 +58,7 @@ #include #include +#include #include #include #include @@ -72,6 +73,7 @@ #include #include +#include #include #include #include @@ -430,6 +432,14 @@ int main(int argc, char **argv) { if (std::strcmp(argv[i], "-v") == 0) verbose = true; if (std::strcmp(argv[i], "--watch-stdin") == 0) watchStdin = true; } + // Nice 5, before any thread starts (they inherit it): we run in the dev container's podman + // scope, beside vrcompositor and vrserver at nice 0, and the gaze service's unit doesn't + // reach us. Not SCHED_BATCH, as ft-eyes is: that would let each wakeup wait out another + // task's turn, and each sample goes on to the pointer. + errno = 0; + const int nice0 = getpriority(PRIO_PROCESS, 0); + if (errno == 0 && nice0 < 5 && setpriority(PRIO_PROCESS, 0, 5) != 0) + std::fprintf(stderr, "ft-gaze: nice: %s\n", std::strerror(errno)); // --watch-stdin: quit when stdin closes. The probe runs us through distrobox, which // passes neither its signals nor a closed stdout on to us, but does pass stdin's end. std::atomic stdinClosed{false}; diff --git a/gaze/tracker/ft-eyes b/gaze/tracker/ft-eyes index 66bbff7..f716f77 100755 --- a/gaze/tracker/ft-eyes +++ b/gaze/tracker/ft-eyes @@ -377,7 +377,24 @@ def serve(sock, tracker): pass +def below_steamvr(): + """Nice 10 and SCHED_BATCH, for this thread and those it starts. ft-eyes runs in the dev + container's podman scope, where the gaze service's unit doesn't reach it, so it ran at + nice 0 on the cores vrcompositor and vrserver use. Batch also lets a waking ft-eyes wait + for the running task's turn instead of taking the core: a frame a few ms late costs the + gaze little, a late compositor frame costs a dropped frame in the headset.""" + try: + os.setpriority(os.PRIO_PROCESS, 0, max(os.getpriority(os.PRIO_PROCESS, 0), 10)) + except OSError as e: + print(f"ft-eyes: nice: {e}", file=sys.stderr, flush=True) + try: + os.sched_setscheduler(0, os.SCHED_BATCH, os.sched_param(0)) + except (OSError, AttributeError) as e: + print(f"ft-eyes: SCHED_BATCH: {e}", file=sys.stderr, flush=True) + + def main(): + below_steamvr() verbose = "-v" in sys.argv if "--watch-stdin" in sys.argv: # Run by ft-gazed through distrobox, which doesn't pass a stop on: quit when our From 0680297efa22b1d8d98ca7f391abbe51e906f92d Mon Sep 17 00:00:00 2001 From: DeeJanuz <45082401+DeeJanuz@users.noreply.github.com> Date: Sat, 3 Oct 2026 09:12:33 -0600 Subject: [PATCH 04/24] Pointer: sleep on the command socket while the pointer is off The main loop slept a fixed 8 ms, about 116 wakeups a second, whether the pointer was awake or not, and every second it looked up every overlay's handle and read a string property from all 64 device slots to find its own device. With the pointer off and hand gestures off, the loop now waits in poll() on its command socket for up to 250 ms, or 20 ms while mapped Frame controller buttons are being read (SteamVR input has no event to wait for). A mouse command ends the wait at once. The headset's activity level, the game check, and the "vrgame" and "gazeawake" repeats keep going at that pace. The 50 ms visibility poll and the 1 s handle lookups run only while the pointer is awake, and waking forces both. The device index is looked for only while it's unknown, and again after SteamVR activates or deactivates a device. The HMD pose history is kept only with hand gestures on, its one user. Co-Authored-By: Claude Opus 5.5 --- docs/design.md | 2 +- pointer/helper/ft-pointer.cpp | 42 ++++++++++++++++++++++++++--------- pointer/helper/vrbuttons.h | 8 +++++++ 3 files changed, 41 insertions(+), 11 deletions(-) diff --git a/docs/design.md b/docs/design.md index fb29cbf..10f697e 100644 --- a/docs/design.md +++ b/docs/design.md @@ -141,7 +141,7 @@ Replacing a loaded driver's files, as re-running the installer used to do, leave The dashboard follows whichever device summoned it or last pressed its trigger. Frametop adds "last used wins": moving a real controller releases the pointer, and the next mouse movement takes the laser back. Moving means faster than 0.35 m/s or 2 rad/s (both times `POINTER_CONTROLLER_PICKUP`, 1 by default) for 100 ms in a row, while the controller is tracked normally. A single sample over the limit used to be enough, and controllers resting on a desk took the laser back on a knock or a tracking jump while the mouse was in use. Small movements don't count; waking needs `POINTER_WAKE_COUNTS` of mouse motion within a second, so desk jitter doesn't steal the laser. While the pointer is awake, a tiny transparent overlay with `MakeOverlaysInteractiveIfVisible` keeps SteamVR's laser mouse on, since otherwise the first click would only switch the laser on. -When the headset comes off, SteamVR reports its activity level as idle at once and turns the displays off 5 seconds later (`power.turnOffScreensTimeout`), unless something keeps it awake. An awake pointer did, and so did the helper's `vrcmd` runs: each is a new SteamVR client, and a new client every second kept SteamVR out of standby. The helper now releases the pointer as soon as the headset is idle, ignores the mouse until you're wearing it again, and pauses the overlay list whenever the pointer is off. +When the headset comes off, SteamVR reports its activity level as idle at once and turns the displays off 5 seconds later (`power.turnOffScreensTimeout`), unless something keeps it awake. An awake pointer did, and so did the helper's `vrcmd` runs: each is a new SteamVR client, and a new client every second kept SteamVR out of standby. The helper now releases the pointer as soon as the headset is idle, ignores the mouse until you're wearing it again, and pauses the overlay list whenever the pointer is off. With the pointer off it also stops running its loop every 8 ms, about 116 wakeups a second for nothing: it waits up to 250 ms for a command on its socket (20 ms while it reads mapped controller buttons, which SteamVR input only offers by polling), and leaves the overlay lookups until the pointer wakes. ### Moving floating windows diff --git a/pointer/helper/ft-pointer.cpp b/pointer/helper/ft-pointer.cpp index 4820c2c..a48c246 100644 --- a/pointer/helper/ft-pointer.cpp +++ b/pointer/helper/ft-pointer.cpp @@ -36,6 +36,13 @@ // pointer is off the helper also stops listing overlays with vrcmd, whose connection every // second kept SteamVR from going to standby. // +// Idle: while the pointer is off (and hand gestures are off), the main loop waits for a command +// on its socket for up to 250 ms instead of running every 8 ms, or 20 ms while it reads mapped +// Frame controller buttons (vrbuttons.h), which have no event to wait for. What has to go on +// meanwhile still does: the headset's activity level, the game check and the relay's "vrgame" +// and "gazeawake" repeats, and a mouse command wakes it at once. The overlay lookups (the 50 ms +// visibility poll, the once-a-second handles) wait for the pointer to wake, and run right then. +// // Last used wins: when a real controller moves (picked up), the pointer is released // (driver "hide", which also drops its hand role hint), so the controller gets // its role and laser back. The next mouse input reconnects and claims the laser again. @@ -329,6 +336,7 @@ extern "C" { #include #include +#include #include #include #include @@ -859,6 +867,8 @@ int main() { std::map sceneGraph; // no texture: plane test instead of ComputeOverlayIntersection std::map visible; // refreshed every 50 ms auto lastVisible = std::chrono::steady_clock::now(); + auto lastSlow = std::chrono::steady_clock::now() - std::chrono::seconds(10); + unsigned listGeneration = 0; // the overlay list the handles were looked up from // The plane of the last panel the cursor was on, and the last point on it (panel edges). Vec3 edgePoint, edgeNormal, edgeLast; std::string edgeKey; @@ -901,6 +911,8 @@ int main() { wokeAt = t; active = true; recenter = true; + // Overlay handles and visibility weren't looked at while it was off (see "Idle" at the top). + lastSlow = lastVisible = t - std::chrono::seconds(10); SendTo(out, "ft_pointer", "role " + role); // POINTER_ROLE, before it takes it SendTo(out, "ft_pointer", "show"); claimPending = true; // take the laser without clicking, once SteamVR has bound the device @@ -1127,8 +1139,6 @@ int main() { return gazeOn && gazeMouseHeld && !inGame && !headsetOff && Clock::now() - gz.at < std::chrono::seconds(1) && !aimHeld && !leftHeld && !tilting && hold.src == Src::None && !clickPress && !clickRelease; }; - auto lastSlow = std::chrono::steady_clock::now() - std::chrono::seconds(10); - unsigned listGeneration = 0; // the overlay list the handles were looked up from // --- Panel placement (see "Placement" at the top of the file) --- // Device pose, given in the standing universe, sent to the driver in raw space. @@ -1669,7 +1679,7 @@ int main() { } // Hands (see the top): pinches and grips from ft-hands. - { + if (handsOn) { vr::TrackedDevicePose_t h0; sys->GetDeviceToAbsoluteTrackingPose(vr::TrackingUniverseStanding, 0, &h0, 1); if (h0.bPoseIsValid) poses.Add(tnow, h0.mDeviceToAbsoluteTracking); @@ -1968,9 +1978,10 @@ int main() { } followAt = tnow; - // Slow work, once a second and when the overlay list is new: overlay handles, our device index. + // Slow work, once a second and when the overlay list is new: overlay handles, our device + // index. Only while the pointer is awake (see "Idle" at the top). const auto now = std::chrono::steady_clock::now(); - if (now - lastSlow > std::chrono::seconds(1) || overlays.Generation() != listGeneration) { + if (active && (now - lastSlow > std::chrono::seconds(1) || overlays.Generation() != listGeneration)) { lastSlow = now; listGeneration = overlays.Generation(); handles.clear(); @@ -1992,16 +2003,16 @@ int main() { sceneGraph[key] = (tw == 0 || th == 0) && tt == vr::VROverlayTransform_Absolute && key.rfind("frametop.", 0) != 0; } - ours = vr::k_unTrackedDeviceIndexInvalid; - for (vr::TrackedDeviceIndex_t i = 0; i < vr::k_unMaxTrackedDeviceCount; ++i) { + // Our device keeps its index; it's looked for again when a device is activated or + // deactivated (the events below). + for (vr::TrackedDeviceIndex_t i = 0; ours == vr::k_unTrackedDeviceIndexInvalid && i < vr::k_unMaxTrackedDeviceCount; ++i) { char type[64] = ""; sys->GetStringTrackedDeviceProperty(i, vr::Prop_ControllerType_String, type, sizeof type); if (std::strcmp(type, "ft_pointer") == 0) ours = i; } - } - if (now - lastVisible > std::chrono::milliseconds(50) || visible.size() != handles.size()) { + if (active && (now - lastVisible > std::chrono::milliseconds(50) || visible.size() != handles.size())) { lastVisible = now; visible.clear(); for (const auto &[key, h] : handles) visible[key] = overlay->IsOverlayVisible(h); @@ -2320,6 +2331,8 @@ int main() { if (ev.eventType == vr::VREvent_DashboardActivated || ev.eventType == vr::VREvent_DashboardDeactivated || ev.eventType == vr::VREvent_DashboardOverlayCreated || ev.eventType == vr::VREvent_SceneApplicationChanged) overlays.Kick(); + if (ev.eventType == vr::VREvent_TrackedDeviceActivated || ev.eventType == vr::VREvent_TrackedDeviceDeactivated) + ours = vr::k_unTrackedDeviceIndexInvalid; if (ev.eventType == vr::VREvent_Quit) { sys->AcknowledgeQuit_Exiting(); @@ -2328,6 +2341,15 @@ int main() { return 0; } } - std::this_thread::sleep_for(std::chrono::milliseconds(8)); + // Idle (see the top): wait for a command instead. Not with anything still to finish (the + // claim pulse, a click's release, a press). + const bool idle = !active && !handsOn && !claimPending && !claimHeld && !clickRelease && !leftHeld && + hold.src == Src::None; + if (idle) { + pollfd p{in, POLLIN, 0}; + poll(&p, 1, controllerButtons.Watching(inGame) ? 20 : 250); + } else { + std::this_thread::sleep_for(std::chrono::milliseconds(8)); + } } } diff --git a/pointer/helper/vrbuttons.h b/pointer/helper/vrbuttons.h index 10d8f17..80d11a8 100644 --- a/pointer/helper/vrbuttons.h +++ b/pointer/helper/vrbuttons.h @@ -111,6 +111,14 @@ class ControllerButtons { } } + // Whether Poll reads any button now (the main loop can't sleep long while it does). + bool Watching(bool inGame) const { + if (!ok_ || (inGame && !games_ && !capture_)) return false; + for (int i = 0; i < kCount; ++i) + if (bound_[i]) return true; + return false; + } + // {"manifest":true,"global":false,"bound":[...],"active":[...]}: active = bound and // delivered (a controller that has the button is on, and SteamVR lets us have it). std::string Status() const { From 3e7248a04e0b505d37ff73a83ba8b30b74c2e7da Mon Sep 17 00:00:00 2001 From: DeeJanuz <45082401+DeeJanuz@users.noreply.github.com> Date: Sat, 3 Oct 2026 09:14:25 -0600 Subject: [PATCH 05/24] Remote desktop: connect FreeRDP only while a VNC viewer is connected vnc-bridge.sh kept FreeRDP connected to krdpserver from the moment remote desktop started, so krdp captured and H.264-encoded every KWin redraw in software (openh264) with nobody watching: krdpserver 55-78% of a core, xfreerdp 16-27%, Xvnc 6-11%, with 0 clients on :5900. krdp 6.7 creates its screencast session per RDP connection and drops it when the connection closes, so krdpserver itself idles without one and stays up. The bridge now counts established connections to Xvnc's port with ss, starts FreeRDP when a viewer appears (the desktop shows about 3 s later; the VNC screen is black until then) and stops it 45 s after the last one leaves (VNC_IDLE_SEC). Xvnc has no client hook, so its log output, which it writes for every connection, wakes the bridge early; otherwise it looks every 5 s while idle (0.1% of a core measured, against 0.9% for ss once a second) and every second while FreeRDP runs. The layout check runs only while FreeRDP runs. While a viewer is connected the bridge sends "watch 15" to ft-screens (@ft_screens) at once and every 5 s, so screens at a reduced frame rate (out of view, headset on a stand) stream at full rate; it lapses by itself if the bridge dies, and an ft-screens without the command just answers an error. The window search after starting FreeRDP now ends when FreeRDP exits instead of polling for 30 s. krdp on 127.0.0.1 with a fresh password, VNC on the tailnet address with VncAuth, and remote-ctl.sh start/stop (pause and resume) are unchanged. Co-Authored-By: Claude Opus 5.5 --- README.md | 2 +- docs/design.md | 2 + docs/reference.md | 2 + session/vnc-bridge.sh | 108 ++++++++++++++++++++++++++++++++++++------ 4 files changed, 98 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index 7fb50a9..4dd22d3 100644 --- a/README.md +++ b/README.md @@ -144,7 +144,7 @@ This is an early release, tested on one Steam Frame (SteamOS 0.3.0 build 2026092 - Dragging something from one panel to another (a screen and a floating window) works, but the dragged item's icon doesn't show while the pointer is between panels. - Gaze mode is only as good as its calibration, and that depends on how the headset sits on your face. If the pointer lands off after you adjust the headset, run Quick check or Calibrate on the Gaze page of Frametop Input Settings. - On SteamVR's Settings page, the 3D mouse shows a laser beam and a larger hit dot, like a controller. SteamVR doesn't tell other programs where that page is (unlike Steam's pages, such as Library), so the mouse used to miss most of it: clicks went through to a desktop screen behind, and the dot disappeared. As a workaround, on that page only, the laser starts near your eye and SteamVR finds the page itself. See docs/design.md. -- Remote desktop over VNC (Frametop Remote Access in the app menu, or `./desktops.sh remote on`) needs Tailscale on the Frame. It shows the primary screen only. The app turns it on and off, shows the address, and shows, copies, or changes the VNC password. The password is made at random on the Frame and kept in `~/.config/frametop-remote` (only you can read it); VNC limits it to 8 characters, and the tailnet encrypts the connection. Turning it on in a desktop that started with it off takes a desktop restart. +- Remote desktop over VNC (Frametop Remote Access in the app menu, or `./desktops.sh remote on`) needs Tailscale on the Frame. It shows the primary screen only. The app turns it on and off, shows the address, and shows, copies, or changes the VNC password. The password is made at random on the Frame and kept in `~/.config/frametop-remote` (only you can read it); VNC limits it to 8 characters, and the tailnet encrypts the connection. Turning it on in a desktop that started with it off takes a desktop restart. It costs almost nothing until a viewer connects; the picture then takes a few seconds to appear. - Turning the displays off on a stand only turns their backlight off. SteamVR has no way for other programs to put the headset in standby, so tracking and rendering keep running, and the headset draws nearly its full power. ## Reporting problems diff --git a/docs/design.md b/docs/design.md index 4725296..a743a22 100644 --- a/docs/design.md +++ b/docs/design.md @@ -177,6 +177,8 @@ The private runtime directory also moves the session's document portal to `$XDG_ A podman container's monitor process (conmon) stays in the cgroup of whatever started the container, and `distrobox enter` starts it on demand. When a Frametop service happened to start the `dev` container, stopping that service stopped the container and everything in it, including the desktop's compositor. `scripts/container-up.sh` starts the container in a systemd scope of its own before anything enters it. It then waits for distrobox-init to log `container_setup_done`, as `distrobox enter` does only for containers it starts itself. A new container's first start takes a minute or more (it installs distrobox's dependencies and sets up passwordless sudo), and an install that entered right away met a sudo password prompt with no terminal to answer it ([#9](https://github.com/DeeJanuz/frametop/issues/9)). +Remote desktop is a chain (krdp, then FreeRDP inside Xvnc) because nothing on SteamOS serves KWin over VNC directly. Kept connected all the time, it cost about a core with nobody watching: krdpserver 55 to 78% (it encodes H.264 in software with openh264: VA-API finds no driver for the Frame's GPU in the container), FreeRDP 16 to 27%, Xvnc 6 to 11%. krdp creates its screencast session per RDP connection and drops it when the connection closes (`SessionController::onNewConnection` in krdp 6.7), so an idle krdpserver costs nothing and can stay up; only the RDP connection has to go. The bridge connects FreeRDP when a VNC client appears and disconnects 45 seconds after the last one leaves. Xvnc has no hook for its clients, so the bridge counts established connections to its port with `ss`, woken early by Xvnc's log output; looking with `ss` once a second cost about 0.9% of a core in bash, against about 0.1% this way. `Xvnc -inetd` from a systemd socket would start a server per connection and lose sharing between viewers. + Program names stay within 15 characters, because Linux truncates process names there and the scripts find programs with `pgrep -x` and `pkill -x`. That's why the prefix is `ft-`. ## Displays off on a stand diff --git a/docs/reference.md b/docs/reference.md index b1273b1..46d9119 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -259,6 +259,8 @@ It listens on port 5900 on the Frame's Tailscale address only, not the LAN, so i No VNC server can capture KWin on SteamOS directly: `krfb` needs `xdg-desktop-portal-kde`, which SteamOS doesn't ship, and `wayvnc` only works with wlroots compositors. So `session/remote-desktop.sh` captures the desktop with KDE's `krdpserver --plasma` on `127.0.0.1:3390`, and `session/vnc-bridge.sh` runs TigerVNC's `Xvnc` on display `:20` with a FreeRDP client inside it and serves that. Both run in the `dev` container, and the extra hop adds a little latency. krdp streams every screen; the VNC screen is the primary's size, and the FreeRDP window is shifted so the primary fills it (`ft-layout remote-view` gives the offset). krdp's own `--monitor` would stream just one screen, but it maps the pointer as if that screen sat at 0,0, so clicks would miss. When the layout changes, the VNC screen resizes and FreeRDP reconnects within a few seconds. +FreeRDP runs only while a VNC viewer is connected, because while it's connected krdp captures and encodes every redraw. With no viewer, krdp has no RDP connection and so captures nothing, and Xvnc shows a black screen. When a viewer connects, the bridge starts FreeRDP, and the desktop appears about 3 seconds later; FreeRDP stops 45 seconds after the last viewer leaves (`VNC_IDLE_SEC` in the bridge's environment). The bridge looks for viewers with `ss` whenever Xvnc logs something, as it does for every connection, and every 5 seconds otherwise. While a viewer is connected, the bridge asks ft-screens to draw every screen at full rate (`watch 15` on `@ft_screens`, renewed every 5 seconds), so screens you aren't looking at in the headset, or a headset on a stand, don't stream at a low rate. It checks the primary screen's place (`ft-layout remote-view`) every 5 seconds, only while FreeRDP runs. + With remote access on, the nested KWin runs with `KWIN_WAYLAND_NO_PERMISSION_CHECKS=1` and `KWIN_SCREENSHOT_NO_PERMISSION_CHECKS=1`, so any app in the Frametop desktop could capture its screens or inject input. The second one lets scripts take screenshots through KWin's `org.kde.KWin.ScreenShot2` D-Bus interface. This applies only to that desktop, not the stock one. Port 3389 is SteamOS's own `xrdp`, which starts a separate X11 session rather than showing the VR desktop. ## Limits diff --git a/session/vnc-bridge.sh b/session/vnc-bridge.sh index 166fd82..02041db 100755 --- a/session/vnc-bridge.sh +++ b/session/vnc-bridge.sh @@ -11,6 +11,16 @@ # screen placed lower or further right. Instead the VNC screen is the primary's size, and # the workspace-sized RDP window inside it is shifted so the primary fills it. The pointer # maps 1:1. When the layout changes, the VNC screen resizes and the RDP client reconnects. +# +# The RDP client runs only while someone watches. While connected, krdp captures and +# H.264-encodes every redraw in software, about 60% of a core, with FreeRDP and Xvnc adding +# about 30% more, even with no VNC viewer. krdp starts its capture per RDP connection and +# stops it when the connection closes, so it idles without one. So FreeRDP starts when a +# VNC client connects (the screen is black for the few seconds that takes) and stops +# VNC_IDLE_SEC (45) seconds after the last one leaves. Xvnc has no hook for clients, so ss +# counts them: whenever Xvnc writes to its log (it logs each connection), every second +# while FreeRDP runs, and every 5 seconds otherwise. The log only wakes this script up; +# what it says doesn't matter. set -eu here=$(dirname "$(readlink -f "$0")") @@ -57,16 +67,45 @@ stop_rdp() { pkill -f "[x]freerdp /v:127.0.0.1:$rdp_port " 2>/dev/null || true; trap 'stop_rdp; pkill -f "[X]vnc $display " 2>/dev/null || true' EXIT box bash -c 'vncpasswd -f < "$1/vnc-password" > "$1/vnc-passwd.bin" && chmod 600 "$1/vnc-passwd.bin"' - "$creds" -box Xvnc "$display" -geometry "${w}x${h}" -depth 24 \ +exec {xlog}< <(box Xvnc "$display" -geometry "${w}x${h}" -depth 24 \ -interface "$addr" -rfbport "$vnc_port" \ -SecurityTypes VncAuth -PasswordFile "$creds/vnc-passwd.bin" \ - -AlwaysShared -desktop "Steam Frame (Frametop)" & + -AlwaysShared -desktop "Steam Frame (Frametop)" 2>&1) xvnc=$! -sleep 2 -# Keep an RDP connection open inside the VNC screen. Reconnect if it drops or the layout changes. +# Wait up to $1 seconds, less if Xvnc logs something; its lines go on to this log. +nap() { + local line rc=0 + IFS= read -rt "$1" -u "$xlog" line || rc=$? + if [ $rc -eq 0 ]; then + printf '%s\n' "$line" + while IFS= read -rt 0.1 -u "$xlog" line; do printf '%s\n' "$line"; done + elif [ $rc -le 128 ]; then + sleep 1 # Xvnc's output closed: it's exiting + fi + return 0 +} +nap 2 + +# A connected VNC client: an established TCP connection to Xvnc's port. Any connection +# counts, authenticated or not; it's on the tailnet only. +clients() { [ -n "$(ss -Htn state established "( sport = :$vnc_port )" 2>/dev/null)" ]; } + +# ft-screens drops screens you aren't looking at to a low frame rate, and krdp would +# stream that. "watch SECONDS" asks it for full rate on every screen for that long: sent +# when a client connects and renewed every few seconds while one stays, so it lapses by +# itself if this script dies. An older ft-screens just answers that it doesn't know it. +watch() { + if command -v socat >/dev/null; then + printf 'watch 15' | socat -u - ABSTRACT-SENDTO:ft_screens 2>/dev/null + else + python3 -c 'import socket; socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM).sendto(b"watch 15", "\0ft_screens")' 2>/dev/null + fi || true +} + +# Start FreeRDP inside the VNC screen, sized and shifted for $v. # /cert:ignore is fine here: the connection never leaves this host. -while kill -0 $xvnc 2>/dev/null; do +start_rdp() { read -r x y w h ww wh <<< "$v" box env DISPLAY=$display bash -c ' size=$1 x=$2 y=$3 ww=$4 wh=$5 creds=$6 rdp_port=$7 @@ -80,6 +119,7 @@ while kill -0 $xvnc 2>/dev/null; do rdp=$! # FreeRDP takes no negative position, so move its window once it is up. for _ in $(seq 60); do + kill -0 $rdp 2>/dev/null || break win=$(xdotool search --class xfreerdp 2>/dev/null | tail -1) [ -n "$win" ] && break sleep 0.5 @@ -88,14 +128,52 @@ while kill -0 $xvnc 2>/dev/null; do wait $rdp ' vnc-rdp "${w}x$h" "$x" "$y" "$ww" "$wh" "$creds" "$rdp_port" || true & rdp=$! - while kill -0 $rdp 2>/dev/null; do - sleep 5 - now=$(view) || continue - [ -n "$now" ] && [ "$now" != "$v" ] || continue - echo "layout changed: $v -> $now" - v=$now - stop_rdp - done - wait $rdp 2>/dev/null || true - sleep 2 +} + +idle_sec=${VNC_IDLE_SEC:-45} +rdp= # FreeRDP's job while it runs +seen=0 # when a client was last seen ($SECONDS) +watched=-99 # when "watch" was last sent +next_view=0 # next time to read the layout +while kill -0 $xvnc 2>/dev/null; do + if clients; then + if [ $((SECONDS - watched)) -ge 5 ]; then watch; watched=$SECONDS; fi + seen=$SECONDS + if [ -z "$rdp" ]; then + echo "VNC client connected, starting the RDP client" + now=$(view) && [ -n "$now" ] && v=$now + next_view=$((SECONDS + 5)) + start_rdp + fi + elif [ "$seen" -ne 0 ]; then + watched=-99 + if [ $((SECONDS - seen)) -ge "$idle_sec" ]; then + seen=0 + if [ -n "$rdp" ]; then + echo "no VNC client for ${idle_sec}s, stopping the RDP client" + stop_rdp + wait "$rdp" 2>/dev/null || true + rdp= + fi + fi + fi + if [ -n "$rdp" ] && ! kill -0 "$rdp" 2>/dev/null; then + # It dropped, or the layout changed: reconnect next round if a client is still there. + wait "$rdp" 2>/dev/null || true + rdp= + nap 2 + continue + fi + if [ -n "$rdp" ]; then + if [ "$SECONDS" -ge "$next_view" ]; then + next_view=$((SECONDS + 5)) + now=$(view) || now= + if [ -n "$now" ] && [ "$now" != "$v" ]; then + echo "layout changed: $v -> $now" + v=$now + stop_rdp + fi + fi + fi + if [ -n "$rdp" ] || [ "$seen" -ne 0 ]; then nap 1; else nap 5; fi done From e1ee5ef395419ab045991a08518b99f71402c747 Mon Sep 17 00:00:00 2001 From: DeeJanuz <45082401+DeeJanuz@users.noreply.github.com> Date: Sat, 3 Oct 2026 09:14:43 -0600 Subject: [PATCH 06/24] Remote desktop: read the layout only after it changes While FreeRDP ran, vnc-bridge.sh called ft-layout remote-view every 5 s, which scans all of /proc for plasmashell and runs kscreen-doctor -j: about 4.4% of a core for a layout that rarely changes. It now stats the two files the answer depends on, the nested KWin's ~/.config/frametop/kwinoutputconfig.json (positions, scales, primary) and ~/.config/frametop-layout.json (screen sizes), once a second while FreeRDP runs. After either changes it reads the layout every 2 s for 10 s, since KWin's outputs follow the file a few seconds later; otherwise once a minute, in case a change touched neither. With no VNC viewer connected nothing runs (previous commit). Co-Authored-By: Claude Opus 5.5 --- docs/design.md | 2 +- docs/reference.md | 2 +- session/vnc-bridge.sh | 18 +++++++++++++++--- 3 files changed, 17 insertions(+), 5 deletions(-) diff --git a/docs/design.md b/docs/design.md index a743a22..435baa9 100644 --- a/docs/design.md +++ b/docs/design.md @@ -177,7 +177,7 @@ The private runtime directory also moves the session's document portal to `$XDG_ A podman container's monitor process (conmon) stays in the cgroup of whatever started the container, and `distrobox enter` starts it on demand. When a Frametop service happened to start the `dev` container, stopping that service stopped the container and everything in it, including the desktop's compositor. `scripts/container-up.sh` starts the container in a systemd scope of its own before anything enters it. It then waits for distrobox-init to log `container_setup_done`, as `distrobox enter` does only for containers it starts itself. A new container's first start takes a minute or more (it installs distrobox's dependencies and sets up passwordless sudo), and an install that entered right away met a sudo password prompt with no terminal to answer it ([#9](https://github.com/DeeJanuz/frametop/issues/9)). -Remote desktop is a chain (krdp, then FreeRDP inside Xvnc) because nothing on SteamOS serves KWin over VNC directly. Kept connected all the time, it cost about a core with nobody watching: krdpserver 55 to 78% (it encodes H.264 in software with openh264: VA-API finds no driver for the Frame's GPU in the container), FreeRDP 16 to 27%, Xvnc 6 to 11%. krdp creates its screencast session per RDP connection and drops it when the connection closes (`SessionController::onNewConnection` in krdp 6.7), so an idle krdpserver costs nothing and can stay up; only the RDP connection has to go. The bridge connects FreeRDP when a VNC client appears and disconnects 45 seconds after the last one leaves. Xvnc has no hook for its clients, so the bridge counts established connections to its port with `ss`, woken early by Xvnc's log output; looking with `ss` once a second cost about 0.9% of a core in bash, against about 0.1% this way. `Xvnc -inetd` from a systemd socket would start a server per connection and lose sharing between viewers. +Remote desktop is a chain (krdp, then FreeRDP inside Xvnc) because nothing on SteamOS serves KWin over VNC directly. Kept connected all the time, it cost about a core with nobody watching: krdpserver 55 to 78% (it encodes H.264 in software with openh264: VA-API finds no driver for the Frame's GPU in the container), FreeRDP 16 to 27%, Xvnc 6 to 11%, and the bridge's layout check every 5 seconds another 4%. krdp creates its screencast session per RDP connection and drops it when the connection closes (`SessionController::onNewConnection` in krdp 6.7), so an idle krdpserver costs nothing and can stay up; only the RDP connection has to go. The bridge connects FreeRDP when a VNC client appears and disconnects 45 seconds after the last one leaves. Xvnc has no hook for its clients, so the bridge counts established connections to its port with `ss`, woken early by Xvnc's log output; looking with `ss` once a second cost about 0.9% of a core in bash, against about 0.1% this way. `Xvnc -inetd` from a systemd socket would start a server per connection and lose sharing between viewers. The layout check (`ft-layout remote-view`, which scans `/proc` for plasmashell and runs `kscreen-doctor -j`) now runs only while FreeRDP runs, and then only after `kwinoutputconfig.json` or `frametop-layout.json` changes, with one check a minute in case a change touched neither. Program names stay within 15 characters, because Linux truncates process names there and the scripts find programs with `pgrep -x` and `pkill -x`. That's why the prefix is `ft-`. diff --git a/docs/reference.md b/docs/reference.md index 46d9119..6bff2ae 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -259,7 +259,7 @@ It listens on port 5900 on the Frame's Tailscale address only, not the LAN, so i No VNC server can capture KWin on SteamOS directly: `krfb` needs `xdg-desktop-portal-kde`, which SteamOS doesn't ship, and `wayvnc` only works with wlroots compositors. So `session/remote-desktop.sh` captures the desktop with KDE's `krdpserver --plasma` on `127.0.0.1:3390`, and `session/vnc-bridge.sh` runs TigerVNC's `Xvnc` on display `:20` with a FreeRDP client inside it and serves that. Both run in the `dev` container, and the extra hop adds a little latency. krdp streams every screen; the VNC screen is the primary's size, and the FreeRDP window is shifted so the primary fills it (`ft-layout remote-view` gives the offset). krdp's own `--monitor` would stream just one screen, but it maps the pointer as if that screen sat at 0,0, so clicks would miss. When the layout changes, the VNC screen resizes and FreeRDP reconnects within a few seconds. -FreeRDP runs only while a VNC viewer is connected, because while it's connected krdp captures and encodes every redraw. With no viewer, krdp has no RDP connection and so captures nothing, and Xvnc shows a black screen. When a viewer connects, the bridge starts FreeRDP, and the desktop appears about 3 seconds later; FreeRDP stops 45 seconds after the last viewer leaves (`VNC_IDLE_SEC` in the bridge's environment). The bridge looks for viewers with `ss` whenever Xvnc logs something, as it does for every connection, and every 5 seconds otherwise. While a viewer is connected, the bridge asks ft-screens to draw every screen at full rate (`watch 15` on `@ft_screens`, renewed every 5 seconds), so screens you aren't looking at in the headset, or a headset on a stand, don't stream at a low rate. It checks the primary screen's place (`ft-layout remote-view`) every 5 seconds, only while FreeRDP runs. +FreeRDP runs only while a VNC viewer is connected, because while it's connected krdp captures and encodes every redraw. With no viewer, krdp has no RDP connection and so captures nothing, and Xvnc shows a black screen. When a viewer connects, the bridge starts FreeRDP, and the desktop appears about 3 seconds later; FreeRDP stops 45 seconds after the last viewer leaves (`VNC_IDLE_SEC` in the bridge's environment). The bridge looks for viewers with `ss` whenever Xvnc logs something, as it does for every connection, and every 5 seconds otherwise. While a viewer is connected, the bridge asks ft-screens to draw every screen at full rate (`watch 15` on `@ft_screens`, renewed every 5 seconds), so screens you aren't looking at in the headset, or a headset on a stand, don't stream at a low rate. It reads the primary screen's place again (`ft-layout remote-view`) only while FreeRDP runs, after `~/.config/frametop/kwinoutputconfig.json` or `~/.config/frametop-layout.json` changes, and once a minute. With remote access on, the nested KWin runs with `KWIN_WAYLAND_NO_PERMISSION_CHECKS=1` and `KWIN_SCREENSHOT_NO_PERMISSION_CHECKS=1`, so any app in the Frametop desktop could capture its screens or inject input. The second one lets scripts take screenshots through KWin's `org.kde.KWin.ScreenShot2` D-Bus interface. This applies only to that desktop, not the stock one. Port 3389 is SteamOS's own `xrdp`, which starts a separate X11 session rather than showing the VR desktop. diff --git a/session/vnc-bridge.sh b/session/vnc-bridge.sh index 02041db..d106b06 100755 --- a/session/vnc-bridge.sh +++ b/session/vnc-bridge.sh @@ -103,6 +103,12 @@ watch() { fi || true } +# What remote-view depends on: KWin's saved outputs (positions, scales, primary) and +# Frametop's layout (screen sizes). It's read again only after one of these changes, and +# once a minute in case a change didn't touch them, and only while FreeRDP runs. +layout_files=("$HOME/.config/frametop/kwinoutputconfig.json" "$HOME/.config/frametop-layout.json") +stamp() { stat -c %y "${layout_files[@]}" 2>/dev/null || true; } + # Start FreeRDP inside the VNC screen, sized and shifted for $v. # /cert:ignore is fine here: the connection never leaves this host. start_rdp() { @@ -134,7 +140,9 @@ idle_sec=${VNC_IDLE_SEC:-45} rdp= # FreeRDP's job while it runs seen=0 # when a client was last seen ($SECONDS) watched=-99 # when "watch" was last sent -next_view=0 # next time to read the layout +stamp_v= # stamp() when $v was read +recheck=0 # read the layout every 2 seconds until then +next_view=0 # next time to read it while kill -0 $xvnc 2>/dev/null; do if clients; then if [ $((SECONDS - watched)) -ge 5 ]; then watch; watched=$SECONDS; fi @@ -142,7 +150,7 @@ while kill -0 $xvnc 2>/dev/null; do if [ -z "$rdp" ]; then echo "VNC client connected, starting the RDP client" now=$(view) && [ -n "$now" ] && v=$now - next_view=$((SECONDS + 5)) + stamp_v=$(stamp) recheck=0 next_view=$((SECONDS + 60)) start_rdp fi elif [ "$seen" -ne 0 ]; then @@ -165,8 +173,12 @@ while kill -0 $xvnc 2>/dev/null; do continue fi if [ -n "$rdp" ]; then + # A layout change shows up in KWin's outputs a few seconds after the files change. + s=$(stamp) + [ "$s" = "$stamp_v" ] || { stamp_v=$s; recheck=$((SECONDS + 10)) next_view=$SECONDS; } if [ "$SECONDS" -ge "$next_view" ]; then - next_view=$((SECONDS + 5)) + next_view=$((SECONDS + 60)) + [ "$SECONDS" -ge "$recheck" ] || next_view=$((SECONDS + 2)) now=$(view) || now= if [ -n "$now" ] && [ "$now" != "$v" ]; then echo "layout changed: $v -> $now" From 7851ce90cc0082bda3a3dfc7d6eff353e8aeafed Mon Sep 17 00:00:00 2001 From: DeeJanuz <45082401+DeeJanuz@users.noreply.github.com> Date: Sat, 3 Oct 2026 09:15:27 -0600 Subject: [PATCH 07/24] Eye tracker: ft-eyes sleeps until the next frame is due ft-eyes looked for new frames about 1,000 times a second: each pass of its loop asked the control socket with a non-blocking recvfrom (a BlockingIOError nearly every time), read both cameras' counters, and slept 1 ms. The frames come every 11.1 ms per camera, and only as counters in ft-eyegrab's shared memory, so there's no fd to wait on. Now each pass ends in select() on the control socket, with a timeout until 2 ms before the next frame of either camera is due (from when its last one was seen), then every 1 ms until it comes. A command wakes it at once. A camera with no frame for 0.1 s (headset off, grabber idle) isn't waited for, and with both stopped it looks every 20 ms. Waiting for the frame grabber's file uses the same select, 0.2 s at a time, instead of sleeping through commands. A new frame is still seen within about 1 ms of when it lands. On a synthetic share at 90 Hz per camera, on a heavily loaded headset (load average 23, so ft-eyes rarely sat idle), its waits went from 178 to 81 a second; unloaded, the old loop's 1 ms sleeps add up to about 1,000. Co-Authored-By: Claude Opus 5.5 --- gaze/tracker/ft-eyes | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/gaze/tracker/ft-eyes b/gaze/tracker/ft-eyes index f716f77..871828e 100755 --- a/gaze/tracker/ft-eyes +++ b/gaze/tracker/ft-eyes @@ -50,6 +50,7 @@ import json import math import mmap import os +import select import socket import struct import sys @@ -87,6 +88,15 @@ GAP = 3.0 # s without frames: the headset was off, and may sit diffe CLICK_BEFORE = 0.3 # a click's frames: the 300 ms before it (like the probe's fixation) CALIB_MIN = 15 # frames an eye needs in a calibration dot's window CALIB_SPREAD = 4.0 # px: more than this and the eye moved during the dot +# Waiting for frames. They come only as a counter in shared memory, so there's nothing to block +# on: sleep until a camera's next frame is due, then look every POLL. ft-eyegrab passes each one +# on within a ms or two of when its camera starts the one after, so they arrive 11.1 ms apart, +# give or take that. +PERIOD = 1 / 90 # s between a camera's frames +EARLY = 0.002 # start looking this long before a frame is due +POLL = 0.001 # then this often until it comes +STALLED = 0.1 # s without a frame: that camera stopped (headset off), and isn't waited for +IDLE_POLL = 0.02 # how often to look while both are stopped class Cams: @@ -357,7 +367,7 @@ def wait_for_cams(sock, tracker, want): return Cams() except (OSError, ValueError, RuntimeError): serve(sock, tracker) - time.sleep(0.2) + select.select([sock], [], [], 0.2) def serve(sock, tracker): @@ -420,6 +430,7 @@ def main(): print("ft-eyes: frames found, tracking", file=sys.stderr, flush=True) seen = [cams.count(0), cams.count(1)] report = time.monotonic() + arrived = [0.0, 0.0] # when each camera's newest frame was seen (monotonic) while True: serve(sock, tracker) want() @@ -429,6 +440,7 @@ def main(): if n == seen[c]: continue seen[c] = n + arrived[c] = time.monotonic() got = cams.frame(c, n - 1) # only the newest: never fall behind if got: eyes[c].feed(*got) @@ -447,8 +459,6 @@ def main(): shifts += [float(v) for v in e.shift.value] pupils += [e.gaze[3], e.gaze[4]] if fresh else [math.nan, math.nan] out.write(latest, (yaw, pitch), flags, per, shifts, pupils) - else: - time.sleep(0.001) now = time.monotonic() if now - report >= 5: if verbose: @@ -466,6 +476,12 @@ def main(): print("ft-eyes: frames went away; waiting", file=sys.stderr, flush=True) cams = wait_for_cams(sock, tracker, want) seen = [cams.count(0), cams.count(1)] + # Until the next frame is due (a command on the socket wakes us sooner). + wait = IDLE_POLL + for c in (0, 1): + if now - arrived[c] < STALLED: + wait = min(wait, max(arrived[c] + PERIOD - EARLY - now, POLL)) + select.select([sock], [], [], wait) if __name__ == "__main__": From d8c2ed0c5806558b5613f7a55cef88aa63655aaa Mon Sep 17 00:00:00 2001 From: DeeJanuz <45082401+DeeJanuz@users.noreply.github.com> Date: Sat, 3 Oct 2026 09:16:46 -0600 Subject: [PATCH 08/24] Screens: frame rates by attention, ticks in step with the display ft-screens gave KWin a frame callback for every committed screen on each tick, and the tick was an 11 ms timer set again after each run, so it slid through the display's frame and came about 85 times a second at 90 Hz: the desktop repeated a frame several times a second (judder in scrolling and video), and KWin drew every screen in one burst at a random point of vrcompositor's frame. Hidden screens got the same 90 Hz unless Frametop was paused for a game. - Ticks run on a timerfd at absolute times, once per display frame, 1 ms after the vsync (IVRSystem::GetTimeSinceLastVsync and the HMD's display frequency, read once a second), so KWin gets its callbacks early in the frame. Measured with --no-vr: 91 wakeups a second instead of about 85. On the Frame the vsync times SteamVR reports lie on a 90 Hz grid. - Each screen's callbacks come at a rate for how much of it you see (vr.cpp, UpdateAttention): every frame while focused (within 12 degrees of where your head points, a laser or the mouse on it in the last 1.5 s, carried, or typed on), 15 a second for the rest of what you see (within 60 degrees), and 1 a second when hidden, behind you, or paused. Levels rise at once and fall after 1.5 s (focused) or 0.5 s (in view). KWin draws a screen only after its callback and its apps wait for theirs, so this throttles the apps too. A screen where nothing changes costs nothing at any rate, as before. - A video in view keeps every frame: 8 commits in a row that each redraw 6% or more of the screen, at 10 a second or more, count as one (from the surface's buffer damage). - "rates F V H" / --rates set the three rates (default 0 15 1, 0 = every frame), "rates?" shows them and each screen's level, "watch S" gives everything full rate for S seconds for a remote viewer (vnc-bridge.sh renews it), and "phase MS" moves the ticks for tuning. - ft-screens' main thread runs at nice -5 after the session starts: SteamOS allows down to -8 once the soft RLIMIT_NICE is raised, and KWin waits on these ticks. It had spent nearly 3 times as long waiting to run as running. Co-Authored-By: Claude Opus 5.5 --- docs/design.md | 2 +- docs/reference.md | 5 +- screens/build.sh | 2 +- screens/compositor.c | 186 +++++++++++++++++++++++++++++++++++++++---- screens/vr.cpp | 89 ++++++++++++++++++++- screens/vr.h | 8 ++ 6 files changed, 272 insertions(+), 20 deletions(-) diff --git a/docs/design.md b/docs/design.md index 4725296..f6f5324 100644 --- a/docs/design.md +++ b/docs/design.md @@ -193,7 +193,7 @@ Staying awake while charging uses Steam's own setting rather than a logind sleep Hiding the screens during a game kept them out of view, but Frametop kept using the headset. Measured on 2026-10-02 with gaze mode off and no game running, in shares of one core: our eye tracker (ft-eyes) about 60%, ft-eyegrab, ft-gaze and ft-gazed about 3 to 4% each; remote desktop (krdpserver, FreeRDP, Xvnc) about 2 cores while it ran; KWin about 13%, ft-screens about 4%. The gaze service ran at full rate whether gaze mode was on or not; now it idles while the gaze isn't used (gaze/README.md), and pausing stops it outright. Reading SteamVR's eye tracking 90 times a second also made it restart every 10 to 13 seconds during Beat Saber, and each restart took input focus from the game, which paused it (PR #13; since then ft-gaze skips SteamVR's gaze action during games, but our own tracker kept running). So pausing stops what costs the most and leaves windows where they are. -- A hidden screen still cost as much as a visible one. ft-screens sent every committed screen its frame callback at 90 Hz whether its overlay showed or not, so KWin kept drawing, and its apps with it. Paused, ft-screens sends the callbacks once a second. A Wayland client draws again only after its last frame's callback, so KWin's output stalls, KWin's own clients stop getting theirs, and the whole desktop idles, without anything losing its connection. A second's pace, rather than none, keeps any client that waits on a callback from waiting forever. Stopping KWin or the apps with SIGSTOP would free the same, but a Wayland peer that stops reading overflows the other side's 4 KB socket buffer, which ends the connection: that's how the live desktop died once when its KWin stalled (`Data too big for buffer`). They also sit in different cgroups (KWin under steam.service when the VR launcher starts it, ft-screens in the dev container's), so no single freeze stops them together. +- A hidden screen still cost as much as a visible one. ft-screens sent every committed screen its frame callback at 90 Hz whether its overlay showed or not (since then, a hidden screen always gets one a second; see the frame rates in reference.md), so KWin kept drawing, and its apps with it. Paused, ft-screens sends the callbacks once a second. A Wayland client draws again only after its last frame's callback, so KWin's output stalls, KWin's own clients stop getting theirs, and the whole desktop idles, without anything losing its connection. A second's pace, rather than none, keeps any client that waits on a callback from waiting forever. Stopping KWin or the apps with SIGSTOP would free the same, but a Wayland peer that stops reading overflows the other side's 4 KB socket buffer, which ends the connection: that's how the live desktop died once when its KWin stalled (`Data too big for buffer`). They also sit in different cgroups (KWin under steam.service when the VR launcher starts it, ft-screens in the dev container's), so no single freeze stops them together. - The relay does the pausing because it's the one part that always runs, and the pointer helper keeps running because stopping it leaves its virtual controller connected with its last pose (the driver has no staleness timeout), maybe holding a hand role, with the 3D mouse dead. Releasing it does the job. The helper already checks for a scene app twice a second, so it's what tells the relay a game started. - The gesture has to work during a game, but SteamVR input reaches only the app with input focus, and an overlay with global input (`steamvr/globalActionSetPriority`) takes the buttons it binds from the game. vrserver's web socket on 127.0.0.1:27062, which its controller binding page uses for the live view, reports every controller component whatever has focus, and reading it takes nothing. The game sees the clicks too, so the default is a gesture games hardly use: both thumbsticks, together, twice. "Together" means within 0.3 seconds of each other, so a stick held down to sprint while the other clicks doesn't count. The stream is about 160 messages a second, nearly all capacitive sensing, so the reader parses only the few that mention a gesture's button. A controller's root path changes while the 3D mouse holds its hand role (`/devices/cv/` instead of `/user/hand/right`), so the reader looks the controllers up again every 3 seconds. - Resuming starts remote desktop through `systemd-run --scope`: started straight from the relay, it would join the relay's cgroup and end with the next relay restart. diff --git a/docs/reference.md b/docs/reference.md index b1273b1..32de70e 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -69,8 +69,11 @@ visibility always|dashboard|gesture|toggle wrist degrees gesture left|righ hide | show | toggle controllers always|outside_games|dashboard ingames hide|visible pause on|off|state conceal N|all reveal N|all concealed cutouts on|off|state cutouts predict on|off cutouts lead ms float N mpp x y w h title unfloat N pose N matrix sub N k x y w h | sub N k off minimized N 0|1 carry N +rates focused in_view hidden rates? watch seconds phase ms ``` +Each screen draws at a frame rate for how much of it you see. KWin draws a screen only after ft-screens gives it a frame callback, and its apps wait for theirs, so the rate of callbacks is the screen's frame rate, for KWin and the apps on it alike. A screen is focused while you look at it (within 12 degrees of where your head points), while a laser or the mouse is on it or was in the last 1.5 seconds, while it's carried, and while you type on it; it gets every display frame. The rest of what you can see (within 60 degrees) gets 15 frames a second, and a hidden screen, one behind you, and everything while paused get one a second. A level goes up at once and comes down after a moment (1.5 s from focused, 0.5 s from in view). A video, or anything moving over a large part of a screen (6% or more of it, redrawn on 8 commits in a row, 10 or more a second), keeps every frame while in view. A floating window is a screen of its own here. Nothing that stands still costs anything at any rate: KWin sends a frame only when something on the screen changed. `rates F V H` sets the three rates in Hz (0: every display frame; default `0 15 1`, also `ft-screens --rates 0,15,1`), and `rates?` shows them, the display's rate, and for each screen its level, its milliseconds between frames, and whether it counts as a video. `watch S` gives every screen full rate for S seconds: remote desktop renews it while a VNC client is connected, since a viewer sees what KWin draws. The ticks (SteamVR events and the callbacks) come once per display frame, 1 ms after the vsync (`phase ms` changes that, for tuning), in step with the display rather than on a timer that drifted through the frame. + `conceal` and `reveal` hide and show one screen on its own (`ft-layout hide` and `show` send them), and `concealed` lists those screens. `pause on` (from the input relay, when Frametop pauses for a VR game) hides every screen and floating window whatever else says, and slows the desktop down; `pause off` undoes it. `cutouts` turns the hand cutouts on and off (`ft-handsctl cutouts`). The last line is ft-floatd's, for floating windows: N is a floating window's panel, numbered on from the screens, one per spare output. `float` gives the window's rectangle in its output, metres per pixel, and the title bar's height, and shows the panel; `unfloat` hides it. `pose` places it (a 3x4 matrix, standing universe), `sub` shows popup or dialog k over it, `minimized` hides it while its window is minimized, and `carry` moves it with the laser that pressed the window's own title bar. ## Input relay @@ -206,7 +209,7 @@ Paused, Frametop leaves the headset's CPU and GPU to a VR game. The input relay - The gaze service stops (`frametop-gaze`: ft-gazed, ft-gaze, our own eye tracker, the gaze panel), so nothing reads SteamVR's eye tracking. Our frame grabber, the root service `ft-eyegrab`, goes idle by itself 3 seconds after our eye tracker stops asking it for frames. - Hand tracking stops if it runs (`frametop-camd`, `frametop-hands`). -- The desktop, as the Game optimization page of Frametop Input Settings says (`pause_desktop`): hidden (the default) or closed. Hidden, ft-screens hides every screen and floating window whatever the visibility mode, the hotkey, or the dashboard says, and gives KWin a frame callback once a second instead of 90 times. KWin draws a screen only after its frame callback, and its apps wait for theirs, so the desktop hardly draws, but its windows stay open. Remote desktop stops if it runs (`session/remote-ctl.sh`). Closed, `desktops.sh stop` closes the desktop and its windows, and resuming starts it again (about 12 seconds), in its start profile if it has one. +- The desktop, as the Game optimization page of Frametop Input Settings says (`pause_desktop`): hidden (the default) or closed. Hidden, ft-screens hides every screen and floating window whatever the visibility mode, the hotkey, or the dashboard says, and gives KWin a frame callback once a second instead of every display frame. KWin draws a screen only after its frame callback, and its apps wait for theirs, so the desktop hardly draws, but its windows stay open. Remote desktop stops if it runs (`session/remote-ctl.sh`). Closed, `desktops.sh stop` closes the desktop and its windows, and resuming starts it again (about 12 seconds), in its start profile if it has one. - The relay lets go of the 3D mouse and feeds pointer devices to its virtual mouse and keyboard, as with `POINTER=0`. Typing goes to Steam. Mapped buttons and key combinations do nothing but pausing, the Steam menu, and commands; a key combination that does nothing is typed as usual. Resuming starts again only what pausing stopped, and plays a second sound. The pointer helper and ft-powerd keep running: they cost little, the helper is what says a game started, and stopping it would leave its virtual controller connected with its last pose. diff --git a/screens/build.sh b/screens/build.sh index bd3cd46..fd626f8 100755 --- a/screens/build.sh +++ b/screens/build.sh @@ -23,6 +23,6 @@ $cxx -c -o build/handcut.o handcut.cpp $cxx -c -o build/handtest.o handtest.cpp vrlibs="$(pkg-config --libs egl glesv2 gbm) -L/opt/steamvr/bin/linuxarm64 -lopenvr_api -Wl,-rpath,/opt/steamvr/bin/linuxarm64" g++ -o build/ft-screens build/compositor.o build/vr.o build/keyboard.o build/handcut.o \ - $(pkg-config --libs wlroots-0.20 wayland-server xkbcommon) $vrlibs + $(pkg-config --libs wlroots-0.20 wayland-server xkbcommon pixman-1) $vrlibs g++ -o build/ft-handtest build/handtest.o build/handcut.o $vrlibs echo "built build/ft-screens build/ft-handtest"' diff --git a/screens/compositor.c b/screens/compositor.c index c635d33..c582903 100644 --- a/screens/compositor.c +++ b/screens/compositor.c @@ -13,7 +13,7 @@ // scale first ("scale ", from ft-layout). // // Usage: ft-screens [--socket NAME] [--control NAME] [--no-vr] [--screen WxH@METRES]... -// [--spares N] [-- COMMAND ARGS...] +// [--spares N] [--rates F,V,H] [-- COMMAND ARGS...] // --socket Wayland socket name in $XDG_RUNTIME_DIR (default ft-screens-0) // --control the control socket's abstract name (default ft_screens) // --no-vr run without SteamVR, for tests next to the running desktop: no panels, no @@ -22,6 +22,8 @@ // --screen one per screen, in KWin's order (default: 3440x1440@2.4) // --spares KWin's outputs after the screens: spares for floating windows (ft-floatd // turns them on and sizes them; see docs/floating-windows.md) +// --rates frame rates in Hz for screens you look at, the rest you see, and hidden ones +// (0: every display frame; default 0,15,1; see frame_interval) // COMMAND run with WAYLAND_DISPLAY set to our socket (e.g. the Frametop session) // Runs in the dev container (wlroots 0.20); KWin connects from the host. #define _GNU_SOURCE @@ -33,7 +35,9 @@ #include #include #include +#include #include +#include #include #include #include @@ -61,6 +65,12 @@ #include "controller-click.h" #define MAX_SCREENS 24 // screens and spare outputs +// A screen counts as playing a video while its last VIDEO_COMMITS commits each redrew at +// least VIDEO_AREA percent of it, at VIDEO_HZ or more; for VIDEO_HOLD ms after that stops. +#define VIDEO_COMMITS 8 +#define VIDEO_AREA 6 +#define VIDEO_HZ 10 +#define VIDEO_HOLD 1500 struct config { int width, height; @@ -80,6 +90,13 @@ struct screen { int buffer_width, buffer_height; struct wlr_xdg_toplevel_decoration_v1 *decoration; // answered on the first commit struct wl_listener commit, destroy, decoration_destroy, set_title; + // Its frame rate (see tick): when its last frame callback went (ms), and what its recent + // commits looked like, to tell a video playing on it. + uint32_t frame_sent; + uint32_t commit_ms[VIDEO_COMMITS]; + unsigned commit_at; + unsigned big; // a bit per recent commit: it redrew at least VIDEO_AREA percent + uint32_t video_until; // counts as a video until then (ms) }; // Per client buffer: forget its import when it goes away. @@ -104,6 +121,14 @@ struct server { int spares; struct wl_list buffers; // tracked_buffer struct wl_event_source *tick; + int tick_fd; // timerfd, at absolute times in step with the display's vsync (see schedule) + int64_t period_ns, base_ns, synced_ns; // the display's frame time, a tick time, last sync + double phase_ms; // ticks come this long after a vsync + // Frame rates in Hz for each attention level (0: every display frame), and a remote + // viewer's lease: until then (ms) every screen gets full rate ("watch"). + int rate[3]; + uint32_t watch_until; + uint32_t typed_ms; // the last key sent to the desktop: its screen counts as focused struct screen *pointer_focus; struct ft_controller_click controller_click; pid_t child; @@ -149,6 +174,25 @@ static void track_buffer(struct server *s, struct wlr_buffer *buffer) { // ---------------------------------------------------------------- screens +// A video (or anything moving over a large area) on a screen you don't look at keeps the +// full frame rate (see frame_interval): its commits keep redrawing much of it, and keep +// coming as fast as its rate lets them. A cursor blinking or a spinner turning redraws a +// small part, and a page changing now and then doesn't keep coming. +static void note_damage(struct screen *sc, struct wlr_surface *surface, struct wlr_buffer *buffer) { + int n = 0; + const pixman_box32_t *r = pixman_region32_rectangles(&surface->buffer_damage, &n); + int64_t area = 0; + for (int i = 0; i < n; ++i) area += (int64_t)(r[i].x2 - r[i].x1) * (r[i].y2 - r[i].y1); + const bool big = area * 100 >= (int64_t)buffer->width * buffer->height * VIDEO_AREA; + const unsigned all = (1u << VIDEO_COMMITS) - 1; + sc->big = ((sc->big << 1) | big) & all; + const uint32_t t = now_ms(); + const uint32_t oldest = sc->commit_ms[sc->commit_at]; // the commit VIDEO_COMMITS ago + sc->commit_ms[sc->commit_at] = t; + sc->commit_at = (sc->commit_at + 1) % VIDEO_COMMITS; + if (sc->big == all && t - oldest <= VIDEO_COMMITS * 1000 / VIDEO_HZ) sc->video_until = t + VIDEO_HOLD; +} + static void screen_commit(struct wl_listener *l, void *data) { struct screen *sc = wl_container_of(l, sc, commit); struct wlr_xdg_surface *xdg = sc->toplevel->base; @@ -172,6 +216,7 @@ static void screen_commit(struct wl_listener *l, void *data) { if (!buffer) return; sc->frame_pending = true; ++sc->commits; + note_damage(sc, xdg->surface, buffer); sc->buffer_width = buffer->width, sc->buffer_height = buffer->height; if (buffer == sc->held) return; struct wlr_dmabuf_attributes a; @@ -381,12 +426,64 @@ static void keys_update(struct server *s) { offsetof(struct sockaddr_un, sun_path) + 1 + sizeof name - 1); } -// Every ~11 ms (90 Hz): SteamVR events, and frame callbacks for screens that committed. While -// Frametop is paused for a VR game (everything hidden), every 100 ms, and the frame callbacks -// once a second: KWin draws a screen only after its callback, and its apps wait for theirs, so -// the desktop hardly draws until it's resumed. -static int tick(void *data) { +// How often a screen gets its frame callback (ms; 0 every tick). KWin draws a screen only +// after its callback, and its apps wait for theirs, so this is the screen's frame rate: +// full where you look (ft_vr_screen_attention) and for a video, lower for the rest of what +// you see, and about once a second for what you don't (hidden, behind you, paused for a VR +// game). A screen nothing changes on costs nothing at any rate: KWin commits only when +// something on it moved. A remote viewer ("watch") sees every screen at full rate. +static uint32_t frame_interval(struct server *s, struct screen *sc, uint32_t t) { + if ((int32_t)(s->watch_until - t) > 0) return 0; + enum ft_attention a = ft_vr_screen_attention(sc->index); + if (a == FT_IN_VIEW && (int32_t)(sc->video_until - t) > 0) a = FT_FOCUSED; + if (a != FT_FOCUSED && t - s->typed_ms < 1500 && + s->seat->keyboard_state.focused_surface == sc->toplevel->base->surface) + a = FT_FOCUSED; // typing on it while looking elsewhere + const int hz = s->rate[a]; + return hz > 0 ? 1000 / hz : 0; +} + +static int64_t mono_ns(void) { + struct timespec ts; + clock_gettime(CLOCK_MONOTONIC, &ts); + return ts.tv_sec * 1000000000LL + ts.tv_nsec; +} + +// Ticks come once per display frame, phase_ms after its vsync: KWin gets its frame callbacks +// early in the frame and has the rest of it to draw before vrcompositor takes the panels. +// The vsync is read from SteamVR once a second, and the ticks run at absolute times between +// reads, so they keep their place (a timer set again after each tick, as before, slid +// through the frame and came about 85 times a second at 90 Hz). Without SteamVR's timing +// they still come at the display's rate (90 Hz until known). While paused, every 100 ms. +static void schedule(struct server *s) { + const int64_t now = mono_ns(); + int64_t next; + if (ft_vr_paused()) { + next = now + 100000000LL; + } else { + double since, hz; + if (now - s->synced_ns >= 1000000000LL) { + s->synced_ns = now; + if (ft_vr_vsync(&since, &hz)) { + const int64_t period = (int64_t)(1e9 / hz); + if (llabs(period - s->period_ns) > 100000) wlr_log(WLR_INFO, "display at %.1f Hz", hz); + s->period_ns = period; + s->base_ns = now - (int64_t)(since * 1e9) + (int64_t)(s->phase_ms * 1e6); + while (s->base_ns > now) s->base_ns -= s->period_ns; + } + } + next = s->base_ns + ((now - s->base_ns) / s->period_ns + 1) * s->period_ns; + } + struct itimerspec its = {.it_value = {.tv_sec = next / 1000000000LL, .tv_nsec = next % 1000000000LL}}; + timerfd_settime(s->tick_fd, TFD_TIMER_ABSTIME, &its, NULL); +} + +// Each tick: SteamVR events, and frame callbacks for screens that committed and are due. +static int tick(int fd, uint32_t mask, void *data) { struct server *s = data; + uint64_t expirations; + const ssize_t got = read(fd, &expirations, sizeof expirations); // clears it; how many doesn't matter + (void)got; ft_vr_poll(handle_vr_event, s); if (++s->ticks % 9 == 0) keys_update(s); if (s->kb_close_at && s->ticks >= s->kb_close_at) { @@ -398,15 +495,17 @@ static int tick(void *data) { } struct timespec now; clock_gettime(CLOCK_MONOTONIC, &now); - const bool paused = ft_vr_paused(); - for (int i = 0; i < MAX_SCREENS && (!paused || s->ticks % 10 == 0); ++i) { + const uint32_t t = now_ms(); + // Due within half a tick counts as due, so 15 Hz is every 6th tick at 90 Hz, not 7th. + const uint32_t slack = ft_vr_paused() ? 50 : (uint32_t)(s->period_ns / 2000000); + for (int i = 0; i < MAX_SCREENS; ++i) { struct screen *sc = s->screens[i]; - if (sc && sc->frame_pending) { - sc->frame_pending = false; - wlr_surface_send_frame_done(sc->toplevel->base->surface, &now); - } + if (!sc || !sc->frame_pending || t - sc->frame_sent + slack < frame_interval(s, sc, t)) continue; + sc->frame_pending = false; + sc->frame_sent = t; + wlr_surface_send_frame_done(sc->toplevel->base->surface, &now); } - wl_event_source_timer_update(s->tick, paused ? 100 : 11); + schedule(s); return 0; } @@ -431,6 +530,7 @@ static bool key_held(const struct wlr_keyboard *kb, uint32_t code) { // One key to the focused screen, through the seat's keyboard so its xkb state and // modifiers stay right. static void send_key(struct server *s, uint32_t code, int pressed) { + s->typed_ms = now_ms(); struct wlr_keyboard_key_event ev = { .time_msec = now_ms(), .keycode = code, .update_state = true, .state = pressed ? WL_KEYBOARD_KEY_STATE_PRESSED : WL_KEYBOARD_KEY_STATE_RELEASED}; @@ -584,6 +684,38 @@ static int control_readable(int fd, uint32_t mask, void *data) { handle_vr_event(&e, s); snprintf(reply, sizeof reply, "ok"); } + } else if (sscanf(buf, "watch %d", &value) == 1) { + // A remote viewer is watching for that many seconds (vnc-bridge.sh renews it). + s->watch_until = now_ms() + (uint32_t)(value < 0 ? 0 : value > 60 ? 60 : value) * 1000; + snprintf(reply, sizeof reply, "ok"); + } else if (sscanf(buf, "rates %d %d %d", &index, &w, &h) == 3) { + // Frame rates in Hz: focused, in view, hidden (0: every display frame). + if (index < 0 || w < 0 || h < 0 || index > 240 || w > 240 || h > 240) { + snprintf(reply, sizeof reply, "error rates (Hz, 0 full)"); + } else { + s->rate[FT_FOCUSED] = index, s->rate[FT_IN_VIEW] = w, s->rate[FT_HIDDEN] = h; + wlr_log(WLR_INFO, "frame rates: focused %d, in view %d, hidden %d (0 full)", index, w, h); + snprintf(reply, sizeof reply, "ok"); + } + } else if (sscanf(buf, "phase %lf", &scale) == 1) { + // Ticks this long after the vsync (ms), for tuning. + if (!(scale >= 0 && scale < 20)) snprintf(reply, sizeof reply, "error phase "); + else s->phase_ms = scale, s->synced_ns = 0, snprintf(reply, sizeof reply, "ok"); + } else if (strcmp(buf, "rates?") == 0) { + // "ok " and a line per + // screen: " hidden|view|focused