mirror of
https://github.com/saphid/frame-control.git
synced 2026-10-06 04:04:21 +02:00
Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 saphidandClaude Opus 5.5
|
c46bf68dd0 | ||
|
Alex SouthwellandClaude Opus 5.5
|
e702adbd77 |
No files matched your search
@@ -35,8 +35,8 @@ See what the headset sees, install games and Android apps, move files and text a
|
||||
<tr>
|
||||
<td width="50%" valign="top">
|
||||
|
||||
**👓 Headset view**<br>
|
||||
Live video of what the lenses show (about 30 fps), or a still of both eyes. Zoom, pan, full screen, save as PNG.
|
||||
**👓 Headset view and Desktop**<br>
|
||||
Live video of what the lenses show, or of the app panel in use, flat and still however the wearer looks around. Turn on Control and tap or click right on it to use the Frame from your phone or computer.
|
||||
|
||||
</td>
|
||||
<td width="50%" valign="top">
|
||||
@@ -93,11 +93,7 @@ SSH, SFTP, Steam Link, remote desktop, volume, sleep, restart and shut down.
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
The optional [Family and comfort](docs/family-comfort.md) card adds session
|
||||
limits, breaks, local alerts and one-click casting. A session copies a small
|
||||
Frame Control worker into your headset user account.
|
||||
|
||||
For the other features, nothing is installed on the Frame: the app uses what SteamOS
|
||||
Nothing is installed on the Frame for any of this: the app uses what SteamOS
|
||||
already ships (sideloading a game copies Valve's own devkit scripts to
|
||||
`~/devkit-utils`, as Valve's Devkit Client does). [How each feature works](docs/frame-control.md).
|
||||
|
||||
|
||||
+2
-17
@@ -1,7 +1,7 @@
|
||||
// Frame Control as a desktop app (macOS, Windows, Linux): starts ui/server.py on
|
||||
// a free loopback port and shows it in a native window. The server does all the
|
||||
// work over the `frame` SSH alias; this file only hosts it.
|
||||
const { app, BrowserWindow, Menu, Notification, clipboard, dialog, ipcMain, shell } = require("electron");
|
||||
const { app, BrowserWindow, Menu, clipboard, dialog, ipcMain, shell } = require("electron");
|
||||
const { execFile, spawn } = require("child_process");
|
||||
const { promisify } = require("util");
|
||||
const fs = require("fs");
|
||||
@@ -254,21 +254,6 @@ ipcMain.handle("update:get", (e) => fromUi(e) ? publicUpdate() : null);
|
||||
ipcMain.handle("update:check", (e) => fromUi(e) ? checkForUpdate({ manual: true }).then(publicUpdate) : null);
|
||||
ipcMain.handle("update:install", (e) => { if (fromUi(e)) installUpdate(); });
|
||||
|
||||
ipcMain.handle("comfort:notify", (e, message) => {
|
||||
if (!fromUi(e) || typeof message !== "string" || message.length > 500) throw new Error("Invalid notification");
|
||||
if (!Notification.isSupported()) throw new Error("System notifications are unavailable");
|
||||
return new Promise((resolve, reject) => {
|
||||
const notification = new Notification({title: "Frame Control", body: message});
|
||||
const timer = setTimeout(() => reject(new Error("Notification delivery was not confirmed. Check system notification settings.")), 5000);
|
||||
notification.once("show", () => { clearTimeout(timer); resolve(true); });
|
||||
notification.once("failed", (_event, error) => {
|
||||
clearTimeout(timer);
|
||||
reject(new Error("Notification delivery failed. Check system notification settings: " + error));
|
||||
});
|
||||
notification.show();
|
||||
});
|
||||
});
|
||||
|
||||
// frame-control://install links from websites (docs/web-install.md). They can
|
||||
// arrive before the window or server exists (macOS open-url on a cold launch),
|
||||
// so they wait here until the page asks for them. The page checks the link with
|
||||
@@ -391,7 +376,7 @@ function createWindow() {
|
||||
title: "Frame Control", backgroundColor: BG, show: false,
|
||||
...(IS_MAC ? { titleBarStyle: "hiddenInset", trafficLightPosition: { x: 18, y: 26 } }
|
||||
: { icon: path.join(__dirname, "build", "icon.png") }),
|
||||
webPreferences: { contextIsolation: true, nodeIntegration: false, sandbox: true, backgroundThrottling: false,
|
||||
webPreferences: { contextIsolation: true, nodeIntegration: false, sandbox: true,
|
||||
preload: path.join(__dirname, "preload.js") },
|
||||
});
|
||||
win.once("ready-to-show", () => win.show());
|
||||
|
||||
@@ -9,7 +9,6 @@
|
||||
const { contextBridge, ipcRenderer, webUtils } = require("electron");
|
||||
|
||||
contextBridge.exposeInMainWorld("frameApp", {
|
||||
notify: (message, request) => ipcRenderer.invoke("comfort:notify", message, request),
|
||||
readClipboard: () => ipcRenderer.invoke("clipboard:read"),
|
||||
setUpConnection: () => ipcRenderer.invoke("connection:setup"),
|
||||
// While the keyboard-and-trackpad panel holds the keyboard, ⌘W, ⌘R and the rest go to the Frame.
|
||||
|
||||
@@ -168,6 +168,43 @@ on this branch (run 36421345682).
|
||||
**Verified on the same Frame/build:** both Ctrl-C and SIGTERM close the dedicated
|
||||
browser profile and SSH tunnel and remove the profile and panel log.
|
||||
|
||||
**Verified end to end on the same Frame/build (2026-09-29), with mutations:**
|
||||
a stdio MCP client started `ui/frame_mcp.py` in its default mode (private
|
||||
backend, no API key, no prestarted server) and a human approved or rejected
|
||||
each change in the approval page in a real Chrome window:
|
||||
|
||||
| Tool | Result on the Frame |
|
||||
|---|---|
|
||||
| `send_file` | Approved; the file arrived in `~/Downloads` with identical contents |
|
||||
| `install` | Approved; `io.github.fizzyizzy05.binary` job finished in about 35 s |
|
||||
| `panel` | Approved; gamescope listed a new panel window, and `computer_state` reported the same window ID and PID. The app rendered in that window (below) |
|
||||
| `launch` | Approved; Keep Talking and Nobody Explodes (341800) started under Proton and `computer_state` reported it as the focused app |
|
||||
| `uninstall` | Approved; app and locale removed |
|
||||
| `power` | Rejected in the page. Unapproved retries, the same token used for `uninstall`, and a retry after rejection were all refused. Nothing was powered off |
|
||||
| `send_text` | Approved, then refused because the Plasma desktop was not open (documented requirement) |
|
||||
| `keep_awake` | `status` reports the script unavailable until PR #16 lands |
|
||||
|
||||
A separate Claude Code CLI session, with only this server configured, read
|
||||
status, `computer_state` and a headset capture, and requested an install. It
|
||||
received an approval URL and did not execute anything.
|
||||
|
||||
The assistant opened as a Frame panel through `scripts/assistant-on-frame.py`.
|
||||
Against a loopback stub model, a send without consent made zero requests. With
|
||||
consent it made exactly one, carrying the text and a fresh Frame screenshot.
|
||||
Consent unticked itself after sending. SIGTERM removed the panel, profile, log
|
||||
and tunnel.
|
||||
|
||||
**Not verified while unworn:** every headset capture was a uniform dark frame,
|
||||
so SteamVR's rendered view of panels and the game could not be checked; window
|
||||
captures (`xwd`) were used instead. MCP can launch a game or panel but has no
|
||||
tool to stop one: the tester stopped them over SSH. Removing an app leaves any
|
||||
runtime it pulled in; Flatpak may also remove related extensions when that
|
||||
runtime is removed by hand.
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
## Computer-use coverage
|
||||
|
||||
@@ -1,70 +0,0 @@
|
||||
{
|
||||
"tested_head": "be1ab129c9b8663be8d4cd9d5aea0a4c587a16ba",
|
||||
"date": "2026-09-29",
|
||||
"steam_os_build": "20260925.6191901",
|
||||
"battery_percent_start": 100,
|
||||
"battery_percent_end": 100,
|
||||
"activity_level": 3,
|
||||
"warning_to_home_seconds": 60.76079535484314,
|
||||
"after": {
|
||||
"path": "/routes/library/home",
|
||||
"visible": true,
|
||||
"apps": []
|
||||
},
|
||||
"session": {
|
||||
"id": "2f03a8c2eb88482ca28b10246ace496b",
|
||||
"boot": "6c5ddc81-747f-44e5-b8e0-00e0f13fd08e",
|
||||
"active": false,
|
||||
"options": {
|
||||
"minutes": 1,
|
||||
"breakMinutes": 1,
|
||||
"stillMinutes": 1,
|
||||
"batteryAlert": true,
|
||||
"heatAlert": true
|
||||
},
|
||||
"started": 81160.254818568,
|
||||
"deadline": 81220.254818568,
|
||||
"lastSample": 81230.770869338,
|
||||
"used": 0,
|
||||
"nextBreak": 60,
|
||||
"stillSent": false,
|
||||
"warned": 81170.389722976,
|
||||
"events": [
|
||||
{
|
||||
"id": "2f03a8c2eb88482ca28b10246ace496b:1",
|
||||
"kind": "started",
|
||||
"message": "Session started. Steam Home opens at the limit; games are not closed.",
|
||||
"time": 1790676762.9003663
|
||||
},
|
||||
{
|
||||
"id": "2f03a8c2eb88482ca28b10246ace496b:2",
|
||||
"kind": "warning",
|
||||
"message": "One minute left. Save your progress; Steam Home will open.",
|
||||
"time": 1790676773.0352101
|
||||
},
|
||||
{
|
||||
"id": "2f03a8c2eb88482ca28b10246ace496b:3",
|
||||
"kind": "finished",
|
||||
"message": "Session ended: Steam Home opened. Your game is still running.",
|
||||
"time": 1790676833.7960055
|
||||
}
|
||||
],
|
||||
"seq": 3,
|
||||
"latched": [],
|
||||
"error": null,
|
||||
"time": 1790676838.9175165,
|
||||
"remaining": 0,
|
||||
"heartbeat": 81230.770869338,
|
||||
"unavailable": [],
|
||||
"activity": 3
|
||||
},
|
||||
"cleanup": {
|
||||
"prior_page_restored": true,
|
||||
"prior_dashboard_visibility_restored": true,
|
||||
"worker_exited": true,
|
||||
"temporary_simulator_key_removed": true,
|
||||
"phone_server_exited": true,
|
||||
"lock_released": true,
|
||||
"owned_simulator_shutdown": true
|
||||
}
|
||||
}
|
||||
@@ -1,30 +0,0 @@
|
||||
# Family and comfort: locked real-Frame verification
|
||||
|
||||
**Verified 2026-09-29**, tested feature source at `be1ab12`, SteamOS 0.4.1 build
|
||||
`20260925.6191901`, iOS 26.5 Simulator. The test acquired
|
||||
`/tmp/frame-test.lock` before deploying or launching anything.
|
||||
|
||||
The Simulator installed the bundled server in the Frame user account, connected
|
||||
over its SSH tunnel, requested notification permission, displayed a native test
|
||||
banner, and started a one-minute session through the shared UI API. The real
|
||||
session warning reached the Simulator as a native banner. Steam Home opened
|
||||
**60.76 seconds after the successful warning**. The running-app list was empty
|
||||
before and after; this run does not prove preservation of a running game.
|
||||
|
||||
|
||||
|
||||
[Recorded session and cleanup result](comfort-session-20260929.json).
|
||||
Local raw logs and headset/Simulator captures are retained at
|
||||
`/tmp/frame-comfort-evidence/locked-run/` on the verification Mac.
|
||||
|
||||
**Verified cleanup:** previous page and hidden-dashboard state restored; comfort
|
||||
worker and phone server exited; temporary Simulator SSH key removed; test lock
|
||||
released; only the owned Simulator shut down. Battery stayed at 100%. No global
|
||||
settings, power state, Steam or SteamVR lifecycle changes were made.
|
||||
|
||||
**Unverified while unworn:** SteamVR activity was 3 (standby), so active-use
|
||||
break/check-in clocks stayed at zero. Headset captures returned blank standby
|
||||
frames. The casting shortcut was invoked, but this run does not establish
|
||||
visible output or frame rate. No low-battery or overheating condition was
|
||||
induced. Earlier worn/active-device results remain separately documented in
|
||||
[Family and comfort](../family-comfort.md).
|
||||
@@ -1,122 +0,0 @@
|
||||
# Family and comfort
|
||||
|
||||
Frame Control's Home tab has a **Family and comfort** card, on desktop and
|
||||
on iPhone. No third-party notification or parental-control app is needed.
|
||||
This is Frame Control code using Python, Steam and SteamVR already on the Frame.
|
||||
|
||||
|
||||
|
||||
## Sessions
|
||||
|
||||
Set a limit of 1–240 minutes, optional break and check-in intervals, then
|
||||
**Start session**. Break and check-in intervals of 0 turn those reminders off.
|
||||
**Cancel session** cancels the timer and monitoring without changing the game.
|
||||
Cancel before starting a session with different settings.
|
||||
|
||||
The Frame shows a one-minute warning, then opens Steam Home in its dashboard.
|
||||
**Games stay running**: save and pause before the limit. Some games pause when
|
||||
the dashboard opens; others do not. There is no kill, power-off, Steam restart,
|
||||
account restriction or parental lock. The wearer can return to the game.
|
||||
|
||||
**Documented implementation:** the timer is a single, opt-in Python worker in
|
||||
the Frame user's account. Desktop and iPhone share its state. It keeps going
|
||||
when the companion disconnects, closes or is suspended. It exits after
|
||||
completion or cancellation (normally within five seconds). Cancellation waits
|
||||
for any in-flight SteamVR action to finish within its timeout; it is not a boot
|
||||
service. A Frame reboot invalidates the session. Suspend counts toward the
|
||||
limit, using Linux's boot-time clock. If a warning was delayed by suspend or a
|
||||
SteamVR failure, Home waits until at least a full minute after a successful
|
||||
warning. A failed Home transition remains active and retries, with an error
|
||||
shown in the companion. A stale worker is reported as unverified enforcement.
|
||||
|
||||
## Alerts and breaks
|
||||
|
||||
During a session, battery, overheating and check-in alerts go to connected
|
||||
companions. Break reminders and session warnings also appear on the headset.
|
||||
|
||||
- **Low battery:** 15% or below while discharging. One alert until charging or
|
||||
recovery to 20%, so values around 15% do not produce repeated notifications.
|
||||
- **Overheating:** a thermal zone reaches its own kernel-reported hot/critical
|
||||
trip, or the battery reports `Overheat`. Missing sensors mean unknown, not
|
||||
safe. These are status alerts, not medical advice or an extra thermal governor.
|
||||
- **Check in:** an alert after the chosen number of active minutes.
|
||||
- **Breaks:** a SteamVR reminder and companion notification at the chosen interval.
|
||||
|
||||
**Inferred:** SteamVR activity levels 1 and 2 are a useful proxy for use, not
|
||||
proof someone is wearing the headset. Inactive readings reset continuous use;
|
||||
missing readings add no time. Long gaps count at most 30 seconds. Breaks and
|
||||
check-ins are distinct from the elapsed-time session limit.
|
||||
|
||||
Click **Enable / test notifications** on each companion. iOS asks for permission;
|
||||
macOS, Windows and Linux follow their notification settings. The page also shows
|
||||
recent events and errors. Keep Frame Control open and connected for companion
|
||||
alerts. **Phone alerts are local, not push notifications:** iOS suspension,
|
||||
force-quit or a lost SSH connection prevents live delivery. Old alerts are not
|
||||
replayed as a notification burst on reconnect. Headset warnings and the session
|
||||
limit continue without the phone. A physical iPhone's background delivery has
|
||||
not been verified and is not guaranteed.
|
||||
|
||||
## Casting
|
||||
|
||||
**Cast headset view** starts the existing headset Live view and requests full
|
||||
screen where supported. Show that screen to people in the room, or use the
|
||||
computer/phone's own screen mirroring. It creates no new stream transport,
|
||||
public URL or LAN server. iPhone uses the inline viewer if full screen is not
|
||||
available. The image includes private content visible to the wearer.
|
||||
|
||||
## What is installed
|
||||
|
||||
The shared authenticated `/api/comfort` endpoint copies three bundled Python
|
||||
files to `~/.cache/frame-control/comfort/<content-hash>/`. Session state and
|
||||
locks live in `~/.local/state/frame-control/comfort/`, with a private directory
|
||||
and 0600 state file. There is no network listener or system service. Cancel a
|
||||
session before removing these directories. The iPhone's normal server still
|
||||
exits on disconnect; the explicitly started comfort worker is the exception.
|
||||
|
||||
## Verification
|
||||
|
||||
**Verified 2026-09-28**, SteamOS 0.4.1, build `20260925.6191901`: shipped
|
||||
`/opt/steamvr/bin/linuxarm64/vrcmd --notify TEXT` reported success for a custom
|
||||
reminder. Steam's CDP `SteamUIStore.Navigate('/library/home')` and
|
||||
`SteamClient.OpenVR.VROverlay.ShowDashboard('valve.steam.gamepadui.main')`
|
||||
opened Home while the running app ID stayed unchanged. Prior page and dashboard
|
||||
visibility were restored. Kernel hot/critical trips and SteamVR activity were
|
||||
read from the real device. No temperature or battery fault was induced.
|
||||
|
||||
**Verified locally:** deterministic fake-Frame tests cover late warnings,
|
||||
failed warnings/Home actions, cancellation, activity gaps, thresholds, duplicate
|
||||
suppression, reboot invalidation, shared session state and the exact Home
|
||||
JavaScript. `python3 -m unittest discover -s tests` runs them. The iOS Simulator
|
||||
build tests notification content and bounds. Physical iPhone delivery and
|
||||
wearer-perceived headset notification visibility remain unverified.
|
||||
|
||||
**Verified end to end on the same Frame:** a two-minute session with no companion
|
||||
connection for 135 seconds emitted its warning, break and check-in, then opened
|
||||
Home. The running app ID was unchanged; the test restored the previous page and
|
||||
dashboard visibility and confirmed the worker exited. Casting through the Home
|
||||
shortcut decoded the existing headset stream at 30 fps.
|
||||
|
||||
**Verified on the iOS 26.5 Simulator:** connected to the real Frame, approved the
|
||||
notification prompt, and saw the native Frame Control test banner. Seven iOS
|
||||
tests passed.
|
||||
|
||||
|
||||
|
||||
Desktop and 390-pixel phone layouts had no horizontal overflow.
|
||||
On macOS the development Electron app's real notification attempt was denied
|
||||
(`UNErrorDomain` 1); the bridge now returns that failure instead of reporting
|
||||
success. Successful macOS/Windows/Linux notification display remains unverified.
|
||||
|
||||
**Verified on the real Frame:** its naturally discharging 15% battery produced
|
||||
one low-battery event during a short session; the test then cancelled the
|
||||
session. Overheating alerts use fake sensor samples in tests: the shared
|
||||
headset was not deliberately overheated.
|
||||
|
||||
**Verified 2026-09-29 on the same Frame:** a fresh one-minute session opened
|
||||
Home more than 60 seconds after the successful warning. The test restored the
|
||||
previous page and dashboard visibility. Local regression coverage now includes
|
||||
slow notification delivery, a total Home-action timeout, failed worker startup,
|
||||
unreadable saved state, malformed activity samples and notification UX: 173
|
||||
Python tests passed. Desktop and 390-pixel layouts were checked again; system
|
||||
notification-denial guidance stayed visible across polls. Initial event history
|
||||
did not replay notifications, and only the latest new event was announced.
|
||||
@@ -40,6 +40,8 @@ Lepton (Android 11, podman container "lepton-dev") ← its own panel, app 305600
|
||||
| Present: `rsync`, `flatpak`, `python3`, `git`, `qdbus6`, `xrdp`, `xprop`, `xwininfo`, `xterm`, `konsole`, `dolphin`, `gamescopectl`. Missing: `wl-copy`, `xclip`, `xsel`, `kdeconnect-cli`, `tailscale` (installable in `~`, see below), `krfb`, `wayvnc`. | Script design |
|
||||
| **SteamOS updates arrive on their own.** The Frame went from 0.3.0 (build 20260922.6101926) to **0.4.1, build 20260925.6191901**, between 2026-09-27 and 2026-09-28 with no action from us; `~` (keys, user Flatpaks, `~/.local/share`) survived. **Verified 2026-09-28.** | Keep changes in `~` |
|
||||
| **Valve's package repository has more than the image.** `pacman -Si` / `pacman -Sp` work as `steamos` without root and list Valve's own builds, such as `kdeconnect` 24.02.2 and `python-evdev` 1.7.0 in `extra`. Unpacking those packages into `~` runs them without touching the read-only root. The repository URLs say not to share them, so never write them down; Valve also publishes each build's source package there (`sources/packages/`), which is how Frame Control got the complete source for the KDE Connect it ships. **Verified 2026-09-28**, SteamOS 0.4.1. | [streaming.md](streaming.md#input-type-and-point-in-the-frame-from-the-mac-or-iphone) |
|
||||
| **gamescope has its own input injection.** An EIS socket at `/run/user/1000/gamescope-0-ei` (libei 1.4.1 is on the image) offers "Gamescope Virtual Input": relative and absolute pointer, buttons, scroll, keyboard. It drives the panel that has focus in the headset, on either X display. Focus moves only with the controller's laser (or to a new panel when none has it); `gamescopectl focus_info` prints the focus state to the journal. **Verified 2026-09-29.** | [streaming.md](streaming.md#live-view-and-control-watch-a-panel-and-tap-on-it) |
|
||||
| **A panel's own pixels:** `ffmpeg -f x11grab -window_id <window> -i :<display>` captures one window (x11grab of the root is black under gamescope). Panels live on `:0` (Steam's UI, windows tagged by `panel-on-frame.sh`) or `:1` (apps Steam starts). **Verified 2026-09-29.** | Frame Control's Desktop view |
|
||||
| **gamescope runs two Xwayland displays.** `:0` holds Steam's VR bar and menus (`valve.steam.gamepadui.*`) and ignores XTest pointer motion; `:1` holds apps such as Chromium and takes it. There's also a libei socket, `/run/user/1000/gamescope-0-ei`. **Verified 2026-09-28**, SteamOS 0.4.1. | Keyboard and trackpad |
|
||||
| Flathub is a **system** remote. `--user` installs over SSH work and show up in the desktop menu. | `install-apps.sh` |
|
||||
| `/` is 10 GB and read-only. `/home` is 929 GB. | Where to put things |
|
||||
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 192 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 1.0 MiB |
Binary file not shown.
|
Before Width: | Height: | Size: 71 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 81 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 20 KiB |
+1
-12
@@ -26,10 +26,7 @@ as its transport too), so the desktop and phone share one code path. Android
|
||||
display settings use `podman exec` into each Lepton container instead of adb,
|
||||
which the Frame doesn't have.
|
||||
|
||||
The app server stops after the phone disconnects. An explicitly started
|
||||
[comfort session](family-comfort.md) keeps its timer and headset reminders running
|
||||
until the session ends or is cancelled; phone notifications require the app to
|
||||
remain connected and running. The copied
|
||||
Nothing is left running on the Frame after the phone disconnects; the copied
|
||||
files stay in `~/.cache/frame-control` (delete it any time).
|
||||
|
||||
## Pairing
|
||||
@@ -111,11 +108,3 @@ running), a real sleep/restart/shut down on the Frame, and a physical iPhone.
|
||||
Debug builds have Simulator test hooks (`FRAME_TEST_HOST`, `FRAME_TEST_PAGE`,
|
||||
`FRAME_TEST_JS`, and the tunnel URL in the app's Caches folder); release builds
|
||||
don't.
|
||||
|
||||
## Family and comfort
|
||||
|
||||
The shared Home card sets session limits, breaks and check-ins, and offers
|
||||
**Cast headset view**. **Enable / test notifications** requests iOS notification
|
||||
permission and sends a local test. These are local notifications, not APNs push;
|
||||
iOS background suspension can interrupt phone alerts. The headset timer still
|
||||
runs. See [the behavior and verification limits](family-comfort.md).
|
||||
+62
-2
@@ -8,6 +8,7 @@ This covers three directions, plus input:
|
||||
- **PC VR from Linux**: [feasibility and options](linux-vr-streaming.md),
|
||||
including Valve's streaming and USB support. No Linux host tested yet.
|
||||
- **Input**: type and point in the Frame from the Mac or iPhone.
|
||||
- **Live view and Control**: watch a panel flat and tap on it to use it.
|
||||
|
||||
The confidence labels are the same as in [ssh.md](ssh.md).
|
||||
|
||||
@@ -184,13 +185,72 @@ on the Frame, which talks KDE Connect's own LAN protocol to the Frame's
|
||||
and ignores injected pointer motion; `:1` holds apps such as Chromium and
|
||||
takes it. KDE Connect runs on `:1`, so it reaches apps, not Steam's own menus.
|
||||
There's also a `gamescope-0-ei` (libei) socket.
|
||||
- **Not yet tested:** typing and clicking as seen in the headset, and whether
|
||||
it reaches the KDE desktop panel (Plasma is its own session).
|
||||
- Typing through KDE Connect lands in a Chromium panel on `:1` (seen in the
|
||||
panel's own capture, 2026-09-29). It **can't reach panels on `:0`** (Frame
|
||||
Control's own panels, Steam's UI) and, since XTest positions are clamped to
|
||||
`:1`'s 1280×720 root, can't reach beyond that in a bigger window. Control on
|
||||
the live view (below) has neither limit.
|
||||
- **Not yet tested:** whether it reaches the KDE desktop panel (Plasma is its
|
||||
own session).
|
||||
- **Known limit:** keys and clicks typed while the link is reconnecting wait
|
||||
and are sent once it's back, but anything sent in the moment the Wi-Fi
|
||||
drops, before SSH notices, can be lost. Confirming every event would add a
|
||||
round trip to each pointer move.
|
||||
|
||||
## Live view and Control: watch a panel and tap on it
|
||||
|
||||
**Built: Home → Desktop / Headset view → Control.** The live view has two
|
||||
sources:
|
||||
|
||||
- **Headset view**: what the lenses show (SteamVR's mirror, `/dev/video99`). It
|
||||
moves with the wearer's head, so Control makes the view a trackpad: drag to
|
||||
move the pointer, tap to click, press and hold to right-click, two fingers to
|
||||
scroll. With a mouse, moving over the view moves the pointer.
|
||||
- **Desktop**: the app panel in use in the headset, from its own window, so it
|
||||
stays still however the wearer looks around. Control makes taps and clicks
|
||||
land exactly where you put them. Dragging is a mouse drag, press and hold is a
|
||||
right-click, two fingers scroll, and on a computer the mouse, wheel and
|
||||
keyboard work directly on it (⌘ is sent as Ctrl on a Mac). A picker shows any
|
||||
other panel, view only.
|
||||
|
||||
Below the view, a text field and key buttons type on the Frame from a phone.
|
||||
|
||||
How (**verified 2026-09-29**, SteamOS 0.4.1, build 20260925.6191901):
|
||||
|
||||
- **Input goes through gamescope's own injection.** gamescope serves an EIS
|
||||
socket (`/run/user/1000/gamescope-0-ei`; Steam feeds Remote Play input through
|
||||
it), and `libei` 1.4.1 is on the image. [`ui/frame_touch.py`](../ui/frame_touch.py)
|
||||
talks to it with `ctypes`: nothing to install. gamescope offers one device,
|
||||
"Gamescope Virtual Input", with relative and absolute pointer, buttons,
|
||||
scroll and keyboard (Linux key codes; no text capability, so the text field
|
||||
types printable ASCII on a US layout). Its absolute region is unbounded; the
|
||||
pointer uses the focused panel's display coordinates, and gamescope fits each
|
||||
window to its display, so a 1920×1080 window on the 1280×720 `:1` takes
|
||||
positions at two thirds scale. Taps on a 1280×720 page landed on the exact
|
||||
pixel.
|
||||
- **It reaches the panel that has focus** (`GAMESCOPE_FOCUSED_WINDOW` on `:0`'s
|
||||
root), on either X display. In the OpenVR backend focus moves only on SteamVR
|
||||
overlay events (the controller's laser entering or clicking a panel), or to a
|
||||
new panel when none holds it (read from gamescope's `OpenVRBackend.cpp`, seen
|
||||
with `gamescopectl focus_info`, which writes to the journal). Neither
|
||||
`GAMESCOPECTRL_BASELAYER_WINDOW`/`_APPID` nor X focus moves it, and no
|
||||
gamescope command does. So Control follows the wearer: whatever they last
|
||||
used is what your taps reach. A window without a Steam app id (`STEAM_GAME`)
|
||||
gets a connector of its own and doesn't hold focus.
|
||||
- **Keys in a burst can arrive out of order**, so the helper paces them (8 ms
|
||||
apart).
|
||||
- **Known limit:** if focus moves to another panel in the middle of a drag, the
|
||||
release goes to the panel that has focus then. Whether gamescope hands it to
|
||||
the window that got the press isn't known yet. When the session ends, the
|
||||
helper lets go of every button and key it still holds.
|
||||
- **The Desktop picture is the window's own pixels**: `ffmpeg -f x11grab
|
||||
-window_id <window> -i :<display>` works on gamescope's redirected windows,
|
||||
while grabbing the root gives black. It streams as H.264 like the headset view
|
||||
(about 30 fps at 720p).
|
||||
- Tested from the iPhone app (Simulator): a tap on the Desktop view focused a
|
||||
text box in the panel and the text field typed into it; a trackpad move went
|
||||
exactly (+40, +25).
|
||||
|
||||
Our own `uinput` keyboard and mouse would also work (`steamos` is in the
|
||||
`input` group and `/dev/uinput` is group-writable, verified 2026-09-27), and
|
||||
remains the fallback if the bundled KDE Connect ever stops working on a new SteamOS.
|
||||
|
||||
@@ -170,12 +170,3 @@ assistant against an in-process HTTP endpoint with canned responses (no keys or
|
||||
external calls). `tests/e2e/test_agents.py` runs the MCP/HTTP/SSH path against the
|
||||
fake Frame for approved installs, clipboard and file transfer. Headset Chromium
|
||||
rendering and real screenshots still need a device; see [agent evidence](agents.md#evidence-and-limits).
|
||||
|
||||
## Family and comfort
|
||||
|
||||
`tests/test_comfort.py` uses an injected clock, fake headset sensor readings and
|
||||
actions, plus a Node fake of Steam's Home API. It covers warnings before Home,
|
||||
late/suspended sessions, cancellation, failed actions, duplicate alerts, reboot
|
||||
invalidation, per-zone thermal trips and shared on-headset state. The server
|
||||
guards reject invalid session settings before SSH. See
|
||||
[real-device evidence and limits](family-comfort.md#verification).
|
||||
@@ -17,7 +17,6 @@
|
||||
78427FC66780623F31E7501E /* FrameControlApp.swift in Sources */ = {isa = PBXBuildFile; fileRef = 93C8E0D7C3F4F628941B3D5A /* FrameControlApp.swift */; };
|
||||
84423CB45629465420180A64 /* Assets.xcassets in Resources */ = {isa = PBXBuildFile; fileRef = 8F2CB550FC81C01E6BDD5A71 /* Assets.xcassets */; };
|
||||
9657F7BC23E3352E5AB30777 /* SetupView.swift in Sources */ = {isa = PBXBuildFile; fileRef = DB544223FC60A59CC3E8EF5F /* SetupView.swift */; };
|
||||
A0C5B00E257230A38DBD9E54 /* ComfortNotificationTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = E81218B75FEEE47B8D8BAE20 /* ComfortNotificationTests.swift */; };
|
||||
A8C7AED25A6280682FCE45DC /* Citadel in Frameworks */ = {isa = PBXBuildFile; productRef = 6BA549B6CC0A0CB847126456 /* Citadel */; };
|
||||
DC043FB74BE2D23F3A5826BF /* FrameControlTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 1740B691F9C25E5FB6F9EFC3 /* FrameControlTests.swift */; };
|
||||
E6898C714A92D3979F73B6E1 /* FrameFinder.swift in Sources */ = {isa = PBXBuildFile; fileRef = 2F288DF6636A417F0CA3A6CD /* FrameFinder.swift */; };
|
||||
@@ -49,7 +48,6 @@
|
||||
BF0FCA7117DA3ABA449B4EE0 /* InstallLink.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = InstallLink.swift; sourceTree = "<group>"; };
|
||||
D6C4E6C28315CA8729FCAAEA /* WebShell.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = WebShell.swift; sourceTree = "<group>"; };
|
||||
DB544223FC60A59CC3E8EF5F /* SetupView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SetupView.swift; sourceTree = "<group>"; };
|
||||
E81218B75FEEE47B8D8BAE20 /* ComfortNotificationTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ComfortNotificationTests.swift; sourceTree = "<group>"; };
|
||||
EDC7BA8014DBC302D08FD397 /* HeadsetServer.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = HeadsetServer.swift; sourceTree = "<group>"; };
|
||||
F3E2F5607DD877272483D64E /* FrameControl.app */ = {isa = PBXFileReference; includeInIndex = 0; lastKnownFileType = wrapper.application; path = FrameControl.app; sourceTree = BUILT_PRODUCTS_DIR; };
|
||||
/* End PBXFileReference section */
|
||||
@@ -109,7 +107,6 @@
|
||||
75A17B1C79C8C3C60FEABBA6 /* FrameControlTests */ = {
|
||||
isa = PBXGroup;
|
||||
children = (
|
||||
E81218B75FEEE47B8D8BAE20 /* ComfortNotificationTests.swift */,
|
||||
1740B691F9C25E5FB6F9EFC3 /* FrameControlTests.swift */,
|
||||
);
|
||||
path = FrameControlTests;
|
||||
@@ -277,7 +274,6 @@
|
||||
isa = PBXSourcesBuildPhase;
|
||||
buildActionMask = 2147483647;
|
||||
files = (
|
||||
A0C5B00E257230A38DBD9E54 /* ComfortNotificationTests.swift in Sources */,
|
||||
DC043FB74BE2D23F3A5826BF /* FrameControlTests.swift in Sources */,
|
||||
);
|
||||
runOnlyForDeploymentPostprocessing = 0;
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
import SwiftUI
|
||||
import UIKit
|
||||
import WebKit
|
||||
import UserNotifications
|
||||
|
||||
/// The Frame Control page, served by the server on the headset, in a web view.
|
||||
/// window.frameApp (the same bridge the desktop app's preload.js provides) lets
|
||||
@@ -49,7 +48,6 @@ struct WebShell: UIViewRepresentable {
|
||||
let installCb = null;
|
||||
window.frameApp = {
|
||||
platform: "ios",
|
||||
notify: (message, request) => call("notify", { message, request }),
|
||||
readClipboard: () => call("readClipboard"),
|
||||
setUpConnection: () => call("setUpConnection"),
|
||||
open: (what) => call("open", what),
|
||||
@@ -60,31 +58,13 @@ struct WebShell: UIViewRepresentable {
|
||||
})();
|
||||
"""
|
||||
|
||||
final class Coordinator: NSObject, WKScriptMessageHandlerWithReply, WKNavigationDelegate, WKUIDelegate, UNUserNotificationCenterDelegate {
|
||||
final class Coordinator: NSObject, WKScriptMessageHandlerWithReply, WKNavigationDelegate, WKUIDelegate {
|
||||
let model: AppModel
|
||||
weak var web: WKWebView?
|
||||
var loaded: URL?
|
||||
private var installReady = false
|
||||
|
||||
init(model: AppModel) {
|
||||
self.model = model
|
||||
super.init()
|
||||
UNUserNotificationCenter.current().delegate = self
|
||||
}
|
||||
|
||||
func userNotificationCenter(_ center: UNUserNotificationCenter, willPresent notification: UNNotification,
|
||||
withCompletionHandler completionHandler: @escaping (UNNotificationPresentationOptions) -> Void) {
|
||||
completionHandler([.banner, .sound, .list])
|
||||
}
|
||||
|
||||
static func notificationContent(_ message: String) -> UNMutableNotificationContent? {
|
||||
guard !message.isEmpty, message.count <= 500 else { return nil }
|
||||
let content = UNMutableNotificationContent()
|
||||
content.title = "Frame Control"
|
||||
content.body = message
|
||||
content.sound = .default
|
||||
return content
|
||||
}
|
||||
init(model: AppModel) { self.model = model }
|
||||
|
||||
// MARK: bridge
|
||||
|
||||
@@ -96,28 +76,6 @@ struct WebShell: UIViewRepresentable {
|
||||
}
|
||||
let arg = body["arg"]
|
||||
switch name {
|
||||
case "notify":
|
||||
guard message.frameInfo.isMainFrame,
|
||||
message.frameInfo.securityOrigin.host == "127.0.0.1",
|
||||
let args = arg as? [String: Any], let text = args["message"] as? String,
|
||||
let content = Self.notificationContent(text) else {
|
||||
return replyHandler(nil, "Invalid notification")
|
||||
}
|
||||
let center = UNUserNotificationCenter.current()
|
||||
let send: (Bool, Error?) -> Void = { allowed, error in
|
||||
guard allowed else {
|
||||
return replyHandler(nil, error?.localizedDescription ?? "Notifications are off. Enable them in iOS Settings.")
|
||||
}
|
||||
let request = UNNotificationRequest(identifier: UUID().uuidString, content: content, trigger: nil)
|
||||
center.add(request) { error in replyHandler(error == nil, error?.localizedDescription) }
|
||||
}
|
||||
if args["request"] as? Bool == true {
|
||||
center.requestAuthorization(options: [.alert, .sound], completionHandler: send)
|
||||
} else {
|
||||
center.getNotificationSettings { settings in
|
||||
send(settings.authorizationStatus == .authorized || settings.authorizationStatus == .provisional, nil)
|
||||
}
|
||||
}
|
||||
case "readClipboard":
|
||||
replyHandler(UIPasteboard.general.string ?? "", nil)
|
||||
case "setUpConnection":
|
||||
|
||||
@@ -1,14 +0,0 @@
|
||||
import XCTest
|
||||
import UserNotifications
|
||||
@testable import Frame_Control
|
||||
|
||||
final class ComfortNotificationTests: XCTestCase {
|
||||
func testNotificationContentAndBounds() {
|
||||
let content = WebShell.Coordinator.notificationContent("Time for a break")
|
||||
XCTAssertEqual(content?.title, "Frame Control")
|
||||
XCTAssertEqual(content?.body, "Time for a break")
|
||||
XCTAssertNotNil(content?.sound)
|
||||
XCTAssertNil(WebShell.Coordinator.notificationContent(""))
|
||||
XCTAssertNil(WebShell.Coordinator.notificationContent(String(repeating: "x", count: 501)))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,244 @@
|
||||
// Control on the live view (tap, drag, hold, scroll, type), run in node against the real
|
||||
// functions from ui/index.html with a fake canvas and a fake server.
|
||||
import { readFileSync } from "fs";
|
||||
const src = readFileSync(new URL("../../ui/index.html", import.meta.url), "utf8");
|
||||
const grab = name => {
|
||||
const one = src.match(new RegExp(`\\n((?:async )?function ${name}\\(.*\\}\\n)`)); // one-line function
|
||||
if (one) return one[1];
|
||||
const m = src.match(new RegExp(`(?:\\nconst ${name} = [^\\n]*\\n)|((?:async )?function ${name}\\([\\s\\S]*?\\n}\\n)`));
|
||||
if (!m) throw new Error("not found: " + name);
|
||||
return m[0];
|
||||
};
|
||||
const NAMES = ["panelKey", "ctrlAimedAt", "ctrlSameTarget", "isMoveEvent", "isRelease", "ctrlKeepable", "TAP_MOVE", "ctrlAim", "ctrlKeyEvent", "ctrlTouchCancel", "ctrlSend", "ctrlFlush", "ctrlMoveTo", "ctrlMoveBy", "ctrlSchedule",
|
||||
"ctrlFlushMoves", "ctrlButton", "ctrlClick", "ctrlRelease", "ctrlFraction", "ctrlTouchDown", "centroid",
|
||||
"ctrlTouchMove", "ctrlTouchUp", "ctrlTap", "ctrlText"];
|
||||
const code = NAMES.map(grab).join("");
|
||||
const fail = msg => { console.log("FAIL " + msg); process.exit(1); };
|
||||
const tick = (ms = 0) => new Promise(r => setTimeout(r, ms));
|
||||
|
||||
function page({ mode = "abs", rect = { left: 0, top: 0, width: 640, height: 360 }, api } = {}) {
|
||||
const target = { panel: { window: 42, display: ":1" } };
|
||||
const ctrl = { on: true, queue: [], sending: false, state: "ready", message: "", move: null, rel: [0, 0], raf: 0,
|
||||
held: new Set(), keys: new Set(), pointers: new Map(), g: null, retry: null };
|
||||
const sent = [];
|
||||
const canvas = { width: 1280, height: 720, getBoundingClientRect: () => rect };
|
||||
const env = {
|
||||
ctrl, $: () => canvas, ctrlMode: () => mode, ctrlShow: () => {}, toast: () => {},
|
||||
ctrlTarget: () => (mode !== "abs" ? { ok: true } : target.panel ? { ok: true, panel: target.panel } : { why: "gone" }),
|
||||
api: api || (async (path, body) => { sent.push(...body.events); return { state: "ready", sent: true }; }),
|
||||
requestAnimationFrame: cb => { setTimeout(cb, 0); return 1; },
|
||||
navigator: {},
|
||||
};
|
||||
const fns = new Function(...Object.keys(env), `let ctrlWarned = false;\n${code}
|
||||
return { ctrlTouchDown, ctrlTouchMove, ctrlTouchUp, ctrlTouchCancel, ctrlSend, ctrlText, ctrlButton, ctrlKeyEvent, ctrlRelease, ctrlFlushMoves };`)(...Object.values(env));
|
||||
const at = (id, x, y) => ({ pointerId: id, clientX: x, clientY: y });
|
||||
return { ctrl, sent, target, ...fns, at };
|
||||
}
|
||||
|
||||
// A tap lands where it was tapped: pointer there first, then the click.
|
||||
{
|
||||
const p = page();
|
||||
p.ctrlTouchDown(p.at(1, 320, 90)); p.ctrlTouchUp(p.at(1, 320, 90));
|
||||
await tick(10);
|
||||
const [move, down, up] = p.sent;
|
||||
if (!(move.fx === 0.5 && Math.abs(move.fy - 0.25) < 1e-9 && move.window === 42 && move.display === ":1")) fail("tap position " + JSON.stringify(move));
|
||||
if (!(down.window === 42 && down.display === ":1")) fail("a press names its panel " + JSON.stringify(down));
|
||||
if (!(down.button === "left" && down.down && up.button === "left" && up.down === false && p.sent.length === 3))
|
||||
fail("tap click " + JSON.stringify(p.sent));
|
||||
}
|
||||
// The panel is letterboxed in a taller view: taps map inside it, taps on the bars do nothing.
|
||||
{
|
||||
const p = page({ rect: { left: 0, top: 0, width: 640, height: 480 } }); // 640x360 picture, 60px bars
|
||||
p.ctrlTouchDown(p.at(1, 320, 150)); p.ctrlTouchUp(p.at(1, 320, 150));
|
||||
p.ctrlTouchDown(p.at(2, 320, 10)); p.ctrlTouchUp(p.at(2, 320, 10));
|
||||
await tick(10);
|
||||
if (!(p.sent.length === 3 && Math.abs(p.sent[0].fy - 0.25) < 1e-9)) fail("letterbox " + JSON.stringify(p.sent));
|
||||
}
|
||||
// A drag is a mouse drag: down where it started, moves, up at the end.
|
||||
{
|
||||
const p = page();
|
||||
p.ctrlTouchDown(p.at(1, 100, 100));
|
||||
p.ctrlTouchMove(p.at(1, 140, 100)); await tick(5);
|
||||
p.ctrlTouchMove(p.at(1, 200, 120)); await tick(5);
|
||||
p.ctrlTouchUp(p.at(1, 200, 120)); await tick(10);
|
||||
const kinds = p.sent.map(e => "fx" in e ? "move" : `${e.button}-${e.down ? "down" : "up"}`);
|
||||
if (!(kinds[0] === "move" && kinds[1] === "left-down" && kinds.at(-1) === "left-up" && kinds.includes("move", 2)))
|
||||
fail("drag " + kinds.join(","));
|
||||
if (Math.abs(p.sent[0].fx - 100 / 640) > 1e-9) fail("drag starts where the finger went down");
|
||||
if (p.ctrl.held.size) fail("drag left the button held");
|
||||
}
|
||||
// Press and hold is a right-click where the finger is.
|
||||
{
|
||||
const p = page();
|
||||
p.ctrlTouchDown(p.at(1, 64, 36));
|
||||
await tick(620);
|
||||
p.ctrlTouchUp(p.at(1, 64, 36)); await tick(10);
|
||||
const buttons = p.sent.filter(e => "button" in e).map(e => `${e.button}-${e.down}`);
|
||||
if (buttons.join() !== "right-true,right-false") fail("hold " + buttons.join());
|
||||
if (!(p.sent[0].fx === 0.1)) fail("hold position " + JSON.stringify(p.sent[0]));
|
||||
}
|
||||
// Two fingers scroll, and the content follows them (fingers up scrolls down).
|
||||
{
|
||||
const p = page();
|
||||
p.ctrlTouchDown(p.at(1, 100, 200)); p.ctrlTouchDown(p.at(2, 200, 200));
|
||||
p.ctrlTouchMove(p.at(1, 100, 180)); p.ctrlTouchMove(p.at(2, 200, 180));
|
||||
p.ctrlTouchUp(p.at(1, 100, 180)); p.ctrlTouchUp(p.at(2, 200, 180));
|
||||
await tick(10);
|
||||
const dy = p.sent.filter(e => e.scroll).reduce((a, e) => a + e.scroll[1], 0);
|
||||
if (!(dy === 40 && !p.sent.some(e => "button" in e))) fail("scroll " + JSON.stringify(p.sent));
|
||||
}
|
||||
// On the headset view it's a trackpad: drags move the pointer, taps click where it is.
|
||||
{
|
||||
const p = page({ mode: "rel" });
|
||||
p.ctrlTouchDown(p.at(1, 100, 100));
|
||||
p.ctrlTouchMove(p.at(1, 110, 100)); p.ctrlTouchMove(p.at(1, 120, 105));
|
||||
p.ctrlTouchUp(p.at(1, 120, 105)); await tick(10);
|
||||
p.ctrlTouchDown(p.at(1, 50, 50)); p.ctrlTouchUp(p.at(1, 50, 50)); await tick(10);
|
||||
const moved = p.sent.filter(e => "dx" in e).reduce((a, e) => [a[0] + e.dx, a[1] + e.dy], [0, 0]);
|
||||
if (!(moved[0] === 32 && moved[1] === 8)) fail("trackpad move " + JSON.stringify(moved));
|
||||
if (p.sent.some(e => "fx" in e)) fail("trackpad sent an absolute position");
|
||||
if (p.sent.filter(e => e.button === "left").length !== 2) fail("trackpad tap " + JSON.stringify(p.sent));
|
||||
}
|
||||
// Text: plain ASCII only, in chunks the server takes.
|
||||
{
|
||||
const p = page();
|
||||
p.ctrlText("héllo " + "x".repeat(600));
|
||||
await tick(10);
|
||||
const text = p.sent.map(e => e.text).join("");
|
||||
if (!(text === "hllo " + "x".repeat(600) && p.sent.every(e => e.text.length <= 500))) fail("text " + text.length);
|
||||
}
|
||||
// Not connected yet: clicks and keys wait with their position. The request fails: only releases wait.
|
||||
{
|
||||
let calls = 0;
|
||||
const p = page({ api: async () => { calls++; if (calls === 1) return { state: "starting", sent: false }; throw new Error("offline"); } });
|
||||
p.ctrlSend([{ fx: 0.1, fy: 0.1, window: 42 }, { button: "left", down: true }, { key: 30, down: true }]);
|
||||
await tick(5);
|
||||
const q = p.ctrl.queue;
|
||||
if (!(q.length === 3 && "fx" in q[0] && q[1].button === "left" && q[2].key === 30)) fail("kept while starting " + JSON.stringify(q));
|
||||
clearTimeout(p.ctrl.retry);
|
||||
p.ctrl.queue = []; p.ctrl.retryAt = 0;
|
||||
p.ctrlSend([{ button: "left", down: false }, { key: 31, down: true }, { dx: 3, dy: 1 }]);
|
||||
await tick(5);
|
||||
if (!(p.ctrl.queue.length === 1 && p.ctrl.queue[0].button === "left" && p.ctrl.queue[0].down === false))
|
||||
fail("release kept on failure " + JSON.stringify(p.ctrl.queue));
|
||||
if (calls !== 2) fail("retried in a tight loop: " + calls + " requests");
|
||||
clearTimeout(p.ctrl.retry);
|
||||
}
|
||||
// Turning Control off (or the view losing focus) lets go of anything held.
|
||||
{
|
||||
const p = page();
|
||||
p.ctrlButton("left", true);
|
||||
p.ctrlRelease(); await tick(10);
|
||||
if (!(p.sent.at(-1).button === "left" && p.sent.at(-1).down === false && !p.ctrl.held.size)) fail("release " + JSON.stringify(p.sent));
|
||||
}
|
||||
// Keys held on the Frame are let go too.
|
||||
{
|
||||
const p = page();
|
||||
p.ctrlKeyEvent(42, true);
|
||||
p.ctrlRelease(); await tick(10);
|
||||
if (!(p.sent.at(-1).key === 42 && p.sent.at(-1).down === false && !p.ctrl.keys.size)) fail("key release " + JSON.stringify(p.sent));
|
||||
}
|
||||
// A long queue sheds moves and old scrolls, never a release.
|
||||
{
|
||||
const p = page({ api: () => new Promise(() => {}) }); // stuck request
|
||||
p.ctrlSend([{ dx: 1, dy: 1 }]);
|
||||
p.ctrlSend([{ button: "left", down: false }]);
|
||||
for (let i = 0; i < 320; i++) p.ctrlSend([{ scroll: [0, 1] }]);
|
||||
if (!p.ctrl.queue.some(e => e.button === "left" && e.down === false)) fail("trim dropped a release");
|
||||
if (p.ctrl.queue.length > 300) fail("trim kept " + p.ctrl.queue.length);
|
||||
}
|
||||
// Broken on the Frame side: no hammering, and only releases wait.
|
||||
{
|
||||
let calls = 0;
|
||||
const p = page({ api: async () => { calls++; return { state: "error", sent: false }; } });
|
||||
p.ctrlSend([{ button: "left", down: true }, { button: "left", down: false }]);
|
||||
await tick(50);
|
||||
if (calls !== 1) fail("error response reposted " + calls + " times");
|
||||
if (!(p.ctrl.queue.length === 1 && p.ctrl.queue[0].down === false)) fail("error kept " + JSON.stringify(p.ctrl.queue));
|
||||
clearTimeout(p.ctrl.retry);
|
||||
}
|
||||
// Connecting: a tap keeps its position, so it lands where it was made.
|
||||
{
|
||||
const p = page({ api: async () => ({ state: "starting", sent: false }) });
|
||||
p.ctrlTouchDown(p.at(1, 320, 90)); p.ctrlTouchUp(p.at(1, 320, 90));
|
||||
await tick(10);
|
||||
const q = p.ctrl.queue;
|
||||
if (!("fx" in q[0] && q[1].button === "left")) fail("connecting tap " + JSON.stringify(q));
|
||||
clearTimeout(p.ctrl.retry);
|
||||
}
|
||||
// Lifting one of two scrolling fingers doesn't jump the scroll or start a drag.
|
||||
{
|
||||
const p = page();
|
||||
p.ctrlTouchDown(p.at(1, 100, 200)); p.ctrlTouchDown(p.at(2, 300, 200));
|
||||
p.ctrlTouchUp(p.at(1, 100, 200));
|
||||
p.ctrlTouchMove(p.at(2, 300, 199));
|
||||
p.ctrlTouchUp(p.at(2, 300, 199)); await tick(10);
|
||||
if (p.sent.length) fail("one finger left after scrolling " + JSON.stringify(p.sent));
|
||||
}
|
||||
// A cancelled touch isn't a tap.
|
||||
{
|
||||
const p = page();
|
||||
p.ctrlTouchDown(p.at(1, 100, 100)); p.ctrlTouchCancel(p.at(1, 100, 100)); await tick(10);
|
||||
if (p.sent.length) fail("cancel clicked " + JSON.stringify(p.sent));
|
||||
}
|
||||
// Press and hold on the bars around the picture does nothing.
|
||||
{
|
||||
const p = page({ rect: { left: 0, top: 0, width: 640, height: 480 } });
|
||||
p.ctrlTouchDown(p.at(1, 320, 10)); await tick(620); p.ctrlTouchUp(p.at(1, 320, 10)); await tick(10);
|
||||
if (p.sent.length) fail("hold on the bars " + JSON.stringify(p.sent));
|
||||
}
|
||||
// Taps only reach the panel in use, and only once its picture is the one on screen.
|
||||
{
|
||||
const fns = ["panelKey", "deskPanel", "ctrlTarget"].map(grab).join("");
|
||||
const check = (desk, live = true) => new Function("desk", "live", "ctrlMode", fns + "; return ctrlTarget();")(desk, live, () => "abs");
|
||||
const a = { display: ":1", window: 5 }, b = { display: ":0", window: 5 };
|
||||
const base = { panels: [a, b], loaded: true, pick: "", focus: ":1/5", shown: ":1/5" };
|
||||
if (!check(base).ok) fail("target: shown and in use");
|
||||
if (check({ ...base, shown: ":0/5" }).ok) fail("target: the picture is another panel with the same id");
|
||||
if (!/Capture or Live/.test(check({ ...base, shown: null }, false).why)) fail("target: stale picture message");
|
||||
if (check({ ...base, pick: ":0/5", shown: ":0/5" }).ok) fail("target: a watched panel that isn't in use");
|
||||
if (check({ ...base, focus: null }).ok) fail("target: nothing in use");
|
||||
}
|
||||
// Focus moves to another panel mid-gesture: the tap or hold does nothing.
|
||||
{
|
||||
const p = page();
|
||||
p.ctrlTouchDown(p.at(1, 100, 100));
|
||||
p.target.panel = { window: 43, display: ":1" };
|
||||
p.ctrlTouchUp(p.at(1, 100, 100)); await tick(10);
|
||||
p.target.panel = { window: 42, display: ":1" };
|
||||
p.ctrlTouchDown(p.at(1, 100, 100));
|
||||
p.target.panel = null;
|
||||
await tick(620); p.ctrlTouchUp(p.at(1, 100, 100)); await tick(10);
|
||||
if (p.sent.some(e => "button" in e)) fail("gesture outlived its panel " + JSON.stringify(p.sent));
|
||||
}
|
||||
// Trimming keeps a click together with its position.
|
||||
{
|
||||
const p = page({ api: () => new Promise(() => {}) });
|
||||
p.ctrlSend([{ dx: 1, dy: 0 }]); // in flight forever
|
||||
for (let i = 0; i < 100; i++) p.ctrlSend([{ scroll: [0, 1] }]);
|
||||
p.ctrlSend([{ fx: 0.5, fy: 0.5, window: 42, display: ":1" }, { button: "left", down: true }, { button: "left", down: false }]);
|
||||
for (let i = 0; i < 198; i++) p.ctrlSend([{ scroll: [0, 1] }]);
|
||||
const q = p.ctrl.queue, i = q.findIndex(e => e.button === "left" && e.down);
|
||||
if (i < 1 || !("fx" in q[i - 1])) fail("trim split a click from its position");
|
||||
}
|
||||
// Backing off holds for new input too.
|
||||
{
|
||||
let calls = 0;
|
||||
const p = page({ api: async () => { calls++; return { state: "error", sent: false }; } });
|
||||
p.ctrlSend([{ button: "left", down: false }]);
|
||||
await tick(5);
|
||||
for (let i = 0; i < 10; i++) { p.ctrlSend([{ key: 30, down: false }]); await tick(2); }
|
||||
if (calls !== 1) fail("new input bypassed the backoff: " + calls + " requests");
|
||||
clearTimeout(p.ctrl.retry);
|
||||
}
|
||||
// A long paste goes in several requests, so a release never waits behind all of it.
|
||||
{
|
||||
const batches = [];
|
||||
const p = page({ api: async (path, body) => { batches.push(body.events); return { state: "ready", sent: true }; } });
|
||||
p.ctrlText("y".repeat(450));
|
||||
p.ctrlButton("left", false);
|
||||
await tick(30);
|
||||
if (!batches.every(b => b.reduce((a, e) => a + (e.text?.length || 0), 0) <= 100)) fail("a batch carried too much text");
|
||||
if (batches.length < 5) fail("paste went in " + batches.length + " requests");
|
||||
}
|
||||
console.log("control gestures ok");
|
||||
@@ -1,257 +0,0 @@
|
||||
"""Fake-Frame session clock/actions plus real helper serialization and sensor probes."""
|
||||
import os
|
||||
import shutil
|
||||
import json
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'ui'))
|
||||
import frame_comfort as comfort
|
||||
import frame_status as status
|
||||
|
||||
OPTIONS = {'action': 'start', 'minutes': 3, 'breakMinutes': 1, 'stillMinutes': 1,
|
||||
'batteryAlert': True, 'heatAlert': True}
|
||||
|
||||
|
||||
class SessionTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.s = comfort.new_session(OPTIONS, 0, 'boot-one')
|
||||
self.warn, self.home = Mock(), Mock()
|
||||
|
||||
def step(self, now, **sample):
|
||||
comfort.tick(self.s, now, sample, self.warn, self.home, read_clock=lambda: now)
|
||||
|
||||
def test_warning_then_home_never_closes_a_game(self):
|
||||
self.step(119)
|
||||
self.warn.assert_not_called()
|
||||
self.step(120)
|
||||
self.warn.assert_called_once()
|
||||
self.step(179)
|
||||
self.home.assert_not_called()
|
||||
self.step(180)
|
||||
self.home.assert_called_once()
|
||||
self.assertFalse(self.s['active'])
|
||||
self.step(181)
|
||||
self.home.assert_called_once()
|
||||
|
||||
def test_late_wakeup_always_gets_a_full_warning_minute(self):
|
||||
self.step(400)
|
||||
self.home.assert_not_called()
|
||||
self.step(459)
|
||||
self.home.assert_not_called()
|
||||
self.step(460)
|
||||
self.home.assert_called_once()
|
||||
|
||||
def test_slow_warning_still_leaves_a_full_minute(self):
|
||||
comfort.tick(self.s, 120, {}, self.warn, self.home, read_clock=lambda: 140)
|
||||
self.assertEqual(self.s['warned'], 140)
|
||||
self.step(180)
|
||||
self.home.assert_not_called()
|
||||
self.step(199)
|
||||
self.home.assert_not_called()
|
||||
self.step(200)
|
||||
self.home.assert_called_once()
|
||||
|
||||
def test_failed_warning_never_stops_session(self):
|
||||
self.warn.side_effect = RuntimeError('offline')
|
||||
with self.assertRaises(RuntimeError):
|
||||
self.step(200)
|
||||
self.assertIsNone(self.s['warned'])
|
||||
self.home.assert_not_called()
|
||||
self.warn.side_effect = None
|
||||
self.step(300)
|
||||
self.step(359)
|
||||
self.home.assert_not_called()
|
||||
self.step(360)
|
||||
self.home.assert_called_once()
|
||||
|
||||
def test_failed_home_stays_active_and_retries(self):
|
||||
self.step(120)
|
||||
self.home.side_effect = RuntimeError('Steam offline')
|
||||
with self.assertRaises(RuntimeError):
|
||||
self.step(180)
|
||||
self.assertTrue(self.s['active'])
|
||||
self.home.side_effect = None
|
||||
self.step(185)
|
||||
self.assertFalse(self.s['active'])
|
||||
|
||||
def test_cancel_prevents_all_actions(self):
|
||||
self.s['active'] = False
|
||||
self.step(999, battery={'percent': 1, 'status': 'Discharging'})
|
||||
self.warn.assert_not_called()
|
||||
self.home.assert_not_called()
|
||||
self.assertEqual(self.s['events'], [])
|
||||
|
||||
def test_breaks_and_checkin_require_measured_activity(self):
|
||||
self.step(20, activity=1)
|
||||
self.step(40, activity=2)
|
||||
self.step(60, activity=1)
|
||||
self.assertEqual([e['kind'] for e in self.s['events']], ['break', 'still'])
|
||||
self.step(70, activity=1)
|
||||
self.assertEqual(len(self.s['events']), 2)
|
||||
self.step(75, activity=3)
|
||||
self.assertEqual(self.s['used'], 0)
|
||||
self.assertFalse(self.s['stillSent'])
|
||||
|
||||
def test_unknown_activity_and_gaps_do_not_count_as_wear(self):
|
||||
self.step(25)
|
||||
self.assertEqual(self.s['used'], 0)
|
||||
self.assertEqual(self.s['unavailable'], ['battery', 'temperature', 'activity'])
|
||||
self.step(100, activity=1)
|
||||
self.assertEqual(self.s['used'], 30)
|
||||
|
||||
def test_alerts_latch_and_rearm_without_battery_chatter(self):
|
||||
low = {'percent': 10, 'status': 'Discharging'}
|
||||
self.step(1, battery=low, thermal=['cpu'])
|
||||
self.step(2, battery=low, thermal=['cpu'])
|
||||
self.step(3)
|
||||
self.assertEqual(len(self.s['events']), 2)
|
||||
self.step(4, battery={'percent': 16, 'status': 'Discharging'}, thermal=[])
|
||||
self.step(5, battery=low, thermal=[])
|
||||
self.assertEqual(len(self.s['events']), 2)
|
||||
self.step(6, battery={'percent': 22, 'status': 'Discharging'}, thermal=[])
|
||||
self.step(7, battery=low, thermal=['cpu'])
|
||||
self.assertEqual([e['kind'] for e in self.s['events']], ['battery', 'heat', 'battery', 'heat'])
|
||||
|
||||
def test_disabled_alerts_and_charging(self):
|
||||
self.s['options']['heatAlert'] = False
|
||||
self.step(1, battery={'percent': 2, 'status': 'Charging'}, thermal=['cpu'])
|
||||
self.assertEqual(self.s['events'], [])
|
||||
|
||||
def test_invalid_options(self):
|
||||
for key, value in [('minutes', 0), ('minutes', 241), ('minutes', True), ('minutes', 2.5),
|
||||
('breakMinutes', -1), ('stillMinutes', '1'), ('heatAlert', 1)]:
|
||||
with self.subTest(key=key, value=value), self.assertRaises(ValueError):
|
||||
comfort.validate({**OPTIONS, key: value})
|
||||
for value in (None, [], {'action': 'shutdown'}):
|
||||
with self.assertRaises(ValueError):
|
||||
comfort.validate(value)
|
||||
|
||||
def test_restart_invalidates_session_and_stale_worker_is_explicit(self):
|
||||
with patch.object(comfort, 'boot', return_value='boot-one'), patch.object(comfort.time, 'time', return_value=999):
|
||||
current = comfort.current(self.s, 100)
|
||||
self.assertIn('not responding', current['error'])
|
||||
self.assertEqual(current['time'], 999)
|
||||
with patch.object(comfort, 'boot', return_value='boot-two'):
|
||||
result = comfort.current(self.s, 100)
|
||||
self.assertFalse(result['active'])
|
||||
self.assertIn('restarted', result['error'])
|
||||
|
||||
@unittest.skipUnless(os.name == "posix", "on-headset state uses POSIX flock")
|
||||
def test_real_state_commands_share_one_session_and_cancel(self):
|
||||
with tempfile.TemporaryDirectory() as tmp, patch.object(comfort, 'ROOT', Path(tmp)), \
|
||||
patch.object(comfort, 'boot', return_value='boot-one'), \
|
||||
patch.object(comfort, 'clock', return_value=0), patch.object(comfort.subprocess, 'Popen') as spawn:
|
||||
started = comfort.command(OPTIONS)
|
||||
self.assertEqual(comfort.command({'action': 'status'})['id'], started['id'])
|
||||
with self.assertRaises(ValueError):
|
||||
comfort.command(OPTIONS)
|
||||
self.assertFalse(comfort.command({'action': 'cancel'})['active'])
|
||||
spawn.assert_called_once()
|
||||
self.assertEqual((Path(tmp) / 'session.json').stat().st_mode & 0o777, 0o600)
|
||||
|
||||
@unittest.skipUnless(os.name == "posix", "on-headset state uses POSIX flock")
|
||||
def test_cancel_clears_stale_worker_error(self):
|
||||
with tempfile.TemporaryDirectory() as tmp, patch.object(comfort, 'ROOT', Path(tmp)), \
|
||||
patch.object(comfort, 'boot', return_value='boot-one'), \
|
||||
patch.object(comfort, 'clock', return_value=200):
|
||||
with comfort.locked():
|
||||
comfort.save(self.s)
|
||||
self.assertIn('not responding', comfort.command({'action': 'status'})['error'])
|
||||
cancelled = comfort.command({'action': 'cancel'})
|
||||
self.assertFalse(cancelled['active'])
|
||||
self.assertIsNone(cancelled['error'])
|
||||
self.assertIsNone(comfort.command({'action': 'status'})['error'])
|
||||
|
||||
@unittest.skipUnless(os.name == "posix", "on-headset state uses POSIX flock")
|
||||
def test_failed_spawn_leaves_session_inactive_and_retryable(self):
|
||||
with tempfile.TemporaryDirectory() as tmp, patch.object(comfort, 'ROOT', Path(tmp)), \
|
||||
patch.object(comfort, 'boot', return_value='boot-one'), \
|
||||
patch.object(comfort, 'clock', return_value=0), patch.object(comfort.subprocess, 'Popen') as spawn:
|
||||
spawn.side_effect = OSError('process limit')
|
||||
with self.assertRaises(OSError):
|
||||
comfort.command(OPTIONS)
|
||||
failed = comfort.command({'action': 'status'})
|
||||
self.assertFalse(failed['active'])
|
||||
self.assertIn('Could not start', failed['error'])
|
||||
spawn.side_effect = None
|
||||
self.assertTrue(comfort.command(OPTIONS)['active'])
|
||||
|
||||
@unittest.skipUnless(os.name == "posix", "on-headset state uses POSIX flock")
|
||||
def test_unreadable_state_is_preserved_and_can_be_replaced(self):
|
||||
for contents in (b'{broken', b'\xff', b'null', b'[]', b'42', b'"x"'):
|
||||
with self.subTest(contents=contents):
|
||||
with tempfile.TemporaryDirectory() as tmp, patch.object(comfort, 'ROOT', Path(tmp)), \
|
||||
patch.object(comfort, 'boot', return_value='boot-one'), \
|
||||
patch.object(comfort, 'clock', return_value=0), patch.object(comfort.subprocess, 'Popen'):
|
||||
(Path(tmp) / 'session.json').write_bytes(contents)
|
||||
failed = comfort.command({'action': 'status'})
|
||||
self.assertFalse(failed['active'])
|
||||
self.assertIn('unreadable', failed['error'])
|
||||
backups = list(Path(tmp).glob('session-unreadable-*.json'))
|
||||
self.assertEqual(len(backups), 1)
|
||||
self.assertEqual(backups[0].read_bytes(), contents)
|
||||
self.assertTrue(comfort.command(OPTIONS)['active'])
|
||||
|
||||
def test_home_has_total_process_deadline_and_propagates_timeout(self):
|
||||
with patch.object(comfort.subprocess, 'run', side_effect=subprocess.TimeoutExpired('home', 15)) as run:
|
||||
with self.assertRaises(subprocess.TimeoutExpired):
|
||||
comfort.home()
|
||||
self.assertEqual(run.call_args.kwargs['timeout'], 15)
|
||||
self.assertEqual(run.call_args.args[0][-1], '--home')
|
||||
|
||||
def test_native_warning_reports_failures_and_quotes_as_one_argument(self):
|
||||
with patch.object(comfort.subprocess, 'run') as run:
|
||||
run.return_value = subprocess.CompletedProcess([], 0, 'Notification succeeded', '')
|
||||
comfort.notify('Save "now"; $(nothing)')
|
||||
args = run.call_args.args[0]
|
||||
self.assertEqual(args, [comfort.VRCMD, '--notify', 'Frame Control: Save "now"; $(nothing)'])
|
||||
run.return_value.stdout = 'Notification failed with error 1'
|
||||
with self.assertRaises(RuntimeError):
|
||||
comfort.notify('test')
|
||||
|
||||
@unittest.skipUnless(shutil.which("node"), "Node exercises the fake Steam JS context")
|
||||
def test_home_javascript_against_fake_steam_preserves_game(self):
|
||||
# Same JS runs in Steam CDP. This fake records navigation and refuses any
|
||||
# unexpected API call; it offers no shutdown or terminate-game primitive.
|
||||
js = '''let running = [123], path = '/routes/library/app/123', visible = false;
|
||||
const location = {get pathname() {return path;}};
|
||||
const SteamUIStore = {Navigate(p) {path = '/routes' + p;}};
|
||||
const SteamClient = {OpenVR: {VROverlay: {
|
||||
async ShowDashboard(key) {if (key !== 'valve.steam.gamepadui.main') throw Error(key); visible = true;},
|
||||
async IsDashboardVisible() {return visible;}
|
||||
}}};
|
||||
'''
|
||||
js += comfort.HOME_JS + '.then(result => console.log(JSON.stringify({result, running, visible})));'
|
||||
r = subprocess.run(['node', '-e', js], capture_output=True, text=True, check=True)
|
||||
result = json.loads(r.stdout)
|
||||
self.assertEqual(result['running'], [123])
|
||||
self.assertTrue(result['visible'])
|
||||
self.assertEqual(result['result']['path'], '/routes/library/home')
|
||||
|
||||
|
||||
class SensorTests(unittest.TestCase):
|
||||
def test_hot_trip_uses_its_own_zone_not_hottest_unrelated_chip(self):
|
||||
values = {'/z/a/temp': '90000', '/z/a/trip_point_0_type': 'hot', '/z/a/trip_point_0_temp': '110000',
|
||||
'/z/b/temp': '45000', '/z/b/trip_point_0_type': 'hot', '/z/b/trip_point_0_temp': '44000', '/z/b/type': 'battery'}
|
||||
def glob(pattern):
|
||||
if pattern.endswith('thermal_zone*'):
|
||||
return ['/z/a', '/z/b']
|
||||
return [pattern.replace('*', '0')]
|
||||
with patch.object(status.glob, 'glob', side_effect=glob), patch.object(status, 'read', side_effect=values.get):
|
||||
self.assertEqual(status.thermal_alerts(), [{'zone': 'battery', 'tempC': 45, 'limitC': 44}])
|
||||
|
||||
def test_missing_thermal_and_activity_are_unknown(self):
|
||||
with patch.object(status.glob, 'glob', return_value=[]):
|
||||
self.assertIsNone(status.thermal_alerts())
|
||||
with patch.object(status, 'run', return_value='unavailable'):
|
||||
self.assertIsNone(status.activity_level())
|
||||
for malformed in ('{}', '[null, 42, "bad"]'):
|
||||
with patch.object(status, 'run', return_value=malformed):
|
||||
self.assertIsNone(status.activity_level())
|
||||
with patch.object(status, 'run', return_value='[{"operation":"status","activity_level":3}]'):
|
||||
self.assertEqual(status.activity_level(), 3)
|
||||
@@ -1,60 +0,0 @@
|
||||
"""Run the actual shared page's comfort renderer against a minimal DOM/bridge."""
|
||||
import pathlib
|
||||
import shutil
|
||||
import subprocess
|
||||
import unittest
|
||||
|
||||
ROOT = pathlib.Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
@unittest.skipUnless(shutil.which('node'), 'Node exercises the shared page JS')
|
||||
class ComfortUI(unittest.TestCase):
|
||||
def test_notification_failure_survives_poll_until_success(self):
|
||||
page = (ROOT / 'ui/index.html').read_text(encoding='utf-8')
|
||||
code = page[page.index('let comfortBusy ='):page.index('async function pollComfort()')]
|
||||
setup = r'''
|
||||
const assert = require('node:assert/strict');
|
||||
const elements = new Map();
|
||||
const $ = id => {
|
||||
if (!elements.has(id)) elements.set(id, {textContent:'', hidden:true, disabled:false, type: 'number'});
|
||||
return elements.get(id);
|
||||
};
|
||||
let denied = 0;
|
||||
const window = {frameApp:{notify:async()=>{denied++;throw Error('permission denied');}}};
|
||||
const log = ()=>{}, toast = ()=>{};
|
||||
'''
|
||||
checks = r'''
|
||||
(async()=>{
|
||||
const active = {id:'session-one',active:true,time:100,remaining:120,
|
||||
options:{minutes:2,breakMinutes:1,stillMinutes:1,batteryAlert:true,heatAlert:true},
|
||||
events:[{id:'event-one',kind:'battery',time:99,message:'Low battery'}]};
|
||||
renderComfort(active); // initial history must not replay even a fresh event
|
||||
assert.equal(denied,0);
|
||||
assert.equal($('comfortAnnouncement').textContent,'');
|
||||
active.events.push({id:'event-two',kind:'break',time:100,message:'Take a break'});
|
||||
renderComfort(active);
|
||||
await new Promise(resolve=>setImmediate(resolve));
|
||||
assert.equal(denied,1);
|
||||
assert.equal($('comfortAnnouncement').textContent,'Take a break');
|
||||
assert.equal($('comfortNotificationStatus').hidden,false);
|
||||
assert.match($('comfortNotificationStatus').textContent,/notification settings/);
|
||||
renderComfort({...active,time:105}); // the next normal poll must not erase failure
|
||||
assert.equal($('comfortNotificationStatus').hidden,false);
|
||||
assert.equal($('sessionStart').disabled,true);
|
||||
assert.equal($('sessionMinutes').disabled,true);
|
||||
assert.equal($('sessionCancel').disabled,false);
|
||||
let requests=0;
|
||||
window.frameApp.notify=async()=>{requests++;};
|
||||
renderComfort({...active,time:106});
|
||||
assert.equal($('comfortAnnouncement').textContent,'Take a break');
|
||||
assert.equal(requests,0); // polling does not replay an already-seen event
|
||||
await localNotification('test',true);
|
||||
assert.equal($('comfortNotificationStatus').hidden,true);
|
||||
renderComfort({...active,active:false});
|
||||
assert.equal($('sessionStart').disabled,false);
|
||||
assert.equal($('sessionMinutes').disabled,false);
|
||||
assert.equal($('sessionCancel').disabled,true);
|
||||
})().catch(e=>{console.error(e);process.exitCode=1;});
|
||||
'''
|
||||
result = subprocess.run(['node', '-e', setup + code + checks], capture_output=True, text=True)
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
@@ -84,7 +84,6 @@ class ServerGuards(unittest.TestCase):
|
||||
|
||||
def test_api_needs_custom_header(self):
|
||||
# <img src> and plain form posts from other sites can't set it.
|
||||
self.assertEqual(self.request("POST", "/api/comfort", {"action": "start"})[0], 403)
|
||||
self.assertEqual(self.request("GET", "/api/status")[0], 403)
|
||||
self.assertEqual(self.request("GET", "/api/screenshot?view=headset")[0], 403)
|
||||
self.assertEqual(self.request("GET", "/api/shots")[0], 403)
|
||||
@@ -100,8 +99,6 @@ class ServerGuards(unittest.TestCase):
|
||||
|
||||
def test_input_validation(self):
|
||||
cases = [
|
||||
("/api/comfort", {"action": "poweroff"}),
|
||||
("/api/comfort", {"action": "start", "minutes": 0}),
|
||||
("/api/launch", {"appid": "620; rm -rf ~"}),
|
||||
("/api/launch", {"appid": ""}),
|
||||
("/api/flatpak", {"id": "org.example.App;id", "action": "install"}),
|
||||
|
||||
@@ -0,0 +1,283 @@
|
||||
"""Touch: the live view's Control (ui/frame_touch.py on the Frame, and the server's checks).
|
||||
|
||||
Run: python3 -m unittest discover -s tests
|
||||
"""
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
sys.path.insert(0, str(ROOT / "ui"))
|
||||
|
||||
|
||||
class Mapping(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
import frame_touch
|
||||
cls.t = frame_touch
|
||||
|
||||
def test_panel_bigger_than_its_display_is_scaled(self):
|
||||
# Verified 2026-09-29: a 1920x1080 window on :1 (1280x720) took Accept at
|
||||
# window (1828, 1020) as pointer (1219, 680).
|
||||
panel = {"root": [1280, 720], "width": 1920, "height": 1080}
|
||||
x, y = self.t.to_root(panel, 1828 / 1920, 1020 / 1080)
|
||||
self.assertAlmostEqual(x, 1218.7, delta=1)
|
||||
self.assertAlmostEqual(y, 679.4, delta=1)
|
||||
|
||||
def test_same_size_is_one_to_one_and_clamped(self):
|
||||
panel = {"root": [1280, 720], "width": 1280, "height": 720}
|
||||
self.assertEqual(self.t.to_root(panel, 0, 0), (0, 0))
|
||||
self.assertEqual(self.t.to_root(panel, 1, 1), (1279, 719))
|
||||
self.assertEqual(self.t.to_root(panel, -3, 7), (0, 719))
|
||||
|
||||
def test_other_shapes_are_letterboxed(self):
|
||||
panel = {"root": [1280, 720], "width": 800, "height": 800} # square: bars left and right
|
||||
x, y = self.t.to_root(panel, 0, 0.5)
|
||||
self.assertAlmostEqual(x, 280, delta=0.5)
|
||||
self.assertAlmostEqual(y, 359.5, delta=0.5)
|
||||
|
||||
def test_same_id_on_both_displays_is_told_apart_by_pid(self):
|
||||
t = self.t
|
||||
saved = t.displays, t.window_info, t.window_pid
|
||||
self.addCleanup(lambda: (setattr(t, "displays", saved[0]), setattr(t, "window_info", saved[1]),
|
||||
setattr(t, "window_pid", saved[2])))
|
||||
t.displays = lambda: [":0", ":1"]
|
||||
t.window_info = lambda d, w: {"name": f"on {d}", "width": 1280 if w != "root" else 1920, "height": 720}
|
||||
t.window_pid = lambda d, w: {":0": 111, ":1": 222}[d]
|
||||
self.assertEqual(t.locate(5, 222)["display"], ":1")
|
||||
self.assertEqual(t.locate(5, 111)["display"], ":0")
|
||||
self.assertEqual(t.locate(5, None)["display"], ":0")
|
||||
|
||||
def test_focus_display_is_decoded(self):
|
||||
t = self.t
|
||||
saved = t.xprop_root
|
||||
self.addCleanup(lambda: setattr(t, "xprop_root", saved))
|
||||
for values, want in (([12602, 0, 58], ":1"), ([12346], ":0"), ([], None), ([0x41], None)):
|
||||
t.xprop_root = lambda d, n, v=values: v
|
||||
self.assertEqual(t.focus_display(), want)
|
||||
|
||||
def test_ascii_table_covers_printable_characters(self):
|
||||
for code in range(0x20, 0x7F):
|
||||
self.assertIn(chr(code), self.t.ASCII, chr(code))
|
||||
self.assertEqual(self.t.ASCII["a"], (30, False))
|
||||
self.assertEqual(self.t.ASCII["A"], (30, True))
|
||||
self.assertEqual(self.t.ASCII["?"], (53, True))
|
||||
self.assertEqual(self.t.ASCII["1"], (2, False))
|
||||
self.assertEqual(self.t.ASCII["0"], (11, False))
|
||||
|
||||
def test_events_parsing(self):
|
||||
self.assertEqual(self.t.events(b'{"dx": 1}'), [{"dx": 1}])
|
||||
self.assertEqual(self.t.events(b'[{"dx": 1}, 5]'), [{"dx": 1}])
|
||||
self.assertEqual(self.t.events(b"nope"), [])
|
||||
|
||||
|
||||
class FakeGamescope:
|
||||
def __init__(self):
|
||||
self.calls = []
|
||||
|
||||
def __getattr__(self, name):
|
||||
return lambda *a: self.calls.append((name, *a))
|
||||
|
||||
|
||||
class Apply(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
import frame_touch
|
||||
cls.t = frame_touch
|
||||
|
||||
def setUp(self):
|
||||
self.focused = {"window": 7, "display": ":1", "root": [1280, 720], "width": 1280, "height": 720, "name": "x"}
|
||||
saved = self.t.focus, self.t.say, self.t.focus_now
|
||||
self.t.STALE[0] = False
|
||||
self.said = []
|
||||
self.t.focus = lambda: dict(self.focused)
|
||||
self.t.focus_now = lambda: (self.focused["window"], self.focused["display"])
|
||||
self.t.say = lambda state, **more: self.said.append((state, more))
|
||||
self.addCleanup(lambda: (setattr(self.t, "focus", saved[0]), setattr(self.t, "say", saved[1]),
|
||||
setattr(self.t, "focus_now", saved[2])))
|
||||
|
||||
def test_tap_moves_then_clicks_in_order(self):
|
||||
gs, panel = FakeGamescope(), None
|
||||
for e in ({"fx": 0.5, "fy": 0.5, "window": 7, "display": ":1"}, {"button": "left", "down": True, "window": 7, "display": ":1"},
|
||||
{"button": "left", "down": False}):
|
||||
panel = self.t.apply(gs, e, panel)
|
||||
self.assertEqual([c[0] for c in gs.calls], ["move_to", "button", "button"])
|
||||
self.assertEqual(gs.calls[1][1:], ("left", True))
|
||||
|
||||
def test_a_tap_meant_for_another_panel_goes_nowhere(self):
|
||||
# Focus moved on: the position and the press are dropped; the release still goes.
|
||||
gs, panel = FakeGamescope(), None
|
||||
for e in ({"fx": 0.5, "fy": 0.5, "window": 99, "display": ":1"}, {"button": "left", "down": True, "window": 99, "display": ":1"},
|
||||
{"key": 30, "down": True, "window": 7, "display": ":0"}, {"button": "left", "down": False, "window": 99, "display": ":1"}):
|
||||
panel = self.t.apply(gs, e, panel)
|
||||
self.assertEqual(gs.calls, [("button", "left", False)])
|
||||
self.assertEqual(self.said[0], ("ready", {"focus": 7, "display": ":1", "stale": True}))
|
||||
|
||||
def test_presses_read_focus_afresh(self):
|
||||
# A move may use a recent reading; a press checks again, so a panel that just
|
||||
# took focus doesn't get a click meant for another.
|
||||
gs, calls = FakeGamescope(), []
|
||||
self.t.focus = lambda: calls.append(1) or dict(self.focused)
|
||||
panel = self.t.apply(gs, {"fx": 0.5, "fy": 0.5, "window": 7, "display": ":1"}, None)
|
||||
panel = self.t.apply(gs, {"fx": 0.6, "fy": 0.5, "window": 7, "display": ":1"}, panel)
|
||||
self.assertEqual(len(calls), 1)
|
||||
self.t.apply(gs, {"button": "left", "down": True, "window": 7, "display": ":1"}, panel)
|
||||
self.assertEqual(len(calls), 1) # same panel still: the quick check was enough
|
||||
self.focused["window"] = 8
|
||||
self.t.apply(gs, {"button": "left", "down": True, "window": 7, "display": ":1"}, panel)
|
||||
self.assertEqual(len(calls), 2)
|
||||
self.assertEqual([c[0] for c in gs.calls], ["move_to", "move_to", "button"])
|
||||
|
||||
def test_stale_is_said_once_and_cleared(self):
|
||||
gs, panel = FakeGamescope(), None
|
||||
for e in ({"fx": 0.5, "fy": 0.5, "window": 99, "display": ":1"}, {"fx": 0.5, "fy": 0.5, "window": 7, "display": ":1"},
|
||||
{"fx": 0.6, "fy": 0.5, "window": 7, "display": ":1"}):
|
||||
panel = self.t.apply(gs, e, panel)
|
||||
self.assertEqual(self.said, [("ready", {"focus": 7, "display": ":1", "stale": True}),
|
||||
("ready", {"focus": 7, "display": ":1"})])
|
||||
|
||||
def test_stale_clears_on_a_trackpad_move_but_not_on_a_stale_release(self):
|
||||
gs = FakeGamescope()
|
||||
panel = self.t.apply(gs, {"fx": 0.5, "fy": 0.5, "window": 99, "display": ":1"}, None)
|
||||
panel = self.t.apply(gs, {"button": "left", "down": False, "window": 99, "display": ":1"}, panel)
|
||||
self.assertTrue(self.t.STALE[0]) # that release was still aimed at the old panel
|
||||
self.t.apply(gs, {"dx": 3, "dy": 0}, panel)
|
||||
self.assertFalse(self.t.STALE[0])
|
||||
self.assertEqual(self.said[-1][1].get("stale"), None)
|
||||
|
||||
def test_same_window_id_on_the_other_display_is_another_panel(self):
|
||||
gs = FakeGamescope()
|
||||
self.t.apply(gs, {"fx": 0.5, "fy": 0.5, "window": 7, "display": ":0"}, None)
|
||||
self.assertEqual(gs.calls, [])
|
||||
|
||||
def test_relative_scroll_keys_text(self):
|
||||
gs = FakeGamescope()
|
||||
for e in ({"dx": 5, "dy": -3}, {"scroll": [0, 120]}, {"key": 30, "down": True}, {"text": "hi"},
|
||||
{"key": True}, {"button": "sideways"}):
|
||||
self.t.apply(gs, e, None)
|
||||
self.assertEqual([c[0] for c in gs.calls], ["move_by", "scroll", "key", "text"])
|
||||
|
||||
def test_everything_held_is_let_go(self):
|
||||
class Held(self.t.Gamescope):
|
||||
def __init__(self):
|
||||
self.held, self.keys, self.log = set(), set(), []
|
||||
|
||||
def frame(self):
|
||||
pass
|
||||
|
||||
g = Held()
|
||||
g.L = type("L", (), {"ei_device_button_button": lambda *a: g.log.append(("button",) + a[2:]),
|
||||
"ei_device_keyboard_key": lambda *a: g.log.append(("key",) + a[2:])})()
|
||||
g.device = object()
|
||||
g.button("left", True)
|
||||
g.key(42, True)
|
||||
g.release_all()
|
||||
self.assertEqual(g.log[-2:], [("button", 0x110, False), ("key", 42, False)])
|
||||
self.assertEqual((g.held, g.keys), (set(), set()))
|
||||
|
||||
def test_resuming_after_a_pause_lets_go_of_everything(self):
|
||||
t = self.t
|
||||
log, queue = [], [1]
|
||||
|
||||
class L:
|
||||
def __getattr__(self, name):
|
||||
if name == "ei_get_event":
|
||||
return lambda ei: queue.pop(0) if queue else None
|
||||
return {"ei_event_get_type": lambda ev: t.EV_DEVICE_RESUMED, "ei_event_get_device": lambda ev: "dev",
|
||||
"ei_device_has_capability": lambda d, c: True,
|
||||
"ei_device_button_button": lambda d, c, down: log.append(("button", c, down)),
|
||||
"ei_device_keyboard_key": lambda d, c, down: log.append(("key", c, down))}.get(name, lambda *a: 0)
|
||||
|
||||
g = t.Gamescope.__new__(t.Gamescope)
|
||||
g.L, g.ei, g.fd, g.device, g.sequence, g.alive = L(), None, None, None, 0, True
|
||||
g.held, g.keys = {0x110}, {42}
|
||||
g.frame = lambda: None
|
||||
old = t.select.select
|
||||
t.select.select = lambda *a: ([], [], [])
|
||||
try:
|
||||
g.pump()
|
||||
finally:
|
||||
t.select.select = old
|
||||
self.assertEqual(sorted(log), [("button", 0x110, False), ("key", 42, False)])
|
||||
self.assertEqual(g.device, "dev")
|
||||
|
||||
def test_text_uses_shift_for_capitals(self):
|
||||
class Keys(self.t.Gamescope):
|
||||
def __init__(self):
|
||||
self.pressed = []
|
||||
|
||||
def key(self, code, down):
|
||||
self.pressed.append((code, down))
|
||||
k = Keys()
|
||||
k.text("Hié") # the é has no key on a US layout and is left out
|
||||
self.assertEqual(k.pressed, [(42, True), (35, True), (35, False), (42, False), (23, True), (23, False)])
|
||||
|
||||
|
||||
class ServerChecks(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
import server
|
||||
cls.s = server
|
||||
|
||||
def test_touch_event_keeps_known_fields(self):
|
||||
ev = self.s.touch_event
|
||||
self.assertEqual(ev({"fx": 0.5, "fy": 2, "window": 5, "display": ":1"}), {"fx": 0.5, "fy": 1.0, "window": 5, "display": ":1"})
|
||||
self.assertEqual(ev({"button": "left", "window": 5, "display": ":0"}), {"button": "left", "down": True, "window": 5, "display": ":0"})
|
||||
self.assertEqual(ev({"button": "right"}), {"button": "right", "down": True})
|
||||
self.assertEqual(ev({"key": 30, "down": False}), {"key": 30, "down": False})
|
||||
self.assertEqual(ev({"scroll": [0, 1e9]}), {"scroll": [0.0, 5000.0]})
|
||||
self.assertEqual(ev({"dx": 3, "other": 1}), {"dx": 3.0})
|
||||
|
||||
def test_touch_event_rejects_bad_ones(self):
|
||||
for bad in (None, {}, {"fx": 0.5, "fy": 0.5}, {"fx": "1", "fy": 0, "window": 1, "display": ":1"}, {"button": "side"},
|
||||
{"fx": 0.5, "fy": 0.5, "window": 1}, {"fx": 0.5, "fy": 0.5, "window": 1, "display": ":1;x"},
|
||||
{"window": 1, "display": ":1"},
|
||||
{"key": 0}, {"key": 999}, {"key": True}, {"scroll": [1]}, {"text": ""}, {"text": "x" * 501},
|
||||
{"fx": 0.1, "fy": 0.1, "window": True, "display": ":1"}):
|
||||
with self.assertRaises(self.s.Failure, msg=repr(bad)):
|
||||
self.s.touch_event(bad)
|
||||
|
||||
def test_panel_stream_is_the_window_and_checked(self):
|
||||
cmd = self.s.stream_command("src=panel&window=10485777&display=:1&h=720&fps=30")
|
||||
self.assertIn("DISPLAY=:1 ffmpeg", cmd)
|
||||
self.assertIn("-window_id 10485777 -i :1", cmd)
|
||||
for bad in ("src=panel&window=1;rm&display=:1", "src=panel&window=1&display=:1;x", "src=panel&display=:1"):
|
||||
with self.assertRaises(self.s.Failure):
|
||||
self.s.stream_command(bad + "&h=720&fps=30")
|
||||
|
||||
def test_touch_agent_runs_the_helper_with_nothing_to_copy(self):
|
||||
agent = self.s.TouchAgent()
|
||||
self.assertEqual(agent.deliver(lambda m: None), "")
|
||||
cmd = agent.command()
|
||||
self.assertTrue(cmd.startswith("python3 -u -c '"))
|
||||
self.assertIn("frame_touch", cmd)
|
||||
|
||||
def test_batch_limit(self):
|
||||
with self.assertRaises(self.s.Failure):
|
||||
self.s.remote_touch({"events": [{"dx": 1}] * (self.s.INPUT_BATCH_LIMIT + 1)})
|
||||
|
||||
|
||||
@unittest.skipUnless(shutil.which("node"), "needs node")
|
||||
class PageGestures(unittest.TestCase):
|
||||
"""Control's gestures and queue (tests/page/ctrl_gestures.mjs runs the real functions from index.html)."""
|
||||
|
||||
def test_gestures(self):
|
||||
r = subprocess.run(["node", str(ROOT / "tests" / "page" / "ctrl_gestures.mjs")], capture_output=True, text=True,
|
||||
timeout=60)
|
||||
self.assertEqual(r.returncode, 0, r.stdout + r.stderr)
|
||||
|
||||
|
||||
class Script(unittest.TestCase):
|
||||
def test_helper_compiles_on_the_frames_python(self):
|
||||
# The Frame runs it with its own python3 (3.13 on SteamOS 0.4.1); stdlib and ctypes only.
|
||||
src = (ROOT / "ui/frame_touch.py").read_text()
|
||||
compile(src, "frame_touch.py", "exec")
|
||||
for mod in ("import ctypes", "import json", "import select"):
|
||||
self.assertIn(mod, src)
|
||||
self.assertNotIn("import requests", src)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -1,264 +0,0 @@
|
||||
"""Opt-in session worker ON the Frame; no root, extra apps, or power actions.
|
||||
|
||||
One worker per user, shared by desktop and phone. State survives companion
|
||||
connections, not headset reboots. See docs/family-comfort.md for guarantees.
|
||||
"""
|
||||
import contextlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import uuid
|
||||
|
||||
from frame_steam import Page
|
||||
from frame_status import battery, thermal_alerts, activity_level
|
||||
|
||||
ROOT = Path.home() / '.local/state/frame-control/comfort'
|
||||
VRCMD = '/opt/steamvr/bin/linuxarm64/vrcmd'
|
||||
HOME_JS = """(async () => {
|
||||
SteamUIStore.Navigate('/library/home');
|
||||
await SteamClient.OpenVR.VROverlay.ShowDashboard('valve.steam.gamepadui.main');
|
||||
if (!await SteamClient.OpenVR.VROverlay.IsDashboardVisible()) throw Error('Steam dashboard did not open');
|
||||
return {path: location.pathname};
|
||||
})()"""
|
||||
|
||||
|
||||
def clock():
|
||||
# CLOCK_BOOTTIME includes headset suspend; wall-clock corrections don't alter limits.
|
||||
return time.clock_gettime(time.CLOCK_BOOTTIME)
|
||||
|
||||
|
||||
def boot():
|
||||
return Path('/proc/sys/kernel/random/boot_id').read_text().strip()
|
||||
|
||||
|
||||
def validate(body):
|
||||
if not isinstance(body, dict) or body.get('action') not in ('status', 'start', 'cancel'):
|
||||
raise ValueError('Choose status, start or cancel')
|
||||
if body['action'] == 'start':
|
||||
for key, low, high in (('minutes', 1, 240), ('breakMinutes', 0, 120), ('stillMinutes', 0, 240)):
|
||||
n = body.get(key)
|
||||
if type(n) is not int or not low <= n <= high:
|
||||
raise ValueError(f'{key} must be a whole number from {low} to {high}')
|
||||
for key in ('batteryAlert', 'heatAlert'):
|
||||
if type(body.get(key)) is not bool:
|
||||
raise ValueError(f'{key} must be true or false')
|
||||
return body
|
||||
|
||||
|
||||
def new_session(body, now, boot_id):
|
||||
return {'id': uuid.uuid4().hex, 'boot': boot_id, 'active': True,
|
||||
'options': {k: body[k] for k in ('minutes', 'breakMinutes', 'stillMinutes', 'batteryAlert', 'heatAlert')},
|
||||
'started': now, 'deadline': now + body['minutes'] * 60, 'lastSample': now,
|
||||
'used': 0, 'nextBreak': body['breakMinutes'] * 60, 'stillSent': False,
|
||||
'warned': None, 'events': [], 'seq': 0, 'latched': [], 'error': None}
|
||||
|
||||
|
||||
def event(s, kind, message):
|
||||
s['seq'] += 1
|
||||
s['events'].append({'id': s['id'] + ':' + str(s['seq']), 'kind': kind,
|
||||
'message': message, 'time': time.time()})
|
||||
s['events'] = s['events'][-40:]
|
||||
|
||||
|
||||
def notify(message):
|
||||
r = subprocess.run([VRCMD, '--notify', 'Frame Control: ' + message],
|
||||
capture_output=True, text=True, timeout=20)
|
||||
if r.returncode or 'succeeded' not in r.stdout:
|
||||
raise RuntimeError('SteamVR could not show the reminder: ' + (r.stderr or r.stdout)[-300:])
|
||||
|
||||
|
||||
def home():
|
||||
# A total process deadline also bounds a CDP peer that keeps sending events
|
||||
# without completing the request. Keep cancellation ordered after this action.
|
||||
r = subprocess.run([sys.executable, str(Path(__file__).resolve()), '--home'],
|
||||
capture_output=True, text=True, timeout=15)
|
||||
if r.returncode:
|
||||
raise RuntimeError('Steam Home failed: ' + (r.stdout or r.stderr)[-300:])
|
||||
|
||||
|
||||
def open_home():
|
||||
page = Page()
|
||||
try:
|
||||
result = page.eval(HOME_JS)
|
||||
if result.get('path') != '/routes/library/home':
|
||||
raise RuntimeError('Steam did not navigate Home')
|
||||
finally:
|
||||
page.sock.close()
|
||||
|
||||
|
||||
def tick(s, now, sample, warn=notify, go_home=home, read_clock=clock):
|
||||
"""One deterministic step; injected actions/samples also exercise a fake Frame."""
|
||||
if not s.get('active'):
|
||||
return
|
||||
o = s['options']
|
||||
s['heartbeat'] = now
|
||||
delta = max(0, min(30, now - s['lastSample']))
|
||||
s['lastSample'] = now
|
||||
level = sample.get('activity')
|
||||
b = sample.get('battery') or {}
|
||||
s['unavailable'] = []
|
||||
if o['batteryAlert'] and b.get('percent') is None:
|
||||
s['unavailable'].append('battery')
|
||||
if o['heatAlert'] and sample.get('thermal') is None:
|
||||
s['unavailable'].append('temperature')
|
||||
if (o['breakMinutes'] or o['stillMinutes']) and level is None:
|
||||
s['unavailable'].append('activity')
|
||||
s['activity'] = level
|
||||
if level in (1, 2):
|
||||
s['used'] += delta
|
||||
elif level is not None:
|
||||
s['used'] = 0
|
||||
s['nextBreak'] = o['breakMinutes'] * 60
|
||||
s['stillSent'] = False
|
||||
# Missing samples never count as time worn. No catch-up burst after a disconnect.
|
||||
if now >= s['deadline'] - 60 and s['warned'] is None:
|
||||
warn('One minute left. Save your progress; Steam Home will open.')
|
||||
s['warned'] = max(now, read_clock())
|
||||
event(s, 'warning', 'One minute left. Save your progress; Steam Home will open.')
|
||||
if s['warned'] is not None and now >= max(s['deadline'], s['warned'] + 60):
|
||||
go_home()
|
||||
s['active'] = False
|
||||
event(s, 'finished', 'Session ended: Steam Home opened. Your game is still running.')
|
||||
return
|
||||
if o['breakMinutes'] and s['used'] >= s['nextBreak']:
|
||||
warn('Time for a break. Take off the headset and rest your eyes.')
|
||||
event(s, 'break', 'Time for a break. Take off the headset and rest your eyes.')
|
||||
s['nextBreak'] = s['used'] + o['breakMinutes'] * 60
|
||||
if o['stillMinutes'] and not s['stillSent'] and s['used'] >= o['stillMinutes'] * 60:
|
||||
event(s, 'still', f"Headset still active after {o['stillMinutes']} active minute(s). Check in with the wearer.")
|
||||
s['stillSent'] = True
|
||||
low = b.get('percent') is not None and b['percent'] <= 15 and b.get('status') == 'Discharging'
|
||||
hot = sample.get('thermal')
|
||||
for kind, enabled, value, message in (
|
||||
('battery', o['batteryAlert'], low if b else None, 'Frame battery is low (15% or less).'),
|
||||
('heat', o['heatAlert'], bool(hot) if hot is not None else None,
|
||||
'Frame reports a hot/critical thermal trip or battery overheat. Ask the wearer to take a break.')):
|
||||
if enabled and value and kind not in s['latched']:
|
||||
event(s, kind, message)
|
||||
s['latched'].append(kind)
|
||||
elif value is False and kind in s['latched']:
|
||||
# Battery hysteresis prevents repeated alerts around 15%.
|
||||
if kind != 'battery' or b.get('status') == 'Charging' or (b.get('percent') or 0) >= 20:
|
||||
s['latched'].remove(kind)
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def locked(name='state.lock', nonblocking=False):
|
||||
import fcntl # only needed ON the Linux headset, not by desktop validation/tests
|
||||
ROOT.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||
with (ROOT / name).open('a') as f:
|
||||
fcntl.flock(f, fcntl.LOCK_EX | (fcntl.LOCK_NB if nonblocking else 0))
|
||||
yield f
|
||||
|
||||
|
||||
def read_state():
|
||||
try:
|
||||
state = json.loads((ROOT / 'session.json').read_text())
|
||||
if not isinstance(state, dict):
|
||||
raise ValueError('Saved session must be an object')
|
||||
return state
|
||||
except FileNotFoundError:
|
||||
return {'active': False, 'events': []}
|
||||
except (ValueError, UnicodeDecodeError):
|
||||
# Preserve the unreadable state for diagnosis, then allow a new session.
|
||||
(ROOT / 'session.json').replace(ROOT / ('session-unreadable-' + uuid.uuid4().hex + '.json'))
|
||||
return {'active': False, 'events': [],
|
||||
'error': 'Saved session was unreadable. Start a new session.'}
|
||||
|
||||
|
||||
def save(s):
|
||||
p = ROOT / 'session.tmp'
|
||||
p.write_text(json.dumps(s))
|
||||
p.chmod(0o600)
|
||||
p.replace(ROOT / 'session.json')
|
||||
|
||||
|
||||
def current(s, now):
|
||||
if s.get('active') and s.get('boot') != boot():
|
||||
s['active'] = False
|
||||
s['error'] = 'Headset restarted. Start a new session.'
|
||||
out = dict(s)
|
||||
out['time'] = time.time() # event age uses the Frame's clock, not the phone's
|
||||
out['remaining'] = max(0, max(s.get('deadline', now), (s.get('warned') or 0) + 60) - now) if s.get('active') else 0
|
||||
beat = s.get('heartbeat', s.get('started', now)) # a hand-edited state may lack either
|
||||
if s.get('active') and now - beat > 90:
|
||||
out['error'] = 'Session worker is not responding. Timer enforcement is unverified; cancel and start again.'
|
||||
return out
|
||||
|
||||
|
||||
def watch():
|
||||
try:
|
||||
with locked('worker.lock', nonblocking=True) as worker:
|
||||
while True:
|
||||
with locked():
|
||||
s = current(read_state(), clock())
|
||||
if not s.get('active'):
|
||||
save(s)
|
||||
# Release ownership before state.lock: a concurrent start
|
||||
# cannot miss the gap between an old worker and its exit.
|
||||
import fcntl
|
||||
fcntl.flock(worker, fcntl.LOCK_UN)
|
||||
return
|
||||
try:
|
||||
b = battery()
|
||||
hot = thermal_alerts()
|
||||
if b and b.get('health') == 'Overheat':
|
||||
hot = (hot or []) + ['battery']
|
||||
tick(s, clock(), {'battery': b, 'thermal': hot, 'activity': activity_level()})
|
||||
s['error'] = None
|
||||
except Exception as e:
|
||||
error = str(e)
|
||||
if s.get('error') != error:
|
||||
event(s, 'error', 'Session action failed: ' + error)
|
||||
s['error'] = error
|
||||
save(s)
|
||||
time.sleep(5)
|
||||
except BlockingIOError:
|
||||
pass # another connection already started the single worker
|
||||
|
||||
|
||||
def command(body):
|
||||
validate(body)
|
||||
with locked():
|
||||
s = current(read_state(), clock())
|
||||
if body['action'] == 'start':
|
||||
if s.get('active'):
|
||||
raise ValueError('A session is already running. Cancel it before starting another.')
|
||||
s = new_session(body, clock(), boot())
|
||||
event(s, 'started', 'Session started. Steam Home opens at the limit; games are not closed.')
|
||||
elif body['action'] == 'cancel':
|
||||
s['active'] = False
|
||||
s['error'] = None
|
||||
if s.get('id'):
|
||||
event(s, 'cancelled', 'Session timer and monitoring cancelled.')
|
||||
save(s)
|
||||
if body['action'] == 'start':
|
||||
try:
|
||||
subprocess.Popen([sys.executable, str(Path(__file__).resolve()), '--watch'],
|
||||
stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
|
||||
start_new_session=True, close_fds=True)
|
||||
except OSError as e:
|
||||
s['active'] = False
|
||||
s['error'] = 'Could not start session worker: ' + str(e)
|
||||
event(s, 'error', s['error'])
|
||||
save(s)
|
||||
raise
|
||||
return current(s, clock())
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
if sys.argv[1:] == ['--watch']:
|
||||
watch()
|
||||
else:
|
||||
try:
|
||||
if sys.argv[1:] == ['--home']:
|
||||
open_home()
|
||||
print(json.dumps({'home': True}))
|
||||
else:
|
||||
print(json.dumps(command(json.loads(sys.argv[1]))))
|
||||
except Exception as e:
|
||||
print(json.dumps({'error': str(e)}))
|
||||
sys.exit(1)
|
||||
+24
-56
@@ -156,59 +156,27 @@ def flatpaks():
|
||||
return out
|
||||
|
||||
|
||||
def thermal_alerts():
|
||||
"""Use the kernel's per-zone hot/critical trips, never a guessed chip limit."""
|
||||
alerts, known = [], False
|
||||
for z in glob.glob("/sys/class/thermal/thermal_zone*"):
|
||||
t = num(z + "/temp", 0.001)
|
||||
for trip in glob.glob(z + "/trip_point_*_type"):
|
||||
if read(trip) not in ("hot", "critical"):
|
||||
continue
|
||||
limit = num(trip[:-4] + "temp", 0.001)
|
||||
if t is not None and limit is not None and limit > 0:
|
||||
known = True
|
||||
if t >= limit:
|
||||
alerts.append({"zone": read(z + "/type"), "tempC": t, "limitC": limit})
|
||||
return alerts if known else None
|
||||
|
||||
|
||||
def activity_level():
|
||||
try:
|
||||
rows = json.loads(run("/opt/steamvr/bin/linuxarm64/vrcmd", "--stats"))
|
||||
if not isinstance(rows, list):
|
||||
return None
|
||||
return next((r.get("activity_level") for r in rows
|
||||
if isinstance(r, dict) and r.get("operation") == "status"), None)
|
||||
except (ValueError, TypeError):
|
||||
return None
|
||||
|
||||
|
||||
def main():
|
||||
uptime = read("/proc/uptime")
|
||||
procs = process_names()
|
||||
print(json.dumps({
|
||||
"time": time.time(),
|
||||
"hostname": socket.gethostname(),
|
||||
"os": os_release(),
|
||||
"uptime": float(uptime.split()[0]) if uptime else None,
|
||||
"battery": battery(),
|
||||
"power": power_source(),
|
||||
"disk": {"root": disk("/"), "home": disk("/home")},
|
||||
"memory": memory(),
|
||||
"temp": max_temp(),
|
||||
"wifi": wifi(),
|
||||
"ip": ip_addr(),
|
||||
"volume": volume(),
|
||||
"services": {
|
||||
"steamvr": "vrserver" in procs,
|
||||
"desktop": "plasmashell" in procs,
|
||||
"lepton": port_listening(5555),
|
||||
"rdp": "xrdp" in procs,
|
||||
},
|
||||
"games": games(),
|
||||
"flatpaks": flatpaks(),
|
||||
}))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
uptime = read("/proc/uptime")
|
||||
procs = process_names()
|
||||
print(json.dumps({
|
||||
"time": time.time(),
|
||||
"hostname": socket.gethostname(),
|
||||
"os": os_release(),
|
||||
"uptime": float(uptime.split()[0]) if uptime else None,
|
||||
"battery": battery(),
|
||||
"power": power_source(),
|
||||
"disk": {"root": disk("/"), "home": disk("/home")},
|
||||
"memory": memory(),
|
||||
"temp": max_temp(),
|
||||
"wifi": wifi(),
|
||||
"ip": ip_addr(),
|
||||
"volume": volume(),
|
||||
"services": {
|
||||
"steamvr": "vrserver" in procs,
|
||||
"desktop": "plasmashell" in procs,
|
||||
"lepton": port_listening(5555),
|
||||
"rdp": "xrdp" in procs,
|
||||
},
|
||||
"games": games(),
|
||||
"flatpaks": flatpaks(),
|
||||
}))
|
||||
@@ -0,0 +1,446 @@
|
||||
"""Touch and direct input for the Steam Frame's panels. Frame Control's server runs this ON the Frame.
|
||||
|
||||
gamescope, the Frame's compositor, serves Valve's own input injection: an EIS
|
||||
socket (libei's server side), which Steam uses to feed it Remote Play input.
|
||||
This connects to it with libei, which is on the SteamOS image, and points,
|
||||
clicks, scrolls and types into the panel that has focus in the headset: the
|
||||
one the wearer last used. No install, and it reaches every panel, on either
|
||||
of gamescope's X displays (see docs/streaming.md).
|
||||
|
||||
python3 frame_touch.py focus print the focused panel as JSON
|
||||
python3 frame_touch.py panels print every app panel as JSON, and which has focus
|
||||
python3 frame_touch.py read events on stdin, one JSON object (or list) per line:
|
||||
{"fx": 0.5, "fy": 0.2, "window": 123, "display": ":1"}
|
||||
pointer to that fraction of that panel; any event can
|
||||
name its panel, and goes nowhere if another has focus
|
||||
{"dx": 4, "dy": -2} pointer by that much
|
||||
{"button": "left", "down": true} left, right or middle; "down" false releases
|
||||
{"scroll": [0, 120]} by pixels; positive y scrolls down
|
||||
{"key": 30, "down": true} a Linux (evdev) key code, as the page maps KeyboardEvent.code
|
||||
{"text": "hello"} printable ASCII, typed on a US layout
|
||||
|
||||
Status goes to stdout, one JSON object per line: {"state": "ready" | "error", ...}.
|
||||
Standard library only (ctypes for libei), like the rest of what runs on the Frame.
|
||||
"""
|
||||
import ctypes
|
||||
import json
|
||||
import os
|
||||
import select
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
SOCKET = "/run/user/{uid}/gamescope-0-ei" # filled in on the Frame (Windows has no getuid; the tests import this)
|
||||
BUTTONS = {"left": 0x110, "right": 0x111, "middle": 0x112} # BTN_LEFT, BTN_RIGHT, BTN_MIDDLE
|
||||
SHIFT = 42 # KEY_LEFTSHIFT
|
||||
# Printable ASCII on a US layout: character -> (evdev key code, shifted).
|
||||
ROWS = [("1234567890-=", "!@#$%^&*()_+", 2), ("qwertyuiop[]", "QWERTYUIOP{}", 16),
|
||||
("asdfghjkl;'`", 'ASDFGHJKL:"~', 30), ("\\zxcvbnm,./", "|ZXCVBNM<>?", 43)]
|
||||
ASCII = {" ": (57, False), "\n": (28, False), "\t": (15, False)}
|
||||
for plain, shifted, first in ROWS:
|
||||
for i, (a, b) in enumerate(zip(plain, shifted)):
|
||||
ASCII[a], ASCII[b] = (first + i, False), (first + i, True)
|
||||
|
||||
# libei's event types and device capabilities (libei.h, libei 1.4).
|
||||
EV_CONNECT, EV_DISCONNECT, EV_SEAT_ADDED, EV_DEVICE_ADDED, EV_DEVICE_REMOVED = 1, 2, 3, 5, 6
|
||||
EV_DEVICE_PAUSED, EV_DEVICE_RESUMED = 7, 8
|
||||
CAP_POINTER, CAP_ABSOLUTE, CAP_KEYBOARD, CAP_SCROLL, CAP_BUTTON = 1, 2, 4, 16, 32
|
||||
|
||||
|
||||
def say(state, **more):
|
||||
print(json.dumps({"state": state, **more}), flush=True)
|
||||
|
||||
|
||||
# ---- which panel has focus -----------------------------------------------------
|
||||
|
||||
def xprop_root(display, name):
|
||||
try:
|
||||
out = subprocess.run(["xprop", "-root", name], env=dict(os.environ, DISPLAY=display),
|
||||
capture_output=True, text=True, timeout=5).stdout
|
||||
except (OSError, subprocess.SubprocessError):
|
||||
return []
|
||||
values = out.split("=", 1)[1] if "=" in out else ""
|
||||
return [int(v) for v in values.replace(",", " ").split() if v.isdigit()]
|
||||
|
||||
|
||||
def window_info(display, window):
|
||||
"""Name and geometry of a window on one X display, or None if it isn't there."""
|
||||
try:
|
||||
which = ["-root"] if window == "root" else ["-id", str(window)]
|
||||
out = subprocess.run(["xwininfo", *which], env=dict(os.environ, DISPLAY=display),
|
||||
capture_output=True, text=True, timeout=5).stdout
|
||||
except (OSError, subprocess.SubprocessError):
|
||||
return None
|
||||
if "IsViewable" not in out:
|
||||
return None
|
||||
info = {}
|
||||
for line in out.splitlines():
|
||||
line = line.strip()
|
||||
if line.startswith("xwininfo: Window id:"):
|
||||
info["name"] = line.split('"', 1)[1].rsplit('"', 1)[0] if '"' in line else ""
|
||||
for key, field in (("Absolute upper-left X:", "x"), ("Absolute upper-left Y:", "y"),
|
||||
("Width:", "width"), ("Height:", "height")):
|
||||
if line.startswith(key):
|
||||
info[field] = int(line.split(":", 1)[1])
|
||||
return info if "width" in info else None
|
||||
|
||||
|
||||
def displays():
|
||||
return sorted(f":{n[1:]}" for n in os.listdir("/tmp/.X11-unix") if n[1:].isdigit())
|
||||
|
||||
|
||||
def window_pid(display, window):
|
||||
try:
|
||||
out = subprocess.run(["xprop", "-id", str(window), "_NET_WM_PID"], env=dict(os.environ, DISPLAY=display),
|
||||
capture_output=True, text=True, timeout=5).stdout
|
||||
except (OSError, subprocess.SubprocessError):
|
||||
return None
|
||||
value = out.rsplit("=", 1)[-1].strip() if "=" in out else ""
|
||||
return int(value) if value.isdigit() else None
|
||||
|
||||
|
||||
def locate(window, pid):
|
||||
"""The display a focusable window is on, with its name and geometry.
|
||||
|
||||
Window ids are per X server, so :0 and :1 can both have one; the pid gamescope
|
||||
lists with it (GAMESCOPE_FOCUSABLE_WINDOWS) tells them apart.
|
||||
"""
|
||||
found = []
|
||||
for display in displays():
|
||||
info = window_info(display, window)
|
||||
if info:
|
||||
found.append((display, info))
|
||||
if len(found) > 1 and pid:
|
||||
found = [f for f in found if window_pid(f[0], window) == pid] or found
|
||||
if not found:
|
||||
return None
|
||||
display, info = found[0]
|
||||
root = window_info(display, "root") or {}
|
||||
return {"window": window, "display": display, **info,
|
||||
"root": [root.get("width", info["width"]), root.get("height", info["height"])]}
|
||||
|
||||
|
||||
def focusable():
|
||||
"""gamescope's focusable windows as (window, app id, pid)."""
|
||||
t = xprop_root(":0", "GAMESCOPE_FOCUSABLE_WINDOWS")
|
||||
return [tuple(t[i:i + 3]) for i in range(0, len(t) - 2, 3)]
|
||||
|
||||
|
||||
def focus_display():
|
||||
"""The display of the focused window, from GAMESCOPE_FOCUS_DISPLAY on :0's root.
|
||||
|
||||
gamescope writes the name (":1") as 32-bit items, so its first four bytes land,
|
||||
little-endian, in the first value: 12602 is 0x313A, ":1" (steamcompmgr.cpp;
|
||||
seen 2026-09-29).
|
||||
"""
|
||||
values = xprop_root(":0", "GAMESCOPE_FOCUS_DISPLAY")
|
||||
if not values:
|
||||
return None
|
||||
name = (values[0] & 0xFFFFFFFF).to_bytes(4, "little").split(b"\0", 1)[0].decode("ascii", "replace")
|
||||
return name if name[:1] == ":" and name[1:].isdigit() else None
|
||||
|
||||
|
||||
def focus_now():
|
||||
"""Just which window and display have focus: two property reads, for checking a press."""
|
||||
window = (xprop_root(":0", "GAMESCOPE_FOCUSED_WINDOW") or [0])[0]
|
||||
return (window or None, focus_display() if window else None)
|
||||
|
||||
|
||||
def focus():
|
||||
"""The panel that has focus in the headset: window, display, name and sizes (gamescope
|
||||
publishes the window and its display on :0's root)."""
|
||||
window = (xprop_root(":0", "GAMESCOPE_FOCUSED_WINDOW") or [0])[0]
|
||||
if not window:
|
||||
return {"window": None}
|
||||
app, pid = next(((a, p) for w, a, p in focusable() if w == window), (None, None))
|
||||
display = focus_display()
|
||||
info = window_info(display, window) if display else None
|
||||
if info:
|
||||
root = window_info(display, "root") or {}
|
||||
panel = {"window": window, "display": display, **info,
|
||||
"root": [root.get("width", info["width"]), root.get("height", info["height"])]}
|
||||
else:
|
||||
panel = locate(window, pid) # no display published: tell them apart by pid
|
||||
return {**panel, "app": app} if panel else {"window": None}
|
||||
|
||||
|
||||
def panels():
|
||||
"""Every app panel (gamescope's focusable windows), for watching one that hasn't focus."""
|
||||
now = focus()
|
||||
found = []
|
||||
for window, app, pid in focusable():
|
||||
panel = locate(window, pid)
|
||||
if panel and panel["width"] > 1 and panel["height"] > 1 and \
|
||||
not any(f["window"] == window and f["display"] == panel["display"] for f in found):
|
||||
panel.pop("root", None)
|
||||
found.append({**panel, "app": app, "focused": (window, panel["display"]) ==
|
||||
(now.get("window"), now.get("display"))})
|
||||
return {"focus": now.get("window"), "focus_display": now.get("display"), "panels": found}
|
||||
|
||||
|
||||
def to_root(panel, fx, fy):
|
||||
"""A point given as a fraction of the panel, in the root coordinates gamescope's pointer uses.
|
||||
|
||||
gamescope fits each panel's window to its display, so a 1920x1080 window on a
|
||||
1280x720 display takes pointer positions at two thirds scale (verified 2026-09-29).
|
||||
"""
|
||||
rw, rh = panel["root"]
|
||||
w, h = panel["width"], panel["height"]
|
||||
s = min(rw / w, rh / h)
|
||||
ox, oy = (rw - w * s) / 2, (rh - h * s) / 2
|
||||
fx, fy = min(max(fx, 0.0), 1.0), min(max(fy, 0.0), 1.0)
|
||||
return ox + fx * (w * s - 1), oy + fy * (h * s - 1)
|
||||
|
||||
|
||||
# ---- gamescope's input socket ----------------------------------------------------
|
||||
|
||||
def libei():
|
||||
L = ctypes.CDLL("libei.so.1")
|
||||
vp, c = ctypes.c_void_p, ctypes
|
||||
sig = {
|
||||
"ei_new_sender": (vp, [vp]), "ei_configure_name": (None, [vp, c.c_char_p]),
|
||||
"ei_setup_backend_socket": (c.c_int, [vp, c.c_char_p]), "ei_get_fd": (c.c_int, [vp]),
|
||||
"ei_dispatch": (None, [vp]), "ei_get_event": (vp, [vp]), "ei_event_get_type": (c.c_int, [vp]),
|
||||
"ei_event_unref": (vp, [vp]), "ei_event_get_seat": (vp, [vp]), "ei_event_get_device": (vp, [vp]),
|
||||
"ei_device_has_capability": (c.c_bool, [vp, c.c_int]), "ei_now": (c.c_uint64, [vp]),
|
||||
"ei_device_start_emulating": (None, [vp, c.c_uint32]), "ei_device_stop_emulating": (None, [vp]),
|
||||
"ei_device_frame": (None, [vp, c.c_uint64]),
|
||||
"ei_device_pointer_motion": (None, [vp, c.c_double, c.c_double]),
|
||||
"ei_device_pointer_motion_absolute": (None, [vp, c.c_double, c.c_double]),
|
||||
"ei_device_button_button": (None, [vp, c.c_uint32, c.c_bool]),
|
||||
"ei_device_scroll_delta": (None, [vp, c.c_double, c.c_double]),
|
||||
"ei_device_keyboard_key": (None, [vp, c.c_uint32, c.c_bool]),
|
||||
"ei_unref": (vp, [vp]),
|
||||
}
|
||||
for name, (res, args) in sig.items():
|
||||
f = getattr(L, name)
|
||||
f.restype, f.argtypes = res, args
|
||||
return L
|
||||
|
||||
|
||||
class Gamescope:
|
||||
"""One connection to gamescope's EIS socket and its virtual input device."""
|
||||
|
||||
def __init__(self):
|
||||
self.L = L = libei()
|
||||
self.ei = L.ei_new_sender(None)
|
||||
L.ei_configure_name(self.ei, b"Frame Control")
|
||||
if L.ei_setup_backend_socket(self.ei, SOCKET.format(uid=os.getuid()).encode()) != 0:
|
||||
raise RuntimeError("Couldn't reach gamescope's input socket. Is the headset on?")
|
||||
self.fd = L.ei_get_fd(self.ei)
|
||||
self.device, self.sequence, self.held, self.keys, self.alive = None, 0, set(), set(), True
|
||||
|
||||
def pump(self, wait=0.0):
|
||||
"""Handle gamescope's events; False once it has disconnected."""
|
||||
select.select([self.fd], [], [], wait)
|
||||
self.L.ei_dispatch(self.ei)
|
||||
alive = True
|
||||
while True:
|
||||
ev = self.L.ei_get_event(self.ei)
|
||||
if not ev:
|
||||
return alive
|
||||
kind = self.L.ei_event_get_type(ev)
|
||||
if kind == EV_SEAT_ADDED:
|
||||
seat = self.L.ei_event_get_seat(ev)
|
||||
# Variadic, ending in 0 (NULL): ask for everything we send.
|
||||
self.L.ei_seat_bind_capabilities(ctypes.c_void_p(seat), *map(ctypes.c_int, (
|
||||
CAP_POINTER, CAP_ABSOLUTE, CAP_BUTTON, CAP_SCROLL, CAP_KEYBOARD, 0)))
|
||||
elif kind == EV_DEVICE_RESUMED:
|
||||
device = self.L.ei_event_get_device(ev)
|
||||
if self.L.ei_device_has_capability(device, CAP_ABSOLUTE):
|
||||
self.sequence += 1
|
||||
self.L.ei_device_start_emulating(device, self.sequence)
|
||||
self.device = device
|
||||
# Releases that arrived while it was paused were dropped: let go of
|
||||
# everything now, so the headset and this agent agree nothing is held.
|
||||
if self.held or self.keys:
|
||||
self.release_all()
|
||||
elif kind in (EV_DEVICE_PAUSED, EV_DEVICE_REMOVED):
|
||||
if self.L.ei_event_get_device(ev) == self.device:
|
||||
self.device = None
|
||||
elif kind == EV_DISCONNECT:
|
||||
self.device, alive, self.alive = None, False, False
|
||||
self.L.ei_event_unref(ev)
|
||||
|
||||
def wait_ready(self, timeout=5):
|
||||
end = time.time() + timeout
|
||||
while self.device is None and time.time() < end:
|
||||
if not self.pump(0.1):
|
||||
break
|
||||
if self.device is None:
|
||||
raise RuntimeError("gamescope closed its input socket" if not self.alive
|
||||
else "gamescope didn't offer an input device")
|
||||
|
||||
def frame(self):
|
||||
self.L.ei_device_frame(self.device, self.L.ei_now(self.ei))
|
||||
self.L.ei_dispatch(self.ei)
|
||||
|
||||
def move_to(self, x, y):
|
||||
self.L.ei_device_pointer_motion_absolute(self.device, x, y)
|
||||
self.frame()
|
||||
|
||||
def move_by(self, dx, dy):
|
||||
self.L.ei_device_pointer_motion(self.device, dx, dy)
|
||||
self.frame()
|
||||
|
||||
def button(self, name, down):
|
||||
code = BUTTONS[name]
|
||||
if down == (code in self.held):
|
||||
return # already in that state
|
||||
self.L.ei_device_button_button(self.device, code, down)
|
||||
self.frame()
|
||||
(self.held.add if down else self.held.discard)(code)
|
||||
|
||||
def scroll(self, dx, dy):
|
||||
self.L.ei_device_scroll_delta(self.device, dx, dy)
|
||||
self.frame()
|
||||
|
||||
def key(self, code, down):
|
||||
self.L.ei_device_keyboard_key(self.device, code, down)
|
||||
self.frame()
|
||||
(self.keys.add if down else self.keys.discard)(code)
|
||||
# Paced: a burst of keys can reach the app out of order (seen 2026-09-29).
|
||||
time.sleep(0.008)
|
||||
|
||||
def text(self, text):
|
||||
for ch in text:
|
||||
if ch not in ASCII:
|
||||
continue
|
||||
code, shifted = ASCII[ch]
|
||||
if shifted:
|
||||
self.key(SHIFT, True)
|
||||
self.key(code, True)
|
||||
self.key(code, False)
|
||||
if shifted:
|
||||
self.key(SHIFT, False)
|
||||
|
||||
def release_all(self):
|
||||
"""Let go of every button and key still down, so nothing stays held in the headset."""
|
||||
for code in list(self.held):
|
||||
name = next(n for n, c in BUTTONS.items() if c == code)
|
||||
self.button(name, False)
|
||||
for code in list(self.keys):
|
||||
self.key(code, False)
|
||||
|
||||
|
||||
# ---- events from the server --------------------------------------------------------
|
||||
|
||||
def events(line):
|
||||
try:
|
||||
data = json.loads(line)
|
||||
except ValueError:
|
||||
return []
|
||||
return [e for e in (data if isinstance(data, list) else [data]) if isinstance(e, dict)]
|
||||
|
||||
|
||||
def number(value, limit=100000.0):
|
||||
if isinstance(value, bool) or not isinstance(value, (int, float)) or value != value:
|
||||
raise ValueError("not a number")
|
||||
return max(-limit, min(limit, float(value)))
|
||||
|
||||
|
||||
STALE = [False] # whether the last status said a tap went nowhere
|
||||
|
||||
|
||||
def aimed_elsewhere(event, panel):
|
||||
"""Whether an event names a panel that isn't the one with focus now."""
|
||||
if "window" not in event:
|
||||
return False
|
||||
return (panel.get("window"), panel.get("display")) != (event.get("window"), event.get("display"))
|
||||
|
||||
|
||||
def apply(gs, event, panel):
|
||||
"""Send one event; returns the focused panel it checked against (looked up at most once a second).
|
||||
|
||||
Positions and presses name the panel they were meant for. If focus has moved to
|
||||
another panel since, they go nowhere, so a tap can't land on the wrong one;
|
||||
releases always go, so nothing stays held.
|
||||
"""
|
||||
# Moves may use a focus reading up to a second old; anything that acts (a press, key,
|
||||
# text or scroll) reads it afresh, so it can't land on a panel that took focus since.
|
||||
acts = any(k in event for k in ("button", "key", "text", "scroll")) and event.get("down") is not False
|
||||
if "window" in event:
|
||||
if panel and acts and focus_now() == (panel.get("window"), panel.get("display")):
|
||||
pass # still the same panel (its geometry is re-read on the usual one-second schedule)
|
||||
elif acts or not panel or time.time() - panel.get("_at", 0) > 1 or aimed_elsewhere(event, panel):
|
||||
panel = {**focus(), "_at": time.time()}
|
||||
stale = aimed_elsewhere(event, panel) if "window" in event else False
|
||||
if stale and not (event.get("down") is False and ("button" in event or "key" in event)):
|
||||
say("ready", focus=panel.get("window"), display=panel.get("display"), stale=True) # the page re-syncs
|
||||
STALE[0] = True
|
||||
return panel
|
||||
if STALE[0] and not stale:
|
||||
# Anything that goes through (a trackpad move names no panel) means caught up: stop re-syncing.
|
||||
STALE[0] = False
|
||||
say("ready", focus=(panel or {}).get("window"), display=(panel or {}).get("display"))
|
||||
if "fx" in event and panel and panel.get("window"):
|
||||
gs.move_to(*to_root(panel, number(event["fx"], 1), number(event["fy"], 1)))
|
||||
if "dx" in event or "dy" in event:
|
||||
gs.move_by(number(event.get("dx", 0), 2000), number(event.get("dy", 0), 2000))
|
||||
if event.get("button") in BUTTONS:
|
||||
gs.button(event["button"], event.get("down") is not False)
|
||||
if isinstance(event.get("scroll"), list) and len(event["scroll"]) == 2:
|
||||
gs.scroll(number(event["scroll"][0], 5000), number(event["scroll"][1], 5000))
|
||||
if isinstance(event.get("key"), int) and not isinstance(event["key"], bool) and 0 < event["key"] < 768:
|
||||
gs.key(event["key"], event.get("down") is not False)
|
||||
if isinstance(event.get("text"), str):
|
||||
gs.text(event["text"][:500])
|
||||
return panel
|
||||
|
||||
|
||||
def main():
|
||||
if sys.argv[1:] == ["focus"]:
|
||||
print(json.dumps(focus()))
|
||||
return 0
|
||||
if sys.argv[1:] == ["panels"]:
|
||||
print(json.dumps(panels()))
|
||||
return 0
|
||||
try:
|
||||
gs = Gamescope()
|
||||
gs.wait_ready()
|
||||
except (OSError, RuntimeError) as e:
|
||||
say("error", message=str(e))
|
||||
return 1
|
||||
say("ready", focus=focus().get("window"))
|
||||
stdin, pending, panel = sys.stdin.fileno(), b"", None
|
||||
try:
|
||||
while True:
|
||||
ready, _, _ = select.select([stdin, gs.fd], [], [], 30)
|
||||
if gs.fd in ready and not gs.pump():
|
||||
say("error", message="gamescope closed its input socket")
|
||||
return 1
|
||||
if stdin not in ready:
|
||||
continue
|
||||
chunk = os.read(stdin, 65536)
|
||||
if not chunk:
|
||||
return 0 # the server went away
|
||||
*lines, pending = (pending + chunk).split(b"\n")
|
||||
for line in lines:
|
||||
waited = False
|
||||
for event in events(line):
|
||||
if gs.device is None and waited:
|
||||
continue # still paused: don't wait again for each event of this batch
|
||||
if gs.device is None:
|
||||
waited = True
|
||||
# Paused (gamescope can pause the device): wait a moment; drop this
|
||||
# event if it doesn't come back. Only a disconnect ends the session.
|
||||
try:
|
||||
gs.wait_ready(2)
|
||||
except RuntimeError:
|
||||
if not gs.alive:
|
||||
raise
|
||||
continue
|
||||
try:
|
||||
panel = apply(gs, event, panel)
|
||||
except (ValueError, KeyError, TypeError, OSError):
|
||||
continue # the server checks events; skip anything odd
|
||||
except RuntimeError as e:
|
||||
say("error", message=str(e))
|
||||
return 1
|
||||
finally:
|
||||
if gs.device is not None:
|
||||
gs.release_all() # never leave a button held down in the headset
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
+509
-150
@@ -102,7 +102,6 @@
|
||||
.shelf-head .count { color: var(--muted); font-size: 13px; }
|
||||
.shelf-head .spacer { flex: 1; }
|
||||
.sub { color: var(--muted); font-size: 12.5px; }
|
||||
.sr-only { position: absolute; width: 1px; height: 1px; overflow: hidden; clip-path: inset(50%); white-space: nowrap; }
|
||||
.hint { color: var(--muted); font-size: 12.5px; margin-top: 11px; line-height: 1.5; }
|
||||
|
||||
/* ---- buttons ---- */
|
||||
@@ -148,6 +147,18 @@
|
||||
.viewer .badge { position: absolute; left: 12px; top: 12px; font-size: 11px; letter-spacing: 1.2px; text-transform: uppercase;
|
||||
background: rgba(0,0,0,.6); color: var(--text); padding: 4px 9px; border-radius: 2px; }
|
||||
.viewer .badge.live { left: auto; right: 12px; background: var(--bad); color: #fff; font-weight: 700; }
|
||||
.view-actions { display: flex; gap: 10px; }
|
||||
.ctrl-status { margin-top: 10px; font-size: 13px; color: var(--blue); }
|
||||
.ctrl-status::before { content: "● "; }
|
||||
.ctrl-status.warn { color: var(--warn); }
|
||||
/* Control: the view takes taps, clicks, drags and keys, and none of them zoom or pan it. */
|
||||
.viewer.control { outline: 1px solid var(--blue); }
|
||||
.viewer.control canvas { cursor: crosshair; touch-action: none; }
|
||||
.viewer.control.rel canvas { cursor: move; }
|
||||
.viewer.immersive { position: fixed; z-index: 40; inset: env(safe-area-inset-top) env(safe-area-inset-right) 0 env(safe-area-inset-left);
|
||||
max-height: none; aspect-ratio: auto; border-radius: 0; }
|
||||
.ctrl-keys { margin-top: 10px; }
|
||||
#panelPick { max-width: 240px; height: 30px; }
|
||||
.viewer .stamp { position: absolute; right: 12px; bottom: 12px; font-size: 11px; color: #c7d5e0;
|
||||
background: rgba(0,0,0,.6); padding: 3px 8px; border-radius: 2px; }
|
||||
.viewer .asleep { position: absolute; inset: auto 0 0 0; padding: 14px 18px; text-align: center; color: var(--text);
|
||||
@@ -349,7 +360,7 @@
|
||||
.disp .k2 { color: var(--muted); font-size: 11px; letter-spacing: 1px; text-transform: uppercase; margin: 12px 0 5px; }
|
||||
.disp .seg { flex-wrap: wrap; }
|
||||
.disp .seg button { padding: 0 10px; }
|
||||
.disp input[type=number], #comfort input[type=number] { width: 72px; background: rgba(0,0,0,.28); color: var(--text); border: 1px solid transparent;
|
||||
.disp input[type=number] { width: 72px; background: rgba(0,0,0,.28); color: var(--text); border: 1px solid transparent;
|
||||
border-radius: 3px; height: 32px; padding: 0 8px; font: inherit; font-size: 13px; }
|
||||
/* Touch screens can't hover: keep the library's name and Play button showing. */
|
||||
@media (hover: none) { .capsule .over { opacity: 1; } .capsule:hover { transform: none; } }
|
||||
@@ -391,6 +402,9 @@
|
||||
.grid-top, .grid-3, .and-grid, .and-col { gap: 14px; }
|
||||
.toolbar { gap: 8px; }
|
||||
.toolbar .spacer { display: none; }
|
||||
.view-actions { flex: 1 1 100%; display: grid; grid-template-columns: repeat(4, 1fr); gap: 6px; }
|
||||
.view-actions button { justify-content: center; padding: 0 4px; }
|
||||
#panelPick { flex: 1 1 100%; max-width: none; }
|
||||
.shelf-head { flex-wrap: wrap; row-gap: 8px; }
|
||||
.shelf { grid-template-columns: repeat(auto-fill, minmax(100px, 1fr)); gap: 10px; }
|
||||
.cat-grid { grid-template-columns: 1fr; }
|
||||
@@ -470,16 +484,20 @@
|
||||
<div class="toolbar">
|
||||
<div class="seg" aria-label="What to capture">
|
||||
<button data-view="headset" class="on" title="What the headset shows, via the SteamVR compositor">Headset view</button>
|
||||
<button data-view="flat" title="gamescope's 2D layer: the desktop panel and Steam's flat UI">Desktop panel</button>
|
||||
<button data-view="flat" title="The app panel in use, flat: it stays still however the wearer looks around">Desktop</button>
|
||||
</div>
|
||||
<select id="panelPick" class="small" hidden aria-label="Which panel to show"></select>
|
||||
<div class="seg" id="eyeSeg" aria-label="Crop">
|
||||
<button data-eye="left" class="on" title="One eye, like pointing a camera into one lens">Left eye</button>
|
||||
<button data-eye="both" title="Both eyes side by side">Both eyes</button>
|
||||
</div>
|
||||
<div class="spacer"></div>
|
||||
<button class="action" id="shotBtn">Capture</button>
|
||||
<button id="liveBtn" title="Keep updating: video of the headset view, or repeated captures of the desktop panel">Live</button>
|
||||
<button id="saveBtn" disabled>Save</button>
|
||||
<span class="view-actions">
|
||||
<button class="action" id="shotBtn">Capture</button>
|
||||
<button id="liveBtn" title="Keep updating, as video">Live</button>
|
||||
<button id="ctrlBtn" title="Control the Frame by tapping or clicking on the view (C)">Control</button>
|
||||
<button id="saveBtn" disabled>Save</button>
|
||||
</span>
|
||||
</div>
|
||||
<div class="viewer" id="viewer">
|
||||
<div class="empty" id="viewerEmpty">
|
||||
@@ -501,6 +519,25 @@
|
||||
<div class="asleep" id="asleep" hidden>The headset is showing a blank frame. It's probably asleep or not being worn. Put it on and capture again.</div>
|
||||
<div class="loading"><div class="spinner"></div></div>
|
||||
</div>
|
||||
<div class="ctrl-status" id="ctrlBadge" role="status" hidden></div>
|
||||
<div class="pad-keys ctrl-keys" id="ctrlKeys" hidden aria-label="Keys and clicks for the Frame">
|
||||
<input id="ctrlType" type="text" placeholder="Type here to type on the Frame" autocomplete="off" autocapitalize="off"
|
||||
autocorrect="off" spellcheck="false" enterkeyhint="send">
|
||||
<span class="seg pad-clicks">
|
||||
<button data-ctrl-click="left" title="Left click where the pointer is">Click</button>
|
||||
<button data-ctrl-click="right" title="Right click where the pointer is">Right-click</button>
|
||||
</span>
|
||||
<span class="seg pad-special">
|
||||
<button data-ctrl-key="1" title="Escape">Esc</button>
|
||||
<button data-ctrl-key="15" title="Tab">Tab</button>
|
||||
<button data-ctrl-key="105" title="Left arrow" aria-label="Left arrow">←</button>
|
||||
<button data-ctrl-key="103" title="Up arrow" aria-label="Up arrow">↑</button>
|
||||
<button data-ctrl-key="108" title="Down arrow" aria-label="Down arrow">↓</button>
|
||||
<button data-ctrl-key="106" title="Right arrow" aria-label="Right arrow">→</button>
|
||||
<button data-ctrl-key="14" title="Backspace" aria-label="Backspace">⌫</button>
|
||||
<button data-ctrl-key="28" title="Enter">Enter</button>
|
||||
</span>
|
||||
</div>
|
||||
<div class="hint" id="viewHint"></div>
|
||||
</section>
|
||||
|
||||
@@ -538,36 +575,6 @@
|
||||
</section>
|
||||
</div>
|
||||
|
||||
<section class="panel" id="comfort" style="margin:20px 0;padding:16px">
|
||||
<div class="shelf-head"><h2>Family and comfort</h2><span class="spacer"></span>
|
||||
<button class="action small" id="castBtn">Cast headset view</button>
|
||||
</div>
|
||||
<p class="hint">Share this screen with people in the room. Casting shows everything the wearer sees, including private content.</p>
|
||||
<form id="comfortForm">
|
||||
<div class="row" style="flex-wrap:wrap;gap:12px">
|
||||
<label>Session limit (minutes) <input id="sessionMinutes" type="number" min="1" max="240" value="30" required style="width:75px"></label>
|
||||
<label>Break every <input id="breakMinutes" type="number" min="0" max="120" value="20" required style="width:75px"> minutes</label>
|
||||
<label>Check in after <input id="stillMinutes" type="number" min="0" max="240" value="30" required style="width:75px"> active minutes</label>
|
||||
</div>
|
||||
<div class="row" style="flex-wrap:wrap;margin:14px 0;gap:14px">
|
||||
<label><input id="batteryAlert" type="checkbox" checked> Alert on low battery</label>
|
||||
<label><input id="heatAlert" type="checkbox" checked> Alert on overheating</label>
|
||||
<button class="action small" id="sessionStart" type="submit">Start session</button>
|
||||
<button class="small" id="sessionCancel" type="button" disabled>Cancel session</button>
|
||||
<button class="small" id="testNotification" type="button">Enable / test notifications</button>
|
||||
</div>
|
||||
</form>
|
||||
<p id="comfortStatus" role="status">Checking session…</p>
|
||||
<p id="comfortNotificationStatus" class="hint" role="status" hidden></p>
|
||||
<p class="hint">A one-minute warning, then Steam Home. Games stay running: save and pause first. Keep this app open and connected for notifications.</p>
|
||||
<details class="hint"><summary>How sessions and alerts work</summary>
|
||||
<p>This is a reminder, not a parental lock. The timer continues on the Frame if you disconnect; a headset restart cancels it. Cancel before changing settings. Set break/check-in to 0 to turn them off.</p>
|
||||
<p>Battery, heat and check-in alerts appear on connected companions during a session. Headset warnings and break reminders continue without a companion. iOS may suspend phone notifications in the background. Breaks and check-ins count SteamVR activity, not confirmed wear time.</p>
|
||||
</details>
|
||||
<div id="comfortEvents" class="hint"></div>
|
||||
<span id="comfortAnnouncement" class="sr-only" aria-live="polite"></span>
|
||||
</section>
|
||||
|
||||
<section class="panel" id="pad">
|
||||
<div class="shelf-head"><h2>Keyboard and trackpad</h2><span class="sub" id="padState"></span><span class="spacer"></span>
|
||||
<button class="small" id="padOn">Turn on</button>
|
||||
@@ -974,9 +981,9 @@ const QUICK = [["Remmina", "org.remmina.Remmina"], ["Moonlight", "com.moonlight_
|
||||
const CDN = "https://cdn.cloudflare.steamstatic.com/steam/apps";
|
||||
const HINTS = {
|
||||
headset: "What the lenses show, composited by SteamVR: the room, floating panels, dashboard and controllers. Live streams it as video (one eye, about 30 fps); Capture takes a still of both eyes. Captures show everything on screen, including anything private.",
|
||||
flat: "gamescope's 2D layer: the desktop panel and Steam's flat UI, without the room or VR scene.",
|
||||
flat: "The app panel in use in the headset, flat and still: it doesn't move as the wearer looks around. Pick another panel to watch it. Control lets you tap or click right where you want on it.",
|
||||
};
|
||||
const SOURCE_LABEL = { steamvr: "Headset view", gamescope: "Desktop panel", shot: "Screenshot" };
|
||||
const SOURCE_LABEL = { steamvr: "Headset view", gamescope: "Desktop panel", panel: "Desktop", shot: "Screenshot" };
|
||||
let state = null, view = "headset", eye = "left", live = false, liveTimer = null, volTimer = null;
|
||||
let lastImg = null, lastSource = null, lastShot = null, viewGen = 0;
|
||||
|
||||
@@ -1128,101 +1135,6 @@ function refresh() {
|
||||
return refreshing;
|
||||
}
|
||||
|
||||
// The Frame owns the clock. Poll independently of status and the selected tab.
|
||||
let comfortBusy = false, comfortSeen = new Set(), comfortSession = null, comfortHydrated = false;
|
||||
async function localNotification(message, request = false) {
|
||||
try {
|
||||
if (window.frameApp?.notify) await window.frameApp.notify(message, request);
|
||||
else {
|
||||
if (!("Notification" in window)) throw new Error("This browser has no notifications; use the Frame Control app.");
|
||||
const permission = request ? await Notification.requestPermission() : Notification.permission;
|
||||
if (permission !== "granted") throw new Error("Notifications are off. Enable them in system settings.");
|
||||
new Notification("Frame Control", {body: message});
|
||||
}
|
||||
$("comfortNotificationStatus").hidden = true;
|
||||
} catch (e) {
|
||||
$("comfortNotificationStatus").hidden = false;
|
||||
$("comfortNotificationStatus").textContent = "Notifications aren't available on this device. Check system notification settings, then use Enable / test notifications. Headset reminders continue during an active session.";
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
function renderComfort(s) {
|
||||
const firstSnapshot = !comfortHydrated;
|
||||
comfortHydrated = true;
|
||||
$("sessionStart").disabled = !!s.active;
|
||||
for (const id of ["sessionMinutes", "breakMinutes", "stillMinutes", "batteryAlert", "heatAlert"])
|
||||
$(id).disabled = !!s.active;
|
||||
$("sessionCancel").disabled = !s.active;
|
||||
if (s.id !== comfortSession) {
|
||||
comfortSession = s.id;
|
||||
comfortSeen.clear();
|
||||
if (s.options) for (const [key, value] of Object.entries(s.options)) {
|
||||
const el = $(key === "minutes" ? "sessionMinutes" : key);
|
||||
if (!el) continue;
|
||||
if (el.type === "checkbox") el.checked = value; else el.value = value;
|
||||
}
|
||||
}
|
||||
let statusText = s.error || (s.active
|
||||
? `${Math.ceil(s.remaining / 60)} min until Steam Home · ${s.activity == null ? "activity unknown" : s.activity === 3 ? "headset in standby" : "monitoring"}`
|
||||
: "No session running.");
|
||||
if (s.active && s.unavailable?.length)
|
||||
statusText += " · No readings: " + s.unavailable.join(", ");
|
||||
if ($("comfortStatus").textContent !== statusText) $("comfortStatus").textContent = statusText;
|
||||
let latest = null;
|
||||
for (const e of s.events || []) {
|
||||
if (comfortSeen.has(e.id)) continue;
|
||||
comfortSeen.add(e.id);
|
||||
// Historical events remain visible but never produce a burst on reconnect.
|
||||
if (!firstSnapshot && s.time - e.time >= 0 && s.time - e.time < 30 && !["started", "cancelled"].includes(e.kind)) {
|
||||
latest = e.message;
|
||||
log(e.message); toast(e.message, e.kind === "error");
|
||||
localNotification(e.message).catch(() => {}); // the notification status keeps the failure visible
|
||||
}
|
||||
}
|
||||
// Keep only the server's bounded history; a new page seeds it without replay.
|
||||
comfortSeen = new Set((s.events || []).map(e => e.id));
|
||||
if (latest !== null) $("comfortAnnouncement").textContent = latest;
|
||||
const history = (s.events || []).slice(-3).map(e => e.message).join(" · ");
|
||||
if ($("comfortEvents").textContent !== history) $("comfortEvents").textContent = history;
|
||||
}
|
||||
async function pollComfort() {
|
||||
if (!comfortBusy) {
|
||||
comfortBusy = true;
|
||||
try { renderComfort(await api("/api/comfort", {action: "status"})); }
|
||||
catch (e) {
|
||||
const message = "Session status unavailable: " + e.message;
|
||||
if ($("comfortStatus").textContent !== message) $("comfortStatus").textContent = message;
|
||||
}
|
||||
finally { comfortBusy = false; }
|
||||
}
|
||||
setTimeout(pollComfort, 5000);
|
||||
}
|
||||
$("comfortForm").onsubmit = async e => {
|
||||
e.preventDefault();
|
||||
const result = await act("Start session", () => api("/api/comfort", {
|
||||
action: "start", minutes: Number($("sessionMinutes").value), breakMinutes: Number($("breakMinutes").value),
|
||||
stillMinutes: Number($("stillMinutes").value), batteryAlert: $("batteryAlert").checked, heatAlert: $("heatAlert").checked,
|
||||
}), $("sessionStart"));
|
||||
if (result) renderComfort(result);
|
||||
};
|
||||
$("sessionCancel").onclick = async () => {
|
||||
const result = await act("Cancel session", () => api("/api/comfort", {action: "cancel"}), $("sessionCancel"));
|
||||
if (result) renderComfort(result);
|
||||
};
|
||||
$("testNotification").onclick = () => act("Test notification", async () => {
|
||||
await localNotification("Comfort notifications are enabled on this device.", true);
|
||||
return {message: "Test notification sent. Check your device's notification settings if it didn't appear."};
|
||||
});
|
||||
$("castBtn").onclick = () => {
|
||||
setView("headset");
|
||||
if (!live) toggleLive(true);
|
||||
$("viewer").scrollIntoView({behavior: "smooth", block: "center"});
|
||||
// Fullscreen is a direct user gesture; iPhone falls back to its inline viewer.
|
||||
$("viewer").requestFullscreen?.().catch(() => {});
|
||||
};
|
||||
pollComfort();
|
||||
|
||||
// ---- battery ----
|
||||
function battery(b, power) {
|
||||
const pct = b?.percent;
|
||||
@@ -1331,8 +1243,12 @@ function setView(v) {
|
||||
view = v;
|
||||
document.querySelectorAll("[data-view]").forEach(b => b.classList.toggle("on", b.dataset.view === v));
|
||||
$("eyeSeg").style.visibility = v === "headset" && !video.ctl ? "visible" : "hidden";
|
||||
$("eyeSeg").style.display = v === "flat" ? "none" : ""; // the Desktop view has no eyes to pick
|
||||
$("viewHint").textContent = HINTS[v];
|
||||
if (live && changed) { toggleLive(false); toggleLive(true); } // video for the headset, captures for the panel
|
||||
$("panelPick").hidden = v !== "flat";
|
||||
if (v === "flat") watchPanels(); else clearTimeout(desk.timer);
|
||||
if (live && changed) { toggleLive(false); toggleLive(true); }
|
||||
ctrlShow();
|
||||
}
|
||||
function setEye(e) {
|
||||
eye = e;
|
||||
@@ -1359,20 +1275,27 @@ function setZoom(z, cx, cy) {
|
||||
$("zoomIn").onclick = () => setZoom(zoom * 1.5);
|
||||
$("zoomOut").onclick = () => setZoom(zoom / 1.5);
|
||||
$("zoomFit").onclick = () => setZoom(1);
|
||||
$("fullBtn").onclick = () => document.fullscreenElement ? document.exitFullscreen() : $("viewer").requestFullscreen();
|
||||
// Full screen, or on phones (no element full screen there) the viewer filling the window.
|
||||
$("fullBtn").onclick = () => {
|
||||
if (document.fullscreenElement) return document.exitFullscreen();
|
||||
if ($("viewer").requestFullscreen && !TOUCH) return $("viewer").requestFullscreen().catch(() => $("viewer").classList.toggle("immersive"));
|
||||
$("viewer").classList.toggle("immersive");
|
||||
setZoom(1);
|
||||
};
|
||||
document.addEventListener("fullscreenchange", () => setZoom(1));
|
||||
$("viewer").addEventListener("wheel", e => {
|
||||
if ($("canvas").hidden) return;
|
||||
if ($("canvas").hidden || ctrl.on) return;
|
||||
e.preventDefault();
|
||||
const v = $("viewer").getBoundingClientRect();
|
||||
setZoom(zoom * Math.exp(-e.deltaY * 0.002), e.clientX - v.left, e.clientY - v.top);
|
||||
}, { passive: false });
|
||||
$("canvas").addEventListener("dblclick", e => {
|
||||
if (ctrl.on) return;
|
||||
const v = $("viewer").getBoundingClientRect();
|
||||
setZoom(zoom > 1 ? 1 : 2.5, e.clientX - v.left, e.clientY - v.top);
|
||||
});
|
||||
$("canvas").addEventListener("pointerdown", e => {
|
||||
if (zoom === 1) return;
|
||||
if (zoom === 1 || ctrl.on) return;
|
||||
const start = { x: e.clientX - panX, y: e.clientY - panY };
|
||||
$("viewer").classList.add("panning");
|
||||
$("canvas").setPointerCapture(e.pointerId);
|
||||
@@ -1382,7 +1305,10 @@ $("canvas").addEventListener("pointerdown", e => {
|
||||
$("canvas").addEventListener("pointerup", up, { once: true });
|
||||
});
|
||||
document.addEventListener("keydown", e => {
|
||||
if (page !== "home" || $("canvas").hidden || /INPUT|TEXTAREA|SELECT/.test(document.activeElement.tagName) || e.metaKey || e.ctrlKey) return;
|
||||
if (page !== "home" || /INPUT|TEXTAREA|SELECT/.test(document.activeElement.tagName) || e.metaKey || e.ctrlKey) return;
|
||||
if (ctrl.on && document.activeElement === $("viewer")) return; // keys are going to the Frame
|
||||
if (e.key === "c") return $("ctrlBtn").click();
|
||||
if ($("canvas").hidden) return;
|
||||
if (e.key === "+" || e.key === "=") setZoom(zoom * 1.5);
|
||||
else if (e.key === "-") setZoom(zoom / 1.5);
|
||||
else if (e.key === "0") setZoom(1);
|
||||
@@ -1424,8 +1350,14 @@ async function capture() {
|
||||
if (!live) $("viewer").classList.add("busy"); // no spinner flashing over a live stream
|
||||
let url = null;
|
||||
try {
|
||||
const r = await fetch(view === "headset" ? "/api/screenshot?view=headset" : "/api/screenshot",
|
||||
{ headers: {"X-Frame-UI": UI_KEY} });
|
||||
let url_ = "/api/screenshot?view=headset";
|
||||
var shotPanel = null;
|
||||
if (view === "flat") {
|
||||
shotPanel = deskPanel() || (await loadPanels(), deskPanel());
|
||||
if (!shotPanel) throw new Error("no app panel is open in the headset");
|
||||
url_ = `/api/screenshot?view=panel&window=${shotPanel.window}&display=${encodeURIComponent(shotPanel.display)}`;
|
||||
}
|
||||
const r = await fetch(url_, { headers: {"X-Frame-UI": UI_KEY} });
|
||||
if (!r.ok) throw new Error((await r.json().catch(() => ({}))).error || `HTTP ${r.status}`);
|
||||
const source = r.headers.get("X-Capture-Source") || "gamescope";
|
||||
url = URL.createObjectURL(await r.blob());
|
||||
@@ -1434,6 +1366,9 @@ async function capture() {
|
||||
if (gen !== viewGen) return true;
|
||||
lastImg = img; lastSource = source; lastShot = null;
|
||||
draw();
|
||||
desk.shown = source === "panel" ? panelKey(shotPanel) : null;
|
||||
if (source === "panel") $("srcBadge").textContent = `Desktop · ${shotPanel?.name || "panel"}`;
|
||||
ctrlShow();
|
||||
$("stamp").hidden = false; $("stamp").textContent = new Date().toLocaleTimeString();
|
||||
$("saveBtn").disabled = false;
|
||||
return true;
|
||||
@@ -1446,26 +1381,29 @@ async function capture() {
|
||||
$("viewer").classList.remove("busy");
|
||||
}
|
||||
}
|
||||
let liveFailures = 0;
|
||||
async function liveLoop() {
|
||||
if (!live) return;
|
||||
liveFailures = (await capture()) ? 0 : liveFailures + 1;
|
||||
let liveFailures = 0, liveGen = 0;
|
||||
async function liveLoop(gen = liveGen) {
|
||||
if (!live || gen !== liveGen) return;
|
||||
const ok = await capture();
|
||||
if (gen !== liveGen) return; // Live was restarted (say, as video) while this capture ran
|
||||
liveFailures = ok ? 0 : liveFailures + 1;
|
||||
if (liveFailures >= 5) {
|
||||
toggleLive(false);
|
||||
toast("Live stopped after 5 failed captures. Is the headset on?", true);
|
||||
toast(view === "flat" ? "Live stopped: couldn't capture the panel." : "Live stopped after 5 failed captures. Is the headset on?", true);
|
||||
return;
|
||||
}
|
||||
if (live) liveTimer = setTimeout(liveLoop, view === "headset" ? 300 : 700);
|
||||
if (live) liveTimer = setTimeout(() => liveLoop(gen), view === "headset" ? 300 : 700);
|
||||
}
|
||||
function toggleLive(on) {
|
||||
live = on;
|
||||
liveGen++;
|
||||
liveFailures = 0;
|
||||
$("liveBtn").classList.toggle("on", live);
|
||||
$("liveBadge").hidden = !live;
|
||||
clearTimeout(liveTimer);
|
||||
stopVideo();
|
||||
if (!live) return;
|
||||
if (view === "headset" && "VideoDecoder" in window) {
|
||||
if ("VideoDecoder" in window && (view === "headset" || deskPanel())) {
|
||||
const started = startVideo();
|
||||
const ctl = video.ctl;
|
||||
started.catch(e => {
|
||||
@@ -1510,9 +1448,10 @@ async function startVideo() {
|
||||
if (!shown++) {
|
||||
viewGen++; // a capture still in flight mustn't replace the video
|
||||
lastImg = null; lastSource = "video"; lastShot = null;
|
||||
desk.shown = panelKey(panel); ctrlShow();
|
||||
$("viewer").classList.remove("busy");
|
||||
c.hidden = false; $("viewerEmpty").hidden = true; $("zoombar").hidden = false; $("asleep").hidden = true;
|
||||
$("srcBadge").hidden = false; $("srcBadge").textContent = "Headset view · video";
|
||||
$("srcBadge").hidden = false; $("srcBadge").textContent = `${video.label} · video`;
|
||||
$("stamp").hidden = false; $("saveBtn").disabled = false;
|
||||
}
|
||||
frames++;
|
||||
@@ -1524,7 +1463,11 @@ async function startVideo() {
|
||||
},
|
||||
error: e => log("Video decoder: " + e.message, "e"),
|
||||
});
|
||||
const r = await fetch(`/api/stream?${STREAM_QUERY}`, { headers: {"X-Frame-UI": UI_KEY}, signal: ctl.signal });
|
||||
const panel = view === "flat" ? deskPanel() : null;
|
||||
desk.shown = null; // until this stream's first picture
|
||||
video.label = panel ? `Desktop · ${panel.name || "panel"}` : "Headset view";
|
||||
const query = panel ? `src=panel&window=${panel.window}&display=${encodeURIComponent(panel.display)}&${STREAM_QUERY}` : STREAM_QUERY;
|
||||
const r = await fetch(`/api/stream?${query}`, { headers: {"X-Frame-UI": UI_KEY}, signal: ctl.signal });
|
||||
if (!r.ok) throw new Error((await r.json().catch(() => ({}))).error || `HTTP ${r.status}`);
|
||||
const reader = r.body.getReader();
|
||||
let buf = new Uint8Array(0), scan = 0, auStart = -1;
|
||||
@@ -1569,7 +1512,7 @@ function stopVideo() {
|
||||
if (video.dec && video.dec.state !== "closed") video.dec.close();
|
||||
video.ctl = video.dec = null;
|
||||
$("viewer").classList.remove("busy");
|
||||
if (lastSource === "video") { $("srcBadge").textContent = "Headset view · video (stopped)"; $("stamp").hidden = true; }
|
||||
if (lastSource === "video") { $("srcBadge").textContent = `${video.label || "Headset view"} · video (stopped)`; $("stamp").hidden = true; }
|
||||
$("eyeSeg").style.visibility = view === "headset" ? "visible" : "hidden";
|
||||
}
|
||||
$("shotBtn").onclick = () => capture();
|
||||
@@ -1589,6 +1532,421 @@ function download(blob, name) {
|
||||
document.querySelectorAll("[data-view]").forEach(b => b.onclick = () => setView(b.dataset.view));
|
||||
document.querySelectorAll("[data-eye]").forEach(b => b.onclick = () => setEye(b.dataset.eye));
|
||||
|
||||
// ---- Desktop view: the app panel in use in the headset (or one picked to watch) ----
|
||||
// gamescope says which panel has focus: the one the wearer last used. That's the one
|
||||
// Control reaches, so the view follows it unless you pick another to watch.
|
||||
// Panels are "display/window": window ids are per X display, and the Frame has two.
|
||||
// desk.shown is the panel whose picture is on screen now; Control only acts on that one.
|
||||
const desk = { panels: [], focus: null, pick: "", timer: null, loaded: false, shown: null };
|
||||
const panelKey = p => p ? `${p.display}/${p.window}` : null;
|
||||
function deskPanel() {
|
||||
const want = desk.pick || desk.focus;
|
||||
// Nothing in use and nothing picked: show the first panel rather than nothing.
|
||||
return desk.panels.find(p => panelKey(p) === want) || (!desk.pick && !desk.focus && desk.panels[0]) || null;
|
||||
}
|
||||
async function loadPanels() {
|
||||
try {
|
||||
const before = panelKey(deskPanel());
|
||||
const r = await api("/api/panels");
|
||||
desk.panels = r.panels || []; desk.loaded = true;
|
||||
desk.focus = r.focus ? `${r.focus_display}/${r.focus}` : null;
|
||||
if (desk.pick && !desk.panels.some(p => panelKey(p) === desk.pick)) desk.pick = "";
|
||||
renderPanelPick();
|
||||
const now = panelKey(deskPanel());
|
||||
if (view === "flat" && live && now !== before) { toggleLive(false); toggleLive(true); }
|
||||
ctrlShow();
|
||||
} catch (e) {
|
||||
log("Couldn't list the headset's panels: " + e.message, "e");
|
||||
}
|
||||
}
|
||||
function watchPanels() {
|
||||
clearTimeout(desk.timer);
|
||||
if (view !== "flat") return;
|
||||
loadPanels().finally(() => { if (view === "flat") desk.timer = setTimeout(watchPanels, document.hidden ? 10000 : 3000); });
|
||||
}
|
||||
function renderPanelPick() {
|
||||
const sel = $("panelPick");
|
||||
sel.innerHTML = `<option value="">${desk.focus ? "Panel in use" : "Panel in use (none yet)"}</option>` +
|
||||
desk.panels.map(p => `<option value="${esc(panelKey(p))}">${esc(p.name || "Untitled")}${p.focused ? " (in use)" : ""}</option>`).join("");
|
||||
sel.value = desk.pick;
|
||||
}
|
||||
$("panelPick").onchange = () => {
|
||||
desk.pick = $("panelPick").value;
|
||||
ctrlShow();
|
||||
if (live) { toggleLive(false); toggleLive(true); } else if (lastImg || lastSource) capture();
|
||||
};
|
||||
|
||||
// ---- Control: tap or click on the view to use the Frame ----
|
||||
// Through gamescope's own input (frame_touch.py). On the Desktop view a tap lands
|
||||
// exactly where you tapped; on the headset view, which moves with the wearer's head,
|
||||
// the view is a trackpad for the pointer instead.
|
||||
const ctrl = { on: false, queue: [], sending: false, state: "off", message: "", move: null, rel: [0, 0], raf: 0,
|
||||
held: new Set(), keys: new Set(), pointers: new Map(), g: null, retry: null };
|
||||
const EVDEV = { Escape: 1, Minus: 12, Equal: 13, Backspace: 14, Tab: 15, BracketLeft: 26, BracketRight: 27, Enter: 28,
|
||||
ControlLeft: 29, Semicolon: 39, Quote: 40, Backquote: 41, ShiftLeft: 42, Backslash: 43, Comma: 51, Period: 52,
|
||||
Slash: 53, ShiftRight: 54, NumpadMultiply: 55, AltLeft: 56, Space: 57, CapsLock: 58, F11: 87, F12: 88,
|
||||
IntlBackslash: 86, NumpadEnter: 96, ControlRight: 97, NumpadDivide: 98, AltRight: 100, Home: 102, ArrowUp: 103,
|
||||
PageUp: 104, ArrowLeft: 105, ArrowRight: 106, End: 107, ArrowDown: 108, PageDown: 109, Insert: 110, Delete: 111,
|
||||
MetaLeft: 125, MetaRight: 126, ContextMenu: 127, NumpadSubtract: 74, NumpadAdd: 78, NumpadDecimal: 83 };
|
||||
[..."qwertyuiop"].forEach((c, i) => EVDEV["Key" + c.toUpperCase()] = 16 + i);
|
||||
[..."asdfghjkl"].forEach((c, i) => EVDEV["Key" + c.toUpperCase()] = 30 + i);
|
||||
[..."zxcvbnm"].forEach((c, i) => EVDEV["Key" + c.toUpperCase()] = 44 + i);
|
||||
[..."1234567890"].forEach((c, i) => EVDEV["Digit" + c] = 2 + i);
|
||||
for (let i = 1; i <= 10; i++) EVDEV["F" + i] = 58 + i;
|
||||
[[7, 71], [8, 72], [9, 73], [4, 75], [5, 76], [6, 77], [1, 79], [2, 80], [3, 81], [0, 82]].forEach(([n, k]) => EVDEV["Numpad" + n] = k);
|
||||
const CTRL_BUTTONS = ["left", "middle", "right"]; // MouseEvent.button 0, 1, 2
|
||||
|
||||
function ctrlMode() { return view === "flat" ? "abs" : "rel"; }
|
||||
// Where taps go on the Desktop view, or why they can't.
|
||||
function ctrlTarget() {
|
||||
if (ctrlMode() === "rel") return { ok: true };
|
||||
const p = deskPanel();
|
||||
if (!desk.loaded) return { why: "Finding the headset's panels…" };
|
||||
if (!p) return { why: "No app panel is open in the headset." };
|
||||
if (!desk.focus) return { why: "Nobody's using a panel in the headset yet. Taps work once one is in use." };
|
||||
if (panelKey(p) !== desk.focus) return { why: "Only the panel in use takes taps. Pick “Panel in use” to control it." };
|
||||
if (desk.shown !== desk.focus) return { why: live ? "Waiting for the picture of the panel in use…"
|
||||
: "The panel in use has changed. Press Capture or Live to see it first." };
|
||||
return { ok: true, panel: p };
|
||||
}
|
||||
function ctrlShow() {
|
||||
const v = $("viewer"), b = $("ctrlBadge");
|
||||
$("ctrlBtn").classList.toggle("on", ctrl.on);
|
||||
v.classList.toggle("control", ctrl.on);
|
||||
v.classList.toggle("rel", ctrl.on && ctrlMode() === "rel");
|
||||
$("ctrlKeys").hidden = !ctrl.on;
|
||||
b.hidden = !ctrl.on;
|
||||
if (!ctrl.on) return;
|
||||
const t = ctrlTarget();
|
||||
let text = ctrlMode() === "abs" ? `Tap or click to use ${t.panel ? t.panel.name || "the panel" : "the panel"}`
|
||||
: "Drag to move the pointer, tap to click, two fingers to scroll";
|
||||
let warn = false;
|
||||
if (!t.ok) { text = t.why; warn = true; }
|
||||
else if (ctrl.state === "error") { text = ctrl.message || "Control isn't working"; warn = true; }
|
||||
else if (ctrl.state === "starting") text = "Connecting…";
|
||||
b.textContent = text;
|
||||
b.classList.toggle("warn", warn);
|
||||
}
|
||||
function ctrlToggle(on) {
|
||||
if (!on) ctrlRelease();
|
||||
ctrl.on = on;
|
||||
if (on) {
|
||||
setZoom(1);
|
||||
ctrlPoll(0);
|
||||
if (view === "flat" && !desk.loaded) watchPanels();
|
||||
if (!TOUCH) $("viewer").focus({ preventScroll: true });
|
||||
}
|
||||
if (window.frameApp && window.frameApp.captureKeys) window.frameApp.captureKeys(on && document.activeElement === $("viewer"));
|
||||
ctrlShow();
|
||||
}
|
||||
$("ctrlBtn").onclick = () => ctrlToggle(!ctrl.on);
|
||||
// Until it's ready, check on the Frame side every half second (for up to 15 s).
|
||||
async function ctrlPoll(n) {
|
||||
if (!ctrl.on) return;
|
||||
try {
|
||||
const r = await api(n ? "/api/touch" : "/api/touch?start=1");
|
||||
ctrl.state = r.state; ctrl.message = r.message || "";
|
||||
} catch (e) { ctrl.state = "error"; ctrl.message = e.message; }
|
||||
if (!ctrl.on) return; // turned off while connecting
|
||||
ctrlShow();
|
||||
if (ctrl.state !== "ready" && ctrl.state !== "error" && n < 30) return setTimeout(() => ctrlPoll(n + 1), 500);
|
||||
if (ctrl.state === "ready") return ctrlFlush();
|
||||
if (ctrl.state !== "error") {
|
||||
ctrl.state = "error"; ctrl.message = "The Frame didn't answer. Turn Control off and on to try again.";
|
||||
ctrlShow();
|
||||
}
|
||||
}
|
||||
$("viewer").tabIndex = 0;
|
||||
$("viewer").addEventListener("focus", () => { if (ctrl.on && window.frameApp?.captureKeys) window.frameApp.captureKeys(true); });
|
||||
$("viewer").addEventListener("blur", () => { ctrlRelease(); if (window.frameApp?.captureKeys) window.frameApp.captureKeys(false); });
|
||||
document.addEventListener("visibilitychange", () => { if (document.hidden) ctrlRelease(); });
|
||||
window.addEventListener("blur", () => ctrlRelease()); // another app took the keyboard
|
||||
|
||||
const isMoveEvent = e => ("fx" in e || "dx" in e) && !("button" in e) && !("key" in e);
|
||||
const isRelease = e => ("button" in e || "key" in e) && e.down === false;
|
||||
// While it connects, keep clicks and keys, each with the last position before it, so a
|
||||
// tap still lands where it was made; lone moves are stale by then.
|
||||
function ctrlKeepable(events) {
|
||||
const out = [];
|
||||
let move = null;
|
||||
for (const e of events) {
|
||||
if (isMoveEvent(e)) { if ("fx" in e) move = e; continue; }
|
||||
if (move && "button" in e && e.down !== false) out.push(move);
|
||||
move = null;
|
||||
out.push(e);
|
||||
}
|
||||
if (move) out.push(move); // the last position: a click queued after it needs it
|
||||
return out;
|
||||
}
|
||||
function ctrlSend(events) {
|
||||
ctrl.queue.push(...events.map(e => isMoveEvent(e) ? e : ctrlAim(e)).filter(Boolean));
|
||||
// Keep order; if the Frame is slow to answer, drop pointer moves first, then the oldest
|
||||
// presses and scrolls, never a release.
|
||||
if (ctrl.queue.length > 300) {
|
||||
const q = ctrlKeepable(ctrl.queue), keep = new Set();
|
||||
q.forEach((e, i) => { if (isRelease(e) || i >= q.length - 200) keep.add(i); });
|
||||
// A kept press keeps the position before it, so it lands where it was made.
|
||||
for (const i of [...keep]) if (i > 0 && "button" in q[i] && q[i].down !== false && "fx" in q[i - 1]) keep.add(i - 1);
|
||||
ctrl.queue = q.filter((e, i) => keep.has(i));
|
||||
}
|
||||
ctrlFlush();
|
||||
}
|
||||
async function ctrlFlush() {
|
||||
if (ctrl.sending || !ctrl.queue.length) return;
|
||||
// Backing off after a failure: new input waits for the same retry.
|
||||
const wait = (ctrl.retryAt || 0) - Date.now();
|
||||
if (wait > 0) {
|
||||
clearTimeout(ctrl.retry);
|
||||
ctrl.retry = setTimeout(ctrlFlush, wait);
|
||||
return;
|
||||
}
|
||||
ctrl.sending = true;
|
||||
// Up to 200 events, and at most about 100 characters of text: the Frame types a batch
|
||||
// key by key before reading the next, and a release mustn't wait behind a long paste.
|
||||
let n = 0, chars = 0;
|
||||
while (n < ctrl.queue.length && n < 200 && (n === 0 || chars + (ctrl.queue[n].text?.length || 0) <= 100)) {
|
||||
chars += ctrl.queue[n].text?.length || 0;
|
||||
n++;
|
||||
}
|
||||
const batch = ctrl.queue.splice(0, n);
|
||||
let again = 0;
|
||||
try {
|
||||
const r = await api("/api/touch", { events: batch });
|
||||
ctrl.state = r.state; ctrl.message = r.message || "";
|
||||
if (r.stale && typeof loadPanels === "function") loadPanels(); // focus moved on: catch up now
|
||||
if (!r.sent && r.state === "error") {
|
||||
// Broken: presses would be stale by the time it's back; releases still matter.
|
||||
ctrl.queue.unshift(...batch.filter(isRelease));
|
||||
again = 2000;
|
||||
} else if (!r.sent) {
|
||||
ctrl.queue.unshift(...ctrlKeepable(batch)); // connecting: send them once it's ready
|
||||
again = 300;
|
||||
}
|
||||
} catch (e) {
|
||||
ctrl.state = "error"; ctrl.message = e.message;
|
||||
ctrl.queue.unshift(...batch.filter(isRelease));
|
||||
again = 1000; // try the releases again in a moment, not in a tight loop
|
||||
} finally {
|
||||
ctrl.sending = false;
|
||||
ctrlShow();
|
||||
clearTimeout(ctrl.retry);
|
||||
ctrl.retryAt = again ? Date.now() + again : 0;
|
||||
if (again) { if (ctrl.queue.length) ctrl.retry = setTimeout(ctrlFlush, again); }
|
||||
else if (ctrl.queue.length) ctrlFlush();
|
||||
}
|
||||
}
|
||||
// Pointer moves: at most one per animation frame.
|
||||
function ctrlMoveTo(fx, fy) {
|
||||
const p = ctrlTarget().panel;
|
||||
if (!p) return;
|
||||
ctrl.move = { fx, fy, window: p.window, display: p.display };
|
||||
ctrlSchedule();
|
||||
}
|
||||
function ctrlMoveBy(dx, dy) { ctrl.rel[0] += dx; ctrl.rel[1] += dy; ctrlSchedule(); }
|
||||
function ctrlSchedule() { if (!ctrl.raf) ctrl.raf = requestAnimationFrame(ctrlFlushMoves); }
|
||||
function ctrlFlushMoves() {
|
||||
ctrl.raf = 0;
|
||||
const out = [];
|
||||
if (ctrl.move) { out.push(ctrl.move); ctrl.move = null; }
|
||||
if (ctrl.rel[0] || ctrl.rel[1]) { out.push({ dx: Math.round(ctrl.rel[0]), dy: Math.round(ctrl.rel[1]) }); ctrl.rel = [0, 0]; }
|
||||
if (out.length) ctrlSend(out);
|
||||
}
|
||||
// On the Desktop view, presses name the panel they're for, so the Frame drops them if
|
||||
// focus has moved on (releases always go).
|
||||
// A press with no panel to aim at (the Desktop view lost its target) isn't sent at all.
|
||||
function ctrlAim(e) {
|
||||
if (ctrlMode() !== "abs" || e.down === false) return e;
|
||||
const p = ctrlTarget().panel;
|
||||
return p ? { ...e, window: p.window, display: p.display } : null;
|
||||
}
|
||||
function ctrlButton(name, down) {
|
||||
ctrlFlushMoves(); // the click lands where the pointer was sent
|
||||
if (down) ctrl.held.add(name); else ctrl.held.delete(name);
|
||||
ctrlSend([ctrlAim({ button: name, down })]);
|
||||
}
|
||||
function ctrlKeyEvent(code, down) {
|
||||
ctrlFlushMoves();
|
||||
if (down) ctrl.keys.add(code); else ctrl.keys.delete(code);
|
||||
ctrlSend([ctrlAim({ key: code, down })]);
|
||||
}
|
||||
function ctrlClick(name = "left") { ctrlButton(name, true); ctrlButton(name, false); }
|
||||
function ctrlRelease() {
|
||||
for (const b of [...ctrl.held]) ctrlButton(b, false);
|
||||
for (const k of [...ctrl.keys]) ctrlKeyEvent(k, false);
|
||||
if (ctrl.g && ctrl.g.timer) clearTimeout(ctrl.g.timer);
|
||||
ctrl.pointers.clear(); ctrl.g = null;
|
||||
}
|
||||
// Where a pointer event falls on the panel, as fractions (the canvas box includes any letterboxing).
|
||||
function ctrlFraction(e) {
|
||||
const c = $("canvas"), r = c.getBoundingClientRect();
|
||||
const scale = Math.min(r.width / c.width, r.height / c.height);
|
||||
const w = c.width * scale, h = c.height * scale;
|
||||
const x = (e.clientX - r.left - (r.width - w) / 2) / w, y = (e.clientY - r.top - (r.height - h) / 2) / h;
|
||||
return x >= 0 && x <= 1 && y >= 0 && y <= 1 ? [x, y] : null;
|
||||
}
|
||||
|
||||
// Mouse (and pen) on a computer: the view is the Frame's screen.
|
||||
function ctrlMouse(e) {
|
||||
if (ctrlMode() === "abs") {
|
||||
const f = ctrlFraction(e);
|
||||
if (f) ctrlMoveTo(...f);
|
||||
} else if (e.type === "pointermove" && (e.movementX || e.movementY)) {
|
||||
ctrlMoveBy(e.movementX * 1.5, e.movementY * 1.5);
|
||||
}
|
||||
}
|
||||
// Touch: tap to click, press and hold to right-click, two fingers to scroll. Dragging is a
|
||||
// mouse drag on the Desktop view (sliders, selecting text) and moves the pointer on the headset view.
|
||||
const TAP_MOVE = 8, HOLD_MS = 550, TOUCH_SPEED = 1.6, SCROLL_SPEED = 2;
|
||||
function ctrlTouchDown(e) {
|
||||
ctrl.pointers.set(e.pointerId, { x: e.clientX, y: e.clientY });
|
||||
if (ctrl.pointers.size === 1) {
|
||||
const g = ctrl.g = { kind: "maybe", x: e.clientX, y: e.clientY, e, target: ctrlAimedAt() };
|
||||
g.timer = setTimeout(() => {
|
||||
if (ctrl.g !== g || g.kind !== "maybe" || !ctrlSameTarget(g)) return;
|
||||
g.kind = "held";
|
||||
if (ctrlMode() === "abs") {
|
||||
const f = ctrlFraction(g.e);
|
||||
if (!f) return; // on the bars around the picture: nothing to press
|
||||
ctrlMoveTo(...f);
|
||||
}
|
||||
ctrlClick("right");
|
||||
navigator.vibrate?.(10);
|
||||
}, HOLD_MS);
|
||||
} else if (ctrl.pointers.size === 2) {
|
||||
if (ctrl.g?.timer) clearTimeout(ctrl.g.timer);
|
||||
if (ctrl.held.has("left")) ctrlButton("left", false);
|
||||
ctrl.g = { kind: "scroll", ...centroid() };
|
||||
}
|
||||
}
|
||||
// The panel a gesture started on; if that stops being the one in use, the gesture
|
||||
// does nothing more (except let go of what it holds).
|
||||
function ctrlAimedAt() { return ctrlMode() === "abs" ? panelKey(ctrlTarget().panel) : "headset"; }
|
||||
function ctrlSameTarget(g) { return g.target && g.target === ctrlAimedAt(); }
|
||||
function centroid() {
|
||||
const ps = [...ctrl.pointers.values()];
|
||||
return { x: ps.reduce((a, p) => a + p.x, 0) / ps.length, y: ps.reduce((a, p) => a + p.y, 0) / ps.length };
|
||||
}
|
||||
function ctrlTouchMove(e) {
|
||||
const prev = ctrl.pointers.get(e.pointerId);
|
||||
if (!prev || !ctrl.g) return;
|
||||
const g = ctrl.g, dx = e.clientX - prev.x, dy = e.clientY - prev.y;
|
||||
ctrl.pointers.set(e.pointerId, { x: e.clientX, y: e.clientY });
|
||||
if (g.kind === "scroll") {
|
||||
const c = centroid();
|
||||
// Content follows the fingers, as on the phone itself.
|
||||
ctrlSend([{ scroll: [-(c.x - g.x) * SCROLL_SPEED, -(c.y - g.y) * SCROLL_SPEED] }]);
|
||||
g.x = c.x; g.y = c.y;
|
||||
return;
|
||||
}
|
||||
if (g.kind === "maybe" && Math.hypot(e.clientX - g.x, e.clientY - g.y) > TAP_MOVE) {
|
||||
clearTimeout(g.timer);
|
||||
if (!ctrlSameTarget(g)) { g.kind = "done"; return; }
|
||||
if (ctrlMode() === "abs") {
|
||||
const f = ctrlFraction(g.e);
|
||||
if (f) { ctrlMoveTo(...f); ctrlButton("left", true); }
|
||||
g.kind = "drag";
|
||||
} else g.kind = "move";
|
||||
}
|
||||
if (g.kind === "drag") { const f = ctrlFraction(e); if (f) ctrlMoveTo(...f); }
|
||||
else if (g.kind === "move") ctrlMoveBy(dx * TOUCH_SPEED, dy * TOUCH_SPEED);
|
||||
}
|
||||
function ctrlTouchUp(e) {
|
||||
const g = ctrl.g;
|
||||
ctrl.pointers.delete(e.pointerId);
|
||||
if (!g) return;
|
||||
if (g.kind === "scroll") g.kind = "done"; // the finger left behind doesn't jump the scroll or drag
|
||||
if (g.kind === "maybe" && !ctrlSameTarget(g)) g.kind = "done";
|
||||
if (g.kind === "maybe" && ctrl.pointers.size === 0) {
|
||||
clearTimeout(g.timer);
|
||||
if (ctrlMode() === "abs") { const f = ctrlFraction(g.e); if (!f) { ctrl.g = null; return; } ctrlMoveTo(...f); }
|
||||
ctrlClick("left");
|
||||
} else if (g.kind === "drag") ctrlButton("left", false);
|
||||
if (ctrl.pointers.size === 0) ctrl.g = null;
|
||||
}
|
||||
// The system took the touch (a swipe home, a call): no tap, and let go of anything held.
|
||||
function ctrlTouchCancel(e) {
|
||||
ctrl.pointers.delete(e.pointerId);
|
||||
if (ctrl.g?.timer) clearTimeout(ctrl.g.timer);
|
||||
if (ctrl.held.has("left")) ctrlButton("left", false);
|
||||
if (ctrl.g) ctrl.g.kind = "done";
|
||||
if (ctrl.pointers.size === 0) ctrl.g = null;
|
||||
}
|
||||
const canvasEl = $("canvas");
|
||||
canvasEl.addEventListener("pointerdown", e => {
|
||||
if (!ctrl.on) return;
|
||||
e.preventDefault();
|
||||
$("viewer").focus({ preventScroll: true });
|
||||
try { canvasEl.setPointerCapture(e.pointerId); } catch { /* a pointer that's already gone */ }
|
||||
if (!ctrlTarget().ok) return ctrlShow();
|
||||
if (e.pointerType === "touch") return ctrlTouchDown(e);
|
||||
if (ctrlMode() === "abs" && !ctrlFraction(e)) return; // on the bars around the picture
|
||||
ctrlMouse(e);
|
||||
ctrlButton(CTRL_BUTTONS[e.button] || "left", true);
|
||||
});
|
||||
canvasEl.addEventListener("pointermove", e => {
|
||||
if (!ctrl.on || !ctrlTarget().ok) return;
|
||||
if (e.pointerType === "touch") return ctrlTouchMove(e);
|
||||
ctrlMouse(e);
|
||||
});
|
||||
for (const type of ["pointerup", "pointercancel"]) canvasEl.addEventListener(type, e => {
|
||||
if (!ctrl.on) return;
|
||||
if (e.pointerType === "touch") return type === "pointercancel" ? ctrlTouchCancel(e) : ctrlTouchUp(e);
|
||||
if (type === "pointercancel") return [...ctrl.held].forEach(b => ctrlButton(b, false)); // no button is named
|
||||
const name = CTRL_BUTTONS[e.button] || "left";
|
||||
if (ctrl.held.has(name)) ctrlButton(name, false);
|
||||
});
|
||||
canvasEl.addEventListener("contextmenu", e => { if (ctrl.on) e.preventDefault(); });
|
||||
$("viewer").addEventListener("wheel", e => {
|
||||
if (!ctrl.on || !ctrlTarget().ok) return;
|
||||
e.preventDefault();
|
||||
const unit = e.deltaMode === 1 ? 40 : e.deltaMode === 2 ? 800 : 1;
|
||||
ctrlSend([{ scroll: [e.deltaX * unit, e.deltaY * unit] }]);
|
||||
}, { passive: false });
|
||||
// Keys while the view has focus go to the Frame (⌘ as Ctrl on a Mac); click elsewhere to stop.
|
||||
function ctrlKey(e, down) {
|
||||
if (!ctrl.on || document.activeElement !== $("viewer") || (down && !ctrlTarget().ok)) return;
|
||||
let code = EVDEV[e.code];
|
||||
if (IS_MAC && (e.code === "MetaLeft" || e.code === "MetaRight")) code = 29;
|
||||
if (!code) return;
|
||||
e.preventDefault();
|
||||
e.stopPropagation(); // it's for the Frame, not Frame Control's own shortcuts
|
||||
if (down && e.repeat && [29, 42, 54, 56, 97, 100, 125, 126].includes(code)) return;
|
||||
if (!down && !ctrl.keys.has(code)) return;
|
||||
ctrlKeyEvent(code, down);
|
||||
}
|
||||
$("viewer").addEventListener("keydown", e => ctrlKey(e, true));
|
||||
$("viewer").addEventListener("keyup", e => ctrlKey(e, false));
|
||||
|
||||
// Phone keyboard: the field types plain text; the buttons send single keys and clicks.
|
||||
function ctrlTap(code) { ctrlKeyEvent(code, true); ctrlKeyEvent(code, false); }
|
||||
let ctrlWarned = false;
|
||||
function ctrlText(text) {
|
||||
const plain = text.replace(/[^\x20-\x7e\n\t]/g, "");
|
||||
if (plain !== text && !ctrlWarned) {
|
||||
ctrlWarned = true;
|
||||
toast("Control types plain letters, numbers and symbols. For accents and emoji, use Keyboard and trackpad below.");
|
||||
}
|
||||
// Small pieces: the Frame types them key by key, and a release queued behind mustn't wait long.
|
||||
for (const part of plain.match(/[\s\S]{1,100}/g) || []) ctrlSend([{ text: part }]);
|
||||
}
|
||||
$("ctrlType").addEventListener("beforeinput", e => {
|
||||
if (e.isComposing) return;
|
||||
e.preventDefault();
|
||||
if (!ctrlTarget().ok) return ctrlShow();
|
||||
if (e.inputType === "insertLineBreak" || e.inputType === "insertParagraph") return ctrlTap(28);
|
||||
if (e.inputType === "deleteContentBackward") return ctrlTap(14);
|
||||
if (e.inputType === "deleteContentForward") return ctrlTap(111);
|
||||
const text = e.data ?? e.dataTransfer?.getData("text/plain") ?? "";
|
||||
if (text) ctrlText(text);
|
||||
});
|
||||
$("ctrlType").addEventListener("compositionend", e => { if (e.data) ctrlText(e.data); $("ctrlType").value = ""; });
|
||||
$("ctrlKeys").addEventListener("click", e => {
|
||||
const k = e.target.closest("[data-ctrl-key]"), c = e.target.closest("[data-ctrl-click]");
|
||||
if (!ctrlTarget().ok) return ctrlShow();
|
||||
if (k) ctrlTap(+k.dataset.ctrlKey);
|
||||
if (c) ctrlClick(c.dataset.ctrlClick);
|
||||
});
|
||||
|
||||
// ---- controls ----
|
||||
$("refreshAll").onclick = () => { refresh(); loadTitles(); }; // titles too: the Frame may have been asleep at start
|
||||
document.addEventListener("keydown", e => {
|
||||
@@ -2517,6 +2875,7 @@ async function openShot(s) {
|
||||
await new Promise((ok, bad) => { img.onload = ok; img.onerror = () => bad(new Error("not an image")); img.src = url; });
|
||||
if (gen !== viewGen) return;
|
||||
lastImg = img; lastSource = "shot"; lastShot = { blob, file: s.file };
|
||||
desk.shown = null; // a screenshot, not a panel: Control has nothing to act on here
|
||||
draw();
|
||||
$("srcBadge").textContent = `Screenshot · ${shotApp(s.appid)}`;
|
||||
$("stamp").hidden = false; $("stamp").textContent = new Date(s.time * 1000).toLocaleString();
|
||||
|
||||
+134
-38
@@ -44,7 +44,6 @@ import frame_assistant # noqa: E402
|
||||
import frame_android # noqa: E402
|
||||
import frame_apk_versions # noqa: E402
|
||||
import frame_catalog # noqa: E402
|
||||
import frame_comfort # noqa: E402
|
||||
import frame_host # noqa: E402
|
||||
import frame_macview # noqa: E402
|
||||
import frame_media # noqa: E402
|
||||
@@ -271,37 +270,6 @@ def status(_body):
|
||||
return s
|
||||
|
||||
|
||||
def comfort(body):
|
||||
try:
|
||||
frame_comfort.validate(body)
|
||||
except ValueError as e:
|
||||
raise Failure(str(e), 400)
|
||||
# Content-addressed, user-only helper bundle. Desktop and phone use the same
|
||||
# on-headset state/lock; no listener, service registration or third-party app.
|
||||
import hashlib
|
||||
files = {name: (HERE / name).read_text() for name in
|
||||
("frame_comfort.py", "frame_status.py", "frame_steam.py")}
|
||||
version = hashlib.sha256(json.dumps(files, sort_keys=True).encode()).hexdigest()[:16]
|
||||
script = """import json, os, pathlib, subprocess, sys
|
||||
os.umask(0o077)
|
||||
files = %r
|
||||
root = pathlib.Path.home() / '.cache/frame-control/comfort' / %r
|
||||
root.mkdir(parents=True, exist_ok=True)
|
||||
for name, source in files.items():
|
||||
path = root / name
|
||||
if not path.exists():
|
||||
tmp = root / (name + '.' + str(os.getpid()))
|
||||
tmp.write_text(source)
|
||||
tmp.replace(path)
|
||||
r = subprocess.run([sys.executable, str(root / 'frame_comfort.py'), %r], capture_output=True, text=True)
|
||||
print(r.stdout, end='')
|
||||
""" % (files, version, json.dumps(body))
|
||||
out = json.loads(ssh("python3 -", stdin=script, timeout=65))
|
||||
if out.get("error") and "active" not in out:
|
||||
raise Failure(out["error"], 409)
|
||||
return out
|
||||
|
||||
|
||||
def headset_view():
|
||||
"""Both eyes as SteamVR composites them (see frame_vrshot.py); PNG bytes."""
|
||||
# `timeout`: VR_Init can block if SteamVR is restarting.
|
||||
@@ -432,6 +400,7 @@ _stream_proc = None
|
||||
|
||||
|
||||
def stream_command(query):
|
||||
"""The ffmpeg that streams H.264: the headset view, or one panel's own window (src=panel)."""
|
||||
q = parse_qs(query)
|
||||
try:
|
||||
height = int((q.get("h") or ["720"])[0])
|
||||
@@ -441,6 +410,16 @@ def stream_command(query):
|
||||
if height not in STREAM_HEIGHTS or fps not in STREAM_FPS:
|
||||
raise Failure(f"h must be one of {STREAM_HEIGHTS} and fps one of {STREAM_FPS}", 400)
|
||||
rate = 3 if height == 720 else 6 # Mbit/s
|
||||
if q.get("src") == ["panel"]:
|
||||
# The panel's own pixels (x11grab of its window: gamescope keeps them), fitted
|
||||
# to the height asked for. It stays still however the wearer moves their head.
|
||||
window, display = panel_target(q)
|
||||
return (f"DISPLAY={display} ffmpeg -hide_banner -loglevel error -nostdin -f x11grab -framerate {fps} "
|
||||
f"-window_id {window} -i {display} "
|
||||
f"-vf \"scale=-2:'trunc(min({height},ih)/2)*2',format=yuv420p\" -c:v libx264 -preset ultrafast "
|
||||
f"-tune zerolatency -g {fps * 2} -bf 0 -b:v {rate}M -maxrate {rate}M -bufsize {rate // 2 or 1}M "
|
||||
f"-x264-params aud=1:repeat-headers=1 -f h264 - & p=$!; "
|
||||
f"exec >&-; cat >/dev/null; kill $p 2>/dev/null; wait $p")
|
||||
return (f"[ -e {STREAM_DEVICE} ] || {{ echo 'No headset view device ({STREAM_DEVICE}). Is SteamVR running?' >&2; exit 3; }}; "
|
||||
f"ffmpeg -hide_banner -loglevel error -nostdin -f v4l2 -video_size 1920x1080 -i {STREAM_DEVICE} "
|
||||
f"-vf fps={fps},scale=-2:{height},format=yuv420p -c:v libx264 -preset ultrafast -tune zerolatency "
|
||||
@@ -660,7 +639,7 @@ class InputAgent:
|
||||
"""
|
||||
|
||||
def __init__(self, source=HERE / "frame_input_agent.py", packages=None):
|
||||
self.source, self.proc, self.lock = source, None, threading.Lock()
|
||||
self.source, self.proc, self.lock, self.write_lock = source, None, threading.Lock(), threading.Lock()
|
||||
self.packages = kdeconnect_packages() if packages is None else packages
|
||||
# generation counts stop()s; launching is the generation a launch is under way for.
|
||||
self.status, self.launching, self.generation = {"state": "off"}, None, 0
|
||||
@@ -833,8 +812,10 @@ class InputAgent:
|
||||
self.start()
|
||||
elif ready and events:
|
||||
try:
|
||||
proc.stdin.write((json.dumps(events) + "\n").encode())
|
||||
proc.stdin.flush()
|
||||
# One writer at a time: two devices sending at once mustn't tear a line.
|
||||
with self.write_lock:
|
||||
proc.stdin.write((json.dumps(events) + "\n").encode())
|
||||
proc.stdin.flush()
|
||||
sent = True
|
||||
except (BrokenPipeError, OSError, ValueError):
|
||||
pass # _watch reports how it ended
|
||||
@@ -874,6 +855,115 @@ def remote_input(body):
|
||||
return _input.send([input_event(e) for e in events])
|
||||
|
||||
|
||||
# ---- touch: the headset's panels, through gamescope's own input (frame_touch.py) ----
|
||||
|
||||
PANEL_WINDOW = re.compile(r"^\d{1,10}$")
|
||||
PANEL_DISPLAY = re.compile(r"^:\d{1,2}$")
|
||||
TOUCH_BUTTONS = ("left", "right", "middle")
|
||||
|
||||
|
||||
def panels():
|
||||
"""The headset's app panels (window, display, name, size) and which has focus."""
|
||||
return json.loads(ssh("python3 - panels", stdin=(HERE / "frame_touch.py").read_text(), timeout=20))
|
||||
|
||||
|
||||
def panel_target(q):
|
||||
window, display = (q.get("window") or [""])[0], (q.get("display") or [""])[0]
|
||||
if not PANEL_WINDOW.match(window) or not PANEL_DISPLAY.match(display):
|
||||
raise Failure("window must be a window id and display an X display such as :1", 400)
|
||||
return window, display
|
||||
|
||||
|
||||
def panel_capture(query):
|
||||
"""One frame of a panel's own window, as PNG (its pixels, without the room around it)."""
|
||||
window, display = panel_target(parse_qs(query))
|
||||
return ssh(f"DISPLAY={display} timeout 10 ffmpeg -hide_banner -loglevel error -nostdin -f x11grab "
|
||||
f"-window_id {window} -i {display} -frames:v 1 -f image2pipe -c:v png -", timeout=20, text=False)
|
||||
|
||||
|
||||
def touch_event(event):
|
||||
"""A frame_touch.py event with only the fields it knows, in range."""
|
||||
if not isinstance(event, dict):
|
||||
raise Failure("each touch event must be an object", 400)
|
||||
|
||||
def num(name, limit):
|
||||
value = event.get(name)
|
||||
if isinstance(value, bool) or not isinstance(value, (int, float)) or value != value:
|
||||
raise Failure(f"{name} must be a number", 400)
|
||||
return max(-limit, min(limit, round(float(value), 4)))
|
||||
out = {}
|
||||
if "fx" in event or "fy" in event:
|
||||
if "window" not in event:
|
||||
raise Failure("a position needs the panel's window and display", 400)
|
||||
out.update(fx=num("fx", 1), fy=num("fy", 1))
|
||||
# Any event can name the panel it's meant for; the Frame drops it if another has focus.
|
||||
if "window" in event:
|
||||
window, display = event.get("window"), event.get("display")
|
||||
if isinstance(window, bool) or not isinstance(window, int) or window <= 0:
|
||||
raise Failure("window must be the panel's window id", 400)
|
||||
if not isinstance(display, str) or not PANEL_DISPLAY.match(display):
|
||||
raise Failure("display must be an X display such as :1", 400)
|
||||
out.update(window=window, display=display)
|
||||
for name in ("dx", "dy"):
|
||||
if name in event:
|
||||
out[name] = num(name, INPUT_MOVE_LIMIT)
|
||||
if "button" in event:
|
||||
if event["button"] not in TOUCH_BUTTONS:
|
||||
raise Failure(f"button must be one of {', '.join(TOUCH_BUTTONS)}", 400)
|
||||
out["button"], out["down"] = event["button"], event.get("down") is not False
|
||||
if "scroll" in event:
|
||||
sc = event["scroll"]
|
||||
if not isinstance(sc, list) or len(sc) != 2:
|
||||
raise Failure("scroll must be [dx, dy]", 400)
|
||||
out["scroll"] = [num_value(v, 5000) for v in sc]
|
||||
if "key" in event:
|
||||
key = event["key"]
|
||||
if isinstance(key, bool) or not isinstance(key, int) or not 0 < key < 768:
|
||||
raise Failure("key must be a Linux key code", 400)
|
||||
out["key"], out["down"] = key, event.get("down") is not False
|
||||
if "text" in event:
|
||||
text = event["text"]
|
||||
if not isinstance(text, str) or not 0 < len(text) <= INPUT_TEXT_LIMIT:
|
||||
raise Failure(f"text must be 1 to {INPUT_TEXT_LIMIT} characters", 400)
|
||||
out["text"] = text
|
||||
if not set(out) - {"window", "display"}:
|
||||
raise Failure("touch event has nothing to do", 400)
|
||||
return out
|
||||
|
||||
|
||||
def num_value(value, limit):
|
||||
if isinstance(value, bool) or not isinstance(value, (int, float)) or value != value:
|
||||
raise Failure("scroll values must be numbers", 400)
|
||||
return max(-limit, min(limit, round(float(value), 2)))
|
||||
|
||||
|
||||
class TouchAgent(InputAgent):
|
||||
"""frame_touch.py on the Frame, fed events over one long-lived ssh. Nothing to install:
|
||||
it uses gamescope's own input socket and the libei that's on the image."""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(source=HERE / "frame_touch.py", packages=[])
|
||||
|
||||
def deliver(self, report, force=False):
|
||||
return ""
|
||||
|
||||
def command(self, folder=""):
|
||||
code = base64.b64encode(self.source.read_bytes()).decode()
|
||||
return "python3 -u -c " + shlex.quote(
|
||||
f"import base64;exec(compile(base64.b64decode('{code}'),'frame_touch','exec'))")
|
||||
|
||||
|
||||
_touch = TouchAgent()
|
||||
|
||||
|
||||
def remote_touch(body):
|
||||
"""{"events": [...]} points, clicks, scrolls and types into the focused panel."""
|
||||
events = body.get("events", [])
|
||||
if not isinstance(events, list) or len(events) > INPUT_BATCH_LIMIT:
|
||||
raise Failure(f"events must be a list of at most {INPUT_BATCH_LIMIT}", 400)
|
||||
return _touch.send([touch_event(e) for e in events])
|
||||
|
||||
|
||||
def flatpak(body):
|
||||
app, action = str(body.get("id", "")), body.get("action")
|
||||
if not FLATPAK_ID.match(app):
|
||||
@@ -1777,9 +1867,9 @@ def agent_approval(body):
|
||||
return frame_agent.approvals.decide(body.get("confirmation"), body.get("accept"))
|
||||
|
||||
|
||||
POST = {"/api/comfort": comfort, "/api/media": media, "/api/agent/call": agent_call, "/api/agent/approval": agent_approval,
|
||||
POST = {"/api/media": media, "/api/agent/call": agent_call, "/api/agent/approval": agent_approval,
|
||||
"/api/assistant/chat": assistant_chat, "/api/android/display": android_display, "/api/android": android, "/api/titles": titles, "/api/launch": launch, "/api/steam": steam, "/api/volume": set_volume, "/api/clipboard": clipboard,
|
||||
"/api/input": remote_input,
|
||||
"/api/input": remote_input, "/api/touch": remote_touch,
|
||||
"/api/flatpak": flatpak, "/api/open": open_thing, "/api/shots/save": save_shots,
|
||||
"/api/webinstall/check": webinstall_check, "/api/webinstall/start": webinstall_start,
|
||||
"/api/webinstall/cancel": webinstall_cancel,
|
||||
@@ -1913,6 +2003,10 @@ class Handler(BaseHTTPRequestHandler):
|
||||
self.send_json(title_job(url.query))
|
||||
elif path == "/api/licenses":
|
||||
self.send_json({"notices": licenses()})
|
||||
elif path == "/api/panels":
|
||||
self.send_json(panels())
|
||||
elif path == "/api/touch":
|
||||
self.send_json(_touch.send([]) if parse_qs(url.query).get("start") == ["1"] else dict(_touch.status))
|
||||
elif path == "/api/input":
|
||||
self.send_json(_input.send([]) if parse_qs(url.query).get("start") == ["1"] else dict(_input.status))
|
||||
elif path == "/api/job":
|
||||
@@ -1944,6 +2038,8 @@ class Handler(BaseHTTPRequestHandler):
|
||||
self.send_bytes(*shot_image(url.query))
|
||||
elif path == "/api/stream":
|
||||
self.stream_video(url.query)
|
||||
elif path == "/api/screenshot" and parse_qs(url.query).get("view") == ["panel"]:
|
||||
self.send_bytes(panel_capture(url.query), "image/png", headers=[("X-Capture-Source", "panel")])
|
||||
elif path == "/api/screenshot" and parse_qs(url.query).get("view") == ["headset"]:
|
||||
self.send_bytes(headset_view(), "image/png", headers=[("X-Capture-Source", "steamvr")])
|
||||
elif path == "/api/screenshot":
|
||||
@@ -2020,7 +2116,7 @@ class Handler(BaseHTTPRequestHandler):
|
||||
proc.wait()
|
||||
errors.seek(0)
|
||||
err = strip_ansi(errors.read().decode(errors="replace")).strip()
|
||||
raise Failure(err or "The headset view sent no video for 20 s")
|
||||
raise Failure(err or "The Frame sent no video for 20 s")
|
||||
chunk = first
|
||||
try:
|
||||
self.send_response(200)
|
||||
|
||||
Reference in new issue
Block a user