mirror of
https://github.com/saphid/frame-control.git
synced 2026-10-06 02:00:19 +02:00
Merge main into devices: switching headset stops the keyboard agent and retargets the Mac view
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
saphidandClaude Opus 5.5 committed
commit
08fbe730c6
74 files changed
+9786
-167
No files matched your search
@@ -0,0 +1,469 @@
|
||||
"""Keyboard and pointer for the Steam Frame. Frame Control's server runs this ON the Frame.
|
||||
|
||||
It speaks KDE Connect's LAN protocol (version 7, as in KDE Connect 24.02) to the
|
||||
Frame's own kdeconnectd, as a phone would, and forwards remote-input events read
|
||||
from stdin: one JSON object (or list of them) per line, each a KDE Connect
|
||||
"mousepad" request body such as {"dx": 4, "dy": -2} or {"key": "hello"}.
|
||||
KDE Connect does the typing and clicking.
|
||||
|
||||
KDE Connect isn't installed on the Frame. Frame Control ships Valve's build of it
|
||||
for the Frame and the few libraries the Frame lacks (frame/kdeconnect); the server
|
||||
copies them over the SSH connection and this unpacks them into
|
||||
~/.local/share/frame-control/kdeconnect: no root, no internet, and SteamOS
|
||||
updates leave it alone.
|
||||
|
||||
argv: client id, client name, the folder holding the packages, and a JSON list
|
||||
of [file, sha256] naming them (see frame/kdeconnect/packages.json).
|
||||
|
||||
Status goes to stdout, one JSON object per line:
|
||||
{"state": "installing" | "starting" | "pairing" | "ready" | "error" | "need-packages", ...}.
|
||||
|
||||
Standard library only: this runs on the Frame's own Python.
|
||||
"""
|
||||
import fcntl
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import selectors
|
||||
import shutil
|
||||
import signal
|
||||
import socket
|
||||
import ssl
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
BASE = Path.home() / ".local/share/frame-control/kdeconnect"
|
||||
ROOT = BASE / "root"
|
||||
BRIDGE = BASE / "bridge"
|
||||
STAMP = ".frame-control-packages" # in ROOT: which packages it was unpacked from
|
||||
PORT = int(os.environ.get("FRAME_INPUT_PORT", "1716"))
|
||||
UID = os.getuid()
|
||||
MOUSEPAD = "kdeconnect.mousepad.request"
|
||||
|
||||
|
||||
def say(state, **more):
|
||||
print(json.dumps({"state": state, **more}), flush=True)
|
||||
|
||||
|
||||
def packet(kind, body):
|
||||
return (json.dumps({"id": int(time.time() * 1000), "type": kind, "body": body}) + "\n").encode()
|
||||
|
||||
|
||||
# ---- KDE Connect on the Frame ------------------------------------------------
|
||||
|
||||
SYSTEM_DAEMON = Path("/usr/lib/kdeconnectd")
|
||||
|
||||
|
||||
def stamp(packages):
|
||||
"""What ROOT/STAMP holds once these packages are unpacked (the server checks it too)."""
|
||||
return "".join(f"{sha} {name}\n" for name, sha in packages)
|
||||
|
||||
|
||||
def installed(packages):
|
||||
try:
|
||||
return (ROOT / STAMP).read_text() == stamp(packages)
|
||||
except OSError:
|
||||
return False
|
||||
|
||||
|
||||
def sha256(path):
|
||||
digest = hashlib.sha256()
|
||||
with open(path, "rb") as f:
|
||||
for block in iter(lambda: f.read(1 << 20), b""):
|
||||
digest.update(block)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def install(folder, packages):
|
||||
"""Unpack the packages the server copied to `folder` into ROOT, checking each one first."""
|
||||
if not packages:
|
||||
raise RuntimeError("This copy of Frame Control doesn't include KDE Connect")
|
||||
say("installing", message="Unpacking KDE Connect on the Frame")
|
||||
stage = BASE / "root.new"
|
||||
shutil.rmtree(stage, ignore_errors=True)
|
||||
stage.mkdir(parents=True)
|
||||
for name, sha in packages:
|
||||
path = Path(folder) / name
|
||||
if not path.is_file():
|
||||
raise RuntimeError(f"{name} didn't reach the Frame")
|
||||
if sha256(path) != sha:
|
||||
raise RuntimeError(f"{name} arrived damaged (its SHA-256 doesn't match)")
|
||||
if subprocess.run(["tar", "--zstd", "-xf", str(path), "-C", str(stage)], capture_output=True).returncode:
|
||||
subprocess.run(["bsdtar", "-xf", str(path), "-C", str(stage)], check=True, capture_output=True)
|
||||
(stage / STAMP).write_text(stamp(packages))
|
||||
stop_daemon() # an older copy may still be running from ROOT
|
||||
shutil.rmtree(ROOT, ignore_errors=True)
|
||||
stage.rename(ROOT)
|
||||
|
||||
|
||||
def app_display():
|
||||
"""The X display that apps (not Steam's own VR menus) are on.
|
||||
|
||||
gamescope runs two Xwayland servers: on 2026-09-28 :0 held Steam's VR bar and
|
||||
menus and ignored XTest pointer motion, while :1 held apps such as Chromium and
|
||||
took it. Inferred to hold in general.
|
||||
"""
|
||||
return ":1" if Path("/tmp/.X11-unix/X1").exists() else ":0"
|
||||
|
||||
|
||||
def daemon_env(daemon):
|
||||
env = dict(os.environ, DBUS_SESSION_BUS_ADDRESS=f"unix:path=/run/user/{UID}/bus",
|
||||
XDG_RUNTIME_DIR=f"/run/user/{UID}", DISPLAY=app_display(), QT_QPA_PLATFORM="xcb")
|
||||
if str(daemon).startswith(str(ROOT)):
|
||||
env.update(LD_LIBRARY_PATH=str(ROOT / "usr/lib"), QT_PLUGIN_PATH=str(ROOT / "usr/lib/qt6/plugins"),
|
||||
QML_IMPORT_PATH=str(ROOT / "usr/lib/qt6/qml"),
|
||||
XDG_DATA_DIRS=f"{ROOT / 'usr/share'}:/usr/share")
|
||||
return env
|
||||
|
||||
|
||||
def listening():
|
||||
try:
|
||||
socket.create_connection(("127.0.0.1", PORT), 1).close()
|
||||
return True
|
||||
except OSError:
|
||||
return False
|
||||
|
||||
|
||||
def our_daemons():
|
||||
"""Process ids of the kdeconnectd that Frame Control installed (never a system one)."""
|
||||
pids = []
|
||||
for proc in Path("/proc").iterdir():
|
||||
if proc.name.isdigit():
|
||||
try:
|
||||
if os.readlink(proc / "exe").startswith(str(ROOT) + "/"):
|
||||
pids.append(int(proc.name))
|
||||
except OSError:
|
||||
pass
|
||||
return pids
|
||||
|
||||
|
||||
def stop_daemon():
|
||||
"""Stop our kdeconnectd and wait until it's gone (so its port is closed too)."""
|
||||
for sig, wait in ((signal.SIGTERM, 30), (signal.SIGKILL, 30)): # tenths of a second
|
||||
for pid in our_daemons():
|
||||
try:
|
||||
os.kill(pid, sig)
|
||||
except ProcessLookupError:
|
||||
pass
|
||||
for _ in range(wait):
|
||||
if not our_daemons() and not listening():
|
||||
return
|
||||
time.sleep(0.1)
|
||||
|
||||
|
||||
class NeedPackages(Exception):
|
||||
"""This build of KDE Connect isn't unpacked and the server didn't send it (it thought it was there)."""
|
||||
|
||||
|
||||
def ensure_daemon(folder, packages):
|
||||
"""Start KDE Connect: the Frame's own if it ever has one, else ours, unpacked first if needed.
|
||||
|
||||
A copy from another Frame Control version that another device is using right
|
||||
now is left running and used as it is (they speak the same protocol); it's
|
||||
replaced the next time nobody is using it.
|
||||
"""
|
||||
system = SYSTEM_DAEMON.exists()
|
||||
if not system and not installed(packages) and not (listening() and our_daemons()):
|
||||
if not folder or not Path(folder).is_dir():
|
||||
raise NeedPackages()
|
||||
install(folder, packages)
|
||||
if listening():
|
||||
return
|
||||
BASE.mkdir(parents=True, exist_ok=True)
|
||||
daemon = SYSTEM_DAEMON if system else ROOT / "usr/lib/kdeconnectd"
|
||||
say("starting", message="Starting KDE Connect on the Frame")
|
||||
log = open(BASE / "kdeconnectd.log", "ab")
|
||||
# Its own session, so it outlives this connection and serves the next one.
|
||||
subprocess.Popen([str(daemon)], env=daemon_env(daemon), cwd=str(Path.home()), stdin=subprocess.DEVNULL,
|
||||
stdout=log, stderr=log, start_new_session=True)
|
||||
for _ in range(40):
|
||||
if listening():
|
||||
return
|
||||
time.sleep(0.25)
|
||||
raise RuntimeError(f"KDE Connect didn't start; see {BASE / 'kdeconnectd.log'} on the Frame")
|
||||
|
||||
|
||||
def qdbus(device, method):
|
||||
"""Call a method on KDE Connect's D-Bus object for our device; its output, or None."""
|
||||
env = dict(os.environ, DBUS_SESSION_BUS_ADDRESS=f"unix:path=/run/user/{UID}/bus")
|
||||
try:
|
||||
r = subprocess.run(["qdbus6", "org.kde.kdeconnect", f"/modules/kdeconnect/devices/{device}",
|
||||
f"org.kde.kdeconnect.device.{method}"], capture_output=True, text=True, env=env, timeout=5)
|
||||
except (OSError, subprocess.TimeoutExpired):
|
||||
return None
|
||||
return r.stdout.strip() if r.returncode == 0 else None
|
||||
|
||||
|
||||
# ---- our identity --------------------------------------------------------------
|
||||
|
||||
def identity(client):
|
||||
"""A device id and certificate for this client, made once and kept (pairing is tied to them).
|
||||
|
||||
Each computer or phone gets its own: KDE Connect keeps one connection per device,
|
||||
so a shared identity would make them knock each other off.
|
||||
"""
|
||||
folder = BRIDGE / client
|
||||
folder.mkdir(parents=True, exist_ok=True)
|
||||
id_file, cert, key = folder / "id", folder / "cert.pem", folder / "key.pem"
|
||||
if not (id_file.exists() and cert.exists() and key.exists()):
|
||||
device = "framecontrol_" + os.urandom(12).hex() # KDE Connect wants 32-38 of [A-Za-z0-9_]
|
||||
subprocess.run(["openssl", "req", "-x509", "-newkey", "ec", "-pkeyopt", "ec_paramgen_curve:prime256v1",
|
||||
"-nodes", "-days", "3650", "-subj", f"/O=KDE/OU=Kde connect/CN={device}",
|
||||
"-keyout", str(key), "-out", str(cert)], check=True, capture_output=True)
|
||||
os.chmod(key, 0o600)
|
||||
id_file.write_text(device)
|
||||
return id_file.read_text().strip(), cert, key
|
||||
|
||||
|
||||
# ---- the link ------------------------------------------------------------------
|
||||
|
||||
class Link:
|
||||
"""One TLS connection to kdeconnectd, as a paired device that sends remote input."""
|
||||
|
||||
def __init__(self, device, cert, key, port=PORT, name="Frame Control"):
|
||||
self.device, self.buf, self.keyboard = device, b"", None
|
||||
raw = socket.create_connection(("127.0.0.1", port), 5)
|
||||
raw.sendall(packet("kdeconnect.identity", {
|
||||
"deviceId": device, "deviceName": name, "deviceType": "phone", "protocolVersion": 7,
|
||||
"incomingCapabilities": [], "outgoingCapabilities": [MOUSEPAD], "tcpPort": port}))
|
||||
# KDE Connect's rule: whoever opened the TCP connection is the TLS server.
|
||||
ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
|
||||
ctx.load_cert_chain(str(cert), str(key))
|
||||
ctx.verify_mode = ssl.CERT_NONE # both sides are on this machine
|
||||
self.sock = ctx.wrap_socket(raw, server_side=True)
|
||||
self.sock.setblocking(False)
|
||||
|
||||
def send(self, body):
|
||||
# Bounded: if KDE Connect stops reading, fail (and be restarted) rather than hang.
|
||||
self.sock.settimeout(5)
|
||||
try:
|
||||
self.sock.sendall(packet(MOUSEPAD, body))
|
||||
finally:
|
||||
self.sock.setblocking(False)
|
||||
|
||||
def pair(self, paired, accept, timeout=15):
|
||||
"""Ask to pair and accept it on KDE Connect's side (we control both ends).
|
||||
|
||||
Only asks when not already paired: a pair request to a device that is
|
||||
already paired makes KDE Connect unpair it.
|
||||
"""
|
||||
if paired():
|
||||
return
|
||||
self.sock.settimeout(5)
|
||||
self.sock.sendall(packet("kdeconnect.pair", {"pair": True}))
|
||||
self.sock.setblocking(False)
|
||||
end = time.time() + timeout
|
||||
while time.time() < end:
|
||||
accept()
|
||||
self.read(0.5)
|
||||
if paired():
|
||||
return
|
||||
raise RuntimeError("KDE Connect didn't accept the pairing")
|
||||
|
||||
def read(self, wait=0.0):
|
||||
"""Packets waiting from kdeconnectd; None once it has closed the connection."""
|
||||
if wait:
|
||||
sel = selectors.DefaultSelector()
|
||||
sel.register(self.sock, selectors.EVENT_READ)
|
||||
sel.select(wait)
|
||||
sel.close()
|
||||
try:
|
||||
while True:
|
||||
chunk = self.sock.recv(65536)
|
||||
if not chunk:
|
||||
return None
|
||||
self.buf += chunk
|
||||
except (ssl.SSLWantReadError, BlockingIOError):
|
||||
pass
|
||||
out = []
|
||||
while b"\n" in self.buf:
|
||||
line, self.buf = self.buf.split(b"\n", 1)
|
||||
if line.strip():
|
||||
p = json.loads(line)
|
||||
if p.get("type") == "kdeconnect.mousepad.keyboardstate":
|
||||
self.keyboard = bool(p.get("body", {}).get("state"))
|
||||
out.append(p)
|
||||
return out
|
||||
|
||||
|
||||
def events(line):
|
||||
"""The event bodies in one stdin line (an object or a list of objects)."""
|
||||
try:
|
||||
value = json.loads(line)
|
||||
except ValueError:
|
||||
return []
|
||||
return [e for e in (value if isinstance(value, list) else [value]) if isinstance(e, dict) and e]
|
||||
|
||||
|
||||
def connect(device, cert, key, name):
|
||||
link = Link(device, cert, key, name=name)
|
||||
link.pair(lambda: qdbus(device, "isPaired") == "true", lambda: qdbus(device, "acceptPairing"))
|
||||
link.read(0.5) # its hello, including whether it can type
|
||||
return link
|
||||
|
||||
|
||||
def client_args():
|
||||
"""argv: a folder-safe id for the computer or phone, the name KDE Connect shows for it,
|
||||
the folder holding the packages, and their [file, sha256] list."""
|
||||
client = sys.argv[1] if len(sys.argv) > 1 else "default"
|
||||
client = "".join(c for c in client if c.isalnum() or c in "-_")[:64] or "default"
|
||||
name = (sys.argv[2] if len(sys.argv) > 2 else "")[:60].strip()
|
||||
folder = os.path.expanduser(sys.argv[3]) if len(sys.argv) > 3 else ""
|
||||
try:
|
||||
packages = [(str(f), str(h)) for f, h in json.loads(sys.argv[4])] if len(sys.argv) > 4 else []
|
||||
except (ValueError, TypeError):
|
||||
packages = []
|
||||
return client, f"Frame Control ({name})" if name else "Frame Control", folder, packages
|
||||
|
||||
|
||||
def main():
|
||||
client, name, folder, packages = client_args()
|
||||
# A dropped ssh (the Frame slept, the app quit) hangs up on us: exit through the
|
||||
# clean-up below rather than dying on the spot.
|
||||
for sig in (signal.SIGHUP, signal.SIGTERM):
|
||||
signal.signal(sig, lambda *_: sys.exit(0))
|
||||
BASE.mkdir(parents=True, exist_ok=True)
|
||||
# Every agent holds this lock shared while it runs. The last one out gets it
|
||||
# exclusively and stops KDE Connect, so it runs, and shows up on the network,
|
||||
# only while something is using the keyboard and trackpad.
|
||||
clients = open(BASE / "clients.lock", "w")
|
||||
fcntl.flock(clients, fcntl.LOCK_SH)
|
||||
try:
|
||||
return run(client, name, folder, packages)
|
||||
finally:
|
||||
# Finish the clean-up even if a second hang-up or TERM arrives meanwhile.
|
||||
for sig in (signal.SIGHUP, signal.SIGTERM):
|
||||
signal.signal(sig, signal.SIG_IGN)
|
||||
fcntl.flock(clients, fcntl.LOCK_UN)
|
||||
try:
|
||||
fcntl.flock(clients, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
except OSError:
|
||||
pass # another device is still using it
|
||||
else:
|
||||
with daemon_lock():
|
||||
stop_daemon()
|
||||
|
||||
|
||||
def tidy_incoming(folder):
|
||||
"""Remove this start's copy of the packages, and others nobody is using.
|
||||
|
||||
Another copy goes only if no agent holds its .in-use lock and it's over an
|
||||
hour old (so not one a server is still copying, before its agent starts).
|
||||
"""
|
||||
incoming = BASE / "incoming"
|
||||
if folder.startswith(str(incoming) + "/"):
|
||||
shutil.rmtree(folder, ignore_errors=True)
|
||||
try:
|
||||
others = list(incoming.iterdir())
|
||||
except OSError:
|
||||
return
|
||||
for other in others:
|
||||
try:
|
||||
if time.time() - other.stat().st_mtime < 3600:
|
||||
continue
|
||||
with open(other / ".in-use", "a") as lock:
|
||||
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
shutil.rmtree(other, ignore_errors=True)
|
||||
except OSError:
|
||||
pass # in use, or already gone
|
||||
try:
|
||||
incoming.rmdir()
|
||||
except OSError:
|
||||
pass # another start's copy is still there
|
||||
|
||||
|
||||
def hold_incoming(folder):
|
||||
"""Mark this start's copy as in use (tidy_incoming leaves it alone); the lock lasts as long as the file."""
|
||||
if not folder.startswith(str(BASE / "incoming") + "/"):
|
||||
return None
|
||||
try:
|
||||
lock = open(Path(folder) / ".in-use", "a")
|
||||
fcntl.flock(lock, fcntl.LOCK_SH)
|
||||
return lock
|
||||
except OSError:
|
||||
return None
|
||||
|
||||
|
||||
class daemon_lock:
|
||||
"""Installing, starting and restarting KDE Connect happen one agent at a time."""
|
||||
|
||||
def __enter__(self):
|
||||
self.file = open(BASE / "daemon.lock", "w")
|
||||
fcntl.flock(self.file, fcntl.LOCK_EX)
|
||||
|
||||
def __exit__(self, *_):
|
||||
self.file.close()
|
||||
|
||||
|
||||
def run(client, name, folder, packages):
|
||||
"""Set up, pair and forward events. This start's copy of the packages stays
|
||||
until it ends, however it ends: restarting KDE Connect may need to unpack it."""
|
||||
held = hold_incoming(folder)
|
||||
try:
|
||||
return serve(client, name, folder, packages)
|
||||
finally:
|
||||
if held:
|
||||
held.close()
|
||||
tidy_incoming(folder)
|
||||
|
||||
|
||||
def serve(client, name, folder, packages):
|
||||
try:
|
||||
with daemon_lock():
|
||||
ensure_daemon(folder, packages)
|
||||
device, cert, key = identity(client)
|
||||
say("pairing")
|
||||
seen = our_daemons()
|
||||
try:
|
||||
link = connect(device, cert, key, name)
|
||||
except (OSError, RuntimeError):
|
||||
if not seen:
|
||||
raise
|
||||
# Ours, but not answering (KDE Connect 24.02 can hang, for one after
|
||||
# unpairing a device that's offline): start it afresh, once. If another
|
||||
# agent already replaced it, just use the new one.
|
||||
say("starting", message="Restarting KDE Connect on the Frame")
|
||||
with daemon_lock():
|
||||
if set(our_daemons()) & set(seen):
|
||||
stop_daemon()
|
||||
ensure_daemon(folder, packages)
|
||||
link = connect(device, cert, key, name)
|
||||
except NeedPackages:
|
||||
say("need-packages") # the server copies them and starts again
|
||||
return 1
|
||||
except (OSError, RuntimeError, subprocess.SubprocessError) as e:
|
||||
say("error", message=str(e))
|
||||
return 1
|
||||
say("ready", keyboard=link.keyboard is not False)
|
||||
stdin, pending = sys.stdin.fileno(), b""
|
||||
sel = selectors.DefaultSelector()
|
||||
sel.register(stdin, selectors.EVENT_READ)
|
||||
sel.register(link.sock, selectors.EVENT_READ)
|
||||
while True:
|
||||
for key_, _ in sel.select(30):
|
||||
if key_.fileobj == stdin:
|
||||
chunk = os.read(stdin, 65536) # raw reads: a buffered readline could strand lines select can't see
|
||||
if not chunk: # the server went away
|
||||
return 0
|
||||
*lines, pending = (pending + chunk).split(b"\n")
|
||||
try:
|
||||
for line in lines:
|
||||
for body in events(line):
|
||||
link.send(body)
|
||||
except OSError as e:
|
||||
say("error", message=f"Lost KDE Connect: {e}")
|
||||
return 1
|
||||
else:
|
||||
packets = link.read()
|
||||
if packets is None:
|
||||
say("error", message="KDE Connect closed the connection")
|
||||
return 1
|
||||
if any(p.get("type") == "kdeconnect.pair" and not p.get("body", {}).get("pair") for p in packets):
|
||||
say("error", message="KDE Connect unpaired Frame Control")
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,67 @@
|
||||
"""Media planning shared by Frame Control and its own Frame-side player.
|
||||
|
||||
No viewer dependencies. Filename hints are suggestions, never guesses from
|
||||
resolution. Explicit layout wins; conflicting hints require a choice.
|
||||
"""
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
LAYOUTS = ('auto', 'mono', 'sbs', 'ou', 'full-sbs', 'full-ou')
|
||||
VIDEO = {'.mp4', '.mkv', '.mov', '.webm', '.m4v'}
|
||||
PHOTO = {'.png', '.jpg', '.jpeg'}
|
||||
|
||||
|
||||
def plan(name, layout='auto', metadata=None):
|
||||
if layout not in LAYOUTS:
|
||||
raise ValueError('Choose auto, mono, sbs, ou, full-sbs or full-ou')
|
||||
suffix = Path(name).suffix.lower()
|
||||
if suffix in {'.heic', '.heif', '.avif', '.mpo'}:
|
||||
raise ValueError('Native spatial-photo containers are not supported yet; export both eyes as SBS or OU PNG/JPEG')
|
||||
if suffix == '.splat':
|
||||
return {'kind': 'splat', 'layout': 'sbs', 'source': 'renderer'}
|
||||
if suffix not in VIDEO | PHOTO:
|
||||
raise ValueError('Use MP4/MKV/MOV/WebM video, PNG/JPEG stereo photos, or a .splat file')
|
||||
source = 'explicit'
|
||||
if layout == 'auto':
|
||||
tokens = set(re.split(r'[^a-z0-9]+', Path(name).stem.lower()))
|
||||
hints = set()
|
||||
for value, tags in [('full-sbs', {'fsbs'}), ('full-ou', {'fou', 'ftb'}),
|
||||
('sbs', {'sbs', 'hsbs', 'lr'}), ('ou', {'ou', 'hou', 'tb', 'htb'})]:
|
||||
if tokens & tags:
|
||||
hints.add(value)
|
||||
if len(hints) > 1:
|
||||
raise ValueError('Conflicting stereo filename tags; choose the layout explicitly')
|
||||
layout = next(iter(hints), None)
|
||||
source = 'filename'
|
||||
if not layout:
|
||||
# Matroska StereoMode/FFmpeg stereo_mode: only known left-first modes.
|
||||
mode = (metadata or {}).get('stereo_mode')
|
||||
layout = {'left_right': 'full-sbs', 'top_bottom': 'full-ou', 'mono': 'mono'}.get(mode)
|
||||
source = 'metadata'
|
||||
if mode and layout is None:
|
||||
raise ValueError('Unsupported stereo metadata; choose the eye order/layout explicitly')
|
||||
if not layout:
|
||||
raise ValueError('No stereo layout found; choose mono, SBS or OU (left/top eye first)')
|
||||
return {'kind': 'video' if suffix in VIDEO else 'photo', 'layout': layout, 'source': source}
|
||||
|
||||
|
||||
def geometry(width, height, layout):
|
||||
"""Bound transfer to 1920x1080; return packed dimensions and texel aspect."""
|
||||
if not 0 < width <= 32768 or not 0 < height <= 32768:
|
||||
raise ValueError('Invalid media dimensions')
|
||||
if layout not in LAYOUTS[1:]:
|
||||
raise ValueError('Resolve the layout before playback')
|
||||
scale = min(1, 1920 / width, 1080 / height)
|
||||
w, h = max(2, int(width * scale) // 2 * 2), max(2, int(height * scale) // 2 * 2)
|
||||
return w, h, {'mono': 1, 'sbs': 2, 'ou': .5, 'full-sbs': 1, 'full-ou': 1}[layout]
|
||||
|
||||
|
||||
def stereo_pixels(data, width, height, layout):
|
||||
"""Normalize top/bottom to OpenVR's left/right texture; preserve eye order."""
|
||||
if len(data) != width * height * 4:
|
||||
raise ValueError('Incomplete RGBA frame')
|
||||
if layout not in ('ou', 'full-ou'):
|
||||
return data, width, height
|
||||
stride, half = width * 4, height // 2
|
||||
return b''.join(data[y*stride:(y+1)*stride] +
|
||||
data[(y+half)*stride:(y+half+1)*stride] for y in range(half)), width*2, half
|
||||
@@ -0,0 +1,46 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Send local media to Frame Control's own OpenVR player."""
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
import sys
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
import frame_media
|
||||
import server
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description=__doc__)
|
||||
ap.add_argument('files', nargs='*', type=Path)
|
||||
ap.add_argument('--launch', action='store_true', help='play the one file being sent')
|
||||
ap.add_argument('--layout', choices=frame_media.LAYOUTS, default='auto')
|
||||
ap.add_argument('--theatre', action='store_true', help='bigger screen and dark surround')
|
||||
ap.add_argument('--list', action='store_true')
|
||||
ap.add_argument('--stop', action='store_true')
|
||||
args = ap.parse_args()
|
||||
if args.launch and len(args.files) != 1:
|
||||
ap.error('--launch needs exactly one file')
|
||||
if not args.files and not (args.list or args.stop):
|
||||
ap.error('choose files, --list or --stop')
|
||||
for path in args.files:
|
||||
if not path.is_file():
|
||||
ap.error('not a file: %s' % path)
|
||||
frame_media.plan(path.name, 'mono')
|
||||
if args.stop:
|
||||
print(json.dumps(server.media({'action': 'stop'})))
|
||||
for path in args.files:
|
||||
result = server.push_media(path.resolve())
|
||||
print(json.dumps(result))
|
||||
if args.launch:
|
||||
print(json.dumps(server.media({'action': 'play', 'id': result['id'],
|
||||
'layout': args.layout, 'theatre': args.theatre})))
|
||||
if args.list:
|
||||
print(json.dumps(server.media({'action': 'list'})))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
try:
|
||||
main()
|
||||
except (ValueError, server.Failure) as e:
|
||||
sys.exit(str(e))
|
||||
@@ -0,0 +1,213 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Frame Control's local-media OpenVR player. Runs on the Frame, no third-party app.
|
||||
|
||||
SteamOS ffmpeg does hardware video decoding, scaling and audio output. OpenVR
|
||||
owns only our screen and optional black surround. Exiting destroys both.
|
||||
"""
|
||||
import argparse
|
||||
import ctypes as C
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import signal
|
||||
import subprocess
|
||||
import time
|
||||
|
||||
import frame_media
|
||||
import frame_splat
|
||||
|
||||
LIB = '/opt/steamvr/bin/linuxarm64/libopenvr_api.so'
|
||||
H = C.c_uint64
|
||||
# Slots from Valve's openvr_capi.h, IVROverlay_028. Fail closed on another ABI.
|
||||
SLOTS = {
|
||||
'CreateOverlay': (1, [C.c_char_p, C.c_char_p, C.POINTER(H)]),
|
||||
'DestroyOverlay': (3, [H]),
|
||||
'SetOverlayFlag': (11, [H, C.c_int, C.c_bool]),
|
||||
'SetOverlayAlpha': (16, [H, C.c_float]),
|
||||
'SetOverlayTexelAspect': (18, [H, C.c_float]),
|
||||
'SetOverlaySortOrder': (20, [H, C.c_uint32]),
|
||||
'SetOverlayWidthInMeters': (22, [H, C.c_float]),
|
||||
'SetOverlayTransformTrackedDeviceRelative': (35, [H, C.c_uint32, C.c_void_p]),
|
||||
'ShowOverlay': (43, [H]),
|
||||
'SetOverlayRaw': (62, [H, C.c_void_p, C.c_uint32, C.c_uint32, C.c_uint32]),
|
||||
}
|
||||
|
||||
|
||||
class Overlay:
|
||||
def __init__(self):
|
||||
self.handles = []
|
||||
self.vr = C.CDLL(LIB)
|
||||
self.vr.VR_InitInternal2.argtypes = [C.POINTER(C.c_int), C.c_int, C.c_char_p]
|
||||
self.vr.VR_GetGenericInterface.argtypes = [C.c_char_p, C.POINTER(C.c_int)]
|
||||
self.vr.VR_GetGenericInterface.restype = C.c_void_p
|
||||
err = C.c_int()
|
||||
self.vr.VR_InitInternal2(C.byref(err), 2, None)
|
||||
if err.value:
|
||||
raise RuntimeError('SteamVR init failed: %s' % err.value)
|
||||
ptr = self.vr.VR_GetGenericInterface(b'FnTable:IVROverlay_028', C.byref(err))
|
||||
if not ptr or err.value:
|
||||
self.vr.VR_ShutdownInternal()
|
||||
raise RuntimeError('SteamVR needs IVROverlay_028: %s' % err.value)
|
||||
self.table = C.cast(ptr, C.POINTER(C.c_void_p))
|
||||
|
||||
def call(self, name, *values):
|
||||
slot, args = SLOTS[name]
|
||||
rc = C.CFUNCTYPE(C.c_int, *args)(self.table[slot])(*values)
|
||||
if rc:
|
||||
raise RuntimeError('OpenVR %s failed: %s' % (name, rc))
|
||||
|
||||
def create(self, key, width, distance, stereo=False, aspect=1, order=1):
|
||||
handle = H()
|
||||
self.call('CreateOverlay', key.encode(), b'Frame Control media', C.byref(handle))
|
||||
self.handles.append(handle)
|
||||
self.call('SetOverlayWidthInMeters', handle, width)
|
||||
self.call('SetOverlaySortOrder', handle, order)
|
||||
self.call('SetOverlayTexelAspect', handle, aspect)
|
||||
if stereo:
|
||||
self.call('SetOverlayFlag', handle, 1024, True) # SideBySide_Parallel
|
||||
matrix = (C.c_float * 12)(1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, -distance)
|
||||
self.call('SetOverlayTransformTrackedDeviceRelative', handle, 0, matrix)
|
||||
return handle
|
||||
|
||||
def pixels(self, handle, data, width, height):
|
||||
buf = C.create_string_buffer(data)
|
||||
self.call('SetOverlayRaw', handle, buf, width, height, 4)
|
||||
self.call('ShowOverlay', handle)
|
||||
|
||||
def close(self):
|
||||
try:
|
||||
for h in reversed(self.handles):
|
||||
self.call('DestroyOverlay', h)
|
||||
finally:
|
||||
self.vr.VR_ShutdownInternal()
|
||||
|
||||
|
||||
def probe(path):
|
||||
result = subprocess.run(['ffprobe', '-v', 'error', '-show_streams', '-of', 'json', str(path)],
|
||||
capture_output=True, text=True, timeout=30)
|
||||
if result.returncode:
|
||||
raise ValueError(result.stderr[-2000:] or 'Cannot read media')
|
||||
streams = json.loads(result.stdout)['streams']
|
||||
video = next((s for s in streams if s['codec_type'] == 'video'), None)
|
||||
if not video:
|
||||
raise ValueError('No image or video stream')
|
||||
return video, any(s['codec_type'] == 'audio' for s in streams)
|
||||
|
||||
|
||||
def decoder_command(path, info, width, height, audio, photo=False):
|
||||
cmd = ['ffmpeg', '-nostdin', '-hide_banner', '-loglevel', 'error']
|
||||
if not photo:
|
||||
cmd += ['-re', '-readrate_initial_burst', '0']
|
||||
codec = {'h264': 'h264_v4l2m2m', 'hevc': 'hevc_v4l2m2m'}.get(info['codec_name'])
|
||||
if not codec:
|
||||
raise ValueError('Hardware playback currently supports H.264 and H.265 only')
|
||||
cmd += ['-c:v', codec]
|
||||
cmd += ['-i', str(path), '-map', '0:v:0', '-vf', 'scale=%s:%s' % (width, height),
|
||||
'-pix_fmt', 'rgba']
|
||||
if photo:
|
||||
cmd += ['-frames:v', '1']
|
||||
else:
|
||||
cmd += ['-r', '30']
|
||||
cmd += ['-f', 'rawvideo', 'pipe:1']
|
||||
if audio and not photo:
|
||||
cmd += ['-map', '0:a:0', '-f', 'pulse', 'Frame Control Media']
|
||||
return cmd
|
||||
|
||||
|
||||
def write_status(path, **values):
|
||||
tmp = path.with_suffix('.tmp')
|
||||
tmp.write_text(json.dumps(values))
|
||||
tmp.replace(path)
|
||||
|
||||
|
||||
def play(args):
|
||||
path = Path(args.file).resolve(strict=True)
|
||||
status = Path(args.status)
|
||||
splat = path.suffix.lower() == '.splat'
|
||||
if splat:
|
||||
data, width, height = frame_splat.render(path)
|
||||
plan = frame_media.plan(path.name)
|
||||
aspect, photo, command = 1, True, None
|
||||
else:
|
||||
info, audio = probe(path)
|
||||
plan = frame_media.plan(path.name, args.layout, info.get('tags'))
|
||||
width, height, aspect = frame_media.geometry(info['width'], info['height'], plan['layout'])
|
||||
photo = plan['kind'] == 'photo'
|
||||
command = decoder_command(path, info, width, height, audio, photo)
|
||||
vr, proc, frames, started = None, None, 0, time.monotonic()
|
||||
# systemd sends SIGTERM to the whole unit, including ffmpeg. Python unwinds
|
||||
# ownership; no unrelated Steam/SteamVR process or setting is touched.
|
||||
def stop(signum, frame):
|
||||
raise InterruptedError('Stopped')
|
||||
signal.signal(signal.SIGTERM, stop)
|
||||
signal.signal(signal.SIGINT, stop)
|
||||
try:
|
||||
vr = Overlay()
|
||||
if args.theatre:
|
||||
surround = vr.create('framecontrol.media.surround', 40, 4, order=0)
|
||||
vr.call('SetOverlayAlpha', surround, .85)
|
||||
vr.pixels(surround, b'\x00\x00\x00\xff', 1, 1)
|
||||
screen = vr.create('framecontrol.media.screen', 3 if args.theatre else 1.6, 2,
|
||||
plan['layout'] != 'mono', aspect)
|
||||
if splat:
|
||||
vr.pixels(screen, data, width, height)
|
||||
write_status(status, state='playing', file=path.name, frames=1, **plan)
|
||||
while True:
|
||||
time.sleep(1)
|
||||
proc = subprocess.Popen(command, stdout=subprocess.PIPE)
|
||||
video_start = time.monotonic()
|
||||
while True:
|
||||
data = proc.stdout.read(width * height * 4)
|
||||
if not data:
|
||||
break
|
||||
data, outw, outh = frame_media.stereo_pixels(data, width, height, plan['layout'])
|
||||
if not photo:
|
||||
time.sleep(max(0, video_start + frames/30 - time.monotonic()))
|
||||
vr.pixels(screen, data, outw, outh)
|
||||
frames += 1
|
||||
if frames == 1 or frames % 30 == 0:
|
||||
write_status(status, state='playing', file=path.name, frames=frames,
|
||||
seconds=time.monotonic()-started, **plan)
|
||||
if not photo:
|
||||
time.sleep(max(0, video_start + frames/30 - time.monotonic()))
|
||||
rc = proc.wait(timeout=10)
|
||||
if rc:
|
||||
raise RuntimeError('ffmpeg exited %s; see media log' % rc)
|
||||
if not frames:
|
||||
raise RuntimeError('Decoder produced no frames')
|
||||
if photo:
|
||||
while True:
|
||||
time.sleep(1)
|
||||
write_status(status, state='ended', frames=frames, seconds=time.monotonic()-started)
|
||||
except InterruptedError:
|
||||
write_status(status, state='stopped', frames=frames)
|
||||
finally:
|
||||
if proc:
|
||||
if proc.poll() is None:
|
||||
proc.terminate()
|
||||
try:
|
||||
proc.wait(timeout=5)
|
||||
except subprocess.TimeoutExpired:
|
||||
proc.kill()
|
||||
proc.wait()
|
||||
proc.stdout.close()
|
||||
if vr:
|
||||
vr.close()
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description=__doc__)
|
||||
ap.add_argument('file')
|
||||
ap.add_argument('--layout', choices=frame_media.LAYOUTS, default='auto')
|
||||
ap.add_argument('--theatre', action='store_true')
|
||||
ap.add_argument('--status', required=True)
|
||||
args = ap.parse_args()
|
||||
try:
|
||||
play(args)
|
||||
except Exception as e:
|
||||
write_status(Path(args.status), state='error', error=str(e))
|
||||
raise
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,109 @@
|
||||
"""Frame-side library and process ownership for Frame Control media.
|
||||
|
||||
Only the dedicated systemd user unit is controlled. No SteamVR settings change.
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
import frame_media
|
||||
from frame_media_player import probe
|
||||
|
||||
ROOT = Path.home() / 'Videos' / 'FrameControl'
|
||||
RUNTIME = Path.home() / '.local' / 'share' / 'frame-control' / 'media'
|
||||
UNIT = 'frame-control-media.service'
|
||||
STATUS = RUNTIME / 'status.json'
|
||||
|
||||
|
||||
def media_path(identity):
|
||||
if not isinstance(identity, str) or '\\' in identity or '\x00' in identity:
|
||||
raise ValueError('Invalid media id')
|
||||
parts = Path(identity).parts
|
||||
if len(parts) != 2 or not re.fullmatch('[0-9a-f]{32}', parts[0]) or parts[1].startswith('.'):
|
||||
raise ValueError('Invalid media id')
|
||||
candidate = ROOT / identity
|
||||
if candidate.is_symlink() or candidate.parent.is_symlink():
|
||||
raise ValueError('Media links are not supported')
|
||||
path = candidate.resolve(strict=True)
|
||||
if not path.is_file() or ROOT.resolve() not in path.parents:
|
||||
raise ValueError('Media file is outside the library')
|
||||
return path
|
||||
|
||||
|
||||
def active():
|
||||
return subprocess.run(['systemctl', '--user', 'is-active', '--quiet', UNIT]).returncode == 0
|
||||
|
||||
|
||||
def status():
|
||||
running = active()
|
||||
try:
|
||||
state = json.loads(STATUS.read_text())
|
||||
except (OSError, ValueError):
|
||||
state = {'state': 'idle'}
|
||||
if not running and state.get('state') in ('playing', 'starting', 'paused'):
|
||||
state = {'state': 'stopped', 'message': 'Player exited; check the media log if this was unexpected'}
|
||||
return dict(state, running=running)
|
||||
|
||||
|
||||
def run(body):
|
||||
action = body.get('action')
|
||||
if action == 'list':
|
||||
files = []
|
||||
if ROOT.exists():
|
||||
for folder in sorted(ROOT.iterdir()):
|
||||
if not re.fullmatch('[0-9a-f]{32}', folder.name) or not folder.is_dir() or folder.is_symlink():
|
||||
continue
|
||||
for path in sorted(folder.iterdir()):
|
||||
if path.is_file() and not path.is_symlink() and not path.name.startswith('.'):
|
||||
files.append({'id': folder.name+'/'+path.name, 'name': path.name, 'bytes': path.stat().st_size})
|
||||
return {'files': files, 'player': status()}
|
||||
if action == 'status':
|
||||
return status()
|
||||
if action == 'stop':
|
||||
# --collect unloads the unit after it exits; systemctl then exits 5
|
||||
# ("not loaded", verified on the Frame). That's a finished player, not an error.
|
||||
stopped = subprocess.run(['systemctl', '--user', 'stop', UNIT], capture_output=True, text=True, timeout=15)
|
||||
if stopped.returncode not in (0, 5):
|
||||
raise RuntimeError('Could not stop the media player: ' + (stopped.stderr.strip() or 'exit %s' % stopped.returncode))
|
||||
return {'message': 'Media player stopped', **status()}
|
||||
if action != 'play':
|
||||
raise ValueError('Media action must be list, status, play or stop')
|
||||
path = media_path(body.get('id'))
|
||||
if type(body.get('theatre', False)) is not bool:
|
||||
raise ValueError('theatre must be true or false')
|
||||
info = {} if path.suffix.lower() == '.splat' else probe(path)[0]
|
||||
plan = frame_media.plan(path.name, body.get('layout', 'auto'), info.get('tags'))
|
||||
if active():
|
||||
raise ValueError('Stop the current media before starting another file')
|
||||
# systemd owns the process group and refuses a concurrent start of this name.
|
||||
# The runtime cap also cleans up if the controlling computer disconnects.
|
||||
subprocess.run(['systemctl', '--user', 'reset-failed', UNIT], stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL, timeout=10)
|
||||
STATUS.write_text(json.dumps({'state': 'starting', 'file': path.name}))
|
||||
command = ['systemd-run', '--user', '--quiet', '--collect', '--unit='+UNIT,
|
||||
'--property=RuntimeMaxSec=14400', '--property=TimeoutStopSec=8',
|
||||
'--property=StandardOutput=append:'+str(RUNTIME/'player.log'),
|
||||
'--property=StandardError=append:'+str(RUNTIME/'player.log'),
|
||||
'python3', str(RUNTIME/'frame_media_player.py'), str(path),
|
||||
'--layout', plan['layout'], '--status', str(STATUS)]
|
||||
if body.get('theatre'):
|
||||
command.append('--theatre')
|
||||
started = subprocess.run(command, capture_output=True, text=True, timeout=15)
|
||||
if started.returncode:
|
||||
raise RuntimeError('Could not start the media player: ' + (started.stderr.strip() or 'systemd-run exited %s' % started.returncode))
|
||||
return {'message': 'Starting Frame Control media', 'plan': plan}
|
||||
|
||||
|
||||
def main():
|
||||
try:
|
||||
print(json.dumps(run(json.load(sys.stdin))))
|
||||
except Exception as e:
|
||||
print(json.dumps({'error': str(e)}))
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,83 @@
|
||||
"""Small, bounded CPU Gaussian-splat preview renderer (Frame Control-owned).
|
||||
|
||||
Reads the common 32-byte .splat record: position/scale float32 triplets,
|
||||
RGBA bytes, then normalized quaternion bytes (wxyz). Two perspective cameras,
|
||||
projected 3D covariance, back-to-front alpha compositing. This is a stationary
|
||||
stereo preview, not a six-degree-of-freedom scene or a large-scene renderer.
|
||||
"""
|
||||
import math
|
||||
from pathlib import Path
|
||||
import struct
|
||||
|
||||
MAX_SPLATS = 20000
|
||||
RECORD = struct.Struct('<6f8B')
|
||||
|
||||
|
||||
def read(path):
|
||||
size = Path(path).stat().st_size
|
||||
if not size or size % RECORD.size or size > MAX_SPLATS * RECORD.size:
|
||||
raise ValueError('Use a 32-byte .splat file with 1–20,000 Gaussians; PLY/SPZ and larger scenes are not supported yet')
|
||||
values = []
|
||||
with open(path, 'rb') as stream:
|
||||
for row in RECORD.iter_unpack(stream.read(MAX_SPLATS * RECORD.size + 1)):
|
||||
xyz, scales = row[:3], row[3:6]
|
||||
if not all(math.isfinite(v) and abs(v) <= 1e6 for v in row[:6]) or min(scales) <= 0:
|
||||
raise ValueError('Invalid splat position or scale')
|
||||
q = [(v - 128) / 128 for v in row[10:14]]
|
||||
length = math.sqrt(sum(v*v for v in q))
|
||||
if length < .01:
|
||||
raise ValueError('Invalid splat quaternion')
|
||||
w, x, y, z = [v / length for v in q]
|
||||
rotation = ((1-2*(y*y+z*z), 2*(x*y-z*w), 2*(x*z+y*w)),
|
||||
(2*(x*y+z*w), 1-2*(x*x+z*z), 2*(y*z-x*w)),
|
||||
(2*(x*z-y*w), 2*(y*z+x*w), 1-2*(x*x+y*y)))
|
||||
cov = [[sum(rotation[i][k]*rotation[j][k]*scales[k]**2 for k in range(3))
|
||||
for j in range(3)] for i in range(3)]
|
||||
values.append((xyz, cov, row[6:10]))
|
||||
return values
|
||||
|
||||
|
||||
def render(path, width=320, height=240):
|
||||
values = read(path)
|
||||
lo = [min(p[0][i] for p in values) for i in range(3)]
|
||||
hi = [max(p[0][i] for p in values) for i in range(3)]
|
||||
center = [(a+b)/2 for a, b in zip(lo, hi)]
|
||||
radius = max(max(b-a for a, b in zip(lo, hi))/2, .01)
|
||||
# Normalize captures to a two-metre box. Source units are not assumed metres.
|
||||
normalized = [([(xyz[i]-center[i])/radius for i in range(3)],
|
||||
[[v/radius**2 for v in row] for row in cov], color)
|
||||
for xyz, cov, color in values]
|
||||
normalized.sort(key=lambda p: p[0][2]) # camera is at z=3; farthest first
|
||||
focal = width * .8
|
||||
eyes = []
|
||||
for eye in (-.032, .032):
|
||||
pixels = bytearray(b'\x00\x00\x00\xff' * (width*height))
|
||||
for (x, y, z), cov, color in normalized:
|
||||
x -= eye
|
||||
depth = 3-z
|
||||
px, py = width/2+focal*x/depth, height/2-focal*y/depth
|
||||
jac = ((focal/depth, 0, focal*x/depth**2),
|
||||
(0, -focal/depth, -focal*y/depth**2))
|
||||
screen = [[sum(jac[i][a]*cov[a][b]*jac[j][b] for a in range(3) for b in range(3))
|
||||
for j in range(2)] for i in range(2)]
|
||||
a, b, c = screen[0][0]+.3, screen[0][1], screen[1][1]+.3
|
||||
det = a*c-b*b
|
||||
if det <= 0 or not math.isfinite(det):
|
||||
raise ValueError('Splat covariance is not renderable')
|
||||
# A footprint cap bounds work on malformed or oversized Gaussians.
|
||||
rx, ry = min(32, math.ceil(3*math.sqrt(a))), min(32, math.ceil(3*math.sqrt(c)))
|
||||
for sy in range(max(0, int(py)-ry), min(height, int(py)+ry+1)):
|
||||
dy = sy+.5-py
|
||||
for sx in range(max(0, int(px)-rx), min(width, int(px)+rx+1)):
|
||||
dx = sx+.5-px
|
||||
power = (c*dx*dx-2*b*dx*dy+a*dy*dy)/det
|
||||
if power > 9:
|
||||
continue
|
||||
alpha = color[3]/255 * math.exp(-.5*power)
|
||||
offset = (sy*width+sx)*4
|
||||
for k in range(3):
|
||||
pixels[offset+k] = round(color[k]*alpha+pixels[offset+k]*(1-alpha))
|
||||
eyes.append(pixels)
|
||||
stride = width*4
|
||||
return b''.join(eyes[0][y*stride:(y+1)*stride]+eyes[1][y*stride:(y+1)*stride]
|
||||
for y in range(height)), width*2, height
|
||||
+353
-3
@@ -139,6 +139,7 @@
|
||||
.found { margin-top: 10px; }
|
||||
|
||||
/* ---- drop anywhere ---- */
|
||||
#media select { min-width: 0; max-width: 100%; flex: 1; }
|
||||
.dropzone { position: fixed; inset: 0; z-index: 40; display: grid; place-items: center; pointer-events: none;
|
||||
background: rgba(14,20,27,.82); backdrop-filter: blur(3px); }
|
||||
.dropzone > div { padding: 42px 60px; border: 2px dashed var(--blue); border-radius: 8px; text-align: center;
|
||||
@@ -317,9 +318,9 @@
|
||||
.and-grid { display: grid; grid-template-columns: minmax(0, 1fr) minmax(0, 2fr); gap: 22px; align-items: start; }
|
||||
.and-col { display: grid; gap: 22px; align-content: start; }
|
||||
.rep-item .s { white-space: normal; }
|
||||
#bugDlg, #repDlg, #titleDlg, #wiDlg, #pwDlg, #apkAltDlg { background: #1e2329; color: var(--text); border: 1px solid rgba(255,255,255,.1); border-radius: 4px;
|
||||
#bugDlg, #repDlg, #titleDlg, #wiDlg, #pwDlg, #apkAltDlg, #aboutDlg { background: #1e2329; color: var(--text); border: 1px solid rgba(255,255,255,.1); border-radius: 4px;
|
||||
padding: 22px; width: min(560px, 92vw); box-shadow: 0 20px 60px rgba(0,0,0,.6); }
|
||||
#bugDlg::backdrop, #repDlg::backdrop, #titleDlg::backdrop, #wiDlg::backdrop, #pwDlg::backdrop, #apkAltDlg::backdrop { background: rgba(0,0,0,.55); }
|
||||
#bugDlg::backdrop, #repDlg::backdrop, #titleDlg::backdrop, #wiDlg::backdrop, #pwDlg::backdrop, #apkAltDlg::backdrop, #aboutDlg::backdrop { background: rgba(0,0,0,.55); }
|
||||
#bugDlg { width: min(640px, calc(100vw - 40px)); }
|
||||
#bugForm label.field { display: block; font-size: 12.5px; color: var(--muted); margin-top: 10px; }
|
||||
#bugForm label.field input, #bugForm label.field textarea, #bugForm select { margin-top: 5px; }
|
||||
@@ -328,7 +329,10 @@
|
||||
#bugForm .popt { margin: 14px 0 0; }
|
||||
#bugForm .sentlog { max-height: 200px; }
|
||||
#bugWarn { color: var(--muted); font-size: 12.5px; line-height: 1.45; margin: 12px 0 0; }
|
||||
#bugDlg h2, #repDlg h2, #titleDlg h2, #wiDlg h2, #pwDlg h2, #apkAltDlg h2 { margin: 0 0 14px; font-size: 15px; letter-spacing: 1.5px; text-transform: uppercase; color: var(--bright); }
|
||||
#bugDlg h2, #repDlg h2, #titleDlg h2, #wiDlg h2, #pwDlg h2, #apkAltDlg h2, #aboutDlg h2 { margin: 0 0 14px; font-size: 15px; letter-spacing: 1.5px; text-transform: uppercase; color: var(--bright); }
|
||||
.about-text { max-height: 55vh; overflow: auto; }
|
||||
.about-text pre { white-space: pre-wrap; font-size: 12px; color: var(--muted); }
|
||||
.about-text summary { cursor: pointer; margin: 8px 0; }
|
||||
#repForm label, #titleForm label { display: block; font-size: 12.5px; color: var(--muted); margin-top: 10px; }
|
||||
#repForm label input[type=text], #repForm textarea, #titleForm label input, #titleForm label select { margin-top: 5px; }
|
||||
#titleForm select { width: 100%; background: rgba(0,0,0,.28); color: var(--text); border: 1px solid transparent;
|
||||
@@ -400,6 +404,19 @@
|
||||
border-radius: 3px; height: 32px; padding: 0 8px; font: inherit; font-size: 13px; }
|
||||
/* Touch screens can't hover: keep the library's name and Play button showing. */
|
||||
@media (hover: none) { .capsule .over { opacity: 1; } .capsule:hover { transform: none; } }
|
||||
/* ---- keyboard and trackpad ---- */
|
||||
.pad-area { position: relative; height: 190px; margin: 12px 0; border-radius: 4px; background: rgba(0,0,0,.28);
|
||||
border: 1px dashed rgba(255,255,255,.14); display: grid; place-items: center; cursor: pointer;
|
||||
touch-action: none; user-select: none; -webkit-user-select: none; -webkit-touch-callout: none; outline: none; }
|
||||
.pad-area:focus-visible, .pad-area.captured { border: 1px solid var(--blue); box-shadow: 0 0 0 1px var(--blue) inset; }
|
||||
.pad-area.off { cursor: default; opacity: .55; }
|
||||
.pad-hint { max-width: 420px; padding: 0 16px; text-align: center; color: var(--muted); font-size: 13px; pointer-events: none; }
|
||||
.pad-keys { display: flex; flex-wrap: wrap; gap: 8px; align-items: center; }
|
||||
.pad-keys input { flex: 1 1 220px; min-width: 0; height: 32px; padding: 0 10px; border-radius: 3px; border: 1px solid transparent;
|
||||
background: rgba(0,0,0,.28); color: var(--text); font: inherit; }
|
||||
.pad-keys input:focus { outline: none; border-color: var(--blue); }
|
||||
#pad .hint { margin-top: 10px; }
|
||||
|
||||
/* ---- phones, upright or on their side: tabs move to a bottom bar, as in iOS apps. Every edge keeps clear of
|
||||
the safe area (notch or Dynamic Island, rounded corners, home indicator); env() is zero on desktops. ---- */
|
||||
@media (max-width: 640px), (max-height: 500px) and (hover: none) {
|
||||
@@ -436,6 +453,10 @@
|
||||
button.small { height: 32px; }
|
||||
.actions button { height: 46px; }
|
||||
.cat-tools select { max-width: none; flex: 1 1 100%; }
|
||||
.pad-area { height: 240px; }
|
||||
.pad-keys .seg { flex: 1 1 100%; display: grid; grid-template-columns: repeat(4, 1fr); }
|
||||
.pad-keys .pad-clicks { grid-template-columns: repeat(2, 1fr); }
|
||||
.pad-keys .seg button { padding: 0 4px; justify-content: center; text-align: center; }
|
||||
/* The on-screen keyboard covers the bottom of the screen; the tab bar would ride on top of it. */
|
||||
body.typing nav { display: none; }
|
||||
}
|
||||
@@ -571,6 +592,37 @@
|
||||
</section>
|
||||
</div>
|
||||
|
||||
<section class="panel" id="pad">
|
||||
<div class="shelf-head"><h2>Keyboard and trackpad</h2><span class="sub" id="padState"></span><span class="spacer"></span>
|
||||
<button class="small" id="padOn">Turn on</button>
|
||||
</div>
|
||||
<div class="pad-area" id="padArea" tabindex="0" role="application" aria-label="Trackpad for the Frame">
|
||||
<div class="pad-hint" id="padHint"></div>
|
||||
</div>
|
||||
<div class="pad-keys" aria-label="Keys and clicks">
|
||||
<input id="padType" type="text" placeholder="Type here to type on the Frame" autocomplete="off" autocapitalize="off"
|
||||
autocorrect="off" spellcheck="false" enterkeyhint="send">
|
||||
<span class="seg pad-clicks">
|
||||
<button data-pad-click="singleclick" title="Left click">Click</button>
|
||||
<button data-pad-click="rightclick" title="Right click">Right-click</button>
|
||||
</span>
|
||||
<span class="seg pad-special">
|
||||
<button data-pad-key="14" title="Escape">Esc</button>
|
||||
<button data-pad-key="2" title="Tab">Tab</button>
|
||||
<button data-pad-key="4" title="Left arrow" aria-label="Left arrow">←</button>
|
||||
<button data-pad-key="5" title="Up arrow" aria-label="Up arrow">↑</button>
|
||||
<button data-pad-key="7" title="Down arrow" aria-label="Down arrow">↓</button>
|
||||
<button data-pad-key="6" title="Right arrow" aria-label="Right arrow">→</button>
|
||||
<button data-pad-key="1" title="Backspace" aria-label="Backspace">⌫</button>
|
||||
<button data-pad-key="12" title="Enter">Enter</button>
|
||||
</span>
|
||||
</div>
|
||||
<div class="hint">Types and points in apps on the Frame, such as Chromium or the desktop's Linux apps; Steam's own VR menus
|
||||
don't take it. It works through KDE Connect, which comes with Frame Control: the first time, it copies Valve's build
|
||||
for the Frame (3.6 MB, 18 MB unpacked) into your home folder there and pairs with it. No internet needed, and nothing
|
||||
else to install. <a href="#" data-about>Licences</a></div>
|
||||
</section>
|
||||
|
||||
<section id="shots">
|
||||
<div class="shelf-head"><h2>Screenshots</h2><span class="count" id="shotCount"></span><span class="spacer"></span>
|
||||
<button class="small" id="shotsRefresh">Refresh</button>
|
||||
@@ -718,6 +770,31 @@
|
||||
<div class="hint">Clipboard needs the desktop panel open in the headset.</div>
|
||||
</section>
|
||||
|
||||
<section class="panel" id="media">
|
||||
<div class="shelf-head"><h2>Media in the headset</h2><span class="count">Preview</span></div>
|
||||
<p class="sub">Play your movies and stereo photos with Frame Control's own player.</p>
|
||||
<div class="row"><button class="small" id="mediaChoose">Send media…</button>
|
||||
<button class="small" id="mediaRefresh">Refresh</button></div>
|
||||
<input type="file" id="mediaInput" accept=".mp4,.mkv,.mov,.m4v,.webm,.png,.jpg,.jpeg,.splat" hidden>
|
||||
<div class="row" style="margin-top:12px"><label for="mediaFile">On the Frame</label>
|
||||
<select id="mediaFile"><option value="">Refresh to load your media</option></select></div>
|
||||
<div class="row" style="margin-top:8px"><label for="mediaLayout">Layout</label>
|
||||
<select id="mediaLayout">
|
||||
<option value="auto">Detect from name or metadata</option><option value="mono">2D</option>
|
||||
<option value="sbs">Half SBS · left / right</option><option value="ou">Half OU · top / bottom</option>
|
||||
<option value="full-sbs">Full SBS · left / right</option><option value="full-ou">Full OU · top / bottom</option>
|
||||
</select></div>
|
||||
<div class="row" style="margin-top:8px"><label><input type="checkbox" id="mediaTheatre" checked>
|
||||
Theatre · bigger screen, dark surround</label></div>
|
||||
<div class="row" style="margin-top:12px"><button class="action small" id="mediaPlay">Play</button>
|
||||
<button class="small" id="mediaStop">Stop</button></div>
|
||||
<p class="sub" id="mediaState" role="status" aria-live="polite">Choose a file to begin.</p>
|
||||
<div class="hint">H.264/H.265 video; SBS/OU PNG or JPEG photos. Screens follow your head.
|
||||
Small .splat files show a stationary stereo preview (up to 20,000 Gaussians).
|
||||
Native HEIC/MPO photos, large splat scenes, seeking and subtitles aren't supported yet.
|
||||
Stop works from here; playback also ends after four hours.</div>
|
||||
</section>
|
||||
|
||||
<section class="panel" id="apps">
|
||||
<div class="shelf-head"><h2>Linux apps</h2><span class="count">Flatpak</span></div>
|
||||
<div class="list" id="flatpaks"><div class="sub">Loading…</div></div>
|
||||
@@ -743,6 +820,7 @@
|
||||
<div class="hint desk-only">Sleep, Restart and Shut down open a terminal window for the Developer Mode password.</div>
|
||||
<div class="hint mobile-only">Sleep, Restart and Shut down ask for the Developer Mode password. SSH, SFTP, Steam Link and remote desktop open in the app that handles them.</div>
|
||||
<div class="row mobile-only" style="margin-top:14px"><button class="small" id="changeHeadset">Change headset…</button></div>
|
||||
<div class="row" style="margin-top:14px"><button class="small" data-about>About and licences</button></div>
|
||||
<div class="links desk-only">
|
||||
<div><a href="https://store.steampowered.com/remoteplay" target="_blank">Steam Link</a>: Valve's remote view of the headset</div>
|
||||
<div><a href="https://streamframe.app/" target="_blank">Stream Frame</a>: third-party recorder (macOS 14+)</div>
|
||||
@@ -831,6 +909,14 @@
|
||||
<div class="actions"><button id="apkAltClose">Close</button></div>
|
||||
</dialog>
|
||||
|
||||
<dialog id="aboutDlg" aria-labelledby="aboutTitle">
|
||||
<h2 id="aboutTitle">About and licences</h2>
|
||||
<p>Frame Control is MIT-licensed. It ships other people's software, each under its own licence: the notices and
|
||||
licence texts follow, with where to get the source.</p>
|
||||
<div id="aboutText" class="about-text">Loading…</div>
|
||||
<div class="actions"><button id="aboutClose">Close</button></div>
|
||||
</dialog>
|
||||
|
||||
<dialog id="repDlg" aria-labelledby="repTitle">
|
||||
<form method="dialog" id="repForm">
|
||||
<h2 id="repTitle">Report an APK</h2>
|
||||
@@ -1681,6 +1767,42 @@ function upload(file, mode, dev = window.connDevice) {
|
||||
xhr.send(file);
|
||||
});
|
||||
}
|
||||
async function refreshMedia(selectId) {
|
||||
const data = await api("/api/media", { action: "list" });
|
||||
const selected = selectId || $("mediaFile").value;
|
||||
$("mediaFile").replaceChildren();
|
||||
if (!data.files.length) $("mediaFile").add(new Option("Send media to begin", ""));
|
||||
for (const file of data.files) $("mediaFile").add(new Option(file.name, file.id));
|
||||
if (data.files.some(f => f.id === selected)) $("mediaFile").value = selected;
|
||||
const p = data.player;
|
||||
$("mediaState").textContent = p.error || `${p.state}${p.file ? ": " + p.file : ""}`;
|
||||
return data;
|
||||
}
|
||||
$("mediaChoose").onclick = () => $("mediaInput").click();
|
||||
$("mediaInput").onchange = () => act("Send media", async () => {
|
||||
const file = $("mediaInput").files[0];
|
||||
if (!file) return;
|
||||
const result = await upload(file, "media");
|
||||
await refreshMedia(result.id);
|
||||
$("mediaInput").value = "";
|
||||
return result;
|
||||
}, $("mediaChoose"));
|
||||
$("mediaRefresh").onclick = () => act("Refresh media", () => refreshMedia(), $("mediaRefresh"));
|
||||
$("mediaPlay").onclick = () => act("Play media", async () => {
|
||||
const id = $("mediaFile").value;
|
||||
if (!id) throw new Error("Send or select a media file first");
|
||||
const result = await api("/api/media", { action: "play", id,
|
||||
layout: $("mediaLayout").value, theatre: $("mediaTheatre").checked });
|
||||
$("mediaState").textContent = "Starting…";
|
||||
setTimeout(() => refreshMedia().catch(e => { $("mediaState").textContent = e.message; }), 1500);
|
||||
return result;
|
||||
}, $("mediaPlay"));
|
||||
$("mediaStop").onclick = () => act("Stop media", async () => {
|
||||
const result = await api("/api/media", { action: "stop" });
|
||||
await refreshMedia();
|
||||
return result;
|
||||
}, $("mediaStop"));
|
||||
|
||||
let apkLookup = 0;
|
||||
async function checkApkAlternatives(apk, dev = window.connDevice) {
|
||||
const lookup = ++apkLookup;
|
||||
@@ -1726,6 +1848,21 @@ function showApkAlternatives(reason, result, dev = window.connDevice) {
|
||||
}
|
||||
$("apkAltClose").onclick = () => $("apkAltDlg").close();
|
||||
|
||||
// ---- about and licences ----
|
||||
document.addEventListener("click", async e => {
|
||||
if (!e.target.closest("[data-about]")) return;
|
||||
e.preventDefault();
|
||||
$("aboutDlg").showModal();
|
||||
try {
|
||||
const { notices } = await api("/api/licenses");
|
||||
$("aboutText").innerHTML = notices.map((n, i) =>
|
||||
`<details${i ? "" : " open"}><summary>${esc(n.title)}</summary><pre>${esc(n.text)}</pre></details>`).join("");
|
||||
} catch (err) {
|
||||
$("aboutText").textContent = `Couldn't load the licences: ${err.message}. They're also in THIRD_PARTY_NOTICES.md and frame/kdeconnect in Frame Control's source.`;
|
||||
}
|
||||
});
|
||||
$("aboutClose").onclick = () => $("aboutDlg").close();
|
||||
|
||||
const TITLE_EXT = /\.(zip|exe)$/i;
|
||||
async function sendFiles(files, dirs = new Set()) {
|
||||
const dev = window.connDevice; // the whole batch is for the headset it was dropped on
|
||||
@@ -2532,6 +2669,219 @@ window.addEventListener("hashchange", showPage);
|
||||
const typesText = el => el?.matches?.('textarea, input:not([type=range], [type=checkbox], [type=radio], [type=file], [type=button])');
|
||||
document.addEventListener("focusin", e => document.body.classList.toggle("typing", !!typesText(e.target)));
|
||||
document.addEventListener("focusout", () => document.body.classList.remove("typing"));
|
||||
|
||||
// ---- keyboard and trackpad: events go to KDE Connect on the Frame (see ui/frame_input_agent.py) ----
|
||||
const pad = { state: "off", queue: [], sending: false, poll: null, captured: false };
|
||||
const PAD_STATES = {
|
||||
off: "", starting: "Connecting…", installing: "Setting up KDE Connect on the Frame (first time only)…",
|
||||
pairing: "Pairing with KDE Connect…", ready: "On",
|
||||
};
|
||||
// KDE Connect's specialKey numbers (plugins/mousepad in its source).
|
||||
const PAD_SPECIAL = { Backspace: 1, Tab: 2, ArrowLeft: 4, ArrowUp: 5, ArrowRight: 6, ArrowDown: 7, PageUp: 8,
|
||||
PageDown: 9, Home: 10, End: 11, Enter: 12, Delete: 13, Escape: 14,
|
||||
...Object.fromEntries(Array.from({ length: 12 }, (_, i) => [`F${i + 1}`, 21 + i])) };
|
||||
const IS_MAC = /^Mac/.test(navigator.platform);
|
||||
const TOUCH = matchMedia("(hover: none)").matches;
|
||||
|
||||
function padShow(status) {
|
||||
pad.state = status.state || "off";
|
||||
const ready = pad.state === "ready";
|
||||
const busy = !ready && pad.state !== "off" && pad.state !== "error";
|
||||
$("padState").textContent = pad.state === "error" ? "" : PAD_STATES[pad.state] || "";
|
||||
$("padOn").hidden = ready || busy;
|
||||
$("padOn").textContent = pad.state === "error" ? "Try again" : "Turn on";
|
||||
$("padArea").classList.toggle("off", !ready);
|
||||
$("padHint").textContent = pad.state === "error" ? status.message || "Couldn't reach KDE Connect on the Frame."
|
||||
: !ready ? (busy ? PAD_STATES[pad.state] : "Turn on to use this as a trackpad for the Frame.")
|
||||
: TOUCH ? "Drag to move the pointer · tap to click · two fingers to scroll · two-finger tap to right-click"
|
||||
: pad.captured ? "Your mouse and keyboard now control the Frame. Press Esc to stop."
|
||||
: "Click here to use your mouse and keyboard on the Frame (Esc to stop), or drag like a trackpad.";
|
||||
clearTimeout(pad.poll);
|
||||
if (ready && pad.queue.length) padFlush();
|
||||
if (busy) pad.poll = setTimeout(async () => padShow(await api("/api/input").catch(e => ({ state: "error", message: e.message }))), 1000);
|
||||
}
|
||||
async function padStart() {
|
||||
localStorage.padUsed = "1"; // from now on it turns on with the page
|
||||
padShow({ state: "starting" });
|
||||
padShow(await api("/api/input?start=1").catch(e => ({ state: "error", message: e.message })));
|
||||
}
|
||||
$("padOn").onclick = padStart;
|
||||
|
||||
// Queue events and send them one request at a time, merging pointer moves made meanwhile.
|
||||
const isMove = e => Object.keys(e).every(k => k === "dx" || k === "dy");
|
||||
function padSend(event) {
|
||||
// While it reconnects, keys and clicks wait in the queue; pointer moves would be stale.
|
||||
// A button release is always kept, so a drag never leaves the button held on the Frame.
|
||||
const room = pad.queue.length < 200 || event.singlerelease;
|
||||
const reconnecting = ["starting", "pairing", "installing"].includes(pad.state) && room;
|
||||
// Even after an error: a button pressed on the Frame must be let go once it's back.
|
||||
const release = event.singlerelease && pad.state !== "off";
|
||||
if (pad.state !== "ready" && !(reconnecting && !isMove(event)) && !release) return;
|
||||
const last = pad.queue[pad.queue.length - 1];
|
||||
if (last && isMove(last) && isMove(event)) { last.dx = (last.dx || 0) + (event.dx || 0); last.dy = (last.dy || 0) + (event.dy || 0); }
|
||||
else pad.queue.push(event);
|
||||
padFlush();
|
||||
}
|
||||
// Events that didn't reach the Frame (the link had dropped, and is reconnecting): keep
|
||||
// the keys and clicks for when it's back, but not pointer moves, which would be stale.
|
||||
// Button releases are kept whatever the queue's length.
|
||||
function padKeep(events) {
|
||||
pad.queue = [...events, ...pad.queue].filter(e => !isMove(e)).filter((e, i) => i < 200 || e.singlerelease);
|
||||
}
|
||||
async function padFlush() {
|
||||
if (pad.sending || !pad.queue.length || pad.state !== "ready") return;
|
||||
pad.sending = true;
|
||||
const events = pad.queue.splice(0, 200);
|
||||
try {
|
||||
const status = await api("/api/input", { events });
|
||||
if (!status.sent) padKeep(events);
|
||||
if (status.state !== "ready") padShow(status);
|
||||
} catch (e) {
|
||||
padKeep(events);
|
||||
padShow({ state: "error", message: e.message });
|
||||
} finally {
|
||||
pad.sending = false;
|
||||
if (pad.queue.length) padFlush();
|
||||
}
|
||||
}
|
||||
function padKey(e) {
|
||||
const mods = { ...(e.ctrlKey || (IS_MAC && e.metaKey) ? { ctrl: true } : {}), ...(e.altKey ? { alt: true } : {}),
|
||||
...(!IS_MAC && e.metaKey ? { super: true } : {}) };
|
||||
if (PAD_SPECIAL[e.key]) return { specialKey: PAD_SPECIAL[e.key], ...mods, ...(e.shiftKey ? { shift: true } : {}) };
|
||||
if (e.key.length === 1) return { key: e.key, ...mods }; // already shifted, so no shift flag
|
||||
return null;
|
||||
}
|
||||
|
||||
// Touch: one finger moves, a quick tap clicks; two fingers scroll, or right-click when tapped.
|
||||
const PAD_SPEED = 1.6, PAD_SCROLL_STEP = 18;
|
||||
const touches = new Map();
|
||||
let gesture = null;
|
||||
$("padArea").addEventListener("pointerdown", e => {
|
||||
if (pad.state === "off" || pad.state === "error") { padStart(); return; }
|
||||
if (pad.state !== "ready") return;
|
||||
if (e.pointerType === "mouse") {
|
||||
if (pad.captured) return; // mousedown and mouseup below send the clicks
|
||||
// Lock the pointer to the pad and pass the whole mouse and keyboard through; where
|
||||
// that isn't allowed, the mouse drags like a finger on a trackpad instead.
|
||||
if (e.button === 0 && !pad.noLock && $("padArea").requestPointerLock) { $("padArea").requestPointerLock(); return; }
|
||||
// Only the left button drags and taps like a finger; the others are clicks of their own.
|
||||
if (e.button === 1 || e.button === 2) { padSend(e.button === 2 ? { rightclick: true } : { middleclick: true }); return; }
|
||||
if (e.button !== 0) return;
|
||||
}
|
||||
try { $("padArea").setPointerCapture(e.pointerId); } catch {} // keeps the drag if the finger leaves the pad
|
||||
touches.set(e.pointerId, { x: e.clientX, y: e.clientY });
|
||||
if (touches.size === 1) gesture = { start: performance.now(), moved: 0, fingers: 1, scroll: 0 };
|
||||
else if (gesture) gesture.fingers = Math.max(gesture.fingers, touches.size);
|
||||
e.preventDefault();
|
||||
});
|
||||
$("padArea").addEventListener("pointermove", e => {
|
||||
if (pad.captured) { padSend({ dx: e.movementX, dy: e.movementY }); return; }
|
||||
const t = touches.get(e.pointerId);
|
||||
if (!t || !gesture) return;
|
||||
const dx = e.clientX - t.x, dy = e.clientY - t.y;
|
||||
t.x = e.clientX; t.y = e.clientY;
|
||||
gesture.moved += Math.abs(dx) + Math.abs(dy);
|
||||
if (touches.size >= 2) {
|
||||
// Scroll with the lead finger only, a notch per step, like a laptop trackpad (content follows the fingers).
|
||||
if (e.pointerId !== touches.keys().next().value) return;
|
||||
gesture.scroll += dy;
|
||||
while (Math.abs(gesture.scroll) >= PAD_SCROLL_STEP) {
|
||||
const up = gesture.scroll > 0;
|
||||
padSend({ scroll: true, dy: up ? 1 : -1 });
|
||||
gesture.scroll -= up ? PAD_SCROLL_STEP : -PAD_SCROLL_STEP;
|
||||
}
|
||||
} else if (gesture.fingers === 1) {
|
||||
padSend({ dx: dx * PAD_SPEED, dy: dy * PAD_SPEED });
|
||||
}
|
||||
});
|
||||
function padLift(e) {
|
||||
if (!touches.delete(e.pointerId) || touches.size || !gesture) return;
|
||||
const quick = performance.now() - gesture.start < 250 && gesture.moved < 10;
|
||||
if (quick) padSend(gesture.fingers >= 2 ? { rightclick: true } : { singleclick: true });
|
||||
gesture = null;
|
||||
}
|
||||
$("padArea").addEventListener("pointerup", padLift);
|
||||
$("padArea").addEventListener("pointercancel", e => { touches.delete(e.pointerId); if (!touches.size) gesture = null; });
|
||||
|
||||
// A computer's mouse and keyboard: captured while the pointer is locked to the pad.
|
||||
document.addEventListener("pointerlockerror", () => { pad.noLock = true; });
|
||||
document.addEventListener("pointerlockchange", () => {
|
||||
pad.captured = document.pointerLockElement === $("padArea");
|
||||
window.frameApp?.captureKeys?.(pad.captured); // the desktop app's menu shortcuts go to the Frame too
|
||||
// Esc or switching windows mid-drag: let go of the button on the Frame as well.
|
||||
if (!pad.captured && pad.held) { padSend({ singlerelease: true }); pad.held = false; }
|
||||
$("padArea").classList.toggle("captured", pad.captured);
|
||||
padShow({ state: pad.state });
|
||||
});
|
||||
$("padArea").addEventListener("mousedown", e => {
|
||||
if (!pad.captured) return;
|
||||
if (e.button === 0) { padSend({ singlehold: true }); pad.held = true; }
|
||||
else if (e.button === 1) padSend({ middleclick: true });
|
||||
else if (e.button === 2) padSend({ rightclick: true });
|
||||
});
|
||||
$("padArea").addEventListener("mouseup", e => { if (pad.held && e.button === 0) { padSend({ singlerelease: true }); pad.held = false; } });
|
||||
$("padArea").addEventListener("contextmenu", e => e.preventDefault());
|
||||
$("padArea").addEventListener("wheel", e => {
|
||||
if (pad.state !== "ready") return;
|
||||
e.preventDefault();
|
||||
if (e.deltaY) padSend({ scroll: true, dy: e.deltaY > 0 ? -1 : 1 });
|
||||
}, { passive: false });
|
||||
// Capture phase, so the page's own shortcuts (tabs 1-4, R, zoom) don't also fire.
|
||||
window.addEventListener("keydown", e => {
|
||||
if (!pad.captured) return;
|
||||
e.preventDefault(); e.stopImmediatePropagation();
|
||||
const event = padKey(e);
|
||||
if (event) padSend(event);
|
||||
}, true);
|
||||
|
||||
// The text field: each keystroke goes to the Frame. While focused it holds one space,
|
||||
// so Backspace always has something to delete; empty otherwise, to show its prompt.
|
||||
const padType = $("padType");
|
||||
padType.addEventListener("beforeinput", e => {
|
||||
if (e.inputType === "insertCompositionText") return; // IME composing; sent on compositionend
|
||||
e.preventDefault();
|
||||
if (pad.state === "off" || pad.state === "error") padStart(); // what's typed meanwhile waits in the queue
|
||||
const text = e.data ?? e.dataTransfer?.getData("text/plain") ?? "";
|
||||
if (/^insert(Text|ReplacementText|FromPaste|FromDrop)$/.test(e.inputType) && text) padText(text);
|
||||
else if (e.inputType === "deleteContentBackward") padSend({ specialKey: 1 });
|
||||
else if (e.inputType === "deleteWordBackward") padSend({ specialKey: 1, ctrl: true }); // Ctrl+Backspace: a word, on Linux
|
||||
else if (e.inputType === "deleteContentForward") padSend({ specialKey: 13 });
|
||||
else if (e.inputType === "deleteWordForward") padSend({ specialKey: 13, ctrl: true });
|
||||
else if (e.inputType === "insertLineBreak" || e.inputType === "insertParagraph") padSend({ specialKey: 12 });
|
||||
});
|
||||
padType.addEventListener("compositionend", e => { if (e.data) padText(e.data); padType.value = " "; padCaret(); });
|
||||
// Text in pieces the server takes (500 characters), with Enter for each line break.
|
||||
function padText(text) {
|
||||
text.replace(/\r\n?/g, "\n").split("\n").forEach((line, i) => {
|
||||
if (i) padSend({ specialKey: 12 });
|
||||
const chars = Array.from(line); // whole characters, so an emoji isn't cut in two
|
||||
for (let at = 0; at < chars.length; at += 500) padSend({ key: chars.slice(at, at + 500).join("") });
|
||||
});
|
||||
}
|
||||
padType.addEventListener("keydown", e => {
|
||||
const special = PAD_SPECIAL[e.key];
|
||||
// Text and deletions arrive as beforeinput; this takes the keys that don't edit the field.
|
||||
if (!special || e.key === "Backspace" || e.key === "Delete" || e.isComposing) return;
|
||||
e.preventDefault();
|
||||
padSend(padKey(e));
|
||||
});
|
||||
// The caret stays after the space, so Backspace always has something to delete.
|
||||
const padCaret = () => requestAnimationFrame(() => padType.setSelectionRange(1, 1));
|
||||
padType.addEventListener("focus", () => { padType.value = " "; padCaret(); if (pad.state === "off") padStart(); });
|
||||
padType.addEventListener("blur", () => { padType.value = ""; });
|
||||
padType.addEventListener("input", e => {
|
||||
if (e.isComposing) return; // an input method is still composing; compositionend sends it
|
||||
if (padType.value !== " ") padType.value = " ";
|
||||
padCaret();
|
||||
});
|
||||
padType.addEventListener("click", padCaret);
|
||||
document.querySelector("#pad .pad-keys").addEventListener("click", e => {
|
||||
const b = e.target.closest("button");
|
||||
if (!b) return;
|
||||
if (b.dataset.padClick) padSend({ [b.dataset.padClick]: true });
|
||||
else if (b.dataset.padKey) padSend({ specialKey: +b.dataset.padKey });
|
||||
});
|
||||
api("/api/input").then(s => s.state === "off" && localStorage.padUsed ? padStart() : padShow(s), () => padShow({ state: "off" }));
|
||||
showPage();
|
||||
document.addEventListener("keydown", e => {
|
||||
if (e.metaKey || e.ctrlKey || e.altKey || /INPUT|TEXTAREA|SELECT/.test(document.activeElement.tagName) || document.querySelector("dialog[open]")) return;
|
||||
|
||||
+404
-2
@@ -11,10 +11,12 @@ Env: FRAME_ALIAS (default frame)
|
||||
FRAME_LOCAL=1 run on the Frame itself (the iPhone app starts it there over SSH)
|
||||
FRAME_UI_KEY required X-Frame-UI value (the iPhone app passes a fresh one)
|
||||
FRAME_DEVICE what to call the device the page runs on (e.g. iPhone)
|
||||
FRAME_CLIENT a stable id for that device (keyboard-and-trackpad pairing is kept per id)
|
||||
"""
|
||||
import argparse
|
||||
import base64
|
||||
import contextlib
|
||||
import hashlib
|
||||
import http.client
|
||||
import json
|
||||
import os
|
||||
@@ -48,6 +50,7 @@ import frame_devices # noqa: E402
|
||||
import frame_host # noqa: E402
|
||||
import frame_link # noqa: E402
|
||||
import frame_macview # noqa: E402
|
||||
import frame_media # noqa: E402
|
||||
import frame_report # noqa: E402
|
||||
import frame_store # noqa: E402
|
||||
import frame_telemetry # noqa: E402
|
||||
@@ -65,6 +68,8 @@ if LOCAL:
|
||||
os.environ["PATH"] = f"{HERE / 'local-bin'}{os.pathsep}{os.environ.get('PATH', '')}"
|
||||
UI_KEY = os.environ.get("FRAME_UI_KEY") or "1"
|
||||
DEVICE = os.environ.get("FRAME_DEVICE") or "phone"
|
||||
# What the Frame's KDE Connect calls this device (keyboard and trackpad).
|
||||
INPUT_NAME = DEVICE if LOCAL else socket.gethostname().split(".")[0]
|
||||
FRAME = os.environ.get("FRAME_ALIAS", "frame")
|
||||
FRAME_FROM_ENV = "FRAME_ALIAS" in os.environ
|
||||
if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._-]*", FRAME):
|
||||
@@ -97,16 +102,21 @@ def route(alias, host_opts):
|
||||
them follows; frame_titles reads frame_android.SSH_OPTS at call time."""
|
||||
global FRAME, HOST_OPTS
|
||||
with _route_lock:
|
||||
if alias != FRAME:
|
||||
moved = alias != FRAME
|
||||
if moved:
|
||||
frame_catalog._env.clear() # the SteamOS and Lepton builds reports record are per headset
|
||||
FRAME = frame_android.FRAME = alias
|
||||
HOST_OPTS = list(host_opts)
|
||||
MUX[:] = [*MUX_BASE, *HOST_OPTS]
|
||||
SSH[:] = [*MUX, *SSH_TAIL]
|
||||
frame_android.SSH_OPTS = SSH[1:]
|
||||
# Long-lived ssh processes started on the old route (outside the lock: they take their own).
|
||||
mv = globals().get("macview")
|
||||
if mv:
|
||||
mv.retarget(alias, host_opts) # its tunnel is its own ssh: it must follow the headset too
|
||||
agent = globals().get("_input")
|
||||
if moved and agent:
|
||||
agent.stop() # typing and pointing mustn't go on reaching the headset switched away from
|
||||
|
||||
|
||||
LINK = None # the connector (frame_link.Link); None on the Frame itself
|
||||
@@ -574,6 +584,322 @@ def clipboard(body):
|
||||
return {"message": ssh(PASTE_CMD, stdin=text, timeout=30).strip()}
|
||||
|
||||
|
||||
# ---- keyboard and pointer (KDE Connect on the Frame, see frame_input_agent.py) ----
|
||||
|
||||
INPUT_FLAGS = ("singleclick", "doubleclick", "middleclick", "rightclick", "singlehold", "singlerelease",
|
||||
"scroll", "ctrl", "alt", "shift", "super")
|
||||
INPUT_MOVE_LIMIT = 2000 # pixels per event
|
||||
INPUT_TEXT_LIMIT = 500 # characters per event
|
||||
INPUT_BATCH_LIMIT = 200 # events per request
|
||||
|
||||
|
||||
def input_event(event):
|
||||
"""A KDE Connect remote-input body with only the fields it knows, in range."""
|
||||
if not isinstance(event, dict):
|
||||
raise Failure("each input event must be an object", 400)
|
||||
out = {}
|
||||
for name in ("dx", "dy"):
|
||||
value = event.get(name)
|
||||
if value is None:
|
||||
continue
|
||||
if isinstance(value, bool) or not isinstance(value, (int, float)) or value != value:
|
||||
raise Failure(f"{name} must be a number", 400)
|
||||
out[name] = max(-INPUT_MOVE_LIMIT, min(INPUT_MOVE_LIMIT, round(float(value), 2)))
|
||||
for name in INPUT_FLAGS:
|
||||
if event.get(name) is True:
|
||||
out[name] = True
|
||||
key = event.get("key")
|
||||
if key is not None:
|
||||
if not isinstance(key, str) or not 0 < len(key) <= INPUT_TEXT_LIMIT:
|
||||
raise Failure(f"key must be text of 1 to {INPUT_TEXT_LIMIT} characters", 400)
|
||||
out["key"] = key
|
||||
special = event.get("specialKey")
|
||||
if special is not None:
|
||||
# KDE Connect's numbering: 1 Backspace … 14 Escape, 21-32 F1-F12.
|
||||
if isinstance(special, bool) or not isinstance(special, int) or not 1 <= special <= 32:
|
||||
raise Failure("specialKey must be a whole number from 1 to 32", 400)
|
||||
out["specialKey"] = special
|
||||
if not set(out) - {"ctrl", "alt", "shift", "super"}:
|
||||
raise Failure("input event has nothing to do", 400)
|
||||
return out
|
||||
|
||||
|
||||
def input_client():
|
||||
"""A stable id for this device, so KDE Connect on the Frame keeps its pairing apart.
|
||||
|
||||
The iPhone app passes one (FRAME_CLIENT). A computer makes one the first time
|
||||
and keeps it: host names alone can clash (desk.home and desk.office).
|
||||
"""
|
||||
if os.environ.get("FRAME_CLIENT"):
|
||||
return os.environ["FRAME_CLIENT"]
|
||||
if LOCAL:
|
||||
return DEVICE
|
||||
path = frame_host.data_dir("input-client-id")
|
||||
try:
|
||||
saved = path.read_text().strip()
|
||||
if re.fullmatch(r"[A-Za-z0-9_-]{4,64}", saved):
|
||||
return saved
|
||||
except OSError:
|
||||
pass
|
||||
made = (re.sub(r"[^A-Za-z0-9-]", "", INPUT_NAME)[:24] or "computer") + "-" + secrets.token_hex(4)
|
||||
try:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(made)
|
||||
except OSError:
|
||||
pass # still works this time; it pairs again next time
|
||||
return made
|
||||
|
||||
|
||||
# KDE Connect for the Frame, as Frame Control ships it (frame/kdeconnect/NOTICE.md).
|
||||
KDECONNECT = HERE.parent / "frame" / "kdeconnect"
|
||||
KDECONNECT_HOME = ".local/share/frame-control/kdeconnect"
|
||||
|
||||
|
||||
def kdeconnect_packages():
|
||||
"""[(file, sha256), ...] from frame/kdeconnect/packages.json; [] if it's missing."""
|
||||
try:
|
||||
manifest = json.loads((KDECONNECT / "packages.json").read_text())
|
||||
return [(p["file"], p["sha256"]) for p in manifest["packages"]]
|
||||
except (OSError, ValueError, KeyError, TypeError):
|
||||
return []
|
||||
|
||||
|
||||
def kdeconnect_stamp(packages):
|
||||
"""What the agent writes once these are unpacked (frame_input_agent.stamp)."""
|
||||
return "".join(f"{sha} {name}\n" for name, sha in packages)
|
||||
|
||||
|
||||
def file_sha256(path):
|
||||
digest = hashlib.sha256()
|
||||
with open(path, "rb") as f:
|
||||
for block in iter(lambda: f.read(1 << 20), b""):
|
||||
digest.update(block)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
class InputAgent:
|
||||
"""frame_input_agent.py running on the Frame, fed events over one long-lived ssh.
|
||||
|
||||
Before starting it, copies KDE Connect to the Frame if it isn't there yet. The
|
||||
agent unpacks it, pairs, and reports its state ({"state": "off" | "installing" |
|
||||
"starting" | "pairing" | "ready" | "error"}).
|
||||
"""
|
||||
|
||||
def __init__(self, source=HERE / "frame_input_agent.py", packages=None):
|
||||
self.source, self.proc, self.lock = source, None, threading.Lock()
|
||||
self.packages = kdeconnect_packages() if packages is None else packages
|
||||
# generation counts stop()s; launching is the generation a launch is under way for.
|
||||
self.status, self.launching, self.generation = {"state": "off"}, None, 0
|
||||
|
||||
def command(self, folder=""):
|
||||
code = base64.b64encode(self.source.read_bytes()).decode()
|
||||
client = input_client()
|
||||
return ("python3 -u -c " + shlex.quote(
|
||||
f"import base64;exec(compile(base64.b64decode('{code}'),'frame_input_agent','exec'))")
|
||||
+ f" {shlex.quote(client)} {shlex.quote(INPUT_NAME)} {shlex.quote(folder)}"
|
||||
+ f" {shlex.quote(json.dumps(self.packages, separators=(',', ':')))}")
|
||||
|
||||
def deliver(self, report, force=False):
|
||||
"""Where the agent finds the packages on the Frame, copying them there first if needed.
|
||||
|
||||
On the Frame itself (the iPhone app) they came with the bundle. Otherwise they
|
||||
go over the SSH connection, unless the Frame already has them unpacked (`force`:
|
||||
the agent found it didn't after all).
|
||||
"""
|
||||
if LOCAL:
|
||||
return str(KDECONNECT / "packages")
|
||||
if not self.packages:
|
||||
return "" # nothing to copy (the agent says so if it needed them)
|
||||
if not force:
|
||||
# Bytes, so Windows doesn't turn the stamp's line ends into CRLF.
|
||||
have = ssh(f"{{ test -x /usr/lib/kdeconnectd || cmp -s - {KDECONNECT_HOME}/root/.frame-control-packages; }}"
|
||||
" && echo yes || true", stdin=kdeconnect_stamp(self.packages).encode(), text=False, timeout=20)
|
||||
if have.strip() == b"yes":
|
||||
return ""
|
||||
# A folder of its own: a cancelled start's agent may still be cleaning up another.
|
||||
folder = f"{KDECONNECT_HOME}/incoming/{secrets.token_hex(8)}"
|
||||
ssh(f"mkdir -p {folder}", timeout=20)
|
||||
try:
|
||||
for name, sha in self.packages:
|
||||
path = KDECONNECT / "packages" / name
|
||||
if not path.is_file() or file_sha256(path) != sha:
|
||||
raise Failure(f"{name} is missing or damaged in this copy of Frame Control"
|
||||
" (a build runs app/build/fetch-deps.js to add it)", 500)
|
||||
report(f"Copying KDE Connect to the Frame ({name.rsplit('-', 3)[0]})")
|
||||
quoted = shlex.quote(name)
|
||||
ssh(f"cd {folder} && cat > {quoted}.part && mv {quoted}.part {quoted}",
|
||||
stdin=path.read_bytes(), text=False, timeout=600)
|
||||
except (Failure, OSError):
|
||||
self.discard(folder) # a partial copy is no use to anyone
|
||||
raise
|
||||
return f"~/{folder}"
|
||||
|
||||
def discard(self, folder):
|
||||
"""Remove a copy no agent will take over (best effort; agents tidy up old ones too)."""
|
||||
folder = folder.removeprefix("~/")
|
||||
if folder.startswith(f"{KDECONNECT_HOME}/incoming/") and not LOCAL:
|
||||
try:
|
||||
ssh(f"rm -rf {folder}", timeout=20)
|
||||
except (Failure, OSError):
|
||||
pass # never let tidying up get in the way of reporting and retrying
|
||||
|
||||
def start(self):
|
||||
with self.lock:
|
||||
if self.launching == self.generation or (self.proc and self.proc.poll() is None):
|
||||
return
|
||||
# (A launch from before a stop() may still be finishing; it ends itself.)
|
||||
self.launching, self.status = self.generation, {"state": "starting"}
|
||||
generation = self.generation
|
||||
threading.Thread(target=self._launch, args=(generation,), daemon=True).start()
|
||||
|
||||
def _launch(self, generation, force=False):
|
||||
try:
|
||||
self._launch_once(generation, force)
|
||||
except Exception as e: # whatever went wrong, never leave it stuck "starting"
|
||||
with self.lock:
|
||||
if self.launching == generation:
|
||||
self.launching = None
|
||||
if self.generation == generation and self.status.get("state") in ("starting", "installing"):
|
||||
self.status = {"state": "error", "message": f"Couldn't start the keyboard and trackpad: {e}"}
|
||||
|
||||
def _launch_once(self, generation, force):
|
||||
def report(message):
|
||||
with self.lock:
|
||||
if self.generation == generation:
|
||||
self.status = {"state": "installing", "message": message}
|
||||
folder = ""
|
||||
try:
|
||||
errors = tempfile.TemporaryFile()
|
||||
ensure_master()
|
||||
folder = self.deliver(report, force)
|
||||
with self.lock:
|
||||
stopped = self.generation != generation
|
||||
if stopped: # turned off while copying
|
||||
raise Failure("stopped")
|
||||
proc = subprocess.Popen([*SSH, FRAME, self.command(folder)], stdin=subprocess.PIPE,
|
||||
stdout=subprocess.PIPE, stderr=errors)
|
||||
except (Failure, OSError) as e:
|
||||
self.discard(folder) # no agent will take the copy over
|
||||
message = str(e)
|
||||
friendly = unreachable(message)
|
||||
with self.lock:
|
||||
if self.launching == generation:
|
||||
self.launching = None
|
||||
if self.generation == generation:
|
||||
self.status = {"state": "error", "message": friendly or message,
|
||||
**({"offline": True} if friendly else {})}
|
||||
return
|
||||
_live_tunnels.add(proc)
|
||||
with self.lock:
|
||||
if self.launching == generation:
|
||||
self.launching = None
|
||||
stale = self.generation != generation
|
||||
if not stale:
|
||||
self.proc = proc
|
||||
if stale: # turned off meanwhile
|
||||
proc.terminate()
|
||||
wanted, heard = self._watch(proc, errors, retry=not force)
|
||||
if not heard:
|
||||
# The agent never started (or was stopped first), so it can't tidy the copy up.
|
||||
self.discard(folder)
|
||||
if wanted and not force:
|
||||
# It needed the packages after all (another device changed what's
|
||||
# installed after we looked): copy them and start once more.
|
||||
with self.lock:
|
||||
# Unless a start() already took over (it launches, and copies if still needed).
|
||||
if self.proc is not proc or self.generation != generation or self.launching is not None:
|
||||
return
|
||||
self.proc, self.launching, self.status = None, generation, {"state": "starting"}
|
||||
self._launch(generation, force=True)
|
||||
|
||||
def _watch(self, proc, errors, retry=False):
|
||||
"""Follow the agent's status until it exits. Returns whether it asked for the
|
||||
packages (`retry`: the caller will send them, so that isn't an error yet), and
|
||||
whether it said anything at all (then it holds its copy and tidies it up)."""
|
||||
wanted = heard = False
|
||||
for line in proc.stdout:
|
||||
try:
|
||||
status = json.loads(line)
|
||||
except ValueError:
|
||||
continue
|
||||
if isinstance(status, dict) and isinstance(status.get("state"), str):
|
||||
heard = True
|
||||
if status["state"] == "need-packages":
|
||||
wanted = True
|
||||
continue
|
||||
with self.lock:
|
||||
if self.proc is proc:
|
||||
self.status = status
|
||||
proc.wait()
|
||||
_live_tunnels.discard(proc)
|
||||
try:
|
||||
errors.seek(0)
|
||||
detail = strip_ansi(errors.read().decode(errors="replace")).strip()
|
||||
except OSError:
|
||||
detail = ""
|
||||
with self.lock:
|
||||
if self.proc is proc and self.status.get("state") != "error" and not (wanted and retry):
|
||||
message = detail.splitlines()[-1] if detail else "The connection to the Frame ended"
|
||||
if wanted:
|
||||
message = "KDE Connect didn't reach the Frame"
|
||||
friendly = unreachable(message)
|
||||
self.status = {"state": "error", "message": friendly or message, **({"offline": True} if friendly else {})}
|
||||
return wanted, heard
|
||||
|
||||
def send(self, events):
|
||||
"""Forward events if the agent is ready; start it if it isn't running.
|
||||
|
||||
Returns the state, with "sent" saying whether the events went; if not,
|
||||
the page keeps them and sends them again once the state is "ready".
|
||||
"""
|
||||
with self.lock:
|
||||
proc, ready = self.proc, self.status.get("state") == "ready"
|
||||
sent = False
|
||||
if not (proc and proc.poll() is None):
|
||||
self.start()
|
||||
elif ready and events:
|
||||
try:
|
||||
proc.stdin.write((json.dumps(events) + "\n").encode())
|
||||
proc.stdin.flush()
|
||||
sent = True
|
||||
except (BrokenPipeError, OSError, ValueError):
|
||||
pass # _watch reports how it ended
|
||||
with self.lock:
|
||||
return {**self.status, "sent": sent}
|
||||
|
||||
def stop(self):
|
||||
with self.lock:
|
||||
proc, self.proc, self.status = self.proc, None, {"state": "off"}
|
||||
self.generation += 1
|
||||
if proc and proc.poll() is None:
|
||||
proc.terminate()
|
||||
|
||||
|
||||
_input = InputAgent()
|
||||
|
||||
|
||||
def licenses():
|
||||
"""The notices and licence texts for what Frame Control ships (the About dialog)."""
|
||||
found = [("Third-party notices", HERE.parent / "THIRD_PARTY_NOTICES.md"), ("KDE Connect for the Frame", KDECONNECT / "NOTICE.md"),
|
||||
("Frame Control (MIT)", HERE.parent / "LICENSE")]
|
||||
found += [(f"{p.parent.name}: {p.stem}", p) for p in sorted((KDECONNECT / "LICENSES").glob("*/*.txt"))]
|
||||
notices = []
|
||||
for title, path in found:
|
||||
try:
|
||||
notices.append({"title": title, "text": path.read_text(errors="replace")})
|
||||
except OSError:
|
||||
pass
|
||||
return notices
|
||||
|
||||
|
||||
def remote_input(body):
|
||||
"""{"events": [...]} sends keyboard and pointer events; {} (or none yet) just starts the agent."""
|
||||
events = body.get("events", [])
|
||||
if not isinstance(events, list) or len(events) > INPUT_BATCH_LIMIT:
|
||||
raise Failure(f"events must be a list of at most {INPUT_BATCH_LIMIT}", 400)
|
||||
return _input.send([input_event(e) for e in events])
|
||||
|
||||
|
||||
def flatpak(body):
|
||||
app, action = str(body.get("id", "")), body.get("action")
|
||||
if not FLATPAK_ID.match(app):
|
||||
@@ -1351,6 +1677,75 @@ def _sweep_one(prefix, d):
|
||||
pass
|
||||
|
||||
|
||||
# ---- Our Frame-side media player -----------------------------------------
|
||||
|
||||
_MEDIA_LOCK = threading.Lock()
|
||||
|
||||
|
||||
def media(body):
|
||||
action = body.get("action")
|
||||
if action not in ("list", "status", "play", "stop"):
|
||||
raise Failure("Media action must be list, status, play or stop", 400)
|
||||
if action == "play":
|
||||
identity = body.get("id")
|
||||
if not isinstance(identity, str) or not re.fullmatch(r"[0-9a-f]{32}/[^/\\\x00]+", identity):
|
||||
raise Failure("Invalid media id", 400)
|
||||
if body.get("layout", "auto") not in frame_media.LAYOUTS:
|
||||
raise Failure("Invalid media layout", 400)
|
||||
if type(body.get("theatre", False)) is not bool:
|
||||
raise Failure("theatre must be true or false", 400)
|
||||
with _MEDIA_LOCK:
|
||||
# Ship only our small stdlib modules, atomically, to the user account.
|
||||
sources = {name: (HERE / name).read_text() for name in (
|
||||
"frame_media.py", "frame_media_player.py", "frame_media_remote.py", "frame_splat.py")}
|
||||
installer = """import json, os, pathlib, sys, tempfile
|
||||
root = pathlib.Path.home()/'.local/share/frame-control/media'
|
||||
root.mkdir(parents=True, exist_ok=True)
|
||||
for name, source in json.load(sys.stdin).items():
|
||||
path = root/name
|
||||
fd, temp = tempfile.mkstemp(dir=root, prefix=name+'.')
|
||||
with os.fdopen(fd, 'w') as f:
|
||||
f.write(source)
|
||||
os.replace(temp, path)
|
||||
"""
|
||||
ssh("python3 -c " + shlex.quote(installer), stdin=json.dumps(sources))
|
||||
try:
|
||||
out = ssh("python3 ~/.local/share/frame-control/media/frame_media_remote.py",
|
||||
stdin=json.dumps(body), timeout=75) # remote worst case: ffprobe 30 + reset-failed 10 + systemd-run 15 s
|
||||
except Failure as e:
|
||||
for line in reversed(getattr(e, "stdout", "").splitlines()):
|
||||
try:
|
||||
detail = json.loads(line).get("error")
|
||||
except (ValueError, AttributeError):
|
||||
continue
|
||||
if detail:
|
||||
raise Failure(detail) from None
|
||||
raise
|
||||
return json.loads(out)
|
||||
|
||||
|
||||
def push_media(path):
|
||||
# Validate the format, but leave layout selection until playback (ffprobe
|
||||
# can then read metadata on the Frame, where it is installed).
|
||||
name = Path(path).name
|
||||
frame_media.plan(name, "mono")
|
||||
if name.startswith(".") or "\\" in name:
|
||||
raise Failure("Rename the file: media names can't start with a dot or contain a backslash", 400)
|
||||
token = secrets.token_hex(16)
|
||||
dest = "Videos/FrameControl/" + token + "/"
|
||||
ssh("mkdir -p ~/" + dest)
|
||||
try:
|
||||
push_file(path, dest)
|
||||
except Exception:
|
||||
try:
|
||||
ssh("rm -rf ~/" + dest)
|
||||
except Exception:
|
||||
pass # keep the copy error; an empty folder isn't listed as media
|
||||
raise
|
||||
return {"message": "Media sent. Choose its layout and press Play.",
|
||||
"id": token + "/" + name}
|
||||
|
||||
|
||||
# ---- Mac in the headset (frame_macview.py) ----------------------------------
|
||||
|
||||
# The tunnel gets its own connection: the shared master's options would win
|
||||
@@ -1417,8 +1812,9 @@ def agent_approval(body):
|
||||
return frame_agent.approvals.decide(body.get("confirmation"), body.get("accept"))
|
||||
|
||||
|
||||
POST = {"/api/agent/call": agent_call, "/api/agent/approval": agent_approval,
|
||||
POST = {"/api/media": media, "/api/agent/call": agent_call, "/api/agent/approval": agent_approval,
|
||||
"/api/assistant/chat": assistant_chat, "/api/android/display": android_display, "/api/android": android, "/api/titles": titles, "/api/launch": launch, "/api/steam": steam, "/api/volume": set_volume, "/api/clipboard": clipboard,
|
||||
"/api/input": remote_input,
|
||||
"/api/flatpak": flatpak, "/api/open": open_thing, "/api/shots/save": save_shots,
|
||||
"/api/webinstall/check": webinstall_check, "/api/webinstall/start": webinstall_start,
|
||||
"/api/webinstall/cancel": webinstall_cancel,
|
||||
@@ -1603,6 +1999,10 @@ class Handler(BaseHTTPRequestHandler):
|
||||
self.send_json({"titles": frame_titles.list_titles()})
|
||||
elif path == "/api/titles/job":
|
||||
self.send_json(title_job(url.query))
|
||||
elif path == "/api/licenses":
|
||||
self.send_json({"notices": licenses()})
|
||||
elif path == "/api/input":
|
||||
self.send_json(_input.send([]) if parse_qs(url.query).get("start") == ["1"] else dict(_input.status))
|
||||
elif path == "/api/job":
|
||||
self.send_json(job_status(url.query))
|
||||
elif path == "/api/android/displays":
|
||||
@@ -1790,6 +2190,8 @@ class Handler(BaseHTTPRequestHandler):
|
||||
raise Failure("upload interrupted", 400)
|
||||
f.write(chunk)
|
||||
remaining -= len(chunk)
|
||||
if mode == "media":
|
||||
return push_media(dest)
|
||||
if mode == "apkinfo":
|
||||
# Read an APK for a report without installing it.
|
||||
try:
|
||||
|
||||
Reference in new issue
Block a user