Live H.264 video of the headset view in Frame Control

Live in the headset view now streams video instead of polling stereo
screenshots (~2 fps). SteamVR's steamvr-v4l2cam.service mirrors the headset
view into /dev/video99; ffmpeg on the Frame encodes it with x264 (720p30 by
default, AUD + repeated SPS/PPS), /api/stream relays the raw H.264 over SSH,
and the page splits it on access unit delimiters and decodes it with
WebCodecs into the existing viewer. Capture still takes a stereo still; the
desktop panel keeps capture polling, and the page falls back to it if the
video can't start.

The remote ffmpeg runs under a shell that kills it when the SSH channel
closes, stderr goes to a temp file, and a 10 s stall ends the stream. One
stream at a time; a new one supersedes the last.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

ui/index.html

@@ -321,7 +321,7 @@
         </div>
         <div class="spacer"></div>
         <button class="action" id="shotBtn">Capture</button>
-        <button id="liveBtn" title="Keep capturing">Live</button>
+        <button id="liveBtn" title="Keep updating: video of the headset view, or repeated captures of the desktop panel">Live</button>
         <button id="saveBtn" disabled>Save</button>
       </div>
       <div class="viewer" id="viewer">
@@ -575,7 +575,7 @@ const QUICK = [["Remmina", "org.remmina.Remmina"], ["Moonlight", "com.moonlight_
                ["Firefox", "org.mozilla.firefox"], ["VLC", "org.videolan.VLC"]];
 const CDN = "https://cdn.cloudflare.steamstatic.com/steam/apps";
 const HINTS = {
-  headset: "What the lenses show, composited by SteamVR: the room, floating panels, dashboard and controllers. Live refreshes about twice a second. Captures show everything on screen, including anything private.",
+  headset: "What the lenses show, composited by SteamVR: the room, floating panels, dashboard and controllers. Live streams it as video (one eye, about 30 fps); Capture takes a still of both eyes. Captures show everything on screen, including anything private.",
   flat: "gamescope's 2D layer: the desktop panel and Steam's flat UI, without the room or VR scene.",
 };
 const SOURCE_LABEL = { steamvr: "Headset view", gamescope: "Desktop panel", shot: "Screenshot" };
@@ -751,10 +751,12 @@ function render(s) {
 
 // ---- view ----
 function setView(v) {
+  const changed = v !== view;
   view = v;
   document.querySelectorAll("[data-view]").forEach(b => b.classList.toggle("on", b.dataset.view === v));
-  $("eyeSeg").style.visibility = v === "headset" ? "visible" : "hidden";
+  $("eyeSeg").style.visibility = v === "headset" && !video.ctl ? "visible" : "hidden";
   $("viewHint").textContent = HINTS[v];
+  if (live && changed) { toggleLive(false); toggleLive(true); }  // video for the headset, captures for the panel
 }
 function setEye(e) {
   eye = e;
@@ -841,6 +843,7 @@ function isBlank(ctx, w, h) {
   return max - min < 6;
 }
 async function capture() {
+  if (video.ctl) toggleLive(false);  // Capture during live video takes a still of both eyes instead
   const gen = viewGen;  // a screenshot opened meanwhile wins over this capture
   if (!live) $("viewer").classList.add("busy");  // no spinner flashing over a live stream
   let url = null;
@@ -883,7 +886,115 @@ function toggleLive(on) {
   liveFailures = 0;
   $("liveBtn").classList.toggle("on", live);
   $("liveBadge").hidden = !live;
-  if (live) liveLoop(); else clearTimeout(liveTimer);
+  clearTimeout(liveTimer);
+  stopVideo();
+  if (!live) return;
+  if (view === "headset" && "VideoDecoder" in window) {
+    const started = startVideo();
+    const ctl = video.ctl;
+    started.catch(e => {
+      if (!live || ctl !== video.ctl || e.name === "AbortError") return;
+      log("Live video failed: " + e.message, "e");
+      toast("Live video failed, using captures instead: " + e.message, true);
+      stopVideo();
+      liveLoop();
+    });
+  } else liveLoop();
+}
+
+// ---- live video of the headset view ----
+// The server relays raw H.264 (Annex B) from SteamVR's headset-view device.
+// Every frame starts with an access unit delimiter (NAL type 9), which is how
+// the stream is cut into frames for WebCodecs.
+const STREAM_QUERY = "h=720&fps=30", STREAM_FPS = 30;
+const video = { ctl: null, dec: null, gen: 0 };
+function startCode(b, i) { return b[i] === 0 && b[i + 1] === 0 && b[i + 2] === 1; }
+function nalTypes(au) {
+  const types = [];
+  for (let i = 0; i + 3 < au.length; i++) if (startCode(au, i)) { types.push(au[i + 3] & 0x1f); i += 3; }
+  return types;
+}
+async function startVideo() {
+  const gen = ++video.gen, ctl = new AbortController();
+  video.ctl = ctl;
+  $("eyeSeg").style.visibility = "hidden";
+  if (!lastImg) $("viewer").classList.add("busy");
+  let frames = 0, shown = 0, second = performance.now(), needKey = true, ts = 0;
+  const c = $("canvas"), ctx = c.getContext("2d");
+  const dec = video.dec = new VideoDecoder({
+    output: frame => {
+      if (gen !== video.gen) return frame.close();
+      if (c.width !== frame.displayWidth || c.height !== frame.displayHeight) {
+        c.width = frame.displayWidth; c.height = frame.displayHeight;
+        $("viewer").style.aspectRatio = `${c.width} / ${c.height}`;
+        setZoom(1);
+      }
+      ctx.drawImage(frame, 0, 0);
+      frame.close();
+      if (!shown++) {
+        viewGen++;  // a capture still in flight mustn't replace the video
+        lastImg = null; lastSource = "video"; lastShot = null;
+        $("viewer").classList.remove("busy");
+        c.hidden = false; $("viewerEmpty").hidden = true; $("zoombar").hidden = false; $("asleep").hidden = true;
+        $("srcBadge").hidden = false; $("srcBadge").textContent = "Headset view · video";
+        $("stamp").hidden = false; $("saveBtn").disabled = false;
+      }
+      frames++;
+      const now = performance.now();
+      if (now - second >= 1000) {
+        $("stamp").textContent = `${Math.round(frames * 1000 / (now - second))} fps`;
+        frames = 0; second = now;
+      }
+    },
+    error: e => log("Video decoder: " + e.message, "e"),
+  });
+  const r = await fetch(`/api/stream?${STREAM_QUERY}`, { headers: {"X-Frame-UI": "1"}, signal: ctl.signal });
+  if (!r.ok) throw new Error((await r.json().catch(() => ({}))).error || `HTTP ${r.status}`);
+  const reader = r.body.getReader();
+  let buf = new Uint8Array(0), scan = 0, auStart = -1;
+  const onAU = au => {
+    const types = nalTypes(au), key = types.includes(5);
+    if (dec.state === "unconfigured") {
+      const sps = types.indexOf(7);
+      if (sps < 0) return;
+      let i = 0, n = -1;  // find the SPS bytes: profile, constraints, level follow its header
+      for (; i + 3 < au.length; i++) if (startCode(au, i) && ++n === sps) break;
+      if (i + 6 >= au.length) return;
+      const hex = [au[i + 4], au[i + 5], au[i + 6]].map(b => b.toString(16).padStart(2, "0")).join("");
+      dec.configure({ codec: `avc1.${hex}`, optimizeForLatency: true });
+    }
+    // If decoding falls behind, drop frames until the next keyframe rather than lag.
+    if (dec.decodeQueueSize > 3) needKey = true;
+    if (needKey && !key) return;
+    needKey = false;
+    dec.decode(new EncodedVideoChunk({ type: key ? "key" : "delta", timestamp: ts, data: au }));
+    ts += 1e6 / STREAM_FPS;
+  };
+  for (;;) {
+    const { value, done } = await reader.read();
+    if (done || gen !== video.gen) break;
+    const next = new Uint8Array(buf.length + value.length);
+    next.set(buf); next.set(value, buf.length);
+    buf = next;
+    if (buf.length > 8 << 20) throw new Error("the stream isn't split into frames");
+    for (; scan + 3 < buf.length; scan++) {
+      if (!startCode(buf, scan) || (buf[scan + 3] & 0x1f) !== 9) continue;
+      if (auStart >= 0) onAU(buf.subarray(auStart, scan));
+      auStart = scan;
+      scan += 3;
+    }
+    if (auStart > 0) { buf = buf.slice(auStart); scan -= auStart; auStart = 0; }
+  }
+  if (gen === video.gen && live) throw new Error(shown ? "the stream ended" : "no video arrived");
+}
+function stopVideo() {
+  video.gen++;
+  if (video.ctl) video.ctl.abort();
+  if (video.dec && video.dec.state !== "closed") video.dec.close();
+  video.ctl = video.dec = null;
+  $("viewer").classList.remove("busy");
+  if (lastSource === "video") { $("srcBadge").textContent = "Headset view · video (stopped)"; $("stamp").hidden = true; }
+  $("eyeSeg").style.visibility = view === "headset" ? "visible" : "hidden";
 }
 $("shotBtn").onclick = () => capture();
 $("liveBtn").onclick = () => toggleLive(!live);

ui/server.py

@@ -12,6 +12,7 @@ import http.client
 import json
 import os
 import re
+import select
 import shlex
 import shutil
 import signal
@@ -272,6 +273,41 @@ def save_shots(body):
     return {"message": msg + (f" ({skipped} already there)" if skipped else ""), "saved": n}
 
 
+# Live video of the headset view. SteamVR's steamvr-v4l2cam.service copies the
+# headset view (one undistorted 1920x1080 image) into the v4l2loopback device
+# /dev/video99. ffmpeg encodes it with x264 (the hardware encoder crashes
+# ffmpeg) and the raw H.264 comes back over SSH for the page to decode with
+# WebCodecs. An access unit delimiter starts every frame so the page can split
+# the stream, and repeated SPS/PPS let it start at any keyframe. ffmpeg runs in
+# the background while the shell waits for our stdin to close: when the local
+# ssh goes, the channel closes and the shell kills ffmpeg, even one that has
+# stopped writing (and so would never get SIGPIPE).
+STREAM_DEVICE = "/dev/video99"
+STREAM_HEIGHTS = (720, 1080)
+STREAM_FPS = (30, 60)
+STREAM_STALL = 10  # seconds without video before the stream is dropped
+_stream_lock = threading.Lock()
+_stream_proc = None
+
+
+def stream_command(query):
+    q = parse_qs(query)
+    try:
+        height = int((q.get("h") or ["720"])[0])
+        fps = int((q.get("fps") or ["30"])[0])
+    except ValueError:
+        raise Failure("h and fps must be integers", 400)
+    if height not in STREAM_HEIGHTS or fps not in STREAM_FPS:
+        raise Failure(f"h must be one of {STREAM_HEIGHTS} and fps one of {STREAM_FPS}", 400)
+    rate = 3 if height == 720 else 6  # Mbit/s
+    return (f"[ -e {STREAM_DEVICE} ] || {{ echo 'No headset view device ({STREAM_DEVICE}). Is SteamVR running?' >&2; exit 3; }}; "
+            f"ffmpeg -hide_banner -loglevel error -nostdin -f v4l2 -video_size 1920x1080 -i {STREAM_DEVICE} "
+            f"-vf fps={fps},scale=-2:{height},format=yuv420p -c:v libx264 -preset ultrafast -tune zerolatency "
+            f"-g {fps * 2} -bf 0 -b:v {rate}M -maxrate {rate}M -bufsize {rate // 2 or 1}M "
+            f"-x264-params aud=1:repeat-headers=1 -f h264 - & p=$!; "
+            f"exec >&-; cat >/dev/null; kill $p 2>/dev/null; wait $p")
+
+
 def launch(body):
     appid = str(body.get("appid", ""))
     if not APPID.match(appid):
@@ -423,7 +459,7 @@ FONT_RANGE = (0.5, 2.0)
 KNOWN_LABELS = {"com.t3tools.t3code": "T3 Code", "org.fdroid.fdroid": "F-Droid"}
 # One ADB session at a time: requests are rare, and it keeps adb's state simple.
 _adb_lock = threading.Lock()
-_live_tunnels = set()  # ssh processes to kill if the server stops mid-request
+_live_tunnels = set()  # ssh processes (ADB forwards, live video) to kill if the server stops mid-request
 
 
 def adb_path():
@@ -770,6 +806,8 @@ class Handler(BaseHTTPRequestHandler):
                 self.send_json(list_shots())
             elif path == "/api/shots/image":
                 self.send_bytes(*shot_image(url.query))
+            elif path == "/api/stream":
+                self.stream_video(url.query)
             elif path == "/api/screenshot" and parse_qs(url.query).get("view") == ["headset"]:
                 self.send_bytes(headset_view(), "image/png", headers=[("X-Capture-Source", "steamvr")])
             elif path == "/api/screenshot":
@@ -808,6 +846,64 @@ class Handler(BaseHTTPRequestHandler):
         except Exception as e:
             self.send_json({"error": f"{type(e).__name__}: {e}"}, 500)
 
+    def stream_video(self, query):
+        """Raw H.264 of the headset view until the page disconnects (see stream_command)."""
+        global _stream_proc
+        remote = stream_command(query)
+        ensure_master()
+        # stderr goes to a file: nothing reads it while streaming, and a full
+        # pipe would stall ffmpeg. It's only read if the stream fails to start.
+        errors = tempfile.TemporaryFile()
+        proc = subprocess.Popen([*SSH, FRAME, remote], stdin=subprocess.PIPE,
+                                stdout=subprocess.PIPE, stderr=errors)
+        try:
+            _live_tunnels.add(proc)
+            # One viewer at a time: a new stream (another tab, a reload) ends the last one.
+            with _stream_lock:
+                old, _stream_proc = _stream_proc, proc
+            if old and old.poll() is None:
+                old.terminate()
+            fd = proc.stdout.fileno()
+            # Nothing is sent until the first bytes arrive, so a failure to
+            # start still comes back as a JSON error.
+            ready, _, _ = select.select([fd], [], [], 20)
+            first = os.read(fd, 1 << 16) if ready else b""
+            if not first:
+                proc.kill()
+                proc.wait()
+                errors.seek(0)
+                err = strip_ansi(errors.read().decode(errors="replace")).strip()
+                raise Failure(err or "The headset view sent no video for 20 s")
+            chunk = first
+            try:
+                self.send_response(200)
+                self.send_header("Content-Type", "video/h264")
+                self.send_header("Cache-Control", "no-store")
+                self.send_header("X-Frame-Options", "DENY")
+                self.send_header("Content-Security-Policy", "frame-ancestors 'none'")
+                self.end_headers()
+                self.close_connection = True  # the body ends when the connection does
+                while chunk:
+                    self.wfile.write(chunk)
+                    self.wfile.flush()
+                    # A stalled headset view ends the stream rather than
+                    # holding this thread (and the page) forever.
+                    ready, _, _ = select.select([fd], [], [], STREAM_STALL)
+                    chunk = os.read(fd, 1 << 16) if ready else b""
+            except OSError:
+                pass  # the page stopped watching (or stopped reading); the body has started, so no JSON
+        finally:
+            if proc.poll() is None:
+                proc.terminate()
+                try:
+                    proc.wait(timeout=5)
+                except subprocess.TimeoutExpired:
+                    proc.kill()
+                    proc.wait()
+            for f in (proc.stdin, proc.stdout, errors):
+                f.close()
+            _live_tunnels.discard(proc)
+
     def upload(self):
         """Raw file body. X-Filename names it; X-Mode is 'push', 'apk' (install) or 'apkinfo' (read only)."""
         name = os.path.basename(unquote(self.headers.get("X-Filename", "")))
@@ -871,7 +967,7 @@ def main():
         subprocess.run([*MUX, "-O", "exit", FRAME], capture_output=True)
         if _master and _master.poll() is None:
             _master.terminate()
-        for proc in list(_live_tunnels):  # ADB forwards of requests cut off mid-way
+        for proc in list(_live_tunnels):  # ADB forwards and video streams cut off mid-way
             if proc.poll() is None:
                 proc.terminate()