saphid--frame-control/ui/mac-view.html

<!doctype html>
<!-- Frame Control: one Mac window (or display) inside the Steam Frame.
     Served by the Mac's frame-mac-view agent through an SSH tunnel and opened
     on the Frame as its own Chromium app window, which gamescope turns into a
     SteamVR panel. Video arrives over a WebSocket (H.264 Annex B for
     WebCodecs, or JPEG); pointer, wheel and keys go back the same way. -->
<html lang="en">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>Mac</title>
<style>
  html, body { margin: 0; height: 100%; background: #000; overflow: hidden; cursor: default; }
  canvas { position: fixed; inset: 0; width: 100%; height: 100%; object-fit: contain; image-rendering: auto; }
  #note { position: fixed; left: 50%; top: 16px; transform: translateX(-50%); max-width: 80%;
          font: 15px/1.4 system-ui, sans-serif; color: #eee; background: rgba(20,22,26,.88);
          padding: 8px 14px; border-radius: 10px; pointer-events: none; transition: opacity .4s; }
  #note[hidden] { display: block; opacity: 0; }
  #hud { position: fixed; left: 8px; top: 8px; font: 12px/1.35 ui-monospace, monospace; color: #dfe;
         background: rgba(0,0,0,.72); padding: 6px 9px; border-radius: 6px; pointer-events: none; white-space: pre; }
</style>
</head>
<body>
<canvas id="c" width="16" height="9"></canvas>
<div id="note">Connecting to the Mac…</div>
<div id="hud" hidden></div>
<script>
"use strict";
const q = new URLSearchParams(location.search);
// t: a single-use ticket from Frame Control. After the first connection the
// Mac sends a reconnect key, kept only in memory.
const src = q.get("src") || "test", ticket = q.get("t") || "", tag = q.get("tag") || "";
let reconnectKey = "";
const c = document.getElementById("c"), ctx = c.getContext("2d", { alpha: false, desynchronized: true });
const note = document.getElementById("note");
// Counters for Frame Control's tests (read over DevTools).
const stats = window.stats = { frames: 0, keyFrames: 0, bytes: 0, dropped: 0, codec: "", errors: [], info: null };

// ---- timing: the Mac measures each frame's journey on its own clock ----
// Clock sync, NTP-style: the Mac's time minus ours, from the ping with the
// shortest round trip lately (the least queueing, so the least error).
const clock = { off: null, rtt: 0, samples: [], reported: 0 };
function toMac(ms) { return clock.off === null ? null : Math.round(ms * 1000 + clock.off); }
function onPong(m) {
  const now = performance.now(), rtt = now - m.c;
  clock.samples.push({ rtt, off: m.a - (m.c + now) / 2 * 1000 });
  if (clock.samples.length > 16) clock.samples.shift();
  const best = clock.samples.reduce((a, b) => (b.rtt < a.rtt ? b : a));
  clock.off = best.off;
  clock.rtt = best.rtt;
  if (now - clock.reported > 1000) {
    clock.reported = now;
    send({ t: "clock", rtt: +best.rtt.toFixed(2), dec: `${decoderInfo}; ${glInfo}; raf ${rafHz} Hz` });
  }
}
let pingTimer = 0;
function startPings() {
  clearInterval(pingTimer);
  clock.samples = [];
  for (let i = 0; i < 10; i++) setTimeout(() => send({ t: "ping", c: performance.now() }), i * 40);
  pingTimer = setInterval(() => send({ t: "ping", c: performance.now() }), 1000);
}
// Decoded, drawn and next-animation-frame times go back in batches.
let reports = [], dropsToReport = 0, shownPending = null, rafQueued = false;
setInterval(() => {
  if (!reports.length && !dropsToReport) return;
  send({ t: "fd", f: reports, drop: dropsToReport });
  reports = [];
  dropsToReport = 0;
}, 250);
function dropped() { stats.dropped++; dropsToReport++; }
// A drawn frame is on screen from the next animation frame, unless another
// replaces it first (then it was never seen).
function shown(seq, decodedAt, drawnAt) {
  if (shownPending) reports.push([shownPending.seq, toMac(shownPending.dec), toMac(shownPending.drawn), 0]);
  shownPending = { seq, dec: decodedAt, drawn: drawnAt };
  if (rafQueued) return;
  rafQueued = true;
  requestAnimationFrame(() => {
    rafQueued = false;
    const p = shownPending, now = performance.now();
    shownPending = null;
    if (p && clock.off !== null) reports.push([p.seq, toMac(p.dec), toMac(p.drawn), toMac(now)]);
  });
}
let decoderInfo = "";
// What this browser draws with, and how often it gives an animation frame
// when nothing else is going on (both only for the numbers).
let glInfo = "", rafHz = 0;
try {
  const gl = document.createElement("canvas").getContext("webgl");
  const ext = gl && gl.getExtension("WEBGL_debug_renderer_info");
  glInfo = ext ? gl.getParameter(ext.UNMASKED_RENDERER_WEBGL) : gl ? "webgl" : "no webgl";
} catch (e) { glInfo = "?"; }
(function measureRaf() {
  let n = 0, t0 = 0;
  const tick = t => {
    if (!t0) t0 = t;
    if (t - t0 < 1000) { n++; return requestAnimationFrame(tick); }
    rafHz = Math.round(n * 1000 / (t - t0));
  };
  requestAnimationFrame(tick);
})();
const hud = document.getElementById("hud");
function showHud(on) { hud.hidden = !on; }
function ms(o) { return o && o.p50 !== undefined ? `${o.p50}/${o.p95}` : "–"; }
function onStats(m) {
  if (hud.hidden) return;
  hud.textContent = `${m.size}  ${m.fps} fps shown (${m.sentFps} sent)  ${m.mbps} Mbit/s` +
    (m.bitrate ? ` of ${(m.bitrate / 1e6).toFixed(1)}` : "") + (m.tier !== undefined ? `  tier ${m.tier}` : "") +
    `\nlatency p50/p95 ms  total ${ms(m.total)}\n  capture ${ms(m.capture)}  queue ${ms(m.queue)}  encode ${ms(m.encode)}` +
    `\n  network ${ms(m.network)}  decode ${ms(m.decode)}  draw ${ms(m.draw)}` +
    `\ninput→shown p50 ${m.input ? m.input.p50.toFixed(0) + " ms" : "–"}  rtt ${m.rtt} ms` +
    `\nskipped ${m.skipped}/${m.captured}  dropped ${m.dropped}  ${decoderInfo}`;
}
showHud(q.get("stats") === "1");
// Frame Control finds this window on the Frame by the tag in its title.
document.title = tag ? `Mac [${tag}]` : "Mac";
let failedStarts = 0;
let ws = null, dec = null, codecString = "", needKey = true, closing = false, retry = 0, noteTimer = 0, lastError = "";

function show(text, ms) {
  note.textContent = text;
  note.hidden = false;
  clearTimeout(noteTimer);
  if (ms) noteTimer = setTimeout(() => { note.hidden = true; }, ms);
}

function send(obj) { if (ws && ws.readyState === 1) ws.send(JSON.stringify(obj)); }
let lastKeyAsk = 0;
function askKeyFrame() {
  const now = performance.now();
  if (now - lastKeyAsk < 500) return;
  lastKeyAsk = now;
  send({ t: "key-frame" });
}

let hideNoteOnFrame = true;  // "Connecting…"/"Reconnecting…" go once video flows again
function drawFrame(img, w, h, seq, decodedAt) {
  if (c.width !== w || c.height !== h) { c.width = w; c.height = h; }
  ctx.drawImage(img, 0, 0);
  shown(seq, decodedAt, performance.now());
  stats.frames++;
  if (hideNoteOnFrame) { hideNoteOnFrame = false; note.hidden = true; }
}

// ---- H.264 ----
function startCode(b, i) { return b[i] === 0 && b[i + 1] === 0 && (b[i + 2] === 1 || (b[i + 2] === 0 && b[i + 3] === 1)); }
function spsCodec(au) {
  for (let i = 0; i + 7 < au.length; i++) {
    if (!startCode(au, i)) continue;
    const n = au[i + 2] === 1 ? i + 3 : i + 4;
    if ((au[n] & 0x1f) === 7) return "avc1." + [au[n + 1], au[n + 2], au[n + 3]].map(b => b.toString(16).padStart(2, "0")).join("");
    i = n;
  }
  return "";
}
function makeDecoder() {
  if (dec) try { dec.close(); } catch (e) {}
  codecString = "";
  dec = new VideoDecoder({
    // The chunk's timestamp is the Mac's sequence number, to match reports up.
    output: frame => { drawFrame(frame, frame.displayWidth, frame.displayHeight, frame.timestamp, performance.now()); frame.close(); },
    error: e => {
      stats.errors.push(String(e.message || e));
      needKey = true;
      makeDecoder();
      askKeyFrame();
    },
  });
}
function onH264(isKey, seq, au) {
  if (isKey) {
    const cs = spsCodec(au);
    if (cs && (cs !== codecString || dec.state !== "configured")) {
      if (dec.state === "closed") makeDecoder();
      dec.configure({ codec: cs, optimizeForLatency: true, hardwareAcceleration: "no-preference" });
      codecString = stats.codec = cs;
    }
    stats.keyFrames++;
  }
  if (dec.state !== "configured" || (needKey && !isKey)) { dropped(); askKeyFrame(); return; }
  // Far behind: skip to the next keyframe rather than show old pictures late.
  // A few queued frames are fine: decoding catches up faster than a keyframe
  // crosses a slow link, and only the newest decoded frame gets drawn.
  if (!isKey && dec.decodeQueueSize > 10) { needKey = true; dropped(); askKeyFrame(); return; }
  needKey = false;
  dec.decode(new EncodedVideoChunk({ type: isKey ? "key" : "delta", timestamp: seq, data: au }));
}

// ---- JPEG ----
let jpegBusy = false;
function onJPEG(seq, data) {
  if (jpegBusy) { dropped(); return; }
  jpegBusy = true;
  createImageBitmap(new Blob([data], { type: "image/jpeg" })).then(bmp => {
    drawFrame(bmp, bmp.width, bmp.height, seq, performance.now());
    bmp.close();
  }).catch(e => stats.errors.push(String(e))).finally(() => { jpegBusy = false; });
}

async function pickCodec() {
  const want = q.get("codec");
  if (want === "jpeg") return "jpeg";
  if (!("VideoDecoder" in window)) return "jpeg";
  try {
    const r = await VideoDecoder.isConfigSupported({ codec: "avc1.640028", optimizeForLatency: true });
    if (!r.supported) return "jpeg";
    // Whether this browser has a hardware H.264 decoder (for the numbers).
    const hw = await VideoDecoder.isConfigSupported({ codec: "avc1.640028", hardwareAcceleration: "prefer-hardware" })
      .catch(() => ({ supported: false }));
    decoderInfo = hw.supported ? "h264 hardware" : "h264 software";
    return "h264";
  } catch (e) { return "jpeg"; }
}

async function connect() {
  const codec = await pickCodec();
  stats.codec = codec;
  if (codec === "h264") makeDecoder();
  needKey = true;
  const p = new URLSearchParams({ src, codec, max: q.get("max") || "1920", fps: q.get("fps") || "60" });
  if (reconnectKey) p.set("r", reconnectKey); else p.set("t", ticket);
  if (q.get("bpp")) p.set("bpp", q.get("bpp"));
  ws = new WebSocket(`ws://${location.host}/stream?${p}`);
  ws.binaryType = "arraybuffer";
  ws.onopen = () => { retry = 0; lastError = ""; startPings(); };
  ws.onmessage = ev => {
    if (typeof ev.data === "string") return control(JSON.parse(ev.data));
    const now = performance.now(), b = new Uint8Array(ev.data);
    stats.bytes += b.length;
    if (b.length < 18) return;
    // flags (1 = keyframe), pts µs, sequence number, input echoed; big-endian.
    const v = new DataView(ev.data), isKey = (b[0] & 1) === 1, seq = v.getUint32(9);
    send({ t: "rx", s: seq, r: toMac(now) });
    const payload = b.subarray(17);
    if (codec === "h264") onH264(isKey, seq, payload); else onJPEG(seq, payload);
  };
  ws.onclose = () => {
    ws = null;
    clearInterval(pingTimer);
    if (closing) return;
    // Refused before ever getting a key: the ticket expired or was revoked,
    // and retrying can't help.
    if (!reconnectKey && ++failedStarts >= 3) {
      show("This view has expired. Press Show in Frame Control on the Mac to open it again.");
      return;
    }
    hideNoteOnFrame = true;
    // The Mac may be asleep or the tunnel restarting: keep trying.
    retry = Math.min(retry + 1, 6);
    // Keep the Mac's reason (a missing permission, a closed window) on screen.
    show(lastError ? `${lastError} Retrying…` : "Lost the Mac. Reconnecting…");
    setTimeout(connect, 500 * 2 ** retry);
  };
}

function control(m) {
  if (m.t === "pong") return onPong(m);
  if (m.t === "stats") return onStats(m);
  if (m.t === "bench") return bench(m);
  if (m.t === "hello") {
    reconnectKey = m.r;
    send({ t: "ack" });  // the ticket is spent only now
  } else if (m.t === "info") {
    stats.info = m;
    warmMs = Math.max(0, +m.warm || 0);
    const name = m.title && m.app && m.title !== m.app ? `${m.title} — ${m.app}` : (m.title || m.app || "Mac");
    document.title = tag ? `${name} [${tag}]` : name;
    if (!m.input) {
      hideNoteOnFrame = false;  // a warning, not a connection notice: let it stay its 8 s
      show("Clicks and keys need Accessibility permission on the Mac (Frame Control asks for it).", 8000);
    }
  } else if (m.t === "error") {
    stats.errors.push(m.message);
    lastError = m.message.replace(/\.?$/, ".");
    show(m.message);
  } else if (m.t === "close" || m.t === "closed") {
    closing = true;
    if (ws) ws.close();
    show(m.reason ? `Stopped: ${m.reason}.` : "Stopped.");
    window.close();
  }
}

// ---- input ----
// Where the picture sits inside the window (object-fit: contain letterboxes it).
function toPicture(e) {
  const r = c.getBoundingClientRect(), s = Math.min(r.width / c.width, r.height / c.height);
  const w = c.width * s, h = c.height * s, left = r.left + (r.width - w) / 2, top = r.top + (r.height - h) / 2;
  const x = (e.clientX - left) / w, y = (e.clientY - top) / h;
  return { x, y, inside: x >= 0 && x <= 1 && y >= 0 && y <= 1 };
}
// Discrete input carries an id and when it happened (the Mac's clock), so
// the Mac can tag the first frame that could show its effect.
let inputId = 0;
// Keep the Frame's Wi-Fi awake for a few seconds after input, when the agent asks.
let warmMs = 0, warmUntil = 0, warmTimer = 0;
function keepWarm() {
  if (!warmMs) return;
  warmUntil = performance.now() + 5000;
  if (warmTimer) return;
  warmTimer = setInterval(() => {
    if (performance.now() > warmUntil) { clearInterval(warmTimer); warmTimer = 0; return; }
    send({ t: "w" });
  }, warmMs);
}
function stamp(m, e) {
  keepWarm();
  m.i = ++inputId;
  m.tv = toMac(e && e.timeStamp ? e.timeStamp : performance.now());
  return m;
}
// Moves go at most every 8 ms, on a timer rather than the next animation frame:
// the Frame throttles a panel's animation frames to 15-36 Hz when it thinks
// nobody is looking, which would hold a move back by up to 60 ms.
let pendingMove = null, moveQueued = false, lastMove = 0;
function flushMove() {
  moveQueued = false;
  if (pendingMove) { send(pendingMove); lastMove = performance.now(); }
  pendingMove = null;
}
addEventListener("pointermove", e => {
  const p = toPicture(e);
  if (!p.inside && !e.buttons) return;
  pendingMove = { t: "m", e: "move", x: p.x, y: p.y };
  if (moveQueued) return;
  const wait = lastMove + 8 - performance.now();
  if (wait <= 0) return flushMove();
  moveQueued = true;
  setTimeout(flushMove, wait);
});
addEventListener("pointerdown", e => {
  const p = toPicture(e);
  if (!p.inside) return;
  if (e.pointerId !== undefined) try { c.setPointerCapture(e.pointerId); } catch (err) {}
  pendingMove = null;
  send(stamp({ t: "m", e: "down", b: e.button < 0 ? 0 : e.button, x: p.x, y: p.y }, e));
  e.preventDefault();
});
addEventListener("pointerup", e => {
  const p = toPicture(e);
  send({ t: "m", e: "up", b: e.button < 0 ? 0 : e.button, x: p.x, y: p.y });
});
// Let go of any held button where the pointer is on the Mac, not at a corner.
addEventListener("pointercancel", () => send({ t: "release" }));
addEventListener("contextmenu", e => e.preventDefault());
addEventListener("wheel", e => {
  const p = toPicture(e), unit = e.deltaMode === 1 ? 16 : e.deltaMode === 2 ? innerHeight : 1;
  send(stamp({ t: "wheel", dx: e.deltaX * unit, dy: e.deltaY * unit, x: p.x, y: p.y }, e));
  e.preventDefault();
}, { passive: false });
function mods(e) {
  return ["shift", "ctrl", "alt", "meta"].filter(m => e[m + "Key"]);
}
function onKey(e, down) {
  // Ctrl+Alt+Shift+S shows the numbers; it isn't sent to the Mac.
  if (e.code === "KeyS" && e.ctrlKey && e.altKey && e.shiftKey) {
    if (down) showHud(hud.hidden);
    return e.preventDefault();
  }
  const m = { t: "k", e: down ? "down" : "up", code: e.code, key: e.key, mods: mods(e) };
  send(down ? stamp(m, e) : m);
  e.preventDefault();
}

// ---- benchmarks: the Mac asks the viewer to act as if someone did ----
function keyCode(ch) {
  if (/[a-z]/i.test(ch)) return "Key" + ch.toUpperCase();
  if (/[0-9]/.test(ch)) return "Digit" + ch;
  return { " ": "Space", "\n": "Enter", ".": "Period", ",": "Comma" }[ch] || "";
}
function bench(m) {
  if (m.action === "overlay") return showHud(m.on !== 0 && m.on !== "0");
  if (m.action === "click") {
    const at = { x: +m.x || 0.5, y: +m.y || 0.5 };
    send(stamp({ t: "m", e: "down", b: 0, ...at }));
    setTimeout(() => send({ t: "m", e: "up", b: 0, ...at }), 40);
  } else if (m.action === "type") {
    const text = String(m.text || ""), gap = +m.interval || 150;
    [...text].forEach((ch, n) => setTimeout(() => {
      const code = keyCode(ch);
      if (!code) return send(stamp({ t: "text", s: ch }));
      const mods = ch !== ch.toLowerCase() ? ["shift"] : [];
      send(stamp({ t: "k", e: "down", code, key: ch, mods }));
      setTimeout(() => send({ t: "k", e: "up", code, key: ch, mods }), 30);
    }, n * gap));
  }
}
addEventListener("keydown", e => onKey(e, true));
addEventListener("keyup", e => onKey(e, false));
addEventListener("blur", () => send({ t: "release" }));

show("Connecting to the Mac…");
connect();
</script>
</body>
</html>