From 83d74548cd486000b4b35a9818820f15e4159117 Mon Sep 17 00:00:00 2001 From: saphid <4596216+saphid@users.noreply.github.com> Date: Tue, 29 Sep 2026 15:08:36 +1000 Subject: [PATCH 1/9] Screenshots: Copy button, right-click menu, and new shots appear on their own Each screenshot card and the viewer get a Copy button that puts the image on the clipboard (natively in the desktop app, as PNG in a browser). Right-click a screenshot to open, copy, save, show it in Finder, or copy its path or name; right-click the viewer to copy or save. The shelf re-lists the Frame's screenshots every 8 s while the window is visible and connected, redraws only when something changed, and keeps thumbnails it already has. Switching headsets clears the list and ignores answers still on their way. Co-Authored-By: Claude Opus 5.5 (1M context) --- app/main.js | 10 ++- app/preload.js | 5 +- tests/test_server.py | 6 ++ ui/frame_host.py | 13 +++ ui/index.html | 199 +++++++++++++++++++++++++++++++++++++------ ui/server.py | 6 ++ 6 files changed, 208 insertions(+), 31 deletions(-) diff --git a/app/main.js b/app/main.js index 4497136..3c07d2b 100644 --- a/app/main.js +++ b/app/main.js @@ -1,7 +1,7 @@ // Frame Control as a desktop app (macOS, Windows, Linux): starts ui/server.py on // a free loopback port and shows it in a native window. The server does all the // work over the `frame` SSH alias; this file only hosts it. -const { app, BrowserWindow, Menu, Notification, clipboard, dialog, ipcMain, shell } = require("electron"); +const { app, BrowserWindow, Menu, Notification, clipboard, dialog, ipcMain, nativeImage, shell } = require("electron"); const { execFile, spawn } = require("child_process"); const { promisify } = require("util"); const fs = require("fs"); @@ -264,6 +264,14 @@ function fromUi(e) { } ipcMain.handle("clipboard:read", (e) => fromUi(e) ? clipboard.readText() : ""); +// A PNG or JPEG (a screenshot) onto the clipboard as an image. +ipcMain.handle("clipboard:writeImage", (e, bytes) => { + if (!fromUi(e) || !(bytes instanceof Uint8Array)) return false; + const img = nativeImage.createFromBuffer(Buffer.from(bytes)); + if (img.isEmpty()) throw new Error("not an image"); + clipboard.writeImage(img); + return true; +}); ipcMain.handle("connection:setup", (e) => { if (fromUi(e)) setUpConnection(); }); ipcMain.on("keys:capture", (e, on) => { if (fromUi(e)) win.webContents.setIgnoreMenuShortcuts(on === true); }); ipcMain.handle("update:get", (e) => fromUi(e) ? publicUpdate() : null); diff --git a/app/preload.js b/app/preload.js index 8ab5b8f..fb86f63 100644 --- a/app/preload.js +++ b/app/preload.js @@ -2,8 +2,8 @@ // to the Frame needs no pbpaste, PowerShell, xclip or wl-clipboard. Also tells // the page where a dropped file or folder lives, so a folder can be sideloaded // as a title without zipping it (the local server reads it from there). -// It can open Set Up Connection when the headset can't be reached, and keeps the -// Frame menu's list of headsets up to date. +// It can put a screenshot on the clipboard as an image, open Set Up Connection when +// the headset can't be reached, and keeps the Frame menu's list of headsets up to date. // It also receives frame-control://install links (docs/web-install.md): only // what the link asked for, never an install; the page asks the user first. // And it passes update state both ways: see app/updater.js. @@ -12,6 +12,7 @@ const { contextBridge, ipcRenderer, webUtils } = require("electron"); contextBridge.exposeInMainWorld("frameApp", { notify: (message, request) => ipcRenderer.invoke("comfort:notify", message, request), readClipboard: () => ipcRenderer.invoke("clipboard:read"), + writeImage: (bytes) => ipcRenderer.invoke("clipboard:writeImage", bytes), setUpConnection: () => ipcRenderer.invoke("connection:setup"), // The Frame menu's headset switcher: the page tells it the headsets, and hears picks. devicesChanged: (list) => ipcRenderer.send("devices:changed", list), diff --git a/tests/test_server.py b/tests/test_server.py index 9805806..99dc85a 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -111,6 +111,8 @@ class ServerGuards(unittest.TestCase): ("/api/volume", {"level": 1.5}), ("/api/clipboard", {"text": ""}), ("/api/open", {"what": "anything-else"}), + ("/api/open", {"what": "shot", "id": "1/250820/../../.ssh/id_ed25519"}), + ("/api/open", {"what": "shot"}), ("/api/shots/save", {"ids": []}), ("/api/shots/save", {"ids": "1/250820/20260925225208_1.jpg"}), ("/api/shots/save", {"ids": [1]}), @@ -121,6 +123,10 @@ class ServerGuards(unittest.TestCase): status, payload = self.post(path, body) self.assertEqual(status, 400, f"{path} {body} -> {payload}") + def test_showing_a_shot_needs_it_saved_here(self): + status, payload = self.post("/api/open", {"what": "shot", "id": "1/250820/19990101000000_1.jpg"}) + self.assertEqual(status, 404, payload) + def test_screenshot_ids_checked_before_ssh(self): for shot in ("../../etc/passwd", "1/250820/x.jpg", "1/2/20260925225208_1.jpg;id", "1/250820/20260925225208_1.gif"): status, _, _ = self.request("GET", f"/api/shots/image?id={quote(shot)}", headers={"X-Frame-UI": "1"}) diff --git a/ui/frame_host.py b/ui/frame_host.py index e5c1914..beb5f8c 100644 --- a/ui/frame_host.py +++ b/ui/frame_host.py @@ -153,6 +153,19 @@ def open_path(path): stderr=subprocess.DEVNULL, **DETACHED) +def reveal_path(path): + """Show a file selected in its folder (Linux file managers vary, so there the folder opens).""" + path = Path(path) + if MAC: + cmd = ["open", "-R", str(path)] + elif WINDOWS: + cmd = ["explorer", f"/select,{path}"] + else: + return open_path(path.parent) + subprocess.Popen(cmd, stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, **DETACHED) + + open_url = open_path # the same openers hand URLs to the default browser diff --git a/ui/index.html b/ui/index.html index 0d7a686..1c19901 100644 --- a/ui/index.html +++ b/ui/index.html @@ -258,7 +258,12 @@ .shot-card .row { flex-wrap: nowrap; } .shot-card .grow { flex: 1; min-width: 0; } .shot-card .t { color: var(--bright); font-size: 13px; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } - .shot-card .s { color: var(--muted); font-size: 12px; } + .shot-card .s { color: var(--muted); font-size: 12px; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } + .ctx-menu { position: fixed; z-index: 1000; min-width: 200px; padding: 4px; border-radius: 4px; background: #232c38; + box-shadow: 0 10px 28px rgba(0,0,0,.6), 0 0 0 1px rgba(255,255,255,.08); } + .ctx-menu button { display: block; width: 100%; height: 30px; padding: 0 10px; text-align: left; background: none; } + .ctx-menu button:hover, .ctx-menu button:focus-visible { background: var(--blue); color: #fff; outline: none; } + .ctx-menu hr { border: 0; border-top: 1px solid rgba(255,255,255,.1); margin: 4px 2px; } /* ---- library shelf (portrait capsules, like Steam's library home) ---- */ .shelf { display: grid; grid-template-columns: repeat(auto-fill, minmax(150px, 1fr)); gap: 16px; } @@ -701,6 +706,7 @@ + @@ -867,7 +873,7 @@
Loading…
-
Screenshots you take in the headset with Steam's screenshot shortcut. Click one to open it in the viewer; Save copies it to ~/Pictures/SteamFrame.
+
Screenshots you take in the headset with Steam's screenshot shortcut. New ones appear on their own. Click one to open it in the viewer, Copy puts it on the clipboard, and Save copies it to ~/Pictures/SteamFrame. Right-click for more.
@@ -1869,7 +1875,7 @@ async function capture() { if (source === "panel") $("srcBadge").textContent = `Desktop · ${shotPanel?.name || "panel"}`; ctrlShow(); $("stamp").hidden = false; $("stamp").textContent = new Date().toLocaleTimeString(); - $("saveBtn").disabled = false; + $("saveBtn").disabled = $("copyBtn").disabled = false; return true; } catch (e) { log("Capture failed: " + e.message, "e"); @@ -1951,7 +1957,7 @@ async function startVideo() { $("viewer").classList.remove("busy"); c.hidden = false; $("viewerEmpty").hidden = true; $("zoombar").hidden = false; $("asleep").hidden = true; $("srcBadge").hidden = false; $("srcBadge").textContent = `${video.label} · video`; - $("stamp").hidden = false; $("saveBtn").disabled = false; + $("stamp").hidden = false; $("saveBtn").disabled = $("copyBtn").disabled = false; } frames++; const now = performance.now(); @@ -2020,6 +2026,35 @@ $("saveBtn").onclick = () => lastShot ? download(lastShot.blob, lastShot.file) : if (!b) return toast("Couldn't encode the image", true); download(b, `frame-${view}-${new Date().toISOString().replace(/[:.]/g, "-")}.png`); }, "image/png"); +const copyViewer = () => act("Copy image", () => copyImage(lastShot ? lastShot.blob + : new Promise((ok, bad) => $("canvas").toBlob(b => b ? ok(b) : bad(new Error("couldn't encode the image")), "image/png")))); +$("copyBtn").onclick = copyViewer; +// Right-click the viewer to copy or save what it shows. +$("canvas").oncontextmenu = e => { + if ($("saveBtn").disabled) return; + e.preventDefault(); + showMenu(e, [["Copy image", copyViewer], ["Save image", () => $("saveBtn").click()]]); +}; +// An image onto the clipboard. The app does it natively (JPEG too); a browser +// takes PNG only, and the blob is handed over as a promise so the click still counts. +async function copyImage(blob) { + if (window.frameApp?.writeImage) { + await window.frameApp.writeImage(new Uint8Array(await (await blob).arrayBuffer())); + return { message: "Copied the image" }; + } + if (!navigator.clipboard?.write || !window.ClipboardItem) throw new Error("This browser can't copy images here"); + await navigator.clipboard.write([new ClipboardItem({ "image/png": Promise.resolve(blob).then(pngBlob) })]); + return { message: "Copied the image" }; +} +async function pngBlob(blob) { + if (blob.type === "image/png") return blob; + const bmp = await createImageBitmap(blob); + const c = document.createElement("canvas"); + c.width = bmp.width; c.height = bmp.height; + c.getContext("2d").drawImage(bmp, 0, 0); + bmp.close(); + return new Promise((ok, bad) => c.toBlob(b => b ? ok(b) : bad(new Error("couldn't encode the image")), "image/png")); +} function download(blob, name) { if (savesToDevice()) return act("Save image", () => saveToDevice([{ blob, name }])); const a = document.createElement("a"); @@ -3497,8 +3532,11 @@ loadReports(); api("/api/host").then(applyHostWording).catch(() => {}); // ---- Steam screenshots from the headset ---- -const shots = { list: [], urls: [] }; +// thumbs: id -> promise of an object URL, kept across reloads so a refresh +// that finds a new shot fetches only that one's thumbnail. +const shots = { list: [], sig: null, thumbs: new Map(), loading: null, gen: 0 }; const STEAMVR_APPID = "250820"; +const SHOTS_POLL_MS = 8000; function shotApp(appid) { if (appid === STEAMVR_APPID) return "SteamVR"; const g = state?.games?.find(x => x.appid === appid); @@ -3510,35 +3548,71 @@ async function shotBlob(id, thumb) { if (!r.ok) throw new Error((await r.json().catch(() => ({}))).error || `HTTP ${r.status}`); return r.blob(); } -async function loadShots() { - $("shotsRefresh").disabled = true; +function shotThumb(id) { + if (!shots.thumbs.has(id)) { + const p = shotBlob(id, true).then(b => URL.createObjectURL(b)); + p.catch(() => shots.thumbs.delete(id)); // try again next time + shots.thumbs.set(id, p); + } + return shots.thumbs.get(id); +} +// quiet: a background check. It keeps what's shown if the Frame can't be read, +// and redraws only when the shots (or whether they're saved here) changed. +function loadShots(quiet) { + if (!shots.loading) { + const p = readShots(quiet === true).finally(() => { if (shots.loading === p) shots.loading = null; }); + shots.loading = p; + } + return shots.loading; +} +// Forget the shots of a headset we've switched away from, and ignore its answers still on their way. +function resetShots() { + shots.gen++; shots.loading = null; shots.list = []; shots.sig = null; + for (const p of shots.thumbs.values()) p.then(URL.revokeObjectURL, () => {}); + shots.thumbs = new Map(); +} +async function readShots(quiet) { + const gen = shots.gen; + if (!quiet) $("shotsRefresh").disabled = true; + let got; try { - shots.list = (await api("/api/shots")).shots; + got = await api("/api/shots"); } catch (e) { - return failed($("shotGrid"), e); + if (!quiet && gen === shots.gen) { shots.sig = null; failed($("shotGrid"), e); } + return; } finally { $("shotsRefresh").disabled = false; } - shots.urls.forEach(URL.revokeObjectURL); shots.urls = []; + if (gen !== shots.gen) return; + ({ shots: shots.list, folder: shots.folder } = got); + const sig = JSON.stringify(shots.list.map(s => [s.id, s.saved, shotApp(s.appid)])); + if (sig !== shots.sig) { shots.sig = sig; drawShots(); } + // Thumbnails one at a time over the shared SSH connection; one that failed is tried again next time. + for (const img of document.querySelectorAll("#shotGrid img[data-shot]:not([src])")) { + const s = shots.list[+img.dataset.shot]; + try { img.src = await shotThumb(s.id); } catch (e) { img.alt = "Preview failed"; } + if (!img.isConnected || gen !== shots.gen) return; // redrawn, or another headset, meanwhile + } +} +// "Sep 28, 10:37 PM": short enough to sit beside the card's buttons (the full date is its tooltip). +function shotTime(t) { + return new Date(t * 1000).toLocaleString([], { month: "short", day: "numeric", hour: "numeric", minute: "2-digit" }); +} +function drawShots() { + const ids = new Set(shots.list.map(s => s.id)); + for (const [id, p] of shots.thumbs) { + if (!ids.has(id)) { shots.thumbs.delete(id); p.then(URL.revokeObjectURL, () => {}); } + } const unsaved = shots.list.filter(s => !s.saved).length; $("shotCount").textContent = shots.list.length ? `${shots.list.length} on the Frame` + (unsaved && !HOST.mobile ? ` · ${unsaved} not on this ${HOST.computer}` : "") : ""; $("shotsSaveNew").disabled = HOST.mobile ? !shots.list.length : !unsaved; - $("shotGrid").innerHTML = shots.list.length ? shots.list.map((s, i) => `
- Screenshot from ${esc(shotApp(s.appid))} + $("shotGrid").innerHTML = shots.list.length ? shots.list.map((s, i) => `
+ Screenshot from ${esc(shotApp(s.appid))}
${esc(shotApp(s.appid))}
-
${esc(new Date(s.time * 1000).toLocaleString())}
+
${esc(shotTime(s.time))}
+ ${s.saved && !HOST.mobile ? `On ${HOST.computer}` : ``}
`).join("") : `
No screenshots on the Frame yet.
`; - // Thumbnails one at a time over the shared SSH connection. - for (const img of document.querySelectorAll("#shotGrid img[data-shot]")) { - const s = shots.list[+img.dataset.shot]; - try { - const url = URL.createObjectURL(await shotBlob(s.id, true)); - shots.urls.push(url); - img.src = url; - } catch (e) { img.alt = "Preview failed"; } - if (!img.isConnected) return; // the list was reloaded meanwhile - } } async function openShot(s) { if (live) toggleLive(false); @@ -3556,7 +3630,7 @@ async function openShot(s) { draw(); $("srcBadge").textContent = `Screenshot · ${shotApp(s.appid)}`; $("stamp").hidden = false; $("stamp").textContent = new Date(s.time * 1000).toLocaleString(); - $("saveBtn").disabled = false; + $("saveBtn").disabled = $("copyBtn").disabled = false; $("view").scrollIntoView({ behavior: "smooth" }); } catch (e) { toast("Couldn't open the screenshot: " + e.message, true); @@ -3575,15 +3649,84 @@ async function saveShots(list, btn) { () => api("/api/shots/save", { ids: list.map(s => s.id) }), btn); if (res) loadShots(); } +const copyShot = (s, btn) => act("Copy screenshot", () => copyImage(shotBlob(s.id, false)), btn); $("shotGrid").onclick = e => { const img = e.target.closest("img[data-shot]"); if (img) return openShot(shots.list[+img.dataset.shot]); + const c = e.target.closest("[data-shot-copy]"); + if (c) return copyShot(shots.list[+c.dataset.shotCopy], c); const b = e.target.closest("[data-shot-save]"); if (b) saveShots([shots.list[+b.dataset.shotSave]], b); }; -$("shotsRefresh").onclick = loadShots; +// A small right-click menu. items: [label, fn] pairs, null for a divider. +function closeMenu() { document.querySelector(".ctx-menu")?.remove(); } +function showMenu(e, items) { + closeMenu(); + const menu = document.createElement("div"); + menu.className = "ctx-menu"; + menu.setAttribute("role", "menu"); + for (const it of items) { + if (!it) { menu.append(document.createElement("hr")); continue; } + const b = document.createElement("button"); + b.textContent = it[0]; + b.setAttribute("role", "menuitem"); + b.onclick = () => { closeMenu(); it[1](); }; + menu.append(b); + } + (document.fullscreenElement || document.body).append(menu); // in front of a fullscreen viewer + const r = menu.getBoundingClientRect(); + menu.style.left = Math.max(4, Math.min(e.clientX, innerWidth - r.width - 4)) + "px"; + menu.style.top = Math.max(4, Math.min(e.clientY, innerHeight - r.height - 4)) + "px"; + menu.querySelector("button").focus(); +} +document.addEventListener("pointerdown", e => { if (!e.target.closest(".ctx-menu")) closeMenu(); }, true); +document.addEventListener("keydown", e => { + const menu = document.querySelector(".ctx-menu"); + if (!menu) return; + if (e.key === "Escape") return closeMenu(); + if (e.key !== "ArrowDown" && e.key !== "ArrowUp") return; + e.preventDefault(); // arrows move through the menu, not the page + const items = [...menu.querySelectorAll("button")]; + const at = items.indexOf(document.activeElement), step = e.key === "ArrowDown" ? 1 : -1; + items[at < 0 ? (step > 0 ? 0 : items.length - 1) : (at + step + items.length) % items.length].focus(); +}); +addEventListener("blur", closeMenu); +document.addEventListener("fullscreenchange", closeMenu); +addEventListener("scroll", closeMenu, true); +// Right-click a screenshot for everything it can do. +$("shotGrid").oncontextmenu = e => { + const card = e.target.closest("[data-card]"); + if (!card) return; + e.preventDefault(); + const s = shots.list[+card.dataset.card]; + const items = [ + ["Open in viewer", () => openShot(s)], + ["Copy image", () => copyShot(s)], + ]; + if (!s.saved || HOST.mobile) items.push(null, [`Save to ${HOST.computer}`, () => saveShots([s])]); + else { + const sep = shots.folder.includes("\\") ? "\\" : "/"; + items.push(null); + items.push([`Show in ${!HOST.fileManager || HOST.fileManager === "your file manager" ? "folder" : HOST.fileManager}`, + () => act("Show screenshot", () => api("/api/open", { what: "shot", id: s.id }))]); + items.push(["Copy file path", () => act("Copy file path", async () => { + await navigator.clipboard.writeText(shots.folder + sep + s.file); + return { message: "Copied the file path" }; + })]); + } + items.push(null, ["Copy file name", () => act("Copy file name", async () => { + await navigator.clipboard.writeText(s.file); + return { message: "Copied " + s.file }; + })]); + showMenu(e, items); +}; +$("shotsRefresh").onclick = () => loadShots(); $("shotsSaveNew").onclick = e => saveShots(shots.list.filter(s => !s.saved), e.currentTarget); $("shotsFolder").onclick = e => act($("shotsFolder").textContent, () => api("/api/open", { what: "shots" }), e.currentTarget); +// Watch for new shots: a cheap listing over the shared SSH connection while the +// window is visible and the Frame is reachable, and again on coming back to it. +setInterval(() => { if (!document.hidden && online) loadShots(true); }, SHOTS_POLL_MS); +document.addEventListener("visibilitychange", () => { if (!document.hidden && online) loadShots(true); }); // ---- Panel switcher: our UI over SteamVR's panel API ---- let panelSeq = 0; @@ -4177,7 +4320,7 @@ if (window.frameApp && window.frameApp.onInstallLink) { } setView("headset"); -refresh().then(loadShots); // after status, so app names resolve +refresh().then(() => loadShots()); // after status, so app names resolve document.addEventListener("visibilitychange", () => { if (!document.hidden && online === false) refresh(); });