// frame-mac-view: streams Mac windows or displays to viewers on the Steam // Frame and plays their pointer and key input back on the Mac. // // frame-mac-view serve --port 47811 --page ui/mac-view.html (token in FRAME_MAC_VIEW_TOKEN) // frame-mac-view windows | displays | permissions (JSON on stdout) // frame-mac-view request-permissions (shows macOS's prompts) // // It listens on 127.0.0.1 only. Frame Control reaches it from the Frame // through an SSH reverse tunnel, and every request must carry the token. // // HTTP. Frame Control's key (?k=, from FRAME_MAC_VIEW_TOKEN) never leaves the // Mac; the Frame gets a single-use ticket per viewer instead. // GET /ping, /view open: tunnel check, viewer page // GET /stream?src=...&t=TICKET|r=KEY WebSocket (&codec=h264|jpeg&max=&fps=&bpp=) // GET /status, /windows, /displays ?k=: permissions, streams, sources // POST /ticket?src=... ?k=: a ticket for one viewer of src // POST /close[?src=...] ?k=: end those streams, close their windows // POST /permissions ?k=: show macOS's permission prompts // GET /stats[?id=&since=] ?k=: per-frame timing records (for benchmarks) // POST /bench?src=&action=... ?k=: ask viewers of src to type, click or show their overlay // src is window:, display: or test. import AppKit import ApplicationServices import Foundation import IOKit.pwr_mgt import ScreenCaptureKit import Security let version = "1" func windowsJSON() -> [[String: Any]] { let me = ProcessInfo.processInfo.processIdentifier let skip: Set = ["Window Server", "Dock", "Control Centre", "Control Center", "Notification Centre", "Notification Center", "Spotlight", "SystemUIServer", "Wallpaper", "WindowManager"] return WindowInfo.all().filter { $0.layer == 0 && $0.pid != me && !skip.contains($0.app) && $0.bounds.width >= 120 && $0.bounds.height >= 80 }.map { ["id": $0.id, "src": "window:\($0.id)", "pid": $0.pid, "app": $0.app, "title": $0.title, "w": Int($0.bounds.width), "h": Int($0.bounds.height)] } } func displaysJSON() -> [[String: Any]] { var ids = [CGDirectDisplayID](repeating: 0, count: 16) var n: UInt32 = 0 // Online, not active: a display that's asleep is still one you can stream. CGGetOnlineDisplayList(16, &ids, &n) return ids.prefix(Int(n)).filter { CGDisplayMirrorsDisplay($0) == kCGNullDirectDisplay && !VirtualDisplay.isOurs($0) // not a separated window's }.map { id in let b = CGDisplayBounds(id) return ["id": id, "src": "display:\(id)", "name": displayName(id), "w": Int(b.width), "h": Int(b.height), "main": CGDisplayIsMain(id) != 0] } } func permissionsJSON() -> [String: Any] { ["screen": CGPreflightScreenCaptureAccess(), "accessibility": AXIsProcessTrusted()] } func printJSON(_ obj: Any) { let d = (try? JSONSerialization.data(withJSONObject: obj, options: [.sortedKeys, .prettyPrinted])) ?? Data() FileHandle.standardOutput.write(d + Data("\n".utf8)) } /// A number from a JSON message, whatever its JSON type. func num(_ v: Any?) -> Double? { (v as? NSNumber)?.doubleValue } /// One viewer watching one source. final class Session { let id: Int let source: Source let capture: CaptureSource let encoder: Encoder let ws: WebSocket let codec: Codec let reconnectKey: String let stats = StreamStats() let controller: RateController private var appliedScale = 1.0 private var lastSubmit: Int64 = 0 private var paceScheduled = false private let lock = NSLock() /// A captured picture waiting to be encoded (or the last one encoded). private struct Picture { let pb: CVPixelBuffer let pts: CMTime let capture: Int64 let arrived: Int64 var echo: UInt32 // the input this picture is the first reply to } private var last: Picture? private var skipped = false private var stopped = false private var inFlight = 0 /// Input posted to the Mac whose effect hasn't been captured yet: the /// next picture composited after `after` is tagged with it. private var pendingEcho: (id: UInt32, after: Int64)? private var statsTimer: DispatchSourceTimer? /// Frames already queued for the network; beyond this, or with two frames /// already in the encoder, new frames are skipped (before encoding, so no /// reference frame goes missing) and the newest picture is sent once /// things catch up. Only that newest picture is kept meanwhile. let maxPending: Int static let maxInFlight = 2 var onEnd: ((Session) -> Void)? var onAck: (() -> Void)? var onFinished: ((String) -> Void)? private let encodeQueue = DispatchQueue(label: "frame-mac-view.encode", qos: .userInteractive) init(id: Int, source: Source, capture: CaptureSource, ws: WebSocket, codec: Codec, fps: Int, bitsPerPixel: Double, reconnectKey: String) { self.id = id self.source = source self.capture = capture self.ws = ws self.codec = codec self.reconnectKey = reconnectKey encoder = Encoder(codec: codec, fps: fps, bitsPerPixel: bitsPerPixel) controller = RateController(maxFps: fps) maxPending = codec == .jpeg ? 3 << 20 : 1 << 20 } /// Frames come faster than the current tier's frame rate: this one waits, /// and goes when its turn comes (unless a newer one replaces it). /// Call with `lock` held. private func paced(now: Int64) -> Bool { let fps = controller.fps guard fps < controller.maxFps, lastSubmit > 0 else { return false } let interval = Int64(1_000_000 / fps), due = lastSubmit + interval * 9 / 10 guard now < due else { return false } if !paceScheduled { paceScheduled = true encodeQueue.asyncAfter(deadline: .now() + .microseconds(Int(due - now))) { [weak self] in guard let self else { return } self.lock.lock(); self.paceScheduled = false; self.lock.unlock() self.resendIfRoom() } } return true } /// Encodes `pb` unless the link or the encoder is busy, in which case it /// becomes the picture to send next. private func offer(_ pb: CVPixelBuffer, pts: CMTime, capture shown: Int64) { let arrived = nowUs() stats.withLock { stats.captured += 1 } controller.captured(at: arrived) lock.lock() // A reply to input stays with whichever picture ends up being sent. var echo = skipped ? last?.echo ?? 0 : 0 if let p = pendingEcho, shown >= p.after { echo = p.id pendingEcho = nil } let picture = Picture(pb: pb, pts: pts, capture: shown, arrived: arrived, echo: echo) last = picture let busy = stopped || ws.pendingBytes > maxPending || inFlight >= Session.maxInFlight || paced(now: arrived) || !controller.maySend(now: arrived) if busy { skipped = true } else { inFlight += 1; last?.echo = 0; lastSubmit = arrived } lock.unlock() if busy { stats.withLock { stats.skipped += 1 } } else { submit(picture) } } private func submit(_ p: Picture) { encodeQueue.async { var r = FrameRecord() r.capture = p.capture r.arrived = p.arrived r.echo = p.echo r.bitrate = self.encoder.bitrate r.tier = self.controller.tier r.encodeStart = nowUs() let seq = self.stats.newFrame(r) if p.echo != 0 { self.stats.updateInput(p.echo) { $0.frame = seq; $0.capture = p.capture } } if !self.encoder.encode(p.pb, pts: p.pts, seq: seq) { self.finished() } } } /// An encode finished (or failed): send the newest skipped picture if /// there's room now. private func finished() { lock.lock() inFlight = max(0, inFlight - 1) lock.unlock() resendIfRoom() } private func resendIfRoom() { lock.lock() var next: Picture? let now = nowUs() if !stopped, skipped, ws.pendingBytes <= maxPending / 2, inFlight < Session.maxInFlight, let l = last, !paced(now: now), controller.maySend(now: now, counts: false) { next = l skipped = false inFlight += 1 last?.echo = 0 lastSubmit = now } lock.unlock() // Stamped now: VideoToolbox needs increasing times, and the capture // time stays in the record, so the wait counts as latency. if let p = next { submit(Picture(pb: p.pb, pts: CMClockGetTime(CMClockGetHostTimeClock()), capture: p.capture, arrived: p.arrived, echo: p.echo)) } } func start(maxLong: Int, fps: Int) { encoder.onFrame = { [weak self] data, key, pts, seq in guard let self else { return } let t = nowUs() // The quality setting's bitrate, at full size, is the most it gets. if self.appliedScale == 1 { self.controller.setCeiling(self.encoder.defaultBitrate(width: self.encoder.width, height: self.encoder.height)) } self.controller.sent(seq: seq, bytes: data.count + 17, at: t) var echo: UInt32 = 0 let (w, h) = (self.encoder.width, self.encoder.height) self.stats.update(seq) { $0.encodeEnd = t $0.sent = t $0.bytes = data.count $0.key = key $0.width = w $0.height = h echo = $0.echo } // Header: flags (1 = keyframe), pts µs, sequence number, and the // input this frame is the first reply to; all big-endian. var msg = Data(capacity: data.count + 17) msg.append(key ? 1 : 0) var us = UInt64(max(0, CMTimeGetSeconds(pts)) * 1_000_000).bigEndian msg.append(Data(bytes: &us, count: 8)) var s = seq.bigEndian, e = echo.bigEndian msg.append(Data(bytes: &s, count: 4)) msg.append(Data(bytes: &e, count: 4)) msg.append(data) let stats = self.stats self.ws.sendBinary(msg) { stats.update(seq) { $0.wire = nowUs() } } } encoder.onDone = { [weak self] _ in self?.finished() } encoder.onError = { [weak self] message in self?.ws.sendJSON(["t": "error", "message": message]) } capture.onFrame = { [weak self] pb, pts, shown in self?.offer(pb, pts: pts, capture: shown) } capture.onEnded = { [weak self] reason in guard let self else { return } self.ws.sendJSON(["t": "closed", "reason": reason]) self.onFinished?(self.source.key) self.end() } ws.onDrain = { [weak self] in self?.resendIfRoom() } ws.onText = { [weak self] text in self?.handle(text) } ws.onClose = { [weak self] in self?.end() } ws.start() // Reconnect with this (kept in the page's memory, never on a command line). ws.sendJSON(["t": "hello", "r": reconnectKey]) capture.onChange = { [weak self] in self?.sendInfo() } // The numbers, for the viewer's overlay. // Adapt ten times a second; the numbers go to the viewer once a second. let t = DispatchSource.makeTimerSource(queue: encodeQueue) t.schedule(deadline: .now() + 0.1, repeating: 0.1) var ticks = 0 t.setEventHandler { [weak self] in guard let self, !self.isStopped else { return } self.adapt(maxLong: maxLong) ticks += 1 guard ticks % 10 == 0 else { return } var s = self.stats.summary() s.merge(self.controller.state()) { a, _ in a } s["t"] = "stats" s["bitrate"] = self.encoder.bitrate s["size"] = "\(self.encoder.width)×\(self.encoder.height)" self.ws.sendJSON(s) } t.resume() statsTimer = t capture.start(maxLong: maxLong, fps: fps) { [weak self] error in guard let self else { return } if let error { if self.capture.gone { // Final, like a window closing mid-stream: the viewer closes // and its key is revoked, rather than retrying for ever. self.ws.sendJSON(["t": "closed", "reason": error]) self.onFinished?(self.source.key) } else { self.ws.sendJSON(["t": "error", "message": error]) } self.end() return } self.sendInfo() } } /// Applies the controller's bitrate and tier. On the encoding queue. private func adapt(maxLong: Int) { guard let bps = controller.update(now: nowUs()) else { return } encoder.setBitrate(bps) let scale = controller.scale if scale != appliedScale { appliedScale = scale capture.setMaxLong(max(320, Int(Double(maxLong) * scale))) } resendIfRoom() // a lower tier may let a held frame go now } /// While someone is typing or clicking, the viewer sends a tiny message this /// often (ms, 0 = off), so the Frame's Wi-Fi doesn't doze between the input /// and the frame that answers it (power saving is on there). static let warmMs = Int(ProcessInfo.processInfo.environment["FRAME_MAC_VIEW_WARM"] ?? "") ?? 0 func sendInfo() { ws.sendJSON(["t": "info", "src": source.key, "title": capture.title, "app": capture.app, "codec": codec.rawValue, "input": source == .test || Input.allowed, "warm": Session.warmMs, "aspect": Double(capture.frameRect.width / max(capture.frameRect.height, 1))]) } var isStopped: Bool { lock.lock(); defer { lock.unlock() }; return stopped } func end() { lock.lock() let was = stopped stopped = true last = nil lock.unlock() guard !was else { return } statsTimer?.cancel() capture.stop() encodeQueue.async { self.encoder.invalidate() } let owner = id DispatchQueue.main.async { Input.releaseAll(owner: owner) } ws.close() onEnd?(self) } /// A point in the picture (0...1 each way) -> Mac global coordinates. private func point(_ x: Double, _ y: Double) -> CGPoint { let r = capture.frameRect return CGPoint(x: r.minX + min(max(x, 0), 1) * r.width, y: r.minY + min(max(y, 0), 1) * r.height) } /// Input with an id (from the viewer) has been posted: the next picture /// the Mac composites is its first chance to show. private func injected(_ m: [String: Any], kind: String) { guard let iid = (m["i"] as? NSNumber)?.uint32Value, iid != 0 else { return } let now = nowUs() stats.addInput(InputRecord(id: iid, kind: kind, viewer: Int64(num(m["tv"]) ?? 0), injected: now)) lock.lock(); pendingEcho = (iid, now); lock.unlock() } /// Timing reports from the viewer, in the agent's clock. private func report(_ t: String, _ m: [String: Any]) -> Bool { switch t { case "ping": ws.sendJSON(["t": "pong", "c": m["c"] ?? 0, "a": nowUs()]) case "rx": guard let s = (m["s"] as? NSNumber)?.uint32Value else { break } if let r = num(m["r"]) { stats.update(s) { $0.received = Int64(r) } } if controller.acked(seq: s, at: nowUs()) { resendIfRoom() } case "fd": for f in m["f"] as? [[NSNumber]] ?? [] where f.count >= 4 { stats.update(f[0].uint32Value) { $0.decoded = f[1].int64Value $0.drawn = f[2].int64Value $0.vsync = f[3].int64Value } } let dropped = Int(num(m["drop"]) ?? 0) stats.withLock { stats.viewerDropped += dropped } case "w": break // keep-warm filler, see sendInfo case "clock": stats.withLock { stats.rtt = num(m["rtt"]) ?? 0 stats.clockSynced = true if let d = m["dec"] as? String { stats.decoder = d } } default: return false } return true } /// Asks the viewer to do something for a benchmark (type, click, show its overlay). func bench(_ m: [String: Any]) { ws.sendJSON(m.merging(["t": "bench"]) { a, _ in a }) } private func handle(_ text: String) { guard !isStopped, let d = text.data(using: .utf8), let m = try? JSONSerialization.jsonObject(with: d) as? [String: Any], let t = m["t"] as? String else { return } if report(t, m) { return } let x = m["x"] as? Double ?? -1, y = m["y"] as? Double ?? -1 if t == "ack" { onAck?() onAck = nil return } if t == "key-frame" { encodeQueue.async { self.encoder.requestKeyFrame() } // before the resend, same queue lock.lock(); skipped = last != nil; lock.unlock() resendIfRoom() return } if source == .test { let desc: String? switch t { case "m": desc = (m["e"] as? String) == "move" ? nil : "mouse \(m["e"] ?? "") button \(m["b"] ?? 0)" case "wheel": desc = "wheel \(m["dx"] ?? 0), \(m["dy"] ?? 0)" case "k": desc = (m["e"] as? String) == "down" ? "key \(m["code"] ?? "") \"\(m["key"] ?? "")\"" : nil case "text": desc = "text \"\(m["s"] ?? "")\"" default: desc = nil } capture.pointer(x: x, y: y, text: desc) if desc != nil { injected(m, kind: t) } return } DispatchQueue.main.async { [self] in guard !isStopped else { return } // Stop revokes input at once switch t { case "m": let kind = m["e"] as? String ?? "move", b = m["b"] as? Int ?? 0 let p = point(x, y) if kind == "down", case .window(let wid) = source, let pid = capture.pid { Input.focus(window: wid, pid: pid) } Input.mouse(kind, button: b, at: p, owner: id) if kind == "down" { injected(m, kind: "click") } case "wheel": Input.scroll(dx: m["dx"] as? Double ?? 0, dy: m["dy"] as? Double ?? 0, at: point(x, y), owner: id) injected(m, kind: "wheel") case "k": let down = (m["e"] as? String) == "down" Input.key(code: m["code"] as? String ?? "", key: m["key"] as? String ?? "", down: down, mods: m["mods"] as? [String] ?? [], owner: id) if down { injected(m, kind: "key") } case "text": if let s = m["s"] as? String, s.count <= 4096 { Input.text(s); injected(m, kind: "text") } case "release": Input.releaseAll(owner: id) case "focus": if case .window(let wid) = source, let pid = capture.pid { Input.focus(window: wid, pid: pid) } default: break } } } } func randomKey() -> String { var bytes = [UInt8](repeating: 0, count: 24) _ = SecRandomCopyBytes(kSecRandomDefault, bytes.count, &bytes) return Data(bytes).base64EncodedString().replacingOccurrences(of: "+", with: "-") .replacingOccurrences(of: "/", with: "_").replacingOccurrences(of: "=", with: "") } func sameSecret(_ a: String, _ b: String) -> Bool { guard !a.isEmpty, a.utf8.count == b.utf8.count else { return false } return zip(a.utf8, b.utf8).reduce(0, { $0 | ($1.0 ^ $1.1) }) == 0 } final class Agent { /// Frame Control's own key. It stays on the Mac: the Frame only ever sees /// single-use tickets and per-stream reconnect keys, both tied to one source. let token: String let page: URL? var sessions: [Int: Session] = [:] { didSet { keepDisplayAwake(!sessions.isEmpty) } } var nextId = 1 let lock = NSLock() private var assertion: IOPMAssertionID = 0 /// ticket -> (source, expiry, reconnect key once redeemed). A ticket opens /// one viewer within a minute; it may be redeemed again, for the same key, /// only until that viewer confirms it has the key ("ack"). private var tickets: [String: (src: String, expiry: Date, key: String?)] = [:] /// reconnect key -> source, until Stop for that source. private var reconnectKeys: [String: String] = [:] /// Sources that ended for good (the window closed), for Frame Control. private var finished = Set() /// Ends every stream, then exits once separated windows are back on the /// Mac's own screens (they're moved back, then their displays go 0.3 s later). /// Idempotent: SIGTERM and the parent going away can both call it, and a /// session already ended by /close may still be putting its window back. private var quitting = false func quit() { lock.lock() let all = Array(sessions.values), again = quitting quitting = true lock.unlock() guard !again else { return } for s in all { s.ws.sendJSON(["t": "close"]); s.end() } DispatchQueue.main.asyncAfter(deadline: .now() + 0.8) { exit(0) } } /// While anyone watches, keep the Mac's display on: a sleeping display /// stops being drawn, so there'd be nothing to capture (and it would lock). private func keepDisplayAwake(_ on: Bool) { if on, assertion == 0 { IOPMAssertionCreateWithName(kIOPMAssertionTypePreventUserIdleDisplaySleep as CFString, IOPMAssertionLevel(kIOPMAssertionLevelOn), "Frame Control is showing this Mac in a Steam Frame" as CFString, &assertion) } else if !on, assertion != 0 { IOPMAssertionRelease(assertion) assertion = 0 } } init(token: String, page: URL?) { self.token = token self.page = page } /// Whether this request may open a stream of `src`: Frame Control's key, /// an unused ticket for that source, or a live reconnect key for it. private func mayStream(_ req: Request, src: String) -> String? { lock.lock() defer { lock.unlock() } let now = Date() tickets = tickets.filter { $0.value.expiry > now } if sameSecret(req.query["k"] ?? "", token) { return randomKey() } if let t = req.query["t"], let entry = tickets[t], entry.src == src { // A retry before the viewer got its key gets the same key back. let key = entry.key ?? randomKey() tickets[t] = (src, entry.expiry, key) reconnectKeys[key] = src return key } if let r = req.query["r"], reconnectKeys[r] == src { return r } return nil } /// The viewer has its reconnect key, so the ticket that led to it can't /// be used again (also when the ack comes over a reconnection). func acknowledged(key: String) { lock.lock(); tickets = tickets.filter { $0.value.key != key }; lock.unlock() } func handle(_ req: Request, _ c: HTTPConnection) { // Open to anything that reaches the port: a liveness check for the // tunnel, and the viewer page, which holds no secrets. switch (req.method, req.path) { case ("GET", "/ping"): return c.respond(200, text: "frame-mac-view") case ("GET", "/view"): guard let page, let body = try? Data(contentsOf: page) else { return c.respond(404, text: "no viewer page") } return c.respond(200, body: body, type: "text/html; charset=utf-8") case ("GET", "/stream"): guard let src = Source(req.query["src"] ?? "") else { return c.respond(400, text: "bad src") } guard let key = mayStream(req, src: src.key) else { return c.respond(403, text: "forbidden") } return stream(req, c, src: src, key: key) default: break } guard sameSecret(req.query["k"] ?? req.headers["x-token"] ?? "", token) else { return c.respond(403, text: "forbidden") } switch (req.method, req.path) { case ("GET", "/status"): lock.lock() let list = sessions.values.map { s -> [String: Any] in var e: [String: Any] = ["id": s.id, "src": s.source.key, "title": s.capture.title, "app": s.capture.app, "stats": s.stats.summary(), "bitrate": s.encoder.bitrate, "controller": s.controller.state()] if let d = (s.capture as? SeparateSource)?.displayID { e["display"] = d } return e } lock.unlock() var s = permissionsJSON() s["version"] = version s["streams"] = list lock.lock(); s["finished"] = Array(finished); lock.unlock() c.respond(json: s) case ("GET", "/windows"): c.respond(json: ["windows": windowsJSON(), "screen": CGPreflightScreenCaptureAccess()]) case ("GET", "/displays"): c.respond(json: ["displays": displaysJSON()]) case ("POST", "/ticket"): guard let src = Source(req.query["src"] ?? "") else { return c.respond(400, text: "bad src") } let t = randomKey() lock.lock() tickets[t] = (src.key, Date().addingTimeInterval(60), nil) finished.remove(src.key) lock.unlock() c.respond(json: ["ticket": t]) case ("POST", "/close"): // Tell viewers to close their windows, but don't rely on them: // the sessions end here and can't reconnect. let src = req.query["src"] lock.lock() let matching = sessions.values.filter { src == nil || $0.source.key == src } reconnectKeys = reconnectKeys.filter { src != nil && $0.value != src } tickets = tickets.filter { src != nil && $0.value.src != src } // not yet used ones too lock.unlock() for s in matching { s.ws.sendJSON(["t": "close"]) } DispatchQueue.global().asyncAfter(deadline: .now() + 0.3) { for s in matching { s.end() } } c.respond(json: ["closed": matching.count]) case ("GET", "/stats"): // Frames the viewer has had time to report on, oldest first. let since = UInt32(req.query["since"] ?? "") ?? 0 let settle = Int64(req.query["settle"] ?? "") ?? 1_500_000 lock.lock() let list = sessions.values.filter { req.query["id"] == nil || "\($0.id)" == req.query["id"] } lock.unlock() c.respond(json: ["now": nowUs(), "streams": list.map { s -> [String: Any] in ["id": s.id, "src": s.source.key, "frames": s.stats.settled(since: since, settle: settle).map(\.json), "inputs": s.stats.inputList().map(\.json), "summary": s.stats.summary(), "captured": s.stats.withLock { s.stats.captured }, "controller": s.controller.state(), "events": s.controller.eventList()] }]) case ("POST", "/bench"): lock.lock() let list = sessions.values.filter { $0.source.key == req.query["src"] } lock.unlock() var m: [String: Any] = [:] for (k, v) in req.query where k != "k" && k != "src" { m[k] = Double(v) ?? v as Any } for s in list { s.bench(m) } c.respond(json: ["sent": list.count]) case ("GET", "/snapshot"): // One JPEG of a display (for checks and thumbnails): ?display=ID guard let id = UInt32(req.query["display"] ?? "") else { return c.respond(400, text: "display=ID") } snapshot(display: id) { data, error in if let data { c.respond(200, body: data, type: "image/jpeg") } else { c.respond(500, text: error ?? "failed") } } case ("POST", "/permissions"): DispatchQueue.main.async { requestPermissions() } c.respond(json: permissionsJSON()) default: c.respond(404, text: "not found") } } private func stream(_ req: Request, _ c: HTTPConnection, src: Source, key: String) { let codec = Codec(rawValue: req.query["codec"] ?? "h264") ?? .h264 let maxLong = min(max(Int(req.query["max"] ?? "") ?? 1920, 320), 3840) let fps = min(max(Int(req.query["fps"] ?? "") ?? 60, 5), 120) let bpp = min(max(Double(req.query["bpp"] ?? "") ?? 0.1, 0.02), 0.5) guard let ws = c.upgrade(req) else { return } let capture: CaptureSource switch src { case .test: capture = TestSource() case .separate(let id): capture = SeparateSource(windowID: id) default: capture = SCKSource(src) } lock.lock() // One live viewer per key: a reconnection (or a second use of an // unacknowledged ticket) replaces the one before. let replaced = sessions.values.filter { $0.reconnectKey == key } let session = Session(id: nextId, source: src, capture: capture, ws: ws, codec: codec, fps: fps, bitsPerPixel: bpp, reconnectKey: key) nextId += 1 sessions[session.id] = session lock.unlock() for old in replaced { old.end() } session.onEnd = { [weak self] s in guard let self else { return } self.lock.lock(); self.sessions[s.id] = nil; self.lock.unlock() } // The source is gone (its window closed): its viewers can't come back. session.onFinished = { [weak self] src in guard let self else { return } self.lock.lock() self.finished.insert(src) self.reconnectKeys = self.reconnectKeys.filter { $0.value != src } self.lock.unlock() } session.onAck = { [weak self] in self?.acknowledged(key: key) } session.start(maxLong: maxLong, fps: fps) } } func snapshot(display id: CGDirectDisplayID, completion: @escaping (Data?, String?) -> Void) { SCShareableContent.getExcludingDesktopWindows(false, onScreenWindowsOnly: true) { content, error in guard let d = content?.displays.first(where: { $0.displayID == id }) else { return completion(nil, error?.localizedDescription ?? "no such display") } let filter = SCContentFilter(display: d, excludingWindows: []) let cfg = SCStreamConfiguration() cfg.width = Int(Double(d.width) * Double(filter.pointPixelScale)) cfg.height = Int(Double(d.height) * Double(filter.pointPixelScale)) cfg.showsCursor = true SCScreenshotManager.captureImage(contentFilter: filter, configuration: cfg) { image, error in guard let image else { return completion(nil, error?.localizedDescription ?? "no image") } let rep = NSBitmapImageRep(cgImage: image) completion(rep.representation(using: .jpeg, properties: [.compressionFactor: 0.85]), nil) } } } func requestPermissions() { if !CGPreflightScreenCaptureAccess() { CGRequestScreenCaptureAccess() } if !AXIsProcessTrusted() { AXIsProcessTrustedWithOptions([kAXTrustedCheckOptionPrompt.takeUnretainedValue() as String: true] as CFDictionary) } } func argument(_ name: String, in args: [String]) -> String? { guard let i = args.firstIndex(of: name), i + 1 < args.count else { return nil } return args[i + 1] } let args = Array(CommandLine.arguments.dropFirst()) switch args.first { case "windows": printJSON(["windows": windowsJSON(), "screen": CGPreflightScreenCaptureAccess()]) case "displays": printJSON(["displays": displaysJSON()]) case "permissions": printJSON(permissionsJSON()) case "request-permissions": requestPermissions() printJSON(permissionsJSON()) case "serve": let port = UInt16(argument("--port", in: args) ?? "") ?? 0 let token = ProcessInfo.processInfo.environment["FRAME_MAC_VIEW_TOKEN"] ?? "" guard !token.isEmpty else { FileHandle.standardError.write(Data("frame-mac-view: set FRAME_MAC_VIEW_TOKEN\n".utf8)) exit(2) } let page = argument("--page", in: args).map { URL(fileURLWithPath: $0) } let agent = Agent(token: token, page: page) // Quit when the parent goes away (it holds our stdin open), or on SIGTERM, // but first end every stream, so separated windows go back where they // were before their displays disappear. if args.contains("--exit-on-eof") { DispatchQueue.global().async { while FileHandle.standardInput.availableData.count > 0 {} agent.quit() } } signal(SIGTERM, SIG_IGN) let sigterm = DispatchSource.makeSignalSource(signal: SIGTERM, queue: .main) sigterm.setEventHandler { agent.quit() } sigterm.resume() let server: Server do { server = try Server(port: port) { req, c in agent.handle(req, c) } } catch { FileHandle.standardError.write(Data("frame-mac-view: \(error)\n".utf8)) exit(1) } server.start { [server] error in if let error { FileHandle.standardError.write(Data("frame-mac-view: can't listen on 127.0.0.1:\(port): \(error)\n".utf8)) exit(1) } print("frame-mac-view listening on 127.0.0.1:\(server.port ?? port)") fflush(stdout) } // A real (background, no Dock icon) AppKit event loop, not just a run // loop: without it this process never hears that displays were added or // changed mode, so NSScreen and CGDisplayCopyDisplayMode stay stale for // the displays Separate mode creates. let app = NSApplication.shared app.setActivationPolicy(.prohibited) withExtendedLifetime((server, sigterm)) { app.run() } default: FileHandle.standardError.write(Data("usage: frame-mac-view serve|windows|displays|permissions|request-permissions\n".utf8)) exit(2) }