mirror of
https://github.com/saphid/frame-control.git
synced 2026-10-06 08:00:32 +02:00
Mac in the headset: stream Mac windows and screens into the Frame as panels
Frame Control can now show any Mac window, or a whole screen, as its own SteamVR panel on the Steam Frame (Tools -> Mac in the headset, macOS only). Place it with the SteamVR dashboard; the laser clicks and scrolls, and the Mac's own keyboard types. - mac/frame-mac-view (Swift, no dependencies): ScreenCaptureKit capture per window or display, VideoToolbox H.264 with low-latency rate control (JPEG fallback), a loopback HTTP/WebSocket server, CGEvent/AX input playback, and a display-awake assertion while anyone watches. - ui/frame_macview.py: starts the agent, runs an ssh -R tunnel with a supervisor that reopens it on the same port, and launches a Chromium app window per stream on gamescope's :0, tagged with STEAM_GAME for its own panel. - Frame Control's key never leaves the Mac: viewers get single-use, per-source tickets and reconnect keys that Stop revokes. - ui/mac-view.html: WebCodecs decode, keyframe recovery, pointer/wheel/keys back. - Bundled in the Mac app build; tests/test_macview.py builds and drives the agent on macOS. Verified on the Frame (build 20260925.6191901) with the test pattern: panel in about 1.5 s, about 60 fps, Mac-to-window about 11-17 ms, tunnel recovery in 4 s. Laser input and real window capture still need a person in the headset. Also commits the other thread's first-party rule (steam-frame skill) and the first-party options table in docs/streaming.md. Reviewed by GPT-6 Astra (xhigh, read-only) over six rounds; all findings fixed. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
saphidandClaude Opus 5.5 committed
1 parent
0016d9200c
commit
a3c6e5c984
20 files changed
+2708
-6
No files matched your search
@@ -0,0 +1,339 @@
|
||||
// Where pictures come from: one Mac window or one display (ScreenCaptureKit),
|
||||
// or a generated test pattern that needs no permissions and shows the input
|
||||
// the viewer sends, for checking the whole path without touching the Mac.
|
||||
import AppKit
|
||||
import CoreMedia
|
||||
import CoreVideo
|
||||
import Foundation
|
||||
import ScreenCaptureKit
|
||||
|
||||
enum Source: Equatable {
|
||||
case window(CGWindowID)
|
||||
case display(CGDirectDisplayID)
|
||||
case test
|
||||
|
||||
init?(_ s: String) {
|
||||
let parts = s.split(separator: ":", maxSplits: 1).map(String.init)
|
||||
switch (parts.first, parts.count > 1 ? UInt32(parts[1]) : nil) {
|
||||
case ("window", let id?): self = .window(id)
|
||||
case ("display", let id?): self = .display(id)
|
||||
case ("test", _): self = .test
|
||||
default: return nil
|
||||
}
|
||||
}
|
||||
|
||||
var key: String {
|
||||
switch self {
|
||||
case .window(let id): return "window:\(id)"
|
||||
case .display(let id): return "display:\(id)"
|
||||
case .test: return "test"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Size to capture at: the source's pixel size, shrunk to fit a box whose
|
||||
/// long side is `maxLong` (default 1920), even numbers for the encoder.
|
||||
func fitSize(width: Double, height: Double, maxLong: Int) -> (Int, Int) {
|
||||
let scale = min(1, Double(maxLong) / max(width, height, 1))
|
||||
let w = max(16, Int((width * scale / 2).rounded()) * 2), h = max(16, Int((height * scale / 2).rounded()) * 2)
|
||||
return (w, h)
|
||||
}
|
||||
|
||||
/// What CGWindowList says about a window right now (no permission needed for
|
||||
/// bounds; titles need Screen Recording).
|
||||
struct WindowInfo {
|
||||
let id: CGWindowID
|
||||
let pid: pid_t
|
||||
let app: String
|
||||
let title: String
|
||||
let bounds: CGRect
|
||||
let layer: Int
|
||||
let onScreen: Bool
|
||||
|
||||
static func all(onScreenOnly: Bool = true) -> [WindowInfo] {
|
||||
let opts: CGWindowListOption = onScreenOnly ? [.optionOnScreenOnly, .excludeDesktopElements] : [.excludeDesktopElements]
|
||||
let list = CGWindowListCopyWindowInfo(opts, kCGNullWindowID) as? [[CFString: Any]] ?? []
|
||||
return list.compactMap(WindowInfo.init)
|
||||
}
|
||||
|
||||
static func find(_ id: CGWindowID) -> WindowInfo? {
|
||||
let list = CGWindowListCopyWindowInfo(.optionIncludingWindow, id) as? [[CFString: Any]] ?? []
|
||||
return list.compactMap(WindowInfo.init).first { $0.id == id }
|
||||
}
|
||||
|
||||
init?(_ d: [CFString: Any]) {
|
||||
guard let id = d[kCGWindowNumber] as? CGWindowID, let pid = d[kCGWindowOwnerPID] as? pid_t,
|
||||
let b = d[kCGWindowBounds] as? [String: Any], let rect = CGRect(dictionaryRepresentation: b as CFDictionary)
|
||||
else { return nil }
|
||||
self.id = id
|
||||
self.pid = pid
|
||||
app = d[kCGWindowOwnerName] as? String ?? ""
|
||||
title = d[kCGWindowName] as? String ?? ""
|
||||
bounds = rect
|
||||
layer = d[kCGWindowLayer] as? Int ?? 0
|
||||
onScreen = d[kCGWindowIsOnscreen] as? Bool ?? false
|
||||
}
|
||||
}
|
||||
|
||||
protocol CaptureSource: AnyObject {
|
||||
var onFrame: ((CVPixelBuffer, CMTime) -> Void)? { get set }
|
||||
var onEnded: ((String) -> Void)? { get set }
|
||||
/// Points on the Mac's global display space that the picture covers.
|
||||
var frameRect: CGRect { get }
|
||||
var title: String { get }
|
||||
var app: String { get }
|
||||
var pid: pid_t? { get }
|
||||
func start(maxLong: Int, fps: Int, completion: @escaping (String?) -> Void)
|
||||
func stop()
|
||||
func pointer(x: Double, y: Double, text: String?) // for the test pattern
|
||||
}
|
||||
|
||||
extension CaptureSource {
|
||||
func pointer(x: Double, y: Double, text: String?) {}
|
||||
}
|
||||
|
||||
/// ScreenCaptureKit, for a window or a display.
|
||||
final class SCKSource: NSObject, CaptureSource, SCStreamOutput, SCStreamDelegate {
|
||||
let source: Source
|
||||
var onFrame: ((CVPixelBuffer, CMTime) -> Void)?
|
||||
var onEnded: ((String) -> Void)?
|
||||
private(set) var frameRect = CGRect.zero
|
||||
private(set) var title = ""
|
||||
private(set) var app = ""
|
||||
private(set) var pid: pid_t?
|
||||
private var stream: SCStream?
|
||||
private var config = SCStreamConfiguration()
|
||||
private var maxLong = 1920
|
||||
private var scale = 2.0
|
||||
private var poll: DispatchSourceTimer?
|
||||
private let queue = DispatchQueue(label: "frame-mac-view.capture", qos: .userInteractive)
|
||||
/// Set by stop(), on `queue`; a start still enumerating windows checks it
|
||||
/// before it starts capturing, so a viewer that left early leaves nothing on.
|
||||
private var cancelled = false
|
||||
/// The window or display no longer exists, so retrying can't help.
|
||||
private(set) var gone = false
|
||||
var onChange: (() -> Void)? // title or size changed
|
||||
|
||||
init(_ source: Source) { self.source = source }
|
||||
|
||||
func start(maxLong: Int, fps: Int, completion: @escaping (String?) -> Void) {
|
||||
self.maxLong = maxLong
|
||||
SCShareableContent.getExcludingDesktopWindows(true, onScreenWindowsOnly: false) { [self] content, error in
|
||||
queue.async { self.begin(content, error, fps: fps, completion: completion) }
|
||||
}
|
||||
}
|
||||
|
||||
private func begin(_ content: SCShareableContent?, _ error: Error?, fps: Int, completion: @escaping (String?) -> Void) {
|
||||
if cancelled { return }
|
||||
guard let content else {
|
||||
return completion("Screen Recording isn't allowed for Frame Control (\(error?.localizedDescription ?? "no content"))")
|
||||
}
|
||||
let filter: SCContentFilter
|
||||
switch source {
|
||||
case .window(let id):
|
||||
guard let w = content.windows.first(where: { $0.windowID == id }) else {
|
||||
gone = true
|
||||
return completion("that window has closed")
|
||||
}
|
||||
filter = SCContentFilter(desktopIndependentWindow: w)
|
||||
title = w.title ?? ""
|
||||
app = w.owningApplication?.applicationName ?? ""
|
||||
pid = w.owningApplication?.processID
|
||||
frameRect = w.frame
|
||||
case .display(let id):
|
||||
guard let d = content.displays.first(where: { $0.displayID == id }) else {
|
||||
gone = true
|
||||
return completion("that display isn't connected")
|
||||
}
|
||||
filter = SCContentFilter(display: d, excludingWindows: [])
|
||||
title = displayName(id)
|
||||
app = "Mac"
|
||||
frameRect = CGDisplayBounds(id)
|
||||
case .test:
|
||||
return completion("not a ScreenCaptureKit source")
|
||||
}
|
||||
scale = Double(filter.pointPixelScale)
|
||||
let (w, h) = fitSize(width: frameRect.width * scale, height: frameRect.height * scale, maxLong: maxLong)
|
||||
config.width = w
|
||||
config.height = h
|
||||
config.minimumFrameInterval = CMTime(value: 1, timescale: CMTimeScale(fps))
|
||||
config.pixelFormat = kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange
|
||||
config.colorMatrix = CGDisplayStream.yCbCrMatrix_ITU_R_709_2
|
||||
config.colorSpaceName = CGColorSpace.sRGB
|
||||
config.showsCursor = true
|
||||
config.queueDepth = 5
|
||||
config.scalesToFit = true
|
||||
config.preservesAspectRatio = true
|
||||
config.capturesAudio = false
|
||||
let stream = SCStream(filter: filter, configuration: config, delegate: self)
|
||||
do {
|
||||
try stream.addStreamOutput(self, type: .screen, sampleHandlerQueue: queue)
|
||||
} catch {
|
||||
return completion("couldn't capture: \(error.localizedDescription)")
|
||||
}
|
||||
self.stream = stream
|
||||
stream.startCapture { error in
|
||||
self.queue.async {
|
||||
if self.cancelled {
|
||||
stream.stopCapture { _ in }
|
||||
return
|
||||
}
|
||||
if let error { return completion("couldn't capture: \(error.localizedDescription)") }
|
||||
self.startPolling()
|
||||
completion(nil)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func stop() {
|
||||
queue.async {
|
||||
self.cancelled = true
|
||||
self.poll?.cancel()
|
||||
self.poll = nil
|
||||
self.stream?.stopCapture { _ in }
|
||||
self.stream = nil
|
||||
}
|
||||
}
|
||||
|
||||
/// Windows move, resize, retitle and close; ScreenCaptureKit doesn't say.
|
||||
private func startPolling() {
|
||||
guard case .window(let id) = source else { return }
|
||||
let t = DispatchSource.makeTimerSource(queue: queue)
|
||||
t.schedule(deadline: .now() + 1, repeating: 1)
|
||||
t.setEventHandler { [weak self] in
|
||||
guard let self else { return }
|
||||
guard let info = WindowInfo.find(id) else {
|
||||
self.stop()
|
||||
self.onEnded?("the window closed")
|
||||
return
|
||||
}
|
||||
var changed = false
|
||||
if !info.title.isEmpty, info.title != self.title { self.title = info.title; changed = true }
|
||||
let old = self.frameRect
|
||||
self.frameRect = info.bounds
|
||||
if abs(old.width - info.bounds.width) > 1 || abs(old.height - info.bounds.height) > 1 {
|
||||
let (w, h) = fitSize(width: info.bounds.width * self.scale, height: info.bounds.height * self.scale, maxLong: self.maxLong)
|
||||
self.config.width = w
|
||||
self.config.height = h
|
||||
self.stream?.updateConfiguration(self.config) { _ in }
|
||||
changed = true
|
||||
}
|
||||
if changed { self.onChange?() }
|
||||
}
|
||||
t.resume()
|
||||
poll = t
|
||||
}
|
||||
|
||||
func stream(_ stream: SCStream, didOutputSampleBuffer sample: CMSampleBuffer, of type: SCStreamOutputType) {
|
||||
guard type == .screen, sample.isValid, let pb = CMSampleBufferGetImageBuffer(sample) else { return }
|
||||
// Only complete frames carry new pixels; idle and blank ones don't.
|
||||
if let atts = CMSampleBufferGetSampleAttachmentsArray(sample, createIfNecessary: false) as? [[SCStreamFrameInfo: Any]],
|
||||
let raw = atts.first?[.status] as? Int, let status = SCFrameStatus(rawValue: raw), status != .complete {
|
||||
return
|
||||
}
|
||||
onFrame?(pb, CMSampleBufferGetPresentationTimeStamp(sample))
|
||||
}
|
||||
|
||||
func stream(_ stream: SCStream, didStopWithError error: Error) {
|
||||
onEnded?("capture stopped: \(error.localizedDescription)")
|
||||
}
|
||||
}
|
||||
|
||||
func displayName(_ id: CGDirectDisplayID) -> String {
|
||||
for screen in NSScreen.screens {
|
||||
if (screen.deviceDescription[NSDeviceDescriptionKey("NSScreenNumber")] as? NSNumber)?.uint32Value == id {
|
||||
return screen.localizedName
|
||||
}
|
||||
}
|
||||
return "Display \(id)"
|
||||
}
|
||||
|
||||
/// A moving test card: bars, a clock and a frame counter, plus a dot where the
|
||||
/// viewer's pointer is and the last key it sent, so input can be checked too.
|
||||
final class TestSource: CaptureSource {
|
||||
var onFrame: ((CVPixelBuffer, CMTime) -> Void)?
|
||||
var onEnded: ((String) -> Void)?
|
||||
let frameRect = CGRect(x: 0, y: 0, width: 1280, height: 720)
|
||||
let title = "Test pattern"
|
||||
let app = "Frame Control"
|
||||
let pid: pid_t? = nil
|
||||
private var timer: DispatchSourceTimer?
|
||||
private var pool: CVPixelBufferPool?
|
||||
private var n = 0
|
||||
private var dot: (Double, Double)?
|
||||
private var lastText = "Point or type in the headset"
|
||||
private let queue = DispatchQueue(label: "frame-mac-view.test")
|
||||
private var size = (1280, 720)
|
||||
|
||||
func start(maxLong: Int, fps: Int, completion: @escaping (String?) -> Void) {
|
||||
size = fitSize(width: 1280, height: 720, maxLong: maxLong)
|
||||
let attrs: [CFString: Any] = [kCVPixelBufferPixelFormatTypeKey: kCVPixelFormatType_32BGRA,
|
||||
kCVPixelBufferWidthKey: size.0, kCVPixelBufferHeightKey: size.1,
|
||||
kCVPixelBufferIOSurfacePropertiesKey: [:] as CFDictionary]
|
||||
CVPixelBufferPoolCreate(nil, nil, attrs as CFDictionary, &pool)
|
||||
let t = DispatchSource.makeTimerSource(queue: queue)
|
||||
t.schedule(deadline: .now(), repeating: 1.0 / Double(fps))
|
||||
t.setEventHandler { [weak self] in self?.draw() }
|
||||
t.resume()
|
||||
timer = t
|
||||
completion(nil)
|
||||
}
|
||||
|
||||
func stop() {
|
||||
timer?.cancel()
|
||||
timer = nil
|
||||
}
|
||||
|
||||
func pointer(x: Double, y: Double, text: String?) {
|
||||
queue.async {
|
||||
if x >= 0 { self.dot = (x, y) }
|
||||
if let text { self.lastText = text }
|
||||
}
|
||||
}
|
||||
|
||||
private func draw() {
|
||||
guard let pool else { return }
|
||||
var out: CVPixelBuffer?
|
||||
CVPixelBufferPoolCreatePixelBuffer(nil, pool, &out)
|
||||
guard let pb = out else { return }
|
||||
CVPixelBufferLockBaseAddress(pb, [])
|
||||
defer { CVPixelBufferUnlockBaseAddress(pb, []) }
|
||||
let (w, h) = size
|
||||
guard let ctx = CGContext(data: CVPixelBufferGetBaseAddress(pb), width: w, height: h, bitsPerComponent: 8,
|
||||
bytesPerRow: CVPixelBufferGetBytesPerRow(pb), space: CGColorSpace(name: CGColorSpace.sRGB)!,
|
||||
bitmapInfo: CGImageAlphaInfo.premultipliedFirst.rawValue | CGBitmapInfo.byteOrder32Little.rawValue)
|
||||
else { return }
|
||||
let colors: [(CGFloat, CGFloat, CGFloat)] = [(0.75, 0.75, 0.75), (0.75, 0.75, 0), (0, 0.75, 0.75), (0, 0.75, 0),
|
||||
(0.75, 0, 0.75), (0.75, 0, 0), (0, 0, 0.75)]
|
||||
let bw = CGFloat(w) / CGFloat(colors.count)
|
||||
for (i, c) in colors.enumerated() {
|
||||
ctx.setFillColor(red: c.0, green: c.1, blue: c.2, alpha: 1)
|
||||
ctx.fill(CGRect(x: CGFloat(i) * bw, y: CGFloat(h) * 0.35, width: bw + 1, height: CGFloat(h) * 0.65))
|
||||
}
|
||||
ctx.setFillColor(red: 0.08, green: 0.09, blue: 0.11, alpha: 1)
|
||||
ctx.fill(CGRect(x: 0, y: 0, width: w, height: Int(Double(h) * 0.35)))
|
||||
// A bar sweeping once a second shows motion and dropped frames.
|
||||
let x = CGFloat(n % 60) / 60 * CGFloat(w)
|
||||
ctx.setFillColor(red: 1, green: 1, blue: 1, alpha: 1)
|
||||
ctx.fill(CGRect(x: x, y: CGFloat(h) * 0.35, width: max(4, CGFloat(w) / 120), height: CGFloat(h) * 0.65))
|
||||
let f = DateFormatter()
|
||||
f.dateFormat = "HH:mm:ss.SSS"
|
||||
let label = "Frame Control test pattern \(f.string(from: Date())) frame \(n)\n\(lastText)"
|
||||
let text = NSAttributedString(string: label, attributes: [
|
||||
.font: NSFont.monospacedSystemFont(ofSize: CGFloat(h) / 24, weight: .medium),
|
||||
.foregroundColor: NSColor.white,
|
||||
])
|
||||
let ns = NSGraphicsContext(cgContext: ctx, flipped: false)
|
||||
NSGraphicsContext.saveGraphicsState()
|
||||
NSGraphicsContext.current = ns
|
||||
text.draw(at: CGPoint(x: CGFloat(w) * 0.03, y: CGFloat(h) * 0.08))
|
||||
NSGraphicsContext.restoreGraphicsState()
|
||||
if let (px, py) = dot {
|
||||
let r = CGFloat(h) / 40
|
||||
ctx.setFillColor(red: 1, green: 0.2, blue: 0.2, alpha: 1)
|
||||
ctx.fillEllipse(in: CGRect(x: CGFloat(px) * CGFloat(w) - r, y: (1 - CGFloat(py)) * CGFloat(h) - r, width: 2 * r, height: 2 * r))
|
||||
}
|
||||
n += 1
|
||||
onFrame?(pb, CMClockGetTime(CMClockGetHostTimeClock()))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,172 @@
|
||||
// VideoToolbox encoding: H.264 for the normal path (hardware, low-latency rate
|
||||
// control, no B-frames, Annex B output that WebCodecs takes without a
|
||||
// description), or JPEG stills for viewers that can't decode H.264.
|
||||
import CoreMedia
|
||||
import Foundation
|
||||
import VideoToolbox
|
||||
|
||||
enum Codec: String {
|
||||
case h264, jpeg
|
||||
}
|
||||
|
||||
final class Encoder {
|
||||
let codec: Codec
|
||||
let fps: Int
|
||||
let bitsPerPixel: Double
|
||||
private(set) var width = 0
|
||||
private(set) var height = 0
|
||||
private var session: VTCompressionSession?
|
||||
private var forceKey = true
|
||||
private var lastPts = CMTime.invalid
|
||||
/// Called on VideoToolbox's thread with one access unit (or JPEG) per frame.
|
||||
var onFrame: ((Data, Bool, CMTime) -> Void)?
|
||||
var onError: ((String) -> Void)?
|
||||
/// Called once per frame handed to VideoToolbox, however it went.
|
||||
var onDone: (() -> Void)?
|
||||
|
||||
init(codec: Codec, fps: Int, bitsPerPixel: Double) {
|
||||
self.codec = codec
|
||||
self.fps = fps
|
||||
self.bitsPerPixel = bitsPerPixel
|
||||
}
|
||||
|
||||
deinit { invalidate() }
|
||||
|
||||
func requestKeyFrame() { forceKey = true }
|
||||
|
||||
func invalidate() {
|
||||
if let s = session {
|
||||
VTCompressionSessionCompleteFrames(s, untilPresentationTimeStamp: .invalid)
|
||||
VTCompressionSessionInvalidate(s)
|
||||
}
|
||||
session = nil
|
||||
}
|
||||
|
||||
private func makeSession(width w: Int, height h: Int) -> Bool {
|
||||
invalidate()
|
||||
var spec: [CFString: Any] = [:]
|
||||
if codec == .h264 { spec[kVTVideoEncoderSpecification_EnableLowLatencyRateControl] = true }
|
||||
var s: VTCompressionSession?
|
||||
let type = codec == .h264 ? kCMVideoCodecType_H264 : kCMVideoCodecType_JPEG
|
||||
func create(_ spec: [CFString: Any]) -> OSStatus {
|
||||
VTCompressionSessionCreate(allocator: nil, width: Int32(w), height: Int32(h), codecType: type,
|
||||
encoderSpecification: spec as CFDictionary, imageBufferAttributes: nil,
|
||||
compressedDataAllocator: nil, outputCallback: nil, refcon: nil,
|
||||
compressionSessionOut: &s)
|
||||
}
|
||||
var err = create(spec)
|
||||
// Low-latency rate control needs Apple's hardware encoder; without it
|
||||
// (some VMs), plain real-time encoding still works.
|
||||
if err != noErr, !spec.isEmpty { err = create([:]) }
|
||||
guard err == noErr, let s else {
|
||||
onError?("couldn't start the \(codec.rawValue) encoder (VideoToolbox \(err))")
|
||||
return false
|
||||
}
|
||||
func set(_ key: CFString, _ value: Any) { VTSessionSetProperty(s, key: key, value: value as CFTypeRef) }
|
||||
set(kVTCompressionPropertyKey_RealTime, true)
|
||||
set(kVTCompressionPropertyKey_ColorPrimaries, kCVImageBufferColorPrimaries_ITU_R_709_2)
|
||||
set(kVTCompressionPropertyKey_TransferFunction, kCVImageBufferTransferFunction_ITU_R_709_2)
|
||||
set(kVTCompressionPropertyKey_YCbCrMatrix, kCVImageBufferYCbCrMatrix_ITU_R_709_2)
|
||||
if codec == .h264 {
|
||||
let bps = min(max(Double(w * h * fps) * bitsPerPixel, 2_000_000), 60_000_000)
|
||||
set(kVTCompressionPropertyKey_ProfileLevel, kVTProfileLevel_H264_ConstrainedHigh_AutoLevel)
|
||||
set(kVTCompressionPropertyKey_AllowFrameReordering, false)
|
||||
set(kVTCompressionPropertyKey_AverageBitRate, Int(bps))
|
||||
set(kVTCompressionPropertyKey_ExpectedFrameRate, fps)
|
||||
// A keyframe every 10 s at most, so a viewer that lost one recovers
|
||||
// even if it never asks. Viewers ask for one when they start.
|
||||
set(kVTCompressionPropertyKey_MaxKeyFrameIntervalDuration, 10)
|
||||
} else {
|
||||
set(kVTCompressionPropertyKey_Quality, 0.8)
|
||||
}
|
||||
VTCompressionSessionPrepareToEncodeFrames(s)
|
||||
session = s
|
||||
lastPts = .invalid
|
||||
width = w
|
||||
height = h
|
||||
forceKey = true
|
||||
return true
|
||||
}
|
||||
|
||||
/// False if the frame never reached VideoToolbox (then onDone won't come).
|
||||
@discardableResult
|
||||
func encode(_ pb: CVPixelBuffer, pts given: CMTime) -> Bool {
|
||||
// VideoToolbox needs strictly increasing timestamps; a resent picture
|
||||
// stamped "now" can be followed by a capture stamped a moment earlier.
|
||||
var pts = given
|
||||
if lastPts.isValid, CMTimeCompare(pts, lastPts) <= 0 { pts = CMTimeAdd(lastPts, CMTime(value: 1, timescale: 1_000_000)) }
|
||||
let w = CVPixelBufferGetWidth(pb), h = CVPixelBufferGetHeight(pb)
|
||||
if session == nil || w != width || h != height {
|
||||
guard makeSession(width: w, height: h) else { return false }
|
||||
}
|
||||
guard let s = session else { return false }
|
||||
var props: CFDictionary?
|
||||
if forceKey {
|
||||
props = [kVTEncodeFrameOptionKey_ForceKeyFrame: true] as CFDictionary
|
||||
forceKey = false
|
||||
}
|
||||
let codec = self.codec
|
||||
lastPts = pts
|
||||
let status = VTCompressionSessionEncodeFrame(s, imageBuffer: pb, presentationTimeStamp: pts, duration: .invalid,
|
||||
frameProperties: props, infoFlagsOut: nil) { [weak self] status, _, sample in
|
||||
guard let self else { return }
|
||||
defer { self.onDone?() }
|
||||
guard status == noErr, let sample else { return }
|
||||
if codec == .jpeg {
|
||||
if let data = Self.bytes(sample) { self.onFrame?(data, true, pts) }
|
||||
} else if let (data, key) = Self.annexB(sample) {
|
||||
self.onFrame?(data, key, pts)
|
||||
}
|
||||
}
|
||||
if status != noErr { forceKey = true }
|
||||
return status == noErr
|
||||
}
|
||||
|
||||
private static func bytes(_ sample: CMSampleBuffer) -> Data? {
|
||||
guard let block = CMSampleBufferGetDataBuffer(sample) else { return nil }
|
||||
var length = 0
|
||||
var ptr: UnsafeMutablePointer<CChar>?
|
||||
guard CMBlockBufferGetDataPointer(block, atOffset: 0, lengthAtOffsetOut: nil, totalLengthOut: &length,
|
||||
dataPointerOut: &ptr) == noErr, let ptr else { return nil }
|
||||
return Data(bytes: ptr, count: length)
|
||||
}
|
||||
|
||||
/// AVCC sample -> Annex B access unit, with SPS and PPS before keyframes.
|
||||
static func annexB(_ sample: CMSampleBuffer) -> (Data, Bool)? {
|
||||
guard let avcc = bytes(sample) else { return nil }
|
||||
var key = true
|
||||
if let atts = CMSampleBufferGetSampleAttachmentsArray(sample, createIfNecessary: false) as? [[CFString: Any]],
|
||||
let first = atts.first, first[kCMSampleAttachmentKey_NotSync] as? Bool == true {
|
||||
key = false
|
||||
}
|
||||
let start: [UInt8] = [0, 0, 0, 1]
|
||||
var out = Data()
|
||||
if key, let fmt = CMSampleBufferGetFormatDescription(sample) {
|
||||
var count = 0
|
||||
CMVideoFormatDescriptionGetH264ParameterSetAtIndex(fmt, parameterSetIndex: 0, parameterSetPointerOut: nil,
|
||||
parameterSetSizeOut: nil, parameterSetCountOut: &count,
|
||||
nalUnitHeaderLengthOut: nil)
|
||||
for i in 0..<count {
|
||||
var p: UnsafePointer<UInt8>?
|
||||
var n = 0
|
||||
if CMVideoFormatDescriptionGetH264ParameterSetAtIndex(fmt, parameterSetIndex: i, parameterSetPointerOut: &p,
|
||||
parameterSetSizeOut: &n, parameterSetCountOut: nil,
|
||||
nalUnitHeaderLengthOut: nil) == noErr, let p {
|
||||
out.append(contentsOf: start)
|
||||
out.append(p, count: n)
|
||||
}
|
||||
}
|
||||
}
|
||||
let bytes = [UInt8](avcc)
|
||||
var i = 0
|
||||
while i + 4 <= bytes.count {
|
||||
let n = Int(bytes[i]) << 24 | Int(bytes[i + 1]) << 16 | Int(bytes[i + 2]) << 8 | Int(bytes[i + 3])
|
||||
i += 4
|
||||
guard n > 0, i + n <= bytes.count else { break }
|
||||
out.append(contentsOf: start)
|
||||
out.append(contentsOf: bytes[i..<(i + n)])
|
||||
i += n
|
||||
}
|
||||
return (out, key)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,168 @@
|
||||
// Turns the viewer's pointer and key events into real Mac input. Needs the
|
||||
// Accessibility permission ("control your computer"); without it macOS drops
|
||||
// the events silently, so the agent reports the permission to the viewer.
|
||||
import AppKit
|
||||
import ApplicationServices
|
||||
import Foundation
|
||||
|
||||
@_silgen_name("_AXUIElementGetWindow")
|
||||
private func _AXUIElementGetWindow(_ element: AXUIElement, _ id: UnsafeMutablePointer<CGWindowID>) -> AXError
|
||||
|
||||
enum Input {
|
||||
static let source = CGEventSource(stateID: .hidSystemState)
|
||||
/// What each viewer (session id) is holding down, so one panel closing
|
||||
/// lets go of its own keys and buttons and nobody else's. Main thread.
|
||||
private static var buttonsHeld: [Int: Set<Int>] = [:]
|
||||
private static var keysHeld: [Int: Set<CGKeyCode>] = [:]
|
||||
|
||||
private static var lastDown: (time: TimeInterval, point: CGPoint, button: Int, count: Int)?
|
||||
|
||||
static var allowed: Bool { AXIsProcessTrusted() }
|
||||
|
||||
/// Brings a window to the front so a click lands on it, not on whatever
|
||||
/// covers it, and typing goes to it.
|
||||
static func focus(window id: CGWindowID, pid: pid_t) {
|
||||
if frontWindow() == id { return }
|
||||
let app = AXUIElementCreateApplication(pid)
|
||||
var value: CFTypeRef?
|
||||
if AXUIElementCopyAttributeValue(app, kAXWindowsAttribute as CFString, &value) == .success,
|
||||
let windows = value as? [AXUIElement] {
|
||||
for w in windows {
|
||||
var wid: CGWindowID = 0
|
||||
if _AXUIElementGetWindow(w, &wid) == .success, wid == id {
|
||||
AXUIElementPerformAction(w, kAXRaiseAction as CFString)
|
||||
AXUIElementSetAttributeValue(w, kAXMainAttribute as CFString, kCFBooleanTrue)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
AXUIElementSetAttributeValue(app, kAXFrontmostAttribute as CFString, kCFBooleanTrue)
|
||||
NSRunningApplication(processIdentifier: pid)?.activate()
|
||||
}
|
||||
|
||||
static func frontWindow() -> CGWindowID? {
|
||||
WindowInfo.all().first { $0.layer == 0 }?.id
|
||||
}
|
||||
|
||||
/// `button` uses the browser's numbering: 0 left, 1 middle, 2 right.
|
||||
static func mouse(_ kind: String, button: Int, at p: CGPoint, owner: Int) {
|
||||
let b: CGMouseButton = button == 2 ? .right : button == 1 ? .center : .left
|
||||
let type: CGEventType
|
||||
switch kind {
|
||||
case "down":
|
||||
type = b == .left ? .leftMouseDown : b == .right ? .rightMouseDown : .otherMouseDown
|
||||
buttonsHeld[owner, default: []].insert(button)
|
||||
case "up":
|
||||
type = b == .left ? .leftMouseUp : b == .right ? .rightMouseUp : .otherMouseUp
|
||||
buttonsHeld[owner]?.remove(button)
|
||||
// Another viewer still holding it keeps it down.
|
||||
if buttonsHeld.values.contains(where: { $0.contains(button) }) { return }
|
||||
default: // a drag is whatever this viewer is holding
|
||||
let mine = buttonsHeld[owner] ?? []
|
||||
if mine.contains(0) { type = .leftMouseDragged }
|
||||
else if mine.contains(2) { type = .rightMouseDragged }
|
||||
else if mine.contains(1) { type = .otherMouseDragged }
|
||||
else { type = .mouseMoved }
|
||||
}
|
||||
let mine = buttonsHeld[owner] ?? []
|
||||
let held: CGMouseButton = mine.contains(0) ? .left : mine.contains(2) ? .right : mine.contains(1) ? .center : .left
|
||||
guard let e = CGEvent(mouseEventSource: source, mouseType: type, mouseCursorPosition: p,
|
||||
mouseButton: kind == "move" ? held : b) else { return }
|
||||
if kind == "down" || kind == "up" {
|
||||
let now = ProcessInfo.processInfo.systemUptime
|
||||
var count = 1
|
||||
if kind == "down" {
|
||||
if let l = lastDown, l.button == button, now - l.time < NSEvent.doubleClickInterval,
|
||||
abs(l.point.x - p.x) < 5, abs(l.point.y - p.y) < 5 { count = l.count + 1 }
|
||||
lastDown = (now, p, button, count)
|
||||
} else if let l = lastDown, l.button == button {
|
||||
count = l.count
|
||||
}
|
||||
e.setIntegerValueField(.mouseEventClickState, value: Int64(count))
|
||||
}
|
||||
e.post(tap: .cghidEventTap)
|
||||
}
|
||||
|
||||
static func scroll(dx: Double, dy: Double, at p: CGPoint, owner: Int) {
|
||||
mouse("move", button: 0, at: p, owner: owner)
|
||||
// Browsers report pixels with +y meaning "scroll down"; macOS's +y is up.
|
||||
guard let e = CGEvent(scrollWheelEvent2Source: source, units: .pixel, wheelCount: 2,
|
||||
wheel1: Int32(-dy.rounded()), wheel2: Int32(-dx.rounded()), wheel3: 0) else { return }
|
||||
e.location = p
|
||||
e.post(tap: .cghidEventTap)
|
||||
}
|
||||
|
||||
/// Lets go of every button and key this viewer is holding down, so a
|
||||
/// dropped connection can't leave, say, Shift or ⌘ stuck on.
|
||||
static func releaseAll(owner: Int) {
|
||||
let p = CGEvent(source: nil)?.location ?? .zero
|
||||
let buttons = buttonsHeld.removeValue(forKey: owner) ?? []
|
||||
let keys = keysHeld.removeValue(forKey: owner) ?? []
|
||||
// Only what no other viewer is still holding.
|
||||
for b in buttons where !buttonsHeld.values.contains(where: { $0.contains(b) }) {
|
||||
let type: CGEventType = b == 2 ? .rightMouseUp : b == 1 ? .otherMouseUp : .leftMouseUp
|
||||
CGEvent(mouseEventSource: source, mouseType: type, mouseCursorPosition: p,
|
||||
mouseButton: b == 2 ? .right : b == 1 ? .center : .left)?.post(tap: .cghidEventTap)
|
||||
}
|
||||
for k in keys where !keysHeld.values.contains(where: { $0.contains(k) }) {
|
||||
CGEvent(keyboardEventSource: source, virtualKey: k, keyDown: false)?.post(tap: .cghidEventTap)
|
||||
}
|
||||
}
|
||||
|
||||
static func key(code: String, key: String, down: Bool, mods: [String], owner: Int) {
|
||||
var flags = CGEventFlags()
|
||||
if mods.contains("shift") { flags.insert(.maskShift) }
|
||||
if mods.contains("ctrl") { flags.insert(.maskControl) }
|
||||
if mods.contains("alt") { flags.insert(.maskAlternate) }
|
||||
if mods.contains("meta") { flags.insert(.maskCommand) }
|
||||
if let vk = keyCodes[code] {
|
||||
guard let e = CGEvent(keyboardEventSource: source, virtualKey: vk, keyDown: down) else { return }
|
||||
if down {
|
||||
keysHeld[owner, default: []].insert(vk)
|
||||
} else {
|
||||
keysHeld[owner]?.remove(vk)
|
||||
if keysHeld.values.contains(where: { $0.contains(vk) }) { return } // still held elsewhere
|
||||
}
|
||||
e.flags = flags
|
||||
e.post(tap: .cghidEventTap)
|
||||
} else if down, !key.isEmpty, key.count <= 4, key.unicodeScalars.allSatisfy({ $0.value >= 0x20 }) {
|
||||
// A key the table doesn't know (a non-US layout, the headset's
|
||||
// on-screen keyboard): type its character instead.
|
||||
text(key)
|
||||
}
|
||||
}
|
||||
|
||||
static func text(_ s: String) {
|
||||
let units = Array(s.utf16)
|
||||
for chunk in stride(from: 0, to: units.count, by: 16) {
|
||||
let part = Array(units[chunk..<min(chunk + 16, units.count)])
|
||||
for down in [true, false] {
|
||||
guard let e = CGEvent(keyboardEventSource: source, virtualKey: 0, keyDown: down) else { continue }
|
||||
e.keyboardSetUnicodeString(stringLength: part.count, unicodeString: part)
|
||||
e.post(tap: .cghidEventTap)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// DOM KeyboardEvent.code -> macOS virtual key code (ANSI positions).
|
||||
static let keyCodes: [String: CGKeyCode] = [
|
||||
"KeyA": 0x00, "KeyS": 0x01, "KeyD": 0x02, "KeyF": 0x03, "KeyH": 0x04, "KeyG": 0x05, "KeyZ": 0x06, "KeyX": 0x07,
|
||||
"KeyC": 0x08, "KeyV": 0x09, "IntlBackslash": 0x0A, "KeyB": 0x0B, "KeyQ": 0x0C, "KeyW": 0x0D, "KeyE": 0x0E,
|
||||
"KeyR": 0x0F, "KeyY": 0x10, "KeyT": 0x11, "Digit1": 0x12, "Digit2": 0x13, "Digit3": 0x14, "Digit4": 0x15,
|
||||
"Digit6": 0x16, "Digit5": 0x17, "Equal": 0x18, "Digit9": 0x19, "Digit7": 0x1A, "Minus": 0x1B, "Digit8": 0x1C,
|
||||
"Digit0": 0x1D, "BracketRight": 0x1E, "KeyO": 0x1F, "KeyU": 0x20, "BracketLeft": 0x21, "KeyI": 0x22,
|
||||
"KeyP": 0x23, "Enter": 0x24, "KeyL": 0x25, "KeyJ": 0x26, "Quote": 0x27, "KeyK": 0x28, "Semicolon": 0x29,
|
||||
"Backslash": 0x2A, "Comma": 0x2B, "Slash": 0x2C, "KeyN": 0x2D, "KeyM": 0x2E, "Period": 0x2F, "Tab": 0x30,
|
||||
"Space": 0x31, "Backquote": 0x32, "Backspace": 0x33, "Escape": 0x35, "MetaRight": 0x36, "MetaLeft": 0x37,
|
||||
"ShiftLeft": 0x38, "CapsLock": 0x39, "AltLeft": 0x3A, "ControlLeft": 0x3B, "ShiftRight": 0x3C,
|
||||
"AltRight": 0x3D, "ControlRight": 0x3E, "F17": 0x40, "NumpadDecimal": 0x41, "NumpadMultiply": 0x43,
|
||||
"NumpadAdd": 0x45, "NumLock": 0x47, "NumpadDivide": 0x4B, "NumpadEnter": 0x4C, "NumpadSubtract": 0x4E,
|
||||
"F18": 0x4F, "F19": 0x50, "NumpadEqual": 0x51, "Numpad0": 0x52, "Numpad1": 0x53, "Numpad2": 0x54,
|
||||
"Numpad3": 0x55, "Numpad4": 0x56, "Numpad5": 0x57, "Numpad6": 0x58, "Numpad7": 0x59, "F20": 0x5A,
|
||||
"Numpad8": 0x5B, "Numpad9": 0x5C, "F5": 0x60, "F6": 0x61, "F7": 0x62, "F3": 0x63, "F8": 0x64, "F9": 0x65,
|
||||
"F11": 0x67, "F13": 0x69, "F16": 0x6A, "F14": 0x6B, "F10": 0x6D, "ContextMenu": 0x6E, "F12": 0x6F,
|
||||
"F15": 0x71, "Insert": 0x72, "Help": 0x72, "Home": 0x73, "PageUp": 0x74, "Delete": 0x75, "F4": 0x76,
|
||||
"End": 0x77, "F2": 0x78, "PageDown": 0x79, "F1": 0x7A, "ArrowLeft": 0x7B, "ArrowRight": 0x7C,
|
||||
"ArrowDown": 0x7D, "ArrowUp": 0x7E, "OSLeft": 0x37, "OSRight": 0x36,
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,267 @@
|
||||
// A small HTTP/1.1 + WebSocket server on Network.framework, loopback only.
|
||||
// Enough for the agent's JSON endpoints, the viewer page and one WebSocket per
|
||||
// stream; not a general web server.
|
||||
import CryptoKit
|
||||
import Foundation
|
||||
import Network
|
||||
|
||||
struct Request {
|
||||
let method: String
|
||||
let path: String
|
||||
let query: [String: String]
|
||||
let headers: [String: String] // lower-cased names
|
||||
}
|
||||
|
||||
final class Server {
|
||||
let queue = DispatchQueue(label: "frame-mac-view.server")
|
||||
private let listener: NWListener
|
||||
private let handle: (Request, HTTPConnection) -> Void
|
||||
|
||||
init(port: UInt16, handle: @escaping (Request, HTTPConnection) -> Void) throws {
|
||||
let tcp = NWProtocolTCP.Options()
|
||||
tcp.noDelay = true
|
||||
let params = NWParameters(tls: nil, tcp: tcp)
|
||||
// Port 0 lets the system pick; `port` then says which.
|
||||
params.requiredLocalEndpoint = .hostPort(host: "127.0.0.1", port: NWEndpoint.Port(rawValue: port) ?? .any)
|
||||
params.allowLocalEndpointReuse = true
|
||||
listener = try NWListener(using: params)
|
||||
self.handle = handle
|
||||
}
|
||||
|
||||
var port: UInt16? { listener.port?.rawValue }
|
||||
|
||||
func start(ready: @escaping (Error?) -> Void) {
|
||||
listener.stateUpdateHandler = { state in
|
||||
switch state {
|
||||
case .ready: ready(nil)
|
||||
case .failed(let e): ready(e)
|
||||
default: break
|
||||
}
|
||||
}
|
||||
listener.newConnectionHandler = { [weak self] conn in
|
||||
guard let self else { return }
|
||||
HTTPConnection(conn, queue: self.queue, handle: self.handle).start()
|
||||
}
|
||||
listener.start(queue: queue)
|
||||
}
|
||||
}
|
||||
|
||||
final class HTTPConnection {
|
||||
let conn: NWConnection
|
||||
let queue: DispatchQueue
|
||||
private let handle: (Request, HTTPConnection) -> Void
|
||||
private var buffer = Data()
|
||||
private var retained: HTTPConnection? // alive until the response is sent
|
||||
|
||||
init(_ conn: NWConnection, queue: DispatchQueue, handle: @escaping (Request, HTTPConnection) -> Void) {
|
||||
self.conn = conn
|
||||
self.queue = queue
|
||||
self.handle = handle
|
||||
}
|
||||
|
||||
func start() {
|
||||
retained = self
|
||||
conn.start(queue: queue)
|
||||
readHead()
|
||||
}
|
||||
|
||||
private func readHead() {
|
||||
conn.receive(minimumIncompleteLength: 1, maximumLength: 16384) { [self] data, _, done, error in
|
||||
if let data { buffer.append(data) }
|
||||
if let end = buffer.range(of: Data("\r\n\r\n".utf8)) {
|
||||
guard let req = Self.parse(buffer[..<end.lowerBound]) else { return respond(400, text: "bad request") }
|
||||
handle(req, self)
|
||||
} else if error != nil || done || buffer.count > 16384 {
|
||||
close()
|
||||
} else {
|
||||
readHead()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static func parse(_ head: Data) -> Request? {
|
||||
guard let text = String(data: head, encoding: .utf8) else { return nil }
|
||||
let lines = text.components(separatedBy: "\r\n")
|
||||
let parts = lines[0].split(separator: " ")
|
||||
guard parts.count >= 2, let url = URLComponents(string: String(parts[1])) else { return nil }
|
||||
var headers: [String: String] = [:]
|
||||
for line in lines.dropFirst() {
|
||||
guard let colon = line.firstIndex(of: ":") else { continue }
|
||||
headers[line[..<colon].lowercased()] = line[line.index(after: colon)...].trimmingCharacters(in: .whitespaces)
|
||||
}
|
||||
var query: [String: String] = [:]
|
||||
for item in url.queryItems ?? [] { query[item.name] = item.value ?? "" }
|
||||
return Request(method: String(parts[0]), path: url.path, query: query, headers: headers)
|
||||
}
|
||||
|
||||
func respond(_ status: Int, body: Data, type: String) {
|
||||
let reason = [200: "OK", 400: "Bad Request", 403: "Forbidden", 404: "Not Found", 409: "Conflict", 500: "Internal Server Error"][status] ?? "Error"
|
||||
var head = "HTTP/1.1 \(status) \(reason)\r\nContent-Type: \(type)\r\nContent-Length: \(body.count)\r\n"
|
||||
head += "Cache-Control: no-store\r\nConnection: close\r\n\r\n"
|
||||
conn.send(content: Data(head.utf8) + body, isComplete: true, completion: .contentProcessed { [self] _ in close() })
|
||||
}
|
||||
|
||||
func respond(_ status: Int, text: String) {
|
||||
respond(status, body: Data(text.utf8), type: "text/plain; charset=utf-8")
|
||||
}
|
||||
|
||||
func respond(_ status: Int = 200, json: Any) {
|
||||
let body = (try? JSONSerialization.data(withJSONObject: json, options: [.sortedKeys])) ?? Data("{}".utf8)
|
||||
respond(status, body: body, type: "application/json")
|
||||
}
|
||||
|
||||
func close() {
|
||||
conn.cancel()
|
||||
retained = nil
|
||||
}
|
||||
|
||||
/// Completes the WebSocket handshake and hands the connection over.
|
||||
func upgrade(_ req: Request) -> WebSocket? {
|
||||
guard req.headers["upgrade"]?.lowercased() == "websocket", let key = req.headers["sec-websocket-key"] else {
|
||||
respond(400, text: "expected a WebSocket upgrade")
|
||||
return nil
|
||||
}
|
||||
let accept = Data(Insecure.SHA1.hash(data: Data((key + "258EAFA5-E914-47DA-95CA-C5AB0DC85B11").utf8))).base64EncodedString()
|
||||
let head = "HTTP/1.1 101 Switching Protocols\r\nUpgrade: websocket\r\nConnection: Upgrade\r\nSec-WebSocket-Accept: \(accept)\r\n\r\n"
|
||||
conn.send(content: Data(head.utf8), completion: .contentProcessed { _ in })
|
||||
let ws = WebSocket(conn, queue: queue)
|
||||
retained = nil
|
||||
return ws
|
||||
}
|
||||
}
|
||||
|
||||
/// Server side of RFC 6455. Sends are counted until the network stack has
|
||||
/// taken them, so the stream can skip frames instead of queueing seconds of
|
||||
/// video on a slow link.
|
||||
final class WebSocket {
|
||||
static let maxMessage: UInt64 = 1 << 20
|
||||
let conn: NWConnection
|
||||
let queue: DispatchQueue
|
||||
var onText: ((String) -> Void)?
|
||||
var onClose: (() -> Void)?
|
||||
private var buffer = Data()
|
||||
private var fragments = Data()
|
||||
private var fragmentOpcode: UInt8 = 0
|
||||
private var closed = false
|
||||
private var retained: WebSocket?
|
||||
private let lock = NSLock()
|
||||
private var _pending = 0
|
||||
var onDrain: (() -> Void)?
|
||||
|
||||
/// Bytes handed to the connection that it hasn't sent yet. Any thread.
|
||||
var pendingBytes: Int { lock.lock(); defer { lock.unlock() }; return _pending }
|
||||
|
||||
init(_ conn: NWConnection, queue: DispatchQueue) {
|
||||
self.conn = conn
|
||||
self.queue = queue
|
||||
}
|
||||
|
||||
func start() {
|
||||
retained = self
|
||||
read()
|
||||
}
|
||||
|
||||
func sendText(_ s: String) { send(opcode: 1, Data(s.utf8)) }
|
||||
func sendJSON(_ obj: Any) {
|
||||
if let d = try? JSONSerialization.data(withJSONObject: obj), let s = String(data: d, encoding: .utf8) { sendText(s) }
|
||||
}
|
||||
func sendBinary(_ d: Data) { send(opcode: 2, d) }
|
||||
|
||||
func send(opcode: UInt8, _ payload: Data) {
|
||||
var frame = Data([0x80 | opcode])
|
||||
let n = payload.count
|
||||
if n < 126 {
|
||||
frame.append(UInt8(n))
|
||||
} else if n < 65536 {
|
||||
frame.append(126)
|
||||
frame.append(contentsOf: [UInt8(n >> 8), UInt8(n & 0xff)])
|
||||
} else {
|
||||
frame.append(127)
|
||||
for shift in stride(from: 56, through: 0, by: -8) { frame.append(UInt8((UInt64(n) >> UInt64(shift)) & 0xff)) }
|
||||
}
|
||||
frame.append(payload)
|
||||
let size = frame.count
|
||||
lock.lock(); _pending += size; lock.unlock()
|
||||
conn.send(content: frame, completion: .contentProcessed { [weak self] _ in
|
||||
guard let self else { return }
|
||||
self.lock.lock(); self._pending -= size; self.lock.unlock()
|
||||
self.onDrain?()
|
||||
})
|
||||
}
|
||||
|
||||
func close() {
|
||||
guard !closed else { return }
|
||||
closed = true
|
||||
send(opcode: 8, Data([0x03, 0xe8])) // 1000, normal closure
|
||||
conn.send(content: nil, isComplete: true, completion: .contentProcessed { [weak self] _ in self?.conn.cancel() })
|
||||
finish()
|
||||
}
|
||||
|
||||
private func finish() {
|
||||
let cb = onClose
|
||||
onClose = nil
|
||||
onText = nil
|
||||
onDrain = nil
|
||||
cb?()
|
||||
retained = nil
|
||||
}
|
||||
|
||||
private func read() {
|
||||
conn.receive(minimumIncompleteLength: 1, maximumLength: 65536) { [weak self] data, _, done, error in
|
||||
guard let self, !self.closed else { return }
|
||||
if let data { self.buffer.append(data) }
|
||||
self.parse()
|
||||
if error != nil || done {
|
||||
self.closed = true
|
||||
self.conn.cancel()
|
||||
self.finish()
|
||||
} else if !self.closed {
|
||||
self.read()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private func parse() {
|
||||
while buffer.count >= 2 {
|
||||
let b = [UInt8](buffer.prefix(14))
|
||||
let fin = b[0] & 0x80 != 0, opcode = b[0] & 0x0f, masked = b[1] & 0x80 != 0
|
||||
// Lengths are unsigned and clients only send small control
|
||||
// messages, so anything big (or a 64-bit length with the top bit
|
||||
// set) is refused before it's turned into an Int.
|
||||
var len64 = UInt64(b[1] & 0x7f), off = 2
|
||||
if len64 == 126 {
|
||||
guard b.count >= 4 else { return }
|
||||
len64 = UInt64(b[2]) << 8 | UInt64(b[3]); off = 4
|
||||
} else if len64 == 127 {
|
||||
guard b.count >= 10 else { return }
|
||||
len64 = 0
|
||||
for i in 2..<10 { len64 = len64 << 8 | UInt64(b[i]) }
|
||||
off = 10
|
||||
}
|
||||
guard len64 <= WebSocket.maxMessage else { return close() }
|
||||
let len = Int(len64)
|
||||
let maskOff = off
|
||||
if masked { off += 4 }
|
||||
guard buffer.count >= off + len else { return }
|
||||
let start = buffer.startIndex
|
||||
var payload = Data(buffer[(start + off)..<(start + off + len)])
|
||||
if masked {
|
||||
let mask = [UInt8](buffer[(start + maskOff)..<(start + maskOff + 4)])
|
||||
payload.withUnsafeMutableBytes { p in
|
||||
for i in 0..<len { p[i] ^= mask[i & 3] }
|
||||
}
|
||||
}
|
||||
buffer.removeFirst(off + len)
|
||||
switch opcode {
|
||||
case 0, 1, 2:
|
||||
if opcode != 0 { fragmentOpcode = opcode; fragments = Data() }
|
||||
guard fragments.count + payload.count <= Int(WebSocket.maxMessage) else { return close() }
|
||||
fragments.append(payload)
|
||||
if fin, fragmentOpcode == 1, let s = String(data: fragments, encoding: .utf8) { onText?(s) }
|
||||
case 8: close(); return
|
||||
case 9: send(opcode: 10, payload)
|
||||
default: break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,490 @@
|
||||
// frame-mac-view: streams Mac windows or displays to viewers on the Steam
|
||||
// Frame and plays their pointer and key input back on the Mac.
|
||||
//
|
||||
// frame-mac-view serve --port 47811 --page ui/mac-view.html (token in FRAME_MAC_VIEW_TOKEN)
|
||||
// frame-mac-view windows | displays | permissions (JSON on stdout)
|
||||
// frame-mac-view request-permissions (shows macOS's prompts)
|
||||
//
|
||||
// It listens on 127.0.0.1 only. Frame Control reaches it from the Frame
|
||||
// through an SSH reverse tunnel, and every request must carry the token.
|
||||
//
|
||||
// HTTP. Frame Control's key (?k=, from FRAME_MAC_VIEW_TOKEN) never leaves the
|
||||
// Mac; the Frame gets a single-use ticket per viewer instead.
|
||||
// GET /ping, /view open: tunnel check, viewer page
|
||||
// GET /stream?src=...&t=TICKET|r=KEY WebSocket (&codec=h264|jpeg&max=&fps=&bpp=)
|
||||
// GET /status, /windows, /displays ?k=: permissions, streams, sources
|
||||
// POST /ticket?src=... ?k=: a ticket for one viewer of src
|
||||
// POST /close[?src=...] ?k=: end those streams, close their windows
|
||||
// POST /permissions ?k=: show macOS's permission prompts
|
||||
// src is window:<CGWindowID>, display:<CGDirectDisplayID> or test.
|
||||
import AppKit
|
||||
import ApplicationServices
|
||||
import Foundation
|
||||
import IOKit.pwr_mgt
|
||||
import ScreenCaptureKit
|
||||
import Security
|
||||
|
||||
let version = "1"
|
||||
|
||||
func windowsJSON() -> [[String: Any]] {
|
||||
let me = ProcessInfo.processInfo.processIdentifier
|
||||
let skip: Set<String> = ["Window Server", "Dock", "Control Centre", "Control Center", "Notification Centre",
|
||||
"Notification Center", "Spotlight", "SystemUIServer", "Wallpaper", "WindowManager"]
|
||||
return WindowInfo.all().filter {
|
||||
$0.layer == 0 && $0.pid != me && !skip.contains($0.app) && $0.bounds.width >= 120 && $0.bounds.height >= 80
|
||||
}.map {
|
||||
["id": $0.id, "src": "window:\($0.id)", "pid": $0.pid, "app": $0.app, "title": $0.title,
|
||||
"w": Int($0.bounds.width), "h": Int($0.bounds.height)]
|
||||
}
|
||||
}
|
||||
|
||||
func displaysJSON() -> [[String: Any]] {
|
||||
var ids = [CGDirectDisplayID](repeating: 0, count: 16)
|
||||
var n: UInt32 = 0
|
||||
// Online, not active: a display that's asleep is still one you can stream.
|
||||
CGGetOnlineDisplayList(16, &ids, &n)
|
||||
return ids.prefix(Int(n)).filter { CGDisplayMirrorsDisplay($0) == kCGNullDirectDisplay }.map { id in
|
||||
let b = CGDisplayBounds(id)
|
||||
return ["id": id, "src": "display:\(id)", "name": displayName(id), "w": Int(b.width), "h": Int(b.height),
|
||||
"main": CGDisplayIsMain(id) != 0]
|
||||
}
|
||||
}
|
||||
|
||||
func permissionsJSON() -> [String: Any] {
|
||||
["screen": CGPreflightScreenCaptureAccess(), "accessibility": AXIsProcessTrusted()]
|
||||
}
|
||||
|
||||
func printJSON(_ obj: Any) {
|
||||
let d = (try? JSONSerialization.data(withJSONObject: obj, options: [.sortedKeys, .prettyPrinted])) ?? Data()
|
||||
FileHandle.standardOutput.write(d + Data("\n".utf8))
|
||||
}
|
||||
|
||||
/// One viewer watching one source.
|
||||
final class Session {
|
||||
let id: Int
|
||||
let source: Source
|
||||
let capture: CaptureSource
|
||||
let encoder: Encoder
|
||||
let ws: WebSocket
|
||||
let codec: Codec
|
||||
let reconnectKey: String
|
||||
private let lock = NSLock()
|
||||
private var last: (CVPixelBuffer, CMTime)?
|
||||
private var skipped = false
|
||||
private var stopped = false
|
||||
private var inFlight = 0
|
||||
/// Frames already queued for the network; beyond this, or with two frames
|
||||
/// already in the encoder, new frames are skipped (before encoding, so no
|
||||
/// reference frame goes missing) and the newest picture is sent once
|
||||
/// things catch up. Only that newest picture is kept meanwhile.
|
||||
let maxPending: Int
|
||||
static let maxInFlight = 2
|
||||
var onEnd: ((Session) -> Void)?
|
||||
var onAck: (() -> Void)?
|
||||
var onFinished: ((String) -> Void)?
|
||||
private let encodeQueue = DispatchQueue(label: "frame-mac-view.encode", qos: .userInteractive)
|
||||
|
||||
init(id: Int, source: Source, capture: CaptureSource, ws: WebSocket, codec: Codec, fps: Int, bitsPerPixel: Double,
|
||||
reconnectKey: String) {
|
||||
self.id = id
|
||||
self.source = source
|
||||
self.capture = capture
|
||||
self.ws = ws
|
||||
self.codec = codec
|
||||
self.reconnectKey = reconnectKey
|
||||
encoder = Encoder(codec: codec, fps: fps, bitsPerPixel: bitsPerPixel)
|
||||
maxPending = codec == .jpeg ? 3 << 20 : 1 << 20
|
||||
}
|
||||
|
||||
/// Encodes `pb` unless the link or the encoder is busy, in which case it
|
||||
/// becomes the picture to send next.
|
||||
private func offer(_ pb: CVPixelBuffer, pts: CMTime) {
|
||||
lock.lock()
|
||||
last = (pb, pts)
|
||||
let busy = stopped || ws.pendingBytes > maxPending || inFlight >= Session.maxInFlight
|
||||
if busy { skipped = true } else { inFlight += 1 }
|
||||
lock.unlock()
|
||||
if !busy { submit(pb, pts: pts) }
|
||||
}
|
||||
|
||||
private func submit(_ pb: CVPixelBuffer, pts: CMTime) {
|
||||
encodeQueue.async {
|
||||
if !self.encoder.encode(pb, pts: pts) { self.finished() }
|
||||
}
|
||||
}
|
||||
|
||||
/// An encode finished (or failed): send the newest skipped picture if
|
||||
/// there's room now.
|
||||
private func finished() {
|
||||
lock.lock()
|
||||
inFlight = max(0, inFlight - 1)
|
||||
lock.unlock()
|
||||
resendIfRoom()
|
||||
}
|
||||
|
||||
private func resendIfRoom() {
|
||||
lock.lock()
|
||||
var next: (CVPixelBuffer, CMTime)?
|
||||
if !stopped, skipped, ws.pendingBytes <= maxPending / 2, inFlight < Session.maxInFlight, let l = last {
|
||||
next = l
|
||||
skipped = false
|
||||
inFlight += 1
|
||||
}
|
||||
lock.unlock()
|
||||
if let (pb, _) = next { submit(pb, pts: CMClockGetTime(CMClockGetHostTimeClock())) }
|
||||
}
|
||||
|
||||
func start(maxLong: Int, fps: Int) {
|
||||
encoder.onFrame = { [weak self] data, key, pts in
|
||||
guard let self else { return }
|
||||
var msg = Data([key ? 1 : 0])
|
||||
var us = UInt64(max(0, CMTimeGetSeconds(pts)) * 1_000_000).bigEndian
|
||||
msg.append(Data(bytes: &us, count: 8))
|
||||
msg.append(data)
|
||||
self.ws.sendBinary(msg)
|
||||
}
|
||||
encoder.onDone = { [weak self] in self?.finished() }
|
||||
encoder.onError = { [weak self] message in self?.ws.sendJSON(["t": "error", "message": message]) }
|
||||
capture.onFrame = { [weak self] pb, pts in self?.offer(pb, pts: pts) }
|
||||
capture.onEnded = { [weak self] reason in
|
||||
guard let self else { return }
|
||||
self.ws.sendJSON(["t": "closed", "reason": reason])
|
||||
self.onFinished?(self.source.key)
|
||||
self.end()
|
||||
}
|
||||
ws.onDrain = { [weak self] in self?.resendIfRoom() }
|
||||
ws.onText = { [weak self] text in self?.handle(text) }
|
||||
ws.onClose = { [weak self] in self?.end() }
|
||||
ws.start()
|
||||
// Reconnect with this (kept in the page's memory, never on a command line).
|
||||
ws.sendJSON(["t": "hello", "r": reconnectKey])
|
||||
if let sck = capture as? SCKSource {
|
||||
sck.onChange = { [weak self] in self?.sendInfo() }
|
||||
}
|
||||
capture.start(maxLong: maxLong, fps: fps) { [weak self] error in
|
||||
guard let self else { return }
|
||||
if let error {
|
||||
if (self.capture as? SCKSource)?.gone == true {
|
||||
// Final, like a window closing mid-stream: the viewer closes
|
||||
// and its key is revoked, rather than retrying for ever.
|
||||
self.ws.sendJSON(["t": "closed", "reason": error])
|
||||
self.onFinished?(self.source.key)
|
||||
} else {
|
||||
self.ws.sendJSON(["t": "error", "message": error])
|
||||
}
|
||||
self.end()
|
||||
return
|
||||
}
|
||||
self.sendInfo()
|
||||
}
|
||||
}
|
||||
|
||||
func sendInfo() {
|
||||
ws.sendJSON(["t": "info", "src": source.key, "title": capture.title, "app": capture.app, "codec": codec.rawValue,
|
||||
"input": source == .test || Input.allowed,
|
||||
"aspect": Double(capture.frameRect.width / max(capture.frameRect.height, 1))])
|
||||
}
|
||||
|
||||
var isStopped: Bool { lock.lock(); defer { lock.unlock() }; return stopped }
|
||||
|
||||
func end() {
|
||||
lock.lock()
|
||||
let was = stopped
|
||||
stopped = true
|
||||
last = nil
|
||||
lock.unlock()
|
||||
guard !was else { return }
|
||||
capture.stop()
|
||||
encodeQueue.async { self.encoder.invalidate() }
|
||||
let owner = id
|
||||
DispatchQueue.main.async { Input.releaseAll(owner: owner) }
|
||||
ws.close()
|
||||
onEnd?(self)
|
||||
}
|
||||
|
||||
/// A point in the picture (0...1 each way) -> Mac global coordinates.
|
||||
private func point(_ x: Double, _ y: Double) -> CGPoint {
|
||||
let r = capture.frameRect
|
||||
return CGPoint(x: r.minX + min(max(x, 0), 1) * r.width, y: r.minY + min(max(y, 0), 1) * r.height)
|
||||
}
|
||||
|
||||
private func handle(_ text: String) {
|
||||
guard !isStopped, let d = text.data(using: .utf8),
|
||||
let m = try? JSONSerialization.jsonObject(with: d) as? [String: Any], let t = m["t"] as? String else { return }
|
||||
let x = m["x"] as? Double ?? -1, y = m["y"] as? Double ?? -1
|
||||
if t == "ack" {
|
||||
onAck?()
|
||||
onAck = nil
|
||||
return
|
||||
}
|
||||
if t == "key-frame" {
|
||||
encodeQueue.async { self.encoder.requestKeyFrame() } // before the resend, same queue
|
||||
lock.lock(); skipped = last != nil; lock.unlock()
|
||||
resendIfRoom()
|
||||
return
|
||||
}
|
||||
if source == .test {
|
||||
let desc: String?
|
||||
switch t {
|
||||
case "m": desc = (m["e"] as? String) == "move" ? nil : "mouse \(m["e"] ?? "") button \(m["b"] ?? 0)"
|
||||
case "wheel": desc = "wheel \(m["dx"] ?? 0), \(m["dy"] ?? 0)"
|
||||
case "k": desc = (m["e"] as? String) == "down" ? "key \(m["code"] ?? "") \"\(m["key"] ?? "")\"" : nil
|
||||
case "text": desc = "text \"\(m["s"] ?? "")\""
|
||||
default: desc = nil
|
||||
}
|
||||
capture.pointer(x: x, y: y, text: desc)
|
||||
return
|
||||
}
|
||||
DispatchQueue.main.async { [self] in
|
||||
guard !isStopped else { return } // Stop revokes input at once
|
||||
switch t {
|
||||
case "m":
|
||||
let kind = m["e"] as? String ?? "move", b = m["b"] as? Int ?? 0
|
||||
let p = point(x, y)
|
||||
if kind == "down", case .window(let wid) = source, let pid = capture.pid {
|
||||
Input.focus(window: wid, pid: pid)
|
||||
}
|
||||
Input.mouse(kind, button: b, at: p, owner: id)
|
||||
case "wheel":
|
||||
Input.scroll(dx: m["dx"] as? Double ?? 0, dy: m["dy"] as? Double ?? 0, at: point(x, y), owner: id)
|
||||
case "k":
|
||||
Input.key(code: m["code"] as? String ?? "", key: m["key"] as? String ?? "",
|
||||
down: (m["e"] as? String) == "down", mods: m["mods"] as? [String] ?? [], owner: id)
|
||||
case "text":
|
||||
if let s = m["s"] as? String, s.count <= 4096 { Input.text(s) }
|
||||
case "release":
|
||||
Input.releaseAll(owner: id)
|
||||
case "focus":
|
||||
if case .window(let wid) = source, let pid = capture.pid { Input.focus(window: wid, pid: pid) }
|
||||
default: break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func randomKey() -> String {
|
||||
var bytes = [UInt8](repeating: 0, count: 24)
|
||||
_ = SecRandomCopyBytes(kSecRandomDefault, bytes.count, &bytes)
|
||||
return Data(bytes).base64EncodedString().replacingOccurrences(of: "+", with: "-")
|
||||
.replacingOccurrences(of: "/", with: "_").replacingOccurrences(of: "=", with: "")
|
||||
}
|
||||
|
||||
func sameSecret(_ a: String, _ b: String) -> Bool {
|
||||
guard !a.isEmpty, a.utf8.count == b.utf8.count else { return false }
|
||||
return zip(a.utf8, b.utf8).reduce(0, { $0 | ($1.0 ^ $1.1) }) == 0
|
||||
}
|
||||
|
||||
final class Agent {
|
||||
/// Frame Control's own key. It stays on the Mac: the Frame only ever sees
|
||||
/// single-use tickets and per-stream reconnect keys, both tied to one source.
|
||||
let token: String
|
||||
let page: URL?
|
||||
var sessions: [Int: Session] = [:] { didSet { keepDisplayAwake(!sessions.isEmpty) } }
|
||||
var nextId = 1
|
||||
let lock = NSLock()
|
||||
private var assertion: IOPMAssertionID = 0
|
||||
/// ticket -> (source, expiry, reconnect key once redeemed). A ticket opens
|
||||
/// one viewer within a minute; it may be redeemed again, for the same key,
|
||||
/// only until that viewer confirms it has the key ("ack").
|
||||
private var tickets: [String: (src: String, expiry: Date, key: String?)] = [:]
|
||||
/// reconnect key -> source, until Stop for that source.
|
||||
private var reconnectKeys: [String: String] = [:]
|
||||
/// Sources that ended for good (the window closed), for Frame Control.
|
||||
private var finished = Set<String>()
|
||||
|
||||
/// While anyone watches, keep the Mac's display on: a sleeping display
|
||||
/// stops being drawn, so there'd be nothing to capture (and it would lock).
|
||||
private func keepDisplayAwake(_ on: Bool) {
|
||||
if on, assertion == 0 {
|
||||
IOPMAssertionCreateWithName(kIOPMAssertionTypePreventUserIdleDisplaySleep as CFString,
|
||||
IOPMAssertionLevel(kIOPMAssertionLevelOn),
|
||||
"Frame Control is showing this Mac in a Steam Frame" as CFString, &assertion)
|
||||
} else if !on, assertion != 0 {
|
||||
IOPMAssertionRelease(assertion)
|
||||
assertion = 0
|
||||
}
|
||||
}
|
||||
|
||||
init(token: String, page: URL?) {
|
||||
self.token = token
|
||||
self.page = page
|
||||
}
|
||||
|
||||
/// Whether this request may open a stream of `src`: Frame Control's key,
|
||||
/// an unused ticket for that source, or a live reconnect key for it.
|
||||
private func mayStream(_ req: Request, src: String) -> String? {
|
||||
lock.lock()
|
||||
defer { lock.unlock() }
|
||||
let now = Date()
|
||||
tickets = tickets.filter { $0.value.expiry > now }
|
||||
if sameSecret(req.query["k"] ?? "", token) { return randomKey() }
|
||||
if let t = req.query["t"], let entry = tickets[t], entry.src == src {
|
||||
// A retry before the viewer got its key gets the same key back.
|
||||
let key = entry.key ?? randomKey()
|
||||
tickets[t] = (src, entry.expiry, key)
|
||||
reconnectKeys[key] = src
|
||||
return key
|
||||
}
|
||||
if let r = req.query["r"], reconnectKeys[r] == src { return r }
|
||||
return nil
|
||||
}
|
||||
|
||||
/// The viewer has its reconnect key, so the ticket that led to it can't
|
||||
/// be used again (also when the ack comes over a reconnection).
|
||||
func acknowledged(key: String) {
|
||||
lock.lock(); tickets = tickets.filter { $0.value.key != key }; lock.unlock()
|
||||
}
|
||||
|
||||
func handle(_ req: Request, _ c: HTTPConnection) {
|
||||
// Open to anything that reaches the port: a liveness check for the
|
||||
// tunnel, and the viewer page, which holds no secrets.
|
||||
switch (req.method, req.path) {
|
||||
case ("GET", "/ping"): return c.respond(200, text: "frame-mac-view")
|
||||
case ("GET", "/view"):
|
||||
guard let page, let body = try? Data(contentsOf: page) else { return c.respond(404, text: "no viewer page") }
|
||||
return c.respond(200, body: body, type: "text/html; charset=utf-8")
|
||||
case ("GET", "/stream"):
|
||||
guard let src = Source(req.query["src"] ?? "") else { return c.respond(400, text: "bad src") }
|
||||
guard let key = mayStream(req, src: src.key) else { return c.respond(403, text: "forbidden") }
|
||||
return stream(req, c, src: src, key: key)
|
||||
default: break
|
||||
}
|
||||
guard sameSecret(req.query["k"] ?? req.headers["x-token"] ?? "", token) else {
|
||||
return c.respond(403, text: "forbidden")
|
||||
}
|
||||
switch (req.method, req.path) {
|
||||
case ("GET", "/status"):
|
||||
lock.lock()
|
||||
let list = sessions.values.map { ["id": $0.id, "src": $0.source.key, "title": $0.capture.title, "app": $0.capture.app] }
|
||||
lock.unlock()
|
||||
var s = permissionsJSON()
|
||||
s["version"] = version
|
||||
s["streams"] = list
|
||||
lock.lock(); s["finished"] = Array(finished); lock.unlock()
|
||||
c.respond(json: s)
|
||||
case ("GET", "/windows"):
|
||||
c.respond(json: ["windows": windowsJSON(), "screen": CGPreflightScreenCaptureAccess()])
|
||||
case ("GET", "/displays"):
|
||||
c.respond(json: ["displays": displaysJSON()])
|
||||
case ("POST", "/ticket"):
|
||||
guard let src = Source(req.query["src"] ?? "") else { return c.respond(400, text: "bad src") }
|
||||
let t = randomKey()
|
||||
lock.lock()
|
||||
tickets[t] = (src.key, Date().addingTimeInterval(60), nil)
|
||||
finished.remove(src.key)
|
||||
lock.unlock()
|
||||
c.respond(json: ["ticket": t])
|
||||
case ("POST", "/close"):
|
||||
// Tell viewers to close their windows, but don't rely on them:
|
||||
// the sessions end here and can't reconnect.
|
||||
let src = req.query["src"]
|
||||
lock.lock()
|
||||
let matching = sessions.values.filter { src == nil || $0.source.key == src }
|
||||
reconnectKeys = reconnectKeys.filter { src != nil && $0.value != src }
|
||||
tickets = tickets.filter { src != nil && $0.value.src != src } // not yet used ones too
|
||||
lock.unlock()
|
||||
for s in matching { s.ws.sendJSON(["t": "close"]) }
|
||||
DispatchQueue.global().asyncAfter(deadline: .now() + 0.3) { for s in matching { s.end() } }
|
||||
c.respond(json: ["closed": matching.count])
|
||||
case ("POST", "/permissions"):
|
||||
DispatchQueue.main.async { requestPermissions() }
|
||||
c.respond(json: permissionsJSON())
|
||||
default:
|
||||
c.respond(404, text: "not found")
|
||||
}
|
||||
}
|
||||
|
||||
private func stream(_ req: Request, _ c: HTTPConnection, src: Source, key: String) {
|
||||
let codec = Codec(rawValue: req.query["codec"] ?? "h264") ?? .h264
|
||||
let maxLong = min(max(Int(req.query["max"] ?? "") ?? 1920, 320), 3840)
|
||||
let fps = min(max(Int(req.query["fps"] ?? "") ?? 60, 5), 120)
|
||||
let bpp = min(max(Double(req.query["bpp"] ?? "") ?? 0.1, 0.02), 0.5)
|
||||
guard let ws = c.upgrade(req) else { return }
|
||||
let capture: CaptureSource = src == .test ? TestSource() : SCKSource(src)
|
||||
lock.lock()
|
||||
// One live viewer per key: a reconnection (or a second use of an
|
||||
// unacknowledged ticket) replaces the one before.
|
||||
let replaced = sessions.values.filter { $0.reconnectKey == key }
|
||||
let session = Session(id: nextId, source: src, capture: capture, ws: ws, codec: codec, fps: fps,
|
||||
bitsPerPixel: bpp, reconnectKey: key)
|
||||
nextId += 1
|
||||
sessions[session.id] = session
|
||||
lock.unlock()
|
||||
for old in replaced { old.end() }
|
||||
session.onEnd = { [weak self] s in
|
||||
guard let self else { return }
|
||||
self.lock.lock(); self.sessions[s.id] = nil; self.lock.unlock()
|
||||
}
|
||||
// The source is gone (its window closed): its viewers can't come back.
|
||||
session.onFinished = { [weak self] src in
|
||||
guard let self else { return }
|
||||
self.lock.lock()
|
||||
self.finished.insert(src)
|
||||
self.reconnectKeys = self.reconnectKeys.filter { $0.value != src }
|
||||
self.lock.unlock()
|
||||
}
|
||||
session.onAck = { [weak self] in self?.acknowledged(key: key) }
|
||||
session.start(maxLong: maxLong, fps: fps)
|
||||
}
|
||||
}
|
||||
|
||||
func requestPermissions() {
|
||||
if !CGPreflightScreenCaptureAccess() { CGRequestScreenCaptureAccess() }
|
||||
if !AXIsProcessTrusted() {
|
||||
AXIsProcessTrustedWithOptions([kAXTrustedCheckOptionPrompt.takeUnretainedValue() as String: true] as CFDictionary)
|
||||
}
|
||||
}
|
||||
|
||||
func argument(_ name: String, in args: [String]) -> String? {
|
||||
guard let i = args.firstIndex(of: name), i + 1 < args.count else { return nil }
|
||||
return args[i + 1]
|
||||
}
|
||||
|
||||
let args = Array(CommandLine.arguments.dropFirst())
|
||||
switch args.first {
|
||||
case "windows":
|
||||
printJSON(["windows": windowsJSON(), "screen": CGPreflightScreenCaptureAccess()])
|
||||
case "displays":
|
||||
printJSON(["displays": displaysJSON()])
|
||||
case "permissions":
|
||||
printJSON(permissionsJSON())
|
||||
case "request-permissions":
|
||||
requestPermissions()
|
||||
printJSON(permissionsJSON())
|
||||
case "serve":
|
||||
let port = UInt16(argument("--port", in: args) ?? "") ?? 0
|
||||
let token = ProcessInfo.processInfo.environment["FRAME_MAC_VIEW_TOKEN"] ?? ""
|
||||
guard !token.isEmpty else {
|
||||
FileHandle.standardError.write(Data("frame-mac-view: set FRAME_MAC_VIEW_TOKEN\n".utf8))
|
||||
exit(2)
|
||||
}
|
||||
let page = argument("--page", in: args).map { URL(fileURLWithPath: $0) }
|
||||
let agent = Agent(token: token, page: page)
|
||||
// Quit when the parent goes away (it holds our stdin open).
|
||||
if args.contains("--exit-on-eof") {
|
||||
DispatchQueue.global().async {
|
||||
while FileHandle.standardInput.availableData.count > 0 {}
|
||||
exit(0)
|
||||
}
|
||||
}
|
||||
let server: Server
|
||||
do {
|
||||
server = try Server(port: port) { req, c in agent.handle(req, c) }
|
||||
} catch {
|
||||
FileHandle.standardError.write(Data("frame-mac-view: \(error)\n".utf8))
|
||||
exit(1)
|
||||
}
|
||||
server.start { [server] error in
|
||||
if let error {
|
||||
FileHandle.standardError.write(Data("frame-mac-view: can't listen on 127.0.0.1:\(port): \(error)\n".utf8))
|
||||
exit(1)
|
||||
}
|
||||
print("frame-mac-view listening on 127.0.0.1:\(server.port ?? port)")
|
||||
fflush(stdout)
|
||||
}
|
||||
_ = NSApplication.shared // AppKit for NSScreen names and app activation
|
||||
withExtendedLifetime(server) { RunLoop.main.run() }
|
||||
default:
|
||||
FileHandle.standardError.write(Data("usage: frame-mac-view serve|windows|displays|permissions|request-permissions\n".utf8))
|
||||
exit(2)
|
||||
}
|
||||
Reference in new issue
Block a user