// VideoToolbox encoding: H.264 for the normal path (hardware, low-latency rate // control, no B-frames, Annex B output that WebCodecs takes without a // description), or JPEG stills for viewers that can't decode H.264. import CoreMedia import Foundation import VideoToolbox enum Codec: String { case h264, jpeg } final class Encoder { let codec: Codec let fps: Int let bitsPerPixel: Double private(set) var width = 0 private(set) var height = 0 private var session: VTCompressionSession? private var forceKey = true private var lastPts = CMTime.invalid private var lastGiven = CMTime.invalid /// Bits per second to aim for; nil means from the size, frame rate and /// bits per pixel. Changing it takes effect on the next frame. private var target: Int? private(set) var bitrate = 0 /// Called on VideoToolbox's thread with one access unit (or JPEG) per /// frame, and the sequence number it was submitted with. var onFrame: ((Data, Bool, CMTime, UInt32) -> Void)? var onError: ((String) -> Void)? /// Called once per frame handed to VideoToolbox, however it went. var onDone: ((UInt32) -> Void)? /// Experiment switches (FRAME_MAC_VIEW_ENCODER, comma-separated), so a /// benchmark can compare them without a rebuild. static let options = Set((ProcessInfo.processInfo.environment["FRAME_MAC_VIEW_ENCODER"] ?? "") .split(separator: ",").map(String.init)) init(codec: Codec, fps: Int, bitsPerPixel: Double) { self.codec = codec self.fps = fps self.bitsPerPixel = bitsPerPixel } /// The bitrate the size and quality setting would give. func defaultBitrate(width w: Int, height h: Int) -> Int { Int(min(max(Double(w * h * fps) * bitsPerPixel, 2_000_000), 60_000_000)) } /// Live, without a new keyframe. Call on the encoding queue. func setBitrate(_ bps: Int?) { target = bps guard codec == .h264, let s = session else { return } let b = bps ?? defaultBitrate(width: width, height: height) guard b != bitrate else { return } bitrate = b VTSessionSetProperty(s, key: kVTCompressionPropertyKey_AverageBitRate, value: b as CFTypeRef) // A hard ceiling too: no more than 200 ms' worth of bits in any 200 ms, // so a keyframe can't hold the link for long. if Encoder.options.contains("cap") { VTSessionSetProperty(s, key: kVTCompressionPropertyKey_DataRateLimits, value: [b / 8 / 5, 0.2] as CFArray) } } deinit { invalidate() } func requestKeyFrame() { forceKey = true } func invalidate() { if let s = session { VTCompressionSessionCompleteFrames(s, untilPresentationTimeStamp: .invalid) VTCompressionSessionInvalidate(s) } session = nil } private func makeSession(width w: Int, height h: Int) -> Bool { invalidate() var spec: [CFString: Any] = [:] if codec == .h264 { spec[kVTVideoEncoderSpecification_EnableLowLatencyRateControl] = true } if Encoder.options.contains("hw") { spec[kVTVideoEncoderSpecification_RequireHardwareAcceleratedVideoEncoder] = true } var s: VTCompressionSession? let type = codec == .h264 ? kCMVideoCodecType_H264 : kCMVideoCodecType_JPEG func create(_ spec: [CFString: Any]) -> OSStatus { VTCompressionSessionCreate(allocator: nil, width: Int32(w), height: Int32(h), codecType: type, encoderSpecification: spec as CFDictionary, imageBufferAttributes: nil, compressedDataAllocator: nil, outputCallback: nil, refcon: nil, compressionSessionOut: &s) } var err = create(spec) // Low-latency rate control needs Apple's hardware encoder; without it // (some VMs), plain real-time encoding still works. if err != noErr, !spec.isEmpty { err = create([:]) } guard err == noErr, let s else { onError?("couldn't start the \(codec.rawValue) encoder (VideoToolbox \(err))") return false } func set(_ key: CFString, _ value: Any) { VTSessionSetProperty(s, key: key, value: value as CFTypeRef) } set(kVTCompressionPropertyKey_RealTime, true) set(kVTCompressionPropertyKey_ColorPrimaries, kCVImageBufferColorPrimaries_ITU_R_709_2) set(kVTCompressionPropertyKey_TransferFunction, kCVImageBufferTransferFunction_ITU_R_709_2) set(kVTCompressionPropertyKey_YCbCrMatrix, kCVImageBufferYCbCrMatrix_ITU_R_709_2) if codec == .h264 { set(kVTCompressionPropertyKey_ProfileLevel, kVTProfileLevel_H264_ConstrainedHigh_AutoLevel) set(kVTCompressionPropertyKey_AllowFrameReordering, false) set(kVTCompressionPropertyKey_ExpectedFrameRate, fps) if Encoder.options.contains("nodelay") { set(kVTCompressionPropertyKey_MaxFrameDelayCount, 0) } if Encoder.options.contains("speed") { set(kVTCompressionPropertyKey_PrioritizeEncodingSpeedOverQuality, true) } // A keyframe every 10 s at most, so a viewer that lost one recovers // even if it never asks. Viewers ask for one when they start. set(kVTCompressionPropertyKey_MaxKeyFrameIntervalDuration, 10) } else { set(kVTCompressionPropertyKey_Quality, 0.8) } VTCompressionSessionPrepareToEncodeFrames(s) session = s lastPts = .invalid lastGiven = .invalid width = w height = h forceKey = true bitrate = 0 setBitrate(target) return true } /// False if the frame never reached VideoToolbox (then onDone won't come). @discardableResult func encode(_ pb: CVPixelBuffer, pts given: CMTime, seq: UInt32) -> Bool { // The encoder's own timeline: strictly increasing (a resent picture // stamped "now" can be followed by a capture stamped a moment earlier), // and never more than two frames on from the last one. Rate control // budgets bits by elapsed time, so after a pause (the link held frames // back, or nothing changed) one frame would otherwise get a quarter // second's worth of bits: 200 KB that then hold a slow link for half a second. let w = CVPixelBufferGetWidth(pb), h = CVPixelBufferGetHeight(pb) if session == nil || w != width || h != height { guard makeSession(width: w, height: h) else { return false } // a new timeline too } guard let s = session else { return false } var pts = given if lastPts.isValid, lastGiven.isValid { let gap = CMTimeSubtract(given, lastGiven) let most = CMTime(value: 2, timescale: CMTimeScale(fps)) let step = CMTimeCompare(gap, most) > 0 ? most : gap pts = CMTimeAdd(lastPts, CMTimeMaximum(step, CMTime(value: 1, timescale: 1_000_000))) } lastGiven = given var props: CFDictionary? if forceKey { props = [kVTEncodeFrameOptionKey_ForceKeyFrame: true] as CFDictionary forceKey = false } let codec = self.codec lastPts = pts let status = VTCompressionSessionEncodeFrame(s, imageBuffer: pb, presentationTimeStamp: pts, duration: .invalid, frameProperties: props, infoFlagsOut: nil) { [weak self] status, _, sample in guard let self else { return } defer { self.onDone?(seq) } guard status == noErr, let sample else { return } if codec == .jpeg { if let data = Self.bytes(sample) { self.onFrame?(data, true, pts, seq) } } else if let (data, key) = Self.annexB(sample) { self.onFrame?(data, key, pts, seq) } } if status != noErr { forceKey = true } return status == noErr } private static func bytes(_ sample: CMSampleBuffer) -> Data? { guard let block = CMSampleBufferGetDataBuffer(sample) else { return nil } var length = 0 var ptr: UnsafeMutablePointer? guard CMBlockBufferGetDataPointer(block, atOffset: 0, lengthAtOffsetOut: nil, totalLengthOut: &length, dataPointerOut: &ptr) == noErr, let ptr else { return nil } return Data(bytes: ptr, count: length) } /// AVCC sample -> Annex B access unit, with SPS and PPS before keyframes. static func annexB(_ sample: CMSampleBuffer) -> (Data, Bool)? { guard let avcc = bytes(sample) else { return nil } var key = true if let atts = CMSampleBufferGetSampleAttachmentsArray(sample, createIfNecessary: false) as? [[CFString: Any]], let first = atts.first, first[kCMSampleAttachmentKey_NotSync] as? Bool == true { key = false } let start: [UInt8] = [0, 0, 0, 1] var out = Data() if key, let fmt = CMSampleBufferGetFormatDescription(sample) { var count = 0 CMVideoFormatDescriptionGetH264ParameterSetAtIndex(fmt, parameterSetIndex: 0, parameterSetPointerOut: nil, parameterSetSizeOut: nil, parameterSetCountOut: &count, nalUnitHeaderLengthOut: nil) for i in 0..? var n = 0 if CMVideoFormatDescriptionGetH264ParameterSetAtIndex(fmt, parameterSetIndex: i, parameterSetPointerOut: &p, parameterSetSizeOut: &n, parameterSetCountOut: nil, nalUnitHeaderLengthOut: nil) == noErr, let p { out.append(contentsOf: start) out.append(p, count: n) } } } let bytes = [UInt8](avcc) var i = 0 while i + 4 <= bytes.count { let n = Int(bytes[i]) << 24 | Int(bytes[i + 1]) << 16 | Int(bytes[i + 2]) << 8 | Int(bytes[i + 3]) i += 4 guard n > 0, i + n <= bytes.count else { break } out.append(contentsOf: start) out.append(contentsOf: bytes[i..<(i + n)]) i += n } return (out, key) } }