Tweak the API, document export params, validate inputs better

This commit is contained in:
Sami Samhuri 2024-07-07 22:25:14 -07:00
parent a44627b971
commit 2fca0fb7fd
No known key found for this signature in database
5 changed files with 214 additions and 151 deletions

View file

@ -16,6 +16,7 @@
7BC5FC772C3B8C5A0090B757 /* SendableWrapper.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7BC5FC762C3B8C5A0090B757 /* SendableWrapper.swift */; };
7BC5FC792C3B90F70090B757 /* AutoDestructingURL.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7BC5FC782C3B90F70090B757 /* AutoDestructingURL.swift */; };
7BC5FC7B2C3B93270090B757 /* AVFoundation+sending.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7BC5FC7A2C3B93270090B757 /* AVFoundation+sending.swift */; };
7BC5FC8A2C3BAA150090B757 /* ExportSession+Error.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7BC5FC892C3BAA150090B757 /* ExportSession+Error.swift */; };
/* End PBXBuildFile section */
/* Begin PBXContainerItemProxy section */
@ -39,6 +40,7 @@
7BC5FC762C3B8C5A0090B757 /* SendableWrapper.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SendableWrapper.swift; sourceTree = "<group>"; };
7BC5FC782C3B90F70090B757 /* AutoDestructingURL.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AutoDestructingURL.swift; sourceTree = "<group>"; };
7BC5FC7A2C3B93270090B757 /* AVFoundation+sending.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "AVFoundation+sending.swift"; sourceTree = "<group>"; };
7BC5FC892C3BAA150090B757 /* ExportSession+Error.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "ExportSession+Error.swift"; sourceTree = "<group>"; };
/* End PBXFileReference section */
/* Begin PBXFileSystemSynchronizedRootGroup section */
@ -85,10 +87,11 @@
7B9BC00B2C305D2C00C160C2 /* SJSAssetExportSession */ = {
isa = PBXGroup;
children = (
7B9BC0272C30612C00C160C2 /* ExportSession.swift */,
7BC5FC892C3BAA150090B757 /* ExportSession+Error.swift */,
7B7AE3082C36615700DB7391 /* SampleWriter.swift */,
7B9BC00C2C305D2C00C160C2 /* SJSAssetExportSession.h */,
7B9BC00D2C305D2C00C160C2 /* SJSAssetExportSession.docc */,
7B9BC0272C30612C00C160C2 /* ExportSession.swift */,
7B7AE3082C36615700DB7391 /* SampleWriter.swift */,
);
path = SJSAssetExportSession;
sourceTree = "<group>";
@ -220,6 +223,7 @@
7B7AE3092C36615700DB7391 /* SampleWriter.swift in Sources */,
7B9BC0282C30612C00C160C2 /* ExportSession.swift in Sources */,
7B9BC00E2C305D2C00C160C2 /* SJSAssetExportSession.docc in Sources */,
7BC5FC8A2C3BAA150090B757 /* ExportSession+Error.swift in Sources */,
);
runOnlyForDeploymentPostprocessing = 0;
};

View file

@ -0,0 +1,75 @@
//
// ExportSession+Error.swift
// SJSAssetExportSession
//
// Created by Sami Samhuri on 2024-07-07.
//
import Foundation
extension ExportSession {
public enum SetupFailureReason: String, Sendable, CustomStringConvertible {
case audioSettingsEmpty
case audioSettingsInvalid
case cannotAddAudioInput
case cannotAddAudioOutput
case cannotAddVideoInput
case cannotAddVideoOutput
case videoSettingsEmpty
case videoSettingsInvalid
case videoTracksEmpty
public var description: String {
switch self {
case .audioSettingsEmpty:
"Must provide audio output settings"
case .audioSettingsInvalid:
"Invalid audio output settings"
case .cannotAddAudioInput:
"Can't add audio input to writer"
case .cannotAddAudioOutput:
"Can't add audio output to reader"
case .cannotAddVideoInput:
"Can't add video input to writer"
case .cannotAddVideoOutput:
"Can't add video output to reader"
case .videoSettingsEmpty:
"Must provide video output settings"
case .videoSettingsInvalid:
"Invalid video output settings"
case .videoTracksEmpty:
"No video track"
}
}
}
public enum Error: LocalizedError, Equatable {
case setupFailure(SetupFailureReason)
case readFailure((any Swift.Error)?)
case writeFailure((any Swift.Error)?)
public var errorDescription: String? {
switch self {
case let .setupFailure(reason):
reason.description
case let .readFailure(underlyingError):
underlyingError?.localizedDescription ?? "Unknown read failure"
case let .writeFailure(underlyingError):
underlyingError?.localizedDescription ?? "Unknown write failure"
}
}
public static func == (lhs: ExportSession.Error, rhs: ExportSession.Error) -> Bool {
switch (lhs, rhs) {
case let (.setupFailure(lhsReason), .setupFailure(rhsReason)):
lhsReason == rhsReason
case let (.readFailure(lhsError), .readFailure(rhsError)):
String(describing: lhsError) == String(describing: rhsError)
case let (.writeFailure(lhsError), .writeFailure(rhsError)):
String(describing: lhsError) == String(describing: rhsError)
default:
false
}
}
}
}

View file

@ -8,72 +8,8 @@
public import AVFoundation
// @unchecked Sendable because progress properties are mutable, it's safe though.
public final class ExportSession: @unchecked Sendable {
public enum SetupFailureReason: String, Sendable, CustomStringConvertible {
case audioSettingsEmpty
case audioSettingsInvalid
case cannotAddAudioInput
case cannotAddAudioOutput
case cannotAddVideoInput
case cannotAddVideoOutput
case videoSettingsEmpty
case videoSettingsInvalid
case videoTracksEmpty
public var description: String {
switch self {
case .audioSettingsEmpty:
"Must provide audio output settings"
case .audioSettingsInvalid:
"Invalid audio output settings"
case .cannotAddAudioInput:
"Can't add audio input to writer"
case .cannotAddAudioOutput:
"Can't add audio output to reader"
case .cannotAddVideoInput:
"Can't add video input to writer"
case .cannotAddVideoOutput:
"Can't add video output to reader"
case .videoSettingsEmpty:
"Must provide video output settings"
case .videoSettingsInvalid:
"Invalid video output settings"
case .videoTracksEmpty:
"No video track"
}
}
}
public enum Error: LocalizedError, Equatable {
case setupFailure(SetupFailureReason)
case readFailure((any Swift.Error)?)
case writeFailure((any Swift.Error)?)
public var errorDescription: String? {
switch self {
case let .setupFailure(reason):
reason.description
case let .readFailure(underlyingError):
underlyingError?.localizedDescription ?? "Unknown read failure"
case let .writeFailure(underlyingError):
underlyingError?.localizedDescription ?? "Unknown write failure"
}
}
public static func == (lhs: ExportSession.Error, rhs: ExportSession.Error) -> Bool {
switch (lhs, rhs) {
case let (.setupFailure(lhsReason), .setupFailure(rhsReason)):
lhsReason == rhsReason
case let (.readFailure(lhsError), .readFailure(rhsError)):
String(describing: lhsError) == String(describing: rhsError)
case let (.writeFailure(lhsError), .writeFailure(rhsError)):
String(describing: lhsError) == String(describing: rhsError)
default:
false
}
}
}
// @unchecked Sendable because progress properties are mutable, it's safe though.
public typealias ProgressStream = AsyncStream<Float>
@ -87,6 +23,33 @@ public final class ExportSession: @unchecked Sendable {
}
}
/**
Exports the given asset using all of the other parameters to transform it in some way.
- Parameters:
- asset: The source asset to export. This can be any kind of `AVAsset` including subclasses such as `AVComposition`.
- audioMix: An optional mix that can be used to manipulate the audio in some way.
- audioOutputSettings: Audio settings using [audio settings keys from AVFoundation](https://developer.apple.com/documentation/avfoundation/audio_settings) and values must be suitable for consumption by Objective-C. Required keys are:
- `AVFormatIDKey` with the typical value `kAudioFormatMPEG4AAC`
- `AVNumberOfChannelsKey` with the typical value `NSNumber(value: 2)` or `AVChannelLayoutKey` with an instance of `AVAudioChannelLayout`
- videoComposition: Used to manipulate the video in some way. This can be used to scale the video, apply filters, amongst other edits.
- videoOutputSettings: Video settings using [video settings keys from AVFoundation](https://developer.apple.com/documentation/avfoundation/video_settings) and values must be suitable for consumption by Objective-C. Required keys are:
- `AVVideoCodecKey` with the typical value `AVVideoCodecType.h264.rawValue` or `AVVideoCodecType.hevc.rawValue`
- `AVVideoWidthKey` with an integer as an `NSNumber`
- `AVVideoHeightKey` with an integer as an `NSNumber`
- timeRange: Providing a time range exports a subset of the asset instead of the entire duration, which is the default behaviour.
- optimizeForNetworkUse: Setting this value to `true` writes the output file in a form that enables a player to begin playing the media after downloading only a small portion of it. Defaults to `false`.
- outputURL: The file URL where the exported video will be written.
- fileType: The type of of video file to export. This will typically be one of `AVFileType.mp4`, `AVFileType.m4v`, or `AVFileType.mov`.
*/
public func export(
asset: sending AVAsset,
audioMix: sending AVAudioMix?,
@ -100,11 +63,11 @@ public final class ExportSession: @unchecked Sendable {
) async throws {
let sampleWriter = try await SampleWriter(
asset: asset,
timeRange: timeRange ?? CMTimeRange(start: .zero, duration: .positiveInfinity),
audioMix: audioMix,
audioOutputSettings: audioOutputSettings,
videoComposition: videoComposition,
videoOutputSettings: videoOutputSettings,
timeRange: timeRange,
optimizeForNetworkUse: optimizeForNetworkUse,
outputURL: outputURL,
fileType: fileType

View file

@ -19,92 +19,66 @@ private extension AVAsset {
actor SampleWriter {
typealias Error = ExportSession.Error
// MARK: - Actor executor
private let queue = DispatchSerialQueue(
label: "SJSAssetExportSession.SampleWriter",
autoreleaseFrequency: .workItem,
target: .global()
)
// Execute this actor on the same queue we use to request media data so we can use
// `assumeIsolated` to ensure that we
public nonisolated var unownedExecutor: UnownedSerialExecutor {
queue.asUnownedSerialExecutor()
}
private let audioMix: AVAudioMix?
private let audioOutputSettings: [String: (any Sendable)]
private let videoComposition: AVVideoComposition?
private let videoOutputSettings: [String: (any Sendable)]
private let reader: AVAssetReader
private let writer: AVAssetWriter
private let duration: CMTime
private let timeRange: CMTimeRange
private var audioOutput: AVAssetReaderAudioMixOutput?
private var audioInput: AVAssetWriterInput?
private var videoOutput: AVAssetReaderVideoCompositionOutput?
private var videoInput: AVAssetWriterInput?
lazy var progressStream: AsyncStream<Float> = AsyncStream { continuation in
progressContinuation = continuation
}
private var progressContinuation: AsyncStream<Float>.Continuation?
private let audioMix: AVAudioMix?
private let audioOutputSettings: [String: (any Sendable)]
private let videoComposition: AVVideoComposition?
private let videoOutputSettings: [String: (any Sendable)]
private let reader: AVAssetReader
private let writer: AVAssetWriter
private let duration: CMTime
private let timeRange: CMTimeRange
private var audioOutput: AVAssetReaderAudioMixOutput?
private var audioInput: AVAssetWriterInput?
private var videoOutput: AVAssetReaderVideoCompositionOutput?
private var videoInput: AVAssetWriterInput?
init(
asset: sending AVAsset,
timeRange: CMTimeRange,
audioMix: AVAudioMix?,
audioOutputSettings: sending [String: (any Sendable)],
videoComposition: AVVideoComposition,
videoOutputSettings: sending [String: (any Sendable)],
optimizeForNetworkUse: Bool,
timeRange: CMTimeRange? = nil,
optimizeForNetworkUse: Bool = false,
outputURL: URL,
fileType: AVFileType
) async throws {
guard !videoOutputSettings.isEmpty else {
throw Error.setupFailure(.videoSettingsEmpty)
}
let duration =
if timeRange.duration.isValid && !timeRange.duration.isPositiveInfinity {
timeRange.duration
} else {
try await asset.load(.duration)
}
if let timeRange { timeRange.duration } else { try await asset.load(.duration) }
let reader = try AVAssetReader(asset: asset)
reader.timeRange = timeRange
if let timeRange {
reader.timeRange = timeRange
}
let writer = try AVAssetWriter(outputURL: outputURL, fileType: fileType)
writer.shouldOptimizeForNetworkUse = optimizeForNetworkUse
let audioTracks = try await asset.sendTracks(withMediaType: .audio)
if !audioTracks.isEmpty {
guard !audioOutputSettings.isEmpty else {
throw Error.setupFailure(.audioSettingsEmpty)
}
guard writer.canApply(outputSettings: audioOutputSettings, forMediaType: .audio) else {
throw Error.setupFailure(.audioSettingsInvalid)
}
}
try Self.validateAudio(tracks: audioTracks, outputSettings: audioOutputSettings, writer: writer)
let videoTracks = try await asset.sendTracks(withMediaType: .video)
guard !videoTracks.isEmpty else {
throw Error.setupFailure(.videoTracksEmpty)
}
guard writer.canApply(outputSettings: videoOutputSettings, forMediaType: .video) else {
throw Error.setupFailure(.videoSettingsInvalid)
}
try Self.validateVideo(tracks: videoTracks, outputSettings: videoOutputSettings, writer: writer)
Self.warnAboutMismatchedVideoDimensions(
renderSize: videoComposition.renderSize,
settings: videoOutputSettings
)
self.audioMix = audioMix
self.audioOutputSettings = audioOutputSettings
@ -113,7 +87,7 @@ actor SampleWriter {
self.reader = reader
self.writer = writer
self.duration = duration
self.timeRange = timeRange
self.timeRange = timeRange ?? CMTimeRange(start: .zero, duration: duration)
try setUpAudio(audioTracks: audioTracks)
try setUpVideo(videoTracks: videoTracks)
@ -156,6 +130,8 @@ actor SampleWriter {
await Task.yield()
}
// MARK: - Setup
private func setUpAudio(audioTracks: [AVAssetTrack]) throws {
guard !audioTracks.isEmpty else { return }
@ -201,34 +177,10 @@ actor SampleWriter {
self.videoInput = videoInput
}
private func writeReadySamples(output: AVAssetReaderOutput, input: AVAssetWriterInput) -> Bool {
while input.isReadyForMoreMediaData {
guard reader.status == .reading && writer.status == .writing,
let sampleBuffer = output.copyNextSampleBuffer() else {
input.markAsFinished()
log.debug("Finished encoding ready audio samples from \(output)")
return false
}
let samplePresentationTime = CMSampleBufferGetPresentationTimeStamp(sampleBuffer) - timeRange.start
let progress = Float(samplePresentationTime.seconds / duration.seconds)
progressContinuation?.yield(progress)
guard input.append(sampleBuffer) else {
log.error("""
Failed to append audio sample buffer \(String(describing: sampleBuffer)) to
input \(input.debugDescription)
""")
return false
}
}
// Everything was appended successfully, return true indicating there's more to do.
log.debug("Completed encoding ready audio samples, more to come...")
return true
}
// MARK: - Encoding
private func encodeAudioTracks() async {
// Don't do anything when we have no audio to encode.
guard audioInput != nil, audioOutput != nil else { return }
return await withCheckedContinuation { continuation in
@ -255,4 +207,73 @@ actor SampleWriter {
}
}
}
private func writeReadySamples(output: AVAssetReaderOutput, input: AVAssetWriterInput) -> Bool {
while input.isReadyForMoreMediaData {
guard reader.status == .reading && writer.status == .writing,
let sampleBuffer = output.copyNextSampleBuffer() else {
input.markAsFinished()
return false
}
// Only yield progress values for video. Audio is insignificant in comparison.
if output == videoOutput {
let samplePresentationTime = CMSampleBufferGetPresentationTimeStamp(sampleBuffer) - timeRange.start
let progress = Float(samplePresentationTime.seconds / duration.seconds)
progressContinuation?.yield(progress)
}
guard input.append(sampleBuffer) else {
log.error("""
Failed to append audio sample buffer \(String(describing: sampleBuffer)) to
input \(input.debugDescription)
""")
return false
}
}
// Everything was appended successfully, return true indicating there's more to do.
return true
}
// MARK: Input validation
private static func validateAudio(
tracks: [AVAssetTrack],
outputSettings: [String: any Sendable],
writer: AVAssetWriter
) throws {
guard !tracks.isEmpty else { return } // Audio is optional so this isn't a failure.
guard !outputSettings.isEmpty else { throw Error.setupFailure(.audioSettingsEmpty) }
guard writer.canApply(outputSettings: outputSettings, forMediaType: .audio) else {
throw Error.setupFailure(.audioSettingsInvalid)
}
}
private static func validateVideo(
tracks: [AVAssetTrack],
outputSettings: [String: any Sendable],
writer: AVAssetWriter
) throws {
guard !tracks.isEmpty else { throw Error.setupFailure(.videoTracksEmpty) }
guard !outputSettings.isEmpty else { throw Error.setupFailure(.videoSettingsEmpty) }
guard writer.canApply(outputSettings: outputSettings, forMediaType: .video) else {
throw Error.setupFailure(.videoSettingsInvalid)
}
}
private static func warnAboutMismatchedVideoDimensions(
renderSize: CGSize,
settings: [String: any Sendable]
) {
guard let settingsWidth = (settings[AVVideoWidthKey] as? NSNumber)?.intValue,
let settingsHeight = (settings[AVVideoHeightKey] as? NSNumber)?.intValue
else { return }
let renderWidth = Int(renderSize.width)
let renderHeight = Int(renderSize.height)
if renderWidth != settingsWidth || renderHeight != settingsHeight {
log.warning("Video composition's render size (\(renderWidth)\(renderHeight)) will be overriden by video output settings (\(settingsWidth)\(settingsHeight))")
}
}
}

View file

@ -8,7 +8,7 @@
import Foundation
import OSLog
private let log = Logger(OSLog(subsystem: "SJSAssetExportSessionTests", category: "AutoDestructingURL"))
private let log = Logger(subsystem: "SJSAssetExportSessionTests", category: "AutoDestructingURL")
/// Wraps a URL and deletes it when this instance is deallocated. Failures to delete the file are logged.
final class AutoDestructingURL: Hashable, Sendable {