diff --git a/Sources/SwiftAudioEx/AVPlayerWrapper/AVPlayerWrapper.swift b/Sources/SwiftAudioEx/AVPlayerWrapper/AVPlayerWrapper.swift index a7b168e..d08866f 100755 --- a/Sources/SwiftAudioEx/AVPlayerWrapper/AVPlayerWrapper.swift +++ b/Sources/SwiftAudioEx/AVPlayerWrapper/AVPlayerWrapper.swift @@ -24,6 +24,7 @@ class AVPlayerWrapper: AVPlayerWrapperProtocol { // MARK: - Properties fileprivate var avPlayer = AVPlayer() + internal var audioTap: AudioTap? = nil private let playerObserver = AVPlayerObserver() internal let playerTimeObserver: AVPlayerTimeObserver private let playerItemNotificationObserver = AVPlayerItemNotificationObserver() @@ -385,6 +386,7 @@ class AVPlayerWrapper: AVPlayerWrapperProtocol { private func startObservingAVPlayer(item: AVPlayerItem) { playerItemObserver.startObserving(item: item) playerItemNotificationObserver.startObserving(item: item) + attachTap(audioTap, to: item) } private func stopObservingAVPlayerItem() { diff --git a/Sources/SwiftAudioEx/AVPlayerWrapper/AVPlayerWrapperProtocol.swift b/Sources/SwiftAudioEx/AVPlayerWrapper/AVPlayerWrapperProtocol.swift index 0903339..0716eca 100755 --- a/Sources/SwiftAudioEx/AVPlayerWrapper/AVPlayerWrapperProtocol.swift +++ b/Sources/SwiftAudioEx/AVPlayerWrapper/AVPlayerWrapperProtocol.swift @@ -13,6 +13,8 @@ protocol AVPlayerWrapperProtocol: AnyObject { var state: AVPlayerWrapperState { get set } + var audioTap: AudioTap? { get set } + var playWhenReady: Bool { get set } var currentItem: AVPlayerItem? { get } diff --git a/Sources/SwiftAudioEx/AudioPlayer.swift b/Sources/SwiftAudioEx/AudioPlayer.swift index 966c295..7c57d70 100755 --- a/Sources/SwiftAudioEx/AudioPlayer.swift +++ b/Sources/SwiftAudioEx/AudioPlayer.swift @@ -13,6 +13,14 @@ public typealias AudioPlayerState = AVPlayerWrapperState public class AudioPlayer: AVPlayerWrapperDelegate { /// The wrapper around the underlying AVPlayer let wrapper: AVPlayerWrapperProtocol = AVPlayerWrapper() + + /** + Set an instance of AudioTap, to receive frame information and audio buffer access during playback. + */ + public var audioTap: AudioTap? { + get { return wrapper.audioTap } + set(value) { wrapper.audioTap = value } + } public let nowPlayingInfoController: NowPlayingInfoControllerProtocol public let remoteCommandController: RemoteCommandController diff --git a/Sources/SwiftAudioEx/AudioTap.swift b/Sources/SwiftAudioEx/AudioTap.swift new file mode 100644 index 0000000..f350269 --- /dev/null +++ b/Sources/SwiftAudioEx/AudioTap.swift @@ -0,0 +1,98 @@ +// +// AudioTap.swift +// +// +// Created by Brandon Sneed on 3/31/24. +// + +import Foundation +import AVFoundation + +/** + Subclass this and set the AudioPlayer's `audioTap` property to start receiving the + audio stream. + */ +open class AudioTap { + // Called at tap initialization for a given player item. Use this to setup anything you might need. + open func initialize() { print("audioTap: initialize") } + // Called at teardown of the internal tap. Use this to reset any memory buffers you have created, etc. + open func finalize() { print("audioTap: finalize") } + // Called just before playback so you can perform setup based on the stream description. + open func prepare(description: AudioStreamBasicDescription) { print("audioTap: prepare") } + // Called just before finalize. + open func unprepare() { print("audioTap: unprepare") } + /** + Called periodically during audio stream playback. + + Example: + + ``` + func process(numberOfFrames: Int, buffer: UnsafeMutableAudioBufferListPointer) { + for channel in buffer { + // process audio samples here + //memset(channel.mData, 0, Int(channel.mDataByteSize)) + } + } + ``` + */ + open func process(numberOfFrames: Int, buffer: UnsafeMutableAudioBufferListPointer) { print("audioTap: process") } +} + +extension AVPlayerWrapper { + internal func attachTap(_ tap: AudioTap?, to item: AVPlayerItem) { + guard let tap else { return } + guard let track = item.asset.tracks(withMediaType: .audio).first else { + return + } + + let audioMix = AVMutableAudioMix() + let params = AVMutableAudioMixInputParameters(track: track) + + // we need to retain this pointer so it doesn't disappear out from under us. + // we'll then let it go after we finalize. If the tap changed upstream, we + // aren't going to pick up the new one until after this player item goes away. + let client = UnsafeMutableRawPointer(Unmanaged.passRetained(tap).toOpaque()) + + var callbacks = MTAudioProcessingTapCallbacks(version: kMTAudioProcessingTapCallbacksVersion_0, clientInfo: client) + { tapRef, clientInfo, tapStorageOut in + // initial tap setup + guard let clientInfo else { return } + tapStorageOut.pointee = clientInfo + let audioTap = Unmanaged.fromOpaque(clientInfo).takeUnretainedValue() + audioTap.initialize() + } finalize: { tapRef in + // clean up + let audioTap = Unmanaged.fromOpaque(MTAudioProcessingTapGetStorage(tapRef)).takeUnretainedValue() + audioTap.finalize() + // we're done, we can let go of the pointer we retained. + Unmanaged.passUnretained(audioTap).release() + } prepare: { tapRef, maxFrames, processingFormat in + // allocate memory for sound processing + let audioTap = Unmanaged.fromOpaque(MTAudioProcessingTapGetStorage(tapRef)).takeUnretainedValue() + audioTap.prepare(description: processingFormat.pointee) + } unprepare: { tapRef in + // deallocate memory for sound processing + let audioTap = Unmanaged.fromOpaque(MTAudioProcessingTapGetStorage(tapRef)).takeUnretainedValue() + audioTap.unprepare() + } process: { tapRef, numberFrames, flags, bufferListInOut, numberFramesOut, flagsOut in + guard noErr == MTAudioProcessingTapGetSourceAudio(tapRef, numberFrames, bufferListInOut, flagsOut, nil, numberFramesOut) else { + return + } + + // process sound data + let audioTap = Unmanaged.fromOpaque(MTAudioProcessingTapGetStorage(tapRef)).takeUnretainedValue() + audioTap.process(numberOfFrames: numberFrames, buffer: UnsafeMutableAudioBufferListPointer(bufferListInOut)) + } + + var tapRef: Unmanaged? + let error = MTAudioProcessingTapCreate(kCFAllocatorDefault, &callbacks, kMTAudioProcessingTapCreationFlag_PreEffects, &tapRef) + assert(error == noErr) + + params.audioTapProcessor = tapRef?.takeUnretainedValue() + tapRef?.release() + + audioMix.inputParameters = [params] + item.audioMix = audioMix + } +} + diff --git a/Sources/SwiftAudioEx/Observer/AVPlayerItemObserver.swift b/Sources/SwiftAudioEx/Observer/AVPlayerItemObserver.swift index f46e3d3..4b93d0b 100644 --- a/Sources/SwiftAudioEx/Observer/AVPlayerItemObserver.swift +++ b/Sources/SwiftAudioEx/Observer/AVPlayerItemObserver.swift @@ -63,6 +63,7 @@ class AVPlayerItemObserver: NSObject { self.isObserving = true self.observingItem = item + item.addObserver(self, forKeyPath: AVPlayerItemKeyPath.duration, options: [.new], context: &AVPlayerItemObserver.context) item.addObserver(self, forKeyPath: AVPlayerItemKeyPath.loadedTimeRanges, options: [.new], context: &AVPlayerItemObserver.context) item.addObserver(self, forKeyPath: AVPlayerItemKeyPath.playbackLikelyToKeepUp, options: [.new], context: &AVPlayerItemObserver.context) @@ -79,6 +80,9 @@ class AVPlayerItemObserver: NSObject { return } + // BKS: remove a tap if we had one. + observingItem.audioMix = nil + observingItem.removeObserver(self, forKeyPath: AVPlayerItemKeyPath.duration, context: &AVPlayerItemObserver.context) observingItem.removeObserver(self, forKeyPath: AVPlayerItemKeyPath.loadedTimeRanges, context: &AVPlayerItemObserver.context) observingItem.removeObserver(self, forKeyPath: AVPlayerItemKeyPath.playbackLikelyToKeepUp, context: &AVPlayerItemObserver.context) diff --git a/Sources/SwiftAudioEx/Utils/Devices.swift b/Sources/SwiftAudioEx/Utils/Devices.swift new file mode 100644 index 0000000..21bccf0 --- /dev/null +++ b/Sources/SwiftAudioEx/Utils/Devices.swift @@ -0,0 +1,8 @@ +// +// File.swift +// +// +// Created by Brandon Sneed on 4/1/24. +// + +import Foundation diff --git a/Tests/SwiftAudioExTests/AudioPlayerTests.swift b/Tests/SwiftAudioExTests/AudioPlayerTests.swift index 82a8a35..774ff0c 100644 --- a/Tests/SwiftAudioExTests/AudioPlayerTests.swift +++ b/Tests/SwiftAudioExTests/AudioPlayerTests.swift @@ -110,6 +110,38 @@ class AudioPlayerTests: XCTestCase { XCTAssertEqual(audioPlayer.duration, 0) } + // MARK: - Audio Tap testing + + func testAudioTapSwitching() { + listener.onSecondsElapse = { position in + if position > 4 { + // swap it out part-way through the first track. + self.audioPlayer.audioTap = DummyAudioTap(tapIndex: 2) + } + } + + audioPlayer.audioTap = DummyAudioTap(tapIndex: 1) + audioPlayer.load(item: FiveSecondSource.getAudioItem()) + audioPlayer.play() + + RunLoop.current.run(until: Date(timeIntervalSinceNow: 6)) + + audioPlayer.load(item: FiveSecondSource.getAudioItem()) + audioPlayer.play() + + RunLoop.current.run(until: Date(timeIntervalSinceNow: 6)) + + let tap1Active = DummyAudioTap.outputs.contains { output in + return output.contains("audioTap 1: process") + } + + let tap2Active = DummyAudioTap.outputs.contains { output in + return output.contains("audioTap 2: process") + } + XCTAssertTrue(tap1Active) + XCTAssertTrue(tap2Active) + } + // MARK: - Failure func testFailEventOnLoadWithNonMalformedURL() { diff --git a/Tests/SwiftAudioExTests/Mocks/DummyAudioTap.swift b/Tests/SwiftAudioExTests/Mocks/DummyAudioTap.swift new file mode 100644 index 0000000..e952e62 --- /dev/null +++ b/Tests/SwiftAudioExTests/Mocks/DummyAudioTap.swift @@ -0,0 +1,40 @@ +// +// File.swift +// +// +// Created by Brandon Sneed on 4/1/24. +// + +import Foundation +import CoreAudio +@testable import SwiftAudioEx + +class DummyAudioTap: AudioTap { + static var outputs = [String]() + + let tapIndex: Int + + init(tapIndex: Int) { + self.tapIndex = tapIndex + } + + override func initialize() { + Self.outputs.append("audioTap \(tapIndex): initialize") + } + + override func finalize() { + Self.outputs.append("audioTap \(tapIndex): finalize") + } + + override func prepare(description: AudioStreamBasicDescription) { + Self.outputs.append("audioTap \(tapIndex): prepare") + } + + override func unprepare() { + Self.outputs.append("audioTap \(tapIndex): unprepare") + } + + override func process(numberOfFrames: Int, buffer: UnsafeMutableAudioBufferListPointer) { + Self.outputs.append("audioTap \(tapIndex): process") + } +}