Added AudioTap functionality

Sources/SwiftAudioEx/AVPlayerWrapper/AVPlayerWrapper.swift

@@ -24,6 +24,7 @@ class AVPlayerWrapper: AVPlayerWrapperProtocol {
     // MARK: - Properties
 
     fileprivate var avPlayer = AVPlayer()
+    internal var audioTap: AudioTap? = nil
     private let playerObserver = AVPlayerObserver()
     internal let playerTimeObserver: AVPlayerTimeObserver
     private let playerItemNotificationObserver = AVPlayerItemNotificationObserver()
@@ -385,6 +386,7 @@ class AVPlayerWrapper: AVPlayerWrapperProtocol {
     private func startObservingAVPlayer(item: AVPlayerItem) {
         playerItemObserver.startObserving(item: item)
         playerItemNotificationObserver.startObserving(item: item)
+        attachTap(audioTap, to: item)
     }
 
     private func stopObservingAVPlayerItem() {

Sources/SwiftAudioEx/AVPlayerWrapper/AVPlayerWrapperProtocol.swift

@@ -13,6 +13,8 @@ protocol AVPlayerWrapperProtocol: AnyObject {
 
     var state: AVPlayerWrapperState { get set }
 
+    var audioTap: AudioTap? { get set }
+
     var playWhenReady: Bool { get set }
 
     var currentItem: AVPlayerItem? { get }

Sources/SwiftAudioEx/AudioPlayer.swift

@@ -13,6 +13,14 @@ public typealias AudioPlayerState = AVPlayerWrapperState
 public class AudioPlayer: AVPlayerWrapperDelegate {
     /// The wrapper around the underlying AVPlayer
     let wrapper: AVPlayerWrapperProtocol = AVPlayerWrapper()
+
+    /**
+     Set an instance of AudioTap, to receive frame information and audio buffer access during playback.
+     */
+    public var audioTap: AudioTap? {
+        get { return wrapper.audioTap }
+        set(value) { wrapper.audioTap = value }
+    }
 
     public let nowPlayingInfoController: NowPlayingInfoControllerProtocol
     public let remoteCommandController: RemoteCommandController

Sources/SwiftAudioEx/AudioTap.swift

@@ -0,0 +1,98 @@
+//
+//  AudioTap.swift
+//
+//
+//  Created by Brandon Sneed on 3/31/24.
+//
+
+import Foundation
+import AVFoundation
+
+/**
+ Subclass this and set the AudioPlayer's `audioTap` property to start receiving the
+ audio stream.
+ */
+open class AudioTap {
+    // Called at tap initialization for a given player item. Use this to setup anything you might need.
+    open func initialize() { print("audioTap: initialize") }
+    // Called at teardown of the internal tap.  Use this to reset any memory buffers you have created, etc.
+    open func finalize() { print("audioTap: finalize") }
+    // Called just before playback so you can perform setup based on the stream description.
+    open func prepare(description: AudioStreamBasicDescription) { print("audioTap: prepare") }
+    // Called just before finalize.
+    open func unprepare() { print("audioTap: unprepare") }
+    /**
+     Called periodically during audio stream playback.
+
+     Example:
+
+     ```
+     func process(numberOfFrames: Int, buffer: UnsafeMutableAudioBufferListPointer) {
+         for channel in buffer {
+             // process audio samples here
+             //memset(channel.mData, 0, Int(channel.mDataByteSize))
+         }
+     }
+     ```
+    */
+    open func process(numberOfFrames: Int, buffer: UnsafeMutableAudioBufferListPointer) { print("audioTap: process") }
+}
+
+extension AVPlayerWrapper {
+    internal func attachTap(_ tap: AudioTap?, to item: AVPlayerItem) {
+        guard let tap else { return }
+        guard let track = item.asset.tracks(withMediaType: .audio).first else {
+            return
+        }
+
+        let audioMix = AVMutableAudioMix()
+        let params = AVMutableAudioMixInputParameters(track: track)
+
+        // we need to retain this pointer so it doesn't disappear out from under us.
+        // we'll then let it go after we finalize.  If the tap changed upstream, we
+        // aren't going to pick up the new one until after this player item goes away.
+        let client = UnsafeMutableRawPointer(Unmanaged.passRetained(tap).toOpaque())
+
+        var callbacks = MTAudioProcessingTapCallbacks(version: kMTAudioProcessingTapCallbacksVersion_0, clientInfo: client)
+        { tapRef, clientInfo, tapStorageOut in
+            // initial tap setup
+            guard let clientInfo else { return }
+            tapStorageOut.pointee = clientInfo
+            let audioTap = Unmanaged<AudioTap>.fromOpaque(clientInfo).takeUnretainedValue()
+            audioTap.initialize()
+        } finalize: { tapRef in
+            // clean up
+            let audioTap = Unmanaged<AudioTap>.fromOpaque(MTAudioProcessingTapGetStorage(tapRef)).takeUnretainedValue()
+            audioTap.finalize()
+            // we're done, we can let go of the pointer we retained.
+            Unmanaged.passUnretained(audioTap).release()
+        } prepare: { tapRef, maxFrames, processingFormat in
+            // allocate memory for sound processing
+            let audioTap = Unmanaged<AudioTap>.fromOpaque(MTAudioProcessingTapGetStorage(tapRef)).takeUnretainedValue()
+            audioTap.prepare(description: processingFormat.pointee)
+        } unprepare: { tapRef in
+            // deallocate memory for sound processing
+            let audioTap = Unmanaged<AudioTap>.fromOpaque(MTAudioProcessingTapGetStorage(tapRef)).takeUnretainedValue()
+            audioTap.unprepare()
+        } process: { tapRef, numberFrames, flags, bufferListInOut, numberFramesOut, flagsOut in
+            guard noErr == MTAudioProcessingTapGetSourceAudio(tapRef, numberFrames, bufferListInOut, flagsOut, nil, numberFramesOut) else {
+                return
+            }
+
+            // process sound data
+            let audioTap = Unmanaged<AudioTap>.fromOpaque(MTAudioProcessingTapGetStorage(tapRef)).takeUnretainedValue()
+            audioTap.process(numberOfFrames: numberFrames, buffer: UnsafeMutableAudioBufferListPointer(bufferListInOut))
+        }
+
+        var tapRef: Unmanaged<MTAudioProcessingTap>?
+        let error = MTAudioProcessingTapCreate(kCFAllocatorDefault, &callbacks, kMTAudioProcessingTapCreationFlag_PreEffects, &tapRef)
+        assert(error == noErr)
+
+        params.audioTapProcessor = tapRef?.takeUnretainedValue()
+        tapRef?.release()
+
+        audioMix.inputParameters = [params]
+        item.audioMix = audioMix
+    }
+}
+

Sources/SwiftAudioEx/Observer/AVPlayerItemObserver.swift

@@ -63,6 +63,7 @@ class AVPlayerItemObserver: NSObject {
 
         self.isObserving = true
         self.observingItem = item
+
         item.addObserver(self, forKeyPath: AVPlayerItemKeyPath.duration, options: [.new], context: &AVPlayerItemObserver.context)
         item.addObserver(self, forKeyPath: AVPlayerItemKeyPath.loadedTimeRanges, options: [.new], context: &AVPlayerItemObserver.context)
         item.addObserver(self, forKeyPath: AVPlayerItemKeyPath.playbackLikelyToKeepUp, options: [.new], context: &AVPlayerItemObserver.context)
@@ -79,6 +80,9 @@ class AVPlayerItemObserver: NSObject {
             return
         }
 
+        // BKS: remove a tap if we had one.
+        observingItem.audioMix = nil
+
         observingItem.removeObserver(self, forKeyPath: AVPlayerItemKeyPath.duration, context: &AVPlayerItemObserver.context)
         observingItem.removeObserver(self, forKeyPath: AVPlayerItemKeyPath.loadedTimeRanges, context: &AVPlayerItemObserver.context)
         observingItem.removeObserver(self, forKeyPath: AVPlayerItemKeyPath.playbackLikelyToKeepUp, context: &AVPlayerItemObserver.context)

Sources/SwiftAudioEx/Utils/Devices.swift

@@ -0,0 +1,8 @@
+//
+//  File.swift
+//  
+//
+//  Created by Brandon Sneed on 4/1/24.
+//
+
+import Foundation

Tests/SwiftAudioExTests/AudioPlayerTests.swift

@@ -110,6 +110,38 @@ class AudioPlayerTests: XCTestCase {
         XCTAssertEqual(audioPlayer.duration, 0)
     }
 
+    // MARK: - Audio Tap testing
+
+    func testAudioTapSwitching() {
+        listener.onSecondsElapse = { position in
+            if position > 4 {
+                // swap it out part-way through the first track.
+                self.audioPlayer.audioTap = DummyAudioTap(tapIndex: 2)
+            }
+        }
+
+        audioPlayer.audioTap = DummyAudioTap(tapIndex: 1)
+        audioPlayer.load(item: FiveSecondSource.getAudioItem())
+        audioPlayer.play()
+
+        RunLoop.current.run(until: Date(timeIntervalSinceNow: 6))
+
+        audioPlayer.load(item: FiveSecondSource.getAudioItem())
+        audioPlayer.play()
+
+        RunLoop.current.run(until: Date(timeIntervalSinceNow: 6))
+
+        let tap1Active = DummyAudioTap.outputs.contains { output in
+            return output.contains("audioTap 1: process")
+        }
+
+        let tap2Active = DummyAudioTap.outputs.contains { output in
+            return output.contains("audioTap 2: process")
+        }
+        XCTAssertTrue(tap1Active)
+        XCTAssertTrue(tap2Active)
+    }
+
     // MARK: - Failure
 
     func testFailEventOnLoadWithNonMalformedURL() {

Tests/SwiftAudioExTests/Mocks/DummyAudioTap.swift

@@ -0,0 +1,40 @@
+//
+//  File.swift
+//  
+//
+//  Created by Brandon Sneed on 4/1/24.
+//
+
+import Foundation
+import CoreAudio
+@testable import SwiftAudioEx
+
+class DummyAudioTap: AudioTap {
+    static var outputs = [String]()
+
+    let tapIndex: Int
+
+    init(tapIndex: Int) {
+        self.tapIndex = tapIndex
+    }
+
+    override func initialize() { 
+        Self.outputs.append("audioTap \(tapIndex): initialize")
+    }
+
+    override func finalize() {
+        Self.outputs.append("audioTap \(tapIndex): finalize")
+    }
+
+    override func prepare(description: AudioStreamBasicDescription) {
+        Self.outputs.append("audioTap \(tapIndex): prepare")
+    }
+
+    override func unprepare() {
+        Self.outputs.append("audioTap \(tapIndex): unprepare")
+    }
+
+    override func process(numberOfFrames: Int, buffer: UnsafeMutableAudioBufferListPointer) {
+        Self.outputs.append("audioTap \(tapIndex): process")
+    }
+}