diff --git a/ios/.swift-format b/ios/.swift-format new file mode 100644 index 00000000..fb8d86a2 --- /dev/null +++ b/ios/.swift-format @@ -0,0 +1,75 @@ +{ + "fileScopedDeclarationPrivacy" : { + "accessLevel" : "private" + }, + "indentConditionalCompilationBlocks" : true, + "indentSwitchCaseLabels" : false, + "indentation" : { + "spaces" : 2 + }, + "lineBreakAroundMultilineExpressionChainComponents" : false, + "lineBreakBeforeControlFlowKeywords" : false, + "lineBreakBeforeEachArgument" : false, + "lineBreakBeforeEachGenericRequirement" : false, + "lineBreakBetweenDeclarationAttributes" : false, + "lineLength" : 120, + "maximumBlankLines" : 1, + "multiElementCollectionTrailingCommas" : true, + "noAssignmentInExpressions" : { + "allowedFunctions" : [ + "XCTAssertNoThrow" + ] + }, + "prioritizeKeepingFunctionOutputTogether" : false, + "reflowMultilineStringLiterals" : "never", + "respectsExistingLineBreaks" : true, + "rules" : { + "AllPublicDeclarationsHaveDocumentation" : false, + "AlwaysUseLiteralForEmptyCollectionInit" : false, + "AlwaysUseLowerCamelCase" : true, + "AmbiguousTrailingClosureOverload" : true, + "AvoidRetroactiveConformances" : true, + "BeginDocumentationCommentWithOneLineSummary" : false, + "DoNotUseSemicolons" : true, + "DontRepeatTypeInStaticProperties" : true, + "FileScopedDeclarationPrivacy" : true, + "FullyIndirectEnum" : true, + "GroupNumericLiterals" : true, + "IdentifiersMustBeASCII" : true, + "NeverForceUnwrap" : false, + "NeverUseForceTry" : false, + "NeverUseImplicitlyUnwrappedOptionals" : false, + "NoAccessLevelOnExtensionDeclaration" : true, + "NoAssignmentInExpressions" : true, + "NoBlockComments" : true, + "NoCasesWithOnlyFallthrough" : true, + "NoEmptyLinesOpeningClosingBraces" : false, + "NoEmptyTrailingClosureParentheses" : true, + "NoLabelsInCasePatterns" : true, + "NoLeadingUnderscores" : false, + "NoParensAroundConditions" : true, + "NoPlaygroundLiterals" : true, + "NoVoidReturnOnFunctionSignature" : true, + "OmitExplicitReturns" : false, + "OneCasePerLine" : true, + "OneVariableDeclarationPerLine" : true, + "OnlyOneTrailingClosureArgument" : true, + "OrderedImports" : true, + "ReplaceForEachWithForLoop" : true, + "ReturnVoidInsteadOfEmptyTuple" : true, + "TypeNamesShouldBeCapitalized" : true, + "UseEarlyExits" : false, + "UseExplicitNilCheckInConditions" : true, + "UseLetInEveryBoundCaseVariable" : true, + "UseShorthandTypeNames" : true, + "UseSingleLinePropertyGetter" : true, + "UseSynthesizedInitializer" : true, + "UseTripleSlashForDocumentationComments" : true, + "UseWhereClausesInForLoops" : false, + "ValidateDocumentationComments" : false + }, + "spacesAroundRangeFormationOperators" : false, + "spacesBeforeEndOfLineComments" : 2, + "tabWidth" : 4, + "version" : 1 +} diff --git a/ios/.swift-version b/ios/.swift-version new file mode 100644 index 00000000..95ee81a4 --- /dev/null +++ b/ios/.swift-version @@ -0,0 +1 @@ +5.9 diff --git a/ios/AudioUtils.swift b/ios/AudioUtils.swift index a84fcb2c..4bce612c 100644 --- a/ios/AudioUtils.swift +++ b/ios/AudioUtils.swift @@ -1,84 +1,86 @@ import AVFoundation public class AudioUtils { - public static func audioSessionModeFromString(_ mode: String) -> AVAudioSession.Mode { - let retMode: AVAudioSession.Mode = switch mode { - case "default_": - .default - case "voicePrompt": - if #available(iOS 12.0, *) { - .voicePrompt - } else { - .default - } - case "videoRecording": - .videoRecording - case "videoChat": - .videoChat - case "voiceChat": - .voiceChat - case "gameChat": - .gameChat - case "measurement": - .measurement - case "moviePlayback": - .moviePlayback - case "spokenAudio": - .spokenAudio - default: - .default + public static func audioSessionModeFromString(_ mode: String) -> AVAudioSession.Mode { + let retMode: AVAudioSession.Mode = + switch mode { + case "default_": + .default + case "voicePrompt": + if #available(iOS 12.0, *) { + .voicePrompt + } else { + .default } - return retMode - } + case "videoRecording": + .videoRecording + case "videoChat": + .videoChat + case "voiceChat": + .voiceChat + case "gameChat": + .gameChat + case "measurement": + .measurement + case "moviePlayback": + .moviePlayback + case "spokenAudio": + .spokenAudio + default: + .default + } + return retMode + } - public static func audioSessionCategoryFromString(_ category: String) -> AVAudioSession.Category { - let retCategory: AVAudioSession.Category = switch category { - case "ambient": - .ambient - case "soloAmbient": - .soloAmbient - case "playback": - .playback - case "record": - .record - case "playAndRecord": - .playAndRecord - case "multiRoute": - .multiRoute - default: - .soloAmbient - } - return retCategory - } + public static func audioSessionCategoryFromString(_ category: String) -> AVAudioSession.Category { + let retCategory: AVAudioSession.Category = + switch category { + case "ambient": + .ambient + case "soloAmbient": + .soloAmbient + case "playback": + .playback + case "record": + .record + case "playAndRecord": + .playAndRecord + case "multiRoute": + .multiRoute + default: + .soloAmbient + } + return retCategory + } - public static func audioSessionCategoryOptionsFromStrings(_ options: [String]) -> AVAudioSession.CategoryOptions { - var categoryOptions: AVAudioSession.CategoryOptions = [] - for option in options { - switch option { - case "mixWithOthers": - categoryOptions.insert(.mixWithOthers) - case "duckOthers": - categoryOptions.insert(.duckOthers) - case "allowBluetooth": - categoryOptions.insert(.allowBluetooth) - case "allowBluetoothA2DP": - categoryOptions.insert(.allowBluetoothA2DP) - case "allowAirPlay": - categoryOptions.insert(.allowAirPlay) - case "defaultToSpeaker": - categoryOptions.insert(.defaultToSpeaker) - case "interruptSpokenAudioAndMixWithOthers": - if #available(iOS 13.0, *) { - categoryOptions.insert(.interruptSpokenAudioAndMixWithOthers) - } - case "overrideMutedMicrophoneInterruption": - if #available(iOS 14.5, *) { - categoryOptions.insert(.overrideMutedMicrophoneInterruption) - } - default: - break - } + public static func audioSessionCategoryOptionsFromStrings(_ options: [String]) -> AVAudioSession.CategoryOptions { + var categoryOptions: AVAudioSession.CategoryOptions = [] + for option in options { + switch option { + case "mixWithOthers": + categoryOptions.insert(.mixWithOthers) + case "duckOthers": + categoryOptions.insert(.duckOthers) + case "allowBluetooth": + categoryOptions.insert(.allowBluetooth) + case "allowBluetoothA2DP": + categoryOptions.insert(.allowBluetoothA2DP) + case "allowAirPlay": + categoryOptions.insert(.allowAirPlay) + case "defaultToSpeaker": + categoryOptions.insert(.defaultToSpeaker) + case "interruptSpokenAudioAndMixWithOthers": + if #available(iOS 13.0, *) { + categoryOptions.insert(.interruptSpokenAudioAndMixWithOthers) + } + case "overrideMutedMicrophoneInterruption": + if #available(iOS 14.5, *) { + categoryOptions.insert(.overrideMutedMicrophoneInterruption) } - return categoryOptions + default: + break + } } + return categoryOptions + } } diff --git a/ios/LiveKitReactNativeModule.swift b/ios/LiveKitReactNativeModule.swift index b3c3ef47..cba2175e 100644 --- a/ios/LiveKitReactNativeModule.swift +++ b/ios/LiveKitReactNativeModule.swift @@ -1,259 +1,260 @@ -import livekit_react_native_webrtc -import AVFoundation import AVFAudio +import AVFoundation import React +import livekit_react_native_webrtc struct LKEvents { - static let kEventVolumeProcessed = "LK_VOLUME_PROCESSED"; - static let kEventMultibandProcessed = "LK_MULTIBAND_PROCESSED"; - static let kEventAudioData = "LK_AUDIO_DATA"; + static let kEventVolumeProcessed = "LK_VOLUME_PROCESSED" + static let kEventMultibandProcessed = "LK_MULTIBAND_PROCESSED" + static let kEventAudioData = "LK_AUDIO_DATA" } @objc(LivekitReactNativeModule) public class LivekitReactNativeModule: RCTEventEmitter { - // This cannot be initialized in init as self.bridge is given afterwards. - private var _audioRendererManager: AudioRendererManager? = nil - public var audioRendererManager: AudioRendererManager { - get { - if _audioRendererManager == nil { - _audioRendererManager = AudioRendererManager(bridge: self.bridge) - } - - return _audioRendererManager! - } - } - - @objc - public override init() { - super.init() - let config = RTCAudioSessionConfiguration() - config.category = AVAudioSession.Category.playAndRecord.rawValue - config.categoryOptions = [.allowAirPlay, .allowBluetooth, .allowBluetoothA2DP, .defaultToSpeaker] - config.mode = AVAudioSession.Mode.videoChat.rawValue - - RTCAudioSessionConfiguration.setWebRTC(config) + // This cannot be initialized in init as self.bridge is given afterwards. + private var _audioRendererManager: AudioRendererManager? = nil + public var audioRendererManager: AudioRendererManager { + if _audioRendererManager == nil { + _audioRendererManager = AudioRendererManager(bridge: self.bridge) } - @objc - override public static func requiresMainQueueSetup() -> Bool { - return false + return _audioRendererManager! + } + + @objc + public override init() { + super.init() + let config = RTCAudioSessionConfiguration() + config.category = AVAudioSession.Category.playAndRecord.rawValue + config.categoryOptions = [.allowAirPlay, .allowBluetooth, .allowBluetoothA2DP, .defaultToSpeaker] + config.mode = AVAudioSession.Mode.videoChat.rawValue + + RTCAudioSessionConfiguration.setWebRTC(config) + } + + @objc + override public static func requiresMainQueueSetup() -> Bool { + return false + } + + @objc + public static func setup() { + let videoEncoderFactory = RTCDefaultVideoEncoderFactory() + let simulcastVideoEncoderFactory = RTCVideoEncoderFactorySimulcast( + primary: videoEncoderFactory, fallback: videoEncoderFactory) + let options = WebRTCModuleOptions.sharedInstance() + options.videoEncoderFactory = simulcastVideoEncoderFactory + options.audioProcessingModule = LKAudioProcessingManager.sharedInstance().audioProcessingModule + } + + @objc(configureAudio:) + public func configureAudio(_ config: NSDictionary) { + guard let iOSConfig = config["ios"] as? NSDictionary + else { + return } - @objc - public static func setup() { - let videoEncoderFactory = RTCDefaultVideoEncoderFactory() - let simulcastVideoEncoderFactory = RTCVideoEncoderFactorySimulcast(primary: videoEncoderFactory, fallback: videoEncoderFactory) - let options = WebRTCModuleOptions.sharedInstance() - options.videoEncoderFactory = simulcastVideoEncoderFactory - options.audioProcessingModule = LKAudioProcessingManager.sharedInstance().audioProcessingModule - } - - @objc(configureAudio:) - public func configureAudio(_ config: NSDictionary) { - guard let iOSConfig = config["ios"] as? NSDictionary - else { - return - } + let defaultOutput = iOSConfig["defaultOutput"] as? String ?? "speaker" - let defaultOutput = iOSConfig["defaultOutput"] as? String ?? "speaker" + let rtcConfig = RTCAudioSessionConfiguration() + rtcConfig.category = AVAudioSession.Category.playAndRecord.rawValue - let rtcConfig = RTCAudioSessionConfiguration() - rtcConfig.category = AVAudioSession.Category.playAndRecord.rawValue - - if (defaultOutput == "earpiece") { - rtcConfig.categoryOptions = [.allowAirPlay, .allowBluetooth, .allowBluetoothA2DP]; - rtcConfig.mode = AVAudioSession.Mode.voiceChat.rawValue - } else { - rtcConfig.categoryOptions = [.allowAirPlay, .allowBluetooth, .allowBluetoothA2DP, .defaultToSpeaker] - rtcConfig.mode = AVAudioSession.Mode.videoChat.rawValue - } - RTCAudioSessionConfiguration.setWebRTC(rtcConfig) + if defaultOutput == "earpiece" { + rtcConfig.categoryOptions = [.allowAirPlay, .allowBluetooth, .allowBluetoothA2DP] + rtcConfig.mode = AVAudioSession.Mode.voiceChat.rawValue + } else { + rtcConfig.categoryOptions = [.allowAirPlay, .allowBluetooth, .allowBluetoothA2DP, .defaultToSpeaker] + rtcConfig.mode = AVAudioSession.Mode.videoChat.rawValue } - - @objc(startAudioSession:withRejecter:) - public func startAudioSession(resolve: RCTPromiseResolveBlock, reject: RCTPromiseRejectBlock) { - let session = RTCAudioSession.sharedInstance() - session.lockForConfiguration() - defer { - session.unlockForConfiguration() - } - - do { - try session.setActive(true) - resolve(nil) - } catch { - reject("startAudioSession", "Error activating audio session: \(error.localizedDescription)", error) - } - } - - @objc(stopAudioSession:withRejecter:) - public func stopAudioSession(resolve: RCTPromiseResolveBlock, reject: RCTPromiseRejectBlock) { - let session = RTCAudioSession.sharedInstance() - session.lockForConfiguration() - defer { - session.unlockForConfiguration() - } - - do { - try session.setActive(false) - resolve(nil) - } catch { - reject("stopAudioSession", "Error deactivating audio session: \(error.localizedDescription)", error) - } + RTCAudioSessionConfiguration.setWebRTC(rtcConfig) + } + + @objc(startAudioSession:withRejecter:) + public func startAudioSession(resolve: RCTPromiseResolveBlock, reject: RCTPromiseRejectBlock) { + let session = RTCAudioSession.sharedInstance() + session.lockForConfiguration() + defer { + session.unlockForConfiguration() } - @objc(showAudioRoutePicker) - public func showAudioRoutePicker() { - if #available(iOS 11.0, *) { - let routePickerView = AVRoutePickerView() - let subviews = routePickerView.subviews - for subview in subviews { - if subview.isKind(of: UIButton.self) { - let button = subview as! UIButton - button.sendActions(for: .touchUpInside) - break - } - } - } + do { + try session.setActive(true) + resolve(nil) + } catch { + reject("startAudioSession", "Error activating audio session: \(error.localizedDescription)", error) } - - @objc(getAudioOutputsWithResolver:withRejecter:) - public func getAudioOutputs(resolve: RCTPromiseResolveBlock, reject: RCTPromiseRejectBlock){ - resolve(["default", "force_speaker"]) + } + + @objc(stopAudioSession:withRejecter:) + public func stopAudioSession(resolve: RCTPromiseResolveBlock, reject: RCTPromiseRejectBlock) { + let session = RTCAudioSession.sharedInstance() + session.lockForConfiguration() + defer { + session.unlockForConfiguration() } - @objc(selectAudioOutput:withResolver:withRejecter:) - public func selectAudioOutput(_ deviceId: String, resolve: RCTPromiseResolveBlock, reject: RCTPromiseRejectBlock) { - let session = AVAudioSession.sharedInstance() - do { - if (deviceId == "default") { - try session.overrideOutputAudioPort(.none) - } else if (deviceId == "force_speaker") { - try session.overrideOutputAudioPort(.speaker) - } - } catch { - reject("selectAudioOutput error", error.localizedDescription, error) - return - } - - resolve(nil) + do { + try session.setActive(false) + resolve(nil) + } catch { + reject("stopAudioSession", "Error deactivating audio session: \(error.localizedDescription)", error) } - - @objc(setAppleAudioConfiguration:withResolver:withRejecter:) - public func setAppleAudioConfiguration(_ configuration: NSDictionary, resolve: RCTPromiseResolveBlock, reject: RCTPromiseRejectBlock) { - let session = RTCAudioSession.sharedInstance() - let config = RTCAudioSessionConfiguration.webRTC() - - let appleAudioCategory = configuration["audioCategory"] as? String - let appleAudioCategoryOptions = configuration["audioCategoryOptions"] as? [String] - let appleAudioMode = configuration["audioMode"] as? String - - session.lockForConfiguration() - defer { - session.unlockForConfiguration() - } - - if let appleAudioCategory = appleAudioCategory { - config.category = AudioUtils.audioSessionCategoryFromString(appleAudioCategory).rawValue + } + + @objc(showAudioRoutePicker) + public func showAudioRoutePicker() { + if #available(iOS 11.0, *) { + let routePickerView = AVRoutePickerView() + let subviews = routePickerView.subviews + for subview in subviews { + if subview.isKind(of: UIButton.self) { + let button = subview as! UIButton + button.sendActions(for: .touchUpInside) + break } - - if let appleAudioCategoryOptions = appleAudioCategoryOptions { - config.categoryOptions = AudioUtils.audioSessionCategoryOptionsFromStrings(appleAudioCategoryOptions) - } - - if let appleAudioMode = appleAudioMode { - config.mode = AudioUtils.audioSessionModeFromString(appleAudioMode).rawValue - } - - do { - try session.setConfiguration(config) - resolve(nil) - } catch { - reject("setAppleAudioConfiguration", "Error setting category: \(error.localizedDescription)", error) - return - } - + } } - - @objc(createAudioSinkListener:trackId:) - public func createAudioSinkListener(_ pcId: NSNumber, trackId: String) -> String { - let renderer = AudioSinkRenderer(eventEmitter: self) - let reactTag = self.audioRendererManager.registerRenderer(renderer) - renderer.reactTag = reactTag - self.audioRendererManager.attach(renderer: renderer, pcId: pcId, trackId: trackId) - - return reactTag + } + + @objc(getAudioOutputsWithResolver:withRejecter:) + public func getAudioOutputs(resolve: RCTPromiseResolveBlock, reject: RCTPromiseRejectBlock) { + resolve(["default", "force_speaker"]) + } + + @objc(selectAudioOutput:withResolver:withRejecter:) + public func selectAudioOutput(_ deviceId: String, resolve: RCTPromiseResolveBlock, reject: RCTPromiseRejectBlock) { + let session = AVAudioSession.sharedInstance() + do { + if deviceId == "default" { + try session.overrideOutputAudioPort(.none) + } else if deviceId == "force_speaker" { + try session.overrideOutputAudioPort(.speaker) + } + } catch { + reject("selectAudioOutput error", error.localizedDescription, error) + return } - @objc(deleteAudioSinkListener:pcId:trackId:) - public func deleteAudioSinkListener(_ reactTag: String, pcId: NSNumber, trackId: String) -> Any? { - self.audioRendererManager.detach(rendererByTag: reactTag, pcId: pcId, trackId: trackId) - self.audioRendererManager.unregisterRenderer(forReactTag: reactTag) + resolve(nil) + } - return nil - } + @objc(setAppleAudioConfiguration:withResolver:withRejecter:) + public func setAppleAudioConfiguration( + _ configuration: NSDictionary, resolve: RCTPromiseResolveBlock, reject: RCTPromiseRejectBlock + ) { + let session = RTCAudioSession.sharedInstance() + let config = RTCAudioSessionConfiguration.webRTC() - @objc(createVolumeProcessor:trackId:) - public func createVolumeProcessor(_ pcId: NSNumber, trackId: String) -> String { - let renderer = VolumeAudioRenderer(intervalMs: 40.0, eventEmitter: self) - let reactTag = self.audioRendererManager.registerRenderer(renderer) - renderer.reactTag = reactTag - self.audioRendererManager.attach(renderer: renderer, pcId: pcId, trackId: trackId) + let appleAudioCategory = configuration["audioCategory"] as? String + let appleAudioCategoryOptions = configuration["audioCategoryOptions"] as? [String] + let appleAudioMode = configuration["audioMode"] as? String - return reactTag + session.lockForConfiguration() + defer { + session.unlockForConfiguration() } - @objc(deleteVolumeProcessor:pcId:trackId:) - public func deleteVolumeProcessor(_ reactTag: String, pcId: NSNumber, trackId: String) -> Any? { - self.audioRendererManager.detach(rendererByTag: reactTag, pcId: pcId, trackId: trackId) - self.audioRendererManager.unregisterRenderer(forReactTag: reactTag) - - return nil + if let appleAudioCategory = appleAudioCategory { + config.category = AudioUtils.audioSessionCategoryFromString(appleAudioCategory).rawValue } - @objc(createMultibandVolumeProcessor:pcId:trackId:) - public func createMultibandVolumeProcessor(_ options: NSDictionary, pcId: NSNumber, trackId: String) -> String { - let bands = (options["bands"] as? NSNumber)?.intValue ?? 5 - let minFrequency = (options["minFrequency"] as? NSNumber)?.floatValue ?? 1000 - let maxFrequency = (options["maxFrequency"] as? NSNumber)?.floatValue ?? 8000 - let intervalMs = (options["updateInterval"] as? NSNumber)?.floatValue ?? 40 - - let renderer = MultibandVolumeAudioRenderer( - bands: bands, - minFrequency: minFrequency, - maxFrequency: maxFrequency, - intervalMs: intervalMs, - eventEmitter: self - ) - let reactTag = self.audioRendererManager.registerRenderer(renderer) - renderer.reactTag = reactTag - self.audioRendererManager.attach(renderer: renderer, pcId: pcId, trackId: trackId) - - return reactTag + if let appleAudioCategoryOptions = appleAudioCategoryOptions { + config.categoryOptions = AudioUtils.audioSessionCategoryOptionsFromStrings(appleAudioCategoryOptions) } - @objc(deleteMultibandVolumeProcessor:pcId:trackId:) - public func deleteMultibandVolumeProcessor(_ reactTag: String, pcId: NSNumber, trackId: String) -> Any? { - self.audioRendererManager.detach(rendererByTag: reactTag, pcId: pcId, trackId: trackId) - self.audioRendererManager.unregisterRenderer(forReactTag: reactTag) - - return nil + if let appleAudioMode = appleAudioMode { + config.mode = AudioUtils.audioSessionModeFromString(appleAudioMode).rawValue } - @objc(setDefaultAudioTrackVolume:) - public func setDefaultAudioTrackVolume(_ volume: NSNumber) -> Any? { - let options = WebRTCModuleOptions.sharedInstance() - options.defaultTrackVolume = volume.doubleValue - - return nil + do { + try session.setConfiguration(config) + resolve(nil) + } catch { + reject("setAppleAudioConfiguration", "Error setting category: \(error.localizedDescription)", error) + return } - override public func supportedEvents() -> [String]! { - return [ - LKEvents.kEventVolumeProcessed, - LKEvents.kEventMultibandProcessed, - LKEvents.kEventAudioData, - ] - } + } + + @objc(createAudioSinkListener:trackId:) + public func createAudioSinkListener(_ pcId: NSNumber, trackId: String) -> String { + let renderer = AudioSinkRenderer(eventEmitter: self) + let reactTag = self.audioRendererManager.registerRenderer(renderer) + renderer.reactTag = reactTag + self.audioRendererManager.attach(renderer: renderer, pcId: pcId, trackId: trackId) + + return reactTag + } + + @objc(deleteAudioSinkListener:pcId:trackId:) + public func deleteAudioSinkListener(_ reactTag: String, pcId: NSNumber, trackId: String) -> Any? { + self.audioRendererManager.detach(rendererByTag: reactTag, pcId: pcId, trackId: trackId) + self.audioRendererManager.unregisterRenderer(forReactTag: reactTag) + + return nil + } + + @objc(createVolumeProcessor:trackId:) + public func createVolumeProcessor(_ pcId: NSNumber, trackId: String) -> String { + let renderer = VolumeAudioRenderer(intervalMs: 40.0, eventEmitter: self) + let reactTag = self.audioRendererManager.registerRenderer(renderer) + renderer.reactTag = reactTag + self.audioRendererManager.attach(renderer: renderer, pcId: pcId, trackId: trackId) + + return reactTag + } + + @objc(deleteVolumeProcessor:pcId:trackId:) + public func deleteVolumeProcessor(_ reactTag: String, pcId: NSNumber, trackId: String) -> Any? { + self.audioRendererManager.detach(rendererByTag: reactTag, pcId: pcId, trackId: trackId) + self.audioRendererManager.unregisterRenderer(forReactTag: reactTag) + + return nil + } + + @objc(createMultibandVolumeProcessor:pcId:trackId:) + public func createMultibandVolumeProcessor(_ options: NSDictionary, pcId: NSNumber, trackId: String) -> String { + let bands = (options["bands"] as? NSNumber)?.intValue ?? 5 + let minFrequency = (options["minFrequency"] as? NSNumber)?.floatValue ?? 1000 + let maxFrequency = (options["maxFrequency"] as? NSNumber)?.floatValue ?? 8000 + let intervalMs = (options["updateInterval"] as? NSNumber)?.floatValue ?? 40 + + let renderer = MultibandVolumeAudioRenderer( + bands: bands, + minFrequency: minFrequency, + maxFrequency: maxFrequency, + intervalMs: intervalMs, + eventEmitter: self + ) + let reactTag = self.audioRendererManager.registerRenderer(renderer) + renderer.reactTag = reactTag + self.audioRendererManager.attach(renderer: renderer, pcId: pcId, trackId: trackId) + + return reactTag + } + + @objc(deleteMultibandVolumeProcessor:pcId:trackId:) + public func deleteMultibandVolumeProcessor(_ reactTag: String, pcId: NSNumber, trackId: String) -> Any? { + self.audioRendererManager.detach(rendererByTag: reactTag, pcId: pcId, trackId: trackId) + self.audioRendererManager.unregisterRenderer(forReactTag: reactTag) + + return nil + } + + @objc(setDefaultAudioTrackVolume:) + public func setDefaultAudioTrackVolume(_ volume: NSNumber) -> Any? { + let options = WebRTCModuleOptions.sharedInstance() + options.defaultTrackVolume = volume.doubleValue + + return nil + } + + override public func supportedEvents() -> [String]! { + return [ + LKEvents.kEventVolumeProcessed, + LKEvents.kEventMultibandProcessed, + LKEvents.kEventAudioData, + ] + } } diff --git a/ios/audio/AVAudioPCMBuffer.swift b/ios/audio/AVAudioPCMBuffer.swift index 5e8fd493..fb9b2515 100644 --- a/ios/audio/AVAudioPCMBuffer.swift +++ b/ios/audio/AVAudioPCMBuffer.swift @@ -14,123 +14,130 @@ * limitations under the License. */ -import Accelerate import AVFoundation +import Accelerate + +extension AVAudioPCMBuffer { + public func resample(toSampleRate targetSampleRate: Double) -> AVAudioPCMBuffer? { + let sourceFormat = format + + if sourceFormat.sampleRate == targetSampleRate { + // Already targetSampleRate. + return self + } -public extension AVAudioPCMBuffer { - func resample(toSampleRate targetSampleRate: Double) -> AVAudioPCMBuffer? { - let sourceFormat = format - - if sourceFormat.sampleRate == targetSampleRate { - // Already targetSampleRate. - return self - } - - // Define the source format (from the input buffer) and the target format. - guard let targetFormat = AVAudioFormat(commonFormat: sourceFormat.commonFormat, - sampleRate: targetSampleRate, - channels: sourceFormat.channelCount, - interleaved: sourceFormat.isInterleaved) - else { - print("Failed to create target format.") - return nil - } - - guard let converter = AVAudioConverter(from: sourceFormat, to: targetFormat) else { - print("Failed to create audio converter.") - return nil - } - - let capacity = targetFormat.sampleRate * Double(frameLength) / sourceFormat.sampleRate - - guard let convertedBuffer = AVAudioPCMBuffer(pcmFormat: targetFormat, frameCapacity: AVAudioFrameCount(capacity)) else { - print("Failed to create converted buffer.") - return nil - } - - var isDone = false - let inputBlock: AVAudioConverterInputBlock = { _, outStatus in - if isDone { - outStatus.pointee = .noDataNow - return nil - } - outStatus.pointee = .haveData - isDone = true - return self - } - - var error: NSError? - let status = converter.convert(to: convertedBuffer, error: &error, withInputFrom: inputBlock) - - if status == .error { - print("Conversion failed: \(error?.localizedDescription ?? "Unknown error")") - return nil - } - - // Adjust frame length to the actual amount of data written - convertedBuffer.frameLength = convertedBuffer.frameCapacity - - return convertedBuffer + // Define the source format (from the input buffer) and the target format. + guard + let targetFormat = AVAudioFormat( + commonFormat: sourceFormat.commonFormat, + sampleRate: targetSampleRate, + channels: sourceFormat.channelCount, + interleaved: sourceFormat.isInterleaved) + else { + print("Failed to create target format.") + return nil } - /// Convert PCM buffer to specified common format. - /// Currently supports conversion from Int16 to Float32. - func convert(toCommonFormat commonFormat: AVAudioCommonFormat) -> AVAudioPCMBuffer? { - // Check if conversion is needed - guard format.commonFormat != commonFormat else { - return self - } - - // Check if the conversion is supported - guard format.commonFormat == .pcmFormatInt16, commonFormat == .pcmFormatFloat32 else { - print("Unsupported conversion: only Int16 to Float32 is supported") - return nil - } - - // Create output format - guard let outputFormat = AVAudioFormat(commonFormat: commonFormat, - sampleRate: format.sampleRate, - channels: format.channelCount, - interleaved: false) - else { - print("Failed to create output audio format") - return nil - } - - // Create output buffer - guard let outputBuffer = AVAudioPCMBuffer(pcmFormat: outputFormat, - frameCapacity: frameCapacity) - else { - print("Failed to create output PCM buffer") - return nil - } - - outputBuffer.frameLength = frameLength - - let channelCount = Int(format.channelCount) - let frameCount = Int(frameLength) - - // Ensure the source buffer has Int16 data - guard let int16Data = int16ChannelData else { - print("Source buffer doesn't contain Int16 data") - return nil - } - - // Ensure the output buffer has Float32 data - guard let floatData = outputBuffer.floatChannelData else { - print("Failed to get float channel data from output buffer") - return nil - } - - // Convert Int16 to Float32 and normalize to [-1.0, 1.0] - let scale = Float(Int16.max) - var scalar = 1.0 / scale - - for channel in 0 ..< channelCount { - vDSP_vflt16(int16Data[channel], 1, floatData[channel], 1, vDSP_Length(frameCount)) - vDSP_vsmul(floatData[channel], 1, &scalar, floatData[channel], 1, vDSP_Length(frameCount)) - } - - return outputBuffer + guard let converter = AVAudioConverter(from: sourceFormat, to: targetFormat) else { + print("Failed to create audio converter.") + return nil + } + + let capacity = targetFormat.sampleRate * Double(frameLength) / sourceFormat.sampleRate + + guard let convertedBuffer = AVAudioPCMBuffer(pcmFormat: targetFormat, frameCapacity: AVAudioFrameCount(capacity)) + else { + print("Failed to create converted buffer.") + return nil + } + + var isDone = false + let inputBlock: AVAudioConverterInputBlock = { _, outStatus in + if isDone { + outStatus.pointee = .noDataNow + return nil + } + outStatus.pointee = .haveData + isDone = true + return self + } + + var error: NSError? + let status = converter.convert(to: convertedBuffer, error: &error, withInputFrom: inputBlock) + + if status == .error { + print("Conversion failed: \(error?.localizedDescription ?? "Unknown error")") + return nil + } + + // Adjust frame length to the actual amount of data written + convertedBuffer.frameLength = convertedBuffer.frameCapacity + + return convertedBuffer + } + + /// Convert PCM buffer to specified common format. + /// Currently supports conversion from Int16 to Float32. + public func convert(toCommonFormat commonFormat: AVAudioCommonFormat) -> AVAudioPCMBuffer? { + // Check if conversion is needed + guard format.commonFormat != commonFormat else { + return self } + + // Check if the conversion is supported + guard format.commonFormat == .pcmFormatInt16, commonFormat == .pcmFormatFloat32 else { + print("Unsupported conversion: only Int16 to Float32 is supported") + return nil + } + + // Create output format + guard + let outputFormat = AVAudioFormat( + commonFormat: commonFormat, + sampleRate: format.sampleRate, + channels: format.channelCount, + interleaved: false) + else { + print("Failed to create output audio format") + return nil + } + + // Create output buffer + guard + let outputBuffer = AVAudioPCMBuffer( + pcmFormat: outputFormat, + frameCapacity: frameCapacity) + else { + print("Failed to create output PCM buffer") + return nil + } + + outputBuffer.frameLength = frameLength + + let channelCount = Int(format.channelCount) + let frameCount = Int(frameLength) + + // Ensure the source buffer has Int16 data + guard let int16Data = int16ChannelData else { + print("Source buffer doesn't contain Int16 data") + return nil + } + + // Ensure the output buffer has Float32 data + guard let floatData = outputBuffer.floatChannelData else { + print("Failed to get float channel data from output buffer") + return nil + } + + // Convert Int16 to Float32 and normalize to [-1.0, 1.0] + let scale = Float(Int16.max) + var scalar = 1.0 / scale + + for channel in 0.. AVAudioPCMBuffer? { - guard let audioFormat = AVAudioFormat(commonFormat: .pcmFormatInt16, - sampleRate: Double(frames * 100), - channels: AVAudioChannelCount(channels), - interleaved: false), - let pcmBuffer = AVAudioPCMBuffer(pcmFormat: audioFormat, - frameCapacity: AVAudioFrameCount(frames)) - else { return nil } - - pcmBuffer.frameLength = AVAudioFrameCount(frames) - - guard let targetBufferPointer = pcmBuffer.int16ChannelData else { return nil } - - for i in 0 ..< channels { - let sourceBuffer = rawBuffer(forChannel: i) - let targetBuffer = targetBufferPointer[i] - // sourceBuffer is in the format of [Int16] but is stored in 32-bit alignment, we need to pack the Int16 data correctly. - - for frame in 0 ..< frames { - // Cast and pack the source 32-bit Int16 data into the target 16-bit buffer - let clampedValue = max(Float(Int16.min), min(Float(Int16.max), sourceBuffer[frame])) - targetBuffer[frame] = Int16(clampedValue) - } - } - - return pcmBuffer +extension RTCAudioBuffer { + /// Convert to AVAudioPCMBuffer Int16 format. + @objc + public func toAVAudioPCMBuffer() -> AVAudioPCMBuffer? { + guard + let audioFormat = AVAudioFormat( + commonFormat: .pcmFormatInt16, + sampleRate: Double(frames * 100), + channels: AVAudioChannelCount(channels), + interleaved: false), + let pcmBuffer = AVAudioPCMBuffer( + pcmFormat: audioFormat, + frameCapacity: AVAudioFrameCount(frames)) + else { return nil } + + pcmBuffer.frameLength = AVAudioFrameCount(frames) + + guard let targetBufferPointer = pcmBuffer.int16ChannelData else { return nil } + + for i in 0.. [AudioLevel] { - var result: [AudioLevel] = [] - guard let data = floatChannelData else { - // Not containing float data - return result - } - - for i in 0 ..< Int(format.channelCount) { - let channelData = data[i] - var max: Float = 0.0 - vDSP_maxv(channelData, stride, &max, vDSP_Length(frameLength)) - var rms: Float = 0.0 - vDSP_rmsqv(channelData, stride, &rms, vDSP_Length(frameLength)) - - // No conversion to dB, return linear scale values directly - result.append(AudioLevel(average: rms, peak: max)) - } - - return result - } + return pcmBuffer + } } -public extension Sequence where Iterator.Element == AudioLevel { - /// Combines all elements into a single audio level by computing the average value of all elements. - func combine() -> AudioLevel? { - var count = 0 - let totalSums: (averageSum: Float, peakSum: Float) = reduce((averageSum: 0.0, peakSum: 0.0)) { totals, audioLevel in - count += 1 - return (totals.averageSum + audioLevel.average, - totals.peakSum + audioLevel.peak) - } - - guard count > 0 else { return nil } - - return AudioLevel(average: totalSums.averageSum / Float(count), - peak: totalSums.peakSum / Float(count)) +extension AVAudioPCMBuffer { + /// Computes Peak and Linear Scale RMS Value (Average) for all channels. + public func audioLevels() -> [AudioLevel] { + var result: [AudioLevel] = [] + guard let data = floatChannelData else { + // Not containing float data + return result } -} - -public class AudioVisualizeProcessor { - static let bufferSize = 1024 - - // MARK: - Public - public let minFrequency: Float - public let maxFrequency: Float - public let minDB: Float - public let maxDB: Float - public let bandsCount: Int + for i in 0..(size: AudioVisualizeProcessor.bufferSize) - private let processor: FFTProcessor +extension Sequence where Iterator.Element == AudioLevel { + /// Combines all elements into a single audio level by computing the average value of all elements. + public func combine() -> AudioLevel? { + var count = 0 + let totalSums: (averageSum: Float, peakSum: Float) = reduce((averageSum: 0.0, peakSum: 0.0)) { totals, audioLevel in + count += 1 + return ( + totals.averageSum + audioLevel.average, + totals.peakSum + audioLevel.peak + ) + } - public init(minFrequency: Float = 10, - maxFrequency: Float = 8000, - minDB: Float = -32.0, - maxDB: Float = 32.0, - bandsCount: Int = 100) - { - self.minFrequency = minFrequency - self.maxFrequency = maxFrequency - self.minDB = minDB - self.maxDB = maxDB - self.bandsCount = bandsCount + guard count > 0 else { return nil } - processor = FFTProcessor(bufferSize: Self.bufferSize) - bands = [Float](repeating: 0.0, count: bandsCount) - } + return AudioLevel( + average: totalSums.averageSum / Float(count), + peak: totalSums.peakSum / Float(count)) + } +} - public func process(pcmBuffer: AVAudioPCMBuffer) -> [Float]? { - guard let pcmBuffer = pcmBuffer.convert(toCommonFormat: .pcmFormatFloat32) else { return nil } - guard let floatChannelData = pcmBuffer.floatChannelData else { return nil } - - // Get the float array. - let floats = Array(UnsafeBufferPointer(start: floatChannelData[0], count: Int(pcmBuffer.frameLength))) - ringBuffer.write(floats) - - // Get full-size buffer if available, otherwise return - guard let buffer = ringBuffer.read() else { return nil } - - // Process FFT and compute frequency bands - let fftRes = processor.process(buffer: buffer) - let bands = fftRes.computeBands( - minFrequency: minFrequency, - maxFrequency: maxFrequency, - bandsCount: bandsCount, - sampleRate: Float(pcmBuffer.format.sampleRate) - ) - - let headroom = maxDB - minDB - - // Normalize magnitudes (already in decibels) - return bands.magnitudes.map { magnitude in - let adjustedMagnitude = max(0, magnitude + abs(minDB)) - return min(1.0, adjustedMagnitude / headroom) - } +public class AudioVisualizeProcessor { + static let bufferSize = 1024 + + // MARK: - Public + + public let minFrequency: Float + public let maxFrequency: Float + public let minDB: Float + public let maxDB: Float + public let bandsCount: Int + + private var bands: [Float]? + + // MARK: - Private + + private let ringBuffer = RingBuffer(size: AudioVisualizeProcessor.bufferSize) + private let processor: FFTProcessor + + public init( + minFrequency: Float = 10, + maxFrequency: Float = 8000, + minDB: Float = -32.0, + maxDB: Float = 32.0, + bandsCount: Int = 100 + ) { + self.minFrequency = minFrequency + self.maxFrequency = maxFrequency + self.minDB = minDB + self.maxDB = maxDB + self.bandsCount = bandsCount + + processor = FFTProcessor(bufferSize: Self.bufferSize) + bands = [Float](repeating: 0.0, count: bandsCount) + } + + public func process(pcmBuffer: AVAudioPCMBuffer) -> [Float]? { + guard let pcmBuffer = pcmBuffer.convert(toCommonFormat: .pcmFormatFloat32) else { return nil } + guard let floatChannelData = pcmBuffer.floatChannelData else { return nil } + + // Get the float array. + let floats = Array(UnsafeBufferPointer(start: floatChannelData[0], count: Int(pcmBuffer.frameLength))) + ringBuffer.write(floats) + + // Get full-size buffer if available, otherwise return + guard let buffer = ringBuffer.read() else { return nil } + + // Process FFT and compute frequency bands + let fftRes = processor.process(buffer: buffer) + let bands = fftRes.computeBands( + minFrequency: minFrequency, + maxFrequency: maxFrequency, + bandsCount: bandsCount, + sampleRate: Float(pcmBuffer.format.sampleRate) + ) + + let headroom = maxDB - minDB + + // Normalize magnitudes (already in decibels) + return bands.magnitudes.map { magnitude in + let adjustedMagnitude = max(0, magnitude + abs(minDB)) + return min(1.0, adjustedMagnitude / headroom) } + } } diff --git a/ios/audio/AudioRendererManager.swift b/ios/audio/AudioRendererManager.swift index fc7bab49..7a426c7c 100644 --- a/ios/audio/AudioRendererManager.swift +++ b/ios/audio/AudioRendererManager.swift @@ -2,70 +2,70 @@ import livekit_react_native_webrtc @objc public class AudioRendererManager: NSObject { - private let bridge: RCTBridge - public private(set) var renderers: [String: RTCAudioRenderer] = [:] - - init(bridge: RCTBridge) { - self.bridge = bridge - } - - @objc - public func registerRenderer(_ audioRenderer: RTCAudioRenderer) -> String { - let reactTag = NSUUID().uuidString - self.renderers[reactTag] = audioRenderer - return reactTag - } - - @objc - public func unregisterRenderer(forReactTag: String) { - self.renderers.removeValue(forKey: forReactTag) + private let bridge: RCTBridge + public private(set) var renderers: [String: RTCAudioRenderer] = [:] + + init(bridge: RCTBridge) { + self.bridge = bridge + } + + @objc + public func registerRenderer(_ audioRenderer: RTCAudioRenderer) -> String { + let reactTag = NSUUID().uuidString + self.renderers[reactTag] = audioRenderer + return reactTag + } + + @objc + public func unregisterRenderer(forReactTag: String) { + self.renderers.removeValue(forKey: forReactTag) + } + + @objc + public func unregisterRenderer(_ audioRenderer: RTCAudioRenderer) { + self.renderers = self.renderers.filter({ $0.value !== audioRenderer }) + } + + @objc + public func attach(renderer: RTCAudioRenderer, pcId: NSNumber, trackId: String) { + let webrtcModule = self.bridge.module(for: WebRTCModule.self) as! WebRTCModule + guard let track = webrtcModule.track(forId: trackId, pcId: pcId) as? RTCAudioTrack + else { + lklog("couldn't find audio track: pcId: \(pcId), trackId: \(trackId)") + return } - - @objc - public func unregisterRenderer(_ audioRenderer: RTCAudioRenderer) { - self.renderers = self.renderers.filter({ $0.value !== audioRenderer }) + + if pcId == -1 { + LKAudioProcessingManager.sharedInstance().addLocalAudioRenderer(renderer) + } else { + track.add(renderer) } - - @objc - public func attach(renderer: RTCAudioRenderer, pcId: NSNumber, trackId: String) { - let webrtcModule = self.bridge.module(for: WebRTCModule.self) as! WebRTCModule - guard let track = webrtcModule.track(forId: trackId, pcId: pcId) as? RTCAudioTrack - else { - lklog("couldn't find audio track: pcId: \(pcId), trackId: \(trackId)") - return - } - - if (pcId == -1) { - LKAudioProcessingManager.sharedInstance().addLocalAudioRenderer(renderer); - } else { - track.add(renderer) - } + } + + @objc + public func detach(rendererByTag reactTag: String, pcId: NSNumber, trackId: String) { + guard let renderer = self.renderers[reactTag] + else { + lklog("couldn't find renderer: tag: \(reactTag)") + return } - - @objc - public func detach(rendererByTag reactTag:String, pcId: NSNumber, trackId: String){ - guard let renderer = self.renderers[reactTag] - else { - lklog("couldn't find renderer: tag: \(reactTag)") - return - } - - detach(renderer: renderer, pcId: pcId, trackId: trackId) + + detach(renderer: renderer, pcId: pcId, trackId: trackId) + } + + @objc + public func detach(renderer: RTCAudioRenderer, pcId: NSNumber, trackId: String) { + let webrtcModule = self.bridge.module(for: WebRTCModule.self) as! WebRTCModule + guard let track = webrtcModule.track(forId: trackId, pcId: pcId) as? RTCAudioTrack + else { + lklog("couldn't find audio track: pcId: \(pcId), trackId: \(trackId)") + return } - - @objc - public func detach(renderer: RTCAudioRenderer, pcId: NSNumber, trackId: String) { - let webrtcModule = self.bridge.module(for: WebRTCModule.self) as! WebRTCModule - guard let track = webrtcModule.track(forId: trackId, pcId: pcId) as? RTCAudioTrack - else { - lklog("couldn't find audio track: pcId: \(pcId), trackId: \(trackId)") - return - } - - if (pcId == -1) { - LKAudioProcessingManager.sharedInstance().removeLocalAudioRenderer(renderer); - } else { - track.remove(renderer) - } + + if pcId == -1 { + LKAudioProcessingManager.sharedInstance().removeLocalAudioRenderer(renderer) + } else { + track.remove(renderer) } + } } diff --git a/ios/audio/AudioSinkRenderer.swift b/ios/audio/AudioSinkRenderer.swift index e3dbc65d..ebda478c 100644 --- a/ios/audio/AudioSinkRenderer.swift +++ b/ios/audio/AudioSinkRenderer.swift @@ -1,51 +1,54 @@ -import livekit_react_native_webrtc import React +import livekit_react_native_webrtc @objc public class AudioSinkRenderer: BaseAudioSinkRenderer { - private let eventEmitter: RCTEventEmitter - - @objc - public var reactTag: String? = nil - - @objc - public init(eventEmitter: RCTEventEmitter) { - self.eventEmitter = eventEmitter - super.init() - } - - override public func onData(_ pcmBuffer: AVAudioPCMBuffer) { - guard pcmBuffer.format.commonFormat == .pcmFormatInt16, - let channelData = pcmBuffer.int16ChannelData else { - return - } - let channelCount = Int(pcmBuffer.format.channelCount) - let channels = UnsafeBufferPointer(start: channelData, count: channelCount) - let length = Int(pcmBuffer.frameCapacity * pcmBuffer.format.streamDescription.pointee.mBytesPerFrame) - let data = NSData(bytes: channels[0], length: length) - let base64 = data.base64EncodedString() - NSLog("AUDIO DATA!!!!") - NSLog("\(data.length)") - NSLog(base64) - NSLog("\(base64.count)") - NSLog("\(length)") - eventEmitter.sendEvent(withName: LKEvents.kEventAudioData, body: [ - "data": base64, - "id": reactTag - ]) + private let eventEmitter: RCTEventEmitter + + @objc + public var reactTag: String? = nil + + @objc + public init(eventEmitter: RCTEventEmitter) { + self.eventEmitter = eventEmitter + super.init() + } + + override public func onData(_ pcmBuffer: AVAudioPCMBuffer) { + guard pcmBuffer.format.commonFormat == .pcmFormatInt16, + let channelData = pcmBuffer.int16ChannelData + else { + return } + let channelCount = Int(pcmBuffer.format.channelCount) + let channels = UnsafeBufferPointer(start: channelData, count: channelCount) + let length = Int(pcmBuffer.frameCapacity * pcmBuffer.format.streamDescription.pointee.mBytesPerFrame) + let data = NSData(bytes: channels[0], length: length) + let base64 = data.base64EncodedString() + NSLog("AUDIO DATA!!!!") + NSLog("\(data.length)") + NSLog(base64) + NSLog("\(base64.count)") + NSLog("\(length)") + eventEmitter.sendEvent( + withName: LKEvents.kEventAudioData, + body: [ + "data": base64, + "id": reactTag, + ]) + } } public class BaseAudioSinkRenderer: NSObject, RTCAudioRenderer { - - public override init() { - super.init() - } - - public func render(pcmBuffer: AVAudioPCMBuffer) { - onData(pcmBuffer) - } - - public func onData(_ pcmBuffer: AVAudioPCMBuffer) { - } + + public override init() { + super.init() + } + + public func render(pcmBuffer: AVAudioPCMBuffer) { + onData(pcmBuffer) + } + + public func onData(_ pcmBuffer: AVAudioPCMBuffer) { + } } diff --git a/ios/audio/FFTProcessor.swift b/ios/audio/FFTProcessor.swift index 2ee10179..a9906097 100755 --- a/ios/audio/FFTProcessor.swift +++ b/ios/audio/FFTProcessor.swift @@ -14,134 +14,137 @@ * limitations under the License. */ -import Accelerate import AVFoundation +import Accelerate extension Float { - var nyquistFrequency: Float { self / 2.0 } + var nyquistFrequency: Float { self / 2.0 } } public struct FFTComputeBandsResult { - let count: Int - let magnitudes: [Float] - let frequencies: [Float] + let count: Int + let magnitudes: [Float] + let frequencies: [Float] } public class FFTResult { - public let magnitudes: [Float] + public let magnitudes: [Float] + + init(magnitudes: [Float]) { + self.magnitudes = magnitudes + } + + func computeBands(minFrequency: Float, maxFrequency: Float, bandsCount: Int, sampleRate: Float) + -> FFTComputeBandsResult + { + let actualMaxFrequency = min(sampleRate.nyquistFrequency, maxFrequency) + var bandMagnitudes = [Float](repeating: 0.0, count: bandsCount) + var bandFrequencies = [Float](repeating: 0.0, count: bandsCount) + + let magLowerRange = _magnitudeIndex(for: minFrequency, sampleRate: sampleRate) + let magUpperRange = _magnitudeIndex(for: actualMaxFrequency, sampleRate: sampleRate) + let ratio = Float(magUpperRange - magLowerRange) / Float(bandsCount) + + return magnitudes.withUnsafeBufferPointer { magnitudesPtr in + for i in 0.. 0 { + var sum: Float = 0 + vDSP_sve(magnitudesPtr.baseAddress! + Int(magsStartIdx), 1, &sum, count) + bandMagnitudes[i] = sum / Float(count) + } else { + bandMagnitudes[i] = magnitudes[Int(magsStartIdx)] + } - init(magnitudes: [Float]) { - self.magnitudes = magnitudes - } + // Compute average frequency + let bandwidth = sampleRate.nyquistFrequency / Float(magnitudes.count) + bandFrequencies[i] = (bandwidth * Float(magsStartIdx) + bandwidth * Float(magsEndIdx)) / 2 + } - func computeBands(minFrequency: Float, maxFrequency: Float, bandsCount: Int, sampleRate: Float) -> FFTComputeBandsResult { - let actualMaxFrequency = min(sampleRate.nyquistFrequency, maxFrequency) - var bandMagnitudes = [Float](repeating: 0.0, count: bandsCount) - var bandFrequencies = [Float](repeating: 0.0, count: bandsCount) - - let magLowerRange = _magnitudeIndex(for: minFrequency, sampleRate: sampleRate) - let magUpperRange = _magnitudeIndex(for: actualMaxFrequency, sampleRate: sampleRate) - let ratio = Float(magUpperRange - magLowerRange) / Float(bandsCount) - - return magnitudes.withUnsafeBufferPointer { magnitudesPtr in - for i in 0 ..< bandsCount { - let magsStartIdx = vDSP_Length(floorf(Float(i) * ratio)) + magLowerRange - let magsEndIdx = vDSP_Length(floorf(Float(i + 1) * ratio)) + magLowerRange - - let count = magsEndIdx - magsStartIdx - if count > 0 { - var sum: Float = 0 - vDSP_sve(magnitudesPtr.baseAddress! + Int(magsStartIdx), 1, &sum, count) - bandMagnitudes[i] = sum / Float(count) - } else { - bandMagnitudes[i] = magnitudes[Int(magsStartIdx)] - } - - // Compute average frequency - let bandwidth = sampleRate.nyquistFrequency / Float(magnitudes.count) - bandFrequencies[i] = (bandwidth * Float(magsStartIdx) + bandwidth * Float(magsEndIdx)) / 2 - } - - return FFTComputeBandsResult(count: bandsCount, magnitudes: bandMagnitudes, frequencies: bandFrequencies) - } + return FFTComputeBandsResult(count: bandsCount, magnitudes: bandMagnitudes, frequencies: bandFrequencies) } + } - @inline(__always) private func _magnitudeIndex(for frequency: Float, sampleRate: Float) -> vDSP_Length { - vDSP_Length(Float(magnitudes.count) * frequency / sampleRate.nyquistFrequency) - } + @inline(__always) private func _magnitudeIndex(for frequency: Float, sampleRate: Float) -> vDSP_Length { + vDSP_Length(Float(magnitudes.count) * frequency / sampleRate.nyquistFrequency) + } } class FFTProcessor { - public enum WindowType { - case none - case hanning - case hamming + public enum WindowType { + case none + case hanning + case hamming + } + + public let bufferSize: vDSP_Length + public let windowType: WindowType + + private let bufferHalfSize: vDSP_Length + private let bufferLog2Size: vDSP_Length + private var window: [Float] = [] + private var fftSetup: FFTSetup + private var realBuffer: [Float] + private var imaginaryBuffer: [Float] + private var zeroDBReference: Float = 1.0 + + init(bufferSize: Int, windowType: WindowType = .hanning) { + self.bufferSize = vDSP_Length(bufferSize) + self.windowType = windowType + + bufferHalfSize = vDSP_Length(bufferSize / 2) + bufferLog2Size = vDSP_Length(log2f(Float(bufferSize))) + + realBuffer = [Float](repeating: 0.0, count: Int(bufferHalfSize)) + imaginaryBuffer = [Float](repeating: 0.0, count: Int(bufferHalfSize)) + window = [Float](repeating: 1.0, count: Int(bufferSize)) + + fftSetup = vDSP_create_fftsetup(UInt(bufferLog2Size), FFTRadix(FFT_RADIX2))! + + switch windowType { + case .none: + break + case .hanning: + vDSP_hann_window(&window, vDSP_Length(bufferSize), Int32(vDSP_HANN_NORM)) + case .hamming: + vDSP_hamm_window(&window, vDSP_Length(bufferSize), 0) } + } - public let bufferSize: vDSP_Length - public let windowType: WindowType - - private let bufferHalfSize: vDSP_Length - private let bufferLog2Size: vDSP_Length - private var window: [Float] = [] - private var fftSetup: FFTSetup - private var realBuffer: [Float] - private var imaginaryBuffer: [Float] - private var zeroDBReference: Float = 1.0 - - init(bufferSize: Int, windowType: WindowType = .hanning) { - self.bufferSize = vDSP_Length(bufferSize) - self.windowType = windowType - - bufferHalfSize = vDSP_Length(bufferSize / 2) - bufferLog2Size = vDSP_Length(log2f(Float(bufferSize))) - - realBuffer = [Float](repeating: 0.0, count: Int(bufferHalfSize)) - imaginaryBuffer = [Float](repeating: 0.0, count: Int(bufferHalfSize)) - window = [Float](repeating: 1.0, count: Int(bufferSize)) - - fftSetup = vDSP_create_fftsetup(UInt(bufferLog2Size), FFTRadix(FFT_RADIX2))! - - switch windowType { - case .none: - break - case .hanning: - vDSP_hann_window(&window, vDSP_Length(bufferSize), Int32(vDSP_HANN_NORM)) - case .hamming: - vDSP_hamm_window(&window, vDSP_Length(bufferSize), 0) - } - } - - deinit { - vDSP_destroy_fftsetup(fftSetup) - } + deinit { + vDSP_destroy_fftsetup(fftSetup) + } - func process(buffer: [Float]) -> FFTResult { - precondition(buffer.count == Int(bufferSize), "Input buffer size mismatch.") + func process(buffer: [Float]) -> FFTResult { + precondition(buffer.count == Int(bufferSize), "Input buffer size mismatch.") - var windowedBuffer = [Float](repeating: 0.0, count: Int(bufferSize)) + var windowedBuffer = [Float](repeating: 0.0, count: Int(bufferSize)) - vDSP_vmul(buffer, 1, window, 1, &windowedBuffer, 1, bufferSize) + vDSP_vmul(buffer, 1, window, 1, &windowedBuffer, 1, bufferSize) - return realBuffer.withUnsafeMutableBufferPointer { realPtr in - imaginaryBuffer.withUnsafeMutableBufferPointer { imagPtr in - var complexBuffer = DSPSplitComplex(realp: realPtr.baseAddress!, imagp: imagPtr.baseAddress!) + return realBuffer.withUnsafeMutableBufferPointer { realPtr in + imaginaryBuffer.withUnsafeMutableBufferPointer { imagPtr in + var complexBuffer = DSPSplitComplex(realp: realPtr.baseAddress!, imagp: imagPtr.baseAddress!) - windowedBuffer.withUnsafeBufferPointer { bufferPtr in - let complexPtr = UnsafeRawPointer(bufferPtr.baseAddress!).bindMemory(to: DSPComplex.self, capacity: Int(bufferHalfSize)) - vDSP_ctoz(complexPtr, 2, &complexBuffer, 1, bufferHalfSize) - } + windowedBuffer.withUnsafeBufferPointer { bufferPtr in + let complexPtr = UnsafeRawPointer(bufferPtr.baseAddress!).bindMemory( + to: DSPComplex.self, capacity: Int(bufferHalfSize)) + vDSP_ctoz(complexPtr, 2, &complexBuffer, 1, bufferHalfSize) + } - vDSP_fft_zrip(fftSetup, &complexBuffer, 1, bufferLog2Size, FFTDirection(FFT_FORWARD)) + vDSP_fft_zrip(fftSetup, &complexBuffer, 1, bufferLog2Size, FFTDirection(FFT_FORWARD)) - var magnitudes = [Float](repeating: 0.0, count: Int(bufferHalfSize)) - vDSP_zvabs(&complexBuffer, 1, &magnitudes, 1, bufferHalfSize) + var magnitudes = [Float](repeating: 0.0, count: Int(bufferHalfSize)) + vDSP_zvabs(&complexBuffer, 1, &magnitudes, 1, bufferHalfSize) - // Convert magnitudes to decibels - vDSP_vdbcon(magnitudes, 1, &zeroDBReference, &magnitudes, 1, vDSP_Length(magnitudes.count), 1) + // Convert magnitudes to decibels + vDSP_vdbcon(magnitudes, 1, &zeroDBReference, &magnitudes, 1, vDSP_Length(magnitudes.count), 1) - return FFTResult(magnitudes: magnitudes) - } - } + return FFTResult(magnitudes: magnitudes) + } } + } } diff --git a/ios/audio/MultibandVolumeAudioRenderer.swift b/ios/audio/MultibandVolumeAudioRenderer.swift index 64f9ccf6..54a76a03 100644 --- a/ios/audio/MultibandVolumeAudioRenderer.swift +++ b/ios/audio/MultibandVolumeAudioRenderer.swift @@ -1,67 +1,71 @@ -import livekit_react_native_webrtc import React +import livekit_react_native_webrtc @objc public class MultibandVolumeAudioRenderer: BaseMultibandVolumeAudioRenderer { - private let eventEmitter: RCTEventEmitter - - @objc - public var reactTag: String? = nil + private let eventEmitter: RCTEventEmitter + + @objc + public var reactTag: String? = nil + + @objc + public init( + bands: Int, + minFrequency: Float, + maxFrequency: Float, + intervalMs: Float, + eventEmitter: RCTEventEmitter + ) { + self.eventEmitter = eventEmitter + super.init( + bands: bands, + minFrequency: minFrequency, + maxFrequency: maxFrequency, + intervalMs: intervalMs) + } + + override func onMagnitudesCalculated(_ magnitudes: [Float]) { + guard !magnitudes.isEmpty, let reactTag = self.reactTag + else { return } + eventEmitter.sendEvent( + withName: LKEvents.kEventMultibandProcessed, + body: [ + "magnitudes": magnitudes, + "id": reactTag, + ]) + } - @objc - public init( - bands: Int, - minFrequency: Float, - maxFrequency: Float, - intervalMs: Float, - eventEmitter: RCTEventEmitter - ) { - self.eventEmitter = eventEmitter - super.init(bands: bands, - minFrequency: minFrequency, - maxFrequency: maxFrequency, - intervalMs: intervalMs) - } - - override func onMagnitudesCalculated(_ magnitudes: [Float]) { - guard !magnitudes.isEmpty, let reactTag = self.reactTag - else { return } - eventEmitter.sendEvent(withName: LKEvents.kEventMultibandProcessed, body: [ - "magnitudes": magnitudes, - "id": reactTag - ]) - } - } public class BaseMultibandVolumeAudioRenderer: NSObject, RTCAudioRenderer { - private let frameInterval: Int - private var skippedFrames = 0 - private let audioProcessor: AudioVisualizeProcessor - - init( - bands: Int, - minFrequency: Float, - maxFrequency: Float, - intervalMs: Float - ) { - self.frameInterval = Int((intervalMs / 10.0).rounded()) - self.audioProcessor = AudioVisualizeProcessor(minFrequency: minFrequency, maxFrequency: maxFrequency, bandsCount: bands) + private let frameInterval: Int + private var skippedFrames = 0 + private let audioProcessor: AudioVisualizeProcessor + + init( + bands: Int, + minFrequency: Float, + maxFrequency: Float, + intervalMs: Float + ) { + self.frameInterval = Int((intervalMs / 10.0).rounded()) + self.audioProcessor = AudioVisualizeProcessor( + minFrequency: minFrequency, maxFrequency: maxFrequency, bandsCount: bands) + } + + public func render(pcmBuffer: AVAudioPCMBuffer) { + if skippedFrames < frameInterval - 1 { + skippedFrames += 1 + return } - - public func render(pcmBuffer: AVAudioPCMBuffer) { - if(skippedFrames < frameInterval - 1) { - skippedFrames += 1 - return - } - - skippedFrames = 0 - guard let magnitudes = audioProcessor.process(pcmBuffer: pcmBuffer) - else { - return - } - onMagnitudesCalculated(magnitudes) + + skippedFrames = 0 + guard let magnitudes = audioProcessor.process(pcmBuffer: pcmBuffer) + else { + return } - - func onMagnitudesCalculated(_ magnitudes: [Float]) { } + onMagnitudesCalculated(magnitudes) + } + + func onMagnitudesCalculated(_ magnitudes: [Float]) {} } diff --git a/ios/audio/RingBuffer.swift b/ios/audio/RingBuffer.swift index 7f9847a5..b1136523 100644 --- a/ios/audio/RingBuffer.swift +++ b/ios/audio/RingBuffer.swift @@ -18,34 +18,34 @@ import Foundation // Simple ring-buffer used for internal audio processing. Not thread-safe. class RingBuffer { - private var _isFull = false - private var _buffer: [T] - private var _head: Int = 0 + private var _isFull = false + private var _buffer: [T] + private var _head: Int = 0 - init(size: Int) { - _buffer = [T](repeating: 0, count: size) - } + init(size: Int) { + _buffer = [T](repeating: 0, count: size) + } - func write(_ value: T) { - _buffer[_head] = value - _head = (_head + 1) % _buffer.count - if _head == 0 { _isFull = true } - } + func write(_ value: T) { + _buffer[_head] = value + _head = (_head + 1) % _buffer.count + if _head == 0 { _isFull = true } + } - func write(_ sequence: [T]) { - for value in sequence { - write(value) - } + func write(_ sequence: [T]) { + for value in sequence { + write(value) } + } - func read() -> [T]? { - guard _isFull else { return nil } + func read() -> [T]? { + guard _isFull else { return nil } - if _head == 0 { - return _buffer // Return the entire buffer if _head is at the start - } else { - // Return the buffer in the correct order - return Array(_buffer[_head ..< _buffer.count] + _buffer[0 ..< _head]) - } + if _head == 0 { + return _buffer // Return the entire buffer if _head is at the start + } else { + // Return the buffer in the correct order + return Array(_buffer[_head..<_buffer.count] + _buffer[0..<_head]) } + } } diff --git a/ios/audio/VolumeAudioRenderer.swift b/ios/audio/VolumeAudioRenderer.swift index f2506f28..648562ec 100644 --- a/ios/audio/VolumeAudioRenderer.swift +++ b/ios/audio/VolumeAudioRenderer.swift @@ -1,50 +1,52 @@ -import livekit_react_native_webrtc import React +import livekit_react_native_webrtc @objc public class VolumeAudioRenderer: BaseVolumeAudioRenderer { - private let eventEmitter: RCTEventEmitter - - @objc - public var reactTag: String? = nil - - @objc - public init(intervalMs: Double, eventEmitter: RCTEventEmitter) { - self.eventEmitter = eventEmitter - super.init(intervalMs: intervalMs) - } - - override public func onVolumeCalculated(_ audioLevels: [AudioLevel]) { - guard let rmsAvg = audioLevels.combine()?.average, - let reactTag = self.reactTag - else { return } - eventEmitter.sendEvent(withName: LKEvents.kEventVolumeProcessed, body: [ - "volume": rmsAvg, - "id": reactTag - ]) - } + private let eventEmitter: RCTEventEmitter + + @objc + public var reactTag: String? = nil + + @objc + public init(intervalMs: Double, eventEmitter: RCTEventEmitter) { + self.eventEmitter = eventEmitter + super.init(intervalMs: intervalMs) + } + + override public func onVolumeCalculated(_ audioLevels: [AudioLevel]) { + guard let rmsAvg = audioLevels.combine()?.average, + let reactTag = self.reactTag + else { return } + eventEmitter.sendEvent( + withName: LKEvents.kEventVolumeProcessed, + body: [ + "volume": rmsAvg, + "id": reactTag, + ]) + } } public class BaseVolumeAudioRenderer: NSObject, RTCAudioRenderer { - private let frameInterval: Int - private var skippedFrames = 0 - public init(intervalMs: Double = 30) { - self.frameInterval = Int((intervalMs / 10.0).rounded()) - } - - public func render(pcmBuffer: AVAudioPCMBuffer) { - if(skippedFrames < frameInterval - 1) { - skippedFrames += 1 - return - } - - skippedFrames = 0 - guard let pcmBuffer = pcmBuffer.convert(toCommonFormat: .pcmFormatFloat32) else { return } - let audioLevels = pcmBuffer.audioLevels() - onVolumeCalculated(audioLevels) - } - - public func onVolumeCalculated(_ audioLevels: [AudioLevel]) { - + private let frameInterval: Int + private var skippedFrames = 0 + public init(intervalMs: Double = 30) { + self.frameInterval = Int((intervalMs / 10.0).rounded()) + } + + public func render(pcmBuffer: AVAudioPCMBuffer) { + if skippedFrames < frameInterval - 1 { + skippedFrames += 1 + return } + + skippedFrames = 0 + guard let pcmBuffer = pcmBuffer.convert(toCommonFormat: .pcmFormatFloat32) else { return } + let audioLevels = pcmBuffer.audioLevels() + onVolumeCalculated(audioLevels) + } + + public func onVolumeCalculated(_ audioLevels: [AudioLevel]) { + + } }