[FirebaseAI] Add Live Audio screen

firebaseai/FirebaseAIExample/ContentView.swift

@@ -50,6 +50,11 @@ struct ContentView: View {
         }
 
         Section("Examples") {
+          NavigationLink {
+            LiveAudioScreen(firebaseService: firebaseService, backend: selectedBackend)
+          } label: {
+            Label("Live Audio", systemImage: "microphone")
+          }
           NavigationLink {
             GenerateContentScreen(firebaseService: firebaseService)
           } label: {

firebaseai/FirebaseAIExample/FirebaseAIExampleApp.swift

@@ -22,6 +22,9 @@ class AppDelegate: NSObject, UIApplicationDelegate {
     // Recommendation: Protect your Vertex AI API resources from abuse by preventing unauthorized
     // clients using App Check; see https://firebase.google.com/docs/app-check#get_started.
 
+    // let providerFactor = AppCheckDebugProviderFactory()
+    // AppCheck.setAppCheckProviderFactory(providerFactor)
+
     FirebaseApp.configure()
 
     if let firebaseApp = FirebaseApp.app(), firebaseApp.options.projectID == "mockproject-1234" {

firebaseai/LiveAudioExample/ApplicationError.swift

@@ -0,0 +1,30 @@
+// Copyright 2025 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import Foundation
+
+/// Generic error for issues that occur within the application.
+public struct ApplicationError: Error, Sendable, CustomNSError {
+  let localizedDescription: String
+
+  init(_ localizedDescription: String) {
+    self.localizedDescription = localizedDescription
+  }
+
+  public var errorUserInfo: [String: Any] {
+    [
+      NSLocalizedDescriptionKey: localizedDescription,
+    ]
+  }
+}

firebaseai/LiveAudioExample/Assets.xcassets/Contents.json

@@ -0,0 +1,6 @@
+{
+  "info" : {
+    "author" : "xcode",
+    "version" : 1
+  }
+}

firebaseai/LiveAudioExample/Assets.xcassets/gemini-logo.imageset/Contents.json

@@ -0,0 +1,35 @@
+{
+  "images" : [
+    {
+      "filename" : "gemini-logo.png",
+      "idiom" : "universal",
+      "resizing" : {
+        "cap-insets" : {
+          "bottom" : 512,
+          "left" : 511,
+          "right" : 512,
+          "top" : 511
+        },
+        "center" : {
+          "height" : 1,
+          "mode" : "tile",
+          "width" : 1
+        },
+        "mode" : "9-part"
+      },
+      "scale" : "1x"
+    },
+    {
+      "idiom" : "universal",
+      "scale" : "2x"
+    },
+    {
+      "idiom" : "universal",
+      "scale" : "3x"
+    }
+  ],
+  "info" : {
+    "author" : "xcode",
+    "version" : 1
+  }
+}

firebaseai/LiveAudioExample/Audio/AudioBufferHelpers.swift

@@ -0,0 +1,90 @@
+// Copyright 2025 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import AVFoundation
+
+extension AVAudioPCMBuffer {
+  /// Creates a new `AVAudioPCMBuffer` from a `Data` struct.
+  ///
+  /// Only works with interleaved data.
+  static func fromInterleavedData(data: Data, format: AVAudioFormat) throws -> AVAudioPCMBuffer? {
+    guard format.isInterleaved else {
+      throw ApplicationError("Only interleaved data is supported")
+    }
+
+    let frameCapacity = AVAudioFrameCount(data
+      .count / Int(format.streamDescription.pointee.mBytesPerFrame))
+    guard let buffer = AVAudioPCMBuffer(pcmFormat: format, frameCapacity: frameCapacity) else {
+      return nil
+    }
+
+    buffer.frameLength = frameCapacity
+    data.withUnsafeBytes { bytes in
+      guard let baseAddress = bytes.baseAddress else { return }
+      let dst = buffer.mutableAudioBufferList.pointee.mBuffers
+      dst.mData?.copyMemory(from: baseAddress, byteCount: Int(dst.mDataByteSize))
+    }
+
+    return buffer
+  }
+
+  /// Gets the underlying `Data` in this buffer.
+  ///
+  /// Will throw an error if this buffer doesn't hold int16 data.
+  func int16Data() throws -> Data {
+    guard let bufferPtr = audioBufferList.pointee.mBuffers.mData else {
+      throw ApplicationError("Missing audio buffer list")
+    }
+
+    let audioBufferLenth = Int(audioBufferList.pointee.mBuffers.mDataByteSize)
+    return Data(bytes: bufferPtr, count: audioBufferLenth)
+  }
+}
+
+extension AVAudioConverter {
+  /// Uses the converter to convert the provided `buffer`.
+  ///
+  /// Will handle determining the proper frame capacity, ensuring formats align, and propogating any errors that occur.
+  ///
+  ///   - Returns: A new buffer, with the converted data.
+  func convertBuffer(_ buffer: AVAudioPCMBuffer) throws -> AVAudioPCMBuffer {
+    if buffer.format == outputFormat { return buffer }
+    guard buffer.format == inputFormat else {
+      throw ApplicationError("The buffer's format was different than the converter's input format")
+    }
+
+    let frameCapacity = AVAudioFrameCount(
+      ceil(Double(buffer.frameLength) * outputFormat.sampleRate / inputFormat.sampleRate)
+    )
+
+    guard let output = AVAudioPCMBuffer(
+      pcmFormat: outputFormat,
+      frameCapacity: frameCapacity
+    ) else {
+      throw ApplicationError("Failed to create output buffer")
+    }
+
+    var error: NSError?
+    convert(to: output, error: &error) { _, status in
+      status.pointee = .haveData
+      return buffer
+    }
+
+    if let error {
+      throw ApplicationError("Failed to convert buffer: \(error.localizedDescription)")
+    }
+
+    return output
+  }
+}