Pull audio track from avatar worker in useVoiceAssistant (#1130)

Author: bcherry

.changeset/tender-eyes-run.md

@@ -0,0 +1,5 @@
+---
+"@livekit/components-react": patch
+---
+
+Pull audio track from avatar worker in useVoiceAssistant

packages/core/etc/components-core.api.md

@@ -86,10 +86,7 @@ export type ChatOptions = {
 };
 
 // @public (undocumented)
-export function computeMenuPosition(button: HTMLElement, menu: HTMLElement): Promise<{
-    x: number;
-    y: number;
-}>;
+export function computeMenuPosition(button: HTMLElement, menu: HTMLElement, onUpdate?: (x: number, y: number) => void): () => void;
 
 // @public (undocumented)
 export function connectedParticipantObserver(room: Room, identity: string, options?: ConnectedParticipantObserverOptions): Observable<RemoteParticipant | undefined>;

packages/react/etc/components-react.api.md

@@ -1256,16 +1256,12 @@ export interface VideoTrackProps extends React_2.VideoHTMLAttributes<HTMLVideoEl
 
 // @beta (undocumented)
 export interface VoiceAssistant {
-    // (undocumented)
     agent: RemoteParticipant | undefined;
-    // (undocumented)
     agentAttributes: RemoteParticipant['attributes'] | undefined;
-    // (undocumented)
     agentTranscriptions: ReceivedTranscriptionSegment[];
-    // (undocumented)
     audioTrack: TrackReference_3 | undefined;
-    // (undocumented)
     state: AgentState;
+    videoTrack: TrackReference_3 | undefined;
 }
 
 // @beta (undocumented)

packages/react/src/hooks/useVoiceAssistant.ts

@@ -23,10 +23,29 @@ export type AgentState =
  * @beta
  */
 export interface VoiceAssistant {
+  /**
+   * The agent participant.
+   */
   agent: RemoteParticipant | undefined;
+  /**
+   * The current state of the agent.
+   */
   state: AgentState;
+  /**
+   * The microphone track published by the agent or associated avatar worker (if any).
+   */
   audioTrack: TrackReference | undefined;
+  /**
+   * The camera track published by the agent or associated avatar worker (if any).
+   */
+  videoTrack: TrackReference | undefined;
+  /**
+   * The transcriptions of the agent's microphone track (if any).
+   */
   agentTranscriptions: ReceivedTranscriptionSegment[];
+  /**
+   * The agent's participant attributes.
+   */
   agentAttributes: RemoteParticipant['attributes'] | undefined;
 }
 
@@ -42,8 +61,28 @@ const state_attribute = 'lk.agent.state';
  * @beta
  */
 export function useVoiceAssistant(): VoiceAssistant {
-  const agent = useRemoteParticipants().find((p) => p.kind === ParticipantKind.AGENT);
-  const audioTrack = useParticipantTracks([Track.Source.Microphone], agent?.identity)[0];
+  const remoteParticipants = useRemoteParticipants();
+  const agent = remoteParticipants.find(
+    (p) => p.kind === ParticipantKind.AGENT && !('lk.publish_on_behalf' in p.attributes),
+  );
+  const worker = remoteParticipants.find(
+    (p) =>
+      p.kind === ParticipantKind.AGENT && p.attributes['lk.publish_on_behalf'] === agent?.identity,
+  );
+  const agentTracks = useParticipantTracks(
+    [Track.Source.Microphone, Track.Source.Camera],
+    agent?.identity,
+  );
+  const workerTracks = useParticipantTracks(
+    [Track.Source.Microphone, Track.Source.Camera],
+    worker?.identity,
+  );
+  const audioTrack =
+    agentTracks.find((t) => t.source === Track.Source.Microphone) ??
+    workerTracks.find((t) => t.source === Track.Source.Microphone);
+  const videoTrack =
+    agentTracks.find((t) => t.source === Track.Source.Camera) ??
+    workerTracks.find((t) => t.source === Track.Source.Camera);
   const { segments: agentTranscriptions } = useTrackTranscription(audioTrack);
   const connectionState = useConnectionState();
   const { attributes } = useParticipantAttributes({ participant: agent });
@@ -66,6 +105,7 @@ export function useVoiceAssistant(): VoiceAssistant {
     agent,
     state,
     audioTrack,
+    videoTrack,
     agentTranscriptions,
     agentAttributes: attributes,
   };