Skip to content

Commit a6b11f0

Browse files
Oceania2018Oceania2018
authored
Merge pull request #1057 from iceljc/test/google-realtime
Test/google realtime
2 parents 968bac6 + d91a552 commit a6b11f0

File tree

52 files changed

+1089
-671
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

52 files changed

+1089
-671
lines changed

‎src/Infrastructure/BotSharp.Abstraction/Conversations/Models/Conversation.cs

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,15 +47,19 @@ public class DialogElement
4747
public string Content { get; set; } = default!;
4848

4949
[JsonPropertyName("secondary_content")]
50+
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
5051
public string? SecondaryContent { get; set; }
5152

5253
[JsonPropertyName("rich_content")]
54+
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
5355
public string? RichContent { get; set; }
5456

5557
[JsonPropertyName("secondary_rich_content")]
58+
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
5659
public string? SecondaryRichContent { get; set; }
5760

5861
[JsonPropertyName("payload")]
62+
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
5963
public string? Payload { get; set; }
6064

6165
public DialogElement()
@@ -95,8 +99,17 @@ public class DialogMetaData
9599
public string MessageType { get; set; } = default!;
96100

97101
[JsonPropertyName("function_name")]
102+
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
98103
public string? FunctionName { get; set; }
99104

105+
[JsonPropertyName("function_args")]
106+
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
107+
public string? FunctionArgs { get; set; }
108+
109+
[JsonPropertyName("tool_call_id")]
110+
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
111+
public string? ToolCallId { get; set; }
112+
100113
[JsonPropertyName("sender_id")]
101114
public string? SenderId { get; set; }
102115

‎src/Infrastructure/BotSharp.Abstraction/Hooks/HookProvider.cs

Lines changed: 11 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -1,25 +1,19 @@
11
using BotSharp.Abstraction.Conversations;
22
using Microsoft.Extensions.DependencyInjection;
3-
using System;
4-
using System.Collections.Generic;
5-
using System.Linq;
6-
using System.Text;
7-
using System.Threading.Tasks;
83

9-
namespace BotSharp.Abstraction.Hooks
4+
namespace BotSharp.Abstraction.Hooks;
5+
6+
public static class HookProvider
107
{
11-
public static class HookProvider
8+
public static List<T> GetHooks<T>(this IServiceProvider services, string agentId) where T : IHookBase
129
{
13-
public static List<T> GetHooks<T>(this IServiceProvider services, string agentId) where T : IHookBase
14-
{
15-
var hooks = services.GetServices<T>().Where(p => p.IsMatch(agentId));
16-
return hooks.ToList();
17-
}
10+
var hooks = services.GetServices<T>().Where(p => p.IsMatch(agentId));
11+
return hooks.ToList();
12+
}
1813

19-
public static List<T> GetHooksOrderByPriority<T>(this IServiceProvider services, string agentId) where T: IConversationHook
20-
{
21-
var hooks = services.GetServices<T>().Where(p => p.IsMatch(agentId));
22-
return hooks.OrderBy(p => p.Priority).ToList();
23-
}
14+
public static List<T> GetHooksOrderByPriority<T>(this IServiceProvider services, string agentId) where T: IConversationHook
15+
{
16+
var hooks = services.GetServices<T>().Where(p => p.IsMatch(agentId));
17+
return hooks.OrderBy(p => p.Priority).ToList();
2418
}
2519
}

‎src/Infrastructure/BotSharp.Abstraction/Hooks/IHookBase.cs

Lines changed: 7 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,10 @@
1-
using System;
2-
using System.Collections.Generic;
3-
using System.Linq;
4-
using System.Text;
5-
using System.Threading.Tasks;
1+
namespace BotSharp.Abstraction.Hooks;
62

7-
namespace BotSharp.Abstraction.Hooks
3+
public interface IHookBase
84
{
9-
public interface IHookBase
10-
{
11-
/// <summary>
12-
/// Agent Id
13-
/// </summary>
14-
string SelfId => string.Empty;
15-
bool IsMatch(string agentId) => string.IsNullOrEmpty(SelfId) || SelfId == agentId;
16-
}
5+
/// <summary>
6+
/// Agent Id
7+
/// </summary>
8+
string SelfId => string.Empty;
9+
bool IsMatch(string agentId) => string.IsNullOrEmpty(SelfId) || SelfId == agentId;
1710
}

‎src/Infrastructure/BotSharp.Abstraction/MCP/Services/IMcpService.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,5 +2,5 @@ namespace BotSharp.Abstraction.MCP.Services;
22

33
public interface IMcpService
44
{
5-
IEnumerable<McpServerOptionModel> GetServerConfigs() => [];
5+
Task<IEnumerable<McpServerOptionModel>> GetServerConfigsAsync() => Task.FromResult<IEnumerable<McpServerOptionModel>>([]);
66
}

‎src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs

Lines changed: 9 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -10,14 +10,15 @@ public interface IRealTimeCompletion
1010

1111
Task Connect(
1212
RealtimeHubConnection conn,
13-
Action onModelReady,
14-
Action<string, string> onModelAudioDeltaReceived,
15-
Action onModelAudioResponseDone,
16-
Action<string> onAudioTranscriptDone,
17-
Action<List<RoleDialogModel>> onModelResponseDone,
18-
Action<string> onConversationItemCreated,
19-
Action<RoleDialogModel> onInputAudioTranscriptionCompleted,
20-
Action onInterruptionDetected);
13+
Func<Task> onModelReady,
14+
Func<string, string, Task> onModelAudioDeltaReceived,
15+
Func<Task> onModelAudioResponseDone,
16+
Func<string, Task> onModelAudioTranscriptDone,
17+
Func<List<RoleDialogModel>, Task> onModelResponseDone,
18+
Func<string, Task> onConversationItemCreated,
19+
Func<RoleDialogModel, Task> onInputAudioTranscriptionDone,
20+
Func<Task> onInterruptionDetected);
21+
2122
Task AppenAudioBuffer(string message);
2223
Task AppenAudioBuffer(ArraySegment<byte> data, int length);
2324

@@ -29,6 +30,4 @@ Task Connect(
2930
Task RemoveConversationItem(string itemId);
3031
Task TriggerModelInference(string? instructions = null);
3132
Task CancelModelResponse();
32-
Task<List<RoleDialogModel>> OnResponsedDone(RealtimeHubConnection conn, string response);
33-
Task<RoleDialogModel> OnConversationItemCreated(RealtimeHubConnection conn, string response);
3433
}

‎src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeModelSettings.cs

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,12 @@ public class RealtimeModelSettings
1212
public string Voice { get; set; } = "alloy";
1313
public float Temperature { get; set; } = 0.8f;
1414
public int MaxResponseOutputTokens { get; set; } = 512;
15-
public int ModelResponseTimeout { get; set; } = 30;
15+
public int ModelResponseTimeoutSeconds { get; set; } = 30;
16+
17+
/// <summary>
18+
/// Whether the target event arrives after ModelResponseTimeoutSeconds, e.g., "response.done"
19+
/// </summary>
20+
public string? ModelResponseTimeoutEndEvent { get; set; }
1621
public AudioTranscription InputAudioTranscription { get; set; } = new();
1722
public ModelTurnDetection TurnDetection { get; set; } = new();
1823
}

‎src/Infrastructure/BotSharp.Core/Routing/Executor/IFunctionExecutor.cs renamed to ‎src/Infrastructure/BotSharp.Abstraction/Routing/Executor/IFunctionExecutor.cs

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,7 @@
1-
namespace BotSharp.Core.Routing.Executor;
1+
namespace BotSharp.Abstraction.Routing.Executor;
22

33
public interface IFunctionExecutor
44
{
55
public Task<bool> ExecuteAsync(RoleDialogModel message);
6-
76
public Task<string> GetIndicatorAsync(RoleDialogModel message);
87
}

‎src/Infrastructure/BotSharp.Core.Realtime/Hooks/RealtimeConversationHook.cs

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -72,7 +72,8 @@ public async Task OnFunctionExecuted(RoleDialogModel message)
7272
{
7373
return;
7474
}
75-
else if (message.StopCompletion)
75+
76+
if (message.StopCompletion)
7677
{
7778
await hub.Completer.TriggerModelInference($"Say to user: \"{message.Content}\"");
7879
}

‎src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -77,8 +77,8 @@ await HookEmitter.Emit<IRealtimeHook>(_services, async hook => await hook.OnMode
7777
{
7878
var data = _conn.OnModelAudioResponseDone();
7979
await (responseToUser?.Invoke(data) ?? Task.CompletedTask);
80-
},
81-
onAudioTranscriptDone: async transcript =>
80+
},
81+
onModelAudioTranscriptDone: async transcript =>
8282
{
8383

8484
},
@@ -98,6 +98,8 @@ await HookEmitter.Emit<IRoutingHook>(_services, async hook => await hook.OnRouti
9898
}
9999

100100
await routing.InvokeFunction(message.FunctionName, message);
101+
dialogs.Add(message);
102+
storage.Append(_conn.ConversationId, message);
101103
}
102104
else
103105
{
@@ -120,7 +122,7 @@ await HookEmitter.Emit<IRoutingHook>(_services, async hook => await hook.OnRouti
120122
{
121123

122124
},
123-
onInputAudioTranscriptionCompleted: async message =>
125+
onInputAudioTranscriptionDone: async message =>
124126
{
125127
// append input audio transcript to conversation
126128
dialogs.Add(message);

‎src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -96,6 +96,8 @@
9696
<None Remove="data\agents\dfd9b46d-d00c-40af-8a75-3fbdc2b89869\templates\instruction.simulator.liquid" />
9797
<None Remove="data\agents\dfd9b46d-d00c-40af-8a75-3fbdc2b89869\templates\instruction.simulator.liquid" />
9898
<None Remove="data\plugins\config.json" />
99+
100+
<None Remove="data\agents\01e2fc5c-2c89-4ec7-8470-7688608b496c\functions\get_weather.json" />
99101
</ItemGroup>
100102

101103
<ItemGroup>
@@ -204,6 +206,11 @@
204206
<Content Include="data\plugins\config.json">
205207
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
206208
</Content>
209+
210+
211+
<Content Include="data\agents\01e2fc5c-2c89-4ec7-8470-7688608b496c\functions\get_weather.json">
212+
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
213+
</Content>
207214
</ItemGroup>
208215

209216
<ItemGroup>

‎src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStorage.cs

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,8 @@ public void Append(string conversationId, IEnumerable<RoleDialogModel> dialogs)
4040
MessageId = dialog.MessageId,
4141
MessageType = dialog.MessageType,
4242
FunctionName = dialog.FunctionName,
43+
FunctionArgs = dialog.FunctionArgs,
44+
ToolCallId = dialog.ToolCallId,
4345
CreatedTime = dialog.CreatedAt
4446
};
4547

@@ -109,7 +111,6 @@ public List<RoleDialogModel> GetDialogs(string conversationId)
109111
var currentAgentId = meta.AgentId;
110112
var messageId = meta.MessageId;
111113
var messageType = meta.MessageType;
112-
var function = meta.FunctionName;
113114
var senderId = role == AgentRole.Function ? currentAgentId : meta.SenderId;
114115
var createdAt = meta.CreatedTime;
115116
var richContent = !string.IsNullOrEmpty(dialog.RichContent) ?
@@ -124,7 +125,9 @@ public List<RoleDialogModel> GetDialogs(string conversationId)
124125
MessageType = messageType,
125126
CreatedAt = createdAt,
126127
SenderId = senderId,
127-
FunctionName = function,
128+
FunctionName = meta.FunctionName,
129+
FunctionArgs = meta.FunctionArgs,
130+
ToolCallId = meta.ToolCallId,
128131
RichContent = richContent,
129132
SecondaryContent = secondaryContent,
130133
SecondaryRichContent = secondaryRichContent,

‎src/Infrastructure/BotSharp.Core/Demo/Functions/GetWeatherFn.cs

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
using BotSharp.Abstraction.Functions;
2+
3+
namespace BotSharp.Core.Demo.Functions;
4+
5+
public class GetWeatherFn : IFunctionCallback
6+
{
7+
private readonly IServiceProvider _services;
8+
9+
public GetWeatherFn(IServiceProvider services)
10+
{
11+
_services = services;
12+
}
13+
14+
public string Name => "get_weather";
15+
public string Indication => "Querying weather";
16+
17+
public async Task<bool> Execute(RoleDialogModel message)
18+
{
19+
message.Content = $"It is a sunny day!";
20+
//message.StopCompletion = true;
21+
return true;
22+
}
23+
}

‎src/Infrastructure/BotSharp.Core/Infrastructures/Websocket/AsyncWebsocketDataResultEnumerator.cs

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,3 @@
1-
using BotSharp.Abstraction.Realtime.Models.Session;
21
using System.Buffers;
32
using System.ClientModel;
43
using System.Net.WebSockets;
@@ -44,6 +43,9 @@ public async ValueTask<bool> MoveNextAsync()
4443

4544
if (receivedResult.CloseStatus.HasValue)
4645
{
46+
#if DEBUG
47+
Console.WriteLine($"Websocket close: {receivedResult.CloseStatus} {receivedResult.CloseStatusDescription}");
48+
#endif
4749
Current = null;
4850
return false;
4951
}

‎src/Infrastructure/BotSharp.Core/MCP/BotSharpMCPExtensions.cs

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -18,15 +18,11 @@ public static IServiceCollection AddBotSharpMCP(this IServiceCollection services
1818
{
1919
var settings = config.GetSection("MCP").Get<McpSettings>();
2020
services.AddScoped(provider => settings);
21+
services.AddScoped<IMcpService, McpService>();
2122

2223
if (settings != null && settings.Enabled && !settings.McpServerConfigs.IsNullOrEmpty())
2324
{
24-
services.AddScoped<IMcpService, McpService>();
25-
26-
var clientManager = new McpClientManager(settings);
27-
services.AddScoped(provider => clientManager);
28-
29-
// Register hooks
25+
services.AddScoped<McpClientManager>();
3026
services.AddScoped<IAgentHook, McpToolAgentHook>();
3127
}
3228
return services;

‎src/Infrastructure/BotSharp.Core/MCP/Helpers/AiFunctionHelper.cs

Lines changed: 16 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,19 +1,28 @@
1-
using System.Text.Json;
21
using ModelContextProtocol.Client;
32

43
namespace BotSharp.Core.MCP.Helpers;
54

65
internal static class AiFunctionHelper
76
{
8-
public static FunctionDef MapToFunctionDef(McpClientTool tool)
7+
public static FunctionDef? MapToFunctionDef(McpClientTool tool)
98
{
109
if (tool == null)
1110
{
12-
throw new ArgumentNullException(nameof(tool));
11+
return null;
1312
}
1413

15-
var properties = tool.JsonSchema.GetProperty("properties");
16-
var required = tool.JsonSchema.GetProperty("required");
14+
var properties = "{}";
15+
var required = "[]";
16+
17+
if (tool.JsonSchema.TryGetProperty("properties", out var p))
18+
{
19+
properties = p.GetRawText();
20+
}
21+
22+
if (tool.JsonSchema.TryGetProperty("required", out var r))
23+
{
24+
required = r.GetRawText();
25+
}
1726

1827
var funDef = new FunctionDef
1928
{
@@ -23,8 +32,8 @@ public static FunctionDef MapToFunctionDef(McpClientTool tool)
2332
Parameters = new FunctionParametersDef
2433
{
2534
Type = "object",
26-
Properties = JsonDocument.Parse(properties.GetRawText()),
27-
Required = JsonSerializer.Deserialize<List<string>>(required.GetRawText())
35+
Properties = JsonDocument.Parse(properties),
36+
Required = JsonSerializer.Deserialize<List<string>>(required) ?? []
2837
}
2938
};
3039

‎src/Infrastructure/BotSharp.Core/MCP/Hooks/MCPToolAgentHook.cs

Lines changed: 15 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -41,18 +41,26 @@ private async Task<IEnumerable<FunctionDef>> GetMcpContent(Agent agent)
4141
return functionDefs;
4242
}
4343

44-
var mcpClientManager = _services.GetRequiredService<McpClientManager>();
45-
var mcps = agent.McpTools.Where(x => !x.Disabled);
44+
var mcpClientManager = _services.GetService<McpClientManager>();
45+
if (mcpClientManager == null)
46+
{
47+
return functionDefs;
48+
}
49+
50+
var mcps = agent.McpTools?.Where(x => !x.Disabled) ?? [];
4651
foreach (var item in mcps)
4752
{
4853
var mcpClient = await mcpClientManager.GetMcpClientAsync(item.ServerId);
49-
if (mcpClient != null)
54+
if (mcpClient == null) continue;
55+
56+
var tools = await mcpClient.ListToolsAsync();
57+
var toolNames = item.Functions.Select(x => x.Name).ToList();
58+
var targetTools = tools.Where(x => toolNames.Contains(x.Name, StringComparer.OrdinalIgnoreCase));
59+
foreach (var tool in targetTools)
5060
{
51-
var tools = await mcpClient.ListToolsAsync();
52-
var toolnames = item.Functions.Select(x => x.Name).ToList();
53-
foreach (var tool in tools.Where(x => toolnames.Contains(x.Name, StringComparer.OrdinalIgnoreCase)))
61+
var funDef = AiFunctionHelper.MapToFunctionDef(tool);
62+
if (funDef != null)
5463
{
55-
var funDef = AiFunctionHelper.MapToFunctionDef(tool);
5664
functionDefs.Add(funDef);
5765
}
5866
}

0 commit comments

Comments
 (0)