Test/google realtime

src/Infrastructure/BotSharp.Abstraction/Conversations/Models/Conversation.cs

@@ -47,15 +47,19 @@ public class DialogElement
     public string Content { get; set; } = default!;
 
     [JsonPropertyName("secondary_content")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
     public string? SecondaryContent { get; set; }
 
     [JsonPropertyName("rich_content")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
     public string? RichContent { get; set; }
 
     [JsonPropertyName("secondary_rich_content")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
     public string? SecondaryRichContent { get; set; }
 
     [JsonPropertyName("payload")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
     public string? Payload { get; set; }
 
     public DialogElement()
@@ -95,8 +99,17 @@ public class DialogMetaData
     public string MessageType { get; set; } = default!;
 
     [JsonPropertyName("function_name")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
     public string? FunctionName { get; set; }
 
+    [JsonPropertyName("function_args")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public string? FunctionArgs { get; set; }
+
+    [JsonPropertyName("tool_call_id")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public string? ToolCallId { get; set; }
+
     [JsonPropertyName("sender_id")]
     public string? SenderId { get; set; }
 

src/Infrastructure/BotSharp.Abstraction/Hooks/HookProvider.cs

@@ -1,25 +1,19 @@
 using BotSharp.Abstraction.Conversations;
 using Microsoft.Extensions.DependencyInjection;
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
 
-namespace BotSharp.Abstraction.Hooks
+namespace BotSharp.Abstraction.Hooks;
+
+public static class HookProvider    
 {
-    public static class HookProvider    
+    public static List<T> GetHooks<T>(this IServiceProvider services, string agentId) where T : IHookBase
     {
-        public static List<T> GetHooks<T>(this IServiceProvider services, string agentId) where T : IHookBase
-        {
-            var hooks = services.GetServices<T>().Where(p => p.IsMatch(agentId));
-            return hooks.ToList();
-        }
+        var hooks = services.GetServices<T>().Where(p => p.IsMatch(agentId));
+        return hooks.ToList();
+    }
 
-        public static List<T> GetHooksOrderByPriority<T>(this IServiceProvider services, string agentId) where T: IConversationHook
-        {
-            var hooks = services.GetServices<T>().Where(p => p.IsMatch(agentId));
-            return hooks.OrderBy(p => p.Priority).ToList();
-        }
+    public static List<T> GetHooksOrderByPriority<T>(this IServiceProvider services, string agentId) where T: IConversationHook
+    {
+        var hooks = services.GetServices<T>().Where(p => p.IsMatch(agentId));
+        return hooks.OrderBy(p => p.Priority).ToList();
     }
 }

src/Infrastructure/BotSharp.Abstraction/Hooks/IHookBase.cs

@@ -1,17 +1,10 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
+namespace BotSharp.Abstraction.Hooks;
 
-namespace BotSharp.Abstraction.Hooks
+public interface IHookBase
 {
-    public interface IHookBase
-    {
-        /// <summary>
-        /// Agent Id
-        /// </summary>
-        string SelfId => string.Empty;
-        bool IsMatch(string agentId) => string.IsNullOrEmpty(SelfId) || SelfId == agentId;
-    }
+    /// <summary>
+    /// Agent Id
+    /// </summary>
+    string SelfId => string.Empty;
+    bool IsMatch(string agentId) => string.IsNullOrEmpty(SelfId) || SelfId == agentId;
 }

src/Infrastructure/BotSharp.Abstraction/MCP/Services/IMcpService.cs

@@ -2,5 +2,5 @@ namespace BotSharp.Abstraction.MCP.Services;
 
 public interface IMcpService
 {
-    IEnumerable<McpServerOptionModel> GetServerConfigs() => [];
+    Task<IEnumerable<McpServerOptionModel>> GetServerConfigsAsync() => Task.FromResult<IEnumerable<McpServerOptionModel>>([]);
 }

src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs

@@ -10,14 +10,15 @@ public interface IRealTimeCompletion
 
     Task Connect(
         RealtimeHubConnection conn,
-        Action onModelReady,
-        Action<string, string> onModelAudioDeltaReceived,
-        Action onModelAudioResponseDone,
-        Action<string> onAudioTranscriptDone,
-        Action<List<RoleDialogModel>> onModelResponseDone,
-        Action<string> onConversationItemCreated,
-        Action<RoleDialogModel> onInputAudioTranscriptionCompleted,
-        Action onInterruptionDetected);
+        Func<Task> onModelReady,
+        Func<string, string, Task> onModelAudioDeltaReceived,
+        Func<Task> onModelAudioResponseDone,
+        Func<string, Task> onModelAudioTranscriptDone,
+        Func<List<RoleDialogModel>, Task> onModelResponseDone,
+        Func<string, Task> onConversationItemCreated,
+        Func<RoleDialogModel, Task> onInputAudioTranscriptionDone,
+        Func<Task> onInterruptionDetected);
+
     Task AppenAudioBuffer(string message);
     Task AppenAudioBuffer(ArraySegment<byte> data, int length);
 
@@ -29,6 +30,4 @@ Task Connect(
     Task RemoveConversationItem(string itemId);
     Task TriggerModelInference(string? instructions = null);
     Task CancelModelResponse();
-    Task<List<RoleDialogModel>> OnResponsedDone(RealtimeHubConnection conn, string response);
-    Task<RoleDialogModel> OnConversationItemCreated(RealtimeHubConnection conn, string response);
 }

src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeModelSettings.cs

@@ -12,7 +12,12 @@ public class RealtimeModelSettings
     public string Voice { get; set; } = "alloy";
     public float Temperature { get; set; } = 0.8f;
     public int MaxResponseOutputTokens { get; set; } = 512;
-    public int ModelResponseTimeout { get; set; } = 30;
+    public int ModelResponseTimeoutSeconds { get; set; } = 30;
+
+    /// <summary>
+    /// Whether the target event arrives after ModelResponseTimeoutSeconds, e.g., "response.done"
+    /// </summary>
+    public string? ModelResponseTimeoutEndEvent { get; set; }
     public AudioTranscription InputAudioTranscription { get; set; } = new();
     public ModelTurnDetection TurnDetection { get; set; } = new();
 }

src/Infrastructure/BotSharp.Core/Routing/Executor/IFunctionExecutor.cs → src/Infrastructure/BotSharp.Abstraction/Routing/Executor/IFunctionExecutor.cs

@@ -1,8 +1,7 @@
-namespace BotSharp.Core.Routing.Executor;
+namespace BotSharp.Abstraction.Routing.Executor;
 
 public interface IFunctionExecutor
 {
     public Task<bool> ExecuteAsync(RoleDialogModel message);
-
     public Task<string> GetIndicatorAsync(RoleDialogModel message);
 }

src/Infrastructure/BotSharp.Core.Realtime/Hooks/RealtimeConversationHook.cs

@@ -72,7 +72,8 @@ public async Task OnFunctionExecuted(RoleDialogModel message)
             {
                 return;
             }
-            else if (message.StopCompletion)
+
+            if (message.StopCompletion)
             {
                 await hub.Completer.TriggerModelInference($"Say to user: \"{message.Content}\"");
             }

src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs

@@ -77,8 +77,8 @@ await HookEmitter.Emit<IRealtimeHook>(_services, async hook => await hook.OnMode
             {
                 var data = _conn.OnModelAudioResponseDone();
                 await (responseToUser?.Invoke(data) ?? Task.CompletedTask);
-            }, 
-            onAudioTranscriptDone: async transcript =>
+            },
+            onModelAudioTranscriptDone: async transcript =>
             {
 
             },
@@ -98,6 +98,8 @@ await HookEmitter.Emit<IRoutingHook>(_services, async hook => await hook.OnRouti
                         }
 
                         await routing.InvokeFunction(message.FunctionName, message);
+                        dialogs.Add(message);
+                        storage.Append(_conn.ConversationId, message);
                     }
                     else
                     {
@@ -120,7 +122,7 @@ await HookEmitter.Emit<IRoutingHook>(_services, async hook => await hook.OnRouti
             {
 
             },
-            onInputAudioTranscriptionCompleted: async message =>
+            onInputAudioTranscriptionDone: async message =>
             {
                 // append input audio transcript to conversation
                 dialogs.Add(message);

src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj

@@ -96,6 +96,8 @@
     <None Remove="data\agents\dfd9b46d-d00c-40af-8a75-3fbdc2b89869\templates\instruction.simulator.liquid" />
     <None Remove="data\agents\dfd9b46d-d00c-40af-8a75-3fbdc2b89869\templates\instruction.simulator.liquid" />
     <None Remove="data\plugins\config.json" />
+
+    <None Remove="data\agents\01e2fc5c-2c89-4ec7-8470-7688608b496c\functions\get_weather.json" />
   </ItemGroup>
 
   <ItemGroup>
@@ -204,6 +206,11 @@
     <Content Include="data\plugins\config.json">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </Content>
+
+
+    <Content Include="data\agents\01e2fc5c-2c89-4ec7-8470-7688608b496c\functions\get_weather.json">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content>
   </ItemGroup>
 
   <ItemGroup>

src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStorage.cs

@@ -40,6 +40,8 @@ public void Append(string conversationId, IEnumerable<RoleDialogModel> dialogs)
                     MessageId = dialog.MessageId,
                     MessageType = dialog.MessageType,
                     FunctionName = dialog.FunctionName,
+                    FunctionArgs = dialog.FunctionArgs,
+                    ToolCallId = dialog.ToolCallId,
                     CreatedTime = dialog.CreatedAt
                 };
 
@@ -109,7 +111,6 @@ public List<RoleDialogModel> GetDialogs(string conversationId)
             var currentAgentId = meta.AgentId;
             var messageId = meta.MessageId;
             var messageType = meta.MessageType;
-            var function = meta.FunctionName;
             var senderId = role == AgentRole.Function ? currentAgentId : meta.SenderId;
             var createdAt = meta.CreatedTime;
             var richContent = !string.IsNullOrEmpty(dialog.RichContent) ? 
@@ -124,7 +125,9 @@ public List<RoleDialogModel> GetDialogs(string conversationId)
                 MessageType = messageType,
                 CreatedAt = createdAt,
                 SenderId = senderId,
-                FunctionName = function,
+                FunctionName = meta.FunctionName,
+                FunctionArgs = meta.FunctionArgs,
+                ToolCallId = meta.ToolCallId,
                 RichContent = richContent,
                 SecondaryContent = secondaryContent,
                 SecondaryRichContent = secondaryRichContent,

src/Infrastructure/BotSharp.Core/Demo/Functions/GetWeatherFn.cs

@@ -0,0 +1,23 @@
+using BotSharp.Abstraction.Functions;
+
+namespace BotSharp.Core.Demo.Functions;
+
+public class GetWeatherFn : IFunctionCallback
+{
+    private readonly IServiceProvider _services;
+
+    public GetWeatherFn(IServiceProvider services)
+    {
+        _services = services;
+    }
+
+    public string Name => "get_weather";
+    public string Indication => "Querying weather";
+
+    public async Task<bool> Execute(RoleDialogModel message)
+    {
+        message.Content = $"It is a sunny day!";
+        //message.StopCompletion = true;
+        return true;
+    }
+}

src/Infrastructure/BotSharp.Core/Infrastructures/Websocket/AsyncWebsocketDataResultEnumerator.cs

@@ -1,4 +1,3 @@
-using BotSharp.Abstraction.Realtime.Models.Session;
 using System.Buffers;
 using System.ClientModel;
 using System.Net.WebSockets;
@@ -44,6 +43,9 @@ public async ValueTask<bool> MoveNextAsync()
 
             if (receivedResult.CloseStatus.HasValue)
             {
+#if DEBUG
+                Console.WriteLine($"Websocket close: {receivedResult.CloseStatus} {receivedResult.CloseStatusDescription}");
+#endif
                 Current = null;
                 return false;
             }

src/Infrastructure/BotSharp.Core/MCP/BotSharpMCPExtensions.cs

@@ -18,15 +18,11 @@ public static IServiceCollection AddBotSharpMCP(this IServiceCollection services
     {
         var settings = config.GetSection("MCP").Get<McpSettings>();
         services.AddScoped(provider => settings);
+        services.AddScoped<IMcpService, McpService>();
 
         if (settings != null && settings.Enabled && !settings.McpServerConfigs.IsNullOrEmpty())
         {
-            services.AddScoped<IMcpService, McpService>();
-
-            var clientManager = new McpClientManager(settings);
-            services.AddScoped(provider => clientManager);
-
-            // Register hooks
+            services.AddScoped<McpClientManager>();
             services.AddScoped<IAgentHook, McpToolAgentHook>();
         }
         return services;

src/Infrastructure/BotSharp.Core/MCP/Helpers/AiFunctionHelper.cs

@@ -1,19 +1,28 @@
-using System.Text.Json;
 using ModelContextProtocol.Client;
 
 namespace BotSharp.Core.MCP.Helpers;
 
 internal static class AiFunctionHelper
 {
-    public static FunctionDef MapToFunctionDef(McpClientTool tool)
+    public static FunctionDef? MapToFunctionDef(McpClientTool tool)
     {
         if (tool == null)
         {
-            throw new ArgumentNullException(nameof(tool));
+            return null;
         }
 
-        var properties = tool.JsonSchema.GetProperty("properties");
-        var required = tool.JsonSchema.GetProperty("required");
+        var properties = "{}";
+        var required = "[]";
+
+        if (tool.JsonSchema.TryGetProperty("properties", out var p))
+        {
+            properties = p.GetRawText();
+        }
+
+        if (tool.JsonSchema.TryGetProperty("required", out var r))
+        {
+            required = r.GetRawText();
+        }
 
         var funDef = new FunctionDef
         {
@@ -23,8 +32,8 @@ public static FunctionDef MapToFunctionDef(McpClientTool tool)
             Parameters = new FunctionParametersDef
             {
                 Type = "object",
-                Properties = JsonDocument.Parse(properties.GetRawText()),
-                Required = JsonSerializer.Deserialize<List<string>>(required.GetRawText())
+                Properties = JsonDocument.Parse(properties),
+                Required = JsonSerializer.Deserialize<List<string>>(required) ?? []
             }
         };
 

src/Infrastructure/BotSharp.Core/MCP/Hooks/MCPToolAgentHook.cs

@@ -41,18 +41,26 @@ private async Task<IEnumerable<FunctionDef>> GetMcpContent(Agent agent)
             return functionDefs;
         }
 
-        var mcpClientManager = _services.GetRequiredService<McpClientManager>();
-        var mcps = agent.McpTools.Where(x => !x.Disabled);
+        var mcpClientManager = _services.GetService<McpClientManager>();
+        if (mcpClientManager == null)
+        {
+            return functionDefs;
+        }
+
+        var mcps = agent.McpTools?.Where(x => !x.Disabled) ?? [];
         foreach (var item in mcps)
         {
             var mcpClient =  await mcpClientManager.GetMcpClientAsync(item.ServerId);
-            if (mcpClient != null)
+            if (mcpClient == null) continue;
+
+            var tools = await mcpClient.ListToolsAsync();
+            var toolNames = item.Functions.Select(x => x.Name).ToList();
+            var targetTools = tools.Where(x => toolNames.Contains(x.Name, StringComparer.OrdinalIgnoreCase));
+            foreach (var tool in targetTools)
             {
-                var tools = await mcpClient.ListToolsAsync();
-                var toolnames = item.Functions.Select(x => x.Name).ToList();
-                foreach (var tool in tools.Where(x => toolnames.Contains(x.Name, StringComparer.OrdinalIgnoreCase)))
+                var funDef = AiFunctionHelper.MapToFunctionDef(tool);
+                if (funDef != null)
                 {
-                    var funDef = AiFunctionHelper.MapToFunctionDef(tool);
                     functionDefs.Add(funDef);
                 }
             }