Merge branch 'master' of https://github.com/SciSharp/BotSharp

Author: Oceania2018

Directory.Packages.props

@@ -111,7 +111,8 @@
     <PackageVersion Include="MSTest.TestFramework" Version="3.1.1" />
     <PackageVersion Include="Microsoft.Extensions.DependencyInjection" Version="8.0.1" />
     <PackageVersion Include="Microsoft.Extensions.Hosting" Version="8.0.0" />
-    <PackageVersion Include="ModelContextProtocol" Version="0.1.0-preview.2" />
+    <PackageVersion Include="ModelContextProtocol" Version="0.1.0-preview.5" /> 
+    <PackageVersion Include="ModelContextProtocol.AspNetCore" Version="0.1.0-preview.5" />
   </ItemGroup>
   <ItemGroup>
     <PackageVersion Include="BotSharp.Core" Version="$(BotSharpVersion)" />

src/Infrastructure/BotSharp.Abstraction/Browsing/Models/ElementLocatingArgs.cs

@@ -36,4 +36,6 @@ public class ElementLocatingArgs
     /// </summary>
     public bool Highlight { get; set; }
     public string HighlightColor { get; set; } = "red";
+    [JsonPropertyName("is_read_content")]
+    public bool IsReadContent { get;set; }
 }

src/Infrastructure/BotSharp.Abstraction/MCP/Models/McpServerConfigModel.cs

@@ -24,11 +24,6 @@ public class McpServerConfigModel
     /// </summary>
     public string? Location { get; set; }
 
-    /// <summary>
-    /// Arguments (if any) to pass to the executable.
-    /// </summary>
-    public string[]? Arguments { get; set; }
-
     /// <summary>
     /// Additional transport-specific configuration.
     /// </summary>

src/Infrastructure/BotSharp.Core.MCP/BotSharpMCPExtensions.cs

@@ -1,10 +1,11 @@
 using BotSharp.Core.MCP.Functions;
-using BotSharp.Core.MCP.Settings;
 using BotSharp.Core.MCP.Hooks;
+using BotSharp.Core.MCP.Managers;
+using BotSharp.Core.MCP.Services;
+using BotSharp.Core.MCP.Settings;
 using Microsoft.Extensions.Configuration;
-using ModelContextProtocol.Configuration;
+using ModelContextProtocol;
 using ModelContextProtocol.Client;
-using BotSharp.Core.MCP.Managers;
 
 namespace BotSharp.Core.MCP;
 
@@ -18,6 +19,7 @@ public static class BotSharpMcpExtensions
     /// <returns></returns>
     public static IServiceCollection AddBotSharpMCP(this IServiceCollection services, IConfiguration config)
     {
+        services.AddScoped<IMcpService, McpService>();
         var settings = config.GetSection("MCP").Get<McpSettings>();
         services.AddScoped(provider => { return settings; });
 

src/Infrastructure/BotSharp.Core.MCP/McpPlugin.cs

@@ -26,7 +26,6 @@ public IEnumerable<McpServerConfigModel> GetServerConfigs()
             Name = x.Name,
             TransportType = x.TransportType,
             TransportOptions = x.TransportOptions,
-            Arguments = x.Arguments,
             Location = x.Location
         });
     }

src/Infrastructure/BotSharp.Core.MCP/Services/McpService.cs

@@ -1,5 +1,5 @@
 using ModelContextProtocol.Client;
-using ModelContextProtocol.Configuration;
+using ModelContextProtocol;
 
 namespace BotSharp.Core.MCP.Settings;
 

src/Infrastructure/BotSharp.Core.MCP/Settings/MCPSettings.cs

@@ -1,4 +1,5 @@
 using BotSharp.Abstraction.Utilities;
+using BotSharp.Core.Infrastructures;
 
 namespace BotSharp.Core.Realtime.Hooks;
 
@@ -29,6 +30,10 @@ public async Task OnFunctionExecuted(RoleDialogModel message)
         {
             return;
         }
+
+        // Clear cache to force to rebuild the agent instruction
+        Utilities.ClearCache();
+
         var routing = _services.GetRequiredService<IRoutingService>();
 
         message.Role = AgentRole.Function;

src/Infrastructure/BotSharp.Core.Realtime/Hooks/RealtimeConversationHook.cs

@@ -49,6 +49,9 @@ public async Task Listen(WebSocket userWebSocket,
             {
                 await _completer.AppenAudioBuffer(_conn.Data);
             }
+            else if (_conn.Event == "user_dtmf_receiving")
+            {
+            }
             else if (_conn.Event == "user_dtmf_received")
             {
                 await HandleUserDtmfReceived();

src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs

@@ -144,13 +144,28 @@ private async Task ReceiveMessage(RealtimeHubConnection conn,
         Action onUserInterrupted)
     {
         var buffer = new byte[1024 * 32];
-        WebSocketReceiveResult result;
+        // Model response timeout
+        var timeout = 30;
+        WebSocketReceiveResult? result = default;
 
         do
         {
             Array.Clear(buffer, 0, buffer.Length);
-            result = await _webSocket.ReceiveAsync(
-                new ArraySegment<byte>(buffer), CancellationToken.None);
+            
+            var taskWorker = _webSocket.ReceiveAsync(new ArraySegment<byte>(buffer), CancellationToken.None);
+            var taskTimer = Task.Delay(1000 * timeout);
+            var completedTask = await Task.WhenAny(taskWorker, taskTimer);
+
+            if (completedTask == taskWorker)
+            {
+                result = taskWorker.Result;
+            }
+            else
+            {
+                _logger.LogWarning($"Timeout {timeout} seconds waiting for Model response.");
+                await TriggerModelInference("Response user immediately");
+                continue;
+            }
 
             // Convert received data to text/audio (Twilio sends Base64-encoded audio)
             string receivedText = Encoding.UTF8.GetString(buffer, 0, result.Count);
@@ -164,6 +179,11 @@ private async Task ReceiveMessage(RealtimeHubConnection conn,
             if (response.Type == "error")
             {
                 _logger.LogError($"{response.Type}: {receivedText}");
+                var error = JsonSerializer.Deserialize<ServerEventErrorResponse>(receivedText);
+                if (error?.Body.Type == "server_error")
+                {
+                    break;
+                }
             }
             else if (response.Type == "session.created")
             {
@@ -182,7 +202,6 @@ private async Task ReceiveMessage(RealtimeHubConnection conn,
             {
                 _logger.LogInformation($"{response.Type}: {receivedText}");
                 var data = JsonSerializer.Deserialize<ResponseAudioTranscript>(receivedText);
-                await Task.Delay(1000);
                 onModelAudioTranscriptDone(data.Transcript);
             }
             else if (response.Type == "response.audio.delta")

src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs

@@ -1,5 +1,4 @@
 using BotSharp.Abstraction.Realtime;
-using BotSharp.Abstraction.Realtime.Models;
 using BotSharp.Plugin.Twilio.Interfaces;
 using BotSharp.Plugin.Twilio.Models.Stream;
 using Microsoft.AspNetCore.Http;
@@ -100,6 +99,7 @@ await hub.Listen(webSocket, (receivedText) =>
                     }
                     else
                     {
+                        conn.Event = "user_dtmf_receiving";
                         conn.KeypadInputBuffer += dtmfResponse.Body.Digit;
                     }
                     break;
@@ -115,19 +115,29 @@ await hub.Listen(webSocket, (receivedText) =>
                     streamSid = response.StreamSid,
                     media = new { payload = message }
                 };
+
             conn.OnModelAudioResponseDone = () =>
                 new
                 {
                     @event = "mark",
                     streamSid = response.StreamSid,
                     mark = new { name = "responsePart" }
                 };
+
             conn.OnModelUserInterrupted = () =>
                 new
                 {
                     @event = "clear",
                     streamSid = response.StreamSid
                 };
+
+            /*if (response.Event == "dtmf")
+            {
+                // Send a Stop command to Twilio
+                string stopPlaybackCommand = "{ \"action\": \"stop_playback\" }";
+                var stopBytes = Encoding.UTF8.GetBytes(stopPlaybackCommand);
+                webSocket.SendAsync(new ArraySegment<byte>(stopBytes), WebSocketMessageType.Text, true, CancellationToken.None);
+            }*/
         });
     }
 }

src/Plugins/BotSharp.Plugin.Twilio/TwilioStreamMiddleware.cs

@@ -32,7 +32,11 @@ public async Task<bool> Execute(RoleDialogModel message)
         };
         var result = await browser.LocateElement(msg, locatorArgs);
 
-        message.Content = $"Locating element {(result.IsSuccess ? "success" : "failed")}";
+        message.Content = $"Locating element {(result.IsSuccess ? "success" : "failed")}. ";
+        if (locatorArgs.IsReadContent && result.IsSuccess && !string.IsNullOrWhiteSpace(result.Body))
+        {
+            message.Content += $"Content is: \n{result.Body}";
+        }
 
         var path = webDriverService.GetScreenshotFilePath(message.MessageId);
 

src/Plugins/BotSharp.Plugin.WebDriver/UtilFunctions/UtilWebLocateElementFn.cs

@@ -7,6 +7,10 @@
       "selector": {
         "type": "string",
         "description": "element selector in XPath, use syntax of Playwright in .NET"
+      },
+      "is_read_content": {
+        "type": "boolean",
+        "description": "read the content"
       }
     },
     "required": [ "selector" ]

src/Plugins/BotSharp.Plugin.WebDriver/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-web-locate_element.json

@@ -10,6 +10,7 @@
     <PackageReference Include="Microsoft.Extensions.Hosting" />
     <PackageReference Include="Microsoft.Extensions.Logging" />
     <PackageReference Include="ModelContextProtocol" />
+    <PackageReference Include="ModelContextProtocol.AspNetCore" />
     <PackageReference Include="Serilog.Extensions.Logging" />
     <PackageReference Include="Serilog.Sinks.File" />
   </ItemGroup>