Merge pull request #910 from hchen2020/master

add welcome for realtime

Author: Oceania2018

src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs

@@ -26,7 +26,9 @@ Task Connect(RealtimeHubConnection conn,
     Task<RealtimeSession> CreateSession(Agent agent, List<RoleDialogModel> conversations);
     Task UpdateSession(RealtimeHubConnection conn);
     Task InsertConversationItem(RoleDialogModel message);
+    Task RemoveConversationItem(string itemId);
     Task TriggerModelInference(string? instructions = null);
+    Task CancelModelResponse();
     Task<List<RoleDialogModel>> OnResponsedDone(RealtimeHubConnection conn, string response);
     Task<RoleDialogModel> OnConversationItemCreated(RealtimeHubConnection conn, string response);
 }

src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs

@@ -3,6 +3,7 @@
 using BotSharp.Abstraction.Realtime.Models;
 using BotSharp.Abstraction.MLTasks;
 using BotSharp.Abstraction.Conversations.Enums;
+using BotSharp.Abstraction.Routing.Models;
 
 namespace BotSharp.Core.Realtime;
 
@@ -97,7 +98,7 @@ await completer.Connect(conn,
 
                 if (dialogs.LastOrDefault()?.Role == AgentRole.Assistant)
                 {
-                    // await completer.TriggerModelInference($"Rephase your last response:\r\n{dialogs.LastOrDefault()?.Content}");
+                    await completer.TriggerModelInference($"Rephase your last response:\r\n{dialogs.LastOrDefault()?.Content}");
                 }
                 else
                 {
@@ -127,19 +128,32 @@ await completer.Connect(conn,
                     {
                         await routing.InvokeFunction(message.FunctionName, message);
                         message.Role = AgentRole.Function;
-                        if (message.FunctionName == "route_to_agent" ||
-                            message.FunctionName == "util-routing-fallback_to_router")
+
+                        if (message.FunctionName == "route_to_agent")
                         {
-                            var routedAgentId = routing.Context.GetCurrentAgentId();
-                            if (conn.CurrentAgentId != routedAgentId)
-                            {
-                                conn.CurrentAgentId = routedAgentId;
-                                await completer.UpdateSession(conn);
-                            }
+                            var inst = JsonSerializer.Deserialize<RoutingArgs>(message.FunctionArgs ?? "{}");
+                            message.Content = $"Connected to agent of {inst.AgentName}";
+                            conn.CurrentAgentId = routing.Context.GetCurrentAgentId();
+
+                            await completer.UpdateSession(conn);
+                            await completer.InsertConversationItem(message);
+                            await completer.TriggerModelInference($"Continue to proceed user request in {inst.AgentName}.");
                         }
+                        else if (message.FunctionName == "util-routing-fallback_to_router")
+                        {
+                            var inst = JsonSerializer.Deserialize<FallbackArgs>(message.FunctionArgs ?? "{}");
+                            message.Content = $"Returned to Router due to {inst.Reason}";
+                            conn.CurrentAgentId = routing.Context.GetCurrentAgentId();
 
-                        await completer.InsertConversationItem(message);
-                        await completer.TriggerModelInference("Reply based on the function's output.");
+                            await completer.UpdateSession(conn);
+                            await completer.InsertConversationItem(message);
+                            await completer.TriggerModelInference("Reply user request.");
+                        }
+                        else
+                        {
+                            await completer.InsertConversationItem(message);
+                            await completer.TriggerModelInference("Reply based on the function's output.");
+                        }
                     }
                     else
                     {

src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/functions/route_to_agent.json

@@ -26,6 +26,6 @@
         "description": "Required parameters of next action agent"
       }
     },
-    "required": [ "next_action_agent", "user_goal_agent", "args" ]
+    "required": [ "next_action_agent", "user_goal_agent", "next_action_reason", "args" ]
   }
 }

src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/instructions/instruction.liquid

@@ -5,7 +5,9 @@ Follow these steps to handle user request:
 2. Determine which agent is suitable to handle this conversation. Try to minimize the routing of human service.
 3. Extract and populate agent required arguments, think carefully, leave it as blank object if user didn't provide the specific arguments.
 4. You must include all required args for the selected agent, but you must not make up any parameters when there is no exact value provided, those parameters must set value as null if not declared.
+{% if routing_mode != 'lazy' %}
 5. Response must be in JSON format.
+{% endif %}
 
 {% if routing_requirements and routing_requirements != empty %}
 [REQUIREMENTS]

src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-routing-fallback_to_router.json

@@ -13,6 +13,6 @@
         "description": "User question or statement."
       }
     },
-    "required": [ "user_question" ]
+    "required": [ "fallback_reason" ]
   }
 }

src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs

@@ -102,6 +102,23 @@ await SendEventToModel(new
         });
     }
 
+    public async Task CancelModelResponse()
+    {
+        await SendEventToModel(new
+        {
+            type = "response.cancel"
+        });
+    }
+
+    public async Task RemoveConversationItem(string itemId)
+    {
+        await SendEventToModel(new
+        {
+            type = "conversation.item.delete",
+            item_id = itemId
+        });
+    }
+
     private async Task ReceiveMessage(RealtimeHubConnection conn, 
         Action<string> onModelAudioDeltaReceived,
         Action onModelAudioResponseDone,
@@ -169,7 +186,6 @@ private async Task ReceiveMessage(RealtimeHubConnection conn,
             else if (response.Type == "response.done")
             {
                 _logger.LogInformation($"{response.Type}: {receivedText}");
-                await Task.Delay(1000);
                 var messages = await OnResponsedDone(conn, receivedText);
                 onModelResponseDone(messages);
             }
@@ -296,7 +312,13 @@ public async Task UpdateSession(RealtimeHubConnection conn)
                 ToolChoice = "auto",
                 Tools = functions,
                 Modalities = [ "text", "audio" ],
-                Temperature = Math.Max(options.Temperature ?? 0f, 0.6f)
+                Temperature = Math.Max(options.Temperature ?? 0f, 0.6f),
+                MaxResponseOutputTokens = 512,
+                TurnDetection = new RealtimeSessionTurnDetection
+                {
+                    Threshold = 0.8f,
+                    SilenceDuration = 800
+                }
             }
         };
 
@@ -565,6 +587,11 @@ public async Task<List<RoleDialogModel>> OnResponsedDone(RealtimeHubConnection c
         var outputs = new List<RoleDialogModel>();
 
         var data = JsonSerializer.Deserialize<ResponseDone>(response).Body;
+        if (data.Status != "completed")
+        {
+            return [];
+        }
+
         foreach (var output in data.Outputs)
         {
             if (output.Type == "function_call")
@@ -575,6 +602,7 @@ public async Task<List<RoleDialogModel>> OnResponsedDone(RealtimeHubConnection c
                     FunctionName = output.Name,
                     FunctionArgs = output.Arguments,
                     ToolCallId = output.CallId,
+                    MessageId = output.Id,
                     MessageType = MessageTypeName.FunctionCall
                 });
             }