init conv simulation

src/Infrastructure/BotSharp.Abstraction/Agents/Enums/BuiltInAgentId.cs

@@ -46,4 +46,9 @@ public class BuiltInAgentId
     /// Programming source code generation
     /// </summary>
     public const string CodeDriver = "c0ded7d9-3f9d-4ef6-b7ce-56a892dcef62";
+
+    /// <summary>
+    /// Evaluate prompt and conversation
+    /// </summary>
+    public const string Evaluator = "dfd9b46d-d00c-40af-8a75-3fbdc2b89869";
 }

src/Infrastructure/BotSharp.Abstraction/Evaluations/Models/EvaluationRequest.cs

@@ -1,6 +1,18 @@
+using BotSharp.Abstraction.Processors.Models;
+
 namespace BotSharp.Abstraction.Evaluations.Models;
 
-public class EvaluationRequest
+public class EvaluationRequest : LlmBaseRequest
 {
-    public string AgentId { get; set; }
+    [JsonPropertyName("agent_id")]
+    public new string AgentId { get; set; }
+
+    [JsonPropertyName("states")]
+    public IEnumerable<MessageState> States { get; set; } = [];
+
+    [JsonPropertyName("max_rounds")]
+    public int MaxRounds { get; set; } = 20;
+
+    [JsonPropertyName("ref_conversation_id")]
+    public string RefConversationId { get; set; } = null!;
 }

src/Infrastructure/BotSharp.Abstraction/Evaluations/Models/EvaluationResult.cs

@@ -5,4 +5,5 @@ public class EvaluationResult
     public List<RoleDialogModel> Dialogs { get; set; }
     public string TaskInstruction { get; set; }
     public string SystemPrompt { get; set; }
+    public string GeneratedConversationId { get; set; }
 }

src/Infrastructure/BotSharp.Abstraction/Evaluations/Models/SimulationResult.cs

@@ -0,0 +1,13 @@
+namespace BotSharp.Abstraction.Evaluations.Models;
+
+public class SimulationResult
+{
+    [JsonPropertyName("generated_message")]
+    public string GeneratedMessage { get; set; }
+
+    [JsonPropertyName("stop")]
+    public bool Stop { get; set; }
+
+    [JsonPropertyName("reason")]
+    public string? Reason { get; set; }
+}

src/Infrastructure/BotSharp.Abstraction/Plugins/Models/PluginDef.cs

@@ -7,7 +7,7 @@ public class PluginDef
     public string Description { get; set; }
     public string Assembly { get; set; }
     [JsonPropertyName("is_core")]
-    public bool IsCore => Assembly == "BotSharp.Core";
+    public bool IsCore => Assembly == "BotSharp.Core" || Assembly == "BotSharp.Core.SideCar";
 
     [JsonPropertyName("icon_url")]
     public string? IconUrl { get; set; }

src/Infrastructure/BotSharp.Abstraction/Processors/Models/LlmBaseRequest.cs

@@ -2,8 +2,15 @@ namespace BotSharp.Abstraction.Processors.Models;
 
 public class LlmBaseRequest
 {
+    [JsonPropertyName("provider")]
     public string Provider { get; set; }
+
+    [JsonPropertyName("model")]
     public string Model { get; set; }
+
+    [JsonPropertyName("agent_id")]
     public string? AgentId { get; set; }
+
+    [JsonPropertyName("template_name")]
     public string? TemplateName { get; set; }
 }

src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj

@@ -83,7 +83,9 @@
     <None Remove="data\agents\dfd9b46d-d00c-40af-8a75-3fbdc2b89869\agent.json" />
     <None Remove="data\agents\dfd9b46d-d00c-40af-8a75-3fbdc2b89869\instructions\instruction.liquid" />
     <None Remove="data\agents\dfd9b46d-d00c-40af-8a75-3fbdc2b89869\templates\instruction.executor.liquid" />
+    <None Remove="data\agents\dfd9b46d-d00c-40af-8a75-3fbdc2b89869\templates\instruction.metrics.liquid" />
     <None Remove="data\agents\dfd9b46d-d00c-40af-8a75-3fbdc2b89869\templates\instruction.reviewer.liquid" />
+    <None Remove="data\agents\dfd9b46d-d00c-40af-8a75-3fbdc2b89869\templates\instruction.simulator.liquid" />
     <None Remove="data\plugins\config.json" />
   </ItemGroup>
 
@@ -163,6 +165,12 @@
     <Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\select_file_prompt.liquid">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </Content>
+    <Content Include="data\agents\dfd9b46d-d00c-40af-8a75-3fbdc2b89869\templates\instruction.simulator.liquid">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content>
+    <Content Include="data\agents\dfd9b46d-d00c-40af-8a75-3fbdc2b89869\templates\instruction.metrics.liquid">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content>
     <Content Include="data\plugins\config.json">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </Content>

src/Infrastructure/BotSharp.Core/Evaluations/EvaluationPlugin.cs

@@ -1,8 +1,8 @@
 using BotSharp.Abstraction.Evaluations.Settings;
 using BotSharp.Abstraction.Evaluations;
 using BotSharp.Abstraction.Settings;
-using BotSharp.Core.Evaluatings;
 using Microsoft.Extensions.Configuration;
+using BotSharp.Core.Evaluations.Services;
 
 namespace BotSharp.Core.Evaluations;
 
@@ -21,7 +21,6 @@ public void RegisterDI(IServiceCollection services, IConfiguration config)
             return settingService.Bind<EvaluatorSetting>("Evaluator");
         });
 
-        services.AddScoped<IConversationHook, EvaluationConversationHook>();
         services.AddScoped<IEvaluatingService, EvaluatingService>();
         services.AddScoped<IExecutionLogger, ExecutionLogger>();
     }

src/Infrastructure/BotSharp.Core/Evaluations/Services/EvaluatingService.Evaluate.cs

@@ -0,0 +1,93 @@
+using BotSharp.Abstraction.Evaluations.Models;
+using BotSharp.Abstraction.Instructs;
+using BotSharp.Abstraction.Instructs.Models;
+
+namespace BotSharp.Core.Evaluations.Services;
+
+public partial class EvaluatingService
+{
+    public async Task<EvaluationResult> Evaluate(string conversationId, EvaluationRequest request)
+    {
+        var storage = _services.GetRequiredService<IConversationStorage>();
+        var refDialogs = storage.GetDialogs(request.RefConversationId);
+        var refDialogContents = GetConversationContent(refDialogs);
+
+        var initDialog = refDialogs.FirstOrDefault(x => x.Role == AgentRole.User);
+        var initMessage = initDialog?.RichContent?.Message?.Text ?? initDialog?.Content ?? "Hello";
+
+        var generatedConvId = await SimulateConversation(initMessage, refDialogContents, request);
+
+        return new EvaluationResult
+        {
+            GeneratedConversationId = generatedConvId
+        };
+    }
+
+    private async Task<string> SimulateConversation(string initMessage, IEnumerable<string> refConversation, EvaluationRequest request)
+    {
+        var count = 0;
+        var curConvId = Guid.NewGuid().ToString();
+        var curConversation = new List<string>();
+        var curMessage = initMessage;
+
+        var storage = _services.GetRequiredService<IConversationStorage>();
+        var agentService = _services.GetRequiredService<IAgentService>();
+        var instructService = _services.GetRequiredService<IInstructService>();
+
+        var query = "Please take yourself as a user and follow the instruction to generate a message in the user tone.";
+        var targetAgentId = request.AgentId;
+        var evaluatorAgent = await agentService.GetAgent(BuiltInAgentId.Evaluator);
+        var simulatorPrompt = evaluatorAgent.Templates.FirstOrDefault(x => x.Name == "instruction.simulator")?.Content ?? string.Empty;
+
+        while (true)
+        {
+            curConversation.Add($"{AgentRole.User}: {curMessage}");
+            var dialog = await SendMessage(targetAgentId, curConvId, curMessage);
+            var botMessage = dialog?.RichContent?.Message?.Text ?? dialog?.Content ?? string.Empty;
+            curConversation.Add($"{AgentRole.Assistant}: {botMessage}");
+            count++;
+
+            var result = await instructService.Instruct<SimulationResult>(simulatorPrompt, BuiltInAgentId.Evaluator,
+                            new InstructOptions
+                            {
+                                Provider = request.Provider,
+                                Model = request.Model,
+                                Message = query,
+                                Data = new Dictionary<string, object>
+                                {
+                                    { "ref_conversation", refConversation },
+                                    { "cur_conversation", curConversation },
+                                }
+                            });
+
+            if (count > request.MaxRounds || (result != null && result.Stop))
+            {
+                break;
+            }
+
+            curMessage = result?.GeneratedMessage ?? string.Empty;
+        }
+
+        return curConvId;
+    }
+
+    private IEnumerable<string> GetConversationContent(IEnumerable<RoleDialogModel> dialogs)
+    {
+        var contents = new List<string>();
+
+        foreach (var dialog in dialogs)
+        {
+            var role = dialog.Role;
+            if (role == AgentRole.Function) continue;
+
+            if (role != AgentRole.User)
+            {
+                role = AgentRole.Assistant;
+            }
+
+            contents.Add($"{role}: {dialog.RichContent?.Message?.Text ?? dialog.Content}");
+        }
+
+        return contents;
+    }
+}

src/Infrastructure/BotSharp.Core/Evaluations/EvaluatingService.cs → src/Infrastructure/BotSharp.Core/Evaluations/Services/EvaluatingService.cs

@@ -4,17 +4,22 @@
 using BotSharp.Abstraction.Evaluations.Settings;
 using BotSharp.Abstraction.Models;
 using BotSharp.Abstraction.Templating;
-using System.Drawing;
 
-namespace BotSharp.Core.Evaluatings;
+namespace BotSharp.Core.Evaluations.Services;
 
-public class EvaluatingService : IEvaluatingService
+public partial class EvaluatingService : IEvaluatingService
 {
     private readonly IServiceProvider _services;
+    private readonly ILogger<EvaluatingService> _logger;
     private readonly EvaluatorSetting _settings;
-    public EvaluatingService(IServiceProvider services, EvaluatorSetting settings)
+
+    public EvaluatingService(
+        IServiceProvider services,
+        ILogger<EvaluatingService> logger,
+        EvaluatorSetting settings)
     {
         _services = services;
+        _logger = logger;
         _settings = settings;
     }
 
@@ -81,11 +86,6 @@ public async Task<Conversation> Execute(string task, EvaluationRequest request)
         return conv;
     }
 
-    public async Task<EvaluationResult> Evaluate(string conversationId, EvaluationRequest request)
-    {
-        throw new NotImplementedException();
-    }
-
     private async Task<RoleDialogModel> SendMessage(string agentId, string conversationId, string text)
     {
         var conv = _services.GetRequiredService<IConversationService>();

src/Infrastructure/BotSharp.Core/data/agents/dfd9b46d-d00c-40af-8a75-3fbdc2b89869/templates/instruction.metrics.liquid

@@ -0,0 +1 @@
+You are a conversation evaluator.

src/Infrastructure/BotSharp.Core/data/agents/dfd9b46d-d00c-40af-8a75-3fbdc2b89869/templates/instruction.simulator.liquid

@@ -0,0 +1,30 @@
+You are a conversaton simulator.
+Please take the content in the [REFERENCE CONVERSATION] section as a reference, and focus on the [ONGOING CONVERSATION] section to generate a message based on the context.
+
+** You need to take a close look at the content in both [REFERENCE CONVERSATION] and [ONGOING CONVERSATION], and determine whether to generate a text message or stop the ongoing conversation.
+** When you generate a message, please assume you as the user and reply as the user perceptive.
+** Please do not generate or append messages with similar meaning that you have already mentioned in the [ONGOING CONVERSATION].
+** If you see the assistant replies two or more than two similar messages in the [ONGOING CONVERSATION], please stop the [ONGOING CONVERSATION] immediately and turn the "stop" to true in the output json.
+** The output must be in JSON format:
+{
+    "generated_message": the generated text message as the user tone,
+    "stop": a boolean to indicate whether to stop the [ONGOING CONVERSATION],
+    "reason": the reason why you generate the message or stop the [ONGOING CONVERSATION]
+}
+
+
+=================
+[REFERENCE CONVERSATION]
+
+{% for text in ref_conversation -%}
+{{ text }}{{ "\r\n" }}
+{%- endfor %}
+
+
+
+=================
+[ONGOING CONVERSATION]
+
+{% for text in cur_conversation -%}
+{{ text }}{{ "\r\n" }}
+{%- endfor %}