Update dictionary look up

src/Plugins/BotSharp.Plugin.Planner/BotSharp.Plugin.Planner.csproj

@@ -16,6 +16,7 @@
     <None Remove="data\agents\282a7128-69a1-44b0-878c-a9159b88f3b9\functions\plan_secondary_stage.json" />
     <None Remove="data\agents\282a7128-69a1-44b0-878c-a9159b88f3b9\functions\plan_summary.json" />
     <None Remove="data\agents\282a7128-69a1-44b0-878c-a9159b88f3b9\instructions\instruction.liquid" />
+    <None Remove="data\agents\282a7128-69a1-44b0-878c-a9159b88f3b9\templates\database.dictionary.sql.liquid" />
     <None Remove="data\agents\282a7128-69a1-44b0-878c-a9159b88f3b9\templates\database.summarize.MySql.liquid" />
     <None Remove="data\agents\282a7128-69a1-44b0-878c-a9159b88f3b9\templates\database.summarize.SqlServer.liquid" />
     <None Remove="data\agents\282a7128-69a1-44b0-878c-a9159b88f3b9\templates\two_stage.2nd.plan.liquid" />
@@ -49,6 +50,9 @@
     <Content Include="data\agents\282a7128-69a1-44b0-878c-a9159b88f3b9\templates\database.summarize.sqlserver.liquid">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </Content>
+    <Content Include="data\agents\282a7128-69a1-44b0-878c-a9159b88f3b9\templates\database.dictionary.sql.liquid">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content>
     <Content Include="data\agents\282a7128-69a1-44b0-878c-a9159b88f3b9\templates\two_stage.next.liquid">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </Content>

src/Plugins/BotSharp.Plugin.Planner/Functions/SecondaryStagePlanFn.cs

@@ -21,34 +21,26 @@ public async Task<bool> Execute(RoleDialogModel message)
         var agentService = _services.GetRequiredService<IAgentService>();
         var knowledgeService = _services.GetRequiredService<IKnowledgeService>();
         var knowledgeSettings = _services.GetRequiredService<KnowledgeBaseSettings>();
-
+        var states = _services.GetRequiredService<IConversationStateService>();
+
         var msgSecondary = RoleDialogModel.From(message);
-        var taskPrimary = JsonSerializer.Deserialize<PrimaryRequirementRequest>(message.FunctionArgs);
         var collectionName = knowledgeSettings.Default.CollectionName ?? KnowledgeCollectionName.BotSharp;
-
-        msgSecondary.FunctionArgs = JsonSerializer.Serialize(new SecondaryBreakdownTask
-        {
-            TaskDescription = taskPrimary.Requirements
-        });
+        var planPrimary = states.GetState("planning_result");
+        var taskPrimary = states.GetState("requirement_detail");
 
         var taskSecondary = JsonSerializer.Deserialize<SecondaryBreakdownTask>(msgSecondary.FunctionArgs);
-        var items = msgSecondary.Content.JsonArrayContent<FirstStagePlan>();
-
+
         // Search knowledgebase
-        foreach (var item in items)
+        var knowledges = await knowledgeService.SearchVectorKnowledge(taskSecondary.SolutionQuestion, collectionName, new VectorSearchOptions
         {
-            if (!item.NeedAdditionalInformation) continue;
-
-            var knowledges = await knowledgeService.SearchVectorKnowledge(item.Task, collectionName, new VectorSearchOptions
-            {
-                Confidence = 0.6f
-            });
-            message.Content += string.Join("\r\n\r\n=====\r\n", knowledges.Select(x => x.ToQuestionAnswer()));
-        }
+            Confidence = 0.6f
+        });
+        var knowledgeResults = "";
+        knowledgeResults = string.Join("\r\n\r\n=====\r\n", knowledges.Select(x => x.ToQuestionAnswer()));
 
         // Get second stage planning prompt
         var currentAgent = await agentService.LoadAgent(message.CurrentAgentId);
-        var secondPlanningPrompt = await GetSecondStagePlanPrompt(taskSecondary.TaskDescription, message);
+        var secondPlanningPrompt = await GetSecondStagePlanPrompt(taskSecondary.TaskDescription, planPrimary, knowledgeResults, message);
         _logger.LogInformation(secondPlanningPrompt);
 
         var plannerAgent = new Agent
@@ -64,12 +56,11 @@ public async Task<bool> Execute(RoleDialogModel message)
         message.Content = response.Content;
         _logger.LogInformation(response.Content);
 
-        var states = _services.GetRequiredService<IConversationStateService>();
         states.SetState("planning_result", response.Content);
         return true;
     }
 
-    private async Task<string> GetSecondStagePlanPrompt(string taskDescription, RoleDialogModel message)
+    private async Task<string> GetSecondStagePlanPrompt(string taskDescription, string planPrimary, string knowledgeResults, RoleDialogModel message)
     {
         var agentService = _services.GetRequiredService<IAgentService>();
         var render = _services.GetRequiredService<ITemplateRender>();
@@ -85,7 +76,8 @@ private async Task<string> GetSecondStagePlanPrompt(string taskDescription, Role
         return render.Render(template, new Dictionary<string, object>
         {
             { "task_description", taskDescription },
-            { "primary_plan", new[]{ message.Content } },
+            { "primary_plan", planPrimary },
+            { "additional_knowledge", knowledgeResults },
             { "response_format",  responseFormat }
         });
     }

src/Plugins/BotSharp.Plugin.Planner/Functions/SummaryPlanFn.cs

@@ -35,7 +35,7 @@ public async Task<bool> Execute(RoleDialogModel message)
         var allTables = new List<string>();
         var ddlStatements = "";
         var relevantKnowledge = states.GetState("planning_result");
-        relevantKnowledge += states.GetState("dictionary_items");
+        var dictionaryItems = states.GetState("dictionary_items");
 
         foreach (var step in steps)
         {
@@ -49,12 +49,12 @@ public async Task<bool> Execute(RoleDialogModel message)
             { 
                 table = table,
             });
-            await fn.InvokeFunction("get_table_definition", msgCopy);
+            await fn.InvokeFunction("sql_table_definition", msgCopy);
             ddlStatements += "\r\n" + msgCopy.Content;
         }
 
         // Summarize and generate query
-        var summaryPlanPrompt = await GetSummaryPlanPrompt(taskRequirement, relevantKnowledge, ddlStatements);
+        var summaryPlanPrompt = await GetSummaryPlanPrompt(taskRequirement, relevantKnowledge, dictionaryItems, ddlStatements);
         _logger.LogInformation($"Summary plan prompt:\r\n{summaryPlanPrompt}");
 
         var plannerAgent = new Agent
@@ -74,7 +74,7 @@ await HookEmitter.Emit<IPlanningHook>(_services, x =>
         return true;
     }
 
-    private async Task<string> GetSummaryPlanPrompt(string taskDescription, string relevantKnowledge, string ddlStatement)
+    private async Task<string> GetSummaryPlanPrompt(string taskDescription, string relevantKnowledge, string dictionaryItems, string ddlStatement)
     {
         var agentService = _services.GetRequiredService<IAgentService>();
         var render = _services.GetRequiredService<ITemplateRender>();
@@ -94,6 +94,7 @@ await HookEmitter.Emit<IPlanningHook>(_services, async x =>
             { "task_description", taskDescription },
             { "summary_requirements", string.Join("\r\n",additionalRequirements) },
             { "relevant_knowledges", relevantKnowledge },
+            { "dictionary_items", dictionaryItems },
             { "table_structure", ddlStatement },
         });
     }

src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/functions/plan_primary_stage.json

@@ -13,7 +13,7 @@
         "description": "User requirements in detail, don't miss any information especially for those line items, values and numbers.",
         "items": {
           "type": "string",
-          "description": "Question converted from requirement in different ways to search in the knowledge base, be short"
+          "description": "Question converted from requirement in different ways to search in the knowledge base, be short and you can refer to the global knowledge."
         }
       }
     },

src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/database.dictionary.sql.liquid

@@ -0,0 +1,10 @@
+You are a sql statement corrector. You will need to refer to the table structure and rewrite the original sql statement so it's using the correct information, e.g. column name.
+Output the sql statement only without comments, in JSON format: {{ response_format }}
+
+=====
+Original Sql:
+{{ original_sql }}
+
+=====
+Table Structure:
+{{ table_structure }}

src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/database.summarize.mysql.liquid

@@ -18,3 +18,4 @@ For example, you should use SET @id = select max(id) from table;
 *** the generated sql query MUST be basedd on the provided table structure. ***
 *** All queries return a maximum of 20 records. ***
 *** Only select user friendly columns. ***
+*** Try to use id instead of string in where clause if you have the dictionary. ***

src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/two_stage.1st.plan.liquid

@@ -8,7 +8,7 @@ Thinking process:
     - If there is extra knowledge or relationship needed between steps, set the need_additional_information to true for both steps.
     - If the solution mentioned "related solutions" is needed, set the need_additional_information to true.
     - You should find the relationships between data structure based on the task knowledge strictly. If lack of information, set the need_additional_information to true.
-    - If you need to verify or get the enum/term/dictionary value, set the need_additional_information to true.
+    - If you need to lookup the dictionary to verify or get the enum/term/dictionary value, set the need_additional_information to true.
 3. Input argument must reference to corresponding variable name that retrieved by previous steps, variable name must start with '@';
 4. Output all the subtasks as much detail as possible in JSON: [{{ response_format }}]
 5. You can NOT generate the final query before calling function plan_summary.

src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/two_stage.2nd.plan.liquid

@@ -11,8 +11,13 @@ Additional Requirements:
 * "output_results" is variable name that needed to be used in the next step.
 
 =====
-TASK: {{ task_description }}
+Sub Task Description: 
+{{ task_description }}
 
 =====
 Primary Planning: 
-{{ primary_plan }}
+{{ primary_plan }}
+
+=====
+Additional Knowledge: 
+{{ additional_knowledge }}

src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/two_stage.summarize.liquid

@@ -11,6 +11,10 @@ Task description:
 Relevant Knowledges:
 {{ relevant_knowledges }}
 
+=====
+Dictionary Items:
+{{ dictionary_items }}
+
 =====
 Table Structure:
 {{ table_structure }}

src/Plugins/BotSharp.Plugin.SqlDriver/BotSharp.Plugin.SqlDriver.csproj

@@ -17,32 +17,34 @@
   </ItemGroup>
 
   <ItemGroup>
-    <None Remove="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\functions\get_table_definition.json" />
     <None Remove="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\functions\sql_dictionary_lookup.json" />
     <None Remove="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\functions\sql_select.json" />
-    <None Remove="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\get_table_definition.fn.liquid" />
+    <None Remove="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\functions\sql_table_definition.json" />
     <None Remove="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\sql_dictionary_lookup.fn.liquid" />
     <None Remove="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\sql_executor.fn.liquid" />
+    <None Remove="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\sql_table_definition.fn.liquid" />
     <None Remove="data\agents\beda4c12-e1ec-4b4b-b328-3df4a6687c4f\agent.json" />
     <None Remove="data\agents\beda4c12-e1ec-4b4b-b328-3df4a6687c4f\functions\execute_sql.json" />
     <None Remove="data\agents\beda4c12-e1ec-4b4b-b328-3df4a6687c4f\functions\lookup_dictionary.json" />
     <None Remove="data\agents\beda4c12-e1ec-4b4b-b328-3df4a6687c4f\functions\sql_insert.json" />
     <None Remove="data\agents\beda4c12-e1ec-4b4b-b328-3df4a6687c4f\functions\sql_select.json" />
     <None Remove="data\agents\beda4c12-e1ec-4b4b-b328-3df4a6687c4f\instructions\instruction.liquid" />
-    <None Remove="data\agents\beda4c12-e1ec-4b4b-b328-3df4a6687c4f\templates\sql_dictionary_lookup.liquid" />
   </ItemGroup>
 
   <ItemGroup>
+    <Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\functions\sql_table_definition.json">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content>
     <Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\functions\sql_dictionary_lookup.json">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </Content>
     <Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\sql_dictionary_lookup.fn.liquid">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </Content>
-    <Content Include="data\agents\beda4c12-e1ec-4b4b-b328-3df4a6687c4f\functions\get_table_definition.json">
+    <Content Include="data\agents\beda4c12-e1ec-4b4b-b328-3df4a6687c4f\functions\sql_table_definition.json">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </Content>
-    <Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\get_table_definition.fn.liquid">
+    <Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\sql_table_definition.fn.liquid">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </Content>
     <Content Include="data\agents\beda4c12-e1ec-4b4b-b328-3df4a6687c4f\agent.json">
@@ -54,9 +56,6 @@
     <Content Include="data\agents\beda4c12-e1ec-4b4b-b328-3df4a6687c4f\instructions\instruction.liquid">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </Content>
-    <Content Include="data\agents\beda4c12-e1ec-4b4b-b328-3df4a6687c4f\templates\sql_dictionary_lookup.liquid">
-      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
-    </Content>
     <Content Include="data\agents\beda4c12-e1ec-4b4b-b328-3df4a6687c4f\functions\sql_insert.json">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </Content>
@@ -81,4 +80,8 @@
     <ProjectReference Include="..\..\Infrastructure\BotSharp.Core\BotSharp.Core.csproj" />
   </ItemGroup>
 
+  <ItemGroup>
+    <Folder Include="data\agents\beda4c12-e1ec-4b4b-b328-3df4a6687c4f\templates\" />
+  </ItemGroup>
+
 </Project>

src/Plugins/BotSharp.Plugin.SqlDriver/Enum/Utility.cs

@@ -4,4 +4,5 @@ public  class Utility
 {
     public const string SqlExecutor = "sql-executor";
     public const string SqlDictionaryLookup = "sql-dictionary-lookup";
+    public const string SqlTableDefinition = "sql-table-definition";
 }

src/Plugins/BotSharp.Plugin.SqlDriver/Functions/GetTableDefinitionFn.cs

@@ -8,7 +8,7 @@ namespace BotSharp.Plugin.SqlDriver.Functions;
 
 public class GetTableDefinitionFn : IFunctionCallback
 {
-    public string Name => "get_table_definition";
+    public string Name => "sql_table_definition";
     public string Indication => "Obtain the relevant data structure definitions.";
     private readonly IServiceProvider _services;
     private readonly ILogger<GetTableDefinitionFn> _logger;
@@ -38,6 +38,9 @@ public async Task<bool> Execute(RoleDialogModel message)
 
         message.Content = string.Join("\r\n\r\n", tableDdls);
 
+        //var states = _services.GetRequiredService<IConversationStateService>();
+        //states.SetState($"table_definition_{args.Table}", message.Content);
+
         return true;
     }
 

src/Plugins/BotSharp.Plugin.SqlDriver/Functions/LookupDictionaryFn.cs

@@ -1,10 +1,14 @@
 using Azure;
 using BotSharp.Abstraction.Agents.Enums;
 using BotSharp.Abstraction.MLTasks;
+using BotSharp.Abstraction.Routing;
+using BotSharp.Core.Agents.Services;
 using BotSharp.Core.Infrastructures;
 using BotSharp.Plugin.SqlDriver.Models;
 using MySqlConnector;
+using System.Text.RegularExpressions;
 using static Dapper.SqlMapper;
+using static System.Net.Mime.MediaTypeNames;
 
 namespace BotSharp.Plugin.SqlDriver.Functions;
 
@@ -22,6 +26,26 @@ public async Task<bool> Execute(RoleDialogModel message)
     {
         var args = JsonSerializer.Deserialize<LookupDictionary>(message.FunctionArgs);
 
+        // get table DDL
+        var fn = _services.GetRequiredService<IRoutingService>();
+        var msgCopy = RoleDialogModel.From(message);
+        await fn.InvokeFunction("sql_table_definition", msgCopy);
+
+        // refine SQL
+        var agentService = _services.GetRequiredService<IAgentService>();
+        var currentAgent = await agentService.LoadAgent(message.CurrentAgentId);
+        var dictionarySqlPrompt = await GetDictionarySQLPrompt(args.SqlStatement, msgCopy.Content);
+        var plannerAgent = new Agent
+        {
+            Id = string.Empty,
+            Name = "sqlDriver_DictionarySearch",
+            Instruction = dictionarySqlPrompt,
+            TemplateDict = new Dictionary<string, object>(),
+            LlmConfig = currentAgent.LlmConfig
+        };
+        var response = await GetAiResponse(plannerAgent);
+        args = JsonSerializer.Deserialize<LookupDictionary>(response.Content);
+
         // check if need to instantely
         var settings = _services.GetRequiredService<SqlDriverSetting>();
         using var connection = new MySqlConnection(settings.MySqlExecutionConnectionString);
@@ -37,9 +61,37 @@ public async Task<bool> Execute(RoleDialogModel message)
         }
         var states = _services.GetRequiredService<IConversationStateService>();
         var dictionaryItems = states.GetState("dictionary_items", "");
-        dictionaryItems += "\r\n\r\n" +  args.Reason + ":\r\n" + message.Content + "\r\n";
+        dictionaryItems += "\r\n\r\n" + args.Table + ":\r\n" + args.Reason + ":\r\n" + message.Content + "\r\n";
         states.SetState("dictionary_items", dictionaryItems);
 
         return true;
     }
+    private async Task<string> GetDictionarySQLPrompt(string originalSql, string tableStructure)
+    {
+        var agentService = _services.GetRequiredService<IAgentService>();
+        var render = _services.GetRequiredService<ITemplateRender>();
+        var knowledgeHooks = _services.GetServices<IKnowledgeHook>();
+
+        var agent = await agentService.GetAgent(BuiltInAgentId.Planner);
+        var template = agent.Templates.FirstOrDefault(x => x.Name == "database.dictionary.sql")?.Content ?? string.Empty;
+        var responseFormat = JsonSerializer.Serialize(new LookupDictionary{ });
+
+        return render.Render(template, new Dictionary<string, object>
+        {
+            { "original_sql", originalSql },
+            { "table_structure", tableStructure },
+            { "response_format", responseFormat }
+        });
+    }
+    private async Task<RoleDialogModel> GetAiResponse(Agent plannerAgent)
+    {
+        var text = "Check and correct the SQL statement.";
+        var message = new RoleDialogModel(AgentRole.User, text);
+
+        var completion = CompletionProvider.GetChatCompletion(_services,
+            provider: plannerAgent.LlmConfig.Provider,
+            model: plannerAgent.LlmConfig.Model);
+
+        return await completion.GetChatCompletions(plannerAgent, new List<RoleDialogModel> { message });
+    }
 }