Merge pull request SciSharp#1074 from Joannall/master

Refactor SQL Driver Plugin and Update Database Handling

Author: Oceania2018

src/Plugins/BotSharp.Plugin.SqlDriver/BotSharp.Plugin.SqlDriver.csproj

@@ -108,4 +108,8 @@
     <ProjectReference Include="..\..\Infrastructure\BotSharp.Core\BotSharp.Core.csproj" />
   </ItemGroup>
 
+  <ItemGroup>
+    <Folder Include="Helpers\" />
+  </ItemGroup>
+
 </Project>

src/Plugins/BotSharp.Plugin.SqlDriver/Helpers/SqlDriverHelper.cs

@@ -4,12 +4,18 @@ namespace BotSharp.Plugin.SqlDriver.Models;
 
 public class SqlStatement
 {
+    [JsonPropertyName("db_provider")]
+    public string DBProvider { get; set; } = null!;
+
     [JsonPropertyName("sql_statement")]
     public string Statement { get; set; } = null!;
 
     [JsonPropertyName("reason")]
     public string Reason { get; set; } = null!;
 
+    [JsonPropertyName("schema")]
+    public string Schema { get; set; } = null!;
+
     [JsonPropertyName("tables")]
     public string[] Tables { get; set; } = null!;
 

src/Plugins/BotSharp.Plugin.SqlDriver/Hooks/SqlDriverAgentHook.cs

@@ -27,7 +27,6 @@ public void RegisterDI(IServiceCollection services, IConfiguration config)
         services.AddScoped<DbKnowledgeService>();
         services.AddScoped<IPlanningHook, SqlDriverPlanningHook>();
         services.AddScoped<IKnowledgeHook, SqlDriverKnowledgeHook>();
-        services.AddScoped<IAgentHook, SqlDriverAgentHook>();
         services.AddScoped<IConversationHook, SqlDriverConversationHook>();
         services.AddScoped<IAgentUtilityHook, SqlUtilityHook>();
         services.AddScoped<ICrontabHook, SqlDriverCrontabHook>();

src/Plugins/BotSharp.Plugin.SqlDriver/Models/SqlStatement.cs

@@ -30,5 +30,4 @@
 global using BotSharp.Plugin.SqlDriver.Hooks;
 global using BotSharp.Plugin.SqlDriver.Services;
 global using BotSharp.Plugin.SqlDriver.Interfaces;
-global using BotSharp.Plugin.SqlDriver.Helpers;
 global using BotSharp.Plugin.SqlDriver.Settings;

src/Plugins/BotSharp.Plugin.SqlDriver/SqlDriverPlugin.cs

@@ -23,16 +23,16 @@ public async Task<bool> Execute(RoleDialogModel message)
     {
         var args = JsonSerializer.Deserialize<SqlStatement>(message.FunctionArgs);
         var tables = args.Tables;
+        var dbType = args.DBProvider;
+        var schema = args.Schema;
         var agentService = _services.GetRequiredService<IAgentService>();
-        var dbHook = _services.GetRequiredService<ISqlDriverHook>();
-        var dbType = dbHook.GetDatabaseType(message);
 
         // Get table DDL from database
         var tableDdls = dbType switch
         {
             "mysql" => GetDdlFromMySql(tables),
             "sqlserver" => GetDdlFromSqlServer(tables),
-            "redshift" => GetDdlFromRedshift(tables),
+            "redshift" => GetDdlFromRedshift(tables,schema),
             _ => throw new NotImplementedException($"Database type {dbType} is not supported.")
         };
 
@@ -127,7 +127,7 @@ FROM INFORMATION_SCHEMA.COLUMNS
         return tableDdls;
     }
 
-    private List<string> GetDdlFromRedshift(string[] tables)
+    private List<string> GetDdlFromRedshift(string[] tables, string schema)
     {
         var settings = _services.GetRequiredService<SqlDriverSetting>();
         var tableDdls = new List<string>();

src/Plugins/BotSharp.Plugin.SqlDriver/Using.cs

@@ -25,9 +25,7 @@ public async Task<bool> Execute(RoleDialogModel message)
             return false;
         }
 
-        // check if need to instantely
-        var dbHook = _services.GetRequiredService<ISqlDriverHook>();
-        var dbType = dbHook.GetDatabaseType(message);
+        var dbType = args.DBProvider.ToLowerInvariant();
 
         var result = dbType switch
         {

src/Plugins/BotSharp.Plugin.SqlDriver/UtilFunctions/GetTableDefinitionFn.cs

@@ -4,6 +4,16 @@
   "parameters": {
     "type": "object",
     "properties": {
+      "db_provider": {
+        "type": "string",
+        "enum": [
+          "mysql",
+          "postgresql",
+          "mssql",
+          "redshift"
+        ],
+        "description": "The database engine."
+      },
       "sql_statement": {
         "type": "string",
         "description": "SQL statement with SELECT"
@@ -55,6 +65,6 @@
         "required": [ "name", "value" ]
       }
     },
-    "required": [ "sql_statement", "reason", "table", "parameters", "return_field" ]
+    "required": [ "db_provider", "sql_statement", "reason", "table", "parameters", "return_field" ]
   }
 }

src/Plugins/BotSharp.Plugin.SqlDriver/UtilFunctions/SqlSelect.cs

@@ -4,6 +4,24 @@
   "parameters": {
     "type": "object",
     "properties": {
+      "db_provider": {
+        "type": "string",
+        "enum": [
+          "mysql",
+          "postgresql",
+          "mssql",
+          "redshift"
+        ],
+        "description": "The database engine."
+      },
+      "schema": {
+        "type": "string",
+        "description": "schema name for tables. Typically, the part before the dot is the schema name, e.g.smsonebi.affiliate_profile, schema name is smsonebi.",
+        "items": {
+          "type": "string",
+          "description": "schema name"
+        }
+      },
       "tables": {
         "type": "array",
         "description": "table name in planning steps",
@@ -17,6 +35,6 @@
         "description": "the reason why you need to call sql_table_definition"
       }
     },
-    "required": [ "tables", "reason" ]
+    "required": [ "db_provider", "schema", "tables", "reason" ]
   }
 }

src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-db-sql_select.json

@@ -1,6 +1,9 @@
-You are connecting to {{ db_type }} database. You can run provided SQL statements by following {{ db_type }} rules.
-
-Please call function util-db-sql_select if user wants to get or retrieve data from data tables.
-If there are any parameters, please add them in the WHERE clause, each of which starts with "@".
-Avoid returning the entire record and only return the fields you need.
-For example, SELECT Id FROM table WHERE Id=@Id AND Name=@Name
+Please call function util-db-sql_select if user wants to retrieve data from database tables.
+- Ensure you obtain the exact provider, schema, table and column names.
+    If user provide them, directly call util-db-sql_select to execute the query.
+    If user didn't provide the provider(e.g. mysql, redshift), schema or table name, ask user to provide.
+    If the user doesn't provide exact column name, try using other available functions to infer the information.
+    If that fails or no function can use, prompt the user to provide the missing information.
+- If there are any parameters, please add them in the WHERE clause, each of which starts with "@".
+- Avoid returning the entire record and only return the fields you need.
+    For example, SELECT Id FROM table WHERE Id=@Id AND Name=@Name

src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-db-sql_table_definition.json

@@ -4,8 +4,6 @@ Rules to call util-db-verify_dictionary_term:
 2. You must return the id and name/code for existing dictionary. 
 3. Call the function only if need_lookup_dictionary is true.
 
-You are connecting to {{ db_type }} database. You can run provided SQL statements by following {{ db_type }} rules.
-
 The dictionary table is identified by a name that begins with "data_".
 You are only allowed to query the dictionary table without joining it with other non-dictionary tables.