Merge pull request #865 from hchen2020/master

WebUtility improvement

Author: Oceania2018

src/Infrastructure/BotSharp.Abstraction/Browsing/Models/ElementActionArgs.cs

@@ -6,8 +6,11 @@ namespace BotSharp.Abstraction.Browsing.Models;
 [DebuggerStepThrough]
 public class ElementActionArgs
 {
+    [JsonPropertyName("action")]
+    [JsonConverter(typeof(JsonStringEnumConverter))]
     public BroswerActionEnum Action { get; set; }
 
+    [JsonPropertyName("content")]
     public string? Content { get; set; }
 
     public ElementPosition? Position { get; set; }
@@ -16,6 +19,8 @@ public class ElementActionArgs
     /// Delay milliseconds before pressing key
     /// </summary>
     public int DelayBeforePressingKey { get; set; }
+
+    [JsonPropertyName("press_key")]
     public string? PressKey { get; set; }
 
     /// <summary>

src/Plugins/BotSharp.Plugin.SqlDriver/Enum/UtilityName.cs

@@ -14,21 +14,19 @@ public void AddUtilities(List<AgentUtility> utilities)
         {
             new AgentUtility
             {
-                Name = UtilityName.SqlTableDefinition,
-                Functions = [new(SQL_TABLE_DEFINITION_FN)],
-                Templates = [new($"{SQL_TABLE_DEFINITION_FN}.fn")]
-            },
-            new AgentUtility
-            {
-                Name = UtilityName.SqlDictionaryLookup,
-                Functions = [new(VERIFY_DICTIONARY_TERM_FN)],
-                Templates = [new($"{VERIFY_DICTIONARY_TERM_FN}.fn")]
-            },
-            new AgentUtility
-            {
-                Name = UtilityName.SqlExecutor,
-                Functions = [new(SQL_SELECT_FN), new(SQL_TABLE_DEFINITION_FN)],
-                Templates = [new($"{SQL_EXECUTOR_FN}.fn")]
+                Name = "db.tools",
+                Functions = 
+                [
+                    new(SQL_TABLE_DEFINITION_FN),
+                    new(VERIFY_DICTIONARY_TERM_FN),
+                    new(SQL_SELECT_FN),
+                ],
+                Templates = 
+                [
+                    new($"{VERIFY_DICTIONARY_TERM_FN}.fn"),
+                    new($"{SQL_TABLE_DEFINITION_FN}.fn"),
+                    new($"{SQL_EXECUTOR_FN}.fn")
+                ]
             }
         };
 

src/Plugins/BotSharp.Plugin.SqlDriver/Hooks/SqlUtilityHook.cs

@@ -23,7 +23,6 @@
 global using BotSharp.Abstraction.Settings;
 global using BotSharp.Plugin.SqlDriver.Hooks;
 global using BotSharp.Plugin.SqlDriver.Services;
-global using BotSharp.Plugin.SqlDriver.Enum;
 global using BotSharp.Abstraction.Agents.Enums;
 global using BotSharp.Abstraction.Instructs;
 global using BotSharp.Abstraction.Instructs.Models;

src/Plugins/BotSharp.Plugin.SqlDriver/Using.cs

@@ -38,6 +38,21 @@
   </ItemGroup>
 
   <ItemGroup>
+    <Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\functions\util-web-action_on_element.json">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content>
+    <Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\functions\util-web-close_browser.json">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content>
+    <Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\functions\util-web-go_to_page.json">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content>
+    <Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\util-web-action_on_element.fn.liquid">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content>
+    <Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\util-web-go_to_page.fn.liquid">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content>
     <Content Include="data\agents\f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b\agent.json">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </Content>

src/Plugins/BotSharp.Plugin.WebDriver/BotSharp.Plugin.WebDriver.csproj

@@ -33,6 +33,10 @@ public void SetServiceProvider(IServiceProvider services)
 
     public async Task<IBrowserContext> GetContext(string ctxId)
     {
+        if (!_contexts.ContainsKey(ctxId))
+        {
+            await InitContext(ctxId, new BrowserActionArgs());
+        }
         return _contexts[ctxId];
     }
 
@@ -235,6 +239,11 @@ public async Task Close(string ctxId)
 
     public async Task CloseCurrentPage(string ctxId)
     {
+        if (!_pages.ContainsKey(ctxId))
+        {
+            return;
+        }
+
         var pages = _pages[ctxId].ToArray();
         for (var i = 0; i < pages.Length; i++)
         {

src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs

@@ -0,0 +1,31 @@
+namespace BotSharp.Plugin.WebDriver.Hooks;
+
+public class WebUtilityHook : IAgentUtilityHook
+{
+    private const string PREFIX = "util-web-";
+    private const string GO_TO_PAGE_FN = $"{PREFIX}go_to_page";
+    private const string ACTION_ON_ELEMENT_FN = $"{PREFIX}action_on_element";
+
+    public void AddUtilities(List<AgentUtility> utilities)
+    {
+        var items = new List<AgentUtility>
+        {
+            new AgentUtility
+            {
+                Name = "web.tools",
+                Functions = 
+                [
+                    new(GO_TO_PAGE_FN),
+                    new(ACTION_ON_ELEMENT_FN)
+                ],
+                Templates = 
+                [
+                    new($"{GO_TO_PAGE_FN}.fn"),
+                    new($"{ACTION_ON_ELEMENT_FN}.fn")
+                ]
+            }
+        };
+
+        utilities.AddRange(items);
+    }
+}

src/Plugins/BotSharp.Plugin.WebDriver/Hooks/WebUtilityHook.cs

@@ -0,0 +1,42 @@
+namespace BotSharp.Plugin.WebDriver.UtilFunctions;
+
+public class UtilWebActionOnElementFn : IFunctionCallback
+{
+    public string Name => "util-web-action_on_element";
+    public string Indication => "Do action on element.";
+    private readonly IServiceProvider _services;
+    private readonly ILogger _logger;
+
+    public UtilWebActionOnElementFn(
+        IServiceProvider services,
+        ILogger<UtilWebActionOnElementFn> logger)
+    {
+        _services = services;
+        _logger = logger;
+    }
+
+    public async Task<bool> Execute(RoleDialogModel message)
+    {
+        var locatorArgs = JsonSerializer.Deserialize<ElementLocatingArgs>(message.FunctionArgs);
+        var actionArgs = JsonSerializer.Deserialize<ElementActionArgs>(message.FunctionArgs);
+        if (actionArgs.Action == BroswerActionEnum.InputText)
+        {
+            actionArgs.PressKey = "Enter";
+        }
+        actionArgs.WaitTime = 2;
+        
+        var conv = _services.GetRequiredService<IConversationService>();
+
+        var browser = _services.GetRequiredService<IWebBrowser>();
+        var result = await browser.ActionOnElement(new MessageInfo
+        {
+            AgentId = message.CurrentAgentId,
+            MessageId = message.MessageId,
+            ContextId = message.CurrentAgentId,
+        }, locatorArgs, actionArgs);
+
+        message.Content = $"{actionArgs.Action} executed {(result.IsSuccess ? "success" : "failed")}";
+
+        return true;
+    }
+}

src/Plugins/BotSharp.Plugin.WebDriver/UtilFunctions/UtilWebActionOnElementFn.cs

@@ -0,0 +1,29 @@
+namespace BotSharp.Plugin.WebDriver.UtilFunctions;
+
+public class UtilWebCloseBrowserFn : IFunctionCallback
+{
+    public string Name => "util-web-close_browser";
+    public string Indication => "Closing web browser.";
+    private readonly IServiceProvider _services;
+    private readonly ILogger _logger;
+
+    public UtilWebCloseBrowserFn(
+        IServiceProvider services,
+        ILogger<UtilWebCloseBrowserFn> logger)
+    {
+        _services = services;
+        _logger = logger;
+    }
+
+    public async Task<bool> Execute(RoleDialogModel message)
+    {
+        var conv = _services.GetRequiredService<IConversationService>();
+
+        var browser = _services.GetRequiredService<IWebBrowser>();
+        await browser.CloseBrowser(message.CurrentAgentId);
+
+        message.Content = $"Browser closed.";
+
+        return true;
+    }
+}

src/Plugins/BotSharp.Plugin.WebDriver/UtilFunctions/UtilWebCloseBrowserFn.cs

@@ -0,0 +1,44 @@
+namespace BotSharp.Plugin.WebDriver.UtilFunctions;
+
+public class UtilWebGoToPageFn : IFunctionCallback
+{
+    public string Name => "util-web-go_to_page";
+    public string Indication => "Open web page.";
+    private readonly IServiceProvider _services;
+    private readonly ILogger _logger;
+
+    public UtilWebGoToPageFn(
+        IServiceProvider services,
+        ILogger<UtilWebGoToPageFn> logger)
+    {
+        _services = services;
+        _logger = logger;
+    }
+
+    public async Task<bool> Execute(RoleDialogModel message)
+    {
+        var args = JsonSerializer.Deserialize<PageActionArgs>(message.FunctionArgs, new JsonSerializerOptions
+        {
+            PropertyNamingPolicy = JsonNamingPolicy.CamelCase
+        });
+        args.WaitForNetworkIdle = false;
+        args.WaitTime = 5;
+        args.OpenNewTab = true;
+
+        var conv = _services.GetRequiredService<IConversationService>();
+
+        var browser = _services.GetRequiredService<IWebBrowser>();
+        var msg = new MessageInfo
+        {
+            AgentId = message.CurrentAgentId,
+            MessageId = message.MessageId,
+            ContextId = message.CurrentAgentId,
+        };
+        await browser.CloseCurrentPage(msg);
+        await browser.GoToPage(msg, args);
+
+        message.Content = $"Open web page successfully.";
+
+        return true;
+    }
+}

src/Plugins/BotSharp.Plugin.WebDriver/UtilFunctions/UtilWebGoToPageFn.cs

@@ -39,5 +39,7 @@ public void RegisterDI(IServiceCollection services, IConfiguration config)
 
         services.AddScoped<WebDriverService>();
         services.AddScoped<IConversationHook, WebDriverConversationHook>();
+
+        services.AddScoped<IAgentUtilityHook, WebUtilityHook>();
     }
 }

src/Plugins/BotSharp.Plugin.WebDriver/WebDriverPlugin.cs

@@ -0,0 +1,23 @@
+{
+  "name": "util-web-action_on_element",
+  "description": "Do action on specific element located by selector",
+  "parameters": {
+    "type": "object",
+    "properties": {
+      "selector": {
+        "type": "string",
+        "description": "element selector in XPath, use syntax of Playwright in .NET"
+      },
+      "action": {
+        "type": "string",
+        "description": "action to perform on element",
+        "enum": [ "Click", "InputText" ]
+      },
+      "content": {
+        "type": "string",
+        "description": "content to input in element"
+      }
+    },
+    "required": [ "selector", "action" ]
+  }
+}

src/Plugins/BotSharp.Plugin.WebDriver/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-web-action_on_element.json

@@ -0,0 +1,10 @@
+{
+  "name": "util-web-close_browser",
+  "description": "Close browser.",
+  "parameters": {
+    "type": "object",
+    "properties": {
+    },
+    "required": [ ]
+  }
+}

src/Plugins/BotSharp.Plugin.WebDriver/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-web-close_browser.json

@@ -0,0 +1,14 @@
+{
+  "name": "util-web-go_to_page",
+  "description": "Go to web page in specific url.",
+  "parameters": {
+    "type": "object",
+    "properties": {
+      "url": {
+        "type": "string",
+        "description": "Web URL"
+      }
+    },
+    "required": [ "url" ]
+  }
+}

src/Plugins/BotSharp.Plugin.WebDriver/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-web-go_to_page.json

@@ -0,0 +1 @@
+When user asks to do specific actions (click, fill content, scroll) on a web page, use tool of util-web-action_on_element.

src/Plugins/BotSharp.Plugin.WebDriver/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-web-action_on_element.fn.liquid

@@ -0,0 +1 @@
+When user asks to open a web page, use tool of util-web-go_to_page.