.Net: Add additional integration tests for the common agent invoke api

dotnet/src/IntegrationTests/Agents/CommonInterfaceConformance/AgentFixture.cs

@@ -13,7 +13,7 @@ namespace SemanticKernel.IntegrationTests.Agents.CommonInterfaceConformance;
 /// </summary>
 public abstract class AgentFixture : IAsyncLifetime
 {
-    public abstract Agent Agent { get; }
+    public abstract KernelAgent Agent { get; }
 
     public abstract AgentThread AgentThread { get; }
 

dotnet/src/IntegrationTests/Agents/CommonInterfaceConformance/AzureAIAgentFixture.cs

@@ -30,7 +30,7 @@ public class AzureAIAgentFixture : AgentFixture
     private AzureAIAgentThread? _serviceFailingAgentThread;
     private AzureAIAgentThread? _createdServiceFailingAgentThread;
 
-    public override Agent Agent => this._agent!;
+    public override KernelAgent Agent => this._agent!;
 
     public override AgentThread AgentThread => this._thread!;
 

dotnet/src/IntegrationTests/Agents/CommonInterfaceConformance/BedrockAgentFixture.cs

@@ -37,7 +37,7 @@ internal sealed class BedrockAgentFixture : AgentFixture, IAsyncDisposable
     private readonly AmazonBedrockAgentClient _client = new();
     private readonly AmazonBedrockAgentRuntimeClient _runtimeClient = new();
 
-    public override Microsoft.SemanticKernel.Agents.Agent Agent => this._agent!;
+    public override Microsoft.SemanticKernel.Agents.KernelAgent Agent => this._agent!;
 
     public override AgentThread AgentThread => this._thread!;
 

dotnet/src/IntegrationTests/Agents/CommonInterfaceConformance/ChatCompletionAgentFixture.cs

@@ -26,7 +26,7 @@ public class ChatCompletionAgentFixture : AgentFixture
     private ChatHistoryAgentThread? _thread;
     private ChatHistoryAgentThread? _createdThread;
 
-    public override Agent Agent => this._agent!;
+    public override KernelAgent Agent => this._agent!;
 
     public override AgentThread AgentThread => this._thread!;
 

dotnet/src/IntegrationTests/Agents/CommonInterfaceConformance/InvokeConformance/BedrockAgentInvokeTests.cs

@@ -39,4 +39,10 @@ public override Task InvokeWithoutThreadCreatesThreadAsync()
     {
         return base.InvokeWithoutThreadCreatesThreadAsync();
     }
+
+    [Fact(Skip = "This test is for manual verification.")]
+    public override Task MultiStepInvokeWithPluginAndArgOverridesAsync()
+    {
+        return base.MultiStepInvokeWithPluginAndArgOverridesAsync();
+    }
 }

dotnet/src/IntegrationTests/Agents/CommonInterfaceConformance/InvokeConformance/InvokeTests.cs

@@ -25,48 +25,60 @@ public abstract class InvokeTests(Func<AgentFixture> createAgentFixture) : IAsyn
     [Fact]
     public virtual async Task InvokeReturnsResultAsync()
     {
+        // Arrange
         var agent = this.Fixture.Agent;
+
+        // Act
         var asyncResults = agent.InvokeAsync(new ChatMessageContent(AuthorRole.User, "What is the capital of France."), this.Fixture.AgentThread);
         var results = await asyncResults.ToListAsync();
-        Assert.Single(results);
 
+        // Assert
+        Assert.Single(results);
         var firstResult = results.First();
+
         Assert.Contains("Paris", firstResult.Message.Content);
         Assert.NotNull(firstResult.Thread);
     }
 
     [Fact]
     public virtual async Task InvokeWithoutThreadCreatesThreadAsync()
     {
+        // Arrange
         var agent = this.Fixture.Agent;
+
+        // Act
         var asyncResults = agent.InvokeAsync(new ChatMessageContent(AuthorRole.User, "What is the capital of France."));
         var results = await asyncResults.ToListAsync();
-        Assert.Single(results);
 
+        // Assert
+        Assert.Single(results);
         var firstResult = results.First();
         Assert.Contains("Paris", firstResult.Message.Content);
         Assert.NotNull(firstResult.Thread);
 
+        // Cleanup
         await this.Fixture.DeleteThread(firstResult.Thread);
     }
 
     [Fact]
     public virtual async Task ConversationMaintainsHistoryAsync()
     {
+        // Arrange
         var q1 = "What is the capital of France.";
         var q2 = "What is the capital of Austria.";
-
         var agent = this.Fixture.Agent;
+
+        // Act
         var asyncResults1 = agent.InvokeAsync(new ChatMessageContent(AuthorRole.User, q1), this.Fixture.AgentThread);
         var result1 = await asyncResults1.FirstAsync();
         var asyncResults2 = agent.InvokeAsync(new ChatMessageContent(AuthorRole.User, q2), result1.Thread);
         var result2 = await asyncResults2.FirstAsync();
 
+        // Assert
         Assert.Contains("Paris", result1.Message.Content);
         Assert.Contains("Austria", result2.Message.Content);
 
         var chatHistory = await this.Fixture.GetChatHistory();
-
         Assert.Equal(4, chatHistory.Count);
         Assert.Equal(2, chatHistory.Count(x => x.Role == AuthorRole.User));
         Assert.Equal(2, chatHistory.Count(x => x.Role == AuthorRole.Assistant));
@@ -76,6 +88,48 @@ public virtual async Task ConversationMaintainsHistoryAsync()
         Assert.Contains("Vienna", chatHistory[3].Content);
     }
 
+    /// <summary>
+    /// Verifies that the agent can invoke a plugin and respects the override
+    /// Kernel and KernelArguments provided in the options.
+    /// The step does multiple iterations to make sure that the agent
+    /// also manages the chat history correctly.
+    /// </summary>
+    [Fact]
+    public virtual async Task MultiStepInvokeWithPluginAndArgOverridesAsync()
+    {
+        // Arrange
+        var questionsAndAnswers = new[]
+        {
+            ("Hello", string.Empty),
+            ("What is the special soup?", "Clam Chowder"),
+            ("What is the special drink?", "Chai Tea"),
+            ("What is the special salad?", "Cobb Salad"),
+            ("Thank you", string.Empty)
+        };
+
+        var agent = this.Fixture.Agent;
+        var kernel = agent.Kernel.Clone();
+        kernel.Plugins.AddFromType<MenuPlugin>();
+
+        foreach (var questionAndAnswer in questionsAndAnswers)
+        {
+            // Act
+            var asyncResults = agent.InvokeAsync(
+                new ChatMessageContent(AuthorRole.User, questionAndAnswer.Item1),
+                this.Fixture.AgentThread,
+                options: new()
+                {
+                    Kernel = kernel,
+                    KernelArguments = new KernelArguments(new PromptExecutionSettings() { FunctionChoiceBehavior = FunctionChoiceBehavior.Auto() })
+                });
+
+            // Assert
+            var result = await asyncResults.FirstAsync();
+            Assert.NotNull(result);
+            Assert.Contains(questionAndAnswer.Item2, result.Message.Content);
+        }
+    }
+
     public Task InitializeAsync()
     {
         this._agentFixture = createAgentFixture();

dotnet/src/IntegrationTests/Agents/CommonInterfaceConformance/InvokeStreamingConformance/AzureAIAgentInvokeStreamingTests.cs

@@ -0,0 +1,7 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+namespace SemanticKernel.IntegrationTests.Agents.CommonInterfaceConformance.InvokeStreamingConformance;
+
+public class AzureAIAgentInvokeStreamingTests() : InvokeStreamingTests(() => new AzureAIAgentFixture())
+{
+}

dotnet/src/IntegrationTests/Agents/CommonInterfaceConformance/InvokeStreamingConformance/ChatCompletionAgentInvokeStreamingTests.cs

@@ -0,0 +1,7 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+namespace SemanticKernel.IntegrationTests.Agents.CommonInterfaceConformance.InvokeStreamingConformance;
+
+public class ChatCompletionAgentInvokeStreamingTests() : InvokeStreamingTests(() => new ChatCompletionAgentFixture())
+{
+}

dotnet/src/IntegrationTests/Agents/CommonInterfaceConformance/InvokeStreamingConformance/InvokeStreamingTests.cs

@@ -0,0 +1,148 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using System;
+using System.Linq;
+using System.Threading.Tasks;
+using Microsoft.SemanticKernel;
+using Microsoft.SemanticKernel.Agents;
+using Microsoft.SemanticKernel.ChatCompletion;
+using Xunit;
+
+namespace SemanticKernel.IntegrationTests.Agents.CommonInterfaceConformance.InvokeStreamingConformance;
+
+/// <summary>
+/// Base test class for testing the <see cref="Agent.InvokeStreamingAsync(ChatMessageContent, AgentThread?, AgentInvokeOptions?, System.Threading.CancellationToken)"/> method of agents.
+/// Each agent type should have its own derived class.
+/// </summary>
+public abstract class InvokeStreamingTests(Func<AgentFixture> createAgentFixture) : IAsyncLifetime
+{
+#pragma warning disable CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider adding the 'required' modifier or declaring as nullable.
+    private AgentFixture _agentFixture;
+#pragma warning restore CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider adding the 'required' modifier or declaring as nullable.
+
+    protected AgentFixture Fixture => this._agentFixture;
+
+    [Fact]
+    public virtual async Task InvokeStreamingAsyncReturnsResultAsync()
+    {
+        // Arrange
+        var agent = this.Fixture.Agent;
+
+        // Act
+        var asyncResults = agent.InvokeStreamingAsync(new ChatMessageContent(AuthorRole.User, "What is the capital of France."), this.Fixture.AgentThread);
+        var results = await asyncResults.ToListAsync();
+
+        // Assert
+        var firstResult = results.First();
+        var resultString = string.Join(string.Empty, results.Select(x => x.Message.Content));
+
+        Assert.Contains("Paris", resultString);
+        Assert.NotNull(firstResult.Thread);
+    }
+
+    [Fact]
+    public virtual async Task InvokeStreamingAsyncWithoutThreadCreatesThreadAsync()
+    {
+        // Arrange
+        var agent = this.Fixture.Agent;
+
+        // Act
+        var asyncResults = agent.InvokeStreamingAsync(new ChatMessageContent(AuthorRole.User, "What is the capital of France."));
+        var results = await asyncResults.ToListAsync();
+
+        // Assert
+        var firstResult = results.First();
+        var resultString = string.Join(string.Empty, results.Select(x => x.Message.Content));
+
+        Assert.Contains("Paris", resultString);
+        Assert.NotNull(firstResult.Thread);
+
+        // Cleanup
+        await this.Fixture.DeleteThread(firstResult.Thread);
+    }
+
+    [Fact]
+    public virtual async Task ConversationMaintainsHistoryAsync()
+    {
+        // Arrange
+        var q1 = "What is the capital of France.";
+        var q2 = "What is the capital of Austria.";
+        var agent = this.Fixture.Agent;
+
+        // Act
+        var asyncResults1 = agent.InvokeStreamingAsync(new ChatMessageContent(AuthorRole.User, q1), this.Fixture.AgentThread);
+        var results1 = await asyncResults1.ToListAsync();
+        var resultString1 = string.Join(string.Empty, results1.Select(x => x.Message.Content));
+        var result1 = results1.First();
+
+        var asyncResults2 = agent.InvokeStreamingAsync(new ChatMessageContent(AuthorRole.User, q2), result1.Thread);
+        var results2 = await asyncResults2.ToListAsync();
+        var resultString2 = string.Join(string.Empty, results2.Select(x => x.Message.Content));
+
+        // Assert
+        Assert.Contains("Paris", resultString1);
+        Assert.Contains("Austria", resultString2);
+
+        var chatHistory = await this.Fixture.GetChatHistory();
+        Assert.Equal(4, chatHistory.Count);
+        Assert.Equal(2, chatHistory.Count(x => x.Role == AuthorRole.User));
+        Assert.Equal(2, chatHistory.Count(x => x.Role == AuthorRole.Assistant));
+        Assert.Equal(q1, chatHistory[0].Content);
+        Assert.Equal(q2, chatHistory[2].Content);
+        Assert.Contains("Paris", chatHistory[1].Content);
+        Assert.Contains("Vienna", chatHistory[3].Content);
+    }
+
+    /// <summary>
+    /// Verifies that the agent can invoke a plugin and respects the override
+    /// Kernel and KernelArguments provided in the options.
+    /// The step does multiple iterations to make sure that the agent
+    /// also manages the chat history correctly.
+    /// </summary>
+    [Fact]
+    public virtual async Task MultiStepInvokeStreamingAsyncWithPluginAndArgOverridesAsync()
+    {
+        // Arrange
+        var questionsAndAnswers = new[]
+        {
+            ("Hello", string.Empty),
+            ("What is the special soup?", "Clam Chowder"),
+            ("What is the special drink?", "Chai Tea"),
+            ("What is the special salad?", "Cobb Salad"),
+            ("Thank you", string.Empty)
+        };
+
+        var agent = this.Fixture.Agent;
+        var kernel = agent.Kernel.Clone();
+        kernel.Plugins.AddFromType<MenuPlugin>();
+
+        foreach (var questionAndAnswer in questionsAndAnswers)
+        {
+            // Act
+            var asyncResults = agent.InvokeStreamingAsync(
+                new ChatMessageContent(AuthorRole.User, questionAndAnswer.Item1),
+                this.Fixture.AgentThread,
+                options: new()
+                {
+                    Kernel = kernel,
+                    KernelArguments = new KernelArguments(new PromptExecutionSettings() { FunctionChoiceBehavior = FunctionChoiceBehavior.Auto() })
+                });
+            var results = await asyncResults.ToListAsync();
+
+            // Assert
+            var resultString = string.Join(string.Empty, results.Select(x => x.Message.Content));
+            Assert.Contains(questionAndAnswer.Item2, resultString);
+        }
+    }
+
+    public Task InitializeAsync()
+    {
+        this._agentFixture = createAgentFixture();
+        return this._agentFixture.InitializeAsync();
+    }
+
+    public Task DisposeAsync()
+    {
+        return this._agentFixture.DisposeAsync();
+    }
+}

dotnet/src/IntegrationTests/Agents/CommonInterfaceConformance/InvokeStreamingConformance/OpenAIAssistantAgentInvokeStreamingTests.cs

@@ -0,0 +1,7 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+namespace SemanticKernel.IntegrationTests.Agents.CommonInterfaceConformance.InvokeStreamingConformance;
+
+public class OpenAIAssistantAgentInvokeStreamingTests() : InvokeStreamingTests(() => new OpenAIAssistantAgentFixture())
+{
+}

dotnet/src/IntegrationTests/Agents/CommonInterfaceConformance/MenuPlugin.cs

@@ -0,0 +1,34 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using System.ComponentModel;
+using Microsoft.SemanticKernel;
+
+namespace SemanticKernel.IntegrationTests.Agents.CommonInterfaceConformance;
+
+/// <summary>
+/// A test plugin used to verify the ability of Semantic Kernel's Common Agent Interface to invoke plugins.
+/// </summary>
+#pragma warning disable CA1812 // Avoid uninstantiated internal classes
+internal sealed class MenuPlugin
+{
+    [KernelFunction, Description("Provides a list of specials from the menu.")]
+    [System.Diagnostics.CodeAnalysis.SuppressMessage("Design", "CA1024:Use properties where appropriate", Justification = "Too smart")]
+    public string GetSpecials()
+    {
+        return
+            """
+                Special Soup: Clam Chowder
+                Special Salad: Cobb Salad
+                Special Drink: Chai Tea
+                """;
+    }
+
+    [KernelFunction, Description("Provides the price of the requested menu item.")]
+    public string GetItemPrice(
+        [Description("The name of the menu item.")]
+        string menuItem)
+    {
+        return "$9.99";
+    }
+}
+#pragma warning restore CA1812 // Avoid uninstantiated internal classes

dotnet/src/IntegrationTests/Agents/CommonInterfaceConformance/OpenAIAssistantAgentFixture.cs

@@ -34,7 +34,7 @@ public class OpenAIAssistantAgentFixture : AgentFixture
     private OpenAIAssistantAgentThread? _serviceFailingAgentThread;
     private OpenAIAssistantAgentThread? _createdServiceFailingAgentThread;
 
-    public override Agent Agent => this._agent!;
+    public override KernelAgent Agent => this._agent!;
 
     public override AgentThread AgentThread => this._thread!;