.Net: Merge Tavily feature branch to main

dotnet/samples/Concepts/Search/Tavily_TextSearch.cs

@@ -0,0 +1,198 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using System.Text.Json;
+using Microsoft.SemanticKernel.Data;
+using Microsoft.SemanticKernel.Plugins.Web.Tavily;
+
+namespace Search;
+
+/// <summary>
+/// This example shows how to create and use a <see cref="TavilyTextSearch"/>.
+/// </summary>
+public class Tavily_TextSearch(ITestOutputHelper output) : BaseTest(output)
+{
+    /// <summary>
+    /// Show how to create a <see cref="TavilyTextSearch"/> and use it to perform a text search.
+    /// </summary>
+    [Fact]
+    public async Task UsingTavilyTextSearch()
+    {
+        // Create a logging handler to output HTTP requests and responses
+        LoggingHandler handler = new(new HttpClientHandler(), this.Output);
+        using HttpClient httpClient = new(handler);
+
+        // Create an ITextSearch instance using Tavily search
+        var textSearch = new TavilyTextSearch(apiKey: TestConfiguration.Tavily.ApiKey, options: new() { HttpClient = httpClient, IncludeRawContent = true });
+
+        var query = "What is the Semantic Kernel?";
+
+        // Search and return results as a string items
+        KernelSearchResults<string> stringResults = await textSearch.SearchAsync(query, new() { Top = 4 });
+        Console.WriteLine("--- String Results ---\n");
+        await foreach (string result in stringResults.Results)
+        {
+            Console.WriteLine(result);
+            WriteHorizontalRule();
+        }
+
+        // Search and return results as TextSearchResult items
+        KernelSearchResults<TextSearchResult> textResults = await textSearch.GetTextSearchResultsAsync(query, new() { Top = 4 });
+        Console.WriteLine("\n--- Text Search Results ---\n");
+        await foreach (TextSearchResult result in textResults.Results)
+        {
+            Console.WriteLine($"Name:  {result.Name}");
+            Console.WriteLine($"Value: {result.Value}");
+            Console.WriteLine($"Link:  {result.Link}");
+            WriteHorizontalRule();
+        }
+
+        // Search and return s results as TavilySearchResult items
+        KernelSearchResults<object> fullResults = await textSearch.GetSearchResultsAsync(query, new() { Top = 4 });
+        Console.WriteLine("\n--- Tavily Web Page Results ---\n");
+        await foreach (TavilySearchResult result in fullResults.Results)
+        {
+            Console.WriteLine($"Name:            {result.Title}");
+            Console.WriteLine($"Content:         {result.Content}");
+            Console.WriteLine($"Url:             {result.Url}");
+            Console.WriteLine($"RawContent:      {result.RawContent}");
+            Console.WriteLine($"Score:           {result.Score}");
+            WriteHorizontalRule();
+        }
+    }
+
+    /// <summary>
+    /// Show how to create a <see cref="TavilyTextSearch"/> and use it to perform a text search which returns an answer.
+    /// </summary>
+    [Fact]
+    public async Task UsingTavilyTextSearchToGetAnAnswer()
+    {
+        // Create a logging handler to output HTTP requests and responses
+        LoggingHandler handler = new(new HttpClientHandler(), this.Output);
+        using HttpClient httpClient = new(handler);
+
+        // Create an ITextSearch instance using Tavily search
+        var textSearch = new TavilyTextSearch(apiKey: TestConfiguration.Tavily.ApiKey, options: new() { HttpClient = httpClient, IncludeAnswer = true });
+
+        var query = "What is the Semantic Kernel?";
+
+        // Search and return results as a string items
+        KernelSearchResults<string> stringResults = await textSearch.SearchAsync(query, new() { Top = 1 });
+        Console.WriteLine("--- String Results ---\n");
+        await foreach (string result in stringResults.Results)
+        {
+            Console.WriteLine(result);
+            WriteHorizontalRule();
+        }
+    }
+
+    /// <summary>
+    /// Show how to create a <see cref="TavilyTextSearch"/> and use it to perform a text search.
+    /// </summary>
+    [Fact]
+    public async Task UsingTavilyTextSearchAndIncludeEverything()
+    {
+        // Create a logging handler to output HTTP requests and responses
+        LoggingHandler handler = new(new HttpClientHandler(), this.Output);
+        using HttpClient httpClient = new(handler);
+
+        // Create an ITextSearch instance using Tavily search
+        var textSearch = new TavilyTextSearch(
+            apiKey: TestConfiguration.Tavily.ApiKey,
+            options: new()
+            {
+                HttpClient = httpClient,
+                IncludeRawContent = true,
+                IncludeImages = true,
+                IncludeImageDescriptions = true,
+                IncludeAnswer = true,
+            });
+
+        var query = "What is the Semantic Kernel?";
+
+        // Search and return s results as TavilySearchResult items
+        KernelSearchResults<object> fullResults = await textSearch.GetSearchResultsAsync(query, new() { Top = 4, Skip = 0 });
+        Console.WriteLine("\n--- Tavily Web Page Results ---\n");
+        await foreach (TavilySearchResult result in fullResults.Results)
+        {
+            Console.WriteLine($"Name:            {result.Title}");
+            Console.WriteLine($"Content:         {result.Content}");
+            Console.WriteLine($"Url:             {result.Url}");
+            Console.WriteLine($"RawContent:      {result.RawContent}");
+            Console.WriteLine($"Score:           {result.Score}");
+            WriteHorizontalRule();
+        }
+    }
+
+    /// <summary>
+    /// Show how to create a <see cref="TavilyTextSearch"/> with a custom mapper and use it to perform a text search.
+    /// </summary>
+    [Fact]
+    public async Task UsingTavilyTextSearchWithACustomMapperAsync()
+    {
+        // Create a logging handler to output HTTP requests and responses
+        LoggingHandler handler = new(new HttpClientHandler(), this.Output);
+        using HttpClient httpClient = new(handler);
+
+        // Create an ITextSearch instance using Tavily search
+        var textSearch = new TavilyTextSearch(apiKey: TestConfiguration.Tavily.ApiKey, options: new()
+        {
+            HttpClient = httpClient,
+            StringMapper = new TestTextSearchStringMapper(),
+        });
+
+        var query = "What is the Semantic Kernel?";
+
+        // Search with TextSearchResult textResult type
+        KernelSearchResults<string> stringResults = await textSearch.SearchAsync(query, new() { Top = 2 });
+        Console.WriteLine("--- Serialized JSON Results ---");
+        await foreach (string result in stringResults.Results)
+        {
+            Console.WriteLine(result);
+            WriteHorizontalRule();
+        }
+    }
+
+    /// <summary>
+    /// Show how to create a <see cref="TavilyTextSearch"/> with a custom mapper and use it to perform a text search.
+    /// </summary>
+    [Fact]
+    public async Task UsingTavilyTextSearchWithAnIncludeDomainFilterAsync()
+    {
+        // Create a logging handler to output HTTP requests and responses
+        LoggingHandler handler = new(new HttpClientHandler(), this.Output);
+        using HttpClient httpClient = new(handler);
+
+        // Create an ITextSearch instance using Tavily search
+        var textSearch = new TavilyTextSearch(apiKey: TestConfiguration.Tavily.ApiKey, options: new()
+        {
+            HttpClient = httpClient,
+            StringMapper = new TestTextSearchStringMapper(),
+        });
+
+        var query = "What is the Semantic Kernel?";
+
+        // Search with TextSearchResult textResult type
+        TextSearchOptions searchOptions = new() { Top = 4, Filter = new TextSearchFilter().Equality("include_domain", "devblogs.microsoft.com") };
+        KernelSearchResults<TextSearchResult> textResults = await textSearch.GetTextSearchResultsAsync(query, searchOptions);
+        Console.WriteLine("--- Microsoft Developer Blogs Results ---");
+        await foreach (TextSearchResult result in textResults.Results)
+        {
+            Console.WriteLine(result.Link);
+            WriteHorizontalRule();
+        }
+    }
+
+    #region private
+    /// <summary>
+    /// Test mapper which converts an arbitrary search result to a string using JSON serialization.
+    /// </summary>
+    private sealed class TestTextSearchStringMapper : ITextSearchStringMapper
+    {
+        /// <inheritdoc />
+        public string MapFromResultToString(object result)
+        {
+            return JsonSerializer.Serialize(result);
+        }
+    }
+    #endregion
+}

dotnet/src/IntegrationTests/Data/BaseTextSearchTests.cs

@@ -18,7 +18,7 @@ namespace SemanticKernel.IntegrationTests.Data;
 /// </summary>
 public abstract class BaseTextSearchTests : BaseIntegrationTest
 {
-    [Fact(Skip = "Failing in integration tests pipeline with - HTTP 429 (insufficient_quota) error.")]
+    [Fact(Skip = "For manual verification only.")]
     public virtual async Task CanSearchAsync()
     {
         // Arrange
@@ -42,7 +42,7 @@ public virtual async Task CanSearchAsync()
         }
     }
 
-    [Fact(Skip = "Failing in integration tests pipeline with - HTTP 429 (insufficient_quota) error.")]
+    [Fact(Skip = "For manual verification only.")]
     public virtual async Task CanGetTextSearchResultsAsync()
     {
         // Arrange
@@ -72,7 +72,7 @@ public virtual async Task CanGetTextSearchResultsAsync()
         }
     }
 
-    [Fact(Skip = "Failing in integration tests pipeline with - HTTP 429 (insufficient_quota) error.")]
+    [Fact(Skip = "For manual verification only.")]
     public virtual async Task CanGetSearchResultsAsync()
     {
         // Arrange
@@ -92,7 +92,7 @@ public virtual async Task CanGetSearchResultsAsync()
         Assert.True(this.VerifySearchResults(results, query));
     }
 
-    [Fact(Skip = "Failing in integration tests pipeline with - HTTP 429 (insufficient_quota) error.")]
+    [Fact(Skip = "For manual verification only.")]
     public virtual async Task UsingTextSearchWithAFilterAsync()
     {
         // Arrange
@@ -113,7 +113,7 @@ public virtual async Task UsingTextSearchWithAFilterAsync()
         Assert.True(this.VerifySearchResults(results, query, filter));
     }
 
-    [Fact(Skip = "Failing in integration tests pipeline with - HTTP 429 (insufficient_quota) error.")]
+    [Fact(Skip = "For manual verification only.")]
     public virtual async Task FunctionCallingUsingCreateWithSearchAsync()
     {
         // Arrange
@@ -142,7 +142,7 @@ public virtual async Task FunctionCallingUsingCreateWithSearchAsync()
         Assert.NotEmpty(results);
     }
 
-    [Fact(Skip = "Failing in integration tests pipeline with - HTTP 429 (insufficient_quota) error.")]
+    [Fact(Skip = "For manual verification only.")]
     public virtual async Task FunctionCallingUsingCreateWithGetSearchResultsAsync()
     {
         // Arrange
@@ -171,7 +171,7 @@ public virtual async Task FunctionCallingUsingCreateWithGetSearchResultsAsync()
         Assert.NotEmpty(results);
     }
 
-    [Fact(Skip = "Failing in integration tests pipeline with - HTTP 429 (insufficient_quota) error.")]
+    [Fact(Skip = "For manual verification only.")]
     public virtual async Task FunctionCallingUsingGetTextSearchResultsAsync()
     {
         // Arrange

dotnet/src/IntegrationTests/Plugins/Web/Tavily/TavilyTextSearchTests.cs

@@ -0,0 +1,48 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using System.Threading.Tasks;
+using Microsoft.Extensions.Configuration;
+using Microsoft.SemanticKernel.Data;
+using Microsoft.SemanticKernel.Plugins.Web.Tavily;
+using SemanticKernel.IntegrationTests.Data;
+using SemanticKernel.IntegrationTests.TestSettings;
+using Xunit;
+
+namespace SemanticKernel.IntegrationTests.Plugins.Web.Tavily;
+
+/// <summary>
+/// Integration tests for <see cref="TavilyTextSearch"/>.
+/// </summary>
+public class TavilyTextSearchTests : BaseTextSearchTests
+{
+    /// <inheritdoc/>
+    public override Task<ITextSearch> CreateTextSearchAsync()
+    {
+        var configuration = this.Configuration.GetSection("Tavily").Get<TavilyConfiguration>();
+        Assert.NotNull(configuration);
+        Assert.NotNull(configuration.ApiKey);
+
+        return Task.FromResult<ITextSearch>(new TavilyTextSearch(apiKey: configuration.ApiKey));
+    }
+
+    /// <inheritdoc/>
+    public override string GetQuery() => "What is the Semantic Kernel?";
+
+    /// <inheritdoc/>
+    public override TextSearchFilter GetTextSearchFilter() => new TextSearchFilter().Equality("include_domain", "devblogs.microsoft.com");
+
+    /// <inheritdoc/>
+    public override bool VerifySearchResults(object[] results, string query, TextSearchFilter? filter = null)
+    {
+        Assert.NotNull(results);
+        Assert.NotEmpty(results);
+        Assert.Equal(4, results.Length);
+        foreach (var result in results)
+        {
+            Assert.NotNull(result);
+            Assert.IsType<TavilySearchResult>(result);
+        }
+
+        return true;
+    }
+}

dotnet/src/IntegrationTests/TestSettings/TavilyConfiguration.cs

@@ -0,0 +1,8 @@
+// Copyright (c) Microsoft. All rights reserved.
+namespace SemanticKernel.IntegrationTests.TestSettings;
+
+#pragma warning disable CA1812 // Configuration classes are instantiated through IConfiguration.
+internal sealed class TavilyConfiguration(string apiKey)
+{
+    public string ApiKey { get; init; } = apiKey;
+}

dotnet/src/InternalUtilities/samples/InternalUtilities/TestConfiguration.cs

@@ -35,6 +35,7 @@ public static void Initialize(IConfigurationRoot configRoot)
     public static PineconeConfig Pinecone => LoadSection<PineconeConfig>();
     public static BingConfig Bing => LoadSection<BingConfig>();
     public static GoogleConfig Google => LoadSection<GoogleConfig>();
+    public static TavilyConfig Tavily => LoadSection<TavilyConfig>();
     public static GithubConfig Github => LoadSection<GithubConfig>();
     public static PostgresConfig Postgres => LoadSection<PostgresConfig>();
     public static RedisConfig Redis => LoadSection<RedisConfig>();
@@ -176,6 +177,12 @@ public class GoogleConfig
         public string SearchEngineId { get; set; }
     }
 
+    public class TavilyConfig
+    {
+        public string Endpoint { get; set; } = "https://api.tavily.com/search";
+        public string ApiKey { get; set; }
+    }
+
     public class GithubConfig
     {
         public string PAT { get; set; }

dotnet/src/Plugins/Plugins.UnitTests/Plugins.UnitTests.csproj

@@ -44,5 +44,9 @@
       <CopyToOutputDirectory>Always</CopyToOutputDirectory>
     </None>
   </ItemGroup>
+
+  <ItemGroup>
+    <Folder Include="Web\Tavily\" />
+  </ItemGroup>
 
 </Project>

dotnet/src/Plugins/Plugins.UnitTests/TestData/tavily_site_filter_devblogs_microsoft.com.json

@@ -0,0 +1,37 @@
+{
+  "query": "What is the Semantic Kernel?",
+  "follow_up_questions": null,
+  "answer": null,
+  "images": [],
+  "results": [
+    {
+      "title": "Hello, Semantic Kernel! | Semantic Kernel - devblogs.microsoft.com",
+      "url": "https://devblogs.microsoft.com/semantic-kernel/hello-world/",
+      "content": "Semantic Kernel (SK) is a lightweight SDK that lets you mix conventional programming languages, like C# and Python, with the latest in Large Language Model (LLM) AI \"prompts\" with prompt templating, chaining, and planning capabilities. This enables you to build new experiences into your apps to bring unparalleled productivity for your users",
+      "score": 0.944878,
+      "raw_content": null
+    },
+    {
+      "title": "Building Generative AI apps with .NET 8 and Semantic Kernel",
+      "url": "https://devblogs.microsoft.com/semantic-kernel/building-generative-ai-apps-with-net-8-and-semantic-kernel/",
+      "content": "What is Semantic Kernel and why should I use it? In many of our samples, you'll see us using Semantic Kernel (SK) - SK is an open-source library that lets you easily build AI solutions that can call your existing code. As a highly extensible SDK, you can use Semantic Kernel to work with models from OpenAI, Azure OpenAI, Hugging Face, and",
+      "score": 0.8412138,
+      "raw_content": null
+    },
+    {
+      "title": "Semantic Kernel and AutoGen Part 2 - devblogs.microsoft.com",
+      "url": "https://devblogs.microsoft.com/semantic-kernel/semantic-kernel-and-autogen-part-2/",
+      "content": "Following on from our blog post a couple months ago: Microsoft's Agentic AI Frameworks: AutoGen and Semantic Kernel, Microsoft's agentic AI story is evolving at a steady pace.Both Azure AI Foundry's Semantic Kernel and AI Frontier's AutoGen are designed to empower developers to build advanced multi-agent systems. The AI Frontier's team is charging ahead pushing the boundaries of",
+      "score": 0.78631157,
+      "raw_content": null
+    },
+    {
+      "title": "Introducing Semantic Kernel for Java | Semantic Kernel",
+      "url": "https://devblogs.microsoft.com/semantic-kernel/introducing-semantic-kernel-for-java/",
+      "content": "Semantic Kernel for Java is an open source library that empowers developers to harness the power of AI while coding in Java. It is compatible with Java 8 and above, ensuring flexibility and accessibility to a wide range of Java developers. By integrating AI services into your Java applications, you can unlock the full potential of artificial",
+      "score": 0.7703443,
+      "raw_content": null
+    }
+  ],
+  "response_time": 3.16
+}