.Net: [Feature Branch] Vector Store Logging

dotnet/SK-dotnet.sln

@@ -112,12 +112,15 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Diagnostics", "Diagnostics"
 		src\InternalUtilities\src\Diagnostics\ExceptionExtensions.cs = src\InternalUtilities\src\Diagnostics\ExceptionExtensions.cs
 		src\InternalUtilities\src\Diagnostics\ExperimentalAttribute.cs = src\InternalUtilities\src\Diagnostics\ExperimentalAttribute.cs
 		src\InternalUtilities\src\Diagnostics\IsExternalInit.cs = src\InternalUtilities\src\Diagnostics\IsExternalInit.cs
+		src\InternalUtilities\src\Diagnostics\KernelVerify.cs = src\InternalUtilities\src\Diagnostics\KernelVerify.cs
 		src\InternalUtilities\src\Diagnostics\NullableAttributes.cs = src\InternalUtilities\src\Diagnostics\NullableAttributes.cs
 		src\InternalUtilities\src\Diagnostics\RequiresDynamicCodeAttribute.cs = src\InternalUtilities\src\Diagnostics\RequiresDynamicCodeAttribute.cs
 		src\InternalUtilities\src\Diagnostics\RequiresUnreferencedCodeAttribute.cs = src\InternalUtilities\src\Diagnostics\RequiresUnreferencedCodeAttribute.cs
 		src\InternalUtilities\src\Diagnostics\UnconditionalSuppressMessageAttribute.cs = src\InternalUtilities\src\Diagnostics\UnconditionalSuppressMessageAttribute.cs
-		src\InternalUtilities\src\Diagnostics\Verify.cs = src\InternalUtilities\src\Diagnostics\Verify.cs
 		src\InternalUtilities\src\Diagnostics\UnreachableException.cs = src\InternalUtilities\src\Diagnostics\UnreachableException.cs
+		src\InternalUtilities\src\Diagnostics\Verify.cs = src\InternalUtilities\src\Diagnostics\Verify.cs
+		src\InternalUtilities\src\Diagnostics\ActivityExtensions.cs = src\InternalUtilities\src\Diagnostics\ActivityExtensions.cs
+		src\InternalUtilities\src\Diagnostics\LoggingExtensions.cs = src\InternalUtilities\src\Diagnostics\LoggingExtensions.cs
 	EndProjectSection
 EndProject
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Linq", "Linq", "{B00AD427-0047-4850-BEF9-BA8237EA9D8B}"
@@ -138,10 +141,10 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "System", "System", "{3CDE10
 	ProjectSection(SolutionItems) = preProject
 		src\InternalUtilities\src\System\EnvExtensions.cs = src\InternalUtilities\src\System\EnvExtensions.cs
 		src\InternalUtilities\src\System\IListExtensions.cs = src\InternalUtilities\src\System\IListExtensions.cs
+		src\InternalUtilities\src\System\IndexRange.cs = src\InternalUtilities\src\System\IndexRange.cs
 		src\InternalUtilities\src\System\InternalTypeConverter.cs = src\InternalUtilities\src\System\InternalTypeConverter.cs
 		src\InternalUtilities\src\System\NonNullCollection.cs = src\InternalUtilities\src\System\NonNullCollection.cs
 		src\InternalUtilities\src\System\TypeConverterFactory.cs = src\InternalUtilities\src\System\TypeConverterFactory.cs
-		src\InternalUtilities\src\System\IndexRange.cs = src\InternalUtilities\src\System\IndexRange.cs
 	EndProjectSection
 EndProject
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Type", "Type", "{E85EA4D0-BB7E-4DFD-882F-A76EB8C0B8FF}"
@@ -492,8 +495,12 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Agents.Bedrock", "src\Agent
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ModelContextProtocol", "samples\Demos\ModelContextProtocol\ModelContextProtocol.csproj", "{B16AC373-3DA8-4505-9510-110347CD635D}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "VectorData.UnitTests", "src\Connectors\VectorData.UnitTests\VectorData.UnitTests.csproj", "{89FC596F-CB81-4733-829B-4527D0FFC291}"
+EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SqlServerIntegrationTests", "src\VectorDataIntegrationTests\SqlServerIntegrationTests\SqlServerIntegrationTests.csproj", "{A5E6193C-8431-4C6E-B674-682CB41EAA0C}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "VectorData", "src\Connectors\VectorData\VectorData.csproj", "{8A40AE00-4A7D-4ED0-A9DA-BB7A98EFABD3}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -1359,12 +1366,24 @@ Global
 		{B16AC373-3DA8-4505-9510-110347CD635D}.Publish|Any CPU.Build.0 = Debug|Any CPU
 		{B16AC373-3DA8-4505-9510-110347CD635D}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{B16AC373-3DA8-4505-9510-110347CD635D}.Release|Any CPU.Build.0 = Release|Any CPU
+		{89FC596F-CB81-4733-829B-4527D0FFC291}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{89FC596F-CB81-4733-829B-4527D0FFC291}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{89FC596F-CB81-4733-829B-4527D0FFC291}.Publish|Any CPU.ActiveCfg = Publish|Any CPU
+		{89FC596F-CB81-4733-829B-4527D0FFC291}.Publish|Any CPU.Build.0 = Publish|Any CPU
+		{89FC596F-CB81-4733-829B-4527D0FFC291}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{89FC596F-CB81-4733-829B-4527D0FFC291}.Release|Any CPU.Build.0 = Release|Any CPU
 		{A5E6193C-8431-4C6E-B674-682CB41EAA0C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{A5E6193C-8431-4C6E-B674-682CB41EAA0C}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{A5E6193C-8431-4C6E-B674-682CB41EAA0C}.Publish|Any CPU.ActiveCfg = Debug|Any CPU
 		{A5E6193C-8431-4C6E-B674-682CB41EAA0C}.Publish|Any CPU.Build.0 = Debug|Any CPU
 		{A5E6193C-8431-4C6E-B674-682CB41EAA0C}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{A5E6193C-8431-4C6E-B674-682CB41EAA0C}.Release|Any CPU.Build.0 = Release|Any CPU
+		{8A40AE00-4A7D-4ED0-A9DA-BB7A98EFABD3}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{8A40AE00-4A7D-4ED0-A9DA-BB7A98EFABD3}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{8A40AE00-4A7D-4ED0-A9DA-BB7A98EFABD3}.Publish|Any CPU.ActiveCfg = Publish|Any CPU
+		{8A40AE00-4A7D-4ED0-A9DA-BB7A98EFABD3}.Publish|Any CPU.Build.0 = Publish|Any CPU
+		{8A40AE00-4A7D-4ED0-A9DA-BB7A98EFABD3}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{8A40AE00-4A7D-4ED0-A9DA-BB7A98EFABD3}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -1550,7 +1569,9 @@ Global
 		{DAD5FC6A-8CA0-43AC-87E1-032DFBD6B02A} = {3F260A77-B6C9-97FD-1304-4B34DA936CF4}
 		{8C658E1E-83C8-4127-B8BF-27A638A45DDD} = {6823CD5E-2ABE-41EB-B865-F86EC13F0CF9}
 		{B16AC373-3DA8-4505-9510-110347CD635D} = {5D4C0700-BBB5-418F-A7B2-F392B9A18263}
+		{89FC596F-CB81-4733-829B-4527D0FFC291} = {5A7028A7-4DDF-4E4F-84A9-37CE8F8D7E89}
 		{A5E6193C-8431-4C6E-B674-682CB41EAA0C} = {4F381919-F1BE-47D8-8558-3187ED04A84F}
+		{8A40AE00-4A7D-4ED0-A9DA-BB7A98EFABD3} = {24503383-A8C4-4255-9998-28D70FE8E99A}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {FBDC56A3-86AD-4323-AA0F-201E59123B83}

dotnet/samples/Concepts/Concepts.csproj

@@ -77,6 +77,7 @@
     <ProjectReference Include="..\..\src\Connectors\Connectors.Onnx\Connectors.Onnx.csproj" />
     <ProjectReference Include="..\..\src\Connectors\Connectors.Ollama\Connectors.Ollama.csproj" />
     <ProjectReference Include="..\..\src\Connectors\Connectors.OpenAI\Connectors.OpenAI.csproj" />
+    <ProjectReference Include="..\..\src\Connectors\VectorData\VectorData.csproj" />
     <ProjectReference Include="..\..\src\Experimental\Orchestration.Flow\Experimental.Orchestration.Flow.csproj" />
     <ProjectReference Include="..\..\src\Extensions\PromptTemplates.Handlebars\PromptTemplates.Handlebars.csproj" />
     <ProjectReference Include="..\..\src\Extensions\PromptTemplates.Liquid\PromptTemplates.Liquid.csproj" />

dotnet/samples/Concepts/Memory/VectorStore_Telemetry.cs

@@ -0,0 +1,180 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using Azure.Identity;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.VectorData;
+using Microsoft.SemanticKernel;
+using Microsoft.SemanticKernel.Connectors.AzureOpenAI;
+using Microsoft.SemanticKernel.Connectors.InMemory;
+using Microsoft.SemanticKernel.Embeddings;
+
+namespace Memory;
+
+/// <summary>
+/// A simple example showing how to ingest data into a vector store and then use vector search to find related records to a given string
+/// with enabled telemetry.
+/// </summary>
+public class VectorStore_Telemetry(ITestOutputHelper output) : BaseTest(output)
+{
+    [Fact]
+    public async Task LoggingManualRegistrationAsync()
+    {
+        // Create an embedding generation service.
+        var textEmbeddingGenerationService = new AzureOpenAITextEmbeddingGenerationService(
+                TestConfiguration.AzureOpenAIEmbeddings.DeploymentName,
+                TestConfiguration.AzureOpenAIEmbeddings.Endpoint,
+                new AzureCliCredential());
+
+        // Manually construct an InMemory vector store with enabled logging.
+        var vectorStore = new InMemoryVectorStore()
+            .AsBuilder()
+            .UseLogging(this.LoggerFactory)
+            .Build();
+
+        await RunExampleAsync(textEmbeddingGenerationService, vectorStore);
+
+        // Output:
+        // CreateCollectionIfNotExistsAsync invoked.
+        // CreateCollectionIfNotExistsAsync completed.
+        // UpsertAsync invoked.
+        // UpsertAsync completed.
+        // UpsertAsync invoked.
+        // UpsertAsync completed.
+        // UpsertAsync invoked.
+        // UpsertAsync completed.
+        // VectorizedSearchAsync invoked.
+        // VectorizedSearchAsync completed.
+
+        // Search string: What is an Application Programming Interface
+        // Result: Application Programming Interface. A set of rules and specifications that allow software components to communicate and exchange data.
+    }
+
+    [Fact]
+    public async Task LoggingDependencyInjectionAsync()
+    {
+        var serviceCollection = new ServiceCollection();
+
+        // Add an embedding generation service.
+        serviceCollection.AddAzureOpenAITextEmbeddingGeneration(
+            TestConfiguration.AzureOpenAIEmbeddings.DeploymentName,
+            TestConfiguration.AzureOpenAIEmbeddings.Endpoint,
+            new AzureCliCredential());
+
+        // Add InMemory vector store
+        serviceCollection.AddInMemoryVectorStore();
+
+        // Register InMemoryVectorStore with enabled logging.
+        serviceCollection
+            .AddVectorStore(s => s.GetRequiredService<InMemoryVectorStore>())
+            .UseLogging(this.LoggerFactory);
+
+        var services = serviceCollection.BuildServiceProvider();
+
+        var vectorStore = services.GetRequiredService<IVectorStore>();
+        var textEmbeddingGenerationService = services.GetRequiredService<ITextEmbeddingGenerationService>();
+
+        await RunExampleAsync(textEmbeddingGenerationService, vectorStore);
+
+        // Output:
+        // CreateCollectionIfNotExistsAsync invoked.
+        // CreateCollectionIfNotExistsAsync completed.
+        // UpsertAsync invoked.
+        // UpsertAsync completed.
+        // UpsertAsync invoked.
+        // UpsertAsync completed.
+        // UpsertAsync invoked.
+        // UpsertAsync completed.
+        // VectorizedSearchAsync invoked.
+        // VectorizedSearchAsync completed.
+
+        // Search string: What is an Application Programming Interface
+        // Result: Application Programming Interface. A set of rules and specifications that allow software components to communicate and exchange data.
+    }
+
+    private async Task RunExampleAsync(
+        ITextEmbeddingGenerationService textEmbeddingGenerationService,
+        IVectorStore vectorStore)
+    {
+        // Get and create collection if it doesn't exist.
+        var collection = vectorStore.GetCollection<ulong, Glossary>("skglossary");
+        await collection.CreateCollectionIfNotExistsAsync();
+
+        // Create glossary entries and generate embeddings for them.
+        var glossaryEntries = CreateGlossaryEntries().ToList();
+        var tasks = glossaryEntries.Select(entry => Task.Run(async () =>
+        {
+            entry.DefinitionEmbedding = await textEmbeddingGenerationService.GenerateEmbeddingAsync(entry.Definition);
+        }));
+        await Task.WhenAll(tasks);
+
+        // Upsert the glossary entries into the collection and return their keys.
+        var upsertedKeysTasks = glossaryEntries.Select(x => collection.UpsertAsync(x));
+        var upsertedKeys = await Task.WhenAll(upsertedKeysTasks);
+
+        // Search the collection using a vector search.
+        var searchString = "What is an Application Programming Interface";
+        var searchVector = await textEmbeddingGenerationService.GenerateEmbeddingAsync(searchString);
+        var searchResult = await collection.VectorizedSearchAsync(searchVector, new() { Top = 1 });
+        var resultRecords = await searchResult.Results.ToListAsync();
+
+        Console.WriteLine("Search string: " + searchString);
+        Console.WriteLine("Result: " + resultRecords.First().Record.Definition);
+        Console.WriteLine();
+    }
+
+    /// <summary>
+    /// Sample model class that represents a glossary entry.
+    /// </summary>
+    /// <remarks>
+    /// Note that each property is decorated with an attribute that specifies how the property should be treated by the vector store.
+    /// This allows us to create a collection in the vector store and upsert and retrieve instances of this class without any further configuration.
+    /// </remarks>
+    private sealed class Glossary
+    {
+        [VectorStoreRecordKey]
+        public ulong Key { get; set; }
+
+        [VectorStoreRecordData(IsFilterable = true)]
+        public string Category { get; set; }
+
+        [VectorStoreRecordData]
+        public string Term { get; set; }
+
+        [VectorStoreRecordData]
+        public string Definition { get; set; }
+
+        [VectorStoreRecordVector(1536)]
+        public ReadOnlyMemory<float> DefinitionEmbedding { get; set; }
+    }
+
+    /// <summary>
+    /// Create some sample glossary entries.
+    /// </summary>
+    /// <returns>A list of sample glossary entries.</returns>
+    private static IEnumerable<Glossary> CreateGlossaryEntries()
+    {
+        yield return new Glossary
+        {
+            Key = 1,
+            Category = "External Definitions",
+            Term = "API",
+            Definition = "Application Programming Interface. A set of rules and specifications that allow software components to communicate and exchange data."
+        };
+
+        yield return new Glossary
+        {
+            Key = 2,
+            Category = "Core Definitions",
+            Term = "Connectors",
+            Definition = "Connectors allow you to integrate with various services provide AI capabilities, including LLM, AudioToText, TextToAudio, Embedding generation, etc."
+        };
+
+        yield return new Glossary
+        {
+            Key = 3,
+            Category = "External Definitions",
+            Term = "RAG",
+            Definition = "Retrieval Augmented Generation - a term that refers to the process of retrieving additional data to provide as context to an LLM to use when generating a response (completion) to a user’s question (prompt)."
+        };
+    }
+}

dotnet/src/Connectors/VectorData.Abstractions/PACKAGE.md

@@ -32,4 +32,4 @@ The main types provided by this library are:
 
 ## Feedback & Contributing
 
-Microsoft.Extensions.DependencyInjection.Abstractions is released as open source under the [MIT license](https://licenses.nuget.org/MIT). Bug reports and contributions are welcome at [the GitHub repository](https://github.com/microsoft/semantic-kernel).
+Microsoft.Extensions.VectorData.Abstractions is released as open source under the [MIT license](https://licenses.nuget.org/MIT). Bug reports and contributions are welcome at [the GitHub repository](https://github.com/microsoft/semantic-kernel).

dotnet/src/Connectors/VectorData.UnitTests/.editorconfig

@@ -0,0 +1,6 @@
+# Suppressing errors for Test projects under dotnet folder
+[*.cs]
+dotnet_diagnostic.CA2007.severity = none # Do not directly await a Task
+dotnet_diagnostic.VSTHRD111.severity = none # Use .ConfigureAwait(bool) is hidden by default, set to none to prevent IDE from changing on autosave
+dotnet_diagnostic.CS1591.severity = none # Missing XML comment for publicly visible type or member
+dotnet_diagnostic.IDE1006.severity = warning # Naming rule violations

dotnet/src/Connectors/VectorData.UnitTests/VectorData.UnitTests.csproj

@@ -0,0 +1,41 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <AssemblyName>VectorData.UnitTests</AssemblyName>
+    <RootNamespace>VectorData.UnitTests</RootNamespace>
+    <TargetFramework>net8.0</TargetFramework>
+    <IsTestProject>true</IsTestProject>
+    <Nullable>enable</Nullable>
+    <ImplicitUsings>disable</ImplicitUsings>
+    <IsPackable>false</IsPackable>
+    <NoWarn>$(NoWarn);SKEXP0001,SKEXP0020,VSTHRD111,CA2007,CS1591</NoWarn>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Microsoft.Extensions.DependencyInjection" />
+    <PackageReference Include="Microsoft.NET.Test.Sdk" />
+    <PackageReference Include="Moq" />
+    <PackageReference Include="xunit" />
+    <PackageReference Include="xunit.runner.visualstudio">
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+      <PrivateAssets>all</PrivateAssets>
+    </PackageReference>
+    <PackageReference Include="coverlet.collector">
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+      <PrivateAssets>all</PrivateAssets>
+    </PackageReference>
+  </ItemGroup>
+
+  <ItemGroup>
+    <Compile Include="$(RepoRoot)/dotnet/src/InternalUtilities/test/AssertExtensions.cs" Link="%(RecursiveDir)%(Filename)%(Extension)" />
+    <Compile Include="$(RepoRoot)/dotnet/src/InternalUtilities/src/Linq/AsyncEnumerable.cs" Link="%(RecursiveDir)%(Filename)%(Extension)" />
+    <Compile Include="$(RepoRoot)/dotnet/src/InternalUtilities/src/Diagnostics/Verify.cs" Link="Diagnostics/Verify.cs" />
+    <Compile Include="$(RepoRoot)/dotnet/src/InternalUtilities/src/Diagnostics/NullableAttributes.cs" Link="Diagnostics/NullableAttributes.cs" />
+    <Compile Include="$(RepoRoot)/dotnet/src/InternalUtilities/src/Diagnostics/CompilerServicesAttributes.cs" Link="Diagnostics/CompilerServicesAttributes.cs" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\VectorData\VectorData.csproj" />
+  </ItemGroup>
+
+</Project>

dotnet/src/Connectors/VectorData.UnitTests/VectorSearch/KeywordHybridSearchBuilderExtensionsTests.cs

@@ -0,0 +1,24 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using Microsoft.Extensions.VectorData;
+using Moq;
+using Xunit;
+
+namespace VectorData.UnitTests;
+
+public class KeywordHybridSearchBuilderExtensionsTests
+{
+    [Fact]
+    public void AsBuilderReturnsKeywordHybridSearchBuilder()
+    {
+        // Arrange
+        var search = new Mock<IKeywordHybridSearch<string>>().Object;
+
+        // Act
+        var builder = search.AsBuilder();
+
+        // Assert
+        Assert.IsType<KeywordHybridSearchBuilder<string>>(builder);
+        Assert.Same(search, builder.Build());
+    }
+}