.Net: Represent all embeddings as ReadOnlyMemory<float> (#2419)

### Motivation and Context

Across AI-related libraries, we'd like a consistent representation of an
embedding. It needs to support efficient handling of the data, with the
ability to get a span over the data so that the data can be manipulated
efficiently, which rules out interfaces like `IEnumerable<float>` and
`IList<float>`. It should ideally encourage immutability but not enforce
it, such that developers have escape hatches that don't require them to
continually allocate and copy around large objects. It should be usable
in both synchronous and asynchronous contexts. And it should be a
ubiquitous core type that's available everywhere. Based on that,
`ReadOnlyMemory<float>` is the winning candidate.

### Description

This PR overhauls SK to use `ReadOnlyMemory<float>` for embeddings. In
addition to the advantages listed above, I think it also cleans things
up nicely.

Note that in the System.Text.Json v8.0.0 nuget package, ReadOnlyMemory
is implicitly supported. Once SK moves from having a v6 to a v8
dependency, we can delete the ReadOnlyMemoryConverter and all mentions
of it.

### Contribution Checklist

<!-- Before submitting this PR, please make sure: -->

- [x] The code builds clean without any errors or warnings
- [x] The PR follows the [SK Contribution
Guidelines](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md)
and the [pre-submission formatting
script](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md#development-scripts)
raises no violations
- [x] All unit tests pass, and I have added new tests where possible
- [ ] I didn't break anyone 😄

Co-authored-by: Shawn Callegari <[email protected]>

Author: stephentoub

dotnet/samples/KernelSyntaxExamples/Example25_ReadOnlyMemoryStore.cs

@@ -4,10 +4,10 @@
 using System.Collections.Generic;
 using System.Linq;
 using System.Runtime.CompilerServices;
+using System.Runtime.InteropServices;
 using System.Text.Json;
 using System.Threading;
 using System.Threading.Tasks;
-using Microsoft.SemanticKernel.AI.Embeddings;
 using Microsoft.SemanticKernel.AI.Embeddings.VectorOperations;
 using Microsoft.SemanticKernel.Memory;
 using Microsoft.SemanticKernel.Memory.Collections;
@@ -30,20 +30,20 @@ public static async Task RunAsync()
     {
         var store = new ReadOnlyMemoryStore(s_jsonVectorEntries);
 
-        var embedding = new Embedding<float>(new float[] { 22, 4, 6 });
+        var embedding = new ReadOnlyMemory<float>(new float[] { 22, 4, 6 });
 
         Console.WriteLine("Reading data from custom read-only memory store");
         var memoryRecord = await store.GetAsync("collection", "key3");
         if (memoryRecord != null)
         {
-            Console.WriteLine("ID = {0}, Embedding = {1}", memoryRecord.Metadata.Id, string.Join(", ", memoryRecord.Embedding.Vector));
+            Console.WriteLine("ID = {0}, Embedding = {1}", memoryRecord.Metadata.Id, string.Join(", ", MemoryMarshal.ToEnumerable(memoryRecord.Embedding)));
         }
 
-        Console.WriteLine("Getting most similar vector to {0}", string.Join(", ", embedding.Vector));
+        Console.WriteLine("Getting most similar vector to {0}", string.Join(", ", MemoryMarshal.ToEnumerable(embedding)));
         var result = await store.GetNearestMatchAsync("collection", embedding, 0.0);
         if (result.HasValue)
         {
-            Console.WriteLine("Embedding = {0}, Similarity = {1}", string.Join(", ", result.Value.Item1.Embedding.Vector), result.Value.Item2);
+            Console.WriteLine("Embedding = {0}, Similarity = {1}", string.Join(", ", MemoryMarshal.ToEnumerable(result.Value.Item1.Embedding)), result.Value.Item2);
         }
     }
 
@@ -105,7 +105,7 @@ public IAsyncEnumerable<string> GetCollectionsAsync(CancellationToken cancellati
             throw new System.NotImplementedException();
         }
 
-        public async Task<(MemoryRecord, double)?> GetNearestMatchAsync(string collectionName, Embedding<float> embedding, double minRelevanceScore = 0,
+        public async Task<(MemoryRecord, double)?> GetNearestMatchAsync(string collectionName, ReadOnlyMemory<float> embedding, double minRelevanceScore = 0,
             bool withEmbedding = false, CancellationToken cancellationToken = default)
         {
             // Note: with this simple implementation, the MemoryRecord will always contain the embedding.
@@ -123,7 +123,7 @@ public IAsyncEnumerable<string> GetCollectionsAsync(CancellationToken cancellati
             return default;
         }
 
-        public async IAsyncEnumerable<(MemoryRecord, double)> GetNearestMatchesAsync(string collectionName, Embedding<float> embedding, int limit,
+        public async IAsyncEnumerable<(MemoryRecord, double)> GetNearestMatchesAsync(string collectionName, ReadOnlyMemory<float> embedding, int limit,
             double minRelevanceScore = 0, bool withEmbeddings = false, [EnumeratorCancellation] CancellationToken cancellationToken = default)
         {
             // Note: with this simple implementation, the MemoryRecord will always contain the embedding.
@@ -132,16 +132,16 @@ public IAsyncEnumerable<string> GetCollectionsAsync(CancellationToken cancellati
                 yield break;
             }
 
-            if (embedding.Count != this._vectorSize)
+            if (embedding.Length != this._vectorSize)
             {
-                throw new Exception($"Embedding vector size {embedding.Count} does not match expected size of {this._vectorSize}");
+                throw new Exception($"Embedding vector size {embedding.Length} does not match expected size of {this._vectorSize}");
             }
 
             TopNCollection<MemoryRecord> embeddings = new(limit);
 
             foreach (var item in this._memoryRecords)
             {
-                double similarity = embedding.AsReadOnlySpan().CosineSimilarity(item.Embedding.AsReadOnlySpan());
+                double similarity = embedding.Span.CosineSimilarity(item.Embedding.Span);
                 if (similarity >= minRelevanceScore)
                 {
                     embeddings.Add(new(item, similarity));

dotnet/src/Connectors/Connectors.AI.HuggingFace/TextEmbedding/HuggingFaceTextEmbeddingGeneration.cs

@@ -81,7 +81,7 @@ public HuggingFaceTextEmbeddingGeneration(string model, HttpClient httpClient, s
     }
 
     /// <inheritdoc/>
-    public async Task<IList<Embedding<float>>> GenerateEmbeddingsAsync(IList<string> data, CancellationToken cancellationToken = default)
+    public async Task<IList<ReadOnlyMemory<float>>> GenerateEmbeddingsAsync(IList<string> data, CancellationToken cancellationToken = default)
     {
         return await this.ExecuteEmbeddingRequestAsync(data, cancellationToken).ConfigureAwait(false);
     }
@@ -95,7 +95,7 @@ public async Task<IList<Embedding<float>>> GenerateEmbeddingsAsync(IList<string>
     /// <param name="cancellationToken">The <see cref="CancellationToken"/> to monitor for cancellation requests. The default is <see cref="CancellationToken.None"/>.</param>
     /// <returns>List of generated embeddings.</returns>
     /// <exception cref="AIException">Exception when backend didn't respond with generated embeddings.</exception>
-    private async Task<IList<Embedding<float>>> ExecuteEmbeddingRequestAsync(IList<string> data, CancellationToken cancellationToken)
+    private async Task<IList<ReadOnlyMemory<float>>> ExecuteEmbeddingRequestAsync(IList<string> data, CancellationToken cancellationToken)
     {
         try
         {
@@ -113,7 +113,7 @@ private async Task<IList<Embedding<float>>> ExecuteEmbeddingRequestAsync(IList<s
 
             var embeddingResponse = JsonSerializer.Deserialize<TextEmbeddingResponse>(body);
 
-            return embeddingResponse?.Embeddings?.Select(l => new Embedding<float>(l.Embedding!, transferOwnership: true)).ToList()!;
+            return embeddingResponse?.Embeddings?.Select(l => l.Embedding).ToList()!;
         }
         catch (Exception e) when (e is not AIException && !e.IsCriticalException())
         {

dotnet/src/Connectors/Connectors.AI.HuggingFace/TextEmbedding/TextEmbeddingResponse.cs

@@ -1,7 +1,9 @@
 // Copyright (c) Microsoft. All rights reserved.
 
+using System;
 using System.Collections.Generic;
 using System.Text.Json.Serialization;
+using Microsoft.SemanticKernel.Text;
 
 namespace Microsoft.SemanticKernel.Connectors.AI.HuggingFace.TextEmbedding;
 
@@ -16,7 +18,8 @@ public sealed class TextEmbeddingResponse
     public sealed class EmbeddingVector
     {
         [JsonPropertyName("embedding")]
-        public IList<float>? Embedding { get; set; }
+        [JsonConverter(typeof(ReadOnlyMemoryConverter))]
+        public ReadOnlyMemory<float> Embedding { get; set; }
     }
 
     /// <summary>

dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/ClientBase.cs

@@ -12,7 +12,6 @@
 using Microsoft.Extensions.Logging.Abstractions;
 using Microsoft.SemanticKernel.AI;
 using Microsoft.SemanticKernel.AI.ChatCompletion;
-using Microsoft.SemanticKernel.AI.Embeddings;
 using Microsoft.SemanticKernel.AI.TextCompletion;
 using Microsoft.SemanticKernel.Connectors.AI.OpenAI.ChatCompletion;
 using Microsoft.SemanticKernel.Diagnostics;
@@ -115,11 +114,11 @@ private protected async IAsyncEnumerable<TextStreamingResult> InternalGetTextStr
     /// <param name="data">List of strings to generate embeddings for</param>
     /// <param name="cancellationToken">The <see cref="CancellationToken"/> to monitor for cancellation requests. The default is <see cref="CancellationToken.None"/>.</param>
     /// <returns>List of embeddings</returns>
-    private protected async Task<IList<Embedding<float>>> InternalGetEmbeddingsAsync(
+    private protected async Task<IList<ReadOnlyMemory<float>>> InternalGetEmbeddingsAsync(
         IList<string> data,
         CancellationToken cancellationToken = default)
     {
-        var result = new List<Embedding<float>>();
+        var result = new List<ReadOnlyMemory<float>>(data.Count);
         foreach (string text in data)
         {
             var options = new EmbeddingsOptions(text);
@@ -137,9 +136,7 @@ private protected async Task<IList<Embedding<float>>> InternalGetEmbeddingsAsync
                 throw new OpenAIInvalidResponseException<Embeddings>(response.Value, "Text embedding not found");
             }
 
-            EmbeddingItem x = response.Value.Data[0];
-
-            result.Add(new Embedding<float>(x.Embedding, transferOwnership: true));
+            result.Add(response.Value.Data[0].Embedding.ToArray());
         }
 
         return result;

dotnet/src/Connectors/Connectors.AI.OpenAI/CustomClient/OpenAIClientBase.cs

@@ -12,7 +12,6 @@
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Logging.Abstractions;
 using Microsoft.SemanticKernel.AI;
-using Microsoft.SemanticKernel.AI.Embeddings;
 using Microsoft.SemanticKernel.Connectors.AI.OpenAI.ImageGeneration;
 using Microsoft.SemanticKernel.Connectors.AI.OpenAI.TextEmbedding;
 using Microsoft.SemanticKernel.Diagnostics;
@@ -48,7 +47,7 @@ private protected virtual void AddRequestHeaders(HttpRequestMessage request)
     /// <param name="cancellationToken">The <see cref="CancellationToken"/> to monitor for cancellation requests. The default is <see cref="CancellationToken.None"/>.</param>
     /// <returns>List of text embeddings</returns>
     /// <exception cref="AIException">AIException thrown during the request.</exception>
-    private protected async Task<IList<Embedding<float>>> ExecuteTextEmbeddingRequestAsync(
+    private protected async Task<IList<ReadOnlyMemory<float>>> ExecuteTextEmbeddingRequestAsync(
         string url,
         string requestBody,
         CancellationToken cancellationToken = default)
@@ -61,7 +60,7 @@ private protected async Task<IList<Embedding<float>>> ExecuteTextEmbeddingReques
                 "Embeddings not found");
         }
 
-        return result.Embeddings.Select(e => new Embedding<float>(e.Values, transferOwnership: true)).ToList();
+        return result.Embeddings.Select(e => e.Values).ToList();
     }
 
     /// <summary>

dotnet/src/Connectors/Connectors.AI.OpenAI/TextEmbedding/AzureTextEmbeddingGeneration.cs

@@ -1,5 +1,6 @@
 // Copyright (c) Microsoft. All rights reserved.
 
+using System;
 using System.Collections.Generic;
 using System.Net.Http;
 using System.Threading;
@@ -56,7 +57,7 @@ public AzureTextEmbeddingGeneration(
     /// <param name="data">List of strings to generate embeddings for</param>
     /// <param name="cancellationToken">The <see cref="CancellationToken"/> to monitor for cancellation requests. The default is <see cref="CancellationToken.None"/>.</param>
     /// <returns>List of embeddings</returns>
-    public Task<IList<Embedding<float>>> GenerateEmbeddingsAsync(
+    public Task<IList<ReadOnlyMemory<float>>> GenerateEmbeddingsAsync(
         IList<string> data,
         CancellationToken cancellationToken = default)
     {

dotnet/src/Connectors/Connectors.AI.OpenAI/TextEmbedding/OpenAITextEmbeddingGeneration.cs

@@ -1,5 +1,6 @@
 // Copyright (c) Microsoft. All rights reserved.
 
+using System;
 using System.Collections.Generic;
 using System.Net.Http;
 using System.Threading;
@@ -39,7 +40,7 @@ public OpenAITextEmbeddingGeneration(
     /// <param name="data">List of strings to generate embeddings for</param>
     /// <param name="cancellationToken">The <see cref="CancellationToken"/> to monitor for cancellation requests. The default is <see cref="CancellationToken.None"/>.</param>
     /// <returns>List of embeddings</returns>
-    public Task<IList<Embedding<float>>> GenerateEmbeddingsAsync(
+    public Task<IList<ReadOnlyMemory<float>>> GenerateEmbeddingsAsync(
         IList<string> data,
         CancellationToken cancellationToken = default)
     {

dotnet/src/Connectors/Connectors.AI.OpenAI/TextEmbedding/TextEmbeddingResponse.cs

@@ -1,7 +1,9 @@
 // Copyright (c) Microsoft. All rights reserved.
 
+using System;
 using System.Collections.Generic;
 using System.Text.Json.Serialization;
+using Microsoft.SemanticKernel.Text;
 
 namespace Microsoft.SemanticKernel.Connectors.AI.OpenAI.TextEmbedding;
 
@@ -19,7 +21,8 @@ public sealed class EmbeddingResponseIndex
         /// The embedding vector
         /// </summary>
         [JsonPropertyName("embedding")]
-        public IList<float> Values { get; set; } = new List<float>();
+        [JsonConverter(typeof(ReadOnlyMemoryConverter))]
+        public ReadOnlyMemory<float> Values { get; set; }
 
         /// <summary>
         /// Index of the embedding vector

dotnet/src/Connectors/Connectors.Memory.AzureCognitiveSearch/AzureCognitiveSearchMemoryRecord.cs

@@ -1,12 +1,10 @@
 // Copyright (c) Microsoft. All rights reserved.
 
 using System;
-using System.Collections.Generic;
-using System.Linq;
 using System.Text;
 using System.Text.Json.Serialization;
-using Microsoft.SemanticKernel.AI.Embeddings;
 using Microsoft.SemanticKernel.Memory;
+using Microsoft.SemanticKernel.Text;
 
 namespace Microsoft.SemanticKernel.Connectors.Memory.AzureCognitiveSearch;
 
@@ -41,7 +39,8 @@ public class AzureCognitiveSearchMemoryRecord
     /// Content embedding
     /// </summary>
     [JsonPropertyName(EmbeddingField)]
-    public List<float> Embedding { get; set; } = Array.Empty<float>().ToList();
+    [JsonConverter(typeof(ReadOnlyMemoryConverter))]
+    public ReadOnlyMemory<float> Embedding { get; set; }
 
     /// <summary>
     /// Optional description of the content, e.g. a title. This can be useful when
@@ -87,13 +86,13 @@ public AzureCognitiveSearchMemoryRecord(
         string text,
         string externalSourceName,
         bool isReference,
-        Embedding<float> embedding,
+        ReadOnlyMemory<float> embedding,
         string? description = null,
         string? additionalMetadata = null)
     {
         this.Id = EncodeId(id);
         this.IsReference = isReference;
-        this.Embedding = embedding.Vector.ToList();
+        this.Embedding = embedding;
         this.Text = text;
         this.ExternalSourceName = externalSourceName;
         this.Description = description;
@@ -128,7 +127,7 @@ public MemoryRecord ToMemoryRecord(bool withEmbeddings = true)
     {
         return new MemoryRecord(
             metadata: this.ToMemoryRecordMetadata(),
-            embedding: new Embedding<float>(withEmbeddings ? this.Embedding : Array.Empty<float>()),
+            embedding: withEmbeddings ? this.Embedding : default,
             key: this.Id);
     }
 

dotnet/src/Connectors/Connectors.Memory.AzureCognitiveSearch/AzureCognitiveSearchMemoryStore.cs

@@ -5,6 +5,7 @@
 using System.Collections.Generic;
 using System.Linq;
 using System.Runtime.CompilerServices;
+using System.Runtime.InteropServices;
 using System.Text.RegularExpressions;
 using System.Threading;
 using System.Threading.Tasks;
@@ -14,7 +15,6 @@
 using Azure.Search.Documents.Indexes;
 using Azure.Search.Documents.Indexes.Models;
 using Azure.Search.Documents.Models;
-using Microsoft.SemanticKernel.AI.Embeddings;
 using Microsoft.SemanticKernel.Diagnostics;
 using Microsoft.SemanticKernel.Memory;
 
@@ -134,7 +134,7 @@ public async IAsyncEnumerable<MemoryRecord> GetBatchAsync(
     /// <inheritdoc />
     public async Task<(MemoryRecord, double)?> GetNearestMatchAsync(
         string collectionName,
-        Embedding<float> embedding,
+        ReadOnlyMemory<float> embedding,
         double minRelevanceScore = 0,
         bool withEmbedding = false,
         CancellationToken cancellationToken = default)
@@ -147,7 +147,7 @@ public async IAsyncEnumerable<MemoryRecord> GetBatchAsync(
     /// <inheritdoc />
     public async IAsyncEnumerable<(MemoryRecord, double)> GetNearestMatchesAsync(
         string collectionName,
-        Embedding<float> embedding,
+        ReadOnlyMemory<float> embedding,
         int limit,
         double minRelevanceScore = 0,
         bool withEmbeddings = false,
@@ -161,7 +161,7 @@ public async IAsyncEnumerable<MemoryRecord> GetBatchAsync(
         {
             KNearestNeighborsCount = limit,
             Fields = AzureCognitiveSearchMemoryRecord.EmbeddingField,
-            Value = embedding.Vector.ToList()
+            Value = MemoryMarshal.TryGetArray(embedding, out var array) && array.Count == embedding.Length ? array.Array! : embedding.ToArray(),
         };
 
         SearchOptions options = new() { Vector = vectorQuery };
@@ -305,7 +305,7 @@ private async Task<List<string>> UpsertBatchAsync(
 
         if (records.Count < 1) { return keys; }
 
-        var embeddingSize = records[0].Embedding.Count;
+        var embeddingSize = records[0].Embedding.Length;
 
         var client = this.GetSearchClient(indexName);
 

dotnet/src/Connectors/Connectors.Memory.Chroma/ChromaClient.cs

@@ -107,7 +107,7 @@ public async IAsyncEnumerable<string> ListCollectionsAsync([EnumeratorCancellati
     }
 
     /// <inheritdoc />
-    public async Task UpsertEmbeddingsAsync(string collectionId, string[] ids, float[][] embeddings, object[]? metadatas = null, CancellationToken cancellationToken = default)
+    public async Task UpsertEmbeddingsAsync(string collectionId, string[] ids, ReadOnlyMemory<float>[] embeddings, object[]? metadatas = null, CancellationToken cancellationToken = default)
     {
         this._logger.LogDebug("Upserting embeddings to collection with id: {0}", collectionId);
 
@@ -141,7 +141,7 @@ public async Task DeleteEmbeddingsAsync(string collectionId, string[] ids, Cance
     }
 
     /// <inheritdoc />
-    public async Task<ChromaQueryResultModel> QueryEmbeddingsAsync(string collectionId, float[][] queryEmbeddings, int nResults, string[]? include = null, CancellationToken cancellationToken = default)
+    public async Task<ChromaQueryResultModel> QueryEmbeddingsAsync(string collectionId, ReadOnlyMemory<float>[] queryEmbeddings, int nResults, string[]? include = null, CancellationToken cancellationToken = default)
     {
         this._logger.LogDebug("Query embeddings in collection with id: {0}", collectionId);