Merge branch 'main' into feature_single_agent_patterns_nochat

Author: crickman

dotnet/src/Connectors/Connectors.Memory.SqlServer/SqlServerClient.cs

@@ -33,18 +33,31 @@ public SqlServerClient(SqlConnection connection, string schema)
         this._schema = schema;
     }
 
+    private async Task<bool> HasJsonNativeTypeAsync(CancellationToken cancellationToken = default)
+    {
+        using (await this.OpenConnectionAsync(cancellationToken).ConfigureAwait(false))
+        {
+            using var cmd = this._connection.CreateCommand();
+            cmd.CommandText = "select [name] from sys.types where system_type_id = 244 and user_type_id = 244";
+            var typeName = (string)await cmd.ExecuteScalarAsync(cancellationToken).ConfigureAwait(false);
+            return string.Equals(typeName, "json", StringComparison.OrdinalIgnoreCase);
+        }
+    }
+
     /// <inheritdoc/>
     public async Task CreateTableAsync(string tableName, CancellationToken cancellationToken = default)
     {
         var fullTableName = this.GetSanitizedFullTableName(tableName);
+        var metadataType = await this.HasJsonNativeTypeAsync(cancellationToken).ConfigureAwait(false) ? "json" : "nvarchar(max)";
+
         using (await this.OpenConnectionAsync(cancellationToken).ConfigureAwait(false))
         {
             using var cmd = this._connection.CreateCommand();
             cmd.CommandText = $"""
                 IF OBJECT_ID(N'{fullTableName}', N'U') IS NULL
                 CREATE TABLE {fullTableName} (
                     [key] nvarchar(255) collate latin1_general_bin2 not null,
-                    [metadata] nvarchar(max) not null,
+                    [metadata] {metadataType} not null,
                     [embedding] varbinary(8000),
                     [timestamp] datetimeoffset,
                     PRIMARY KEY NONCLUSTERED ([key]),
@@ -138,9 +151,9 @@ WHEN NOT MATCHED THEN
     /// <inheritdoc/>
     public async IAsyncEnumerable<SqlServerMemoryEntry> ReadBatchAsync(string tableName, IEnumerable<string> keys, bool withEmbeddings = false, [EnumeratorCancellation] CancellationToken cancellationToken = default)
     {
-        var queryColumns = withEmbeddings
-            ? "[key], [metadata], [timestamp], VECTOR_TO_JSON_ARRAY([embedding]) AS [embedding]"
-            : "[key], [metadata], [timestamp]";
+        var queryColumns = "[key], [metadata], [timestamp]" +
+            (withEmbeddings ? ", VECTOR_TO_JSON_ARRAY([embedding]) AS [embedding]" : string.Empty);
+
         var fullTableName = this.GetSanitizedFullTableName(tableName);
         var keysList = keys.ToList();
         var keysParams = string.Join(", ", keysList.Select((_, i) => $"@k{i}"));
@@ -189,9 +202,8 @@ WHERE [key] IN ({keysParams})
     /// <inheritdoc/>
     public async IAsyncEnumerable<(SqlServerMemoryEntry, double)> GetNearestMatchesAsync(string tableName, ReadOnlyMemory<float> embedding, int limit, double minRelevanceScore = 0, bool withEmbeddings = false, [EnumeratorCancellation] CancellationToken cancellationToken = default)
     {
-        var queryColumns = withEmbeddings
-            ? "[key], [metadata], [timestamp], 1 - VECTOR_DISTANCE('cosine', [embedding], JSON_ARRAY_TO_VECTOR(@e)) AS [cosine_similarity], VECTOR_TO_JSON_ARRAY([embedding]) AS [embedding]"
-            : "[key], [metadata], [timestamp], 1 - VECTOR_DISTANCE('cosine', [embedding], JSON_ARRAY_TO_VECTOR(@e)) AS [cosine_similarity]";
+        var queryColumns = "[key], [metadata], [timestamp], 1 - VECTOR_DISTANCE('cosine', [embedding], JSON_ARRAY_TO_VECTOR(@e)) AS [cosine_similarity]" +
+            (withEmbeddings ? ", VECTOR_TO_JSON_ARRAY([embedding]) AS [embedding]" : string.Empty);
         var fullTableName = this.GetSanitizedFullTableName(tableName);
         using (await this.OpenConnectionAsync(cancellationToken).ConfigureAwait(false))
         {
@@ -221,6 +233,7 @@ ORDER BY [cosine_similarity] DESC
     private string GetSanitizedFullTableName(string tableName) => $"{DelimitIdentifier(this._schema)}.{DelimitIdentifier(tableName)}";
 
     private string SerializeEmbedding(ReadOnlyMemory<float> embedding) => JsonSerializer.Serialize(embedding);
+
     private ReadOnlyMemory<float> DeserializeEmbedding(string embedding) => JsonSerializer.Deserialize<ReadOnlyMemory<float>>(embedding);
 
     private SqlServerMemoryEntry ReadEntry(SqlDataReader reader, bool hasEmbedding)
@@ -247,6 +260,7 @@ private async Task<IDisposable> OpenConnectionAsync(CancellationToken cancellati
     }
 
     private static string DelimitIdentifier(string identifier) => $"[{EscapeIdentifier(identifier)}]";
+
     private static string EscapeIdentifier(string identifier) => identifier.Replace("]", "]]");
 
     private readonly struct Closer(SqlServerClient client, bool shouldClose) : IDisposable

dotnet/src/Connectors/Connectors.Memory.Sqlite/Database.cs

@@ -168,6 +168,27 @@ DELETE FROM {TableName}
         return cmd.ExecuteNonQueryAsync(cancellationToken);
     }
 
+    public Task DeleteBatchAsync(SqliteConnection conn, string collectionName, string[] keys, CancellationToken cancellationToken = default)
+    {
+        using SqliteCommand cmd = conn.CreateCommand();
+        var keyParameters = keys.Select((key, index) => $"@key{index}");
+        var parameters = string.Join(", ", keyParameters);
+
+#pragma warning disable CA2100 // Review SQL queries for security vulnerabilities
+        cmd.CommandText = $@"
+         DELETE FROM {TableName}
+         WHERE collection=@collection
+            AND key IN ({parameters})";
+#pragma warning restore CA2100 // Review SQL queries for security vulnerabilities
+
+        cmd.Parameters.Add(new SqliteParameter("@collection", collectionName));
+        for (int i = 0; i < keys.Length; i++)
+        {
+            cmd.Parameters.Add(new SqliteParameter($"@key{i}", keys[i]));
+        }
+        return cmd.ExecuteNonQueryAsync(cancellationToken);
+    }
+
     public Task DeleteEmptyAsync(SqliteConnection conn, string collectionName, CancellationToken cancellationToken = default)
     {
         using SqliteCommand cmd = conn.CreateCommand();

dotnet/src/Connectors/Connectors.Memory.Sqlite/SqliteMemoryStore.cs

@@ -113,7 +113,7 @@ public async Task RemoveAsync(string collectionName, string key, CancellationTok
     /// <inheritdoc/>
     public async Task RemoveBatchAsync(string collectionName, IEnumerable<string> keys, CancellationToken cancellationToken = default)
     {
-        await Task.WhenAll(keys.Select(k => this._dbConnector.DeleteAsync(this._dbConnection, collectionName, k, cancellationToken))).ConfigureAwait(false);
+        await this._dbConnector.DeleteBatchAsync(this._dbConnection, collectionName, keys.ToArray(), cancellationToken).ConfigureAwait(false);
     }
 
     /// <inheritdoc/>
@@ -218,8 +218,7 @@ private SqliteMemoryStore(string filename)
 
     private static DateTimeOffset? ParseTimestamp(string? str)
     {
-        if (!string.IsNullOrEmpty(str)
-            && DateTimeOffset.TryParse(str, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal, out DateTimeOffset timestamp))
+        if (DateTimeOffset.TryParse(str, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal, out DateTimeOffset timestamp))
         {
             return timestamp;
         }

dotnet/src/IntegrationTests/IntegrationTests.csproj

@@ -92,9 +92,6 @@
     <None Update="testsettings.json">
       <CopyToOutputDirectory>Always</CopyToOutputDirectory>
     </None>
-    <None Update="testsettings.development.json" Condition="Exists('testsettings.development.json')">
-      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
-    </None>
     <Content Include="CrossLanguage\Data\LightBulbApi.json">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </Content>