Merge pull request #598 from iceljc/features/refine-vector-db

iceljc · web-flow · commit 89fab1259c93 · 2024-08-20T13:32:42.000-05:00
1. split knowledge menu
diff --git a/src/Infrastructure/BotSharp.Abstraction/Plugins/Models/PluginMenuDef.cs b/src/Infrastructure/BotSharp.Abstraction/Plugins/Models/PluginMenuDef.cs
@@ -25,9 +25,9 @@ public class PluginMenuDef
     [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
     public List<PluginMenuDef>? SubMenu {  get; set; }
 
-    public PluginMenuDef(string lable, string? link = null, string? icon = null, int weight = 0) 
+    public PluginMenuDef(string label, string? link = null, string? icon = null, int weight = 0) 
     { 
-        Label = lable;
+        Label = label;
         Link = link;
         Icon = icon;
         Weight = weight;
diff --git a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs
@@ -9,7 +9,7 @@ public interface IVectorDb
     Task<IEnumerable<string>> GetCollections();
     Task<StringIdPagedItems<VectorCollectionData>> GetCollectionData(string collectionName, VectorFilter filter);
     Task CreateCollection(string collectionName, int dim);
-    Task<bool> Upsert(string collectionName, string id, float[] vector, string text, Dictionary<string, string>? payload = null);
+    Task<bool> Upsert(string collectionName, Guid id, float[] vector, string text, Dictionary<string, string>? payload = null);
     Task<IEnumerable<VectorCollectionData>> Search(string collectionName, float[] vector, IEnumerable<string>? fields, int limit = 5, float confidence = 0.5f, bool withVector = false);
-    Task<bool> DeleteCollectionData(string collectionName, string id);
+    Task<bool> DeleteCollectionData(string collectionName, Guid id);
 }
diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/MemorizeKnowledgeFn.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/MemorizeKnowledgeFn.cs
@@ -29,9 +29,8 @@ public async Task<bool> Execute(RoleDialogModel message)
         var collectionName = !string.IsNullOrWhiteSpace(_settings.DefaultCollection) ? _settings.DefaultCollection : KnowledgeCollectionName.BotSharp;
         await vectorDb.CreateCollection(collectionName, vector[0].Length);
 
-        var id = Guid.NewGuid().ToString();
-        var result = await vectorDb.Upsert(collectionName, id, vector[0], 
-            args.Question, 
+        var result = await vectorDb.Upsert(collectionName, Guid.NewGuid(), vector[0], 
+            args.Question,
             new Dictionary<string, string> 
             { 
                 { KnowledgePayloadName.Answer, args.Answer } 
diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/KnowledgeBasePlugin.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/KnowledgeBasePlugin.cs
@@ -20,8 +20,6 @@ public void RegisterDI(IServiceCollection services, IConfiguration config)
             return settingService.Bind<KnowledgeBaseSettings>("KnowledgeBase");
         });
 
-        var a = config["KnowledgeBase"];
-
         services.AddScoped<ITextChopper, TextChopperService>();
         services.AddScoped<IKnowledgeService, KnowledgeService>();
         services.AddSingleton<IPdf2TextConverter, PigPdf2TextConverter>();
@@ -32,7 +30,14 @@ public void RegisterDI(IServiceCollection services, IConfiguration config)
     public bool AttachMenu(List<PluginMenuDef> menu)
     {
         var section = menu.First(x => x.Label == "Apps");
-        menu.Add(new PluginMenuDef("Knowledge Base", link: "/page/knowledge-base", icon: "bx bx-book-open", weight: section.Weight + 1));
+        menu.Add(new PluginMenuDef("Knowledge Base", icon: "bx bx-book-open", weight: section.Weight + 1)
+        {
+            SubMenu = new List<PluginMenuDef>
+            {
+                new PluginMenuDef("Vector", link: "page/knowledge-base/vector"),
+                new PluginMenuDef("Graph", link: "page/knowledge-base/graph")
+            }
+        });
         return true;
     }
 }
diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemoryVectorDb.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemoryVectorDb.cs
@@ -53,19 +53,19 @@ public async Task<IEnumerable<VectorCollectionData>> Search(string collectionNam
         return await Task.FromResult(results);
     }
 
-    public async Task<bool> Upsert(string collectionName, string id, float[] vector, string text, Dictionary<string, string>? payload = null)
+    public async Task<bool> Upsert(string collectionName, Guid id, float[] vector, string text, Dictionary<string, string>? payload = null)
     {
         _vectors[collectionName].Add(new VecRecord
         {
-            Id = id,
+            Id = id.ToString(),
             Vector = vector,
             Text = text
         });
 
         return true;
     }
 
-    public async Task<bool> DeleteCollectionData(string collectionName, string id)
+    public async Task<bool> DeleteCollectionData(string collectionName, Guid id)
     {
         return await Task.FromResult(false);
     }
diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Create.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Create.cs
@@ -19,8 +19,7 @@ public async Task FeedVectorKnowledge(string collectionName, KnowledgeCreationMo
         foreach (var line in lines)
         {
             var vec = await textEmbedding.GetVectorAsync(line);
-            var id = Guid.NewGuid().ToString();
-            await db.Upsert(collectionName, id, vec, line);
+            await db.Upsert(collectionName, Guid.NewGuid(), vec, line);
             index++;
             Console.WriteLine($"Saved vector {index}/{lines.Count}: {line}\n");
         }
diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Delete.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Delete.cs
@@ -6,8 +6,13 @@ public async Task<bool> DeleteVectorCollectionData(string collectionName, string
     {
         try
         {
+            if (!Guid.TryParse(id, out var guid))
+            {
+                return false;
+            }
+
             var db = GetVectorDb();
-            return await db.DeleteCollectionData(collectionName, id);
+            return await db.DeleteCollectionData(collectionName, guid);
         }
         catch (Exception ex)
         {
diff --git a/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs b/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs
@@ -32,12 +32,12 @@ public Task<IEnumerable<VectorCollectionData>> Search(string collectionName, flo
         throw new NotImplementedException();
     }
 
-    public Task<bool> Upsert(string collectionName, string id, float[] vector, string text, Dictionary<string, string>? payload = null)
+    public Task<bool> Upsert(string collectionName, Guid id, float[] vector, string text, Dictionary<string, string>? payload = null)
     {
         throw new NotImplementedException();
     }
 
-    public Task<bool> DeleteCollectionData(string collectionName, string id)
+    public Task<bool> DeleteCollectionData(string collectionName, Guid id)
     {
         throw new NotImplementedException();
     }
diff --git a/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs b/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs
@@ -92,14 +92,14 @@ public async Task CreateCollection(string collectionName, int dim)
         }
     }
 
-    public async Task<bool> Upsert(string collectionName, string id, float[] vector, string text, Dictionary<string, string>? payload = null)
+    public async Task<bool> Upsert(string collectionName, Guid id, float[] vector, string text, Dictionary<string, string>? payload = null)
     {
         // Insert vectors
         var point = new PointStruct()
         {
             Id = new PointId()
             {
-                Uuid = id
+                Uuid = id.ToString()
             },
             Vectors = vector,
             Payload =
@@ -137,7 +137,11 @@ public async Task<IEnumerable<VectorCollectionData>> Search(string collectionNam
             return results;
         }
 
-        var points = await client.SearchAsync(collectionName, vector, limit: (ulong)limit, scoreThreshold: confidence);
+        var points = await client.SearchAsync(collectionName,
+            vector,
+            limit: (ulong)limit,
+            scoreThreshold: confidence,
+            vectorsSelector: new WithVectorsSelector { Enable = withVector });
 
         var pickFields = fields != null;
         foreach (var point in points)
@@ -174,15 +178,10 @@ public async Task<IEnumerable<VectorCollectionData>> Search(string collectionNam
         return results;
     }
 
-    public async Task<bool> DeleteCollectionData(string collectionName, string id)
+    public async Task<bool> DeleteCollectionData(string collectionName, Guid id)
     {
-        if (!Guid.TryParse(id, out var guid))
-        {
-            return false;
-        }
-
         var client = GetClient();
-        var result = await client.DeleteAsync(collectionName, guid);
+        var result = await client.DeleteAsync(collectionName, id);
         return result.Status == UpdateStatus.Completed;
     }
 
diff --git a/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs b/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs
@@ -2,6 +2,7 @@
 using BotSharp.Abstraction.VectorStorage;
 using BotSharp.Abstraction.VectorStorage.Models;
 using Microsoft.SemanticKernel.Memory;
+using System;
 using System.Collections.Generic;
 using System.Threading.Tasks;
 
@@ -62,21 +63,21 @@ public async Task<IEnumerable<VectorCollectionData>> Search(string collectionNam
             return resultTexts;
         }
 
-        public async Task<bool> Upsert(string collectionName, string id, float[] vector, string text, Dictionary<string, string>? payload)
+        public async Task<bool> Upsert(string collectionName, Guid id, float[] vector, string text, Dictionary<string, string>? payload)
         {
 #pragma warning disable SKEXP0001 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed.
             await _memoryStore.UpsertAsync(collectionName, MemoryRecord.LocalRecord(id.ToString(), text, null, vector));
 #pragma warning restore SKEXP0001 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed.
             return true;
         }
 
-        public async Task<bool> DeleteCollectionData(string collectionName, string id)
+        public async Task<bool> DeleteCollectionData(string collectionName, Guid id)
         {
             var exist = await _memoryStore.DoesCollectionExistAsync(collectionName);
 
             if (exist)
             {
-                await _memoryStore.RemoveAsync(collectionName, id);
+                await _memoryStore.RemoveAsync(collectionName, id.ToString());
                 return true;
             }
             return false;

Original file line number	Diff line number	Diff line change
`@@ -53,19 +53,19 @@ public async Task<IEnumerable<VectorCollectionData>> Search(string collectionNam`
`53`	`53`	`return await Task.FromResult(results);`
`54`	`54`	`}`
`55`	`55`
`56`		`- public async Task<bool> Upsert(string collectionName, string id, float[] vector, string text, Dictionary<string, string>? payload = null)`
	`56`	`+ public async Task<bool> Upsert(string collectionName, Guid id, float[] vector, string text, Dictionary<string, string>? payload = null)`
`57`	`57`	`{`
`58`	`58`	`_vectors[collectionName].Add(new VecRecord`
`59`	`59`	`{`
`60`		`- Id = id,`
	`60`	`+ Id = id.ToString(),`
`61`	`61`	`Vector = vector,`
`62`	`62`	`Text = text`
`63`	`63`	`});`
`64`	`64`
`65`	`65`	`return true;`
`66`	`66`	`}`
`67`	`67`
`68`		`- public async Task<bool> DeleteCollectionData(string collectionName, string id)`
	`68`	`+ public async Task<bool> DeleteCollectionData(string collectionName, Guid id)`
`69`	`69`	`{`
`70`	`70`	`return await Task.FromResult(false);`
`71`	`71`	`}`
Original file line number	Diff line number	Diff line change
`@@ -19,8 +19,7 @@ public async Task FeedVectorKnowledge(string collectionName, KnowledgeCreationMo`
`19`	`19`	`foreach (var line in lines)`
`20`	`20`	`{`
`21`	`21`	`var vec = await textEmbedding.GetVectorAsync(line);`
`22`		`- var id = Guid.NewGuid().ToString();`
`23`		`- await db.Upsert(collectionName, id, vec, line);`
	`22`	`+ await db.Upsert(collectionName, Guid.NewGuid(), vec, line);`
`24`	`23`	`index++;`
`25`	`24`	`Console.WriteLine($"Saved vector {index}/{lines.Count}: {line}\n");`
`26`	`25`	`}`
Original file line number	Diff line number	Diff line change
`@@ -6,8 +6,13 @@ public async Task<bool> DeleteVectorCollectionData(string collectionName, string`
`6`	`6`	`{`
`7`	`7`	`try`
`8`	`8`	`{`
	`9`	`+ if (!Guid.TryParse(id, out var guid))`
	`10`	`+ {`
	`11`	`+ return false;`
	`12`	`+ }`
	`13`	`+`
`9`	`14`	`var db = GetVectorDb();`
`10`		`- return await db.DeleteCollectionData(collectionName, id);`
	`15`	`+ return await db.DeleteCollectionData(collectionName, guid);`
`11`	`16`	`}`
`12`	`17`	`catch (Exception ex)`
`13`	`18`	`{`
Original file line number	Diff line number	Diff line change
`@@ -32,12 +32,12 @@ public Task<IEnumerable<VectorCollectionData>> Search(string collectionName, flo`
`32`	`32`	`throw new NotImplementedException();`
`33`	`33`	`}`
`34`	`34`
`35`		`- public Task<bool> Upsert(string collectionName, string id, float[] vector, string text, Dictionary<string, string>? payload = null)`
	`35`	`+ public Task<bool> Upsert(string collectionName, Guid id, float[] vector, string text, Dictionary<string, string>? payload = null)`
`36`	`36`	`{`
`37`	`37`	`throw new NotImplementedException();`
`38`	`38`	`}`
`39`	`39`
`40`		`- public Task<bool> DeleteCollectionData(string collectionName, string id)`
	`40`	`+ public Task<bool> DeleteCollectionData(string collectionName, Guid id)`
`41`	`41`	`{`
`42`	`42`	`throw new NotImplementedException();`
`43`	`43`	`}`
Original file line number	Diff line number	Diff line change
`@@ -92,14 +92,14 @@ public async Task CreateCollection(string collectionName, int dim)`
`92`	`92`	`}`
`93`	`93`	`}`
`94`	`94`
`95`		`- public async Task<bool> Upsert(string collectionName, string id, float[] vector, string text, Dictionary<string, string>? payload = null)`
	`95`	`+ public async Task<bool> Upsert(string collectionName, Guid id, float[] vector, string text, Dictionary<string, string>? payload = null)`
`96`	`96`	`{`
`97`	`97`	`// Insert vectors`
`98`	`98`	`var point = new PointStruct()`
`99`	`99`	`{`
`100`	`100`	`Id = new PointId()`
`101`	`101`	`{`
`102`		`- Uuid = id`
	`102`	`+ Uuid = id.ToString()`
`103`	`103`	`},`
`104`	`104`	`Vectors = vector,`
`105`	`105`	`Payload =`
`@@ -137,7 +137,11 @@ public async Task<IEnumerable<VectorCollectionData>> Search(string collectionNam`
`137`	`137`	`return results;`
`138`	`138`	`}`
`139`	`139`
`140`		`- var points = await client.SearchAsync(collectionName, vector, limit: (ulong)limit, scoreThreshold: confidence);`
	`140`	`+ var points = await client.SearchAsync(collectionName,`
	`141`	`+ vector,`
	`142`	`+ limit: (ulong)limit,`
	`143`	`+ scoreThreshold: confidence,`
	`144`	`+ vectorsSelector: new WithVectorsSelector { Enable = withVector });`
`141`	`145`
`142`	`146`	`var pickFields = fields != null;`
`143`	`147`	`foreach (var point in points)`
`@@ -174,15 +178,10 @@ public async Task<IEnumerable<VectorCollectionData>> Search(string collectionNam`
`174`	`178`	`return results;`
`175`	`179`	`}`
`176`	`180`
`177`		`- public async Task<bool> DeleteCollectionData(string collectionName, string id)`
	`181`	`+ public async Task<bool> DeleteCollectionData(string collectionName, Guid id)`
`178`	`182`	`{`
`179`		`- if (!Guid.TryParse(id, out var guid))`
`180`		`- {`
`181`		`- return false;`
`182`		`- }`
`183`		`-`
`184`	`183`	`var client = GetClient();`
`185`		`- var result = await client.DeleteAsync(collectionName, guid);`
	`184`	`+ var result = await client.DeleteAsync(collectionName, id);`
`186`	`185`	`return result.Status == UpdateStatus.Completed;`
`187`	`186`	`}`
`188`	`187`
Original file line number	Diff line number	Diff line change
`@@ -2,6 +2,7 @@`
`2`	`2`	`using BotSharp.Abstraction.VectorStorage;`
`3`	`3`	`using BotSharp.Abstraction.VectorStorage.Models;`
`4`	`4`	`using Microsoft.SemanticKernel.Memory;`
	`5`	`+using System;`
`5`	`6`	`using System.Collections.Generic;`
`6`	`7`	`using System.Threading.Tasks;`
`7`	`8`
`@@ -62,21 +63,21 @@ public async Task<IEnumerable<VectorCollectionData>> Search(string collectionNam`
`62`	`63`	`return resultTexts;`
`63`	`64`	`}`
`64`	`65`
`65`		`- public async Task<bool> Upsert(string collectionName, string id, float[] vector, string text, Dictionary<string, string>? payload)`
	`66`	`+ public async Task<bool> Upsert(string collectionName, Guid id, float[] vector, string text, Dictionary<string, string>? payload)`
`66`	`67`	`{`
`67`	`68`	`#pragma warning disable SKEXP0001 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed.`
`68`	`69`	`await _memoryStore.UpsertAsync(collectionName, MemoryRecord.LocalRecord(id.ToString(), text, null, vector));`
`69`	`70`	`#pragma warning restore SKEXP0001 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed.`
`70`	`71`	`return true;`
`71`	`72`	`}`
`72`	`73`
`73`		`- public async Task<bool> DeleteCollectionData(string collectionName, string id)`
	`74`	`+ public async Task<bool> DeleteCollectionData(string collectionName, Guid id)`
`74`	`75`	`{`
`75`	`76`	`var exist = await _memoryStore.DoesCollectionExistAsync(collectionName);`
`76`	`77`
`77`	`78`	`if (exist)`
`78`	`79`	`{`
`79`		`- await _memoryStore.RemoveAsync(collectionName, id);`
	`80`	`+ await _memoryStore.RemoveAsync(collectionName, id.ToString());`
`80`	`81`	`return true;`
`81`	`82`	`}`
`82`	`83`	`return false;`