Merge pull request #647 from iceljc/features/add-knowledge-docs

iceljc · web-flow · commit b51399c17e46 · 2024-09-18T10:45:49.000-05:00
Features/add knowledge docs
diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs b/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs
@@ -74,6 +74,6 @@ public interface IFileStorageService
     /// <returns></returns>
     bool DeleteKnowledgeFile(string collectionName, string vectorStoreProvider, Guid? fileId = null);
     string GetKnowledgeBaseFileUrl(string collectionName, string vectorStoreProvider, Guid fileId, string fileName);
-    BinaryData? GetKnowledgeBaseFileBinaryData(string collectionName, string vectorStoreProvider, Guid fileId, string fileName);
+    BinaryData GetKnowledgeBaseFileBinaryData(string collectionName, string vectorStoreProvider, Guid fileId, string fileName);
     #endregion
 }
diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Models/KnowledgeFileModel.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Models/KnowledgeFileModel.cs
@@ -7,4 +7,5 @@ public class KnowledgeFileModel
     public string FileExtension { get; set; }
     public string ContentType { get; set; }
     public string FileUrl { get; set; }
+    public DocMetaRefData? RefData { get; set; }
 }
diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs
@@ -24,7 +24,7 @@ public interface IKnowledgeService
     Task<UploadKnowledgeResponse> UploadKnowledgeDocuments(string collectionName, IEnumerable<ExternalFileModel> files);
     Task<bool> DeleteKnowledgeDocument(string collectionName, Guid fileId);
     Task<PagedItems<KnowledgeFileModel>> GetPagedKnowledgeDocuments(string collectionName, KnowledgeFileFilter filter);
-    Task<FileBinaryDataModel?> GetKnowledgeDocumentBinaryData(string collectionName, Guid fileId);
+    Task<FileBinaryDataModel> GetKnowledgeDocumentBinaryData(string collectionName, Guid fileId);
     #endregion
 
     #region Common
diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeDocMetaData.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeDocMetaData.cs
@@ -23,13 +23,29 @@ public class KnowledgeDocMetaData
     [JsonPropertyName("vector_data_ids")]
     public IEnumerable<string> VectorDataIds { get; set; } = new List<string>();
 
-    [JsonPropertyName("web_url")]
+    [JsonPropertyName("ref_data")]
     [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
-    public string? WebUrl { get; set; }
+    public DocMetaRefData? RefData { get; set; }
 
     [JsonPropertyName("create_date")]
     public DateTime CreateDate { get; set; } = DateTime.UtcNow;
 
     [JsonPropertyName("create_user_id")]
     public string CreateUserId { get; set; }
 }
+
+public class DocMetaRefData
+{
+    [JsonPropertyName("id")]
+    public string Id { get; set; }
+
+    [JsonPropertyName("name")]
+    public string Name { get; set; }
+
+    [JsonPropertyName("url")]
+    public string Url { get; set; }
+
+    [JsonPropertyName("json_content")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public string? JsonContent { get; set; }
+}
diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.KnowledgeBase.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.KnowledgeBase.cs
@@ -68,28 +68,39 @@ public bool DeleteKnowledgeFile(string collectionName, string vectorStoreProvide
     public string GetKnowledgeBaseFileUrl(string collectionName, string vectorStoreProvider, Guid fileId, string fileName)
     {
         if (string.IsNullOrWhiteSpace(collectionName)
-            || string.IsNullOrWhiteSpace(vectorStoreProvider))
+            || string.IsNullOrWhiteSpace(vectorStoreProvider)
+            || string.IsNullOrWhiteSpace(fileName))
+        {
+            return string.Empty;
+        }
+
+        var docDir = BuildKnowledgeCollectionFileDir(collectionName, vectorStoreProvider);
+        var file = Path.Combine(docDir, fileId.ToString(), fileName);
+        if (!File.Exists(file))
         {
             return string.Empty;
         }
 
         return $"/knowledge/document/{collectionName}/file/{fileId}";
     }
 
-    public BinaryData? GetKnowledgeBaseFileBinaryData(string collectionName, string vectorStoreProvider, Guid fileId, string fileName)
+    public BinaryData GetKnowledgeBaseFileBinaryData(string collectionName, string vectorStoreProvider, Guid fileId, string fileName)
     {
         if (string.IsNullOrWhiteSpace(collectionName)
             || string.IsNullOrWhiteSpace(vectorStoreProvider)
             || string.IsNullOrWhiteSpace(fileName))
         {
-            return null;
+            return BinaryData.Empty;
         }
 
         var docDir = BuildKnowledgeCollectionFileDir(collectionName, vectorStoreProvider);
-        var fileDir = Path.Combine(docDir, fileId.ToString());
-        if (!ExistDirectory(fileDir)) return null;
+        var file = Path.Combine(docDir, fileId.ToString(), fileName);
 
-        var file = Path.Combine(fileDir, fileName);
+        if (!File.Exists(file))
+        {
+            return BinaryData.Empty;
+        }
+        
         using var stream = new FileStream(file, FileMode.Open, FileAccess.Read);
         stream.Position = 0;
 
diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeFileViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeFileViewModel.cs
@@ -19,6 +19,10 @@ public class KnowledgeFileViewModel
     [JsonPropertyName("file_url")]
     public string FileUrl { get; set; }
 
+    [JsonPropertyName("ref_data")]
+    public DocMetaRefData? RefData { get; set; }
+
+
     public static KnowledgeFileViewModel From(KnowledgeFileModel model)
     {
         return new KnowledgeFileViewModel
@@ -27,7 +31,8 @@ public static KnowledgeFileViewModel From(KnowledgeFileModel model)
             FileName = model.FileName,
             FileExtension = model.FileExtension,
             ContentType = model.ContentType,
-            FileUrl = model.FileUrl
+            FileUrl = model.FileUrl,
+            RefData = model.RefData
         };
     }
 }
diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Document.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Document.cs
@@ -150,19 +150,14 @@ public async Task<PagedItems<KnowledgeFileModel>> GetPagedKnowledgeDocuments(str
             Size = filter.Size
         });
 
-        var files = pagedData.Items?.Select(x =>
+        var files = pagedData.Items?.Select(x => new KnowledgeFileModel
         {
-            var fileUrl = x.ContentType == MediaTypeNames.Text.Html ?
-                        x.WebUrl : fileStorage.GetKnowledgeBaseFileUrl(collectionName, vectorStoreProvider, x.FileId, x.FileName);  
-
-            return new KnowledgeFileModel
-            {
-                FileId = x.FileId,
-                FileName = x.FileName,
-                FileExtension = Path.GetExtension(x.FileName),
-                ContentType = x.ContentType,
-                FileUrl = fileUrl
-            };
+            FileId = x.FileId,
+            FileName = x.FileName,
+            FileExtension = Path.GetExtension(x.FileName),
+            ContentType = x.ContentType,
+            FileUrl = fileStorage.GetKnowledgeBaseFileUrl(collectionName, vectorStoreProvider, x.FileId, x.FileName),
+            RefData = x.RefData
         })?.ToList() ?? new List<KnowledgeFileModel>();
 
         return new PagedItems<KnowledgeFileModel>
@@ -172,7 +167,7 @@ public async Task<PagedItems<KnowledgeFileModel>> GetPagedKnowledgeDocuments(str
         };
     }
 
-    public async Task<FileBinaryDataModel?> GetKnowledgeDocumentBinaryData(string collectionName, Guid fileId)
+    public async Task<FileBinaryDataModel> GetKnowledgeDocumentBinaryData(string collectionName, Guid fileId)
     {
         var db = _services.GetRequiredService<IBotSharpRepository>();
         var fileStorage = _services.GetRequiredService<IFileStorageService>();
@@ -186,7 +181,15 @@ public async Task<PagedItems<KnowledgeFileModel>> GetPagedKnowledgeDocuments(str
         });
 
         var metaData = pageData?.Items?.FirstOrDefault();
-        if (metaData == null) return null;
+        if (metaData == null)
+        {
+            return new FileBinaryDataModel
+            {
+                FileName = "error.txt",
+                ContentType = "text/plain",
+                FileBinaryData = BinaryData.Empty
+            };
+        };
 
         var binaryData = fileStorage.GetKnowledgeBaseFileBinaryData(collectionName, vectorStoreProvider, fileId, metaData.FileName);
         return new FileBinaryDataModel
diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/KnowledgeCollectionFileMetaDocument.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/KnowledgeCollectionFileMetaDocument.cs
@@ -9,7 +9,7 @@ public class KnowledgeCollectionFileMetaDocument : MongoBase
     public string ContentType { get; set; }
     public string VectorStoreProvider { get; set; }
     public IEnumerable<string> VectorDataIds { get; set; } = new List<string>();
-    public string? WebUrl { get; set; }
+    public KnowledgeFileMetaRefMongoModel? RefData { get; set; }
     public DateTime CreateDate { get; set; }
     public string CreateUserId { get; set; }
 }
diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Models/KnowledgeFileMetaRefMongoModel.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/KnowledgeFileMetaRefMongoModel.cs
@@ -0,0 +1,37 @@
+using BotSharp.Abstraction.Knowledges.Models;
+
+namespace BotSharp.Plugin.MongoStorage.Models;
+
+public class KnowledgeFileMetaRefMongoModel
+{
+    public string Id { get; set; }
+    public string Name { get; set; }
+    public string Url { get; set; }
+    public string? JsonContent { get; set; }
+
+    public static KnowledgeFileMetaRefMongoModel? ToMongoModel(DocMetaRefData? model)
+    {
+        if (model == null) return null;
+
+        return new KnowledgeFileMetaRefMongoModel
+        {
+            Id = model.Id,
+            Name = model.Name,
+            Url = model.Url,
+            JsonContent = model.JsonContent
+        };
+    }
+
+    public static DocMetaRefData? ToDomainModel(KnowledgeFileMetaRefMongoModel? model)
+    {
+        if (model == null) return null;
+
+        return new DocMetaRefData
+        {
+            Id = model.Id,
+            Name = model.Name,
+            Url = model.Url,
+            JsonContent = model.JsonContent
+        };
+    }
+}
diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.KnowledgeBase.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.KnowledgeBase.cs
@@ -135,7 +135,7 @@ public bool SaveKnolwedgeBaseFileMeta(KnowledgeDocMetaData metaData)
             ContentType = metaData.ContentType,
             VectorStoreProvider = metaData.VectorStoreProvider,
             VectorDataIds = metaData.VectorDataIds,
-            WebUrl = metaData.WebUrl,
+            RefData = KnowledgeFileMetaRefMongoModel.ToMongoModel(metaData.RefData),
             CreateDate = metaData.CreateDate,
             CreateUserId = metaData.CreateUserId
         };
@@ -203,10 +203,10 @@ public PagedItems<KnowledgeDocMetaData> GetKnowledgeBaseFileMeta(string collecti
             ContentType = x.ContentType,
             VectorStoreProvider = x.VectorStoreProvider,
             VectorDataIds = x.VectorDataIds,
-            WebUrl = x.WebUrl,
+            RefData = KnowledgeFileMetaRefMongoModel.ToDomainModel(x.RefData),
             CreateDate = x.CreateDate,
             CreateUserId = x.CreateUserId
-        })?.ToList() ?? new List<KnowledgeDocMetaData>();
+        })?.ToList() ?? new();
 
         return new PagedItems<KnowledgeDocMetaData>
         {
diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.KnowledgeBase.cs b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.KnowledgeBase.cs
@@ -68,28 +68,49 @@ public string GetKnowledgeBaseFileUrl(string collectionName, string vectorStoreP
             return string.Empty;
         }
 
-        var dir = BuildKnowledgeCollectionFileDir(vectorStoreProvider, collectionName);
-        return $"https://{_fullBuketName}.cos.{_settings.Region}.myqcloud.com/{dir}/{fileId}/{fileName}"; ;
+        var docDir = BuildKnowledgeCollectionFileDir(vectorStoreProvider, collectionName);
+        var fileDir = $"{docDir}/{fileId}";
+        if (!ExistDirectory(fileDir))
+        {
+            return string.Empty;
+        }
+
+        return $"https://{_fullBuketName}.cos.{_settings.Region}.myqcloud.com/{fileDir}/{fileName}"; ;
     }
 
-    public BinaryData? GetKnowledgeBaseFileBinaryData(string collectionName, string vectorStoreProvider, Guid fileId, string fileName)
+    public BinaryData GetKnowledgeBaseFileBinaryData(string collectionName, string vectorStoreProvider, Guid fileId, string fileName)
     {
         if (string.IsNullOrWhiteSpace(collectionName)
             || string.IsNullOrWhiteSpace(vectorStoreProvider)
             || string.IsNullOrWhiteSpace(fileName))
         {
-            return null;
+            return BinaryData.Empty;
         }
 
-        var docDir = BuildKnowledgeCollectionFileDir(collectionName, vectorStoreProvider);
-        var fileDir = $"{docDir}/{fileId}";
-        if (!ExistDirectory(fileDir)) return null;
+        try
+        {
+            var docDir = BuildKnowledgeCollectionFileDir(collectionName, vectorStoreProvider);
+            var fileDir = $"{docDir}/{fileId}";
+            if (!ExistDirectory(fileDir))
+            {
+                return BinaryData.Empty;
+            }
 
-        var file = $"{fileDir}/{fileName}";
-        var bytes = _cosClient.BucketClient.DownloadFileBytes(file);
-        if (bytes == null) return null;
+            var file = $"{fileDir}/{fileName}";
+            var bytes = _cosClient.BucketClient.DownloadFileBytes(file);
+            if (bytes == null)
+            {
+                return BinaryData.Empty;
+            }
 
-        return BinaryData.FromBytes(bytes);
+            return BinaryData.FromBytes(bytes);
+        }
+        catch (Exception ex)
+        {
+            _logger.LogWarning($"Error when downloading collection file ({collectionName}-{vectorStoreProvider}-{fileId}-{fileName})" +
+                $"\r\n{ex.Message}\r\n{ex.InnerException}");
+            return BinaryData.Empty;
+        }
     }
 
 

Original file line number	Diff line number	Diff line change
`@@ -74,6 +74,6 @@ public interface IFileStorageService`
`74`	`74`	`/// <returns></returns>`
`75`	`75`	`bool DeleteKnowledgeFile(string collectionName, string vectorStoreProvider, Guid? fileId = null);`
`76`	`76`	`string GetKnowledgeBaseFileUrl(string collectionName, string vectorStoreProvider, Guid fileId, string fileName);`
`77`		`- BinaryData? GetKnowledgeBaseFileBinaryData(string collectionName, string vectorStoreProvider, Guid fileId, string fileName);`
	`77`	`+ BinaryData GetKnowledgeBaseFileBinaryData(string collectionName, string vectorStoreProvider, Guid fileId, string fileName);`
`78`	`78`	`#endregion`
`79`	`79`	`}`
Original file line number	Diff line number	Diff line change
`@@ -7,4 +7,5 @@ public class KnowledgeFileModel`
`7`	`7`	`public string FileExtension { get; set; }`
`8`	`8`	`public string ContentType { get; set; }`
`9`	`9`	`public string FileUrl { get; set; }`
	`10`	`+ public DocMetaRefData? RefData { get; set; }`
`10`	`11`	`}`
Original file line number	Diff line number	Diff line change
`@@ -19,6 +19,10 @@ public class KnowledgeFileViewModel`
`19`	`19`	`[JsonPropertyName("file_url")]`
`20`	`20`	`public string FileUrl { get; set; }`
`21`	`21`
	`22`	`+ [JsonPropertyName("ref_data")]`
	`23`	`+ public DocMetaRefData? RefData { get; set; }`
	`24`	`+`
	`25`	`+`
`22`	`26`	`public static KnowledgeFileViewModel From(KnowledgeFileModel model)`
`23`	`27`	`{`
`24`	`28`	`return new KnowledgeFileViewModel`
`@@ -27,7 +31,8 @@ public static KnowledgeFileViewModel From(KnowledgeFileModel model)`
`27`	`31`	`FileName = model.FileName,`
`28`	`32`	`FileExtension = model.FileExtension,`
`29`	`33`	`ContentType = model.ContentType,`
`30`		`- FileUrl = model.FileUrl`
	`34`	`+ FileUrl = model.FileUrl,`
	`35`	`+ RefData = model.RefData`
`31`	`36`	`};`
`32`	`37`	`}`
`33`	`38`	`}`
Original file line number	Diff line number	Diff line change
`@@ -9,7 +9,7 @@ public class KnowledgeCollectionFileMetaDocument : MongoBase`
`9`	`9`	`public string ContentType { get; set; }`
`10`	`10`	`public string VectorStoreProvider { get; set; }`
`11`	`11`	`public IEnumerable<string> VectorDataIds { get; set; } = new List<string>();`
`12`		`- public string? WebUrl { get; set; }`
	`12`	`+ public KnowledgeFileMetaRefMongoModel? RefData { get; set; }`
`13`	`13`	`public DateTime CreateDate { get; set; }`
`14`	`14`	`public string CreateUserId { get; set; }`
`15`	`15`	`}`