Skip to content

Commit acf0e9f

Browse files
author
Jicheng Lu
committed
add collection exist
1 parent 0e2a457 commit acf0e9f

File tree

10 files changed

+81
-33
lines changed

10 files changed

+81
-33
lines changed

‎src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ namespace BotSharp.Abstraction.Knowledges;
66
public interface IKnowledgeService
77
{
88
#region Vector
9+
Task<bool> ExistVectorCollection(string collectionName);
910
Task<bool> CreateVectorCollection(string collectionName, string collectionType, int dimension, string provider, string model);
1011
Task<bool> DeleteVectorCollection(string collectionName);
1112
Task<IEnumerable<string>> GetVectorCollections(string type);

‎src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -116,7 +116,6 @@ public interface IBotSharpRepository
116116
bool AddKnowledgeCollectionConfigs(List<VectorCollectionConfig> configs, bool reset = false);
117117
bool DeleteKnowledgeCollectionConfig(string collectionName);
118118
IEnumerable<VectorCollectionConfig> GetKnowledgeCollectionConfigs(VectorCollectionConfigFilter filter);
119-
120119
bool SaveKnolwedgeBaseFileMeta(KnowledgeDocMetaData metaData);
121120
/// <summary>
122121
/// Delete file meta data in a knowledge collection, given the vector store provider. If "fileId" is null, delete all in the collection.

‎src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,8 @@ namespace BotSharp.Abstraction.VectorStorage;
55
public interface IVectorDb
66
{
77
string Provider { get; }
8-
8+
9+
Task<bool> DoesCollectionExist(string collectionName);
910
Task<IEnumerable<string>> GetCollections();
1011
Task<StringIdPagedItems<VectorCollectionData>> GetPagedCollectionData(string collectionName, VectorFilter filter);
1112
Task<IEnumerable<VectorCollectionData>> GetCollectionData(string collectionName, IEnumerable<Guid> ids, bool withPayload = false, bool withVector = false);

‎src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.KnowledgeBase.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -213,7 +213,7 @@ public PagedItems<KnowledgeDocMetaData> GetKnowledgeBaseFileMeta(string collecti
213213

214214
return new PagedItems<KnowledgeDocMetaData>
215215
{
216-
Items = records.Skip(filter.Offset).Take(filter.Size),
216+
Items = records.OrderByDescending(x => x.CreateDate).Skip(filter.Offset).Take(filter.Size),
217217
Count = records.Count
218218
};
219219
}

‎src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
using BotSharp.Abstraction.Files.Utilities;
22
using BotSharp.Abstraction.Graph.Models;
3-
using BotSharp.Abstraction.Knowledges.Models;
43
using BotSharp.Abstraction.VectorStorage.Models;
54
using BotSharp.OpenAPI.ViewModels.Knowledges;
65

@@ -20,6 +19,12 @@ public KnowledgeBaseController(IKnowledgeService knowledgeService, IServiceProvi
2019
}
2120

2221
#region Vector
22+
[HttpGet("knowledge/vector/{collection}/exist")]
23+
public async Task<bool> ExistVectorCollection([FromRoute] string collection)
24+
{
25+
return await _knowledgeService.ExistVectorCollection(collection);
26+
}
27+
2328
[HttpGet("knowledge/vector/collections")]
2429
public async Task<IEnumerable<string>> GetVectorCollections([FromQuery] string type)
2530
{

‎src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemoryVectorDb.cs

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,12 @@ public class MemoryVectorDb : IVectorDb
1010

1111
public string Provider => "MemoryVector";
1212

13+
14+
public async Task<bool> DoesCollectionExist(string collectionName)
15+
{
16+
return false;
17+
}
18+
1319
public async Task<bool> CreateCollection(string collectionName, int dimension)
1420
{
1521
_collections[collectionName] = dimension;

‎src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Document.cs

Lines changed: 16 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@
33
using BotSharp.Abstraction.Files.Utilities;
44
using BotSharp.Abstraction.Knowledges.Helpers;
55
using BotSharp.Abstraction.VectorStorage.Enums;
6-
using System.Collections;
76
using System.Net.Http;
87
using System.Net.Mime;
98

@@ -13,13 +12,21 @@ public partial class KnowledgeService
1312
{
1413
public async Task<UploadKnowledgeResponse> UploadDocumentsToKnowledge(string collectionName, IEnumerable<ExternalFileModel> files)
1514
{
15+
var res = new UploadKnowledgeResponse
16+
{
17+
Success = [],
18+
Failed = files?.Select(x => x.FileName) ?? new List<string>()
19+
};
20+
1621
if (string.IsNullOrWhiteSpace(collectionName) || files.IsNullOrEmpty())
1722
{
18-
return new UploadKnowledgeResponse
19-
{
20-
Success = [],
21-
Failed = files?.Select(x => x.FileName) ?? new List<string>()
22-
};
23+
return res;
24+
}
25+
26+
var exist = await ExistVectorCollection(collectionName);
27+
if (!exist)
28+
{
29+
return res;
2330
}
2431

2532
var db = _services.GetRequiredService<IBotSharpRepository>();
@@ -103,6 +110,9 @@ public async Task<bool> ImportDocumentContentToKnowledge(string collectionName,
103110

104111
try
105112
{
113+
var exist = await ExistVectorCollection(collectionName);
114+
if (!exist) return false;
115+
106116
var db = _services.GetRequiredService<IBotSharpRepository>();
107117
var userId = await GetUserId();
108118
var vectorStoreProvider = _settings.VectorDb.Provider;

‎src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Vector.cs

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,23 @@ namespace BotSharp.Plugin.KnowledgeBase.Services;
77
public partial class KnowledgeService
88
{
99
#region Collection
10+
public async Task<bool> ExistVectorCollection(string collectionName)
11+
{
12+
var db = _services.GetRequiredService<IBotSharpRepository>();
13+
var vectorDb = GetVectorDb();
14+
15+
var exist = await vectorDb.DoesCollectionExist(collectionName);
16+
if (exist) return true;
17+
18+
var configs = db.GetKnowledgeCollectionConfigs(new VectorCollectionConfigFilter
19+
{
20+
CollectionNames = [collectionName],
21+
VectorStroageProviders = [_settings.VectorDb.Provider]
22+
});
23+
24+
return !configs.IsNullOrEmpty();
25+
}
26+
1027
public async Task<bool> CreateVectorCollection(string collectionName, string collectionType, int dimension, string provider, string model)
1128
{
1229
try

‎src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs

Lines changed: 26 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -39,16 +39,22 @@ private QdrantClient GetClient()
3939
return _client;
4040
}
4141

42-
public async Task<bool> CreateCollection(string collectionName, int dimension)
42+
public async Task<bool> DoesCollectionExist(string collectionName)
4343
{
4444
var client = GetClient();
45-
var exist = await DoesCollectionExist(client, collectionName);
45+
return await client.CollectionExistsAsync(collectionName);
46+
}
47+
48+
public async Task<bool> CreateCollection(string collectionName, int dimension)
49+
{
50+
var exist = await DoesCollectionExist(collectionName);
4651

4752
if (exist) return false;
4853

4954
try
5055
{
5156
// Create a new collection
57+
var client = GetClient();
5258
await client.CreateCollectionAsync(collectionName, new VectorParams()
5359
{
5460
Size = (ulong)dimension,
@@ -65,11 +71,11 @@ public async Task<bool> CreateCollection(string collectionName, int dimension)
6571

6672
public async Task<bool> DeleteCollection(string collectionName)
6773
{
68-
var client = GetClient();
69-
var exist = await DoesCollectionExist(client, collectionName);
74+
var exist = await DoesCollectionExist(collectionName);
7075

7176
if (!exist) return false;
7277

78+
var client = GetClient();
7379
await client.DeleteCollectionAsync(collectionName);
7480
return true;
7581
}
@@ -83,8 +89,7 @@ public async Task<IEnumerable<string>> GetCollections()
8389

8490
public async Task<StringIdPagedItems<VectorCollectionData>> GetPagedCollectionData(string collectionName, VectorFilter filter)
8591
{
86-
var client = GetClient();
87-
var exist = await DoesCollectionExist(client, collectionName);
92+
var exist = await DoesCollectionExist(collectionName);
8893
if (!exist)
8994
{
9095
return new StringIdPagedItems<VectorCollectionData>();
@@ -126,6 +131,7 @@ public async Task<StringIdPagedItems<VectorCollectionData>> GetPagedCollectionDa
126131
};
127132
}
128133

134+
var client = GetClient();
129135
var totalPointCount = await client.CountAsync(collectionName, filter: queryFilter);
130136
var response = await client.ScrollAsync(collectionName, limit: (uint)filter.Size,
131137
offset: !string.IsNullOrWhiteSpace(filter.StartId) ? new PointId { Uuid = filter.StartId } : null,
@@ -152,15 +158,18 @@ public async Task<StringIdPagedItems<VectorCollectionData>> GetPagedCollectionDa
152158
public async Task<IEnumerable<VectorCollectionData>> GetCollectionData(string collectionName, IEnumerable<Guid> ids,
153159
bool withPayload = false, bool withVector = false)
154160
{
155-
if (ids.IsNullOrEmpty()) return Enumerable.Empty<VectorCollectionData>();
156-
157-
var client = GetClient();
158-
var exist = await DoesCollectionExist(client, collectionName);
161+
if (ids.IsNullOrEmpty())
162+
{
163+
return Enumerable.Empty<VectorCollectionData>();
164+
}
165+
166+
var exist = await DoesCollectionExist(collectionName);
159167
if (!exist)
160168
{
161169
return Enumerable.Empty<VectorCollectionData>();
162170
}
163171

172+
var client = GetClient();
164173
var pointIds = ids.Select(x => new PointId { Uuid = x.ToString() }).Distinct().ToList();
165174
var points = await client.RetrieveAsync(collectionName, pointIds, withPayload, withVector);
166175
return points.Select(x => new VectorCollectionData
@@ -209,8 +218,7 @@ public async Task<IEnumerable<VectorCollectionData>> Search(string collectionNam
209218
{
210219
var results = new List<VectorCollectionData>();
211220

212-
var client = GetClient();
213-
var exist = await DoesCollectionExist(client, collectionName);
221+
var exist = await DoesCollectionExist(collectionName);
214222
if (!exist)
215223
{
216224
return results;
@@ -221,7 +229,8 @@ public async Task<IEnumerable<VectorCollectionData>> Search(string collectionNam
221229
{
222230
payloadSelector.Include = new PayloadIncludeSelector { Fields = { fields.ToArray() } };
223231
}
224-
232+
233+
var client = GetClient();
225234
var points = await client.SearchAsync(collectionName,
226235
vector,
227236
limit: (ulong)limit,
@@ -244,33 +253,27 @@ public async Task<bool> DeleteCollectionData(string collectionName, List<Guid> i
244253
{
245254
if (ids.IsNullOrEmpty()) return false;
246255

247-
var client = GetClient();
248-
var exist = await DoesCollectionExist(client, collectionName);
256+
var exist = await DoesCollectionExist(collectionName);
249257
if (!exist)
250258
{
251259
return false;
252260
}
253261

262+
var client = GetClient();
254263
var result = await client.DeleteAsync(collectionName, ids);
255264
return result.Status == UpdateStatus.Completed;
256265
}
257266

258267
public async Task<bool> DeleteCollectionAllData(string collectionName)
259268
{
260-
var client = GetClient();
261-
var exist = await DoesCollectionExist(client, collectionName);
269+
var exist = await DoesCollectionExist(collectionName);
262270
if (!exist)
263271
{
264272
return false;
265273
}
266274

275+
var client = GetClient();
267276
var result = await client.DeleteAsync(collectionName, new Filter());
268277
return result.Status == UpdateStatus.Completed;
269278
}
270-
271-
272-
private async Task<bool> DoesCollectionExist(QdrantClient client, string collectionName)
273-
{
274-
return await client.CollectionExistsAsync(collectionName);
275-
}
276279
}

‎src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,12 @@ public SemanticKernelMemoryStoreProvider(IMemoryStore memoryStore)
2525

2626
public string Provider => "SemanticKernel";
2727

28+
29+
public async Task<bool> DoesCollectionExist(string collectionName)
30+
{
31+
return false;
32+
}
33+
2834
public async Task<bool> CreateCollection(string collectionName, int dimension)
2935
{
3036
await _memoryStore.CreateCollectionAsync(collectionName);

0 commit comments

Comments
 (0)