Skip to content

Commit 18a08ce

Browse files
Oceania2018Oceania2018
authored
Merge pull request #529 from iceljc/features/refine-file-loading
refine message file loading
2 parents 9fa0621 + 423ed95 commit 18a08ce

File tree

13 files changed

+123
-75
lines changed

13 files changed

+123
-75
lines changed

‎src/Infrastructure/BotSharp.Abstraction/Files/IBotSharpFileService.cs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ public interface IBotSharpFileService
66
Task<IEnumerable<MessageFileModel>> GetChatImages(string conversationId, string source, IEnumerable<string> fileTypes, List<RoleDialogModel> conversations, int? offset = null);
77
IEnumerable<MessageFileModel> GetMessageFiles(string conversationId, IEnumerable<string> messageIds, string source, bool imageOnly = false);
88
string GetMessageFile(string conversationId, string messageId, string source, string index, string fileName);
9+
IEnumerable<MessageFileModel> GetMessagesWithFile(string conversationId, IEnumerable<string> messageIds);
910
bool SaveMessageFiles(string conversationId, string messageId, string source, List<BotSharpFile> files);
1011

1112
string GetUserAvatar();

‎src/Infrastructure/BotSharp.Abstraction/Repositories/Filters/ConversationFilter.cs

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,4 +28,9 @@ public class KeyValue
2828
{
2929
public string Key { get; set; }
3030
public string? Value { get; set; }
31+
32+
public override string ToString()
33+
{
34+
return $"Key: {Key}, Value: {Value}";
35+
}
3136
}

‎src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.Conversation.cs

Lines changed: 28 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
using AspectInjector.Broker;
12
using BotSharp.Abstraction.Files.Converters;
23
using Microsoft.EntityFrameworkCore;
34
using System.IO;
@@ -126,7 +127,7 @@ public IEnumerable<MessageFileModel> GetMessageFiles(string conversationId, IEnu
126127
string source, bool imageOnly = false)
127128
{
128129
var files = new List<MessageFileModel>();
129-
if (messageIds.IsNullOrEmpty()) return files;
130+
if (string.IsNullOrWhiteSpace(conversationId) || messageIds.IsNullOrEmpty()) return files;
130131

131132
foreach (var messageId in messageIds)
132133
{
@@ -159,7 +160,8 @@ public IEnumerable<MessageFileModel> GetMessageFiles(string conversationId, IEnu
159160
FileStorageUrl = file,
160161
FileName = fileName,
161162
FileType = fileType,
162-
ContentType = contentType
163+
ContentType = contentType,
164+
FileSource = source
163165
};
164166
files.Add(model);
165167
}
@@ -181,6 +183,30 @@ public string GetMessageFile(string conversationId, string messageId, string sou
181183
return found;
182184
}
183185

186+
public IEnumerable<MessageFileModel> GetMessagesWithFile(string conversationId, IEnumerable<string> messageIds)
187+
{
188+
var foundMsgs = new List<MessageFileModel>();
189+
if (string.IsNullOrWhiteSpace(conversationId) || messageIds.IsNullOrEmpty()) return foundMsgs;
190+
191+
foreach (var messageId in messageIds)
192+
{
193+
var prefix = Path.Combine(_baseDir, CONVERSATION_FOLDER, conversationId, FILE_FOLDER, messageId);
194+
var userDir = Path.Combine(prefix, FileSourceType.User);
195+
if (ExistDirectory(userDir))
196+
{
197+
foundMsgs.Add(new MessageFileModel { MessageId = messageId, FileSource = FileSourceType.User });
198+
}
199+
200+
var botDir = Path.Combine(prefix, FileSourceType.Bot);
201+
if (ExistDirectory(botDir))
202+
{
203+
foundMsgs.Add(new MessageFileModel { MessageId = messageId, FileSource = FileSourceType.Bot });
204+
}
205+
}
206+
207+
return foundMsgs;
208+
}
209+
184210
public bool SaveMessageFiles(string conversationId, string messageId, string source, List<BotSharpFile> files)
185211
{
186212
if (files.IsNullOrEmpty()) return false;

‎src/Infrastructure/BotSharp.OpenAPI/Controllers/AgentController.cs

Lines changed: 21 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ public AgentSettings GetSettings()
3131
var agents = await GetAgents(new AgentFilter
3232
{
3333
AgentIds = new List<string> { id }
34-
});
34+
}, useHook: true);
3535

3636
var targetAgent = agents.Items.FirstOrDefault();
3737
if (targetAgent == null) return null;
@@ -63,26 +63,35 @@ public AgentSettings GetSettings()
6363
targetAgent.Editable = editable;
6464
return targetAgent;
6565
}
66-
66+
6767
[HttpGet("/agents")]
68-
public async Task<PagedItems<AgentViewModel>> GetAgents([FromQuery] AgentFilter filter)
68+
public async Task<PagedItems<AgentViewModel>> GetAgents([FromQuery] AgentFilter filter, [FromQuery] bool useHook = false)
6969
{
7070
var agentSetting = _services.GetRequiredService<AgentSettings>();
7171
var pagedAgents = await _agentService.GetAgents(filter);
7272

73-
// prerender agent
7473
var items = new List<Agent>();
75-
foreach (var agent in pagedAgents.Items)
74+
var agents = new List<AgentViewModel>();
75+
if (useHook)
7676
{
77-
var renderedAgent = await _agentService.LoadAgent(agent.Id);
78-
items.Add(renderedAgent);
77+
// prerender agent
78+
foreach (var agent in pagedAgents.Items)
79+
{
80+
var renderedAgent = await _agentService.LoadAgent(agent.Id);
81+
items.Add(renderedAgent);
82+
}
83+
84+
// Set IsHost
85+
agents = items.Select(x => AgentViewModel.FromAgent(x)).ToList();
86+
foreach (var agent in agents)
87+
{
88+
agent.IsHost = agentSetting.HostAgentId == agent.Id;
89+
}
7990
}
80-
81-
// Set IsHost
82-
var agents = items.Select(x => AgentViewModel.FromAgent(x)).ToList();
83-
foreach(var agent in agents)
91+
else
8492
{
85-
agent.IsHost = agentSetting.HostAgentId == agent.Id;
93+
items = pagedAgents.Items.ToList();
94+
agents = items.Select(x => AgentViewModel.FromAgent(x)).ToList();
8695
}
8796

8897
return new PagedItems<AgentViewModel>

‎src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,10 @@ public async Task<IEnumerable<ChatResponseModel>> GetDialogs([FromRoute] string
8080

8181
var userService = _services.GetRequiredService<IUserService>();
8282
var agentService = _services.GetRequiredService<IAgentService>();
83+
var fileService = _services.GetRequiredService<IBotSharpFileService>();
84+
85+
var messageIds = history.Select(x => x.MessageId).Distinct().ToList();
86+
var fileMessages = fileService.GetMessagesWithFile(conversationId, messageIds);
8387

8488
var dialogs = new List<ChatResponseModel>();
8589
foreach (var message in history)
@@ -96,7 +100,8 @@ public async Task<IEnumerable<ChatResponseModel>> GetDialogs([FromRoute] string
96100
Text = !string.IsNullOrEmpty(message.SecondaryContent) ? message.SecondaryContent : message.Content,
97101
Data = message.Data,
98102
Sender = UserViewModel.FromUser(user),
99-
Payload = message.Payload
103+
Payload = message.Payload,
104+
HasMessageFiles = fileMessages.Any(x => x.MessageId.IsEqualTo(message.MessageId) && x.FileSource == FileSourceType.User)
100105
});
101106
}
102107
else if (message.Role == AgentRole.Assistant)
@@ -115,11 +120,11 @@ public async Task<IEnumerable<ChatResponseModel>> GetDialogs([FromRoute] string
115120
FirstName = agent?.Name ?? "Unkown",
116121
Role = message.Role,
117122
},
118-
RichContent = message.SecondaryRichContent ?? message.RichContent
123+
RichContent = message.SecondaryRichContent ?? message.RichContent,
124+
HasMessageFiles = fileMessages.Any(x => x.MessageId.IsEqualTo(message.MessageId) && x.FileSource == FileSourceType.Bot)
119125
});
120126
}
121127
}
122-
123128
return dialogs;
124129
}
125130

‎src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/ChatResponseModel.cs

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,9 @@ public class ChatResponseModel : InstructResult
3232
[JsonPropertyName("payload")]
3333
public string? Payload { get; set; }
3434

35+
[JsonPropertyName("has_message_files")]
36+
public bool HasMessageFiles { get; set; }
37+
3538
[JsonPropertyName("created_at")]
3639
public DateTime CreatedAt { get; set; } = DateTime.UtcNow;
3740
}

‎src/Infrastructure/BotSharp.OpenAPI/ViewModels/Files/MessageFileViewModel.cs

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,9 @@ public class MessageFileViewModel
1616
[JsonPropertyName("content_type")]
1717
public string ContentType { get; set; }
1818

19+
[JsonPropertyName("file_source")]
20+
public string FileSource { get; set; }
21+
1922
public MessageFileViewModel()
2023
{
2124

@@ -28,7 +31,8 @@ public static MessageFileViewModel Transform(MessageFileModel model)
2831
FileUrl = model.FileUrl,
2932
FileName = model.FileName,
3033
FileType = model.FileType,
31-
ContentType = model.ContentType
34+
ContentType = model.ContentType,
35+
FileSource = model.FileSource
3236
};
3337
}
3438
}

‎src/Plugins/BotSharp.Plugin.MongoStorage/Collections/AgentDocument.cs

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,3 @@
1-
using BotSharp.Plugin.MongoStorage.Models;
2-
31
namespace BotSharp.Plugin.MongoStorage.Collections;
42

53
public class AgentDocument : MongoBase

‎src/Plugins/BotSharp.Plugin.MongoStorage/Collections/ConversationDialogDocument.cs

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,3 @@
1-
using BotSharp.Plugin.MongoStorage.Models;
2-
31
namespace BotSharp.Plugin.MongoStorage.Collections;
42

53
public class ConversationDialogDocument : MongoBase

‎src/Plugins/BotSharp.Plugin.MongoStorage/Collections/ConversationStateDocument.cs

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,3 @@
1-
using BotSharp.Plugin.MongoStorage.Models;
2-
31
namespace BotSharp.Plugin.MongoStorage.Collections;
42

53
public class ConversationStateDocument : MongoBase

‎src/Plugins/BotSharp.Plugin.MongoStorage/Collections/LlmCompletionLogDocument.cs

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,3 @@
1-
using BotSharp.Plugin.MongoStorage.Models;
2-
31
namespace BotSharp.Plugin.MongoStorage.Collections;
42

53
public class LlmCompletionLogDocument : MongoBase

‎src/Plugins/BotSharp.Plugin.MongoStorage/MongoDbContext.cs

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,19 @@ private IMongoCollection<ConversationDocument> CreateConversationIndex()
4040
return collection;
4141
}
4242

43+
private IMongoCollection<ConversationStateDocument> CreateConversationStateIndex()
44+
{
45+
var collection = Database.GetCollection<ConversationStateDocument>($"{_collectionPrefix}_ConversationStates");
46+
var indexes = collection.Indexes.List().ToList();
47+
var stateIndex = indexes.FirstOrDefault(x => x.GetElement("name").ToString().StartsWith("States.Key"));
48+
if (stateIndex == null)
49+
{
50+
var indexDef = Builders<ConversationStateDocument>.IndexKeys.Ascending("States.Key");
51+
collection.Indexes.CreateOne(new CreateIndexModel<ConversationStateDocument>(indexDef));
52+
}
53+
return collection;
54+
}
55+
4356
private IMongoCollection<AgentTaskDocument> CreateAgentTaskIndex()
4457
{
4558
var collection = Database.GetCollection<AgentTaskDocument>($"{_collectionPrefix}_AgentTasks");
@@ -93,7 +106,7 @@ public IMongoCollection<ConversationDialogDocument> ConversationDialogs
93106
=> Database.GetCollection<ConversationDialogDocument>($"{_collectionPrefix}_ConversationDialogs");
94107

95108
public IMongoCollection<ConversationStateDocument> ConversationStates
96-
=> Database.GetCollection<ConversationStateDocument>($"{_collectionPrefix}_ConversationStates");
109+
=> CreateConversationStateIndex();
97110

98111
public IMongoCollection<ExecutionLogDocument> ExectionLogs
99112
=> Database.GetCollection<ExecutionLogDocument>($"{_collectionPrefix}_ExecutionLogs");

‎src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs

Lines changed: 38 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,9 @@
1+
using Amazon.Util.Internal;
12
using BotSharp.Abstraction.Conversations.Models;
23
using BotSharp.Abstraction.Repositories.Filters;
4+
using MongoDB.Bson.Serialization;
5+
using MongoDB.Driver;
6+
using System.Collections.Immutable;
37

48
namespace BotSharp.Plugin.MongoStorage.Repository;
59

@@ -225,91 +229,77 @@ public Conversation GetConversation(string conversationId)
225229

226230
public PagedItems<Conversation> GetConversations(ConversationFilter filter)
227231
{
228-
var conversations = new List<Conversation>();
229-
var builder = Builders<ConversationDocument>.Filter;
230-
var filters = new List<FilterDefinition<ConversationDocument>>() { builder.Empty };
232+
var convBuilder = Builders<ConversationDocument>.Filter;
233+
var convFilters = new List<FilterDefinition<ConversationDocument>>() { convBuilder.Empty };
231234

235+
// Filter conversations
232236
if (!string.IsNullOrEmpty(filter?.Id))
233237
{
234-
filters.Add(builder.Eq(x => x.Id, filter.Id));
238+
convFilters.Add(convBuilder.Eq(x => x.Id, filter.Id));
235239
}
236240
if (!string.IsNullOrEmpty(filter?.AgentId))
237241
{
238-
filters.Add(builder.Eq(x => x.AgentId, filter.AgentId));
242+
convFilters.Add(convBuilder.Eq(x => x.AgentId, filter.AgentId));
239243
}
240244
if (!string.IsNullOrEmpty(filter?.Status))
241245
{
242-
filters.Add(builder.Eq(x => x.Status, filter.Status));
246+
convFilters.Add(convBuilder.Eq(x => x.Status, filter.Status));
243247
}
244248
if (!string.IsNullOrEmpty(filter?.Channel))
245249
{
246-
filters.Add(builder.Eq(x => x.Channel, filter.Channel));
250+
convFilters.Add(convBuilder.Eq(x => x.Channel, filter.Channel));
247251
}
248252
if (!string.IsNullOrEmpty(filter?.UserId))
249253
{
250-
filters.Add(builder.Eq(x => x.UserId, filter.UserId));
254+
convFilters.Add(convBuilder.Eq(x => x.UserId, filter.UserId));
251255
}
252256
if (!string.IsNullOrEmpty(filter?.TaskId))
253257
{
254-
filters.Add(builder.Eq(x => x.TaskId, filter.TaskId));
258+
convFilters.Add(convBuilder.Eq(x => x.TaskId, filter.TaskId));
255259
}
256260
if (filter?.StartTime != null)
257261
{
258-
filters.Add(builder.Gte(x => x.CreatedTime, filter.StartTime.Value));
262+
convFilters.Add(convBuilder.Gte(x => x.CreatedTime, filter.StartTime.Value));
259263
}
260264

261-
// Check states
262-
if (filter != null && !filter.States.IsNullOrEmpty())
265+
// Filter states
266+
var stateFilters = new List<FilterDefinition<ConversationStateDocument>>();
267+
if (filter != null && string.IsNullOrEmpty(filter.Id) && !filter.States.IsNullOrEmpty())
263268
{
264-
var targetConvIds = new List<string>();
265-
266269
foreach (var pair in filter.States)
267270
{
268-
if (pair == null || string.IsNullOrWhiteSpace(pair.Key)) continue;
269-
270-
var query = _dc.ConversationStates.AsQueryable();
271-
var convIds = query.AsEnumerable().Where(x =>
271+
var elementFilters = new List<FilterDefinition<StateMongoElement>> { Builders<StateMongoElement>.Filter.Eq(x => x.Key, pair.Key) };
272+
if (!string.IsNullOrEmpty(pair.Value))
272273
{
273-
var foundState = x.States.FirstOrDefault(s => s.Key.IsEqualTo(pair.Key));
274-
if (foundState == null) return false;
275-
276-
if (!string.IsNullOrWhiteSpace(pair.Value))
277-
{
278-
return pair.Value.IsEqualTo(foundState.Values.LastOrDefault()?.Data);
279-
}
280-
281-
return true;
282-
}).Select(x => x.ConversationId).ToList();
283-
284-
targetConvIds = targetConvIds.Concat(convIds).Distinct().ToList();
274+
elementFilters.Add(Builders<StateMongoElement>.Filter.Eq("Values.Data", pair.Value));
275+
}
276+
stateFilters.Add(Builders<ConversationStateDocument>.Filter.ElemMatch(x => x.States, Builders<StateMongoElement>.Filter.And(elementFilters)));
285277
}
286278

287-
filters.Add(builder.In(x => x.Id, targetConvIds));
279+
var targetConvIds = _dc.ConversationStates.Find(Builders<ConversationStateDocument>.Filter.And(stateFilters)).ToEnumerable().Select(x => x.ConversationId).Distinct().ToList();
280+
convFilters.Add(convBuilder.In(x => x.Id, targetConvIds));
288281
}
289282

290-
var filterDef = builder.And(filters);
283+
// Sort and paginate
284+
var filterDef = convBuilder.And(convFilters);
291285
var sortDef = Builders<ConversationDocument>.Sort.Descending(x => x.CreatedTime);
292286
var pager = filter?.Pager ?? new Pagination();
293287
var conversationDocs = _dc.Conversations.Find(filterDef).Sort(sortDef).Skip(pager.Offset).Limit(pager.Size).ToList();
294288
var count = _dc.Conversations.CountDocuments(filterDef);
295289

296-
foreach (var conv in conversationDocs)
290+
var conversations = conversationDocs.Select(x => new Conversation
297291
{
298-
var convId = conv.Id.ToString();
299-
conversations.Add(new Conversation
300-
{
301-
Id = convId,
302-
AgentId = conv.AgentId.ToString(),
303-
UserId = conv.UserId.ToString(),
304-
TaskId = conv.TaskId,
305-
Title = conv.Title,
306-
Channel = conv.Channel,
307-
Status = conv.Status,
308-
DialogCount = conv.DialogCount,
309-
CreatedTime = conv.CreatedTime,
310-
UpdatedTime = conv.UpdatedTime
311-
});
312-
}
292+
Id = x.Id.ToString(),
293+
AgentId = x.AgentId.ToString(),
294+
UserId = x.UserId.ToString(),
295+
TaskId = x.TaskId,
296+
Title = x.Title,
297+
Channel = x.Channel,
298+
Status = x.Status,
299+
DialogCount = x.DialogCount,
300+
CreatedTime = x.CreatedTime,
301+
UpdatedTime = x.UpdatedTime
302+
}).ToList();
313303

314304
return new PagedItems<Conversation>
315305
{

0 commit comments

Comments
 (0)