Skip to content

Commit f41820f

Browse files
iceljciceljc
authored
Merge pull request #928 from iceljc/master
refine state key search
2 parents a006f6a + 9f71816 commit f41820f

File tree

24 files changed

+246
-79
lines changed

24 files changed

+246
-79
lines changed

‎src/Infrastructure/BotSharp.Abstraction/Agents/IAgentService.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ public interface IAgentService
1212
Task<Agent> CreateAgent(Agent agent);
1313
Task<string> RefreshAgents();
1414
Task<PagedItems<Agent>> GetAgents(AgentFilter filter);
15-
Task<List<IdName>> GetAgentOptions();
15+
Task<List<IdName>> GetAgentOptions(List<string>? agentIds = null);
1616

1717
/// <summary>
1818
/// Load agent configurations and trigger hooks

‎src/Infrastructure/BotSharp.Abstraction/Conversations/Enums/MessageTypeName.cs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,4 +6,5 @@ public static class MessageTypeName
66
public const string Notification = "notification";
77
public const string FunctionCall = "function";
88
public const string Audio = "audio";
9+
public const string Error = "error";
910
}

‎src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationService.cs

Lines changed: 1 addition & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -62,14 +62,7 @@ Task<bool> SendMessage(string agentId,
6262

6363
void SaveStates();
6464

65-
/// <summary>
66-
/// Get conversation keys for searching
67-
/// </summary>
68-
/// <param name="query">search query</param>
69-
/// <param name="convLimit">conversation limit</param>
70-
/// <param name="preLoad">if pre-loading, then keys are not filter by the search query</param>
71-
/// <returns></returns>
72-
Task<List<string>> GetConversationStateSearhKeys(string query, int convLimit = 100, int keyLimit = 10, bool preload = false);
65+
Task<List<string>> GetConversationStateSearhKeys(ConversationStateKeysFilter filter);
7366

7467
Task<bool> MigrateLatestStates(int batchSize = 100, int errorLimit = 10);
7568
}

‎src/Infrastructure/BotSharp.Abstraction/Loggers/Services/ILoggerService.cs

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
1-
using BotSharp.Abstraction.Instructs.Models;
21
using BotSharp.Abstraction.Loggers.Models;
2+
using BotSharp.Abstraction.Repositories.Filters;
33

44
namespace BotSharp.Abstraction.Loggers.Services;
55

@@ -12,5 +12,6 @@ public interface ILoggerService
1212

1313
#region Instruction
1414
Task<PagedItems<InstructionLogModel>> GetInstructionLogs(InstructLogFilter filter);
15+
Task<List<string>> GetInstructionLogSearchKeys(InstructLogKeysFilter filter);
1516
#endregion
1617
}

‎src/Infrastructure/BotSharp.Abstraction/Repositories/Filters/ConversationFilter.cs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,9 +23,9 @@ public class ConversationFilter
2323
/// <summary>
2424
/// Check whether each key in the list is in the conversation states and its value equals to target value if not empty
2525
/// </summary>
26-
public IEnumerable<KeyValue>? States { get; set; } = [];
26+
public List<KeyValue>? States { get; set; }
2727

28-
public IEnumerable<string>? Tags { get; set; } = [];
28+
public List<string>? Tags { get; set; }
2929

3030
public static ConversationFilter Empty()
3131
{

‎src/Infrastructure/BotSharp.Abstraction/Repositories/Filters/ConversationStateKeysFilter.cs

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
namespace BotSharp.Abstraction.Repositories.Filters;
2+
3+
public class ConversationStateKeysFilter
4+
{
5+
public string? Query { get; set; }
6+
public int KeyLimit { get; set; } = 10;
7+
public int ConvLimit { get; set; } = 100;
8+
public bool PreLoad { get; set; }
9+
public List<string>? AgentIds { get; set; }
10+
public List<string>? UserIds { get; set; }
11+
12+
public ConversationStateKeysFilter()
13+
{
14+
15+
}
16+
17+
public static ConversationStateKeysFilter Empty()
18+
{
19+
return new();
20+
}
21+
}

‎src/Infrastructure/BotSharp.Abstraction/Instructs/Models/InstructLogFilter.cs renamed to ‎src/Infrastructure/BotSharp.Abstraction/Repositories/Filters/InstructLogFilter.cs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
namespace BotSharp.Abstraction.Instructs.Models;
1+
namespace BotSharp.Abstraction.Repositories.Filters;
22

33
public class InstructLogFilter : Pagination
44
{
@@ -11,6 +11,6 @@ public class InstructLogFilter : Pagination
1111

1212
public static InstructLogFilter Empty()
1313
{
14-
return new InstructLogFilter();
14+
return new();
1515
}
1616
}

‎src/Infrastructure/BotSharp.Abstraction/Repositories/Filters/InstructLogKeysFilter.cs

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
namespace BotSharp.Abstraction.Repositories.Filters;
2+
3+
public class InstructLogKeysFilter
4+
{
5+
public string? Query { get; set; }
6+
public int KeyLimit { get; set; } = 10;
7+
public int LogLimit { get; set; } = 100;
8+
public bool PreLoad { get; set; }
9+
public List<string>? AgentIds { get; set; }
10+
public List<string>? UserIds { get; set; }
11+
12+
public InstructLogKeysFilter()
13+
{
14+
15+
}
16+
17+
public static InstructLogKeysFilter Empty()
18+
{
19+
return new();
20+
}
21+
}

‎src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,3 @@
1-
using BotSharp.Abstraction.Instructs.Models;
21
using BotSharp.Abstraction.Loggers.Models;
32
using BotSharp.Abstraction.Plugins.Models;
43
using BotSharp.Abstraction.Repositories.Filters;
@@ -149,7 +148,7 @@ List<string> GetIdleConversations(int batchSize, int messageLimit, int bufferHou
149148
=> throw new NotImplementedException();
150149
List<string> TruncateConversation(string conversationId, string messageId, bool cleanLog = false)
151150
=> throw new NotImplementedException();
152-
List<string> GetConversationStateSearchKeys(int messageLowerLimit = 2, int convUpperLimit = 100)
151+
List<string> GetConversationStateSearchKeys(ConversationStateKeysFilter filter)
153152
=> throw new NotImplementedException();
154153
List<string> GetConversationsToMigrate(int batchSize = 100)
155154
=> throw new NotImplementedException();
@@ -182,6 +181,9 @@ bool SaveInstructionLogs(IEnumerable<InstructionLogModel> logs)
182181

183182
PagedItems<InstructionLogModel> GetInstructionLogs(InstructLogFilter filter)
184183
=> throw new NotImplementedException();
184+
185+
List<string> GetInstructionLogSearchKeys(InstructLogKeysFilter filter)
186+
=> throw new NotImplementedException();
185187
#endregion
186188

187189
#region Statistics

‎src/Infrastructure/BotSharp.Abstraction/Users/IUserService.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ public interface IUserService
1010
Task<PagedItems<User>> GetUsers(UserFilter filter);
1111
Task<List<User>> SearchLoginUsers(User filter);
1212
Task<User?> GetUserDetails(string userId, bool includeAgent = false);
13-
Task<bool> IsAdminUser(string userId);
13+
Task<(bool, User?)> IsAdminUser(string userId);
1414
Task<UserAuthorization> GetUserAuthorizations(IEnumerable<string>? agentIds = null);
1515
Task<bool> UpdateUser(User user, bool isUpdateUserAgents = false);
1616
Task<User> CreateUser(User user);

‎src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.GetAgents.cs

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -30,9 +30,12 @@ public async Task<PagedItems<Agent>> GetAgents(AgentFilter filter)
3030
#if !DEBUG
3131
[SharpCache(10, perInstanceCache: true)]
3232
#endif
33-
public async Task<List<IdName>> GetAgentOptions()
33+
public async Task<List<IdName>> GetAgentOptions(List<string>? agentIds)
3434
{
35-
var agents = _db.GetAgents(AgentFilter.Empty());
35+
var agents = _db.GetAgents(new AgentFilter
36+
{
37+
AgentIds = !agentIds.IsNullOrEmpty() ? agentIds : null
38+
});
3639
return agents?.Select(x => new IdName(x.Id, x.Name))?.OrderBy(x => x.Name)?.ToList() ?? [];
3740
}
3841

‎src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.RefreshAgents.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ public async Task<string> RefreshAgents()
1818

1919
var userIdentity = _services.GetRequiredService<IUserIdentity>();
2020
var userService = _services.GetRequiredService<IUserService>();
21-
var isValid = await userService.IsAdminUser(userIdentity.Id);
21+
var (isValid, _) = await userService.IsAdminUser(userIdentity.Id);
2222
if (!isValid)
2323
{
2424
return "Unauthorized user.";

‎src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs

Lines changed: 14 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -222,17 +222,26 @@ public void SaveStates()
222222
_state.Save();
223223
}
224224

225-
public async Task<List<string>> GetConversationStateSearhKeys(string query, int convLimit = 100, int keyLimit = 10, bool preload = false)
225+
public async Task<List<string>> GetConversationStateSearhKeys(ConversationStateKeysFilter filter)
226226
{
227+
if (filter == null)
228+
{
229+
filter = ConversationStateKeysFilter.Empty();
230+
}
231+
227232
var keys = new List<string>();
228-
if (!preload && string.IsNullOrWhiteSpace(query))
233+
if (!filter.PreLoad && string.IsNullOrWhiteSpace(filter.Query))
229234
{
230235
return keys;
231236
}
232237

238+
var userService = _services.GetRequiredService<IUserService>();
233239
var db = _services.GetRequiredService<IBotSharpRepository>();
234-
keys = db.GetConversationStateSearchKeys(convUpperLimit: convLimit);
235-
keys = preload ? keys : keys.Where(x => x.Contains(query, StringComparison.OrdinalIgnoreCase)).ToList();
236-
return keys.OrderBy(x => x).Take(keyLimit).ToList();
240+
241+
var (isAdmin, user) = await userService.IsAdminUser(_user.Id);
242+
filter.UserIds = !isAdmin && user?.Id != null ? [user.Id] : [];
243+
keys = db.GetConversationStateSearchKeys(filter);
244+
keys = filter.PreLoad ? keys : keys.Where(x => x.Contains(filter.Query ?? string.Empty, StringComparison.OrdinalIgnoreCase)).ToList();
245+
return keys.OrderBy(x => x).Take(filter.KeyLimit).ToList();
237246
}
238247
}

‎src/Infrastructure/BotSharp.Core/Loggers/Services/LoggerService.Instruction.cs

Lines changed: 25 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,4 @@
1-
using BotSharp.Abstraction.Instructs.Models;
21
using BotSharp.Abstraction.Loggers.Models;
3-
using BotSharp.Abstraction.Users.Enums;
42
using BotSharp.Abstraction.Users.Models;
53

64
namespace BotSharp.Core.Loggers.Services;
@@ -15,11 +13,10 @@ public async Task<PagedItems<InstructionLogModel>> GetInstructionLogs(InstructLo
1513
}
1614

1715
var userService = _services.GetRequiredService<IUserService>();
18-
var user = await userService.GetUser(_user.Id);
19-
var isAdmin = UserConstant.AdminRoles.Contains(user?.Role);
16+
var (isAdmin, user) = await userService.IsAdminUser(_user.Id);
2017
if (!isAdmin && user?.Id == null) return new();
2118

22-
filter.UserIds = isAdmin ? [] : user?.Id != null ? [user.Id] : [];
19+
filter.UserIds = !isAdmin && user?.Id != null ? [user.Id] : null;
2320

2421
var agents = new List<Agent>();
2522
var users = new List<User>();
@@ -61,4 +58,27 @@ public async Task<PagedItems<InstructionLogModel>> GetInstructionLogs(InstructLo
6158
Count = logs.Count
6259
};
6360
}
61+
62+
public async Task<List<string>> GetInstructionLogSearchKeys(InstructLogKeysFilter filter)
63+
{
64+
if (filter == null)
65+
{
66+
filter = InstructLogKeysFilter.Empty();
67+
}
68+
69+
var keys = new List<string>();
70+
if (!filter.PreLoad && string.IsNullOrWhiteSpace(filter.Query))
71+
{
72+
return keys;
73+
}
74+
75+
var userService = _services.GetRequiredService<IUserService>();
76+
var db = _services.GetRequiredService<IBotSharpRepository>();
77+
78+
var (isAdmin, user) = await userService.IsAdminUser(_user.Id);
79+
filter.UserIds = !isAdmin && user?.Id != null ? [user.Id] : null;
80+
keys = db.GetInstructionLogSearchKeys(filter);
81+
keys = filter.PreLoad ? keys : keys.Where(x => x.Contains(filter.Query ?? string.Empty, StringComparison.OrdinalIgnoreCase)).ToList();
82+
return keys.OrderBy(x => x).Take(filter.KeyLimit).ToList();
83+
}
6484
}

‎src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs

Lines changed: 12 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
using BotSharp.Abstraction.Loggers.Models;
2+
using BotSharp.Abstraction.Users.Models;
23
using System.IO;
34

45
namespace BotSharp.Core.Repository;
@@ -647,7 +648,7 @@ public List<string> TruncateConversation(string conversationId, string messageId
647648
#if !DEBUG
648649
[SharpCache(10)]
649650
#endif
650-
public List<string> GetConversationStateSearchKeys(int messageLowerLimit = 2, int convUpperLimit = 100)
651+
public List<string> GetConversationStateSearchKeys(ConversationStateKeysFilter filter)
651652
{
652653
var dir = Path.Combine(_dbSettings.FileRepository, _conversationSettings.DataDir);
653654
if (!Directory.Exists(dir)) return [];
@@ -658,26 +659,29 @@ public List<string> GetConversationStateSearchKeys(int messageLowerLimit = 2, in
658659
foreach (var d in Directory.GetDirectories(dir))
659660
{
660661
var convFile = Path.Combine(d, CONVERSATION_FILE);
661-
var stateFile = Path.Combine(d, STATE_FILE);
662-
if (!File.Exists(convFile) || !File.Exists(stateFile))
662+
var latestStateFile = Path.Combine(d, CONV_LATEST_STATE_FILE);
663+
if (!File.Exists(convFile) || !File.Exists(latestStateFile))
663664
{
664665
continue;
665666
}
666667

667668
var convJson = File.ReadAllText(convFile);
668-
var stateJson = File.ReadAllText(stateFile);
669+
var stateJson = File.ReadAllText(latestStateFile);
669670
var conv = JsonSerializer.Deserialize<Conversation>(convJson, _options);
670-
var states = JsonSerializer.Deserialize<List<StateKeyValue>>(stateJson, _options);
671-
if (conv == null || conv.DialogCount < messageLowerLimit)
671+
var states = JsonSerializer.Deserialize<Dictionary<string, JsonDocument>>(stateJson, _options);
672+
if (conv == null
673+
|| states.IsNullOrEmpty()
674+
|| (!filter.AgentIds.IsNullOrEmpty() && !filter.AgentIds.Contains(conv.AgentId))
675+
|| (!filter.UserIds.IsNullOrEmpty() && !filter.UserIds.Contains(conv.UserId)))
672676
{
673677
continue;
674678
}
675679

676-
var stateKeys = states?.Select(x => x.Key)?.Distinct()?.ToList() ?? [];
680+
var stateKeys = states?.Select(x => x.Key)?.ToList() ?? [];
677681
keys.AddRange(stateKeys);
678682
count++;
679683

680-
if (count >= convUpperLimit)
684+
if (count >= filter.ConvLimit)
681685
{
682686
break;
683687
}

‎src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Log.cs

Lines changed: 38 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,3 @@
1-
using BotSharp.Abstraction.Instructs.Models;
21
using BotSharp.Abstraction.Loggers.Models;
32
using System.IO;
43

@@ -271,6 +270,44 @@ public PagedItems<InstructionLogModel> GetInstructionLogs(InstructLogFilter filt
271270
Count = logs.Count()
272271
};
273272
}
273+
274+
public List<string> GetInstructionLogSearchKeys(InstructLogKeysFilter filter)
275+
{
276+
var keys = new List<string>();
277+
var baseDir = Path.Combine(_dbSettings.FileRepository, INSTRUCTION_LOG_FOLDER);
278+
if (!Directory.Exists(baseDir))
279+
{
280+
return keys;
281+
}
282+
283+
var count = 0;
284+
var files = Directory.GetFiles(baseDir);
285+
foreach (var file in files)
286+
{
287+
var json = File.ReadAllText(file);
288+
var log = JsonSerializer.Deserialize<InstructionLogModel>(json, _options);
289+
if (log == null) continue;
290+
291+
if (log == null
292+
|| log.InnerStates.IsNullOrEmpty()
293+
|| (!filter.UserIds.IsNullOrEmpty() && !filter.UserIds.Contains(log.UserId))
294+
|| (!filter.AgentIds.IsNullOrEmpty() && !filter.AgentIds.Contains(log.AgentId)))
295+
{
296+
continue;
297+
}
298+
299+
var stateKeys = log.InnerStates.Select(x => x.Key)?.ToList() ?? [];
300+
keys.AddRange(stateKeys);
301+
count++;
302+
303+
if (count >= filter.LogLimit)
304+
{
305+
break;
306+
}
307+
}
308+
309+
return keys.Distinct().ToList();
310+
}
274311
#endregion
275312

276313
#region Private methods

‎src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -418,23 +418,24 @@ public async Task<PagedItems<User>> GetUsers(UserFilter filter)
418418
return users;
419419
}
420420

421-
public async Task<bool> IsAdminUser(string userId)
421+
[SharpCache(10, perInstanceCache: true)]
422+
public async Task<(bool, User?)> IsAdminUser(string userId)
422423
{
423424
var db = _services.GetRequiredService<IBotSharpRepository>();
424425
var user = db.GetUserById(userId);
425-
return user != null && UserConstant.AdminRoles.Contains(user.Role);
426+
var isAdmin = user != null && UserConstant.AdminRoles.Contains(user.Role);
427+
return (isAdmin, user);
426428
}
427429

428430
public async Task<UserAuthorization> GetUserAuthorizations(IEnumerable<string>? agentIds = null)
429431
{
430432
var db = _services.GetRequiredService<IBotSharpRepository>();
431-
var user = db.GetUserById(_user.Id);
433+
var (isAdmin, user) = await IsAdminUser(_user.Id);
432434
var auth = new UserAuthorization();
433435

434436
if (user == null) return auth;
435437

436-
auth.IsAdmin = UserConstant.AdminRoles.Contains(user.Role);
437-
438+
auth.IsAdmin = isAdmin;
438439
var role = db.GetRoles(new RoleFilter { Names = [user.Role] }).FirstOrDefault();
439440
var permissions = user.Permissions?.Any() == true ? user.Permissions : role?.Permissions ?? [];
440441
auth.Permissions = permissions;

0 commit comments

Comments
 (0)