Merge pull request #241 from iceljc/bugfix/refine-llm-prompt-log

Oceania2018 · web-flow · commit 365593595469 · 2023-12-20T15:37:19.000-06:00
Bugfix/refine llm prompt log
diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/LlmCompletionLog.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/LlmCompletionLog.cs
@@ -2,7 +2,6 @@ namespace BotSharp.Abstraction.Conversations.Models;
 
 public class LlmCompletionLog
 {
-    public string Id { get; set; } = string.Empty;
     public string ConversationId { get; set; } = string.Empty;
     public string MessageId { get; set; } = string.Empty;
     public string AgentId { get; set; } = string.Empty;
diff --git a/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs b/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs
@@ -1,4 +1,5 @@
 using BotSharp.Abstraction.Repositories.Filters;
+using BotSharp.Abstraction.Repositories.Models;
 using BotSharp.Abstraction.Users.Models;
 
 namespace BotSharp.Abstraction.Repositories;
@@ -30,6 +31,7 @@ public interface IBotSharpRepository
     void CreateNewConversation(Conversation conversation);
     bool DeleteConversation(string conversationId);
     List<DialogElement> GetConversationDialogs(string conversationId);
+    void UpdateConversationDialogElements(string conversationId, List<DialogContentUpdateModel> updateElements);
     void AppendConversationDialogs(string conversationId, List<DialogElement> dialogs);
     List<StateKeyValue> GetConversationStates(string conversationId);
     void UpdateConversationStates(string conversationId, List<StateKeyValue> states);
diff --git a/src/Infrastructure/BotSharp.Abstraction/Repositories/Models/DialogContentUpdateModel.cs b/src/Infrastructure/BotSharp.Abstraction/Repositories/Models/DialogContentUpdateModel.cs
@@ -0,0 +1,8 @@
+namespace BotSharp.Abstraction.Repositories.Models
+{
+    public class DialogContentUpdateModel
+    {
+        public int Index { get; set; }
+        public string UpdateContent { get; set; }
+    }
+}
diff --git a/src/Infrastructure/BotSharp.Abstraction/Utilities/StringExtensions.cs b/src/Infrastructure/BotSharp.Abstraction/Utilities/StringExtensions.cs
@@ -37,8 +37,10 @@ public static string RemoveNewLine(this string input)
         return input.Replace("\r", " ").Replace("\n", " ").Trim();
     }
 
-    public static bool IsEqualTo(this string str1, string str2, StringComparison option = StringComparison.OrdinalIgnoreCase)
+    public static bool IsEqualTo(this string? str1, string? str2, StringComparison option = StringComparison.OrdinalIgnoreCase)
     {
+        if (str1 == null) return str2 == null;
+
         return str1.Equals(str2, option);
     }
 
diff --git a/src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs b/src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs
@@ -1,6 +1,7 @@
 using BotSharp.Abstraction.Agents.Models;
 using BotSharp.Abstraction.Repositories;
 using BotSharp.Abstraction.Repositories.Filters;
+using BotSharp.Abstraction.Repositories.Models;
 using BotSharp.Abstraction.Users.Models;
 using Microsoft.EntityFrameworkCore.Infrastructure;
 
@@ -146,6 +147,11 @@ public List<DialogElement> GetConversationDialogs(string conversationId)
         throw new NotImplementedException();
     }
 
+    public void UpdateConversationDialogElements(string conversationId, List<DialogContentUpdateModel> updateElements)
+    {
+        throw new NotImplementedException();
+    }
+
     public List<StateKeyValue> GetConversationStates(string conversationId)
     {
         throw new NotImplementedException();
diff --git a/src/Infrastructure/BotSharp.Core/Repository/FileRepository.cs b/src/Infrastructure/BotSharp.Core/Repository/FileRepository.cs
@@ -6,6 +6,9 @@
 using MongoDB.Driver;
 using BotSharp.Abstraction.Routing.Models;
 using BotSharp.Abstraction.Repositories.Filters;
+using BotSharp.Abstraction.Utilities;
+using BotSharp.Abstraction.Conversations.Models;
+using BotSharp.Abstraction.Repositories.Models;
 
 namespace BotSharp.Core.Repository;
 
@@ -650,6 +653,33 @@ public List<DialogElement> GetConversationDialogs(string conversationId)
         return dialogs;
     }
 
+    public void UpdateConversationDialogElements(string conversationId, List<DialogContentUpdateModel> updateElements)
+    {
+        var dialogElements = GetConversationDialogs(conversationId);
+        if (dialogElements.IsNullOrEmpty() || updateElements.IsNullOrEmpty()) return;
+
+        var convDir = FindConversationDirectory(conversationId);
+        if (!string.IsNullOrEmpty(convDir))
+        {
+            var dialogDir = Path.Combine(convDir, "dialogs.txt");
+            if (File.Exists(dialogDir))
+            {
+                var updated = dialogElements.Select((x, idx) =>
+                {
+                    var found = updateElements.FirstOrDefault(e => e.Index == idx);
+                    if (found != null)
+                    {
+                        x.Content = found.UpdateContent;
+                    }
+                    return x;
+                }).ToList();
+
+                var texts = ParseDialogElements(updated);
+                File.WriteAllLines(dialogDir, texts);
+            }
+        }
+    }
+
     public void AppendConversationDialogs(string conversationId, List<DialogElement> dialogs)
     {
         var convDir = FindConversationDirectory(conversationId);
@@ -860,18 +890,24 @@ public List<string> GetExecutionLogs(string conversationId)
     #region LLM Completion Log
     public void SaveLlmCompletionLog(LlmCompletionLog log)
     {
-        if (log == null || string.IsNullOrEmpty(log.ConversationId)) return;
+        if (log == null) return;
+
+        log.ConversationId = log.ConversationId.IfNullOrEmptyAs(Guid.NewGuid().ToString());
+        log.MessageId = log.MessageId.IfNullOrEmptyAs(Guid.NewGuid().ToString());
 
         var convDir = FindConversationDirectory(log.ConversationId);
-        if (string.IsNullOrEmpty(convDir)) return;
+        if (string.IsNullOrEmpty(convDir))
+        {
+            convDir = Path.Combine(_dbSettings.FileRepository, _conversationSettings.DataDir, log.ConversationId);
+            Directory.CreateDirectory(convDir);
+        }
 
         var logDir = Path.Combine(convDir, "llm_prompt_log");
         if (!Directory.Exists(logDir))
         {
             Directory.CreateDirectory(logDir);
         }
 
-        log.Id = Guid.NewGuid().ToString();
         var index = GetNextLlmCompletionLogIndex(logDir, log.MessageId);
         var file = Path.Combine(logDir, $"{log.MessageId}.{index}.log");
         File.WriteAllText(file, JsonSerializer.Serialize(log, _options));
diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/LlmCompletionLogDocument.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/LlmCompletionLogDocument.cs
@@ -1,11 +1,9 @@
+using BotSharp.Plugin.MongoStorage.Models;
+
 namespace BotSharp.Plugin.MongoStorage.Collections;
 
 public class LlmCompletionLogDocument : MongoBase
 {
     public string ConversationId { get; set; }
-    public string MessageId { get; set; }
-    public string AgentId { get; set; }
-    public string Prompt { get; set; }
-    public string? Response { get; set; }
-    public DateTime CreateDateTime { get; set; }
+    public List<PromptLogElement> Logs { get; set; }
 }
diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Models/PromptLogElement.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/PromptLogElement.cs
@@ -0,0 +1,10 @@
+namespace BotSharp.Plugin.MongoStorage.Models;
+
+public class PromptLogElement
+{
+    public string MessageId { get; set; }
+    public string AgentId { get; set; }
+    public string Prompt { get; set; }
+    public string? Response { get; set; }
+    public DateTime CreateDateTime { get; set; }
+}
diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.cs
@@ -2,6 +2,7 @@
 using BotSharp.Abstraction.Conversations.Models;
 using BotSharp.Abstraction.Functions.Models;
 using BotSharp.Abstraction.Repositories.Filters;
+using BotSharp.Abstraction.Repositories.Models;
 using BotSharp.Abstraction.Routing.Models;
 using BotSharp.Abstraction.Users.Models;
 using BotSharp.Plugin.MongoStorage.Collections;
@@ -635,7 +636,8 @@ public bool DeleteConversation(string conversationId)
         var promptLogDeleted = _dc.LlmCompletionLogs.DeleteMany(filterPromptLog);
         var dialogDeleted = _dc.ConversationDialogs.DeleteMany(filterDialog);
         var convDeleted = _dc.Conversations.DeleteMany(filterConv);
-        return convDeleted.DeletedCount > 0 || dialogDeleted.DeletedCount > 0;
+        return convDeleted.DeletedCount > 0 || dialogDeleted.DeletedCount > 0
+            || exeLogDeleted.DeletedCount > 0 || promptLogDeleted.DeletedCount > 0;
     }
 
     public List<DialogElement> GetConversationDialogs(string conversationId)
@@ -651,6 +653,27 @@ public List<DialogElement> GetConversationDialogs(string conversationId)
         return formattedDialog ?? new List<DialogElement>();
     }
 
+    public void UpdateConversationDialogElements(string conversationId, List<DialogContentUpdateModel> updateElements)
+    {
+        if (string.IsNullOrEmpty(conversationId) || updateElements.IsNullOrEmpty()) return;
+
+        var filterDialog = Builders<ConversationDialogDocument>.Filter.Eq(x => x.ConversationId, conversationId);
+        var foundDialog = _dc.ConversationDialogs.Find(filterDialog).FirstOrDefault();
+        if (foundDialog == null || foundDialog.Dialogs.IsNullOrEmpty()) return;
+
+        foundDialog.Dialogs = foundDialog.Dialogs.Select((x, idx) =>
+        {
+            var found = updateElements.FirstOrDefault(e => e.Index == idx);
+            if (found != null)
+            {
+                x.Content = found.UpdateContent;
+            }
+            return x;
+        }).ToList();
+        
+        _dc.ConversationDialogs.ReplaceOne(filterDialog, foundDialog);
+    }
+
     public void AppendConversationDialogs(string conversationId, List<DialogElement> dialogs)
     {
         if (string.IsNullOrEmpty(conversationId)) return;
@@ -892,20 +915,26 @@ public List<string> GetExecutionLogs(string conversationId)
     #region LLM Completion Log
     public void SaveLlmCompletionLog(LlmCompletionLog log)
     {
-        if (log == null || string.IsNullOrEmpty(log.ConversationId)) return;
+        if (log == null) return;
+
+        var conversationId = log.ConversationId.IfNullOrEmptyAs(Guid.NewGuid().ToString());
+        var messageId = log.MessageId.IfNullOrEmptyAs(Guid.NewGuid().ToString());
 
-        var completiongLog = new LlmCompletionLogDocument
+        var logElement = new PromptLogElement
         {
-            Id = string.IsNullOrEmpty(log.Id) ? Guid.NewGuid().ToString() : log.Id,
-            ConversationId = log.ConversationId,
-            MessageId = log.MessageId,
+            MessageId = messageId,
             AgentId = log.AgentId,
             Prompt = log.Prompt,
             Response = log.Response,
             CreateDateTime = log.CreateDateTime
         };
 
-        _dc.LlmCompletionLogs.InsertOne(completiongLog);
+        var filter = Builders<LlmCompletionLogDocument>.Filter.Eq(x => x.ConversationId, conversationId);
+        var update = Builders<LlmCompletionLogDocument>.Update
+                                                       .SetOnInsert(x => x.Id, Guid.NewGuid().ToString())
+                                                       .Push(x => x.Logs, logElement);
+
+        _dc.LlmCompletionLogs.UpdateOne(filter, update, _options);
     }
     #endregion
 }

Original file line number	Diff line number	Diff line change
`@@ -2,7 +2,6 @@ namespace BotSharp.Abstraction.Conversations.Models;`
`2`	`2`
`3`	`3`	`public class LlmCompletionLog`
`4`	`4`	`{`
`5`		`- public string Id { get; set; } = string.Empty;`
`6`	`5`	`public string ConversationId { get; set; } = string.Empty;`
`7`	`6`	`public string MessageId { get; set; } = string.Empty;`
`8`	`7`	`public string AgentId { get; set; } = string.Empty;`
Original file line number	Diff line number	Diff line change
`@@ -37,8 +37,10 @@ public static string RemoveNewLine(this string input)`
`37`	`37`	`return input.Replace("\r", " ").Replace("\n", " ").Trim();`
`38`	`38`	`}`
`39`	`39`
`40`		`- public static bool IsEqualTo(this string str1, string str2, StringComparison option = StringComparison.OrdinalIgnoreCase)`
	`40`	`+ public static bool IsEqualTo(this string? str1, string? str2, StringComparison option = StringComparison.OrdinalIgnoreCase)`
`41`	`41`	`{`
	`42`	`+ if (str1 == null) return str2 == null;`
	`43`	`+`
`42`	`44`	`return str1.Equals(str2, option);`
`43`	`45`	`}`
`44`	`46`
Original file line number	Diff line number	Diff line change
`@@ -1,11 +1,9 @@`
	`1`	`+using BotSharp.Plugin.MongoStorage.Models;`
	`2`	`+`
`1`	`3`	`namespace BotSharp.Plugin.MongoStorage.Collections;`
`2`	`4`
`3`	`5`	`public class LlmCompletionLogDocument : MongoBase`
`4`	`6`	`{`
`5`	`7`	`public string ConversationId { get; set; }`
`6`		`- public string MessageId { get; set; }`
`7`		`- public string AgentId { get; set; }`
`8`		`- public string Prompt { get; set; }`
`9`		`- public string? Response { get; set; }`
`10`		`- public DateTime CreateDateTime { get; set; }`
	`8`	`+ public List<PromptLogElement> Logs { get; set; }`
`11`	`9`	`}`