Merge pull request #412 from iceljc/features/fix-file-serialization

fix file serialization

Author: Oceania2018

src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStorage.cs

@@ -8,8 +8,8 @@ namespace BotSharp.Core.Conversations.Services;
 public class ConversationStorage : IConversationStorage
 {
     private readonly BotSharpDatabaseSettings _dbSettings;
+    private readonly BotSharpOptions _options;
     private readonly IServiceProvider _services;
-    private readonly JsonSerializerOptions _jsonOptions;
 
     public ConversationStorage(
         BotSharpDatabaseSettings dbSettings,
@@ -18,7 +18,7 @@ public ConversationStorage(
     {
         _dbSettings = dbSettings;
         _services = services;
-        _jsonOptions = InitJsonSerilizerOptions(options);
+        _options = options;
     }
 
     public void Append(string conversationId, RoleDialogModel dialog)
@@ -70,7 +70,7 @@ public void Append(string conversationId, RoleDialogModel dialog)
                 return;
             }
 
-            var richContent = dialog.RichContent != null ? JsonSerializer.Serialize(dialog.RichContent, _jsonOptions) : null;
+            var richContent = dialog.RichContent != null ? JsonSerializer.Serialize(dialog.RichContent, _options.JsonSerializerOptions) : null;
             dialogElements.Add(new DialogElement(meta, content, richContent));
         }
 
@@ -95,7 +95,7 @@ public List<RoleDialogModel> GetDialogs(string conversationId)
             var senderId = role == AgentRole.Function ? currentAgentId : meta.SenderId;
             var createdAt = meta.CreateTime;
             var richContent = !string.IsNullOrEmpty(dialog.RichContent) ? 
-                                JsonSerializer.Deserialize<RichContent<IRichMessage>>(dialog.RichContent, _jsonOptions) : null;
+                                JsonSerializer.Deserialize<RichContent<IRichMessage>>(dialog.RichContent, _options.JsonSerializerOptions) : null;
 
             var record = new RoleDialogModel(role, content)
             {
@@ -140,21 +140,4 @@ private string GetStorageFile(string conversationId)
         }
         return Path.Combine(dir, "dialogs.txt");
     }
-
-    private JsonSerializerOptions InitJsonSerilizerOptions(BotSharpOptions botSharOptions)
-    {
-        var options = botSharOptions.JsonSerializerOptions;
-        var jsonOptions = new JsonSerializerOptions
-        {
-            PropertyNameCaseInsensitive = options.PropertyNameCaseInsensitive,
-            PropertyNamingPolicy = options.PropertyNamingPolicy ?? JsonNamingPolicy.CamelCase,
-            AllowTrailingCommas = options.AllowTrailingCommas,
-        };
-
-        foreach (var converter in options.Converters)
-        {
-            jsonOptions.Converters.Add(converter);
-        }
-        return jsonOptions;
-    }
 }

src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs

@@ -521,7 +521,7 @@ private List<DialogElement> CollectDialogElements(string dialogDir)
                         CreateTime = DateTime.Parse(blocks[0])
                     };
 
-                    var richContent = blocks.Count() > 6 ? blocks[6] : null;
+                    var richContent = blocks.Count() > 6 ? DecodeRichContent(blocks[6]) : null;
                     dialogs.Add(new DialogElement(meta, trimmed, richContent));
                 }
             }
@@ -537,7 +537,8 @@ private List<string> ParseDialogElements(List<DialogElement> dialogs)
             {
                 var meta = element.MetaData;
                 var createTime = meta.CreateTime.ToString("MM/dd/yyyy hh:mm:ss.ffffff tt", CultureInfo.InvariantCulture);
-                var metaStr = $"{createTime}|{meta.Role}|{meta.AgentId}|{meta.MessageId}|{meta.SenderId}|{meta.FunctionName}|{element.RichContent}";
+                var encodedRichContent = EncodeRichContent(element.RichContent);
+                var metaStr = $"{createTime}|{meta.Role}|{meta.AgentId}|{meta.MessageId}|{meta.SenderId}|{meta.FunctionName}|{encodedRichContent}";
                 dialogTexts.Add(metaStr);
                 var content = $"  - {element.Content}";
                 dialogTexts.Add(content);
@@ -686,6 +687,24 @@ private bool SaveTruncatedBreakpoints(string breakpointDir, List<ConversationBre
             File.WriteAllText(breakpointDir, breakpointStr);
             return true;
         }
+
+        private string? EncodeRichContent(string? content)
+        {
+            if (string.IsNullOrEmpty(content)) return content;
+
+            var bytes = Encoding.UTF8.GetBytes(content);
+            var encoded = Convert.ToBase64String(bytes);
+            return encoded;
+        }
+
+        private string? DecodeRichContent(string? content)
+        {
+            if (string.IsNullOrEmpty(content)) return content;
+
+            var decoded = Convert.FromBase64String(content);
+            var origin = Encoding.UTF8.GetString(decoded);
+            return origin;
+        }
         #endregion
     }
 }