Merge pull request #616 from iceljc/features/refine-audio-handler

Features/refine audio handler

Author: iceljc

BotSharp.sln

@@ -111,7 +111,9 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BotSharp.Plugin.PythonInter
 EndProject
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Graph", "Graph", "{97A0B191-64D7-4F8A-BFE8-1BFCC5E247E1}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.Plugin.Graph", "src\Plugins\BotSharp.Plugin.Graph\BotSharp.Plugin.Graph.csproj", "{EBFE97DA-D0BA-48BA-8B5D-083B60348D1D}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BotSharp.Plugin.Graph", "src\Plugins\BotSharp.Plugin.Graph\BotSharp.Plugin.Graph.csproj", "{EBFE97DA-D0BA-48BA-8B5D-083B60348D1D}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BotSharp.Plugin.AudioHandler", "src\Plugins\BotSharp.Plugin.AudioHandler\BotSharp.Plugin.AudioHandler.csproj", "{F57F4862-F8D4-44A1-AC12-5C131B5C9785}"
 EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
@@ -449,6 +451,14 @@ Global
 		{EBFE97DA-D0BA-48BA-8B5D-083B60348D1D}.Release|Any CPU.Build.0 = Release|Any CPU
 		{EBFE97DA-D0BA-48BA-8B5D-083B60348D1D}.Release|x64.ActiveCfg = Release|Any CPU
 		{EBFE97DA-D0BA-48BA-8B5D-083B60348D1D}.Release|x64.Build.0 = Release|Any CPU
+		{F57F4862-F8D4-44A1-AC12-5C131B5C9785}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{F57F4862-F8D4-44A1-AC12-5C131B5C9785}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{F57F4862-F8D4-44A1-AC12-5C131B5C9785}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{F57F4862-F8D4-44A1-AC12-5C131B5C9785}.Debug|x64.Build.0 = Debug|Any CPU
+		{F57F4862-F8D4-44A1-AC12-5C131B5C9785}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{F57F4862-F8D4-44A1-AC12-5C131B5C9785}.Release|Any CPU.Build.0 = Release|Any CPU
+		{F57F4862-F8D4-44A1-AC12-5C131B5C9785}.Release|x64.ActiveCfg = Release|Any CPU
+		{F57F4862-F8D4-44A1-AC12-5C131B5C9785}.Release|x64.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -503,6 +513,7 @@ Global
 		{05E6E405-5021-406E-8A5E-0A7CEC881F6D} = {C4C59872-3C8A-450D-83D5-2BE402D610D5}
 		{97A0B191-64D7-4F8A-BFE8-1BFCC5E247E1} = {2635EC9B-2E5F-4313-AC21-0B847F31F36C}
 		{EBFE97DA-D0BA-48BA-8B5D-083B60348D1D} = {97A0B191-64D7-4F8A-BFE8-1BFCC5E247E1}
+		{F57F4862-F8D4-44A1-AC12-5C131B5C9785} = {51AFE054-AE99-497D-A593-69BAEFB5106F}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {A9969D89-C98B-40A5-A12B-FC87E55B3A19}

src/BotSharp.AppHost/BotSharp.AppHost.csproj

@@ -10,7 +10,7 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Aspire.Hosting.AppHost" Version="8.0.0" />
+    <PackageReference Include="Aspire.Hosting.AppHost" Version="8.2.0" />
     <PackageReference Include="Aspire.Hosting.NodeJs" Version="8.0.1" />
   </ItemGroup>
 

src/Infrastructure/BotSharp.Abstraction/BotSharp.Abstraction.csproj

@@ -1,4 +1,4 @@
-<Project Sdk="Microsoft.NET.Sdk">
+<Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
     <TargetFramework>$(TargetFramework)</TargetFramework>

src/Infrastructure/BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs

@@ -1,7 +1,6 @@
 using BotSharp.Abstraction.Functions.Models;
 using BotSharp.Abstraction.Messaging;
 using BotSharp.Abstraction.Messaging.Models.RichContent;
-using BotSharp.Abstraction.MLTasks;
 
 namespace BotSharp.Abstraction.Conversations.Models;
 
@@ -75,6 +74,9 @@ public class RoleDialogModel : ITrackableMessage
     [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
     public RichContent<IRichMessage>? RichContent { get; set; }
 
+    /// <summary>
+    /// Rich content for secondary language
+    /// </summary>
     [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
     public RichContent<IRichMessage>? SecondaryRichContent { get; set; }
 
@@ -86,6 +88,9 @@ public class RoleDialogModel : ITrackableMessage
 
     public FunctionCallFromLlm Instruction { get; set; }
 
+    /// <summary>
+    /// Files to be used in conversation
+    /// </summary>
     public List<BotSharpFile> Files { get; set; } = new List<BotSharpFile>();
 
     /// <summary>
@@ -94,7 +99,8 @@ public class RoleDialogModel : ITrackableMessage
     [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
     [JsonPropertyName("generated_images")]
     public List<ImageGeneration> GeneratedImages { get; set; } = new List<ImageGeneration>();
-    public float KnowledgeConfidence { get; set; } = 0.5f;
+
+
     private RoleDialogModel()
     {
     }
@@ -134,8 +140,7 @@ public static RoleDialogModel From(RoleDialogModel source,
             Payload = source.Payload,
             StopCompletion = source.StopCompletion,
             Instruction = source.Instruction,
-            Data = source.Data,
-            KnowledgeConfidence = source.KnowledgeConfidence
+            Data = source.Data
         };
     }
 }

src/Infrastructure/BotSharp.Abstraction/Files/Converters/IPdf2ImageConverter.cs

@@ -2,7 +2,7 @@ namespace BotSharp.Abstraction.Files.Converters;
 
 public interface IPdf2ImageConverter
 {
-    public string Name { get; }
+    public string Provider { get; }
 
     /// <summary>
     /// Convert pdf pages to images, and return a list of image file paths

src/Infrastructure/BotSharp.Abstraction/Files/FileCoreSettings.cs

@@ -5,6 +5,11 @@ namespace BotSharp.Abstraction.Files;
 public class FileCoreSettings
 {
     public string Storage { get; set; } = FileStorageEnum.LocalFileStorage;
-    public string Pdf2TextConverter { get; set; }
-    public string Pdf2ImageConverter { get; set; }
+    public SettingBase Pdf2TextConverter { get; set; }
+    public SettingBase Pdf2ImageConverter { get; set; }
 }
+
+public class SettingBase
+{
+    public string Provider { get; set; }
+}

src/Infrastructure/BotSharp.Abstraction/Files/IFileInstructService.cs

@@ -3,11 +3,11 @@ namespace BotSharp.Abstraction.Files;
 public interface IFileInstructService
 {
     #region Image
-    Task<RoleDialogModel> ReadImages(string? provider, string? model, string text, IEnumerable<BotSharpFile> images);
+    Task<string> ReadImages(string? provider, string? model, string text, IEnumerable<InstructFileModel> images);
     Task<RoleDialogModel> GenerateImage(string? provider, string? model, string text);
-    Task<RoleDialogModel> VaryImage(string? provider, string? model, BotSharpFile image);
-    Task<RoleDialogModel> EditImage(string? provider, string? model, string text, BotSharpFile image);
-    Task<RoleDialogModel> EditImage(string? provider, string? model, string text, BotSharpFile image, BotSharpFile mask);
+    Task<RoleDialogModel> VaryImage(string? provider, string? model, InstructFileModel image);
+    Task<RoleDialogModel> EditImage(string? provider, string? model, string text, InstructFileModel image);
+    Task<RoleDialogModel> EditImage(string? provider, string? model, string text, InstructFileModel image, InstructFileModel mask);
     #endregion
 
     #region Pdf
@@ -17,7 +17,11 @@ public interface IFileInstructService
     /// <param name="prompt"></param>
     /// <param name="files">Pdf files</param>
     /// <returns></returns>
-    Task<string> ReadPdf(string? provider, string? model, string? modelId, string prompt, List<BotSharpFile> files);
+    Task<string> ReadPdf(string? provider, string? model, string? modelId, string prompt, List<InstructFileModel> files);
+    #endregion
+
+    #region Audio
+    Task<string> SpeechToText(string? provider, string? model, InstructFileModel audio, string? text = null);
     #endregion
 
     #region Select file

src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs

@@ -6,6 +6,7 @@ public interface IFileStorageService
 {
     #region Common
     string GetDirectory(string conversationId);
+    IEnumerable<string> GetFiles(string relativePath, string? searchQuery = null);
     byte[] GetFileBytes(string fileStorageUrl);
     bool SaveFileStreamToPath(string filePath, Stream stream);
     bool SaveFileBytesToPath(string filePath, byte[] bytes);
@@ -16,15 +17,14 @@ public interface IFileStorageService
     string BuildDirectory(params string[] segments);
     #endregion
 
-
     #region Conversation
     /// <summary>
     /// Get the message file screenshots for specific content types, e.g., pdf
     /// </summary>
     /// <param name="conversationId"></param>
     /// <param name="messageIds"></param>
     /// <returns></returns>
-    Task<IEnumerable<MessageFileModel>> GetMessageFileScreenshots(string conversationId, IEnumerable<string> messageIds);
+    Task<IEnumerable<MessageFileModel>> GetMessageFileScreenshotsAsync(string conversationId, IEnumerable<string> messageIds);
 
     /// <summary>
     /// Get the files that have been uploaded in the chat. No screenshot images are included.
@@ -37,7 +37,7 @@ public interface IFileStorageService
     IEnumerable<MessageFileModel> GetMessageFiles(string conversationId, IEnumerable<string> messageIds, string source, IEnumerable<string>? contentTypes = null);
     string GetMessageFile(string conversationId, string messageId, string source, string index, string fileName);
     IEnumerable<MessageFileModel> GetMessagesWithFile(string conversationId, IEnumerable<string> messageIds);
-    bool SaveMessageFiles(string conversationId, string messageId, string source, List<BotSharpFile> files);
+    bool SaveMessageFiles(string conversationId, string messageId, string source, List<InputFileModel> files);
 
     /// <summary>
     /// Delete files under messages
@@ -54,10 +54,11 @@ public interface IFileStorageService
 
     #region User
     string GetUserAvatar();
-    bool SaveUserAvatar(BotSharpFile file);
+    bool SaveUserAvatar(InputFileModel file);
     #endregion
+
     #region Speech
-    Task SaveSpeechFileAsync(string conversationId, string fileName, BinaryData data);
-    Task<BinaryData> RetrieveSpeechFileAsync(string conversationId, string fileName);
+    bool SaveSpeechFile(string conversationId, string fileName, BinaryData data);
+    BinaryData GetSpeechFile(string conversationId, string fileName);
     #endregion
 }

src/Infrastructure/BotSharp.Abstraction/Files/Models/BotSharpFile.cs

@@ -1,7 +1,12 @@
 
 namespace BotSharp.Abstraction.Files.Models;
 
-public class BotSharpFile : FileBase
+public class BotSharpFile : FileInformation
 {
-    
+    /// <summary>
+    /// File data => format: "data:image/png;base64,aaaaaaaa"
+    /// </summary>
+    [JsonPropertyName("file_data")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public string? FileData { get; set; } = string.Empty;
 }

src/Infrastructure/BotSharp.Abstraction/Files/Models/FileBase.cs

@@ -2,20 +2,6 @@ namespace BotSharp.Abstraction.Files.Models;
 
 public class FileBase
 {
-    /// <summary>
-    /// External file url
-    /// </summary>
-    [JsonPropertyName("file_url")]
-    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
-    public string? FileUrl { get; set; } = string.Empty;
-
-    /// <summary>
-    /// Internal file storage url
-    /// </summary>
-    [JsonPropertyName("file_storage_url")]
-    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
-    public string? FileStorageUrl { get; set; } = string.Empty;
-
     /// <summary>
     /// File name without extension
     /// </summary>
@@ -29,18 +15,4 @@ public class FileBase
     [JsonPropertyName("file_data")]
     [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
     public string? FileData { get; set; } = string.Empty;
-
-    /// <summary>
-    /// File content type
-    /// </summary>
-    [JsonPropertyName("content_type")]
-    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
-    public string? ContentType { get; set; } = string.Empty;
-
-    /// <summary>
-    /// File extension without dot
-    /// </summary>
-    [JsonPropertyName("file_extension")]
-    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
-    public string? FileExtension { get; set; } = string.Empty;
 }

src/Infrastructure/BotSharp.Abstraction/Files/Models/FileInformation.cs

@@ -0,0 +1,38 @@
+namespace BotSharp.Abstraction.Files.Models;
+
+public class FileInformation
+{
+    /// <summary>
+    /// External file url
+    /// </summary>
+    [JsonPropertyName("file_url")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public string? FileUrl { get; set; } = string.Empty;
+
+    /// <summary>
+    /// Internal file storage url
+    /// </summary>
+    [JsonPropertyName("file_storage_url")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public string? FileStorageUrl { get; set; } = string.Empty;
+
+    /// <summary>
+    /// File content type
+    /// </summary>
+    [JsonPropertyName("content_type")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public string? ContentType { get; set; } = string.Empty;
+
+    /// <summary>
+    /// File name without extension
+    /// </summary>
+    [JsonPropertyName("file_name")]
+    public string FileName { get; set; } = string.Empty;
+
+    /// <summary>
+    /// File extension without dot
+    /// </summary>
+    [JsonPropertyName("file_extension")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public string? FileExtension { get; set; } = string.Empty;
+}

src/Infrastructure/BotSharp.Abstraction/Files/Models/InputFileModel.cs

@@ -0,0 +1,16 @@
+namespace BotSharp.Abstraction.Files.Models;
+
+public class InputFileModel : FileBase
+{
+    /// <summary>
+    /// File name with extension
+    /// </summary>
+    [JsonPropertyName("file_name")]
+    public new string FileName { get; set; } = string.Empty;
+
+    /// <summary>
+    /// File data => format: "data:image/png;base64,aaaaaaaa"
+    /// </summary>
+    [JsonPropertyName("file_data")]
+    public new string FileData { get; set; } = string.Empty;
+}

src/Infrastructure/BotSharp.Abstraction/Files/Models/InputMessageFiles.cs

@@ -0,0 +1,18 @@
+namespace BotSharp.Abstraction.Files.Models;
+
+public class InstructFileModel : FileBase
+{
+    /// <summary>
+    /// File extension without dot
+    /// </summary>
+    [JsonPropertyName("file_extension")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public string? FileExtension { get; set; } = string.Empty;
+
+    /// <summary>
+    /// External file url
+    /// </summary>
+    [JsonPropertyName("file_url")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public string? FileUrl { get; set; } = string.Empty;
+}

src/Infrastructure/BotSharp.Abstraction/Files/Models/InstructFileModel.cs

@@ -1,6 +1,6 @@
 namespace BotSharp.Abstraction.Files.Models;
 
-public class MessageFileModel : FileBase
+public class MessageFileModel : FileInformation
 {
     [JsonPropertyName("message_id")]
     public string MessageId { get; set; }

src/Infrastructure/BotSharp.Abstraction/Files/Models/MessageFileModel.cs

@@ -1,4 +1,6 @@
+using Microsoft.AspNetCore.Http;
 using Microsoft.AspNetCore.StaticFiles;
+using System.IO;
 
 namespace BotSharp.Abstraction.Files.Utilities;
 
@@ -26,11 +28,30 @@ public static (string, byte[]) GetFileInfoFromData(string data)
         return (contentType, Convert.FromBase64String(base64Str));
     }
 
-    public static string GetFileContentType(string filePath)
+    public static string BuildFileDataFromFile(string fileName, byte[] bytes)
+    {
+        var contentType = GetFileContentType(fileName);
+        var base64 = Convert.ToBase64String(bytes);
+        return $"data:{contentType};base64,{base64}";
+    }
+
+    public static string BuildFileDataFromFile(IFormFile file)
+    {
+        using var stream = new MemoryStream();
+        file.CopyTo(stream);
+        stream.Position = 0;
+        var contentType = GetFileContentType(file.FileName);
+        var base64 = Convert.ToBase64String(stream.ToArray());
+        stream.Close();
+
+        return $"data:{contentType};base64,{base64}";
+    }
+
+    public static string GetFileContentType(string fileName)
     {
         string contentType;
         var provider = new FileExtensionContentTypeProvider();
-        if (!provider.TryGetContentType(filePath, out contentType))
+        if (!provider.TryGetContentType(fileName, out contentType))
         {
             contentType = string.Empty;
         }

src/Infrastructure/BotSharp.Abstraction/Files/Utilities/FileUtility.cs

@@ -4,7 +4,7 @@ namespace BotSharp.Abstraction.Graph;
 
 public interface IGraphDb
 {
-    public string Name { get; }
+    public string Provider { get; }
 
     Task<GraphSearchData> Search(string query, GraphSearchOptions options);
 }