Merge pull request #964 from hchen2020/master

IAudioSynthesis

Author: Oceania2018

src/Infrastructure/BotSharp.Abstraction/MLTasks/IAudioCompletion.cs

@@ -0,0 +1,15 @@
+namespace BotSharp.Abstraction.MLTasks;
+
+/// <summary>
+/// Text to speech synthesis
+/// </summary>
+public interface IAudioSynthesis
+{
+    string Provider { get; }
+
+    string Model { get; }
+
+    void SetModelName(string model);
+
+    Task<BinaryData> GenerateAudioAsync(string text, string? voice = "alloy", string? format = "mp3", string? instructions = null);
+}

src/Infrastructure/BotSharp.Abstraction/MLTasks/IAudioSynthesis.cs

@@ -0,0 +1,17 @@
+using System.IO;
+
+namespace BotSharp.Abstraction.MLTasks;
+
+/// <summary>
+/// Audio transcription service
+/// </summary>
+public interface IAudioTranscription
+{
+    string Provider { get; }
+
+    string Model { get; }
+
+    Task<string> TranscriptTextAsync(Stream audio, string audioFileName, string? text = null);
+
+    void SetModelName(string model);
+}

src/Infrastructure/BotSharp.Abstraction/MLTasks/IAudioTranscription.cs

@@ -3,14 +3,14 @@ namespace BotSharp.Abstraction.MLTasks.Settings;
 public class LlmModelSetting
 {
     /// <summary>
-    /// Model Id, like "gpt-3.5" and "gpt-4".
+    /// Model Id, like "gpt-4", "gpt-4o", "o1".
     /// </summary>
-    public string? Id { get; set; }
+    public string Id { get; set; } = null!;
 
     /// <summary>
     /// Deployment model name
     /// </summary>
-    public string Name { get; set; }
+    public string Name { get; set; } = null!;
 
     /// <summary>
     /// Model version
@@ -28,8 +28,8 @@ public class LlmModelSetting
     /// </summary>
     public string? Group { get; set; }
 
-    public string ApiKey { get; set; }
-    public string Endpoint { get; set; }
+    public string ApiKey { get; set; } = null!;
+    public string? Endpoint { get; set; }
     public LlmModelType Type { get; set; } = LlmModelType.Chat;
 
     /// <summary>

src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs

@@ -11,6 +11,6 @@ public class ModelTurnDetection
 
 public class AudioTranscription
 {
-    public string Model { get; set; } = "whisper-1";
-    public string Language { get; set; } = "en";
+    public string Model { get; set; } = "gpt-4o-mini-transcribe";
+    public string? Language { get; set; }
 }

src/Infrastructure/BotSharp.Abstraction/Realtime/Models/ModelTurnDetection.cs

@@ -74,7 +74,7 @@ private async Task ConnectToModel(WebSocket userWebSocket)
         if (!model.Contains("-realtime-"))
         {
             var llmProviderService = _services.GetRequiredService<ILlmProviderService>();
-            model = llmProviderService.GetProviderModel("openai", "gpt-4", realTime: true).Name;
+            model = llmProviderService.GetProviderModel("openai", "gpt-4o", realTime: true).Name;
         }
 
         _completer.SetModelName(model);

src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs

@@ -6,14 +6,14 @@ public partial class FileInstructService
 {
     public async Task<string> SpeechToText(string? provider, string? model, InstructFileModel audio, string? text = null)
     {
-        var completion = CompletionProvider.GetAudioCompletion(_services, provider: provider ?? "openai", model: model ?? "whisper-1");
+        var completion = CompletionProvider.GetAudioTranscriber(_services, provider: provider, model: model);
         var audioBytes = await DownloadFile(audio);
         using var stream = new MemoryStream();
         stream.Write(audioBytes, 0, audioBytes.Length);
         stream.Position = 0;
 
         var fileName = $"{audio.FileName ?? "audio"}.{audio.FileExtension ?? "wav"}";
-        var content = await completion.GenerateTextFromAudioAsync(stream, fileName, text);
+        var content = await completion.TranscriptTextAsync(stream, fileName, text);
         stream.Close();
         return content;
     }

src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Audio.cs

@@ -27,7 +27,7 @@ public async Task<string> ReadPdf(string? provider, string? model, string? model
 
             var innerAgentId = agentId ?? Guid.Empty.ToString();
             var completion = CompletionProvider.GetChatCompletion(_services, provider: provider ?? "openai",
-                model: model, modelId: modelId ?? "gpt-4", multiModal: true);
+                model: model, modelId: modelId ?? "gpt-4o", multiModal: true);
             var message = await completion.GetChatCompletions(new Agent()
             {
                 Id = innerAgentId,

src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs

@@ -93,7 +93,7 @@ private async Task<IEnumerable<MessageFileModel>> SelectFiles(IEnumerable<Messag
             }
 
             var providerName = options.Provider ?? "openai";
-            var modelId = options?.ModelId ?? "gpt-4";
+            var modelId = options?.ModelId ?? "gpt-4o";
             var provider = llmProviderService.GetProviders().FirstOrDefault(x => x == providerName);
             var model = llmProviderService.GetProviderModel(provider: provider, id: modelId);
             var completion = CompletionProvider.GetChatCompletion(_services, provider: provider, model: model.Name);

src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.SelectFile.cs

@@ -30,7 +30,7 @@ public static object GetCompletion(IServiceProvider services,
         }
         else if (settings.Type == LlmModelType.Audio)
         {
-            return GetAudioCompletion(services, provider: provider, model: model);
+            return GetAudioTranscriber(services, provider: provider, model: model);
         }
         else
         {
@@ -126,20 +126,39 @@ public static ITextEmbedding GetTextEmbedding(IServiceProvider services,
         return completer;
     }
 
-    public static IAudioCompletion GetAudioCompletion(
+    public static IAudioTranscription GetAudioTranscriber(
         IServiceProvider services,
-        string provider,
-        string model)
+        string? provider = null,
+        string? model = null)
     {
-        var completions = services.GetServices<IAudioCompletion>();
-        var completer = completions.FirstOrDefault(x => x.Provider == provider);
+        var completions = services.GetServices<IAudioTranscription>();
+        var completer = completions.FirstOrDefault(x => x.Provider == (provider ?? "openai"));
         if (completer == null)
         {
             var logger = services.GetRequiredService<ILogger<CompletionProvider>>();
-            logger.LogError($"Can't resolve audio-completion provider by {provider}");
+            logger.LogError($"Can't resolve audio-transcriber provider by {provider}");
+            return default!;
         }
 
-        completer.SetModelName(model);
+        completer.SetModelName(model ?? "gpt-4o-mini-transcribe");
+        return completer;
+    }
+
+    public static IAudioSynthesis GetAudioSynthesizer(
+        IServiceProvider services,
+        string? provider = null,
+        string? model = null)
+    {
+        var completions = services.GetServices<IAudioSynthesis>();
+        var completer = completions.FirstOrDefault(x => x.Provider == (provider ?? "openai"));
+        if (completer == null)
+        {
+            var logger = services.GetRequiredService<ILogger<CompletionProvider>>();
+            logger.LogError($"Can't resolve audio-synthesizer provider by {provider}");
+            return default!;
+        }
+
+        completer.SetModelName(model ?? "gpt-4o-mini-tts");
         return completer;
     }
 

src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs

@@ -499,8 +499,8 @@ public async Task<SpeechToTextViewModel> SpeechToText(IFormFile file, [FromForm]
             file.CopyTo(stream);
             stream.Position = 0;
 
-            var completion = CompletionProvider.GetAudioCompletion(_services, provider: provider ?? "openai", model: model ?? "whisper-1");
-            var content = await completion.GenerateTextFromAudioAsync(stream, file.FileName, text);
+            var completion = CompletionProvider.GetAudioTranscriber(_services, provider: provider, model: model);
+            var content = await completion.TranscriptTextAsync(stream, file.FileName, text);
             viewModel.Content = content;
             stream.Close();
             return viewModel;
@@ -520,8 +520,8 @@ public async Task<IActionResult> TextToSpeech([FromBody] TextToSpeechRequest inp
         var state = _services.GetRequiredService<IConversationStateService>();
         input.States.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External));
 
-        var completion = CompletionProvider.GetAudioCompletion(_services, provider: input.Provider ?? "openai", model: input.Model ?? "tts-1");
-        var binaryData = await completion.GenerateAudioFromTextAsync(input.Text);
+        var completion = CompletionProvider.GetAudioSynthesizer(_services, provider: input.Provider, model: input.Model);
+        var binaryData = await completion.GenerateAudioAsync(input.Text);
         var stream = binaryData.ToStream();
         stream.Position = 0;
 

src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs

@@ -22,7 +22,7 @@ public RealtimeController(IServiceProvider services)
     [HttpGet("/agent/{agentId}/realtime/session")]
     public async Task<RealtimeSession> CreateSession(string agentId)
     {
-        var completion = CompletionProvider.GetRealTimeCompletion(_services, provider: "openai", modelId: "gpt-4");
+        var completion = CompletionProvider.GetRealTimeCompletion(_services, provider: "openai", modelId: "gpt-4o");
 
         var agentService = _services.GetRequiredService<IAgentService>();
         var agent = await agentService.LoadAgent(agentId);

src/Infrastructure/BotSharp.OpenAPI/Controllers/RealtimeController.cs

@@ -16,7 +16,7 @@ public void RegisterDI(IServiceCollection services, IConfiguration config)
             return settingService.Bind<AudioHandlerSettings>("AudioHandler");
         });
 
-        services.AddScoped<IAudioCompletion, NativeWhisperProvider>();
+        services.AddScoped<IAudioTranscription, NativeWhisperProvider>();
         services.AddScoped<IAgentUtilityHook, AudioHandlerUtilityHook>();
     }
 }

src/Plugins/BotSharp.Plugin.AudioHandler/AudioHandlerPlugin.cs

@@ -91,7 +91,7 @@ private async Task<string> GetResponeFromDialogs(List<RoleDialogModel> dialogs)
             using var stream = new MemoryStream(bytes);
             stream.Position = 0;
 
-            var result = await audioCompletion.GenerateTextFromAudioAsync(stream, fileName);
+            var result = await audioCompletion.TranscriptTextAsync(stream, fileName);
             transcripts.Add(result);
             stream.Close();
         }
@@ -104,9 +104,9 @@ private async Task<string> GetResponeFromDialogs(List<RoleDialogModel> dialogs)
         return string.Join("\r\n\r\n", transcripts);
     }
 
-    private IAudioCompletion PrepareModel()
+    private IAudioTranscription PrepareModel()
     {
-        return CompletionProvider.GetAudioCompletion(_serviceProvider, provider: "openai", model: "whisper-1");
+        return CompletionProvider.GetAudioTranscriber(_serviceProvider);
     }
 
     private bool ParseAudioFileType(string fileName)

src/Plugins/BotSharp.Plugin.AudioHandler/Functions/HandleAudioRequestFn.cs

@@ -6,7 +6,7 @@ namespace BotSharp.Plugin.AudioHandler.Provider;
 /// <summary>
 /// Native Whisper provider for speech to text conversion
 /// </summary>
-public class NativeWhisperProvider : IAudioCompletion
+public class NativeWhisperProvider : IAudioTranscription
 {
     private static WhisperProcessor _whisperProcessor;
 
@@ -29,7 +29,7 @@ public NativeWhisperProvider(
         _logger = logger;
     }
 
-    public async Task<string> GenerateTextFromAudioAsync(Stream audio, string audioFileName, string? text = null)
+    public async Task<string> TranscriptTextAsync(Stream audio, string audioFileName, string? text = null)
     {
         var textResult = new List<SegmentData>();
 

src/Plugins/BotSharp.Plugin.AudioHandler/Provider/NativeWhisperProvider.cs

@@ -31,6 +31,6 @@ public void RegisterDI(IServiceCollection services, IConfiguration config)
         services.AddScoped<IChatCompletion, ChatCompletionProvider>();
         services.AddScoped<ITextEmbedding, TextEmbeddingProvider>();
         services.AddScoped<IImageCompletion, ImageCompletionProvider>();
-        services.AddScoped<IAudioCompletion, AudioCompletionProvider>();
+        services.AddScoped<IAudioTranscription, AudioCompletionProvider>();
     }
 }

src/Plugins/BotSharp.Plugin.AzureOpenAI/AzureOpenAiPlugin.cs

@@ -4,7 +4,7 @@ namespace BotSharp.Plugin.AzureOpenAI.Providers.Audio;
 
 public partial class AudioCompletionProvider
 {
-    public async Task<string> GenerateTextFromAudioAsync(Stream audio, string audioFileName, string? text = null)
+    public async Task<string> TranscriptTextAsync(Stream audio, string audioFileName, string? text = null)
     {
         var audioClient = ProviderHelper.GetClient(Provider, _model, _services)
                                         .GetAudioClient(_model);

src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Audio/AudioCompletionProvider.SpeechToText.cs

@@ -1,6 +1,6 @@
 namespace BotSharp.Plugin.AzureOpenAI.Providers.Audio;
 
-public partial class AudioCompletionProvider : IAudioCompletion
+public partial class AudioCompletionProvider : IAudioTranscription
 {
     private readonly IServiceProvider _services;
 

src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Audio/AudioCompletionProvider.cs

@@ -67,7 +67,7 @@ public async Task<bool> Execute(RoleDialogModel message)
 
                     var llmProviderService = _services.GetRequiredService<ILlmProviderService>();
                     var provider = llmProviderService.GetProviders().FirstOrDefault(x => x == "openai");
-                    var model = llmProviderService.GetProviderModel(provider: provider ?? "openai", id: "gpt-4");
+                    var model = llmProviderService.GetProviderModel(provider: provider ?? "openai", id: "gpt-4o");
                     var completion = CompletionProvider.GetChatCompletion(_services, provider: provider, model: model.Name);
                     var convService = _services.GetRequiredService<IConversationService>();
                     var conversationId = convService.ConversationId;

src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailReaderFn.cs

@@ -100,7 +100,7 @@ private async Task<string> GetChatCompletion(Agent agent, List<RoleDialogModel>
         {
             var llmProviderService = _services.GetRequiredService<ILlmProviderService>();
             var provider = llmProviderService.GetProviders().FirstOrDefault(x => x == "openai");
-            var model = llmProviderService.GetProviderModel(provider: provider, id: "gpt-4", multiModal: true);
+            var model = llmProviderService.GetProviderModel(provider: provider, id: "gpt-4o", multiModal: true);
             var completion = CompletionProvider.GetChatCompletion(_services, provider: provider, model: model.Name);
             var response = await completion.GetChatCompletions(agent, dialogs);
             return response.Content;

src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadImageFn.cs

@@ -78,7 +78,7 @@ private async Task<string> GetChatCompletion(Agent agent, List<RoleDialogModel>
         {
             var llmProviderService = _services.GetRequiredService<ILlmProviderService>();
             var provider = llmProviderService.GetProviders().FirstOrDefault(x => x == "openai");
-            var model = llmProviderService.GetProviderModel(provider: provider, id: "gpt-4", multiModal: true);
+            var model = llmProviderService.GetProviderModel(provider: provider, id: "gpt-4o", multiModal: true);
             var completion = CompletionProvider.GetChatCompletion(_services, provider: provider, model: model.Name);
             var response = await completion.GetChatCompletions(agent, dialogs);
             return response.Content;

src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadPdfFn.cs

@@ -61,26 +61,30 @@ public class RealtimeSessionTurnDetection
     /// <summary>
     /// Milliseconds
     /// </summary>
-    [JsonPropertyName("prefix_padding_ms")]
+    /*[JsonPropertyName("prefix_padding_ms")]
     public int PrefixPadding { get; set; } = 300;
 
     [JsonPropertyName("silence_duration_ms")]
     public int SilenceDuration { get; set; } = 500;
 
     [JsonPropertyName("threshold")]
-    public float Threshold { get; set; } = 0.5f;
+    public float Threshold { get; set; } = 0.5f;*/
 
     [JsonPropertyName("type")]
-    public string Type { get; set; } = "server_vad";
+    public string Type { get; set; } = "semantic_vad";
+
+    [JsonPropertyName("eagerness")]
+    public string eagerness { get;set; } = "auto";
 }
 
 public class InputAudioTranscription
 {
     [JsonPropertyName("model")]
-    public string Model { get; set; } = "whisper-1";
+    public string Model { get; set; } = "gpt-4o-transcribe";
 
     [JsonPropertyName("language")]
-    public string Language { get; set; } = "en";
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public string? Language { get; set; }
 
     [JsonPropertyName("prompt")]
     [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]

src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionBody.cs

@@ -33,7 +33,8 @@ public void RegisterDI(IServiceCollection services, IConfiguration config)
         services.AddScoped<IChatCompletion, ChatCompletionProvider>();
         services.AddScoped<ITextEmbedding, TextEmbeddingProvider>();
         services.AddScoped<IImageCompletion, ImageCompletionProvider>();
-        services.AddScoped<IAudioCompletion, AudioCompletionProvider>();
+        services.AddScoped<IAudioTranscription, AudioTranscriptionProvider>();
+        services.AddScoped<IAudioSynthesis, AudioSynthesisProvider>();
         services.AddScoped<IRealTimeCompletion, RealTimeCompletionProvider>();
 
         services.AddRefitClient<IOpenAiRealtimeApi>()