Merge branch 'main' into feature_agent_filters

Author: crickman

.github/_typos.toml

@@ -15,6 +15,7 @@ extend-exclude = [
     "CodeTokenizerTests.cs",
     "test_code_tokenizer.py",
     "*response.json",
+    "test_content.txt",
 ]
 
 [default.extend-words]

docs/decisions/0046-azure-model-as-a-service.md

@@ -0,0 +1,44 @@
+---
+# These are optional elements. Feel free to remove any of them.
+status: { accepted }
+contact: { rogerbarreto, taochen }
+date: { 2024-06-20 }
+deciders: { alliscode, moonbox3, eavanvalkenburg }
+consulted: {}
+informed: {}
+---
+
+# Support for Azure Model-as-a-Service in SK
+
+## Context and Problem Statement
+
+There has been a demand from customers for the implementation of Model-as-a-Service (MaaS) in SK. MaaS, which is also referred to as [serverless API](https://learn.microsoft.com/en-us/azure/ai-studio/how-to/model-catalog-overview#model-deployment-managed-compute-and-serverless-api-pay-as-you-go), is available in [Azure AI Studio](https://learn.microsoft.com/en-us/azure/ai-studio/what-is-ai-studio). This mode of consumption operates on a pay-as-you-go basis, typically using tokens for billing purposes. Clients can access the service via the [Azure AI Model Inference API](https://learn.microsoft.com/en-us/azure/ai-studio/reference/reference-model-inference-api?tabs=azure-studio) or client SDKs.
+
+At present, there is no official support for MaaS in SK. The purpose of this ADR is to examine the constraints of the service and explore potential solutions to enable support for the service in SK via the development of a new AI connector.
+
+## Client SDK
+
+The Azure team will be providing a new client library, namely `Azure.AI.Inference` in .Net and `azure-ai-inference` in Python, for effectively interacting with the service. While the service API is OpenAI-compatible, it is not permissible to use the OpenAI and the Azure OpenAI client libraries for interacting with the service as they are not independent with respect to both the models and their providers. This is because Azure AI Studio features a diverse range of open-source models, other than OpenAI models.
+
+### Limitations
+
+The initial release of the client SDK will only support chat completion and text/image embedding generation, with image generation to be added later.
+
+Plans to support for text completion are currently unclear, and it is highly unlikely that the SDK will ever include support for text completion. As a result, the new AI connector will **NOT** support text completions in the initial version until we get more customer signals or the client SDK adds support.
+
+## AI Connector
+
+### Naming options
+
+- Azure
+- AzureAI
+- AzureAIInference
+- AzureAIModelInference
+
+  Decision: `AzureAIInference`
+
+### Support for model-specific parameters
+
+Models can possess supplementary parameters that are not part of the default API. The service API and the client SDK enable the provision of model-specific parameters. Users can provide model-specific settings via a dedicated argument along with other settings, such as `temperature` and `top_p`, among others.
+
+In the context of SK, execution parameters are categorized under `PromptExecutionSettings`, which is inherited by all connector-specific setting classes. The settings of the new connector will contain a member of type `dictionary`, which will group together the model-specific parameters.

dotnet/samples/Concepts/Agents/OpenAIAssistant_FileService.cs

@@ -1,13 +1,12 @@
 // Copyright (c) Microsoft. All rights reserved.
-using Azure.AI.OpenAI.Assistants;
 using Microsoft.SemanticKernel;
 using Microsoft.SemanticKernel.Connectors.OpenAI;
 using Resources;
 
 namespace Agents;
 
 /// <summary>
-/// Demonstrate uploading and retrieving files with <see cref="OpenAIFileService"/> .
+/// Demonstrate using <see cref="OpenAIFileService"/> .
 /// </summary>
 public class OpenAIAssistant_FileService(ITestOutputHelper output) : BaseTest(output)
 {
@@ -19,7 +18,6 @@ public class OpenAIAssistant_FileService(ITestOutputHelper output) : BaseTest(ou
     [Fact]
     public async Task UploadAndRetrieveFilesAsync()
     {
-        var openAIClient = new AssistantsClient(TestConfiguration.OpenAI.ApiKey);
         OpenAIFileService fileService = new(TestConfiguration.OpenAI.ApiKey);
 
         BinaryContent[] files = [
@@ -29,41 +27,40 @@ public async Task UploadAndRetrieveFilesAsync()
             new BinaryContent(data: await EmbeddedResource.ReadAllAsync("travelinfo.txt"), mimeType: "text/plain") { InnerContent = "travelinfo.txt" }
         ];
 
-        var fileIds = new Dictionary<string, BinaryContent>();
-        foreach (var file in files)
+        var fileContents = new Dictionary<string, BinaryContent>();
+        foreach (BinaryContent file in files)
         {
-            var result = await openAIClient.UploadFileAsync(new BinaryData(file.Data), Azure.AI.OpenAI.Assistants.OpenAIFilePurpose.FineTune);
-            fileIds.Add(result.Value.Id, file);
+            OpenAIFileReference result = await fileService.UploadContentAsync(file, new(file.InnerContent!.ToString()!, OpenAIFilePurpose.FineTune));
+            fileContents.Add(result.Id, file);
         }
 
-        foreach (var file in (await openAIClient.GetFilesAsync(Azure.AI.OpenAI.Assistants.OpenAIFilePurpose.FineTune)).Value)
+        foreach (OpenAIFileReference fileReference in await fileService.GetFilesAsync(OpenAIFilePurpose.FineTune))
         {
-            if (!fileIds.ContainsKey(file.Id))
+            // Only interested in the files we uploaded
+            if (!fileContents.ContainsKey(fileReference.Id))
             {
                 continue;
             }
 
-            var data = (await openAIClient.GetFileContentAsync(file.Id)).Value;
+            BinaryContent content = await fileService.GetFileContentAsync(fileReference.Id);
 
-            var mimeType = fileIds[file.Id].MimeType;
-            var fileName = fileIds[file.Id].InnerContent!.ToString();
-            var metadata = new Dictionary<string, object?> { ["id"] = file.Id };
-            var uri = new Uri($"https://api.openai.com/v1/files/{file.Id}/content");
-            var content = mimeType switch
+            string? mimeType = fileContents[fileReference.Id].MimeType;
+            string? fileName = fileContents[fileReference.Id].InnerContent!.ToString();
+            ReadOnlyMemory<byte> data = content.Data ?? new();
+
+            var typedContent = mimeType switch
             {
-                "image/jpeg" => new ImageContent(data, mimeType) { Uri = uri, InnerContent = fileName, Metadata = metadata },
-                "audio/wav" => new AudioContent(data, mimeType) { Uri = uri, InnerContent = fileName, Metadata = metadata },
-                _ => new BinaryContent(data, mimeType) { Uri = uri, InnerContent = fileName, Metadata = metadata }
+                "image/jpeg" => new ImageContent(data, mimeType) { Uri = content.Uri, InnerContent = fileName, Metadata = content.Metadata },
+                "audio/wav" => new AudioContent(data, mimeType) { Uri = content.Uri, InnerContent = fileName, Metadata = content.Metadata },
+                _ => new BinaryContent(data, mimeType) { Uri = content.Uri, InnerContent = fileName, Metadata = content.Metadata }
             };
 
-            // Display the the file-name and mime-tyupe for each content type.
-            Console.WriteLine($"File: {fileName} - {mimeType}");
-
-            // Display the each content type-name.
-            Console.WriteLine($"Type: {content}");
+            Console.WriteLine($"\nFile: {fileName} - {mimeType}");
+            Console.WriteLine($"Type: {typedContent}");
+            Console.WriteLine($"Uri: {typedContent.Uri}");
 
             // Delete the test file remotely
-            await openAIClient.DeleteFileAsync(file.Id);
+            await fileService.DeleteFileAsync(fileReference.Id);
         }
     }
 }

dotnet/samples/Concepts/Functions/MethodFunctions_Yaml.cs

@@ -0,0 +1,73 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using System.Reflection;
+using Microsoft.SemanticKernel;
+
+namespace Functions;
+
+public class MethodFunctions_Yaml(ITestOutputHelper output) : BaseTest(output)
+{
+    private const string FunctionConfig = """
+        name: ValidateTaskId
+        description: Validate a task id.
+        input_variables:
+          - name: kernel
+            description: Kernel instance.
+          - name: taskId
+            description: Task identifier.
+            is_required: true
+        output_variable:
+          description: String indicating whether or not the task id is valid.
+        """;
+
+    /// <summary>
+    /// This example create a plugin and uses a separate configuration file for the function metadata.
+    /// </summary>
+    /// <remarks>
+    /// Some reasons you would want to do this:
+    /// 1. It's not possible to modify the existing code to add the KernelFunction attribute.
+    /// 2. You want to keep the function metadata separate from the function implementation.
+    /// </remarks>
+    [Fact]
+    public async Task CreateFunctionFromMethodWithYamlConfigAsync()
+    {
+        var kernel = new Kernel();
+
+        var config = KernelFunctionYaml.ToPromptTemplateConfig(FunctionConfig);
+
+        var target = new ValidatorPlugin();
+        MethodInfo method = target.GetType().GetMethod(config.Name!)!;
+        var functions = new List<KernelFunction>();
+        var functionName = config.Name;
+        var description = config.Description;
+        var parameters = config.InputVariables;
+        functions.Add(KernelFunctionFactory.CreateFromMethod(method, target, new()
+        {
+            FunctionName = functionName,
+            Description = description,
+            Parameters = parameters.Select(p => new KernelParameterMetadata(p.Name) { Description = p.Description, IsRequired = p.IsRequired }).ToList(),
+        }));
+
+        var plugin = kernel.ImportPluginFromFunctions("ValidatorPlugin", functions);
+
+        var function = plugin["ValidateTaskId"];
+        var result = await kernel.InvokeAsync(function, new() { { "taskId", "1234" } });
+        Console.WriteLine(result.GetValue<string>());
+
+        Console.WriteLine("Function Metadata:");
+        Console.WriteLine(function.Metadata.Description);
+        Console.WriteLine(function.Metadata.Parameters[0].Description);
+        Console.WriteLine(function.Metadata.Parameters[1].Description);
+    }
+
+    /// <summary>
+    /// Plugin example with no KernelFunction or Description attributes.
+    /// </summary>
+    private sealed class ValidatorPlugin
+    {
+        public string ValidateTaskId(Kernel kernel, string taskId)
+        {
+            return taskId.Equals("1234", StringComparison.Ordinal) ? "Valid task id" : "Invalid task id";
+        }
+    }
+}

dotnet/samples/Concepts/README.md

@@ -78,6 +78,7 @@ Down below you can find the code snippets that demonstrate the usage of many Sem
 - [MethodFunctions](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/Functions/MethodFunctions.cs)
 - [MethodFunctions_Advanced](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/Functions/MethodFunctions_Advanced.cs)
 - [MethodFunctions_Types](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/Functions/MethodFunctions_Types.cs)
+- [MethodFunctions_Yaml](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/Functions/MethodFunctions_Yaml.cs)
 - [PromptFunctions_Inline](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/Functions/PromptFunctions_Inline.cs)
 - [PromptFunctions_MultipleArguments](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/Functions/PromptFunctions_MultipleArguments.cs)
 

dotnet/samples/Demos/FunctionInvocationApproval/README.md

@@ -0,0 +1,44 @@
+# Function Invocation Approval
+
+This console application shows how to use function invocation filter (`IFunctionInvocationFilter`) to invoke a Kernel Function only if such operation was approved.
+If function invocation was rejected, the result will contain the reason why, so the LLM can respond appropriately.
+
+The application uses a sample plugin which builds software by following these development stages: collection of requirements, design, implementation, testing and deployment.
+
+Each step can be approved or rejected. Based on that, the LLM will decide how to proceed.
+
+## Configuring Secrets
+
+The example requires credentials to access OpenAI or Azure OpenAI.
+
+If you have set up those credentials as secrets within Secret Manager or through environment variables for other samples from the solution in which this project is found, they will be re-used.
+
+### To set your secrets with Secret Manager:
+
+```
+cd dotnet/samples/Demos/FunctionInvocationApproval
+
+dotnet user-secrets init
+
+dotnet user-secrets set "OpenAI:ChatModelId" "..."
+dotnet user-secrets set "OpenAI:ApiKey" "..."
+
+dotnet user-secrets set "AzureOpenAI:ChatDeploymentName" "..."
+dotnet user-secrets set "AzureOpenAI:Endpoint" "https://... .openai.azure.com/"
+dotnet user-secrets set "AzureOpenAI:ApiKey" "..."
+```
+
+### To set your secrets with environment variables
+
+Use these names:
+
+```
+# OpenAI
+OpenAI__ChatModelId
+OpenAI__ApiKey
+
+# Azure OpenAI
+AzureOpenAI__ChatDeploymentName
+AzureOpenAI__Endpoint
+AzureOpenAI__ApiKey
+```

dotnet/samples/Demos/HomeAutomation/README.md

@@ -12,7 +12,7 @@ If you have set up those credentials as secrets within Secret Manager or through
 ### To set your secrets with Secret Manager:
 
 ```
-cd dotnet/samples/HouseAutomation
+cd dotnet/samples/Demos/HouseAutomation
 
 dotnet user-secrets init
 

dotnet/samples/GettingStartedWithAgents/README.md

@@ -22,7 +22,7 @@ Example|Description
 [Step1_Agent](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/GettingStartedWithAgents/Step1_Agent.cs)|How to create and use an agent.
 [Step2_Plugins](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/GettingStartedWithAgents/Step2_Plugins.cs)|How to associate plug-ins with an agent.
 [Step3_Chat](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/GettingStartedWithAgents/Step3_Chat.cs)|How to create a conversation between agents.
-[Step4_KernelFunctionStrategies](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Step4_KernelFunctionStrategies/Step1_Agent.cs)|How to utilize a `KernelFunction` as a _chat strategy_.
+[Step4_KernelFunctionStrategies](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/GettingStartedWithAgents/Step4_KernelFunctionStrategies.cs)|How to utilize a `KernelFunction` as a _chat strategy_.
 [Step5_JsonResult](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/GettingStartedWithAgents/Step5_JsonResult.cs)|How to have an agent produce JSON.
 [Step6_DependencyInjection](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/GettingStartedWithAgents/Step6_DependencyInjection.cs)|How to define dependency injection patterns for agents.
 [Step7_OpenAIAssistant](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/GettingStartedWithAgents/Step7_OpenAIAssistant.cs)|How to create an Open AI Assistant agent.

dotnet/src/Connectors/Connectors.HuggingFace/Core/HuggingFaceMessageApiClient.cs

@@ -85,9 +85,8 @@ internal async IAsyncEnumerable<StreamingChatMessageContent> StreamCompleteChatM
         var endpoint = this.GetChatGenerationEndpoint();
 
         var huggingFaceExecutionSettings = HuggingFacePromptExecutionSettings.FromExecutionSettings(executionSettings);
-        huggingFaceExecutionSettings.ModelId ??= this._clientCore.ModelId;
 
-        var request = this.CreateChatRequest(chatHistory, huggingFaceExecutionSettings);
+        var request = this.CreateChatRequest(chatHistory, huggingFaceExecutionSettings, modelId);
         request.Stream = true;
 
         using var activity = ModelDiagnostics.StartCompletionActivity(endpoint, modelId, this._clientCore.ModelProvider, chatHistory, huggingFaceExecutionSettings);
@@ -149,8 +148,7 @@ internal async Task<IReadOnlyList<ChatMessageContent>> CompleteChatMessageAsync(
         var endpoint = this.GetChatGenerationEndpoint();
 
         var huggingFaceExecutionSettings = HuggingFacePromptExecutionSettings.FromExecutionSettings(executionSettings);
-        huggingFaceExecutionSettings.ModelId ??= this._clientCore.ModelId;
-        var request = this.CreateChatRequest(chatHistory, huggingFaceExecutionSettings);
+        var request = this.CreateChatRequest(chatHistory, huggingFaceExecutionSettings, modelId);
 
         using var activity = ModelDiagnostics.StartCompletionActivity(endpoint, modelId, this._clientCore.ModelProvider, chatHistory, huggingFaceExecutionSettings);
         using var httpRequestMessage = this._clientCore.CreatePost(request, endpoint, this._clientCore.ApiKey);
@@ -276,7 +274,8 @@ private async IAsyncEnumerable<StreamingChatMessageContent> ProcessChatResponseS
 
     private ChatCompletionRequest CreateChatRequest(
         ChatHistory chatHistory,
-        HuggingFacePromptExecutionSettings huggingFaceExecutionSettings)
+        HuggingFacePromptExecutionSettings huggingFaceExecutionSettings,
+        string modelId)
     {
         HuggingFaceClient.ValidateMaxTokens(huggingFaceExecutionSettings.MaxTokens);
 
@@ -287,7 +286,7 @@ private ChatCompletionRequest CreateChatRequest(
                 JsonSerializer.Serialize(huggingFaceExecutionSettings));
         }
 
-        var request = ChatCompletionRequest.FromChatHistoryAndExecutionSettings(chatHistory, huggingFaceExecutionSettings);
+        var request = ChatCompletionRequest.FromChatHistoryAndExecutionSettings(chatHistory, huggingFaceExecutionSettings, modelId);
         return request;
     }
 

dotnet/src/Connectors/Connectors.HuggingFace/Core/Models/ChatCompletionRequest.cs

@@ -102,8 +102,9 @@ internal sealed class ChatCompletionRequest
     /// </summary>
     /// <param name="chatHistory">Chat history to be used for the request.</param>
     /// <param name="executionSettings">Execution settings to be used for the request.</param>
-    /// <returns>TexGenerationtRequest object.</returns>
-    internal static ChatCompletionRequest FromChatHistoryAndExecutionSettings(ChatHistory chatHistory, HuggingFacePromptExecutionSettings executionSettings)
+    /// <param name="modelId">Model id to use if value in prompt execution settings is not set.</param>
+    /// <returns>TexGenerationRequest object.</returns>
+    internal static ChatCompletionRequest FromChatHistoryAndExecutionSettings(ChatHistory chatHistory, HuggingFacePromptExecutionSettings executionSettings, string modelId)
     {
         return new ChatCompletionRequest
         {
@@ -118,7 +119,7 @@ internal static ChatCompletionRequest FromChatHistoryAndExecutionSettings(ChatHi
             Temperature = executionSettings.Temperature,
             Stop = executionSettings.Stop,
             MaxTokens = executionSettings.MaxTokens,
-            Model = executionSettings.ModelId ?? TextGenerationInferenceDefaultModel,
+            Model = executionSettings.ModelId ?? modelId ?? TextGenerationInferenceDefaultModel,
             TopP = executionSettings.TopP,
             TopLogProbs = executionSettings.TopLogProbs
         };