Skip to content

Commit b5439b1

Browse files
Oceania2018Oceania2018
authored
Merge pull request #562 from iceljc/features/refine-json-format
Features/refine json format
2 parents da79eec + bc76a49 commit b5439b1

File tree

14 files changed

+156
-88
lines changed

14 files changed

+156
-88
lines changed

‎src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,11 @@ public class LlmModelSetting
1717
/// </summary>
1818
public string Version { get; set; } = "1106-Preview";
1919

20+
/// <summary>
21+
/// Api version
22+
/// </summary>
23+
public string? ApiVersion { get; set; }
24+
2025
/// <summary>
2126
/// Deployment same functional model in a group.
2227
/// It can be used to deploy same model in different regions.

‎src/Infrastructure/BotSharp.Abstraction/Utilities/JsonExtensions.cs

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
using Newtonsoft.Json.Linq;
2+
using Newtonsoft.Json;
3+
4+
namespace BotSharp.Abstraction.Utilities;
5+
6+
public static class JsonExtensions
7+
{
8+
public static string FormatJson(this string? json, Formatting format = Formatting.Indented)
9+
{
10+
if (string.IsNullOrWhiteSpace(json))
11+
{
12+
return "{}";
13+
}
14+
15+
try
16+
{
17+
var parsedJson = JObject.Parse(json);
18+
foreach (var item in parsedJson)
19+
{
20+
try
21+
{
22+
var key = item.Key;
23+
var value = parsedJson[key].ToString();
24+
var parsedValue = JObject.Parse(value);
25+
parsedJson[key] = parsedValue;
26+
}
27+
catch { continue; }
28+
}
29+
30+
var jsonSettings = new JsonSerializerSettings
31+
{
32+
Formatting = format
33+
};
34+
return JsonConvert.SerializeObject(parsedJson, jsonSettings);
35+
}
36+
catch
37+
{
38+
return json;
39+
}
40+
}
41+
}

‎src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Text/TextCompletionProvider.cs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -132,11 +132,11 @@ private async Task<TextCompletionResponse> GetTextCompletion(string apiUrl, stri
132132

133133
private string BuildApiUrl(LlmModelSetting modelSetting)
134134
{
135-
var url = string.Empty;
135+
var apiVersion = !string.IsNullOrWhiteSpace(modelSetting.ApiVersion) ? modelSetting.ApiVersion : "2023-09-15-preview";
136136
var endpoint = modelSetting.Endpoint.EndsWith("/") ?
137137
modelSetting.Endpoint.Substring(0, modelSetting.Endpoint.Length - 1) : modelSetting.Endpoint;
138138

139-
url = $"{endpoint}/openai/deployments/{_model}/completions?api-version=2023-09-15-preview";
139+
var url = $"{endpoint}/openai/deployments/{_model}/completions?api-version={apiVersion}";
140140
return url;
141141
}
142142

‎src/Plugins/BotSharp.Plugin.ChatHub/Hooks/ChatHubConversationHook.cs

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,3 @@
1-
using BotSharp.Abstraction.Messaging.Enums;
2-
using BotSharp.Abstraction.Options;
31
using Microsoft.AspNetCore.SignalR;
42

53
namespace BotSharp.Plugin.ChatHub.Hooks;

‎src/Plugins/BotSharp.Plugin.ChatHub/Hooks/StreamingLogHook.cs

Lines changed: 1 addition & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,3 @@
1-
using BotSharp.Abstraction.Agents.Models;
2-
using BotSharp.Abstraction.Functions.Models;
3-
using BotSharp.Abstraction.Loggers;
4-
using BotSharp.Abstraction.Loggers.Enums;
5-
using BotSharp.Abstraction.Loggers.Models;
6-
using BotSharp.Abstraction.Options;
7-
using BotSharp.Abstraction.Repositories;
8-
using BotSharp.Abstraction.Routing;
91
using Microsoft.AspNetCore.SignalR;
102
using System.Text.Encodings.Web;
113
using System.Text.Unicode;
@@ -126,7 +118,7 @@ public override async Task OnFunctionExecuting(RoleDialogModel message)
126118

127119
var agent = await _agentService.LoadAgent(message.CurrentAgentId);
128120
message.FunctionArgs = message.FunctionArgs ?? "{}";
129-
var args = FormatJson(message.FunctionArgs);
121+
var args = message.FunctionArgs.FormatJson();
130122
var log = $"{message.FunctionName} <u>executing</u>\r\n```json\r\n{args}\r\n```";
131123

132124
var input = new ContentLogInputModel(conversationId, message)
@@ -567,40 +559,5 @@ private JsonSerializerOptions InitLocalJsonOptions(BotSharpOptions options)
567559

568560
return localOptions;
569561
}
570-
571-
private string FormatJson(string? json)
572-
{
573-
var defaultJson = "{}";
574-
if (string.IsNullOrWhiteSpace(json))
575-
{
576-
return defaultJson;
577-
}
578-
579-
try
580-
{
581-
var parsedJson = JObject.Parse(json);
582-
foreach (var item in parsedJson)
583-
{
584-
try
585-
{
586-
var key = item.Key;
587-
var value = parsedJson[key].ToString();
588-
var parsedValue = JObject.Parse(value);
589-
parsedJson[key] = parsedValue;
590-
}
591-
catch { continue; }
592-
}
593-
594-
var jsonSettings = new JsonSerializerSettings
595-
{
596-
Formatting = Newtonsoft.Json.Formatting.Indented
597-
};
598-
return JsonConvert.SerializeObject(parsedJson, jsonSettings) ?? defaultJson;
599-
}
600-
catch
601-
{
602-
return defaultJson;
603-
}
604-
}
605562
#endregion
606563
}

‎src/Plugins/BotSharp.Plugin.ChatHub/Hooks/WelcomeHook.cs

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,4 @@
1-
using BotSharp.Abstraction.Messaging.Models.RichContent;
2-
using BotSharp.Abstraction.Messaging;
3-
using BotSharp.Abstraction.Templating;
41
using Microsoft.AspNetCore.SignalR;
5-
using BotSharp.Abstraction.Options;
62

73
namespace BotSharp.Plugin.ChatHub.Hooks;
84

‎src/Plugins/BotSharp.Plugin.ChatHub/Using.cs

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,4 +19,16 @@
1919
global using BotSharp.Abstraction.Agents.Enums;
2020
global using BotSharp.Abstraction.Conversations.Models;
2121
global using BotSharp.OpenAPI.ViewModels.Conversations;
22-
global using BotSharp.OpenAPI.ViewModels.Users;
22+
global using BotSharp.OpenAPI.ViewModels.Users;
23+
global using BotSharp.Abstraction.Agents.Models;
24+
global using BotSharp.Abstraction.Functions.Models;
25+
global using BotSharp.Abstraction.Loggers;
26+
global using BotSharp.Abstraction.Loggers.Enums;
27+
global using BotSharp.Abstraction.Loggers.Models;
28+
global using BotSharp.Abstraction.Options;
29+
global using BotSharp.Abstraction.Repositories;
30+
global using BotSharp.Abstraction.Routing;
31+
global using BotSharp.Abstraction.Messaging;
32+
global using BotSharp.Abstraction.Messaging.Enums;
33+
global using BotSharp.Abstraction.Messaging.Models.RichContent;
34+
global using BotSharp.Abstraction.Templating;

‎src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailRequestFn.cs

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -78,8 +78,9 @@ private async Task<IEnumerable<MessageFileModel>> GetConversationFiles()
7878
var conversationId = convService.ConversationId;
7979
var dialogs = convService.GetDialogHistory(fromBreakpoint: false);
8080
var messageIds = dialogs.Select(x => x.MessageId).Distinct().ToList();
81-
var files = fileService.GetMessageFiles(conversationId, messageIds, FileSourceType.User);
82-
return await SelectFiles(files, dialogs);
81+
var userFiles = fileService.GetMessageFiles(conversationId, messageIds, FileSourceType.User);
82+
var botFiles = fileService.GetMessageFiles(conversationId, messageIds, FileSourceType.Bot);
83+
return await SelectFiles(userFiles.Concat(botFiles), dialogs);
8384
}
8485

8586
private async Task<IEnumerable<MessageFileModel>> SelectFiles(IEnumerable<MessageFileModel> files, List<RoleDialogModel> dialogs)
@@ -94,7 +95,7 @@ private async Task<IEnumerable<MessageFileModel>> SelectFiles(IEnumerable<Messag
9495
{
9596
var promptFiles = files.Select((x, idx) =>
9697
{
97-
return $"id: {idx + 1}, file_name: {x.FileName}.{x.FileType}, content_type: {x.ContentType}";
98+
return $"id: {idx + 1}, file_name: {x.FileName}.{x.FileType}, author: {x.FileSource}, content_type: {x.ContentType}";
9899
}).ToList();
99100
var prompt = db.GetAgentTemplate(BuiltInAgentId.UtilityAssistant, "select_attachment_prompt");
100101
prompt = render.Render(prompt, new Dictionary<string, object>
@@ -114,7 +115,8 @@ private async Task<IEnumerable<MessageFileModel>> SelectFiles(IEnumerable<Messag
114115
var completion = CompletionProvider.GetChatCompletion(_services, provider: provider, model: model.Name);
115116
var response = await completion.GetChatCompletions(agent, dialogs);
116117
var content = response?.Content ?? string.Empty;
117-
var fids = JsonSerializer.Deserialize<List<int>>(content) ?? new List<int>();
118+
var selecteds = JsonSerializer.Deserialize<LlmContextOut>(content);
119+
var fids = selecteds?.Selecteds ?? new List<int>();
118120
return files.Where((x, idx) => fids.Contains(idx + 1)).ToList();
119121
}
120122
catch (Exception ex)
@@ -146,6 +148,6 @@ private async Task<string> SendEmailBySMTP(MimeMessage mailMessage)
146148
await smtpClient.ConnectAsync(_emailSettings.SMTPServer, _emailSettings.SMTPPort, SecureSocketOptions.StartTls);
147149
await smtpClient.AuthenticateAsync(_emailSettings.EmailAddress, _emailSettings.Password);
148150
var response = await smtpClient.SendAsync(mailMessage);
149-
return response;
151+
return response ?? "Email sent";
150152
}
151153
}

‎src/Plugins/BotSharp.Plugin.EmailHandler/LlmContexts/LlmContextOut.cs

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
using System.Text.Json.Serialization;
2+
3+
namespace BotSharp.Plugin.EmailHandler.LlmContexts;
4+
5+
public class LlmContextOut
6+
{
7+
[JsonPropertyName("selected_ids")]
8+
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
9+
public IEnumerable<int> Selecteds { get; set; }
10+
}

‎src/Plugins/BotSharp.Plugin.EmailHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/handle_email_request.fn.liquid

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,2 @@
1-
Please call handle_email_request if user wants to send out an email.
2-
** Please take a look at the conversation, and decide whether user wants to send email with attachments or not.
1+
Please call handle_email_request if user wants to send email.
2+
** Please take a look at the conversation and decide whether user wants to send email with files/attachments/images or not.

‎src/Plugins/BotSharp.Plugin.EmailHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/select_attachment_prompt.liquid

Lines changed: 35 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,42 @@
11
Please take a look at the files in the [FILES] section from the conversation and select the files based on the conversation with user.
2-
Your response must be a list of file ids.
3-
** Please only output the list. Do not prepend or append anything.
42

5-
For example:
6-
Suppose there are three files:
3+
** Ensure the output is only in JSON format without any additional text.
4+
** If no files are selected, you must output an empty list [].
5+
** You may need to look at the file_name as a reference to find the correct file id.
76

8-
id: 1, file_name: example_file.png, content_type: image/png
9-
id: 2, file_name: example_file.jpeg, content_type: image/jpeg
10-
id: 3, file_name: example_file.pdf, content_type: application/pdf
7+
Here is the JSON format to use:
8+
{
9+
"selected_ids": a list of id selected from the [FILES] section
10+
}
1111

12-
If user wants the first file and the third file, the ouput should be [1, 3].
13-
If user wants the all the images, the output should be [1, 2].
14-
If user wants the pdf file, the output should be [3].
15-
If user does not want any files, the ouput should be [];
12+
Suppose there are 4 files:
13+
14+
id: 1, file_name: example_file.jpg, author: user, content_type: image/jpeg
15+
id: 2, file_name: example_file.pdf, author: user, content_type: application/pdf
16+
id: 3, file_name: example_file.png, author: bot, content_type: image/png
17+
id: 4, file_name: example_file.png, author: bot, content_type: image/png
18+
19+
=====
20+
Example 1:
21+
USER: I want to send the first file and the third file.
22+
OUTPUT: { "selected_ids": [1, 3] }
23+
24+
Example 2:
25+
USER: Send all the images.
26+
OUTPUT: { "selected_ids": [1, 2, 4] }
27+
28+
Example 3:
29+
USER: Send all the images I uploaded.
30+
OUTPUT: { "selected_ids": [1] }
31+
32+
Example 4:
33+
USER: Send the image and the pdf file.
34+
OUTPUT: { "selected_ids": [1, 2] }
35+
36+
Example 5:
37+
USER: Send the images generated by bot
38+
OUTPUT: { "selected_ids": [3, 4] }
39+
=====
1640

1741
[FILES]
1842
{% for file in file_list -%}

‎src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs

Lines changed: 15 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ public async Task<bool> Execute(RoleDialogModel message)
2828
Init(message);
2929
SetImageOptions();
3030

31-
var image = await SelectConversationImage();
31+
var image = await SelectConversationImage(descrpition);
3232
var response = await GetImageEditGeneration(message, descrpition, image);
3333
message.Content = response;
3434
return true;
@@ -48,17 +48,17 @@ private void SetImageOptions()
4848
state.SetState("image_count", "1");
4949
}
5050

51-
private async Task<MessageFileModel?> SelectConversationImage()
51+
private async Task<MessageFileModel?> SelectConversationImage(string? description)
5252
{
5353
var convService = _services.GetRequiredService<IConversationService>();
5454
var fileService = _services.GetRequiredService<IBotSharpFileService>();
55-
var dialogs = convService.GetDialogHistory(fromBreakpoint: false);
55+
var dialogs = convService.GetDialogHistory();
5656
var messageIds = dialogs.Select(x => x.MessageId).Distinct().ToList();
57-
var images = fileService.GetMessageFiles(_conversationId, messageIds, FileSourceType.User, imageOnly: true);
58-
return await SelectImage(images, dialogs);
57+
var userImages = fileService.GetMessageFiles(_conversationId, messageIds, FileSourceType.User, imageOnly: true);
58+
return await SelectImage(userImages, dialogs.LastOrDefault(), description);
5959
}
6060

61-
private async Task<MessageFileModel?> SelectImage(IEnumerable<MessageFileModel> images, List<RoleDialogModel> dialogs)
61+
private async Task<MessageFileModel?> SelectImage(IEnumerable<MessageFileModel> images, RoleDialogModel message, string? description)
6262
{
6363
if (images.IsNullOrEmpty()) return null;
6464

@@ -91,10 +91,15 @@ private void SetImageOptions()
9191
var provider = llmProviderService.GetProviders().FirstOrDefault(x => x == "openai");
9292
var model = llmProviderService.GetProviderModel(provider: provider, id: "gpt-4");
9393
var completion = CompletionProvider.GetChatCompletion(_services, provider: provider, model: model.Name);
94-
var response = await completion.GetChatCompletions(agent, dialogs);
94+
95+
var text = !string.IsNullOrWhiteSpace(description) ? description : message.Content;
96+
var dialog = RoleDialogModel.From(message, AgentRole.User, text);
97+
98+
var response = await completion.GetChatCompletions(agent, new List<RoleDialogModel> { dialog });
9599
var content = response?.Content ?? string.Empty;
96-
var fid = JsonSerializer.Deserialize<int?>(content);
97-
return images.Where((x, idx) => idx == fid - 1).FirstOrDefault();
100+
var selected = JsonSerializer.Deserialize<LlmContextOut>(content);
101+
var fid = selected?.Selected ?? -1;
102+
return fid > 0 ? images.Where((x, idx) => idx == fid - 1).FirstOrDefault() : null;
98103
}
99104
catch (Exception ex)
100105
{
@@ -126,7 +131,7 @@ private async Task<string> GetImageEditGeneration(RoleDialogModel message, strin
126131
stream.Close();
127132
SaveGeneratedImage(result?.GeneratedImages?.FirstOrDefault());
128133

129-
return !string.IsNullOrWhiteSpace(result?.Content) ? result.Content : "Image edit is completed.";
134+
return $"Image \"{image.FileName}.{image.FileType}\" is successfylly editted.";
130135
}
131136
catch (Exception ex)
132137
{

‎src/Plugins/BotSharp.Plugin.FileHandler/LlmContexts/LlmContextOut.cs

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
using System.Text.Json.Serialization;
2+
3+
namespace BotSharp.Plugin.FileHandler.LlmContexts;
4+
5+
public class LlmContextOut
6+
{
7+
[JsonPropertyName("selected_id")]
8+
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
9+
public int? Selected { get; set; }
10+
}

‎src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/select_edit_image_prompt.liquid

Lines changed: 15 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,30 +1,38 @@
11
Please take a look at the images in the [IMAGES] section from the conversation and select ONLY one image based on the conversation with user.
22
Your response must be an interger number.
3-
** Please ONLY output the interger number. Do not prepend or append anything else.
4-
** If you think user requests multiple images. Please ONLY select the first image and output its id.
3+
4+
** Ensure the output is only in JSON format without any additional text.
5+
** You may need to look at the image_name as a reference to find the correct image id.
6+
7+
Here is the JSON format to use:
8+
{
9+
"selected_id": the id selected from the [IMAGES] section
10+
}
11+
512

613
Suppose there are three images:
714

815
id: 1, image_name: example_image_a.png
916
id: 2, image_name: example_image_b.png
1017
id: 3, image_name: example_image_c.png
18+
id: 4, image_name: example_image_d.png
1119

1220
=====
1321
Example 1:
1422
USER: I want to add a dog in the first file.
15-
OUTPUT: 1
23+
OUTPUT: { "selected_id": 1 }
1624

1725
Example 2:
1826
USER: Add a coffee cup in the second image I uploaded.
19-
OUTPUT: 2
27+
OUTPUT: { "selected_id": 2 }
2028

2129
Example 3:
2230
USER: Please remove the left tree in the third and the first images.
23-
OUTPUT: 3
31+
OUTPUT: { "selected_id": 3 }
2432

2533
Example 4:
26-
USER: Add a boat in the images.
27-
OUTPUT: 1
34+
USER: Circle the head of the dog in example_image_b.png.
35+
OUTPUT: { "selected_id": 4 }
2836
=====
2937

3038

0 commit comments

Comments
 (0)