Skip to content

refine global stats #848

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Jan 28, 2025
Merged

refine global stats #848

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 5 additions & 2 deletions src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
using BotSharp.Abstraction.Repositories.Filters;
using BotSharp.Abstraction.Roles.Models;
using BotSharp.Abstraction.Shared;
using BotSharp.Abstraction.Statistics.Enums;
using BotSharp.Abstraction.Statistics.Models;
using BotSharp.Abstraction.Tasks.Models;
using BotSharp.Abstraction.Translation.Models;
Expand Down Expand Up @@ -120,8 +121,10 @@ public interface IBotSharpRepository : IHaveServiceProvider
#endregion

#region Statistics
BotSharpStats? GetGlobalStats(string metric, string dimension, DateTime recordTime) => throw new NotImplementedException();
bool SaveGlobalStats(BotSharpStats body) => throw new NotImplementedException();
BotSharpStats? GetGlobalStats(string metric, string dimension, DateTime recordTime, StatsInterval interval)
=> throw new NotImplementedException();
bool SaveGlobalStats(BotSharpStats body)
=> throw new NotImplementedException();

#endregion

Expand Down
9 changes: 9 additions & 0 deletions src/Infrastructure/BotSharp.Abstraction/Statistics/Enums/StatsInterval.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
namespace BotSharp.Abstraction.Statistics.Enums;

public enum StatsInterval
{
Hour = 1,
Day = 2,
Week = 3,
Month = 4
}
30 changes: 22 additions & 8 deletions src/Infrastructure/BotSharp.Abstraction/Statistics/Models/BotSharpStats.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
using BotSharp.Abstraction.Statistics.Enums;

namespace BotSharp.Abstraction.Statistics.Models;

public class BotSharpStats
Expand All @@ -11,24 +13,36 @@ public class BotSharpStats
[JsonPropertyName("data")]
public IDictionary<string, double> Data { get; set; } = new Dictionary<string, double>();

private DateTime innerRecordTime;

[JsonPropertyName("record_time")]
public DateTime RecordTime
public DateTime RecordTime { get; set; } = DateTime.UtcNow;

[JsonIgnore]
public StatsInterval IntervalType { get; set; }

[JsonPropertyName("interval")]
public string Interval
{
get
{
return innerRecordTime;
}
return IntervalType.ToString();
}
set
{
var date = new DateTime(value.Year, value.Month, value.Day, value.Hour, 0, 0);
innerRecordTime = DateTime.SpecifyKind(date, DateTimeKind.Utc);
if (Enum.TryParse(value, out StatsInterval type))
{
IntervalType = type;
}
}
}

[JsonPropertyName("start_time")]
public DateTime StartTime { get; set; }

[JsonPropertyName("end_time")]
public DateTime EndTime { get; set; }

public override string ToString()
{
return $"{Metric}-{Dimension}: {Data?.Count ?? 0} ({RecordTime})";
return $"{Metric}-{Dimension} ({Interval}): {Data?.Count ?? 0}";
}
}
5 changes: 4 additions & 1 deletion src/Infrastructure/BotSharp.Abstraction/Statistics/Models/BotSharpStatsInput.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,12 @@
using BotSharp.Abstraction.Statistics.Enums;

namespace BotSharp.Abstraction.Statistics.Models;

public class BotSharpStatsInput
{
public string Metric { get; set; }
public string Dimension { get; set; }
public List<StatsKeyValuePair> Data { get; set; } = [];
public DateTime RecordTime { get; set; }
public DateTime RecordTime { get; set; } = DateTime.UtcNow;
public StatsInterval IntervalType { get; set; } = StatsInterval.Day;
}
1 change: 1 addition & 0 deletions src/Infrastructure/BotSharp.Core/Conversations/Services/TokenStatistics.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,7 @@ public void AddToken(TokenStatsModel stats, RoleDialogModel message)
Metric = StatsCategory.AgentLlmCost,
Dimension = message.CurrentAgentId,
RecordTime = DateTime.UtcNow,
IntervalType = StatsInterval.Day,
Data = [
new StatsKeyValuePair("prompt_token_count_total", stats.PromptCount),
new StatsKeyValuePair("completion_token_count_total", stats.CompletionCount),
Expand Down
55 changes: 45 additions & 10 deletions src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Stats.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,28 +4,34 @@ namespace BotSharp.Core.Repository;

public partial class FileRepository
{
public BotSharpStats? GetGlobalStats(string metric, string dimension, DateTime recordTime)
public BotSharpStats? GetGlobalStats(string metric, string dimension, DateTime recordTime, StatsInterval interval)
{
var baseDir = Path.Combine(_dbSettings.FileRepository, STATS_FOLDER);
var dir = Path.Combine(baseDir, metric, recordTime.Year.ToString(), recordTime.Month.ToString("D2"));
var (startTime, endTime) = BuildTimeInterval(recordTime, interval);
var dir = Path.Combine(baseDir, metric, startTime.Year.ToString(), startTime.Month.ToString("D2"));
if (!Directory.Exists(dir)) return null;

var file = Directory.GetFiles(dir).FirstOrDefault(x => Path.GetFileName(x) == STATS_FILE);
if (file == null) return null;

var time = BuildRecordTime(recordTime);
var text = File.ReadAllText(file);
var list = JsonSerializer.Deserialize<List<BotSharpStats>>(text, _options);
var found = list?.FirstOrDefault(x => x.Metric.IsEqualTo(metric)
&& x.Dimension.IsEqualTo(dimension)
&& x.RecordTime == time);
&& x.StartTime == startTime
&& x.EndTime == endTime);

return found;
}

public bool SaveGlobalStats(BotSharpStats body)
{
var baseDir = Path.Combine(_dbSettings.FileRepository, STATS_FOLDER);
var dir = Path.Combine(baseDir, body.Metric, body.RecordTime.Year.ToString(), body.RecordTime.Month.ToString("D2"));
var (startTime, endTime) = BuildTimeInterval(body.RecordTime, body.IntervalType);
body.StartTime = startTime;
body.EndTime = endTime;

var dir = Path.Combine(baseDir, body.Metric, startTime.Year.ToString(), startTime.Month.ToString("D2"));
if (!Directory.Exists(dir))
{
Directory.CreateDirectory(dir);
Expand All @@ -39,19 +45,22 @@ public bool SaveGlobalStats(BotSharpStats body)
}
else
{
var time = BuildRecordTime(body.RecordTime);
var text = File.ReadAllText(file);
var list = JsonSerializer.Deserialize<List<BotSharpStats>>(text, _options);
var found = list?.FirstOrDefault(x => x.Metric.IsEqualTo(body.Metric)
&& x.Dimension.IsEqualTo(body.Dimension)
&& x.RecordTime == time);
&& x.StartTime == startTime
&& x.EndTime == endTime);

if (found != null)
{
found.Metric = body.Metric;
found.Dimension = body.Dimension;
found.Data = body.Data;
found.RecordTime = body.RecordTime;
found.StartTime = body.StartTime;
found.EndTime = body.EndTime;
found.Interval = body.Interval;
}
else if (list != null)
{
Expand All @@ -69,10 +78,36 @@ public bool SaveGlobalStats(BotSharpStats body)
}

#region Private methods
private DateTime BuildRecordTime(DateTime date)
private (DateTime, DateTime) BuildTimeInterval(DateTime recordTime, StatsInterval interval)
{
var recordDate = new DateTime(date.Year, date.Month, date.Day, date.Hour, 0, 0);
return DateTime.SpecifyKind(recordDate, DateTimeKind.Utc);
DateTime startTime = recordTime;
DateTime endTime = DateTime.UtcNow;

switch (interval)
{
case StatsInterval.Hour:
startTime = new DateTime(recordTime.Year, recordTime.Month, recordTime.Day, recordTime.Hour, 0, 0);
endTime = startTime.AddHours(1);
break;
case StatsInterval.Week:
var dayOfWeek = startTime.DayOfWeek;
var firstDayOfWeek = startTime.AddDays(-(int)dayOfWeek);
startTime = new DateTime(firstDayOfWeek.Year, firstDayOfWeek.Month, firstDayOfWeek.Day, 0, 0, 0);
endTime = startTime.AddDays(7);
break;
case StatsInterval.Month:
startTime = new DateTime(recordTime.Year, recordTime.Month, 1);
endTime = startTime.AddMonths(1);
break;
default:
startTime = new DateTime(recordTime.Year, recordTime.Month, recordTime.Day, 0, 0, 0);
endTime = startTime.AddDays(1);
break;
}

startTime = DateTime.SpecifyKind(startTime, DateTimeKind.Utc);
endTime = DateTime.SpecifyKind(endTime, DateTimeKind.Utc);
return (startTime, endTime);
}
#endregion
}
6 changes: 4 additions & 2 deletions src/Infrastructure/BotSharp.Core/Statistics/Services/BotSharpStatsService.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,8 @@ public bool UpdateStats(string resourceKey, BotSharpStatsInput input)
|| string.IsNullOrEmpty(resourceKey)
|| input == null
|| string.IsNullOrEmpty(input.Metric)
|| string.IsNullOrEmpty(input.Dimension))
|| string.IsNullOrEmpty(input.Dimension)
|| input.Data.IsNullOrEmpty())
{
return false;
}
Expand All @@ -39,14 +40,15 @@ public bool UpdateStats(string resourceKey, BotSharpStatsInput input)
var res = locker.Lock(resourceKey, () =>
{
var db = _services.GetRequiredService<IBotSharpRepository>();
var body = db.GetGlobalStats(input.Metric, input.Dimension, input.RecordTime);
var body = db.GetGlobalStats(input.Metric, input.Dimension, input.RecordTime, input.IntervalType);
if (body == null)
{
var stats = new BotSharpStats
{
Metric = input.Metric,
Dimension = input.Dimension,
RecordTime = input.RecordTime,
IntervalType = input.IntervalType,
Data = input.Data.ToDictionary(x => x.Key, x => x.Value)
};
db.SaveGlobalStats(stats);
Expand Down
5 changes: 3 additions & 2 deletions src/Infrastructure/BotSharp.Logger/Hooks/GlobalStatsConversationHook.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,10 +33,11 @@ private void UpdateAgentCall(RoleDialogModel message)
{
Metric = StatsCategory.AgentCall,
Dimension = message.CurrentAgentId,
RecordTime = DateTime.UtcNow,
IntervalType = StatsInterval.Day,
Data = [
new StatsKeyValuePair("agent_call_count", 1)
],
RecordTime = DateTime.UtcNow
]
};
globalStats.UpdateStats("global-agent-call", body);
}
Expand Down
5 changes: 5 additions & 0 deletions src/Plugins/BotSharp.Plugin.MongoStorage/Collections/GlobalStatisticsDocument.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
using BotSharp.Abstraction.Statistics.Enums;

namespace BotSharp.Plugin.MongoStorage.Collections;

public class GlobalStatisticsDocument : MongoBase
Expand All @@ -6,4 +8,7 @@ public class GlobalStatisticsDocument : MongoBase
public string Dimension { get; set; }
public IDictionary<string, double> Data { get; set; } = new Dictionary<string, double>();
public DateTime RecordTime { get; set; }
public DateTime StartTime { get; set; }
public DateTime EndTime { get; set; }
public string Interval { get; set; }
}
59 changes: 49 additions & 10 deletions src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Stats.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,19 +1,21 @@
using BotSharp.Abstraction.Statistics.Enums;
using BotSharp.Abstraction.Statistics.Models;

namespace BotSharp.Plugin.MongoStorage.Repository;

public partial class MongoRepository
{
public BotSharpStats? GetGlobalStats(string metric, string dimension, DateTime recordTime)
public BotSharpStats? GetGlobalStats(string metric, string dimension, DateTime recordTime, StatsInterval interval)
{
var time = BuildRecordTime(recordTime);
var (startTime, endTime) = BuildTimeInterval(recordTime, interval);

var builder = Builders<GlobalStatisticsDocument>.Filter;
var filters = new List<FilterDefinition<GlobalStatisticsDocument>>()
{
builder.Eq(x => x.Metric, metric),
builder.Eq(x => x.Dimension, dimension),
builder.Eq(x => x.RecordTime, time)
builder.Eq(x => x.StartTime, startTime),
builder.Eq(x => x.EndTime, endTime)
};

var filterDef = builder.And(filters);
Expand All @@ -25,19 +27,27 @@ public partial class MongoRepository
Metric = found.Metric,
Dimension = found.Dimension,
Data = found.Data,
RecordTime = found.RecordTime
RecordTime = found.RecordTime,
StartTime = startTime,
EndTime = endTime,
Interval = interval.ToString()
};
}

public bool SaveGlobalStats(BotSharpStats body)
{
var time = BuildRecordTime(body.RecordTime);
var (startTime, endTime) = BuildTimeInterval(body.RecordTime, body.IntervalType);
body.RecordTime = DateTime.SpecifyKind(body.RecordTime, DateTimeKind.Utc);
body.StartTime = startTime;
body.EndTime = endTime;

var builder = Builders<GlobalStatisticsDocument>.Filter;
var filters = new List<FilterDefinition<GlobalStatisticsDocument>>()
{
builder.Eq(x => x.Metric, body.Metric),
builder.Eq(x => x.Dimension, body.Dimension),
builder.Eq(x => x.RecordTime, time)
builder.Eq(x => x.StartTime, startTime),
builder.Eq(x => x.EndTime, endTime)
};

var filterDef = builder.And(filters);
Expand All @@ -46,17 +56,46 @@ public bool SaveGlobalStats(BotSharpStats body)
.Set(x => x.Metric, body.Metric)
.Set(x => x.Dimension, body.Dimension)
.Set(x => x.Data, body.Data)
.Set(x => x.RecordTime, time);
.Set(x => x.StartTime, body.StartTime)
.Set(x => x.EndTime, body.EndTime)
.Set(x => x.Interval, body.Interval)
.Set(x => x.RecordTime, body.RecordTime);

_dc.GlobalStatistics.UpdateOne(filterDef, updateDef, _options);
return true;
}

#region Private methods
private DateTime BuildRecordTime(DateTime date)
private (DateTime, DateTime) BuildTimeInterval(DateTime recordTime, StatsInterval interval)
{
var recordDate = new DateTime(date.Year, date.Month, date.Day, date.Hour, 0, 0);
return DateTime.SpecifyKind(recordDate, DateTimeKind.Utc);
DateTime startTime = recordTime;
DateTime endTime = DateTime.UtcNow;

switch (interval)
{
case StatsInterval.Hour:
startTime = new DateTime(recordTime.Year, recordTime.Month, recordTime.Day, recordTime.Hour, 0, 0);
endTime = startTime.AddHours(1);
break;
case StatsInterval.Week:
var dayOfWeek = startTime.DayOfWeek;
var firstDayOfWeek = startTime.AddDays(-(int)dayOfWeek);
startTime = new DateTime(firstDayOfWeek.Year, firstDayOfWeek.Month, firstDayOfWeek.Day, 0, 0, 0);
endTime = startTime.AddDays(7);
break;
case StatsInterval.Month:
startTime = new DateTime(recordTime.Year, recordTime.Month, 1);
endTime = startTime.AddMonths(1);
break;
default:
startTime = new DateTime(recordTime.Year, recordTime.Month, recordTime.Day, 0, 0, 0);
endTime = startTime.AddDays(1);
break;
}

startTime = DateTime.SpecifyKind(startTime, DateTimeKind.Utc);
endTime = DateTime.SpecifyKind(endTime, DateTimeKind.Utc);
return (startTime, endTime);
}
#endregion
}