Skip to content

Commit 872b57f

Browse files
Oceania2018Oceania2018
authored
Merge pull request #1050 from iceljc/features/refine-global-stats
refine global stats
2 parents d57da2d + 4865e31 commit 872b57f

File tree

15 files changed

+246
-202
lines changed

15 files changed

+246
-202
lines changed

‎src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -187,9 +187,9 @@ List<string> GetInstructionLogSearchKeys(InstructLogKeysFilter filter)
187187
#endregion
188188

189189
#region Statistics
190-
BotSharpStats? GetGlobalStats(string metric, string dimension, string dimRefVal, DateTime recordTime, StatsInterval interval)
190+
BotSharpStats? GetGlobalStats(string agentId, DateTime recordTime, StatsInterval interval)
191191
=> throw new NotImplementedException();
192-
bool SaveGlobalStats(BotSharpStats body)
192+
bool SaveGlobalStats(BotSharpStatsDelta delta)
193193
=> throw new NotImplementedException();
194194

195195
#endregion

‎src/Infrastructure/BotSharp.Abstraction/Statistics/Models/BotSharpStats.cs

Lines changed: 30 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -4,17 +4,14 @@ namespace BotSharp.Abstraction.Statistics.Models;
44

55
public class BotSharpStats
66
{
7-
[JsonPropertyName("metric")]
8-
public string Metric { get; set; } = null!;
7+
[JsonPropertyName("agent_id")]
8+
public string AgentId { get; set; } = null!;
99

10-
[JsonPropertyName("dimension")]
11-
public string Dimension { get; set; } = null!;
10+
[JsonPropertyName("count")]
11+
public StatsCount Count { get; set; } = new();
1212

13-
[JsonPropertyName("dim_ref_val")]
14-
public string DimRefVal { get; set; } = null!;
15-
16-
[JsonPropertyName("data")]
17-
public IDictionary<string, double> Data { get; set; } = new Dictionary<string, double>();
13+
[JsonPropertyName("llm_cost")]
14+
public StatsLlmCost LlmCost { get; set; } = new();
1815

1916
[JsonPropertyName("record_time")]
2017
public DateTime RecordTime { get; set; } = DateTime.UtcNow;
@@ -44,11 +41,13 @@ public string Interval
4441
[JsonPropertyName("end_time")]
4542
public DateTime EndTime { get; set; }
4643

44+
4745
public override string ToString()
4846
{
49-
return $"{Metric}-{Dimension}-{DimRefVal} ({Interval}): {Data?.Count ?? 0}";
47+
return $"Stats: {AgentId}-{IntervalType}";
5048
}
5149

50+
5251
public static (DateTime, DateTime) BuildTimeInterval(DateTime recordTime, StatsInterval interval)
5352
{
5453
DateTime startTime = recordTime;
@@ -75,4 +74,25 @@ public static (DateTime, DateTime) BuildTimeInterval(DateTime recordTime, StatsI
7574
endTime = DateTime.SpecifyKind(endTime, DateTimeKind.Utc);
7675
return (startTime, endTime);
7776
}
77+
}
78+
79+
public class StatsCount
80+
{
81+
[JsonPropertyName("agent_call_count")]
82+
public long AgentCallCount { get; set; }
83+
}
84+
85+
public class StatsLlmCost
86+
{
87+
[JsonPropertyName("prompt_tokens")]
88+
public long PromptTokens { get; set; }
89+
90+
[JsonPropertyName("completion_tokens")]
91+
public long CompletionTokens { get; set; }
92+
93+
[JsonPropertyName("prompt_total_cost")]
94+
public float PromptTotalCost { get; set; }
95+
96+
[JsonPropertyName("completion_total_cost")]
97+
public float CompletionTotalCost { get; set; }
7898
}

‎src/Infrastructure/BotSharp.Abstraction/Statistics/Models/BotSharpStatsDelta.cs

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
using BotSharp.Abstraction.Statistics.Enums;
2+
3+
namespace BotSharp.Abstraction.Statistics.Models;
4+
5+
public class BotSharpStatsDelta
6+
{
7+
public string AgentId { get; set; } = null!;
8+
public StatsCountDelta CountDelta { get; set; } = new();
9+
public StatsLlmCostDelta LlmCostDelta { get; set; } = new();
10+
public DateTime RecordTime { get; set; } = DateTime.UtcNow;
11+
public StatsInterval IntervalType { get; set; } = StatsInterval.Day;
12+
13+
public string Interval
14+
{
15+
get
16+
{
17+
return IntervalType.ToString();
18+
}
19+
set
20+
{
21+
if (Enum.TryParse(value, out StatsInterval type))
22+
{
23+
IntervalType = type;
24+
}
25+
}
26+
}
27+
}
28+
29+
public class StatsCountDelta
30+
{
31+
public int AgentCallCountDelta { get; set; }
32+
}
33+
34+
public class StatsLlmCostDelta
35+
{
36+
public int PromptTokensDelta { get; set; }
37+
public int CompletionTokensDelta { get; set; }
38+
public float PromptTotalCostDelta { get; set; }
39+
public float CompletionTotalCostDelta { get; set; }
40+
}

‎src/Infrastructure/BotSharp.Abstraction/Statistics/Models/BotSharpStatsInput.cs

Lines changed: 0 additions & 13 deletions
This file was deleted.

‎src/Infrastructure/BotSharp.Abstraction/Statistics/Models/StatsKeyValuePair.cs

Lines changed: 0 additions & 27 deletions
This file was deleted.

‎src/Infrastructure/BotSharp.Abstraction/Statistics/Services/IBotSharpStatsService.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,5 +4,5 @@ namespace BotSharp.Abstraction.Statistics.Services;
44

55
public interface IBotSharpStatsService
66
{
7-
bool UpdateStats(string resourceKey, BotSharpStatsInput input);
7+
bool UpdateStats(string @event, BotSharpStatsDelta delta);
88
}

‎src/Infrastructure/BotSharp.Core/Conversations/Services/TokenStatistics.cs

Lines changed: 10 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -73,21 +73,20 @@ public void AddToken(TokenStatsModel stats, RoleDialogModel message)
7373
var dim = "agent";
7474
var agentId = message.CurrentAgentId ?? string.Empty;
7575
var globalStats = _services.GetRequiredService<IBotSharpStatsService>();
76-
var body = new BotSharpStatsInput
76+
var delta = new BotSharpStatsDelta
7777
{
78-
Metric = metric,
79-
Dimension = dim,
80-
DimRefVal = agentId,
78+
AgentId = agentId,
8179
RecordTime = DateTime.UtcNow,
8280
IntervalType = StatsInterval.Day,
83-
Data = [
84-
new StatsKeyValuePair("prompt_token_count_total", stats.TotalInputTokens),
85-
new StatsKeyValuePair("completion_token_count_total", stats.TotalOutputTokens),
86-
new StatsKeyValuePair("prompt_cost_total", deltaPromptCost),
87-
new StatsKeyValuePair("completion_cost_total", deltaCompletionCost)
88-
]
81+
LlmCostDelta = new()
82+
{
83+
PromptTokensDelta = stats.TotalInputTokens,
84+
CompletionTokensDelta = stats.TotalOutputTokens,
85+
PromptTotalCostDelta = deltaPromptCost,
86+
CompletionTotalCostDelta = deltaCompletionCost
87+
}
8988
};
90-
globalStats.UpdateStats($"global-{metric}-{dim}-{agentId}", body);
89+
globalStats.UpdateStats($"global-{metric}-{dim}-{agentId}", delta);
9190
}
9291

9392
public void PrintStatistics()

‎src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Stats.cs

Lines changed: 58 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -4,74 +4,106 @@ namespace BotSharp.Core.Repository;
44

55
public partial class FileRepository
66
{
7-
public BotSharpStats? GetGlobalStats(string metric, string dimension, string dimRefVal, DateTime recordTime, StatsInterval interval)
7+
public BotSharpStats? GetGlobalStats(string agentId, DateTime recordTime, StatsInterval interval)
88
{
9+
if (string.IsNullOrWhiteSpace(agentId))
10+
{
11+
return null;
12+
}
13+
914
var baseDir = Path.Combine(_dbSettings.FileRepository, STATS_FOLDER);
1015
var (startTime, endTime) = BotSharpStats.BuildTimeInterval(recordTime, interval);
11-
var dir = Path.Combine(baseDir, metric, startTime.Year.ToString(), startTime.Month.ToString("D2"));
12-
if (!Directory.Exists(dir)) return null;
16+
var dir = Path.Combine(baseDir, agentId, startTime.Year.ToString(), startTime.Month.ToString("D2"));
17+
if (!Directory.Exists(dir))
18+
{
19+
return null;
20+
}
1321

1422
var file = Directory.GetFiles(dir).FirstOrDefault(x => Path.GetFileName(x) == STATS_FILE);
15-
if (file == null) return null;
23+
if (file == null)
24+
{
25+
return null;
26+
}
1627

1728
var text = File.ReadAllText(file);
1829
var list = JsonSerializer.Deserialize<List<BotSharpStats>>(text, _options);
19-
var found = list?.FirstOrDefault(x => x.Metric.IsEqualTo(metric)
20-
&& x.Dimension.IsEqualTo(dimension)
21-
&& x.DimRefVal.IsEqualTo(dimRefVal)
30+
var found = list?.FirstOrDefault(x => x.AgentId.IsEqualTo(agentId)
2231
&& x.StartTime == startTime
2332
&& x.EndTime == endTime);
2433

2534
return found;
2635
}
2736

28-
public bool SaveGlobalStats(BotSharpStats body)
37+
public bool SaveGlobalStats(BotSharpStatsDelta delta)
2938
{
39+
if (delta == null || string.IsNullOrWhiteSpace(delta.AgentId))
40+
{
41+
return false;
42+
}
43+
3044
var baseDir = Path.Combine(_dbSettings.FileRepository, STATS_FOLDER);
31-
var (startTime, endTime) = BotSharpStats.BuildTimeInterval(body.RecordTime, body.IntervalType);
32-
body.StartTime = startTime;
33-
body.EndTime = endTime;
45+
var (startTime, endTime) = BotSharpStats.BuildTimeInterval(delta.RecordTime, delta.IntervalType);
3446

35-
var dir = Path.Combine(baseDir, body.Metric, startTime.Year.ToString(), startTime.Month.ToString("D2"));
47+
var dir = Path.Combine(baseDir, delta.AgentId, startTime.Year.ToString(), startTime.Month.ToString("D2"));
3648
if (!Directory.Exists(dir))
3749
{
3850
Directory.CreateDirectory(dir);
3951
}
4052

53+
var newItem = new BotSharpStats
54+
{
55+
AgentId = delta.AgentId,
56+
Count = new()
57+
{
58+
AgentCallCount = delta.CountDelta.AgentCallCountDelta
59+
},
60+
LlmCost = new()
61+
{
62+
PromptTokens = delta.LlmCostDelta.PromptTokensDelta,
63+
CompletionTokens = delta.LlmCostDelta.CompletionTokensDelta,
64+
PromptTotalCost = delta.LlmCostDelta.PromptTotalCostDelta,
65+
CompletionTotalCost = delta.LlmCostDelta.CompletionTotalCostDelta,
66+
},
67+
RecordTime = delta.RecordTime,
68+
StartTime = startTime,
69+
EndTime = endTime,
70+
Interval = delta.Interval
71+
};
72+
4173
var file = Path.Combine(dir, STATS_FILE);
4274
if (!File.Exists(file))
4375
{
44-
var list = new List<BotSharpStats> { body };
76+
var list = new List<BotSharpStats> { newItem };
4577
File.WriteAllText(file, JsonSerializer.Serialize(list, _options));
4678
}
4779
else
4880
{
4981
var text = File.ReadAllText(file);
5082
var list = JsonSerializer.Deserialize<List<BotSharpStats>>(text, _options);
51-
var found = list?.FirstOrDefault(x => x.Metric.IsEqualTo(body.Metric)
52-
&& x.Dimension.IsEqualTo(body.Dimension)
53-
&& x.DimRefVal.IsEqualTo(body.DimRefVal)
83+
var found = list?.FirstOrDefault(x => x.AgentId.IsEqualTo(delta.AgentId)
5484
&& x.StartTime == startTime
5585
&& x.EndTime == endTime);
5686

5787
if (found != null)
5888
{
59-
found.Metric = body.Metric;
60-
found.Dimension = body.Dimension;
61-
found.DimRefVal = body.DimRefVal;
62-
found.Data = body.Data;
63-
found.RecordTime = body.RecordTime;
64-
found.StartTime = body.StartTime;
65-
found.EndTime = body.EndTime;
66-
found.Interval = body.Interval;
89+
found.AgentId = delta.AgentId;
90+
found.RecordTime = delta.RecordTime;
91+
found.Count.AgentCallCount += delta.CountDelta.AgentCallCountDelta;
92+
found.LlmCost.PromptTokens += delta.LlmCostDelta.PromptTokensDelta;
93+
found.LlmCost.CompletionTokens += delta.LlmCostDelta.CompletionTokensDelta;
94+
found.LlmCost.PromptTotalCost += delta.LlmCostDelta.PromptTotalCostDelta;
95+
found.LlmCost.CompletionTotalCost += delta.LlmCostDelta.CompletionTotalCostDelta;
96+
found.StartTime = startTime;
97+
found.EndTime = endTime;
98+
found.Interval = delta.Interval;
6799
}
68100
else if (list != null)
69101
{
70-
list.Add(body);
102+
list.Add(newItem);
71103
}
72104
else if (list == null)
73105
{
74-
list = new List<BotSharpStats> { body };
106+
list = [newItem];
75107
}
76108

77109
File.WriteAllText(file, JsonSerializer.Serialize(list, _options));

0 commit comments

Comments
 (0)