Skip to content

Re-add routes to AI stats #643

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 5 commits into
base: main
Choose a base branch
from
Open

Re-add routes to AI stats #643

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
d592429
Re-add routes to AI stats
timokoessler Jun 17, 2025
32bc126
Extract getRoute to helper
timokoessler Jun 18, 2025
f00539d
Update library/sinks/OpenAI.ts
bitterpanda63 Jun 18, 2025
3b6d932
Update library/sinks/AwsSDKVersion3.ts
bitterpanda63 Jun 18, 2025
02693bf
Fix unit test
timokoessler Jun 18, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
261 changes: 261 additions & 0 deletions library/agent/AIStatistics.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ t.test("it tracks basic AI calls", async () => {
model: "gpt-4",
inputTokens: 100,
outputTokens: 50,
route: undefined,
});

const result = stats.getStats();
Expand All @@ -29,6 +30,7 @@ t.test("it tracks basic AI calls", async () => {
output: 50,
total: 150,
},
routes: [],
});

t.equal(stats.isEmpty(), false);
Expand All @@ -42,13 +44,15 @@ t.test("it tracks multiple calls to the same provider/model", async () => {
model: "gpt-4",
inputTokens: 100,
outputTokens: 50,
route: undefined,
});

stats.onAICall({
provider: "openai",
model: "gpt-4",
inputTokens: 200,
outputTokens: 75,
route: undefined,
});

const result = stats.getStats();
Expand All @@ -62,6 +66,7 @@ t.test("it tracks multiple calls to the same provider/model", async () => {
output: 125,
total: 425,
},
routes: [],
});
});

Expand All @@ -75,20 +80,23 @@ t.test(
model: "gpt-4",
inputTokens: 100,
outputTokens: 50,
route: undefined,
});

stats.onAICall({
provider: "openai",
model: "gpt-3.5-turbo",
inputTokens: 80,
outputTokens: 40,
route: undefined,
});

stats.onAICall({
provider: "anthropic",
model: "claude-3",
inputTokens: 120,
outputTokens: 60,
route: undefined,
});

const result = stats.getStats();
Expand All @@ -108,6 +116,7 @@ t.test(
output: 60,
total: 180,
},
routes: [],
});

t.same(result[1], {
Expand All @@ -119,6 +128,7 @@ t.test(
output: 40,
total: 120,
},
routes: [],
});

t.same(result[2], {
Expand All @@ -130,6 +140,7 @@ t.test(
output: 50,
total: 150,
},
routes: [],
});
}
);
Expand All @@ -142,13 +153,15 @@ t.test("it resets all statistics", async () => {
model: "gpt-4",
inputTokens: 100,
outputTokens: 50,
route: undefined,
});

stats.onAICall({
provider: "anthropic",
model: "claude-3",
inputTokens: 120,
outputTokens: 60,
route: undefined,
});

t.equal(stats.isEmpty(), false);
Expand All @@ -168,6 +181,7 @@ t.test("it handles zero token inputs", async () => {
model: "gpt-4",
inputTokens: 0,
outputTokens: 0,
route: undefined,
});

const result = stats.getStats();
Expand All @@ -187,6 +201,7 @@ t.test("called with empty provider", async () => {
model: "gpt-4",
inputTokens: 100,
outputTokens: 50,
route: undefined,
});

t.same(true, stats.isEmpty());
Expand All @@ -200,7 +215,253 @@ t.test("called with empty model", async () => {
model: "",
inputTokens: 100,
outputTokens: 50,
route: undefined,
});

t.same(true, stats.isEmpty());
});

t.test("it tracks route-specific statistics", async () => {
const stats = new AIStatistics();

stats.onAICall({
provider: "openai",
model: "gpt-4",
route: {
path: "/api/chat",
method: "POST",
},
inputTokens: 100,
outputTokens: 50,
});

const result = stats.getStats();
t.equal(result.length, 1);
t.same(result[0], {
provider: "openai",
model: "gpt-4",
calls: 1,
tokens: {
input: 100,
output: 50,
total: 150,
},
routes: [
{
path: "/api/chat",
method: "POST",
calls: 1,
tokens: {
input: 100,
output: 50,
total: 150,
},
},
],
});
});

t.test(
"it tracks multiple route calls for the same provider/model",
async () => {
const stats = new AIStatistics();

// First call to /api/chat
stats.onAICall({
provider: "openai",
model: "gpt-4",
route: {
path: "/api/chat",
method: "POST",
},
inputTokens: 100,
outputTokens: 50,
});

// Second call to /api/chat
stats.onAICall({
provider: "openai",
model: "gpt-4",
route: {
path: "/api/chat",
method: "POST",
},
inputTokens: 120,
outputTokens: 60,
});

// Call to different route
stats.onAICall({
provider: "openai",
model: "gpt-4",
route: {
path: "/api/summary",
method: "GET",
},
inputTokens: 80,
outputTokens: 40,
});

const result = stats.getStats();
t.equal(result.length, 1);
t.same(result[0].calls, 3);
t.same(result[0].tokens.total, 450);
t.same(result[0].routes.length, 2);

t.same(result[0].routes[0], {
path: "/api/chat",
method: "POST",
calls: 2,
tokens: {
input: 220,
output: 110,
total: 330,
},
});

t.same(result[0].routes[1], {
path: "/api/summary",
method: "GET",
calls: 1,
tokens: {
input: 80,
output: 40,
total: 120,
},
});
}
);

t.test("it mixes calls with and without routes", async () => {
const stats = new AIStatistics();

// Call without route
stats.onAICall({
provider: "openai",
model: "gpt-4",
inputTokens: 100,
outputTokens: 50,
route: undefined,
});

// Call with route
stats.onAICall({
provider: "openai",
model: "gpt-4",
route: {
path: "/api/chat",
method: "POST",
},
inputTokens: 120,
outputTokens: 60,
});

const result = stats.getStats();
t.same(result.length, 1);
t.same(result[0].calls, 2);
t.same(result[0].tokens.total, 330);
t.same(result[0].routes.length, 1);

t.same(result[0].routes[0], {
path: "/api/chat",
method: "POST",
calls: 1,
tokens: {
input: 120,
output: 60,
total: 180,
},
});
});

t.test("it respects LRU limit for routes", async () => {
const maxRoutes = 2;
const stats = new AIStatistics(maxRoutes);

// Add three different routes to exceed the limit
stats.onAICall({
provider: "openai",
model: "gpt-4",
route: {
path: "/api/route1",
method: "GET",
},
inputTokens: 100,
outputTokens: 50,
});

stats.onAICall({
provider: "openai",
model: "gpt-4",
route: {
path: "/api/route2",
method: "GET",
},
inputTokens: 100,
outputTokens: 50,
});

stats.onAICall({
provider: "openai",
model: "gpt-4",
route: {
path: "/api/route3",
method: "GET",
},
inputTokens: 100,
outputTokens: 50,
});

const result = stats.getStats();
t.equal(result.length, 1);
// All calls should be tracked in the provider stats
t.same(result[0].calls, 3);
t.same(result[0].tokens.total, 450);

// But only the most recent routes should be kept (LRU eviction)
t.same(result[0].routes.length, 2);

// The first route should have been evicted, keeping route2 and route3
const routePaths = result[0].routes.map((r) => r.path);
t.notOk(routePaths.includes("/api/route1"));
t.ok(routePaths.includes("/api/route2"));
t.ok(routePaths.includes("/api/route3"));
});

t.test("called with empty path", async () => {
const stats = new AIStatistics();

stats.onAICall({
provider: "openai",
model: "gpt-4",
route: {
path: "",
method: "POST",
},
inputTokens: 100,
outputTokens: 50,
});

const result = stats.getStats();
t.equal(result.length, 1);
t.same(result[0].routes.length, 0);
});

t.test("called with empty method", async () => {
const stats = new AIStatistics();

stats.onAICall({
provider: "openai",
model: "gpt-4",
route: {
path: "/api/chat",
method: "",
},
inputTokens: 100,
outputTokens: 50,
});

const result = stats.getStats();
t.equal(result.length, 1);
t.same(result[0].routes.length, 0);
});
Loading
Loading