From 4c84a14bc811c9dd91ed99d0ede64e3c5a8331e1 Mon Sep 17 00:00:00 2001 From: Matteo Morelli Date: Tue, 6 Oct 2026 09:57:52 +0200 Subject: [PATCH] Invocation log: count repeated failures on one row, short payloads past an hourly budget One workspace (1885Data) is 802,628 of the 859,784 rows in tool_invocations and about 1.8 GB of it: their backend calls at up to 197 a minute and their own gateway answers 40% of those calls with the same 429. - A failure that repeats within 60 s with the same connector, tool and error (numbers blanked) is counted on the row already stored, in a new repeat_count column, instead of being stored again. Counts are written when the window closes, every 30 s and on shutdown. Successes are always stored. - Past INVOCATION_FULL_PAYLOADS_PER_HOUR (default 1000) rows per organisation per hour, input and output keep a 512-byte excerpt; status, timing and error stay complete. - Stats, analytics, usage breakdowns and server usage sum repeat_count, so the numbers shown stay the number of calls. The migration adds a column with a constant default: catalog-only on the cloud's Postgres 17, no table rewrite. --- .../migration.sql | 7 + packages/backend/prisma/schema.prisma | 4 + .../src/audit/audit.breakdowns.spec.ts | 11 +- .../backend/src/audit/audit.service.spec.ts | 107 ++++++++-- packages/backend/src/audit/audit.service.ts | 197 ++++++++++++++---- .../src/mcp-servers/mcp-servers.service.ts | 4 +- 6 files changed, 272 insertions(+), 58 deletions(-) create mode 100644 packages/backend/prisma/migrations/20261006090000_tool_invocation_repeat_count/migration.sql diff --git a/packages/backend/prisma/migrations/20261006090000_tool_invocation_repeat_count/migration.sql b/packages/backend/prisma/migrations/20261006090000_tool_invocation_repeat_count/migration.sql new file mode 100644 index 00000000..a3407bbe --- /dev/null +++ b/packages/backend/prisma/migrations/20261006090000_tool_invocation_repeat_count/migration.sql @@ -0,0 +1,7 @@ +-- How many identical calls this row stands for. A failing call that repeats +-- within a minute (same connector, tool and error) is counted on the first +-- row instead of being stored again: one workspace's own rate limit put +-- 21,000 identical 429 rows a day into this table. Existing rows stand for one +-- call each. Adding a column with a constant default is a catalog-only change +-- in Postgres 11+, so this does not rewrite the table. +ALTER TABLE "tool_invocations" ADD COLUMN "repeat_count" INTEGER NOT NULL DEFAULT 1; diff --git a/packages/backend/prisma/schema.prisma b/packages/backend/prisma/schema.prisma index 81d21eca..5290f6be 100644 --- a/packages/backend/prisma/schema.prisma +++ b/packages/backend/prisma/schema.prisma @@ -711,6 +711,10 @@ model ToolInvocation { clientInfo String? @map("client_info") // JSON: {authMethod, apiKeyName, userEmail, mcpServerName} + // How many identical calls this row stands for: a failure repeating within + // a minute is counted here instead of stored again (see AuditService). + repeatCount Int @default(1) @map("repeat_count") + createdAt DateTime @default(now()) @map("created_at") @@index([toolId, createdAt]) diff --git a/packages/backend/src/audit/audit.breakdowns.spec.ts b/packages/backend/src/audit/audit.breakdowns.spec.ts index 2982f830..b0746806 100644 --- a/packages/backend/src/audit/audit.breakdowns.spec.ts +++ b/packages/backend/src/audit/audit.breakdowns.spec.ts @@ -20,13 +20,14 @@ describe('AuditService.getBreakdowns', () => { .mockResolvedValueOnce([{ userId: 'u1', _count: { _all: 10 } }]) // byUserErr .mockResolvedValueOnce([]); - const count = jest + // Calls are summed over repeat_count, so totals come from aggregate(). + const aggregate = jest .fn() - .mockResolvedValueOnce(10) // total - .mockResolvedValueOnce(3) // errors - .mockResolvedValueOnce(4); // proxyCalls + .mockResolvedValueOnce({ _sum: { repeatCount: 10 } }) // total + .mockResolvedValueOnce({ _sum: { repeatCount: 3 } }) // errors + .mockResolvedValueOnce({ _sum: { repeatCount: 4 } }); // proxyCalls return { - toolInvocation: { groupBy, count }, + toolInvocation: { groupBy, aggregate }, connector: { findMany: jest.fn().mockResolvedValue([{ id: 'c1', name: 'Koch ERP' }]) }, mcpServerConfig: { findMany: jest.fn().mockResolvedValue([{ id: 's1', name: 'KOCH Superpowers' }]) }, user: { findMany: jest.fn().mockResolvedValue([{ id: 'u1', email: 'a@b.c', name: 'Alice' }]) }, diff --git a/packages/backend/src/audit/audit.service.spec.ts b/packages/backend/src/audit/audit.service.spec.ts index 7b8cdd10..6d174150 100644 --- a/packages/backend/src/audit/audit.service.spec.ts +++ b/packages/backend/src/audit/audit.service.spec.ts @@ -10,6 +10,8 @@ describe('AuditService', () => { create: jest.fn().mockResolvedValue({}), findMany: jest.fn().mockResolvedValue([]), count: jest.fn().mockResolvedValue(0), + aggregate: jest.fn().mockResolvedValue({ _sum: { repeatCount: 0 } }), + update: jest.fn().mockResolvedValue({}), }, // resolveUserId consults users to satisfy the FK before insert. // Default: the test user exists. Individual tests override @@ -21,6 +23,10 @@ describe('AuditService', () => { service = new AuditService(mockPrisma); }); + afterEach(async () => { + await service.onModuleDestroy(); + }); + describe('logInvocation', () => { it('should persist an invocation record', async () => { await service.logInvocation({ @@ -32,14 +38,16 @@ describe('AuditService', () => { durationMs: 150, }); - expect(mockPrisma.toolInvocation.create).toHaveBeenCalledWith({ - data: expect.objectContaining({ - toolId: 'tool-1', - userId: 'user-1', - status: 'SUCCESS', - durationMs: 150, + expect(mockPrisma.toolInvocation.create).toHaveBeenCalledWith( + expect.objectContaining({ + data: expect.objectContaining({ + toolId: 'tool-1', + userId: 'user-1', + status: 'SUCCESS', + durationMs: 150, + }), }), - }); + ); }); it('should not throw if persistence fails', async () => { @@ -87,11 +95,11 @@ describe('AuditService', () => { describe('getStats', () => { it('should return aggregated stats', async () => { - mockPrisma.toolInvocation.count - .mockResolvedValueOnce(10) // 24h invocations - .mockResolvedValueOnce(2) // 24h errors - .mockResolvedValueOnce(50) // 7d invocations - .mockResolvedValueOnce(100); // total + mockPrisma.toolInvocation.aggregate + .mockResolvedValueOnce({ _sum: { repeatCount: 10 } }) // 24h invocations + .mockResolvedValueOnce({ _sum: { repeatCount: 2 } }) // 24h errors + .mockResolvedValueOnce({ _sum: { repeatCount: 50 } }) // 7d invocations + .mockResolvedValueOnce({ _sum: { repeatCount: 100 } }); // total const stats = await service.getStats(); @@ -101,7 +109,80 @@ describe('AuditService', () => { invocations7d: 50, totalInvocations: 100, }); - expect(mockPrisma.toolInvocation.count).toHaveBeenCalledTimes(4); + expect(mockPrisma.toolInvocation.aggregate).toHaveBeenCalledTimes(4); + }); + }); + + describe('repeated failures', () => { + const fail = (error = '429 Too Many Requests: body={"code":429,"msg":"Request rate limit reached"}') => + service.logInvocation({ + toolId: 'tool-1', + connectorId: 'conn-1', + organizationId: 'org-1', + input: { view: 'api.pgb_v1' }, + status: 'ERROR', + error, + }); + + beforeEach(() => { + let n = 0; + mockPrisma.toolInvocation.create.mockImplementation(async () => ({ id: `row-${++n}` })); + }); + + it('stores the first failure and counts identical ones on it', async () => { + await fail(); + await fail(); + await fail(); + expect(mockPrisma.toolInvocation.create).toHaveBeenCalledTimes(1); + + await service.flushRepeats(true); + expect(mockPrisma.toolInvocation.update).toHaveBeenCalledWith({ + where: { id: 'row-1' }, + data: { repeatCount: { increment: 2 } }, + }); + }); + + it('treats errors that differ only in numbers as the same', async () => { + await fail('upstream timeout after 3012 ms (request 8812)'); + await fail('upstream timeout after 2977 ms (request 9001)'); + expect(mockPrisma.toolInvocation.create).toHaveBeenCalledTimes(1); + }); + + it('stores a different error, or a success, as its own row', async () => { + await fail(); + await fail('403 Forbidden: Access denied'); + await service.logInvocation({ toolId: 'tool-1', connectorId: 'conn-1', input: {}, status: 'SUCCESS' }); + expect(mockPrisma.toolInvocation.create).toHaveBeenCalledTimes(3); + }); + + it('stores the failure again once the window has passed, carrying over the count', async () => { + const now = jest.spyOn(Date, 'now'); + now.mockReturnValue(1_000_000); + await fail(); + await fail(); + now.mockReturnValue(1_000_000 + 61_000); + await fail(); + expect(mockPrisma.toolInvocation.create).toHaveBeenCalledTimes(2); + expect(mockPrisma.toolInvocation.update).toHaveBeenCalledWith({ + where: { id: 'row-1' }, + data: { repeatCount: { increment: 1 } }, + }); + now.mockRestore(); + }); + }); + + describe('payload budget', () => { + it('keeps only a short excerpt once an organisation has logged its hourly budget', async () => { + const big = { rows: 'x'.repeat(20_000) }; + for (let i = 0; i < 1001; i++) { + await service.logInvocation({ toolId: 't', organizationId: 'busy-org', input: {}, output: big, status: 'SUCCESS' }); + } + const calls = mockPrisma.toolInvocation.create.mock.calls; + const size = (call: any) => JSON.stringify(call[0].data.output).length; + // Within the budget the normal excerpt rules apply; past it, 512 bytes. + expect(size(calls[999])).toBeGreaterThan(1_500); + expect(size(calls[1000])).toBeLessThanOrEqual(512); + expect(calls[1000][0].data.status).toBe('SUCCESS'); }); }); }); diff --git a/packages/backend/src/audit/audit.service.ts b/packages/backend/src/audit/audit.service.ts index 80ed7024..e7259fd8 100644 --- a/packages/backend/src/audit/audit.service.ts +++ b/packages/backend/src/audit/audit.service.ts @@ -1,4 +1,4 @@ -import { Injectable, Logger } from '@nestjs/common'; +import { Injectable, Logger, OnModuleDestroy } from '@nestjs/common'; import { boundPayload, boundText } from './bound-payload'; /** Bytes of a call's input / output kept in tool_invocations, and error chars. */ @@ -9,14 +9,43 @@ const envInt = (name: string, fallback: number) => { const INVOCATION_LOG_INPUT_BYTES = envInt('INVOCATION_LOG_INPUT_BYTES', 8 * 1024); const INVOCATION_LOG_OUTPUT_BYTES = envInt('INVOCATION_LOG_OUTPUT_BYTES', 16 * 1024); const INVOCATION_LOG_ERROR_CHARS = envInt('INVOCATION_LOG_ERROR_CHARS', 4000); +/** + * A failure that repeats with the same connector, tool and error inside this + * window is counted on the row already stored (repeat_count) instead of being + * stored again. 1885Data's own gateway answered 21,000 calls a day with the + * same 429; every one of them used to become a row. + */ +const INVOCATION_REPEAT_WINDOW_MS = envInt('INVOCATION_REPEAT_WINDOW_SECONDS', 60) * 1000; +/** + * Calls an organisation may log per hour with their full input/output + * excerpts. Past it, rows keep status, timing and error but only a short + * excerpt: a backend calling at 80 a minute filled 1.8 GB of payloads that + * nobody reads, while counts and errors are what the dashboards use. + */ +const INVOCATION_FULL_PAYLOADS_PER_HOUR = envInt('INVOCATION_FULL_PAYLOADS_PER_HOUR', 1000); +const INVOCATION_LOG_VOLUME_EXCERPT_BYTES = 512; +const REPEAT_KEYS_MAX = 10_000; import { PrismaService } from '../common/prisma.service'; -import { InvocationStatus } from '../generated/prisma/client'; +import { InvocationStatus, Prisma } from '../generated/prisma/client'; @Injectable() -export class AuditService { +export class AuditService implements OnModuleDestroy { private readonly logger = new Logger(AuditService.name); + /** Recent failures by signature: the row they were stored as, and how many repeats since. */ + private readonly repeats = new Map(); + /** Rows per organisation in the current hour, for the full-payload budget. */ + private volume = { hour: -1, perOrg: new Map() }; + private readonly flushTimer: NodeJS.Timeout; - constructor(private readonly prisma: PrismaService) {} + constructor(private readonly prisma: PrismaService) { + this.flushTimer = setInterval(() => void this.flushRepeats(false), 30_000); + this.flushTimer.unref?.(); + } + + async onModuleDestroy(): Promise { + clearInterval(this.flushTimer); + await this.flushRepeats(true); + } async logInvocation(data: { toolId: string; @@ -46,15 +75,24 @@ export class AuditService { error?: string; clientInfo?: string; }): Promise { + const repeatKey = data.status === 'SUCCESS' ? null : repeatSignature(data); + if (repeatKey && this.countRepeat(repeatKey)) return; + const resolvedUserId = await this.resolveUserId(data.userId, data.userEmail); // Store an excerpt, not the whole payload (see bound-payload.ts). The // caller already has the full response; this is only the log. - const input = boundPayload(data.input, { maxBytes: INVOCATION_LOG_INPUT_BYTES }); - const output = boundPayload(data.output, { maxBytes: INVOCATION_LOG_OUTPUT_BYTES }); + const overBudget = this.overPayloadBudget(data.organizationId); + const input = boundPayload(data.input, { + maxBytes: overBudget ? INVOCATION_LOG_VOLUME_EXCERPT_BYTES : INVOCATION_LOG_INPUT_BYTES, + }); + const output = boundPayload(data.output, { + maxBytes: overBudget ? INVOCATION_LOG_VOLUME_EXCERPT_BYTES : INVOCATION_LOG_OUTPUT_BYTES, + }); const error = boundText(data.error, INVOCATION_LOG_ERROR_CHARS); try { - await this.prisma.toolInvocation.create({ + const row = await this.prisma.toolInvocation.create({ + select: { id: true }, data: { toolId: data.toolId, userId: resolvedUserId, @@ -71,6 +109,7 @@ export class AuditService { clientInfo: data.clientInfo, }, }); + if (repeatKey && row?.id) await this.startRepeat(repeatKey, row.id); // Activation milestone: stamp the user's first successful call. The // conditional where makes this a no-op after the first success, so it // stays cheap on the hot path and never overwrites the original time. @@ -115,6 +154,70 @@ export class AuditService { ); } + /** + * True when this failure repeats one stored less than a window ago: it is + * then counted on that row and not stored again. + */ + private countRepeat(key: string): boolean { + const seen = this.repeats.get(key); + if (!seen || Date.now() - seen.since >= INVOCATION_REPEAT_WINDOW_MS) return false; + seen.extra += 1; + return true; + } + + /** Remember a stored failure as the row its repeats count on. */ + private async startRepeat(key: string, rowId: string): Promise { + const previous = this.repeats.get(key); + if (previous?.extra) await this.writeRepeatCount(previous.rowId, previous.extra); + this.repeats.delete(key); + this.repeats.set(key, { rowId, since: Date.now(), extra: 0 }); + if (this.repeats.size > REPEAT_KEYS_MAX) await this.flushRepeats(false, true); + } + + /** + * Write the counted repeats onto their rows. `all` flushes everything (on + * shutdown); `trim` also drops the oldest half when the map is full. + */ + async flushRepeats(all: boolean, trim = false): Promise { + const now = Date.now(); + const entries = [...this.repeats.entries()]; + const drop = trim ? new Set(entries.slice(0, Math.ceil(entries.length / 2)).map(([k]) => k)) : new Set(); + for (const [key, seen] of entries) { + const expired = now - seen.since >= INVOCATION_REPEAT_WINDOW_MS; + if (!all && !expired && !drop.has(key)) continue; + this.repeats.delete(key); + if (seen.extra) await this.writeRepeatCount(seen.rowId, seen.extra); + } + } + + private async writeRepeatCount(rowId: string, extra: number): Promise { + try { + await this.prisma.toolInvocation.update({ + where: { id: rowId }, + data: { repeatCount: { increment: extra } }, + }); + } catch (err: any) { + // The row may have been pruned meanwhile; a lost count is not worth more. + this.logger.debug(`Could not record ${extra} repeats on ${rowId}: ${err?.message ?? err}`); + } + } + + /** Calls matching `where`, counting the repeats a row stands for. */ + private async countCalls(where: Prisma.ToolInvocationWhereInput): Promise { + const agg = await this.prisma.toolInvocation.aggregate({ where, _sum: { repeatCount: true } }); + return agg._sum.repeatCount ?? 0; + } + + /** Whether this organisation has used its hourly budget of full payloads. */ + private overPayloadBudget(organizationId?: string): boolean { + if (!organizationId) return false; + const hour = Math.floor(Date.now() / 3_600_000); + if (this.volume.hour !== hour) this.volume = { hour, perOrg: new Map() }; + const n = (this.volume.perOrg.get(organizationId) ?? 0) + 1; + this.volume.perOrg.set(organizationId, n); + return n > INVOCATION_FULL_PAYLOADS_PER_HOUR; + } + /** * Return a `user_id` that is guaranteed to satisfy the FK constraint, * or `undefined`. Order of preference: @@ -231,16 +334,10 @@ export class AuditService { const scope = this.orgScope(organizationId); const [total24h, errors24h, total7d, totalAll] = await Promise.all([ - this.prisma.toolInvocation.count({ - where: { createdAt: { gte: last24h }, ...scope }, - }), - this.prisma.toolInvocation.count({ - where: { createdAt: { gte: last24h }, status: 'ERROR', ...scope }, - }), - this.prisma.toolInvocation.count({ - where: { createdAt: { gte: last7d }, ...scope }, - }), - this.prisma.toolInvocation.count({ where: scope }), + this.countCalls({ createdAt: { gte: last24h }, ...scope }), + this.countCalls({ createdAt: { gte: last24h }, status: 'ERROR', ...scope }), + this.countCalls({ createdAt: { gte: last7d }, ...scope }), + this.countCalls(scope), ]); return { @@ -268,6 +365,7 @@ export class AuditService { status: true, durationMs: true, createdAt: true, + repeatCount: true, tool: { select: { name: true } }, }, orderBy: { createdAt: 'asc' }, @@ -284,9 +382,11 @@ export class AuditService { dailyMap.set(dayKey, { success: 0, error: 0, timeout: 0, totalDuration: 0, count: 0 }); } const day = dailyMap.get(dayKey)!; - if (inv.status === 'SUCCESS') day.success++; - else if (inv.status === 'ERROR') day.error++; - else if (inv.status === 'TIMEOUT') day.timeout++; + // A row can stand for repeats of the same failure (repeat_count). + const n = inv.repeatCount ?? 1; + if (inv.status === 'SUCCESS') day.success += n; + else if (inv.status === 'ERROR') day.error += n; + else if (inv.status === 'TIMEOUT') day.timeout += n; day.totalDuration += inv.durationMs || 0; day.count++; @@ -296,8 +396,8 @@ export class AuditService { toolUsageMap.set(toolName, { count: 0, errors: 0, avgDuration: 0, totalDuration: 0 }); } const toolStats = toolUsageMap.get(toolName)!; - toolStats.count++; - if (inv.status === 'ERROR') toolStats.errors++; + toolStats.count += n; + if (inv.status === 'ERROR') toolStats.errors += n; toolStats.totalDuration += inv.durationMs || 0; } @@ -327,13 +427,15 @@ export class AuditService { .sort((a, b) => b.count - a.count) .slice(0, 10); + const totalCalls = invocations.reduce((sum, i) => sum + (i.repeatCount ?? 1), 0); + const successCalls = invocations + .filter((i) => i.status === 'SUCCESS') + .reduce((sum, i) => sum + (i.repeatCount ?? 1), 0); return { daily, topTools, - totalInvocations: invocations.length, - successRate: invocations.length > 0 - ? Math.round((invocations.filter(i => i.status === 'SUCCESS').length / invocations.length) * 100) - : 0, + totalInvocations: totalCalls, + successRate: totalCalls > 0 ? Math.round((successCalls / totalCalls) * 100) : 0, avgDuration: invocations.length > 0 ? Math.round(invocations.reduce((sum, i) => sum + (i.durationMs || 0), 0) / invocations.length) : 0, @@ -361,15 +463,15 @@ export class AuditService { byUser, byUserErr, total, errors, proxyCalls, ] = await Promise.all([ - this.prisma.toolInvocation.groupBy({ by: ['connectorId'], where, _count: { _all: true } }), - this.prisma.toolInvocation.groupBy({ by: ['connectorId'], where: errWhere, _count: { _all: true } }), - this.prisma.toolInvocation.groupBy({ by: ['mcpServerId'], where, _count: { _all: true } }), - this.prisma.toolInvocation.groupBy({ by: ['mcpServerId'], where: errWhere, _count: { _all: true } }), - this.prisma.toolInvocation.groupBy({ by: ['userId'], where, _count: { _all: true } }), - this.prisma.toolInvocation.groupBy({ by: ['userId'], where: errWhere, _count: { _all: true } }), - this.prisma.toolInvocation.count({ where }), - this.prisma.toolInvocation.count({ where: errWhere }), - this.prisma.toolInvocation.count({ where: { ...where, usedProxy: true } }), + this.prisma.toolInvocation.groupBy({ by: ['connectorId'], where, _count: { _all: true }, _sum: { repeatCount: true } }), + this.prisma.toolInvocation.groupBy({ by: ['connectorId'], where: errWhere, _count: { _all: true }, _sum: { repeatCount: true } }), + this.prisma.toolInvocation.groupBy({ by: ['mcpServerId'], where, _count: { _all: true }, _sum: { repeatCount: true } }), + this.prisma.toolInvocation.groupBy({ by: ['mcpServerId'], where: errWhere, _count: { _all: true }, _sum: { repeatCount: true } }), + this.prisma.toolInvocation.groupBy({ by: ['userId'], where, _count: { _all: true }, _sum: { repeatCount: true } }), + this.prisma.toolInvocation.groupBy({ by: ['userId'], where: errWhere, _count: { _all: true }, _sum: { repeatCount: true } }), + this.countCalls(where), + this.countCalls(errWhere), + this.countCalls({ ...where, usedProxy: true }), ]); // Resolve display names for the grouped ids (one query per dimension). @@ -392,19 +494,21 @@ export class AuditService { const userName = new Map(users.map((u) => [u.id, u.name || u.email])); const merge = ( - rows: Array<{ _count: { _all: number } } & Record>, - errRows: Array<{ _count: { _all: number } } & Record>, + rows: Array<{ _count: { _all: number }; _sum?: { repeatCount: number | null } } & Record>, + errRows: Array<{ _count: { _all: number }; _sum?: { repeatCount: number | null } } & Record>, key: string, label: (id: string | null) => string, ) => { - const errById = new Map(errRows.map((r) => [r[key] ?? '__null__', r._count._all])); + const calls = (r: { _count: { _all: number }; _sum?: { repeatCount: number | null } }) => + r._sum?.repeatCount ?? r._count._all; + const errById = new Map(errRows.map((r) => [r[key] ?? '__null__', calls(r)])); return rows .map((r) => { const id = r[key] as string | null; return { id, label: label(id), - count: r._count._all, + count: calls(r), errors: errById.get(id ?? '__null__') ?? 0, }; }) @@ -434,3 +538,18 @@ export class AuditService { }; } } + +/** + * What makes two failures "the same" for repeat counting: connector, tool, + * status and the error text with numbers blanked (ids, timestamps and counts + * differ between otherwise identical upstream errors). + */ +export function repeatSignature(data: { + connectorId?: string; + toolId: string; + status: string; + error?: string; +}): string { + const text = (data.error ?? '').slice(0, 300).replace(/\d+/g, '#'); + return `${data.connectorId ?? ''}|${data.toolId}|${data.status}|${text}`; +} diff --git a/packages/backend/src/mcp-servers/mcp-servers.service.ts b/packages/backend/src/mcp-servers/mcp-servers.service.ts index 733d4ba2..f1b187bc 100644 --- a/packages/backend/src/mcp-servers/mcp-servers.service.ts +++ b/packages/backend/src/mcp-servers/mcp-servers.service.ts @@ -59,12 +59,14 @@ export class McpServersService { by: ['mcpServerId'], where: { mcpServerId: { in: serverIds }, createdAt: { gte: since } }, _count: { _all: true }, + _sum: { repeatCount: true }, _max: { createdAt: true }, }); for (const row of rows) { if (!row.mcpServerId) continue; usage.set(row.mcpServerId, { - calls30d: row._count._all, + // A row can stand for repeats of the same failure (repeat_count). + calls30d: row._sum?.repeatCount ?? row._count._all, lastCallAt: row._max.createdAt ?? null, }); }