From 78bc559897232b8682020266afee2767a11171dd Mon Sep 17 00:00:00 2001 From: JeremyFunk Date: Fri, 11 Sep 2026 04:36:15 +0200 Subject: [PATCH 01/16] feat(agent-sessions): overview api Two internal reads behind the Agent Sessions overview page, both off `ai_trace_index` alone: `/overview/summary` returns the window's measures, the equal-length window before it, and both as a bucketed series; `/overview/breakdown` returns the busiest keys of one dimension, each measured over both windows. Every number reconciles with the sessions list over the same window, because it is derived the same way: the session key is resolved per trace with `sessionKey`, the filters are the list's per-trace existence tests, and usage runs through `usageReportersExpr` / `sessionUsageSum`, so a wrapper's roll-up, a gateway's mirror of a call and a provider retry each count once. A session is filed under the bucket its first span started in, so the series sums to the totals; the totals are their own un-bucketed read because quantiles do not merge. The breakdown keys a row by the value the span itself carries and nets inside that key, so a session that used two models is a session under each while its tokens are charged to the call that reported them. `model` reads model calls, `tool` reads tool calls, and a span that names no value keys under `''`. `sessionPricedLlmCalls` joins the shared span columns: coverage for a cost that only exists where the instrumentation reported one. --- .../routes/internal/ai-sessions.http.test.ts | 248 ++++ .../src/routes/internal/ai-sessions.http.ts | 195 +++ .../ai-overview.clickhouse.e2e.test.ts | 560 +++++++++ packages/domain/src/http/ai-sessions.ts | 217 ++++ packages/domain/src/http/query-engine.ts | 2 +- .../src/__sql_baseline__/integrations.sql | 1092 +++++++++++++++++ .../src/ai/ai-overview.test.ts | 214 ++++ .../src/ai/ai-overview.ts | 540 ++++++++ .../src/ai/ai-sessions.ts | 2 +- .../src/ai/ai-span-columns.ts | 21 + .../query-engine-integrations/src/ai/index.ts | 14 + .../src/benchmark/index.ts | 71 ++ 12 files changed, 3174 insertions(+), 2 deletions(-) create mode 100644 apps/api/src/services/warehouse/ai-overview.clickhouse.e2e.test.ts create mode 100644 packages/query-engine-integrations/src/ai/ai-overview.test.ts create mode 100644 packages/query-engine-integrations/src/ai/ai-overview.ts diff --git a/apps/api/src/routes/internal/ai-sessions.http.test.ts b/apps/api/src/routes/internal/ai-sessions.http.test.ts index 7b1c49baa..33a0f6cb8 100644 --- a/apps/api/src/routes/internal/ai-sessions.http.test.ts +++ b/apps/api/src/routes/internal/ai-sessions.http.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it } from "@effect/vitest" import { AiSessionsInternalApiGroup, + AI_OVERVIEW_BREAKDOWN_MAX, AI_SESSION_SPANS_MAX_SPANS, AI_SESSION_SUMMARY_MAX_TURNS, CurrentTenant, @@ -1094,3 +1095,250 @@ describe("POST /internal/ai-sessions/summary", () => { } }) }) + +/** + * The overview's two reads. What matters here is the composition the route + * does and not the SQL: the previous window is computed server-side, the + * summary is two reads folded into one response, and the breakdown pairs each + * key's two periods and zero-fills the one that has no row. + */ +const OVERVIEW_MEASURES = { + sessions: "4", + erroredSessions: "1", + llmCalls: 9, + erroredLlmCalls: "2", + toolCalls: "6", + erroredToolCalls: "1", + cost: 0.42, + pricedLlmCalls: 7, + tokens: 18_400, + inputTokens: 12_000, + cacheReadTokens: 4_000, + cacheWriteTokens: 0, + outputTokens: 2_000, + reasoningTokens: 400, + sessionDurationP50Ns: 600_000_000, + sessionDurationP95Ns: 900_000_000, + llmDurationP50Ns: 4_000_000, + llmDurationP95Ns: 8_000_000, +} + +/** One row of the totals or series union, in the wire shape it decodes from. */ +const overviewRow = (overrides: Record) => ({ ...OVERVIEW_MEASURES, ...overrides }) + +describe("POST /internal/ai-sessions/overview/summary", () => { + const SUMMARY_BODY = { ...WINDOW, bucketSeconds: 300 } + + const TOTALS = [ + overviewRow({ period: "current" }), + overviewRow({ period: "previous", sessions: "2", cost: 0.2 }), + ] + const SERIES = [ + overviewRow({ period: "current", bucket: "2026-08-19T09:00:00.000Z", sessions: "1" }), + overviewRow({ period: "current", bucket: "2026-08-19T10:00:00.000Z", sessions: "3" }), + overviewRow({ period: "previous", bucket: "2026-08-19T07:00:00.000Z", sessions: "2" }), + ] + + const summaryHarness = () => { + const contexts: Array = [] + const sqlByContext = new Map() + const harness = makeHarness({ + compiledQuery: (_tenant, compiled, options) => { + const context = options?.context ?? "" + contexts.push(context) + sqlByContext.set(context, compiledQueryOf(compiled).sql) + return compiledQueryOf(compiled) + .decodeRows(context === "aiOverviewTotals" ? TOTALS : SERIES) + .pipe(Effect.orDie) + }, + }) + return { harness, contexts, sqlByContext } + } + + it("answers from two index reads, the tiles beside the chart", async () => { + const { harness, contexts, sqlByContext } = summaryHarness() + + try { + const response = await harness.post("/internal/ai-sessions/overview/summary", SUMMARY_BODY) + expect(response.status).toBe(200) + // Two groupings of one population: quantiles do not merge, so the + // tiles cannot be folded from the chart. + expect([...contexts].sort()).toEqual(["aiOverviewSeries", "aiOverviewTotals"]) + for (const sql of sqlByContext.values()) { + expect(sql).toContain("FROM ai_trace_index") + expect(sql).not.toContain("trace_detail_spans") + expect(sql).not.toContain("__PARAM_") + } + // The chart's bucket reaches the read as the interval it was asked for. + expect(sqlByContext.get("aiOverviewSeries")).toContain("INTERVAL 300 SECOND") + expect(sqlByContext.get("aiOverviewTotals")).not.toContain("INTERVAL") + } finally { + await harness.dispose() + } + }) + + it("compares against the window of equal length ending where the caller's begins", async () => { + const { harness, sqlByContext } = summaryHarness() + + try { + const response = await harness.post("/internal/ai-sessions/overview/summary", SUMMARY_BODY) + expect(response.status).toBe(200) + // The caller asked for 09:00–11:00, so the comparison is 07:00–09:00 — + // computed here, never asked for, so a delta cannot be taken against a + // window of a different size. + const sql = sqlByContext.get("aiOverviewTotals") ?? "" + expect(sql).toContain("Timestamp >= '2026-08-19 07:00:00'") + expect(sql).toContain("Timestamp <= '2026-08-19 09:00:00'") + expect(sql).toContain(`Timestamp <= '${WINDOW.endTime}'`) + } finally { + await harness.dispose() + } + }) + + it("folds the two reads into the window, its comparison, and the two series", async () => { + const { harness } = summaryHarness() + + try { + const response = await harness.post("/internal/ai-sessions/overview/summary", SUMMARY_BODY) + expect(response.status).toBe(200) + expect(response.body).toMatchObject({ + bucketSeconds: 300, + current: { sessions: 4, erroredSessions: 1, llmCalls: 9, cost: 0.42, tokens: 18_400 }, + previous: { sessions: 2, cost: 0.2 }, + }) + const series = response.body.series as ReadonlyArray> + expect(series.map((point) => point.bucket)).toEqual([ + "2026-08-19T09:00:00.000Z", + "2026-08-19T10:00:00.000Z", + ]) + // A session is filed under the bucket it started in, so the buckets sum + // to the tile. + expect(series.reduce((total, point) => total + Number(point.sessions), 0)).toBe(4) + const previousSeries = response.body.previousSeries as ReadonlyArray> + expect(previousSeries.map((point) => point.bucket)).toEqual(["2026-08-19T07:00:00.000Z"]) + } finally { + await harness.dispose() + } + }) + + it("answers an empty window with zeros rather than a missing period", async () => { + const harness = makeHarness({ + compiledQuery: (_tenant, compiled) => compiledQueryOf(compiled).decodeRows([]).pipe(Effect.orDie), + }) + + try { + const response = await harness.post("/internal/ai-sessions/overview/summary", SUMMARY_BODY) + expect(response.status).toBe(200) + expect(response.body).toMatchObject({ + current: { sessions: 0, cost: 0, llmDurationP95Ns: 0 }, + previous: { sessions: 0 }, + series: [], + previousSeries: [], + }) + } finally { + await harness.dispose() + } + }) + + it("refuses a fractional bucket with a 400 rather than a 500", async () => { + const harness = makeHarness({ + compiledQuery: () => Effect.die("the read must never run"), + }) + + try { + const response = await harness.post("/internal/ai-sessions/overview/summary", { + ...WINDOW, + bucketSeconds: 1.5, + }) + // `param.int` rejects a fraction inside the builder, which would be a + // 500 — the contract catches it at the boundary instead. + expect(response.status).toBe(400) + } finally { + await harness.dispose() + } + }) +}) + +describe("POST /internal/ai-sessions/overview/breakdown", () => { + const BREAKDOWN_BODY = { ...WINDOW, dimension: "model" } + + const ROWS = [ + overviewRow({ period: "current", key: "gpt-5.5", keyCount: 0, sessions: "3", cost: 0.9 }), + overviewRow({ period: "current", key: "claude-sonnet-5", keyCount: 0, sessions: "7", cost: 0.3 }), + overviewRow({ period: "current", key: "", keyCount: 0, sessions: "3", cost: 0.1 }), + overviewRow({ period: "previous", key: "gpt-5.5", keyCount: 0, sessions: "2", cost: 0.5 }), + overviewRow({ period: "keys", key: "", keyCount: 9, sessions: "0" }), + ] + + const breakdownHarness = (rows: ReadonlyArray> = ROWS) => { + let sql: string | undefined + const contexts: Array = [] + const harness = makeHarness({ + compiledQuery: (_tenant, compiled, options) => { + contexts.push(options?.context) + sql = compiledQueryOf(compiled).sql + return compiledQueryOf(compiled).decodeRows(rows).pipe(Effect.orDie) + }, + }) + return { harness, contexts, readSql: () => sql ?? "" } + } + + it("reads both windows and the window's key count in one query", async () => { + const { harness, contexts, readSql } = breakdownHarness() + + try { + const response = await harness.post("/internal/ai-sessions/overview/breakdown", BREAKDOWN_BODY) + expect(response.status).toBe(200) + expect(contexts).toEqual(["aiOverviewBreakdown"]) + // The dimension picks the column, and a model is read over model calls. + expect(readSql()).toContain("toString(ai_trace_index.Model) AS key") + expect(readSql()).toContain("AND ai_trace_index.IsLlmCall = 1") + expect(response.body).toMatchObject({ dimension: "model", totalKeys: 9 }) + } finally { + await harness.dispose() + } + }) + + it("ranks the keys by sessions, pairs the two periods, and keeps the unattributed row", async () => { + const { harness } = breakdownHarness() + + try { + const response = await harness.post("/internal/ai-sessions/overview/breakdown", BREAKDOWN_BODY) + expect(response.status).toBe(200) + const rows = response.body.rows as ReadonlyArray> + // Busiest first, cost breaking the tie; `''` is a real key the page + // renders as unattributed rather than a gap. + expect(rows.map((row) => row.key)).toEqual(["claude-sonnet-5", "gpt-5.5", ""]) + expect(rows[1]).toMatchObject({ + key: "gpt-5.5", + current: { sessions: 3, cost: 0.9 }, + previous: { sessions: 2, cost: 0.5 }, + }) + // A key the previous window never saw reads as zeros, not as a missing + // row — the client shows it as new rather than as a -100%. + expect(rows[0]).toMatchObject({ previous: { sessions: 0, cost: 0 } }) + } finally { + await harness.dispose() + } + }) + + it("refuses a limit past the table's cap with a 400 rather than a 500", async () => { + const harness = makeHarness({ compiledQuery: () => Effect.die("the read must never run") }) + + try { + const tooMany = await harness.post("/internal/ai-sessions/overview/breakdown", { + ...BREAKDOWN_BODY, + limit: AI_OVERVIEW_BREAKDOWN_MAX + 1, + }) + expect(tooMany.status).toBe(400) + // And a dimension that is not a column of the index at all. + const unknown = await harness.post("/internal/ai-sessions/overview/breakdown", { + ...WINDOW, + dimension: "customer", + }) + expect(unknown.status).toBe(400) + } finally { + await harness.dispose() + } + }) +}) diff --git a/apps/api/src/routes/internal/ai-sessions.http.ts b/apps/api/src/routes/internal/ai-sessions.http.ts index 3c5856675..b4ed0f4e4 100644 --- a/apps/api/src/routes/internal/ai-sessions.http.ts +++ b/apps/api/src/routes/internal/ai-sessions.http.ts @@ -1,5 +1,7 @@ import { HttpApiBuilder } from "effect/unstable/httpapi" import { + AiOverviewBreakdownResponse, + AiOverviewSummaryResponse, AiSessionTooLargeError, AI_SESSION_SPANS_MAX_SPANS, AI_SESSION_SUMMARY_MAX_TURNS, @@ -11,6 +13,8 @@ import { ListAiSessionsResponse, MapleInternalApi, MAX_AI_SESSION_SPANS_RESPONSE_BYTES, + type AiOverviewBreakdownRow, + type AiOverviewMeasures, type AiSessionTokenReporting, type AiSessionTokenTotals, type AiSessionTurnSummary, @@ -400,9 +404,200 @@ export const HttpAiSessionsInternalLive = HttpApiBuilder.group( return summary }), ) + .handle("overviewSummary", ({ payload }) => + Effect.gen(function* () { + const tenant = yield* CurrentTenant.Context + yield* Effect.annotateCurrentSpan({ + orgId: tenant.orgId, + "maple.ai.overview.bucket_seconds": payload.bucketSeconds, + }) + // The comparison window is the caller's, shifted back by its own + // length: `previous` ends where `current` begins, so the two + // never overlap and the delta is over equal spans. + const params = { + orgId: tenant.orgId, + startTime: payload.startTime, + endTime: payload.endTime, + ...previousWindow(payload.startTime, payload.endTime), + } + const selection = overviewSelection(payload) + // Two reads and not one: the tiles' percentiles cannot be folded + // from the chart's, so the window has to be grouped twice — and + // side by side that costs one read's latency rather than two. + const [totals, series] = yield* Effect.all( + [ + warehouse.compiledQuery( + tenant, + CH.compileUnion(Integrations.aiOverviewTotalsQuery(selection), params), + { context: "aiOverviewTotals" }, + ), + warehouse.compiledQuery( + tenant, + CH.compileUnion(Integrations.aiOverviewSeriesQuery(selection), { + ...params, + bucketSeconds: payload.bucketSeconds, + }), + { context: "aiOverviewSeries" }, + ), + ], + { concurrency: 2 }, + ) + const points = (period: Integrations.AiOverviewPeriod) => + series + .filter((row) => row.period === period) + .map((row) => ({ bucket: row.bucket, ...overviewMeasures(row) })) + return new AiOverviewSummaryResponse({ + bucketSeconds: payload.bucketSeconds, + // An aggregate over no rows still yields one row per branch, so + // a missing period is a shape failure rather than an empty + // window — the zeros are what a client renders as "no + // comparison". + current: overviewMeasures(totals.find((row) => row.period === "current")), + previous: overviewMeasures(totals.find((row) => row.period === "previous")), + series: points("current"), + previousSeries: points("previous"), + }) + }), + ) + .handle("overviewBreakdown", ({ payload }) => + Effect.gen(function* () { + const tenant = yield* CurrentTenant.Context + yield* Effect.annotateCurrentSpan({ + orgId: tenant.orgId, + "maple.ai.overview.dimension": payload.dimension, + }) + const rows = yield* warehouse.compiledQuery( + tenant, + CH.compileUnion( + Integrations.aiOverviewBreakdownQuery({ + ...overviewSelection(payload), + dimension: payload.dimension, + limit: payload.limit, + }), + { + orgId: tenant.orgId, + startTime: payload.startTime, + endTime: payload.endTime, + ...previousWindow(payload.startTime, payload.endTime), + }, + ), + { context: "aiOverviewBreakdown" }, + ) + const previous = new Map( + rows.filter((row) => row.period === "previous").map((row) => [row.key, row]), + ) + // The busiest first, and the ranking the query made is by sessions + // alone — cost orders the keys it tied. + const ranked = rows + .filter((row) => row.period === "current") + .sort((a, b) => b.sessions - a.sessions || b.cost - a.cost || a.key.localeCompare(b.key)) + const breakdown: ReadonlyArray = ranked.map((row) => ({ + key: row.key, + current: overviewMeasures(row), + // Zeros for a key that did not appear before, which reads as + // "new" rather than as a missing row. + previous: overviewMeasures(previous.get(row.key)), + })) + return new AiOverviewBreakdownResponse({ + dimension: payload.dimension, + rows: breakdown, + totalKeys: rows.find((row) => row.period === "keys")?.keyCount ?? 0, + }) + }), + ) }), ) +/** + * The overview's selection, as both of its reads take it — the sessions list's + * counted filters, so the two pages measure the same sessions. + */ +const overviewSelection = (payload: { + readonly vendorIds?: ReadonlyArray + readonly serviceNames?: ReadonlyArray + readonly deploymentEnvs?: ReadonlyArray + readonly models?: ReadonlyArray + readonly agentNames?: ReadonlyArray + readonly toolNames?: ReadonlyArray + readonly hasErrors?: boolean +}) => ({ + vendorIds: payload.vendorIds, + serviceNames: payload.serviceNames, + deploymentEnvs: payload.deploymentEnvs, + models: payload.models, + agentNames: payload.agentNames, + toolNames: payload.toolNames, + hasErrors: payload.hasErrors, +}) + +/** `TinybirdDateTime` is UTC without a zone marker. */ +const warehouseDateTimeMs = (value: string): number => Date.parse(`${value.replace(" ", "T")}Z`) + +/** Back to warehouse shape, seconds precision — what the params take. */ +const warehouseDateTime = (ms: number): string => new Date(ms).toISOString().replace("T", " ").slice(0, 19) + +/** + * The window of equal length ending where the caller's begins — the tiles' + * comparison. Computed here rather than asked for, so the delta cannot be + * quietly taken against a window of a different size. + */ +const previousWindow = (startTime: string, endTime: string) => { + const start = warehouseDateTimeMs(startTime) + const span = warehouseDateTimeMs(endTime) - start + return { + prevStartTime: warehouseDateTime(start - span), + prevEndTime: warehouseDateTime(start), + } +} + +const NO_OVERVIEW_MEASURES: AiOverviewMeasures = { + sessions: 0, + erroredSessions: 0, + llmCalls: 0, + erroredLlmCalls: 0, + toolCalls: 0, + erroredToolCalls: 0, + cost: 0, + pricedLlmCalls: 0, + tokens: 0, + inputTokens: 0, + cacheReadTokens: 0, + cacheWriteTokens: 0, + outputTokens: 0, + reasoningTokens: 0, + sessionDurationP50Ns: 0, + sessionDurationP95Ns: 0, + llmDurationP50Ns: 0, + llmDurationP95Ns: 0, +} + +/** One row's measures, or zeros for a period or a key that has no row. */ +const overviewMeasures = ( + row: Integrations.AiOverviewMeasuresOutput | undefined, +): AiOverviewMeasures => { + if (row === undefined) return NO_OVERVIEW_MEASURES + return { + sessions: row.sessions, + erroredSessions: row.erroredSessions, + llmCalls: row.llmCalls, + erroredLlmCalls: row.erroredLlmCalls, + toolCalls: row.toolCalls, + erroredToolCalls: row.erroredToolCalls, + cost: row.cost, + pricedLlmCalls: row.pricedLlmCalls, + tokens: row.tokens, + inputTokens: row.inputTokens, + cacheReadTokens: row.cacheReadTokens, + cacheWriteTokens: row.cacheWriteTokens, + outputTokens: row.outputTokens, + reasoningTokens: row.reasoningTokens, + sessionDurationP50Ns: row.sessionDurationP50Ns, + sessionDurationP95Ns: row.sessionDurationP95Ns, + llmDurationP50Ns: row.llmDurationP50Ns, + llmDurationP95Ns: row.llmDurationP95Ns, + } +} + const NO_TOKENS: AiSessionTokenTotals = { input: 0, output: 0, cacheRead: 0 } const emptySummary = () => diff --git a/apps/api/src/services/warehouse/ai-overview.clickhouse.e2e.test.ts b/apps/api/src/services/warehouse/ai-overview.clickhouse.e2e.test.ts new file mode 100644 index 000000000..b86d663ce --- /dev/null +++ b/apps/api/src/services/warehouse/ai-overview.clickhouse.e2e.test.ts @@ -0,0 +1,560 @@ +// SAFETY-FILE: JSON in this test is emitted by the fixture or unit under test before its fields are asserted. +// Agent Sessions › Overview, against real rows. +// +// The overview's whole contract is that its numbers reconcile with the +// sessions LIST over the same window, and nothing about that can be proved +// from SQL text: +// +// - the SESSION a row belongs to is `max(SessionId)` per TRACE. Keyed per +// row instead, every span that carries no session id becomes its own +// session and every count is wrong by an order of magnitude behind a +// healthy 200. +// - USAGE is netted. A wrapper that rolls up its children's tokens, a +// gateway that files a second trace of the same call under the same +// session, and a provider retry beneath the call each have to count once, +// and each of those three is an array pass over real rows. +// - the buckets have to SUM to the totals, which is a statement about where +// a session that ran across a bucket boundary lands. +// +// So this suite seeds spans into `traces`, lets the real migration chain's +// `ai_trace_index_mv` materialize them, and runs the real compiled builders +// over the result — beside `aiSessionPageQuery` over the same window, which is +// what the numbers are checked against. It reads with +// `use_variant_as_common_type = 0`, the setting managed Tinybird runs, so a +// `UNION ALL` branch whose types only agree on a modern analyzer fails here. + +import { afterAll, assert, beforeAll, describe, it } from "@effect/vitest" +import { Effect } from "effect" +import { compileUnionUnsafe, compileUnsafe } from "@maple-dev/effect-clickhouse" +import { MAPLE_AI_SESSION_ID_ATTR, MAPLE_AI_VENDOR_ID_ATTR } from "@maple/domain/gen-ai" +import * as Integrations from "@maple/query-engine-integrations" +import { normalizeSqlForClickHouseClient } from "@maple/query-engine/execution" +import { + ANALYZER_STRICTNESS, + applyRealMigrations, + clickhouseE2eEnabled, + clickhouseExec, + uniqueDatabase, +} from "./clickhouse-e2e-support" + +const database = uniqueDatabase("maple_ai_overview_e2e") +const ORG_ID = "org_ai_overview_e2e" +const FOREIGN_ORG_ID = "org_ai_overview_e2e_other" + +// Anchored to now: `traces` and `ai_trace_index` both enforce a 30-day TTL at +// insert, so a hardcoded date would one day drop every seed and leave the suite +// comparing nothing to nothing. +const HOUR_MS = 3_600_000 +const BASE_MS = Math.floor((Date.now() - 2 * HOUR_MS) / 1000) * 1000 + +/** Half an hour on — far enough to land in its own five-minute bucket. */ +const LATER_MS = BASE_MS + 1_800_000 + +/** Inside the comparison window, which ends where the caller's begins. */ +const EARLIER_MS = BASE_MS - 2 * HOUR_MS + +const SESSION_ID = `${ORG_ID}:overview-1` +/** The ordinary shape: a turn span that rolls up its two model calls, and a + * tool call that failed. */ +const TRACE_TURN = "aioverve2e00000000000000000000001" +/** The gateway's own trace of the first model call — same session, same + * response id, a price the app's SDK did not have. */ +const TRACE_MIRROR = "aioverve2e00000000000000000000002" +/** No session id anywhere, so the trace IS the session. Its model call failed. */ +const TRACE_SESSIONLESS = "aioverve2e00000000000000000000003" +/** A row written before the token buckets existed — inserted into the index + * directly, because the materialized view derives the buckets from the same + * attributes as the total and cannot produce one. */ +const TRACE_PRE_BUCKETS = "aioverve2e00000000000000000000004" +/** The comparison window's only session. */ +const TRACE_EARLIER = "aioverve2e00000000000000000000005" +const TRACE_FOREIGN = "aioverve2e00000000000000000000006" + +const GPT = "gpt-5" +const CLAUDE = "claude-sonnet-5" +/** The response id the app's SDK and the gateway both report for one call. */ +const SHARED_RESPONSE_ID = "resp-shared-1" + +interface SeedSpan { + readonly traceId: string + readonly spanId: string + readonly parentSpanId?: string + readonly name: string + readonly ms: number + readonly durationNs: number + readonly status: string + readonly attrs: Readonly> +} + +const agentSpan = (attrs: Readonly>) => ({ + [MAPLE_AI_VENDOR_ID_ATTR]: "eve", + ...attrs, +}) + +/** Tokens and a price, under the canonical semconv keys. */ +const usage = (input: number, output: number, cost: number, responseId?: string) => ({ + "gen_ai.usage.input_tokens": String(input), + "gen_ai.usage.output_tokens": String(output), + "gen_ai.usage.cost": String(cost), + ...(responseId === undefined ? {} : { "gen_ai.response.id": responseId }), +}) + +const SEED_SPANS: ReadonlyArray = [ + // The turn span carries the session id AND its children's usage summed onto + // it — the roll-up the netting has to cancel. + { + traceId: TRACE_TURN, + spanId: "overview-turn-1", + name: "invoke_agent slack-agent", + ms: BASE_MS, + durationNs: 10_000_000, + status: "Ok", + attrs: agentSpan({ + [MAPLE_AI_SESSION_ID_ATTR]: SESSION_ID, + "gen_ai.operation.name": "invoke_agent", + "gen_ai.agent.name": "slack-agent", + ...usage(120, 60, 0.03), + }), + }, + { + traceId: TRACE_TURN, + spanId: "overview-chat-gpt", + parentSpanId: "overview-turn-1", + name: "chat gpt-5", + ms: BASE_MS + 1, + durationNs: 4_000_000, + status: "Ok", + attrs: agentSpan({ + "gen_ai.operation.name": "chat", + "gen_ai.response.model": GPT, + ...usage(100, 50, 0.02, SHARED_RESPONSE_ID), + }), + }, + // A second model, in the same session — what makes the breakdown's rows + // overlap. + { + traceId: TRACE_TURN, + spanId: "overview-chat-claude", + parentSpanId: "overview-turn-1", + name: "chat claude-sonnet-5", + ms: BASE_MS + 4, + durationNs: 3_000_000, + status: "Ok", + attrs: agentSpan({ + "gen_ai.operation.name": "chat", + "gen_ai.response.model": CLAUDE, + ...usage(20, 10, 0.01, "resp-claude-1"), + }), + }, + { + traceId: TRACE_TURN, + spanId: "overview-tool-1", + parentSpanId: "overview-chat-gpt", + name: "execute_tool search_traces", + ms: BASE_MS + 2, + durationNs: 1_000_000, + status: "Error", + attrs: agentSpan({ + "gen_ai.operation.name": "execute_tool", + "gen_ai.tool.name": "search_traces", + }), + }, + // The gateway's mirror: its own trace of the GPT call, under the same + // session id and the same response id, priced higher. + { + traceId: TRACE_MIRROR, + spanId: "overview-chat-mirror", + name: "chat gpt-5", + ms: BASE_MS + 3, + durationNs: 5_000_000, + status: "Ok", + attrs: agentSpan({ + [MAPLE_AI_SESSION_ID_ATTR]: SESSION_ID, + "gen_ai.operation.name": "chat", + "gen_ai.response.model": GPT, + ...usage(100, 50, 0.05, SHARED_RESPONSE_ID), + }), + }, + // A sessionless trace, half an hour later, whose model call failed. + { + traceId: TRACE_SESSIONLESS, + spanId: "overview-chat-late", + name: "chat claude-sonnet-5", + ms: LATER_MS, + durationNs: 6_000_000, + status: "Error", + attrs: agentSpan({ + "gen_ai.operation.name": "chat", + "gen_ai.response.model": CLAUDE, + ...usage(200, 100, 0.1, "resp-late-1"), + }), + }, + // The comparison window's only session. + { + traceId: TRACE_EARLIER, + spanId: "overview-chat-earlier", + name: "chat gpt-5", + ms: EARLIER_MS, + durationNs: 2_000_000, + status: "Ok", + attrs: agentSpan({ + "gen_ai.operation.name": "chat", + "gen_ai.response.model": GPT, + ...usage(10, 5, 0.01, "resp-earlier-1"), + }), + }, +] + +/** Another org's session, in the same window — the reads must never see it. */ +const FOREIGN_SPAN: SeedSpan = { + traceId: TRACE_FOREIGN, + spanId: "overview-chat-foreign", + name: "chat gpt-5", + ms: BASE_MS + 5, + durationNs: 9_000_000, + status: "Ok", + attrs: agentSpan({ + "gen_ai.operation.name": "chat", + "gen_ai.response.model": GPT, + ...usage(999, 999, 9.99, "resp-foreign-1"), + }), +} + +const quote = (value: string): string => `'${value.replaceAll("'", "\\'")}'` +const chDateTime = (epochMs: number): string => new Date(epochMs).toISOString().replace("T", " ").slice(0, 23) +const chMap = (attrs: Readonly>): string => + `map(${Object.entries(attrs) + .flatMap(([key, value]) => [quote(key), quote(value)]) + .join(", ")})` + +const seed = async (): Promise => { + const rows = [ + ...SEED_SPANS.map((span) => [ORG_ID, span] as const), + [FOREIGN_ORG_ID, FOREIGN_SPAN] as const, + ] + .map( + ([orgId, span]) => + `(${quote(orgId)}, ${quote(chDateTime(span.ms))}, ${quote(span.traceId)}, ${quote(span.spanId)}, ${quote(span.parentSpanId ?? "")}, ${quote(span.name)}, 'Internal', 'agent-service', ${span.durationNs}, ${quote(span.status)}, 1, ${chMap(span.attrs)}, ${chMap({ "deployment.environment.name": "production" })})`, + ) + .join("\n,") + + await clickhouseExec( + `INSERT INTO traces + (OrgId, Timestamp, TraceId, SpanId, ParentSpanId, SpanName, SpanKind, ServiceName, Duration, StatusCode, SampleRate, SpanAttributes, ResourceAttributes) + VALUES\n${rows}`, + database, + ) + + // The pre-0031 row, written straight into the index: a total with five + // empty buckets, which is what every row materialized before the bucket + // columns existed still looks like. The overview must report the total and + // leave the buckets summing to nothing rather than inventing a split. + await clickhouseExec( + `INSERT INTO ai_trace_index + (OrgId, Timestamp, TraceId, SessionId, VendorId, ServiceName, DeploymentEnv, Model, AgentName, ToolName, + SpanId, ParentSpanId, Duration, IsError, IsLlmCall, IsToolCall, Tokens, Cost, ResponseId, VendorVersion, + InputTokens, CacheReadTokens, CacheWriteTokens, OutputTokens, ReasoningTokens) + VALUES (${quote(ORG_ID)}, ${quote(chDateTime(BASE_MS + 6))}, ${quote(TRACE_PRE_BUCKETS)}, '', 'eve', + 'agent-service', 'production', '', '', '', 'overview-chat-legacy', '', 2000000, 0, 1, 0, 500, 0, '', '', + 0, 0, 0, 0, 0)`, + database, + ) +} + +const runJson = async (sql: string): Promise>> => { + const body = await clickhouseExec(normalizeSqlForClickHouseClient(sql), database, { + default_format: "JSON", + output_format_json_quote_64bit_integers: "0", + ...ANALYZER_STRICTNESS, + }) + const parsed = JSON.parse(body) as { readonly data?: ReadonlyArray> } + return parsed.data ?? [] +} + +const window = { + orgId: ORG_ID, + startTime: chDateTime(BASE_MS - HOUR_MS), + endTime: chDateTime(BASE_MS + HOUR_MS), +} + +/** The comparison window the route computes: equal length, ending where the + * caller's begins. `TRACE_EARLIER` is the only session inside it. */ +const compareWindow = { + ...window, + prevStartTime: chDateTime(BASE_MS - 3 * HOUR_MS), + prevEndTime: chDateTime(BASE_MS - HOUR_MS), +} + +const totals = async (opts: Integrations.AiOverviewFilterOpts = {}) => { + const compiled = compileUnionUnsafe(Integrations.aiOverviewTotalsQuery(opts), compareWindow) + const rows = Effect.runSync(compiled.decodeRows(await runJson(compiled.sql))) + return { + current: rows.find((row) => row.period === "current"), + previous: rows.find((row) => row.period === "previous"), + } +} + +/** The sessions list over the same window — what every number here is checked + * against. */ +const listRows = async (opts: Integrations.AiSessionPageOpts = {}) => { + const compiled = compileUnsafe(Integrations.aiSessionPageQuery(opts), window) + return Effect.runSync(compiled.decodeRows(await runJson(compiled.sql))) +} + +const sumOf = (rows: ReadonlyArray, read: (row: T) => number) => + rows.reduce((total, row) => total + read(row), 0) + +describe.skipIf(!clickhouseE2eEnabled)("agent overview reads", () => { + beforeAll(async () => { + await clickhouseExec(`CREATE DATABASE ${database}`) + await applyRealMigrations(database) + await seed() + }, 180_000) + + afterAll(async () => { + await clickhouseExec(`DROP DATABASE IF EXISTS ${database}`) + }, 30_000) + + it("counts the sessions the list would have listed, and nobody else's", async () => { + const { current } = await totals() + const list = await listRows() + + // Three sessions: the vendor's own (two traces, the turn's and the + // gateway's mirror of it), the sessionless trace, and the pre-0031 row. + // The foreign org's session is in neither. + assert.strictEqual(current?.sessions, list.length) + assert.strictEqual(current?.sessions, 3) + assert.deepStrictEqual( + [...list].map((row) => row.sessionId).sort(), + [SESSION_ID, `trace:${TRACE_PRE_BUCKETS}`, `trace:${TRACE_SESSIONLESS}`].sort(), + ) + }) + + it("nets usage exactly as the list nets it — the roll-up, the mirror and the retry", async () => { + const { current } = await totals() + const list = await listRows() + + // The turn span reports its children's tokens as its own and the gateway + // reports the GPT call a second time: 180 raw tokens on the turn, 180 on + // its two model calls and 150 on the mirror, which is 510 summed and 180 + // netted — 150 for the GPT call (once, priced at the higher of the two + // claims) and 30 for the Claude call. + assert.strictEqual(current?.tokens, sumOf(list, (row) => row.totalTokens)) + assert.strictEqual(current?.tokens, 180 + 300 + 500) + assert.closeTo( + current!.cost, + sumOf(list, (row) => row.cost), + 1e-9, + ) + // 0.05 for the mirrored call (the gateway's price, not the SDK's), 0.01 + // for the Claude call, 0.10 for the late one; the roll-up adds nothing. + assert.closeTo(current!.cost, 0.16, 1e-9) + assert.strictEqual(current?.llmCalls, sumOf(list, (row) => row.llmCalls)) + assert.strictEqual(current?.llmCalls, 4) + // Coverage, not a price: every call but the pre-0031 one carried one. + assert.strictEqual(current?.pricedLlmCalls, 3) + assert.strictEqual(current?.toolCalls, sumOf(list, (row) => row.toolCalls)) + assert.strictEqual(current?.toolCalls, 1) + }) + + it("leaves a pre-bucket row's total whole and its buckets empty", async () => { + const { current } = await totals() + + // The five buckets are the disjoint split of the total for every row + // materialized since migration 0031, and zeros for the rows before it. + // Summing them to the total in SQL would invent a split the row never + // reported; the client falls back to the total instead. + assert.strictEqual(current?.inputTokens, 120 + 200) + assert.strictEqual(current?.outputTokens, 60 + 100) + assert.strictEqual(current?.cacheReadTokens, 0) + assert.strictEqual(current?.cacheWriteTokens, 0) + assert.strictEqual(current?.reasoningTokens, 0) + const buckets = + current!.inputTokens + + current!.cacheReadTokens + + current!.cacheWriteTokens + + current!.outputTokens + + current!.reasoningTokens + assert.strictEqual(buckets, 480) + assert.isBelow(buckets, current!.tokens) + }) + + it("counts failures against the population each belongs to", async () => { + const { current } = await totals() + const list = await listRows() + + // Two of the three sessions carry a failed agent span — the same two the + // list's own `hasErrors` matches. + assert.strictEqual(current?.erroredSessions, list.filter((row) => row.errorAgentSpans > 0).length) + assert.strictEqual(current?.erroredSessions, 2) + // One failed tool call, out of one; one failed model call, out of four. + assert.strictEqual(current?.erroredToolCalls, 1) + assert.strictEqual(current?.erroredLlmCalls, 1) + + // And the filter selects exactly those sessions. + const failing = await totals({ hasErrors: true }) + assert.strictEqual(failing.current?.sessions, 2) + assert.strictEqual(failing.current?.erroredSessions, 2) + }) + + it("measures the session's extent and the model call's own duration", async () => { + const { current } = await totals() + const list = await listRows() + + // The session's extent is its first span to its last span's END: 10ms for + // the turn's session (its own span outlives every call beneath it), 6ms + // and 2ms for the other two — the same three the list reports. + const extents = [...list].map((row) => row.agentDurationMs).sort((a, b) => a - b) + assert.deepStrictEqual(extents, [2, 6, 10]) + assert.strictEqual(current?.sessionDurationP50Ns, extents[1]! * 1_000_000) + assert.isAbove(current!.sessionDurationP95Ns, extents[1]! * 1_000_000) + assert.isAtMost(current!.sessionDurationP95Ns, extents[2]! * 1_000_000) + // The model calls took 2, 3, 4, 5 and 6ms: a span-level quantile, taken + // at a level whose rows are sessions. + assert.strictEqual(current?.llmDurationP50Ns, 4_000_000) + assert.isAbove(current!.llmDurationP95Ns, 5_000_000) + assert.isAtMost(current!.llmDurationP95Ns, 6_000_000) + }) + + it("measures the window before the caller's in the same read", async () => { + const { previous } = await totals() + + // One session, an hour before the window opens. Nothing of the current + // window leaks into it. + assert.strictEqual(previous?.sessions, 1) + assert.strictEqual(previous?.tokens, 15) + assert.closeTo(previous!.cost, 0.01, 1e-9) + assert.strictEqual(previous?.toolCalls, 0) + }) + + it("files a session under the bucket it started in, so the buckets sum to the totals", async () => { + const compiled = compileUnionUnsafe(Integrations.aiOverviewSeriesQuery(), { + ...compareWindow, + bucketSeconds: 300, + }) + const rows = Effect.runSync(compiled.decodeRows(await runJson(compiled.sql))) + const series = rows.filter((row) => row.period === "current") + const { current } = await totals() + + // Two buckets: the turn's session and the pre-0031 row in the first, the + // sessionless trace half an hour later in its own. + assert.strictEqual(series.length, 2) + assert.isTrue(series[0]!.bucket < series[1]!.bucket, `${series[0]!.bucket} < ${series[1]!.bucket}`) + assert.strictEqual( + sumOf(series, (row) => row.sessions), + current?.sessions, + ) + assert.strictEqual( + sumOf(series, (row) => row.tokens), + current?.tokens, + ) + assert.closeTo( + sumOf(series, (row) => row.cost), + current!.cost, + 1e-9, + ) + assert.strictEqual( + sumOf(series, (row) => row.llmCalls), + current?.llmCalls, + ) + // The session the gateway mirrored spans both its traces and still lands + // in one bucket — the one its first span started in. + assert.strictEqual(series[0]?.sessions, 2) + assert.strictEqual(rows.filter((row) => row.period === "previous").length, 1) + }) + + it("files a session under every model it used, and its tokens under the call that reported them", async () => { + const compiled = compileUnionUnsafe( + Integrations.aiOverviewBreakdownQuery({ dimension: "model" }), + compareWindow, + ) + const rows = Effect.runSync(compiled.decodeRows(await runJson(compiled.sql))) + const byKey = new Map(rows.filter((row) => row.period === "current").map((row) => [row.key, row])) + const { current } = await totals() + + // The turn's session used two models, so it is a session under each — + // rows overlap and do not sum to the totals. + assert.deepStrictEqual([...byKey.keys()].sort(), ["", CLAUDE, GPT]) + assert.strictEqual(byKey.get(GPT)?.sessions, 1) + assert.strictEqual(byKey.get(CLAUDE)?.sessions, 2) + assert.isAbove( + sumOf([...byKey.values()], (row) => row.sessions), + current!.sessions, + ) + + // The usage does NOT overlap: a call's tokens are charged to the model + // that reported them, with the mirror still collapsed onto one claim and + // the turn span's roll-up — which names no model — never counted. + assert.strictEqual(byKey.get(GPT)?.tokens, 150) + assert.closeTo(byKey.get(GPT)!.cost, 0.05, 1e-9) + assert.strictEqual(byKey.get(CLAUDE)?.tokens, 30 + 300) + assert.closeTo(byKey.get(CLAUDE)!.cost, 0.11, 1e-9) + // The pre-0031 row names no model and is the unattributed key, not a gap. + assert.strictEqual(byKey.get("")?.tokens, 500) + assert.strictEqual( + sumOf([...byKey.values()], (row) => row.tokens), + current?.tokens, + ) + + // The previous window is measured over the same keys, and the third + // branch counts what the table is not showing. + const previous = rows.filter((row) => row.period === "previous") + assert.deepStrictEqual( + previous.map((row) => row.key), + [GPT], + ) + assert.strictEqual(previous[0]?.tokens, 15) + assert.strictEqual(rows.find((row) => row.period === "keys")?.keyCount, 3) + }) + + it("reads a tool breakdown over tool calls and an agent breakdown over every span", async () => { + const read = async (dimension: Integrations.AiOverviewBreakdownOpts["dimension"]) => { + const compiled = compileUnionUnsafe( + Integrations.aiOverviewBreakdownQuery({ dimension }), + compareWindow, + ) + const rows = Effect.runSync(compiled.decodeRows(await runJson(compiled.sql))) + return rows.filter((row) => row.period === "current") + } + + // One tool, called once, and it failed. A tool span reports no usage, so + // its row costs nothing — which is the honest answer, not a missing one. + const tools = await read("tool") + assert.deepStrictEqual( + tools.map((row) => ({ key: row.key, toolCalls: row.toolCalls, errored: row.erroredToolCalls })), + [{ key: "search_traces", toolCalls: 1, errored: 1 }], + ) + assert.strictEqual(tools[0]?.cost, 0) + + // Agent names sit on the turn span alone, so the rest of the spans key + // under '' — the unattributed row the page shows beside the named one. + const agents = await read("agent") + assert.deepStrictEqual([...agents].map((row) => row.key).sort(), ["", "slack-agent"]) + + // Every agent span carries a service and a vendor, so those two never + // have an unattributed row. + const services = await read("service") + assert.deepStrictEqual( + services.map((row) => ({ key: row.key, sessions: row.sessions })), + [{ key: "agent-service", sessions: 3 }], + ) + }) + + it("selects sessions the way the list selects them, by any span of the trace", async () => { + // A model filter and a tool filter together: they are matched by + // DIFFERENT spans of the same trace, which a row predicate could never + // do — and the session the two select is the one the list selects. + const { current } = await totals({ models: [GPT], toolNames: ["search_traces"] }) + const list = await listRows({ models: [GPT], toolNames: ["search_traces"] }) + + assert.strictEqual(current?.sessions, list.length) + assert.strictEqual(current?.sessions, 1) + assert.strictEqual(current?.tokens, sumOf(list, (row) => row.totalTokens)) + + // A filter no span carries selects nothing, rather than everything. + const none = await totals({ vendorIds: ["vercel_ai_sdk"] }) + assert.strictEqual(none.current?.sessions, 0) + assert.strictEqual(none.current?.cost, 0) + assert.strictEqual(none.current?.sessionDurationP50Ns, 0) + }) +}) diff --git a/packages/domain/src/http/ai-sessions.ts b/packages/domain/src/http/ai-sessions.ts index a44b45134..bdad9f12d 100644 --- a/packages/domain/src/http/ai-sessions.ts +++ b/packages/domain/src/http/ai-sessions.ts @@ -2,6 +2,7 @@ import { HttpApiEndpoint, HttpApiGroup } from "effect/unstable/httpapi" import { Schema } from "effect" import { AiAgentSpanSchema, AiGenAiValuesSchema } from "../gen-ai" import { TinybirdDateTime } from "../query-engine" +import { BucketSeconds } from "./query-engine" import { SessionAuthorization } from "./current-tenant" import { HttpTaggedError } from "./error-policy" import { warehouseReadHttpErrors } from "./warehouse" @@ -530,6 +531,208 @@ export class AiSessionTooLargeError extends HttpTaggedError + +export const AiOverviewSeriesPoint = Schema.Struct({ + /** ISO-8601 with a literal `Z`, the shape every Maple timeseries emits. */ + bucket: Schema.String, + ...aiOverviewMeasures, +}) +export type AiOverviewSeriesPoint = Schema.Schema.Type + +export class AiOverviewSummaryRequest extends Schema.Class( + "AiOverviewSummaryRequest", +)({ + startTime: TinybirdDateTime, + endTime: TinybirdDateTime, + /** Whole seconds, greater than zero — it reaches `toStartOfInterval` as an + * `INTERVAL n SECOND` literal, so a fraction is a 400 and not a 500. */ + bucketSeconds: BucketSeconds, + ...aiOverviewSelection, +}) {} + +export class AiOverviewSummaryResponse extends Schema.Class( + "AiOverviewSummaryResponse", +)({ + /** Echoed back, so a client rendering an axis reads the width the buckets + * were actually cut at rather than re-deriving it. */ + bucketSeconds: Schema.Number, + /** The whole selected window. */ + current: AiOverviewMeasures, + /** + * The window of equal length immediately before the caller's, measured by + * the same read — the deltas the tiles show. Zeros where nothing ran then, + * which the client renders as "no comparison" rather than a -100%. + */ + previous: AiOverviewMeasures, + /** + * One point per bucket that had a session, oldest first. A session — and + * its netted usage, its calls and its failures — belongs to the bucket its + * FIRST span started in, so the points sum to `current` rather than + * counting a long session in every bucket it touched. Quantiles are the + * exception and cannot be summed at all, which is why `current` comes from + * its own un-bucketed read and not from these. + */ + series: Schema.Array(AiOverviewSeriesPoint), + /** The same, over the previous window and at the same bucket width. */ + previousSeries: Schema.Array(AiOverviewSeriesPoint), +}) {} + +/** Which dimension the breakdown groups by. Each is a column of + * `ai_trace_index`; there is no provider column — a model maps to its + * provider client-side. */ +export const AiOverviewDimension = Schema.Literals([ + "model", + "agent", + "service", + "environment", + "vendor", + "tool", +]) +export type AiOverviewDimension = Schema.Schema.Type + +/** Rows one breakdown returns, and the default. The page shows a table, not a + * catalogue: `totalKeys` is what lets it say "+ N more" off the same read. */ +export const AI_OVERVIEW_BREAKDOWN_MAX = 12 + +export const AiOverviewBreakdownRow = Schema.Struct({ + /** + * The dimension's value. `''` is a real key, not a gap — a span that + * carries no value for this dimension — and the page renders it as + * unattributed rather than hiding it. + */ + key: Schema.String, + current: AiOverviewMeasures, + /** The same key over the previous window; zeros where it did not appear. */ + previous: AiOverviewMeasures, +}) +export type AiOverviewBreakdownRow = Schema.Schema.Type + +export class AiOverviewBreakdownRequest extends Schema.Class( + "AiOverviewBreakdownRequest", +)({ + startTime: TinybirdDateTime, + endTime: TinybirdDateTime, + dimension: AiOverviewDimension, + limit: Schema.optionalKey( + Schema.Number.check( + Schema.isInt(), + Schema.isBetween({ minimum: 1, maximum: AI_OVERVIEW_BREAKDOWN_MAX }), + ), + ), + ...aiOverviewSelection, +}) {} + +export class AiOverviewBreakdownResponse extends Schema.Class( + "AiOverviewBreakdownResponse", +)({ + dimension: AiOverviewDimension, + /** + * The busiest keys by session count, most sessions first. + * + * Rows OVERLAP and need not sum to the totals: a session that used two + * models is a session under each of them. What does not overlap is the + * usage — a model call's tokens are netted under the model that reported + * them, so the cost column splits rather than repeats. + */ + rows: Schema.Array(AiOverviewBreakdownRow), + /** Distinct keys in the current window, so the table can say how many it + * is not showing. */ + totalKeys: Schema.Number, +}) {} + export class AiSessionsInternalApiGroup extends HttpApiGroup.make("aiSessionsInternal") .add( HttpApiEndpoint.post("list", "/list", { @@ -566,5 +769,19 @@ export class AiSessionsInternalApiGroup extends HttpApiGroup.make("aiSessionsInt error: warehouseReadHttpErrors, }), ) + .add( + HttpApiEndpoint.post("overviewSummary", "/overview/summary", { + payload: AiOverviewSummaryRequest, + success: AiOverviewSummaryResponse, + error: warehouseReadHttpErrors, + }), + ) + .add( + HttpApiEndpoint.post("overviewBreakdown", "/overview/breakdown", { + payload: AiOverviewBreakdownRequest, + success: AiOverviewBreakdownResponse, + error: warehouseReadHttpErrors, + }), + ) .prefix("/internal/ai-sessions") .middleware(SessionAuthorization) {} diff --git a/packages/domain/src/http/query-engine.ts b/packages/domain/src/http/query-engine.ts index a76088017..93a07bb81 100644 --- a/packages/domain/src/http/query-engine.ts +++ b/packages/domain/src/http/query-engine.ts @@ -36,7 +36,7 @@ import { FunnelBreakdownBy, FunnelKeyBy, FunnelStep } from "@maple/query-model" * of a 400. `packages/domain/src/query-engine.ts` already had this right; these * declarations did not. */ -const BucketSeconds = Schema.Number.check(Schema.isInt(), Schema.isGreaterThan(0)).pipe( +export const BucketSeconds = Schema.Number.check(Schema.isInt(), Schema.isGreaterThan(0)).pipe( Schema.annotate({ identifier: "BucketSeconds", description: "Timeseries bucket width in whole seconds, greater than zero.", diff --git a/packages/query-engine-integrations/src/__sql_baseline__/integrations.sql b/packages/query-engine-integrations/src/__sql_baseline__/integrations.sql index f39c1bbd9..050458e45 100644 --- a/packages/query-engine-integrations/src/__sql_baseline__/integrations.sql +++ b/packages/query-engine-integrations/src/__sql_baseline__/integrations.sql @@ -1,3 +1,1095 @@ +-- builder:ai-overview:aiOverviewBreakdownQuery:model +SELECT + 'current' AS period, + key AS key, + 0 AS keyCount, + count() AS sessions, + countIf(errorSpans > 0) AS erroredSessions, + sum(toFloat64(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 2)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, toFloat64(n.2)), arrayFilter(n -> n.1 != '', netted))))))) AS llmCalls, + sum(erroredLlmCalls) AS erroredLlmCalls, + sum(toolCalls) AS toolCalls, + sum(erroredToolCalls) AS erroredToolCalls, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 4)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.4), arrayFilter(n -> n.1 != '', netted)))))) AS cost, + sum(arraySum(arrayMap(n -> toFloat64(n.2 AND n.4 > 0), arrayFilter(n -> n.1 = '', netted))) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, toFloat64(n.2 AND n.4 > 0)), arrayFilter(n -> n.1 != '', netted)))))) AS pricedLlmCalls, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 3)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.3), arrayFilter(n -> n.1 != '', netted)))))) AS tokens, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 5)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.5), arrayFilter(n -> n.1 != '', netted)))))) AS inputTokens, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 6)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.6), arrayFilter(n -> n.1 != '', netted)))))) AS cacheReadTokens, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 7)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.7), arrayFilter(n -> n.1 != '', netted)))))) AS cacheWriteTokens, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 8)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.8), arrayFilter(n -> n.1 != '', netted)))))) AS outputTokens, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 9)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.9), arrayFilter(n -> n.1 != '', netted)))))) AS reasoningTokens, + ifNull(ifNotFinite(quantile(0.5)(sessionDurationNs), 0), 0) AS sessionDurationP50Ns, + ifNull(ifNotFinite(quantile(0.95)(sessionDurationNs), 0), 0) AS sessionDurationP95Ns, + ifNull(ifNotFinite(quantileArray(0.5)(llmDurations), 0), 0) AS llmDurationP50Ns, + ifNull(ifNotFinite(quantileArray(0.95)(llmDurations), 0), 0) AS llmDurationP95Ns + FROM (SELECT + key AS key, + sessionStart AS sessionStart, + sessionDurationNs AS sessionDurationNs, + errorSpans AS errorSpans, + toolCalls AS toolCalls, + erroredToolCalls AS erroredToolCalls, + erroredLlmCalls AS erroredLlmCalls, + llmDurations AS llmDurations, + arrayMap(r -> tuple(r.5, r.6 = 1 AND if((r.3 > 0 OR r.4 > 0), greatest(0., r.3 - arrayElement(tupleElement(childClaims, 2), indexOf(tupleElement(childClaims, 1), r.1))) > 0 OR greatest(0., r.4 - arrayElement(tupleElement(childClaims, 3), indexOf(tupleElement(childClaims, 1), r.1))) > 0, NOT has(reportingIds, r.2)), greatest(0., r.3 - arrayElement(tupleElement(childClaims, 2), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.4 - arrayElement(tupleElement(childClaims, 3), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.7 - arrayElement(tupleElement(childClaims, 4), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.8 - arrayElement(tupleElement(childClaims, 5), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.9 - arrayElement(tupleElement(childClaims, 6), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.10 - arrayElement(tupleElement(childClaims, 7), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.11 - arrayElement(tupleElement(childClaims, 8), indexOf(tupleElement(childClaims, 1), r.1)))), reporters) AS netted + FROM (SELECT + if(trace.rawSessionId = '', concat('trace:', ai_trace_index.TraceId), trace.rawSessionId) AS sessionId, + toString(ai_trace_index.Model) AS key, + min(ai_trace_index.Timestamp) AS sessionStart, + max(toUnixTimestamp64Nano(ai_trace_index.Timestamp) + toInt64(ai_trace_index.Duration)) - toUnixTimestamp64Nano(min(ai_trace_index.Timestamp)) AS sessionDurationNs, + sum(ai_trace_index.IsError) AS errorSpans, + sum(ai_trace_index.IsToolCall) AS toolCalls, + sumIf(ai_trace_index.IsError, ai_trace_index.IsToolCall = 1) AS erroredToolCalls, + sumIf(ai_trace_index.IsError, ai_trace_index.IsLlmCall = 1) AS erroredLlmCalls, + groupArrayIf(2000)(ai_trace_index.Duration, ai_trace_index.IsLlmCall = 1) AS llmDurations, + groupArrayIf(2000)(tuple(ai_trace_index.SpanId, ai_trace_index.ParentSpanId, ai_trace_index.Tokens, ai_trace_index.Cost, ai_trace_index.ResponseId, ai_trace_index.IsLlmCall, ai_trace_index.InputTokens, ai_trace_index.CacheReadTokens, ai_trace_index.CacheWriteTokens, ai_trace_index.OutputTokens, ai_trace_index.ReasoningTokens), ((ai_trace_index.Tokens > 0 OR ai_trace_index.Cost > 0) OR ai_trace_index.IsLlmCall = 1)) AS reporters, + arrayReduce('sumMap', arrayMap(c -> [c.2], reporters), arrayMap(c -> [c.3], reporters), arrayMap(c -> [c.4], reporters), arrayMap(c -> [c.7], reporters), arrayMap(c -> [c.8], reporters), arrayMap(c -> [c.9], reporters), arrayMap(c -> [c.10], reporters), arrayMap(c -> [c.11], reporters)) AS childClaims, + tupleElement(arrayFilter(p -> p.3 > 0 OR p.4 > 0, reporters), 1) AS reportingIds + FROM ai_trace_index + INNER JOIN (SELECT + TraceId AS TraceId, + max(SessionId) AS rawSessionId + FROM ai_trace_index + WHERE OrgId = 'org_sql_catalog' + AND Timestamp >= '2026-01-01 10:30:00' + AND Timestamp <= '2026-01-03 14:15:00' + GROUP BY TraceId) AS trace ON ai_trace_index.TraceId = trace.TraceId + WHERE ai_trace_index.OrgId = 'org_sql_catalog' + AND ai_trace_index.Timestamp >= '2026-01-01 10:30:00' + AND ai_trace_index.Timestamp <= '2026-01-03 14:15:00' + AND ai_trace_index.IsLlmCall = 1 + GROUP BY sessionId, key) AS session_rows) AS netted_current + WHERE key IN (SELECT + rankKey AS topKey + FROM (SELECT + toString(ai_trace_index.Model) AS rankKey, + uniqExact(if(trace.rawSessionId = '', concat('trace:', ai_trace_index.TraceId), trace.rawSessionId)) AS rankSessions + FROM ai_trace_index + INNER JOIN (SELECT + TraceId AS TraceId, + max(SessionId) AS rawSessionId + FROM ai_trace_index + WHERE OrgId = 'org_sql_catalog' + AND Timestamp >= '2026-01-01 10:30:00' + AND Timestamp <= '2026-01-03 14:15:00' + GROUP BY TraceId) AS trace ON ai_trace_index.TraceId = trace.TraceId + WHERE ai_trace_index.OrgId = 'org_sql_catalog' + AND ai_trace_index.Timestamp >= '2026-01-01 10:30:00' + AND ai_trace_index.Timestamp <= '2026-01-03 14:15:00' + AND ai_trace_index.IsLlmCall = 1 + GROUP BY rankKey + ORDER BY rankSessions DESC, rankKey ASC + LIMIT 12) AS top_keys) + GROUP BY key +UNION ALL +SELECT + 'previous' AS period, + key AS key, + 0 AS keyCount, + count() AS sessions, + countIf(errorSpans > 0) AS erroredSessions, + sum(toFloat64(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 2)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, toFloat64(n.2)), arrayFilter(n -> n.1 != '', netted))))))) AS llmCalls, + sum(erroredLlmCalls) AS erroredLlmCalls, + sum(toolCalls) AS toolCalls, + sum(erroredToolCalls) AS erroredToolCalls, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 4)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.4), arrayFilter(n -> n.1 != '', netted)))))) AS cost, + sum(arraySum(arrayMap(n -> toFloat64(n.2 AND n.4 > 0), arrayFilter(n -> n.1 = '', netted))) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, toFloat64(n.2 AND n.4 > 0)), arrayFilter(n -> n.1 != '', netted)))))) AS pricedLlmCalls, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 3)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.3), arrayFilter(n -> n.1 != '', netted)))))) AS tokens, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 5)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.5), arrayFilter(n -> n.1 != '', netted)))))) AS inputTokens, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 6)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.6), arrayFilter(n -> n.1 != '', netted)))))) AS cacheReadTokens, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 7)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.7), arrayFilter(n -> n.1 != '', netted)))))) AS cacheWriteTokens, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 8)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.8), arrayFilter(n -> n.1 != '', netted)))))) AS outputTokens, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 9)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.9), arrayFilter(n -> n.1 != '', netted)))))) AS reasoningTokens, + ifNull(ifNotFinite(quantile(0.5)(sessionDurationNs), 0), 0) AS sessionDurationP50Ns, + ifNull(ifNotFinite(quantile(0.95)(sessionDurationNs), 0), 0) AS sessionDurationP95Ns, + ifNull(ifNotFinite(quantileArray(0.5)(llmDurations), 0), 0) AS llmDurationP50Ns, + ifNull(ifNotFinite(quantileArray(0.95)(llmDurations), 0), 0) AS llmDurationP95Ns + FROM (SELECT + key AS key, + sessionStart AS sessionStart, + sessionDurationNs AS sessionDurationNs, + errorSpans AS errorSpans, + toolCalls AS toolCalls, + erroredToolCalls AS erroredToolCalls, + erroredLlmCalls AS erroredLlmCalls, + llmDurations AS llmDurations, + arrayMap(r -> tuple(r.5, r.6 = 1 AND if((r.3 > 0 OR r.4 > 0), greatest(0., r.3 - arrayElement(tupleElement(childClaims, 2), indexOf(tupleElement(childClaims, 1), r.1))) > 0 OR greatest(0., r.4 - arrayElement(tupleElement(childClaims, 3), indexOf(tupleElement(childClaims, 1), r.1))) > 0, NOT has(reportingIds, r.2)), greatest(0., r.3 - arrayElement(tupleElement(childClaims, 2), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.4 - arrayElement(tupleElement(childClaims, 3), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.7 - arrayElement(tupleElement(childClaims, 4), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.8 - arrayElement(tupleElement(childClaims, 5), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.9 - arrayElement(tupleElement(childClaims, 6), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.10 - arrayElement(tupleElement(childClaims, 7), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.11 - arrayElement(tupleElement(childClaims, 8), indexOf(tupleElement(childClaims, 1), r.1)))), reporters) AS netted + FROM (SELECT + if(trace.rawSessionId = '', concat('trace:', ai_trace_index.TraceId), trace.rawSessionId) AS sessionId, + toString(ai_trace_index.Model) AS key, + min(ai_trace_index.Timestamp) AS sessionStart, + max(toUnixTimestamp64Nano(ai_trace_index.Timestamp) + toInt64(ai_trace_index.Duration)) - toUnixTimestamp64Nano(min(ai_trace_index.Timestamp)) AS sessionDurationNs, + sum(ai_trace_index.IsError) AS errorSpans, + sum(ai_trace_index.IsToolCall) AS toolCalls, + sumIf(ai_trace_index.IsError, ai_trace_index.IsToolCall = 1) AS erroredToolCalls, + sumIf(ai_trace_index.IsError, ai_trace_index.IsLlmCall = 1) AS erroredLlmCalls, + groupArrayIf(2000)(ai_trace_index.Duration, ai_trace_index.IsLlmCall = 1) AS llmDurations, + groupArrayIf(2000)(tuple(ai_trace_index.SpanId, ai_trace_index.ParentSpanId, ai_trace_index.Tokens, ai_trace_index.Cost, ai_trace_index.ResponseId, ai_trace_index.IsLlmCall, ai_trace_index.InputTokens, ai_trace_index.CacheReadTokens, ai_trace_index.CacheWriteTokens, ai_trace_index.OutputTokens, ai_trace_index.ReasoningTokens), ((ai_trace_index.Tokens > 0 OR ai_trace_index.Cost > 0) OR ai_trace_index.IsLlmCall = 1)) AS reporters, + arrayReduce('sumMap', arrayMap(c -> [c.2], reporters), arrayMap(c -> [c.3], reporters), arrayMap(c -> [c.4], reporters), arrayMap(c -> [c.7], reporters), arrayMap(c -> [c.8], reporters), arrayMap(c -> [c.9], reporters), arrayMap(c -> [c.10], reporters), arrayMap(c -> [c.11], reporters)) AS childClaims, + tupleElement(arrayFilter(p -> p.3 > 0 OR p.4 > 0, reporters), 1) AS reportingIds + FROM ai_trace_index + INNER JOIN (SELECT + TraceId AS TraceId, + max(SessionId) AS rawSessionId + FROM ai_trace_index + WHERE OrgId = 'org_sql_catalog' + AND Timestamp >= '2025-12-30 06:45:00' + AND Timestamp <= '2026-01-01 10:30:00' + GROUP BY TraceId) AS trace ON ai_trace_index.TraceId = trace.TraceId + WHERE ai_trace_index.OrgId = 'org_sql_catalog' + AND ai_trace_index.Timestamp >= '2025-12-30 06:45:00' + AND ai_trace_index.Timestamp <= '2026-01-01 10:30:00' + AND ai_trace_index.IsLlmCall = 1 + GROUP BY sessionId, key) AS session_rows) AS netted_previous + WHERE key IN (SELECT + rankKey AS topKey + FROM (SELECT + toString(ai_trace_index.Model) AS rankKey, + uniqExact(if(trace.rawSessionId = '', concat('trace:', ai_trace_index.TraceId), trace.rawSessionId)) AS rankSessions + FROM ai_trace_index + INNER JOIN (SELECT + TraceId AS TraceId, + max(SessionId) AS rawSessionId + FROM ai_trace_index + WHERE OrgId = 'org_sql_catalog' + AND Timestamp >= '2026-01-01 10:30:00' + AND Timestamp <= '2026-01-03 14:15:00' + GROUP BY TraceId) AS trace ON ai_trace_index.TraceId = trace.TraceId + WHERE ai_trace_index.OrgId = 'org_sql_catalog' + AND ai_trace_index.Timestamp >= '2026-01-01 10:30:00' + AND ai_trace_index.Timestamp <= '2026-01-03 14:15:00' + AND ai_trace_index.IsLlmCall = 1 + GROUP BY rankKey + ORDER BY rankSessions DESC, rankKey ASC + LIMIT 12) AS top_keys) + GROUP BY key +UNION ALL +SELECT + 'keys' AS period, + '' AS key, + uniqExact(toString(ai_trace_index.Model)) AS keyCount, + 0 AS sessions, + 0 AS erroredSessions, + 0 AS llmCalls, + 0 AS erroredLlmCalls, + 0 AS toolCalls, + 0 AS erroredToolCalls, + 0 AS cost, + 0 AS pricedLlmCalls, + 0 AS tokens, + 0 AS inputTokens, + 0 AS cacheReadTokens, + 0 AS cacheWriteTokens, + 0 AS outputTokens, + 0 AS reasoningTokens, + 0 AS sessionDurationP50Ns, + 0 AS sessionDurationP95Ns, + 0 AS llmDurationP50Ns, + 0 AS llmDurationP95Ns + FROM ai_trace_index + INNER JOIN (SELECT + TraceId AS TraceId, + max(SessionId) AS rawSessionId + FROM ai_trace_index + WHERE OrgId = 'org_sql_catalog' + AND Timestamp >= '2026-01-01 10:30:00' + AND Timestamp <= '2026-01-03 14:15:00' + GROUP BY TraceId) AS trace ON ai_trace_index.TraceId = trace.TraceId + WHERE ai_trace_index.OrgId = 'org_sql_catalog' + AND ai_trace_index.Timestamp >= '2026-01-01 10:30:00' + AND ai_trace_index.Timestamp <= '2026-01-03 14:15:00' + AND ai_trace_index.IsLlmCall = 1 +FORMAT JSON + +-- builder:ai-overview:aiOverviewBreakdownQuery:service +SELECT + 'current' AS period, + key AS key, + 0 AS keyCount, + count() AS sessions, + countIf(errorSpans > 0) AS erroredSessions, + sum(toFloat64(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 2)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, toFloat64(n.2)), arrayFilter(n -> n.1 != '', netted))))))) AS llmCalls, + sum(erroredLlmCalls) AS erroredLlmCalls, + sum(toolCalls) AS toolCalls, + sum(erroredToolCalls) AS erroredToolCalls, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 4)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.4), arrayFilter(n -> n.1 != '', netted)))))) AS cost, + sum(arraySum(arrayMap(n -> toFloat64(n.2 AND n.4 > 0), arrayFilter(n -> n.1 = '', netted))) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, toFloat64(n.2 AND n.4 > 0)), arrayFilter(n -> n.1 != '', netted)))))) AS pricedLlmCalls, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 3)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.3), arrayFilter(n -> n.1 != '', netted)))))) AS tokens, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 5)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.5), arrayFilter(n -> n.1 != '', netted)))))) AS inputTokens, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 6)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.6), arrayFilter(n -> n.1 != '', netted)))))) AS cacheReadTokens, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 7)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.7), arrayFilter(n -> n.1 != '', netted)))))) AS cacheWriteTokens, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 8)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.8), arrayFilter(n -> n.1 != '', netted)))))) AS outputTokens, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 9)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.9), arrayFilter(n -> n.1 != '', netted)))))) AS reasoningTokens, + ifNull(ifNotFinite(quantile(0.5)(sessionDurationNs), 0), 0) AS sessionDurationP50Ns, + ifNull(ifNotFinite(quantile(0.95)(sessionDurationNs), 0), 0) AS sessionDurationP95Ns, + ifNull(ifNotFinite(quantileArray(0.5)(llmDurations), 0), 0) AS llmDurationP50Ns, + ifNull(ifNotFinite(quantileArray(0.95)(llmDurations), 0), 0) AS llmDurationP95Ns + FROM (SELECT + key AS key, + sessionStart AS sessionStart, + sessionDurationNs AS sessionDurationNs, + errorSpans AS errorSpans, + toolCalls AS toolCalls, + erroredToolCalls AS erroredToolCalls, + erroredLlmCalls AS erroredLlmCalls, + llmDurations AS llmDurations, + arrayMap(r -> tuple(r.5, r.6 = 1 AND if((r.3 > 0 OR r.4 > 0), greatest(0., r.3 - arrayElement(tupleElement(childClaims, 2), indexOf(tupleElement(childClaims, 1), r.1))) > 0 OR greatest(0., r.4 - arrayElement(tupleElement(childClaims, 3), indexOf(tupleElement(childClaims, 1), r.1))) > 0, NOT has(reportingIds, r.2)), greatest(0., r.3 - arrayElement(tupleElement(childClaims, 2), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.4 - arrayElement(tupleElement(childClaims, 3), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.7 - arrayElement(tupleElement(childClaims, 4), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.8 - arrayElement(tupleElement(childClaims, 5), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.9 - arrayElement(tupleElement(childClaims, 6), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.10 - arrayElement(tupleElement(childClaims, 7), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.11 - arrayElement(tupleElement(childClaims, 8), indexOf(tupleElement(childClaims, 1), r.1)))), reporters) AS netted + FROM (SELECT + if(trace.rawSessionId = '', concat('trace:', ai_trace_index.TraceId), trace.rawSessionId) AS sessionId, + toString(ai_trace_index.ServiceName) AS key, + min(ai_trace_index.Timestamp) AS sessionStart, + max(toUnixTimestamp64Nano(ai_trace_index.Timestamp) + toInt64(ai_trace_index.Duration)) - toUnixTimestamp64Nano(min(ai_trace_index.Timestamp)) AS sessionDurationNs, + sum(ai_trace_index.IsError) AS errorSpans, + sum(ai_trace_index.IsToolCall) AS toolCalls, + sumIf(ai_trace_index.IsError, ai_trace_index.IsToolCall = 1) AS erroredToolCalls, + sumIf(ai_trace_index.IsError, ai_trace_index.IsLlmCall = 1) AS erroredLlmCalls, + groupArrayIf(2000)(ai_trace_index.Duration, ai_trace_index.IsLlmCall = 1) AS llmDurations, + groupArrayIf(2000)(tuple(ai_trace_index.SpanId, ai_trace_index.ParentSpanId, ai_trace_index.Tokens, ai_trace_index.Cost, ai_trace_index.ResponseId, ai_trace_index.IsLlmCall, ai_trace_index.InputTokens, ai_trace_index.CacheReadTokens, ai_trace_index.CacheWriteTokens, ai_trace_index.OutputTokens, ai_trace_index.ReasoningTokens), ((ai_trace_index.Tokens > 0 OR ai_trace_index.Cost > 0) OR ai_trace_index.IsLlmCall = 1)) AS reporters, + arrayReduce('sumMap', arrayMap(c -> [c.2], reporters), arrayMap(c -> [c.3], reporters), arrayMap(c -> [c.4], reporters), arrayMap(c -> [c.7], reporters), arrayMap(c -> [c.8], reporters), arrayMap(c -> [c.9], reporters), arrayMap(c -> [c.10], reporters), arrayMap(c -> [c.11], reporters)) AS childClaims, + tupleElement(arrayFilter(p -> p.3 > 0 OR p.4 > 0, reporters), 1) AS reportingIds + FROM ai_trace_index + INNER JOIN (SELECT + TraceId AS TraceId, + max(SessionId) AS rawSessionId + FROM ai_trace_index + WHERE OrgId = 'org_sql_catalog' + AND Timestamp >= '2026-01-01 10:30:00' + AND Timestamp <= '2026-01-03 14:15:00' + GROUP BY TraceId) AS trace ON ai_trace_index.TraceId = trace.TraceId + WHERE ai_trace_index.OrgId = 'org_sql_catalog' + AND ai_trace_index.Timestamp >= '2026-01-01 10:30:00' + AND ai_trace_index.Timestamp <= '2026-01-03 14:15:00' + AND if(trace.rawSessionId = '', concat('trace:', ai_trace_index.TraceId), trace.rawSessionId) IN (SELECT + if(trace.rawSessionId = '', concat('trace:', ai_trace_index.TraceId), trace.rawSessionId) AS sessionId + FROM ai_trace_index + INNER JOIN (SELECT + TraceId AS TraceId, + max(SessionId) AS rawSessionId + FROM ai_trace_index + WHERE OrgId = 'org_sql_catalog' + AND Timestamp >= '2026-01-01 10:30:00' + AND Timestamp <= '2026-01-03 14:15:00' + GROUP BY TraceId) AS trace ON ai_trace_index.TraceId = trace.TraceId + WHERE ai_trace_index.OrgId = 'org_sql_catalog' + AND ai_trace_index.Timestamp >= '2026-01-01 10:30:00' + AND ai_trace_index.Timestamp <= '2026-01-03 14:15:00' + GROUP BY sessionId + HAVING sum(ai_trace_index.IsError) > 0) + GROUP BY sessionId, key) AS session_rows) AS netted_current + WHERE key IN (SELECT + rankKey AS topKey + FROM (SELECT + toString(ai_trace_index.ServiceName) AS rankKey, + uniqExact(if(trace.rawSessionId = '', concat('trace:', ai_trace_index.TraceId), trace.rawSessionId)) AS rankSessions + FROM ai_trace_index + INNER JOIN (SELECT + TraceId AS TraceId, + max(SessionId) AS rawSessionId + FROM ai_trace_index + WHERE OrgId = 'org_sql_catalog' + AND Timestamp >= '2026-01-01 10:30:00' + AND Timestamp <= '2026-01-03 14:15:00' + GROUP BY TraceId) AS trace ON ai_trace_index.TraceId = trace.TraceId + WHERE ai_trace_index.OrgId = 'org_sql_catalog' + AND ai_trace_index.Timestamp >= '2026-01-01 10:30:00' + AND ai_trace_index.Timestamp <= '2026-01-03 14:15:00' + AND if(trace.rawSessionId = '', concat('trace:', ai_trace_index.TraceId), trace.rawSessionId) IN (SELECT + if(trace.rawSessionId = '', concat('trace:', ai_trace_index.TraceId), trace.rawSessionId) AS sessionId + FROM ai_trace_index + INNER JOIN (SELECT + TraceId AS TraceId, + max(SessionId) AS rawSessionId + FROM ai_trace_index + WHERE OrgId = 'org_sql_catalog' + AND Timestamp >= '2026-01-01 10:30:00' + AND Timestamp <= '2026-01-03 14:15:00' + GROUP BY TraceId) AS trace ON ai_trace_index.TraceId = trace.TraceId + WHERE ai_trace_index.OrgId = 'org_sql_catalog' + AND ai_trace_index.Timestamp >= '2026-01-01 10:30:00' + AND ai_trace_index.Timestamp <= '2026-01-03 14:15:00' + GROUP BY sessionId + HAVING sum(ai_trace_index.IsError) > 0) + GROUP BY rankKey + ORDER BY rankSessions DESC, rankKey ASC + LIMIT 12) AS top_keys) + GROUP BY key +UNION ALL +SELECT + 'previous' AS period, + key AS key, + 0 AS keyCount, + count() AS sessions, + countIf(errorSpans > 0) AS erroredSessions, + sum(toFloat64(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 2)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, toFloat64(n.2)), arrayFilter(n -> n.1 != '', netted))))))) AS llmCalls, + sum(erroredLlmCalls) AS erroredLlmCalls, + sum(toolCalls) AS toolCalls, + sum(erroredToolCalls) AS erroredToolCalls, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 4)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.4), arrayFilter(n -> n.1 != '', netted)))))) AS cost, + sum(arraySum(arrayMap(n -> toFloat64(n.2 AND n.4 > 0), arrayFilter(n -> n.1 = '', netted))) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, toFloat64(n.2 AND n.4 > 0)), arrayFilter(n -> n.1 != '', netted)))))) AS pricedLlmCalls, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 3)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.3), arrayFilter(n -> n.1 != '', netted)))))) AS tokens, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 5)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.5), arrayFilter(n -> n.1 != '', netted)))))) AS inputTokens, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 6)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.6), arrayFilter(n -> n.1 != '', netted)))))) AS cacheReadTokens, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 7)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.7), arrayFilter(n -> n.1 != '', netted)))))) AS cacheWriteTokens, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 8)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.8), arrayFilter(n -> n.1 != '', netted)))))) AS outputTokens, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 9)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.9), arrayFilter(n -> n.1 != '', netted)))))) AS reasoningTokens, + ifNull(ifNotFinite(quantile(0.5)(sessionDurationNs), 0), 0) AS sessionDurationP50Ns, + ifNull(ifNotFinite(quantile(0.95)(sessionDurationNs), 0), 0) AS sessionDurationP95Ns, + ifNull(ifNotFinite(quantileArray(0.5)(llmDurations), 0), 0) AS llmDurationP50Ns, + ifNull(ifNotFinite(quantileArray(0.95)(llmDurations), 0), 0) AS llmDurationP95Ns + FROM (SELECT + key AS key, + sessionStart AS sessionStart, + sessionDurationNs AS sessionDurationNs, + errorSpans AS errorSpans, + toolCalls AS toolCalls, + erroredToolCalls AS erroredToolCalls, + erroredLlmCalls AS erroredLlmCalls, + llmDurations AS llmDurations, + arrayMap(r -> tuple(r.5, r.6 = 1 AND if((r.3 > 0 OR r.4 > 0), greatest(0., r.3 - arrayElement(tupleElement(childClaims, 2), indexOf(tupleElement(childClaims, 1), r.1))) > 0 OR greatest(0., r.4 - arrayElement(tupleElement(childClaims, 3), indexOf(tupleElement(childClaims, 1), r.1))) > 0, NOT has(reportingIds, r.2)), greatest(0., r.3 - arrayElement(tupleElement(childClaims, 2), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.4 - arrayElement(tupleElement(childClaims, 3), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.7 - arrayElement(tupleElement(childClaims, 4), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.8 - arrayElement(tupleElement(childClaims, 5), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.9 - arrayElement(tupleElement(childClaims, 6), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.10 - arrayElement(tupleElement(childClaims, 7), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.11 - arrayElement(tupleElement(childClaims, 8), indexOf(tupleElement(childClaims, 1), r.1)))), reporters) AS netted + FROM (SELECT + if(trace.rawSessionId = '', concat('trace:', ai_trace_index.TraceId), trace.rawSessionId) AS sessionId, + toString(ai_trace_index.ServiceName) AS key, + min(ai_trace_index.Timestamp) AS sessionStart, + max(toUnixTimestamp64Nano(ai_trace_index.Timestamp) + toInt64(ai_trace_index.Duration)) - toUnixTimestamp64Nano(min(ai_trace_index.Timestamp)) AS sessionDurationNs, + sum(ai_trace_index.IsError) AS errorSpans, + sum(ai_trace_index.IsToolCall) AS toolCalls, + sumIf(ai_trace_index.IsError, ai_trace_index.IsToolCall = 1) AS erroredToolCalls, + sumIf(ai_trace_index.IsError, ai_trace_index.IsLlmCall = 1) AS erroredLlmCalls, + groupArrayIf(2000)(ai_trace_index.Duration, ai_trace_index.IsLlmCall = 1) AS llmDurations, + groupArrayIf(2000)(tuple(ai_trace_index.SpanId, ai_trace_index.ParentSpanId, ai_trace_index.Tokens, ai_trace_index.Cost, ai_trace_index.ResponseId, ai_trace_index.IsLlmCall, ai_trace_index.InputTokens, ai_trace_index.CacheReadTokens, ai_trace_index.CacheWriteTokens, ai_trace_index.OutputTokens, ai_trace_index.ReasoningTokens), ((ai_trace_index.Tokens > 0 OR ai_trace_index.Cost > 0) OR ai_trace_index.IsLlmCall = 1)) AS reporters, + arrayReduce('sumMap', arrayMap(c -> [c.2], reporters), arrayMap(c -> [c.3], reporters), arrayMap(c -> [c.4], reporters), arrayMap(c -> [c.7], reporters), arrayMap(c -> [c.8], reporters), arrayMap(c -> [c.9], reporters), arrayMap(c -> [c.10], reporters), arrayMap(c -> [c.11], reporters)) AS childClaims, + tupleElement(arrayFilter(p -> p.3 > 0 OR p.4 > 0, reporters), 1) AS reportingIds + FROM ai_trace_index + INNER JOIN (SELECT + TraceId AS TraceId, + max(SessionId) AS rawSessionId + FROM ai_trace_index + WHERE OrgId = 'org_sql_catalog' + AND Timestamp >= '2025-12-30 06:45:00' + AND Timestamp <= '2026-01-01 10:30:00' + GROUP BY TraceId) AS trace ON ai_trace_index.TraceId = trace.TraceId + WHERE ai_trace_index.OrgId = 'org_sql_catalog' + AND ai_trace_index.Timestamp >= '2025-12-30 06:45:00' + AND ai_trace_index.Timestamp <= '2026-01-01 10:30:00' + AND if(trace.rawSessionId = '', concat('trace:', ai_trace_index.TraceId), trace.rawSessionId) IN (SELECT + if(trace.rawSessionId = '', concat('trace:', ai_trace_index.TraceId), trace.rawSessionId) AS sessionId + FROM ai_trace_index + INNER JOIN (SELECT + TraceId AS TraceId, + max(SessionId) AS rawSessionId + FROM ai_trace_index + WHERE OrgId = 'org_sql_catalog' + AND Timestamp >= '2025-12-30 06:45:00' + AND Timestamp <= '2026-01-01 10:30:00' + GROUP BY TraceId) AS trace ON ai_trace_index.TraceId = trace.TraceId + WHERE ai_trace_index.OrgId = 'org_sql_catalog' + AND ai_trace_index.Timestamp >= '2025-12-30 06:45:00' + AND ai_trace_index.Timestamp <= '2026-01-01 10:30:00' + GROUP BY sessionId + HAVING sum(ai_trace_index.IsError) > 0) + GROUP BY sessionId, key) AS session_rows) AS netted_previous + WHERE key IN (SELECT + rankKey AS topKey + FROM (SELECT + toString(ai_trace_index.ServiceName) AS rankKey, + uniqExact(if(trace.rawSessionId = '', concat('trace:', ai_trace_index.TraceId), trace.rawSessionId)) AS rankSessions + FROM ai_trace_index + INNER JOIN (SELECT + TraceId AS TraceId, + max(SessionId) AS rawSessionId + FROM ai_trace_index + WHERE OrgId = 'org_sql_catalog' + AND Timestamp >= '2026-01-01 10:30:00' + AND Timestamp <= '2026-01-03 14:15:00' + GROUP BY TraceId) AS trace ON ai_trace_index.TraceId = trace.TraceId + WHERE ai_trace_index.OrgId = 'org_sql_catalog' + AND ai_trace_index.Timestamp >= '2026-01-01 10:30:00' + AND ai_trace_index.Timestamp <= '2026-01-03 14:15:00' + AND if(trace.rawSessionId = '', concat('trace:', ai_trace_index.TraceId), trace.rawSessionId) IN (SELECT + if(trace.rawSessionId = '', concat('trace:', ai_trace_index.TraceId), trace.rawSessionId) AS sessionId + FROM ai_trace_index + INNER JOIN (SELECT + TraceId AS TraceId, + max(SessionId) AS rawSessionId + FROM ai_trace_index + WHERE OrgId = 'org_sql_catalog' + AND Timestamp >= '2026-01-01 10:30:00' + AND Timestamp <= '2026-01-03 14:15:00' + GROUP BY TraceId) AS trace ON ai_trace_index.TraceId = trace.TraceId + WHERE ai_trace_index.OrgId = 'org_sql_catalog' + AND ai_trace_index.Timestamp >= '2026-01-01 10:30:00' + AND ai_trace_index.Timestamp <= '2026-01-03 14:15:00' + GROUP BY sessionId + HAVING sum(ai_trace_index.IsError) > 0) + GROUP BY rankKey + ORDER BY rankSessions DESC, rankKey ASC + LIMIT 12) AS top_keys) + GROUP BY key +UNION ALL +SELECT + 'keys' AS period, + '' AS key, + uniqExact(toString(ai_trace_index.ServiceName)) AS keyCount, + 0 AS sessions, + 0 AS erroredSessions, + 0 AS llmCalls, + 0 AS erroredLlmCalls, + 0 AS toolCalls, + 0 AS erroredToolCalls, + 0 AS cost, + 0 AS pricedLlmCalls, + 0 AS tokens, + 0 AS inputTokens, + 0 AS cacheReadTokens, + 0 AS cacheWriteTokens, + 0 AS outputTokens, + 0 AS reasoningTokens, + 0 AS sessionDurationP50Ns, + 0 AS sessionDurationP95Ns, + 0 AS llmDurationP50Ns, + 0 AS llmDurationP95Ns + FROM ai_trace_index + INNER JOIN (SELECT + TraceId AS TraceId, + max(SessionId) AS rawSessionId + FROM ai_trace_index + WHERE OrgId = 'org_sql_catalog' + AND Timestamp >= '2026-01-01 10:30:00' + AND Timestamp <= '2026-01-03 14:15:00' + GROUP BY TraceId) AS trace ON ai_trace_index.TraceId = trace.TraceId + WHERE ai_trace_index.OrgId = 'org_sql_catalog' + AND ai_trace_index.Timestamp >= '2026-01-01 10:30:00' + AND ai_trace_index.Timestamp <= '2026-01-03 14:15:00' + AND if(trace.rawSessionId = '', concat('trace:', ai_trace_index.TraceId), trace.rawSessionId) IN (SELECT + if(trace.rawSessionId = '', concat('trace:', ai_trace_index.TraceId), trace.rawSessionId) AS sessionId + FROM ai_trace_index + INNER JOIN (SELECT + TraceId AS TraceId, + max(SessionId) AS rawSessionId + FROM ai_trace_index + WHERE OrgId = 'org_sql_catalog' + AND Timestamp >= '2026-01-01 10:30:00' + AND Timestamp <= '2026-01-03 14:15:00' + GROUP BY TraceId) AS trace ON ai_trace_index.TraceId = trace.TraceId + WHERE ai_trace_index.OrgId = 'org_sql_catalog' + AND ai_trace_index.Timestamp >= '2026-01-01 10:30:00' + AND ai_trace_index.Timestamp <= '2026-01-03 14:15:00' + GROUP BY sessionId + HAVING sum(ai_trace_index.IsError) > 0) +FORMAT JSON + +-- builder:ai-overview:aiOverviewBreakdownQuery:tool +SELECT + 'current' AS period, + key AS key, + 0 AS keyCount, + count() AS sessions, + countIf(errorSpans > 0) AS erroredSessions, + sum(toFloat64(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 2)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, toFloat64(n.2)), arrayFilter(n -> n.1 != '', netted))))))) AS llmCalls, + sum(erroredLlmCalls) AS erroredLlmCalls, + sum(toolCalls) AS toolCalls, + sum(erroredToolCalls) AS erroredToolCalls, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 4)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.4), arrayFilter(n -> n.1 != '', netted)))))) AS cost, + sum(arraySum(arrayMap(n -> toFloat64(n.2 AND n.4 > 0), arrayFilter(n -> n.1 = '', netted))) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, toFloat64(n.2 AND n.4 > 0)), arrayFilter(n -> n.1 != '', netted)))))) AS pricedLlmCalls, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 3)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.3), arrayFilter(n -> n.1 != '', netted)))))) AS tokens, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 5)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.5), arrayFilter(n -> n.1 != '', netted)))))) AS inputTokens, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 6)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.6), arrayFilter(n -> n.1 != '', netted)))))) AS cacheReadTokens, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 7)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.7), arrayFilter(n -> n.1 != '', netted)))))) AS cacheWriteTokens, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 8)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.8), arrayFilter(n -> n.1 != '', netted)))))) AS outputTokens, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 9)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.9), arrayFilter(n -> n.1 != '', netted)))))) AS reasoningTokens, + ifNull(ifNotFinite(quantile(0.5)(sessionDurationNs), 0), 0) AS sessionDurationP50Ns, + ifNull(ifNotFinite(quantile(0.95)(sessionDurationNs), 0), 0) AS sessionDurationP95Ns, + ifNull(ifNotFinite(quantileArray(0.5)(llmDurations), 0), 0) AS llmDurationP50Ns, + ifNull(ifNotFinite(quantileArray(0.95)(llmDurations), 0), 0) AS llmDurationP95Ns + FROM (SELECT + key AS key, + sessionStart AS sessionStart, + sessionDurationNs AS sessionDurationNs, + errorSpans AS errorSpans, + toolCalls AS toolCalls, + erroredToolCalls AS erroredToolCalls, + erroredLlmCalls AS erroredLlmCalls, + llmDurations AS llmDurations, + arrayMap(r -> tuple(r.5, r.6 = 1 AND if((r.3 > 0 OR r.4 > 0), greatest(0., r.3 - arrayElement(tupleElement(childClaims, 2), indexOf(tupleElement(childClaims, 1), r.1))) > 0 OR greatest(0., r.4 - arrayElement(tupleElement(childClaims, 3), indexOf(tupleElement(childClaims, 1), r.1))) > 0, NOT has(reportingIds, r.2)), greatest(0., r.3 - arrayElement(tupleElement(childClaims, 2), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.4 - arrayElement(tupleElement(childClaims, 3), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.7 - arrayElement(tupleElement(childClaims, 4), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.8 - arrayElement(tupleElement(childClaims, 5), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.9 - arrayElement(tupleElement(childClaims, 6), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.10 - arrayElement(tupleElement(childClaims, 7), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.11 - arrayElement(tupleElement(childClaims, 8), indexOf(tupleElement(childClaims, 1), r.1)))), reporters) AS netted + FROM (SELECT + if(trace.rawSessionId = '', concat('trace:', ai_trace_index.TraceId), trace.rawSessionId) AS sessionId, + toString(ai_trace_index.ToolName) AS key, + min(ai_trace_index.Timestamp) AS sessionStart, + max(toUnixTimestamp64Nano(ai_trace_index.Timestamp) + toInt64(ai_trace_index.Duration)) - toUnixTimestamp64Nano(min(ai_trace_index.Timestamp)) AS sessionDurationNs, + sum(ai_trace_index.IsError) AS errorSpans, + sum(ai_trace_index.IsToolCall) AS toolCalls, + sumIf(ai_trace_index.IsError, ai_trace_index.IsToolCall = 1) AS erroredToolCalls, + sumIf(ai_trace_index.IsError, ai_trace_index.IsLlmCall = 1) AS erroredLlmCalls, + groupArrayIf(2000)(ai_trace_index.Duration, ai_trace_index.IsLlmCall = 1) AS llmDurations, + groupArrayIf(2000)(tuple(ai_trace_index.SpanId, ai_trace_index.ParentSpanId, ai_trace_index.Tokens, ai_trace_index.Cost, ai_trace_index.ResponseId, ai_trace_index.IsLlmCall, ai_trace_index.InputTokens, ai_trace_index.CacheReadTokens, ai_trace_index.CacheWriteTokens, ai_trace_index.OutputTokens, ai_trace_index.ReasoningTokens), ((ai_trace_index.Tokens > 0 OR ai_trace_index.Cost > 0) OR ai_trace_index.IsLlmCall = 1)) AS reporters, + arrayReduce('sumMap', arrayMap(c -> [c.2], reporters), arrayMap(c -> [c.3], reporters), arrayMap(c -> [c.4], reporters), arrayMap(c -> [c.7], reporters), arrayMap(c -> [c.8], reporters), arrayMap(c -> [c.9], reporters), arrayMap(c -> [c.10], reporters), arrayMap(c -> [c.11], reporters)) AS childClaims, + tupleElement(arrayFilter(p -> p.3 > 0 OR p.4 > 0, reporters), 1) AS reportingIds + FROM ai_trace_index + INNER JOIN (SELECT + TraceId AS TraceId, + max(SessionId) AS rawSessionId + FROM ai_trace_index + WHERE OrgId = 'org_sql_catalog' + AND Timestamp >= '2026-01-01 10:30:00' + AND Timestamp <= '2026-01-03 14:15:00' + GROUP BY TraceId) AS trace ON ai_trace_index.TraceId = trace.TraceId + WHERE ai_trace_index.OrgId = 'org_sql_catalog' + AND ai_trace_index.Timestamp >= '2026-01-01 10:30:00' + AND ai_trace_index.Timestamp <= '2026-01-03 14:15:00' + AND ai_trace_index.IsToolCall = 1 + GROUP BY sessionId, key) AS session_rows) AS netted_current + WHERE key IN (SELECT + rankKey AS topKey + FROM (SELECT + toString(ai_trace_index.ToolName) AS rankKey, + uniqExact(if(trace.rawSessionId = '', concat('trace:', ai_trace_index.TraceId), trace.rawSessionId)) AS rankSessions + FROM ai_trace_index + INNER JOIN (SELECT + TraceId AS TraceId, + max(SessionId) AS rawSessionId + FROM ai_trace_index + WHERE OrgId = 'org_sql_catalog' + AND Timestamp >= '2026-01-01 10:30:00' + AND Timestamp <= '2026-01-03 14:15:00' + GROUP BY TraceId) AS trace ON ai_trace_index.TraceId = trace.TraceId + WHERE ai_trace_index.OrgId = 'org_sql_catalog' + AND ai_trace_index.Timestamp >= '2026-01-01 10:30:00' + AND ai_trace_index.Timestamp <= '2026-01-03 14:15:00' + AND ai_trace_index.IsToolCall = 1 + GROUP BY rankKey + ORDER BY rankSessions DESC, rankKey ASC + LIMIT 5) AS top_keys) + GROUP BY key +UNION ALL +SELECT + 'previous' AS period, + key AS key, + 0 AS keyCount, + count() AS sessions, + countIf(errorSpans > 0) AS erroredSessions, + sum(toFloat64(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 2)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, toFloat64(n.2)), arrayFilter(n -> n.1 != '', netted))))))) AS llmCalls, + sum(erroredLlmCalls) AS erroredLlmCalls, + sum(toolCalls) AS toolCalls, + sum(erroredToolCalls) AS erroredToolCalls, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 4)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.4), arrayFilter(n -> n.1 != '', netted)))))) AS cost, + sum(arraySum(arrayMap(n -> toFloat64(n.2 AND n.4 > 0), arrayFilter(n -> n.1 = '', netted))) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, toFloat64(n.2 AND n.4 > 0)), arrayFilter(n -> n.1 != '', netted)))))) AS pricedLlmCalls, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 3)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.3), arrayFilter(n -> n.1 != '', netted)))))) AS tokens, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 5)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.5), arrayFilter(n -> n.1 != '', netted)))))) AS inputTokens, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 6)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.6), arrayFilter(n -> n.1 != '', netted)))))) AS cacheReadTokens, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 7)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.7), arrayFilter(n -> n.1 != '', netted)))))) AS cacheWriteTokens, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 8)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.8), arrayFilter(n -> n.1 != '', netted)))))) AS outputTokens, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 9)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.9), arrayFilter(n -> n.1 != '', netted)))))) AS reasoningTokens, + ifNull(ifNotFinite(quantile(0.5)(sessionDurationNs), 0), 0) AS sessionDurationP50Ns, + ifNull(ifNotFinite(quantile(0.95)(sessionDurationNs), 0), 0) AS sessionDurationP95Ns, + ifNull(ifNotFinite(quantileArray(0.5)(llmDurations), 0), 0) AS llmDurationP50Ns, + ifNull(ifNotFinite(quantileArray(0.95)(llmDurations), 0), 0) AS llmDurationP95Ns + FROM (SELECT + key AS key, + sessionStart AS sessionStart, + sessionDurationNs AS sessionDurationNs, + errorSpans AS errorSpans, + toolCalls AS toolCalls, + erroredToolCalls AS erroredToolCalls, + erroredLlmCalls AS erroredLlmCalls, + llmDurations AS llmDurations, + arrayMap(r -> tuple(r.5, r.6 = 1 AND if((r.3 > 0 OR r.4 > 0), greatest(0., r.3 - arrayElement(tupleElement(childClaims, 2), indexOf(tupleElement(childClaims, 1), r.1))) > 0 OR greatest(0., r.4 - arrayElement(tupleElement(childClaims, 3), indexOf(tupleElement(childClaims, 1), r.1))) > 0, NOT has(reportingIds, r.2)), greatest(0., r.3 - arrayElement(tupleElement(childClaims, 2), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.4 - arrayElement(tupleElement(childClaims, 3), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.7 - arrayElement(tupleElement(childClaims, 4), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.8 - arrayElement(tupleElement(childClaims, 5), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.9 - arrayElement(tupleElement(childClaims, 6), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.10 - arrayElement(tupleElement(childClaims, 7), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.11 - arrayElement(tupleElement(childClaims, 8), indexOf(tupleElement(childClaims, 1), r.1)))), reporters) AS netted + FROM (SELECT + if(trace.rawSessionId = '', concat('trace:', ai_trace_index.TraceId), trace.rawSessionId) AS sessionId, + toString(ai_trace_index.ToolName) AS key, + min(ai_trace_index.Timestamp) AS sessionStart, + max(toUnixTimestamp64Nano(ai_trace_index.Timestamp) + toInt64(ai_trace_index.Duration)) - toUnixTimestamp64Nano(min(ai_trace_index.Timestamp)) AS sessionDurationNs, + sum(ai_trace_index.IsError) AS errorSpans, + sum(ai_trace_index.IsToolCall) AS toolCalls, + sumIf(ai_trace_index.IsError, ai_trace_index.IsToolCall = 1) AS erroredToolCalls, + sumIf(ai_trace_index.IsError, ai_trace_index.IsLlmCall = 1) AS erroredLlmCalls, + groupArrayIf(2000)(ai_trace_index.Duration, ai_trace_index.IsLlmCall = 1) AS llmDurations, + groupArrayIf(2000)(tuple(ai_trace_index.SpanId, ai_trace_index.ParentSpanId, ai_trace_index.Tokens, ai_trace_index.Cost, ai_trace_index.ResponseId, ai_trace_index.IsLlmCall, ai_trace_index.InputTokens, ai_trace_index.CacheReadTokens, ai_trace_index.CacheWriteTokens, ai_trace_index.OutputTokens, ai_trace_index.ReasoningTokens), ((ai_trace_index.Tokens > 0 OR ai_trace_index.Cost > 0) OR ai_trace_index.IsLlmCall = 1)) AS reporters, + arrayReduce('sumMap', arrayMap(c -> [c.2], reporters), arrayMap(c -> [c.3], reporters), arrayMap(c -> [c.4], reporters), arrayMap(c -> [c.7], reporters), arrayMap(c -> [c.8], reporters), arrayMap(c -> [c.9], reporters), arrayMap(c -> [c.10], reporters), arrayMap(c -> [c.11], reporters)) AS childClaims, + tupleElement(arrayFilter(p -> p.3 > 0 OR p.4 > 0, reporters), 1) AS reportingIds + FROM ai_trace_index + INNER JOIN (SELECT + TraceId AS TraceId, + max(SessionId) AS rawSessionId + FROM ai_trace_index + WHERE OrgId = 'org_sql_catalog' + AND Timestamp >= '2025-12-30 06:45:00' + AND Timestamp <= '2026-01-01 10:30:00' + GROUP BY TraceId) AS trace ON ai_trace_index.TraceId = trace.TraceId + WHERE ai_trace_index.OrgId = 'org_sql_catalog' + AND ai_trace_index.Timestamp >= '2025-12-30 06:45:00' + AND ai_trace_index.Timestamp <= '2026-01-01 10:30:00' + AND ai_trace_index.IsToolCall = 1 + GROUP BY sessionId, key) AS session_rows) AS netted_previous + WHERE key IN (SELECT + rankKey AS topKey + FROM (SELECT + toString(ai_trace_index.ToolName) AS rankKey, + uniqExact(if(trace.rawSessionId = '', concat('trace:', ai_trace_index.TraceId), trace.rawSessionId)) AS rankSessions + FROM ai_trace_index + INNER JOIN (SELECT + TraceId AS TraceId, + max(SessionId) AS rawSessionId + FROM ai_trace_index + WHERE OrgId = 'org_sql_catalog' + AND Timestamp >= '2026-01-01 10:30:00' + AND Timestamp <= '2026-01-03 14:15:00' + GROUP BY TraceId) AS trace ON ai_trace_index.TraceId = trace.TraceId + WHERE ai_trace_index.OrgId = 'org_sql_catalog' + AND ai_trace_index.Timestamp >= '2026-01-01 10:30:00' + AND ai_trace_index.Timestamp <= '2026-01-03 14:15:00' + AND ai_trace_index.IsToolCall = 1 + GROUP BY rankKey + ORDER BY rankSessions DESC, rankKey ASC + LIMIT 5) AS top_keys) + GROUP BY key +UNION ALL +SELECT + 'keys' AS period, + '' AS key, + uniqExact(toString(ai_trace_index.ToolName)) AS keyCount, + 0 AS sessions, + 0 AS erroredSessions, + 0 AS llmCalls, + 0 AS erroredLlmCalls, + 0 AS toolCalls, + 0 AS erroredToolCalls, + 0 AS cost, + 0 AS pricedLlmCalls, + 0 AS tokens, + 0 AS inputTokens, + 0 AS cacheReadTokens, + 0 AS cacheWriteTokens, + 0 AS outputTokens, + 0 AS reasoningTokens, + 0 AS sessionDurationP50Ns, + 0 AS sessionDurationP95Ns, + 0 AS llmDurationP50Ns, + 0 AS llmDurationP95Ns + FROM ai_trace_index + INNER JOIN (SELECT + TraceId AS TraceId, + max(SessionId) AS rawSessionId + FROM ai_trace_index + WHERE OrgId = 'org_sql_catalog' + AND Timestamp >= '2026-01-01 10:30:00' + AND Timestamp <= '2026-01-03 14:15:00' + GROUP BY TraceId) AS trace ON ai_trace_index.TraceId = trace.TraceId + WHERE ai_trace_index.OrgId = 'org_sql_catalog' + AND ai_trace_index.Timestamp >= '2026-01-01 10:30:00' + AND ai_trace_index.Timestamp <= '2026-01-03 14:15:00' + AND ai_trace_index.IsToolCall = 1 +FORMAT JSON + +-- builder:ai-overview:aiOverviewSeriesQuery:default +SELECT * FROM ( +SELECT + 'current' AS period, + formatDateTime(toStartOfInterval(sessionStart, INTERVAL 300 SECOND), '%Y-%m-%dT%H:%i:%S.%fZ') AS bucket, + count() AS sessions, + countIf(errorSpans > 0) AS erroredSessions, + sum(toFloat64(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 2)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, toFloat64(n.2)), arrayFilter(n -> n.1 != '', netted))))))) AS llmCalls, + sum(erroredLlmCalls) AS erroredLlmCalls, + sum(toolCalls) AS toolCalls, + sum(erroredToolCalls) AS erroredToolCalls, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 4)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.4), arrayFilter(n -> n.1 != '', netted)))))) AS cost, + sum(arraySum(arrayMap(n -> toFloat64(n.2 AND n.4 > 0), arrayFilter(n -> n.1 = '', netted))) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, toFloat64(n.2 AND n.4 > 0)), arrayFilter(n -> n.1 != '', netted)))))) AS pricedLlmCalls, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 3)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.3), arrayFilter(n -> n.1 != '', netted)))))) AS tokens, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 5)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.5), arrayFilter(n -> n.1 != '', netted)))))) AS inputTokens, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 6)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.6), arrayFilter(n -> n.1 != '', netted)))))) AS cacheReadTokens, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 7)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.7), arrayFilter(n -> n.1 != '', netted)))))) AS cacheWriteTokens, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 8)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.8), arrayFilter(n -> n.1 != '', netted)))))) AS outputTokens, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 9)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.9), arrayFilter(n -> n.1 != '', netted)))))) AS reasoningTokens, + ifNull(ifNotFinite(quantile(0.5)(sessionDurationNs), 0), 0) AS sessionDurationP50Ns, + ifNull(ifNotFinite(quantile(0.95)(sessionDurationNs), 0), 0) AS sessionDurationP95Ns, + ifNull(ifNotFinite(quantileArray(0.5)(llmDurations), 0), 0) AS llmDurationP50Ns, + ifNull(ifNotFinite(quantileArray(0.95)(llmDurations), 0), 0) AS llmDurationP95Ns + FROM (SELECT + key AS key, + sessionStart AS sessionStart, + sessionDurationNs AS sessionDurationNs, + errorSpans AS errorSpans, + toolCalls AS toolCalls, + erroredToolCalls AS erroredToolCalls, + erroredLlmCalls AS erroredLlmCalls, + llmDurations AS llmDurations, + arrayMap(r -> tuple(r.5, r.6 = 1 AND if((r.3 > 0 OR r.4 > 0), greatest(0., r.3 - arrayElement(tupleElement(childClaims, 2), indexOf(tupleElement(childClaims, 1), r.1))) > 0 OR greatest(0., r.4 - arrayElement(tupleElement(childClaims, 3), indexOf(tupleElement(childClaims, 1), r.1))) > 0, NOT has(reportingIds, r.2)), greatest(0., r.3 - arrayElement(tupleElement(childClaims, 2), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.4 - arrayElement(tupleElement(childClaims, 3), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.7 - arrayElement(tupleElement(childClaims, 4), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.8 - arrayElement(tupleElement(childClaims, 5), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.9 - arrayElement(tupleElement(childClaims, 6), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.10 - arrayElement(tupleElement(childClaims, 7), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.11 - arrayElement(tupleElement(childClaims, 8), indexOf(tupleElement(childClaims, 1), r.1)))), reporters) AS netted + FROM (SELECT + if(trace.rawSessionId = '', concat('trace:', ai_trace_index.TraceId), trace.rawSessionId) AS sessionId, + '' AS key, + min(ai_trace_index.Timestamp) AS sessionStart, + max(toUnixTimestamp64Nano(ai_trace_index.Timestamp) + toInt64(ai_trace_index.Duration)) - toUnixTimestamp64Nano(min(ai_trace_index.Timestamp)) AS sessionDurationNs, + sum(ai_trace_index.IsError) AS errorSpans, + sum(ai_trace_index.IsToolCall) AS toolCalls, + sumIf(ai_trace_index.IsError, ai_trace_index.IsToolCall = 1) AS erroredToolCalls, + sumIf(ai_trace_index.IsError, ai_trace_index.IsLlmCall = 1) AS erroredLlmCalls, + groupArrayIf(2000)(ai_trace_index.Duration, ai_trace_index.IsLlmCall = 1) AS llmDurations, + groupArrayIf(2000)(tuple(ai_trace_index.SpanId, ai_trace_index.ParentSpanId, ai_trace_index.Tokens, ai_trace_index.Cost, ai_trace_index.ResponseId, ai_trace_index.IsLlmCall, ai_trace_index.InputTokens, ai_trace_index.CacheReadTokens, ai_trace_index.CacheWriteTokens, ai_trace_index.OutputTokens, ai_trace_index.ReasoningTokens), ((ai_trace_index.Tokens > 0 OR ai_trace_index.Cost > 0) OR ai_trace_index.IsLlmCall = 1)) AS reporters, + arrayReduce('sumMap', arrayMap(c -> [c.2], reporters), arrayMap(c -> [c.3], reporters), arrayMap(c -> [c.4], reporters), arrayMap(c -> [c.7], reporters), arrayMap(c -> [c.8], reporters), arrayMap(c -> [c.9], reporters), arrayMap(c -> [c.10], reporters), arrayMap(c -> [c.11], reporters)) AS childClaims, + tupleElement(arrayFilter(p -> p.3 > 0 OR p.4 > 0, reporters), 1) AS reportingIds + FROM ai_trace_index + INNER JOIN (SELECT + TraceId AS TraceId, + max(SessionId) AS rawSessionId + FROM ai_trace_index + WHERE OrgId = 'org_sql_catalog' + AND Timestamp >= '2026-01-01 10:30:00' + AND Timestamp <= '2026-01-03 14:15:00' + GROUP BY TraceId) AS trace ON ai_trace_index.TraceId = trace.TraceId + WHERE ai_trace_index.OrgId = 'org_sql_catalog' + AND ai_trace_index.Timestamp >= '2026-01-01 10:30:00' + AND ai_trace_index.Timestamp <= '2026-01-03 14:15:00' + GROUP BY sessionId) AS session_rows) AS netted_current + GROUP BY bucket +UNION ALL +SELECT + 'previous' AS period, + formatDateTime(toStartOfInterval(sessionStart, INTERVAL 300 SECOND), '%Y-%m-%dT%H:%i:%S.%fZ') AS bucket, + count() AS sessions, + countIf(errorSpans > 0) AS erroredSessions, + sum(toFloat64(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 2)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, toFloat64(n.2)), arrayFilter(n -> n.1 != '', netted))))))) AS llmCalls, + sum(erroredLlmCalls) AS erroredLlmCalls, + sum(toolCalls) AS toolCalls, + sum(erroredToolCalls) AS erroredToolCalls, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 4)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.4), arrayFilter(n -> n.1 != '', netted)))))) AS cost, + sum(arraySum(arrayMap(n -> toFloat64(n.2 AND n.4 > 0), arrayFilter(n -> n.1 = '', netted))) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, toFloat64(n.2 AND n.4 > 0)), arrayFilter(n -> n.1 != '', netted)))))) AS pricedLlmCalls, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 3)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.3), arrayFilter(n -> n.1 != '', netted)))))) AS tokens, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 5)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.5), arrayFilter(n -> n.1 != '', netted)))))) AS inputTokens, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 6)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.6), arrayFilter(n -> n.1 != '', netted)))))) AS cacheReadTokens, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 7)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.7), arrayFilter(n -> n.1 != '', netted)))))) AS cacheWriteTokens, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 8)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.8), arrayFilter(n -> n.1 != '', netted)))))) AS outputTokens, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 9)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.9), arrayFilter(n -> n.1 != '', netted)))))) AS reasoningTokens, + ifNull(ifNotFinite(quantile(0.5)(sessionDurationNs), 0), 0) AS sessionDurationP50Ns, + ifNull(ifNotFinite(quantile(0.95)(sessionDurationNs), 0), 0) AS sessionDurationP95Ns, + ifNull(ifNotFinite(quantileArray(0.5)(llmDurations), 0), 0) AS llmDurationP50Ns, + ifNull(ifNotFinite(quantileArray(0.95)(llmDurations), 0), 0) AS llmDurationP95Ns + FROM (SELECT + key AS key, + sessionStart AS sessionStart, + sessionDurationNs AS sessionDurationNs, + errorSpans AS errorSpans, + toolCalls AS toolCalls, + erroredToolCalls AS erroredToolCalls, + erroredLlmCalls AS erroredLlmCalls, + llmDurations AS llmDurations, + arrayMap(r -> tuple(r.5, r.6 = 1 AND if((r.3 > 0 OR r.4 > 0), greatest(0., r.3 - arrayElement(tupleElement(childClaims, 2), indexOf(tupleElement(childClaims, 1), r.1))) > 0 OR greatest(0., r.4 - arrayElement(tupleElement(childClaims, 3), indexOf(tupleElement(childClaims, 1), r.1))) > 0, NOT has(reportingIds, r.2)), greatest(0., r.3 - arrayElement(tupleElement(childClaims, 2), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.4 - arrayElement(tupleElement(childClaims, 3), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.7 - arrayElement(tupleElement(childClaims, 4), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.8 - arrayElement(tupleElement(childClaims, 5), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.9 - arrayElement(tupleElement(childClaims, 6), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.10 - arrayElement(tupleElement(childClaims, 7), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.11 - arrayElement(tupleElement(childClaims, 8), indexOf(tupleElement(childClaims, 1), r.1)))), reporters) AS netted + FROM (SELECT + if(trace.rawSessionId = '', concat('trace:', ai_trace_index.TraceId), trace.rawSessionId) AS sessionId, + '' AS key, + min(ai_trace_index.Timestamp) AS sessionStart, + max(toUnixTimestamp64Nano(ai_trace_index.Timestamp) + toInt64(ai_trace_index.Duration)) - toUnixTimestamp64Nano(min(ai_trace_index.Timestamp)) AS sessionDurationNs, + sum(ai_trace_index.IsError) AS errorSpans, + sum(ai_trace_index.IsToolCall) AS toolCalls, + sumIf(ai_trace_index.IsError, ai_trace_index.IsToolCall = 1) AS erroredToolCalls, + sumIf(ai_trace_index.IsError, ai_trace_index.IsLlmCall = 1) AS erroredLlmCalls, + groupArrayIf(2000)(ai_trace_index.Duration, ai_trace_index.IsLlmCall = 1) AS llmDurations, + groupArrayIf(2000)(tuple(ai_trace_index.SpanId, ai_trace_index.ParentSpanId, ai_trace_index.Tokens, ai_trace_index.Cost, ai_trace_index.ResponseId, ai_trace_index.IsLlmCall, ai_trace_index.InputTokens, ai_trace_index.CacheReadTokens, ai_trace_index.CacheWriteTokens, ai_trace_index.OutputTokens, ai_trace_index.ReasoningTokens), ((ai_trace_index.Tokens > 0 OR ai_trace_index.Cost > 0) OR ai_trace_index.IsLlmCall = 1)) AS reporters, + arrayReduce('sumMap', arrayMap(c -> [c.2], reporters), arrayMap(c -> [c.3], reporters), arrayMap(c -> [c.4], reporters), arrayMap(c -> [c.7], reporters), arrayMap(c -> [c.8], reporters), arrayMap(c -> [c.9], reporters), arrayMap(c -> [c.10], reporters), arrayMap(c -> [c.11], reporters)) AS childClaims, + tupleElement(arrayFilter(p -> p.3 > 0 OR p.4 > 0, reporters), 1) AS reportingIds + FROM ai_trace_index + INNER JOIN (SELECT + TraceId AS TraceId, + max(SessionId) AS rawSessionId + FROM ai_trace_index + WHERE OrgId = 'org_sql_catalog' + AND Timestamp >= '2025-12-30 06:45:00' + AND Timestamp <= '2026-01-01 10:30:00' + GROUP BY TraceId) AS trace ON ai_trace_index.TraceId = trace.TraceId + WHERE ai_trace_index.OrgId = 'org_sql_catalog' + AND ai_trace_index.Timestamp >= '2025-12-30 06:45:00' + AND ai_trace_index.Timestamp <= '2026-01-01 10:30:00' + GROUP BY sessionId) AS session_rows) AS netted_previous + GROUP BY bucket +) +ORDER BY period ASC, bucket ASC +FORMAT JSON + +-- builder:ai-overview:aiOverviewTotalsQuery:default +SELECT + 'current' AS period, + count() AS sessions, + countIf(errorSpans > 0) AS erroredSessions, + sum(toFloat64(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 2)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, toFloat64(n.2)), arrayFilter(n -> n.1 != '', netted))))))) AS llmCalls, + sum(erroredLlmCalls) AS erroredLlmCalls, + sum(toolCalls) AS toolCalls, + sum(erroredToolCalls) AS erroredToolCalls, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 4)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.4), arrayFilter(n -> n.1 != '', netted)))))) AS cost, + sum(arraySum(arrayMap(n -> toFloat64(n.2 AND n.4 > 0), arrayFilter(n -> n.1 = '', netted))) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, toFloat64(n.2 AND n.4 > 0)), arrayFilter(n -> n.1 != '', netted)))))) AS pricedLlmCalls, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 3)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.3), arrayFilter(n -> n.1 != '', netted)))))) AS tokens, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 5)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.5), arrayFilter(n -> n.1 != '', netted)))))) AS inputTokens, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 6)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.6), arrayFilter(n -> n.1 != '', netted)))))) AS cacheReadTokens, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 7)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.7), arrayFilter(n -> n.1 != '', netted)))))) AS cacheWriteTokens, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 8)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.8), arrayFilter(n -> n.1 != '', netted)))))) AS outputTokens, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 9)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.9), arrayFilter(n -> n.1 != '', netted)))))) AS reasoningTokens, + ifNull(ifNotFinite(quantile(0.5)(sessionDurationNs), 0), 0) AS sessionDurationP50Ns, + ifNull(ifNotFinite(quantile(0.95)(sessionDurationNs), 0), 0) AS sessionDurationP95Ns, + ifNull(ifNotFinite(quantileArray(0.5)(llmDurations), 0), 0) AS llmDurationP50Ns, + ifNull(ifNotFinite(quantileArray(0.95)(llmDurations), 0), 0) AS llmDurationP95Ns + FROM (SELECT + key AS key, + sessionStart AS sessionStart, + sessionDurationNs AS sessionDurationNs, + errorSpans AS errorSpans, + toolCalls AS toolCalls, + erroredToolCalls AS erroredToolCalls, + erroredLlmCalls AS erroredLlmCalls, + llmDurations AS llmDurations, + arrayMap(r -> tuple(r.5, r.6 = 1 AND if((r.3 > 0 OR r.4 > 0), greatest(0., r.3 - arrayElement(tupleElement(childClaims, 2), indexOf(tupleElement(childClaims, 1), r.1))) > 0 OR greatest(0., r.4 - arrayElement(tupleElement(childClaims, 3), indexOf(tupleElement(childClaims, 1), r.1))) > 0, NOT has(reportingIds, r.2)), greatest(0., r.3 - arrayElement(tupleElement(childClaims, 2), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.4 - arrayElement(tupleElement(childClaims, 3), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.7 - arrayElement(tupleElement(childClaims, 4), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.8 - arrayElement(tupleElement(childClaims, 5), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.9 - arrayElement(tupleElement(childClaims, 6), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.10 - arrayElement(tupleElement(childClaims, 7), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.11 - arrayElement(tupleElement(childClaims, 8), indexOf(tupleElement(childClaims, 1), r.1)))), reporters) AS netted + FROM (SELECT + if(trace.rawSessionId = '', concat('trace:', ai_trace_index.TraceId), trace.rawSessionId) AS sessionId, + '' AS key, + min(ai_trace_index.Timestamp) AS sessionStart, + max(toUnixTimestamp64Nano(ai_trace_index.Timestamp) + toInt64(ai_trace_index.Duration)) - toUnixTimestamp64Nano(min(ai_trace_index.Timestamp)) AS sessionDurationNs, + sum(ai_trace_index.IsError) AS errorSpans, + sum(ai_trace_index.IsToolCall) AS toolCalls, + sumIf(ai_trace_index.IsError, ai_trace_index.IsToolCall = 1) AS erroredToolCalls, + sumIf(ai_trace_index.IsError, ai_trace_index.IsLlmCall = 1) AS erroredLlmCalls, + groupArrayIf(2000)(ai_trace_index.Duration, ai_trace_index.IsLlmCall = 1) AS llmDurations, + groupArrayIf(2000)(tuple(ai_trace_index.SpanId, ai_trace_index.ParentSpanId, ai_trace_index.Tokens, ai_trace_index.Cost, ai_trace_index.ResponseId, ai_trace_index.IsLlmCall, ai_trace_index.InputTokens, ai_trace_index.CacheReadTokens, ai_trace_index.CacheWriteTokens, ai_trace_index.OutputTokens, ai_trace_index.ReasoningTokens), ((ai_trace_index.Tokens > 0 OR ai_trace_index.Cost > 0) OR ai_trace_index.IsLlmCall = 1)) AS reporters, + arrayReduce('sumMap', arrayMap(c -> [c.2], reporters), arrayMap(c -> [c.3], reporters), arrayMap(c -> [c.4], reporters), arrayMap(c -> [c.7], reporters), arrayMap(c -> [c.8], reporters), arrayMap(c -> [c.9], reporters), arrayMap(c -> [c.10], reporters), arrayMap(c -> [c.11], reporters)) AS childClaims, + tupleElement(arrayFilter(p -> p.3 > 0 OR p.4 > 0, reporters), 1) AS reportingIds + FROM ai_trace_index + INNER JOIN (SELECT + TraceId AS TraceId, + max(SessionId) AS rawSessionId + FROM ai_trace_index + WHERE OrgId = 'org_sql_catalog' + AND Timestamp >= '2026-01-01 10:30:00' + AND Timestamp <= '2026-01-03 14:15:00' + GROUP BY TraceId) AS trace ON ai_trace_index.TraceId = trace.TraceId + WHERE ai_trace_index.OrgId = 'org_sql_catalog' + AND ai_trace_index.Timestamp >= '2026-01-01 10:30:00' + AND ai_trace_index.Timestamp <= '2026-01-03 14:15:00' + GROUP BY sessionId) AS session_rows) AS netted_current +UNION ALL +SELECT + 'previous' AS period, + count() AS sessions, + countIf(errorSpans > 0) AS erroredSessions, + sum(toFloat64(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 2)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, toFloat64(n.2)), arrayFilter(n -> n.1 != '', netted))))))) AS llmCalls, + sum(erroredLlmCalls) AS erroredLlmCalls, + sum(toolCalls) AS toolCalls, + sum(erroredToolCalls) AS erroredToolCalls, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 4)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.4), arrayFilter(n -> n.1 != '', netted)))))) AS cost, + sum(arraySum(arrayMap(n -> toFloat64(n.2 AND n.4 > 0), arrayFilter(n -> n.1 = '', netted))) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, toFloat64(n.2 AND n.4 > 0)), arrayFilter(n -> n.1 != '', netted)))))) AS pricedLlmCalls, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 3)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.3), arrayFilter(n -> n.1 != '', netted)))))) AS tokens, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 5)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.5), arrayFilter(n -> n.1 != '', netted)))))) AS inputTokens, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 6)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.6), arrayFilter(n -> n.1 != '', netted)))))) AS cacheReadTokens, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 7)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.7), arrayFilter(n -> n.1 != '', netted)))))) AS cacheWriteTokens, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 8)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.8), arrayFilter(n -> n.1 != '', netted)))))) AS outputTokens, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 9)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.9), arrayFilter(n -> n.1 != '', netted)))))) AS reasoningTokens, + ifNull(ifNotFinite(quantile(0.5)(sessionDurationNs), 0), 0) AS sessionDurationP50Ns, + ifNull(ifNotFinite(quantile(0.95)(sessionDurationNs), 0), 0) AS sessionDurationP95Ns, + ifNull(ifNotFinite(quantileArray(0.5)(llmDurations), 0), 0) AS llmDurationP50Ns, + ifNull(ifNotFinite(quantileArray(0.95)(llmDurations), 0), 0) AS llmDurationP95Ns + FROM (SELECT + key AS key, + sessionStart AS sessionStart, + sessionDurationNs AS sessionDurationNs, + errorSpans AS errorSpans, + toolCalls AS toolCalls, + erroredToolCalls AS erroredToolCalls, + erroredLlmCalls AS erroredLlmCalls, + llmDurations AS llmDurations, + arrayMap(r -> tuple(r.5, r.6 = 1 AND if((r.3 > 0 OR r.4 > 0), greatest(0., r.3 - arrayElement(tupleElement(childClaims, 2), indexOf(tupleElement(childClaims, 1), r.1))) > 0 OR greatest(0., r.4 - arrayElement(tupleElement(childClaims, 3), indexOf(tupleElement(childClaims, 1), r.1))) > 0, NOT has(reportingIds, r.2)), greatest(0., r.3 - arrayElement(tupleElement(childClaims, 2), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.4 - arrayElement(tupleElement(childClaims, 3), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.7 - arrayElement(tupleElement(childClaims, 4), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.8 - arrayElement(tupleElement(childClaims, 5), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.9 - arrayElement(tupleElement(childClaims, 6), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.10 - arrayElement(tupleElement(childClaims, 7), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.11 - arrayElement(tupleElement(childClaims, 8), indexOf(tupleElement(childClaims, 1), r.1)))), reporters) AS netted + FROM (SELECT + if(trace.rawSessionId = '', concat('trace:', ai_trace_index.TraceId), trace.rawSessionId) AS sessionId, + '' AS key, + min(ai_trace_index.Timestamp) AS sessionStart, + max(toUnixTimestamp64Nano(ai_trace_index.Timestamp) + toInt64(ai_trace_index.Duration)) - toUnixTimestamp64Nano(min(ai_trace_index.Timestamp)) AS sessionDurationNs, + sum(ai_trace_index.IsError) AS errorSpans, + sum(ai_trace_index.IsToolCall) AS toolCalls, + sumIf(ai_trace_index.IsError, ai_trace_index.IsToolCall = 1) AS erroredToolCalls, + sumIf(ai_trace_index.IsError, ai_trace_index.IsLlmCall = 1) AS erroredLlmCalls, + groupArrayIf(2000)(ai_trace_index.Duration, ai_trace_index.IsLlmCall = 1) AS llmDurations, + groupArrayIf(2000)(tuple(ai_trace_index.SpanId, ai_trace_index.ParentSpanId, ai_trace_index.Tokens, ai_trace_index.Cost, ai_trace_index.ResponseId, ai_trace_index.IsLlmCall, ai_trace_index.InputTokens, ai_trace_index.CacheReadTokens, ai_trace_index.CacheWriteTokens, ai_trace_index.OutputTokens, ai_trace_index.ReasoningTokens), ((ai_trace_index.Tokens > 0 OR ai_trace_index.Cost > 0) OR ai_trace_index.IsLlmCall = 1)) AS reporters, + arrayReduce('sumMap', arrayMap(c -> [c.2], reporters), arrayMap(c -> [c.3], reporters), arrayMap(c -> [c.4], reporters), arrayMap(c -> [c.7], reporters), arrayMap(c -> [c.8], reporters), arrayMap(c -> [c.9], reporters), arrayMap(c -> [c.10], reporters), arrayMap(c -> [c.11], reporters)) AS childClaims, + tupleElement(arrayFilter(p -> p.3 > 0 OR p.4 > 0, reporters), 1) AS reportingIds + FROM ai_trace_index + INNER JOIN (SELECT + TraceId AS TraceId, + max(SessionId) AS rawSessionId + FROM ai_trace_index + WHERE OrgId = 'org_sql_catalog' + AND Timestamp >= '2025-12-30 06:45:00' + AND Timestamp <= '2026-01-01 10:30:00' + GROUP BY TraceId) AS trace ON ai_trace_index.TraceId = trace.TraceId + WHERE ai_trace_index.OrgId = 'org_sql_catalog' + AND ai_trace_index.Timestamp >= '2025-12-30 06:45:00' + AND ai_trace_index.Timestamp <= '2026-01-01 10:30:00' + GROUP BY sessionId) AS session_rows) AS netted_previous +FORMAT JSON + +-- builder:ai-overview:aiOverviewTotalsQuery:every-filter +SELECT + 'current' AS period, + count() AS sessions, + countIf(errorSpans > 0) AS erroredSessions, + sum(toFloat64(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 2)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, toFloat64(n.2)), arrayFilter(n -> n.1 != '', netted))))))) AS llmCalls, + sum(erroredLlmCalls) AS erroredLlmCalls, + sum(toolCalls) AS toolCalls, + sum(erroredToolCalls) AS erroredToolCalls, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 4)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.4), arrayFilter(n -> n.1 != '', netted)))))) AS cost, + sum(arraySum(arrayMap(n -> toFloat64(n.2 AND n.4 > 0), arrayFilter(n -> n.1 = '', netted))) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, toFloat64(n.2 AND n.4 > 0)), arrayFilter(n -> n.1 != '', netted)))))) AS pricedLlmCalls, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 3)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.3), arrayFilter(n -> n.1 != '', netted)))))) AS tokens, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 5)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.5), arrayFilter(n -> n.1 != '', netted)))))) AS inputTokens, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 6)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.6), arrayFilter(n -> n.1 != '', netted)))))) AS cacheReadTokens, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 7)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.7), arrayFilter(n -> n.1 != '', netted)))))) AS cacheWriteTokens, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 8)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.8), arrayFilter(n -> n.1 != '', netted)))))) AS outputTokens, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 9)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.9), arrayFilter(n -> n.1 != '', netted)))))) AS reasoningTokens, + ifNull(ifNotFinite(quantile(0.5)(sessionDurationNs), 0), 0) AS sessionDurationP50Ns, + ifNull(ifNotFinite(quantile(0.95)(sessionDurationNs), 0), 0) AS sessionDurationP95Ns, + ifNull(ifNotFinite(quantileArray(0.5)(llmDurations), 0), 0) AS llmDurationP50Ns, + ifNull(ifNotFinite(quantileArray(0.95)(llmDurations), 0), 0) AS llmDurationP95Ns + FROM (SELECT + key AS key, + sessionStart AS sessionStart, + sessionDurationNs AS sessionDurationNs, + errorSpans AS errorSpans, + toolCalls AS toolCalls, + erroredToolCalls AS erroredToolCalls, + erroredLlmCalls AS erroredLlmCalls, + llmDurations AS llmDurations, + arrayMap(r -> tuple(r.5, r.6 = 1 AND if((r.3 > 0 OR r.4 > 0), greatest(0., r.3 - arrayElement(tupleElement(childClaims, 2), indexOf(tupleElement(childClaims, 1), r.1))) > 0 OR greatest(0., r.4 - arrayElement(tupleElement(childClaims, 3), indexOf(tupleElement(childClaims, 1), r.1))) > 0, NOT has(reportingIds, r.2)), greatest(0., r.3 - arrayElement(tupleElement(childClaims, 2), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.4 - arrayElement(tupleElement(childClaims, 3), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.7 - arrayElement(tupleElement(childClaims, 4), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.8 - arrayElement(tupleElement(childClaims, 5), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.9 - arrayElement(tupleElement(childClaims, 6), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.10 - arrayElement(tupleElement(childClaims, 7), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.11 - arrayElement(tupleElement(childClaims, 8), indexOf(tupleElement(childClaims, 1), r.1)))), reporters) AS netted + FROM (SELECT + if(trace.rawSessionId = '', concat('trace:', ai_trace_index.TraceId), trace.rawSessionId) AS sessionId, + '' AS key, + min(ai_trace_index.Timestamp) AS sessionStart, + max(toUnixTimestamp64Nano(ai_trace_index.Timestamp) + toInt64(ai_trace_index.Duration)) - toUnixTimestamp64Nano(min(ai_trace_index.Timestamp)) AS sessionDurationNs, + sum(ai_trace_index.IsError) AS errorSpans, + sum(ai_trace_index.IsToolCall) AS toolCalls, + sumIf(ai_trace_index.IsError, ai_trace_index.IsToolCall = 1) AS erroredToolCalls, + sumIf(ai_trace_index.IsError, ai_trace_index.IsLlmCall = 1) AS erroredLlmCalls, + groupArrayIf(2000)(ai_trace_index.Duration, ai_trace_index.IsLlmCall = 1) AS llmDurations, + groupArrayIf(2000)(tuple(ai_trace_index.SpanId, ai_trace_index.ParentSpanId, ai_trace_index.Tokens, ai_trace_index.Cost, ai_trace_index.ResponseId, ai_trace_index.IsLlmCall, ai_trace_index.InputTokens, ai_trace_index.CacheReadTokens, ai_trace_index.CacheWriteTokens, ai_trace_index.OutputTokens, ai_trace_index.ReasoningTokens), ((ai_trace_index.Tokens > 0 OR ai_trace_index.Cost > 0) OR ai_trace_index.IsLlmCall = 1)) AS reporters, + arrayReduce('sumMap', arrayMap(c -> [c.2], reporters), arrayMap(c -> [c.3], reporters), arrayMap(c -> [c.4], reporters), arrayMap(c -> [c.7], reporters), arrayMap(c -> [c.8], reporters), arrayMap(c -> [c.9], reporters), arrayMap(c -> [c.10], reporters), arrayMap(c -> [c.11], reporters)) AS childClaims, + tupleElement(arrayFilter(p -> p.3 > 0 OR p.4 > 0, reporters), 1) AS reportingIds + FROM ai_trace_index + INNER JOIN (SELECT + TraceId AS TraceId, + max(SessionId) AS rawSessionId + FROM ai_trace_index + WHERE OrgId = 'org_sql_catalog' + AND Timestamp >= '2026-01-01 10:30:00' + AND Timestamp <= '2026-01-03 14:15:00' + GROUP BY TraceId + HAVING countIf(VendorId IN ('eve')) > 0 + AND countIf(ServiceName IN ('maple-slack-agent')) > 0 + AND countIf(DeploymentEnv IN ('production')) > 0 + AND countIf(Model IN ('gpt-5.5')) > 0 + AND countIf(AgentName IN ('billing-agent')) > 0 + AND countIf(ToolName IN ('send_email')) > 0) AS trace ON ai_trace_index.TraceId = trace.TraceId + WHERE ai_trace_index.OrgId = 'org_sql_catalog' + AND ai_trace_index.Timestamp >= '2026-01-01 10:30:00' + AND ai_trace_index.Timestamp <= '2026-01-03 14:15:00' + AND if(trace.rawSessionId = '', concat('trace:', ai_trace_index.TraceId), trace.rawSessionId) IN (SELECT + if(trace.rawSessionId = '', concat('trace:', ai_trace_index.TraceId), trace.rawSessionId) AS sessionId + FROM ai_trace_index + INNER JOIN (SELECT + TraceId AS TraceId, + max(SessionId) AS rawSessionId + FROM ai_trace_index + WHERE OrgId = 'org_sql_catalog' + AND Timestamp >= '2026-01-01 10:30:00' + AND Timestamp <= '2026-01-03 14:15:00' + GROUP BY TraceId + HAVING countIf(VendorId IN ('eve')) > 0 + AND countIf(ServiceName IN ('maple-slack-agent')) > 0 + AND countIf(DeploymentEnv IN ('production')) > 0 + AND countIf(Model IN ('gpt-5.5')) > 0 + AND countIf(AgentName IN ('billing-agent')) > 0 + AND countIf(ToolName IN ('send_email')) > 0) AS trace ON ai_trace_index.TraceId = trace.TraceId + WHERE ai_trace_index.OrgId = 'org_sql_catalog' + AND ai_trace_index.Timestamp >= '2026-01-01 10:30:00' + AND ai_trace_index.Timestamp <= '2026-01-03 14:15:00' + GROUP BY sessionId + HAVING sum(ai_trace_index.IsError) > 0) + GROUP BY sessionId) AS session_rows) AS netted_current +UNION ALL +SELECT + 'previous' AS period, + count() AS sessions, + countIf(errorSpans > 0) AS erroredSessions, + sum(toFloat64(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 2)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, toFloat64(n.2)), arrayFilter(n -> n.1 != '', netted))))))) AS llmCalls, + sum(erroredLlmCalls) AS erroredLlmCalls, + sum(toolCalls) AS toolCalls, + sum(erroredToolCalls) AS erroredToolCalls, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 4)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.4), arrayFilter(n -> n.1 != '', netted)))))) AS cost, + sum(arraySum(arrayMap(n -> toFloat64(n.2 AND n.4 > 0), arrayFilter(n -> n.1 = '', netted))) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, toFloat64(n.2 AND n.4 > 0)), arrayFilter(n -> n.1 != '', netted)))))) AS pricedLlmCalls, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 3)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.3), arrayFilter(n -> n.1 != '', netted)))))) AS tokens, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 5)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.5), arrayFilter(n -> n.1 != '', netted)))))) AS inputTokens, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 6)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.6), arrayFilter(n -> n.1 != '', netted)))))) AS cacheReadTokens, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 7)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.7), arrayFilter(n -> n.1 != '', netted)))))) AS cacheWriteTokens, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 8)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.8), arrayFilter(n -> n.1 != '', netted)))))) AS outputTokens, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 9)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.9), arrayFilter(n -> n.1 != '', netted)))))) AS reasoningTokens, + ifNull(ifNotFinite(quantile(0.5)(sessionDurationNs), 0), 0) AS sessionDurationP50Ns, + ifNull(ifNotFinite(quantile(0.95)(sessionDurationNs), 0), 0) AS sessionDurationP95Ns, + ifNull(ifNotFinite(quantileArray(0.5)(llmDurations), 0), 0) AS llmDurationP50Ns, + ifNull(ifNotFinite(quantileArray(0.95)(llmDurations), 0), 0) AS llmDurationP95Ns + FROM (SELECT + key AS key, + sessionStart AS sessionStart, + sessionDurationNs AS sessionDurationNs, + errorSpans AS errorSpans, + toolCalls AS toolCalls, + erroredToolCalls AS erroredToolCalls, + erroredLlmCalls AS erroredLlmCalls, + llmDurations AS llmDurations, + arrayMap(r -> tuple(r.5, r.6 = 1 AND if((r.3 > 0 OR r.4 > 0), greatest(0., r.3 - arrayElement(tupleElement(childClaims, 2), indexOf(tupleElement(childClaims, 1), r.1))) > 0 OR greatest(0., r.4 - arrayElement(tupleElement(childClaims, 3), indexOf(tupleElement(childClaims, 1), r.1))) > 0, NOT has(reportingIds, r.2)), greatest(0., r.3 - arrayElement(tupleElement(childClaims, 2), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.4 - arrayElement(tupleElement(childClaims, 3), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.7 - arrayElement(tupleElement(childClaims, 4), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.8 - arrayElement(tupleElement(childClaims, 5), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.9 - arrayElement(tupleElement(childClaims, 6), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.10 - arrayElement(tupleElement(childClaims, 7), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.11 - arrayElement(tupleElement(childClaims, 8), indexOf(tupleElement(childClaims, 1), r.1)))), reporters) AS netted + FROM (SELECT + if(trace.rawSessionId = '', concat('trace:', ai_trace_index.TraceId), trace.rawSessionId) AS sessionId, + '' AS key, + min(ai_trace_index.Timestamp) AS sessionStart, + max(toUnixTimestamp64Nano(ai_trace_index.Timestamp) + toInt64(ai_trace_index.Duration)) - toUnixTimestamp64Nano(min(ai_trace_index.Timestamp)) AS sessionDurationNs, + sum(ai_trace_index.IsError) AS errorSpans, + sum(ai_trace_index.IsToolCall) AS toolCalls, + sumIf(ai_trace_index.IsError, ai_trace_index.IsToolCall = 1) AS erroredToolCalls, + sumIf(ai_trace_index.IsError, ai_trace_index.IsLlmCall = 1) AS erroredLlmCalls, + groupArrayIf(2000)(ai_trace_index.Duration, ai_trace_index.IsLlmCall = 1) AS llmDurations, + groupArrayIf(2000)(tuple(ai_trace_index.SpanId, ai_trace_index.ParentSpanId, ai_trace_index.Tokens, ai_trace_index.Cost, ai_trace_index.ResponseId, ai_trace_index.IsLlmCall, ai_trace_index.InputTokens, ai_trace_index.CacheReadTokens, ai_trace_index.CacheWriteTokens, ai_trace_index.OutputTokens, ai_trace_index.ReasoningTokens), ((ai_trace_index.Tokens > 0 OR ai_trace_index.Cost > 0) OR ai_trace_index.IsLlmCall = 1)) AS reporters, + arrayReduce('sumMap', arrayMap(c -> [c.2], reporters), arrayMap(c -> [c.3], reporters), arrayMap(c -> [c.4], reporters), arrayMap(c -> [c.7], reporters), arrayMap(c -> [c.8], reporters), arrayMap(c -> [c.9], reporters), arrayMap(c -> [c.10], reporters), arrayMap(c -> [c.11], reporters)) AS childClaims, + tupleElement(arrayFilter(p -> p.3 > 0 OR p.4 > 0, reporters), 1) AS reportingIds + FROM ai_trace_index + INNER JOIN (SELECT + TraceId AS TraceId, + max(SessionId) AS rawSessionId + FROM ai_trace_index + WHERE OrgId = 'org_sql_catalog' + AND Timestamp >= '2025-12-30 06:45:00' + AND Timestamp <= '2026-01-01 10:30:00' + GROUP BY TraceId + HAVING countIf(VendorId IN ('eve')) > 0 + AND countIf(ServiceName IN ('maple-slack-agent')) > 0 + AND countIf(DeploymentEnv IN ('production')) > 0 + AND countIf(Model IN ('gpt-5.5')) > 0 + AND countIf(AgentName IN ('billing-agent')) > 0 + AND countIf(ToolName IN ('send_email')) > 0) AS trace ON ai_trace_index.TraceId = trace.TraceId + WHERE ai_trace_index.OrgId = 'org_sql_catalog' + AND ai_trace_index.Timestamp >= '2025-12-30 06:45:00' + AND ai_trace_index.Timestamp <= '2026-01-01 10:30:00' + AND if(trace.rawSessionId = '', concat('trace:', ai_trace_index.TraceId), trace.rawSessionId) IN (SELECT + if(trace.rawSessionId = '', concat('trace:', ai_trace_index.TraceId), trace.rawSessionId) AS sessionId + FROM ai_trace_index + INNER JOIN (SELECT + TraceId AS TraceId, + max(SessionId) AS rawSessionId + FROM ai_trace_index + WHERE OrgId = 'org_sql_catalog' + AND Timestamp >= '2025-12-30 06:45:00' + AND Timestamp <= '2026-01-01 10:30:00' + GROUP BY TraceId + HAVING countIf(VendorId IN ('eve')) > 0 + AND countIf(ServiceName IN ('maple-slack-agent')) > 0 + AND countIf(DeploymentEnv IN ('production')) > 0 + AND countIf(Model IN ('gpt-5.5')) > 0 + AND countIf(AgentName IN ('billing-agent')) > 0 + AND countIf(ToolName IN ('send_email')) > 0) AS trace ON ai_trace_index.TraceId = trace.TraceId + WHERE ai_trace_index.OrgId = 'org_sql_catalog' + AND ai_trace_index.Timestamp >= '2025-12-30 06:45:00' + AND ai_trace_index.Timestamp <= '2026-01-01 10:30:00' + GROUP BY sessionId + HAVING sum(ai_trace_index.IsError) > 0) + GROUP BY sessionId) AS session_rows) AS netted_previous +FORMAT JSON + -- builder:ai-sessions:aiSessionDetailsQuery:default SELECT if(index_traces.rawSessionId = '', concat('trace:', session_traces.traceId), index_traces.rawSessionId) AS sessionId, diff --git a/packages/query-engine-integrations/src/ai/ai-overview.test.ts b/packages/query-engine-integrations/src/ai/ai-overview.test.ts new file mode 100644 index 000000000..816eb3101 --- /dev/null +++ b/packages/query-engine-integrations/src/ai/ai-overview.test.ts @@ -0,0 +1,214 @@ +import { describe, expect, it } from "vitest" +import { compileUnionUnsafe } from "@maple-dev/effect-clickhouse" +import { AI_OVERVIEW_BREAKDOWN_MAX } from "@maple/domain/http" +import { aiOverviewBreakdownQuery, aiOverviewSeriesQuery, aiOverviewTotalsQuery } from "./ai-overview" + +const params = { + orgId: "org_1", + startTime: "2026-08-18 00:00:00", + endTime: "2026-08-19 23:59:59", + prevStartTime: "2026-08-16 00:00:01", + prevEndTime: "2026-08-18 00:00:00", +} + +/** The series is the only read that cuts buckets. */ +const seriesParams = { ...params, bucketSeconds: 300 } + +/** The sessions list's key, resolved per trace — the same expression + * `aiSessionPageQuery` groups on, so a number here reconciles with a row + * there. */ +const SESSION_KEY = + "if(trace.rawSessionId = '', concat('trace:', ai_trace_index.TraceId), trace.rawSessionId)" + +/** `OrgId = 'x'` on every level that reads a table — a subquery contributes + * nothing to the outer query's scope. */ +const orgPredicateCount = (sql: string) => sql.split("OrgId = 'org_1'").length - 1 + +const totalsSql = (opts = {}) => compileUnionUnsafe(aiOverviewTotalsQuery(opts), params).sql +const seriesSql = (opts = {}) => compileUnionUnsafe(aiOverviewSeriesQuery(opts), seriesParams).sql + +describe("overview population", () => { + it("reads ai_trace_index alone", () => { + for (const sql of [totalsSql(), seriesSql()]) { + expect(sql).toContain("FROM ai_trace_index") + // The moment an overview read reaches the span tables it costs what the + // sessions fan-out costs — seconds, per partition, over a month. + expect(sql).not.toContain("trace_detail_spans") + expect(sql).not.toContain("SpanAttributes") + expect(sql).not.toContain("__PARAM_") + } + }) + + it("resolves the session per trace, with the sessions list's own key", () => { + const sql = totalsSql() + + // `max(SessionId)` per trace, because the id sits on the turn-owning span + // and every other row of the trace reads ''. + expect(sql).toContain("max(SessionId) AS rawSessionId") + expect(sql).toContain(`${SESSION_KEY} AS sessionId`) + }) + + it("nets usage and model calls the way the sessions list nets them", () => { + const sql = totalsSql() + + // The reporters, the two lookups taken off them once per session, and the + // netting — a naive sum(Cost) double-counts every wrapper roll-up. + expect(sql).toContain("AS reporters") + expect(sql).toContain("AS childClaims") + expect(sql).toContain("AS reportingIds") + expect(sql).toContain("AS netted") + expect(sql).toContain("maxMap") + expect(sql).not.toContain("sum(Cost)") + expect(sql).not.toContain("sum(Tokens)") + }) + + it("scopes every level that reads the table to the org", () => { + // Two levels per branch — the trace keys and the session rows — and two + // branches. + expect(orgPredicateCount(totalsSql())).toBe(4) + expect(compileUnionUnsafe(aiOverviewTotalsQuery(), params).tenantScope).toBe("single-tenant") + expect(compileUnionUnsafe(aiOverviewSeriesQuery(), seriesParams).tenantScope).toBe("single-tenant") + expect(compileUnionUnsafe(aiOverviewBreakdownQuery({ dimension: "model" }), params).tenantScope).toBe( + "single-tenant", + ) + }) + + it("applies each filter as a per-trace existence test, and none when none is given", () => { + const sql = totalsSql({ + vendorIds: ["eve"], + serviceNames: ["agent-runner"], + deploymentEnvs: ["production"], + models: ["gpt-5.5"], + agentNames: ["billing-agent"], + toolNames: ["send_email"], + }) + + // HAVING, not WHERE: a row predicate would narrow the rows the session id + // is read from, and a model ANDed with a tool can never match one row. + expect(sql).toContain("countIf(VendorId IN ('eve')) > 0") + expect(sql).toContain("countIf(ServiceName IN ('agent-runner')) > 0") + expect(sql).toContain("countIf(DeploymentEnv IN ('production')) > 0") + expect(sql).toContain("countIf(Model IN ('gpt-5.5')) > 0") + expect(sql).toContain("countIf(AgentName IN ('billing-agent')) > 0") + expect(sql).toContain("countIf(ToolName IN ('send_email')) > 0") + + const unfiltered = totalsSql() + expect(unfiltered).not.toContain("HAVING") + expect(unfiltered).not.toContain("countIf(VendorId") + }) + + it("selects the sessions that failed with the list's session-level rule", () => { + const sql = totalsSql({ hasErrors: true }) + + // A session-level test, not a trace-level one: a session spans traces and + // the list matches it when any of its agent spans failed. + expect(sql).toContain("HAVING sum(ai_trace_index.IsError) > 0") + expect(sql).toContain(`${SESSION_KEY} IN (SELECT`) + expect(totalsSql()).not.toContain("sum(ai_trace_index.IsError) > 0") + }) +}) + +describe("the measures every grouping reports", () => { + it("counts a session once and files it under the bucket it started in", () => { + const sql = seriesSql() + + // `count()`, not `uniqExact`: the level below is already one row per + // session, and a session has exactly one first span — so the buckets sum + // to the totals. + expect(sql).toContain("count() AS sessions") + expect(sql).toContain("countIf(errorSpans > 0) AS erroredSessions") + expect(sql).toContain("min(ai_trace_index.Timestamp) AS sessionStart") + expect(sql).toContain("toStartOfInterval(sessionStart, INTERVAL 300 SECOND)") + expect(sql).toContain("GROUP BY bucket") + // The totals are their own un-bucketed read, because quantiles do not + // merge — a p95 folded from the series is not a p95. + expect(totalsSql()).not.toContain("toStartOfInterval") + }) + + it("guards every quantile against the empty group", () => { + const sql = totalsSql() + + // A quantile over no rows is NULL, which the row schema refuses. + for (const measure of [ + "sessionDurationP50Ns", + "sessionDurationP95Ns", + "llmDurationP50Ns", + "llmDurationP95Ns", + ]) { + expect(sql).toContain(`AS ${measure}`) + } + expect(sql).toContain("ifNull(ifNotFinite(quantile(0.95)(sessionDurationNs), 0), 0)") + // The model-call latency is a SPAN quantile taken at a level whose rows + // are sessions, which is what the array carries it up for. + expect(sql).toContain("quantileArray(0.5)(llmDurations)") + expect(sql).toContain("groupArrayIf(2000)(ai_trace_index.Duration, ai_trace_index.IsLlmCall = 1)") + }) + + it("measures the extent of the session, not the start of its last span", () => { + // Without the `+ Duration` a session whose trace is one long span reports + // a duration of 0, and every other session under-reports by the + // last-starting span's own duration. + expect(totalsSql()).toContain( + "max(toUnixTimestamp64Nano(ai_trace_index.Timestamp) + toInt64(ai_trace_index.Duration)) - toUnixTimestamp64Nano(min(ai_trace_index.Timestamp)) AS sessionDurationNs", + ) + }) +}) + +describe("the breakdown's dimensions", () => { + const breakdownSql = (opts: Parameters[0]) => + compileUnionUnsafe(aiOverviewBreakdownQuery(opts), params).sql + + it("keys each dimension by the column the span carries it in", () => { + expect(breakdownSql({ dimension: "model" })).toContain("toString(ai_trace_index.Model) AS key") + expect(breakdownSql({ dimension: "agent" })).toContain("toString(ai_trace_index.AgentName) AS key") + expect(breakdownSql({ dimension: "service" })).toContain( + "toString(ai_trace_index.ServiceName) AS key", + ) + expect(breakdownSql({ dimension: "environment" })).toContain( + "toString(ai_trace_index.DeploymentEnv) AS key", + ) + expect(breakdownSql({ dimension: "vendor" })).toContain("toString(ai_trace_index.VendorId) AS key") + expect(breakdownSql({ dimension: "tool" })).toContain("toString(ai_trace_index.ToolName) AS key") + }) + + it("reads a model over model calls and a tool over tool calls, and the rest over every span", () => { + // The predicate, not the `sumIf` measures every read carries: a model + // keys off a column only a model call fills, and a tool off one only a + // tool call fills, so the population is narrowed rather than left to + // answer `''` for every other span. + expect(breakdownSql({ dimension: "model" })).toContain("AND ai_trace_index.IsLlmCall = 1") + expect(breakdownSql({ dimension: "tool" })).toContain("AND ai_trace_index.IsToolCall = 1") + const byService = breakdownSql({ dimension: "service" }) + expect(byService).not.toContain("AND ai_trace_index.IsLlmCall = 1") + expect(byService).not.toContain("AND ai_trace_index.IsToolCall = 1") + }) + + it("measures both windows over the keys the current window ranked, and counts the rest", () => { + const sql = breakdownSql({ dimension: "model" }) + + // The previous branch is restricted to the same keys, so a key that + // stopped being used still shows what it cost. + expect(sql.split("key IN (SELECT").length - 1).toBe(2) + expect(sql).toContain(`LIMIT ${AI_OVERVIEW_BREAKDOWN_MAX}`) + expect(sql).toContain("uniqExact(toString(ai_trace_index.Model)) AS keyCount") + expect(sql).toContain("'keys' AS period") + }) + + it("never ranks more keys than the table can show", () => { + expect(breakdownSql({ dimension: "tool", limit: 3 })).toContain("LIMIT 3") + expect(breakdownSql({ dimension: "tool", limit: 500 })).toContain( + `LIMIT ${AI_OVERVIEW_BREAKDOWN_MAX}`, + ) + }) + + it("groups by the key and by nothing else, so a session counts once per key", () => { + const sql = breakdownSql({ dimension: "model" }) + + expect(sql).toContain("GROUP BY sessionId, key") + expect(sql).toContain("GROUP BY key") + // The totals never group by a key — theirs is the constant every read + // carries so the levels have one shape. + expect(totalsSql()).toContain("GROUP BY sessionId") + expect(totalsSql()).not.toContain("GROUP BY sessionId, key") + }) +}) diff --git a/packages/query-engine-integrations/src/ai/ai-overview.ts b/packages/query-engine-integrations/src/ai/ai-overview.ts new file mode 100644 index 000000000..c69c74e6e --- /dev/null +++ b/packages/query-engine-integrations/src/ai/ai-overview.ts @@ -0,0 +1,540 @@ +// Agent Sessions › Overview — the warehouse reads behind the overview page. +// +// Everything here is `ai_trace_index` and nothing else (see `ai-sessions.ts` +// for what that index is and what it costs). The page asks three questions of +// one population — what the window totals, how it moved, and where it went — +// and the answer to all three has to be the same numbers the sessions LIST +// shows for the same window, or the two pages describe different products. +// +// That constraint is what shapes the file. Four rules, all borrowed rather +// than re-derived: +// +// 1. A SESSION is a trace-level key: `max(SessionId)` per trace, then +// `sessionKey(…)` from `ai-sessions.ts`, which files a trace whose vendor +// exposes no session key under `trace:`. Counted per row instead, +// every sessionless span collapses into one phantom session. +// 2. A FILTER selects sessions, as a per-trace existence test — "some agent +// span of this trace carries this value" — never a row predicate. The +// three GenAI identity columns are mutually exclusive by construction, so +// a row predicate ANDing a model with a tool can only match a row +// carrying both, and two facets with non-zero counts would return +// nothing. `traceKeys` is that test, and it is the same one `indexTraces` +// makes for the list. +// 3. USAGE is netted per session — `usageReportersExpr` collected over the +// session's spans, then `nettedReportersExpr` and `sessionUsageSum`. +// A wrapper that rolls up its children's tokens, a gateway's second trace +// of the same call, and a provider retry beneath the call each count +// once. A bucketed `sum(Cost)` is one line of SQL and is wrong by +// whatever the org's roll-up rate is. +// 4. A SESSION BELONGS TO ONE BUCKET — the one its first span started in — +// so the series sums to the totals instead of counting a long session in +// every bucket it touched. Quantiles are the exception: they do not +// merge, which is why the totals are their own un-bucketed read rather +// than a client-side fold of the series. +// +// The breakdown adds a fifth. A key is the value the SPAN ITSELF carries, and +// the netting runs per (session, key): a session that used two models is a +// session under each of them (rows overlap and do not sum to the totals), but +// its tokens are charged to the model whose call reported them rather than +// repeated under both. That is the most faithful per-model attribution the +// reporter mechanism allows without a second netting implementation, and it is +// what keeps a wrapper's roll-up and a gateway's mirror from being counted +// twice inside a key. Its one gap: a reporter whose dimension value differs +// from its children's — an eve turn span rolling up a Vercel model call under a +// `vendor` breakdown — is netted only within its own key, so those two rows +// can together exceed the window's cost. `model` and `tool` are read over the +// spans that can carry them (`IsLlmCall = 1`, `IsToolCall = 1`); the rest are +// read over every agent span, and a span that names no value keys under `''`, +// which the page renders as unattributed rather than hiding. +// +// Durations stay in NANOSECONDS, like every other AI read — `Duration` is what +// the index stores and the client formats. + +import * as CH from "@maple-dev/effect-clickhouse/expr" +import * as T from "@maple-dev/effect-clickhouse/types" +import { compile } from "@maple-dev/effect-clickhouse/sql" +import { from, fromQuery, inSubquery, param, unionAll, type CHUnionQuery } from "@maple-dev/effect-clickhouse" +import { AI_OVERVIEW_BREAKDOWN_MAX, type AiOverviewDimension } from "@maple/domain/http" +import { AiTraceIndex } from "@maple/query-engine/ch/tables" +import { finiteOrZero, isoBucket } from "@maple/query-engine/ch/format" +import { sessionKey } from "./ai-sessions" +import { + childClaimsExpr, + MAX_USAGE_REPORTERS_PER_TRACE, + nettedReportersExpr, + reportingSpanIdsExpr, + sessionLlmCalls, + sessionPricedLlmCalls, + sessionUsageSum, + usageReportersExpr, +} from "./ai-span-columns" + +/** + * The page's selection, as every read here takes it — the sessions list's + * counted filters by the same names, so the two pages select the same + * sessions. Each is a per-trace existence test; see rule 2 in the header. + */ +export interface AiOverviewFilterOpts { + readonly vendorIds?: readonly string[] + readonly serviceNames?: readonly string[] + readonly deploymentEnvs?: readonly string[] + readonly models?: readonly string[] + readonly agentNames?: readonly string[] + readonly toolNames?: readonly string[] + /** Sessions with at least one failed agent span — the list's own rule. */ + readonly hasErrors?: boolean +} + +export interface AiOverviewBreakdownOpts extends AiOverviewFilterOpts { + readonly dimension: AiOverviewDimension + /** Keys returned per period. Defaults to — and is capped at — + * {@link AI_OVERVIEW_BREAKDOWN_MAX}. */ + readonly limit?: number +} + +/** + * Which pair of params bounds a read: the caller's window, or the window of + * equal length immediately before it. Both reads use both — one `UNION ALL` + * branch each — and every branch is built from the same expression functions, + * because a `LowCardinality(String)` on one branch against a `String` on + * another is a `NO_COMMON_TYPE`. + */ +export type AiOverviewWindow = "current" | "previous" + +/** Which window a row measures. `keys` is the breakdown's third branch: how + * many distinct keys the current window has, before the top-N cut. */ +export type AiOverviewPeriod = "current" | "previous" | "keys" + +const startParam = (window: AiOverviewWindow) => + param.dateTimeString(window === "current" ? "startTime" : "prevStartTime") +const endParam = (window: AiOverviewWindow) => + param.dateTimeString(window === "current" ? "endTime" : "prevEndTime") + +/** + * One row per agent trace of the window that passes the selection: its id and + * the session it is filed under. + * + * `max(SessionId)` because the id sits on the turn-owning span alone and every + * other row of the trace reads `''`, which `max` discards. The filters are + * `HAVING countIf(…) > 0` for the same reason `indexTraces` applies them + * there — a row predicate would also narrow the rows the session id is read + * from, and would file a trace under `trace:` whenever its session-bearing + * span belonged to another vendor. + */ +const traceKeys = (opts: AiOverviewFilterOpts, window: AiOverviewWindow) => { + const values = (list: readonly string[] | undefined) => (list?.length ? list : undefined) + const carries = (cond: CH.Condition) => CH.countIf(cond).gt(0) + return from(AiTraceIndex) + .select(($) => ({ TraceId: $.TraceId, rawSessionId: CH.max_($.SessionId) })) + .where(($) => [ + $.OrgId.eq(param.string("orgId")), + $.Timestamp.gte(startParam(window)), + $.Timestamp.lte(endParam(window)), + ]) + .groupBy("TraceId") + .having(($) => [ + CH.when(values(opts.vendorIds), (v) => carries(CH.inList($.VendorId, v))), + CH.when(values(opts.serviceNames), (v) => carries(CH.inList($.ServiceName, v))), + CH.when(values(opts.deploymentEnvs), (v) => carries(CH.inList($.DeploymentEnv, v))), + CH.when(values(opts.models), (v) => carries(CH.inList($.Model, v))), + CH.when(values(opts.agentNames), (v) => carries(CH.inList($.AgentName, v))), + CH.when(values(opts.toolNames), (v) => carries(CH.inList($.ToolName, v))), + ]) +} + +/** + * The session keys of the window with a failed agent span — the `hasErrors` + * filter, as the list applies it. + * + * A session-level test and not a trace-level one: a session spans traces, and + * the list matches it when ANY of its agent spans failed. It reads the whole + * population rather than the dimension's, so a `model` breakdown under + * `hasErrors` measures the sessions the list would have listed. + */ +const erroredSessionKeys = (opts: AiOverviewFilterOpts, window: AiOverviewWindow) => + from(AiTraceIndex) + .innerJoinQuery(traceKeys(opts, window), "trace", (row, trace) => row.TraceId.eq(trace.TraceId)) + .select(($) => ({ sessionId: sessionKey($.trace.rawSessionId, $.TraceId) })) + .where(($) => [ + $.OrgId.eq(param.string("orgId")), + $.Timestamp.gte(startParam(window)), + $.Timestamp.lte(endParam(window)), + ]) + .groupBy("sessionId") + .having(($) => [CH.sum($.IsError).gt(0)]) + +interface CallColumns { + readonly IsLlmCall: CH.Expr + readonly IsToolCall: CH.Expr +} + +/** The spans a dimension's keys can come from. `Model` sits on model calls and + * `ToolName` on tool calls; the rest are properties of every agent span. */ +const dimensionPopulation = ( + dimension: AiOverviewDimension, +): (($: CallColumns) => CH.Condition) | undefined => { + if (dimension === "model") return ($) => $.IsLlmCall.eq(1) + if (dimension === "tool") return ($) => $.IsToolCall.eq(1) + return undefined +} + +/** + * The value a row is filed under, as a plain `String`. + * + * `toString` because four of the six columns are `LowCardinality(String)` and + * the breakdown unions them against a `String` literal on its third branch, + * which is a `NO_COMMON_TYPE` without it. + */ +const dimensionKey = (dimension: AiOverviewDimension) => { + switch (dimension) { + case "model": + return ($: DimensionColumns) => CH.toString_($.Model) + case "agent": + return ($: DimensionColumns) => CH.toString_($.AgentName) + case "service": + return ($: DimensionColumns) => CH.toString_($.ServiceName) + case "environment": + return ($: DimensionColumns) => CH.toString_($.DeploymentEnv) + case "vendor": + return ($: DimensionColumns) => CH.toString_($.VendorId) + case "tool": + return ($: DimensionColumns) => CH.toString_($.ToolName) + } +} + +interface DimensionColumns { + readonly Model: CH.Expr + readonly AgentName: CH.Expr + readonly ServiceName: CH.Expr + readonly DeploymentEnv: CH.Expr + readonly VendorId: CH.Expr + readonly ToolName: CH.Expr +} + +/** One session's model-call durations, for the quantiles two levels up. Raw + * SQL because the cap is a parameter of the aggregate (`groupArrayIf(N)(…)`), + * a shape the builder's function-call helper does not render. */ +const llmDurationsExpr = ($: { + readonly Duration: CH.Expr + readonly IsLlmCall: CH.Expr +}): CH.Expr => + CH.untypedExpr( + `groupArrayIf(${MAX_USAGE_REPORTERS_PER_TRACE})(${compile($.Duration.toFragment())}, ${compile( + $.IsLlmCall.eq(1).toFragment(), + )})`, + ) + +/** A quantile over every element of an array column — ClickHouse's `-Array` + * combinator, which reads the arrays as if they had been `arrayJoin`ed. + * The only way to take a SPAN-level quantile at a level whose rows are + * sessions, and not in the builder's function set. */ +const quantileOfArrays = (level: number, column: string): CH.Expr => + CH.rawExpr(`quantileArray(${level})(${column})`, T.float64) + +/** + * One row per session (or per session and key), with everything the index + * carries about it: the measures summed over its spans, and its usage still as + * reporters, netted one level up and summed at the grouping level. + * + * `key` is the breakdown's grouping; without it the rows are sessions, which + * is what the totals and the series aggregate. Column names are deliberately + * not the names the levels above select (`sessionStart`, not `bucket`): an + * outer alias shadows a derived column of the same name, and an aggregate over + * the shadowed name becomes a cyclic alias rather than the aggregate meant. + */ +const sessionRows = ( + opts: AiOverviewFilterOpts, + window: AiOverviewWindow, + dimension?: AiOverviewDimension, +) => { + const population = dimension === undefined ? undefined : dimensionPopulation(dimension) + const key = dimension === undefined ? undefined : dimensionKey(dimension) + const rows = from(AiTraceIndex) + .innerJoinQuery(traceKeys(opts, window), "trace", (row, trace) => row.TraceId.eq(trace.TraceId)) + .select(($) => ({ + sessionId: sessionKey($.trace.rawSessionId, $.TraceId), + // The totals and the series are a breakdown of one key, so the column + // is always there and is `''` for them — a constant, which needs no + // place in the GROUP BY and costs the read nothing. + key: key === undefined ? CH.lit("") : key($), + // The bucket the session is filed under is cut from this one level + // up — the session's FIRST span, so it lands in exactly one bucket. + sessionStart: CH.min_($.Timestamp), + // `Timestamp` is the span's START, so the extent ends where the + // last-starting span ended. Without the `+ Duration` a session whose + // trace is one long span reports a duration of 0. + sessionDurationNs: CH.max_(CH.toUnixTimestamp64Nano($.Timestamp).add(CH.toInt64($.Duration))).sub( + CH.toUnixTimestamp64Nano(CH.min_($.Timestamp)), + ), + errorSpans: CH.sum($.IsError), + toolCalls: CH.sum($.IsToolCall), + erroredToolCalls: CH.sumIf($.IsError, $.IsToolCall.eq(1)), + // The failed model-call SPANS. Not netted — the index carries no + // error flag into the reporters — so a framework that echoes a + // failure onto the span wrapping the call reports it twice. + erroredLlmCalls: CH.sumIf($.IsError, $.IsLlmCall.eq(1)), + llmDurations: llmDurationsExpr($), + // Usage AND model calls travel as reporters: both are counted above, + // where every span of the session is in hand — see `ai-span-columns`. + // The two lookups the netting makes are taken off the reporters here, + // once per session, rather than once per reporter inside the netting. + reporters: usageReportersExpr($), + childClaims: childClaimsExpr("reporters"), + reportingIds: reportingSpanIdsExpr("reporters"), + })) + .where(($) => [ + $.OrgId.eq(param.string("orgId")), + $.Timestamp.gte(startParam(window)), + $.Timestamp.lte(endParam(window)), + population === undefined ? undefined : population($), + CH.whenTrue(opts.hasErrors, () => + inSubquery(sessionKey($.trace.rawSessionId, $.TraceId), erroredSessionKeys(opts, window)), + ), + ]) + return key === undefined ? rows.groupBy("sessionId") : rows.groupBy("sessionId", "key") +} + +/** The reporters netted into claims — its own level, because the netting reads + * the three columns below it inside lambdas and an alias of the same level + * would be evaluated once per reporter. */ +const nettedRows = (opts: AiOverviewFilterOpts, window: AiOverviewWindow, dimension?: AiOverviewDimension) => + fromQuery(sessionRows(opts, window, dimension), "session_rows").select(($) => ({ + key: $.key, + sessionStart: $.sessionStart, + sessionDurationNs: $.sessionDurationNs, + errorSpans: $.errorSpans, + toolCalls: $.toolCalls, + erroredToolCalls: $.erroredToolCalls, + erroredLlmCalls: $.erroredLlmCalls, + llmDurations: $.llmDurations, + netted: nettedReportersExpr("reporters", "childClaims", "reportingIds"), + })) + +/** The accessor shape {@link measures} reads off {@link nettedRows}. */ +interface SessionColumns { + readonly sessionDurationNs: CH.Expr + readonly errorSpans: CH.Expr + readonly toolCalls: CH.Expr + readonly erroredToolCalls: CH.Expr + readonly erroredLlmCalls: CH.Expr +} + +/** + * The measures every grouping reports, so a tile, a point on the chart and a + * breakdown row are the same numbers under different `GROUP BY`s. + * + * `count()` rather than `uniqExact`: the level below is already one row per + * session (or per session and key), so a session is counted once and exactly + * once — and a session lands in one bucket, so the series sums to the totals. + * + * The usage sums are the netting evaluated per session and summed over the + * group. Written as one pass per measure over the netted claims, the same + * eight the sessions list makes, because what a level computes is what it + * costs: the warehouse analyses every lambda in a SELECT before it reads a + * row. + */ +const measures = ($: SessionColumns) => ({ + sessions: CH.count(), + erroredSessions: CH.countIf($.errorSpans.gt(0)), + llmCalls: CH.sum(sessionLlmCalls("netted")), + erroredLlmCalls: CH.sum($.erroredLlmCalls), + toolCalls: CH.sum($.toolCalls), + erroredToolCalls: CH.sum($.erroredToolCalls), + cost: CH.sum(sessionUsageSum("netted", "cost")), + pricedLlmCalls: CH.sum(sessionPricedLlmCalls("netted")), + tokens: CH.sum(sessionUsageSum("netted", "tokens")), + inputTokens: CH.sum(sessionUsageSum("netted", "inputTokens")), + cacheReadTokens: CH.sum(sessionUsageSum("netted", "cacheReadTokens")), + cacheWriteTokens: CH.sum(sessionUsageSum("netted", "cacheWriteTokens")), + outputTokens: CH.sum(sessionUsageSum("netted", "outputTokens")), + reasoningTokens: CH.sum(sessionUsageSum("netted", "reasoningTokens")), + // A quantile over an empty group is NULL, which the row schema refuses. + sessionDurationP50Ns: finiteOrZero(CH.quantile(0.5)($.sessionDurationNs)), + sessionDurationP95Ns: finiteOrZero(CH.quantile(0.95)($.sessionDurationNs)), + llmDurationP50Ns: finiteOrZero(quantileOfArrays(0.5, "llmDurations")), + llmDurationP95Ns: finiteOrZero(quantileOfArrays(0.95, "llmDurations")), +}) + +/** Every measure at zero — the shape a branch that measures something else + * still has to project, since a `UNION ALL`'s branches share one row. */ +const noMeasures = () => ({ + sessions: CH.lit(0), + erroredSessions: CH.lit(0), + llmCalls: CH.lit(0), + erroredLlmCalls: CH.lit(0), + toolCalls: CH.lit(0), + erroredToolCalls: CH.lit(0), + cost: CH.lit(0), + pricedLlmCalls: CH.lit(0), + tokens: CH.lit(0), + inputTokens: CH.lit(0), + cacheReadTokens: CH.lit(0), + cacheWriteTokens: CH.lit(0), + outputTokens: CH.lit(0), + reasoningTokens: CH.lit(0), + sessionDurationP50Ns: CH.lit(0), + sessionDurationP95Ns: CH.lit(0), + llmDurationP50Ns: CH.lit(0), + llmDurationP95Ns: CH.lit(0), +}) + +export interface AiOverviewMeasuresOutput { + readonly sessions: number + readonly erroredSessions: number + readonly llmCalls: number + readonly erroredLlmCalls: number + readonly toolCalls: number + readonly erroredToolCalls: number + readonly cost: number + readonly pricedLlmCalls: number + readonly tokens: number + readonly inputTokens: number + readonly cacheReadTokens: number + readonly cacheWriteTokens: number + readonly outputTokens: number + readonly reasoningTokens: number + readonly sessionDurationP50Ns: number + readonly sessionDurationP95Ns: number + readonly llmDurationP50Ns: number + readonly llmDurationP95Ns: number +} + +export interface AiOverviewTotalsOutput extends AiOverviewMeasuresOutput { + readonly period: string +} + +export interface AiOverviewSeriesOutput extends AiOverviewTotalsOutput { + /** ISO-8601 with a literal `Z`. */ + readonly bucket: string +} + +export interface AiOverviewBreakdownOutput extends AiOverviewTotalsOutput { + readonly key: string + /** Distinct keys the current window has — carried by the `keys` branch + * alone, 0 on the two that measure. */ + readonly keyCount: number +} + +/** + * The KPI tiles: every measure over the caller's window, and over the window of + * equal length immediately before it. + * + * One read rather than two requests, and not folded from the series either: + * quantiles cannot be merged after the fact, so a p95 for the window is only + * available from a read that grouped the window. The previous branch is bounded + * by its own pair of params (`prevStartTime`/`prevEndTime`), which the caller + * computes — the query has no opinion about what "previous" means beyond + * reading a second window. + */ +export function aiOverviewTotalsQuery(opts: AiOverviewFilterOpts = {}): CHUnionQuery { + const branch = (window: AiOverviewWindow) => + fromQuery(nettedRows(opts, window), `netted_${window}`).select(($) => ({ + period: CH.lit(window), + ...measures($), + })) + return unionAll(branch("current"), branch("previous")).format("JSON") +} + +/** + * The chart: the same measures, cut into buckets. + * + * A session is filed under the bucket its FIRST span started in, so the points + * sum to the totals — every other reading counts a session that ran across a + * bucket boundary twice. The session's whole netted usage goes with it, which + * is the simplification the attribution makes: at bucket widths of five + * minutes and up a session's spans are inside one bucket or the next. + */ +export function aiOverviewSeriesQuery(opts: AiOverviewFilterOpts = {}): CHUnionQuery { + const branch = (window: AiOverviewWindow) => + fromQuery(nettedRows(opts, window), `netted_${window}`) + .select(($) => ({ + period: CH.lit(window), + bucket: isoBucket($.sessionStart), + ...measures($), + })) + .groupBy("bucket") + // Oldest first, so a client plots the points in the order they arrive. + return unionAll(branch("current"), branch("previous")) + .orderBy(["period", "asc"], ["bucket", "asc"]) + .format("JSON") +} + +/** + * The busiest keys of the current window, as a one-column subquery for `IN`. + * + * Ranked on sessions alone, off the raw index rows rather than the netted + * pipeline: which keys the table shows is a question about counts, and running + * the netting a third time to break a tie by cost would cost more than the tie + * is worth. The key breaks ties instead, so two keys with the same session + * count cannot swap places between loads. + */ +const topKeys = (opts: AiOverviewBreakdownOpts) => { + const population = dimensionPopulation(opts.dimension) + const key = dimensionKey(opts.dimension) + const ranked = from(AiTraceIndex) + .innerJoinQuery(traceKeys(opts, "current"), "trace", (row, trace) => row.TraceId.eq(trace.TraceId)) + .select(($) => ({ + rankKey: key($), + rankSessions: CH.uniqExact(sessionKey($.trace.rawSessionId, $.TraceId)), + })) + .where(($) => [ + $.OrgId.eq(param.string("orgId")), + $.Timestamp.gte(startParam("current")), + $.Timestamp.lte(endParam("current")), + population === undefined ? undefined : population($), + CH.whenTrue(opts.hasErrors, () => + inSubquery(sessionKey($.trace.rawSessionId, $.TraceId), erroredSessionKeys(opts, "current")), + ), + ]) + .groupBy("rankKey") + .orderBy(["rankSessions", "desc"], ["rankKey", "asc"]) + .limit(Math.min(opts.limit ?? AI_OVERVIEW_BREAKDOWN_MAX, AI_OVERVIEW_BREAKDOWN_MAX)) + return fromQuery(ranked, "top_keys").select(($) => ({ topKey: $.rankKey })) +} + +/** + * The breakdown table: the busiest keys of the current window, each measured + * over both windows. + * + * Three branches. Two measure the keys the ranking picked — the previous one + * over the same keys, so a key that stopped being used still shows what it + * cost. The third counts the window's distinct keys, which is what lets the + * table say how many it is not showing; it reads the same population off the + * index rather than the netted pipeline, because it is a count of keys and not + * of anything a session did. + */ +export function aiOverviewBreakdownQuery( + opts: AiOverviewBreakdownOpts, +): CHUnionQuery { + const keys = topKeys(opts) + const branch = (window: AiOverviewWindow) => + fromQuery(nettedRows(opts, window, opts.dimension), `netted_${window}`) + .select(($) => ({ + period: CH.lit(window), + key: $.key, + keyCount: CH.lit(0), + ...measures($), + })) + .where(($) => [inSubquery($.key, keys)]) + .groupBy("key") + const population = dimensionPopulation(opts.dimension) + const key = dimensionKey(opts.dimension) + const keyCount = from(AiTraceIndex) + .innerJoinQuery(traceKeys(opts, "current"), "trace", (row, trace) => row.TraceId.eq(trace.TraceId)) + .select(($) => ({ + period: CH.lit("keys"), + key: CH.lit(""), + keyCount: CH.uniqExact(key($)), + ...noMeasures(), + })) + .where(($) => [ + $.OrgId.eq(param.string("orgId")), + $.Timestamp.gte(startParam("current")), + $.Timestamp.lte(endParam("current")), + population === undefined ? undefined : population($), + CH.whenTrue(opts.hasErrors, () => + inSubquery(sessionKey($.trace.rawSessionId, $.TraceId), erroredSessionKeys(opts, "current")), + ), + ]) + return unionAll(branch("current"), branch("previous"), keyCount).format("JSON") +} diff --git a/packages/query-engine-integrations/src/ai/ai-sessions.ts b/packages/query-engine-integrations/src/ai/ai-sessions.ts index fe0fdf1aa..45fd25e27 100644 --- a/packages/query-engine-integrations/src/ai/ai-sessions.ts +++ b/packages/query-engine-integrations/src/ai/ai-sessions.ts @@ -237,7 +237,7 @@ const orderTuple = (...parts: ReadonlyArray): CH.Expr => * a session-bearing trace carry no session id themselves, and keying on that * would file each of them as its own sessionless trace. */ -const sessionKey = (rawSessionId: CH.Expr, traceId: CH.Expr): CH.Expr => +export const sessionKey = (rawSessionId: CH.Expr, traceId: CH.Expr): CH.Expr => CH.if_(rawSessionId.eq(""), CH.concat(MAPLE_AI_TRACE_SESSION_PREFIX, traceId), rawSessionId) /** diff --git a/packages/query-engine-integrations/src/ai/ai-span-columns.ts b/packages/query-engine-integrations/src/ai/ai-span-columns.ts index d7f6b817e..9e5fd0319 100644 --- a/packages/query-engine-integrations/src/ai/ai-span-columns.ts +++ b/packages/query-engine-integrations/src/ai/ai-span-columns.ts @@ -211,3 +211,24 @@ export function sessionUsageSum(netted: string, measure: SessionUsageMeasure): E export function sessionLlmCalls(netted: string): Expr { return CH.rawExpr(`toFloat64(${nettedSum(netted, 2, "toFloat64(n.2)")})`, T.float64) } + +/** + * The session's model calls that carried a PRICE — {@link sessionLlmCalls} + * restricted to the reporters whose netted cost is above zero. + * + * The coverage behind a cost figure, and it has to be netted to be a share of + * anything: `Cost` is whatever the instrumentation reported and nothing prices + * a call Maple-side, so a window's cost is only as complete as the calls that + * carried one — and a gateway that prices the call the app's SDK could not is + * the same call twice until the response id collapses it. + * + * Written out rather than passed through {@link nettedSum}, whose unkeyed half + * reads one element of the tuple: this claim is a condition over two of them. + */ +export function sessionPricedLlmCalls(netted: string): Expr { + const priced = "toFloat64(n.2 AND n.4 > 0)" + return CH.rawExpr( + `arraySum(arrayMap(n -> ${priced}, arrayFilter(n -> n.1 = '', ${netted}))) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, ${priced}), arrayFilter(n -> n.1 != '', ${netted})))))`, + T.float64, + ) +} diff --git a/packages/query-engine-integrations/src/ai/index.ts b/packages/query-engine-integrations/src/ai/index.ts index 6acc7ee69..b170918ed 100644 --- a/packages/query-engine-integrations/src/ai/index.ts +++ b/packages/query-engine-integrations/src/ai/index.ts @@ -40,6 +40,20 @@ export { type AiSessionWindowOutput, } from "./ai-sessions" +export { + aiOverviewBreakdownQuery, + aiOverviewSeriesQuery, + aiOverviewTotalsQuery, + type AiOverviewBreakdownOpts, + type AiOverviewBreakdownOutput, + type AiOverviewFilterOpts, + type AiOverviewMeasuresOutput, + type AiOverviewPeriod, + type AiOverviewSeriesOutput, + type AiOverviewTotalsOutput, + type AiOverviewWindow, +} from "./ai-overview" + export { aiFieldSourceKeys, aiSpanAttributeKeys, diff --git a/packages/query-engine-integrations/src/benchmark/index.ts b/packages/query-engine-integrations/src/benchmark/index.ts index 980c0ae8d..db9ae1b0e 100644 --- a/packages/query-engine-integrations/src/benchmark/index.ts +++ b/packages/query-engine-integrations/src/benchmark/index.ts @@ -64,6 +64,13 @@ const traceWindow = { * because the page was ranked inside it. */ const AI_PAGE_SESSION_IDS = ["wrun_sql_catalog", `${MAPLE_AI_TRACE_SESSION_PREFIX}${AI_TRACE_ID}`] +/** The overview's reads see two windows at once: the caller's, and the one of + * equal length immediately before it that the tiles compare against. */ +const aiCompare = { ...window, prevStartTime: "2025-12-30 06:45:00", prevEndTime: START_TIME } + +/** The same, plus the bucket the chart is cut at. */ +const aiCompareBucketed = { ...aiCompare, bucketSeconds: 300 } + /** Stage two's whole param set — it never sees the caller's window: the * page's bounds for the index levels, and one slice of the padded extent * (`aiSessionDetailsSlices`) for the fan-out. */ @@ -190,6 +197,70 @@ export const integrationFixtures: ReadonlyArray = [ label: "default", compile: () => compileUnionUnsafe(CH.aiSessionFacetsQuery(), window), }, + { + // The overview's tiles: every measure over the window and over the one + // before it, in one read. The netting runs inside an aggregate here, + // which is a shape no other builder emits. + module: "ai-overview", + name: "aiOverviewTotalsQuery", + label: "default", + compile: () => compileUnionUnsafe(CH.aiOverviewTotalsQuery(), aiCompare), + }, + { + // Every filter the sidebar can send at once: the per-trace existence + // tests, plus the session-level `hasErrors` subquery, which is its own + // SQL shape. + module: "ai-overview", + name: "aiOverviewTotalsQuery", + label: "every-filter", + compile: () => + compileUnionUnsafe( + CH.aiOverviewTotalsQuery({ + vendorIds: ["eve"], + serviceNames: ["maple-slack-agent"], + deploymentEnvs: ["production"], + models: ["gpt-5.5"], + agentNames: ["billing-agent"], + toolNames: ["send_email"], + hasErrors: true, + }), + aiCompare, + ), + }, + { + module: "ai-overview", + name: "aiOverviewSeriesQuery", + label: "default", + compile: () => compileUnionUnsafe(CH.aiOverviewSeriesQuery(), aiCompareBucketed), + }, + { + // A model breakdown reads model calls alone and keys off `Model`. + module: "ai-overview", + name: "aiOverviewBreakdownQuery", + label: "model", + compile: () => + compileUnionUnsafe(CH.aiOverviewBreakdownQuery({ dimension: "model" }), aiCompare), + }, + { + // A tool breakdown reads tool calls alone and keys off `ToolName`. + module: "ai-overview", + name: "aiOverviewBreakdownQuery", + label: "tool", + compile: () => + compileUnionUnsafe(CH.aiOverviewBreakdownQuery({ dimension: "tool", limit: 5 }), aiCompare), + }, + { + // The other four dimensions share one shape: every agent span, keyed by + // a column the span always carries. + module: "ai-overview", + name: "aiOverviewBreakdownQuery", + label: "service", + compile: () => + compileUnionUnsafe( + CH.aiOverviewBreakdownQuery({ dimension: "service", hasErrors: true }), + aiCompare, + ), + }, { module: "ai-sessions", name: "aiSessionSpansQuery", From c2c2304cbfad3176a6303315ef258255c94ed4b9 Mon Sep 17 00:00:00 2001 From: JeremyFunk Date: Fri, 11 Sep 2026 05:04:29 +0200 Subject: [PATCH 02/16] fix(agent-sessions): overview error-rate population and previous window MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A model call's failures and its volume are different populations, and the LLM error rate was dividing one by the other. `erroredLlmCalls` is a raw `sumIf` over the model-call SPANS — a failure cannot be netted, the index carries no error flag into the reporters — while `llmCalls` is the netted volume, so a gateway's mirror of a failed call is two failures of one call and the rate passes 100%. The measures carry `llmCallSpans` now, the same spans counted, and the rate is `erroredLlmCalls / llmCallSpans`. The e2e seeds the mirrored call and the call it mirrors as failed and pins all three numbers: three failures over five spans, which net to four calls, and two over two under the model they belong to. The comparison window is `[start - length, start)`. Its upper bound is exclusive now, so the second where the two windows meet is measured in the caller's window alone rather than in both. `AiOverviewBreakdownRow` documents which measures a row can mean under which dimension, since `model` and `tool` restrict the population to their own spans and the rest of the columns are structurally zero. The breakdown's `limit` keeps the one cap the request contract enforces instead of clamping again behind it. --- .../routes/internal/ai-sessions.http.test.ts | 9 ++- .../src/routes/internal/ai-sessions.http.ts | 6 ++ .../ai-overview.clickhouse.e2e.test.ts | 39 ++++++++-- packages/domain/src/http/ai-sessions.ts | 37 +++++++++- .../src/__sql_baseline__/integrations.sql | 71 ++++++++++++++----- .../src/ai/ai-overview.test.ts | 36 ++++++++-- .../src/ai/ai-overview.ts | 71 +++++++++++++------ .../query-engine-integrations/src/ai/index.ts | 1 - 8 files changed, 218 insertions(+), 52 deletions(-) diff --git a/apps/api/src/routes/internal/ai-sessions.http.test.ts b/apps/api/src/routes/internal/ai-sessions.http.test.ts index 33a0f6cb8..7d4c1b188 100644 --- a/apps/api/src/routes/internal/ai-sessions.http.test.ts +++ b/apps/api/src/routes/internal/ai-sessions.http.test.ts @@ -1106,6 +1106,9 @@ const OVERVIEW_MEASURES = { sessions: "4", erroredSessions: "1", llmCalls: 9, + // The failures' own population: the model-call SPANS, mirrors included, so + // the rate the client takes cannot pass 100%. + llmCallSpans: "11", erroredLlmCalls: "2", toolCalls: "6", erroredToolCalls: "1", @@ -1188,7 +1191,11 @@ describe("POST /internal/ai-sessions/overview/summary", () => { // window of a different size. const sql = sqlByContext.get("aiOverviewTotals") ?? "" expect(sql).toContain("Timestamp >= '2026-08-19 07:00:00'") - expect(sql).toContain("Timestamp <= '2026-08-19 09:00:00'") + // `[07:00, 09:00)`: the comparison ends where the caller's window + // begins, so 09:00:00 itself is measured in one window and not in two. + expect(sql).toContain("Timestamp < '2026-08-19 09:00:00'") + expect(sql).not.toContain("Timestamp <= '2026-08-19 09:00:00'") + expect(sql).toContain(`Timestamp >= '${WINDOW.startTime}'`) expect(sql).toContain(`Timestamp <= '${WINDOW.endTime}'`) } finally { await harness.dispose() diff --git a/apps/api/src/routes/internal/ai-sessions.http.ts b/apps/api/src/routes/internal/ai-sessions.http.ts index b4ed0f4e4..0e02bb375 100644 --- a/apps/api/src/routes/internal/ai-sessions.http.ts +++ b/apps/api/src/routes/internal/ai-sessions.http.ts @@ -540,6 +540,10 @@ const warehouseDateTime = (ms: number): string => new Date(ms).toISOString().rep * The window of equal length ending where the caller's begins — the tiles' * comparison. Computed here rather than asked for, so the delta cannot be * quietly taken against a window of a different size. + * + * `prevEndTime` IS the caller's `startTime`: the read bounds the previous + * branch half-open (`[prevStartTime, prevEndTime)`), so the boundary second + * belongs to the current window alone and no session is measured in both. */ const previousWindow = (startTime: string, endTime: string) => { const start = warehouseDateTimeMs(startTime) @@ -554,6 +558,7 @@ const NO_OVERVIEW_MEASURES: AiOverviewMeasures = { sessions: 0, erroredSessions: 0, llmCalls: 0, + llmCallSpans: 0, erroredLlmCalls: 0, toolCalls: 0, erroredToolCalls: 0, @@ -580,6 +585,7 @@ const overviewMeasures = ( sessions: row.sessions, erroredSessions: row.erroredSessions, llmCalls: row.llmCalls, + llmCallSpans: row.llmCallSpans, erroredLlmCalls: row.erroredLlmCalls, toolCalls: row.toolCalls, erroredToolCalls: row.erroredToolCalls, diff --git a/apps/api/src/services/warehouse/ai-overview.clickhouse.e2e.test.ts b/apps/api/src/services/warehouse/ai-overview.clickhouse.e2e.test.ts index b86d663ce..238531e56 100644 --- a/apps/api/src/services/warehouse/ai-overview.clickhouse.e2e.test.ts +++ b/apps/api/src/services/warehouse/ai-overview.clickhouse.e2e.test.ts @@ -55,10 +55,11 @@ const EARLIER_MS = BASE_MS - 2 * HOUR_MS const SESSION_ID = `${ORG_ID}:overview-1` /** The ordinary shape: a turn span that rolls up its two model calls, and a - * tool call that failed. */ + * tool call that failed. Its GPT call failed too. */ const TRACE_TURN = "aioverve2e00000000000000000000001" /** The gateway's own trace of the first model call — same session, same - * response id, a price the app's SDK did not have. */ + * response id, a price the app's SDK did not have, and the SAME failure: one + * call, netted, but two failed model-call spans. */ const TRACE_MIRROR = "aioverve2e00000000000000000000002" /** No session id anywhere, so the trace IS the session. Its model call failed. */ const TRACE_SESSIONLESS = "aioverve2e00000000000000000000003" @@ -116,6 +117,8 @@ const SEED_SPANS: ReadonlyArray = [ ...usage(120, 60, 0.03), }), }, + // The call the gateway mirrors below, and it FAILED — so the same failure is + // on the wire twice while the netting collapses the two spans into one call. { traceId: TRACE_TURN, spanId: "overview-chat-gpt", @@ -123,7 +126,7 @@ const SEED_SPANS: ReadonlyArray = [ name: "chat gpt-5", ms: BASE_MS + 1, durationNs: 4_000_000, - status: "Ok", + status: "Error", attrs: agentSpan({ "gen_ai.operation.name": "chat", "gen_ai.response.model": GPT, @@ -160,14 +163,15 @@ const SEED_SPANS: ReadonlyArray = [ }), }, // The gateway's mirror: its own trace of the GPT call, under the same - // session id and the same response id, priced higher. + // session id and the same response id, priced higher — and carrying the + // call's failure a second time. { traceId: TRACE_MIRROR, spanId: "overview-chat-mirror", name: "chat gpt-5", ms: BASE_MS + 3, durationNs: 5_000_000, - status: "Ok", + status: "Error", attrs: agentSpan({ [MAPLE_AI_SESSION_ID_ATTR]: SESSION_ID, "gen_ai.operation.name": "chat", @@ -387,9 +391,23 @@ describe.skipIf(!clickhouseE2eEnabled)("agent overview reads", () => { // list's own `hasErrors` matches. assert.strictEqual(current?.erroredSessions, list.filter((row) => row.errorAgentSpans > 0).length) assert.strictEqual(current?.erroredSessions, 2) - // One failed tool call, out of one; one failed model call, out of four. + // One failed tool call, out of one. The list counts the DEEPEST failure + // (a failed tool whose child also failed is the child's echo) while this + // counts the failed tool spans; the two agree here because no failed span + // sits under the failed tool. assert.strictEqual(current?.erroredToolCalls, 1) - assert.strictEqual(current?.erroredLlmCalls, 1) + assert.strictEqual(current?.erroredToolCalls, sumOf(list, (row) => row.toolErrors)) + + // The GPT call failed and the gateway mirrored that failure into its own + // trace, so three of the five model-call SPANS failed — while those five + // spans net to four calls. The rate is the failures over the population + // they were counted in, which cannot pass 100%; over `llmCalls` the + // mirrored call would be counted twice against itself. + assert.strictEqual(current?.erroredLlmCalls, 3) + assert.strictEqual(current?.llmCallSpans, 5) + assert.strictEqual(current?.llmCalls, 4) + assert.closeTo(current!.erroredLlmCalls / current!.llmCallSpans, 3 / 5, 1e-9) + assert.isAtMost(current!.erroredLlmCalls / current!.llmCallSpans, 1) // And the filter selects exactly those sessions. const failing = await totals({ hasErrors: true }) @@ -489,6 +507,13 @@ describe.skipIf(!clickhouseE2eEnabled)("agent overview reads", () => { assert.closeTo(byKey.get(GPT)!.cost, 0.05, 1e-9) assert.strictEqual(byKey.get(CLAUDE)?.tokens, 30 + 300) assert.closeTo(byKey.get(CLAUDE)!.cost, 0.11, 1e-9) + // And the mirror is where the two model-call populations part: under GPT, + // two failed spans over two spans, which net to one call. A rate taken + // against the netted call would read 200%. + assert.strictEqual(byKey.get(GPT)?.erroredLlmCalls, 2) + assert.strictEqual(byKey.get(GPT)?.llmCallSpans, 2) + assert.strictEqual(byKey.get(GPT)?.llmCalls, 1) + // The pre-0031 row names no model and is the unattributed key, not a gap. assert.strictEqual(byKey.get("")?.tokens, 500) assert.strictEqual( diff --git a/packages/domain/src/http/ai-sessions.ts b/packages/domain/src/http/ai-sessions.ts index bdad9f12d..3d8e77abe 100644 --- a/packages/domain/src/http/ai-sessions.ts +++ b/packages/domain/src/http/ai-sessions.ts @@ -590,11 +590,20 @@ const aiOverviewMeasures = { sessions: Schema.Number, /** Sessions with at least one failed agent span. */ erroredSessions: Schema.Number, - /** Model calls, netted — the list row's `llmCalls`. */ + /** Model calls, netted — the list row's `llmCalls`. The VOLUME: a wrapper's + * roll-up, a gateway's mirror and a provider retry of one call are one + * call. */ llmCalls: Schema.Number, + /** Model-call SPANS, counted raw — the denominator of the LLM error rate, + * which is `erroredLlmCalls / llmCallSpans` and never `/ llmCalls`. The two + * populations differ by every mirror and wrapper the netting collapses, so + * a mirrored call that failed twice reads as a rate above 100% against the + * netted volume. */ + llmCallSpans: Schema.Number, /** Model-call spans that failed. Not netted: the index carries no error * flag into the netting, so a framework that echoes a failure onto the - * span wrapping the call reports it twice. */ + * span wrapping the call reports it twice — the same span population as + * `llmCallSpans`, which is why those two divide. */ erroredLlmCalls: Schema.Number, toolCalls: Schema.Number, erroredToolCalls: Schema.Number, @@ -687,6 +696,30 @@ export type AiOverviewDimension = Schema.Schema.Type * catalogue: `totalKeys` is what lets it say "+ N more" off the same read. */ export const AI_OVERVIEW_BREAKDOWN_MAX = 12 +/** + * One key of a breakdown, measured over both windows. + * + * A row carries the whole measure set, but which of them MEAN anything depends + * on the dimension, because `model` and `tool` restrict the population to the + * spans that can carry the key — model calls and tool calls respectively: + * + * - `model`: `toolCalls` and `erroredToolCalls` are structurally 0 (a model + * call is not a tool call), and the usage, call and model-latency measures + * are the row's subject. + * - `tool`: `llmCalls`, `llmCallSpans`, `erroredLlmCalls` and + * `llmDurationP*Ns` are structurally 0 (a tool call is not a model call), + * and `cost`, `tokens` and `pricedLlmCalls` are 0 for every tool span that + * reports no usage, which is all of them in practice. `toolCalls` and + * `erroredToolCalls` are the row's subject. + * - `agent`, `service`, `environment`, `vendor`: every agent span carries the + * key, so every measure is meaningful. + * + * `sessions`, `erroredSessions` and `sessionDurationP*Ns` are always over THIS + * key's spans: a session appears under every key it used, its failures are the + * ones its spans under this key carried, and its extent runs from the first of + * those spans to the last rather than across the whole session. A client + * renders the columns the dimension supports rather than a column of zeros. + */ export const AiOverviewBreakdownRow = Schema.Struct({ /** * The dimension's value. `''` is a real key, not a gap — a span that diff --git a/packages/query-engine-integrations/src/__sql_baseline__/integrations.sql b/packages/query-engine-integrations/src/__sql_baseline__/integrations.sql index 050458e45..aba85c62a 100644 --- a/packages/query-engine-integrations/src/__sql_baseline__/integrations.sql +++ b/packages/query-engine-integrations/src/__sql_baseline__/integrations.sql @@ -6,6 +6,7 @@ SELECT count() AS sessions, countIf(errorSpans > 0) AS erroredSessions, sum(toFloat64(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 2)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, toFloat64(n.2)), arrayFilter(n -> n.1 != '', netted))))))) AS llmCalls, + sum(llmCallSpans) AS llmCallSpans, sum(erroredLlmCalls) AS erroredLlmCalls, sum(toolCalls) AS toolCalls, sum(erroredToolCalls) AS erroredToolCalls, @@ -29,6 +30,7 @@ SELECT toolCalls AS toolCalls, erroredToolCalls AS erroredToolCalls, erroredLlmCalls AS erroredLlmCalls, + llmCallSpans AS llmCallSpans, llmDurations AS llmDurations, arrayMap(r -> tuple(r.5, r.6 = 1 AND if((r.3 > 0 OR r.4 > 0), greatest(0., r.3 - arrayElement(tupleElement(childClaims, 2), indexOf(tupleElement(childClaims, 1), r.1))) > 0 OR greatest(0., r.4 - arrayElement(tupleElement(childClaims, 3), indexOf(tupleElement(childClaims, 1), r.1))) > 0, NOT has(reportingIds, r.2)), greatest(0., r.3 - arrayElement(tupleElement(childClaims, 2), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.4 - arrayElement(tupleElement(childClaims, 3), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.7 - arrayElement(tupleElement(childClaims, 4), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.8 - arrayElement(tupleElement(childClaims, 5), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.9 - arrayElement(tupleElement(childClaims, 6), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.10 - arrayElement(tupleElement(childClaims, 7), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.11 - arrayElement(tupleElement(childClaims, 8), indexOf(tupleElement(childClaims, 1), r.1)))), reporters) AS netted FROM (SELECT @@ -40,6 +42,7 @@ SELECT sum(ai_trace_index.IsToolCall) AS toolCalls, sumIf(ai_trace_index.IsError, ai_trace_index.IsToolCall = 1) AS erroredToolCalls, sumIf(ai_trace_index.IsError, ai_trace_index.IsLlmCall = 1) AS erroredLlmCalls, + sum(ai_trace_index.IsLlmCall) AS llmCallSpans, groupArrayIf(2000)(ai_trace_index.Duration, ai_trace_index.IsLlmCall = 1) AS llmDurations, groupArrayIf(2000)(tuple(ai_trace_index.SpanId, ai_trace_index.ParentSpanId, ai_trace_index.Tokens, ai_trace_index.Cost, ai_trace_index.ResponseId, ai_trace_index.IsLlmCall, ai_trace_index.InputTokens, ai_trace_index.CacheReadTokens, ai_trace_index.CacheWriteTokens, ai_trace_index.OutputTokens, ai_trace_index.ReasoningTokens), ((ai_trace_index.Tokens > 0 OR ai_trace_index.Cost > 0) OR ai_trace_index.IsLlmCall = 1)) AS reporters, arrayReduce('sumMap', arrayMap(c -> [c.2], reporters), arrayMap(c -> [c.3], reporters), arrayMap(c -> [c.4], reporters), arrayMap(c -> [c.7], reporters), arrayMap(c -> [c.8], reporters), arrayMap(c -> [c.9], reporters), arrayMap(c -> [c.10], reporters), arrayMap(c -> [c.11], reporters)) AS childClaims, @@ -88,6 +91,7 @@ SELECT count() AS sessions, countIf(errorSpans > 0) AS erroredSessions, sum(toFloat64(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 2)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, toFloat64(n.2)), arrayFilter(n -> n.1 != '', netted))))))) AS llmCalls, + sum(llmCallSpans) AS llmCallSpans, sum(erroredLlmCalls) AS erroredLlmCalls, sum(toolCalls) AS toolCalls, sum(erroredToolCalls) AS erroredToolCalls, @@ -111,6 +115,7 @@ SELECT toolCalls AS toolCalls, erroredToolCalls AS erroredToolCalls, erroredLlmCalls AS erroredLlmCalls, + llmCallSpans AS llmCallSpans, llmDurations AS llmDurations, arrayMap(r -> tuple(r.5, r.6 = 1 AND if((r.3 > 0 OR r.4 > 0), greatest(0., r.3 - arrayElement(tupleElement(childClaims, 2), indexOf(tupleElement(childClaims, 1), r.1))) > 0 OR greatest(0., r.4 - arrayElement(tupleElement(childClaims, 3), indexOf(tupleElement(childClaims, 1), r.1))) > 0, NOT has(reportingIds, r.2)), greatest(0., r.3 - arrayElement(tupleElement(childClaims, 2), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.4 - arrayElement(tupleElement(childClaims, 3), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.7 - arrayElement(tupleElement(childClaims, 4), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.8 - arrayElement(tupleElement(childClaims, 5), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.9 - arrayElement(tupleElement(childClaims, 6), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.10 - arrayElement(tupleElement(childClaims, 7), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.11 - arrayElement(tupleElement(childClaims, 8), indexOf(tupleElement(childClaims, 1), r.1)))), reporters) AS netted FROM (SELECT @@ -122,6 +127,7 @@ SELECT sum(ai_trace_index.IsToolCall) AS toolCalls, sumIf(ai_trace_index.IsError, ai_trace_index.IsToolCall = 1) AS erroredToolCalls, sumIf(ai_trace_index.IsError, ai_trace_index.IsLlmCall = 1) AS erroredLlmCalls, + sum(ai_trace_index.IsLlmCall) AS llmCallSpans, groupArrayIf(2000)(ai_trace_index.Duration, ai_trace_index.IsLlmCall = 1) AS llmDurations, groupArrayIf(2000)(tuple(ai_trace_index.SpanId, ai_trace_index.ParentSpanId, ai_trace_index.Tokens, ai_trace_index.Cost, ai_trace_index.ResponseId, ai_trace_index.IsLlmCall, ai_trace_index.InputTokens, ai_trace_index.CacheReadTokens, ai_trace_index.CacheWriteTokens, ai_trace_index.OutputTokens, ai_trace_index.ReasoningTokens), ((ai_trace_index.Tokens > 0 OR ai_trace_index.Cost > 0) OR ai_trace_index.IsLlmCall = 1)) AS reporters, arrayReduce('sumMap', arrayMap(c -> [c.2], reporters), arrayMap(c -> [c.3], reporters), arrayMap(c -> [c.4], reporters), arrayMap(c -> [c.7], reporters), arrayMap(c -> [c.8], reporters), arrayMap(c -> [c.9], reporters), arrayMap(c -> [c.10], reporters), arrayMap(c -> [c.11], reporters)) AS childClaims, @@ -133,11 +139,11 @@ SELECT FROM ai_trace_index WHERE OrgId = 'org_sql_catalog' AND Timestamp >= '2025-12-30 06:45:00' - AND Timestamp <= '2026-01-01 10:30:00' + AND Timestamp < '2026-01-01 10:30:00' GROUP BY TraceId) AS trace ON ai_trace_index.TraceId = trace.TraceId WHERE ai_trace_index.OrgId = 'org_sql_catalog' AND ai_trace_index.Timestamp >= '2025-12-30 06:45:00' - AND ai_trace_index.Timestamp <= '2026-01-01 10:30:00' + AND ai_trace_index.Timestamp < '2026-01-01 10:30:00' AND ai_trace_index.IsLlmCall = 1 GROUP BY sessionId, key) AS session_rows) AS netted_previous WHERE key IN (SELECT @@ -170,6 +176,7 @@ SELECT 0 AS sessions, 0 AS erroredSessions, 0 AS llmCalls, + 0 AS llmCallSpans, 0 AS erroredLlmCalls, 0 AS toolCalls, 0 AS erroredToolCalls, @@ -208,6 +215,7 @@ SELECT count() AS sessions, countIf(errorSpans > 0) AS erroredSessions, sum(toFloat64(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 2)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, toFloat64(n.2)), arrayFilter(n -> n.1 != '', netted))))))) AS llmCalls, + sum(llmCallSpans) AS llmCallSpans, sum(erroredLlmCalls) AS erroredLlmCalls, sum(toolCalls) AS toolCalls, sum(erroredToolCalls) AS erroredToolCalls, @@ -231,6 +239,7 @@ SELECT toolCalls AS toolCalls, erroredToolCalls AS erroredToolCalls, erroredLlmCalls AS erroredLlmCalls, + llmCallSpans AS llmCallSpans, llmDurations AS llmDurations, arrayMap(r -> tuple(r.5, r.6 = 1 AND if((r.3 > 0 OR r.4 > 0), greatest(0., r.3 - arrayElement(tupleElement(childClaims, 2), indexOf(tupleElement(childClaims, 1), r.1))) > 0 OR greatest(0., r.4 - arrayElement(tupleElement(childClaims, 3), indexOf(tupleElement(childClaims, 1), r.1))) > 0, NOT has(reportingIds, r.2)), greatest(0., r.3 - arrayElement(tupleElement(childClaims, 2), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.4 - arrayElement(tupleElement(childClaims, 3), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.7 - arrayElement(tupleElement(childClaims, 4), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.8 - arrayElement(tupleElement(childClaims, 5), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.9 - arrayElement(tupleElement(childClaims, 6), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.10 - arrayElement(tupleElement(childClaims, 7), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.11 - arrayElement(tupleElement(childClaims, 8), indexOf(tupleElement(childClaims, 1), r.1)))), reporters) AS netted FROM (SELECT @@ -242,6 +251,7 @@ SELECT sum(ai_trace_index.IsToolCall) AS toolCalls, sumIf(ai_trace_index.IsError, ai_trace_index.IsToolCall = 1) AS erroredToolCalls, sumIf(ai_trace_index.IsError, ai_trace_index.IsLlmCall = 1) AS erroredLlmCalls, + sum(ai_trace_index.IsLlmCall) AS llmCallSpans, groupArrayIf(2000)(ai_trace_index.Duration, ai_trace_index.IsLlmCall = 1) AS llmDurations, groupArrayIf(2000)(tuple(ai_trace_index.SpanId, ai_trace_index.ParentSpanId, ai_trace_index.Tokens, ai_trace_index.Cost, ai_trace_index.ResponseId, ai_trace_index.IsLlmCall, ai_trace_index.InputTokens, ai_trace_index.CacheReadTokens, ai_trace_index.CacheWriteTokens, ai_trace_index.OutputTokens, ai_trace_index.ReasoningTokens), ((ai_trace_index.Tokens > 0 OR ai_trace_index.Cost > 0) OR ai_trace_index.IsLlmCall = 1)) AS reporters, arrayReduce('sumMap', arrayMap(c -> [c.2], reporters), arrayMap(c -> [c.3], reporters), arrayMap(c -> [c.4], reporters), arrayMap(c -> [c.7], reporters), arrayMap(c -> [c.8], reporters), arrayMap(c -> [c.9], reporters), arrayMap(c -> [c.10], reporters), arrayMap(c -> [c.11], reporters)) AS childClaims, @@ -320,6 +330,7 @@ SELECT count() AS sessions, countIf(errorSpans > 0) AS erroredSessions, sum(toFloat64(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 2)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, toFloat64(n.2)), arrayFilter(n -> n.1 != '', netted))))))) AS llmCalls, + sum(llmCallSpans) AS llmCallSpans, sum(erroredLlmCalls) AS erroredLlmCalls, sum(toolCalls) AS toolCalls, sum(erroredToolCalls) AS erroredToolCalls, @@ -343,6 +354,7 @@ SELECT toolCalls AS toolCalls, erroredToolCalls AS erroredToolCalls, erroredLlmCalls AS erroredLlmCalls, + llmCallSpans AS llmCallSpans, llmDurations AS llmDurations, arrayMap(r -> tuple(r.5, r.6 = 1 AND if((r.3 > 0 OR r.4 > 0), greatest(0., r.3 - arrayElement(tupleElement(childClaims, 2), indexOf(tupleElement(childClaims, 1), r.1))) > 0 OR greatest(0., r.4 - arrayElement(tupleElement(childClaims, 3), indexOf(tupleElement(childClaims, 1), r.1))) > 0, NOT has(reportingIds, r.2)), greatest(0., r.3 - arrayElement(tupleElement(childClaims, 2), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.4 - arrayElement(tupleElement(childClaims, 3), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.7 - arrayElement(tupleElement(childClaims, 4), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.8 - arrayElement(tupleElement(childClaims, 5), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.9 - arrayElement(tupleElement(childClaims, 6), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.10 - arrayElement(tupleElement(childClaims, 7), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.11 - arrayElement(tupleElement(childClaims, 8), indexOf(tupleElement(childClaims, 1), r.1)))), reporters) AS netted FROM (SELECT @@ -354,6 +366,7 @@ SELECT sum(ai_trace_index.IsToolCall) AS toolCalls, sumIf(ai_trace_index.IsError, ai_trace_index.IsToolCall = 1) AS erroredToolCalls, sumIf(ai_trace_index.IsError, ai_trace_index.IsLlmCall = 1) AS erroredLlmCalls, + sum(ai_trace_index.IsLlmCall) AS llmCallSpans, groupArrayIf(2000)(ai_trace_index.Duration, ai_trace_index.IsLlmCall = 1) AS llmDurations, groupArrayIf(2000)(tuple(ai_trace_index.SpanId, ai_trace_index.ParentSpanId, ai_trace_index.Tokens, ai_trace_index.Cost, ai_trace_index.ResponseId, ai_trace_index.IsLlmCall, ai_trace_index.InputTokens, ai_trace_index.CacheReadTokens, ai_trace_index.CacheWriteTokens, ai_trace_index.OutputTokens, ai_trace_index.ReasoningTokens), ((ai_trace_index.Tokens > 0 OR ai_trace_index.Cost > 0) OR ai_trace_index.IsLlmCall = 1)) AS reporters, arrayReduce('sumMap', arrayMap(c -> [c.2], reporters), arrayMap(c -> [c.3], reporters), arrayMap(c -> [c.4], reporters), arrayMap(c -> [c.7], reporters), arrayMap(c -> [c.8], reporters), arrayMap(c -> [c.9], reporters), arrayMap(c -> [c.10], reporters), arrayMap(c -> [c.11], reporters)) AS childClaims, @@ -365,11 +378,11 @@ SELECT FROM ai_trace_index WHERE OrgId = 'org_sql_catalog' AND Timestamp >= '2025-12-30 06:45:00' - AND Timestamp <= '2026-01-01 10:30:00' + AND Timestamp < '2026-01-01 10:30:00' GROUP BY TraceId) AS trace ON ai_trace_index.TraceId = trace.TraceId WHERE ai_trace_index.OrgId = 'org_sql_catalog' AND ai_trace_index.Timestamp >= '2025-12-30 06:45:00' - AND ai_trace_index.Timestamp <= '2026-01-01 10:30:00' + AND ai_trace_index.Timestamp < '2026-01-01 10:30:00' AND if(trace.rawSessionId = '', concat('trace:', ai_trace_index.TraceId), trace.rawSessionId) IN (SELECT if(trace.rawSessionId = '', concat('trace:', ai_trace_index.TraceId), trace.rawSessionId) AS sessionId FROM ai_trace_index @@ -379,11 +392,11 @@ SELECT FROM ai_trace_index WHERE OrgId = 'org_sql_catalog' AND Timestamp >= '2025-12-30 06:45:00' - AND Timestamp <= '2026-01-01 10:30:00' + AND Timestamp < '2026-01-01 10:30:00' GROUP BY TraceId) AS trace ON ai_trace_index.TraceId = trace.TraceId WHERE ai_trace_index.OrgId = 'org_sql_catalog' AND ai_trace_index.Timestamp >= '2025-12-30 06:45:00' - AND ai_trace_index.Timestamp <= '2026-01-01 10:30:00' + AND ai_trace_index.Timestamp < '2026-01-01 10:30:00' GROUP BY sessionId HAVING sum(ai_trace_index.IsError) > 0) GROUP BY sessionId, key) AS session_rows) AS netted_previous @@ -432,6 +445,7 @@ SELECT 0 AS sessions, 0 AS erroredSessions, 0 AS llmCalls, + 0 AS llmCallSpans, 0 AS erroredLlmCalls, 0 AS toolCalls, 0 AS erroredToolCalls, @@ -485,6 +499,7 @@ SELECT count() AS sessions, countIf(errorSpans > 0) AS erroredSessions, sum(toFloat64(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 2)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, toFloat64(n.2)), arrayFilter(n -> n.1 != '', netted))))))) AS llmCalls, + sum(llmCallSpans) AS llmCallSpans, sum(erroredLlmCalls) AS erroredLlmCalls, sum(toolCalls) AS toolCalls, sum(erroredToolCalls) AS erroredToolCalls, @@ -508,6 +523,7 @@ SELECT toolCalls AS toolCalls, erroredToolCalls AS erroredToolCalls, erroredLlmCalls AS erroredLlmCalls, + llmCallSpans AS llmCallSpans, llmDurations AS llmDurations, arrayMap(r -> tuple(r.5, r.6 = 1 AND if((r.3 > 0 OR r.4 > 0), greatest(0., r.3 - arrayElement(tupleElement(childClaims, 2), indexOf(tupleElement(childClaims, 1), r.1))) > 0 OR greatest(0., r.4 - arrayElement(tupleElement(childClaims, 3), indexOf(tupleElement(childClaims, 1), r.1))) > 0, NOT has(reportingIds, r.2)), greatest(0., r.3 - arrayElement(tupleElement(childClaims, 2), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.4 - arrayElement(tupleElement(childClaims, 3), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.7 - arrayElement(tupleElement(childClaims, 4), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.8 - arrayElement(tupleElement(childClaims, 5), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.9 - arrayElement(tupleElement(childClaims, 6), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.10 - arrayElement(tupleElement(childClaims, 7), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.11 - arrayElement(tupleElement(childClaims, 8), indexOf(tupleElement(childClaims, 1), r.1)))), reporters) AS netted FROM (SELECT @@ -519,6 +535,7 @@ SELECT sum(ai_trace_index.IsToolCall) AS toolCalls, sumIf(ai_trace_index.IsError, ai_trace_index.IsToolCall = 1) AS erroredToolCalls, sumIf(ai_trace_index.IsError, ai_trace_index.IsLlmCall = 1) AS erroredLlmCalls, + sum(ai_trace_index.IsLlmCall) AS llmCallSpans, groupArrayIf(2000)(ai_trace_index.Duration, ai_trace_index.IsLlmCall = 1) AS llmDurations, groupArrayIf(2000)(tuple(ai_trace_index.SpanId, ai_trace_index.ParentSpanId, ai_trace_index.Tokens, ai_trace_index.Cost, ai_trace_index.ResponseId, ai_trace_index.IsLlmCall, ai_trace_index.InputTokens, ai_trace_index.CacheReadTokens, ai_trace_index.CacheWriteTokens, ai_trace_index.OutputTokens, ai_trace_index.ReasoningTokens), ((ai_trace_index.Tokens > 0 OR ai_trace_index.Cost > 0) OR ai_trace_index.IsLlmCall = 1)) AS reporters, arrayReduce('sumMap', arrayMap(c -> [c.2], reporters), arrayMap(c -> [c.3], reporters), arrayMap(c -> [c.4], reporters), arrayMap(c -> [c.7], reporters), arrayMap(c -> [c.8], reporters), arrayMap(c -> [c.9], reporters), arrayMap(c -> [c.10], reporters), arrayMap(c -> [c.11], reporters)) AS childClaims, @@ -567,6 +584,7 @@ SELECT count() AS sessions, countIf(errorSpans > 0) AS erroredSessions, sum(toFloat64(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 2)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, toFloat64(n.2)), arrayFilter(n -> n.1 != '', netted))))))) AS llmCalls, + sum(llmCallSpans) AS llmCallSpans, sum(erroredLlmCalls) AS erroredLlmCalls, sum(toolCalls) AS toolCalls, sum(erroredToolCalls) AS erroredToolCalls, @@ -590,6 +608,7 @@ SELECT toolCalls AS toolCalls, erroredToolCalls AS erroredToolCalls, erroredLlmCalls AS erroredLlmCalls, + llmCallSpans AS llmCallSpans, llmDurations AS llmDurations, arrayMap(r -> tuple(r.5, r.6 = 1 AND if((r.3 > 0 OR r.4 > 0), greatest(0., r.3 - arrayElement(tupleElement(childClaims, 2), indexOf(tupleElement(childClaims, 1), r.1))) > 0 OR greatest(0., r.4 - arrayElement(tupleElement(childClaims, 3), indexOf(tupleElement(childClaims, 1), r.1))) > 0, NOT has(reportingIds, r.2)), greatest(0., r.3 - arrayElement(tupleElement(childClaims, 2), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.4 - arrayElement(tupleElement(childClaims, 3), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.7 - arrayElement(tupleElement(childClaims, 4), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.8 - arrayElement(tupleElement(childClaims, 5), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.9 - arrayElement(tupleElement(childClaims, 6), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.10 - arrayElement(tupleElement(childClaims, 7), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.11 - arrayElement(tupleElement(childClaims, 8), indexOf(tupleElement(childClaims, 1), r.1)))), reporters) AS netted FROM (SELECT @@ -601,6 +620,7 @@ SELECT sum(ai_trace_index.IsToolCall) AS toolCalls, sumIf(ai_trace_index.IsError, ai_trace_index.IsToolCall = 1) AS erroredToolCalls, sumIf(ai_trace_index.IsError, ai_trace_index.IsLlmCall = 1) AS erroredLlmCalls, + sum(ai_trace_index.IsLlmCall) AS llmCallSpans, groupArrayIf(2000)(ai_trace_index.Duration, ai_trace_index.IsLlmCall = 1) AS llmDurations, groupArrayIf(2000)(tuple(ai_trace_index.SpanId, ai_trace_index.ParentSpanId, ai_trace_index.Tokens, ai_trace_index.Cost, ai_trace_index.ResponseId, ai_trace_index.IsLlmCall, ai_trace_index.InputTokens, ai_trace_index.CacheReadTokens, ai_trace_index.CacheWriteTokens, ai_trace_index.OutputTokens, ai_trace_index.ReasoningTokens), ((ai_trace_index.Tokens > 0 OR ai_trace_index.Cost > 0) OR ai_trace_index.IsLlmCall = 1)) AS reporters, arrayReduce('sumMap', arrayMap(c -> [c.2], reporters), arrayMap(c -> [c.3], reporters), arrayMap(c -> [c.4], reporters), arrayMap(c -> [c.7], reporters), arrayMap(c -> [c.8], reporters), arrayMap(c -> [c.9], reporters), arrayMap(c -> [c.10], reporters), arrayMap(c -> [c.11], reporters)) AS childClaims, @@ -612,11 +632,11 @@ SELECT FROM ai_trace_index WHERE OrgId = 'org_sql_catalog' AND Timestamp >= '2025-12-30 06:45:00' - AND Timestamp <= '2026-01-01 10:30:00' + AND Timestamp < '2026-01-01 10:30:00' GROUP BY TraceId) AS trace ON ai_trace_index.TraceId = trace.TraceId WHERE ai_trace_index.OrgId = 'org_sql_catalog' AND ai_trace_index.Timestamp >= '2025-12-30 06:45:00' - AND ai_trace_index.Timestamp <= '2026-01-01 10:30:00' + AND ai_trace_index.Timestamp < '2026-01-01 10:30:00' AND ai_trace_index.IsToolCall = 1 GROUP BY sessionId, key) AS session_rows) AS netted_previous WHERE key IN (SELECT @@ -649,6 +669,7 @@ SELECT 0 AS sessions, 0 AS erroredSessions, 0 AS llmCalls, + 0 AS llmCallSpans, 0 AS erroredLlmCalls, 0 AS toolCalls, 0 AS erroredToolCalls, @@ -687,6 +708,7 @@ SELECT count() AS sessions, countIf(errorSpans > 0) AS erroredSessions, sum(toFloat64(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 2)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, toFloat64(n.2)), arrayFilter(n -> n.1 != '', netted))))))) AS llmCalls, + sum(llmCallSpans) AS llmCallSpans, sum(erroredLlmCalls) AS erroredLlmCalls, sum(toolCalls) AS toolCalls, sum(erroredToolCalls) AS erroredToolCalls, @@ -710,6 +732,7 @@ SELECT toolCalls AS toolCalls, erroredToolCalls AS erroredToolCalls, erroredLlmCalls AS erroredLlmCalls, + llmCallSpans AS llmCallSpans, llmDurations AS llmDurations, arrayMap(r -> tuple(r.5, r.6 = 1 AND if((r.3 > 0 OR r.4 > 0), greatest(0., r.3 - arrayElement(tupleElement(childClaims, 2), indexOf(tupleElement(childClaims, 1), r.1))) > 0 OR greatest(0., r.4 - arrayElement(tupleElement(childClaims, 3), indexOf(tupleElement(childClaims, 1), r.1))) > 0, NOT has(reportingIds, r.2)), greatest(0., r.3 - arrayElement(tupleElement(childClaims, 2), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.4 - arrayElement(tupleElement(childClaims, 3), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.7 - arrayElement(tupleElement(childClaims, 4), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.8 - arrayElement(tupleElement(childClaims, 5), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.9 - arrayElement(tupleElement(childClaims, 6), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.10 - arrayElement(tupleElement(childClaims, 7), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.11 - arrayElement(tupleElement(childClaims, 8), indexOf(tupleElement(childClaims, 1), r.1)))), reporters) AS netted FROM (SELECT @@ -721,6 +744,7 @@ SELECT sum(ai_trace_index.IsToolCall) AS toolCalls, sumIf(ai_trace_index.IsError, ai_trace_index.IsToolCall = 1) AS erroredToolCalls, sumIf(ai_trace_index.IsError, ai_trace_index.IsLlmCall = 1) AS erroredLlmCalls, + sum(ai_trace_index.IsLlmCall) AS llmCallSpans, groupArrayIf(2000)(ai_trace_index.Duration, ai_trace_index.IsLlmCall = 1) AS llmDurations, groupArrayIf(2000)(tuple(ai_trace_index.SpanId, ai_trace_index.ParentSpanId, ai_trace_index.Tokens, ai_trace_index.Cost, ai_trace_index.ResponseId, ai_trace_index.IsLlmCall, ai_trace_index.InputTokens, ai_trace_index.CacheReadTokens, ai_trace_index.CacheWriteTokens, ai_trace_index.OutputTokens, ai_trace_index.ReasoningTokens), ((ai_trace_index.Tokens > 0 OR ai_trace_index.Cost > 0) OR ai_trace_index.IsLlmCall = 1)) AS reporters, arrayReduce('sumMap', arrayMap(c -> [c.2], reporters), arrayMap(c -> [c.3], reporters), arrayMap(c -> [c.4], reporters), arrayMap(c -> [c.7], reporters), arrayMap(c -> [c.8], reporters), arrayMap(c -> [c.9], reporters), arrayMap(c -> [c.10], reporters), arrayMap(c -> [c.11], reporters)) AS childClaims, @@ -746,6 +770,7 @@ SELECT count() AS sessions, countIf(errorSpans > 0) AS erroredSessions, sum(toFloat64(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 2)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, toFloat64(n.2)), arrayFilter(n -> n.1 != '', netted))))))) AS llmCalls, + sum(llmCallSpans) AS llmCallSpans, sum(erroredLlmCalls) AS erroredLlmCalls, sum(toolCalls) AS toolCalls, sum(erroredToolCalls) AS erroredToolCalls, @@ -769,6 +794,7 @@ SELECT toolCalls AS toolCalls, erroredToolCalls AS erroredToolCalls, erroredLlmCalls AS erroredLlmCalls, + llmCallSpans AS llmCallSpans, llmDurations AS llmDurations, arrayMap(r -> tuple(r.5, r.6 = 1 AND if((r.3 > 0 OR r.4 > 0), greatest(0., r.3 - arrayElement(tupleElement(childClaims, 2), indexOf(tupleElement(childClaims, 1), r.1))) > 0 OR greatest(0., r.4 - arrayElement(tupleElement(childClaims, 3), indexOf(tupleElement(childClaims, 1), r.1))) > 0, NOT has(reportingIds, r.2)), greatest(0., r.3 - arrayElement(tupleElement(childClaims, 2), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.4 - arrayElement(tupleElement(childClaims, 3), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.7 - arrayElement(tupleElement(childClaims, 4), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.8 - arrayElement(tupleElement(childClaims, 5), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.9 - arrayElement(tupleElement(childClaims, 6), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.10 - arrayElement(tupleElement(childClaims, 7), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.11 - arrayElement(tupleElement(childClaims, 8), indexOf(tupleElement(childClaims, 1), r.1)))), reporters) AS netted FROM (SELECT @@ -780,6 +806,7 @@ SELECT sum(ai_trace_index.IsToolCall) AS toolCalls, sumIf(ai_trace_index.IsError, ai_trace_index.IsToolCall = 1) AS erroredToolCalls, sumIf(ai_trace_index.IsError, ai_trace_index.IsLlmCall = 1) AS erroredLlmCalls, + sum(ai_trace_index.IsLlmCall) AS llmCallSpans, groupArrayIf(2000)(ai_trace_index.Duration, ai_trace_index.IsLlmCall = 1) AS llmDurations, groupArrayIf(2000)(tuple(ai_trace_index.SpanId, ai_trace_index.ParentSpanId, ai_trace_index.Tokens, ai_trace_index.Cost, ai_trace_index.ResponseId, ai_trace_index.IsLlmCall, ai_trace_index.InputTokens, ai_trace_index.CacheReadTokens, ai_trace_index.CacheWriteTokens, ai_trace_index.OutputTokens, ai_trace_index.ReasoningTokens), ((ai_trace_index.Tokens > 0 OR ai_trace_index.Cost > 0) OR ai_trace_index.IsLlmCall = 1)) AS reporters, arrayReduce('sumMap', arrayMap(c -> [c.2], reporters), arrayMap(c -> [c.3], reporters), arrayMap(c -> [c.4], reporters), arrayMap(c -> [c.7], reporters), arrayMap(c -> [c.8], reporters), arrayMap(c -> [c.9], reporters), arrayMap(c -> [c.10], reporters), arrayMap(c -> [c.11], reporters)) AS childClaims, @@ -791,11 +818,11 @@ SELECT FROM ai_trace_index WHERE OrgId = 'org_sql_catalog' AND Timestamp >= '2025-12-30 06:45:00' - AND Timestamp <= '2026-01-01 10:30:00' + AND Timestamp < '2026-01-01 10:30:00' GROUP BY TraceId) AS trace ON ai_trace_index.TraceId = trace.TraceId WHERE ai_trace_index.OrgId = 'org_sql_catalog' AND ai_trace_index.Timestamp >= '2025-12-30 06:45:00' - AND ai_trace_index.Timestamp <= '2026-01-01 10:30:00' + AND ai_trace_index.Timestamp < '2026-01-01 10:30:00' GROUP BY sessionId) AS session_rows) AS netted_previous GROUP BY bucket ) @@ -808,6 +835,7 @@ SELECT count() AS sessions, countIf(errorSpans > 0) AS erroredSessions, sum(toFloat64(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 2)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, toFloat64(n.2)), arrayFilter(n -> n.1 != '', netted))))))) AS llmCalls, + sum(llmCallSpans) AS llmCallSpans, sum(erroredLlmCalls) AS erroredLlmCalls, sum(toolCalls) AS toolCalls, sum(erroredToolCalls) AS erroredToolCalls, @@ -831,6 +859,7 @@ SELECT toolCalls AS toolCalls, erroredToolCalls AS erroredToolCalls, erroredLlmCalls AS erroredLlmCalls, + llmCallSpans AS llmCallSpans, llmDurations AS llmDurations, arrayMap(r -> tuple(r.5, r.6 = 1 AND if((r.3 > 0 OR r.4 > 0), greatest(0., r.3 - arrayElement(tupleElement(childClaims, 2), indexOf(tupleElement(childClaims, 1), r.1))) > 0 OR greatest(0., r.4 - arrayElement(tupleElement(childClaims, 3), indexOf(tupleElement(childClaims, 1), r.1))) > 0, NOT has(reportingIds, r.2)), greatest(0., r.3 - arrayElement(tupleElement(childClaims, 2), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.4 - arrayElement(tupleElement(childClaims, 3), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.7 - arrayElement(tupleElement(childClaims, 4), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.8 - arrayElement(tupleElement(childClaims, 5), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.9 - arrayElement(tupleElement(childClaims, 6), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.10 - arrayElement(tupleElement(childClaims, 7), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.11 - arrayElement(tupleElement(childClaims, 8), indexOf(tupleElement(childClaims, 1), r.1)))), reporters) AS netted FROM (SELECT @@ -842,6 +871,7 @@ SELECT sum(ai_trace_index.IsToolCall) AS toolCalls, sumIf(ai_trace_index.IsError, ai_trace_index.IsToolCall = 1) AS erroredToolCalls, sumIf(ai_trace_index.IsError, ai_trace_index.IsLlmCall = 1) AS erroredLlmCalls, + sum(ai_trace_index.IsLlmCall) AS llmCallSpans, groupArrayIf(2000)(ai_trace_index.Duration, ai_trace_index.IsLlmCall = 1) AS llmDurations, groupArrayIf(2000)(tuple(ai_trace_index.SpanId, ai_trace_index.ParentSpanId, ai_trace_index.Tokens, ai_trace_index.Cost, ai_trace_index.ResponseId, ai_trace_index.IsLlmCall, ai_trace_index.InputTokens, ai_trace_index.CacheReadTokens, ai_trace_index.CacheWriteTokens, ai_trace_index.OutputTokens, ai_trace_index.ReasoningTokens), ((ai_trace_index.Tokens > 0 OR ai_trace_index.Cost > 0) OR ai_trace_index.IsLlmCall = 1)) AS reporters, arrayReduce('sumMap', arrayMap(c -> [c.2], reporters), arrayMap(c -> [c.3], reporters), arrayMap(c -> [c.4], reporters), arrayMap(c -> [c.7], reporters), arrayMap(c -> [c.8], reporters), arrayMap(c -> [c.9], reporters), arrayMap(c -> [c.10], reporters), arrayMap(c -> [c.11], reporters)) AS childClaims, @@ -865,6 +895,7 @@ SELECT count() AS sessions, countIf(errorSpans > 0) AS erroredSessions, sum(toFloat64(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 2)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, toFloat64(n.2)), arrayFilter(n -> n.1 != '', netted))))))) AS llmCalls, + sum(llmCallSpans) AS llmCallSpans, sum(erroredLlmCalls) AS erroredLlmCalls, sum(toolCalls) AS toolCalls, sum(erroredToolCalls) AS erroredToolCalls, @@ -888,6 +919,7 @@ SELECT toolCalls AS toolCalls, erroredToolCalls AS erroredToolCalls, erroredLlmCalls AS erroredLlmCalls, + llmCallSpans AS llmCallSpans, llmDurations AS llmDurations, arrayMap(r -> tuple(r.5, r.6 = 1 AND if((r.3 > 0 OR r.4 > 0), greatest(0., r.3 - arrayElement(tupleElement(childClaims, 2), indexOf(tupleElement(childClaims, 1), r.1))) > 0 OR greatest(0., r.4 - arrayElement(tupleElement(childClaims, 3), indexOf(tupleElement(childClaims, 1), r.1))) > 0, NOT has(reportingIds, r.2)), greatest(0., r.3 - arrayElement(tupleElement(childClaims, 2), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.4 - arrayElement(tupleElement(childClaims, 3), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.7 - arrayElement(tupleElement(childClaims, 4), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.8 - arrayElement(tupleElement(childClaims, 5), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.9 - arrayElement(tupleElement(childClaims, 6), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.10 - arrayElement(tupleElement(childClaims, 7), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.11 - arrayElement(tupleElement(childClaims, 8), indexOf(tupleElement(childClaims, 1), r.1)))), reporters) AS netted FROM (SELECT @@ -899,6 +931,7 @@ SELECT sum(ai_trace_index.IsToolCall) AS toolCalls, sumIf(ai_trace_index.IsError, ai_trace_index.IsToolCall = 1) AS erroredToolCalls, sumIf(ai_trace_index.IsError, ai_trace_index.IsLlmCall = 1) AS erroredLlmCalls, + sum(ai_trace_index.IsLlmCall) AS llmCallSpans, groupArrayIf(2000)(ai_trace_index.Duration, ai_trace_index.IsLlmCall = 1) AS llmDurations, groupArrayIf(2000)(tuple(ai_trace_index.SpanId, ai_trace_index.ParentSpanId, ai_trace_index.Tokens, ai_trace_index.Cost, ai_trace_index.ResponseId, ai_trace_index.IsLlmCall, ai_trace_index.InputTokens, ai_trace_index.CacheReadTokens, ai_trace_index.CacheWriteTokens, ai_trace_index.OutputTokens, ai_trace_index.ReasoningTokens), ((ai_trace_index.Tokens > 0 OR ai_trace_index.Cost > 0) OR ai_trace_index.IsLlmCall = 1)) AS reporters, arrayReduce('sumMap', arrayMap(c -> [c.2], reporters), arrayMap(c -> [c.3], reporters), arrayMap(c -> [c.4], reporters), arrayMap(c -> [c.7], reporters), arrayMap(c -> [c.8], reporters), arrayMap(c -> [c.9], reporters), arrayMap(c -> [c.10], reporters), arrayMap(c -> [c.11], reporters)) AS childClaims, @@ -910,11 +943,11 @@ SELECT FROM ai_trace_index WHERE OrgId = 'org_sql_catalog' AND Timestamp >= '2025-12-30 06:45:00' - AND Timestamp <= '2026-01-01 10:30:00' + AND Timestamp < '2026-01-01 10:30:00' GROUP BY TraceId) AS trace ON ai_trace_index.TraceId = trace.TraceId WHERE ai_trace_index.OrgId = 'org_sql_catalog' AND ai_trace_index.Timestamp >= '2025-12-30 06:45:00' - AND ai_trace_index.Timestamp <= '2026-01-01 10:30:00' + AND ai_trace_index.Timestamp < '2026-01-01 10:30:00' GROUP BY sessionId) AS session_rows) AS netted_previous FORMAT JSON @@ -924,6 +957,7 @@ SELECT count() AS sessions, countIf(errorSpans > 0) AS erroredSessions, sum(toFloat64(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 2)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, toFloat64(n.2)), arrayFilter(n -> n.1 != '', netted))))))) AS llmCalls, + sum(llmCallSpans) AS llmCallSpans, sum(erroredLlmCalls) AS erroredLlmCalls, sum(toolCalls) AS toolCalls, sum(erroredToolCalls) AS erroredToolCalls, @@ -947,6 +981,7 @@ SELECT toolCalls AS toolCalls, erroredToolCalls AS erroredToolCalls, erroredLlmCalls AS erroredLlmCalls, + llmCallSpans AS llmCallSpans, llmDurations AS llmDurations, arrayMap(r -> tuple(r.5, r.6 = 1 AND if((r.3 > 0 OR r.4 > 0), greatest(0., r.3 - arrayElement(tupleElement(childClaims, 2), indexOf(tupleElement(childClaims, 1), r.1))) > 0 OR greatest(0., r.4 - arrayElement(tupleElement(childClaims, 3), indexOf(tupleElement(childClaims, 1), r.1))) > 0, NOT has(reportingIds, r.2)), greatest(0., r.3 - arrayElement(tupleElement(childClaims, 2), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.4 - arrayElement(tupleElement(childClaims, 3), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.7 - arrayElement(tupleElement(childClaims, 4), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.8 - arrayElement(tupleElement(childClaims, 5), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.9 - arrayElement(tupleElement(childClaims, 6), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.10 - arrayElement(tupleElement(childClaims, 7), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.11 - arrayElement(tupleElement(childClaims, 8), indexOf(tupleElement(childClaims, 1), r.1)))), reporters) AS netted FROM (SELECT @@ -958,6 +993,7 @@ SELECT sum(ai_trace_index.IsToolCall) AS toolCalls, sumIf(ai_trace_index.IsError, ai_trace_index.IsToolCall = 1) AS erroredToolCalls, sumIf(ai_trace_index.IsError, ai_trace_index.IsLlmCall = 1) AS erroredLlmCalls, + sum(ai_trace_index.IsLlmCall) AS llmCallSpans, groupArrayIf(2000)(ai_trace_index.Duration, ai_trace_index.IsLlmCall = 1) AS llmDurations, groupArrayIf(2000)(tuple(ai_trace_index.SpanId, ai_trace_index.ParentSpanId, ai_trace_index.Tokens, ai_trace_index.Cost, ai_trace_index.ResponseId, ai_trace_index.IsLlmCall, ai_trace_index.InputTokens, ai_trace_index.CacheReadTokens, ai_trace_index.CacheWriteTokens, ai_trace_index.OutputTokens, ai_trace_index.ReasoningTokens), ((ai_trace_index.Tokens > 0 OR ai_trace_index.Cost > 0) OR ai_trace_index.IsLlmCall = 1)) AS reporters, arrayReduce('sumMap', arrayMap(c -> [c.2], reporters), arrayMap(c -> [c.3], reporters), arrayMap(c -> [c.4], reporters), arrayMap(c -> [c.7], reporters), arrayMap(c -> [c.8], reporters), arrayMap(c -> [c.9], reporters), arrayMap(c -> [c.10], reporters), arrayMap(c -> [c.11], reporters)) AS childClaims, @@ -1009,6 +1045,7 @@ SELECT count() AS sessions, countIf(errorSpans > 0) AS erroredSessions, sum(toFloat64(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 2)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, toFloat64(n.2)), arrayFilter(n -> n.1 != '', netted))))))) AS llmCalls, + sum(llmCallSpans) AS llmCallSpans, sum(erroredLlmCalls) AS erroredLlmCalls, sum(toolCalls) AS toolCalls, sum(erroredToolCalls) AS erroredToolCalls, @@ -1032,6 +1069,7 @@ SELECT toolCalls AS toolCalls, erroredToolCalls AS erroredToolCalls, erroredLlmCalls AS erroredLlmCalls, + llmCallSpans AS llmCallSpans, llmDurations AS llmDurations, arrayMap(r -> tuple(r.5, r.6 = 1 AND if((r.3 > 0 OR r.4 > 0), greatest(0., r.3 - arrayElement(tupleElement(childClaims, 2), indexOf(tupleElement(childClaims, 1), r.1))) > 0 OR greatest(0., r.4 - arrayElement(tupleElement(childClaims, 3), indexOf(tupleElement(childClaims, 1), r.1))) > 0, NOT has(reportingIds, r.2)), greatest(0., r.3 - arrayElement(tupleElement(childClaims, 2), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.4 - arrayElement(tupleElement(childClaims, 3), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.7 - arrayElement(tupleElement(childClaims, 4), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.8 - arrayElement(tupleElement(childClaims, 5), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.9 - arrayElement(tupleElement(childClaims, 6), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.10 - arrayElement(tupleElement(childClaims, 7), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.11 - arrayElement(tupleElement(childClaims, 8), indexOf(tupleElement(childClaims, 1), r.1)))), reporters) AS netted FROM (SELECT @@ -1043,6 +1081,7 @@ SELECT sum(ai_trace_index.IsToolCall) AS toolCalls, sumIf(ai_trace_index.IsError, ai_trace_index.IsToolCall = 1) AS erroredToolCalls, sumIf(ai_trace_index.IsError, ai_trace_index.IsLlmCall = 1) AS erroredLlmCalls, + sum(ai_trace_index.IsLlmCall) AS llmCallSpans, groupArrayIf(2000)(ai_trace_index.Duration, ai_trace_index.IsLlmCall = 1) AS llmDurations, groupArrayIf(2000)(tuple(ai_trace_index.SpanId, ai_trace_index.ParentSpanId, ai_trace_index.Tokens, ai_trace_index.Cost, ai_trace_index.ResponseId, ai_trace_index.IsLlmCall, ai_trace_index.InputTokens, ai_trace_index.CacheReadTokens, ai_trace_index.CacheWriteTokens, ai_trace_index.OutputTokens, ai_trace_index.ReasoningTokens), ((ai_trace_index.Tokens > 0 OR ai_trace_index.Cost > 0) OR ai_trace_index.IsLlmCall = 1)) AS reporters, arrayReduce('sumMap', arrayMap(c -> [c.2], reporters), arrayMap(c -> [c.3], reporters), arrayMap(c -> [c.4], reporters), arrayMap(c -> [c.7], reporters), arrayMap(c -> [c.8], reporters), arrayMap(c -> [c.9], reporters), arrayMap(c -> [c.10], reporters), arrayMap(c -> [c.11], reporters)) AS childClaims, @@ -1054,7 +1093,7 @@ SELECT FROM ai_trace_index WHERE OrgId = 'org_sql_catalog' AND Timestamp >= '2025-12-30 06:45:00' - AND Timestamp <= '2026-01-01 10:30:00' + AND Timestamp < '2026-01-01 10:30:00' GROUP BY TraceId HAVING countIf(VendorId IN ('eve')) > 0 AND countIf(ServiceName IN ('maple-slack-agent')) > 0 @@ -1064,7 +1103,7 @@ SELECT AND countIf(ToolName IN ('send_email')) > 0) AS trace ON ai_trace_index.TraceId = trace.TraceId WHERE ai_trace_index.OrgId = 'org_sql_catalog' AND ai_trace_index.Timestamp >= '2025-12-30 06:45:00' - AND ai_trace_index.Timestamp <= '2026-01-01 10:30:00' + AND ai_trace_index.Timestamp < '2026-01-01 10:30:00' AND if(trace.rawSessionId = '', concat('trace:', ai_trace_index.TraceId), trace.rawSessionId) IN (SELECT if(trace.rawSessionId = '', concat('trace:', ai_trace_index.TraceId), trace.rawSessionId) AS sessionId FROM ai_trace_index @@ -1074,7 +1113,7 @@ SELECT FROM ai_trace_index WHERE OrgId = 'org_sql_catalog' AND Timestamp >= '2025-12-30 06:45:00' - AND Timestamp <= '2026-01-01 10:30:00' + AND Timestamp < '2026-01-01 10:30:00' GROUP BY TraceId HAVING countIf(VendorId IN ('eve')) > 0 AND countIf(ServiceName IN ('maple-slack-agent')) > 0 @@ -1084,7 +1123,7 @@ SELECT AND countIf(ToolName IN ('send_email')) > 0) AS trace ON ai_trace_index.TraceId = trace.TraceId WHERE ai_trace_index.OrgId = 'org_sql_catalog' AND ai_trace_index.Timestamp >= '2025-12-30 06:45:00' - AND ai_trace_index.Timestamp <= '2026-01-01 10:30:00' + AND ai_trace_index.Timestamp < '2026-01-01 10:30:00' GROUP BY sessionId HAVING sum(ai_trace_index.IsError) > 0) GROUP BY sessionId) AS session_rows) AS netted_previous diff --git a/packages/query-engine-integrations/src/ai/ai-overview.test.ts b/packages/query-engine-integrations/src/ai/ai-overview.test.ts index 816eb3101..59872d484 100644 --- a/packages/query-engine-integrations/src/ai/ai-overview.test.ts +++ b/packages/query-engine-integrations/src/ai/ai-overview.test.ts @@ -97,6 +97,20 @@ describe("overview population", () => { expect(unfiltered).not.toContain("countIf(VendorId") }) + it("bounds the comparison window half-open, so the boundary is measured once", () => { + const sql = totalsSql() + + // `[prevStartTime, startTime)`: the previous window ends where the + // caller's begins, and every level of the current branch still takes the + // closed window every other Maple read takes. + expect(sql).toContain(`Timestamp >= '${params.prevStartTime}'`) + expect(sql).toContain(`Timestamp < '${params.prevEndTime}'`) + expect(sql).toContain(`Timestamp <= '${params.endTime}'`) + // `prevEndTime` IS `startTime`, so a row on it would otherwise land in + // both windows. + expect(sql).not.toContain(`Timestamp <= '${params.prevEndTime}'`) + }) + it("selects the sessions that failed with the list's session-level rule", () => { const sql = totalsSql({ hasErrors: true }) @@ -125,6 +139,20 @@ describe("the measures every grouping reports", () => { expect(totalsSql()).not.toContain("toStartOfInterval") }) + it("gives the model-call failures a denominator of their own population", () => { + const sql = totalsSql() + + // The numerator is a span `sumIf` — a failure cannot be netted, the index + // carries no error flag into the reporters — so the denominator counts + // the same spans. Against the netted `llmCalls`, a mirrored call that + // failed on both observations is a rate above 100%. + expect(sql).toContain( + "sumIf(ai_trace_index.IsError, ai_trace_index.IsLlmCall = 1) AS erroredLlmCalls", + ) + expect(sql).toContain("sum(ai_trace_index.IsLlmCall) AS llmCallSpans") + expect(sql).toContain("sum(llmCallSpans) AS llmCallSpans") + }) + it("guards every quantile against the empty group", () => { const sql = totalsSql() @@ -194,11 +222,11 @@ describe("the breakdown's dimensions", () => { expect(sql).toContain("'keys' AS period") }) - it("never ranks more keys than the table can show", () => { + it("ranks the keys the caller asked for, and the table's own cap by default", () => { expect(breakdownSql({ dimension: "tool", limit: 3 })).toContain("LIMIT 3") - expect(breakdownSql({ dimension: "tool", limit: 500 })).toContain( - `LIMIT ${AI_OVERVIEW_BREAKDOWN_MAX}`, - ) + // The cap is the request contract's — a `limit` past it is a 400 and + // never reaches the builder, so nothing re-clamps it here. + expect(breakdownSql({ dimension: "tool" })).toContain(`LIMIT ${AI_OVERVIEW_BREAKDOWN_MAX}`) }) it("groups by the key and by nothing else, so a session counts once per key", () => { diff --git a/packages/query-engine-integrations/src/ai/ai-overview.ts b/packages/query-engine-integrations/src/ai/ai-overview.ts index c69c74e6e..6ddf45117 100644 --- a/packages/query-engine-integrations/src/ai/ai-overview.ts +++ b/packages/query-engine-integrations/src/ai/ai-overview.ts @@ -47,9 +47,19 @@ // read over every agent span, and a span that names no value keys under `''`, // which the page renders as unattributed rather than hiding. // +// MODEL CALLS are counted over two populations, because volume and failures +// cannot share one. `llmCalls` is the netted volume: a wrapper's roll-up, a +// gateway's mirror and a provider retry of one call are one call. Failures +// cannot be netted at all — the index carries no error flag into the reporters +// — so `erroredLlmCalls` is a raw `sumIf` over the model-call SPANS, and +// `llmCallSpans` counts exactly those spans so the two divide. Read against +// `llmCalls`, a mirrored call that failed on both observations is two failures +// of one call and the rate passes 100%. +// // Durations stay in NANOSECONDS, like every other AI read — `Duration` is what // the index stores and the client formats. +import type { DateTime } from "effect" import * as CH from "@maple-dev/effect-clickhouse/expr" import * as T from "@maple-dev/effect-clickhouse/types" import { compile } from "@maple-dev/effect-clickhouse/sql" @@ -87,8 +97,9 @@ export interface AiOverviewFilterOpts { export interface AiOverviewBreakdownOpts extends AiOverviewFilterOpts { readonly dimension: AiOverviewDimension - /** Keys returned per period. Defaults to — and is capped at — - * {@link AI_OVERVIEW_BREAKDOWN_MAX}. */ + /** Keys returned per period. Defaults to {@link AI_OVERVIEW_BREAKDOWN_MAX}, + * which is also where the request contract caps it — a larger `limit` is a + * 400 and never reaches here, so there is nothing to clamp twice. */ readonly limit?: number } @@ -99,7 +110,7 @@ export interface AiOverviewBreakdownOpts extends AiOverviewFilterOpts { * because a `LowCardinality(String)` on one branch against a `String` on * another is a `NO_COMMON_TYPE`. */ -export type AiOverviewWindow = "current" | "previous" +type AiOverviewWindow = "current" | "previous" /** Which window a row measures. `keys` is the breakdown's third branch: how * many distinct keys the current window has, before the top-N cut. */ @@ -110,6 +121,24 @@ const startParam = (window: AiOverviewWindow) => const endParam = (window: AiOverviewWindow) => param.dateTimeString(window === "current" ? "endTime" : "prevEndTime") +/** + * The window's bounds on a row's timestamp, on every level that reads the + * index. + * + * The caller's window is CLOSED at both ends, the way every other Maple read + * takes one. The comparison window is `[start − length, start)`: it ends where + * the caller's begins, so its upper bound is EXCLUSIVE and a row sitting + * exactly on the boundary belongs to the current window alone rather than to + * both. + */ +const withinWindow = ( + timestamp: CH.Expr, + window: AiOverviewWindow, +): ReadonlyArray => [ + timestamp.gte(startParam(window)), + window === "current" ? timestamp.lte(endParam(window)) : timestamp.lt(endParam(window)), +] + /** * One row per agent trace of the window that passes the selection: its id and * the session it is filed under. @@ -126,11 +155,7 @@ const traceKeys = (opts: AiOverviewFilterOpts, window: AiOverviewWindow) => { const carries = (cond: CH.Condition) => CH.countIf(cond).gt(0) return from(AiTraceIndex) .select(($) => ({ TraceId: $.TraceId, rawSessionId: CH.max_($.SessionId) })) - .where(($) => [ - $.OrgId.eq(param.string("orgId")), - $.Timestamp.gte(startParam(window)), - $.Timestamp.lte(endParam(window)), - ]) + .where(($) => [$.OrgId.eq(param.string("orgId")), ...withinWindow($.Timestamp, window)]) .groupBy("TraceId") .having(($) => [ CH.when(values(opts.vendorIds), (v) => carries(CH.inList($.VendorId, v))), @@ -155,11 +180,7 @@ const erroredSessionKeys = (opts: AiOverviewFilterOpts, window: AiOverviewWindow from(AiTraceIndex) .innerJoinQuery(traceKeys(opts, window), "trace", (row, trace) => row.TraceId.eq(trace.TraceId)) .select(($) => ({ sessionId: sessionKey($.trace.rawSessionId, $.TraceId) })) - .where(($) => [ - $.OrgId.eq(param.string("orgId")), - $.Timestamp.gte(startParam(window)), - $.Timestamp.lte(endParam(window)), - ]) + .where(($) => [$.OrgId.eq(param.string("orgId")), ...withinWindow($.Timestamp, window)]) .groupBy("sessionId") .having(($) => [CH.sum($.IsError).gt(0)]) @@ -273,6 +294,11 @@ const sessionRows = ( // error flag into the reporters — so a framework that echoes a // failure onto the span wrapping the call reports it twice. erroredLlmCalls: CH.sumIf($.IsError, $.IsLlmCall.eq(1)), + // Its denominator: the SAME spans, counted. The netted `llmCalls` + // below measures a different population — one mirrored call is one + // call there and two failures above — so an error rate taken against + // it can exceed 100%. + llmCallSpans: CH.sum($.IsLlmCall), llmDurations: llmDurationsExpr($), // Usage AND model calls travel as reporters: both are counted above, // where every span of the session is in hand — see `ai-span-columns`. @@ -284,8 +310,7 @@ const sessionRows = ( })) .where(($) => [ $.OrgId.eq(param.string("orgId")), - $.Timestamp.gte(startParam(window)), - $.Timestamp.lte(endParam(window)), + ...withinWindow($.Timestamp, window), population === undefined ? undefined : population($), CH.whenTrue(opts.hasErrors, () => inSubquery(sessionKey($.trace.rawSessionId, $.TraceId), erroredSessionKeys(opts, window)), @@ -306,6 +331,7 @@ const nettedRows = (opts: AiOverviewFilterOpts, window: AiOverviewWindow, dimens toolCalls: $.toolCalls, erroredToolCalls: $.erroredToolCalls, erroredLlmCalls: $.erroredLlmCalls, + llmCallSpans: $.llmCallSpans, llmDurations: $.llmDurations, netted: nettedReportersExpr("reporters", "childClaims", "reportingIds"), })) @@ -317,6 +343,7 @@ interface SessionColumns { readonly toolCalls: CH.Expr readonly erroredToolCalls: CH.Expr readonly erroredLlmCalls: CH.Expr + readonly llmCallSpans: CH.Expr } /** @@ -337,6 +364,7 @@ const measures = ($: SessionColumns) => ({ sessions: CH.count(), erroredSessions: CH.countIf($.errorSpans.gt(0)), llmCalls: CH.sum(sessionLlmCalls("netted")), + llmCallSpans: CH.sum($.llmCallSpans), erroredLlmCalls: CH.sum($.erroredLlmCalls), toolCalls: CH.sum($.toolCalls), erroredToolCalls: CH.sum($.erroredToolCalls), @@ -361,6 +389,7 @@ const noMeasures = () => ({ sessions: CH.lit(0), erroredSessions: CH.lit(0), llmCalls: CH.lit(0), + llmCallSpans: CH.lit(0), erroredLlmCalls: CH.lit(0), toolCalls: CH.lit(0), erroredToolCalls: CH.lit(0), @@ -382,6 +411,7 @@ export interface AiOverviewMeasuresOutput { readonly sessions: number readonly erroredSessions: number readonly llmCalls: number + readonly llmCallSpans: number readonly erroredLlmCalls: number readonly toolCalls: number readonly erroredToolCalls: number @@ -424,7 +454,8 @@ export interface AiOverviewBreakdownOutput extends AiOverviewTotalsOutput { * available from a read that grouped the window. The previous branch is bounded * by its own pair of params (`prevStartTime`/`prevEndTime`), which the caller * computes — the query has no opinion about what "previous" means beyond - * reading a second window. + * reading a second window, half-open at its upper bound so a session on the + * boundary is measured once (see {@link withinWindow}). */ export function aiOverviewTotalsQuery(opts: AiOverviewFilterOpts = {}): CHUnionQuery { const branch = (window: AiOverviewWindow) => @@ -479,8 +510,7 @@ const topKeys = (opts: AiOverviewBreakdownOpts) => { })) .where(($) => [ $.OrgId.eq(param.string("orgId")), - $.Timestamp.gte(startParam("current")), - $.Timestamp.lte(endParam("current")), + ...withinWindow($.Timestamp, "current"), population === undefined ? undefined : population($), CH.whenTrue(opts.hasErrors, () => inSubquery(sessionKey($.trace.rawSessionId, $.TraceId), erroredSessionKeys(opts, "current")), @@ -488,7 +518,7 @@ const topKeys = (opts: AiOverviewBreakdownOpts) => { ]) .groupBy("rankKey") .orderBy(["rankSessions", "desc"], ["rankKey", "asc"]) - .limit(Math.min(opts.limit ?? AI_OVERVIEW_BREAKDOWN_MAX, AI_OVERVIEW_BREAKDOWN_MAX)) + .limit(opts.limit ?? AI_OVERVIEW_BREAKDOWN_MAX) return fromQuery(ranked, "top_keys").select(($) => ({ topKey: $.rankKey })) } @@ -529,8 +559,7 @@ export function aiOverviewBreakdownQuery( })) .where(($) => [ $.OrgId.eq(param.string("orgId")), - $.Timestamp.gte(startParam("current")), - $.Timestamp.lte(endParam("current")), + ...withinWindow($.Timestamp, "current"), population === undefined ? undefined : population($), CH.whenTrue(opts.hasErrors, () => inSubquery(sessionKey($.trace.rawSessionId, $.TraceId), erroredSessionKeys(opts, "current")), diff --git a/packages/query-engine-integrations/src/ai/index.ts b/packages/query-engine-integrations/src/ai/index.ts index b170918ed..67940aab2 100644 --- a/packages/query-engine-integrations/src/ai/index.ts +++ b/packages/query-engine-integrations/src/ai/index.ts @@ -51,7 +51,6 @@ export { type AiOverviewPeriod, type AiOverviewSeriesOutput, type AiOverviewTotalsOutput, - type AiOverviewWindow, } from "./ai-overview" export { From bdd7b8f55d6ed4ed2c54e373d26a3ee13433d138 Mon Sep 17 00:00:00 2001 From: JeremyFunk Date: Fri, 11 Sep 2026 05:08:57 +0200 Subject: [PATCH 03/16] test(agent-sessions): build the overview e2e usage attrs without a conditional spread --- .../warehouse/ai-overview.clickhouse.e2e.test.ts | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/apps/api/src/services/warehouse/ai-overview.clickhouse.e2e.test.ts b/apps/api/src/services/warehouse/ai-overview.clickhouse.e2e.test.ts index 238531e56..79aefa1ec 100644 --- a/apps/api/src/services/warehouse/ai-overview.clickhouse.e2e.test.ts +++ b/apps/api/src/services/warehouse/ai-overview.clickhouse.e2e.test.ts @@ -93,12 +93,14 @@ const agentSpan = (attrs: Readonly>) => ({ }) /** Tokens and a price, under the canonical semconv keys. */ -const usage = (input: number, output: number, cost: number, responseId?: string) => ({ - "gen_ai.usage.input_tokens": String(input), - "gen_ai.usage.output_tokens": String(output), - "gen_ai.usage.cost": String(cost), - ...(responseId === undefined ? {} : { "gen_ai.response.id": responseId }), -}) +const usage = (input: number, output: number, cost: number, responseId?: string) => { + const base = { + "gen_ai.usage.input_tokens": String(input), + "gen_ai.usage.output_tokens": String(output), + "gen_ai.usage.cost": String(cost), + } + return responseId === undefined ? base : { ...base, "gen_ai.response.id": responseId } +} const SEED_SPANS: ReadonlyArray = [ // The turn span carries the session id AND its children's usage summed onto From 5dc2b0f06c0dfca954459c97e1fe224cf522a5c8 Mon Sep 17 00:00:00 2001 From: JeremyFunk Date: Fri, 11 Sep 2026 06:00:54 +0200 Subject: [PATCH 04/16] feat(agent-sessions): overview model-mix series MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `/overview/model-mix` returns the window's model-call SPANS split by model, bucket by bucket — the band chart beside the tiles. It takes the summary's request shape (the window, a bucket width, the same six filters and `hasErrors`, now one shared bag) and answers the current window alone: the chart has no comparison band. The population is the raw one, not the netted one. `llmCallSpans` is what the tiles already report and what the LLM error rate divides by, so a gateway's mirror of a call is a band under the model it names, twice; netting would charge a mirrored call to one model and leave the bands disagreeing with the rate above them. A call whose instrumentation named no model has no share of a model mix and is left out, which is the only difference between this read's total and the tile's. A span is filed under the bucket its own timestamp falls in — the rows are spans, so there is no session to keep whole — while sessions are selected the way every other overview read selects them, so the mix describes the sessions the tiles measure. `LIMIT AI_OVERVIEW_MODEL_MIX_MAX_ROWS` is a blow-up guard and not a top-N: the client folds the minor models into an "other" band and needs every model of every bucket to do it. --- .../routes/internal/ai-sessions.http.test.ts | 71 +++++++++++++++++ .../src/routes/internal/ai-sessions.http.ts | 30 +++++++ .../ai-overview.clickhouse.e2e.test.ts | 55 +++++++++++++ packages/domain/src/http/ai-sessions.ts | 55 ++++++++++++- .../src/__sql_baseline__/integrations.sql | 24 ++++++ .../src/ai/ai-overview.test.ts | 79 ++++++++++++++++++- .../src/ai/ai-overview.ts | 56 +++++++++++++ .../query-engine-integrations/src/ai/index.ts | 2 + .../src/benchmark/index.ts | 9 +++ 9 files changed, 375 insertions(+), 6 deletions(-) diff --git a/apps/api/src/routes/internal/ai-sessions.http.test.ts b/apps/api/src/routes/internal/ai-sessions.http.test.ts index 7d4c1b188..7f2194cca 100644 --- a/apps/api/src/routes/internal/ai-sessions.http.test.ts +++ b/apps/api/src/routes/internal/ai-sessions.http.test.ts @@ -1349,3 +1349,74 @@ describe("POST /internal/ai-sessions/overview/breakdown", () => { } }) }) + +describe("POST /internal/ai-sessions/overview/model-mix", () => { + const MODEL_MIX_BODY = { ...WINDOW, bucketSeconds: 300 } + + /** `count()` arrives quoted from a BYO-ClickHouse cluster and as a number + * from managed Tinybird; the row schema has to take both. */ + const ROWS = [ + { bucket: "2026-08-19T09:00:00.000Z", model: "gpt-5.5", llmCallSpans: "5" }, + { bucket: "2026-08-19T09:00:00.000Z", model: "claude-sonnet-5", llmCallSpans: 2 }, + { bucket: "2026-08-19T10:00:00.000Z", model: "gpt-5.5", llmCallSpans: "3" }, + ] + + const modelMixHarness = () => { + const contexts: Array = [] + let sql: string | undefined + const harness = makeHarness({ + compiledQuery: (_tenant, compiled, options) => { + contexts.push(options?.context) + sql = compiledQueryOf(compiled).sql + return compiledQueryOf(compiled).decodeRows(ROWS).pipe(Effect.orDie) + }, + }) + return { harness, contexts, readSql: () => sql ?? "" } + } + + it("counts the model-call spans of the window, bucket by bucket", async () => { + const { harness, contexts, readSql } = modelMixHarness() + + try { + const response = await harness.post("/internal/ai-sessions/overview/model-mix", MODEL_MIX_BODY) + expect(response.status).toBe(200) + expect(contexts).toEqual(["aiOverviewModelMix"]) + // One index read, cut at the width the caller asked for, over the + // model-call spans alone. + expect(readSql()).toContain("FROM ai_trace_index") + expect(readSql()).toContain("INTERVAL 300 SECOND") + expect(readSql()).toContain("AND ai_trace_index.IsLlmCall = 1") + expect(readSql()).not.toContain("__PARAM_") + // The caller's window alone — no comparison band, so no second pair of + // bounds. + expect(readSql()).toContain(`Timestamp >= '${WINDOW.startTime}'`) + expect(readSql()).not.toContain("2026-08-19 07:00:00") + expect(response.body).toMatchObject({ + bucketSeconds: 300, + rows: [ + { bucket: "2026-08-19T09:00:00.000Z", model: "gpt-5.5", llmCallSpans: 5 }, + { bucket: "2026-08-19T09:00:00.000Z", model: "claude-sonnet-5", llmCallSpans: 2 }, + { bucket: "2026-08-19T10:00:00.000Z", model: "gpt-5.5", llmCallSpans: 3 }, + ], + }) + } finally { + await harness.dispose() + } + }) + + it("refuses a fractional bucket with a 400 rather than a 500", async () => { + const harness = makeHarness({ compiledQuery: () => Effect.die("the read must never run") }) + + try { + const response = await harness.post("/internal/ai-sessions/overview/model-mix", { + ...WINDOW, + bucketSeconds: 1.5, + }) + // `param.int` rejects a fraction inside the builder, which would be a + // 500 — the contract catches it at the boundary instead. + expect(response.status).toBe(400) + } finally { + await harness.dispose() + } + }) +}) diff --git a/apps/api/src/routes/internal/ai-sessions.http.ts b/apps/api/src/routes/internal/ai-sessions.http.ts index 0e02bb375..46e1303de 100644 --- a/apps/api/src/routes/internal/ai-sessions.http.ts +++ b/apps/api/src/routes/internal/ai-sessions.http.ts @@ -1,6 +1,7 @@ import { HttpApiBuilder } from "effect/unstable/httpapi" import { AiOverviewBreakdownResponse, + AiOverviewModelMixResponse, AiOverviewSummaryResponse, AiSessionTooLargeError, AI_SESSION_SPANS_MAX_SPANS, @@ -505,6 +506,35 @@ export const HttpAiSessionsInternalLive = HttpApiBuilder.group( }) }), ) + .handle("overviewModelMix", ({ payload }) => + Effect.gen(function* () { + const tenant = yield* CurrentTenant.Context + yield* Effect.annotateCurrentSpan({ + orgId: tenant.orgId, + "maple.ai.overview.bucket_seconds": payload.bucketSeconds, + }) + // No comparison window: the chart plots the selected window's + // bands and nothing behind them. + const rows = yield* warehouse.compiledQuery( + tenant, + CH.compile(Integrations.aiOverviewModelMixQuery(overviewSelection(payload)), { + orgId: tenant.orgId, + startTime: payload.startTime, + endTime: payload.endTime, + bucketSeconds: payload.bucketSeconds, + }), + { context: "aiOverviewModelMix" }, + ) + return new AiOverviewModelMixResponse({ + bucketSeconds: payload.bucketSeconds, + rows: rows.map((row) => ({ + bucket: row.bucket, + model: row.model, + llmCallSpans: row.llmCallSpans, + })), + }) + }), + ) }), ) diff --git a/apps/api/src/services/warehouse/ai-overview.clickhouse.e2e.test.ts b/apps/api/src/services/warehouse/ai-overview.clickhouse.e2e.test.ts index 79aefa1ec..84c961dcb 100644 --- a/apps/api/src/services/warehouse/ai-overview.clickhouse.e2e.test.ts +++ b/apps/api/src/services/warehouse/ai-overview.clickhouse.e2e.test.ts @@ -567,6 +567,61 @@ describe.skipIf(!clickhouseE2eEnabled)("agent overview reads", () => { ) }) + it("splits the window's model-call spans by model, without netting a single one", async () => { + const modelMix = async (opts: Integrations.AiOverviewFilterOpts = {}) => { + const compiled = compileUnsafe(Integrations.aiOverviewModelMixQuery(opts), { + ...window, + bucketSeconds: 300, + }) + return Effect.runSync(compiled.decodeRows(await runJson(compiled.sql))) + } + + const rows = await modelMix() + const { current } = await totals() + + // Two buckets, half an hour apart, and the busiest model of a bucket + // first. The gateway's mirror is a SPAN of its own here — the netting + // that makes it one call never runs — so GPT carries two. + assert.deepStrictEqual( + rows.map((row) => ({ model: row.model, spans: row.llmCallSpans })), + [ + { model: GPT, spans: 2 }, + { model: CLAUDE, spans: 1 }, + { model: CLAUDE, spans: 1 }, + ], + ) + assert.strictEqual(rows[0]!.bucket, rows[1]!.bucket) + assert.isTrue(rows[1]!.bucket < rows[2]!.bucket, `${rows[1]!.bucket} < ${rows[2]!.bucket}`) + + // The tool call is not a model call, and the pre-0031 row is a model call + // that named no model — so the mix is exactly the summary's SPAN + // population less that one row. GPT's two spans against the one call they + // net to is the whole difference between this read and the breakdown. + assert.strictEqual( + sumOf(rows, (row) => row.llmCallSpans), + current!.llmCallSpans - 1, + ) + assert.strictEqual( + sumOf(rows, (row) => row.llmCallSpans), + 4, + ) + assert.isFalse(rows.some((row) => row.model === "" || row.model === "search_traces")) + + // A model filter is the per-trace existence test every other overview + // read applies: it drops the sessionless trace, which never called GPT, + // and keeps every model span of the traces it selected — so Claude is + // still a band under a GPT filter. + const gptOnly = await modelMix({ models: [GPT] }) + assert.deepStrictEqual( + gptOnly.map((row) => ({ model: row.model, spans: row.llmCallSpans })), + [ + { model: GPT, spans: 2 }, + { model: CLAUDE, spans: 1 }, + ], + ) + assert.strictEqual(gptOnly[0]!.bucket, rows[0]!.bucket) + }) + it("selects sessions the way the list selects them, by any span of the trace", async () => { // A model filter and a tool filter together: they are matched by // DIFFERENT spans of the same trace, which a row predicate could never diff --git a/packages/domain/src/http/ai-sessions.ts b/packages/domain/src/http/ai-sessions.ts index 3d8e77abe..9154357bb 100644 --- a/packages/domain/src/http/ai-sessions.ts +++ b/packages/domain/src/http/ai-sessions.ts @@ -641,16 +641,20 @@ export const AiOverviewSeriesPoint = Schema.Struct({ }) export type AiOverviewSeriesPoint = Schema.Schema.Type -export class AiOverviewSummaryRequest extends Schema.Class( - "AiOverviewSummaryRequest", -)({ +/** The selection every bucketed overview read takes: the window, the width its + * buckets are cut at, and the filters. */ +const aiOverviewBucketedSelection = { startTime: TinybirdDateTime, endTime: TinybirdDateTime, /** Whole seconds, greater than zero — it reaches `toStartOfInterval` as an * `INTERVAL n SECOND` literal, so a fraction is a 400 and not a 500. */ bucketSeconds: BucketSeconds, ...aiOverviewSelection, -}) {} +} + +export class AiOverviewSummaryRequest extends Schema.Class( + "AiOverviewSummaryRequest", +)(aiOverviewBucketedSelection) {} export class AiOverviewSummaryResponse extends Schema.Class( "AiOverviewSummaryResponse", @@ -766,6 +770,42 @@ export class AiOverviewBreakdownResponse extends Schema.Class + +export class AiOverviewModelMixRequest extends Schema.Class( + "AiOverviewModelMixRequest", +)(aiOverviewBucketedSelection) {} + +export class AiOverviewModelMixResponse extends Schema.Class( + "AiOverviewModelMixResponse", +)({ + /** Echoed back, so a client rendering an axis reads the width the buckets + * were actually cut at rather than re-deriving it. */ + bucketSeconds: Schema.Number, + /** + * One row per (bucket, model) the window saw, oldest bucket first and the + * busiest model of a bucket first. + * + * The share of model SPANS — the raw population the summary reports as + * `llmCallSpans`, so a gateway's mirror of a call is counted under the model + * it names, twice. Netting would charge a mirrored call to one model alone + * and leave the bands disagreeing with the error rate above them. A band's + * share is its count over its bucket's, and the client folds the minor + * models into an "other" band rather than plotting a line per model. + */ + rows: Schema.Array(AiOverviewModelMixPoint), +}) {} + export class AiSessionsInternalApiGroup extends HttpApiGroup.make("aiSessionsInternal") .add( HttpApiEndpoint.post("list", "/list", { @@ -816,5 +856,12 @@ export class AiSessionsInternalApiGroup extends HttpApiGroup.make("aiSessionsInt error: warehouseReadHttpErrors, }), ) + .add( + HttpApiEndpoint.post("overviewModelMix", "/overview/model-mix", { + payload: AiOverviewModelMixRequest, + success: AiOverviewModelMixResponse, + error: warehouseReadHttpErrors, + }), + ) .prefix("/internal/ai-sessions") .middleware(SessionAuthorization) {} diff --git a/packages/query-engine-integrations/src/__sql_baseline__/integrations.sql b/packages/query-engine-integrations/src/__sql_baseline__/integrations.sql index aba85c62a..69f238c75 100644 --- a/packages/query-engine-integrations/src/__sql_baseline__/integrations.sql +++ b/packages/query-engine-integrations/src/__sql_baseline__/integrations.sql @@ -700,6 +700,30 @@ SELECT AND ai_trace_index.IsToolCall = 1 FORMAT JSON +-- builder:ai-overview:aiOverviewModelMixQuery:default +SELECT + formatDateTime(toStartOfInterval(ai_trace_index.Timestamp, INTERVAL 300 SECOND), '%Y-%m-%dT%H:%i:%S.%fZ') AS bucket, + toString(ai_trace_index.Model) AS model, + count() AS llmCallSpans + FROM ai_trace_index + INNER JOIN (SELECT + TraceId AS TraceId, + max(SessionId) AS rawSessionId + FROM ai_trace_index + WHERE OrgId = 'org_sql_catalog' + AND Timestamp >= '2026-01-01 10:30:00' + AND Timestamp <= '2026-01-03 14:15:00' + GROUP BY TraceId) AS trace ON ai_trace_index.TraceId = trace.TraceId + WHERE ai_trace_index.OrgId = 'org_sql_catalog' + AND ai_trace_index.Timestamp >= '2026-01-01 10:30:00' + AND ai_trace_index.Timestamp <= '2026-01-03 14:15:00' + AND ai_trace_index.IsLlmCall = 1 + AND ai_trace_index.Model != '' + GROUP BY bucket, model + ORDER BY bucket ASC, llmCallSpans DESC + LIMIT 4000 + FORMAT JSON + -- builder:ai-overview:aiOverviewSeriesQuery:default SELECT * FROM ( SELECT diff --git a/packages/query-engine-integrations/src/ai/ai-overview.test.ts b/packages/query-engine-integrations/src/ai/ai-overview.test.ts index 59872d484..3269f8c4f 100644 --- a/packages/query-engine-integrations/src/ai/ai-overview.test.ts +++ b/packages/query-engine-integrations/src/ai/ai-overview.test.ts @@ -1,7 +1,13 @@ import { describe, expect, it } from "vitest" -import { compileUnionUnsafe } from "@maple-dev/effect-clickhouse" +import { compileUnionUnsafe, compileUnsafe } from "@maple-dev/effect-clickhouse" import { AI_OVERVIEW_BREAKDOWN_MAX } from "@maple/domain/http" -import { aiOverviewBreakdownQuery, aiOverviewSeriesQuery, aiOverviewTotalsQuery } from "./ai-overview" +import { + aiOverviewBreakdownQuery, + aiOverviewModelMixQuery, + aiOverviewSeriesQuery, + aiOverviewTotalsQuery, + AI_OVERVIEW_MODEL_MIX_MAX_ROWS, +} from "./ai-overview" const params = { orgId: "org_1", @@ -240,3 +246,72 @@ describe("the breakdown's dimensions", () => { expect(totalsSql()).not.toContain("GROUP BY sessionId, key") }) }) + +describe("the model mix", () => { + const modelMixSql = (opts: Parameters[0] = {}) => + compileUnsafe(aiOverviewModelMixQuery(opts), seriesParams).sql + + it("counts the model-call spans that name a model, per bucket and model", () => { + const sql = modelMixSql() + + expect(sql).toContain("FROM ai_trace_index") + expect(sql).not.toContain("trace_detail_spans") + expect(sql).not.toContain("__PARAM_") + // The population: model-call spans that named a model. The netting never + // runs here, so a gateway's mirror is a span of its own — the summary's + // `llmCallSpans` population, less the calls that named nothing. + expect(sql).toContain("AND ai_trace_index.IsLlmCall = 1") + expect(sql).toContain("AND ai_trace_index.Model != ''") + expect(sql).toContain("count() AS llmCallSpans") + expect(sql).toContain("toString(ai_trace_index.Model) AS model") + expect(sql).toContain("GROUP BY bucket, model") + expect(sql).not.toContain("AS netted") + }) + + it("buckets the span's own timestamp, at the width the caller asked for", () => { + const sql = modelMixSql() + + // The span's timestamp and not the session's start: the rows are spans, + // so there is no session to keep inside one bucket. + expect(sql).toContain("toStartOfInterval(ai_trace_index.Timestamp, INTERVAL 300 SECOND)") + expect(sql).toContain("ORDER BY bucket ASC, llmCallSpans DESC") + // A guard and not a top-N — the client folds the minor models into + // "other" and needs every model of every bucket to do it. + expect(sql).toContain(`LIMIT ${AI_OVERVIEW_MODEL_MIX_MAX_ROWS}`) + }) + + it("reads the current window alone, scoped to the org on every level", () => { + const sql = modelMixSql() + + expect(sql).toContain(`Timestamp >= '${params.startTime}'`) + expect(sql).toContain(`Timestamp <= '${params.endTime}'`) + // The chart has no comparison band, so the previous window's params are + // never resolved. + expect(sql).not.toContain(params.prevStartTime) + // The trace keys and the spans themselves. + expect(orgPredicateCount(sql)).toBe(2) + expect(compileUnsafe(aiOverviewModelMixQuery(), seriesParams).tenantScope).toBe("single-tenant") + }) + + it("selects sessions with the same tests every other overview read applies", () => { + const sql = modelMixSql({ + vendorIds: ["eve"], + models: ["gpt-5.5"], + toolNames: ["send_email"], + hasErrors: true, + }) + + // The per-trace existence tests, so a session that used the model is + // measured across every model it used — and the session-level failure + // test, which adds its own two levels to the org scoping. + expect(sql).toContain("countIf(VendorId IN ('eve')) > 0") + expect(sql).toContain("countIf(Model IN ('gpt-5.5')) > 0") + expect(sql).toContain("countIf(ToolName IN ('send_email')) > 0") + expect(sql).toContain(`${SESSION_KEY} IN (SELECT`) + expect(orgPredicateCount(sql)).toBe(4) + + const unfiltered = modelMixSql() + expect(unfiltered).not.toContain("HAVING") + expect(unfiltered).not.toContain(`${SESSION_KEY} IN (SELECT`) + }) +}) diff --git a/packages/query-engine-integrations/src/ai/ai-overview.ts b/packages/query-engine-integrations/src/ai/ai-overview.ts index 6ddf45117..fdbd19daa 100644 --- a/packages/query-engine-integrations/src/ai/ai-overview.ts +++ b/packages/query-engine-integrations/src/ai/ai-overview.ts @@ -567,3 +567,59 @@ export function aiOverviewBreakdownQuery( ]) return unionAll(branch("current"), branch("previous"), keyCount).format("JSON") } + +/** + * Rows one model mix returns, across every bucket and model together. + * + * Not a top-N: the client folds the minor models into an "other" band and + * needs every model of every bucket to do it. The cap is there so a month at a + * one-minute bucket, in an org that routes across a long model list, cannot + * answer with a response nothing can render. + */ +export const AI_OVERVIEW_MODEL_MIX_MAX_ROWS = 4000 + +/** + * The model mix: the window's model-call SPANS, split by model, bucket by + * bucket. + * + * The share of model SPANS and not of netted calls — this is a plain GROUP BY + * over the index, where the netting is a per-session array pass — so a + * gateway's mirror of a call is counted under the model it names, twice. It is + * the same population the summary counts as `llmCallSpans`, less the calls + * whose instrumentation named no model: those carry no share of a model mix, + * so the two totals differ by exactly them. + * + * A span is filed under the bucket ITS OWN timestamp falls in, where the + * summary's series files a whole session under the bucket it started in. The + * rows here are spans, so there is no session to keep whole. + * + * Sessions are selected the way every other read in this file selects them — + * `traceKeys`, plus the session-level `hasErrors` test — so the mix describes + * the sessions the tiles above it measure. The current window alone: the chart + * has no comparison band. + */ +export function aiOverviewModelMixQuery(opts: AiOverviewFilterOpts = {}) { + return from(AiTraceIndex) + .innerJoinQuery(traceKeys(opts, "current"), "trace", (row, trace) => row.TraceId.eq(trace.TraceId)) + .select(($) => ({ + bucket: isoBucket($.Timestamp), + // `toString` for the reason the breakdown's key takes it: `Model` is + // `LowCardinality(String)` in the index, and a model key is a plain + // `String` everywhere else the page reads one. + model: CH.toString_($.Model), + llmCallSpans: CH.count(), + })) + .where(($) => [ + $.OrgId.eq(param.string("orgId")), + ...withinWindow($.Timestamp, "current"), + $.IsLlmCall.eq(1), + $.Model.neq(""), + CH.whenTrue(opts.hasErrors, () => + inSubquery(sessionKey($.trace.rawSessionId, $.TraceId), erroredSessionKeys(opts, "current")), + ), + ]) + .groupBy("bucket", "model") + .orderBy(["bucket", "asc"], ["llmCallSpans", "desc"]) + .limit(AI_OVERVIEW_MODEL_MIX_MAX_ROWS) + .format("JSON") +} diff --git a/packages/query-engine-integrations/src/ai/index.ts b/packages/query-engine-integrations/src/ai/index.ts index 67940aab2..c3aa7b35f 100644 --- a/packages/query-engine-integrations/src/ai/index.ts +++ b/packages/query-engine-integrations/src/ai/index.ts @@ -42,8 +42,10 @@ export { export { aiOverviewBreakdownQuery, + aiOverviewModelMixQuery, aiOverviewSeriesQuery, aiOverviewTotalsQuery, + AI_OVERVIEW_MODEL_MIX_MAX_ROWS, type AiOverviewBreakdownOpts, type AiOverviewBreakdownOutput, type AiOverviewFilterOpts, diff --git a/packages/query-engine-integrations/src/benchmark/index.ts b/packages/query-engine-integrations/src/benchmark/index.ts index db9ae1b0e..79e052569 100644 --- a/packages/query-engine-integrations/src/benchmark/index.ts +++ b/packages/query-engine-integrations/src/benchmark/index.ts @@ -261,6 +261,15 @@ export const integrationFixtures: ReadonlyArray = [ aiCompare, ), }, + { + // The model mix: model-call SPANS per bucket and model, off the same + // selection and with no netting at all — the one overview read that is a + // plain GROUP BY over the index. + module: "ai-overview", + name: "aiOverviewModelMixQuery", + label: "default", + compile: () => compileUnsafe(CH.aiOverviewModelMixQuery(), bucketed), + }, { module: "ai-sessions", name: "aiSessionSpansQuery", From 68ccc2ce598ad26ecc719b25e1243c670284f229 Mon Sep 17 00:00:00 2001 From: JeremyFunk Date: Fri, 11 Sep 2026 06:37:42 +0200 Subject: [PATCH 05/16] feat(agent-sessions): overview page data layer, route and lab --- .../api/warehouse/ai-agent-overview.test.ts | 78 ++ .../src/api/warehouse/ai-agent-overview.ts | 238 ++++ .../overview/agent-overview-view.test.tsx | 191 +++ .../overview/agent-overview-view.tsx | 188 +++ .../overview/overview-breakdowns.tsx | 187 +++ .../overview/overview-filter-toolbar.tsx | 156 +++ .../overview/overview-metric-strip.tsx | 65 + .../overview/overview-movers-rail.tsx | 104 ++ .../overview/overview-scope-row.tsx | 63 + .../overview/overview-top-sessions.tsx | 167 +++ .../overview/overview-trends.tsx | 115 ++ .../tools/agent-sessions-tabs.tsx | 84 ++ apps/web/src/lab/agent-overview-fixture.ts | 562 +++++++++ apps/web/src/lab/agent-overview-lab.tsx | 109 ++ apps/web/src/lab/registry.ts | 8 + .../agent-sessions/overview-analytics.test.ts | 431 +++++++ .../lib/agent-sessions/overview-analytics.ts | 1071 +++++++++++++++++ .../agent-sessions/overview-search.test.ts | 112 ++ .../src/lib/agent-sessions/overview-search.ts | 198 +++ .../agent-sessions/use-agent-overview.test.ts | 73 ++ .../lib/agent-sessions/use-agent-overview.ts | 189 +++ .../services/atoms/warehouse-query-atoms.ts | 21 + apps/web/src/routeTree.gen.ts | 42 + apps/web/src/routes/agent-sessions/index.tsx | 10 +- .../src/routes/agent-sessions/overview.tsx | 222 ++++ apps/web/src/routes/lab/agent-overview.tsx | 5 + 26 files changed, 4688 insertions(+), 1 deletion(-) create mode 100644 apps/web/src/api/warehouse/ai-agent-overview.test.ts create mode 100644 apps/web/src/api/warehouse/ai-agent-overview.ts create mode 100644 apps/web/src/components/agent-sessions/overview/agent-overview-view.test.tsx create mode 100644 apps/web/src/components/agent-sessions/overview/agent-overview-view.tsx create mode 100644 apps/web/src/components/agent-sessions/overview/overview-breakdowns.tsx create mode 100644 apps/web/src/components/agent-sessions/overview/overview-filter-toolbar.tsx create mode 100644 apps/web/src/components/agent-sessions/overview/overview-metric-strip.tsx create mode 100644 apps/web/src/components/agent-sessions/overview/overview-movers-rail.tsx create mode 100644 apps/web/src/components/agent-sessions/overview/overview-scope-row.tsx create mode 100644 apps/web/src/components/agent-sessions/overview/overview-top-sessions.tsx create mode 100644 apps/web/src/components/agent-sessions/overview/overview-trends.tsx create mode 100644 apps/web/src/components/agent-sessions/tools/agent-sessions-tabs.tsx create mode 100644 apps/web/src/lab/agent-overview-fixture.ts create mode 100644 apps/web/src/lab/agent-overview-lab.tsx create mode 100644 apps/web/src/lib/agent-sessions/overview-analytics.test.ts create mode 100644 apps/web/src/lib/agent-sessions/overview-analytics.ts create mode 100644 apps/web/src/lib/agent-sessions/overview-search.test.ts create mode 100644 apps/web/src/lib/agent-sessions/overview-search.ts create mode 100644 apps/web/src/lib/agent-sessions/use-agent-overview.test.ts create mode 100644 apps/web/src/lib/agent-sessions/use-agent-overview.ts create mode 100644 apps/web/src/routes/agent-sessions/overview.tsx create mode 100644 apps/web/src/routes/lab/agent-overview.tsx diff --git a/apps/web/src/api/warehouse/ai-agent-overview.test.ts b/apps/web/src/api/warehouse/ai-agent-overview.test.ts new file mode 100644 index 000000000..72eadd6cb --- /dev/null +++ b/apps/web/src/api/warehouse/ai-agent-overview.test.ts @@ -0,0 +1,78 @@ +import { describe, expect, it } from "vitest" + +import { AiOverviewMeasures } from "@maple/domain/http" + +import { + mapOverviewBreakdown, + mapOverviewMeasures, + mapOverviewModelMix, + mapOverviewSeries, +} from "./ai-agent-overview" + +const wire = (overrides: Partial = {}): AiOverviewMeasures => ({ + sessions: 10, + erroredSessions: 1, + llmCalls: 80, + llmCallSpans: 94, + erroredLlmCalls: 3, + toolCalls: 63, + erroredToolCalls: 2, + cost: 2.64, + pricedLlmCalls: 88, + tokens: 674_000, + inputTokens: 152_000, + cacheReadTokens: 373_000, + cacheWriteTokens: 27_000, + outputTokens: 81_000, + reasoningTokens: 41_000, + sessionDurationP50Ns: 42_000_000_000, + sessionDurationP95Ns: 96_000_000_000, + llmDurationP50Ns: 1_900_000_000, + llmDurationP95Ns: 7_400_000_000, + ...overrides, +}) + +describe("mapOverviewMeasures", () => { + it("converts every quantile from nanoseconds to milliseconds", () => { + const row = mapOverviewMeasures(wire()) + expect(row.sessionDurationP50Ms).toBe(42_000) + expect(row.sessionDurationP95Ms).toBe(96_000) + expect(row.llmDurationP50Ms).toBe(1_900) + expect(row.llmDurationP95Ms).toBe(7_400) + }) + + it("carries the raw span population separately from the netted volume", () => { + const row = mapOverviewMeasures(wire()) + expect(row.llmCalls).toBe(80) + expect(row.llmCallSpans).toBe(94) + }) +}) + +describe("mapOverviewSeries", () => { + it("reads a bucket as UTC rather than as local time", () => { + const [point] = mapOverviewSeries([{ bucket: "2026-09-10T12:00:00.000Z", ...wire() }]) + expect(point.bucket).toBe(Date.UTC(2026, 8, 10, 12, 0, 0)) + }) +}) + +describe("mapOverviewBreakdown", () => { + it("keeps both windows per key, and `''` as a real key", () => { + const [row] = mapOverviewBreakdown([ + { key: "", current: wire(), previous: wire({ sessions: 4 }) }, + ]) + expect(row.key).toBe("") + expect(row.current.sessions).toBe(10) + expect(row.previous.sessions).toBe(4) + }) +}) + +describe("mapOverviewModelMix", () => { + it("reads the bucket as UTC and leaves the model alone", () => { + const [row] = mapOverviewModelMix([ + { bucket: "2026-09-10T18:00:00.000Z", model: "claude-opus-5", llmCallSpans: 42 }, + ]) + expect(row.bucket).toBe(Date.UTC(2026, 8, 10, 18, 0, 0)) + expect(row.model).toBe("claude-opus-5") + expect(row.llmCallSpans).toBe(42) + }) +}) diff --git a/apps/web/src/api/warehouse/ai-agent-overview.ts b/apps/web/src/api/warehouse/ai-agent-overview.ts new file mode 100644 index 000000000..9ae57808f --- /dev/null +++ b/apps/web/src/api/warehouse/ai-agent-overview.ts @@ -0,0 +1,238 @@ +// The three warehouse reads behind `/agent-sessions/overview`, and the only +// place in the web app that sees their wire shape. +// +// Two conversions happen here and nowhere else. Buckets arrive as ISO-8601 with +// a literal `Z`; `toEpochMs` reads them as UTC, where `new Date(value)` would +// read a bare warehouse datetime as local time. Durations arrive in +// nanoseconds and leave in milliseconds, because every formatter downstream +// takes milliseconds. +// +// The page's filters are single-valued — one model, one agent, one tool — and +// the contract takes arrays. The widening happens in `selectionFields`, so a +// dimension that later becomes multi-valued changes one function. + +import { Effect, Schema } from "effect" +import { + AI_OVERVIEW_BREAKDOWN_MAX, + AiOverviewBreakdownRequest, + AiOverviewDimension, + AiOverviewModelMixRequest, + AiOverviewSummaryRequest, + type AiOverviewBreakdownRow, + type AiOverviewMeasures, + type AiOverviewModelMixPoint, + type AiOverviewSeriesPoint, +} from "@maple/domain/http" +import { toEpochMs } from "@maple/ui/lib/time-format" + +import type { + OverviewBreakdownEntry, + OverviewMeasurePoint, + OverviewMeasures, + OverviewModelMixRow, +} from "@/lib/agent-sessions/overview-analytics" +import { MapleInternalAtomClient } from "@/lib/services/common/internal-atom-client" + +import { WarehouseDateTimeString, decodeInput, runWarehouseQuery } from "./effect-utils" + +/** + * The page's selection, as all three reads take it. + * + * Sent identically to every one of them so the numbers can never disagree about + * what they are counting: a summary narrower than the breakdown would make the + * tiles and the table tell different stories about one window. + */ +const AiOverviewSelection = Schema.Struct({ + startTime: WarehouseDateTimeString, + endTime: WarehouseDateTimeString, + /** The SDK or gateway — `vendorIds` on the wire. */ + framework: Schema.optional(Schema.String), + model: Schema.optional(Schema.String), + agent: Schema.optional(Schema.String), + service: Schema.optional(Schema.String), + environment: Schema.optional(Schema.String), + tool: Schema.optional(Schema.String), + hasErrors: Schema.optional(Schema.Boolean), +}) +export type AiOverviewSelection = Schema.Schema.Type + +const AiOverviewBucketedInput = Schema.Struct({ + ...AiOverviewSelection.fields, + bucketSeconds: Schema.Number, +}) +export type AiOverviewBucketedInput = Schema.Schema.Type + +/** + * The bounds here MIRROR the domain request's: `decodeInput` turns a violation + * into a `WarehouseDecodeError` the page can render, where the domain + * constructor throws — a defect that would crash the page rather than fail one + * read. + */ +const AiOverviewBreakdownInput = Schema.Struct({ + ...AiOverviewSelection.fields, + dimension: AiOverviewDimension, + limit: Schema.optional( + Schema.Number.check( + Schema.isInt(), + Schema.isBetween({ minimum: 1, maximum: AI_OVERVIEW_BREAKDOWN_MAX }), + ), + ), +}) +export type AiOverviewBreakdownInput = Schema.Schema.Type + +/** The selection minus the window, spread into a request payload. */ +const selectionFields = (input: AiOverviewSelection) => ({ + ...(input.framework !== undefined && { vendorIds: [input.framework] }), + ...(input.service !== undefined && { serviceNames: [input.service] }), + ...(input.environment !== undefined && { deploymentEnvs: [input.environment] }), + ...(input.model !== undefined && { models: [input.model] }), + ...(input.agent !== undefined && { agentNames: [input.agent] }), + ...(input.tool !== undefined && { toolNames: [input.tool] }), + ...(input.hasErrors !== undefined && { hasErrors: input.hasErrors }), +}) + +/* ------------------------------------------------------------------------------------------------- + * Mappers + * -----------------------------------------------------------------------------------------------*/ + +const NS_PER_MS = 1_000_000 + +export function mapOverviewMeasures(row: AiOverviewMeasures): OverviewMeasures { + return { + sessions: row.sessions, + erroredSessions: row.erroredSessions, + llmCalls: row.llmCalls, + llmCallSpans: row.llmCallSpans, + erroredLlmCalls: row.erroredLlmCalls, + toolCalls: row.toolCalls, + erroredToolCalls: row.erroredToolCalls, + cost: row.cost, + pricedLlmCalls: row.pricedLlmCalls, + tokens: row.tokens, + inputTokens: row.inputTokens, + cacheReadTokens: row.cacheReadTokens, + cacheWriteTokens: row.cacheWriteTokens, + outputTokens: row.outputTokens, + reasoningTokens: row.reasoningTokens, + sessionDurationP50Ms: row.sessionDurationP50Ns / NS_PER_MS, + sessionDurationP95Ms: row.sessionDurationP95Ns / NS_PER_MS, + llmDurationP50Ms: row.llmDurationP50Ns / NS_PER_MS, + llmDurationP95Ms: row.llmDurationP95Ns / NS_PER_MS, + } +} + +export function mapOverviewSeries( + rows: ReadonlyArray, +): ReadonlyArray { + return rows.map((row) => ({ bucket: toEpochMs(row.bucket), ...mapOverviewMeasures(row) })) +} + +export function mapOverviewBreakdown( + rows: ReadonlyArray, +): ReadonlyArray { + return rows.map((row) => ({ + key: row.key, + current: mapOverviewMeasures(row.current), + previous: mapOverviewMeasures(row.previous), + })) +} + +export function mapOverviewModelMix( + rows: ReadonlyArray, +): ReadonlyArray { + return rows.map((row) => ({ + bucket: toEpochMs(row.bucket), + model: row.model, + llmCallSpans: row.llmCallSpans, + })) +} + +/* ------------------------------------------------------------------------------------------------- + * The reads + * -----------------------------------------------------------------------------------------------*/ + +/** The window and the one before it, whole and bucketed — the tiles and the grid. */ +export const getAiOverviewSummary = Effect.fn("AiAgentOverview.summary")(function* ({ + data, +}: { + data: AiOverviewBucketedInput +}) { + const input = yield* decodeInput(AiOverviewBucketedInput, data, "aiOverviewSummary") + const result = yield* runWarehouseQuery("aiOverviewSummary", () => + Effect.gen(function* () { + const client = yield* MapleInternalAtomClient + return yield* client.aiSessionsInternal.overviewSummary({ + payload: new AiOverviewSummaryRequest({ + startTime: input.startTime, + endTime: input.endTime, + bucketSeconds: input.bucketSeconds, + ...selectionFields(input), + }), + }) + }), + ) + return { + bucketSeconds: result.bucketSeconds, + current: mapOverviewMeasures(result.current), + // Zeros where nothing ran then, which the tiles render as "no comparison" + // rather than as a -100%. + previous: mapOverviewMeasures(result.previous), + series: mapOverviewSeries(result.series), + previousSeries: mapOverviewSeries(result.previousSeries), + } +}) + +/** One dimension's busiest keys, each over both windows. */ +export const getAiOverviewBreakdown = Effect.fn("AiAgentOverview.breakdown")(function* ({ + data, +}: { + data: AiOverviewBreakdownInput +}) { + const input = yield* decodeInput(AiOverviewBreakdownInput, data, "aiOverviewBreakdown") + const result = yield* runWarehouseQuery("aiOverviewBreakdown", () => + Effect.gen(function* () { + const client = yield* MapleInternalAtomClient + return yield* client.aiSessionsInternal.overviewBreakdown({ + payload: new AiOverviewBreakdownRequest({ + startTime: input.startTime, + endTime: input.endTime, + dimension: input.dimension, + ...(input.limit !== undefined && { limit: input.limit }), + ...selectionFields(input), + }), + }) + }), + ) + return { + dimension: result.dimension, + entries: mapOverviewBreakdown(result.rows), + totalKeys: result.totalKeys, + } +}) + +/** Model-call spans per bucket per model — the 100% stack. Current window only. */ +export const getAiOverviewModelMix = Effect.fn("AiAgentOverview.modelMix")(function* ({ + data, +}: { + data: AiOverviewBucketedInput +}) { + const input = yield* decodeInput(AiOverviewBucketedInput, data, "aiOverviewModelMix") + const result = yield* runWarehouseQuery("aiOverviewModelMix", () => + Effect.gen(function* () { + const client = yield* MapleInternalAtomClient + return yield* client.aiSessionsInternal.overviewModelMix({ + payload: new AiOverviewModelMixRequest({ + startTime: input.startTime, + endTime: input.endTime, + bucketSeconds: input.bucketSeconds, + ...selectionFields(input), + }), + }) + }), + ) + return { bucketSeconds: result.bucketSeconds, rows: mapOverviewModelMix(result.rows) } +}) + +export type AiOverviewSummaryData = Effect.Success> +export type AiOverviewBreakdownData = Effect.Success> +export type AiOverviewModelMixData = Effect.Success> diff --git a/apps/web/src/components/agent-sessions/overview/agent-overview-view.test.tsx b/apps/web/src/components/agent-sessions/overview/agent-overview-view.test.tsx new file mode 100644 index 000000000..d218ce5d5 --- /dev/null +++ b/apps/web/src/components/agent-sessions/overview/agent-overview-view.test.tsx @@ -0,0 +1,191 @@ +// @vitest-environment jsdom +// TEST-SEAM: the router has no instance-level injection seam, so `Link` is +// replaced at the module boundary. What is under test is the page's own wiring +// — which control writes which search param, which row links where, and what +// the sections say about the data they are given. + +import { cleanup, fireEvent, render, screen, within } from "@testing-library/react" +import { afterEach, describe, expect, it, vi } from "vitest" + +import { buildOverviewFixture } from "@/lab/agent-overview-fixture" +import { buildAgentOverviewData } from "@/lib/agent-sessions/overview-analytics" +import { + compareEnabled, + type AgentOverviewSearch, +} from "@/lib/agent-sessions/overview-search" + +import { AgentOverviewView } from "./agent-overview-view" + +vi.mock("@tanstack/react-router", () => ({ + Link: ({ + children, + to, + params, + search, + ...props + }: React.PropsWithChildren>) => ( + )} + > + {children} + + ), +})) + +const NOW = Date.UTC(2026, 8, 10, 12, 0, 0) + +function renderView(search: AgentOverviewSearch = {}, scenario: "healthy7d" | "regression24h" = "regression24h") { + const fixture = buildOverviewFixture(scenario, NOW) + const data = buildAgentOverviewData({ ...fixture.input, compare: compareEnabled(search) }) + const onSearchChange = vi.fn() + render( + , + ) + return { onSearchChange, data, fixture } +} + +/** The section a heading owns — the page repeats labels across sections. */ +const sectionOf = (heading: string) => + screen.getByRole("heading", { name: heading }).closest("section")! + +afterEach(cleanup) + +describe("AgentOverviewView", () => { + it("renders the seven KPI tiles and the nine small multiples", () => { + const { data } = renderView() + for (const tile of data.tiles) { + expect(screen.getAllByText(tile.label).length).toBeGreaterThan(0) + } + expect(document.querySelectorAll("[data-chart]")).toHaveLength(9) + }) + + it("draws the previous-period ghost only while the comparison is on", () => { + renderView({}) + expect(screen.getAllByText(/ghost/).length).toBeGreaterThan(0) + cleanup() + renderView({ compare: false }) + expect(screen.queryByText(/ghost/)).toBeNull() + }) + + it("turns the comparison off through the URL rather than through local state", () => { + const { onSearchChange } = renderView({}) + fireEvent.click(screen.getByRole("button", { name: /compare prev/ })) + expect(onSearchChange).toHaveBeenCalledWith({ compare: false }) + }) + + it("writes the failing-only toggle to the URL", () => { + const { onSearchChange } = renderView({}) + fireEvent.click(screen.getByRole("button", { name: "Failing only" })) + expect(onSearchChange).toHaveBeenCalledWith({ hasErrors: true }) + }) + + it("shows one chip per active filter and clears them all at once", () => { + const { onSearchChange } = renderView({ model: "claude-opus-5", environment: "production" }) + expect(screen.getAllByText("claude-opus-5").length).toBeGreaterThan(0) + fireEvent.click(screen.getByRole("button", { name: "Clear all" })) + expect(onSearchChange).toHaveBeenCalledWith({ + model: undefined, + agent: undefined, + service: undefined, + framework: undefined, + environment: undefined, + tool: undefined, + }) + }) + + it("removes one chip without touching the others", () => { + const { onSearchChange } = renderView({ model: "claude-opus-5" }) + fireEvent.click(screen.getByRole("button", { name: /Remove model filter/ })) + expect(onSearchChange).toHaveBeenCalledWith({ model: undefined }) + }) + + it("filters the whole page from a breakdown row", () => { + const { onSearchChange, data } = renderView({}) + const row = data.breakdowns.find((b) => b.dimension === "model")?.rows[0] + expect(row).toBeDefined() + fireEvent.click(within(sectionOf("Breakdowns")).getAllByText(row!.label)[0]) + expect(onSearchChange).toHaveBeenCalledWith({ model: row!.key }) + }) + + it("switches the breakdown table without touching the URL", () => { + const { onSearchChange } = renderView({}) + fireEvent.click( + within(sectionOf("Breakdowns")).getByRole("button", { name: /^tool/ }), + ) + expect(screen.getByText("Share of calls")).toBeTruthy() + expect(onSearchChange).not.toHaveBeenCalled() + }) + + it("filters the page from a mover line", () => { + const { onSearchChange, data } = renderView({}) + const mover = data.movers.find((candidate) => candidate.key !== "") + expect(mover).toBeDefined() + const rail = screen.getByRole("heading", { name: "What changed" }).closest("aside")! + fireEvent.click(within(rail).getAllByText(mover!.label)[0]) + expect(onSearchChange).toHaveBeenCalledWith({ [mover!.dimension]: mover!.key }) + }) + + it("links each top session to its own detail page, with the session's bounds", () => { + const { fixture } = renderView({}) + const first = fixture.topSessions.cost[0] + const link = screen.getByText(first.sessionId).closest("a") + expect(link?.getAttribute("data-to")).toBe("/agent-sessions/$sessionId") + expect(link?.getAttribute("data-params")).toBe(JSON.stringify({ sessionId: first.sessionId })) + expect(link?.getAttribute("data-search")).toContain("\"t\"") + }) + + it("carries the board's filters into the Sessions list", () => { + renderView({ model: "claude-opus-5", hasErrors: true }) + const link = screen.getByText(/Open in Sessions/).closest("a") + expect(JSON.parse(link?.getAttribute("data-search") ?? "{}")).toMatchObject({ + models: ["claude-opus-5"], + hasErrors: true, + }) + }) + + it("switches the top-sessions tab without touching the URL", () => { + const { onSearchChange } = renderView({}) + fireEvent.click(screen.getByRole("button", { name: /longest/ })) + expect(onSearchChange).not.toHaveBeenCalled() + }) + + it("keeps the chips up and shows the empty block when the scope matches nothing", () => { + const fixture = buildOverviewFixture("healthy7d", NOW) + const data = buildAgentOverviewData({ + ...fixture.input, + current: { ...fixture.input.current, sessions: 0 }, + series: [], + compare: true, + }) + render( + , + ) + expect(screen.getByText("No agent sessions in this range")).toBeTruthy() + expect(screen.getAllByText("claude-opus-5").length).toBeGreaterThan(0) + expect(screen.queryByText("Top sessions")).toBeNull() + }) + + it("names the two tabs the page can be read under", () => { + renderView({}) + const nav = screen.getByRole("navigation", { name: "Agent sessions views" }) + expect(within(nav).getByText("Overview")).toBeTruthy() + expect(within(nav).getByText("Sessions")).toBeTruthy() + }) +}) diff --git a/apps/web/src/components/agent-sessions/overview/agent-overview-view.tsx b/apps/web/src/components/agent-sessions/overview/agent-overview-view.tsx new file mode 100644 index 000000000..a0615c01a --- /dev/null +++ b/apps/web/src/components/agent-sessions/overview/agent-overview-view.tsx @@ -0,0 +1,188 @@ +import { useMemo, type ReactNode } from "react" + +import { + Empty, + EmptyDescription, + EmptyHeader, + EmptyMedia, + EmptyTitle, +} from "@maple/ui/components/ui/empty" + +import { SquareSparkleIcon } from "@/components/icons" +import type { AgentSessionRow } from "@/components/agent-sessions/agent-sessions-list" +import { AgentSessionsTabs } from "@/components/agent-sessions/tools/agent-sessions-tabs" +import type { TimeRangeSearch } from "@/components/time-range-picker/search" +import { + bucketWidthLabel, + overviewScopeSummary, + type AgentOverviewData, + type OverviewMover, +} from "@/lib/agent-sessions/overview-analytics" +import { + activeOverviewFilters, + clearOverviewFilters, + compareEnabled, + sessionsLinkSearch, + toggleOverviewFilter, + type AgentOverviewSearch, + type OverviewDimension, + type OverviewFacets, +} from "@/lib/agent-sessions/overview-search" +import type { OverviewTopSessionTab } from "@/lib/agent-sessions/use-agent-overview" + +import { OverviewBreakdowns } from "./overview-breakdowns" +import { OverviewFilterToolbar } from "./overview-filter-toolbar" +import { OverviewMetricStrip } from "./overview-metric-strip" +import { OverviewMoversRail } from "./overview-movers-rail" +import { OverviewScopeRow } from "./overview-scope-row" +import { OverviewTopSessions } from "./overview-top-sessions" +import { OverviewTrends } from "./overview-trends" + +export interface AgentOverviewViewProps { + search: AgentOverviewSearch + /** Applied to the URL by the route. Keys set to `undefined` are cleared. */ + onSearchChange: (patch: Partial) => void + data: AgentOverviewData + facets: OverviewFacets + topSessions: Record> + /** Names the window and its comparison in the tiles, e.g. `7d`. */ + windowLabel: string + /** The window, carried by the tab strip's link to the Sessions list. */ + timeRange?: TimeRangeSearch + /** The time-range picker, or whatever the host wants beside the title. */ + headerControls?: ReactNode + /** Dim the data surfaces while a refetch is in flight. */ + waiting?: boolean +} + +/** + * The whole `/agent-sessions/overview` page below the layout chrome, over data + * that has already resolved. + * + * Presentational on purpose: the route hands it resolved values and the lab + * hands it fixtures, so the page can be looked at and reviewed without a + * warehouse behind it — `ai_trace_index` does not exist in the local Tinybird + * container. Every control writes a search param and nothing filters rows + * locally: the toolbar's predicates are server-side on every read, so the + * tiles, the grid and the tables always describe the same sessions. + * + * One column of full-bleed sections divided by hairlines, not a stack of cards: + * the page is one instrument, and every section is a different reading of the + * same scope. Reading order is the order the questions get asked: what am I + * looking at, over which sessions, narrowed to what, how much of it, how it + * moved and what moved most, grouped how, and finally which sessions. + */ +export function AgentOverviewView({ + search, + onSearchChange, + data, + facets, + topSessions, + windowLabel, + timeRange, + headerControls, + waiting, +}: AgentOverviewViewProps) { + const chips = useMemo(() => activeOverviewFilters(search), [search]) + const selectDimension = (dimension: OverviewDimension, key: string) => + onSearchChange(toggleOverviewFilter(search, dimension, key)) + + const note = [ + compareEnabled(search) ? `previous ${windowLabel}` : `last ${windowLabel}`, + `${bucketWidthLabel(data.bucketSeconds)} buckets`, + ].join(" · ") + + return ( +
+
+
+

+ Overview +

+

+ What your agents cost, how much they ran, and how often they failed. +

+
+ {headerControls ? ( +
{headerControls}
+ ) : null} +
+ + + + + + {/* The chips stay up when the scope matches nothing: a reader looking at + an empty board needs to see what emptied it. */} + selectDimension(chip.dimension, chip.value)} + onClearAll={() => onSearchChange(clearOverviewFilters())} + /> + + {data.current.sessions === 0 ? ( + + + + + + No agent sessions in this range + + {chips.length === 0 + ? "Nothing your agents ran was recorded in this window. Widen the range, or check that the SDK is reporting." + : "Nothing in this window matches the scope above. Remove a filter to widen it."} + + + + ) : ( + <> + + + + selectDimension(mover.dimension, mover.key) + } + /> + } + /> + + + + + + )} +
+ ) +} diff --git a/apps/web/src/components/agent-sessions/overview/overview-breakdowns.tsx b/apps/web/src/components/agent-sessions/overview/overview-breakdowns.tsx new file mode 100644 index 000000000..9891d605a --- /dev/null +++ b/apps/web/src/components/agent-sessions/overview/overview-breakdowns.tsx @@ -0,0 +1,187 @@ +import { useState } from "react" + +import { formatErrorRate, formatNumber, formatPercent } from "@maple/ui/lib/format" +import { cn } from "@maple/ui/lib/utils" + +import { + formatOverviewCount, + type OverviewBreakdown, + type OverviewBreakdownRow, +} from "@/lib/agent-sessions/overview-analytics" +import { formatCost } from "@/lib/agent-sessions/session-summary" +import { + selectedDimensionValue, + type AgentOverviewSearch, + type OverviewDimension, +} from "@/lib/agent-sessions/overview-search" + +export interface OverviewBreakdownsProps { + /** All six, in the dimensions' own order. */ + breakdowns: ReadonlyArray + search: AgentOverviewSearch + /** Toggles that dimension's filter for the whole page. */ + onSelectRow: (dimension: OverviewDimension, key: string) => void + waiting?: boolean +} + +/** The columns a dimension can actually attribute — see `AiOverviewBreakdownRow`. */ +const USAGE_COLUMNS = [ + "Share of cost", + "Sessions", + "LLM calls", + "Tok / sess", + "Cost", + "$ / sess", + "Error rate", + "Δ prev", +] as const + +const TOOL_COLUMNS = [ + "Share of calls", + "Sessions", + "Calls", + "Errors", + "Error rate", + "Δ prev", +] as const + +/** + * The same window, grouped six ways. + * + * A row is not a link but a filter: clicking one narrows every reading on the + * page to that key, and clicking it again gives the page back. Rows OVERLAP — + * a session that used two models is a session under each — which the footer + * says out loud rather than leaving a reader to reconcile the columns. + */ +export function OverviewBreakdowns({ + breakdowns, + search, + onSelectRow, + waiting = false, +}: OverviewBreakdownsProps) { + const [active, setActive] = useState("model") + const breakdown = breakdowns.find((item) => item.dimension === active) + const isTool = active === "tool" + const columns = isTool ? TOOL_COLUMNS : USAGE_COLUMNS + const selected = selectedDimensionValue(search, active) + + return ( +
+
+

+ Breakdowns +

+ + click a row to filter the whole page + +
+ +
+ {breakdowns.map((item) => ( + + ))} +
+ + {breakdown === undefined || breakdown.rows.length === 0 ? ( +

+ No {active} activity in this range. +

+ ) : ( + <> + + + + + {columns.map((column) => ( + + ))} + + + + {breakdown.rows.map((row) => ( + onSelectRow(active, row.key)} + data-selected={row.key === selected ? "" : undefined} + className={cn( + "cursor-pointer border-b border-border/60 transition-colors hover:bg-accent/40", + row.key === selected && "bg-primary/10", + )} + > + + {isTool ? : } + + ))} + +
+ {active} + + {column} +
+ {row.label} +
+

+ {breakdown.totalKeys} keys · session counts overlap where a session used more than + one {active} + {breakdown.totalKeys > breakdown.rows.length + ? ` · + ${breakdown.totalKeys - breakdown.rows.length} more` + : ""} +

+ + )} +
+ ) +} + +const Cell = ({ children }: { children: React.ReactNode }) => ( + {children} +) + +function UsageCells({ row }: { row: OverviewBreakdownRow }) { + return ( + <> + {formatPercent(row.shareOfCost)} + {formatOverviewCount(row.sessions)} + {formatOverviewCount(row.llmCalls)} + {formatNumber(row.tokensPerSession)} + {formatCost(row.cost)} + {formatCost(row.costPerSession)} + {formatErrorRate(row.errorRate)} + {formatDeltaPp(row.errorRateDeltaPp)} + + ) +} + +function ToolCells({ row }: { row: OverviewBreakdownRow }) { + return ( + <> + {formatPercent(row.shareOfCalls)} + {formatOverviewCount(row.sessions)} + {formatOverviewCount(row.toolCalls)} + {formatOverviewCount(row.toolErrors)} + {formatErrorRate(row.errorRate)} + {formatDeltaPp(row.errorRateDeltaPp)} + + ) +} + +/** A key with no previous window has no move to show, which is not a zero. */ +const formatDeltaPp = (pp: number | null): string => + pp === null ? "—" : `${pp < 0 ? "-" : "+"}${Math.abs(pp).toFixed(1)}pp` diff --git a/apps/web/src/components/agent-sessions/overview/overview-filter-toolbar.tsx b/apps/web/src/components/agent-sessions/overview/overview-filter-toolbar.tsx new file mode 100644 index 000000000..a2f0c1c6e --- /dev/null +++ b/apps/web/src/components/agent-sessions/overview/overview-filter-toolbar.tsx @@ -0,0 +1,156 @@ +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from "@maple/ui/components/ui/select" +import { cn } from "@maple/ui/lib/utils" + +import { + OVERVIEW_DIMENSIONS, + compareEnabled, + failingOnly, + overviewFilterPatch, + selectedDimensionValue, + type AgentOverviewSearch, + type OverviewFacetOption, + type OverviewFacets, +} from "@/lib/agent-sessions/overview-search" +import { formatOverviewCount } from "@/lib/agent-sessions/overview-analytics" + +/** Base UI selects need a real value for "no filter"; this never reaches the URL. */ +const ALL = "__all__" + +export interface OverviewFilterToolbarProps { + search: AgentOverviewSearch + facets: OverviewFacets + onSearchChange: (patch: Partial) => void + /** Names the comparison the toggle turns on, e.g. `7d`. */ + windowLabel: string + waiting?: boolean +} + +/** + * Which sessions the board is about. + * + * Six dimensions, one value each: this page is read by narrowing to one thing + * at a time, and every control is the same 30px pill so a control drawn in the + * primary tint reads as "this is narrowing the page" at a glance. + */ +export function OverviewFilterToolbar({ + search, + facets, + onSearchChange, + windowLabel, + waiting = false, +}: OverviewFilterToolbarProps) { + const failing = failingOnly(search) + const compare = compareEnabled(search) + return ( +
+ {OVERVIEW_DIMENSIONS.map((dimension) => ( + onSearchChange(overviewFilterPatch(dimension, value))} + /> + ))} + + + + +
+ ) +} + +function FacetSelect({ + label, + value, + options, + onChange, +}: { + label: string + value: string | undefined + options: ReadonlyArray + onChange: (value: string | undefined) => void +}) { + const set = value !== undefined + return ( + + ) +} diff --git a/apps/web/src/components/agent-sessions/overview/overview-metric-strip.tsx b/apps/web/src/components/agent-sessions/overview/overview-metric-strip.tsx new file mode 100644 index 000000000..d0d0492c1 --- /dev/null +++ b/apps/web/src/components/agent-sessions/overview/overview-metric-strip.tsx @@ -0,0 +1,65 @@ +import { Skeleton } from "@maple/ui/components/ui/skeleton" +import { cn } from "@maple/ui/lib/utils" + +import { deltaToneClass, type OverviewTile } from "@/lib/agent-sessions/overview-analytics" + +export interface OverviewMetricStripProps { + tiles: ReadonlyArray + waiting?: boolean +} + +/** + * Seven readings of the window, left to right, and not a selector: nothing + * below the strip changes when one is read. The grid underneath already draws + * all nine series at once, so a tile that took over a chart would be a step + * backwards from what is already on screen. + */ +export function OverviewMetricStrip({ tiles, waiting = false }: OverviewMetricStripProps) { + return ( +
+ {tiles.map((tile) => ( +
+ + {tile.label} + + + + {tile.value} + + {tile.unit === undefined ? null : ( + {tile.unit} + )} + + + {tile.delta === null ? null : ( + + {tile.delta.text} + + )} + {tile.sub} + +
+ ))} +
+ ) +} + +/** The strip's shape while the summary read is in flight. */ +export function OverviewMetricStripLoading() { + return ( +
+ {Array.from({ length: 7 }).map((_, index) => ( +
+ + + +
+ ))} +
+ ) +} diff --git a/apps/web/src/components/agent-sessions/overview/overview-movers-rail.tsx b/apps/web/src/components/agent-sessions/overview/overview-movers-rail.tsx new file mode 100644 index 000000000..d3f5c5232 --- /dev/null +++ b/apps/web/src/components/agent-sessions/overview/overview-movers-rail.tsx @@ -0,0 +1,104 @@ +import { formatPercent } from "@maple/ui/lib/format" +import { cn } from "@maple/ui/lib/utils" + +import { + OVERVIEW_MOVER_MIN_SESSIONS, + deltaToneClass, + type OverviewCoverage, + type OverviewMover, +} from "@/lib/agent-sessions/overview-analytics" + +export interface OverviewMoversRailProps { + movers: ReadonlyArray + coverage: OverviewCoverage + /** Names the window the moves are measured against, e.g. `7d`. */ + windowLabel: string + /** Applies that mover's dimension filter to the whole board. */ + onSelect: (mover: OverviewMover) => void +} + +/** + * What changed, ranked across every dimension at once. + * + * One line per key rather than per metric, and a magnitude bar measured against + * the worst move on the rail — the rail answers "where do I look first", which + * is an ordering question and not a measurement. + */ +export function OverviewMoversRail({ + movers, + coverage, + windowLabel, + onSelect, +}: OverviewMoversRailProps) { + const worst = movers.reduce((max, mover) => Math.max(max, mover.score), 0) + return ( + + ) +} diff --git a/apps/web/src/components/agent-sessions/overview/overview-scope-row.tsx b/apps/web/src/components/agent-sessions/overview/overview-scope-row.tsx new file mode 100644 index 000000000..93dc523a8 --- /dev/null +++ b/apps/web/src/components/agent-sessions/overview/overview-scope-row.tsx @@ -0,0 +1,63 @@ +import { XmarkIcon } from "@/components/icons" +import { cn } from "@maple/ui/lib/utils" + +import type { OverviewFilterChip } from "@/lib/agent-sessions/overview-search" + +export interface OverviewScopeRowProps { + chips: ReadonlyArray + /** What the filters matched, in one sentence. */ + summary: string + onRemove: (chip: OverviewFilterChip) => void + onClearAll: () => void +} + +/** + * Everything narrowing the board, stated once. + * + * The chips stay visible when the filters match nothing — a reader who has + * narrowed to an empty set needs to see what they narrowed by, not an empty + * page with no explanation. + */ +export function OverviewScopeRow({ chips, summary, onRemove, onClearAll }: OverviewScopeRowProps) { + return ( +
+ + Scope + + {chips.length === 0 ? ( + + all sessions in range + + ) : ( + chips.map((chip) => ( + + )) + )} + {chips.length === 0 ? null : ( + + )} + + {summary} + +
+ ) +} diff --git a/apps/web/src/components/agent-sessions/overview/overview-top-sessions.tsx b/apps/web/src/components/agent-sessions/overview/overview-top-sessions.tsx new file mode 100644 index 000000000..e92f0e890 --- /dev/null +++ b/apps/web/src/components/agent-sessions/overview/overview-top-sessions.tsx @@ -0,0 +1,167 @@ +import { useState } from "react" +import { Link } from "@tanstack/react-router" + +import { formatNumber } from "@maple/ui/lib/format" +import { formatRelativeShort } from "@maple/ui/lib/time-format" +import { cn } from "@maple/ui/lib/utils" + +import type { AgentSessionsSearchState } from "@/components/agent-sessions/agent-sessions-filter-inputs" +import type { AgentSessionRow } from "@/components/agent-sessions/agent-sessions-list" +import { + formatOverviewCount, + formatOverviewDuration, +} from "@/lib/agent-sessions/overview-analytics" +import { formatCost } from "@/lib/agent-sessions/session-summary" +import { sessionLinkWindow } from "@/lib/agent-sessions/session-window" +import { + OVERVIEW_TOP_SESSION_TABS, + type OverviewTopSessionTab, +} from "@/lib/agent-sessions/use-agent-overview" + +const TAB_LABEL = { + cost: "most expensive", + duration: "longest", + errored: "errored", +} satisfies Record + +const COLUMNS = [ + "Agent", + "Model", + "Service", + "Cost", + "Tokens", + "LLM", + "Tools", + "Errors", + "Duration", + "Started", +] as const + +export interface OverviewTopSessionsProps { + sessions: Record> + /** Sessions in the window that failed — what the Errored tab is a sample of. */ + erroredCount: number + /** The board's filters, as the Sessions list takes them. */ + sessionsSearch: AgentSessionsSearchState + waiting?: boolean +} + +/** + * The concrete examples behind the trends above. + * + * Six rows from the sessions list itself, under the board's own filters — the + * same endpoint the list pages, so a row here and the same row there agree. The + * link out carries the filters and drops the ranking: the list is where the + * rest of them are. + */ +export function OverviewTopSessions({ + sessions, + erroredCount, + sessionsSearch, + waiting = false, +}: OverviewTopSessionsProps) { + const [active, setActive] = useState("cost") + const rows = sessions[active] + + return ( +
+
+

+ Top sessions +

+ + the concrete examples behind the trends above + +
+ +
+ {OVERVIEW_TOP_SESSION_TABS.map((tab) => ( + + ))} + + Open in Sessions ↗ + +
+ + {rows.length === 0 ? ( +

+ No sessions match this scope. +

+ ) : ( + + + + + {COLUMNS.map((column) => ( + + ))} + + + + {rows.map((row) => ( + + + {row.firstAgentName || "—"} + {row.models[0] ?? "—"} + {row.serviceNames[0] ?? "—"} + {formatCost(row.cost)} + {formatNumber(row.totalTokens)} + {formatOverviewCount(row.llmCalls)} + {formatOverviewCount(row.toolCalls)} + {formatOverviewCount(row.errorSpanCount)} + {formatOverviewDuration(row.durationMs)} + {formatRelativeShort(row.startTime)} + + ))} + +
+ Session + + {column} +
+ + {row.sessionId} + +
+ )} +
+ ) +} + +const Cell = ({ children }: { children: React.ReactNode }) => ( + + {children} + +) diff --git a/apps/web/src/components/agent-sessions/overview/overview-trends.tsx b/apps/web/src/components/agent-sessions/overview/overview-trends.tsx new file mode 100644 index 000000000..7e3c7af18 --- /dev/null +++ b/apps/web/src/components/agent-sessions/overview/overview-trends.tsx @@ -0,0 +1,115 @@ +import type { ReactNode } from "react" + +import { cn } from "@maple/ui/lib/utils" + +import { + deltaToneClass, + type OverviewChartSummary, + type OverviewModelMix, + type OverviewSeriesPoint, +} from "@/lib/agent-sessions/overview-analytics" + +export interface OverviewTrendsProps { + charts: ReadonlyArray + /** One point per bucket of the selected window, oldest first. */ + series: ReadonlyArray + /** The previous period, already shifted onto this axis. Empty with compare off. */ + previousSeries: ReadonlyArray + modelMix: OverviewModelMix + /** The bucket width and range, e.g. `6h buckets · previous 7 days`. */ + note: string + /** The "What changed" rail, placed beside the grid. */ + rail: ReactNode + waiting?: boolean +} + +/** + * Nine readings of one window at one bucket width, in one grid. + * + * Small multiples rather than one chart with a metric picker: the questions a + * regression raises are "did anything else move at the same instant", and that + * is a comparison across charts, not a sequence of them. + */ +export function OverviewTrends({ + charts, + series, + previousSeries, + modelMix, + note, + rail, + waiting = false, +}: OverviewTrendsProps) { + return ( +
+
+

Trends

+ {note} +
+ +
+
+ {charts.map((chart) => ( + 0} + /> + ))} +
+
{rail}
+
+
+ ) +} + +/** + * One small multiple. + * + * The plot area is a placeholder in this pass — the headline, the unit and the + * delta are the parts the rest of the page is wired to, and the marks land here + * without moving anything above them. + */ +function ChartCell({ + chart, + buckets, + ghost, +}: { + chart: OverviewChartSummary + buckets: number + ghost: boolean +}) { + return ( +
+
+ {chart.title} + + + {chart.value} + + {chart.delta === null ? null : ( + + {chart.delta.text} + + )} + + {chart.unit} +
+
+ + {buckets === 0 ? "no data in range" : `${buckets} buckets${ghost ? " · ghost" : ""}`} + +
+
+ ) +} diff --git a/apps/web/src/components/agent-sessions/tools/agent-sessions-tabs.tsx b/apps/web/src/components/agent-sessions/tools/agent-sessions-tabs.tsx new file mode 100644 index 000000000..0006ca46f --- /dev/null +++ b/apps/web/src/components/agent-sessions/tools/agent-sessions-tabs.tsx @@ -0,0 +1,84 @@ +import { Link } from "@tanstack/react-router" + +import { cn } from "@maple/ui/lib/utils" + +import { ChartBarTrendUpIcon, LayersIcon } from "@/components/icons" +import { pickTimeRangeSearch, type TimeRangeSearch } from "@/components/time-range-picker/search" + +/** The two readings of the same spans: the whole population, or one session at a time. */ +export type AgentSessionsTab = "overview" | "sessions" + +/** + * The tab strip both Agent Sessions pages carry. + * + * Real links rather than a `Tabs` widget, because the two tabs are two routes: + * middle-click and Copy link have to work, and the browser's own Back is what + * undoes the switch. Only the window travels between them — the overview's + * dimension filters mean nothing to a list that pages one session at a time. + * + * The Sessions list has no time picker (it is fixed to a rolling week), so a + * jump from there carries no window and this page falls back to its own default. + */ +export function AgentSessionsTabs({ + active, + search, + className, +}: { + active: AgentSessionsTab + /** The current window, carried across. Absent from the Sessions list, which has none. */ + search?: TimeRangeSearch + className?: string +}) { + const window = search === undefined ? {} : pickTimeRangeSearch(search) + return ( + + ) +} + +function TabLink({ + to, + search, + active, + icon, + children, +}: { + to: "/agent-sessions" | "/agent-sessions/overview" + search: Record + active: boolean + icon: React.ReactNode + children: React.ReactNode +}) { + return ( + + {icon} + {children} + + ) +} diff --git a/apps/web/src/lab/agent-overview-fixture.ts b/apps/web/src/lab/agent-overview-fixture.ts new file mode 100644 index 000000000..41d2575c1 --- /dev/null +++ b/apps/web/src/lab/agent-overview-fixture.ts @@ -0,0 +1,562 @@ +// Two boards' worth of synthetic agent traffic, for `/lab/agent-overview` and +// for the view's own test. +// +// `ai_trace_index` does not exist in the local Tinybird container, so the real +// page has nothing to draw locally; this is where its layout gets looked at. +// +// Every figure is built from PER-SESSION rates rather than typed in as totals, +// so the counts, the ratios and the rates in a bucket can never contradict each +// other the way hand-written fixture numbers do. The scenarios are the two the +// design was drawn for: a healthy week, and a day with a step change at 14:00 +// UTC that the movers rail is supposed to find. +// +// The filters are NOT applied to these numbers — the lab writes them to local +// state so every control, chip and row highlight works, and the readings stay +// put so a layout change is the only thing that moves on screen. + +import { formatWarehouseDateTime } from "@maple/query-engine" + +import type { AgentSessionRow } from "@/components/agent-sessions/agent-sessions-list" +import { + EMPTY_OVERVIEW_MEASURES, + type AgentOverviewInput, + type OverviewBreakdownEntry, + type OverviewMeasurePoint, + type OverviewMeasures, + type OverviewModelMixRow, +} from "@/lib/agent-sessions/overview-analytics" +import { + OVERVIEW_DIMENSIONS, + type OverviewDimension, + type OverviewFacets, +} from "@/lib/agent-sessions/overview-search" +import type { OverviewTopSessionTab } from "@/lib/agent-sessions/use-agent-overview" + +export const OVERVIEW_SCENARIOS = ["healthy7d", "regression24h"] as const +export type OverviewScenario = (typeof OVERVIEW_SCENARIOS)[number] + +export interface OverviewFixture { + readonly scenario: OverviewScenario + readonly windowLabel: string + readonly window: { readonly startTime: string; readonly endTime: string } + /** Ready for `buildAgentOverviewData`, minus the `compare` the lab owns. */ + readonly input: Omit + readonly facets: OverviewFacets + readonly topSessions: Record> +} + +/* ------------------------------------------------------------------------------------------------- + * Shapes + * -----------------------------------------------------------------------------------------------*/ + +/** One bucket's behaviour, as a reader would describe it. */ +interface SessionRates { + sessions: number + /** Sessions with at least one failed span, 0–1. */ + errorRate: number + llmPerSession: number + /** Model-call SPANS per netted call — gateway mirrors and wrapper roll-ups. */ + spanFactor: number + llmErrorRate: number + toolsPerSession: number + toolErrorRate: number + costPerSession: number + tokensPerSession: number + /** Cache reads over everything that could have been a prompt read, 0–1. */ + cacheShare: number + pricedShare: number + p50Ms: number + p95Ms: number + llmP50Ms: number + llmP95Ms: number +} + +const HEALTHY: SessionRates = { + sessions: 114, + errorRate: 0.024, + llmPerSession: 8.4, + spanFactor: 1.18, + llmErrorRate: 0.019, + toolsPerSession: 6.3, + toolErrorRate: 0.031, + costPerSession: 0.264, + tokensPerSession: 67_400, + cacheShare: 0.71, + pricedShare: 0.94, + p50Ms: 42_000, + p95Ms: 96_000, + llmP50Ms: 1_900, + llmP95Ms: 7_400, +} + +/** The step the investigating board is drawn around. Sessions barely move. */ +const REGRESSED: SessionRates = { + ...HEALTHY, + errorRate: 0.26, + llmErrorRate: 0.161, + toolsPerSession: 15.8, + toolErrorRate: 0.19, + costPerSession: 0.378, + tokensPerSession: 118_000, + cacheShare: 0.12, + p95Ms: 227_000, + llmP50Ms: 3_100, + llmP95Ms: 18_600, +} + +/** A deterministic wobble, so a board looks like traffic and not like a ruler. */ +const wobble = (index: number, amplitude: number): number => + 1 + amplitude * Math.sin(index * 1.7) + (amplitude / 2) * Math.sin(index * 0.53) + +const scaleRates = (rates: SessionRates, index: number): SessionRates => ({ + ...rates, + sessions: Math.round(rates.sessions * wobble(index, 0.18)), + costPerSession: rates.costPerSession * wobble(index, 0.09), + tokensPerSession: rates.tokensPerSession * wobble(index, 0.07), + toolsPerSession: rates.toolsPerSession * wobble(index, 0.08), + errorRate: rates.errorRate * wobble(index, 0.22), + llmErrorRate: rates.llmErrorRate * wobble(index, 0.2), + p95Ms: rates.p95Ms * wobble(index, 0.11), +}) + +/** + * One bucket's rates, as the API would report them. + * + * The token split is disjoint and adds to the total, and `cacheShare` is + * exactly the ratio the cache-hit chart divides — the numbers agree because + * they come from one place. + */ +function measuresOf(rates: SessionRates): OverviewMeasures { + const sessions = Math.max(0, Math.round(rates.sessions)) + const llmCalls = Math.round(sessions * rates.llmPerSession) + const llmCallSpans = Math.round(llmCalls * rates.spanFactor) + const toolCalls = Math.round(sessions * rates.toolsPerSession) + const tokens = Math.round(sessions * rates.tokensPerSession) + const prompt = tokens * 0.78 + return { + ...EMPTY_OVERVIEW_MEASURES, + sessions, + erroredSessions: Math.round(sessions * rates.errorRate), + llmCalls, + llmCallSpans, + erroredLlmCalls: Math.round(llmCallSpans * rates.llmErrorRate), + toolCalls, + erroredToolCalls: Math.round(toolCalls * rates.toolErrorRate), + cost: Number((sessions * rates.costPerSession).toFixed(2)), + pricedLlmCalls: Math.round(llmCallSpans * rates.pricedShare), + tokens, + inputTokens: Math.round(prompt * (1 - rates.cacheShare)), + cacheReadTokens: Math.round(prompt * rates.cacheShare), + cacheWriteTokens: Math.round(tokens * 0.04), + outputTokens: Math.round(tokens * 0.12), + reasoningTokens: Math.round(tokens * 0.06), + sessionDurationP50Ms: rates.p50Ms, + sessionDurationP95Ms: rates.p95Ms, + llmDurationP50Ms: rates.llmP50Ms, + llmDurationP95Ms: rates.llmP95Ms, + } +} + +/** Counts sum; quantiles do not, so the window's own are passed in. */ +function foldMeasures( + points: ReadonlyArray, + quantiles: Pick< + OverviewMeasures, + | "sessionDurationP50Ms" + | "sessionDurationP95Ms" + | "llmDurationP50Ms" + | "llmDurationP95Ms" + >, +): OverviewMeasures { + const sum = points.reduce( + (total, point) => ({ + ...total, + sessions: total.sessions + point.sessions, + erroredSessions: total.erroredSessions + point.erroredSessions, + llmCalls: total.llmCalls + point.llmCalls, + llmCallSpans: total.llmCallSpans + point.llmCallSpans, + erroredLlmCalls: total.erroredLlmCalls + point.erroredLlmCalls, + toolCalls: total.toolCalls + point.toolCalls, + erroredToolCalls: total.erroredToolCalls + point.erroredToolCalls, + cost: total.cost + point.cost, + pricedLlmCalls: total.pricedLlmCalls + point.pricedLlmCalls, + tokens: total.tokens + point.tokens, + inputTokens: total.inputTokens + point.inputTokens, + cacheReadTokens: total.cacheReadTokens + point.cacheReadTokens, + cacheWriteTokens: total.cacheWriteTokens + point.cacheWriteTokens, + outputTokens: total.outputTokens + point.outputTokens, + reasoningTokens: total.reasoningTokens + point.reasoningTokens, + }), + EMPTY_OVERVIEW_MEASURES, + ) + return { ...sum, cost: Number(sum.cost.toFixed(2)), ...quantiles } +} + +/* ------------------------------------------------------------------------------------------------- + * Breakdowns + * -----------------------------------------------------------------------------------------------*/ + +/** One key's story: its share of the window, and what it did differently. */ +interface KeyStory { + key: string + share: number + current?: Partial + previous?: Partial +} + +const stories = (...entries: ReadonlyArray) => entries + +const STORIES = { + model: stories( + { + key: "claude-opus-5", + share: 0.42, + // The regression the rail is supposed to put first. + current: { llmErrorRate: 0.161, costPerSession: 0.41 }, + previous: { llmErrorRate: 0.019, costPerSession: 0.29 }, + }, + { key: "gpt-5.5", share: 0.23 }, + { key: "claude-sonnet-5", share: 0.16, current: { costPerSession: 0.09 } }, + { key: "gemini-3-pro", share: 0.11, current: { tokensPerSession: 94_000 } }, + { key: "gpt-5.6", share: 0.05 }, + { key: "llama-4-70b", share: 0.03, current: { costPerSession: 0.004 } }, + ), + agent: stories( + { + key: "release-captain", + share: 0.31, + current: { toolsPerSession: 15.8, errorRate: 0.24 }, + previous: { toolsPerSession: 6.1, errorRate: 0.022 }, + }, + { key: "code-reviewer", share: 0.27 }, + { key: "docs-writer", share: 0.18, current: { tokensPerSession: 122_000 } }, + { key: "triage-bot", share: 0.14 }, + { key: "", share: 0.1 }, + ), + service: stories( + { key: "api", share: 0.46, current: { p95Ms: 188_000 }, previous: { p95Ms: 94_000 } }, + { key: "worker", share: 0.29 }, + { key: "cli", share: 0.17 }, + { key: "landing", share: 0.08 }, + ), + framework: stories( + { key: "eve", share: 0.58 }, + { key: "openrouter", share: 0.24, current: { costPerSession: 0.39 } }, + { key: "langchain", share: 0.13 }, + { key: "vercel-ai", share: 0.05 }, + ), + environment: stories( + { + key: "production", + share: 0.64, + current: { errorRate: 0.19 }, + previous: { errorRate: 0.021 }, + }, + { key: "staging", share: 0.26 }, + { key: "development", share: 0.1 }, + ), + tool: stories( + { + key: "run_tests", + share: 0.34, + current: { toolErrorRate: 0.28, toolsPerSession: 9.4 }, + previous: { toolErrorRate: 0.04, toolsPerSession: 3.6 }, + }, + { key: "read_file", share: 0.26 }, + { key: "search_code", share: 0.19 }, + { key: "apply_patch", share: 0.13, current: { toolErrorRate: 0.09 } }, + { key: "web_fetch", share: 0.08 }, + ), +} satisfies Record> + +/** + * One dimension's rows. + * + * The stories above are the REGRESSED board's; a healthy week gets a small + * deterministic drift per key instead, so its movers rail has the handful of + * modest moves a healthy week actually has rather than a copy of the outage. + */ +function breakdownEntries( + dimension: OverviewDimension, + current: SessionRates, + previous: SessionRates, + regressed: boolean, +): ReadonlyArray { + // Seeded per dimension as well as per row, so six tables do not print six + // copies of one key's drift and the rail ranks six different things. + const seed = OVERVIEW_DIMENSIONS.indexOf(dimension) * 7 + const drift = (rates: SessionRates, at: number): Partial => ({ + costPerSession: rates.costPerSession * wobble(seed + at, 0.16), + tokensPerSession: rates.tokensPerSession * wobble(seed + at + 2, 0.24), + errorRate: rates.errorRate * wobble(seed + at + 1, 0.3), + toolsPerSession: rates.toolsPerSession * wobble(seed + at + 3, 0.18), + }) + return STORIES[dimension].map((story, index) => ({ + key: story.key, + current: measuresOf({ + ...current, + sessions: current.sessions * story.share, + ...drift(current, index * 2), + ...(regressed ? story.current : undefined), + }), + previous: measuresOf({ + ...previous, + sessions: previous.sessions * story.share, + ...drift(previous, index * 2 + 11), + ...(regressed ? story.previous : undefined), + }), + })) +} + +/* ------------------------------------------------------------------------------------------------- + * Model mix + * -----------------------------------------------------------------------------------------------*/ + +/** Seven models, so the sixth and the seventh fold into the `other` band. */ +const MIX_MODELS = [ + { model: "claude-opus-5", base: 0.34, regressed: 0.62 }, + { model: "gpt-5.5", base: 0.24, regressed: 0.14 }, + { model: "claude-sonnet-5", base: 0.16, regressed: 0.09 }, + { model: "gemini-3-pro", base: 0.12, regressed: 0.07 }, + { model: "gpt-5.6", base: 0.07, regressed: 0.04 }, + { model: "llama-4-70b", base: 0.04, regressed: 0.02 }, + { model: "mistral-large-3", base: 0.03, regressed: 0.02 }, +] as const + +function modelMixRows( + buckets: ReadonlyArray<{ bucket: number; spans: number; regressed: boolean }>, +): ReadonlyArray { + return buckets.flatMap(({ bucket, spans, regressed }, index) => + MIX_MODELS.map((model) => ({ + bucket, + model: model.model, + llmCallSpans: Math.max( + 1, + Math.round(spans * (regressed ? model.regressed : model.base) * wobble(index, 0.06)), + ), + })), + ) +} + +/* ------------------------------------------------------------------------------------------------- + * Top sessions + * -----------------------------------------------------------------------------------------------*/ + +const SESSION_SEEDS = [ + { agent: "release-captain", model: "claude-opus-5", service: "api", vendor: "eve" }, + { agent: "code-reviewer", model: "gpt-5.5", service: "api", vendor: "openrouter" }, + { agent: "docs-writer", model: "gemini-3-pro", service: "worker", vendor: "eve" }, + { agent: "triage-bot", model: "claude-sonnet-5", service: "worker", vendor: "langchain" }, + { agent: "release-captain", model: "claude-opus-5", service: "cli", vendor: "eve" }, + { agent: "code-reviewer", model: "gpt-5.6", service: "api", vendor: "vercel-ai" }, +] as const + +function topSessions( + tab: OverviewTopSessionTab, + endMs: number, + rates: SessionRates, +): ReadonlyArray { + return SESSION_SEEDS.map((seed, index) => { + const rank = SESSION_SEEDS.length - index + const startMs = endMs - (index + 1) * 37 * 60_000 + const durationMs = + tab === "duration" ? rates.p95Ms * (1.6 + index * 0.2) : rates.p50Ms * (1 + index * 0.1) + const errors = tab === "errored" ? rank * 3 : index === 0 ? 2 : 0 + const llmCalls = Math.round(rates.llmPerSession * (tab === "cost" ? rank * 1.6 : 1.2)) + return { + sessionId: `${seed.agent}-${(2261 + index * 17).toString(16)}`, + vendorId: seed.vendor, + traceCount: 1 + (index % 3), + spanCount: 40 + index * 11, + errorSpanCount: errors, + toolErrorCount: Math.round(errors * 0.6), + turnErrorCount: errors - Math.round(errors * 0.6), + serviceNames: [seed.service], + models: [seed.model], + agentNames: [seed.agent], + firstAgentName: seed.agent, + llmCalls, + toolCalls: Math.round(rates.toolsPerSession * (tab === "cost" ? rank : 1.4)), + totalTokens: Math.round(rates.tokensPerSession * (tab === "cost" ? rank * 1.4 : 1.1)), + inputTokens: 0, + cacheReadTokens: 0, + cacheWriteTokens: 0, + outputTokens: 0, + reasoningTokens: 0, + cost: Number((rates.costPerSession * (tab === "cost" ? rank * 2.4 : 1.3)).toFixed(2)), + startTime: formatWarehouseDateTime(startMs), + endTime: formatWarehouseDateTime(startMs + durationMs), + durationMs: Math.round(durationMs), + hasDetails: true, + } + }) +} + +/* ------------------------------------------------------------------------------------------------- + * The scenarios + * -----------------------------------------------------------------------------------------------*/ + +const HOUR_MS = 60 * 60_000 + +/** The hour the investigating board's step happens, in UTC so a board looks the + * same wherever it is opened. */ +export const OVERVIEW_REGRESSION_HOUR_UTC = 14 + +interface ScenarioSpec { + readonly windowLabel: string + readonly bucketSeconds: number + readonly buckets: number + readonly current: SessionRates + readonly previous: SessionRates + /** True where the bucket is past the step. Healthy weeks have none. */ + readonly regressedAt: (bucketMs: number) => boolean +} + +const SPECS = { + healthy7d: { + windowLabel: "7d", + bucketSeconds: 6 * 3_600, + buckets: 28, + current: HEALTHY, + previous: { ...HEALTHY, sessions: 121, costPerSession: 0.276, errorRate: 0.027 }, + regressedAt: (_bucketMs: number) => false, + }, + regression24h: { + windowLabel: "24h", + bucketSeconds: 3_600, + buckets: 24, + current: { ...HEALTHY, sessions: 52 }, + previous: { ...HEALTHY, sessions: 51 }, + regressedAt: (bucketMs: number) => + new Date(bucketMs).getUTCHours() >= OVERVIEW_REGRESSION_HOUR_UTC, + }, +} satisfies Record + +/** + * One board's worth of data, from one frozen timestamp. + * + * The window ends on the hour so the regression scenario's buckets line up with + * the step it is drawn around. + */ +export function buildOverviewFixture(scenario: OverviewScenario, nowMs: number): OverviewFixture { + const spec = SPECS[scenario] + const bucketMs = spec.bucketSeconds * 1_000 + const endMs = Math.floor(nowMs / HOUR_MS) * HOUR_MS + const startMs = endMs - spec.buckets * bucketMs + const windowMs = endMs - startMs + + const buckets = Array.from({ length: spec.buckets }, (_, index) => { + const bucket = startMs + index * bucketMs + const regressed = spec.regressedAt(bucket) + return { bucket, index, regressed } + }) + + const series: ReadonlyArray = buckets.map( + ({ bucket, index, regressed }) => ({ + bucket, + ...measuresOf( + scaleRates( + regressed ? { ...REGRESSED, sessions: spec.current.sessions } : spec.current, + index, + ), + ), + }), + ) + const previousSeries: ReadonlyArray = buckets.map( + ({ bucket, index }) => ({ + bucket: bucket - windowMs, + ...measuresOf(scaleRates(spec.previous, index + 3)), + }), + ) + + // The regressed scenario's window mixes both shapes, so the tiles read the + // whole window while the grid shows where it turned. + const regressedShare = buckets.filter((b) => b.regressed).length / spec.buckets + const blend = (healthy: number, bad: number) => + healthy * (1 - regressedShare) + bad * regressedShare + const currentRates: SessionRates = { + ...spec.current, + errorRate: blend(spec.current.errorRate, REGRESSED.errorRate), + llmErrorRate: blend(spec.current.llmErrorRate, REGRESSED.llmErrorRate), + toolsPerSession: blend(spec.current.toolsPerSession, REGRESSED.toolsPerSession), + toolErrorRate: blend(spec.current.toolErrorRate, REGRESSED.toolErrorRate), + costPerSession: blend(spec.current.costPerSession, REGRESSED.costPerSession), + cacheShare: blend(spec.current.cacheShare, REGRESSED.cacheShare), + p95Ms: blend(spec.current.p95Ms, REGRESSED.p95Ms), + } + + const current = foldMeasures(series, { + sessionDurationP50Ms: currentRates.p50Ms, + sessionDurationP95Ms: currentRates.p95Ms, + llmDurationP50Ms: currentRates.llmP50Ms, + llmDurationP95Ms: currentRates.llmP95Ms, + }) + const previous = foldMeasures(previousSeries, { + sessionDurationP50Ms: spec.previous.p50Ms, + sessionDurationP95Ms: spec.previous.p95Ms, + llmDurationP50Ms: spec.previous.llmP50Ms, + llmDurationP95Ms: spec.previous.llmP95Ms, + }) + + // Breakdown rows are measured over the WINDOW, like the tiles above them — + // a table summing to a fraction of the strip would read as a bug. + const breakdownCurrent: SessionRates = { ...currentRates, sessions: current.sessions } + const breakdownPrevious: SessionRates = { ...spec.previous, sessions: previous.sessions } + const breakdowns = OVERVIEW_DIMENSIONS.map((dimension) => { + const entries = breakdownEntries( + dimension, + breakdownCurrent, + breakdownPrevious, + regressedShare > 0, + ) + return { dimension, entries, totalKeys: entries.length + (dimension === "tool" ? 9 : 4) } + }) + + const facetsFor = (dimension: OverviewDimension) => + STORIES[dimension] + .filter((story) => story.key !== "") + .map((story) => ({ + name: story.key, + count: Math.round(current.sessions * story.share), + })) + const facets: OverviewFacets = { + model: facetsFor("model"), + agent: facetsFor("agent"), + service: facetsFor("service"), + framework: facetsFor("framework"), + environment: facetsFor("environment"), + tool: facetsFor("tool"), + } + + return { + scenario, + windowLabel: spec.windowLabel, + window: { + startTime: formatWarehouseDateTime(startMs), + endTime: formatWarehouseDateTime(endMs), + }, + input: { + current, + previous, + series, + previousSeries, + modelMix: modelMixRows( + buckets.map(({ bucket, index, regressed }) => ({ + bucket, + spans: series[index].llmCallSpans, + regressed, + })), + ), + breakdowns, + bucketSeconds: spec.bucketSeconds, + windowMs: { startMs, endMs }, + windowLabel: spec.windowLabel, + }, + facets, + topSessions: { + cost: topSessions("cost", endMs, currentRates), + duration: topSessions("duration", endMs, currentRates), + errored: topSessions("errored", endMs, currentRates), + }, + } +} diff --git a/apps/web/src/lab/agent-overview-lab.tsx b/apps/web/src/lab/agent-overview-lab.tsx new file mode 100644 index 000000000..1328c2359 --- /dev/null +++ b/apps/web/src/lab/agent-overview-lab.tsx @@ -0,0 +1,109 @@ +import { useMemo, useState } from "react" + +import { AgentOverviewView } from "@/components/agent-sessions/overview/agent-overview-view" +import { buildAgentOverviewData } from "@/lib/agent-sessions/overview-analytics" +import { compareEnabled, type AgentOverviewSearch } from "@/lib/agent-sessions/overview-search" + +import { + OVERVIEW_SCENARIOS, + buildOverviewFixture, + type OverviewScenario, +} from "./agent-overview-fixture" + +/** + * The overview board without a warehouse behind it. + * + * The page is the real one — the route mounts this same view — over two + * synthetic boards: a healthy week, and a day whose 14:00 UTC step is what the + * movers rail and the error charts are for. The URL is stood in for by local + * state, so every control works: the selects and the toggles narrow the scope + * row, a breakdown row filters the page, and a mover line does the same. + * + * The width buttons matter here: the strip folds from seven columns to four to + * two, and the trends grid from three columns to two to one, at container + * widths the layout's content column does not have in this page. + */ +const WIDTHS = [ + { label: "Full", value: null }, + { label: "1400px", value: 1400 }, + { label: "1100px", value: 1100 }, + { label: "820px", value: 820 }, +] as const + +const SCENARIO_LABEL = { + healthy7d: "healthy · 7d", + regression24h: "regression · 24h", +} satisfies Record + +export function AgentOverviewLab() { + // One timestamp for the life of the mount: the whole fixture is derived from + // it, and a re-derived "now" while you look at a spacing change is noise. + const [nowMs] = useState(() => Date.now()) + const [scenario, setScenario] = useState("healthy7d") + const [search, setSearch] = useState({}) + const [width, setWidth] = useState(null) + + const fixture = useMemo(() => buildOverviewFixture(scenario, nowMs), [scenario, nowMs]) + const data = useMemo( + () => buildAgentOverviewData({ ...fixture.input, compare: compareEnabled(search) }), + [fixture, search], + ) + + const onSearchChange = (patch: Partial) => + setSearch((previous) => ({ ...previous, ...patch })) + + return ( +
+
+ {OVERVIEW_SCENARIOS.map((option) => ( + + ))} + + {WIDTHS.map((option) => ( + + ))} + + {JSON.stringify(search)} + +
+ + {/* `@container/page` because the page's breakpoints are container queries + against the layout's content column, which is not mounted here. */} +
+ +
+
+ ) +} diff --git a/apps/web/src/lab/registry.ts b/apps/web/src/lab/registry.ts index c50c661d7..e705c9751 100644 --- a/apps/web/src/lab/registry.ts +++ b/apps/web/src/lab/registry.ts @@ -99,6 +99,14 @@ export const LAB_ENTRIES: ReadonlyArray = [ kind: "lab", session: "none", }, + { + path: "/lab/agent-overview", + title: "Agent overview", + description: + "The `/agent-sessions/overview` board over two synthetic weeks — a healthy one, and a day whose 14:00 step lifts the error rate, the tool calls per session and the cost per session while the cache-read band collapses.", + kind: "lab", + session: "none", + }, { path: "/lab/agent-sessions", title: "Agent sessions list", diff --git a/apps/web/src/lib/agent-sessions/overview-analytics.test.ts b/apps/web/src/lib/agent-sessions/overview-analytics.test.ts new file mode 100644 index 000000000..85699671e --- /dev/null +++ b/apps/web/src/lib/agent-sessions/overview-analytics.test.ts @@ -0,0 +1,431 @@ +import { describe, expect, it } from "vitest" + +import { + EMPTY_OVERVIEW_MEASURES, + OVERVIEW_MOVER_LIMIT, + buildAgentOverviewData, + buildBreakdownRows, + buildModelMix, + buildMovers, + buildOverviewSeries, + buildOverviewTiles, + cacheHitRatio, + costPerSession, + formatOverviewCount, + formatOverviewDuration, + formatPerSession, + llmErrorRate, + overviewDelta, + overviewScopeSummary, + pricedShare, + sessionErrorRate, + shiftOverviewSeries, + tokenBandValues, + toolErrorRate, + tokensPerSession, + type OverviewMeasures, +} from "./overview-analytics" + +const measures = (overrides: Partial): OverviewMeasures => ({ + ...EMPTY_OVERVIEW_MEASURES, + ...overrides, +}) + +describe("derivations", () => { + it("divides by zero as zero rather than as NaN", () => { + const empty = EMPTY_OVERVIEW_MEASURES + for (const value of [ + sessionErrorRate(empty), + llmErrorRate(empty), + toolErrorRate(empty), + costPerSession(empty), + tokensPerSession(empty), + cacheHitRatio(empty), + pricedShare(empty), + ]) { + expect(value).toBe(0) + } + }) + + it("divides the LLM error rate by the raw span population, not the netted volume", () => { + // A mirrored call that failed twice reads above 100% against `llmCalls`. + const row = measures({ llmCalls: 5, llmCallSpans: 10, erroredLlmCalls: 2 }) + expect(llmErrorRate(row)).toBe(0.2) + }) + + it("measures the cache hit ratio against everything that could have been a prompt read", () => { + expect(cacheHitRatio(measures({ inputTokens: 30, cacheReadTokens: 70 }))).toBe(0.7) + }) +}) + +describe("tokenBandValues", () => { + it("splits the five bands when they carry anything", () => { + const bands = tokenBandValues( + measures({ + tokens: 100, + inputTokens: 40, + cacheReadTokens: 30, + cacheWriteTokens: 10, + outputTokens: 15, + reasoningTokens: 5, + }), + ) + expect(bands).toEqual({ + input: 40, + cacheRead: 30, + cacheWrite: 10, + output: 15, + reasoning: 5, + total: 0, + }) + }) + + it("falls back to one band for a row materialized before the bucket columns", () => { + const bands = tokenBandValues(measures({ tokens: 900 })) + expect(bands.total).toBe(900) + expect(bands.input).toBe(0) + }) + + it("leaves every band at zero when there are no tokens at all", () => { + expect(tokenBandValues(EMPTY_OVERVIEW_MEASURES).total).toBe(0) + }) +}) + +describe("overviewDelta", () => { + it("moves a rate in percentage points, never in percent", () => { + const delta = overviewDelta(0.024, 0.26, { unit: "points", riseIs: "bad" }) + expect(delta?.pp).toBeCloseTo(23.6, 5) + expect(delta?.percent).toBeNull() + expect(delta?.text).toBe("+23.6pp") + expect(delta?.tone).toBe("bad") + }) + + it("grades a fall in a rise-is-bad metric as good", () => { + expect(overviewDelta(0.26, 0.024, { unit: "points", riseIs: "bad" })?.tone).toBe("good") + }) + + it("keeps a totals metric neutral in both directions", () => { + expect(overviewDelta(100, 180, { unit: "percent", riseIs: "neutral" })?.tone).toBe("neutral") + expect(overviewDelta(180, 100, { unit: "percent", riseIs: "neutral" })?.tone).toBe("neutral") + }) + + it("grades a rise in a rise-is-good metric as good", () => { + expect(overviewDelta(0.12, 0.71, { unit: "points", riseIs: "good" })?.tone).toBe("good") + }) + + it("refuses a percentage against a window of zero", () => { + expect(overviewDelta(0, 42, { unit: "percent", riseIs: "bad" })).toBeNull() + }) + + it("reads a move too small to matter as flat and neutral", () => { + const delta = overviewDelta(0.2, 0.2001, { unit: "points", riseIs: "bad" }) + expect(delta?.direction).toBe("flat") + expect(delta?.tone).toBe("neutral") + }) + + it("moves a duration by a duration and still reports its percent", () => { + const delta = overviewDelta(96_000, 227_000, { unit: "duration", riseIs: "bad" }) + expect(delta?.absolute).toBe(131_000) + expect(delta?.text).toBe("+2.2min") + expect(delta?.percent).toBeCloseTo(1.3646, 3) + }) + + it("signs a fall", () => { + expect(overviewDelta(200, 100, { unit: "percent", riseIs: "neutral" })?.text).toBe("-50%") + }) +}) + +describe("formatters", () => { + it("prints counts in full up to a million and compacts past it", () => { + expect(formatOverviewCount(1243)).toBe((1243).toLocaleString()) + expect(formatOverviewCount(2_400_000)).toBe("2.4M") + }) + + it("keeps a decimal on a per-session ratio while it has one to keep", () => { + expect(formatPerSession(6.34)).toBe("6.3") + expect(formatPerSession(420)).toBe((420).toLocaleString()) + }) + + it("reads a zero duration as nothing measured rather than as 0μs", () => { + expect(formatOverviewDuration(0)).toBe("—") + }) +}) + +describe("buildOverviewTiles", () => { + const current = measures({ + sessions: 100, + erroredSessions: 12, + cost: 40, + tokens: 1_000, + toolCalls: 500, + llmCallSpans: 200, + pricedLlmCalls: 188, + sessionDurationP50Ms: 40_000, + sessionDurationP95Ms: 90_000, + }) + const previous = measures({ sessions: 80, erroredSessions: 4, cost: 40, tokens: 800 }) + + it("builds seven tiles in the strip's order", () => { + expect(buildOverviewTiles(current, previous, { compare: true, windowLabel: "7d" }).map((t) => t.id)).toEqual([ + "sessions", + "cost", + "costPerSession", + "tokens", + "errorRate", + "toolCallsPerSession", + "durationP95", + ]) + }) + + it("drops every delta when the comparison is off", () => { + const tiles = buildOverviewTiles(current, previous, { compare: false, windowLabel: "7d" }) + expect(tiles.every((tile) => tile.delta === null)).toBe(true) + }) + + it("grades cost per session but leaves the cost total neutral", () => { + const tiles = buildOverviewTiles(current, previous, { compare: true, windowLabel: "7d" }) + const byId = new Map(tiles.map((tile) => [tile.id, tile])) + expect(byId.get("cost")?.delta?.tone).toBe("neutral") + // $0.50 → $0.40 a session is an improvement even though the bill held. + expect(byId.get("costPerSession")?.delta?.tone).toBe("good") + }) + + it("states the priced coverage under the cost tile", () => { + const tiles = buildOverviewTiles(current, previous, { compare: true, windowLabel: "7d" }) + expect(tiles.find((tile) => tile.id === "cost")?.sub).toBe("priced 94%") + }) +}) + +describe("buildOverviewSeries", () => { + it("derives every per-bucket reading and reads an empty bucket as zero", () => { + const [busy, quiet] = buildOverviewSeries([ + { + bucket: 1_000, + ...measures({ + sessions: 10, + erroredSessions: 1, + cost: 5, + tokens: 1_000, + inputTokens: 300, + cacheReadTokens: 700, + toolCalls: 40, + erroredToolCalls: 4, + llmCalls: 80, + llmCallSpans: 100, + erroredLlmCalls: 5, + sessionDurationP50Ms: 1_000, + sessionDurationP95Ms: 4_000, + }), + }, + { bucket: 2_000, ...EMPTY_OVERVIEW_MEASURES }, + ]) + expect(busy.costPerSession).toBe(0.5) + expect(busy.tokensPerSession).toBe(100) + expect(busy.toolCallsPerSession).toBe(4) + expect(busy.llmCallsPerSession).toBe(8) + expect(busy.sessionErrorRate).toBe(0.1) + expect(busy.llmErrorRate).toBe(0.05) + expect(busy.toolErrorRate).toBe(0.1) + expect(busy.cacheHitRatio).toBe(0.7) + expect(busy.sessionP95Ms).toBe(4_000) + expect(busy.tokenBandShares.cacheRead).toBe(0.7) + expect(busy.tokenBands.cacheRead).toBe(70) + expect(Object.values(quiet.tokenBandShares).every((share) => share === 0)).toBe(true) + expect(quiet.costPerSession).toBe(0) + }) +}) + +describe("shiftOverviewSeries", () => { + it("moves the previous period onto the current period's axis", () => { + const points = buildOverviewSeries([{ bucket: 100, ...EMPTY_OVERVIEW_MEASURES }]) + expect(shiftOverviewSeries(points, 900)[0].bucket).toBe(1_000) + }) +}) + +describe("buildModelMix", () => { + const rows = [1, 2, 3, 4, 5, 6, 7].flatMap((rank) => + [0, 1].map((bucket) => ({ + bucket, + model: `model-${rank}`, + llmCallSpans: 100 - rank * 10, + })), + ) + + it("keeps the five busiest models and folds the tail into one grey band", () => { + const mix = buildModelMix(rows) + expect(mix.models).toEqual([ + "model-1", + "model-2", + "model-3", + "model-4", + "model-5", + "other", + ]) + }) + + it("stacks each bucket to one", () => { + const mix = buildModelMix(rows) + for (const point of mix.points) { + const total = mix.models.reduce((sum, model) => sum + point.shares[model], 0) + expect(total).toBeCloseTo(1, 10) + } + }) + + it("leaves out the other band when nothing was folded", () => { + expect(buildModelMix([{ bucket: 0, model: "solo", llmCallSpans: 4 }]).models).toEqual(["solo"]) + }) + + it("has no points and no models for a window that ran nothing", () => { + expect(buildModelMix([])).toEqual({ models: [], points: [] }) + }) +}) + +describe("buildBreakdownRows", () => { + const entries = [ + { + key: "opus", + current: measures({ sessions: 100, erroredSessions: 10, cost: 60, tokens: 1_000, llmCalls: 200 }), + previous: measures({ sessions: 80, erroredSessions: 4, cost: 40 }), + }, + { + key: "", + current: measures({ sessions: 50, erroredSessions: 0, cost: 40, tokens: 500 }), + previous: EMPTY_OVERVIEW_MEASURES, + }, + ] + + it("measures the cost share against the rows it is showing", () => { + const rows = buildBreakdownRows("model", entries) + expect(rows[0].shareOfCost).toBe(0.6) + expect(rows[1].shareOfCost).toBe(0.4) + }) + + it("names the unattributed key rather than hiding it", () => { + expect(buildBreakdownRows("model", entries)[1].label).toBe("Unattributed") + }) + + it("reports the session error rate for a usage dimension", () => { + const rows = buildBreakdownRows("model", entries) + expect(rows[0].errorRate).toBe(0.1) + expect(rows[0].errorRateDeltaPp).toBeCloseTo(5, 5) + }) + + it("reports the CALL error rate for the tool dimension", () => { + const rows = buildBreakdownRows("tool", [ + { + key: "run_tests", + current: measures({ sessions: 20, toolCalls: 100, erroredToolCalls: 28 }), + previous: measures({ sessions: 20, toolCalls: 50, erroredToolCalls: 2 }), + }, + ]) + expect(rows[0].errorRate).toBe(0.28) + expect(rows[0].errorRateDeltaPp).toBeCloseTo(24, 5) + }) + + it("has no move to show for a key the previous window never saw", () => { + expect(buildBreakdownRows("model", entries)[1].errorRateDeltaPp).toBeNull() + }) +}) + +describe("buildMovers", () => { + const quiet = { + key: "quiet", + current: measures({ sessions: 9, erroredSessions: 9 }), + previous: measures({ sessions: 9 }), + } + const failing = { + key: "opus", + current: measures({ sessions: 100, llmCallSpans: 1_000, erroredLlmCalls: 161 }), + previous: measures({ sessions: 100, llmCallSpans: 1_000, erroredLlmCalls: 19 }), + } + const pricier = { + key: "gpt", + current: measures({ sessions: 100, cost: 40 }), + previous: measures({ sessions: 100, cost: 28 }), + } + + it("drops a key too small to read in either window", () => { + expect(buildMovers([{ dimension: "model", entries: [quiet] }])).toEqual([]) + }) + + it("ranks a rate's points above a ratio's percent", () => { + const movers = buildMovers([{ dimension: "model", entries: [failing, pricier] }]) + expect(movers.map((mover) => mover.key)).toEqual(["opus", "gpt"]) + expect(movers[0].metric).toBe("llmErrorRate") + expect(movers[0].deltaText).toBe("+14.2pp") + expect(movers[0].tone).toBe("bad") + }) + + it("keeps one line per key, its worst metric", () => { + const both = { + key: "opus", + current: measures({ sessions: 100, cost: 80, llmCallSpans: 1_000, erroredLlmCalls: 161 }), + previous: measures({ sessions: 100, cost: 40, llmCallSpans: 1_000, erroredLlmCalls: 19 }), + } + const movers = buildMovers([{ dimension: "model", entries: [both] }]) + expect(movers).toHaveLength(1) + expect(movers[0].metric).toBe("llmErrorRate") + }) + + it("scores the tool error rate only under the tool dimension", () => { + const entry = { + key: "run_tests", + current: measures({ sessions: 100, toolCalls: 100, erroredToolCalls: 28 }), + previous: measures({ sessions: 100, toolCalls: 100, erroredToolCalls: 4 }), + } + expect(buildMovers([{ dimension: "tool", entries: [entry] }])[0]?.metric).toBe("toolErrorRate") + // Under `model` the tool measures are structurally zero, so nothing ranks. + expect(buildMovers([{ dimension: "model", entries: [entry] }])).toEqual([]) + }) + + it("shows at most six lines however many dimensions moved", () => { + const entries = Array.from({ length: 5 }, (_, index) => ({ + ...failing, + key: `key-${index}`, + })) + const movers = buildMovers([ + { dimension: "model", entries }, + { dimension: "agent", entries }, + ]) + expect(movers).toHaveLength(OVERVIEW_MOVER_LIMIT) + }) +}) + +describe("overviewScopeSummary", () => { + it("names the three populations the rest of the board divides by", () => { + expect( + overviewScopeSummary(measures({ sessions: 1_284, llmCalls: 10_842, toolCalls: 8_101 })), + ).toBe( + `${(1284).toLocaleString()} sessions · ${(10842).toLocaleString()} LLM calls · ${(8101).toLocaleString()} tool calls`, + ) + }) +}) + +describe("buildAgentOverviewData", () => { + const input = { + current: measures({ sessions: 100, cost: 40, llmCallSpans: 100, pricedLlmCalls: 94 }), + previous: measures({ sessions: 80, cost: 40 }), + series: [{ bucket: 2_000, ...measures({ sessions: 10 }) }], + previousSeries: [{ bucket: 1_000, ...measures({ sessions: 8 }) }], + modelMix: [{ bucket: 2_000, model: "opus", llmCallSpans: 10 }], + breakdowns: [{ dimension: "model" as const, entries: [], totalKeys: 3 }], + bucketSeconds: 3_600, + windowMs: { startMs: 2_000, endMs: 3_000 }, + windowLabel: "24h", + } + + it("shifts the previous series onto the current window's axis", () => { + const data = buildAgentOverviewData({ ...input, compare: true }) + expect(data.previousSeries[0].bucket).toBe(2_000) + }) + + it("drops the previous series entirely when the comparison is off", () => { + expect(buildAgentOverviewData({ ...input, compare: false }).previousSeries).toEqual([]) + }) + + it("builds nine charts and the priced coverage line", () => { + const data = buildAgentOverviewData({ ...input, compare: true }) + expect(data.charts).toHaveLength(9) + expect(data.coverage.share).toBe(0.94) + }) +}) diff --git a/apps/web/src/lib/agent-sessions/overview-analytics.ts b/apps/web/src/lib/agent-sessions/overview-analytics.ts new file mode 100644 index 000000000..7ebb47043 --- /dev/null +++ b/apps/web/src/lib/agent-sessions/overview-analytics.ts @@ -0,0 +1,1071 @@ +// The view model behind `/agent-sessions/overview`: every number the board +// prints, derived from the two reads that produce it, with no React and no wire +// shape in sight. +// +// Three rules shape this module. +// +// The API reports COUNTS; the page reads RATIOS — cost per session, tokens per +// session, an error rate. Every one of those divides, and a window with no +// sessions in it divides by zero, so `ratio` is the only division here and it +// answers 0 rather than NaN. A tile that reads "NaN" is worse than one that +// reads "0". +// +// Durations arrive in milliseconds (the adapter converts the wire's +// nanoseconds), and quantiles never fold: the window's p95 is the summary's own +// un-bucketed figure, never the mean of the buckets'. +// +// A delta is expressed in the unit its metric is read in. A rate moves in +// percentage POINTS — 2% to 26% is "up 24 points", not "up 1200%" — a ratio +// moves in percent, and a duration moves by a duration. + +import { formatErrorRate, formatLatency, formatNumber, formatPercent } from "@maple/ui/lib/format" + +import { formatCost } from "./session-summary" +import { OVERVIEW_DIMENSIONS, type OverviewDimension } from "./overview-search" + +/* ------------------------------------------------------------------------------------------------- + * Measures — the API's numbers, in the units the client reads + * -----------------------------------------------------------------------------------------------*/ + +/** + * What every overview read reports, so a tile, a point on a chart and a table + * row are the same numbers under different groupings. + * + * Identical to the wire's `AiOverviewMeasures` except that the four quantiles + * are milliseconds here; see `api/warehouse/ai-agent-overview.ts`. + */ +export interface OverviewMeasures { + readonly sessions: number + readonly erroredSessions: number + /** Model calls, netted — a wrapper's roll-up, a gateway's mirror and a + * provider retry of one call are one call. */ + readonly llmCalls: number + /** Model-call SPANS, counted raw. The denominator of the LLM error rate. */ + readonly llmCallSpans: number + readonly erroredLlmCalls: number + readonly toolCalls: number + readonly erroredToolCalls: number + readonly cost: number + /** Netted model calls that carried a price — the coverage behind `cost`. */ + readonly pricedLlmCalls: number + readonly tokens: number + readonly inputTokens: number + readonly cacheReadTokens: number + readonly cacheWriteTokens: number + readonly outputTokens: number + readonly reasoningTokens: number + readonly sessionDurationP50Ms: number + readonly sessionDurationP95Ms: number + readonly llmDurationP50Ms: number + readonly llmDurationP95Ms: number +} + +export const EMPTY_OVERVIEW_MEASURES: OverviewMeasures = { + sessions: 0, + erroredSessions: 0, + llmCalls: 0, + llmCallSpans: 0, + erroredLlmCalls: 0, + toolCalls: 0, + erroredToolCalls: 0, + cost: 0, + pricedLlmCalls: 0, + tokens: 0, + inputTokens: 0, + cacheReadTokens: 0, + cacheWriteTokens: 0, + outputTokens: 0, + reasoningTokens: 0, + sessionDurationP50Ms: 0, + sessionDurationP95Ms: 0, + llmDurationP50Ms: 0, + llmDurationP95Ms: 0, +} + +/** One bucket of a summary series. `bucket` is epoch milliseconds. */ +export interface OverviewMeasurePoint extends OverviewMeasures { + readonly bucket: number +} + +/** One key of a breakdown, over both windows. */ +export interface OverviewBreakdownEntry { + /** `''` is a real key — a span carrying no value for this dimension. */ + readonly key: string + readonly current: OverviewMeasures + readonly previous: OverviewMeasures +} + +/** One (bucket, model) pair of the model-mix read. */ +export interface OverviewModelMixRow { + readonly bucket: number + readonly model: string + readonly llmCallSpans: number +} + +/* ------------------------------------------------------------------------------------------------- + * Derivations + * -----------------------------------------------------------------------------------------------*/ + +/** The only division in this module. A window that ran nothing reads 0. */ +const ratio = (numerator: number, denominator: number): number => + denominator > 0 ? numerator / denominator : 0 + +export const sessionErrorRate = (m: OverviewMeasures): number => + ratio(m.erroredSessions, m.sessions) + +/** `erroredLlmCalls / llmCallSpans` and never `/ llmCalls`: the two populations + * differ by every mirror and wrapper the netting collapses. */ +export const llmErrorRate = (m: OverviewMeasures): number => + ratio(m.erroredLlmCalls, m.llmCallSpans) + +export const toolErrorRate = (m: OverviewMeasures): number => + ratio(m.erroredToolCalls, m.toolCalls) + +export const costPerSession = (m: OverviewMeasures): number => ratio(m.cost, m.sessions) +export const tokensPerSession = (m: OverviewMeasures): number => ratio(m.tokens, m.sessions) +export const toolCallsPerSession = (m: OverviewMeasures): number => ratio(m.toolCalls, m.sessions) +export const llmCallsPerSession = (m: OverviewMeasures): number => ratio(m.llmCalls, m.sessions) + +/** Cache reads over everything that could have been a prompt read. */ +export const cacheHitRatio = (m: OverviewMeasures): number => + ratio(m.cacheReadTokens, m.inputTokens + m.cacheReadTokens) + +/** `cost` is 0 for "nobody priced it" and not for "free" — this is how much of + * the window it actually covers. */ +export const pricedShare = (m: OverviewMeasures): number => + ratio(m.pricedLlmCalls, m.llmCallSpans) + +/* ------------------------------------------------------------------------------------------------- + * Token bands + * -----------------------------------------------------------------------------------------------*/ + +export const OVERVIEW_TOKEN_BANDS = [ + "input", + "cacheRead", + "cacheWrite", + "output", + "reasoning", +] as const +export type OverviewTokenBand = (typeof OVERVIEW_TOKEN_BANDS)[number] + +/** + * The band a row falls back to. Rows materialized before the bucket columns + * existed carry a total and five zeros; showing five empty bands over a + * non-zero total would read as "no tokens". + */ +export const OVERVIEW_TOKEN_FALLBACK_BAND = "total" +export type OverviewTokenBandKey = OverviewTokenBand | typeof OVERVIEW_TOKEN_FALLBACK_BAND + +export const OVERVIEW_TOKEN_BAND_KEYS = [ + ...OVERVIEW_TOKEN_BANDS, + OVERVIEW_TOKEN_FALLBACK_BAND, +] as const + +export const OVERVIEW_TOKEN_BAND_LABEL = { + input: "input", + cacheRead: "cache read", + cacheWrite: "cache write", + output: "output", + reasoning: "reasoning", + total: "tokens", +} satisfies Record + +const emptyBands = (): Record => ({ + input: 0, + cacheRead: 0, + cacheWrite: 0, + output: 0, + reasoning: 0, + total: 0, +}) + +/** Raw token counts per band, with the fallback applied. */ +export function tokenBandValues(m: OverviewMeasures): Record { + const bands = emptyBands() + const split = + m.inputTokens + m.cacheReadTokens + m.cacheWriteTokens + m.outputTokens + m.reasoningTokens + if (split === 0) { + bands.total = m.tokens + return bands + } + bands.input = m.inputTokens + bands.cacheRead = m.cacheReadTokens + bands.cacheWrite = m.cacheWriteTokens + bands.output = m.outputTokens + bands.reasoning = m.reasoningTokens + return bands +} + +/* ------------------------------------------------------------------------------------------------- + * Deltas + * -----------------------------------------------------------------------------------------------*/ + +export type DeltaDirection = "up" | "down" | "flat" +/** How the move reads, not which way it went: a rise in cost is `bad`, a rise + * in sessions is `neutral`, a rise in cache hits is `good`. */ +export type DeltaTone = "good" | "bad" | "neutral" +/** Which unit the change is expressed in. */ +export type DeltaUnit = "percent" | "points" | "duration" + +/** The colour a graded move is drawn in. A neutral move is just a number. */ +export const deltaToneClass = (tone: DeltaTone): string => + tone === "bad" + ? "text-[var(--severity-error)]" + : tone === "good" + ? "text-[var(--severity-info)]" + : "text-muted-foreground" + +export interface OverviewDelta { + /** `after - before`, in the metric's own unit (ms for durations). */ + readonly absolute: number + /** Fractional change (0.43 = +43%); `null` against a zero baseline. */ + readonly percent: number | null + /** Percentage-point change (24 = +24pp); `null` unless the metric is a rate. */ + readonly pp: number | null + readonly direction: DeltaDirection + readonly tone: DeltaTone + /** As a tile prints it, sign included: `+43%`, `24.0pp`, `+2.4s`. */ + readonly text: string +} + +const FLAT_POINTS = 0.05 +const FLAT_PERCENT = 0.001 +const FLAT_MS = 1 + +const signed = (value: number, text: string): string => (value < 0 ? `-${text}` : `+${text}`) + +/** + * The change against the previous window. + * + * `null` where there is no reading to give: a percentage against a baseline of + * zero is "up ∞%", which is not a number anybody acts on. + */ +export function overviewDelta( + before: number, + after: number, + options: { unit: DeltaUnit; riseIs: DeltaTone }, +): OverviewDelta | null { + if (!Number.isFinite(before) || !Number.isFinite(after)) return null + const absolute = after - before + const flatAt = + options.unit === "points" ? FLAT_POINTS / 100 : options.unit === "duration" ? FLAT_MS : 0 + const percent = before === 0 ? null : absolute / before + + if (options.unit === "percent" && percent === null) return null + + const direction: DeltaDirection = + options.unit === "percent" + ? Math.abs(percent ?? 0) < FLAT_PERCENT + ? "flat" + : absolute > 0 + ? "up" + : "down" + : Math.abs(absolute) < flatAt + ? "flat" + : absolute > 0 + ? "up" + : "down" + + const tone: DeltaTone = + direction === "flat" || options.riseIs === "neutral" + ? "neutral" + : direction === "up" + ? options.riseIs + : options.riseIs === "bad" + ? "good" + : "bad" + + if (options.unit === "points") { + const pp = absolute * 100 + return { + absolute, + percent: null, + pp, + direction, + // A move too small to read is not a signed zero: "+0.0pp" reads as a + // rise that rounded away, which is a different claim from "flat". + text: direction === "flat" ? "0pp" : signed(pp, `${Math.abs(pp).toFixed(1)}pp`), + tone, + } + } + if (options.unit === "duration") { + return { + absolute, + percent, + pp: null, + direction, + text: direction === "flat" ? "0s" : signed(absolute, formatLatency(Math.abs(absolute))), + tone, + } + } + return { + absolute, + percent, + pp: null, + direction, + text: direction === "flat" ? "0%" : signed(absolute, formatPercent(Math.abs(percent ?? 0))), + tone, + } +} + +/** The bucket width, as the Trends note states it: `15m`, `6h`, `1d`. */ +export function bucketWidthLabel(seconds: number): string { + if (seconds >= 86_400) return `${Math.round(seconds / 86_400)}d` + if (seconds >= 3_600) return `${Math.round(seconds / 3_600)}h` + return `${Math.round(seconds / 60)}m` +} + +/* ------------------------------------------------------------------------------------------------- + * Formatters + * -----------------------------------------------------------------------------------------------*/ + +/** + * Counts read as themselves up to a million. "1.2K" and "1,243" are the same + * number to a reader; compaction starts where the exact digits stop being + * something anyone holds in their head. + */ +export function formatOverviewCount(value: number): string { + return Math.abs(value) >= 1_000_000 ? formatNumber(value) : Math.round(value).toLocaleString() +} + +/** A per-session ratio: one decimal while the number is small enough to have one. */ +export function formatPerSession(value: number): string { + if (!Number.isFinite(value)) return "—" + return value >= 100 ? formatOverviewCount(value) : value.toFixed(1) +} + +/** A duration a session or a call took. Zero means "nothing measured". */ +export function formatOverviewDuration(ms: number): string { + if (!Number.isFinite(ms) || ms <= 0) return "—" + return formatLatency(ms) +} + +/** `''` is a real breakdown key, shown as unattributed rather than hidden. */ +export const UNATTRIBUTED_LABEL = "Unattributed" +export const breakdownKeyLabel = (key: string): string => (key === "" ? UNATTRIBUTED_LABEL : key) + +/** + * What the current scope matched, in one line. + * + * The three populations the rest of the board divides by, so a reader can see + * at a glance whether a rate is measured over a thousand sessions or over four. + */ +export function overviewScopeSummary(current: OverviewMeasures): string { + return [ + `${formatOverviewCount(current.sessions)} sessions`, + `${formatOverviewCount(current.llmCalls)} LLM calls`, + `${formatOverviewCount(current.toolCalls)} tool calls`, + ].join(" · ") +} + +/* ------------------------------------------------------------------------------------------------- + * KPI tiles + * -----------------------------------------------------------------------------------------------*/ + +export const OVERVIEW_TILES = [ + "sessions", + "cost", + "costPerSession", + "tokens", + "errorRate", + "toolCallsPerSession", + "durationP95", +] as const +export type OverviewTileId = (typeof OVERVIEW_TILES)[number] + +export interface OverviewTile { + readonly id: OverviewTileId + /** The eyebrow, in the page's own words. */ + readonly label: string + readonly value: string + /** A short suffix beside the value, where one helps read it. */ + readonly unit?: string + /** `null` when the comparison is off, or when there is no reading to give. */ + readonly delta: OverviewDelta | null + /** The second half of the delta line: what the number is made of. */ + readonly sub: string +} + +/** + * The seven tiles, left to right. + * + * Totals are neutral — more sessions is neither good nor bad news, and a bill + * that rose because usage rose is not a regression. What is graded is the unit + * economics and the failures: cost per session, tokens per session, the error + * rate, tool calls per session and the p95. + */ +export function buildOverviewTiles( + current: OverviewMeasures, + previous: OverviewMeasures, + options: { compare: boolean; windowLabel: string }, +): ReadonlyArray { + const delta = ( + before: number, + after: number, + unit: DeltaUnit, + riseIs: DeltaTone, + ): OverviewDelta | null => + options.compare ? overviewDelta(before, after, { unit, riseIs }) : null + + return [ + { + id: "sessions", + label: "Sessions", + value: formatOverviewCount(current.sessions), + delta: delta(previous.sessions, current.sessions, "percent", "neutral"), + sub: options.compare + ? `vs ${formatOverviewCount(previous.sessions)} prev` + : `over ${options.windowLabel}`, + }, + { + id: "cost", + label: "Cost", + value: formatCost(current.cost), + delta: delta(previous.cost, current.cost, "percent", "neutral"), + sub: `priced ${formatPercent(pricedShare(current))}`, + }, + { + id: "costPerSession", + label: "Cost / session", + value: formatCost(costPerSession(current)), + delta: delta(costPerSession(previous), costPerSession(current), "percent", "bad"), + sub: options.compare + ? `vs ${formatCost(costPerSession(previous))}` + : `${formatOverviewCount(current.sessions)} sessions`, + }, + { + id: "tokens", + label: "Tokens", + value: formatNumber(current.tokens), + unit: "tok", + delta: delta(previous.tokens, current.tokens, "percent", "neutral"), + sub: `${formatNumber(tokensPerSession(current))} / session`, + }, + { + id: "errorRate", + label: "Error rate", + value: formatErrorRate(sessionErrorRate(current)), + delta: delta(sessionErrorRate(previous), sessionErrorRate(current), "points", "bad"), + sub: `${formatOverviewCount(current.erroredSessions)} errored`, + }, + { + id: "toolCallsPerSession", + label: "Tool calls / sess", + value: formatPerSession(toolCallsPerSession(current)), + delta: delta( + toolCallsPerSession(previous), + toolCallsPerSession(current), + "percent", + "bad", + ), + sub: `${formatOverviewCount(current.toolCalls)} calls`, + }, + { + id: "durationP95", + label: "Duration p95", + value: formatOverviewDuration(current.sessionDurationP95Ms), + delta: delta( + previous.sessionDurationP95Ms, + current.sessionDurationP95Ms, + "duration", + "bad", + ), + sub: `p50 ${formatOverviewDuration(current.sessionDurationP50Ms)}`, + }, + ] +} + +/* ------------------------------------------------------------------------------------------------- + * The series behind the small multiples + * -----------------------------------------------------------------------------------------------*/ + +export interface OverviewSeriesPoint { + /** Epoch milliseconds, the bucket's start. */ + readonly bucket: number + readonly sessions: number + readonly costPerSession: number + readonly tokensPerSession: number + /** Tokens per session, split by band — the stacked chart's values. */ + readonly tokenBands: Record + /** Each band's share of the bucket's tokens, 0–1. */ + readonly tokenBandShares: Record + readonly toolCallsPerSession: number + readonly sessionErrorRate: number + readonly llmErrorRate: number + readonly toolErrorRate: number + readonly sessionP50Ms: number + readonly sessionP95Ms: number + readonly llmCallsPerSession: number + readonly cacheHitRatio: number +} + +export function buildOverviewSeries( + points: ReadonlyArray, +): ReadonlyArray { + return points.map((point) => { + const bands = tokenBandValues(point) + const total = OVERVIEW_TOKEN_BAND_KEYS.reduce((sum, key) => sum + bands[key], 0) + const perSession = emptyBands() + const shares = emptyBands() + for (const key of OVERVIEW_TOKEN_BAND_KEYS) { + perSession[key] = ratio(bands[key], point.sessions) + shares[key] = ratio(bands[key], total) + } + return { + bucket: point.bucket, + sessions: point.sessions, + costPerSession: costPerSession(point), + tokensPerSession: tokensPerSession(point), + tokenBands: perSession, + tokenBandShares: shares, + toolCallsPerSession: toolCallsPerSession(point), + sessionErrorRate: sessionErrorRate(point), + llmErrorRate: llmErrorRate(point), + toolErrorRate: toolErrorRate(point), + sessionP50Ms: point.sessionDurationP50Ms, + sessionP95Ms: point.sessionDurationP95Ms, + llmCallsPerSession: llmCallsPerSession(point), + cacheHitRatio: cacheHitRatio(point), + } + }) +} + +/** + * The previous period's points moved onto the current period's x-axis. + * + * The comparison window is the equal-length one immediately before, so adding + * the window's length puts each of its buckets under the current bucket it is + * being compared with — which is what lets the ghost line share one axis. + */ +export function shiftOverviewSeries( + points: ReadonlyArray, + offsetMs: number, +): ReadonlyArray { + return points.map((point) => ({ ...point, bucket: point.bucket + offsetMs })) +} + +/* ------------------------------------------------------------------------------------------------- + * Model mix + * -----------------------------------------------------------------------------------------------*/ + +/** Models plotted as their own band; everything past this is folded. */ +export const OVERVIEW_MODEL_MIX_LIMIT = 5 +export const OVERVIEW_MODEL_MIX_OTHER = "other" + +export interface OverviewModelMixPoint { + readonly bucket: number + /** Spans per band key, over the models {@link OverviewModelMix.models} names. */ + readonly spans: Record + /** Each band's share of the bucket, 0–1 — the 100% stack. */ + readonly shares: Record +} + +export interface OverviewModelMix { + /** The plotted bands, busiest first, with `other` last when the tail exists. */ + readonly models: ReadonlyArray + readonly points: ReadonlyArray +} + +/** + * The top models by span count, with the rest folded into one grey band. + * + * A line per model is unreadable past a handful and the tail is a residue + * rather than a thing; what the chart is for is noticing that one band took + * over. + */ +export function buildModelMix(rows: ReadonlyArray): OverviewModelMix { + const totals = new Map() + for (const row of rows) totals.set(row.model, (totals.get(row.model) ?? 0) + row.llmCallSpans) + + const ranked = [...totals.entries()] + .sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0])) + .map(([model]) => model) + const top = ranked.slice(0, OVERVIEW_MODEL_MIX_LIMIT) + const kept = new Set(top) + const models = ranked.length > top.length ? [...top, OVERVIEW_MODEL_MIX_OTHER] : top + + const byBucket = new Map>() + for (const row of rows) { + const band = kept.has(row.model) ? row.model : OVERVIEW_MODEL_MIX_OTHER + let bucket = byBucket.get(row.bucket) + if (bucket === undefined) { + bucket = Object.fromEntries(models.map((model) => [model, 0])) + byBucket.set(row.bucket, bucket) + } + bucket[band] += row.llmCallSpans + } + + const points = [...byBucket.entries()] + .sort((a, b) => a[0] - b[0]) + .map(([bucket, spans]) => { + const total = models.reduce((sum, model) => sum + spans[model], 0) + const shares: Record = {} + for (const model of models) shares[model] = ratio(spans[model], total) + return { bucket, spans, shares } + }) + + return { models, points } +} + +/* ------------------------------------------------------------------------------------------------- + * Breakdown tables + * -----------------------------------------------------------------------------------------------*/ + +export interface OverviewBreakdownRow { + readonly key: string + readonly label: string + /** Share of the table's cost, 0–1 — the usage dimensions' bar. */ + readonly shareOfCost: number + /** Share of the table's tool calls, 0–1 — the tool dimension's bar. */ + readonly shareOfCalls: number + readonly sessions: number + readonly llmCalls: number + readonly tokensPerSession: number + readonly cost: number + readonly costPerSession: number + readonly toolCalls: number + readonly toolErrors: number + /** Tool calls that failed for the `tool` dimension, sessions that failed for + * every other — the failure the dimension can actually attribute. */ + readonly errorRate: number + /** The same rate's move in percentage points; `null` where the key did not + * appear in the previous window. */ + readonly errorRateDeltaPp: number | null +} + +export interface OverviewBreakdown { + readonly dimension: OverviewDimension + readonly rows: ReadonlyArray + /** Distinct keys the window had, so the table can say what it is not showing. */ + readonly totalKeys: number +} + +/** A tool row's failures are its calls'; every other dimension's are its sessions'. */ +const dimensionErrorRate = (dimension: OverviewDimension, m: OverviewMeasures): number => + dimension === "tool" ? toolErrorRate(m) : sessionErrorRate(m) + +const dimensionPopulation = (dimension: OverviewDimension, m: OverviewMeasures): number => + dimension === "tool" ? m.toolCalls : m.sessions + +/** + * One table's rows, in the order the server ranked them (busiest first). + * + * Shares are of the ROWS, not of the window: the table shows at most a dozen + * keys and a bar measured against a total it is not showing would never fill. + */ +export function buildBreakdownRows( + dimension: OverviewDimension, + entries: ReadonlyArray, +): ReadonlyArray { + const totalCost = entries.reduce((sum, entry) => sum + entry.current.cost, 0) + const totalCalls = entries.reduce((sum, entry) => sum + entry.current.toolCalls, 0) + return entries.map((entry) => { + const rate = dimensionErrorRate(dimension, entry.current) + const hadPrevious = dimensionPopulation(dimension, entry.previous) > 0 + return { + key: entry.key, + label: breakdownKeyLabel(entry.key), + shareOfCost: ratio(entry.current.cost, totalCost), + shareOfCalls: ratio(entry.current.toolCalls, totalCalls), + sessions: entry.current.sessions, + llmCalls: entry.current.llmCalls, + tokensPerSession: tokensPerSession(entry.current), + cost: entry.current.cost, + costPerSession: costPerSession(entry.current), + toolCalls: entry.current.toolCalls, + toolErrors: entry.current.erroredToolCalls, + errorRate: rate, + errorRateDeltaPp: hadPrevious + ? (rate - dimensionErrorRate(dimension, entry.previous)) * 100 + : null, + } + }) +} + +/* ------------------------------------------------------------------------------------------------- + * "What changed" — the movers rail + * -----------------------------------------------------------------------------------------------*/ + +/** A key this quiet moved by accident, not by regression. */ +export const OVERVIEW_MOVER_MIN_SESSIONS = 10 +export const OVERVIEW_MOVER_LIMIT = 6 + +export type OverviewMoverMetric = + | "sessionErrorRate" + | "llmErrorRate" + | "toolErrorRate" + | "costPerSession" + | "toolCallsPerSession" + | "tokensPerSession" + | "durationP95" + +export interface OverviewMover { + readonly dimension: OverviewDimension + readonly key: string + readonly label: string + readonly metric: OverviewMoverMetric + readonly metricLabel: string + /** Formatted in the metric's own unit. */ + readonly before: string + readonly after: string + readonly deltaText: string + readonly tone: DeltaTone + /** Percentage points for a rate, tenths of a percent for a ratio — the one + * scale the rail ranks on, and the magnitude bar's length. */ + readonly score: number +} + +interface MoverMetric { + readonly id: OverviewMoverMetric + readonly label: string + readonly unit: DeltaUnit + readonly riseIs: DeltaTone + readonly value: (m: OverviewMeasures) => number + readonly format: (value: number) => string + /** Absent means every dimension. */ + readonly only?: OverviewDimension +} + +const MOVER_METRICS: ReadonlyArray = [ + { + id: "sessionErrorRate", + label: "session error rate", + unit: "points", + riseIs: "bad", + value: sessionErrorRate, + format: formatErrorRate, + }, + { + id: "llmErrorRate", + label: "LLM error rate", + unit: "points", + riseIs: "bad", + value: llmErrorRate, + format: formatErrorRate, + }, + { + id: "toolErrorRate", + label: "tool error rate", + unit: "points", + riseIs: "bad", + value: toolErrorRate, + format: formatErrorRate, + only: "tool", + }, + { + id: "costPerSession", + label: "cost / session", + unit: "percent", + riseIs: "bad", + value: costPerSession, + format: formatCost, + }, + { + id: "toolCallsPerSession", + label: "tool calls / session", + unit: "percent", + riseIs: "bad", + value: toolCallsPerSession, + format: formatPerSession, + }, + { + id: "tokensPerSession", + label: "tokens / session", + unit: "percent", + riseIs: "bad", + value: tokensPerSession, + format: formatNumber, + }, + { + id: "durationP95", + label: "p95 duration", + unit: "percent", + riseIs: "bad", + value: (m) => m.sessionDurationP95Ms, + format: formatOverviewDuration, + }, +] + +/** Points for a rate, a tenth of a percent for a ratio — so an 8-point jump in + * failures outranks an 80% rise in tokens, which is the order a reader wants. */ +const moverScore = (delta: OverviewDelta): number => + delta.pp !== null ? Math.abs(delta.pp) : Math.abs((delta.percent ?? 0) * 100) / 10 + +/** + * The biggest movers across every dimension, worst first. + * + * One line per key, not per metric: a key whose cost and tokens both doubled + * moved once, and printing it twice would push a different regression off the + * rail. Keys too small to read are dropped in both windows — a group that ran + * three sessions last week and four this week can post any rate at all. + */ +export function buildMovers( + breakdowns: ReadonlyArray<{ + readonly dimension: OverviewDimension + readonly entries: ReadonlyArray + }>, +): ReadonlyArray { + const best: OverviewMover[] = [] + for (const breakdown of breakdowns) { + for (const entry of breakdown.entries) { + if ( + entry.current.sessions < OVERVIEW_MOVER_MIN_SESSIONS || + entry.previous.sessions < OVERVIEW_MOVER_MIN_SESSIONS + ) { + continue + } + let winner: OverviewMover | undefined + for (const metric of MOVER_METRICS) { + if (metric.only !== undefined && metric.only !== breakdown.dimension) continue + const before = metric.value(entry.previous) + const after = metric.value(entry.current) + const delta = overviewDelta(before, after, { + unit: metric.unit, + riseIs: metric.riseIs, + }) + if (delta === null || delta.direction === "flat") continue + const score = moverScore(delta) + if (winner !== undefined && score <= winner.score) continue + winner = { + dimension: breakdown.dimension, + key: entry.key, + label: breakdownKeyLabel(entry.key), + metric: metric.id, + metricLabel: metric.label, + before: metric.format(before), + after: metric.format(after), + deltaText: delta.text, + tone: delta.tone, + score, + } + } + if (winner !== undefined) best.push(winner) + } + } + return best + .sort( + (a, b) => + b.score - a.score || + OVERVIEW_DIMENSIONS.indexOf(a.dimension) - OVERVIEW_DIMENSIONS.indexOf(b.dimension) || + a.key.localeCompare(b.key), + ) + .slice(0, OVERVIEW_MOVER_LIMIT) +} + +/* ------------------------------------------------------------------------------------------------- + * Coverage + * -----------------------------------------------------------------------------------------------*/ + +/** The one coverage line the API can answer: how much of the window has a price. */ +export interface OverviewCoverage { + readonly label: string + readonly share: number +} + +export const overviewCoverage = (current: OverviewMeasures): OverviewCoverage => ({ + label: "LLM calls with a cost", + share: pricedShare(current), +}) + +/* ------------------------------------------------------------------------------------------------- + * The nine small multiples + * -----------------------------------------------------------------------------------------------*/ + +export const OVERVIEW_CHARTS = [ + "sessions", + "costPerSession", + "tokensPerSession", + "toolCallsPerSession", + "errorRate", + "sessionDuration", + "llmCallsPerSession", + "modelMix", + "cacheHitRatio", +] as const +export type OverviewChartId = (typeof OVERVIEW_CHARTS)[number] + +export interface OverviewChartSummary { + readonly id: OverviewChartId + readonly title: string + /** The unit sub-label under the title. */ + readonly unit: string + /** The window's headline, from the un-bucketed summary — never folded from + * the buckets, because quantiles do not merge. */ + readonly value: string + readonly delta: OverviewDelta | null +} + +/** The chart headlines, in the grid's reading order. */ +export function buildOverviewCharts( + current: OverviewMeasures, + previous: OverviewMeasures, + options: { compare: boolean; modelMix: OverviewModelMix }, +): ReadonlyArray { + const delta = ( + before: number, + after: number, + unit: DeltaUnit, + riseIs: DeltaTone, + ): OverviewDelta | null => + options.compare ? overviewDelta(before, after, { unit, riseIs }) : null + + const leadModel = options.modelMix.models[0] + const spansOf = (model: string) => + options.modelMix.points.reduce((sum, point) => sum + point.spans[model], 0) + const mixSpans = options.modelMix.models.reduce((sum, model) => sum + spansOf(model), 0) + const leadShare = leadModel === undefined ? 0 : ratio(spansOf(leadModel), mixSpans) + + return [ + { + id: "sessions", + title: "Sessions started", + unit: "sessions / bucket", + value: formatOverviewCount(current.sessions), + delta: delta(previous.sessions, current.sessions, "percent", "neutral"), + }, + { + id: "costPerSession", + title: "Cost per session", + unit: "USD / session", + value: formatCost(costPerSession(current)), + delta: delta(costPerSession(previous), costPerSession(current), "percent", "bad"), + }, + { + id: "tokensPerSession", + title: "Tokens per session", + unit: "tokens / session, by band", + value: formatNumber(tokensPerSession(current)), + delta: delta(tokensPerSession(previous), tokensPerSession(current), "percent", "bad"), + }, + { + id: "toolCallsPerSession", + title: "Tool calls per session", + unit: "calls / session", + value: formatPerSession(toolCallsPerSession(current)), + delta: delta( + toolCallsPerSession(previous), + toolCallsPerSession(current), + "percent", + "bad", + ), + }, + { + id: "errorRate", + title: "Error rate", + unit: "sessions · LLM calls · tool calls", + value: formatErrorRate(sessionErrorRate(current)), + delta: delta(sessionErrorRate(previous), sessionErrorRate(current), "points", "bad"), + }, + { + id: "sessionDuration", + title: "Session duration", + unit: "p50 with p50–p95 band", + value: formatOverviewDuration(current.sessionDurationP95Ms), + delta: delta( + previous.sessionDurationP95Ms, + current.sessionDurationP95Ms, + "duration", + "bad", + ), + }, + { + id: "llmCallsPerSession", + title: "LLM calls per session", + unit: "calls / session", + value: formatPerSession(llmCallsPerSession(current)), + delta: delta( + llmCallsPerSession(previous), + llmCallsPerSession(current), + "percent", + "neutral", + ), + }, + { + id: "modelMix", + title: "Model mix", + unit: "share of LLM call spans", + value: leadModel === undefined ? "—" : `${leadModel} ${formatPercent(leadShare)}`, + delta: null, + }, + { + id: "cacheHitRatio", + title: "Cache hit ratio", + unit: "cache reads / prompt tokens", + value: formatPercent(cacheHitRatio(current)), + delta: delta(cacheHitRatio(previous), cacheHitRatio(current), "points", "good"), + }, + ] +} + +/* ------------------------------------------------------------------------------------------------- + * The whole board + * -----------------------------------------------------------------------------------------------*/ + +export interface AgentOverviewData { + readonly current: OverviewMeasures + readonly previous: OverviewMeasures + readonly compare: boolean + readonly tiles: ReadonlyArray + readonly charts: ReadonlyArray + readonly series: ReadonlyArray + /** Already shifted onto the current window's axis; empty when compare is off. */ + readonly previousSeries: ReadonlyArray + readonly modelMix: OverviewModelMix + readonly movers: ReadonlyArray + readonly coverage: OverviewCoverage + /** All six, in `OVERVIEW_DIMENSIONS` order — the tabs are component state, + * and the movers rail reads every one of them anyway. */ + readonly breakdowns: ReadonlyArray + readonly bucketSeconds: number +} + +export interface AgentOverviewInput { + readonly current: OverviewMeasures + readonly previous: OverviewMeasures + readonly series: ReadonlyArray + readonly previousSeries: ReadonlyArray + readonly modelMix: ReadonlyArray + readonly breakdowns: ReadonlyArray<{ + readonly dimension: OverviewDimension + readonly entries: ReadonlyArray + readonly totalKeys: number + }> + readonly bucketSeconds: number + /** The resolved window, in epoch ms — its length is the ghost's shift. */ + readonly windowMs: { readonly startMs: number; readonly endMs: number } + /** Names the comparison in the tiles, e.g. `7d`. */ + readonly windowLabel: string + readonly compare: boolean +} + +/** Everything the view renders, from everything the reads returned. */ +export function buildAgentOverviewData(input: AgentOverviewInput): AgentOverviewData { + const modelMix = buildModelMix(input.modelMix) + const windowMs = input.windowMs.endMs - input.windowMs.startMs + return { + current: input.current, + previous: input.previous, + compare: input.compare, + tiles: buildOverviewTiles(input.current, input.previous, { + compare: input.compare, + windowLabel: input.windowLabel, + }), + charts: buildOverviewCharts(input.current, input.previous, { + compare: input.compare, + modelMix, + }), + series: buildOverviewSeries(input.series), + previousSeries: input.compare + ? shiftOverviewSeries(buildOverviewSeries(input.previousSeries), windowMs) + : [], + modelMix, + movers: buildMovers(input.breakdowns), + coverage: overviewCoverage(input.current), + breakdowns: input.breakdowns.map((breakdown) => ({ + dimension: breakdown.dimension, + rows: buildBreakdownRows(breakdown.dimension, breakdown.entries), + totalKeys: breakdown.totalKeys, + })), + bucketSeconds: input.bucketSeconds, + } +} diff --git a/apps/web/src/lib/agent-sessions/overview-search.test.ts b/apps/web/src/lib/agent-sessions/overview-search.test.ts new file mode 100644 index 000000000..0c996c273 --- /dev/null +++ b/apps/web/src/lib/agent-sessions/overview-search.test.ts @@ -0,0 +1,112 @@ +import { describe, expect, it } from "vitest" + +import { + activeOverviewFilters, + clearOverviewFilters, + compareEnabled, + failingOnly, + overviewApiDimension, + sessionsLinkSearch, + toggleOverviewFilter, + type AgentOverviewSearch, +} from "./overview-search" + +describe("overviewApiDimension", () => { + it("renames the page's framework to the warehouse's vendor and leaves the rest", () => { + expect(overviewApiDimension("framework")).toBe("vendor") + expect(overviewApiDimension("model")).toBe("model") + expect(overviewApiDimension("tool")).toBe("tool") + }) +}) + +describe("compareEnabled", () => { + it("is on when the URL says nothing, and only `false` turns it off", () => { + expect(compareEnabled({})).toBe(true) + expect(compareEnabled({ compare: true })).toBe(true) + expect(compareEnabled({ compare: false })).toBe(false) + }) +}) + +describe("failingOnly", () => { + it("reads only an explicit true", () => { + expect(failingOnly({})).toBe(false) + expect(failingOnly({ hasErrors: false })).toBe(false) + expect(failingOnly({ hasErrors: true })).toBe(true) + }) +}) + +describe("activeOverviewFilters", () => { + it("lists the set dimensions in the dimensions' own order", () => { + const search: AgentOverviewSearch = { tool: "run_tests", model: "opus", environment: "prd" } + expect(activeOverviewFilters(search)).toEqual([ + { dimension: "model", value: "opus" }, + { dimension: "environment", value: "prd" }, + { dimension: "tool", value: "run_tests" }, + ]) + }) + + it("is empty when only the toggles are set", () => { + expect(activeOverviewFilters({ hasErrors: true, compare: false })).toEqual([]) + }) +}) + +describe("clearOverviewFilters", () => { + it("clears every dimension and leaves the toggles alone", () => { + const patch = clearOverviewFilters() + expect(patch).toEqual({ + model: undefined, + agent: undefined, + service: undefined, + framework: undefined, + environment: undefined, + tool: undefined, + }) + expect("hasErrors" in patch).toBe(false) + expect("compare" in patch).toBe(false) + }) +}) + +describe("toggleOverviewFilter", () => { + it("selects a key that is not the current one", () => { + expect(toggleOverviewFilter({}, "model", "opus")).toEqual({ model: "opus" }) + }) + + it("clears the dimension when the key is already selected", () => { + expect(toggleOverviewFilter({ model: "opus" }, "model", "opus")).toEqual({ model: undefined }) + }) + + it("clears rather than selects the unattributed key, which has no spelling", () => { + expect(toggleOverviewFilter({ agent: "a" }, "agent", "")).toEqual({ agent: undefined }) + }) +}) + +describe("sessionsLinkSearch", () => { + it("widens each single value into the list's array-valued key", () => { + expect( + sessionsLinkSearch({ + framework: "eve", + model: "opus", + agent: "captain", + service: "api", + environment: "prd", + tool: "run_tests", + }), + ).toEqual({ + vendors: ["eve"], + services: ["api"], + environments: ["prd"], + models: ["opus"], + agents: ["captain"], + tools: ["run_tests"], + hasErrors: undefined, + }) + }) + + it("carries the board's failing-only toggle", () => { + expect(sessionsLinkSearch({ hasErrors: true }).hasErrors).toBe(true) + }) + + it("lets the errored tab ask for failures the board is not filtered to", () => { + expect(sessionsLinkSearch({}, { hasErrors: true }).hasErrors).toBe(true) + }) +}) diff --git a/apps/web/src/lib/agent-sessions/overview-search.ts b/apps/web/src/lib/agent-sessions/overview-search.ts new file mode 100644 index 000000000..a2cd968dd --- /dev/null +++ b/apps/web/src/lib/agent-sessions/overview-search.ts @@ -0,0 +1,198 @@ +// The URL is the whole state of `/agent-sessions/overview`. Every control — +// the six dimension selects, the two toggles, a breakdown row, a mover line — +// writes a search param and reads it back, so a link carries exactly the board +// someone was looking at and Back undoes one decision at a time. +// +// Declared here rather than in the route file because the hook, the view and +// the lab all need the decoded shape, and only the route needs the schema. + +import { Schema } from "effect" + +import type { AiOverviewDimension } from "@maple/domain/http" +import type { AgentSessionsSearchState } from "@/components/agent-sessions/agent-sessions-filter-inputs" +import { BooleanFromStringParam } from "@/lib/search-params" + +const BooleanParam = Schema.optional(Schema.Union([Schema.Boolean, BooleanFromStringParam])) + +/** + * The six dimensions the page groups and filters by, in the order the + * breakdown tabs show them. + * + * `framework` is the page's word for what the warehouse calls a vendor — the + * SDK or gateway that produced the spans. The URL key and the dimension id are + * the same string on purpose; {@link overviewApiDimension} is the one place + * the rename happens. + */ +export const OVERVIEW_DIMENSIONS = [ + "model", + "agent", + "service", + "framework", + "environment", + "tool", +] as const +export type OverviewDimension = (typeof OVERVIEW_DIMENSIONS)[number] + +/** The dimension as the breakdown endpoint spells it. */ +export function overviewApiDimension(dimension: OverviewDimension): AiOverviewDimension { + return dimension === "framework" ? "vendor" : dimension +} + +/** + * Spread into the route's own `Schema.Struct`, ahead of `TimeRangeSearchFields`. + * + * `Schema.optional` throughout (not `optionalKey`) for the reason the + * time-range fields give: TanStack Router hands back keys that are + * present-but-`undefined`, and clearing a filter writes `undefined` explicitly. + * + * One value per dimension, not an array: this page is read by narrowing to one + * thing at a time, and a row click that appended to a set would need a second + * gesture to mean "only this". + */ +export const OverviewSearchFields = { + /** The SDK or gateway, as the gateway stamps it (e.g. `eve`), not a label. */ + framework: Schema.optional(Schema.String), + model: Schema.optional(Schema.String), + agent: Schema.optional(Schema.String), + service: Schema.optional(Schema.String), + environment: Schema.optional(Schema.String), + tool: Schema.optional(Schema.String), + /** Sessions with at least one failed span. */ + hasErrors: BooleanParam, + /** The previous-period comparison. On unless the URL says `false`. */ + compare: BooleanParam, +} + +export const AgentOverviewSearch = Schema.Struct(OverviewSearchFields) +export type AgentOverviewSearch = Schema.Schema.Type + +/** Wide enough that a nightly agent shows up at all. */ +export const AGENT_OVERVIEW_DEFAULT_PRESET = "7d" + +/** The value in force for one dimension, or nothing. */ +export const selectedDimensionValue = ( + search: AgentOverviewSearch, + dimension: OverviewDimension, +): string | undefined => search[dimension] + +/** The comparison is on by default, so only `compare=false` turns it off. */ +export const compareEnabled = (search: AgentOverviewSearch): boolean => search.compare !== false + +export const failingOnly = (search: AgentOverviewSearch): boolean => search.hasErrors === true + +/** One option in a dimension select, with the sessions behind it. */ +export interface OverviewFacetOption { + readonly name: string + readonly count: number +} + +/** The window's facet values per dimension, unfiltered — picking one model must + * not erase the others from the select. */ +export type OverviewFacets = Record> + +export const EMPTY_OVERVIEW_FACETS = { + model: [], + agent: [], + service: [], + framework: [], + environment: [], + tool: [], +} satisfies OverviewFacets + +export interface OverviewFilterChip { + readonly dimension: OverviewDimension + readonly value: string +} + +/** The active dimension filters, in the dimensions' own order — the scope row. */ +export function activeOverviewFilters( + search: AgentOverviewSearch, +): ReadonlyArray { + return OVERVIEW_DIMENSIONS.flatMap((dimension) => { + const value = search[dimension] + return value === undefined ? [] : [{ dimension, value }] + }) +} + +/** The patch "Clear all" applies: every dimension filter off, the toggles kept. */ +export function clearOverviewFilters(): Partial { + return { + model: undefined, + agent: undefined, + service: undefined, + framework: undefined, + environment: undefined, + tool: undefined, + } +} + +/** + * The patch a breakdown row or a mover line applies: pick this key, or clear + * the dimension when it is already the selected one. + * + * `''` is a real breakdown key — a span that carries no value for the dimension + * — but the selection contract has no spelling for "the unnamed one", so a row + * under it clears the dimension rather than selecting nothing. + */ +export function toggleOverviewFilter( + search: AgentOverviewSearch, + dimension: OverviewDimension, + key: string, +): Partial { + return overviewFilterPatch( + dimension, + key === "" || search[dimension] === key ? undefined : key, + ) +} + +/** + * The patch that sets one dimension. + * + * Written out rather than built from a computed key: a computed key widens the + * patch to an open dictionary, and the whole point of the patch type is that + * only a real search field can reach the URL. + */ +export function overviewFilterPatch( + dimension: OverviewDimension, + value: string | undefined, +): Partial { + switch (dimension) { + case "model": + return { model: value } + case "agent": + return { agent: value } + case "service": + return { service: value } + case "framework": + return { framework: value } + case "environment": + return { environment: value } + case "tool": + return { tool: value } + } +} + +/** + * What travels from this page into the Sessions list. + * + * The list filters by the same six dimensions under array-valued keys, so a + * single value becomes a one-element array. The window does NOT travel: the + * list has no picker and reads a rolling week of its own, and a window param it + * does not validate is dropped by the router rather than honoured. + */ +export function sessionsLinkSearch( + search: AgentOverviewSearch, + options?: { hasErrors?: boolean }, +): AgentSessionsSearchState { + const one = (value: string | undefined) => (value === undefined ? undefined : [value]) + const errors = options?.hasErrors ?? search.hasErrors === true + return { + vendors: one(search.framework), + services: one(search.service), + environments: one(search.environment), + models: one(search.model), + agents: one(search.agent), + tools: one(search.tool), + hasErrors: errors ? true : undefined, + } +} diff --git a/apps/web/src/lib/agent-sessions/use-agent-overview.test.ts b/apps/web/src/lib/agent-sessions/use-agent-overview.test.ts new file mode 100644 index 000000000..989aaa88e --- /dev/null +++ b/apps/web/src/lib/agent-sessions/use-agent-overview.test.ts @@ -0,0 +1,73 @@ +import { describe, expect, it } from "vitest" + +import { overviewSelection, overviewSessionsInput } from "./use-agent-overview" + +const WINDOW = { startTime: "2026-09-04 00:00:00", endTime: "2026-09-11 00:00:00" } + +describe("overviewSelection", () => { + it("sends the window and nothing else for an untouched page", () => { + expect(overviewSelection({}, WINDOW)).toEqual({ + ...WINDOW, + framework: undefined, + model: undefined, + agent: undefined, + service: undefined, + environment: undefined, + tool: undefined, + hasErrors: undefined, + }) + }) + + it("puts every dimension filter in the selection, so each is in the cache key", () => { + const selection = overviewSelection( + { + framework: "eve", + model: "claude-opus-5", + agent: "release-captain", + service: "api", + environment: "production", + tool: "run_tests", + }, + WINDOW, + ) + expect(selection.framework).toBe("eve") + expect(selection.model).toBe("claude-opus-5") + expect(selection.agent).toBe("release-captain") + expect(selection.service).toBe("api") + expect(selection.environment).toBe("production") + expect(selection.tool).toBe("run_tests") + }) + + it("drops an explicitly false failing-only rather than sending it", () => { + expect(overviewSelection({ hasErrors: false }, WINDOW).hasErrors).toBeUndefined() + expect(overviewSelection({ hasErrors: true }, WINDOW).hasErrors).toBe(true) + }) + + it("leaves the comparison out — it is a client-side reading of one read", () => { + expect(overviewSelection({ compare: false }, WINDOW)).toEqual(overviewSelection({}, WINDOW)) + }) +}) + +describe("overviewSessionsInput", () => { + it("widens each single value into the list endpoint's array key", () => { + const input = overviewSessionsInput({ model: "claude-opus-5", framework: "eve" }, WINDOW, { + sortBy: "cost", + }) + expect(input.models).toEqual(["claude-opus-5"]) + expect(input.vendorIds).toEqual(["eve"]) + expect(input.agentNames).toBeUndefined() + }) + + it("asks for one short page, worst first", () => { + const input = overviewSessionsInput({}, WINDOW, { sortBy: "durationMs" }) + expect(input.sortBy).toBe("durationMs") + expect(input.sortDir).toBe("desc") + expect(input.limit).toBe(6) + }) + + it("lets the errored tab ask for failures the board is not filtered to", () => { + expect( + overviewSessionsInput({}, WINDOW, { sortBy: "errorSpanCount", hasErrors: true }).hasErrors, + ).toBe(true) + }) +}) diff --git a/apps/web/src/lib/agent-sessions/use-agent-overview.ts b/apps/web/src/lib/agent-sessions/use-agent-overview.ts new file mode 100644 index 000000000..7a1750f74 --- /dev/null +++ b/apps/web/src/lib/agent-sessions/use-agent-overview.ts @@ -0,0 +1,189 @@ +// Every warehouse read `/agent-sessions/overview` makes, behind one hook. +// +// The page itself never touches an atom: it takes the `Result`s this returns +// and renders them. That is what lets the lab mount the same view over +// fixtures, and it keeps the wire shape confined to the mappers in +// `api/warehouse/ai-agent-overview.ts`. +// +// Eleven reads, because the board is eleven questions: the summary, one +// breakdown per dimension (the tabs are component state and the movers rail +// reads all six anyway), the model mix, and three pages of six sessions. + +import { useMemo } from "react" + +import type { Effect } from "effect" +import type { AiSessionSortKey } from "@maple/domain/http" + +import type { + AiOverviewBreakdownData, + AiOverviewModelMixData, + AiOverviewSelection, + AiOverviewSummaryData, +} from "@/api/warehouse/ai-agent-overview" +import type { ListAiSessionsInput, listAiSessions } from "@/api/warehouse/ai-sessions" +import { chartBucketSeconds } from "@/components/infra/chart-utils" +import { useRefreshableAtomValue } from "@/hooks/use-refreshable-atom-value" +import type { Result } from "@/lib/effect-atom" +import type { QueryAtomFailure } from "@/lib/services/atoms/warehouse-query-atoms" +import { + aiOverviewBreakdownResultAtom, + aiOverviewModelMixResultAtom, + aiOverviewSummaryResultAtom, + listAiSessionsResultAtom, +} from "@/lib/services/atoms/warehouse-query-atoms" + +import { + overviewApiDimension, + type AgentOverviewSearch, + type OverviewDimension, +} from "./overview-search" + +export interface AgentOverviewWindow { + readonly startTime: string + readonly endTime: string +} + +/** Enough rows to recognise a pattern, few enough to read without scrolling. */ +export const OVERVIEW_TOP_SESSIONS_LIMIT = 6 + +/** The three readings of "show me the sessions behind this". */ +export const OVERVIEW_TOP_SESSION_TABS = ["cost", "duration", "errored"] as const +export type OverviewTopSessionTab = (typeof OVERVIEW_TOP_SESSION_TABS)[number] + +/** The list read's own page shape — the rows the Top sessions table renders. */ +export type AgentOverviewSessionsPage = Effect.Success> +type SessionsResult = Result.Result + +export interface AgentOverviewResults { + readonly summary: Result.Result + /** In `OVERVIEW_DIMENSIONS` order. */ + readonly breakdowns: ReadonlyArray<{ + readonly dimension: OverviewDimension + readonly result: Result.Result + }> + readonly modelMix: Result.Result + readonly topSessions: Record + readonly bucketSeconds: number +} + +/** + * The search params as the three overview endpoints take them. + * + * All six filters are server-side, so every read re-scopes to the toolbar and + * the tiles, the grid and the tables always describe the same sessions. That + * also puts every filter in every atom's cache key, which is what makes a + * chosen model a new read rather than a stale one. + */ +export function overviewSelection( + search: AgentOverviewSearch, + window: AgentOverviewWindow, +): AiOverviewSelection { + return { + startTime: window.startTime, + endTime: window.endTime, + framework: search.framework, + model: search.model, + agent: search.agent, + service: search.service, + environment: search.environment, + tool: search.tool, + hasErrors: search.hasErrors === true ? true : undefined, + } +} + +/** + * One page of the sessions list, under the board's own scope. + * + * The list is the concrete end of every number above it, so it must filter by + * exactly the same six dimensions — under the array-valued names the list + * endpoint uses. + */ +export function overviewSessionsInput( + search: AgentOverviewSearch, + window: AgentOverviewWindow, + options: { sortBy: AiSessionSortKey; hasErrors?: boolean }, +): ListAiSessionsInput { + const one = (value: string | undefined) => (value === undefined ? undefined : [value]) + const errors = options.hasErrors ?? search.hasErrors === true + return { + startTime: window.startTime, + endTime: window.endTime, + vendorIds: one(search.framework), + serviceNames: one(search.service), + deploymentEnvs: one(search.environment), + models: one(search.model), + agentNames: one(search.agent), + toolNames: one(search.tool), + hasErrors: errors ? true : undefined, + sortBy: options.sortBy, + sortDir: "desc", + limit: OVERVIEW_TOP_SESSIONS_LIMIT, + } +} + +export function useAgentOverview( + search: AgentOverviewSearch, + window: AgentOverviewWindow, +): AgentOverviewResults { + const selection = useMemo(() => overviewSelection(search, window), [search, window]) + const bucketSeconds = chartBucketSeconds(window.startTime, window.endTime) + const bucketed = { ...selection, bucketSeconds } + + const summary = useRefreshableAtomValue(aiOverviewSummaryResultAtom({ data: bucketed })) + const modelMix = useRefreshableAtomValue(aiOverviewModelMixResultAtom({ data: bucketed })) + + // One call per dimension, written out: a loop over the dimensions would be a + // hook in a loop. + const model = useRefreshableAtomValue( + aiOverviewBreakdownResultAtom({ data: { ...selection, dimension: "model" } }), + ) + const agent = useRefreshableAtomValue( + aiOverviewBreakdownResultAtom({ data: { ...selection, dimension: "agent" } }), + ) + const service = useRefreshableAtomValue( + aiOverviewBreakdownResultAtom({ data: { ...selection, dimension: "service" } }), + ) + const framework = useRefreshableAtomValue( + aiOverviewBreakdownResultAtom({ + data: { ...selection, dimension: overviewApiDimension("framework") }, + }), + ) + const environment = useRefreshableAtomValue( + aiOverviewBreakdownResultAtom({ data: { ...selection, dimension: "environment" } }), + ) + const tool = useRefreshableAtomValue( + aiOverviewBreakdownResultAtom({ data: { ...selection, dimension: "tool" } }), + ) + + const byCost = useRefreshableAtomValue( + listAiSessionsResultAtom({ data: overviewSessionsInput(search, window, { sortBy: "cost" }) }), + ) + const byDuration = useRefreshableAtomValue( + listAiSessionsResultAtom({ + data: overviewSessionsInput(search, window, { sortBy: "durationMs" }), + }), + ) + const errored = useRefreshableAtomValue( + listAiSessionsResultAtom({ + data: overviewSessionsInput(search, window, { + sortBy: "errorSpanCount", + hasErrors: true, + }), + }), + ) + + return { + summary, + modelMix, + breakdowns: [ + { dimension: "model", result: model }, + { dimension: "agent", result: agent }, + { dimension: "service", result: service }, + { dimension: "framework", result: framework }, + { dimension: "environment", result: environment }, + { dimension: "tool", result: tool }, + ], + topSessions: { cost: byCost, duration: byDuration, errored }, + bucketSeconds, + } +} diff --git a/apps/web/src/lib/services/atoms/warehouse-query-atoms.ts b/apps/web/src/lib/services/atoms/warehouse-query-atoms.ts index c6eb0b65c..a3c9f5abf 100644 --- a/apps/web/src/lib/services/atoms/warehouse-query-atoms.ts +++ b/apps/web/src/lib/services/atoms/warehouse-query-atoms.ts @@ -115,6 +115,11 @@ import { listReplays, } from "@/api/warehouse/replays" import { getAiSessionSpans, getAiSessionSummary, getAiSessionsFacets, listAiSessions } from "@/api/warehouse/ai-sessions" +import { + getAiOverviewBreakdown, + getAiOverviewModelMix, + getAiOverviewSummary, +} from "@/api/warehouse/ai-agent-overview" import { getWebAnalyticsBreakdowns, getWebAnalyticsEvents, @@ -350,6 +355,22 @@ export const aiSessionSummaryResultAtom = makeQueryAtomFamily(getAiSessionSummar staleTime: 60_000, }) +// The overview board's three reads. 30s like every other filtered analytics +// atom: the whole input is the cache key, so each of the six breakdown +// dimensions keys separately and a filter change is a new read rather than a +// stale one. +export const aiOverviewSummaryResultAtom = makeQueryAtomFamily(getAiOverviewSummary, { + staleTime: 30_000, +}) + +export const aiOverviewBreakdownResultAtom = makeQueryAtomFamily(getAiOverviewBreakdown, { + staleTime: 30_000, +}) + +export const aiOverviewModelMixResultAtom = makeQueryAtomFamily(getAiOverviewModelMix, { + staleTime: 30_000, +}) + export const replaysFacetsResultAtom = makeQueryAtomFamily(getReplaysFacets, { staleTime: 30_000, }) diff --git a/apps/web/src/routeTree.gen.ts b/apps/web/src/routeTree.gen.ts index f91b3da1f..b36a7f901 100644 --- a/apps/web/src/routeTree.gen.ts +++ b/apps/web/src/routeTree.gen.ts @@ -28,6 +28,7 @@ import { Route as SignInRouteImport } from './routes/sign-in' import { Route as SignUpRouteImport } from './routes/sign-up' import { Route as AgentSessionsIndexRouteImport } from './routes/agent-sessions/index' import { Route as AgentSessionsSessionIdRouteImport } from './routes/agent-sessions/$sessionId' +import { Route as AgentSessionsOverviewRouteImport } from './routes/agent-sessions/overview' import { Route as AlertsIndexRouteImport } from './routes/alerts/index' import { Route as AlertsRuleIdRouteImport } from './routes/alerts/$ruleId' import { Route as AlertsCreateRouteImport } from './routes/alerts/create' @@ -44,6 +45,7 @@ import { Route as InfraDiscoverRouteImport } from './routes/infra/discover' import { Route as InvestigationsIndexRouteImport } from './routes/investigations/index' import { Route as InvestigationsIdRouteImport } from './routes/investigations/$id' import { Route as LabIndexRouteImport } from './routes/lab/index' +import { Route as LabAgentOverviewRouteImport } from './routes/lab/agent-overview' import { Route as LabAgentSessionRouteImport } from './routes/lab/agent-session' import { Route as LabAgentSessionsRouteImport } from './routes/lab/agent-sessions' import { Route as LabChartsRouteImport } from './routes/lab/charts' @@ -194,6 +196,11 @@ const AgentSessionsSessionIdRoute = AgentSessionsSessionIdRouteImport.update({ path: '/agent-sessions/$sessionId', getParentRoute: () => rootRouteImport, } as any) +const AgentSessionsOverviewRoute = AgentSessionsOverviewRouteImport.update({ + id: '/agent-sessions/overview', + path: '/agent-sessions/overview', + getParentRoute: () => rootRouteImport, +} as any) const AlertsIndexRoute = AlertsIndexRouteImport.update({ id: '/alerts/', path: '/alerts/', @@ -274,6 +281,11 @@ const LabIndexRoute = LabIndexRouteImport.update({ path: '/', getParentRoute: () => LabRouteRoute, } as any) +const LabAgentOverviewRoute = LabAgentOverviewRouteImport.update({ + id: '/agent-overview', + path: '/agent-overview', + getParentRoute: () => LabRouteRoute, +} as any) const LabAgentSessionRoute = LabAgentSessionRouteImport.update({ id: '/agent-session', path: '/agent-session', @@ -576,6 +588,7 @@ export interface FileRoutesByFullPath { '/sign-in': typeof SignInRoute '/sign-up': typeof SignUpRoute '/agent-sessions/$sessionId': typeof AgentSessionsSessionIdRoute + '/agent-sessions/overview': typeof AgentSessionsOverviewRoute '/alerts/$ruleId': typeof AlertsRuleIdRoute '/alerts/create': typeof AlertsCreateRoute '/anomalies/$incidentId': typeof AnomaliesIncidentIdRoute @@ -584,6 +597,7 @@ export interface FileRoutesByFullPath { '/infra/$hostName': typeof InfraHostNameRoute '/infra/discover': typeof InfraDiscoverRoute '/investigations/$id': typeof InvestigationsIdRoute + '/lab/agent-overview': typeof LabAgentOverviewRoute '/lab/agent-session': typeof LabAgentSessionRoute '/lab/agent-sessions': typeof LabAgentSessionsRoute '/lab/charts': typeof LabChartsRoute @@ -666,6 +680,7 @@ export interface FileRoutesByTo { '/sign-in': typeof SignInRoute '/sign-up': typeof SignUpRoute '/agent-sessions/$sessionId': typeof AgentSessionsSessionIdRoute + '/agent-sessions/overview': typeof AgentSessionsOverviewRoute '/alerts/$ruleId': typeof AlertsRuleIdRoute '/alerts/create': typeof AlertsCreateRoute '/anomalies/$incidentId': typeof AnomaliesIncidentIdRoute @@ -674,6 +689,7 @@ export interface FileRoutesByTo { '/infra/$hostName': typeof InfraHostNameRoute '/infra/discover': typeof InfraDiscoverRoute '/investigations/$id': typeof InvestigationsIdRoute + '/lab/agent-overview': typeof LabAgentOverviewRoute '/lab/agent-session': typeof LabAgentSessionRoute '/lab/agent-sessions': typeof LabAgentSessionsRoute '/lab/charts': typeof LabChartsRoute @@ -758,6 +774,7 @@ export interface FileRoutesById { '/sign-in': typeof SignInRoute '/sign-up': typeof SignUpRoute '/agent-sessions/$sessionId': typeof AgentSessionsSessionIdRoute + '/agent-sessions/overview': typeof AgentSessionsOverviewRoute '/alerts/$ruleId': typeof AlertsRuleIdRoute '/alerts/create': typeof AlertsCreateRoute '/anomalies/$incidentId': typeof AnomaliesIncidentIdRoute @@ -766,6 +783,7 @@ export interface FileRoutesById { '/infra/$hostName': typeof InfraHostNameRoute '/infra/discover': typeof InfraDiscoverRoute '/investigations/$id': typeof InvestigationsIdRoute + '/lab/agent-overview': typeof LabAgentOverviewRoute '/lab/agent-session': typeof LabAgentSessionRoute '/lab/agent-sessions': typeof LabAgentSessionsRoute '/lab/charts': typeof LabChartsRoute @@ -851,6 +869,7 @@ export interface FileRouteTypes { | '/sign-in' | '/sign-up' | '/agent-sessions/$sessionId' + | '/agent-sessions/overview' | '/alerts/$ruleId' | '/alerts/create' | '/anomalies/$incidentId' @@ -859,6 +878,7 @@ export interface FileRouteTypes { | '/infra/$hostName' | '/infra/discover' | '/investigations/$id' + | '/lab/agent-overview' | '/lab/agent-session' | '/lab/agent-sessions' | '/lab/charts' @@ -941,6 +961,7 @@ export interface FileRouteTypes { | '/sign-in' | '/sign-up' | '/agent-sessions/$sessionId' + | '/agent-sessions/overview' | '/alerts/$ruleId' | '/alerts/create' | '/anomalies/$incidentId' @@ -949,6 +970,7 @@ export interface FileRouteTypes { | '/infra/$hostName' | '/infra/discover' | '/investigations/$id' + | '/lab/agent-overview' | '/lab/agent-session' | '/lab/agent-sessions' | '/lab/charts' @@ -1032,6 +1054,7 @@ export interface FileRouteTypes { | '/sign-in' | '/sign-up' | '/agent-sessions/$sessionId' + | '/agent-sessions/overview' | '/alerts/$ruleId' | '/alerts/create' | '/anomalies/$incidentId' @@ -1040,6 +1063,7 @@ export interface FileRouteTypes { | '/infra/$hostName' | '/infra/discover' | '/investigations/$id' + | '/lab/agent-overview' | '/lab/agent-session' | '/lab/agent-sessions' | '/lab/charts' @@ -1124,6 +1148,7 @@ export interface RootRouteChildren { SignInRoute: typeof SignInRoute SignUpRoute: typeof SignUpRoute AgentSessionsSessionIdRoute: typeof AgentSessionsSessionIdRoute + AgentSessionsOverviewRoute: typeof AgentSessionsOverviewRoute AlertsRuleIdRoute: typeof AlertsRuleIdRoute AlertsCreateRoute: typeof AlertsCreateRoute AnomaliesIncidentIdRoute: typeof AnomaliesIncidentIdRoute @@ -1310,6 +1335,13 @@ declare module '@tanstack/react-router' { preLoaderRoute: typeof AgentSessionsSessionIdRouteImport parentRoute: typeof rootRouteImport } + '/agent-sessions/overview': { + id: '/agent-sessions/overview' + path: '/agent-sessions/overview' + fullPath: '/agent-sessions/overview' + preLoaderRoute: typeof AgentSessionsOverviewRouteImport + parentRoute: typeof rootRouteImport + } '/alerts/': { id: '/alerts/' path: '/alerts' @@ -1422,6 +1454,13 @@ declare module '@tanstack/react-router' { preLoaderRoute: typeof LabIndexRouteImport parentRoute: typeof LabRouteRoute } + '/lab/agent-overview': { + id: '/lab/agent-overview' + path: '/agent-overview' + fullPath: '/lab/agent-overview' + preLoaderRoute: typeof LabAgentOverviewRouteImport + parentRoute: typeof LabRouteRoute + } '/lab/agent-session': { id: '/lab/agent-session' path: '/agent-session' @@ -1804,6 +1843,7 @@ declare module '@tanstack/react-router' { } interface LabRouteRouteChildren { + LabAgentOverviewRoute: typeof LabAgentOverviewRoute LabAgentSessionRoute: typeof LabAgentSessionRoute LabAgentSessionsRoute: typeof LabAgentSessionsRoute LabChartsRoute: typeof LabChartsRoute @@ -1829,6 +1869,7 @@ interface LabRouteRouteChildren { } const LabRouteRouteChildren: LabRouteRouteChildren = { + LabAgentOverviewRoute: LabAgentOverviewRoute, LabAgentSessionRoute: LabAgentSessionRoute, LabAgentSessionsRoute: LabAgentSessionsRoute, LabChartsRoute: LabChartsRoute, @@ -1876,6 +1917,7 @@ const rootRouteChildren: RootRouteChildren = { SignInRoute: SignInRoute, SignUpRoute: SignUpRoute, AgentSessionsSessionIdRoute: AgentSessionsSessionIdRoute, + AgentSessionsOverviewRoute: AgentSessionsOverviewRoute, AlertsRuleIdRoute: AlertsRuleIdRoute, AlertsCreateRoute: AlertsCreateRoute, AnomaliesIncidentIdRoute: AnomaliesIncidentIdRoute, diff --git a/apps/web/src/routes/agent-sessions/index.tsx b/apps/web/src/routes/agent-sessions/index.tsx index deefb93f1..f3c379b2f 100644 --- a/apps/web/src/routes/agent-sessions/index.tsx +++ b/apps/web/src/routes/agent-sessions/index.tsx @@ -7,6 +7,7 @@ import { DashboardLayout } from "@/components/layout/dashboard-layout" import { AgentSessionsList } from "@/components/agent-sessions/agent-sessions-list" import { AgentSessionsFilterSidebar } from "@/components/agent-sessions/agent-sessions-filter-sidebar" import { AgentSessionsToolbar } from "@/components/agent-sessions/agent-sessions-toolbar" +import { AgentSessionsTabs } from "@/components/agent-sessions/tools/agent-sessions-tabs" import { agentSessionsFilterInputs, sortOptionFor, @@ -181,7 +182,14 @@ function AgentSessionsBody() { - {toolbar} + + {/* The Overview tab reads the whole population of these spans; this + page reads one session at a time. Two routes, one strip. */} +
+ + {toolbar} +
+ {Result.builder(firstPageResult) .onInitial(() => ( diff --git a/apps/web/src/routes/agent-sessions/overview.tsx b/apps/web/src/routes/agent-sessions/overview.tsx new file mode 100644 index 000000000..540a6b11b --- /dev/null +++ b/apps/web/src/routes/agent-sessions/overview.tsx @@ -0,0 +1,222 @@ +import { useMemo, type ReactNode } from "react" +import { createFileRoute, useNavigate } from "@tanstack/react-router" +import { Schema } from "effect" + +import { Skeleton } from "@maple/ui/components/ui/skeleton" +import { toEpochMs } from "@maple/ui/lib/time-format" + +import { AgentOverviewView } from "@/components/agent-sessions/overview/agent-overview-view" +import { OverviewMetricStripLoading } from "@/components/agent-sessions/overview/overview-metric-strip" +import { QueryErrorState } from "@/components/common/query-error-state" +import { DashboardLayout } from "@/components/layout/dashboard-layout" +import { NotFoundError } from "@/components/route-error" +import { PageRefreshProvider } from "@/components/time-range-picker/page-refresh-context" +import { TimeRangeSearchFields, applyTimeRangeSearch } from "@/components/time-range-picker/search" +import { sessionTimeRangeSearchMiddleware } from "@/components/time-range-picker/session-time-range" +import { TimeRangeHeaderControls } from "@/components/time-range-picker/time-range-header-controls" +import { useEffectiveTimeRange } from "@/hooks/use-effective-time-range" +import { useOrganizationFeatureFlags } from "@/hooks/use-organization-feature-flags" +import { Result, useAtomValue } from "@/lib/effect-atom" +import { buildAgentOverviewData } from "@/lib/agent-sessions/overview-analytics" +import { + AGENT_OVERVIEW_DEFAULT_PRESET, + EMPTY_OVERVIEW_FACETS, + OverviewSearchFields, + compareEnabled, + type AgentOverviewSearch, + type OverviewFacets, +} from "@/lib/agent-sessions/overview-search" +import { useAgentOverview } from "@/lib/agent-sessions/use-agent-overview" +import { aiSessionsFacetsResultAtom } from "@/lib/services/atoms/warehouse-query-atoms" + +const overviewSearchSchema = Schema.Struct({ + ...OverviewSearchFields, + ...TimeRangeSearchFields, +}) + +export const Route = createFileRoute("/agent-sessions/overview")({ + component: AgentOverviewPage, + validateSearch: Schema.toStandardSchemaV1(overviewSearchSchema), + search: { middlewares: [sessionTimeRangeSearchMiddleware()] }, +}) + +/** + * Behind the `agent_tracing` org rollout flag, gated exactly as the list and + * detail pages are: in the component rather than `beforeLoad` (router context + * carries no flags), `isLoaded` first so an entitled org gets no not-found + * flash, and no route `loader` — a loader would fire eleven warehouse reads for + * orgs that are not entitled to the page at all. + */ +function AgentOverviewPage() { + const { flags, isLoaded } = useOrganizationFeatureFlags() + if (!isLoaded) return null + if (!flags.agentTracing) return + return +} + +function AgentOverviewPageContent() { + const search = Route.useSearch() + const navigate = useNavigate({ from: Route.fullPath }) + const preset = search.timePreset ?? AGENT_OVERVIEW_DEFAULT_PRESET + const { startTime, endTime } = useEffectiveTimeRange(search.startTime, search.endTime, preset) + // One object for the whole render tree below: it is a dependency of every + // selection memo down there, and a fresh literal defeats all of them. + const window = useMemo(() => ({ startTime, endTime }), [startTime, endTime]) + + const onSearchChange = (patch: Partial) => { + navigate({ search: (prev) => ({ ...prev, ...patch }) }) + } + + const handleTimeChange = ( + range: { startTime?: string; endTime?: string; presetValue?: string }, + options?: { replace?: boolean }, + ) => { + navigate({ + replace: options?.replace, + search: (prev) => ({ ...applyTimeRangeSearch(prev, range) }), + }) + } + + return ( + + + + + + + + } + /> + + + + + + ) +} + +/** + * The eleven reads, resolved. + * + * The **summary** is the one the page waits on: it is what the tiles, the chart + * headlines and the empty state are made of. Everything else degrades to empty + * rather than to a skeleton, so a slow breakdown leaves an empty table under a + * strip that is already drawn, not a page of grey boxes. + */ +function AgentOverviewBody({ + search, + window, + preset, + onSearchChange, + headerControls, +}: { + search: AgentOverviewSearch + window: { startTime: string; endTime: string } + preset: string + onSearchChange: (patch: Partial) => void + headerControls: ReactNode +}) { + const results = useAgentOverview(search, window) + const windowMs = useMemo( + () => ({ startMs: toEpochMs(window.startTime), endMs: toEpochMs(window.endTime) }), + [window.startTime, window.endTime], + ) + const timeRange = useMemo( + () => ({ + startTime: search.startTime, + endTime: search.endTime, + timePreset: search.timePreset, + }), + [search.startTime, search.endTime, search.timePreset], + ) + + // The selects' options come from the sessions facets — the same counted + // lists the list page's sidebar uses, unfiltered so picking one model does + // not erase the others. Plain `useAtomValue` keeps them off the Reload + // subscription, so a manual refresh cannot rebuild a select under a click. + const facetsResult = useAtomValue(aiSessionsFacetsResultAtom({ data: window })) + const facets: OverviewFacets = Result.builder(facetsResult) + .onSuccess((value) => ({ + model: value.models, + agent: value.agents, + service: value.services, + framework: value.vendors, + environment: value.environments, + tool: value.tools, + })) + .orElse(() => EMPTY_OVERVIEW_FACETS) + + const breakdowns = results.breakdowns.map((breakdown) => ({ + dimension: breakdown.dimension, + ...Result.builder(breakdown.result) + .onSuccess((value) => ({ entries: value.entries, totalKeys: value.totalKeys })) + .orElse(() => ({ entries: [], totalKeys: 0 })), + })) + const modelMix = Result.builder(results.modelMix) + .onSuccess((value) => value.rows) + .orElse(() => []) + const sessionsOf = (result: (typeof results.topSessions)["cost"]) => + Result.builder(result) + .onSuccess((value) => value.data) + .orElse(() => []) + + return Result.builder(results.summary) + .onInitial(() => ( +
+ + + + +
+ )) + .onError((error) => ( + + )) + .onSuccess((summary, result) => ( + + )) + .render() +} diff --git a/apps/web/src/routes/lab/agent-overview.tsx b/apps/web/src/routes/lab/agent-overview.tsx new file mode 100644 index 000000000..894cabaec --- /dev/null +++ b/apps/web/src/routes/lab/agent-overview.tsx @@ -0,0 +1,5 @@ +import { createFileRoute } from "@tanstack/react-router" + +import { AgentOverviewLab } from "@/lab/agent-overview-lab" + +export const Route = createFileRoute("/lab/agent-overview")({ component: AgentOverviewLab }) From c42d8884ed057de0c4a4581b13b94ec20193ae1e Mon Sep 17 00:00:00 2001 From: JeremyFunk Date: Fri, 11 Sep 2026 07:15:15 +0200 Subject: [PATCH 06/16] feat(agent-sessions): overview page visuals --- .../overview/agent-overview-view.test.tsx | 40 +- .../overview/agent-overview-view.tsx | 18 +- .../overview/overview-breakdowns.tsx | 346 +++++++++++++----- .../overview/overview-delta.tsx | 31 ++ .../overview/overview-filter-toolbar.tsx | 107 +++--- .../overview/overview-metric-strip.tsx | 55 +-- .../overview/overview-movers-rail.tsx | 86 +++-- .../overview/overview-scope-row.tsx | 65 ++-- .../overview/overview-small-multiple.tsx | 183 +++++++++ .../overview/overview-top-sessions.tsx | 252 ++++++++----- .../overview/overview-trends.tsx | 260 ++++++++++--- .../overview-chart-specs.test.ts | 108 ++++++ .../agent-sessions/overview-chart-specs.ts | 343 +++++++++++++++++ .../src/routes/agent-sessions/overview.tsx | 99 ++--- 14 files changed, 1533 insertions(+), 460 deletions(-) create mode 100644 apps/web/src/components/agent-sessions/overview/overview-delta.tsx create mode 100644 apps/web/src/components/agent-sessions/overview/overview-small-multiple.tsx create mode 100644 apps/web/src/lib/agent-sessions/overview-chart-specs.test.ts create mode 100644 apps/web/src/lib/agent-sessions/overview-chart-specs.ts diff --git a/apps/web/src/components/agent-sessions/overview/agent-overview-view.test.tsx b/apps/web/src/components/agent-sessions/overview/agent-overview-view.test.tsx index d218ce5d5..4e0c25682 100644 --- a/apps/web/src/components/agent-sessions/overview/agent-overview-view.test.tsx +++ b/apps/web/src/components/agent-sessions/overview/agent-overview-view.test.tsx @@ -9,21 +9,20 @@ import { afterEach, describe, expect, it, vi } from "vitest" import { buildOverviewFixture } from "@/lab/agent-overview-fixture" import { buildAgentOverviewData } from "@/lib/agent-sessions/overview-analytics" -import { - compareEnabled, - type AgentOverviewSearch, -} from "@/lib/agent-sessions/overview-search" +import { compareEnabled, type AgentOverviewSearch } from "@/lib/agent-sessions/overview-search" import { AgentOverviewView } from "./agent-overview-view" +// TEST-SEAM: the plots themselves are a canvas/ResizeObserver story jsdom cannot +// tell. What the cell puts around them — title, headline, unit, delta, legend — +// is real, and the legend is where the previous-period ghost shows up. +vi.mock("./overview-small-multiple", () => ({ + OVERVIEW_PLOT_HEIGHT: 104, + OverviewSmallMultiple: ({ chartId }: { chartId: string }) =>
, +})) + vi.mock("@tanstack/react-router", () => ({ - Link: ({ - children, - to, - params, - search, - ...props - }: React.PropsWithChildren>) => ( + Link: ({ children, to, params, search, ...props }: React.PropsWithChildren>) => ( ({ const NOW = Date.UTC(2026, 8, 10, 12, 0, 0) -function renderView(search: AgentOverviewSearch = {}, scenario: "healthy7d" | "regression24h" = "regression24h") { +function renderView( + search: AgentOverviewSearch = {}, + scenario: "healthy7d" | "regression24h" = "regression24h", +) { const fixture = buildOverviewFixture(scenario, NOW) const data = buildAgentOverviewData({ ...fixture.input, compare: compareEnabled(search) }) const onSearchChange = vi.fn() @@ -55,8 +57,7 @@ function renderView(search: AgentOverviewSearch = {}, scenario: "healthy7d" | "r } /** The section a heading owns — the page repeats labels across sections. */ -const sectionOf = (heading: string) => - screen.getByRole("heading", { name: heading }).closest("section")! +const sectionOf = (heading: string) => screen.getByRole("heading", { name: heading }).closest("section")! afterEach(cleanup) @@ -67,14 +68,15 @@ describe("AgentOverviewView", () => { expect(screen.getAllByText(tile.label).length).toBeGreaterThan(0) } expect(document.querySelectorAll("[data-chart]")).toHaveLength(9) + expect(document.querySelectorAll("[data-plot]")).toHaveLength(9) }) it("draws the previous-period ghost only while the comparison is on", () => { renderView({}) - expect(screen.getAllByText(/ghost/).length).toBeGreaterThan(0) + expect(screen.getAllByText("prev").length).toBeGreaterThan(0) cleanup() renderView({ compare: false }) - expect(screen.queryByText(/ghost/)).toBeNull() + expect(screen.queryByText("prev")).toBeNull() }) it("turns the comparison off through the URL rather than through local state", () => { @@ -119,9 +121,7 @@ describe("AgentOverviewView", () => { it("switches the breakdown table without touching the URL", () => { const { onSearchChange } = renderView({}) - fireEvent.click( - within(sectionOf("Breakdowns")).getByRole("button", { name: /^tool/ }), - ) + fireEvent.click(within(sectionOf("Breakdowns")).getByRole("button", { name: /^tool/ })) expect(screen.getByText("Share of calls")).toBeTruthy() expect(onSearchChange).not.toHaveBeenCalled() }) @@ -141,7 +141,7 @@ describe("AgentOverviewView", () => { const link = screen.getByText(first.sessionId).closest("a") expect(link?.getAttribute("data-to")).toBe("/agent-sessions/$sessionId") expect(link?.getAttribute("data-params")).toBe(JSON.stringify({ sessionId: first.sessionId })) - expect(link?.getAttribute("data-search")).toContain("\"t\"") + expect(link?.getAttribute("data-search")).toContain('"t"') }) it("carries the board's filters into the Sessions list", () => { diff --git a/apps/web/src/components/agent-sessions/overview/agent-overview-view.tsx b/apps/web/src/components/agent-sessions/overview/agent-overview-view.tsx index a0615c01a..577cfb62f 100644 --- a/apps/web/src/components/agent-sessions/overview/agent-overview-view.tsx +++ b/apps/web/src/components/agent-sessions/overview/agent-overview-view.tsx @@ -1,12 +1,6 @@ import { useMemo, type ReactNode } from "react" -import { - Empty, - EmptyDescription, - EmptyHeader, - EmptyMedia, - EmptyTitle, -} from "@maple/ui/components/ui/empty" +import { Empty, EmptyDescription, EmptyHeader, EmptyMedia, EmptyTitle } from "@maple/ui/components/ui/empty" import { SquareSparkleIcon } from "@/components/icons" import type { AgentSessionRow } from "@/components/agent-sessions/agent-sessions-list" @@ -100,7 +94,8 @@ export function AgentOverviewView({ Overview

- What your agents cost, how much they ran, and how often they failed. + Volume, cost, tokens, reliability and latency for every agent session — all on one + clock.

{headerControls ? ( @@ -108,11 +103,7 @@ export function AgentOverviewView({ ) : null} - + + /** The plotted bands, so a model's chip here is its band in the mix chart. */ + modelMix: OverviewModelMix search: AgentOverviewSearch /** Toggles that dimension's filter for the whole page. */ onSelectRow: (dimension: OverviewDimension, key: string) => void waiting?: boolean } +/** A lane: its header label, its width, and the width it sheds at. */ +interface Column { + readonly label: string + readonly className: string +} + /** The columns a dimension can actually attribute — see `AiOverviewBreakdownRow`. */ -const USAGE_COLUMNS = [ - "Share of cost", - "Sessions", - "LLM calls", - "Tok / sess", - "Cost", - "$ / sess", - "Error rate", - "Δ prev", -] as const - -const TOOL_COLUMNS = [ - "Share of calls", - "Sessions", - "Calls", - "Errors", - "Error rate", - "Δ prev", -] as const +const USAGE_COLUMNS: ReadonlyArray = [ + { label: "Share of cost", className: "w-[170px] @max-[900px]/table:hidden" }, + { label: "Sessions", className: "w-[84px] text-right" }, + { label: "LLM calls", className: "w-[92px] text-right @max-[700px]/table:hidden" }, + { label: "Tok / sess", className: "w-[92px] text-right @max-[820px]/table:hidden" }, + { label: "Cost", className: "w-[92px] text-right" }, + { label: "$ / sess", className: "w-[88px] text-right @max-[640px]/table:hidden" }, + { label: "Error rate", className: "w-[116px] text-right" }, + { label: "Δ prev", className: "w-[86px] text-right @max-[560px]/table:hidden" }, +] + +const TOOL_COLUMNS: ReadonlyArray = [ + { label: "Share of calls", className: "w-[170px] @max-[900px]/table:hidden" }, + { label: "Sessions", className: "w-[84px] text-right" }, + { label: "Calls", className: "w-[92px] text-right" }, + { label: "Errors", className: "w-[92px] text-right @max-[700px]/table:hidden" }, + { label: "Error rate", className: "w-[116px] text-right" }, + { label: "Δ prev", className: "w-[86px] text-right @max-[560px]/table:hidden" }, +] /** * The same window, grouped six ways. @@ -55,6 +67,7 @@ const TOOL_COLUMNS = [ */ export function OverviewBreakdowns({ breakdowns, + modelMix, search, onSelectRow, waiting = false, @@ -64,19 +77,23 @@ export function OverviewBreakdowns({ const isTool = active === "tool" const columns = isTool ? TOOL_COLUMNS : USAGE_COLUMNS const selected = selectedDimensionValue(search, active) + const rows = breakdown?.rows ?? [] + const worstRate = rows.reduce((max, row) => Math.max(max, row.errorRate), 0) return ( -
-
-

- Breakdowns -

- - click a row to filter the whole page - +
+
+
+

+ Breakdowns +

+ + click a row to filter the whole page + +
-
+
{breakdowns.map((item) => ( ))}
- {breakdown === undefined || breakdown.rows.length === 0 ? ( -

+ {rows.length === 0 ? ( +

No {active} activity in this range.

) : ( - <> - - - - - {columns.map((column) => ( - - ))} - - - - {breakdown.rows.map((row) => ( - onSelectRow(active, row.key)} - data-selected={row.key === selected ? "" : undefined} - className={cn( - "cursor-pointer border-b border-border/60 transition-colors hover:bg-accent/40", - row.key === selected && "bg-primary/10", - )} - > - - {isTool ? : } - - ))} - -
- {active} - - {column} -
- {row.label} -
-

- {breakdown.totalKeys} keys · session counts overlap where a session used more than - one {active} - {breakdown.totalKeys > breakdown.rows.length - ? ` · + ${breakdown.totalKeys - breakdown.rows.length} more` - : ""} +

+
+ {active} + {columns.map((column) => ( + + {column.label} + + ))} + +
+ + {rows.map((row) => ( + + ))} + +

+ {footerTotals(active, rows)} + + · + + + session counts overlap where a session used more than one {active} + {breakdown !== undefined && breakdown.totalKeys > rows.length + ? ` · +${breakdown.totalKeys - rows.length} more` + : ""} +

- +
)}
) } -const Cell = ({ children }: { children: React.ReactNode }) => ( - {children} -) +const HEAD = "font-mono text-[10.5px] leading-[14px] tracking-[0.06em] text-muted-foreground/60 uppercase" +const NUM = "shrink-0 font-mono text-[12px] leading-4 tabular-nums" + +/** + * A model wears the colour of its band in the mix chart; a framework wears its + * vendor's mark. Nothing else gets a glyph — a lane of generic marks would + * indent every name without naming anything. + */ +function Glyph({ + dimension, + row, + modelMix, +}: { + dimension: OverviewDimension + row: OverviewBreakdownRow + modelMix: OverviewModelMix +}) { + if (dimension === "framework") { + const Icon = vendorIcon(row.key) + return + } + if (dimension !== "model") return null + const band = modelMix.models.indexOf(row.key) + return ( + + ) +} -function UsageCells({ row }: { row: OverviewBreakdownRow }) { +function UsageCells({ row, worstRate }: { row: OverviewBreakdownRow; worstRate: number }) { return ( <> - {formatPercent(row.shareOfCost)} - {formatOverviewCount(row.sessions)} - {formatOverviewCount(row.llmCalls)} - {formatNumber(row.tokensPerSession)} - {formatCost(row.cost)} - {formatCost(row.costPerSession)} - {formatErrorRate(row.errorRate)} - {formatDeltaPp(row.errorRateDeltaPp)} + + + {formatOverviewCount(row.sessions)} + + + {formatOverviewCount(row.llmCalls)} + + + {formatNumber(row.tokensPerSession)} + + {formatCost(row.cost)} + + {formatCost(row.costPerSession)} + + + ) } -function ToolCells({ row }: { row: OverviewBreakdownRow }) { +function ToolCells({ row, worstRate }: { row: OverviewBreakdownRow; worstRate: number }) { return ( <> - {formatPercent(row.shareOfCalls)} - {formatOverviewCount(row.sessions)} - {formatOverviewCount(row.toolCalls)} - {formatOverviewCount(row.toolErrors)} - {formatErrorRate(row.errorRate)} - {formatDeltaPp(row.errorRateDeltaPp)} + + + {formatOverviewCount(row.sessions)} + + + {formatOverviewCount(row.toolCalls)} + + + {formatOverviewCount(row.toolErrors)} + + + ) } +/** The row's share of what the table lists, drawn against the full width. */ +function ShareCell({ share, className }: { share: number; className: string }) { + return ( + + + 0 ? 2 : 0)}%` }} + /> + + + {formatPercent(share)} + + + ) +} + +/** + * The error rate, drawn against the WORST row rather than against 100%: a table + * where every model fails under 3% would otherwise be a column of invisible + * slivers, and ranking these rows against each other is the column's whole job. + * The tone is absolute, so the colour still says how bad 3% is. + */ +function RateCell({ rate, worst }: { rate: number; worst: number }) { + const tone = rate >= 0.1 ? "--severity-error" : rate >= 0.01 ? "--severity-warn" : "--severity-info" + return ( + + + 0 ? 4 : 0)}%`, + backgroundColor: `var(${tone})`, + }} + /> + + {formatErrorRate(rate)} + + ) +} + /** A key with no previous window has no move to show, which is not a zero. */ -const formatDeltaPp = (pp: number | null): string => - pp === null ? "—" : `${pp < 0 ? "-" : "+"}${Math.abs(pp).toFixed(1)}pp` +function DeltaCell({ pp }: { pp: number | null }) { + if (pp === null) { + return ( + + — + + ) + } + const flat = Math.abs(pp) < 0.05 + return ( + 0 + ? "text-[var(--severity-error)]" + : "text-[var(--severity-info)]", + )} + > + {flat ? "0.0pp" : `${pp < 0 ? "-" : "+"}${Math.abs(pp).toFixed(1)}pp`} + + ) +} + +/** What the listed rows add up to, as the closing line states it. */ +function footerTotals(dimension: OverviewDimension, rows: ReadonlyArray): string { + const plural = rows.length === 1 ? dimension : `${dimension}s` + if (dimension === "tool") { + const calls = rows.reduce((sum, row) => sum + row.toolCalls, 0) + return `${rows.length} ${plural} · ${formatOverviewCount(calls)} tool calls` + } + const calls = rows.reduce((sum, row) => sum + row.llmCalls, 0) + const cost = rows.reduce((sum, row) => sum + row.cost, 0) + return `${rows.length} ${plural} · ${formatOverviewCount(calls)} LLM calls · ${formatCost(cost)}` +} diff --git a/apps/web/src/components/agent-sessions/overview/overview-delta.tsx b/apps/web/src/components/agent-sessions/overview/overview-delta.tsx new file mode 100644 index 000000000..d11693c42 --- /dev/null +++ b/apps/web/src/components/agent-sessions/overview/overview-delta.tsx @@ -0,0 +1,31 @@ +import { cn } from "@maple/ui/lib/utils" + +import { deltaToneClass, type OverviewDelta } from "@/lib/agent-sessions/overview-analytics" + +const ARROW = { up: "↑", down: "↓", flat: "→" } as const + +/** + * How a reading moved: an arrow for the direction, a colour for whether that + * was good news. + * + * Two signals, never one — colour alone is unreadable to a reader who cannot + * separate the greens from the reds, and an arrow alone cannot say that a + * falling cache-hit ratio is the bad kind of falling. The sign is dropped from + * the number because the arrow already carries it, and `↓ -4.9%` says it twice. + */ +export function DeltaReading({ delta, className }: { delta: OverviewDelta | null; className?: string }) { + if (delta === null) return null + return ( + + {ARROW[delta.direction]} + {delta.text.replace(/^[+-]/, "")} + + ) +} diff --git a/apps/web/src/components/agent-sessions/overview/overview-filter-toolbar.tsx b/apps/web/src/components/agent-sessions/overview/overview-filter-toolbar.tsx index a2f0c1c6e..093ade1b8 100644 --- a/apps/web/src/components/agent-sessions/overview/overview-filter-toolbar.tsx +++ b/apps/web/src/components/agent-sessions/overview/overview-filter-toolbar.tsx @@ -1,12 +1,7 @@ -import { - Select, - SelectContent, - SelectItem, - SelectTrigger, - SelectValue, -} from "@maple/ui/components/ui/select" +import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@maple/ui/components/ui/select" import { cn } from "@maple/ui/lib/utils" +import { CheckIcon } from "@/components/icons" import { OVERVIEW_DIMENSIONS, compareEnabled, @@ -22,6 +17,11 @@ import { formatOverviewCount } from "@/lib/agent-sessions/overview-analytics" /** Base UI selects need a real value for "no filter"; this never reaches the URL. */ const ALL = "__all__" +/** Every control on the row is the same pill, so a lit one reads as a narrowing. */ +const PILL = "inline-flex h-[30px] items-center rounded-md border px-2.5 font-mono text-xs" +const PILL_SET = "border-primary/40 bg-primary/10 text-primary" +const PILL_IDLE = "border-border bg-card text-muted-foreground hover:text-foreground" + export interface OverviewFilterToolbarProps { search: AgentOverviewSearch facets: OverviewFacets @@ -36,7 +36,9 @@ export interface OverviewFilterToolbarProps { * * Six dimensions, one value each: this page is read by narrowing to one thing * at a time, and every control is the same 30px pill so a control drawn in the - * primary tint reads as "this is narrowing the page" at a glance. + * primary tint reads as "this is narrowing the page" at a glance. The two + * switches sit apart from the dimensions because they are not dimensions — one + * is a predicate on sessions, the other changes what the page compares against. */ export function OverviewFilterToolbar({ search, @@ -50,55 +52,56 @@ export function OverviewFilterToolbar({ return (
- {OVERVIEW_DIMENSIONS.map((dimension) => ( - onSearchChange(overviewFilterPatch(dimension, value))} - /> - ))} +
+ {OVERVIEW_DIMENSIONS.map((dimension) => ( + onSearchChange(overviewFilterPatch(dimension, value))} + /> + ))} +
- + > + + Failing only + - + +
) } @@ -128,9 +131,7 @@ function FacetSelect({ aria-label={label} className={cn( "h-[30px] gap-2 rounded-md px-2.5 font-mono text-xs", - set - ? "border-primary/40 bg-primary/10 text-primary [&_svg]:text-primary" - : "bg-card text-foreground", + set ? cn(PILL_SET, "[&_svg]:text-primary") : "bg-card text-foreground", )} > diff --git a/apps/web/src/components/agent-sessions/overview/overview-metric-strip.tsx b/apps/web/src/components/agent-sessions/overview/overview-metric-strip.tsx index d0d0492c1..6708fbb6a 100644 --- a/apps/web/src/components/agent-sessions/overview/overview-metric-strip.tsx +++ b/apps/web/src/components/agent-sessions/overview/overview-metric-strip.tsx @@ -1,13 +1,30 @@ import { Skeleton } from "@maple/ui/components/ui/skeleton" import { cn } from "@maple/ui/lib/utils" -import { deltaToneClass, type OverviewTile } from "@/lib/agent-sessions/overview-analytics" +import type { OverviewTile } from "@/lib/agent-sessions/overview-analytics" + +import { DeltaReading } from "./overview-delta" export interface OverviewMetricStripProps { tiles: ReadonlyArray waiting?: boolean } +/** + * Seven tiles, hairline-divided, in one band. + * + * `gap-px` over a border-coloured ground rather than `divide-x`: the strip + * folds to four columns and then to two, and a divide rule would leave a stray + * hairline at the page's own edge on every wrapped row. The last tile spans two + * columns where seven does not divide, so the ground never shows through as a + * missing tile. + */ +const STRIP = cn( + "grid grid-cols-2 gap-px border-b border-border bg-border", + "@min-[900px]/page:grid-cols-4 @min-[1200px]/page:grid-cols-7", + "[&>*:last-child]:col-span-2 @min-[1200px]/page:[&>*:last-child]:col-span-1", +) + /** * Seven readings of the window, left to right, and not a selector: nothing * below the strip changes when one is read. The grid underneath already draws @@ -16,32 +33,28 @@ export interface OverviewMetricStripProps { */ export function OverviewMetricStrip({ tiles, waiting = false }: OverviewMetricStripProps) { return ( -
+
{tiles.map((tile) => ( -
- +
+ {tile.label} - - + + {tile.value} {tile.unit === undefined ? null : ( - {tile.unit} - )} - - - {tile.delta === null ? null : ( - - {tile.delta.text} + + {tile.unit} )} - {tile.sub} + + + + {tile.sub}
))} @@ -52,9 +65,9 @@ export function OverviewMetricStrip({ tiles, waiting = false }: OverviewMetricSt /** The strip's shape while the summary read is in flight. */ export function OverviewMetricStripLoading() { return ( -
+
{Array.from({ length: 7 }).map((_, index) => ( -
+
diff --git a/apps/web/src/components/agent-sessions/overview/overview-movers-rail.tsx b/apps/web/src/components/agent-sessions/overview/overview-movers-rail.tsx index d3f5c5232..9104e0647 100644 --- a/apps/web/src/components/agent-sessions/overview/overview-movers-rail.tsx +++ b/apps/web/src/components/agent-sessions/overview/overview-movers-rail.tsx @@ -22,82 +22,96 @@ export interface OverviewMoversRailProps { * * One line per key rather than per metric, and a magnitude bar measured against * the worst move on the rail — the rail answers "where do I look first", which - * is an ordering question and not a measurement. + * is an ordering question and not a measurement. The bar stays grey for the + * same reason: the ranking is the bar's whole message, and the delta beside it + * already says whether the move was bad news. */ -export function OverviewMoversRail({ - movers, - coverage, - windowLabel, - onSelect, -}: OverviewMoversRailProps) { +export function OverviewMoversRail({ movers, coverage, windowLabel, onSelect }: OverviewMoversRailProps) { const worst = movers.reduce((max, mover) => Math.max(max, mover.score), 0) return ( -
)}
) } -const Cell = ({ children }: { children: React.ReactNode }) => ( - - {children} - -) +const HEAD = "font-mono text-[10.5px] leading-[14px] tracking-[0.06em] text-muted-foreground/60 uppercase" + +function Cell({ + className, + tone = "text-muted-foreground", + children, +}: { + className: string + tone?: string + children: React.ReactNode +}) { + return ( + + {children} + + ) +} diff --git a/apps/web/src/components/agent-sessions/overview/overview-trends.tsx b/apps/web/src/components/agent-sessions/overview/overview-trends.tsx index 7e3c7af18..dcc3703df 100644 --- a/apps/web/src/components/agent-sessions/overview/overview-trends.tsx +++ b/apps/web/src/components/agent-sessions/overview/overview-trends.tsx @@ -1,13 +1,25 @@ -import type { ReactNode } from "react" +import { useMemo, type ReactNode } from "react" +import { Skeleton } from "@maple/ui/components/ui/skeleton" import { cn } from "@maple/ui/lib/utils" +import { makeBucketAxis } from "@/components/infra/chart-utils" +import { useLinkedCursor } from "@/hooks/use-linked-cursor" +import { useTimezonePreference } from "@/hooks/use-timezone-preference" import { - deltaToneClass, + OVERVIEW_CHARTS, type OverviewChartSummary, type OverviewModelMix, type OverviewSeriesPoint, } from "@/lib/agent-sessions/overview-analytics" +import { + buildOverviewPlotSpec, + type OverviewPlotLegendItem, + type OverviewPlotSpec, +} from "@/lib/agent-sessions/overview-chart-specs" + +import { DeltaReading } from "./overview-delta" +import { OVERVIEW_PLOT_HEIGHT, OverviewSmallMultiple } from "./overview-small-multiple" export interface OverviewTrendsProps { charts: ReadonlyArray @@ -28,7 +40,10 @@ export interface OverviewTrendsProps { * * Small multiples rather than one chart with a metric picker: the questions a * regression raises are "did anything else move at the same instant", and that - * is a comparison across charts, not a sequence of them. + * is a comparison across charts, not a sequence of them. One axis is built here + * for all nine — a shared bucket domain is what lets a rule drawn at the same + * fraction of every plot land on the same instant, which is the whole basis of + * the linked cursor below. */ export function OverviewTrends({ charts, @@ -39,77 +54,230 @@ export function OverviewTrends({ rail, waiting = false, }: OverviewTrendsProps) { + const { effectiveTimezone } = useTimezonePreference() + const { containerProps } = useLinkedCursor(true) + + const axis = useMemo(() => { + const buckets = new Set() + for (const point of series) buckets.add(point.bucket) + for (const point of previousSeries) buckets.add(point.bucket) + for (const point of modelMix.points) buckets.add(point.bucket) + const sorted = [...buckets].sort((a, b) => a - b) + const base = makeBucketAxis( + sorted.map((ms) => new Date(ms).toISOString()), + effectiveTimezone, + ) + const spanMs = sorted.length < 2 ? 0 : sorted[sorted.length - 1]! - sorted[0]! + return { + ...base, + // The shared tick format is written for a full-width chart: "Sep 5, 12:00 + // AM" is most of a 280px plot, so one label survived thinning and the axis + // read as unlabelled. These carry the terse form the design uses. + x: { + ...base.x, + axis: { + ...base.x.axis, + ticks: { ...base.x.axis.ticks, format: terseTick(spanMs, effectiveTimezone) }, + }, + }, + } + }, [series, previousSeries, modelMix, effectiveTimezone]) + + const specs = useMemo(() => { + const input = { series, previousSeries, modelMix } + return new Map(OVERVIEW_CHARTS.map((id) => [id, buildOverviewPlotSpec(id, input)])) + }, [series, previousSeries, modelMix]) + return ( -
-
-

Trends

- {note} +
+
+
+

+ Trends +

+ + nine metrics, one clock — hover any chart to move the crosshair on all nine + +
+ {note}
-
+
{charts.map((chart) => ( - 0} - /> + ))}
-
{rail}
+
+ {rail} +
) } +const DAY_MS = 86_400_000 + +/** + * An x tick as one of these plots can afford to print it: a clock alone inside + * a day, a date alone on a midnight boundary, and the pair only where a tick + * lands mid-day in a window that spans several. + */ +function terseTick(spanMs: number, timeZone: string | undefined): (value: Date) => string { + const day = new Intl.DateTimeFormat(undefined, { timeZone, month: "short", day: "numeric" }) + const clock = new Intl.DateTimeFormat(undefined, { + timeZone, + hour: "2-digit", + minute: "2-digit", + hour12: false, + }) + return (value: Date) => { + const time = clock.format(value).replace(/^24:/, "00:") + if (spanMs <= DAY_MS) return time + return time === "00:00" ? day.format(value) : `${day.format(value)} ${time}` + } +} + /** - * One small multiple. - * - * The plot area is a placeholder in this pass — the headline, the unit and the - * delta are the parts the rest of the page is wired to, and the marks land here - * without moving anything above them. + * Hairlines between columns, never between rows — the grid is one instrument + * and a full lattice would read as nine cards. The stacked breakpoints keep + * that true at two and three columns and fall back to row rules at one, where + * there is no column to divide. + */ +const CHART_GRID = cn( + "grid grid-cols-1 @min-[700px]/page:grid-cols-2 @min-[1000px]/page:grid-cols-3", + "[&>figure]:border-b [&>figure]:border-border [&>figure:last-child]:border-b-0", + "@min-[700px]/page:[&>figure]:border-r @min-[700px]/page:[&>figure]:border-b-0", + "@min-[700px]/page:[&>figure:nth-child(2n)]:border-r-0", + "@min-[1000px]/page:[&>figure:nth-child(2n)]:border-r", + "@min-[1000px]/page:[&>figure:nth-child(3n)]:border-r-0", + "@min-[700px]/page:[&>figure:last-child]:border-r-0", +) + +/** + * One small multiple: what it measures, where it stands now, how it moved, what + * the marks mean, and then the marks. */ function ChartCell({ chart, - buckets, - ghost, + spec, + axis, }: { chart: OverviewChartSummary - buckets: number - ghost: boolean + spec: OverviewPlotSpec | undefined + axis: ReturnType }) { return ( -
-
- {chart.title} - - +
+
+ + + {chart.title} + + {chart.value} - {chart.delta === null ? null : ( - - {chart.delta.text} - - )} - {chart.unit} -
-
- - {buckets === 0 ? "no data in range" : `${buckets} buckets${ghost ? " · ghost" : ""}`} + + + {chart.unit} + + -
+
+ + {spec === undefined ? null : ( + <> +
+ {spec.legend.map((item) => ( + + ))} + {spec.legendMore === 0 ? null : ( + + +{spec.legendMore} + + )} +
+
+ +
+ + )}
) } + +function LegendItem({ item }: { item: OverviewPlotLegendItem }) { + return ( + + + + {item.label} + + + ) +} + +function Swatch({ item }: { item: OverviewPlotLegendItem }) { + if (item.kind === "ghost") { + return ( + + ) + } + if (item.kind === "line") { + return ( + + ) + } + return ( + + ) +} + +/** The grid's shape while the summary read is in flight — nine cells, not one box. */ +export function OverviewTrendsLoading() { + return ( +
+
+ +
+
+ {OVERVIEW_CHARTS.map((id) => ( +
+ + + +
+ ))} +
+
+ ) +} diff --git a/apps/web/src/lib/agent-sessions/overview-chart-specs.test.ts b/apps/web/src/lib/agent-sessions/overview-chart-specs.test.ts new file mode 100644 index 000000000..3b5768007 --- /dev/null +++ b/apps/web/src/lib/agent-sessions/overview-chart-specs.test.ts @@ -0,0 +1,108 @@ +import { describe, expect, it } from "vitest" + +import { + EMPTY_OVERVIEW_MEASURES, + buildModelMix, + buildOverviewSeries, + shiftOverviewSeries, + type OverviewMeasurePoint, + type OverviewMeasures, + type OverviewModelMix, +} from "./overview-analytics" +import { buildOverviewPlotSpec, type OverviewPlotInput } from "./overview-chart-specs" + +const HOUR = 3_600_000 +const START = Date.UTC(2026, 8, 10, 0, 0, 0) + +const point = (index: number, overrides: Partial): OverviewMeasurePoint => ({ + ...EMPTY_OVERVIEW_MEASURES, + ...overrides, + bucket: START + index * HOUR, +}) + +const EMPTY_MIX: OverviewModelMix = { models: [], points: [] } + +const input = (overrides: Partial): OverviewPlotInput => ({ + series: [], + previousSeries: [], + modelMix: EMPTY_MIX, + ...overrides, +}) + +const series = buildOverviewSeries([ + point(0, { sessions: 10, cost: 5, tokens: 300, inputTokens: 200, cacheReadTokens: 100 }), + point(1, { sessions: 20, cost: 30, tokens: 900, inputTokens: 600, outputTokens: 300 }), +]) + +describe("buildOverviewPlotSpec", () => { + it("draws the previous period only when there is one to draw", () => { + const without = buildOverviewPlotSpec("sessions", input({ series })) + expect(without.marks.map((mark) => mark.kind)).toEqual(["line"]) + + const withGhost = buildOverviewPlotSpec( + "sessions", + input({ series, previousSeries: shiftOverviewSeries(series, -2 * HOUR) }), + ) + expect(withGhost.marks.map((mark) => mark.kind)).toEqual(["line", "ghost"]) + expect(withGhost.legend.at(-1)?.label).toBe("prev") + }) + + it("tops the axis on a round number above the data, and at 1 for an empty window", () => { + expect(buildOverviewPlotSpec("sessions", input({ series })).yMax).toBe(20) + expect(buildOverviewPlotSpec("toolCallsPerSession", input({ series })).yMax).toBe(1) + expect(buildOverviewPlotSpec("sessions", input({})).yMax).toBe(1) + }) + + it("pins the share charts to a full axis so a mix is read against 100%", () => { + expect(buildOverviewPlotSpec("cacheHitRatio", input({ series })).yMax).toBe(1) + expect(buildOverviewPlotSpec("modelMix", input({})).yMax).toBe(1) + }) + + it("stacks the token bands, each layer sitting on the one below", () => { + const spec = buildOverviewPlotSpec("tokensPerSession", input({ series })) + expect(spec.marks.map((mark) => mark.key)).toEqual([ + "input", + "cacheRead", + "cacheWrite", + "output", + "reasoning", + ]) + // 200 input + 100 cache read over 10 sessions: 20 then 30. + expect(spec.rows[0]?.input).toBe(20) + expect(spec.rows[0]?.cacheRead_base).toBe(20) + expect(spec.rows[0]?.cacheRead).toBe(30) + expect(spec.yMax).toBe(50) + }) + + it("falls back to one band when no bucket reported a breakdown", () => { + const flat = buildOverviewSeries([point(0, { sessions: 4, tokens: 400 })]) + const spec = buildOverviewPlotSpec("tokensPerSession", input({ series: flat })) + expect(spec.marks.map((mark) => mark.key)).toEqual(["total"]) + expect(spec.rows[0]?.total).toBe(100) + }) + + it("names the leading models in the legend and counts the rest", () => { + const mix = buildModelMix( + ["a", "b", "c", "d"].map((model, index) => ({ + bucket: START, + model, + llmCallSpans: 10 - index, + })), + ) + const spec = buildOverviewPlotSpec("modelMix", input({ modelMix: mix })) + expect(spec.marks).toHaveLength(4) + expect(spec.legend).toHaveLength(3) + expect(spec.legendMore).toBe(1) + expect(spec.legend[0]?.label).toBe("a 29%") + }) + + it("spreads the duration band from p50 up to p95", () => { + const durations = buildOverviewSeries([ + point(0, { sessions: 1, sessionDurationP50Ms: 1_000, sessionDurationP95Ms: 4_000 }), + ]) + const spec = buildOverviewPlotSpec("sessionDuration", input({ series: durations })) + expect(spec.marks.map((mark) => mark.kind)).toEqual(["spread", "line"]) + expect(spec.rows[0]?.p95_base).toBe(1_000) + expect(spec.rows[0]?.p95).toBe(4_000) + }) +}) diff --git a/apps/web/src/lib/agent-sessions/overview-chart-specs.ts b/apps/web/src/lib/agent-sessions/overview-chart-specs.ts new file mode 100644 index 000000000..4a4da17fc --- /dev/null +++ b/apps/web/src/lib/agent-sessions/overview-chart-specs.ts @@ -0,0 +1,343 @@ +import { formatWarehouseDateTime } from "@maple/query-engine" +import { formatErrorRate, formatNumber, formatPercent } from "@maple/ui/lib/format" + +import { + OVERVIEW_MODEL_MIX_OTHER, + OVERVIEW_TOKEN_BANDS, + OVERVIEW_TOKEN_FALLBACK_BAND, + formatOverviewCount, + formatOverviewDuration, + formatPerSession, + type OverviewChartId, + type OverviewModelMix, + type OverviewSeriesPoint, + type OverviewTokenBandKey, +} from "./overview-analytics" +import { formatCost } from "./session-summary" + +/* ------------------------------------------------------------------------------------------------- + * The shape a small multiple is drawn from + * -----------------------------------------------------------------------------------------------*/ + +/** One bucket, wide: every mark on the chart reads its own field off this row. */ +export interface OverviewPlotRow extends Record { + bucket: string + date: Date +} + +/** + * `line` is the subject, `ghost` the previous period behind it, `band` a layer + * of a stack, `spread` the faint region a line of the same colour reads over. + */ +export type OverviewPlotKind = "line" | "ghost" | "band" | "spread" + +export interface OverviewPlotMark { + /** The row field this mark plots — a band's TOP edge. */ + readonly key: string + readonly color: string + readonly kind: OverviewPlotKind + /** A band's floor field. A line sits on the axis and omits it. */ + readonly base?: string + /** The tooltip's name for the series. */ + readonly label: string +} + +export interface OverviewPlotLegendItem { + readonly label: string + readonly color: string + readonly kind: OverviewPlotKind +} + +export interface OverviewPlotSpec { + readonly rows: ReadonlyArray + /** Painted in order — bands first, then the lines that read over them. */ + readonly marks: ReadonlyArray + readonly legend: ReadonlyArray + /** Bands the legend did not name, as the design's trailing `+2`. */ + readonly legendMore: number + /** The axis top. Every plot draws exactly `0` and this. */ + readonly yMax: number + readonly format: (value: number) => string +} + +export interface OverviewPlotInput { + readonly series: ReadonlyArray + /** Already shifted onto this axis; empty with the comparison off. */ + readonly previousSeries: ReadonlyArray + readonly modelMix: OverviewModelMix +} + +/* ------------------------------------------------------------------------------------------------- + * Colours + * -----------------------------------------------------------------------------------------------*/ + +const PRIMARY = "var(--primary)" +/** The previous period: present, and never mistakable for the subject. */ +const GHOST = "var(--muted-foreground)" + +/** + * The token buckets wear their own designated hues rather than `--chart-1..5` + * slots — the same five the session detail's usage bar draws, so a band here + * and a segment there are the same colour for the same tokens. + */ +const TOKEN_BAND_COLOR = { + input: "var(--chart-tok-input)", + cacheRead: "var(--chart-tok-cache-read)", + cacheWrite: "var(--chart-tok-cache-write)", + output: "var(--chart-tok-output)", + reasoning: "var(--chart-tok-reasoning)", + total: PRIMARY, +} satisfies Record + +/** Short enough that five of them fit on one legend line at this width. */ +const TOKEN_BAND_SHORT = { + input: "in", + cacheRead: "cache r", + cacheWrite: "cache w", + output: "out", + reasoning: "reason", + total: "tokens", +} satisfies Record + +const MODEL_MIX_COLORS = [ + "var(--chart-1)", + "var(--chart-2)", + "var(--chart-3)", + "var(--chart-4)", + "var(--chart-5)", +] as const +/** The folded tail is a residue, not a model — it wears no chart slot. */ +const MODEL_MIX_OTHER_COLOR = "var(--muted-foreground)" +/** Model names are long; the rest of them are counted instead of listed. */ +const MODEL_MIX_LEGEND_LIMIT = 3 + +export const overviewModelMixColor = (index: number, model: string): string => + model === OVERVIEW_MODEL_MIX_OTHER + ? MODEL_MIX_OTHER_COLOR + : (MODEL_MIX_COLORS[index % MODEL_MIX_COLORS.length] ?? MODEL_MIX_OTHER_COLOR) + +/** A band's floor field, beside the field carrying its top edge. */ +const baseKey = (key: string): string => `${key}_base` + +/* ------------------------------------------------------------------------------------------------- + * The nine specs + * -----------------------------------------------------------------------------------------------*/ + +/** + * What one small multiple draws, as data. + * + * Pure so the awkward parts — which token bands survive the fallback, where a + * stack's layers sit, what the axis tops out at — are testable without a + * rendered chart, and so the plot component stays a translation of this into + * marks rather than nine branches of chart code. + */ +export function buildOverviewPlotSpec(id: OverviewChartId, input: OverviewPlotInput): OverviewPlotSpec { + switch (id) { + case "sessions": + return lineSpec(input, (point) => point.sessions, "sessions", formatOverviewCount) + case "costPerSession": + return lineSpec(input, (point) => point.costPerSession, "$ / session", formatCost) + case "tokensPerSession": + return tokenSpec(input) + case "toolCallsPerSession": + return lineSpec(input, (point) => point.toolCallsPerSession, "calls / session", formatPerSession) + case "errorRate": + return errorRateSpec(input) + case "sessionDuration": + return durationSpec(input) + case "llmCallsPerSession": + return lineSpec(input, (point) => point.llmCallsPerSession, "calls / session", formatPerSession) + case "modelMix": + return modelMixSpec(input) + case "cacheHitRatio": + return lineSpec(input, (point) => point.cacheHitRatio, "hit ratio", formatPercent, 1) + } +} + +/** One series, plus the previous period behind it when the comparison is on. */ +function lineSpec( + input: OverviewPlotInput, + read: (point: OverviewSeriesPoint) => number, + label: string, + format: (value: number) => string, + fixedMax?: number, +): OverviewPlotSpec { + const previous = new Map(input.previousSeries.map((point) => [point.bucket, read(point)])) + const rows = input.series.map((point) => + row(point.bucket, { value: read(point), prev: previous.get(point.bucket) ?? null }), + ) + const marks: OverviewPlotMark[] = [{ key: "value", label, color: PRIMARY, kind: "line" }] + if (input.previousSeries.length > 0) { + marks.push({ key: "prev", label: "prev", color: GHOST, kind: "ghost" }) + } + return spec(rows, marks, fixedMax ?? axisTop(rows, marks), format) +} + +/** + * Tokens per session, split five ways. + * + * The band set is decided once for the whole window rather than per bucket: an + * SDK that reports no breakdown reports none all window, and a stack that + * changed its own vocabulary mid-chart would read as a shift in usage. + */ +function tokenSpec(input: OverviewPlotInput): OverviewPlotSpec { + const split = input.series.some((point) => + OVERVIEW_TOKEN_BANDS.some((band) => point.tokenBands[band] > 0), + ) + const bands: ReadonlyArray = split + ? OVERVIEW_TOKEN_BANDS + : [OVERVIEW_TOKEN_FALLBACK_BAND] + + const rows = input.series.map((point) => + stackRow( + point.bucket, + bands.map((band) => ({ key: band, value: point.tokenBands[band] })), + ), + ) + const marks = bands.map((band) => bandMark(band, TOKEN_BAND_COLOR[band], TOKEN_BAND_SHORT[band])) + return spec(rows, marks, axisTop(rows, marks), formatNumber) +} + +/** The three layers a session can fail at, on one rate axis. */ +function errorRateSpec(input: OverviewPlotInput): OverviewPlotSpec { + const rows = input.series.map((point) => + row(point.bucket, { + sessions: point.sessionErrorRate, + llm: point.llmErrorRate, + tool: point.toolErrorRate, + }), + ) + const marks: ReadonlyArray = [ + { key: "sessions", label: "sessions", color: "var(--severity-error)", kind: "line" }, + { key: "llm", label: "llm calls", color: "var(--chart-2)", kind: "line" }, + { key: "tool", label: "tool calls", color: "var(--chart-5)", kind: "line" }, + ] + return spec(rows, marks, axisTop(rows, marks), formatErrorRate) +} + +/** The median with the spread above it — the tail is the question. */ +function durationSpec(input: OverviewPlotInput): OverviewPlotSpec { + const rows = input.series.map((point) => + row(point.bucket, { + p50: point.sessionP50Ms, + p95: point.sessionP95Ms, + p95_base: point.sessionP50Ms, + }), + ) + const marks: ReadonlyArray = [ + { key: "p95", base: "p95_base", label: "p50 – p95", color: PRIMARY, kind: "spread" }, + { key: "p50", label: "p50", color: PRIMARY, kind: "line" }, + ] + return spec(rows, marks, axisTop(rows, marks), formatOverviewDuration) +} + +/** Every bucket normalised to 1, so the question is share and not volume. */ +function modelMixSpec(input: OverviewPlotInput): OverviewPlotSpec { + const { models, points } = input.modelMix + const rows = points.map((point) => + stackRow( + point.bucket, + models.map((model) => ({ key: model, value: point.shares[model] ?? 0 })), + ), + ) + const marks = models.map((model, index) => bandMark(model, overviewModelMixColor(index, model), model)) + const legend = marks.slice(0, MODEL_MIX_LEGEND_LIMIT).map((mark, index) => ({ + label: `${models[index] ?? ""} ${formatPercent(modelShare(input.modelMix, models[index] ?? ""))}`, + color: mark.color, + kind: mark.kind, + })) + return { + rows, + marks, + legend, + legendMore: Math.max(0, marks.length - legend.length), + yMax: 1, + format: formatPercent, + } +} + +/** A model's share of the window's plotted spans, for its legend entry. */ +function modelShare(mix: OverviewModelMix, model: string): number { + let total = 0 + let own = 0 + for (const point of mix.points) { + for (const band of mix.models) total += point.spans[band] ?? 0 + own += point.spans[model] ?? 0 + } + return total === 0 ? 0 : own / total +} + +/* ------------------------------------------------------------------------------------------------- + * Row and axis plumbing + * -----------------------------------------------------------------------------------------------*/ + +function row(bucketMs: number, values: Record): OverviewPlotRow { + return { bucket: formatWarehouseDateTime(bucketMs), date: new Date(bucketMs), ...values } +} + +/** A stack's layers, each carrying the top edge it landed on and its floor. */ +function stackRow(bucketMs: number, layers: ReadonlyArray<{ key: string; value: number }>): OverviewPlotRow { + const values: Record = {} + let floor = 0 + for (const layer of layers) { + values[baseKey(layer.key)] = floor + floor += layer.value + values[layer.key] = floor + } + return row(bucketMs, values) +} + +const bandMark = (key: string, color: string, label: string): OverviewPlotMark => ({ + key, + base: baseKey(key), + color, + kind: "band", + label, +}) + +function spec( + rows: ReadonlyArray, + marks: ReadonlyArray, + yMax: number, + format: (value: number) => string, +): OverviewPlotSpec { + return { + rows, + marks, + legend: marks.map((mark) => ({ label: mark.label, color: mark.color, kind: mark.kind })), + legendMore: 0, + yMax, + format, + } +} + +/** + * A finer ladder than a chart library's: these plots are 86px tall, so a top + * two steps above the data spends a quarter of the height on empty air. Every + * rung still prints as a round number through the chart's own formatter. + */ +const CEILING_STEPS = [1, 1.2, 1.5, 1.8, 2, 2.5, 3, 4, 5, 6, 8, 10] + +/** + * The axis top: the next round number above the largest plotted value. + * + * Above, not equal to — a flat series at exactly the top rides the ceiling and + * reads as clipped rather than as steady, which is the one thing these nine + * charts exist to show. A stack's layers carry their cumulative top, so the + * same maximum covers both shapes; an all-zero window still gets a `1` so the + * scale has a domain, which draws the flat floor it should. + */ +function axisTop(rows: ReadonlyArray, marks: ReadonlyArray): number { + let max = 0 + for (const plotRow of rows) { + for (const mark of marks) { + const value = plotRow[mark.key] + if (typeof value === "number" && value > max) max = value + } + } + if (max <= 0) return 1 + const magnitude = 10 ** Math.floor(Math.log10(max)) + const scaled = max / magnitude + const step = CEILING_STEPS.find((candidate) => scaled <= candidate * 1.000_001) ?? 10 + return step * magnitude +} diff --git a/apps/web/src/routes/agent-sessions/overview.tsx b/apps/web/src/routes/agent-sessions/overview.tsx index 540a6b11b..435fc9af2 100644 --- a/apps/web/src/routes/agent-sessions/overview.tsx +++ b/apps/web/src/routes/agent-sessions/overview.tsx @@ -7,6 +7,7 @@ import { toEpochMs } from "@maple/ui/lib/time-format" import { AgentOverviewView } from "@/components/agent-sessions/overview/agent-overview-view" import { OverviewMetricStripLoading } from "@/components/agent-sessions/overview/overview-metric-strip" +import { OverviewTrendsLoading } from "@/components/agent-sessions/overview/overview-trends" import { QueryErrorState } from "@/components/common/query-error-state" import { DashboardLayout } from "@/components/layout/dashboard-layout" import { NotFoundError } from "@/components/route-error" @@ -176,47 +177,59 @@ function AgentOverviewBody({ .onSuccess((value) => value.data) .orElse(() => []) - return Result.builder(results.summary) - .onInitial(() => ( -
- - - - -
- )) - .onError((error) => ( - - )) - .onSuccess((summary, result) => ( - - )) - .render() + return ( + Result.builder(results.summary) + // Shaped like what lands, so nothing reflows when it does: the strip keeps + // its seven tiles and the grid its nine cells. + .onInitial(() => ( +
+
+ + +
+ + +
+ + {Array.from({ length: 5 }).map((_, index) => ( + + ))} +
+
+ )) + .onError((error) => ( + + )) + .onSuccess((summary, result) => ( + + )) + .render() + ) } From 7dc09e8736296a62f7e6df096b0654c1eeb032a7 Mon Sep 17 00:00:00 2001 From: JeremyFunk Date: Fri, 11 Sep 2026 07:30:08 +0200 Subject: [PATCH 07/16] fix(agent-sessions): overview priced share, window label and bucket ladder MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The priced share divided the netted priced calls by the raw span count, so a fully priced window read as the netting factor rather than 100%. The window label came from the page's default preset, which the resolver ignores once both endpoints are in the URL — an absolute range was labelled "prev 7d" whatever its length. The grid asked for ~100 buckets, which nine ~104px plots cannot separate, at widths like 105 minutes an axis had to call "2h". It now snaps to a ladder a reader recognises: a day at 1h, a week at 6h, a month at 1d. Also: the bucketed input mirrors the domain's `BucketSeconds`, the active tab merges its search instead of replacing it, and the two per-call duration quantiles leave the view model, which never rendered them. --- .../api/warehouse/ai-agent-overview.test.ts | 10 +++-- .../src/api/warehouse/ai-agent-overview.ts | 9 +++-- .../tools/agent-sessions-tabs.tsx | 7 +++- .../src/components/infra/chart-utils.test.ts | 37 +++++++++++++++++++ apps/web/src/components/infra/chart-utils.ts | 36 ++++++++++++++++++ apps/web/src/lab/agent-overview-fixture.ts | 22 +---------- .../agent-sessions/overview-analytics.test.ts | 35 +++++++++++++++++- .../lib/agent-sessions/overview-analytics.ts | 24 +++++++----- .../agent-sessions/overview-search.test.ts | 24 ++++++++++++ .../src/lib/agent-sessions/overview-search.ts | 21 +++++++++++ .../lib/agent-sessions/use-agent-overview.ts | 6 ++- .../src/routes/agent-sessions/overview.tsx | 20 ++++++---- 12 files changed, 202 insertions(+), 49 deletions(-) diff --git a/apps/web/src/api/warehouse/ai-agent-overview.test.ts b/apps/web/src/api/warehouse/ai-agent-overview.test.ts index 72eadd6cb..4e319c793 100644 --- a/apps/web/src/api/warehouse/ai-agent-overview.test.ts +++ b/apps/web/src/api/warehouse/ai-agent-overview.test.ts @@ -33,12 +33,16 @@ const wire = (overrides: Partial = {}): AiOverviewMeasures = }) describe("mapOverviewMeasures", () => { - it("converts every quantile from nanoseconds to milliseconds", () => { + it("converts the session quantiles from nanoseconds to milliseconds", () => { const row = mapOverviewMeasures(wire()) expect(row.sessionDurationP50Ms).toBe(42_000) expect(row.sessionDurationP95Ms).toBe(96_000) - expect(row.llmDurationP50Ms).toBe(1_900) - expect(row.llmDurationP95Ms).toBe(7_400) + }) + + it("drops the per-call quantiles, which nothing on the board reads", () => { + const row = mapOverviewMeasures(wire()) + expect(row).not.toHaveProperty("llmDurationP50Ms") + expect(row).not.toHaveProperty("llmDurationP95Ms") }) it("carries the raw span population separately from the netted volume", () => { diff --git a/apps/web/src/api/warehouse/ai-agent-overview.ts b/apps/web/src/api/warehouse/ai-agent-overview.ts index 9ae57808f..2913d69eb 100644 --- a/apps/web/src/api/warehouse/ai-agent-overview.ts +++ b/apps/web/src/api/warehouse/ai-agent-overview.ts @@ -5,7 +5,8 @@ // a literal `Z`; `toEpochMs` reads them as UTC, where `new Date(value)` would // read a bare warehouse datetime as local time. Durations arrive in // nanoseconds and leave in milliseconds, because every formatter downstream -// takes milliseconds. +// takes milliseconds — the per-call quantiles are dropped here rather than +// converted, since nothing on the board reads them. // // The page's filters are single-valued — one model, one agent, one tool — and // the contract takes arrays. The widening happens in `selectionFields`, so a @@ -18,6 +19,7 @@ import { AiOverviewDimension, AiOverviewModelMixRequest, AiOverviewSummaryRequest, + BucketSeconds, type AiOverviewBreakdownRow, type AiOverviewMeasures, type AiOverviewModelMixPoint, @@ -58,7 +60,8 @@ export type AiOverviewSelection = Schema.Schema.Type const AiOverviewBucketedInput = Schema.Struct({ ...AiOverviewSelection.fields, - bucketSeconds: Schema.Number, + /** The domain's own bound, for the reason the breakdown input gives below. */ + bucketSeconds: BucketSeconds, }) export type AiOverviewBucketedInput = Schema.Schema.Type @@ -116,8 +119,6 @@ export function mapOverviewMeasures(row: AiOverviewMeasures): OverviewMeasures { reasoningTokens: row.reasoningTokens, sessionDurationP50Ms: row.sessionDurationP50Ns / NS_PER_MS, sessionDurationP95Ms: row.sessionDurationP95Ns / NS_PER_MS, - llmDurationP50Ms: row.llmDurationP50Ns / NS_PER_MS, - llmDurationP95Ms: row.llmDurationP95Ns / NS_PER_MS, } } diff --git a/apps/web/src/components/agent-sessions/tools/agent-sessions-tabs.tsx b/apps/web/src/components/agent-sessions/tools/agent-sessions-tabs.tsx index 0006ca46f..62586b902 100644 --- a/apps/web/src/components/agent-sessions/tools/agent-sessions-tabs.tsx +++ b/apps/web/src/components/agent-sessions/tools/agent-sessions-tabs.tsx @@ -16,6 +16,11 @@ export type AgentSessionsTab = "overview" | "sessions" * undoes the switch. Only the window travels between them — the overview's * dimension filters mean nothing to a list that pages one session at a time. * + * The tab already showing is the exception: it navigates to where it already + * is, so it MERGES rather than replaces. A `search` object replaces the whole + * search, and clicking the tab you are on is not how anyone asks for their + * filters to be cleared. + * * The Sessions list has no time picker (it is fixed to a rolling week), so a * jump from there carries no window and this page falls back to its own default. */ @@ -68,7 +73,7 @@ function TabLink({ return ( ) => ({ ...prev, ...search }) : search} aria-current={active ? "page" : undefined} className={cn( "flex h-9 items-center gap-[7px] border-b-2 px-3 font-mono text-[12.5px] transition-colors first:pl-0.5", diff --git a/apps/web/src/components/infra/chart-utils.test.ts b/apps/web/src/components/infra/chart-utils.test.ts index 2031356f9..3918902c2 100644 --- a/apps/web/src/components/infra/chart-utils.test.ts +++ b/apps/web/src/components/infra/chart-utils.test.ts @@ -4,6 +4,7 @@ import { formatValueWithUnit, isoToLabel, makeBucketLabeler, + smallMultipleBucketSeconds, transformRows, } from "./chart-utils" @@ -94,3 +95,39 @@ describe("transformRows", () => { expect(data[0]?.time).not.toBe(data[1]?.time) }) }) + +describe("smallMultipleBucketSeconds", () => { + /** A window of `hours`, spelled the way the warehouse spells one. */ + const window = (hours: number) => { + const endMs = Date.UTC(2026, 8, 11, 0, 0, 0) + const iso = (ms: number) => new Date(ms).toISOString().replace("T", " ").slice(0, 19) + return [iso(endMs - hours * 3_600_000), iso(endMs)] as const + } + const widthOf = (hours: number) => smallMultipleBucketSeconds(...window(hours)) + + it("cuts the usual windows at a width a reader recognises", () => { + expect(widthOf(24)).toBe(3_600) + expect(widthOf(24 * 7)).toBe(21_600) + expect(widthOf(24 * 30)).toBe(86_400) + }) + + it("keeps every window inside the count a ~100px plot can separate", () => { + for (const hours of [1, 6, 12, 24, 72, 24 * 7, 24 * 30]) { + const points = (hours * 3_600) / widthOf(hours) + expect(points).toBeLessThanOrEqual(36) + } + }) + + it("stays on the five-minute grid the API's bucket bound needs", () => { + for (const hours of [0.25, 1, 6, 12, 24, 72, 24 * 7, 24 * 30, 24 * 90]) { + const width = widthOf(hours) + expect(width % 300).toBe(0) + expect(Number.isInteger(width)).toBe(true) + } + }) + + it("floors at five minutes and tops out at a day", () => { + expect(widthOf(0.25)).toBe(300) + expect(widthOf(24 * 365)).toBe(86_400) + }) +}) diff --git a/apps/web/src/components/infra/chart-utils.ts b/apps/web/src/components/infra/chart-utils.ts index 81840a412..752764198 100644 --- a/apps/web/src/components/infra/chart-utils.ts +++ b/apps/web/src/components/infra/chart-utils.ts @@ -28,6 +28,42 @@ export function chartBucketSeconds(startTime: string, endTime: string): number { return Math.max(300, Math.ceil(windowSeconds / 100 / 300) * 300) } +/** + * Bucket widths a small multiple can be read at: whole, recognisable steps from + * five minutes to a day, every one of them a multiple of 300 so the width also + * satisfies the API's `BucketSeconds`. + */ +const SMALL_MULTIPLE_CEILING = 86_400 +const SMALL_MULTIPLE_LADDER: ReadonlyArray = [ + 300, + 900, + 1_800, + 3_600, + 10_800, + 21_600, + 43_200, + SMALL_MULTIPLE_CEILING, +] + +/** Points a ~100px-tall plot can separate. A hundred of them land under a pixel each. */ +const SMALL_MULTIPLE_TARGET_POINTS = 30 + +/** + * Bucket width for a grid of small multiples: about thirty points, snapped up + * to the ladder above. + * + * Two things differ from `chartBucketSeconds`, which a full-width chart wants. + * The count: a plot barely a hundred pixels tall cannot show a hundred buckets. + * And the snapping: dividing a window into a hundred equal parts produces + * widths like 105 minutes, which an axis note has to call "2h" while the + * buckets are something else. A day reads at 1h, a week at 6h, a month at 1d. + */ +export function smallMultipleBucketSeconds(startTime: string, endTime: string): number { + const windowSeconds = Math.max((toEpochMs(endTime) - toEpochMs(startTime)) / 1000, 300) + const target = windowSeconds / SMALL_MULTIPLE_TARGET_POINTS + return SMALL_MULTIPLE_LADDER.find((width) => width >= target) ?? SMALL_MULTIPLE_CEILING +} + /** Every value unit an infra chart can carry. Drives unit-aware formatting. */ export type ChartUnit = | "percent" diff --git a/apps/web/src/lab/agent-overview-fixture.ts b/apps/web/src/lab/agent-overview-fixture.ts index 41d2575c1..871568d45 100644 --- a/apps/web/src/lab/agent-overview-fixture.ts +++ b/apps/web/src/lab/agent-overview-fixture.ts @@ -67,8 +67,6 @@ interface SessionRates { pricedShare: number p50Ms: number p95Ms: number - llmP50Ms: number - llmP95Ms: number } const HEALTHY: SessionRates = { @@ -85,8 +83,6 @@ const HEALTHY: SessionRates = { pricedShare: 0.94, p50Ms: 42_000, p95Ms: 96_000, - llmP50Ms: 1_900, - llmP95Ms: 7_400, } /** The step the investigating board is drawn around. Sessions barely move. */ @@ -100,8 +96,6 @@ const REGRESSED: SessionRates = { tokensPerSession: 118_000, cacheShare: 0.12, p95Ms: 227_000, - llmP50Ms: 3_100, - llmP95Ms: 18_600, } /** A deterministic wobble, so a board looks like traffic and not like a ruler. */ @@ -143,7 +137,7 @@ function measuresOf(rates: SessionRates): OverviewMeasures { toolCalls, erroredToolCalls: Math.round(toolCalls * rates.toolErrorRate), cost: Number((sessions * rates.costPerSession).toFixed(2)), - pricedLlmCalls: Math.round(llmCallSpans * rates.pricedShare), + pricedLlmCalls: Math.round(llmCalls * rates.pricedShare), tokens, inputTokens: Math.round(prompt * (1 - rates.cacheShare)), cacheReadTokens: Math.round(prompt * rates.cacheShare), @@ -152,21 +146,13 @@ function measuresOf(rates: SessionRates): OverviewMeasures { reasoningTokens: Math.round(tokens * 0.06), sessionDurationP50Ms: rates.p50Ms, sessionDurationP95Ms: rates.p95Ms, - llmDurationP50Ms: rates.llmP50Ms, - llmDurationP95Ms: rates.llmP95Ms, } } /** Counts sum; quantiles do not, so the window's own are passed in. */ function foldMeasures( points: ReadonlyArray, - quantiles: Pick< - OverviewMeasures, - | "sessionDurationP50Ms" - | "sessionDurationP95Ms" - | "llmDurationP50Ms" - | "llmDurationP95Ms" - >, + quantiles: Pick, ): OverviewMeasures { const sum = points.reduce( (total, point) => ({ @@ -488,14 +474,10 @@ export function buildOverviewFixture(scenario: OverviewScenario, nowMs: number): const current = foldMeasures(series, { sessionDurationP50Ms: currentRates.p50Ms, sessionDurationP95Ms: currentRates.p95Ms, - llmDurationP50Ms: currentRates.llmP50Ms, - llmDurationP95Ms: currentRates.llmP95Ms, }) const previous = foldMeasures(previousSeries, { sessionDurationP50Ms: spec.previous.p50Ms, sessionDurationP95Ms: spec.previous.p95Ms, - llmDurationP50Ms: spec.previous.llmP50Ms, - llmDurationP95Ms: spec.previous.llmP95Ms, }) // Breakdown rows are measured over the WINDOW, like the tiles above them — diff --git a/apps/web/src/lib/agent-sessions/overview-analytics.test.ts b/apps/web/src/lib/agent-sessions/overview-analytics.test.ts index 85699671e..501ec0bbe 100644 --- a/apps/web/src/lib/agent-sessions/overview-analytics.test.ts +++ b/apps/web/src/lib/agent-sessions/overview-analytics.test.ts @@ -9,6 +9,7 @@ import { buildMovers, buildOverviewSeries, buildOverviewTiles, + bucketWidthLabel, cacheHitRatio, costPerSession, formatOverviewCount, @@ -56,6 +57,14 @@ describe("derivations", () => { it("measures the cache hit ratio against everything that could have been a prompt read", () => { expect(cacheHitRatio(measures({ inputTokens: 30, cacheReadTokens: 70 }))).toBe(0.7) }) + + it("divides the priced share by the netted volume the server priced", () => { + // 47 of 50 netted calls carried a price. The 59 spans behind them did not + // each need one, and dividing by those would read as 80% coverage. + expect(pricedShare(measures({ llmCalls: 50, llmCallSpans: 59, pricedLlmCalls: 47 }))).toBe( + 0.94, + ) + }) }) describe("tokenBandValues", () => { @@ -149,6 +158,21 @@ describe("formatters", () => { it("reads a zero duration as nothing measured rather than as 0μs", () => { expect(formatOverviewDuration(0)).toBe("—") }) + + it("names every width the grid's buckets are actually cut at", () => { + // The ladder `smallMultipleBucketSeconds` snaps to, one unit each. + const ladder = [300, 900, 1_800, 3_600, 10_800, 21_600, 43_200, 86_400] + expect(ladder.map(bucketWidthLabel)).toEqual([ + "5m", + "15m", + "30m", + "1h", + "3h", + "6h", + "12h", + "1d", + ]) + }) }) describe("buildOverviewTiles", () => { @@ -158,7 +182,8 @@ describe("buildOverviewTiles", () => { cost: 40, tokens: 1_000, toolCalls: 500, - llmCallSpans: 200, + llmCalls: 200, + llmCallSpans: 236, pricedLlmCalls: 188, sessionDurationP50Ms: 40_000, sessionDurationP95Ms: 90_000, @@ -403,7 +428,13 @@ describe("overviewScopeSummary", () => { describe("buildAgentOverviewData", () => { const input = { - current: measures({ sessions: 100, cost: 40, llmCallSpans: 100, pricedLlmCalls: 94 }), + current: measures({ + sessions: 100, + cost: 40, + llmCalls: 100, + llmCallSpans: 118, + pricedLlmCalls: 94, + }), previous: measures({ sessions: 80, cost: 40 }), series: [{ bucket: 2_000, ...measures({ sessions: 10 }) }], previousSeries: [{ bucket: 1_000, ...measures({ sessions: 8 }) }], diff --git a/apps/web/src/lib/agent-sessions/overview-analytics.ts b/apps/web/src/lib/agent-sessions/overview-analytics.ts index 7ebb47043..82914ef5d 100644 --- a/apps/web/src/lib/agent-sessions/overview-analytics.ts +++ b/apps/web/src/lib/agent-sessions/overview-analytics.ts @@ -31,8 +31,10 @@ import { OVERVIEW_DIMENSIONS, type OverviewDimension } from "./overview-search" * What every overview read reports, so a tile, a point on a chart and a table * row are the same numbers under different groupings. * - * Identical to the wire's `AiOverviewMeasures` except that the four quantiles - * are milliseconds here; see `api/warehouse/ai-agent-overview.ts`. + * A subset of the wire's `AiOverviewMeasures`: the session quantiles are + * milliseconds here rather than nanoseconds, and the per-call ones are dropped + * because nothing on the board reads them. See + * `api/warehouse/ai-agent-overview.ts`. */ export interface OverviewMeasures { readonly sessions: number @@ -56,8 +58,6 @@ export interface OverviewMeasures { readonly reasoningTokens: number readonly sessionDurationP50Ms: number readonly sessionDurationP95Ms: number - readonly llmDurationP50Ms: number - readonly llmDurationP95Ms: number } export const EMPTY_OVERVIEW_MEASURES: OverviewMeasures = { @@ -78,8 +78,6 @@ export const EMPTY_OVERVIEW_MEASURES: OverviewMeasures = { reasoningTokens: 0, sessionDurationP50Ms: 0, sessionDurationP95Ms: 0, - llmDurationP50Ms: 0, - llmDurationP95Ms: 0, } /** One bucket of a summary series. `bucket` is epoch milliseconds. */ @@ -130,10 +128,16 @@ export const llmCallsPerSession = (m: OverviewMeasures): number => ratio(m.llmCa export const cacheHitRatio = (m: OverviewMeasures): number => ratio(m.cacheReadTokens, m.inputTokens + m.cacheReadTokens) -/** `cost` is 0 for "nobody priced it" and not for "free" — this is how much of - * the window it actually covers. */ -export const pricedShare = (m: OverviewMeasures): number => - ratio(m.pricedLlmCalls, m.llmCallSpans) +/** + * `cost` is 0 for "nobody priced it" and not for "free" — this is how much of + * the window it actually covers. + * + * Over `llmCalls` and never `llmCallSpans` — the mirror image of the LLM error + * rate above. The server nets the priced calls exactly as it nets the volume, + * so the two are one population and a fully priced window reads 100% rather + * than the netting factor. + */ +export const pricedShare = (m: OverviewMeasures): number => ratio(m.pricedLlmCalls, m.llmCalls) /* ------------------------------------------------------------------------------------------------- * Token bands diff --git a/apps/web/src/lib/agent-sessions/overview-search.test.ts b/apps/web/src/lib/agent-sessions/overview-search.test.ts index 0c996c273..3166a9d6a 100644 --- a/apps/web/src/lib/agent-sessions/overview-search.test.ts +++ b/apps/web/src/lib/agent-sessions/overview-search.test.ts @@ -6,6 +6,7 @@ import { compareEnabled, failingOnly, overviewApiDimension, + overviewWindowLabel, sessionsLinkSearch, toggleOverviewFilter, type AgentOverviewSearch, @@ -110,3 +111,26 @@ describe("sessionsLinkSearch", () => { expect(sessionsLinkSearch({}, { hasErrors: true }).hasErrors).toBe(true) }) }) + +describe("overviewWindowLabel", () => { + const hours = (count: number) => count * 3_600_000 + /** An absolute range in the URL, which is what makes the default irrelevant. */ + const ABSOLUTE = { startTime: "2026-09-10 00:00:00", endTime: "2026-09-10 03:00:00" } + + it("lets a preset name itself", () => { + expect(overviewWindowLabel({ timePreset: "24h" }, hours(24))).toBe("24h") + }) + + it("falls back to the page's default only while the URL carries no window", () => { + expect(overviewWindowLabel({}, hours(3))).toBe("7d") + // Half a range is not a range: the resolver would still use the default. + expect(overviewWindowLabel({ startTime: ABSOLUTE.startTime }, hours(3))).toBe("7d") + }) + + it("names an absolute range after its own length, not after the default", () => { + expect(overviewWindowLabel(ABSOLUTE, hours(3))).toBe("3h") + expect(overviewWindowLabel(ABSOLUTE, hours(24))).toBe("24h") + expect(overviewWindowLabel(ABSOLUTE, hours(72))).toBe("3d") + expect(overviewWindowLabel(ABSOLUTE, 45 * 60_000)).toBe("45m") + }) +}) diff --git a/apps/web/src/lib/agent-sessions/overview-search.ts b/apps/web/src/lib/agent-sessions/overview-search.ts index a2cd968dd..6f58a85f7 100644 --- a/apps/web/src/lib/agent-sessions/overview-search.ts +++ b/apps/web/src/lib/agent-sessions/overview-search.ts @@ -10,6 +10,7 @@ import { Schema } from "effect" import type { AiOverviewDimension } from "@maple/domain/http" import type { AgentSessionsSearchState } from "@/components/agent-sessions/agent-sessions-filter-inputs" +import type { TimeRangeSearch } from "@/components/time-range-picker/search" import { BooleanFromStringParam } from "@/lib/search-params" const BooleanParam = Schema.optional(Schema.Union([Schema.Boolean, BooleanFromStringParam])) @@ -69,6 +70,26 @@ export type AgentOverviewSearch = Schema.Schema.Type /** Wide enough that a nightly agent shows up at all. */ export const AGENT_OVERVIEW_DEFAULT_PRESET = "7d" +/** + * What the page calls the window it is showing — `7d`, `24h`, `45m`. + * + * A preset names itself. An absolute range has no preset, and the default above + * does not name it either: the resolver hands a start/end pair straight back + * and never looks at the default, so a two-hour range picked by hand would + * otherwise be labelled "prev 7d". It is named after its own length instead, to + * the nearest whole unit. + */ +export function overviewWindowLabel(search: TimeRangeSearch, windowMs: number): string { + if (search.timePreset !== undefined) return search.timePreset + if (search.startTime === undefined || search.endTime === undefined) { + return AGENT_OVERVIEW_DEFAULT_PRESET + } + const minutes = Math.max(1, Math.round(windowMs / 60_000)) + if (minutes < 60) return `${minutes}m` + const hours = Math.round(minutes / 60) + return hours < 48 ? `${hours}h` : `${Math.round(hours / 24)}d` +} + /** The value in force for one dimension, or nothing. */ export const selectedDimensionValue = ( search: AgentOverviewSearch, diff --git a/apps/web/src/lib/agent-sessions/use-agent-overview.ts b/apps/web/src/lib/agent-sessions/use-agent-overview.ts index 7a1750f74..b5c2c355d 100644 --- a/apps/web/src/lib/agent-sessions/use-agent-overview.ts +++ b/apps/web/src/lib/agent-sessions/use-agent-overview.ts @@ -21,7 +21,7 @@ import type { AiOverviewSummaryData, } from "@/api/warehouse/ai-agent-overview" import type { ListAiSessionsInput, listAiSessions } from "@/api/warehouse/ai-sessions" -import { chartBucketSeconds } from "@/components/infra/chart-utils" +import { smallMultipleBucketSeconds } from "@/components/infra/chart-utils" import { useRefreshableAtomValue } from "@/hooks/use-refreshable-atom-value" import type { Result } from "@/lib/effect-atom" import type { QueryAtomFailure } from "@/lib/services/atoms/warehouse-query-atoms" @@ -126,7 +126,9 @@ export function useAgentOverview( window: AgentOverviewWindow, ): AgentOverviewResults { const selection = useMemo(() => overviewSelection(search, window), [search, window]) - const bucketSeconds = chartBucketSeconds(window.startTime, window.endTime) + // The grid is nine ~104px-tall plots rather than one wide chart, so the + // buckets are cut at a width that reads at that size. + const bucketSeconds = smallMultipleBucketSeconds(window.startTime, window.endTime) const bucketed = { ...selection, bucketSeconds } const summary = useRefreshableAtomValue(aiOverviewSummaryResultAtom({ data: bucketed })) diff --git a/apps/web/src/routes/agent-sessions/overview.tsx b/apps/web/src/routes/agent-sessions/overview.tsx index 435fc9af2..d30e37def 100644 --- a/apps/web/src/routes/agent-sessions/overview.tsx +++ b/apps/web/src/routes/agent-sessions/overview.tsx @@ -12,7 +12,11 @@ import { QueryErrorState } from "@/components/common/query-error-state" import { DashboardLayout } from "@/components/layout/dashboard-layout" import { NotFoundError } from "@/components/route-error" import { PageRefreshProvider } from "@/components/time-range-picker/page-refresh-context" -import { TimeRangeSearchFields, applyTimeRangeSearch } from "@/components/time-range-picker/search" +import { + TimeRangeSearchFields, + applyTimeRangeSearch, + type TimeRangeSearch, +} from "@/components/time-range-picker/search" import { sessionTimeRangeSearchMiddleware } from "@/components/time-range-picker/session-time-range" import { TimeRangeHeaderControls } from "@/components/time-range-picker/time-range-header-controls" import { useEffectiveTimeRange } from "@/hooks/use-effective-time-range" @@ -24,6 +28,7 @@ import { EMPTY_OVERVIEW_FACETS, OverviewSearchFields, compareEnabled, + overviewWindowLabel, type AgentOverviewSearch, type OverviewFacets, } from "@/lib/agent-sessions/overview-search" @@ -90,7 +95,6 @@ function AgentOverviewPageContent() { ) => void headerControls: ReactNode }) { @@ -146,6 +148,10 @@ function AgentOverviewBody({ }), [search.startTime, search.endTime, search.timePreset], ) + // Read off the resolved window, not off the page's default preset: the + // resolver hands an absolute range straight back, so the default never + // named it and "prev 7d" beside a two-hour range would be a fiction. + const windowLabel = overviewWindowLabel(timeRange, windowMs.endMs - windowMs.startMs) // The selects' options come from the sessions facets — the same counted // lists the list page's sidebar uses, unfiltered so picking one model does @@ -215,7 +221,7 @@ function AgentOverviewBody({ // than re-derived for the axis. bucketSeconds: summary.bucketSeconds, windowMs, - windowLabel: preset, + windowLabel, compare: compareEnabled(search), })} facets={facets} @@ -224,7 +230,7 @@ function AgentOverviewBody({ duration: sessionsOf(results.topSessions.duration), errored: sessionsOf(results.topSessions.errored), }} - windowLabel={preset} + windowLabel={windowLabel} timeRange={timeRange} headerControls={headerControls} waiting={result.waiting} From 04f7335908983c71e9ca5664bc56334439e9af72 Mon Sep 17 00:00:00 2001 From: JeremyFunk Date: Fri, 11 Sep 2026 07:48:43 +0200 Subject: [PATCH 08/16] fix(agent-sessions): overview axis gutters and duration formatting The nine small multiples reserved a fixed 32px gutter, so any y-axis label wider than four characters was drawn off the canvas's left edge and lost its leading character: $0.50 read as 0.50, 150.0K as 50.0K. The gutter is now measured from the widest top-tick label in the grid and shared by all nine, so the plots still line up column to column. Durations on the board went through formatLatency (42.00s, 2.5min) while the Sessions list reads them in clock units. The page's own wrapper now delegates to formatSessionDuration above a minute and keeps a tenth of a second below one, and the STARTED column matches the list's relative-time rendering. --- .../api/warehouse/ai-agent-overview.test.ts | 4 +- .../overview/overview-small-multiple.tsx | 24 +++-- .../overview/overview-top-sessions.tsx | 8 +- .../overview/overview-trends.tsx | 16 +++- apps/web/src/lab/agent-overview-fixture.ts | 40 +++----- apps/web/src/lab/agent-overview-lab.tsx | 6 +- .../agent-sessions/overview-analytics.test.ts | 44 ++++----- .../lib/agent-sessions/overview-analytics.ts | 95 ++++++------------- .../overview-chart-specs.test.ts | 40 +++++++- .../agent-sessions/overview-chart-specs.ts | 41 ++++++++ .../src/lib/agent-sessions/overview-search.ts | 18 +--- .../lib/agent-sessions/use-agent-overview.ts | 6 +- 12 files changed, 180 insertions(+), 162 deletions(-) diff --git a/apps/web/src/api/warehouse/ai-agent-overview.test.ts b/apps/web/src/api/warehouse/ai-agent-overview.test.ts index 4e319c793..22085b882 100644 --- a/apps/web/src/api/warehouse/ai-agent-overview.test.ts +++ b/apps/web/src/api/warehouse/ai-agent-overview.test.ts @@ -61,9 +61,7 @@ describe("mapOverviewSeries", () => { describe("mapOverviewBreakdown", () => { it("keeps both windows per key, and `''` as a real key", () => { - const [row] = mapOverviewBreakdown([ - { key: "", current: wire(), previous: wire({ sessions: 4 }) }, - ]) + const [row] = mapOverviewBreakdown([{ key: "", current: wire(), previous: wire({ sessions: 4 }) }]) expect(row.key).toBe("") expect(row.current.sessions).toBe(10) expect(row.previous.sessions).toBe(4) diff --git a/apps/web/src/components/agent-sessions/overview/overview-small-multiple.tsx b/apps/web/src/components/agent-sessions/overview/overview-small-multiple.tsx index 555290bf1..e0f60605a 100644 --- a/apps/web/src/components/agent-sessions/overview/overview-small-multiple.tsx +++ b/apps/web/src/components/agent-sessions/overview/overview-small-multiple.tsx @@ -19,12 +19,15 @@ import { import type { makeBucketAxis } from "@/components/infra/chart-utils" import { LinkedCursorOverlay, linkedCursorChartProps } from "@/hooks/use-linked-cursor" -import type { OverviewPlotRow, OverviewPlotSpec } from "@/lib/agent-sessions/overview-chart-specs" +import { + OVERVIEW_TICK_PADDING, + overviewAxisTick, + type OverviewPlotRow, + type OverviewPlotSpec, +} from "@/lib/agent-sessions/overview-chart-specs" /** The plot box, x-axis labels included — the design's 86px svg over its ticks. */ export const OVERVIEW_PLOT_HEIGHT = 104 -/** Wide enough for `100%` and `$0.40`, narrow enough to leave the plot room. */ -const Y_AXIS_WIDTH = 32 const STROKE_WIDTH = 1.5 const GHOST_STROKE_WIDTH = 1.2 /** A stack layer is read by its area, so it is nearly opaque. */ @@ -39,6 +42,9 @@ export interface OverviewSmallMultipleProps { spec: OverviewPlotSpec /** Built once for the whole grid, so all nine agree on where an instant sits. */ axis: ReturnType + /** The y-axis gutter, sized by `overviewAxisGutter` over every spec in the + * grid rather than this one alone, so the nine plots line up. */ + gutter: number } /** @@ -48,7 +54,7 @@ export interface OverviewSmallMultipleProps { * differ (lines, a stack, a spread) but the chrome must not, because the grid is * read across as much as down. */ -export function OverviewSmallMultiple({ chartId, title, spec, axis }: OverviewSmallMultipleProps) { +export function OverviewSmallMultiple({ chartId, title, spec, axis, gutter }: OverviewSmallMultipleProps) { const chromeColors = usePlotChromeColors() const focusStore = useMemo(() => createTooltipFocusStore(), []) @@ -121,25 +127,23 @@ export function OverviewSmallMultiple({ chartId, title, spec, axis }: OverviewSm line: false, ticks: { size: 0, - padding: 6, + padding: OVERVIEW_TICK_PADDING, // Two labels, the extremes — nine charts of laddered ticks is a // wall of digits, and the question here is shape. values: [0, spec.yMax], - // A duration or a cost renders zero as an em dash, which is right - // for a headline and wrong for an axis floor. - format: (value: number) => (value === 0 ? "0" : spec.format(value)), + format: (value: number) => overviewAxisTick(spec, value), }, }, }, }, // The top tick sits on the highest plotted value, so the margin is what // keeps its label — and the peak under it — inside the frame. - margin: { left: Y_AXIS_WIDTH, right: 6, top: 8 }, + margin: { left: gutter, right: 6, top: 8 }, focus: "group-x", focusRing: false, tooltip: cursorTooltip(focusStore.anchor), }) - }, [spec, axis, chromeColors, focusStore]) + }, [spec, axis, chromeColors, focusStore, gutter]) if (spec.rows.length === 0) { return No data in this range. diff --git a/apps/web/src/components/agent-sessions/overview/overview-top-sessions.tsx b/apps/web/src/components/agent-sessions/overview/overview-top-sessions.tsx index 2d7be5d8e..88429a22d 100644 --- a/apps/web/src/components/agent-sessions/overview/overview-top-sessions.tsx +++ b/apps/web/src/components/agent-sessions/overview/overview-top-sessions.tsx @@ -2,12 +2,13 @@ import { useState } from "react" import { Link } from "@tanstack/react-router" import { formatNumber } from "@maple/ui/lib/format" -import { formatRelativeShort } from "@maple/ui/lib/time-format" +import { formatRelativeTimeOrDate } from "@maple/ui/lib/time-format" import { cn } from "@maple/ui/lib/utils" import { ExternalLinkIcon } from "@/components/icons" import type { AgentSessionsSearchState } from "@/components/agent-sessions/agent-sessions-filter-inputs" import type { AgentSessionRow } from "@/components/agent-sessions/agent-sessions-list" +import { useTimezonePreference } from "@/hooks/use-timezone-preference" import { formatOverviewCount, formatOverviewDuration } from "@/lib/agent-sessions/overview-analytics" import { formatCost } from "@/lib/agent-sessions/session-summary" import { sessionLinkWindow } from "@/lib/agent-sessions/session-window" @@ -59,6 +60,7 @@ export function OverviewTopSessions({ sessionsSearch, waiting = false, }: OverviewTopSessionsProps) { + const { effectiveTimezone } = useTimezonePreference() const [active, setActive] = useState("cost") const rows = sessions[active] @@ -165,7 +167,9 @@ export function OverviewTopSessions({ {formatOverviewDuration(row.durationMs)} - {formatRelativeShort(row.startTime)} + {/* The Sessions list's own reading: relative inside the week, + an absolute date once "23d ago" stops being the easier one. */} + {formatRelativeTimeOrDate(row.startTime, undefined, effectiveTimezone)} [id, buildOverviewPlotSpec(id, input)])) }, [series, previousSeries, modelMix]) + // One gutter for all nine, so the plots line up column to column however wide + // this window's labels turn out to be. + const gutter = useMemo(() => overviewAxisGutter(specs.values()), [specs]) + return (
@@ -111,7 +116,13 @@ export function OverviewTrends({ >
{charts.map((chart) => ( - + ))}
@@ -168,10 +179,12 @@ function ChartCell({ chart, spec, axis, + gutter, }: { chart: OverviewChartSummary spec: OverviewPlotSpec | undefined axis: ReturnType + gutter: number }) { return (
@@ -210,6 +223,7 @@ function ChartCell({ title={chart.title} spec={spec} axis={axis} + gutter={gutter} />
diff --git a/apps/web/src/lab/agent-overview-fixture.ts b/apps/web/src/lab/agent-overview-fixture.ts index 871568d45..e929efef4 100644 --- a/apps/web/src/lab/agent-overview-fixture.ts +++ b/apps/web/src/lab/agent-overview-fixture.ts @@ -413,8 +413,7 @@ const SPECS = { buckets: 24, current: { ...HEALTHY, sessions: 52 }, previous: { ...HEALTHY, sessions: 51 }, - regressedAt: (bucketMs: number) => - new Date(bucketMs).getUTCHours() >= OVERVIEW_REGRESSION_HOUR_UTC, + regressedAt: (bucketMs: number) => new Date(bucketMs).getUTCHours() >= OVERVIEW_REGRESSION_HOUR_UTC, }, } satisfies Record @@ -437,29 +436,21 @@ export function buildOverviewFixture(scenario: OverviewScenario, nowMs: number): return { bucket, index, regressed } }) - const series: ReadonlyArray = buckets.map( - ({ bucket, index, regressed }) => ({ - bucket, - ...measuresOf( - scaleRates( - regressed ? { ...REGRESSED, sessions: spec.current.sessions } : spec.current, - index, - ), - ), - }), - ) - const previousSeries: ReadonlyArray = buckets.map( - ({ bucket, index }) => ({ - bucket: bucket - windowMs, - ...measuresOf(scaleRates(spec.previous, index + 3)), - }), - ) + const series: ReadonlyArray = buckets.map(({ bucket, index, regressed }) => ({ + bucket, + ...measuresOf( + scaleRates(regressed ? { ...REGRESSED, sessions: spec.current.sessions } : spec.current, index), + ), + })) + const previousSeries: ReadonlyArray = buckets.map(({ bucket, index }) => ({ + bucket: bucket - windowMs, + ...measuresOf(scaleRates(spec.previous, index + 3)), + })) // The regressed scenario's window mixes both shapes, so the tiles read the // whole window while the grid shows where it turned. const regressedShare = buckets.filter((b) => b.regressed).length / spec.buckets - const blend = (healthy: number, bad: number) => - healthy * (1 - regressedShare) + bad * regressedShare + const blend = (healthy: number, bad: number) => healthy * (1 - regressedShare) + bad * regressedShare const currentRates: SessionRates = { ...spec.current, errorRate: blend(spec.current.errorRate, REGRESSED.errorRate), @@ -485,12 +476,7 @@ export function buildOverviewFixture(scenario: OverviewScenario, nowMs: number): const breakdownCurrent: SessionRates = { ...currentRates, sessions: current.sessions } const breakdownPrevious: SessionRates = { ...spec.previous, sessions: previous.sessions } const breakdowns = OVERVIEW_DIMENSIONS.map((dimension) => { - const entries = breakdownEntries( - dimension, - breakdownCurrent, - breakdownPrevious, - regressedShare > 0, - ) + const entries = breakdownEntries(dimension, breakdownCurrent, breakdownPrevious, regressedShare > 0) return { dimension, entries, totalKeys: entries.length + (dimension === "tool" ? 9 : 4) } }) diff --git a/apps/web/src/lab/agent-overview-lab.tsx b/apps/web/src/lab/agent-overview-lab.tsx index 1328c2359..26eaf31dc 100644 --- a/apps/web/src/lab/agent-overview-lab.tsx +++ b/apps/web/src/lab/agent-overview-lab.tsx @@ -4,11 +4,7 @@ import { AgentOverviewView } from "@/components/agent-sessions/overview/agent-ov import { buildAgentOverviewData } from "@/lib/agent-sessions/overview-analytics" import { compareEnabled, type AgentOverviewSearch } from "@/lib/agent-sessions/overview-search" -import { - OVERVIEW_SCENARIOS, - buildOverviewFixture, - type OverviewScenario, -} from "./agent-overview-fixture" +import { OVERVIEW_SCENARIOS, buildOverviewFixture, type OverviewScenario } from "./agent-overview-fixture" /** * The overview board without a warehouse behind it. diff --git a/apps/web/src/lib/agent-sessions/overview-analytics.test.ts b/apps/web/src/lib/agent-sessions/overview-analytics.test.ts index 501ec0bbe..a5900780b 100644 --- a/apps/web/src/lib/agent-sessions/overview-analytics.test.ts +++ b/apps/web/src/lib/agent-sessions/overview-analytics.test.ts @@ -61,9 +61,7 @@ describe("derivations", () => { it("divides the priced share by the netted volume the server priced", () => { // 47 of 50 netted calls carried a price. The 59 spans behind them did not // each need one, and dividing by those would read as 80% coverage. - expect(pricedShare(measures({ llmCalls: 50, llmCallSpans: 59, pricedLlmCalls: 47 }))).toBe( - 0.94, - ) + expect(pricedShare(measures({ llmCalls: 50, llmCallSpans: 59, pricedLlmCalls: 47 }))).toBe(0.94) }) }) @@ -135,10 +133,14 @@ describe("overviewDelta", () => { it("moves a duration by a duration and still reports its percent", () => { const delta = overviewDelta(96_000, 227_000, { unit: "duration", riseIs: "bad" }) expect(delta?.absolute).toBe(131_000) - expect(delta?.text).toBe("+2.2min") + expect(delta?.text).toBe("+2m 11s") expect(delta?.percent).toBeCloseTo(1.3646, 3) }) + it("keeps a sub-minute move in seconds rather than rounding it to a clock", () => { + expect(overviewDelta(96_000, 150_580, { unit: "duration", riseIs: "bad" })?.text).toBe("+54.6s") + }) + it("signs a fall", () => { expect(overviewDelta(200, 100, { unit: "percent", riseIs: "neutral" })?.text).toBe("-50%") }) @@ -159,19 +161,16 @@ describe("formatters", () => { expect(formatOverviewDuration(0)).toBe("—") }) + it("reads a duration in the clock units the Sessions list uses", () => { + expect(formatOverviewDuration(58_200)).toBe("58.2s") + expect(formatOverviewDuration(724_000)).toBe("12m 4s") + expect(formatOverviewDuration(5_400_000)).toBe("1h 30m") + }) + it("names every width the grid's buckets are actually cut at", () => { // The ladder `smallMultipleBucketSeconds` snaps to, one unit each. const ladder = [300, 900, 1_800, 3_600, 10_800, 21_600, 43_200, 86_400] - expect(ladder.map(bucketWidthLabel)).toEqual([ - "5m", - "15m", - "30m", - "1h", - "3h", - "6h", - "12h", - "1d", - ]) + expect(ladder.map(bucketWidthLabel)).toEqual(["5m", "15m", "30m", "1h", "3h", "6h", "12h", "1d"]) }) }) @@ -191,7 +190,9 @@ describe("buildOverviewTiles", () => { const previous = measures({ sessions: 80, erroredSessions: 4, cost: 40, tokens: 800 }) it("builds seven tiles in the strip's order", () => { - expect(buildOverviewTiles(current, previous, { compare: true, windowLabel: "7d" }).map((t) => t.id)).toEqual([ + expect( + buildOverviewTiles(current, previous, { compare: true, windowLabel: "7d" }).map((t) => t.id), + ).toEqual([ "sessions", "cost", "costPerSession", @@ -278,14 +279,7 @@ describe("buildModelMix", () => { it("keeps the five busiest models and folds the tail into one grey band", () => { const mix = buildModelMix(rows) - expect(mix.models).toEqual([ - "model-1", - "model-2", - "model-3", - "model-4", - "model-5", - "other", - ]) + expect(mix.models).toEqual(["model-1", "model-2", "model-3", "model-4", "model-5", "other"]) }) it("stacks each bucket to one", () => { @@ -418,9 +412,7 @@ describe("buildMovers", () => { describe("overviewScopeSummary", () => { it("names the three populations the rest of the board divides by", () => { - expect( - overviewScopeSummary(measures({ sessions: 1_284, llmCalls: 10_842, toolCalls: 8_101 })), - ).toBe( + expect(overviewScopeSummary(measures({ sessions: 1_284, llmCalls: 10_842, toolCalls: 8_101 }))).toBe( `${(1284).toLocaleString()} sessions · ${(10842).toLocaleString()} LLM calls · ${(8101).toLocaleString()} tool calls`, ) }) diff --git a/apps/web/src/lib/agent-sessions/overview-analytics.ts b/apps/web/src/lib/agent-sessions/overview-analytics.ts index 82914ef5d..bae500bee 100644 --- a/apps/web/src/lib/agent-sessions/overview-analytics.ts +++ b/apps/web/src/lib/agent-sessions/overview-analytics.ts @@ -18,7 +18,8 @@ // percentage POINTS — 2% to 26% is "up 24 points", not "up 1200%" — a ratio // moves in percent, and a duration moves by a duration. -import { formatErrorRate, formatLatency, formatNumber, formatPercent } from "@maple/ui/lib/format" +import { formatErrorRate, formatNumber, formatPercent } from "@maple/ui/lib/format" +import { formatSessionDuration } from "@maple/ui/lib/replay-format" import { formatCost } from "./session-summary" import { OVERVIEW_DIMENSIONS, type OverviewDimension } from "./overview-search" @@ -108,16 +109,13 @@ export interface OverviewModelMixRow { const ratio = (numerator: number, denominator: number): number => denominator > 0 ? numerator / denominator : 0 -export const sessionErrorRate = (m: OverviewMeasures): number => - ratio(m.erroredSessions, m.sessions) +export const sessionErrorRate = (m: OverviewMeasures): number => ratio(m.erroredSessions, m.sessions) /** `erroredLlmCalls / llmCallSpans` and never `/ llmCalls`: the two populations * differ by every mirror and wrapper the netting collapses. */ -export const llmErrorRate = (m: OverviewMeasures): number => - ratio(m.erroredLlmCalls, m.llmCallSpans) +export const llmErrorRate = (m: OverviewMeasures): number => ratio(m.erroredLlmCalls, m.llmCallSpans) -export const toolErrorRate = (m: OverviewMeasures): number => - ratio(m.erroredToolCalls, m.toolCalls) +export const toolErrorRate = (m: OverviewMeasures): number => ratio(m.erroredToolCalls, m.toolCalls) export const costPerSession = (m: OverviewMeasures): number => ratio(m.cost, m.sessions) export const tokensPerSession = (m: OverviewMeasures): number => ratio(m.tokens, m.sessions) @@ -143,13 +141,7 @@ export const pricedShare = (m: OverviewMeasures): number => ratio(m.pricedLlmCal * Token bands * -----------------------------------------------------------------------------------------------*/ -export const OVERVIEW_TOKEN_BANDS = [ - "input", - "cacheRead", - "cacheWrite", - "output", - "reasoning", -] as const +export const OVERVIEW_TOKEN_BANDS = ["input", "cacheRead", "cacheWrite", "output", "reasoning"] as const export type OverviewTokenBand = (typeof OVERVIEW_TOKEN_BANDS)[number] /** @@ -160,10 +152,7 @@ export type OverviewTokenBand = (typeof OVERVIEW_TOKEN_BANDS)[number] export const OVERVIEW_TOKEN_FALLBACK_BAND = "total" export type OverviewTokenBandKey = OverviewTokenBand | typeof OVERVIEW_TOKEN_FALLBACK_BAND -export const OVERVIEW_TOKEN_BAND_KEYS = [ - ...OVERVIEW_TOKEN_BANDS, - OVERVIEW_TOKEN_FALLBACK_BAND, -] as const +export const OVERVIEW_TOKEN_BAND_KEYS = [...OVERVIEW_TOKEN_BANDS, OVERVIEW_TOKEN_FALLBACK_BAND] as const export const OVERVIEW_TOKEN_BAND_LABEL = { input: "input", @@ -186,8 +175,7 @@ const emptyBands = (): Record => ({ /** Raw token counts per band, with the fallback applied. */ export function tokenBandValues(m: OverviewMeasures): Record { const bands = emptyBands() - const split = - m.inputTokens + m.cacheReadTokens + m.cacheWriteTokens + m.outputTokens + m.reasoningTokens + const split = m.inputTokens + m.cacheReadTokens + m.cacheWriteTokens + m.outputTokens + m.reasoningTokens if (split === 0) { bands.total = m.tokens return bands @@ -251,8 +239,7 @@ export function overviewDelta( ): OverviewDelta | null { if (!Number.isFinite(before) || !Number.isFinite(after)) return null const absolute = after - before - const flatAt = - options.unit === "points" ? FLAT_POINTS / 100 : options.unit === "duration" ? FLAT_MS : 0 + const flatAt = options.unit === "points" ? FLAT_POINTS / 100 : options.unit === "duration" ? FLAT_MS : 0 const percent = before === 0 ? null : absolute / before if (options.unit === "percent" && percent === null) return null @@ -298,7 +285,7 @@ export function overviewDelta( percent, pp: null, direction, - text: direction === "flat" ? "0s" : signed(absolute, formatLatency(Math.abs(absolute))), + text: direction === "flat" ? "0s" : signed(absolute, formatOverviewDuration(Math.abs(absolute))), tone, } } @@ -338,10 +325,19 @@ export function formatPerSession(value: number): string { return value >= 100 ? formatOverviewCount(value) : value.toFixed(1) } -/** A duration a session or a call took. Zero means "nothing measured". */ +/** + * A duration a session took, in the clock units the Sessions list reads them + * in — `2m 30s`, `1h 4m` — so a session's row there and its cell here are the + * same string. Zero means "nothing measured". + * + * The shared formatter starts at whole seconds, which is a reading on a list + * row and a rounding on a delta: `+55s` and `+54.6s` are the same move only + * until you compare two of them. The tenth therefore survives until the minutes + * arrive to carry the magnitude instead. + */ export function formatOverviewDuration(ms: number): string { if (!Number.isFinite(ms) || ms <= 0) return "—" - return formatLatency(ms) + return ms < 60_000 ? `${(ms / 1000).toFixed(1)}s` : formatSessionDuration(ms) } /** `''` is a real breakdown key, shown as unattributed rather than hidden. */ @@ -403,12 +399,7 @@ export function buildOverviewTiles( previous: OverviewMeasures, options: { compare: boolean; windowLabel: string }, ): ReadonlyArray { - const delta = ( - before: number, - after: number, - unit: DeltaUnit, - riseIs: DeltaTone, - ): OverviewDelta | null => + const delta = (before: number, after: number, unit: DeltaUnit, riseIs: DeltaTone): OverviewDelta | null => options.compare ? overviewDelta(before, after, { unit, riseIs }) : null return [ @@ -456,24 +447,14 @@ export function buildOverviewTiles( id: "toolCallsPerSession", label: "Tool calls / sess", value: formatPerSession(toolCallsPerSession(current)), - delta: delta( - toolCallsPerSession(previous), - toolCallsPerSession(current), - "percent", - "bad", - ), + delta: delta(toolCallsPerSession(previous), toolCallsPerSession(current), "percent", "bad"), sub: `${formatOverviewCount(current.toolCalls)} calls`, }, { id: "durationP95", label: "Duration p95", value: formatOverviewDuration(current.sessionDurationP95Ms), - delta: delta( - previous.sessionDurationP95Ms, - current.sessionDurationP95Ms, - "duration", - "bad", - ), + delta: delta(previous.sessionDurationP95Ms, current.sessionDurationP95Ms, "duration", "bad"), sub: `p50 ${formatOverviewDuration(current.sessionDurationP50Ms)}`, }, ] @@ -905,12 +886,7 @@ export function buildOverviewCharts( previous: OverviewMeasures, options: { compare: boolean; modelMix: OverviewModelMix }, ): ReadonlyArray { - const delta = ( - before: number, - after: number, - unit: DeltaUnit, - riseIs: DeltaTone, - ): OverviewDelta | null => + const delta = (before: number, after: number, unit: DeltaUnit, riseIs: DeltaTone): OverviewDelta | null => options.compare ? overviewDelta(before, after, { unit, riseIs }) : null const leadModel = options.modelMix.models[0] @@ -946,12 +922,7 @@ export function buildOverviewCharts( title: "Tool calls per session", unit: "calls / session", value: formatPerSession(toolCallsPerSession(current)), - delta: delta( - toolCallsPerSession(previous), - toolCallsPerSession(current), - "percent", - "bad", - ), + delta: delta(toolCallsPerSession(previous), toolCallsPerSession(current), "percent", "bad"), }, { id: "errorRate", @@ -965,24 +936,14 @@ export function buildOverviewCharts( title: "Session duration", unit: "p50 with p50–p95 band", value: formatOverviewDuration(current.sessionDurationP95Ms), - delta: delta( - previous.sessionDurationP95Ms, - current.sessionDurationP95Ms, - "duration", - "bad", - ), + delta: delta(previous.sessionDurationP95Ms, current.sessionDurationP95Ms, "duration", "bad"), }, { id: "llmCallsPerSession", title: "LLM calls per session", unit: "calls / session", value: formatPerSession(llmCallsPerSession(current)), - delta: delta( - llmCallsPerSession(previous), - llmCallsPerSession(current), - "percent", - "neutral", - ), + delta: delta(llmCallsPerSession(previous), llmCallsPerSession(current), "percent", "neutral"), }, { id: "modelMix", diff --git a/apps/web/src/lib/agent-sessions/overview-chart-specs.test.ts b/apps/web/src/lib/agent-sessions/overview-chart-specs.test.ts index 3b5768007..c5f9bd787 100644 --- a/apps/web/src/lib/agent-sessions/overview-chart-specs.test.ts +++ b/apps/web/src/lib/agent-sessions/overview-chart-specs.test.ts @@ -9,7 +9,14 @@ import { type OverviewMeasures, type OverviewModelMix, } from "./overview-analytics" -import { buildOverviewPlotSpec, type OverviewPlotInput } from "./overview-chart-specs" +import { + OVERVIEW_TICK_PADDING, + buildOverviewPlotSpec, + overviewAxisGutter, + overviewAxisTick, + type OverviewPlotInput, + type OverviewPlotSpec, +} from "./overview-chart-specs" const HOUR = 3_600_000 const START = Date.UTC(2026, 8, 10, 0, 0, 0) @@ -106,3 +113,34 @@ describe("buildOverviewPlotSpec", () => { expect(spec.rows[0]?.p95).toBe(4_000) }) }) + +describe("overviewAxisGutter", () => { + const axis = (yMax: number, format: (value: number) => string): OverviewPlotSpec => + ({ yMax, format }) as OverviewPlotSpec + + it("prints the floor as a digit and the top through the chart's formatter", () => { + const spec = axis(0.5, (value) => `$${value.toFixed(2)}`) + expect(overviewAxisTick(spec, 0)).toBe("0") + expect(overviewAxisTick(spec, spec.yMax)).toBe("$0.50") + }) + + it("holds the design's width for the labels it was drawn around", () => { + expect(overviewAxisGutter([axis(1, () => "100%"), axis(8, () => "8.0")])).toBe(32) + }) + + it("widens to the widest label in the grid, and gives every plot the same one", () => { + const narrow = axis(1, () => "100%") + const wide = axis(150_000, () => "150.0K") + expect(overviewAxisGutter([narrow])).toBe(32) + expect(overviewAxisGutter([narrow, wide])).toBe(overviewAxisGutter([wide])) + expect(overviewAxisGutter([narrow, wide])).toBeGreaterThan(overviewAxisGutter([narrow])) + }) + + it("leaves every label room to sit right-aligned off the plot", () => { + // 6.02px per character at the 10px mono tick size, plus the tick padding. + for (const label of ["$0.50", "80.0K", "10.0%", "2m 30s", "100%"]) { + const gutter = overviewAxisGutter([axis(1, () => label)]) + expect(gutter - OVERVIEW_TICK_PADDING).toBeGreaterThanOrEqual(label.length * 6.02) + } + }) +}) diff --git a/apps/web/src/lib/agent-sessions/overview-chart-specs.ts b/apps/web/src/lib/agent-sessions/overview-chart-specs.ts index 4a4da17fc..89b6df528 100644 --- a/apps/web/src/lib/agent-sessions/overview-chart-specs.ts +++ b/apps/web/src/lib/agent-sessions/overview-chart-specs.ts @@ -67,6 +67,47 @@ export interface OverviewPlotInput { readonly modelMix: OverviewModelMix } +/* ------------------------------------------------------------------------------------------------- + * The y axis's two ticks, and the gutter they need + * -----------------------------------------------------------------------------------------------*/ + +/** The gap a tick label keeps from the plot it labels. */ +export const OVERVIEW_TICK_PADDING = 6 +/** One character of the 10px mono tick label, rounded up from the 0.6em advance + * the face actually uses — the axis is painted onto a canvas, so the gutter has + * to be decided before there is anything to measure. */ +const TICK_CHAR_WIDTH = 6.2 +/** Four characters — `100%`, `4.0%`, `10.0` — and the width the design drew. */ +const MIN_GUTTER = 32 + +/** + * A tick as the axis prints it. A duration or a cost renders zero as an em + * dash, which is right for a headline and wrong for an axis floor. + */ +export const overviewAxisTick = (spec: OverviewPlotSpec, value: number): string => + value === 0 ? "0" : spec.format(value) + +/** + * The left gutter the nine plots share, wide enough for the widest label any of + * them will print. + * + * A fixed gutter only ever fits the formatter it was written for: `100%` and + * `150.0K` are the same axis and not the same width, and a label that does not + * fit is drawn straight off the canvas's left edge. Right-aligned text overflows + * leftwards, so what is lost is the leading character — the one carrying the + * magnitude, which leaves `$0.50` reading as `0.50`. + * + * One width for the whole grid rather than one per chart: the board is read + * across as much as down, and a plot starting further right than the one beside + * it reads as a different instrument. Only the top tick is measured; the floor + * is always `0`. + */ +export function overviewAxisGutter(specs: Iterable): number { + let widest = 0 + for (const spec of specs) widest = Math.max(widest, overviewAxisTick(spec, spec.yMax).length) + return Math.max(MIN_GUTTER, Math.ceil(widest * TICK_CHAR_WIDTH) + OVERVIEW_TICK_PADDING) +} + /* ------------------------------------------------------------------------------------------------- * Colours * -----------------------------------------------------------------------------------------------*/ diff --git a/apps/web/src/lib/agent-sessions/overview-search.ts b/apps/web/src/lib/agent-sessions/overview-search.ts index 6f58a85f7..63800f5f6 100644 --- a/apps/web/src/lib/agent-sessions/overview-search.ts +++ b/apps/web/src/lib/agent-sessions/overview-search.ts @@ -24,14 +24,7 @@ const BooleanParam = Schema.optional(Schema.Union([Schema.Boolean, BooleanFromSt * the same string on purpose; {@link overviewApiDimension} is the one place * the rename happens. */ -export const OVERVIEW_DIMENSIONS = [ - "model", - "agent", - "service", - "framework", - "environment", - "tool", -] as const +export const OVERVIEW_DIMENSIONS = ["model", "agent", "service", "framework", "environment", "tool"] as const export type OverviewDimension = (typeof OVERVIEW_DIMENSIONS)[number] /** The dimension as the breakdown endpoint spells it. */ @@ -126,9 +119,7 @@ export interface OverviewFilterChip { } /** The active dimension filters, in the dimensions' own order — the scope row. */ -export function activeOverviewFilters( - search: AgentOverviewSearch, -): ReadonlyArray { +export function activeOverviewFilters(search: AgentOverviewSearch): ReadonlyArray { return OVERVIEW_DIMENSIONS.flatMap((dimension) => { const value = search[dimension] return value === undefined ? [] : [{ dimension, value }] @@ -160,10 +151,7 @@ export function toggleOverviewFilter( dimension: OverviewDimension, key: string, ): Partial { - return overviewFilterPatch( - dimension, - key === "" || search[dimension] === key ? undefined : key, - ) + return overviewFilterPatch(dimension, key === "" || search[dimension] === key ? undefined : key) } /** diff --git a/apps/web/src/lib/agent-sessions/use-agent-overview.ts b/apps/web/src/lib/agent-sessions/use-agent-overview.ts index b5c2c355d..6f95b4916 100644 --- a/apps/web/src/lib/agent-sessions/use-agent-overview.ts +++ b/apps/web/src/lib/agent-sessions/use-agent-overview.ts @@ -32,11 +32,7 @@ import { listAiSessionsResultAtom, } from "@/lib/services/atoms/warehouse-query-atoms" -import { - overviewApiDimension, - type AgentOverviewSearch, - type OverviewDimension, -} from "./overview-search" +import { overviewApiDimension, type AgentOverviewSearch, type OverviewDimension } from "./overview-search" export interface AgentOverviewWindow { readonly startTime: string From 4ee4ff020ac3063499ea7c85a154e1a6fb83484f Mon Sep 17 00:00:00 2001 From: JeremyFunk Date: Fri, 11 Sep 2026 07:51:16 +0200 Subject: [PATCH 09/16] chore(agent-sessions): keep the overview token band label module-private --- apps/web/src/lib/agent-sessions/overview-analytics.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/web/src/lib/agent-sessions/overview-analytics.ts b/apps/web/src/lib/agent-sessions/overview-analytics.ts index bae500bee..d1a49c761 100644 --- a/apps/web/src/lib/agent-sessions/overview-analytics.ts +++ b/apps/web/src/lib/agent-sessions/overview-analytics.ts @@ -154,7 +154,7 @@ export type OverviewTokenBandKey = OverviewTokenBand | typeof OVERVIEW_TOKEN_FAL export const OVERVIEW_TOKEN_BAND_KEYS = [...OVERVIEW_TOKEN_BANDS, OVERVIEW_TOKEN_FALLBACK_BAND] as const -export const OVERVIEW_TOKEN_BAND_LABEL = { +const OVERVIEW_TOKEN_BAND_LABEL = { input: "input", cacheRead: "cache read", cacheWrite: "cache write", From e2fcd1f9678d36ffb5156946a5358131ab7fbb33 Mon Sep 17 00:00:00 2001 From: JeremyFunk Date: Fri, 11 Sep 2026 08:07:54 +0200 Subject: [PATCH 10/16] fix(agent-sessions): overview type errors MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - `withinWindow` takes a `CH.Expr`: the index's `Timestamp` is the string flavour every Maple warehouse timestamp is, which is what `param.dateTimeString` compares against. SQL is unchanged. - the facet select normalises the `null` Base UI reports for a cleared selection onto the `undefined` the toolbar spells "no filter" with. - a link into the Sessions list carries `AgentSessionsLinkSearch`, whose arrays are the mutable flavour the route's search schema declares. - the tab strip's `TabLink` takes a `TimeRangeSearch` — an interface has no index signature to satisfy a `Record`. - drop the unused token band label. --- .../overview/overview-filter-toolbar.tsx | 4 +++- .../overview/overview-top-sessions.tsx | 4 ++-- .../tools/agent-sessions-tabs.tsx | 3 ++- .../lib/agent-sessions/overview-analytics.ts | 9 --------- .../src/lib/agent-sessions/overview-search.ts | 20 +++++++++++++++++-- .../src/ai/ai-overview.ts | 7 +++++-- 6 files changed, 30 insertions(+), 17 deletions(-) diff --git a/apps/web/src/components/agent-sessions/overview/overview-filter-toolbar.tsx b/apps/web/src/components/agent-sessions/overview/overview-filter-toolbar.tsx index 093ade1b8..2c3d49b2d 100644 --- a/apps/web/src/components/agent-sessions/overview/overview-filter-toolbar.tsx +++ b/apps/web/src/components/agent-sessions/overview/overview-filter-toolbar.tsx @@ -121,7 +121,9 @@ function FacetSelect({ return (