From 3604ae834097aa3554651e6bf2886f70c88f1c1e Mon Sep 17 00:00:00 2001 From: JeremyFunk Date: Fri, 18 Sep 2026 01:35:18 +0200 Subject: [PATCH 1/3] feat(agent-sessions): grade the list's Errors cell as the Overview grades its checks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Errors column showed "N turn" / "N tool" counts, both read as red, so a session that retried a rate limit looked like one whose prompt outgrew the window. The Overview (#920) already splits failures from warnings by one rule — red when the run died on it or its kind always needs a fix, amber when it was survived — but only from the session's spans, which the list cannot read. Since migration 0032 `ai_trace_index` carries every field the classifier reads off a failed span (`ErrorType`, `StatusMessage`, `FailedToolCallResult`, `ToolName`, `VendorId`, `ResponseId`), so the page query now ships the session's deepest failed spans (the same filter `toolErrors`/`turnErrors` count on, capped at 100) plus which trace ends last and whether its turn failed. The backend classifies them with the same `classifyFailureSignal` and `failureSeverity` the detail page uses — `classifyFailure` is now that function over a span — and the row carries `failures`: label, count, severity, terminal. The cell is a red "N failures" chip and an amber "N warnings" chip; hovering either lists the breakdown by label (`context_length_exceeded`, `tool_error · run_tests ×3`, "ended the run") so a reader triages without opening the session. The column widens 100→140 so a chip stays on one line, and the responsive thresholds shift with it. `list_agent_sessions` prints the same breakdown, `!` on the red ones. Terminal is the Overview's verdict one trace deep: the last trace had a non-tool failure, and the failure it died on is that trace's last. Finish reasons, loops and stalls are not in the index and stay the detail page's; all are amber there unless terminal. Rows materialized before 0032 classify as a plain `error`. Verified against real ClickHouse: the materialization e2e decodes the tuple and classifies both fixture sessions, and the analyzer sweep passes. --- apps/ai/src/mcp/tools/list-agent-sessions.ts | 29 +++- .../agent-sessions-list.test.tsx | 39 ++++- .../agent-sessions/agent-sessions-list.tsx | 148 ++++++++++++----- apps/web/src/lab/agent-sessions-list-lab.tsx | 34 +++- apps/web/src/lab/agent-tools-fixture.ts | 155 ++++++++++------- bun.lock | 1 + packages/agent-sessions/src/failure-text.ts | 22 ++- .../agent-sessions/src/index-failures.test.ts | 156 +++++++++++++++++ packages/agent-sessions/src/index-failures.ts | 157 ++++++++++++++++++ packages/agent-sessions/src/index.ts | 1 + .../agent-sessions/src/session-findings.ts | 9 +- .../agent-sessions/src/session-summary.ts | 68 +++++--- packages/backend/package.json | 1 + .../services/ai-sessions/ai-session-reads.ts | 20 +++ ...dex-materialization.clickhouse.e2e.test.ts | 75 +++++++++ packages/domain/src/http/ai-sessions.ts | 37 +++++ .../src/__sql_baseline__/integrations.sql | 50 ++++-- .../src/ai/ai-sessions.test.ts | 80 ++++++++- .../src/ai/ai-sessions.ts | 118 ++++++++++--- .../query-engine-integrations/src/ai/index.ts | 1 + 20 files changed, 1025 insertions(+), 176 deletions(-) create mode 100644 packages/agent-sessions/src/index-failures.test.ts create mode 100644 packages/agent-sessions/src/index-failures.ts diff --git a/apps/ai/src/mcp/tools/list-agent-sessions.ts b/apps/ai/src/mcp/tools/list-agent-sessions.ts index 6758c5d73..d0727e061 100644 --- a/apps/ai/src/mcp/tools/list-agent-sessions.ts +++ b/apps/ai/src/mcp/tools/list-agent-sessions.ts @@ -22,6 +22,7 @@ import { RangeBound, } from "@maple/domain/http" import { formatCost } from "@maple/agent-sessions" +import type { AiSessionFailureSummary } from "@maple/domain/http" import { splitCsv } from "@maple/domain/where-clause" import { listAiSessions } from "@maple/backend/services/ai-sessions/ai-session-reads" import { warehouseReadToMcpHandlers } from "../lib/map-warehouse-error" @@ -150,7 +151,8 @@ export function registerListAgentSessionsTool(server: McpToolRegistrar) { formatDurationFromMs(session.durationMs), formatNumber(session.llmCalls), formatNumber(session.toolCalls), - `${session.errorSpanCount}/${session.toolErrorCount}/${session.turnErrorCount}`, + failuresCell(session.failures) ?? + `${session.errorSpanCount}/${session.toolErrorCount}/${session.turnErrorCount}`, formatNumber(session.totalTokens), session.cost > 0 ? formatCost(session.cost) : "—", truncate(session.models.join(", "), 40), @@ -160,7 +162,7 @@ export function registerListAgentSessionsTool(server: McpToolRegistrar) { const lines: string[] = [ `## AI agent sessions (showing ${offset + 1}–${offset + sessions.length})`, `Time range: ${st} — ${et}`, - `Every figure is over the session's AGENT spans; the app's own spans in the same traces are not counted. Errors are agent/tool/turn.`, + `Every figure is over the session's AGENT spans; the app's own spans in the same traces are not counted. Failures are by label, ×count; "!" marks one that needs a fix (the run died on it, or its kind always does), the rest were survived. A bare a/b/c is errored agent/tool/turn spans the index could not classify.`, ``, formatTable( [ @@ -171,7 +173,7 @@ export function registerListAgentSessionsTool(server: McpToolRegistrar) { "Duration", "LLM calls", "Tool calls", - "Errors", + "Failures", "Tokens", "Cost", "Models", @@ -180,7 +182,10 @@ export function registerListAgentSessionsTool(server: McpToolRegistrar) { rows, ), ...(sessions.length === limit - ? [``, `The page is full; more sessions may match — call again with offset=${offset + sessions.length}.`] + ? [ + ``, + `The page is full; more sessions may match — call again with offset=${offset + sessions.length}.`, + ] : []), formatNextSteps( sessions.slice(0, 3).map( @@ -197,3 +202,19 @@ export function registerListAgentSessionsTool(server: McpToolRegistrar) { }), ) } + +/** `!context_length_exceeded, tool_error · run_tests ×2` — the row's failures + * by label, a `!` on each one that needs a fix. `undefined` when the index + * classified none, and the raw counts say what it saw. */ +function failuresCell(failures: ReadonlyArray): string | undefined { + if (failures.length === 0) return undefined + return truncate( + failures + .map( + (failure) => + `${failure.severity === "failure" ? "!" : ""}${failure.label}${failure.count > 1 ? ` ×${failure.count}` : ""}`, + ) + .join(", "), + 80, + ) +} diff --git a/apps/web/src/components/agent-sessions/agent-sessions-list.test.tsx b/apps/web/src/components/agent-sessions/agent-sessions-list.test.tsx index 4ff284487..4d529b364 100644 --- a/apps/web/src/components/agent-sessions/agent-sessions-list.test.tsx +++ b/apps/web/src/components/agent-sessions/agent-sessions-list.test.tsx @@ -50,6 +50,7 @@ const session: AgentSessionRow = { errorSpanCount: 0, toolErrorCount: 0, turnErrorCount: 0, + failures: [], serviceNames: ["maple-slack-agent"], models: ["claude-sonnet-5"], agentNames: ["web-fetcher", "slack-agent"], @@ -114,7 +115,13 @@ describe("AgentSessionsList", () => { it("asks for the next page when the sentinel comes into view, but not while one is in flight", () => { const onReachEnd = vi.fn() const view = renderList( - , + , ) const first = MockIntersectionObserver.instances[0]! @@ -141,7 +148,7 @@ describe("AgentSessionsList", () => { expect(MockIntersectionObserver.instances).toHaveLength(0) }) - it("names the framework by its mark alone, and splits the failures by kind", () => { + it("names the framework by its mark alone, and splits the failures by severity", () => { const view = renderList( { errorSpanCount: 5, toolErrorCount: 2, turnErrorCount: 1, + failures: [ + { + kind: "contextExceeded", + label: "context_length_exceeded", + count: 1, + severity: "failure", + terminal: true, + }, + { + kind: "error", + label: "tool_error · run_tests", + tool: "run_tests", + count: 2, + severity: "anomaly", + terminal: false, + }, + ], }, ]} />, @@ -164,8 +188,8 @@ describe("AgentSessionsList", () => { // count and noun into fixed-width slots, so match on the whole chip's text. const chip = (label: string) => (_: string, element: Element | null) => element?.classList.contains("rounded-full") === true && element.textContent === label - expect(view.getAllByText(chip("2 tools"))).toHaveLength(1) - expect(view.getAllByText(chip("1 turn"))).toHaveLength(1) + expect(view.getAllByText(chip("1 failure"))).toHaveLength(1) + expect(view.getAllByText(chip("2 warnings"))).toHaveLength(1) expect(view.getByText("18.4k")).toBeTruthy() expect(view.getByText("maple-slack-agent")).toBeTruthy() }) @@ -199,7 +223,12 @@ describe("AgentSessionsList", () => { it("sorts through the column headers, marking the one the rows are in", () => { const onSortChange = vi.fn() const view = renderList( - , + , ) const cost = view.getByRole("columnheader", { name: "Cost" }) expect(cost.getAttribute("aria-sort")).toBe("descending") diff --git a/apps/web/src/components/agent-sessions/agent-sessions-list.tsx b/apps/web/src/components/agent-sessions/agent-sessions-list.tsx index 2df58be2b..3b635b08a 100644 --- a/apps/web/src/components/agent-sessions/agent-sessions-list.tsx +++ b/apps/web/src/components/agent-sessions/agent-sessions-list.tsx @@ -8,7 +8,7 @@ import { useTable, } from "@tanstack/react-table" import { useVirtualizer } from "@tanstack/react-virtual" -import type { AiSessionSortDir, AiSessionSortKey } from "@maple/domain/http" +import type { AiSessionFailureSummary, AiSessionSortDir, AiSessionSortKey } from "@maple/domain/http" import { Empty, EmptyDescription, EmptyHeader, EmptyMedia, EmptyTitle } from "@maple/ui/components/ui/empty" import { TableSkeleton } from "@maple/ui/components/ui/table-skeleton" @@ -53,6 +53,8 @@ export interface AgentSessionRow { readonly toolErrorCount: number /** Failed model calls and turn spans that failed on their own. */ readonly turnErrorCount: number + /** The failures by label, red first — the Errors cell's chips and hover. */ + readonly failures: ReadonlyArray readonly serviceNames: ReadonlyArray readonly models: ReadonlyArray readonly agentNames: ReadonlyArray @@ -104,7 +106,8 @@ const ROW_HEIGHT = 53 // No wrap: a two-word label ("LLM calls") breaking onto a second line would // make the whole header row taller. -const HEADER_CELL_CLASS = "h-10 whitespace-nowrap px-2 text-left align-middle font-medium text-muted-foreground" +const HEADER_CELL_CLASS = + "h-10 whitespace-nowrap px-2 text-left align-middle font-medium text-muted-foreground" /** * Column layout, shared by the real table and the loading skeleton so the two can't drift apart. @@ -114,11 +117,12 @@ const HEADER_CELL_CLASS = "h-10 whitespace-nowrap px-2 text-left align-middle fo * queries against `@container/page` (declared by PageLayout.Content), as on the traces table: the * app sidebar and the filter rail take width the viewport knows nothing about. * - * Budget: Errors (100) is always on — the triage signal, as Status is for traces. Every other column - * joins where Session keeps ≥200px beside it, the sortable measures first, so a width that shows a - * measure can also sort by it: Started (96) at 400, Duration (100) at 500, Cost (80) at 580, LLM - * calls (110) at 690, Tool calls (116) at 810 and Tokens (130) at 940. Services (170) at 1110 and - * Model (160) at 1270 come last — the filter rail answers both for the whole list. A sortable + * Budget: Errors (140, room for a "12 warnings" chip on one line) is always on — the triage signal, + * as Status is for traces. Every other column joins where Session keeps ≥200px beside it, the + * sortable measures first, so a width that shows a measure can also sort by it: Started (96) at + * 440, Duration (100) at 540, Cost (80) at 620, LLM calls (110) at 730, Tool calls (116) at 850 and + * Tokens (130) at 980. Services (170) at 1150 and Model (160) at 1310 come last — the filter rail + * answers both for the whole list. A sortable * column is at least as wide as its label and arrow at text-sm plus the cell's padding. */ interface SessionColumnLayout { @@ -138,57 +142,57 @@ const SESSION_COLUMNS: readonly SessionColumnLayout[] = [ header: "Services", width: 170, skeleton: "w-24", - responsive: "hidden @min-[1110px]/page:table-cell", + responsive: "hidden @min-[1150px]/page:table-cell", }, { id: "model", header: "Model", width: 160, skeleton: "w-24", - responsive: "hidden @min-[1270px]/page:table-cell", + responsive: "hidden @min-[1310px]/page:table-cell", }, { id: "durationMs", header: "Duration", width: 100, skeleton: "w-12", - responsive: "hidden @min-[500px]/page:table-cell", + responsive: "hidden @min-[540px]/page:table-cell", }, { id: "llmCalls", header: "LLM calls", width: 110, skeleton: "w-8", - responsive: "hidden @min-[690px]/page:table-cell", + responsive: "hidden @min-[730px]/page:table-cell", }, { id: "toolCalls", header: "Tool calls", width: 116, skeleton: "w-8", - responsive: "hidden @min-[810px]/page:table-cell", + responsive: "hidden @min-[850px]/page:table-cell", }, { id: "totalTokens", header: "Tokens", width: 130, skeleton: "w-20", - responsive: "hidden @min-[940px]/page:table-cell", + responsive: "hidden @min-[980px]/page:table-cell", }, { id: "cost", header: "Cost", width: 80, skeleton: "w-10", - responsive: "hidden @min-[580px]/page:table-cell", + responsive: "hidden @min-[620px]/page:table-cell", }, - { id: "errorSpanCount", header: "Errors", width: 100, skeleton: "w-12" }, + { id: "errorSpanCount", header: "Errors", width: 140, skeleton: "w-12" }, { id: "startTime", header: "Started", width: 96, skeleton: "w-14", - responsive: "hidden @min-[400px]/page:table-cell", + responsive: "hidden @min-[440px]/page:table-cell", }, ] @@ -316,7 +320,11 @@ export function AgentSessionsList({ } > - + ) }, @@ -370,20 +378,31 @@ export function AgentSessionsList({ }, { id: "cost", - header: sortHeader("Cost", "cost", "As priced by the instrumentation; blank where it reported none"), + header: sortHeader( + "Cost", + "cost", + "As priced by the instrumentation; blank where it reported none", + ), size: 80, // Blank where nothing was reported — a "$0.00" would read as "measured, // and it was free". cell: ({ row }) => row.original.cost > 0 ? ( - + {formatCost(row.original.cost)} ) : null, }, { id: "errorSpanCount", - header: sortHeader("Errors", "errorSpanCount", "Failed turns and tool calls"), + header: sortHeader( + "Errors", + "errorSpanCount", + "Failures the run needs fixed, and warnings it survived", + ), size: 100, cell: ({ row }) => , }, @@ -466,7 +485,10 @@ export function AgentSessionsList({ : "descending" : undefined } - className={cn(HEADER_CELL_CLASS, COLUMN_LAYOUT.get(header.id)?.responsive)} + className={cn( + HEADER_CELL_CLASS, + COLUMN_LAYOUT.get(header.id)?.responsive, + )} style={{ width: header.getSize() !== 150 ? header.getSize() : undefined, }} @@ -481,7 +503,10 @@ export function AgentSessionsList({ {firstItem && ( - + )} @@ -505,7 +530,10 @@ export function AgentSessionsList({ {row.getAllCells().map((cell) => ( {flexRender(cell.column.columnDef.cell, cell.getContext())} @@ -643,11 +671,14 @@ function SessionCell({ session, timeZone }: { session: AgentSessionRow; timeZone - } className="mt-0.5 flex min-w-0 items-baseline gap-1.5 text-xs"> + } + className="mt-0.5 flex min-w-0 items-baseline gap-1.5 text-xs" + > {id.kind === "trace" ? "Trace" : "Session"} @@ -743,7 +774,10 @@ function TokenBar({ session }: { session: AgentSessionRow }) { {/* A session that reported only a total draws no bar — an empty track would read as "measured, and it was nothing". */} {bucketTotal > 0 && ( - + {drawn.map((bucket) => ( — } + const failures = session.failures.filter((failure) => failure.severity === "failure") + const warnings = session.failures.filter((failure) => failure.severity === "anomaly") const classified = session.toolErrorCount + session.turnErrorCount const other = session.errorSpanCount - classified return (
- {session.turnErrorCount > 0 && ( + {failures.length > 0 && ( + } className="border-destructive/30 bg-destructive/10 text-destructive" /> )} - {session.toolErrorCount > 0 && ( + {warnings.length > 0 && ( } className="border-severity-warn/40 bg-severity-warn/10 text-severity-warn" /> )} @@ -806,6 +849,29 @@ function ErrorChips({ session }: { session: AgentSessionRow }) { ) } +const sumCounts = (rows: ReadonlyArray) => + rows.reduce((total, row) => total + row.count, 0) + +/** The hover: one line per label, in the Overview's own words and order. */ +function FailureBreakdown({ rows, lede }: { rows: ReadonlyArray; lede: string }) { + return ( +
+ {lede} +
    + {rows.map((row) => ( +
  • + {row.label} + {row.count > 1 && ( + ×{row.count} + )} + {row.terminal && ended the run} +
    • + ))} +
+
+ ) +} + function ErrorChip({ icon: Icon, count, @@ -816,13 +882,13 @@ function ErrorChip({ icon?: IconComponent count: number noun: string - hint: string + hint: ReactNode className: string }) { return ( { errorSpanCount: 26, toolErrorCount: 24, turnErrorCount: 1, + failures: [ + { + kind: "toolUnavailable", + label: "tool_unavailable · github_search", + tool: "github_search", + count: 3, + severity: "failure", + terminal: false, + }, + { + kind: "providerError", + label: "provider_error", + count: 1, + severity: "failure", + terminal: true, + }, + { kind: "rateLimited", label: "rate_limit", count: 9, severity: "anomaly", terminal: false }, + { + kind: "error", + label: "tool_error · read_file", + tool: "read_file", + count: 12, + severity: "anomaly", + terminal: false, + }, + ], totalTokens: 22_400_000, inputTokens: 3_100_000, cacheReadTokens: 17_800_000, @@ -146,6 +173,9 @@ function buildRows(nowMs: number): ReadonlyArray { errorSpanCount: 12, toolErrorCount: 0, turnErrorCount: 12, + failures: [ + { kind: "rateLimited", label: "rate_limit", count: 12, severity: "anomaly", terminal: false }, + ], totalTokens: 812_000, inputTokens: 0, cacheReadTokens: 0, diff --git a/apps/web/src/lab/agent-tools-fixture.ts b/apps/web/src/lab/agent-tools-fixture.ts index cf7896847..5592c4f64 100644 --- a/apps/web/src/lab/agent-tools-fixture.ts +++ b/apps/web/src/lab/agent-tools-fixture.ts @@ -194,9 +194,7 @@ export function buildToolCells(nowMs: number): ReadonlyArray { // `run_tests` regresses through the back half of the window. It is the // one thing this page exists to make visible, so the fixture has one. const regression = - profile.name === "run_tests" && index > BUCKETS * 0.6 - ? 1 + (index - BUCKETS * 0.6) / 14 - : 1 + profile.name === "run_tests" && index > BUCKETS * 0.6 ? 1 + (index - BUCKETS * 0.6) / 14 : 1 profile.models.forEach((model, modelIndex) => { const share = 1 / profile.models.length @@ -411,10 +409,7 @@ export function buildToolAnalyticsFixture( p95: totals.p95 * 0.7, } - const lastSeenBy = ( - keyOf: (cell: ToolFixtureCell) => string, - pick: (a: number, b: number) => number, - ) => { + const lastSeenBy = (keyOf: (cell: ToolFixtureCell) => string, pick: (a: number, b: number) => number) => { const out = new Map() for (const cell of filtered) { const key = keyOf(cell) @@ -518,9 +513,9 @@ interface ErrorGroupSpec { const envelope = (text: string) => JSON.stringify({ result: text }) const FINDINGS_CLAIM = - "Confirmed: the incident's failure is a disk-capacity exhaustion — the embedded chDB store on the local filesystem hit its \"no space left on device\" ceiling, and at least one trace ingest (POST /v1/traces) was rejected with HTTP 500 as a result." + 'Confirmed: the incident\'s failure is a disk-capacity exhaustion — the embedded chDB store on the local filesystem hit its "no space left on device" ceiling, and at least one trace ingest (POST /v1/traces) was rejected with HTTP 500 as a result.' const LOG_PATTERN = - "chDB insert (traces): Code: 1001. DB::Exception: filesystem error: in create_directories: No space left on device [\"/var/lib/maple/store/traces\"]" + 'chDB insert (traces): Code: 1001. DB::Exception: filesystem error: in create_directories: No space left on device ["/var/lib/maple/store/traces"]' const evidenceItem = (index: number, omit?: "traceIds" | "logPatterns") => ({ ...(omit !== "logPatterns" && { logPatterns: [LOG_PATTERN] }), @@ -596,7 +591,10 @@ const ERROR_GROUPS = new Map>([ callsSince: 402, arguments: (index) => candidate({ - evidence: index % 3 === 2 ? [evidenceItem(0), evidenceItem(1, "logPatterns")] : [evidenceItem(0, "logPatterns"), evidenceItem(1)], + evidence: + index % 3 === 2 + ? [evidenceItem(0), evidenceItem(1, "logPatterns")] + : [evidenceItem(0, "logPatterns"), evidenceItem(1)], }), }, { @@ -613,7 +611,12 @@ const ERROR_GROUPS = new Map>([ callsSince: 284, arguments: (index) => candidate({ - evidence: [0, 1, 2].map((item) => evidenceItem(item, item === variantIndex(index, [4, 3, 2]) ? "traceIds" : undefined)), + evidence: [0, 1, 2].map((item) => + evidenceItem( + item, + item === variantIndex(index, [4, 3, 2]) ? "traceIds" : undefined, + ), + ), }), }, { @@ -623,7 +626,12 @@ const ERROR_GROUPS = new Map>([ sessions: 6, perDay: [1, 2, 2, 1, 0, 0, 0, 0], callsSince: 300, - arguments: () => JSON.stringify({ claim: FINDINGS_CLAIM, confidence: "medium", evidence: [evidenceItem(0)] }), + arguments: () => + JSON.stringify({ + claim: FINDINGS_CLAIM, + confidence: "medium", + evidence: [evidenceItem(0)], + }), }, { message: expectedAt("array", '["suggestedActions"]'), @@ -829,7 +837,10 @@ const fingerprintOf = (message: string, index: number) => message === "" ? "0" : String( - [...message].reduce((hash, char) => (hash * 31n + BigInt(char.charCodeAt(0))) % 18_446_744_073_709_551_557n, BigInt(index + 7)), + [...message].reduce( + (hash, char) => (hash * 31n + BigInt(char.charCodeAt(0))) % 18_446_744_073_709_551_557n, + BigInt(index + 7), + ), ) const dayStart = (ms: number) => Math.floor(ms / DAY) * DAY @@ -838,7 +849,10 @@ const dayStart = (ms: number) => Math.floor(ms / DAY) * DAY export function buildToolErrorsFixture(tool: string, nowMs: number): ReadonlyArray { const today = dayStart(nowMs) return (ERROR_GROUPS.get(tool) ?? []).map((spec, index) => { - const days = spec.perDay.map((calls, day) => ({ bucket: today - (spec.perDay.length - 1 - day) * DAY, calls })) + const days = spec.perDay.map((calls, day) => ({ + bucket: today - (spec.perDay.length - 1 - day) * DAY, + calls, + })) const active = days.filter((day) => day.calls > 0) const lastDay = active[active.length - 1]?.bucket ?? today return { @@ -850,7 +864,10 @@ export function buildToolErrorsFixture(tool: string, nowMs: number): ReadonlyArr variants: spec.variants?.length ?? 1, firstSeen: (active[0]?.bucket ?? today) + 9 * 3_600_000 + index * 60_000, // The newest day at 23:48, or the morning for a group still failing today. - lastSeen: Math.min(lastDay + 23 * 3_600_000 + 48 * 60_000 - index * 67_000, nowMs - 22 * 3_600_000 - index * 60_000), + lastSeen: Math.min( + lastDay + 23 * 3_600_000 + 48 * 60_000 - index * 67_000, + nowMs - 22 * 3_600_000 - index * 60_000, + ), callsSince: spec.callsSince, trend: days.filter((day) => day.calls > 0), } @@ -872,9 +889,16 @@ export function buildToolErrorDetailFixture( tool: string, row: ToolErrorRow, options: { readonly session?: string; readonly variant?: string; readonly pages: number }, -): { readonly detail: ToolErrorDetailData; readonly occurrences: ReadonlyArray; readonly hasMore: boolean } { - const spec = (ERROR_GROUPS.get(tool) ?? []).find((candidate, index) => fingerprintOf(candidate.message, index) === row.fingerprint) - if (spec === undefined) return { detail: { sessions: [], variants: [], breakdown: [] }, occurrences: [], hasMore: false } +): { + readonly detail: ToolErrorDetailData + readonly occurrences: ReadonlyArray + readonly hasMore: boolean +} { + const spec = (ERROR_GROUPS.get(tool) ?? []).find( + (candidate, index) => fingerprintOf(candidate.message, index) === row.fingerprint, + ) + if (spec === undefined) + return { detail: { sessions: [], variants: [], breakdown: [] }, occurrences: [], hasMore: false } const variants = spec.variants ?? [{ message: spec.message, calls: spec.calls }] const where = spec.where ?? [["z-ai/glm-5.3-flash:nitro", "maple-investigations", 1] as const] const sessionIds = SAMPLE_SESSION_IDS.slice(0, Math.min(SAMPLE_SESSION_IDS.length, spec.sessions)) @@ -889,7 +913,13 @@ export function buildToolErrorDetailFixture( })) const all: ReadonlyArray = Array.from({ length: spec.calls }, (_, index) => { - const variant = variants[variantIndex(index, variants.map((candidate) => candidate.calls))]! + const variant = + variants[ + variantIndex( + index, + variants.map((candidate) => candidate.calls), + ) + ]! const [model, service] = where[index % where.length]! const args = spec.arguments(index) const result = spec.message.startsWith("{") ? variant.message : "" @@ -920,8 +950,18 @@ export function buildToolErrorDetailFixture( return { detail: { sessions, - variants: spec.variants === undefined ? [{ message: spec.message, calls: spec.calls, lastSeen: row.lastSeen }] : variants.map((candidate, index) => ({ ...candidate, lastSeen: row.lastSeen - index * 3_600_000 })), - breakdown: where.map(([model, service, share]) => ({ model, service, calls: Math.max(1, Math.round(spec.calls * share)) })), + variants: + spec.variants === undefined + ? [{ message: spec.message, calls: spec.calls, lastSeen: row.lastSeen }] + : variants.map((candidate, index) => ({ + ...candidate, + lastSeen: row.lastSeen - index * 3_600_000, + })), + breakdown: where.map(([model, service, share]) => ({ + model, + service, + calls: Math.max(1, Math.round(spec.calls * share)), + })), }, occurrences: narrowed.slice(0, 25 * options.pages), hasMore: narrowed.length > 25 * options.pages, @@ -946,38 +986,37 @@ function detailSessions( ((seed.tools as ReadonlyArray).includes(tool) && scoped.some((cell) => cell.service === seed.serviceName))) && (model === undefined || seed.model === model), - ).map( - (seed, index) => { - const startedAt = nowMs - seed.minutesAgo * 60_000 - const durationMs = seed.maxMs * 4 + 12_000 - return { - sessionId: seed.sessionId, - vendorId: ["eve", "claude_agent_sdk", "vercel_ai_sdk", "langchain"][index % 4]!, - vendorVersion: ["v1.4.2", "v0.9.1", "v5.0.4", "v0.3.27"][index % 4]!, - traceCount: 1 + (index % 6), - spanCount: 22 + index * 97, - errorSpanCount: seed.errors, - toolErrorCount: seed.errors, - turnErrorCount: 0, - serviceNames: [seed.serviceName], - models: [seed.model], - agentNames: seed.agentName === "" ? [] : [seed.agentName], - firstAgentName: seed.agentName, - llmCalls: Math.round(seed.calls / 3), - toolCalls: seed.calls, - totalTokens: seed.calls * 900, - inputTokens: seed.calls * 600, - cacheReadTokens: seed.calls * 200, - cacheWriteTokens: 0, - outputTokens: seed.calls * 100, - reasoningTokens: 0, - cost: seed.calls * 0.004, - startTime: new Date(startedAt).toISOString().replace("T", " ").slice(0, 23), - endTime: new Date(startedAt + durationMs).toISOString().replace("T", " ").slice(0, 23), - durationMs, - } - }, - ) + ).map((seed, index) => { + const startedAt = nowMs - seed.minutesAgo * 60_000 + const durationMs = seed.maxMs * 4 + 12_000 + return { + sessionId: seed.sessionId, + vendorId: ["eve", "claude_agent_sdk", "vercel_ai_sdk", "langchain"][index % 4]!, + vendorVersion: ["v1.4.2", "v0.9.1", "v5.0.4", "v0.3.27"][index % 4]!, + traceCount: 1 + (index % 6), + spanCount: 22 + index * 97, + errorSpanCount: seed.errors, + toolErrorCount: seed.errors, + turnErrorCount: 0, + failures: [], + serviceNames: [seed.serviceName], + models: [seed.model], + agentNames: seed.agentName === "" ? [] : [seed.agentName], + firstAgentName: seed.agentName, + llmCalls: Math.round(seed.calls / 3), + toolCalls: seed.calls, + totalTokens: seed.calls * 900, + inputTokens: seed.calls * 600, + cacheReadTokens: seed.calls * 200, + cacheWriteTokens: 0, + outputTokens: seed.calls * 100, + reasoningTokens: 0, + cost: seed.calls * 0.004, + startTime: new Date(startedAt).toISOString().replace("T", " ").slice(0, 23), + endTime: new Date(startedAt + durationMs).toISOString().replace("T", " ").slice(0, 23), + durationMs, + } + }) } /** `/agent-sessions/tools/$toolName` over the same week the overview draws. */ @@ -994,18 +1033,16 @@ export function buildToolDetailFixture( (search.service === undefined || cell.service === search.service) && (search.env === undefined || cell.env === search.env), ) - const series: ToolSeriesPoint[] = [ - ...rollup(scoped, (cell) => `${cell.bucket}`).entries(), - ].map(([bucket, value]) => ({ bucket: Number(bucket), seriesKey: tool, ...value })) + const series: ToolSeriesPoint[] = [...rollup(scoped, (cell) => `${cell.bucket}`).entries()].map( + ([bucket, value]) => ({ bucket: Number(bucket), seriesKey: tool, ...value }), + ) const totals: ToolTotals = rollup(scoped, () => "all").get("all") ?? EMPTY_MEASURES const sessions = detailSessions(tool, nowMs, scoped, search.model) return { series, totals, - scopeCalls: cells - .filter((cell) => cell.tool === tool) - .reduce((sum, cell) => sum + cell.calls, 0), + scopeCalls: cells.filter((cell) => cell.tool === tool).reduce((sum, cell) => sum + cell.calls, 0), firstSeen: scoped.reduce((min, cell) => (min === 0 ? cell.bucket : Math.min(min, cell.bucket)), 0), lastSeen: scoped.reduce((max, cell) => Math.max(max, cell.bucket), 0), description: `Runs ${tool} in the agent's workspace and returns its output, truncated to the last 4,000 characters.`, diff --git a/bun.lock b/bun.lock index 8f38b0f70..34236c3c7 100644 --- a/bun.lock +++ b/bun.lock @@ -580,6 +580,7 @@ "@effect-agent/sandbox": "0.1.0-beta.85", "@maple-dev/effect-clickhouse": "0.1.0", "@maple-dev/effect-clickhouse-http": "workspace:*", + "@maple/agent-sessions": "workspace:*", "@maple/auth": "workspace:*", "@maple/cache": "workspace:*", "@maple/db": "workspace:*", diff --git a/packages/agent-sessions/src/failure-text.ts b/packages/agent-sessions/src/failure-text.ts index 4bd3db535..9984408bc 100644 --- a/packages/agent-sessions/src/failure-text.ts +++ b/packages/agent-sessions/src/failure-text.ts @@ -104,11 +104,27 @@ function wholeProse(value: unknown, depth = 0): string | undefined { * call's recorded result. `undefined` when the span said nothing at all. */ export function rawFailureText(span: AiSessionSpan): string | undefined { - const message = span.statusMessage.trim() - const errorType = span.genAi.errorType + return rawFailureTextOf({ + statusMessage: span.statusMessage, + errorType: span.genAi.errorType, + toolCallResult: span.genAi.toolCallResult, + }) +} + +/** The three fields {@link rawFailureText} reads, as a span or an + * `ai_trace_index` row supplies them — the list classifies off the index. */ +export interface FailureText { + readonly statusMessage: string + readonly errorType: string | undefined + readonly toolCallResult: unknown +} + +export function rawFailureTextOf(signal: FailureText): string | undefined { + const message = signal.statusMessage.trim() + const errorType = signal.errorType const informative = message !== "" && message !== errorType && !GENERIC_TOOL_MESSAGE.test(message) if (informative) return message - const result = span.genAi.toolCallResult + const result = signal.toolCallResult const prose = result === undefined ? undefined : wholeProse(result) if (prose !== undefined) return prose return message === "" || message === errorType ? undefined : message diff --git a/packages/agent-sessions/src/index-failures.test.ts b/packages/agent-sessions/src/index-failures.test.ts new file mode 100644 index 000000000..bb387c9f3 --- /dev/null +++ b/packages/agent-sessions/src/index-failures.test.ts @@ -0,0 +1,156 @@ +import { describe, expect, it } from "vitest" +import { summarizeIndexFailures, type IndexFailedSpan } from "./index-failures" + +const failed = (overrides: Partial & { spanId: string }): IndexFailedSpan => ({ + traceId: "trace-1", + isToolCall: false, + isLlmCall: true, + errorType: "", + toolName: "", + vendorId: "eve", + statusMessage: "", + failedToolCallResult: "", + responseId: "", + atMs: 1_000, + ...overrides, +}) + +describe("summarizeIndexFailures", () => { + it("classifies index rows the way the detail page classifies spans, and grades them by the one rule", () => { + const rows = summarizeIndexFailures( + [ + failed({ spanId: "a", statusMessage: "429 Too Many Requests", atMs: 1_000 }), + failed({ spanId: "b", statusMessage: "429 Too Many Requests", atMs: 2_000 }), + failed({ + spanId: "c", + isToolCall: true, + isLlmCall: false, + toolName: "run_tests", + errorType: "tool_error", + statusMessage: "Tool execution failed", + failedToolCallResult: '{"error":"exit code 1"}', + atMs: 3_000, + }), + failed({ + spanId: "d", + errorType: "context_length_exceeded", + statusMessage: "prompt is too long: 210000 tokens > 200000 maximum", + atMs: 4_000, + }), + ], + { traceId: "trace-1", turnFailed: false }, + ) + expect(rows).toEqual([ + { + kind: "contextExceeded", + label: "context_length_exceeded", + tool: undefined, + count: 1, + severity: "failure", + terminal: false, + }, + { + kind: "rateLimited", + label: "rate_limit", + tool: undefined, + count: 2, + severity: "anomaly", + terminal: false, + }, + { + kind: "error", + label: "tool_error · run_tests", + tool: "run_tests", + count: 1, + severity: "anomaly", + terminal: false, + }, + ]) + }) + + it("marks the last failure of the last trace terminal when that trace's turn failed, and reds it", () => { + const rows = summarizeIndexFailures( + [ + failed({ spanId: "early", traceId: "trace-1", statusMessage: "rate limit", atMs: 1_000 }), + failed({ spanId: "late", traceId: "trace-2", errorType: "provider_error", atMs: 9_000 }), + failed({ spanId: "later", traceId: "trace-2", statusMessage: "rate limit", atMs: 9_500 }), + ], + { traceId: "trace-2", turnFailed: true }, + ) + // The rate limit the run died on leads, red; the provider error it + // survived is amber even though it is in the same trace. + expect(rows.map((row) => [row.label, row.severity, row.terminal, row.count])).toEqual([ + ["rate_limit", "failure", true, 2], + ["provider_error", "anomaly", false, 1], + ]) + }) + + it("leaves a survived last trace amber whatever it failed on", () => { + const rows = summarizeIndexFailures( + [ + failed({ + spanId: "tool", + isToolCall: true, + isLlmCall: false, + toolName: "grep", + errorType: "tool_error", + }), + ], + { traceId: "trace-1", turnFailed: false }, + ) + expect(rows).toEqual([ + { + kind: "error", + label: "tool_error · grep", + tool: "grep", + count: 1, + severity: "anomaly", + terminal: false, + }, + ]) + }) + + it("counts a call the app and a gateway mirror both observed once, keeping the observation that named the cause", () => { + const rows = summarizeIndexFailures( + [ + failed({ spanId: "app", responseId: "gen-1", errorType: "provider_error", atMs: 1_000 }), + failed({ + spanId: "mirror", + traceId: "mirror-trace", + vendorId: "openrouter", + responseId: "gen-1", + statusMessage: "Provider overloaded, retry later", + atMs: 1_001, + }), + ], + { traceId: "mirror-trace", turnFailed: false }, + ) + expect(rows).toEqual([ + { + kind: "rateLimited", + label: "rate_limit", + tool: undefined, + count: 1, + severity: "anomaly", + terminal: false, + }, + ]) + }) + + it("reads a row that predates migration 0032 as a plain error", () => { + const rows = summarizeIndexFailures([failed({ spanId: "old" })], { + traceId: "trace-1", + turnFailed: false, + }) + expect(rows).toEqual([ + { + kind: "error", + label: "error", + tool: undefined, + count: 1, + severity: "anomaly", + terminal: false, + }, + ]) + }) +}) diff --git a/packages/agent-sessions/src/index-failures.ts b/packages/agent-sessions/src/index-failures.ts new file mode 100644 index 000000000..2fddbde80 --- /dev/null +++ b/packages/agent-sessions/src/index-failures.ts @@ -0,0 +1,157 @@ +// The list's breakdown of a session's failures, off `ai_trace_index` alone. +// +// The detail page names what went wrong from the session's spans +// (`failureEvents` → `buildSessionFindings`). The list cannot read spans — +// a page is one index query, and the fan-out costs seconds per partition — +// but since migration 0032 the index carries every field the classifier +// reads off a failed span, so the page query ships the failed rows and this +// module classifies them with the SAME `classifyFailureSignal` and the SAME +// severity rule. A session the detail page calls failed on +// `context_length_exceeded` is red in the list for the same reason. +// +// What the index cannot say, the list does not claim: refusals and truncated +// replies (finish reasons), loops and stalls (the turn timeline) are the +// detail page's, and all of them are amber there unless the run died on one. + +import { rawFailureTextOf } from "./failure-text" +import { failureSeverity, type FindingSeverity } from "./session-findings" +import { + classifyFailureSignal, + failureSpecificity, + type SessionFailureClass, + type SessionFailureKind, +} from "./session-summary" + +/** One failed agent span as the page query ships it: the deepest span of a + * roll-up, with the index columns the classifier reads. */ +export interface IndexFailedSpan { + readonly spanId: string + readonly traceId: string + readonly isToolCall: boolean + readonly isLlmCall: boolean + /** `''` where the span stamped none, or the row predates migration 0032. */ + readonly errorType: string + readonly toolName: string + readonly vendorId: string + readonly statusMessage: string + readonly failedToolCallResult: string + readonly responseId: string + readonly atMs: number +} + +/** One line of the list's breakdown: a failure label, how often, how bad. */ +export interface SessionFailureSummary { + readonly kind: SessionFailureKind + /** `context_length_exceeded`, `tool_error · run_tests` — the finding's label. */ + readonly label: string + readonly tool: string | undefined + readonly count: number + readonly severity: FindingSeverity + /** The session's last turn died on it. */ + readonly terminal: boolean +} + +/** + * Failures grouped by label, red ones first and the terminal one leading — + * the order `buildSessionFindings` gives its failure rows. + * + * `terminal` is the detail page's verdict approximated one trace deep: the + * session's last trace had a turn-level failure (`lastTraceTurnFailed`, from + * every failed span of that trace, echoes included), and the failure it died + * on is the last one in that trace. A turn that crosses traces can differ. + */ +export function summarizeIndexFailures( + spans: readonly IndexFailedSpan[], + lastTrace: { readonly traceId: string; readonly turnFailed: boolean }, +): readonly SessionFailureSummary[] { + const events = dedupeByResponseId( + [...spans] + .sort((a, b) => a.atMs - b.atMs) + .map((span) => ({ span, ...classifyFailureSignal(signalOf(span)) })), + ) + const cause = lastTrace.turnFailed + ? events.findLast((event) => event.span.traceId === lastTrace.traceId) + : undefined + + const groups = new Map< + string, + { kind: SessionFailureKind; tool: string | undefined; count: number; terminal: boolean; atMs: number } + >() + for (const event of events) { + const group = groups.get(event.label) ?? { + kind: event.kind, + tool: event.tool, + count: 0, + terminal: false, + atMs: event.span.atMs, + } + group.count += 1 + group.terminal ||= event === cause + groups.set(event.label, group) + } + + return [...groups] + .map(([label, group]) => ({ + kind: group.kind, + label, + tool: group.tool, + count: group.count, + severity: failureSeverity(group.kind, group.terminal), + terminal: group.terminal, + atMs: group.atMs, + })) + .sort( + (a, b) => + Number(a.severity === "anomaly") - Number(b.severity === "anomaly") || + Number(b.terminal) - Number(a.terminal) || + a.atMs - b.atMs, + ) + .map(({ atMs: _atMs, ...summary }) => summary) +} + +function signalOf(span: IndexFailedSpan) { + return { + errorType: span.errorType === "" ? undefined : span.errorType, + responseStatus: undefined, + statusMessage: span.statusMessage, + // The view keeps the result only on failed tool calls, as text. + toolCallResult: span.failedToolCallResult === "" ? undefined : span.failedToolCallResult, + tool: span.toolName !== "" ? span.toolName : span.isToolCall ? "tool" : undefined, + isLlmCall: span.isLlmCall, + vendorId: span.vendorId === "" ? undefined : span.vendorId, + } +} + +type IndexFailureEvent = SessionFailureClass & { readonly span: IndexFailedSpan } + +/** Same rule as `session-summary.ts`'s `dedupeByResponseId`, over index rows: + * a call the app and a gateway mirror both observed is one failure, and the + * observation that named the cause keeps it. */ +function dedupeByResponseId(events: readonly IndexFailureEvent[]): readonly IndexFailureEvent[] { + const slots = new Map() + const kept: IndexFailureEvent[] = [] + for (const event of events) { + const id = event.span.responseId + if (id === "") { + kept.push(event) + continue + } + const slot = slots.get(id) + if (slot === undefined) { + slots.set(id, { index: kept.length, event }) + kept.push(event) + continue + } + const specific = failureSpecificity(event) - failureSpecificity(slot.event) + const longer = textLength(event.span) - textLength(slot.event.span) + if (specific > 0 || (specific === 0 && longer > 0)) { + kept[slot.index] = event + slot.event = event + } + } + return kept +} + +function textLength(span: IndexFailedSpan): number { + return (rawFailureTextOf(signalOf(span)) ?? "").length +} diff --git a/packages/agent-sessions/src/index.ts b/packages/agent-sessions/src/index.ts index 83e3bc30a..b9ff1b0b7 100644 --- a/packages/agent-sessions/src/index.ts +++ b/packages/agent-sessions/src/index.ts @@ -7,6 +7,7 @@ export * from "./session-turns" export * from "./session-summary" export * from "./session-findings" export * from "./session-checks" +export * from "./index-failures" export * from "./session-transcript" export * from "./span-detail" export * from "./session-window" diff --git a/packages/agent-sessions/src/session-findings.ts b/packages/agent-sessions/src/session-findings.ts index 3478c6c0d..2fd63be5e 100644 --- a/packages/agent-sessions/src/session-findings.ts +++ b/packages/agent-sessions/src/session-findings.ts @@ -69,6 +69,13 @@ const FAILURE_KINDS: ReadonlySet = new Set([ * split reads the same field, so the two never disagree. */ export type FindingSeverity = "failure" | "anomaly" +/** The one severity rule, for the findings here and for the list's breakdown + * off the index (`index-failures.ts`): red when the run died on it or its + * kind is in {@link FAILURE_KINDS}, amber otherwise. */ +export function failureSeverity(kind: SessionFailureKind, terminal: boolean): FindingSeverity { + return terminal || FAILURE_KINDS.has(kind) ? "failure" : "anomaly" +} + /** Which detector produced the row: a failure kind, or one of the four * session-shape detectors below. What the checklist groups rows on. */ export type SessionFindingKind = SessionFailureKind | "providerRetry" | "truncation" | "repetition" | "stall" @@ -199,7 +206,7 @@ function failureFindings( return { id: `failure:${label}`, kind: group.kind, - severity: terminal || FAILURE_KINDS.has(group.kind) ? ("failure" as const) : ("anomaly" as const), + severity: failureSeverity(group.kind, terminal), label, tool: group.tool, count: group.members.length, diff --git a/packages/agent-sessions/src/session-summary.ts b/packages/agent-sessions/src/session-summary.ts index ec3f8db5f..517f2770d 100644 --- a/packages/agent-sessions/src/session-summary.ts +++ b/packages/agent-sessions/src/session-summary.ts @@ -12,7 +12,7 @@ // and a total is always the plain sum of the buckets. import { genAiUsageConvention } from "@maple/domain/gen-ai" -import type { AiSessionSpan } from "@maple/domain/http" +import type { AiSessionFailureKind, AiSessionSpan } from "@maple/domain/http" import { formatCurrency, formatDuration, formatNumber } from "@maple/domain/format" import { @@ -20,6 +20,7 @@ import { failureDetailText, incompleteRunTool, rawFailureText, + rawFailureTextOf, stripFailurePrefixes, toolNamedBySchemaError, } from "./failure-text" @@ -155,17 +156,7 @@ export interface SessionToolUsage { * - `toolTimeout`: a tool's backend gave up. * - `incomplete`: the run ended without the completion the agent required. */ -export type SessionFailureKind = - | "error" - | "rateLimited" - | "contextExceeded" - | "refusal" - | "providerError" - | "invalidOutput" - | "toolArguments" - | "toolUnavailable" - | "toolTimeout" - | "incomplete" +export type SessionFailureKind = AiSessionFailureKind export interface SessionFailureEvent { readonly kind: SessionFailureKind @@ -1046,7 +1037,7 @@ function dedupeByResponseId(events: readonly SessionFailureEvent[]): readonly Se } /** Whether the event's kind says what happened, or is one of the catch-alls. */ -function failureSpecificity(event: SessionFailureEvent): number { +export function failureSpecificity(event: { readonly kind: SessionFailureKind }): number { return event.kind === "error" || event.kind === "providerError" ? 0 : 1 } @@ -1068,21 +1059,54 @@ const TOOL_ARGUMENTS_PATTERN = const TOOL_UNAVAILABLE_PATTERN = /not configured|not connected|not available|unavailable|not enabled|not installed|unauthori[sz]ed|forbidden|permission denied|access denied|no sandbox|integration/i -function classifyFailure(span: AiSessionSpan): Omit { - const signal = errorSignal(span) +function classifyFailure(span: AiSessionSpan): SessionFailureClass { + return classifyFailureSignal({ + errorType: span.genAi.errorType, + responseStatus: span.genAi.responseStatus, + statusMessage: span.statusMessage, + toolCallResult: span.genAi.toolCallResult, + tool: span.genAi.toolName ?? (classifyAiSpan(span) === "tool" ? span.spanName : undefined), + isLlmCall: isLlmCall(span), + vendorId: span.vendorId, + }) +} + +/** What a failure is, less the span that carried it. */ +export type SessionFailureClass = Omit + +/** + * What {@link classifyFailureSignal} reads off a failed span — the fields as a + * span carries them, or as an `ai_trace_index` row does since migration 0032, + * which is how the list names a session's failures without reading its spans. + */ +export interface FailureSignal { + readonly errorType: string | undefined + readonly responseStatus: string | undefined + readonly statusMessage: string + readonly toolCallResult: unknown + /** The tool the span is about — `gen_ai.tool.name`, else a tool span's name. */ + readonly tool: string | undefined + readonly isLlmCall: boolean + readonly vendorId: string | undefined +} + +export function classifyFailureSignal(span: FailureSignal): SessionFailureClass { + const signal = [span.errorType, span.responseStatus, span.statusMessage] + .filter((value): value is string => value !== undefined && value !== "") + .join(" ") if (RATE_LIMIT_PATTERN.test(signal)) return { kind: "rateLimited", label: "rate_limit" } if (CONTEXT_EXCEEDED_PATTERN.test(signal)) { return { kind: "contextExceeded", label: "context_length_exceeded" } } - const raw = (rawFailureText(span) ?? "").slice(0, CLASSIFIED_TEXT_CHARS) + const raw = (rawFailureTextOf(span) ?? "").slice(0, CLASSIFIED_TEXT_CHARS) const text = stripFailurePrefixes(raw) if (incompleteRunTool(text) !== undefined) return { kind: "incomplete", label: "incomplete" } // `error.type` is the instrumentation's own word for it; the tool name is // what separates one failing tool from another under a shared `tool_error`. - const name = span.genAi.errorType ?? "error" - const tool = span.genAi.toolName ?? (classifyAiSpan(span) === "tool" ? span.spanName : undefined) + const name = span.errorType ?? "error" + const tool = span.tool if (tool !== undefined) { // A parameter error names the tool whose schema was violated; a batch // of calls can stamp a sibling's name on the span. `error.type` joins @@ -1090,7 +1114,7 @@ function classifyFailure(span: AiSessionSpan): Omit // has still said what happened. The schema wrapper is one of the // prefixes stripping removes, so it is read off the raw text. const named = toolNamedBySchemaError(raw) ?? tool - const words = `${span.genAi.errorType ?? ""} ${text}` + const words = `${span.errorType ?? ""} ${text}` // The schema cue first: a rejected parameter named `timeout` is still // the model's arguments. if (TOOL_SCHEMA_PATTERN.test(raw)) @@ -1107,15 +1131,15 @@ function classifyFailure(span: AiSessionSpan): Omit return { kind: "error", label: `${name} · ${named}`, tool: named } } - if (span.genAi.errorType === "invalid_output" || describeSchemaFailure(text) !== undefined) { + if (span.errorType === "invalid_output" || describeSchemaFailure(text) !== undefined) { return { kind: "invalidOutput", label: "invalid_output" } } // A model call that failed at the provider: Maple's own agents stamp // `provider_error`; a gateway mirror's generation span stamps nothing and is // known by where it came from. A more specific `error.type` keeps its name. if ( - span.genAi.errorType === "provider_error" || - (span.genAi.errorType === undefined && span.vendorId === "openrouter" && isLlmCall(span)) + span.errorType === "provider_error" || + (span.errorType === undefined && span.vendorId === "openrouter" && span.isLlmCall) ) { return { kind: "providerError", label: "provider_error" } } diff --git a/packages/backend/package.json b/packages/backend/package.json index e6b0b84a5..f162ddea8 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -25,6 +25,7 @@ "@effect-agent/sandbox": "0.1.0-beta.85", "@maple-dev/effect-clickhouse": "0.1.0", "@maple-dev/effect-clickhouse-http": "workspace:*", + "@maple/agent-sessions": "workspace:*", "@maple/auth": "workspace:*", "@maple/cache": "workspace:*", "@maple/db": "workspace:*", diff --git a/packages/backend/src/services/ai-sessions/ai-session-reads.ts b/packages/backend/src/services/ai-sessions/ai-session-reads.ts index 752230dae..e382b6ccb 100644 --- a/packages/backend/src/services/ai-sessions/ai-session-reads.ts +++ b/packages/backend/src/services/ai-sessions/ai-session-reads.ts @@ -35,6 +35,7 @@ import { type ListAiSessionsRequest, } from "@maple/domain/http" import { traceSessionTraceId } from "@maple/domain/gen-ai" +import { summarizeIndexFailures, type IndexFailedSpan } from "@maple/agent-sessions" import { Array as Arr, Effect } from "effect" import { CH, formatWarehouseDateTime, parseWarehouseDateTime } from "@maple/query-engine" import * as Integrations from "@maple/query-engine-integrations" @@ -222,6 +223,10 @@ export const listAiSessions = Effect.fn("aiSessions.list")(function* ( errorSpanCount: row.errorAgentSpans, toolErrorCount: row.toolErrors, turnErrorCount: row.turnErrors, + failures: summarizeIndexFailures(row.failures.map(indexFailedSpan), { + traceId: row.lastTraceId, + turnFailed: row.lastTraceTurnFailed === 1, + }), serviceNames: row.serviceNames, models: row.models, agentNames: row.agentNames, @@ -603,6 +608,21 @@ export const readAiToolsBreakdowns = Effect.fn("aiSessions.toolsBreakdowns")(fun return new AiToolsBreakdownsResponse({ tools: rows.map(breakdownItem) }) }) +/** A `failedSpansExpr` tuple as the page query ships it, by position. */ +export const indexFailedSpan = (tuple: Integrations.IndexFailedSpanTuple): IndexFailedSpan => ({ + spanId: tuple[0], + traceId: tuple[10], + isToolCall: tuple[2] === 1, + isLlmCall: tuple[3] === 1, + errorType: tuple[4], + toolName: tuple[5], + vendorId: tuple[6], + statusMessage: tuple[7], + failedToolCallResult: tuple[8], + responseId: tuple[9], + atMs: tuple[11], +}) + export const readAiToolErrors = Effect.fn("aiSessions.toolErrors")(function* ( tenant: TenantContext, payload: AiToolErrorsRequest, diff --git a/packages/backend/src/services/warehouse/ai-trace-index-materialization.clickhouse.e2e.test.ts b/packages/backend/src/services/warehouse/ai-trace-index-materialization.clickhouse.e2e.test.ts index f967bd843..641b3507f 100644 --- a/packages/backend/src/services/warehouse/ai-trace-index-materialization.clickhouse.e2e.test.ts +++ b/packages/backend/src/services/warehouse/ai-trace-index-materialization.clickhouse.e2e.test.ts @@ -26,6 +26,8 @@ import { } from "@maple/domain/gen-ai" import { genAiErrorFingerprintText } from "@maple/domain/tinybird/gen-ai-columns" import * as Integrations from "@maple/query-engine-integrations" +import { summarizeIndexFailures } from "@maple/agent-sessions" +import { indexFailedSpan } from "@maple/backend/services/ai-sessions/ai-session-reads" import type { AiSessionPageOpts } from "@maple/query-engine-integrations" import { normalizeSqlForClickHouseClient } from "@maple/query-engine/execution" import { @@ -806,6 +808,44 @@ describe.skipIf(!clickhouseE2eEnabled)("ai_trace_index materialization", () => { assert.strictEqual(sessionless!.errorAgentSpans, 1) // The one failed span is a model call: a turn failure, not a tool's. assert.deepStrictEqual([sessionless!.toolErrors, sessionless!.turnErrors], [0, 1]) + // The failed span, shipped for the breakdown as the tuple's positions — + // its trace is the session's only one, so its turn failure is terminal. + assert.deepStrictEqual(sessionless!.failures, [ + [ + "span-agent-2", + "", + 0, + 1, + "", + "", + "vercel_ai_sdk", + "", + "", + "", + SESSIONLESS_TRACE, + BASE_MS + 60_123, + ], + ]) + assert.deepStrictEqual( + [sessionless!.lastTraceId, sessionless!.lastTraceTurnFailed], + [SESSIONLESS_TRACE, 1], + ) + assert.deepStrictEqual( + summarizeIndexFailures(sessionless!.failures.map(indexFailedSpan), { + traceId: sessionless!.lastTraceId, + turnFailed: sessionless!.lastTraceTurnFailed === 1, + }), + [ + { + kind: "error", + label: "error", + tool: undefined, + count: 1, + severity: "failure", + terminal: true, + }, + ], + ) // One span of 1ms: the extent is its own duration. assert.strictEqual(sessionless!.agentDurationMs, 1) @@ -834,6 +874,41 @@ describe.skipIf(!clickhouseE2eEnabled)("ai_trace_index materialization", () => { // The failed tool span under an `Ok` turn: one tool error, and no turn // error echoed off it. The failure lambda is raw SQL too. assert.deepStrictEqual([session!.toolErrors, session!.turnErrors], [1, 0]) + // The 0032 columns ride along, and the session's last trace — the second + // one, 30s on — closed cleanly, so the timeout is a warning, not red. + assert.deepStrictEqual(session!.failures, [ + [ + "span-tool-1", + "span-agent-1", + 1, + 0, + "TimeoutError", + "search_traces", + "eve", + "search timed out", + "", + "", + AGENT_TRACE, + BASE_MS + 2_000, + ], + ]) + assert.deepStrictEqual([session!.lastTraceId, session!.lastTraceTurnFailed], [AGENT_TRACE_2, 0]) + assert.deepStrictEqual( + summarizeIndexFailures(session!.failures.map(indexFailedSpan), { + traceId: session!.lastTraceId, + turnFailed: session!.lastTraceTurnFailed === 1, + }), + [ + { + kind: "toolTimeout", + label: "tool_timeout · search_traces", + tool: "search_traces", + count: 1, + severity: "anomaly", + terminal: false, + }, + ], + ) // From the first turn span to the end of the second trace's turn span. assert.strictEqual(session!.agentDurationMs, 30_001) }) diff --git a/packages/domain/src/http/ai-sessions.ts b/packages/domain/src/http/ai-sessions.ts index 0f1c0cdec..96e7f58a3 100644 --- a/packages/domain/src/http/ai-sessions.ts +++ b/packages/domain/src/http/ai-sessions.ts @@ -134,6 +134,40 @@ export const AiSessionDetailsItem = Schema.Struct({ }) export type AiSessionDetailsItem = Schema.Schema.Type +/** What a failed span was classified as — `classifyFailureSignal` in + * `@maple/agent-sessions`, whose `SessionFailureKind` is this type. */ +export const AiSessionFailureKind = Schema.Literals([ + "error", + "rateLimited", + "contextExceeded", + "refusal", + "providerError", + "invalidOutput", + "toolArguments", + "toolUnavailable", + "toolTimeout", + "incomplete", +]) +export type AiSessionFailureKind = Schema.Schema.Type + +/** + * One line of a list row's failure breakdown: the failures sharing a label, + * classified off the index the way the detail page classifies them off the + * spans, and graded by the same rule — `failure` needs a fix (the run died + * on it, or its kind always does), `anomaly` was survived. + */ +export const AiSessionFailureSummary = Schema.Struct({ + kind: AiSessionFailureKind, + /** `context_length_exceeded`, `tool_error · run_tests` — the finding's label. */ + label: Schema.String, + tool: Schema.optionalKey(Schema.String), + count: Schema.Number, + severity: Schema.Literals(["failure", "anomaly"]), + /** The session's last turn died on it. */ + terminal: Schema.Boolean, +}) +export type AiSessionFailureSummary = Schema.Schema.Type + export const AiSessionListItem = Schema.Struct({ /** The vendor's own session id, or `trace:` for an agent trace whose * vendor exposes no session key — see `MAPLE_AI_TRACE_SESSION_PREFIX`. */ @@ -153,6 +187,9 @@ export const AiSessionListItem = Schema.Struct({ toolErrorCount: Schema.Number, /** Failed model calls and turn spans that failed on their own. */ turnErrorCount: Schema.Number, + /** The failures by label, red first — what the Errors cell's chips count + * and its hover lists. Empty when nothing failed. */ + failures: Schema.Array(AiSessionFailureSummary), /** Services the agent spans came from until the details land, then every service touched. */ serviceNames: Schema.Array(Schema.String), /** Every model any agent span of the session ran on, dialects coalesced. */ diff --git a/packages/query-engine-integrations/src/__sql_baseline__/integrations.sql b/packages/query-engine-integrations/src/__sql_baseline__/integrations.sql index dbd45d5bc..1dbdbbdaf 100644 --- a/packages/query-engine-integrations/src/__sql_baseline__/integrations.sql +++ b/packages/query-engine-integrations/src/__sql_baseline__/integrations.sql @@ -37,7 +37,7 @@ SELECT min(if(AgentName != '', Timestamp, toDateTime('2106-01-01 00:00:00'))) AS firstAgentAt, sum(IsToolCall) AS toolCalls, sum(IsError) AS errorAgentSpans, - groupArrayIf(2000)(tuple(SpanId, ParentSpanId, IsToolCall), IsError = 1) AS failedSpans, + groupArrayIf(2000)(tuple(SpanId, ParentSpanId, IsToolCall, IsLlmCall, ErrorType, ToolName, VendorId, StatusMessage, FailedToolCallResult, ResponseId, TraceId, toUnixTimestamp64Milli(Timestamp)), IsError = 1) AS failedSpans, groupArrayIf(2000)(tuple(SpanId, ParentSpanId, Tokens, Cost, ResponseId, IsLlmCall, InputTokens, CacheReadTokens, CacheWriteTokens, OutputTokens, ReasoningTokens), ((Tokens > 0 OR Cost > 0) OR IsLlmCall = 1)) AS usageReporters FROM ai_trace_index WHERE OrgId = 'org_sql_catalog' @@ -66,7 +66,7 @@ SELECT min(if(AgentName != '', Timestamp, toDateTime('2106-01-01 00:00:00'))) AS firstAgentAt, sum(IsToolCall) AS toolCalls, sum(IsError) AS errorAgentSpans, - groupArrayIf(2000)(tuple(SpanId, ParentSpanId, IsToolCall), IsError = 1) AS failedSpans, + groupArrayIf(2000)(tuple(SpanId, ParentSpanId, IsToolCall, IsLlmCall, ErrorType, ToolName, VendorId, StatusMessage, FailedToolCallResult, ResponseId, TraceId, toUnixTimestamp64Milli(Timestamp)), IsError = 1) AS failedSpans, groupArrayIf(2000)(tuple(SpanId, ParentSpanId, Tokens, Cost, ResponseId, IsLlmCall, InputTokens, CacheReadTokens, CacheWriteTokens, OutputTokens, ReasoningTokens), ((Tokens > 0 OR Cost > 0) OR IsLlmCall = 1)) AS usageReporters FROM ai_trace_index WHERE OrgId = 'org_sql_catalog' @@ -117,7 +117,7 @@ SELECT min(if(AgentName != '', Timestamp, toDateTime('2106-01-01 00:00:00'))) AS firstAgentAt, sum(IsToolCall) AS toolCalls, sum(IsError) AS errorAgentSpans, - groupArrayIf(2000)(tuple(SpanId, ParentSpanId, IsToolCall), IsError = 1) AS failedSpans, + groupArrayIf(2000)(tuple(SpanId, ParentSpanId, IsToolCall, IsLlmCall, ErrorType, ToolName, VendorId, StatusMessage, FailedToolCallResult, ResponseId, TraceId, toUnixTimestamp64Milli(Timestamp)), IsError = 1) AS failedSpans, groupArrayIf(2000)(tuple(SpanId, ParentSpanId, Tokens, Cost, ResponseId, IsLlmCall, InputTokens, CacheReadTokens, CacheWriteTokens, OutputTokens, ReasoningTokens), ((Tokens > 0 OR Cost > 0) OR IsLlmCall = 1)) AS usageReporters FROM ai_trace_index WHERE OrgId = 'org_sql_catalog' @@ -151,7 +151,7 @@ SELECT min(if(AgentName != '', Timestamp, toDateTime('2106-01-01 00:00:00'))) AS firstAgentAt, sum(IsToolCall) AS toolCalls, sum(IsError) AS errorAgentSpans, - groupArrayIf(2000)(tuple(SpanId, ParentSpanId, IsToolCall), IsError = 1) AS failedSpans, + groupArrayIf(2000)(tuple(SpanId, ParentSpanId, IsToolCall, IsLlmCall, ErrorType, ToolName, VendorId, StatusMessage, FailedToolCallResult, ResponseId, TraceId, toUnixTimestamp64Milli(Timestamp)), IsError = 1) AS failedSpans, groupArrayIf(2000)(tuple(SpanId, ParentSpanId, Tokens, Cost, ResponseId, IsLlmCall, InputTokens, CacheReadTokens, CacheWriteTokens, OutputTokens, ReasoningTokens), ((Tokens > 0 OR Cost > 0) OR IsLlmCall = 1)) AS usageReporters FROM ai_trace_index WHERE OrgId = 'org_sql_catalog' @@ -207,7 +207,7 @@ SELECT min(if(AgentName != '', Timestamp, toDateTime('2106-01-01 00:00:00'))) AS firstAgentAt, sum(IsToolCall) AS toolCalls, sum(IsError) AS errorAgentSpans, - groupArrayIf(2000)(tuple(SpanId, ParentSpanId, IsToolCall), IsError = 1) AS failedSpans, + groupArrayIf(2000)(tuple(SpanId, ParentSpanId, IsToolCall, IsLlmCall, ErrorType, ToolName, VendorId, StatusMessage, FailedToolCallResult, ResponseId, TraceId, toUnixTimestamp64Milli(Timestamp)), IsError = 1) AS failedSpans, groupArrayIf(2000)(tuple(SpanId, ParentSpanId, Tokens, Cost, ResponseId, IsLlmCall, InputTokens, CacheReadTokens, CacheWriteTokens, OutputTokens, ReasoningTokens), ((Tokens > 0 OR Cost > 0) OR IsLlmCall = 1)) AS usageReporters FROM ai_trace_index WHERE OrgId = 'org_sql_catalog' @@ -238,7 +238,7 @@ SELECT min(if(AgentName != '', Timestamp, toDateTime('2106-01-01 00:00:00'))) AS firstAgentAt, sum(IsToolCall) AS toolCalls, sum(IsError) AS errorAgentSpans, - groupArrayIf(2000)(tuple(SpanId, ParentSpanId, IsToolCall), IsError = 1) AS failedSpans, + groupArrayIf(2000)(tuple(SpanId, ParentSpanId, IsToolCall, IsLlmCall, ErrorType, ToolName, VendorId, StatusMessage, FailedToolCallResult, ResponseId, TraceId, toUnixTimestamp64Milli(Timestamp)), IsError = 1) AS failedSpans, groupArrayIf(2000)(tuple(SpanId, ParentSpanId, Tokens, Cost, ResponseId, IsLlmCall, InputTokens, CacheReadTokens, CacheWriteTokens, OutputTokens, ReasoningTokens), ((Tokens > 0 OR Cost > 0) OR IsLlmCall = 1)) AS usageReporters FROM ai_trace_index WHERE OrgId = 'org_sql_catalog' @@ -286,6 +286,9 @@ SELECT sum(errorAgentSpans) AS errorAgentSpans, sum(arrayCount(f -> f.3 = 1 AND NOT has(tupleElement(failedSpans, 2), f.1), failedSpans)) AS toolErrors, sum(arrayCount(f -> f.3 != 1 AND NOT has(tupleElement(failedSpans, 2), f.1), failedSpans)) AS turnErrors, + arraySlice(groupArrayArray(arrayFilter(f -> NOT has(tupleElement(failedSpans, 2), f.1), failedSpans)), 1, 100) AS failures, + argMax(traceId, traceAgentEndNanos) AS lastTraceId, + argMax(arrayExists(f -> f.3 != 1, failedSpans), traceAgentEndNanos) AS lastTraceTurnFailed, intDiv(max(traceAgentEndNanos) - toUnixTimestamp64Nano(min(traceAgentStart)), 1000000) AS agentDurationMs, arraySlice(arrayFlatten(groupArray(usageReporters)), 1, 2000) AS reporters, arrayReduce('sumMap', arrayMap(c -> [c.2], reporters), arrayMap(c -> [c.3], reporters), arrayMap(c -> [c.4], reporters), arrayMap(c -> [c.7], reporters), arrayMap(c -> [c.8], reporters), arrayMap(c -> [c.9], reporters), arrayMap(c -> [c.10], reporters), arrayMap(c -> [c.11], reporters)) AS childClaims, @@ -307,7 +310,7 @@ SELECT min(if(AgentName != '', Timestamp, toDateTime('2106-01-01 00:00:00'))) AS firstAgentAt, sum(IsToolCall) AS toolCalls, sum(IsError) AS errorAgentSpans, - groupArrayIf(2000)(tuple(SpanId, ParentSpanId, IsToolCall), IsError = 1) AS failedSpans, + groupArrayIf(2000)(tuple(SpanId, ParentSpanId, IsToolCall, IsLlmCall, ErrorType, ToolName, VendorId, StatusMessage, FailedToolCallResult, ResponseId, TraceId, toUnixTimestamp64Milli(Timestamp)), IsError = 1) AS failedSpans, groupArrayIf(2000)(tuple(SpanId, ParentSpanId, Tokens, Cost, ResponseId, IsLlmCall, InputTokens, CacheReadTokens, CacheWriteTokens, OutputTokens, ReasoningTokens), ((Tokens > 0 OR Cost > 0) OR IsLlmCall = 1)) AS usageReporters FROM ai_trace_index WHERE OrgId = 'org_sql_catalog' @@ -440,6 +443,9 @@ SELECT errorAgentSpans AS errorAgentSpans, toolErrors AS toolErrors, turnErrors AS turnErrors, + failures AS failures, + lastTraceId AS lastTraceId, + lastTraceTurnFailed AS lastTraceTurnFailed, agentDurationMs AS agentDurationMs, toFloat64(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 2)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, toFloat64(n.2)), arrayFilter(n -> n.1 != '', netted)))))) AS llmCalls, arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 3)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.3), arrayFilter(n -> n.1 != '', netted))))) AS totalTokens, @@ -465,6 +471,9 @@ SELECT errorAgentSpans AS errorAgentSpans, toolErrors AS toolErrors, turnErrors AS turnErrors, + failures AS failures, + lastTraceId AS lastTraceId, + lastTraceTurnFailed AS lastTraceTurnFailed, agentDurationMs AS agentDurationMs, arrayMap(r -> tuple(r.5, r.6 = 1 AND if((r.3 > 0 OR r.4 > 0), greatest(0., r.3 - arrayElement(tupleElement(childClaims, 2), indexOf(tupleElement(childClaims, 1), r.1))) > 0 OR greatest(0., r.4 - arrayElement(tupleElement(childClaims, 3), indexOf(tupleElement(childClaims, 1), r.1))) > 0, NOT has(reportingIds, r.2)), greatest(0., r.3 - arrayElement(tupleElement(childClaims, 2), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.4 - arrayElement(tupleElement(childClaims, 3), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.7 - arrayElement(tupleElement(childClaims, 4), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.8 - arrayElement(tupleElement(childClaims, 5), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.9 - arrayElement(tupleElement(childClaims, 6), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.10 - arrayElement(tupleElement(childClaims, 7), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.11 - arrayElement(tupleElement(childClaims, 8), indexOf(tupleElement(childClaims, 1), r.1)))), reporters) AS netted FROM (SELECT @@ -483,6 +492,9 @@ SELECT sum(errorAgentSpans) AS errorAgentSpans, sum(arrayCount(f -> f.3 = 1 AND NOT has(tupleElement(failedSpans, 2), f.1), failedSpans)) AS toolErrors, sum(arrayCount(f -> f.3 != 1 AND NOT has(tupleElement(failedSpans, 2), f.1), failedSpans)) AS turnErrors, + arraySlice(groupArrayArray(arrayFilter(f -> NOT has(tupleElement(failedSpans, 2), f.1), failedSpans)), 1, 100) AS failures, + argMax(traceId, traceAgentEndNanos) AS lastTraceId, + argMax(arrayExists(f -> f.3 != 1, failedSpans), traceAgentEndNanos) AS lastTraceTurnFailed, intDiv(max(traceAgentEndNanos) - toUnixTimestamp64Nano(min(traceAgentStart)), 1000000) AS agentDurationMs, arraySlice(arrayFlatten(groupArray(usageReporters)), 1, 2000) AS reporters, arrayReduce('sumMap', arrayMap(c -> [c.2], reporters), arrayMap(c -> [c.3], reporters), arrayMap(c -> [c.4], reporters), arrayMap(c -> [c.7], reporters), arrayMap(c -> [c.8], reporters), arrayMap(c -> [c.9], reporters), arrayMap(c -> [c.10], reporters), arrayMap(c -> [c.11], reporters)) AS childClaims, @@ -504,7 +516,7 @@ SELECT min(if(AgentName != '', Timestamp, toDateTime('2106-01-01 00:00:00'))) AS firstAgentAt, sum(IsToolCall) AS toolCalls, sum(IsError) AS errorAgentSpans, - groupArrayIf(2000)(tuple(SpanId, ParentSpanId, IsToolCall), IsError = 1) AS failedSpans, + groupArrayIf(2000)(tuple(SpanId, ParentSpanId, IsToolCall, IsLlmCall, ErrorType, ToolName, VendorId, StatusMessage, FailedToolCallResult, ResponseId, TraceId, toUnixTimestamp64Milli(Timestamp)), IsError = 1) AS failedSpans, groupArrayIf(2000)(tuple(SpanId, ParentSpanId, Tokens, Cost, ResponseId, IsLlmCall, InputTokens, CacheReadTokens, CacheWriteTokens, OutputTokens, ReasoningTokens), ((Tokens > 0 OR Cost > 0) OR IsLlmCall = 1)) AS usageReporters FROM ai_trace_index WHERE OrgId = 'org_sql_catalog' @@ -534,6 +546,9 @@ SELECT errorAgentSpans AS errorAgentSpans, toolErrors AS toolErrors, turnErrors AS turnErrors, + failures AS failures, + lastTraceId AS lastTraceId, + lastTraceTurnFailed AS lastTraceTurnFailed, agentDurationMs AS agentDurationMs, toFloat64(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 2)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, toFloat64(n.2)), arrayFilter(n -> n.1 != '', netted)))))) AS llmCalls, arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 3)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.3), arrayFilter(n -> n.1 != '', netted))))) AS totalTokens, @@ -559,6 +574,9 @@ SELECT errorAgentSpans AS errorAgentSpans, toolErrors AS toolErrors, turnErrors AS turnErrors, + failures AS failures, + lastTraceId AS lastTraceId, + lastTraceTurnFailed AS lastTraceTurnFailed, agentDurationMs AS agentDurationMs, arrayMap(r -> tuple(r.5, r.6 = 1 AND if((r.3 > 0 OR r.4 > 0), greatest(0., r.3 - arrayElement(tupleElement(childClaims, 2), indexOf(tupleElement(childClaims, 1), r.1))) > 0 OR greatest(0., r.4 - arrayElement(tupleElement(childClaims, 3), indexOf(tupleElement(childClaims, 1), r.1))) > 0, NOT has(reportingIds, r.2)), greatest(0., r.3 - arrayElement(tupleElement(childClaims, 2), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.4 - arrayElement(tupleElement(childClaims, 3), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.7 - arrayElement(tupleElement(childClaims, 4), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.8 - arrayElement(tupleElement(childClaims, 5), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.9 - arrayElement(tupleElement(childClaims, 6), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.10 - arrayElement(tupleElement(childClaims, 7), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.11 - arrayElement(tupleElement(childClaims, 8), indexOf(tupleElement(childClaims, 1), r.1)))), reporters) AS netted FROM (SELECT @@ -577,6 +595,9 @@ SELECT sum(errorAgentSpans) AS errorAgentSpans, sum(arrayCount(f -> f.3 = 1 AND NOT has(tupleElement(failedSpans, 2), f.1), failedSpans)) AS toolErrors, sum(arrayCount(f -> f.3 != 1 AND NOT has(tupleElement(failedSpans, 2), f.1), failedSpans)) AS turnErrors, + arraySlice(groupArrayArray(arrayFilter(f -> NOT has(tupleElement(failedSpans, 2), f.1), failedSpans)), 1, 100) AS failures, + argMax(traceId, traceAgentEndNanos) AS lastTraceId, + argMax(arrayExists(f -> f.3 != 1, failedSpans), traceAgentEndNanos) AS lastTraceTurnFailed, intDiv(max(traceAgentEndNanos) - toUnixTimestamp64Nano(min(traceAgentStart)), 1000000) AS agentDurationMs, arraySlice(arrayFlatten(groupArray(usageReporters)), 1, 2000) AS reporters, arrayReduce('sumMap', arrayMap(c -> [c.2], reporters), arrayMap(c -> [c.3], reporters), arrayMap(c -> [c.4], reporters), arrayMap(c -> [c.7], reporters), arrayMap(c -> [c.8], reporters), arrayMap(c -> [c.9], reporters), arrayMap(c -> [c.10], reporters), arrayMap(c -> [c.11], reporters)) AS childClaims, @@ -598,7 +619,7 @@ SELECT min(if(AgentName != '', Timestamp, toDateTime('2106-01-01 00:00:00'))) AS firstAgentAt, sum(IsToolCall) AS toolCalls, sum(IsError) AS errorAgentSpans, - groupArrayIf(2000)(tuple(SpanId, ParentSpanId, IsToolCall), IsError = 1) AS failedSpans, + groupArrayIf(2000)(tuple(SpanId, ParentSpanId, IsToolCall, IsLlmCall, ErrorType, ToolName, VendorId, StatusMessage, FailedToolCallResult, ResponseId, TraceId, toUnixTimestamp64Milli(Timestamp)), IsError = 1) AS failedSpans, groupArrayIf(2000)(tuple(SpanId, ParentSpanId, Tokens, Cost, ResponseId, IsLlmCall, InputTokens, CacheReadTokens, CacheWriteTokens, OutputTokens, ReasoningTokens), ((Tokens > 0 OR Cost > 0) OR IsLlmCall = 1)) AS usageReporters FROM ai_trace_index WHERE OrgId = 'org_sql_catalog' @@ -647,6 +668,9 @@ SELECT errorAgentSpans AS errorAgentSpans, toolErrors AS toolErrors, turnErrors AS turnErrors, + failures AS failures, + lastTraceId AS lastTraceId, + lastTraceTurnFailed AS lastTraceTurnFailed, agentDurationMs AS agentDurationMs, toFloat64(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 2)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, toFloat64(n.2)), arrayFilter(n -> n.1 != '', netted)))))) AS llmCalls, arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 3)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.3), arrayFilter(n -> n.1 != '', netted))))) AS totalTokens, @@ -672,6 +696,9 @@ SELECT errorAgentSpans AS errorAgentSpans, toolErrors AS toolErrors, turnErrors AS turnErrors, + failures AS failures, + lastTraceId AS lastTraceId, + lastTraceTurnFailed AS lastTraceTurnFailed, agentDurationMs AS agentDurationMs, arrayMap(r -> tuple(r.5, r.6 = 1 AND if((r.3 > 0 OR r.4 > 0), greatest(0., r.3 - arrayElement(tupleElement(childClaims, 2), indexOf(tupleElement(childClaims, 1), r.1))) > 0 OR greatest(0., r.4 - arrayElement(tupleElement(childClaims, 3), indexOf(tupleElement(childClaims, 1), r.1))) > 0, NOT has(reportingIds, r.2)), greatest(0., r.3 - arrayElement(tupleElement(childClaims, 2), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.4 - arrayElement(tupleElement(childClaims, 3), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.7 - arrayElement(tupleElement(childClaims, 4), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.8 - arrayElement(tupleElement(childClaims, 5), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.9 - arrayElement(tupleElement(childClaims, 6), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.10 - arrayElement(tupleElement(childClaims, 7), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.11 - arrayElement(tupleElement(childClaims, 8), indexOf(tupleElement(childClaims, 1), r.1)))), reporters) AS netted FROM (SELECT @@ -690,6 +717,9 @@ SELECT sum(errorAgentSpans) AS errorAgentSpans, sum(arrayCount(f -> f.3 = 1 AND NOT has(tupleElement(failedSpans, 2), f.1), failedSpans)) AS toolErrors, sum(arrayCount(f -> f.3 != 1 AND NOT has(tupleElement(failedSpans, 2), f.1), failedSpans)) AS turnErrors, + arraySlice(groupArrayArray(arrayFilter(f -> NOT has(tupleElement(failedSpans, 2), f.1), failedSpans)), 1, 100) AS failures, + argMax(traceId, traceAgentEndNanos) AS lastTraceId, + argMax(arrayExists(f -> f.3 != 1, failedSpans), traceAgentEndNanos) AS lastTraceTurnFailed, intDiv(max(traceAgentEndNanos) - toUnixTimestamp64Nano(min(traceAgentStart)), 1000000) AS agentDurationMs, arraySlice(arrayFlatten(groupArray(usageReporters)), 1, 2000) AS reporters, arrayReduce('sumMap', arrayMap(c -> [c.2], reporters), arrayMap(c -> [c.3], reporters), arrayMap(c -> [c.4], reporters), arrayMap(c -> [c.7], reporters), arrayMap(c -> [c.8], reporters), arrayMap(c -> [c.9], reporters), arrayMap(c -> [c.10], reporters), arrayMap(c -> [c.11], reporters)) AS childClaims, @@ -711,7 +741,7 @@ SELECT min(if(AgentName != '', Timestamp, toDateTime('2106-01-01 00:00:00'))) AS firstAgentAt, sum(IsToolCall) AS toolCalls, sum(IsError) AS errorAgentSpans, - groupArrayIf(2000)(tuple(SpanId, ParentSpanId, IsToolCall), IsError = 1) AS failedSpans, + groupArrayIf(2000)(tuple(SpanId, ParentSpanId, IsToolCall, IsLlmCall, ErrorType, ToolName, VendorId, StatusMessage, FailedToolCallResult, ResponseId, TraceId, toUnixTimestamp64Milli(Timestamp)), IsError = 1) AS failedSpans, groupArrayIf(2000)(tuple(SpanId, ParentSpanId, Tokens, Cost, ResponseId, IsLlmCall, InputTokens, CacheReadTokens, CacheWriteTokens, OutputTokens, ReasoningTokens), ((Tokens > 0 OR Cost > 0) OR IsLlmCall = 1)) AS usageReporters FROM ai_trace_index WHERE OrgId = 'org_sql_catalog' diff --git a/packages/query-engine-integrations/src/ai/ai-sessions.test.ts b/packages/query-engine-integrations/src/ai/ai-sessions.test.ts index eda4c29ad..2e5bec4c6 100644 --- a/packages/query-engine-integrations/src/ai/ai-sessions.test.ts +++ b/packages/query-engine-integrations/src/ai/ai-sessions.test.ts @@ -277,6 +277,26 @@ describe("aiSessionPageQuery", () => { errorAgentSpans: "1", toolErrors: 1, turnErrors: 0, + // A tuple on the JSON wire is an array; the 64-bit instant is quoted + // where the setting is refused, like every other UInt64/Int64. + failures: [ + [ + "span-tool-1", + "span-agent-1", + 1, + 0, + "TimeoutError", + "search_traces", + "eve", + "search timed out", + "", + "", + "trace-1", + "1755599605825", + ], + ], + lastTraceId: "trace-3", + lastTraceTurnFailed: 0, totalTokens: 184_320, inputTokens: 120_000, cacheReadTokens: 60_000, @@ -305,6 +325,24 @@ describe("aiSessionPageQuery", () => { errorAgentSpans: 1, toolErrors: 1, turnErrors: 0, + failures: [ + [ + "span-tool-1", + "span-agent-1", + 1, + 0, + "TimeoutError", + "search_traces", + "eve", + "search timed out", + "", + "", + "trace-1", + 1_755_599_605_825, + ], + ], + lastTraceId: "trace-3", + lastTraceTurnFailed: 0, totalTokens: 184_320, inputTokens: 120_000, cacheReadTokens: 60_000, @@ -368,10 +406,14 @@ describe("aiSessionPageQuery", () => { // The session's reporters, every trace's flattened, so a gateway's mirror // trace of a call is in hand next to the app's own span of it — and the // two lookups the netting makes, taken off them once per session. - expect(sessions).toContain("arraySlice(arrayFlatten(groupArray(usageReporters)), 1, 2000) AS reporters") + expect(sessions).toContain( + "arraySlice(arrayFlatten(groupArray(usageReporters)), 1, 2000) AS reporters", + ) expect(sessions).toContain("arrayReduce('sumMap', arrayMap(c -> [c.2], reporters)") expect(sessions).toContain(") AS childClaims") - expect(sessions).toContain("tupleElement(arrayFilter(p -> p.3 > 0 OR p.4 > 0, reporters), 1) AS reportingIds") + expect(sessions).toContain( + "tupleElement(arrayFilter(p -> p.3 > 0 OR p.4 > 0, reporters), 1) AS reportingIds", + ) expect(sessions).toContain( "intDiv(max(traceAgentEndNanos) - toUnixTimestamp64Nano(min(traceAgentStart)), 1000000) AS agentDurationMs", ) @@ -425,7 +467,7 @@ describe("aiSessionPageQuery", () => { const { sessions: outer, traces: inner } = levels(sql) expect(inner).toContain( - "groupArrayIf(2000)(tuple(SpanId, ParentSpanId, IsToolCall), IsError = 1) AS failedSpans", + "groupArrayIf(2000)(tuple(SpanId, ParentSpanId, IsToolCall, IsLlmCall, ErrorType, ToolName, VendorId, StatusMessage, FailedToolCallResult, ResponseId, TraceId, toUnixTimestamp64Milli(Timestamp)), IsError = 1) AS failedSpans", ) // A failed span whose own child also failed is the child's echo, not a // second failure — the turn span a framework fails alongside its call. @@ -435,6 +477,15 @@ describe("aiSessionPageQuery", () => { expect(outer).toContain( "sum(arrayCount(f -> f.3 != 1 AND NOT has(tupleElement(failedSpans, 2), f.1), failedSpans)) AS turnErrors", ) + // The same deepest spans, kept for the row's breakdown and capped; and + // whether the last trace's turn failed, off every failed span of it. + expect(outer).toContain( + "arraySlice(groupArrayArray(arrayFilter(f -> NOT has(tupleElement(failedSpans, 2), f.1), failedSpans)), 1, 100) AS failures", + ) + expect(outer).toContain("argMax(traceId, traceAgentEndNanos) AS lastTraceId") + expect(outer).toContain( + "argMax(arrayExists(f -> f.3 != 1, failedSpans), traceAgentEndNanos) AS lastTraceTurnFailed", + ) }) it("filters the ranked row with HAVING, after the session grouping", () => { @@ -497,7 +548,9 @@ describe("aiSessionPageQuery", () => { expect(byCost.split("ORDER BY").length - 1).toBe(1) // A session-level sort ranks and cuts the page before the netting. const byDuration = compileUnsafe(aiSessionPageQuery({ sortBy: "durationMs" }), params).sql - expect(byDuration.split("ORDER BY agentDurationMs DESC, agentStart DESC, sessionId ASC").length - 1).toBe(2) + expect( + byDuration.split("ORDER BY agentDurationMs DESC, agentStart DESC, sessionId ASC").length - 1, + ).toBe(2) expect(byDuration.indexOf("LIMIT 50")).toBeLessThan(byDuration.indexOf(") AS ranked_sessions")) expect(compileUnsafe(aiSessionPageQuery({ sortBy: "errorSpanCount" }), params).sql).toContain( "ORDER BY errorAgentSpans DESC, agentStart DESC, sessionId ASC", @@ -1379,7 +1432,12 @@ describe("aiSessionSummaryQuery", () => { it("reads usage across every vendor spelling, per call and in total", () => { const { sql } = compileUnsafe(aiSessionSummaryQuery(), summaryParams) - for (const key of ["gen_ai.usage.input_tokens", "gen_ai.usage.prompt_tokens", "ai.usage.inputTokens", "llm.token_count.prompt"]) { + for (const key of [ + "gen_ai.usage.input_tokens", + "gen_ai.usage.prompt_tokens", + "ai.usage.inputTokens", + "llm.token_count.prompt", + ]) { expect(sql, key).toContain(`SpanAttributes['${key}']`) } expect(sql).toContain("AS inputTokens") @@ -1406,7 +1464,9 @@ describe("aiSessionSummaryQuery", () => { it("guards every usage sum against a non-finite attribute", () => { const { sql } = compileUnsafe(aiSessionSummaryQuery(), summaryParams) for (const alias of ["inputTokens", "llmInputTokens", "cost", "llmCost"]) { - expect(sql, alias).toMatch(new RegExp(`ifNotFinite\\(sum(If)?\\(toFloat64OrZero\\([^\\n]*, 0\\) AS ${alias},`)) + expect(sql, alias).toMatch( + new RegExp(`ifNotFinite\\(sum(If)?\\(toFloat64OrZero\\([^\\n]*, 0\\) AS ${alias},`), + ) } }) @@ -1455,6 +1515,12 @@ describe("aiSessionSummaryQuery", () => { }, ]) - expect(row).toMatchObject({ spanCount: 12, durationMs: 1000, inputTokens: 300, cost: 0.0123, models: ["gpt-5"] }) + expect(row).toMatchObject({ + spanCount: 12, + durationMs: 1000, + inputTokens: 300, + cost: 0.0123, + models: ["gpt-5"], + }) }) }) diff --git a/packages/query-engine-integrations/src/ai/ai-sessions.ts b/packages/query-engine-integrations/src/ai/ai-sessions.ts index 32b5f9a09..15c748528 100644 --- a/packages/query-engine-integrations/src/ai/ai-sessions.ts +++ b/packages/query-engine-integrations/src/ai/ai-sessions.ts @@ -241,13 +241,19 @@ export const sessionKey = (rawSessionId: CH.Expr, traceId: CH.Expr @@ -256,9 +262,62 @@ const failedSpansExpr = ($: { readonly IsError: CH.Expr }): CH.Expr => CH.untypedExpr( - `groupArrayIf(${MAX_USAGE_REPORTERS_PER_TRACE})(tuple(SpanId, ParentSpanId, IsToolCall), IsError = 1)`, + `groupArrayIf(${MAX_USAGE_REPORTERS_PER_TRACE})(tuple(SpanId, ParentSpanId, IsToolCall, IsLlmCall, ErrorType, ToolName, VendorId, StatusMessage, FailedToolCallResult, ResponseId, TraceId, toUnixTimestamp64Milli(Timestamp)), IsError = 1)`, ) +/** The most failures a page row ships for its breakdown — the deepest failed + * spans of the session, in no order. A session past it is triaged on its + * first hundred, which is what a chip and a hover can say anyway. */ +const MAX_FAILURES_PER_SESSION = 100 + +/** + * The session's deepest failed spans (`deepestFailureCount`'s filter, kept + * rather than counted), flattened across its traces and cut at + * `MAX_FAILURES_PER_SESSION`. Decoded as the tuple's positions. + */ +const sessionFailuresExpr = (failedSpans: string): CH.Expr => + CH.rawExpr( + `arraySlice(groupArrayArray(arrayFilter(f -> NOT has(tupleElement(${failedSpans}, 2), f.1), ${failedSpans})), 1, ${MAX_FAILURES_PER_SESSION})`, + FAILED_SPAN_TUPLES, + ) + +/** `(spanId, parentSpanId, isToolCall, isLlmCall, errorType, toolName, + * vendorId, statusMessage, failedToolCallResult, responseId, traceId, atMs)` + * — an element of `failedSpansExpr`, as the JSON wire renders a tuple. */ +export type IndexFailedSpanTuple = readonly [ + string, + string, + number, + number, + string, + string, + string, + string, + string, + string, + string, + number, +] +const FAILED_SPAN_TUPLES = T.array( + T.custom( + "Tuple(String, String, UInt8, UInt8, LowCardinality(String), LowCardinality(String), LowCardinality(String), String, String, String, String, Int64)", + Schema.Tuple([ + Schema.String, + Schema.String, + CHNumber, + CHNumber, + Schema.String, + Schema.String, + Schema.String, + Schema.String, + Schema.String, + Schema.String, + Schema.String, + CHNumber, + ]), + ), +) + /** * Failed spans of one kind, summed over the traces' `failedSpans`, with a * failed span whose own child also failed left out: the child is the failure, @@ -383,6 +442,13 @@ export interface AiSessionPageOutput { readonly toolErrors: number /** Failed model calls and turn spans that failed on their own — the rest. */ readonly turnErrors: number + /** The deepest failed spans, for the row's breakdown — see `sessionFailuresExpr`. */ + readonly failures: readonly IndexFailedSpanTuple[] + /** The trace whose agent spans end last — where the session's final turn is. */ + readonly lastTraceId: string + /** That trace had a failed span that was not a tool call — the session's + * last turn did not close cleanly, one trace deep. 0/1. */ + readonly lastTraceTurnFailed: number /** Tokens across every bucket, deepest reporter counted, one claim per response id — see `sessionUsageSum`. */ readonly totalTokens: number // The five disjoint buckets `totalTokens` is the sum of, counted the same @@ -596,6 +662,9 @@ const SESSION_COLUMNS = [ "errorAgentSpans", "toolErrors", "turnErrors", + "failures", + "lastTraceId", + "lastTraceTurnFailed", "agentDurationMs", ] as const type SessionColumn = (typeof SESSION_COLUMNS)[number] @@ -640,6 +709,12 @@ const indexSessions = (opts: AiSessionFilterOpts) => errorAgentSpans: CH.sum($.errorAgentSpans), toolErrors: deepestFailureCount("failedSpans", "tool"), turnErrors: deepestFailureCount("failedSpans", "turn"), + failures: sessionFailuresExpr("failedSpans"), + lastTraceId: CH.argMax($.traceId, $.traceAgentEndNanos), + lastTraceTurnFailed: CH.rawExpr( + `argMax(arrayExists(f -> f.3 != 1, failedSpans), traceAgentEndNanos)`, + T.uint8, + ), // Nanoseconds first, wrapped in `intDiv` — see `durationMs` in // `aiSessionDetailsQuery` for both. agentDurationMs: CH.intDiv( @@ -769,8 +844,7 @@ export function aiSessionPageQuery(opts: AiSessionPageOpts = {}) { ]) // A String order, and a correct one: the literal is fixed-width // `YYYY-MM-DD hh:mm:ss.nnnnnnnnn`, so it sorts as the instant does. - const ranked = - sortsOnSession(order) && !filtersOnUsage ? paged(sessions.orderBy(...order)) : sessions + const ranked = sortsOnSession(order) && !filtersOnUsage ? paged(sessions.orderBy(...order)) : sessions const netted = fromQuery(ranked, "ranked_sessions").select(($) => ({ ...carry($), @@ -840,7 +914,8 @@ export function aiSessionDetailsQuery(opts: AiSessionDetailsOpts) { // its page is empty — see `AiSessionDetailsOpts`. if (opts.sessionIds.length === 0) { throw new QueryBuilderDefect({ - message: "aiSessionDetailsQuery needs the page's session ids; an empty page has nothing to detail", + message: + "aiSessionDetailsQuery needs the page's session ids; an empty page has nothing to detail", }) } // The page's traces, keyed as the page keyed them — read twice below, once @@ -944,8 +1019,7 @@ const NANOS_PER_MS = 1_000_000n const warehouseNanos = (literal: string): bigint => { const [datetime = "", fraction = ""] = literal.split(".") return ( - BigInt(Date.parse(`${datetime.replace(" ", "T")}Z`)) * NANOS_PER_MS + - BigInt(fraction.padEnd(9, "0")) + BigInt(Date.parse(`${datetime.replace(" ", "T")}Z`)) * NANOS_PER_MS + BigInt(fraction.padEnd(9, "0")) ) } @@ -1615,15 +1689,15 @@ const summaryMeasures_ = ($: SpanColumns) => { const model = attr([...aiFieldSourceKeys("responseModel"), ...aiFieldSourceKeys("requestModel")]) const toolName = field("toolName") const agentName = field("agentName") - const isLlmCall = operation - .in_(...AI_INFERENCE_OPERATIONS) - .or( - operation - .notIn(...AI_RETRIEVAL_OPERATIONS, ...AI_TOOL_OPERATIONS, ...AI_AGENT_OPERATIONS) - .and(model.neq("")) - .and(toolName.eq("")), - ) - const isToolCall = operation.in_(...AI_TOOL_OPERATIONS).or(operation.eq("").and(isAi).and(toolName.neq(""))) + const isLlmCall = operation.in_(...AI_INFERENCE_OPERATIONS).or( + operation + .notIn(...AI_RETRIEVAL_OPERATIONS, ...AI_TOOL_OPERATIONS, ...AI_AGENT_OPERATIONS) + .and(model.neq("")) + .and(toolName.eq("")), + ) + const isToolCall = operation + .in_(...AI_TOOL_OPERATIONS) + .or(operation.eq("").and(isAi).and(toolName.neq(""))) // The list query's error rule, so the summary and the list badge agree. const failed = $.StatusCode.eq("Error").or( isAi.and( diff --git a/packages/query-engine-integrations/src/ai/index.ts b/packages/query-engine-integrations/src/ai/index.ts index d2dc2b4f1..6fcf02551 100644 --- a/packages/query-engine-integrations/src/ai/index.ts +++ b/packages/query-engine-integrations/src/ai/index.ts @@ -41,6 +41,7 @@ export { type AiSessionSummaryOutput, type AiSessionTotalsOutput, type AiSessionWindowOutput, + type IndexFailedSpanTuple, } from "./ai-sessions" export { From e6e7d58af5027e6047e6216797c291e91ae4ebfd Mon Sep 17 00:00:00 2001 From: JeremyFunk Date: Fri, 18 Sep 2026 01:41:21 +0200 Subject: [PATCH 2/3] test(api): stub the page row's failure tuple in the ai-sessions list route test --- .../routes/internal/ai-sessions.http.test.ts | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/apps/api/src/routes/internal/ai-sessions.http.test.ts b/apps/api/src/routes/internal/ai-sessions.http.test.ts index 09bd33752..bc77808ad 100644 --- a/apps/api/src/routes/internal/ai-sessions.http.test.ts +++ b/apps/api/src/routes/internal/ai-sessions.http.test.ts @@ -454,6 +454,26 @@ describe("POST /internal/ai-sessions/list", () => { errorAgentSpans: "1", toolErrors: 1, turnErrors: 0, + // The one failed span, as the wire renders the tuple: a tool that timed + // out, in a trace that is not the session's last, so a warning. + failures: [ + [ + "span-tool-1", + "span-agent-1", + 1, + 0, + "TimeoutError", + "search_traces", + "eve", + "search timed out", + "", + "", + "trace-1", + "1755599605825", + ], + ], + lastTraceId: "trace-2", + lastTraceTurnFailed: 0, totalTokens: 18_400, inputTokens: 12_000, cacheReadTokens: 4_000, @@ -523,6 +543,17 @@ describe("POST /internal/ai-sessions/list", () => { errorSpanCount: 1, toolErrorCount: 1, turnErrorCount: 0, + // Classified off the shipped tuple by the detail page's own rule. + failures: [ + { + kind: "toolTimeout", + label: "tool_timeout · search_traces", + tool: "search_traces", + count: 1, + severity: "anomaly", + terminal: false, + }, + ], models: ["claude-sonnet-5"], agentNames: ["web-fetcher", "slack-agent"], firstAgentName: "slack-agent", From 11c84e91d0ef386dce3479859cd71b1397348e05 Mon Sep 17 00:00:00 2001 From: JeremyFunk Date: Fri, 18 Sep 2026 02:04:01 +0200 Subject: [PATCH 3/3] fix(agent-sessions): grade the list's failures by span identity, and ship less of them MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review round on #927. A failure with no tool — a rate limit, a context overflow, most of them — 500'd the list: the summary carried `tool: undefined` as a present key, and the wire schema's `optionalKey` rejects that. The key is now absent, and the route test stubs a tool-less failure so the encode path is covered. The verdict is now by identity. The page query resolves ONE span the last turn died on (`terminalSpanId`): a turn-root failure in the trace — a failed span that is neither a tool nor a model call, or a failed root — then that trace's last deepest failure, resolved over the session's last non-mirror trace, so a retried-and-recovered model call is amber as the Overview has it, and an OpenRouter mirror trace ending last cannot take the verdict with it. `summarizeIndexFailures` marks the group holding that span, under whichever observation the dedupe kept; a span the query did not ship marks nothing, never a neighbour by position. The page query collects less: the split and the verdict read a thin `(SpanId, ParentSpanId, IsToolCall, IsLlmCall, atMs)` tuple with the deepest filter hoisted to one pass per trace, and only the page query collects the detail tuple, 50 per trace with both texts clipped to 400 chars — the details fan-out and the distributions no longer materialize 2000 × 1.4 KB per trace they never read. The shipped breakdown is the latest 100 deepest failures, ordered before the cut. The list span annotates how many failures a page classified and how many rows hit the cap. One column list now generates the detail tuple's SQL, its ClickHouse type and the decoder beside them (`indexFailedSpanFromTuple`), with a positional test; the failed-span record and the severity literal live in the domain and the agent-sessions types alias them, like the kind already did. UI: the TanStack column def was still 100px (only the layout table had moved), so the chip overflowed; both are 140 now, the lab's width stops follow. Chips carry no icon — severity is the colour, as on the Overview, and a red chip may be a tool — and their accessible name is the breakdown, so it is reachable without a hover. A row whose failures the index could not classify shows its errored-span count instead of an empty cell. The MCP cell cuts on a label boundary with `+N more`. Documented, not fixed: the index carries no `gen_ai.response.status` and no provider-attempt marker, so a rate limit reported only there is a plain `error` here, and a gateway's attempts count where the Overview folds them. --- .../tools/__tests__/agent-sessions.test.ts | 20 +- apps/ai/src/mcp/tools/list-agent-sessions.ts | 27 +- .../routes/internal/ai-sessions.http.test.ts | 32 ++- .../agent-sessions-list.test.tsx | 18 ++ .../agent-sessions/agent-sessions-list.tsx | 74 ++--- apps/web/src/lab/agent-sessions-list-lab.tsx | 4 +- .../agent-sessions/src/index-failures.test.ts | 139 +++++++--- packages/agent-sessions/src/index-failures.ts | 119 ++++---- .../agent-sessions/src/session-findings.ts | 4 +- .../services/ai-sessions/ai-session-reads.ts | 38 ++- ...dex-materialization.clickhouse.e2e.test.ts | 41 +-- packages/domain/src/http/ai-sessions.ts | 31 ++- .../src/__sql_baseline__/integrations.sql | 123 ++++++--- .../src/ai/ai-sessions.test.ts | 66 ++++- .../src/ai/ai-sessions.ts | 259 +++++++++++++----- .../query-engine-integrations/src/ai/index.ts | 2 + 16 files changed, 674 insertions(+), 323 deletions(-) diff --git a/apps/ai/src/mcp/tools/__tests__/agent-sessions.test.ts b/apps/ai/src/mcp/tools/__tests__/agent-sessions.test.ts index 770d993db..1f5725838 100644 --- a/apps/ai/src/mcp/tools/__tests__/agent-sessions.test.ts +++ b/apps/ai/src/mcp/tools/__tests__/agent-sessions.test.ts @@ -61,6 +61,23 @@ const listRow = { errorAgentSpans: 1, toolErrors: 1, turnErrors: 0, + failures: [ + [ + "span-tool-1", + "span-agent-1", + 1, + 0, + "TimeoutError", + "search_traces", + "eve", + "search timed out", + "", + "", + "trace-1", + 1755599605825, + ], + ], + terminalSpanId: "", serviceNames: ["agent-runner"], models: ["gpt-5"], agentNames: ["maple"], @@ -291,7 +308,8 @@ describe("list_agent_sessions rendering", () => { expect(cells?.[1]).toBe("maple") expect(cells?.[2]).toBe("eve") // Errors are agent/tool/turn, in that order. - expect(cells?.[7]).toBe("1/1/0") + // The breakdown by label, not the raw counts, once the index classified one. + expect(cells?.[7]).toBe("tool_timeout · search_traces") // The hint is the row's bounds PADDED the way the page pads them: a row's // bounds are its agent spans' extent, and both read levels bound on // `Timestamp`, so handing them over verbatim would drop the app spans diff --git a/apps/ai/src/mcp/tools/list-agent-sessions.ts b/apps/ai/src/mcp/tools/list-agent-sessions.ts index d0727e061..9651fb8c1 100644 --- a/apps/ai/src/mcp/tools/list-agent-sessions.ts +++ b/apps/ai/src/mcp/tools/list-agent-sessions.ts @@ -14,6 +14,7 @@ import { formatNextSteps } from "../lib/next-steps" import { windowHint } from "../lib/agent-sessions" import { Effect, Schema } from "effect" import { + type AiSessionFailureSummary, AI_SESSION_SEARCH_MAX_CHARS, AiSessionSortDir, AiSessionSortKey, @@ -22,7 +23,6 @@ import { RangeBound, } from "@maple/domain/http" import { formatCost } from "@maple/agent-sessions" -import type { AiSessionFailureSummary } from "@maple/domain/http" import { splitCsv } from "@maple/domain/where-clause" import { listAiSessions } from "@maple/backend/services/ai-sessions/ai-session-reads" import { warehouseReadToMcpHandlers } from "../lib/map-warehouse-error" @@ -204,17 +204,22 @@ export function registerListAgentSessionsTool(server: McpToolRegistrar) { } /** `!context_length_exceeded, tool_error · run_tests ×2` — the row's failures - * by label, a `!` on each one that needs a fix. `undefined` when the index - * classified none, and the raw counts say what it saw. */ + * by label, a `!` on each one that needs a fix, whole labels only and + * `+N more` past the cell's width. `undefined` when the index classified + * none, and the raw counts say what it saw. */ function failuresCell(failures: ReadonlyArray): string | undefined { if (failures.length === 0) return undefined - return truncate( - failures - .map( - (failure) => - `${failure.severity === "failure" ? "!" : ""}${failure.label}${failure.count > 1 ? ` ×${failure.count}` : ""}`, - ) - .join(", "), - 80, + const labels = failures.map( + (failure) => + `${failure.severity === "failure" ? "!" : ""}${failure.label}${failure.count > 1 ? ` ×${failure.count}` : ""}`, ) + const shown: string[] = [] + for (const label of labels) { + const next = [...shown, label].join(", ") + if (shown.length > 0 && next.length > FAILURES_CELL_MAX) break + shown.push(label) + } + const more = labels.length - shown.length + return shown.join(", ") + (more > 0 ? ` +${more} more` : "") } +const FAILURES_CELL_MAX = 80 diff --git a/apps/api/src/routes/internal/ai-sessions.http.test.ts b/apps/api/src/routes/internal/ai-sessions.http.test.ts index bc77808ad..b035437d1 100644 --- a/apps/api/src/routes/internal/ai-sessions.http.test.ts +++ b/apps/api/src/routes/internal/ai-sessions.http.test.ts @@ -454,9 +454,25 @@ describe("POST /internal/ai-sessions/list", () => { errorAgentSpans: "1", toolErrors: 1, turnErrors: 0, - // The one failed span, as the wire renders the tuple: a tool that timed - // out, in a trace that is not the session's last, so a warning. + // The failed spans, as the wire renders the tuple: a rate-limited model + // call — no tool, so the summary carries no `tool` key at all — and a + // tool that timed out, both in a trace that is not the session's last, + // so both warnings. failures: [ + [ + "span-llm-1", + "span-agent-1", + 0, + 1, + "", + "", + "eve", + "429 Too Many Requests", + "", + "", + "trace-1", + "1755599604825", + ], [ "span-tool-1", "span-agent-1", @@ -472,8 +488,7 @@ describe("POST /internal/ai-sessions/list", () => { "1755599605825", ], ], - lastTraceId: "trace-2", - lastTraceTurnFailed: 0, + terminalSpanId: "", totalTokens: 18_400, inputTokens: 12_000, cacheReadTokens: 4_000, @@ -543,8 +558,15 @@ describe("POST /internal/ai-sessions/list", () => { errorSpanCount: 1, toolErrorCount: 1, turnErrorCount: 0, - // Classified off the shipped tuple by the detail page's own rule. + // Classified off the shipped tuples by the detail page's own rule. failures: [ + { + kind: "rateLimited", + label: "rate_limit", + count: 1, + severity: "anomaly", + terminal: false, + }, { kind: "toolTimeout", label: "tool_timeout · search_traces", diff --git a/apps/web/src/components/agent-sessions/agent-sessions-list.test.tsx b/apps/web/src/components/agent-sessions/agent-sessions-list.test.tsx index 4d529b364..7ff1e2e22 100644 --- a/apps/web/src/components/agent-sessions/agent-sessions-list.test.tsx +++ b/apps/web/src/components/agent-sessions/agent-sessions-list.test.tsx @@ -190,6 +190,24 @@ describe("AgentSessionsList", () => { element?.classList.contains("rounded-full") === true && element.textContent === label expect(view.getAllByText(chip("1 failure"))).toHaveLength(1) expect(view.getAllByText(chip("2 warnings"))).toHaveLength(1) + // The breakdown is the chip's accessible name: the labels exist nowhere + // else until the tooltip opens. + expect(view.getByLabelText("context_length_exceeded (ended the run)")).toBeTruthy() + expect(view.getByLabelText("tool_error · run_tests ×2")).toBeTruthy() + }) + + it("counts the errored spans when the index classified none of them", () => { + const view = renderList( + , + ) + const chip = (label: string) => (_: string, element: Element | null) => + element?.classList.contains("rounded-full") === true && element.textContent === label + expect(view.getAllByText(chip("3 spans"))).toHaveLength(1) expect(view.getByText("18.4k")).toBeTruthy() expect(view.getByText("maple-slack-agent")).toBeTruthy() }) diff --git a/apps/web/src/components/agent-sessions/agent-sessions-list.tsx b/apps/web/src/components/agent-sessions/agent-sessions-list.tsx index 3b635b08a..16b770741 100644 --- a/apps/web/src/components/agent-sessions/agent-sessions-list.tsx +++ b/apps/web/src/components/agent-sessions/agent-sessions-list.tsx @@ -17,13 +17,7 @@ import { formatRelativeTimeOrDate, toEpochMs } from "@maple/ui/lib/time-format" import { formatSessionDuration } from "@maple/ui/lib/replay-format" import { formatCount } from "@maple/ui/components/filters/range-filter-section" import { cn } from "@maple/ui/lib/utils" -import { - FaceRobotIcon, - GearIcon, - PixelSparkleIcon, - SquareSparkleIcon, - type IconComponent, -} from "@/components/icons" +import { GearIcon, PixelSparkleIcon, SquareSparkleIcon, type IconComponent } from "@/components/icons" import { ServicePills } from "@/components/common/service-pills" import { SortableHeader } from "@/components/common/sortable-header" import { useDetectedModels } from "@/hooks/use-detected-models" @@ -403,7 +397,7 @@ export function AgentSessionsList({ "errorSpanCount", "Failures the run needs fixed, and warnings it survived", ), - size: 100, + size: 140, cell: ({ row }) => , }, { @@ -583,15 +577,19 @@ export function AgentSessionsList({ function Hint({ content, className, + label, children, }: { content: ReactNode className?: string + /** The tooltip's words for a reader who cannot hover, when the trigger's + * own text does not carry them. */ + label?: string children: ReactNode }) { return ( - } className={className}> + } className={className} aria-label={label}> {children} {content} @@ -810,13 +808,24 @@ function ErrorChips({ session }: { session: AgentSessionRow }) { } const failures = session.failures.filter((failure) => failure.severity === "failure") const warnings = session.failures.filter((failure) => failure.severity === "anomaly") - const classified = session.toolErrorCount + session.turnErrorCount - const other = session.errorSpanCount - classified + // Nothing classified: the errored spans are outside the agent's own (the + // details read counts every span), or the row predates the breakdown. + if (failures.length === 0 && warnings.length === 0) { + return ( + + ) + } + // Severity is the colour, as on the Overview's checklist: no icon, because + // a red chip may be a tool and an amber one a model call. return (
{failures.length > 0 && ( } + label={breakdownText(failures)} className="border-destructive/30 bg-destructive/10 text-destructive" /> )} {warnings.length > 0 && ( } + label={breakdownText(warnings)} className="border-severity-warn/40 bg-severity-warn/10 text-severity-warn" /> )} - {classified === 0 && other > 0 && ( - - )}
) } @@ -852,15 +854,24 @@ function ErrorChips({ session }: { session: AgentSessionRow }) { const sumCounts = (rows: ReadonlyArray) => rows.reduce((total, row) => total + row.count, 0) +/** The breakdown as one line, for the chip's accessible name. */ +const breakdownText = (rows: ReadonlyArray) => + rows + .map( + (row) => + `${row.label}${row.count > 1 ? ` ×${row.count}` : ""}${row.terminal ? " (ended the run)" : ""}`, + ) + .join(", ") + /** The hover: one line per label, in the Overview's own words and order. */ function FailureBreakdown({ rows, lede }: { rows: ReadonlyArray; lede: string }) { return ( -
+
{lede}
    {rows.map((row) => (
  • - {row.label} + {row.label} {row.count > 1 && ( ×{row.count} )} @@ -873,16 +884,16 @@ function FailureBreakdown({ rows, lede }: { rows: ReadonlyArray - {Icon ? ( - - ) : ( - - )} + {/* Two digits of room, and the noun always as wide as its plural: a - row's "1 tool" above the next row's "12 tools" otherwise makes two - chips that never line up. Past 99 the chip does widen — a third + row's "1 failure" above the next row's "12 failures" otherwise makes + two chips that never line up. Past 99 the chip does widen — a third digit costs every row space for a count almost no session reaches. */} {count}{" "} - {count === 1 ? noun : `${noun}s`} + {count === 1 ? noun : `${noun}s`} ) diff --git a/apps/web/src/lab/agent-sessions-list-lab.tsx b/apps/web/src/lab/agent-sessions-list-lab.tsx index 8b5493796..130de0346 100644 --- a/apps/web/src/lab/agent-sessions-list-lab.tsx +++ b/apps/web/src/lab/agent-sessions-list-lab.tsx @@ -39,9 +39,9 @@ import { buildToolAnalyticsFixture, buildToolCells } from "./agent-tools-fixture * into the Session cell. */ const WIDTHS = [ { label: "Full", value: null }, - { label: "1300px", value: 1300 }, + { label: "1340px", value: 1340 }, { label: "1000px", value: 1000 }, - { label: "700px", value: 700 }, + { label: "740px", value: 740 }, { label: "380px", value: 380 }, ] as const diff --git a/packages/agent-sessions/src/index-failures.test.ts b/packages/agent-sessions/src/index-failures.test.ts index bb387c9f3..a0713c4b0 100644 --- a/packages/agent-sessions/src/index-failures.test.ts +++ b/packages/agent-sessions/src/index-failures.test.ts @@ -1,5 +1,7 @@ import { describe, expect, it } from "vitest" import { summarizeIndexFailures, type IndexFailedSpan } from "./index-failures" +import { failureSeverity } from "./session-findings" +import type { SessionFailureKind } from "./session-summary" const failed = (overrides: Partial & { spanId: string }): IndexFailedSpan => ({ traceId: "trace-1", @@ -38,25 +40,17 @@ describe("summarizeIndexFailures", () => { atMs: 4_000, }), ], - { traceId: "trace-1", turnFailed: false }, + "", ) expect(rows).toEqual([ { kind: "contextExceeded", label: "context_length_exceeded", - tool: undefined, count: 1, severity: "failure", terminal: false, }, - { - kind: "rateLimited", - label: "rate_limit", - tool: undefined, - count: 2, - severity: "anomaly", - terminal: false, - }, + { kind: "rateLimited", label: "rate_limit", count: 2, severity: "anomaly", terminal: false }, { kind: "error", label: "tool_error · run_tests", @@ -66,26 +60,29 @@ describe("summarizeIndexFailures", () => { terminal: false, }, ]) + // No `tool` key at all on the rows without one: the wire schema's + // `optionalKey` rejects a present `undefined`. + expect(Object.hasOwn(rows[0]!, "tool")).toBe(false) }) - it("marks the last failure of the last trace terminal when that trace's turn failed, and reds it", () => { + it("marks the group holding the terminal span, and reds it whatever its kind", () => { const rows = summarizeIndexFailures( [ failed({ spanId: "early", traceId: "trace-1", statusMessage: "rate limit", atMs: 1_000 }), failed({ spanId: "late", traceId: "trace-2", errorType: "provider_error", atMs: 9_000 }), failed({ spanId: "later", traceId: "trace-2", statusMessage: "rate limit", atMs: 9_500 }), ], - { traceId: "trace-2", turnFailed: true }, + "later", ) - // The rate limit the run died on leads, red; the provider error it - // survived is amber even though it is in the same trace. + // The rate limit the run died on leads, red, its count whole; the + // provider error it survived is amber even though it is in the same trace. expect(rows.map((row) => [row.label, row.severity, row.terminal, row.count])).toEqual([ ["rate_limit", "failure", true, 2], ["provider_error", "anomaly", false, 1], ]) }) - it("leaves a survived last trace amber whatever it failed on", () => { + it("leaves everything amber when the last turn closed cleanly, whatever failed", () => { const rows = summarizeIndexFailures( [ failed({ @@ -96,7 +93,7 @@ describe("summarizeIndexFailures", () => { errorType: "tool_error", }), ], - { traceId: "trace-1", turnFailed: false }, + "", ) expect(rows).toEqual([ { @@ -110,6 +107,14 @@ describe("summarizeIndexFailures", () => { ]) }) + it("marks nothing for a terminal span it was not shipped", () => { + const rows = summarizeIndexFailures( + [failed({ spanId: "shipped", statusMessage: "rate limit" })], + "dropped", + ) + expect(rows.map((row) => [row.severity, row.terminal])).toEqual([["anomaly", false]]) + }) + it("counts a call the app and a gateway mirror both observed once, keeping the observation that named the cause", () => { const rows = summarizeIndexFailures( [ @@ -123,34 +128,94 @@ describe("summarizeIndexFailures", () => { atMs: 1_001, }), ], - { traceId: "mirror-trace", turnFailed: false }, + "", ) expect(rows).toEqual([ - { - kind: "rateLimited", - label: "rate_limit", - tool: undefined, - count: 1, - severity: "anomaly", - terminal: false, - }, + { kind: "rateLimited", label: "rate_limit", count: 1, severity: "anomaly", terminal: false }, ]) }) + it("finds the terminal span under whichever observation the dedupe kept", () => { + const rows = summarizeIndexFailures( + [ + failed({ spanId: "app", responseId: "gen-1", errorType: "provider_error", atMs: 1_000 }), + failed({ + spanId: "mirror", + traceId: "mirror-trace", + vendorId: "openrouter", + responseId: "gen-1", + statusMessage: "Provider overloaded, retry later", + atMs: 1_001, + }), + ], + // The query resolved the app's trace as the last one; the mirror's + // observation is the one the dedupe kept. + "app", + ) + expect(rows.map((row) => [row.label, row.severity, row.terminal])).toEqual([ + ["rate_limit", "failure", true], + ]) + }) + + it("reads a failed tool call's result as the span carries it, JSON where it parses", () => { + const rows = summarizeIndexFailures( + [ + failed({ + spanId: "t", + isToolCall: true, + isLlmCall: false, + toolName: "submit_findings", + errorType: "tool_error", + failedToolCallResult: JSON.stringify({ + result: 'Invalid tool input: Missing key\n at ["evidence"][0]["traceIds"]', + }), + }), + ], + "", + ) + expect(rows.map((row) => row.label)).toEqual(["tool_arguments · submit_findings"]) + }) + + it("orders the same rows the same whatever order they arrive in", () => { + const a = failed({ spanId: "a", statusMessage: "rate limit", atMs: 5_000 }) + const b = failed({ spanId: "b", errorType: "provider_error", atMs: 5_000 }) + expect(summarizeIndexFailures([a, b], "")).toEqual(summarizeIndexFailures([b, a], "")) + }) + it("reads a row that predates migration 0032 as a plain error", () => { - const rows = summarizeIndexFailures([failed({ spanId: "old" })], { - traceId: "trace-1", - turnFailed: false, - }) + const rows = summarizeIndexFailures([failed({ spanId: "old" })], "") expect(rows).toEqual([ - { - kind: "error", - label: "error", - tool: undefined, - count: 1, - severity: "anomaly", - terminal: false, - }, + { kind: "error", label: "error", count: 1, severity: "anomaly", terminal: false }, ]) }) }) + +describe("failureSeverity", () => { + it("is the one rule both surfaces grade on", () => { + const kinds: readonly SessionFailureKind[] = [ + "error", + "rateLimited", + "contextExceeded", + "refusal", + "providerError", + "invalidOutput", + "toolArguments", + "toolUnavailable", + "toolTimeout", + "incomplete", + ] + expect(Object.fromEntries(kinds.map((kind) => [kind, failureSeverity(kind, false)]))).toEqual({ + error: "anomaly", + rateLimited: "anomaly", + contextExceeded: "failure", + refusal: "anomaly", + providerError: "anomaly", + invalidOutput: "failure", + toolArguments: "anomaly", + toolUnavailable: "failure", + toolTimeout: "anomaly", + incomplete: "failure", + }) + for (const kind of kinds) expect(failureSeverity(kind, true)).toBe("failure") + }) +}) diff --git a/packages/agent-sessions/src/index-failures.ts b/packages/agent-sessions/src/index-failures.ts index 2fddbde80..0fa1351c9 100644 --- a/packages/agent-sessions/src/index-failures.ts +++ b/packages/agent-sessions/src/index-failures.ts @@ -12,39 +12,35 @@ // What the index cannot say, the list does not claim: refusals and truncated // replies (finish reasons), loops and stalls (the turn timeline) are the // detail page's, and all of them are amber there unless the run died on one. +// The index also carries no `gen_ai.response.status`, so a rate limit a span +// reports there and nowhere else is a plain `error` here and `rate_limit` on +// the detail page; and no attempt marker, so a gateway's provider attempts +// (`isProviderAttempt`) count here where the detail page folds them into +// one retry. +import type { AiSessionIndexFailedSpan } from "@maple/domain/http" +import { Option, Schema } from "effect" import { rawFailureTextOf } from "./failure-text" import { failureSeverity, type FindingSeverity } from "./session-findings" import { classifyFailureSignal, failureSpecificity, + type FailureSignal, type SessionFailureClass, type SessionFailureKind, } from "./session-summary" -/** One failed agent span as the page query ships it: the deepest span of a - * roll-up, with the index columns the classifier reads. */ -export interface IndexFailedSpan { - readonly spanId: string - readonly traceId: string - readonly isToolCall: boolean - readonly isLlmCall: boolean - /** `''` where the span stamped none, or the row predates migration 0032. */ - readonly errorType: string - readonly toolName: string - readonly vendorId: string - readonly statusMessage: string - readonly failedToolCallResult: string - readonly responseId: string - readonly atMs: number -} +/** One failed agent span as the page query ships it — see the domain type. */ +export type IndexFailedSpan = AiSessionIndexFailedSpan /** One line of the list's breakdown: a failure label, how often, how bad. */ export interface SessionFailureSummary { readonly kind: SessionFailureKind /** `context_length_exceeded`, `tool_error · run_tests` — the finding's label. */ readonly label: string - readonly tool: string | undefined + /** Absent, not `undefined`: the wire schema's `optionalKey` rejects a + * present key holding `undefined`. */ + readonly tool?: string readonly count: number readonly severity: FindingSeverity /** The session's last turn died on it. */ @@ -55,27 +51,33 @@ export interface SessionFailureSummary { * Failures grouped by label, red ones first and the terminal one leading — * the order `buildSessionFindings` gives its failure rows. * - * `terminal` is the detail page's verdict approximated one trace deep: the - * session's last trace had a turn-level failure (`lastTraceTurnFailed`, from - * every failed span of that trace, echoes included), and the failure it died - * on is the last one in that trace. A turn that crosses traces can differ. + * `terminalSpanId` is the span the page query resolved the session's last + * turn died on (`terminalSpanIdExpr`), or `''`; the group holding that span + * — under whichever observation of the call the dedupe kept — is terminal. + * A terminal span the query did not ship (past its per-trace detail cap) + * marks nothing: the verdict is by identity, never by position. */ export function summarizeIndexFailures( spans: readonly IndexFailedSpan[], - lastTrace: { readonly traceId: string; readonly turnFailed: boolean }, + terminalSpanId: string, ): readonly SessionFailureSummary[] { const events = dedupeByResponseId( [...spans] - .sort((a, b) => a.atMs - b.atMs) - .map((span) => ({ span, ...classifyFailureSignal(signalOf(span)) })), + // Span id breaks a same-millisecond tie: the array arrives in no order. + .sort((a, b) => a.atMs - b.atMs || a.spanId.localeCompare(b.spanId)) + .map((span) => ({ span, spanIds: [span.spanId], ...classifyFailureSignal(signalOf(span)) })), ) - const cause = lastTrace.turnFailed - ? events.findLast((event) => event.span.traceId === lastTrace.traceId) - : undefined const groups = new Map< string, - { kind: SessionFailureKind; tool: string | undefined; count: number; terminal: boolean; atMs: number } + { + kind: SessionFailureKind + tool: string | undefined + count: number + terminal: boolean + atMs: number + spanId: string + } >() for (const event of events) { const group = groups.get(event.label) ?? { @@ -84,45 +86,66 @@ export function summarizeIndexFailures( count: 0, terminal: false, atMs: event.span.atMs, + spanId: event.span.spanId, } group.count += 1 - group.terminal ||= event === cause + group.terminal ||= terminalSpanId !== "" && event.spanIds.includes(terminalSpanId) groups.set(event.label, group) } return [...groups] - .map(([label, group]) => ({ - kind: group.kind, - label, - tool: group.tool, - count: group.count, - severity: failureSeverity(group.kind, group.terminal), - terminal: group.terminal, - atMs: group.atMs, - })) + .map(([label, group]) => { + const summary = { + kind: group.kind, + label, + count: group.count, + severity: failureSeverity(group.kind, group.terminal), + terminal: group.terminal, + atMs: group.atMs, + spanId: group.spanId, + } + return group.tool === undefined ? summary : { ...summary, tool: group.tool } + }) .sort( (a, b) => Number(a.severity === "anomaly") - Number(b.severity === "anomaly") || Number(b.terminal) - Number(a.terminal) || - a.atMs - b.atMs, + a.atMs - b.atMs || + a.spanId.localeCompare(b.spanId), ) - .map(({ atMs: _atMs, ...summary }) => summary) + .map(({ atMs: _atMs, spanId: _spanId, ...summary }) => summary) } -function signalOf(span: IndexFailedSpan) { +function signalOf(span: IndexFailedSpan): FailureSignal { return { errorType: span.errorType === "" ? undefined : span.errorType, responseStatus: undefined, statusMessage: span.statusMessage, - // The view keeps the result only on failed tool calls, as text. - toolCallResult: span.failedToolCallResult === "" ? undefined : span.failedToolCallResult, + // The view keeps a failed tool call's result as text, clipped; the + // classifier reads it as the span carries it — JSON where it still + // parses, else the text itself. + toolCallResult: + span.failedToolCallResult === "" ? undefined : toolCallResultOf(span.failedToolCallResult), tool: span.toolName !== "" ? span.toolName : span.isToolCall ? "tool" : undefined, isLlmCall: span.isLlmCall, vendorId: span.vendorId === "" ? undefined : span.vendorId, } } -type IndexFailureEvent = SessionFailureClass & { readonly span: IndexFailedSpan } +type ToolCallResult = FailureSignal["toolCallResult"] +const decodeJsonText = Schema.decodeUnknownOption(Schema.fromJsonString(Schema.Unknown)) +/** The clipped text parsed where it still is JSON, else as it is. */ +function toolCallResultOf(text: string): ToolCallResult { + return Option.getOrElse(decodeJsonText(text), () => text) +} + +interface IndexFailureEvent extends SessionFailureClass { + readonly span: IndexFailedSpan + /** Every observation's span this event stands for — its own, plus those + * of the duplicates dropped under it, so the terminal span is found under + * whichever observation the dedupe kept. */ + readonly spanIds: string[] +} /** Same rule as `session-summary.ts`'s `dedupeByResponseId`, over index rows: * a call the app and a gateway mirror both observed is one failure, and the @@ -144,10 +167,10 @@ function dedupeByResponseId(events: readonly IndexFailureEvent[]): readonly Inde } const specific = failureSpecificity(event) - failureSpecificity(slot.event) const longer = textLength(event.span) - textLength(slot.event.span) - if (specific > 0 || (specific === 0 && longer > 0)) { - kept[slot.index] = event - slot.event = event - } + const winner = specific > 0 || (specific === 0 && longer > 0) ? event : slot.event + const merged = { ...winner, spanIds: [...slot.event.spanIds, ...event.spanIds] } + kept[slot.index] = merged + slot.event = merged } return kept } diff --git a/packages/agent-sessions/src/session-findings.ts b/packages/agent-sessions/src/session-findings.ts index 2fd63be5e..0046388ef 100644 --- a/packages/agent-sessions/src/session-findings.ts +++ b/packages/agent-sessions/src/session-findings.ts @@ -6,7 +6,7 @@ // instrumentation's own vocabulary. Nothing is scored, sampled or modeled: a // finding the reader clicks through to must be exactly what the spans say. -import type { AiSessionSpan } from "@maple/domain/http" +import type { AiSessionFailureSeverity, AiSessionSpan } from "@maple/domain/http" import { formatNumber, formatSessionDuration } from "@maple/domain/format" import { canonicalJSON } from "@maple/query-engine" @@ -67,7 +67,7 @@ const FAILURE_KINDS: ReadonlySet = new Set([ /** Red or amber: whether the thing found ended the run or names a class that * needs a fix regardless, or was survived. The checklist's failed/warning * split reads the same field, so the two never disagree. */ -export type FindingSeverity = "failure" | "anomaly" +export type FindingSeverity = AiSessionFailureSeverity /** The one severity rule, for the findings here and for the list's breakdown * off the index (`index-failures.ts`): red when the run died on it or its diff --git a/packages/backend/src/services/ai-sessions/ai-session-reads.ts b/packages/backend/src/services/ai-sessions/ai-session-reads.ts index e382b6ccb..b29cae89c 100644 --- a/packages/backend/src/services/ai-sessions/ai-session-reads.ts +++ b/packages/backend/src/services/ai-sessions/ai-session-reads.ts @@ -35,7 +35,7 @@ import { type ListAiSessionsRequest, } from "@maple/domain/http" import { traceSessionTraceId } from "@maple/domain/gen-ai" -import { summarizeIndexFailures, type IndexFailedSpan } from "@maple/agent-sessions" +import { summarizeIndexFailures } from "@maple/agent-sessions" import { Array as Arr, Effect } from "effect" import { CH, formatWarehouseDateTime, parseWarehouseDateTime } from "@maple/query-engine" import * as Integrations from "@maple/query-engine-integrations" @@ -207,7 +207,18 @@ export const listAiSessions = Effect.fn("aiSessions.list")(function* ( ) // Rows returned, not rows asked for — annotated before the empty answer // leaves, so a window that ranks nothing is visible as such. - yield* Effect.annotateCurrentSpan({ "maple.ai.page_size": page.length }) + // The breakdown's two failure modes are otherwise invisible in a trace: a + // page whose rows classified nothing renders like a clean one, and a row + // cut at the cap may have lost the failure it died on. + yield* Effect.annotateCurrentSpan({ + "maple.ai.page_size": page.length, + "maple.ai.failures_classified": Arr.reduce(page, 0, (n, row) => n + row.failures.length), + "maple.ai.failure_rows_capped": Arr.reduce( + page, + 0, + (n, row) => n + (row.failures.length >= Integrations.MAX_FAILURES_PER_SESSION ? 1 : 0), + ), + }) if (page.length === 0) { return new ListAiSessionsResponse({ data: [] }) } @@ -223,10 +234,10 @@ export const listAiSessions = Effect.fn("aiSessions.list")(function* ( errorSpanCount: row.errorAgentSpans, toolErrorCount: row.toolErrors, turnErrorCount: row.turnErrors, - failures: summarizeIndexFailures(row.failures.map(indexFailedSpan), { - traceId: row.lastTraceId, - turnFailed: row.lastTraceTurnFailed === 1, - }), + failures: summarizeIndexFailures( + row.failures.map(Integrations.indexFailedSpanFromTuple), + row.terminalSpanId, + ), serviceNames: row.serviceNames, models: row.models, agentNames: row.agentNames, @@ -608,21 +619,6 @@ export const readAiToolsBreakdowns = Effect.fn("aiSessions.toolsBreakdowns")(fun return new AiToolsBreakdownsResponse({ tools: rows.map(breakdownItem) }) }) -/** A `failedSpansExpr` tuple as the page query ships it, by position. */ -export const indexFailedSpan = (tuple: Integrations.IndexFailedSpanTuple): IndexFailedSpan => ({ - spanId: tuple[0], - traceId: tuple[10], - isToolCall: tuple[2] === 1, - isLlmCall: tuple[3] === 1, - errorType: tuple[4], - toolName: tuple[5], - vendorId: tuple[6], - statusMessage: tuple[7], - failedToolCallResult: tuple[8], - responseId: tuple[9], - atMs: tuple[11], -}) - export const readAiToolErrors = Effect.fn("aiSessions.toolErrors")(function* ( tenant: TenantContext, payload: AiToolErrorsRequest, diff --git a/packages/backend/src/services/warehouse/ai-trace-index-materialization.clickhouse.e2e.test.ts b/packages/backend/src/services/warehouse/ai-trace-index-materialization.clickhouse.e2e.test.ts index 641b3507f..d5d822b7c 100644 --- a/packages/backend/src/services/warehouse/ai-trace-index-materialization.clickhouse.e2e.test.ts +++ b/packages/backend/src/services/warehouse/ai-trace-index-materialization.clickhouse.e2e.test.ts @@ -27,7 +27,6 @@ import { import { genAiErrorFingerprintText } from "@maple/domain/tinybird/gen-ai-columns" import * as Integrations from "@maple/query-engine-integrations" import { summarizeIndexFailures } from "@maple/agent-sessions" -import { indexFailedSpan } from "@maple/backend/services/ai-sessions/ai-session-reads" import type { AiSessionPageOpts } from "@maple/query-engine-integrations" import { normalizeSqlForClickHouseClient } from "@maple/query-engine/execution" import { @@ -809,7 +808,7 @@ describe.skipIf(!clickhouseE2eEnabled)("ai_trace_index materialization", () => { // The one failed span is a model call: a turn failure, not a tool's. assert.deepStrictEqual([sessionless!.toolErrors, sessionless!.turnErrors], [0, 1]) // The failed span, shipped for the breakdown as the tuple's positions — - // its trace is the session's only one, so its turn failure is terminal. + // a root span that failed, so the turn died on it: terminal. assert.deepStrictEqual(sessionless!.failures, [ [ "span-agent-2", @@ -826,25 +825,13 @@ describe.skipIf(!clickhouseE2eEnabled)("ai_trace_index materialization", () => { BASE_MS + 60_123, ], ]) + assert.strictEqual(sessionless!.terminalSpanId, "span-agent-2") assert.deepStrictEqual( - [sessionless!.lastTraceId, sessionless!.lastTraceTurnFailed], - [SESSIONLESS_TRACE, 1], - ) - assert.deepStrictEqual( - summarizeIndexFailures(sessionless!.failures.map(indexFailedSpan), { - traceId: sessionless!.lastTraceId, - turnFailed: sessionless!.lastTraceTurnFailed === 1, - }), - [ - { - kind: "error", - label: "error", - tool: undefined, - count: 1, - severity: "failure", - terminal: true, - }, - ], + summarizeIndexFailures( + sessionless!.failures.map(Integrations.indexFailedSpanFromTuple), + sessionless!.terminalSpanId, + ), + [{ kind: "error", label: "error", count: 1, severity: "failure", terminal: true }], ) // One span of 1ms: the extent is its own duration. assert.strictEqual(sessionless!.agentDurationMs, 1) @@ -874,8 +861,8 @@ describe.skipIf(!clickhouseE2eEnabled)("ai_trace_index materialization", () => { // The failed tool span under an `Ok` turn: one tool error, and no turn // error echoed off it. The failure lambda is raw SQL too. assert.deepStrictEqual([session!.toolErrors, session!.turnErrors], [1, 0]) - // The 0032 columns ride along, and the session's last trace — the second - // one, 30s on — closed cleanly, so the timeout is a warning, not red. + // The 0032 columns ride along, and the tool failed under an `Ok` turn + // span, so no turn root failed and nothing is terminal: a warning. assert.deepStrictEqual(session!.failures, [ [ "span-tool-1", @@ -892,12 +879,12 @@ describe.skipIf(!clickhouseE2eEnabled)("ai_trace_index materialization", () => { BASE_MS + 2_000, ], ]) - assert.deepStrictEqual([session!.lastTraceId, session!.lastTraceTurnFailed], [AGENT_TRACE_2, 0]) + assert.strictEqual(session!.terminalSpanId, "") assert.deepStrictEqual( - summarizeIndexFailures(session!.failures.map(indexFailedSpan), { - traceId: session!.lastTraceId, - turnFailed: session!.lastTraceTurnFailed === 1, - }), + summarizeIndexFailures( + session!.failures.map(Integrations.indexFailedSpanFromTuple), + session!.terminalSpanId, + ), [ { kind: "toolTimeout", diff --git a/packages/domain/src/http/ai-sessions.ts b/packages/domain/src/http/ai-sessions.ts index 96e7f58a3..9b6d132f2 100644 --- a/packages/domain/src/http/ai-sessions.ts +++ b/packages/domain/src/http/ai-sessions.ts @@ -150,6 +150,35 @@ export const AiSessionFailureKind = Schema.Literals([ ]) export type AiSessionFailureKind = Schema.Schema.Type +/** Red or amber: `failure` needs a fix (the run died on it, or its kind always + * does), `anomaly` was survived. `FindingSeverity` in `@maple/agent-sessions` + * is this type. */ +export const AiSessionFailureSeverity = Schema.Literals(["failure", "anomaly"]) +export type AiSessionFailureSeverity = Schema.Schema.Type + +/** + * One failed agent span as `ai_trace_index` carries it since migration 0032 — + * the deepest span of a roll-up, with the columns the failure classifier + * reads. The page query ships it as a positional tuple + * (`@maple/query-engine-integrations`' `indexFailedSpanFromTuple` names the + * positions) and `summarizeIndexFailures` in `@maple/agent-sessions` grades + * it. `''` where the span stamped nothing, or the row predates 0032. + */ +export interface AiSessionIndexFailedSpan { + readonly spanId: string + readonly traceId: string + readonly isToolCall: boolean + readonly isLlmCall: boolean + readonly errorType: string + readonly toolName: string + readonly vendorId: string + readonly statusMessage: string + readonly failedToolCallResult: string + readonly responseId: string + /** The span's start, epoch ms. */ + readonly atMs: number +} + /** * One line of a list row's failure breakdown: the failures sharing a label, * classified off the index the way the detail page classifies them off the @@ -162,7 +191,7 @@ export const AiSessionFailureSummary = Schema.Struct({ label: Schema.String, tool: Schema.optionalKey(Schema.String), count: Schema.Number, - severity: Schema.Literals(["failure", "anomaly"]), + severity: AiSessionFailureSeverity, /** The session's last turn died on it. */ terminal: Schema.Boolean, }) diff --git a/packages/query-engine-integrations/src/__sql_baseline__/integrations.sql b/packages/query-engine-integrations/src/__sql_baseline__/integrations.sql index 1dbdbbdaf..8278340f8 100644 --- a/packages/query-engine-integrations/src/__sql_baseline__/integrations.sql +++ b/packages/query-engine-integrations/src/__sql_baseline__/integrations.sql @@ -37,7 +37,11 @@ SELECT min(if(AgentName != '', Timestamp, toDateTime('2106-01-01 00:00:00'))) AS firstAgentAt, sum(IsToolCall) AS toolCalls, sum(IsError) AS errorAgentSpans, - groupArrayIf(2000)(tuple(SpanId, ParentSpanId, IsToolCall, IsLlmCall, ErrorType, ToolName, VendorId, StatusMessage, FailedToolCallResult, ResponseId, TraceId, toUnixTimestamp64Milli(Timestamp)), IsError = 1) AS failedSpans, + groupArrayIf(2000)(tuple(SpanId, ParentSpanId, IsToolCall, IsLlmCall, toUnixTimestamp64Milli(Timestamp)), IsError = 1) AS failedSpans, + arrayFilter(f -> NOT has(tupleElement(failedSpans, 2), f.1), failedSpans) AS deepestFailedSpans, + arrayExists(f -> (f.3 != 1 AND f.4 != 1) OR f.2 = '', failedSpans) AS turnRootFailed, + tupleElement(arrayReverseSort(f -> f.5, deepestFailedSpans)[1], 1) AS lastFailedSpanId, + vendorId = 'openrouter' AS traceIsMirror, groupArrayIf(2000)(tuple(SpanId, ParentSpanId, Tokens, Cost, ResponseId, IsLlmCall, InputTokens, CacheReadTokens, CacheWriteTokens, OutputTokens, ReasoningTokens), ((Tokens > 0 OR Cost > 0) OR IsLlmCall = 1)) AS usageReporters FROM ai_trace_index WHERE OrgId = 'org_sql_catalog' @@ -66,7 +70,11 @@ SELECT min(if(AgentName != '', Timestamp, toDateTime('2106-01-01 00:00:00'))) AS firstAgentAt, sum(IsToolCall) AS toolCalls, sum(IsError) AS errorAgentSpans, - groupArrayIf(2000)(tuple(SpanId, ParentSpanId, IsToolCall, IsLlmCall, ErrorType, ToolName, VendorId, StatusMessage, FailedToolCallResult, ResponseId, TraceId, toUnixTimestamp64Milli(Timestamp)), IsError = 1) AS failedSpans, + groupArrayIf(2000)(tuple(SpanId, ParentSpanId, IsToolCall, IsLlmCall, toUnixTimestamp64Milli(Timestamp)), IsError = 1) AS failedSpans, + arrayFilter(f -> NOT has(tupleElement(failedSpans, 2), f.1), failedSpans) AS deepestFailedSpans, + arrayExists(f -> (f.3 != 1 AND f.4 != 1) OR f.2 = '', failedSpans) AS turnRootFailed, + tupleElement(arrayReverseSort(f -> f.5, deepestFailedSpans)[1], 1) AS lastFailedSpanId, + vendorId = 'openrouter' AS traceIsMirror, groupArrayIf(2000)(tuple(SpanId, ParentSpanId, Tokens, Cost, ResponseId, IsLlmCall, InputTokens, CacheReadTokens, CacheWriteTokens, OutputTokens, ReasoningTokens), ((Tokens > 0 OR Cost > 0) OR IsLlmCall = 1)) AS usageReporters FROM ai_trace_index WHERE OrgId = 'org_sql_catalog' @@ -117,7 +125,11 @@ SELECT min(if(AgentName != '', Timestamp, toDateTime('2106-01-01 00:00:00'))) AS firstAgentAt, sum(IsToolCall) AS toolCalls, sum(IsError) AS errorAgentSpans, - groupArrayIf(2000)(tuple(SpanId, ParentSpanId, IsToolCall, IsLlmCall, ErrorType, ToolName, VendorId, StatusMessage, FailedToolCallResult, ResponseId, TraceId, toUnixTimestamp64Milli(Timestamp)), IsError = 1) AS failedSpans, + groupArrayIf(2000)(tuple(SpanId, ParentSpanId, IsToolCall, IsLlmCall, toUnixTimestamp64Milli(Timestamp)), IsError = 1) AS failedSpans, + arrayFilter(f -> NOT has(tupleElement(failedSpans, 2), f.1), failedSpans) AS deepestFailedSpans, + arrayExists(f -> (f.3 != 1 AND f.4 != 1) OR f.2 = '', failedSpans) AS turnRootFailed, + tupleElement(arrayReverseSort(f -> f.5, deepestFailedSpans)[1], 1) AS lastFailedSpanId, + vendorId = 'openrouter' AS traceIsMirror, groupArrayIf(2000)(tuple(SpanId, ParentSpanId, Tokens, Cost, ResponseId, IsLlmCall, InputTokens, CacheReadTokens, CacheWriteTokens, OutputTokens, ReasoningTokens), ((Tokens > 0 OR Cost > 0) OR IsLlmCall = 1)) AS usageReporters FROM ai_trace_index WHERE OrgId = 'org_sql_catalog' @@ -151,7 +163,11 @@ SELECT min(if(AgentName != '', Timestamp, toDateTime('2106-01-01 00:00:00'))) AS firstAgentAt, sum(IsToolCall) AS toolCalls, sum(IsError) AS errorAgentSpans, - groupArrayIf(2000)(tuple(SpanId, ParentSpanId, IsToolCall, IsLlmCall, ErrorType, ToolName, VendorId, StatusMessage, FailedToolCallResult, ResponseId, TraceId, toUnixTimestamp64Milli(Timestamp)), IsError = 1) AS failedSpans, + groupArrayIf(2000)(tuple(SpanId, ParentSpanId, IsToolCall, IsLlmCall, toUnixTimestamp64Milli(Timestamp)), IsError = 1) AS failedSpans, + arrayFilter(f -> NOT has(tupleElement(failedSpans, 2), f.1), failedSpans) AS deepestFailedSpans, + arrayExists(f -> (f.3 != 1 AND f.4 != 1) OR f.2 = '', failedSpans) AS turnRootFailed, + tupleElement(arrayReverseSort(f -> f.5, deepestFailedSpans)[1], 1) AS lastFailedSpanId, + vendorId = 'openrouter' AS traceIsMirror, groupArrayIf(2000)(tuple(SpanId, ParentSpanId, Tokens, Cost, ResponseId, IsLlmCall, InputTokens, CacheReadTokens, CacheWriteTokens, OutputTokens, ReasoningTokens), ((Tokens > 0 OR Cost > 0) OR IsLlmCall = 1)) AS usageReporters FROM ai_trace_index WHERE OrgId = 'org_sql_catalog' @@ -207,7 +223,11 @@ SELECT min(if(AgentName != '', Timestamp, toDateTime('2106-01-01 00:00:00'))) AS firstAgentAt, sum(IsToolCall) AS toolCalls, sum(IsError) AS errorAgentSpans, - groupArrayIf(2000)(tuple(SpanId, ParentSpanId, IsToolCall, IsLlmCall, ErrorType, ToolName, VendorId, StatusMessage, FailedToolCallResult, ResponseId, TraceId, toUnixTimestamp64Milli(Timestamp)), IsError = 1) AS failedSpans, + groupArrayIf(2000)(tuple(SpanId, ParentSpanId, IsToolCall, IsLlmCall, toUnixTimestamp64Milli(Timestamp)), IsError = 1) AS failedSpans, + arrayFilter(f -> NOT has(tupleElement(failedSpans, 2), f.1), failedSpans) AS deepestFailedSpans, + arrayExists(f -> (f.3 != 1 AND f.4 != 1) OR f.2 = '', failedSpans) AS turnRootFailed, + tupleElement(arrayReverseSort(f -> f.5, deepestFailedSpans)[1], 1) AS lastFailedSpanId, + vendorId = 'openrouter' AS traceIsMirror, groupArrayIf(2000)(tuple(SpanId, ParentSpanId, Tokens, Cost, ResponseId, IsLlmCall, InputTokens, CacheReadTokens, CacheWriteTokens, OutputTokens, ReasoningTokens), ((Tokens > 0 OR Cost > 0) OR IsLlmCall = 1)) AS usageReporters FROM ai_trace_index WHERE OrgId = 'org_sql_catalog' @@ -238,7 +258,11 @@ SELECT min(if(AgentName != '', Timestamp, toDateTime('2106-01-01 00:00:00'))) AS firstAgentAt, sum(IsToolCall) AS toolCalls, sum(IsError) AS errorAgentSpans, - groupArrayIf(2000)(tuple(SpanId, ParentSpanId, IsToolCall, IsLlmCall, ErrorType, ToolName, VendorId, StatusMessage, FailedToolCallResult, ResponseId, TraceId, toUnixTimestamp64Milli(Timestamp)), IsError = 1) AS failedSpans, + groupArrayIf(2000)(tuple(SpanId, ParentSpanId, IsToolCall, IsLlmCall, toUnixTimestamp64Milli(Timestamp)), IsError = 1) AS failedSpans, + arrayFilter(f -> NOT has(tupleElement(failedSpans, 2), f.1), failedSpans) AS deepestFailedSpans, + arrayExists(f -> (f.3 != 1 AND f.4 != 1) OR f.2 = '', failedSpans) AS turnRootFailed, + tupleElement(arrayReverseSort(f -> f.5, deepestFailedSpans)[1], 1) AS lastFailedSpanId, + vendorId = 'openrouter' AS traceIsMirror, groupArrayIf(2000)(tuple(SpanId, ParentSpanId, Tokens, Cost, ResponseId, IsLlmCall, InputTokens, CacheReadTokens, CacheWriteTokens, OutputTokens, ReasoningTokens), ((Tokens > 0 OR Cost > 0) OR IsLlmCall = 1)) AS usageReporters FROM ai_trace_index WHERE OrgId = 'org_sql_catalog' @@ -284,11 +308,10 @@ SELECT argMin(firstAgentName, firstAgentAt) AS firstAgentName, sum(toolCalls) AS toolCalls, sum(errorAgentSpans) AS errorAgentSpans, - sum(arrayCount(f -> f.3 = 1 AND NOT has(tupleElement(failedSpans, 2), f.1), failedSpans)) AS toolErrors, - sum(arrayCount(f -> f.3 != 1 AND NOT has(tupleElement(failedSpans, 2), f.1), failedSpans)) AS turnErrors, - arraySlice(groupArrayArray(arrayFilter(f -> NOT has(tupleElement(failedSpans, 2), f.1), failedSpans)), 1, 100) AS failures, - argMax(traceId, traceAgentEndNanos) AS lastTraceId, - argMax(arrayExists(f -> f.3 != 1, failedSpans), traceAgentEndNanos) AS lastTraceTurnFailed, + sum(arrayCount(f -> f.3 = 1, deepestFailedSpans)) AS toolErrors, + sum(arrayCount(f -> f.3 != 1, deepestFailedSpans)) AS turnErrors, + [] AS failures, + argMax(if(turnRootFailed = 1, lastFailedSpanId, ''), tuple(traceIsMirror != 1, traceAgentEndNanos)) AS terminalSpanId, intDiv(max(traceAgentEndNanos) - toUnixTimestamp64Nano(min(traceAgentStart)), 1000000) AS agentDurationMs, arraySlice(arrayFlatten(groupArray(usageReporters)), 1, 2000) AS reporters, arrayReduce('sumMap', arrayMap(c -> [c.2], reporters), arrayMap(c -> [c.3], reporters), arrayMap(c -> [c.4], reporters), arrayMap(c -> [c.7], reporters), arrayMap(c -> [c.8], reporters), arrayMap(c -> [c.9], reporters), arrayMap(c -> [c.10], reporters), arrayMap(c -> [c.11], reporters)) AS childClaims, @@ -310,7 +333,11 @@ SELECT min(if(AgentName != '', Timestamp, toDateTime('2106-01-01 00:00:00'))) AS firstAgentAt, sum(IsToolCall) AS toolCalls, sum(IsError) AS errorAgentSpans, - groupArrayIf(2000)(tuple(SpanId, ParentSpanId, IsToolCall, IsLlmCall, ErrorType, ToolName, VendorId, StatusMessage, FailedToolCallResult, ResponseId, TraceId, toUnixTimestamp64Milli(Timestamp)), IsError = 1) AS failedSpans, + groupArrayIf(2000)(tuple(SpanId, ParentSpanId, IsToolCall, IsLlmCall, toUnixTimestamp64Milli(Timestamp)), IsError = 1) AS failedSpans, + arrayFilter(f -> NOT has(tupleElement(failedSpans, 2), f.1), failedSpans) AS deepestFailedSpans, + arrayExists(f -> (f.3 != 1 AND f.4 != 1) OR f.2 = '', failedSpans) AS turnRootFailed, + tupleElement(arrayReverseSort(f -> f.5, deepestFailedSpans)[1], 1) AS lastFailedSpanId, + vendorId = 'openrouter' AS traceIsMirror, groupArrayIf(2000)(tuple(SpanId, ParentSpanId, Tokens, Cost, ResponseId, IsLlmCall, InputTokens, CacheReadTokens, CacheWriteTokens, OutputTokens, ReasoningTokens), ((Tokens > 0 OR Cost > 0) OR IsLlmCall = 1)) AS usageReporters FROM ai_trace_index WHERE OrgId = 'org_sql_catalog' @@ -444,8 +471,7 @@ SELECT toolErrors AS toolErrors, turnErrors AS turnErrors, failures AS failures, - lastTraceId AS lastTraceId, - lastTraceTurnFailed AS lastTraceTurnFailed, + terminalSpanId AS terminalSpanId, agentDurationMs AS agentDurationMs, toFloat64(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 2)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, toFloat64(n.2)), arrayFilter(n -> n.1 != '', netted)))))) AS llmCalls, arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 3)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.3), arrayFilter(n -> n.1 != '', netted))))) AS totalTokens, @@ -472,8 +498,7 @@ SELECT toolErrors AS toolErrors, turnErrors AS turnErrors, failures AS failures, - lastTraceId AS lastTraceId, - lastTraceTurnFailed AS lastTraceTurnFailed, + terminalSpanId AS terminalSpanId, agentDurationMs AS agentDurationMs, arrayMap(r -> tuple(r.5, r.6 = 1 AND if((r.3 > 0 OR r.4 > 0), greatest(0., r.3 - arrayElement(tupleElement(childClaims, 2), indexOf(tupleElement(childClaims, 1), r.1))) > 0 OR greatest(0., r.4 - arrayElement(tupleElement(childClaims, 3), indexOf(tupleElement(childClaims, 1), r.1))) > 0, NOT has(reportingIds, r.2)), greatest(0., r.3 - arrayElement(tupleElement(childClaims, 2), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.4 - arrayElement(tupleElement(childClaims, 3), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.7 - arrayElement(tupleElement(childClaims, 4), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.8 - arrayElement(tupleElement(childClaims, 5), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.9 - arrayElement(tupleElement(childClaims, 6), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.10 - arrayElement(tupleElement(childClaims, 7), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.11 - arrayElement(tupleElement(childClaims, 8), indexOf(tupleElement(childClaims, 1), r.1)))), reporters) AS netted FROM (SELECT @@ -490,11 +515,10 @@ SELECT argMin(firstAgentName, firstAgentAt) AS firstAgentName, sum(toolCalls) AS toolCalls, sum(errorAgentSpans) AS errorAgentSpans, - sum(arrayCount(f -> f.3 = 1 AND NOT has(tupleElement(failedSpans, 2), f.1), failedSpans)) AS toolErrors, - sum(arrayCount(f -> f.3 != 1 AND NOT has(tupleElement(failedSpans, 2), f.1), failedSpans)) AS turnErrors, - arraySlice(groupArrayArray(arrayFilter(f -> NOT has(tupleElement(failedSpans, 2), f.1), failedSpans)), 1, 100) AS failures, - argMax(traceId, traceAgentEndNanos) AS lastTraceId, - argMax(arrayExists(f -> f.3 != 1, failedSpans), traceAgentEndNanos) AS lastTraceTurnFailed, + sum(arrayCount(f -> f.3 = 1, deepestFailedSpans)) AS toolErrors, + sum(arrayCount(f -> f.3 != 1, deepestFailedSpans)) AS turnErrors, + arraySlice(arrayReverseSort(d -> d.12, groupArrayArray(arrayFilter(d -> NOT has(tupleElement(failedSpans, 2), d.1), failedSpanDetails))), 1, 100) AS failures, + argMax(if(turnRootFailed = 1, lastFailedSpanId, ''), tuple(traceIsMirror != 1, traceAgentEndNanos)) AS terminalSpanId, intDiv(max(traceAgentEndNanos) - toUnixTimestamp64Nano(min(traceAgentStart)), 1000000) AS agentDurationMs, arraySlice(arrayFlatten(groupArray(usageReporters)), 1, 2000) AS reporters, arrayReduce('sumMap', arrayMap(c -> [c.2], reporters), arrayMap(c -> [c.3], reporters), arrayMap(c -> [c.4], reporters), arrayMap(c -> [c.7], reporters), arrayMap(c -> [c.8], reporters), arrayMap(c -> [c.9], reporters), arrayMap(c -> [c.10], reporters), arrayMap(c -> [c.11], reporters)) AS childClaims, @@ -516,8 +540,13 @@ SELECT min(if(AgentName != '', Timestamp, toDateTime('2106-01-01 00:00:00'))) AS firstAgentAt, sum(IsToolCall) AS toolCalls, sum(IsError) AS errorAgentSpans, - groupArrayIf(2000)(tuple(SpanId, ParentSpanId, IsToolCall, IsLlmCall, ErrorType, ToolName, VendorId, StatusMessage, FailedToolCallResult, ResponseId, TraceId, toUnixTimestamp64Milli(Timestamp)), IsError = 1) AS failedSpans, - groupArrayIf(2000)(tuple(SpanId, ParentSpanId, Tokens, Cost, ResponseId, IsLlmCall, InputTokens, CacheReadTokens, CacheWriteTokens, OutputTokens, ReasoningTokens), ((Tokens > 0 OR Cost > 0) OR IsLlmCall = 1)) AS usageReporters + groupArrayIf(2000)(tuple(SpanId, ParentSpanId, IsToolCall, IsLlmCall, toUnixTimestamp64Milli(Timestamp)), IsError = 1) AS failedSpans, + arrayFilter(f -> NOT has(tupleElement(failedSpans, 2), f.1), failedSpans) AS deepestFailedSpans, + arrayExists(f -> (f.3 != 1 AND f.4 != 1) OR f.2 = '', failedSpans) AS turnRootFailed, + tupleElement(arrayReverseSort(f -> f.5, deepestFailedSpans)[1], 1) AS lastFailedSpanId, + vendorId = 'openrouter' AS traceIsMirror, + groupArrayIf(2000)(tuple(SpanId, ParentSpanId, Tokens, Cost, ResponseId, IsLlmCall, InputTokens, CacheReadTokens, CacheWriteTokens, OutputTokens, ReasoningTokens), ((Tokens > 0 OR Cost > 0) OR IsLlmCall = 1)) AS usageReporters, + groupArrayIf(50)(tuple(SpanId, ParentSpanId, IsToolCall, IsLlmCall, ErrorType, ToolName, VendorId, leftUTF8(StatusMessage, 400), leftUTF8(FailedToolCallResult, 400), ResponseId, TraceId, toUnixTimestamp64Milli(Timestamp)), IsError = 1) AS failedSpanDetails FROM ai_trace_index WHERE OrgId = 'org_sql_catalog' AND Timestamp >= '2026-01-01 10:30:00' @@ -547,8 +576,7 @@ SELECT toolErrors AS toolErrors, turnErrors AS turnErrors, failures AS failures, - lastTraceId AS lastTraceId, - lastTraceTurnFailed AS lastTraceTurnFailed, + terminalSpanId AS terminalSpanId, agentDurationMs AS agentDurationMs, toFloat64(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 2)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, toFloat64(n.2)), arrayFilter(n -> n.1 != '', netted)))))) AS llmCalls, arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 3)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.3), arrayFilter(n -> n.1 != '', netted))))) AS totalTokens, @@ -575,8 +603,7 @@ SELECT toolErrors AS toolErrors, turnErrors AS turnErrors, failures AS failures, - lastTraceId AS lastTraceId, - lastTraceTurnFailed AS lastTraceTurnFailed, + terminalSpanId AS terminalSpanId, agentDurationMs AS agentDurationMs, arrayMap(r -> tuple(r.5, r.6 = 1 AND if((r.3 > 0 OR r.4 > 0), greatest(0., r.3 - arrayElement(tupleElement(childClaims, 2), indexOf(tupleElement(childClaims, 1), r.1))) > 0 OR greatest(0., r.4 - arrayElement(tupleElement(childClaims, 3), indexOf(tupleElement(childClaims, 1), r.1))) > 0, NOT has(reportingIds, r.2)), greatest(0., r.3 - arrayElement(tupleElement(childClaims, 2), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.4 - arrayElement(tupleElement(childClaims, 3), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.7 - arrayElement(tupleElement(childClaims, 4), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.8 - arrayElement(tupleElement(childClaims, 5), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.9 - arrayElement(tupleElement(childClaims, 6), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.10 - arrayElement(tupleElement(childClaims, 7), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.11 - arrayElement(tupleElement(childClaims, 8), indexOf(tupleElement(childClaims, 1), r.1)))), reporters) AS netted FROM (SELECT @@ -593,11 +620,10 @@ SELECT argMin(firstAgentName, firstAgentAt) AS firstAgentName, sum(toolCalls) AS toolCalls, sum(errorAgentSpans) AS errorAgentSpans, - sum(arrayCount(f -> f.3 = 1 AND NOT has(tupleElement(failedSpans, 2), f.1), failedSpans)) AS toolErrors, - sum(arrayCount(f -> f.3 != 1 AND NOT has(tupleElement(failedSpans, 2), f.1), failedSpans)) AS turnErrors, - arraySlice(groupArrayArray(arrayFilter(f -> NOT has(tupleElement(failedSpans, 2), f.1), failedSpans)), 1, 100) AS failures, - argMax(traceId, traceAgentEndNanos) AS lastTraceId, - argMax(arrayExists(f -> f.3 != 1, failedSpans), traceAgentEndNanos) AS lastTraceTurnFailed, + sum(arrayCount(f -> f.3 = 1, deepestFailedSpans)) AS toolErrors, + sum(arrayCount(f -> f.3 != 1, deepestFailedSpans)) AS turnErrors, + arraySlice(arrayReverseSort(d -> d.12, groupArrayArray(arrayFilter(d -> NOT has(tupleElement(failedSpans, 2), d.1), failedSpanDetails))), 1, 100) AS failures, + argMax(if(turnRootFailed = 1, lastFailedSpanId, ''), tuple(traceIsMirror != 1, traceAgentEndNanos)) AS terminalSpanId, intDiv(max(traceAgentEndNanos) - toUnixTimestamp64Nano(min(traceAgentStart)), 1000000) AS agentDurationMs, arraySlice(arrayFlatten(groupArray(usageReporters)), 1, 2000) AS reporters, arrayReduce('sumMap', arrayMap(c -> [c.2], reporters), arrayMap(c -> [c.3], reporters), arrayMap(c -> [c.4], reporters), arrayMap(c -> [c.7], reporters), arrayMap(c -> [c.8], reporters), arrayMap(c -> [c.9], reporters), arrayMap(c -> [c.10], reporters), arrayMap(c -> [c.11], reporters)) AS childClaims, @@ -619,8 +645,13 @@ SELECT min(if(AgentName != '', Timestamp, toDateTime('2106-01-01 00:00:00'))) AS firstAgentAt, sum(IsToolCall) AS toolCalls, sum(IsError) AS errorAgentSpans, - groupArrayIf(2000)(tuple(SpanId, ParentSpanId, IsToolCall, IsLlmCall, ErrorType, ToolName, VendorId, StatusMessage, FailedToolCallResult, ResponseId, TraceId, toUnixTimestamp64Milli(Timestamp)), IsError = 1) AS failedSpans, - groupArrayIf(2000)(tuple(SpanId, ParentSpanId, Tokens, Cost, ResponseId, IsLlmCall, InputTokens, CacheReadTokens, CacheWriteTokens, OutputTokens, ReasoningTokens), ((Tokens > 0 OR Cost > 0) OR IsLlmCall = 1)) AS usageReporters + groupArrayIf(2000)(tuple(SpanId, ParentSpanId, IsToolCall, IsLlmCall, toUnixTimestamp64Milli(Timestamp)), IsError = 1) AS failedSpans, + arrayFilter(f -> NOT has(tupleElement(failedSpans, 2), f.1), failedSpans) AS deepestFailedSpans, + arrayExists(f -> (f.3 != 1 AND f.4 != 1) OR f.2 = '', failedSpans) AS turnRootFailed, + tupleElement(arrayReverseSort(f -> f.5, deepestFailedSpans)[1], 1) AS lastFailedSpanId, + vendorId = 'openrouter' AS traceIsMirror, + groupArrayIf(2000)(tuple(SpanId, ParentSpanId, Tokens, Cost, ResponseId, IsLlmCall, InputTokens, CacheReadTokens, CacheWriteTokens, OutputTokens, ReasoningTokens), ((Tokens > 0 OR Cost > 0) OR IsLlmCall = 1)) AS usageReporters, + groupArrayIf(50)(tuple(SpanId, ParentSpanId, IsToolCall, IsLlmCall, ErrorType, ToolName, VendorId, leftUTF8(StatusMessage, 400), leftUTF8(FailedToolCallResult, 400), ResponseId, TraceId, toUnixTimestamp64Milli(Timestamp)), IsError = 1) AS failedSpanDetails FROM ai_trace_index WHERE OrgId = 'org_sql_catalog' AND Timestamp >= '2026-01-01 10:30:00' @@ -669,8 +700,7 @@ SELECT toolErrors AS toolErrors, turnErrors AS turnErrors, failures AS failures, - lastTraceId AS lastTraceId, - lastTraceTurnFailed AS lastTraceTurnFailed, + terminalSpanId AS terminalSpanId, agentDurationMs AS agentDurationMs, toFloat64(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 2)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, toFloat64(n.2)), arrayFilter(n -> n.1 != '', netted)))))) AS llmCalls, arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 3)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.3), arrayFilter(n -> n.1 != '', netted))))) AS totalTokens, @@ -697,8 +727,7 @@ SELECT toolErrors AS toolErrors, turnErrors AS turnErrors, failures AS failures, - lastTraceId AS lastTraceId, - lastTraceTurnFailed AS lastTraceTurnFailed, + terminalSpanId AS terminalSpanId, agentDurationMs AS agentDurationMs, arrayMap(r -> tuple(r.5, r.6 = 1 AND if((r.3 > 0 OR r.4 > 0), greatest(0., r.3 - arrayElement(tupleElement(childClaims, 2), indexOf(tupleElement(childClaims, 1), r.1))) > 0 OR greatest(0., r.4 - arrayElement(tupleElement(childClaims, 3), indexOf(tupleElement(childClaims, 1), r.1))) > 0, NOT has(reportingIds, r.2)), greatest(0., r.3 - arrayElement(tupleElement(childClaims, 2), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.4 - arrayElement(tupleElement(childClaims, 3), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.7 - arrayElement(tupleElement(childClaims, 4), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.8 - arrayElement(tupleElement(childClaims, 5), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.9 - arrayElement(tupleElement(childClaims, 6), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.10 - arrayElement(tupleElement(childClaims, 7), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.11 - arrayElement(tupleElement(childClaims, 8), indexOf(tupleElement(childClaims, 1), r.1)))), reporters) AS netted FROM (SELECT @@ -715,11 +744,10 @@ SELECT argMin(firstAgentName, firstAgentAt) AS firstAgentName, sum(toolCalls) AS toolCalls, sum(errorAgentSpans) AS errorAgentSpans, - sum(arrayCount(f -> f.3 = 1 AND NOT has(tupleElement(failedSpans, 2), f.1), failedSpans)) AS toolErrors, - sum(arrayCount(f -> f.3 != 1 AND NOT has(tupleElement(failedSpans, 2), f.1), failedSpans)) AS turnErrors, - arraySlice(groupArrayArray(arrayFilter(f -> NOT has(tupleElement(failedSpans, 2), f.1), failedSpans)), 1, 100) AS failures, - argMax(traceId, traceAgentEndNanos) AS lastTraceId, - argMax(arrayExists(f -> f.3 != 1, failedSpans), traceAgentEndNanos) AS lastTraceTurnFailed, + sum(arrayCount(f -> f.3 = 1, deepestFailedSpans)) AS toolErrors, + sum(arrayCount(f -> f.3 != 1, deepestFailedSpans)) AS turnErrors, + arraySlice(arrayReverseSort(d -> d.12, groupArrayArray(arrayFilter(d -> NOT has(tupleElement(failedSpans, 2), d.1), failedSpanDetails))), 1, 100) AS failures, + argMax(if(turnRootFailed = 1, lastFailedSpanId, ''), tuple(traceIsMirror != 1, traceAgentEndNanos)) AS terminalSpanId, intDiv(max(traceAgentEndNanos) - toUnixTimestamp64Nano(min(traceAgentStart)), 1000000) AS agentDurationMs, arraySlice(arrayFlatten(groupArray(usageReporters)), 1, 2000) AS reporters, arrayReduce('sumMap', arrayMap(c -> [c.2], reporters), arrayMap(c -> [c.3], reporters), arrayMap(c -> [c.4], reporters), arrayMap(c -> [c.7], reporters), arrayMap(c -> [c.8], reporters), arrayMap(c -> [c.9], reporters), arrayMap(c -> [c.10], reporters), arrayMap(c -> [c.11], reporters)) AS childClaims, @@ -741,8 +769,13 @@ SELECT min(if(AgentName != '', Timestamp, toDateTime('2106-01-01 00:00:00'))) AS firstAgentAt, sum(IsToolCall) AS toolCalls, sum(IsError) AS errorAgentSpans, - groupArrayIf(2000)(tuple(SpanId, ParentSpanId, IsToolCall, IsLlmCall, ErrorType, ToolName, VendorId, StatusMessage, FailedToolCallResult, ResponseId, TraceId, toUnixTimestamp64Milli(Timestamp)), IsError = 1) AS failedSpans, - groupArrayIf(2000)(tuple(SpanId, ParentSpanId, Tokens, Cost, ResponseId, IsLlmCall, InputTokens, CacheReadTokens, CacheWriteTokens, OutputTokens, ReasoningTokens), ((Tokens > 0 OR Cost > 0) OR IsLlmCall = 1)) AS usageReporters + groupArrayIf(2000)(tuple(SpanId, ParentSpanId, IsToolCall, IsLlmCall, toUnixTimestamp64Milli(Timestamp)), IsError = 1) AS failedSpans, + arrayFilter(f -> NOT has(tupleElement(failedSpans, 2), f.1), failedSpans) AS deepestFailedSpans, + arrayExists(f -> (f.3 != 1 AND f.4 != 1) OR f.2 = '', failedSpans) AS turnRootFailed, + tupleElement(arrayReverseSort(f -> f.5, deepestFailedSpans)[1], 1) AS lastFailedSpanId, + vendorId = 'openrouter' AS traceIsMirror, + groupArrayIf(2000)(tuple(SpanId, ParentSpanId, Tokens, Cost, ResponseId, IsLlmCall, InputTokens, CacheReadTokens, CacheWriteTokens, OutputTokens, ReasoningTokens), ((Tokens > 0 OR Cost > 0) OR IsLlmCall = 1)) AS usageReporters, + groupArrayIf(50)(tuple(SpanId, ParentSpanId, IsToolCall, IsLlmCall, ErrorType, ToolName, VendorId, leftUTF8(StatusMessage, 400), leftUTF8(FailedToolCallResult, 400), ResponseId, TraceId, toUnixTimestamp64Milli(Timestamp)), IsError = 1) AS failedSpanDetails FROM ai_trace_index WHERE OrgId = 'org_sql_catalog' AND Timestamp >= '2026-01-01 10:30:00' diff --git a/packages/query-engine-integrations/src/ai/ai-sessions.test.ts b/packages/query-engine-integrations/src/ai/ai-sessions.test.ts index 2e5bec4c6..fb7beaa97 100644 --- a/packages/query-engine-integrations/src/ai/ai-sessions.test.ts +++ b/packages/query-engine-integrations/src/ai/ai-sessions.test.ts @@ -12,6 +12,7 @@ import { aiSessionDistributionsQuery, aiSessionFacetsQuery, idSearchPattern, + indexFailedSpanFromTuple, mergeAiSessionDetails, aiSessionPageQuery, aiSessionSpansQuery, @@ -295,8 +296,7 @@ describe("aiSessionPageQuery", () => { "1755599605825", ], ], - lastTraceId: "trace-3", - lastTraceTurnFailed: 0, + terminalSpanId: "", totalTokens: 184_320, inputTokens: 120_000, cacheReadTokens: 60_000, @@ -341,8 +341,7 @@ describe("aiSessionPageQuery", () => { 1_755_599_605_825, ], ], - lastTraceId: "trace-3", - lastTraceTurnFailed: 0, + terminalSpanId: "", totalTokens: 184_320, inputTokens: 120_000, cacheReadTokens: 60_000, @@ -467,25 +466,64 @@ describe("aiSessionPageQuery", () => { const { sessions: outer, traces: inner } = levels(sql) expect(inner).toContain( - "groupArrayIf(2000)(tuple(SpanId, ParentSpanId, IsToolCall, IsLlmCall, ErrorType, ToolName, VendorId, StatusMessage, FailedToolCallResult, ResponseId, TraceId, toUnixTimestamp64Milli(Timestamp)), IsError = 1) AS failedSpans", + "groupArrayIf(2000)(tuple(SpanId, ParentSpanId, IsToolCall, IsLlmCall, toUnixTimestamp64Milli(Timestamp)), IsError = 1) AS failedSpans", ) // A failed span whose own child also failed is the child's echo, not a // second failure — the turn span a framework fails alongside its call. - expect(outer).toContain( - "sum(arrayCount(f -> f.3 = 1 AND NOT has(tupleElement(failedSpans, 2), f.1), failedSpans)) AS toolErrors", + // Filtered once per trace; the counts read the result. + expect(inner).toContain( + "arrayFilter(f -> NOT has(tupleElement(failedSpans, 2), f.1), failedSpans) AS deepestFailedSpans", ) - expect(outer).toContain( - "sum(arrayCount(f -> f.3 != 1 AND NOT has(tupleElement(failedSpans, 2), f.1), failedSpans)) AS turnErrors", + expect(outer).toContain("sum(arrayCount(f -> f.3 = 1, deepestFailedSpans)) AS toolErrors") + expect(outer).toContain("sum(arrayCount(f -> f.3 != 1, deepestFailedSpans)) AS turnErrors") + }) + + it("ships the page's failure breakdown and the span the last turn died on", () => { + const { sql } = compileUnsafe(aiSessionPageQuery(), params) + const { sessions: outer, traces: inner } = levels(sql) + + // The detail tuple, capped per trace with its texts clipped, only on + // the page query; the deepest filter reads the thin array. + expect(inner).toContain( + "groupArrayIf(50)(tuple(SpanId, ParentSpanId, IsToolCall, IsLlmCall, ErrorType, ToolName, VendorId, leftUTF8(StatusMessage, 400), leftUTF8(FailedToolCallResult, 400), ResponseId, TraceId, toUnixTimestamp64Milli(Timestamp)), IsError = 1) AS failedSpanDetails", ) - // The same deepest spans, kept for the row's breakdown and capped; and - // whether the last trace's turn failed, off every failed span of it. expect(outer).toContain( - "arraySlice(groupArrayArray(arrayFilter(f -> NOT has(tupleElement(failedSpans, 2), f.1), failedSpans)), 1, 100) AS failures", + "arraySlice(arrayReverseSort(d -> d.12, groupArrayArray(arrayFilter(d -> NOT has(tupleElement(failedSpans, 2), d.1), failedSpanDetails))), 1, 100) AS failures", + ) + // The verdict: a turn-root failure in the trace, that trace's last + // failure, resolved over the session's last non-mirror trace. + expect(inner).toContain( + "arrayExists(f -> (f.3 != 1 AND f.4 != 1) OR f.2 = '', failedSpans) AS turnRootFailed", + ) + expect(inner).toContain( + "tupleElement(arrayReverseSort(f -> f.5, deepestFailedSpans)[1], 1) AS lastFailedSpanId", ) - expect(outer).toContain("argMax(traceId, traceAgentEndNanos) AS lastTraceId") expect(outer).toContain( - "argMax(arrayExists(f -> f.3 != 1, failedSpans), traceAgentEndNanos) AS lastTraceTurnFailed", + "argMax(if(turnRootFailed = 1, lastFailedSpanId, ''), tuple(traceIsMirror != 1, traceAgentEndNanos)) AS terminalSpanId", ) + + // The distributions read the same session level and decode none of it. + const distributions = compileUnsafe(aiSessionDistributionsQuery(), params).sql + expect(distributions).not.toContain("failedSpanDetails") + expect(distributions).toContain("[] AS failures") + }) + + it("names a failure tuple's positions the way the SQL orders them", () => { + expect( + indexFailedSpanFromTuple(["s", "p", 1, 0, "et", "tn", "vid", "msg", "res", "rid", "tid", 42]), + ).toEqual({ + spanId: "s", + traceId: "tid", + isToolCall: true, + isLlmCall: false, + errorType: "et", + toolName: "tn", + vendorId: "vid", + statusMessage: "msg", + failedToolCallResult: "res", + responseId: "rid", + atMs: 42, + }) }) it("filters the ranked row with HAVING, after the session grouping", () => { diff --git a/packages/query-engine-integrations/src/ai/ai-sessions.ts b/packages/query-engine-integrations/src/ai/ai-sessions.ts index 15c748528..a0f587578 100644 --- a/packages/query-engine-integrations/src/ai/ai-sessions.ts +++ b/packages/query-engine-integrations/src/ai/ai-sessions.ts @@ -140,6 +140,7 @@ import { CHNumber } from "@maple/query-engine/ch/schema" import { AI_SESSION_SPANS_MAX_SPANS, AI_SESSION_SUMMARY_MAX_TURNS, + type AiSessionIndexFailedSpan, type AiSessionSortDir, type AiSessionSortKey, type AiSessionSpanScope, @@ -241,66 +242,151 @@ export const sessionKey = (rawSessionId: CH.Expr, traceId: CH.Expr - readonly ParentSpanId: CH.Expr - readonly IsToolCall: CH.Expr - readonly IsError: CH.Expr -}): CH.Expr => +const failedSpansExpr = (): CH.Expr => CH.untypedExpr( - `groupArrayIf(${MAX_USAGE_REPORTERS_PER_TRACE})(tuple(SpanId, ParentSpanId, IsToolCall, IsLlmCall, ErrorType, ToolName, VendorId, StatusMessage, FailedToolCallResult, ResponseId, TraceId, toUnixTimestamp64Milli(Timestamp)), IsError = 1)`, + `groupArrayIf(${MAX_USAGE_REPORTERS_PER_TRACE})(tuple(SpanId, ParentSpanId, IsToolCall, IsLlmCall, toUnixTimestamp64Milli(Timestamp)), IsError = 1)`, + ) + +/** `failedSpans` less every span whose own child also failed: the child is + * the failure, the parent its echo. One pass per trace, read by the two + * counts, the verdict and the breakdown above it. */ +const deepestFailedSpansExpr = (failedSpans: string): CH.Expr => + CH.untypedExpr(`arrayFilter(f -> NOT has(tupleElement(${failedSpans}, 2), f.1), ${failedSpans})`) + +/** + * Whether a trace's turn did not close cleanly, as `session-turns.ts` reads + * it off the spans — a failed AI span that roots the turn — approximated + * from the index rows alone: a failed span that is neither a tool call nor + * a model call (the turn's own wrapper), or a failed root span. A model call + * that failed under an `Ok` turn is a retry the turn survived, not this. + */ +const turnRootFailedExpr = (failedSpans: string): CH.Expr => + CH.rawExpr(`arrayExists(f -> (f.3 != 1 AND f.4 != 1) OR f.2 = '', ${failedSpans})`, T.uint8) + +/** The span the trace's last failure is on — the deepest failed span that + * started last — or `''` when nothing failed. */ +const lastFailedSpanIdExpr = (deepestFailedSpans: string): CH.Expr => + CH.rawExpr(`tupleElement(arrayReverseSort(f -> f.5, ${deepestFailedSpans})[1], 1)`, T.string) + +/** + * The span the session's last turn died on, or `''` when it closed cleanly: + * the last trace's last failure, when that trace's turn failed. The last + * trace is the one whose agent spans end last, a gateway mirror trace + * (OpenRouter Broadcast, `traceIsMirror`) losing to any trace of + * the app's own — the mirror is a second observer of the same call, and its + * observation may be the one the breakdown drops as a duplicate. The + * verdict's proxy for `buildSessionFindings`' cause, one trace deep: a turn + * that crosses traces can differ. + */ +const terminalSpanIdExpr = (): CH.Expr => + CH.rawExpr( + `argMax(if(turnRootFailed = 1, lastFailedSpanId, ''), tuple(traceIsMirror != 1, traceAgentEndNanos))`, + T.string, ) -/** The most failures a page row ships for its breakdown — the deepest failed - * spans of the session, in no order. A session past it is triaged on its - * first hundred, which is what a chip and a hover can say anyway. */ -const MAX_FAILURES_PER_SESSION = 100 +/** The gateway whose Broadcast mirrors a session's model calls as traces of + * its own — see `dedupeByResponseId` in `@maple/agent-sessions`. */ +const MIRROR_VENDOR_ID = "openrouter" + +/** + * A failed span with what the list's breakdown classifies it on: the columns + * migration 0032 added, which are what the detail page's classifier reads + * off a span, plus the trace and the instant. Only the page query collects + * it (`FailureDetail`), and at most `MAX_FAILURE_DETAILS_PER_TRACE` per + * trace with the two texts clipped to `FAILURE_TEXT_CHARS`: a trace's + * failures otherwise cost the query up to 2000 × 1.4 KB of aggregate state + * for every trace in the window, before any page is cut. The classifier + * reads the head of a message. `''`/0 on rows that predate 0032, which + * classify as a plain `error`. + */ +const failedSpanDetailsExpr = (): CH.Expr => + CH.untypedExpr(`groupArrayIf(${MAX_FAILURE_DETAILS_PER_TRACE})(${FAILED_SPAN_TUPLE_SQL}, IsError = 1)`) +const MAX_FAILURE_DETAILS_PER_TRACE = 50 +const FAILURE_TEXT_CHARS = 400 + +/** + * How much of a failed span the trace level collects: `split` is what the + * tool/turn counts and the verdict read; `breakdown` adds the detail tuple + * the page's failure breakdown classifies on, which only the page query + * decodes. The details fan-out and the distributions read `split`: a level's + * unused aggregate is still computed. + */ +type FailureDetail = "split" | "breakdown" + +/** The most failures a page row ships for its breakdown — the latest of the + * session's deepest failed spans, so the one the run died on is among them. + * A session past it is triaged on those, which is what a chip and a hover + * can say anyway; the page read's span counts the rows it happened to. */ +export const MAX_FAILURES_PER_SESSION = 100 /** - * The session's deepest failed spans (`deepestFailureCount`'s filter, kept - * rather than counted), flattened across its traces and cut at - * `MAX_FAILURES_PER_SESSION`. Decoded as the tuple's positions. + * The session's deepest failed spans with their detail, latest first, cut at + * `MAX_FAILURES_PER_SESSION`. The deepest filter reads the trace's thin + * `failedSpans`, which sees every failure; the detail tuple is capped per + * trace, so a trace past `MAX_FAILURE_DETAILS_PER_TRACE` ships a sample of + * its failures, and may not ship the one the turn died on. Decoded as the + * tuple's positions. */ -const sessionFailuresExpr = (failedSpans: string): CH.Expr => +const sessionFailuresExpr = ( + failedSpans: string, + failedSpanDetails: string, +): CH.Expr => CH.rawExpr( - `arraySlice(groupArrayArray(arrayFilter(f -> NOT has(tupleElement(${failedSpans}, 2), f.1), ${failedSpans})), 1, ${MAX_FAILURES_PER_SESSION})`, + `arraySlice(arrayReverseSort(d -> d.12, groupArrayArray(arrayFilter(d -> NOT has(tupleElement(${failedSpans}, 2), d.1), ${failedSpanDetails}))), 1, ${MAX_FAILURES_PER_SESSION})`, FAILED_SPAN_TUPLES, ) -/** `(spanId, parentSpanId, isToolCall, isLlmCall, errorType, toolName, - * vendorId, statusMessage, failedToolCallResult, responseId, traceId, atMs)` - * — an element of `failedSpansExpr`, as the JSON wire renders a tuple. */ +/** + * The detail tuple's columns, in the order `failedSpanDetailsExpr` selects + * them, the ClickHouse type string declares them and + * {@link indexFailedSpanFromTuple} reads them: one list, so a column added + * or moved changes all three together. `d.1` and `d.12` in the lambdas + * above are the span id and the instant. + */ +const FAILED_SPAN_COLUMNS = [ + ["SpanId", "String"], + ["ParentSpanId", "String"], + ["IsToolCall", "UInt8"], + ["IsLlmCall", "UInt8"], + ["ErrorType", "LowCardinality(String)"], + ["ToolName", "LowCardinality(String)"], + ["VendorId", "LowCardinality(String)"], + [`leftUTF8(StatusMessage, ${FAILURE_TEXT_CHARS})`, "String"], + [`leftUTF8(FailedToolCallResult, ${FAILURE_TEXT_CHARS})`, "String"], + ["ResponseId", "String"], + ["TraceId", "String"], + ["toUnixTimestamp64Milli(Timestamp)", "Int64"], +] as const +const FAILED_SPAN_TUPLE_SQL = `tuple(${FAILED_SPAN_COLUMNS.map(([sql]) => sql).join(", ")})` + +/** An element of `failures` as the JSON wire renders a tuple — the + * {@link FAILED_SPAN_COLUMNS} by position. */ export type IndexFailedSpanTuple = readonly [ - string, - string, - number, - number, - string, - string, - string, - string, - string, - string, - string, - number, + spanId: string, + parentSpanId: string, + isToolCall: number, + isLlmCall: number, + errorType: string, + toolName: string, + vendorId: string, + statusMessage: string, + failedToolCallResult: string, + responseId: string, + traceId: string, + atMs: number, ] const FAILED_SPAN_TUPLES = T.array( T.custom( - "Tuple(String, String, UInt8, UInt8, LowCardinality(String), LowCardinality(String), LowCardinality(String), String, String, String, String, Int64)", + `Tuple(${FAILED_SPAN_COLUMNS.map(([, type]) => type).join(", ")})`, Schema.Tuple([ Schema.String, Schema.String, @@ -318,11 +404,25 @@ const FAILED_SPAN_TUPLES = T.array( ), ) +/** A `failures` element by name — the positions of {@link FAILED_SPAN_COLUMNS}. */ +export const indexFailedSpanFromTuple = (tuple: IndexFailedSpanTuple): AiSessionIndexFailedSpan => ({ + spanId: tuple[0], + traceId: tuple[10], + isToolCall: tuple[2] === 1, + isLlmCall: tuple[3] === 1, + errorType: tuple[4], + toolName: tuple[5], + vendorId: tuple[6], + statusMessage: tuple[7], + failedToolCallResult: tuple[8], + responseId: tuple[9], + atMs: tuple[11], +}) + /** - * Failed spans of one kind, summed over the traces' `failedSpans`, with a - * failed span whose own child also failed left out: the child is the failure, - * the parent its echo. `tool` counts the failed tool calls; the rest — failed - * model calls and turn spans that failed on their own — are the turn's. + * Failed spans of one kind, summed over the traces' `deepestFailedSpans`. + * `tool` counts the failed tool calls; the rest — failed model calls and + * turn spans that failed on their own — are the turn's. * * One level, and without the signal, where `shadowedAncestorIds` walks every * ancestor and shadows only a match: the index carries no error signal (its @@ -333,9 +433,9 @@ const FAILED_SPAN_TUPLES = T.array( * detail counts both, this counts one), which reads as one turn failing * either way; the cost of an exact copy is an error column on the index. */ -const deepestFailureCount = (failedSpans: string, kind: "tool" | "turn"): CH.Expr => +const deepestFailureCount = (deepestFailedSpans: string, kind: "tool" | "turn"): CH.Expr => CH.rawExpr( - `sum(arrayCount(f -> f.3 ${kind === "tool" ? "=" : "!="} 1 AND NOT has(tupleElement(${failedSpans}, 2), f.1), ${failedSpans}))`, + `sum(arrayCount(f -> f.3 ${kind === "tool" ? "=" : "!="} 1, ${deepestFailedSpans}))`, T.float64, ) @@ -442,13 +542,10 @@ export interface AiSessionPageOutput { readonly toolErrors: number /** Failed model calls and turn spans that failed on their own — the rest. */ readonly turnErrors: number - /** The deepest failed spans, for the row's breakdown — see `sessionFailuresExpr`. */ + /** The deepest failed spans, latest first, for the row's breakdown — see `sessionFailuresExpr`. */ readonly failures: readonly IndexFailedSpanTuple[] - /** The trace whose agent spans end last — where the session's final turn is. */ - readonly lastTraceId: string - /** That trace had a failed span that was not a tool call — the session's - * last turn did not close cleanly, one trace deep. 0/1. */ - readonly lastTraceTurnFailed: number + /** The span the session's last turn died on, `''` when it closed cleanly — see `terminalSpanIdExpr`. */ + readonly terminalSpanId: string /** Tokens across every bucket, deepest reporter counted, one claim per response id — see `sessionUsageSum`. */ readonly totalTokens: number // The five disjoint buckets `totalTokens` is the sum of, counted the same @@ -553,7 +650,7 @@ const MAX_NAMES_PER_TRACE = 20 * wrapper's roll-up of its children cannot be undone one row at a time — see * `usageReportersExpr`. */ -const indexTraces = (opts: AiSessionFilterOpts, bounds: IndexBounds) => { +const indexTraces = (opts: AiSessionFilterOpts, bounds: IndexBounds, detail: FailureDetail) => { const values = (list: readonly string[] | undefined) => (list?.length ? list : undefined) const search = opts.search?.trim() || undefined const carries = (cond: CH.Condition) => CH.countIf(cond).gt(0) @@ -582,7 +679,7 @@ const indexTraces = (opts: AiSessionFilterOpts, bounds: IndexBounds) => { // ties at one rank fall through to time rather than to whichever row // ClickHouse read first. const vendorOrder = orderTuple(CH.if_($.SessionId.neq(""), CH.lit(0), CH.lit(1)), $.Timestamp) - return { + const columns = { traceId: $.TraceId, rawSessionId: CH.max_($.SessionId), vendorId: CH.argMin($.VendorId, vendorOrder), @@ -617,11 +714,21 @@ const indexTraces = (opts: AiSessionFilterOpts, bounds: IndexBounds) => { firstAgentAt: CH.min_(agentOrder), toolCalls: CH.sum($.IsToolCall), errorAgentSpans: CH.sum($.IsError), - failedSpans: failedSpansExpr($), + failedSpans: failedSpansExpr(), + deepestFailedSpans: deepestFailedSpansExpr("failedSpans"), + turnRootFailed: turnRootFailedExpr("failedSpans"), + lastFailedSpanId: lastFailedSpanIdExpr("deepestFailedSpans"), + // Named apart from `vendorId`, which the session level re-aliases — + // an outer alias shadows the derived table's column of the same + // name inside an aggregate (see `traceAgentStart`). + traceIsMirror: CH.rawExpr(`vendorId = '${MIRROR_VENDOR_ID}'`, T.uint8), // Usage AND model calls travel as reporters: both are counted one level // up, where every trace of the session is in hand — see `ai-span-columns`. usageReporters: usageReportersExpr($), } + return detail === "breakdown" + ? { ...columns, failedSpanDetails: failedSpanDetailsExpr() } + : columns }) .where(($) => [ $.OrgId.eq(param.string("orgId")), @@ -663,8 +770,7 @@ const SESSION_COLUMNS = [ "toolErrors", "turnErrors", "failures", - "lastTraceId", - "lastTraceTurnFailed", + "terminalSpanId", "agentDurationMs", ] as const type SessionColumn = (typeof SESSION_COLUMNS)[number] @@ -678,8 +784,8 @@ const carry = >(row: Row): Pick - fromQuery(indexTraces(opts, "window"), "index_traces") +const indexSessions = (opts: AiSessionFilterOpts, detail: FailureDetail) => + fromQuery(indexTraces(opts, "window", detail), "index_traces") .select(($) => ({ // The grouping key, and the only level that can compute it: the // derived table is one row per trace, so a trace with no session id @@ -707,14 +813,15 @@ const indexSessions = (opts: AiSessionFilterOpts) => firstAgentName: CH.argMin($.firstAgentName, $.firstAgentAt), toolCalls: CH.sum($.toolCalls), errorAgentSpans: CH.sum($.errorAgentSpans), - toolErrors: deepestFailureCount("failedSpans", "tool"), - turnErrors: deepestFailureCount("failedSpans", "turn"), - failures: sessionFailuresExpr("failedSpans"), - lastTraceId: CH.argMax($.traceId, $.traceAgentEndNanos), - lastTraceTurnFailed: CH.rawExpr( - `argMax(arrayExists(f -> f.3 != 1, failedSpans), traceAgentEndNanos)`, - T.uint8, - ), + toolErrors: deepestFailureCount("deepestFailedSpans", "tool"), + turnErrors: deepestFailureCount("deepestFailedSpans", "turn"), + // `[]` where the trace level collected no detail: the distributions + // read this level and decode nothing of it. + failures: + detail === "breakdown" + ? sessionFailuresExpr("failedSpans", "failedSpanDetails") + : CH.rawExpr("[]", FAILED_SPAN_TUPLES), + terminalSpanId: terminalSpanIdExpr(), // Nanoseconds first, wrapped in `intDiv` — see `durationMs` in // `aiSessionDetailsQuery` for both. agentDurationMs: CH.intDiv( @@ -832,7 +939,7 @@ export function aiSessionPageQuery(opts: AiSessionPageOpts = {}) { const paged = (query: Q): Q => offset > 0 ? query.limit(limit).offset(offset) : query.limit(limit) - const sessions = indexSessions(opts).having(() => [ + const sessions = indexSessions(opts, "breakdown").having(() => [ CH.whenTrue(opts.hasErrors, () => column.errorAgentSpans.gt(0)), CH.whenTrue(opts.excludeTraceSessions, () => CH.not(column.sessionId.like(`${MAPLE_AI_TRACE_SESSION_PREFIX}%`)), @@ -923,10 +1030,10 @@ export function aiSessionDetailsQuery(opts: AiSessionDetailsOpts) { // page's bounds rather than the caller's window (see `indexTraces`). const onPage = ($: { rawSessionId: CH.Expr; traceId: CH.Expr }) => CH.inList(sessionKey($.rawSessionId, $.traceId), opts.sessionIds) - const pageTraceIds = fromQuery(indexTraces(opts, "page"), "agent_traces") + const pageTraceIds = fromQuery(indexTraces(opts, "page", "split"), "agent_traces") .select(($) => ({ traceId: $.traceId })) .where(($) => [onPage($)]) - const pageTraces = fromQuery(indexTraces(opts, "page"), "agent_traces") + const pageTraces = fromQuery(indexTraces(opts, "page", "split"), "agent_traces") .select(($) => ({ traceId: $.traceId, rawSessionId: $.rawSessionId })) .where(($) => [onPage($)]) @@ -1251,7 +1358,7 @@ const DISTRIBUTION_BUCKET_FLOORS = { */ export function aiSessionDistributionsQuery() { // The page's netting and sums, less every column the histograms do not read. - const netted = fromQuery(indexSessions({}), "window_sessions").select(($) => ({ + const netted = fromQuery(indexSessions({}, "split"), "window_sessions").select(($) => ({ agentDurationMs: $.agentDurationMs, toolCalls: $.toolCalls, netted: nettedReportersExpr("reporters", "childClaims", "reportingIds"), diff --git a/packages/query-engine-integrations/src/ai/index.ts b/packages/query-engine-integrations/src/ai/index.ts index 6fcf02551..57e62cc69 100644 --- a/packages/query-engine-integrations/src/ai/index.ts +++ b/packages/query-engine-integrations/src/ai/index.ts @@ -26,6 +26,8 @@ export { aiTraceTotalsQuery, aiTraceWindowQuery, idSearchPattern, + indexFailedSpanFromTuple, + MAX_FAILURES_PER_SESSION, type AiSessionDistributionMeasure, type AiSessionDistributionsOutput, type AiSessionFacetType,