Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 19 additions & 1 deletion apps/ai/src/mcp/tools/__tests__/agent-sessions.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,23 @@ const listRow = {
errorAgentSpans: 1,
toolErrors: 1,
turnErrors: 0,
failures: [
[
"span-tool-1",
"span-agent-1",
1,
0,
"TimeoutError",
"search_traces",
"eve",
"search timed out",
"",
"",
"trace-1",
1755599605825,
],
],
terminalSpanId: "",
serviceNames: ["agent-runner"],
models: ["gpt-5"],
agentNames: ["maple"],
Expand Down Expand Up @@ -291,7 +308,8 @@ describe("list_agent_sessions rendering", () => {
expect(cells?.[1]).toBe("maple")
expect(cells?.[2]).toBe("eve")
// Errors are agent/tool/turn, in that order.
expect(cells?.[7]).toBe("1/1/0")
// The breakdown by label, not the raw counts, once the index classified one.
expect(cells?.[7]).toBe("tool_timeout · search_traces")
// The hint is the row's bounds PADDED the way the page pads them: a row's
// bounds are its agent spans' extent, and both read levels bound on
// `Timestamp`, so handing them over verbatim would drop the app spans
Expand Down
34 changes: 30 additions & 4 deletions apps/ai/src/mcp/tools/list-agent-sessions.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import { formatNextSteps } from "../lib/next-steps"
import { windowHint } from "../lib/agent-sessions"
import { Effect, Schema } from "effect"
import {
type AiSessionFailureSummary,
AI_SESSION_SEARCH_MAX_CHARS,
AiSessionSortDir,
AiSessionSortKey,
Expand Down Expand Up @@ -150,7 +151,8 @@ export function registerListAgentSessionsTool(server: McpToolRegistrar) {
formatDurationFromMs(session.durationMs),
formatNumber(session.llmCalls),
formatNumber(session.toolCalls),
`${session.errorSpanCount}/${session.toolErrorCount}/${session.turnErrorCount}`,
failuresCell(session.failures) ??
`${session.errorSpanCount}/${session.toolErrorCount}/${session.turnErrorCount}`,
formatNumber(session.totalTokens),
session.cost > 0 ? formatCost(session.cost) : "—",
truncate(session.models.join(", "), 40),
Expand All @@ -160,7 +162,7 @@ export function registerListAgentSessionsTool(server: McpToolRegistrar) {
const lines: string[] = [
`## AI agent sessions (showing ${offset + 1}–${offset + sessions.length})`,
`Time range: ${st} — ${et}`,
`Every figure is over the session's AGENT spans; the app's own spans in the same traces are not counted. Errors are agent/tool/turn.`,
`Every figure is over the session's AGENT spans; the app's own spans in the same traces are not counted. Failures are by label, ×count; "!" marks one that needs a fix (the run died on it, or its kind always does), the rest were survived. A bare a/b/c is errored agent/tool/turn spans the index could not classify.`,
``,
formatTable(
[
Expand All @@ -171,7 +173,7 @@ export function registerListAgentSessionsTool(server: McpToolRegistrar) {
"Duration",
"LLM calls",
"Tool calls",
"Errors",
"Failures",
"Tokens",
"Cost",
"Models",
Expand All @@ -180,7 +182,10 @@ export function registerListAgentSessionsTool(server: McpToolRegistrar) {
rows,
),
...(sessions.length === limit
? [``, `The page is full; more sessions may match — call again with offset=${offset + sessions.length}.`]
? [
``,
`The page is full; more sessions may match — call again with offset=${offset + sessions.length}.`,
]
: []),
formatNextSteps(
sessions.slice(0, 3).map(
Expand All @@ -197,3 +202,24 @@ export function registerListAgentSessionsTool(server: McpToolRegistrar) {
}),
)
}

/** `!context_length_exceeded, tool_error · run_tests ×2` — the row's failures
* by label, a `!` on each one that needs a fix, whole labels only and
* `+N more` past the cell's width. `undefined` when the index classified
* none, and the raw counts say what it saw. */
function failuresCell(failures: ReadonlyArray<AiSessionFailureSummary>): string | undefined {
if (failures.length === 0) return undefined
const labels = failures.map(
(failure) =>
`${failure.severity === "failure" ? "!" : ""}${failure.label}${failure.count > 1 ? ` ×${failure.count}` : ""}`,
)
const shown: string[] = []
for (const label of labels) {
const next = [...shown, label].join(", ")
if (shown.length > 0 && next.length > FAILURES_CELL_MAX) break
shown.push(label)
}
const more = labels.length - shown.length
return shown.join(", ") + (more > 0 ? ` +${more} more` : "")
}
const FAILURES_CELL_MAX = 80
53 changes: 53 additions & 0 deletions apps/api/src/routes/internal/ai-sessions.http.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -454,6 +454,41 @@ describe("POST /internal/ai-sessions/list", () => {
errorAgentSpans: "1",
toolErrors: 1,
turnErrors: 0,
// The failed spans, as the wire renders the tuple: a rate-limited model
// call — no tool, so the summary carries no `tool` key at all — and a
// tool that timed out, both in a trace that is not the session's last,
// so both warnings.
failures: [
[
"span-llm-1",
"span-agent-1",
0,
1,
"",
"",
"eve",
"429 Too Many Requests",
"",
"",
"trace-1",
"1755599604825",
],
[
"span-tool-1",
"span-agent-1",
1,
0,
"TimeoutError",
"search_traces",
"eve",
"search timed out",
"",
"",
"trace-1",
"1755599605825",
],
],
terminalSpanId: "",
totalTokens: 18_400,
inputTokens: 12_000,
cacheReadTokens: 4_000,
Expand Down Expand Up @@ -523,6 +558,24 @@ describe("POST /internal/ai-sessions/list", () => {
errorSpanCount: 1,
toolErrorCount: 1,
turnErrorCount: 0,
// Classified off the shipped tuples by the detail page's own rule.
failures: [
{
kind: "rateLimited",
label: "rate_limit",
count: 1,
severity: "anomaly",
terminal: false,
},
{
kind: "toolTimeout",
label: "tool_timeout · search_traces",
tool: "search_traces",
count: 1,
severity: "anomaly",
terminal: false,
},
],
models: ["claude-sonnet-5"],
agentNames: ["web-fetcher", "slack-agent"],
firstAgentName: "slack-agent",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@ const session: AgentSessionRow = {
errorSpanCount: 0,
toolErrorCount: 0,
turnErrorCount: 0,
failures: [],
serviceNames: ["maple-slack-agent"],
models: ["claude-sonnet-5"],
agentNames: ["web-fetcher", "slack-agent"],
Expand Down Expand Up @@ -114,7 +115,13 @@ describe("AgentSessionsList", () => {
it("asks for the next page when the sentinel comes into view, but not while one is in flight", () => {
const onReachEnd = vi.fn()
const view = renderList(
<AgentSessionsList {...sort} sessions={[session]} hasMore onReachEnd={onReachEnd} loadingMore={false} />,
<AgentSessionsList
{...sort}
sessions={[session]}
hasMore
onReachEnd={onReachEnd}
loadingMore={false}
/>,
)

const first = MockIntersectionObserver.instances[0]!
Expand All @@ -141,7 +148,7 @@ describe("AgentSessionsList", () => {
expect(MockIntersectionObserver.instances).toHaveLength(0)
})

it("names the framework by its mark alone, and splits the failures by kind", () => {
it("names the framework by its mark alone, and splits the failures by severity", () => {
const view = renderList(
<AgentSessionsList
{...sort}
Expand All @@ -151,6 +158,23 @@ describe("AgentSessionsList", () => {
errorSpanCount: 5,
toolErrorCount: 2,
turnErrorCount: 1,
failures: [
{
kind: "contextExceeded",
label: "context_length_exceeded",
count: 1,
severity: "failure",
terminal: true,
},
{
kind: "error",
label: "tool_error · run_tests",
tool: "run_tests",
count: 2,
severity: "anomaly",
terminal: false,
},
],
},
]}
/>,
Expand All @@ -164,8 +188,26 @@ describe("AgentSessionsList", () => {
// count and noun into fixed-width slots, so match on the whole chip's text.
const chip = (label: string) => (_: string, element: Element | null) =>
element?.classList.contains("rounded-full") === true && element.textContent === label
expect(view.getAllByText(chip("2 tools"))).toHaveLength(1)
expect(view.getAllByText(chip("1 turn"))).toHaveLength(1)
expect(view.getAllByText(chip("1 failure"))).toHaveLength(1)
expect(view.getAllByText(chip("2 warnings"))).toHaveLength(1)
// The breakdown is the chip's accessible name: the labels exist nowhere
// else until the tooltip opens.
expect(view.getByLabelText("context_length_exceeded (ended the run)")).toBeTruthy()
expect(view.getByLabelText("tool_error · run_tests ×2")).toBeTruthy()
})

it("counts the errored spans when the index classified none of them", () => {
const view = renderList(
<AgentSessionsList
{...sort}
sessions={[
{ ...session, errorSpanCount: 3, toolErrorCount: 0, turnErrorCount: 0, failures: [] },
]}
/>,
)
const chip = (label: string) => (_: string, element: Element | null) =>
element?.classList.contains("rounded-full") === true && element.textContent === label
expect(view.getAllByText(chip("3 spans"))).toHaveLength(1)
expect(view.getByText("18.4k")).toBeTruthy()
expect(view.getByText("maple-slack-agent")).toBeTruthy()
})
Expand Down Expand Up @@ -199,7 +241,12 @@ describe("AgentSessionsList", () => {
it("sorts through the column headers, marking the one the rows are in", () => {
const onSortChange = vi.fn()
const view = renderList(
<AgentSessionsList sessions={[session]} sortBy="cost" sortDir="desc" onSortChange={onSortChange} />,
<AgentSessionsList
sessions={[session]}
sortBy="cost"
sortDir="desc"
onSortChange={onSortChange}
/>,
)
const cost = view.getByRole("columnheader", { name: "Cost" })
expect(cost.getAttribute("aria-sort")).toBe("descending")
Expand Down
Loading
Loading