Add local agent trace DAG and deterministic evals

packages/app/package.json

@@ -27,6 +27,7 @@
     "test:all": "vitest run",
     "memory:eval": "tsx scripts/memory-eval.mts",
     "memory:eval:real": "tsx scripts/memory-eval.mts --real",
+    "trace:eval": "tsx scripts/trace-eval.mts",
     "automation": "WDIO_LOG_LEVEL=silent tsx automation/server.ts",
     "automation:prepare": "tsx automation/check-runtime.ts && electron-forge package && tsx automation/prepare-driver.ts",
     "automation:typecheck": "tsc --noEmit -p automation/tsconfig.json"

packages/app/scripts/trace-eval.mts

@@ -0,0 +1,187 @@
+import { randomUUID } from "node:crypto";
+import { mkdir, writeFile } from "node:fs/promises";
+import { resolve } from "node:path";
+import { evaluateTrace } from "@/electron/trace/evaluation";
+import { projectTrace } from "@/electron/trace/projector";
+import { LocalTraceStore } from "@/electron/trace/store";
+import type {
+  TraceEvent,
+  TraceEventInput,
+  TraceSpanKind,
+} from "@/shared/types/trace";
+import { TRACE_SCHEMA_VERSION } from "@/shared/types/trace";
+
+const pathArgument = process.argv.find((argument) =>
+  argument.startsWith("--path="),
+);
+const requestedPath = pathArgument?.slice("--path=".length);
+const runId = new Date().toISOString().replaceAll(/[:.]/g, "-");
+const outputDirectory = resolve(
+  ".automation",
+  "artifacts",
+  "trace-eval",
+  runId,
+);
+
+function syntheticTrace(
+  name: string,
+  parallelTasks: number,
+  terminalStatus: "ok" | "interrupted" = "ok",
+): TraceEvent[] {
+  const traceId = `synthetic:${name}`;
+  const occurredAt = "2026-01-01T00:00:00.000Z";
+  const inputs: TraceEventInput[] = [];
+  const add = (
+    spanId: string,
+    spanKind: TraceSpanKind,
+    type: "span.start" | "span.end",
+    parentSpanId?: string,
+    metrics?: Record<string, number>,
+  ) => {
+    inputs.push({
+      eventId: `${spanId}:${type}`,
+      traceId,
+      spanId,
+      parentSpanId,
+      occurredAt,
+      emitter:
+        spanKind === "model"
+          ? "provider"
+          : spanKind === "tool"
+            ? "tool"
+            : "main",
+      type,
+      spanKind,
+      name: spanKind,
+      status: type === "span.end" ? terminalStatus : undefined,
+      attributes:
+        spanKind === "tool" ? { toolName: `tool-${spanId.at(-1)}` } : undefined,
+      metrics,
+      classification: "P0",
+    });
+  };
+
+  add("mission", "mission", "span.start");
+  for (let index = 0; index < parallelTasks; index += 1) {
+    const suffix = String(index + 1);
+    add(`task-${suffix}`, "task", "span.start", "mission");
+    add(`run-${suffix}`, "run", "span.start", `task-${suffix}`);
+    add(`turn-${suffix}`, "turn", "span.start", `run-${suffix}`);
+    add(`model-${suffix}`, "model", "span.start", `turn-${suffix}`);
+    add(`tool-${suffix}`, "tool", "span.start", `model-${suffix}`);
+    add(`tool-${suffix}`, "tool", "span.end", `model-${suffix}`);
+    add(`model-${suffix}`, "model", "span.end", `turn-${suffix}`, {
+      inputTokens: 100,
+      outputTokens: 50,
+      totalTokens: 150,
+    });
+    add(`turn-${suffix}`, "turn", "span.end", `run-${suffix}`);
+    add(`run-${suffix}`, "run", "span.end", `task-${suffix}`);
+  }
+  return inputs.map((input, index) => ({
+    ...input,
+    eventId: input.eventId ?? randomUUID(),
+    schemaVersion: TRACE_SCHEMA_VERSION,
+    sequence: index + 1,
+    recordedAt: occurredAt,
+  })) as TraceEvent[];
+}
+
+function structuredCollaborationTrace(): TraceEvent[] {
+  const events = syntheticTrace("structured-collaboration", 1);
+  let sequence = events.length;
+  const add = (
+    spanId: string,
+    spanKind: "task" | "handoff",
+    parentSpanId: string,
+    attributes: Record<string, string | number | boolean>,
+  ) => {
+    for (const type of ["span.start", "span.end"] as const) {
+      events.push({
+        schemaVersion: TRACE_SCHEMA_VERSION,
+        eventId: `${spanId}:${type}`,
+        traceId: "synthetic:structured-collaboration",
+        spanId,
+        parentSpanId,
+        links:
+          spanKind === "task"
+            ? [{ spanId: "run-1", relation: "triggered_by" }]
+            : [{ spanId: "run-1", relation: "handoff" }],
+        sequence: ++sequence,
+        occurredAt: "2026-01-01T00:00:01.000Z",
+        recordedAt: "2026-01-01T00:00:01.000Z",
+        emitter: "main",
+        type,
+        spanKind,
+        name: spanKind,
+        status: type === "span.end" ? "ok" : undefined,
+        attributes,
+        classification: "P0",
+      });
+    }
+  };
+  add("task-delegated", "task", "task-1", {
+    collaborationKind: "delegation",
+    collaborationOperationId: "delegation-1",
+    taskDepth: 1,
+    resultMessageCount: 1,
+  });
+  add("handoff-1", "handoff", "task-delegated", {
+    operationId: "handoff-1",
+    committed: true,
+  });
+  return events;
+}
+
+const cases = [];
+if (requestedPath) {
+  const store = new LocalTraceStore(resolve(requestedPath));
+  for (const traceId of await store.listTraceIds()) {
+    cases.push(evaluateTrace(await store.graph(traceId)));
+  }
+} else {
+  for (const [name, tasks, terminalStatus] of [
+    ["single-task", 1, "ok"],
+    ["parallel-two-task", 2, "ok"],
+    ["parallel-four-task", 4, "ok"],
+    ["interrupted-recovery", 1, "interrupted"],
+  ] as const) {
+    const events = syntheticTrace(name, tasks, terminalStatus);
+    cases.push(evaluateTrace(projectTrace(events, `synthetic:${name}`)));
+  }
+  const structured = structuredCollaborationTrace();
+  cases.push(
+    evaluateTrace(
+      projectTrace(structured, "synthetic:structured-collaboration"),
+    ),
+  );
+}
+
+const report = {
+  schemaVersion: 1,
+  runId,
+  generatedAt: new Date().toISOString(),
+  source: requestedPath
+    ? resolve(requestedPath)
+    : "built-in synthetic scenarios",
+  summary: {
+    traces: cases.length,
+    passed: cases.filter((entry) => entry.summary.score === 1).length,
+    meanScore:
+      cases.length === 0
+        ? 0
+        : Math.round(
+            (cases.reduce((total, entry) => total + entry.summary.score, 0) /
+              cases.length) *
+              10_000,
+          ) / 10_000,
+  },
+  cases,
+};
+
+await mkdir(outputDirectory, { recursive: true });
+const outputPath = resolve(outputDirectory, "report.json");
+await writeFile(outputPath, `${JSON.stringify(report, null, 2)}\n`, "utf8");
+console.log(JSON.stringify({ ...report.summary, outputPath }, null, 2));
+
+if (report.summary.passed !== report.summary.traces) process.exitCode = 1;

packages/app/scripts/web-bridge-dev.mts

@@ -23,6 +23,8 @@ import { JsonAgentHostJobRepository } from "@/electron/agent-host/repository";
 import { AgentHostRendererBridge } from "@/electron/agent-host/renderer-bridge";
 import { LocalAiAgentHostExecutor } from "@/electron/agent-host/executor";
 import { withAgentHostTools } from "@/electron/ai/agent-host-tools";
+import { AgentHostTraceRecorder } from "@/electron/trace/agent-host";
+import { LocalTraceStore } from "@/electron/trace/store";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
 import {
@@ -67,8 +69,12 @@ const runtime = new LocalAiRuntime({
 
 // The bridge owns one sender per connected tab; the host talks to whichever
 // one is live.
-/** Readable from outside the browser, unlike anything in the renderer's DB. */
-const TURN_LOG_PATH = join(tmpdir(), "convera-dev-agent-turns.jsonl");
+const traceStore = new LocalTraceStore(
+  join(tmpdir(), "convera-dev-agent-traces.jsonl"),
+);
+const traceRecorder = new AgentHostTraceRecorder(traceStore, {
+  onError: (error) => console.warn("[agent-trace] write failed", error),
+});
 
 const agentHostBridge = new AgentHostRendererBridge(() => {
   // The newest live tab, not the first ever seen: a stale sender from a closed
@@ -89,12 +95,18 @@ const agentHost = new AgentHost({
   executor: new LocalAiAgentHostExecutor(
     runtime,
     agentHostBridge,
-    TURN_LOG_PATH,
+    traceRecorder,
   ),
   startPaused: true,
 });
-agentHost.subscribe((event) => agentHostBridge.emit(event));
+agentHost.subscribe((event) => {
+  void traceRecorder.record(event);
+  agentHostBridge.emit(event);
+});
 await agentHost.initialize();
+await Promise.all(
+  (await agentHost.listJobs()).map((job) => traceRecorder.recordJob(job)),
+);
 
 const recordingIPC = createRecordingIpcMain({
   handle: () => {},

packages/app/src/electron/agent-host/executor.ts

@@ -10,9 +10,9 @@ import type {
   LocalAIStreamEvent,
 } from "@/shared/types/local-ai";
 import { WORKSPACE_QUERY_INTERACTION } from "@/shared/types/workspace-perception";
+import type { AgentHostTraceRecorder } from "@/electron/trace/agent-host";
 import type { AgentHostExecutor } from "./host";
 import type { AgentHostRendererBridge } from "./renderer-bridge";
-import { TurnLogger } from "./turn-log";
 
 /**
  * A turn that ends without calling the speech tool said nothing at all: the
@@ -69,8 +69,7 @@ export class LocalAiAgentHostExecutor implements AgentHostExecutor {
   constructor(
     private readonly runtime: LocalAIRuntimeService,
     private readonly bridge: AgentHostRendererBridge,
-    /** Where each turn's tool sequence is written; omit to record nothing. */
-    private readonly turnLogPath?: string,
+    private readonly trace?: AgentHostTraceRecorder,
   ) {}
 
   async execute(
@@ -98,37 +97,19 @@ export class LocalAiAgentHostExecutor implements AgentHostExecutor {
         },
       };
       this.activeRequests.set(job.id, request.requestId);
-      const log = this.turnLogPath
-        ? new TurnLogger(this.turnLogPath)
-        : undefined;
-      const flush = () =>
-        log?.flush({
-          id: job.id,
-          requestId: request.requestId,
-          conversationId: job.conversationId,
-          memberId: job.agentMemberId,
-          mode: job.mode,
-        });
-      if (await this.runTurn(request, job, emit, log)) {
-        await flush();
-        return;
-      }
+      if (await this.runTurn(request, job, emit)) return;
       // Silence is a complete answer on an open floor; only a direct question
       // left unanswered is worth a second ask.
       if (job.mode !== "direct") {
-        await flush();
         return;
       }
       // Announced before the second turn starts, so the renderer can hold the
       // typing indicator across the gap rather than retiring it and lighting
       // it again for the same reply.
       emit({ type: "retrying", jobId: job.id });
-      log?.noteRetry();
-      if (await this.runTurn(remind(request), job, emit, log)) {
-        await flush();
+      if (await this.runTurn(remind(request), job, emit, request.turnId)) {
         return;
       }
-      await flush();
       throw new Error(
         "The agent completed a direct offer without sending a message.",
       );
@@ -142,20 +123,27 @@ export class LocalAiAgentHostExecutor implements AgentHostExecutor {
     request: LocalAIChatRequest,
     job: AgentHostJob,
     emit: (event: AgentHostEvent) => void,
-    log?: TurnLogger,
+    previousTurnId?: string,
   ): Promise<boolean> {
     let streamError: Error | undefined;
     let spoke = false;
-    await this.runtime.startChat(request, (event) => {
-      log?.record(event);
-      if (event.type === "error") {
-        streamError = new Error(event.error.message);
-      }
-      if (isSpeech(event)) spoke = true;
-      emit({ type: "stream", jobId: job.id, event });
-    });
-    if (streamError) throw streamError;
-    return spoke;
+    const trace = await this.trace?.beginTurn(job, request, previousTurnId);
+    try {
+      await this.runtime.startChat(request, (event) => {
+        trace?.record(event);
+        if (event.type === "error") {
+          streamError = new Error(event.error.message);
+        }
+        if (isSpeech(event)) spoke = true;
+        emit({ type: "stream", jobId: job.id, event });
+      });
+      if (streamError) throw streamError;
+      await trace?.complete();
+      return spoke;
+    } catch (error) {
+      await trace?.complete(error);
+      throw error;
+    }
   }
 
   async cancel(job: AgentHostJob): Promise<boolean> {