diff --git a/packages/core/src/trace-projection.test.ts b/packages/core/src/trace-projection.test.ts index 4a34d38..d27ee39 100644 --- a/packages/core/src/trace-projection.test.ts +++ b/packages/core/src/trace-projection.test.ts @@ -224,4 +224,49 @@ describe('projectTrace (V9.1)', () => { const toolStep = trace.steps.find((s) => s.kind === 'tool' && s.tool?.id === 'live-run'); expect(toolStep?.status).toBe('running'); }); + + it('projects the benchmark case prompt as the trace input, never the answer', () => { + const caseDef: EvaluationCase = { + id: 'case-1', + name: 'NVDA research', + category: 'tool-selection', + difficulty: 'golden', + input: { prompt: 'Research NVDA' }, + expected: {}, + tags: [], + source: 'hand-authored', + }; + const evalRun: EvaluationRun = { + id: 'eval-run-1', + experimentId: 'exp-1', + caseId: 'case-1', + datasetId: 'ds-1', + status: 'completed', + startedAt: 1000, + completedAt: 9000, + answer: 'NVDA looks fine', + toolCalls: [], + failureModes: [], + }; + const trace = projectTrace({ evaluationRun: evalRun, evaluationCase: caseDef }); + expect(trace.input).toBe('Research NVDA'); + expect(trace.input).not.toBe(evalRun.answer); + }); + + it('leaves the trace input empty when no authoritative prompt was recorded', () => { + const evalRun: EvaluationRun = { + id: 'eval-run-2', + experimentId: 'exp-1', + caseId: 'case-2', + datasetId: 'ds-1', + status: 'completed', + startedAt: 1000, + completedAt: 9000, + answer: 'NVDA looks fine', + toolCalls: [], + failureModes: [], + }; + const trace = projectTrace({ evaluationRun: evalRun }); + expect(trace.input).toBe(''); + }); }); diff --git a/packages/core/src/trace-projection.ts b/packages/core/src/trace-projection.ts index 6997bba..10bc573 100644 --- a/packages/core/src/trace-projection.ts +++ b/packages/core/src/trace-projection.ts @@ -291,7 +291,11 @@ export function projectTrace(input: TraceProjectionInput): FolioTrace { startedAt, completedAt, latencyMs, - input: run?.input ?? evaluationRun?.answer ?? '', + // The evaluation run record carries no input of its own; the benchmark + // case prompt is the authoritative input for that run (spec ยง9). Falling + // back to `answer` would project the OUTPUT as the input, which is the + // exact reconstruction-from-run-state the projection must never do. + input: run?.input ?? evaluationCase?.input.prompt ?? '', answer, error: run?.error?.message ?? evaluationRun?.error?.message, completeness: deriveCompleteness(Boolean(run || evaluationRun), tools, traceEvents, traceRefForOutput),