From 60223d6c7100873bdf6580e093c6d872c9306ede Mon Sep 17 00:00:00 2001 From: wangzhenjia Date: Tue, 22 Sep 2026 11:41:07 +0800 Subject: [PATCH] =?UTF-8?q?fix(trace):=20=E8=AF=84=E4=BC=B0=E8=BF=90?= =?UTF-8?q?=E8=A1=8C=E7=9A=84=20trace=20input=20=E4=B8=8D=E5=86=8D?= =?UTF-8?q?=E5=9B=9E=E9=80=80=E6=88=90=20answer?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `projectTrace()` 组装 `FolioTrace.input` 时: ```ts input: run?.input ?? evaluationRun?.answer ?? '' ``` `EvaluationRun` 没有 `input` 字段(它的输入在 benchmark case 的 `input.prompt` 里),于是评估类 trace 会把 agent 的**输出**当成**输入** 投影出去。TraceInspector 里 "Input" 一行因此显示成回答文本,而且永远 落不到 `t('trace.notRecorded')` 的兜底。 这与 trace-projection 的既定纪律直接冲突——文件头写明"NEVER reconstruct historical context from current app state … anything else is not-recorded", 且同文件 `contextFromCase()` 的注释明确"benchmark case input — authoritative (spec §9)"。 修复:改为从权威来源取输入 ```ts input: run?.input ?? evaluationCase?.input.prompt ?? '' ``` - 会话运行:仍用 `run.input`(未变) - 评估运行:用 case 的 `prompt` - 两者都没有:空串,UI 显示"未记录" 范围:仅 `projectTrace()` 的 `input` 字段赋值,其余投影逻辑未动。 有可见 UI 变化:评估类 trace 的 Input 行从"回答文本"变为"case prompt" (无 case 时显示"未记录")。 测试:`packages/core/src/trace-projection.test.ts` 新增 2 个用例 (用 case prompt;无权威输入时为空串)。旧代码下 11 pass / 2 fail, 修复后 51 pass / 0 fail(core 全量)。 --- packages/core/src/trace-projection.test.ts | 45 ++++++++++++++++++++++ packages/core/src/trace-projection.ts | 6 ++- 2 files changed, 50 insertions(+), 1 deletion(-) diff --git a/packages/core/src/trace-projection.test.ts b/packages/core/src/trace-projection.test.ts index 4a34d385..d27ee39a 100644 --- a/packages/core/src/trace-projection.test.ts +++ b/packages/core/src/trace-projection.test.ts @@ -224,4 +224,49 @@ describe('projectTrace (V9.1)', () => { const toolStep = trace.steps.find((s) => s.kind === 'tool' && s.tool?.id === 'live-run'); expect(toolStep?.status).toBe('running'); }); + + it('projects the benchmark case prompt as the trace input, never the answer', () => { + const caseDef: EvaluationCase = { + id: 'case-1', + name: 'NVDA research', + category: 'tool-selection', + difficulty: 'golden', + input: { prompt: 'Research NVDA' }, + expected: {}, + tags: [], + source: 'hand-authored', + }; + const evalRun: EvaluationRun = { + id: 'eval-run-1', + experimentId: 'exp-1', + caseId: 'case-1', + datasetId: 'ds-1', + status: 'completed', + startedAt: 1000, + completedAt: 9000, + answer: 'NVDA looks fine', + toolCalls: [], + failureModes: [], + }; + const trace = projectTrace({ evaluationRun: evalRun, evaluationCase: caseDef }); + expect(trace.input).toBe('Research NVDA'); + expect(trace.input).not.toBe(evalRun.answer); + }); + + it('leaves the trace input empty when no authoritative prompt was recorded', () => { + const evalRun: EvaluationRun = { + id: 'eval-run-2', + experimentId: 'exp-1', + caseId: 'case-2', + datasetId: 'ds-1', + status: 'completed', + startedAt: 1000, + completedAt: 9000, + answer: 'NVDA looks fine', + toolCalls: [], + failureModes: [], + }; + const trace = projectTrace({ evaluationRun: evalRun }); + expect(trace.input).toBe(''); + }); }); diff --git a/packages/core/src/trace-projection.ts b/packages/core/src/trace-projection.ts index 6997bbab..10bc573c 100644 --- a/packages/core/src/trace-projection.ts +++ b/packages/core/src/trace-projection.ts @@ -291,7 +291,11 @@ export function projectTrace(input: TraceProjectionInput): FolioTrace { startedAt, completedAt, latencyMs, - input: run?.input ?? evaluationRun?.answer ?? '', + // The evaluation run record carries no input of its own; the benchmark + // case prompt is the authoritative input for that run (spec §9). Falling + // back to `answer` would project the OUTPUT as the input, which is the + // exact reconstruction-from-run-state the projection must never do. + input: run?.input ?? evaluationCase?.input.prompt ?? '', answer, error: run?.error?.message ?? evaluationRun?.error?.message, completeness: deriveCompleteness(Boolean(run || evaluationRun), tools, traceEvents, traceRefForOutput),