diff --git a/bench/lib/eval/workspace.mjs b/bench/lib/eval/workspace.mjs index 2c27c39..d814257 100644 --- a/bench/lib/eval/workspace.mjs +++ b/bench/lib/eval/workspace.mjs @@ -20,6 +20,19 @@ export function evalAssessmentPath(root, id, phase) { return path.join(evalWorkspaceDir(root, id, phase), 'assessment.json') } +// Blinding: harness-written inputs must not name the run. The verification +// record carries the real runId (and could embed it in check output or +// workspace paths), so every string in it is rewritten to the blind handle +// before it reaches the evaluator. +function scrubRunIdentity(value, id, blindId) { + if (typeof value === 'string') return value.split(id).join(blindId) + if (Array.isArray(value)) return value.map((entry) => scrubRunIdentity(entry, id, blindId)) + if (value && typeof value === 'object') { + return Object.fromEntries(Object.entries(value).map(([key, entry]) => [key.split(id).join(blindId), scrubRunIdentity(entry, id, blindId)])) + } + return value +} + function copyInput(root, relativePath, inputRoot) { const source = path.resolve(root, relativePath) if (!source.startsWith(`${path.resolve(root)}${path.sep}`)) throw new Error(`Input escapes repository: ${relativePath}`) @@ -78,7 +91,7 @@ export function createEvalWorkspace(root, { id, phase, variantId, stageDir, veri writeJson(path.join(inputs, 'specs', 'nord-stage-4.variants.json'), projectVariants(variants, variantId)) writeJson(path.join(inputs, 'rubric.json'), rubric) if (verificationPath && fs.existsSync(verificationPath)) { - fs.copyFileSync(verificationPath, path.join(inputs, 'verification.json')) + writeJson(path.join(inputs, 'verification.json'), scrubRunIdentity(readJson(verificationPath), id, blindId)) } // Blind template: identified by the handle, never the model id. writeJson(path.join(workspace, 'assessment.json'), createAssessmentTemplate(rubric, blindId, phase)) diff --git a/tests/bench.test.mjs b/tests/bench.test.mjs index f951adb..f4eb358 100644 --- a/tests/bench.test.mjs +++ b/tests/bench.test.mjs @@ -180,6 +180,24 @@ test('a run flows new → start → seal → score with sealed digests and a sco assert.ok(!evalWorkspace.workspace.includes(created.id), 'the model id is not printed in the evaluator path') assert.ok(fs.readFileSync(path.join(evalWorkspace.workspace, 'EVAL.md'), 'utf8').includes('blind'), 'EVAL.md states the evaluation is blind') assert.equal(readJson(evalWorkspace.assessment).runId, blindRunCode(created.id), 'the template is identified by the blind handle') + assert.equal(readJson(path.join(evalWorkspace.inputs, 'verification.json')).runId, evalWorkspace.blindId, 'the verification record is re-identified by the blind handle') + // No harness-written file in the evaluator workspace may name the real run. + // artifact/ is exempt: it is the candidate's own sealed bytes, pinned by the + // digest check, so the harness cannot rewrite it. + const harnessFiles = [] + const collectHarnessFiles = (dir) => { + for (const entry of fs.readdirSync(dir, { withFileTypes: true })) { + const full = path.join(dir, entry.name) + if (entry.isDirectory()) { + if (full !== evalWorkspace.artifact) collectHarnessFiles(full) + } else harnessFiles.push(full) + } + } + collectHarnessFiles(evalWorkspace.workspace) + assert.ok(harnessFiles.some((file) => file.endsWith('verification.json')), 'the blinding sweep covers the verification record') + for (const file of harnessFiles) { + assert.ok(!fs.readFileSync(file).includes(created.id), `${path.relative(evalWorkspace.workspace, file)} leaks the run id`) + } const assessment = readJson(evalWorkspace.assessment) assessment.evaluator = 'Fixture evaluator'