From afac0acaafe34202d03fd5803330c5d2f4c38d86 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Fri, 24 Jul 2026 18:22:06 -0600 Subject: [PATCH 1/2] fix(bench): honor absolute Terminal-Bench venv paths --- bench/scripts/verify-packed-consumer.mjs | 65 +++++++++++++++++++++++- bench/src/benchmarks/_harness.ts | 9 +++- bench/src/benchmarks/terminal-bench.ts | 3 +- 3 files changed, 74 insertions(+), 3 deletions(-) diff --git a/bench/scripts/verify-packed-consumer.mjs b/bench/scripts/verify-packed-consumer.mjs index 21fde2ab..869f577b 100644 --- a/bench/scripts/verify-packed-consumer.mjs +++ b/bench/scripts/verify-packed-consumer.mjs @@ -54,9 +54,53 @@ try { const packDir = path.join(scratch, 'pack') const runtimePackDir = path.join(scratch, 'runtime-pack') const consumerDir = path.join(scratch, 'consumer') + const terminalBenchVenv = path.join(scratch, 'terminal-bench-venv') + const terminalBenchBinDir = path.join(terminalBenchVenv, 'bin') await mkdir(packDir) await mkdir(runtimePackDir) await mkdir(consumerDir) + await mkdir(terminalBenchBinDir, { recursive: true }) + await writeFile( + path.join(terminalBenchBinDir, 'python'), + String.raw`#!/usr/bin/env node +const rows = [{ + id: 'installed-absolute-venv', + instruction: 'prove the installed path', + task_dir: '/tmp/terminal-bench-task', + solution: 'echo ok\n', +}] +process.stdout.write(JSON.stringify(rows) + '\n') +`, + { mode: 0o755 }, + ) + await writeFile( + path.join(terminalBenchBinDir, 'tb'), + String.raw`#!/usr/bin/env node +import { mkdirSync, writeFileSync } from 'node:fs' +import { join } from 'node:path' + +const args = process.argv.slice(2) +function value(flag) { + const index = args.indexOf(flag) + if (index === -1 || index + 1 >= args.length) throw new Error('missing ' + flag) + return args[index + 1] +} + +const outputPath = value('--output-path') +const runId = value('--run-id') +const taskId = value('-t') +const reportDir = join(outputPath, runId) +mkdirSync(reportDir, { recursive: true }) +writeFileSync( + join(reportDir, 'results.json'), + JSON.stringify({ + resolved_ids: [taskId], + results: [{ task_id: taskId, is_resolved: true, parser_results: { installedVenv: true } }], + }), +) +`, + { mode: 0o755 }, + ) // Build explicitly so verification cannot inherit a machine-level // ignore-scripts setting and accidentally pack stale or missing output. @@ -122,6 +166,21 @@ try { path.join(consumerDir, 'index.mjs'), "import { resolveAdapter, runBenchmarks } from '@tangle-network/agent-bench'\nimport { ADAPTERS } from '@tangle-network/agent-bench/adapters'\nimport { createCragAdapter } from '@tangle-network/agent-bench/benchmarks/crag'\n\nif (typeof resolveAdapter !== 'function' || typeof runBenchmarks !== 'function') throw new Error('root exports are not executable')\nif (typeof ADAPTERS !== 'object' || typeof createCragAdapter !== 'function') throw new Error('subpath exports are not executable')\nif (resolveAdapter('crag').name !== 'crag') throw new Error('compiled adapter registry returned the wrong adapter')\nprocess.env.TOOLLM_FIXTURES = '1'\nconst toolLlmTasks = await resolveAdapter('toollm').loadTasks({ limit: 1 })\nif (toolLlmTasks.length !== 1 || toolLlmTasks[0]?.id !== '1') throw new Error('packed ToolLLM fixture loading failed')\n", ) + await writeFile( + path.join(consumerDir, 'terminal-bench-absolute-venv.mjs'), + `import { createTerminalBenchAdapter } from '@tangle-network/agent-bench/benchmarks/terminal-bench' + +const adapter = createTerminalBenchAdapter() +const tasks = await adapter.loadTasks({ ids: ['installed-absolute-venv'] }) +if (tasks.length !== 1 || tasks[0]?.id !== 'installed-absolute-venv') { + throw new Error('packed Terminal-Bench adapter did not load through the absolute venv') +} +const score = await adapter.judge(tasks[0], 'echo installed-package-path') +if (!score.resolved || score.score !== 1) { + throw new Error(\`packed Terminal-Bench adapter did not judge through the absolute venv: \${JSON.stringify(score)}\`) +} +`, + ) await writeFile( path.join(consumerDir, 'tsconfig.json'), `${JSON.stringify( @@ -165,6 +224,10 @@ for name in sorted(expected): consumerDir, ) await run('node', ['index.mjs'], consumerDir) + await run('node', ['terminal-bench-absolute-venv.mjs'], consumerDir, { + ...process.env, + TERMINAL_BENCH_VENV: terminalBenchVenv, + }) await run('npm', ['exec', '--', 'tsc', '-p', 'tsconfig.json'], consumerDir) const typescript5 = await run('npm', ['exec', '--', 'tsc', '--version'], consumerDir) if (typescript5.stdout.trim() !== `Version ${TYPESCRIPT_5}`) { @@ -228,7 +291,7 @@ for name in sorted(expected): }) } console.log( - `packed consumer verified: ${manifest.name}@${manifest.version} with @tangle-network/agent-runtime@${runtimeManifest.version}, TypeScript ${TYPESCRIPT_5} and ${TYPESCRIPT_6}; prepared ${prepareProof.executionPlanDigest}`, + `packed consumer verified: ${manifest.name}@${manifest.version} with @tangle-network/agent-runtime@${runtimeManifest.version}, TypeScript ${TYPESCRIPT_5} and ${TYPESCRIPT_6}, absolute Terminal-Bench venv; prepared ${prepareProof.executionPlanDigest}`, ) } finally { await rm(scratch, { recursive: true, force: true }) diff --git a/bench/src/benchmarks/_harness.ts b/bench/src/benchmarks/_harness.ts index f123db89..e77b8ab0 100644 --- a/bench/src/benchmarks/_harness.ts +++ b/bench/src/benchmarks/_harness.ts @@ -82,7 +82,14 @@ export const venvPython = resolveBenchPython() /** Interpreter for a NAMED isolated venv (e.g. `.venv-commit0`). Benches whose pip * deps conflict with the shared `.venv` (commit0 downgrades pydantic/sqlalchemy) * get their own venv and pass its python explicitly — keeping the shared one clean. */ -export const venvPythonAt = (venvDir: string): string => join(benchRoot, venvDir, 'bin', 'python') +export const venvPythonAt = (venvDir: string): string => venvBinAt(venvDir, 'python') + +/** Resolve an executable in an isolated venv. Relative venv paths are package-owned; + * absolute paths allow installed consumers to keep large environments elsewhere. */ +export function venvBinAt(venvDir: string, name: string): string { + return join(resolve(benchRoot, venvDir), 'bin', name) +} + /** Report/transcript reads are large; 256 MiB matches the SWE harness budget. */ export const bigBuffer = 1024 * 1024 * 256 diff --git a/bench/src/benchmarks/terminal-bench.ts b/bench/src/benchmarks/terminal-bench.ts index f73b05b5..42e458ad 100644 --- a/bench/src/benchmarks/terminal-bench.ts +++ b/bench/src/benchmarks/terminal-bench.ts @@ -25,6 +25,7 @@ import { runVenvPython, safeRunId, stageFile, + venvBinAt, venvPythonAt, } from './_harness' import type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types' @@ -34,7 +35,7 @@ import type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './typ // override the env without reloading this module. const terminalBenchVenvDir = (): string => process.env.TERMINAL_BENCH_VENV ?? '.venv-terminal-bench' const terminalBenchPython = (): string => venvPythonAt(terminalBenchVenvDir()) -const terminalBenchBin = (): string => join(benchRoot, terminalBenchVenvDir(), 'bin', 'tb') +const terminalBenchBin = (): string => venvBinAt(terminalBenchVenvDir(), 'tb') // Pinned dataset: the 0.1.1 core set is patched for terminal-bench >=0.2.4 (the // installed CLI) and is the published launch task set. name==version is what `tb From 8c3279e87653190d816e90fad7ee5cbf0bb058f6 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Fri, 24 Jul 2026 18:53:10 -0600 Subject: [PATCH 2/2] test(bench): harden venv path coverage --- bench/scripts/verify-packed-consumer.mjs | 1 + bench/src/benchmarks/_harness.test.mts | 17 ++++++++++++++++- bench/src/benchmarks/_harness.ts | 2 +- 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/bench/scripts/verify-packed-consumer.mjs b/bench/scripts/verify-packed-consumer.mjs index 869f577b..1b529216 100644 --- a/bench/scripts/verify-packed-consumer.mjs +++ b/bench/scripts/verify-packed-consumer.mjs @@ -60,6 +60,7 @@ try { await mkdir(runtimePackDir) await mkdir(consumerDir) await mkdir(terminalBenchBinDir, { recursive: true }) + await writeFile(path.join(terminalBenchVenv, 'package.json'), '{"type":"module"}\n') await writeFile( path.join(terminalBenchBinDir, 'python'), String.raw`#!/usr/bin/env node diff --git a/bench/src/benchmarks/_harness.test.mts b/bench/src/benchmarks/_harness.test.mts index a0101dc5..488b7f3a 100644 --- a/bench/src/benchmarks/_harness.test.mts +++ b/bench/src/benchmarks/_harness.test.mts @@ -4,7 +4,13 @@ import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises' import test from 'node:test' import { tmpdir } from 'node:os' import { join } from 'node:path' -import { resolveBenchPython, runStagedJudge } from './_harness' +import { + benchRoot, + resolveBenchPython, + runStagedJudge, + venvBin, + venvBinAt, +} from './_harness' const digest = (bytes: Uint8Array): `sha256:${string}` => `sha256:${createHash('sha256').update(bytes).digest('hex')}` @@ -31,6 +37,15 @@ test('resolveBenchPython rejects relative and empty interpreter paths', () => { ) }) +test('venv executable paths support package-owned and external environments', () => { + assert.equal(venvBinAt('/srv/terminal-bench', 'tb'), join('/srv/terminal-bench', 'bin', 'tb')) + assert.equal( + venvBinAt('.venv-commit0', 'python'), + join(benchRoot, '.venv-commit0', 'bin', 'python'), + ) + assert.equal(venvBin('python'), venvBinAt('.venv', 'python')) +}) + test('runStagedJudge terminates a hung evaluator at its explicit timeout', async () => { const started = Date.now() await assert.rejects( diff --git a/bench/src/benchmarks/_harness.ts b/bench/src/benchmarks/_harness.ts index e77b8ab0..dc06c83d 100644 --- a/bench/src/benchmarks/_harness.ts +++ b/bench/src/benchmarks/_harness.ts @@ -95,7 +95,7 @@ export const bigBuffer = 1024 * 1024 * 256 /** Path to a named executable inside the bench venv (e.g. `venvBin('tb')`). */ export function venvBin(name: string): string { - return join(benchRoot, '.venv', 'bin', name) + return venvBinAt('.venv', name) } /**