diff --git a/bench/CHANGELOG.md b/bench/CHANGELOG.md index e59f70db..80d87e20 100644 --- a/bench/CHANGELOG.md +++ b/bench/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## 0.3.8 + +- Resolve package-owned fixtures and scripts from both source and compiled installs, and prove ToolLLM fixture loading from the packed package. + ## 0.3.7 - Declare the public executable search path for Pier candidate entrypoints so the exact process contract can replay them against `@tangle-network/agent-runtime@0.102.0`. diff --git a/bench/package.json b/bench/package.json index f185f3bb..1bf21f0b 100644 --- a/bench/package.json +++ b/bench/package.json @@ -1,6 +1,6 @@ { "name": "@tangle-network/agent-bench", - "version": "0.3.7", + "version": "0.3.8", "type": "module", "description": "The unified benchmark suite for agent-runtime agents: 31 adapters (commit0, enterpriseops-gym, ragbench, crag, nomiracl, open-rag-bench, t2-ragbench, tau3-banking, bfcl, finresearchbench, …) behind one resolveAdapter registry, each with a real judge or fail-loud unsupported scorer. Score any profile/skill/prompt change against them. Map: bench/HARNESS.md.", "repository": { diff --git a/bench/scripts/verify-packed-consumer.mjs b/bench/scripts/verify-packed-consumer.mjs index ccfe2c4f..21fde2ab 100644 --- a/bench/scripts/verify-packed-consumer.mjs +++ b/bench/scripts/verify-packed-consumer.mjs @@ -120,7 +120,7 @@ try { ) await writeFile( path.join(consumerDir, 'index.mjs'), - "import { resolveAdapter, runBenchmarks } from '@tangle-network/agent-bench'\nimport { ADAPTERS } from '@tangle-network/agent-bench/adapters'\nimport { createCragAdapter } from '@tangle-network/agent-bench/benchmarks/crag'\n\nif (typeof resolveAdapter !== 'function' || typeof runBenchmarks !== 'function') throw new Error('root exports are not executable')\nif (typeof ADAPTERS !== 'object' || typeof createCragAdapter !== 'function') throw new Error('subpath exports are not executable')\nif (resolveAdapter('crag').name !== 'crag') throw new Error('compiled adapter registry returned the wrong adapter')\n", + "import { resolveAdapter, runBenchmarks } from '@tangle-network/agent-bench'\nimport { ADAPTERS } from '@tangle-network/agent-bench/adapters'\nimport { createCragAdapter } from '@tangle-network/agent-bench/benchmarks/crag'\n\nif (typeof resolveAdapter !== 'function' || typeof runBenchmarks !== 'function') throw new Error('root exports are not executable')\nif (typeof ADAPTERS !== 'object' || typeof createCragAdapter !== 'function') throw new Error('subpath exports are not executable')\nif (resolveAdapter('crag').name !== 'crag') throw new Error('compiled adapter registry returned the wrong adapter')\nprocess.env.TOOLLM_FIXTURES = '1'\nconst toolLlmTasks = await resolveAdapter('toollm').loadTasks({ limit: 1 })\nif (toolLlmTasks.length !== 1 || toolLlmTasks[0]?.id !== '1') throw new Error('packed ToolLLM fixture loading failed')\n", ) await writeFile( path.join(consumerDir, 'tsconfig.json'), diff --git a/bench/src/benchmarks/_harness.ts b/bench/src/benchmarks/_harness.ts index 7f82c53c..f123db89 100644 --- a/bench/src/benchmarks/_harness.ts +++ b/bench/src/benchmarks/_harness.ts @@ -18,6 +18,7 @@ import { execFile, spawn } from 'node:child_process' import { createHash } from 'node:crypto' +import { existsSync, readFileSync } from 'node:fs' import { cp, lstat, @@ -42,8 +43,25 @@ import type { const execFileAsync = promisify(execFile) -/** Repo root for the bench package (…/bench), so `.venv` and `fixtures` resolve. */ -export const benchRoot = fileURLToPath(new URL('../..', import.meta.url)) +/** Locate the package by identity because source files and compiled chunks have different depths. */ +function resolveBenchRoot(moduleUrl: string): string { + let current = dirname(fileURLToPath(moduleUrl)) + while (true) { + const manifestPath = join(current, 'package.json') + if (existsSync(manifestPath)) { + const manifest = JSON.parse(readFileSync(manifestPath, 'utf8')) as { name?: unknown } + if (manifest.name === '@tangle-network/agent-bench') return current + } + const parent = dirname(current) + if (parent === current) { + throw new Error(`Unable to locate @tangle-network/agent-bench from ${moduleUrl}`) + } + current = parent + } +} + +/** Package root for agent-bench, so `.venv`, scripts, and fixtures resolve. */ +export const benchRoot = resolveBenchRoot(import.meta.url) /** Resolve the shared interpreter without requiring an installed package to contain a venv. */ export function resolveBenchPython(