Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions bench/CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,5 +1,9 @@
# Changelog

## 0.3.8

- Resolve package-owned fixtures and scripts from both source and compiled installs, and prove ToolLLM fixture loading from the packed package.

## 0.3.7

- Declare the public executable search path for Pier candidate entrypoints so the exact process contract can replay them against `@tangle-network/agent-runtime@0.102.0`.
Expand Down
2 changes: 1 addition & 1 deletion bench/package.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "@tangle-network/agent-bench",
"version": "0.3.7",
"version": "0.3.8",
"type": "module",
"description": "The unified benchmark suite for agent-runtime agents: 31 adapters (commit0, enterpriseops-gym, ragbench, crag, nomiracl, open-rag-bench, t2-ragbench, tau3-banking, bfcl, finresearchbench, …) behind one resolveAdapter registry, each with a real judge or fail-loud unsupported scorer. Score any profile/skill/prompt change against them. Map: bench/HARNESS.md.",
"repository": {
Expand Down
2 changes: 1 addition & 1 deletion bench/scripts/verify-packed-consumer.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -120,7 +120,7 @@ try {
)
await writeFile(
path.join(consumerDir, 'index.mjs'),
"import { resolveAdapter, runBenchmarks } from '@tangle-network/agent-bench'\nimport { ADAPTERS } from '@tangle-network/agent-bench/adapters'\nimport { createCragAdapter } from '@tangle-network/agent-bench/benchmarks/crag'\n\nif (typeof resolveAdapter !== 'function' || typeof runBenchmarks !== 'function') throw new Error('root exports are not executable')\nif (typeof ADAPTERS !== 'object' || typeof createCragAdapter !== 'function') throw new Error('subpath exports are not executable')\nif (resolveAdapter('crag').name !== 'crag') throw new Error('compiled adapter registry returned the wrong adapter')\n",
"import { resolveAdapter, runBenchmarks } from '@tangle-network/agent-bench'\nimport { ADAPTERS } from '@tangle-network/agent-bench/adapters'\nimport { createCragAdapter } from '@tangle-network/agent-bench/benchmarks/crag'\n\nif (typeof resolveAdapter !== 'function' || typeof runBenchmarks !== 'function') throw new Error('root exports are not executable')\nif (typeof ADAPTERS !== 'object' || typeof createCragAdapter !== 'function') throw new Error('subpath exports are not executable')\nif (resolveAdapter('crag').name !== 'crag') throw new Error('compiled adapter registry returned the wrong adapter')\nprocess.env.TOOLLM_FIXTURES = '1'\nconst toolLlmTasks = await resolveAdapter('toollm').loadTasks({ limit: 1 })\nif (toolLlmTasks.length !== 1 || toolLlmTasks[0]?.id !== '1') throw new Error('packed ToolLLM fixture loading failed')\n",
)
await writeFile(
path.join(consumerDir, 'tsconfig.json'),
Expand Down
22 changes: 20 additions & 2 deletions bench/src/benchmarks/_harness.ts
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@

import { execFile, spawn } from 'node:child_process'
import { createHash } from 'node:crypto'
import { existsSync, readFileSync } from 'node:fs'
import {
cp,
lstat,
Expand All @@ -42,8 +43,25 @@ import type {

const execFileAsync = promisify(execFile)

/** Repo root for the bench package (…/bench), so `.venv` and `fixtures` resolve. */
export const benchRoot = fileURLToPath(new URL('../..', import.meta.url))
/** Locate the package by identity because source files and compiled chunks have different depths. */
function resolveBenchRoot(moduleUrl: string): string {
let current = dirname(fileURLToPath(moduleUrl))
while (true) {
const manifestPath = join(current, 'package.json')
if (existsSync(manifestPath)) {
const manifest = JSON.parse(readFileSync(manifestPath, 'utf8')) as { name?: unknown }
if (manifest.name === '@tangle-network/agent-bench') return current
}
const parent = dirname(current)
if (parent === current) {
throw new Error(`Unable to locate @tangle-network/agent-bench from ${moduleUrl}`)
}
current = parent
}
}

/** Package root for agent-bench, so `.venv`, scripts, and fixtures resolve. */
export const benchRoot = resolveBenchRoot(import.meta.url)

/** Resolve the shared interpreter without requiring an installed package to contain a venv. */
export function resolveBenchPython(
Expand Down
Loading