diff --git a/CHANGELOG.md b/CHANGELOG.md index 9effc868..9f50964c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,12 @@ All notable changes to `@tangle-network/agent-eval` and its sibling `agent-eval- --- +## [0.126.6] - 2026-07-24 - optimizer model provenance + +### Added + +- Proxied GEPA and SkillOpt runs now record the configured optimizer model in `OptimizationMethodProvenance.optimizerModel`; GEPA engines without a configured optimizer omit it. + ## [0.126.5] - 2026-07-24 - published GEPA compatibility ### Fixed diff --git a/README.md b/README.md index aa19d28b..ab82ae77 100644 --- a/README.md +++ b/README.md @@ -283,7 +283,7 @@ console.table(result.scores) Read `scores` for final-case lift and intervals. Read `pairwise` before claiming one method beat another. Read `totalCost.accountingComplete` before using the reported dollars as a complete total. -Each official method score records the optimizer and bridge package versions, source revisions and source-tree hashes, Python runtime, custom engine module hashes, compatible run ID, exact attempt ID, resume status, evaluation count, artifact directory, and available optimizer token usage in `provenance`. +Each official method score records the optimizer and bridge package versions, source revisions and source-tree hashes, Python runtime, configured optimizer model when present, custom engine module hashes, compatible run ID, exact attempt ID, resume status, evaluation count, artifact directory, and available optimizer token usage in `provenance`. The [optimizer guide](./docs/campaign-proposers.md) covers recipes, budgets, resuming, and data separation. The [runnable comparison](./examples/compare-optimization-methods/) can run GEPA, SkillOpt, or both. diff --git a/clients/python/pyproject.toml b/clients/python/pyproject.toml index 842c7ca4..3c2b5503 100644 --- a/clients/python/pyproject.toml +++ b/clients/python/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "agent-eval-rpc" -version = "0.126.5" +version = "0.126.6" description = "Python RPC client, official optimizer bridge, and DSPy metric adapter for @tangle-network/agent-eval." readme = "README.md" requires-python = ">=3.10" diff --git a/clients/python/src/agent_eval_rpc/__init__.py b/clients/python/src/agent_eval_rpc/__init__.py index dc94df26..33b370c6 100644 --- a/clients/python/src/agent_eval_rpc/__init__.py +++ b/clients/python/src/agent_eval_rpc/__init__.py @@ -53,7 +53,7 @@ try: __version__ = version("agent-eval-rpc") except PackageNotFoundError: - __version__ = "0.126.5" + __version__ = "0.126.6" __all__ = [ "Client", diff --git a/clients/python/uv.lock b/clients/python/uv.lock index e4f13100..01d9c09b 100644 --- a/clients/python/uv.lock +++ b/clients/python/uv.lock @@ -34,7 +34,7 @@ conflicts = [[ [[package]] name = "agent-eval-rpc" -version = "0.126.5" +version = "0.126.6" source = { editable = "." } dependencies = [ { name = "filelock" }, diff --git a/package.json b/package.json index e9b1ddca..f72e044f 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@tangle-network/agent-eval", - "version": "0.126.5", + "version": "0.126.6", "description": "Evaluate and improve AI agents from runs, traces, judges, and feedback. Compare candidates, cluster failures, measure lift, and gate releases.", "homepage": "https://github.com/tangle-network/agent-eval#readme", "repository": { diff --git a/scripts/verify-package-exports.mjs b/scripts/verify-package-exports.mjs index 7b652184..2e7179df 100644 --- a/scripts/verify-package-exports.mjs +++ b/scripts/verify-package-exports.mjs @@ -113,6 +113,7 @@ try { import { type CostLedgerHandle as CampaignCostLedgerHandle, type LlmJudgeOptions as CampaignLlmJudgeOptions, + type OptimizationMethodProvenance, type ReferenceEquivalenceJudgeOptions as CampaignReferenceEquivalenceJudgeOptions, type SurfaceProposer as CampaignSurfaceProposer, } from '@tangle-network/agent-eval/campaign' @@ -179,6 +180,19 @@ try { const rootCostLedger: RootCostLedgerHandle = costLedger const campaignCostLedger: CampaignCostLedgerHandle = costLedger const contractCostLedger: ContractCostLedgerHandle = costLedger + const optimizationProvenance: OptimizationMethodProvenance = { + source: { + kind: 'package', + evidence: 'observed', + package: 'optimizer', + version: '1.0.0', + }, + optimizerModel: 'provider/model@2026-07-24', + runId: 'run', + resumed: false, + evaluationCount: 1, + artifactDir: '/tmp/optimizer', + } const contextTokens = contextInputTokens({ inputTokens: 10, cachedTokens: 20 }) const inputTokens = firstNumberAttr( { 'gen_ai.usage.input_tokens': '10' }, @@ -204,6 +218,7 @@ try { rootCostLedger, campaignCostLedger, contractCostLedger, + optimizationProvenance, LLM_INPUT_TOKENS, LLM_CONTEXT_TOKENS, contextTokens, diff --git a/src/campaign/gepa-optimization-method.ts b/src/campaign/gepa-optimization-method.ts index 1abd03bc..30e8760d 100644 --- a/src/campaign/gepa-optimization-method.ts +++ b/src/campaign/gepa-optimization-method.ts @@ -476,6 +476,7 @@ export function gepaOptimizationMethod( durationMs: Date.now() - started, provenance: { ...runtime, + ...(config.optimizer ? { optimizerModel: config.optimizer.model } : {}), compatibleRunId, runId, resumed: result.resumed, diff --git a/src/campaign/presets/compare-optimization-methods.ts b/src/campaign/presets/compare-optimization-methods.ts index 70b23556..6134da06 100644 --- a/src/campaign/presets/compare-optimization-methods.ts +++ b/src/campaign/presets/compare-optimization-methods.ts @@ -82,6 +82,8 @@ export interface OptimizationMethodProvenance { modules?: OptimizationModuleSource[] /** Python implementation used by the bridge process. */ python?: OptimizationPythonRuntime + /** Exact model identifier configured for optimizer-owned model calls. */ + optimizerModel?: string runId: string /** Content identity shared by compatible resumptions. */ compatibleRunId?: string @@ -558,6 +560,14 @@ function assertOptimizationProvenance( if (entry !== undefined && (typeof entry !== 'string' || !entry.trim())) fail(`source.${field}`) } if (typeof value.runId !== 'string' || !value.runId.trim()) fail('runId') + if ( + value.optimizerModel !== undefined && + (typeof value.optimizerModel !== 'string' || + !value.optimizerModel.trim() || + value.optimizerModel.trim() !== value.optimizerModel) + ) { + fail('optimizerModel') + } if (typeof value.resumed !== 'boolean') fail('resumed') if (!Number.isSafeInteger(value.evaluationCount) || value.evaluationCount < 0) { fail('evaluationCount') diff --git a/src/campaign/skillopt-optimization-method.ts b/src/campaign/skillopt-optimization-method.ts index 2ff66a7f..90c511bb 100644 --- a/src/campaign/skillopt-optimization-method.ts +++ b/src/campaign/skillopt-optimization-method.ts @@ -359,6 +359,7 @@ export function skillOptOptimizationMethod { sourceUrl: 'https://example.test/official-test', revision: 'abc123', }, + optimizerModel: 'test-model@2026-07-24', runId: 'official-run', resumed: false, evaluationCount: 6, @@ -209,10 +210,14 @@ describe('selfImprove — complete optimization methods', () => { package: 'official-test', revision: 'abc123', }), + optimizerModel: 'test-model@2026-07-24', runId: 'official-run', }), }) expect(result.provenance.optimizationMethod).toEqual(result.optimization) + expect(result.provenance.optimizationMethod?.provenance?.optimizerModel).toBe( + 'test-model@2026-07-24', + ) expect(result.receipts).toEqual([ expect.objectContaining({ channel: 'optimizer', diff --git a/tests/campaign/compare-optimization-methods.test.ts b/tests/campaign/compare-optimization-methods.test.ts index 502bc338..ec0f7db4 100644 --- a/tests/campaign/compare-optimization-methods.test.ts +++ b/tests/campaign/compare-optimization-methods.test.ts @@ -2,6 +2,7 @@ import { mkdtempSync, rmSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import type { OptimizationMethodProvenance } from '../../src/campaign/index' import { type CompareOptimizationMethodsOptions, compareOptimizationMethods, @@ -31,6 +32,22 @@ afterEach(() => { rmSync(runDir, { recursive: true, force: true }) }) +function optimizerProvenance(optimizerModel: string): OptimizationMethodProvenance { + return { + source: { + kind: 'package', + evidence: 'observed', + package: 'test-optimizer', + version: '1.0.0', + }, + optimizerModel, + runId: 'test-run', + resumed: false, + evaluationCount: 1, + artifactDir: '/tmp/test-optimizer', + } +} + describe('compareOptimizationMethods', () => { it('ranks methods by final-test lift and scores every winner uniformly', async () => { // Baseline solves nothing. The strong method solves all 4; the weak method solves 1. @@ -96,6 +113,37 @@ describe('compareOptimizationMethods', () => { ) }) + it('clones optimizer model provenance into score and best outputs', async () => { + const provenance = optimizerProvenance('provider/model@2026-07-24') + const result = await compareOptimizationMethods({ + methods: [ + { + name: 'model-backed', + optimize: async () => ({ + winnerSurface: 'SOLVE_h1', + cost: completeCost(0), + provenance, + }), + }, + ], + baselineSurface: 'nothing', + ...PARTITIONS, + dispatchWithSurface: async (surface) => ({ text: String(surface) }), + judges: [judge], + runDir, + expectUsage: 'off', + }) + + expect(result.scores[0]!.provenance).toEqual(provenance) + expect(result.scores[0]!.provenance).not.toBe(provenance) + expect(result.scores[0]!.provenance?.source).not.toBe(provenance.source) + + provenance.optimizerModel = 'mutated' + provenance.source.version = 'mutated' + expect(result.best.provenance?.optimizerModel).toBe('provider/model@2026-07-24') + expect(result.best.provenance?.source.version).toBe('1.0.0') + }) + it('reports a tie when two methods solve the same scenarios', async () => { const result = await compareOptimizationMethods({ methods: [fixedMethod('a', 'SOLVE_h1 SOLVE_h2', 1), fixedMethod('b', 'SOLVE_h1 SOLVE_h2', 1)], @@ -425,6 +473,34 @@ describe('compareOptimizationMethods', () => { expect(testDispatches).toBe(0) }) + it('rejects untrimmed optimizer model provenance before final scoring', async () => { + let testDispatches = 0 + await expect( + compareOptimizationMethods({ + methods: [ + { + name: 'invalid-provenance', + optimize: async () => ({ + winnerSurface: 'x', + cost: completeCost(0), + provenance: optimizerProvenance(' untrimmed-model'), + }), + }, + ], + baselineSurface: 'x', + ...PARTITIONS, + dispatchWithSurface: async (surface) => { + testDispatches += 1 + return { text: String(surface) } + }, + judges: [judge], + runDir, + expectUsage: 'off', + }), + ).rejects.toThrow(/provenance\.optimizerModel/) + expect(testDispatches).toBe(0) + }) + it('uses compareOptimizationMethods in downstream validation errors', async () => { const flakeyJudge: JudgeConfig = { name: 'flakey', diff --git a/tests/campaign/gepa-optimization-method.test.ts b/tests/campaign/gepa-optimization-method.test.ts index 27e455c3..6e8c12ed 100644 --- a/tests/campaign/gepa-optimization-method.test.ts +++ b/tests/campaign/gepa-optimization-method.test.ts @@ -271,11 +271,14 @@ describe('gepaOptimizationMethod', () => { accountingComplete: true, incompleteReasons: [], }) - expect(result.scores[0]!.provenance?.tokenUsage).toEqual({ - inputTokens: 10, - outputTokens: 5, - totalTokens: 15, - calls: 1, + expect(result.scores[0]!.provenance).toMatchObject({ + optimizerModel: 'model', + tokenUsage: { + inputTokens: 10, + outputTokens: 5, + totalTokens: 15, + calls: 1, + }, }) }) @@ -440,6 +443,7 @@ describe('gepaOptimizationMethod', () => { expect(result.scores[0]!.provenance?.runId).toMatch( new RegExp(`^${result.scores[0]!.provenance?.compatibleRunId}-[0-9a-f]{32}$`), ) + expect(result.scores[0]!.provenance).not.toHaveProperty('optimizerModel') expect(result.scores[0]!.provenance?.artifactDir).toContain('/gepa/external') }) diff --git a/tests/campaign/skillopt-optimization-method.test.ts b/tests/campaign/skillopt-optimization-method.test.ts index f6e341a8..acf3d220 100644 --- a/tests/campaign/skillopt-optimization-method.test.ts +++ b/tests/campaign/skillopt-optimization-method.test.ts @@ -245,6 +245,7 @@ describe('skillOptOptimizationMethod', () => { implementation: 'CPython', version: '3.12.0', }, + optimizerModel: 'model', compatibleRunId: expect.stringMatching(/^[0-9a-f]{64}$/), runId: expect.stringMatching(/^[0-9a-f]{64}-[0-9a-f]{32}$/), resumed: false,