diff --git a/src/contract/index.ts b/src/contract/index.ts index 83b3cf13..38333e5f 100644 --- a/src/contract/index.ts +++ b/src/contract/index.ts @@ -175,7 +175,12 @@ export { export { type CandidateExperimentExecutionInput, type CompareCandidateExperimentOptions, + type EvaluatePairedMeasurementsOptions, + evaluatePairedMeasurements, measuredComparisonFromCandidateExperiment, + type PairedMeasurement, + type PairedMeasurementAdapter, + type PairedMeasurementEvaluation, type RunCandidateExperimentOptions, runCandidateExperiment, type SealCandidateBenchmarkSuiteOptions, diff --git a/src/contract/measured-comparison.test.ts b/src/contract/measured-comparison.test.ts index 0edf9a33..0763a31b 100644 --- a/src/contract/measured-comparison.test.ts +++ b/src/contract/measured-comparison.test.ts @@ -11,6 +11,7 @@ import type { import { canonicalCandidateDigest } from '@tangle-network/agent-interface' import { describe, expect, it } from 'vitest' import { + evaluatePairedMeasurements, measuredComparisonFromCandidateExperiment, runCandidateExperiment, sealCandidateBenchmarkSuite, @@ -399,7 +400,187 @@ function neverOutput(): never { throw new Error('fixture task must use an output contract') } +interface PlatformProfileRun { + score: number + dimensions: Array<{ name: string; score: number }> + costUsd: number + latencyMs: number + completed: boolean + passed: boolean +} + +function profileMeasurements(): Array<{ + cellId: string + baseline: PlatformProfileRun + candidate: PlatformProfileRun +}> { + return [0, 1, 2].map((index) => { + const baseline = 0.2 + index * 0.05 + const candidate = baseline + 0.5 + const run = (score: number): PlatformProfileRun => ({ + score, + dimensions: [{ name: 'reliability', score }], + costUsd: 0.01, + latencyMs: 100, + completed: true, + passed: true, + }) + return { + cellId: `platform-case:${index}`, + baseline: run(baseline), + candidate: run(candidate), + } + }) +} + +const profileRunAdapter = { + score: (run: PlatformProfileRun) => run.score, + dimensions: (run: PlatformProfileRun) => run.dimensions, + costUsd: (run: PlatformProfileRun) => run.costUsd, + latencyMs: (run: PlatformProfileRun) => run.latencyMs, + completed: (run: PlatformProfileRun) => run.completed, + passed: (run: PlatformProfileRun) => run.passed, +} + describe('candidate experiment comparison', () => { + it('evaluates ordinary profile receipts through the same paired decision path', () => { + const policy = experiment().policy + const measurements = profileMeasurements() + + const result = evaluatePairedMeasurements({ + measurements, + policy, + adapter: profileRunAdapter, + sharedScorerChannel: true, + additionalCostUsd: 0.25, + }) + + expect(result.overall).toMatchObject({ baseline: 0.25, candidate: 0.75, delta: 0.5, n: 3 }) + expect(result.decision.outcome).toBe('ship') + expect(result.executionCostUsd).toBeCloseTo(0.06) + expect(result.totalCostUsd).toBeCloseTo(0.31) + expect(result.executionDurationMs).toBe(600) + expect(() => + evaluatePairedMeasurements({ + measurements: [measurements[0]!, measurements[0]!], + policy, + adapter: profileRunAdapter, + sharedScorerChannel: true, + }), + ).toThrow(/cell ids must be unique/) + expect(() => + evaluatePairedMeasurements({ + measurements, + policy: { ...policy, resamples: 0 } as unknown as typeof policy, + adapter: profileRunAdapter, + sharedScorerChannel: true, + }), + ).toThrow(/resamples/) + }) + + it.each([ + { + label: 'an empty matrix', + input: () => ({ measurements: [] as ReturnType }), + error: /requires at least one paired cell/, + }, + { + label: 'negative additional cost', + input: () => ({ additionalCostUsd: -0.01 }), + error: /additionalCostUsd must be a non-negative number/, + }, + { + label: 'a non-boolean scorer-channel declaration', + input: () => ({ sharedScorerChannel: 'shared' as never }), + error: /sharedScorerChannel must be a boolean/, + }, + { + label: 'a blank cell id', + input: () => ({ + measurements: [ + { ...profileMeasurements()[0]!, cellId: '' }, + ...profileMeasurements().slice(1), + ], + }), + error: /requires a cell id/, + }, + { + label: 'a non-finite score', + input: () => ({ adapter: { ...profileRunAdapter, score: () => Number.NaN } }), + error: /score must be finite/, + }, + { + label: 'non-array dimensions', + input: () => ({ + adapter: { ...profileRunAdapter, dimensions: () => 'reliability' as never }, + }), + error: /dimensions must be an array/, + }, + { + label: 'an unnamed dimension', + input: () => ({ + adapter: { ...profileRunAdapter, dimensions: () => [{ name: '', score: 0.5 }] }, + }), + error: /contains an unnamed dimension/, + }, + { + label: 'a repeated dimension', + input: () => ({ + adapter: { + ...profileRunAdapter, + dimensions: () => [ + { name: 'reliability', score: 0.5 }, + { name: 'reliability', score: 0.5 }, + ], + }, + }), + error: /repeats dimension 'reliability'/, + }, + { + label: 'different arm dimensions', + input: () => ({ + measurements: profileMeasurements().map((measurement, index) => + index === 0 + ? { + ...measurement, + candidate: { + ...measurement.candidate, + dimensions: [{ name: 'different', score: measurement.candidate.score }], + }, + } + : measurement, + ), + }), + error: /dimensions do not match the suite/, + }, + { + label: 'negative run cost', + input: () => ({ adapter: { ...profileRunAdapter, costUsd: () => -0.01 } }), + error: /cost must be non-negative/, + }, + { + label: 'negative run latency', + input: () => ({ adapter: { ...profileRunAdapter, latencyMs: () => -1 } }), + error: /latency must be non-negative/, + }, + { + label: 'non-boolean completion', + input: () => ({ adapter: { ...profileRunAdapter, completed: () => 'yes' as never } }), + error: /completion and pass values must be booleans/, + }, + ])('rejects $label from generic receipt adapters', ({ input, error }) => { + const policy = experiment().policy + expect(() => + evaluatePairedMeasurements({ + measurements: profileMeasurements(), + policy, + adapter: profileRunAdapter, + sharedScorerChannel: false, + ...input(), + }), + ).toThrow(error) + }) + it('rejects an experiment whose candidate is identical to its baseline', () => { const frozen = experiment() const { digest: _digest, ...material } = frozen @@ -460,6 +641,9 @@ describe('candidate experiment comparison', () => { }) expect(comparison.evaluation.executionCostUsd).toBeCloseTo(0.06) expect(comparison.evaluation.totalCostUsd).toBeCloseTo(0.31) + expect(canonicalCandidateDigest(comparison)).toBe( + 'sha256:a75abc57ab4ab7c02bd384a29be1771c93ae921d73411ae6b485b6973cf2f3d5', + ) expect(comparison.objectives).toEqual( expect.arrayContaining([ expect.objectContaining({ kind: 'cost', baseline: 0.01, candidate: 0.01 }), diff --git a/src/contract/measured-comparison.ts b/src/contract/measured-comparison.ts index 25bf710c..8ede6a23 100644 --- a/src/contract/measured-comparison.ts +++ b/src/contract/measured-comparison.ts @@ -4,6 +4,7 @@ import { type AgentCandidateBenchmarkTask, type AgentCandidateBenchmarkTaskMaterial, type AgentCandidateBundle, + type AgentCandidateEvaluationPolicy, type AgentCandidateExperiment, type AgentCandidateExperimentMaterial, type AgentCandidateExperimentMeasurement, @@ -11,6 +12,7 @@ import { agentCandidateBenchmarkSuiteSchema, agentCandidateBenchmarkTaskSchema, agentCandidateBundleSchema, + agentCandidateEvaluationPolicySchema, agentCandidateExperimentSchema, agentImprovementMeasuredComparisonSchema, type CandidateExecutionEvidence, @@ -57,6 +59,43 @@ export interface CompareCandidateExperimentOptions { metadata?: AgentImprovementMeasuredComparison['metadata'] } +/** One exact baseline/candidate observation of the same held-out cell. */ +export interface PairedMeasurement { + cellId: string + baseline: TRun + candidate: TRun +} + +/** Maps a product-owned run receipt into the measurements required for a fair paired decision. */ +export interface PairedMeasurementAdapter { + score(run: TRun): number + dimensions(run: TRun): readonly { name: string; score: number }[] + costUsd(run: TRun): number + latencyMs(run: TRun): number + completed(run: TRun): boolean + passed(run: TRun): boolean +} + +export interface EvaluatePairedMeasurementsOptions { + measurements: readonly PairedMeasurement[] + policy: AgentCandidateEvaluationPolicy + adapter: PairedMeasurementAdapter + /** Whether both arms use the same scorer family as the promotion decision. */ + sharedScorerChannel: boolean + /** Search or preparation spend that belongs to the same frozen budget. */ + additionalCostUsd?: number +} + +/** Statistical and operational result derived from complete paired receipts. */ +export type PairedMeasurementEvaluation = Pick< + AgentImprovementMeasuredComparison, + 'overall' | 'objectives' | 'decision' | 'power' +> & { + executionCostUsd: number + totalCostUsd: number + executionDurationMs: number +} + /** Content-address one task before any measured execution can see it. */ export function sealCandidateBenchmarkTask( material: AgentCandidateBenchmarkTaskMaterial, @@ -170,23 +209,37 @@ export async function runCandidateExperiment( return measurements } -/** Build the only publishable comparison: paired statistics over Runtime receipts. */ -export function measuredComparisonFromCandidateExperiment( - options: CompareCandidateExperimentOptions, -): AgentImprovementMeasuredComparison { - const experiment = verifyCandidateExperiment(options.experiment) +/** + * Calculate the shared paired decision from any complete receipt shape. + * + * Callers still own sealing their tasks, verifying each receipt against its + * expected arm and state, and proving every expected cell exists. This function + * only validates the projected measurements and derives their shared decision. + */ +export function evaluatePairedMeasurements( + options: EvaluatePairedMeasurementsOptions, +): PairedMeasurementEvaluation { + if (options.measurements.length === 0) { + throw new Error('paired measurement evaluation requires at least one paired cell') + } + const additionalCostUsd = options.additionalCostUsd ?? 0 + if (!Number.isFinite(additionalCostUsd) || additionalCostUsd < 0) { + throw new Error('paired measurement evaluation additionalCostUsd must be a non-negative number') + } + if (typeof options.sharedScorerChannel !== 'boolean') { + throw new Error('paired measurement evaluation sharedScorerChannel must be a boolean') + } + const policy = agentCandidateEvaluationPolicySchema.parse(options.policy) const measurements = options.measurements.map((measurement, index) => - verifyMeasurement(experiment, measurement, index), + projectPairedMeasurement(measurement, index, options.adapter), ) - const expectedN = experiment.benchmark.suite.taskDigests.length * experiment.benchmark.suite.reps - if (measurements.length !== expectedN) { - throw new Error( - `candidate experiment is incomplete (${measurements.length}/${expectedN} paired cells)`, - ) + const cellIds = measurements.map((measurement) => measurement.cellId) + if (new Set(cellIds).size !== cellIds.length) { + throw new Error('paired measurement evaluation cell ids must be unique') } - verifyStableProfileMaterialization(measurements) - if (!options.runId.trim()) throw new Error('candidate experiment runId is required') - + const dimensions = sharedProjectedDimensions(measurements) + const baselineScores = measurements.map((measurement) => measurement.baseline.score) + const candidateScores = measurements.map((measurement) => measurement.candidate.score) const { confidenceLevel: confidence, resamples, @@ -196,15 +249,23 @@ export function measuredComparisonFromCandidateExperiment( budgetUsd, criticalDimensions, regressionTolerance, - } = experiment.policy - const baselineScores = measurements.map(scoreOf('baseline')) - const candidateScores = measurements.map(scoreOf('candidate')) + } = policy + const significance = heldoutSignificance( + { before: baselineScores, after: candidateScores, cellIds }, + { + confidence, + resamples, + seed: bootstrapSeed, + statistic: 'mean', + deltaThreshold, + minProductiveRuns, + }, + ) const overall = measuredEstimate(baselineScores, candidateScores, { confidence, resamples, seed: bootstrapSeed, }) - const dimensions = sharedDimensions(measurements) const objectives: AgentImprovementMeasuredComparison['objectives'] = [ { kind: 'objective', @@ -222,21 +283,33 @@ export function measuredComparisonFromCandidateExperiment( unit: 'score' as const, availability: 'measured' as const, ...measuredEstimate( - measurements.map(dimensionOf('baseline', name)), - measurements.map(dimensionOf('candidate', name)), - { confidence, resamples, seed: bootstrapSeed + index + 1 }, + measurements.map((measurement) => dimensionScore(measurement.baseline, name)), + measurements.map((measurement) => dimensionScore(measurement.candidate, name)), + { + confidence, + resamples, + seed: bootstrapSeed + index + 1, + }, ), })), ] const cost = measuredEstimate( - measurements.map(costOf('baseline')), - measurements.map(costOf('candidate')), - { confidence, resamples, seed: bootstrapSeed + dimensions.length + 1 }, + measurements.map((measurement) => measurement.baseline.costUsd), + measurements.map((measurement) => measurement.candidate.costUsd), + { + confidence, + resamples, + seed: bootstrapSeed + dimensions.length + 1, + }, ) const latency = measuredEstimate( - measurements.map(latencyOf('baseline')), - measurements.map(latencyOf('candidate')), - { confidence, resamples, seed: bootstrapSeed + dimensions.length + 2 }, + measurements.map((measurement) => measurement.baseline.latencyMs), + measurements.map((measurement) => measurement.candidate.latencyMs), + { + confidence, + resamples, + seed: bootstrapSeed + dimensions.length + 2, + }, ) objectives.push( { @@ -257,17 +330,6 @@ export function measuredComparisonFromCandidateExperiment( }, ) - const significance = heldoutSignificance( - { before: baselineScores, after: candidateScores, cellIds: cellIds(experiment) }, - { - confidence, - resamples, - seed: bootstrapSeed, - statistic: 'mean', - deltaThreshold, - minProductiveRuns, - }, - ) const power = baselineScores.length >= 3 ? powerPreflight({ @@ -275,7 +337,7 @@ export function measuredComparisonFromCandidateExperiment( pairedN: baselineScores.length, deltaThreshold, confidence, - sharedScorerChannel: true, + sharedScorerChannel: options.sharedScorerChannel, }) : undefined const powerSufficient = @@ -292,21 +354,21 @@ export function measuredComparisonFromCandidateExperiment( objective.confidenceInterval.lower < -regressionTolerance, ) const executionCostUsd = measurements.reduce( - (sum, measurement) => - sum + costFromEvidence(measurement.baseline) + costFromEvidence(measurement.candidate), + (sum, measurement) => sum + measurement.baseline.costUsd + measurement.candidate.costUsd, + 0, + ) + const executionDurationMs = measurements.reduce( + (sum, measurement) => sum + measurement.baseline.latencyMs + measurement.candidate.latencyMs, 0, ) - const searchCostUsd = options.searchCostUsd ?? 0 - const totalCostUsd = executionCostUsd + searchCostUsd - const budgetPassed = budgetUsd === undefined || totalCostUsd <= budgetUsd const completedRuns = measurements.flatMap((measurement) => [ measurement.baseline, measurement.candidate, ]) - const incompleteRuns = completedRuns.filter((evidence) => !completedSuccessfully(evidence)) - const failedCandidateResults = measurements.filter( - (measurement) => !measurement.candidate.receipt.benchmarkResult.material.passed, - ) + const incompleteRuns = completedRuns.filter((run) => !run.completed) + const failedCandidateResults = measurements.filter((measurement) => !measurement.candidate.passed) + const totalCostUsd = executionCostUsd + additionalCostUsd + const budgetPassed = budgetUsd === undefined || totalCostUsd <= budgetUsd const checks = [ { name: 'paired-significance', passed: significance.significant }, { name: 'statistical-power', passed: powerSufficient }, @@ -344,18 +406,8 @@ export function measuredComparisonFromCandidateExperiment( : [`candidate failed ${failedCandidateResults.length} benchmark tasks`]), ...(budgetPassed ? [] : [`total cost ${totalCostUsd} exceeded budget ${budgetUsd}`]), ] - const diff = deriveCandidateBundleDiff(experiment) - const executionDurationMs = measurements.reduce( - (sum, measurement) => - sum + latencyFromEvidence(measurement.baseline) + latencyFromEvidence(measurement.candidate), - 0, - ) - const searchDurationMs = options.searchDurationMs ?? 0 - const durationMs = executionDurationMs + searchDurationMs - const provisional = agentImprovementMeasuredComparisonSchema.parse({ - kind: 'agent-improvement-measured-comparison', - experiment, - measurements, + + return { overall: { name: 'composite', direction: 'higher-is-better', @@ -363,7 +415,6 @@ export function measuredComparisonFromCandidateExperiment( ...overall, }, objectives, - ...(options.candidate ? { candidate: options.candidate } : {}), decision: { outcome: shipped ? 'ship' @@ -379,10 +430,56 @@ export function measuredComparisonFromCandidateExperiment( minimumDetectableDelta: power?.mde ?? 1, confidenceLevel: confidence, scaleAssumed: power?.scaleAssumed ?? true, - sharedScorerChannel: true, + sharedScorerChannel: options.sharedScorerChannel, reason: power?.recommendation ?? `need at least ${Math.max(3, minProductiveRuns)} paired runs`, }, + executionCostUsd, + totalCostUsd, + executionDurationMs, + } +} + +/** Build the only publishable comparison: paired statistics over Runtime receipts. */ +export function measuredComparisonFromCandidateExperiment( + options: CompareCandidateExperimentOptions, +): AgentImprovementMeasuredComparison { + const experiment = verifyCandidateExperiment(options.experiment) + const measurements = options.measurements.map((measurement, index) => + verifyMeasurement(experiment, measurement, index), + ) + const expectedN = experiment.benchmark.suite.taskDigests.length * experiment.benchmark.suite.reps + if (measurements.length !== expectedN) { + throw new Error( + `candidate experiment is incomplete (${measurements.length}/${expectedN} paired cells)`, + ) + } + verifyStableProfileMaterialization(measurements) + if (!options.runId.trim()) throw new Error('candidate experiment runId is required') + const searchCostUsd = options.searchCostUsd ?? 0 + const evaluation = evaluatePairedMeasurements({ + measurements: measurements.map((measurement, index) => ({ + cellId: cellIds(experiment)[index]!, + ...measurement, + })), + policy: experiment.policy, + adapter: candidateExecutionEvidenceAdapter, + sharedScorerChannel: true, + additionalCostUsd: searchCostUsd, + }) + const diff = deriveCandidateBundleDiff(experiment) + const searchDurationMs = options.searchDurationMs ?? 0 + const totalCostUsd = evaluation.totalCostUsd + const durationMs = evaluation.executionDurationMs + searchDurationMs + const provisional = agentImprovementMeasuredComparisonSchema.parse({ + kind: 'agent-improvement-measured-comparison', + experiment, + measurements, + overall: evaluation.overall, + objectives: evaluation.objectives, + ...(options.candidate ? { candidate: options.candidate } : {}), + decision: evaluation.decision, + power: evaluation.power, provenance: { kind: 'agent-eval-loop', schema: 'agent-candidate-experiment', @@ -395,10 +492,10 @@ export function measuredComparisonFromCandidateExperiment( evaluation: { generationsExplored: options.generationsExplored ?? 0, searchDurationMs, - executionDurationMs, + executionDurationMs: evaluation.executionDurationMs, durationMs, searchCostUsd, - executionCostUsd, + executionCostUsd: evaluation.executionCostUsd, totalCostUsd, }, ...(options.metadata ? { metadata: options.metadata } : {}), @@ -671,6 +768,92 @@ function verifyMaterialAddressed( } } +interface ProjectedRun { + score: number + dimensions: Map + costUsd: number + latencyMs: number + completed: boolean + passed: boolean +} + +interface ProjectedPairedMeasurement { + cellId: string + baseline: ProjectedRun + candidate: ProjectedRun +} + +function projectPairedMeasurement( + measurement: PairedMeasurement, + index: number, + adapter: PairedMeasurementAdapter, +): ProjectedPairedMeasurement { + if (typeof measurement.cellId !== 'string' || !measurement.cellId.trim()) { + throw new Error(`paired measurement ${index} requires a cell id`) + } + return { + cellId: measurement.cellId, + baseline: projectRun(measurement.baseline, adapter, `paired measurement ${index} baseline`), + candidate: projectRun(measurement.candidate, adapter, `paired measurement ${index} candidate`), + } +} + +function projectRun( + run: TRun, + adapter: PairedMeasurementAdapter, + label: string, +): ProjectedRun { + const suppliedDimensions = adapter.dimensions(run) + if (!Array.isArray(suppliedDimensions)) { + throw new Error(`${label} dimensions must be an array`) + } + const dimensions = new Map() + for (const dimension of suppliedDimensions) { + if (typeof dimension.name !== 'string' || !dimension.name.trim()) { + throw new Error(`${label} contains an unnamed dimension`) + } + if (dimensions.has(dimension.name)) { + throw new Error(`${label} repeats dimension '${dimension.name}'`) + } + dimensions.set(dimension.name, finiteMeasurement(dimension.score, `${label} ${dimension.name}`)) + } + const completed = adapter.completed(run) + const passed = adapter.passed(run) + if (typeof completed !== 'boolean' || typeof passed !== 'boolean') { + throw new Error(`${label} completion and pass values must be booleans`) + } + return { + score: finiteMeasurement(adapter.score(run), `${label} score`), + dimensions, + costUsd: nonNegativeMeasurement(adapter.costUsd(run), `${label} cost`), + latencyMs: nonNegativeMeasurement(adapter.latencyMs(run), `${label} latency`), + completed, + passed, + } +} + +function sharedProjectedDimensions(measurements: readonly ProjectedPairedMeasurement[]): string[] { + const expected = [...measurements[0]!.baseline.dimensions.keys()] + for (const [index, measurement] of measurements.entries()) { + for (const [arm, run] of [ + ['baseline', measurement.baseline], + ['candidate', measurement.candidate], + ] as const) { + const actual = [...run.dimensions.keys()] + if (JSON.stringify(actual) !== JSON.stringify(expected)) { + throw new Error(`paired measurement ${index} ${arm} dimensions do not match the suite`) + } + } + } + return expected +} + +function dimensionScore(run: ProjectedRun, name: string): number { + const value = run.dimensions.get(name) + if (value === undefined) throw new Error(`paired measurement is missing dimension '${name}'`) + return value +} + function measuredEstimate( baseline: number[], candidate: number[], @@ -704,53 +887,23 @@ function measuredEstimate( } } -function sharedDimensions(measurements: AgentCandidateExperimentMeasurement[]): string[] { - const expected = dimensionNames(measurements[0]?.baseline) - for (const [index, measurement] of measurements.entries()) { - for (const [arm, evidence] of [ - ['baseline', measurement.baseline], - ['candidate', measurement.candidate], - ] as const) { - const actual = dimensionNames(evidence) - if (JSON.stringify(actual) !== JSON.stringify(expected)) { - throw new Error( - `candidate experiment measurement ${index} ${arm} dimensions do not match the suite`, - ) - } - } - } - return expected -} - -function dimensionNames(evidence: CandidateExecutionEvidence | undefined): string[] { - return (evidence?.receipt.benchmarkResult.material.dimensions ?? []).map( - (dimension) => dimension.name, - ) -} - -function scoreOf(arm: 'baseline' | 'candidate') { - return (measurement: AgentCandidateExperimentMeasurement): number => - measurement[arm].receipt.benchmarkResult.material.score -} - -function dimensionOf(arm: 'baseline' | 'candidate', name: string) { - return (measurement: AgentCandidateExperimentMeasurement): number => { - const dimension = measurement[arm].receipt.benchmarkResult.material.dimensions.find( - (entry) => entry.name === name, - ) - if (!dimension) throw new Error(`candidate experiment is missing dimension '${name}'`) - return dimension.score - } +function finiteMeasurement(value: number, label: string): number { + if (!Number.isFinite(value)) throw new Error(`${label} must be finite`) + return value } -function costOf(arm: 'baseline' | 'candidate') { - return (measurement: AgentCandidateExperimentMeasurement): number => - costFromEvidence(measurement[arm]) +function nonNegativeMeasurement(value: number, label: string): number { + if (!Number.isFinite(value) || value < 0) throw new Error(`${label} must be non-negative`) + return value } -function latencyOf(arm: 'baseline' | 'candidate') { - return (measurement: AgentCandidateExperimentMeasurement): number => - latencyFromEvidence(measurement[arm]) +const candidateExecutionEvidenceAdapter: PairedMeasurementAdapter = { + score: (evidence) => evidence.receipt.benchmarkResult.material.score, + dimensions: (evidence) => evidence.receipt.benchmarkResult.material.dimensions, + costUsd: costFromEvidence, + latencyMs: latencyFromEvidence, + completed: completedSuccessfully, + passed: (evidence) => evidence.receipt.benchmarkResult.material.passed, } function costFromEvidence(evidence: CandidateExecutionEvidence): number {