From 03fee7055c908eb65be7b5adc4ab82d5231710d6 Mon Sep 17 00:00:00 2001 From: Varun S G Date: Fri, 4 Sep 2026 22:47:47 +0530 Subject: [PATCH] fix(cli): forward solver config to all benchmarks and improve error resilience --- src/cli/index.test.ts | 60 +++++++++++++++ src/cli/index.ts | 165 +++++++++++++++--------------------------- 2 files changed, 119 insertions(+), 106 deletions(-) diff --git a/src/cli/index.test.ts b/src/cli/index.test.ts index 3a65c45..0226728 100644 --- a/src/cli/index.test.ts +++ b/src/cli/index.test.ts @@ -47,6 +47,21 @@ describe("bench-harness CLI", () => { ); }); + it("rejects non-numeric values for numeric flags", () => { + expect(() => parseArgs(["--concurrency", "abc"])).toThrow( + "--concurrency must be a valid number, got: abc" + ); + expect(() => parseArgs(["--limit", "xyz"])).toThrow( + "--limit must be a valid number, got: xyz" + ); + expect(() => parseArgs(["--start", "foo"])).toThrow( + "--start must be a valid number, got: foo" + ); + expect(() => parseArgs(["--epochs", "bar"])).toThrow( + "--epochs must be a valid number, got: bar" + ); + }); + it("passes reasoning effort to hand-built model benchmark configs", () => { for (const benchmarkId of [ "gpqa_diamond", @@ -290,4 +305,49 @@ describe("bench-harness CLI", () => { retrievalConfig: "bm25_grep", }); }); + + it("forwards solver-config options to gpqa_diamond, mmlu_pro, mmmu_pro_vision, and ifstruct", () => { + for (const benchmarkId of [ + "gpqa_diamond", + "mmlu_pro", + "mmmu_pro_vision", + "ifstruct", + ] as const) { + const config = buildBenchmarkConfig({ + benchmarkId, + model: "openai/gpt-5", + panelConfig: { + providerOnly: ["together"], + maxTokens: 2048, + timeoutMs: 30000, + }, + artifactDir: undefined, + endpointId: undefined, + imageDetail: benchmarkId === "mmmu_pro_vision" ? "high" : undefined, + reasoningEffort: "low", + }); + expect(config).toMatchObject({ + benchmarkId, + providerOnly: ["together"], + maxTokens: 2048, + timeoutMs: 30000, + reasoningEffort: "low", + ...(benchmarkId === "mmmu_pro_vision" ? { imageDetail: "high" } : {}), + }); + } + }); + + it("rejects unknown solver-config options for gpqa_diamond and mmlu_pro", () => { + expect(() => + buildBenchmarkConfig({ + benchmarkId: "gpqa_diamond", + model: "openai/gpt-5", + panelConfig: { unknownOption: true }, + artifactDir: undefined, + endpointId: undefined, + imageDetail: undefined, + reasoningEffort: "high", + }) + ).toThrow("Unknown gpqa_diamond solver-config option(s): unknownOption"); + }); }); diff --git a/src/cli/index.ts b/src/cli/index.ts index 922f378..9f7ba55 100644 --- a/src/cli/index.ts +++ b/src/cli/index.ts @@ -56,7 +56,14 @@ export function parseArgs(argv: readonly string[]): CliArgs { }; const num = (flag: string): number | undefined => { const raw = get(flag); - return raw !== undefined ? Number(raw) : undefined; + if (raw === undefined) { + return undefined; + } + const val = Number(raw); + if (!Number.isFinite(val)) { + throw new TypeError(`${flag} must be a valid number, got: ${raw}`); + } + return val; }; return { benchmark: get("--benchmark") ?? "gpqa_diamond", @@ -203,39 +210,47 @@ function main(): Promise { Presets.shades_classic ); let currentSample = ""; + let barStarted = false; if (total !== undefined) { bar.start(total, 0, { sample: "" }); + barStarted = true; + } + let result; + try { + result = yield* promise(() => + runBenchmarkById( + definedValues({ + benchmarkId: args.benchmark, + apiKey, + benchmarkConfig: benchmarkRunConfig, + epochs, + maxConcurrency: args.concurrency, + baseUrl: baseUrl ? baseUrl : undefined, + range, + sessionId, + resultStore: makeLocalResultStore({ + dir: join(process.cwd(), "bench-results"), + }), + progressReporter: makeProgressReporter({ + onSampleComplete: (completed) => + bar.update(completed, { sample: currentSample }), + onSampleStart: (event) => { + currentSample = `#${event.sampleIndex}`; + bar.update({ sample: currentSample }); + }, + onSampleEnd: () => { + currentSample = ""; + bar.update({ sample: currentSample }); + }, + }), + }) + ) + ); + } finally { + if (barStarted) { + bar.stop(); + } } - const result = yield* promise(() => - runBenchmarkById( - definedValues({ - benchmarkId: args.benchmark, - apiKey, - benchmarkConfig: benchmarkRunConfig, - epochs, - maxConcurrency: args.concurrency, - baseUrl: baseUrl ? baseUrl : undefined, - range, - sessionId, - resultStore: makeLocalResultStore({ - dir: join(process.cwd(), "bench-results"), - }), - progressReporter: makeProgressReporter({ - onSampleComplete: (completed) => - bar.update(completed, { sample: currentSample }), - onSampleStart: (event) => { - currentSample = `#${event.sampleIndex}`; - bar.update({ sample: currentSample }); - }, - onSampleEnd: () => { - currentSample = ""; - bar.update({ sample: currentSample }); - }, - }), - }) - ) - ); - bar.stop(); if (Either.isLeft(result)) { process.stderr.write(`Benchmark failed: ${result.left}\n`); process.exitCode = 1; @@ -330,6 +345,7 @@ function buildSchemaValidatedConfig(opts: { panelConfig: unknown; costTier?: CostTier; reasoningEffort: ReasoningEffort; + imageDetail?: ImageDetail; }): BenchmarkRunConfig { const { benchmarkId, @@ -338,6 +354,7 @@ function buildSchemaValidatedConfig(opts: { panelConfig, costTier, reasoningEffort, + imageDetail, } = opts; const merged: Record = definedValues({ benchmarkId, @@ -345,6 +362,7 @@ function buildSchemaValidatedConfig(opts: { endpointId, costTier, reasoningEffort, + imageDetail, }); if (typeof panelConfig === "object" && panelConfig !== null) { const known = isModelBenchmarkId(benchmarkId) @@ -419,58 +437,6 @@ export function buildBenchmarkConfig(opts: { reasoningEffort, } = opts; switch (benchmarkId) { - case "gpqa_diamond": { - return { - benchmarkId: "gpqa_diamond", - model: requireModel("gpqa_diamond", model), - ...definedValues({ - endpointId, - costTier, - }), - reasoningEffort, - }; - } - case "mmlu_pro": { - return { - benchmarkId: "mmlu_pro", - model: requireModel("mmlu_pro", model), - ...definedValues({ - endpointId, - costTier, - }), - reasoningEffort, - }; - } - case "tau_bench_verified_airline": { - return buildSchemaValidatedConfig({ - benchmarkId: "tau_bench_verified_airline", - model: requireModel("tau_bench_verified_airline", model), - endpointId, - panelConfig, - costTier, - reasoningEffort, - }); - } - case "tau3_bench_banking": { - return buildSchemaValidatedConfig({ - benchmarkId: "tau3_bench_banking", - model: requireModel("tau3_bench_banking", model), - endpointId, - panelConfig, - costTier, - reasoningEffort, - }); - } - case "terminal_bench": { - return buildSchemaValidatedConfig({ - benchmarkId: "terminal_bench", - model: requireModel("terminal_bench", model), - endpointId, - panelConfig, - costTier, - reasoningEffort, - }); - } case "draco": { const panel = parseSchema(DracoPanelConfigSchema, panelConfig); if (Either.isLeft(panel)) { @@ -484,29 +450,13 @@ export function buildBenchmarkConfig(opts: { }), }; } - case "mmmu_pro_vision": { - return { - benchmarkId: "mmmu_pro_vision", - model: requireModel("mmmu_pro_vision", model), - ...definedValues({ - endpointId, - imageDetail: opts.imageDetail, - costTier, - }), - reasoningEffort, - }; - } - case "ifstruct": { - return { - benchmarkId: "ifstruct", - model: requireModel("ifstruct", model), - ...definedValues({ - endpointId, - costTier, - }), - reasoningEffort, - }; - } + case "gpqa_diamond": + case "mmlu_pro": + case "tau_bench_verified_airline": + case "tau3_bench_banking": + case "terminal_bench": + case "mmmu_pro_vision": + case "ifstruct": case "swe_atlas_qa": case "swe_atlas_tw": case "swe_atlas_rf": @@ -524,6 +474,9 @@ export function buildBenchmarkConfig(opts: { panelConfig, costTier, reasoningEffort, + ...(benchmarkId === "mmmu_pro_vision" + ? definedValues({ imageDetail: opts.imageDetail }) + : {}), }); } default: {