From 62348724f5e1588531ad50f9095e1de339e45be1 Mon Sep 17 00:00:00 2001 From: Steve Calvert Date: Sat, 20 Jun 2026 18:12:01 -0700 Subject: [PATCH] fix(evals): warn once when testInfo is missing The 'testInfo not provided' warning fired on every runEvalDataset call without testInfo, spamming output when runVariantExperiment or scripted loops call it many times. Gate it behind a process-level flag so it warns once. Co-Authored-By: Claude Fable 5 --- src/evals/evalRunner.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/evals/evalRunner.ts b/src/evals/evalRunner.ts index c3fff3a..8083dd6 100644 --- a/src/evals/evalRunner.ts +++ b/src/evals/evalRunner.ts @@ -1078,6 +1078,10 @@ async function getGitHash(): Promise { return result.status === 0 ? result.stdout.trim() : undefined; } +// ponytail: warn once per process, not per call — the message is identical and +// runVariantExperiment / scripted loops call this many times. +let warnedNoTestInfo = false; + export async function runEvalDataset( options: EvalRunnerOptions, context: EvalContext @@ -1328,7 +1332,8 @@ export async function runEvalDataset( contentType: 'application/json', body: Buffer.from(JSON.stringify({ caseResults })), }); - } else if (caseResults.length > 0) { + } else if (caseResults.length > 0 && !warnedNoTestInfo) { + warnedNoTestInfo = true; console.warn( '[mcp-server-tester] runEvalDataset: testInfo not provided — results will not appear in the MCP reporter.\n' + 'To enable reporting, pass testInfo from the Playwright test function:\n' +