stg <- dev

CHANGELOG.md

@@ -34,6 +34,11 @@ The config file `skill-benchmark.json` is no longer auto-detected. Rename it to
 - **benchmark:** Strip provider prefix from model ID when using direct `anthropic` or `openai` formats. Previously, `anthropic/claude-sonnet-4-6` was sent as-is to the Anthropic API, which expects `claude-sonnet-4-6`. The `pi` format is unaffected.
 - **model IDs:** OpenRouter model slugs now preserve dots in version numbers (e.g. `openrouter/anthropic/claude-sonnet-4.6`). Presets updated to match OpenRouter's catalog exactly. The dot→hyphen rewrite in `validate`/`fix` now applies only to the `anthropic/` direct-API prefix; `openrouter/` and `openai/` slugs are exempt.
 - **error message:** `E_MODEL_ID_FORMAT` now lists all three valid provider prefixes (`openrouter/`, `anthropic/`, `openai/`) instead of directing all users to use `openrouter/`.
+- Prompt-surface benchmarks no longer hard-FAIL on `scopeCoverage.coverageViolation`; coverage is informational for prompt runs (`src/benchmark/scoring.ts`).
+- Prompt-surface tasks are now scored against the specific capability they exercise via a required `capabilityId` on `GeneratedTask`. Previously every task was scored against the first discovered capability (`src/benchmark/runner.ts`, `src/benchmark/prompt-criteria.ts`, `src/tasks/generate.ts`).
+- Prompt evaluator surfaces `noActiveCriteria: true` (score 0, runner-level FAIL with an actionable message) when a capability's section produces empty criteria, replacing the previous vacuous 1.0 pass (`src/benchmark/prompt-evaluator.ts`).
+- `openai/` direct-API model IDs are exempt from dot→hyphen rewriting in `applyFixes`. OpenAI's API slugs use dots (`gpt-5.4`, `gpt-4.1`). (`src/project/fix.ts`)
+- Removed dead `src/discovery/prompt.ts`. Active discovery path is `src/project/discover-prompt.ts`.
 
 ## 1.0.0 — 2026-04-14
 

README.md

@@ -4,6 +4,8 @@ Benchmark and self-optimize SDK, CLI, and MCP guidance so every agent model can
 
 skill-optimizer runs your SDK / CLI / MCP docs against multiple LLMs, measures whether they call the right actions with the right arguments, and iteratively rewrites your `SKILL.md` / docs until a floor score is met across every model.
 
+Built by the team at [Fast](https://fast.xyz/) — payment infrastructure for AI agents. [Give your agent a wallet](https://github.com/fastxyz/fast-sdk) in 3 lines of code.
+
 **Requirements:** Node.js 20+, plus either an [OpenRouter](https://openrouter.ai) API key or a local Codex login when using direct OpenAI models.
 
 ## How it works — at a glance
@@ -132,10 +134,14 @@ skill-optimizer run
 
 The prompt surface discovers phases and capabilities from your SKILL.md,
 generates scenario-based tasks, and evaluates output quality — not just
-tool calls. It scores responses on required sections, format patterns,
-forbidden keywords, and structural elements (code blocks, numbered lists,
-tables). This lets you optimize prompt templates the same way you optimize
-SDK/CLI/MCP guidance.
+tool calls. Each task is tagged with the specific capability it exercises
+(`capabilityId`), and scoring is performed against that capability's
+criteria — not the first discovered capability. It scores responses on
+required sections, format patterns, forbidden keywords, and structural
+elements (code blocks, numbered lists, tables). Coverage violations do
+not hard-fail prompt runs; coverage is informational for the prompt
+surface. This lets you optimize prompt templates the same way you
+optimize SDK/CLI/MCP guidance.
 
 ## How it works
 

package.json

@@ -49,7 +49,7 @@
     "build": "tsc && npm run gen-docs && chmod +x dist/cli.js",
     "typecheck": "tsc --noEmit",
     "lint": "tsc --noUnusedLocals --noEmit",
-    "test": "tsx tests/smoke-code.ts && tsx tests/smoke-sdk-python.ts && tsx tests/smoke-sdk-rust.ts && tsx tests/smoke-cli.ts && tsx tests/smoke-cli-entry.ts && tsx tests/smoke-mcp.ts && tsx tests/smoke-llm.ts && tsx tests/smoke-discovery-sdk.ts && tsx tests/smoke-discovery-cli.ts && tsx tests/smoke-discovery-mcp.ts && tsx tests/smoke-discovery-prompt.ts && tsx tests/smoke-prompt-evaluator.ts && tsx tests/smoke-snapshot-prompt.ts && tsx tests/smoke-generation.ts && tsx tests/smoke-optimize.ts && tsx tests/smoke-mock-repos.ts && tsx tests/smoke-release.ts && tsx tests/smoke-scoring.ts && tsx tests/smoke-scope.ts && tsx tests/smoke-coverage.ts && tsx tests/smoke-feedback.ts && tsx tests/smoke-verdict.ts && tsx tests/smoke-dry-run.ts && tsx tests/smoke-errors.ts && tsx tests/smoke-e2e.ts && tsx tests/smoke-import.ts && tsx tests/smoke-init.ts && tsx tests/smoke-gen-docs.ts && tsx tests/smoke-actions.ts"
+    "test": "tsx tests/smoke-code.ts && tsx tests/smoke-sdk-python.ts && tsx tests/smoke-sdk-rust.ts && tsx tests/smoke-cli.ts && tsx tests/smoke-cli-entry.ts && tsx tests/smoke-mcp.ts && tsx tests/smoke-llm.ts && tsx tests/smoke-discovery-sdk.ts && tsx tests/smoke-discovery-cli.ts && tsx tests/smoke-discovery-mcp.ts && tsx tests/smoke-prompt-evaluator.ts && tsx tests/smoke-prompt-criteria.ts && tsx tests/smoke-snapshot-prompt.ts && tsx tests/smoke-generation.ts && tsx tests/smoke-optimize.ts && tsx tests/smoke-mock-repos.ts && tsx tests/smoke-release.ts && tsx tests/smoke-changelog-coverage.ts && tsx tests/smoke-scoring.ts && tsx tests/smoke-scope.ts && tsx tests/smoke-coverage.ts && tsx tests/smoke-feedback.ts && tsx tests/smoke-verdict.ts && tsx tests/smoke-verdict-prompt.ts && tsx tests/smoke-dry-run.ts && tsx tests/smoke-errors.ts && tsx tests/smoke-model-ids.ts && tsx tests/smoke-e2e.ts && tsx tests/smoke-import.ts && tsx tests/smoke-init.ts && tsx tests/smoke-gen-docs.ts && tsx tests/smoke-actions.ts"
   },
   "dependencies": {
     "@clack/prompts": "^1.2.0",

src/benchmark/config.ts

@@ -50,7 +50,7 @@ export function loadTasks(tasksPath: string, baseDir?: string): TaskDefinition[]
     throw new Error(`Failed to read tasks: ${resolved}: ${err instanceof Error ? err.message : err}`);
   }
 
-  let parsed: { tasks: Array<{ id?: unknown; prompt?: unknown; expected_actions?: unknown; verify?: unknown; expected_fetches?: unknown }> };
+  let parsed: { tasks: Array<{ id?: unknown; prompt?: unknown; expected_actions?: unknown; verify?: unknown; expected_fetches?: unknown; capabilityId?: unknown }> };
   try {
     parsed = JSON.parse(raw) as typeof parsed;
   } catch (err) {
@@ -65,7 +65,7 @@ export function loadTasks(tasksPath: string, baseDir?: string): TaskDefinition[]
 }
 
 function normalizeTaskDefinition(
-  task: { id?: unknown; prompt?: unknown; expected_actions?: unknown; verify?: unknown; expected_fetches?: unknown },
+  task: { id?: unknown; prompt?: unknown; expected_actions?: unknown; verify?: unknown; expected_fetches?: unknown; capabilityId?: unknown },
   resolvedPath: string,
   index: number,
 ): TaskDefinition {
@@ -105,12 +105,15 @@ function normalizeTaskDefinition(
     }
   }
 
+  const capabilityId = typeof task.capabilityId === 'string' ? task.capabilityId : undefined;
+
   return {
     id: task.id,
     prompt: task.prompt,
     expected_actions,
     verify: rawVerify as TaskDefinition['verify'] | undefined,
     expected_fetches: rawFetches as string[] | undefined,
+    ...(capabilityId !== undefined ? { capabilityId } : {}),
   };
 }
 

src/benchmark/prompt-criteria.ts

@@ -0,0 +1,41 @@
+import type { GeneratedTask } from '../tasks/types.js';
+import type { PromptCapabilityWithSection } from '../project/discover-prompt.js';
+import type { PromptEvaluationCriteria } from './prompt-evaluator.js';
+import { generateCriteriaFromCapability } from './prompt-evaluator.js';
+
+export interface ResolvedPromptCriteria {
+  criteria: PromptEvaluationCriteria;
+  noActiveCriteria: boolean;
+}
+
+function isEmptyCriteria(c: PromptEvaluationCriteria): boolean {
+  const s = (c.requiredSections?.length ?? 0) === 0;
+  const k = (c.requiredKeywords?.length ?? 0) === 0 && (c.forbiddenKeywords?.length ?? 0) === 0;
+  const f = (c.formatPatterns?.length ?? 0) === 0 && (c.minLength ?? 0) === 0;
+  const structure =
+    c.hasCodeBlocks === undefined &&
+    c.hasNumberedList === undefined &&
+    c.hasTable === undefined;
+  return s && k && f && structure;
+}
+
+export function resolveCriteriaForTask(
+  task: GeneratedTask,
+  caps: readonly PromptCapabilityWithSection[],
+): ResolvedPromptCriteria {
+  if (!task.capabilityId) {
+    throw new Error(
+      `Task ${task.id}: prompt-surface task is missing capabilityId. ` +
+      `Regenerate tasks with \`skill-optimizer generate-tasks\`.`,
+    );
+  }
+  const cap = caps.find((c) => c.action.key === task.capabilityId);
+  if (!cap) {
+    const known = caps.map((c) => c.action.key).join(', ') || '(none discovered)';
+    throw new Error(
+      `Task ${task.id}: capabilityId "${task.capabilityId}" is not in the discovered capability set. Known: ${known}`,
+    );
+  }
+  const criteria = generateCriteriaFromCapability(cap.action, cap.section);
+  return { criteria, noActiveCriteria: isEmptyCriteria(criteria) };
+}

src/benchmark/prompt-evaluator.ts

@@ -44,6 +44,12 @@ export interface PromptEvaluationResult {
     keywords: number;
     structure: number;
   };
+  /**
+   * True when every criteria category is empty. Evaluation cannot produce a
+   * meaningful score in this case; runner treats this as an evaluation error
+   * rather than a pass.
+   */
+  noActiveCriteria: boolean;
 }
 
 // ── Weights ───────────────────────────────────────────────────────────────────
@@ -237,9 +243,11 @@ export function evaluatePromptResponse(
   if (structureTotal > 0) activeParts.push({ weight: WEIGHT_STRUCTURE, score: structureScore });
 
   let score: number;
+  let noActiveCriteria = false;
   if (activeParts.length === 0) {
-    // No criteria specified at all — vacuously pass
-    score = 1.0;
+    // No criteria are active — treat as evaluation error, not vacuous pass.
+    score = 0;
+    noActiveCriteria = true;
   } else {
     const totalActiveWeight = activeParts.reduce((s, p) => s + p.weight, 0);
     score = activeParts.reduce((s, p) => s + (p.weight / totalActiveWeight) * p.score, 0);
@@ -255,6 +263,7 @@ export function evaluatePromptResponse(
       keywords: keywordScore,
       structure: structureScore,
     },
+    noActiveCriteria,
   };
 }
 

src/benchmark/runner.ts

@@ -25,8 +25,8 @@ import { evaluateTask } from './evaluator.js';
 import { computeCoverage } from './coverage.js';
 import { computeVerdict } from './scoring.js';
 import { buildSystemPrompt, buildTaskPrompt } from './prompts.js';
-import { evaluatePromptResponse, generateCriteriaFromCapability } from './prompt-evaluator.js';
-import type { PromptEvaluationCriteria } from './prompt-evaluator.js';
+import { evaluatePromptResponse } from './prompt-evaluator.js';
+import { resolveCriteriaForTask } from './prompt-criteria.js';
 import { discoverPromptCapabilitiesWithSections } from '../project/discover-prompt.js';
 
 function buildWebFetchTool(): McpToolDefinition {
@@ -178,15 +178,12 @@ export async function runBenchmark(options: RunnerOptions = {}): Promise<Benchma
     console.log('[skill] No skill configured — using generic system prompt\n');
   }
 
-  // Pre-compute evaluation criteria for prompt surface.
-  let promptEvalCriteria: PromptEvaluationCriteria | undefined;
-  if (config.surface === 'prompt' && skill) {
-    const caps = discoverPromptCapabilitiesWithSections(skill.content);
-    if (caps.length > 0) {
-      // Use the primary capability's section to derive criteria.
-      promptEvalCriteria = generateCriteriaFromCapability(caps[0].action, caps[0].section);
-      console.log(`[prompt] Evaluation criteria derived from ${caps.length} discovered capabilities`);
-    }
+  // Discover prompt capabilities once; criteria are resolved per-task inside the loop.
+  const promptCaps = (config.surface === 'prompt' && skill)
+    ? discoverPromptCapabilitiesWithSections(skill.content)
+    : [];
+  if (config.surface === 'prompt') {
+    console.log(`[prompt] ${promptCaps.length} capabilities discovered`);
   }
 
 
@@ -365,16 +362,28 @@ export async function runBenchmark(options: RunnerOptions = {}): Promise<Benchma
         surface: config.surface,
       });
 
-      // Prompt surface: replace vacuous tool-recall with content-based score.
-      if (config.surface === 'prompt' && promptEvalCriteria) {
+      // Prompt surface: replace vacuous tool-recall with per-capability content score.
+      if (config.surface === 'prompt') {
         try {
-          const promptResult = evaluatePromptResponse(rawResponse, promptEvalCriteria);
-          taskResult.metrics.toolRecall = promptResult.score;
-          taskResult.metrics.taskPassed = promptResult.score >= 0.5;
-          console.log(`  [${slug}] Prompt score: ${promptResult.score.toFixed(3)} → ${taskResult.metrics.taskPassed ? 'PASS' : 'FAIL'}`);
+          const { criteria } = resolveCriteriaForTask(task, promptCaps);
+          const promptResult = evaluatePromptResponse(rawResponse, criteria);
+          if (promptResult.noActiveCriteria) {
+            const msg = `Task "${task.id}" has no extractable criteria — fix SKILL.md section for that action`;
+            taskResult.metrics.toolRecall = 0;
+            taskResult.metrics.taskPassed = false;
+            taskResult.error = taskResult.error ?? msg;
+            console.error(`  [${slug}] Prompt eval error: ${msg}`);
+          } else {
+            taskResult.metrics.toolRecall = promptResult.score;
+            taskResult.metrics.taskPassed = promptResult.score >= 0.5;
+            console.log(`  [${slug}] Prompt score: ${promptResult.score.toFixed(3)} → ${taskResult.metrics.taskPassed ? 'PASS' : 'FAIL'}`);
+          }
         } catch (err) {
-          console.error(`  [${slug}] Prompt eval error: ${err instanceof Error ? err.message : err}`);
-          // Leave vacuous scores in place — task shows as PASS rather than crashing the run
+          const msg = err instanceof Error ? err.message : String(err);
+          console.error(`  [${slug}] Prompt eval error: ${msg}`);
+          taskResult.metrics.toolRecall = 0;
+          taskResult.metrics.taskPassed = false;
+          taskResult.error = taskResult.error ?? msg;
         }
       }
 

src/benchmark/scoring.ts

@@ -50,7 +50,10 @@ export function computeVerdict(
     );
   }
 
-  if (report.scopeCoverage?.coverageViolation) {
+  // Prompt surface: coverage is reported but does not veto the verdict.
+  // Prompt tasks are not required to map 1:1 to capabilities, so a
+  // coverage gap here is informational rather than a failure condition.
+  if (report.scopeCoverage?.coverageViolation && report.config.surface !== 'prompt') {
     reasons.push('coverage violation: some in-scope actions have zero tasks');
   }
 

src/benchmark/types.ts

@@ -145,6 +145,7 @@ export interface TaskDefinition {
   expected_actions: ExpectedAction[];
   verify?: TaskVerification[];
   expected_fetches?: string[];
+  capabilityId?: string;
 }
 
 // === Extracted from generated code or tool_calls ===