diff --git a/CHANGELOG.md b/CHANGELOG.md index 89a5b19..23e237d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,6 +34,11 @@ The config file `skill-benchmark.json` is no longer auto-detected. Rename it to - **benchmark:** Strip provider prefix from model ID when using direct `anthropic` or `openai` formats. Previously, `anthropic/claude-sonnet-4-6` was sent as-is to the Anthropic API, which expects `claude-sonnet-4-6`. The `pi` format is unaffected. - **model IDs:** OpenRouter model slugs now preserve dots in version numbers (e.g. `openrouter/anthropic/claude-sonnet-4.6`). Presets updated to match OpenRouter's catalog exactly. The dot→hyphen rewrite in `validate`/`fix` now applies only to the `anthropic/` direct-API prefix; `openrouter/` and `openai/` slugs are exempt. - **error message:** `E_MODEL_ID_FORMAT` now lists all three valid provider prefixes (`openrouter/`, `anthropic/`, `openai/`) instead of directing all users to use `openrouter/`. +- Prompt-surface benchmarks no longer hard-FAIL on `scopeCoverage.coverageViolation`; coverage is informational for prompt runs (`src/benchmark/scoring.ts`). +- Prompt-surface tasks are now scored against the specific capability they exercise via a required `capabilityId` on `GeneratedTask`. Previously every task was scored against the first discovered capability (`src/benchmark/runner.ts`, `src/benchmark/prompt-criteria.ts`, `src/tasks/generate.ts`). +- Prompt evaluator surfaces `noActiveCriteria: true` (score 0, runner-level FAIL with an actionable message) when a capability's section produces empty criteria, replacing the previous vacuous 1.0 pass (`src/benchmark/prompt-evaluator.ts`). +- `openai/` direct-API model IDs are exempt from dot→hyphen rewriting in `applyFixes`. OpenAI's API slugs use dots (`gpt-5.4`, `gpt-4.1`). (`src/project/fix.ts`) +- Removed dead `src/discovery/prompt.ts`. Active discovery path is `src/project/discover-prompt.ts`. ## 1.0.0 — 2026-04-14 diff --git a/README.md b/README.md index 8e22aff..28b28bb 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,8 @@ Benchmark and self-optimize SDK, CLI, and MCP guidance so every agent model can skill-optimizer runs your SDK / CLI / MCP docs against multiple LLMs, measures whether they call the right actions with the right arguments, and iteratively rewrites your `SKILL.md` / docs until a floor score is met across every model. +Built by the team at [Fast](https://fast.xyz/) — payment infrastructure for AI agents. [Give your agent a wallet](https://github.com/fastxyz/fast-sdk) in 3 lines of code. + **Requirements:** Node.js 20+, plus either an [OpenRouter](https://openrouter.ai) API key or a local Codex login when using direct OpenAI models. ## How it works — at a glance @@ -132,10 +134,14 @@ skill-optimizer run The prompt surface discovers phases and capabilities from your SKILL.md, generates scenario-based tasks, and evaluates output quality — not just -tool calls. It scores responses on required sections, format patterns, -forbidden keywords, and structural elements (code blocks, numbered lists, -tables). This lets you optimize prompt templates the same way you optimize -SDK/CLI/MCP guidance. +tool calls. Each task is tagged with the specific capability it exercises +(`capabilityId`), and scoring is performed against that capability's +criteria — not the first discovered capability. It scores responses on +required sections, format patterns, forbidden keywords, and structural +elements (code blocks, numbered lists, tables). Coverage violations do +not hard-fail prompt runs; coverage is informational for the prompt +surface. This lets you optimize prompt templates the same way you +optimize SDK/CLI/MCP guidance. ## How it works diff --git a/package.json b/package.json index 0a96499..8422dea 100644 --- a/package.json +++ b/package.json @@ -49,7 +49,7 @@ "build": "tsc && npm run gen-docs && chmod +x dist/cli.js", "typecheck": "tsc --noEmit", "lint": "tsc --noUnusedLocals --noEmit", - "test": "tsx tests/smoke-code.ts && tsx tests/smoke-sdk-python.ts && tsx tests/smoke-sdk-rust.ts && tsx tests/smoke-cli.ts && tsx tests/smoke-cli-entry.ts && tsx tests/smoke-mcp.ts && tsx tests/smoke-llm.ts && tsx tests/smoke-discovery-sdk.ts && tsx tests/smoke-discovery-cli.ts && tsx tests/smoke-discovery-mcp.ts && tsx tests/smoke-discovery-prompt.ts && tsx tests/smoke-prompt-evaluator.ts && tsx tests/smoke-snapshot-prompt.ts && tsx tests/smoke-generation.ts && tsx tests/smoke-optimize.ts && tsx tests/smoke-mock-repos.ts && tsx tests/smoke-release.ts && tsx tests/smoke-scoring.ts && tsx tests/smoke-scope.ts && tsx tests/smoke-coverage.ts && tsx tests/smoke-feedback.ts && tsx tests/smoke-verdict.ts && tsx tests/smoke-dry-run.ts && tsx tests/smoke-errors.ts && tsx tests/smoke-e2e.ts && tsx tests/smoke-import.ts && tsx tests/smoke-init.ts && tsx tests/smoke-gen-docs.ts && tsx tests/smoke-actions.ts" + "test": "tsx tests/smoke-code.ts && tsx tests/smoke-sdk-python.ts && tsx tests/smoke-sdk-rust.ts && tsx tests/smoke-cli.ts && tsx tests/smoke-cli-entry.ts && tsx tests/smoke-mcp.ts && tsx tests/smoke-llm.ts && tsx tests/smoke-discovery-sdk.ts && tsx tests/smoke-discovery-cli.ts && tsx tests/smoke-discovery-mcp.ts && tsx tests/smoke-prompt-evaluator.ts && tsx tests/smoke-prompt-criteria.ts && tsx tests/smoke-snapshot-prompt.ts && tsx tests/smoke-generation.ts && tsx tests/smoke-optimize.ts && tsx tests/smoke-mock-repos.ts && tsx tests/smoke-release.ts && tsx tests/smoke-changelog-coverage.ts && tsx tests/smoke-scoring.ts && tsx tests/smoke-scope.ts && tsx tests/smoke-coverage.ts && tsx tests/smoke-feedback.ts && tsx tests/smoke-verdict.ts && tsx tests/smoke-verdict-prompt.ts && tsx tests/smoke-dry-run.ts && tsx tests/smoke-errors.ts && tsx tests/smoke-model-ids.ts && tsx tests/smoke-e2e.ts && tsx tests/smoke-import.ts && tsx tests/smoke-init.ts && tsx tests/smoke-gen-docs.ts && tsx tests/smoke-actions.ts" }, "dependencies": { "@clack/prompts": "^1.2.0", diff --git a/src/benchmark/config.ts b/src/benchmark/config.ts index b8ac3fa..b5f9923 100644 --- a/src/benchmark/config.ts +++ b/src/benchmark/config.ts @@ -50,7 +50,7 @@ export function loadTasks(tasksPath: string, baseDir?: string): TaskDefinition[] throw new Error(`Failed to read tasks: ${resolved}: ${err instanceof Error ? err.message : err}`); } - let parsed: { tasks: Array<{ id?: unknown; prompt?: unknown; expected_actions?: unknown; verify?: unknown; expected_fetches?: unknown }> }; + let parsed: { tasks: Array<{ id?: unknown; prompt?: unknown; expected_actions?: unknown; verify?: unknown; expected_fetches?: unknown; capabilityId?: unknown }> }; try { parsed = JSON.parse(raw) as typeof parsed; } catch (err) { @@ -65,7 +65,7 @@ export function loadTasks(tasksPath: string, baseDir?: string): TaskDefinition[] } function normalizeTaskDefinition( - task: { id?: unknown; prompt?: unknown; expected_actions?: unknown; verify?: unknown; expected_fetches?: unknown }, + task: { id?: unknown; prompt?: unknown; expected_actions?: unknown; verify?: unknown; expected_fetches?: unknown; capabilityId?: unknown }, resolvedPath: string, index: number, ): TaskDefinition { @@ -105,12 +105,15 @@ function normalizeTaskDefinition( } } + const capabilityId = typeof task.capabilityId === 'string' ? task.capabilityId : undefined; + return { id: task.id, prompt: task.prompt, expected_actions, verify: rawVerify as TaskDefinition['verify'] | undefined, expected_fetches: rawFetches as string[] | undefined, + ...(capabilityId !== undefined ? { capabilityId } : {}), }; } diff --git a/src/benchmark/prompt-criteria.ts b/src/benchmark/prompt-criteria.ts new file mode 100644 index 0000000..7c57360 --- /dev/null +++ b/src/benchmark/prompt-criteria.ts @@ -0,0 +1,41 @@ +import type { GeneratedTask } from '../tasks/types.js'; +import type { PromptCapabilityWithSection } from '../project/discover-prompt.js'; +import type { PromptEvaluationCriteria } from './prompt-evaluator.js'; +import { generateCriteriaFromCapability } from './prompt-evaluator.js'; + +export interface ResolvedPromptCriteria { + criteria: PromptEvaluationCriteria; + noActiveCriteria: boolean; +} + +function isEmptyCriteria(c: PromptEvaluationCriteria): boolean { + const s = (c.requiredSections?.length ?? 0) === 0; + const k = (c.requiredKeywords?.length ?? 0) === 0 && (c.forbiddenKeywords?.length ?? 0) === 0; + const f = (c.formatPatterns?.length ?? 0) === 0 && (c.minLength ?? 0) === 0; + const structure = + c.hasCodeBlocks === undefined && + c.hasNumberedList === undefined && + c.hasTable === undefined; + return s && k && f && structure; +} + +export function resolveCriteriaForTask( + task: GeneratedTask, + caps: readonly PromptCapabilityWithSection[], +): ResolvedPromptCriteria { + if (!task.capabilityId) { + throw new Error( + `Task ${task.id}: prompt-surface task is missing capabilityId. ` + + `Regenerate tasks with \`skill-optimizer generate-tasks\`.`, + ); + } + const cap = caps.find((c) => c.action.key === task.capabilityId); + if (!cap) { + const known = caps.map((c) => c.action.key).join(', ') || '(none discovered)'; + throw new Error( + `Task ${task.id}: capabilityId "${task.capabilityId}" is not in the discovered capability set. Known: ${known}`, + ); + } + const criteria = generateCriteriaFromCapability(cap.action, cap.section); + return { criteria, noActiveCriteria: isEmptyCriteria(criteria) }; +} diff --git a/src/benchmark/prompt-evaluator.ts b/src/benchmark/prompt-evaluator.ts index 0067f44..8946629 100644 --- a/src/benchmark/prompt-evaluator.ts +++ b/src/benchmark/prompt-evaluator.ts @@ -44,6 +44,12 @@ export interface PromptEvaluationResult { keywords: number; structure: number; }; + /** + * True when every criteria category is empty. Evaluation cannot produce a + * meaningful score in this case; runner treats this as an evaluation error + * rather than a pass. + */ + noActiveCriteria: boolean; } // ── Weights ─────────────────────────────────────────────────────────────────── @@ -237,9 +243,11 @@ export function evaluatePromptResponse( if (structureTotal > 0) activeParts.push({ weight: WEIGHT_STRUCTURE, score: structureScore }); let score: number; + let noActiveCriteria = false; if (activeParts.length === 0) { - // No criteria specified at all — vacuously pass - score = 1.0; + // No criteria are active — treat as evaluation error, not vacuous pass. + score = 0; + noActiveCriteria = true; } else { const totalActiveWeight = activeParts.reduce((s, p) => s + p.weight, 0); score = activeParts.reduce((s, p) => s + (p.weight / totalActiveWeight) * p.score, 0); @@ -255,6 +263,7 @@ export function evaluatePromptResponse( keywords: keywordScore, structure: structureScore, }, + noActiveCriteria, }; } diff --git a/src/benchmark/runner.ts b/src/benchmark/runner.ts index 9aa8edd..3976ed5 100644 --- a/src/benchmark/runner.ts +++ b/src/benchmark/runner.ts @@ -25,8 +25,8 @@ import { evaluateTask } from './evaluator.js'; import { computeCoverage } from './coverage.js'; import { computeVerdict } from './scoring.js'; import { buildSystemPrompt, buildTaskPrompt } from './prompts.js'; -import { evaluatePromptResponse, generateCriteriaFromCapability } from './prompt-evaluator.js'; -import type { PromptEvaluationCriteria } from './prompt-evaluator.js'; +import { evaluatePromptResponse } from './prompt-evaluator.js'; +import { resolveCriteriaForTask } from './prompt-criteria.js'; import { discoverPromptCapabilitiesWithSections } from '../project/discover-prompt.js'; function buildWebFetchTool(): McpToolDefinition { @@ -178,15 +178,12 @@ export async function runBenchmark(options: RunnerOptions = {}): Promise 0) { - // Use the primary capability's section to derive criteria. - promptEvalCriteria = generateCriteriaFromCapability(caps[0].action, caps[0].section); - console.log(`[prompt] Evaluation criteria derived from ${caps.length} discovered capabilities`); - } + // Discover prompt capabilities once; criteria are resolved per-task inside the loop. + const promptCaps = (config.surface === 'prompt' && skill) + ? discoverPromptCapabilitiesWithSections(skill.content) + : []; + if (config.surface === 'prompt') { + console.log(`[prompt] ${promptCaps.length} capabilities discovered`); } @@ -365,16 +362,28 @@ export async function runBenchmark(options: RunnerOptions = {}): Promise= 0.5; - console.log(` [${slug}] Prompt score: ${promptResult.score.toFixed(3)} → ${taskResult.metrics.taskPassed ? 'PASS' : 'FAIL'}`); + const { criteria } = resolveCriteriaForTask(task, promptCaps); + const promptResult = evaluatePromptResponse(rawResponse, criteria); + if (promptResult.noActiveCriteria) { + const msg = `Task "${task.id}" has no extractable criteria — fix SKILL.md section for that action`; + taskResult.metrics.toolRecall = 0; + taskResult.metrics.taskPassed = false; + taskResult.error = taskResult.error ?? msg; + console.error(` [${slug}] Prompt eval error: ${msg}`); + } else { + taskResult.metrics.toolRecall = promptResult.score; + taskResult.metrics.taskPassed = promptResult.score >= 0.5; + console.log(` [${slug}] Prompt score: ${promptResult.score.toFixed(3)} → ${taskResult.metrics.taskPassed ? 'PASS' : 'FAIL'}`); + } } catch (err) { - console.error(` [${slug}] Prompt eval error: ${err instanceof Error ? err.message : err}`); - // Leave vacuous scores in place — task shows as PASS rather than crashing the run + const msg = err instanceof Error ? err.message : String(err); + console.error(` [${slug}] Prompt eval error: ${msg}`); + taskResult.metrics.toolRecall = 0; + taskResult.metrics.taskPassed = false; + taskResult.error = taskResult.error ?? msg; } } diff --git a/src/benchmark/scoring.ts b/src/benchmark/scoring.ts index dbf5fec..4e041fb 100644 --- a/src/benchmark/scoring.ts +++ b/src/benchmark/scoring.ts @@ -50,7 +50,10 @@ export function computeVerdict( ); } - if (report.scopeCoverage?.coverageViolation) { + // Prompt surface: coverage is reported but does not veto the verdict. + // Prompt tasks are not required to map 1:1 to capabilities, so a + // coverage gap here is informational rather than a failure condition. + if (report.scopeCoverage?.coverageViolation && report.config.surface !== 'prompt') { reasons.push('coverage violation: some in-scope actions have zero tasks'); } diff --git a/src/benchmark/types.ts b/src/benchmark/types.ts index ba63cd9..c1c77ab 100644 --- a/src/benchmark/types.ts +++ b/src/benchmark/types.ts @@ -145,6 +145,7 @@ export interface TaskDefinition { expected_actions: ExpectedAction[]; verify?: TaskVerification[]; expected_fetches?: string[]; + capabilityId?: string; } // === Extracted from generated code or tool_calls === diff --git a/src/discovery/prompt.ts b/src/discovery/prompt.ts deleted file mode 100644 index c17106f..0000000 --- a/src/discovery/prompt.ts +++ /dev/null @@ -1,180 +0,0 @@ -/** - * Prompt surface discoverer. - * - * Parses SKILL.md files to extract phases, capabilities, and structural - * information. This enables benchmarking how well models follow prompt - * templates — not just tool calls. - */ - -import { existsSync, readFileSync } from 'node:fs'; -import { resolve } from 'node:path'; - -import type { DiscoveryOptions } from './types.js'; - -// ── Types ───────────────────────────────────────────────────────────────── - -export interface PromptPhase { - name: string; - description: string; - hasCodeBlocks: boolean; - hasNumberedSteps: boolean; - hasDecisionPoints: boolean; -} - -export interface PromptCapability { - name: string; - description: string; - source: 'phase' | 'instruction'; -} - -export interface PromptDiscoverySnapshot { - surface: 'prompt'; - phases: PromptPhase[]; - capabilities: PromptCapability[]; - sources: string[]; -} - -// ── Phase parsing ───────────────────────────────────────────────────────── - -const PHASE_HEADER_RE = /^##\s+(?:Phase\s+\d+\s*[—–-]\s*)?(.+)$/gm; -const CODE_BLOCK_RE = /```[\s\S]*?```/g; -const NUMBERED_STEP_RE = /^\d+\.\s+/m; -const DECISION_POINT_RE = /\b(?:if|when|unless|otherwise|decide|choose|either)\b/i; - -function extractPhases(content: string): PromptPhase[] { - const phases: PromptPhase[] = []; - const headers: { name: string; start: number }[] = []; - - // Find all phase headers (## headings) - let match: RegExpExecArray | null; - const headerRe = new RegExp(PHASE_HEADER_RE.source, PHASE_HEADER_RE.flags); - while ((match = headerRe.exec(content)) !== null) { - headers.push({ name: match[1].trim(), start: match.index }); - } - - for (let i = 0; i < headers.length; i++) { - const start = headers[i].start; - const end = i + 1 < headers.length ? headers[i + 1].start : content.length; - const sectionContent = content.slice(start, end); - - // Extract description: first non-empty, non-heading line - const lines = sectionContent.split('\n').slice(1); // skip the heading itself - const descLine = lines.find((l) => l.trim().length > 0 && !l.startsWith('#')); - const description = descLine?.trim() ?? ''; - - phases.push({ - name: headers[i].name, - description, - hasCodeBlocks: CODE_BLOCK_RE.test(sectionContent), - hasNumberedSteps: NUMBERED_STEP_RE.test(sectionContent), - hasDecisionPoints: DECISION_POINT_RE.test(sectionContent), - }); - - // Reset lastIndex for the stateful regex - CODE_BLOCK_RE.lastIndex = 0; - } - - return phases; -} - -// ── Capability extraction ───────────────────────────────────────────────── - -function extractCapabilities(content: string, phases: PromptPhase[]): PromptCapability[] { - const capabilities: PromptCapability[] = []; - - if (phases.length > 0) { - // Derive capabilities from phases - for (const phase of phases) { - capabilities.push({ - name: phase.name, - description: phase.description, - source: 'phase', - }); - } - } else if (content.trim().length > 0) { - // No phases found — extract capabilities from top-level instructions - // Look for bullet points or imperative sentences - const lines = content.split('\n'); - for (const line of lines) { - const trimmed = line.trim(); - // Match bullet points: "- Do something" or "* Do something" - const bulletMatch = trimmed.match(/^[-*]\s+(.+)$/); - if (bulletMatch) { - capabilities.push({ - name: bulletMatch[1].slice(0, 60), - description: bulletMatch[1], - source: 'instruction', - }); - } - } - - // If still no capabilities, treat the whole content as one instruction block - if (capabilities.length === 0) { - const firstLine = lines.find((l) => l.trim().length > 0 && !l.startsWith('#') && !l.startsWith('---')); - if (firstLine) { - capabilities.push({ - name: firstLine.trim().slice(0, 60), - description: firstLine.trim(), - source: 'instruction', - }); - } - } - } - - return capabilities; -} - -// ── Frontmatter stripping ───────────────────────────────────────────────── - -function stripFrontmatter(content: string): string { - if (!content.startsWith('---')) return content; - // Match the closing --- that appears on its own line (after the opening ---) - const closingMatch = content.slice(3).match(/\n---\s*(\n|$)/); - if (!closingMatch || closingMatch.index === undefined) return content; - const endIdx = 3 + closingMatch.index + closingMatch[0].length; - return content.slice(endIdx); -} - -// ── Public API ──────────────────────────────────────────────────────────── - -export function discoverPromptSurfaceFromContent(content: string): PromptDiscoverySnapshot { - const body = stripFrontmatter(content); - const phases = extractPhases(body); - const capabilities = extractCapabilities(body, phases); - - return { - surface: 'prompt', - phases, - capabilities, - sources: [], - }; -} - -export function discoverPromptSurfaceFromSources( - sources: string[], - options: DiscoveryOptions = {}, -): PromptDiscoverySnapshot { - const baseDir = options.baseDir ?? process.cwd(); - const resolvedSources = sources.map((source) => resolve(baseDir, source)); - - const allPhases: PromptPhase[] = []; - const allCapabilities: PromptCapability[] = []; - - for (const sourcePath of resolvedSources) { - if (!existsSync(sourcePath)) { - throw new Error(`Prompt discovery source file does not exist: ${sourcePath}`); - } - - const content = readFileSync(sourcePath, 'utf-8'); - const snapshot = discoverPromptSurfaceFromContent(content); - allPhases.push(...snapshot.phases); - allCapabilities.push(...snapshot.capabilities); - } - - return { - surface: 'prompt', - phases: allPhases, - capabilities: allCapabilities, - sources: resolvedSources, - }; -} diff --git a/src/project/discover-prompt.ts b/src/project/discover-prompt.ts index ba4a384..7f242af 100644 --- a/src/project/discover-prompt.ts +++ b/src/project/discover-prompt.ts @@ -312,7 +312,7 @@ export function discoverPromptCapabilitiesWithSections( addWithSection({ name: outputName, description: `Expected output format: ${preview}`, - section: snippet, + section: section.body, type: 'output', }); } diff --git a/src/project/fix.ts b/src/project/fix.ts index f143323..28ad3a3 100644 --- a/src/project/fix.ts +++ b/src/project/fix.ts @@ -31,8 +31,9 @@ export function applyFixes( if (issue.code === 'model-id-bad-format' && !prefixFixedIndices.has(idx)) { const currentId = models[idx]!.id as string; - // Never rewrite openrouter/ slugs — they're passed verbatim to OpenRouter's API. - if (!currentId.startsWith('openrouter/')) { + // Only anthropic/ direct-API IDs get dots rewritten to hyphens. + // openrouter/ slugs are passed verbatim; openai/ direct-API IDs use dots (e.g. gpt-5.4). + if (!currentId.startsWith('openrouter/') && !currentId.startsWith('openai/')) { models[idx]!.id = currentId.replace(/(\d+)\.(\d+)/g, '$1-$2'); } } diff --git a/src/tasks/coverage.ts b/src/tasks/coverage.ts index 503b4c9..ac791cd 100644 --- a/src/tasks/coverage.ts +++ b/src/tasks/coverage.ts @@ -3,7 +3,11 @@ import type { GeneratedTask } from './types.js'; import type { CoverageReport } from '../benchmark/types.js'; function actionNamesOf(task: GeneratedTask): string[] { - return task.expected_actions.map((a) => a.name).filter(Boolean); + const fromActions = task.expected_actions.map((a) => a.name).filter(Boolean); + if (fromActions.length > 0) return fromActions; + // Prompt surface: tasks use capabilityId instead of expected_actions. + // action.key === action.name for prompt capabilities, so capabilityId matches. + return task.capabilityId ? [task.capabilityId] : []; } export function computeCoverage( diff --git a/src/tasks/generate.ts b/src/tasks/generate.ts index 6c7b420..25f2586 100644 --- a/src/tasks/generate.ts +++ b/src/tasks/generate.ts @@ -27,7 +27,14 @@ export async function generateCandidateTasks( const prompt = buildPrompt(surface, config); const completion = await deps.complete({ system, prompt }); - const tasks = parseGeneratedTasks(completion); + + // For prompt surface, pass the known capability keys so parseGeneratedTasks + // can attach capabilityId to each task; membership validation is in ground.ts. + const knownCapabilityKeys = surface.snapshot.surface === 'prompt' + ? surface.snapshot.actions.map((a) => a.name) + : undefined; + + const tasks = parseGeneratedTasks(completion, knownCapabilityKeys); return tasks.slice(0, Math.max(1, Math.floor(config.maxTasks))); } @@ -35,18 +42,22 @@ function buildPrompt(surface: DiscoveredTaskSurface, config: TaskGeneratorConfig const clampedMax = Math.max(1, Math.floor(config.maxTasks)); if (surface.snapshot.surface === 'prompt') { + const capKeys = surface.snapshot.actions.map((a) => a.name); return [ 'Generate benchmark evaluation tasks for a prompt/skill document.', 'These tasks will be evaluated by content quality, not action matching.', '', 'Return a JSON object with EXACTLY this shape:', - '{"tasks":[{"id":"string","prompt":"string","expected_actions":[]}]}', + '{"tasks":[{"id":"string","prompt":"string","expected_actions":[],"capabilityId":"string"}]}', '', 'RULES:', - '- Each task has EXACTLY three keys: id, prompt, expected_actions.', + '- Each task has EXACTLY four keys: id, prompt, expected_actions, capabilityId.', '- expected_actions MUST always be an empty array [].', '- id: short snake_case identifier (e.g. "deploy_service_to_staging").', '- prompt: ask the model to perform a realistic task from the skill.', + '- capabilityId: set to the action key of the discovered capability this task exercises.', + `- Valid capabilityId values: ${capKeys.join(', ')}.`, + '- Every task MUST have a capabilityId from the valid list above — no other values are accepted.', `- Produce at most ${clampedMax} tasks. Seed: ${config.seed}.`, '', 'Full SKILL.md:', @@ -102,7 +113,7 @@ function stripCodeFence(raw: string): string { return match ? match[1].trim() : trimmed; } -function parseGeneratedTasks(raw: string): GeneratedTask[] { +function parseGeneratedTasks(raw: string, knownCapabilityKeys?: string[]): GeneratedTask[] { let parsed: unknown; try { parsed = JSON.parse(stripCodeFence(raw)); @@ -119,7 +130,7 @@ function parseGeneratedTasks(raw: string): GeneratedTask[] { throw new Error('Task generator response must contain a top-level "tasks" array'); } - const validated = tasks.map((task, index) => validateTask(task, index)); + const validated = tasks.map((task, index) => validateTask(task, index, knownCapabilityKeys)); // Sort by (id, prompt) before deduplication so the numeric suffix assigned to // colliding IDs is determined by content order, not by the LLM's output order. @@ -164,7 +175,7 @@ function pickLongestStringValue(obj: Record): string | null { return best; } -function validateTask(task: unknown, index: number): GeneratedTask { +function validateTask(task: unknown, index: number, knownCapabilityKeys?: string[]): GeneratedTask { if (!task || typeof task !== 'object') { throw new Error(`Task at index ${index} must be an object`); } @@ -221,10 +232,16 @@ function validateTask(task: unknown, index: number): GeneratedTask { const expected_actions = rawExpectedActions.map((action, actionIndex) => validateExpectedAction(taskId, action, actionIndex)); + // Extract capabilityId for prompt-surface tasks. The field is stored as-is here; + // grounding validates it against the known capability keys and rejects bad values. + const rawCapabilityId = typeof candidate['capabilityId'] === 'string' ? candidate['capabilityId'].trim() : undefined; + const capabilityId = knownCapabilityKeys !== undefined && rawCapabilityId ? rawCapabilityId : undefined; + return { id: taskId, prompt: taskPrompt, expected_actions, + ...(capabilityId !== undefined ? { capabilityId } : {}), }; } diff --git a/src/tasks/ground.ts b/src/tasks/ground.ts index d1e3055..5b97f6d 100644 --- a/src/tasks/ground.ts +++ b/src/tasks/ground.ts @@ -36,10 +36,18 @@ function getRejectionReason( } // Prompt surface tasks must have expected_actions: [] — evaluated on content, not tool calls. + // They must also carry a valid capabilityId referencing a known discovered capability. if (surface === 'prompt') { if (expectedActions.length > 0) { return `prompt task "${task.id}" must have empty expected_actions, got ${expectedActions.length}`; } + const knownKeys = [...actions.keys()]; + if (!task.capabilityId) { + return `prompt task "${task.id}" is missing capabilityId (known: ${knownKeys.join(', ')})`; + } + if (!actions.has(task.capabilityId)) { + return `prompt task "${task.id}" has unknown capabilityId "${task.capabilityId}" (known: ${knownKeys.join(', ')})`; + } return null; } diff --git a/src/tasks/types.ts b/src/tasks/types.ts index 43351b6..cbd51bb 100644 --- a/src/tasks/types.ts +++ b/src/tasks/types.ts @@ -5,6 +5,7 @@ export interface GeneratedTask { id: string; prompt: string; expected_actions: ExpectedAction[]; + capabilityId?: string; // prompt surface only; SDK/CLI/MCP don't set this } export interface TaskGeneratorConfig { diff --git a/tests/smoke-changelog-coverage.ts b/tests/smoke-changelog-coverage.ts new file mode 100644 index 0000000..4041e73 --- /dev/null +++ b/tests/smoke-changelog-coverage.ts @@ -0,0 +1,87 @@ +import { strict as assert } from 'node:assert'; +import { readFileSync, readdirSync } from 'node:fs'; +import { resolve } from 'node:path'; + +/** + * Parses the top version block of CHANGELOG.md and checks that every + * item mentioned in Added/Fixed has at least one test file containing + * a relevant token. + * + * Guards against "shipped feature, forgot the test" — the class that + * let P1/P2/P3 escape in the first place. + */ + +const repoRoot = resolve(process.cwd()); +const changelog = readFileSync(resolve(repoRoot, 'CHANGELOG.md'), 'utf-8'); + +// Grab the first ## block +const blocks = changelog.split(/^##\s+/m).slice(1); +assert.ok(blocks.length > 0, 'CHANGELOG.md must have at least one ## version heading'); +const topBlock = blocks[0]!; + +function extractSection(block: string, name: string): string[] { + // Split block on ### headings and find the named section + const parts = block.split(/^###\s+/m); + for (const part of parts) { + if (part.trimStart().toLowerCase().startsWith(name.toLowerCase())) { + return part + .split('\n') + .slice(1) // skip the section heading line + .map((l) => l.trim()) + .filter((l) => l.startsWith('-')) + .map((l) => l.replace(/^-\s*/, '')); + } + } + return []; +} + +const added = extractSection(topBlock, 'Added'); +const fixed = extractSection(topBlock, 'Fixed'); +const items = [...added, ...fixed]; + +// If no items exist in the top block, skip the check (pre-release state). +if (items.length === 0) { + console.log('SKIP: no Added/Fixed items in top CHANGELOG block'); + process.exit(0); +} + +const STOP = new Set([ + 'this', 'that', 'from', 'with', 'into', 'when', 'then', + 'some', 'have', 'been', 'does', 'must', 'will', 'true', 'false', + 'none', 'more', 'less', 'only', 'each', 'other', 'also', + 'added', 'fixed', 'remove', 'removed', 'change', 'changed', + 'every', 'their', 'where', 'which', 'about', 'bench', 'mark', +]); + +const testFiles = readdirSync(resolve(repoRoot, 'tests')) + .filter((f) => f.startsWith('smoke-') && f.endsWith('.ts')) + .map((f) => readFileSync(resolve(repoRoot, 'tests', f), 'utf-8')); + +let failures = 0; +for (const item of items) { + const tokens = item + .toLowerCase() + .split(/[^a-z0-9]+/) + .filter((t) => t.length >= 4 && !STOP.has(t)) + .slice(0, 8); + if (tokens.length === 0) continue; + // Require at least 2 tokens to co-occur in a single test file (whole-word match). + // Prevents false-passes where a lone generic word like "prompt" or "coverage" + // appears somewhere in the corpus but no test actually covers the claimed behavior. + const minMatch = Math.min(2, tokens.length); + const hit = testFiles.some((content) => { + const matched = tokens.filter((t) => new RegExp(`\\b${t}\\b`, 'i').test(content)); + return matched.length >= minMatch; + }); + if (!hit) { + console.error(`[FAIL] CHANGELOG entry has no test reference: "${item}"`); + console.error(` searched for tokens: ${tokens.join(', ')}`); + failures += 1; + } +} + +assert.strictEqual(failures, 0, + `${failures} CHANGELOG item(s) have no matching test file — ` + + `either add a test or remove the CHANGELOG claim`); + +console.log(`PASS: smoke-changelog-coverage (${items.length} items, all with tests)`); diff --git a/tests/smoke-coverage.ts b/tests/smoke-coverage.ts index 890421c..9df5146 100644 --- a/tests/smoke-coverage.ts +++ b/tests/smoke-coverage.ts @@ -20,6 +20,10 @@ function mkTask(id: string, actions: string[]): GeneratedTask { }; } +function mkPromptTask(id: string, capabilityId: string): GeneratedTask { + return { id, prompt: `do ${id}`, expected_actions: [], capabilityId }; +} + function testFullCoverage() { const actions = [mkAction('Wallet.send'), mkAction('Wallet.receive')]; const tasks = [mkTask('t1', ['Wallet.send']), mkTask('t2', ['Wallet.receive'])]; @@ -54,11 +58,24 @@ function testRetryPromptMentionsActions() { console.log('PASS: retry prompt names uncovered actions'); } +function testPromptCoverageFromCapabilityId() { + // Regression guard for Issue 2: prompt tasks have expected_actions:[] but + // must count as covering their declared capabilityId capability. + const actions = [mkAction('summarize'), mkAction('translate')]; + const tasks = [mkPromptTask('t1', 'summarize'), mkPromptTask('t2', 'translate')]; + const coverage = computeCoverage(actions, tasks); + assert.strictEqual(coverage.coverageViolation, false); + assert.strictEqual(coverage.uncoveredActions.length, 0); + assert.deepStrictEqual(coverage.coveredActions.sort(), ['summarize', 'translate']); + console.log('PASS: prompt tasks covered via capabilityId (Issue 2 guard)'); +} + async function main() { testFullCoverage(); testPartialCoverage(); testUncoveredDriver(); testRetryPromptMentionsActions(); + testPromptCoverageFromCapabilityId(); console.log('\nALL PASS: smoke-coverage'); } diff --git a/tests/smoke-discovery-prompt.ts b/tests/smoke-discovery-prompt.ts deleted file mode 100644 index 2145104..0000000 --- a/tests/smoke-discovery-prompt.ts +++ /dev/null @@ -1,321 +0,0 @@ -/** - * Smoke tests for prompt surface discovery (discovery/prompt.ts). - * Mirrors the structure of smoke-discovery-mcp.ts. - */ - -import { readFileSync } from 'node:fs'; -import { resolve } from 'node:path'; - -import { - discoverPromptSurfaceFromContent, - discoverPromptSurfaceFromSources, -} from '../src/discovery/prompt.js'; -import { discoverPromptCapabilities } from '../src/project/discover-prompt.js'; - -let passed = 0; -let failed = 0; - -async function test(name: string, fn: () => Promise | void) { - try { - await fn(); - passed++; - console.log(` + ${name}`); - } catch (error: any) { - failed++; - console.log(` - ${name}`); - console.log(` ${error.message}`); - } -} - -function assert(condition: boolean, message: string): void { - if (!condition) { - throw new Error(`Assertion failed: ${message}`); - } -} - -function assertEqual(actual: T, expected: T, message: string): void { - if (actual !== expected) { - throw new Error(`${message}: expected ${JSON.stringify(expected)}, got ${JSON.stringify(actual)}`); - } -} - -// --------------------------------------------------------------------------- -// Inline sample SKILL.md for unit-level tests -// --------------------------------------------------------------------------- - -const sampleSkill = `--- -name: test-skill -description: A test skill for benchmarking ---- - -# Test Skill - -## Phase 1 — Requirements Discovery - -Ask clarifying questions until the stopping condition is met. - -## Phase 2 — Implementation - -Write the code changes: - -\`\`\`go -func handleRequest(ctx context.Context, req *Request) (*Response, error) { - // implementation -} -\`\`\` - -## Phase 3 — Testing - -Create table-driven tests with named fixtures. -`; - -// --------------------------------------------------------------------------- -// Tests -// --------------------------------------------------------------------------- - -console.log('\n=== Prompt Discovery Smoke Tests ===\n'); - -await test('discovers 3 phases from sample SKILL.md', () => { - const snapshot = discoverPromptSurfaceFromContent(sampleSkill); - - assertEqual(snapshot.surface, 'prompt', 'surface should be prompt'); - assertEqual(snapshot.phases.length, 3, 'should discover 3 phases'); - - const phaseNames = snapshot.phases.map((p) => p.name); - assert(phaseNames.includes('Requirements Discovery'), 'should include Requirements Discovery phase'); - assert(phaseNames.includes('Implementation'), 'should include Implementation phase'); - assert(phaseNames.includes('Testing'), 'should include Testing phase'); -}); - -await test('each phase has a name and description', () => { - const snapshot = discoverPromptSurfaceFromContent(sampleSkill); - - for (const phase of snapshot.phases) { - assert(phase.name.length > 0, `phase should have a non-empty name, got: "${phase.name}"`); - assert(phase.description.length > 0, `phase "${phase.name}" should have a non-empty description`); - } -}); - -await test('code block detected in Phase 2 (Implementation)', () => { - const snapshot = discoverPromptSurfaceFromContent(sampleSkill); - const implPhase = snapshot.phases.find((p) => p.name === 'Implementation'); - - assert(implPhase !== undefined, 'Implementation phase should exist'); - assert(implPhase!.hasCodeBlocks, 'Implementation phase should have code blocks'); - - // Phase 1 and Phase 3 should NOT have code blocks - const discoveryPhase = snapshot.phases.find((p) => p.name === 'Requirements Discovery'); - assert(discoveryPhase !== undefined, 'Requirements Discovery phase should exist'); - assert(!discoveryPhase!.hasCodeBlocks, 'Requirements Discovery phase should NOT have code blocks'); -}); - -await test('empty skill returns empty capabilities', () => { - const snapshot = discoverPromptSurfaceFromContent(''); - - assertEqual(snapshot.phases.length, 0, 'empty skill should have 0 phases'); - assertEqual(snapshot.capabilities.length, 0, 'empty skill should have 0 capabilities'); -}); - -await test('skill with no phases returns capabilities from instructions', () => { - const noPhaseSkill = `--- -name: simple-skill ---- - -# Simple Skill - -- Validate the input format -- Transform data to JSON -- Send the result to the API -`; - - const snapshot = discoverPromptSurfaceFromContent(noPhaseSkill); - - assertEqual(snapshot.phases.length, 0, 'should have 0 phases'); - assert(snapshot.capabilities.length > 0, 'should have capabilities extracted from bullet points'); - assert( - snapshot.capabilities.every((c) => c.source === 'instruction'), - 'all capabilities should have source "instruction"', - ); -}); - -await test('phases with capabilities have source "phase"', () => { - const snapshot = discoverPromptSurfaceFromContent(sampleSkill); - - assertEqual(snapshot.capabilities.length, 3, 'should have 3 capabilities (one per phase)'); - assert( - snapshot.capabilities.every((c) => c.source === 'phase'), - 'all capabilities should have source "phase" when phases exist', - ); -}); - -await test('discoverPromptSurfaceFromSources reads fixture file', () => { - const fixturePath = resolve(process.cwd(), 'tests/fixtures/sample-skill.md'); - const snapshot = discoverPromptSurfaceFromSources([fixturePath]); - - assertEqual(snapshot.surface, 'prompt', 'surface should be prompt'); - assertEqual(snapshot.phases.length, 3, 'fixture should have 3 phases'); - assertEqual(snapshot.sources.length, 1, 'should track one source file'); - - // Verify structural detection on the fixture - const phase2 = snapshot.phases.find((p) => p.name === 'Manifest Generation'); - assert(phase2 !== undefined, 'fixture should have Manifest Generation phase'); - assert(phase2!.hasCodeBlocks, 'Manifest Generation should have code blocks'); - assert(phase2!.hasNumberedSteps, 'Manifest Generation should have numbered steps'); - - const phase1 = snapshot.phases.find((p) => p.name === 'Requirements Discovery'); - assert(phase1 !== undefined, 'fixture should have Requirements Discovery phase'); - assert(phase1!.hasDecisionPoints, 'Requirements Discovery should have decision points'); - assert(phase1!.hasNumberedSteps, 'Requirements Discovery should have numbered steps'); -}); - -await test('discoverPromptSurfaceFromSources throws on missing file', () => { - let threw = false; - try { - discoverPromptSurfaceFromSources(['/tmp/nonexistent-skill-12345.md']); - } catch (error: any) { - threw = true; - assert( - error.message.includes('does not exist'), - `error should mention missing path, got: ${error.message}`, - ); - } - assert(threw, 'should throw on missing source file'); -}); - -await test('frontmatter is stripped before parsing phases', () => { - const skillWithFrontmatter = `--- -name: fm-test -description: frontmatter test -custom_field: true ---- - -## Phase 1 — Setup - -Initialize the environment. -`; - - const snapshot = discoverPromptSurfaceFromContent(skillWithFrontmatter); - assertEqual(snapshot.phases.length, 1, 'should discover 1 phase after stripping frontmatter'); - assertEqual(snapshot.phases[0].name, 'Setup', 'phase name should be "Setup"'); -}); - -await test('decision points detected from conditional keywords', () => { - const skillWithDecisions = `--- -name: decision-test ---- - -## Phase 1 — Routing - -If the request is a GET, return cached data. -When the cache is stale, fetch fresh data. -Otherwise fall back to the default response. -`; - - const snapshot = discoverPromptSurfaceFromContent(skillWithDecisions); - assertEqual(snapshot.phases.length, 1, 'should discover 1 phase'); - assert(snapshot.phases[0].hasDecisionPoints, 'phase should detect decision points'); -}); - -await test('stripFrontmatter: does not terminate early when YAML value contains ---', () => { - // YAML value "A --- B" contains ---, which indexOf finds first - const content = `---\ntitle: "A --- B"\n---\n\n## Section\nBody text.`; - const result = discoverPromptSurfaceFromContent(content); - assert(result.phases.length > 0, 'should have at least one phase'); - const phaseName = result.phases[0].name.toLowerCase(); - assert(phaseName.includes('section'), - `phase name should be "Section" but was: "${result.phases[0].name}"`); -}); - -// Bug 6: preamble contamination -await test('discoverPromptCapabilities: preamble lines do not contaminate first section body', () => { - const content = [ - '# My Skill', - 'Run this preamble step before anything else.', - '', - '## Phase 1: Do the thing', - 'Write the implementation here.', - ].join('\n'); - const actions = discoverPromptCapabilities(content); - const hasSpuriousInstruction = actions.some(a => - a.description.toLowerCase().includes('preamble'), - ); - // Should be false — preamble should not bleed into instructions - if (hasSpuriousInstruction) throw new Error('preamble content bled into discovered instructions'); -}); - -// Bug 7: frontmatter -await test('discoverPromptCapabilities: strips YAML frontmatter before scanning', () => { - const content = [ - '---', - 'run: my-skill-command', - '---', - '', - '## Phase 1: Setup', - 'Configure the environment.', - ].join('\n'); - const actions = discoverPromptCapabilities(content); - const hasFrontmatterCap = actions.some(a => - a.description.toLowerCase().includes('run: my-skill-command'), - ); - if (hasFrontmatterCap) throw new Error('frontmatter must be stripped before discovery'); - if (actions.length === 0) throw new Error('should still find Phase 1 capability'); -}); - -// Bug 4: heading-less files -await test('discoverPromptCapabilities: handles heading-less instruction-only files', () => { - const content = [ - '# Deployment Skill', - '', - 'This skill deploys services to production.', - '', - '- Ask for the service name and environment', - '- Generate the deployment manifest', - '- Validate the configuration before applying', - '- Run the deployment command', - ].join('\n'); - const actions = discoverPromptCapabilities(content); - assert(actions.length > 0, `should extract at least one capability from bullet list, got ${actions.length}`); -}); - -await test('discoverPromptCapabilities: returns valid ActionDefinition[] shape', () => { - const content = [ - '## Phase 1: Requirements', - 'Ask the user clarifying questions.', - 'Generate a requirements document.', - '', - '## Phase 2: Implementation', - 'Write the implementation code.', - ].join('\n'); - - const actions = discoverPromptCapabilities(content); - assert(Array.isArray(actions), 'should return an array'); - assert(actions.length >= 2, `expected ≥2 capabilities, got ${actions.length}`); - - for (const action of actions) { - assert(typeof action.name === 'string' && action.name.length > 0, 'name must be non-empty'); - assert(typeof action.description === 'string', 'description must be string'); - assert(Array.isArray(action.args), 'args must be array'); - } -}); - -// Regression guard: both discovery modules exist intentionally for different layers. -// This test documents the interface difference so future refactors stay aware. -await test('production path (discover-prompt) vs standalone path (discovery/prompt) have different shapes', () => { - const content = '## Phase 1: Do the thing\nRun this step.'; - - // Production path: ActionDefinition[] — has args field, no source field - const productionResult = discoverPromptCapabilities(content); - assert(productionResult.length > 0, 'production path must return at least one capability'); - assert('args' in productionResult[0], 'production path must have args field'); - - // Standalone discovery path: PromptDiscoverySnapshot — capabilities have source field, not args - const discoveryResult = discoverPromptSurfaceFromContent(content); - if (discoveryResult.capabilities.length > 0) { - const cap = discoveryResult.capabilities[0]; - assert('source' in cap, 'discovery path capability must have source field'); - assert(!('args' in cap), 'discovery path must NOT have args field'); - } -}); - -console.log(`\n${passed} passed, ${failed} failed\n`); -process.exit(failed > 0 ? 1 : 0); diff --git a/tests/smoke-generation.ts b/tests/smoke-generation.ts index 28502e2..f786705 100644 --- a/tests/smoke-generation.ts +++ b/tests/smoke-generation.ts @@ -529,5 +529,167 @@ await test('cli discovery/task generation canonicalizes option keys for extracti } }); +// ── Prompt surface fixture ──────────────────────────────────────────────────── + +function makePromptFixture(): { + root: string; + benchmarkConfigPath: string; + skillPath: string; + // Capability keys produced by the phase headings in the skill file. + capabilityKeys: { summarize: string; translate: string }; +} { + const root = mkdtempSync(join(tmpdir(), 'skill-optimizer-prompt-')); + const skillPath = join(root, 'SKILL.md'); + const benchmarkConfigPath = join(root, 'skill-optimizer.json'); + + // Two ## Phase headings produce exactly two capabilities: + // phase_1_summarize and phase_2_translate + // Body text avoids imperative verbs and decision-point words so no extra + // instruction / decision capabilities are generated alongside the phases. + writeFileSync(skillPath, [ + '# Translation Service Skill', + '', + '## Phase 1: Summarize', + 'Condenses long documents into brief summaries for quick reading.', + '', + '## Phase 2: Translate', + 'Converts text from one language to another while preserving meaning.', + ].join('\n'), 'utf-8'); + + writeFileSync(benchmarkConfigPath, JSON.stringify({ + name: 'prompt-smoke', + target: { + surface: 'prompt', + repoPath: '.', + skill: './SKILL.md', + }, + benchmark: { + tasks: './tasks.json', + format: 'pi', + models: [{ id: 'openai/test', name: 'Test', tier: 'flagship' }], + }, + }, null, 2), 'utf-8'); + + return { + root, + benchmarkConfigPath, + skillPath, + capabilityKeys: { summarize: 'phase_1_summarize', translate: 'phase_2_translate' }, + }; +} + +// ── Prompt surface: capabilityId tagging ───────────────────────────────────── + +await test('prompt surface: generator tags tasks with capabilityId', async () => { + const fixture = makePromptFixture(); + try { + const surface = await discoverTaskSurface(fixture.benchmarkConfigPath); + assertEqual(surface.snapshot.surface, 'prompt', 'surface should be prompt'); + + const { summarize, translate } = fixture.capabilityKeys; + const deps: TaskGeneratorDeps = { + async complete() { + return JSON.stringify({ + tasks: [ + { + id: 'summarize_long_doc', + prompt: 'Summarize this long research paper into three bullet points.', + expected_actions: [], + capabilityId: summarize, + }, + { + id: 'translate_spanish', + prompt: 'Translate the following paragraph from English to Spanish.', + expected_actions: [], + capabilityId: translate, + }, + ], + }); + }, + }; + + const generated = await generateCandidateTasks(surface, { maxTasks: 5, seed: 7 }, deps); + assertEqual(generated.length, 2, 'should produce 2 tasks'); + + const summarizeTask = generated.find((t) => t.capabilityId === summarize); + const translateTask = generated.find((t) => t.capabilityId === translate); + + assert(summarizeTask !== undefined, `task with capabilityId "${summarize}" should exist`); + assert(translateTask !== undefined, `task with capabilityId "${translate}" should exist`); + } finally { + rmSync(fixture.root, { recursive: true, force: true }); + } +}); + +// ── Prompt surface: grounding rejects unknown capabilityId ─────────────────── + +await test('prompt surface: grounding rejects unknown capabilityId', async () => { + const fixture = makePromptFixture(); + try { + const surface = await discoverTaskSurface(fixture.benchmarkConfigPath); + const { summarize } = fixture.capabilityKeys; + + const deps: TaskGeneratorDeps = { + async complete() { + return JSON.stringify({ + tasks: [ + { + id: 'bad_task', + prompt: 'Use an unknown capability.', + expected_actions: [], + capabilityId: 'not-real', + }, + { + id: 'good_task', + prompt: 'Summarize this document into bullet points.', + expected_actions: [], + capabilityId: summarize, + }, + ], + }); + }, + }; + + const generated = await generateCandidateTasks(surface, { maxTasks: 5, seed: 7 }, deps); + + // Only the task with a valid capabilityId passes grounding. + const grounded = groundTasks(generated, surface.snapshot); + assertEqual(grounded.kept.length, 1, 'only the valid capabilityId task should be kept'); + assertEqual(grounded.rejected.length, 1, 'task with unknown capabilityId should be rejected'); + assert( + grounded.rejected[0].reason.includes('unknown capabilityId'), + `rejection reason should mention unknown capabilityId, got: ${grounded.rejected[0].reason}`, + ); + assertEqual(grounded.kept[0].capabilityId, summarize, 'kept task should have the valid capabilityId'); + } finally { + rmSync(fixture.root, { recursive: true, force: true }); + } +}); + +await test('prompt surface: grounding rejects task with missing capabilityId', async () => { + const fixture = makePromptFixture(); + try { + const surface = await discoverTaskSurface(fixture.benchmarkConfigPath); + const deps: TaskGeneratorDeps = { + async complete() { + return JSON.stringify({ + tasks: [ + // capabilityId field is entirely absent from this task + { id: 't1', prompt: 'Do something.', expected_actions: [] }, + ], + }); + }, + }; + const generated = await generateCandidateTasks(surface, { maxTasks: 5, seed: 7 }, deps); + const result = groundTasks(generated, surface.snapshot); + assertEqual(result.kept.length, 0, 'task without capabilityId must be rejected'); + assertEqual(result.rejected.length, 1, 'rejected list must have one entry'); + assert(result.rejected[0]!.reason.includes('capabilityId'), + `rejection reason must mention capabilityId, got: "${result.rejected[0]!.reason}"`); + } finally { + rmSync(fixture.root, { recursive: true, force: true }); + } +}); + console.log(`\n${passed} passed, ${failed} failed\n`); process.exit(failed > 0 ? 1 : 0); diff --git a/tests/smoke-model-ids.ts b/tests/smoke-model-ids.ts index bd0feb7..6ca1ba8 100644 --- a/tests/smoke-model-ids.ts +++ b/tests/smoke-model-ids.ts @@ -60,3 +60,26 @@ await test('openai/ direct-API IDs with dots are NOT rewritten', async () => { assert.equal(result, 'openai/gpt-5.4', 'openai/ direct API dots must be preserved (OpenAI uses gpt-5.4 not gpt-5-4)'); }); + +await test('applyFixes directly: openai/ IDs are NOT rewritten even if model-id-bad-format issue is present', async () => { + const { applyFixes } = await import('../src/project/fix.js'); + const raw = { + name: 'test', + target: { surface: 'mcp' as const, repoPath: '.' }, + benchmark: { + format: 'pi', + models: [{ id: 'openai/gpt-5.4', name: 'GPT-5.4', tier: 'flagship' as const }], + taskGeneration: { enabled: true, maxTasks: 5 }, + }, + }; + // Manufacture the issue that validate.ts would normally never emit for openai/, + // so we exercise fix.ts's defense-in-depth exemption directly. + // fixable: true is required so the filter in applyFixes actually processes it. + const issues = [ + { code: 'model-id-bad-format' as const, field: 'benchmark.models[0].id', message: 'synthetic', severity: 'warning' as const, fixable: true }, + ]; + const fixed = applyFixes(raw as never, issues as never, '/tmp'); + const id = (fixed as { benchmark: { models: Array<{ id: string }> } }).benchmark.models[0]!.id; + assert.equal(id, 'openai/gpt-5.4', + 'fix.ts must exempt openai/ from dot→hyphen rewrite (defense-in-depth; OpenAI API uses dots)'); +}); diff --git a/tests/smoke-prompt-criteria.ts b/tests/smoke-prompt-criteria.ts new file mode 100644 index 0000000..becd39f --- /dev/null +++ b/tests/smoke-prompt-criteria.ts @@ -0,0 +1,104 @@ +import { strict as assert } from 'node:assert'; +import { resolveCriteriaForTask } from '../src/benchmark/prompt-criteria.js'; +import type { GeneratedTask } from '../src/tasks/types.js'; +import type { PromptCapabilityWithSection } from '../src/project/discover-prompt.js'; +import type { ActionDefinition } from '../src/actions/types.js'; + +function cap(actionKey: string, section: string): PromptCapabilityWithSection { + const action: ActionDefinition = { key: actionKey, name: actionKey, args: [], source: 'prompt' }; + return { action, section }; +} + +function task(id: string, capabilityId: string): GeneratedTask { + return { id, prompt: `do ${capabilityId}`, expected_actions: [], capabilityId }; +} + +function testResolvesCriteriaForMatchingCapability() { + const caps = [ + cap('summarize', '## summarize\n\nInclude: date, author. Use a numbered list.'), + cap('translate', '## translate\n\nInclude: source language, target.'), + ]; + const result = resolveCriteriaForTask(task('t1', 'summarize'), caps); + assert.ok(result.criteria, 'criteria must be returned for matched capability'); + assert.strictEqual(result.noActiveCriteria, false); + console.log('PASS: resolves criteria for matching capability'); +} + +function testDistinctCriteriaPerCapability() { + const caps = [ + cap('alpha', '## alpha\n\nInclude: x, y. Numbered list required.'), + cap('beta', '## beta\n\nInclude: totally-different-thing.'), + ]; + const a = resolveCriteriaForTask(task('t1', 'alpha'), caps); + const b = resolveCriteriaForTask(task('t2', 'beta'), caps); + assert.notDeepStrictEqual(a.criteria, b.criteria, + 'different capabilities must produce different criteria (caps[0]-collapse guard)'); + console.log('PASS: distinct criteria per capability'); +} + +function testThrowsOnUnknownCapabilityId() { + const caps = [cap('known', '## known\n\nInclude: foo.')]; + assert.throws( + () => resolveCriteriaForTask(task('t1', 'unknown'), caps), + /capabilityId "unknown"/, + 'unknown capabilityId must throw loudly — no silent fallback', + ); + console.log('PASS: throws on unknown capabilityId'); +} + +function testThrowsOnMissingCapabilityId() { + const caps = [cap('known', '## known\n\nInclude: foo.')]; + const taskWithoutId: GeneratedTask = { id: 't1', prompt: 'test', expected_actions: [] }; + assert.throws( + () => resolveCriteriaForTask(taskWithoutId, caps), + /missing capabilityId/, + 'task without capabilityId must throw loudly', + ); + console.log('PASS: throws on missing capabilityId'); +} + +function testNoActiveCriteriaFlag() { + const caps = [cap('empty', '')]; + const result = resolveCriteriaForTask(task('t1', 'empty'), caps); + assert.strictEqual(result.noActiveCriteria, true, + 'capability with no extractable criteria must set noActiveCriteria: true'); + console.log('PASS: flags noActiveCriteria when criteria are empty'); +} + +function testOutputCapabilityWithCodeBlockSectionProducesCriteria() { + // Regression guard for Issue 1: _output capabilities store section: section.body + // (full markdown with fences), not section: snippet (stripped content without fences). + // generateCriteriaFromCapability requires fences to extract format patterns. + const outputSection = [ + '## Output Format', + '', + 'Respond with this structure:', + '', + '```json', + '{', + ' "name": "",', + ' "count": ', + '}', + '```', + ].join('\n'); + const caps = [cap('my_output', outputSection)]; + const result = resolveCriteriaForTask(task('t1', 'my_output'), caps); + assert.strictEqual(result.noActiveCriteria, false, + 'output capability whose section contains a fenced code block must produce non-empty criteria'); + console.log('PASS: output capability with code block section produces criteria (Issue 1 guard)'); +} + +async function main() { + testResolvesCriteriaForMatchingCapability(); + testDistinctCriteriaPerCapability(); + testThrowsOnUnknownCapabilityId(); + testThrowsOnMissingCapabilityId(); + testNoActiveCriteriaFlag(); + testOutputCapabilityWithCodeBlockSectionProducesCriteria(); + console.log('\nALL PASS: smoke-prompt-criteria'); +} + +main().catch((err) => { + console.error('FAIL: smoke-prompt-criteria', err); + process.exit(1); +}); diff --git a/tests/smoke-prompt-evaluator.ts b/tests/smoke-prompt-evaluator.ts index c8170c2..8024404 100644 --- a/tests/smoke-prompt-evaluator.ts +++ b/tests/smoke-prompt-evaluator.ts @@ -238,6 +238,7 @@ kubectl apply -f manifests/ assertEqual(result.categoryScores.keywords, 1.0, 'all keywords present, no forbidden'); assertEqual(result.categoryScores.structure, 1.0, 'code blocks and numbered list found'); assertEqual(result.score, 1.0, 'overall score should be 1.0 when all criteria met'); + assertEqual(result.noActiveCriteria, false, 'populated criteria must have noActiveCriteria: false'); }); // --------------------------------------------------------------------------- @@ -336,13 +337,20 @@ await test('minLength enforced as format check', () => { }); // --------------------------------------------------------------------------- -// No criteria -> vacuous pass +// No criteria -> noActiveCriteria + score 0 (regression guard for P3) // --------------------------------------------------------------------------- -await test('no criteria specified -> score 1.0 (vacuous pass)', () => { +await test('no criteria specified -> noActiveCriteria: true, score: 0', () => { const criteria: PromptEvaluationCriteria = {}; const result = evaluatePromptResponse('Any response at all.', criteria); - assertEqual(result.score, 1.0, 'no criteria should vacuously pass'); + assertEqual(result.noActiveCriteria, true, 'empty criteria must set noActiveCriteria: true'); + assertEqual(result.score, 0, 'empty criteria must score 0, not vacuously pass'); +}); + +await test('non-empty criteria -> noActiveCriteria: false', () => { + const criteria: PromptEvaluationCriteria = { requiredKeywords: ['alpha'] }; + const result = evaluatePromptResponse('alpha present here.', criteria); + assertEqual(result.noActiveCriteria, false, 'non-empty criteria must not set noActiveCriteria'); }); console.log(`\n${passed} passed, ${failed} failed\n`); diff --git a/tests/smoke-release.ts b/tests/smoke-release.ts index 1f19ae3..df53ad3 100644 --- a/tests/smoke-release.ts +++ b/tests/smoke-release.ts @@ -59,5 +59,14 @@ await test('mock-repos README matches the tracked templates', () => { assert(!readme.includes('mcp-demo'), 'mock-repos README should not mention removed mcp-demo template'); }); +// CHANGELOG must have a heading for current package version +await test('CHANGELOG.md has a section for the current package version', () => { + const pkgVersion = (JSON.parse(readFileSync(join(process.cwd(), 'package.json'), 'utf-8')) as { version: string }).version; + const changelogContent = readFileSync(join(process.cwd(), 'CHANGELOG.md'), 'utf-8'); + const versionHeaderRe = new RegExp(`^##\\s*\\[?${pkgVersion.replace(/\./g, '\\.')}\\]?`, 'm'); + assert(versionHeaderRe.test(changelogContent), `CHANGELOG.md must have a section for version ${pkgVersion}`); + console.log(`PASS: CHANGELOG has section for v${pkgVersion}`); +}); + console.log(`\n${passed} passed, ${failed} failed\n`); process.exit(failed > 0 ? 1 : 0); diff --git a/tests/smoke-verdict-prompt.ts b/tests/smoke-verdict-prompt.ts new file mode 100644 index 0000000..f6b7112 --- /dev/null +++ b/tests/smoke-verdict-prompt.ts @@ -0,0 +1,213 @@ +import { strict as assert } from 'node:assert'; +import { computeVerdict } from '../src/benchmark/scoring.js'; +import type { BenchmarkReport, ModelConfig, BenchmarkSurface } from '../src/benchmark/types.js'; +import { resolveCriteriaForTask } from '../src/benchmark/prompt-criteria.js'; +import { evaluatePromptResponse } from '../src/benchmark/prompt-evaluator.js'; +import type { PromptCapabilityWithSection } from '../src/project/discover-prompt.js'; + +function syntheticReport( + perModel: Record, + surface: BenchmarkSurface, + opts?: { coverageViolation?: boolean; weightedAverage?: number }, +): BenchmarkReport { + const entries = Object.entries(perModel); + const summaryPerModel: Record = {}; + for (const [id, rate] of entries) { + summaryPerModel[id] = { + passRate: rate, avgRecall: 0, avgPrecision: 0, + avgToolSelectionAccuracy: 0, avgArgAccuracy: 0, + avgHallucinationRate: 0, tasksRun: 10, + }; + } + const overall = entries.reduce((a, [, r]) => a + r, 0) / Math.max(1, entries.length); + const wavg = opts?.weightedAverage ?? overall; + return { + timestamp: new Date().toISOString(), + config: { name: 'syn', surface }, + skillVersion: { source: 'local', commitSha: 'local', ref: 'file', fetchedAt: new Date().toISOString() }, + results: [], + coverage: [], + scopeCoverage: opts?.coverageViolation + ? { + coverageViolation: true, + inScopeActions: ['a', 'b'], + outOfScopeActions: [], + coveredActions: ['a'], + uncoveredActions: ['b'], + tasksPerAction: { a: 3, b: 0 }, + } + : undefined, + summary: { + totalTasks: 10, totalModels: entries.length, totalEvaluations: 10 * entries.length, + overallPassRate: overall, weightedAverage: wavg, + avgToolRecall: 0, avgToolPrecision: 0, avgToolSelectionAccuracy: 0, + avgArgAccuracy: 0, avgHallucinationRate: 0, methodCoveragePercent: 1, + perModel: summaryPerModel, perTask: {}, + perTier: { + flagship: { passRate: 0, avgRecall: 0, avgToolSelectionAccuracy: 0, avgArgAccuracy: 0 }, + mid: { passRate: 0, avgRecall: 0, avgToolSelectionAccuracy: 0, avgArgAccuracy: 0 }, + low: { passRate: 0, avgRecall: 0, avgToolSelectionAccuracy: 0, avgArgAccuracy: 0 }, + }, + }, + }; +} + +function testPromptSurfaceIgnoresCoverageViolation() { + const models: ModelConfig[] = [ + { id: 'a', name: 'A', tier: 'flagship' }, + { id: 'b', name: 'B', tier: 'mid' }, + ]; + const report = syntheticReport({ a: 0.9, b: 0.85 }, 'prompt', { + coverageViolation: true, + weightedAverage: 0.875, + }); + const verdict = computeVerdict(report, models, { perModelFloor: 0.6, targetWeightedAverage: 0.7 }); + assert.strictEqual(verdict.result, 'PASS', + 'prompt surface with scores above floor must PASS despite coverageViolation=true'); + assert.ok(!verdict.reasons.some(r => r.includes('coverage')), + 'verdict reasons must not mention coverage for prompt surface'); + console.log('PASS: prompt surface ignores coverage violation'); +} + +function testMcpSurfaceStillBlocksOnCoverageViolation() { + const models: ModelConfig[] = [ + { id: 'a', name: 'A', tier: 'flagship' }, + { id: 'b', name: 'B', tier: 'mid' }, + ]; + const report = syntheticReport({ a: 0.9, b: 0.85 }, 'mcp', { + coverageViolation: true, + weightedAverage: 0.875, + }); + const verdict = computeVerdict(report, models, { perModelFloor: 0.6, targetWeightedAverage: 0.7 }); + assert.strictEqual(verdict.result, 'FAIL', + 'mcp surface must still FAIL on coverage violation (regression guard)'); + assert.ok(verdict.reasons.some(r => r.includes('coverage')), + 'verdict reasons must mention coverage for non-prompt surfaces'); + console.log('PASS: mcp surface still blocks on coverage violation'); +} + +// ── Helpers for scenarios 6-7 ──────────────────────────────────────────────── + +function makeCap(key: string, section: string): PromptCapabilityWithSection { + return { + action: { + key, + name: key, + description: `Capability: ${key}`, + args: [], + }, + section, + }; +} + +// ── Scenario 6: distinct criteria per capability (caps[0]-collapse guard) ──── + +function testDistinctCriteriaPerCapability() { + const caps: PromptCapabilityWithSection[] = [ + makeCap('summarize', '## Summary\nProvide a summary section.\nInclude key points.'), + makeCap('translate', '## Translation\nList the translated output.\nSpecify the target language.'), + makeCap('classify', '## Classification\nShow a numbered list of categories.\nInclude confidence score.'), + ]; + + const tasks = caps.map((cap) => ({ + id: `task_${cap.action.key}`, + prompt: `Perform ${cap.action.key} on the given text.`, + expected_actions: [] as Array<{ name: string; args?: Record }>, + capabilityId: cap.action.key, + })); + + const criteriaList = tasks.map((task) => resolveCriteriaForTask(task, caps).criteria); + + // All 3 criteria must be mutually distinct — none should be equal to another. + for (let i = 0; i < criteriaList.length; i++) { + for (let j = i + 1; j < criteriaList.length; j++) { + const ci = JSON.stringify(criteriaList[i]); + const cj = JSON.stringify(criteriaList[j]); + assert.notStrictEqual( + ci, + cj, + `caps[${i}] and caps[${j}] criteria must be distinct (caps[0]-collapse guard): ` + + `got identical criteria ${ci}`, + ); + } + } + console.log('PASS: distinct criteria per capability (caps[0] collapse guard)'); +} + +// ── Scenario 7: empty criteria → noActiveCriteria via evaluator (P3 guard) ─── + +function testNoActiveCriteriaViaEvaluator() { + // A cap with empty section produces no extractable criteria. + const cap = makeCap('empty_cap', ''); + const task = { + id: 'task_empty', + prompt: 'Do something with empty_cap.', + expected_actions: [] as Array<{ name: string; args?: Record }>, + capabilityId: 'empty_cap', + }; + + const { criteria } = resolveCriteriaForTask(task, [cap]); + const result = evaluatePromptResponse('any response text', criteria); + + assert.strictEqual(result.score, 0, 'empty criteria → score must be 0'); + assert.strictEqual(result.noActiveCriteria, true, 'empty criteria → noActiveCriteria must be true'); + console.log('PASS: empty criteria → noActiveCriteria via evaluator (P3 regression guard)'); +} + +// ── Scenario 8: mock-LLM verdict matrix (threshold off-by-one + weight math) ─ + +function testVerdictMatrix() { + const models2: ModelConfig[] = [ + { id: 'a', name: 'A', tier: 'flagship' }, + { id: 'b', name: 'B', tier: 'mid' }, + ]; + const policy = { perModelFloor: 0.6, targetWeightedAverage: 0.7 }; + + // 8a: both 1.0 → PASS + const r8a = syntheticReport({ a: 1.0, b: 1.0 }, 'prompt', { weightedAverage: 1.0 }); + assert.strictEqual(computeVerdict(r8a, models2, policy).result, 'PASS', 'both 1.0 → PASS'); + + // 8b: floor inclusive (0.60 == floor) + const r8b = syntheticReport({ a: 1.0, b: 0.60 }, 'prompt', { weightedAverage: 0.80 }); + assert.strictEqual(computeVerdict(r8b, models2, policy).result, 'PASS', 'floor inclusive at 0.60'); + + // 8c: below floor (0.59) + const r8c = syntheticReport({ a: 1.0, b: 0.59 }, 'prompt', { weightedAverage: 0.795 }); + assert.strictEqual(computeVerdict(r8c, models2, policy).result, 'FAIL', '0.59 < floor → FAIL'); + + // 8d: weights 2:1 → wavg 0.733 (a=0.80, b=0.60, weights 2:1) + const models2d: ModelConfig[] = [ + { id: 'a', name: 'A', tier: 'flagship', weight: 2 }, + { id: 'b', name: 'B', tier: 'mid', weight: 1 }, + ]; + const r8d = syntheticReport({ a: 0.80, b: 0.60 }, 'prompt', { weightedAverage: (0.80 * 2 + 0.60 * 1) / 3 }); + assert.strictEqual( + computeVerdict(r8d, models2d, { perModelFloor: 0.6, targetWeightedAverage: 0.7 }).result, + 'PASS', + 'weight 2:1 wavg 0.733 > 0.7 → PASS', + ); + + // 8e: wavg below target (a=0.70, b=0.60, weights 1:1 → wavg 0.65) + const r8e = syntheticReport({ a: 0.70, b: 0.60 }, 'prompt', { weightedAverage: 0.65 }); + assert.strictEqual(computeVerdict(r8e, models2, policy).result, 'FAIL', 'wavg 0.65 < target 0.70 → FAIL'); + + console.log('PASS: verdict matrix (threshold off-by-one + weight math guards)'); +} + +async function main() { + testPromptSurfaceIgnoresCoverageViolation(); + testMcpSurfaceStillBlocksOnCoverageViolation(); + testDistinctCriteriaPerCapability(); + testNoActiveCriteriaViaEvaluator(); + testVerdictMatrix(); + console.log('\nALL PASS: smoke-verdict-prompt'); +} + +main().catch((err) => { + console.error('FAIL: smoke-verdict-prompt', err); + process.exit(1); +});