From fdb33a53ab12c9f92389ef2334740f5489651570 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Selser?= Date: Tue, 14 Apr 2026 19:12:12 -0400 Subject: [PATCH 1/7] fix: strip provider prefix from model ID for direct Anthropic/OpenAI API formats (#12) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(benchmark): strip provider prefix from model ID for direct API formats When `benchmark.format` is `anthropic` or `openai` with a direct `baseUrl`, the model ID (e.g. `anthropic/claude-sonnet-4-6`) includes a provider prefix that OpenRouter expects but direct APIs reject. The Anthropic API returns: 404 {"error":{"message":"model: anthropic/claude-sonnet-4-6"}} Root cause: `createLLMClient` in `src/benchmark/llm/index.ts` passes `model.id` verbatim to the API request body. The config validator requires the prefix (`anthropic/...`) but the direct API expects the bare name (`claude-sonnet-4-6`). Fix: added `stripProviderPrefix()` that removes everything up to the first `/` when format is `anthropic` or `openai`. Applied in all three call paths (chat, chatWithTools, chatAgentLoop). The `pi` format is unaffected. Tests: 4 new cases in `smoke-llm.ts`: - anthropic: strips prefix in chat - anthropic: strips prefix in chatWithTools - anthropic: no-op when prefix absent - openai: strips prefix in chat * fix: use prefix-aware stripping instead of indexOf in stripProviderPrefix indexOf('/') only strips the first slash segment, so openrouter/anthropic/ model IDs would become anthropic/claude-... (still wrong for direct Anthropic API). validate.ts accepts openrouter/ prefixes as valid for any format, making this reachable by documented usage. New approach: only strip the direct-provider prefixes (anthropic/, openai/). openrouter/ IDs are left intact — they signal format:'pi' usage, so encountering one in a direct-API context is a misconfiguration that should fail visibly at the API level rather than being silently misrouted. Also adds a test confirming openrouter/-prefixed IDs are passed through unchanged. Co-Authored-By: Claude Sonnet 4.6 * test: fix env var restore and add chatAgentLoop prefix-stripping coverage - Restore OPENAI_API_KEY to its original value (or delete it if absent) in a finally block so CI environments don't lose a pre-existing key - Add chatAgentLoop test to cover the third call path for prefix stripping, matching the chat and chatWithTools cases Co-Authored-By: Claude Sonnet 4.6 * test: add missing models field to openaiConfig test fixture LLMConfig.models is required; the inline config in the OpenAI prefix-stripping test was missing it, causing a structural type error in editors. Co-Authored-By: Claude Sonnet 4.6 --------- Co-authored-by: OpenClaw Agent (basd) Co-authored-by: Claude Sonnet 4.6 --- CHANGELOG.md | 4 + src/benchmark/llm/index.ts | 35 +++++-- tests/smoke-llm.ts | 201 +++++++++++++++++++++++++++++++++++++ 3 files changed, 234 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ffea8c7..7fa08dc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ ## Unreleased +### Fixed + +- **benchmark:** Strip provider prefix from model ID when using direct `anthropic` or `openai` formats. Previously, `anthropic/claude-sonnet-4-6` was sent as-is to the Anthropic API, which expects `claude-sonnet-4-6`. The `pi` format is unaffected. + ## 1.0.0 — 2026-04-14 First public release. diff --git a/src/benchmark/llm/index.ts b/src/benchmark/llm/index.ts index 248d947..3c88ebf 100644 --- a/src/benchmark/llm/index.ts +++ b/src/benchmark/llm/index.ts @@ -15,6 +15,21 @@ export interface LLMClient { ): Promise; } +/** + * Strip the provider prefix from a model ID when talking directly to a provider API. + * + * Only strips "anthropic/" and "openai/" prefixes — the prefixes that belong to + * direct-API configs. "openrouter/" prefixes are intentionally left intact: they + * signal that the ID belongs to format:'pi' (OpenRouter), so encountering one here + * means the config is misconfigured and we want a fast, visible API error rather + * than silently misrouting the request. + */ +function stripProviderPrefix(modelId: string): string { + if (modelId.startsWith('anthropic/')) return modelId.slice('anthropic/'.length); + if (modelId.startsWith('openai/')) return modelId.slice('openai/'.length); + return modelId; +} + /** * Create an LLM client from config. */ @@ -29,33 +44,41 @@ export function createLLMClient(config: LLMConfig): LLMClient { const extraHeaders = config.headers ?? {}; + // When format is 'anthropic' or 'openai', we're talking directly to a provider + // API that doesn't understand prefixed model IDs like "anthropic/claude-sonnet-4-6". + // Strip the prefix so only the bare model name (e.g. "claude-sonnet-4-6") is sent. + const shouldStripPrefix = config.format === 'anthropic' || config.format === 'openai'; + return { async chat(modelId, system, user) { + const resolvedModelId = shouldStripPrefix ? stripProviderPrefix(modelId) : modelId; if (config.format === 'pi') { return chatPi({ timeout, modelId, system, user, apiKeyOverride: apiKey, headers: config.headers }); } if (config.format === 'anthropic') { - return chatAnthropic({ baseUrl: baseUrl!, apiKey, timeout, extraHeaders, modelId, system, user }); + return chatAnthropic({ baseUrl: baseUrl!, apiKey, timeout, extraHeaders, modelId: resolvedModelId, system, user }); } - return chatOpenAI({ baseUrl: baseUrl!, apiKey, timeout, extraHeaders, modelId, system, user }); + return chatOpenAI({ baseUrl: baseUrl!, apiKey, timeout, extraHeaders, modelId: resolvedModelId, system, user }); }, async chatWithTools(modelId, system, user, tools) { + const resolvedModelId = shouldStripPrefix ? stripProviderPrefix(modelId) : modelId; if (config.format === 'pi') { return chatWithToolsPi({ timeout, modelId, system, user, tools, apiKeyOverride: apiKey, headers: config.headers }); } if (config.format === 'anthropic') { - return chatWithToolsAnthropic({ baseUrl: baseUrl!, apiKey, timeout, extraHeaders, modelId, system, user, tools }); + return chatWithToolsAnthropic({ baseUrl: baseUrl!, apiKey, timeout, extraHeaders, modelId: resolvedModelId, system, user, tools }); } - return chatWithToolsOpenAI({ baseUrl: baseUrl!, apiKey, timeout, extraHeaders, modelId, system, user, tools }); + return chatWithToolsOpenAI({ baseUrl: baseUrl!, apiKey, timeout, extraHeaders, modelId: resolvedModelId, system, user, tools }); }, async chatAgentLoop(modelId, system, user, tools, executor, maxTurns = 5) { + const resolvedModelId = shouldStripPrefix ? stripProviderPrefix(modelId) : modelId; if (config.format === 'pi') { return chatAgentLoopPi({ timeout, modelId, system, user, tools, executor, maxTurns, apiKeyOverride: apiKey, headers: config.headers }); } if (config.format === 'anthropic') { - return chatAgentLoopAnthropic({ baseUrl: baseUrl!, apiKey, timeout, extraHeaders, modelId, system, user, tools, executor, maxTurns }); + return chatAgentLoopAnthropic({ baseUrl: baseUrl!, apiKey, timeout, extraHeaders, modelId: resolvedModelId, system, user, tools, executor, maxTurns }); } - return chatAgentLoopOpenAI({ baseUrl: baseUrl!, apiKey, timeout, extraHeaders, modelId, system, user, tools, executor, maxTurns }); + return chatAgentLoopOpenAI({ baseUrl: baseUrl!, apiKey, timeout, extraHeaders, modelId: resolvedModelId, system, user, tools, executor, maxTurns }); }, }; } diff --git a/tests/smoke-llm.ts b/tests/smoke-llm.ts index cd18006..64fc0d7 100644 --- a/tests/smoke-llm.ts +++ b/tests/smoke-llm.ts @@ -591,6 +591,207 @@ await test('anthropic format: converts McpToolDefinition to Anthropic tool forma ); }); +// --------------------------------------------------------------------------- +// Provider prefix stripping for direct API formats +// --------------------------------------------------------------------------- + +await test('anthropic format: strips provider prefix from model ID', async () => { + let capturedBody: any = null; + + globalThis.fetch = mockFetch((_url, init) => { + capturedBody = JSON.parse(init.body as string); + return { + status: 200, + body: { + content: [{ type: 'text', text: 'ok' }], + }, + }; + }) as any; + + const client = createLLMClient(anthropicConfig); + // Pass a prefixed model ID like the config validation requires + await client.chat('anthropic/claude-sonnet-4-6', 'system', 'user'); + + assert(capturedBody !== null, 'fetch should have been called'); + assertEqual( + capturedBody.model, + 'claude-sonnet-4-6', + 'model field should have provider prefix stripped for direct Anthropic API', + ); +}); + +await test('anthropic format: strips provider prefix in chatWithTools', async () => { + let capturedBody: any = null; + + globalThis.fetch = mockFetch((_url, init) => { + capturedBody = JSON.parse(init.body as string); + return { + status: 200, + body: { + content: [{ type: 'text', text: 'no tools needed' }], + stop_reason: 'end_turn', + }, + }; + }) as any; + + const tools: McpToolDefinition[] = [ + { + type: 'function', + function: { + name: 'test_tool', + description: 'A test tool', + parameters: { type: 'object', properties: {} }, + }, + }, + ]; + + const client = createLLMClient(anthropicConfig); + await client.chatWithTools('anthropic/claude-sonnet-4-6', 'system', 'user', tools); + + assert(capturedBody !== null, 'fetch should have been called'); + assertEqual( + capturedBody.model, + 'claude-sonnet-4-6', + 'model field should have provider prefix stripped in chatWithTools', + ); +}); + +await test('anthropic format: strips provider prefix in chatAgentLoop', async () => { + let capturedBody: any = null; + + globalThis.fetch = mockFetch((_url, init) => { + capturedBody = JSON.parse(init.body as string); + return { + status: 200, + body: { + content: [{ type: 'text', text: 'no tools needed' }], + stop_reason: 'end_turn', + }, + }; + }) as any; + + const tools: McpToolDefinition[] = [ + { + type: 'function', + function: { + name: 'test_tool', + description: 'A test tool', + parameters: { type: 'object', properties: {} }, + }, + }, + ]; + + const client = createLLMClient(anthropicConfig); + await client.chatAgentLoop('anthropic/claude-sonnet-4-6', 'system', 'user', tools, async () => 'result'); + + assert(capturedBody !== null, 'fetch should have been called'); + assertEqual( + capturedBody.model, + 'claude-sonnet-4-6', + 'model field should have provider prefix stripped in chatAgentLoop', + ); +}); + +await test('anthropic format: does not strip when no prefix present', async () => { + let capturedBody: any = null; + + globalThis.fetch = mockFetch((_url, init) => { + capturedBody = JSON.parse(init.body as string); + return { + status: 200, + body: { + content: [{ type: 'text', text: 'ok' }], + }, + }; + }) as any; + + const client = createLLMClient(anthropicConfig); + await client.chat('claude-sonnet-4-6', 'system', 'user'); + + assert(capturedBody !== null, 'fetch should have been called'); + assertEqual( + capturedBody.model, + 'claude-sonnet-4-6', + 'model field should remain unchanged when no prefix', + ); +}); + +await test('openai format: strips provider prefix from model ID', async () => { + let capturedBody: any = null; + + const openaiConfig: LLMConfig = { + format: 'openai', + baseUrl: 'https://api.openai.com', + apiKeyEnv: 'OPENAI_API_KEY', + timeout: 5000, + models: [], + }; + + globalThis.fetch = mockFetch((_url, init) => { + capturedBody = JSON.parse(init.body as string); + return { + status: 200, + body: { + choices: [{ message: { role: 'assistant', content: 'ok' } }], + }, + }; + }) as any; + + // Temporarily set the env var so createLLMClient doesn't throw. + // Capture and restore the original value so we don't clobber it in CI. + const hadOpenAiApiKey = Object.prototype.hasOwnProperty.call(process.env, 'OPENAI_API_KEY'); + const previousOpenAiApiKey = process.env.OPENAI_API_KEY; + process.env.OPENAI_API_KEY = 'test-key'; + const client = createLLMClient(openaiConfig); + try { + await client.chat('openai/gpt-4o', 'system', 'user'); + } finally { + if (hadOpenAiApiKey) { + process.env.OPENAI_API_KEY = previousOpenAiApiKey; + } else { + delete process.env.OPENAI_API_KEY; + } + } + + assert(capturedBody !== null, 'fetch should have been called'); + assertEqual( + capturedBody.model, + 'gpt-4o', + 'model field should have provider prefix stripped for direct OpenAI API', + ); +}); + +await test('anthropic format: leaves openrouter/-prefixed model ID unchanged', async () => { + // openrouter/ IDs belong to format:'pi'. If one appears with format:'anthropic', + // the config is misconfigured — we pass it through unchanged so the Anthropic API + // returns a fast, visible error rather than silently misrouting the request. + let capturedBody: any = null; + + globalThis.fetch = mockFetch((_url, init) => { + capturedBody = JSON.parse(init.body as string); + return { + status: 200, + body: { + content: [{ type: 'text', text: 'ok' }], + }, + }; + }) as any; + + const client = createLLMClient(anthropicConfig); + await client.chat('openrouter/anthropic/claude-sonnet-4-6', 'system', 'user'); + + assert(capturedBody !== null, 'fetch should have been called'); + assertEqual( + capturedBody.model, + 'openrouter/anthropic/claude-sonnet-4-6', + 'openrouter/ prefix should be left intact — not silently stripped', + ); +}); + +// NOTE: pi format prefix preservation is already covered by existing +// "pi format: uses provider/model id" tests above. The prefix stripping +// only applies to anthropic and openai direct formats. + // --------------------------------------------------------------------------- // Restore original fetch // --------------------------------------------------------------------------- From 215bb6bdc743bb1177c8d52790d722b65663d617 Mon Sep 17 00:00:00 2001 From: Chris <25006584+seichris@users.noreply.github.com> Date: Wed, 15 Apr 2026 07:56:10 +0800 Subject: [PATCH 2/7] Add current OpenRouter model presets (#10) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add current OpenRouter model presets * fix: align optimize model ID to dot notation used by benchmark presets commonOptimize.model was left as 'openrouter/anthropic/claude-sonnet-4-6' (dashes) while the rest of this PR uses 'openrouter/anthropic/claude-sonnet-4.6' (dots). A mismatched model ID would cause `skill-optimizer optimize` to call a different or invalid OpenRouter ref than the benchmark uses. Co-Authored-By: Claude Sonnet 4.6 * fix: use hyphen notation in model IDs to match codebase convention validate.ts flags digit.digit patterns in model IDs as bad format (warning: "OpenRouter expects hyphens") and fix.ts auto-corrects them. All new model IDs in this PR used dot notation (e.g. claude-sonnet-4.6) which would trigger warnings on every init-generated config. Convert all version segments in model ID strings to hyphens to match the existing convention. Display names (name/label fields) keep dots for readability (e.g. 'Claude Sonnet 4.6'). Also reverts the previous incorrect fix to commonOptimize.model — the dash format was already correct. Co-Authored-By: Claude Sonnet 4.6 * docs(claude): document model ID hyphen convention and display name distinction validate.ts warns on digit.digit patterns in model IDs and fix.ts auto-corrects them. This tripped up PR #10 where new presets used dot notation. Document the rule clearly so it's not re-discovered the hard way. Co-Authored-By: Claude Sonnet 4.6 * test: enforce hyphen convention for model ID version segments Adds a check to the MODEL_PRESETS block in smoke-init.ts that fails if any preset value contains a digit.digit pattern (e.g. claude-sonnet-4.6). validate.ts already flags these at runtime; this makes CI catch them at commit time so the convention is enforced on every PR without manual review. Co-Authored-By: Claude Sonnet 4.6 * ci: rename workflow to 'Build & Test', add development and staging branches Co-Authored-By: Claude Sonnet 4.6 * ci: restore job ID to build-test to preserve required check names Renaming the job broke branch protection rules that require build-test (20) and build-test (22). Keep the job ID as-is; the display name still shows 'Build & Test (node X)' in the GitHub UI. Co-Authored-By: Claude Sonnet 4.6 --------- Co-authored-by: OpenClaw Agent (basd) Co-authored-by: Claude Sonnet 4.6 Co-authored-by: dmn --- .github/workflows/ci.yml | 5 +++-- CLAUDE.md | 17 +++++++++++++++++ README.md | 12 +++++++++++- src/benchmark/init.ts | 5 +++-- src/init/answers.ts | 5 +++-- src/init/scaffold.ts | 30 +++++++++++++++++------------- src/init/wizard.ts | 32 ++++++++++++++++++-------------- tests/smoke-init.ts | 12 +++++++++++- 8 files changed, 83 insertions(+), 35 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3e29c2d..d577d04 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,12 +1,13 @@ -name: CI +name: Build & Test on: pull_request: push: - branches: [main] + branches: [main, development, staging] jobs: build-test: + name: "Build & Test (node ${{ matrix.node }})" runs-on: ubuntu-latest strategy: fail-fast: false diff --git a/CLAUDE.md b/CLAUDE.md index 415b64e..f1d5189 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -86,6 +86,23 @@ npx tsx src/cli.ts optimize --config ./.tmp/mock-repos/mcp-tracker-demo/skill-op - Run `npm test` before finishing when behavior changes may affect extraction, evaluation, reporting, or optimizer flow. - For CLI-only or docs-only changes, at minimum verify `npx tsx src/cli.ts --help` still works if the touched docs reference CLI behavior. +## Model ID Convention + +Model IDs in configs and presets follow a strict format: + +``` +openrouter// +``` + +**Version segments use hyphens, not dots.** Examples: +- `openrouter/anthropic/claude-sonnet-4-6` ✓ (not `claude-sonnet-4.6`) +- `openrouter/google/gemini-2-5-flash` ✓ (not `gemini-2.5-flash`) +- `openrouter/deepseek/deepseek-v3-2` ✓ (not `deepseek-v3.2`) + +`src/project/validate.ts` warns on dot-notation version segments (`model-id-bad-format`) and `src/project/fix.ts` auto-corrects them. When adding new model presets to `src/init/scaffold.ts`, `src/init/wizard.ts`, or `src/benchmark/init.ts`, always use hyphens in the model ID path. + +Display names (`name:` / `label:` fields) are human-readable and should keep dots (e.g. `'Claude Sonnet 4.6'`, `'Gemini 2.5 Flash'`). + ## Environment Notes - Do not commit `.env` or secrets. diff --git a/README.md b/README.md index bd90911..ef6488b 100644 --- a/README.md +++ b/README.md @@ -72,7 +72,17 @@ npx skill-optimizer init --answers answers.json { "surface": "cli", "repoPath": "/absolute/path/to/your-repo", - "models": ["openrouter/anthropic/claude-sonnet-4.6", "openrouter/openai/gpt-4o"], + "models": [ + "openrouter/anthropic/claude-sonnet-4-6", + "openrouter/deepseek/deepseek-v3-2", + "openrouter/google/gemini-2-5-flash", + "openrouter/qwen/qwen3-5-397b-a17b", + "openrouter/moonshotai/kimi-k2-5", + "openrouter/z-ai/glm-5-1", + "openrouter/minimax/minimax-m2-7", + "openrouter/google/gemma-4-31b-it", + "openrouter/meta-llama/llama-4-maverick" + ], "maxTasks": 20, "maxIterations": 5, "entryFile": "src/cli.ts" diff --git a/src/benchmark/init.ts b/src/benchmark/init.ts index 55c5896..5f449df 100644 --- a/src/benchmark/init.ts +++ b/src/benchmark/init.ts @@ -119,8 +119,9 @@ function buildConfig(surface: 'sdk' | 'cli' | 'mcp'): object { outputDir: '.', }, models: [ - { id: 'openrouter/openai/gpt-4o', name: 'GPT-4o', tier: 'flagship' }, - { id: 'openrouter/google/gemini-2.0-flash-001', name: 'Gemini 2.0 Flash', tier: 'mid' }, + { id: 'openrouter/anthropic/claude-sonnet-4-6', name: 'Claude Sonnet 4.6', tier: 'flagship' }, + { id: 'openrouter/deepseek/deepseek-v3-2', name: 'DeepSeek V3.2', tier: 'flagship' }, + { id: 'openrouter/google/gemini-2-5-flash', name: 'Gemini 2.5 Flash', tier: 'mid' }, ], output: { dir: '../benchmark-results', diff --git a/src/init/answers.ts b/src/init/answers.ts index 1a5f6cf..b1efaab 100644 --- a/src/init/answers.ts +++ b/src/init/answers.ts @@ -16,8 +16,9 @@ export interface WizardAnswers { } const DEFAULT_MODELS = [ - 'openrouter/openai/gpt-4o', - 'openrouter/google/gemini-2.0-flash-001', + 'openrouter/anthropic/claude-sonnet-4-6', + 'openrouter/deepseek/deepseek-v3-2', + 'openrouter/google/gemini-2-5-flash', ]; export function buildDefaultAnswers(surface: 'sdk' | 'cli' | 'mcp' = 'sdk', repoPath?: string): WizardAnswers { diff --git a/src/init/scaffold.ts b/src/init/scaffold.ts index 86622d0..641965e 100644 --- a/src/init/scaffold.ts +++ b/src/init/scaffold.ts @@ -5,35 +5,39 @@ import { importCommands } from '../import/index.js'; const KNOWN_MODELS: Record = { // DeepSeek - 'openrouter/deepseek/deepseek-v3.2': { name: 'DeepSeek V3.2', tier: 'flagship' }, + 'openrouter/deepseek/deepseek-v3-2': { name: 'DeepSeek V3.2', tier: 'flagship' }, // Anthropic - 'openrouter/anthropic/claude-opus-4.6': { name: 'Claude Opus 4.6', tier: 'flagship' }, - 'openrouter/anthropic/claude-sonnet-4.6': { name: 'Claude Sonnet 4.6', tier: 'flagship' }, + 'openrouter/anthropic/claude-opus-4-6': { name: 'Claude Opus 4.6', tier: 'flagship' }, + 'openrouter/anthropic/claude-sonnet-4-6': { name: 'Claude Sonnet 4.6', tier: 'flagship' }, // MiniMax - 'openrouter/minimax/minimax-m2.7': { name: 'MiniMax M2.7', tier: 'flagship' }, - 'openrouter/minimax/minimax-m2.5': { name: 'MiniMax M2.5', tier: 'mid' }, + 'openrouter/minimax/minimax-m2-7': { name: 'MiniMax M2.7', tier: 'flagship' }, + 'openrouter/minimax/minimax-m2-5': { name: 'MiniMax M2.5', tier: 'mid' }, // Google 'openrouter/google/gemini-3-flash-preview': { name: 'Gemini 3 Flash Preview', tier: 'mid' }, - 'openrouter/google/gemini-3.1-pro-preview': { name: 'Gemini 3.1 Pro Preview', tier: 'flagship' }, - 'openrouter/google/gemini-2.5-flash': { name: 'Gemini 2.5 Flash', tier: 'mid' }, - 'openrouter/google/gemini-2.5-flash-lite': { name: 'Gemini 2.5 Flash Lite', tier: 'low' }, + 'openrouter/google/gemini-3-1-pro-preview': { name: 'Gemini 3.1 Pro Preview', tier: 'flagship' }, + 'openrouter/google/gemini-2-5-flash': { name: 'Gemini 2.5 Flash', tier: 'mid' }, + 'openrouter/google/gemini-2-5-flash-lite': { name: 'Gemini 2.5 Flash Lite', tier: 'low' }, + 'openrouter/google/gemma-4-31b-it': { name: 'Gemma 4 31B-it', tier: 'mid' }, // Qwen - 'openrouter/qwen/qwen3.6-plus': { name: 'Qwen 3.6 Plus', tier: 'mid' }, + 'openrouter/qwen/qwen3-5-397b-a17b': { name: 'Qwen3.5 397B A17B', tier: 'flagship' }, + 'openrouter/qwen/qwen3-6-plus': { name: 'Qwen 3.6 Plus', tier: 'mid' }, // Xiaomi 'openrouter/xiaomi/mimo-v2-pro': { name: 'MiMo-V2-Pro', tier: 'mid' }, // Nvidia 'openrouter/nvidia/nemotron-3-super-120b-a12b': { name: 'Nemotron 3 Super 120B', tier: 'mid' }, // Moonshot - 'openrouter/moonshotai/kimi-k2.5': { name: 'Kimi K2.5', tier: 'flagship' }, + 'openrouter/moonshotai/kimi-k2-5': { name: 'Kimi K2.5', tier: 'flagship' }, // xAI - 'openrouter/x-ai/grok-4.1-fast': { name: 'Grok 4.1 Fast', tier: 'flagship' }, + 'openrouter/x-ai/grok-4-1-fast': { name: 'Grok 4.1 Fast', tier: 'flagship' }, // OpenAI - 'openrouter/openai/gpt-5.4': { name: 'GPT-5.4', tier: 'flagship' }, + 'openrouter/openai/gpt-5-4': { name: 'GPT-5.4', tier: 'flagship' }, 'openrouter/openai/gpt-4o-mini': { name: 'GPT-4o Mini', tier: 'mid' }, 'openrouter/openai/gpt-oss-120b': { name: 'GPT-OSS 120B', tier: 'mid' }, + // Meta + 'openrouter/meta-llama/llama-4-maverick': { name: 'Llama 4 Maverick', tier: 'mid' }, // Z-AI 'openrouter/z-ai/glm-5': { name: 'GLM 5', tier: 'mid' }, - 'openrouter/z-ai/glm-5.1': { name: 'GLM 5.1', tier: 'mid' }, + 'openrouter/z-ai/glm-5-1': { name: 'GLM 5.1', tier: 'mid' }, 'openrouter/z-ai/glm-5-turbo': { name: 'GLM 5 Turbo', tier: 'low' }, }; diff --git a/src/init/wizard.ts b/src/init/wizard.ts index 2afdea5..ff6816e 100644 --- a/src/init/wizard.ts +++ b/src/init/wizard.ts @@ -7,35 +7,39 @@ import { scaffoldInit } from './scaffold.js'; // Ordered by OpenRouter weekly token volume (top 20). export const MODEL_PRESETS = [ // DeepSeek - { value: 'openrouter/deepseek/deepseek-v3.2', label: 'DeepSeek V3.2 · DeepSeek', hint: '#1' }, + { value: 'openrouter/deepseek/deepseek-v3-2', label: 'DeepSeek V3.2 · DeepSeek', hint: '#1' }, // Anthropic - { value: 'openrouter/anthropic/claude-opus-4.6', label: 'Claude Opus 4.6 · Anthropic', hint: 'flagship' }, - { value: 'openrouter/anthropic/claude-sonnet-4.6', label: 'Claude Sonnet 4.6 · Anthropic' }, + { value: 'openrouter/anthropic/claude-opus-4-6', label: 'Claude Opus 4.6 · Anthropic', hint: 'flagship' }, + { value: 'openrouter/anthropic/claude-sonnet-4-6', label: 'Claude Sonnet 4.6 · Anthropic' }, // MiniMax - { value: 'openrouter/minimax/minimax-m2.7', label: 'MiniMax M2.7 · MiniMax', hint: 'flagship' }, - { value: 'openrouter/minimax/minimax-m2.5', label: 'MiniMax M2.5 · MiniMax' }, + { value: 'openrouter/minimax/minimax-m2-7', label: 'MiniMax M2.7 · MiniMax', hint: 'flagship' }, + { value: 'openrouter/minimax/minimax-m2-5', label: 'MiniMax M2.5 · MiniMax' }, // Google { value: 'openrouter/google/gemini-3-flash-preview', label: 'Gemini 3 Flash Preview · Google', hint: 'fast' }, - { value: 'openrouter/google/gemini-3.1-pro-preview', label: 'Gemini 3.1 Pro Preview · Google', hint: 'flagship' }, - { value: 'openrouter/google/gemini-2.5-flash', label: 'Gemini 2.5 Flash · Google', hint: 'fast' }, - { value: 'openrouter/google/gemini-2.5-flash-lite', label: 'Gemini 2.5 Flash Lite · Google', hint: 'fast' }, + { value: 'openrouter/google/gemini-3-1-pro-preview', label: 'Gemini 3.1 Pro Preview · Google', hint: 'flagship' }, + { value: 'openrouter/google/gemini-2-5-flash', label: 'Gemini 2.5 Flash · Google', hint: 'fast' }, + { value: 'openrouter/google/gemini-2-5-flash-lite', label: 'Gemini 2.5 Flash Lite · Google', hint: 'fast' }, + { value: 'openrouter/google/gemma-4-31b-it', label: 'Gemma 4 31B-it · Google', hint: 'open' }, // Qwen - { value: 'openrouter/qwen/qwen3.6-plus', label: 'Qwen 3.6 Plus · Alibaba', hint: 'open' }, + { value: 'openrouter/qwen/qwen3-5-397b-a17b', label: 'Qwen3.5 397B A17B · Alibaba', hint: 'MoE' }, + { value: 'openrouter/qwen/qwen3-6-plus', label: 'Qwen 3.6 Plus · Alibaba', hint: 'open' }, // Xiaomi { value: 'openrouter/xiaomi/mimo-v2-pro', label: 'MiMo-V2-Pro · Xiaomi' }, // Nvidia { value: 'openrouter/nvidia/nemotron-3-super-120b-a12b', label: 'Nemotron 3 Super 120B · Nvidia', hint: 'open' }, // Moonshot - { value: 'openrouter/moonshotai/kimi-k2.5', label: 'Kimi K2.5 · Moonshot', hint: 'flagship' }, + { value: 'openrouter/moonshotai/kimi-k2-5', label: 'Kimi K2.5 · Moonshot', hint: 'flagship' }, // xAI - { value: 'openrouter/x-ai/grok-4.1-fast', label: 'Grok 4.1 Fast · xAI' }, + { value: 'openrouter/x-ai/grok-4-1-fast', label: 'Grok 4.1 Fast · xAI' }, // OpenAI - { value: 'openrouter/openai/gpt-5.4', label: 'GPT-5.4 · OpenAI', hint: 'flagship' }, + { value: 'openrouter/openai/gpt-5-4', label: 'GPT-5.4 · OpenAI', hint: 'flagship' }, { value: 'openrouter/openai/gpt-4o-mini', label: 'GPT-4o Mini · OpenAI', hint: 'fast' }, { value: 'openrouter/openai/gpt-oss-120b', label: 'GPT-OSS 120B · OpenAI', hint: 'open' }, + // Meta + { value: 'openrouter/meta-llama/llama-4-maverick', label: 'Llama 4 Maverick · Meta', hint: 'open' }, // Z-AI { value: 'openrouter/z-ai/glm-5', label: 'GLM 5 · Z-AI' }, - { value: 'openrouter/z-ai/glm-5.1', label: 'GLM 5.1 · Z-AI', hint: 'new' }, + { value: 'openrouter/z-ai/glm-5-1', label: 'GLM 5.1 · Z-AI', hint: 'new' }, { value: 'openrouter/z-ai/glm-5-turbo', label: 'GLM 5 Turbo · Z-AI', hint: 'fast' }, ]; @@ -111,7 +115,7 @@ export async function runWizard(cwd: string, preseed?: Partial): message: 'Which models to benchmark? (space to toggle, enter to confirm)', options: MODEL_PRESETS, required: true, - initialValues: ['openrouter/anthropic/claude-sonnet-4.6', 'openrouter/deepseek/deepseek-v3.2', 'openrouter/google/gemini-2.5-flash'], + initialValues: ['openrouter/anthropic/claude-sonnet-4-6', 'openrouter/deepseek/deepseek-v3-2', 'openrouter/google/gemini-2-5-flash'], }) as string[]); const models: string[] = selectedPresets; diff --git a/tests/smoke-init.ts b/tests/smoke-init.ts index 65189c9..8faadab 100644 --- a/tests/smoke-init.ts +++ b/tests/smoke-init.ts @@ -176,8 +176,18 @@ assert.strictEqual(typeof _a.surface, 'string'); // MODEL_PRESETS count check { const { MODEL_PRESETS } = await import('../src/init/wizard.js'); - assert.strictEqual(MODEL_PRESETS.length, 20, `Expected 20 presets, got ${MODEL_PRESETS.length}`); + assert.strictEqual(MODEL_PRESETS.length, 23, `Expected 23 presets, got ${MODEL_PRESETS.length}`); assert.ok(MODEL_PRESETS.every(p => p.value.startsWith('openrouter/')), 'All presets should be openrouter/ IDs'); + + // Model ID paths must use hyphens in version segments, not dots. + // validate.ts flags digit.digit as 'model-id-bad-format' and fix.ts auto-corrects them. + // Display names (label/name fields) are exempt — they keep dots for readability. + const dotVersionPattern = /\d+\.\d+/; + const bad = MODEL_PRESETS.filter(p => dotVersionPattern.test(p.value)); + assert.strictEqual( + bad.length, 0, + `Model ID paths must use hyphens not dots in version segments. Fix: ${bad.map(p => p.value).join(', ')}`, + ); } // detectProject: TypeScript SDK (package.json with main, no bin) From dcbff262713f35850aa9ed8805bda7691acf3235 Mon Sep 17 00:00:00 2001 From: Chris <25006584+seichris@users.noreply.github.com> Date: Thu, 16 Apr 2026 01:24:04 +0800 Subject: [PATCH 3/7] Add Codex auth support (#11) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add Codex auth support * Fix Codex auth CI smoke test * Fix codex auth in optimize task generation * Fix Codex MCP schema handling * fix(auth): prioritize tokens.access_token over static OPENAI_API_KEY in readCodexApiKey The browser-login JWT (tokens.access_token) now takes highest priority, followed by tokens.OPENAI_API_KEY, then the top-level OPENAI_API_KEY. Previously, a stale static key could shadow a valid browser-login token, defeating the purpose of Codex auth. The expiry check on access_token is preserved and still applied. Co-Authored-By: Claude Sonnet 4.6 * fix(llm): resolve OpenAI credential per-call to avoid stale Codex token Move resolvedOpenAICredential and useCodexBridgeForOpenAI out of the createLLMClient closure and into each method body, so long-running benchmarks pick up refreshed Codex tokens instead of a stale snapshot. Also pass apiKeyOverride (pre-resolved key) to Pi bridge calls instead of authMode+apiKeyEnv, preventing a second auth.json read on every call. * test(llm): cover chatWithTools+chatAgentLoop codex bridge; narrow resolveDirectApiKey return type Co-Authored-By: Claude Sonnet 4.6 * fix(doctor): probe each model individually instead of bailing when first model is non-openrouter Co-Authored-By: Claude Sonnet 4.6 * fix(tests): restore HOME env var with delete-vs-assign to avoid "undefined" string pollution Co-Authored-By: Claude Sonnet 4.6 * fix(validate): use benchmark.authMode as field for codex auth warning instead of benchmark.apiKeyEnv Co-Authored-By: Claude Sonnet 4.6 * refactor(optimizer): reference PiAuthMode instead of duplicating literal union Co-Authored-By: Claude Sonnet 4.6 * docs: clarify Codex auth paths and format/apiKeyEnv descriptions Co-Authored-By: Claude Sonnet 4.6 * refactor: simplify code clarity in Codex auth changes Drop the redundant useCodexBridgeForOpenAI variable in createLLMClient. Since resolveOpenAICredential() returns undefined unless format is 'openai', checking openAICredential?.source === 'codex' is sufficient on its own. Also regenerate docs/reference/config-schema.md to reflect the schema.ts description updates from the PR. Co-Authored-By: Claude Sonnet 4.6 * fix(review): add format guard comment, accurate doctor skip message, and auto-mode priority test Co-Authored-By: Claude Sonnet 4.6 * fix(auth): skip apiKeyEnv lookup when authMode=codex; avoid writing OPENROUTER_API_KEY in openai-format scaffold Co-Authored-By: Claude Sonnet 4.6 * fix(validate): use integer index instead of model.id in error field paths Replace `for...of` loops over `benchmark.models` with `entries()` variants so error field paths use the numeric index (e.g. `benchmark.models[0].weight`) instead of the model ID string, which produced malformed paths like `benchmark.models[openai/gpt-5.4].weight`. Co-Authored-By: Claude Sonnet 4.6 * fix(validate): check all providers in mixed-model configs and validate optimize auth Fix 2: when benchmark.format is 'pi', models can carry different provider prefixes (openrouter/*, openai/*, anthropic/*). The old code derived benchmarkProvider from models[0] and validated only that one credential. Now all unique provider prefixes are collected and each is checked separately, so a mixed config like ["openai/gpt-4", "openrouter/…"] warns when either key is missing. Fix 3: the optimize section's authMode/apiKeyEnv was never validated for key presence — only codex provider-compatibility was checked. Now the optimize credential is resolved and a warning is pushed if no key is found, parallel to the benchmark check. Both fixes share a warnMissingApiKey helper that formats the hint and field path consistently for both benchmark and optimize sections. Co-Authored-By: Claude Sonnet 4.6 * fix(scaffold): drop redundant benchmark.apiKeyEnv from scaffold defaults benchmark.apiKeyEnv: 'OPENROUTER_API_KEY' was written into every scaffolded config by both scaffold.ts (commonBenchmark) and init.ts (commonBenchmark + commonOptimize). The field is redundant: resolveApiKey already falls back to OPENROUTER_API_KEY for the openrouter provider via defaultApiKeyEnvForProvider. With authMode:'auto' now in the PR, the explicit field becomes a footgun: when a user edits their config to use openai/* models and forgets to remove apiKeyEnv, resolveApiCredential reads OPENROUTER_API_KEY from the env (present and non-empty) and returns it as the OpenAI key — skipping the Codex fallback and sending a wrong key to the OpenAI API. Fix 4 already removed it from scaffold.ts commonOptimize. This commit removes it from the three remaining sites. Co-Authored-By: Claude Sonnet 4.6 * fix(llm): restore authMode:codex in bridge calls to preserve Pi routing signal The Codex bridge was passing apiKeyOverride to chatPi, which caused resolveApiCredential to short-circuit with source:'override'. The routing decision in resolvePiModel checks credential.source === 'codex' to switch to the 'openai-codex' provider (synthesizeOpenAICodexModel, Codex endpoint at chatgpt.com/backend-api/codex). With source:'override' that branch was never taken — the browser JWT was being sent to the standard OpenAI API endpoint instead, causing a 401. Restore the original approach: pass authMode:'codex' so Pi re-reads ~/.codex/auth.json, returns source:'codex', and routes to openai-codex. The second disk read is load-bearing for the provider routing decision. Update the three smoke tests that were asserting the old (incorrect) apiKeyOverride behavior. Co-Authored-By: Claude Sonnet 4.6 * docs: fix dot-notation model IDs in README; extend CLAUDE.md ID convention README used openai/gpt-5.4 (dot notation) in the Codex auth example — two occurrences. Corrected to openai/gpt-5-4 to match the project's model ID convention (hyphens in version segments). CLAUDE.md model ID convention section only documented the openrouter/ format. Extended it to also cover the direct-provider formats (openai/, anthropic/) that this PR introduces, with the same hyphen rule and an openai/gpt-5-4 example. Co-Authored-By: Claude Sonnet 4.6 * fix(validate,models): restore dot-version warning for anthropic/; add OpenAI OPENROUTER_API_KEY guard Finding 1: validate.ts narrowed the model-id-bad-format check to openrouter/* only, which stopped flagging anthropic/claude-sonnet-4.6 — a real error since the Anthropic API expects hyphens. The exemption should only apply to openai/ IDs, where OpenAI's own API uses dots in some model slugs (e.g. gpt-4.5). Change the guard to !startsWith('openai/') and update the message to be provider-agnostic. Finding 2: resolvePiModel had an Anthropic-specific guard that throws when apiKeyEnv is OPENROUTER_API_KEY, preventing silent 401s from stale scaffolded configs. No equivalent existed for the openai provider. Add a parallel guard so direct openai/* calls with an OpenRouter key fail fast with a clear error and a migration hint. Co-Authored-By: Claude Sonnet 4.6 * fix(cli,auth,validate): three preflight and credential correctness gaps cli.ts: run preflight checked only models[0] provider for pi format, so a mixed-provider config (e.g. openrouter + openai models) would pass startup and then silently record per-model failures for whichever provider's key was missing. Now all unique providers are checked, matching what validate.ts already does across the model list. auth.ts: readCodexApiKey returned {} immediately on an expired JWT, making the tokens.OPENAI_API_KEY and top-level OPENAI_API_KEY fallbacks unreachable when access_token was present-but-expired. The comment's intent was correct (don't shadow valid JWT with stale static key) but the inverse was wrong (expired JWT was shadowing valid static key). Now falls through to the static key branches on JWT expiry. validate.ts: optimize auth check ran even when optimize.enabled === false, causing spurious auth warnings for benchmark-only setups. Added the same enabled !== false guard that neighboring optimize checks already use. Co-Authored-By: Claude Sonnet 4.6 --------- Co-authored-by: OpenClaw Agent (basd) Co-authored-by: Claude Sonnet 4.6 --- CLAUDE.md | 11 +- README.md | 21 +- docs/reference/config-schema.md | 6 +- src/actions/discover.ts | 3 + src/actions/snapshot.ts | 27 ++ src/actions/types.ts | 1 + src/benchmark/init.ts | 2 - src/benchmark/llm/index.ts | 172 ++++++++- src/benchmark/llm/openai-format.ts | 47 ++- src/benchmark/llm/pi-format.ts | 73 +++- src/benchmark/llm/tool-name-aliases.ts | 65 ++++ src/benchmark/types.ts | 1 + src/cli.ts | 31 +- src/discovery/mcp.ts | 1 + src/discovery/types.ts | 1 + src/doctor/checks.ts | 36 +- src/init/scaffold.ts | 2 - src/optimizer/main.ts | 1 + src/optimizer/mutation/pi-coding.ts | 1 + src/optimizer/types.ts | 2 + src/project/adapters.ts | 3 + src/project/resolve.ts | 3 + src/project/schema.ts | 6 +- src/project/snapshot.ts | 1 + src/project/types.ts | 4 + src/project/validate.ts | 110 +++++- src/runtime/pi/auth.ts | 154 +++++++- src/runtime/pi/benchmark-agent.ts | 2 + src/runtime/pi/coding-orchestrator.ts | 6 +- src/runtime/pi/index.ts | 4 +- src/runtime/pi/models.ts | 54 ++- src/tasks/default-pi-critic.ts | 2 + src/tasks/default-pi-generator.ts | 2 + src/tasks/freeze.ts | 2 +- tests/smoke-actions.ts | 73 +++- tests/smoke-code.ts | 133 +++++++ tests/smoke-generation.ts | 5 +- tests/smoke-llm.ts | 510 ++++++++++++++++++++++++- 38 files changed, 1487 insertions(+), 91 deletions(-) create mode 100644 src/benchmark/llm/tool-name-aliases.ts diff --git a/CLAUDE.md b/CLAUDE.md index f1d5189..2439da2 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -88,18 +88,21 @@ npx tsx src/cli.ts optimize --config ./.tmp/mock-repos/mcp-tracker-demo/skill-op ## Model ID Convention -Model IDs in configs and presets follow a strict format: +Model IDs use a provider-prefixed format. The prefix determines how the request is routed: ``` -openrouter// +openrouter// — routed through OpenRouter (format: "pi") +anthropic/ — direct Anthropic API (format: "anthropic") +openai/ — direct OpenAI API (format: "openai") ``` -**Version segments use hyphens, not dots.** Examples: +**Version segments use hyphens, not dots**, regardless of provider. Examples: - `openrouter/anthropic/claude-sonnet-4-6` ✓ (not `claude-sonnet-4.6`) - `openrouter/google/gemini-2-5-flash` ✓ (not `gemini-2.5-flash`) - `openrouter/deepseek/deepseek-v3-2` ✓ (not `deepseek-v3.2`) +- `openai/gpt-5-4` ✓ (not `openai/gpt-5.4`) -`src/project/validate.ts` warns on dot-notation version segments (`model-id-bad-format`) and `src/project/fix.ts` auto-corrects them. When adding new model presets to `src/init/scaffold.ts`, `src/init/wizard.ts`, or `src/benchmark/init.ts`, always use hyphens in the model ID path. +`src/project/validate.ts` warns on dot-notation version segments for `openrouter/` model IDs (`model-id-bad-format`) and `src/project/fix.ts` auto-corrects them. When adding new model presets to `src/init/scaffold.ts`, `src/init/wizard.ts`, or `src/benchmark/init.ts`, always use hyphens in the model ID path. Display names (`name:` / `label:` fields) are human-readable and should keep dots (e.g. `'Claude Sonnet 4.6'`, `'Gemini 2.5 Flash'`). diff --git a/README.md b/README.md index ef6488b..ab85d61 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ Benchmark and self-optimize SDK, CLI, and MCP guidance so every agent model can skill-optimizer runs your SDK / CLI / MCP docs against multiple LLMs, measures whether they call the right actions with the right arguments, and iteratively rewrites your `SKILL.md` / docs until a floor score is met across every model. -**Requirements:** Node.js 20+, an [OpenRouter](https://openrouter.ai) API key. +**Requirements:** Node.js 20+, plus either an [OpenRouter](https://openrouter.ai) API key or a local Codex login when using direct OpenAI models. ## Installation @@ -22,6 +22,20 @@ npm link # makes `skill-optimizer` available globally export OPENROUTER_API_KEY=sk-or-... ``` +For direct OpenAI runs you can also use your local Codex browser login instead of exporting `OPENAI_API_KEY`: + +```json +{ + "benchmark": { + "format": "pi", + "authMode": "codex", + "models": [ + { "id": "openai/gpt-5-4", "name": "GPT-5.4", "tier": "flagship" } + ] + } +} +``` + **Step 1 — Scaffold config** (run from your project root): ```bash @@ -97,6 +111,7 @@ npx skill-optimizer init --answers answers.json | `target.discovery.sources` | Source files to scan for callable methods/commands/tools | e.g. `["../src/index.ts"]` or `["../src/server.ts"]` | | `target.skill` | Docs file the optimizer will edit | Path to your `SKILL.md` or equivalent guidance doc | | `benchmark.models` | Models to benchmark | Valid [OpenRouter](https://openrouter.ai/models) model IDs | +| `benchmark.authMode` | How model auth is resolved | `env` (default), `codex`, or `auto` | ## How it works @@ -160,7 +175,7 @@ No per-failure LLM calls — feedback is deterministic (structured failure detai ## Dependencies -The optimizer's coding agent is powered by `@mariozechner/pi-coding-agent` — a small OSS wrapper around OpenRouter that handles agent sessions and tool loops. Models are accessed through [OpenRouter](https://openrouter.ai/) — you need one API key for everything. +The optimizer's coding agent is powered by `@mariozechner/pi-coding-agent`. OpenRouter-backed runs still use your configured API key env var. Direct OpenAI runs can use either `OPENAI_API_KEY` or the browser-login tokens that Codex stores in `~/.codex/auth.json`. ## Troubleshooting @@ -169,6 +184,8 @@ The optimizer's coding agent is powered by `@mariozechner/pi-coding-agent` — a export OPENROUTER_API_KEY=sk-or-... ``` +**Using Codex auth**: Set `benchmark.authMode` (and optionally `optimize.authMode`) to `"codex"` or `"auto"` and use direct OpenAI model refs such as `openai/gpt-5-4`. Codex auth only applies to the `openai` provider and reads either a browser-login access token or `OPENAI_API_KEY` from `~/.codex/auth.json`. Alternatively, set `benchmark.format` to `"openai"` with `authMode: "codex"` and `openai/...` model IDs — the client bridges to the Pi/Codex path automatically. + **Dirty git**: The optimizer requires a clean git state in the target repo (`requireCleanGit: true` by default). Commit or stash uncommitted changes before running. Note: the optimizer never writes to the target repo's skill file — it works from local versioned copies in `.skill-optimizer/`. **`maxTasks < scope_size`**: `benchmark.taskGeneration.maxTasks` must be >= the number of in-scope actions. Run `npx skill-optimizer --dry-run --config ./skill-optimizer.json` to see the count without making LLM calls. diff --git a/docs/reference/config-schema.md b/docs/reference/config-schema.md index e6ea968..a4a60ce 100644 --- a/docs/reference/config-schema.md +++ b/docs/reference/config-schema.md @@ -22,8 +22,9 @@ Paths in the config are relative to the config file location. | `target.mcp.tools` | `string` | — | Path to MCP tools manifest JSON (OpenAI function tool definitions) | | `target.scope.include` | `string[]` | — | Glob patterns for actions to include (default ["*"]) | | `target.scope.exclude` | `string[]` | — | Glob patterns for actions to exclude (default []) | -| `benchmark.format` | `"pi" | "openai" | "anthropic"` | — | LLM transport format: pi, openai, or anthropic | -| `benchmark.apiKeyEnv` | `string` | — | Env var name for the API key (default OPENROUTER_API_KEY) | +| `benchmark.format` | `"pi" | "openai" | "anthropic"` | — | LLM transport format: "pi" routes through OpenRouter/Pi (use openrouter/* or openai/* model refs); "openai" calls the OpenAI API directly (supports Codex auth); "anthropic" calls the Anthropic API directly | +| `benchmark.authMode` | `"env" | "codex" | "auto"` | — | How to resolve credentials: env var, ~/.codex/auth.json browser-login tokens, or env-then-codex fallback | +| `benchmark.apiKeyEnv` | `string` | — | Env var name for the API key (default: OPENROUTER_API_KEY for format:pi, OPENAI_API_KEY for format:openai, ANTHROPIC_API_KEY for format:anthropic) | | `benchmark.timeout` | `integer` | — | Milliseconds per model call (default 240000) | | `benchmark.models` | `object[]` | — | Models to benchmark — at least one required | | `benchmark.taskGeneration.enabled` | `boolean` | — | Whether to generate tasks automatically (default false) | @@ -34,6 +35,7 @@ Paths in the config are relative to the config file location. | `benchmark.verdict.perModelFloor` | `number` | — | Minimum per-model pass fraction for PASS verdict (default 0.6) | | `benchmark.verdict.targetWeightedAverage` | `number` | — | Minimum weighted average across all models for PASS (default 0.7) | | `optimize.model` | `string` | — | Model for mutation, e.g. openrouter/anthropic/claude-sonnet-4-6 | +| `optimize.authMode` | `"env" | "codex" | "auto"` | — | How to resolve optimizer credentials: env var, ~/.codex/auth.json browser-login tokens, or env-then-codex fallback | | `optimize.apiKeyEnv` | `string` | — | Env var for the optimizer API key | | `optimize.thinkingLevel` | `"off" | "minimal" | "low" | "medium" | "high" | "xhigh"` | — | Reasoning depth for mutation calls (default "medium") | | `optimize.allowedPaths` | `string[]` | — | Paths the optimizer may edit — safety boundary | diff --git a/src/actions/discover.ts b/src/actions/discover.ts index 68c672e..07a6629 100644 --- a/src/actions/discover.ts +++ b/src/actions/discover.ts @@ -95,6 +95,9 @@ export function discoverActions(project: ResolvedProjectConfig): ActionCatalog { required: (tool.function.parameters?.required ?? []).includes(name), type: typeof schema === 'object' && schema && 'type' in schema ? String((schema as { type?: unknown }).type ?? '') || undefined : undefined, description: typeof schema === 'object' && schema && 'description' in schema ? String((schema as { description?: unknown }).description ?? '') || undefined : undefined, + schema: typeof schema === 'object' && schema && !Array.isArray(schema) + ? schema as Record + : undefined, })), source: 'mcp.tools', })), diff --git a/src/actions/snapshot.ts b/src/actions/snapshot.ts index f98a4ec..b8009e3 100644 --- a/src/actions/snapshot.ts +++ b/src/actions/snapshot.ts @@ -37,12 +37,16 @@ function validateActionArgs(snapshotPath: string, actionIndex: number, args: unk if (candidate.description !== undefined && typeof candidate.description !== 'string') { invalidSnapshot(snapshotPath, `${path}.description must be a string when provided`); } + if (candidate.schema !== undefined && (!candidate.schema || typeof candidate.schema !== 'object' || Array.isArray(candidate.schema))) { + invalidSnapshot(snapshotPath, `${path}.schema must be an object when provided`); + } return { name: candidate.name, required: candidate.required, type: candidate.type, description: candidate.description, + schema: candidate.schema as Record | undefined, }; }); } @@ -81,6 +85,28 @@ function validateCatalogActions(snapshotPath: string, actions: unknown): ActionD }); } +function normalizeSchemaValue(value: unknown): unknown { + if (Array.isArray(value)) { + return value.map(normalizeSchemaValue); + } + + if (!value || typeof value !== 'object') { + return value; + } + + const normalizedEntries = Object.entries(value as Record) + .sort(([left], [right]) => left.localeCompare(right)) + .map(([key, childValue]) => { + if (key === 'required' && Array.isArray(childValue) && childValue.every((entry) => typeof entry === 'string')) { + return [key, [...childValue].sort()] as const; + } + + return [key, normalizeSchemaValue(childValue)] as const; + }); + + return Object.fromEntries(normalizedEntries); +} + export function normalizeActionArgSchema(args: ActionArgSchema[]): ActionArgSchema[] { return [...args] .map((arg) => ({ @@ -88,6 +114,7 @@ export function normalizeActionArgSchema(args: ActionArgSchema[]): ActionArgSche required: Boolean(arg.required), type: arg.type, description: arg.description, + schema: arg.schema ? normalizeSchemaValue(arg.schema) as Record : undefined, })) .sort((a, b) => a.name.localeCompare(b.name)); } diff --git a/src/actions/types.ts b/src/actions/types.ts index 1a92df9..ed5a2c5 100644 --- a/src/actions/types.ts +++ b/src/actions/types.ts @@ -5,6 +5,7 @@ export interface ActionArgSchema { required: boolean; type?: string; description?: string; + schema?: Record; } export interface ActionDefinition { diff --git a/src/benchmark/init.ts b/src/benchmark/init.ts index 5f449df..4c0b0e9 100644 --- a/src/benchmark/init.ts +++ b/src/benchmark/init.ts @@ -110,7 +110,6 @@ export function initBenchmark(targetDir: string = process.cwd(), surface: 'sdk' function buildConfig(surface: 'sdk' | 'cli' | 'mcp'): object { const commonBenchmark = { - apiKeyEnv: 'OPENROUTER_API_KEY', format: 'pi', timeout: 240000, taskGeneration: { @@ -134,7 +133,6 @@ function buildConfig(surface: 'sdk' | 'cli' | 'mcp'): object { const commonOptimize = { model: 'openrouter/anthropic/claude-sonnet-4-6', - apiKeyEnv: 'OPENROUTER_API_KEY', allowedPaths: ['./SKILL.md'], validation: [], maxIterations: 5, diff --git a/src/benchmark/llm/index.ts b/src/benchmark/llm/index.ts index 3c88ebf..8606f93 100644 --- a/src/benchmark/llm/index.ts +++ b/src/benchmark/llm/index.ts @@ -2,6 +2,7 @@ import type { LLMConfig, LLMResponse, McpToolDefinition, ToolExecutor } from '.. import { chatOpenAI, chatWithToolsOpenAI, chatAgentLoopOpenAI } from './openai-format.js'; import { chatAnthropic, chatWithToolsAnthropic, chatAgentLoopAnthropic } from './anthropic-format.js'; import { chatPi, chatWithToolsPi, chatAgentLoopPi } from './pi-format.js'; +import { requireConfiguredApiKey, resolveApiCredential } from '../../runtime/pi/index.js'; export interface LLMClient { /** Regular chat — LLM returns text output (SDK/CLI surfaces) */ @@ -36,13 +37,27 @@ function stripProviderPrefix(modelId: string): string { export function createLLMClient(config: LLMConfig): LLMClient { const baseUrl = config.baseUrl?.replace(/\/+$/, ''); // strip trailing slash const timeout = config.timeout ?? 240_000; - const apiKey = config.apiKeyEnv ? process.env[config.apiKeyEnv] : undefined; + const extraHeaders = config.headers ?? {}; - if (config.apiKeyEnv && !apiKey) { - throw new Error(`Environment variable ${config.apiKeyEnv} is not set`); + function resolveOpenAICredential() { + if (config.format !== 'openai') return undefined; + return resolveApiCredential({ + provider: 'openai', + authMode: config.authMode, + apiKeyEnv: config.apiKeyEnv, + }); } - const extraHeaders = config.headers ?? {}; + const resolveDirectApiKey = (provider: 'openai' | 'anthropic', openAICredential?: ReturnType): string => + provider === 'openai' && openAICredential?.apiKey + ? openAICredential.apiKey + : requireConfiguredApiKey({ + provider, + authMode: config.authMode, + apiKeyEnv: config.apiKeyEnv, + }); + const toOpenAIProviderModelRef = (modelId: string): string => + modelId.includes('/') ? modelId : `openai/${modelId}`; // When format is 'anthropic' or 'openai', we're talking directly to a provider // API that doesn't understand prefixed model IDs like "anthropic/claude-sonnet-4-6". @@ -51,34 +66,165 @@ export function createLLMClient(config: LLMConfig): LLMClient { return { async chat(modelId, system, user) { + const openAICredential = resolveOpenAICredential(); const resolvedModelId = shouldStripPrefix ? stripProviderPrefix(modelId) : modelId; if (config.format === 'pi') { - return chatPi({ timeout, modelId, system, user, apiKeyOverride: apiKey, headers: config.headers }); + return chatPi({ + timeout, + modelId, + system, + user, + authMode: config.authMode, + apiKeyEnv: config.apiKeyEnv, + headers: config.headers, + }); + } + // openAICredential is only non-undefined when config.format === 'openai' (see resolveOpenAICredential) + if (openAICredential?.source === 'codex') { + // Pass authMode:'codex' so Pi re-reads ~/.codex/auth.json and sets source:'codex', + // which is required for resolvePiModel to route to the openai-codex provider + // (synthesizeOpenAICodexModel guards on provider === 'openai-codex'). Using + // apiKeyOverride here would return source:'override' and break that routing. + return chatPi({ + timeout, + modelId: toOpenAIProviderModelRef(modelId), + system, + user, + authMode: 'codex', + apiKeyEnv: config.apiKeyEnv, + headers: config.headers, + }); } if (config.format === 'anthropic') { - return chatAnthropic({ baseUrl: baseUrl!, apiKey, timeout, extraHeaders, modelId: resolvedModelId, system, user }); + return chatAnthropic({ + baseUrl: baseUrl!, + apiKey: resolveDirectApiKey('anthropic'), + timeout, + extraHeaders, + modelId: resolvedModelId, + system, + user, + }); } - return chatOpenAI({ baseUrl: baseUrl!, apiKey, timeout, extraHeaders, modelId: resolvedModelId, system, user }); + return chatOpenAI({ + baseUrl: baseUrl!, + apiKey: resolveDirectApiKey('openai', openAICredential), + timeout, + extraHeaders, + modelId: resolvedModelId, + system, + user, + }); }, async chatWithTools(modelId, system, user, tools) { + const openAICredential = resolveOpenAICredential(); const resolvedModelId = shouldStripPrefix ? stripProviderPrefix(modelId) : modelId; if (config.format === 'pi') { - return chatWithToolsPi({ timeout, modelId, system, user, tools, apiKeyOverride: apiKey, headers: config.headers }); + return chatWithToolsPi({ + timeout, + modelId, + system, + user, + tools, + authMode: config.authMode, + apiKeyEnv: config.apiKeyEnv, + headers: config.headers, + }); + } + // openAICredential is only non-undefined when config.format === 'openai' (see resolveOpenAICredential) + if (openAICredential?.source === 'codex') { + return chatWithToolsPi({ + timeout, + modelId: toOpenAIProviderModelRef(modelId), + system, + user, + tools, + authMode: 'codex', + apiKeyEnv: config.apiKeyEnv, + headers: config.headers, + }); } if (config.format === 'anthropic') { - return chatWithToolsAnthropic({ baseUrl: baseUrl!, apiKey, timeout, extraHeaders, modelId: resolvedModelId, system, user, tools }); + return chatWithToolsAnthropic({ + baseUrl: baseUrl!, + apiKey: resolveDirectApiKey('anthropic'), + timeout, + extraHeaders, + modelId: resolvedModelId, + system, + user, + tools, + }); } - return chatWithToolsOpenAI({ baseUrl: baseUrl!, apiKey, timeout, extraHeaders, modelId: resolvedModelId, system, user, tools }); + return chatWithToolsOpenAI({ + baseUrl: baseUrl!, + apiKey: resolveDirectApiKey('openai', openAICredential), + timeout, + extraHeaders, + modelId: resolvedModelId, + system, + user, + tools, + }); }, async chatAgentLoop(modelId, system, user, tools, executor, maxTurns = 5) { + const openAICredential = resolveOpenAICredential(); const resolvedModelId = shouldStripPrefix ? stripProviderPrefix(modelId) : modelId; if (config.format === 'pi') { - return chatAgentLoopPi({ timeout, modelId, system, user, tools, executor, maxTurns, apiKeyOverride: apiKey, headers: config.headers }); + return chatAgentLoopPi({ + timeout, + modelId, + system, + user, + tools, + executor, + maxTurns, + authMode: config.authMode, + apiKeyEnv: config.apiKeyEnv, + headers: config.headers, + }); + } + // openAICredential is only non-undefined when config.format === 'openai' (see resolveOpenAICredential) + if (openAICredential?.source === 'codex') { + return chatAgentLoopPi({ + timeout, + modelId: toOpenAIProviderModelRef(modelId), + system, + user, + tools, + executor, + maxTurns, + authMode: 'codex', + apiKeyEnv: config.apiKeyEnv, + headers: config.headers, + }); } if (config.format === 'anthropic') { - return chatAgentLoopAnthropic({ baseUrl: baseUrl!, apiKey, timeout, extraHeaders, modelId: resolvedModelId, system, user, tools, executor, maxTurns }); + return chatAgentLoopAnthropic({ + baseUrl: baseUrl!, + apiKey: resolveDirectApiKey('anthropic'), + timeout, + extraHeaders, + modelId: resolvedModelId, + system, + user, + tools, + executor, + maxTurns, + }); } - return chatAgentLoopOpenAI({ baseUrl: baseUrl!, apiKey, timeout, extraHeaders, modelId: resolvedModelId, system, user, tools, executor, maxTurns }); + return chatAgentLoopOpenAI({ + baseUrl: baseUrl!, + apiKey: resolveDirectApiKey('openai', openAICredential), + timeout, + extraHeaders, + modelId: resolvedModelId, + system, + user, + tools, + executor, + maxTurns, + }); }, }; } diff --git a/src/benchmark/llm/openai-format.ts b/src/benchmark/llm/openai-format.ts index a959a41..8a7c7b9 100644 --- a/src/benchmark/llm/openai-format.ts +++ b/src/benchmark/llm/openai-format.ts @@ -1,4 +1,5 @@ import type { LLMResponse, McpToolDefinition, ToolExecutor } from '../types.js'; +import { createToolNameAliasCodec } from './tool-name-aliases.js'; interface CallParams { baseUrl: string; @@ -35,17 +36,18 @@ export async function chatOpenAI(params: CallParams): Promise { * POST {baseUrl}/chat/completions with tools array */ export async function chatWithToolsOpenAI(params: CallWithToolsParams): Promise { + const toolCodec = createToolNameAliasCodec(params.tools); const body = { model: params.modelId, messages: [ { role: 'system', content: params.system }, { role: 'user', content: params.user }, ], - tools: params.tools, + tools: toolCodec.tools, tool_choice: 'auto', temperature: 0.2, }; - return callWithRetry(params, body); + return callWithRetry(params, body, toolCodec.toCanonical); } async function doFetch(params: CallParams, body: Record): Promise { @@ -132,6 +134,7 @@ interface AgentLoopParams extends CallWithToolsParams { } export async function chatAgentLoopOpenAI(params: AgentLoopParams): Promise { + const toolCodec = createToolNameAliasCodec(params.tools); const messages: Array> = [ { role: 'system', content: params.system }, { role: 'user', content: params.user }, @@ -141,9 +144,9 @@ export async function chatAgentLoopOpenAI(params: AgentLoopParams): Promise = { - model: params.modelId, messages, tools: params.tools, tool_choice: 'auto', temperature: 0.2, + model: params.modelId, messages, tools: toolCodec.tools, tool_choice: 'auto', temperature: 0.2, }; - const response = await callWithRetry(params, body); + const response = await callWithRetry(params, body, toolCodec.toCanonical); if (response.usage) { totalUsage.prompt += response.usage.prompt; totalUsage.completion += response.usage.completion; @@ -155,7 +158,14 @@ export async function chatAgentLoopOpenAI(params: AgentLoopParams): Promise ({ id: `call_${turn}_${i}`, type: 'function', function: { name: tc.name, arguments: JSON.stringify(tc.arguments) } })), + tool_calls: response.toolCalls.map((tc, i) => ({ + id: `call_${turn}_${i}`, + type: 'function', + function: { + name: toolCodec.toProvider(tc.name), + arguments: JSON.stringify(tc.arguments), + }, + })), }); const MAX_TOOL_RESULT_CHARS = 50_000; for (let i = 0; i < response.toolCalls.length; i++) { @@ -175,9 +185,21 @@ export async function chatAgentLoopOpenAI(params: AgentLoopParams): Promise, + toCanonicalToolName: (name: string) => string = (name) => name, ): Promise { try { - return await doFetch(params, body); + const response = await doFetch(params, body); + if (!response.toolCalls || response.toolCalls.length === 0) { + return response; + } + + return { + ...response, + toolCalls: response.toolCalls.map((toolCall) => ({ + ...toolCall, + name: toCanonicalToolName(toolCall.name), + })), + }; } catch (err) { if (err && typeof err === 'object' && 'name' in err && err.name === 'AbortError') { throw err; @@ -191,6 +213,17 @@ async function callWithRetry( } // Retry once after 3s await new Promise((resolve) => setTimeout(resolve, 3_000)); - return doFetch(params, body); + const response = await doFetch(params, body); + if (!response.toolCalls || response.toolCalls.length === 0) { + return response; + } + + return { + ...response, + toolCalls: response.toolCalls.map((toolCall) => ({ + ...toolCall, + name: toCanonicalToolName(toolCall.name), + })), + }; } } diff --git a/src/benchmark/llm/pi-format.ts b/src/benchmark/llm/pi-format.ts index 2132d2d..763d9a8 100644 --- a/src/benchmark/llm/pi-format.ts +++ b/src/benchmark/llm/pi-format.ts @@ -4,9 +4,12 @@ import { complete, completeSimple } from '@mariozechner/pi-ai'; import type { LLMResponse, McpToolDefinition, ToolExecutor } from '../types.js'; import { resolvePiModelByRef } from '../../runtime/pi/index.js'; +import { createToolNameAliasCodec } from './tool-name-aliases.js'; interface PiCallParams { + authMode?: import('../../runtime/pi/auth.js').PiAuthMode; apiKeyOverride?: string; + apiKeyEnv?: string; headers?: Record; timeout: number; modelId: string; @@ -27,7 +30,14 @@ interface ResolvedPiRequest { } type PiImplementationSet = { - resolve(modelId: string, apiKeyOverride?: string): Promise; + resolve( + modelId: string, + authOptions?: { + authMode?: import('../../runtime/pi/auth.js').PiAuthMode; + apiKeyEnv?: string; + apiKeyOverride?: string; + }, + ): Promise; completeSimple(model: Model, context: Context, options?: SimpleStreamOptions): Promise; complete(model: Model, context: Context, options?: SimpleStreamOptions): Promise; }; @@ -40,7 +50,11 @@ export function __setPiImplementationsForTest(implementations: PiImplementationS export async function chatPi(params: PiCallParams): Promise { const impl = getPiImplementations(); - const { model, auth } = await impl.resolve(params.modelId, params.apiKeyOverride); + const { model, auth } = await impl.resolve(params.modelId, { + authMode: params.authMode, + apiKeyEnv: params.apiKeyEnv, + apiKeyOverride: params.apiKeyOverride, + }); const response = await impl.completeSimple( model, { @@ -49,22 +63,29 @@ export async function chatPi(params: PiCallParams): Promise { }, buildPiOptions(params.timeout, auth, params.headers), ); + assertPiResponseSucceeded(response); return toLLMResponse(response); } export async function chatWithToolsPi(params: PiCallWithToolsParams): Promise { const impl = getPiImplementations(); - const { model, auth } = await impl.resolve(params.modelId, params.apiKeyOverride); + const toolCodec = createToolNameAliasCodec(params.tools); + const { model, auth } = await impl.resolve(params.modelId, { + authMode: params.authMode, + apiKeyEnv: params.apiKeyEnv, + apiKeyOverride: params.apiKeyOverride, + }); const response = await impl.complete( model, { systemPrompt: params.system, messages: [{ role: 'user', content: params.user, timestamp: Date.now() }], - tools: params.tools.map(toPiTool), + tools: toolCodec.tools.map(toPiTool), }, buildPiOptions(params.timeout, auth, params.headers), ); - return toLLMResponse(response); + assertPiResponseSucceeded(response); + return toLLMResponse(response, toolCodec.toCanonical); } interface PiAgentLoopParams extends PiCallWithToolsParams { @@ -74,7 +95,12 @@ interface PiAgentLoopParams extends PiCallWithToolsParams { export async function chatAgentLoopPi(params: PiAgentLoopParams): Promise { const impl = getPiImplementations(); - const { model, auth } = await impl.resolve(params.modelId, params.apiKeyOverride); + const toolCodec = createToolNameAliasCodec(params.tools); + const { model, auth } = await impl.resolve(params.modelId, { + authMode: params.authMode, + apiKeyEnv: params.apiKeyEnv, + apiKeyOverride: params.apiKeyOverride, + }); const messages: Context['messages'] = [ { role: 'user', content: params.user, timestamp: Date.now() }, ]; @@ -87,12 +113,13 @@ export async function chatAgentLoopPi(params: PiAgentLoopParams): Promise { - const resolved = await resolvePiModelByRef(modelId, { apiKeyOverride }); +async function resolvePiRequest( + modelId: string, + authOptions?: { + authMode?: import('../../runtime/pi/auth.js').PiAuthMode; + apiKeyEnv?: string; + apiKeyOverride?: string; + }, +): Promise { + const resolved = await resolvePiModelByRef(modelId, authOptions); return { model: resolved.model, auth: { @@ -180,7 +215,16 @@ function toPiTool(tool: McpToolDefinition): PiTool { }; } -function toLLMResponse(message: AssistantMessage): LLMResponse { +function assertPiResponseSucceeded(message: AssistantMessage): void { + if (message.stopReason === 'error') { + throw new Error(message.errorMessage ?? 'PI model returned an unknown error'); + } +} + +function toLLMResponse( + message: AssistantMessage, + toCanonicalToolName: (name: string) => string = (name) => name, +): LLMResponse { const content = message.content .filter((block): block is Extract => block.type === 'text') .map((block) => block.text) @@ -188,7 +232,10 @@ function toLLMResponse(message: AssistantMessage): LLMResponse { const toolCalls = message.content .filter((block): block is Extract => block.type === 'toolCall') - .map((block) => ({ name: block.name, arguments: block.arguments as Record })); + .map((block) => ({ + name: toCanonicalToolName(block.name), + arguments: block.arguments as Record, + })); return { content, diff --git a/src/benchmark/llm/tool-name-aliases.ts b/src/benchmark/llm/tool-name-aliases.ts new file mode 100644 index 0000000..e8cab6e --- /dev/null +++ b/src/benchmark/llm/tool-name-aliases.ts @@ -0,0 +1,65 @@ +import type { McpToolDefinition } from '../types.js'; + +const PROVIDER_TOOL_NAME_PATTERN = /[^a-zA-Z0-9_-]/g; + +export interface ToolNameAliasCodec { + tools: McpToolDefinition[]; + toCanonical(name: string): string; + toProvider(name: string): string; +} + +export function createToolNameAliasCodec( + tools: McpToolDefinition[], +): ToolNameAliasCodec { + const usedProviderNames = new Set(); + const canonicalToProvider = new Map(); + const providerToCanonical = new Map(); + + const aliasedTools = tools.map((tool) => { + const canonicalName = tool.function.name; + let providerName = sanitizeToolName(canonicalName); + + if (providerName.length === 0) { + providerName = 'tool'; + } + + if (usedProviderNames.has(providerName)) { + const baseName = providerName; + let suffix = 1; + while (usedProviderNames.has(`${baseName}__${suffix}`)) { + suffix += 1; + } + providerName = `${baseName}__${suffix}`; + } + + usedProviderNames.add(providerName); + canonicalToProvider.set(canonicalName, providerName); + providerToCanonical.set(providerName, canonicalName); + + if (providerName === canonicalName) { + return tool; + } + + return { + ...tool, + function: { + ...tool.function, + name: providerName, + }, + }; + }); + + return { + tools: aliasedTools, + toCanonical(name: string) { + return providerToCanonical.get(name) ?? name; + }, + toProvider(name: string) { + return canonicalToProvider.get(name) ?? name; + }, + }; +} + +function sanitizeToolName(name: string): string { + return name.replace(PROVIDER_TOOL_NAME_PATTERN, '_'); +} diff --git a/src/benchmark/types.ts b/src/benchmark/types.ts index ad29fff..0246f34 100644 --- a/src/benchmark/types.ts +++ b/src/benchmark/types.ts @@ -87,6 +87,7 @@ export interface SkillConfig { export interface LLMConfig { baseUrl?: string; // required for direct openai/anthropic formats + authMode?: 'env' | 'codex' | 'auto'; apiKeyEnv?: string; // e.g. "OPENROUTER_API_KEY" — reads from process.env format: 'openai' | 'anthropic' | 'pi'; timeout?: number; // ms, default 240000 diff --git a/src/cli.ts b/src/cli.ts index 1019679..319723f 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -27,6 +27,7 @@ import type { WizardAnswers } from './init/answers.js'; import { runWizard } from './init/wizard.js'; import { detectProject, detectedToPreseed, printDetectionSummary } from './init/detect-project.js'; import { ERRORS, SkillOptimizerError, printError } from './errors.js'; +import { requireConfiguredApiKey } from './runtime/pi/index.js'; // ── Error handling ──────────────────────────────────────────────────────────── @@ -393,6 +394,7 @@ async function main(): Promise { const criticDeps = createDefaultPiCritic({ provider: mutation.provider, model: mutation.model, + authMode: mutation.authMode, apiKeyEnv: mutation.apiKeyEnv, }); recs = await generateRecommendations( @@ -426,6 +428,7 @@ async function main(): Promise { const deps = createDefaultPiTaskGenerator({ provider, model, + authMode: project.optimize?.authMode ?? project.benchmark.authMode, apiKeyEnv: project.optimize?.apiKeyEnv ?? project.benchmark.apiKeyEnv, }); const result = await generateTasksForProject({ @@ -482,11 +485,27 @@ async function main(): Promise { `Edit "target.repoPath" in ${project.configPath}.`, ); } - if (project.benchmark.format !== 'anthropic' && !process.env[project.benchmark.apiKeyEnv ?? 'OPENROUTER_API_KEY']) { - throw new Error( - `Missing ${project.benchmark.apiKeyEnv ?? 'OPENROUTER_API_KEY'} environment variable. ` + - `Set it in your shell or in a .env file alongside ${project.configPath}.`, - ); + // Preflight: check credentials for every unique provider in the benchmark config. + // For direct-API formats there is one provider; for pi format each model may need + // a different key (e.g. openrouter + openai in the same run). + const benchmarkProviders = project.benchmark.format === 'openai' + ? ['openai'] + : project.benchmark.format === 'anthropic' + ? ['anthropic'] + : [...new Set(project.benchmark.models.map(m => parseModelRef(m.id).provider))]; + for (const benchmarkProvider of benchmarkProviders) { + try { + requireConfiguredApiKey({ + provider: benchmarkProvider, + authMode: project.benchmark.authMode, + apiKeyEnv: project.benchmark.apiKeyEnv, + }); + } catch (error) { + throw new Error( + `${error instanceof Error ? error.message : String(error)} ` + + `Configure auth in ${project.configPath} before running the benchmark.`, + ); + } } if (project.benchmark.taskGeneration.enabled) { const modelRef = project.optimize?.model ?? project.benchmark.models[0]!.id; @@ -494,6 +513,7 @@ async function main(): Promise { const deps = createDefaultPiTaskGenerator({ provider, model, + authMode: project.optimize?.authMode ?? project.benchmark.authMode, apiKeyEnv: project.optimize?.apiKeyEnv ?? project.benchmark.apiKeyEnv, }); const generation = await generateTasksForProject({ @@ -544,6 +564,7 @@ async function main(): Promise { const criticDeps = createDefaultPiCritic({ provider, model, + authMode: project.optimize?.authMode ?? project.benchmark.authMode, apiKeyEnv: project.optimize?.apiKeyEnv ?? project.benchmark.apiKeyEnv, }); recommendations = await generateRecommendations( diff --git a/src/discovery/mcp.ts b/src/discovery/mcp.ts index 3585214..292f029 100644 --- a/src/discovery/mcp.ts +++ b/src/discovery/mcp.ts @@ -301,6 +301,7 @@ function toDiscoveredAction(toolDefinition: LiteralObject, source: string): Disc required: requiredNames.includes(argName), type: typeof schemaValue.type === 'string' ? schemaValue.type : undefined, description: typeof schemaValue.description === 'string' ? schemaValue.description : undefined, + schema: schemaValue, }); } diff --git a/src/discovery/types.ts b/src/discovery/types.ts index a24f2dc..c91eac7 100644 --- a/src/discovery/types.ts +++ b/src/discovery/types.ts @@ -3,6 +3,7 @@ export interface DiscoveredActionArg { required: boolean; type?: string; description?: string; + schema?: Record; } export interface DiscoveredAction { diff --git a/src/doctor/checks.ts b/src/doctor/checks.ts index 233240b..bae26f5 100644 --- a/src/doctor/checks.ts +++ b/src/doctor/checks.ts @@ -1,6 +1,7 @@ import type { ResolvedProjectConfig } from '../project/types.js'; import type { Issue } from '../project/validate.js'; import { discoverActionsOnly, resolveScope } from '../tasks/index.js'; +import { requireConfiguredApiKey } from '../runtime/pi/index.js'; /** * Tier-2: discover actions and verify scope + maxTasks. @@ -63,21 +64,46 @@ export function checkDiscovery(project: ResolvedProjectConfig): Issue[] { */ export async function checkModelReachability(project: ResolvedProjectConfig): Promise { const issues: Issue[] = []; - const apiKey = process.env[project.benchmark.apiKeyEnv ?? 'OPENROUTER_API_KEY']; - if (!apiKey) return issues; // already reported by checkConfig // Only PI format uses OpenRouter; skip reachability for other formats if (project.benchmark.format && project.benchmark.format !== 'pi') { issues.push({ code: 'reachability-skipped', severity: 'info', field: 'benchmark.format', - message: `Skipping reachability check (--check-models only supports format "pi" for now)`, + message: `Skipping reachability check — model probing is only implemented for openrouter/* model IDs via the OpenRouter API`, fixable: false, }); return issues; } - for (let i = 0; i < project.benchmark.models.length; i++) { - const model = project.benchmark.models[i]!; + const openrouterEntries = project.benchmark.models + .map((model, i) => ({ model, i })) + .filter(({ model }) => model.id.startsWith('openrouter/')); + const skippedCount = project.benchmark.models.length - openrouterEntries.length; + + if (skippedCount > 0) { + issues.push({ + code: 'reachability-skipped', severity: 'info', field: 'benchmark.models', + message: `Skipping reachability for ${skippedCount} non-OpenRouter model(s) (only OpenRouter models can be probed)`, + fixable: false, + }); + } + + if (openrouterEntries.length === 0) { + return issues; + } + + let apiKey: string; + try { + apiKey = requireConfiguredApiKey({ + provider: 'openrouter', + authMode: project.benchmark.authMode, + apiKeyEnv: project.benchmark.apiKeyEnv, + }); + } catch { + return issues; // already reported by checkConfig + } + + for (const { model, i } of openrouterEntries) { try { const res = await fetch('https://openrouter.ai/api/v1/chat/completions', { method: 'POST', diff --git a/src/init/scaffold.ts b/src/init/scaffold.ts index 641965e..67019d0 100644 --- a/src/init/scaffold.ts +++ b/src/init/scaffold.ts @@ -67,7 +67,6 @@ export function buildConfigFromAnswers(answers: WizardAnswers, configDir: string const skillAllowedPath = relative(repoPath, skillAbsPath); const commonBenchmark = { - apiKeyEnv: 'OPENROUTER_API_KEY', format: 'pi', timeout: 240000, taskGeneration: { enabled: true, maxTasks, outputDir: '.' }, @@ -78,7 +77,6 @@ export function buildConfigFromAnswers(answers: WizardAnswers, configDir: string const commonOptimize = { model: 'openrouter/anthropic/claude-sonnet-4-6', - apiKeyEnv: 'OPENROUTER_API_KEY', allowedPaths: [skillAllowedPath], validation: [], maxIterations, diff --git a/src/optimizer/main.ts b/src/optimizer/main.ts index eb28861..8ba6a47 100644 --- a/src/optimizer/main.ts +++ b/src/optimizer/main.ts @@ -101,6 +101,7 @@ export async function runOptimizeFromConfig( const deps = createDefaultPiTaskGenerator({ provider: mutation.provider, model: mutation.model, + authMode: mutation.authMode, apiKeyEnv: mutation.apiKeyEnv, }); const generation = await generateTasksForProject({ diff --git a/src/optimizer/mutation/pi-coding.ts b/src/optimizer/mutation/pi-coding.ts index bece9b3..d067dfd 100644 --- a/src/optimizer/mutation/pi-coding.ts +++ b/src/optimizer/mutation/pi-coding.ts @@ -28,6 +28,7 @@ export class PiCodingMutationExecutor { const { session } = await createCodingOrchestratorSession({ cwd: agentCwd, modelRef: `${mutation.provider}/${mutation.model}`, + authMode: mutation.authMode, apiKeyEnv: mutation.apiKeyEnv, thinkingLevel: mutation.thinkingLevel ?? 'medium', }); diff --git a/src/optimizer/types.ts b/src/optimizer/types.ts index 3aba82e..09222d7 100644 --- a/src/optimizer/types.ts +++ b/src/optimizer/types.ts @@ -1,4 +1,5 @@ import type { BenchmarkReport, BenchmarkSurface, ModelConfig, ModelSummary } from '../benchmark/types.js'; +import type { PiAuthMode } from '../runtime/pi/auth.js'; export type FailureBucketKind = 'missing-tool' | 'bad-args' | 'hallucination' | 'error'; export type StopReason = 'max-iterations' | 'stable'; @@ -35,6 +36,7 @@ export interface OptimizeMutationConfig { provider: string; model: string; thinkingLevel?: 'off' | 'minimal' | 'low' | 'medium' | 'high' | 'xhigh'; + authMode?: PiAuthMode; apiKeyEnv?: string; reportContextMaxBytes?: number; } diff --git a/src/project/adapters.ts b/src/project/adapters.ts index bdc42c9..8151991 100644 --- a/src/project/adapters.ts +++ b/src/project/adapters.ts @@ -21,6 +21,7 @@ export function toBenchmarkConfig(project: ResolvedProjectConfig): BenchmarkConf llm: { format: project.benchmark.format, baseUrl: project.benchmark.baseUrl, + authMode: project.benchmark.authMode, apiKeyEnv: project.benchmark.apiKeyEnv, timeout: project.benchmark.timeout, headers: project.benchmark.headers, @@ -78,6 +79,7 @@ export function toOptimizeManifest(project: ResolvedProjectConfig): ResolvedOpti mutation: { provider: mutationModel.provider, model: mutationModel.model, + authMode: optimize.authMode, apiKeyEnv: optimize.apiKeyEnv, thinkingLevel: optimize.thinkingLevel, reportContextMaxBytes: optimize.reportContextMaxBytes, @@ -115,6 +117,7 @@ export function toLegacyOptimizeManifest(project: ResolvedProjectConfig): Optimi }, mutation: { ...parseModelRef(optimize.model), + authMode: optimize.authMode, apiKeyEnv: optimize.apiKeyEnv, thinkingLevel: optimize.thinkingLevel, reportContextMaxBytes: optimize.reportContextMaxBytes, diff --git a/src/project/resolve.ts b/src/project/resolve.ts index 2f9dc3c..cbc3ecf 100644 --- a/src/project/resolve.ts +++ b/src/project/resolve.ts @@ -3,6 +3,7 @@ import { dirname, resolve } from 'node:path'; import type { ProjectConfig, ResolvedProjectConfig } from './types.js'; const DEFAULT_BENCHMARK_FORMAT = 'pi'; +const DEFAULT_AUTH_MODE = 'env'; const DEFAULT_TIMEOUT = 240_000; const DEFAULT_OUTPUT_DIR = 'benchmark-results'; const DEFAULT_GENERATION_OUTPUT_DIR = '.skill-optimizer'; @@ -97,6 +98,7 @@ export function resolveProjectConfig(config: ProjectConfig, configPath: string): benchmark: { format: config.benchmark.format ?? DEFAULT_BENCHMARK_FORMAT, baseUrl: config.benchmark.baseUrl, + authMode: config.benchmark.authMode ?? DEFAULT_AUTH_MODE, apiKeyEnv: config.benchmark.apiKeyEnv, timeout: config.benchmark.timeout ?? DEFAULT_TIMEOUT, headers: config.benchmark.headers, @@ -123,6 +125,7 @@ export function resolveProjectConfig(config: ProjectConfig, configPath: string): enabled: config.optimize.enabled ?? true, mode: config.optimize.mode ?? 'stable-surface', model: config.optimize.model ?? config.benchmark.models[0]!.id, + authMode: config.optimize.authMode ?? config.benchmark.authMode ?? DEFAULT_AUTH_MODE, apiKeyEnv: config.optimize.apiKeyEnv ?? config.benchmark.apiKeyEnv, thinkingLevel: config.optimize.thinkingLevel ?? 'medium', allowedPaths: [...(config.optimize.allowedPaths ?? [])], diff --git a/src/project/schema.ts b/src/project/schema.ts index 3e923e8..193ce4d 100644 --- a/src/project/schema.ts +++ b/src/project/schema.ts @@ -62,8 +62,9 @@ const VerdictConfigSchema = z.object({ }); const BenchmarkConfigSchema = z.object({ - format: z.enum(['pi', 'openai', 'anthropic']).optional().describe('LLM transport format: pi, openai, or anthropic'), - apiKeyEnv: z.string().optional().describe('Env var name for the API key (default OPENROUTER_API_KEY)'), + format: z.enum(['pi', 'openai', 'anthropic']).optional().describe('LLM transport format: "pi" routes through OpenRouter/Pi (use openrouter/* or openai/* model refs); "openai" calls the OpenAI API directly (supports Codex auth); "anthropic" calls the Anthropic API directly'), + authMode: z.enum(['env', 'codex', 'auto']).optional().describe('How to resolve credentials: env var, ~/.codex/auth.json browser-login tokens, or env-then-codex fallback'), + apiKeyEnv: z.string().optional().describe('Env var name for the API key (default: OPENROUTER_API_KEY for format:pi, OPENAI_API_KEY for format:openai, ANTHROPIC_API_KEY for format:anthropic)'), timeout: z.number().int().positive().optional().describe('Milliseconds per model call (default 240000)'), models: z.array(ModelConfigSchema).describe('Models to benchmark — at least one required'), taskGeneration: TaskGenerationConfigSchema.optional().describe('Automatic task generation config'), @@ -75,6 +76,7 @@ const BenchmarkConfigSchema = z.object({ const OptimizeConfigSchema = z.object({ model: z.string().optional().describe('Model for mutation, e.g. openrouter/anthropic/claude-sonnet-4-6'), + authMode: z.enum(['env', 'codex', 'auto']).optional().describe('How to resolve optimizer credentials: env var, ~/.codex/auth.json browser-login tokens, or env-then-codex fallback'), apiKeyEnv: z.string().optional().describe('Env var for the optimizer API key'), thinkingLevel: z.enum(['off', 'minimal', 'low', 'medium', 'high', 'xhigh']).optional() .describe('Reasoning depth for mutation calls (default "medium")'), diff --git a/src/project/snapshot.ts b/src/project/snapshot.ts index 7dfbc00..76c7a46 100644 --- a/src/project/snapshot.ts +++ b/src/project/snapshot.ts @@ -106,6 +106,7 @@ export function buildMcpToolDefinitionsFromSnapshot(snapshot: SurfaceSnapshot): action.args.map((arg) => [ arg.name, { + ...(arg.schema ?? {}), ...(arg.type ? { type: arg.type } : {}), ...(arg.description ? { description: arg.description } : {}), }, diff --git a/src/project/types.ts b/src/project/types.ts index a81276a..b0cde89 100644 --- a/src/project/types.ts +++ b/src/project/types.ts @@ -52,6 +52,7 @@ export interface ProjectBenchmarkVerdictConfig { export interface ProjectBenchmarkConfig { format?: LLMConfig['format']; baseUrl?: string; + authMode?: LLMConfig['authMode']; apiKeyEnv?: string; timeout?: number; headers?: Record; @@ -68,6 +69,7 @@ export interface ProjectOptimizeConfig { enabled?: boolean; mode?: 'stable-surface' | 'surface-changing'; model?: string; + authMode?: LLMConfig['authMode']; apiKeyEnv?: string; thinkingLevel?: 'off' | 'minimal' | 'low' | 'medium' | 'high' | 'xhigh'; allowedPaths?: string[]; @@ -114,6 +116,7 @@ export interface ResolvedProjectTargetConfig { export interface ResolvedProjectBenchmarkConfig { format: LLMConfig['format']; baseUrl?: string; + authMode: NonNullable; apiKeyEnv?: string; timeout: number; headers?: Record; @@ -130,6 +133,7 @@ export interface ResolvedProjectOptimizeConfig { enabled: boolean; mode: 'stable-surface' | 'surface-changing'; model: string; + authMode: NonNullable; apiKeyEnv?: string; thinkingLevel: 'off' | 'minimal' | 'low' | 'medium' | 'high' | 'xhigh'; allowedPaths: string[]; diff --git a/src/project/validate.ts b/src/project/validate.ts index 4676deb..923b481 100644 --- a/src/project/validate.ts +++ b/src/project/validate.ts @@ -4,7 +4,8 @@ import { execFile } from 'node:child_process'; import { promisify } from 'node:util'; import type { ProjectConfig } from './types.js'; -import { isSdkLanguage } from './types.js'; +import { isSdkLanguage, parseModelRef } from './types.js'; +import { resolveApiKey } from '../runtime/pi/auth.js'; const execFileAsync = promisify(execFile); @@ -134,9 +135,27 @@ export async function checkConfig( } } - for (const model of benchmark.models) { + for (const [i, model] of benchmark.models.entries()) { if (model.weight !== undefined && (!Number.isFinite(model.weight) || model.weight < 0)) { - err('invalid-model-weight', `benchmark.models[${model.id}].weight`, `model "${model.id}" has invalid weight; must be a non-negative number`); + err('invalid-model-weight', `benchmark.models[${i}].weight`, `model "${model.id}" has invalid weight; must be a non-negative number`); + } + } + + if (benchmark.authMode === 'codex') { + for (const [i, model] of benchmark.models.entries()) { + try { + const { provider } = parseModelRef(model.id); + if (provider !== 'openai') { + err( + 'codex-auth-provider-mismatch', + `benchmark.models[${i}].id`, + `benchmark.authMode="codex" only supports openai/* models, but found "${model.id}"`, + 'Use openai/* model IDs with codex auth, or switch benchmark.authMode to "env" / "auto"', + ); + } + } catch { + // model-id format issues are reported separately + } } } } @@ -204,6 +223,27 @@ export async function checkConfig( if (optimize.requireCleanGit === false) { err('require-clean-git-disabled', 'optimize.requireCleanGit', '"optimize.requireCleanGit" must remain true in v1'); } + + const effectiveOptimizeAuthMode = optimize.authMode ?? benchmark.authMode; + if (effectiveOptimizeAuthMode === 'codex') { + const optimizeModelRef = optimize.model + ?? (Array.isArray(benchmark.models) && benchmark.models.length > 0 ? benchmark.models[0]!.id : undefined); + if (typeof optimizeModelRef === 'string') { + try { + const { provider } = parseModelRef(optimizeModelRef); + if (provider !== 'openai') { + err( + 'codex-auth-provider-mismatch', + 'optimize.model', + `optimize.authMode="codex" only supports openai/* models, but found "${optimizeModelRef}"`, + 'Use an openai/* optimize.model with codex auth, or switch optimize.authMode to "env" / "auto"', + ); + } + } catch { + // model-id format issues are reported separately + } + } + } } const configDir = dirname(_configPath); @@ -295,11 +335,14 @@ export async function checkConfig( }); } - if (/\d+\.\d+/.test(model.id)) { + // OpenAI's own API uses dots in some model slugs (e.g. gpt-4.5), so skip the + // dot check for openai/ IDs. All other providers (openrouter/, anthropic/, etc.) + // expect hyphens in version segments. + if (!model.id.startsWith('openai/') && /\d+\.\d+/.test(model.id)) { const corrected = model.id.replace(/(\d+)\.(\d+)/g, '$1-$2'); issues.push({ code: 'model-id-bad-format', severity: 'warning', field: `benchmark.models[${i}].id`, - message: `model ID "${model.id}" uses dots in version segment — OpenRouter expects hyphens`, + message: `model ID "${model.id}" uses dots in version segment — use hyphens instead`, hint: `Change to: ${corrected}`, fixable: true, }); @@ -323,17 +366,62 @@ export async function checkConfig( } } - // Check: API key env var - const apiKeyEnv = benchmark.apiKeyEnv ?? 'OPENROUTER_API_KEY'; - if (!process.env[apiKeyEnv]) { + // Check: API key env var / Codex auth + const authMode = benchmark.authMode ?? 'env'; + // Helper: push a missing-credential warning for a given provider + function warnMissingApiKey(provider: string, effectiveAuthMode: typeof authMode, apiKeyEnv: string | undefined, fieldPrefix: 'benchmark' | 'optimize'): void { + const defaultEnvName = apiKeyEnv + ?? (provider === 'openai' ? 'OPENAI_API_KEY' + : provider === 'anthropic' ? 'ANTHROPIC_API_KEY' + : 'OPENROUTER_API_KEY'); + const hint = effectiveAuthMode === 'codex' + ? `Sign in with Codex so ~/.codex/auth.json contains a browser-login access token or OPENAI_API_KEY, or switch ${fieldPrefix}.authMode to "env"` + : effectiveAuthMode === 'auto' && provider === 'openai' + ? `Run: export ${defaultEnvName}=... or sign in with Codex` + : `Run: export ${defaultEnvName}=...`; issues.push({ - code: 'api-key-not-set', severity: 'warning', field: 'benchmark.apiKeyEnv', - message: `env var "${apiKeyEnv}" is not set`, - hint: `Run: export ${apiKeyEnv}=sk-or-...`, + code: 'api-key-not-set', severity: 'warning', + field: effectiveAuthMode === 'codex' ? `${fieldPrefix}.authMode` : `${fieldPrefix}.apiKeyEnv`, + message: effectiveAuthMode === 'codex' + ? 'Codex auth is enabled but no usable browser-login access token or OPENAI_API_KEY was found in ~/.codex/auth.json' + : `No API key was found for authMode "${effectiveAuthMode}"`, + hint, fixable: false, }); } + if (benchmark.format === 'openai' || benchmark.format === 'anthropic') { + // Single direct-API provider — one credential to check + const benchmarkProvider = benchmark.format === 'openai' ? 'openai' : 'anthropic'; + const apiKey = resolveApiKey({ provider: benchmarkProvider, authMode, apiKeyEnv: benchmark.apiKeyEnv }); + if (!apiKey) warnMissingApiKey(benchmarkProvider, authMode, benchmark.apiKeyEnv, 'benchmark'); + } else { + // Pi format: each model may route through a different provider — check all unique ones + const modelList = Array.isArray(benchmark.models) ? benchmark.models : []; + const providers = Array.from(new Set( + modelList.length > 0 + ? modelList.map(m => String(m.id ?? '').split('/')[0] || 'openrouter') + : ['openrouter'], + )); + for (const provider of providers) { + const apiKey = resolveApiKey({ provider, authMode, apiKeyEnv: benchmark.apiKeyEnv }); + if (!apiKey) warnMissingApiKey(provider, authMode, benchmark.apiKeyEnv, 'benchmark'); + } + } + + // Check: optimize API key env var / Codex auth (skip when optimization is disabled) + if (optimize !== undefined && optimize.enabled !== false) { + const optimizeAuthMode = optimize.authMode ?? benchmark.authMode ?? 'env'; + const optimizeModelRef = optimize.model + ?? (Array.isArray(benchmark.models) && benchmark.models.length > 0 ? benchmark.models[0]!.id : undefined); + const optimizeProvider = typeof optimizeModelRef === 'string' + ? (optimizeModelRef.split('/')[0] || 'openrouter') + : 'openrouter'; + const optimizeApiKeyEnv = optimize.apiKeyEnv ?? benchmark.apiKeyEnv; + const optimizeApiKey = resolveApiKey({ provider: optimizeProvider, authMode: optimizeAuthMode, apiKeyEnv: optimizeApiKeyEnv }); + if (!optimizeApiKey) warnMissingApiKey(optimizeProvider, optimizeAuthMode, optimizeApiKeyEnv, 'optimize'); + } + // Check: dirty git (injection-safe: fixed arg array, no shell) // Skipped when called from within the optimizer benchmark loop — the loop manages // git state itself via ensureReady (run once before the loop starts). diff --git a/src/runtime/pi/auth.ts b/src/runtime/pi/auth.ts index fe4a81c..d8b1256 100644 --- a/src/runtime/pi/auth.ts +++ b/src/runtime/pi/auth.ts @@ -1,24 +1,106 @@ +import { readFileSync } from 'node:fs'; +import { homedir } from 'node:os'; +import { resolve } from 'node:path'; + import { AuthStorage } from '@mariozechner/pi-coding-agent'; +export type PiAuthMode = 'env' | 'codex' | 'auto'; + export interface PiAuthOptions { provider: string; + authMode?: PiAuthMode; apiKeyEnv?: string; apiKeyOverride?: string; } +export interface ResolvedApiCredential { + apiKey?: string; + source?: 'override' | 'env' | 'codex'; +} + +const CODEX_ACCESS_TOKEN_REFRESH_SKEW_MS = 120_000; + export function createPiAuthStorage(options?: PiAuthOptions): ReturnType { const authStorage = AuthStorage.create(); if (!options) { return authStorage; } - const apiKey = options.apiKeyOverride ?? (options.apiKeyEnv ? process.env[options.apiKeyEnv] : undefined); + const apiKey = resolveApiKey({ + provider: options.provider, + authMode: options.authMode, + apiKeyEnv: options.apiKeyEnv, + apiKeyOverride: options.apiKeyOverride, + }); if (apiKey) { authStorage.setRuntimeApiKey(options.provider as never, apiKey); } return authStorage; } +export function resolveApiKey(options: PiAuthOptions): string | undefined { + return resolveApiCredential(options).apiKey; +} + +export function resolveApiCredential(options: PiAuthOptions): ResolvedApiCredential { + if (options.apiKeyOverride) { + return { apiKey: options.apiKeyOverride, source: 'override' }; + } + + const authMode = options.authMode ?? 'env'; + const envName = options.apiKeyEnv ?? defaultApiKeyEnvForProvider(options.provider); + const envApiKey = envName ? process.env[envName] : undefined; + + if (authMode === 'env') { + return envApiKey ? { apiKey: envApiKey, source: 'env' } : {}; + } + + if (authMode === 'codex') { + return readCodexApiKey(options.provider); + } + + if (options.authMode !== 'codex' && envApiKey) { + return { apiKey: envApiKey, source: 'env' }; + } + + return readCodexApiKey(options.provider); +} + +export function requireConfiguredApiKey(options: PiAuthOptions): string { + const apiKey = resolveApiKey(options); + if (apiKey) { + return apiKey; + } + + const authMode = options.authMode ?? 'env'; + if (authMode === 'codex') { + if (options.provider !== 'openai') { + throw new Error( + `Codex auth only supports the "openai" provider, got "${options.provider}". ` + + `Use authMode: "env" with an appropriate API key env var instead.`, + ); + } + throw new Error( + `Codex auth is enabled for provider "${options.provider}" but no usable access token or OPENAI_API_KEY was found in ~/.codex/auth.json.`, + ); + } + + if (authMode === 'auto' && options.provider === 'openai') { + const envName = options.apiKeyEnv ?? 'OPENAI_API_KEY'; + throw new Error( + `Could not resolve auth for provider "${options.provider}". ` + + `Checked env var "${envName}" and ~/.codex/auth.json for a browser-login access token or OPENAI_API_KEY.`, + ); + } + + const envName = options.apiKeyEnv ?? defaultApiKeyEnvForProvider(options.provider); + if (!envName) { + throw new Error(`No default API key env var is known for provider "${options.provider}"`); + } + + throw new Error(`Missing API key env var: ${envName}`); +} + export function requireApiKeyFromEnv(apiKeyEnv: string): string { const apiKey = process.env[apiKeyEnv]; if (!apiKey) { @@ -26,3 +108,73 @@ export function requireApiKeyFromEnv(apiKeyEnv: string): string { } return apiKey; } + +function readCodexApiKey(provider: string): ResolvedApiCredential { + if (provider !== 'openai') { + return {}; + } + + const authPath = resolve(homedir(), '.codex', 'auth.json'); + let raw: string; + try { + raw = readFileSync(authPath, 'utf-8'); + } catch { + return {}; + } + + try { + const parsed = JSON.parse(raw) as { + OPENAI_API_KEY?: unknown; + tokens?: { OPENAI_API_KEY?: unknown; access_token?: unknown }; + }; + // Browser-login JWT takes highest priority: it represents an active user session. + // A stale static key must not shadow a valid browser-login token. + // However, an expired JWT must not shadow a valid static key — fall through on expiry. + if (typeof parsed.tokens?.access_token === 'string' && parsed.tokens.access_token.trim()) { + if (!isJwtExpired(parsed.tokens.access_token)) { + return { apiKey: parsed.tokens.access_token, source: 'codex' }; + } + // JWT is expired — fall through to static key fallbacks below + } + if (typeof parsed.tokens?.OPENAI_API_KEY === 'string' && parsed.tokens.OPENAI_API_KEY.trim()) { + return { apiKey: parsed.tokens.OPENAI_API_KEY, source: 'codex' }; + } + if (typeof parsed.OPENAI_API_KEY === 'string' && parsed.OPENAI_API_KEY.trim()) { + return { apiKey: parsed.OPENAI_API_KEY, source: 'codex' }; + } + return {}; + } catch { + return {}; + } +} + +function isJwtExpired(token: string): boolean { + const parts = token.split('.'); + if (parts.length !== 3) { + return false; + } + + try { + const payload = JSON.parse(Buffer.from(parts[1]!, 'base64url').toString('utf-8')) as { exp?: unknown }; + if (typeof payload.exp !== 'number') { + return false; + } + return payload.exp * 1000 <= Date.now() + CODEX_ACCESS_TOKEN_REFRESH_SKEW_MS; + } catch { + return false; + } +} + +function defaultApiKeyEnvForProvider(provider: string): string | undefined { + switch (provider) { + case 'openai': + case 'openai-codex': + return 'OPENAI_API_KEY'; + case 'anthropic': + return 'ANTHROPIC_API_KEY'; + case 'openrouter': + return 'OPENROUTER_API_KEY'; + default: + return undefined; + } +} diff --git a/src/runtime/pi/benchmark-agent.ts b/src/runtime/pi/benchmark-agent.ts index 93bf0b3..634cc92 100644 --- a/src/runtime/pi/benchmark-agent.ts +++ b/src/runtime/pi/benchmark-agent.ts @@ -2,10 +2,12 @@ import { resolvePiModelByRef } from './models.js'; export async function createReadOnlyBenchmarkModel(params: { modelRef: string; + authMode?: import('./auth.js').PiAuthMode; apiKeyEnv?: string; apiKeyOverride?: string; }) { return resolvePiModelByRef(params.modelRef, { + authMode: params.authMode, apiKeyEnv: params.apiKeyEnv, apiKeyOverride: params.apiKeyOverride, }); diff --git a/src/runtime/pi/coding-orchestrator.ts b/src/runtime/pi/coding-orchestrator.ts index bc32256..e21fc49 100644 --- a/src/runtime/pi/coding-orchestrator.ts +++ b/src/runtime/pi/coding-orchestrator.ts @@ -10,11 +10,15 @@ import { resolvePiModel } from './models.js'; export async function createCodingOrchestratorSession(params: { cwd: string; modelRef: string; + authMode?: import('./auth.js').PiAuthMode; apiKeyEnv?: string; thinkingLevel?: 'off' | 'minimal' | 'low' | 'medium' | 'high' | 'xhigh'; }) { const { provider, model } = parseModelRef(params.modelRef); - const resolved = await resolvePiModel(provider, model, { apiKeyEnv: params.apiKeyEnv }); + const resolved = await resolvePiModel(provider, model, { + authMode: params.authMode, + apiKeyEnv: params.apiKeyEnv, + }); return createAgentSession({ cwd: params.cwd, diff --git a/src/runtime/pi/index.ts b/src/runtime/pi/index.ts index 12905b7..c4c8a02 100644 --- a/src/runtime/pi/index.ts +++ b/src/runtime/pi/index.ts @@ -1,7 +1,7 @@ -export { createPiAuthStorage, requireApiKeyFromEnv } from './auth.js'; +export { createPiAuthStorage, requireApiKeyFromEnv, requireConfiguredApiKey, resolveApiCredential, resolveApiKey } from './auth.js'; export { createReadOnlyBenchmarkModel } from './benchmark-agent.js'; export { createCodingOrchestratorSession } from './coding-orchestrator.js'; export { resolvePiModel, resolvePiModelByRef } from './models.js'; -export type { PiAuthOptions } from './auth.js'; +export type { PiAuthMode, PiAuthOptions } from './auth.js'; export type { ResolvedPiModelRequest } from './models.js'; diff --git a/src/runtime/pi/models.ts b/src/runtime/pi/models.ts index 6972427..faf940f 100644 --- a/src/runtime/pi/models.ts +++ b/src/runtime/pi/models.ts @@ -2,7 +2,7 @@ import type { Model } from '@mariozechner/pi-ai'; import { getModel } from '@mariozechner/pi-ai'; import { ModelRegistry } from '@mariozechner/pi-coding-agent'; -import { createPiAuthStorage } from './auth.js'; +import { createPiAuthStorage, resolveApiCredential } from './auth.js'; import { parseModelRef } from '../../project/types.js'; export interface ResolvedPiModelRequest { @@ -17,7 +17,7 @@ export interface ResolvedPiModelRequest { export async function resolvePiModelByRef( modelRef: string, - options?: { apiKeyEnv?: string; apiKeyOverride?: string }, + options?: { authMode?: import('./auth.js').PiAuthMode; apiKeyEnv?: string; apiKeyOverride?: string }, ): Promise { const { provider, model } = parseModelRef(modelRef); return resolvePiModel(provider, model, options); @@ -45,32 +45,66 @@ function synthesizeOpenRouterModel(provider: string, modelName: string): Model | undefined { + if (provider !== 'openai-codex') return undefined; + return { + id: modelName, + name: modelName, + api: 'openai-codex-responses' as const, + provider: 'openai-codex' as const, + baseUrl: 'https://chatgpt.com/backend-api/codex', + reasoning: true, + input: ['text', 'image'] as ('text' | 'image')[], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 272000, + maxTokens: 32768, + }; +} + export async function resolvePiModel( provider: string, modelName: string, - options?: { apiKeyEnv?: string; apiKeyOverride?: string }, + options?: { authMode?: import('./auth.js').PiAuthMode; apiKeyEnv?: string; apiKeyOverride?: string }, ): Promise { - // Guard: direct Anthropic provider + OpenRouter key = guaranteed 401. - // Users who want Claude should use openrouter/anthropic/claude-* instead. + // Guard: direct-provider model + OpenRouter key = guaranteed 401. + // Catches stale scaffolded configs that had apiKeyEnv:"OPENROUTER_API_KEY" and were + // later updated to use a direct openai/ or anthropic/ model without fixing the key. if (provider === 'anthropic' && options?.apiKeyEnv === 'OPENROUTER_API_KEY') { throw new Error( `Model "${provider}/${modelName}" routes through the Anthropic API directly and requires ANTHROPIC_API_KEY. ` + `To use Claude via OpenRouter, change the model ID to "openrouter/anthropic/${modelName}".`, ); } + if (provider === 'openai' && options?.apiKeyEnv === 'OPENROUTER_API_KEY') { + throw new Error( + `Model "${provider}/${modelName}" routes through the OpenAI API directly and requires OPENAI_API_KEY. ` + + `To use this model via OpenRouter, change the model ID to "openrouter/openai/${modelName}".`, + ); + } - const authStorage = createPiAuthStorage({ + const credential = resolveApiCredential({ provider, + authMode: options?.authMode, apiKeyEnv: options?.apiKeyEnv, apiKeyOverride: options?.apiKeyOverride, }); + const resolvedProvider = provider === 'openai' && credential.source === 'codex' + ? 'openai-codex' + : provider; + const authStorage = createPiAuthStorage({ + provider: resolvedProvider, + authMode: options?.authMode, + apiKeyEnv: options?.apiKeyEnv, + apiKeyOverride: credential.apiKey, + }); const modelRegistry = ModelRegistry.create(authStorage); - const registryModel = modelRegistry.find(provider, modelName); + const registryModel = modelRegistry.find(resolvedProvider, modelName); const resolvedModel = registryModel - ?? getModel(provider as never, modelName) - ?? synthesizeOpenRouterModel(provider, modelName); + ?? getModel(resolvedProvider as never, modelName) + ?? synthesizeOpenRouterModel(resolvedProvider, modelName) + ?? synthesizeOpenAICodexModel(resolvedProvider, modelName); if (!resolvedModel) { - throw new Error(`Could not resolve pi model ${provider}/${modelName}`); + throw new Error(`Could not resolve pi model ${resolvedProvider}/${modelName}`); } const auth = await modelRegistry.getApiKeyAndHeaders(resolvedModel); diff --git a/src/tasks/default-pi-critic.ts b/src/tasks/default-pi-critic.ts index 794f46a..1348b3a 100644 --- a/src/tasks/default-pi-critic.ts +++ b/src/tasks/default-pi-critic.ts @@ -6,6 +6,7 @@ import type { CriticDeps } from '../verdict/recommendations.js'; export interface DefaultPiCriticOptions { provider: string; model: string; + authMode?: import('../runtime/pi/auth.js').PiAuthMode; apiKeyEnv?: string; timeoutMs?: number; headers?: Record; @@ -15,6 +16,7 @@ export function createDefaultPiCritic(options: DefaultPiCriticOptions): CriticDe return { async complete(input) { const resolved = await resolvePiModel(options.provider, options.model, { + authMode: options.authMode, apiKeyEnv: options.apiKeyEnv, }); diff --git a/src/tasks/default-pi-generator.ts b/src/tasks/default-pi-generator.ts index b9523e7..ce9b02e 100644 --- a/src/tasks/default-pi-generator.ts +++ b/src/tasks/default-pi-generator.ts @@ -10,6 +10,7 @@ type ThinkingLevel = NonNullable; export interface DefaultPiGeneratorOptions { provider: string; model: string; + authMode?: import('../runtime/pi/auth.js').PiAuthMode; apiKeyEnv?: string; timeoutMs?: number; headers?: Record; @@ -20,6 +21,7 @@ export function createDefaultPiTaskGenerator(options: DefaultPiGeneratorOptions) return { async complete(input) { const resolved = await resolvePiModel(options.provider, options.model, { + authMode: options.authMode, apiKeyEnv: options.apiKeyEnv, }); diff --git a/src/tasks/freeze.ts b/src/tasks/freeze.ts index 7954059..dafd498 100644 --- a/src/tasks/freeze.ts +++ b/src/tasks/freeze.ts @@ -47,6 +47,7 @@ export function freezeTaskArtifacts(params: FreezeTaskArtifactsParams): FrozenTa benchmark: { format: params.project.benchmark.format, baseUrl: params.project.benchmark.baseUrl, + authMode: params.project.benchmark.authMode, apiKeyEnv: params.project.benchmark.apiKeyEnv, timeout: params.project.benchmark.timeout, headers: params.project.benchmark.headers, @@ -63,7 +64,6 @@ export function freezeTaskArtifacts(params: FreezeTaskArtifactsParams): FrozenTa output: params.project.benchmark.output, agentic: params.project.benchmark.agentic, }, - optimize: params.project.optimize, }; writeFileSync(benchmarkPath, JSON.stringify(generatedProject, null, 2), 'utf-8'); diff --git a/tests/smoke-actions.ts b/tests/smoke-actions.ts index 267f362..a23bfcf 100644 --- a/tests/smoke-actions.ts +++ b/tests/smoke-actions.ts @@ -10,7 +10,7 @@ import { writeActionSnapshotFile, } from '../src/actions/snapshot.js'; import { diffActionCatalog } from '../src/actions/diff.js'; -import { buildSurfaceSnapshot, loadProjectConfig, loadSurfaceSnapshotFile } from '../src/project/index.js'; +import { buildMcpToolDefinitionsFromSnapshot, buildSurfaceSnapshot, loadProjectConfig, loadSurfaceSnapshotFile } from '../src/project/index.js'; import type { SurfaceSnapshot } from '../src/project/types.js'; let passed = 0; @@ -69,6 +69,40 @@ await test('snapshot write/load roundtrip includes artifact version', () => { } }); +await test('snapshot roundtrip preserves nested MCP arg schemas', () => { + const root = mkdtempSync(join(tmpdir(), 'skill-optimizer-actions-schema-')); + try { + const snapshotPath = join(root, 'actions.snapshot.json'); + writeActionSnapshotFile(snapshotPath, { + surface: 'mcp', + actions: [ + { + key: 'folders.update', + name: 'folders.update', + args: [ + { + name: 'folderIds', + required: true, + type: 'array', + schema: { + type: 'array', + items: { type: 'integer' }, + description: 'Folder ids to update', + }, + }, + ], + }, + ], + }); + + const loaded = loadActionSnapshotFile(snapshotPath); + const schema = loaded.catalog.actions[0].args[0].schema as { items?: { type?: string } }; + assertEqual(schema.items?.type, 'integer', 'nested array items should survive snapshot roundtrip'); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); + await test('diffActionCatalog ignores arg reordering and catches schema changes', () => { const before = { surface: 'mcp' as const, @@ -390,6 +424,43 @@ await test('buildSurfaceSnapshot returns expected legacy snapshot fields from di } }); +await test('buildMcpToolDefinitionsFromSnapshot preserves nested arg schemas', () => { + const snapshot: SurfaceSnapshot = { + surface: 'mcp', + actions: [ + { + name: 'folders.update', + args: [ + { + name: 'folderIds', + required: true, + type: 'array', + schema: { + type: 'array', + items: { type: 'integer' }, + description: 'Folder ids to update', + }, + }, + { + name: 'peers', + required: false, + type: 'array', + schema: { + type: 'array', + items: { type: 'string' }, + }, + }, + ], + }, + ], + }; + + const definitions = buildMcpToolDefinitionsFromSnapshot(snapshot); + const properties = definitions[0].function.parameters?.properties as Record; + assertEqual(properties.folderIds.items.type, 'integer', 'array items should be preserved in rebuilt tool schema'); + assertEqual(properties.peers.items.type, 'string', 'optional array items should be preserved in rebuilt tool schema'); +}); + await test('loadSurfaceSnapshotFile supports legacy plain snapshot shape', () => { const root = mkdtempSync(join(tmpdir(), 'skill-optimizer-snapshot-legacy-')); try { diff --git a/tests/smoke-code.ts b/tests/smoke-code.ts index d1a471d..bf6a3f8 100644 --- a/tests/smoke-code.ts +++ b/tests/smoke-code.ts @@ -751,6 +751,22 @@ await test('checkConfig: model ID with dot version → fixable warning', async ( assert(warn!.hint?.includes('4-6'), `hint should show hyphen version, got: ${warn!.hint}`); }); +await test('checkConfig: direct openai model IDs do not get OpenRouter dot-version warning', async () => { + const { checkConfig } = await import('../src/project/validate.js'); + const config = { + name: 'test', + target: { surface: 'cli' as const, discovery: { sources: ['./src/cli.ts'] } }, + benchmark: { + format: 'pi' as const, + models: [{ id: 'openai/gpt-5.4', name: 'GPT-5.4', tier: 'flagship' as const }], + taskGeneration: { enabled: true, maxTasks: 5 }, + }, + }; + const issues = await checkConfig(config as any, '/fake/skill-optimizer.json'); + const warn = issues.find(i => i.code === 'model-id-bad-format'); + assert(warn === undefined, 'direct openai model IDs should not be warned as OpenRouter dot versions'); +}); + await test('checkConfig: deprecated benchmark.tasks field → fixable warning', async () => { const { checkConfig } = await import('../src/project/validate.js'); const config = { @@ -770,6 +786,71 @@ await test('checkConfig: deprecated benchmark.tasks field → fixable warning', assert(warn!.fixable === true, 'deprecated-tasks-field should be fixable'); }); +await test('checkConfig: codex auth rejects non-openai benchmark models', async () => { + const { checkConfig } = await import('../src/project/validate.js'); + const config = { + name: 'test', + target: { surface: 'sdk' as const, discovery: { sources: ['./src/index.ts'], language: 'typescript' as const } }, + benchmark: { + format: 'pi' as const, + authMode: 'codex' as const, + models: [ + { id: 'openai/gpt-5.4', name: 'GPT-5.4', tier: 'flagship' as const }, + { id: 'openrouter/openai/gpt-5.4', name: 'OpenRouter GPT-5.4', tier: 'flagship' as const }, + ], + tasks: './tasks.json', + }, + }; + const issues = await checkConfig(config as any, '/fake/skill-optimizer.json'); + const err = issues.find(i => i.code === 'codex-auth-provider-mismatch' && i.field.includes('benchmark.models')); + assert(err !== undefined, 'expected codex-auth-provider-mismatch for benchmark.models'); + assert(err!.severity === 'error', 'benchmark model/provider mismatch should be an error'); +}); + +await test('checkConfig: codex auth rejects non-openai optimize model', async () => { + const { checkConfig } = await import('../src/project/validate.js'); + const config = { + name: 'test', + target: { surface: 'sdk' as const, discovery: { sources: ['./src/index.ts'], language: 'typescript' as const } }, + benchmark: { + format: 'pi' as const, + models: [{ id: 'openai/gpt-5.4', name: 'GPT-5.4', tier: 'flagship' as const }], + tasks: './tasks.json', + }, + optimize: { + authMode: 'codex' as const, + model: 'openrouter/anthropic/claude-sonnet-4-6', + allowedPaths: ['SKILL.md'], + }, + }; + const issues = await checkConfig(config as any, '/fake/skill-optimizer.json'); + const err = issues.find(i => i.code === 'codex-auth-provider-mismatch' && i.field === 'optimize.model'); + assert(err !== undefined, 'expected codex-auth-provider-mismatch for optimize.model'); + assert(err!.severity === 'error', 'optimize model/provider mismatch should be an error'); +}); + +await test('checkConfig: inherited codex auth rejects non-openai optimize model', async () => { + const { checkConfig } = await import('../src/project/validate.js'); + const config = { + name: 'test', + target: { surface: 'sdk' as const, discovery: { sources: ['./src/index.ts'], language: 'typescript' as const } }, + benchmark: { + format: 'pi' as const, + authMode: 'codex' as const, + models: [{ id: 'openai/gpt-5.4', name: 'GPT-5.4', tier: 'flagship' as const }], + tasks: './tasks.json', + }, + optimize: { + model: 'openrouter/anthropic/claude-sonnet-4-6', + allowedPaths: ['SKILL.md'], + }, + }; + const issues = await checkConfig(config as any, '/fake/skill-optimizer.json'); + const err = issues.find(i => i.code === 'codex-auth-provider-mismatch' && i.field === 'optimize.model'); + assert(err !== undefined, 'expected inherited codex-auth-provider-mismatch for optimize.model'); + assert(err!.severity === 'error', 'inherited optimize model/provider mismatch should be an error'); +}); + await test('applyFixes: adds openrouter/ prefix to model IDs', async () => { const { applyFixes } = await import('../src/project/fix.js'); const { checkConfig } = await import('../src/project/validate.js'); @@ -918,6 +999,58 @@ await test('doctor --fix: corrects model ID in-place', async () => { } }); +await test('checkModelReachability: mixed list skips non-openrouter and probes only openrouter models', async () => { + const { checkModelReachability } = await import('../src/doctor/checks.js'); + + const project = { + configPath: '/fake/skill-optimizer.json', + configDir: '/fake', + name: 'test-mixed', + target: { + surface: 'mcp', + repoPath: '/fake', + scope: { include: ['.*'], exclude: [] }, + }, + benchmark: { + format: 'pi', + authMode: 'env', + timeout: 30000, + models: [ + { id: 'anthropic/claude-sonnet-4-6', name: 'Claude', tier: 'flagship' as const }, + { id: 'openrouter/openai/gpt-4o', name: 'GPT-4o', tier: 'flagship' as const }, + ], + taskGeneration: { enabled: false, maxTasks: 10, useExisting: false }, + output: { dir: '/fake/.results' }, + verdict: { perModelFloor: 0.5, targetWeightedAverage: 0.6 }, + }, + } as unknown as ResolvedProjectConfig; + + // Without OPENROUTER_API_KEY set the key resolution throws, so the function + // returns early after emitting reachability-skipped. We verify: + // 1. A reachability-skipped issue appears (for the 1 non-openrouter model) + // 2. No reachability-skipped with field 'benchmark.format' (wrong early-exit path) + // 3. No model-unreachable for the anthropic model + const savedKey = process.env['OPENROUTER_API_KEY']; + delete process.env['OPENROUTER_API_KEY']; + try { + const issues = await checkModelReachability(project); + const skipped = issues.filter((i) => i.code === 'reachability-skipped'); + assert(skipped.length >= 1, 'should have at least one reachability-skipped issue'); + const modelSkipped = skipped.find((i) => i.field === 'benchmark.models'); + assert(modelSkipped !== undefined, 'reachability-skipped issue should reference benchmark.models field'); + assert( + modelSkipped!.message.includes('1 non-OpenRouter'), + `message should count 1 skipped, got: ${modelSkipped!.message}`, + ); + const anthropicUnreachable = issues.find( + (i) => i.code === 'model-unreachable' && i.message.includes('anthropic/'), + ); + assert(anthropicUnreachable === undefined, 'anthropic model must not produce a model-unreachable issue'); + } finally { + if (savedKey !== undefined) process.env['OPENROUTER_API_KEY'] = savedKey; + } +}); + // ── Summary ──────────────────────────────────────────────────────────────── console.log(`\n${passed} passed, ${failed} failed\n`); diff --git a/tests/smoke-generation.ts b/tests/smoke-generation.ts index 925e8b6..3ae2c3a 100644 --- a/tests/smoke-generation.ts +++ b/tests/smoke-generation.ts @@ -322,13 +322,16 @@ await test('freezeGeneratedBenchmark: writes artifacts and absolute paths', asyn const benchmark = JSON.parse(readFileSync(frozen.benchmarkPath, 'utf-8')) as { target: { skill: { source: string; cache: boolean } }; - benchmark: { tasks: string; surfaceSnapshot: string }; + benchmark: { authMode?: string; tasks: string; surfaceSnapshot: string }; + optimize?: unknown; }; assert(benchmark.target.skill.source.startsWith('/'), 'target.skill.source should be absolute'); assertEqual(benchmark.target.skill.cache, false, 'target.skill.cache should be preserved'); + assertEqual(benchmark.benchmark.authMode, 'env', 'benchmark authMode should be preserved in generated config'); assertEqual(benchmark.benchmark.tasks, frozen.tasksPath, 'tasks should point at generated tasks path'); assertEqual(benchmark.benchmark.surfaceSnapshot, frozen.snapshotPath, 'surface snapshot should be pinned in generated config'); + assertEqual(benchmark.optimize, undefined, 'generated benchmark config should omit optimize-only settings'); } finally { rmSync(fixture.root, { recursive: true, force: true }); } diff --git a/tests/smoke-llm.ts b/tests/smoke-llm.ts index 64fc0d7..ff6308d 100644 --- a/tests/smoke-llm.ts +++ b/tests/smoke-llm.ts @@ -5,6 +5,7 @@ import { createLLMClient } from '../src/benchmark/llm/index.js'; import { __setPiImplementationsForTest } from '../src/benchmark/llm/pi-format.js'; +import { resolveApiKey } from '../src/runtime/pi/auth.js'; import type { LLMConfig, McpToolDefinition } from '../src/benchmark/types.js'; // --------------------------------------------------------------------------- @@ -102,6 +103,20 @@ const sampleTools: McpToolDefinition[] = [ }, ]; +const dottedSampleTools: McpToolDefinition[] = [ + { + type: 'function', + function: { + name: 'auth.status', + description: 'Check session auth state', + parameters: { + type: 'object', + properties: {}, + }, + }, + }, +]; + // --------------------------------------------------------------------------- // Save original fetch so we can restore it after all tests // --------------------------------------------------------------------------- @@ -134,10 +149,11 @@ await test('createLLMClient: creates client for anthropic format', () => { assert(typeof client.chatWithTools === 'function', 'client.chatWithTools should be a function'); }); -await test('createLLMClient: throws when apiKeyEnv is set but env var is missing', () => { +await test('createLLMClient: throws when apiKeyEnv is set but env var is missing', async () => { let threw = false; try { - createLLMClient({ ...openaiConfig, apiKeyEnv: '__MISSING_ENV_VAR_XYZ__' }); + const client = createLLMClient({ ...openaiConfig, apiKeyEnv: '__MISSING_ENV_VAR_XYZ__' }); + await client.chat('gpt-5.4', 'sys', 'user'); } catch (e: any) { threw = true; assert( @@ -150,15 +166,19 @@ await test('createLLMClient: throws when apiKeyEnv is set but env var is missing await test('pi format: uses provider/model id and runtime auth override for text chat', async () => { let capturedModel: string | null = null; + let capturedAuthMode: string | undefined; + let capturedApiKeyEnv: string | undefined; let capturedApiKey: string | undefined; __setPiImplementationsForTest({ - async resolve(modelId, apiKeyOverride) { + async resolve(modelId, authOptions) { capturedModel = modelId; - capturedApiKey = apiKeyOverride; + capturedAuthMode = authOptions?.authMode; + capturedApiKeyEnv = authOptions?.apiKeyEnv; + capturedApiKey = authOptions?.apiKeyOverride; return { model: { id: 'openai/gpt-5.4', provider: 'openrouter', api: 'openai-completions', name: 'GPT-5.4' } as any, - auth: { apiKey: apiKeyOverride, headers: { 'x-test-header': 'yes' } }, + auth: { apiKey: 'test-key-12345', headers: { 'x-test-header': 'yes' } }, }; }, async completeSimple(_model, context, options) { @@ -186,7 +206,9 @@ await test('pi format: uses provider/model id and runtime auth override for text const result = await client.chat('openrouter/openai/gpt-5.4', 'sys', 'user'); assertEqual(capturedModel, 'openrouter/openai/gpt-5.4', 'model id should be resolved through pi'); - assertEqual(capturedApiKey, 'test-key-12345', 'api key override should be passed into pi resolution'); + assertEqual(capturedAuthMode, undefined, 'default auth mode should be passed through as undefined'); + assertEqual(capturedApiKeyEnv, '__TEST_API_KEY__', 'api key env should be forwarded to pi resolution'); + assertEqual(capturedApiKey, undefined, 'pi resolution should now read the configured env var itself'); assertEqual(result.content, 'hello from pi', 'pi text response should be surfaced'); __setPiImplementationsForTest(null); @@ -231,6 +253,440 @@ await test('pi format: converts tool calls for MCP chat', async () => { __setPiImplementationsForTest(null); }); +await test('pi format: sanitizes dotted MCP tool names and maps them back', async () => { + __setPiImplementationsForTest({ + async resolve() { + return { + model: { id: 'gpt-5.4', provider: 'openai-codex', api: 'openai-codex-responses', name: 'GPT-5.4' } as any, + auth: { apiKey: 'codex-access-token', headers: {} }, + }; + }, + async completeSimple() { + throw new Error('unexpected completeSimple() call'); + }, + async complete(_model, context) { + assert(Array.isArray(context.tools), 'tools should be passed to pi complete()'); + assertEqual(context.tools?.[0]?.name, 'auth_status', 'tool name should be sanitized for provider request'); + return { + role: 'assistant', + api: 'openai-codex-responses', + provider: 'openai-codex', + model: 'gpt-5.4', + stopReason: 'toolUse', + timestamp: Date.now(), + usage: { input: 1, output: 1, cacheRead: 0, cacheWrite: 0, totalTokens: 2, cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 } }, + content: [ + { type: 'toolCall', id: 'call-1', name: 'auth_status', arguments: {} }, + ], + } as any; + }, + }); + + const client = createLLMClient({ ...piConfig, authMode: 'codex' }); + const result = await client.chatWithTools('openai/gpt-5.4', 'sys', 'user', dottedSampleTools); + + assertEqual(result.toolCalls?.[0]?.name, 'auth.status', 'tool call name should map back to canonical form'); + + __setPiImplementationsForTest(null); +}); + +await test('pi format: throws provider-side errors instead of returning empty output', async () => { + __setPiImplementationsForTest({ + async resolve() { + return { + model: { id: 'gpt-5.4', provider: 'openai-codex', api: 'openai-codex-responses', name: 'GPT-5.4' } as any, + auth: { apiKey: 'codex-access-token', headers: {} }, + }; + }, + async completeSimple() { + throw new Error('unexpected completeSimple() call'); + }, + async complete() { + return { + role: 'assistant', + api: 'openai-codex-responses', + provider: 'openai-codex', + model: 'gpt-5.4', + stopReason: 'error', + errorMessage: 'Invalid tool schema', + timestamp: Date.now(), + usage: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, totalTokens: 0, cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 } }, + content: [], + } as any; + }, + }); + + const client = createLLMClient({ ...piConfig, authMode: 'codex' }); + let threw = false; + try { + await client.chatWithTools('openai/gpt-5.4', 'sys', 'user', dottedSampleTools); + } catch (e: any) { + threw = true; + assert( + e.message.includes('Invalid tool schema'), + 'provider-side error message should surface to the caller', + ); + } + assert(threw, 'chatWithTools should throw when the provider reports an error'); + + __setPiImplementationsForTest(null); +}); + +await test('resolveApiKey: codex auth reads browser-login access token from ~/.codex/auth.json', async () => { + const originalHome = process.env.HOME; + const dir = await import('node:fs/promises').then(({ mkdtemp, mkdir, writeFile }) => ({ mkdtemp, mkdir, writeFile })); + const os = await import('node:os'); + const path = await import('node:path'); + const tmpHome = await dir.mkdtemp(path.join(os.tmpdir(), 'codex-auth-')); + const codexDir = path.join(tmpHome, '.codex'); + await dir.mkdir(codexDir, { recursive: true }); + const futureExp = Math.floor(Date.now() / 1000) + 3600; + const jwt = [ + Buffer.from(JSON.stringify({ alg: 'none', typ: 'JWT' })).toString('base64url'), + Buffer.from(JSON.stringify({ exp: futureExp })).toString('base64url'), + 'sig', + ].join('.'); + await dir.writeFile( + path.join(codexDir, 'auth.json'), + JSON.stringify({ auth_mode: 'chatgpt', tokens: { access_token: jwt } }), + 'utf-8', + ); + process.env.HOME = tmpHome; + + try { + const result = resolveApiKey({ provider: 'openai', authMode: 'codex' }); + assertEqual(result, jwt, 'browser-login access token should be returned'); + } finally { + if (originalHome === undefined) { + delete process.env.HOME; + } else { + process.env.HOME = originalHome; + } + } +}); + +await test('openai format: codex auth bridges through pi with openai provider refs', async () => { + const originalHome = process.env.HOME; + const dir = await import('node:fs/promises').then(({ mkdtemp, mkdir, writeFile }) => ({ mkdtemp, mkdir, writeFile })); + const os = await import('node:os'); + const path = await import('node:path'); + const tmpHome = await dir.mkdtemp(path.join(os.tmpdir(), 'codex-bridge-auth-')); + const codexDir = path.join(tmpHome, '.codex'); + await dir.mkdir(codexDir, { recursive: true }); + const futureExp = Math.floor(Date.now() / 1000) + 3600; + const jwt = [ + Buffer.from(JSON.stringify({ alg: 'none', typ: 'JWT' })).toString('base64url'), + Buffer.from(JSON.stringify({ exp: futureExp })).toString('base64url'), + 'sig', + ].join('.'); + await dir.writeFile( + path.join(codexDir, 'auth.json'), + JSON.stringify({ auth_mode: 'chatgpt', tokens: { access_token: jwt } }), + 'utf-8', + ); + process.env.HOME = tmpHome; + + let capturedModel: string | null = null; + + __setPiImplementationsForTest({ + async resolve(modelId) { + capturedModel = modelId; + return { + model: { id: 'gpt-5.4', provider: 'openai-codex', api: 'openai-codex-responses', name: 'GPT-5.4' } as any, + auth: { apiKey: 'codex-access-token', headers: {} }, + }; + }, + async completeSimple(_model, context, options) { + assertEqual(context.systemPrompt, 'sys', 'system prompt passed through'); + assertEqual((context.messages[0] as any).content, 'user', 'user content passed through'); + assertEqual(options?.apiKey, 'codex-access-token', 'codex token should flow through pi'); + return { + role: 'assistant', + api: 'openai-codex-responses', + provider: 'openai-codex', + model: 'gpt-5.4', + stopReason: 'stop', + timestamp: Date.now(), + usage: { input: 1, output: 1, cacheRead: 0, cacheWrite: 0, totalTokens: 2, cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 } }, + content: [{ type: 'text', text: 'hello from codex auth' }], + } as any; + }, + async complete() { + throw new Error('unexpected complete() call'); + }, + }); + + try { + const client = createLLMClient({ + format: 'openai', + baseUrl: 'https://api.openai.com/v1', + authMode: 'codex', + models: [], + }); + const result = await client.chat('gpt-5.4', 'sys', 'user'); + assertEqual(capturedModel, 'openai/gpt-5.4', 'openai-format codex auth should bridge to pi using provider/model form'); + assertEqual(result.content, 'hello from codex auth', 'codex-auth bridged response should be returned'); + } finally { + if (originalHome === undefined) { + delete process.env.HOME; + } else { + process.env.HOME = originalHome; + } + __setPiImplementationsForTest(null); + } +}); + +await test('openai format: codex bridge passes authMode:codex (not apiKeyOverride) to pi call', async () => { + const originalHome = process.env.HOME; + const dir = await import('node:fs/promises').then(({ mkdtemp, mkdir, writeFile }) => ({ mkdtemp, mkdir, writeFile })); + const os = await import('node:os'); + const path = await import('node:path'); + const tmpHome = await dir.mkdtemp(path.join(os.tmpdir(), 'codex-override-auth-')); + const codexDir = path.join(tmpHome, '.codex'); + await dir.mkdir(codexDir, { recursive: true }); + const futureExp = Math.floor(Date.now() / 1000) + 3600; + const jwt = [ + Buffer.from(JSON.stringify({ alg: 'none', typ: 'JWT' })).toString('base64url'), + Buffer.from(JSON.stringify({ exp: futureExp })).toString('base64url'), + 'sig', + ].join('.'); + await dir.writeFile( + path.join(codexDir, 'auth.json'), + JSON.stringify({ auth_mode: 'chatgpt', tokens: { access_token: jwt } }), + 'utf-8', + ); + process.env.HOME = tmpHome; + + let capturedAuthMode: string | undefined = 'NOT_SET'; + let capturedApiKeyEnv: string | undefined = 'NOT_SET'; + let capturedApiKeyOverride: string | undefined = 'NOT_SET'; + + __setPiImplementationsForTest({ + async resolve(_modelId, authOptions) { + capturedAuthMode = authOptions?.authMode; + capturedApiKeyEnv = authOptions?.apiKeyEnv; + capturedApiKeyOverride = authOptions?.apiKeyOverride; + return { + model: { id: 'gpt-5.4', provider: 'openai-codex', api: 'openai-codex-responses', name: 'GPT-5.4' } as any, + auth: { apiKey: jwt, headers: {} }, + }; + }, + async completeSimple() { + return { + role: 'assistant', + api: 'openai-codex-responses', + provider: 'openai-codex', + model: 'gpt-5.4', + stopReason: 'stop', + timestamp: Date.now(), + usage: { input: 1, output: 1, cacheRead: 0, cacheWrite: 0, totalTokens: 2, cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 } }, + content: [{ type: 'text', text: 'ok' }], + } as any; + }, + async complete() { + throw new Error('unexpected complete() call'); + }, + }); + + try { + const client = createLLMClient({ + format: 'openai', + baseUrl: 'https://api.openai.com/v1', + authMode: 'codex', + models: [], + }); + await client.chat('gpt-5.4', 'sys', 'user'); + + assertEqual(capturedAuthMode, 'codex', 'codex bridge should pass authMode:codex to pi for correct provider routing'); + assert(capturedApiKeyOverride === undefined, `codex bridge should NOT pass apiKeyOverride to pi (would break source:'codex' routing signal) (got: ${capturedApiKeyOverride})`); + } finally { + if (originalHome === undefined) { + delete process.env.HOME; + } else { + process.env.HOME = originalHome; + } + __setPiImplementationsForTest(null); + } +}); + +await test('openai format: codex bridge passes authMode:codex (not apiKeyOverride) to pi call for chatWithTools', async () => { + const originalHome = process.env.HOME; + const dir = await import('node:fs/promises').then(({ mkdtemp, mkdir, writeFile }) => ({ mkdtemp, mkdir, writeFile })); + const os = await import('node:os'); + const path = await import('node:path'); + const tmpHome = await dir.mkdtemp(path.join(os.tmpdir(), 'codex-override-tools-')); + const codexDir = path.join(tmpHome, '.codex'); + await dir.mkdir(codexDir, { recursive: true }); + const futureExp = Math.floor(Date.now() / 1000) + 3600; + const jwt = [ + Buffer.from(JSON.stringify({ alg: 'none', typ: 'JWT' })).toString('base64url'), + Buffer.from(JSON.stringify({ exp: futureExp })).toString('base64url'), + 'sig', + ].join('.'); + await dir.writeFile( + path.join(codexDir, 'auth.json'), + JSON.stringify({ auth_mode: 'chatgpt', tokens: { access_token: jwt } }), + 'utf-8', + ); + process.env.HOME = tmpHome; + + let capturedAuthMode: string | undefined = 'NOT_SET'; + let capturedApiKeyEnv: string | undefined = 'NOT_SET'; + let capturedApiKeyOverride: string | undefined = 'NOT_SET'; + + __setPiImplementationsForTest({ + async resolve(_modelId, authOptions) { + capturedAuthMode = authOptions?.authMode; + capturedApiKeyEnv = authOptions?.apiKeyEnv; + capturedApiKeyOverride = authOptions?.apiKeyOverride; + return { + model: { id: 'gpt-5.4', provider: 'openai-codex', api: 'openai-codex-responses', name: 'GPT-5.4' } as any, + auth: { apiKey: jwt, headers: {} }, + }; + }, + async completeSimple() { + throw new Error('unexpected completeSimple() call'); + }, + async complete() { + return { + role: 'assistant', + api: 'openai-codex-responses', + provider: 'openai-codex', + model: 'gpt-5.4', + stopReason: 'stop', + timestamp: Date.now(), + usage: { input: 1, output: 1, cacheRead: 0, cacheWrite: 0, totalTokens: 2, cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 } }, + content: [{ type: 'text', text: 'ok' }], + } as any; + }, + }); + + try { + const client = createLLMClient({ + format: 'openai', + baseUrl: 'https://api.openai.com/v1', + authMode: 'codex', + models: [], + }); + await client.chatWithTools('gpt-5.4', 'sys', 'user', sampleTools); + + assertEqual(capturedAuthMode, 'codex', 'codex bridge should pass authMode:codex to pi for correct provider routing (chatWithTools)'); + assert(capturedApiKeyOverride === undefined, `codex bridge should NOT pass apiKeyOverride to pi (chatWithTools) (got: ${capturedApiKeyOverride})`); + } finally { + if (originalHome === undefined) { + delete process.env.HOME; + } else { + process.env.HOME = originalHome; + } + __setPiImplementationsForTest(null); + } +}); + +await test('openai format: codex bridge passes authMode:codex (not apiKeyOverride) to pi call for chatAgentLoop', async () => { + const originalHome = process.env.HOME; + const dir = await import('node:fs/promises').then(({ mkdtemp, mkdir, writeFile }) => ({ mkdtemp, mkdir, writeFile })); + const os = await import('node:os'); + const path = await import('node:path'); + const tmpHome = await dir.mkdtemp(path.join(os.tmpdir(), 'codex-override-agent-')); + const codexDir = path.join(tmpHome, '.codex'); + await dir.mkdir(codexDir, { recursive: true }); + const futureExp = Math.floor(Date.now() / 1000) + 3600; + const jwt = [ + Buffer.from(JSON.stringify({ alg: 'none', typ: 'JWT' })).toString('base64url'), + Buffer.from(JSON.stringify({ exp: futureExp })).toString('base64url'), + 'sig', + ].join('.'); + await dir.writeFile( + path.join(codexDir, 'auth.json'), + JSON.stringify({ auth_mode: 'chatgpt', tokens: { access_token: jwt } }), + 'utf-8', + ); + process.env.HOME = tmpHome; + + let capturedAuthMode: string | undefined = 'NOT_SET'; + let capturedApiKeyEnv: string | undefined = 'NOT_SET'; + let capturedApiKeyOverride: string | undefined = 'NOT_SET'; + + __setPiImplementationsForTest({ + async resolve(_modelId, authOptions) { + capturedAuthMode = authOptions?.authMode; + capturedApiKeyEnv = authOptions?.apiKeyEnv; + capturedApiKeyOverride = authOptions?.apiKeyOverride; + return { + model: { id: 'gpt-5.4', provider: 'openai-codex', api: 'openai-codex-responses', name: 'GPT-5.4' } as any, + auth: { apiKey: jwt, headers: {} }, + }; + }, + async completeSimple() { + throw new Error('unexpected completeSimple() call'); + }, + async complete() { + return { + role: 'assistant', + api: 'openai-codex-responses', + provider: 'openai-codex', + model: 'gpt-5.4', + stopReason: 'stop', + timestamp: Date.now(), + usage: { input: 1, output: 1, cacheRead: 0, cacheWrite: 0, totalTokens: 2, cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 } }, + content: [{ type: 'text', text: 'ok' }], + } as any; + }, + }); + + try { + const client = createLLMClient({ + format: 'openai', + baseUrl: 'https://api.openai.com/v1', + authMode: 'codex', + models: [], + }); + const dummyExecutor = async () => 'result'; + await client.chatAgentLoop('gpt-5.4', 'sys', 'user', sampleTools, dummyExecutor); + + assertEqual(capturedAuthMode, 'codex', 'codex bridge should pass authMode:codex to pi for correct provider routing (chatAgentLoop)'); + assert(capturedApiKeyOverride === undefined, `codex bridge should NOT pass apiKeyOverride to pi (chatAgentLoop) (got: ${capturedApiKeyOverride})`); + } finally { + if (originalHome === undefined) { + delete process.env.HOME; + } else { + process.env.HOME = originalHome; + } + __setPiImplementationsForTest(null); + } +}); + +await test('resolveApiKey: auto mode prefers env var over codex token when both present', async () => { + const originalHome = process.env.HOME; + const originalEnv = process.env.OPENAI_API_KEY; + const dir = await import('node:fs/promises').then(({ mkdtemp, mkdir, writeFile }) => ({ mkdtemp, mkdir, writeFile })); + const os = await import('node:os'); + const path = await import('node:path'); + const tmpHome = await dir.mkdtemp(path.join(os.tmpdir(), 'codex-auto-priority-')); + const codexDir = path.join(tmpHome, '.codex'); + await dir.mkdir(codexDir, { recursive: true }); + const futureExp = Math.floor(Date.now() / 1000) + 3600; + const jwt = [ + Buffer.from(JSON.stringify({ alg: 'none', typ: 'JWT' })).toString('base64url'), + Buffer.from(JSON.stringify({ exp: futureExp })).toString('base64url'), + 'sig', + ].join('.'); + await dir.writeFile( + path.join(codexDir, 'auth.json'), + JSON.stringify({ auth_mode: 'chatgpt', tokens: { access_token: jwt } }), + 'utf-8', + ); + process.env.HOME = tmpHome; + process.env.OPENAI_API_KEY = 'env-api-key'; + try { + const result = resolveApiKey({ provider: 'openai', authMode: 'auto' }); + assertEqual(result, 'env-api-key', 'auto mode should prefer env var over codex token'); + } finally { + if (originalHome === undefined) { delete process.env.HOME; } else { process.env.HOME = originalHome; } + if (originalEnv === undefined) { delete process.env.OPENAI_API_KEY; } else { process.env.OPENAI_API_KEY = originalEnv; } + } +}); + await test('pi format: agent loop feeds tool results back with original tool call ids', async () => { const toolCallIds: string[] = []; let callCount = 0; @@ -400,6 +856,48 @@ await test('openai format: parses tool_calls response', async () => { ); }); +await test('openai format: sanitizes dotted tool names in requests and maps them back in responses', async () => { + let capturedBody: any = null; + + globalThis.fetch = mockFetch((_url, init) => { + capturedBody = JSON.parse(init.body as string); + return { + status: 200, + body: { + choices: [ + { + message: { + content: '', + tool_calls: [ + { + function: { + name: 'auth_status', + arguments: '{}', + }, + }, + ], + }, + }, + ], + }, + }; + }) as any; + + const client = createLLMClient(openaiConfig); + const result = await client.chatWithTools('gpt-4o', 'sys', 'user', dottedSampleTools); + + assertEqual( + capturedBody.tools[0].function.name, + 'auth_status', + 'dotted tool name should be sanitized in the request body', + ); + assertEqual( + result.toolCalls?.[0]?.name, + 'auth.status', + 'sanitized tool call names should map back to canonical form', + ); +}); + await test('openai format: handles non-200 response', async () => { // The retry logic retries once after 3s on any non-AbortError. // To avoid a 3s delay in tests, we make both attempts fail immediately. From fc7ee88d3b3a3e4ab3594ea185a00efd86335234 Mon Sep 17 00:00:00 2001 From: dmn Date: Wed, 15 Apr 2026 10:49:22 -0700 Subject: [PATCH 4/7] feat: add SKILL folder for AI agent guidance (#14) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * docs: add SKILL design spec for skill-optimizer Spec for a multi-file SKILL/ folder that guides AI agents through setting up, benchmarking, and optimizing SDK/CLI/MCP documentation. Co-Authored-By: Claude Opus 4.6 (1M context) * docs: add implementation plan for SKILL folder 7-task plan covering branch creation, all 5 markdown files with complete content, verification, and PR creation. Co-Authored-By: Claude Opus 4.6 (1M context) * feat(skill): add SKILL.md entry point with context detection and routing * feat(skill): add setup reference — prerequisites, init wizard, discovery * feat(skill): add benchmark reference — run, interpret, diagnose, compare * feat(skill): add optimize reference — loop mechanics, safety, troubleshooting * feat(skill): add config reference — schema, models, scope, errors * fix(skill): address Copilot review — config path, safety boundary, minImprovement, gitignore dev docs * docs(skill): update routing concept; add direct-provider auth to prerequisites; fix dot-notation * docs(skill): expand compare section in benchmark reference --------- Co-authored-by: OpenClaw Agent (basd) Co-authored-by: Claude Opus 4.6 (1M context) --- .gitignore | 4 +- SKILL/SKILL.md | 67 +++++++++++++ SKILL/references/benchmark.md | 134 ++++++++++++++++++++++++++ SKILL/references/config.md | 174 ++++++++++++++++++++++++++++++++++ SKILL/references/optimize.md | 101 ++++++++++++++++++++ SKILL/references/setup.md | 145 ++++++++++++++++++++++++++++ 6 files changed, 624 insertions(+), 1 deletion(-) create mode 100644 SKILL/SKILL.md create mode 100644 SKILL/references/benchmark.md create mode 100644 SKILL/references/config.md create mode 100644 SKILL/references/optimize.md create mode 100644 SKILL/references/setup.md diff --git a/.gitignore b/.gitignore index 4c5c2be..84361ce 100644 --- a/.gitignore +++ b/.gitignore @@ -48,8 +48,10 @@ coverage/ tmp/ temp/ -# Superpowers planning artifacts +# Superpowers planning artifacts and internal dev docs docs/superpowers/ +docs/plans/ +docs/specs/ # Skill-optimizer generated artifacts .skill-optimizer/ diff --git a/SKILL/SKILL.md b/SKILL/SKILL.md new file mode 100644 index 0000000..d81afd1 --- /dev/null +++ b/SKILL/SKILL.md @@ -0,0 +1,67 @@ +--- +name: skill-optimizer +description: > + Benchmark and optimize SDK, CLI, and MCP documentation so every LLM model + can reliably call the right actions with correct arguments. Use when setting + up skill-optimizer for a project, running benchmarks, interpreting results, + optimizing SKILL.md files, or diagnosing configuration issues. Also use when + working inside the skill-optimizer repository itself — for running against + mock repos, testing changes, or understanding the codebase. +--- + +# skill-optimizer + +Benchmark your SDK / CLI / MCP docs against multiple LLMs, measure whether they call the right actions with the right arguments, and iteratively rewrite your guidance until a quality floor is met across every model. + +## Context Detection + +Before doing anything, figure out where you are: + +1. **Look for `skill-optimizer.json`** (in CWD or parent directories). If found, you are in a **configured target project**. Use that file path as `` in all commands below. + +2. **Look for `src/cli.ts` and a `package.json` with `"name": "skill-optimizer"`**. If found, you are in the **optimizer repo itself**. You can use dev commands directly (`npm run build`, `npm test`, `npx tsx src/cli.ts`). To benchmark a target, either use the mock repos in `mock-repos/` or point `--config` at an external project's config. + +3. **Neither found** — you are in an **unconfigured target project**. Read `references/setup.md` to scaffold a config before proceeding. + +## Quick Reference + +| Task | Command | +|------|---------| +| Init config | `npx skill-optimizer init cli\|sdk\|mcp` | +| Init (non-interactive) | `npx skill-optimizer init cli --yes` | +| Import CLI commands | `npx skill-optimizer import-commands --from ./src/cli.ts` | +| Import (binary scrape) | `npx skill-optimizer import-commands --from my-cli --scrape` | +| Diagnose config | `npx skill-optimizer doctor --config ` | +| Auto-fix config | `npx skill-optimizer doctor --fix --config ` | +| Dry run (no LLM calls) | `npx skill-optimizer run --dry-run --config ` | +| Run benchmark | `npx skill-optimizer run --config ` | +| Generate tasks only | `npx skill-optimizer generate-tasks --config ` | +| Run optimizer | `npx skill-optimizer optimize --config ` | +| Compare two runs | `npx skill-optimizer compare --baseline a.json --current b.json` | + +`` is the path to your `skill-optimizer.json` — typically `./skill-optimizer/skill-optimizer.json` after running `init`, or wherever you placed it. + +## What Do You Need? + +Read the reference file that matches your current goal: + +| Goal | Reference | +|------|-----------| +| Set up skill-optimizer for a project (first time) | Read `references/setup.md` | +| Run a benchmark or understand results | Read `references/benchmark.md` | +| Automatically optimize a SKILL.md | Read `references/optimize.md` | +| Understand config options | Read `references/config.md` | + +If you are in an **unconfigured project** (context detection case 3), start with `references/setup.md`. + +## Key Concepts + +**Surfaces** — The callable interface of your project: SDK methods, CLI commands, or MCP tools. Skill-optimizer discovers these via tree-sitter code analysis or from a hand-written manifest file. + +**Static evaluation** — Benchmark evaluation never executes generated code. Actions are extracted from model responses via pattern matching and compared structurally against expected calls. This makes benchmarks safe and repeatable. + +**Verdict gates** — Two thresholds must both pass for a benchmark to receive a PASS verdict: `perModelFloor` (each model individually meets a minimum score) and `targetWeightedAverage` (the weighted mean across all models meets a target). A single model below the floor fails the entire run. + +**Safety boundary** — The optimizer never modifies your original SKILL.md. It creates versioned copies in `.skill-optimizer/skill-v{N}.md` and only accepts mutations that improve scores without dropping any model below the floor. It does not modify tracked source files, but the generated artifacts appear under `.skill-optimizer/` — add that directory to your `.gitignore`. + +**LLM routing** — By default (`format: "pi"`), all benchmark calls route through [OpenRouter](https://openrouter.ai) and need `OPENROUTER_API_KEY`. You can also call providers directly: `format: "anthropic"` uses the Anthropic API directly (`ANTHROPIC_API_KEY`), and `format: "openai"` uses the OpenAI API directly (`OPENAI_API_KEY`), with optional Codex browser-login auth via `authMode: "codex"`. The model ID prefix must match the format — see `references/config.md` for the full mapping. diff --git a/SKILL/references/benchmark.md b/SKILL/references/benchmark.md new file mode 100644 index 0000000..51a54f7 --- /dev/null +++ b/SKILL/references/benchmark.md @@ -0,0 +1,134 @@ +# Running & Interpreting Benchmarks + +This guide covers running benchmarks, reading results, diagnosing failures, and comparing runs. + +## 1. Pre-flight Check + +Before running a benchmark, verify: + +```bash +# Config is valid +npx skill-optimizer doctor --config + +# API key is set +echo $OPENROUTER_API_KEY # should print sk-or-... + +# Git is clean (if requireCleanGit is true, which is the default) +git status # should show "nothing to commit, working tree clean" +``` + +## 2. Dry Run First + +Always start with a dry run to check scope and estimate cost: + +```bash +npx skill-optimizer run --dry-run --config +``` + +This shows: +- How many actions were discovered +- How many are in scope after filtering +- How many tasks would be generated +- Which models would be called + +No LLM calls are made. Use this to verify your scope and estimate cost (N models x M tasks = total calls). + +## 3. Run the Benchmark + +```bash +npx skill-optimizer run --config +``` + +What happens at each stage: + +1. **Discover** — find callable actions via tree-sitter or manifest +2. **Scope** — apply `include`/`exclude` filters +3. **Generate tasks** — create one prompt per in-scope action (coverage-guaranteed: every action gets at least one task) +4. **Call models** — each configured model attempts each task +5. **Extract** — pull action calls from model responses via pattern matching +6. **Evaluate** — compare extracted actions against expected actions +7. **Verdict** — PASS or FAIL based on two gates + +## 4. Reading the Output + +The benchmark produces: + +- **Per-model score table** — each model's pass rate as a fraction (e.g., `Claude Sonnet: 18/20 (0.90)`) +- **Weighted average** — computed from individual scores and model weights +- **Verdict** — `PASS` (both gates satisfied) or `FAIL` (at least one gate missed) +- **Exit code** — `0` for PASS, `1` for FAIL + +## 5. Verdict Gates + +Two gates must **both** pass for a PASS verdict: + +**`perModelFloor`** (default: `0.6`) +Every model must individually score at or above this threshold. If any single model scores below, the entire benchmark fails — regardless of how well other models did. This prevents one weak model from hiding behind a strong average. + +**`targetWeightedAverage`** (default: `0.7`) +The weighted mean across all models must reach this threshold. Models with higher `weight` values count more. This ensures overall quality, not just per-model minimums. + +**Model `weight`** (default: `1.0`) +Controls how much each model influences the weighted average. Set flagship models to `2.0` and budget models to `0.5` if you care more about flagship performance. + +## 6. Diagnosing Failures + +When a benchmark fails, look at the per-task breakdown to identify patterns: + +**Hallucinated actions** — the model calls functions that don't exist in your API. +- *Cause:* SKILL.md describes features ambiguously or mentions non-existent methods +- *Fix:* Tighten your docs. Remove references to deprecated methods. Be explicit about what exists. + +**Missing arguments** — the model calls the right action but with wrong or missing arguments. +- *Cause:* Documentation doesn't clearly specify required parameters or their types +- *Fix:* Add explicit parameter sections with types, defaults, and examples + +**Wrong tool selection** — the model calls a related but incorrect action (e.g., `deleteTask` instead of `removeTask`). +- *Cause:* Action names are ambiguous or the docs don't distinguish between similar actions +- *Fix:* Add disambiguation notes or rename actions to be more distinct + +**One model fails, others pass** — a specific model consistently underperforms. +- *Cause:* That model may need more explicit guidance or has known weaknesses with your API style +- *Fix:* Consider adjusting its `weight`, adding model-specific notes to your docs, or accepting the floor as-is + +## 7. Comparing Runs + +After making changes to your SKILL.md, compare before and after: + +```bash +npx skill-optimizer compare --baseline report-before.json --current report-after.json +``` + +This shows: +- Per-model score deltas (e.g. `Claude Sonnet: 0.75 → 0.90 (+0.15)`) +- Per-task deltas — which tasks improved, which regressed +- Overall weighted average change + +**Finding the report files:** The benchmark writes its report JSON to the `output.dir` configured in your `skill-optimizer.json` (default: `benchmark-results/`). Each run creates a timestamped file there. + +## 8. Cost Awareness + +Each benchmark run makes `N models x M tasks` LLM calls. To minimize cost while iterating: + +- **Start narrow** — use `scope.include` to benchmark only your most important actions first +- **Few models first** — start with 2-3 models, expand after the skill stabilizes +- **Dry run** — always check scope size with `--dry-run` before committing to a full run +- **Iterate on docs first** — fix obvious SKILL.md gaps before re-running. Each run costs real money. + +## 9. CI Integration + +The exit code (`0` = PASS, `1` = FAIL) makes skill-optimizer suitable for CI pipelines: + +```bash +# In a CI script or Makefile +npx skill-optimizer run --config +# Exits 0 on PASS, 1 on FAIL — use as a gate step +``` + +This lets you catch regressions in documentation quality as part of your CI workflow. + +## Next Steps + +If the benchmark fails and the issues are scattered (not one obvious fix), read `references/optimize.md` to run the automatic optimization loop. + +If you need to adjust config (models, scope, thresholds), read `references/config.md`. diff --git a/SKILL/references/config.md b/SKILL/references/config.md new file mode 100644 index 0000000..a91f555 --- /dev/null +++ b/SKILL/references/config.md @@ -0,0 +1,174 @@ +# Configuration Reference + +Complete reference for `skill-optimizer.json`. For auto-generated schema docs, see `docs/reference/config-schema.md` in the skill-optimizer repo. + +## Minimal Working Configs + +### CLI surface + +```json +{ + "name": "my-cli-tool", + "target": { + "surface": "cli", + "repoPath": "/path/to/my-project", + "skill": "./SKILL.md", + "discovery": { + "mode": "auto", + "sources": ["src/cli.ts"] + } + }, + "benchmark": { + "format": "pi", + "models": [ + { "id": "openrouter/anthropic/claude-sonnet-4-6", "name": "Claude Sonnet", "tier": "flagship" } + ] + } +} +``` + +### SDK surface + +```json +{ + "name": "my-sdk", + "target": { + "surface": "sdk", + "repoPath": "/path/to/my-sdk", + "skill": "./SKILL.md", + "discovery": { + "mode": "auto", + "sources": ["src/index.ts"], + "language": "typescript" + } + }, + "benchmark": { + "format": "pi", + "models": [ + { "id": "openrouter/anthropic/claude-sonnet-4-6", "name": "Claude Sonnet", "tier": "flagship" } + ] + } +} +``` + +### MCP surface + +```json +{ + "name": "my-mcp-server", + "target": { + "surface": "mcp", + "repoPath": "/path/to/my-mcp-server", + "skill": "./SKILL.md", + "discovery": { + "mode": "auto", + "sources": ["src/server.ts"] + } + }, + "benchmark": { + "format": "pi", + "models": [ + { "id": "openrouter/anthropic/claude-sonnet-4-6", "name": "Claude Sonnet", "tier": "flagship" } + ] + } +} +``` + +## Field-by-Field Reference + +### `target` — What You're Benchmarking + +| Field | Required | Default | Description | +|-------|----------|---------|-------------| +| `surface` | Yes | — | `"cli"`, `"sdk"`, or `"mcp"` | +| `repoPath` | Yes | — | Absolute or config-relative path to your project root | +| `skill` | Yes | — | Path to your SKILL.md or guidance doc, relative to `repoPath` | +| `discovery.mode` | No | `"auto"` | `"auto"` (tree-sitter) or `"manifest"` (hand-written JSON) | +| `discovery.sources` | No | `[]` | Source files for tree-sitter to parse, relative to `repoPath` | +| `discovery.language` | No | — | SDK only: `"typescript"`, `"python"`, or `"rust"` | +| `scope.include` | No | `["*"]` | Glob patterns for actions to include | +| `scope.exclude` | No | `[]` | Glob patterns for actions to exclude | + +### `benchmark` — How to Test + +| Field | Required | Default | Description | +|-------|----------|---------|-------------| +| `format` | No | `"pi"` | `"pi"` — route through OpenRouter (default); `"openai"` — call OpenAI API directly; `"anthropic"` — call Anthropic API directly | +| `authMode` | No | `"env"` | `"env"` — read key from env var (default); `"codex"` — read from `~/.codex/auth.json` (OpenAI only); `"auto"` — try env first, fall back to codex for OpenAI | +| `apiKeyEnv` | No | provider default | Env var holding the API key. Defaults: `OPENROUTER_API_KEY` for `pi`, `OPENAI_API_KEY` for `openai`, `ANTHROPIC_API_KEY` for `anthropic` | +| `baseUrl` | No | — | Override the API base URL (e.g. for a custom OpenAI-compatible endpoint) | +| `models[].id` | Yes | — | Model ID with provider prefix: `openrouter/

/` (OpenRouter), `anthropic/` (direct Anthropic), `openai/` (direct OpenAI). Use hyphens in version segments for `openrouter/` and `anthropic/` (e.g. `claude-sonnet-4-6`, `gemini-2-5-flash`). | +| `models[].name` | No | — | Human-readable label for output tables | +| `models[].tier` | No | — | `"flagship"`, `"mid"`, or `"low"` (informational only) | +| `models[].weight` | No | `1.0` | Influence on weighted average (higher = counts more) | +| `verdict.perModelFloor` | No | `0.6` | Minimum score each model must reach individually | +| `verdict.targetWeightedAverage` | No | `0.7` | Minimum weighted average across all models | +| `taskGeneration.enabled` | No | `true` | Whether to auto-generate tasks | +| `taskGeneration.maxTasks` | No | `20` | Upper bound on tasks (must be >= in-scope action count) | +| `taskGeneration.outputDir` | No | `".skill-optimizer"` | Where to write task artifacts | + +### `optimize` — How to Improve + +| Field | Required | Default | Description | +|-------|----------|---------|-------------| +| `enabled` | No | `true` | Whether optimization is allowed | +| `mode` | No | `"stable-surface"` | `"stable-surface"` (reuse tasks) or `"surface-changing"` (regenerate per iteration) | +| `model` | No | `"openrouter/anthropic/claude-opus-4-6"` | Which LLM writes mutations | +| `maxIterations` | No | `5` | Maximum optimization rounds | +| `minImprovement` | No | `0.02` | Minimum delta in weighted average required to accept a mutation | +| `allowedPaths` | No | `["SKILL.md"]` | Files the mutation agent may edit | +| `requireCleanGit` | No | `true` | Block optimizer if target repo has uncommitted changes | + +## Model ID and Auth Quick Guide + +The `benchmark.format` field controls which API receives requests. Pick the combination that matches your setup: + +| Format | Model ID prefix | Required env var | Notes | +|--------|----------------|------------------|-------| +| `"pi"` (default) | `openrouter//` | `OPENROUTER_API_KEY` | All providers via OpenRouter | +| `"openai"` | `openai/` | `OPENAI_API_KEY` | Direct OpenAI API | +| `"openai"` + `authMode: "codex"` | `openai/` | `~/.codex/auth.json` | Codex browser-login auth | +| `"anthropic"` | `anthropic/` | `ANTHROPIC_API_KEY` | Direct Anthropic API | + +**Codex auth** (`authMode: "codex"`) reads credentials from `~/.codex/auth.json` in this priority order: +1. `tokens.access_token` — browser-login JWT (checked for expiry) +2. `tokens.OPENAI_API_KEY` — static key nested under tokens +3. `OPENAI_API_KEY` — root-level static key + +Codex auth only works with the OpenAI provider. For `openrouter/` or `anthropic/` models, use `authMode: "env"`. + +**`authMode: "auto"`** tries the env var first; for OpenAI models, falls back to `~/.codex/auth.json` if the env var is unset. + +## Model Configuration Tips + +- Browse available models at [openrouter.ai/models](https://openrouter.ai/models) +- **Recommended starter set:** one flagship (Claude Sonnet or GPT-4o) + one budget model (Gemini Flash or Haiku) to test both capability ends +- **Weighting strategy:** set flagship models to `weight: 2.0` and budget to `weight: 0.5` if flagship performance matters most to you +- `tier` is informational only — it appears in output tables but doesn't affect scoring + +## Scope Patterns + +The `*` wildcard matches any sequence of characters, including dots and slashes. It is not limited to a single path segment like filesystem globs. + +| Pattern | Matches | +|---------|---------| +| `"Wallet.*"` | All Wallet methods (`Wallet.create`, `Wallet.balance`, etc.) | +| `"*.internal*"` | Anything with "internal" in the name | +| `"get_*"` | Only getter actions | +| `["create_*", "update_*", "delete_*"]` | Only mutation actions | + +Task generation is **coverage-guaranteed**: every in-scope action gets at least one task. If coverage fails after retries, an error names the uncovered actions and suggests either fixing SKILL.md guidance or excluding them. + +## Common Error Codes + +| Code | Meaning | Fix | +|------|---------|-----| +| `E_MISSING_SKILL` | `target.skill` file not found | Create the file or fix the path in config | +| `E_INVALID_SURFACE` | `target.surface` is not cli/sdk/mcp | Use one of the three valid values | +| `E_DIRTY_GIT` | Uncommitted changes in target repo | Commit or stash, or set `requireCleanGit: false` | +| `E_EMPTY_SCOPE` | Scope filters matched no actions | Check your `include`/`exclude` patterns | +| `E_MISSING_API_KEY` | API key env var not set | `export OPENROUTER_API_KEY=sk-or-...` (or `OPENAI_API_KEY` / `ANTHROPIC_API_KEY` for direct formats) | + +Full error reference with detailed descriptions: `docs/reference/errors.md` + +Full config schema reference (auto-generated from Zod): `docs/reference/config-schema.md` diff --git a/SKILL/references/optimize.md b/SKILL/references/optimize.md new file mode 100644 index 0000000..b54675d --- /dev/null +++ b/SKILL/references/optimize.md @@ -0,0 +1,101 @@ +# Optimization Loop + +This guide covers when and how to use the automatic optimizer, how to interpret its results, and what to do when it doesn't converge. + +## 1. When to Optimize vs. Fix Manually + +**Fix manually** when the benchmark reveals a clear, localized problem — a missing section, a wrong example, an outdated method name. Manual fixes are faster and more precise for known issues. + +**Run the optimizer** when failures are scattered across multiple models and tasks with no obvious single fix. The optimizer systematically tries mutations to your SKILL.md and keeps only changes that improve scores. + +A good workflow: run a benchmark, fix the obvious stuff by hand, re-benchmark, then let the optimizer handle whatever's left. + +## 2. How the Loop Works + +1. **Baseline benchmark** — establish starting scores for all models +2. **Copy** — your SKILL.md is copied to `.skill-optimizer/skill-v0.md` (original is never touched) +3. **Failure analysis** — identify patterns in what models get wrong +4. **Mutation** — a mutation agent (powered by `optimize.model`, defaults to Claude Opus via OpenRouter) proposes edits to the versioned copy +5. **Re-benchmark** — run all models against all tasks using the mutated skill +6. **Accept or reject** — the mutation is accepted only if: + - The weighted average improves by at least `minImprovement` + - No model that was above the floor drops below it +7. **Rollback** if rejected — revert to the previous version +8. **Repeat** up to `maxIterations` times +9. **Progress table** — final output shows Baseline -> each iteration -> Final -> delta per model + +## 3. Safety Guarantees + +The optimizer is designed to be safe to run: + +- **Your original SKILL.md is never modified.** All edits happen on versioned copies in `.skill-optimizer/skill-v0.md`, `skill-v1.md`, etc. +- **`requireCleanGit`** is enforced by default — the optimizer won't run if your target repo has uncommitted changes +- **`allowedPaths`** constrains which files the mutation agent can edit (defaults to just the skill file) +- **Stabilization window** prevents oscillation — if the same mutation keeps getting accepted and rejected, the optimizer exits early + +## 4. Running the Optimizer + +```bash +npx skill-optimizer optimize --config +``` + +Output during the run: +- Current iteration number and total +- Per-model scores after each mutation attempt +- Accept/reject decision with reasoning +- Running progress table + +The optimizer can take several minutes per iteration (it runs a full benchmark each time). + +## 5. Key Config Knobs + +| Setting | Default | What it controls | +|---------|---------|------------------| +| `optimize.maxIterations` | `5` | Upper bound on optimization rounds | +| `optimize.mode` | `"stable-surface"` | `"stable-surface"`: reuse tasks across iterations (faster, apples-to-apples). `"surface-changing"`: regenerate tasks each iteration (if skill changes might affect task phrasing) | +| `optimize.model` | `"openrouter/anthropic/claude-opus-4.6"` | Which LLM writes mutations | +| `optimize.enabled` | `true` | Set to `false` to skip optimization (useful in CI) | +| `optimize.requireCleanGit` | `true` | Block optimizer if target repo has uncommitted changes | + +## 6. Interpreting Results + +**Progress table** — rows are models, columns are iterations. Shows the score trajectory for each model across the optimization run. + +**Accepted iteration** — the mutation improved scores without violating either gate. The versioned copy advances to `skill-v{N+1}.md`. + +**Rejected iteration** — the mutation either didn't improve the weighted average enough, or it caused a model to drop below the floor. The previous version is kept and the optimizer tries a different mutation. + +**Early exit** — if scores plateau for consecutive iterations, the optimizer may stop before reaching `maxIterations`. This is normal and means further mutations aren't producing meaningful improvements. + +## 7. After Optimization + +The best version is the highest-numbered `skill-v{N}.md` in `.skill-optimizer/`. To apply it: + +```bash +# 1. See what changed +diff SKILL.md .skill-optimizer/skill-v3.md # adjust N to your highest version + +# 2. Review the diff — the optimizer is a tool, not an oracle +# Look for: overly specific examples, removed important context, awkward phrasing + +# 3. Copy it back +cp .skill-optimizer/skill-v3.md SKILL.md + +# 4. Commit +git add SKILL.md +git commit -m "docs: apply skill-optimizer improvements (v3)" +``` + +## 8. When It Doesn't Converge + +If the optimizer oscillates or plateaus without reaching your target scores: + +**Narrow the scope** — exclude actions that are inherently ambiguous or rarely used. A smaller, cleaner scope gives the optimizer more room to improve what matters. + +**Improve discovery** — make sure `discovery.sources` points at the right files. If the surface is incomplete (missing actions), the optimizer is working with bad data. + +**Manual intervention** — read the failure analysis output from the last iteration. It often reveals patterns that a targeted manual edit can fix more effectively than automated mutation. + +**Adjust gates** — if `perModelFloor` or `targetWeightedAverage` are set very high, lower them to something achievable first. Optimize to hit that floor, then ratchet up gradually. + +**Try different models** — change `optimize.model` to a different LLM. Different models have different strengths in rewriting documentation. diff --git a/SKILL/references/setup.md b/SKILL/references/setup.md new file mode 100644 index 0000000..e5a38b4 --- /dev/null +++ b/SKILL/references/setup.md @@ -0,0 +1,145 @@ +# Setup & Init + +This guide walks through setting up skill-optimizer for your project, from prerequisites to a verified configuration. + +## 1. Prerequisites + +Before starting, verify these three requirements: + +**Node.js 20+:** +```bash +node --version +# Expected: v20.x.x or higher +``` + +**API key** (which one depends on your `benchmark.format`): +```bash +# Default — OpenRouter (format: "pi"): +export OPENROUTER_API_KEY=sk-or-your-key-here + +# Direct OpenAI (format: "openai"): +export OPENAI_API_KEY=sk-your-key-here + +# Direct Anthropic (format: "anthropic"): +export ANTHROPIC_API_KEY=sk-ant-your-key-here +``` +If you're just getting started, use OpenRouter — one key covers all providers. + +**skill-optimizer available:** +```bash +npx skill-optimizer --help +# Expected: Usage information +# If not installed globally, install from the repo: +# cd /path/to/skill-optimizer && npm install && npm run build && npm link +``` + +## 2. Determine Your Surface Type + +skill-optimizer supports three surface types. Pick the one that matches your project: + +| Surface | Your project exposes... | Examples | +|---------|------------------------|----------| +| `cli` | CLI commands or a binary | Yargs, Commander, @optique/core, argparse, Click, Clap | +| `sdk` | Library methods users call in code | TypeScript/Python/Rust SDKs | +| `mcp` | MCP tool handlers | MCP servers with `server.tool()` definitions | + +If unsure: does your user run commands in a terminal (`cli`), import your package and call functions (`sdk`), or connect an AI agent to your tool server (`mcp`)? + +## 3. Run the Init Wizard + +From your project root: + +```bash +npx skill-optimizer init +# Example: npx skill-optimizer init cli +``` + +The wizard prompts for: + +- **Repo path** — absolute path to your project root (defaults to CWD) +- **Models** — model IDs to benchmark against (e.g., `openrouter/anthropic/claude-sonnet-4-6`) +- **SKILL.md location** — path to your existing documentation or guidance file +- **Discovery sources** — source files for tree-sitter to parse (e.g., `src/cli.ts`, `src/index.ts`) +- **Max tasks** — upper bound on generated benchmark tasks (default: 20) + +**Non-interactive mode** (for CI or scripting): + +```bash +# Accept all defaults +npx skill-optimizer init cli --yes + +# Load answers from a file +npx skill-optimizer init --answers answers.json +``` + +`answers.json` format: +```json +{ + "surface": "cli", + "repoPath": "/absolute/path/to/your-repo", + "models": ["openrouter/anthropic/claude-sonnet-4-6", "openrouter/openai/gpt-4o"], + "maxTasks": 20, + "maxIterations": 5, + "entryFile": "src/cli.ts" +} +``` + +## 4. Surface Discovery + +After init, skill-optimizer needs to know what actions your project exposes. There are two discovery modes: + +**Code-first (auto)** — tree-sitter parses your source files automatically. This works for: +- TypeScript: Yargs, Commander, @optique/core CLI frameworks +- TypeScript/Python/Rust: SDK method extraction +- TypeScript: MCP `server.tool()` definitions + +If auto-discovery finds your actions, you're done. Check with: +```bash +npx skill-optimizer run --dry-run --config +# Look for "Discovered N actions" in the output +``` + +**Manual / import** — if auto-discovery yields nothing or misses actions: + +```bash +# Extract from TypeScript source +npx skill-optimizer import-commands --from ./src/cli.ts + +# Extract from a compiled binary's help text +npx skill-optimizer import-commands --from my-cli --scrape +``` + +This populates `.skill-optimizer/cli-commands.json` (CLI) or `.skill-optimizer/tools.json` (MCP). You can also edit these manifest files by hand. + +## 5. Verify with Doctor + +Run the config diagnostics to catch problems early: + +```bash +npx skill-optimizer doctor --config +``` + +If issues are found, auto-fix what's fixable: + +```bash +npx skill-optimizer doctor --fix --config +``` + +## 6. What You Should Have Now + +After successful setup: + +- **`skill-optimizer.json`** — main config file (commit this); when created by `init`, the default location is `./skill-optimizer/skill-optimizer.json` +- **`.skill-optimizer/`** — working directory for task artifacts, surface manifests, and versioned skill copies (gitignored) + +Your project is ready for benchmarking. Read `references/benchmark.md` for next steps. + +## 7. Common Pitfalls + +| Problem | Cause | Fix | +|---------|-------|-----| +| "Config not found" | Wrong path to `skill-optimizer.json` | Use `--config` with the full path | +| "No actions discovered" | `discovery.sources` points at wrong files | Check paths are relative to `repoPath` | +| "Skill file not found" | `target.skill` path is wrong | Path is relative to `repoPath` — verify it exists | +| "Unknown config format" | File is named `skill-benchmark.json` | Rename to `skill-optimizer.json` | +| "repoPath not found" | Relative path resolved wrong | Use absolute path, or make it relative to config file location | From 382792a18dcf5b35692141ac45f7d778eb54c6d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Selser?= Date: Thu, 16 Apr 2026 00:41:08 -0400 Subject: [PATCH 5/7] feat(prompt): add prompt surface type for skill/template benchmarking (#20) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * docs: fix repo URL in CONTRIBUTING.md (bucurdavid → fastxyz) Fixes #18 * feat(prompt): add prompt surface type for skill/template benchmarking Discovers phases, instructions, output formats, and decision points from markdown skill files. Evaluates model responses with content-based criteria (required sections, format patterns, keywords, structural elements) instead of tool-call matching. New files: - src/project/discover-prompt.ts — markdown parser for capabilities - src/benchmark/prompt-evaluator.ts — content-based scorer (weighted) - src/discovery/prompt.ts — file-based discoverer + types - tests/smoke-discovery-prompt.ts — 10 tests - tests/smoke-prompt-evaluator.ts — 15 tests - tests/fixtures/sample-skill.md — 3-phase sample Modified: types.ts, validate.ts, snapshot.ts, runner.ts, evaluator.ts, extractors/index.ts, prompts.ts, cli.ts, init/, CHANGELOG.md, README.md * fix(prompt): guard snapshot surface, fix format inconsistency, remove dead interface, fix stale comment * fix(discovery): anchor frontmatter closing delimiter to line boundary Replace indexOf('---', 3) with a regex that matches --- only at the start of a line, preventing early termination when a YAML value contains ---. Co-Authored-By: Claude Sonnet 4.6 * fix(discover-prompt): clear preamble on first heading, strip frontmatter before section split * fix(discover-prompt): fallback to whole-content instruction extraction for heading-less files Co-Authored-By: Claude Sonnet 4.6 * test(discover-prompt): document interface difference between production and standalone paths Co-Authored-By: Claude Sonnet 4.6 * fix(prompt): wire evaluatePromptResponse into runner, expose section text, fix generate schema Co-Authored-By: Claude Sonnet 4.6 * fix(prompt): unique temp dir + cleanup guard in test, enforce empty actions in grounding, bypass maxTasks constraint for prompt Co-Authored-By: Claude Sonnet 4.6 --------- Co-authored-by: OpenClaw Agent (basd) Co-authored-by: Claude Sonnet 4.6 --- CHANGELOG.md | 3 + CONTRIBUTING.md | 2 +- README.md | 18 +- package.json | 2 +- src/actions/snapshot.ts | 4 +- src/actions/types.ts | 2 +- src/benchmark/evaluator.ts | 2 +- src/benchmark/extractors/index.ts | 7 +- src/benchmark/init.ts | 17 +- src/benchmark/prompt-evaluator.ts | 394 ++++++++++++++++++++++++++++++ src/benchmark/prompts.ts | 15 +- src/benchmark/runner.ts | 112 ++++++--- src/benchmark/types.ts | 2 +- src/cli.ts | 11 +- src/discovery/index.ts | 7 + src/discovery/prompt.ts | 180 ++++++++++++++ src/init/answers.ts | 8 +- src/init/scaffold.ts | 13 + src/init/wizard.ts | 5 +- src/project/discover-prompt.ts | 369 ++++++++++++++++++++++++++++ src/project/snapshot.ts | 34 ++- src/project/validate.ts | 10 +- src/tasks/generate.ts | 27 ++ src/tasks/ground.ts | 11 +- src/tasks/index.ts | 32 ++- tests/fixtures/sample-skill.md | 72 ++++++ tests/smoke-discovery-prompt.ts | 321 ++++++++++++++++++++++++ tests/smoke-prompt-evaluator.ts | 349 ++++++++++++++++++++++++++ tests/smoke-snapshot-prompt.ts | 25 ++ 29 files changed, 1979 insertions(+), 75 deletions(-) create mode 100644 src/benchmark/prompt-evaluator.ts create mode 100644 src/discovery/prompt.ts create mode 100644 src/project/discover-prompt.ts create mode 100644 tests/fixtures/sample-skill.md create mode 100644 tests/smoke-discovery-prompt.ts create mode 100644 tests/smoke-prompt-evaluator.ts create mode 100644 tests/smoke-snapshot-prompt.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 7fa08dc..2cc6331 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,9 @@ ## Unreleased +### Added +- **prompt surface type** — benchmark and optimize prompt templates, Claude Code skills, and agent instructions. Discovers phases and capabilities from markdown, evaluates output quality with content-based criteria. + ### Fixed - **benchmark:** Strip provider prefix from model ID when using direct `anthropic` or `openai` formats. Previously, `anthropic/claude-sonnet-4-6` was sent as-is to the Anthropic API, which expects `claude-sonnet-4-6`. The `pi` format is unaffected. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6d285b5..c894b5a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -5,7 +5,7 @@ Thanks for contributing! This project is a small, opinionated tool — changes s ## Local workflow ```bash -git clone https://github.com/bucurdavid/skill-optimizer +git clone https://github.com/fastxyz/skill-optimizer cd skill-optimizer npm install npm run typecheck diff --git a/README.md b/README.md index ab85d61..300654a 100644 --- a/README.md +++ b/README.md @@ -113,9 +113,25 @@ npx skill-optimizer init --answers answers.json | `benchmark.models` | Models to benchmark | Valid [OpenRouter](https://openrouter.ai/models) model IDs | | `benchmark.authMode` | How model auth is resolved | `env` (default), `codex`, or `auto` | +### Prompt templates / Claude Code skills + +Benchmark how well models follow your prompt templates: + +```bash +skill-optimizer init prompt +skill-optimizer benchmark +``` + +The prompt surface discovers phases and capabilities from your SKILL.md, +generates scenario-based tasks, and evaluates output quality — not just +tool calls. It scores responses on required sections, format patterns, +forbidden keywords, and structural elements (code blocks, numbered lists, +tables). This lets you optimize prompt templates the same way you optimize +SDK/CLI/MCP guidance. + ## How it works -1. **Discover** callable surface (SDK methods / CLI commands / MCP tools) via tree-sitter or a manifest. +1. **Discover** callable surface (SDK methods / CLI commands / MCP tools / prompt phases) via tree-sitter, manifest, or markdown parsing. 2. **Scope** the surface with `target.scope.include` / `target.scope.exclude` globs. 3. **Generate tasks** — one prompt per in-scope action, coverage-guaranteed. 4. **Benchmark** — every configured model attempts every task; static evaluator checks action calls + args. diff --git a/package.json b/package.json index 1da30c1..5bc6f96 100644 --- a/package.json +++ b/package.json @@ -48,7 +48,7 @@ "gen-docs": "tsx scripts/gen-docs.ts", "build": "tsc && npm run gen-docs && chmod +x dist/cli.js", "typecheck": "tsc --noEmit", - "test": "tsx tests/smoke-code.ts && tsx tests/smoke-sdk-python.ts && tsx tests/smoke-sdk-rust.ts && tsx tests/smoke-cli.ts && tsx tests/smoke-cli-entry.ts && tsx tests/smoke-mcp.ts && tsx tests/smoke-llm.ts && tsx tests/smoke-discovery-sdk.ts && tsx tests/smoke-discovery-cli.ts && tsx tests/smoke-discovery-mcp.ts && tsx tests/smoke-generation.ts && tsx tests/smoke-optimize.ts && tsx tests/smoke-mock-repos.ts && tsx tests/smoke-release.ts && tsx tests/smoke-scoring.ts && tsx tests/smoke-scope.ts && tsx tests/smoke-coverage.ts && tsx tests/smoke-feedback.ts && tsx tests/smoke-verdict.ts && tsx tests/smoke-dry-run.ts && tsx tests/smoke-errors.ts && tsx tests/smoke-e2e.ts && tsx tests/smoke-import.ts && tsx tests/smoke-init.ts && tsx tests/smoke-gen-docs.ts" + "test": "tsx tests/smoke-code.ts && tsx tests/smoke-sdk-python.ts && tsx tests/smoke-sdk-rust.ts && tsx tests/smoke-cli.ts && tsx tests/smoke-cli-entry.ts && tsx tests/smoke-mcp.ts && tsx tests/smoke-llm.ts && tsx tests/smoke-discovery-sdk.ts && tsx tests/smoke-discovery-cli.ts && tsx tests/smoke-discovery-mcp.ts && tsx tests/smoke-discovery-prompt.ts && tsx tests/smoke-prompt-evaluator.ts && tsx tests/smoke-snapshot-prompt.ts && tsx tests/smoke-generation.ts && tsx tests/smoke-optimize.ts && tsx tests/smoke-mock-repos.ts && tsx tests/smoke-release.ts && tsx tests/smoke-scoring.ts && tsx tests/smoke-scope.ts && tsx tests/smoke-coverage.ts && tsx tests/smoke-feedback.ts && tsx tests/smoke-verdict.ts && tsx tests/smoke-dry-run.ts && tsx tests/smoke-errors.ts && tsx tests/smoke-e2e.ts && tsx tests/smoke-import.ts && tsx tests/smoke-init.ts && tsx tests/smoke-gen-docs.ts" }, "dependencies": { "@clack/prompts": "^1.2.0", diff --git a/src/actions/snapshot.ts b/src/actions/snapshot.ts index b8009e3..df0ac45 100644 --- a/src/actions/snapshot.ts +++ b/src/actions/snapshot.ts @@ -171,8 +171,8 @@ export function loadActionSnapshotFile(snapshotPath: string): ActionSnapshotArti } const catalog = candidate.catalog as Partial; - if (catalog.surface !== 'sdk' && catalog.surface !== 'cli' && catalog.surface !== 'mcp') { - invalidSnapshot(snapshotPath, 'catalog.surface must be one of sdk|cli|mcp'); + if (catalog.surface !== 'sdk' && catalog.surface !== 'cli' && catalog.surface !== 'mcp' && catalog.surface !== 'prompt') { + invalidSnapshot(snapshotPath, 'catalog.surface must be one of sdk|cli|mcp|prompt'); } const validatedActions = validateCatalogActions(snapshotPath, catalog.actions); diff --git a/src/actions/types.ts b/src/actions/types.ts index ed5a2c5..1ed78c4 100644 --- a/src/actions/types.ts +++ b/src/actions/types.ts @@ -1,4 +1,4 @@ -export type ActionSurface = 'sdk' | 'cli' | 'mcp'; +export type ActionSurface = 'sdk' | 'cli' | 'mcp' | 'prompt'; export interface ActionArgSchema { name: string; diff --git a/src/benchmark/evaluator.ts b/src/benchmark/evaluator.ts index 693b96a..aa6fe62 100644 --- a/src/benchmark/evaluator.ts +++ b/src/benchmark/evaluator.ts @@ -418,7 +418,7 @@ export function evaluateTask(params: { error?: string; knownMethods: Set; bindings?: Map; // variable → source function/class from extractor - surface: 'sdk' | 'cli' | 'mcp'; + surface: 'sdk' | 'cli' | 'mcp' | 'prompt'; }): TaskResult { const { task, diff --git a/src/benchmark/extractors/index.ts b/src/benchmark/extractors/index.ts index 355f677..9629b19 100644 --- a/src/benchmark/extractors/index.ts +++ b/src/benchmark/extractors/index.ts @@ -16,7 +16,7 @@ export async function extract( config: BenchmarkConfig, ): Promise<{ calls: ExtractedCall[]; generatedCode: string | null; bindings?: Map }> { const extended = config as BenchmarkConfig & { - surface?: 'sdk' | 'cli' | 'mcp'; + surface?: 'sdk' | 'cli' | 'mcp' | 'prompt'; mode?: 'code' | 'mcp'; sdk?: unknown; cli?: { commandDefinitions?: Array<{ command: string }> }; @@ -28,6 +28,11 @@ export async function extract( ? extended.cli.commandDefinitions.map((definition) => definition.command) : undefined; + if (surface === 'prompt') { + // Prompt surface: no extraction — response is plain text, not tool calls or code. + return { calls: [], generatedCode: null }; + } + if (surface === 'mcp' || extended.mode === 'mcp') { const calls = extractFromToolCalls(response); return { calls, generatedCode: null }; diff --git a/src/benchmark/init.ts b/src/benchmark/init.ts index 4c0b0e9..e7f2997 100644 --- a/src/benchmark/init.ts +++ b/src/benchmark/init.ts @@ -1,7 +1,7 @@ import { writeFileSync, existsSync, mkdirSync } from 'node:fs'; import { resolve } from 'node:path'; -export function initBenchmark(targetDir: string = process.cwd(), surface: 'sdk' | 'cli' | 'mcp' = 'sdk'): void { +export function initBenchmark(targetDir: string = process.cwd(), surface: 'sdk' | 'cli' | 'mcp' | 'prompt' = 'sdk'): void { const generatedDir = resolve(targetDir, '.skill-optimizer'); mkdirSync(generatedDir, { recursive: true }); @@ -108,7 +108,7 @@ export function initBenchmark(targetDir: string = process.cwd(), surface: 'sdk' console.log(' 3. Run: skill-optimizer optimize --config ./.skill-optimizer/skill-optimizer.json'); } -function buildConfig(surface: 'sdk' | 'cli' | 'mcp'): object { +function buildConfig(surface: 'sdk' | 'cli' | 'mcp' | 'prompt'): object { const commonBenchmark = { format: 'pi', timeout: 240000, @@ -175,6 +175,19 @@ function buildConfig(surface: 'sdk' | 'cli' | 'mcp'): object { }; } + if (surface === 'prompt') { + return { + name: 'my-prompt', + target: { + surface: 'prompt', + repoPath: '..', + skill: '../SKILL.md', + }, + benchmark: commonBenchmark, + optimize: commonOptimize, + }; + } + // mcp return { name: 'my-mcp', diff --git a/src/benchmark/prompt-evaluator.ts b/src/benchmark/prompt-evaluator.ts new file mode 100644 index 0000000..0067f44 --- /dev/null +++ b/src/benchmark/prompt-evaluator.ts @@ -0,0 +1,394 @@ +import type { ActionDefinition } from '../actions/types.js'; + +// ── Types ───────────────────────────────────────────────────────────────────── + +export interface FormatPattern { + name: string; + pattern: string; +} + +export interface PromptEvaluationCriteria { + /** Required sections in the output (case-insensitive heading match). */ + requiredSections?: string[]; + /** Required format patterns (regex). */ + formatPatterns?: FormatPattern[]; + /** Minimum content length in characters. */ + minLength?: number; + /** Keywords that must appear (case-insensitive). */ + requiredKeywords?: string[]; + /** Keywords that must NOT appear (hallucination check, case-insensitive). */ + forbiddenKeywords?: string[]; + /** Structural checks. */ + hasCodeBlocks?: boolean; + hasNumberedList?: boolean; + hasTable?: boolean; +} + +export interface PromptCheckDetail { + check: string; + passed: boolean; + detail: string; +} + +export interface PromptEvaluationResult { + /** Overall score 0.0-1.0 (weighted across all criteria categories). */ + score: number; + /** Human-readable detail for each sub-check. */ + details: string[]; + /** Structured breakdown of individual checks. */ + checks: PromptCheckDetail[]; + /** Per-category scores before weighting. */ + categoryScores: { + sections: number; + format: number; + keywords: number; + structure: number; + }; +} + +// ── Weights ─────────────────────────────────────────────────────────────────── + +const WEIGHT_SECTIONS = 0.4; +const WEIGHT_FORMAT = 0.2; +const WEIGHT_KEYWORDS = 0.2; +const WEIGHT_STRUCTURE = 0.2; + +// ── Helpers ─────────────────────────────────────────────────────────────────── + +/** + * Build a regex that matches a markdown heading (any level) whose text + * contains the given section name. Also matches bold lines and plain + * uppercase lines that act as section headers. + */ +function sectionRegex(section: string): RegExp { + const escaped = section.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); + // Match: ## Section Name, **Section Name**, or SECTION NAME on its own line + return new RegExp( + `(?:^#{1,6}\\s+.*${escaped}.*$)|(?:^\\*\\*.*${escaped}.*\\*\\*$)|(?:^${escaped}\\s*$)`, + 'im', + ); +} + +function hasCodeBlock(text: string): boolean { + return /```[\s\S]*?```/.test(text); +} + +function hasNumberedList(text: string): boolean { + // At least two consecutive numbered items + return /(?:^|\n)\s*\d+[\.\)]\s+\S.*\n\s*\d+[\.\)]\s+\S/m.test(text); +} + +function hasTable(text: string): boolean { + // Markdown table: header row, separator row, at least one data row + return /\|.+\|[\r\n]+\|[\s:|-]+\|[\r\n]+\|.+\|/.test(text); +} + +// ── Main evaluator ──────────────────────────────────────────────────────────── + +/** + * Evaluate a model's response against prompt-based criteria. + * Returns a score 0.0-1.0 (recall equivalent for prompt surface). + */ +export function evaluatePromptResponse( + response: string, + criteria: PromptEvaluationCriteria, +): PromptEvaluationResult { + const checks: PromptCheckDetail[] = []; + const details: string[] = []; + + // Track per-category numerator / denominator + let sectionHits = 0; + let sectionTotal = 0; + let formatHits = 0; + let formatTotal = 0; + let keywordHits = 0; + let keywordTotal = 0; + let structureHits = 0; + let structureTotal = 0; + + // ── 1. Required sections (weight: 40%) ────────────────────────────────── + if (criteria.requiredSections && criteria.requiredSections.length > 0) { + sectionTotal = criteria.requiredSections.length; + for (const section of criteria.requiredSections) { + const found = sectionRegex(section).test(response); + if (found) sectionHits++; + const msg = found + ? `section "${section}": found` + : `section "${section}": MISSING`; + checks.push({ check: `section:${section}`, passed: found, detail: msg }); + details.push(msg); + } + } + + // ── 2. Format patterns (part of format weight: 20%) ───────────────────── + if (criteria.formatPatterns && criteria.formatPatterns.length > 0) { + formatTotal += criteria.formatPatterns.length; + for (const fp of criteria.formatPatterns) { + let matched = false; + try { + const re = new RegExp(fp.pattern, 'm'); + matched = re.test(response); + } catch { + // Invalid regex — treat as not matched + } + if (matched) formatHits++; + const msg = matched + ? `format "${fp.name}": matched` + : `format "${fp.name}": NOT matched`; + checks.push({ check: `format:${fp.name}`, passed: matched, detail: msg }); + details.push(msg); + } + } + + // Minimum length is also a format check + if (criteria.minLength !== undefined && criteria.minLength > 0) { + formatTotal++; + const lengthOk = response.length >= criteria.minLength; + if (lengthOk) formatHits++; + const msg = lengthOk + ? `minLength (${criteria.minLength}): OK (${response.length} chars)` + : `minLength (${criteria.minLength}): TOO SHORT (${response.length} chars)`; + checks.push({ check: 'format:minLength', passed: lengthOk, detail: msg }); + details.push(msg); + } + + // ── 3. Keywords (weight: 20%) ─────────────────────────────────────────── + const responseLower = response.toLowerCase(); + + if (criteria.requiredKeywords && criteria.requiredKeywords.length > 0) { + keywordTotal += criteria.requiredKeywords.length; + for (const kw of criteria.requiredKeywords) { + const found = responseLower.includes(kw.toLowerCase()); + if (found) keywordHits++; + const msg = found + ? `keyword "${kw}": found` + : `keyword "${kw}": MISSING`; + checks.push({ check: `keyword:${kw}`, passed: found, detail: msg }); + details.push(msg); + } + } + + if (criteria.forbiddenKeywords && criteria.forbiddenKeywords.length > 0) { + keywordTotal += criteria.forbiddenKeywords.length; + for (const kw of criteria.forbiddenKeywords) { + const absent = !responseLower.includes(kw.toLowerCase()); + if (absent) keywordHits++; + const msg = absent + ? `forbidden "${kw}": absent (good)` + : `forbidden "${kw}": PRESENT (hallucination)`; + checks.push({ check: `forbidden:${kw}`, passed: absent, detail: msg }); + details.push(msg); + } + } + + // ── 4. Structural checks (weight: 20%) ───────────────────────────────── + if (criteria.hasCodeBlocks !== undefined) { + structureTotal++; + const found = hasCodeBlock(response); + const pass = criteria.hasCodeBlocks ? found : !found; + if (pass) structureHits++; + const label = criteria.hasCodeBlocks ? 'expected' : 'unexpected'; + const msg = pass + ? `codeBlocks (${label}): OK` + : `codeBlocks (${label}): ${found ? 'PRESENT' : 'MISSING'}`; + checks.push({ check: 'structure:codeBlocks', passed: pass, detail: msg }); + details.push(msg); + } + + if (criteria.hasNumberedList !== undefined) { + structureTotal++; + const found = hasNumberedList(response); + const pass = criteria.hasNumberedList ? found : !found; + if (pass) structureHits++; + const label = criteria.hasNumberedList ? 'expected' : 'unexpected'; + const msg = pass + ? `numberedList (${label}): OK` + : `numberedList (${label}): ${found ? 'PRESENT' : 'MISSING'}`; + checks.push({ check: 'structure:numberedList', passed: pass, detail: msg }); + details.push(msg); + } + + if (criteria.hasTable !== undefined) { + structureTotal++; + const found = hasTable(response); + const pass = criteria.hasTable ? found : !found; + if (pass) structureHits++; + const label = criteria.hasTable ? 'expected' : 'unexpected'; + const msg = pass + ? `table (${label}): OK` + : `table (${label}): ${found ? 'PRESENT' : 'MISSING'}`; + checks.push({ check: 'structure:table', passed: pass, detail: msg }); + details.push(msg); + } + + // ── Compute category scores ───────────────────────────────────────────── + + const sectionScore = sectionTotal > 0 ? sectionHits / sectionTotal : 1.0; + const formatScore = formatTotal > 0 ? formatHits / formatTotal : 1.0; + const keywordScore = keywordTotal > 0 ? keywordHits / keywordTotal : 1.0; + const structureScore = structureTotal > 0 ? structureHits / structureTotal : 1.0; + + // If a category has no checks, redistribute its weight proportionally + // to the categories that do have checks. + const activeParts: { weight: number; score: number }[] = []; + if (sectionTotal > 0) activeParts.push({ weight: WEIGHT_SECTIONS, score: sectionScore }); + if (formatTotal > 0) activeParts.push({ weight: WEIGHT_FORMAT, score: formatScore }); + if (keywordTotal > 0) activeParts.push({ weight: WEIGHT_KEYWORDS, score: keywordScore }); + if (structureTotal > 0) activeParts.push({ weight: WEIGHT_STRUCTURE, score: structureScore }); + + let score: number; + if (activeParts.length === 0) { + // No criteria specified at all — vacuously pass + score = 1.0; + } else { + const totalActiveWeight = activeParts.reduce((s, p) => s + p.weight, 0); + score = activeParts.reduce((s, p) => s + (p.weight / totalActiveWeight) * p.score, 0); + } + + return { + score, + details, + checks, + categoryScores: { + sections: sectionScore, + format: formatScore, + keywords: keywordScore, + structure: structureScore, + }, + }; +} + +// ── Auto-generation from capability ─────────────────────────────────────────── + +/** + * Instruction verbs that signal the model should produce something specific. + * We look for "verb " patterns to extract required keywords. + */ +const INSTRUCTION_VERBS = [ + 'include', 'list', 'show', 'display', 'provide', 'output', 'generate', + 'create', 'write', 'produce', 'return', 'describe', 'explain', 'summarize', + 'format', 'use', 'add', 'specify', 'mention', 'contain', +]; + +/** + * Generate evaluation criteria from a prompt capability's section content. + * Extracts expected output patterns from code blocks and instruction verbs + * found in the skill section text. + */ +export function generateCriteriaFromCapability( + capability: ActionDefinition, + skillSection: string, +): PromptEvaluationCriteria { + const criteria: PromptEvaluationCriteria = {}; + + // ── 1. Extract required sections from markdown headings in the section ── + // Look for sub-headings that describe expected output structure + const headingRe = /^#{2,6}\s+(.+)$/gm; + const sections: string[] = []; + let headingMatch: RegExpExecArray | null; + while ((headingMatch = headingRe.exec(skillSection)) !== null) { + const heading = headingMatch[1].trim(); + // Skip headings that are clearly meta / instructional + if (!/^(example|note|tip|warning|usage|syntax|overview|description)s?$/i.test(heading)) { + sections.push(heading); + } + } + if (sections.length > 0) { + criteria.requiredSections = sections; + } + + // ── 2. Extract format patterns from code fences ── + // Code blocks in the skill section often show expected output templates. + const codeBlockRe = /```\w*\n([\s\S]*?)```/g; + const formatPatterns: FormatPattern[] = []; + let codeBlockCount = 0; + let codeBlockMatch: RegExpExecArray | null; + while ((codeBlockMatch = codeBlockRe.exec(skillSection)) !== null) { + codeBlockCount++; + const content = codeBlockMatch[1].trim(); + // Skip very long code blocks (likely full examples, not format constraints) + if (content.length > 500) continue; + // Extract lines that look like template patterns (contain placeholders or fixed structure) + const templateLines = content.split('\n').filter( + line => /[{<\[].+[}>\]]/.test(line) || /^\s*\w+\s*[:=]/.test(line), + ); + for (const line of templateLines) { + // Convert template placeholders to regex wildcards + const escaped = line.trim() + .replace(/[.*+?^${}()|[\]\\]/g, '\\$&') + .replace(/\\[{<]\\?[^}>\\]+\\?[}>]/g, '.+'); + if (escaped.length > 5) { + formatPatterns.push({ + name: `template-line-${formatPatterns.length + 1}`, + pattern: escaped, + }); + } + } + } + if (formatPatterns.length > 0) { + criteria.formatPatterns = formatPatterns; + } + + // ── 3. Detect structural expectations ── + if (codeBlockCount > 0 || /code block|code example|snippet/i.test(skillSection)) { + criteria.hasCodeBlocks = true; + } + if (/numbered list|ordered list|step[- ]by[- ]step|\d+\.\s/i.test(skillSection)) { + criteria.hasNumberedList = true; + } + if (/\btable\b|markdown table|\|.*\|.*\|/i.test(skillSection)) { + criteria.hasTable = true; + } + + // ── 4. Extract required keywords from instruction patterns ── + const requiredKeywords: string[] = []; + const verbPattern = new RegExp( + `\\b(?:${INSTRUCTION_VERBS.join('|')})\\b\\s+(?:the\\s+|a\\s+|an\\s+)?["']?([\\w][\\w\\s-]{2,30}?)["']?(?:\\s*[,;.]|\\s+(?:in|to|for|as|with|from|using))`, + 'gi', + ); + const seenKeywords = new Set(); + let verbMatch: RegExpExecArray | null; + while ((verbMatch = verbPattern.exec(skillSection)) !== null) { + const keyword = verbMatch[1].trim().toLowerCase(); + // Skip very generic words + if (keyword.length < 3) continue; + if (/^(the|this|that|your|each|all|any|it|them|these|those)$/i.test(keyword)) continue; + if (!seenKeywords.has(keyword)) { + seenKeywords.add(keyword); + requiredKeywords.push(keyword); + } + } + if (requiredKeywords.length > 0) { + criteria.requiredKeywords = requiredKeywords; + } + + // ── 5. Extract forbidden keywords from explicit "do not" / "never" instructions ── + const forbiddenKeywords: string[] = []; + const forbiddenPattern = /\b(?:do\s+not|don't|never|avoid|must\s+not|should\s+not|shouldn't)\b\s+(?:\w+\s+)?["']?(\w[\w\s-]{2,30}?)["']?\b/gi; + const seenForbidden = new Set(); + let forbiddenMatch: RegExpExecArray | null; + while ((forbiddenMatch = forbiddenPattern.exec(skillSection)) !== null) { + const keyword = forbiddenMatch[1].trim().toLowerCase(); + if (keyword.length < 3) continue; + if (/^(the|this|that|your|each|all|any|it|them|use|include|mention)$/i.test(keyword)) continue; + if (!seenForbidden.has(keyword)) { + seenForbidden.add(keyword); + forbiddenKeywords.push(keyword); + } + } + if (forbiddenKeywords.length > 0) { + criteria.forbiddenKeywords = forbiddenKeywords; + } + + // ── 6. Infer minimum length from the capability description ── + // Longer descriptions with many args suggest non-trivial output + const descLength = (capability.description ?? '').length; + const argCount = capability.args.length; + if (descLength > 200 || argCount > 3) { + criteria.minLength = 200; + } else if (descLength > 50 || argCount > 1) { + criteria.minLength = 100; + } + + return criteria; +} diff --git a/src/benchmark/prompts.ts b/src/benchmark/prompts.ts index 74ee28a..5d5029c 100644 --- a/src/benchmark/prompts.ts +++ b/src/benchmark/prompts.ts @@ -1,6 +1,6 @@ import type { FetchedSkill, TaskDefinition, SdkLanguage } from './types.js'; -export type PromptSurface = 'sdk' | 'cli' | 'mcp'; +export type PromptSurface = 'sdk' | 'cli' | 'mcp' | 'prompt'; export interface PromptOptions { surface: PromptSurface; @@ -37,6 +37,15 @@ export function buildSystemPrompt( ? `\n\nOptional guidance context (SKILL.md):\n--- GUIDANCE ---\n${skill.content}\n--- END GUIDANCE ---` : ''; + if (options.surface === 'prompt') { + // For prompt surface, the skill IS the system prompt. + // If skill content is available, use it directly; otherwise use a generic wrapper. + if (skill) { + return skill.content; + } + return `You are a helpful assistant for ${sdkName}. Follow the instructions and complete the task.`; + } + if (options.surface === 'mcp') { return ( `You are a helpful assistant with access to tools for ${sdkName}.\n` + @@ -84,6 +93,10 @@ export function buildTaskPrompt( task: TaskDefinition, options: PromptOptions, ): string { + if (options.surface === 'prompt') { + return task.prompt; + } + if (options.surface === 'mcp') { return ( `Task: ${task.prompt}\n\n` + diff --git a/src/benchmark/runner.ts b/src/benchmark/runner.ts index 14ed41b..ddb0687 100644 --- a/src/benchmark/runner.ts +++ b/src/benchmark/runner.ts @@ -26,6 +26,9 @@ import { evaluateTask } from './evaluator.js'; import { computeCoverage } from './coverage.js'; import { computeVerdict } from './scoring.js'; import { buildSystemPrompt, buildTaskPrompt } from './prompts.js'; +import { evaluatePromptResponse, generateCriteriaFromCapability } from './prompt-evaluator.js'; +import type { PromptEvaluationCriteria } from './prompt-evaluator.js'; +import { discoverPromptCapabilitiesWithSections } from '../project/discover-prompt.js'; function buildWebFetchTool(): McpToolDefinition { return { @@ -128,7 +131,10 @@ export async function runBenchmark(options: RunnerOptions = {}): Promise; let cliCommands: ReturnType | undefined = undefined; let mcpToolDefs: ReturnType | undefined = undefined; - if (config.surface === 'sdk') { + if (config.surface === 'prompt') { + // Prompt surface: no tool/action definitions — evaluation is content-based. + knownMethods = new Set(); + } else if (config.surface === 'sdk') { // Derive from task expected actions + optional sdk.apiSurface const fromTasks = new Set(); for (const task of tasks) { @@ -157,7 +163,9 @@ export async function runBenchmark(options: RunnerOptions = {}): Promise 0) { + // Use the primary capability's section to derive criteria. + promptEvalCriteria = generateCriteriaFromCapability(caps[0].action, caps[0].section); + console.log(`[prompt] Evaluation criteria derived from ${caps.length} discovered capabilities`); + } + } + // 6. Build system prompt const promptOptions = { surface: config.surface, @@ -297,47 +316,53 @@ export async function runBenchmark(options: RunnerOptions = {}): Promise | undefined; - try { - const extractionConfig = config.surface === 'cli' && cliCommands - ? { - ...config, - cli: { - ...config.cli, - commandDefinitions: cliCommands, - }, + if (config.surface === 'prompt') { + // Prompt surface: no extraction — evaluation is content-based. + // The raw text response is the output; no tool calls or code blocks to parse. + console.log(` [${slug}] Prompt response: ${rawResponse.length} chars`); + } else { + try { + const extractionConfig = config.surface === 'cli' && cliCommands + ? { + ...config, + cli: { + ...config.cli, + commandDefinitions: cliCommands, + }, + } + : config; + + const extracted = await extract(llmResponse!, extractionConfig as BenchmarkConfig); + extractedCalls = extracted.calls; + generatedCode = extracted.generatedCode; + bindings = extracted.bindings; + + if (config.surface === 'sdk') { + const sdkLanguage = config.sdk?.language ?? 'typescript'; + if (generatedCode) { + console.log(` [${slug}] ${sdkLanguage} code extracted: ${generatedCode.length} chars`); + } else if (!error) { + console.log(` [${slug}] WARNING: No ${sdkLanguage} code block found`); + error = error ?? `No ${sdkLanguage} code block in response`; } - : config; - - const extracted = await extract(llmResponse!, extractionConfig as BenchmarkConfig); - extractedCalls = extracted.calls; - generatedCode = extracted.generatedCode; - bindings = extracted.bindings; - - if (config.surface === 'sdk') { - const sdkLanguage = config.sdk?.language ?? 'typescript'; - if (generatedCode) { - console.log(` [${slug}] ${sdkLanguage} code extracted: ${generatedCode.length} chars`); - } else if (!error) { - console.log(` [${slug}] WARNING: No ${sdkLanguage} code block found`); - error = error ?? `No ${sdkLanguage} code block in response`; } - } - if (config.surface === 'cli') { - if (generatedCode) { - console.log(` [${slug}] Command block extracted: ${generatedCode.length} chars`); - } else if (!error) { - console.log(` [${slug}] WARNING: No shell command block found`); - error = error ?? 'No shell command block in response'; + if (config.surface === 'cli') { + if (generatedCode) { + console.log(` [${slug}] Command block extracted: ${generatedCode.length} chars`); + } else if (!error) { + console.log(` [${slug}] WARNING: No shell command block found`); + error = error ?? 'No shell command block in response'; + } } - } - if (extractedCalls.length > 0) { - console.log(` [${slug}] Extracted ${extractedCalls.length} calls: ${extractedCalls.map(c => c.method).join(', ')}`); + if (extractedCalls.length > 0) { + console.log(` [${slug}] Extracted ${extractedCalls.length} calls: ${extractedCalls.map(c => c.method).join(', ')}`); + } + } catch (err) { + console.error(` [${slug}] Extraction error: ${err instanceof Error ? err.message : err}`); + error = error ?? `Extraction failed: ${err instanceof Error ? err.message : err}`; } - } catch (err) { - console.error(` [${slug}] Extraction error: ${err instanceof Error ? err.message : err}`); - error = error ?? `Extraction failed: ${err instanceof Error ? err.message : err}`; } // Evaluate @@ -355,6 +380,19 @@ export async function runBenchmark(options: RunnerOptions = {}): Promise= 0.5; + console.log(` [${slug}] Prompt score: ${promptResult.score.toFixed(3)} → ${taskResult.metrics.taskPassed ? 'PASS' : 'FAIL'}`); + } catch (err) { + console.error(` [${slug}] Prompt eval error: ${err instanceof Error ? err.message : err}`); + // Leave vacuous scores in place — task shows as PASS rather than crashing the run + } + } + if (config.agentic && task.expected_fetches) { const actualFetches = fetchedPaths; const expectedSet = new Set(task.expected_fetches); diff --git a/src/benchmark/types.ts b/src/benchmark/types.ts index 0246f34..2a9b680 100644 --- a/src/benchmark/types.ts +++ b/src/benchmark/types.ts @@ -23,7 +23,7 @@ export interface TokenUsage { // === Config (loaded from benchmark.config.json) === -export type BenchmarkSurface = 'sdk' | 'cli' | 'mcp'; +export type BenchmarkSurface = 'sdk' | 'cli' | 'mcp' | 'prompt'; export type SdkLanguage = 'typescript' | 'python' | 'rust'; export interface BenchmarkConfig { diff --git a/src/cli.ts b/src/cli.ts index 319723f..46d019d 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -124,10 +124,10 @@ export function positionals(args: string[]): string[] { function printUsage(): void { console.log(` -Skill Optimizer CLI — Benchmark and optimize SDK/CLI/MCP guidance +Skill Optimizer CLI — Benchmark and optimize SDK/CLI/MCP/prompt guidance Usage: - skill-optimizer init [sdk|cli|mcp] Interactive wizard — scaffold config for the given surface + skill-optimizer init [sdk|cli|mcp|prompt] Interactive wizard — scaffold config for the given surface skill-optimizer init [surface] --yes Accept all defaults non-interactively skill-optimizer init --answers Load wizard answers from a JSON file (CI mode) skill-optimizer init --auto Auto-detect surface from CWD and pre-fill wizard @@ -179,6 +179,7 @@ Examples: skill-optimizer init cli skill-optimizer init sdk skill-optimizer init mcp + skill-optimizer init prompt skill-optimizer import-commands --from ./src/cli.ts skill-optimizer import-commands --from fast-cli --scrape skill-optimizer doctor --config ./skill-optimizer.json @@ -251,9 +252,9 @@ async function main(): Promise { // ── Init mode ──────────────────────────────────────────────────────────────── if (command === 'init') { - const surfaceArg = pos[1] as 'sdk' | 'cli' | 'mcp' | undefined; - if (surfaceArg && !['sdk', 'cli', 'mcp'].includes(surfaceArg)) { - console.error(`ERROR: Unknown surface '${surfaceArg}'. Must be: sdk | cli | mcp`); + const surfaceArg = pos[1] as 'sdk' | 'cli' | 'mcp' | 'prompt' | undefined; + if (surfaceArg && !['sdk', 'cli', 'mcp', 'prompt'].includes(surfaceArg)) { + console.error(`ERROR: Unknown surface '${surfaceArg}'. Must be: sdk | cli | mcp | prompt`); process.exit(1); } const answersFlag = getFlag(args, '--answers'); diff --git a/src/discovery/index.ts b/src/discovery/index.ts index d19efde..5591003 100644 --- a/src/discovery/index.ts +++ b/src/discovery/index.ts @@ -1,5 +1,6 @@ export { discoverCliSurfaceFromSources } from './cli.js'; export { discoverMcpSurfaceFromSources } from './mcp.js'; +export { discoverPromptSurfaceFromContent, discoverPromptSurfaceFromSources } from './prompt.js'; export { discoverSdkSurfaceFromSources } from './sdk.js'; export type { @@ -11,3 +12,9 @@ export type { McpDiscoverySnapshot, SdkDiscoverySnapshot, } from './types.js'; + +export type { + PromptCapability, + PromptDiscoverySnapshot, + PromptPhase, +} from './prompt.js'; diff --git a/src/discovery/prompt.ts b/src/discovery/prompt.ts new file mode 100644 index 0000000..c17106f --- /dev/null +++ b/src/discovery/prompt.ts @@ -0,0 +1,180 @@ +/** + * Prompt surface discoverer. + * + * Parses SKILL.md files to extract phases, capabilities, and structural + * information. This enables benchmarking how well models follow prompt + * templates — not just tool calls. + */ + +import { existsSync, readFileSync } from 'node:fs'; +import { resolve } from 'node:path'; + +import type { DiscoveryOptions } from './types.js'; + +// ── Types ───────────────────────────────────────────────────────────────── + +export interface PromptPhase { + name: string; + description: string; + hasCodeBlocks: boolean; + hasNumberedSteps: boolean; + hasDecisionPoints: boolean; +} + +export interface PromptCapability { + name: string; + description: string; + source: 'phase' | 'instruction'; +} + +export interface PromptDiscoverySnapshot { + surface: 'prompt'; + phases: PromptPhase[]; + capabilities: PromptCapability[]; + sources: string[]; +} + +// ── Phase parsing ───────────────────────────────────────────────────────── + +const PHASE_HEADER_RE = /^##\s+(?:Phase\s+\d+\s*[—–-]\s*)?(.+)$/gm; +const CODE_BLOCK_RE = /```[\s\S]*?```/g; +const NUMBERED_STEP_RE = /^\d+\.\s+/m; +const DECISION_POINT_RE = /\b(?:if|when|unless|otherwise|decide|choose|either)\b/i; + +function extractPhases(content: string): PromptPhase[] { + const phases: PromptPhase[] = []; + const headers: { name: string; start: number }[] = []; + + // Find all phase headers (## headings) + let match: RegExpExecArray | null; + const headerRe = new RegExp(PHASE_HEADER_RE.source, PHASE_HEADER_RE.flags); + while ((match = headerRe.exec(content)) !== null) { + headers.push({ name: match[1].trim(), start: match.index }); + } + + for (let i = 0; i < headers.length; i++) { + const start = headers[i].start; + const end = i + 1 < headers.length ? headers[i + 1].start : content.length; + const sectionContent = content.slice(start, end); + + // Extract description: first non-empty, non-heading line + const lines = sectionContent.split('\n').slice(1); // skip the heading itself + const descLine = lines.find((l) => l.trim().length > 0 && !l.startsWith('#')); + const description = descLine?.trim() ?? ''; + + phases.push({ + name: headers[i].name, + description, + hasCodeBlocks: CODE_BLOCK_RE.test(sectionContent), + hasNumberedSteps: NUMBERED_STEP_RE.test(sectionContent), + hasDecisionPoints: DECISION_POINT_RE.test(sectionContent), + }); + + // Reset lastIndex for the stateful regex + CODE_BLOCK_RE.lastIndex = 0; + } + + return phases; +} + +// ── Capability extraction ───────────────────────────────────────────────── + +function extractCapabilities(content: string, phases: PromptPhase[]): PromptCapability[] { + const capabilities: PromptCapability[] = []; + + if (phases.length > 0) { + // Derive capabilities from phases + for (const phase of phases) { + capabilities.push({ + name: phase.name, + description: phase.description, + source: 'phase', + }); + } + } else if (content.trim().length > 0) { + // No phases found — extract capabilities from top-level instructions + // Look for bullet points or imperative sentences + const lines = content.split('\n'); + for (const line of lines) { + const trimmed = line.trim(); + // Match bullet points: "- Do something" or "* Do something" + const bulletMatch = trimmed.match(/^[-*]\s+(.+)$/); + if (bulletMatch) { + capabilities.push({ + name: bulletMatch[1].slice(0, 60), + description: bulletMatch[1], + source: 'instruction', + }); + } + } + + // If still no capabilities, treat the whole content as one instruction block + if (capabilities.length === 0) { + const firstLine = lines.find((l) => l.trim().length > 0 && !l.startsWith('#') && !l.startsWith('---')); + if (firstLine) { + capabilities.push({ + name: firstLine.trim().slice(0, 60), + description: firstLine.trim(), + source: 'instruction', + }); + } + } + } + + return capabilities; +} + +// ── Frontmatter stripping ───────────────────────────────────────────────── + +function stripFrontmatter(content: string): string { + if (!content.startsWith('---')) return content; + // Match the closing --- that appears on its own line (after the opening ---) + const closingMatch = content.slice(3).match(/\n---\s*(\n|$)/); + if (!closingMatch || closingMatch.index === undefined) return content; + const endIdx = 3 + closingMatch.index + closingMatch[0].length; + return content.slice(endIdx); +} + +// ── Public API ──────────────────────────────────────────────────────────── + +export function discoverPromptSurfaceFromContent(content: string): PromptDiscoverySnapshot { + const body = stripFrontmatter(content); + const phases = extractPhases(body); + const capabilities = extractCapabilities(body, phases); + + return { + surface: 'prompt', + phases, + capabilities, + sources: [], + }; +} + +export function discoverPromptSurfaceFromSources( + sources: string[], + options: DiscoveryOptions = {}, +): PromptDiscoverySnapshot { + const baseDir = options.baseDir ?? process.cwd(); + const resolvedSources = sources.map((source) => resolve(baseDir, source)); + + const allPhases: PromptPhase[] = []; + const allCapabilities: PromptCapability[] = []; + + for (const sourcePath of resolvedSources) { + if (!existsSync(sourcePath)) { + throw new Error(`Prompt discovery source file does not exist: ${sourcePath}`); + } + + const content = readFileSync(sourcePath, 'utf-8'); + const snapshot = discoverPromptSurfaceFromContent(content); + allPhases.push(...snapshot.phases); + allCapabilities.push(...snapshot.capabilities); + } + + return { + surface: 'prompt', + phases: allPhases, + capabilities: allCapabilities, + sources: resolvedSources, + }; +} diff --git a/src/init/answers.ts b/src/init/answers.ts index b1efaab..c8f240b 100644 --- a/src/init/answers.ts +++ b/src/init/answers.ts @@ -1,7 +1,7 @@ import { readFileSync } from 'node:fs'; export interface WizardAnswers { - surface: 'sdk' | 'cli' | 'mcp'; + surface: 'sdk' | 'cli' | 'mcp' | 'prompt'; repoPath: string; models: string[]; maxTasks: number; @@ -21,7 +21,7 @@ const DEFAULT_MODELS = [ 'openrouter/google/gemini-2-5-flash', ]; -export function buildDefaultAnswers(surface: 'sdk' | 'cli' | 'mcp' = 'sdk', repoPath?: string): WizardAnswers { +export function buildDefaultAnswers(surface: 'sdk' | 'cli' | 'mcp' | 'prompt' = 'sdk', repoPath?: string): WizardAnswers { return { surface, repoPath: repoPath ?? process.cwd(), @@ -34,8 +34,8 @@ export function buildDefaultAnswers(surface: 'sdk' | 'cli' | 'mcp' = 'sdk', repo export function readAnswersFile(filePath: string): WizardAnswers { const raw = JSON.parse(readFileSync(filePath, 'utf-8')) as Partial; - if (!raw.surface || !['sdk', 'cli', 'mcp'].includes(raw.surface)) { - throw new Error(`answers file must have surface: sdk | cli | mcp (got: ${JSON.stringify(raw.surface)})`); + if (!raw.surface || !['sdk', 'cli', 'mcp', 'prompt'].includes(raw.surface)) { + throw new Error(`answers file must have surface: sdk | cli | mcp | prompt (got: ${JSON.stringify(raw.surface)})`); } if (!raw.repoPath) { throw new Error('answers file must have repoPath'); diff --git a/src/init/scaffold.ts b/src/init/scaffold.ts index 67019d0..8d3eb04 100644 --- a/src/init/scaffold.ts +++ b/src/init/scaffold.ts @@ -118,6 +118,19 @@ export function buildConfigFromAnswers(answers: WizardAnswers, configDir: string }; } + if (surface === 'prompt') { + return { + name: projectName, + target: { + surface: 'prompt', + repoPath: relRepo, + skill: skillConfigPath, + }, + benchmark: commonBenchmark, + optimize: commonOptimize, + }; + } + // mcp return { name: projectName, diff --git a/src/init/wizard.ts b/src/init/wizard.ts index ff6816e..d8d2461 100644 --- a/src/init/wizard.ts +++ b/src/init/wizard.ts @@ -55,7 +55,7 @@ export async function runWizard(cwd: string, preseed?: Partial): p.intro('skill-optimizer init'); // 1. Surface - let surface: 'sdk' | 'cli' | 'mcp'; + let surface: 'sdk' | 'cli' | 'mcp' | 'prompt'; if (preseed?.surface) { surface = preseed.surface; p.log.info(`Surface: ${surface}`); @@ -66,8 +66,9 @@ export async function runWizard(cwd: string, preseed?: Partial): { value: 'sdk', label: 'sdk', hint: 'TypeScript / Python / Rust library' }, { value: 'cli', label: 'cli', hint: 'command-line tool with subcommands' }, { value: 'mcp', label: 'mcp', hint: 'MCP server with tools' }, + { value: 'prompt', label: 'prompt', hint: 'markdown skill/prompt file' }, ], - }) as 'sdk' | 'cli' | 'mcp'); + }) as 'sdk' | 'cli' | 'mcp' | 'prompt'); } // 2. Target repo path diff --git a/src/project/discover-prompt.ts b/src/project/discover-prompt.ts new file mode 100644 index 0000000..ba4a384 --- /dev/null +++ b/src/project/discover-prompt.ts @@ -0,0 +1,369 @@ +import type { ActionDefinition } from '../actions/types.js'; + +export type PromptCapabilityType = 'phase' | 'instruction' | 'output' | 'decision'; + +export interface PromptCapability { + name: string; + description: string; + section: string; + type: PromptCapabilityType; +} + +// Imperative verbs that start an instruction line. +const IMPERATIVE_VERBS = [ + 'write', 'generate', 'create', 'check', 'verify', 'ensure', 'add', + 'remove', 'delete', 'update', 'modify', 'set', 'configure', 'run', + 'execute', 'build', 'deploy', 'test', 'validate', 'send', 'fetch', + 'parse', 'extract', 'transform', 'convert', 'compute', 'calculate', + 'define', 'implement', 'install', 'import', 'export', 'open', 'close', + 'start', 'stop', 'initialize', 'list', 'search', 'find', 'filter', + 'sort', 'map', 'reduce', 'merge', 'split', 'read', 'load', 'save', + 'store', 'output', 'print', 'log', 'return', 'emit', 'publish', + 'subscribe', 'ask', 'prompt', 'collect', 'gather', 'summarize', + 'analyze', 'review', 'approve', 'reject', 'iterate', 'loop', 'repeat', + 'wait', 'retry', 'handle', 'catch', 'throw', 'raise', 'assert', + 'call', 'invoke', 'trigger', 'notify', 'alert', 'warn', +]; + +const IMPERATIVE_RE = new RegExp( + `^(?:[-*]\\s+)?(?:\\*\\*)?(?:${IMPERATIVE_VERBS.join('|')})\\b`, + 'i', +); + +// Heading patterns for phases/steps. +const PHASE_HEADING_RE = /^##\s+(?:phase|step)\s+(\d+)[:\s—–-]*\s*(.*)/i; + +// General ## heading (non-phase). +const HEADING_RE = /^##\s+(.+)/; + +// Decision-point patterns (if/when/then, conditional logic). +const DECISION_RE = /\b(?:if|when|unless|otherwise|then|else|in case|provided that|assuming)\b/i; + +// Code block detection. +const CODE_BLOCK_OPEN_RE = /^```(\w*)/; +const CODE_BLOCK_CLOSE_RE = /^```\s*$/; + +interface MarkdownSection { + heading: string; + level: number; + body: string; + isPhase: boolean; + phaseNumber?: number; +} + +function stripFrontmatter(content: string): string { + if (!content.startsWith('---')) return content; + const closingMatch = content.slice(3).match(/\n---\s*(\n|$)/); + if (!closingMatch || closingMatch.index === undefined) return content; + return content.slice(3 + closingMatch.index + closingMatch[0].length); +} + +/** + * Split markdown content into sections by ## headings. + */ +function splitSections(content: string): MarkdownSection[] { + const lines = content.split('\n'); + const sections: MarkdownSection[] = []; + let current: MarkdownSection | null = null; + const bodyLines: string[] = []; + + function flushCurrent(): void { + if (current) { + current.body = bodyLines.join('\n').trim(); + sections.push(current); + bodyLines.length = 0; + } + } + + for (const line of lines) { + const headingMatch = HEADING_RE.exec(line); + if (headingMatch) { + flushCurrent(); + bodyLines.length = 0; // discard preamble lines before first heading + const phaseMatch = PHASE_HEADING_RE.exec(line); + current = { + heading: headingMatch[1]!.trim(), + level: 2, + body: '', + isPhase: Boolean(phaseMatch), + phaseNumber: phaseMatch ? parseInt(phaseMatch[1]!, 10) : undefined, + }; + } else if (current) { + bodyLines.push(line); + } else { + // Lines before the first ## heading — accumulate in case there is a preamble. + bodyLines.push(line); + } + } + + flushCurrent(); + return sections; +} + +/** + * Slugify a heading string into a snake_case identifier. + * "Phase 1: Requirements Discovery" -> "phase_1_requirements_discovery" + */ +function slugify(text: string): string { + return text + .toLowerCase() + .replace(/[^a-z0-9]+/g, '_') + .replace(/^_|_$/g, ''); +} + +/** + * Extract imperative instructions from section body text. + * Returns an array of instruction sentences. + */ +function extractInstructions(body: string): string[] { + const results: string[] = []; + let inCodeBlock = false; + + for (const line of body.split('\n')) { + if (CODE_BLOCK_OPEN_RE.test(line) && !inCodeBlock) { + inCodeBlock = true; + continue; + } + if (CODE_BLOCK_CLOSE_RE.test(line) && inCodeBlock) { + inCodeBlock = false; + continue; + } + if (inCodeBlock) continue; + + const trimmed = line.trim(); + if (IMPERATIVE_RE.test(trimmed)) { + // Strip leading markdown list markers and bold markers for the description. + const clean = trimmed.replace(/^[-*]\s+/, '').replace(/\*\*/g, ''); + results.push(clean); + } + } + + return results; +} + +/** + * Extract code blocks that represent expected output formats. + * Returns the raw code block content strings. + */ +function extractOutputFormats(body: string): string[] { + const blocks: string[] = []; + let inCodeBlock = false; + let currentBlock: string[] = []; + + for (const line of body.split('\n')) { + if (CODE_BLOCK_OPEN_RE.test(line) && !inCodeBlock) { + inCodeBlock = true; + currentBlock = []; + continue; + } + if (CODE_BLOCK_CLOSE_RE.test(line) && inCodeBlock) { + inCodeBlock = false; + if (currentBlock.length > 0) { + blocks.push(currentBlock.join('\n')); + } + continue; + } + if (inCodeBlock) { + currentBlock.push(line); + } + } + + return blocks; +} + +/** + * Detect whether the section body contains decision-point language. + */ +function hasDecisionPoints(body: string): boolean { + let inCodeBlock = false; + + for (const line of body.split('\n')) { + if (CODE_BLOCK_OPEN_RE.test(line) && !inCodeBlock) { + inCodeBlock = true; + continue; + } + if (CODE_BLOCK_CLOSE_RE.test(line) && inCodeBlock) { + inCodeBlock = false; + continue; + } + if (inCodeBlock) continue; + + if (DECISION_RE.test(line)) { + return true; + } + } + + return false; +} + +/** + * Convert a PromptCapability into an ActionDefinition compatible with the + * benchmark runner and surface snapshot system. + */ +function capabilityToAction(cap: PromptCapability): ActionDefinition { + const key = slugify(cap.name); + + // Build args based on the capability type. + const args: ActionDefinition['args'] = []; + + if (cap.type === 'phase') { + args.push({ + name: 'user_brief', + required: true, + type: 'string', + description: 'The product brief from the user', + }); + } else if (cap.type === 'instruction') { + args.push({ + name: 'input', + required: true, + type: 'string', + description: 'The input to process for this instruction', + }); + } else if (cap.type === 'output') { + args.push({ + name: 'content', + required: true, + type: 'string', + description: 'The content to format into the expected output', + }); + } else if (cap.type === 'decision') { + args.push( + { + name: 'condition', + required: true, + type: 'string', + description: 'The condition or context to evaluate', + }, + { + name: 'context', + required: false, + type: 'string', + description: 'Additional context for the decision', + }, + ); + } + + return { + key, + name: key, + description: cap.description, + args, + source: 'prompt', + }; +} + +export interface PromptCapabilityWithSection { + action: ActionDefinition; + /** Raw markdown body text of the section this capability was extracted from. */ + section: string; +} + +/** + * Discover capabilities from a markdown prompt/skill file, returning each + * capability paired with its raw section body text. + */ +export function discoverPromptCapabilitiesWithSections( + skillContent: string, +): PromptCapabilityWithSection[] { + const content = stripFrontmatter(skillContent); + const sections = splitSections(content); + const result: PromptCapabilityWithSection[] = []; + const seenNames = new Set(); + + function addWithSection(cap: PromptCapability): void { + const key = slugify(cap.name); + if (seenNames.has(key)) return; + seenNames.add(key); + result.push({ action: capabilityToAction(cap), section: cap.section }); + } + + for (const section of sections) { + // 1. Phase capabilities (## Phase N / ## Step N headings). + if (section.isPhase && section.phaseNumber !== undefined) { + const phaseName = `phase_${section.phaseNumber}_${slugify(section.heading.replace(/^(?:phase|step)\s+\d+[:\s—–-]*/i, ''))}`; + const firstSentence = section.body.split(/[.!?\n]/)[0]?.trim() ?? section.heading; + addWithSection({ + name: phaseName, + description: firstSentence || section.heading, + section: section.body, + type: 'phase', + }); + } + + // 2. Instruction capabilities from imperative sentences. + const instructions = extractInstructions(section.body); + for (const instruction of instructions) { + const instructionName = `${slugify(section.heading)}_${slugify(instruction.slice(0, 60))}`; + addWithSection({ + name: instructionName, + description: instruction, + section: section.body, + type: 'instruction', + }); + } + + // 3. Output format capabilities from code blocks. + const outputs = extractOutputFormats(section.body); + for (let i = 0; i < outputs.length; i++) { + const snippet = outputs[i]!; + const outputName = `${slugify(section.heading)}_output${outputs.length > 1 ? `_${i + 1}` : ''}`; + const preview = snippet.split('\n')[0]?.trim().slice(0, 80) ?? 'code block'; + addWithSection({ + name: outputName, + description: `Expected output format: ${preview}`, + section: snippet, + type: 'output', + }); + } + + // 4. Decision-point capabilities. + if (hasDecisionPoints(section.body)) { + const decisionName = `${slugify(section.heading)}_decision`; + addWithSection({ + name: decisionName, + description: `Decision point in "${section.heading}" — evaluate conditional logic`, + section: section.body, + type: 'decision', + }); + } + } + + // Fallback: no ## sections were found — extract from whole content. + if (sections.length === 0) { + const fallbackInstructions = extractInstructions(content); + for (const instruction of fallbackInstructions) { + addWithSection({ + name: slugify(instruction.slice(0, 60)) || 'instruction', + description: instruction, + section: content, + type: 'instruction', + }); + } + // Last resort: use the first non-empty content line as a single capability. + // Only do this for non-empty content; empty files should still return [] so + // buildPromptSurfaceSnapshot's 0-capability guard fires correctly. + if (result.length === 0 && content.trim().length > 0) { + const firstLine = content.trim().split('\n').find(l => l.trim().length > 0) ?? 'skill'; + const cleaned = firstLine.replace(/^#+\s*/, '').trim(); + addWithSection({ + name: slugify(cleaned) || 'skill', + description: cleaned || 'Skill capability', + section: content, + type: 'instruction', + }); + } + } + + return result; +} + +/** + * Discover capabilities from a markdown prompt/skill file. + * Parses headings, imperative instructions, code-block output formats, + * and decision-point logic into ActionDefinition[] compatible with the + * benchmark runner. + */ +export function discoverPromptCapabilities(skillContent: string): ActionDefinition[] { + return discoverPromptCapabilitiesWithSections(skillContent).map(({ action }) => action); +} diff --git a/src/project/snapshot.ts b/src/project/snapshot.ts index 76c7a46..cab7d50 100644 --- a/src/project/snapshot.ts +++ b/src/project/snapshot.ts @@ -1,19 +1,49 @@ import { existsSync, readFileSync } from 'node:fs'; +import { resolve, isAbsolute } from 'node:path'; import type { SurfaceSnapshot } from './types.js'; import type { ResolvedProjectConfig } from './types.js'; import type { McpToolDefinition } from '../benchmark/types.js'; import { discoverActions } from '../actions/discover.js'; import { loadActionSnapshotFile, toSurfaceSnapshot } from '../actions/snapshot.js'; +import { discoverPromptCapabilities } from './discover-prompt.js'; export function buildSurfaceSnapshot(project: ResolvedProjectConfig): SurfaceSnapshot { if (project.benchmark.surfaceSnapshot) { return loadSurfaceSnapshotFile(project.benchmark.surfaceSnapshot); } + if (project.target.surface === 'prompt') { + return buildPromptSurfaceSnapshot(project); + } + return toSurfaceSnapshot(discoverActions(project)); } +function buildPromptSurfaceSnapshot(project: ResolvedProjectConfig): SurfaceSnapshot { + const skillSource = project.target.skill?.source; + if (!skillSource) { + throw new Error('Prompt surface requires target.skill to point at a markdown skill file'); + } + + const absPath = isAbsolute(skillSource) ? skillSource : resolve(project.configDir, skillSource); + if (!existsSync(absPath)) { + throw new Error(`Prompt skill file not found: ${absPath}`); + } + + const content = readFileSync(absPath, 'utf-8'); + const actions = discoverPromptCapabilities(content); + + if (actions.length === 0) { + throw new Error(`Prompt discovery found 0 capabilities in ${absPath}`); + } + + return { + surface: 'prompt', + actions: actions.map(({ key: _key, ...rest }) => rest), + }; +} + function normalizeCliArgName(name: string): string { return name.replace(/^-+/, ''); } @@ -49,7 +79,7 @@ function isLegacySurfaceSnapshot(value: unknown): value is SurfaceSnapshot { && 'surface' in value && 'actions' in value && Array.isArray((value as { actions?: unknown }).actions) - && ['sdk', 'cli', 'mcp'].includes(String((value as { surface?: unknown }).surface)), + && ['sdk', 'cli', 'mcp', 'prompt'].includes(String((value as { surface?: unknown }).surface)), ); } @@ -69,7 +99,7 @@ function isActionSnapshotArtifactShape(value: unknown): value is { version: numb return typeof candidate.version === 'number' && Boolean(candidate.catalog) && typeof candidate.catalog === 'object' - && ['sdk', 'cli', 'mcp'].includes(String(candidate.catalog.surface)) + && ['sdk', 'cli', 'mcp', 'prompt'].includes(String(candidate.catalog.surface)) && Array.isArray(candidate.catalog.actions); } diff --git a/src/project/validate.ts b/src/project/validate.ts index 923b481..e787ec9 100644 --- a/src/project/validate.ts +++ b/src/project/validate.ts @@ -45,8 +45,8 @@ export async function checkConfig( const { target, benchmark, optimize } = cfg; - if (target.surface !== 'sdk' && target.surface !== 'cli' && target.surface !== 'mcp') { - err('invalid-surface', 'target.surface', '"target.surface" must be sdk, cli, or mcp'); + if (target.surface !== 'sdk' && target.surface !== 'cli' && target.surface !== 'mcp' && target.surface !== 'prompt') { + err('invalid-surface', 'target.surface', '"target.surface" must be sdk, cli, mcp, or prompt'); } if (target.skill !== undefined) { @@ -109,6 +109,12 @@ export async function checkConfig( } } + if (target.surface === 'prompt') { + if (target.skill === undefined) { + err('missing-prompt-skill', 'target.skill', 'Prompt targets require target.skill (path to the markdown file)'); + } + } + if (target.discovery) { if (target.discovery.mode && target.discovery.mode !== 'auto' && target.discovery.mode !== 'manifest') { err('invalid-discovery-mode', 'target.discovery.mode', '"target.discovery.mode" must be auto or manifest'); diff --git a/src/tasks/generate.ts b/src/tasks/generate.ts index 4aaa893..2536df8 100644 --- a/src/tasks/generate.ts +++ b/src/tasks/generate.ts @@ -31,6 +31,33 @@ export async function generateCandidateTasks( function buildPrompt(surface: DiscoveredTaskSurface, config: TaskGeneratorConfig): string { const clampedMax = Math.max(1, Math.floor(config.maxTasks)); + if (surface.snapshot.surface === 'prompt') { + return [ + 'Generate benchmark evaluation tasks for a prompt/skill document.', + 'These tasks will be evaluated by content quality, not action matching.', + '', + 'Return a JSON object with EXACTLY this shape:', + '{"tasks":[{"id":"string","prompt":"string","expected_actions":[]}]}', + '', + 'RULES:', + '- Each task has EXACTLY three keys: id, prompt, expected_actions.', + '- expected_actions MUST always be an empty array [].', + '- id: short snake_case identifier (e.g. "deploy_service_to_staging").', + '- prompt: ask the model to perform a realistic task from the skill.', + `- Produce at most ${clampedMax} tasks. Seed: ${config.seed}.`, + '', + 'Full SKILL.md:', + '---BEGIN SKILL---', + surface.skillMarkdown, + '---END SKILL---', + '', + 'Discovered prompt surface snapshot (capabilities for reference):', + '---BEGIN SURFACE SNAPSHOT---', + JSON.stringify(surface.snapshot, null, 2), + '---END SURFACE SNAPSHOT---', + ].join('\n'); + } + return [ `Generate benchmark tasks for a ${surface.snapshot.surface} callable surface.`, '', diff --git a/src/tasks/ground.ts b/src/tasks/ground.ts index 010bd60..ba255a3 100644 --- a/src/tasks/ground.ts +++ b/src/tasks/ground.ts @@ -11,7 +11,7 @@ export function groundTasks(tasks: GeneratedTask[], snapshot: SurfaceSnapshot): const actions = new Map(snapshot.actions.map((action) => [action.name, action])); for (const task of tasks) { - const rejection = getRejectionReason(task, seenIds, actions); + const rejection = getRejectionReason(task, seenIds, actions, snapshot.surface); if (rejection) { rejected.push({ task, reason: rejection }); continue; @@ -28,12 +28,21 @@ function getRejectionReason( task: GeneratedTask, seenIds: Set, actions: Map, + surface: SurfaceSnapshot['surface'], ): string | null { const expectedActions = task.expected_actions ?? task.expected_tools ?? []; if (seenIds.has(task.id)) { return `duplicate task id "${task.id}"`; } + // Prompt surface tasks must have expected_actions: [] — evaluated on content, not tool calls. + if (surface === 'prompt') { + if (expectedActions.length > 0) { + return `prompt task "${task.id}" must have empty expected_actions, got ${expectedActions.length}`; + } + return null; + } + if (!Array.isArray(expectedActions) || expectedActions.length === 0) { return `task "${task.id}" has empty expected_actions`; } diff --git a/src/tasks/index.ts b/src/tasks/index.ts index 2313494..193d849 100644 --- a/src/tasks/index.ts +++ b/src/tasks/index.ts @@ -1,11 +1,11 @@ import { discoverTaskSurface } from './discover.js'; import { freezeTaskArtifacts } from './freeze.js'; -import { generateCandidateTasksWithCoverage } from './generate.js'; +import { generateCandidateTasks, generateCandidateTasksWithCoverage } from './generate.js'; import { groundTasks } from './ground.js'; import { resolveScope } from './scope.js'; import { computeCoverage } from './coverage.js'; -import type { GenerateTasksForProjectResult, TaskGeneratorDeps } from './types.js'; +import type { GenerateTasksForProjectResult, GeneratedTask, TaskGeneratorDeps } from './types.js'; import { buildSurfaceSnapshot } from '../project/index.js'; import type { ResolvedProjectConfig } from '../project/types.js'; import type { SurfaceSnapshotAction } from '../project/types.js'; @@ -49,7 +49,7 @@ export async function generateTasksForProject( console.log(`[optimize] Scope filter: ${inScope.length} in scope, ${outOfScope.length} out of scope.`); const maxTasks = params.maxTasks; - if (maxTasks < inScope.length) { + if (surface.snapshot.surface !== 'prompt' && maxTasks < inScope.length) { throw new Error( `benchmark.taskGeneration.maxTasks (${maxTasks}) is smaller than in-scope action count (${inScope.length}). ` + `Raise maxTasks in ${params.configPath} or tighten target.scope.exclude.`, @@ -70,13 +70,25 @@ export async function generateTasksForProject( // Coverage matching only reads action.name so key=name is always correct here. const inScopeActions = inScope.map((a) => ({ key: a.name, ...a })); const outOfScopeActions = outOfScope.map((a) => ({ key: a.name, ...a })); - const { tasks: generated } = await generateCandidateTasksWithCoverage( - filteredSurface, - { maxTasks: params.maxTasks, seed: params.seed }, - params.deps, - inScopeActions, - outOfScopeActions, - ); + + let generated: GeneratedTask[]; + if (filteredSurface.snapshot.surface === 'prompt') { + // Prompt surface: tasks have expected_actions:[] and are evaluated on content. + // Coverage enforcement does not apply — skip the retry/coverage-gap loop. + generated = await generateCandidateTasks( + filteredSurface, + { maxTasks: params.maxTasks, seed: params.seed }, + params.deps, + ); + } else { + ({ tasks: generated } = await generateCandidateTasksWithCoverage( + filteredSurface, + { maxTasks: params.maxTasks, seed: params.seed }, + params.deps, + inScopeActions, + outOfScopeActions, + )); + } console.log(`[optimize] Model proposed ${generated.length} tasks.`); console.log('[optimize] Grounding generated tasks against the discovered surface snapshot...'); diff --git a/tests/fixtures/sample-skill.md b/tests/fixtures/sample-skill.md new file mode 100644 index 0000000..7343437 --- /dev/null +++ b/tests/fixtures/sample-skill.md @@ -0,0 +1,72 @@ +--- +name: deploy-service +description: A skill for deploying microservices to Kubernetes +--- + +# Deploy Service + +A comprehensive skill for deploying, validating, and monitoring microservice deployments. + +## Phase 1 — Requirements Discovery + +Ask clarifying questions about the deployment target until you have enough information: + +1. Which environment? (staging, production, canary) +2. Which service name and version? +3. Any special resource limits or scaling requirements? + +If the user specifies "production", require explicit confirmation before proceeding. +When the user says "canary", ask for the traffic percentage split. + +## Phase 2 — Manifest Generation + +Generate the Kubernetes manifests based on gathered requirements: + +```yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: + namespace: +spec: + replicas: + template: + spec: + containers: + - name: + image: /: + resources: + limits: + cpu: + memory: +``` + +1. Create the Deployment resource +2. Create the Service resource +3. Create the HorizontalPodAutoscaler if scaling is requested +4. Apply namespace-specific overrides + +Do not include deprecated API versions. +Never use `latest` as an image tag. + +## Phase 3 — Validation and Testing + +Run pre-deployment checks to ensure correctness: + +```bash +kubectl apply --dry-run=server -f manifests/ +kubectl diff -f manifests/ +``` + +Either run a smoke test suite or skip if the user explicitly opts out. + +Verify the following before proceeding: +- All container images exist in the registry +- Resource limits are within cluster quotas +- No conflicting service names in the target namespace + +| Check | Tool | Pass Criteria | +|-------|------|---------------| +| Image exists | crane digest | Exit code 0 | +| Quota check | kubectl describe quota | Used < Limit | +| Name conflict | kubectl get svc | Not found | diff --git a/tests/smoke-discovery-prompt.ts b/tests/smoke-discovery-prompt.ts new file mode 100644 index 0000000..2145104 --- /dev/null +++ b/tests/smoke-discovery-prompt.ts @@ -0,0 +1,321 @@ +/** + * Smoke tests for prompt surface discovery (discovery/prompt.ts). + * Mirrors the structure of smoke-discovery-mcp.ts. + */ + +import { readFileSync } from 'node:fs'; +import { resolve } from 'node:path'; + +import { + discoverPromptSurfaceFromContent, + discoverPromptSurfaceFromSources, +} from '../src/discovery/prompt.js'; +import { discoverPromptCapabilities } from '../src/project/discover-prompt.js'; + +let passed = 0; +let failed = 0; + +async function test(name: string, fn: () => Promise | void) { + try { + await fn(); + passed++; + console.log(` + ${name}`); + } catch (error: any) { + failed++; + console.log(` - ${name}`); + console.log(` ${error.message}`); + } +} + +function assert(condition: boolean, message: string): void { + if (!condition) { + throw new Error(`Assertion failed: ${message}`); + } +} + +function assertEqual(actual: T, expected: T, message: string): void { + if (actual !== expected) { + throw new Error(`${message}: expected ${JSON.stringify(expected)}, got ${JSON.stringify(actual)}`); + } +} + +// --------------------------------------------------------------------------- +// Inline sample SKILL.md for unit-level tests +// --------------------------------------------------------------------------- + +const sampleSkill = `--- +name: test-skill +description: A test skill for benchmarking +--- + +# Test Skill + +## Phase 1 — Requirements Discovery + +Ask clarifying questions until the stopping condition is met. + +## Phase 2 — Implementation + +Write the code changes: + +\`\`\`go +func handleRequest(ctx context.Context, req *Request) (*Response, error) { + // implementation +} +\`\`\` + +## Phase 3 — Testing + +Create table-driven tests with named fixtures. +`; + +// --------------------------------------------------------------------------- +// Tests +// --------------------------------------------------------------------------- + +console.log('\n=== Prompt Discovery Smoke Tests ===\n'); + +await test('discovers 3 phases from sample SKILL.md', () => { + const snapshot = discoverPromptSurfaceFromContent(sampleSkill); + + assertEqual(snapshot.surface, 'prompt', 'surface should be prompt'); + assertEqual(snapshot.phases.length, 3, 'should discover 3 phases'); + + const phaseNames = snapshot.phases.map((p) => p.name); + assert(phaseNames.includes('Requirements Discovery'), 'should include Requirements Discovery phase'); + assert(phaseNames.includes('Implementation'), 'should include Implementation phase'); + assert(phaseNames.includes('Testing'), 'should include Testing phase'); +}); + +await test('each phase has a name and description', () => { + const snapshot = discoverPromptSurfaceFromContent(sampleSkill); + + for (const phase of snapshot.phases) { + assert(phase.name.length > 0, `phase should have a non-empty name, got: "${phase.name}"`); + assert(phase.description.length > 0, `phase "${phase.name}" should have a non-empty description`); + } +}); + +await test('code block detected in Phase 2 (Implementation)', () => { + const snapshot = discoverPromptSurfaceFromContent(sampleSkill); + const implPhase = snapshot.phases.find((p) => p.name === 'Implementation'); + + assert(implPhase !== undefined, 'Implementation phase should exist'); + assert(implPhase!.hasCodeBlocks, 'Implementation phase should have code blocks'); + + // Phase 1 and Phase 3 should NOT have code blocks + const discoveryPhase = snapshot.phases.find((p) => p.name === 'Requirements Discovery'); + assert(discoveryPhase !== undefined, 'Requirements Discovery phase should exist'); + assert(!discoveryPhase!.hasCodeBlocks, 'Requirements Discovery phase should NOT have code blocks'); +}); + +await test('empty skill returns empty capabilities', () => { + const snapshot = discoverPromptSurfaceFromContent(''); + + assertEqual(snapshot.phases.length, 0, 'empty skill should have 0 phases'); + assertEqual(snapshot.capabilities.length, 0, 'empty skill should have 0 capabilities'); +}); + +await test('skill with no phases returns capabilities from instructions', () => { + const noPhaseSkill = `--- +name: simple-skill +--- + +# Simple Skill + +- Validate the input format +- Transform data to JSON +- Send the result to the API +`; + + const snapshot = discoverPromptSurfaceFromContent(noPhaseSkill); + + assertEqual(snapshot.phases.length, 0, 'should have 0 phases'); + assert(snapshot.capabilities.length > 0, 'should have capabilities extracted from bullet points'); + assert( + snapshot.capabilities.every((c) => c.source === 'instruction'), + 'all capabilities should have source "instruction"', + ); +}); + +await test('phases with capabilities have source "phase"', () => { + const snapshot = discoverPromptSurfaceFromContent(sampleSkill); + + assertEqual(snapshot.capabilities.length, 3, 'should have 3 capabilities (one per phase)'); + assert( + snapshot.capabilities.every((c) => c.source === 'phase'), + 'all capabilities should have source "phase" when phases exist', + ); +}); + +await test('discoverPromptSurfaceFromSources reads fixture file', () => { + const fixturePath = resolve(process.cwd(), 'tests/fixtures/sample-skill.md'); + const snapshot = discoverPromptSurfaceFromSources([fixturePath]); + + assertEqual(snapshot.surface, 'prompt', 'surface should be prompt'); + assertEqual(snapshot.phases.length, 3, 'fixture should have 3 phases'); + assertEqual(snapshot.sources.length, 1, 'should track one source file'); + + // Verify structural detection on the fixture + const phase2 = snapshot.phases.find((p) => p.name === 'Manifest Generation'); + assert(phase2 !== undefined, 'fixture should have Manifest Generation phase'); + assert(phase2!.hasCodeBlocks, 'Manifest Generation should have code blocks'); + assert(phase2!.hasNumberedSteps, 'Manifest Generation should have numbered steps'); + + const phase1 = snapshot.phases.find((p) => p.name === 'Requirements Discovery'); + assert(phase1 !== undefined, 'fixture should have Requirements Discovery phase'); + assert(phase1!.hasDecisionPoints, 'Requirements Discovery should have decision points'); + assert(phase1!.hasNumberedSteps, 'Requirements Discovery should have numbered steps'); +}); + +await test('discoverPromptSurfaceFromSources throws on missing file', () => { + let threw = false; + try { + discoverPromptSurfaceFromSources(['/tmp/nonexistent-skill-12345.md']); + } catch (error: any) { + threw = true; + assert( + error.message.includes('does not exist'), + `error should mention missing path, got: ${error.message}`, + ); + } + assert(threw, 'should throw on missing source file'); +}); + +await test('frontmatter is stripped before parsing phases', () => { + const skillWithFrontmatter = `--- +name: fm-test +description: frontmatter test +custom_field: true +--- + +## Phase 1 — Setup + +Initialize the environment. +`; + + const snapshot = discoverPromptSurfaceFromContent(skillWithFrontmatter); + assertEqual(snapshot.phases.length, 1, 'should discover 1 phase after stripping frontmatter'); + assertEqual(snapshot.phases[0].name, 'Setup', 'phase name should be "Setup"'); +}); + +await test('decision points detected from conditional keywords', () => { + const skillWithDecisions = `--- +name: decision-test +--- + +## Phase 1 — Routing + +If the request is a GET, return cached data. +When the cache is stale, fetch fresh data. +Otherwise fall back to the default response. +`; + + const snapshot = discoverPromptSurfaceFromContent(skillWithDecisions); + assertEqual(snapshot.phases.length, 1, 'should discover 1 phase'); + assert(snapshot.phases[0].hasDecisionPoints, 'phase should detect decision points'); +}); + +await test('stripFrontmatter: does not terminate early when YAML value contains ---', () => { + // YAML value "A --- B" contains ---, which indexOf finds first + const content = `---\ntitle: "A --- B"\n---\n\n## Section\nBody text.`; + const result = discoverPromptSurfaceFromContent(content); + assert(result.phases.length > 0, 'should have at least one phase'); + const phaseName = result.phases[0].name.toLowerCase(); + assert(phaseName.includes('section'), + `phase name should be "Section" but was: "${result.phases[0].name}"`); +}); + +// Bug 6: preamble contamination +await test('discoverPromptCapabilities: preamble lines do not contaminate first section body', () => { + const content = [ + '# My Skill', + 'Run this preamble step before anything else.', + '', + '## Phase 1: Do the thing', + 'Write the implementation here.', + ].join('\n'); + const actions = discoverPromptCapabilities(content); + const hasSpuriousInstruction = actions.some(a => + a.description.toLowerCase().includes('preamble'), + ); + // Should be false — preamble should not bleed into instructions + if (hasSpuriousInstruction) throw new Error('preamble content bled into discovered instructions'); +}); + +// Bug 7: frontmatter +await test('discoverPromptCapabilities: strips YAML frontmatter before scanning', () => { + const content = [ + '---', + 'run: my-skill-command', + '---', + '', + '## Phase 1: Setup', + 'Configure the environment.', + ].join('\n'); + const actions = discoverPromptCapabilities(content); + const hasFrontmatterCap = actions.some(a => + a.description.toLowerCase().includes('run: my-skill-command'), + ); + if (hasFrontmatterCap) throw new Error('frontmatter must be stripped before discovery'); + if (actions.length === 0) throw new Error('should still find Phase 1 capability'); +}); + +// Bug 4: heading-less files +await test('discoverPromptCapabilities: handles heading-less instruction-only files', () => { + const content = [ + '# Deployment Skill', + '', + 'This skill deploys services to production.', + '', + '- Ask for the service name and environment', + '- Generate the deployment manifest', + '- Validate the configuration before applying', + '- Run the deployment command', + ].join('\n'); + const actions = discoverPromptCapabilities(content); + assert(actions.length > 0, `should extract at least one capability from bullet list, got ${actions.length}`); +}); + +await test('discoverPromptCapabilities: returns valid ActionDefinition[] shape', () => { + const content = [ + '## Phase 1: Requirements', + 'Ask the user clarifying questions.', + 'Generate a requirements document.', + '', + '## Phase 2: Implementation', + 'Write the implementation code.', + ].join('\n'); + + const actions = discoverPromptCapabilities(content); + assert(Array.isArray(actions), 'should return an array'); + assert(actions.length >= 2, `expected ≥2 capabilities, got ${actions.length}`); + + for (const action of actions) { + assert(typeof action.name === 'string' && action.name.length > 0, 'name must be non-empty'); + assert(typeof action.description === 'string', 'description must be string'); + assert(Array.isArray(action.args), 'args must be array'); + } +}); + +// Regression guard: both discovery modules exist intentionally for different layers. +// This test documents the interface difference so future refactors stay aware. +await test('production path (discover-prompt) vs standalone path (discovery/prompt) have different shapes', () => { + const content = '## Phase 1: Do the thing\nRun this step.'; + + // Production path: ActionDefinition[] — has args field, no source field + const productionResult = discoverPromptCapabilities(content); + assert(productionResult.length > 0, 'production path must return at least one capability'); + assert('args' in productionResult[0], 'production path must have args field'); + + // Standalone discovery path: PromptDiscoverySnapshot — capabilities have source field, not args + const discoveryResult = discoverPromptSurfaceFromContent(content); + if (discoveryResult.capabilities.length > 0) { + const cap = discoveryResult.capabilities[0]; + assert('source' in cap, 'discovery path capability must have source field'); + assert(!('args' in cap), 'discovery path must NOT have args field'); + } +}); + +console.log(`\n${passed} passed, ${failed} failed\n`); +process.exit(failed > 0 ? 1 : 0); diff --git a/tests/smoke-prompt-evaluator.ts b/tests/smoke-prompt-evaluator.ts new file mode 100644 index 0000000..c8170c2 --- /dev/null +++ b/tests/smoke-prompt-evaluator.ts @@ -0,0 +1,349 @@ +/** + * Smoke tests for the prompt surface evaluator (benchmark/prompt-evaluator.ts). + * Mirrors the structure of smoke-scoring.ts and smoke-llm.ts. + */ + +import { + evaluatePromptResponse, + type PromptEvaluationCriteria, +} from '../src/benchmark/prompt-evaluator.js'; + +let passed = 0; +let failed = 0; + +async function test(name: string, fn: () => Promise | void) { + try { + await fn(); + passed++; + console.log(` + ${name}`); + } catch (error: any) { + failed++; + console.log(` - ${name}`); + console.log(` ${error.message}`); + } +} + +function assert(condition: boolean, message: string): void { + if (!condition) { + throw new Error(`Assertion failed: ${message}`); + } +} + +function assertEqual(actual: T, expected: T, message: string): void { + if (actual !== expected) { + throw new Error(`${message}: expected ${JSON.stringify(expected)}, got ${JSON.stringify(actual)}`); + } +} + +function assertInRange(actual: number, min: number, max: number, message: string): void { + if (actual < min || actual > max) { + throw new Error(`${message}: expected ${actual} to be in [${min}, ${max}]`); + } +} + +console.log('\n=== Prompt Evaluator Smoke Tests ===\n'); + +// --------------------------------------------------------------------------- +// Required sections +// --------------------------------------------------------------------------- + +await test('required sections present -> high section score', () => { + const criteria: PromptEvaluationCriteria = { + requiredSections: ['Overview', 'Implementation'], + }; + + const response = `## Overview + +Here is the overview of the system. + +## Implementation + +The implementation uses a factory pattern. +`; + + const result = evaluatePromptResponse(response, criteria); + assertEqual(result.categoryScores.sections, 1.0, 'all required sections present should yield section score 1.0'); + assertInRange(result.score, 0.8, 1.0, 'overall score should be high when sections are all present'); +}); + +await test('required sections missing -> low section score', () => { + const criteria: PromptEvaluationCriteria = { + requiredSections: ['Overview', 'Implementation', 'Testing'], + }; + + const response = `## Overview + +Here is the overview. + +Some text about implementation without a heading. +`; + + const result = evaluatePromptResponse(response, criteria); + // Only 1 of 3 sections found + assertInRange(result.categoryScores.sections, 0.3, 0.4, 'only 1/3 sections found should yield ~0.33'); + assert(result.details.some((d) => d.includes('MISSING')), 'details should mention missing sections'); +}); + +// --------------------------------------------------------------------------- +// Format patterns +// --------------------------------------------------------------------------- + +await test('format pattern match -> positive format score', () => { + const criteria: PromptEvaluationCriteria = { + formatPatterns: [ + { name: 'has-yaml-key', pattern: '^\\w+:\\s+' }, + { name: 'has-heading', pattern: '^#+\\s+' }, + ], + }; + + const response = `# Configuration + +name: my-service +replicas: 3 +`; + + const result = evaluatePromptResponse(response, criteria); + assertEqual(result.categoryScores.format, 1.0, 'both format patterns matched should yield 1.0'); +}); + +await test('format pattern mismatch -> low format score', () => { + const criteria: PromptEvaluationCriteria = { + formatPatterns: [ + { name: 'has-json', pattern: '\\{[\\s\\S]*"\\w+"\\s*:' }, + { name: 'has-array', pattern: '\\[\\s*\\{' }, + ], + }; + + const response = 'Just plain text without any JSON or arrays.'; + + const result = evaluatePromptResponse(response, criteria); + assertEqual(result.categoryScores.format, 0, 'no format patterns matched should yield 0'); +}); + +// --------------------------------------------------------------------------- +// Forbidden keywords +// --------------------------------------------------------------------------- + +await test('forbidden keywords present -> keyword score penalty', () => { + const criteria: PromptEvaluationCriteria = { + forbiddenKeywords: ['deprecated', 'latest'], + }; + + const response = 'Use the deprecated API with the latest image tag.'; + + const result = evaluatePromptResponse(response, criteria); + assertEqual(result.categoryScores.keywords, 0, 'both forbidden keywords present should yield 0'); + assert( + result.checks.filter((c) => c.check.startsWith('forbidden:') && !c.passed).length === 2, + 'should have 2 failed forbidden checks', + ); +}); + +await test('forbidden keywords absent -> keyword score 1.0', () => { + const criteria: PromptEvaluationCriteria = { + forbiddenKeywords: ['deprecated', 'hack', 'workaround'], + }; + + const response = 'A clean implementation using standard patterns.'; + + const result = evaluatePromptResponse(response, criteria); + assertEqual(result.categoryScores.keywords, 1.0, 'no forbidden keywords should yield 1.0'); +}); + +// --------------------------------------------------------------------------- +// Code blocks (structural) +// --------------------------------------------------------------------------- + +await test('code blocks detected -> structural score boost', () => { + const criteria: PromptEvaluationCriteria = { + hasCodeBlocks: true, + }; + + const response = `Here is the implementation: + +\`\`\`go +func main() { + fmt.Println("hello") +} +\`\`\` +`; + + const result = evaluatePromptResponse(response, criteria); + assertEqual(result.categoryScores.structure, 1.0, 'expected code blocks found should yield structure 1.0'); +}); + +await test('code blocks expected but missing -> structural score drop', () => { + const criteria: PromptEvaluationCriteria = { + hasCodeBlocks: true, + }; + + const response = 'Just text, no code blocks at all.'; + + const result = evaluatePromptResponse(response, criteria); + assertEqual(result.categoryScores.structure, 0, 'expected code blocks missing should yield structure 0'); +}); + +// --------------------------------------------------------------------------- +// Empty response +// --------------------------------------------------------------------------- + +await test('empty response -> score 0', () => { + const criteria: PromptEvaluationCriteria = { + requiredSections: ['Overview'], + forbiddenKeywords: ['error'], + }; + + // With required sections set, an empty string scores 0 on section checks. + const resultEmpty = evaluatePromptResponse('', criteria); + // Empty string: heading regex won't match, so sections score = 0 + assertInRange(resultEmpty.categoryScores.sections, 0, 0, 'empty response should have section score 0'); + + const resultWhitespace = evaluatePromptResponse(' \n \n ', criteria); + assertInRange(resultWhitespace.categoryScores.sections, 0, 0, 'whitespace-only response should have section score 0'); +}); + +// --------------------------------------------------------------------------- +// All criteria met -> score 1.0 +// --------------------------------------------------------------------------- + +await test('all criteria met -> score 1.0', () => { + const criteria: PromptEvaluationCriteria = { + requiredSections: ['Setup', 'Deploy'], + formatPatterns: [{ name: 'has-heading', pattern: '^##\\s+' }], + requiredKeywords: ['kubernetes', 'namespace'], + forbiddenKeywords: ['deprecated'], + hasCodeBlocks: true, + hasNumberedList: true, + }; + + const response = `## Setup + +Configure the kubernetes cluster and target namespace. + +1. Create the namespace +2. Apply RBAC rules + +## Deploy + +Deploy the service: + +\`\`\`bash +kubectl apply -f manifests/ +\`\`\` +`; + + const result = evaluatePromptResponse(response, criteria); + assertEqual(result.categoryScores.sections, 1.0, 'all sections found'); + assertEqual(result.categoryScores.format, 1.0, 'format pattern matched'); + assertEqual(result.categoryScores.keywords, 1.0, 'all keywords present, no forbidden'); + assertEqual(result.categoryScores.structure, 1.0, 'code blocks and numbered list found'); + assertEqual(result.score, 1.0, 'overall score should be 1.0 when all criteria met'); +}); + +// --------------------------------------------------------------------------- +// Mixed criteria: partial scores +// --------------------------------------------------------------------------- + +await test('mixed criteria produce weighted intermediate score', () => { + const criteria: PromptEvaluationCriteria = { + requiredSections: ['Setup', 'Missing Section'], + requiredKeywords: ['deploy'], + forbiddenKeywords: ['hack'], + hasCodeBlocks: true, + }; + + const response = `## Setup + +We will deploy the service. + +\`\`\`bash +kubectl apply -f deploy.yaml +\`\`\` +`; + + const result = evaluatePromptResponse(response, criteria); + // Sections: 1/2 = 0.5 + assertInRange(result.categoryScores.sections, 0.49, 0.51, 'sections should be ~0.5 (1/2 found)'); + // Keywords: deploy found + hack absent = 2/2 = 1.0 + assertEqual(result.categoryScores.keywords, 1.0, 'keywords should be 1.0'); + // Structure: code blocks present = 1.0 + assertEqual(result.categoryScores.structure, 1.0, 'structure should be 1.0'); + // Overall: weighted mix, should be between 0.5 and 1.0 + assertInRange(result.score, 0.5, 1.0, 'overall score should be between 0.5 and 1.0'); +}); + +// --------------------------------------------------------------------------- +// Numbered list and table detection +// --------------------------------------------------------------------------- + +await test('numbered list detection works', () => { + const criteria: PromptEvaluationCriteria = { + hasNumberedList: true, + }; + + const withList = `Steps: + +1. First step +2. Second step +3. Third step +`; + + const withoutList = 'No numbered items here, just prose.'; + + const resultWith = evaluatePromptResponse(withList, criteria); + assertEqual(resultWith.categoryScores.structure, 1.0, 'numbered list found should yield 1.0'); + + const resultWithout = evaluatePromptResponse(withoutList, criteria); + assertEqual(resultWithout.categoryScores.structure, 0, 'no numbered list should yield 0'); +}); + +await test('table detection works', () => { + const criteria: PromptEvaluationCriteria = { + hasTable: true, + }; + + const withTable = `| Name | Value | +|------|-------| +| foo | bar | +`; + + const withoutTable = 'No table here.'; + + const resultWith = evaluatePromptResponse(withTable, criteria); + assertEqual(resultWith.categoryScores.structure, 1.0, 'table found should yield 1.0'); + + const resultWithout = evaluatePromptResponse(withoutTable, criteria); + assertEqual(resultWithout.categoryScores.structure, 0, 'no table should yield 0'); +}); + +// --------------------------------------------------------------------------- +// minLength check +// --------------------------------------------------------------------------- + +await test('minLength enforced as format check', () => { + const criteria: PromptEvaluationCriteria = { + minLength: 100, + }; + + const shortResponse = 'Too short.'; + const longResponse = 'A'.repeat(150) + ' — this response is well over the minimum length requirement and should pass the check.'; + + const resultShort = evaluatePromptResponse(shortResponse, criteria); + assertEqual(resultShort.categoryScores.format, 0, 'short response should fail minLength'); + + const resultLong = evaluatePromptResponse(longResponse, criteria); + assertEqual(resultLong.categoryScores.format, 1.0, 'long response should pass minLength'); +}); + +// --------------------------------------------------------------------------- +// No criteria -> vacuous pass +// --------------------------------------------------------------------------- + +await test('no criteria specified -> score 1.0 (vacuous pass)', () => { + const criteria: PromptEvaluationCriteria = {}; + const result = evaluatePromptResponse('Any response at all.', criteria); + assertEqual(result.score, 1.0, 'no criteria should vacuously pass'); +}); + +console.log(`\n${passed} passed, ${failed} failed\n`); +process.exit(failed > 0 ? 1 : 0); diff --git a/tests/smoke-snapshot-prompt.ts b/tests/smoke-snapshot-prompt.ts new file mode 100644 index 0000000..8df054f --- /dev/null +++ b/tests/smoke-snapshot-prompt.ts @@ -0,0 +1,25 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { writeFileSync, mkdirSync, mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { loadActionSnapshotFile } from '../src/actions/snapshot.js'; + +await test('loadActionSnapshotFile accepts prompt surface', async () => { + const dir = mkdtempSync(join(tmpdir(), 'smoke-snapshot-prompt-')); + try { + const p = join(dir, 'snapshot.json'); + writeFileSync(p, JSON.stringify({ + version: 1, + catalog: { + surface: 'prompt', + actions: [], + }, + })); + // Must not throw — previously threw "catalog.surface must be one of sdk|cli|mcp" + const result = loadActionSnapshotFile(p); + assert.equal(result.catalog.surface, 'prompt'); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); From c898bd341807eb1f0ca3a6ba4329104092e2c4c8 Mon Sep 17 00:00:00 2001 From: dmn Date: Wed, 15 Apr 2026 23:00:33 -0700 Subject: [PATCH 6/7] Refactoring & Removal of legacy code (#15) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * refactor: DRY deduplication across LLM format handlers and Pi task helpers Extract shared retry logic (isAbortError, isRetryableError, sleep), tool-result truncation, and agent-loop accumulator helpers into src/benchmark/llm/shared.ts, eliminating identical code that existed in both openai-format.ts and anthropic-format.ts. Extract the Pi completeSimple call sequence (timeout setup, error check, text block extraction) into src/tasks/pi-simple-complete.ts, used by both default-pi-generator.ts and default-pi-critic.ts to replace ~40 lines of near-identical implementation in each file. Net: -129 lines of duplication, no behavior changes, all tests pass. Co-Authored-By: Claude Sonnet 4.6 * refactor(types): consolidate duplicate type definitions - DiscoveredActionArg (discovery/types) was identical to ActionArgSchema (actions/types); replaced with a re-export alias to eliminate the copy - BenchmarkSurface now aliases ActionSurface instead of re-declaring the same 'sdk' | 'cli' | 'mcp' union literal - Removed dead SurfaceActionArg alias from benchmark/types (never imported) - Removed dead SurfaceSnapshotArg alias from project/types and project/index (exported but never consumed) - Replaced all inline import('...').PiAuthMode dynamic-import type refs with proper named import type statements in pi-format, models, coding-orchestrator, benchmark-agent, default-pi-critic, default-pi-generator Co-Authored-By: Claude Sonnet 4.6 * chore: remove unused code found by knip - Delete src/runtime/pi/benchmark-agent.ts (createReadOnlyBenchmarkModel was dead — exported but never imported) - Delete src/discovery/index.ts and src/optimizer/mutation/types.ts (barrel re-exports with no consumers) - Remove requireApiKeyFromEnv from auth.ts (never called internally or imported from outside) - Remove 5 unused re-export lines from benchmark/extractors/index.ts (extractCodeBlock, extractShellBlock etc. — tests import directly from sub-files) - Unexport 11 internal-only types/interfaces (MAX_TOOL_RESULT_CHARS, buildConfigFromAnswers, LLMClient, ToolNameAliasCodec, PromptSurface/Options, DoctorOptions, DetectedProject, BenchmarkAdapterRunOptions/Result, FeedbackPackage, RawExtraction, PiSimpleCompleteOptions/Input) - Remove @mariozechner/pi-agent-core from package.json dependencies (transitive dep, never directly imported) - Add smoke-actions.ts to the npm test script (17 passing tests were orphaned from the suite) Co-Authored-By: Claude Sonnet 4.6 * fix(deps): break circular dependency between actions/discover and benchmark/config Move loadCliCommands and loadMcpTools from benchmark/config.ts into a new actions/loaders.ts file, resolving the cycle: actions/discover → benchmark/config → project → project/snapshot → actions/discover benchmark/config.ts re-exports both functions for backward compatibility. Co-Authored-By: Claude Sonnet 4.6 * refactor(types): replace weak types with precise types throughout - Replace `Model` with `Model` and specific API literal types (`Model<'openai-completions'>`, `Model<'openai-codex-responses'>`) in models.ts and pi-format.ts - Replace `(err as any).status` with typed intersection cast `Error & { status: number }` in anthropic-format.ts and openai-format.ts - Remove redundant `as unknown[]` cast on `session.state.messages` and import `AgentMessage` from `@mariozechner/pi-agent-core` in pi-coding.ts; update `extractLatestAssistantText` and `extractToolActivity` parameter types accordingly - Remove unnecessary double casts `(c as unknown as { method: string }).method` in failure-details.ts and passing-failing-diff.ts — `ExtractedCall` is `ActionAttempt` which already has `method: string` and `args` - Remove `as any` from `Type.Unsafe(...)` call in pi-format.ts — the argument type is compatible with `UnsafeOptions` (SchemaOptions index signature) - Fix `catch (error: any)` to `catch (error)` in two snapshot files, using `instanceof Error` guard to access `.message` Co-Authored-By: Claude Sonnet 4.6 * fix: remove unnecessary optional chaining on non-optional reportConfig reportConfig is always defined (report.config is non-optional per BenchmarkReport). The ?. on reportConfig before .outputDir was defensive against a value that TypeScript knows can never be null. Co-Authored-By: Claude Sonnet 4.6 * chore: remove deprecated and legacy code paths - Remove toLegacyOptimizeManifest (never called anywhere in src or tests) - Remove deprecated SdkSurfaceConfig fields: classes, functions, functionReturns, methods - Remove dead config.sdk?.methods fallback in runner.ts (no config ever sets methods) Co-Authored-By: Claude Sonnet 4.6 * chore: remove AI slop comments and stubs Removed or rewrote ~75 comments across 15 files: - Dropped "// NEW:", "// Transitional alias", numbered step comments (1-14 in runner.ts), and other in-motion migration language - Removed redundant comments that restated the immediately-following code - Replaced multi-line comment pairs with single consolidated explanations where the context was non-obvious and worth preserving - Kept all comments explaining invariants, non-obvious behavior, or security rationale (e.g. git injection safety, OpenAI dot-format exemption) Co-Authored-By: Claude Sonnet 4.6 * chore: remove backwards-compatibility shims — SemVer allows breaking changes - Remove LEGACY_PROJECT_CONFIG_NAME and skill-benchmark.json detection in load.ts - Remove legacy-config-name warning from validate.ts (old filename alongside new one) - Remove E_LEGACY_CONFIG error definition from errors.ts and its docs entry - Remove DiscoveredActionArg deprecated alias from discovery/types.ts; update all four discovery modules (cli, mcp, sdk, optique) to import ActionArgSchema directly - Remove CodeModeConfig, McpModeConfig, ExpectedTool, ToolMatch backward-compat type aliases from benchmark/types.ts and benchmark/index.ts - Remove toolMatches dual-field from TaskResult; make actionMatches required - Remove unnecessaryCalls/hallucinatedCalls alias fields from TaskResult.metrics; consolidate to unnecessaryActions/hallucinatedActions throughout - Remove method? alias from ExpectedAction; make name required - Remove expected_tools? alias from TaskDefinition; make expected_actions required - Simplify getExpectedActionName and getExpectedActions helper functions - Remove deprecated extractFromCode function from code-analyzer.ts - Update all internal consumers (optimizer, evaluator, tasks, benchmark) to use canonical field names without the ?? fallback patterns - Update all test files to use canonical field names and types - Rewrite smoke-code.ts Group 2 tests for extractAllFromCode (the modern API) Co-Authored-By: Claude Sonnet 4.6 * fix: address Copilot review issues from simplify PR - Restore @mariozechner/pi-agent-core as explicit dependency (still imported as a type in src/optimizer/mutation/pi-coding.ts; relying on transitive availability is fragile) - Fix piSimpleComplete: throw unconditionally when stopReason === 'error' (previously silent fallthrough when errorMessage was falsy) and throw with content-type diagnostic when the response has no text blocks - Remove redundant empty-text guard in default-pi-generator.ts now that piSimpleComplete handles that case itself - Replace unsafe `as` casts for verify/expected_fetches in config.ts with Array.isArray guards before assignment Co-Authored-By: Claude Sonnet 4.6 * fix: address secondary reviewer issues - loadReport: detect pre-1.x report format (toolMatches field) and throw a clear error instead of letting callers hit silent undefined-access later - loadProjectConfig: when skill-optimizer.json is missing, check for skill-benchmark.json and surface a targeted rename hint instead of the generic "config not found" error - CHANGELOG: add BREAKING CHANGES section documenting all removed public API exports and renamed TaskResult fields Co-Authored-By: Claude Sonnet 4.6 * chore: rename minOverallPassDelta, fix mock repo configs, remove ghost repo names - Part F: rename `minOverallPassDelta` → `minImprovement` in optimizer types, loop resolution, adapters, and tests to unify with the public ProjectOptimizeConfig field name - Part G: fix dot-notation model IDs (claude-sonnet-4.6 → 4-6, gpt-5.4 → 5-4) in all three mock-repo skill-optimizer.json files; delete stale legacy benchmark.config.json and optimize.config.json from mcp-tracker-demo (superseded by skill-optimizer.json) - Part H: remove ghost names sdk-demo, cli-demo, mcp-demo from MOCK_REPO_NAMES and update materialize-mock-repo.ts usage string to only list the three templates that actually exist Co-Authored-By: Claude Sonnet 4.6 * chore: remove expected_tools/method input shims and benchmark.tasks deprecated field Co-Authored-By: Claude Sonnet 4.6 * fix: reapply Copilot comment fixes that were lost before agent commits - loadProjectConfig: only check for skill-benchmark.json when configPath was not explicitly supplied; use dirname() for clarity - default-pi-critic: catch no-text-blocks error and return '[]' (graceful degradation) while re-throwing real provider errors Co-Authored-By: Claude Sonnet 4.6 * chore: rename matchTools→matchActions, remove pre-1.x report guard, clean straddling test comments - Renamed exported matchTools → matchActions in evaluator.ts; updated internal vars (toolMatches→actionMatches, unnecessaryCalls→unnecessaryActions, hallucinatedCalls→hallucinatedActions) for consistency - Stripped pre-1.x schema guard from loadReport in compare.ts — old format check on 'toolMatches' field was pure backwards-compat dead code - Removed "Accept either old or new error" branching from two smoke tests; both now only assert the current validation path - Updated CLAUDE.md: manifest fallback is a supported feature, not a transitional shim Co-Authored-By: Claude Sonnet 4.6 * chore: remove skill-benchmark.json legacy filename check from loadProjectConfig The check for the old config filename was a migration hint for users upgrading from a pre-1.x version. Current config is skill-optimizer.json; drop the dead code. Co-Authored-By: Claude Sonnet 4.6 * chore: rename misleading 'legacy' test names in smoke-actions Test names said 'legacy' and 'legacy shape' but were testing current SurfaceSnapshot ↔ ActionCatalog conversion behaviour. Rename variables and assertions to remove the legacy framing. Co-Authored-By: Claude Sonnet 4.6 * chore: remove/reword remaining legacy and old language - README.md: remove skill-benchmark.json migration hint (the detection code was removed in the previous commit) - cli.ts: rename example path results/old/ → results/baseline/ (clearer) - snapshot.ts: rephrase error from "old format" to "not supported" - smoke-actions.ts: align test name and assertion strings with the updated error message; drop "legacy" framing from assertion strings Co-Authored-By: Claude Sonnet 4.6 * chore: final cleanup — remove unused exports, trivial comments, stale doc entries Unused exports: - Make extractPythonSdk/extractRustSdk/extractTypeScriptSdk unexported; they are private implementation details assigned to adapter objects in the same file, not part of the public API Trivial JSDoc removal: - shared.ts: remove one-liner comments that restate function names - reporter.ts: remove pad/center JSDoc; remove redundant generateMarkdown doc - anthropic-format.ts: remove comment restating toAnthropicTool's name - openai-format.ts: remove HTTP-method comments that add no information Docs: - README.md: fix benchmark.models description to cover all three providers (openrouter, anthropic, openai) — not OpenRouter only - CLAUDE.md: fix layer count (5, not 4); fix dot-notation validator description (skips openai/, not just openrouter/) - SKILL/references/setup.md: remove troubleshooting row for skill-benchmark.json detection that was removed last commit Co-Authored-By: Claude Sonnet 4.6 * fix: correct inaccurate CHANGELOG claim and wrong GitHub URL in CONTRIBUTING - CHANGELOG: loadReport does not validate old field names (the validation guard was intentionally removed); update wording to match actual behavior - CONTRIBUTING: fix clone URL from bucurdavid fork to fastxyz canonical repo Co-Authored-By: Claude Sonnet 4.6 * fix: repair three regressions in the generate-tasks → run workflow Issue A (freeze.ts): surface.snapshot.json was written as a plain {surface,actions} SurfaceSnapshot, which loadSurfaceSnapshotFile now rejects. Switch to writeActionSnapshotFile + fromSurfaceSnapshot so the file is written in the versioned {version,catalog} format. Issue B (shared.ts): truncateToolResult sliced to MAX_TOOL_RESULT_CHARS then appended the 16-char suffix, producing 50,016-char strings. Subtract TRUNCATED_SUFFIX.length from the slice index so the final string is exactly MAX_TOOL_RESULT_CHARS. Issue C (types/resolve/adapters): benchmark.tasks was removed from ProjectBenchmarkConfig during the simplify cleanup. freeze.ts still writes tasks: tasksPath into benchmark.generated.json, but the field was silently dropped on load and adapters.ts hard-coded tasks: '__generated__', causing the runner to throw "run generate-tasks first" even after generate-tasks had succeeded. Re-add tasks?: string to both benchmark config interfaces, thread it through resolve.ts, and use project.benchmark.tasks ?? '__generated__' in adapters.ts. Co-Authored-By: Claude Sonnet 4.6 * typos * license typo * fix(validation): restore missing-tasks guard, validate verify/expected_fetches elements, document tasks.json migration Co-Authored-By: Claude Sonnet 4.6 * docs: add prompt surface to init comment, E_INVALID_SURFACE fix text, and config-schema reference Co-Authored-By: Claude Sonnet 4.6 --------- Co-authored-by: OpenClaw Agent (basd) Co-authored-by: Claude Sonnet 4.6 --- CHANGELOG.md | 20 +++ CLAUDE.md | 6 +- LICENSE | 2 +- README.md | 6 +- SKILL/references/setup.md | 1 - docs/reference/config-schema.md | 2 +- docs/reference/errors.md | 17 +-- .../cli-taskfile-demo/skill-optimizer.json | 4 +- .../mcp-tracker-demo/benchmark.config.json | 32 ----- .../mcp-tracker-demo/optimize.config.json | 30 ---- .../mcp-tracker-demo/skill-optimizer.json | 6 +- .../sdk-counter-demo/skill-optimizer.json | 4 +- package.json | 2 +- src/actions/discover.ts | 2 +- src/actions/index.ts | 1 + src/actions/loaders.ts | 82 +++++++++++ src/actions/snapshot.ts | 4 +- src/benchmark/compare.ts | 11 -- src/benchmark/config.ts | 126 ++++------------- src/benchmark/evaluator.ts | 69 ++++----- src/benchmark/extractors/code-analyzer.ts | 34 +---- src/benchmark/extractors/index.ts | 7 - src/benchmark/extractors/sdk/python.ts | 2 +- src/benchmark/extractors/sdk/rust.ts | 2 +- src/benchmark/extractors/sdk/typescript.ts | 2 +- src/benchmark/index.ts | 4 +- src/benchmark/llm/anthropic-format.ts | 41 +++--- src/benchmark/llm/index.ts | 8 +- src/benchmark/llm/openai-format.ts | 73 ++++------ src/benchmark/llm/pi-format.ts | 17 +-- src/benchmark/llm/shared.ts | 54 ++++++++ src/benchmark/llm/tool-name-aliases.ts | 2 +- src/benchmark/prompts.ts | 2 +- src/benchmark/reporter.ts | 6 - src/benchmark/runner.ts | 23 +-- src/benchmark/scoring.ts | 2 +- src/benchmark/types.ts | 42 ++---- src/cli.ts | 23 +-- src/discovery/cli.ts | 7 +- src/discovery/index.ts | 20 --- src/discovery/mcp.ts | 5 +- src/discovery/optique.ts | 11 +- src/discovery/sdk.ts | 7 +- src/discovery/types.ts | 10 +- src/doctor/index.ts | 2 +- src/errors.ts | 15 +- src/import/extractors/py-click.ts | 1 - src/import/extractors/rs-clap.ts | 8 +- src/import/extractors/ts-commander.ts | 3 - src/init/detect-project.ts | 2 +- src/init/scaffold.ts | 2 +- src/optimizer/benchmark-adapter.ts | 4 +- src/optimizer/failure-analysis.ts | 6 +- src/optimizer/feedback/failure-details.ts | 10 +- src/optimizer/feedback/mutation-context.ts | 2 +- .../feedback/passing-failing-diff.ts | 4 +- src/optimizer/loop.ts | 16 +-- src/optimizer/materialize-mock-repo.ts | 6 +- src/optimizer/mock-repos.ts | 2 +- src/optimizer/mutation/pi-coding.ts | 14 +- src/optimizer/mutation/types.ts | 1 - src/optimizer/types.ts | 4 +- src/project/adapters.ts | 42 +----- src/project/fix.ts | 4 - src/project/index.ts | 5 +- src/project/load.ts | 10 -- src/project/schema.ts | 2 +- src/project/snapshot.ts | 35 ++--- src/project/types.ts | 2 - src/project/validate.ts | 30 +--- src/runtime/pi/auth.ts | 8 -- src/runtime/pi/benchmark-agent.ts | 14 -- src/runtime/pi/coding-orchestrator.ts | 3 +- src/runtime/pi/index.ts | 3 +- src/runtime/pi/models.ts | 13 +- src/tasks/coverage.ts | 3 +- src/tasks/default-pi-critic.ts | 62 ++++----- src/tasks/default-pi-generator.ts | 56 ++------ src/tasks/freeze.ts | 3 +- src/tasks/generate.ts | 15 +- src/tasks/ground.ts | 4 +- src/tasks/index.ts | 4 +- src/tasks/pi-simple-complete.ts | 81 +++++++++++ src/tasks/types.ts | 3 +- tests/smoke-actions.ts | 71 ++++------ tests/smoke-code.ts | 131 +++++++----------- tests/smoke-coverage.ts | 3 +- tests/smoke-discovery-mcp.ts | 21 +-- tests/smoke-e2e.ts | 2 +- tests/smoke-errors.ts | 21 --- tests/smoke-feedback.ts | 7 +- tests/smoke-generation.ts | 6 +- tests/smoke-mcp.ts | 42 +++--- tests/smoke-optimize.ts | 18 +-- tests/smoke-sdk-python.ts | 6 +- tests/smoke-sdk-rust.ts | 6 +- 96 files changed, 661 insertions(+), 1008 deletions(-) delete mode 100644 mock-repos/mcp-tracker-demo/benchmark.config.json delete mode 100644 mock-repos/mcp-tracker-demo/optimize.config.json create mode 100644 src/actions/loaders.ts create mode 100644 src/benchmark/llm/shared.ts delete mode 100644 src/discovery/index.ts delete mode 100644 src/optimizer/mutation/types.ts delete mode 100644 src/runtime/pi/benchmark-agent.ts create mode 100644 src/tasks/pi-simple-complete.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 2cc6331..b76d149 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,26 @@ ## Unreleased +### Breaking Changes + +The following public API exports have been removed. Update imports to use the canonical names: + +| Removed | Replacement | +|---------|-------------| +| `CodeModeConfig` | `SdkSurfaceConfig` | +| `McpModeConfig` | `McpSurfaceConfig` | +| `ExpectedTool` | `ExpectedAction` | +| `ToolMatch` | `ActionMatch` | +| `LEGACY_PROJECT_CONFIG_NAME` | hard-code `"skill-optimizer.json"` | +| `toLegacyOptimizeManifest` | removed with no replacement | +| `SurfaceSnapshotArg` | removed with no replacement | + +`TaskResult` fields renamed: `toolMatches` → `actionMatches`, `hallucinatedCalls` → `hallucinatedActions` (on `metrics`), `unnecessaryCalls` → `unnecessaryActions` (on `metrics`). `loadReport` does not validate old field names — old report JSON files may produce unexpected output in detail views. Re-run the benchmark to generate a current-format report. + +Existing `tasks.json` files using `expected_tools` (instead of `expected_actions`) or `method` (instead of `name`) on action entries will now fail to load with an error. Update affected task files: rename `expected_tools` to `expected_actions` and rename each action's `method` field to `name`. + +The config file `skill-benchmark.json` is no longer auto-detected. Rename it to `skill-optimizer.json`. + ### Added - **prompt surface type** — benchmark and optimize prompt templates, Claude Code skills, and agent instructions. Discovers phases and capabilities from markdown, evaluates output quality with content-based criteria. diff --git a/CLAUDE.md b/CLAUDE.md index 2439da2..ba0b03d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -4,7 +4,7 @@ `skill-optimizer` measures whether LLMs pick the right SDK methods, CLI commands, or MCP tools from docs and task prompts, and can run a benchmark-driven optimization loop over an allowed target repo. -The repo now has four important layers: +The repo has five layers: - `src/project/`: unified `skill-optimizer.json` config loading, validation, and path resolution - `src/runtime/pi/`: shared Pi auth/model/runtime helpers @@ -77,7 +77,7 @@ npx tsx src/cli.ts optimize --config ./.tmp/mock-repos/mcp-tracker-demo/skill-op - Prefer small changes in the existing architecture over broad refactors. - When updating config or project types, also update the README examples and any scaffolding in `src/benchmark/init.ts` if needed. - When changing optimizer behavior, verify both the loop and the unified project defaults still agree. -- Code-first surface discovery is now active for `sdk`, `cli`, and `mcp` via `target.discovery.sources`. Explicit manifests/declared surface metadata still exist as transitional internal fallbacks, but new public examples should prefer code-first discovery. +- Code-first surface discovery is the preferred mode for `sdk`, `cli`, and `mcp` via `target.discovery.sources`. Explicit manifest files (`target.cli.commands`, `target.mcp.tools`, `target.discovery.fallbackManifest`) are supported for projects that cannot use code-first discovery. - Be careful around mock repo references: code may support template names that are not currently present in the working tree. ## Testing Guidance @@ -102,7 +102,7 @@ openai/ — direct OpenAI API (format: "opena - `openrouter/deepseek/deepseek-v3-2` ✓ (not `deepseek-v3.2`) - `openai/gpt-5-4` ✓ (not `openai/gpt-5.4`) -`src/project/validate.ts` warns on dot-notation version segments for `openrouter/` model IDs (`model-id-bad-format`) and `src/project/fix.ts` auto-corrects them. When adding new model presets to `src/init/scaffold.ts`, `src/init/wizard.ts`, or `src/benchmark/init.ts`, always use hyphens in the model ID path. +`src/project/validate.ts` warns on dot-notation version segments for all non-`openai/` model IDs (`model-id-bad-format`) and `src/project/fix.ts` auto-corrects them. (OpenAI's own slugs use dots in some cases, so `openai/` is exempt.) When adding new model presets to `src/init/scaffold.ts`, `src/init/wizard.ts`, or `src/benchmark/init.ts`, always use hyphens in the model ID path. Display names (`name:` / `label:` fields) are human-readable and should keep dots (e.g. `'Claude Sonnet 4.6'`, `'Gemini 2.5 Flash'`). diff --git a/LICENSE b/LICENSE index a219a97..728fcf3 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2026 David Bucur +Copyright (c) 2026 Pi Squared Inc. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/README.md b/README.md index 300654a..1e65c31 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,7 @@ For direct OpenAI runs you can also use your local Codex browser login instead o **Step 1 — Scaffold config** (run from your project root): ```bash -npx skill-optimizer init cli # or: init sdk, init mcp +npx skill-optimizer init cli # or: init sdk, init mcp, init prompt ``` The wizard asks for your repo path, models to benchmark, and where your `SKILL.md` lives. It creates a `skill-optimizer/` directory: @@ -110,7 +110,7 @@ npx skill-optimizer init --answers answers.json | `target.repoPath` | Root of the project being benchmarked | Absolute or relative path to your repo | | `target.discovery.sources` | Source files to scan for callable methods/commands/tools | e.g. `["../src/index.ts"]` or `["../src/server.ts"]` | | `target.skill` | Docs file the optimizer will edit | Path to your `SKILL.md` or equivalent guidance doc | -| `benchmark.models` | Models to benchmark | Valid [OpenRouter](https://openrouter.ai/models) model IDs | +| `benchmark.models` | Models to benchmark | Model IDs with provider prefix: `openrouter//` (via OpenRouter), `anthropic/` (direct Anthropic), `openai/` (direct OpenAI) | | `benchmark.authMode` | How model auth is resolved | `env` (default), `codex`, or `auto` | ### Prompt templates / Claude Code skills @@ -208,8 +208,6 @@ export OPENROUTER_API_KEY=sk-or-... **Empty scope**: `target.scope.include` matched nothing. Check your glob patterns — remember `*` matches everything including dots. -**Legacy `skill-benchmark.json`**: Rename it to `skill-optimizer.json`. The loader will tell you if it finds the old name. - ## Contributing See [CONTRIBUTING.md](./CONTRIBUTING.md). diff --git a/SKILL/references/setup.md b/SKILL/references/setup.md index e5a38b4..e13d836 100644 --- a/SKILL/references/setup.md +++ b/SKILL/references/setup.md @@ -141,5 +141,4 @@ Your project is ready for benchmarking. Read `references/benchmark.md` for next | "Config not found" | Wrong path to `skill-optimizer.json` | Use `--config` with the full path | | "No actions discovered" | `discovery.sources` points at wrong files | Check paths are relative to `repoPath` | | "Skill file not found" | `target.skill` path is wrong | Path is relative to `repoPath` — verify it exists | -| "Unknown config format" | File is named `skill-benchmark.json` | Rename to `skill-optimizer.json` | | "repoPath not found" | Relative path resolved wrong | Use absolute path, or make it relative to config file location | diff --git a/docs/reference/config-schema.md b/docs/reference/config-schema.md index a4a60ce..0a73c2c 100644 --- a/docs/reference/config-schema.md +++ b/docs/reference/config-schema.md @@ -9,7 +9,7 @@ Paths in the config are relative to the config file location. | Field | Type | Default | Description | |---|---|---|---| | `name` | `string` | — | Human-readable project name | -| `target.surface` | `"sdk" | "cli" | "mcp"` | — | Type of callable surface | +| `target.surface` | `"sdk" | "cli" | "mcp" | "prompt"` | — | Type of callable surface | | `target.repoPath` | `string` | — | Path to the target repo (default ".") | | `target.skill` | `string | object` | — | Path to SKILL.md or { source, cache } object | | `target.discovery.mode` | `"auto" | "manifest"` | — | "auto" = code-first tree-sitter; "manifest" = use provided file only | diff --git a/docs/reference/errors.md b/docs/reference/errors.md index d1f04ef..f59f2a6 100644 --- a/docs/reference/errors.md +++ b/docs/reference/errors.md @@ -10,7 +10,7 @@ The catch-all `E_UNEXPECTED` appears if an error slips past the known list. | Code | Description | Quick fix | |---|---|---| -| `E_INVALID_SURFACE` | Invalid surface value | Set target.surface to one of: sdk, cli, mcp | +| `E_INVALID_SURFACE` | Invalid surface value | Set target.surface to one of: sdk, cli, mcp, prompt | | `E_MODELS_EMPTY` | benchmark.models is empty or missing | Add at least one model to benchmark.models, e.g.: | | `E_MODEL_ID_FORMAT` | Model ID is missing the openrouter/ prefix | Prefix all model IDs with openrouter/, e.g. openrouter/anthropic/claude-sonnet-4-6 | | `E_VERDICT_OUT_OF_RANGE` | Verdict threshold is out of range | Set benchmark.verdict.perModelFloor and targetWeightedAverage to values between 0.0 and 1.0 | @@ -24,7 +24,6 @@ The catch-all `E_UNEXPECTED` appears if an error slips past the known list. | `E_ALLOWED_PATHS_ESCAPE` | optimize.allowedPaths contains a path outside target.repoPath | All paths in optimize.allowedPaths must be inside target.repoPath | | `E_OUTPUT_DIR_NOT_WRITABLE` | benchmark.output.dir is not writable | Check directory permissions for the path set in benchmark.output.dir | | `E_MISSING_API_KEY` | API key environment variable is not set | Export your OpenRouter API key before running: export OPENROUTER_API_KEY=sk-or-... | -| `E_LEGACY_CONFIG` | Found skill-benchmark.json instead of skill-optimizer.json | Rename skill-benchmark.json to skill-optimizer.json | | `E_DISCOVERY_EMPTY` | Discovery found zero callable actions | Check that target.discovery.sources points at the right entry file | | `E_MAXTASKS_TOO_LOW` | benchmark.taskGeneration.maxTasks is less than the in-scope action count | Raise benchmark.taskGeneration.maxTasks to at least the number of in-scope actions | | `E_COVERAGE_EXHAUSTED` | Task generation could not cover all in-scope actions after 2 retry passes | Add guidance for the uncovered actions to your SKILL.md | @@ -41,8 +40,8 @@ The catch-all `E_UNEXPECTED` appears if an error slips past the known list. **Invalid surface value** **How to fix:** -- Set target.surface to one of: sdk, cli, mcp -- sdk = TypeScript/Python/Rust library, cli = command-line tool, mcp = MCP server +- Set target.surface to one of: sdk, cli, mcp, prompt +- sdk = TypeScript/Python/Rust library, cli = command-line tool, mcp = MCP server, prompt = prompt template / skill document ### `E_MODELS_EMPTY` @@ -150,14 +149,6 @@ The catch-all `E_UNEXPECTED` appears if an error slips past the known list. - Or add it to a .env file alongside your skill-optimizer.json - Get a key at https://openrouter.ai/keys -### `E_LEGACY_CONFIG` - -**Found skill-benchmark.json instead of skill-optimizer.json** - -**How to fix:** -- Rename skill-benchmark.json to skill-optimizer.json -- See CHANGELOG.md for any field renames between versions - ### `E_DISCOVERY_EMPTY` **Discovery found zero callable actions** @@ -229,4 +220,4 @@ The catch-all `E_UNEXPECTED` appears if an error slips past the known list. **How to fix:** - Check the full error message and stack trace above for details -- File an issue at https://github.com/bucurdavid/skill-optimizer/issues with the full output +- File an issue at https://github.com/fastxyz/skill-optimizer/issues with the full output diff --git a/mock-repos/cli-taskfile-demo/skill-optimizer.json b/mock-repos/cli-taskfile-demo/skill-optimizer.json index 002bd21..97016fb 100644 --- a/mock-repos/cli-taskfile-demo/skill-optimizer.json +++ b/mock-repos/cli-taskfile-demo/skill-optimizer.json @@ -15,14 +15,14 @@ "apiKeyEnv": "OPENROUTER_API_KEY", "models": [ { "id": "openrouter/openai/gpt-4o-mini", "name": "GPT-4o mini", "tier": "low" }, - { "id": "openrouter/anthropic/claude-sonnet-4.6", "name": "Claude 4.6", "tier": "mid" } + { "id": "openrouter/anthropic/claude-sonnet-4-6", "name": "Claude 4.6", "tier": "mid" } ], "verdict": { "perModelFloor": 0.6, "targetWeightedAverage": 0.7 }, "taskGeneration": { "enabled": true, "maxTasks": 10, "seed": 1, "outputDir": "./.skill-optimizer" } }, "optimize": { "enabled": true, - "model": "openrouter/anthropic/claude-sonnet-4.6", + "model": "openrouter/anthropic/claude-sonnet-4-6", "apiKeyEnv": "OPENROUTER_API_KEY", "allowedPaths": ["SKILL.md"], "validation": [], diff --git a/mock-repos/mcp-tracker-demo/benchmark.config.json b/mock-repos/mcp-tracker-demo/benchmark.config.json deleted file mode 100644 index 548641a..0000000 --- a/mock-repos/mcp-tracker-demo/benchmark.config.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "name": "mcp-tracker-demo", - "surface": "mcp", - "mcp": { - "tools": "./tools.json" - }, - "skill": { - "source": "./SKILL.md" - }, - "tasks": "./tasks.json", - "llm": { - "apiKeyEnv": "OPENROUTER_API_KEY", - "format": "pi", - "models": [ - { - "id": "openrouter/openai/gpt-5.4", - "name": "GPT-5.4", - "tier": "flagship" - }, - { - "id": "openrouter/anthropic/claude-sonnet-4.6", - "name": "Claude Sonnet 4.6", - "tier": "mid" - }, - { - "id": "openrouter/google/gemini-3-flash-preview", - "name": "Gemini 3 Flash Preview", - "tier": "low" - } - ] - } -} diff --git a/mock-repos/mcp-tracker-demo/optimize.config.json b/mock-repos/mcp-tracker-demo/optimize.config.json deleted file mode 100644 index 47476e1..0000000 --- a/mock-repos/mcp-tracker-demo/optimize.config.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "benchmarkConfig": "./benchmark.config.json", - "targetRepo": { - "path": ".", - "surface": "mcp", - "allowedPaths": [ - "SKILL.md" - ], - "validation": [], - "requireCleanGit": true - }, - "optimizer": { - "maxIterations": 5, - "stabilityWindow": 2, - "minOverallPassDelta": 0.01, - "taskGeneration": { - "enabled": true, - "maxGenerated": 8, - "seed": 1, - "outputDir": "./.skill-optimizer" - } - }, - "mutation": { - "provider": "openrouter", - "model": "openai/gpt-5.4", - "thinkingLevel": "medium", - "apiKeyEnv": "OPENROUTER_API_KEY", - "reportContextMaxBytes": 16000 - } -} diff --git a/mock-repos/mcp-tracker-demo/skill-optimizer.json b/mock-repos/mcp-tracker-demo/skill-optimizer.json index a11ebab..21c020c 100644 --- a/mock-repos/mcp-tracker-demo/skill-optimizer.json +++ b/mock-repos/mcp-tracker-demo/skill-optimizer.json @@ -20,12 +20,12 @@ "apiKeyEnv": "OPENROUTER_API_KEY", "models": [ { - "id": "openrouter/openai/gpt-5.4", + "id": "openrouter/openai/gpt-5-4", "name": "GPT-5.4", "tier": "flagship" }, { - "id": "openrouter/anthropic/claude-sonnet-4.6", + "id": "openrouter/anthropic/claude-sonnet-4-6", "name": "Claude Sonnet 4.6", "tier": "mid" }, @@ -52,7 +52,7 @@ "optimize": { "enabled": true, "mode": "surface-changing", - "model": "openrouter/openai/gpt-5.4", + "model": "openrouter/openai/gpt-5-4", "apiKeyEnv": "OPENROUTER_API_KEY", "thinkingLevel": "medium", "allowedPaths": [ diff --git a/mock-repos/sdk-counter-demo/skill-optimizer.json b/mock-repos/sdk-counter-demo/skill-optimizer.json index 17bd952..10222fc 100644 --- a/mock-repos/sdk-counter-demo/skill-optimizer.json +++ b/mock-repos/sdk-counter-demo/skill-optimizer.json @@ -17,14 +17,14 @@ "apiKeyEnv": "OPENROUTER_API_KEY", "models": [ { "id": "openrouter/openai/gpt-4o-mini", "name": "GPT-4o mini", "tier": "low" }, - { "id": "openrouter/anthropic/claude-sonnet-4.6", "name": "Claude 4.6", "tier": "mid" } + { "id": "openrouter/anthropic/claude-sonnet-4-6", "name": "Claude 4.6", "tier": "mid" } ], "verdict": { "perModelFloor": 0.6, "targetWeightedAverage": 0.7 }, "taskGeneration": { "enabled": true, "maxTasks": 8, "seed": 1, "outputDir": "./.skill-optimizer" } }, "optimize": { "enabled": true, - "model": "openrouter/anthropic/claude-sonnet-4.6", + "model": "openrouter/anthropic/claude-sonnet-4-6", "apiKeyEnv": "OPENROUTER_API_KEY", "allowedPaths": ["SKILL.md"], "validation": [], diff --git a/package.json b/package.json index 5bc6f96..81d1f7a 100644 --- a/package.json +++ b/package.json @@ -48,7 +48,7 @@ "gen-docs": "tsx scripts/gen-docs.ts", "build": "tsc && npm run gen-docs && chmod +x dist/cli.js", "typecheck": "tsc --noEmit", - "test": "tsx tests/smoke-code.ts && tsx tests/smoke-sdk-python.ts && tsx tests/smoke-sdk-rust.ts && tsx tests/smoke-cli.ts && tsx tests/smoke-cli-entry.ts && tsx tests/smoke-mcp.ts && tsx tests/smoke-llm.ts && tsx tests/smoke-discovery-sdk.ts && tsx tests/smoke-discovery-cli.ts && tsx tests/smoke-discovery-mcp.ts && tsx tests/smoke-discovery-prompt.ts && tsx tests/smoke-prompt-evaluator.ts && tsx tests/smoke-snapshot-prompt.ts && tsx tests/smoke-generation.ts && tsx tests/smoke-optimize.ts && tsx tests/smoke-mock-repos.ts && tsx tests/smoke-release.ts && tsx tests/smoke-scoring.ts && tsx tests/smoke-scope.ts && tsx tests/smoke-coverage.ts && tsx tests/smoke-feedback.ts && tsx tests/smoke-verdict.ts && tsx tests/smoke-dry-run.ts && tsx tests/smoke-errors.ts && tsx tests/smoke-e2e.ts && tsx tests/smoke-import.ts && tsx tests/smoke-init.ts && tsx tests/smoke-gen-docs.ts" + "test": "tsx tests/smoke-code.ts && tsx tests/smoke-sdk-python.ts && tsx tests/smoke-sdk-rust.ts && tsx tests/smoke-cli.ts && tsx tests/smoke-cli-entry.ts && tsx tests/smoke-mcp.ts && tsx tests/smoke-llm.ts && tsx tests/smoke-discovery-sdk.ts && tsx tests/smoke-discovery-cli.ts && tsx tests/smoke-discovery-mcp.ts && tsx tests/smoke-discovery-prompt.ts && tsx tests/smoke-prompt-evaluator.ts && tsx tests/smoke-snapshot-prompt.ts && tsx tests/smoke-generation.ts && tsx tests/smoke-optimize.ts && tsx tests/smoke-mock-repos.ts && tsx tests/smoke-release.ts && tsx tests/smoke-scoring.ts && tsx tests/smoke-scope.ts && tsx tests/smoke-coverage.ts && tsx tests/smoke-feedback.ts && tsx tests/smoke-verdict.ts && tsx tests/smoke-dry-run.ts && tsx tests/smoke-errors.ts && tsx tests/smoke-e2e.ts && tsx tests/smoke-import.ts && tsx tests/smoke-init.ts && tsx tests/smoke-gen-docs.ts && tsx tests/smoke-actions.ts" }, "dependencies": { "@clack/prompts": "^1.2.0", diff --git a/src/actions/discover.ts b/src/actions/discover.ts index 07a6629..4e9d86e 100644 --- a/src/actions/discover.ts +++ b/src/actions/discover.ts @@ -1,4 +1,4 @@ -import { loadCliCommands, loadMcpTools } from '../benchmark/config.js'; +import { loadCliCommands, loadMcpTools } from './loaders.js'; import type { ResolvedProjectConfig } from '../project/types.js'; import type { ActionCatalog } from './types.js'; import { readCliActionsFromSources } from './readers/cli.js'; diff --git a/src/actions/index.ts b/src/actions/index.ts index 50cb055..dda1e60 100644 --- a/src/actions/index.ts +++ b/src/actions/index.ts @@ -1,4 +1,5 @@ export { diffActionCatalog, type ActionCatalogDiff, type ChangedAction } from './diff.js'; +export { loadCliCommands, loadMcpTools } from './loaders.js'; export { discoverActions } from './discover.js'; export { readCliActionsFromSources } from './readers/cli.js'; export { readMcpActionsFromSources } from './readers/mcp.js'; diff --git a/src/actions/loaders.ts b/src/actions/loaders.ts new file mode 100644 index 0000000..f4088f2 --- /dev/null +++ b/src/actions/loaders.ts @@ -0,0 +1,82 @@ +import { readFileSync, existsSync } from 'node:fs'; +import { resolve } from 'node:path'; +import type { McpToolDefinition, CliCommandDefinition } from '../benchmark/types.js'; + +/** + * Load MCP tool definitions from the tools.json path specified in config. + */ +export function loadMcpTools(toolsPath: string, baseDir?: string): McpToolDefinition[] { + const resolved = resolve(baseDir ?? process.cwd(), toolsPath); + if (!existsSync(resolved)) { + throw new Error(`MCP tools file not found: ${resolved}`); + } + + let raw: string; + try { + raw = readFileSync(resolved, 'utf-8'); + } catch (err) { + throw new Error(`Failed to read MCP tools: ${resolved}: ${err instanceof Error ? err.message : err}`); + } + + let tools: McpToolDefinition[]; + try { + tools = JSON.parse(raw) as McpToolDefinition[]; + } catch (err) { + throw new Error(`Invalid JSON in MCP tools file: ${resolved}: ${err instanceof Error ? err.message : err}`); + } + + if (!Array.isArray(tools)) { + throw new Error(`MCP tools file ${resolved}: must be a JSON array of tool definitions`); + } + + return tools; +} + +/** + * Load CLI command definitions from the commands.json path specified in config. + */ +export function loadCliCommands(commandsPath: string, baseDir?: string): CliCommandDefinition[] { + const resolved = resolve(baseDir ?? process.cwd(), commandsPath); + if (!existsSync(resolved)) { + throw new Error(`CLI commands file not found: ${resolved}`); + } + + let raw: string; + try { + raw = readFileSync(resolved, 'utf-8'); + } catch (err) { + throw new Error(`Failed to read CLI commands: ${resolved}: ${err instanceof Error ? err.message : err}`); + } + + let commands: CliCommandDefinition[]; + try { + commands = JSON.parse(raw) as CliCommandDefinition[]; + } catch (err) { + throw new Error(`Invalid JSON in CLI commands file: ${resolved}: ${err instanceof Error ? err.message : err}`); + } + + if (!Array.isArray(commands)) { + throw new Error(`CLI commands file ${resolved}: must be a JSON array of command definitions`); + } + + for (const [index, command] of commands.entries()) { + if (!command || typeof command !== 'object') { + throw new Error(`CLI commands file ${resolved}: entry ${index} must be an object`); + } + if (typeof command.command !== 'string' || command.command.trim() === '') { + throw new Error(`CLI commands file ${resolved}: entry ${index} must include a non-empty "command" string`); + } + if (command.options !== undefined && !Array.isArray(command.options)) { + throw new Error(`CLI commands file ${resolved}: entry ${index} options must be an array when present`); + } + if (Array.isArray(command.options)) { + for (const [optionIndex, option] of command.options.entries()) { + if (!option || typeof option !== 'object' || typeof option.name !== 'string' || option.name.trim() === '') { + throw new Error(`CLI commands file ${resolved}: entry ${index} option ${optionIndex} must include a non-empty "name" string`); + } + } + } + } + + return commands; +} diff --git a/src/actions/snapshot.ts b/src/actions/snapshot.ts index df0ac45..c265a7b 100644 --- a/src/actions/snapshot.ts +++ b/src/actions/snapshot.ts @@ -151,8 +151,8 @@ export function loadActionSnapshotFile(snapshotPath: string): ActionSnapshotArti let parsed: unknown; try { parsed = JSON.parse(raw) as unknown; - } catch (error: any) { - invalidSnapshot(snapshotPath, `invalid JSON: ${error.message}`); + } catch (error) { + invalidSnapshot(snapshotPath, `invalid JSON: ${error instanceof Error ? error.message : String(error)}`); } if (!parsed || typeof parsed !== 'object') { invalidSnapshot(snapshotPath, 'expected object root'); diff --git a/src/benchmark/compare.ts b/src/benchmark/compare.ts index 60f45cc..980ec8f 100644 --- a/src/benchmark/compare.ts +++ b/src/benchmark/compare.ts @@ -15,7 +15,6 @@ export function compareReports( baseline: BenchmarkReport, current: BenchmarkReport, ): ComparisonReport { - // Build lookup maps keyed by "taskId:modelId" const baselineMap = new Map(); for (const r of baseline.results) { const key = `${r.task.id}:${r.model.id}`; @@ -38,7 +37,6 @@ export function compareReports( }); } - // Collect all unique keys from both reports const allKeys = new Set([...baselineMap.keys(), ...currentMap.keys()]); const taskDeltas: TaskDelta[] = []; @@ -79,13 +77,10 @@ export function compareReports( passedNow = inCurrent.passed; recallNow = inCurrent.recall; delta = 'new'; - // 'new' entries don't count toward improved/regressed/unchanged } else { - // only in baseline passedBefore = inBaseline!.passed; recallBefore = inBaseline!.recall; delta = 'removed'; - // 'removed' entries don't count toward improved/regressed/unchanged } taskDeltas.push({ @@ -101,7 +96,6 @@ export function compareReports( }); } - // Sort for stable output: regressions first, then improvements, then unchanged const deltaOrder: Record = { regressed: 0, improved: 1, @@ -117,11 +111,8 @@ export function compareReports( return a.modelId.localeCompare(b.modelId); }); - // Coverage delta const coverageBefore = baseline.summary.methodCoveragePercent; const coverageNow = current.summary.methodCoveragePercent; - - // Accuracy delta — read from overallPassRate const accuracyBefore = baseline.summary.overallPassRate; const accuracyNow = current.summary.overallPassRate; @@ -183,7 +174,6 @@ export function printComparison(comparison: ComparisonReport): void { console.log(header); console.log('─'.repeat(header.length + COL_DELTA)); - // Rows — only print non-unchanged entries first, then unchanged if any const interesting = taskDeltas.filter(d => d.delta !== 'unchanged'); const unchangedDeltas = taskDeltas.filter(d => d.delta === 'unchanged'); @@ -209,7 +199,6 @@ export function printComparison(comparison: ComparisonReport): void { deltaLabel = 'unchanged'; } - // Shorten model ID for display (use last segment after '/') const modelDisplay = d.modelId.includes('/') ? d.modelId.split('/').pop()! : d.modelId; console.log( diff --git a/src/benchmark/config.ts b/src/benchmark/config.ts index 2519703..b8ac3fa 100644 --- a/src/benchmark/config.ts +++ b/src/benchmark/config.ts @@ -3,12 +3,11 @@ import { resolve } from 'node:path'; import type { BenchmarkConfig, TaskDefinition, - McpToolDefinition, - CliCommandDefinition, ModelConfig, ExpectedAction, } from './types.js'; import { DEFAULT_PROJECT_CONFIG_NAME, loadProjectConfig, toBenchmarkConfig } from '../project/index.js'; +export { loadMcpTools, loadCliCommands } from '../actions/loaders.js'; const DEFAULT_CONFIG_NAME = DEFAULT_PROJECT_CONFIG_NAME; const SAFE_TASK_ID = /^[A-Za-z0-9._-]+$/; @@ -51,9 +50,9 @@ export function loadTasks(tasksPath: string, baseDir?: string): TaskDefinition[] throw new Error(`Failed to read tasks: ${resolved}: ${err instanceof Error ? err.message : err}`); } - let parsed: { tasks: Array }; + let parsed: { tasks: Array<{ id?: unknown; prompt?: unknown; expected_actions?: unknown; verify?: unknown; expected_fetches?: unknown }> }; try { - parsed = JSON.parse(raw) as { tasks: TaskDefinition[] }; + parsed = JSON.parse(raw) as typeof parsed; } catch (err) { throw new Error(`Invalid JSON in tasks file: ${resolved}: ${err instanceof Error ? err.message : err}`); } @@ -66,7 +65,7 @@ export function loadTasks(tasksPath: string, baseDir?: string): TaskDefinition[] } function normalizeTaskDefinition( - task: TaskDefinition & { expected_tools?: unknown; expected_actions?: unknown }, + task: { id?: unknown; prompt?: unknown; expected_actions?: unknown; verify?: unknown; expected_fetches?: unknown }, resolvedPath: string, index: number, ): TaskDefinition { @@ -80,11 +79,7 @@ function normalizeTaskDefinition( throw new Error(`Tasks file ${resolvedPath}: task ${task.id} must include a non-empty string prompt`); } - const rawExpectedActions = Array.isArray(task.expected_actions) - ? task.expected_actions - : Array.isArray(task.expected_tools) - ? task.expected_tools - : null; + const rawExpectedActions = Array.isArray(task.expected_actions) ? task.expected_actions : null; if (!rawExpectedActions) { throw new Error(`Tasks file ${resolvedPath}: task at index ${index} must include an expected_actions array`); @@ -92,13 +87,30 @@ function normalizeTaskDefinition( const expected_actions = rawExpectedActions.map((rawAction, actionIndex) => normalizeExpectedAction(rawAction, resolvedPath, index, actionIndex)); + const rawVerify = Array.isArray(task.verify) ? task.verify : undefined; + if (rawVerify !== undefined) { + for (let i = 0; i < rawVerify.length; i++) { + if (!rawVerify[i] || typeof rawVerify[i] !== 'object') { + throw new Error(`Tasks file ${resolvedPath}: task ${task.id} verify[${i}] must be an object`); + } + } + } + + const rawFetches = Array.isArray(task.expected_fetches) ? task.expected_fetches : undefined; + if (rawFetches !== undefined) { + for (let i = 0; i < rawFetches.length; i++) { + if (typeof rawFetches[i] !== 'string' || !(rawFetches[i] as string).trim()) { + throw new Error(`Tasks file ${resolvedPath}: task ${task.id} expected_fetches[${i}] must be a non-empty string`); + } + } + } + return { id: task.id, prompt: task.prompt, expected_actions, - expected_tools: expected_actions, - verify: task.verify, - expected_fetches: task.expected_fetches, + verify: rawVerify as TaskDefinition['verify'] | undefined, + expected_fetches: rawFetches as string[] | undefined, }; } @@ -112,12 +124,8 @@ function normalizeExpectedAction( throw new Error(`Tasks file ${resolvedPath}: task ${taskIndex} action ${actionIndex} must be an object`); } - const candidate = rawAction as { name?: unknown; method?: unknown; args?: unknown }; - const name = typeof candidate.name === 'string' - ? candidate.name - : typeof candidate.method === 'string' - ? candidate.method - : null; + const candidate = rawAction as { name?: unknown; args?: unknown }; + const name = typeof candidate.name === 'string' ? candidate.name : null; if (!name || name.trim() === '') { throw new Error(`Tasks file ${resolvedPath}: task ${taskIndex} action ${actionIndex} must include a non-empty name`); @@ -129,90 +137,10 @@ function normalizeExpectedAction( return { name, - method: name, args: (candidate.args as Record | undefined) ?? {}, }; } -/** - * Load MCP tool definitions from the tools.json path specified in config. - */ -export function loadMcpTools(toolsPath: string, baseDir?: string): McpToolDefinition[] { - const resolved = resolve(baseDir ?? process.cwd(), toolsPath); - if (!existsSync(resolved)) { - throw new Error(`MCP tools file not found: ${resolved}`); - } - - let raw: string; - try { - raw = readFileSync(resolved, 'utf-8'); - } catch (err) { - throw new Error(`Failed to read MCP tools: ${resolved}: ${err instanceof Error ? err.message : err}`); - } - - let tools: McpToolDefinition[]; - try { - tools = JSON.parse(raw) as McpToolDefinition[]; - } catch (err) { - throw new Error(`Invalid JSON in MCP tools file: ${resolved}: ${err instanceof Error ? err.message : err}`); - } - - if (!Array.isArray(tools)) { - throw new Error(`MCP tools file ${resolved}: must be a JSON array of tool definitions`); - } - - return tools; -} - -/** - * Load CLI command definitions from the commands.json path specified in config. - */ -export function loadCliCommands(commandsPath: string, baseDir?: string): CliCommandDefinition[] { - const resolved = resolve(baseDir ?? process.cwd(), commandsPath); - if (!existsSync(resolved)) { - throw new Error(`CLI commands file not found: ${resolved}`); - } - - let raw: string; - try { - raw = readFileSync(resolved, 'utf-8'); - } catch (err) { - throw new Error(`Failed to read CLI commands: ${resolved}: ${err instanceof Error ? err.message : err}`); - } - - let commands: CliCommandDefinition[]; - try { - commands = JSON.parse(raw) as CliCommandDefinition[]; - } catch (err) { - throw new Error(`Invalid JSON in CLI commands file: ${resolved}: ${err instanceof Error ? err.message : err}`); - } - - if (!Array.isArray(commands)) { - throw new Error(`CLI commands file ${resolved}: must be a JSON array of command definitions`); - } - - for (const [index, command] of commands.entries()) { - if (!command || typeof command !== 'object') { - throw new Error(`CLI commands file ${resolved}: entry ${index} must be an object`); - } - if (typeof command.command !== 'string' || command.command.trim() === '') { - throw new Error(`CLI commands file ${resolved}: entry ${index} must include a non-empty "command" string`); - } - if (command.options !== undefined && !Array.isArray(command.options)) { - throw new Error(`CLI commands file ${resolved}: entry ${index} options must be an array when present`); - } - if (Array.isArray(command.options)) { - for (const [optionIndex, option] of command.options.entries()) { - if (!option || typeof option !== 'object' || typeof option.name !== 'string' || option.name.trim() === '') { - throw new Error(`CLI commands file ${resolved}: entry ${index} option ${optionIndex} must include a non-empty "name" string`); - } - } - } - } - - return commands; -} - /** * Helper to get a model by slug (lowercased name with non-alphanumeric replaced by hyphens). */ diff --git a/src/benchmark/evaluator.ts b/src/benchmark/evaluator.ts index aa6fe62..c1bf64e 100644 --- a/src/benchmark/evaluator.ts +++ b/src/benchmark/evaluator.ts @@ -1,7 +1,7 @@ import type { - ExpectedTool, + ExpectedAction, ExtractedCall, - ToolMatch, + ActionMatch, TaskDefinition, TaskResult, ModelConfig, @@ -168,19 +168,19 @@ function matchExpectedValue( // ── Tool matching ────────────────────────────────────────────────────────── /** - * Match extracted calls against expected tools. - * Returns ToolMatch[] with match details. + * Match extracted calls against expected actions. + * Returns ActionMatch[] with match details. * - * Each extracted call can only be matched to ONE expected tool (greedy, first match wins). + * Each extracted call can only be matched to ONE expected action (greedy, first match wins). */ -export function matchTools( - expectedTools: ExpectedTool[], +export function matchActions( + expectedActions: ExpectedAction[], extractedCalls: ExtractedCall[], -): ToolMatch[] { +): ActionMatch[] { // Track which extracted call indices have already been consumed const usedIndices = new Set(); - return expectedTools.map((expected) => { + return expectedActions.map((expected) => { const expectedMethod = getExpectedActionName(expected); // If there are args to check, try to find a perfect match (method + args) first. // Otherwise, find the first unused extracted call that matches the method name. @@ -222,7 +222,7 @@ export function matchTools( argsCorrect: true, matched: true, argResults, - } satisfies ToolMatch; + } satisfies ActionMatch; } // Pass 2: no perfect match — fall back to the first method-name match. @@ -244,7 +244,7 @@ export function matchTools( methodFound: false, argsCorrect: false, matched: false, - } satisfies ToolMatch; + } satisfies ActionMatch; } // Consume the fallback index so it can't be re-matched @@ -264,7 +264,7 @@ export function matchTools( argsCorrect: allArgsMatch, matched: allArgsMatch, argResults, - } satisfies ToolMatch; + } satisfies ActionMatch; } // No args to check — find the first unused extracted call matching the method name. @@ -285,7 +285,7 @@ export function matchTools( methodFound: false, argsCorrect: false, matched: false, - } satisfies ToolMatch; + } satisfies ActionMatch; } // No args to check — method match is sufficient @@ -296,7 +296,7 @@ export function matchTools( methodFound: true, argsCorrect: true, matched: true, - } satisfies ToolMatch; + } satisfies ActionMatch; }); } @@ -316,7 +316,7 @@ export function matchTools( function resolveCallsFromBindings( extractedCalls: ExtractedCall[], bindings: Map, - expectedTools: ExpectedTool[], + expectedTools: ExpectedAction[], ): ExtractedCall[] { // 1. Collect type prefixes from expected tools. const expectedPrefixes = new Set(); @@ -433,17 +433,14 @@ export function evaluateTask(params: { surface, } = params; - // ── Resolve raw calls if bindings provided ── let extractedCalls = params.extractedCalls; const expectedActions = getExpectedActions(task); if (bindings && bindings.size > 0) { extractedCalls = resolveCallsFromBindings(extractedCalls, bindings, expectedActions); } - // ── Tool matching ── - const toolMatches = matchTools(expectedActions, extractedCalls); + const actionMatches = matchActions(expectedActions, extractedCalls); - // ── Code pattern checks ── const codePatternResults: Record = {}; let allCodePatternsPass = true; @@ -463,7 +460,6 @@ export function evaluateTask(params: { } } } else if (task.verify) { - // generatedCode is null — any code_pattern check fails for (const verification of task.verify) { if (verification.code_pattern) { codePatternResults[verification.code_pattern] = false; @@ -472,37 +468,32 @@ export function evaluateTask(params: { } } - // ── Metrics ── const expectedCount = expectedActions.length; - const matchedCount = toolMatches.filter((m) => m.matched).length; + const matchedCount = actionMatches.filter((m) => m.matched).length; - // Recall: matched / expected. If 0 expected, recall = 1.0 + // recall = matched / expected; 1.0 when there are no expectations const toolRecall = expectedCount === 0 ? 1.0 : matchedCount / expectedCount; - // Precision: matched / total known calls extracted. If 0 extracted known calls, precision = 0.0 + // precision = matched / known calls extracted; 0.0 when nothing was extracted const knownCallCount = extractedCalls.filter((c) => knownMethods.has(c.method)).length; const toolPrecision = knownCallCount === 0 ? 0.0 : matchedCount / knownCallCount; - // Task passes if all expected tools matched AND all code_pattern checks passed const hasCodePatterns = Object.keys(codePatternResults).length > 0; const taskPassed = matchedCount === expectedCount && (expectedCount > 0 || matchedCount === 0) && (!hasCodePatterns || allCodePatternsPass); - // Tool selection accuracy: how many expected methods were found (ignoring args)? - const methodsFoundCount = toolMatches.filter(m => m.methodFound).length; + const methodsFoundCount = actionMatches.filter(m => m.methodFound).length; const toolSelectionAccuracy = expectedCount === 0 ? 1.0 : methodsFoundCount / expectedCount; - // Arg accuracy: of methods found, how many had correct args? const argAccuracy = methodsFoundCount === 0 ? 1.0 - : toolMatches.filter(m => m.methodFound && m.argsCorrect).length / methodsFoundCount; + : actionMatches.filter(m => m.methodFound && m.argsCorrect).length / methodsFoundCount; - // Surface-aware hallucination tracking const allExtractedMethods = extractedCalls.map(c => c.method); const expectedMethods = new Set(expectedActions.map((action) => getExpectedActionName(action))); - // Derive SDK prefixes from knownMethods + // Collect known class/type prefixes for SDK hallucination filtering (e.g. "FastClient" from "FastClient.setup") const sdkPrefixes = new Set(); for (const m of knownMethods) { if (m.includes('.')) { @@ -510,12 +501,11 @@ export function evaluateTask(params: { } } - // Unnecessary: valid SDK methods that weren't expected for this task - const unnecessaryCalls = allExtractedMethods.filter( + const unnecessaryActions = allExtractedMethods.filter( m => knownMethods.has(m) && !expectedMethods.has(m) ); - const hallucinatedCalls = allExtractedMethods.filter(m => { + const hallucinatedActions = allExtractedMethods.filter(m => { if (knownMethods.has(m)) return false; if (surface === 'sdk') { @@ -532,7 +522,7 @@ export function evaluateTask(params: { }); const hallucinationRate = extractedCalls.length === 0 ? 0 - : hallucinatedCalls.length / extractedCalls.length; + : hallucinatedActions.length / extractedCalls.length; return { task, @@ -540,8 +530,7 @@ export function evaluateTask(params: { generatedCode, rawResponse, extractedCalls, - actionMatches: toolMatches, - toolMatches, + actionMatches, codePatternResults: hasCodePatterns ? codePatternResults : undefined, metrics: { toolPrecision, @@ -549,10 +538,8 @@ export function evaluateTask(params: { taskPassed, toolSelectionAccuracy, argAccuracy, - unnecessaryActions: unnecessaryCalls, - unnecessaryCalls, - hallucinatedActions: hallucinatedCalls, - hallucinatedCalls, + unnecessaryActions, + hallucinatedActions, hallucinationRate, }, llmLatencyMs, diff --git a/src/benchmark/extractors/code-analyzer.ts b/src/benchmark/extractors/code-analyzer.ts index c310ad0..a97ee25 100644 --- a/src/benchmark/extractors/code-analyzer.ts +++ b/src/benchmark/extractors/code-analyzer.ts @@ -410,7 +410,7 @@ function extractCallExpression( const fnNode = node.childForFieldName('function'); if (!fnNode) return null; - // NEW: standalone function call + // Standalone function call (e.g. x402Pay(...), fast(...)) if (fnNode.type === 'identifier' && knownFunctions.has(fnNode.text)) { const method = fnNode.text; const argsNode = node.childForFieldName('arguments'); @@ -449,7 +449,7 @@ function extractCallExpression( // ── Generic extraction (no config hints) ─────────────────────────────────── -export interface RawExtraction { +interface RawExtraction { calls: ExtractedCall[]; bindings: Map; // variable → source function/class name } @@ -580,33 +580,3 @@ export async function extractAllFromCode(code: string): Promise { return { calls, bindings }; } -// ── Legacy Public API (deprecated — use extractAllFromCode) ──────────────── - -/** - * @deprecated Use extractAllFromCode instead. This function requires config hints. - */ -export async function extractFromCode( - code: string, - classes: string[], - functions: string[] = [], - functionReturns: Record = {}, -): Promise { - const sdkClasses = new Set(classes); - const knownFunctions = new Set(functions); - const fnReturns = new Map(Object.entries(functionReturns)); - const p = await initParser(); - const tree = p.parse(code); - const root = tree.rootNode; - const literalMap = collectLiteralBindings(root); - - // First pass: collect variable bindings - const varMap = collectVariableBindings(root, sdkClasses, knownFunctions, fnReturns); - - // Second pass: collect all calls - const calls = collectCalls(root, varMap, literalMap, code, sdkClasses, knownFunctions); - - // Sort by line number - calls.sort((a, b) => a.line - b.line); - - return calls; -} diff --git a/src/benchmark/extractors/index.ts b/src/benchmark/extractors/index.ts index 9629b19..3604686 100644 --- a/src/benchmark/extractors/index.ts +++ b/src/benchmark/extractors/index.ts @@ -56,10 +56,3 @@ export async function extract( const { calls, bindings } = await extractSdkFromCode(generatedCode, sdkConfig.language); return { calls, generatedCode, bindings }; } - -// Re-export for direct access -export { extractCodeBlock, extractSdkCodeBlock } from './code-extractor.js'; -export { extractShellBlock, extractFromCliMarkdown, parseShellCommands } from './cli-extractor.js'; -export { extractFromCode } from './code-analyzer.js'; -export { extractFromToolCalls } from './mcp-extractor.js'; -export { extractSdkFromCode, getSdkAdapter } from './sdk/registry.js'; diff --git a/src/benchmark/extractors/sdk/python.ts b/src/benchmark/extractors/sdk/python.ts index 2f4a9bb..165d16a 100644 --- a/src/benchmark/extractors/sdk/python.ts +++ b/src/benchmark/extractors/sdk/python.ts @@ -178,7 +178,7 @@ function collectPythonCalls(root: Parser.SyntaxNode, bindings: BindingMap, liter return sortCalls(calls); } -export async function extractPythonSdk(code: string): Promise { +async function extractPythonSdk(code: string): Promise { const parser = await getSdkParser('python'); const tree = parser.parse(code); const root = tree.rootNode; diff --git a/src/benchmark/extractors/sdk/rust.ts b/src/benchmark/extractors/sdk/rust.ts index 50728b0..e0e72cc 100644 --- a/src/benchmark/extractors/sdk/rust.ts +++ b/src/benchmark/extractors/sdk/rust.ts @@ -222,7 +222,7 @@ function collectRustCalls(root: Parser.SyntaxNode, bindings: BindingMap, literal return sortCalls(calls); } -export async function extractRustSdk(code: string): Promise { +async function extractRustSdk(code: string): Promise { const parser = await getSdkParser('rust'); const tree = parser.parse(code); const root = tree.rootNode; diff --git a/src/benchmark/extractors/sdk/typescript.ts b/src/benchmark/extractors/sdk/typescript.ts index 9c160cc..22519e3 100644 --- a/src/benchmark/extractors/sdk/typescript.ts +++ b/src/benchmark/extractors/sdk/typescript.ts @@ -2,7 +2,7 @@ import { extractAllFromCode } from '../code-analyzer.js'; import type { RawSdkExtraction, SdkLanguageAdapter } from './types.js'; -export async function extractTypeScriptSdk(code: string): Promise { +async function extractTypeScriptSdk(code: string): Promise { return extractAllFromCode(code); } diff --git a/src/benchmark/index.ts b/src/benchmark/index.ts index fdba541..3121a70 100644 --- a/src/benchmark/index.ts +++ b/src/benchmark/index.ts @@ -21,10 +21,8 @@ export type { CliCommandDefinition, CliCommandOptionDefinition, McpSurfaceConfig, - CodeModeConfig, - McpModeConfig, LLMConfig, - TaskDefinition, ExpectedTool, ExtractedCall, ToolMatch, + TaskDefinition, ExpectedAction, ExtractedCall, ActionMatch, TaskResult, BenchmarkReport, ComparisonReport, ModelConfig, Tier, LLMResponse, ToolCallResult, } from './types.js'; diff --git a/src/benchmark/llm/anthropic-format.ts b/src/benchmark/llm/anthropic-format.ts index 8b337e4..8df744c 100644 --- a/src/benchmark/llm/anthropic-format.ts +++ b/src/benchmark/llm/anthropic-format.ts @@ -1,4 +1,12 @@ import type { LLMResponse, McpToolDefinition, ToolExecutor } from '../types.js'; +import { + isAbortError, + isRetryableError, + sleep, + truncateToolResult, + undefinedIfEmpty, + normalizeUsage, +} from './shared.js'; interface CallParams { baseUrl: string; @@ -24,7 +32,6 @@ interface AnthropicTool { }; } -/** Transform OpenAI-format McpToolDefinition to Anthropic tool format. */ function toAnthropicTool(tool: McpToolDefinition): AnthropicTool { return { name: tool.function.name, @@ -99,8 +106,8 @@ async function doFetch(params: CallParams, body: Record): Promi if (!response.ok) { const text = await response.text(); - const err = new Error(`Anthropic API error ${response.status}: ${text}`); - (err as any).status = response.status; + const err = new Error(`Anthropic API error ${response.status}: ${text}`) as Error & { status: number }; + err.status = response.status; throw err; } @@ -155,8 +162,8 @@ interface AgentLoopParams extends CallWithToolsParams { export async function chatAgentLoopAnthropic(params: AgentLoopParams): Promise { const messages: Array> = [{ role: 'user', content: params.user }]; - let allToolCalls: Array<{ name: string; arguments: Record }> = []; - let totalUsage = { prompt: 0, completion: 0, total: 0 }; + const allToolCalls: Array<{ name: string; arguments: Record }> = []; + const totalUsage = { prompt: 0, completion: 0, total: 0 }; for (let turn = 0; turn < params.maxTurns; turn++) { const body: Record = { @@ -170,7 +177,7 @@ export async function chatAgentLoopAnthropic(params: AgentLoopParams): Promise 0 ? allToolCalls : undefined, usage: totalUsage.total > 0 ? totalUsage : undefined }; + return { content: response.content, toolCalls: undefinedIfEmpty(allToolCalls), usage: normalizeUsage(totalUsage) }; } allToolCalls.push(...response.toolCalls); const assistantContent = [ @@ -178,21 +185,17 @@ export async function chatAgentLoopAnthropic(params: AgentLoopParams): Promise ({ type: 'tool_use', id: `toolu_${turn}_${i}`, name: tc.name, input: tc.arguments })), ]; messages.push({ role: 'assistant', content: assistantContent }); - const MAX_TOOL_RESULT_CHARS = 50_000; const toolResults = []; for (let i = 0; i < response.toolCalls.length; i++) { const tc = response.toolCalls[i]; let result: string; try { result = await params.executor(tc.name, tc.arguments); } catch (err) { result = `Error: ${err instanceof Error ? err.message : String(err)}`; } - if (result.length > MAX_TOOL_RESULT_CHARS) { - result = result.slice(0, MAX_TOOL_RESULT_CHARS) + '\n\n[... truncated]'; - } - toolResults.push({ type: 'tool_result', tool_use_id: `toolu_${turn}_${i}`, content: result }); + toolResults.push({ type: 'tool_result', tool_use_id: `toolu_${turn}_${i}`, content: truncateToolResult(result) }); } messages.push({ role: 'user', content: toolResults }); } - return { content: '', toolCalls: allToolCalls.length > 0 ? allToolCalls : undefined, usage: totalUsage.total > 0 ? totalUsage : undefined }; + return { content: '', toolCalls: undefinedIfEmpty(allToolCalls), usage: normalizeUsage(totalUsage) }; } async function callWithRetry( @@ -202,18 +205,10 @@ async function callWithRetry( try { return await doFetch(params, body); } catch (err) { - if (err && typeof err === 'object' && 'name' in err && err.name === 'AbortError') { - throw err; - } - // Don't retry client errors (4xx) except 429 (rate limit) - if (err && typeof err === 'object' && 'status' in err) { - const status = (err as any).status; - if (typeof status === 'number' && status >= 400 && status < 500 && status !== 429) { - throw err; - } - } + if (isAbortError(err)) throw err; + if (!isRetryableError(err)) throw err; // Retry once after 3s - await new Promise((resolve) => setTimeout(resolve, 3_000)); + await sleep(3_000); return doFetch(params, body); } } diff --git a/src/benchmark/llm/index.ts b/src/benchmark/llm/index.ts index 8606f93..24c2c23 100644 --- a/src/benchmark/llm/index.ts +++ b/src/benchmark/llm/index.ts @@ -4,7 +4,7 @@ import { chatAnthropic, chatWithToolsAnthropic, chatAgentLoopAnthropic } from '. import { chatPi, chatWithToolsPi, chatAgentLoopPi } from './pi-format.js'; import { requireConfiguredApiKey, resolveApiCredential } from '../../runtime/pi/index.js'; -export interface LLMClient { +interface LLMClient { /** Regular chat — LLM returns text output (SDK/CLI surfaces) */ chat(modelId: string, system: string, user: string): Promise; /** Chat with tools — LLM returns structured tool_calls (MCP surface) */ @@ -79,7 +79,7 @@ export function createLLMClient(config: LLMConfig): LLMClient { headers: config.headers, }); } - // openAICredential is only non-undefined when config.format === 'openai' (see resolveOpenAICredential) + // openAICredential is set only when config.format === 'openai' if (openAICredential?.source === 'codex') { // Pass authMode:'codex' so Pi re-reads ~/.codex/auth.json and sets source:'codex', // which is required for resolvePiModel to route to the openai-codex provider @@ -131,7 +131,7 @@ export function createLLMClient(config: LLMConfig): LLMClient { headers: config.headers, }); } - // openAICredential is only non-undefined when config.format === 'openai' (see resolveOpenAICredential) + // openAICredential is set only when config.format === 'openai' if (openAICredential?.source === 'codex') { return chatWithToolsPi({ timeout, @@ -184,7 +184,7 @@ export function createLLMClient(config: LLMConfig): LLMClient { headers: config.headers, }); } - // openAICredential is only non-undefined when config.format === 'openai' (see resolveOpenAICredential) + // openAICredential is set only when config.format === 'openai' if (openAICredential?.source === 'codex') { return chatAgentLoopPi({ timeout, diff --git a/src/benchmark/llm/openai-format.ts b/src/benchmark/llm/openai-format.ts index 8a7c7b9..a2441f2 100644 --- a/src/benchmark/llm/openai-format.ts +++ b/src/benchmark/llm/openai-format.ts @@ -1,5 +1,13 @@ import type { LLMResponse, McpToolDefinition, ToolExecutor } from '../types.js'; import { createToolNameAliasCodec } from './tool-name-aliases.js'; +import { + isAbortError, + isRetryableError, + sleep, + truncateToolResult, + undefinedIfEmpty, + normalizeUsage, +} from './shared.js'; interface CallParams { baseUrl: string; @@ -15,10 +23,6 @@ interface CallWithToolsParams extends CallParams { tools: McpToolDefinition[]; } -/** - * Regular chat completion (code mode). - * POST {baseUrl}/chat/completions - */ export async function chatOpenAI(params: CallParams): Promise { const body = { model: params.modelId, @@ -31,10 +35,6 @@ export async function chatOpenAI(params: CallParams): Promise { return callWithRetry(params, body); } -/** - * Chat with tools (MCP mode). - * POST {baseUrl}/chat/completions with tools array - */ export async function chatWithToolsOpenAI(params: CallWithToolsParams): Promise { const toolCodec = createToolNameAliasCodec(params.tools); const body = { @@ -76,8 +76,8 @@ async function doFetch(params: CallParams, body: Record): Promi if (!response.ok) { const text = await response.text(); - const err = new Error(`OpenAI API error ${response.status}: ${text}`); - (err as any).status = response.status; + const err = new Error(`OpenAI API error ${response.status}: ${text}`) as Error & { status: number }; + err.status = response.status; throw err; } @@ -139,8 +139,8 @@ export async function chatAgentLoopOpenAI(params: AgentLoopParams): Promise }> = []; - let totalUsage = { prompt: 0, completion: 0, total: 0 }; + const allToolCalls: Array<{ name: string; arguments: Record }> = []; + const totalUsage = { prompt: 0, completion: 0, total: 0 }; for (let turn = 0; turn < params.maxTurns; turn++) { const body: Record = { @@ -153,7 +153,7 @@ export async function chatAgentLoopOpenAI(params: AgentLoopParams): Promise 0 ? allToolCalls : undefined, usage: totalUsage.total > 0 ? totalUsage : undefined }; + return { content: response.content, toolCalls: undefinedIfEmpty(allToolCalls), usage: normalizeUsage(totalUsage) }; } allToolCalls.push(...response.toolCalls); messages.push({ @@ -167,19 +167,15 @@ export async function chatAgentLoopOpenAI(params: AgentLoopParams): Promise MAX_TOOL_RESULT_CHARS) { - result = result.slice(0, MAX_TOOL_RESULT_CHARS) + '\n\n[... truncated]'; - } - messages.push({ role: 'tool', tool_call_id: `call_${turn}_${i}`, content: result }); + messages.push({ role: 'tool', tool_call_id: `call_${turn}_${i}`, content: truncateToolResult(result) }); } } - return { content: '', toolCalls: allToolCalls.length > 0 ? allToolCalls : undefined, usage: totalUsage.total > 0 ? totalUsage : undefined }; + return { content: '', toolCalls: undefinedIfEmpty(allToolCalls), usage: normalizeUsage(totalUsage) }; } async function callWithRetry( @@ -187,12 +183,8 @@ async function callWithRetry( body: Record, toCanonicalToolName: (name: string) => string = (name) => name, ): Promise { - try { - const response = await doFetch(params, body); - if (!response.toolCalls || response.toolCalls.length === 0) { - return response; - } - + const applyCodec = (response: LLMResponse): LLMResponse => { + if (!response.toolCalls || response.toolCalls.length === 0) return response; return { ...response, toolCalls: response.toolCalls.map((toolCall) => ({ @@ -200,30 +192,15 @@ async function callWithRetry( name: toCanonicalToolName(toolCall.name), })), }; + }; + + try { + return applyCodec(await doFetch(params, body)); } catch (err) { - if (err && typeof err === 'object' && 'name' in err && err.name === 'AbortError') { - throw err; - } - // Don't retry client errors (4xx) except 429 (rate limit) - if (err && typeof err === 'object' && 'status' in err) { - const status = (err as any).status; - if (typeof status === 'number' && status >= 400 && status < 500 && status !== 429) { - throw err; - } - } + if (isAbortError(err)) throw err; + if (!isRetryableError(err)) throw err; // Retry once after 3s - await new Promise((resolve) => setTimeout(resolve, 3_000)); - const response = await doFetch(params, body); - if (!response.toolCalls || response.toolCalls.length === 0) { - return response; - } - - return { - ...response, - toolCalls: response.toolCalls.map((toolCall) => ({ - ...toolCall, - name: toCanonicalToolName(toolCall.name), - })), - }; + await sleep(3_000); + return applyCodec(await doFetch(params, body)); } } diff --git a/src/benchmark/llm/pi-format.ts b/src/benchmark/llm/pi-format.ts index 763d9a8..76149ea 100644 --- a/src/benchmark/llm/pi-format.ts +++ b/src/benchmark/llm/pi-format.ts @@ -1,13 +1,14 @@ import { Type } from '@mariozechner/pi-ai'; -import type { Context, Model, AssistantMessage, SimpleStreamOptions, Tool as PiTool } from '@mariozechner/pi-ai'; +import type { Api, Context, Model, AssistantMessage, SimpleStreamOptions, Tool as PiTool } from '@mariozechner/pi-ai'; import { complete, completeSimple } from '@mariozechner/pi-ai'; import type { LLMResponse, McpToolDefinition, ToolExecutor } from '../types.js'; import { resolvePiModelByRef } from '../../runtime/pi/index.js'; +import type { PiAuthMode } from '../../runtime/pi/auth.js'; import { createToolNameAliasCodec } from './tool-name-aliases.js'; interface PiCallParams { - authMode?: import('../../runtime/pi/auth.js').PiAuthMode; + authMode?: PiAuthMode; apiKeyOverride?: string; apiKeyEnv?: string; headers?: Record; @@ -22,7 +23,7 @@ interface PiCallWithToolsParams extends PiCallParams { } interface ResolvedPiRequest { - model: Model; + model: Model; auth: { apiKey?: string; headers?: Record; @@ -33,13 +34,13 @@ type PiImplementationSet = { resolve( modelId: string, authOptions?: { - authMode?: import('../../runtime/pi/auth.js').PiAuthMode; + authMode?: PiAuthMode; apiKeyEnv?: string; apiKeyOverride?: string; }, ): Promise; - completeSimple(model: Model, context: Context, options?: SimpleStreamOptions): Promise; - complete(model: Model, context: Context, options?: SimpleStreamOptions): Promise; + completeSimple(model: Model, context: Context, options?: SimpleStreamOptions): Promise; + complete(model: Model, context: Context, options?: SimpleStreamOptions): Promise; }; let piImplementationsForTest: PiImplementationSet | null = null; @@ -177,7 +178,7 @@ function getPiImplementations(): PiImplementationSet { async function resolvePiRequest( modelId: string, authOptions?: { - authMode?: import('../../runtime/pi/auth.js').PiAuthMode; + authMode?: PiAuthMode; apiKeyEnv?: string; apiKeyOverride?: string; }, @@ -211,7 +212,7 @@ function toPiTool(tool: McpToolDefinition): PiTool { return { name: tool.function.name, description: tool.function.description ?? '', - parameters: Type.Unsafe((tool.function.parameters ?? { type: 'object', properties: {}, required: [] }) as any), + parameters: Type.Unsafe(tool.function.parameters ?? { type: 'object', properties: {}, required: [] }), }; } diff --git a/src/benchmark/llm/shared.ts b/src/benchmark/llm/shared.ts new file mode 100644 index 0000000..f756d8d --- /dev/null +++ b/src/benchmark/llm/shared.ts @@ -0,0 +1,54 @@ +/** + * Shared utilities for LLM format handlers. + * + * Keeping these in one place avoids duplicating identical logic across + * openai-format.ts, anthropic-format.ts, and any future format handlers. + */ + +import type { LLMResponse } from '../types.js'; + +/** Maximum tool-result characters forwarded to the model in agent loops. */ +const MAX_TOOL_RESULT_CHARS = 50_000; +const TRUNCATED_SUFFIX = '\n\n[... truncated]'; + +/** + * Truncate a tool result string so it does not exceed MAX_TOOL_RESULT_CHARS. + * Returns the original string if it is already within the limit. + */ +export function truncateToolResult(result: string): string { + if (result.length <= MAX_TOOL_RESULT_CHARS) return result; + return result.slice(0, MAX_TOOL_RESULT_CHARS - TRUNCATED_SUFFIX.length) + TRUNCATED_SUFFIX; +} + +export function isAbortError(err: unknown): boolean { + return Boolean(err && typeof err === 'object' && 'name' in err && (err as { name: unknown }).name === 'AbortError'); +} + +/** + * Return true if the error is a transient server error that should be retried. + * Does NOT retry abort errors or 4xx client errors (except 429 rate-limit). + */ +export function isRetryableError(err: unknown): boolean { + if (isAbortError(err)) return false; + if (err && typeof err === 'object' && 'status' in err) { + const status = (err as { status: unknown }).status; + if (typeof status === 'number' && status >= 400 && status < 500 && status !== 429) { + return false; + } + } + return true; +} + +export function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +export function undefinedIfEmpty(arr: T[]): T[] | undefined { + return arr.length > 0 ? arr : undefined; +} + +export function normalizeUsage( + usage: { prompt: number; completion: number; total: number }, +): LLMResponse['usage'] { + return usage.total > 0 ? usage : undefined; +} diff --git a/src/benchmark/llm/tool-name-aliases.ts b/src/benchmark/llm/tool-name-aliases.ts index e8cab6e..5ddda1e 100644 --- a/src/benchmark/llm/tool-name-aliases.ts +++ b/src/benchmark/llm/tool-name-aliases.ts @@ -2,7 +2,7 @@ import type { McpToolDefinition } from '../types.js'; const PROVIDER_TOOL_NAME_PATTERN = /[^a-zA-Z0-9_-]/g; -export interface ToolNameAliasCodec { +interface ToolNameAliasCodec { tools: McpToolDefinition[]; toCanonical(name: string): string; toProvider(name: string): string; diff --git a/src/benchmark/prompts.ts b/src/benchmark/prompts.ts index 5d5029c..065e280 100644 --- a/src/benchmark/prompts.ts +++ b/src/benchmark/prompts.ts @@ -2,7 +2,7 @@ import type { FetchedSkill, TaskDefinition, SdkLanguage } from './types.js'; export type PromptSurface = 'sdk' | 'cli' | 'mcp' | 'prompt'; -export interface PromptOptions { +interface PromptOptions { surface: PromptSurface; agentic?: boolean; shell?: 'bash' | 'sh'; diff --git a/src/benchmark/reporter.ts b/src/benchmark/reporter.ts index 710769d..acabc52 100644 --- a/src/benchmark/reporter.ts +++ b/src/benchmark/reporter.ts @@ -11,17 +11,14 @@ function fixed2(n: number): string { return n.toFixed(2); } -/** Pad a string to a fixed width (left-aligned). */ function padR(s: string, w: number): string { return s.length >= w ? s.slice(0, w) : s + ' '.repeat(w - s.length); } -/** Pad a string to a fixed width (right-aligned). */ function padL(s: string, w: number): string { return s.length >= w ? s.slice(0, w) : ' '.repeat(w - s.length) + s; } -/** Center a string in a fixed width. */ function center(s: string, w: number): string { if (s.length >= w) return s.slice(0, w); const total = w - s.length; @@ -32,9 +29,6 @@ function center(s: string, w: number): string { // ── Markdown generation ─────────────────────────────────────────────────────── -/** - * Generate a Markdown report string. - */ export function generateMarkdown(report: BenchmarkReport): string { const lines: string[] = []; const { summary, skillVersion, coverage, results } = report; diff --git a/src/benchmark/runner.ts b/src/benchmark/runner.ts index ddb0687..0457736 100644 --- a/src/benchmark/runner.ts +++ b/src/benchmark/runner.ts @@ -110,14 +110,12 @@ export interface RunnerOptions { * Run the full benchmark. */ export async function runBenchmark(options: RunnerOptions = {}): Promise { - // 1. Load config const { config, configDir } = await loadConfig(options.configPath); console.log('================================================================'); console.log(` Skill Benchmark — ${config.name}`); console.log('================================================================\n'); - // 2. Load tasks first (needed for known action derivation) if (config.tasks === '__generated__') { throw new Error( 'This benchmark config uses task generation (benchmark.taskGeneration.enabled=true). ' + @@ -127,7 +125,6 @@ export async function runBenchmark(options: RunnerOptions = {}): Promise; let cliCommands: ReturnType | undefined = undefined; let mcpToolDefs: ReturnType | undefined = undefined; @@ -135,20 +132,18 @@ export async function runBenchmark(options: RunnerOptions = {}): Promise(); } else if (config.surface === 'sdk') { - // Derive from task expected actions + optional sdk.apiSurface const fromTasks = new Set(); for (const task of tasks) { for (const action of getExpectedActions(task)) { fromTasks.add(getExpectedActionName(action)); } } - const apiSurface = config.sdk?.apiSurface ?? config.sdk?.methods ?? []; + const apiSurface = config.sdk?.apiSurface ?? []; knownMethods = new Set([...fromTasks, ...apiSurface]); } else if (config.surface === 'cli') { cliCommands = loadCliCommands(config.cli!.commands, configDir); knownMethods = new Set(cliCommands.map((c) => c.command)); } else { - // MCP surface: load tools once, reuse for both knownMethods and chatWithTools mcpToolDefs = config.mcpToolDefinitions ?? (config.mcp ? loadMcpTools(config.mcp.tools, configDir) : []); knownMethods = new Set(mcpToolDefs.map(t => t.function.name)); } @@ -162,7 +157,6 @@ export async function runBenchmark(options: RunnerOptions = {}): Promise | undefined; @@ -365,7 +352,6 @@ export async function runBenchmark(options: RunnerOptions = {}): Promise= policy.minImprovement; } diff --git a/src/benchmark/types.ts b/src/benchmark/types.ts index 2a9b680..26edd19 100644 --- a/src/benchmark/types.ts +++ b/src/benchmark/types.ts @@ -3,6 +3,7 @@ import type { ActionAttempt, ActionCatalog, ActionDefinition, + ActionSurface, } from '../actions/types.js'; // === Core === @@ -23,7 +24,8 @@ export interface TokenUsage { // === Config (loaded from benchmark.config.json) === -export type BenchmarkSurface = 'sdk' | 'cli' | 'mcp' | 'prompt'; +/** Canonical surface type — same union as ActionSurface in actions/types. */ +export type BenchmarkSurface = ActionSurface; export type SdkLanguage = 'typescript' | 'python' | 'rust'; export interface BenchmarkConfig { @@ -45,13 +47,8 @@ export interface SdkSurfaceConfig { language: SdkLanguage; style?: 'sdk'; // defaults to 'sdk' if omitted // Optional: explicit API surface for coverage/hallucination reporting only. - // If omitted, derived automatically from task expected_tools. + // If omitted, derived automatically from task expected_actions. apiSurface?: string[]; - // Deprecated fields — kept for backward compat, no longer required - classes?: string[]; - functions?: string[]; - functionReturns?: Record; - methods?: string[]; } export interface CliSurfaceConfig { @@ -76,10 +73,6 @@ export interface McpSurfaceConfig { tools: string; // path to tools.json (OpenAI function calling format) } -// Backward-compatible aliases retained for internal usage. -export type CodeModeConfig = SdkSurfaceConfig; -export type McpModeConfig = McpSurfaceConfig; - export interface SkillConfig { source: string; // "github:org/repo/path", "./file.md", "https://url" cache?: boolean; // default true @@ -139,14 +132,10 @@ export interface FetchedSkill { // === Task Definition (loaded from tasks.json) === export interface ExpectedAction { - name?: string; // Unified action name (SDK method, CLI command, or MCP tool) - method?: string; // Transitional alias for older internal code paths + name: string; // Unified action name (SDK method, CLI command, or MCP tool) args?: Record; // expected arg values (supports nested objects/arrays, strings, regexes, sentinels) } -// Transitional alias retained while internal code paths migrate. -export type ExpectedTool = ExpectedAction; - export interface TaskVerification { code_pattern?: string; // regex pattern to match in generated code } @@ -154,8 +143,7 @@ export interface TaskVerification { export interface TaskDefinition { id: string; prompt: string; - expected_actions?: ExpectedAction[]; - expected_tools?: ExpectedAction[]; // transitional alias populated by the loader + expected_actions: ExpectedAction[]; verify?: TaskVerification[]; expected_fetches?: string[]; } @@ -194,17 +182,13 @@ export interface ActionMatch { }>; } -// Transitional alias retained while internal code paths migrate. -export type ToolMatch = ActionMatch; - export interface TaskResult { task: TaskDefinition; model: ModelConfig; generatedCode: string | null; rawResponse: string; extractedCalls: ExtractedCall[]; - actionMatches?: ActionMatch[]; - toolMatches: ActionMatch[]; // transitional alias + actionMatches: ActionMatch[]; codePatternResults?: Record; metrics: { toolPrecision: number; @@ -212,10 +196,8 @@ export interface TaskResult { taskPassed: boolean; toolSelectionAccuracy: number; argAccuracy: number; - unnecessaryActions?: string[]; - unnecessaryCalls: string[]; // transitional alias - hallucinatedActions?: string[]; - hallucinatedCalls: string[]; // transitional alias + unnecessaryActions: string[]; + hallucinatedActions: string[]; hallucinationRate: number; fetchRecall?: number; fetchPrecision?: number; @@ -234,8 +216,6 @@ export interface MethodCoverage { covered: boolean; } -export type SurfaceActionArg = ActionArgSchema; - export type SurfaceAction = Omit; export interface SurfaceSnapshot extends Omit { @@ -311,11 +291,11 @@ export interface BenchmarkReport { } export function getExpectedActionName(action: ExpectedAction): string { - return action.name || action.method || ''; + return action.name; } export function getExpectedActions(task: TaskDefinition): ExpectedAction[] { - return task.expected_actions ?? task.expected_tools ?? []; + return task.expected_actions; } // === Comparison === diff --git a/src/cli.ts b/src/cli.ts index 46d019d..b7583c4 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -196,7 +196,7 @@ Examples: skill-optimizer run --no-cache skill-optimizer generate-tasks --config ./skill-optimizer.json skill-optimizer optimize --config ./skill-optimizer.json - skill-optimizer compare --baseline results/old/report.json --current results/report.json + skill-optimizer compare --baseline results/baseline/report.json --current results/report.json `); } @@ -233,7 +233,6 @@ async function runDryRun(configPath: string): Promise { // ── Main ────────────────────────────────────────────────────────────────────── async function main(): Promise { - // Strip node + script path from argv const args = process.argv.slice(2); if (hasFlag(args, '--help') || hasFlag(args, '-h')) { @@ -345,12 +344,12 @@ async function main(): Promise { if (!baselinePath) { console.error('ERROR: --baseline is required for compare mode.'); - console.error(' Example: skill-optimizer compare --baseline results/old/report.json --current results/report.json'); + console.error(' Example: skill-optimizer compare --baseline results/baseline/report.json --current results/report.json'); process.exit(1); } if (!currentPath) { console.error('ERROR: --current is required for compare mode.'); - console.error(' Example: skill-optimizer compare --baseline results/old/report.json --current results/report.json'); + console.error(' Example: skill-optimizer compare --baseline results/baseline/report.json --current results/report.json'); process.exit(1); } @@ -486,9 +485,8 @@ async function main(): Promise { `Edit "target.repoPath" in ${project.configPath}.`, ); } - // Preflight: check credentials for every unique provider in the benchmark config. - // For direct-API formats there is one provider; for pi format each model may need - // a different key (e.g. openrouter + openai in the same run). + // Check credentials for each unique provider: direct-API formats have one provider, + // but pi format may route different models to different providers (e.g. openrouter + openai). const benchmarkProviders = project.benchmark.format === 'openai' ? ['openai'] : project.benchmark.format === 'anthropic' @@ -545,18 +543,14 @@ async function main(): Promise { fatalError(err); } - // Print console summary printSummary(report); - - // Print coverage printCoverage(report.coverage); - // Determine output dir — resolve relative to the config file's directory (matching the runner) + // Resolve output dir relative to the config file's directory, matching the runner's behavior const configFileDir = options.configPath ? dirname(resolve(options.configPath)) : process.cwd(); const reportConfig = report.config as { name: string; surface: string; outputDir?: string }; - const outputDir = resolve(configFileDir, reportConfig?.outputDir ?? 'benchmark-results'); + const outputDir = resolve(configFileDir, reportConfig.outputDir ?? 'benchmark-results'); - // Generate recommendations if verdict is FAIL let recommendations: Recommendation[] = []; if (report.verdict?.result === 'FAIL' && project) { try { @@ -578,10 +572,8 @@ async function main(): Promise { } } - // Print verdict console.log(renderVerdictConsole(report, recommendations)); - // Generate and save Markdown report alongside JSON const mdPath = resolve(outputDir, 'report.md'); try { const markdown = generateMarkdown(report) + '\n\n' + renderVerdictMarkdown(report, recommendations); @@ -591,7 +583,6 @@ async function main(): Promise { console.error(`WARNING: Could not write Markdown report: ${err instanceof Error ? err.message : err}`); } - // Final summary line const { summary } = report; const passedCount = Math.round(summary.overallPassRate * summary.totalEvaluations); console.log( diff --git a/src/discovery/cli.ts b/src/discovery/cli.ts index 457da75..95cc603 100644 --- a/src/discovery/cli.ts +++ b/src/discovery/cli.ts @@ -3,7 +3,8 @@ import { resolve } from 'node:path'; import ts from 'typescript'; -import type { CliDiscoverySnapshot, DiscoveryOptions, DiscoveredAction, DiscoveredActionArg } from './types.js'; +import type { ActionArgSchema } from '../actions/types.js'; +import type { CliDiscoverySnapshot, DiscoveryOptions, DiscoveredAction } from './types.js'; import { discoverOptiqueActionsFromFile } from './optique.js'; type LiteralPrimitive = string | number | boolean | null; @@ -279,12 +280,12 @@ function toDiscoveredAction(commandEntry: LiteralObject, source: string): Discov }; } -function extractActionArgs(options: LiteralValue | undefined): DiscoveredActionArg[] { +function extractActionArgs(options: LiteralValue | undefined): ActionArgSchema[] { if (!Array.isArray(options)) { return []; } - const args: DiscoveredActionArg[] = []; + const args: ActionArgSchema[] = []; for (const option of options) { if (!isLiteralObject(option)) { continue; diff --git a/src/discovery/index.ts b/src/discovery/index.ts deleted file mode 100644 index 5591003..0000000 --- a/src/discovery/index.ts +++ /dev/null @@ -1,20 +0,0 @@ -export { discoverCliSurfaceFromSources } from './cli.js'; -export { discoverMcpSurfaceFromSources } from './mcp.js'; -export { discoverPromptSurfaceFromContent, discoverPromptSurfaceFromSources } from './prompt.js'; -export { discoverSdkSurfaceFromSources } from './sdk.js'; - -export type { - CliDiscoverySnapshot, - DiscoveryOptions, - DiscoverySnapshot, - DiscoveredAction, - DiscoveredActionArg, - McpDiscoverySnapshot, - SdkDiscoverySnapshot, -} from './types.js'; - -export type { - PromptCapability, - PromptDiscoverySnapshot, - PromptPhase, -} from './prompt.js'; diff --git a/src/discovery/mcp.ts b/src/discovery/mcp.ts index 292f029..ebf5770 100644 --- a/src/discovery/mcp.ts +++ b/src/discovery/mcp.ts @@ -3,7 +3,8 @@ import { resolve } from 'node:path'; import ts from 'typescript'; -import type { DiscoveryOptions, DiscoveredAction, DiscoveredActionArg, McpDiscoverySnapshot } from './types.js'; +import type { ActionArgSchema } from '../actions/types.js'; +import type { DiscoveryOptions, DiscoveredAction, McpDiscoverySnapshot } from './types.js'; type LiteralPrimitive = string | number | boolean | null; interface LiteralObject { @@ -290,7 +291,7 @@ function toDiscoveredAction(toolDefinition: LiteralObject, source: string): Disc const properties = parameters && isLiteralObject(parameters.properties) ? parameters.properties : {}; const requiredNames = parameters ? asRequiredArray(parameters.required) : []; - const args: DiscoveredActionArg[] = []; + const args: ActionArgSchema[] = []; for (const [argName, schemaValue] of Object.entries(properties)) { if (!isLiteralObject(schemaValue)) { continue; diff --git a/src/discovery/optique.ts b/src/discovery/optique.ts index e0bc4f2..d88c58d 100644 --- a/src/discovery/optique.ts +++ b/src/discovery/optique.ts @@ -12,7 +12,8 @@ import { existsSync, readFileSync } from 'node:fs'; import ts from 'typescript'; -import type { DiscoveredAction, DiscoveredActionArg } from './types.js'; +import type { ActionArgSchema } from '../actions/types.js'; +import type { DiscoveredAction } from './types.js'; export function discoverOptiqueActionsFromFile(filePath: string): DiscoveredAction[] { if (!existsSync(filePath)) { @@ -156,12 +157,12 @@ function resolveExpr(expr: ts.Expression, constants: Map) function extractArgsFromObjectCall( objectCall: ts.CallExpression, constants: Map, -): DiscoveredActionArg[] { +): ActionArgSchema[] { if (objectCall.arguments.length === 0) return []; const objLiteral = unwrap(objectCall.arguments[0]); if (!ts.isObjectLiteralExpression(objLiteral)) return []; - const result: DiscoveredActionArg[] = []; + const result: ActionArgSchema[] = []; for (const prop of objLiteral.properties) { if (!ts.isPropertyAssignment(prop)) continue; const key = ts.isIdentifier(prop.name) ? prop.name.text : null; @@ -182,7 +183,7 @@ function parseArgCall( expr: ts.Expression, constants: Map, forceOptional: boolean, -): DiscoveredActionArg | null { +): ActionArgSchema | null { const node = unwrap(expr); if (!ts.isCallExpression(node)) return null; @@ -213,7 +214,7 @@ function parseArgCall( function extractOption( call: ts.CallExpression, constants: Map, -): DiscoveredActionArg | null { +): ActionArgSchema | null { if (call.arguments.length === 0) return null; const name = extractStringLiteral(call.arguments[0]); if (!name) return null; diff --git a/src/discovery/sdk.ts b/src/discovery/sdk.ts index d2caefd..3485d96 100644 --- a/src/discovery/sdk.ts +++ b/src/discovery/sdk.ts @@ -3,7 +3,8 @@ import { extname, resolve } from 'node:path'; import ts from 'typescript'; -import type { DiscoveryOptions, DiscoveredAction, DiscoveredActionArg, SdkDiscoverySnapshot } from './types.js'; +import type { ActionArgSchema } from '../actions/types.js'; +import type { DiscoveryOptions, DiscoveredAction, SdkDiscoverySnapshot } from './types.js'; export function discoverSdkSurfaceFromSources(sources: string[], options: DiscoveryOptions = {}): SdkDiscoverySnapshot { const baseDir = options.baseDir ?? process.cwd(); @@ -227,8 +228,8 @@ function actionFromFunctionDeclaration(functionDeclaration: ts.FunctionDeclarati }; } -function extractArgsFromParameters(parameters: ts.NodeArray): DiscoveredActionArg[] { - const args: DiscoveredActionArg[] = []; +function extractArgsFromParameters(parameters: ts.NodeArray): ActionArgSchema[] { + const args: ActionArgSchema[] = []; for (const parameter of parameters) { if (!ts.isIdentifier(parameter.name)) { diff --git a/src/discovery/types.ts b/src/discovery/types.ts index c91eac7..ab7bb78 100644 --- a/src/discovery/types.ts +++ b/src/discovery/types.ts @@ -1,15 +1,9 @@ -export interface DiscoveredActionArg { - name: string; - required: boolean; - type?: string; - description?: string; - schema?: Record; -} +import type { ActionArgSchema } from '../actions/types.js'; export interface DiscoveredAction { name: string; description?: string; - args: DiscoveredActionArg[]; + args: ActionArgSchema[]; source?: string; } diff --git a/src/doctor/index.ts b/src/doctor/index.ts index 4c44bab..a2754be 100644 --- a/src/doctor/index.ts +++ b/src/doctor/index.ts @@ -8,7 +8,7 @@ import { checkDiscovery, checkModelReachability } from './checks.js'; import { formatIssues, formatFixResult } from './format.js'; import type { Issue } from '../project/validate.js'; -export interface DoctorOptions { +interface DoctorOptions { staticOnly?: boolean; checkModels?: boolean; fix?: boolean; diff --git a/src/errors.ts b/src/errors.ts index d19a9ce..ef21d1a 100644 --- a/src/errors.ts +++ b/src/errors.ts @@ -1,4 +1,3 @@ -// src/errors.ts export interface ErrorDef { code: string; message: string; @@ -28,8 +27,8 @@ export const ERRORS = { code: 'E_INVALID_SURFACE', message: 'Invalid surface value', fix: [ - 'Set target.surface to one of: sdk, cli, mcp', - 'sdk = TypeScript/Python/Rust library, cli = command-line tool, mcp = MCP server', + 'Set target.surface to one of: sdk, cli, mcp, prompt', + 'sdk = TypeScript/Python/Rust library, cli = command-line tool, mcp = MCP server, prompt = prompt template / skill document', ], }, E_MODELS_EMPTY: { @@ -140,14 +139,6 @@ export const ERRORS = { 'Get a key at https://openrouter.ai/keys', ], }, - E_LEGACY_CONFIG: { - code: 'E_LEGACY_CONFIG', - message: 'Found skill-benchmark.json instead of skill-optimizer.json', - fix: [ - 'Rename skill-benchmark.json to skill-optimizer.json', - 'See CHANGELOG.md for any field renames between versions', - ], - }, // ── Discovery ───────────────────────────────────────────────────────────── E_DISCOVERY_EMPTY: { code: 'E_DISCOVERY_EMPTY', @@ -223,7 +214,7 @@ export const ERRORS = { message: 'An unexpected error occurred', fix: [ 'Check the full error message and stack trace above for details', - 'File an issue at https://github.com/bucurdavid/skill-optimizer/issues with the full output', + 'File an issue at https://github.com/fastxyz/skill-optimizer/issues with the full output', ], }, } as const satisfies Record; diff --git a/src/import/extractors/py-click.ts b/src/import/extractors/py-click.ts index 6773512..465e034 100644 --- a/src/import/extractors/py-click.ts +++ b/src/import/extractors/py-click.ts @@ -92,7 +92,6 @@ function walkTree(node: Parser.SyntaxNode, source: string): CliCommandDefinition } } - // Recurse into children for (const child of node.children) { results.push(...walkTree(child, source)); } diff --git a/src/import/extractors/rs-clap.ts b/src/import/extractors/rs-clap.ts index fcfdcb4..4390afe 100644 --- a/src/import/extractors/rs-clap.ts +++ b/src/import/extractors/rs-clap.ts @@ -33,20 +33,16 @@ export async function extractClap(filePath: string): Promise 0) return 'hallucination'; - if ((result.actionMatches ?? result.toolMatches).some((match) => match.methodFound && !match.argsCorrect)) return 'bad-args'; - if ((result.actionMatches ?? result.toolMatches).some((match) => !match.methodFound)) return 'missing-tool'; + if (result.metrics.hallucinatedActions.length > 0) return 'hallucination'; + if (result.actionMatches.some((match) => match.methodFound && !match.argsCorrect)) return 'bad-args'; + if (result.actionMatches.some((match) => !match.methodFound)) return 'missing-tool'; return null; } diff --git a/src/optimizer/feedback/failure-details.ts b/src/optimizer/feedback/failure-details.ts index ed29750..2f96cd8 100644 --- a/src/optimizer/feedback/failure-details.ts +++ b/src/optimizer/feedback/failure-details.ts @@ -19,8 +19,8 @@ export function extractFailureDetails(results: TaskResult[]): FailureDetail[] { if (r.metrics.taskPassed) continue; const actual = r.extractedCalls.map((c) => ({ - action: (c as unknown as { method: string }).method ?? (c as unknown as { name: string }).name ?? '', - args: ((c as unknown as { args?: Record }).args) ?? {}, + action: c.method, + args: c.args, })); if (r.error) { @@ -36,7 +36,7 @@ export function extractFailureDetails(results: TaskResult[]): FailureDetail[] { continue; } - const matches = r.actionMatches ?? r.toolMatches; + const matches = r.actionMatches; for (const m of matches) { const expectedName = getExpectedActionName(m.expected); // Cross-check whether the expected method actually appears in extracted calls, @@ -72,7 +72,7 @@ export function extractFailureDetails(results: TaskResult[]): FailureDetail[] { } } - if (r.metrics.hallucinatedCalls?.length) { + if (r.metrics.hallucinatedActions.length > 0) { out.push({ task_id: r.task.id, model_id: r.model.id, @@ -80,7 +80,7 @@ export function extractFailureDetails(results: TaskResult[]): FailureDetail[] { expected_action: matches.map((m) => getExpectedActionName(m.expected)).join(', '), expected_args: {}, actual_calls: actual, - mismatch_detail: `hallucinated: ${r.metrics.hallucinatedCalls.join(', ')}`, + mismatch_detail: `hallucinated: ${r.metrics.hallucinatedActions.join(', ')}`, }); } } diff --git a/src/optimizer/feedback/mutation-context.ts b/src/optimizer/feedback/mutation-context.ts index 7e1345d..474a16d 100644 --- a/src/optimizer/feedback/mutation-context.ts +++ b/src/optimizer/feedback/mutation-context.ts @@ -3,7 +3,7 @@ import { extractFailureDetails, type FailureDetail } from './failure-details.js' import { detectPatterns, type Pattern } from './patterns.js'; import { buildPassingFailingDiff, type PassingFailingDiff } from './passing-failing-diff.js'; -export interface FeedbackPackage { +interface FeedbackPackage { failureDetails: FailureDetail[]; patterns: Pattern[]; passingFailingDiffs: PassingFailingDiff[]; diff --git a/src/optimizer/feedback/passing-failing-diff.ts b/src/optimizer/feedback/passing-failing-diff.ts index 2e1e28e..b869ef6 100644 --- a/src/optimizer/feedback/passing-failing-diff.ts +++ b/src/optimizer/feedback/passing-failing-diff.ts @@ -29,11 +29,11 @@ export function buildPassingFailingDiff(results: TaskResult[]): PassingFailingDi failing_models: failing.map((r) => r.model.name), passing_calls: passing.map((r) => ({ model: r.model.name, - actions: r.extractedCalls.map((c) => (c as unknown as { method: string }).method ?? ''), + actions: r.extractedCalls.map((c) => c.method), })), failing_calls: failing.map((r) => ({ model: r.model.name, - actions: r.extractedCalls.map((c) => (c as unknown as { method: string }).method ?? ''), + actions: r.extractedCalls.map((c) => c.method), })), }); } diff --git a/src/optimizer/loop.ts b/src/optimizer/loop.ts index 8bb80f1..b53289b 100644 --- a/src/optimizer/loop.ts +++ b/src/optimizer/loop.ts @@ -141,15 +141,11 @@ export async function runOptimizeLoop( console.log( `[optimize] Changed files: ${changedFiles.length > 0 ? changedFiles.join(', ') : '(none)'}`, ); - // When the mutation writes to a local skill file, the target repo must stay clean. - // The agent runs with cwd = targetRepo and can inadvertently write tracked files there. - // Restore the repo to the accepted checkpoint so any rogue writes are undone — only - // changes to the local skill file (outside the repo) are kept. + // In local-skill mode: restore the repo to undo any rogue writes the agent may have made + // (it runs with cwd=targetRepo), then skip scope validation (the local file is always in scope). if (localSkillPath) { await deps.repo.restoreCheckpoint(resolvedManifest.targetRepo, acceptedCheckpoint); } - // When the mutation writes to a local skill file (not the target repo), - // skip target-repo scope validation — the local file is always in scope. const scopeValidation = localSkillPath ? null : validateChangedFiles(changedFiles, resolvedManifest); @@ -285,7 +281,7 @@ export async function runOptimizeLoop( const accepted = accept(beforeReport, afterReport, resolvedManifest.optimizer.models, { perModelFloor: resolvedManifest.optimizer.perModelFloor, targetWeightedAverage: resolvedManifest.optimizer.targetWeightedAverage, - minImprovement: resolvedManifest.optimizer.minOverallPassDelta, + minImprovement: resolvedManifest.optimizer.minImprovement, }); if (accepted) { @@ -314,7 +310,7 @@ export async function runOptimizeLoop( console.log( `[optimize] Rejected iteration ${index}: gates not satisfied ` + `(weighted ${beforeAvg.toFixed(1)}% -> ${afterAvg.toFixed(1)}%; ` + - `min improvement ${(resolvedManifest.optimizer.minOverallPassDelta * 100).toFixed(1)} pts; ` + + `min improvement ${(resolvedManifest.optimizer.minImprovement * 100).toFixed(1)} pts; ` + `per-model floor ${(resolvedManifest.optimizer.perModelFloor * 100).toFixed(1)}%).`, ); console.log('[optimize] Restoring checkpoint.'); @@ -376,7 +372,7 @@ function summarizeTopFailures(report: BenchmarkReport, limit = 3): string[] { existing.failCount += 1; existing.models.add(result.model.name); - for (const match of result.actionMatches ?? result.toolMatches) { + for (const match of result.actionMatches) { if (!match.methodFound) { existing.missing.add(getExpectedActionName(match.expected)); } else if (!match.argsCorrect) { @@ -511,7 +507,7 @@ function resolveManifest(manifest: OptimizeManifest | ResolvedOptimizeManifest): optimizer: { maxIterations: unresolved.optimizer?.maxIterations ?? 5, stabilityWindow: unresolved.optimizer?.stabilityWindow ?? 2, - minOverallPassDelta: unresolved.optimizer?.minOverallPassDelta ?? 0.02, + minImprovement: unresolved.optimizer?.minImprovement ?? 0.02, taskGeneration: { enabled: unresolved.optimizer?.taskGeneration?.enabled ?? false, maxGenerated: unresolved.optimizer?.taskGeneration?.maxGenerated ?? 10, diff --git a/src/optimizer/materialize-mock-repo.ts b/src/optimizer/materialize-mock-repo.ts index 42df1e8..a30b59a 100644 --- a/src/optimizer/materialize-mock-repo.ts +++ b/src/optimizer/materialize-mock-repo.ts @@ -9,12 +9,12 @@ import { listMockRepoTemplates, materializeMockRepo } from './mock-repos.js'; function printUsage(): void { console.log(` Usage: - tsx src/optimizer/materialize-mock-repo.ts [destination-root] + tsx src/optimizer/materialize-mock-repo.ts [destination-root] Examples: - tsx src/optimizer/materialize-mock-repo.ts sdk-demo - tsx src/optimizer/materialize-mock-repo.ts cli-demo ./.tmp/mock-repos tsx src/optimizer/materialize-mock-repo.ts mcp-tracker-demo + tsx src/optimizer/materialize-mock-repo.ts sdk-counter-demo ./.tmp/mock-repos + tsx src/optimizer/materialize-mock-repo.ts cli-taskfile-demo `); } diff --git a/src/optimizer/mock-repos.ts b/src/optimizer/mock-repos.ts index 7f2bd42..89993d1 100644 --- a/src/optimizer/mock-repos.ts +++ b/src/optimizer/mock-repos.ts @@ -6,7 +6,7 @@ import { promisify } from 'node:util'; const execFileAsync = promisify(execFile); -const MOCK_REPO_NAMES = ['sdk-demo', 'cli-demo', 'mcp-demo', 'mcp-tracker-demo', 'sdk-counter-demo', 'cli-taskfile-demo'] as const; +const MOCK_REPO_NAMES = ['mcp-tracker-demo', 'sdk-counter-demo', 'cli-taskfile-demo'] as const; export type MockRepoName = (typeof MOCK_REPO_NAMES)[number]; diff --git a/src/optimizer/mutation/pi-coding.ts b/src/optimizer/mutation/pi-coding.ts index d067dfd..bae3d08 100644 --- a/src/optimizer/mutation/pi-coding.ts +++ b/src/optimizer/mutation/pi-coding.ts @@ -1,6 +1,7 @@ import { readFileSync, statSync } from 'node:fs'; import { createHash } from 'node:crypto'; import { dirname, basename } from 'node:path'; +import type { AgentMessage } from '@mariozechner/pi-agent-core'; import type { MutationCandidate, MutationContext } from '../types.js'; import { collectGitChangedFiles } from './git-changes.js'; import { buildMutationContext } from '../feedback/mutation-context.js'; @@ -35,7 +36,7 @@ export class PiCodingMutationExecutor { await session.prompt(buildMutationPrompt(context)); - const messages = session.state.messages as unknown[]; + const messages = session.state.messages; const toolActivity = extractToolActivity(messages); const assistantText = extractLatestAssistantText(messages); @@ -62,8 +63,7 @@ export class PiCodingMutationExecutor { console.warn('[mutation] WARNING: skill file content is identical before and after mutation'); } - // If we wrote to a local skill file, return it directly — no git detection needed. - // Otherwise fall back to git status for target-repo mutations. + // Local skill file: return the path directly; git detection only applies to target-repo mutations. const changedFiles = context.localSkillPath ? [context.localSkillPath] : await collectGitChangedFiles(context.manifest.targetRepo.path); @@ -147,9 +147,7 @@ function buildMutationPrompt(context: MutationContext): string { ].join('\n'); } -function extractLatestAssistantText(messages: unknown): string | null { - if (!Array.isArray(messages)) return null; - +function extractLatestAssistantText(messages: AgentMessage[]): string | null { for (let index = messages.length - 1; index >= 0; index--) { const message = messages[index] as { role?: string; @@ -175,9 +173,7 @@ function extractLatestAssistantText(messages: unknown): string | null { return null; } -function extractToolActivity(messages: unknown): string[] { - if (!Array.isArray(messages)) return []; - +function extractToolActivity(messages: AgentMessage[]): string[] { const lines: string[] = []; for (const message of messages as Array<{ role?: string; diff --git a/src/optimizer/mutation/types.ts b/src/optimizer/mutation/types.ts deleted file mode 100644 index b01eb8b..0000000 --- a/src/optimizer/mutation/types.ts +++ /dev/null @@ -1 +0,0 @@ -export type { MutationCandidate, MutationContext } from '../types.js'; diff --git a/src/optimizer/types.ts b/src/optimizer/types.ts index 09222d7..77d2c8e 100644 --- a/src/optimizer/types.ts +++ b/src/optimizer/types.ts @@ -25,7 +25,7 @@ export interface OptimizePolicy { mode?: 'stable-surface' | 'surface-changing'; maxIterations?: number; stabilityWindow?: number; - minOverallPassDelta?: number; + minImprovement?: number; taskGeneration?: OptimizeTaskGenerationConfig; perModelFloor?: number; targetWeightedAverage?: number; @@ -65,7 +65,7 @@ export interface ResolvedOptimizeManifest { mode: 'stable-surface' | 'surface-changing'; maxIterations: number; stabilityWindow: number; - minOverallPassDelta: number; + minImprovement: number; taskGeneration: { enabled: boolean; maxGenerated: number; diff --git a/src/project/adapters.ts b/src/project/adapters.ts index 8151991..f0f3187 100644 --- a/src/project/adapters.ts +++ b/src/project/adapters.ts @@ -1,5 +1,5 @@ import type { BenchmarkConfig, SkillConfig } from '../benchmark/types.js'; -import type { OptimizeManifest, ResolvedOptimizeManifest } from '../optimizer/types.js'; +import type { ResolvedOptimizeManifest } from '../optimizer/types.js'; import type { ResolvedProjectConfig } from './types.js'; import { parseModelRef } from './types.js'; import { buildMcpToolDefinitionsFromSnapshot, buildSurfaceSnapshot } from './snapshot.js'; @@ -65,7 +65,7 @@ export function toOptimizeManifest(project: ResolvedProjectConfig): ResolvedOpti mode: optimize.mode, maxIterations: optimize.maxIterations, stabilityWindow: optimize.stabilityWindow, - minOverallPassDelta: optimize.minImprovement, + minImprovement: optimize.minImprovement, taskGeneration: { enabled: project.benchmark.taskGeneration.enabled, maxGenerated: project.benchmark.taskGeneration.maxTasks, @@ -87,44 +87,6 @@ export function toOptimizeManifest(project: ResolvedProjectConfig): ResolvedOpti }; } -export function toLegacyOptimizeManifest(project: ResolvedProjectConfig): OptimizeManifest { - const optimize = project.optimize; - if (!optimize || !optimize.enabled) { - throw new Error(`Project ${project.configPath} does not have optimization enabled`); - } - - return { - benchmarkConfig: project.configPath, - targetRepo: { - path: project.target.repoPath, - surface: project.target.surface, - allowedPaths: optimize.allowedPaths, - surfacePaths: getProjectSurfacePaths(project), - validation: optimize.validation, - requireCleanGit: optimize.requireCleanGit, - }, - optimizer: { - mode: optimize.mode, - maxIterations: optimize.maxIterations, - stabilityWindow: optimize.stabilityWindow, - minOverallPassDelta: optimize.minImprovement, - taskGeneration: { - enabled: project.benchmark.taskGeneration.enabled, - maxGenerated: project.benchmark.taskGeneration.maxTasks, - seed: project.benchmark.taskGeneration.seed, - outputDir: project.benchmark.taskGeneration.outputDir, - }, - }, - mutation: { - ...parseModelRef(optimize.model), - authMode: optimize.authMode, - apiKeyEnv: optimize.apiKeyEnv, - thinkingLevel: optimize.thinkingLevel, - reportContextMaxBytes: optimize.reportContextMaxBytes, - }, - }; -} - function getProjectSurfacePaths(project: ResolvedProjectConfig): string[] { const paths = new Set(); diff --git a/src/project/fix.ts b/src/project/fix.ts index 4bd4155..83c4b22 100644 --- a/src/project/fix.ts +++ b/src/project/fix.ts @@ -34,10 +34,6 @@ export function applyFixes( } } - if (issue.code === 'deprecated-tasks-field') { - const benchmark = result.benchmark as Record | undefined; - if (benchmark) delete benchmark.tasks; - } } return result; diff --git a/src/project/index.ts b/src/project/index.ts index 550856b..ba0f3d7 100644 --- a/src/project/index.ts +++ b/src/project/index.ts @@ -1,8 +1,8 @@ -export { DEFAULT_PROJECT_CONFIG_NAME, LEGACY_PROJECT_CONFIG_NAME, loadProjectConfig } from './load.js'; +export { DEFAULT_PROJECT_CONFIG_NAME, loadProjectConfig } from './load.js'; export { resolveProjectConfig } from './resolve.js'; export { buildMcpToolDefinitionsFromSnapshot, buildSurfaceSnapshot, loadSurfaceSnapshotFile } from './snapshot.js'; export { checkConfig, validateProjectConfig, type Issue, type IssueSeverity } from './validate.js'; -export { toBenchmarkConfig, toLegacyOptimizeManifest, toOptimizeManifest } from './adapters.js'; +export { toBenchmarkConfig, toOptimizeManifest } from './adapters.js'; export type { ActionCatalog } from '../actions/types.js'; @@ -20,7 +20,6 @@ export type { ResolvedProjectConfig, SurfaceSnapshot, SurfaceSnapshotAction, - SurfaceSnapshotArg, ResolvedProjectOptimizeConfig, ResolvedProjectTargetConfig, ResolvedProjectTaskGenerationConfig, diff --git a/src/project/load.ts b/src/project/load.ts index 1847dfa..8234f1a 100644 --- a/src/project/load.ts +++ b/src/project/load.ts @@ -6,7 +6,6 @@ import { resolveProjectConfig } from './resolve.js'; import { validateProjectConfig } from './validate.js'; export const DEFAULT_PROJECT_CONFIG_NAME = 'skill-optimizer.json'; -export const LEGACY_PROJECT_CONFIG_NAME = 'skill-benchmark.json'; export async function loadProjectConfig(configPath?: string, opts?: { skipDirtyGitCheck?: boolean }): Promise { const resolvedPath = configPath @@ -14,15 +13,6 @@ export async function loadProjectConfig(configPath?: string, opts?: { skipDirtyG : resolve(process.cwd(), DEFAULT_PROJECT_CONFIG_NAME); if (!existsSync(resolvedPath)) { - if (!configPath) { - const legacyCandidate = resolve(process.cwd(), LEGACY_PROJECT_CONFIG_NAME); - if (existsSync(legacyCandidate)) { - throw new Error( - `Found legacy '${LEGACY_PROJECT_CONFIG_NAME}'. ` + - `Rename it to '${DEFAULT_PROJECT_CONFIG_NAME}' — see CHANGELOG.md for migration notes.`, - ); - } - } throw new Error( `Project config not found: ${resolvedPath}\n` + `Run 'skill-optimizer init' to create one, or specify --config .`, diff --git a/src/project/schema.ts b/src/project/schema.ts index 193ce4d..362a768 100644 --- a/src/project/schema.ts +++ b/src/project/schema.ts @@ -36,7 +36,7 @@ const ScopeConfigSchema = z.object({ }); const TargetConfigSchema = z.object({ - surface: z.enum(['sdk', 'cli', 'mcp']).describe('Type of callable surface'), + surface: z.enum(['sdk', 'cli', 'mcp', 'prompt']).describe('Type of callable surface'), repoPath: z.string().optional().describe('Path to the target repo (default ".")'), skill: z.union([ z.string(), diff --git a/src/project/snapshot.ts b/src/project/snapshot.ts index cab7d50..db21ff8 100644 --- a/src/project/snapshot.ts +++ b/src/project/snapshot.ts @@ -57,30 +57,25 @@ export function loadSurfaceSnapshotFile(snapshotPath: string): SurfaceSnapshot { let parsed: unknown; try { parsed = JSON.parse(raw) as unknown; - } catch (error: any) { - throw new Error(`Invalid surface snapshot file: ${snapshotPath} (invalid JSON: ${error.message})`); + } catch (error) { + throw new Error(`Invalid surface snapshot file: ${snapshotPath} (invalid JSON: ${error instanceof Error ? error.message : String(error)})`); } - if (isLegacySurfaceSnapshot(parsed)) { - return normalizeCliArgs(parsed); + if (!isActionSnapshotArtifactShape(parsed)) { + if ( + parsed + && typeof parsed === 'object' + && 'surface' in parsed + && 'actions' in parsed + ) { + throw new Error( + `Snapshot file format is not supported — delete .skill-optimizer/ and re-run the benchmark to regenerate.`, + ); + } + throw new Error(`Invalid surface snapshot file: ${snapshotPath}`); } - if (isActionSnapshotArtifactShape(parsed)) { - return normalizeCliArgs(toSurfaceSnapshot(loadActionSnapshotFile(snapshotPath).catalog)); - } - - throw new Error(`Invalid surface snapshot file: ${snapshotPath}`); -} - -function isLegacySurfaceSnapshot(value: unknown): value is SurfaceSnapshot { - return Boolean( - value - && typeof value === 'object' - && 'surface' in value - && 'actions' in value - && Array.isArray((value as { actions?: unknown }).actions) - && ['sdk', 'cli', 'mcp', 'prompt'].includes(String((value as { surface?: unknown }).surface)), - ); + return normalizeCliArgs(toSurfaceSnapshot(loadActionSnapshotFile(snapshotPath).catalog)); } function isActionSnapshotArtifactShape(value: unknown): value is { version: number; catalog: { surface: string; actions: unknown[] } } { diff --git a/src/project/types.ts b/src/project/types.ts index b0cde89..0266731 100644 --- a/src/project/types.ts +++ b/src/project/types.ts @@ -154,8 +154,6 @@ export interface ResolvedProjectConfig { optimize?: ResolvedProjectOptimizeConfig; } -export type SurfaceSnapshotArg = ActionArgSchema; - export type SurfaceSnapshotAction = Omit; export interface SurfaceSnapshot extends Omit { diff --git a/src/project/validate.ts b/src/project/validate.ts index e787ec9..f2fff86 100644 --- a/src/project/validate.ts +++ b/src/project/validate.ts @@ -185,7 +185,8 @@ export async function checkConfig( err('invalid-format', 'benchmark.format', '"benchmark.format" must be pi, openai, or anthropic'); } - if (!benchmark.taskGeneration?.enabled && !benchmark.tasks) { + + if (benchmark.taskGeneration?.enabled !== true && !benchmark.tasks) { err('missing-tasks', 'benchmark.tasks', '"benchmark.tasks" is required when task generation is disabled'); } @@ -374,7 +375,6 @@ export async function checkConfig( // Check: API key env var / Codex auth const authMode = benchmark.authMode ?? 'env'; - // Helper: push a missing-credential warning for a given provider function warnMissingApiKey(provider: string, effectiveAuthMode: typeof authMode, apiKeyEnv: string | undefined, fieldPrefix: 'benchmark' | 'optimize'): void { const defaultEnvName = apiKeyEnv ?? (provider === 'openai' ? 'OPENAI_API_KEY' @@ -428,9 +428,8 @@ export async function checkConfig( if (!optimizeApiKey) warnMissingApiKey(optimizeProvider, optimizeAuthMode, optimizeApiKeyEnv, 'optimize'); } - // Check: dirty git (injection-safe: fixed arg array, no shell) - // Skipped when called from within the optimizer benchmark loop — the loop manages - // git state itself via ensureReady (run once before the loop starts). + // Check: dirty git (uses a fixed arg array, not a shell string, to prevent injection) + // Skipped inside the optimizer loop — the loop manages git state itself via ensureReady. if (!opts?.skipDirtyGitCheck && optimize !== undefined && optimize.requireCleanGit !== false && target.repoPath) { const absRepo = isAbsolute(target.repoPath) ? target.repoPath : resolve(configDir, target.repoPath); if (existsSync(absRepo)) { @@ -459,27 +458,6 @@ export async function checkConfig( } } - // Check: deprecated benchmark.tasks field - if (benchmark.tasks !== undefined && benchmark.taskGeneration?.enabled === true) { - issues.push({ - code: 'deprecated-tasks-field', severity: 'warning', field: 'benchmark.tasks', - message: '"benchmark.tasks" is set but task generation is enabled — the tasks field is deprecated', - hint: `Remove "tasks" from benchmark — task generation replaces it`, - fixable: true, - }); - } - - // Check: legacy skill-benchmark.json alongside skill-optimizer.json - const legacyPath = resolve(dirname(_configPath), 'skill-benchmark.json'); - if (existsSync(legacyPath)) { - issues.push({ - code: 'legacy-config-name', severity: 'warning', field: '(config file)', - message: `Found legacy "skill-benchmark.json" alongside "skill-optimizer.json"`, - hint: `Delete skill-benchmark.json — it is no longer used`, - fixable: false, - }); - } - return issues; } diff --git a/src/runtime/pi/auth.ts b/src/runtime/pi/auth.ts index d8b1256..573d332 100644 --- a/src/runtime/pi/auth.ts +++ b/src/runtime/pi/auth.ts @@ -101,14 +101,6 @@ export function requireConfiguredApiKey(options: PiAuthOptions): string { throw new Error(`Missing API key env var: ${envName}`); } -export function requireApiKeyFromEnv(apiKeyEnv: string): string { - const apiKey = process.env[apiKeyEnv]; - if (!apiKey) { - throw new Error(`Missing API key env var: ${apiKeyEnv}`); - } - return apiKey; -} - function readCodexApiKey(provider: string): ResolvedApiCredential { if (provider !== 'openai') { return {}; diff --git a/src/runtime/pi/benchmark-agent.ts b/src/runtime/pi/benchmark-agent.ts deleted file mode 100644 index 634cc92..0000000 --- a/src/runtime/pi/benchmark-agent.ts +++ /dev/null @@ -1,14 +0,0 @@ -import { resolvePiModelByRef } from './models.js'; - -export async function createReadOnlyBenchmarkModel(params: { - modelRef: string; - authMode?: import('./auth.js').PiAuthMode; - apiKeyEnv?: string; - apiKeyOverride?: string; -}) { - return resolvePiModelByRef(params.modelRef, { - authMode: params.authMode, - apiKeyEnv: params.apiKeyEnv, - apiKeyOverride: params.apiKeyOverride, - }); -} diff --git a/src/runtime/pi/coding-orchestrator.ts b/src/runtime/pi/coding-orchestrator.ts index e21fc49..ae8f16d 100644 --- a/src/runtime/pi/coding-orchestrator.ts +++ b/src/runtime/pi/coding-orchestrator.ts @@ -6,11 +6,12 @@ import { import { parseModelRef } from '../../project/types.js'; import { resolvePiModel } from './models.js'; +import type { PiAuthMode } from './auth.js'; export async function createCodingOrchestratorSession(params: { cwd: string; modelRef: string; - authMode?: import('./auth.js').PiAuthMode; + authMode?: PiAuthMode; apiKeyEnv?: string; thinkingLevel?: 'off' | 'minimal' | 'low' | 'medium' | 'high' | 'xhigh'; }) { diff --git a/src/runtime/pi/index.ts b/src/runtime/pi/index.ts index c4c8a02..b85353c 100644 --- a/src/runtime/pi/index.ts +++ b/src/runtime/pi/index.ts @@ -1,5 +1,4 @@ -export { createPiAuthStorage, requireApiKeyFromEnv, requireConfiguredApiKey, resolveApiCredential, resolveApiKey } from './auth.js'; -export { createReadOnlyBenchmarkModel } from './benchmark-agent.js'; +export { createPiAuthStorage, requireConfiguredApiKey, resolveApiCredential, resolveApiKey } from './auth.js'; export { createCodingOrchestratorSession } from './coding-orchestrator.js'; export { resolvePiModel, resolvePiModelByRef } from './models.js'; diff --git a/src/runtime/pi/models.ts b/src/runtime/pi/models.ts index faf940f..d4c378b 100644 --- a/src/runtime/pi/models.ts +++ b/src/runtime/pi/models.ts @@ -1,12 +1,13 @@ -import type { Model } from '@mariozechner/pi-ai'; +import type { Api, Model } from '@mariozechner/pi-ai'; import { getModel } from '@mariozechner/pi-ai'; import { ModelRegistry } from '@mariozechner/pi-coding-agent'; import { createPiAuthStorage, resolveApiCredential } from './auth.js'; +import type { PiAuthMode } from './auth.js'; import { parseModelRef } from '../../project/types.js'; export interface ResolvedPiModelRequest { - model: Model; + model: Model; authStorage: ReturnType; modelRegistry: ReturnType; auth: { @@ -17,7 +18,7 @@ export interface ResolvedPiModelRequest { export async function resolvePiModelByRef( modelRef: string, - options?: { authMode?: import('./auth.js').PiAuthMode; apiKeyEnv?: string; apiKeyOverride?: string }, + options?: { authMode?: PiAuthMode; apiKeyEnv?: string; apiKeyOverride?: string }, ): Promise { const { provider, model } = parseModelRef(modelRef); return resolvePiModel(provider, model, options); @@ -29,7 +30,7 @@ export async function resolvePiModelByRef( * it can use the "openai-completions" api type with openrouter.ai/api/v1 as the base URL. * The model ID passed to OpenRouter is the portion after "openrouter/" in the full ref. */ -function synthesizeOpenRouterModel(provider: string, modelName: string): Model | undefined { +function synthesizeOpenRouterModel(provider: string, modelName: string): Model<'openai-completions'> | undefined { if (provider !== 'openrouter') return undefined; return { id: modelName, @@ -45,7 +46,7 @@ function synthesizeOpenRouterModel(provider: string, modelName: string): Model | undefined { +function synthesizeOpenAICodexModel(provider: string, modelName: string): Model<'openai-codex-responses'> | undefined { if (provider !== 'openai-codex') return undefined; return { id: modelName, @@ -64,7 +65,7 @@ function synthesizeOpenAICodexModel(provider: string, modelName: string): Model< export async function resolvePiModel( provider: string, modelName: string, - options?: { authMode?: import('./auth.js').PiAuthMode; apiKeyEnv?: string; apiKeyOverride?: string }, + options?: { authMode?: PiAuthMode; apiKeyEnv?: string; apiKeyOverride?: string }, ): Promise { // Guard: direct-provider model + OpenRouter key = guaranteed 401. // Catches stale scaffolded configs that had apiKeyEnv:"OPENROUTER_API_KEY" and were diff --git a/src/tasks/coverage.ts b/src/tasks/coverage.ts index 7c35b29..503b4c9 100644 --- a/src/tasks/coverage.ts +++ b/src/tasks/coverage.ts @@ -3,8 +3,7 @@ import type { GeneratedTask } from './types.js'; import type { CoverageReport } from '../benchmark/types.js'; function actionNamesOf(task: GeneratedTask): string[] { - const list = task.expected_actions ?? task.expected_tools ?? []; - return list.map((a) => a.name ?? a.method ?? '').filter(Boolean); + return task.expected_actions.map((a) => a.name).filter(Boolean); } export function computeCoverage( diff --git a/src/tasks/default-pi-critic.ts b/src/tasks/default-pi-critic.ts index 1348b3a..85675f7 100644 --- a/src/tasks/default-pi-critic.ts +++ b/src/tasks/default-pi-critic.ts @@ -1,12 +1,11 @@ -import { completeSimple } from '@mariozechner/pi-ai'; - -import { resolvePiModel } from '../runtime/pi/index.js'; import type { CriticDeps } from '../verdict/recommendations.js'; +import { piSimpleComplete } from './pi-simple-complete.js'; +import type { PiAuthMode } from '../runtime/pi/auth.js'; export interface DefaultPiCriticOptions { provider: string; model: string; - authMode?: import('../runtime/pi/auth.js').PiAuthMode; + authMode?: PiAuthMode; apiKeyEnv?: string; timeoutMs?: number; headers?: Record; @@ -15,41 +14,28 @@ export interface DefaultPiCriticOptions { export function createDefaultPiCritic(options: DefaultPiCriticOptions): CriticDeps { return { async complete(input) { - const resolved = await resolvePiModel(options.provider, options.model, { - authMode: options.authMode, - apiKeyEnv: options.apiKeyEnv, - }); - - const controller = new AbortController(); - const timeoutMs = options.timeoutMs ?? 120_000; - const timer = setTimeout(() => controller.abort(), timeoutMs); - timer.unref?.(); - - const response = await completeSimple( - resolved.model, - { - systemPrompt: input.system, - messages: [{ role: 'user', content: input.prompt, timestamp: Date.now() }], - }, - { - signal: controller.signal, - apiKey: resolved.auth.apiKey, - headers: { ...(resolved.auth.headers ?? {}), ...(options.headers ?? {}) }, - reasoning: 'minimal', - }, - ).finally(() => clearTimeout(timer)); - - if (response.stopReason === 'error' && response.errorMessage) { - throw new Error(response.errorMessage); + try { + return await piSimpleComplete( + { + provider: options.provider, + model: options.model, + authMode: options.authMode, + apiKeyEnv: options.apiKeyEnv, + timeoutMs: options.timeoutMs, + headers: options.headers, + reasoning: 'minimal', + }, + { system: input.system, prompt: input.prompt }, + ); + } catch (err) { + // A model that returns no text blocks is treated as "no recommendations" + // rather than a hard failure — the verdict flow continues with an empty list. + // Real provider errors (stopReason === 'error') are re-thrown. + if (err instanceof Error && err.message.startsWith('Model returned no text blocks')) { + return '[]'; + } + throw err; } - - const text = response.content - .filter((block): block is Extract => block.type === 'text') - .map((block) => block.text) - .join('\n') - .trim(); - - return text || '[]'; }, }; } diff --git a/src/tasks/default-pi-generator.ts b/src/tasks/default-pi-generator.ts index ce9b02e..a17bef5 100644 --- a/src/tasks/default-pi-generator.ts +++ b/src/tasks/default-pi-generator.ts @@ -1,16 +1,15 @@ -import { completeSimple } from '@mariozechner/pi-ai'; import type { SimpleStreamOptions } from '@mariozechner/pi-ai'; -import { resolvePiModel } from '../runtime/pi/index.js'; - import type { TaskGeneratorDeps } from './types.js'; +import { piSimpleComplete } from './pi-simple-complete.js'; +import type { PiAuthMode } from '../runtime/pi/auth.js'; type ThinkingLevel = NonNullable; export interface DefaultPiGeneratorOptions { provider: string; model: string; - authMode?: import('../runtime/pi/auth.js').PiAuthMode; + authMode?: PiAuthMode; apiKeyEnv?: string; timeoutMs?: number; headers?: Record; @@ -20,49 +19,18 @@ export interface DefaultPiGeneratorOptions { export function createDefaultPiTaskGenerator(options: DefaultPiGeneratorOptions): TaskGeneratorDeps { return { async complete(input) { - const resolved = await resolvePiModel(options.provider, options.model, { - authMode: options.authMode, - apiKeyEnv: options.apiKeyEnv, - }); - - const controller = new AbortController(); - const timeoutMs = options.timeoutMs ?? 120_000; - const timer = setTimeout(() => controller.abort(), timeoutMs); - timer.unref?.(); - - const response = await completeSimple( - resolved.model, - { - systemPrompt: input.system, - messages: [{ role: 'user', content: input.prompt, timestamp: Date.now() }], - }, + return await piSimpleComplete( { - signal: controller.signal, - apiKey: resolved.auth.apiKey, - headers: { ...(resolved.auth.headers ?? {}), ...(options.headers ?? {}) }, + provider: options.provider, + model: options.model, + authMode: options.authMode, + apiKeyEnv: options.apiKeyEnv, + timeoutMs: options.timeoutMs, + headers: options.headers, reasoning: options.thinkingLevel ?? 'minimal', }, - ).finally(() => clearTimeout(timer)); - - if (response.stopReason === 'error' && response.errorMessage) { - throw new Error(response.errorMessage); - } - - const text = response.content - .filter((block): block is Extract => block.type === 'text') - .map((block) => block.text) - .join('\n') - .trim(); - - if (!text) { - const contentTypes = response.content.map((block) => block.type).join(', '); - throw new Error( - response.errorMessage - ?? `Generation model returned no text blocks${contentTypes ? ` (content types: ${contentTypes})` : ''}`, - ); - } - - return text; + { system: input.system, prompt: input.prompt }, + ); }, }; } diff --git a/src/tasks/freeze.ts b/src/tasks/freeze.ts index dafd498..94a32b3 100644 --- a/src/tasks/freeze.ts +++ b/src/tasks/freeze.ts @@ -1,6 +1,7 @@ import { mkdirSync, writeFileSync } from 'node:fs'; import { join, resolve } from 'node:path'; +import { fromSurfaceSnapshot, writeActionSnapshotFile } from '../actions/snapshot.js'; import type { ResolvedProjectConfig, SurfaceSnapshot } from '../project/types.js'; import type { FrozenTaskArtifacts, GeneratedTask } from './types.js'; @@ -27,7 +28,7 @@ export function freezeTaskArtifacts(params: FreezeTaskArtifactsParams): FrozenTa const snapshotPath = join(outputDir, 'surface.snapshot.json'); writeFileSync(tasksPath, JSON.stringify({ tasks: params.kept }, null, 2), 'utf-8'); - writeFileSync(snapshotPath, JSON.stringify(params.snapshot, null, 2), 'utf-8'); + writeActionSnapshotFile(snapshotPath, fromSurfaceSnapshot(params.snapshot)); const generatedProject = { name: params.project.name, diff --git a/src/tasks/generate.ts b/src/tasks/generate.ts index 2536df8..52cda9d 100644 --- a/src/tasks/generate.ts +++ b/src/tasks/generate.ts @@ -1,5 +1,4 @@ import type { ExpectedAction, CoverageReport } from '../benchmark/types.js'; -import { getExpectedActionName } from '../benchmark/types.js'; import type { ActionDefinition } from '../actions/types.js'; import { computeUncovered, buildRetryPrompt, computeCoverage } from './coverage.js'; @@ -186,7 +185,7 @@ function validateTask(task: unknown, index: number): GeneratedTask { } let rawExpectedActions = ( - ['expected_actions', 'expected_tools', 'actions', 'steps', 'calls', 'expected_calls', 'tool_calls', 'cli_command'] as const + ['expected_actions', 'actions', 'steps', 'calls', 'expected_calls', 'tool_calls', 'cli_command'] as const ) .map((key) => candidate[key]) .find((v) => Array.isArray(v)) as unknown[] | undefined; @@ -195,7 +194,6 @@ function validateTask(task: unknown, index: number): GeneratedTask { if (!rawExpectedActions) { const actionName = typeof candidate['action'] === 'string' ? candidate['action'] : - typeof candidate['method'] === 'string' ? candidate['method'] : typeof candidate['command'] === 'string' ? candidate['command'] : null; if (actionName && actionName.trim()) { rawExpectedActions = [{ name: actionName.trim(), args: candidate['args'] }]; @@ -213,7 +211,6 @@ function validateTask(task: unknown, index: number): GeneratedTask { id: taskId, prompt: taskPrompt, expected_actions, - expected_tools: expected_actions, }; } @@ -222,8 +219,8 @@ function validateExpectedAction(taskId: string, action: unknown, actionIndex: nu throw new Error(`Task ${taskId} expected_actions[${actionIndex}] must be an object`); } - const typed = action as { name?: unknown; method?: unknown; args?: unknown }; - const name = typeof typed.name === 'string' ? typed.name : typeof typed.method === 'string' ? typed.method : null; + const typed = action as { name?: unknown; args?: unknown }; + const name = typeof typed.name === 'string' ? typed.name : null; if (!name || name.trim() === '') { throw new Error(`Task ${taskId} expected_actions[${actionIndex}] must include a non-empty name`); } @@ -232,14 +229,10 @@ function validateExpectedAction(taskId: string, action: unknown, actionIndex: nu throw new Error(`Task ${taskId} expected_actions[${actionIndex}] args must be an object when present`); } - const normalized: ExpectedAction = { + return { name, - method: name, args: typed.args as Record | undefined, }; - // Touch helper so transitional action alias stays normalized consistently. - getExpectedActionName(normalized); - return normalized; } export async function generateCandidateTasksWithCoverage( diff --git a/src/tasks/ground.ts b/src/tasks/ground.ts index ba255a3..d1e3055 100644 --- a/src/tasks/ground.ts +++ b/src/tasks/ground.ts @@ -30,7 +30,7 @@ function getRejectionReason( actions: Map, surface: SurfaceSnapshot['surface'], ): string | null { - const expectedActions = task.expected_actions ?? task.expected_tools ?? []; + const expectedActions = task.expected_actions; if (seenIds.has(task.id)) { return `duplicate task id "${task.id}"`; } @@ -43,7 +43,7 @@ function getRejectionReason( return null; } - if (!Array.isArray(expectedActions) || expectedActions.length === 0) { + if (expectedActions.length === 0) { return `task "${task.id}" has empty expected_actions`; } diff --git a/src/tasks/index.ts b/src/tasks/index.ts index 193d849..c5b4208 100644 --- a/src/tasks/index.ts +++ b/src/tasks/index.ts @@ -37,7 +37,6 @@ export async function generateTasksForProject( const surface = await discoverTaskSurface(params.configPath); console.log(`[optimize] Loaded ${surface.snapshot.surface} surface with ${surface.snapshot.actions.length} actions.`); - // Apply scope filter const { inScope, outOfScope } = resolveScope(surface.snapshot.actions, surface.project.target.scope); if (inScope.length === 0) { console.warn( @@ -56,7 +55,6 @@ export async function generateTasksForProject( ); } - // Replace snapshot actions with in-scope only (for generation context) const filteredSurface = { ...surface, snapshot: { @@ -97,7 +95,7 @@ export async function generateTasksForProject( throw new Error('Task generation produced zero valid tasks after grounding'); } - // Recompute coverage from kept tasks only — pre-grounding coverage is stale if tasks were rejected. + // Recompute from kept tasks only — pre-grounding coverage may include rejected tasks. const taskCoverage = computeCoverage(inScopeActions, grounded.kept, outOfScopeActions); console.log(`[optimize] Grounded ${grounded.kept.length} tasks, rejected ${grounded.rejected.length}.`); console.log('[optimize] Benchmark tasks:'); diff --git a/src/tasks/pi-simple-complete.ts b/src/tasks/pi-simple-complete.ts new file mode 100644 index 0000000..44d9f67 --- /dev/null +++ b/src/tasks/pi-simple-complete.ts @@ -0,0 +1,81 @@ +/** + * Shared helper for tasks that call Pi's completeSimple API. + * + * Both the default task-generator and the default critic perform an identical + * sequence: resolve the model, set up an abort timer, call completeSimple, check + * for errors, and extract text blocks. This module encapsulates that sequence so + * each consumer only needs to handle what's unique to it. + */ + +import { completeSimple } from '@mariozechner/pi-ai'; +import type { SimpleStreamOptions } from '@mariozechner/pi-ai'; + +import { resolvePiModel } from '../runtime/pi/index.js'; +import type { PiAuthMode } from '../runtime/pi/auth.js'; + +interface PiSimpleCompleteOptions { + provider: string; + model: string; + authMode?: PiAuthMode; + apiKeyEnv?: string; + timeoutMs?: number; + headers?: Record; + reasoning?: NonNullable; +} + +interface PiSimpleCompleteInput { + system: string; + prompt: string; +} + +/** + * Resolve a Pi model, call completeSimple with a timeout, check for errors, + * and return the concatenated text from all text blocks. + * + * Throws if the model signals an error or if the response contains no text blocks. + */ +export async function piSimpleComplete( + options: PiSimpleCompleteOptions, + input: PiSimpleCompleteInput, +): Promise { + const resolved = await resolvePiModel(options.provider, options.model, { + authMode: options.authMode, + apiKeyEnv: options.apiKeyEnv, + }); + + const controller = new AbortController(); + const timeoutMs = options.timeoutMs ?? 120_000; + const timer = setTimeout(() => controller.abort(), timeoutMs); + timer.unref?.(); + + const response = await completeSimple( + resolved.model, + { + systemPrompt: input.system, + messages: [{ role: 'user', content: input.prompt, timestamp: Date.now() }], + }, + { + signal: controller.signal, + apiKey: resolved.auth.apiKey, + headers: { ...(resolved.auth.headers ?? {}), ...(options.headers ?? {}) }, + reasoning: options.reasoning ?? 'minimal', + }, + ).finally(() => clearTimeout(timer)); + + if (response.stopReason === 'error') { + throw new Error(response.errorMessage ?? 'Model returned stop reason "error" with no message'); + } + + const text = response.content + .filter((block): block is Extract => block.type === 'text') + .map((block) => block.text) + .join('\n') + .trim(); + + if (!text) { + const contentTypes = response.content.map((b) => b.type).join(', '); + throw new Error(`Model returned no text blocks${contentTypes ? ` (content types: ${contentTypes})` : ''}`); + } + + return text; +} diff --git a/src/tasks/types.ts b/src/tasks/types.ts index 7ff224f..43351b6 100644 --- a/src/tasks/types.ts +++ b/src/tasks/types.ts @@ -4,8 +4,7 @@ import type { ResolvedProjectConfig, SurfaceSnapshot } from '../project/types.js export interface GeneratedTask { id: string; prompt: string; - expected_actions?: ExpectedAction[]; - expected_tools?: ExpectedAction[]; + expected_actions: ExpectedAction[]; } export interface TaskGeneratorConfig { diff --git a/tests/smoke-actions.ts b/tests/smoke-actions.ts index a23bfcf..469bdb4 100644 --- a/tests/smoke-actions.ts +++ b/tests/smoke-actions.ts @@ -150,8 +150,8 @@ await test('diffActionCatalog ignores arg reordering and catches schema changes' assertEqual(changedDiff.changed.length, 1, 'required-flag changes should count as schema changes'); }); -await test('surface snapshot compatibility mapping preserves legacy shape', () => { - const legacy: SurfaceSnapshot = { +await test('fromSurfaceSnapshot maps SurfaceSnapshot action names to ActionCatalog keys', () => { + const snapshot: SurfaceSnapshot = { surface: 'cli', actions: [ { @@ -164,16 +164,16 @@ await test('surface snapshot compatibility mapping preserves legacy shape', () = ], }; - const catalog = fromSurfaceSnapshot(legacy); - assertEqual(catalog.actions[0].key, 'wallet create', 'legacy action name should map to canonical key by default'); + const catalog = fromSurfaceSnapshot(snapshot); + assertEqual(catalog.actions[0].key, 'wallet create', 'action name should map to canonical key'); const roundtrip = toSurfaceSnapshot(catalog); - assertEqual(roundtrip.actions[0].name, 'wallet create', 'legacy action name should roundtrip'); - assert(!('key' in roundtrip.actions[0]), 'legacy surface snapshot actions should not include key field'); + assertEqual(roundtrip.actions[0].name, 'wallet create', 'action name should roundtrip via toSurfaceSnapshot'); + assert(!('key' in roundtrip.actions[0]), 'SurfaceSnapshot actions should not include key field'); }); -await test('fromSurfaceSnapshot trims keys so legacy conversions diff stably', () => { - const legacy: SurfaceSnapshot = { +await test('fromSurfaceSnapshot trims whitespace from derived action keys', () => { + const snapshot: SurfaceSnapshot = { surface: 'mcp', actions: [ { @@ -186,8 +186,8 @@ await test('fromSurfaceSnapshot trims keys so legacy conversions diff stably', ( ], }; - const converted = fromSurfaceSnapshot(legacy); - assertEqual(converted.actions[0].key, 'wallet.send', 'legacy conversion should trim derived canonical keys'); + const converted = fromSurfaceSnapshot(snapshot); + assertEqual(converted.actions[0].key, 'wallet.send', 'action name whitespace should be trimmed to derive canonical key'); const normalized = { surface: 'mcp' as const, @@ -364,7 +364,7 @@ await test('loadActionSnapshotFile includes path on invalid JSON', () => { } }); -await test('buildSurfaceSnapshot returns expected legacy snapshot fields from discovery fixture', async () => { +await test('buildSurfaceSnapshot returns surface snapshot from code-first discovery fixture', async () => { const root = mkdtempSync(join(tmpdir(), 'skill-optimizer-snapshot-bridge-')); try { const sourcePath = join(root, 'server.ts'); @@ -418,7 +418,7 @@ await test('buildSurfaceSnapshot returns expected legacy snapshot fields from di assertEqual(actual.actions[0].args.length, 1, 'snapshot should include tool args'); assertEqual(actual.actions[0].args[0].name, 'label', 'arg name should match discovered schema'); assertEqual(actual.actions[0].args[0].required, true, 'required arg flag should be preserved'); - assert(!('key' in actual.actions[0]), 'legacy snapshot action should not expose canonical key field'); + assert(!('key' in actual.actions[0]), 'SurfaceSnapshot action should not expose the key field'); } finally { rmSync(root, { recursive: true, force: true }); } @@ -461,29 +461,6 @@ await test('buildMcpToolDefinitionsFromSnapshot preserves nested arg schemas', ( assertEqual(properties.peers.items.type, 'string', 'optional array items should be preserved in rebuilt tool schema'); }); -await test('loadSurfaceSnapshotFile supports legacy plain snapshot shape', () => { - const root = mkdtempSync(join(tmpdir(), 'skill-optimizer-snapshot-legacy-')); - try { - const snapshotPath = join(root, 'surface.snapshot.json'); - writeFileSync(snapshotPath, JSON.stringify({ - surface: 'cli', - actions: [ - { - name: 'wallet create', - args: [{ name: '--label', required: true, type: 'string' }], - }, - ], - }, null, 2), 'utf-8'); - - const loaded = loadSurfaceSnapshotFile(snapshotPath); - assertEqual(loaded.surface, 'cli', 'legacy file should load as cli snapshot'); - assertEqual(loaded.actions.length, 1, 'legacy file should include actions'); - assertEqual(loaded.actions[0].args[0].name, 'label', 'legacy cli arg names should be normalized'); - } finally { - rmSync(root, { recursive: true, force: true }); - } -}); - await test('loadSurfaceSnapshotFile supports versioned action snapshot artifact', () => { const root = mkdtempSync(join(tmpdir(), 'skill-optimizer-snapshot-versioned-')); try { @@ -503,14 +480,14 @@ await test('loadSurfaceSnapshotFile supports versioned action snapshot artifact' assertEqual(loaded.surface, 'cli', 'versioned file should map to cli snapshot'); assertEqual(loaded.actions[0].name, 'wallet create', 'action name should map from action catalog'); assertEqual(loaded.actions[0].args[0].name, 'label', 'converted cli arg names should be normalized'); - assert(!('key' in loaded.actions[0]), 'converted legacy snapshot should not expose key field'); + assert(!('key' in loaded.actions[0]), 'converted SurfaceSnapshot action should not expose key field'); } finally { rmSync(root, { recursive: true, force: true }); } }); -await test('loadSurfaceSnapshotFile prefers valid legacy snapshot shape even with version/catalog keys present', () => { - const root = mkdtempSync(join(tmpdir(), 'skill-optimizer-snapshot-legacy-extra-')); +await test('loadSurfaceSnapshotFile throws a clear error for unsupported plain snapshot format', () => { + const root = mkdtempSync(join(tmpdir(), 'skill-optimizer-snapshot-unsupported-format-')); try { const snapshotPath = join(root, 'surface.snapshot.json'); writeFileSync(snapshotPath, JSON.stringify({ @@ -521,16 +498,18 @@ await test('loadSurfaceSnapshotFile prefers valid legacy snapshot shape even wit args: [{ name: '--label', required: true, type: 'string' }], }, ], - version: ACTION_SNAPSHOT_VERSION, - catalog: { - this: 'should be ignored for legacy shape', - }, }, null, 2), 'utf-8'); - const loaded = loadSurfaceSnapshotFile(snapshotPath); - assertEqual(loaded.surface, 'cli', 'valid legacy snapshot should load as legacy shape'); - assertEqual(loaded.actions.length, 1, 'legacy action list should be preserved'); - assertEqual(loaded.actions[0].args[0].name, 'label', 'legacy CLI arg normalization should still apply'); + let threw = false; + try { + loadSurfaceSnapshotFile(snapshotPath); + } catch (error: any) { + threw = true; + assert(error.message.includes('not supported'), 'error should describe the format as unsupported'); + assert(error.message.includes('.skill-optimizer/'), 'error should direct user to delete .skill-optimizer/'); + } + + assert(threw, 'unsupported plain snapshot format should throw'); } finally { rmSync(root, { recursive: true, force: true }); } diff --git a/tests/smoke-code.ts b/tests/smoke-code.ts index bf6a3f8..eb7ddf2 100644 --- a/tests/smoke-code.ts +++ b/tests/smoke-code.ts @@ -4,7 +4,7 @@ import { tmpdir } from 'node:os'; import { extractCodeBlock } from '../src/benchmark/extractors/code-extractor.js'; import { extractSdkCodeBlock } from '../src/benchmark/extractors/code-extractor.js'; -import { extractFromCode, extractAllFromCode } from '../src/benchmark/extractors/code-analyzer.js'; +import { extractAllFromCode } from '../src/benchmark/extractors/code-analyzer.js'; import { extract } from '../src/benchmark/extractors/index.js'; import { evaluateTask } from '../src/benchmark/evaluator.js'; import { computeCoverage } from '../src/benchmark/coverage.js'; @@ -54,7 +54,7 @@ function makeTask(id: string, methods: string[]): TaskDefinition { return { id, prompt: `Task ${id}`, - expected_tools: methods.map((method) => ({ method })), + expected_actions: methods.map((name) => ({ name })), }; } @@ -142,95 +142,99 @@ await test('loadConfig: rejects unsupported sdk language', async () => { } }); -// Group 2: extractFromCode (tree-sitter) +// Group 2: extractAllFromCode (tree-sitter, no config hints needed) -await test('extractFromCode: constructor call', async () => { +await test('extractAllFromCode: constructor call', async () => { const code = 'const provider = new FastProvider("testnet");'; - const calls = await extractFromCode(code, ['FastProvider', 'FastWallet']); + const { calls } = await extractAllFromCode(code); assertEqual(calls.length, 1, 'should find 1 call'); assertEqual(calls[0].method, 'FastProvider.constructor', 'method should be FastProvider.constructor'); assertEqual(calls[0].args['_positional_0'] as string, 'testnet', 'first positional arg should be "testnet"'); }); -await test('extractFromCode: variable tracking', async () => { +await test('extractAllFromCode: variable tracking (raw calls)', async () => { const code = [ 'const provider = new FastProvider("testnet");', 'const wallet = await FastWallet.fromKeyfile(provider);', 'const balance = await wallet.balance();', ].join('\n'); - const calls = await extractFromCode(code, ['FastProvider', 'FastWallet']); + const { calls } = await extractAllFromCode(code); assertEqual(calls.length, 3, 'should find 3 calls'); - // The 3rd call should resolve wallet → FastWallet via variable tracking - assertEqual(calls[2].method, 'FastWallet.balance', 'third call method should be FastWallet.balance'); + // Raw extraction: wallet.balance is unresolved (resolution happens in evaluateTask with bindings) + assertEqual(calls[2].method, 'wallet.balance', 'third raw call method should be wallet.balance'); }); -await test('extractFromCode: static method', async () => { +await test('extractAllFromCode: static method', async () => { const code = 'const wallet = await FastWallet.fromKeyfile(provider, "merchant");'; - const calls = await extractFromCode(code, ['FastProvider', 'FastWallet']); + const { calls } = await extractAllFromCode(code); assertEqual(calls.length, 1, 'should find 1 call'); assertEqual(calls[0].method, 'FastWallet.fromKeyfile', 'method should be FastWallet.fromKeyfile'); assertEqual(calls[0].args['_positional_1'] as string, 'merchant', 'second positional arg should be "merchant"'); }); -await test('extractFromCode: object arguments', async () => { +await test('extractAllFromCode: object arguments', async () => { const code = [ 'const provider = new FastProvider("testnet");', 'const wallet = await FastWallet.fromKeyfile(provider);', 'await wallet.send({ to: "fast1abc", amount: "5", token: "FAST" });', ].join('\n'); - const calls = await extractFromCode(code, ['FastProvider', 'FastWallet']); - // Find the send call - const sendCall = calls.find((c) => c.method === 'FastWallet.send'); - assert(sendCall !== undefined, 'should find a FastWallet.send call'); + const { calls } = await extractAllFromCode(code); + // Raw extraction: wallet.send is unresolved + const sendCall = calls.find((c) => c.method === 'wallet.send'); + assert(sendCall !== undefined, 'should find a wallet.send call (raw, unresolved)'); assertEqual(sendCall!.args['to'] as string, 'fast1abc', 'to arg should be "fast1abc"'); assertEqual(sendCall!.args['amount'] as string, '5', 'amount arg should be "5"'); assertEqual(sendCall!.args['token'] as string, 'FAST', 'token arg should be "FAST"'); }); -await test('extractFromCode: empty code returns empty array', async () => { - const calls = await extractFromCode('', ['FastProvider', 'FastWallet']); +await test('extractAllFromCode: empty code returns empty arrays', async () => { + const { calls, bindings } = await extractAllFromCode(''); assertEqual(calls.length, 0, 'should return empty array for empty code'); + assertEqual(bindings.size, 0, 'should return empty bindings for empty code'); }); -await test('extractFromCode: standalone function call', async () => { +await test('extractAllFromCode: standalone function call', async () => { const code = `const result = await x402Pay({ url: 'https://api.example.com', wallet: { type: 'evm' } });`; - const calls = await extractFromCode(code, [], ['x402Pay']); + const { calls } = await extractAllFromCode(code); assertEqual(calls.length, 1, 'should find 1 call'); assertEqual(calls[0].method, 'x402Pay', 'method should be x402Pay'); assertEqual(calls[0].args['url'] as string, 'https://api.example.com', 'url arg'); }); -await test('extractFromCode: function return tracking', async () => { +await test('extractAllFromCode: bindings capture factory returns', async () => { const code = [ `const f = fast({ network: 'testnet' });`, `await f.setup();`, `const balance = await f.balance({ token: 'FAST' });`, ].join('\n'); - const calls = await extractFromCode(code, [], ['fast'], { fast: 'FastClient' }); + const { calls, bindings } = await extractAllFromCode(code); assertEqual(calls.length, 3, 'should find 3 calls'); assertEqual(calls[0].method, 'fast', 'first call should be fast'); assertEqual(calls[0].args['network'] as string, 'testnet', 'network arg'); - assertEqual(calls[1].method, 'FastClient.setup', 'second call should be FastClient.setup'); - assertEqual(calls[2].method, 'FastClient.balance', 'third call should be FastClient.balance'); + // Raw calls use variable names; bindings map f → fast + assertEqual(calls[1].method, 'f.setup', 'second raw call should be f.setup'); + assertEqual(calls[2].method, 'f.balance', 'third raw call should be f.balance'); + assertEqual(bindings.get('f'), 'fast', 'bindings should map f → fast'); assertEqual(calls[2].args['token'] as string, 'FAST', 'token arg'); }); -await test('extractFromCode: mixed classes and functions', async () => { +await test('extractAllFromCode: mixed classes and functions', async () => { const code = [ `const account = createEvmWallet('~/.evm/keys/default.json');`, `const allset = new AllSetProvider({ network: 'testnet' });`, `await allset.sendToFast({ chain: 'arbitrum', token: 'USDC', amount: '1000000' });`, ].join('\n'); - const calls = await extractFromCode(code, ['AllSetProvider'], ['createEvmWallet']); + const { calls } = await extractAllFromCode(code); assertEqual(calls.length, 3, 'should find 3 calls'); assertEqual(calls[0].method, 'createEvmWallet', 'first call should be createEvmWallet'); assertEqual(calls[0].args['_positional_0'] as string, '~/.evm/keys/default.json', 'keyfile path arg'); assertEqual(calls[1].method, 'AllSetProvider.constructor', 'second call should be AllSetProvider.constructor'); - assertEqual(calls[2].method, 'AllSetProvider.sendToFast', 'third call should be AllSetProvider.sendToFast'); + // Raw: allset.sendToFast (unresolved) + assertEqual(calls[2].method, 'allset.sendToFast', 'third raw call should be allset.sendToFast'); assertEqual(calls[2].args['chain'] as string, 'arbitrum', 'chain arg'); }); -await test('extractFromCode: standalone function with no classes', async () => { +await test('extractAllFromCode: standalone function with no classes', async () => { const code = [ `const result = await x402Pay({`, ` url: 'https://api.example.com/premium',`, @@ -238,27 +242,27 @@ await test('extractFromCode: standalone function with no classes', async () => { ` verbose: true,`, `});`, ].join('\n'); - const calls = await extractFromCode(code, [], ['x402Pay']); + const { calls } = await extractAllFromCode(code); assertEqual(calls.length, 1, 'should find 1 call'); assertEqual(calls[0].method, 'x402Pay', 'method should be x402Pay'); assertEqual(calls[0].args['url'] as string, 'https://api.example.com/premium', 'url arg'); assertEqual(calls[0].args['verbose'] as boolean, true, 'verbose arg'); }); -await test('extractFromCode: nested object arguments', async () => { +await test('extractAllFromCode: nested object arguments', async () => { const code = [ `const result = await x402Pay({`, ` url: 'https://api.example.com/premium',`, ` wallet: { type: 'evm', privateKey: '0x123', address: '0xabc' },`, `});`, ].join('\n'); - const calls = await extractFromCode(code, [], ['x402Pay']); + const { calls } = await extractAllFromCode(code); assertEqual(calls.length, 1, 'should find 1 call'); assertEqual((calls[0].args['wallet'] as Record).type as string, 'evm', 'wallet.type arg'); assertEqual((calls[0].args['wallet'] as Record).address as string, '0xabc', 'wallet.address arg'); }); -await test('extractFromCode: resolves identifier-backed nested arguments', async () => { +await test('extractAllFromCode: resolves identifier-backed nested arguments', async () => { const code = [ `const fastWallet = { type: 'fast', address: 'fast1abc', publicKey: 'pub', privateKey: 'priv' };`, `const evmWallet = { type: 'evm', address: '0xabc', privateKey: '0x123' };`, @@ -267,7 +271,7 @@ await test('extractFromCode: resolves identifier-backed nested arguments', async ` wallet: [fastWallet, evmWallet],`, `});`, ].join('\n'); - const calls = await extractFromCode(code, [], ['x402Pay']); + const { calls } = await extractAllFromCode(code); assertEqual(calls.length, 1, 'should find 1 call'); const wallet = calls[0].args['wallet'] as unknown[]; assertEqual((wallet[0] as Record).type as string, 'fast', 'wallet[0].type arg'); @@ -395,7 +399,7 @@ await test('evaluateTask: hallucinated method → hallucinationRate > 0', () => error: undefined, knownMethods: KNOWN_METHODS, }); - assert(result.metrics.hallucinatedCalls.length > 0, 'hallucinatedCalls should be non-empty'); + assert(result.metrics.hallucinatedActions.length > 0, 'hallucinatedActions should be non-empty'); assert(result.metrics.hallucinationRate > 0, 'hallucinationRate should be > 0'); }); @@ -424,9 +428,9 @@ await test('evaluateTask: nested expected args match recursively', () => { const task: TaskDefinition = { id: 'nested-args', prompt: 'Task nested', - expected_tools: [ + expected_actions: [ { - method: 'x402Pay', + name: 'x402Pay', args: { url: 'https://api.example.com/premium', wallet: { @@ -466,9 +470,9 @@ await test('evaluateTask: nested expected args fail when nested field differs', const task: TaskDefinition = { id: 'nested-args-fail', prompt: 'Task nested fail', - expected_tools: [ + expected_actions: [ { - method: 'x402Pay', + name: 'x402Pay', args: { wallet: { type: 'fast', @@ -565,10 +569,10 @@ await test('evaluateTask: resolves raw calls via bindings + task expectations', const task: TaskDefinition = { id: 'resolve-test', prompt: 'Test resolution', - expected_tools: [ - { method: 'fast', args: { network: 'testnet' } }, - { method: 'FastClient.setup' }, - { method: 'FastClient.balance' }, + expected_actions: [ + { name: 'fast', args: { network: 'testnet' } }, + { name: 'FastClient.setup' }, + { name: 'FastClient.balance' }, ], }; const extractedCalls: ExtractedCall[] = [ @@ -597,9 +601,9 @@ await test('evaluateTask: resolves constructor-based bindings', () => { const task: TaskDefinition = { id: 'constructor-resolve', prompt: 'Test constructor', - expected_tools: [ - { method: 'AllSetProvider.constructor', args: { network: 'testnet' } }, - { method: 'AllSetProvider.sendToFast' }, + expected_actions: [ + { name: 'AllSetProvider.constructor', args: { network: 'testnet' } }, + { name: 'AllSetProvider.sendToFast' }, ], }; const extractedCalls: ExtractedCall[] = [ @@ -767,25 +771,6 @@ await test('checkConfig: direct openai model IDs do not get OpenRouter dot-versi assert(warn === undefined, 'direct openai model IDs should not be warned as OpenRouter dot versions'); }); -await test('checkConfig: deprecated benchmark.tasks field → fixable warning', async () => { - const { checkConfig } = await import('../src/project/validate.js'); - const config = { - name: 'test', - target: { surface: 'cli' as const, discovery: { sources: ['./src/cli.ts'] } }, - benchmark: { - format: 'pi' as const, - models: [{ id: 'openrouter/openai/gpt-4o', name: 'GPT', tier: 'flagship' as const }], - tasks: './tasks.json', - taskGeneration: { enabled: true, maxTasks: 5 }, - }, - }; - const issues = await checkConfig(config as any, '/fake/skill-optimizer.json'); - const warn = issues.find(i => i.code === 'deprecated-tasks-field'); - assert(warn !== undefined, 'expected deprecated-tasks-field issue'); - assert(warn!.severity === 'warning', 'should be warning'); - assert(warn!.fixable === true, 'deprecated-tasks-field should be fixable'); -}); - await test('checkConfig: codex auth rejects non-openai benchmark models', async () => { const { checkConfig } = await import('../src/project/validate.js'); const config = { @@ -891,24 +876,6 @@ await test('applyFixes: normalises dot versions in model IDs', async () => { assertEqual(models[0]!.id, 'openrouter/anthropic/claude-sonnet-4-6', 'dots should be replaced with hyphens'); }); -await test('applyFixes: removes deprecated benchmark.tasks field', async () => { - const { applyFixes } = await import('../src/project/fix.js'); - const { checkConfig } = await import('../src/project/validate.js'); - const rawJson = { - name: 'test', - target: { surface: 'cli', discovery: { sources: ['./src/cli.ts'] } }, - benchmark: { - format: 'pi', - models: [{ id: 'openrouter/openai/gpt-4o', name: 'GPT', tier: 'flagship' }], - tasks: './tasks.json', - taskGeneration: { enabled: true, maxTasks: 5 }, - }, - }; - const issues = await checkConfig(rawJson as any, '/fake/skill-optimizer.json'); - const fixed = applyFixes(rawJson as any, issues, '/fake'); - assert(!(fixed.benchmark as any).tasks, 'deprecated tasks field should be removed'); -}); - await test('applyFixes: does not mutate input', async () => { const { applyFixes } = await import('../src/project/fix.js'); const { checkConfig } = await import('../src/project/validate.js'); diff --git a/tests/smoke-coverage.ts b/tests/smoke-coverage.ts index 31dd941..890421c 100644 --- a/tests/smoke-coverage.ts +++ b/tests/smoke-coverage.ts @@ -16,8 +16,7 @@ function mkTask(id: string, actions: string[]): GeneratedTask { return { id, prompt: `do ${id}`, - expected_actions: actions.map((name) => ({ name, method: name })), - expected_tools: actions.map((name) => ({ name, method: name })), + expected_actions: actions.map((name) => ({ name })), }; } diff --git a/tests/smoke-discovery-mcp.ts b/tests/smoke-discovery-mcp.ts index 40d2c6f..32a7512 100644 --- a/tests/smoke-discovery-mcp.ts +++ b/tests/smoke-discovery-mcp.ts @@ -157,16 +157,12 @@ await test('code-first discovery fails fast when MCP source is missing', async ( // loadProjectConfig now validates paths eagerly — expect it to throw when source is missing let threw = false; try { - const project = await loadProjectConfig(configPath); - // If load succeeds (future behavior change), check that buildSurfaceSnapshot also fails - buildSurfaceSnapshot(project); + await loadProjectConfig(configPath); } catch (error: any) { threw = true; - // Accept either the new validation error or the old discovery error - const msg: string = error.message; assert( - msg.includes('does not exist') || msg.includes('not found'), - `missing source error should mention missing path, got: ${msg}`, + error.message.includes('does not exist'), + `missing source error should mention missing path, got: ${error.message}`, ); } assert(threw, 'missing discovery source should fail fast'); @@ -252,19 +248,14 @@ await test('discoverActions fails fast when MCP discovery source is missing and }, }, null, 2), 'utf-8'); - // loadProjectConfig now validates paths eagerly — expect it to throw when source is missing let threw = false; try { - const project = await loadProjectConfig(configPath); - // If load succeeds (future behavior change), check that discoverActions also fails - discoverActions(project); + await loadProjectConfig(configPath); } catch (error: any) { threw = true; - // Accept either the new validation error or the old discovery error - const msg: string = error.message; assert( - msg.includes('does not exist') || msg.includes('not found'), - `missing source error should mention missing path, got: ${msg}`, + error.message.includes('does not exist'), + `missing source error should mention missing path, got: ${error.message}`, ); } diff --git a/tests/smoke-e2e.ts b/tests/smoke-e2e.ts index 5f15dc3..82ce0ae 100644 --- a/tests/smoke-e2e.ts +++ b/tests/smoke-e2e.ts @@ -118,7 +118,7 @@ async function testFullLoopReachesPass(): Promise { optimizer: { maxIterations: 3, stabilityWindow: 3, // large window so we don't stop early on stable - minOverallPassDelta: 0.0, // accept any improvement (even 0 delta) + minImprovement: 0.0, // accept any improvement (even 0 delta) perModelFloor: 0, targetWeightedAverage: 0, models: [ diff --git a/tests/smoke-errors.ts b/tests/smoke-errors.ts index a0f7b8f..7d538c4 100644 --- a/tests/smoke-errors.ts +++ b/tests/smoke-errors.ts @@ -29,26 +29,6 @@ function testConfigNotFound() { console.log('PASS: config-not-found error has next step'); } -function testLegacyFilenameError() { - const dir = mkdtempSync(join(tmpdir(), 'skill-opt-legacy-')); - try { - const p = join(dir, 'skill-benchmark.json'); - writeFileSync(p, '{}'); - const result = run(['run', '--config', p]); - // This test is about the legacy filename detection which only fires when no --config is given - // and the file is named skill-benchmark.json in cwd. We skip the cwd-based detection here - // and just check that config-not-found is reported for the explicit path. - const combined = result.stderr + result.stdout; - if (combined.includes('legacy') || combined.includes('skill-optimizer.json')) { - console.log('PASS: legacy filename error names new filename'); - } else { - // Config not found is acceptable since we gave an explicit path - console.log('SKIP: legacy test — cwd plumbing varies'); - } - } finally { - rmSync(dir, { recursive: true, force: true }); - } -} function testInvalidJson() { const dir = mkdtempSync(join(tmpdir(), 'skill-opt-inv-')); @@ -153,7 +133,6 @@ function testRepoPathMissing() { async function main() { testConfigNotFound(); - testLegacyFilenameError(); testInvalidJson(); testMissingApiKeyOnRun(); testEmptyScope(); diff --git a/tests/smoke-feedback.ts b/tests/smoke-feedback.ts index fc8dddf..6384268 100644 --- a/tests/smoke-feedback.ts +++ b/tests/smoke-feedback.ts @@ -15,7 +15,7 @@ function mkResult(opts: { error?: string; }): TaskResult { return { - task: { id: opts.taskId, prompt: 'p', expected_actions: [opts.expected], expected_tools: [opts.expected] }, + task: { id: opts.taskId, prompt: 'p', expected_actions: [opts.expected] }, model: { id: opts.modelId, name: opts.modelName, tier: 'mid' }, generatedCode: null, rawResponse: '', @@ -27,15 +27,14 @@ function mkResult(opts: { argsCorrect: opts.passed, matched: opts.passed, }], - toolMatches: [], metrics: { toolPrecision: 0, toolRecall: 0, taskPassed: opts.passed, toolSelectionAccuracy: opts.passed ? 1 : 0, argAccuracy: opts.passed ? 1 : 0, - unnecessaryCalls: [], - hallucinatedCalls: opts.extracted?.filter((c) => c.name !== opts.expected.name).map((c) => c.name) ?? [], + unnecessaryActions: [], + hallucinatedActions: opts.extracted?.filter((c) => c.name !== opts.expected.name).map((c) => c.name) ?? [], hallucinationRate: 0, }, llmLatencyMs: 0, diff --git a/tests/smoke-generation.ts b/tests/smoke-generation.ts index 3ae2c3a..164a4d3 100644 --- a/tests/smoke-generation.ts +++ b/tests/smoke-generation.ts @@ -348,17 +348,17 @@ await test('generateTasksForProject: runs discover -> generate -> ground -> free { id: 'kept-task', prompt: 'Create wallet called beta.', - expected_tools: [{ method: 'create_wallet', args: { label: 'beta' } }], + expected_actions: [{ name: 'create_wallet', args: { label: 'beta' } }], }, { id: 'balance-task', prompt: 'Get balance for address 0x1.', - expected_tools: [{ method: 'get_balance', args: { address: '0x1' } }], + expected_actions: [{ name: 'get_balance', args: { address: '0x1' } }], }, { id: 'rejected-task', prompt: 'Use unknown method.', - expected_tools: [{ method: 'delete_wallet', args: { id: 'x' } }], + expected_actions: [{ name: 'delete_wallet', args: { id: 'x' } }], }, ], }); diff --git a/tests/smoke-mcp.ts b/tests/smoke-mcp.ts index 4f99a63..e165abc 100644 --- a/tests/smoke-mcp.ts +++ b/tests/smoke-mcp.ts @@ -1,14 +1,14 @@ import { extractFromToolCalls } from '../src/benchmark/extractors/mcp-extractor.js'; import { extract } from '../src/benchmark/extractors/index.js'; -import { evaluateTask, matchTools } from '../src/benchmark/evaluator.js'; +import { evaluateTask, matchActions } from '../src/benchmark/evaluator.js'; import type { ExtractedCall, LLMResponse, BenchmarkConfig, TaskDefinition, - ExpectedTool, + ExpectedAction, ModelConfig, - ToolMatch, + ActionMatch, } from '../src/benchmark/types.js'; // ── Test harness ────────────────────────────────────────────────────────── @@ -101,36 +101,36 @@ await test('extractFromToolCalls: undefined toolCalls', () => { assertEqual(calls.length, 0, 'should return empty array when toolCalls is undefined'); }); -// ── Group 2: matchTools (arg validation) ────────────────────────────────── +// ── Group 2: matchActions (arg validation) ────────────────────────────────── -await test('matchTools: regex arg matching', () => { - const expectedTools: ExpectedTool[] = [ - { method: 'send_tokens', args: { to: '/fast1.+/' } }, +await test('matchActions: regex arg matching', () => { + const expectedTools: ExpectedAction[] = [ + { name: 'send_tokens', args: { to: '/fast1.+/' } }, ]; const extractedCalls: ExtractedCall[] = [ { method: 'send_tokens', args: { to: 'fast1abc123def' }, line: 0, raw: '' }, ]; - const matches = matchTools(expectedTools, extractedCalls); + const matches = matchActions(expectedTools, extractedCalls); - assertEqual(matches.length, 1, 'should return 1 ToolMatch'); + assertEqual(matches.length, 1, 'should return 1 ActionMatch'); assert(matches[0].methodFound === true, 'methodFound should be true'); assert(matches[0].argsCorrect === true, 'argsCorrect should be true — regex /fast1.+/ should match "fast1abc123def"'); }); -await test('matchTools: dynamic sentinel passes', () => { - const expectedTools: ExpectedTool[] = [ - { method: 'get_balance', args: { address: '' } }, +await test('matchActions: dynamic sentinel passes', () => { + const expectedTools: ExpectedAction[] = [ + { name: 'get_balance', args: { address: '' } }, ]; const extractedCalls: ExtractedCall[] = [ { method: 'get_balance', args: { address: 'literally_anything' }, line: 0, raw: '' }, ]; - const matches = matchTools(expectedTools, extractedCalls); + const matches = matchActions(expectedTools, extractedCalls); - assertEqual(matches.length, 1, 'should return 1 ToolMatch'); + assertEqual(matches.length, 1, 'should return 1 ActionMatch'); assert(matches[0].methodFound === true, 'methodFound should be true'); // in expected acts as a wildcard — any value from the LLM is acceptable. assert( @@ -153,9 +153,9 @@ await test('evaluateTask: MCP perfect match', () => { const task: TaskDefinition = { id: 'task-perfect', prompt: 'Create a wallet and check its balance', - expected_tools: [ - { method: 'create_wallet' }, - { method: 'get_balance' }, + expected_actions: [ + { name: 'create_wallet' }, + { name: 'get_balance' }, ], }; @@ -185,8 +185,8 @@ await test('evaluateTask: MCP hallucinated tool', () => { const task: TaskDefinition = { id: 'task-hallucination', prompt: 'Send some tokens', - expected_tools: [ - { method: 'send_tokens' }, + expected_actions: [ + { name: 'send_tokens' }, ], }; @@ -208,8 +208,8 @@ await test('evaluateTask: MCP hallucinated tool', () => { }); assert( - result.metrics.hallucinatedCalls.includes('delete_everything'), - 'hallucinatedCalls should include "delete_everything"', + result.metrics.hallucinatedActions.includes('delete_everything'), + 'hallucinatedActions should include "delete_everything"', ); assert(result.metrics.hallucinationRate > 0, 'hallucinationRate should be > 0'); }); diff --git a/tests/smoke-optimize.ts b/tests/smoke-optimize.ts index c149bea..eb0325a 100644 --- a/tests/smoke-optimize.ts +++ b/tests/smoke-optimize.ts @@ -42,7 +42,7 @@ function makeReport(overallPassRate: number, kinds: Array<'missing' | 'args' | ' task: { id: `task-${overallPassRate}-${kinds.join('-') || 'ok'}`, prompt: 'Do the thing', - expected_tools: [{ method: 'Client.doThing', args: { id: '123' } }], + expected_actions: [{ name: 'Client.doThing', args: { id: '123' } }], }, model: { id: 'openai/test', @@ -52,15 +52,15 @@ function makeReport(overallPassRate: number, kinds: Array<'missing' | 'args' | ' generatedCode: 'client.doThing({ id: "123" })', rawResponse: 'ok', extractedCalls: [], - toolMatches: [], + actionMatches: [], metrics: { toolPrecision: 1, toolRecall: 1, taskPassed: true, toolSelectionAccuracy: 1, argAccuracy: 1, - unnecessaryCalls: [], - hallucinatedCalls: [], + unnecessaryActions: [], + hallucinatedActions: [], hallucinationRate: 0, }, llmLatencyMs: 10, @@ -76,9 +76,9 @@ function makeReport(overallPassRate: number, kinds: Array<'missing' | 'args' | ' ...baseResult.task, id: `task-${kind}-${index}`, }, - toolMatches: [ + actionMatches: [ { - expected: { method: 'Client.doThing', args: { id: '123' } }, + expected: { name: 'Client.doThing', args: { id: '123' } }, found: kind === 'missing' ? null : { method: 'Client.doThing', args: { id: kind === 'args' ? '999' : '123' }, line: 1, raw: 'mock' }, methodFound: kind !== 'missing', argsCorrect: kind !== 'args', @@ -90,7 +90,7 @@ function makeReport(overallPassRate: number, kinds: Array<'missing' | 'args' | ' taskPassed: false, toolSelectionAccuracy: kind === 'missing' ? 0 : 1, argAccuracy: kind === 'args' ? 0 : 1, - hallucinatedCalls: kind === 'hallucination' ? ['Client.deleteEverything'] : [], + hallucinatedActions: kind === 'hallucination' ? ['Client.deleteEverything'] : [], hallucinationRate: kind === 'hallucination' ? 1 : 0, }, error: kind === 'error' ? 'provider failed' : undefined, @@ -161,7 +161,7 @@ function makeManifest(): OptimizeManifest { mode: 'stable-surface' as any, maxIterations: 5, stabilityWindow: 2, - minOverallPassDelta: 0.01, + minImprovement: 0.01, taskGeneration: { enabled: false, maxGenerated: 10, @@ -580,7 +580,7 @@ await test('runOptimizeLoop: applies defaults to partially specified manifests', }; const result = await runOptimizeLoop(manifest, deps); - assertEqual(result.iterations[0]?.accepted, true, 'default minOverallPassDelta should still allow acceptance'); + assertEqual(result.iterations[0]?.accepted, true, 'default minImprovement should still allow acceptance'); }); await test('runOptimizeLoop: rejects requireCleanGit=false even for programmatic manifests', async () => { diff --git a/tests/smoke-sdk-python.ts b/tests/smoke-sdk-python.ts index 2f59399..49cc484 100644 --- a/tests/smoke-sdk-python.ts +++ b/tests/smoke-sdk-python.ts @@ -78,9 +78,9 @@ await test('python: factory methods do not mislabel returned instance type', asy const task = { id: 'python-factory-binding', prompt: 'send a payment', - expected_tools: [ - { method: 'FastClient.constructor', args: { _positional_0: 'testnet' } }, - { method: 'FastWallet.send', args: { to: 'fast1abc' } }, + expected_actions: [ + { name: 'FastClient.constructor', args: { _positional_0: 'testnet' } }, + { name: 'FastWallet.send', args: { to: 'fast1abc' } }, ], }; const result = evaluateTask({ diff --git a/tests/smoke-sdk-rust.ts b/tests/smoke-sdk-rust.ts index bcbdce3..66b8925 100644 --- a/tests/smoke-sdk-rust.ts +++ b/tests/smoke-sdk-rust.ts @@ -85,9 +85,9 @@ await test('rust: factory methods on instances resolve through evaluator binding const task = { id: 'rust-factory-binding', prompt: 'send a payment', - expected_tools: [ - { method: 'FastClient.new', args: { _positional_0: 'testnet' } }, - { method: 'FastWallet.send', args: { to: 'fast1abc', amount: '5' } }, + expected_actions: [ + { name: 'FastClient.new', args: { _positional_0: 'testnet' } }, + { name: 'FastWallet.send', args: { to: 'fast1abc', amount: '5' } }, ], }; const result = evaluateTask({ From ff4c3835fd032bcc979070e2203119c3979abcad Mon Sep 17 00:00:00 2001 From: "OpenClaw Agent (basd)" Date: Thu, 16 Apr 2026 06:10:44 +0000 Subject: [PATCH 7/7] chore(release): bump version to 1.1.0, update CHANGELOG and docs Co-Authored-By: Claude Sonnet 4.6 --- CHANGELOG.md | 2 +- CONTRIBUTING.md | 4 ++-- SKILL/SKILL.md | 18 +++++++++--------- SKILL/references/config.md | 23 +++++++++++++++++++++-- SKILL/references/setup.md | 5 +++-- package.json | 2 +- 6 files changed, 37 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b76d149..2b673ce 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog -## Unreleased +## 1.1.0 — 2026-04-16 ### Breaking Changes diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c894b5a..2a4cd71 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -35,11 +35,11 @@ All three commands must pass before opening a PR. ## Adding a surface type -Follow the shape of existing discoverers in `src/project/snapshot.ts`. A surface discoverer returns `ActionDefinition[]`. Then: +A surface discoverer returns `ActionDefinition[]`. To add one: 1. Extend `BenchmarkSurface` in `src/benchmark/types.ts`. 2. Add a branch to `src/project/validate.ts` and `src/project/resolve.ts`. -3. Register the discoverer in `src/project/snapshot.ts`. +3. Implement the new code-first discoverer in `src/actions/discover.ts`, then register it in `src/project/snapshot.ts` (the dispatcher that routes surfaces to discoverers). 4. Add a discovery smoke test. ## Adding an LLM provider diff --git a/SKILL/SKILL.md b/SKILL/SKILL.md index d81afd1..e725323 100644 --- a/SKILL/SKILL.md +++ b/SKILL/SKILL.md @@ -1,17 +1,17 @@ --- name: skill-optimizer description: > - Benchmark and optimize SDK, CLI, and MCP documentation so every LLM model - can reliably call the right actions with correct arguments. Use when setting - up skill-optimizer for a project, running benchmarks, interpreting results, - optimizing SKILL.md files, or diagnosing configuration issues. Also use when - working inside the skill-optimizer repository itself — for running against - mock repos, testing changes, or understanding the codebase. + Benchmark and optimize SDK, CLI, MCP, and prompt documentation so every LLM + model can reliably call the right actions with correct arguments. Use when + setting up skill-optimizer for a project, running benchmarks, interpreting + results, optimizing SKILL.md files, or diagnosing configuration issues. Also + use when working inside the skill-optimizer repository itself — for running + against mock repos, testing changes, or understanding the codebase. --- # skill-optimizer -Benchmark your SDK / CLI / MCP docs against multiple LLMs, measure whether they call the right actions with the right arguments, and iteratively rewrite your guidance until a quality floor is met across every model. +Benchmark your SDK / CLI / MCP / prompt docs against multiple LLMs, measure whether they call the right actions with the right arguments, and iteratively rewrite your guidance until a quality floor is met across every model. ## Context Detection @@ -27,7 +27,7 @@ Before doing anything, figure out where you are: | Task | Command | |------|---------| -| Init config | `npx skill-optimizer init cli\|sdk\|mcp` | +| Init config | `npx skill-optimizer init cli\|sdk\|mcp\|prompt` | | Init (non-interactive) | `npx skill-optimizer init cli --yes` | | Import CLI commands | `npx skill-optimizer import-commands --from ./src/cli.ts` | | Import (binary scrape) | `npx skill-optimizer import-commands --from my-cli --scrape` | @@ -56,7 +56,7 @@ If you are in an **unconfigured project** (context detection case 3), start with ## Key Concepts -**Surfaces** — The callable interface of your project: SDK methods, CLI commands, or MCP tools. Skill-optimizer discovers these via tree-sitter code analysis or from a hand-written manifest file. +**Surfaces** — The callable interface of your project: SDK methods, CLI commands, MCP tools, or prompt templates. Skill-optimizer discovers these via tree-sitter code analysis, manifest files, or markdown parsing. **Static evaluation** — Benchmark evaluation never executes generated code. Actions are extracted from model responses via pattern matching and compared structurally against expected calls. This makes benchmarks safe and repeatable. diff --git a/SKILL/references/config.md b/SKILL/references/config.md index a91f555..652ed85 100644 --- a/SKILL/references/config.md +++ b/SKILL/references/config.md @@ -74,13 +74,32 @@ Complete reference for `skill-optimizer.json`. For auto-generated schema docs, s } ``` +### Prompt surface + +```json +{ + "name": "my-skill-doc", + "target": { + "surface": "prompt", + "repoPath": "/path/to/my-project", + "skill": "./SKILL.md" + }, + "benchmark": { + "format": "pi", + "models": [ + { "id": "openrouter/anthropic/claude-sonnet-4-6", "name": "Claude Sonnet", "tier": "flagship" } + ] + } +} +``` + ## Field-by-Field Reference ### `target` — What You're Benchmarking | Field | Required | Default | Description | |-------|----------|---------|-------------| -| `surface` | Yes | — | `"cli"`, `"sdk"`, or `"mcp"` | +| `surface` | Yes | — | `"cli"`, `"sdk"`, `"mcp"`, or `"prompt"` | | `repoPath` | Yes | — | Absolute or config-relative path to your project root | | `skill` | Yes | — | Path to your SKILL.md or guidance doc, relative to `repoPath` | | `discovery.mode` | No | `"auto"` | `"auto"` (tree-sitter) or `"manifest"` (hand-written JSON) | @@ -164,7 +183,7 @@ Task generation is **coverage-guaranteed**: every in-scope action gets at least | Code | Meaning | Fix | |------|---------|-----| | `E_MISSING_SKILL` | `target.skill` file not found | Create the file or fix the path in config | -| `E_INVALID_SURFACE` | `target.surface` is not cli/sdk/mcp | Use one of the three valid values | +| `E_INVALID_SURFACE` | `target.surface` is not cli/sdk/mcp/prompt | Use one of the four valid values | | `E_DIRTY_GIT` | Uncommitted changes in target repo | Commit or stash, or set `requireCleanGit: false` | | `E_EMPTY_SCOPE` | Scope filters matched no actions | Check your `include`/`exclude` patterns | | `E_MISSING_API_KEY` | API key env var not set | `export OPENROUTER_API_KEY=sk-or-...` (or `OPENAI_API_KEY` / `ANTHROPIC_API_KEY` for direct formats) | diff --git a/SKILL/references/setup.md b/SKILL/references/setup.md index e13d836..9683183 100644 --- a/SKILL/references/setup.md +++ b/SKILL/references/setup.md @@ -35,15 +35,16 @@ npx skill-optimizer --help ## 2. Determine Your Surface Type -skill-optimizer supports three surface types. Pick the one that matches your project: +skill-optimizer supports four surface types. Pick the one that matches your project: | Surface | Your project exposes... | Examples | |---------|------------------------|----------| | `cli` | CLI commands or a binary | Yargs, Commander, @optique/core, argparse, Click, Clap | | `sdk` | Library methods users call in code | TypeScript/Python/Rust SDKs | | `mcp` | MCP tool handlers | MCP servers with `server.tool()` definitions | +| `prompt` | Prompt templates or agent skill docs | SKILL.md files, Claude Code skills, agent instructions | -If unsure: does your user run commands in a terminal (`cli`), import your package and call functions (`sdk`), or connect an AI agent to your tool server (`mcp`)? +If unsure: does your user run commands in a terminal (`cli`), import your package and call functions (`sdk`), connect an AI agent to your tool server (`mcp`), or follow a prompt template / skill document (`prompt`)? ## 3. Run the Init Wizard diff --git a/package.json b/package.json index 81d1f7a..ec03192 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "skill-optimizer", - "version": "1.0.0", + "version": "1.1.0", "description": "Benchmark and optimizer for evaluating SDK, CLI, and MCP guidance with static action matching.", "license": "MIT", "author": "Pi2 Labs",