From b3e519680177172936909585f6d16fcc13e5330a Mon Sep 17 00:00:00 2001 From: Jack Felke Date: Mon, 16 Mar 2026 08:45:19 -0700 Subject: [PATCH 1/4] docs: add usage walkthrough with realistic tool output examples Shows 4 concrete scenarios: vague prompt clarification, multi-step scoping, correction pattern matching, and cross-service awareness. Each example includes the prompt, triage classification, and the actual output users can expect. --- README.md | 123 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 123 insertions(+) diff --git a/README.md b/README.md index 6d03f5d..7e8b3f5 100644 --- a/README.md +++ b/README.md @@ -74,6 +74,129 @@ The pattern is always the same: vague prompt → Claude guesses → wrong output --- +## Usage Walkthrough + +Here's what preflight actually looks like in practice — real tool calls with realistic output. + +### 1. Vague prompt → automatic clarification + +You type into Claude Code: + +``` +> update the dashboard +``` + +Claude calls `preflight_check` with `{ "prompt": "update the dashboard" }`. Triage classifies this as **ambiguous** (< 50 chars, no file refs). You get back: + +``` +⚠️ AMBIGUOUS — needs clarification before proceeding + +📂 Git State: + Branch: feat/analytics + 3 uncommitted files: src/components/Dashboard.tsx, src/hooks/useDashboard.ts, src/api/metrics.ts + +🔍 Ambiguity Signals: + - "update" is vague — add data? change layout? fix a bug? restyle? + - "dashboard" matches 4 files across the project + - No specific file path or line number referenced + +💡 Try something like: + - "add the new revenue chart to src/components/Dashboard.tsx using the metrics API from src/api/metrics.ts" + - "fix the loading spinner that won't dismiss in the analytics dashboard" + - "update dashboard layout to 3-column grid per the Figma mock" +``` + +### 2. Multi-step task → scoped execution plan + +``` +> refactor the payments module to support Stripe and add webhook handling +``` + +Triage: **multi-step** (multiple verbs + cross-concern). `scope_work` and `sequence_tasks` fire automatically: + +``` +🔶 MULTI-STEP — breaking this down before starting + +📋 Execution Plan: + Step 1: Extract payment interface (src/payments/provider.ts) + → Define PaymentProvider interface with charge(), refund(), webhook() + → Risk: LOW — new file, no existing consumers yet + + Step 2: Implement Stripe adapter (src/payments/stripe.ts) + → Implement PaymentProvider using stripe SDK + → Dependencies: stripe npm package + → Risk: MEDIUM — needs API keys, error handling for network failures + + Step 3: Add webhook endpoint (src/api/webhooks/stripe.ts) + → POST /api/webhooks/stripe with signature verification + → Risk: HIGH — security-sensitive, needs crypto verification + → ⚠️ Checkpoint recommended before this step + + Step 4: Update consumers (src/checkout/*, src/subscriptions/*) + → Swap direct calls to new PaymentProvider interface + → 6 files affected + +🧪 Suggested checkpoints: after Step 1, after Step 3 +⏱️ Estimated tokens: ~45,000 across all steps +``` + +### 3. Correction pattern matching + +You previously logged a correction with `log_correction`: + +``` +> log_correction: "When I said 'update types', I meant only the shared types in packages/shared, not the local type files in each service" +``` + +Next time you say: + +``` +> update the types +``` + +Preflight catches it: + +``` +⚠️ PATTERN MATCH — you've corrected this before + +🔄 Previous correction (2 days ago): + "When I said 'update types', I meant only the shared types in packages/shared, + not the local type files in each service" + +Did you mean shared types in packages/shared? If so, try: + "update shared types in packages/shared/types.ts" +``` + +### 4. Cross-service awareness + +``` +> add a loyalty points field to the user profile +``` + +Triage: **cross-service** (detected `user` + field change pattern, config has related projects). Contracts are searched: + +``` +🔗 CROSS-SERVICE — this change affects multiple projects + +📂 Current project: web-app + - src/types/user.ts → UserProfile interface (line 12) + - prisma/schema.prisma → User model (line 34) + +🔗 Related projects: + - mobile-app: src/api/types.ts → UserProfile (mirrors web-app, line 8) + - rewards-service: src/models/user.ts → UserRecord (includes loyalty_points already?) + - analytics-pipeline: src/schemas/user-events.avro → UserEvent schema + +⚠️ Changing UserProfile requires updates in 3 services. + Suggested order: + 1. prisma/schema.prisma (source of truth) + migrate + 2. web-app src/types/user.ts + 3. mobile-app src/api/types.ts (notify mobile team) + 4. analytics-pipeline schema (may need backfill) +``` + +--- + ## Quick Start ### Option A: npx (fastest — no install) From c9707b940beec124a27c60a568b6350629df168e Mon Sep 17 00:00:00 2001 From: Jack Felke Date: Mon, 16 Mar 2026 08:56:35 -0700 Subject: [PATCH 2/4] examples: add .preflight/ starter config files MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The README and examples/README.md referenced examples/.preflight/ but the actual config files didn't exist. Added: - config.yml — profile, related projects, thresholds, embeddings - triage.yml — keyword rules and strictness settings - contracts/api.yml — manual contract definition examples All files are heavily commented so users can copy them into their project root and customize without needing to reference the docs. --- examples/.preflight/config.yml | 63 ++++++++++++++++----------- examples/.preflight/contracts/api.yml | 59 ++++++++++--------------- examples/.preflight/triage.yml | 44 +++++++++---------- 3 files changed, 81 insertions(+), 85 deletions(-) diff --git a/examples/.preflight/config.yml b/examples/.preflight/config.yml index f59170f..297d8d7 100644 --- a/examples/.preflight/config.yml +++ b/examples/.preflight/config.yml @@ -1,35 +1,46 @@ -# .preflight/config.yml — Drop this in your project root -# -# This is an example config for a typical Next.js + microservices setup. -# Every field is optional — preflight works with sensible defaults out of the box. -# Commit this to your repo so the whole team gets the same preflight behavior. - -# Profile controls how much detail preflight returns. -# "minimal" — only flags ambiguous+ prompts, skips clarification detail +# .preflight/config.yml — Drop this in your project root. +# Every field is optional. Preflight works out of the box with sensible defaults. +# Commit this to your repo so your whole team gets the same behavior. + +# Profile controls how much detail preflight surfaces. +# "minimal" — only flag ambiguous+, skip clarification detail # "standard" — balanced (default) # "full" — maximum detail on every non-trivial prompt profile: standard -# Related projects for cross-service awareness. -# Preflight will search these for shared types, routes, and contracts -# so it can warn you when a change might break a consumer. +# Related projects for cross-service contract awareness. +# Preflight scans these for shared types, interfaces, and API routes +# so it can warn when a change in one project might break another. related_projects: - - path: /Users/you/code/auth-service - alias: auth - - path: /Users/you/code/billing-api - alias: billing - - path: /Users/you/code/shared-types - alias: types - -# Behavioral thresholds — tune these to your workflow + # Example: a backend API that your frontend talks to + # - path: /Users/you/projects/my-api + # alias: api + + # Example: a shared types package + # - path: /Users/you/projects/shared-types + # alias: shared + +# Behavioral thresholds — tune these to your workflow. thresholds: - session_stale_minutes: 30 # Warn if no activity for this long - max_tool_calls_before_checkpoint: 100 # Suggest a checkpoint after N tool calls - correction_pattern_threshold: 3 # Min corrections before flagging a pattern + # Warn if no session activity for this many minutes + session_stale_minutes: 30 -# Embedding provider for semantic search over session history. -# "local" uses Xenova transformers (no API key needed, runs on CPU). -# "openai" uses text-embedding-3-small (faster, needs OPENAI_API_KEY). + # Suggest a checkpoint after this many tool calls in one session + max_tool_calls_before_checkpoint: 100 + + # Minimum correction occurrences before flagging as a pattern + # (e.g., if you keep correcting "use the auth module" 3+ times, + # preflight learns to warn you proactively) + correction_pattern_threshold: 3 + +# Embedding configuration for vector search (timeline tools). +# Local embeddings work offline with no API key — they just need +# a one-time ~90MB model download on first use. embeddings: + # "local" — Xenova/all-MiniLM-L6-v2, runs on-device, no API key needed + # "openai" — OpenAI text-embedding-3-small, needs OPENAI_API_KEY + # "ollama" — Local Ollama server, needs `ollama serve` running provider: local - # openai_api_key: sk-... # Uncomment if using openai provider + + # Only needed if provider is "openai": + # openai_api_key: sk-... diff --git a/examples/.preflight/contracts/api.yml b/examples/.preflight/contracts/api.yml index 512543f..7a2ebd3 100644 --- a/examples/.preflight/contracts/api.yml +++ b/examples/.preflight/contracts/api.yml @@ -1,17 +1,15 @@ -# .preflight/contracts/api.yml — Manual contract definitions +# .preflight/contracts/api.yml — Manual contract definitions. # -# Define shared types and interfaces that preflight should know about. -# These supplement auto-extracted contracts from your codebase. -# Manual definitions win on name conflicts with auto-extracted ones. +# These supplement auto-extraction. Use them when: +# - A related service isn't in TypeScript (preflight can't auto-extract) +# - You want to pin a specific contract shape for stability +# - You're documenting an external API your project depends on # -# Why manual contracts? -# - Document cross-service interfaces that live in docs, not code -# - Define contracts for external APIs your services consume -# - Pin down types that are implicit (e.g., event payloads) +# Preflight uses these to warn when a prompt might affect shared types. - name: User kind: interface - description: Core user model shared across all services + description: Core user object shared between frontend and API fields: - name: id type: string @@ -19,40 +17,29 @@ - name: email type: string required: true - - name: tier - type: "'free' | 'pro' | 'enterprise'" + - name: role + type: "'admin' | 'member' | 'viewer'" required: true - name: createdAt type: Date required: true -- name: AuthToken +- name: ApiResponse kind: interface - description: JWT payload structure from auth-service + description: Standard API response envelope fields: - - name: userId - type: string - required: true - - name: permissions - type: string[] - required: true - - name: expiresAt - type: number - required: true - -- name: WebhookPayload - kind: interface - description: Standard webhook envelope for inter-service events - fields: - - name: event - type: string - required: true - - name: timestamp - type: string + - name: success + type: boolean required: true - name: data - type: Record - required: true - - name: source + type: T + required: false + - name: error type: string - required: true + required: false + +# Add your own shared types below: +# - name: OrderStatus +# kind: enum +# description: Order lifecycle states +# values: [pending, confirmed, shipped, delivered, cancelled] diff --git a/examples/.preflight/triage.yml b/examples/.preflight/triage.yml index b3d394e..4aa02d1 100644 --- a/examples/.preflight/triage.yml +++ b/examples/.preflight/triage.yml @@ -1,45 +1,43 @@ -# .preflight/triage.yml — Controls how preflight classifies your prompts -# -# The triage engine routes prompts into categories: -# TRIVIAL → pass through (commit, format, lint) -# CLEAR → well-specified, no intervention needed -# AMBIGUOUS → needs clarification before proceeding -# MULTI-STEP → complex task, preflight suggests a plan -# CROSS-SERVICE → touches multiple projects, pulls in contracts -# -# Customize the keywords below to match your domain. +# .preflight/triage.yml — Controls how preflight classifies your prompts. +# Customize these rules to match your project's domain vocabulary. rules: - # Prompts containing these words are always flagged as AMBIGUOUS. - # Add domain-specific terms that tend to produce vague prompts. + # Prompts containing these keywords are always flagged as AMBIGUOUS or higher. + # Add domain-specific terms that tend to produce vague requests. always_check: - rewards - permissions - migration - schema - - pricing # example: your billing domain - - onboarding # example: multi-step user flows + # Add your own: + # - billing + # - deployment + # - auth - # Prompts containing these words skip checks entirely (TRIVIAL). - # These are safe, mechanical tasks that don't need guardrails. + # Prompts containing these keywords pass through as TRIVIAL. + # These are routine commands where clarification adds no value. skip: - commit - format - lint - - prettier - - "git push" + # Add your own: + # - typecheck + # - "git status" - # Prompts containing these words trigger CROSS-SERVICE classification. - # Preflight will search related_projects for relevant types and routes. + # Prompts containing these keywords trigger CROSS-SERVICE checks. + # These scan related_projects (from config.yml) for shared contracts. cross_service_keywords: - auth - notification - event - webhook - - billing # matches the related_project alias + # Add your own: + # - graphql + # - grpc + # - queue # How aggressively to classify prompts. -# "relaxed" — more prompts pass as clear (experienced users) +# "relaxed" — more prompts pass as clear (less interruption) # "standard" — balanced (default) -# "strict" — more prompts flagged as ambiguous (new teams, complex codebases) +# "strict" — more prompts flagged as ambiguous (maximum savings) strictness: standard From 2b4a020908daaccb699604891431a364036addde Mon Sep 17 00:00:00 2001 From: Jack Felke Date: Mon, 16 Mar 2026 09:30:28 -0700 Subject: [PATCH 3/4] fix: resolve templates path from package root in init CLI MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The init wizard resolved the templates directory relative to dist/cli/, which pointed to dist/templates/ — a path that doesn't exist. Templates live in src/templates/ and ship in the npm tarball via the 'files' field. Fixed by going up three levels (dist/cli/init.js → package root) instead of two, then into src/templates/. --- src/cli/init.ts | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/cli/init.ts b/src/cli/init.ts index dfaaa25..871fe10 100644 --- a/src/cli/init.ts +++ b/src/cli/init.ts @@ -27,10 +27,12 @@ async function createPreflightConfig(): Promise { try { await mkdir(preflightDir, { recursive: true }); - // Get the current module's directory to find templates + // Get the current module's directory to find templates. + // At runtime this file is dist/cli/init.js — go up twice to reach the + // package root, then into src/templates/ (which ships in the npm tarball). const currentFile = fileURLToPath(import.meta.url); - const srcDir = dirname(dirname(currentFile)); // Go up from cli/ to src/ - const templatesDir = join(srcDir, "templates"); + const packageRoot = dirname(dirname(dirname(currentFile))); // dist/cli/ → dist/ → root + const templatesDir = join(packageRoot, "src", "templates"); // Copy template files await copyFile(join(templatesDir, "config.yml"), join(preflightDir, "config.yml")); From 6a2c244e33dadeaa1c9efd404c629dfb9bc401eb Mon Sep 17 00:00:00 2001 From: Jack Felke Date: Mon, 16 Mar 2026 10:04:40 -0700 Subject: [PATCH 4/4] test: add 12 unit tests for lib/config.ts Tests cover: - Default config when no .preflight/ directory - Env var overrides (profile, embedding provider, API key, related projects) - Invalid env var values ignored - config.yml loading and merging with defaults - triage.yml loading and merging - Env vars ignored when .preflight/ exists - Singleton caching behavior - Graceful handling of malformed YAML - hasPreflightConfig() behavior - getRelatedProjects() backward compat helper --- tests/lib/config.test.ts | 215 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 215 insertions(+) create mode 100644 tests/lib/config.test.ts diff --git a/tests/lib/config.test.ts b/tests/lib/config.test.ts new file mode 100644 index 0000000..640a48e --- /dev/null +++ b/tests/lib/config.test.ts @@ -0,0 +1,215 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; +import { existsSync, readFileSync } from "fs"; +import { join } from "path"; + +// We need to mock modules before importing config +vi.mock("fs", () => ({ + existsSync: vi.fn(), + readFileSync: vi.fn(), +})); + +vi.mock("../../src/lib/files.js", () => ({ + PROJECT_DIR: "/fake/project", +})); + +const mockedExistsSync = vi.mocked(existsSync); +const mockedReadFileSync = vi.mocked(readFileSync); + +describe("config", () => { + beforeEach(() => { + vi.resetModules(); + vi.resetAllMocks(); + // Clear env vars + delete process.env.PROMPT_DISCIPLINE_PROFILE; + delete process.env.PREFLIGHT_RELATED; + delete process.env.EMBEDDING_PROVIDER; + delete process.env.OPENAI_API_KEY; + }); + + afterEach(() => { + delete process.env.PROMPT_DISCIPLINE_PROFILE; + delete process.env.PREFLIGHT_RELATED; + delete process.env.EMBEDDING_PROVIDER; + delete process.env.OPENAI_API_KEY; + }); + + async function loadFreshConfig() { + const mod = await import("../../src/lib/config.js"); + return mod; + } + + it("returns default config when no .preflight/ directory exists", async () => { + mockedExistsSync.mockReturnValue(false); + + const { getConfig } = await loadFreshConfig(); + const config = getConfig(); + + expect(config.profile).toBe("standard"); + expect(config.related_projects).toEqual([]); + expect(config.thresholds.session_stale_minutes).toBe(30); + expect(config.embeddings.provider).toBe("local"); + expect(config.triage.strictness).toBe("standard"); + }); + + it("applies env var overrides when no .preflight/ directory", async () => { + mockedExistsSync.mockReturnValue(false); + process.env.PROMPT_DISCIPLINE_PROFILE = "minimal"; + process.env.EMBEDDING_PROVIDER = "openai"; + process.env.OPENAI_API_KEY = "sk-test-123"; + + const { getConfig } = await loadFreshConfig(); + const config = getConfig(); + + expect(config.profile).toBe("minimal"); + expect(config.embeddings.provider).toBe("openai"); + expect(config.embeddings.openai_api_key).toBe("sk-test-123"); + }); + + it("parses PREFLIGHT_RELATED env var into related_projects", async () => { + mockedExistsSync.mockReturnValue(false); + process.env.PREFLIGHT_RELATED = "/path/to/foo, /path/to/bar"; + + const { getConfig } = await loadFreshConfig(); + const config = getConfig(); + + expect(config.related_projects).toEqual([ + { path: "/path/to/foo", alias: "foo" }, + { path: "/path/to/bar", alias: "bar" }, + ]); + }); + + it("ignores invalid env profile values", async () => { + mockedExistsSync.mockReturnValue(false); + process.env.PROMPT_DISCIPLINE_PROFILE = "invalid_profile"; + + const { getConfig } = await loadFreshConfig(); + const config = getConfig(); + + expect(config.profile).toBe("standard"); // stays default + }); + + it("loads config.yml and merges with defaults", async () => { + const configYaml = ` +profile: full +thresholds: + session_stale_minutes: 60 +`; + mockedExistsSync.mockImplementation((p) => { + const path = String(p); + if (path === join("/fake/project", ".preflight")) return true; + if (path === join("/fake/project", ".preflight", "config.yml")) return true; + return false; + }); + mockedReadFileSync.mockReturnValue(configYaml); + + const { getConfig } = await loadFreshConfig(); + const config = getConfig(); + + expect(config.profile).toBe("full"); + expect(config.thresholds.session_stale_minutes).toBe(60); + // Other thresholds should keep defaults + expect(config.thresholds.max_tool_calls_before_checkpoint).toBe(100); + }); + + it("loads triage.yml and merges with defaults", async () => { + const triageYaml = ` +strictness: strict +rules: + always_check: + - payments + - auth +`; + mockedExistsSync.mockImplementation((p) => { + const path = String(p); + if (path === join("/fake/project", ".preflight")) return true; + if (path === join("/fake/project", ".preflight", "triage.yml")) return true; + return false; + }); + mockedReadFileSync.mockReturnValue(triageYaml); + + const { getConfig } = await loadFreshConfig(); + const config = getConfig(); + + expect(config.triage.strictness).toBe("strict"); + expect(config.triage.rules.always_check).toEqual(["payments", "auth"]); + }); + + it("ignores env vars when .preflight/ directory exists", async () => { + mockedExistsSync.mockImplementation((p) => { + const path = String(p); + if (path === join("/fake/project", ".preflight")) return true; + return false; + }); + process.env.PROMPT_DISCIPLINE_PROFILE = "full"; + + const { getConfig } = await loadFreshConfig(); + const config = getConfig(); + + // Should NOT apply env var since .preflight/ exists + expect(config.profile).toBe("standard"); + }); + + it("caches config on second call (singleton)", async () => { + mockedExistsSync.mockReturnValue(false); + + const { getConfig } = await loadFreshConfig(); + const config1 = getConfig(); + const config2 = getConfig(); + + expect(config1).toBe(config2); // same reference + }); + + it("handles malformed config.yml gracefully", async () => { + mockedExistsSync.mockImplementation((p) => { + const path = String(p); + if (path === join("/fake/project", ".preflight")) return true; + if (path === join("/fake/project", ".preflight", "config.yml")) return true; + return false; + }); + mockedReadFileSync.mockImplementation(() => { throw new Error("read error"); }); + + const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {}); + const { getConfig } = await loadFreshConfig(); + const config = getConfig(); + + // Should fall back to defaults + expect(config.profile).toBe("standard"); + warnSpy.mockRestore(); + }); + + it("hasPreflightConfig returns true when .preflight/ exists", async () => { + mockedExistsSync.mockImplementation((p) => { + return String(p) === join("/fake/project", ".preflight"); + }); + + const { hasPreflightConfig } = await loadFreshConfig(); + expect(hasPreflightConfig()).toBe(true); + }); + + it("hasPreflightConfig returns false when .preflight/ missing", async () => { + mockedExistsSync.mockReturnValue(false); + + const { hasPreflightConfig } = await loadFreshConfig(); + expect(hasPreflightConfig()).toBe(false); + }); + + it("getRelatedProjects returns path array", async () => { + const configYaml = ` +related_projects: + - path: /srv/api + alias: api + - path: /srv/web + alias: web +`; + mockedExistsSync.mockImplementation((p) => { + const path = String(p); + if (path === join("/fake/project", ".preflight")) return true; + if (path === join("/fake/project", ".preflight", "config.yml")) return true; + return false; + }); + mockedReadFileSync.mockReturnValue(configYaml); + + const { getRelatedProjects } = await loadFreshConfig(); + expect(getRelatedProjects()).toEqual(["/srv/api", "/srv/web"]); + }); +});