diff --git a/.gitignore b/.gitignore index 27f9536..410acdb 100644 --- a/.gitignore +++ b/.gitignore @@ -29,6 +29,9 @@ docs/AUDIT-PROMPT.md docs/ARCHITECTURE.md docs/ROADMAP.md docs/TOOL-AUDIT.md +docs/LOOP-ENGINEERING.md +docs/VERIFICATION.md +docs/VISION-CHECKLIST.md # Internal launch/marketing working files — never publish marketing/ diff --git a/CLAUDE.md b/CLAUDE.md index 67ac3a6..2bc3fdd 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -27,7 +27,7 @@ Adherence gate: close-the-loop writes (`knitbrain_record_learning`, `knitbrain_s **High-fanout (change carefully):** `src/ccr/store.ts`, `src/tokenizer.ts`, `src/engine/feedback.ts`, `src/engine/memory.ts`, `src/engine/knowledge.ts` — check `knitbrain_query_dependents` before touching. **Largest:** `src/mcp/tools.ts`, `scripts/production-audit.mjs`, `src/learn.ts` -Full architecture map + audit procedure: `docs/AUDIT-PROMPT.md`. +System design (processes, state, flows, failure domains): `docs/ARCHITECTURE.md` · Binding product map (gaps, kill-list, UX law, build order, platform ledger): `docs/LOOP-ENGINEERING.md` (both internal, gitignored). --- diff --git a/README.md b/README.md index a897c53..a8f717f 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@

knitbrain

-

The local-first brain for coding agents — retrieval, lossless compression, persistent memory, and a verify-gated closed loop, in one MCP server.

+

The substrate for agent loops — state, optimization, and enforcement in one local-first MCP server. The loop that can't lie, can't run away, and can't go broke.

npm version @@ -11,25 +11,32 @@

Quick start · - What it does · + Three legs · + Loops · + Receipt · Numbers · - How it works · - Comparison · + Platforms · Commands · Guarantees

--- -Coding agents burn most of their tokens on context — files they didn't need, logs resent every -turn, knowledge re-derived every session. knitbrain is a local-first **MCP server (37 tools)** -that attacks all three leaks at once, for any agent that speaks MCP: Claude Code, Cursor, Copilot, -Codex, Windsurf, Cline, and others. +Everyone is wiring coding agents into loops — goal in, iterate until done. Every loop fails the +same three ways: the agent **lies** ("done!" with red tests), it **runs away** (iteration 47, +nothing converging), and it **goes broke** (context resent and re-derived until the window or the +bill gives out). -- **Send less** — a retrieval layer returns ranked, score-gated function-level chunks instead of whole files. -- **Shrink what's sent, losslessly** — large tool output collapses to a skeleton with a `⟨recall:hash⟩` handle; the exact original is always one call away. -- **Stop re-deriving** — per-project memory, a knowledge graph, and a compounding wiki survive `/clear`, restarts, and even switching tools. -- **Finish what you start** — a closed loop where "done" means your verify command exited 0, never the model's opinion. +knitbrain is the substrate those loops run on — a local-first **MCP server (37 tools)** plus a +hook layer for Claude Code, Codex CLI, Cursor, Gemini CLI, and VS Code Copilot: + +- **Can't lie** — "done" means *your* verify command exited 0. The loop's gate is a real process + exit code, never the model's opinion. Hooks block the agent from stopping while the goal is unmet. +- **Can't run away** — every loop carries hard breaks: max iterations, wall-clock deadline, and a + per-cycle failure history injected into the next attempt so it converges instead of thrashing. +- **Can't go broke** — lossless compression (byte-exact recall, never-expanding), function-level + retrieval instead of whole files, persistent memory instead of re-derivation — and a session + receipt that shows exactly what was saved and where. Pure Node, three runtime dependencies, no Python, no ML runtime. Everything lives under `~/.knitbrain`; the proxy, hub, and dashboard bind `127.0.0.1`. Nothing leaves your machine. @@ -39,57 +46,94 @@ Pure Node, three runtime dependencies, no Python, no ML runtime. Everything live ```bash npx knitbrain profile # 1. measure compression on YOUR transcripts — see the number first npm install -g knitbrain # 2. install -knitbrain setup # 3. wire into your agent(s): MCP config, rules, hooks, slash commands +knitbrain setup # 3. wire into your agent(s): MCP config, hooks, rules, slash commands +knitbrain onboard # 4. scan the repo + import past sessions into the brain ``` -Then open your agent in a project and say **"onboard this project"** — a 5-question interview -writes a Project Charter, a per-part workflow, and a loop-ready `goal.md`. Requires Node ≥ 18. - -## What it does - -### 1. Retrieval — send only the code that matters - -`knitbrain_search_code` turns "find the auth middleware" into ranked function/class-level chunks: -name- and signature-boosted keyword ranking, knowledge-graph expansion (what imports / depends on -each hit), and an adaptive score gate — an empty result beats confidently-wrong context. The agent -reads only the hits, not the tree. +Then open your agent and answer the 5-question interview (or just say **"onboard this project"**) — +it writes a Project Charter, a per-part workflow, and a loop-ready `goal.md`. From that point, +stating a goal in plain words is enough: the ambient frame turns it into a verify-gated loop. +Requires Node ≥ 18. -### 2. Compression — lossless, never-expanding +## The three legs -Tool results (code, logs, diffs, JSON, prose) route through a structure-preserving skeletonizer -(tree-sitter AST + deterministic handlers). The exact original lands in a content-addressed recall -store; the agent sees a skeleton plus a `⟨recall:hash⟩` handle and pages the original back on -demand. Small or incompressible payloads pass through untouched. JSON tool responses are never -skeletonized — machine contracts stay parseable. - -### 3. Memory — one brain, every session, every tool +### STATE — one brain, every session, every tool Learnings ranked by outcome (a learning reported wrong is discredited and sinks), an imports/exports/dependents knowledge graph that re-scans itself on read, session handoffs that -survive `/clear`, and a small interlinked wiki the agent maintains instead of re-reading the -codebase every morning. The same brain serves every MCP tool you use — explain the project once, -Cursor inherits what Claude Code learned. - -### 4. The closed loop — goal until verified done +survive `/clear`, and a compounding wiki. Onboarding scans your **whole toolkit** — skills, agents, +commands, hooks, across project, global, and plugin tiers — and composes a standing workflow (GOAL, +VERIFY, CONSTRAINTS, per-part ROUTING) that re-surfaces every session. The same brain serves every +MCP client: explain the project once, Cursor inherits what Claude Code learned. + +### OPTIMIZATION — lossless, measured, felt + +- **Retrieval**: `knitbrain_search_code` returns ranked, score-gated function-level chunks with + graph context — the agent reads hits, not trees. +- **Compression**: large tool output collapses to a structure-preserving skeleton plus a + `⟨recall:hash⟩` handle; the exact original is content-addressed on disk and one call away. + Small or incompressible payloads pass through untouched. JSON tool responses are never + skeletonized — machine contracts stay parseable. +- **Attribution**: every optimization event — MCP tool, hook, or proxy — lands in one ledger, so + the session receipt can tell you which door saved what. + +### ENFORCEMENT — the workflow is not advice + +All five major agent platforms now ship hook systems. knitbrain's one hook binary auto-detects the +calling platform from the payload itself and speaks its dialect: + +| Enforcement | Claude Code | Codex CLI | Cursor | Gemini CLI | VS Code Copilot | +|---|---|---|---|---|---| +| Deny a violating tool call | ✅ | ✅ | ✅ | ✅ | ✅ | +| Block stop while goal unmet | ✅ | ✅ reason becomes next prompt | ➖ follow-up injection | ✅ deny + auto-retry | ✅ | +| Inject the goal frame | ✅ every prompt | ✅ every prompt | session start only | ✅ | ✅ | +| Rewrite oversized reads | ✅ | context pointer | ✅ (MCP outputs) | context pointer | ✅ | + +Your Project Charter's CONSTRAINTS line is enforced *physically*: write "NEVER npm publish without +OK" during onboarding and the PreToolUse hook denies `npm publish` at the tool boundary — on every +platform above. The differences in the table are each host's documented API ceilings, stated +honestly, not gaps we hide. + +## Loops + +One engine, three ways in: + +- **Ambient** (after onboarding): say what you're working on; the injected frame classifies it — + actionable requests become goals driven through `knitbrain_run_loop` until the verify gate + passes; questions get answered directly. +- **`/goal `** — loop until met. **`/loop --for 2h --iters N`** — same engine + plus a wall-clock and iteration budget. Every iteration runs as a full goal cycle. +- **Headless**: `knitbrain loop goal.md` drives a checkbox goal file outside any editor — survives + laptop-close, ticks boxes only on green verify. `knitbrain fan` runs N workers in parallel, each + in its own git worktree, draining the same queue. + +**Self-healing:** each failed cycle's verify output is persisted (`failures[]`, last 3) and +injected into the next directive — "previous failures — iter 1: …. Address the ROOT CAUSE" — so +loops converge in fewer iterations without shortcuts. An adherence gate blocks memory writes until +a task was classified: unverified "done" cannot enter the brain. + +## The receipt + +Optimization you can't see is optimization you don't trust. When a session ends, the Stop hook +prints an honest receipt (also available mid-session via `/meter`): ``` -goal → judge → iterate → grade (your verify command, exit 0 or not) → review → repeat +— knitbrain session receipt — +consumed ~281k tok · avoided 16.0k tok (5% of what would have been) +top sinks: + Bash: 10.0k → 2.0k tok (saved 8.0k) + request: 9.0k → 6.0k tok (saved 3.0k) + src/big.ts: 6.0k → 1.0k tok (saved 5.0k) +hygiene: + re-read unchanged ×2: /proj/dup.ts + 1 oversized raw read(s) redirected to knitbrain_read +lifetime: 141.7k tok saved · 394 exact recalls ``` -- **In your agent:** `knitbrain_run_loop` runs the verify gate each cycle and hands back "not met — - smallest fix, go again" until it passes. An adherence gate blocks memory writes until a task was - classified — unverified "done" cannot enter the brain. -- **Around an agent:** `knitbrain loop | fan | orchestrate` drive a checkbox goal file headlessly — - `fan` runs N workers in isolated git worktrees. The loop never commits, pushes, or deploys. - -### 5. Workflow — every part of the project has an owner - -Onboarding scans your existing skills, agents, and plugins (project + global), asks what's missing, -scaffolds scoped agents for uncovered parts, and bakes a standing workflow — GOAL, VERIFY, -CONSTRAINTS, TOOLKIT, per-part ROUTING — that re-surfaces verbatim every session. A deterministic -classifier sizes each task (inquiry → trivial → standard → complex) and routes plan-mode for -complex work. `knitbrain_self_check` audits seven invariants (anti-stale, anti-drift, -anti-sycophancy, adherence, context-hygiene…) in one pass. +Honest-math rules, enforced structurally: tokens count as "saved" only when a raw output actually +existed and was replaced or redirected — redirects themselves record zero (the follow-up read +counts once). Estimates are labeled estimates. A session with no savings says so plainly instead +of inventing a number. ## Measured, not promised @@ -101,10 +145,12 @@ Run these on your own data — every number below is reproducible with one comma | Weighted real-shape benchmark (code · logs · JSON · diffs · prose) | 68% | `npm run bench` | | Answer preservation (round-trip · identifiers · error/summary lines) | 100% | `knitbrain evals` | -These are the **ceiling** — what you save when output flows through the optimizer. Your **realized** -number is the live meter (`knitbrain dashboard`), which counts only what actually passed through. -Honest expectations: 60–70% on code/JSON/logs, ~18% on prose, ~48% all-inclusive on measured real -sessions — less inside an already-lean harness, more on raw API traffic. +These are the **ceiling** — what you save when output flows through the optimizer. Your +**realized** number is the receipt and the live meter (`knitbrain dashboard`), which count only +what actually passed through. Honest expectations: 60–70% on code/JSON/logs, ~18% on prose, ~48% +all-inclusive on measured real sessions — less inside an already-lean harness, more on raw API +traffic. And honestly: per-request optimization cannot offset provider cache-cold re-reads or +subagent spawns — the meter warns you when a handoff + fresh session is the cheaper move. ## How it reaches your traffic @@ -115,53 +161,43 @@ The optimizer is identical everywhere; what differs is reach: moved to a marked tail), detects the model's context window, and can inject a terse-output directive (`KNITBRAIN_TERSE=1`). - **Subscription (OAuth)** — the wire can't be intercepted (true for every tool in this space), so - knitbrain works through the MCP + hook surface instead: `knitbrain_read` for files, and on Claude - Code a PostToolUse hook that skeletonizes Bash/Grep/Glob/WebFetch output in place. Assistant - prose lands on disk in the host's transcripts — SessionStart mines new ones into the brain - automatically. - -## How it compares - -Token burn has three taps: **input** (what the model reads), **output** (what it writes), and -**memory** (what gets re-derived every session). Single-tap tools close one; knitbrain closes all -three from one install — and states plainly where the standard criticisms apply: - -| Common criticism of single-tap tools | knitbrain's answer | -|---|---| -| "Compression retrieval is a second call; small payloads can cost more" | **Never-expand is build-gated**: small/incompressible payloads pass through. Retrieval is on-demand, not speculative. | -| "Compressing context breaks the provider's cache discount" | The **CacheAligner** exists for exactly this — stable prefix bytes, `cache_control` added only when the client set none. | -| "Lossy compression makes models confidently wrong" | Compression here is **lossless** — 100% round-trip, 100% identifier fidelity, error lines never elided, gated in CI, byte-for-byte recovery always available. | -| "Inside a lean harness there's little left to squeeze" | True, and measured honestly — that's why the MCP + hook path exists (works *inside* Claude Code) and why `knitbrain profile` measures **your** transcripts first. | -| "Terse output degrades multi-turn quality" | Terse mode never drops technical content, numbers, paths, or decision-changing caveats — and it's opt-in at every layer. | -| "The real fix is discipline, not tools" | Agreed — that's the third tap: tier routing, verify-before-done, self-check invariants, and memory that stops the most expensive burn of all: re-deriving context. | - -Percentages from different taps overlap the same bill; they add, they don't multiply. + knitbrain works through the MCP + hook surface instead: `knitbrain_read` for files, PreToolUse + redirecting oversized raw reads, and PostToolUse skeletonizing Bash/Grep/Glob/WebFetch output in + place. Assistant prose lands in the host's transcripts — SessionStart mines new ones into the + brain automatically. ## Platform support -| Platform | MCP tools | Auto-compression | Slash commands | Notes | +| Platform | MCP tools | Hook enforcement | Auto-compression | Slash commands | |---|---|---|---|---| -| Claude Code | ✅ | ✅ hooks (deepest) | `/goal` `/meter` `/handoff` `/terse` | full lifecycle hooks | -| Cursor · Windsurf · Cline | ✅ | via `knitbrain_read` | — | native config written by `setup` | -| Copilot (VS Code + CLI) | ✅ | via `knitbrain_read` | — | `.vscode/mcp.json` + `.github/instructions` | -| Codex and any MCP client | ✅ | via `knitbrain_read` | — | one universal server entry | -| Any agent, API key | ✅ | ✅ proxy (full wire) | — | `knitbrain wrap ` | +| Claude Code | ✅ | ✅ full (deny · stop-block · inject · rewrite) | ✅ hooks | `/goal` `/loop` `/meter` `/handoff` `/terse` | +| Codex CLI | ✅ | ✅ full (`.codex/hooks.json`) | hooks + `knitbrain_read` | — | +| Cursor | ✅ | ✅ deny + follow-up loop (`.cursor/hooks.json`) | hooks + `knitbrain_read` | — | +| Gemini CLI | ✅ | ✅ deny + AfterAgent loop (`.gemini/settings.json`) | hooks + `knitbrain_read` | — | +| VS Code Copilot | ✅ | ✅ full (reads `.claude/settings.json` natively) | hooks + `knitbrain_read` | — | +| Windsurf · Cline · any MCP client | ✅ | — (advisory; hooks planned where APIs allow) | via `knitbrain_read` | — | +| Any agent, API key | ✅ | — | ✅ proxy (full wire) | — | + +One hook binary serves every row: it detects the calling platform from the payload and answers in +that host's schema. Where a host's API can't do something (Cursor can't block stop; Gemini can't +rewrite output), knitbrain degrades to the nearest honest mechanism instead of claiming otherwise. ## Commands | Command | What it does | |---|---| | `knitbrain` *(no args)* | Start the MCP server on stdio — what your editor invokes. | -| `knitbrain setup` | Wire into your agent(s): MCP config, rules, hooks, slash commands, `AGENTS.md`. | +| `knitbrain setup` | Wire into your agent(s): MCP config, hooks, rules, slash commands, `AGENTS.md`. | +| `knitbrain onboard` | Scan the repo + import past sessions into the brain; start the charter interview. | | `knitbrain profile` | Measure compression on your real transcripts. | | `knitbrain evals` | Answer-preservation gates on your transcripts (exit 1 on failure). | -| `knitbrain orchestrate ` | The closed loop: judge → iterate → grade → review, verify-gated. | -| `knitbrain loop ` | Single-worker loop over a checkbox goal file. | +| `knitbrain loop ` | Headless verify-gated loop over a checkbox goal file. | | `knitbrain fan ` | Parallel loop — N workers in isolated git worktrees. | | `knitbrain dashboard` | Live local dashboard (`127.0.0.1:8790`): meter, graph, wiki, activity, plan usage. | | `knitbrain wrap ` | Launch an agent through the optimizer proxy (API-key setups). | | `knitbrain compress ` | Terse-rewrite a memory file (e.g. `CLAUDE.md`); keeps a backup. | | `knitbrain learn` | Mine past sessions for failure → success corrections. | +| `knitbrain terse [level]` | Print the terse-output guide (lite / full / ultra). | | `knitbrain hub` / `join` | Optional team hub — shared findings over one URL and token. | | `knitbrain statusline` | Tokens-saved badge for your editor's status line. | | `knitbrain prompt` | Print the operating prompt (for non-MCP platforms). | @@ -174,9 +210,11 @@ Gated by tests and CI, not promised: - **Never-expand** — output tokens ≤ input tokens, always. - **Answers survive** — error lines, result summaries, and top-level declarations are never elided (`knitbrain evals`, 100% on real transcripts). - **Machine contracts hold** — JSON tool responses are never skeletonized. -- **No false green** — the loop marks a task done only after a real verify passes. +- **No false green** — the loop marks a task done only after a real verify passes; hooks block premature stops. +- **Honest receipt** — savings are counted only when a raw output was actually replaced or redirected; estimates are labeled; zero is reported as zero. - **Local-first** — proxy, hub, and dashboard bind `127.0.0.1`; credentials are read locally, sent only to the provider's own endpoint, never logged or stored. - **Reproducible** — every number in this README comes from a command you can run on your own data. +- **Self-audited** — `knitbrain_self_check` runs seven invariants (anti-stale ×2, anti-drift ×2, anti-sycophancy, adherence, context-hygiene) in one pass. Two integration notes worth knowing up front: diff --git a/package.json b/package.json index 8eaa77b..c132c07 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "knitbrain", "version": "0.17.0", - "description": "Local-first MCP brain for coding agents: code retrieval, lossless context compression, persistent per-project memory, and a verify-gated closed loop — for Claude Code, Cursor, Copilot, and any MCP client.", + "description": "Loop engineering substrate for coding agents — state, lossless optimization, and hook enforcement: verify-gated loops that can't lie, run away, or go broke. For Claude Code, Codex, Cursor, Gemini CLI, VS Code Copilot, and any MCP client.", "type": "module", "main": "dist/lib.js", "types": "dist/lib.d.ts", @@ -30,6 +30,12 @@ "knowledge-graph", "code-search", "retrieval", + "loop-engineering", + "agent-loop", + "hooks", + "enforcement", + "codex", + "gemini-cli", "token-optimization", "context-engineering", "prompt-compression", diff --git a/scripts/bench.ts b/scripts/bench.ts index e30b1ba..8459249 100644 --- a/scripts/bench.ts +++ b/scripts/bench.ts @@ -22,7 +22,7 @@ import { mkdtempSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { activeTokenizerName } from "../src/tokenizer.js"; -import { measure, summarize, type Measurement } from "../src/measure.js"; +import { measure, summarize, type Measurement } from "./measure.js"; import { createFileCCRStore } from "../src/ccr/store.js"; import { compressJson } from "../src/optimizer/json.js"; import { compressCode } from "../src/optimizer/code.js"; diff --git a/src/measure.ts b/scripts/measure.ts similarity index 97% rename from src/measure.ts rename to scripts/measure.ts index 37acccc..ea9ce36 100644 --- a/src/measure.ts +++ b/scripts/measure.ts @@ -1,4 +1,4 @@ -import { countTokens } from "./tokenizer.js"; +import { countTokens } from "../src/tokenizer.js"; /** Result of measuring a single original→optimized payload pair. */ export interface Measurement { diff --git a/scripts/research-measure.mjs b/scripts/research-measure.mjs deleted file mode 100644 index a19bb5b..0000000 --- a/scripts/research-measure.mjs +++ /dev/null @@ -1,20 +0,0 @@ -/** - * Research worker — ONE measurement in a FRESH process, so WASM/heap state - * never accumulates across experiments (the autoresearch-faithful design: - * each experiment is isolated). Parameter overrides arrive via the KNITBRAIN_* - * env vars that src/optimizer/params.ts already reads at load. Prints a single - * JSON line: {savings, pass, blocks}. Run by scripts/research.mjs. - */ -import { join, dirname } from "node:path"; -import { fileURLToPath, pathToFileURL } from "node:url"; - -const ROOT = join(dirname(fileURLToPath(import.meta.url)), ".."); -const distUrl = (p) => pathToFileURL(join(ROOT, "dist", p)).href; -const { runProfile } = await import(distUrl("profile.js")); -const { runEvals } = await import(distUrl("evals.js")); - -const noop = () => {}; -const corpus = process.argv[2] ? [process.argv[2]] : []; -const savings = await runProfile(corpus, noop); -const evals = await runEvals(corpus, noop); -process.stdout.write(JSON.stringify({ savings, pass: evals.pass, blocks: evals.blocks })); diff --git a/scripts/research.mjs b/scripts/research.mjs deleted file mode 100644 index c3de373..0000000 --- a/scripts/research.mjs +++ /dev/null @@ -1,134 +0,0 @@ -/** - * knitbrain research — an autoresearch-style autonomous tuning loop for - * knitbrain's OWN compression heuristics (inspired by karpathy/autoresearch). - * - * The product optimizes the user's agent loop; this optimizes the product. - * Hand-tuned constants (anchor trigger, min-sentences, never-expand floor …) - * are swept against REAL transcripts. The objective is the same hard number a - * user sees — overall savings % from `profile` — under a HARD CONSTRAINT: the - * fidelity gates (`evals`) must still pass. A setting that saves more but - * breaks an answer is auto-discarded, exactly like a crashed experiment in - * autoresearch. No mocks: every measurement runs on your own ~/.claude - * transcripts. - * - * Coordinate descent: from the current defaults, sweep each knob, adopt the - * value that maximizes savings while keeping every gate green, move on. Every - * experiment is logged to experiments.tsv (keep/discard/crash), and the report - * flags slop — a default that's beatable, or a knob with zero effect (dead - * complexity worth deleting). - * - * Run after `npm run build`: node scripts/research.mjs [corpusDir] - * This CHANGES NOTHING on disk except the ledger — it reports winners for you - * to review and apply (the program.md model: the human owns the decision). - */ -import { appendFileSync, writeFileSync } from "node:fs"; -import { join, dirname } from "node:path"; -import { fileURLToPath } from "node:url"; -import { spawnSync } from "node:child_process"; - -const ROOT = join(dirname(fileURLToPath(import.meta.url)), ".."); -const corpus = process.argv[2] ? [process.argv[2]] : []; -const ledger = join(ROOT, "experiments.tsv"); -const WORKER = join(ROOT, "scripts", "research-measure.mjs"); -writeFileSync(ledger, "knob\tvalue\tsavings\tgates\tstatus\tnote\n"); - -// param name → the env var src/optimizer/params.ts reads. -const ENV_OF = { - anchorTriggerPct: "KNITBRAIN_ANCHOR_TRIGGER_PCT", - minSentences: "KNITBRAIN_MIN_SENTENCES", - anchorMinLines: "KNITBRAIN_ANCHOR_MIN_LINES", - minSavingPct: "KNITBRAIN_MIN_SAVING_PCT", -}; -const PARAMS = { anchorTriggerPct: 35, minSentences: 8, anchorMinLines: 40, minSavingPct: 5 }; - -/** - * One real measurement, in a FRESH child process (zero WASM accumulation - * across experiments). `params` overrides ride the KNITBRAIN_* env vars. - */ -function measure(params = {}) { - const env = { ...process.env }; - for (const [k, v] of Object.entries(params)) env[ENV_OF[k]] = String(v); - const r = spawnSync(process.execPath, [WORKER, ...(corpus[0] ? [corpus[0]] : [])], { - env, - encoding: "utf8", - maxBuffer: 128 << 20, - }); - const line = (r.stdout || "").trim().split("\n").filter(Boolean).pop(); - if (!line) throw new Error(`worker produced no output${r.status ? ` (exit ${r.status})` : ""}`); - return JSON.parse(line); -} - -const log = (knob, value, savings, pass, status, note) => { - const line = `${knob}\t${value}\t${savings.toFixed(2)}\t${pass ? "PASS" : "FAIL"}\t${status}\t${note}`; - console.log(` ${status.padEnd(8)} ${knob}=${value} savings=${savings.toFixed(2)}% gates=${pass ? "green" : "RED"} ${note}`); - appendFileSync(ledger, line + "\n"); -}; - -// The knobs and the grid to sweep each over (defaults included so the baseline -// is always represented; small grids keep the loop to ~minutes). -const KNOBS = [ - { name: "anchorTriggerPct", grid: [25, 30, 35, 40, 45] }, - { name: "minSentences", grid: [6, 7, 8, 10, 12] }, - { name: "anchorMinLines", grid: [30, 40, 50] }, - { name: "minSavingPct", grid: [3, 5, 8] }, -]; - -console.log(`[research] corpus: ${corpus[0] ?? "~/.claude/projects (default)"}`); -console.log("[research] baseline (current hand-tuned defaults)…"); -const baseline = measure(); -console.log(`[research] baseline: savings=${baseline.savings.toFixed(2)}% gates=${baseline.pass ? "green" : "RED"} blocks=${baseline.blocks}`); -if (!baseline.pass) { - console.error("[research] baseline gates are RED — fix fidelity before tuning. Aborting."); - process.exit(1); -} - -let bestOverall = baseline.savings; -const adopted = {}; -const slop = []; - -for (const knob of KNOBS) { - const original = PARAMS[knob.name]; - console.log(`\n[research] sweeping ${knob.name} (default ${original})…`); - let bestVal = original; - let bestSavings = bestOverall; - const effects = new Set(); - for (const value of knob.grid) { - let m; - try { - // Carry adopted winners forward (coordinate descent), each run isolated. - m = measure({ ...adopted, [knob.name]: value }); - } catch (err) { - log(knob.name, value, 0, false, "crash", String(err).slice(0, 60)); - continue; - } - effects.add(m.savings.toFixed(2)); - // KEEP only if it beats the running best AND keeps every gate green. - const better = m.pass && m.savings > bestSavings + 0.01; - log(knob.name, value, m.savings, m.pass, better ? "keep" : "discard", value === original ? "(default)" : !m.pass ? "gate broke" : better ? "improves" : "no gain"); - if (better) { - bestSavings = m.savings; - bestVal = value; - } - } - // Adopt the winner for this knob (coordinate descent), carry it forward. - adopted[knob.name] = bestVal; - bestOverall = bestSavings; - // SLOP FLAG: a knob whose every grid value produced the SAME savings has no - // effect on this corpus — it's dead complexity worth questioning. - if (effects.size === 1) slop.push(`${knob.name}: no measurable effect across ${knob.grid.join("/")} — candidate for removal`); - if (bestVal !== original) slop.push(`${knob.name}: default ${original} beaten by ${bestVal} (+${(bestSavings - baseline.savings).toFixed(2)}pp)`); -} - -console.log("\n[research] ── RESULT ──"); -console.log(`baseline savings: ${baseline.savings.toFixed(2)}%`); -console.log(`best found: ${bestOverall.toFixed(2)}% (+${(bestOverall - baseline.savings).toFixed(2)}pp)`); -console.log(`adopted knobs: ${JSON.stringify(adopted)}`); -if (slop.length === 0) { - console.log("verdict: hand-tuned defaults are already optimal on this corpus (no slop, nothing to change)."); -} else { - console.log("findings (review, then apply by editing src/optimizer/params.ts defaults):"); - for (const s of slop) console.log(` • ${s}`); -} -console.log(`ledger: ${ledger}`); -// This tool reports only — it never writes source. Apply a winner by editing -// the default in src/optimizer/params.ts, then re-run the gates. diff --git a/scripts/shape-profile.mjs b/scripts/shape-profile.mjs deleted file mode 100644 index d8036a3..0000000 --- a/scripts/shape-profile.mjs +++ /dev/null @@ -1,125 +0,0 @@ -/** - * Shape profiler — answers "where does context burn ACTUALLY live?" with data. - * - * Scans real host transcripts (Claude Code JSONL), buckets every sizable - * tool_result by shape, and reports per shape: count, tokens, and what the - * CURRENT optimizer already saves. The under-served heavy buckets are the - * next handlers to build — measurement decides, not guesswork. - * - * Usage: node scripts/shape-profile.mjs [more...] - * (defaults to ~/.claude/projects — all projects, all sessions) - */ -import { createReadStream, readdirSync, statSync, mkdtempSync, rmSync } from "node:fs"; -import { createInterface } from "node:readline"; -import { homedir, tmpdir } from "node:os"; -import { join, dirname } from "node:path"; -import { fileURLToPath, pathToFileURL } from "node:url"; - -const ROOT = join(dirname(fileURLToPath(import.meta.url)), ".."); -const d = (p) => pathToFileURL(join(ROOT, "dist", p)).href; -const { createFileCCRStore, sha256 } = await import(d("ccr/store.js")); -const { compress } = await import(d("optimizer/router.js")); -const { countTokens } = await import(d("tokenizer.js")); -const { ensureAst, astReady } = await import(d("optimizer/ast.js")); -await ensureAst(); -console.log(`[profile] AST parsers: ${astReady() ? "warm" : "FAILED — scanner fallback"}`); - - -// ── shape classification (deterministic, order matters) ── -function dupRatio(lines) { - const uniq = new Set(lines).size; - return lines.length === 0 ? 0 : 1 - uniq / lines.length; -} -function classifyShape(t) { - const head = t.slice(0, 200).trimStart(); - const lines = t.split("\n"); - if (head.startsWith("= lines.length * 0.6) return "numbered-read"; - if (/^diff --git|^@@ |\n@@ /.test(t) || (/^--- /m.test(t) && /^\+\+\+ /m.test(t))) return "diff"; - if (/\b(\d+ (passing|passed|failed)|Tests:|Test Files|PASS|FAIL|✓|✗)\b/.test(t) && lines.length > 15) return "test-output"; - if (dupRatio(lines) > 0.25 && lines.length >= 20) return "repetitive-log"; - if (/[{};]/.test(t) && /\b(function|const|class|import|export|def|return)\b/.test(t)) return "code"; - if (lines.length > 40) return "long-prose"; - return "short-prose"; -} - -// ── collect transcripts ── -const args = process.argv.slice(2); -const roots = args.length > 0 ? args : [join(homedir(), ".claude", "projects")]; -const files = []; -for (const r of roots) { - const st = statSync(r); - if (st.isFile()) { files.push(r); continue; } - for (const proj of readdirSync(r)) { - const pd = join(r, proj); - try { - if (!statSync(pd).isDirectory()) continue; - for (const f of readdirSync(pd)) if (f.endsWith(".jsonl")) files.push(join(pd, f)); - } catch { /* skip */ } - } -} -console.log(`[profile] transcripts: ${files.length}`); - -const store = mkdtempSync(join(tmpdir(), "kb-profile-")); -const ccr = createFileCCRStore(store); -const buckets = new Map(); // shape → {n, before, after, sample} - -let dedupN = 0; -let dedupSaved = 0; - -for (const file of files) { - // Cross-turn dedup is per session: every request re-sends the history, so a - // block whose exact text already appeared earlier in this transcript would - // collapse to a ⟪same as ⟨recall:hash⟩⟫ marker in the proxy. - const seen = new Set(); - const rl = createInterface({ input: createReadStream(file), crlfDelay: Infinity }); - for await (const line of rl) { - let msg; - try { msg = JSON.parse(line); } catch { continue; } - const content = msg?.message?.content; - if (!Array.isArray(content)) continue; - for (const block of content) { - if (block?.type !== "tool_result") continue; - const texts = typeof block.content === "string" ? [block.content] - : Array.isArray(block.content) ? block.content.filter((c) => c?.type === "text").map((c) => c.text) : []; - for (const t of texts) { - if (typeof t !== "string" || t.length < 400) continue; - const shape = classifyShape(t); - const hash = sha256(t); - const repeat = seen.has(hash); - seen.add(hash); - const r = compress(t, ccr); - let after = r.skeletonTokens; - if (repeat) { - const marker = countTokens(`⟪same as earlier ⟨recall:${hash}⟩⟫`); - if (marker < after) { - dedupN++; dedupSaved += after - marker; - after = marker; - } - } - const b = buckets.get(shape) ?? { n: 0, before: 0, after: 0 }; - b.n++; b.before += r.originalTokens; b.after += after; - buckets.set(shape, b); - } - } - } -} -rmSync(store, { recursive: true, force: true }); - -const rows = [...buckets.entries()].sort((a, b) => b[1].before - a[1].before); -const totB = rows.reduce((s, [, v]) => s + v.before, 0); -const totA = rows.reduce((s, [, v]) => s + v.after, 0); -console.log("\nshape n tokens %of-burn saved-now"); -for (const [shape, v] of rows) { - const saved = v.before ? Math.round((1 - v.after / v.before) * 1000) / 10 : 0; - console.log( - `${shape.padEnd(15)} ${String(v.n).padStart(5)} ${String(v.before).padStart(10)} ${String(Math.round((v.before / totB) * 100)).padStart(8)}% ${String(saved).padStart(9)}%`, - ); -} -console.log(`\ncross-turn dedup: ${dedupN} repeated blocks, ${dedupSaved} extra tokens saved`); -console.log(`\nTOTAL ${totB} → ${totA} tokens overall saved=${Math.round((1 - totA / totB) * 1000) / 10}%`); diff --git a/src/engine/receipt.ts b/src/engine/receipt.ts index 00b782d..ae416e0 100644 --- a/src/engine/receipt.ts +++ b/src/engine/receipt.ts @@ -257,6 +257,22 @@ export function buildReceipt(i: ReceiptInput): string { } } + // G3 cold-restart waste: the sink optimization can't touch. The provider's + // prompt cache expires after ~5min idle — every gap >5min in the session's + // event stream means the NEXT turn re-read the whole context uncached. + // Estimated from event timestamps (labeled estimate; skipped when unknowable). + if (mark && events.length >= 2) { + const CACHE_TTL_MS = 5 * 60_000; + const ts = events.map((e) => Date.parse(e.ts)).filter((t) => Number.isFinite(t)).sort((a, b) => a - b); + let coldGaps = 0; + for (let j = 1; j < ts.length; j += 1) if (ts[j]! - ts[j - 1]! > CACHE_TTL_MS) coldGaps += 1; + if (coldGaps > 0 && meter.usedTokens > 0) { + lines.push( + `${coldGaps} idle gap(s) >5min — each next turn re-read ~${fmtTokens(meter.usedTokens)} tok uncached (estimate); a handoff + fresh session is cheaper than a cold return`, + ); + } + } + lines.push(`lifetime: ${fmtTokens(meter.savedTokens)} tok saved · ${retrievalsTotal} exact recalls`); if (eventsTrimmed) { diff --git a/src/fan.ts b/src/fan.ts index 6ec3ac1..125a2fa 100644 --- a/src/fan.ts +++ b/src/fan.ts @@ -1,3 +1,7 @@ +// KEEP ruling (kill-list note): the "own worktree manager" kill targets a +// worktree SERVICE. fan does not manage worktrees as a product surface — it +// uses git's own `git worktree` per parallel task, exactly the coordination +// the loop-engineering doc endorses. fan is the parallelism leg of the loop. import { spawn, spawnSync } from "node:child_process"; import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; import { join } from "node:path"; diff --git a/src/index.ts b/src/index.ts index 06196b3..6f54f36 100644 --- a/src/index.ts +++ b/src/index.ts @@ -9,7 +9,7 @@ import { VERSION } from "./version.js"; const KNOWN_COMMANDS = new Set([ "version", "-v", "--version", "help", "-h", "--help", "setup", "hub", "join", "profile", "wrap", "prompt", "statusline", - "terse", "evals", "compress", "loop", "fan", "orchestrate", "learn", + "terse", "evals", "compress", "loop", "fan", "learn", "onboard", "dashboard", ]); @@ -142,10 +142,6 @@ usage: knitbrain const { runFan } = await import("./fan.js"); process.exit(await runFan(process.argv.slice(3))); } - if (process.argv[2] === "orchestrate") { - const { runOrchestrate } = await import("./orchestrate.js"); - process.exit(runOrchestrate(process.argv.slice(3))); - } if (process.argv[2] === "learn") { const { runLearn } = await import("./learn.js"); await runLearn(process.argv.slice(3)); diff --git a/src/orchestrate.ts b/src/orchestrate.ts deleted file mode 100644 index bce04ce..0000000 --- a/src/orchestrate.ts +++ /dev/null @@ -1,125 +0,0 @@ -import { spawnSync } from "node:child_process"; -import { existsSync, readFileSync } from "node:fs"; -import { runClosedLoop, defaultJudge, makeGrade, makeReview } from "./engine/closed-loop.js"; -import { createWikiStore } from "./engine/wiki.js"; -import { createSkillsStore, type SkillsStore } from "./engine/skills.js"; -import { createKnowledge, type Knowledge } from "./engine/knowledge.js"; -import { proposeAgents } from "./engine/agents.js"; -import { classifyTask } from "./engine/workflow.js"; -import { wikiRoot, skillsRoot, knowledgeRoot } from "./paths.js"; -import { currentContextTokens } from "./engine/usage.js"; - -/** - * Plan for one orchestration cycle — orchestration scales with project - * INTENSITY: trivial/standard tasks get the matched skill only; complex tasks - * also get guardrailed agent proposals briefed into the prompt. Pure (stores - * injected) so the prompt composition is testable without spawning an agent. - */ -export interface CyclePlan { - tier: string; - skillName: string | null; - agentNames: string[]; - prompt: string; -} - -export function buildCyclePlan(goalText: string, skills: SkillsStore, knowledge: Knowledge): CyclePlan { - const cls = classifyTask(goalText, []); - const skill = skills.find(goalText); - const agents = cls.tier === "complex" ? proposeAgents(knowledge.listFiles()).slice(0, 4) : []; - - let prompt = `Goal:\n${goalText}\n\n`; - if (skill) { - prompt += `SKILL — ${skill.name}:\n${skill.body}\n`; - if (skill.constraints.length) prompt += `CONSTRAINTS (non-negotiable):\n${skill.constraints.map((c) => `- ${c}`).join("\n")}\n`; - prompt += "\n"; - } - if (agents.length) { - prompt += `AGENTS to orchestrate (complex task — spawn via your host's sub-agent mechanism):\n`; - prompt += agents - .map((a) => `- ${a.name}: scope \`${a.scope}\` · tools ${a.tools.join("/")}${a.reviewGate ? " · REVIEW-GATED" : ""}`) - .join("\n"); - prompt += "\n\n"; - } - prompt += "Make progress, then stop. Do NOT declare the goal met yourself — the loop grades (verify) + reviews. Do NOT commit, push, or deploy."; - - return { tier: cls.tier, skillName: skill?.name ?? null, agentNames: agents.map((a) => a.name), prompt }; -} - -/** - * `knitbrain orchestrate ` — the closed-loop orchestrator (P3). - * goal → judge → iterate → grade → review → repeat until met (or max cap). - * - * Composes the real steps: the agent does the work (iterate), the verify - * command is the grade (real run, no false green), a rubric is the review, the - * wiki gets a per-cycle audit trail, and the live token window is metered. - * Like the outer loop, it NEVER commits/pushes/deploys — the human does that. - */ - -interface Opts { - goalFile?: string; - max: number; - agent: string; - verify: string | null; -} - -function parseArgs(args: string[]): Opts { - const o: Opts = { max: 6, agent: "claude -p", verify: null }; - for (let i = 0; i < args.length; i += 1) { - const a = args[i]; - if (a === "--max") o.max = Math.max(1, Number(args[(i += 1)]) || 6); - else if (a === "--agent") o.agent = args[(i += 1)] ?? o.agent; - else if (a === "--verify") o.verify = args[(i += 1)] ?? null; - else if (a && !a.startsWith("--")) o.goalFile = a; - } - return o; -} - -const run = (cmd: string, input?: string): boolean => - spawnSync(cmd, { shell: true, input, stdio: input === undefined ? "inherit" : ["pipe", "inherit", "inherit"] }).status === 0; - -export function runOrchestrate(args: string[]): number { - const o = parseArgs(args); - if (!o.goalFile || !existsSync(o.goalFile)) { - console.error('usage: knitbrain orchestrate [--max N=6] [--agent "cmd"] [--verify "cmd"]'); - console.error(" goal → judge → iterate → grade → review → repeat until met. Never commits/pushes/deploys."); - return 1; - } - const goalText = readFileSync(o.goalFile, "utf8"); - const verify = o.verify ?? (existsSync("package.json") ? "npm test" : ""); - const wiki = createWikiStore(wikiRoot()); - // Intensity-based orchestration: the classifier + skill + agents drive the prompt. - const skills = createSkillsStore(skillsRoot()); - const knowledge = createKnowledge(process.cwd(), knowledgeRoot()); - - console.log(`[orchestrate] goal: ${o.goalFile} · verify: ${verify || "(none)"} · max ${o.max} cycles`); - - const result = runClosedLoop( - { - judge: () => defaultJudge(goalText, (verify || "").trim().length > 0), - iterate: (iter) => { - const plan = buildCyclePlan(goalText, skills, knowledge); - console.log(`[orchestrate] cycle ${iter}: iterate · tier=${plan.tier} · skill=${plan.skillName ?? "none"} · agents=${plan.agentNames.length}`); - run(o.agent, plan.prompt); - }, - grade: makeGrade(verify, (cmd) => run(cmd)), - review: makeReview(), - onCycle: (c) => { - wiki.log("cycle", `${o.goalFile} · iter ${c.iter} · grade=${c.graded.pass} · met=${c.met} · tokens=${c.tokens ?? "?"}`); - console.log(`[orchestrate] cycle ${c.iter}: grade=${c.graded.pass} review=${c.reviewed.notes} met=${c.met}${c.tokens != null ? ` · ${c.tokens} ctx tokens` : ""}`); - }, - meter: () => currentContextTokens() ?? 0, - }, - o.max, - ); - - // Synthesize the run into the wiki so the orchestration compounds. - wiki.ingest({ - title: `orchestration ${o.goalFile}`, - kind: "session", - content: `Closed loop on ${o.goalFile}.\n- claim: met = ${result.met}\ncycles: ${result.cycles.length} · ${result.reason}`, - }); - - console.log(`[orchestrate] ${result.met ? "MET ✓" : "NOT met ✗"} — ${result.reason}`); - if (!result.met) console.log("[orchestrate] no false green: goal not marked met. Re-run to continue, or refine the goal."); - return result.met ? 0 : 1; -} diff --git a/tests/orchestrator-polish.test.ts b/tests/orchestrator-polish.test.ts deleted file mode 100644 index 58d4f5e..0000000 --- a/tests/orchestrator-polish.test.ts +++ /dev/null @@ -1,116 +0,0 @@ -import { describe, it, expect, beforeEach, afterEach } from "vitest"; -import { mkdtempSync, rmSync, mkdirSync, writeFileSync, readFileSync } from "node:fs"; -import { tmpdir } from "node:os"; -import { join } from "node:path"; -import { generateAgentMarkdown } from "../src/engine/agents.js"; -import type { StyleProfile } from "../src/engine/host-scan.js"; -import { buildCyclePlan } from "../src/orchestrate.js"; -import { createSkillsStore } from "../src/engine/skills.js"; -import { createKnowledge } from "../src/engine/knowledge.js"; -import { createFileCCRStore } from "../src/ccr/store.js"; -import { createMemory } from "../src/engine/memory.js"; -import { createFeedback } from "../src/engine/feedback.js"; -import { createTeamBoard } from "../src/engine/teams.js"; -import { createMeter } from "../src/engine/meter.js"; -import { createCalibration } from "../src/engine/calibration.js"; -import { TOOLS, dispatch, type ToolContext } from "../src/mcp/tools.js"; - -const style = (over: Partial): StyleProfile => ({ medianBodyLen: 200, terse: false, usesModel: false, usesTriggers: false, headers: [], ...over }); - -// ── (A) agent-frontmatter style-match ── -describe("agent style-match — generateAgentMarkdown honors the StyleProfile", () => { - it("emits `model:` only when the style uses it (with the user's dominant model)", () => { - const withModel = generateAgentMarkdown({ name: "scope-guard", scope: "src/engine" }, style({ usesModel: true, model: "opus" })); - expect(withModel).toMatch(/^model: opus$/m); - - const noModel = generateAgentMarkdown({ name: "scope-guard", scope: "src/engine" }, style({ usesModel: false })); - expect(noModel).not.toMatch(/^model:/m); - - // no style at all → no model line (back-compat) - expect(generateAgentMarkdown({ name: "x" })).not.toMatch(/^model:/m); - }); - - it("emits `triggers:` only when the style uses it", () => { - const withTriggers = generateAgentMarkdown({ name: "x", triggers: ["alpha", "beta"] }, style({ usesTriggers: true })); - expect(withTriggers).toMatch(/^triggers: alpha, beta$/m); - expect(generateAgentMarkdown({ name: "x" }, style({ usesTriggers: false }))).not.toMatch(/^triggers:/m); - }); -}); - -describe("agent style-match — create_agent through real dispatch mirrors the user's agents", () => { - let root: string; - let prior: string; - let ctx: ToolContext; - beforeEach(() => { - root = mkdtempSync(join(tmpdir(), "kb-agentstyle-")); - // a REAL-shaped existing agent that carries model: - mkdirSync(join(root, ".claude", "agents"), { recursive: true }); - writeFileSync(join(root, ".claude", "agents", "reviewer.md"), "---\nname: reviewer\ndescription: reviews code\ntools: Read, Grep\nmodel: opus\n---\n\nYou review.\n"); - const ccr = createFileCCRStore(join(root, "ccr")); - ctx = { - ccr, - memory: createMemory(join(root, "mem")), - knowledge: createKnowledge(root, join(root, "kb")), - feedback: createFeedback(join(root, "fb")), - team: createTeamBoard(join(root, "team"), ccr), - meter: createMeter(join(root, "meter")), - skills: createSkillsStore(join(root, "skills")), - calibration: createCalibration(join(root, "cal")), - }; - prior = process.cwd(); - process.chdir(root); - }); - afterEach(() => { - process.chdir(prior); - rmSync(root, { recursive: true, force: true }); - }); - - it("generated agent carries `model:` because the user's existing agents do", () => { - const out = dispatch(TOOLS.find((t) => t.name === "knitbrain_create_agent")!, { name: "new-agent", scope: "src" }, ctx); - expect(out).toContain("created agent at"); - const body = readFileSync(join(root, ".claude", "agents", "new-agent.md"), "utf8"); - expect(body).toMatch(/^model: opus$/m); // style-matched to the seeded reviewer.md - }); -}); - -// ── (B) intensity-based skill/agent selection ── -describe("orchestrate intensity — buildCyclePlan scales with project intensity", () => { - let root: string; - beforeEach(() => { - root = mkdtempSync(join(tmpdir(), "kb-cycleplan-")); - writeFileSync(join(root, "b.ts"), "export const foo = 1;\n"); - writeFileSync(join(root, "a.ts"), 'import { foo } from "./b.js";\nexport const x = foo;\n'); - mkdirSync(join(root, "src"), { recursive: true }); - writeFileSync(join(root, "src", "p.ts"), 'export const p = 1;\n'); - writeFileSync(join(root, "src", "q.ts"), 'import { p } from "./p.js";\nexport const q = p;\n'); - }); - afterEach(() => rmSync(root, { recursive: true, force: true })); - - it("SIMPLE goal → matched skill in the prompt, NO proposed agents", () => { - const skills = createSkillsStore(join(root, "skills")); - skills.save({ name: "fix-typo", body: "find the typo, fix it, re-read", triggers: ["typo", "readme"] }); - const knowledge = createKnowledge(root, join(root, "kb")); - knowledge.scan(); - - const plan = buildCyclePlan("fix the typo in the readme", skills, knowledge); - expect(plan.tier).toBe("trivial"); - expect(plan.skillName).toBe("fix-typo"); - expect(plan.prompt).toContain("SKILL — fix-typo"); - expect(plan.agentNames).toHaveLength(0); - expect(plan.prompt).not.toContain("AGENTS to orchestrate"); - }); - - it("COMPLEX goal → proposed agents + guardrails AND the skill in the prompt", () => { - const skills = createSkillsStore(join(root, "skills")); - skills.save({ name: "refactor-playbook", body: "map deps, smallest change, verify", triggers: ["refactor", "architecture"] }); - const knowledge = createKnowledge(root, join(root, "kb")); - knowledge.scan(); - - const plan = buildCyclePlan("refactor the authentication architecture and migrate the database schema", skills, knowledge); - expect(plan.tier).toBe("complex"); - expect(plan.agentNames.length).toBeGreaterThan(0); // proposed from the seeded src/ dir - expect(plan.prompt).toContain("AGENTS to orchestrate"); - expect(plan.prompt).toMatch(/scope `.*\*\*`/); // a guardrail (scope glob) is briefed in - expect(plan.prompt).toContain("SKILL — refactor-playbook"); // skill still injected - }); -}); diff --git a/tests/receipt.test.ts b/tests/receipt.test.ts index 5f9f4d0..ba6ede7 100644 --- a/tests/receipt.test.ts +++ b/tests/receipt.test.ts @@ -168,3 +168,26 @@ describe("recordRead / recordRedirect", () => { expect(readSessionMark(root)).toBeNull(); }); }); + +describe("G3 cold-restart waste line", () => { + const meter = (over: Record = {}) => ({ + usedTokens: 100_000, windowTokens: 1_000_000, usedPct: 10, savedTokens: 5_000, + optimizationPct: 5, estimated: false, cacheCold: false, status: "ok" as const, + advice: "", billingMode: "plan" as const, ...over, + }); + const mark = { startTs: new Date(0).toISOString(), savedAtStart: 0, usedAtStart: 0, retrievalsAtStart: 0, reads: {}, redirects: {} }; + const ev = (tsMs: number) => ({ ts: new Date(tsMs).toISOString(), agent: "x", tool: "t", summary: "s", saved: 100, rawTokens: 200, storedTokens: 100 }); + + it("names cold gaps >5min between session events, labeled estimate", async () => { + const { buildReceipt } = await import("../src/engine/receipt.js"); + const r = buildReceipt({ meter: meter(), mark, events: [ev(0), ev(6 * 60_000), ev(7 * 60_000), ev(20 * 60_000)], eventsTrimmed: false, retrievalsTotal: 0 }); + expect(r).toContain("2 idle gap(s) >5min"); + expect(r).toContain("estimate"); + }); + + it("stays silent when no gap exceeds the cache TTL", async () => { + const { buildReceipt } = await import("../src/engine/receipt.js"); + const r = buildReceipt({ meter: meter(), mark, events: [ev(0), ev(60_000), ev(120_000)], eventsTrimmed: false, retrievalsTotal: 0 }); + expect(r).not.toContain("idle gap"); + }); +}); diff --git a/tests/tokenizer.test.ts b/tests/tokenizer.test.ts index 0983ece..17b6fa5 100644 --- a/tests/tokenizer.test.ts +++ b/tests/tokenizer.test.ts @@ -1,6 +1,6 @@ import { describe, it, expect } from "vitest"; import { countTokens, activeTokenizerName } from "../src/tokenizer.js"; -import { measure, summarize } from "../src/measure.js"; +import { measure, summarize } from "../scripts/measure.js"; describe("tokenizer (rung 1)", () => { it("counts empty string as 0 tokens", () => {