From d0d537451425c8d1dd436e1479136d8647612057 Mon Sep 17 00:00:00 2001 From: DAVID AMID Date: Mon, 31 Aug 2026 19:53:48 +0300 Subject: [PATCH 1/2] fix(docs): correct every row of the "What each one runs" preset table, and guard it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every row of the "What each one runs" table in docs/how-to/choose-a-preset.md was stale relative to the `presets` map in config/config.go: - `codesmart`, `coding` and `general` were documented as running `toon`, which was RETIRED from them — it acted 0 times on 5,752 production requests and converted 0 candidates in 11.67M measured tokens. - every row omitted components that do run: the lossless pair `textclean` and `searchfold`, and `linecap`. - the map has presets the table never listed at all (`agentdiet`, `house`, `housellm`). docs/reference/presets.md was correct at the same moment, so the two documents contradicted each other — in the table a reader uses to decide what a preset will do to their context. That is worse than an out-of-date sentence: the answer was wrong in the direction that matters, naming a component that does not run and omitting three that do. The table is now regenerated from the map (existing row order kept, the never-listed presets appended), and config/docdrift_test.go parses BOTH doc tables and diffs each row against `presets`. It reads either formatting — one backtick pair around a comma list (choose-a-preset.md) or per-name backticks joined by arrows (presets.md) — and asserts coverage, because an earlier version of this check looked for individually-backticked names only, matched nothing in choose-a-preset.md, and so silently checked just the single-component presets. Revert-verified (each mutation asserted to have landed in the source before the test was believed). (a) codesmart's row in docs/how-to/choose-a-preset.md put back to its stale form: --- FAIL: TestDocumentedPresetPipelinesMatchTheShippedOnes (0.00s) docdrift_test.go:98: ../docs/how-to/choose-a-preset.md: preset "codesmart" is documented as running [format toon dedup failed_run cmdfilter extract_llm extract cachesplit] but ships [format textclean searchfold dedup failed_run cmdfilter extract_llm extract linecap cachesplit] A reader uses this table to decide what a preset will do to their context; naming a component that does not run, or omitting one that does, is the failure this guard exists for. (b) `safe` reduced to `format` -> `cachesplit` in docs/reference/presets.md: --- FAIL: TestDocumentedPresetPipelinesMatchTheShippedOnes (0.00s) docdrift_test.go:98: ../docs/reference/presets.md: preset "safe" is documented as running [format cachesplit] but ships [format textclean searchfold cachesplit] (c) the coverage assertion itself, checked non-vacuously by deleting 10 rows from the table: --- FAIL: TestDocumentedPresetPipelinesMatchTheShippedOnes (0.00s) docdrift_test.go:114: ../docs/how-to/choose-a-preset.md: this guard only checked 3 preset rows (2 of them multi-component). The table's shape must have changed and the check has silently stopped covering it — which is how the stale rows survived in the first place. Restoring each mutation returns `go test ./config/` to ok. gofmt and `go vet ./config/` are clean. Found while adding a row to this table during unrelated local-distribution work; split out because the defect has nothing to do with distribution. Assisted-By: Claude Opus 5 (1M context) Signed-off-by: DAVID AMID --- config/docdrift_test.go | 119 +++++++++++++++++++++++++++++++++ docs/how-to/choose-a-preset.md | 21 +++--- 2 files changed, 131 insertions(+), 9 deletions(-) create mode 100644 config/docdrift_test.go diff --git a/config/docdrift_test.go b/config/docdrift_test.go new file mode 100644 index 00000000..2c580a01 --- /dev/null +++ b/config/docdrift_test.go @@ -0,0 +1,119 @@ +package config + +import ( + "os" + "regexp" + "strings" + "testing" +) + +// The preset tables in the docs must match the presets the product actually ships. +// +// This guard exists because they did not. Auditing the "What each one runs" table in +// docs/how-to/choose-a-preset.md against the `presets` map above found EVERY row stale: three +// presets (`codesmart`, `coding`, `general`) were still documented as running `toon`, which was +// retired from them after it acted 0 times on 5,752 production requests and converted 0 +// candidates in 11.67M measured tokens, and every row omitted components that do run — the +// lossless pair (`textclean`, `searchfold`) and `linecap`. docs/reference/presets.md was correct +// at the same moment, so the two documents contradicted each other and a reader had no way to +// tell which one was lying. +// +// That is worse than an out-of-date sentence: the table is what somebody reads to decide whether +// a preset does anything they object to, and the answer it gave was wrong in the direction that +// matters — it named a component that does not run and omitted three that do. +// +// Same reasoning as deploy/harbor/pipeline_drift_test.go, applied to the docs instead of the +// benchmark harnesses. +var presetTableDocs = []string{ + "../docs/how-to/choose-a-preset.md", + "../docs/reference/presets.md", +} + +// docRow matches a markdown table row whose first cell is a `preset` name in backticks, and +// captures the rest of the row (where the component list lives, in either `a, b` or `a → b` +// form — the two files use different separators on purpose, so the check reads component names +// rather than trying to normalise the formatting). +var docRow = regexp.MustCompile("(?m)^\\|\\s*`([a-z_]+)`\\s*\\|(.*)$") + +// componentToken matches one component name in a pipeline cell. +// +// The two documents format a pipeline differently — choose-a-preset.md writes the whole list +// inside ONE pair of backticks (`format, textclean, cachesplit`) while presets.md backticks each +// name and joins them with arrows (`format` → `textclean`). An earlier version of this guard +// looked for backticked names only, which silently matched nothing in the first file: every +// multi-component row was skipped, and the guard's coverage there was limited to the presets +// that happen to run exactly one component. So the cell is stripped of backticks and split on +// the separators instead, which reads both forms. +var componentToken = regexp.MustCompile(`^[a-z][a-z_]*$`) + +// pipelineFromCell reads the component list out of a table cell, in either document's format. +func pipelineFromCell(cell string) []string { + cell = strings.ReplaceAll(cell, "`", " ") + cell = strings.ReplaceAll(cell, "→", ",") + cell = strings.ReplaceAll(cell, "->", ",") + var out []string + for _, tok := range strings.Split(cell, ",") { + tok = strings.TrimSpace(tok) + if componentToken.MatchString(tok) { + out = append(out, tok) + } + } + return out +} + +func TestDocumentedPresetPipelinesMatchTheShippedOnes(t *testing.T) { + for _, path := range presetTableDocs { + b, err := os.ReadFile(path) + if err != nil { + t.Fatalf("%s: %v", path, err) + } + seen := map[string]bool{} + for _, m := range docRow.FindAllStringSubmatch(string(b), -1) { + name, row := m[1], m[2] + want, ok := presets[name] + if !ok { + // A row for something that is not a preset (a component reference table, a + // config key) is not this test's business. + continue + } + // The pipeline cell is the one that lists components. In choose-a-preset.md it is + // the whole rest of the row; in presets.md the prose that follows also mentions + // component names, so only the FIRST cell after the name is read. + cell := row + if i := strings.Index(row, "|"); i >= 0 { + cell = row[:i] + } + names := pipelineFromCell(cell) + // A row that lists no components at all is either the `off` passthrough or a row + // that documents something else about the preset; only check the ones that claim + // to list a pipeline. + if len(names) == 0 { + if len(want) > 0 && strings.Contains(cell, "empty") { + t.Errorf("%s: preset %q is documented as empty but runs %v", path, name, want) + } + continue + } + seen[name] = true + if strings.Join(names, ",") != strings.Join(want, ",") { + t.Errorf("%s: preset %q is documented as running\n %v\nbut ships\n %v\n"+ + "A reader uses this table to decide what a preset will do to their context; "+ + "naming a component that does not run, or omitting one that does, is the "+ + "failure this guard exists for.", path, name, names, want) + } + } + // Coverage, not just "something matched". The previous version of this guard passed + // while checking only the single-component presets in one of these files, so a count + // is asserted: the multi-component rows are exactly the ones that were rotting. + multi := 0 + for name := range seen { + if len(presets[name]) > 1 { + multi++ + } + } + if len(seen) == 0 || multi < 5 { + t.Errorf("%s: this guard only checked %d preset rows (%d of them multi-component). "+ + "The table's shape must have changed and the check has silently stopped covering "+ + "it — which is how the stale rows survived in the first place.", path, len(seen), multi) + } + } +} diff --git a/docs/how-to/choose-a-preset.md b/docs/how-to/choose-a-preset.md index 15a32c5a..68beb14f 100644 --- a/docs/how-to/choose-a-preset.md +++ b/docs/how-to/choose-a-preset.md @@ -26,17 +26,20 @@ context-guru-proxy --preset codesmart # or PRESET=codesmart, or preset: in | Preset | Pipeline | |---|---| -| `codesmart` | `format, toon, dedup, failed_run, cmdfilter, extract_llm, extract, cachesplit` | -| `codesafe` | `format, dedup, failed_run, cmdfilter, extract, collapse, cachesplit` | -| `safe` | `format, cachesplit` | -| `balanced` | `format, dedup, failed_run, cmdfilter, cachesplit` | -| `aggressive` | `format, dedup, failed_run, cmdfilter, smartcrush, extract, extract_llm, cachesplit` | -| `coding` | `format, toon, dedup, cmdfilter, extract, cachesplit` | -| `mcp` | `format, smartcrush, cachesplit` | -| `agent` | `format, dedup, failed_run, mask, extract, extract_llm, cachesplit` | -| `general` | `format, toon, dedup, failed_run, cmdfilter, mask, extract, extract_llm, collapse, cachesplit` | +| `codesmart` | `format, textclean, searchfold, dedup, failed_run, cmdfilter, extract_llm, extract, linecap, cachesplit` | +| `codesafe` | `format, textclean, searchfold, dedup, failed_run, cmdfilter, extract, collapse, linecap, cachesplit` | +| `safe` | `format, textclean, searchfold, cachesplit` | +| `balanced` | `format, textclean, searchfold, dedup, failed_run, cmdfilter, linecap, cachesplit` | +| `aggressive` | `format, textclean, searchfold, dedup, failed_run, cmdfilter, smartcrush, extract, extract_llm, linecap, cachesplit` | +| `coding` | `format, textclean, searchfold, dedup, cmdfilter, extract, linecap, cachesplit` | +| `mcp` | `format, textclean, smartcrush, cachesplit` | +| `agent` | `format, textclean, searchfold, dedup, failed_run, mask, extract, extract_llm, cachesplit` | +| `general` | `format, textclean, searchfold, dedup, failed_run, cmdfilter, mask, extract, extract_llm, collapse, linecap, cachesplit` | | `summarize` | `summarize` | | `off` | *(empty)* | +| `agentdiet` | `format, agentdiet, cachesplit` | +| `house` | `format, dedup, toon, cmdfilter, searchfold, textclean, extract, cachesplit, toolfilter` | +| `housellm` | `format, dedup, toon, cmdfilter, searchfold, textclean, extract_llm, extract_llm_sweep, extract, cachesplit, toolfilter` | Order is deliberate: lossless repack first, then the cheap structural offloaders, then anything that costs a model call, cache directives last. From 65b801a8fcd2ced49a9080bd1bebae5a48d6fb76 Mon Sep 17 00:00:00 2001 From: DAVID AMID Date: Tue, 1 Sep 2026 10:45:11 +0300 Subject: [PATCH 2/2] fix(docs): make the preset-doc guard assert set equality, not row correctness MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review of #142 found the guard checked the rows it happened to find, never the preset SET. It iterated doc rows asking "is this row right?" and stood in for completeness with a hand-tuned floor, `multi < 5`, against a population of 12 multi-component presets. Three holes, all in the class this PR had just fixed by hand: - a preset that SHIPS with no row passed (seven of twelve rows could be deleted, `codesmart` among them, before the floor tripped); - a preset DOCUMENTED that does not exist passed — and that is a startup failure, not a graceful degradation: $ PRESET=cache ./context-guru-proxy config: config: unknown preset "cache" # process exits the same class as the skeleton/coding incident this file's comments describe; - a row name with a digit or hyphen was silently skipped by docRow. The loop is inverted: it now walks `presets` asserting every preset has a row with the pipeline it runs, then walks the rows asserting nothing is documented that does not exist. That deletes the seen/multi/`multi < 5` block and its magic number — the check scales with the map instead of needing the floor re-tuned — and docRow widens to `([a-z0-9_-]+)` so an odd name is REPORTED rather than skipped. Both loops iterate sorted keys so failures are stable. Also: renamed to TestPresetDocsDoNotDrift so `go test -run Drift` matches it; noted in choose-a-preset.md that `house`/`housellm` deviate from the lossless-first ordering deliberately (config.go records why, reference/presets.md carries the exemption) and that `house`/`housellm`/`agentdiet` are the service configs and a published baseline, not peer recommendations, so nobody reads a how-to and sets `--preset house`; corrected "the `presets` map above" to name config/config.go. Revert-verified again, each mutation asserted to have landed in the source before its result was believed (an apply that fails its own anchor check aborts the run rather than reporting a vacuous ok). (a) a preset that ships with no row — `agentdiet`'s row deleted: --- FAIL: TestPresetDocsDoNotDrift (0.00s) docdrift_test.go:116: ../docs/how-to/choose-a-preset.md: preset "agentdiet" ships [format agentdiet cachesplit] but has no row in this table. An undocumented preset is the same defect as a misdocumented one: a reader cannot tell it exists, or what it would do to their context. (b) a preset documented that does not exist — a `cache` row added: --- FAIL: TestPresetDocsDoNotDrift (0.00s) docdrift_test.go:145: ../docs/how-to/choose-a-preset.md: preset "cache" is documented but does not exist in the presets map; `--preset cache` exits at startup with `unknown preset "cache"`. (c) the hyphen hole, which trips both directions at once — `codesmart` renamed to `code-smart`: --- FAIL: TestPresetDocsDoNotDrift (0.00s) docdrift_test.go:116: ../docs/how-to/choose-a-preset.md: preset "codesmart" ships [format textclean searchfold dedup failed_run cmdfilter extract_llm extract linecap cachesplit] but has no row in this table. ... docdrift_test.go:145: ../docs/how-to/choose-a-preset.md: preset "code-smart" is documented but does not exist in the presets map; `--preset code-smart` exits at startup with `unknown preset "code-smart"`. (d) a wrong component list — `safe` cut to format -> cachesplit in docs/reference/presets.md: --- FAIL: TestPresetDocsDoNotDrift (0.00s) docdrift_test.go:128: ../docs/reference/presets.md: preset "safe" is documented as running [format cachesplit] but ships [format textclean searchfold cachesplit] A reader uses this table to decide what a preset will do to their context; naming a component that does not run, or omitting one that does, is the failure this guard exists for. Restoring each mutation returns `go test ./config/ -run Drift` to ok; the full `go test ./config/` passes, gofmt and `go vet ./config/` are clean. Assisted-By: Claude Opus 5 (1M context) Signed-off-by: DAVID AMID --- config/docdrift_test.go | 111 +++++++++++++++++++++------------ docs/how-to/choose-a-preset.md | 12 +++- 2 files changed, 82 insertions(+), 41 deletions(-) diff --git a/config/docdrift_test.go b/config/docdrift_test.go index 2c580a01..47e497b0 100644 --- a/config/docdrift_test.go +++ b/config/docdrift_test.go @@ -3,25 +3,41 @@ package config import ( "os" "regexp" + "sort" "strings" "testing" ) -// The preset tables in the docs must match the presets the product actually ships. +// The preset tables in the docs must match the presets the product actually ships — as a SET, +// in both directions: every preset has a row, every row is a preset, and every row's pipeline is +// the one that runs. // // This guard exists because they did not. Auditing the "What each one runs" table in -// docs/how-to/choose-a-preset.md against the `presets` map above found EVERY row stale: three -// presets (`codesmart`, `coding`, `general`) were still documented as running `toon`, which was -// retired from them after it acted 0 times on 5,752 production requests and converted 0 -// candidates in 11.67M measured tokens, and every row omitted components that do run — the -// lossless pair (`textclean`, `searchfold`) and `linecap`. docs/reference/presets.md was correct -// at the same moment, so the two documents contradicted each other and a reader had no way to -// tell which one was lying. +// docs/how-to/choose-a-preset.md against the `presets` map in config/config.go found EVERY row +// stale: three presets (`codesmart`, `coding`, `general`) were still documented as running +// `toon`, which was retired from them after it acted 0 times on 5,752 production requests and +// converted 0 candidates in 11.67M measured tokens, and every row omitted components that do +// run — the lossless pair (`textclean`, `searchfold`) and `linecap`. Three presets had no row at +// all (`agentdiet`, `house`, `housellm`). docs/reference/presets.md was correct at the same +// moment, so the two documents contradicted each other and a reader had no way to tell which one +// was lying. // // That is worse than an out-of-date sentence: the table is what somebody reads to decide whether // a preset does anything they object to, and the answer it gave was wrong in the direction that // matters — it named a component that does not run and omitted three that do. // +// The set check in BOTH directions is deliberate, and it is what the first version of this guard +// got wrong: it iterated the rows it happened to find and asked only "is this row right?", with a +// hand-tuned coverage floor standing in for completeness. Against 12 multi-component presets that +// floor tolerated deleting seven rows, and a preset that ships with no row is exactly the defect +// this PR fixed by hand. The reverse direction matters just as much: a documented preset that +// does not exist is not a cosmetic error, it is a STARTUP failure — +// +// $ PRESET=cache ./context-guru-proxy +// config: config: unknown preset "cache" # process exits +// +// the same class as the `skeleton`/`coding` incident this repo's own comments describe. +// // Same reasoning as deploy/harbor/pipeline_drift_test.go, applied to the docs instead of the // benchmark harnesses. var presetTableDocs = []string{ @@ -33,7 +49,11 @@ var presetTableDocs = []string{ // captures the rest of the row (where the component list lives, in either `a, b` or `a → b` // form — the two files use different separators on purpose, so the check reads component names // rather than trying to normalise the formatting). -var docRow = regexp.MustCompile("(?m)^\\|\\s*`([a-z_]+)`\\s*\\|(.*)$") +// +// The name class is deliberately wider than the presets that exist today: a row naming +// `code-smart` or `preset2` must be REPORTED as documenting a preset that does not exist, not +// skipped for failing to look like a name this file recognises. +var docRow = regexp.MustCompile("(?m)^\\|\\s*`([a-z0-9_-]+)`\\s*\\|(.*)$") // componentToken matches one component name in a pipeline cell. // @@ -61,21 +81,16 @@ func pipelineFromCell(cell string) []string { return out } -func TestDocumentedPresetPipelinesMatchTheShippedOnes(t *testing.T) { +func TestPresetDocsDoNotDrift(t *testing.T) { for _, path := range presetTableDocs { b, err := os.ReadFile(path) if err != nil { t.Fatalf("%s: %v", path, err) } - seen := map[string]bool{} + + documented := map[string][]string{} for _, m := range docRow.FindAllStringSubmatch(string(b), -1) { name, row := m[1], m[2] - want, ok := presets[name] - if !ok { - // A row for something that is not a preset (a component reference table, a - // config key) is not this test's business. - continue - } // The pipeline cell is the one that lists components. In choose-a-preset.md it is // the whole rest of the row; in presets.md the prose that follows also mentions // component names, so only the FIRST cell after the name is read. @@ -83,37 +98,53 @@ func TestDocumentedPresetPipelinesMatchTheShippedOnes(t *testing.T) { if i := strings.Index(row, "|"); i >= 0 { cell = row[:i] } - names := pipelineFromCell(cell) - // A row that lists no components at all is either the `off` passthrough or a row - // that documents something else about the preset; only check the ones that claim - // to list a pipeline. - if len(names) == 0 { - if len(want) > 0 && strings.Contains(cell, "empty") { - t.Errorf("%s: preset %q is documented as empty but runs %v", path, name, want) - } + documented[name] = pipelineFromCell(cell) + } + + // Direction 1: every preset that ships is documented, with the pipeline it runs. + // Iterated over the map, not over the rows, so a missing row FAILS instead of simply + // not being checked. + names := make([]string, 0, len(presets)) + for name := range presets { + names = append(names, name) + } + sort.Strings(names) + for _, name := range names { + want := presets[name] + got, ok := documented[name] + if !ok { + t.Errorf("%s: preset %q ships %v but has no row in this table. "+ + "An undocumented preset is the same defect as a misdocumented one: a reader "+ + "cannot tell it exists, or what it would do to their context.", path, name, want) + continue + } + // `off` is the passthrough: no components on either side, and the table writes it + // as *(empty)*. Anything else that documents an empty pipeline for a preset which + // runs components is caught by the comparison below. + if len(want) == 0 && len(got) == 0 { continue } - seen[name] = true - if strings.Join(names, ",") != strings.Join(want, ",") { + if strings.Join(got, ",") != strings.Join(want, ",") { t.Errorf("%s: preset %q is documented as running\n %v\nbut ships\n %v\n"+ "A reader uses this table to decide what a preset will do to their context; "+ "naming a component that does not run, or omitting one that does, is the "+ - "failure this guard exists for.", path, name, names, want) + "failure this guard exists for.", path, name, got, want) } } - // Coverage, not just "something matched". The previous version of this guard passed - // while checking only the single-component presets in one of these files, so a count - // is asserted: the multi-component rows are exactly the ones that were rotting. - multi := 0 - for name := range seen { - if len(presets[name]) > 1 { - multi++ - } + + // Direction 2: nothing is documented that does not exist. `--preset ` for a name + // that is not in the map does not degrade, it exits at startup, so a row inviting one is + // a break dressed as documentation. + rows := make([]string, 0, len(documented)) + for name := range documented { + rows = append(rows, name) } - if len(seen) == 0 || multi < 5 { - t.Errorf("%s: this guard only checked %d preset rows (%d of them multi-component). "+ - "The table's shape must have changed and the check has silently stopped covering "+ - "it — which is how the stale rows survived in the first place.", path, len(seen), multi) + sort.Strings(rows) + for _, name := range rows { + if _, ok := presets[name]; !ok { + t.Errorf("%s: preset %q is documented but does not exist in the presets map; "+ + "`--preset %s` exits at startup with `unknown preset %q`.", path, name, name, name) + } } } } diff --git a/docs/how-to/choose-a-preset.md b/docs/how-to/choose-a-preset.md index 68beb14f..7f3b501f 100644 --- a/docs/how-to/choose-a-preset.md +++ b/docs/how-to/choose-a-preset.md @@ -42,7 +42,17 @@ context-guru-proxy --preset codesmart # or PRESET=codesmart, or preset: in | `housellm` | `format, dedup, toon, cmdfilter, searchfold, textclean, extract_llm, extract_llm_sweep, extract, cachesplit, toolfilter` | Order is deliberate: lossless repack first, then the cheap structural offloaders, then -anything that costs a model call, cache directives last. +anything that costs a model call, cache directives last — except in `house` and `housellm`, +whose order is the operator's on purpose: `dedup` and `cmdfilter` run ahead of the lossless +pair and `toolfilter` sits after `cachesplit`. That costs per-component attribution in +`/stats`, never content; the reasons are recorded in +[`config/config.go`](../design.md#config-registry) and the exemption is noted in the +[preset reference](../reference/presets.md). + +The last three rows are not options in the chooser above. `house` and `housellm` are the +**service** configs — what a hosted account runs unless it asks otherwise — and `agentdiet` +reproduces a published baseline for A/B comparison, not a recommendation. They are in the +table so it lists every preset that exists; pick from the table above this one. ## Notes on the ones people pick