diff --git a/config/docdrift_test.go b/config/docdrift_test.go new file mode 100644 index 0000000..47e497b --- /dev/null +++ b/config/docdrift_test.go @@ -0,0 +1,150 @@ +package config + +import ( + "os" + "regexp" + "sort" + "strings" + "testing" +) + +// The preset tables in the docs must match the presets the product actually ships — as a SET, +// in both directions: every preset has a row, every row is a preset, and every row's pipeline is +// the one that runs. +// +// This guard exists because they did not. Auditing the "What each one runs" table in +// docs/how-to/choose-a-preset.md against the `presets` map in config/config.go found EVERY row +// stale: three presets (`codesmart`, `coding`, `general`) were still documented as running +// `toon`, which was retired from them after it acted 0 times on 5,752 production requests and +// converted 0 candidates in 11.67M measured tokens, and every row omitted components that do +// run — the lossless pair (`textclean`, `searchfold`) and `linecap`. Three presets had no row at +// all (`agentdiet`, `house`, `housellm`). docs/reference/presets.md was correct at the same +// moment, so the two documents contradicted each other and a reader had no way to tell which one +// was lying. +// +// That is worse than an out-of-date sentence: the table is what somebody reads to decide whether +// a preset does anything they object to, and the answer it gave was wrong in the direction that +// matters — it named a component that does not run and omitted three that do. +// +// The set check in BOTH directions is deliberate, and it is what the first version of this guard +// got wrong: it iterated the rows it happened to find and asked only "is this row right?", with a +// hand-tuned coverage floor standing in for completeness. Against 12 multi-component presets that +// floor tolerated deleting seven rows, and a preset that ships with no row is exactly the defect +// this PR fixed by hand. The reverse direction matters just as much: a documented preset that +// does not exist is not a cosmetic error, it is a STARTUP failure — +// +// $ PRESET=cache ./context-guru-proxy +// config: config: unknown preset "cache" # process exits +// +// the same class as the `skeleton`/`coding` incident this repo's own comments describe. +// +// Same reasoning as deploy/harbor/pipeline_drift_test.go, applied to the docs instead of the +// benchmark harnesses. +var presetTableDocs = []string{ + "../docs/how-to/choose-a-preset.md", + "../docs/reference/presets.md", +} + +// docRow matches a markdown table row whose first cell is a `preset` name in backticks, and +// captures the rest of the row (where the component list lives, in either `a, b` or `a → b` +// form — the two files use different separators on purpose, so the check reads component names +// rather than trying to normalise the formatting). +// +// The name class is deliberately wider than the presets that exist today: a row naming +// `code-smart` or `preset2` must be REPORTED as documenting a preset that does not exist, not +// skipped for failing to look like a name this file recognises. +var docRow = regexp.MustCompile("(?m)^\\|\\s*`([a-z0-9_-]+)`\\s*\\|(.*)$") + +// componentToken matches one component name in a pipeline cell. +// +// The two documents format a pipeline differently — choose-a-preset.md writes the whole list +// inside ONE pair of backticks (`format, textclean, cachesplit`) while presets.md backticks each +// name and joins them with arrows (`format` → `textclean`). An earlier version of this guard +// looked for backticked names only, which silently matched nothing in the first file: every +// multi-component row was skipped, and the guard's coverage there was limited to the presets +// that happen to run exactly one component. So the cell is stripped of backticks and split on +// the separators instead, which reads both forms. +var componentToken = regexp.MustCompile(`^[a-z][a-z_]*$`) + +// pipelineFromCell reads the component list out of a table cell, in either document's format. +func pipelineFromCell(cell string) []string { + cell = strings.ReplaceAll(cell, "`", " ") + cell = strings.ReplaceAll(cell, "→", ",") + cell = strings.ReplaceAll(cell, "->", ",") + var out []string + for _, tok := range strings.Split(cell, ",") { + tok = strings.TrimSpace(tok) + if componentToken.MatchString(tok) { + out = append(out, tok) + } + } + return out +} + +func TestPresetDocsDoNotDrift(t *testing.T) { + for _, path := range presetTableDocs { + b, err := os.ReadFile(path) + if err != nil { + t.Fatalf("%s: %v", path, err) + } + + documented := map[string][]string{} + for _, m := range docRow.FindAllStringSubmatch(string(b), -1) { + name, row := m[1], m[2] + // The pipeline cell is the one that lists components. In choose-a-preset.md it is + // the whole rest of the row; in presets.md the prose that follows also mentions + // component names, so only the FIRST cell after the name is read. + cell := row + if i := strings.Index(row, "|"); i >= 0 { + cell = row[:i] + } + documented[name] = pipelineFromCell(cell) + } + + // Direction 1: every preset that ships is documented, with the pipeline it runs. + // Iterated over the map, not over the rows, so a missing row FAILS instead of simply + // not being checked. + names := make([]string, 0, len(presets)) + for name := range presets { + names = append(names, name) + } + sort.Strings(names) + for _, name := range names { + want := presets[name] + got, ok := documented[name] + if !ok { + t.Errorf("%s: preset %q ships %v but has no row in this table. "+ + "An undocumented preset is the same defect as a misdocumented one: a reader "+ + "cannot tell it exists, or what it would do to their context.", path, name, want) + continue + } + // `off` is the passthrough: no components on either side, and the table writes it + // as *(empty)*. Anything else that documents an empty pipeline for a preset which + // runs components is caught by the comparison below. + if len(want) == 0 && len(got) == 0 { + continue + } + if strings.Join(got, ",") != strings.Join(want, ",") { + t.Errorf("%s: preset %q is documented as running\n %v\nbut ships\n %v\n"+ + "A reader uses this table to decide what a preset will do to their context; "+ + "naming a component that does not run, or omitting one that does, is the "+ + "failure this guard exists for.", path, name, got, want) + } + } + + // Direction 2: nothing is documented that does not exist. `--preset ` for a name + // that is not in the map does not degrade, it exits at startup, so a row inviting one is + // a break dressed as documentation. + rows := make([]string, 0, len(documented)) + for name := range documented { + rows = append(rows, name) + } + sort.Strings(rows) + for _, name := range rows { + if _, ok := presets[name]; !ok { + t.Errorf("%s: preset %q is documented but does not exist in the presets map; "+ + "`--preset %s` exits at startup with `unknown preset %q`.", path, name, name, name) + } + } + } +} diff --git a/docs/how-to/choose-a-preset.md b/docs/how-to/choose-a-preset.md index 15a32c5..7f3b501 100644 --- a/docs/how-to/choose-a-preset.md +++ b/docs/how-to/choose-a-preset.md @@ -26,20 +26,33 @@ context-guru-proxy --preset codesmart # or PRESET=codesmart, or preset: in | Preset | Pipeline | |---|---| -| `codesmart` | `format, toon, dedup, failed_run, cmdfilter, extract_llm, extract, cachesplit` | -| `codesafe` | `format, dedup, failed_run, cmdfilter, extract, collapse, cachesplit` | -| `safe` | `format, cachesplit` | -| `balanced` | `format, dedup, failed_run, cmdfilter, cachesplit` | -| `aggressive` | `format, dedup, failed_run, cmdfilter, smartcrush, extract, extract_llm, cachesplit` | -| `coding` | `format, toon, dedup, cmdfilter, extract, cachesplit` | -| `mcp` | `format, smartcrush, cachesplit` | -| `agent` | `format, dedup, failed_run, mask, extract, extract_llm, cachesplit` | -| `general` | `format, toon, dedup, failed_run, cmdfilter, mask, extract, extract_llm, collapse, cachesplit` | +| `codesmart` | `format, textclean, searchfold, dedup, failed_run, cmdfilter, extract_llm, extract, linecap, cachesplit` | +| `codesafe` | `format, textclean, searchfold, dedup, failed_run, cmdfilter, extract, collapse, linecap, cachesplit` | +| `safe` | `format, textclean, searchfold, cachesplit` | +| `balanced` | `format, textclean, searchfold, dedup, failed_run, cmdfilter, linecap, cachesplit` | +| `aggressive` | `format, textclean, searchfold, dedup, failed_run, cmdfilter, smartcrush, extract, extract_llm, linecap, cachesplit` | +| `coding` | `format, textclean, searchfold, dedup, cmdfilter, extract, linecap, cachesplit` | +| `mcp` | `format, textclean, smartcrush, cachesplit` | +| `agent` | `format, textclean, searchfold, dedup, failed_run, mask, extract, extract_llm, cachesplit` | +| `general` | `format, textclean, searchfold, dedup, failed_run, cmdfilter, mask, extract, extract_llm, collapse, linecap, cachesplit` | | `summarize` | `summarize` | | `off` | *(empty)* | +| `agentdiet` | `format, agentdiet, cachesplit` | +| `house` | `format, dedup, toon, cmdfilter, searchfold, textclean, extract, cachesplit, toolfilter` | +| `housellm` | `format, dedup, toon, cmdfilter, searchfold, textclean, extract_llm, extract_llm_sweep, extract, cachesplit, toolfilter` | Order is deliberate: lossless repack first, then the cheap structural offloaders, then -anything that costs a model call, cache directives last. +anything that costs a model call, cache directives last — except in `house` and `housellm`, +whose order is the operator's on purpose: `dedup` and `cmdfilter` run ahead of the lossless +pair and `toolfilter` sits after `cachesplit`. That costs per-component attribution in +`/stats`, never content; the reasons are recorded in +[`config/config.go`](../design.md#config-registry) and the exemption is noted in the +[preset reference](../reference/presets.md). + +The last three rows are not options in the chooser above. `house` and `housellm` are the +**service** configs — what a hosted account runs unless it asks otherwise — and `agentdiet` +reproduces a published baseline for A/B comparison, not a recommendation. They are in the +table so it lists every preset that exists; pick from the table above this one. ## Notes on the ones people pick