From aa34ff240515a7f22294ddd3f8d5188916743d0a Mon Sep 17 00:00:00 2001 From: Andrei Marinescu Date: Tue, 15 Sep 2026 18:35:31 +0300 Subject: [PATCH 1/3] index: make the BM25 MaxScore bound authority-aware BM25 ranking depended on k: 48 of 60 golden queries returned a different top-10 at k=10 than at k=50 on prod. The scan terminated when remainingMax < theta, where theta is the k-th largest raw score. The final ranking is raw * Multiplier(host), applied later at metadata-resolution time because the multiplier needs the document URL, so an unseen document's real ceiling is remainingMax * maxMult. With the default alpha of 2.0 the bound was 3x too loose and dropped genuine top-k members. An unknown host scores 0.5, i.e. a 2x multiplier, so this was the common case rather than a tail case. resolveTopKPool already carried the equivalent bound on the candidate side and needed no change; the tracker's note to apply it there too was mistaken. Phrase queries now opt out of MaxScore entirely: theta cannot threshold a phrase-filtered subset, so early termination could return empty. No test covered MaxScore and authority together, which is the gap this shipped through: one test exercised MaxScore with a nil scorer and compared only the first hit, the other attached a scorer but disabled MaxScore. Adds that case asserting full top-k membership, a k-independence pin, and an alpha table covering alpha=0 where the new bound must reduce to the old one. Corrects two false comments: top-k membership is approximate, not lossless (break also freezes documents already in scores -- deferred, tracked), and boost seeding rescues only zero-overlap documents in boost sets of 256 or fewer. Measured against a lossless oracle (COSIFT_BM25_DISABLE_MAXSCORE=1, exact by construction) on a frozen 3,070-document store: overlap@10 versus ground truth 0.8362 -> 0.9793, rank displacement 0.844 -> 0.135, worst score ratio 0.5119 -> 0.7637. One query of 58 regressed, 0.80 -> 0.70. No latency change on the shared benchmark: pool +0.46%, resolve-all -0.86%, both within noise. The prod k=10 versus k=50 verdict and the POOL_FACTOR reassessment still need a measurement window on the box. --- docs/ENV.md | 10 +- docs/TUNING.md | 20 ++ internal/index/pebble_bm25.go | 52 +++-- internal/index/pebble_bm25_test.go | 320 +++++++++++++++++++++++++++-- 4 files changed, 350 insertions(+), 52 deletions(-) diff --git a/docs/ENV.md b/docs/ENV.md index 10b8915..f80f7b9 100644 --- a/docs/ENV.md +++ b/docs/ENV.md @@ -136,13 +136,13 @@ not re-enable them without the confidentiality decision in that section. | `COSIFT_BM25_K1` | float | `1.2` (PebbleBM25 default) | BM25 term-frequency saturation `k1`; applied only if parseable and `> 0`. | `serve_search.go:1350` | | `COSIFT_BM25_B` | float | `0.75` (PebbleBM25 default) | BM25 length-normalization `b`; applied only if parseable and `> 0`. | `serve_search.go:1358` | | `COSIFT_BM25_MIN_IDF` | float | `0.5` | IDF floor below which a query term is dropped as a stopword. `0` disables pruning. Must be `>= 0`. | `internal/index/pebble_bm25.go:38` | -| `COSIFT_BM25_DISABLE_MAXSCORE` | bool (non-empty disables) | unset → MaxScore optimization **enabled** | Any non-empty value disables the WAND/MaxScore early-termination optimization for benchmark-grade lossless ranking. | `internal/index/pebble_bm25.go:223` | +| `COSIFT_BM25_DISABLE_MAXSCORE` | bool (non-empty disables) | unset → MaxScore optimization **enabled** | Any non-empty value disables the WAND/MaxScore early-termination optimization for benchmark-grade lossless ranking. Phrase queries skip the optimization unconditionally. | `internal/index/pebble_bm25.go:230` | | `COSIFT_BM25_TOPK_POOL_FACTOR` | int | `50` | Sizes the metadata-resolution pool at `factor*k` candidates (PebbleBM25 top-k pool). Must be `>= 1`. Raise if the pool-cap log line fires on ranking-sensitive traffic. | `internal/index/pebble_bm25.go:50` | -| `COSIFT_BM25_DISABLE_TOPK_POOL` | bool (non-empty disables) | unset → top-k pool **enabled** | Any non-empty value restores the resolve-all metadata path (every scored candidate gets a `GetDocMeta`) — the pre-pool behavior, for lossless A/B comparison. | `internal/index/pebble_bm25.go:295` | +| `COSIFT_BM25_DISABLE_TOPK_POOL` | bool (non-empty disables) | unset → top-k pool **enabled** | Any non-empty value restores the resolve-all metadata path (every scored candidate gets a `GetDocMeta`) — the pre-pool behavior, for lossless A/B comparison. | `internal/index/pebble_bm25.go:294` | | `COSIFT_DEFAULT_DECAY_DAYS` | float | `180` (6-month half-life) | Default recency half-life (days) applied when a request has no explicit `?decay=`. `0` disables decay globally. Must be `>= 0`; explicit `?decay=N` still wins. | `serve_search.go:1592` | -| `COSIFT_AUTHORITY_ALPHA` | float | scorer's built-in default (`authority.New()`) | Authority-score blend weight; applied only if parseable and `>= 0`. | `serve_setup.go:188` | -| `COSIFT_TRANCO_CSV` | string (path) | unset → embedded whitelist + TLD heuristics only | Path to a Tranco rankings CSV to enrich authority scoring. | `serve_setup.go:193` | -| `COSIFT_MAJESTIC_CSV` | string (path) | unset → embedded whitelist + TLD heuristics only | Path to a Majestic Million CSV to enrich authority scoring. | `serve_setup.go:202` | +| `COSIFT_AUTHORITY_ALPHA` | float | scorer's built-in default (`authority.New()`) | Authority-score blend weight; applied only if parseable and `>= 0`. **Also a BM25 scan-cost knob**: it sets the MaxScore pruning ceiling `maxMult = 1+alpha` (`internal/index/pebble_bm25.go:242`), so raising it prunes less (more posting scans, larger per-query score map) and lowering it prunes more. Lower alpha before reaching for `COSIFT_BM25_DISABLE_MAXSCORE`. | `serve_setup.go:135` | +| `COSIFT_TRANCO_CSV` | string (path) | unset → embedded whitelist + TLD heuristics only | Path to a Tranco rankings CSV to enrich authority scoring. | `serve_setup.go:140` | +| `COSIFT_MAJESTIC_CSV` | string (path) | unset → embedded whitelist + TLD heuristics only | Path to a Majestic Million CSV to enrich authority scoring. | `serve_setup.go:149` | | `COSIFT_DISABLE_ENTITY_EXPAND` | bool (non-empty disables) | unset → entity expansion **enabled** | When unset, query is expanded with canonical-attribute rewrites (`qexpand.RewriteEntity`). Any non-empty value disables it. | `serve_search.go:2064` | | `COSIFT_HYDE_CACHE_SIZE` | int | `256` | Capacity of the HyDE hypothetical-document cache; must be `> 0` (warns on bad value). | `serve_setup.go:161` | | `COSIFT_PARA_CACHE_SIZE` | int | `256` | Capacity of the paraphrase cache; must be `> 0` (warns on bad value). | `serve_setup.go:173` | diff --git a/docs/TUNING.md b/docs/TUNING.md index a872f9a..e2b8050 100644 --- a/docs/TUNING.md +++ b/docs/TUNING.md @@ -111,6 +111,26 @@ set (same cost as before the pool, plus a small selection overhead). `COSIFT_BM25_DISABLE_TOPK_POOL=1` restores the resolve-all path for lossless A/B comparison; both vars are read per query, no restart needed. +### MaxScore pruning depth: `COSIFT_AUTHORITY_ALPHA` + +The MaxScore early-break compares the k-th *raw* score against +`remainingMax * (1 + alpha)`, because a doc's final score is +`raw × authority multiplier` and that multiplier tops out at `1 + alpha`. So +alpha is a latency knob as well as a ranking knob: **higher alpha prunes less** +(queries whose `theta / remainingMax` lands in `[1, 1+alpha)` now scan the +lower-IDF term's full posting list and grow the per-query score map +accordingly); **lower alpha prunes more**. + +If BM25 p50 regresses, lower alpha first — it narrows the non-pruning band +proportionally and keeps ranking sound. `COSIFT_BM25_DISABLE_MAXSCORE=1` is the +opposite of a mitigation: it removes pruning entirely and makes common-term +queries strictly slower. Alpha is read once at server start, so changing it +needs a restart (unlike the two pool vars above). + +Phrase queries (`"like this"`) skip MaxScore unconditionally and pay the full +scan regardless of alpha: the threshold is a k-th over all scored docs, which +says nothing about the k-th of the phrase-filtered subset. + ## Latency budget ### Don't enrich what you don't need: `?enrich=false` diff --git a/internal/index/pebble_bm25.go b/internal/index/pebble_bm25.go index 240810d..3e12e08 100644 --- a/internal/index/pebble_bm25.go +++ b/internal/index/pebble_bm25.go @@ -94,6 +94,14 @@ func (b *PebbleBM25) WithAuthority(a *authority.Scorer) *PebbleBM25 { return b } +// maxAuthorityMult upper-bounds Scorer.Multiplier (= 1 + alpha*score, score in [0,1]). +func (b *PebbleBM25) maxAuthorityMult() float64 { + if b.authority == nil { + return 1.0 + } + return 1.0 + b.authority.Alpha() +} + // WithBoost returns a shallow copy of b with the given docID→multiplier map // applied post-scoring. Use this for site= queries: enumerate the site's // docIDs, pass a 50× multiplier, and the site's docs will always appear in @@ -214,13 +222,13 @@ func (b *PebbleBM25) Search(ctx context.Context, q string, k int) ([]Hit, error) // The BM25 contribution of one term to any doc is bounded above by // idf*(k1+1) — saturates as tf grows relative to docLen. Sum of these // upper bounds across remaining terms = the maximum score a doc not - // yet seen can ever accumulate. If that sum drops below the current - // K-th best partial score, no future doc can enter top-K. Top-K - // membership stays lossless; in-top-K ranking can shift (the rerank - // pipeline re-orders anyway, and the score-decay/MMR stages tolerate - // approximate partial scores). COSIFT_BM25_DISABLE_MAXSCORE=1 disables - // the optimization for benchmark-grade lossless ranking. - maxScoreEnabled := os.Getenv("COSIFT_BM25_DISABLE_MAXSCORE") == "" + // yet seen can ever accumulate. Docs already in `scores` stop + // accumulating at the break too, so top-K membership is approximate, + // not lossless. COSIFT_BM25_DISABLE_MAXSCORE=1 disables the + // optimization for benchmark-grade lossless ranking. Phrase queries opt + // out: theta cannot threshold the phrase-filtered subset (empty results). + maxScoreEnabled := os.Getenv("COSIFT_BM25_DISABLE_MAXSCORE") == "" && len(phrases) == 0 + maxMult := b.maxAuthorityMult() remainingMax := 0.0 if maxScoreEnabled { for _, c := range active { @@ -228,18 +236,10 @@ func (b *PebbleBM25) Search(ctx context.Context, q string, k int) ([]Hit, error) } } for i, c := range active { - // Pre-scan MaxScore check: decide whether term i (and everything - // after it, since active is sorted descending by IDF) can still push - // an unseen doc into top-K BEFORE paying to scan term i's postings. - // remainingMax here is the sum of max contributions of terms i..end - // (idf*(k1+1) each). A doc not yet in `scores` can gain at most - // remainingMax from the un-scanned terms; if that's below the current - // K-th best score, scanning i..end can only reorder within top-K — - // which the reranker fixes — so stop. This catches the common-term - // last-term full scan the old post-scan i==len-1 guard always paid. + // theta is raw; an unseen doc's final ceiling is remainingMax*maxMult. if maxScoreEnabled && i > 0 && len(scores) >= k { theta := kthLargest(scores, k) - if remainingMax < theta { + if remainingMax*maxMult < theta { break } } @@ -268,13 +268,12 @@ func (b *PebbleBM25) Search(ctx context.Context, q string, k int) ([]Hit, error) // Apply per-doc boosts (e.g. site= queries) before sorting. Boosted docs // already present in `scores` (i.e. they matched ≥1 query term) get the - // full multiplier. Boosted docs with zero term overlap are absent from - // `scores`; for small boost sets we seed them with a tiny base score - // (boostSeedBase) before multiplying so they still enter the candidate - // pool — landing at boostSeedBase*mult (e.g. 0.05), below any genuine - // single-term BM25 match but enough for the reranker to judge them. This - // also closes the MaxScore early-termination gap: a boosted doc whose only - // matching posting list was skipped still gets seeded and surfaced. + // full multiplier on their (possibly MaxScore-truncated) partial. Boosted + // docs with zero term overlap are absent from `scores`; for small boost + // sets we seed them with a tiny base score (boostSeedBase) before + // multiplying so they still enter the candidate pool — landing at + // boostSeedBase*mult (e.g. 0.05), below any genuine single-term BM25 + // match but enough for the reranker to judge them. // // Seeding is gated by boostSeedMaxIDs: large boost sets (big sites) would // inject a GetDocMeta per zero-overlap doc and flood the pool, so above @@ -342,10 +341,7 @@ const topKResolveSlack = 16 // the cap does bind (more than factor*k candidates inside that band) the // truncation is logged — the operator signal to raise the factor. func (b *PebbleBM25) resolveTopKPool(ctx context.Context, scores map[int64]float64, phrases []string, k int) ([]Hit, error) { - maxMult := 1.0 - if b.authority != nil { - maxMult += b.authority.Alpha() - } + maxMult := b.maxAuthorityMult() poolCap := bm25TopKPoolFactor() * k pool := topCandidates(scores, poolCap) diff --git a/internal/index/pebble_bm25_test.go b/internal/index/pebble_bm25_test.go index 62462ec..e6f34a5 100644 --- a/internal/index/pebble_bm25_test.go +++ b/internal/index/pebble_bm25_test.go @@ -13,7 +13,7 @@ import ( "github.com/pilot-protocol/cosift/internal/store" ) -func newPebbleBM25(t *testing.T) (*store.PebbleStore, *PebbleBM25) { +func newPebbleBM25(t testing.TB) (*store.PebbleStore, *PebbleBM25) { t.Helper() dir := filepath.Join(t.TempDir(), "pebble") p, err := store.OpenPebble(dir) @@ -135,11 +135,8 @@ func TestPebbleBM25IDFStopwordFilter(t *testing.T) { } } -// TestPebbleBM25MaxScorePreservesTopKMembership locks in the Phase-2 -// contract: MaxScore-style early termination must not lose any doc that -// would have been in the lossless top-K. We compare top-K membership -// (URL set) with the optimization on vs off across a corpus where the -// optimization should actually trigger early-break. +// TestPebbleBM25MaxScorePreservesTopKMembership — top hit only, no authority; +// full set membership is TestPebbleBM25MaxScoreAuthorityTopK. func TestPebbleBM25MaxScorePreservesTopKMembership(t *testing.T) { ps, idx := newPebbleBM25(t) ctx := context.Background() @@ -199,6 +196,254 @@ func TestPebbleBM25MaxScorePreservesTopKMembership(t *testing.T) { } } +const ( + maxScoreAuthQuery = "zeta koppa" + maxScoreAuthURL = "https://www.nih.gov/koppa" + maxScorePhraseURL = "https://www.nih.gov/phrase" + maxScorePhraseQuery = `zeta "koppa koppa"` +) + +// maxScoreAuthorityCorpus — spam farm (multiplier 1.0) plus one trusted host +// whose only matching term is the lower-IDF one. +func maxScoreAuthorityCorpus(t testing.TB) (*store.PebbleStore, *PebbleBM25, *authority.Scorer) { + t.Helper() + ps, idx := newPebbleBM25(t) + for i := 0; i < 13; i++ { + upsertAndIndex(t, ps, idx, fmt.Sprintf("https://spam%d.jiali.sbs/p", i), "spam", + "zeta zeta ordinary body prose here"+strings.Repeat(" pad", i)) + } + upsertAndIndex(t, ps, idx, maxScoreAuthURL, "Koppa", "koppa koppa koppa") + for i := 0; i < 165; i++ { + upsertAndIndex(t, ps, idx, fmt.Sprintf("https://fill%d.jiali.sbs/p", i), "fill", + "koppa ordinary body prose here") + } + for i := 0; i < 221; i++ { + upsertAndIndex(t, ps, idx, fmt.Sprintf("https://bg%d.jiali.sbs/p", i), "bg", + "unrelated background prose entirely") + } + sc := authority.New() + sc.SetSubdomainCounts(map[string]int{"jiali.sbs": 200000}) + return ps, idx, sc +} + +// maxScoreDeepPruneCorpus — a rare term plus a near-ubiquitous one, far enough +// apart that the MaxScore bound breaks before the common postings are scanned +// even at maxMult=1+alpha. Only maxScorePhraseURL carries the literal phrase. +func maxScoreDeepPruneCorpus(t testing.TB) (*store.PebbleStore, *PebbleBM25, *authority.Scorer) { + t.Helper() + ps, idx := newPebbleBM25(t) + for i := 0; i < 10; i++ { + upsertAndIndex(t, ps, idx, fmt.Sprintf("https://spam%d.jiali.sbs/p", i), "spam", + "zeta zeta zeta koppa ordinary body prose here"+strings.Repeat(" pad", i)) + } + upsertAndIndex(t, ps, idx, maxScorePhraseURL, "Koppa", "koppa koppa is here") + for i := 0; i < 1800; i++ { + upsertAndIndex(t, ps, idx, fmt.Sprintf("https://fill%d.jiali.sbs/p", i), "fill", + "koppa ordinary body prose here") + } + for i := 0; i < 200; i++ { + upsertAndIndex(t, ps, idx, fmt.Sprintf("https://bg%d.jiali.sbs/p", i), "bg", + "unrelated background prose entirely") + } + sc := authority.New() + sc.SetSubdomainCounts(map[string]int{"jiali.sbs": 200000}) + return ps, idx, sc +} + +// searchEnvAB runs the same query twice: once with envVar cleared, once with +// it set to "1". +func searchEnvAB(t *testing.T, idx *PebbleBM25, envVar, q string, k int) ([]Hit, []Hit) { + t.Helper() + ctx := context.Background() + t.Setenv(envVar, "") + on, err := idx.Search(ctx, q, k) + if err != nil { + t.Fatalf("search (%s unset): %v", envVar, err) + } + t.Setenv(envVar, "1") + off, err := idx.Search(ctx, q, k) + if err != nil { + t.Fatalf("search (%s=1): %v", envVar, err) + } + t.Setenv(envVar, "") + return on, off +} + +// searchMaxScoreAB — arm 2 (COSIFT_BM25_DISABLE_MAXSCORE=1) is the lossless oracle. +func searchMaxScoreAB(t *testing.T, idx *PebbleBM25, q string, k int) ([]Hit, []Hit) { + t.Helper() + return searchEnvAB(t, idx, "COSIFT_BM25_DISABLE_MAXSCORE", q, k) +} + +func containsURL(hits []Hit, url string) bool { + for _, h := range hits { + if h.URL == url { + return true + } + } + return false +} + +// TestPebbleBM25MaxScoreAuthorityTopK — MaxScore ON with authority attached: +// the raw k-th early-stop threshold must carry maxAuthorityMult. +func TestPebbleBM25MaxScoreAuthorityTopK(t *testing.T) { + _, idx, sc := maxScoreAuthorityCorpus(t) + scored := idx.WithAuthority(sc) + + for _, k := range []int{5, 10} { + on, off := searchMaxScoreAB(t, scored, maxScoreAuthQuery, k) + if len(off) != k { + t.Fatalf("k=%d: oracle returned %d hits, want %d", k, len(off), k) + } + if !sameURLSet(urlSet(on), urlSet(off)) { + t.Errorf("k=%d: MaxScore top-k diverges from lossless\n maxscore: %v\n lossless: %v", + k, urlSet(on), urlSet(off)) + } + if !containsURL(on, maxScoreAuthURL) { + t.Errorf("k=%d: authority doc dropped by MaxScore: %v", k, urlSet(on)) + } + } +} + +// TestPebbleBM25MaxScoreKIndependence pins the prod symptom: the top-10 must +// not depend on the requested k. Each k is independently oracle-checked, so +// the test cannot go vacuous if the fixture starts breaking at k=50. +func TestPebbleBM25MaxScoreKIndependence(t *testing.T) { + _, idx, sc := maxScoreAuthorityCorpus(t) + scored := idx.WithAuthority(sc) + + small, smallOracle := searchMaxScoreAB(t, scored, maxScoreAuthQuery, 10) + large, largeOracle := searchMaxScoreAB(t, scored, maxScoreAuthQuery, 50) + if len(small) != 10 || len(large) < 10 || len(largeOracle) < 10 { + t.Fatalf("want >=10 hits every arm, got k=10:%d k=50:%d oracle50:%d", + len(small), len(large), len(largeOracle)) + } + if !sameURLSet(urlSet(small), urlSet(smallOracle)) { + t.Errorf("k=10 diverges from lossless\n maxscore: %v\n lossless: %v", + urlSet(small), urlSet(smallOracle)) + } + if !sameURLSet(urlSet(large[:10]), urlSet(largeOracle[:10])) { + t.Errorf("k=50 diverges from lossless\n maxscore: %v\n lossless: %v", + urlSet(large[:10]), urlSet(largeOracle[:10])) + } + if !sameURLSet(urlSet(small), urlSet(large[:10])) { + t.Errorf("top-10 depends on k\n k=10: %v\n k=50: %v", urlSet(small), urlSet(large[:10])) + } +} + +// TestPebbleBM25MaxScoreAlphaTable sweeps alpha over the same fixture: the +// MaxScore arm must match the lossless oracle at every alpha, including +// alpha=0 where the authority-aware bound collapses to the raw one. +func TestPebbleBM25MaxScoreAlphaTable(t *testing.T) { + ps, _, _ := maxScoreAuthorityCorpus(t) + for _, alpha := range []float64{0, 0.5, 1, 2, 5} { + t.Run(fmt.Sprintf("alpha=%g", alpha), func(t *testing.T) { + sc := authority.New().WithAlpha(alpha) + sc.SetSubdomainCounts(map[string]int{"jiali.sbs": 200000}) + // WithAuthority mutates in place, so each subtest gets its own handle. + on, off := searchMaxScoreAB(t, NewPebbleBM25(ps).WithAuthority(sc), maxScoreAuthQuery, 10) + if !sameURLSet(urlSet(on), urlSet(off)) { + t.Errorf("MaxScore diverges from lossless\n maxscore: %v\n lossless: %v", + urlSet(on), urlSet(off)) + } + }) + } +} + +// TestPebbleBM25MaxScorePrunes pins that the optimization still prunes at the +// production alpha: the truncated scan must under-score at least one hit +// relative to the lossless arm. Goes red if MaxScore stops breaking. +func TestPebbleBM25MaxScorePrunes(t *testing.T) { + _, idx, sc := maxScoreDeepPruneCorpus(t) + scored := idx.WithAuthority(sc) + t.Setenv("COSIFT_BM25_MIN_IDF", "0") + + on, off := searchMaxScoreAB(t, scored, "zeta koppa", 10) + if len(on) != 10 || len(off) != 10 { + t.Fatalf("want 10 hits both arms, got on=%d off=%d", len(on), len(off)) + } + if !sameURLSet(urlSet(on), urlSet(off)) { + t.Fatalf("deep-prune fixture must not change top-k membership\n maxscore: %v\n lossless: %v", + urlSet(on), urlSet(off)) + } + lossless := make(map[string]float64, len(off)) + for _, h := range off { + lossless[h.URL] = h.Score + } + for _, h := range on { + if h.Score < lossless[h.URL]-1e-9 { + return + } + } + t.Errorf("MaxScore never broke: every hit scored identically to the lossless arm (%v)", urlSet(on)) +} + +// TestPebbleBM25MaxScorePhraseQuery — theta is a k-th over all scored docs, so +// it cannot threshold the phrase-filtered subset; the break must be skipped. +func TestPebbleBM25MaxScorePhraseQuery(t *testing.T) { + ps, _, sc := maxScoreDeepPruneCorpus(t) + t.Setenv("COSIFT_BM25_MIN_IDF", "0") + + for _, c := range []struct { + name string + scorer *authority.Scorer + }{{"no-authority", nil}, {"authority", sc}} { + t.Run(c.name, func(t *testing.T) { + on, off := searchMaxScoreAB(t, NewPebbleBM25(ps).WithAuthority(c.scorer), maxScorePhraseQuery, 10) + if !containsURL(off, maxScorePhraseURL) { + t.Fatalf("oracle lost the phrase doc; fixture is wrong: %v", urlSet(off)) + } + if !sameURLSet(urlSet(on), urlSet(off)) { + t.Errorf("phrase query diverges from lossless\n maxscore: %v\n lossless: %v", + urlSet(on), urlSet(off)) + } + }) + } +} + +// TestPebbleBM25TopKPoolAuthorityRiser pins the pool path's authority-aware +// bounds: a doc far outside the raw-score top-k whose raw*(1+alpha) clears the +// k-th raw score must still be resolved and must displace into the top-k. +func TestPebbleBM25TopKPoolAuthorityRiser(t *testing.T) { + ps, idx := newPebbleBM25(t) + const riserURL = "https://www.nih.gov/riser" + for i := 0; i < 60; i++ { + upsertAndIndex(t, ps, idx, fmt.Sprintf("https://spam%d.jiali.sbs/p", i), "spam", + "gopher body prose"+strings.Repeat(" pad", i)) + } + upsertAndIndex(t, ps, idx, riserURL, "riser", "gopher body prose"+strings.Repeat(" pad", 44)) + sc := authority.New() + sc.SetSubdomainCounts(map[string]int{"jiali.sbs": 200000}) + scored := idx.WithAuthority(sc) + t.Setenv("COSIFT_BM25_TOPK_POOL_FACTOR", "10") + + pooled, lossless := searchAB(t, scored, "gopher", 5) + if len(pooled) != 5 || len(lossless) != 5 { + t.Fatalf("want 5 hits both arms, got pool=%d lossless=%d", len(pooled), len(lossless)) + } + if !containsURL(lossless, riserURL) { + t.Fatalf("oracle lost the riser; fixture is wrong: %v", urlSet(lossless)) + } + if !containsURL(pooled, riserURL) { + t.Errorf("pool dropped the authority riser: %v", urlSet(pooled)) + } +} + +// TestPebbleBM25MaxAuthorityMult — the pruning-bound ceiling shared by the +// MaxScore loop and resolveTopKPool. +func TestPebbleBM25MaxAuthorityMult(t *testing.T) { + _, idx := newPebbleBM25(t) + if got := idx.maxAuthorityMult(); got != 1.0 { + t.Errorf("no authority: got %v want 1", got) + } + for _, c := range []struct{ alpha, want float64 }{{0, 1}, {0.5, 1.5}, {2, 3}} { + _, h := newPebbleBM25(t) + if got := h.WithAuthority(authority.New().WithAlpha(c.alpha)).maxAuthorityMult(); got != c.want { + t.Errorf("alpha=%v: got %v want %v", c.alpha, got, c.want) + } + } +} + // TestKthLargest — partial heap-select helper used by MaxScore early-stop. func TestKthLargest(t *testing.T) { cases := []struct { @@ -381,19 +626,7 @@ func upsertAndIndex(t testing.TB, ps *store.PebbleStore, idx *PebbleBM25, url, t // returns both hit lists. func searchAB(t *testing.T, idx *PebbleBM25, q string, k int) ([]Hit, []Hit) { t.Helper() - ctx := context.Background() - t.Setenv("COSIFT_BM25_DISABLE_TOPK_POOL", "") - pooled, err := idx.Search(ctx, q, k) - if err != nil { - t.Fatalf("search (pool on): %v", err) - } - t.Setenv("COSIFT_BM25_DISABLE_TOPK_POOL", "1") - lossless, err := idx.Search(ctx, q, k) - if err != nil { - t.Fatalf("search (pool off): %v", err) - } - t.Setenv("COSIFT_BM25_DISABLE_TOPK_POOL", "") - return pooled, lossless + return searchEnvAB(t, idx, "COSIFT_BM25_DISABLE_TOPK_POOL", q, k) } // TestPebbleBM25TopKPoolPreservesMembership pins the pool refactor: results @@ -660,6 +893,55 @@ func TestTopCandidates(t *testing.T) { } } +// BenchmarkPebbleBM25MaxScoreAuthority — cost of the authority-aware bound: +// "band" is suppressed by maxMult, "deep-prune" breaks either way. +func BenchmarkPebbleBM25MaxScoreAuthority(b *testing.B) { + ps, idx := newPebbleBM25(b) + ctx := context.Background() + + for i := 0; i < 20000; i++ { + text := "filler body prose text content" + if i%20 == 0 { + text += " rare" + } + if i%5 < 2 { + text += " mid mid" + } + upsertAndIndex(b, ps, idx, fmt.Sprintf("https://bench%d.example.com/d", i), "bench doc", text) + } + b.Setenv("COSIFT_BM25_MIN_IDF", "0") + b.Setenv("COSIFT_BM25_TOPK_POOL_FACTOR", "200") + + for _, q := range []struct{ name, query string }{ + {"band", "rare mid"}, + {"deep-prune", "rare filler"}, + {"phrase", `rare "mid mid"`}, + } { + for _, a := range []struct { + name string + scorer *authority.Scorer + }{ + {"no-authority", nil}, + {"authority", authority.New()}, + } { + b.Run(q.name+"/"+a.name, func(b *testing.B) { + scored := idx.WithAuthority(a.scorer) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + hits, err := scored.Search(ctx, q.query, 10) + if err != nil { + b.Fatal(err) + } + if len(hits) != 10 { + b.Fatalf("want 10 hits, got %d", len(hits)) + } + } + }) + } + } +} + // BenchmarkPebbleBM25Search — allocation evidence for the top-k pool: a // common term matching most of a synthetic corpus, pool vs resolve-all. func BenchmarkPebbleBM25Search(b *testing.B) { From 0a6f73101350c775e91f742123cd181412f04a84 Mon Sep 17 00:00:00 2001 From: Andrei Marinescu Date: Tue, 15 Sep 2026 19:26:29 +0300 Subject: [PATCH 2/3] index: floor the top-k resolution pool at a k-independent size The prod A/B for the MaxScore fix came back negative: k-dependence was unchanged (51/60 goldens before, 53/59 after) and p50 at k=10 went 56ms -> 184ms. Measuring why found the real cause, and it is not MaxScore. poolCap = factor*k makes the resolved candidate set proportional to k. When the cap binds, k=10 and k=50 sample the same candidates to different depths and cannot agree. From the cap-bind log at factor 200: a median bound query resolved 2,000 of ~68,500 in-band candidates, under 3%; the worst resolved 2,000 of 403,943. At k=50 the pool is 10,000 -- a 5x deeper look at the same set. That difference is the k-dependence. Cap-bind events were essentially unchanged across the two arms (15 vs 13 over comparable windows), which is what you would expect if the two approximations are independent -- they are. This contradicts the 2026-09-12 reading that "MaxScore dominates": that came from raising the factor 50 -> 200 and seeing little movement, which shows raising the factor does not help, not that the pool path is innocent. Adds COSIFT_BM25_TOPK_POOL_MIN, an absolute floor on the pool. Default 0 keeps today's behaviour exactly, so this is inert until configured. Also gates the MaxScore authority bound behind COSIFT_BM25_MAXSCORE_AUTHORITY (default on, =0 selects the old bound) so the two changes can be measured independently on the box without rebuilding. TestPebbleBM25PoolMinMakesTopKIndependentOfK reproduces the defect on a 400-doc fixture and pins the fix. It fails first if the fixture stops reproducing k-dependence, so it cannot go vacuous. Mutation: making bm25TopKPoolCap ignore the floor fails it with "top-10 still depends on k with the pool floor set". The floor's production value is not settled -- a deeper pool costs resolution work, and that trade is the next measurement. --- internal/index/pebble_bm25.go | 40 ++++++++++++++++++- internal/index/pebble_bm25_test.go | 64 ++++++++++++++++++++++++++++++ 2 files changed, 102 insertions(+), 2 deletions(-) diff --git a/internal/index/pebble_bm25.go b/internal/index/pebble_bm25.go index 3e12e08..b25ea9e 100644 --- a/internal/index/pebble_bm25.go +++ b/internal/index/pebble_bm25.go @@ -55,6 +55,39 @@ func bm25TopKPoolFactor() int { return 50 } +// bm25TopKPoolMin floors the resolution pool at a k-independent size. +// +// factor*k alone makes ranking depend on k: when the cap binds, k=10 and k=50 +// sample the same candidate set to different depths, so one query returns two +// different top-10s. Measured on prod 2026-09-15 at factor 200, a median +// cap-bound query resolved 2,000 of ~68,500 in-band candidates and 51 of 60 +// goldens disagreed between k=10 and k=50. A floor makes the candidate universe +// identical across k. 0 keeps the pure factor*k behaviour. +func bm25TopKPoolMin() int { + if v := os.Getenv("COSIFT_BM25_TOPK_POOL_MIN"); v != "" { + if n, err := strconv.Atoi(v); err == nil && n >= 0 { + return n + } + } + return 0 +} + +// bm25TopKPoolCap is the resolution pool size for a query of depth k. +func bm25TopKPoolCap(k int) int { + c := bm25TopKPoolFactor() * k + if m := bm25TopKPoolMin(); c < m { + c = m + } + return c +} + +// bm25MaxScoreAuthorityBound reports whether the MaxScore early-termination +// bound accounts for the authority multiplier applied after the scan. 0 selects +// the pre-2026-09 behaviour, which prunes harder and is measurably less exact. +func bm25MaxScoreAuthorityBound() bool { + return os.Getenv("COSIFT_BM25_MAXSCORE_AUTHORITY") != "0" +} + // PebbleBM25 mirrors BM25 but reads from a PebbleStore. type PebbleBM25 struct { store *store.PebbleStore @@ -228,7 +261,10 @@ func (b *PebbleBM25) Search(ctx context.Context, q string, k int) ([]Hit, error) // optimization for benchmark-grade lossless ranking. Phrase queries opt // out: theta cannot threshold the phrase-filtered subset (empty results). maxScoreEnabled := os.Getenv("COSIFT_BM25_DISABLE_MAXSCORE") == "" && len(phrases) == 0 - maxMult := b.maxAuthorityMult() + maxMult := 1.0 + if bm25MaxScoreAuthorityBound() { + maxMult = b.maxAuthorityMult() + } remainingMax := 0.0 if maxScoreEnabled { for _, c := range active { @@ -343,7 +379,7 @@ const topKResolveSlack = 16 func (b *PebbleBM25) resolveTopKPool(ctx context.Context, scores map[int64]float64, phrases []string, k int) ([]Hit, error) { maxMult := b.maxAuthorityMult() - poolCap := bm25TopKPoolFactor() * k + poolCap := bm25TopKPoolCap(k) pool := topCandidates(scores, poolCap) if len(pool) == 0 { return nil, nil diff --git a/internal/index/pebble_bm25_test.go b/internal/index/pebble_bm25_test.go index e6f34a5..3ad8bd1 100644 --- a/internal/index/pebble_bm25_test.go +++ b/internal/index/pebble_bm25_test.go @@ -994,3 +994,67 @@ func BenchmarkPebbleBM25Search(b *testing.B) { }) } } + +// TestPebbleBM25PoolMinMakesTopKIndependentOfK pins the defect measured on prod +// 2026-09-15: with poolCap = factor*k, a cap-bound query returns a different +// top-10 at k=10 than at k=50 (51 of 60 goldens did). The floor makes the +// candidate universe identical across k, so the top-10 must agree. +func TestPebbleBM25PoolMinMakesTopKIndependentOfK(t *testing.T) { + ps, idx := newPebbleBM25(t) + // Authority spread is what lets a low-raw-score doc outrank a high one, so + // truncating the pool at different depths changes the answer. + // Embedded-trusted host scores 0.9 vs 0.5 for an unknown one: a 1.9x vs + // 1.5x multiplier at the default alpha, enough for a lower-raw-score doc to + // outrank a higher one once it is actually resolved. + sc := authority.New() + idx = idx.WithAuthority(sc) + + for i := 0; i < 400; i++ { + host := "plain.example" + if i%7 == 0 { + host = "en.wikipedia.org" + } + // Descending term frequency gives a descending raw-score ladder. + body := strings.Repeat("quantum ", 1+(400-i)/8) + strings.Repeat("filler ", i%5) + upsertAndIndex(t, ps, idx, fmt.Sprintf("https://%s/doc%03d", host, i), "Quantum", body) + } + + top := func(k int) []string { + hits, err := idx.Search(context.Background(), "quantum", k) + if err != nil { + t.Fatalf("search k=%d: %v", k, err) + } + if len(hits) > 10 { + hits = hits[:10] + } + return urlSet(hits) + } + + // factor 1 makes the cap bind hard: k=10 resolves 10 candidates, k=50 + // resolves 50, out of 400 scored. + t.Setenv("COSIFT_BM25_TOPK_POOL_FACTOR", "1") + t.Setenv("COSIFT_BM25_TOPK_POOL_MIN", "0") + if sameURLSet(top(10), top(50)) { + t.Fatal("fixture does not reproduce k-dependence: raise the authority spread or the corpus size") + } + + t.Setenv("COSIFT_BM25_TOPK_POOL_MIN", "400") + a, b := top(10), top(50) + if !sameURLSet(a, b) { + t.Errorf("top-10 still depends on k with the pool floor set:\n k=10 %v\n k=50 %v", a, b) + } +} + +func TestBM25TopKPoolCapHonoursFloor(t *testing.T) { + t.Setenv("COSIFT_BM25_TOPK_POOL_FACTOR", "200") + t.Setenv("COSIFT_BM25_TOPK_POOL_MIN", "10000") + for _, tc := range []struct{ k, want int }{{10, 10000}, {50, 10000}, {100, 20000}} { + if got := bm25TopKPoolCap(tc.k); got != tc.want { + t.Errorf("bm25TopKPoolCap(%d) = %d, want %d", tc.k, got, tc.want) + } + } + t.Setenv("COSIFT_BM25_TOPK_POOL_MIN", "0") + if got := bm25TopKPoolCap(10); got != 2000 { + t.Errorf("floor 0 must keep factor*k: got %d, want 2000", got) + } +} From 408cd7ee1b0211404be79f6ebbe16c06d7201d76 Mon Sep 17 00:00:00 2001 From: Andrei Marinescu Date: Tue, 15 Sep 2026 20:42:28 +0300 Subject: [PATCH 3/3] docs: record the k-dependence investigation and its three wrong turns Production measurement of the T0.5 branch came back negative, and finding out why turned up a second, larger defect that no BM25 change can reach. Writing it down so the next attempt does not repeat this one. Findings, all measured on the box at 15.7M docs: Time decay is the bigger defect. /search applies decay after retrieval over a k-sized candidate list, so a deeper fetch lets decay promote different documents into the top 10. On one binary at one moment: 50/59 goldens disagree with decay on, 0/59 with decay=0. On stock v0.2.5 it is 51/60 vs 19/60. Not fixed here, and not fixable from internal/index. BM25 is genuinely k-dependent underneath, 19/60, because two decisions branch on the caller's k: the MaxScore theta and the resolution pool. Flooring both at kEff takes it to 0/59. Depth 15 is the knee: mean overlap@10 vs the k=50 reference goes 0.9450 -> 0.9817 for +31% p50 and +9% p90, and it is where the tracker's example query flips from 0.20 to 1.00. Depth 50 costs 147ms p50 for no further accuracy. Also records why the first two attempts failed -- the authority-aware bound left theta k-shaped and cost 3.3x p50 for no symptom improvement, and flooring the pool alone left theta k-shaped too -- plus the /search fetchK widening rules, which anyone reasoning about "the k the index sees" needs first. No production defaults change: COSIFT_BM25_RANK_DEPTH is 0 unless set. --- docs/K-DEPENDENCE.md | 124 +++++++++++++++++++++++++++++ internal/index/pebble_bm25.go | 53 +++++++----- internal/index/pebble_bm25_test.go | 52 ++++++++---- 3 files changed, 193 insertions(+), 36 deletions(-) create mode 100644 docs/K-DEPENDENCE.md diff --git a/docs/K-DEPENDENCE.md b/docs/K-DEPENDENCE.md new file mode 100644 index 0000000..3c6066a --- /dev/null +++ b/docs/K-DEPENDENCE.md @@ -0,0 +1,124 @@ +# Why the same query returns different results at different k + +Measured on production 2026-09-15: 15,718,478 docs / 78,652,652 vectors, `POOL_FACTOR=200`, +authority alpha 2.0. Five binary arms, the 60 golden queries at k=10 and k=50. + +The short version: **there are two independent defects, and the one everybody was chasing is not the +one that matters most.** This document exists so the next person does not repeat the three wrong +turns below. + +## The symptom + +`GET /search?q=X&k=10` and `GET /search?q=X&k=50` disagree about the top 10. On the goldens, 51 of 60 +queries disagreed. `quantum gate teleportation logical qubits` was the reported example: wrong at +k=10, correct from k=15. + +## Defect 1 — time decay re-ranks a k-sized window (the bigger one) + +`serve_search.go` applies time decay *after* retrieval, over the candidate list it fetched. For a +plain query `fetchK == k`, so k=10 decays a 10-item list and k=50 decays a 50-item list. A deeper +list gives decay more to choose from, and different documents surface into the top 10. + +Isolating it, on one binary at one moment: + +| retriever | queries whose k=10 top-10 ≠ k=50 top-10 | +|---|---| +| `bm25+decay:180d` (the default) | 50/59 | +| `bm25` (`?decay=0`) | **0/59** | + +On stock v0.2.5, turning decay off takes 51/60 → 19/60. So decay accounts for roughly two thirds of +the reported symptom, and it is **not reachable from `internal/index`** — no BM25 change affects it. + +Magnitude, stock v0.2.5, k=10 against the k=50 reference: + +| | decay on | decay off | +|---|---|---| +| mean overlap@10 | 0.7133 | 0.9450 | +| queries with an identical top-10 *set* | 12/60 | 46/60 | + +**Not fixed.** The counterpart fix is to decay over a fixed-depth candidate window rather than a +k-sized one, so the input to decay stops depending on the caller's k. Not attempted. + +## Defect 2 — BM25 makes two k-shaped decisions (the real engine bug) + +With decay off, 19 of 60 goldens still disagree. Two places in `pebble_bm25.go` branch on the +caller's k: + +1. **The MaxScore break.** `theta := kthLargest(scores, k)` — a smaller k gives a *higher* theta, so + the scan stops earlier and produces a different score map before anything is ranked. +2. **The resolution pool.** `poolCap = factor * k` — a smaller k resolves fewer candidates. At + factor 200 and k=10 the pool is 2,000, while the cap-bind log shows a median bound query with + **68,497 in-band candidates** and a worst case of 403,943. Under 3% of eligible candidates get + resolved. + +Running both at `kEff = max(k, COSIFT_BM25_RANK_DEPTH)` and truncating to k afterwards makes the +top-k of any k ≤ depth a prefix of one ranking: **19/60 → 0/59, exactly zero.** + +### Choosing the depth + +k-sweep on stock v0.2.5 with decay off, each k's top-10 against the k=50 reference: + +| k | queries differing | mean overlap@10 | p50 | p90 | +|---|---|---|---|---| +| 10 | 19/60 | 0.9450 | 42 ms | 162 ms | +| **15** | 15/60 | **0.9817** | 55 ms | 176 ms | +| 20 | 16/60 | 0.9800 | 66 ms | 184 ms | +| 30 | 14/60 | 0.9850 | 94 ms | 213 ms | +| 50 | 0/60 | 1.0000 | 147 ms | 324 ms | + +The knee is at **15**: it captures most of the accuracy for +31% p50 and +9% p90, and it is where +`quantum gate teleportation logical qubits` goes from 0.20 to 1.00. Beyond 15 the curve is flat until +50, which is the reference and therefore trivially perfect. + +`COSIFT_BM25_RANK_DEPTH` defaults to 0 (off, current behaviour). Depth 15 has **not** been measured +as its own production arm. + +## Three wrong turns, recorded so they are not repeated + +**1. "MaxScore × authority dominates."** The engine tracker concluded this on 2026-09-12 after +raising the pool factor 50 → 200 and seeing k-dependence barely move (48/60 → 46/60). The correct +reading is *raising the factor does not help*, not *MaxScore is the cause*. Cap-bind rate was +essentially identical across every arm here (15 vs 13 over comparable windows). + +**2. Making the MaxScore bound authority-aware does not fix it, and is expensive.** The bound +`remainingMax < theta` ignores the authority multiplier applied after the scan, so with alpha 2.0 it +is 3× too loose and drops genuine top-k members. Correcting it to `remainingMax*maxMult < theta` is +provably more exact in isolation — against a lossless oracle on a small corpus where the pool never +binds, agreement with exact ranking goes 0.8362 → 0.9793. On production it measured: + +| | k-dependence | k=10 p50 | k=10 p90 | +|---|---|---|---| +| control | 51/60 | 56 ms | 168 ms | +| authority-aware bound | 53/59 | 184 ms | 943 ms | + +No improvement to the symptom, 3.3× p50 and 5.6× p90. It leaves theta k-shaped, so it changes *where* +the scan stops without changing *that it stops somewhere k-dependent*. The correctness gain is real +but is not separately measurable at production scale, and the cost is not. + +**3. Flooring the pool alone does not fix it either.** An absolute floor on `poolCap` makes the +candidate universe k-independent but leaves theta k-shaped, so the score map still differs before the +pool is built: 48/59, and 210 ms p50. Both floors are needed, which is why the knob is a single rank +depth rather than a pool minimum. + +## Things worth knowing that came out of this + +- **`/search` does not pass the caller's k to the index.** It computes `fetchK` from `keepCap`, which + widens 5× for include/exclude filters, 10× for a date filter, a flat 300 for `site=`, and 2× for + rerank, capped at 500. For a plain query `fetchK == k`. Anything reasoning about "the k the index + sees" has to start there. +- **`COSIFT_BM25_DISABLE_MAXSCORE=1` is a usable oracle on small corpora only.** It is exact by + construction and far too slow at 15.7M documents. +- **Latency numbers here are not comparable across sessions.** The later arms ran alongside the 16:00 + UTC scheduled snapshot; measured contention on an unchanged binary was +7% p50 / +8% p90. The + k-sweep ran on a quiet box, which is why its k=10 p50 reads 42 ms against 56 ms elsewhere. + +## Reproducing + +```sh +# On the box. decay=0 is the flag that separates the two defects. +python3 cosift-golden-capture.py -q cosift-golden-queries.txt -o out.jsonl \ + -b http://127.0.0.1:7777 -k 10,50 -t 75 -x decay=0 +``` + +Then compare each query's k=10 top-10 against its k=50 top-10. Raw captures for all five arms are in +the monorepo at `tools/golden/2026-09-15-t0-prod/`. diff --git a/internal/index/pebble_bm25.go b/internal/index/pebble_bm25.go index b25ea9e..0ec3eef 100644 --- a/internal/index/pebble_bm25.go +++ b/internal/index/pebble_bm25.go @@ -55,16 +55,21 @@ func bm25TopKPoolFactor() int { return 50 } -// bm25TopKPoolMin floors the resolution pool at a k-independent size. +// bm25RankDepth floors the depth at which ranking decisions are made, so they +// stop depending on the caller's k. // -// factor*k alone makes ranking depend on k: when the cap binds, k=10 and k=50 -// sample the same candidate set to different depths, so one query returns two -// different top-10s. Measured on prod 2026-09-15 at factor 200, a median -// cap-bound query resolved 2,000 of ~68,500 in-band candidates and 51 of 60 -// goldens disagreed between k=10 and k=50. A floor makes the candidate universe -// identical across k. 0 keeps the pure factor*k behaviour. -func bm25TopKPoolMin() int { - if v := os.Getenv("COSIFT_BM25_TOPK_POOL_MIN"); v != "" { +// Two decisions are k-shaped. The MaxScore break compares against +// kthLargest(scores, k): a small k gives a higher theta, so the scan stops +// earlier and produces a different score map. The resolution pool is factor*k: +// a small k resolves fewer candidates. Measured on prod 2026-09-15 at factor +// 200, 51 of 60 goldens returned a different top-10 at k=10 than at k=50, and a +// median cap-bound query resolved 2,000 of ~68,500 in-band candidates. +// +// Running both at kEff = max(k, depth) and truncating afterwards makes the +// top-k of any k <= depth a prefix of the same ranking. The cost is that every +// query pays the depth-k price. 0 keeps the k-shaped behaviour. +func bm25RankDepth() int { + if v := os.Getenv("COSIFT_BM25_RANK_DEPTH"); v != "" { if n, err := strconv.Atoi(v); err == nil && n >= 0 { return n } @@ -72,13 +77,15 @@ func bm25TopKPoolMin() int { return 0 } -// bm25TopKPoolCap is the resolution pool size for a query of depth k. -func bm25TopKPoolCap(k int) int { - c := bm25TopKPoolFactor() * k - if m := bm25TopKPoolMin(); c < m { - c = m +// bm25EffectiveK is the internal ranking depth for a caller asking for k. +func bm25EffectiveK(k int) int { + if k <= 0 { + return k } - return c + if d := bm25RankDepth(); d > k { + return d + } + return k } // bm25MaxScoreAuthorityBound reports whether the MaxScore early-termination @@ -261,6 +268,7 @@ func (b *PebbleBM25) Search(ctx context.Context, q string, k int) ([]Hit, error) // optimization for benchmark-grade lossless ranking. Phrase queries opt // out: theta cannot threshold the phrase-filtered subset (empty results). maxScoreEnabled := os.Getenv("COSIFT_BM25_DISABLE_MAXSCORE") == "" && len(phrases) == 0 + kEff := bm25EffectiveK(k) maxMult := 1.0 if bm25MaxScoreAuthorityBound() { maxMult = b.maxAuthorityMult() @@ -273,8 +281,8 @@ func (b *PebbleBM25) Search(ctx context.Context, q string, k int) ([]Hit, error) } for i, c := range active { // theta is raw; an unseen doc's final ceiling is remainingMax*maxMult. - if maxScoreEnabled && i > 0 && len(scores) >= k { - theta := kthLargest(scores, k) + if maxScoreEnabled && i > 0 && len(scores) >= kEff { + theta := kthLargest(scores, kEff) if remainingMax*maxMult < theta { break } @@ -328,7 +336,14 @@ func (b *PebbleBM25) Search(ctx context.Context, q string, k int) ([]Hit, error) // resolved; COSIFT_BM25_DISABLE_TOPK_POOL restores the resolve-all // path (also taken for k<=0 = "return everything"). if k > 0 && os.Getenv("COSIFT_BM25_DISABLE_TOPK_POOL") == "" { - return b.resolveTopKPool(ctx, scores, phrases, k) + hits, err := b.resolveTopKPool(ctx, scores, phrases, kEff) + if err != nil { + return nil, err + } + if len(hits) > k { + hits = hits[:k] + } + return hits, nil } hits := make([]Hit, 0, len(scores)) @@ -379,7 +394,7 @@ const topKResolveSlack = 16 func (b *PebbleBM25) resolveTopKPool(ctx context.Context, scores map[int64]float64, phrases []string, k int) ([]Hit, error) { maxMult := b.maxAuthorityMult() - poolCap := bm25TopKPoolCap(k) + poolCap := bm25TopKPoolFactor() * k pool := topCandidates(scores, poolCap) if len(pool) == 0 { return nil, nil diff --git a/internal/index/pebble_bm25_test.go b/internal/index/pebble_bm25_test.go index 3ad8bd1..db1f8cf 100644 --- a/internal/index/pebble_bm25_test.go +++ b/internal/index/pebble_bm25_test.go @@ -995,11 +995,12 @@ func BenchmarkPebbleBM25Search(b *testing.B) { } } -// TestPebbleBM25PoolMinMakesTopKIndependentOfK pins the defect measured on prod -// 2026-09-15: with poolCap = factor*k, a cap-bound query returns a different -// top-10 at k=10 than at k=50 (51 of 60 goldens did). The floor makes the -// candidate universe identical across k, so the top-10 must agree. -func TestPebbleBM25PoolMinMakesTopKIndependentOfK(t *testing.T) { +// TestPebbleBM25RankDepthMakesTopKIndependentOfK pins the defect measured on +// prod 2026-09-15: 51 of 60 goldens returned a different top-10 at k=10 than at +// k=50. Two decisions are k-shaped -- the MaxScore theta and the resolution +// pool -- and flooring only the pool still left 48 of 59 disagreeing on the +// box. Running both at kEff makes the top-10 a prefix of the same ranking. +func TestPebbleBM25RankDepthMakesTopKIndependentOfK(t *testing.T) { ps, idx := newPebbleBM25(t) // Authority spread is what lets a low-raw-score doc outrank a high one, so // truncating the pool at different depths changes the answer. @@ -1033,28 +1034,45 @@ func TestPebbleBM25PoolMinMakesTopKIndependentOfK(t *testing.T) { // factor 1 makes the cap bind hard: k=10 resolves 10 candidates, k=50 // resolves 50, out of 400 scored. t.Setenv("COSIFT_BM25_TOPK_POOL_FACTOR", "1") - t.Setenv("COSIFT_BM25_TOPK_POOL_MIN", "0") + t.Setenv("COSIFT_BM25_RANK_DEPTH", "0") if sameURLSet(top(10), top(50)) { t.Fatal("fixture does not reproduce k-dependence: raise the authority spread or the corpus size") } - t.Setenv("COSIFT_BM25_TOPK_POOL_MIN", "400") + t.Setenv("COSIFT_BM25_RANK_DEPTH", "50") a, b := top(10), top(50) if !sameURLSet(a, b) { - t.Errorf("top-10 still depends on k with the pool floor set:\n k=10 %v\n k=50 %v", a, b) + t.Errorf("top-10 still depends on k at rank depth 50:\n k=10 %v\n k=50 %v", a, b) } } -func TestBM25TopKPoolCapHonoursFloor(t *testing.T) { - t.Setenv("COSIFT_BM25_TOPK_POOL_FACTOR", "200") - t.Setenv("COSIFT_BM25_TOPK_POOL_MIN", "10000") - for _, tc := range []struct{ k, want int }{{10, 10000}, {50, 10000}, {100, 20000}} { - if got := bm25TopKPoolCap(tc.k); got != tc.want { - t.Errorf("bm25TopKPoolCap(%d) = %d, want %d", tc.k, got, tc.want) +func TestBM25EffectiveKHonoursRankDepth(t *testing.T) { + t.Setenv("COSIFT_BM25_RANK_DEPTH", "50") + for _, tc := range []struct{ k, want int }{{10, 50}, {50, 50}, {100, 100}, {0, 0}, {-1, -1}} { + if got := bm25EffectiveK(tc.k); got != tc.want { + t.Errorf("bm25EffectiveK(%d) = %d, want %d", tc.k, got, tc.want) } } - t.Setenv("COSIFT_BM25_TOPK_POOL_MIN", "0") - if got := bm25TopKPoolCap(10); got != 2000 { - t.Errorf("floor 0 must keep factor*k: got %d, want 2000", got) + t.Setenv("COSIFT_BM25_RANK_DEPTH", "0") + if got := bm25EffectiveK(10); got != 10 { + t.Errorf("depth 0 must leave k alone: got %d, want 10", got) + } +} + +// A caller asking for k must still get at most k hits when the engine ranked +// deeper internally. +func TestPebbleBM25RankDepthStillReturnsK(t *testing.T) { + ps, idx := newPebbleBM25(t) + for i := 0; i < 120; i++ { + upsertAndIndex(t, ps, idx, fmt.Sprintf("https://e.example/d%03d", i), "Quantum", + strings.Repeat("quantum ", 1+(120-i)/4)) + } + t.Setenv("COSIFT_BM25_RANK_DEPTH", "100") + hits, err := idx.Search(context.Background(), "quantum", 5) + if err != nil { + t.Fatalf("search: %v", err) + } + if len(hits) != 5 { + t.Errorf("asked for k=5 at rank depth 100, got %d hits", len(hits)) } }