fix(dash): stop the dashboard retry storm that took the service down

dash/api_test.go

@@ -1116,3 +1116,27 @@ func TestUINeverShowsABareCost(t *testing.T) {
 		}
 	}
 }
+
+// The 5 s response cache is the reason /api/facets survived at ~1.5s per call: requested on
+// every tab switch, one miss followed by eleven hits reads as a fast endpoint. X-Cache is what
+// lets a measurement tell the two apart, so it is asserted rather than assumed — an A/B that
+// silently compares a miss against a hit reports a 158x "optimisation" nobody made.
+func TestCachedRoutesLabelHitsAndMisses(t *testing.T) {
+	a, rec := newTestAPI(t, Options{})
+	seed(t, rec, mkEvent(time.Now().UnixMilli(), "sess-1", "aws/claude-sonnet-5", 1000, 800))
+	for _, path := range []string{"/api/stats", "/api/facets", "/api/components"} {
+		w, _ := get(t, a, path, "")
+		if got := w.Header().Get("X-Cache"); got != "miss" {
+			t.Errorf("%s first call: X-Cache = %q, want miss", path, got)
+		}
+		w, _ = get(t, a, path, "")
+		if got := w.Header().Get("X-Cache"); got != "hit" {
+			t.Errorf("%s second call: X-Cache = %q, want hit", path, got)
+		}
+	}
+	// An uncached route must not claim either, or the header becomes noise.
+	w, _ := get(t, a, "/api/requests?limit=1", "")
+	if got := w.Header().Get("X-Cache"); got != "" {
+		t.Errorf("/api/requests: X-Cache = %q, want absent", got)
+	}
+}

dash/archive.go

@@ -113,7 +113,7 @@ func (d *DB) coldSessions(idleBefore int64, kind string, limit int) ([]coldCandi
 	      FROM requests r WHERE r.session_id <> '' ` + notYet + `
 	      GROUP BY r.session_id HAVING MAX(r.ts) < ?
 	      ORDER BY MAX(r.ts) ASC LIMIT ?`
-	rows, err := d.sql.Query(q, idleBefore, limit)
+	rows, err := d.sql.QueryContext(d.readCtx(), q, idleBefore, limit)
 	if err != nil {
 		return nil, err
 	}
@@ -139,7 +139,7 @@ func (d *DB) oldestLocalSessions(limit int) ([]coldCandidate, error) {
 
 // scanCandidates runs a candidate query and scans its rows.
 func (d *DB) scanCandidates(q string, args ...any) ([]coldCandidate, error) {
-	rows, err := d.sql.Query(q, args...)
+	rows, err := d.sql.QueryContext(d.readCtx(), q, args...)
 	if err != nil {
 		return nil, err
 	}
@@ -403,7 +403,7 @@ func (d *DB) ArchivedSessions(f Filter, limit int) ([]ArchivedSession, error) {
 	}
 	q += ` ORDER BY last_ts DESC LIMIT ?`
 	args = append(args, limit)
-	rows, err := d.sql.Query(q, args...)
+	rows, err := d.sql.QueryContext(d.readCtx(), q, args...)
 	if err != nil {
 		return nil, err
 	}
@@ -424,7 +424,7 @@ func (d *DB) ArchivedSessions(f Filter, limit int) ([]ArchivedSession, error) {
 // ArchivedSessionByID reads one index row.
 func (d *DB) ArchivedSessionByID(sessionID string) (ArchivedSession, error) {
 	var a ArchivedSession
-	err := d.sql.QueryRow(`SELECT session_id,tenant_id,first_ts,last_ts,requests,
+	err := d.sql.QueryRowContext(d.readCtx(), `SELECT session_id,tenant_id,first_ts,last_ts,requests,
 	  content_path,content_bytes,full_path,full_bytes,archived_at,remote
 	  FROM archived_sessions WHERE session_id = ?`, sessionID).Scan(
 		&a.SessionID, &a.TenantID, &a.FirstTS, &a.LastTS, &a.Requests,

dash/bench.go

@@ -207,7 +207,7 @@ type BenchArm struct {
 
 // BenchRuns returns every ingested run with its per-arm aggregates.
 func (d *DB) BenchRuns() ([]*BenchRun, error) {
-	rows, err := d.sql.Query(`SELECT id, name, ts, dataset, model, summary FROM bench_runs ORDER BY ts DESC`)
+	rows, err := d.sql.QueryContext(d.readCtx(), `SELECT id, name, ts, dataset, model, summary FROM bench_runs ORDER BY ts DESC`)
 	if err != nil {
 		return nil, err
 	}
@@ -236,7 +236,7 @@ func (d *DB) BenchRuns() ([]*BenchRun, error) {
 }
 
 func (d *DB) benchArms(runID int64) ([]BenchArm, error) {
-	rows, err := d.sql.Query(`SELECT arm, COUNT(*),
+	rows, err := d.sql.QueryContext(d.readCtx(), `SELECT arm, COUNT(*),
 		SUM(CASE WHEN exception = 0 THEN 1 ELSE 0 END),
 		SUM(CASE WHEN reward >= 1 THEN 1 ELSE 0 END),
 		AVG(reward), AVG(steps), SUM(cost_usd), AVG(cost_usd), SUM(norm_cost_usd),
@@ -295,7 +295,7 @@ func (d *DB) BenchTasks(runID int64, arm string) ([]*BenchTask, error) {
 		args = append(args, arm)
 	}
 	q += " ORDER BY task, arm"
-	rows, err := d.sql.Query(q, args...)
+	rows, err := d.sql.QueryContext(d.readCtx(), q, args...)
 	if err != nil {
 		return nil, err
 	}

dash/cachehistory.go

@@ -86,11 +86,22 @@ func (d *DB) CachesplitHistoricalUSD(f Filter, p modelinfo.Pricer) (CachesplitHi
 	// ROW_NUMBER ranks EVERY row in the table by session, unfiltered, so rn=1 means exactly
 	// what the old subquery's NOT EXISTS meant — "nothing earlier in this session, filtered or
 	// not" — before the outer WHERE narrows to the rows this query actually values.
-	rows, err := d.sql.Query(`WITH s AS (
-		SELECT r.*, ROW_NUMBER() OVER (PARTITION BY r.session_id ORDER BY r.ts, r.id) AS rn
+	// The ranking CTE emits (id, rn) and is joined back on the primary key, rather than pulling
+	// `r.*` through the PARTITION BY sort for the sake of one integer per row. Same rewrite as
+	// CompactionResets in overview.go, same reason, and the same caveat: the CTE stays UNFILTERED,
+	// which is the whole correctness argument above — rn=1 has to mean "nothing earlier in this
+	// session, filtered or not", so the filter may only ever be applied outside it.
+	//
+	// Measured: 199-208 ms to 38-44 ms on the 16,444-request corpus, and 1,103 ms to 270 ms
+	// read-only against the production database. Results identical, checked on the 6,697 rows
+	// this predicate actually returns there and on 16,167 rows with the split/cache filters
+	// dropped, plus each tenant scope separately -- the frozen corpus returns ZERO rows for the
+	// shipped predicate, so a check against it alone proves nothing.
+	rows, err := d.sql.QueryContext(d.readCtx(), `WITH rn AS (
+		SELECT r.id AS rid, ROW_NUMBER() OVER (PARTITION BY r.session_id ORDER BY r.ts, r.id) AS n
 		FROM requests r
-	) SELECT r.model, r.cache_read, r.cache_write FROM s r
-		WHERE `+cond+` AND r.split_stable_tokens = 0 AND r.cache_read > 0 AND r.rn = 1`, args...)
+	) SELECT r.model, r.cache_read, r.cache_write FROM requests r JOIN rn ON rn.rid = r.id AND rn.n = 1
+		WHERE `+cond+` AND r.split_stable_tokens = 0 AND r.cache_read > 0`, args...)
 	if err != nil {
 		return out, err
 	}
@@ -162,11 +173,22 @@ func (d *DB) CachesplitHistoricalUSDByTenant(since int64, p modelinfo.Pricer) (m
 		return out, err
 	}
 	cond, args := (Filter{Since: since, TenantAll: true}).where()
-	rows, err := d.sql.Query(`WITH s AS (
-		SELECT r.*, ROW_NUMBER() OVER (PARTITION BY r.session_id ORDER BY r.ts, r.id) AS rn
+	// The ranking CTE emits (id, rn) and is joined back on the primary key, rather than pulling
+	// `r.*` through the PARTITION BY sort for the sake of one integer per row. Same rewrite as
+	// CompactionResets in overview.go, same reason, and the same caveat: the CTE stays UNFILTERED,
+	// which is the whole correctness argument above — rn=1 has to mean "nothing earlier in this
+	// session, filtered or not", so the filter may only ever be applied outside it.
+	//
+	// Measured: 199-208 ms to 38-44 ms on the 16,444-request corpus, and 1,103 ms to 270 ms
+	// read-only against the production database. Results identical, checked on the 6,697 rows
+	// this predicate actually returns there and on 16,167 rows with the split/cache filters
+	// dropped, plus each tenant scope separately -- the frozen corpus returns ZERO rows for the
+	// shipped predicate, so a check against it alone proves nothing.
+	rows, err := d.sql.QueryContext(d.readCtx(), `WITH rn AS (
+		SELECT r.id AS rid, ROW_NUMBER() OVER (PARTITION BY r.session_id ORDER BY r.ts, r.id) AS n
 		FROM requests r
-	) SELECT r.tenant_id, r.model, r.cache_read, r.cache_write FROM s r
-		WHERE `+cond+` AND r.split_stable_tokens = 0 AND r.cache_read > 0 AND r.rn = 1`, args...)
+	) SELECT r.tenant_id, r.model, r.cache_read, r.cache_write FROM requests r JOIN rn ON rn.rid = r.id AND rn.n = 1
+		WHERE `+cond+` AND r.split_stable_tokens = 0 AND r.cache_read > 0`, args...)
 	if err != nil {
 		return out, err
 	}
@@ -234,7 +256,7 @@ func (d *DB) CachesplitHistoricalUSDByTenant(since int64, p modelinfo.Pricer) (m
 // instead of assumed, and it is what decides the refusal rather than merely reporting on it.
 func (d *DB) CachesplitSizeSpread() (map[string][2]int, error) {
 	out := map[string][2]int{}
-	rows, err := d.sql.Query(`SELECT model, MIN(split_stable_tokens), MAX(split_stable_tokens)
+	rows, err := d.sql.QueryContext(d.readCtx(), `SELECT model, MIN(split_stable_tokens), MAX(split_stable_tokens)
 		FROM requests WHERE split_stable_tokens > 0 GROUP BY model`)
 	if err != nil {
 		return nil, err

dash/campaignsavings.go

@@ -117,7 +117,7 @@ func (d *DB) CampaignRealSavings(strategyIDs, tenantIDs []string, since int64) (
 			costArgs = append(costArgs, id)
 		}
 		costArgs = append(costArgs, since)
-		costRows, err := d.sql.Query(`SELECT tenant_id,
+		costRows, err := d.sql.QueryContext(d.readCtx(), `SELECT tenant_id,
 				CAST(strftime('%H', ts/1000, 'unixepoch') AS INTEGER) h,
 				COUNT(*), COALESCE(SUM(cost_usd),0)
 			FROM requests WHERE keepalive = 1 AND keepalive_strategy_id IN (`+
@@ -175,7 +175,7 @@ func (d *DB) CampaignRealSavings(strategyIDs, tenantIDs []string, since int64) (
 			savingArgs = append(savingArgs, id)
 		}
 		savingArgs = append(savingArgs, since)
-		savingRows, err := d.sql.Query(`SELECT r.tenant_id,
+		savingRows, err := d.sql.QueryContext(d.readCtx(), `SELECT r.tenant_id,
 				CAST(strftime('%H', r.ts/1000, 'unixepoch') AS INTEGER) h,
 				COUNT(*), COUNT(DISTINCT r.ts/86400000), `+savedExpr+`
 			FROM requests r WHERE r.keepalive = 0 AND r.tenant_id IN (`+

dash/componentscache_test.go

@@ -7,12 +7,18 @@ import (
 	"time"
 )
 
-// TestComponentsCached guards the fix: /api/components had no cache at all (unlike
-// /api/stats and /api/facets), so every load re-ran all five of its queries cold —
-// measured ~4.4s on a corpus matching production scale. A second call within
-// dashCacheTTL must be served from cache, not re-run against the store: seed one
-// component row, read it, seed a second, and confirm the immediate re-read still
-// reports only the first (the cached body), while a read past the TTL sees both.
+// TestComponentsCached guards the fix: /api/components had no cache at all (unlike /api/stats and
+// /api/facets), so every load re-ran all five of its queries cold — measured ~4.4s on a corpus
+// matching production scale, ~6s on the live one.
+//
+// It now guards the STALE-WHILE-REVALIDATE contract too, which is a deliberate change from "past
+// the TTL, block and return fresh". A TTL of 5s in front of a 6s query cached nothing — every
+// reader missed and paid full price, and nineteen of them at once turned a query that fits inside
+// the handler timeout into 503s all day. So past the TTL a reader is now handed the previous body
+// immediately (X-Cache: stale) while the replacement is computed behind the response, and only
+// past dashCacheStale on top of that does anyone wait again. The staleness cap is the honest half
+// and is asserted here: without it a quiet deployment would serve its last body forever while the
+// page reported it as current.
 func TestComponentsCached(t *testing.T) {
 	a, rec := newTestAPI(t, Options{})
 	e := mkEvent(time.Now().UnixMilli(), "sess-1", "aws/claude-sonnet-5", 1000, 800)
@@ -39,19 +45,59 @@ func TestComponentsCached(t *testing.T) {
 		t.Fatalf("second call within the TTL: got %d component rows, want 1 (cached) -- the cache did not hit", len(rows))
 	}
 
-	// Past the TTL, the same call must see the fresh data. Single-tenant scope() always
-	// returns Principal{Manager: true} (see API.scope), so cacheKey's own rule gives a
-	// deterministic key to backdate.
+	// Single-tenant scope() always returns Principal{Manager: true} (see API.scope), so cacheKey's
+	// own rule gives a deterministic key to backdate.
 	key := cacheKey(Principal{Manager: true}, httptest.NewRequest(http.MethodGet, "/api/components", nil))
-	a.componentsCache.mu.Lock()
-	entry := a.componentsCache.entries[key]
-	entry.at = time.Now().Add(-dashCacheTTL - time.Second)
-	a.componentsCache.entries[key] = entry
-	a.componentsCache.mu.Unlock()
+	backdate := func(d time.Duration) {
+		a.componentsCache.mu.Lock()
+		entry := a.componentsCache.entries[key]
+		entry.at = time.Now().Add(-d)
+		a.componentsCache.entries[key] = entry
+		a.componentsCache.mu.Unlock()
+	}
 
-	_, body = get(t, a, "/api/components", "127.0.0.1:1")
+	// Just past the TTL: the reader is handed the STALE body rather than made to wait, and the
+	// refresh happens behind the response.
+	backdate(dashCacheTTL + time.Second)
+	w, body = get(t, a, "/api/components", "127.0.0.1:1")
+	if got := w.Header().Get("X-Cache"); got != "stale" {
+		t.Errorf("just past the TTL: X-Cache = %q, want \"stale\" — the reader should not wait for a "+
+			"recompute when a servable body exists", got)
+	}
+	rows, _ = body["components"].([]any)
+	if len(rows) != 1 {
+		t.Errorf("just past the TTL: got %d component rows, want 1 (the stale body served immediately)", len(rows))
+	}
+
+	// ...and that background refresh must actually land, or "stale" would be permanent.
+	deadline := time.Now().Add(5 * time.Second)
+	for {
+		_, body = get(t, a, "/api/components", "127.0.0.1:1")
+		rows, _ = body["components"].([]any)
+		if len(rows) == 2 {
+			break
+		}
+		if time.Now().After(deadline) {
+			t.Fatalf("the background refresh never replaced the stale body: still %d rows after 5s. "+
+				"A refresh that cannot complete makes every reader permanently stale, which is worse "+
+				"than the blocking recompute this replaced.", len(rows))
+		}
+		time.Sleep(20 * time.Millisecond)
+	}
+
+	// Past the staleness cap, a reader waits for fresh numbers instead. Seed a third component so
+	// "fresh" is distinguishable from whatever is cached.
+	e3 := mkEvent(time.Now().UnixMilli(), "sess-3", "aws/claude-sonnet-5", 1000, 800)
+	e3.Components = []CompRow{{Component: "dedup", Kind: "reformat", Acted: true}}
+	seed(t, rec, e3)
+	backdate(dashCacheTTL + dashCacheStale + time.Second)
+	w, body = get(t, a, "/api/components", "127.0.0.1:1")
+	if got := w.Header().Get("X-Cache"); got != "miss" {
+		t.Errorf("past dashCacheStale: X-Cache = %q, want \"miss\" — beyond the cap a body is too old "+
+			"to serve, and the freshness line the page prints would be a lie", got)
+	}
 	rows, _ = body["components"].([]any)
-	if len(rows) != 2 {
-		t.Fatalf("call past the TTL: got %d component rows, want 2 (fresh)", len(rows))
+	if len(rows) != 3 {
+		t.Errorf("past dashCacheStale: got %d component rows, want 3 (recomputed)", len(rows))
 	}
 }

dash/dedupetext.go

@@ -138,7 +138,7 @@ const dedupeMigrationDoneKey = "dedupe_declaration_text_done"
 // that silently never runs.
 func (d *DB) dedupeMigrationDone() (bool, error) {
 	var v string
-	err := d.sql.QueryRow(`SELECT value FROM meta WHERE key = ?`, dedupeMigrationDoneKey).Scan(&v)
+	err := d.sql.QueryRowContext(d.readCtx(), `SELECT value FROM meta WHERE key = ?`, dedupeMigrationDoneKey).Scan(&v)
 	if err == sql.ErrNoRows {
 		return false, nil
 	}
@@ -164,7 +164,7 @@ type declBlob struct {
 // pendingDedupe reads the next batch of rows whose text is still on the row itself. rowid is
 // the table's own key, so `rowid > ?` is a seek rather than a scan of what came before it.
 func (d *DB) pendingDedupe(after int64, limit int) ([]declBlob, error) {
-	rows, err := d.sql.Query(`SELECT rowid, text_gz FROM tool_declarations
+	rows, err := d.sql.QueryContext(d.readCtx(), `SELECT rowid, text_gz FROM tool_declarations
 		WHERE rowid > ? AND text_gz IS NOT NULL ORDER BY rowid LIMIT ?`, after, limit)
 	if err != nil {
 		return nil, err

dash/janitor.go

@@ -50,6 +50,36 @@ func (r *Recorder) janitorPass() {
 		slog.Info("dash: pruned old dashboard rows", "requests", n)
 	}
 	r.relieveDiskPressure()
+	// Keep the query planner's statistics current. Cheap, and the one thing here that makes
+	// existing indexes get USED rather than merely exist.
+	//
+	// BEFORE the checkpoint below, not after, and that ordering is load-bearing: PRAGMA optimize
+	// runs ANALYZE, which WRITES sqlite_stat1. Run after the checkpoint, those writes land in a
+	// freshly truncated WAL and stay there until the next pass — so every pass would leave the WAL
+	// growing again, which is the exact thing the checkpoint exists to prevent.
+	// TestJanitorPassCheckpointsWAL caught this, and it is the reason that test asserts on WAL size
+	// rather than on the checkpoint merely having been called.
+	//
+	// This database had never been ANALYZEd at all — sqlite_stat1 did not exist — so SQLite was
+	// planning every join in this package on hard-coded guesses. Measured on a copy of the
+	// production database, /api/components over a 24-hour window: 3.59s without statistics,
+	// 0.81s with them, a 4.4x difference from no code change and no new index. The guess it was
+	// getting wrong is join ORDER: with no row counts it scanned all 1.58M request_components
+	// rows and probed requests per row, instead of driving from the filtered requests window
+	// through idx_rc_request. Two of those queries are pinned explicitly with a CROSS JOIN
+	// barrier (see Components in dash/query.go); statistics fix the rest, including
+	// DecomposeComponentSavedUSD, EstimateComponentSavedUSD and Facets' component list.
+	//
+	// PRAGMA optimize rather than a bare ANALYZE, and here rather than at Open: it re-analyses
+	// only tables whose statistics are missing or stale, so the steady-state cost is near zero
+	// and only the first pass does real work. A bare ANALYZE on this database measures 72s —
+	// fine on a background timer, and exactly what must never sit in front of the listener,
+	// which Open does (cmd/context-guru-proxy opens the store long before it serves).
+	//
+	// Statistics going stale is the failure mode to keep in mind: they are a snapshot, and a
+	// database that doubles in size between passes plans against the old shape. That is what
+	// makes this belong on the recurring pass rather than being run once by hand.
+	r.db.optimize()
 	// Checkpoint on every pass, not only as reclaim()'s side effect of an actual
 	// deletion above: a deployment comfortably inside its retention budget never
 	// deletes anything, so without this the WAL was left to grow until SQLite's own