Add reproducible ranker calibration harness

.github/workflows/ci.yml

@@ -181,6 +181,9 @@ jobs:
         python3 -m pip install --quiet -r pkg/fingerprint/testdata/requirements.txt
         make verify-golden-vectors
 
+    - name: Verify ranker calibration result
+      run: make verify-ranker-calibration
+
   race-test:
     name: Race Tests
     runs-on: ubuntu-latest

Makefile

@@ -358,6 +358,17 @@ verify-golden-vectors: ## Verify pkg/fingerprint/testdata/golden_vectors.json ma
 		     "(pip install -r pkg/fingerprint/testdata/requirements.txt to enable)"; \
 	fi
 
+RANKER_CALIBRATION_TRACE ?= internal/index/calibration/testdata/c1_synthetic_trace.json
+RANKER_CALIBRATION_RESULT ?= internal/index/calibration/testdata/c1_synthetic_result.json
+
+.PHONY: ranker-calibration
+ranker-calibration: ## Replay the checked-in ranker trace and regenerate per-knob calibration curves.
+	$(GO_CMD) run ./hack/ranker-calibration -trace $(RANKER_CALIBRATION_TRACE) -out $(RANKER_CALIBRATION_RESULT)
+
+.PHONY: verify-ranker-calibration
+verify-ranker-calibration: ## Verify the checked-in ranker calibration output matches its trace and sweep.
+	$(GO_CMD) run ./hack/ranker-calibration -trace $(RANKER_CALIBRATION_TRACE) -out $(RANKER_CALIBRATION_RESULT) -check
+
 .PHONY: image-build
 image-build: controller-image server-image subscriber-image ## Build controller, server, and kvevent-subscriber images.
 
@@ -619,7 +630,7 @@ verify-prometheus: promtool kustomize ## Lint + unit-test the Prometheus alertin
 	@echo "✓ Prometheus rules valid"
 
 .PHONY: ci
-ci: verify-naming verify-no-internal-refs verify-dco test-dco reuse-lint verify-syft-pin verify-minimal-base test-minimal-images fmt-check vet ci-lint python-lint verify-prometheus verify-golden-vectors test-docs-sync test-race build ## Local CI gate (naming + internal-refs + DCO/REUSE compliance + Syft/minimal-image policy + Go/Python lint + Prometheus rules + golden vectors + docs-sync tests + race tests + build). Run by the pre-push hook.
+ci: verify-naming verify-no-internal-refs verify-dco test-dco reuse-lint verify-syft-pin verify-minimal-base test-minimal-images fmt-check vet ci-lint python-lint verify-prometheus verify-golden-vectors verify-ranker-calibration test-docs-sync test-race build ## Local CI gate (naming + internal-refs + DCO/REUSE compliance + Syft/minimal-image policy + Go/Python lint + Prometheus rules + calibration/golden fixtures + docs-sync tests + race tests + build). Run by the pre-push hook.
 
 .PHONY: pre-pr
 pre-pr: ci ## Pre-PR gate: CI gate + generated-code drift check + sample admission check + review checklist.

docs/design/lookuproute-ranking.md

@@ -846,6 +846,47 @@ set so that:
   degenerates to 1.0 with one replica); for multi-replica deployments
   it is "pre-floor raw recall with cardinality-adjusted scores."
 
+### Calibration provenance and replay
+
+The reproducible sweep under `internal/index/calibration` calls the production
+`LookupRoute` implementation for every observation and searches the configured
+Cartesian grid. The objective is the macro-average of prefix-hit ratio and
+`TENANT_HOT`-hit ratio; ties prefer gentler score multipliers and the shorter
+fallback window.
+
+Calibration rows are controlled counterfactual experiments, not ordinary
+single-route request logs: every candidate replica must have an available
+ground-truth outcome. Captured traces must measure those outcomes experimentally
+under an equivalent cache snapshot; synthetic traces define them by
+construction. The loader rejects rows without that explicit availability.
+Prefix hashes remain engine-opaque bytes and use standard base64 JSON encoding
+in trace files.
+
+The checked-in `c1-synthetic-mixed-routing-v1` trace contains 22 observations:
+14 prefix-routing cases spanning pressure/locality tradeoffs and tight/loose
+TTFT budgets, plus 8 prefix-miss cases spanning noisy hit-rate reports and
+fresh/stale `TENANT_HOT` candidates. Its provenance is explicitly
+`synthetic`: no production C1 request trace is currently checked into this
+repository, so these values verify the harness and identify a provisional
+candidate, not a production calibration. `DefaultRankerConfig` therefore keeps
+its existing `1.0 / 200 ms / 1.0 / 0.1 / 5 min` tuple. Replace or supplement
+the trace with a sanitized captured fixture before changing those defaults.
+
+```bash
+make ranker-calibration
+make verify-ranker-calibration
+```
+
+The trace and generated per-knob curves live in
+`internal/index/calibration/testdata/c1_synthetic_trace.json` and
+`c1_synthetic_result.json`. On this synthetic boundary fixture, the sweep
+selects the candidate `PressureWeight = 0.5`,
+`SLOTightTTFTMs = 200 ms`, `SLOTightBias = 1.0`,
+`TenantHotMinHitRate = 0.2`, and `TenantHotMaxAge = 2 min`; both measured hit
+ratios are 100%. CI regenerates the result in check mode, and tests keep the
+trace and generated result from silently diverging. This candidate is not
+applied to production defaults without representative captured evidence.
+
 ## 7. The reason-code summary
 
 | Code                  | When it fires                                                                                                                                                                                                                       | What the gateway treats it as           |

hack/ranker-calibration/main.go

@@ -0,0 +1,103 @@
+// SPDX-FileCopyrightText: 2026 The inference-cache Authors
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package main
+
+import (
+	"bytes"
+	"flag"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+
+	"github.com/cachebox-project/inference-cache/internal/index/calibration"
+)
+
+func main() {
+	os.Exit(run(os.Args[1:], os.Stdout, os.Stderr))
+}
+
+func run(args []string, stdout, stderr io.Writer) int {
+	flags := flag.NewFlagSet("ranker-calibration", flag.ContinueOnError)
+	flags.SetOutput(stderr)
+	tracePath := flags.String("trace", "", "path to a ranker calibration trace")
+	outPath := flags.String("out", "", "path to write the calibration result")
+	check := flags.Bool("check", false, "verify that -out already matches the generated result")
+	if err := flags.Parse(args); err != nil {
+		return 2
+	}
+	if *tracePath == "" || *outPath == "" {
+		return failf(stderr, "both -trace and -out are required")
+	}
+
+	traceFile, err := os.Open(*tracePath)
+	if err != nil {
+		return failf(stderr, "open trace: %v", err)
+	}
+	trace, err := calibration.Load(traceFile)
+	closeErr := traceFile.Close()
+	if err != nil {
+		return failf(stderr, "load trace: %v", err)
+	}
+	if closeErr != nil {
+		return failf(stderr, "close trace: %v", closeErr)
+	}
+
+	data, err := calibration.MarshalResult(calibration.Calibrate(trace))
+	if err != nil {
+		return failf(stderr, "render result: %v", err)
+	}
+	if *check {
+		current, err := os.ReadFile(*outPath)
+		if err != nil {
+			return failf(stderr, "read result for check: %v", err)
+		}
+		if !bytes.Equal(current, data) {
+			return failf(stderr, "%s is stale; rerun ranker calibration", *outPath)
+		}
+		fmt.Fprintf(stdout, "ranker calibration is current: %s\n", *outPath)
+		return 0
+	}
+	if err := writeAtomic(*outPath, data); err != nil {
+		return failf(stderr, "write result: %v", err)
+	}
+	fmt.Fprintf(stdout, "wrote ranker calibration: %s\n", *outPath)
+	return 0
+}
+
+func writeAtomic(path string, data []byte) error {
+	dir := filepath.Dir(path)
+	temp, err := os.CreateTemp(dir, "."+filepath.Base(path)+".tmp-*")
+	if err != nil {
+		return fmt.Errorf("create temporary result: %w", err)
+	}
+	tempPath := temp.Name()
+	defer func() { _ = os.Remove(tempPath) }()
+
+	if err := temp.Chmod(0o644); err != nil {
+		_ = temp.Close()
+		return fmt.Errorf("set temporary result mode: %w", err)
+	}
+	if _, err := temp.Write(data); err != nil {
+		_ = temp.Close()
+		return fmt.Errorf("write temporary result: %w", err)
+	}
+	if err := temp.Sync(); err != nil {
+		_ = temp.Close()
+		return fmt.Errorf("sync temporary result: %w", err)
+	}
+	if err := temp.Close(); err != nil {
+		return fmt.Errorf("close temporary result: %w", err)
+	}
+	if err := os.Rename(tempPath, path); err != nil {
+		return fmt.Errorf("replace result: %w", err)
+	}
+	return nil
+}
+
+func failf(stderr io.Writer, format string, args ...any) int {
+	fmt.Fprintf(stderr, "ranker-calibration: "+format+"\n", args...)
+	return 1
+}

hack/ranker-calibration/main_test.go

@@ -0,0 +1,75 @@
+// SPDX-FileCopyrightText: 2026 The inference-cache Authors
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package main
+
+import (
+	"bytes"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+)
+
+func TestRunGenerateAndCheck(t *testing.T) {
+	tracePath := filepath.Join("..", "..", "internal", "index", "calibration", "testdata", "c1_synthetic_trace.json")
+	outPath := filepath.Join(t.TempDir(), "result.json")
+	var stdout, stderr bytes.Buffer
+
+	if code := run([]string{"-trace", tracePath, "-out", outPath}, &stdout, &stderr); code != 0 {
+		t.Fatalf("generate exit = %d, stderr = %q", code, stderr.String())
+	}
+	if !strings.Contains(stdout.String(), "wrote ranker calibration") {
+		t.Fatalf("generate stdout = %q", stdout.String())
+	}
+	if matches, err := filepath.Glob(filepath.Join(filepath.Dir(outPath), ".result.json.tmp-*")); err != nil || len(matches) != 0 {
+		t.Fatalf("temporary results after generation = %v, err = %v", matches, err)
+	}
+
+	stdout.Reset()
+	stderr.Reset()
+	if code := run([]string{"-trace", tracePath, "-out", outPath, "-check"}, &stdout, &stderr); code != 0 {
+		t.Fatalf("current check exit = %d, stderr = %q", code, stderr.String())
+	}
+	if !strings.Contains(stdout.String(), "ranker calibration is current") {
+		t.Fatalf("check stdout = %q", stdout.String())
+	}
+
+	if err := os.WriteFile(outPath, []byte("{}\n"), 0o644); err != nil {
+		t.Fatalf("write stale result: %v", err)
+	}
+	stdout.Reset()
+	stderr.Reset()
+	if code := run([]string{"-trace", tracePath, "-out", outPath, "-check"}, &stdout, &stderr); code != 1 {
+		t.Fatalf("stale check exit = %d, want 1", code)
+	}
+	if !strings.Contains(stderr.String(), "is stale") {
+		t.Fatalf("stale check stderr = %q", stderr.String())
+	}
+}
+
+func TestRunRejectsMalformedTrace(t *testing.T) {
+	dir := t.TempDir()
+	tracePath := filepath.Join(dir, "trace.json")
+	if err := os.WriteFile(tracePath, []byte("not-json"), 0o644); err != nil {
+		t.Fatalf("write malformed trace: %v", err)
+	}
+	var stdout, stderr bytes.Buffer
+	if code := run([]string{"-trace", tracePath, "-out", filepath.Join(dir, "result.json")}, &stdout, &stderr); code != 1 {
+		t.Fatalf("malformed trace exit = %d, want 1", code)
+	}
+	if !strings.Contains(stderr.String(), "load trace") {
+		t.Fatalf("malformed trace stderr = %q", stderr.String())
+	}
+}
+
+func TestRunRequiresPaths(t *testing.T) {
+	var stdout, stderr bytes.Buffer
+	if code := run(nil, &stdout, &stderr); code != 1 {
+		t.Fatalf("missing paths exit = %d, want 1", code)
+	}
+	if !strings.Contains(stderr.String(), "both -trace and -out are required") {
+		t.Fatalf("missing paths stderr = %q", stderr.String())
+	}
+}

internal/index/calibration/README.md

@@ -0,0 +1,61 @@
+# Ranker calibration
+
+This package replays routing observations through the production
+`internal/index.LookupRoute` implementation and searches a Cartesian grid of
+the five `RankerConfig` knobs. It reports two outcome rates:
+
+- `prefix_hit_rate_pct`: the selected top replica produced an observed cache
+  hit for a `prefix` observation.
+- `tenant_hot_hit_rate_pct`: the selected top replica produced an observed
+  cache hit after a `tenant_hot` fallback observation.
+
+`macro_hit_rate_pct` gives both observation classes equal weight, regardless
+of how many rows each class contributes. The deterministic tie-break prefers
+gentler pressure/SLO multipliers and shorter fallback windows.
+
+## Trace shape
+
+Each observation is a self-contained point-in-time view. `reported_prefix`
+records whether the cache plane believed that replica held the requested
+prefix; `prefix_reported_at_ms` records that prefix observation's freshness,
+while `stats_reported_at_ms` independently records when `hit_rate` and
+`pressure` were reported. `matched_tokens` and those timestamps are the signals
+visible to the ranker. `prefix_hash` is standard base64 JSON for the engine's
+opaque bytes, not a human-readable identifier. Replicas without the requested
+prefix still receive a collision-free serving-only entry during replay so
+`TENANT_HOT` can apply its real engine-domain membership guard.
+
+Every replica row must set `outcome_available: true`; `observed_hit` is the
+ground-truth result of routing that request to that replica. Captured traces
+must measure that outcome experimentally for every candidate under an
+equivalent cache snapshot; synthetic traces may define it by construction. A
+normal production request observes only its selected replica and is therefore
+not sufficient calibration input by itself. The harness rejects incomplete
+rows so an unavailable outcome cannot silently turn into a miss. Zero values
+for `slo_tight_ttft_ms` and
+`tenant_hot_max_age_ms` are valid sweep points and exercise the production kill
+switches.
+
+Captured data should contain opaque or one-way prefix hashes only. Do not put
+prompt text, token IDs, customer identifiers, or other request content in a
+trace. Use stable pseudonyms for tenants, models, and replicas. Set
+`provenance.kind` to `captured` only when the observations came from a real
+run; generated and hand-constructed fixtures must say `synthetic`.
+
+The checked-in fixture is intentionally synthetic because no production C1
+trace is available in this repository. It provides deterministic boundary
+coverage and proves the calibration pipeline, but it should be replaced or
+supplemented with a sanitized captured trace before treating the coefficients
+as a production benchmark conclusion. Its selected tuple is therefore a
+candidate only and does not change `DefaultRankerConfig`; production defaults
+remain stable until representative captured data supports a retune.
+
+## Reproduce
+
+```bash
+make ranker-calibration
+make verify-ranker-calibration
+```
+
+Override `RANKER_CALIBRATION_TRACE` and `RANKER_CALIBRATION_RESULT` to replay a
+different trace without changing the tool.