diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8474da85..050b0499 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -181,6 +181,9 @@ jobs: python3 -m pip install --quiet -r pkg/fingerprint/testdata/requirements.txt make verify-golden-vectors + - name: Verify ranker calibration result + run: make verify-ranker-calibration + race-test: name: Race Tests runs-on: ubuntu-latest diff --git a/Makefile b/Makefile index b0a548e8..97b4ba73 100644 --- a/Makefile +++ b/Makefile @@ -358,6 +358,17 @@ verify-golden-vectors: ## Verify pkg/fingerprint/testdata/golden_vectors.json ma "(pip install -r pkg/fingerprint/testdata/requirements.txt to enable)"; \ fi +RANKER_CALIBRATION_TRACE ?= internal/index/calibration/testdata/c1_synthetic_trace.json +RANKER_CALIBRATION_RESULT ?= internal/index/calibration/testdata/c1_synthetic_result.json + +.PHONY: ranker-calibration +ranker-calibration: ## Replay the checked-in ranker trace and regenerate per-knob calibration curves. + $(GO_CMD) run ./hack/ranker-calibration -trace $(RANKER_CALIBRATION_TRACE) -out $(RANKER_CALIBRATION_RESULT) + +.PHONY: verify-ranker-calibration +verify-ranker-calibration: ## Verify the checked-in ranker calibration output matches its trace and sweep. + $(GO_CMD) run ./hack/ranker-calibration -trace $(RANKER_CALIBRATION_TRACE) -out $(RANKER_CALIBRATION_RESULT) -check + .PHONY: image-build image-build: controller-image server-image subscriber-image ## Build controller, server, and kvevent-subscriber images. @@ -619,7 +630,7 @@ verify-prometheus: promtool kustomize ## Lint + unit-test the Prometheus alertin @echo "✓ Prometheus rules valid" .PHONY: ci -ci: verify-naming verify-no-internal-refs verify-dco test-dco reuse-lint verify-syft-pin verify-minimal-base test-minimal-images fmt-check vet ci-lint python-lint verify-prometheus verify-golden-vectors test-docs-sync test-race build ## Local CI gate (naming + internal-refs + DCO/REUSE compliance + Syft/minimal-image policy + Go/Python lint + Prometheus rules + golden vectors + docs-sync tests + race tests + build). Run by the pre-push hook. +ci: verify-naming verify-no-internal-refs verify-dco test-dco reuse-lint verify-syft-pin verify-minimal-base test-minimal-images fmt-check vet ci-lint python-lint verify-prometheus verify-golden-vectors verify-ranker-calibration test-docs-sync test-race build ## Local CI gate (naming + internal-refs + DCO/REUSE compliance + Syft/minimal-image policy + Go/Python lint + Prometheus rules + calibration/golden fixtures + docs-sync tests + race tests + build). Run by the pre-push hook. .PHONY: pre-pr pre-pr: ci ## Pre-PR gate: CI gate + generated-code drift check + sample admission check + review checklist. diff --git a/docs/design/lookuproute-ranking.md b/docs/design/lookuproute-ranking.md index 968e0b01..33f0b4b5 100644 --- a/docs/design/lookuproute-ranking.md +++ b/docs/design/lookuproute-ranking.md @@ -846,6 +846,47 @@ set so that: degenerates to 1.0 with one replica); for multi-replica deployments it is "pre-floor raw recall with cardinality-adjusted scores." +### Calibration provenance and replay + +The reproducible sweep under `internal/index/calibration` calls the production +`LookupRoute` implementation for every observation and searches the configured +Cartesian grid. The objective is the macro-average of prefix-hit ratio and +`TENANT_HOT`-hit ratio; ties prefer gentler score multipliers and the shorter +fallback window. + +Calibration rows are controlled counterfactual experiments, not ordinary +single-route request logs: every candidate replica must have an available +ground-truth outcome. Captured traces must measure those outcomes experimentally +under an equivalent cache snapshot; synthetic traces define them by +construction. The loader rejects rows without that explicit availability. +Prefix hashes remain engine-opaque bytes and use standard base64 JSON encoding +in trace files. + +The checked-in `c1-synthetic-mixed-routing-v1` trace contains 22 observations: +14 prefix-routing cases spanning pressure/locality tradeoffs and tight/loose +TTFT budgets, plus 8 prefix-miss cases spanning noisy hit-rate reports and +fresh/stale `TENANT_HOT` candidates. Its provenance is explicitly +`synthetic`: no production C1 request trace is currently checked into this +repository, so these values verify the harness and identify a provisional +candidate, not a production calibration. `DefaultRankerConfig` therefore keeps +its existing `1.0 / 200 ms / 1.0 / 0.1 / 5 min` tuple. Replace or supplement +the trace with a sanitized captured fixture before changing those defaults. + +```bash +make ranker-calibration +make verify-ranker-calibration +``` + +The trace and generated per-knob curves live in +`internal/index/calibration/testdata/c1_synthetic_trace.json` and +`c1_synthetic_result.json`. On this synthetic boundary fixture, the sweep +selects the candidate `PressureWeight = 0.5`, +`SLOTightTTFTMs = 200 ms`, `SLOTightBias = 1.0`, +`TenantHotMinHitRate = 0.2`, and `TenantHotMaxAge = 2 min`; both measured hit +ratios are 100%. CI regenerates the result in check mode, and tests keep the +trace and generated result from silently diverging. This candidate is not +applied to production defaults without representative captured evidence. + ## 7. The reason-code summary | Code | When it fires | What the gateway treats it as | diff --git a/hack/ranker-calibration/main.go b/hack/ranker-calibration/main.go new file mode 100644 index 00000000..4a0e677a --- /dev/null +++ b/hack/ranker-calibration/main.go @@ -0,0 +1,103 @@ +// SPDX-FileCopyrightText: 2026 The inference-cache Authors +// +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "bytes" + "flag" + "fmt" + "io" + "os" + "path/filepath" + + "github.com/cachebox-project/inference-cache/internal/index/calibration" +) + +func main() { + os.Exit(run(os.Args[1:], os.Stdout, os.Stderr)) +} + +func run(args []string, stdout, stderr io.Writer) int { + flags := flag.NewFlagSet("ranker-calibration", flag.ContinueOnError) + flags.SetOutput(stderr) + tracePath := flags.String("trace", "", "path to a ranker calibration trace") + outPath := flags.String("out", "", "path to write the calibration result") + check := flags.Bool("check", false, "verify that -out already matches the generated result") + if err := flags.Parse(args); err != nil { + return 2 + } + if *tracePath == "" || *outPath == "" { + return failf(stderr, "both -trace and -out are required") + } + + traceFile, err := os.Open(*tracePath) + if err != nil { + return failf(stderr, "open trace: %v", err) + } + trace, err := calibration.Load(traceFile) + closeErr := traceFile.Close() + if err != nil { + return failf(stderr, "load trace: %v", err) + } + if closeErr != nil { + return failf(stderr, "close trace: %v", closeErr) + } + + data, err := calibration.MarshalResult(calibration.Calibrate(trace)) + if err != nil { + return failf(stderr, "render result: %v", err) + } + if *check { + current, err := os.ReadFile(*outPath) + if err != nil { + return failf(stderr, "read result for check: %v", err) + } + if !bytes.Equal(current, data) { + return failf(stderr, "%s is stale; rerun ranker calibration", *outPath) + } + fmt.Fprintf(stdout, "ranker calibration is current: %s\n", *outPath) + return 0 + } + if err := writeAtomic(*outPath, data); err != nil { + return failf(stderr, "write result: %v", err) + } + fmt.Fprintf(stdout, "wrote ranker calibration: %s\n", *outPath) + return 0 +} + +func writeAtomic(path string, data []byte) error { + dir := filepath.Dir(path) + temp, err := os.CreateTemp(dir, "."+filepath.Base(path)+".tmp-*") + if err != nil { + return fmt.Errorf("create temporary result: %w", err) + } + tempPath := temp.Name() + defer func() { _ = os.Remove(tempPath) }() + + if err := temp.Chmod(0o644); err != nil { + _ = temp.Close() + return fmt.Errorf("set temporary result mode: %w", err) + } + if _, err := temp.Write(data); err != nil { + _ = temp.Close() + return fmt.Errorf("write temporary result: %w", err) + } + if err := temp.Sync(); err != nil { + _ = temp.Close() + return fmt.Errorf("sync temporary result: %w", err) + } + if err := temp.Close(); err != nil { + return fmt.Errorf("close temporary result: %w", err) + } + if err := os.Rename(tempPath, path); err != nil { + return fmt.Errorf("replace result: %w", err) + } + return nil +} + +func failf(stderr io.Writer, format string, args ...any) int { + fmt.Fprintf(stderr, "ranker-calibration: "+format+"\n", args...) + return 1 +} diff --git a/hack/ranker-calibration/main_test.go b/hack/ranker-calibration/main_test.go new file mode 100644 index 00000000..0531cd3b --- /dev/null +++ b/hack/ranker-calibration/main_test.go @@ -0,0 +1,75 @@ +// SPDX-FileCopyrightText: 2026 The inference-cache Authors +// +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "bytes" + "os" + "path/filepath" + "strings" + "testing" +) + +func TestRunGenerateAndCheck(t *testing.T) { + tracePath := filepath.Join("..", "..", "internal", "index", "calibration", "testdata", "c1_synthetic_trace.json") + outPath := filepath.Join(t.TempDir(), "result.json") + var stdout, stderr bytes.Buffer + + if code := run([]string{"-trace", tracePath, "-out", outPath}, &stdout, &stderr); code != 0 { + t.Fatalf("generate exit = %d, stderr = %q", code, stderr.String()) + } + if !strings.Contains(stdout.String(), "wrote ranker calibration") { + t.Fatalf("generate stdout = %q", stdout.String()) + } + if matches, err := filepath.Glob(filepath.Join(filepath.Dir(outPath), ".result.json.tmp-*")); err != nil || len(matches) != 0 { + t.Fatalf("temporary results after generation = %v, err = %v", matches, err) + } + + stdout.Reset() + stderr.Reset() + if code := run([]string{"-trace", tracePath, "-out", outPath, "-check"}, &stdout, &stderr); code != 0 { + t.Fatalf("current check exit = %d, stderr = %q", code, stderr.String()) + } + if !strings.Contains(stdout.String(), "ranker calibration is current") { + t.Fatalf("check stdout = %q", stdout.String()) + } + + if err := os.WriteFile(outPath, []byte("{}\n"), 0o644); err != nil { + t.Fatalf("write stale result: %v", err) + } + stdout.Reset() + stderr.Reset() + if code := run([]string{"-trace", tracePath, "-out", outPath, "-check"}, &stdout, &stderr); code != 1 { + t.Fatalf("stale check exit = %d, want 1", code) + } + if !strings.Contains(stderr.String(), "is stale") { + t.Fatalf("stale check stderr = %q", stderr.String()) + } +} + +func TestRunRejectsMalformedTrace(t *testing.T) { + dir := t.TempDir() + tracePath := filepath.Join(dir, "trace.json") + if err := os.WriteFile(tracePath, []byte("not-json"), 0o644); err != nil { + t.Fatalf("write malformed trace: %v", err) + } + var stdout, stderr bytes.Buffer + if code := run([]string{"-trace", tracePath, "-out", filepath.Join(dir, "result.json")}, &stdout, &stderr); code != 1 { + t.Fatalf("malformed trace exit = %d, want 1", code) + } + if !strings.Contains(stderr.String(), "load trace") { + t.Fatalf("malformed trace stderr = %q", stderr.String()) + } +} + +func TestRunRequiresPaths(t *testing.T) { + var stdout, stderr bytes.Buffer + if code := run(nil, &stdout, &stderr); code != 1 { + t.Fatalf("missing paths exit = %d, want 1", code) + } + if !strings.Contains(stderr.String(), "both -trace and -out are required") { + t.Fatalf("missing paths stderr = %q", stderr.String()) + } +} diff --git a/internal/index/calibration/README.md b/internal/index/calibration/README.md new file mode 100644 index 00000000..852816ed --- /dev/null +++ b/internal/index/calibration/README.md @@ -0,0 +1,61 @@ +# Ranker calibration + +This package replays routing observations through the production +`internal/index.LookupRoute` implementation and searches a Cartesian grid of +the five `RankerConfig` knobs. It reports two outcome rates: + +- `prefix_hit_rate_pct`: the selected top replica produced an observed cache + hit for a `prefix` observation. +- `tenant_hot_hit_rate_pct`: the selected top replica produced an observed + cache hit after a `tenant_hot` fallback observation. + +`macro_hit_rate_pct` gives both observation classes equal weight, regardless +of how many rows each class contributes. The deterministic tie-break prefers +gentler pressure/SLO multipliers and shorter fallback windows. + +## Trace shape + +Each observation is a self-contained point-in-time view. `reported_prefix` +records whether the cache plane believed that replica held the requested +prefix; `prefix_reported_at_ms` records that prefix observation's freshness, +while `stats_reported_at_ms` independently records when `hit_rate` and +`pressure` were reported. `matched_tokens` and those timestamps are the signals +visible to the ranker. `prefix_hash` is standard base64 JSON for the engine's +opaque bytes, not a human-readable identifier. Replicas without the requested +prefix still receive a collision-free serving-only entry during replay so +`TENANT_HOT` can apply its real engine-domain membership guard. + +Every replica row must set `outcome_available: true`; `observed_hit` is the +ground-truth result of routing that request to that replica. Captured traces +must measure that outcome experimentally for every candidate under an +equivalent cache snapshot; synthetic traces may define it by construction. A +normal production request observes only its selected replica and is therefore +not sufficient calibration input by itself. The harness rejects incomplete +rows so an unavailable outcome cannot silently turn into a miss. Zero values +for `slo_tight_ttft_ms` and +`tenant_hot_max_age_ms` are valid sweep points and exercise the production kill +switches. + +Captured data should contain opaque or one-way prefix hashes only. Do not put +prompt text, token IDs, customer identifiers, or other request content in a +trace. Use stable pseudonyms for tenants, models, and replicas. Set +`provenance.kind` to `captured` only when the observations came from a real +run; generated and hand-constructed fixtures must say `synthetic`. + +The checked-in fixture is intentionally synthetic because no production C1 +trace is available in this repository. It provides deterministic boundary +coverage and proves the calibration pipeline, but it should be replaced or +supplemented with a sanitized captured trace before treating the coefficients +as a production benchmark conclusion. Its selected tuple is therefore a +candidate only and does not change `DefaultRankerConfig`; production defaults +remain stable until representative captured data supports a retune. + +## Reproduce + +```bash +make ranker-calibration +make verify-ranker-calibration +``` + +Override `RANKER_CALIBRATION_TRACE` and `RANKER_CALIBRATION_RESULT` to replay a +different trace without changing the tool. diff --git a/internal/index/calibration/calibration.go b/internal/index/calibration/calibration.go new file mode 100644 index 00000000..b81e86e6 --- /dev/null +++ b/internal/index/calibration/calibration.go @@ -0,0 +1,516 @@ +// SPDX-FileCopyrightText: 2026 The inference-cache Authors +// +// SPDX-License-Identifier: Apache-2.0 + +// Package calibration replays routing observations through the +// production index ranker and sweeps RankerConfig candidates. +package calibration + +import ( + "encoding/json" + "errors" + "fmt" + "io" + "math" + "sort" + "time" + + "github.com/cachebox-project/inference-cache/internal/index" +) + +const SchemaVersion = 1 + +const maxDurationMillis int64 = (1<<63 - 1) / int64(time.Millisecond) + +const ( + ObservationPrefix = "prefix" + ObservationTenantHot = "tenant_hot" +) + +type Provenance struct { + Kind string `json:"kind"` + Source string `json:"source"` + Description string `json:"description"` +} + +type Sweep struct { + PressureWeights []float64 `json:"pressure_weights"` + SLOTightTTFTMillis []int32 `json:"slo_tight_ttft_ms"` + SLOTightBiases []float64 `json:"slo_tight_biases"` + TenantHotMinHitRates []float64 `json:"tenant_hot_min_hit_rates"` + TenantHotMaxAgeMillis []int64 `json:"tenant_hot_max_age_ms"` +} + +type Trace struct { + SchemaVersion int `json:"schema_version"` + Name string `json:"name"` + Provenance Provenance `json:"provenance"` + TTLMillis int64 `json:"ttl_ms"` + Sweep Sweep `json:"sweep"` + Observations []Observation `json:"observations"` +} + +type Observation struct { + ID string `json:"id"` + Kind string `json:"kind"` + AtMillis int64 `json:"at_ms"` + Tenant string `json:"tenant"` + Model string `json:"model"` + HashScheme string `json:"hash_scheme"` + PrefixHash []byte `json:"prefix_hash"` + TokenCount int32 `json:"token_count"` + TTFTBudgetMillis int32 `json:"ttft_budget_ms,omitempty"` + Replicas []ReplicaObservation `json:"replicas"` +} + +type ReplicaObservation struct { + ID string `json:"id"` + PrefixReportedAtMillis int64 `json:"prefix_reported_at_ms"` + StatsReportedAtMillis int64 `json:"stats_reported_at_ms"` + ReportedPrefix bool `json:"reported_prefix"` + MatchedTokens int32 `json:"matched_tokens"` + HitRate float32 `json:"hit_rate"` + Pressure float32 `json:"pressure"` + OutcomeAvailable bool `json:"outcome_available"` + ObservedHit bool `json:"observed_hit"` +} + +type Config struct { + PressureWeight float64 `json:"pressure_weight"` + SLOTightTTFTMillis int32 `json:"slo_tight_ttft_ms"` + SLOTightBias float64 `json:"slo_tight_bias"` + TenantHotMinHitRate float64 `json:"tenant_hot_min_hit_rate"` + TenantHotMaxAgeMillis int64 `json:"tenant_hot_max_age_ms"` +} + +type Metrics struct { + PrefixRequests int `json:"prefix_requests"` + PrefixHits int `json:"prefix_hits"` + PrefixHitRatePct float64 `json:"prefix_hit_rate_pct"` + TenantHotRequests int `json:"tenant_hot_requests"` + TenantHotHits int `json:"tenant_hot_hits"` + TenantHotHitRatePct float64 `json:"tenant_hot_hit_rate_pct"` + MacroHitRatePct float64 `json:"macro_hit_rate_pct"` +} + +type CurvePoint struct { + Value float64 `json:"value"` + Metrics Metrics `json:"metrics"` +} + +type Result struct { + SchemaVersion int `json:"schema_version"` + TraceName string `json:"trace_name"` + Provenance Provenance `json:"provenance"` + Observations int `json:"observations"` + BestConfig Config `json:"best_config"` + BestMetrics Metrics `json:"best_metrics"` + Curves map[string][]CurvePoint `json:"curves"` +} + +func Load(r io.Reader) (Trace, error) { + var trace Trace + decoder := json.NewDecoder(r) + decoder.DisallowUnknownFields() + if err := decoder.Decode(&trace); err != nil { + return Trace{}, fmt.Errorf("decode calibration trace: %w", err) + } + var trailing any + if err := decoder.Decode(&trailing); !errors.Is(err, io.EOF) { + return Trace{}, errors.New("decode calibration trace: trailing JSON value") + } + if err := trace.Validate(); err != nil { + return Trace{}, err + } + return trace, nil +} + +func (t Trace) Validate() error { + if t.SchemaVersion != SchemaVersion { + return fmt.Errorf("schema_version = %d, want %d", t.SchemaVersion, SchemaVersion) + } + if t.Name == "" { + return errors.New("trace name is required") + } + if t.Provenance.Kind != "captured" && t.Provenance.Kind != "synthetic" { + return fmt.Errorf("provenance kind %q must be captured or synthetic", t.Provenance.Kind) + } + if t.Provenance.Source == "" { + return errors.New("provenance source is required") + } + if t.TTLMillis <= 0 { + return errors.New("ttl_ms must be positive") + } + if t.TTLMillis > maxDurationMillis { + return fmt.Errorf("ttl_ms exceeds maximum representable duration (%d ms)", maxDurationMillis) + } + if err := t.Sweep.validate(); err != nil { + return err + } + if len(t.Observations) == 0 { + return errors.New("at least one observation is required") + } + seen := make(map[string]struct{}, len(t.Observations)) + seenKinds := make(map[string]bool, 2) + for i, observation := range t.Observations { + if err := observation.validate(); err != nil { + return fmt.Errorf("observation %d: %w", i, err) + } + if _, ok := seen[observation.ID]; ok { + return fmt.Errorf("observation %d: duplicate id %q", i, observation.ID) + } + seen[observation.ID] = struct{}{} + seenKinds[observation.Kind] = true + } + if !seenKinds[ObservationPrefix] || !seenKinds[ObservationTenantHot] { + return errors.New("trace must contain at least one prefix and one tenant_hot observation") + } + return nil +} + +func (s Sweep) validate() error { + if len(s.PressureWeights) == 0 || len(s.SLOTightTTFTMillis) == 0 || + len(s.SLOTightBiases) == 0 || len(s.TenantHotMinHitRates) == 0 || + len(s.TenantHotMaxAgeMillis) == 0 { + return errors.New("every sweep dimension must contain at least one value") + } + for _, value := range append(append([]float64{}, s.PressureWeights...), s.SLOTightBiases...) { + if !finiteNonNegativeFloat32(value) { + return fmt.Errorf("sweep contains invalid finite non-negative float32 value %v", value) + } + } + for _, value := range s.TenantHotMinHitRates { + if !finiteRate(value) { + return fmt.Errorf("tenant_hot_min_hit_rates contains invalid rate %v", value) + } + } + for _, value := range s.SLOTightTTFTMillis { + if value < 0 { + return fmt.Errorf("slo_tight_ttft_ms contains negative value %d", value) + } + } + for _, value := range s.TenantHotMaxAgeMillis { + if value < 0 { + return fmt.Errorf("tenant_hot_max_age_ms contains negative value %d", value) + } + if value > maxDurationMillis { + return fmt.Errorf("tenant_hot_max_age_ms contains value exceeding maximum representable duration: %d", value) + } + } + return nil +} + +func (o Observation) validate() error { + if o.ID == "" || o.Tenant == "" || o.Model == "" || o.HashScheme == "" || len(o.PrefixHash) == 0 { + return errors.New("id, tenant, model, hash_scheme, and prefix_hash are required") + } + if o.Kind != ObservationPrefix && o.Kind != ObservationTenantHot { + return fmt.Errorf("kind %q must be prefix or tenant_hot", o.Kind) + } + if o.TokenCount <= 0 { + return errors.New("token_count must be positive") + } + if o.TTFTBudgetMillis < 0 { + return errors.New("ttft_budget_ms must be non-negative") + } + if len(o.Replicas) == 0 { + return errors.New("at least one replica is required") + } + seen := make(map[string]struct{}, len(o.Replicas)) + for i, replica := range o.Replicas { + if replica.ID == "" { + return fmt.Errorf("replica %d: id is required", i) + } + if replica.PrefixReportedAtMillis > o.AtMillis { + return fmt.Errorf("replica %q: prefix_reported_at_ms is after observation", replica.ID) + } + if replica.StatsReportedAtMillis > o.AtMillis { + return fmt.Errorf("replica %q: stats_reported_at_ms is after observation", replica.ID) + } + if replica.MatchedTokens < 0 { + return fmt.Errorf("replica %q: matched_tokens must be non-negative", replica.ID) + } + if replica.ReportedPrefix && replica.MatchedTokens == 0 { + return fmt.Errorf("replica %q: matched_tokens must be positive when reported_prefix is true", replica.ID) + } + if !finiteRate(float64(replica.HitRate)) || !finiteRate(float64(replica.Pressure)) { + return fmt.Errorf("replica %q: hit_rate and pressure must be finite values in [0,1]", replica.ID) + } + if !replica.OutcomeAvailable { + return fmt.Errorf("replica %q: outcome_available must be true", replica.ID) + } + if _, ok := seen[replica.ID]; ok { + return fmt.Errorf("duplicate replica id %q", replica.ID) + } + seen[replica.ID] = struct{}{} + } + return nil +} + +func finiteNonNegative(value float64) bool { + return !math.IsNaN(value) && !math.IsInf(value, 0) && value >= 0 +} + +func finiteNonNegativeFloat32(value float64) bool { + return finiteNonNegative(value) && value <= math.MaxFloat32 +} + +func finiteRate(value float64) bool { + return finiteNonNegative(value) && value <= 1 +} + +func Calibrate(trace Trace) Result { + bestConfig, bestMetrics := bestGridPoint(trace) + return Result{ + SchemaVersion: SchemaVersion, + TraceName: trace.Name, + Provenance: trace.Provenance, + Observations: len(trace.Observations), + BestConfig: bestConfig, + BestMetrics: bestMetrics, + Curves: map[string][]CurvePoint{ + "pressure_weight": pressureCurve(trace, bestConfig), + "slo_tight_ttft_ms": sloThresholdCurve(trace, bestConfig), + "slo_tight_bias": sloBiasCurve(trace, bestConfig), + "tenant_hot_min_hit_rate": tenantHotRateCurve(trace, bestConfig), + "tenant_hot_max_age_ms": tenantHotAgeCurve(trace, bestConfig), + }, + } +} + +func bestGridPoint(trace Trace) (Config, Metrics) { + var best Config + var bestMetrics Metrics + first := true + for _, pressureWeight := range trace.Sweep.PressureWeights { + for _, ttft := range trace.Sweep.SLOTightTTFTMillis { + for _, bias := range trace.Sweep.SLOTightBiases { + for _, hitRate := range trace.Sweep.TenantHotMinHitRates { + for _, maxAge := range trace.Sweep.TenantHotMaxAgeMillis { + candidate := Config{ + PressureWeight: pressureWeight, + SLOTightTTFTMillis: ttft, + SLOTightBias: bias, + TenantHotMinHitRate: hitRate, + TenantHotMaxAgeMillis: maxAge, + } + metrics := Replay(trace, candidate) + if first || better(metrics, candidate, bestMetrics, best) { + best, bestMetrics, first = candidate, metrics, false + } + } + } + } + } + } + return best, bestMetrics +} + +func better(candidateMetrics Metrics, candidate Config, bestMetrics Metrics, best Config) bool { + if candidateMetrics.MacroHitRatePct != bestMetrics.MacroHitRatePct { + return candidateMetrics.MacroHitRatePct > bestMetrics.MacroHitRatePct + } + if candidateMetrics.PrefixHitRatePct != bestMetrics.PrefixHitRatePct { + return candidateMetrics.PrefixHitRatePct > bestMetrics.PrefixHitRatePct + } + if candidateMetrics.TenantHotHitRatePct != bestMetrics.TenantHotHitRatePct { + return candidateMetrics.TenantHotHitRatePct > bestMetrics.TenantHotHitRatePct + } + // Conservative deterministic tie-break: prefer the least invasive score + // multipliers and shortest fallback window among equally accurate points. + if candidate.PressureWeight != best.PressureWeight { + return candidate.PressureWeight < best.PressureWeight + } + if candidate.SLOTightBias != best.SLOTightBias { + return candidate.SLOTightBias < best.SLOTightBias + } + if candidate.SLOTightTTFTMillis != best.SLOTightTTFTMillis { + return candidate.SLOTightTTFTMillis < best.SLOTightTTFTMillis + } + if candidate.TenantHotMaxAgeMillis != best.TenantHotMaxAgeMillis { + return candidate.TenantHotMaxAgeMillis < best.TenantHotMaxAgeMillis + } + return candidate.TenantHotMinHitRate > best.TenantHotMinHitRate +} + +func Replay(trace Trace, config Config) Metrics { + metrics := Metrics{} + for _, observation := range trace.Observations { + hit := replayObservation(trace, observation, config) + switch observation.Kind { + case ObservationPrefix: + metrics.PrefixRequests++ + if hit { + metrics.PrefixHits++ + } + case ObservationTenantHot: + metrics.TenantHotRequests++ + if hit { + metrics.TenantHotHits++ + } + } + } + metrics.PrefixHitRatePct = percentage(metrics.PrefixHits, metrics.PrefixRequests) + metrics.TenantHotHitRatePct = percentage(metrics.TenantHotHits, metrics.TenantHotRequests) + samples := 0 + if metrics.PrefixRequests > 0 { + metrics.MacroHitRatePct += metrics.PrefixHitRatePct + samples++ + } + if metrics.TenantHotRequests > 0 { + metrics.MacroHitRatePct += metrics.TenantHotHitRatePct + samples++ + } + if samples > 0 { + metrics.MacroHitRatePct /= float64(samples) + } + return metrics +} + +func replayObservation(trace Trace, observation Observation, config Config) bool { + anchor := time.UnixMilli(observation.AtMillis) + ttl := time.Duration(trace.TTLMillis) * time.Millisecond + ranker := index.RankerConfig{ + PressureWeight: float32(config.PressureWeight), + SLOTightTTFTMs: config.SLOTightTTFTMillis, + SLOTightBias: float32(config.SLOTightBias), + TenantHotMinHitRate: float32(config.TenantHotMinHitRate), + TenantHotMaxAge: time.Duration(config.TenantHotMaxAgeMillis) * time.Millisecond, + } + idx := index.New( + index.WithTTL(ttl), + index.WithRanker(ranker), + index.WithClock(func() time.Time { return anchor }), + ) + observedHits := make(map[string]bool, len(observation.Replicas)) + for _, replica := range observation.Replicas { + hash := observation.PrefixHash + tokens := replica.MatchedTokens + if !replica.ReportedPrefix { + hash = servingOnlyHash(observation.PrefixHash, replica.ID) + tokens = 1 + } + prefixReportedAt := time.UnixMilli(replica.PrefixReportedAtMillis) + if anchor.Sub(prefixReportedAt) < ttl { + idx.Ingest(index.Update{ + ReplicaID: replica.ID, + Model: observation.Model, + Tenant: observation.Tenant, + HashScheme: observation.HashScheme, + Timestamp: prefixReportedAt, + Prefixes: []index.PrefixRef{{ + PrefixHash: hash, + TokenCount: tokens, + }}, + }) + } + statsReportedAt := time.UnixMilli(replica.StatsReportedAtMillis) + if anchor.Sub(statsReportedAt) < ttl { + idx.Ingest(index.Update{ + ReplicaID: replica.ID, + Model: observation.Model, + Tenant: observation.Tenant, + HashScheme: observation.HashScheme, + Timestamp: statsReportedAt, + Stats: &index.ReplicaStats{ + HitRate: replica.HitRate, + Pressure: replica.Pressure, + }, + }) + } + observedHits[replica.ID] = replica.ObservedHit + } + result := idx.LookupRoute(index.LookupRequest{ + Model: observation.Model, + Tenant: observation.Tenant, + HashScheme: observation.HashScheme, + PrefixHash: observation.PrefixHash, + TokenCount: observation.TokenCount, + TTFTBudgetMs: observation.TTFTBudgetMillis, + }) + wantStrategy := index.StrategyPrefixMatch + if observation.Kind == ObservationTenantHot { + wantStrategy = index.StrategyTenantHot + } + return result.Strategy == wantStrategy && len(result.Scores) > 0 && observedHits[result.Scores[0].ReplicaID] +} + +func servingOnlyHash(requested []byte, replicaID string) []byte { + hash := make([]byte, 0, len(requested)+1+len(replicaID)) + hash = append(hash, requested...) + hash = append(hash, 0) + return append(hash, replicaID...) +} + +func percentage(numerator, denominator int) float64 { + if denominator == 0 { + return 0 + } + return float64(numerator) * 100 / float64(denominator) +} + +func pressureCurve(trace Trace, best Config) []CurvePoint { + return floatCurve(trace.Sweep.PressureWeights, func(value float64) Metrics { + candidate := best + candidate.PressureWeight = value + return Replay(trace, candidate) + }) +} + +func sloBiasCurve(trace Trace, best Config) []CurvePoint { + return floatCurve(trace.Sweep.SLOTightBiases, func(value float64) Metrics { + candidate := best + candidate.SLOTightBias = value + return Replay(trace, candidate) + }) +} + +func tenantHotRateCurve(trace Trace, best Config) []CurvePoint { + return floatCurve(trace.Sweep.TenantHotMinHitRates, func(value float64) Metrics { + candidate := best + candidate.TenantHotMinHitRate = value + return Replay(trace, candidate) + }) +} + +func floatCurve(values []float64, replay func(float64) Metrics) []CurvePoint { + values = append([]float64(nil), values...) + sort.Slice(values, func(i, j int) bool { return values[i] < values[j] }) + points := make([]CurvePoint, 0, len(values)) + for _, value := range values { + points = append(points, CurvePoint{Value: float64(value), Metrics: replay(value)}) + } + return points +} + +func sloThresholdCurve(trace Trace, best Config) []CurvePoint { + values := append([]int32(nil), trace.Sweep.SLOTightTTFTMillis...) + sort.Slice(values, func(i, j int) bool { return values[i] < values[j] }) + points := make([]CurvePoint, 0, len(values)) + for _, value := range values { + candidate := best + candidate.SLOTightTTFTMillis = value + points = append(points, CurvePoint{Value: float64(value), Metrics: Replay(trace, candidate)}) + } + return points +} + +func tenantHotAgeCurve(trace Trace, best Config) []CurvePoint { + values := append([]int64(nil), trace.Sweep.TenantHotMaxAgeMillis...) + sort.Slice(values, func(i, j int) bool { return values[i] < values[j] }) + points := make([]CurvePoint, 0, len(values)) + for _, value := range values { + candidate := best + candidate.TenantHotMaxAgeMillis = value + points = append(points, CurvePoint{Value: float64(value), Metrics: Replay(trace, candidate)}) + } + return points +} + +func MarshalResult(result Result) ([]byte, error) { + data, err := json.MarshalIndent(result, "", " ") + if err != nil { + return nil, fmt.Errorf("marshal calibration result: %w", err) + } + return append(data, '\n'), nil +} diff --git a/internal/index/calibration/calibration_test.go b/internal/index/calibration/calibration_test.go new file mode 100644 index 00000000..c87b9c5d --- /dev/null +++ b/internal/index/calibration/calibration_test.go @@ -0,0 +1,400 @@ +// SPDX-FileCopyrightText: 2026 The inference-cache Authors +// +// SPDX-License-Identifier: Apache-2.0 + +package calibration + +import ( + "bytes" + "encoding/json" + "math" + "os" + "strings" + "testing" + "time" +) + +func TestCheckedInSyntheticTraceSelectsCandidateAndCurrentResult(t *testing.T) { + traceFile, err := os.Open("testdata/c1_synthetic_trace.json") + if err != nil { + t.Fatalf("open trace: %v", err) + } + trace, err := Load(traceFile) + if closeErr := traceFile.Close(); closeErr != nil { + t.Fatalf("close trace: %v", closeErr) + } + if err != nil { + t.Fatalf("Load: %v", err) + } + result := Calibrate(trace) + if trace.Provenance.Kind != "synthetic" { + t.Fatalf("provenance kind = %q, want synthetic", trace.Provenance.Kind) + } + if !bytes.Equal(trace.Observations[0].PrefixHash, []byte("pressure-shed-1")) { + t.Fatalf("decoded prefix hash = %x, want opaque fixture bytes", trace.Observations[0].PrefixHash) + } + want := Config{ + PressureWeight: 0.5, + SLOTightTTFTMillis: 200, + SLOTightBias: 1, + TenantHotMinHitRate: 0.2, + TenantHotMaxAgeMillis: 120_000, + } + if result.BestConfig != want { + t.Fatalf("synthetic candidate = %+v, want %+v", result.BestConfig, want) + } + if result.BestMetrics.PrefixHitRatePct != 100 || result.BestMetrics.TenantHotHitRatePct != 100 { + t.Fatalf("best metrics = %+v, want both fixture hit rates at 100%%", result.BestMetrics) + } + got, err := MarshalResult(result) + if err != nil { + t.Fatalf("MarshalResult: %v", err) + } + committed, err := os.ReadFile("testdata/c1_synthetic_result.json") + if err != nil { + t.Fatalf("read committed result: %v", err) + } + if !bytes.Equal(got, committed) { + t.Fatal("c1_synthetic_result.json is stale; run make ranker-calibration") + } +} + +func TestCalibrateSeparatesKnobEffects(t *testing.T) { + trace := Trace{ + SchemaVersion: SchemaVersion, + Name: "unit", + Provenance: Provenance{Kind: "synthetic", Source: "unit test"}, + TTLMillis: 100_000, + Sweep: Sweep{ + PressureWeights: []float64{0, 0.5, 1}, + SLOTightTTFTMillis: []int32{100, 200}, + SLOTightBiases: []float64{0, 1}, + TenantHotMinHitRates: []float64{0.1, 0.2}, + TenantHotMaxAgeMillis: []int64{60_000, 120_000}, + }, + Observations: []Observation{ + { + ID: "pressure", Kind: ObservationPrefix, AtMillis: 100_000, + Tenant: "tenant-a", Model: "model-a", HashScheme: "vllm", + PrefixHash: []byte("p"), TokenCount: 320, + Replicas: []ReplicaObservation{ + {ID: "hot", PrefixReportedAtMillis: 100_000, StatsReportedAtMillis: 100_000, ReportedPrefix: true, MatchedTokens: 320, HitRate: 0.8, Pressure: 0.8, OutcomeAvailable: true}, + {ID: "cool", PrefixReportedAtMillis: 100_000, StatsReportedAtMillis: 100_000, ReportedPrefix: true, MatchedTokens: 256, HitRate: 0.4, Pressure: 0.1, OutcomeAvailable: true, ObservedHit: true}, + {ID: "decoy", PrefixReportedAtMillis: 100_000, StatsReportedAtMillis: 100_000, HitRate: 0.1, OutcomeAvailable: true}, + }, + }, + { + ID: "tenant-hot", Kind: ObservationTenantHot, AtMillis: 100_000, + Tenant: "tenant-a", Model: "model-a", HashScheme: "vllm", + PrefixHash: []byte("other"), TokenCount: 320, + Replicas: []ReplicaObservation{{ + ID: "hot", PrefixReportedAtMillis: 100_000, StatsReportedAtMillis: 100_000, + HitRate: 1, OutcomeAvailable: true, ObservedHit: true, + }}, + }, + }, + } + if err := trace.Validate(); err != nil { + t.Fatalf("Validate: %v", err) + } + result := Calibrate(trace) + if result.BestConfig.PressureWeight != 0.5 { + t.Fatalf("PressureWeight = %v, want 0.5", result.BestConfig.PressureWeight) + } + if result.BestMetrics.PrefixHitRatePct != 100 { + t.Fatalf("PrefixHitRatePct = %v, want 100", result.BestMetrics.PrefixHitRatePct) + } + if got := len(result.Curves["pressure_weight"]); got != 3 { + t.Fatalf("pressure curve points = %d, want 3", got) + } +} + +func TestLoadRejectsUnknownAndInvalidFields(t *testing.T) { + for _, tc := range []struct { + name string + json string + want string + }{ + {"unknown", `{"schema_version":1,"unknown":true}`, "unknown field"}, + {"version", `{"schema_version":2}`, "schema_version"}, + {"trailing", `{"schema_version":1} {}`, "trailing JSON"}, + } { + t.Run(tc.name, func(t *testing.T) { + _, err := Load(strings.NewReader(tc.json)) + if err == nil || !strings.Contains(err.Error(), tc.want) { + t.Fatalf("Load error = %v, want substring %q", err, tc.want) + } + }) + } +} + +func TestTraceValidationRejectsInvalidFields(t *testing.T) { + valid := func() Trace { + return Trace{ + SchemaVersion: SchemaVersion, + Name: "unit", + Provenance: Provenance{Kind: "synthetic", Source: "unit test"}, + TTLMillis: 1, + Sweep: Sweep{ + PressureWeights: []float64{0.5}, + SLOTightTTFTMillis: []int32{200}, + SLOTightBiases: []float64{1}, + TenantHotMinHitRates: []float64{0.2}, + TenantHotMaxAgeMillis: []int64{60_000}, + }, + Observations: []Observation{ + { + ID: "prefix", Kind: ObservationPrefix, Tenant: "t", Model: "m", + HashScheme: "vllm", PrefixHash: []byte("p"), TokenCount: 1, + Replicas: []ReplicaObservation{{ + ID: "r", ReportedPrefix: true, MatchedTokens: 1, OutcomeAvailable: true, + }}, + }, + { + ID: "tenant-hot", Kind: ObservationTenantHot, Tenant: "t", Model: "m", + HashScheme: "vllm", PrefixHash: []byte("p"), TokenCount: 1, + Replicas: []ReplicaObservation{{ID: "r", OutcomeAvailable: true}}, + }, + }, + } + } + + for _, tc := range []struct { + name string + mutate func(*Trace) + want string + }{ + {"version", func(trace *Trace) { trace.SchemaVersion++ }, "schema_version"}, + {"name", func(trace *Trace) { trace.Name = "" }, "trace name"}, + {"provenance kind", func(trace *Trace) { trace.Provenance.Kind = "unknown" }, "captured or synthetic"}, + {"provenance source", func(trace *Trace) { trace.Provenance.Source = "" }, "provenance source"}, + {"ttl", func(trace *Trace) { trace.TTLMillis = 0 }, "ttl_ms"}, + {"ttl overflow", func(trace *Trace) { trace.TTLMillis = maxDurationMillis + 1 }, "maximum representable"}, + {"empty sweep", func(trace *Trace) { trace.Sweep.PressureWeights = nil }, "every sweep dimension"}, + {"empty observations", func(trace *Trace) { trace.Observations = nil }, "at least one observation"}, + {"missing prefix observation", func(trace *Trace) { trace.Observations = trace.Observations[1:] }, "one prefix and one tenant_hot"}, + {"missing tenant-hot observation", func(trace *Trace) { trace.Observations = trace.Observations[:1] }, "one prefix and one tenant_hot"}, + {"invalid observation", func(trace *Trace) { trace.Observations[0].Kind = "unknown" }, "observation 0"}, + {"duplicate observation", func(trace *Trace) { trace.Observations = append(trace.Observations, trace.Observations[0]) }, "duplicate id"}, + } { + t.Run(tc.name, func(t *testing.T) { + trace := valid() + tc.mutate(&trace) + if err := trace.Validate(); err == nil || !strings.Contains(err.Error(), tc.want) { + t.Fatalf("Validate error = %v, want substring %q", err, tc.want) + } + }) + } +} + +func TestSweepValidationRejectsInvalidValues(t *testing.T) { + valid := func() Sweep { + return Sweep{ + PressureWeights: []float64{0.5}, + SLOTightTTFTMillis: []int32{200}, + SLOTightBiases: []float64{1}, + TenantHotMinHitRates: []float64{0.2}, + TenantHotMaxAgeMillis: []int64{60_000}, + } + } + for _, tc := range []struct { + name string + mutate func(*Sweep) + want string + }{ + {"pressure", func(sweep *Sweep) { sweep.PressureWeights[0] = -1 }, "non-negative"}, + {"pressure float32 overflow", func(sweep *Sweep) { sweep.PressureWeights[0] = math.MaxFloat64 }, "float32"}, + {"bias float32 overflow", func(sweep *Sweep) { sweep.SLOTightBiases[0] = math.MaxFloat64 }, "float32"}, + {"hit rate", func(sweep *Sweep) { sweep.TenantHotMinHitRates[0] = 2 }, "invalid rate"}, + {"ttft", func(sweep *Sweep) { sweep.SLOTightTTFTMillis[0] = -1 }, "negative"}, + {"max age", func(sweep *Sweep) { sweep.TenantHotMaxAgeMillis[0] = -1 }, "negative"}, + {"max age overflow", func(sweep *Sweep) { sweep.TenantHotMaxAgeMillis[0] = maxDurationMillis + 1 }, "maximum representable"}, + } { + t.Run(tc.name, func(t *testing.T) { + sweep := valid() + tc.mutate(&sweep) + if err := sweep.validate(); err == nil || !strings.Contains(err.Error(), tc.want) { + t.Fatalf("validate error = %v, want substring %q", err, tc.want) + } + }) + } + t.Run("zero kill switches", func(t *testing.T) { + sweep := valid() + sweep.SLOTightTTFTMillis[0] = 0 + sweep.TenantHotMaxAgeMillis[0] = 0 + if err := sweep.validate(); err != nil { + t.Fatalf("validate zero kill switches: %v", err) + } + }) +} + +func TestObservationValidationRejectsInvalidReplicas(t *testing.T) { + valid := func() Observation { + return Observation{ + ID: "o", Kind: ObservationPrefix, Tenant: "t", Model: "m", + HashScheme: "vllm", PrefixHash: []byte("p"), TokenCount: 1, + Replicas: []ReplicaObservation{{ID: "r", ReportedPrefix: true, MatchedTokens: 1, OutcomeAvailable: true}}, + } + } + for _, tc := range []struct { + name string + mutate func(*Observation) + want string + }{ + {"identity", func(observation *Observation) { observation.ID = "" }, "required"}, + {"kind", func(observation *Observation) { observation.Kind = "unknown" }, "prefix or tenant_hot"}, + {"tokens", func(observation *Observation) { observation.TokenCount = 0 }, "token_count"}, + {"negative ttft budget", func(observation *Observation) { observation.TTFTBudgetMillis = -1 }, "ttft_budget_ms"}, + {"replicas", func(observation *Observation) { observation.Replicas = nil }, "at least one replica"}, + {"replica id", func(observation *Observation) { observation.Replicas[0].ID = "" }, "id is required"}, + {"matched tokens", func(observation *Observation) { observation.Replicas[0].MatchedTokens = 0 }, "matched_tokens"}, + {"negative unused matched tokens", func(observation *Observation) { + observation.Replicas[0].ReportedPrefix = false + observation.Replicas[0].MatchedTokens = -1 + }, "matched_tokens must be non-negative"}, + {"rate", func(observation *Observation) { observation.Replicas[0].HitRate = 2 }, "finite values"}, + {"outcome unavailable", func(observation *Observation) { observation.Replicas[0].OutcomeAvailable = false }, "outcome_available"}, + {"duplicate replica", func(observation *Observation) { + observation.Replicas = append(observation.Replicas, observation.Replicas[0]) + }, "duplicate replica"}, + } { + t.Run(tc.name, func(t *testing.T) { + observation := valid() + tc.mutate(&observation) + if err := observation.validate(); err == nil || !strings.Contains(err.Error(), tc.want) { + t.Fatalf("validate error = %v, want substring %q", err, tc.want) + } + }) + } +} + +func TestMarshalResultWrapsJSONErrors(t *testing.T) { + _, err := MarshalResult(Result{BestConfig: Config{PressureWeight: math.NaN()}}) + if err == nil || !strings.Contains(err.Error(), "marshal calibration result") { + t.Fatalf("MarshalResult error = %v, want wrapped JSON error", err) + } +} + +func TestObservationPrefixHashRoundTripsOpaqueBytes(t *testing.T) { + want := []byte{0, 0xff, 0x80, 'x'} + encoded, err := json.Marshal(Observation{PrefixHash: want}) + if err != nil { + t.Fatalf("Marshal: %v", err) + } + var got Observation + if err := json.Unmarshal(encoded, &got); err != nil { + t.Fatalf("Unmarshal: %v", err) + } + if !bytes.Equal(got.PrefixHash, want) { + t.Fatalf("prefix hash = %x, want %x", got.PrefixHash, want) + } +} + +func TestReplayTenantHotMissWithoutCandidate(t *testing.T) { + trace := Trace{TTLMillis: int64(time.Minute / time.Millisecond)} + observation := Observation{ + ID: "tenant-hot", Kind: ObservationTenantHot, AtMillis: 100_000, + Tenant: "t", Model: "m", HashScheme: "vllm", PrefixHash: []byte("novel"), TokenCount: 1, + Replicas: []ReplicaObservation{{ + ID: "cold", PrefixReportedAtMillis: 100_000, StatsReportedAtMillis: 100_000, + HitRate: 0.1, ObservedHit: true, + }}, + } + config := Config{TenantHotMinHitRate: 0.2, TenantHotMaxAgeMillis: 60_000} + if replayObservation(trace, observation, config) { + t.Fatal("replayObservation = hit, want miss when every tenant-hot candidate is below the floor") + } +} + +func TestReplayUsesObservationClockAtTenantHotBoundary(t *testing.T) { + trace := Trace{TTLMillis: int64(time.Minute / time.Millisecond)} + observation := Observation{ + ID: "tenant-hot-boundary", Kind: ObservationTenantHot, AtMillis: 100_000, + Tenant: "t", Model: "m", HashScheme: "vllm", PrefixHash: []byte("novel"), TokenCount: 1, + Replicas: []ReplicaObservation{{ + ID: "warm", PrefixReportedAtMillis: 40_001, StatsReportedAtMillis: 40_001, + HitRate: 0.8, ObservedHit: true, + }}, + } + config := Config{TenantHotMinHitRate: 0.2, TenantHotMaxAgeMillis: 60_000} + if !replayObservation(trace, observation, config) { + t.Fatal("replayObservation = miss, want hit for stats one millisecond inside the replay window") + } +} + +func TestReplayUsesIndependentPrefixAndStatsTimestamps(t *testing.T) { + trace := Trace{TTLMillis: int64(time.Minute / time.Millisecond)} + observation := Observation{ + ID: "independent-clocks", Kind: ObservationPrefix, AtMillis: 100_000, + Tenant: "t", Model: "m", HashScheme: "vllm", PrefixHash: []byte("p"), TokenCount: 320, + Replicas: []ReplicaObservation{ + { + ID: "deep-stale-stats", PrefixReportedAtMillis: 100_000, StatsReportedAtMillis: 0, + ReportedPrefix: true, MatchedTokens: 320, Pressure: 1, ObservedHit: true, + }, + { + ID: "shallow-fresh-stats", PrefixReportedAtMillis: 100_000, StatsReportedAtMillis: 100_000, + ReportedPrefix: true, MatchedTokens: 256, + }, + }, + } + config := Config{PressureWeight: 1, TenantHotMaxAgeMillis: 60_000} + if !replayObservation(trace, observation, config) { + t.Fatal("replayObservation = miss, want stale pressure ignored while fresh prefix remains routable") + } +} + +func TestReplayExcludesTTLExpiredServingEntries(t *testing.T) { + trace := Trace{TTLMillis: int64(time.Minute / time.Millisecond)} + observation := Observation{ + ID: "expired-serving", Kind: ObservationTenantHot, AtMillis: 100_000, + Tenant: "t", Model: "m", HashScheme: "vllm", PrefixHash: []byte("novel"), TokenCount: 1, + Replicas: []ReplicaObservation{{ + ID: "stale", PrefixReportedAtMillis: 40_000, StatsReportedAtMillis: 100_000, + HitRate: 1, ObservedHit: true, + }}, + } + config := Config{TenantHotMinHitRate: 0.2, TenantHotMaxAgeMillis: 120_000} + if replayObservation(trace, observation, config) { + t.Fatal("replayObservation = hit, want TTL-expired serving entry evicted before lookup") + } +} + +func TestReplayServingOnlyHashCannotMatchRequestedPrefix(t *testing.T) { + trace := Trace{TTLMillis: int64(time.Minute / time.Millisecond)} + observation := Observation{ + ID: "collision", Kind: ObservationPrefix, AtMillis: 100_000, + Tenant: "t", Model: "m", HashScheme: "vllm", + PrefixHash: []byte("serving/collision/not-holder"), TokenCount: 1, + Replicas: []ReplicaObservation{{ + ID: "not-holder", PrefixReportedAtMillis: 100_000, StatsReportedAtMillis: 100_000, + ObservedHit: true, + }}, + } + if replayObservation(trace, observation, Config{TenantHotMaxAgeMillis: 60_000}) { + t.Fatal("replayObservation = hit, want serving-only key distinct from requested prefix") + } +} + +func TestObservationRejectsFutureReplicaReport(t *testing.T) { + for _, tc := range []struct { + name string + mutate func(*ReplicaObservation) + want string + }{ + {"prefix", func(replica *ReplicaObservation) { replica.PrefixReportedAtMillis = 11 }, "prefix_reported_at_ms"}, + {"stats", func(replica *ReplicaObservation) { replica.StatsReportedAtMillis = 11 }, "stats_reported_at_ms"}, + } { + t.Run(tc.name, func(t *testing.T) { + observation := Observation{ + ID: "future", Kind: ObservationPrefix, AtMillis: 10, + Tenant: "t", Model: "m", HashScheme: "vllm", PrefixHash: []byte("p"), TokenCount: 1, + Replicas: []ReplicaObservation{{ID: "r", ReportedPrefix: true, MatchedTokens: 1, OutcomeAvailable: true}}, + } + tc.mutate(&observation.Replicas[0]) + if err := observation.validate(); err == nil || !strings.Contains(err.Error(), tc.want) { + t.Fatalf("validate error = %v, want substring %q", err, tc.want) + } + }) + } +} diff --git a/internal/index/calibration/testdata/c1_synthetic_result.json b/internal/index/calibration/testdata/c1_synthetic_result.json new file mode 100644 index 00000000..7649e9e9 --- /dev/null +++ b/internal/index/calibration/testdata/c1_synthetic_result.json @@ -0,0 +1,326 @@ +{ + "schema_version": 1, + "trace_name": "c1-synthetic-mixed-routing-v1", + "provenance": { + "kind": "synthetic", + "source": "Deterministic boundary-case replay derived from the C1 ReplicaStats and LookupRoute contracts", + "description": "No production C1 request trace is checked into the repository. This fixture exercises pressure/locality tradeoffs, tight and loose TTFT budgets, stale soft-state observations, and TENANT_HOT rate/age gates without claiming to represent production traffic." + }, + "observations": 22, + "best_config": { + "pressure_weight": 0.5, + "slo_tight_ttft_ms": 200, + "slo_tight_bias": 1, + "tenant_hot_min_hit_rate": 0.2, + "tenant_hot_max_age_ms": 120000 + }, + "best_metrics": { + "prefix_requests": 14, + "prefix_hits": 14, + "prefix_hit_rate_pct": 100, + "tenant_hot_requests": 8, + "tenant_hot_hits": 8, + "tenant_hot_hit_rate_pct": 100, + "macro_hit_rate_pct": 100 + }, + "curves": { + "pressure_weight": [ + { + "value": 0, + "metrics": { + "prefix_requests": 14, + "prefix_hits": 11, + "prefix_hit_rate_pct": 78.57142857142857, + "tenant_hot_requests": 8, + "tenant_hot_hits": 8, + "tenant_hot_hit_rate_pct": 100, + "macro_hit_rate_pct": 89.28571428571428 + } + }, + { + "value": 0.25, + "metrics": { + "prefix_requests": 14, + "prefix_hits": 11, + "prefix_hit_rate_pct": 78.57142857142857, + "tenant_hot_requests": 8, + "tenant_hot_hits": 8, + "tenant_hot_hit_rate_pct": 100, + "macro_hit_rate_pct": 89.28571428571428 + } + }, + { + "value": 0.5, + "metrics": { + "prefix_requests": 14, + "prefix_hits": 14, + "prefix_hit_rate_pct": 100, + "tenant_hot_requests": 8, + "tenant_hot_hits": 8, + "tenant_hot_hit_rate_pct": 100, + "macro_hit_rate_pct": 100 + } + }, + { + "value": 0.75, + "metrics": { + "prefix_requests": 14, + "prefix_hits": 11, + "prefix_hit_rate_pct": 78.57142857142857, + "tenant_hot_requests": 8, + "tenant_hot_hits": 8, + "tenant_hot_hit_rate_pct": 100, + "macro_hit_rate_pct": 89.28571428571428 + } + }, + { + "value": 1, + "metrics": { + "prefix_requests": 14, + "prefix_hits": 11, + "prefix_hit_rate_pct": 78.57142857142857, + "tenant_hot_requests": 8, + "tenant_hot_hits": 8, + "tenant_hot_hit_rate_pct": 100, + "macro_hit_rate_pct": 89.28571428571428 + } + } + ], + "slo_tight_bias": [ + { + "value": 0, + "metrics": { + "prefix_requests": 14, + "prefix_hits": 10, + "prefix_hit_rate_pct": 71.42857142857143, + "tenant_hot_requests": 8, + "tenant_hot_hits": 8, + "tenant_hot_hit_rate_pct": 100, + "macro_hit_rate_pct": 85.71428571428572 + } + }, + { + "value": 0.5, + "metrics": { + "prefix_requests": 14, + "prefix_hits": 10, + "prefix_hit_rate_pct": 71.42857142857143, + "tenant_hot_requests": 8, + "tenant_hot_hits": 8, + "tenant_hot_hit_rate_pct": 100, + "macro_hit_rate_pct": 85.71428571428572 + } + }, + { + "value": 1, + "metrics": { + "prefix_requests": 14, + "prefix_hits": 14, + "prefix_hit_rate_pct": 100, + "tenant_hot_requests": 8, + "tenant_hot_hits": 8, + "tenant_hot_hit_rate_pct": 100, + "macro_hit_rate_pct": 100 + } + }, + { + "value": 1.5, + "metrics": { + "prefix_requests": 14, + "prefix_hits": 12, + "prefix_hit_rate_pct": 85.71428571428571, + "tenant_hot_requests": 8, + "tenant_hot_hits": 8, + "tenant_hot_hit_rate_pct": 100, + "macro_hit_rate_pct": 92.85714285714286 + } + }, + { + "value": 2, + "metrics": { + "prefix_requests": 14, + "prefix_hits": 12, + "prefix_hit_rate_pct": 85.71428571428571, + "tenant_hot_requests": 8, + "tenant_hot_hits": 8, + "tenant_hot_hit_rate_pct": 100, + "macro_hit_rate_pct": 92.85714285714286 + } + } + ], + "slo_tight_ttft_ms": [ + { + "value": 100, + "metrics": { + "prefix_requests": 14, + "prefix_hits": 10, + "prefix_hit_rate_pct": 71.42857142857143, + "tenant_hot_requests": 8, + "tenant_hot_hits": 8, + "tenant_hot_hit_rate_pct": 100, + "macro_hit_rate_pct": 85.71428571428572 + } + }, + { + "value": 150, + "metrics": { + "prefix_requests": 14, + "prefix_hits": 12, + "prefix_hit_rate_pct": 85.71428571428571, + "tenant_hot_requests": 8, + "tenant_hot_hits": 8, + "tenant_hot_hit_rate_pct": 100, + "macro_hit_rate_pct": 92.85714285714286 + } + }, + { + "value": 200, + "metrics": { + "prefix_requests": 14, + "prefix_hits": 14, + "prefix_hit_rate_pct": 100, + "tenant_hot_requests": 8, + "tenant_hot_hits": 8, + "tenant_hot_hit_rate_pct": 100, + "macro_hit_rate_pct": 100 + } + }, + { + "value": 250, + "metrics": { + "prefix_requests": 14, + "prefix_hits": 12, + "prefix_hit_rate_pct": 85.71428571428571, + "tenant_hot_requests": 8, + "tenant_hot_hits": 8, + "tenant_hot_hit_rate_pct": 100, + "macro_hit_rate_pct": 92.85714285714286 + } + }, + { + "value": 300, + "metrics": { + "prefix_requests": 14, + "prefix_hits": 12, + "prefix_hit_rate_pct": 85.71428571428571, + "tenant_hot_requests": 8, + "tenant_hot_hits": 8, + "tenant_hot_hit_rate_pct": 100, + "macro_hit_rate_pct": 92.85714285714286 + } + } + ], + "tenant_hot_max_age_ms": [ + { + "value": 60000, + "metrics": { + "prefix_requests": 14, + "prefix_hits": 14, + "prefix_hit_rate_pct": 100, + "tenant_hot_requests": 8, + "tenant_hot_hits": 6, + "tenant_hot_hit_rate_pct": 75, + "macro_hit_rate_pct": 87.5 + } + }, + { + "value": 120000, + "metrics": { + "prefix_requests": 14, + "prefix_hits": 14, + "prefix_hit_rate_pct": 100, + "tenant_hot_requests": 8, + "tenant_hot_hits": 8, + "tenant_hot_hit_rate_pct": 100, + "macro_hit_rate_pct": 100 + } + }, + { + "value": 300000, + "metrics": { + "prefix_requests": 14, + "prefix_hits": 14, + "prefix_hit_rate_pct": 100, + "tenant_hot_requests": 8, + "tenant_hot_hits": 5, + "tenant_hot_hit_rate_pct": 62.5, + "macro_hit_rate_pct": 81.25 + } + }, + { + "value": 600000, + "metrics": { + "prefix_requests": 14, + "prefix_hits": 14, + "prefix_hit_rate_pct": 100, + "tenant_hot_requests": 8, + "tenant_hot_hits": 5, + "tenant_hot_hit_rate_pct": 62.5, + "macro_hit_rate_pct": 81.25 + } + } + ], + "tenant_hot_min_hit_rate": [ + { + "value": 0.05, + "metrics": { + "prefix_requests": 14, + "prefix_hits": 14, + "prefix_hit_rate_pct": 100, + "tenant_hot_requests": 8, + "tenant_hot_hits": 5, + "tenant_hot_hit_rate_pct": 62.5, + "macro_hit_rate_pct": 81.25 + } + }, + { + "value": 0.1, + "metrics": { + "prefix_requests": 14, + "prefix_hits": 14, + "prefix_hit_rate_pct": 100, + "tenant_hot_requests": 8, + "tenant_hot_hits": 5, + "tenant_hot_hit_rate_pct": 62.5, + "macro_hit_rate_pct": 81.25 + } + }, + { + "value": 0.2, + "metrics": { + "prefix_requests": 14, + "prefix_hits": 14, + "prefix_hit_rate_pct": 100, + "tenant_hot_requests": 8, + "tenant_hot_hits": 8, + "tenant_hot_hit_rate_pct": 100, + "macro_hit_rate_pct": 100 + } + }, + { + "value": 0.3, + "metrics": { + "prefix_requests": 14, + "prefix_hits": 14, + "prefix_hit_rate_pct": 100, + "tenant_hot_requests": 8, + "tenant_hot_hits": 5, + "tenant_hot_hit_rate_pct": 62.5, + "macro_hit_rate_pct": 81.25 + } + }, + { + "value": 0.4, + "metrics": { + "prefix_requests": 14, + "prefix_hits": 14, + "prefix_hit_rate_pct": 100, + "tenant_hot_requests": 8, + "tenant_hot_hits": 0, + "tenant_hot_hit_rate_pct": 0, + "macro_hit_rate_pct": 50 + } + } + ] + } +} diff --git a/internal/index/calibration/testdata/c1_synthetic_trace.json b/internal/index/calibration/testdata/c1_synthetic_trace.json new file mode 100644 index 00000000..a9fa43d3 --- /dev/null +++ b/internal/index/calibration/testdata/c1_synthetic_trace.json @@ -0,0 +1,347 @@ +{ + "schema_version": 1, + "name": "c1-synthetic-mixed-routing-v1", + "provenance": { + "kind": "synthetic", + "source": "Deterministic boundary-case replay derived from the C1 ReplicaStats and LookupRoute contracts", + "description": "No production C1 request trace is checked into the repository. This fixture exercises pressure/locality tradeoffs, tight and loose TTFT budgets, stale soft-state observations, and TENANT_HOT rate/age gates without claiming to represent production traffic." + }, + "ttl_ms": 1800000, + "sweep": { + "pressure_weights": [0, 0.25, 0.5, 0.75, 1], + "slo_tight_ttft_ms": [100, 150, 200, 250, 300], + "slo_tight_biases": [0, 0.5, 1, 1.5, 2], + "tenant_hot_min_hit_rates": [0.05, 0.1, 0.2, 0.3, 0.4], + "tenant_hot_max_age_ms": [60000, 120000, 300000, 600000] + }, + "observations": [ + { + "id": "pressure-shed-1", + "kind": "prefix", + "at_ms": 100000, + "tenant": "tenant-a", + "model": "model-a", + "hash_scheme": "vllm", + "prefix_hash": "cHJlc3N1cmUtc2hlZC0x", + "token_count": 320, + "replicas": [ + {"id": "a-hot", "prefix_reported_at_ms": 100000, "stats_reported_at_ms": 100000, "reported_prefix": true, "matched_tokens": 320, "hit_rate": 0.7, "pressure": 0.8, "outcome_available": true, "observed_hit": false}, + {"id": "b-cool", "prefix_reported_at_ms": 100000, "stats_reported_at_ms": 100000, "reported_prefix": true, "matched_tokens": 256, "hit_rate": 0.4, "pressure": 0.1, "outcome_available": true, "observed_hit": true}, + {"id": "z-decoy", "prefix_reported_at_ms": 100000, "stats_reported_at_ms": 100000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.1, "pressure": 0, "outcome_available": true, "observed_hit": false} + ] + }, + { + "id": "pressure-shed-2", + "kind": "prefix", + "at_ms": 200000, + "tenant": "tenant-a", + "model": "model-a", + "hash_scheme": "vllm", + "prefix_hash": "cHJlc3N1cmUtc2hlZC0y", + "token_count": 320, + "replicas": [ + {"id": "a-hot", "prefix_reported_at_ms": 200000, "stats_reported_at_ms": 200000, "reported_prefix": true, "matched_tokens": 320, "hit_rate": 0.7, "pressure": 0.8, "outcome_available": true, "observed_hit": false}, + {"id": "b-cool", "prefix_reported_at_ms": 200000, "stats_reported_at_ms": 200000, "reported_prefix": true, "matched_tokens": 256, "hit_rate": 0.4, "pressure": 0.1, "outcome_available": true, "observed_hit": true}, + {"id": "z-decoy", "prefix_reported_at_ms": 200000, "stats_reported_at_ms": 200000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.1, "pressure": 0, "outcome_available": true, "observed_hit": false} + ] + }, + { + "id": "pressure-shed-3", + "kind": "prefix", + "at_ms": 300000, + "tenant": "tenant-b", + "model": "model-b", + "hash_scheme": "vllm", + "prefix_hash": "cHJlc3N1cmUtc2hlZC0z", + "token_count": 320, + "replicas": [ + {"id": "a-hot", "prefix_reported_at_ms": 300000, "stats_reported_at_ms": 300000, "reported_prefix": true, "matched_tokens": 320, "hit_rate": 0.65, "pressure": 0.8, "outcome_available": true, "observed_hit": false}, + {"id": "b-cool", "prefix_reported_at_ms": 300000, "stats_reported_at_ms": 300000, "reported_prefix": true, "matched_tokens": 256, "hit_rate": 0.45, "pressure": 0.1, "outcome_available": true, "observed_hit": true}, + {"id": "z-decoy", "prefix_reported_at_ms": 300000, "stats_reported_at_ms": 300000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.1, "pressure": 0, "outcome_available": true, "observed_hit": false} + ] + }, + { + "id": "pressure-preserve-1", + "kind": "prefix", + "at_ms": 400000, + "tenant": "tenant-a", + "model": "model-a", + "hash_scheme": "vllm", + "prefix_hash": "cHJlc3N1cmUtcHJlc2VydmUtMQ==", + "token_count": 512, + "replicas": [ + {"id": "a-local", "prefix_reported_at_ms": 400000, "stats_reported_at_ms": 400000, "reported_prefix": true, "matched_tokens": 512, "hit_rate": 0.75, "pressure": 0.7, "outcome_available": true, "observed_hit": true}, + {"id": "b-cool", "prefix_reported_at_ms": 400000, "stats_reported_at_ms": 400000, "reported_prefix": true, "matched_tokens": 256, "hit_rate": 0.3, "pressure": 0, "outcome_available": true, "observed_hit": false}, + {"id": "z-decoy", "prefix_reported_at_ms": 400000, "stats_reported_at_ms": 400000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.1, "pressure": 0, "outcome_available": true, "observed_hit": false} + ] + }, + { + "id": "pressure-preserve-2", + "kind": "prefix", + "at_ms": 500000, + "tenant": "tenant-a", + "model": "model-a", + "hash_scheme": "vllm", + "prefix_hash": "cHJlc3N1cmUtcHJlc2VydmUtMg==", + "token_count": 512, + "replicas": [ + {"id": "a-local", "prefix_reported_at_ms": 500000, "stats_reported_at_ms": 500000, "reported_prefix": true, "matched_tokens": 512, "hit_rate": 0.75, "pressure": 0.7, "outcome_available": true, "observed_hit": true}, + {"id": "b-cool", "prefix_reported_at_ms": 500000, "stats_reported_at_ms": 500000, "reported_prefix": true, "matched_tokens": 256, "hit_rate": 0.3, "pressure": 0, "outcome_available": true, "observed_hit": false}, + {"id": "z-decoy", "prefix_reported_at_ms": 500000, "stats_reported_at_ms": 500000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.1, "pressure": 0, "outcome_available": true, "observed_hit": false} + ] + }, + { + "id": "pressure-preserve-3", + "kind": "prefix", + "at_ms": 600000, + "tenant": "tenant-b", + "model": "model-b", + "hash_scheme": "vllm", + "prefix_hash": "cHJlc3N1cmUtcHJlc2VydmUtMw==", + "token_count": 512, + "replicas": [ + {"id": "a-local", "prefix_reported_at_ms": 600000, "stats_reported_at_ms": 600000, "reported_prefix": true, "matched_tokens": 512, "hit_rate": 0.7, "pressure": 0.7, "outcome_available": true, "observed_hit": true}, + {"id": "b-cool", "prefix_reported_at_ms": 600000, "stats_reported_at_ms": 600000, "reported_prefix": true, "matched_tokens": 256, "hit_rate": 0.35, "pressure": 0, "outcome_available": true, "observed_hit": false}, + {"id": "z-decoy", "prefix_reported_at_ms": 600000, "stats_reported_at_ms": 600000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.1, "pressure": 0, "outcome_available": true, "observed_hit": false} + ] + }, + { + "id": "slo-promote-120-1", + "kind": "prefix", + "at_ms": 700000, + "tenant": "tenant-a", + "model": "model-a", + "hash_scheme": "vllm", + "prefix_hash": "c2xvLXByb21vdGUtMTIwLTE=", + "token_count": 512, + "ttft_budget_ms": 120, + "replicas": [ + {"id": "a-old", "prefix_reported_at_ms": 160000, "stats_reported_at_ms": 160000, "reported_prefix": true, "matched_tokens": 512, "hit_rate": 0.5, "pressure": 0, "outcome_available": true, "observed_hit": false}, + {"id": "b-fresh", "prefix_reported_at_ms": 700000, "stats_reported_at_ms": 700000, "reported_prefix": true, "matched_tokens": 320, "hit_rate": 0.5, "pressure": 0, "outcome_available": true, "observed_hit": true}, + {"id": "z-decoy", "prefix_reported_at_ms": 700000, "stats_reported_at_ms": 700000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.1, "pressure": 0, "outcome_available": true, "observed_hit": false} + ] + }, + { + "id": "slo-promote-120-2", + "kind": "prefix", + "at_ms": 800000, + "tenant": "tenant-b", + "model": "model-b", + "hash_scheme": "vllm", + "prefix_hash": "c2xvLXByb21vdGUtMTIwLTI=", + "token_count": 512, + "ttft_budget_ms": 120, + "replicas": [ + {"id": "a-old", "prefix_reported_at_ms": 260000, "stats_reported_at_ms": 260000, "reported_prefix": true, "matched_tokens": 512, "hit_rate": 0.5, "pressure": 0, "outcome_available": true, "observed_hit": false}, + {"id": "b-fresh", "prefix_reported_at_ms": 800000, "stats_reported_at_ms": 800000, "reported_prefix": true, "matched_tokens": 320, "hit_rate": 0.5, "pressure": 0, "outcome_available": true, "observed_hit": true}, + {"id": "z-decoy", "prefix_reported_at_ms": 800000, "stats_reported_at_ms": 800000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.1, "pressure": 0, "outcome_available": true, "observed_hit": false} + ] + }, + { + "id": "slo-promote-180-1", + "kind": "prefix", + "at_ms": 900000, + "tenant": "tenant-a", + "model": "model-a", + "hash_scheme": "vllm", + "prefix_hash": "c2xvLXByb21vdGUtMTgwLTE=", + "token_count": 512, + "ttft_budget_ms": 180, + "replicas": [ + {"id": "a-old", "prefix_reported_at_ms": 360000, "stats_reported_at_ms": 360000, "reported_prefix": true, "matched_tokens": 512, "hit_rate": 0.5, "pressure": 0, "outcome_available": true, "observed_hit": false}, + {"id": "b-fresh", "prefix_reported_at_ms": 900000, "stats_reported_at_ms": 900000, "reported_prefix": true, "matched_tokens": 320, "hit_rate": 0.5, "pressure": 0, "outcome_available": true, "observed_hit": true}, + {"id": "z-decoy", "prefix_reported_at_ms": 900000, "stats_reported_at_ms": 900000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.1, "pressure": 0, "outcome_available": true, "observed_hit": false} + ] + }, + { + "id": "slo-promote-180-2", + "kind": "prefix", + "at_ms": 1000000, + "tenant": "tenant-b", + "model": "model-b", + "hash_scheme": "vllm", + "prefix_hash": "c2xvLXByb21vdGUtMTgwLTI=", + "token_count": 512, + "ttft_budget_ms": 180, + "replicas": [ + {"id": "a-old", "prefix_reported_at_ms": 460000, "stats_reported_at_ms": 460000, "reported_prefix": true, "matched_tokens": 512, "hit_rate": 0.5, "pressure": 0, "outcome_available": true, "observed_hit": false}, + {"id": "b-fresh", "prefix_reported_at_ms": 1000000, "stats_reported_at_ms": 1000000, "reported_prefix": true, "matched_tokens": 320, "hit_rate": 0.5, "pressure": 0, "outcome_available": true, "observed_hit": true}, + {"id": "z-decoy", "prefix_reported_at_ms": 1000000, "stats_reported_at_ms": 1000000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.1, "pressure": 0, "outcome_available": true, "observed_hit": false} + ] + }, + { + "id": "slo-preserve-120-1", + "kind": "prefix", + "at_ms": 1100000, + "tenant": "tenant-a", + "model": "model-a", + "hash_scheme": "vllm", + "prefix_hash": "c2xvLXByZXNlcnZlLTEyMC0x", + "token_count": 550, + "ttft_budget_ms": 120, + "replicas": [ + {"id": "a-deep", "prefix_reported_at_ms": 560000, "stats_reported_at_ms": 560000, "reported_prefix": true, "matched_tokens": 550, "hit_rate": 0.6, "pressure": 0, "outcome_available": true, "observed_hit": true}, + {"id": "b-fresh", "prefix_reported_at_ms": 1100000, "stats_reported_at_ms": 1100000, "reported_prefix": true, "matched_tokens": 320, "hit_rate": 0.5, "pressure": 0, "outcome_available": true, "observed_hit": false}, + {"id": "z-decoy", "prefix_reported_at_ms": 1100000, "stats_reported_at_ms": 1100000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.1, "pressure": 0, "outcome_available": true, "observed_hit": false} + ] + }, + { + "id": "slo-preserve-120-2", + "kind": "prefix", + "at_ms": 1200000, + "tenant": "tenant-b", + "model": "model-b", + "hash_scheme": "vllm", + "prefix_hash": "c2xvLXByZXNlcnZlLTEyMC0y", + "token_count": 550, + "ttft_budget_ms": 120, + "replicas": [ + {"id": "a-deep", "prefix_reported_at_ms": 660000, "stats_reported_at_ms": 660000, "reported_prefix": true, "matched_tokens": 550, "hit_rate": 0.6, "pressure": 0, "outcome_available": true, "observed_hit": true}, + {"id": "b-fresh", "prefix_reported_at_ms": 1200000, "stats_reported_at_ms": 1200000, "reported_prefix": true, "matched_tokens": 320, "hit_rate": 0.5, "pressure": 0, "outcome_available": true, "observed_hit": false}, + {"id": "z-decoy", "prefix_reported_at_ms": 1200000, "stats_reported_at_ms": 1200000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.1, "pressure": 0, "outcome_available": true, "observed_hit": false} + ] + }, + { + "id": "slo-loose-220-1", + "kind": "prefix", + "at_ms": 1300000, + "tenant": "tenant-a", + "model": "model-a", + "hash_scheme": "vllm", + "prefix_hash": "c2xvLWxvb3NlLTIyMC0x", + "token_count": 512, + "ttft_budget_ms": 220, + "replicas": [ + {"id": "a-deep", "prefix_reported_at_ms": 760000, "stats_reported_at_ms": 760000, "reported_prefix": true, "matched_tokens": 512, "hit_rate": 0.6, "pressure": 0, "outcome_available": true, "observed_hit": true}, + {"id": "b-fresh", "prefix_reported_at_ms": 1300000, "stats_reported_at_ms": 1300000, "reported_prefix": true, "matched_tokens": 320, "hit_rate": 0.5, "pressure": 0, "outcome_available": true, "observed_hit": false}, + {"id": "z-decoy", "prefix_reported_at_ms": 1300000, "stats_reported_at_ms": 1300000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.1, "pressure": 0, "outcome_available": true, "observed_hit": false} + ] + }, + { + "id": "slo-loose-220-2", + "kind": "prefix", + "at_ms": 1400000, + "tenant": "tenant-b", + "model": "model-b", + "hash_scheme": "vllm", + "prefix_hash": "c2xvLWxvb3NlLTIyMC0y", + "token_count": 512, + "ttft_budget_ms": 220, + "replicas": [ + {"id": "a-deep", "prefix_reported_at_ms": 860000, "stats_reported_at_ms": 860000, "reported_prefix": true, "matched_tokens": 512, "hit_rate": 0.6, "pressure": 0, "outcome_available": true, "observed_hit": true}, + {"id": "b-fresh", "prefix_reported_at_ms": 1400000, "stats_reported_at_ms": 1400000, "reported_prefix": true, "matched_tokens": 320, "hit_rate": 0.5, "pressure": 0, "outcome_available": true, "observed_hit": false}, + {"id": "z-decoy", "prefix_reported_at_ms": 1400000, "stats_reported_at_ms": 1400000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.1, "pressure": 0, "outcome_available": true, "observed_hit": false} + ] + }, + { + "id": "tenant-rate-1", + "kind": "tenant_hot", + "at_ms": 1500000, + "tenant": "tenant-a", + "model": "model-a", + "hash_scheme": "vllm", + "prefix_hash": "dGVuYW50LXJhdGUtMQ==", + "token_count": 64, + "replicas": [ + {"id": "a-noisy", "prefix_reported_at_ms": 1500000, "stats_reported_at_ms": 1500000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.19, "pressure": 0, "outcome_available": true, "observed_hit": false}, + {"id": "z-warm", "prefix_reported_at_ms": 1470000, "stats_reported_at_ms": 1470000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.25, "pressure": 0, "outcome_available": true, "observed_hit": true} + ] + }, + { + "id": "tenant-rate-2", + "kind": "tenant_hot", + "at_ms": 1600000, + "tenant": "tenant-a", + "model": "model-a", + "hash_scheme": "vllm", + "prefix_hash": "dGVuYW50LXJhdGUtMg==", + "token_count": 64, + "replicas": [ + {"id": "a-noisy", "prefix_reported_at_ms": 1600000, "stats_reported_at_ms": 1600000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.19, "pressure": 0, "outcome_available": true, "observed_hit": false}, + {"id": "z-warm", "prefix_reported_at_ms": 1570000, "stats_reported_at_ms": 1570000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.25, "pressure": 0, "outcome_available": true, "observed_hit": true} + ] + }, + { + "id": "tenant-rate-3", + "kind": "tenant_hot", + "at_ms": 1700000, + "tenant": "tenant-b", + "model": "model-b", + "hash_scheme": "vllm", + "prefix_hash": "dGVuYW50LXJhdGUtMw==", + "token_count": 64, + "replicas": [ + {"id": "a-noisy", "prefix_reported_at_ms": 1700000, "stats_reported_at_ms": 1700000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.19, "pressure": 0, "outcome_available": true, "observed_hit": false}, + {"id": "z-warm", "prefix_reported_at_ms": 1670000, "stats_reported_at_ms": 1670000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.25, "pressure": 0, "outcome_available": true, "observed_hit": true} + ] + }, + { + "id": "tenant-age-preserve-1", + "kind": "tenant_hot", + "at_ms": 1800000, + "tenant": "tenant-a", + "model": "model-a", + "hash_scheme": "vllm", + "prefix_hash": "dGVuYW50LWFnZS1wcmVzZXJ2ZS0x", + "token_count": 64, + "replicas": [ + {"id": "a-moderate", "prefix_reported_at_ms": 1725000, "stats_reported_at_ms": 1725000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.3, "pressure": 0, "outcome_available": true, "observed_hit": true} + ] + }, + { + "id": "tenant-age-preserve-2", + "kind": "tenant_hot", + "at_ms": 1900000, + "tenant": "tenant-b", + "model": "model-b", + "hash_scheme": "vllm", + "prefix_hash": "dGVuYW50LWFnZS1wcmVzZXJ2ZS0y", + "token_count": 64, + "replicas": [ + {"id": "a-moderate", "prefix_reported_at_ms": 1825000, "stats_reported_at_ms": 1825000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.3, "pressure": 0, "outcome_available": true, "observed_hit": true} + ] + }, + { + "id": "tenant-age-expire-1", + "kind": "tenant_hot", + "at_ms": 2000000, + "tenant": "tenant-a", + "model": "model-a", + "hash_scheme": "vllm", + "prefix_hash": "dGVuYW50LWFnZS1leHBpcmUtMQ==", + "token_count": 64, + "replicas": [ + {"id": "a-stale", "prefix_reported_at_ms": 1850000, "stats_reported_at_ms": 1850000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.8, "pressure": 0, "outcome_available": true, "observed_hit": false}, + {"id": "z-recent", "prefix_reported_at_ms": 1970000, "stats_reported_at_ms": 1970000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.3, "pressure": 0, "outcome_available": true, "observed_hit": true} + ] + }, + { + "id": "tenant-age-expire-2", + "kind": "tenant_hot", + "at_ms": 2100000, + "tenant": "tenant-a", + "model": "model-a", + "hash_scheme": "vllm", + "prefix_hash": "dGVuYW50LWFnZS1leHBpcmUtMg==", + "token_count": 64, + "replicas": [ + {"id": "a-stale", "prefix_reported_at_ms": 1950000, "stats_reported_at_ms": 1950000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.8, "pressure": 0, "outcome_available": true, "observed_hit": false}, + {"id": "z-recent", "prefix_reported_at_ms": 2070000, "stats_reported_at_ms": 2070000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.3, "pressure": 0, "outcome_available": true, "observed_hit": true} + ] + }, + { + "id": "tenant-age-expire-3", + "kind": "tenant_hot", + "at_ms": 2200000, + "tenant": "tenant-b", + "model": "model-b", + "hash_scheme": "vllm", + "prefix_hash": "dGVuYW50LWFnZS1leHBpcmUtMw==", + "token_count": 64, + "replicas": [ + {"id": "a-stale", "prefix_reported_at_ms": 2050000, "stats_reported_at_ms": 2050000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.8, "pressure": 0, "outcome_available": true, "observed_hit": false}, + {"id": "z-recent", "prefix_reported_at_ms": 2170000, "stats_reported_at_ms": 2170000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.3, "pressure": 0, "outcome_available": true, "observed_hit": true} + ] + } + ] +} diff --git a/internal/index/index.go b/internal/index/index.go index bafaa7dc..e6c46888 100644 --- a/internal/index/index.go +++ b/internal/index/index.go @@ -264,8 +264,17 @@ func WithReservedTenants(tenants ...string) Option { } } -// withClock overrides the time source (tests only). -func withClock(now func() time.Time) Option { return func(i *Index) { i.now = now } } +// WithClock overrides the time source for deterministic replay and tests. +func WithClock(now func() time.Time) Option { + return func(i *Index) { + if now != nil { + i.now = now + } + } +} + +// withClock keeps the package-local test helper concise. +func withClock(now func() time.Time) Option { return WithClock(now) } // New builds an index with the given options. func New(opts ...Option) *Index { diff --git a/internal/index/ingest_test.go b/internal/index/ingest_test.go index be69a6cf..508d6129 100644 --- a/internal/index/ingest_test.go +++ b/internal/index/ingest_test.go @@ -9,6 +9,15 @@ import ( "time" ) +func TestWithClockIgnoresNil(t *testing.T) { + idx := New(WithClock(nil)) + if idx.now == nil { + t.Fatal("WithClock(nil) cleared the default clock") + } + idx.Ingest(Update{ReplicaID: "r", Model: "m", Tenant: "t", HashScheme: "vllm", + Prefixes: []PrefixRef{{PrefixHash: hash("p"), TokenCount: 1}}}) +} + func TestIngestAndLookupRanksByTokensAndFreshness(t *testing.T) { clk := &fakeClock{t: time.Unix(1_000_000, 0)} idx := New(withClock(clk.now), WithTTL(time.Hour)) diff --git a/internal/index/ranking_test.go b/internal/index/ranking_test.go index 24ffb998..e4d7c6e5 100644 --- a/internal/index/ranking_test.go +++ b/internal/index/ranking_test.go @@ -9,6 +9,20 @@ import ( "time" ) +func TestDefaultRankerConfigMatchesStableProductionTuple(t *testing.T) { + got := DefaultRankerConfig() + want := RankerConfig{ + PressureWeight: 1, + SLOTightTTFTMs: 200, + SLOTightBias: 1, + TenantHotMinHitRate: 0.1, + TenantHotMaxAge: 5 * time.Minute, + } + if got != want { + t.Fatalf("DefaultRankerConfig() = %+v, want stable production tuple %+v", got, want) + } +} + // TestLookupPressureAndSLOFactorsCollapseToUnityWhenSignalsAbsent locks in the // contract that the pressure and SLO score factors collapse to 1 when (a) no // replica stats are reported (pressure=0) and (b) the request carries no SLO @@ -257,7 +271,9 @@ func TestWorstTierPrefersLeastLocal(t *testing.T) { // have a chain hit would outrank a fresher idle peer the chain-aware // formula was supposed to demote. func TestChainLookupSharesPressureAndSLOFactorsWithExact(t *testing.T) { - idx := New(WithTTL(time.Hour)) + cfg := DefaultRankerConfig() + cfg.PressureWeight = 1 + idx := New(WithTTL(time.Hour), WithRanker(cfg)) hashes, counts := chain("b1", "b2", "b3") idx.Ingest(Update{ReplicaID: "big-but-hot", Model: "m", Tenant: "t", HashScheme: "vllm",