From 3d4ec5160ac02e118a7a0be0d74d8e434de81643 Mon Sep 17 00:00:00 2001 From: Weiwei Zheng Date: Wed, 19 Aug 2026 12:13:06 -0700 Subject: [PATCH 1/8] Calibrate ranker defaults with replay harness Signed-off-by: Weiwei Zheng --- .github/workflows/ci.yml | 3 + Makefile | 13 +- docs/design/grpc-contract.md | 2 +- docs/design/lookuproute-ranking.md | 72 ++- docs/reference/reason-codes.md | 8 +- hack/ranker-calibration/main.go | 62 +++ internal/index/calibration/README.md | 46 ++ internal/index/calibration/calibration.go | 464 ++++++++++++++++++ .../index/calibration/calibration_test.go | 141 ++++++ .../testdata/c1_synthetic_result.json | 326 ++++++++++++ .../testdata/c1_synthetic_trace.json | 347 +++++++++++++ internal/index/ranking.go | 10 +- internal/index/ranking_test.go | 18 +- .../content/en/docs/reference/reason-codes.md | 6 +- 14 files changed, 1483 insertions(+), 35 deletions(-) create mode 100644 hack/ranker-calibration/main.go create mode 100644 internal/index/calibration/README.md create mode 100644 internal/index/calibration/calibration.go create mode 100644 internal/index/calibration/calibration_test.go create mode 100644 internal/index/calibration/testdata/c1_synthetic_result.json create mode 100644 internal/index/calibration/testdata/c1_synthetic_trace.json diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8474da85..050b0499 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -181,6 +181,9 @@ jobs: python3 -m pip install --quiet -r pkg/fingerprint/testdata/requirements.txt make verify-golden-vectors + - name: Verify ranker calibration result + run: make verify-ranker-calibration + race-test: name: Race Tests runs-on: ubuntu-latest diff --git a/Makefile b/Makefile index b0a548e8..97b4ba73 100644 --- a/Makefile +++ b/Makefile @@ -358,6 +358,17 @@ verify-golden-vectors: ## Verify pkg/fingerprint/testdata/golden_vectors.json ma "(pip install -r pkg/fingerprint/testdata/requirements.txt to enable)"; \ fi +RANKER_CALIBRATION_TRACE ?= internal/index/calibration/testdata/c1_synthetic_trace.json +RANKER_CALIBRATION_RESULT ?= internal/index/calibration/testdata/c1_synthetic_result.json + +.PHONY: ranker-calibration +ranker-calibration: ## Replay the checked-in ranker trace and regenerate per-knob calibration curves. + $(GO_CMD) run ./hack/ranker-calibration -trace $(RANKER_CALIBRATION_TRACE) -out $(RANKER_CALIBRATION_RESULT) + +.PHONY: verify-ranker-calibration +verify-ranker-calibration: ## Verify the checked-in ranker calibration output matches its trace and sweep. + $(GO_CMD) run ./hack/ranker-calibration -trace $(RANKER_CALIBRATION_TRACE) -out $(RANKER_CALIBRATION_RESULT) -check + .PHONY: image-build image-build: controller-image server-image subscriber-image ## Build controller, server, and kvevent-subscriber images. @@ -619,7 +630,7 @@ verify-prometheus: promtool kustomize ## Lint + unit-test the Prometheus alertin @echo "✓ Prometheus rules valid" .PHONY: ci -ci: verify-naming verify-no-internal-refs verify-dco test-dco reuse-lint verify-syft-pin verify-minimal-base test-minimal-images fmt-check vet ci-lint python-lint verify-prometheus verify-golden-vectors test-docs-sync test-race build ## Local CI gate (naming + internal-refs + DCO/REUSE compliance + Syft/minimal-image policy + Go/Python lint + Prometheus rules + golden vectors + docs-sync tests + race tests + build). Run by the pre-push hook. +ci: verify-naming verify-no-internal-refs verify-dco test-dco reuse-lint verify-syft-pin verify-minimal-base test-minimal-images fmt-check vet ci-lint python-lint verify-prometheus verify-golden-vectors verify-ranker-calibration test-docs-sync test-race build ## Local CI gate (naming + internal-refs + DCO/REUSE compliance + Syft/minimal-image policy + Go/Python lint + Prometheus rules + calibration/golden fixtures + docs-sync tests + race tests + build). Run by the pre-push hook. .PHONY: pre-pr pre-pr: ci ## Pre-PR gate: CI gate + generated-code drift check + sample admission check + review checklist. diff --git a/docs/design/grpc-contract.md b/docs/design/grpc-contract.md index 2cbc5685..1d374fe3 100644 --- a/docs/design/grpc-contract.md +++ b/docs/design/grpc-contract.md @@ -116,7 +116,7 @@ unchanged and old clients continue to fail open on a downgrade. **Update — B6 (CacheIndex status surface):** the cluster-wide aggregate is now exposed two ways: an internal HTTP `/snapshot` endpoint on the server (JSON; metadata only — replica/tenant stats + prefix counts, never KV/prompt data), and a cluster-scoped, status-only `CacheIndex` CRD (`kubectl get cacheindex`) that the controller maintains by scraping `/snapshot`. This is outside the gRPC contract (no proto change); see the `CacheIndex` type in `api/v1alpha1` and the `CacheIndexPoller` in `internal/controller`. -**Update — B6 follow-up (`LookupRoute` ranking v2):** the `LookupRoute` ranker layers additive strategies on top of the original `matched_tokens × freshness` baseline, **without any proto change** (all inputs were already on the contract). Today the full score is `matched_tokens × freshness × pressure_factor × slo_bias × distinguishing_power` where `pressure_factor = max(0, 1 - PressureWeight × ReplicaStats.pressure)`, `slo_bias = 1 + freshness × SLOTightBias` when `SLO.ttft_ms` is below a configurable threshold (otherwise 1), and `distinguishing_power = 1 − num_matching_replicas / total_replicas` (1.0 when `total_replicas ≤ 1`; see [`lookuproute-ranking.md` §2.7](./lookuproute-ranking.md#27-the-replica-distinguishing-power-factor)). On a prefix miss, the server falls back to **`TENANT_HOT`**: ranked replicas that are warm for the request's `(tenant, model, hash_scheme)` — i.e. the replica has at least one prefix entry in the requested engine domain AND its latest stats are recent (within a configurable window, default 5m) with `hit_rate` above a floor (default 0.1). `TENANT_HOT` responses carry `matched_tokens=0` because there is no prefix overlap; the gateway must rely on `reason_code`, not `matched_tokens`, to recognize the soft hint. The pressure/SLO factors collapse to 1 when their supporting input is absent (no stats → pressure_factor = 1; no SLO hint → slo_bias = 1; `TenantHotMaxAge=0` disables the TENANT_HOT fallback entirely); the distinguishing-power factor collapses to 1 only for single-replica deployments — multi-replica deployments always see a cardinality-adjusted score. See [`lookuproute-ranking.md`](./lookuproute-ranking.md) and [`reason-codes.md`](../reference/reason-codes.md) for the full knob table. +**Update — B6 follow-up (`LookupRoute` ranking v2):** the `LookupRoute` ranker layers additive strategies on top of the original `matched_tokens × freshness` baseline, **without any proto change** (all inputs were already on the contract). Today the full score is `matched_tokens × freshness × pressure_factor × slo_bias × distinguishing_power` where `pressure_factor = max(0, 1 - PressureWeight × ReplicaStats.pressure)`, `slo_bias = 1 + freshness × SLOTightBias` when `SLO.ttft_ms` is below a configurable threshold (otherwise 1), and `distinguishing_power = 1 − num_matching_replicas / total_replicas` (1.0 when `total_replicas ≤ 1`; see [`lookuproute-ranking.md` §2.7](./lookuproute-ranking.md#27-the-replica-distinguishing-power-factor)). On a prefix miss, the server falls back to **`TENANT_HOT`**: ranked replicas that are warm for the request's `(tenant, model, hash_scheme)` — i.e. the replica has at least one prefix entry in the requested engine domain AND its latest stats are recent (within a configurable window, default 2m) with `hit_rate` above a floor (default 0.2). `TENANT_HOT` responses carry `matched_tokens=0` because there is no prefix overlap; the gateway must rely on `reason_code`, not `matched_tokens`, to recognize the soft hint. The pressure/SLO factors collapse to 1 when their supporting input is absent (no stats → pressure_factor = 1; no SLO hint → slo_bias = 1; `TenantHotMaxAge=0` disables the TENANT_HOT fallback entirely); the distinguishing-power factor collapses to 1 only for single-replica deployments — multi-replica deployments always see a cardinality-adjusted score. See [`lookuproute-ranking.md`](./lookuproute-ranking.md) and [`reason-codes.md`](../reference/reason-codes.md) for the full knob table. **Update — `CachePolicy.spec.strategy`:** the server enforces three per-namespace strategy gates before results leave the handler. `enableChainMatching=false` strips block-hash chain inputs and forces the legacy exact `prefix_hash` path; `requireChain=true` returns empty scores with `POLICY_REQUIRES_CHAIN` before touching the index when a request has no valid wire block-hash chain; `enableTenantHot=false` downgrades a `TENANT_HOT` result to `NO_HINT`. Defaults are `true` / `false` / `true`, preserving the previous behavior. diff --git a/docs/design/lookuproute-ranking.md b/docs/design/lookuproute-ranking.md index 968e0b01..9b820329 100644 --- a/docs/design/lookuproute-ranking.md +++ b/docs/design/lookuproute-ranking.md @@ -584,11 +584,11 @@ pressure_factor = max(0, 1 − PressureWeight × pressure) score = matched_tokens × freshness × pressure_factor ``` -Worked example with `PressureWeight = 1`: +Worked example with the default `PressureWeight = 0.5`: | Replica | tokens | freshness | pressure | baseline score | new score | |---|---|---|---|---|---| -| `big-but-hot` | 100 | 1.0 | 0.9 | 100 | **10** | +| `big-but-hot` | 80 | 1.0 | 0.9 | 80 | **44** | | `small-cool` | 50 | 1.0 | 0.0 | 50 | **50** | Under the baseline `big-but-hot` wins; with pressure folded in, the smaller @@ -623,8 +623,8 @@ When the prefix-match path returns empty, the ranker runs a second strategy: 1. Find replicas under `(tenant, model)` whose stats are recent - (`statsReported` within `TenantHotMaxAge`, default 5 min) AND whose - `hit_rate` is at least a floor (default `0.1`). These are "warm." + (`statsReported` within `TenantHotMaxAge`, default 2 min) AND whose + `hit_rate` is at least a floor (default `0.2`). These are "warm." 2. Restrict to replicas that *actually serve* the requested `hash_scheme` — i.e. they hold at least one prefix entry in the request's engine domain. Without this guard, a stats-only update with @@ -846,6 +846,38 @@ set so that: degenerates to 1.0 with one replica); for multi-replica deployments it is "pre-floor raw recall with cardinality-adjusted scores." +### Calibration provenance and replay + +The default tuple is selected by the reproducible sweep under +`internal/index/calibration`, which calls the production `LookupRoute` +implementation for every observation and searches the configured Cartesian +grid. The objective is the macro-average of prefix-hit ratio and +`TENANT_HOT`-hit ratio; ties prefer gentler score multipliers and the shorter +fallback window. + +The checked-in `c1-synthetic-mixed-routing-v1` trace contains 22 observations: +14 prefix-routing cases spanning pressure/locality tradeoffs and tight/loose +TTFT budgets, plus 8 prefix-miss cases spanning noisy hit-rate reports and +fresh/stale `TENANT_HOT` candidates. Its provenance is explicitly +`synthetic`: no production C1 request trace is currently checked into this +repository, so these values are a deterministic first calibration, not a claim +about production traffic. Replace the trace with a sanitized captured fixture +and rerun the same command when such data is available. + +```bash +make ranker-calibration +make verify-ranker-calibration +``` + +The trace and generated per-knob curves live in +`internal/index/calibration/testdata/c1_synthetic_trace.json` and +`c1_synthetic_result.json`. The sweep selects `PressureWeight = 0.5`, +`SLOTightTTFTMs = 200 ms`, `SLOTightBias = 1.0`, +`TenantHotMinHitRate = 0.2`, and `TenantHotMaxAge = 2 min`; both measured hit +ratios are 100% on this boundary-case fixture. CI regenerates the result in +check mode so trace, curves, documentation, and defaults cannot silently +diverge. + ## 7. The reason-code summary | Code | When it fires | What the gateway treats it as | @@ -870,9 +902,9 @@ carve-outs that keep them on `NO_HINT`). Six concrete scenarios that exercise the strategies in §2–5 end-to-end. Each shows the relevant index state, the request, the score computation, and the response. All examples use the default -`RankerConfig`: `PressureWeight = 1`, `SLOTightTTFTMs = 200 ms`, -`SLOTightBias = 1`, `TenantHotMaxAge = 5 min`, -`TenantHotMinHitRate = 0.1`. +`RankerConfig`: `PressureWeight = 0.5`, `SLOTightTTFTMs = 200 ms`, +`SLOTightBias = 1`, `TenantHotMaxAge = 2 min`, +`TenantHotMinHitRate = 0.2`. ### 8.1. Baseline: one replica holds the prefix @@ -884,7 +916,7 @@ Index state — tenant `team-a`, model `m`, scheme `vllm`: Request: `{tenant=team-a, model=m, hash_scheme=vllm, prefix_hash=p}`, no SLO. -Computation: `100 × 1.0 × (1 − 1 × 0.0) × 1 = 100`. +Computation: `100 × 1.0 × (1 − 0.5 × 0.0) × 1 = 100`. Response: `reason_code=PREFIX_MATCH`, scores `[{r1, score=100, matched_tokens=100}]`. @@ -897,19 +929,19 @@ Index state: | Replica | Prefix | Tokens | Freshness | Pressure | |---|---|---|---|---| -| `big-but-hot` | `p` | 100 | 1.0 | 0.9 | +| `big-but-hot` | `p` | 80 | 1.0 | 0.9 | | `small-cool` | `p` | 50 | 1.0 | 0.0 | Request: same as §8.1, no SLO. Computation: -- `big-but-hot`: `100 × 1.0 × (1 − 1 × 0.9) × 1 = 100 × 0.1 = 10` -- `small-cool`: `50 × 1.0 × (1 − 1 × 0.0) × 1 = 50` +- `big-but-hot`: `80 × 1.0 × (1 − 0.5 × 0.9) × 1 = 80 × 0.55 = 44` +- `small-cool`: `50 × 1.0 × (1 − 0.5 × 0.0) × 1 = 50` -Response: `PREFIX_MATCH`, ranked `[small-cool (50), big-but-hot (10)]`. +Response: `PREFIX_MATCH`, ranked `[small-cool (50), big-but-hot (44)]`. The pure baseline (`tokens × freshness`) would have given `big-but-hot` -a score of `100` vs `small-cool`'s `50` and routed traffic to the +a score of `80` vs `small-cool`'s `50` and routed traffic to the already-saturated replica. The pressure factor flips it: locality weighed against load. @@ -950,16 +982,16 @@ no SLO. Prefix-match path: empty (no replica holds `novel`). -Tenant-hot fallback (defaults `TenantHotMaxAge = 5 min`, -`TenantHotMinHitRate = 0.1`): -- `r-warm` reported 30 s ago (well under 5 min), `hit_rate = 0.8 ≥ 0.1`, +Tenant-hot fallback (defaults `TenantHotMaxAge = 2 min`, +`TenantHotMinHitRate = 0.2`): +- `r-warm` reported 30 s ago (well under 2 min), `hit_rate = 0.8 ≥ 0.2`, and it holds at least one prefix in `vllm` (`other`) — qualifies. -- `recency = 1 − 30 s / 5 min = 0.9` -- `pressure_factor = 1 − 1 × 0.1 = 0.9` +- `recency = 1 − 30 s / 2 min = 0.75` +- `pressure_factor = 1 − 0.5 × 0.1 = 0.95` - `slo_bias = 1` -- `score = 0.8 × 0.9 × 0.9 × 1 = 0.648` +- `score = 0.8 × 0.75 × 0.95 × 1 = 0.57` -Response: `TENANT_HOT`, scores `[{r-warm, score=0.648, matched_tokens=0}]`. +Response: `TENANT_HOT`, scores `[{r-warm, score=0.57, matched_tokens=0}]`. `matched_tokens` is `0` because there's no prefix overlap — the gateway must rely on `reason_code` (not `matched_tokens`) to tell `TENANT_HOT` diff --git a/docs/reference/reason-codes.md b/docs/reference/reason-codes.md index d06ff466..970a70dd 100644 --- a/docs/reference/reason-codes.md +++ b/docs/reference/reason-codes.md @@ -37,7 +37,7 @@ module — until then, treat `NO_HINT` as the only `LookupPDRoute` answer. | `PREFIX_MATCH` | **shipped** | LookupRoute only | The index has at least one replica holding the request's `(tenant, model, hash_scheme, adapter)` prefix — either the exact `prefix_hash` (legacy single-blob path) OR the leading run of `block_hashes[0..k]` (chain longest-prefix path; see [`../design/lookuproute-ranking.md`](../design/lookuproute-ranking.md) §2.5) — the ranker returned a non-empty set, AND at least one replica's realized `matched_tokens` cleared the per-namespace `minimumMatchedTokens` floor (default 64) AND the top surviving replica's score cleared the per-namespace `routingFloorScore` floor (default `0.1`). The score includes the distinguishing-power factor `1 − num_matching_replicas / total_replicas` so an overlap held by every replica (chat-template framing, RAG corpus headers, custom system prompts) collapses to score 0 and is filtered by the score floor. Both floors can downgrade independently. | `replica_scores` non-empty, ranked best-first by `matched_tokens × freshness × pressure_factor × slo_bias × distinguishing_power`. The pressure / SLO / distinguishing-power factors collapse appropriately when their inputs are absent (no stats → pressure_factor = 1; no SLO hint → slo_bias = 1; single-replica deployment → distinguishing_power = 1). Every qualifying replica is returned today (no top-K limit); the gateway typically uses the top entry. | Route to the top-ranked replica → prefix-cache hit; lower TTFT. | | `NO_HINT` | **shipped** | both | The fail-open default. **`LookupPDRoute`**: every call (the handler is a stub). **`LookupRoute`**: the prefix is novel under matching contract keys AND no usable affinity fallback fired (`affinityRouting: Disabled` on the per-namespace policy, OR no replica known to serve the `(tenant, model, hash_scheme)` engine domain, OR no usable seed — empty `block_hashes` and empty `prefix_hash`, OR structurally malformed input — empty `hash_scheme` or chain arrays of mismatched length); the ranker found nothing AND the same affinity-disabled / no-replica / no-seed / malformed clause holds; any of `tenant_id`, `model_id`, or `hash_scheme` was unspecified (a contract violation — set-but-wrong values surface as the matching `UNKNOWN_*` code instead); the index is globally empty (cold-start carve-out); the request was policy-gated below `minimumPrefixTokens` AND `affinityRouting: Disabled` (with `affinityRouting: Enabled` — the default — the same gate surfaces as `AFFINITY_HINT`); **every replica that held the prefix matched fewer tokens than `minimumMatchedTokens` (result-side per-replica floor, default 64) AND `affinityRouting: Disabled`** (with affinity Enabled the downgrade surfaces as `AFFINITY_HINT`); **the top per-replica score from the distinguishing-power-aware ranker fell below `routingFloorScore` (result-side score floor, default `0.1`) AND `affinityRouting: Disabled`** (with affinity Enabled the downgrade surfaces as `AFFINITY_HINT`); or an index-disabled state. | `replica_scores` **empty**. Not an error. | Route per the gateway's default policy (round-robin, least-loaded, …). The cache plane is invisible to this request. | | `POLICY_REQUIRES_CHAIN` | **shipped** | LookupRoute only | The tenant's `CachePolicy.spec.strategy.requireChain` is `true`, but the request did not carry a valid `block_hashes` + `block_token_counts` chain. The server returns before touching the index. This is a policy-gated empty result, separated from `NO_HINT` so operators can see legacy/exact callers hitting a chain-only namespace in `inferencecache_lookup_route_calls_total{reason_code="POLICY_REQUIRES_CHAIN"}`. | Empty `replica_scores`. | Treat as `NO_HINT`; update the caller to send the chain if it should benefit from the cache plane in this namespace. | -| `TENANT_HOT` | **shipped** | LookupRoute only | No exact prefix match for `(tenant, model, hash_scheme, adapter, prefix_hash)`, `CachePolicy.spec.strategy.enableTenantHot` is not `false`, and the tenant has at least one replica that (a) has reported stats recently (within ~5 minutes by default), (b) has a `hit_rate` above a small floor (default 0.1), AND (c) currently has **at least one prefix entry in the requested `(tenant, model, hash_scheme)` in the index** — proving the replica serves the requested engine domain. The "in the index" check is sweep-driven (an entry past TTL stays counted until the next sweep removes it), so for at most one sweep interval a recently-stale entry can briefly still satisfy the check; per soft-state semantics that yields at worst a soft hint that turns into a cache miss, never a wrong answer. A coarser locality signal than `PREFIX_MATCH` — useful when the prefix is novel but the tenant already has servers warm in the cache rotation. When the same fallback is found but the policy disables tenant-hot, the handler downgrades it to `NO_HINT`. | `replica_scores` non-empty (tenant-hot ranked); `matched_tokens` is **0** because there is no prefix overlap (the gateway must rely on `reason_code`, not `matched_tokens`, to recognize this branch). Shape otherwise unchanged. | Treat as a softer hint than `PREFIX_MATCH`; gateway free to use or ignore. | +| `TENANT_HOT` | **shipped** | LookupRoute only | No exact prefix match for `(tenant, model, hash_scheme, adapter, prefix_hash)`, `CachePolicy.spec.strategy.enableTenantHot` is not `false`, and the tenant has at least one replica that (a) has reported stats recently (within ~2 minutes by default), (b) has a `hit_rate` above a small floor (default 0.2), AND (c) currently has **at least one prefix entry in the requested `(tenant, model, hash_scheme)` in the index** — proving the replica serves the requested engine domain. The "in the index" check is sweep-driven (an entry past TTL stays counted until the next sweep removes it), so for at most one sweep interval a recently-stale entry can briefly still satisfy the check; per soft-state semantics that yields at worst a soft hint that turns into a cache miss, never a wrong answer. A coarser locality signal than `PREFIX_MATCH` — useful when the prefix is novel but the tenant already has servers warm in the cache rotation. When the same fallback is found but the policy disables tenant-hot, the handler downgrades it to `NO_HINT`. | `replica_scores` non-empty (tenant-hot ranked); `matched_tokens` is **0** because there is no prefix overlap (the gateway must rely on `reason_code`, not `matched_tokens`, to recognize this branch). Shape otherwise unchanged. | Treat as a softer hint than `PREFIX_MATCH`; gateway free to use or ignore. | | `TIMEOUT` | **shipped** | LookupRoute only | The lookup deadline expired before the index could rank — either the caller's context was already past its deadline on arrival or the per-tenant `CachePolicy.spec.lookupTimeoutMs` budget elapsed during the lookup. Gateway clients also synthesize this locally when *they* cancel a slow `LookupRoute` RPC. | Server: empty `replica_scores`. Client-side synth: same. | Treat as `NO_HINT`. | | `UNKNOWN_TENANT` | **shipped** | LookupRoute only | After a prefix miss (and the `TENANT_HOT` fallback when it applies — non-chain requests; chain requests skip `TENANT_HOT` by design and classify directly), AND the index is **not globally empty**: the request supplied a non-empty `tenant_id` and the index has **zero prefix entries for that tenant** across every model and hash scheme. **Cold-start carve-out:** a globally empty index (server just started, no `ReportCacheState` yet) stays on `NO_HINT` so a fresh deployment does not flood gateways with `UNKNOWN_TENANT`; the diagnostic resumes the moment any replica has reported state. Canonical asymmetric shape: a gateway-SDK querying with `tenant_id="default"` while the producer (kvevent-subscriber sidecar) is publishing under `tenant_id=$(POD_NAMESPACE)`. | Empty `replica_scores`. | Treat as `NO_HINT` for routing (still fail-open — the cache plane is hint-only); surface as a configuration error (log line / metric / SDK warning). **Do not retry under a different key** — the cache plane will not change between calls. | | `UNKNOWN_MODEL` | **shipped** | LookupRoute only | Same precondition as `UNKNOWN_TENANT` above. The tenant is known but the `(tenant_id, model_id)` pair has **zero entries**. The model has never served traffic in this tenant, or the model identifier disagrees between producer and consumer. | Empty `replica_scores`. | Same as `UNKNOWN_TENANT`: fail-open, surface as configuration error. | @@ -65,11 +65,11 @@ the cardinality factor and both floors still run. | Knob | What it does | Default | Off switch | |---|---|---|---| -| `PressureWeight` | Penalty applied to a replica's score from `ReplicaStats.pressure`: `pressure_factor = max(0, 1 - PressureWeight × pressure)`. Avoids blindly preferring a saturated cache holder over a fresher, lower-pressure peer. | `1.0` | `0` → no penalty | +| `PressureWeight` | Penalty applied to a replica's score from `ReplicaStats.pressure`: `pressure_factor = max(0, 1 - PressureWeight × pressure)`. Avoids blindly preferring a saturated cache holder over a fresher, lower-pressure peer. | `0.5` | `0` → no penalty | | `SLOTightTTFTMs` | TTFT budget (ms) below which the request is "tight" and the SLO bias kicks in. Uses `LookupRouteRequest.slo.ttft_ms`. | `200` | `0` → bias never fires | | `SLOTightBias` | Coefficient in the freshness boost: `slo_bias = 1 + freshness × SLOTightBias` when the request is tight. Higher → fresher candidates are favored more aggressively. | `1.0` | `0` → no boost | -| `TenantHotMinHitRate` | Minimum `hit_rate` for a replica to count as "warm" for the `TENANT_HOT` fallback. | `0.1` | n/a (use `TenantHotMaxAge = 0` to disable the fallback) | -| `TenantHotMaxAge` | Maximum stats age for a replica to count as "warm". | `5m` | `0` → fallback disabled (a prefix miss whose contract keys all populate the index lands at `NO_HINT`; mismatched-key misses still diagnose as `UNKNOWN_*` via the miss-classifier) | +| `TenantHotMinHitRate` | Minimum `hit_rate` for a replica to count as "warm" for the `TENANT_HOT` fallback. | `0.2` | n/a (use `TenantHotMaxAge = 0` to disable the fallback) | +| `TenantHotMaxAge` | Maximum stats age for a replica to count as "warm". | `2m` | `0` → fallback disabled (a prefix miss whose contract keys all populate the index lands at `NO_HINT`; mismatched-key misses still diagnose as `UNKNOWN_*` via the miss-classifier) | | `distinguishing_power` factor | Cardinality-aware multiplier: `1 − num_matching_replicas / total_replicas`, per-replica depth-aware for chain matches. Discounts overlaps every replica holds (chat-template framing, RAG corpus headers, custom system prompts). Always on for multi-replica deployments; degrades to `1.0` for single-replica deployments. See [`../design/lookuproute-ranking.md` §2.7](../design/lookuproute-ranking.md#27-the-replica-distinguishing-power-factor). | always on for multi-replica; `1.0` for single-replica | none (operators disable the *floor* it feeds via `CachePolicy.spec.routingFloorScore: "0"`, not the factor itself) | | `CachePolicy.spec.minimumMatchedTokens` | Per-replica matched-tokens floor: filters replicas whose realized `matched_tokens` falls below the threshold. If no replica survives, the response downgrades to `StrategyNone`, which surfaces as `AFFINITY_HINT` under `affinityRouting: Enabled` (the default) with a usable seed + serving replica or as `NO_HINT` under `affinityRouting: Disabled`. | `64` (4 KV blocks) | `0` on the CR → opt-out for that namespace | | `CachePolicy.spec.routingFloorScore` | Per-response score floor on the top surviving replica's score (after the distinguishing-power factor multiplies in). Below the floor → response downgrades to `StrategyNone`, with the same `AFFINITY_HINT` vs `NO_HINT` split as the matched-tokens row above. | `"0.1"` | `"0"` on the CR → opt-out for that namespace | diff --git a/hack/ranker-calibration/main.go b/hack/ranker-calibration/main.go new file mode 100644 index 00000000..4c43a5eb --- /dev/null +++ b/hack/ranker-calibration/main.go @@ -0,0 +1,62 @@ +// SPDX-FileCopyrightText: 2026 The inference-cache Authors +// +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "bytes" + "flag" + "fmt" + "os" + + "github.com/cachebox-project/inference-cache/internal/index/calibration" +) + +func main() { + tracePath := flag.String("trace", "", "path to a ranker calibration trace") + outPath := flag.String("out", "", "path to write the calibration result") + check := flag.Bool("check", false, "verify that -out already matches the generated result") + flag.Parse() + if *tracePath == "" || *outPath == "" { + fatalf("both -trace and -out are required") + } + + traceFile, err := os.Open(*tracePath) + if err != nil { + fatalf("open trace: %v", err) + } + trace, err := calibration.Load(traceFile) + closeErr := traceFile.Close() + if err != nil { + fatalf("load trace: %v", err) + } + if closeErr != nil { + fatalf("close trace: %v", closeErr) + } + + data, err := calibration.MarshalResult(calibration.Calibrate(trace)) + if err != nil { + fatalf("render result: %v", err) + } + if *check { + current, err := os.ReadFile(*outPath) + if err != nil { + fatalf("read result for check: %v", err) + } + if !bytes.Equal(current, data) { + fatalf("%s is stale; rerun ranker calibration", *outPath) + } + fmt.Printf("ranker calibration is current: %s\n", *outPath) + return + } + if err := os.WriteFile(*outPath, data, 0o644); err != nil { + fatalf("write result: %v", err) + } + fmt.Printf("wrote ranker calibration: %s\n", *outPath) +} + +func fatalf(format string, args ...any) { + fmt.Fprintf(os.Stderr, "ranker-calibration: "+format+"\n", args...) + os.Exit(1) +} diff --git a/internal/index/calibration/README.md b/internal/index/calibration/README.md new file mode 100644 index 00000000..d3e44def --- /dev/null +++ b/internal/index/calibration/README.md @@ -0,0 +1,46 @@ +# Ranker calibration + +This package replays routing observations through the production +`internal/index.LookupRoute` implementation and searches a Cartesian grid of +the five `RankerConfig` knobs. It reports two outcome rates: + +- `prefix_hit_rate_pct`: the selected top replica produced an observed cache + hit for a `prefix` observation. +- `tenant_hot_hit_rate_pct`: the selected top replica produced an observed + cache hit after a `tenant_hot` fallback observation. + +`macro_hit_rate_pct` gives both observation classes equal weight, regardless +of how many rows each class contributes. The deterministic tie-break prefers +gentler pressure/SLO multipliers and shorter fallback windows. + +## Trace shape + +Each observation is a self-contained point-in-time view. `reported_prefix` +records whether the cache plane believed that replica held the requested +prefix; `matched_tokens`, `hit_rate`, `pressure`, and `reported_at_ms` are the +signals visible to the ranker. `observed_hit` is the later ground-truth outcome +for routing to that replica. Replicas without the requested prefix still receive +a unique serving-prefix entry during replay so `TENANT_HOT` can apply its real +engine-domain membership guard. + +Captured data should contain opaque or one-way prefix hashes only. Do not put +prompt text, token IDs, customer identifiers, or other request content in a +trace. Use stable pseudonyms for tenants, models, and replicas. Set +`provenance.kind` to `captured` only when the observations came from a real +run; generated and hand-constructed fixtures must say `synthetic`. + +The checked-in fixture is intentionally synthetic because no production C1 +trace is available in this repository. It provides deterministic boundary +coverage and proves the calibration pipeline, but it should be replaced or +supplemented with a sanitized captured trace before treating the coefficients +as a production benchmark conclusion. + +## Reproduce + +```bash +make ranker-calibration +make verify-ranker-calibration +``` + +Override `RANKER_CALIBRATION_TRACE` and `RANKER_CALIBRATION_RESULT` to replay a +different trace without changing the tool. diff --git a/internal/index/calibration/calibration.go b/internal/index/calibration/calibration.go new file mode 100644 index 00000000..b1bc60d2 --- /dev/null +++ b/internal/index/calibration/calibration.go @@ -0,0 +1,464 @@ +// SPDX-FileCopyrightText: 2026 The inference-cache Authors +// +// SPDX-License-Identifier: Apache-2.0 + +// Package calibration replays routing observations through the +// production index ranker and sweeps RankerConfig candidates. +package calibration + +import ( + "encoding/json" + "errors" + "fmt" + "io" + "math" + "sort" + "time" + + "github.com/cachebox-project/inference-cache/internal/index" +) + +const SchemaVersion = 1 + +const ( + ObservationPrefix = "prefix" + ObservationTenantHot = "tenant_hot" +) + +type Provenance struct { + Kind string `json:"kind"` + Source string `json:"source"` + Description string `json:"description"` +} + +type Sweep struct { + PressureWeights []float64 `json:"pressure_weights"` + SLOTightTTFTMillis []int32 `json:"slo_tight_ttft_ms"` + SLOTightBiases []float64 `json:"slo_tight_biases"` + TenantHotMinHitRates []float64 `json:"tenant_hot_min_hit_rates"` + TenantHotMaxAgeMillis []int64 `json:"tenant_hot_max_age_ms"` +} + +type Trace struct { + SchemaVersion int `json:"schema_version"` + Name string `json:"name"` + Provenance Provenance `json:"provenance"` + TTLMillis int64 `json:"ttl_ms"` + Sweep Sweep `json:"sweep"` + Observations []Observation `json:"observations"` +} + +type Observation struct { + ID string `json:"id"` + Kind string `json:"kind"` + AtMillis int64 `json:"at_ms"` + Tenant string `json:"tenant"` + Model string `json:"model"` + HashScheme string `json:"hash_scheme"` + PrefixHash string `json:"prefix_hash"` + TokenCount int32 `json:"token_count"` + TTFTBudgetMillis int32 `json:"ttft_budget_ms,omitempty"` + Replicas []ReplicaObservation `json:"replicas"` +} + +type ReplicaObservation struct { + ID string `json:"id"` + ReportedAtMillis int64 `json:"reported_at_ms"` + ReportedPrefix bool `json:"reported_prefix"` + MatchedTokens int32 `json:"matched_tokens"` + HitRate float32 `json:"hit_rate"` + Pressure float32 `json:"pressure"` + ObservedHit bool `json:"observed_hit"` +} + +type Config struct { + PressureWeight float64 `json:"pressure_weight"` + SLOTightTTFTMillis int32 `json:"slo_tight_ttft_ms"` + SLOTightBias float64 `json:"slo_tight_bias"` + TenantHotMinHitRate float64 `json:"tenant_hot_min_hit_rate"` + TenantHotMaxAgeMillis int64 `json:"tenant_hot_max_age_ms"` +} + +type Metrics struct { + PrefixRequests int `json:"prefix_requests"` + PrefixHits int `json:"prefix_hits"` + PrefixHitRatePct float64 `json:"prefix_hit_rate_pct"` + TenantHotRequests int `json:"tenant_hot_requests"` + TenantHotHits int `json:"tenant_hot_hits"` + TenantHotHitRatePct float64 `json:"tenant_hot_hit_rate_pct"` + MacroHitRatePct float64 `json:"macro_hit_rate_pct"` +} + +type CurvePoint struct { + Value float64 `json:"value"` + Metrics Metrics `json:"metrics"` +} + +type Result struct { + SchemaVersion int `json:"schema_version"` + TraceName string `json:"trace_name"` + Provenance Provenance `json:"provenance"` + Observations int `json:"observations"` + BestConfig Config `json:"best_config"` + BestMetrics Metrics `json:"best_metrics"` + Curves map[string][]CurvePoint `json:"curves"` +} + +func Load(r io.Reader) (Trace, error) { + var trace Trace + decoder := json.NewDecoder(r) + decoder.DisallowUnknownFields() + if err := decoder.Decode(&trace); err != nil { + return Trace{}, fmt.Errorf("decode calibration trace: %w", err) + } + var trailing any + if err := decoder.Decode(&trailing); !errors.Is(err, io.EOF) { + return Trace{}, errors.New("decode calibration trace: trailing JSON value") + } + if err := trace.Validate(); err != nil { + return Trace{}, err + } + return trace, nil +} + +func (t Trace) Validate() error { + if t.SchemaVersion != SchemaVersion { + return fmt.Errorf("schema_version = %d, want %d", t.SchemaVersion, SchemaVersion) + } + if t.Name == "" { + return errors.New("trace name is required") + } + if t.Provenance.Kind != "captured" && t.Provenance.Kind != "synthetic" { + return fmt.Errorf("provenance kind %q must be captured or synthetic", t.Provenance.Kind) + } + if t.Provenance.Source == "" { + return errors.New("provenance source is required") + } + if t.TTLMillis <= 0 { + return errors.New("ttl_ms must be positive") + } + if err := t.Sweep.validate(); err != nil { + return err + } + if len(t.Observations) == 0 { + return errors.New("at least one observation is required") + } + seen := make(map[string]struct{}, len(t.Observations)) + for i, observation := range t.Observations { + if err := observation.validate(); err != nil { + return fmt.Errorf("observation %d: %w", i, err) + } + if _, ok := seen[observation.ID]; ok { + return fmt.Errorf("observation %d: duplicate id %q", i, observation.ID) + } + seen[observation.ID] = struct{}{} + } + return nil +} + +func (s Sweep) validate() error { + if len(s.PressureWeights) == 0 || len(s.SLOTightTTFTMillis) == 0 || + len(s.SLOTightBiases) == 0 || len(s.TenantHotMinHitRates) == 0 || + len(s.TenantHotMaxAgeMillis) == 0 { + return errors.New("every sweep dimension must contain at least one value") + } + for _, value := range append(append([]float64{}, s.PressureWeights...), s.SLOTightBiases...) { + if !finiteNonNegative(value) { + return fmt.Errorf("sweep contains invalid non-negative value %v", value) + } + } + for _, value := range s.TenantHotMinHitRates { + if !finiteRate(value) { + return fmt.Errorf("tenant_hot_min_hit_rates contains invalid rate %v", value) + } + } + for _, value := range s.SLOTightTTFTMillis { + if value <= 0 { + return fmt.Errorf("slo_tight_ttft_ms contains non-positive value %d", value) + } + } + for _, value := range s.TenantHotMaxAgeMillis { + if value <= 0 { + return fmt.Errorf("tenant_hot_max_age_ms contains non-positive value %d", value) + } + } + return nil +} + +func (o Observation) validate() error { + if o.ID == "" || o.Tenant == "" || o.Model == "" || o.HashScheme == "" || o.PrefixHash == "" { + return errors.New("id, tenant, model, hash_scheme, and prefix_hash are required") + } + if o.Kind != ObservationPrefix && o.Kind != ObservationTenantHot { + return fmt.Errorf("kind %q must be prefix or tenant_hot", o.Kind) + } + if o.TokenCount <= 0 { + return errors.New("token_count must be positive") + } + if len(o.Replicas) == 0 { + return errors.New("at least one replica is required") + } + seen := make(map[string]struct{}, len(o.Replicas)) + for i, replica := range o.Replicas { + if replica.ID == "" { + return fmt.Errorf("replica %d: id is required", i) + } + if replica.ReportedAtMillis > o.AtMillis { + return fmt.Errorf("replica %q: reported_at_ms is after observation", replica.ID) + } + if replica.ReportedPrefix && replica.MatchedTokens <= 0 { + return fmt.Errorf("replica %q: matched_tokens must be positive when reported_prefix is true", replica.ID) + } + if !finiteRate(float64(replica.HitRate)) || !finiteRate(float64(replica.Pressure)) { + return fmt.Errorf("replica %q: hit_rate and pressure must be finite values in [0,1]", replica.ID) + } + if _, ok := seen[replica.ID]; ok { + return fmt.Errorf("duplicate replica id %q", replica.ID) + } + seen[replica.ID] = struct{}{} + } + return nil +} + +func finiteNonNegative(value float64) bool { + return !math.IsNaN(value) && !math.IsInf(value, 0) && value >= 0 +} + +func finiteRate(value float64) bool { + return finiteNonNegative(value) && value <= 1 +} + +func Calibrate(trace Trace) Result { + bestConfig, bestMetrics := bestGridPoint(trace) + return Result{ + SchemaVersion: SchemaVersion, + TraceName: trace.Name, + Provenance: trace.Provenance, + Observations: len(trace.Observations), + BestConfig: bestConfig, + BestMetrics: bestMetrics, + Curves: map[string][]CurvePoint{ + "pressure_weight": pressureCurve(trace, bestConfig), + "slo_tight_ttft_ms": sloThresholdCurve(trace, bestConfig), + "slo_tight_bias": sloBiasCurve(trace, bestConfig), + "tenant_hot_min_hit_rate": tenantHotRateCurve(trace, bestConfig), + "tenant_hot_max_age_ms": tenantHotAgeCurve(trace, bestConfig), + }, + } +} + +func bestGridPoint(trace Trace) (Config, Metrics) { + var best Config + var bestMetrics Metrics + first := true + for _, pressureWeight := range trace.Sweep.PressureWeights { + for _, ttft := range trace.Sweep.SLOTightTTFTMillis { + for _, bias := range trace.Sweep.SLOTightBiases { + for _, hitRate := range trace.Sweep.TenantHotMinHitRates { + for _, maxAge := range trace.Sweep.TenantHotMaxAgeMillis { + candidate := Config{ + PressureWeight: pressureWeight, + SLOTightTTFTMillis: ttft, + SLOTightBias: bias, + TenantHotMinHitRate: hitRate, + TenantHotMaxAgeMillis: maxAge, + } + metrics := Replay(trace, candidate) + if first || better(metrics, candidate, bestMetrics, best) { + best, bestMetrics, first = candidate, metrics, false + } + } + } + } + } + } + return best, bestMetrics +} + +func better(candidateMetrics Metrics, candidate Config, bestMetrics Metrics, best Config) bool { + if candidateMetrics.MacroHitRatePct != bestMetrics.MacroHitRatePct { + return candidateMetrics.MacroHitRatePct > bestMetrics.MacroHitRatePct + } + if candidateMetrics.PrefixHitRatePct != bestMetrics.PrefixHitRatePct { + return candidateMetrics.PrefixHitRatePct > bestMetrics.PrefixHitRatePct + } + if candidateMetrics.TenantHotHitRatePct != bestMetrics.TenantHotHitRatePct { + return candidateMetrics.TenantHotHitRatePct > bestMetrics.TenantHotHitRatePct + } + // Conservative deterministic tie-break: prefer the least invasive score + // multipliers and shortest fallback window among equally accurate points. + if candidate.PressureWeight != best.PressureWeight { + return candidate.PressureWeight < best.PressureWeight + } + if candidate.SLOTightBias != best.SLOTightBias { + return candidate.SLOTightBias < best.SLOTightBias + } + if candidate.SLOTightTTFTMillis != best.SLOTightTTFTMillis { + return candidate.SLOTightTTFTMillis < best.SLOTightTTFTMillis + } + if candidate.TenantHotMaxAgeMillis != best.TenantHotMaxAgeMillis { + return candidate.TenantHotMaxAgeMillis < best.TenantHotMaxAgeMillis + } + return candidate.TenantHotMinHitRate > best.TenantHotMinHitRate +} + +func Replay(trace Trace, config Config) Metrics { + metrics := Metrics{} + for _, observation := range trace.Observations { + hit := replayObservation(trace, observation, config) + switch observation.Kind { + case ObservationPrefix: + metrics.PrefixRequests++ + if hit { + metrics.PrefixHits++ + } + case ObservationTenantHot: + metrics.TenantHotRequests++ + if hit { + metrics.TenantHotHits++ + } + } + } + metrics.PrefixHitRatePct = percentage(metrics.PrefixHits, metrics.PrefixRequests) + metrics.TenantHotHitRatePct = percentage(metrics.TenantHotHits, metrics.TenantHotRequests) + samples := 0 + if metrics.PrefixRequests > 0 { + metrics.MacroHitRatePct += metrics.PrefixHitRatePct + samples++ + } + if metrics.TenantHotRequests > 0 { + metrics.MacroHitRatePct += metrics.TenantHotHitRatePct + samples++ + } + if samples > 0 { + metrics.MacroHitRatePct /= float64(samples) + } + return metrics +} + +func replayObservation(trace Trace, observation Observation, config Config) bool { + anchor := time.Now() + ranker := index.RankerConfig{ + PressureWeight: float32(config.PressureWeight), + SLOTightTTFTMs: config.SLOTightTTFTMillis, + SLOTightBias: float32(config.SLOTightBias), + TenantHotMinHitRate: float32(config.TenantHotMinHitRate), + TenantHotMaxAge: time.Duration(config.TenantHotMaxAgeMillis) * time.Millisecond, + } + idx := index.New( + index.WithTTL(time.Duration(trace.TTLMillis)*time.Millisecond), + index.WithRanker(ranker), + ) + observedHits := make(map[string]bool, len(observation.Replicas)) + for _, replica := range observation.Replicas { + hash := observation.PrefixHash + tokens := replica.MatchedTokens + if !replica.ReportedPrefix { + hash = "serving/" + observation.ID + "/" + replica.ID + tokens = 1 + } + age := time.Duration(observation.AtMillis-replica.ReportedAtMillis) * time.Millisecond + idx.Ingest(index.Update{ + ReplicaID: replica.ID, + Model: observation.Model, + Tenant: observation.Tenant, + HashScheme: observation.HashScheme, + Timestamp: anchor.Add(-age), + Prefixes: []index.PrefixRef{{ + PrefixHash: []byte(hash), + TokenCount: tokens, + }}, + Stats: &index.ReplicaStats{ + HitRate: replica.HitRate, + Pressure: replica.Pressure, + }, + }) + observedHits[replica.ID] = replica.ObservedHit + } + result := idx.LookupRoute(index.LookupRequest{ + Model: observation.Model, + Tenant: observation.Tenant, + HashScheme: observation.HashScheme, + PrefixHash: []byte(observation.PrefixHash), + TokenCount: observation.TokenCount, + TTFTBudgetMs: observation.TTFTBudgetMillis, + }) + wantStrategy := index.StrategyPrefixMatch + if observation.Kind == ObservationTenantHot { + wantStrategy = index.StrategyTenantHot + } + return result.Strategy == wantStrategy && len(result.Scores) > 0 && observedHits[result.Scores[0].ReplicaID] +} + +func percentage(numerator, denominator int) float64 { + if denominator == 0 { + return 0 + } + return float64(numerator) * 100 / float64(denominator) +} + +func pressureCurve(trace Trace, best Config) []CurvePoint { + return floatCurve(trace.Sweep.PressureWeights, func(value float64) Metrics { + candidate := best + candidate.PressureWeight = value + return Replay(trace, candidate) + }) +} + +func sloBiasCurve(trace Trace, best Config) []CurvePoint { + return floatCurve(trace.Sweep.SLOTightBiases, func(value float64) Metrics { + candidate := best + candidate.SLOTightBias = value + return Replay(trace, candidate) + }) +} + +func tenantHotRateCurve(trace Trace, best Config) []CurvePoint { + return floatCurve(trace.Sweep.TenantHotMinHitRates, func(value float64) Metrics { + candidate := best + candidate.TenantHotMinHitRate = value + return Replay(trace, candidate) + }) +} + +func floatCurve(values []float64, replay func(float64) Metrics) []CurvePoint { + values = append([]float64(nil), values...) + sort.Slice(values, func(i, j int) bool { return values[i] < values[j] }) + points := make([]CurvePoint, 0, len(values)) + for _, value := range values { + points = append(points, CurvePoint{Value: float64(value), Metrics: replay(value)}) + } + return points +} + +func sloThresholdCurve(trace Trace, best Config) []CurvePoint { + values := append([]int32(nil), trace.Sweep.SLOTightTTFTMillis...) + sort.Slice(values, func(i, j int) bool { return values[i] < values[j] }) + points := make([]CurvePoint, 0, len(values)) + for _, value := range values { + candidate := best + candidate.SLOTightTTFTMillis = value + points = append(points, CurvePoint{Value: float64(value), Metrics: Replay(trace, candidate)}) + } + return points +} + +func tenantHotAgeCurve(trace Trace, best Config) []CurvePoint { + values := append([]int64(nil), trace.Sweep.TenantHotMaxAgeMillis...) + sort.Slice(values, func(i, j int) bool { return values[i] < values[j] }) + points := make([]CurvePoint, 0, len(values)) + for _, value := range values { + candidate := best + candidate.TenantHotMaxAgeMillis = value + points = append(points, CurvePoint{Value: float64(value), Metrics: Replay(trace, candidate)}) + } + return points +} + +func MarshalResult(result Result) ([]byte, error) { + data, err := json.MarshalIndent(result, "", " ") + if err != nil { + return nil, fmt.Errorf("marshal calibration result: %w", err) + } + return append(data, '\n'), nil +} diff --git a/internal/index/calibration/calibration_test.go b/internal/index/calibration/calibration_test.go new file mode 100644 index 00000000..69c70784 --- /dev/null +++ b/internal/index/calibration/calibration_test.go @@ -0,0 +1,141 @@ +// SPDX-FileCopyrightText: 2026 The inference-cache Authors +// +// SPDX-License-Identifier: Apache-2.0 + +package calibration + +import ( + "bytes" + "os" + "strings" + "testing" + "time" + + "github.com/cachebox-project/inference-cache/internal/index" +) + +func TestCheckedInTraceSelectsDefaultConfigAndCurrentResult(t *testing.T) { + traceFile, err := os.Open("testdata/c1_synthetic_trace.json") + if err != nil { + t.Fatalf("open trace: %v", err) + } + trace, err := Load(traceFile) + if closeErr := traceFile.Close(); closeErr != nil { + t.Fatalf("close trace: %v", closeErr) + } + if err != nil { + t.Fatalf("Load: %v", err) + } + result := Calibrate(trace) + defaults := index.DefaultRankerConfig() + best := result.BestConfig + if float32(best.PressureWeight) != defaults.PressureWeight || + best.SLOTightTTFTMillis != defaults.SLOTightTTFTMs || + float32(best.SLOTightBias) != defaults.SLOTightBias || + float32(best.TenantHotMinHitRate) != defaults.TenantHotMinHitRate || + time.Duration(best.TenantHotMaxAgeMillis)*time.Millisecond != defaults.TenantHotMaxAge { + t.Fatalf("calibrated config = %+v, DefaultRankerConfig = %+v", best, defaults) + } + if result.BestMetrics.PrefixHitRatePct != 100 || result.BestMetrics.TenantHotHitRatePct != 100 { + t.Fatalf("best metrics = %+v, want both fixture hit rates at 100%%", result.BestMetrics) + } + got, err := MarshalResult(result) + if err != nil { + t.Fatalf("MarshalResult: %v", err) + } + committed, err := os.ReadFile("testdata/c1_synthetic_result.json") + if err != nil { + t.Fatalf("read committed result: %v", err) + } + if !bytes.Equal(got, committed) { + t.Fatal("c1_synthetic_result.json is stale; run make ranker-calibration") + } +} + +func TestCalibrateSeparatesKnobEffects(t *testing.T) { + trace := Trace{ + SchemaVersion: SchemaVersion, + Name: "unit", + Provenance: Provenance{Kind: "synthetic", Source: "unit test"}, + TTLMillis: 100_000, + Sweep: Sweep{ + PressureWeights: []float64{0, 0.5, 1}, + SLOTightTTFTMillis: []int32{100, 200}, + SLOTightBiases: []float64{0, 1}, + TenantHotMinHitRates: []float64{0.1, 0.2}, + TenantHotMaxAgeMillis: []int64{60_000, 120_000}, + }, + Observations: []Observation{ + { + ID: "pressure", Kind: ObservationPrefix, AtMillis: 100_000, + Tenant: "tenant-a", Model: "model-a", HashScheme: "vllm", + PrefixHash: "p", TokenCount: 320, + Replicas: []ReplicaObservation{ + {ID: "hot", ReportedAtMillis: 100_000, ReportedPrefix: true, MatchedTokens: 320, HitRate: 0.8, Pressure: 0.8}, + {ID: "cool", ReportedAtMillis: 100_000, ReportedPrefix: true, MatchedTokens: 256, HitRate: 0.4, Pressure: 0.1, ObservedHit: true}, + {ID: "decoy", ReportedAtMillis: 100_000, HitRate: 0.1}, + }, + }, + }, + } + if err := trace.Validate(); err != nil { + t.Fatalf("Validate: %v", err) + } + result := Calibrate(trace) + if result.BestConfig.PressureWeight != 0.5 { + t.Fatalf("PressureWeight = %v, want 0.5", result.BestConfig.PressureWeight) + } + if result.BestMetrics.PrefixHitRatePct != 100 { + t.Fatalf("PrefixHitRatePct = %v, want 100", result.BestMetrics.PrefixHitRatePct) + } + if got := len(result.Curves["pressure_weight"]); got != 3 { + t.Fatalf("pressure curve points = %d, want 3", got) + } +} + +func TestLoadRejectsUnknownAndInvalidFields(t *testing.T) { + for _, tc := range []struct { + name string + json string + want string + }{ + {"unknown", `{"schema_version":1,"unknown":true}`, "unknown field"}, + {"version", `{"schema_version":2}`, "schema_version"}, + {"trailing", `{"schema_version":1} {}`, "trailing JSON"}, + } { + t.Run(tc.name, func(t *testing.T) { + _, err := Load(strings.NewReader(tc.json)) + if err == nil || !strings.Contains(err.Error(), tc.want) { + t.Fatalf("Load error = %v, want substring %q", err, tc.want) + } + }) + } +} + +func TestReplayTenantHotMissWithoutCandidate(t *testing.T) { + trace := Trace{TTLMillis: int64(time.Minute / time.Millisecond)} + observation := Observation{ + ID: "tenant-hot", Kind: ObservationTenantHot, AtMillis: 100_000, + Tenant: "t", Model: "m", HashScheme: "vllm", PrefixHash: "novel", TokenCount: 1, + Replicas: []ReplicaObservation{{ + ID: "cold", ReportedAtMillis: 100_000, HitRate: 0.1, ObservedHit: true, + }}, + } + config := Config{TenantHotMinHitRate: 0.2, TenantHotMaxAgeMillis: 60_000} + if replayObservation(trace, observation, config) { + t.Fatal("replayObservation = hit, want miss when every tenant-hot candidate is below the floor") + } +} + +func TestObservationRejectsFutureReplicaReport(t *testing.T) { + observation := Observation{ + ID: "future", Kind: ObservationPrefix, AtMillis: 10, + Tenant: "t", Model: "m", HashScheme: "vllm", PrefixHash: "p", TokenCount: 1, + Replicas: []ReplicaObservation{{ + ID: "r", ReportedAtMillis: 11, ReportedPrefix: true, MatchedTokens: 1, + }}, + } + if err := observation.validate(); err == nil || !strings.Contains(err.Error(), "after observation") { + t.Fatalf("validate error = %v, want future-report rejection", err) + } +} diff --git a/internal/index/calibration/testdata/c1_synthetic_result.json b/internal/index/calibration/testdata/c1_synthetic_result.json new file mode 100644 index 00000000..7649e9e9 --- /dev/null +++ b/internal/index/calibration/testdata/c1_synthetic_result.json @@ -0,0 +1,326 @@ +{ + "schema_version": 1, + "trace_name": "c1-synthetic-mixed-routing-v1", + "provenance": { + "kind": "synthetic", + "source": "Deterministic boundary-case replay derived from the C1 ReplicaStats and LookupRoute contracts", + "description": "No production C1 request trace is checked into the repository. This fixture exercises pressure/locality tradeoffs, tight and loose TTFT budgets, stale soft-state observations, and TENANT_HOT rate/age gates without claiming to represent production traffic." + }, + "observations": 22, + "best_config": { + "pressure_weight": 0.5, + "slo_tight_ttft_ms": 200, + "slo_tight_bias": 1, + "tenant_hot_min_hit_rate": 0.2, + "tenant_hot_max_age_ms": 120000 + }, + "best_metrics": { + "prefix_requests": 14, + "prefix_hits": 14, + "prefix_hit_rate_pct": 100, + "tenant_hot_requests": 8, + "tenant_hot_hits": 8, + "tenant_hot_hit_rate_pct": 100, + "macro_hit_rate_pct": 100 + }, + "curves": { + "pressure_weight": [ + { + "value": 0, + "metrics": { + "prefix_requests": 14, + "prefix_hits": 11, + "prefix_hit_rate_pct": 78.57142857142857, + "tenant_hot_requests": 8, + "tenant_hot_hits": 8, + "tenant_hot_hit_rate_pct": 100, + "macro_hit_rate_pct": 89.28571428571428 + } + }, + { + "value": 0.25, + "metrics": { + "prefix_requests": 14, + "prefix_hits": 11, + "prefix_hit_rate_pct": 78.57142857142857, + "tenant_hot_requests": 8, + "tenant_hot_hits": 8, + "tenant_hot_hit_rate_pct": 100, + "macro_hit_rate_pct": 89.28571428571428 + } + }, + { + "value": 0.5, + "metrics": { + "prefix_requests": 14, + "prefix_hits": 14, + "prefix_hit_rate_pct": 100, + "tenant_hot_requests": 8, + "tenant_hot_hits": 8, + "tenant_hot_hit_rate_pct": 100, + "macro_hit_rate_pct": 100 + } + }, + { + "value": 0.75, + "metrics": { + "prefix_requests": 14, + "prefix_hits": 11, + "prefix_hit_rate_pct": 78.57142857142857, + "tenant_hot_requests": 8, + "tenant_hot_hits": 8, + "tenant_hot_hit_rate_pct": 100, + "macro_hit_rate_pct": 89.28571428571428 + } + }, + { + "value": 1, + "metrics": { + "prefix_requests": 14, + "prefix_hits": 11, + "prefix_hit_rate_pct": 78.57142857142857, + "tenant_hot_requests": 8, + "tenant_hot_hits": 8, + "tenant_hot_hit_rate_pct": 100, + "macro_hit_rate_pct": 89.28571428571428 + } + } + ], + "slo_tight_bias": [ + { + "value": 0, + "metrics": { + "prefix_requests": 14, + "prefix_hits": 10, + "prefix_hit_rate_pct": 71.42857142857143, + "tenant_hot_requests": 8, + "tenant_hot_hits": 8, + "tenant_hot_hit_rate_pct": 100, + "macro_hit_rate_pct": 85.71428571428572 + } + }, + { + "value": 0.5, + "metrics": { + "prefix_requests": 14, + "prefix_hits": 10, + "prefix_hit_rate_pct": 71.42857142857143, + "tenant_hot_requests": 8, + "tenant_hot_hits": 8, + "tenant_hot_hit_rate_pct": 100, + "macro_hit_rate_pct": 85.71428571428572 + } + }, + { + "value": 1, + "metrics": { + "prefix_requests": 14, + "prefix_hits": 14, + "prefix_hit_rate_pct": 100, + "tenant_hot_requests": 8, + "tenant_hot_hits": 8, + "tenant_hot_hit_rate_pct": 100, + "macro_hit_rate_pct": 100 + } + }, + { + "value": 1.5, + "metrics": { + "prefix_requests": 14, + "prefix_hits": 12, + "prefix_hit_rate_pct": 85.71428571428571, + "tenant_hot_requests": 8, + "tenant_hot_hits": 8, + "tenant_hot_hit_rate_pct": 100, + "macro_hit_rate_pct": 92.85714285714286 + } + }, + { + "value": 2, + "metrics": { + "prefix_requests": 14, + "prefix_hits": 12, + "prefix_hit_rate_pct": 85.71428571428571, + "tenant_hot_requests": 8, + "tenant_hot_hits": 8, + "tenant_hot_hit_rate_pct": 100, + "macro_hit_rate_pct": 92.85714285714286 + } + } + ], + "slo_tight_ttft_ms": [ + { + "value": 100, + "metrics": { + "prefix_requests": 14, + "prefix_hits": 10, + "prefix_hit_rate_pct": 71.42857142857143, + "tenant_hot_requests": 8, + "tenant_hot_hits": 8, + "tenant_hot_hit_rate_pct": 100, + "macro_hit_rate_pct": 85.71428571428572 + } + }, + { + "value": 150, + "metrics": { + "prefix_requests": 14, + "prefix_hits": 12, + "prefix_hit_rate_pct": 85.71428571428571, + "tenant_hot_requests": 8, + "tenant_hot_hits": 8, + "tenant_hot_hit_rate_pct": 100, + "macro_hit_rate_pct": 92.85714285714286 + } + }, + { + "value": 200, + "metrics": { + "prefix_requests": 14, + "prefix_hits": 14, + "prefix_hit_rate_pct": 100, + "tenant_hot_requests": 8, + "tenant_hot_hits": 8, + "tenant_hot_hit_rate_pct": 100, + "macro_hit_rate_pct": 100 + } + }, + { + "value": 250, + "metrics": { + "prefix_requests": 14, + "prefix_hits": 12, + "prefix_hit_rate_pct": 85.71428571428571, + "tenant_hot_requests": 8, + "tenant_hot_hits": 8, + "tenant_hot_hit_rate_pct": 100, + "macro_hit_rate_pct": 92.85714285714286 + } + }, + { + "value": 300, + "metrics": { + "prefix_requests": 14, + "prefix_hits": 12, + "prefix_hit_rate_pct": 85.71428571428571, + "tenant_hot_requests": 8, + "tenant_hot_hits": 8, + "tenant_hot_hit_rate_pct": 100, + "macro_hit_rate_pct": 92.85714285714286 + } + } + ], + "tenant_hot_max_age_ms": [ + { + "value": 60000, + "metrics": { + "prefix_requests": 14, + "prefix_hits": 14, + "prefix_hit_rate_pct": 100, + "tenant_hot_requests": 8, + "tenant_hot_hits": 6, + "tenant_hot_hit_rate_pct": 75, + "macro_hit_rate_pct": 87.5 + } + }, + { + "value": 120000, + "metrics": { + "prefix_requests": 14, + "prefix_hits": 14, + "prefix_hit_rate_pct": 100, + "tenant_hot_requests": 8, + "tenant_hot_hits": 8, + "tenant_hot_hit_rate_pct": 100, + "macro_hit_rate_pct": 100 + } + }, + { + "value": 300000, + "metrics": { + "prefix_requests": 14, + "prefix_hits": 14, + "prefix_hit_rate_pct": 100, + "tenant_hot_requests": 8, + "tenant_hot_hits": 5, + "tenant_hot_hit_rate_pct": 62.5, + "macro_hit_rate_pct": 81.25 + } + }, + { + "value": 600000, + "metrics": { + "prefix_requests": 14, + "prefix_hits": 14, + "prefix_hit_rate_pct": 100, + "tenant_hot_requests": 8, + "tenant_hot_hits": 5, + "tenant_hot_hit_rate_pct": 62.5, + "macro_hit_rate_pct": 81.25 + } + } + ], + "tenant_hot_min_hit_rate": [ + { + "value": 0.05, + "metrics": { + "prefix_requests": 14, + "prefix_hits": 14, + "prefix_hit_rate_pct": 100, + "tenant_hot_requests": 8, + "tenant_hot_hits": 5, + "tenant_hot_hit_rate_pct": 62.5, + "macro_hit_rate_pct": 81.25 + } + }, + { + "value": 0.1, + "metrics": { + "prefix_requests": 14, + "prefix_hits": 14, + "prefix_hit_rate_pct": 100, + "tenant_hot_requests": 8, + "tenant_hot_hits": 5, + "tenant_hot_hit_rate_pct": 62.5, + "macro_hit_rate_pct": 81.25 + } + }, + { + "value": 0.2, + "metrics": { + "prefix_requests": 14, + "prefix_hits": 14, + "prefix_hit_rate_pct": 100, + "tenant_hot_requests": 8, + "tenant_hot_hits": 8, + "tenant_hot_hit_rate_pct": 100, + "macro_hit_rate_pct": 100 + } + }, + { + "value": 0.3, + "metrics": { + "prefix_requests": 14, + "prefix_hits": 14, + "prefix_hit_rate_pct": 100, + "tenant_hot_requests": 8, + "tenant_hot_hits": 5, + "tenant_hot_hit_rate_pct": 62.5, + "macro_hit_rate_pct": 81.25 + } + }, + { + "value": 0.4, + "metrics": { + "prefix_requests": 14, + "prefix_hits": 14, + "prefix_hit_rate_pct": 100, + "tenant_hot_requests": 8, + "tenant_hot_hits": 0, + "tenant_hot_hit_rate_pct": 0, + "macro_hit_rate_pct": 50 + } + } + ] + } +} diff --git a/internal/index/calibration/testdata/c1_synthetic_trace.json b/internal/index/calibration/testdata/c1_synthetic_trace.json new file mode 100644 index 00000000..a3fb18e1 --- /dev/null +++ b/internal/index/calibration/testdata/c1_synthetic_trace.json @@ -0,0 +1,347 @@ +{ + "schema_version": 1, + "name": "c1-synthetic-mixed-routing-v1", + "provenance": { + "kind": "synthetic", + "source": "Deterministic boundary-case replay derived from the C1 ReplicaStats and LookupRoute contracts", + "description": "No production C1 request trace is checked into the repository. This fixture exercises pressure/locality tradeoffs, tight and loose TTFT budgets, stale soft-state observations, and TENANT_HOT rate/age gates without claiming to represent production traffic." + }, + "ttl_ms": 1800000, + "sweep": { + "pressure_weights": [0, 0.25, 0.5, 0.75, 1], + "slo_tight_ttft_ms": [100, 150, 200, 250, 300], + "slo_tight_biases": [0, 0.5, 1, 1.5, 2], + "tenant_hot_min_hit_rates": [0.05, 0.1, 0.2, 0.3, 0.4], + "tenant_hot_max_age_ms": [60000, 120000, 300000, 600000] + }, + "observations": [ + { + "id": "pressure-shed-1", + "kind": "prefix", + "at_ms": 100000, + "tenant": "tenant-a", + "model": "model-a", + "hash_scheme": "vllm", + "prefix_hash": "pressure-shed-1", + "token_count": 320, + "replicas": [ + {"id": "a-hot", "reported_at_ms": 100000, "reported_prefix": true, "matched_tokens": 320, "hit_rate": 0.7, "pressure": 0.8, "observed_hit": false}, + {"id": "b-cool", "reported_at_ms": 100000, "reported_prefix": true, "matched_tokens": 256, "hit_rate": 0.4, "pressure": 0.1, "observed_hit": true}, + {"id": "z-decoy", "reported_at_ms": 100000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.1, "pressure": 0, "observed_hit": false} + ] + }, + { + "id": "pressure-shed-2", + "kind": "prefix", + "at_ms": 200000, + "tenant": "tenant-a", + "model": "model-a", + "hash_scheme": "vllm", + "prefix_hash": "pressure-shed-2", + "token_count": 320, + "replicas": [ + {"id": "a-hot", "reported_at_ms": 200000, "reported_prefix": true, "matched_tokens": 320, "hit_rate": 0.7, "pressure": 0.8, "observed_hit": false}, + {"id": "b-cool", "reported_at_ms": 200000, "reported_prefix": true, "matched_tokens": 256, "hit_rate": 0.4, "pressure": 0.1, "observed_hit": true}, + {"id": "z-decoy", "reported_at_ms": 200000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.1, "pressure": 0, "observed_hit": false} + ] + }, + { + "id": "pressure-shed-3", + "kind": "prefix", + "at_ms": 300000, + "tenant": "tenant-b", + "model": "model-b", + "hash_scheme": "vllm", + "prefix_hash": "pressure-shed-3", + "token_count": 320, + "replicas": [ + {"id": "a-hot", "reported_at_ms": 300000, "reported_prefix": true, "matched_tokens": 320, "hit_rate": 0.65, "pressure": 0.8, "observed_hit": false}, + {"id": "b-cool", "reported_at_ms": 300000, "reported_prefix": true, "matched_tokens": 256, "hit_rate": 0.45, "pressure": 0.1, "observed_hit": true}, + {"id": "z-decoy", "reported_at_ms": 300000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.1, "pressure": 0, "observed_hit": false} + ] + }, + { + "id": "pressure-preserve-1", + "kind": "prefix", + "at_ms": 400000, + "tenant": "tenant-a", + "model": "model-a", + "hash_scheme": "vllm", + "prefix_hash": "pressure-preserve-1", + "token_count": 512, + "replicas": [ + {"id": "a-local", "reported_at_ms": 400000, "reported_prefix": true, "matched_tokens": 512, "hit_rate": 0.75, "pressure": 0.7, "observed_hit": true}, + {"id": "b-cool", "reported_at_ms": 400000, "reported_prefix": true, "matched_tokens": 256, "hit_rate": 0.3, "pressure": 0, "observed_hit": false}, + {"id": "z-decoy", "reported_at_ms": 400000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.1, "pressure": 0, "observed_hit": false} + ] + }, + { + "id": "pressure-preserve-2", + "kind": "prefix", + "at_ms": 500000, + "tenant": "tenant-a", + "model": "model-a", + "hash_scheme": "vllm", + "prefix_hash": "pressure-preserve-2", + "token_count": 512, + "replicas": [ + {"id": "a-local", "reported_at_ms": 500000, "reported_prefix": true, "matched_tokens": 512, "hit_rate": 0.75, "pressure": 0.7, "observed_hit": true}, + {"id": "b-cool", "reported_at_ms": 500000, "reported_prefix": true, "matched_tokens": 256, "hit_rate": 0.3, "pressure": 0, "observed_hit": false}, + {"id": "z-decoy", "reported_at_ms": 500000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.1, "pressure": 0, "observed_hit": false} + ] + }, + { + "id": "pressure-preserve-3", + "kind": "prefix", + "at_ms": 600000, + "tenant": "tenant-b", + "model": "model-b", + "hash_scheme": "vllm", + "prefix_hash": "pressure-preserve-3", + "token_count": 512, + "replicas": [ + {"id": "a-local", "reported_at_ms": 600000, "reported_prefix": true, "matched_tokens": 512, "hit_rate": 0.7, "pressure": 0.7, "observed_hit": true}, + {"id": "b-cool", "reported_at_ms": 600000, "reported_prefix": true, "matched_tokens": 256, "hit_rate": 0.35, "pressure": 0, "observed_hit": false}, + {"id": "z-decoy", "reported_at_ms": 600000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.1, "pressure": 0, "observed_hit": false} + ] + }, + { + "id": "slo-promote-120-1", + "kind": "prefix", + "at_ms": 700000, + "tenant": "tenant-a", + "model": "model-a", + "hash_scheme": "vllm", + "prefix_hash": "slo-promote-120-1", + "token_count": 512, + "ttft_budget_ms": 120, + "replicas": [ + {"id": "a-old", "reported_at_ms": 160000, "reported_prefix": true, "matched_tokens": 512, "hit_rate": 0.5, "pressure": 0, "observed_hit": false}, + {"id": "b-fresh", "reported_at_ms": 700000, "reported_prefix": true, "matched_tokens": 320, "hit_rate": 0.5, "pressure": 0, "observed_hit": true}, + {"id": "z-decoy", "reported_at_ms": 700000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.1, "pressure": 0, "observed_hit": false} + ] + }, + { + "id": "slo-promote-120-2", + "kind": "prefix", + "at_ms": 800000, + "tenant": "tenant-b", + "model": "model-b", + "hash_scheme": "vllm", + "prefix_hash": "slo-promote-120-2", + "token_count": 512, + "ttft_budget_ms": 120, + "replicas": [ + {"id": "a-old", "reported_at_ms": 260000, "reported_prefix": true, "matched_tokens": 512, "hit_rate": 0.5, "pressure": 0, "observed_hit": false}, + {"id": "b-fresh", "reported_at_ms": 800000, "reported_prefix": true, "matched_tokens": 320, "hit_rate": 0.5, "pressure": 0, "observed_hit": true}, + {"id": "z-decoy", "reported_at_ms": 800000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.1, "pressure": 0, "observed_hit": false} + ] + }, + { + "id": "slo-promote-180-1", + "kind": "prefix", + "at_ms": 900000, + "tenant": "tenant-a", + "model": "model-a", + "hash_scheme": "vllm", + "prefix_hash": "slo-promote-180-1", + "token_count": 512, + "ttft_budget_ms": 180, + "replicas": [ + {"id": "a-old", "reported_at_ms": 360000, "reported_prefix": true, "matched_tokens": 512, "hit_rate": 0.5, "pressure": 0, "observed_hit": false}, + {"id": "b-fresh", "reported_at_ms": 900000, "reported_prefix": true, "matched_tokens": 320, "hit_rate": 0.5, "pressure": 0, "observed_hit": true}, + {"id": "z-decoy", "reported_at_ms": 900000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.1, "pressure": 0, "observed_hit": false} + ] + }, + { + "id": "slo-promote-180-2", + "kind": "prefix", + "at_ms": 1000000, + "tenant": "tenant-b", + "model": "model-b", + "hash_scheme": "vllm", + "prefix_hash": "slo-promote-180-2", + "token_count": 512, + "ttft_budget_ms": 180, + "replicas": [ + {"id": "a-old", "reported_at_ms": 460000, "reported_prefix": true, "matched_tokens": 512, "hit_rate": 0.5, "pressure": 0, "observed_hit": false}, + {"id": "b-fresh", "reported_at_ms": 1000000, "reported_prefix": true, "matched_tokens": 320, "hit_rate": 0.5, "pressure": 0, "observed_hit": true}, + {"id": "z-decoy", "reported_at_ms": 1000000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.1, "pressure": 0, "observed_hit": false} + ] + }, + { + "id": "slo-preserve-120-1", + "kind": "prefix", + "at_ms": 1100000, + "tenant": "tenant-a", + "model": "model-a", + "hash_scheme": "vllm", + "prefix_hash": "slo-preserve-120-1", + "token_count": 550, + "ttft_budget_ms": 120, + "replicas": [ + {"id": "a-deep", "reported_at_ms": 560000, "reported_prefix": true, "matched_tokens": 550, "hit_rate": 0.6, "pressure": 0, "observed_hit": true}, + {"id": "b-fresh", "reported_at_ms": 1100000, "reported_prefix": true, "matched_tokens": 320, "hit_rate": 0.5, "pressure": 0, "observed_hit": false}, + {"id": "z-decoy", "reported_at_ms": 1100000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.1, "pressure": 0, "observed_hit": false} + ] + }, + { + "id": "slo-preserve-120-2", + "kind": "prefix", + "at_ms": 1200000, + "tenant": "tenant-b", + "model": "model-b", + "hash_scheme": "vllm", + "prefix_hash": "slo-preserve-120-2", + "token_count": 550, + "ttft_budget_ms": 120, + "replicas": [ + {"id": "a-deep", "reported_at_ms": 660000, "reported_prefix": true, "matched_tokens": 550, "hit_rate": 0.6, "pressure": 0, "observed_hit": true}, + {"id": "b-fresh", "reported_at_ms": 1200000, "reported_prefix": true, "matched_tokens": 320, "hit_rate": 0.5, "pressure": 0, "observed_hit": false}, + {"id": "z-decoy", "reported_at_ms": 1200000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.1, "pressure": 0, "observed_hit": false} + ] + }, + { + "id": "slo-loose-220-1", + "kind": "prefix", + "at_ms": 1300000, + "tenant": "tenant-a", + "model": "model-a", + "hash_scheme": "vllm", + "prefix_hash": "slo-loose-220-1", + "token_count": 512, + "ttft_budget_ms": 220, + "replicas": [ + {"id": "a-deep", "reported_at_ms": 760000, "reported_prefix": true, "matched_tokens": 512, "hit_rate": 0.6, "pressure": 0, "observed_hit": true}, + {"id": "b-fresh", "reported_at_ms": 1300000, "reported_prefix": true, "matched_tokens": 320, "hit_rate": 0.5, "pressure": 0, "observed_hit": false}, + {"id": "z-decoy", "reported_at_ms": 1300000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.1, "pressure": 0, "observed_hit": false} + ] + }, + { + "id": "slo-loose-220-2", + "kind": "prefix", + "at_ms": 1400000, + "tenant": "tenant-b", + "model": "model-b", + "hash_scheme": "vllm", + "prefix_hash": "slo-loose-220-2", + "token_count": 512, + "ttft_budget_ms": 220, + "replicas": [ + {"id": "a-deep", "reported_at_ms": 860000, "reported_prefix": true, "matched_tokens": 512, "hit_rate": 0.6, "pressure": 0, "observed_hit": true}, + {"id": "b-fresh", "reported_at_ms": 1400000, "reported_prefix": true, "matched_tokens": 320, "hit_rate": 0.5, "pressure": 0, "observed_hit": false}, + {"id": "z-decoy", "reported_at_ms": 1400000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.1, "pressure": 0, "observed_hit": false} + ] + }, + { + "id": "tenant-rate-1", + "kind": "tenant_hot", + "at_ms": 1500000, + "tenant": "tenant-a", + "model": "model-a", + "hash_scheme": "vllm", + "prefix_hash": "tenant-rate-1", + "token_count": 64, + "replicas": [ + {"id": "a-noisy", "reported_at_ms": 1500000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.19, "pressure": 0, "observed_hit": false}, + {"id": "z-warm", "reported_at_ms": 1470000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.25, "pressure": 0, "observed_hit": true} + ] + }, + { + "id": "tenant-rate-2", + "kind": "tenant_hot", + "at_ms": 1600000, + "tenant": "tenant-a", + "model": "model-a", + "hash_scheme": "vllm", + "prefix_hash": "tenant-rate-2", + "token_count": 64, + "replicas": [ + {"id": "a-noisy", "reported_at_ms": 1600000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.19, "pressure": 0, "observed_hit": false}, + {"id": "z-warm", "reported_at_ms": 1570000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.25, "pressure": 0, "observed_hit": true} + ] + }, + { + "id": "tenant-rate-3", + "kind": "tenant_hot", + "at_ms": 1700000, + "tenant": "tenant-b", + "model": "model-b", + "hash_scheme": "vllm", + "prefix_hash": "tenant-rate-3", + "token_count": 64, + "replicas": [ + {"id": "a-noisy", "reported_at_ms": 1700000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.19, "pressure": 0, "observed_hit": false}, + {"id": "z-warm", "reported_at_ms": 1670000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.25, "pressure": 0, "observed_hit": true} + ] + }, + { + "id": "tenant-age-preserve-1", + "kind": "tenant_hot", + "at_ms": 1800000, + "tenant": "tenant-a", + "model": "model-a", + "hash_scheme": "vllm", + "prefix_hash": "tenant-age-preserve-1", + "token_count": 64, + "replicas": [ + {"id": "a-moderate", "reported_at_ms": 1725000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.3, "pressure": 0, "observed_hit": true} + ] + }, + { + "id": "tenant-age-preserve-2", + "kind": "tenant_hot", + "at_ms": 1900000, + "tenant": "tenant-b", + "model": "model-b", + "hash_scheme": "vllm", + "prefix_hash": "tenant-age-preserve-2", + "token_count": 64, + "replicas": [ + {"id": "a-moderate", "reported_at_ms": 1825000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.3, "pressure": 0, "observed_hit": true} + ] + }, + { + "id": "tenant-age-expire-1", + "kind": "tenant_hot", + "at_ms": 2000000, + "tenant": "tenant-a", + "model": "model-a", + "hash_scheme": "vllm", + "prefix_hash": "tenant-age-expire-1", + "token_count": 64, + "replicas": [ + {"id": "a-stale", "reported_at_ms": 1850000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.8, "pressure": 0, "observed_hit": false}, + {"id": "z-recent", "reported_at_ms": 1970000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.3, "pressure": 0, "observed_hit": true} + ] + }, + { + "id": "tenant-age-expire-2", + "kind": "tenant_hot", + "at_ms": 2100000, + "tenant": "tenant-a", + "model": "model-a", + "hash_scheme": "vllm", + "prefix_hash": "tenant-age-expire-2", + "token_count": 64, + "replicas": [ + {"id": "a-stale", "reported_at_ms": 1950000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.8, "pressure": 0, "observed_hit": false}, + {"id": "z-recent", "reported_at_ms": 2070000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.3, "pressure": 0, "observed_hit": true} + ] + }, + { + "id": "tenant-age-expire-3", + "kind": "tenant_hot", + "at_ms": 2200000, + "tenant": "tenant-b", + "model": "model-b", + "hash_scheme": "vllm", + "prefix_hash": "tenant-age-expire-3", + "token_count": 64, + "replicas": [ + {"id": "a-stale", "reported_at_ms": 2050000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.8, "pressure": 0, "observed_hit": false}, + {"id": "z-recent", "reported_at_ms": 2170000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.3, "pressure": 0, "observed_hit": true} + ] + } + ] +} diff --git a/internal/index/ranking.go b/internal/index/ranking.go index 8ffd624a..da461706 100644 --- a/internal/index/ranking.go +++ b/internal/index/ranking.go @@ -14,9 +14,9 @@ import ( // when no stats are present and no SLO hint is set — see DefaultRankerConfig. const ( // Pressure penalty: pressureFactor = 1 - PressureWeight × pressure. - // 1.0 → a fully-saturated replica (pressure=1.0) drops to score 0, so a - // fresher lower-pressure peer can win. Lower values are gentler. - DefaultPressureWeight = 1.0 + // 0.5 keeps locality meaningful while letting a lower-pressure peer win + // when the reported token advantage is modest. + DefaultPressureWeight = 0.5 // TTFT below this (ms) is treated as "tight" — the SLO bias kicks in. // 200 ms is a conservative threshold; tune per workload. DefaultSLOTightTTFTMs = 200 @@ -25,10 +25,10 @@ const ( // against matched-token count when latency is critical. DefaultSLOTightBias = 1.0 // TENANT_HOT fallback: replicas with hit_rate >= this count as "warm". - DefaultTenantHotMinHitRate = 0.1 + DefaultTenantHotMinHitRate = 0.2 // TENANT_HOT fallback: stats lastSeen within this window count as // "recent" — anything older is treated as cold for the fallback. - DefaultTenantHotMaxAge = 5 * time.Minute + DefaultTenantHotMaxAge = 2 * time.Minute ) // applyChainDistinguishingPower folds the depth-aware distinguishing-power diff --git a/internal/index/ranking_test.go b/internal/index/ranking_test.go index 24ffb998..ceca4c70 100644 --- a/internal/index/ranking_test.go +++ b/internal/index/ranking_test.go @@ -9,6 +9,20 @@ import ( "time" ) +func TestDefaultRankerConfigMatchesCalibratedTuple(t *testing.T) { + got := DefaultRankerConfig() + want := RankerConfig{ + PressureWeight: 0.5, + SLOTightTTFTMs: 200, + SLOTightBias: 1, + TenantHotMinHitRate: 0.2, + TenantHotMaxAge: 2 * time.Minute, + } + if got != want { + t.Fatalf("DefaultRankerConfig() = %+v, want calibrated tuple %+v", got, want) + } +} + // TestLookupPressureAndSLOFactorsCollapseToUnityWhenSignalsAbsent locks in the // contract that the pressure and SLO score factors collapse to 1 when (a) no // replica stats are reported (pressure=0) and (b) the request carries no SLO @@ -257,7 +271,9 @@ func TestWorstTierPrefersLeastLocal(t *testing.T) { // have a chain hit would outrank a fresher idle peer the chain-aware // formula was supposed to demote. func TestChainLookupSharesPressureAndSLOFactorsWithExact(t *testing.T) { - idx := New(WithTTL(time.Hour)) + cfg := DefaultRankerConfig() + cfg.PressureWeight = 1 + idx := New(WithTTL(time.Hour), WithRanker(cfg)) hashes, counts := chain("b1", "b2", "b3") idx.Ingest(Update{ReplicaID: "big-but-hot", Model: "m", Tenant: "t", HashScheme: "vllm", diff --git a/site/content/en/docs/reference/reason-codes.md b/site/content/en/docs/reference/reason-codes.md index 4a032c32..56bb4334 100644 --- a/site/content/en/docs/reference/reason-codes.md +++ b/site/content/en/docs/reference/reason-codes.md @@ -44,11 +44,11 @@ See [LookupRoute & ranking]({{< relref "/docs/concepts/lookuproute/#diagnostics- | `strategy.requireChain` | CachePolicy | false | (n/a) | | `strategy.enableTenantHot` | CachePolicy | true | false | | `affinityRouting` | CachePolicy | `Enabled` | `Disabled` | -| `PressureWeight` | server RankerConfig | 1.0 | 0 | +| `PressureWeight` | server RankerConfig | 0.5 | 0 | | `SLOTightTTFTMs` | server RankerConfig | 200ms | 0 | | `SLOTightBias` | server RankerConfig | 1.0 | 0 | -| `TenantHotMaxAge` | server RankerConfig | 5m | 0 | -| `TenantHotMinHitRate` | server RankerConfig | 0.1 | — | +| `TenantHotMaxAge` | server RankerConfig | 2m | 0 | +| `TenantHotMinHitRate` | server RankerConfig | 0.2 | — | ## RenderTemplate From 95063e32e4ec3870db3ef0b011ecea48d6040eaa Mon Sep 17 00:00:00 2001 From: Weiwei Zheng Date: Wed, 19 Aug 2026 14:29:19 -0700 Subject: [PATCH 2/8] Make ranker replay deterministic Signed-off-by: Weiwei Zheng --- docs/design/lookuproute-ranking.md | 4 ++-- internal/index/calibration/calibration.go | 6 +++--- internal/index/calibration/calibration_test.go | 15 +++++++++++++++ internal/index/index.go | 7 +++++-- 4 files changed, 25 insertions(+), 7 deletions(-) diff --git a/docs/design/lookuproute-ranking.md b/docs/design/lookuproute-ranking.md index 9b820329..7f6856bd 100644 --- a/docs/design/lookuproute-ranking.md +++ b/docs/design/lookuproute-ranking.md @@ -875,8 +875,8 @@ The trace and generated per-knob curves live in `SLOTightTTFTMs = 200 ms`, `SLOTightBias = 1.0`, `TenantHotMinHitRate = 0.2`, and `TenantHotMaxAge = 2 min`; both measured hit ratios are 100% on this boundary-case fixture. CI regenerates the result in -check mode so trace, curves, documentation, and defaults cannot silently -diverge. +check mode, and tests keep the trace, generated result, and code defaults from +silently diverging. Documentation changes remain subject to normal review. ## 7. The reason-code summary diff --git a/internal/index/calibration/calibration.go b/internal/index/calibration/calibration.go index b1bc60d2..531cf3fb 100644 --- a/internal/index/calibration/calibration.go +++ b/internal/index/calibration/calibration.go @@ -337,7 +337,7 @@ func Replay(trace Trace, config Config) Metrics { } func replayObservation(trace Trace, observation Observation, config Config) bool { - anchor := time.Now() + anchor := time.UnixMilli(observation.AtMillis) ranker := index.RankerConfig{ PressureWeight: float32(config.PressureWeight), SLOTightTTFTMs: config.SLOTightTTFTMillis, @@ -348,6 +348,7 @@ func replayObservation(trace Trace, observation Observation, config Config) bool idx := index.New( index.WithTTL(time.Duration(trace.TTLMillis)*time.Millisecond), index.WithRanker(ranker), + index.WithClock(func() time.Time { return anchor }), ) observedHits := make(map[string]bool, len(observation.Replicas)) for _, replica := range observation.Replicas { @@ -357,13 +358,12 @@ func replayObservation(trace Trace, observation Observation, config Config) bool hash = "serving/" + observation.ID + "/" + replica.ID tokens = 1 } - age := time.Duration(observation.AtMillis-replica.ReportedAtMillis) * time.Millisecond idx.Ingest(index.Update{ ReplicaID: replica.ID, Model: observation.Model, Tenant: observation.Tenant, HashScheme: observation.HashScheme, - Timestamp: anchor.Add(-age), + Timestamp: time.UnixMilli(replica.ReportedAtMillis), Prefixes: []index.PrefixRef{{ PrefixHash: []byte(hash), TokenCount: tokens, diff --git a/internal/index/calibration/calibration_test.go b/internal/index/calibration/calibration_test.go index 69c70784..6828c448 100644 --- a/internal/index/calibration/calibration_test.go +++ b/internal/index/calibration/calibration_test.go @@ -127,6 +127,21 @@ func TestReplayTenantHotMissWithoutCandidate(t *testing.T) { } } +func TestReplayUsesObservationClockAtTenantHotBoundary(t *testing.T) { + trace := Trace{TTLMillis: int64(time.Minute / time.Millisecond)} + observation := Observation{ + ID: "tenant-hot-boundary", Kind: ObservationTenantHot, AtMillis: 100_000, + Tenant: "t", Model: "m", HashScheme: "vllm", PrefixHash: "novel", TokenCount: 1, + Replicas: []ReplicaObservation{{ + ID: "warm", ReportedAtMillis: 40_001, HitRate: 0.8, ObservedHit: true, + }}, + } + config := Config{TenantHotMinHitRate: 0.2, TenantHotMaxAgeMillis: 60_000} + if !replayObservation(trace, observation, config) { + t.Fatal("replayObservation = miss, want hit for stats one millisecond inside the replay window") + } +} + func TestObservationRejectsFutureReplicaReport(t *testing.T) { observation := Observation{ ID: "future", Kind: ObservationPrefix, AtMillis: 10, diff --git a/internal/index/index.go b/internal/index/index.go index bafaa7dc..a9199083 100644 --- a/internal/index/index.go +++ b/internal/index/index.go @@ -264,8 +264,11 @@ func WithReservedTenants(tenants ...string) Option { } } -// withClock overrides the time source (tests only). -func withClock(now func() time.Time) Option { return func(i *Index) { i.now = now } } +// WithClock overrides the time source for deterministic replay and tests. +func WithClock(now func() time.Time) Option { return func(i *Index) { i.now = now } } + +// withClock keeps the package-local test helper concise. +func withClock(now func() time.Time) Option { return WithClock(now) } // New builds an index with the given options. func New(opts ...Option) *Index { From 24e2579bed20f588fd252d356f131060dfd07871 Mon Sep 17 00:00:00 2001 From: Weiwei Zheng Date: Wed, 19 Aug 2026 17:26:57 -0700 Subject: [PATCH 3/8] Cover ranker calibration validation Signed-off-by: Weiwei Zheng --- .../index/calibration/calibration_test.go | 121 ++++++++++++++++++ 1 file changed, 121 insertions(+) diff --git a/internal/index/calibration/calibration_test.go b/internal/index/calibration/calibration_test.go index 6828c448..ae72d330 100644 --- a/internal/index/calibration/calibration_test.go +++ b/internal/index/calibration/calibration_test.go @@ -6,6 +6,7 @@ package calibration import ( "bytes" + "math" "os" "strings" "testing" @@ -112,6 +113,126 @@ func TestLoadRejectsUnknownAndInvalidFields(t *testing.T) { } } +func TestTraceValidationRejectsInvalidFields(t *testing.T) { + valid := func() Trace { + return Trace{ + SchemaVersion: SchemaVersion, + Name: "unit", + Provenance: Provenance{Kind: "synthetic", Source: "unit test"}, + TTLMillis: 1, + Sweep: Sweep{ + PressureWeights: []float64{0.5}, + SLOTightTTFTMillis: []int32{200}, + SLOTightBiases: []float64{1}, + TenantHotMinHitRates: []float64{0.2}, + TenantHotMaxAgeMillis: []int64{60_000}, + }, + Observations: []Observation{{ + ID: "o", Kind: ObservationPrefix, Tenant: "t", Model: "m", + HashScheme: "vllm", PrefixHash: "p", TokenCount: 1, + Replicas: []ReplicaObservation{{ + ID: "r", ReportedPrefix: true, MatchedTokens: 1, + }}, + }}, + } + } + + for _, tc := range []struct { + name string + mutate func(*Trace) + want string + }{ + {"version", func(trace *Trace) { trace.SchemaVersion++ }, "schema_version"}, + {"name", func(trace *Trace) { trace.Name = "" }, "trace name"}, + {"provenance kind", func(trace *Trace) { trace.Provenance.Kind = "unknown" }, "captured or synthetic"}, + {"provenance source", func(trace *Trace) { trace.Provenance.Source = "" }, "provenance source"}, + {"ttl", func(trace *Trace) { trace.TTLMillis = 0 }, "ttl_ms"}, + {"empty sweep", func(trace *Trace) { trace.Sweep.PressureWeights = nil }, "every sweep dimension"}, + {"empty observations", func(trace *Trace) { trace.Observations = nil }, "at least one observation"}, + {"invalid observation", func(trace *Trace) { trace.Observations[0].Kind = "unknown" }, "observation 0"}, + {"duplicate observation", func(trace *Trace) { trace.Observations = append(trace.Observations, trace.Observations[0]) }, "duplicate id"}, + } { + t.Run(tc.name, func(t *testing.T) { + trace := valid() + tc.mutate(&trace) + if err := trace.Validate(); err == nil || !strings.Contains(err.Error(), tc.want) { + t.Fatalf("Validate error = %v, want substring %q", err, tc.want) + } + }) + } +} + +func TestSweepValidationRejectsInvalidValues(t *testing.T) { + valid := func() Sweep { + return Sweep{ + PressureWeights: []float64{0.5}, + SLOTightTTFTMillis: []int32{200}, + SLOTightBiases: []float64{1}, + TenantHotMinHitRates: []float64{0.2}, + TenantHotMaxAgeMillis: []int64{60_000}, + } + } + for _, tc := range []struct { + name string + mutate func(*Sweep) + want string + }{ + {"pressure", func(sweep *Sweep) { sweep.PressureWeights[0] = -1 }, "non-negative"}, + {"hit rate", func(sweep *Sweep) { sweep.TenantHotMinHitRates[0] = 2 }, "invalid rate"}, + {"ttft", func(sweep *Sweep) { sweep.SLOTightTTFTMillis[0] = 0 }, "non-positive"}, + {"max age", func(sweep *Sweep) { sweep.TenantHotMaxAgeMillis[0] = 0 }, "non-positive"}, + } { + t.Run(tc.name, func(t *testing.T) { + sweep := valid() + tc.mutate(&sweep) + if err := sweep.validate(); err == nil || !strings.Contains(err.Error(), tc.want) { + t.Fatalf("validate error = %v, want substring %q", err, tc.want) + } + }) + } +} + +func TestObservationValidationRejectsInvalidReplicas(t *testing.T) { + valid := func() Observation { + return Observation{ + ID: "o", Kind: ObservationPrefix, Tenant: "t", Model: "m", + HashScheme: "vllm", PrefixHash: "p", TokenCount: 1, + Replicas: []ReplicaObservation{{ID: "r", ReportedPrefix: true, MatchedTokens: 1}}, + } + } + for _, tc := range []struct { + name string + mutate func(*Observation) + want string + }{ + {"identity", func(observation *Observation) { observation.ID = "" }, "required"}, + {"kind", func(observation *Observation) { observation.Kind = "unknown" }, "prefix or tenant_hot"}, + {"tokens", func(observation *Observation) { observation.TokenCount = 0 }, "token_count"}, + {"replicas", func(observation *Observation) { observation.Replicas = nil }, "at least one replica"}, + {"replica id", func(observation *Observation) { observation.Replicas[0].ID = "" }, "id is required"}, + {"matched tokens", func(observation *Observation) { observation.Replicas[0].MatchedTokens = 0 }, "matched_tokens"}, + {"rate", func(observation *Observation) { observation.Replicas[0].HitRate = 2 }, "finite values"}, + {"duplicate replica", func(observation *Observation) { + observation.Replicas = append(observation.Replicas, observation.Replicas[0]) + }, "duplicate replica"}, + } { + t.Run(tc.name, func(t *testing.T) { + observation := valid() + tc.mutate(&observation) + if err := observation.validate(); err == nil || !strings.Contains(err.Error(), tc.want) { + t.Fatalf("validate error = %v, want substring %q", err, tc.want) + } + }) + } +} + +func TestMarshalResultWrapsJSONErrors(t *testing.T) { + _, err := MarshalResult(Result{BestConfig: Config{PressureWeight: math.NaN()}}) + if err == nil || !strings.Contains(err.Error(), "marshal calibration result") { + t.Fatalf("MarshalResult error = %v, want wrapped JSON error", err) + } +} + func TestReplayTenantHotMissWithoutCandidate(t *testing.T) { trace := Trace{TTLMillis: int64(time.Minute / time.Millisecond)} observation := Observation{ From 789191ce390f3abaab7d0630490646da6a015e3e Mon Sep 17 00:00:00 2001 From: Weiwei Zheng Date: Wed, 19 Aug 2026 17:35:07 -0700 Subject: [PATCH 4/8] Keep production ranker defaults stable Signed-off-by: Weiwei Zheng --- docs/design/grpc-contract.md | 2 +- docs/design/lookuproute-ranking.md | 63 ++++++++++--------- docs/reference/reason-codes.md | 8 +-- internal/index/calibration/README.md | 4 +- .../index/calibration/calibration_test.go | 24 +++---- internal/index/ranking.go | 10 +-- internal/index/ranking_test.go | 10 +-- .../content/en/docs/reference/reason-codes.md | 6 +- 8 files changed, 66 insertions(+), 61 deletions(-) diff --git a/docs/design/grpc-contract.md b/docs/design/grpc-contract.md index 1d374fe3..2cbc5685 100644 --- a/docs/design/grpc-contract.md +++ b/docs/design/grpc-contract.md @@ -116,7 +116,7 @@ unchanged and old clients continue to fail open on a downgrade. **Update — B6 (CacheIndex status surface):** the cluster-wide aggregate is now exposed two ways: an internal HTTP `/snapshot` endpoint on the server (JSON; metadata only — replica/tenant stats + prefix counts, never KV/prompt data), and a cluster-scoped, status-only `CacheIndex` CRD (`kubectl get cacheindex`) that the controller maintains by scraping `/snapshot`. This is outside the gRPC contract (no proto change); see the `CacheIndex` type in `api/v1alpha1` and the `CacheIndexPoller` in `internal/controller`. -**Update — B6 follow-up (`LookupRoute` ranking v2):** the `LookupRoute` ranker layers additive strategies on top of the original `matched_tokens × freshness` baseline, **without any proto change** (all inputs were already on the contract). Today the full score is `matched_tokens × freshness × pressure_factor × slo_bias × distinguishing_power` where `pressure_factor = max(0, 1 - PressureWeight × ReplicaStats.pressure)`, `slo_bias = 1 + freshness × SLOTightBias` when `SLO.ttft_ms` is below a configurable threshold (otherwise 1), and `distinguishing_power = 1 − num_matching_replicas / total_replicas` (1.0 when `total_replicas ≤ 1`; see [`lookuproute-ranking.md` §2.7](./lookuproute-ranking.md#27-the-replica-distinguishing-power-factor)). On a prefix miss, the server falls back to **`TENANT_HOT`**: ranked replicas that are warm for the request's `(tenant, model, hash_scheme)` — i.e. the replica has at least one prefix entry in the requested engine domain AND its latest stats are recent (within a configurable window, default 2m) with `hit_rate` above a floor (default 0.2). `TENANT_HOT` responses carry `matched_tokens=0` because there is no prefix overlap; the gateway must rely on `reason_code`, not `matched_tokens`, to recognize the soft hint. The pressure/SLO factors collapse to 1 when their supporting input is absent (no stats → pressure_factor = 1; no SLO hint → slo_bias = 1; `TenantHotMaxAge=0` disables the TENANT_HOT fallback entirely); the distinguishing-power factor collapses to 1 only for single-replica deployments — multi-replica deployments always see a cardinality-adjusted score. See [`lookuproute-ranking.md`](./lookuproute-ranking.md) and [`reason-codes.md`](../reference/reason-codes.md) for the full knob table. +**Update — B6 follow-up (`LookupRoute` ranking v2):** the `LookupRoute` ranker layers additive strategies on top of the original `matched_tokens × freshness` baseline, **without any proto change** (all inputs were already on the contract). Today the full score is `matched_tokens × freshness × pressure_factor × slo_bias × distinguishing_power` where `pressure_factor = max(0, 1 - PressureWeight × ReplicaStats.pressure)`, `slo_bias = 1 + freshness × SLOTightBias` when `SLO.ttft_ms` is below a configurable threshold (otherwise 1), and `distinguishing_power = 1 − num_matching_replicas / total_replicas` (1.0 when `total_replicas ≤ 1`; see [`lookuproute-ranking.md` §2.7](./lookuproute-ranking.md#27-the-replica-distinguishing-power-factor)). On a prefix miss, the server falls back to **`TENANT_HOT`**: ranked replicas that are warm for the request's `(tenant, model, hash_scheme)` — i.e. the replica has at least one prefix entry in the requested engine domain AND its latest stats are recent (within a configurable window, default 5m) with `hit_rate` above a floor (default 0.1). `TENANT_HOT` responses carry `matched_tokens=0` because there is no prefix overlap; the gateway must rely on `reason_code`, not `matched_tokens`, to recognize the soft hint. The pressure/SLO factors collapse to 1 when their supporting input is absent (no stats → pressure_factor = 1; no SLO hint → slo_bias = 1; `TenantHotMaxAge=0` disables the TENANT_HOT fallback entirely); the distinguishing-power factor collapses to 1 only for single-replica deployments — multi-replica deployments always see a cardinality-adjusted score. See [`lookuproute-ranking.md`](./lookuproute-ranking.md) and [`reason-codes.md`](../reference/reason-codes.md) for the full knob table. **Update — `CachePolicy.spec.strategy`:** the server enforces three per-namespace strategy gates before results leave the handler. `enableChainMatching=false` strips block-hash chain inputs and forces the legacy exact `prefix_hash` path; `requireChain=true` returns empty scores with `POLICY_REQUIRES_CHAIN` before touching the index when a request has no valid wire block-hash chain; `enableTenantHot=false` downgrades a `TENANT_HOT` result to `NO_HINT`. Defaults are `true` / `false` / `true`, preserving the previous behavior. diff --git a/docs/design/lookuproute-ranking.md b/docs/design/lookuproute-ranking.md index 7f6856bd..1d0f5541 100644 --- a/docs/design/lookuproute-ranking.md +++ b/docs/design/lookuproute-ranking.md @@ -584,11 +584,11 @@ pressure_factor = max(0, 1 − PressureWeight × pressure) score = matched_tokens × freshness × pressure_factor ``` -Worked example with the default `PressureWeight = 0.5`: +Worked example with `PressureWeight = 1`: | Replica | tokens | freshness | pressure | baseline score | new score | |---|---|---|---|---|---| -| `big-but-hot` | 80 | 1.0 | 0.9 | 80 | **44** | +| `big-but-hot` | 100 | 1.0 | 0.9 | 100 | **10** | | `small-cool` | 50 | 1.0 | 0.0 | 50 | **50** | Under the baseline `big-but-hot` wins; with pressure folded in, the smaller @@ -623,8 +623,8 @@ When the prefix-match path returns empty, the ranker runs a second strategy: 1. Find replicas under `(tenant, model)` whose stats are recent - (`statsReported` within `TenantHotMaxAge`, default 2 min) AND whose - `hit_rate` is at least a floor (default `0.2`). These are "warm." + (`statsReported` within `TenantHotMaxAge`, default 5 min) AND whose + `hit_rate` is at least a floor (default `0.1`). These are "warm." 2. Restrict to replicas that *actually serve* the requested `hash_scheme` — i.e. they hold at least one prefix entry in the request's engine domain. Without this guard, a stats-only update with @@ -848,10 +848,9 @@ set so that: ### Calibration provenance and replay -The default tuple is selected by the reproducible sweep under -`internal/index/calibration`, which calls the production `LookupRoute` -implementation for every observation and searches the configured Cartesian -grid. The objective is the macro-average of prefix-hit ratio and +The reproducible sweep under `internal/index/calibration` calls the production +`LookupRoute` implementation for every observation and searches the configured +Cartesian grid. The objective is the macro-average of prefix-hit ratio and `TENANT_HOT`-hit ratio; ties prefer gentler score multipliers and the shorter fallback window. @@ -860,9 +859,10 @@ The checked-in `c1-synthetic-mixed-routing-v1` trace contains 22 observations: TTFT budgets, plus 8 prefix-miss cases spanning noisy hit-rate reports and fresh/stale `TENANT_HOT` candidates. Its provenance is explicitly `synthetic`: no production C1 request trace is currently checked into this -repository, so these values are a deterministic first calibration, not a claim -about production traffic. Replace the trace with a sanitized captured fixture -and rerun the same command when such data is available. +repository, so these values verify the harness and identify a provisional +candidate, not a production calibration. `DefaultRankerConfig` therefore keeps +its existing `1.0 / 200 ms / 1.0 / 0.1 / 5 min` tuple. Replace or supplement +the trace with a sanitized captured fixture before changing those defaults. ```bash make ranker-calibration @@ -871,12 +871,13 @@ make verify-ranker-calibration The trace and generated per-knob curves live in `internal/index/calibration/testdata/c1_synthetic_trace.json` and -`c1_synthetic_result.json`. The sweep selects `PressureWeight = 0.5`, +`c1_synthetic_result.json`. On this synthetic boundary fixture, the sweep +selects the candidate `PressureWeight = 0.5`, `SLOTightTTFTMs = 200 ms`, `SLOTightBias = 1.0`, `TenantHotMinHitRate = 0.2`, and `TenantHotMaxAge = 2 min`; both measured hit -ratios are 100% on this boundary-case fixture. CI regenerates the result in -check mode, and tests keep the trace, generated result, and code defaults from -silently diverging. Documentation changes remain subject to normal review. +ratios are 100%. CI regenerates the result in check mode, and tests keep the +trace and generated result from silently diverging. This candidate is not +applied to production defaults without representative captured evidence. ## 7. The reason-code summary @@ -902,9 +903,9 @@ carve-outs that keep them on `NO_HINT`). Six concrete scenarios that exercise the strategies in §2–5 end-to-end. Each shows the relevant index state, the request, the score computation, and the response. All examples use the default -`RankerConfig`: `PressureWeight = 0.5`, `SLOTightTTFTMs = 200 ms`, -`SLOTightBias = 1`, `TenantHotMaxAge = 2 min`, -`TenantHotMinHitRate = 0.2`. +`RankerConfig`: `PressureWeight = 1`, `SLOTightTTFTMs = 200 ms`, +`SLOTightBias = 1`, `TenantHotMaxAge = 5 min`, +`TenantHotMinHitRate = 0.1`. ### 8.1. Baseline: one replica holds the prefix @@ -916,7 +917,7 @@ Index state — tenant `team-a`, model `m`, scheme `vllm`: Request: `{tenant=team-a, model=m, hash_scheme=vllm, prefix_hash=p}`, no SLO. -Computation: `100 × 1.0 × (1 − 0.5 × 0.0) × 1 = 100`. +Computation: `100 × 1.0 × (1 − 1 × 0.0) × 1 = 100`. Response: `reason_code=PREFIX_MATCH`, scores `[{r1, score=100, matched_tokens=100}]`. @@ -929,19 +930,19 @@ Index state: | Replica | Prefix | Tokens | Freshness | Pressure | |---|---|---|---|---| -| `big-but-hot` | `p` | 80 | 1.0 | 0.9 | +| `big-but-hot` | `p` | 100 | 1.0 | 0.9 | | `small-cool` | `p` | 50 | 1.0 | 0.0 | Request: same as §8.1, no SLO. Computation: -- `big-but-hot`: `80 × 1.0 × (1 − 0.5 × 0.9) × 1 = 80 × 0.55 = 44` -- `small-cool`: `50 × 1.0 × (1 − 0.5 × 0.0) × 1 = 50` +- `big-but-hot`: `100 × 1.0 × (1 − 1 × 0.9) × 1 = 100 × 0.1 = 10` +- `small-cool`: `50 × 1.0 × (1 − 1 × 0.0) × 1 = 50` -Response: `PREFIX_MATCH`, ranked `[small-cool (50), big-but-hot (44)]`. +Response: `PREFIX_MATCH`, ranked `[small-cool (50), big-but-hot (10)]`. The pure baseline (`tokens × freshness`) would have given `big-but-hot` -a score of `80` vs `small-cool`'s `50` and routed traffic to the +a score of `100` vs `small-cool`'s `50` and routed traffic to the already-saturated replica. The pressure factor flips it: locality weighed against load. @@ -982,16 +983,16 @@ no SLO. Prefix-match path: empty (no replica holds `novel`). -Tenant-hot fallback (defaults `TenantHotMaxAge = 2 min`, -`TenantHotMinHitRate = 0.2`): -- `r-warm` reported 30 s ago (well under 2 min), `hit_rate = 0.8 ≥ 0.2`, +Tenant-hot fallback (defaults `TenantHotMaxAge = 5 min`, +`TenantHotMinHitRate = 0.1`): +- `r-warm` reported 30 s ago (well under 5 min), `hit_rate = 0.8 ≥ 0.1`, and it holds at least one prefix in `vllm` (`other`) — qualifies. -- `recency = 1 − 30 s / 2 min = 0.75` -- `pressure_factor = 1 − 0.5 × 0.1 = 0.95` +- `recency = 1 − 30 s / 5 min = 0.9` +- `pressure_factor = 1 − 1 × 0.1 = 0.9` - `slo_bias = 1` -- `score = 0.8 × 0.75 × 0.95 × 1 = 0.57` +- `score = 0.8 × 0.9 × 0.9 × 1 = 0.648` -Response: `TENANT_HOT`, scores `[{r-warm, score=0.57, matched_tokens=0}]`. +Response: `TENANT_HOT`, scores `[{r-warm, score=0.648, matched_tokens=0}]`. `matched_tokens` is `0` because there's no prefix overlap — the gateway must rely on `reason_code` (not `matched_tokens`) to tell `TENANT_HOT` diff --git a/docs/reference/reason-codes.md b/docs/reference/reason-codes.md index 970a70dd..d06ff466 100644 --- a/docs/reference/reason-codes.md +++ b/docs/reference/reason-codes.md @@ -37,7 +37,7 @@ module — until then, treat `NO_HINT` as the only `LookupPDRoute` answer. | `PREFIX_MATCH` | **shipped** | LookupRoute only | The index has at least one replica holding the request's `(tenant, model, hash_scheme, adapter)` prefix — either the exact `prefix_hash` (legacy single-blob path) OR the leading run of `block_hashes[0..k]` (chain longest-prefix path; see [`../design/lookuproute-ranking.md`](../design/lookuproute-ranking.md) §2.5) — the ranker returned a non-empty set, AND at least one replica's realized `matched_tokens` cleared the per-namespace `minimumMatchedTokens` floor (default 64) AND the top surviving replica's score cleared the per-namespace `routingFloorScore` floor (default `0.1`). The score includes the distinguishing-power factor `1 − num_matching_replicas / total_replicas` so an overlap held by every replica (chat-template framing, RAG corpus headers, custom system prompts) collapses to score 0 and is filtered by the score floor. Both floors can downgrade independently. | `replica_scores` non-empty, ranked best-first by `matched_tokens × freshness × pressure_factor × slo_bias × distinguishing_power`. The pressure / SLO / distinguishing-power factors collapse appropriately when their inputs are absent (no stats → pressure_factor = 1; no SLO hint → slo_bias = 1; single-replica deployment → distinguishing_power = 1). Every qualifying replica is returned today (no top-K limit); the gateway typically uses the top entry. | Route to the top-ranked replica → prefix-cache hit; lower TTFT. | | `NO_HINT` | **shipped** | both | The fail-open default. **`LookupPDRoute`**: every call (the handler is a stub). **`LookupRoute`**: the prefix is novel under matching contract keys AND no usable affinity fallback fired (`affinityRouting: Disabled` on the per-namespace policy, OR no replica known to serve the `(tenant, model, hash_scheme)` engine domain, OR no usable seed — empty `block_hashes` and empty `prefix_hash`, OR structurally malformed input — empty `hash_scheme` or chain arrays of mismatched length); the ranker found nothing AND the same affinity-disabled / no-replica / no-seed / malformed clause holds; any of `tenant_id`, `model_id`, or `hash_scheme` was unspecified (a contract violation — set-but-wrong values surface as the matching `UNKNOWN_*` code instead); the index is globally empty (cold-start carve-out); the request was policy-gated below `minimumPrefixTokens` AND `affinityRouting: Disabled` (with `affinityRouting: Enabled` — the default — the same gate surfaces as `AFFINITY_HINT`); **every replica that held the prefix matched fewer tokens than `minimumMatchedTokens` (result-side per-replica floor, default 64) AND `affinityRouting: Disabled`** (with affinity Enabled the downgrade surfaces as `AFFINITY_HINT`); **the top per-replica score from the distinguishing-power-aware ranker fell below `routingFloorScore` (result-side score floor, default `0.1`) AND `affinityRouting: Disabled`** (with affinity Enabled the downgrade surfaces as `AFFINITY_HINT`); or an index-disabled state. | `replica_scores` **empty**. Not an error. | Route per the gateway's default policy (round-robin, least-loaded, …). The cache plane is invisible to this request. | | `POLICY_REQUIRES_CHAIN` | **shipped** | LookupRoute only | The tenant's `CachePolicy.spec.strategy.requireChain` is `true`, but the request did not carry a valid `block_hashes` + `block_token_counts` chain. The server returns before touching the index. This is a policy-gated empty result, separated from `NO_HINT` so operators can see legacy/exact callers hitting a chain-only namespace in `inferencecache_lookup_route_calls_total{reason_code="POLICY_REQUIRES_CHAIN"}`. | Empty `replica_scores`. | Treat as `NO_HINT`; update the caller to send the chain if it should benefit from the cache plane in this namespace. | -| `TENANT_HOT` | **shipped** | LookupRoute only | No exact prefix match for `(tenant, model, hash_scheme, adapter, prefix_hash)`, `CachePolicy.spec.strategy.enableTenantHot` is not `false`, and the tenant has at least one replica that (a) has reported stats recently (within ~2 minutes by default), (b) has a `hit_rate` above a small floor (default 0.2), AND (c) currently has **at least one prefix entry in the requested `(tenant, model, hash_scheme)` in the index** — proving the replica serves the requested engine domain. The "in the index" check is sweep-driven (an entry past TTL stays counted until the next sweep removes it), so for at most one sweep interval a recently-stale entry can briefly still satisfy the check; per soft-state semantics that yields at worst a soft hint that turns into a cache miss, never a wrong answer. A coarser locality signal than `PREFIX_MATCH` — useful when the prefix is novel but the tenant already has servers warm in the cache rotation. When the same fallback is found but the policy disables tenant-hot, the handler downgrades it to `NO_HINT`. | `replica_scores` non-empty (tenant-hot ranked); `matched_tokens` is **0** because there is no prefix overlap (the gateway must rely on `reason_code`, not `matched_tokens`, to recognize this branch). Shape otherwise unchanged. | Treat as a softer hint than `PREFIX_MATCH`; gateway free to use or ignore. | +| `TENANT_HOT` | **shipped** | LookupRoute only | No exact prefix match for `(tenant, model, hash_scheme, adapter, prefix_hash)`, `CachePolicy.spec.strategy.enableTenantHot` is not `false`, and the tenant has at least one replica that (a) has reported stats recently (within ~5 minutes by default), (b) has a `hit_rate` above a small floor (default 0.1), AND (c) currently has **at least one prefix entry in the requested `(tenant, model, hash_scheme)` in the index** — proving the replica serves the requested engine domain. The "in the index" check is sweep-driven (an entry past TTL stays counted until the next sweep removes it), so for at most one sweep interval a recently-stale entry can briefly still satisfy the check; per soft-state semantics that yields at worst a soft hint that turns into a cache miss, never a wrong answer. A coarser locality signal than `PREFIX_MATCH` — useful when the prefix is novel but the tenant already has servers warm in the cache rotation. When the same fallback is found but the policy disables tenant-hot, the handler downgrades it to `NO_HINT`. | `replica_scores` non-empty (tenant-hot ranked); `matched_tokens` is **0** because there is no prefix overlap (the gateway must rely on `reason_code`, not `matched_tokens`, to recognize this branch). Shape otherwise unchanged. | Treat as a softer hint than `PREFIX_MATCH`; gateway free to use or ignore. | | `TIMEOUT` | **shipped** | LookupRoute only | The lookup deadline expired before the index could rank — either the caller's context was already past its deadline on arrival or the per-tenant `CachePolicy.spec.lookupTimeoutMs` budget elapsed during the lookup. Gateway clients also synthesize this locally when *they* cancel a slow `LookupRoute` RPC. | Server: empty `replica_scores`. Client-side synth: same. | Treat as `NO_HINT`. | | `UNKNOWN_TENANT` | **shipped** | LookupRoute only | After a prefix miss (and the `TENANT_HOT` fallback when it applies — non-chain requests; chain requests skip `TENANT_HOT` by design and classify directly), AND the index is **not globally empty**: the request supplied a non-empty `tenant_id` and the index has **zero prefix entries for that tenant** across every model and hash scheme. **Cold-start carve-out:** a globally empty index (server just started, no `ReportCacheState` yet) stays on `NO_HINT` so a fresh deployment does not flood gateways with `UNKNOWN_TENANT`; the diagnostic resumes the moment any replica has reported state. Canonical asymmetric shape: a gateway-SDK querying with `tenant_id="default"` while the producer (kvevent-subscriber sidecar) is publishing under `tenant_id=$(POD_NAMESPACE)`. | Empty `replica_scores`. | Treat as `NO_HINT` for routing (still fail-open — the cache plane is hint-only); surface as a configuration error (log line / metric / SDK warning). **Do not retry under a different key** — the cache plane will not change between calls. | | `UNKNOWN_MODEL` | **shipped** | LookupRoute only | Same precondition as `UNKNOWN_TENANT` above. The tenant is known but the `(tenant_id, model_id)` pair has **zero entries**. The model has never served traffic in this tenant, or the model identifier disagrees between producer and consumer. | Empty `replica_scores`. | Same as `UNKNOWN_TENANT`: fail-open, surface as configuration error. | @@ -65,11 +65,11 @@ the cardinality factor and both floors still run. | Knob | What it does | Default | Off switch | |---|---|---|---| -| `PressureWeight` | Penalty applied to a replica's score from `ReplicaStats.pressure`: `pressure_factor = max(0, 1 - PressureWeight × pressure)`. Avoids blindly preferring a saturated cache holder over a fresher, lower-pressure peer. | `0.5` | `0` → no penalty | +| `PressureWeight` | Penalty applied to a replica's score from `ReplicaStats.pressure`: `pressure_factor = max(0, 1 - PressureWeight × pressure)`. Avoids blindly preferring a saturated cache holder over a fresher, lower-pressure peer. | `1.0` | `0` → no penalty | | `SLOTightTTFTMs` | TTFT budget (ms) below which the request is "tight" and the SLO bias kicks in. Uses `LookupRouteRequest.slo.ttft_ms`. | `200` | `0` → bias never fires | | `SLOTightBias` | Coefficient in the freshness boost: `slo_bias = 1 + freshness × SLOTightBias` when the request is tight. Higher → fresher candidates are favored more aggressively. | `1.0` | `0` → no boost | -| `TenantHotMinHitRate` | Minimum `hit_rate` for a replica to count as "warm" for the `TENANT_HOT` fallback. | `0.2` | n/a (use `TenantHotMaxAge = 0` to disable the fallback) | -| `TenantHotMaxAge` | Maximum stats age for a replica to count as "warm". | `2m` | `0` → fallback disabled (a prefix miss whose contract keys all populate the index lands at `NO_HINT`; mismatched-key misses still diagnose as `UNKNOWN_*` via the miss-classifier) | +| `TenantHotMinHitRate` | Minimum `hit_rate` for a replica to count as "warm" for the `TENANT_HOT` fallback. | `0.1` | n/a (use `TenantHotMaxAge = 0` to disable the fallback) | +| `TenantHotMaxAge` | Maximum stats age for a replica to count as "warm". | `5m` | `0` → fallback disabled (a prefix miss whose contract keys all populate the index lands at `NO_HINT`; mismatched-key misses still diagnose as `UNKNOWN_*` via the miss-classifier) | | `distinguishing_power` factor | Cardinality-aware multiplier: `1 − num_matching_replicas / total_replicas`, per-replica depth-aware for chain matches. Discounts overlaps every replica holds (chat-template framing, RAG corpus headers, custom system prompts). Always on for multi-replica deployments; degrades to `1.0` for single-replica deployments. See [`../design/lookuproute-ranking.md` §2.7](../design/lookuproute-ranking.md#27-the-replica-distinguishing-power-factor). | always on for multi-replica; `1.0` for single-replica | none (operators disable the *floor* it feeds via `CachePolicy.spec.routingFloorScore: "0"`, not the factor itself) | | `CachePolicy.spec.minimumMatchedTokens` | Per-replica matched-tokens floor: filters replicas whose realized `matched_tokens` falls below the threshold. If no replica survives, the response downgrades to `StrategyNone`, which surfaces as `AFFINITY_HINT` under `affinityRouting: Enabled` (the default) with a usable seed + serving replica or as `NO_HINT` under `affinityRouting: Disabled`. | `64` (4 KV blocks) | `0` on the CR → opt-out for that namespace | | `CachePolicy.spec.routingFloorScore` | Per-response score floor on the top surviving replica's score (after the distinguishing-power factor multiplies in). Below the floor → response downgrades to `StrategyNone`, with the same `AFFINITY_HINT` vs `NO_HINT` split as the matched-tokens row above. | `"0.1"` | `"0"` on the CR → opt-out for that namespace | diff --git a/internal/index/calibration/README.md b/internal/index/calibration/README.md index d3e44def..dbf4aff5 100644 --- a/internal/index/calibration/README.md +++ b/internal/index/calibration/README.md @@ -33,7 +33,9 @@ The checked-in fixture is intentionally synthetic because no production C1 trace is available in this repository. It provides deterministic boundary coverage and proves the calibration pipeline, but it should be replaced or supplemented with a sanitized captured trace before treating the coefficients -as a production benchmark conclusion. +as a production benchmark conclusion. Its selected tuple is therefore a +candidate only and does not change `DefaultRankerConfig`; production defaults +remain stable until representative captured data supports a retune. ## Reproduce diff --git a/internal/index/calibration/calibration_test.go b/internal/index/calibration/calibration_test.go index ae72d330..e706c7e1 100644 --- a/internal/index/calibration/calibration_test.go +++ b/internal/index/calibration/calibration_test.go @@ -11,11 +11,9 @@ import ( "strings" "testing" "time" - - "github.com/cachebox-project/inference-cache/internal/index" ) -func TestCheckedInTraceSelectsDefaultConfigAndCurrentResult(t *testing.T) { +func TestCheckedInSyntheticTraceSelectsCandidateAndCurrentResult(t *testing.T) { traceFile, err := os.Open("testdata/c1_synthetic_trace.json") if err != nil { t.Fatalf("open trace: %v", err) @@ -28,14 +26,18 @@ func TestCheckedInTraceSelectsDefaultConfigAndCurrentResult(t *testing.T) { t.Fatalf("Load: %v", err) } result := Calibrate(trace) - defaults := index.DefaultRankerConfig() - best := result.BestConfig - if float32(best.PressureWeight) != defaults.PressureWeight || - best.SLOTightTTFTMillis != defaults.SLOTightTTFTMs || - float32(best.SLOTightBias) != defaults.SLOTightBias || - float32(best.TenantHotMinHitRate) != defaults.TenantHotMinHitRate || - time.Duration(best.TenantHotMaxAgeMillis)*time.Millisecond != defaults.TenantHotMaxAge { - t.Fatalf("calibrated config = %+v, DefaultRankerConfig = %+v", best, defaults) + if trace.Provenance.Kind != "synthetic" { + t.Fatalf("provenance kind = %q, want synthetic", trace.Provenance.Kind) + } + want := Config{ + PressureWeight: 0.5, + SLOTightTTFTMillis: 200, + SLOTightBias: 1, + TenantHotMinHitRate: 0.2, + TenantHotMaxAgeMillis: 120_000, + } + if result.BestConfig != want { + t.Fatalf("synthetic candidate = %+v, want %+v", result.BestConfig, want) } if result.BestMetrics.PrefixHitRatePct != 100 || result.BestMetrics.TenantHotHitRatePct != 100 { t.Fatalf("best metrics = %+v, want both fixture hit rates at 100%%", result.BestMetrics) diff --git a/internal/index/ranking.go b/internal/index/ranking.go index da461706..8ffd624a 100644 --- a/internal/index/ranking.go +++ b/internal/index/ranking.go @@ -14,9 +14,9 @@ import ( // when no stats are present and no SLO hint is set — see DefaultRankerConfig. const ( // Pressure penalty: pressureFactor = 1 - PressureWeight × pressure. - // 0.5 keeps locality meaningful while letting a lower-pressure peer win - // when the reported token advantage is modest. - DefaultPressureWeight = 0.5 + // 1.0 → a fully-saturated replica (pressure=1.0) drops to score 0, so a + // fresher lower-pressure peer can win. Lower values are gentler. + DefaultPressureWeight = 1.0 // TTFT below this (ms) is treated as "tight" — the SLO bias kicks in. // 200 ms is a conservative threshold; tune per workload. DefaultSLOTightTTFTMs = 200 @@ -25,10 +25,10 @@ const ( // against matched-token count when latency is critical. DefaultSLOTightBias = 1.0 // TENANT_HOT fallback: replicas with hit_rate >= this count as "warm". - DefaultTenantHotMinHitRate = 0.2 + DefaultTenantHotMinHitRate = 0.1 // TENANT_HOT fallback: stats lastSeen within this window count as // "recent" — anything older is treated as cold for the fallback. - DefaultTenantHotMaxAge = 2 * time.Minute + DefaultTenantHotMaxAge = 5 * time.Minute ) // applyChainDistinguishingPower folds the depth-aware distinguishing-power diff --git a/internal/index/ranking_test.go b/internal/index/ranking_test.go index ceca4c70..e4d7c6e5 100644 --- a/internal/index/ranking_test.go +++ b/internal/index/ranking_test.go @@ -9,17 +9,17 @@ import ( "time" ) -func TestDefaultRankerConfigMatchesCalibratedTuple(t *testing.T) { +func TestDefaultRankerConfigMatchesStableProductionTuple(t *testing.T) { got := DefaultRankerConfig() want := RankerConfig{ - PressureWeight: 0.5, + PressureWeight: 1, SLOTightTTFTMs: 200, SLOTightBias: 1, - TenantHotMinHitRate: 0.2, - TenantHotMaxAge: 2 * time.Minute, + TenantHotMinHitRate: 0.1, + TenantHotMaxAge: 5 * time.Minute, } if got != want { - t.Fatalf("DefaultRankerConfig() = %+v, want calibrated tuple %+v", got, want) + t.Fatalf("DefaultRankerConfig() = %+v, want stable production tuple %+v", got, want) } } diff --git a/site/content/en/docs/reference/reason-codes.md b/site/content/en/docs/reference/reason-codes.md index 56bb4334..4a032c32 100644 --- a/site/content/en/docs/reference/reason-codes.md +++ b/site/content/en/docs/reference/reason-codes.md @@ -44,11 +44,11 @@ See [LookupRoute & ranking]({{< relref "/docs/concepts/lookuproute/#diagnostics- | `strategy.requireChain` | CachePolicy | false | (n/a) | | `strategy.enableTenantHot` | CachePolicy | true | false | | `affinityRouting` | CachePolicy | `Enabled` | `Disabled` | -| `PressureWeight` | server RankerConfig | 0.5 | 0 | +| `PressureWeight` | server RankerConfig | 1.0 | 0 | | `SLOTightTTFTMs` | server RankerConfig | 200ms | 0 | | `SLOTightBias` | server RankerConfig | 1.0 | 0 | -| `TenantHotMaxAge` | server RankerConfig | 2m | 0 | -| `TenantHotMinHitRate` | server RankerConfig | 0.2 | — | +| `TenantHotMaxAge` | server RankerConfig | 5m | 0 | +| `TenantHotMinHitRate` | server RankerConfig | 0.1 | — | ## RenderTemplate From d30cc3aee07599cf03cf170671e37ca7866e1b88 Mon Sep 17 00:00:00 2001 From: Weiwei Zheng Date: Wed, 19 Aug 2026 17:44:18 -0700 Subject: [PATCH 5/8] Harden ranker calibration replay Signed-off-by: Weiwei Zheng --- hack/ranker-calibration/main.go | 79 +++++++++--- hack/ranker-calibration/main_test.go | 75 ++++++++++++ internal/index/calibration/README.md | 10 +- internal/index/calibration/calibration.go | 39 ++++-- .../index/calibration/calibration_test.go | 62 ++++++++-- .../testdata/c1_synthetic_trace.json | 112 +++++++++--------- internal/index/index.go | 8 +- internal/index/ingest_test.go | 9 ++ 8 files changed, 291 insertions(+), 103 deletions(-) create mode 100644 hack/ranker-calibration/main_test.go diff --git a/hack/ranker-calibration/main.go b/hack/ranker-calibration/main.go index 4c43a5eb..4a0e677a 100644 --- a/hack/ranker-calibration/main.go +++ b/hack/ranker-calibration/main.go @@ -8,55 +8,96 @@ import ( "bytes" "flag" "fmt" + "io" "os" + "path/filepath" "github.com/cachebox-project/inference-cache/internal/index/calibration" ) func main() { - tracePath := flag.String("trace", "", "path to a ranker calibration trace") - outPath := flag.String("out", "", "path to write the calibration result") - check := flag.Bool("check", false, "verify that -out already matches the generated result") - flag.Parse() + os.Exit(run(os.Args[1:], os.Stdout, os.Stderr)) +} + +func run(args []string, stdout, stderr io.Writer) int { + flags := flag.NewFlagSet("ranker-calibration", flag.ContinueOnError) + flags.SetOutput(stderr) + tracePath := flags.String("trace", "", "path to a ranker calibration trace") + outPath := flags.String("out", "", "path to write the calibration result") + check := flags.Bool("check", false, "verify that -out already matches the generated result") + if err := flags.Parse(args); err != nil { + return 2 + } if *tracePath == "" || *outPath == "" { - fatalf("both -trace and -out are required") + return failf(stderr, "both -trace and -out are required") } traceFile, err := os.Open(*tracePath) if err != nil { - fatalf("open trace: %v", err) + return failf(stderr, "open trace: %v", err) } trace, err := calibration.Load(traceFile) closeErr := traceFile.Close() if err != nil { - fatalf("load trace: %v", err) + return failf(stderr, "load trace: %v", err) } if closeErr != nil { - fatalf("close trace: %v", closeErr) + return failf(stderr, "close trace: %v", closeErr) } data, err := calibration.MarshalResult(calibration.Calibrate(trace)) if err != nil { - fatalf("render result: %v", err) + return failf(stderr, "render result: %v", err) } if *check { current, err := os.ReadFile(*outPath) if err != nil { - fatalf("read result for check: %v", err) + return failf(stderr, "read result for check: %v", err) } if !bytes.Equal(current, data) { - fatalf("%s is stale; rerun ranker calibration", *outPath) + return failf(stderr, "%s is stale; rerun ranker calibration", *outPath) } - fmt.Printf("ranker calibration is current: %s\n", *outPath) - return + fmt.Fprintf(stdout, "ranker calibration is current: %s\n", *outPath) + return 0 + } + if err := writeAtomic(*outPath, data); err != nil { + return failf(stderr, "write result: %v", err) + } + fmt.Fprintf(stdout, "wrote ranker calibration: %s\n", *outPath) + return 0 +} + +func writeAtomic(path string, data []byte) error { + dir := filepath.Dir(path) + temp, err := os.CreateTemp(dir, "."+filepath.Base(path)+".tmp-*") + if err != nil { + return fmt.Errorf("create temporary result: %w", err) + } + tempPath := temp.Name() + defer func() { _ = os.Remove(tempPath) }() + + if err := temp.Chmod(0o644); err != nil { + _ = temp.Close() + return fmt.Errorf("set temporary result mode: %w", err) + } + if _, err := temp.Write(data); err != nil { + _ = temp.Close() + return fmt.Errorf("write temporary result: %w", err) + } + if err := temp.Sync(); err != nil { + _ = temp.Close() + return fmt.Errorf("sync temporary result: %w", err) + } + if err := temp.Close(); err != nil { + return fmt.Errorf("close temporary result: %w", err) } - if err := os.WriteFile(*outPath, data, 0o644); err != nil { - fatalf("write result: %v", err) + if err := os.Rename(tempPath, path); err != nil { + return fmt.Errorf("replace result: %w", err) } - fmt.Printf("wrote ranker calibration: %s\n", *outPath) + return nil } -func fatalf(format string, args ...any) { - fmt.Fprintf(os.Stderr, "ranker-calibration: "+format+"\n", args...) - os.Exit(1) +func failf(stderr io.Writer, format string, args ...any) int { + fmt.Fprintf(stderr, "ranker-calibration: "+format+"\n", args...) + return 1 } diff --git a/hack/ranker-calibration/main_test.go b/hack/ranker-calibration/main_test.go new file mode 100644 index 00000000..0531cd3b --- /dev/null +++ b/hack/ranker-calibration/main_test.go @@ -0,0 +1,75 @@ +// SPDX-FileCopyrightText: 2026 The inference-cache Authors +// +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "bytes" + "os" + "path/filepath" + "strings" + "testing" +) + +func TestRunGenerateAndCheck(t *testing.T) { + tracePath := filepath.Join("..", "..", "internal", "index", "calibration", "testdata", "c1_synthetic_trace.json") + outPath := filepath.Join(t.TempDir(), "result.json") + var stdout, stderr bytes.Buffer + + if code := run([]string{"-trace", tracePath, "-out", outPath}, &stdout, &stderr); code != 0 { + t.Fatalf("generate exit = %d, stderr = %q", code, stderr.String()) + } + if !strings.Contains(stdout.String(), "wrote ranker calibration") { + t.Fatalf("generate stdout = %q", stdout.String()) + } + if matches, err := filepath.Glob(filepath.Join(filepath.Dir(outPath), ".result.json.tmp-*")); err != nil || len(matches) != 0 { + t.Fatalf("temporary results after generation = %v, err = %v", matches, err) + } + + stdout.Reset() + stderr.Reset() + if code := run([]string{"-trace", tracePath, "-out", outPath, "-check"}, &stdout, &stderr); code != 0 { + t.Fatalf("current check exit = %d, stderr = %q", code, stderr.String()) + } + if !strings.Contains(stdout.String(), "ranker calibration is current") { + t.Fatalf("check stdout = %q", stdout.String()) + } + + if err := os.WriteFile(outPath, []byte("{}\n"), 0o644); err != nil { + t.Fatalf("write stale result: %v", err) + } + stdout.Reset() + stderr.Reset() + if code := run([]string{"-trace", tracePath, "-out", outPath, "-check"}, &stdout, &stderr); code != 1 { + t.Fatalf("stale check exit = %d, want 1", code) + } + if !strings.Contains(stderr.String(), "is stale") { + t.Fatalf("stale check stderr = %q", stderr.String()) + } +} + +func TestRunRejectsMalformedTrace(t *testing.T) { + dir := t.TempDir() + tracePath := filepath.Join(dir, "trace.json") + if err := os.WriteFile(tracePath, []byte("not-json"), 0o644); err != nil { + t.Fatalf("write malformed trace: %v", err) + } + var stdout, stderr bytes.Buffer + if code := run([]string{"-trace", tracePath, "-out", filepath.Join(dir, "result.json")}, &stdout, &stderr); code != 1 { + t.Fatalf("malformed trace exit = %d, want 1", code) + } + if !strings.Contains(stderr.String(), "load trace") { + t.Fatalf("malformed trace stderr = %q", stderr.String()) + } +} + +func TestRunRequiresPaths(t *testing.T) { + var stdout, stderr bytes.Buffer + if code := run(nil, &stdout, &stderr); code != 1 { + t.Fatalf("missing paths exit = %d, want 1", code) + } + if !strings.Contains(stderr.String(), "both -trace and -out are required") { + t.Fatalf("missing paths stderr = %q", stderr.String()) + } +} diff --git a/internal/index/calibration/README.md b/internal/index/calibration/README.md index dbf4aff5..34c9c7f1 100644 --- a/internal/index/calibration/README.md +++ b/internal/index/calibration/README.md @@ -17,10 +17,12 @@ gentler pressure/SLO multipliers and shorter fallback windows. Each observation is a self-contained point-in-time view. `reported_prefix` records whether the cache plane believed that replica held the requested -prefix; `matched_tokens`, `hit_rate`, `pressure`, and `reported_at_ms` are the -signals visible to the ranker. `observed_hit` is the later ground-truth outcome -for routing to that replica. Replicas without the requested prefix still receive -a unique serving-prefix entry during replay so `TENANT_HOT` can apply its real +prefix; `prefix_reported_at_ms` records that prefix observation's freshness, +while `stats_reported_at_ms` independently records when `hit_rate` and +`pressure` were reported. `matched_tokens` and those timestamps are the signals +visible to the ranker. `observed_hit` is the later ground-truth outcome for +routing to that replica. Replicas without the requested prefix still receive a +unique serving-prefix entry during replay so `TENANT_HOT` can apply its real engine-domain membership guard. Captured data should contain opaque or one-way prefix hashes only. Do not put diff --git a/internal/index/calibration/calibration.go b/internal/index/calibration/calibration.go index 531cf3fb..3ebcc475 100644 --- a/internal/index/calibration/calibration.go +++ b/internal/index/calibration/calibration.go @@ -20,6 +20,8 @@ import ( const SchemaVersion = 1 +const maxDurationMillis int64 = (1<<63 - 1) / int64(time.Millisecond) + const ( ObservationPrefix = "prefix" ObservationTenantHot = "tenant_hot" @@ -62,13 +64,14 @@ type Observation struct { } type ReplicaObservation struct { - ID string `json:"id"` - ReportedAtMillis int64 `json:"reported_at_ms"` - ReportedPrefix bool `json:"reported_prefix"` - MatchedTokens int32 `json:"matched_tokens"` - HitRate float32 `json:"hit_rate"` - Pressure float32 `json:"pressure"` - ObservedHit bool `json:"observed_hit"` + ID string `json:"id"` + PrefixReportedAtMillis int64 `json:"prefix_reported_at_ms"` + StatsReportedAtMillis int64 `json:"stats_reported_at_ms"` + ReportedPrefix bool `json:"reported_prefix"` + MatchedTokens int32 `json:"matched_tokens"` + HitRate float32 `json:"hit_rate"` + Pressure float32 `json:"pressure"` + ObservedHit bool `json:"observed_hit"` } type Config struct { @@ -137,6 +140,9 @@ func (t Trace) Validate() error { if t.TTLMillis <= 0 { return errors.New("ttl_ms must be positive") } + if t.TTLMillis > maxDurationMillis { + return fmt.Errorf("ttl_ms exceeds maximum representable duration (%d ms)", maxDurationMillis) + } if err := t.Sweep.validate(); err != nil { return err } @@ -181,6 +187,9 @@ func (s Sweep) validate() error { if value <= 0 { return fmt.Errorf("tenant_hot_max_age_ms contains non-positive value %d", value) } + if value > maxDurationMillis { + return fmt.Errorf("tenant_hot_max_age_ms contains value exceeding maximum representable duration: %d", value) + } } return nil } @@ -203,8 +212,11 @@ func (o Observation) validate() error { if replica.ID == "" { return fmt.Errorf("replica %d: id is required", i) } - if replica.ReportedAtMillis > o.AtMillis { - return fmt.Errorf("replica %q: reported_at_ms is after observation", replica.ID) + if replica.PrefixReportedAtMillis > o.AtMillis { + return fmt.Errorf("replica %q: prefix_reported_at_ms is after observation", replica.ID) + } + if replica.StatsReportedAtMillis > o.AtMillis { + return fmt.Errorf("replica %q: stats_reported_at_ms is after observation", replica.ID) } if replica.ReportedPrefix && replica.MatchedTokens <= 0 { return fmt.Errorf("replica %q: matched_tokens must be positive when reported_prefix is true", replica.ID) @@ -363,11 +375,18 @@ func replayObservation(trace Trace, observation Observation, config Config) bool Model: observation.Model, Tenant: observation.Tenant, HashScheme: observation.HashScheme, - Timestamp: time.UnixMilli(replica.ReportedAtMillis), + Timestamp: time.UnixMilli(replica.PrefixReportedAtMillis), Prefixes: []index.PrefixRef{{ PrefixHash: []byte(hash), TokenCount: tokens, }}, + }) + idx.Ingest(index.Update{ + ReplicaID: replica.ID, + Model: observation.Model, + Tenant: observation.Tenant, + HashScheme: observation.HashScheme, + Timestamp: time.UnixMilli(replica.StatsReportedAtMillis), Stats: &index.ReplicaStats{ HitRate: replica.HitRate, Pressure: replica.Pressure, diff --git a/internal/index/calibration/calibration_test.go b/internal/index/calibration/calibration_test.go index e706c7e1..0ca051fd 100644 --- a/internal/index/calibration/calibration_test.go +++ b/internal/index/calibration/calibration_test.go @@ -74,9 +74,9 @@ func TestCalibrateSeparatesKnobEffects(t *testing.T) { Tenant: "tenant-a", Model: "model-a", HashScheme: "vllm", PrefixHash: "p", TokenCount: 320, Replicas: []ReplicaObservation{ - {ID: "hot", ReportedAtMillis: 100_000, ReportedPrefix: true, MatchedTokens: 320, HitRate: 0.8, Pressure: 0.8}, - {ID: "cool", ReportedAtMillis: 100_000, ReportedPrefix: true, MatchedTokens: 256, HitRate: 0.4, Pressure: 0.1, ObservedHit: true}, - {ID: "decoy", ReportedAtMillis: 100_000, HitRate: 0.1}, + {ID: "hot", PrefixReportedAtMillis: 100_000, StatsReportedAtMillis: 100_000, ReportedPrefix: true, MatchedTokens: 320, HitRate: 0.8, Pressure: 0.8}, + {ID: "cool", PrefixReportedAtMillis: 100_000, StatsReportedAtMillis: 100_000, ReportedPrefix: true, MatchedTokens: 256, HitRate: 0.4, Pressure: 0.1, ObservedHit: true}, + {ID: "decoy", PrefixReportedAtMillis: 100_000, StatsReportedAtMillis: 100_000, HitRate: 0.1}, }, }, }, @@ -149,6 +149,7 @@ func TestTraceValidationRejectsInvalidFields(t *testing.T) { {"provenance kind", func(trace *Trace) { trace.Provenance.Kind = "unknown" }, "captured or synthetic"}, {"provenance source", func(trace *Trace) { trace.Provenance.Source = "" }, "provenance source"}, {"ttl", func(trace *Trace) { trace.TTLMillis = 0 }, "ttl_ms"}, + {"ttl overflow", func(trace *Trace) { trace.TTLMillis = maxDurationMillis + 1 }, "maximum representable"}, {"empty sweep", func(trace *Trace) { trace.Sweep.PressureWeights = nil }, "every sweep dimension"}, {"empty observations", func(trace *Trace) { trace.Observations = nil }, "at least one observation"}, {"invalid observation", func(trace *Trace) { trace.Observations[0].Kind = "unknown" }, "observation 0"}, @@ -183,6 +184,7 @@ func TestSweepValidationRejectsInvalidValues(t *testing.T) { {"hit rate", func(sweep *Sweep) { sweep.TenantHotMinHitRates[0] = 2 }, "invalid rate"}, {"ttft", func(sweep *Sweep) { sweep.SLOTightTTFTMillis[0] = 0 }, "non-positive"}, {"max age", func(sweep *Sweep) { sweep.TenantHotMaxAgeMillis[0] = 0 }, "non-positive"}, + {"max age overflow", func(sweep *Sweep) { sweep.TenantHotMaxAgeMillis[0] = maxDurationMillis + 1 }, "maximum representable"}, } { t.Run(tc.name, func(t *testing.T) { sweep := valid() @@ -241,7 +243,8 @@ func TestReplayTenantHotMissWithoutCandidate(t *testing.T) { ID: "tenant-hot", Kind: ObservationTenantHot, AtMillis: 100_000, Tenant: "t", Model: "m", HashScheme: "vllm", PrefixHash: "novel", TokenCount: 1, Replicas: []ReplicaObservation{{ - ID: "cold", ReportedAtMillis: 100_000, HitRate: 0.1, ObservedHit: true, + ID: "cold", PrefixReportedAtMillis: 100_000, StatsReportedAtMillis: 100_000, + HitRate: 0.1, ObservedHit: true, }}, } config := Config{TenantHotMinHitRate: 0.2, TenantHotMaxAgeMillis: 60_000} @@ -256,7 +259,8 @@ func TestReplayUsesObservationClockAtTenantHotBoundary(t *testing.T) { ID: "tenant-hot-boundary", Kind: ObservationTenantHot, AtMillis: 100_000, Tenant: "t", Model: "m", HashScheme: "vllm", PrefixHash: "novel", TokenCount: 1, Replicas: []ReplicaObservation{{ - ID: "warm", ReportedAtMillis: 40_001, HitRate: 0.8, ObservedHit: true, + ID: "warm", PrefixReportedAtMillis: 40_001, StatsReportedAtMillis: 40_001, + HitRate: 0.8, ObservedHit: true, }}, } config := Config{TenantHotMinHitRate: 0.2, TenantHotMaxAgeMillis: 60_000} @@ -265,15 +269,47 @@ func TestReplayUsesObservationClockAtTenantHotBoundary(t *testing.T) { } } -func TestObservationRejectsFutureReplicaReport(t *testing.T) { +func TestReplayUsesIndependentPrefixAndStatsTimestamps(t *testing.T) { + trace := Trace{TTLMillis: int64(time.Minute / time.Millisecond)} observation := Observation{ - ID: "future", Kind: ObservationPrefix, AtMillis: 10, - Tenant: "t", Model: "m", HashScheme: "vllm", PrefixHash: "p", TokenCount: 1, - Replicas: []ReplicaObservation{{ - ID: "r", ReportedAtMillis: 11, ReportedPrefix: true, MatchedTokens: 1, - }}, + ID: "independent-clocks", Kind: ObservationPrefix, AtMillis: 100_000, + Tenant: "t", Model: "m", HashScheme: "vllm", PrefixHash: "p", TokenCount: 320, + Replicas: []ReplicaObservation{ + { + ID: "deep-stale-stats", PrefixReportedAtMillis: 100_000, StatsReportedAtMillis: 0, + ReportedPrefix: true, MatchedTokens: 320, Pressure: 1, ObservedHit: true, + }, + { + ID: "shallow-fresh-stats", PrefixReportedAtMillis: 100_000, StatsReportedAtMillis: 100_000, + ReportedPrefix: true, MatchedTokens: 256, + }, + }, + } + config := Config{PressureWeight: 1, TenantHotMaxAgeMillis: 60_000} + if !replayObservation(trace, observation, config) { + t.Fatal("replayObservation = miss, want stale pressure ignored while fresh prefix remains routable") } - if err := observation.validate(); err == nil || !strings.Contains(err.Error(), "after observation") { - t.Fatalf("validate error = %v, want future-report rejection", err) +} + +func TestObservationRejectsFutureReplicaReport(t *testing.T) { + for _, tc := range []struct { + name string + mutate func(*ReplicaObservation) + want string + }{ + {"prefix", func(replica *ReplicaObservation) { replica.PrefixReportedAtMillis = 11 }, "prefix_reported_at_ms"}, + {"stats", func(replica *ReplicaObservation) { replica.StatsReportedAtMillis = 11 }, "stats_reported_at_ms"}, + } { + t.Run(tc.name, func(t *testing.T) { + observation := Observation{ + ID: "future", Kind: ObservationPrefix, AtMillis: 10, + Tenant: "t", Model: "m", HashScheme: "vllm", PrefixHash: "p", TokenCount: 1, + Replicas: []ReplicaObservation{{ID: "r", ReportedPrefix: true, MatchedTokens: 1}}, + } + tc.mutate(&observation.Replicas[0]) + if err := observation.validate(); err == nil || !strings.Contains(err.Error(), tc.want) { + t.Fatalf("validate error = %v, want substring %q", err, tc.want) + } + }) } } diff --git a/internal/index/calibration/testdata/c1_synthetic_trace.json b/internal/index/calibration/testdata/c1_synthetic_trace.json index a3fb18e1..34e4cf18 100644 --- a/internal/index/calibration/testdata/c1_synthetic_trace.json +++ b/internal/index/calibration/testdata/c1_synthetic_trace.json @@ -25,9 +25,9 @@ "prefix_hash": "pressure-shed-1", "token_count": 320, "replicas": [ - {"id": "a-hot", "reported_at_ms": 100000, "reported_prefix": true, "matched_tokens": 320, "hit_rate": 0.7, "pressure": 0.8, "observed_hit": false}, - {"id": "b-cool", "reported_at_ms": 100000, "reported_prefix": true, "matched_tokens": 256, "hit_rate": 0.4, "pressure": 0.1, "observed_hit": true}, - {"id": "z-decoy", "reported_at_ms": 100000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.1, "pressure": 0, "observed_hit": false} + {"id": "a-hot", "prefix_reported_at_ms": 100000, "stats_reported_at_ms": 100000, "reported_prefix": true, "matched_tokens": 320, "hit_rate": 0.7, "pressure": 0.8, "observed_hit": false}, + {"id": "b-cool", "prefix_reported_at_ms": 100000, "stats_reported_at_ms": 100000, "reported_prefix": true, "matched_tokens": 256, "hit_rate": 0.4, "pressure": 0.1, "observed_hit": true}, + {"id": "z-decoy", "prefix_reported_at_ms": 100000, "stats_reported_at_ms": 100000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.1, "pressure": 0, "observed_hit": false} ] }, { @@ -40,9 +40,9 @@ "prefix_hash": "pressure-shed-2", "token_count": 320, "replicas": [ - {"id": "a-hot", "reported_at_ms": 200000, "reported_prefix": true, "matched_tokens": 320, "hit_rate": 0.7, "pressure": 0.8, "observed_hit": false}, - {"id": "b-cool", "reported_at_ms": 200000, "reported_prefix": true, "matched_tokens": 256, "hit_rate": 0.4, "pressure": 0.1, "observed_hit": true}, - {"id": "z-decoy", "reported_at_ms": 200000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.1, "pressure": 0, "observed_hit": false} + {"id": "a-hot", "prefix_reported_at_ms": 200000, "stats_reported_at_ms": 200000, "reported_prefix": true, "matched_tokens": 320, "hit_rate": 0.7, "pressure": 0.8, "observed_hit": false}, + {"id": "b-cool", "prefix_reported_at_ms": 200000, "stats_reported_at_ms": 200000, "reported_prefix": true, "matched_tokens": 256, "hit_rate": 0.4, "pressure": 0.1, "observed_hit": true}, + {"id": "z-decoy", "prefix_reported_at_ms": 200000, "stats_reported_at_ms": 200000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.1, "pressure": 0, "observed_hit": false} ] }, { @@ -55,9 +55,9 @@ "prefix_hash": "pressure-shed-3", "token_count": 320, "replicas": [ - {"id": "a-hot", "reported_at_ms": 300000, "reported_prefix": true, "matched_tokens": 320, "hit_rate": 0.65, "pressure": 0.8, "observed_hit": false}, - {"id": "b-cool", "reported_at_ms": 300000, "reported_prefix": true, "matched_tokens": 256, "hit_rate": 0.45, "pressure": 0.1, "observed_hit": true}, - {"id": "z-decoy", "reported_at_ms": 300000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.1, "pressure": 0, "observed_hit": false} + {"id": "a-hot", "prefix_reported_at_ms": 300000, "stats_reported_at_ms": 300000, "reported_prefix": true, "matched_tokens": 320, "hit_rate": 0.65, "pressure": 0.8, "observed_hit": false}, + {"id": "b-cool", "prefix_reported_at_ms": 300000, "stats_reported_at_ms": 300000, "reported_prefix": true, "matched_tokens": 256, "hit_rate": 0.45, "pressure": 0.1, "observed_hit": true}, + {"id": "z-decoy", "prefix_reported_at_ms": 300000, "stats_reported_at_ms": 300000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.1, "pressure": 0, "observed_hit": false} ] }, { @@ -70,9 +70,9 @@ "prefix_hash": "pressure-preserve-1", "token_count": 512, "replicas": [ - {"id": "a-local", "reported_at_ms": 400000, "reported_prefix": true, "matched_tokens": 512, "hit_rate": 0.75, "pressure": 0.7, "observed_hit": true}, - {"id": "b-cool", "reported_at_ms": 400000, "reported_prefix": true, "matched_tokens": 256, "hit_rate": 0.3, "pressure": 0, "observed_hit": false}, - {"id": "z-decoy", "reported_at_ms": 400000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.1, "pressure": 0, "observed_hit": false} + {"id": "a-local", "prefix_reported_at_ms": 400000, "stats_reported_at_ms": 400000, "reported_prefix": true, "matched_tokens": 512, "hit_rate": 0.75, "pressure": 0.7, "observed_hit": true}, + {"id": "b-cool", "prefix_reported_at_ms": 400000, "stats_reported_at_ms": 400000, "reported_prefix": true, "matched_tokens": 256, "hit_rate": 0.3, "pressure": 0, "observed_hit": false}, + {"id": "z-decoy", "prefix_reported_at_ms": 400000, "stats_reported_at_ms": 400000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.1, "pressure": 0, "observed_hit": false} ] }, { @@ -85,9 +85,9 @@ "prefix_hash": "pressure-preserve-2", "token_count": 512, "replicas": [ - {"id": "a-local", "reported_at_ms": 500000, "reported_prefix": true, "matched_tokens": 512, "hit_rate": 0.75, "pressure": 0.7, "observed_hit": true}, - {"id": "b-cool", "reported_at_ms": 500000, "reported_prefix": true, "matched_tokens": 256, "hit_rate": 0.3, "pressure": 0, "observed_hit": false}, - {"id": "z-decoy", "reported_at_ms": 500000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.1, "pressure": 0, "observed_hit": false} + {"id": "a-local", "prefix_reported_at_ms": 500000, "stats_reported_at_ms": 500000, "reported_prefix": true, "matched_tokens": 512, "hit_rate": 0.75, "pressure": 0.7, "observed_hit": true}, + {"id": "b-cool", "prefix_reported_at_ms": 500000, "stats_reported_at_ms": 500000, "reported_prefix": true, "matched_tokens": 256, "hit_rate": 0.3, "pressure": 0, "observed_hit": false}, + {"id": "z-decoy", "prefix_reported_at_ms": 500000, "stats_reported_at_ms": 500000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.1, "pressure": 0, "observed_hit": false} ] }, { @@ -100,9 +100,9 @@ "prefix_hash": "pressure-preserve-3", "token_count": 512, "replicas": [ - {"id": "a-local", "reported_at_ms": 600000, "reported_prefix": true, "matched_tokens": 512, "hit_rate": 0.7, "pressure": 0.7, "observed_hit": true}, - {"id": "b-cool", "reported_at_ms": 600000, "reported_prefix": true, "matched_tokens": 256, "hit_rate": 0.35, "pressure": 0, "observed_hit": false}, - {"id": "z-decoy", "reported_at_ms": 600000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.1, "pressure": 0, "observed_hit": false} + {"id": "a-local", "prefix_reported_at_ms": 600000, "stats_reported_at_ms": 600000, "reported_prefix": true, "matched_tokens": 512, "hit_rate": 0.7, "pressure": 0.7, "observed_hit": true}, + {"id": "b-cool", "prefix_reported_at_ms": 600000, "stats_reported_at_ms": 600000, "reported_prefix": true, "matched_tokens": 256, "hit_rate": 0.35, "pressure": 0, "observed_hit": false}, + {"id": "z-decoy", "prefix_reported_at_ms": 600000, "stats_reported_at_ms": 600000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.1, "pressure": 0, "observed_hit": false} ] }, { @@ -116,9 +116,9 @@ "token_count": 512, "ttft_budget_ms": 120, "replicas": [ - {"id": "a-old", "reported_at_ms": 160000, "reported_prefix": true, "matched_tokens": 512, "hit_rate": 0.5, "pressure": 0, "observed_hit": false}, - {"id": "b-fresh", "reported_at_ms": 700000, "reported_prefix": true, "matched_tokens": 320, "hit_rate": 0.5, "pressure": 0, "observed_hit": true}, - {"id": "z-decoy", "reported_at_ms": 700000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.1, "pressure": 0, "observed_hit": false} + {"id": "a-old", "prefix_reported_at_ms": 160000, "stats_reported_at_ms": 160000, "reported_prefix": true, "matched_tokens": 512, "hit_rate": 0.5, "pressure": 0, "observed_hit": false}, + {"id": "b-fresh", "prefix_reported_at_ms": 700000, "stats_reported_at_ms": 700000, "reported_prefix": true, "matched_tokens": 320, "hit_rate": 0.5, "pressure": 0, "observed_hit": true}, + {"id": "z-decoy", "prefix_reported_at_ms": 700000, "stats_reported_at_ms": 700000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.1, "pressure": 0, "observed_hit": false} ] }, { @@ -132,9 +132,9 @@ "token_count": 512, "ttft_budget_ms": 120, "replicas": [ - {"id": "a-old", "reported_at_ms": 260000, "reported_prefix": true, "matched_tokens": 512, "hit_rate": 0.5, "pressure": 0, "observed_hit": false}, - {"id": "b-fresh", "reported_at_ms": 800000, "reported_prefix": true, "matched_tokens": 320, "hit_rate": 0.5, "pressure": 0, "observed_hit": true}, - {"id": "z-decoy", "reported_at_ms": 800000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.1, "pressure": 0, "observed_hit": false} + {"id": "a-old", "prefix_reported_at_ms": 260000, "stats_reported_at_ms": 260000, "reported_prefix": true, "matched_tokens": 512, "hit_rate": 0.5, "pressure": 0, "observed_hit": false}, + {"id": "b-fresh", "prefix_reported_at_ms": 800000, "stats_reported_at_ms": 800000, "reported_prefix": true, "matched_tokens": 320, "hit_rate": 0.5, "pressure": 0, "observed_hit": true}, + {"id": "z-decoy", "prefix_reported_at_ms": 800000, "stats_reported_at_ms": 800000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.1, "pressure": 0, "observed_hit": false} ] }, { @@ -148,9 +148,9 @@ "token_count": 512, "ttft_budget_ms": 180, "replicas": [ - {"id": "a-old", "reported_at_ms": 360000, "reported_prefix": true, "matched_tokens": 512, "hit_rate": 0.5, "pressure": 0, "observed_hit": false}, - {"id": "b-fresh", "reported_at_ms": 900000, "reported_prefix": true, "matched_tokens": 320, "hit_rate": 0.5, "pressure": 0, "observed_hit": true}, - {"id": "z-decoy", "reported_at_ms": 900000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.1, "pressure": 0, "observed_hit": false} + {"id": "a-old", "prefix_reported_at_ms": 360000, "stats_reported_at_ms": 360000, "reported_prefix": true, "matched_tokens": 512, "hit_rate": 0.5, "pressure": 0, "observed_hit": false}, + {"id": "b-fresh", "prefix_reported_at_ms": 900000, "stats_reported_at_ms": 900000, "reported_prefix": true, "matched_tokens": 320, "hit_rate": 0.5, "pressure": 0, "observed_hit": true}, + {"id": "z-decoy", "prefix_reported_at_ms": 900000, "stats_reported_at_ms": 900000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.1, "pressure": 0, "observed_hit": false} ] }, { @@ -164,9 +164,9 @@ "token_count": 512, "ttft_budget_ms": 180, "replicas": [ - {"id": "a-old", "reported_at_ms": 460000, "reported_prefix": true, "matched_tokens": 512, "hit_rate": 0.5, "pressure": 0, "observed_hit": false}, - {"id": "b-fresh", "reported_at_ms": 1000000, "reported_prefix": true, "matched_tokens": 320, "hit_rate": 0.5, "pressure": 0, "observed_hit": true}, - {"id": "z-decoy", "reported_at_ms": 1000000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.1, "pressure": 0, "observed_hit": false} + {"id": "a-old", "prefix_reported_at_ms": 460000, "stats_reported_at_ms": 460000, "reported_prefix": true, "matched_tokens": 512, "hit_rate": 0.5, "pressure": 0, "observed_hit": false}, + {"id": "b-fresh", "prefix_reported_at_ms": 1000000, "stats_reported_at_ms": 1000000, "reported_prefix": true, "matched_tokens": 320, "hit_rate": 0.5, "pressure": 0, "observed_hit": true}, + {"id": "z-decoy", "prefix_reported_at_ms": 1000000, "stats_reported_at_ms": 1000000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.1, "pressure": 0, "observed_hit": false} ] }, { @@ -180,9 +180,9 @@ "token_count": 550, "ttft_budget_ms": 120, "replicas": [ - {"id": "a-deep", "reported_at_ms": 560000, "reported_prefix": true, "matched_tokens": 550, "hit_rate": 0.6, "pressure": 0, "observed_hit": true}, - {"id": "b-fresh", "reported_at_ms": 1100000, "reported_prefix": true, "matched_tokens": 320, "hit_rate": 0.5, "pressure": 0, "observed_hit": false}, - {"id": "z-decoy", "reported_at_ms": 1100000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.1, "pressure": 0, "observed_hit": false} + {"id": "a-deep", "prefix_reported_at_ms": 560000, "stats_reported_at_ms": 560000, "reported_prefix": true, "matched_tokens": 550, "hit_rate": 0.6, "pressure": 0, "observed_hit": true}, + {"id": "b-fresh", "prefix_reported_at_ms": 1100000, "stats_reported_at_ms": 1100000, "reported_prefix": true, "matched_tokens": 320, "hit_rate": 0.5, "pressure": 0, "observed_hit": false}, + {"id": "z-decoy", "prefix_reported_at_ms": 1100000, "stats_reported_at_ms": 1100000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.1, "pressure": 0, "observed_hit": false} ] }, { @@ -196,9 +196,9 @@ "token_count": 550, "ttft_budget_ms": 120, "replicas": [ - {"id": "a-deep", "reported_at_ms": 660000, "reported_prefix": true, "matched_tokens": 550, "hit_rate": 0.6, "pressure": 0, "observed_hit": true}, - {"id": "b-fresh", "reported_at_ms": 1200000, "reported_prefix": true, "matched_tokens": 320, "hit_rate": 0.5, "pressure": 0, "observed_hit": false}, - {"id": "z-decoy", "reported_at_ms": 1200000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.1, "pressure": 0, "observed_hit": false} + {"id": "a-deep", "prefix_reported_at_ms": 660000, "stats_reported_at_ms": 660000, "reported_prefix": true, "matched_tokens": 550, "hit_rate": 0.6, "pressure": 0, "observed_hit": true}, + {"id": "b-fresh", "prefix_reported_at_ms": 1200000, "stats_reported_at_ms": 1200000, "reported_prefix": true, "matched_tokens": 320, "hit_rate": 0.5, "pressure": 0, "observed_hit": false}, + {"id": "z-decoy", "prefix_reported_at_ms": 1200000, "stats_reported_at_ms": 1200000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.1, "pressure": 0, "observed_hit": false} ] }, { @@ -212,9 +212,9 @@ "token_count": 512, "ttft_budget_ms": 220, "replicas": [ - {"id": "a-deep", "reported_at_ms": 760000, "reported_prefix": true, "matched_tokens": 512, "hit_rate": 0.6, "pressure": 0, "observed_hit": true}, - {"id": "b-fresh", "reported_at_ms": 1300000, "reported_prefix": true, "matched_tokens": 320, "hit_rate": 0.5, "pressure": 0, "observed_hit": false}, - {"id": "z-decoy", "reported_at_ms": 1300000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.1, "pressure": 0, "observed_hit": false} + {"id": "a-deep", "prefix_reported_at_ms": 760000, "stats_reported_at_ms": 760000, "reported_prefix": true, "matched_tokens": 512, "hit_rate": 0.6, "pressure": 0, "observed_hit": true}, + {"id": "b-fresh", "prefix_reported_at_ms": 1300000, "stats_reported_at_ms": 1300000, "reported_prefix": true, "matched_tokens": 320, "hit_rate": 0.5, "pressure": 0, "observed_hit": false}, + {"id": "z-decoy", "prefix_reported_at_ms": 1300000, "stats_reported_at_ms": 1300000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.1, "pressure": 0, "observed_hit": false} ] }, { @@ -228,9 +228,9 @@ "token_count": 512, "ttft_budget_ms": 220, "replicas": [ - {"id": "a-deep", "reported_at_ms": 860000, "reported_prefix": true, "matched_tokens": 512, "hit_rate": 0.6, "pressure": 0, "observed_hit": true}, - {"id": "b-fresh", "reported_at_ms": 1400000, "reported_prefix": true, "matched_tokens": 320, "hit_rate": 0.5, "pressure": 0, "observed_hit": false}, - {"id": "z-decoy", "reported_at_ms": 1400000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.1, "pressure": 0, "observed_hit": false} + {"id": "a-deep", "prefix_reported_at_ms": 860000, "stats_reported_at_ms": 860000, "reported_prefix": true, "matched_tokens": 512, "hit_rate": 0.6, "pressure": 0, "observed_hit": true}, + {"id": "b-fresh", "prefix_reported_at_ms": 1400000, "stats_reported_at_ms": 1400000, "reported_prefix": true, "matched_tokens": 320, "hit_rate": 0.5, "pressure": 0, "observed_hit": false}, + {"id": "z-decoy", "prefix_reported_at_ms": 1400000, "stats_reported_at_ms": 1400000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.1, "pressure": 0, "observed_hit": false} ] }, { @@ -243,8 +243,8 @@ "prefix_hash": "tenant-rate-1", "token_count": 64, "replicas": [ - {"id": "a-noisy", "reported_at_ms": 1500000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.19, "pressure": 0, "observed_hit": false}, - {"id": "z-warm", "reported_at_ms": 1470000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.25, "pressure": 0, "observed_hit": true} + {"id": "a-noisy", "prefix_reported_at_ms": 1500000, "stats_reported_at_ms": 1500000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.19, "pressure": 0, "observed_hit": false}, + {"id": "z-warm", "prefix_reported_at_ms": 1470000, "stats_reported_at_ms": 1470000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.25, "pressure": 0, "observed_hit": true} ] }, { @@ -257,8 +257,8 @@ "prefix_hash": "tenant-rate-2", "token_count": 64, "replicas": [ - {"id": "a-noisy", "reported_at_ms": 1600000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.19, "pressure": 0, "observed_hit": false}, - {"id": "z-warm", "reported_at_ms": 1570000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.25, "pressure": 0, "observed_hit": true} + {"id": "a-noisy", "prefix_reported_at_ms": 1600000, "stats_reported_at_ms": 1600000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.19, "pressure": 0, "observed_hit": false}, + {"id": "z-warm", "prefix_reported_at_ms": 1570000, "stats_reported_at_ms": 1570000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.25, "pressure": 0, "observed_hit": true} ] }, { @@ -271,8 +271,8 @@ "prefix_hash": "tenant-rate-3", "token_count": 64, "replicas": [ - {"id": "a-noisy", "reported_at_ms": 1700000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.19, "pressure": 0, "observed_hit": false}, - {"id": "z-warm", "reported_at_ms": 1670000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.25, "pressure": 0, "observed_hit": true} + {"id": "a-noisy", "prefix_reported_at_ms": 1700000, "stats_reported_at_ms": 1700000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.19, "pressure": 0, "observed_hit": false}, + {"id": "z-warm", "prefix_reported_at_ms": 1670000, "stats_reported_at_ms": 1670000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.25, "pressure": 0, "observed_hit": true} ] }, { @@ -285,7 +285,7 @@ "prefix_hash": "tenant-age-preserve-1", "token_count": 64, "replicas": [ - {"id": "a-moderate", "reported_at_ms": 1725000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.3, "pressure": 0, "observed_hit": true} + {"id": "a-moderate", "prefix_reported_at_ms": 1725000, "stats_reported_at_ms": 1725000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.3, "pressure": 0, "observed_hit": true} ] }, { @@ -298,7 +298,7 @@ "prefix_hash": "tenant-age-preserve-2", "token_count": 64, "replicas": [ - {"id": "a-moderate", "reported_at_ms": 1825000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.3, "pressure": 0, "observed_hit": true} + {"id": "a-moderate", "prefix_reported_at_ms": 1825000, "stats_reported_at_ms": 1825000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.3, "pressure": 0, "observed_hit": true} ] }, { @@ -311,8 +311,8 @@ "prefix_hash": "tenant-age-expire-1", "token_count": 64, "replicas": [ - {"id": "a-stale", "reported_at_ms": 1850000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.8, "pressure": 0, "observed_hit": false}, - {"id": "z-recent", "reported_at_ms": 1970000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.3, "pressure": 0, "observed_hit": true} + {"id": "a-stale", "prefix_reported_at_ms": 1850000, "stats_reported_at_ms": 1850000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.8, "pressure": 0, "observed_hit": false}, + {"id": "z-recent", "prefix_reported_at_ms": 1970000, "stats_reported_at_ms": 1970000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.3, "pressure": 0, "observed_hit": true} ] }, { @@ -325,8 +325,8 @@ "prefix_hash": "tenant-age-expire-2", "token_count": 64, "replicas": [ - {"id": "a-stale", "reported_at_ms": 1950000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.8, "pressure": 0, "observed_hit": false}, - {"id": "z-recent", "reported_at_ms": 2070000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.3, "pressure": 0, "observed_hit": true} + {"id": "a-stale", "prefix_reported_at_ms": 1950000, "stats_reported_at_ms": 1950000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.8, "pressure": 0, "observed_hit": false}, + {"id": "z-recent", "prefix_reported_at_ms": 2070000, "stats_reported_at_ms": 2070000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.3, "pressure": 0, "observed_hit": true} ] }, { @@ -339,8 +339,8 @@ "prefix_hash": "tenant-age-expire-3", "token_count": 64, "replicas": [ - {"id": "a-stale", "reported_at_ms": 2050000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.8, "pressure": 0, "observed_hit": false}, - {"id": "z-recent", "reported_at_ms": 2170000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.3, "pressure": 0, "observed_hit": true} + {"id": "a-stale", "prefix_reported_at_ms": 2050000, "stats_reported_at_ms": 2050000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.8, "pressure": 0, "observed_hit": false}, + {"id": "z-recent", "prefix_reported_at_ms": 2170000, "stats_reported_at_ms": 2170000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.3, "pressure": 0, "observed_hit": true} ] } ] diff --git a/internal/index/index.go b/internal/index/index.go index a9199083..e6c46888 100644 --- a/internal/index/index.go +++ b/internal/index/index.go @@ -265,7 +265,13 @@ func WithReservedTenants(tenants ...string) Option { } // WithClock overrides the time source for deterministic replay and tests. -func WithClock(now func() time.Time) Option { return func(i *Index) { i.now = now } } +func WithClock(now func() time.Time) Option { + return func(i *Index) { + if now != nil { + i.now = now + } + } +} // withClock keeps the package-local test helper concise. func withClock(now func() time.Time) Option { return WithClock(now) } diff --git a/internal/index/ingest_test.go b/internal/index/ingest_test.go index be69a6cf..508d6129 100644 --- a/internal/index/ingest_test.go +++ b/internal/index/ingest_test.go @@ -9,6 +9,15 @@ import ( "time" ) +func TestWithClockIgnoresNil(t *testing.T) { + idx := New(WithClock(nil)) + if idx.now == nil { + t.Fatal("WithClock(nil) cleared the default clock") + } + idx.Ingest(Update{ReplicaID: "r", Model: "m", Tenant: "t", HashScheme: "vllm", + Prefixes: []PrefixRef{{PrefixHash: hash("p"), TokenCount: 1}}}) +} + func TestIngestAndLookupRanksByTokensAndFreshness(t *testing.T) { clk := &fakeClock{t: time.Unix(1_000_000, 0)} idx := New(withClock(clk.now), WithTTL(time.Hour)) From 2c4eed298e5b64850d49577938f8ce090ed78ee6 Mon Sep 17 00:00:00 2001 From: Weiwei Zheng Date: Wed, 19 Aug 2026 17:51:43 -0700 Subject: [PATCH 6/8] Validate ranker calibration inputs Signed-off-by: Weiwei Zheng --- internal/index/calibration/calibration.go | 13 ++++++- .../index/calibration/calibration_test.go | 37 +++++++++++++++---- 2 files changed, 42 insertions(+), 8 deletions(-) diff --git a/internal/index/calibration/calibration.go b/internal/index/calibration/calibration.go index 3ebcc475..7d40efbf 100644 --- a/internal/index/calibration/calibration.go +++ b/internal/index/calibration/calibration.go @@ -150,6 +150,7 @@ func (t Trace) Validate() error { return errors.New("at least one observation is required") } seen := make(map[string]struct{}, len(t.Observations)) + seenKinds := make(map[string]bool, 2) for i, observation := range t.Observations { if err := observation.validate(); err != nil { return fmt.Errorf("observation %d: %w", i, err) @@ -158,6 +159,10 @@ func (t Trace) Validate() error { return fmt.Errorf("observation %d: duplicate id %q", i, observation.ID) } seen[observation.ID] = struct{}{} + seenKinds[observation.Kind] = true + } + if !seenKinds[ObservationPrefix] || !seenKinds[ObservationTenantHot] { + return errors.New("trace must contain at least one prefix and one tenant_hot observation") } return nil } @@ -204,6 +209,9 @@ func (o Observation) validate() error { if o.TokenCount <= 0 { return errors.New("token_count must be positive") } + if o.TTFTBudgetMillis < 0 { + return errors.New("ttft_budget_ms must be non-negative") + } if len(o.Replicas) == 0 { return errors.New("at least one replica is required") } @@ -218,7 +226,10 @@ func (o Observation) validate() error { if replica.StatsReportedAtMillis > o.AtMillis { return fmt.Errorf("replica %q: stats_reported_at_ms is after observation", replica.ID) } - if replica.ReportedPrefix && replica.MatchedTokens <= 0 { + if replica.MatchedTokens < 0 { + return fmt.Errorf("replica %q: matched_tokens must be non-negative", replica.ID) + } + if replica.ReportedPrefix && replica.MatchedTokens == 0 { return fmt.Errorf("replica %q: matched_tokens must be positive when reported_prefix is true", replica.ID) } if !finiteRate(float64(replica.HitRate)) || !finiteRate(float64(replica.Pressure)) { diff --git a/internal/index/calibration/calibration_test.go b/internal/index/calibration/calibration_test.go index 0ca051fd..0e27be54 100644 --- a/internal/index/calibration/calibration_test.go +++ b/internal/index/calibration/calibration_test.go @@ -79,6 +79,15 @@ func TestCalibrateSeparatesKnobEffects(t *testing.T) { {ID: "decoy", PrefixReportedAtMillis: 100_000, StatsReportedAtMillis: 100_000, HitRate: 0.1}, }, }, + { + ID: "tenant-hot", Kind: ObservationTenantHot, AtMillis: 100_000, + Tenant: "tenant-a", Model: "model-a", HashScheme: "vllm", + PrefixHash: "other", TokenCount: 320, + Replicas: []ReplicaObservation{{ + ID: "hot", PrefixReportedAtMillis: 100_000, StatsReportedAtMillis: 100_000, + HitRate: 1, ObservedHit: true, + }}, + }, }, } if err := trace.Validate(); err != nil { @@ -129,13 +138,20 @@ func TestTraceValidationRejectsInvalidFields(t *testing.T) { TenantHotMinHitRates: []float64{0.2}, TenantHotMaxAgeMillis: []int64{60_000}, }, - Observations: []Observation{{ - ID: "o", Kind: ObservationPrefix, Tenant: "t", Model: "m", - HashScheme: "vllm", PrefixHash: "p", TokenCount: 1, - Replicas: []ReplicaObservation{{ - ID: "r", ReportedPrefix: true, MatchedTokens: 1, - }}, - }}, + Observations: []Observation{ + { + ID: "prefix", Kind: ObservationPrefix, Tenant: "t", Model: "m", + HashScheme: "vllm", PrefixHash: "p", TokenCount: 1, + Replicas: []ReplicaObservation{{ + ID: "r", ReportedPrefix: true, MatchedTokens: 1, + }}, + }, + { + ID: "tenant-hot", Kind: ObservationTenantHot, Tenant: "t", Model: "m", + HashScheme: "vllm", PrefixHash: "p", TokenCount: 1, + Replicas: []ReplicaObservation{{ID: "r"}}, + }, + }, } } @@ -152,6 +168,8 @@ func TestTraceValidationRejectsInvalidFields(t *testing.T) { {"ttl overflow", func(trace *Trace) { trace.TTLMillis = maxDurationMillis + 1 }, "maximum representable"}, {"empty sweep", func(trace *Trace) { trace.Sweep.PressureWeights = nil }, "every sweep dimension"}, {"empty observations", func(trace *Trace) { trace.Observations = nil }, "at least one observation"}, + {"missing prefix observation", func(trace *Trace) { trace.Observations = trace.Observations[1:] }, "one prefix and one tenant_hot"}, + {"missing tenant-hot observation", func(trace *Trace) { trace.Observations = trace.Observations[:1] }, "one prefix and one tenant_hot"}, {"invalid observation", func(trace *Trace) { trace.Observations[0].Kind = "unknown" }, "observation 0"}, {"duplicate observation", func(trace *Trace) { trace.Observations = append(trace.Observations, trace.Observations[0]) }, "duplicate id"}, } { @@ -212,9 +230,14 @@ func TestObservationValidationRejectsInvalidReplicas(t *testing.T) { {"identity", func(observation *Observation) { observation.ID = "" }, "required"}, {"kind", func(observation *Observation) { observation.Kind = "unknown" }, "prefix or tenant_hot"}, {"tokens", func(observation *Observation) { observation.TokenCount = 0 }, "token_count"}, + {"negative ttft budget", func(observation *Observation) { observation.TTFTBudgetMillis = -1 }, "ttft_budget_ms"}, {"replicas", func(observation *Observation) { observation.Replicas = nil }, "at least one replica"}, {"replica id", func(observation *Observation) { observation.Replicas[0].ID = "" }, "id is required"}, {"matched tokens", func(observation *Observation) { observation.Replicas[0].MatchedTokens = 0 }, "matched_tokens"}, + {"negative unused matched tokens", func(observation *Observation) { + observation.Replicas[0].ReportedPrefix = false + observation.Replicas[0].MatchedTokens = -1 + }, "matched_tokens must be non-negative"}, {"rate", func(observation *Observation) { observation.Replicas[0].HitRate = 2 }, "finite values"}, {"duplicate replica", func(observation *Observation) { observation.Replicas = append(observation.Replicas, observation.Replicas[0]) From c25c8173552e2bccc3d9f2c06f99d283e4e77cb9 Mon Sep 17 00:00:00 2001 From: Weiwei Zheng Date: Wed, 19 Aug 2026 17:57:52 -0700 Subject: [PATCH 7/8] Align calibration replay with TTL eviction Signed-off-by: Weiwei Zheng --- internal/index/calibration/calibration.go | 61 +++++++++++-------- .../index/calibration/calibration_test.go | 18 ++++++ 2 files changed, 54 insertions(+), 25 deletions(-) diff --git a/internal/index/calibration/calibration.go b/internal/index/calibration/calibration.go index 7d40efbf..fdd81e08 100644 --- a/internal/index/calibration/calibration.go +++ b/internal/index/calibration/calibration.go @@ -174,8 +174,8 @@ func (s Sweep) validate() error { return errors.New("every sweep dimension must contain at least one value") } for _, value := range append(append([]float64{}, s.PressureWeights...), s.SLOTightBiases...) { - if !finiteNonNegative(value) { - return fmt.Errorf("sweep contains invalid non-negative value %v", value) + if !finiteNonNegativeFloat32(value) { + return fmt.Errorf("sweep contains invalid finite non-negative float32 value %v", value) } } for _, value := range s.TenantHotMinHitRates { @@ -247,6 +247,10 @@ func finiteNonNegative(value float64) bool { return !math.IsNaN(value) && !math.IsInf(value, 0) && value >= 0 } +func finiteNonNegativeFloat32(value float64) bool { + return finiteNonNegative(value) && value <= math.MaxFloat32 +} + func finiteRate(value float64) bool { return finiteNonNegative(value) && value <= 1 } @@ -361,6 +365,7 @@ func Replay(trace Trace, config Config) Metrics { func replayObservation(trace Trace, observation Observation, config Config) bool { anchor := time.UnixMilli(observation.AtMillis) + ttl := time.Duration(trace.TTLMillis) * time.Millisecond ranker := index.RankerConfig{ PressureWeight: float32(config.PressureWeight), SLOTightTTFTMs: config.SLOTightTTFTMillis, @@ -369,7 +374,7 @@ func replayObservation(trace Trace, observation Observation, config Config) bool TenantHotMaxAge: time.Duration(config.TenantHotMaxAgeMillis) * time.Millisecond, } idx := index.New( - index.WithTTL(time.Duration(trace.TTLMillis)*time.Millisecond), + index.WithTTL(ttl), index.WithRanker(ranker), index.WithClock(func() time.Time { return anchor }), ) @@ -381,28 +386,34 @@ func replayObservation(trace Trace, observation Observation, config Config) bool hash = "serving/" + observation.ID + "/" + replica.ID tokens = 1 } - idx.Ingest(index.Update{ - ReplicaID: replica.ID, - Model: observation.Model, - Tenant: observation.Tenant, - HashScheme: observation.HashScheme, - Timestamp: time.UnixMilli(replica.PrefixReportedAtMillis), - Prefixes: []index.PrefixRef{{ - PrefixHash: []byte(hash), - TokenCount: tokens, - }}, - }) - idx.Ingest(index.Update{ - ReplicaID: replica.ID, - Model: observation.Model, - Tenant: observation.Tenant, - HashScheme: observation.HashScheme, - Timestamp: time.UnixMilli(replica.StatsReportedAtMillis), - Stats: &index.ReplicaStats{ - HitRate: replica.HitRate, - Pressure: replica.Pressure, - }, - }) + prefixReportedAt := time.UnixMilli(replica.PrefixReportedAtMillis) + if anchor.Sub(prefixReportedAt) < ttl { + idx.Ingest(index.Update{ + ReplicaID: replica.ID, + Model: observation.Model, + Tenant: observation.Tenant, + HashScheme: observation.HashScheme, + Timestamp: prefixReportedAt, + Prefixes: []index.PrefixRef{{ + PrefixHash: []byte(hash), + TokenCount: tokens, + }}, + }) + } + statsReportedAt := time.UnixMilli(replica.StatsReportedAtMillis) + if anchor.Sub(statsReportedAt) < ttl { + idx.Ingest(index.Update{ + ReplicaID: replica.ID, + Model: observation.Model, + Tenant: observation.Tenant, + HashScheme: observation.HashScheme, + Timestamp: statsReportedAt, + Stats: &index.ReplicaStats{ + HitRate: replica.HitRate, + Pressure: replica.Pressure, + }, + }) + } observedHits[replica.ID] = replica.ObservedHit } result := idx.LookupRoute(index.LookupRequest{ diff --git a/internal/index/calibration/calibration_test.go b/internal/index/calibration/calibration_test.go index 0e27be54..760063b5 100644 --- a/internal/index/calibration/calibration_test.go +++ b/internal/index/calibration/calibration_test.go @@ -199,6 +199,8 @@ func TestSweepValidationRejectsInvalidValues(t *testing.T) { want string }{ {"pressure", func(sweep *Sweep) { sweep.PressureWeights[0] = -1 }, "non-negative"}, + {"pressure float32 overflow", func(sweep *Sweep) { sweep.PressureWeights[0] = math.MaxFloat64 }, "float32"}, + {"bias float32 overflow", func(sweep *Sweep) { sweep.SLOTightBiases[0] = math.MaxFloat64 }, "float32"}, {"hit rate", func(sweep *Sweep) { sweep.TenantHotMinHitRates[0] = 2 }, "invalid rate"}, {"ttft", func(sweep *Sweep) { sweep.SLOTightTTFTMillis[0] = 0 }, "non-positive"}, {"max age", func(sweep *Sweep) { sweep.TenantHotMaxAgeMillis[0] = 0 }, "non-positive"}, @@ -314,6 +316,22 @@ func TestReplayUsesIndependentPrefixAndStatsTimestamps(t *testing.T) { } } +func TestReplayExcludesTTLExpiredServingEntries(t *testing.T) { + trace := Trace{TTLMillis: int64(time.Minute / time.Millisecond)} + observation := Observation{ + ID: "expired-serving", Kind: ObservationTenantHot, AtMillis: 100_000, + Tenant: "t", Model: "m", HashScheme: "vllm", PrefixHash: "novel", TokenCount: 1, + Replicas: []ReplicaObservation{{ + ID: "stale", PrefixReportedAtMillis: 40_000, StatsReportedAtMillis: 100_000, + HitRate: 1, ObservedHit: true, + }}, + } + config := Config{TenantHotMinHitRate: 0.2, TenantHotMaxAgeMillis: 120_000} + if replayObservation(trace, observation, config) { + t.Fatal("replayObservation = hit, want TTL-expired serving entry evicted before lookup") + } +} + func TestObservationRejectsFutureReplicaReport(t *testing.T) { for _, tc := range []struct { name string From 6280a83e2882388ad1275dfb3101b09ef89af62d Mon Sep 17 00:00:00 2001 From: Weiwei Zheng Date: Wed, 19 Aug 2026 18:10:00 -0700 Subject: [PATCH 8/8] Harden calibration trace contract Signed-off-by: Weiwei Zheng --- docs/design/lookuproute-ranking.md | 8 + internal/index/calibration/README.md | 19 ++- internal/index/calibration/calibration.go | 29 +++- .../index/calibration/calibration_test.go | 84 +++++++--- .../testdata/c1_synthetic_trace.json | 156 +++++++++--------- 5 files changed, 185 insertions(+), 111 deletions(-) diff --git a/docs/design/lookuproute-ranking.md b/docs/design/lookuproute-ranking.md index 1d0f5541..33f0b4b5 100644 --- a/docs/design/lookuproute-ranking.md +++ b/docs/design/lookuproute-ranking.md @@ -854,6 +854,14 @@ Cartesian grid. The objective is the macro-average of prefix-hit ratio and `TENANT_HOT`-hit ratio; ties prefer gentler score multipliers and the shorter fallback window. +Calibration rows are controlled counterfactual experiments, not ordinary +single-route request logs: every candidate replica must have an available +ground-truth outcome. Captured traces must measure those outcomes experimentally +under an equivalent cache snapshot; synthetic traces define them by +construction. The loader rejects rows without that explicit availability. +Prefix hashes remain engine-opaque bytes and use standard base64 JSON encoding +in trace files. + The checked-in `c1-synthetic-mixed-routing-v1` trace contains 22 observations: 14 prefix-routing cases spanning pressure/locality tradeoffs and tight/loose TTFT budgets, plus 8 prefix-miss cases spanning noisy hit-rate reports and diff --git a/internal/index/calibration/README.md b/internal/index/calibration/README.md index 34c9c7f1..852816ed 100644 --- a/internal/index/calibration/README.md +++ b/internal/index/calibration/README.md @@ -20,10 +20,21 @@ records whether the cache plane believed that replica held the requested prefix; `prefix_reported_at_ms` records that prefix observation's freshness, while `stats_reported_at_ms` independently records when `hit_rate` and `pressure` were reported. `matched_tokens` and those timestamps are the signals -visible to the ranker. `observed_hit` is the later ground-truth outcome for -routing to that replica. Replicas without the requested prefix still receive a -unique serving-prefix entry during replay so `TENANT_HOT` can apply its real -engine-domain membership guard. +visible to the ranker. `prefix_hash` is standard base64 JSON for the engine's +opaque bytes, not a human-readable identifier. Replicas without the requested +prefix still receive a collision-free serving-only entry during replay so +`TENANT_HOT` can apply its real engine-domain membership guard. + +Every replica row must set `outcome_available: true`; `observed_hit` is the +ground-truth result of routing that request to that replica. Captured traces +must measure that outcome experimentally for every candidate under an +equivalent cache snapshot; synthetic traces may define it by construction. A +normal production request observes only its selected replica and is therefore +not sufficient calibration input by itself. The harness rejects incomplete +rows so an unavailable outcome cannot silently turn into a miss. Zero values +for `slo_tight_ttft_ms` and +`tenant_hot_max_age_ms` are valid sweep points and exercise the production kill +switches. Captured data should contain opaque or one-way prefix hashes only. Do not put prompt text, token IDs, customer identifiers, or other request content in a diff --git a/internal/index/calibration/calibration.go b/internal/index/calibration/calibration.go index fdd81e08..b81e86e6 100644 --- a/internal/index/calibration/calibration.go +++ b/internal/index/calibration/calibration.go @@ -57,7 +57,7 @@ type Observation struct { Tenant string `json:"tenant"` Model string `json:"model"` HashScheme string `json:"hash_scheme"` - PrefixHash string `json:"prefix_hash"` + PrefixHash []byte `json:"prefix_hash"` TokenCount int32 `json:"token_count"` TTFTBudgetMillis int32 `json:"ttft_budget_ms,omitempty"` Replicas []ReplicaObservation `json:"replicas"` @@ -71,6 +71,7 @@ type ReplicaObservation struct { MatchedTokens int32 `json:"matched_tokens"` HitRate float32 `json:"hit_rate"` Pressure float32 `json:"pressure"` + OutcomeAvailable bool `json:"outcome_available"` ObservedHit bool `json:"observed_hit"` } @@ -184,13 +185,13 @@ func (s Sweep) validate() error { } } for _, value := range s.SLOTightTTFTMillis { - if value <= 0 { - return fmt.Errorf("slo_tight_ttft_ms contains non-positive value %d", value) + if value < 0 { + return fmt.Errorf("slo_tight_ttft_ms contains negative value %d", value) } } for _, value := range s.TenantHotMaxAgeMillis { - if value <= 0 { - return fmt.Errorf("tenant_hot_max_age_ms contains non-positive value %d", value) + if value < 0 { + return fmt.Errorf("tenant_hot_max_age_ms contains negative value %d", value) } if value > maxDurationMillis { return fmt.Errorf("tenant_hot_max_age_ms contains value exceeding maximum representable duration: %d", value) @@ -200,7 +201,7 @@ func (s Sweep) validate() error { } func (o Observation) validate() error { - if o.ID == "" || o.Tenant == "" || o.Model == "" || o.HashScheme == "" || o.PrefixHash == "" { + if o.ID == "" || o.Tenant == "" || o.Model == "" || o.HashScheme == "" || len(o.PrefixHash) == 0 { return errors.New("id, tenant, model, hash_scheme, and prefix_hash are required") } if o.Kind != ObservationPrefix && o.Kind != ObservationTenantHot { @@ -235,6 +236,9 @@ func (o Observation) validate() error { if !finiteRate(float64(replica.HitRate)) || !finiteRate(float64(replica.Pressure)) { return fmt.Errorf("replica %q: hit_rate and pressure must be finite values in [0,1]", replica.ID) } + if !replica.OutcomeAvailable { + return fmt.Errorf("replica %q: outcome_available must be true", replica.ID) + } if _, ok := seen[replica.ID]; ok { return fmt.Errorf("duplicate replica id %q", replica.ID) } @@ -383,7 +387,7 @@ func replayObservation(trace Trace, observation Observation, config Config) bool hash := observation.PrefixHash tokens := replica.MatchedTokens if !replica.ReportedPrefix { - hash = "serving/" + observation.ID + "/" + replica.ID + hash = servingOnlyHash(observation.PrefixHash, replica.ID) tokens = 1 } prefixReportedAt := time.UnixMilli(replica.PrefixReportedAtMillis) @@ -395,7 +399,7 @@ func replayObservation(trace Trace, observation Observation, config Config) bool HashScheme: observation.HashScheme, Timestamp: prefixReportedAt, Prefixes: []index.PrefixRef{{ - PrefixHash: []byte(hash), + PrefixHash: hash, TokenCount: tokens, }}, }) @@ -420,7 +424,7 @@ func replayObservation(trace Trace, observation Observation, config Config) bool Model: observation.Model, Tenant: observation.Tenant, HashScheme: observation.HashScheme, - PrefixHash: []byte(observation.PrefixHash), + PrefixHash: observation.PrefixHash, TokenCount: observation.TokenCount, TTFTBudgetMs: observation.TTFTBudgetMillis, }) @@ -431,6 +435,13 @@ func replayObservation(trace Trace, observation Observation, config Config) bool return result.Strategy == wantStrategy && len(result.Scores) > 0 && observedHits[result.Scores[0].ReplicaID] } +func servingOnlyHash(requested []byte, replicaID string) []byte { + hash := make([]byte, 0, len(requested)+1+len(replicaID)) + hash = append(hash, requested...) + hash = append(hash, 0) + return append(hash, replicaID...) +} + func percentage(numerator, denominator int) float64 { if denominator == 0 { return 0 diff --git a/internal/index/calibration/calibration_test.go b/internal/index/calibration/calibration_test.go index 760063b5..c87b9c5d 100644 --- a/internal/index/calibration/calibration_test.go +++ b/internal/index/calibration/calibration_test.go @@ -6,6 +6,7 @@ package calibration import ( "bytes" + "encoding/json" "math" "os" "strings" @@ -29,6 +30,9 @@ func TestCheckedInSyntheticTraceSelectsCandidateAndCurrentResult(t *testing.T) { if trace.Provenance.Kind != "synthetic" { t.Fatalf("provenance kind = %q, want synthetic", trace.Provenance.Kind) } + if !bytes.Equal(trace.Observations[0].PrefixHash, []byte("pressure-shed-1")) { + t.Fatalf("decoded prefix hash = %x, want opaque fixture bytes", trace.Observations[0].PrefixHash) + } want := Config{ PressureWeight: 0.5, SLOTightTTFTMillis: 200, @@ -72,20 +76,20 @@ func TestCalibrateSeparatesKnobEffects(t *testing.T) { { ID: "pressure", Kind: ObservationPrefix, AtMillis: 100_000, Tenant: "tenant-a", Model: "model-a", HashScheme: "vllm", - PrefixHash: "p", TokenCount: 320, + PrefixHash: []byte("p"), TokenCount: 320, Replicas: []ReplicaObservation{ - {ID: "hot", PrefixReportedAtMillis: 100_000, StatsReportedAtMillis: 100_000, ReportedPrefix: true, MatchedTokens: 320, HitRate: 0.8, Pressure: 0.8}, - {ID: "cool", PrefixReportedAtMillis: 100_000, StatsReportedAtMillis: 100_000, ReportedPrefix: true, MatchedTokens: 256, HitRate: 0.4, Pressure: 0.1, ObservedHit: true}, - {ID: "decoy", PrefixReportedAtMillis: 100_000, StatsReportedAtMillis: 100_000, HitRate: 0.1}, + {ID: "hot", PrefixReportedAtMillis: 100_000, StatsReportedAtMillis: 100_000, ReportedPrefix: true, MatchedTokens: 320, HitRate: 0.8, Pressure: 0.8, OutcomeAvailable: true}, + {ID: "cool", PrefixReportedAtMillis: 100_000, StatsReportedAtMillis: 100_000, ReportedPrefix: true, MatchedTokens: 256, HitRate: 0.4, Pressure: 0.1, OutcomeAvailable: true, ObservedHit: true}, + {ID: "decoy", PrefixReportedAtMillis: 100_000, StatsReportedAtMillis: 100_000, HitRate: 0.1, OutcomeAvailable: true}, }, }, { ID: "tenant-hot", Kind: ObservationTenantHot, AtMillis: 100_000, Tenant: "tenant-a", Model: "model-a", HashScheme: "vllm", - PrefixHash: "other", TokenCount: 320, + PrefixHash: []byte("other"), TokenCount: 320, Replicas: []ReplicaObservation{{ ID: "hot", PrefixReportedAtMillis: 100_000, StatsReportedAtMillis: 100_000, - HitRate: 1, ObservedHit: true, + HitRate: 1, OutcomeAvailable: true, ObservedHit: true, }}, }, }, @@ -141,15 +145,15 @@ func TestTraceValidationRejectsInvalidFields(t *testing.T) { Observations: []Observation{ { ID: "prefix", Kind: ObservationPrefix, Tenant: "t", Model: "m", - HashScheme: "vllm", PrefixHash: "p", TokenCount: 1, + HashScheme: "vllm", PrefixHash: []byte("p"), TokenCount: 1, Replicas: []ReplicaObservation{{ - ID: "r", ReportedPrefix: true, MatchedTokens: 1, + ID: "r", ReportedPrefix: true, MatchedTokens: 1, OutcomeAvailable: true, }}, }, { ID: "tenant-hot", Kind: ObservationTenantHot, Tenant: "t", Model: "m", - HashScheme: "vllm", PrefixHash: "p", TokenCount: 1, - Replicas: []ReplicaObservation{{ID: "r"}}, + HashScheme: "vllm", PrefixHash: []byte("p"), TokenCount: 1, + Replicas: []ReplicaObservation{{ID: "r", OutcomeAvailable: true}}, }, }, } @@ -202,8 +206,8 @@ func TestSweepValidationRejectsInvalidValues(t *testing.T) { {"pressure float32 overflow", func(sweep *Sweep) { sweep.PressureWeights[0] = math.MaxFloat64 }, "float32"}, {"bias float32 overflow", func(sweep *Sweep) { sweep.SLOTightBiases[0] = math.MaxFloat64 }, "float32"}, {"hit rate", func(sweep *Sweep) { sweep.TenantHotMinHitRates[0] = 2 }, "invalid rate"}, - {"ttft", func(sweep *Sweep) { sweep.SLOTightTTFTMillis[0] = 0 }, "non-positive"}, - {"max age", func(sweep *Sweep) { sweep.TenantHotMaxAgeMillis[0] = 0 }, "non-positive"}, + {"ttft", func(sweep *Sweep) { sweep.SLOTightTTFTMillis[0] = -1 }, "negative"}, + {"max age", func(sweep *Sweep) { sweep.TenantHotMaxAgeMillis[0] = -1 }, "negative"}, {"max age overflow", func(sweep *Sweep) { sweep.TenantHotMaxAgeMillis[0] = maxDurationMillis + 1 }, "maximum representable"}, } { t.Run(tc.name, func(t *testing.T) { @@ -214,14 +218,22 @@ func TestSweepValidationRejectsInvalidValues(t *testing.T) { } }) } + t.Run("zero kill switches", func(t *testing.T) { + sweep := valid() + sweep.SLOTightTTFTMillis[0] = 0 + sweep.TenantHotMaxAgeMillis[0] = 0 + if err := sweep.validate(); err != nil { + t.Fatalf("validate zero kill switches: %v", err) + } + }) } func TestObservationValidationRejectsInvalidReplicas(t *testing.T) { valid := func() Observation { return Observation{ ID: "o", Kind: ObservationPrefix, Tenant: "t", Model: "m", - HashScheme: "vllm", PrefixHash: "p", TokenCount: 1, - Replicas: []ReplicaObservation{{ID: "r", ReportedPrefix: true, MatchedTokens: 1}}, + HashScheme: "vllm", PrefixHash: []byte("p"), TokenCount: 1, + Replicas: []ReplicaObservation{{ID: "r", ReportedPrefix: true, MatchedTokens: 1, OutcomeAvailable: true}}, } } for _, tc := range []struct { @@ -241,6 +253,7 @@ func TestObservationValidationRejectsInvalidReplicas(t *testing.T) { observation.Replicas[0].MatchedTokens = -1 }, "matched_tokens must be non-negative"}, {"rate", func(observation *Observation) { observation.Replicas[0].HitRate = 2 }, "finite values"}, + {"outcome unavailable", func(observation *Observation) { observation.Replicas[0].OutcomeAvailable = false }, "outcome_available"}, {"duplicate replica", func(observation *Observation) { observation.Replicas = append(observation.Replicas, observation.Replicas[0]) }, "duplicate replica"}, @@ -262,11 +275,26 @@ func TestMarshalResultWrapsJSONErrors(t *testing.T) { } } +func TestObservationPrefixHashRoundTripsOpaqueBytes(t *testing.T) { + want := []byte{0, 0xff, 0x80, 'x'} + encoded, err := json.Marshal(Observation{PrefixHash: want}) + if err != nil { + t.Fatalf("Marshal: %v", err) + } + var got Observation + if err := json.Unmarshal(encoded, &got); err != nil { + t.Fatalf("Unmarshal: %v", err) + } + if !bytes.Equal(got.PrefixHash, want) { + t.Fatalf("prefix hash = %x, want %x", got.PrefixHash, want) + } +} + func TestReplayTenantHotMissWithoutCandidate(t *testing.T) { trace := Trace{TTLMillis: int64(time.Minute / time.Millisecond)} observation := Observation{ ID: "tenant-hot", Kind: ObservationTenantHot, AtMillis: 100_000, - Tenant: "t", Model: "m", HashScheme: "vllm", PrefixHash: "novel", TokenCount: 1, + Tenant: "t", Model: "m", HashScheme: "vllm", PrefixHash: []byte("novel"), TokenCount: 1, Replicas: []ReplicaObservation{{ ID: "cold", PrefixReportedAtMillis: 100_000, StatsReportedAtMillis: 100_000, HitRate: 0.1, ObservedHit: true, @@ -282,7 +310,7 @@ func TestReplayUsesObservationClockAtTenantHotBoundary(t *testing.T) { trace := Trace{TTLMillis: int64(time.Minute / time.Millisecond)} observation := Observation{ ID: "tenant-hot-boundary", Kind: ObservationTenantHot, AtMillis: 100_000, - Tenant: "t", Model: "m", HashScheme: "vllm", PrefixHash: "novel", TokenCount: 1, + Tenant: "t", Model: "m", HashScheme: "vllm", PrefixHash: []byte("novel"), TokenCount: 1, Replicas: []ReplicaObservation{{ ID: "warm", PrefixReportedAtMillis: 40_001, StatsReportedAtMillis: 40_001, HitRate: 0.8, ObservedHit: true, @@ -298,7 +326,7 @@ func TestReplayUsesIndependentPrefixAndStatsTimestamps(t *testing.T) { trace := Trace{TTLMillis: int64(time.Minute / time.Millisecond)} observation := Observation{ ID: "independent-clocks", Kind: ObservationPrefix, AtMillis: 100_000, - Tenant: "t", Model: "m", HashScheme: "vllm", PrefixHash: "p", TokenCount: 320, + Tenant: "t", Model: "m", HashScheme: "vllm", PrefixHash: []byte("p"), TokenCount: 320, Replicas: []ReplicaObservation{ { ID: "deep-stale-stats", PrefixReportedAtMillis: 100_000, StatsReportedAtMillis: 0, @@ -320,7 +348,7 @@ func TestReplayExcludesTTLExpiredServingEntries(t *testing.T) { trace := Trace{TTLMillis: int64(time.Minute / time.Millisecond)} observation := Observation{ ID: "expired-serving", Kind: ObservationTenantHot, AtMillis: 100_000, - Tenant: "t", Model: "m", HashScheme: "vllm", PrefixHash: "novel", TokenCount: 1, + Tenant: "t", Model: "m", HashScheme: "vllm", PrefixHash: []byte("novel"), TokenCount: 1, Replicas: []ReplicaObservation{{ ID: "stale", PrefixReportedAtMillis: 40_000, StatsReportedAtMillis: 100_000, HitRate: 1, ObservedHit: true, @@ -332,6 +360,22 @@ func TestReplayExcludesTTLExpiredServingEntries(t *testing.T) { } } +func TestReplayServingOnlyHashCannotMatchRequestedPrefix(t *testing.T) { + trace := Trace{TTLMillis: int64(time.Minute / time.Millisecond)} + observation := Observation{ + ID: "collision", Kind: ObservationPrefix, AtMillis: 100_000, + Tenant: "t", Model: "m", HashScheme: "vllm", + PrefixHash: []byte("serving/collision/not-holder"), TokenCount: 1, + Replicas: []ReplicaObservation{{ + ID: "not-holder", PrefixReportedAtMillis: 100_000, StatsReportedAtMillis: 100_000, + ObservedHit: true, + }}, + } + if replayObservation(trace, observation, Config{TenantHotMaxAgeMillis: 60_000}) { + t.Fatal("replayObservation = hit, want serving-only key distinct from requested prefix") + } +} + func TestObservationRejectsFutureReplicaReport(t *testing.T) { for _, tc := range []struct { name string @@ -344,8 +388,8 @@ func TestObservationRejectsFutureReplicaReport(t *testing.T) { t.Run(tc.name, func(t *testing.T) { observation := Observation{ ID: "future", Kind: ObservationPrefix, AtMillis: 10, - Tenant: "t", Model: "m", HashScheme: "vllm", PrefixHash: "p", TokenCount: 1, - Replicas: []ReplicaObservation{{ID: "r", ReportedPrefix: true, MatchedTokens: 1}}, + Tenant: "t", Model: "m", HashScheme: "vllm", PrefixHash: []byte("p"), TokenCount: 1, + Replicas: []ReplicaObservation{{ID: "r", ReportedPrefix: true, MatchedTokens: 1, OutcomeAvailable: true}}, } tc.mutate(&observation.Replicas[0]) if err := observation.validate(); err == nil || !strings.Contains(err.Error(), tc.want) { diff --git a/internal/index/calibration/testdata/c1_synthetic_trace.json b/internal/index/calibration/testdata/c1_synthetic_trace.json index 34e4cf18..a9fa43d3 100644 --- a/internal/index/calibration/testdata/c1_synthetic_trace.json +++ b/internal/index/calibration/testdata/c1_synthetic_trace.json @@ -22,12 +22,12 @@ "tenant": "tenant-a", "model": "model-a", "hash_scheme": "vllm", - "prefix_hash": "pressure-shed-1", + "prefix_hash": "cHJlc3N1cmUtc2hlZC0x", "token_count": 320, "replicas": [ - {"id": "a-hot", "prefix_reported_at_ms": 100000, "stats_reported_at_ms": 100000, "reported_prefix": true, "matched_tokens": 320, "hit_rate": 0.7, "pressure": 0.8, "observed_hit": false}, - {"id": "b-cool", "prefix_reported_at_ms": 100000, "stats_reported_at_ms": 100000, "reported_prefix": true, "matched_tokens": 256, "hit_rate": 0.4, "pressure": 0.1, "observed_hit": true}, - {"id": "z-decoy", "prefix_reported_at_ms": 100000, "stats_reported_at_ms": 100000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.1, "pressure": 0, "observed_hit": false} + {"id": "a-hot", "prefix_reported_at_ms": 100000, "stats_reported_at_ms": 100000, "reported_prefix": true, "matched_tokens": 320, "hit_rate": 0.7, "pressure": 0.8, "outcome_available": true, "observed_hit": false}, + {"id": "b-cool", "prefix_reported_at_ms": 100000, "stats_reported_at_ms": 100000, "reported_prefix": true, "matched_tokens": 256, "hit_rate": 0.4, "pressure": 0.1, "outcome_available": true, "observed_hit": true}, + {"id": "z-decoy", "prefix_reported_at_ms": 100000, "stats_reported_at_ms": 100000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.1, "pressure": 0, "outcome_available": true, "observed_hit": false} ] }, { @@ -37,12 +37,12 @@ "tenant": "tenant-a", "model": "model-a", "hash_scheme": "vllm", - "prefix_hash": "pressure-shed-2", + "prefix_hash": "cHJlc3N1cmUtc2hlZC0y", "token_count": 320, "replicas": [ - {"id": "a-hot", "prefix_reported_at_ms": 200000, "stats_reported_at_ms": 200000, "reported_prefix": true, "matched_tokens": 320, "hit_rate": 0.7, "pressure": 0.8, "observed_hit": false}, - {"id": "b-cool", "prefix_reported_at_ms": 200000, "stats_reported_at_ms": 200000, "reported_prefix": true, "matched_tokens": 256, "hit_rate": 0.4, "pressure": 0.1, "observed_hit": true}, - {"id": "z-decoy", "prefix_reported_at_ms": 200000, "stats_reported_at_ms": 200000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.1, "pressure": 0, "observed_hit": false} + {"id": "a-hot", "prefix_reported_at_ms": 200000, "stats_reported_at_ms": 200000, "reported_prefix": true, "matched_tokens": 320, "hit_rate": 0.7, "pressure": 0.8, "outcome_available": true, "observed_hit": false}, + {"id": "b-cool", "prefix_reported_at_ms": 200000, "stats_reported_at_ms": 200000, "reported_prefix": true, "matched_tokens": 256, "hit_rate": 0.4, "pressure": 0.1, "outcome_available": true, "observed_hit": true}, + {"id": "z-decoy", "prefix_reported_at_ms": 200000, "stats_reported_at_ms": 200000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.1, "pressure": 0, "outcome_available": true, "observed_hit": false} ] }, { @@ -52,12 +52,12 @@ "tenant": "tenant-b", "model": "model-b", "hash_scheme": "vllm", - "prefix_hash": "pressure-shed-3", + "prefix_hash": "cHJlc3N1cmUtc2hlZC0z", "token_count": 320, "replicas": [ - {"id": "a-hot", "prefix_reported_at_ms": 300000, "stats_reported_at_ms": 300000, "reported_prefix": true, "matched_tokens": 320, "hit_rate": 0.65, "pressure": 0.8, "observed_hit": false}, - {"id": "b-cool", "prefix_reported_at_ms": 300000, "stats_reported_at_ms": 300000, "reported_prefix": true, "matched_tokens": 256, "hit_rate": 0.45, "pressure": 0.1, "observed_hit": true}, - {"id": "z-decoy", "prefix_reported_at_ms": 300000, "stats_reported_at_ms": 300000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.1, "pressure": 0, "observed_hit": false} + {"id": "a-hot", "prefix_reported_at_ms": 300000, "stats_reported_at_ms": 300000, "reported_prefix": true, "matched_tokens": 320, "hit_rate": 0.65, "pressure": 0.8, "outcome_available": true, "observed_hit": false}, + {"id": "b-cool", "prefix_reported_at_ms": 300000, "stats_reported_at_ms": 300000, "reported_prefix": true, "matched_tokens": 256, "hit_rate": 0.45, "pressure": 0.1, "outcome_available": true, "observed_hit": true}, + {"id": "z-decoy", "prefix_reported_at_ms": 300000, "stats_reported_at_ms": 300000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.1, "pressure": 0, "outcome_available": true, "observed_hit": false} ] }, { @@ -67,12 +67,12 @@ "tenant": "tenant-a", "model": "model-a", "hash_scheme": "vllm", - "prefix_hash": "pressure-preserve-1", + "prefix_hash": "cHJlc3N1cmUtcHJlc2VydmUtMQ==", "token_count": 512, "replicas": [ - {"id": "a-local", "prefix_reported_at_ms": 400000, "stats_reported_at_ms": 400000, "reported_prefix": true, "matched_tokens": 512, "hit_rate": 0.75, "pressure": 0.7, "observed_hit": true}, - {"id": "b-cool", "prefix_reported_at_ms": 400000, "stats_reported_at_ms": 400000, "reported_prefix": true, "matched_tokens": 256, "hit_rate": 0.3, "pressure": 0, "observed_hit": false}, - {"id": "z-decoy", "prefix_reported_at_ms": 400000, "stats_reported_at_ms": 400000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.1, "pressure": 0, "observed_hit": false} + {"id": "a-local", "prefix_reported_at_ms": 400000, "stats_reported_at_ms": 400000, "reported_prefix": true, "matched_tokens": 512, "hit_rate": 0.75, "pressure": 0.7, "outcome_available": true, "observed_hit": true}, + {"id": "b-cool", "prefix_reported_at_ms": 400000, "stats_reported_at_ms": 400000, "reported_prefix": true, "matched_tokens": 256, "hit_rate": 0.3, "pressure": 0, "outcome_available": true, "observed_hit": false}, + {"id": "z-decoy", "prefix_reported_at_ms": 400000, "stats_reported_at_ms": 400000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.1, "pressure": 0, "outcome_available": true, "observed_hit": false} ] }, { @@ -82,12 +82,12 @@ "tenant": "tenant-a", "model": "model-a", "hash_scheme": "vllm", - "prefix_hash": "pressure-preserve-2", + "prefix_hash": "cHJlc3N1cmUtcHJlc2VydmUtMg==", "token_count": 512, "replicas": [ - {"id": "a-local", "prefix_reported_at_ms": 500000, "stats_reported_at_ms": 500000, "reported_prefix": true, "matched_tokens": 512, "hit_rate": 0.75, "pressure": 0.7, "observed_hit": true}, - {"id": "b-cool", "prefix_reported_at_ms": 500000, "stats_reported_at_ms": 500000, "reported_prefix": true, "matched_tokens": 256, "hit_rate": 0.3, "pressure": 0, "observed_hit": false}, - {"id": "z-decoy", "prefix_reported_at_ms": 500000, "stats_reported_at_ms": 500000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.1, "pressure": 0, "observed_hit": false} + {"id": "a-local", "prefix_reported_at_ms": 500000, "stats_reported_at_ms": 500000, "reported_prefix": true, "matched_tokens": 512, "hit_rate": 0.75, "pressure": 0.7, "outcome_available": true, "observed_hit": true}, + {"id": "b-cool", "prefix_reported_at_ms": 500000, "stats_reported_at_ms": 500000, "reported_prefix": true, "matched_tokens": 256, "hit_rate": 0.3, "pressure": 0, "outcome_available": true, "observed_hit": false}, + {"id": "z-decoy", "prefix_reported_at_ms": 500000, "stats_reported_at_ms": 500000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.1, "pressure": 0, "outcome_available": true, "observed_hit": false} ] }, { @@ -97,12 +97,12 @@ "tenant": "tenant-b", "model": "model-b", "hash_scheme": "vllm", - "prefix_hash": "pressure-preserve-3", + "prefix_hash": "cHJlc3N1cmUtcHJlc2VydmUtMw==", "token_count": 512, "replicas": [ - {"id": "a-local", "prefix_reported_at_ms": 600000, "stats_reported_at_ms": 600000, "reported_prefix": true, "matched_tokens": 512, "hit_rate": 0.7, "pressure": 0.7, "observed_hit": true}, - {"id": "b-cool", "prefix_reported_at_ms": 600000, "stats_reported_at_ms": 600000, "reported_prefix": true, "matched_tokens": 256, "hit_rate": 0.35, "pressure": 0, "observed_hit": false}, - {"id": "z-decoy", "prefix_reported_at_ms": 600000, "stats_reported_at_ms": 600000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.1, "pressure": 0, "observed_hit": false} + {"id": "a-local", "prefix_reported_at_ms": 600000, "stats_reported_at_ms": 600000, "reported_prefix": true, "matched_tokens": 512, "hit_rate": 0.7, "pressure": 0.7, "outcome_available": true, "observed_hit": true}, + {"id": "b-cool", "prefix_reported_at_ms": 600000, "stats_reported_at_ms": 600000, "reported_prefix": true, "matched_tokens": 256, "hit_rate": 0.35, "pressure": 0, "outcome_available": true, "observed_hit": false}, + {"id": "z-decoy", "prefix_reported_at_ms": 600000, "stats_reported_at_ms": 600000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.1, "pressure": 0, "outcome_available": true, "observed_hit": false} ] }, { @@ -112,13 +112,13 @@ "tenant": "tenant-a", "model": "model-a", "hash_scheme": "vllm", - "prefix_hash": "slo-promote-120-1", + "prefix_hash": "c2xvLXByb21vdGUtMTIwLTE=", "token_count": 512, "ttft_budget_ms": 120, "replicas": [ - {"id": "a-old", "prefix_reported_at_ms": 160000, "stats_reported_at_ms": 160000, "reported_prefix": true, "matched_tokens": 512, "hit_rate": 0.5, "pressure": 0, "observed_hit": false}, - {"id": "b-fresh", "prefix_reported_at_ms": 700000, "stats_reported_at_ms": 700000, "reported_prefix": true, "matched_tokens": 320, "hit_rate": 0.5, "pressure": 0, "observed_hit": true}, - {"id": "z-decoy", "prefix_reported_at_ms": 700000, "stats_reported_at_ms": 700000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.1, "pressure": 0, "observed_hit": false} + {"id": "a-old", "prefix_reported_at_ms": 160000, "stats_reported_at_ms": 160000, "reported_prefix": true, "matched_tokens": 512, "hit_rate": 0.5, "pressure": 0, "outcome_available": true, "observed_hit": false}, + {"id": "b-fresh", "prefix_reported_at_ms": 700000, "stats_reported_at_ms": 700000, "reported_prefix": true, "matched_tokens": 320, "hit_rate": 0.5, "pressure": 0, "outcome_available": true, "observed_hit": true}, + {"id": "z-decoy", "prefix_reported_at_ms": 700000, "stats_reported_at_ms": 700000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.1, "pressure": 0, "outcome_available": true, "observed_hit": false} ] }, { @@ -128,13 +128,13 @@ "tenant": "tenant-b", "model": "model-b", "hash_scheme": "vllm", - "prefix_hash": "slo-promote-120-2", + "prefix_hash": "c2xvLXByb21vdGUtMTIwLTI=", "token_count": 512, "ttft_budget_ms": 120, "replicas": [ - {"id": "a-old", "prefix_reported_at_ms": 260000, "stats_reported_at_ms": 260000, "reported_prefix": true, "matched_tokens": 512, "hit_rate": 0.5, "pressure": 0, "observed_hit": false}, - {"id": "b-fresh", "prefix_reported_at_ms": 800000, "stats_reported_at_ms": 800000, "reported_prefix": true, "matched_tokens": 320, "hit_rate": 0.5, "pressure": 0, "observed_hit": true}, - {"id": "z-decoy", "prefix_reported_at_ms": 800000, "stats_reported_at_ms": 800000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.1, "pressure": 0, "observed_hit": false} + {"id": "a-old", "prefix_reported_at_ms": 260000, "stats_reported_at_ms": 260000, "reported_prefix": true, "matched_tokens": 512, "hit_rate": 0.5, "pressure": 0, "outcome_available": true, "observed_hit": false}, + {"id": "b-fresh", "prefix_reported_at_ms": 800000, "stats_reported_at_ms": 800000, "reported_prefix": true, "matched_tokens": 320, "hit_rate": 0.5, "pressure": 0, "outcome_available": true, "observed_hit": true}, + {"id": "z-decoy", "prefix_reported_at_ms": 800000, "stats_reported_at_ms": 800000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.1, "pressure": 0, "outcome_available": true, "observed_hit": false} ] }, { @@ -144,13 +144,13 @@ "tenant": "tenant-a", "model": "model-a", "hash_scheme": "vllm", - "prefix_hash": "slo-promote-180-1", + "prefix_hash": "c2xvLXByb21vdGUtMTgwLTE=", "token_count": 512, "ttft_budget_ms": 180, "replicas": [ - {"id": "a-old", "prefix_reported_at_ms": 360000, "stats_reported_at_ms": 360000, "reported_prefix": true, "matched_tokens": 512, "hit_rate": 0.5, "pressure": 0, "observed_hit": false}, - {"id": "b-fresh", "prefix_reported_at_ms": 900000, "stats_reported_at_ms": 900000, "reported_prefix": true, "matched_tokens": 320, "hit_rate": 0.5, "pressure": 0, "observed_hit": true}, - {"id": "z-decoy", "prefix_reported_at_ms": 900000, "stats_reported_at_ms": 900000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.1, "pressure": 0, "observed_hit": false} + {"id": "a-old", "prefix_reported_at_ms": 360000, "stats_reported_at_ms": 360000, "reported_prefix": true, "matched_tokens": 512, "hit_rate": 0.5, "pressure": 0, "outcome_available": true, "observed_hit": false}, + {"id": "b-fresh", "prefix_reported_at_ms": 900000, "stats_reported_at_ms": 900000, "reported_prefix": true, "matched_tokens": 320, "hit_rate": 0.5, "pressure": 0, "outcome_available": true, "observed_hit": true}, + {"id": "z-decoy", "prefix_reported_at_ms": 900000, "stats_reported_at_ms": 900000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.1, "pressure": 0, "outcome_available": true, "observed_hit": false} ] }, { @@ -160,13 +160,13 @@ "tenant": "tenant-b", "model": "model-b", "hash_scheme": "vllm", - "prefix_hash": "slo-promote-180-2", + "prefix_hash": "c2xvLXByb21vdGUtMTgwLTI=", "token_count": 512, "ttft_budget_ms": 180, "replicas": [ - {"id": "a-old", "prefix_reported_at_ms": 460000, "stats_reported_at_ms": 460000, "reported_prefix": true, "matched_tokens": 512, "hit_rate": 0.5, "pressure": 0, "observed_hit": false}, - {"id": "b-fresh", "prefix_reported_at_ms": 1000000, "stats_reported_at_ms": 1000000, "reported_prefix": true, "matched_tokens": 320, "hit_rate": 0.5, "pressure": 0, "observed_hit": true}, - {"id": "z-decoy", "prefix_reported_at_ms": 1000000, "stats_reported_at_ms": 1000000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.1, "pressure": 0, "observed_hit": false} + {"id": "a-old", "prefix_reported_at_ms": 460000, "stats_reported_at_ms": 460000, "reported_prefix": true, "matched_tokens": 512, "hit_rate": 0.5, "pressure": 0, "outcome_available": true, "observed_hit": false}, + {"id": "b-fresh", "prefix_reported_at_ms": 1000000, "stats_reported_at_ms": 1000000, "reported_prefix": true, "matched_tokens": 320, "hit_rate": 0.5, "pressure": 0, "outcome_available": true, "observed_hit": true}, + {"id": "z-decoy", "prefix_reported_at_ms": 1000000, "stats_reported_at_ms": 1000000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.1, "pressure": 0, "outcome_available": true, "observed_hit": false} ] }, { @@ -176,13 +176,13 @@ "tenant": "tenant-a", "model": "model-a", "hash_scheme": "vllm", - "prefix_hash": "slo-preserve-120-1", + "prefix_hash": "c2xvLXByZXNlcnZlLTEyMC0x", "token_count": 550, "ttft_budget_ms": 120, "replicas": [ - {"id": "a-deep", "prefix_reported_at_ms": 560000, "stats_reported_at_ms": 560000, "reported_prefix": true, "matched_tokens": 550, "hit_rate": 0.6, "pressure": 0, "observed_hit": true}, - {"id": "b-fresh", "prefix_reported_at_ms": 1100000, "stats_reported_at_ms": 1100000, "reported_prefix": true, "matched_tokens": 320, "hit_rate": 0.5, "pressure": 0, "observed_hit": false}, - {"id": "z-decoy", "prefix_reported_at_ms": 1100000, "stats_reported_at_ms": 1100000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.1, "pressure": 0, "observed_hit": false} + {"id": "a-deep", "prefix_reported_at_ms": 560000, "stats_reported_at_ms": 560000, "reported_prefix": true, "matched_tokens": 550, "hit_rate": 0.6, "pressure": 0, "outcome_available": true, "observed_hit": true}, + {"id": "b-fresh", "prefix_reported_at_ms": 1100000, "stats_reported_at_ms": 1100000, "reported_prefix": true, "matched_tokens": 320, "hit_rate": 0.5, "pressure": 0, "outcome_available": true, "observed_hit": false}, + {"id": "z-decoy", "prefix_reported_at_ms": 1100000, "stats_reported_at_ms": 1100000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.1, "pressure": 0, "outcome_available": true, "observed_hit": false} ] }, { @@ -192,13 +192,13 @@ "tenant": "tenant-b", "model": "model-b", "hash_scheme": "vllm", - "prefix_hash": "slo-preserve-120-2", + "prefix_hash": "c2xvLXByZXNlcnZlLTEyMC0y", "token_count": 550, "ttft_budget_ms": 120, "replicas": [ - {"id": "a-deep", "prefix_reported_at_ms": 660000, "stats_reported_at_ms": 660000, "reported_prefix": true, "matched_tokens": 550, "hit_rate": 0.6, "pressure": 0, "observed_hit": true}, - {"id": "b-fresh", "prefix_reported_at_ms": 1200000, "stats_reported_at_ms": 1200000, "reported_prefix": true, "matched_tokens": 320, "hit_rate": 0.5, "pressure": 0, "observed_hit": false}, - {"id": "z-decoy", "prefix_reported_at_ms": 1200000, "stats_reported_at_ms": 1200000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.1, "pressure": 0, "observed_hit": false} + {"id": "a-deep", "prefix_reported_at_ms": 660000, "stats_reported_at_ms": 660000, "reported_prefix": true, "matched_tokens": 550, "hit_rate": 0.6, "pressure": 0, "outcome_available": true, "observed_hit": true}, + {"id": "b-fresh", "prefix_reported_at_ms": 1200000, "stats_reported_at_ms": 1200000, "reported_prefix": true, "matched_tokens": 320, "hit_rate": 0.5, "pressure": 0, "outcome_available": true, "observed_hit": false}, + {"id": "z-decoy", "prefix_reported_at_ms": 1200000, "stats_reported_at_ms": 1200000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.1, "pressure": 0, "outcome_available": true, "observed_hit": false} ] }, { @@ -208,13 +208,13 @@ "tenant": "tenant-a", "model": "model-a", "hash_scheme": "vllm", - "prefix_hash": "slo-loose-220-1", + "prefix_hash": "c2xvLWxvb3NlLTIyMC0x", "token_count": 512, "ttft_budget_ms": 220, "replicas": [ - {"id": "a-deep", "prefix_reported_at_ms": 760000, "stats_reported_at_ms": 760000, "reported_prefix": true, "matched_tokens": 512, "hit_rate": 0.6, "pressure": 0, "observed_hit": true}, - {"id": "b-fresh", "prefix_reported_at_ms": 1300000, "stats_reported_at_ms": 1300000, "reported_prefix": true, "matched_tokens": 320, "hit_rate": 0.5, "pressure": 0, "observed_hit": false}, - {"id": "z-decoy", "prefix_reported_at_ms": 1300000, "stats_reported_at_ms": 1300000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.1, "pressure": 0, "observed_hit": false} + {"id": "a-deep", "prefix_reported_at_ms": 760000, "stats_reported_at_ms": 760000, "reported_prefix": true, "matched_tokens": 512, "hit_rate": 0.6, "pressure": 0, "outcome_available": true, "observed_hit": true}, + {"id": "b-fresh", "prefix_reported_at_ms": 1300000, "stats_reported_at_ms": 1300000, "reported_prefix": true, "matched_tokens": 320, "hit_rate": 0.5, "pressure": 0, "outcome_available": true, "observed_hit": false}, + {"id": "z-decoy", "prefix_reported_at_ms": 1300000, "stats_reported_at_ms": 1300000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.1, "pressure": 0, "outcome_available": true, "observed_hit": false} ] }, { @@ -224,13 +224,13 @@ "tenant": "tenant-b", "model": "model-b", "hash_scheme": "vllm", - "prefix_hash": "slo-loose-220-2", + "prefix_hash": "c2xvLWxvb3NlLTIyMC0y", "token_count": 512, "ttft_budget_ms": 220, "replicas": [ - {"id": "a-deep", "prefix_reported_at_ms": 860000, "stats_reported_at_ms": 860000, "reported_prefix": true, "matched_tokens": 512, "hit_rate": 0.6, "pressure": 0, "observed_hit": true}, - {"id": "b-fresh", "prefix_reported_at_ms": 1400000, "stats_reported_at_ms": 1400000, "reported_prefix": true, "matched_tokens": 320, "hit_rate": 0.5, "pressure": 0, "observed_hit": false}, - {"id": "z-decoy", "prefix_reported_at_ms": 1400000, "stats_reported_at_ms": 1400000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.1, "pressure": 0, "observed_hit": false} + {"id": "a-deep", "prefix_reported_at_ms": 860000, "stats_reported_at_ms": 860000, "reported_prefix": true, "matched_tokens": 512, "hit_rate": 0.6, "pressure": 0, "outcome_available": true, "observed_hit": true}, + {"id": "b-fresh", "prefix_reported_at_ms": 1400000, "stats_reported_at_ms": 1400000, "reported_prefix": true, "matched_tokens": 320, "hit_rate": 0.5, "pressure": 0, "outcome_available": true, "observed_hit": false}, + {"id": "z-decoy", "prefix_reported_at_ms": 1400000, "stats_reported_at_ms": 1400000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.1, "pressure": 0, "outcome_available": true, "observed_hit": false} ] }, { @@ -240,11 +240,11 @@ "tenant": "tenant-a", "model": "model-a", "hash_scheme": "vllm", - "prefix_hash": "tenant-rate-1", + "prefix_hash": "dGVuYW50LXJhdGUtMQ==", "token_count": 64, "replicas": [ - {"id": "a-noisy", "prefix_reported_at_ms": 1500000, "stats_reported_at_ms": 1500000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.19, "pressure": 0, "observed_hit": false}, - {"id": "z-warm", "prefix_reported_at_ms": 1470000, "stats_reported_at_ms": 1470000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.25, "pressure": 0, "observed_hit": true} + {"id": "a-noisy", "prefix_reported_at_ms": 1500000, "stats_reported_at_ms": 1500000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.19, "pressure": 0, "outcome_available": true, "observed_hit": false}, + {"id": "z-warm", "prefix_reported_at_ms": 1470000, "stats_reported_at_ms": 1470000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.25, "pressure": 0, "outcome_available": true, "observed_hit": true} ] }, { @@ -254,11 +254,11 @@ "tenant": "tenant-a", "model": "model-a", "hash_scheme": "vllm", - "prefix_hash": "tenant-rate-2", + "prefix_hash": "dGVuYW50LXJhdGUtMg==", "token_count": 64, "replicas": [ - {"id": "a-noisy", "prefix_reported_at_ms": 1600000, "stats_reported_at_ms": 1600000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.19, "pressure": 0, "observed_hit": false}, - {"id": "z-warm", "prefix_reported_at_ms": 1570000, "stats_reported_at_ms": 1570000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.25, "pressure": 0, "observed_hit": true} + {"id": "a-noisy", "prefix_reported_at_ms": 1600000, "stats_reported_at_ms": 1600000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.19, "pressure": 0, "outcome_available": true, "observed_hit": false}, + {"id": "z-warm", "prefix_reported_at_ms": 1570000, "stats_reported_at_ms": 1570000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.25, "pressure": 0, "outcome_available": true, "observed_hit": true} ] }, { @@ -268,11 +268,11 @@ "tenant": "tenant-b", "model": "model-b", "hash_scheme": "vllm", - "prefix_hash": "tenant-rate-3", + "prefix_hash": "dGVuYW50LXJhdGUtMw==", "token_count": 64, "replicas": [ - {"id": "a-noisy", "prefix_reported_at_ms": 1700000, "stats_reported_at_ms": 1700000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.19, "pressure": 0, "observed_hit": false}, - {"id": "z-warm", "prefix_reported_at_ms": 1670000, "stats_reported_at_ms": 1670000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.25, "pressure": 0, "observed_hit": true} + {"id": "a-noisy", "prefix_reported_at_ms": 1700000, "stats_reported_at_ms": 1700000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.19, "pressure": 0, "outcome_available": true, "observed_hit": false}, + {"id": "z-warm", "prefix_reported_at_ms": 1670000, "stats_reported_at_ms": 1670000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.25, "pressure": 0, "outcome_available": true, "observed_hit": true} ] }, { @@ -282,10 +282,10 @@ "tenant": "tenant-a", "model": "model-a", "hash_scheme": "vllm", - "prefix_hash": "tenant-age-preserve-1", + "prefix_hash": "dGVuYW50LWFnZS1wcmVzZXJ2ZS0x", "token_count": 64, "replicas": [ - {"id": "a-moderate", "prefix_reported_at_ms": 1725000, "stats_reported_at_ms": 1725000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.3, "pressure": 0, "observed_hit": true} + {"id": "a-moderate", "prefix_reported_at_ms": 1725000, "stats_reported_at_ms": 1725000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.3, "pressure": 0, "outcome_available": true, "observed_hit": true} ] }, { @@ -295,10 +295,10 @@ "tenant": "tenant-b", "model": "model-b", "hash_scheme": "vllm", - "prefix_hash": "tenant-age-preserve-2", + "prefix_hash": "dGVuYW50LWFnZS1wcmVzZXJ2ZS0y", "token_count": 64, "replicas": [ - {"id": "a-moderate", "prefix_reported_at_ms": 1825000, "stats_reported_at_ms": 1825000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.3, "pressure": 0, "observed_hit": true} + {"id": "a-moderate", "prefix_reported_at_ms": 1825000, "stats_reported_at_ms": 1825000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.3, "pressure": 0, "outcome_available": true, "observed_hit": true} ] }, { @@ -308,11 +308,11 @@ "tenant": "tenant-a", "model": "model-a", "hash_scheme": "vllm", - "prefix_hash": "tenant-age-expire-1", + "prefix_hash": "dGVuYW50LWFnZS1leHBpcmUtMQ==", "token_count": 64, "replicas": [ - {"id": "a-stale", "prefix_reported_at_ms": 1850000, "stats_reported_at_ms": 1850000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.8, "pressure": 0, "observed_hit": false}, - {"id": "z-recent", "prefix_reported_at_ms": 1970000, "stats_reported_at_ms": 1970000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.3, "pressure": 0, "observed_hit": true} + {"id": "a-stale", "prefix_reported_at_ms": 1850000, "stats_reported_at_ms": 1850000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.8, "pressure": 0, "outcome_available": true, "observed_hit": false}, + {"id": "z-recent", "prefix_reported_at_ms": 1970000, "stats_reported_at_ms": 1970000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.3, "pressure": 0, "outcome_available": true, "observed_hit": true} ] }, { @@ -322,11 +322,11 @@ "tenant": "tenant-a", "model": "model-a", "hash_scheme": "vllm", - "prefix_hash": "tenant-age-expire-2", + "prefix_hash": "dGVuYW50LWFnZS1leHBpcmUtMg==", "token_count": 64, "replicas": [ - {"id": "a-stale", "prefix_reported_at_ms": 1950000, "stats_reported_at_ms": 1950000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.8, "pressure": 0, "observed_hit": false}, - {"id": "z-recent", "prefix_reported_at_ms": 2070000, "stats_reported_at_ms": 2070000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.3, "pressure": 0, "observed_hit": true} + {"id": "a-stale", "prefix_reported_at_ms": 1950000, "stats_reported_at_ms": 1950000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.8, "pressure": 0, "outcome_available": true, "observed_hit": false}, + {"id": "z-recent", "prefix_reported_at_ms": 2070000, "stats_reported_at_ms": 2070000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.3, "pressure": 0, "outcome_available": true, "observed_hit": true} ] }, { @@ -336,11 +336,11 @@ "tenant": "tenant-b", "model": "model-b", "hash_scheme": "vllm", - "prefix_hash": "tenant-age-expire-3", + "prefix_hash": "dGVuYW50LWFnZS1leHBpcmUtMw==", "token_count": 64, "replicas": [ - {"id": "a-stale", "prefix_reported_at_ms": 2050000, "stats_reported_at_ms": 2050000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.8, "pressure": 0, "observed_hit": false}, - {"id": "z-recent", "prefix_reported_at_ms": 2170000, "stats_reported_at_ms": 2170000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.3, "pressure": 0, "observed_hit": true} + {"id": "a-stale", "prefix_reported_at_ms": 2050000, "stats_reported_at_ms": 2050000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.8, "pressure": 0, "outcome_available": true, "observed_hit": false}, + {"id": "z-recent", "prefix_reported_at_ms": 2170000, "stats_reported_at_ms": 2170000, "reported_prefix": false, "matched_tokens": 0, "hit_rate": 0.3, "pressure": 0, "outcome_available": true, "observed_hit": true} ] } ]